diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 86587ca1087933e9337366fe39092be2197b6377..9dae34e5e18ae4d9fb7106aea800412b0ca7bc5f 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -1,10 +1,215 @@
+stages:
+ - build
+ - test
+ - deploy
-simple_test:
- script:
- - source tests/ci-scripts/test.sh
- tags:
- - fftw3
- - hdf5
- - mpi
- - python3
+image: gitlab-registry.mpcdf.mpg.de/mpcdf/module-image
+
+.load_modules: &load_modules |
+ module purge
+ module load cmake/3.22 ${COMPILER} ${MPI} gsl hdf5-mpi/1.12.1 fftw-mpi anaconda
+ export FFTW_DIR=$FFTW_HOME
+
+.build: &build |
+ mkdir build
+ cd build
+ cmake .. -DNDEBUG=OFF -DTIMING_OUTPUT=OFF -DCMAKE_BUILD_TYPE=Debug
+ make VERBOSE=1
+
+.run_tests: &run_tests |
+ cd build
+ mkdir -p /usr/local/lib/python3.7/site-packages
+ export PYTHONPATH=/usr/local/lib/python3.7/site-packages:${PYTHONPATH}
+ make VERBOSE=1 install
+ export CMAKE_PREFIX_PATH=/usr/local/lib:${CMAKE_PREFIX_PATH}
+ env CTEST_OUTPUT_ON_FAILURE=1 make test
+
+.export_GCC_compilers: &export_GCC_compilers |
+ export CC=${GCC_HOME}/bin/gcc
+ export CXX=${GCC_HOME}/bin/g++
+
+.export_INTEL_compilers: &export_INTEL_compilers |
+ export CC=icc
+ export CXX=icpc
+
+#.allow_docker_run_as_root: &allow_docker_run_as_root |
+# export OMPI_ALLOW_RUN_AS_ROOT=1
+# export OMPI_ALLOW_RUN_AS_ROOT_CONFIRM=1
+
+build-gcc-openmpi:
+ stage: build
+ script:
+ - *load_modules
+ - *export_GCC_compilers
+ - *build
+ variables:
+ COMPILER: "gcc"
+ MPI: "openmpi"
+ MPICXX: "mpicxx"
+ tags:
+ - docker
+ artifacts:
+ paths:
+ - build/
+ expire_in: 1 hour
+ when: always
+
+build-gcc-impi:
+ stage: build
+ script:
+ - *load_modules
+ - *export_GCC_compilers
+ - >
+ export MPI_HOME=${I_MPI_ROOT}
+ - *build
+ variables:
+ COMPILER: "gcc"
+ MPI: "impi"
+ MPICXX: "mpigxx"
+ tags:
+ - docker
+ artifacts:
+ paths:
+ - build/
+ expire_in: 1 hour
+ when: always
+
+build-intel:
+ stage: build
+ script:
+ - *load_modules
+ - *export_INTEL_compilers
+ - >
+ export MPI_HOME=${I_MPI_ROOT}
+ - *build
+ variables:
+ COMPILER: "intel"
+ MPI: "impi"
+ MPICXX: "mpiicpc"
+ tags:
+ - docker
+ artifacts:
+ paths:
+ - build/
+ expire_in: 1 hour
+ when: always
+
+test-gcc-impi:
+ stage: test
+ script:
+ - *load_modules
+ - *export_GCC_compilers
+ - >
+ export MPI_HOME=${I_MPI_ROOT}
+ - *run_tests
+ tags:
+ - docker
+ variables:
+ COMPILER: "gcc"
+ MPI: "impi"
+ MPICXX: "mpigxx"
+ needs:
+ - job: build-gcc-impi
+ artifacts: true
+
+#test-gcc-openmpi:
+# stage: test
+# script:
+# - *load_modules
+# - *export_GCC_compilers
+# - *allow_docker_run_as_root
+# - *run_tests
+# tags:
+# - docker
+# variables:
+# COMPILER: "gcc"
+# MPI: "openmpi"
+# MPICXX: "mpicxx"
+# needs:
+# - job: build-gcc-openmpi
+# artifacts: true
+
+test-intel:
+ stage: test
+ script:
+ - *load_modules
+ - *export_INTEL_compilers
+ - >
+ export MPI_HOME=${I_MPI_ROOT}
+ - *run_tests
+ tags:
+ - docker
+ variables:
+ COMPILER: "intel"
+ MPI: "impi"
+ MPICXX: "mpiicpc"
+ needs:
+ - job: build-intel
+ artifacts: true
+
+build-doc:
+ stage: build
+ script:
+ - *load_modules
+ - mkdir build-doc
+ - cd build-doc
+ - module load git gcc graphviz doxygen
+ - *export_GCC_compilers
+ - pip install breathe
+ - yum -y install gd
+ - cmake .. -DNDEBUG=OFF -DTIMING_OUTPUT=OFF
+ - make VERBOSE=1 doc_html_full
+ #- make VERBOSE=1 doc_latex
+ tags:
+ - docker
+ variables:
+ COMPILER: "gcc"
+ MPI: "impi"
+ MPICXX: "mpigxx"
+ artifacts:
+ paths:
+ - build-doc/
+ when: always
+ expire_in: 12 hrs
+
+ #build-doc-latex:
+ # stage: test
+ # image: gitlab-registry.mpcdf.mpg.de/mpcdf/documentation/docs-ci:latest
+ # script:
+ # - cd build-doc/sphinx_latex
+ # - make
+ # - cd ../..
+ # - cp build-doc/sphinx_latex/TurTLE.pdf TurTLE_manual.pdf
+ # needs:
+ # - job: build-doc
+ # artifacts: true
+ # artifacts:
+ # paths:
+ # - TurTLE_manual.pdf
+ # when: always
+ # expire_in: 12 hrs
+
+pages:
+ stage: deploy
+ dependencies:
+ - build-doc
+ script:
+ - mkdir public
+ - mv build-doc/html public/doxygen_html
+ - mv build-doc/sphinx_html public/sphinx_html
+ #- mv TurTLE_manual.pdf public/
+ - mv build-doc/doc_full_index.html public/index.html
+ - cp documentation/TurTLE_logo.svg public/
+ artifacts:
+ paths:
+ - public
+# only:
+# - tags
+ tags:
+ - docker
+ needs:
+ - job: build-doc
+ artifacts: true
+ #- job: build-doc-latex
+ #artifacts: true
diff --git a/AUTHORS b/AUTHORS
index 921ffeda512e71d1a70c2797e5c676f80967aede..8e5fa7e98c21cf64e6f3dd5e8ed17f44c3872590 100644
--- a/AUTHORS
+++ b/AUTHORS
@@ -1,6 +1,17 @@
-All people who contributed to bfps, in order of the date of their first
-contribution.
+- Cristian C Lalescu <Cristian.Lalescu@mpcdf.mpg.de>
-Cristian C Lalescu <Cristian.Lalescu@ds.mpg.de>
-Dimitar Vlaykov
-Berenger Bramas
+- Dimitar Vlaykov
+
+- Berenger Bramas
+
+- Debarghya Banerjee
+
+- Jose Agustin Arguedas Leiva
+
+- Markus Rampp
+
+- Tobias Baetge
+
+- Lukas Bentkamp
+
+- Michael Wilczek
diff --git a/CMakeLists.txt b/CMakeLists.txt
new file mode 100644
index 0000000000000000000000000000000000000000..3add4d7222c7253285dda95deca73e8fb2e0d72a
--- /dev/null
+++ b/CMakeLists.txt
@@ -0,0 +1,677 @@
+#######################################################################
+# #
+# Copyright 2019 Max Planck Institute #
+# for Dynamics and Self-Organization #
+# #
+# This file is part of TurTLE. #
+# #
+# TurTLE is free software: you can redistribute it and/or modify #
+# it under the terms of the GNU General Public License as published #
+# by the Free Software Foundation, either version 3 of the License, #
+# or (at your option) any later version. #
+# #
+# TurTLE is distributed in the hope that it will be useful, #
+# but WITHOUT ANY WARRANTY; without even the implied warranty of #
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the #
+# GNU General Public License for more details. #
+# #
+# You should have received a copy of the GNU General Public License #
+# along with TurTLE. If not, see <http://www.gnu.org/licenses/> #
+# #
+# Contact: Cristian.Lalescu@ds.mpg.de #
+# #
+#######################################################################
+
+
+
+cmake_minimum_required(VERSION 3.6)
+cmake_policy(VERSION 3.12)
+
+project(TurTLE)
+
+string(TIMESTAMP BUILD_DATE "%Y-%m-%d")
+
+#####################################################################################
+## Python and version
+
+find_program(PYTHON_EXECUTABLE
+ python3)
+
+execute_process(
+ COMMAND ${PYTHON_EXECUTABLE} ${PROJECT_SOURCE_DIR}/get_version.py
+ OUTPUT_VARIABLE TURTLE_VERSION
+ OUTPUT_STRIP_TRAILING_WHITESPACE)
+execute_process(
+ COMMAND ${PYTHON_EXECUTABLE} ${PROJECT_SOURCE_DIR}/get_version_long.py
+ OUTPUT_VARIABLE TURTLE_VERSION_LONG
+ OUTPUT_STRIP_TRAILING_WHITESPACE)
+
+project(TurTLE
+ VERSION ${TURTLE_VERSION}
+ LANGUAGES C CXX)
+#####################################################################################
+
+#####################################################################################
+## CXX Standard
+
+set(CMAKE_CXX_STANDARD 17)
+set(CMAKE_CXX_STANDARD_REQUIRED ON)
+#####################################################################################
+
+#####################################################################################
+## OpenMP
+
+find_package(OpenMP REQUIRED)
+
+set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${OpenMP_CXX_FLAGS}")
+set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} ${OpenMP_EXE_LINKER_FLAGS}")
+#####################################################################################
+
+#####################################################################################
+## MPI
+
+find_package(MPI REQUIRED)
+
+set(CMAKE_CXX_COMPILE_FLAGS "${CMAKE_CXX_COMPILE_FLAGS} ${MPI_CXX_COMPILE_OPTIONS}")
+set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} ${MPI_CXX_LINK_FLAGS}")
+include_directories(${MPI_CXX_INCLUDE_DIRS})
+add_definitions(${MPI_CXX_COMPILE_DEFINITIONS})
+# set(CMAKE_CXX_EXTENSIONS OFF)
+#####################################################################################
+
+
+#####################################################################################
+if (DEFINED ENV{CMAKE_INSTALL_PREFIX})
+ set(CMAKE_INSTALL_PREFIX $ENV{CMAKE_INSTALL_PREFIX})
+endif()
+
+
+set(CMAKE_MODULE_PATH ${PROJECT_SOURCE_DIR}/cmake ${CMAKE_MODULE_PATH})
+set(CMAKE_MODULE_PATH ${PROJECT_SOURCE_DIR}/cmake/morse ${CMAKE_MODULE_PATH})
+#####################################################################################
+
+
+#####################################################################################
+if(NOT CMAKE_BUILD_TYPE)
+ set(CMAKE_BUILD_TYPE Release CACHE STRING "Build type" FORCE)
+endif()
+#####################################################################################
+
+#####################################################################################
+option(NDEBUG "Define NDEBUG macro" ON)
+if(NDEBUG)
+ add_definitions(-DNDEBUG)
+endif()
+#####################################################################################
+
+#####################################################################################
+# taken from https://vicrucann.github.io/tutorials/quick-cmake-doxygen/
+# indicate the documentation build as an option and set it to ON by default
+option(BUILD_DOC "Build documentation" OFF)
+
+# check if Doxygen is installed
+find_package(Doxygen)
+if (DOXYGEN_FOUND)
+ # set input and output files
+ set(DOXYGEN_IN ${CMAKE_CURRENT_SOURCE_DIR}/documentation/cpp/cpp_config)
+ set(DOXYGEN_OUT ${CMAKE_CURRENT_BINARY_DIR}/Doxyfile)
+
+ # request to configure the file
+ configure_file(${DOXYGEN_IN} ${DOXYGEN_OUT} @ONLY)
+ message("Doxygen build started")
+
+ # note the option ALL which allows to build the docs together with the application
+ add_custom_target( doc_doxygen
+ COMMAND ${DOXYGEN_EXECUTABLE} ${DOXYGEN_OUT}
+ WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}
+ COMMENT "Generating API documentation with Doxygen"
+ VERBATIM )
+else (DOXYGEN_FOUND)
+ message("Doxygen needs to be installed to generate the doxygen documentation")
+endif (DOXYGEN_FOUND)
+
+# check if sphinx is available
+find_package(Sphinx)
+if (SPHINX_FOUND)
+ if(NOT DEFINED SPHINX_THEME)
+ set(SPHINX_THEME default)
+ endif()
+
+ if(NOT DEFINED SPHINX_THEME_DIR)
+ set(SPHINX_THEME_DIR)
+ endif()
+
+ # configured documentation tools and intermediate build results
+ set(BINARY_BUILD_DIR "${CMAKE_CURRENT_BINARY_DIR}/sphinx_build")
+
+ # Sphinx cache with pickled ReST documents
+ set(SPHINX_CACHE_DIR "${CMAKE_CURRENT_BINARY_DIR}/sphinx_doctrees")
+
+ # HTML output directory
+ set(SPHINX_HTML_DIR "${CMAKE_CURRENT_BINARY_DIR}/sphinx_html")
+ # LaTeX output directory
+ set(SPHINX_LATEX_DIR "${CMAKE_CURRENT_BINARY_DIR}/sphinx_latex")
+
+ configure_file(
+ "${PROJECT_SOURCE_DIR}/documentation/conf.py.in"
+ "${BINARY_BUILD_DIR}/conf.py"
+ @ONLY)
+
+ add_custom_target(doc_html
+ ${SPHINX_EXECUTABLE}
+ -q -b html
+ -c "${BINARY_BUILD_DIR}"
+ -d "${SPHINX_CACHE_DIR}"
+ "${PROJECT_SOURCE_DIR}/documentation"
+ "${SPHINX_HTML_DIR}"
+ COMMENT "Building HTML documentation with Sphinx")
+
+ add_custom_target(doc_latex
+ ${SPHINX_EXECUTABLE}
+ -q -b latex
+ -c "${BINARY_BUILD_DIR}"
+ -d "${SPHINX_CACHE_DIR}"
+ "${PROJECT_SOURCE_DIR}/documentation"
+ "${SPHINX_LATEX_DIR}"
+ COMMENT "Building LaTeX documentation with Sphinx")
+
+ if (DOXYGEN_FOUND)
+ set(FULL_DOC_BUILD_DIR "${CMAKE_CURRENT_BINARY_DIR}/public")
+ configure_file(
+ "${PROJECT_SOURCE_DIR}/documentation/index.html.in"
+ "${CMAKE_CURRENT_BINARY_DIR}/doc_full_index.html")
+ configure_file(
+ "${PROJECT_SOURCE_DIR}/documentation/merge_doc_html.sh.in"
+ "${CMAKE_CURRENT_BINARY_DIR}/merge_doc_html.sh")
+
+ add_custom_target(doc_html_full
+ COMMAND sh merge_doc_html.sh
+ DEPENDS doc_doxygen doc_html
+ WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}
+ COMMENT "Merging Doxygen and Sphinx HTML documentation")
+ endif()
+else (SPHINX_FOUND)
+ message("Sphinx needs to be installed to generate the full documentation")
+endif (SPHINX_FOUND)
+
+option(TIMING_OUTPUT "Toggle timing output. WARNING: memory usage is proportional to `niter_todo`" OFF)
+if(TIMING_OUTPUT)
+ add_definitions(-DUSE_TIMINGOUTPUT)
+endif()
+#####################################################################################
+
+
+set(TURTLE_LIBS "")
+#####################################################################################
+## HDF5
+
+set(HDF5_STATIC ON)
+if(NOT DEFINED ENV{HDF5_ROOT})
+ message(WARNING "The environment variable HDF5_ROOT is undefined, this might cause trouble in finding the HDF5")
+endif()
+
+set(HDF5_PREFER_PARALLEL TRUE)
+set(HDF5_NO_FIND_PACKAGE_CONFIG_FILE TRUE)
+find_package(HDF5 REQUIRED)
+
+message(STATUS "HDF5_C_INCLUDE_DIRS ${HDF5_C_INCLUDE_DIRS}")
+
+include_directories(${HDF5_C_INCLUDE_DIRS})
+add_definitions(${HDF5_C_DEFINITIONS})
+list(APPEND TURTLE_LIBS "${HDF5_C_LIBRARIES}")
+
+option(TURTLE_HDF5_USE_SZIP "Set to on to also link against SZIP" OFF)
+
+if(TURTLE_HDF5_USE_SZIP)
+ option(TURTLE_HDF5_SZIP_LIB_PATH "Additional lib path for SZIP" "")
+ if(TURTLE_HDF5_SZIP_LIB_PATH)
+ link_directories(${TURTLE_HDF5_SZIP_LIB_PATH})
+ endif()
+ list(APPEND TURTLE_LIBS "z")
+endif()
+#####################################################################################
+
+
+#####################################################################################
+## GSL
+find_package(GSL)
+
+if (GSL_FOUND)
+ include_directories(${GSL_INCLUDE_DIRS})
+ list(APPEND TURTLE_LIBS "${GSL_LIBRARIES}")
+endif()
+#####################################################################################
+
+
+#####################################################################################
+## PINCHECK
+if(DEFINED ENV{PINCHECK_ROOT})
+ set(PINCHECK_INCLUDE $ENV{PINCHECK_ROOT}/include)
+ find_file(PINCHECK_HEADER
+ pincheck.hpp
+ HINTS ${PINCHECK_INCLUDE})
+ if(PINCHECK_HEADER)
+ message("found pincheck header ${PINCHECK_HEADER}")
+ include_directories($ENV{PINCHECK_ROOT}/include)
+ #add_definitions(-DPINCHECK_FOUND)
+ set(CMAKE_CXX_COMPILE_FLAGS "${CMAKE_CXX_COMPILE_FLAGS} -DPINCHECK_FOUND")
+ endif()
+endif()
+#####################################################################################
+
+#####################################################################################
+## Extra flags
+
+set(CMAKE_CXX_COMPILE_FLAGS "${CMAKE_CXX_COMPILE_FLAGS} $ENV{TURTLE_COMPILATION_FLAGS} -Wall -g")
+set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${CMAKE_CXX_COMPILE_FLAGS}")
+#####################################################################################
+
+#####################################################################################
+## FFTW
+
+set(FFTW_STATIC ON)
+if(NOT DEFINED ENV{FFTW_DIR})
+ message(WARNING "The environment variable FFTW_DIR is undefined, this might cause trouble in finding the FFTW")
+endif()
+
+find_package(FFTW REQUIRED)
+
+set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${FFTW_CFLAGS_OTHER}")
+set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} ${FFTW_LDFLAGS_OTHER}")
+list(APPEND TURTLE_LIBS "${FFTW_LIBRARIES}")
+include_directories(${FFTW_INCLUDE_DIRS})
+link_directories(${FFTW_LIBRARY_DIRS})
+
+find_package(FFTW REQUIRED SIMPLE)
+
+set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${FFTW_CFLAGS_OTHER}")
+set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} ${FFTW_LDFLAGS_OTHER}")
+list(APPEND TURTLE_LIBS "${FFTW_LIBRARIES}")
+include_directories(${FFTW_INCLUDE_DIRS})
+link_directories(${FFTW_LIBRARY_DIRS})
+
+# hack for FFTW OMP libs
+find_library(
+ FFTWF_OMP fftw3f_omp
+ HINTS ${FFTW_LIBRARY_DIRS} $ENV{FFTW_LIBDIR} $ENV{FFTW_OPENMP_LIBDIR})
+set(TURTLE_LIBS ${FFTWF_OMP} ${TURTLE_LIBS})
+find_library(
+ FFTW_OMP fftw3_omp
+ HINTS ${FFTW_LIBRARY_DIRS} $ENV{FFTW_LIBDIR} $ENV{FFTW_OPENMP_LIBDIR})
+set(TURTLE_LIBS ${FFTW_OMP} ${TURTLE_LIBS})
+
+# hack for FFTW MPI libs
+find_library(
+ FFTWF_MPI fftw3f_mpi
+ HINTS ${FFTW_LIBRARY_DIRS} $ENV{FFTW_LIBDIR} $ENV{FFTW_MPI_LIBDIR})
+set(TURTLE_LIBS ${FFTWF_MPI} ${TURTLE_LIBS})
+find_library(
+ FFTW_MPI fftw3_mpi
+ HINTS ${FFTW_LIBRARY_DIRS} $ENV{FFTW_LIBDIR} $ENV{FFTW_MPI_LIBDIR})
+set(TURTLE_LIBS ${FFTW_MPI} ${TURTLE_LIBS})
+#####################################################################################
+
+list(APPEND TURTLE_LIBS "${MPI_CXX_LIBRARIES}")
+list(APPEND TURTLE_LIBS "${OpenMP_CXX_LIB_NAMES}")
+
+#####################################################################################
+## Get the links and include from deps
+
+get_property(ALL_INCLUDE_DIRS DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR} PROPERTY INCLUDE_DIRECTORIES)
+get_property(ALL_LINK_DIRS DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR} PROPERTY LINK_DIRECTORIES)
+#####################################################################################
+
+#####################################################################################
+## Build the lib
+
+include_directories(${PROJECT_SOURCE_DIR}/cpp)
+
+#file(GLOB_RECURSE cpp_for_lib ${PROJECT_SOURCE_DIR}/*.cpp)
+set(cpp_for_lib
+ ${PROJECT_SOURCE_DIR}/cpp/full_code/code_base.cpp
+ ${PROJECT_SOURCE_DIR}/cpp/full_code/direct_numerical_simulation.cpp
+ ${PROJECT_SOURCE_DIR}/cpp/full_code/NSE.cpp
+ ${PROJECT_SOURCE_DIR}/cpp/full_code/NSVE.cpp
+ ${PROJECT_SOURCE_DIR}/cpp/full_code/static_field.cpp
+ ${PROJECT_SOURCE_DIR}/cpp/full_code/kraichnan_field.cpp
+ ${PROJECT_SOURCE_DIR}/cpp/full_code/joint_acc_vel_stats.cpp
+ ${PROJECT_SOURCE_DIR}/cpp/full_code/pressure_stats.cpp
+ ${PROJECT_SOURCE_DIR}/cpp/full_code/test.cpp
+ ${PROJECT_SOURCE_DIR}/cpp/full_code/filter_test.cpp
+ ${PROJECT_SOURCE_DIR}/cpp/full_code/field_test.cpp
+ ${PROJECT_SOURCE_DIR}/cpp/full_code/write_filtered_test.cpp
+ ${PROJECT_SOURCE_DIR}/cpp/full_code/dealias_test.cpp
+ ${PROJECT_SOURCE_DIR}/cpp/full_code/Gauss_field_test.cpp
+ ${PROJECT_SOURCE_DIR}/cpp/full_code/symmetrize_test.cpp
+ ${PROJECT_SOURCE_DIR}/cpp/full_code/field_output_test.cpp
+ ${PROJECT_SOURCE_DIR}/cpp/full_code/phase_shift_test.cpp
+ ${PROJECT_SOURCE_DIR}/cpp/full_code/test_tracer_set.cpp
+ ${PROJECT_SOURCE_DIR}/cpp/full_code/get_rfields.cpp
+ ${PROJECT_SOURCE_DIR}/cpp/full_code/get_velocity.cpp
+ ${PROJECT_SOURCE_DIR}/cpp/full_code/get_3D_correlations.cpp
+ ${PROJECT_SOURCE_DIR}/cpp/full_code/write_rpressure.cpp
+ ${PROJECT_SOURCE_DIR}/cpp/full_code/bandpass_stats.cpp
+ ${PROJECT_SOURCE_DIR}/cpp/full_code/field_single_to_double.cpp
+ ${PROJECT_SOURCE_DIR}/cpp/full_code/resize.cpp
+ ${PROJECT_SOURCE_DIR}/cpp/full_code/NSVE_field_stats.cpp
+ ${PROJECT_SOURCE_DIR}/cpp/full_code/native_binary_to_hdf5.cpp
+ ${PROJECT_SOURCE_DIR}/cpp/full_code/postprocess.cpp
+ ${PROJECT_SOURCE_DIR}/cpp/field.cpp
+ ${PROJECT_SOURCE_DIR}/cpp/kspace.cpp
+ ${PROJECT_SOURCE_DIR}/cpp/field_layout.cpp
+ ${PROJECT_SOURCE_DIR}/cpp/hdf5_tools.cpp
+ ${PROJECT_SOURCE_DIR}/cpp/fftw_tools.cpp
+ ${PROJECT_SOURCE_DIR}/cpp/vorticity_equation.cpp
+ ${PROJECT_SOURCE_DIR}/cpp/field_binary_IO.cpp
+ ${PROJECT_SOURCE_DIR}/cpp/particles/interpolation/spline_n0.cpp
+ ${PROJECT_SOURCE_DIR}/cpp/particles/interpolation/spline_n1.cpp
+ ${PROJECT_SOURCE_DIR}/cpp/particles/interpolation/spline_n2.cpp
+ ${PROJECT_SOURCE_DIR}/cpp/particles/interpolation/spline_n3.cpp
+ ${PROJECT_SOURCE_DIR}/cpp/particles/interpolation/spline_n4.cpp
+ ${PROJECT_SOURCE_DIR}/cpp/particles/interpolation/spline_n5.cpp
+ ${PROJECT_SOURCE_DIR}/cpp/particles/interpolation/spline_n6.cpp
+ ${PROJECT_SOURCE_DIR}/cpp/particles/interpolation/spline_n7.cpp
+ ${PROJECT_SOURCE_DIR}/cpp/particles/interpolation/spline_n8.cpp
+ ${PROJECT_SOURCE_DIR}/cpp/particles/interpolation/spline_n9.cpp
+ ${PROJECT_SOURCE_DIR}/cpp/particles/interpolation/spline_n10.cpp
+ ${PROJECT_SOURCE_DIR}/cpp/particles/interpolation/Lagrange_polys.cpp
+ ${PROJECT_SOURCE_DIR}/cpp/particles/interpolation/field_tinterpolator.cpp
+ ${PROJECT_SOURCE_DIR}/cpp/particles/interpolation/particle_set.cpp
+ ${PROJECT_SOURCE_DIR}/cpp/particles/rhs/tracer_rhs.cpp
+ ${PROJECT_SOURCE_DIR}/cpp/particles/rhs/tracer_with_collision_counter_rhs.cpp
+ ${PROJECT_SOURCE_DIR}/cpp/particles/particle_solver.cpp
+ ${PROJECT_SOURCE_DIR}/cpp/scope_timer.cpp
+ ${PROJECT_SOURCE_DIR}/cpp/full_code/test_interpolation.cpp
+ ${PROJECT_SOURCE_DIR}/cpp/full_code/NSVEparticles.cpp
+ ${PROJECT_SOURCE_DIR}/cpp/full_code/NSVEcomplex_particles.cpp
+ ${PROJECT_SOURCE_DIR}/cpp/full_code/NSVE_Stokes_particles.cpp
+ ${PROJECT_SOURCE_DIR}/cpp/full_code/NSVEp_extra_sampling.cpp
+ ${PROJECT_SOURCE_DIR}/cpp/particles/inner/particles_inner_computer.cpp
+ ${PROJECT_SOURCE_DIR}/cpp/full_code/ornstein_uhlenbeck_process.cpp
+ ${PROJECT_SOURCE_DIR}/cpp/full_code/ou_vorticity_equation.cpp
+ ${PROJECT_SOURCE_DIR}/cpp/full_code/test_interpolation_methods.cpp
+ ${PROJECT_SOURCE_DIR}/cpp/full_code/test_particle_integration.cpp)
+
+set(hpp_for_lib
+ ${PROJECT_SOURCE_DIR}/cpp/full_code/code_base.hpp
+ ${PROJECT_SOURCE_DIR}/cpp/full_code/direct_numerical_simulation.hpp
+ ${PROJECT_SOURCE_DIR}/cpp/full_code/NSE.hpp
+ ${PROJECT_SOURCE_DIR}/cpp/full_code/NSVE.hpp
+ ${PROJECT_SOURCE_DIR}/cpp/full_code/static_field.hpp
+ ${PROJECT_SOURCE_DIR}/cpp/full_code/kraichnan_field.hpp
+ ${PROJECT_SOURCE_DIR}/cpp/full_code/joint_acc_vel_stats.hpp
+ ${PROJECT_SOURCE_DIR}/cpp/full_code/pressure_stats.hpp
+ ${PROJECT_SOURCE_DIR}/cpp/full_code/test.hpp
+ ${PROJECT_SOURCE_DIR}/cpp/full_code/filter_test.hpp
+ ${PROJECT_SOURCE_DIR}/cpp/full_code/dealias_test.hpp
+ ${PROJECT_SOURCE_DIR}/cpp/full_code/Gauss_field_test.hpp
+ ${PROJECT_SOURCE_DIR}/cpp/full_code/field_test.hpp
+ ${PROJECT_SOURCE_DIR}/cpp/full_code/write_filtered_test.hpp
+ ${PROJECT_SOURCE_DIR}/cpp/full_code/symmetrize_test.hpp
+ ${PROJECT_SOURCE_DIR}/cpp/full_code/field_output_test.hpp
+ ${PROJECT_SOURCE_DIR}/cpp/full_code/phase_shift_test.hpp
+ ${PROJECT_SOURCE_DIR}/cpp/full_code/test_tracer_set.hpp
+ ${PROJECT_SOURCE_DIR}/cpp/full_code/get_rfields.hpp
+ ${PROJECT_SOURCE_DIR}/cpp/full_code/get_velocity.hpp
+ ${PROJECT_SOURCE_DIR}/cpp/full_code/get_3D_correlations.hpp
+ ${PROJECT_SOURCE_DIR}/cpp/full_code/write_rpressure.hpp
+ ${PROJECT_SOURCE_DIR}/cpp/full_code/bandpass_stats.hpp
+ ${PROJECT_SOURCE_DIR}/cpp/full_code/field_single_to_double.hpp
+ ${PROJECT_SOURCE_DIR}/cpp/full_code/resize.hpp
+ ${PROJECT_SOURCE_DIR}/cpp/full_code/NSVE_field_stats.hpp
+ ${PROJECT_SOURCE_DIR}/cpp/full_code/native_binary_to_hdf5.hpp
+ ${PROJECT_SOURCE_DIR}/cpp/full_code/postprocess.hpp
+ ${PROJECT_SOURCE_DIR}/cpp/field.hpp
+ ${PROJECT_SOURCE_DIR}/cpp/kspace.hpp
+ ${PROJECT_SOURCE_DIR}/cpp/field_layout.hpp
+ ${PROJECT_SOURCE_DIR}/cpp/hdf5_tools.hpp
+ ${PROJECT_SOURCE_DIR}/cpp/fftw_tools.hpp
+ ${PROJECT_SOURCE_DIR}/cpp/vorticity_equation.hpp
+ ${PROJECT_SOURCE_DIR}/cpp/field_binary_IO.hpp
+ ${PROJECT_SOURCE_DIR}/cpp/particles/interpolation/spline.hpp
+ ${PROJECT_SOURCE_DIR}/cpp/particles/interpolation/spline_n0.hpp
+ ${PROJECT_SOURCE_DIR}/cpp/particles/interpolation/spline_n1.hpp
+ ${PROJECT_SOURCE_DIR}/cpp/particles/interpolation/spline_n2.hpp
+ ${PROJECT_SOURCE_DIR}/cpp/particles/interpolation/spline_n3.hpp
+ ${PROJECT_SOURCE_DIR}/cpp/particles/interpolation/spline_n4.hpp
+ ${PROJECT_SOURCE_DIR}/cpp/particles/interpolation/spline_n5.hpp
+ ${PROJECT_SOURCE_DIR}/cpp/particles/interpolation/spline_n6.hpp
+ ${PROJECT_SOURCE_DIR}/cpp/particles/interpolation/spline_n7.hpp
+ ${PROJECT_SOURCE_DIR}/cpp/particles/interpolation/spline_n8.hpp
+ ${PROJECT_SOURCE_DIR}/cpp/particles/interpolation/spline_n9.hpp
+ ${PROJECT_SOURCE_DIR}/cpp/particles/interpolation/spline_n10.hpp
+ ${PROJECT_SOURCE_DIR}/cpp/particles/interpolation/Lagrange_polys.hpp
+ ${PROJECT_SOURCE_DIR}/cpp/particles/interpolation/abstract_particle_set.hpp
+ ${PROJECT_SOURCE_DIR}/cpp/particles/interpolation/particle_set.hpp
+ ${PROJECT_SOURCE_DIR}/cpp/particles/interpolation/field_tinterpolator.hpp
+ ${PROJECT_SOURCE_DIR}/cpp/particles/abstract_particle_rhs.hpp
+ ${PROJECT_SOURCE_DIR}/cpp/particles/rhs/tracer_rhs.hpp
+ ${PROJECT_SOURCE_DIR}/cpp/particles/rhs/deformation_tensor_rhs.hpp
+ ${PROJECT_SOURCE_DIR}/cpp/particles/rhs/tracer_with_collision_counter_rhs.hpp
+ ${PROJECT_SOURCE_DIR}/cpp/particles/particle_solver.hpp
+ ${PROJECT_SOURCE_DIR}/cpp/scope_timer.hpp
+ ${PROJECT_SOURCE_DIR}/cpp/full_code/test_interpolation.hpp
+ ${PROJECT_SOURCE_DIR}/cpp/full_code/NSVEparticles.hpp
+ ${PROJECT_SOURCE_DIR}/cpp/full_code/NSVEcomplex_particles.hpp
+ ${PROJECT_SOURCE_DIR}/cpp/full_code/NSVE_Stokes_particles.hpp
+ ${PROJECT_SOURCE_DIR}/cpp/full_code/NSVEp_extra_sampling.hpp
+ ${PROJECT_SOURCE_DIR}/cpp/particles/inner/particles_inner_computer.hpp
+ ${PROJECT_SOURCE_DIR}/cpp/particles/inner/particles_inner_computer_2nd_order.hpp
+ ${PROJECT_SOURCE_DIR}/cpp/particles/inner/particles_inner_computer_empty.hpp
+ ${PROJECT_SOURCE_DIR}/cpp/particles/abstract_particles_input.hpp
+ ${PROJECT_SOURCE_DIR}/cpp/particles/abstract_particles_output.hpp
+ ${PROJECT_SOURCE_DIR}/cpp/particles/abstract_particles_system.hpp
+ ${PROJECT_SOURCE_DIR}/cpp/particles/abstract_particles_system_with_p2p.hpp
+ ${PROJECT_SOURCE_DIR}/cpp/particles/alltoall_exchanger.hpp
+ ${PROJECT_SOURCE_DIR}/cpp/particles/lock_free_bool_array.hpp
+ ${PROJECT_SOURCE_DIR}/cpp/particles/p2p/p2p_computer_empty.hpp
+ ${PROJECT_SOURCE_DIR}/cpp/particles/p2p/p2p_computer.hpp
+ ${PROJECT_SOURCE_DIR}/cpp/particles/p2p/p2p_ghost_collisions.hpp
+ ${PROJECT_SOURCE_DIR}/cpp/particles/p2p/p2p_distr_mpi.hpp
+ ${PROJECT_SOURCE_DIR}/cpp/particles/p2p/p2p_tree.hpp
+ ${PROJECT_SOURCE_DIR}/cpp/particles/particles_adams_bashforth.hpp
+ ${PROJECT_SOURCE_DIR}/cpp/particles/particles_distr_mpi.hpp
+ ${PROJECT_SOURCE_DIR}/cpp/particles/interpolation/particles_field_computer.hpp
+ ${PROJECT_SOURCE_DIR}/cpp/particles/interpolation/particles_generic_interp.hpp
+ ${PROJECT_SOURCE_DIR}/cpp/particles/particles_input_hdf5.hpp
+ ${PROJECT_SOURCE_DIR}/cpp/particles/particles_output_hdf5.hpp
+ ${PROJECT_SOURCE_DIR}/cpp/particles/particles_output_mpiio.hpp
+ ${PROJECT_SOURCE_DIR}/cpp/particles/particles_output_sampling_hdf5.hpp
+ ${PROJECT_SOURCE_DIR}/cpp/particles/particles_sampling.hpp
+ ${PROJECT_SOURCE_DIR}/cpp/particles/particles_system_builder.hpp
+ ${PROJECT_SOURCE_DIR}/cpp/particles/particles_system.hpp
+ ${PROJECT_SOURCE_DIR}/cpp/particles/particles_utils.hpp
+ ${PROJECT_SOURCE_DIR}/cpp/full_code/main_code.hpp
+ ${PROJECT_SOURCE_DIR}/cpp/full_code/codes_with_no_output.hpp
+ ${PROJECT_SOURCE_DIR}/cpp/full_code/NSVE_no_output.hpp
+ ${PROJECT_SOURCE_DIR}/cpp/full_code/NSVEparticles_no_output.hpp
+ ${PROJECT_SOURCE_DIR}/cpp/base.hpp
+ ${PROJECT_SOURCE_DIR}/cpp/env_utils.hpp
+ ${PROJECT_SOURCE_DIR}/cpp/fftw_interface.hpp
+ ${PROJECT_SOURCE_DIR}/cpp/turtle_timer.hpp
+ ${PROJECT_SOURCE_DIR}/cpp/omputils.hpp
+ ${PROJECT_SOURCE_DIR}/cpp/shared_array.hpp
+ ${PROJECT_SOURCE_DIR}/cpp/spectrum_function.hpp
+ ${PROJECT_SOURCE_DIR}/cpp/full_code/ornstein_uhlenbeck_process.hpp
+ ${PROJECT_SOURCE_DIR}/cpp/full_code/ou_vorticity_equation.hpp
+ ${PROJECT_SOURCE_DIR}/cpp/full_code/test_interpolation_methods.hpp
+ ${PROJECT_SOURCE_DIR}/cpp/full_code/test_particle_integration.hpp)
+
+if(GSL_FOUND)
+ LIST(APPEND cpp_for_lib ${PROJECT_SOURCE_DIR}/cpp/particles/rhs/deformation_tensor_rhs.cpp)
+ LIST(APPEND hpp_for_lib ${PROJECT_SOURCE_DIR}/cpp/particles/rhs/deformation_tensor_rhs.hpp)
+endif()
+
+#file(GLOB_RECURSE hpp_for_lib ${PROJECT_SOURCE_DIR}/*.hpp)
+LIST(APPEND source_files ${hpp_for_lib} ${cpp_for_lib})
+
+add_library(TurTLE ${source_files})
+
+target_link_libraries(TurTLE ${TURTLE_LIBS})
+target_compile_options(TurTLE PRIVATE $<$<CONFIG:DEBUG>:-O1>)
+
+install(TARGETS TurTLE EXPORT TURTLE_EXPORT DESTINATION lib/ )
+install(DIRECTORY ${PROJECT_SOURCE_DIR}/cpp/ DESTINATION include/TurTLE/ FILES_MATCHING PATTERN "*.h*")
+#####################################################################################
+
+#####################################################################################
+## Export the configuration
+
+configure_file(${PROJECT_SOURCE_DIR}/cmake/TurTLEConfig.cmake.in ${PROJECT_BINARY_DIR}/TurTLEConfig.cmake @ONLY)
+
+install(FILES "${PROJECT_BINARY_DIR}/TurTLEConfig.cmake" DESTINATION lib/)
+export(TARGETS TurTLE FILE "${PROJECT_BINARY_DIR}/TurTLELibraryDepends.cmake")
+install(EXPORT TURTLE_EXPORT DESTINATION lib/)
+if(EXISTS "${PROJECT_BINARY_DIR}/bash_setup_for_TurTLE.sh")
+ install(
+ FILES "${PROJECT_BINARY_DIR}/bash_setup_for_TurTLE.sh"
+ PERMISSIONS OWNER_READ GROUP_READ WORLD_READ
+ DESTINATION "lib/"
+ )
+endif()
+install(FILES "${PROJECT_SOURCE_DIR}/TurTLE/test/B32p1e4_checkpoint_0.h5" DESTINATION share/TurTLE-${TURTLE_VERSION_LONG})
+install(DIRECTORY "${PROJECT_SOURCE_DIR}/TurTLE/test/particle_set" DESTINATION share/TurTLE-${TURTLE_VERSION_LONG}/)
+install(DIRECTORY "${PROJECT_SOURCE_DIR}/TurTLE/test/profiler" DESTINATION share/TurTLE-${TURTLE_VERSION_LONG}/)
+install(DIRECTORY "${PROJECT_SOURCE_DIR}/TurTLE/test/collisions" DESTINATION share/TurTLE-${TURTLE_VERSION_LONG}/)
+#####################################################################################
+
+
+#####################################################################################
+## Install the python3 wrapper
+# copy python package
+install(CODE "execute_process(COMMAND ${CMAKE_COMMAND} -E copy_directory ${PROJECT_SOURCE_DIR}/TurTLE ${PROJECT_BINARY_DIR}/python/TurTLE/)")
+# install __init__
+configure_file(
+ "${PROJECT_SOURCE_DIR}/TurTLE/__init__.py.in"
+ "${PROJECT_BINARY_DIR}/python/TurTLE/__init__.py"
+ @ONLY)
+# update setup.py
+configure_file(
+ "${PROJECT_SOURCE_DIR}/setup.py.in"
+ "${PROJECT_BINARY_DIR}/python/setup.py"
+ @ONLY)
+if(EXISTS "${PROJECT_BINARY_DIR}/host_info.py")
+ install(CODE "execute_process(COMMAND ${CMAKE_COMMAND} -E copy ${PROJECT_BINARY_DIR}/host_info.py ${PROJECT_BINARY_DIR}/python/TurTLE/)")
+else()
+ if(EXISTS "${PROJECT_SOURCE_DIR}/host_info.py")
+ install(CODE "execute_process(COMMAND ${CMAKE_COMMAND} -E copy ${PROJECT_SOURCE_DIR}/host_info.py ${PROJECT_BINARY_DIR}/python/TurTLE/)")
+ else()
+ install(CODE "execute_process(COMMAND ${CMAKE_COMMAND} -E copy ${PROJECT_SOURCE_DIR}/pc_host_info.py ${PROJECT_BINARY_DIR}/python/TurTLE/host_info.py)")
+ endif()
+endif()
+install(CODE "execute_process(COMMAND ${PYTHON_EXECUTABLE} ${PROJECT_BINARY_DIR}/python/setup.py install --force --prefix=${CMAKE_INSTALL_PREFIX} WORKING_DIRECTORY ${PROJECT_BINARY_DIR}/python/)")
+#####################################################################################
+
+
+#####################################################################################
+## Add tests
+include(CTest)
+set(TEST_OUTPUT_DIRECTORY "${PROJECT_BINARY_DIR}/test_runs/")
+enable_testing()
+if (BUILD_TESTING)
+ file(MAKE_DIRECTORY ${TEST_OUTPUT_DIRECTORY})
+ ### basic functionality
+ add_test(
+ NAME test_fftw
+ COMMAND turtle.test_fftw
+ WORKING_DIRECTORY ${TEST_OUTPUT_DIRECTORY})
+ add_test(
+ NAME test_Parseval
+ COMMAND turtle.test_Parseval
+ WORKING_DIRECTORY ${TEST_OUTPUT_DIRECTORY})
+ ### basic particle functionality
+ add_test(
+ NAME test_particles
+ COMMAND turtle.test_particles p2p_sampling on
+ WORKING_DIRECTORY ${TEST_OUTPUT_DIRECTORY})
+ set_tests_properties(test_particles PROPERTIES TIMEOUT 2000)
+ add_test(
+ NAME test_collisions
+ COMMAND turtle.test_collisions NSVE_Stokes_growing_subset p2p_sampling on
+ WORKING_DIRECTORY ${TEST_OUTPUT_DIRECTORY})
+ set_tests_properties(test_particles PROPERTIES TIMEOUT 2000)
+ add_test(
+ NAME test_tracer_set
+ COMMAND turtle TEST test_tracer_set -n 32 --np 2 --ntpp 2 --simname tracer_set_testsim
+ WORKING_DIRECTORY ${TEST_OUTPUT_DIRECTORY})
+ ### test copying of parameters
+ add_test(
+ NAME test_parameter_copy
+ COMMAND turtle.test_parameter_copy
+ WORKING_DIRECTORY ${TEST_OUTPUT_DIRECTORY})
+ set_tests_properties(test_parameter_copy PROPERTIES TIMEOUT 100)
+ ### compare DNS output to stored results
+ add_test(
+ NAME test_NSVEparticles
+ COMMAND turtle.test_NSVEparticles --np 2 --ntpp 2 --cpp_random_particles 0
+ WORKING_DIRECTORY ${TEST_OUTPUT_DIRECTORY})
+ set_tests_properties(test_NSVEparticles PROPERTIES TIMEOUT 2000)
+ ### check whether data is messed up by p2p calculations
+ add_test(
+ NAME test_Heun_p2p
+ COMMAND turtle.test_Heun_p2p -n 32 --np 2 --ntpp 2 --simname tracer_set_Heun_p2p --src-simname dns_nsveparticles --src-iteration 0
+ WORKING_DIRECTORY ${TEST_OUTPUT_DIRECTORY})
+ ### check whether particle deletion works
+ add_test(
+ NAME test_particle_deleter
+ COMMAND turtle.test_particle_deleter particle_deleter -n 32 --np 2 --ntpp 2 --simname tracer_set_particle_deleter --src-simname dns_nsveparticles --src-iteration 0
+ WORKING_DIRECTORY ${TEST_OUTPUT_DIRECTORY})
+ ### simple runs of post-processing tools
+ add_test(
+ NAME test_pp_single_to_double
+ COMMAND turtle PP field_single_to_double --simname dns_nsveparticles --iter0 32 --iter1 32
+ WORKING_DIRECTORY ${TEST_OUTPUT_DIRECTORY})
+ add_test(
+ NAME test_pp_get_rfields
+ COMMAND turtle PP get_rfields --simname dns_nsveparticles --iter0 0 --iter1 64
+ WORKING_DIRECTORY ${TEST_OUTPUT_DIRECTORY})
+ add_test(
+ NAME test_pp_get_velocity
+ COMMAND turtle PP get_velocity --simname dns_nsveparticles --iter0 0 --iter1 64
+ WORKING_DIRECTORY ${TEST_OUTPUT_DIRECTORY})
+ add_test(
+ NAME test_pp_get_3D_correlations
+ COMMAND turtle PP get_3D_correlations --simname dns_nsveparticles --iter0 0 --iter1 64
+ WORKING_DIRECTORY ${TEST_OUTPUT_DIRECTORY})
+ add_test(
+ NAME test_pp_write_rpressure
+ COMMAND turtle PP write_rpressure --simname dns_nsveparticles --iter0 0 --iter1 64
+ WORKING_DIRECTORY ${TEST_OUTPUT_DIRECTORY})
+ add_test(
+ NAME test_pp_joint_acc_vel_stats
+ COMMAND turtle PP joint_acc_vel_stats --simname dns_nsveparticles --iter0 0 --iter1 64
+ WORKING_DIRECTORY ${TEST_OUTPUT_DIRECTORY})
+ add_test(
+ NAME test_pp_pressure_stats
+ COMMAND turtle PP pressure_stats --simname dns_nsveparticles --iter0 0 --iter1 64
+ WORKING_DIRECTORY ${TEST_OUTPUT_DIRECTORY})
+ add_test(
+ NAME test_pp_resize
+ COMMAND turtle PP resize --simname dns_nsveparticles --new_nx 96 --new_ny 96 --new_nz 96 --new_simname dns_nsveparticles_resized
+ WORKING_DIRECTORY ${TEST_OUTPUT_DIRECTORY})
+ ### simple runs of different DNS
+ add_test(
+ NAME test_NSVEp_extra_sampling
+ COMMAND turtle DNS NSVEp_extra_sampling -n 32 --np 2 --ntpp 2 --src-simname dns_nsveparticles --src-iteration 32 --simname dns_nsvep_extra_sampling --nparticles 1000
+ WORKING_DIRECTORY ${TEST_OUTPUT_DIRECTORY})
+ add_test(
+ NAME test_NSVEcomplex_particles
+ COMMAND turtle DNS NSVEcomplex_particles -n 32 --np 2 --ntpp 2 --src-simname dns_nsveparticles --src-iteration 32 --simname dns_nsvecomplex_particles --nparticles 1000 --cpp_random_particles 0
+ WORKING_DIRECTORY ${TEST_OUTPUT_DIRECTORY})
+ add_test(
+ NAME test_static_field
+ COMMAND turtle DNS static_field --simname dns_static_field --nparticles 10000 --cpp_random_particles 0
+ WORKING_DIRECTORY ${TEST_OUTPUT_DIRECTORY})
+ add_test(
+ NAME test_kraichnan_field
+ COMMAND turtle DNS kraichnan_field --simname dns_kraichnan_field --dtfactor 0.05 --nparticles 10000 --ntpp 2 --cpp_random_particles 0
+ WORKING_DIRECTORY ${TEST_OUTPUT_DIRECTORY})
+ add_test(
+ NAME test_NSVE_Stokes_particles
+ COMMAND turtle DNS NSVE_Stokes_particles -n 32 --np 2 --ntpp 2 --src-simname dns_nsveparticles --src-iteration 32 --simname dns_nsve_tokes_particles --nparticles 10000 --cpp_random_particles 0
+ WORKING_DIRECTORY ${TEST_OUTPUT_DIRECTORY})
+endif(BUILD_TESTING)
+#####################################################################################
+
+#EOF
+
diff --git a/README.rst b/README.rst
index ddb9f2447db919248100368a9a08b13297d5e3a4..832a27d96a8a525f6df52c8319ce930bee0b2616 100644
--- a/README.rst
+++ b/README.rst
@@ -1,108 +1,366 @@
-================================
-Big Fluid and Particle Simulator
-================================
-
-In brief, this code runs pseudospectral direct numerical simulations
-(DNS) of the incompressible Navier-Stokes equations, using FFTW 3, and
-it can integrate particle trajectories in the resulting fields.
-
-The Navier-Stokes solver has been extensively tested (tests are included
-in the repository), and it is working as expected.
-Parameters and statistics are stored in HDF5 format, together with code
-information, so simulation data should be "future proof" --- suggestions
-of possible improvements to the current approach are always welcome.
-
-The primary aim of bfps is to reduce the time spent on setting up and
-baby sitting DNS, as well as simplify the analysis of the generated
-data.
-The wish is that this Python package provides an easy and general way
-of constructing efficient specialized DNS C++ codes for different
-turbulence problems encountered in research.
-At the same time, the package should provide a unified way of
-postprocessing, and accessing the postprocessing results.
-The code therefore consists of two main parts: the pure C++ code, a set
-of loosely related "building blocks", and the Python code, which can
-generate C++ code using the pure classes, but with a significant degree
-of flexibility.
-
-The code user is expected to write a small python script that will
-properly define the DNS they are interested in running.
-That code will generate an executable that can then be run directly on
-the user's machine, or submitted to a queue on a cluster.
+.. image:: https://gitlab.mpcdf.mpg.de/TurTLE/turtle/badges/develop/pipeline.svg
+ :target: https://gitlab.mpcdf.mpg.de/TurTLE/turtle/commits/develop
+=========================================
+Turbulence Tools: Lagrangian and Eulerian
+=========================================
-.. _sec-installation:
+.. _sec-introduction:
------------
-Installation
+Introduction
------------
-So far, the code has been run on laptops, desktops, and a couple of
-clusters (biggest run so far was 1536^3 on 16 nodes of 32 cores each,
-with about 11 seconds per time step, for a simple incompressible
-Navier-Stokes problem).
-Postprocessing data may not be very computationally intensive, depending
-on the amount of data involved.
+TurTLE implements a number of standard functionality of Fourier-based
+pseudo-spectral numerical simulations, as well as the corresponding
+numerical particle-tracking functionality.
+The package also contains a Navier-Stokes solver, as well as a small
+number of post-processing tools.
+The solver is production-ready, having already been used for a number of
+publications.
+
+TurTLE is written in C++ and it uses a hybrid MPI/OpenMP programming
+paradigm, relying on FFTW3 for an efficient Fourier transform
+implementation.
+HDF5 is used for I/O, including for parameter input and output of
+statistics.
+CMake is used for compilation and installation.
+
+A Python 3 wrapper is used to prepare parameters and initial conditions,
+as well as to generate job submission scripts for clusters and to
+perform basic post-processing of simulation results.
+
+The primary aim of TurTLE is to provide optimal performance, while
+reducing the time spent on setting up and supervising ensembles of DNS,
+with the added benefit of a unified launch-postprocess approach through
+the use of the Python wrapper.
+
+----------------------------
+Background and documentation
+----------------------------
+
+Please find up-to-date documentation at
+http://TurTLE.pages.mpcdf.de/turtle/index.html.
+
+Authors
+-------
+
+TurTLE is developed and maintained by the Wilczek group at the Max Planck
+Institute for Dynamics and Self-Organization and the University of Bayreuth
+in collaboration with the Application Support Group of the Max Planck
+Computing and Data Facility.
+
+TurTLE contains contributions from:
+
+.. include:: AUTHORS
+
+.. _sec-installation:
-**Postprocessing only**
+---------------------------------------------------
+Installation for postprocessing only
+---------------------------------------------------
-Use a console; navigate to the ``bfps`` folder, and type:
+The Python 3 package may be installed directly if only post-processing
+of existing data is desired. Simply clone the repository and install by
+executing
.. code:: bash
+ bash configure_python.sh
python setup.py install
(add `--user` or `sudo` as appropriate).
-`setup.py` should tell you about the various packages you need.
-
-**Full installation**
-
-If you want to run simulations on the machine where you're installing,
-you will need to call `compile_library` before installing.
-Your machine needs to have an MPI compiler installed, the HDF5 C library
-and FFTW >= 3.4.
-The file `machine_settings_py.py` should be modified
-appropriately for your machine (otherwise the `compile_library` command will most
-likely fail).
-This file will be copied the first time you run `setup.py` into
-`$HOME/.config/bfps/machine_settings.py`, **where it will be imported from
-afterwards** --- any future edits **must** be made to the new file.
-You may, obviously, edit it afterwards and rerun the `compile_library` command as
-needed.
+`setup.py` uses the `setuptools` package for dependency resolution.
+
+----------------------
+Full installation
+----------------------
+
+TurTLE can be used on various machines, with laptops routinely being
+used for development and testing, but large production runs using tens
+of thousands of CPU cores on large computing clusters.
+
+The C++ library requires a number of dependencies, that `CMake` will
+search for before compilation and installation.
+In brief, an MPI compiler is required, as well as the HDF5 C library
+compiled with parallel support and FFTW >= 3.3.4.
+We provide instructions for local compilation of FFTW and HDF5, because
+default versions packaged with Linux variants are typically inadequately
+configured.
+
+These installation steps assume that you have a working C/C++ MPI compiler,
+properly configured on your system (i.e. the various configure scripts
+are able to find it), as well as an installation of Python 3.
+The list is a work in progress, please contact us
+(Cristian.Lalescu@mpcdf.mpg.de) if the procedure fails at any step of the
+process.
+We recommend to first read the instructions in full, and only
+afterwards starting to execute the individual steps.
+
+**Creating a virtual environment**
+
+We recommend creating a virtual environment for TurTLE. To do this,
+choose an installation location :code:`<INSTALL_LOC>` on a local fast partition
+(under unix systems, this could be e.g. `~/.local`) and a name for the
+environment :code:`<VENV_NAME>` (e.g. `turtle-production`).
+To create the virtual environment, execute:
.. code:: bash
- python setup.py compile_library
- python setup.py install
+ python -m venv <INSTALL_LOC>/<VENV_NAME>
+
+In the following, we refer to the path :code:`<INSTALL_LOC>` /
+:code:`<VENV_NAME>` as :code:`<TURTLE_DIR>`. For more information on virtual
+environments, please see https://docs.python-guide.org/dev/virtualenvs/.
+
+**Installation of requirements**
+
+TurTLE has the following requirements:
+
+- C/C++ compiler of your choice
+
+- MPI
+
+- Python 3
+
+- FFTW version >= 3.3.4
+
+- HDF5 version >= 1.8. **Important**: must be compiled with the option :code:`--enable-parallel`
+
+- cmake version > 3.12
+
+We will assume you already have a working MPI compiler and an
+installation of Python 3. For the other requirements, we provide
+installation instructions. We recommend installing
+them on a local fast partition. In the following, we refer to their
+locations as :code:`<FFTW_DIR>`, :code:`<HDF5_DIR>` and
+:code:`<CMAKE_DIR>`. You may choose to install the requirements into
+your virtual environment. In that case, all of these placeholders are
+equal to :code:`<TURTLE_DIR>`.
+
+- **FFTW**
+ Download latest version from http://www.fftw.org/.
+ To compile and install it in the custom location :code:`<FFTW_DIR>`,
+ execute the following commands in order (feel free to customize
+ optimisation flags for your own computer, see
+ http://www.fftw.org/fftw3_doc/Installation-on-Unix.html):
+
+ .. code:: bash
+
+ ./configure --prefix=<FFTW_DIR> --enable-float --enable-sse --enable-mpi --enable-openmp --enable-threads
+ make
+ make install
+ ./configure --prefix=<FFTW_DIR> --enable-sse2 --enable-avx512 --enable-mpi --enable-openmp --enable-threads
+ make
+ make install
+
+..
+ TurTLE will try to find FFTW using the FindFFTW from the Morse project.
+ If the package is installed in a non-standard location, make sure the
+ corresponding environment variables are properly exported
+ (see step 3 of installation of TurTLE).
+
+- **HDF5**
+ Download HDF5 from https://www.hdfgroup.org/downloads/hdf5/.
+ To compile and install it in custom location :code:`<HDF5_DIR>`,
+ execute the following commands in order:
+
+ .. code:: bash
+
+ ./configure --prefix=<HDF5_DIR> --enable-parallel
+ make
+ make install
+
+ The :code:`--enable-parallel` flag is required because TurTLE uses
+ parallel I/O.
+
+..
+ TurTLE will try to find HDF5 using the regular FindHDF5, which
+ searches system folders, or `HDF5_ROOT`.
+
+- **cmake**
+ Check if cmake (version >3.12) is available from your default package manager.
+ If not, then download cmake at https://cmake.org/cmake/resources/software.html.
+ To compile and install it in custom location :code:`<CMAKE_DIR>`,
+ execute the following commands in order:
+
+ .. code:: bash
+
+ ./bootstrap --prefix=<CMAKE_DIR>
+ make
+ make install
+
+ The directory :code:`<CMAKE_DIR>` is only relevant to later executing
+ the :code:`cmake` binary (which can be found under :code:`<CMAKE_DIR>/bin` after
+ installation).
+
+
+**Installation of TurTLE**
--------------
-Documentation
--------------
+1. Choose a location for the source code, enter it and clone turtle
+ repository by
-While the code is not fully documented yet, basic information is already
-available, and it is recommended that you generate the manual and go
-through it carefully.
-Please don't be shy about asking for specific improvements to the
-current text.
-In order to generate the manual, navigate to the repository folder, and
-execute the following commands:
+ .. code:: bash
+
+ git clone git@gitlab.mpcdf.mpg.de:TurTLE/turtle.git
+
+ Alternatively, you may visit the website
+ https://gitlab.mpcdf.mpg.de/TurTLE/turtle
+ and download the source manually.
+
+2. Execute
+
+ .. code:: bash
+
+ cd turtle
+ mkdir build
+ cd build
+
+3. Copy the file `bash_setup_template.sh` into the build folder:
+
+ .. code:: bash
+
+ cp ../bash_setup_template.sh ./bash_setup_for_TurTLE.sh
+
+ This file will set all required environment variables.
+ Please replace all placeholders by their corresponding values.
+
+ *Note*: In principle it is possible to add this information to your
+ `.bashrc`, but we recommend against it.
+
+
+4. Copy the file `pc_host_info.py` into the build folder:
+
+ .. code:: bash
+
+ cp ../pc_host_info.py ./host_info.py
+
+ This file contains information about the machine on which TurTLE
+ will run. On desktop machines, no further steps are required. On
+ clusters, please edit the `host_info.py` file according to the
+ instructions in the file.
+
+
+5.
+ Finally, we are ready to install TurTLE. Within the build folder, execute
+
+ .. code:: bash
+
+ source bash_setup_for_TurTLE.sh
+ cmake .. -DCMAKE_INSTALL_PREFIX=<TURTLE_DIR>
+
+ Then compile TurTLE with :code:`<N>` cores using
+
+ .. code:: bash
+
+ make -j<N>
+ make install
+
+Congratulations, you have installed TurTLE! Feel free to checkout the
+`overview <http://turtle.pages.mpcdf.de/turtle/sphinx_html/sphinx_static/overview.html>`_.
+
+
+**Using TurTLE as a library**.
+
+When requiring functionality not provided by TurTLE, we recommend that
+users use TurTLE as a library from "external projects".
+We are yet to write an explicit tutorial of the procedure, but TurTLE
+already contains examples of the approach:
+
+* `TurTLE/test/test_Heun_p2p.py` and related C++ files
+* `TurTLE/test/test_particle_deleter.py` and related C++ files
+
+In order for the procedure to work, please note the 3 files that TurTLE installs
+alongside the C++ headers and library:
+
+.. code:: bash
+
+ TurTLEConfig.cmake
+ TurTLE_EXPORT.cmake
+ TurTLE_EXPORT-noconfig.cmake
+
+These files are installed under :code:`<TURTLE_DIR>/lib`, and they are
+required for such external projects to work (the Python wrapper calls
+`cmake` behind the scenes, and `cmake` will need these files).
+In case you encounter compilation errors even though TurTLE itself works
+without problems, it is probably necessary to update
+the cmake input config file: `turtle/cmake/TurTLEConfig.cmake.in` ---
+you are welcome to contact us with the details.
+
+
+**Uninstall**
+
+If you installed TurTLE in a virtual environment, you may simply remove
+the virtual environment.
+
+If you installed TurTLE in a default Python location, then you should
+navigate to the corresponding `site-packages` folder, and manually
+remove all folders/files containing "TurTLE" in their name.
+On linux systems Python will typically use something like
+`/usr/lib/python3/dist-packages` or
+`~/.local/lib/python3.x/site-packages` (you should be able to find all
+relevant locations in the `sys.path` list).
+This also applies if you used a virtual environment, but you'd like to
+clean it for any reason.
+
+**Documentation**
+
+A local build of the documentation is possible where necessary.
+Doxygen is used for the C++ source code, and the `Sphinx` and `breathe`
+Python packages are used to merge the Python wrapper documentation with
+the C++ documentation.
+The optional `CMake` targets `doc_doxygen`, `doc_html` and `doc_latex`
+generate the required documents in the build directory.
+As long as the full cmake installation is possible (see below), one would
+proceed as follows to generate the documentation locally:
.. code:: bash
- cd documentation
- make latexpdf
+ mkdir build-doc
+ cd build-doc
+ cmake ..
+ make doc_doxygen
+ make doc_html
+ cd sphinx_latex
+ make
+
+After these steps, the (HTML) manual is available under
+`build-doc/sphinx_html/index.html`, and a PDF version of the manual can
+be found at `build-doc/sphinx_latex/TurTLE.pdf`.
+The stand-alone `doxygen`-generated documentation is present at
+`build-doc/html/index.html`.
-Optionally, html documentation can be generated instead if needed, just
-type ``make html`` instead of ``make latexpdf``.
+
+---------------------
+Reference publication
+---------------------
+
+Please see https://arxiv.org/abs/2107.01104 for a description of TurTLE,
+as well as a detailed discussion of the novel particle tracking
+approach.
+This is also the publication to be cited by works that made use of TurTLE.
+
+-------
+Contact
+-------
+
+If you have any questions, comments or suggestions, please contact
+Dr. Cristian C. Lalescu (Cristian.Lalescu@mpcdf.mpg.de).
--------
Comments
--------
+* the `cmake` folder contains files extracted from
+ https://gitlab.inria.fr/solverstack/morse_cmake, a separate project licensed
+ under the "CeCILL-C" license, please see
+ http://www.cecill.info/licences/Licence_CeCILL-C_V1-en.html for
+ details.
+
* particles: initialization of multistep solvers is done with lower
- order methods, so direct convergence tests will fail.
+ order methods, so direct convergence tests will fail (see the
+ "convergence test" tutorial for more information).
-* Code is used mainly with Python 3.4 and 3.5.
- In principle it should be easy to maintain compatibility with Python
- 2.7.x, but as of `bfps 1.8` this is no longer a main concern.
+* code is used mainly with Python 3.5 and later, and it is not tested at
+ all with Python 2.x
diff --git a/TurTLE/DNS.py b/TurTLE/DNS.py
new file mode 100644
index 0000000000000000000000000000000000000000..4b26dd81590097d2f9b40a4fb677ecfbcbcaf745
--- /dev/null
+++ b/TurTLE/DNS.py
@@ -0,0 +1,1228 @@
+################################################################################
+# #
+# Copyright 2015-2019 Max Planck Institute for Dynamics and Self-Organization #
+# #
+# This file is part of TurTLE. #
+# #
+# TurTLE is free software: you can redistribute it and/or modify #
+# it under the terms of the GNU General Public License as published #
+# by the Free Software Foundation, either version 3 of the License, #
+# or (at your option) any later version. #
+# #
+# TurTLE is distributed in the hope that it will be useful, #
+# but WITHOUT ANY WARRANTY; without even the implied warranty of #
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the #
+# GNU General Public License for more details. #
+# #
+# You should have received a copy of the GNU General Public License #
+# along with TurTLE. If not, see <http://www.gnu.org/licenses/> #
+# #
+# Contact: Cristian.Lalescu@ds.mpg.de #
+# #
+################################################################################
+
+
+
+import os
+import sys
+import shutil
+import subprocess
+import h5py
+import math
+import numpy as np
+import warnings
+
+import TurTLE
+from ._code import _code
+from TurTLE import tools
+
+class DNS(_code):
+ """This class is meant to stitch together the C++ code into a final source file,
+ compile it, and handle all job launching.
+ """
+ def __init__(
+ self,
+ work_dir = './',
+ simname = 'test'):
+ _code.__init__(
+ self,
+ work_dir = work_dir,
+ simname = simname)
+ self.list_of_particle_codes = [
+ 'NSVEparticles_no_output',
+ 'NSVEcomplex_particles',
+ 'NSVE_Stokes_particles',
+ 'NSVEparticles',
+ 'NSVEp_extra_sampling',
+ 'static_field',
+ 'static_field_with_ghost_collisions',
+ 'kraichnan_field']
+ self.extra_parameters = {}
+ self.statistics = {}
+ return None
+ def set_precision(
+ self,
+ fluid_dtype):
+ if fluid_dtype in [np.float32, np.float64]:
+ self.fluid_dtype = fluid_dtype
+ elif fluid_dtype in ['single', 'double']:
+ if fluid_dtype == 'single':
+ self.fluid_dtype = np.dtype(np.float32)
+ elif fluid_dtype == 'double':
+ self.fluid_dtype = np.dtype(np.float64)
+ self.rtype = self.fluid_dtype
+ if self.rtype == np.float32:
+ self.ctype = np.dtype(np.complex64)
+ self.C_field_dtype = 'float'
+ self.fluid_precision = 'single'
+ elif self.rtype == np.float64:
+ self.ctype = np.dtype(np.complex128)
+ self.C_field_dtype = 'double'
+ self.fluid_precision = 'double'
+ return None
+ def write_src(
+ self):
+ self.version_message = (
+ '/***********************************************************************\n' +
+ '* this code automatically generated by TurTLE\n' +
+ '* version {0}\n'.format(TurTLE.__version__) +
+ '***********************************************************************/\n\n\n')
+ self.include_list = [
+ '"base.hpp"',
+ '"scope_timer.hpp"',
+ '"fftw_interface.hpp"',
+ '"full_code/main_code.hpp"',
+ '<cmath>',
+ '<iostream>',
+ '<hdf5.h>',
+ '<string>',
+ '<cstring>',
+ '<fftw3-mpi.h>',
+ '<omp.h>',
+ '<cfenv>',
+ '<cstdlib>',
+ '"full_code/{0}.hpp"\n'.format(self.dns_type)]
+ self.main = """
+ int main(int argc, char *argv[])
+ {{
+ bool fpe = (
+ (getenv("TURTLE_FPE_OFF") == nullptr) ||
+ (getenv("TURTLE_FPE_OFF") != std::string("TRUE")));
+ return main_code< {0} >(argc, argv, fpe);
+ }}
+ """.format(self.dns_type + '<{0}>'.format(self.C_field_dtype))
+ self.includes = '\n'.join(
+ ['#include ' + hh
+ for hh in self.include_list])
+ with open(self.name + '.cpp', 'w') as outfile:
+ outfile.write(self.version_message + '\n\n')
+ outfile.write(self.includes + '\n\n')
+ outfile.write(self.main + '\n')
+ return None
+ def generate_default_parameters(self):
+ # these parameters are relevant for all DNS classes
+ self.parameters['fftw_plan_rigor'] = 'FFTW_ESTIMATE'
+ self.parameter_description['fftw_plan_rigor'] = 'FFTW plan rigor to use. One of `FFTW_ESTIMATE`, `FFTW_MEASURE`, `FFTW_PATIENT`. Please see FFTW documentation.'
+ self.parameters['dealias_type'] = int(1)
+ self.parameter_description['dealias_type'] = 'Dealiasing mehtod to use, integer. Options are: two-thirds (0) or smooth (1).'
+ self.parameters['dkx'] = float(1.0)
+ self.parameter_description['dkx'] = 'Smallest wavenumber in the x direction for a pseudo-spectral run.'
+ self.parameters['dky'] = float(1.0)
+ self.parameter_description['dky'] = 'Smallest wavenumber in the y direction for a pseudo-spectral run.'
+ self.parameters['dkz'] = float(1.0)
+ self.parameter_description['dkz'] = 'Smallest wavenumber in the z direction for a pseudo-spectral run.'
+ self.parameters['niter_todo'] = int(8)
+ self.parameter_description['niter_todo'] = 'Number of iterations to compute during a single run.'
+ self.parameters['niter_stat'] = int(1)
+ self.parameter_description['niter_stat'] = 'Interval (in iterations) over which to compute field statistics (i.e. call `do_stats`).'
+ self.parameters['niter_out'] = int(8)
+ self.parameter_description['niter_out'] = 'Output is performed every `NITER_OUT` iterations.'
+ self.parameters['checkpoints_per_file'] = int(1)
+ self.parameter_description['checkpoints_per_file'] = 'Number of checkpoints to store in a single checkpoint file. Rule of thumb: files should hold gigabytes of data, rather than megabytes.'
+ self.parameters['dt'] = float(0.01)
+ self.parameter_description['dt'] = 'Fixed timestep to use. It is strongly recommended not to change this value in between jobs.'
+ self.parameters['nu'] = float(0.1)
+ self.parameter_description['nu'] = 'Viscosity value used in the equations, given in code units.'
+ self.parameters['fmode'] = int(1)
+ self.parameter_description['fmode'] = 'Forcing parameter: mode to use for the Kolmogorov forcing.'
+ self.parameters['famplitude'] = float(0.5)
+ self.parameter_description['famplitude'] = 'Forcing parameter: amplitude of Kolmogorov forcing, in code units.'
+ self.parameters['friction_coefficient'] = float(0.5)
+ self.parameter_description['friction_coefficient'] = 'Forcing parameter: drag coefficient, in code units.'
+ self.parameters['energy'] = float(0.5)
+ self.parameter_description['energy'] = 'Forcing parameter: if fluid is forced by enforcing a constant energy, this is the value (in code units).'
+ self.parameters['injection_rate'] = float(0.4)
+ self.parameter_description['injection_rate'] = 'Forcing parameter: if a fixed energy injection rate is used, this is the value (in code units).'
+ self.parameters['fk0'] = float(2.0)
+ self.parameter_description['fk0'] = 'Forcing parameter: if forcing acts on wavenumber band, this is the smallest wavenumber where it acts (in code units).'
+ self.parameters['fk1'] = float(4.0)
+ self.parameter_description['fk1'] = 'Forcing parameter: if forcing acts on wavenumber band, this is the largest wavenumber where it acts (in code units).'
+ self.parameters['forcing_type'] = 'fixed_energy_injection_rate'
+ self.parameter_description['forcing_type'] = 'Forcing parameter: what type of force to use.'
+ self.parameters['histogram_bins'] = int(256)
+ self.parameter_description['histogram_bins'] = 'During statistics, histograms of real-valued fields are computed using a number of `HISTOGRAM_BINS` bins.'
+ self.parameters['max_velocity_estimate'] = float(1)
+ self.parameter_description['max_velocity_estimate'] = 'During statistics, velocity histogram bins are computed using this estimate (see code for details).'
+ self.parameters['max_vorticity_estimate'] = float(1)
+ self.parameter_description['max_velocity_estimate'] = 'During statistics, vorticity histogram bins are computed using this estimate (see code for details).'
+ # parameters specific to particle version
+ self.NSVEp_extra_parameters = {}
+ self.NSVEp_extra_parameters['niter_part'] = int(1)
+ self.NSVEp_extra_parameters['niter_part_fine_period'] = int(10)
+ self.NSVEp_extra_parameters['niter_part_fine_duration'] = int(0)
+ self.NSVEp_extra_parameters['nparticles'] = int(10)
+ self.NSVEp_extra_parameters['cpp_random_particles'] = int(1)
+ self.NSVEp_extra_parameters['sample_acceleration'] = int(0)
+ self.NSVEp_extra_parameters['tracers0_integration_steps'] = int(4)
+ self.NSVEp_extra_parameters['tracers0_neighbours'] = int(1)
+ self.NSVEp_extra_parameters['tracers0_smoothness'] = int(1)
+ self.NSVEp_extra_parameters['tracers0_enable_p2p'] = int(0)
+ self.NSVEp_extra_parameters['tracers0_enable_inner'] = int(0)
+ self.NSVEp_extra_parameters['tracers0_enable_vorticity_omega'] = int(0)
+ self.NSVEp_extra_parameters['tracers0_cutoff'] = float(1)
+ self.NSVEp_extra_parameters['tracers0_inner_v0'] = float(1)
+ self.NSVEp_extra_parameters['tracers0_lambda'] = float(1)
+ return None
+
+ def generate_extra_parameters(self):
+ # this is to ensure that particle codes have the dictionaries defined
+ for kk in self.list_of_particle_codes:
+ if kk not in self.extra_parameters.keys():
+ self.extra_parameters[kk] = {}
+ self.extra_parameters['kraichnan_field'].update({
+ 'output_velocity': int(1),
+ 'field_random_seed': int(1),
+ 'spectrum_dissipation': float(0.027),
+ 'spectrum_Lint': float(1.3),
+ 'spectrum_etaK': float(0.3),
+ 'spectrum_large_scale_const': float(6.78),
+ 'spectrum_small_scale_const': float(0.40)})
+ self.extra_parameters['NSVE_Stokes_particles'].update({
+ 'initial_field_amplitude': float(0.0),
+ 'initial_particle_vel': float(0.05),
+ 'drag_coefficient': float(0.1)})
+ self.extra_parameters['NSVEp_extra_sampling'].update({
+ 'sample_pressure': int(1),
+ 'sample_pressure_gradient': int(1),
+ 'sample_pressure_Hessian': int(1),
+ 'sample_velocity_gradient': int(1)})
+ return None
+
+ def get_kspace(self):
+ kspace = {}
+ if self.parameters['dealias_type'] == 1:
+ kMx = self.parameters['dkx']*(self.parameters['nx']//2 - 1)
+ kMy = self.parameters['dky']*(self.parameters['ny']//2 - 1)
+ kMz = self.parameters['dkz']*(self.parameters['nz']//2 - 1)
+ else:
+ kMx = self.parameters['dkx']*(self.parameters['nx']//3 - 1)
+ kMy = self.parameters['dky']*(self.parameters['ny']//3 - 1)
+ kMz = self.parameters['dkz']*(self.parameters['nz']//3 - 1)
+ kspace['kM'] = max(kMx, kMy, kMz)
+ kspace['dk'] = min(self.parameters['dkx'],
+ self.parameters['dky'],
+ self.parameters['dkz'])
+ nshells = int(kspace['kM'] / kspace['dk']) + 2
+ kspace['nshell'] = np.zeros(nshells, dtype = np.int64)
+ kspace['kshell'] = np.zeros(nshells, dtype = np.float64)
+ kspace['kx'] = np.arange( 0,
+ self.parameters['nx']//2 + 1).astype(np.float64)*self.parameters['dkx']
+ kspace['ky'] = np.arange(-self.parameters['ny']//2 + 1,
+ self.parameters['ny']//2 + 1).astype(np.float64)*self.parameters['dky']
+ kspace['ky'] = np.roll(kspace['ky'], self.parameters['ny']//2+1)
+ kspace['kz'] = np.arange(-self.parameters['nz']//2 + 1,
+ self.parameters['nz']//2 + 1).astype(np.float64)*self.parameters['dkz']
+ kspace['kz'] = np.roll(kspace['kz'], self.parameters['nz']//2+1)
+ return kspace
+ def get_data_file_name(self):
+ return os.path.join(self.work_dir, self.simname + '.h5')
+ def get_data_file(self):
+ return h5py.File(self.get_data_file_name(), 'r')
+ def get_particle_file_name(self):
+ return os.path.join(self.work_dir, self.simname + '_particles.h5')
+ def get_particle_file(self):
+ return h5py.File(self.get_particle_file_name(), 'r')
+ def get_cache_file_name(self):
+ return os.path.join(self.work_dir, self.simname + '_cache.h5')
+ def get_cache_file(self):
+ return h5py.File(self.get_cache_file_name(), 'r')
+ def get_postprocess_file_name(self):
+ return self.get_cache_file_name()
+ def get_postprocess_file(self):
+ return h5py.File(self.get_postprocess_file_name(), 'r')
+ def compute_statistics(
+ self,
+ iter0 = 0,
+ iter1 = None,
+ strict_Parseval_check = True):
+ """Run basic postprocessing on raw data.
+ The energy spectrum :math:`E(t, k)` and the enstrophy spectrum
+ :math:`\\frac{1}{2}\omega^2(t, k)` are computed from the
+
+ .. math::
+
+ \sum_{k \\leq \\|\\mathbf{k}\\| \\leq k+dk}\\hat{u_i} \\hat{u_j}^*, \\hskip .5cm
+ \sum_{k \\leq \\|\\mathbf{k}\\| \\leq k+dk}\\hat{\omega_i} \\hat{\\omega_j}^*
+
+ tensors, and the enstrophy spectrum is also used to
+ compute the dissipation :math:`\\varepsilon(t)`.
+ These basic quantities are stored in a newly created HDF5 file,
+ ``simname_cache.h5``.
+ """
+ if len(list(self.statistics.keys())) > 0:
+ return None
+ if not os.path.exists(self.get_data_file_name()):
+ if os.path.exists(self.get_cache_file_name()):
+ self.read_parameters(fname = self.get_cache_file_name())
+ pp_file = self.get_cache_file()
+ for k in ['t',
+ 'energy(t)',
+ 'energy(k)',
+ 'enstrophy(t)',
+ 'enstrophy(k)',
+ 'R_ij(t)',
+ 'vel_max(t)',
+ 'renergy(t)',
+ 'renstrophy(t)']:
+ if k in pp_file.keys():
+ self.statistics[k] = pp_file[k][...]
+ self.statistics['kM'] = pp_file['kspace/kM'][...]
+ self.statistics['dk'] = pp_file['kspace/dk'][...]
+ self.statistics['kshell'] = pp_file['kspace/kshell'][...]
+ self.statistics['nshell'] = pp_file['kspace/nshell'][...]
+ else:
+ self.read_parameters()
+ with self.get_data_file() as data_file:
+ if 'moments' not in data_file['statistics'].keys():
+ return None
+ iter0 = min((data_file['statistics/moments/velocity'].shape[0] *
+ self.parameters['niter_stat']-1),
+ iter0)
+ if type(iter1) == type(None):
+ iter1 = data_file['iteration'][...]
+ else:
+ iter1 = min(data_file['iteration'][...], iter1)
+ ii0 = iter0 // self.parameters['niter_stat']
+ ii1 = iter1 // self.parameters['niter_stat']
+ self.statistics['kshell'] = data_file['kspace/kshell'][...]
+ self.statistics['nshell'] = data_file['kspace/nshell'][...]
+ for kk in [-1, -2]:
+ if (self.statistics['kshell'][kk] == 0):
+ self.statistics['kshell'][kk] = np.nan
+ self.statistics['kM'] = data_file['kspace/kM'][...]
+ self.statistics['dk'] = data_file['kspace/dk'][...]
+ computation_needed = True
+ pp_file = h5py.File(self.get_postprocess_file_name(), 'a')
+ if not ('parameters' in pp_file.keys()):
+ data_file.copy('parameters', pp_file)
+ data_file.copy('kspace', pp_file)
+ if 'ii0' in pp_file.keys():
+ computation_needed = not (ii0 == pp_file['ii0'][...] and
+ ii1 == pp_file['ii1'][...])
+ if computation_needed:
+ for k in ['t', 'vel_max(t)',
+ 'renergy(t)',
+ 'renstrophy(t)',
+ 'energy(t)', 'enstrophy(t)',
+ 'energy(k)', 'enstrophy(k)',
+ 'energy(t, k)',
+ 'enstrophy(t, k)',
+ 'R_ij(t)',
+ 'ii0', 'ii1', 'iter0', 'iter1']:
+ if k in pp_file.keys():
+ del pp_file[k]
+ if computation_needed:
+ #TODO figure out whether normalization is sane or not
+ pp_file['iter0'] = iter0
+ pp_file['iter1'] = iter1
+ pp_file['ii0'] = ii0
+ pp_file['ii1'] = ii1
+ pp_file['t'] = (self.parameters['dt']*
+ self.parameters['niter_stat']*
+ (np.arange(ii0, ii1+1).astype(np.float)))
+ # we have an extra division by shell_width because of the Dirac delta restricting integration to the shell
+ phi_ij = data_file['statistics/spectra/velocity_velocity'][ii0:ii1+1] / self.statistics['dk']
+ pp_file['R_ij(t)'] = np.sum(phi_ij*self.statistics['dk'], axis = 1)
+ energy_tk = (
+ phi_ij[:, :, 0, 0] +
+ phi_ij[:, :, 1, 1] +
+ phi_ij[:, :, 2, 2])/2
+ pp_file['energy(t)'] = np.sum(energy_tk*self.statistics['dk'], axis = 1)
+ # normalization factor is (4 pi * shell_width * kshell^2) / (nmodes in shell * dkx*dky*dkz)
+ norm_factor = (4*np.pi*self.statistics['dk']*self.statistics['kshell']**2) / (self.parameters['dkx']*self.parameters['dky']*self.parameters['dkz']*self.statistics['nshell'])
+ pp_file['energy(k)'] = np.mean(energy_tk, axis = 0)*norm_factor
+ phi_vorticity_ij = data_file['statistics/spectra/vorticity_vorticity'][ii0:ii1+1] / self.statistics['dk']
+ enstrophy_tk = (
+ phi_vorticity_ij[:, :, 0, 0] +
+ phi_vorticity_ij[:, :, 1, 1] +
+ phi_vorticity_ij[:, :, 2, 2])/2
+ pp_file['enstrophy(t)'] = np.sum(enstrophy_tk*self.statistics['dk'], axis = 1)
+ pp_file['enstrophy(k)'] = np.mean(enstrophy_tk, axis = 0)*norm_factor
+ pp_file['vel_max(t)'] = data_file['statistics/moments/velocity'][ii0:ii1+1, 9, 3]
+ pp_file['renergy(t)'] = data_file['statistics/moments/velocity'][ii0:ii1+1, 2, 3]/2
+ pp_file['renstrophy(t)'] = data_file['statistics/moments/vorticity'][ii0:ii1+1, 2, 3]/2
+ for k in ['t',
+ 'energy(t)',
+ 'energy(k)',
+ 'enstrophy(t)',
+ 'enstrophy(k)',
+ 'R_ij(t)',
+ 'vel_max(t)',
+ 'renergy(t)',
+ 'renstrophy(t)']:
+ if k in pp_file.keys():
+ self.statistics[k] = pp_file[k][...]
+ # sanity check --- Parseval theorem check
+ energy_error = np.max(np.abs(
+ self.statistics['renergy(t)'] -
+ self.statistics['energy(t)']) / self.statistics['energy(t)'])
+ test_energy = (energy_error < 1e-5)
+ enstrophy_error = np.max(np.abs(
+ self.statistics['renstrophy(t)'] -
+ self.statistics['enstrophy(t)']) / self.statistics['enstrophy(t)'])
+ test_enstrophy = (enstrophy_error < 1e-5)
+ if strict_Parseval_check:
+ #print('energy error', energy_error)
+ #print('enstrophy error', enstrophy_error)
+ assert(test_energy)
+ assert(test_enstrophy)
+ else:
+ if not test_energy:
+ print('!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!')
+ print('warning: Parseval theorem failed for velocity')
+ print('!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!')
+ if not test_energy:
+ print('!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!')
+ print('warning: Parseval theorem failed for vorticity')
+ print('!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!')
+ self.compute_time_averages()
+ return None
+ def compute_Reynolds_stress_invariants(
+ self):
+ """
+ see Choi and Lumley, JFM v436 p59 (2001)
+ """
+ Rij = self.statistics['R_ij(t)']
+ Rij /= (2*self.statistics['energy(t)'][:, None, None])
+ Rij[:, 0, 0] -= 1./3
+ Rij[:, 1, 1] -= 1./3
+ Rij[:, 2, 2] -= 1./3
+ self.statistics['I2(t)'] = np.sqrt(np.einsum('...ij,...ij', Rij, Rij, optimize = True) / 6)
+ self.statistics['I3(t)'] = np.cbrt(np.einsum('...ij,...jk,...ki', Rij, Rij, Rij, optimize = True) / 6)
+ return None
+ def compute_time_averages(self):
+ """Compute easy stats.
+
+ Further computation of statistics based on the contents of
+ ``simname_cache.h5``.
+ Standard quantities are as follows
+ (consistent with [ishihara2007jfm]_):
+
+ .. math::
+
+ U_{\\textrm{int}}(t) = \\sqrt{\\frac{2E(t)}{3}}, \\hskip .5cm
+ L_{\\textrm{int}} = \\frac{\pi}{2U_{int}^2} \\int \\frac{dk}{k} E(k), \\hskip .5cm
+ T_{\\textrm{int}} =
+ \\frac{L_{\\textrm{int}}}{U_{\\textrm{int}}}
+
+ \\eta_K = \\left(\\frac{\\nu^3}{\\varepsilon}\\right)^{1/4}, \\hskip .5cm
+ \\tau_K = \\left(\\frac{\\nu}{\\varepsilon}\\right)^{1/2}, \\hskip .5cm
+ \\lambda = \\sqrt{\\frac{15 \\nu U_{\\textrm{int}}^2}{\\varepsilon}}
+
+ Re = \\frac{U_{\\textrm{int}} L_{\\textrm{int}}}{\\nu}, \\hskip
+ .5cm
+ R_{\\lambda} = \\frac{U_{\\textrm{int}} \\lambda}{\\nu}
+ """
+ self.statistics['Uint(t)'] = np.sqrt(2*self.statistics['energy(t)'] / 3)
+ for key in ['energy',
+ 'enstrophy',
+ 'mean_trS2',
+ 'Uint']:
+ if key + '(t)' in self.statistics.keys():
+ self.statistics[key] = np.average(self.statistics[key + '(t)'], axis = 0)
+ self.statistics['vel_max'] = np.max(self.statistics['vel_max(t)'])
+ for suffix in ['', '(t)']:
+ self.statistics['diss' + suffix] = (self.parameters['nu'] *
+ self.statistics['enstrophy' + suffix]*2)
+ self.statistics['etaK' + suffix] = (self.parameters['nu']**3 /
+ self.statistics['diss' + suffix])**.25
+ self.statistics['tauK' + suffix] = (self.parameters['nu'] /
+ self.statistics['diss' + suffix])**.5
+ self.statistics['lambda' + suffix] = (15 * self.parameters['nu'] *
+ self.statistics['Uint' + suffix]**2 /
+ self.statistics['diss' + suffix])**.5
+ self.statistics['Rlambda' + suffix] = (self.statistics['Uint' + suffix] *
+ self.statistics['lambda' + suffix] /
+ self.parameters['nu'])
+ self.statistics['kMeta' + suffix] = (self.statistics['kM'] *
+ self.statistics['etaK' + suffix])
+ if self.parameters['dealias_type'] == 1:
+ self.statistics['kMeta' + suffix] *= 0.8
+ self.statistics['Lint'] = ((np.pi /
+ (2*self.statistics['Uint']**2)) *
+ np.sum(self.statistics['energy(k)'][1:-2] /
+ self.statistics['kshell'][1:-2]))
+ self.statistics['Re'] = (self.statistics['Uint'] *
+ self.statistics['Lint'] /
+ self.parameters['nu'])
+ self.statistics['Tint'] = self.statistics['Lint'] / self.statistics['Uint']
+ self.statistics['Taylor_microscale'] = self.statistics['lambda']
+ return None
+ def set_plt_style(
+ self,
+ style = {'dashes' : (None, None)}):
+ self.style.update(style)
+ return None
+ def convert_complex_from_binary(
+ self,
+ field_name = 'vorticity',
+ iteration = 0,
+ file_name = None):
+ """read the Fourier representation of a vector field.
+
+ Read the binary file containing iteration ``iteration`` of the
+ field ``field_name``, and write it in a ``.h5`` file.
+ """
+ data = np.memmap(
+ os.path.join(self.work_dir,
+ self.simname + '_{0}_i{1:0>5x}'.format('c' + field_name, iteration)),
+ dtype = self.ctype,
+ mode = 'r',
+ shape = (self.parameters['ny'],
+ self.parameters['nz'],
+ self.parameters['nx']//2+1,
+ 3))
+ if type(file_name) == type(None):
+ file_name = self.simname + '_{0}_i{1:0>5x}.h5'.format('c' + field_name, iteration)
+ file_name = os.path.join(self.work_dir, file_name)
+ f = h5py.File(file_name, 'a')
+ f[field_name + '/complex/{0}'.format(iteration)] = data
+ f.close()
+ return None
+ def write_par(
+ self,
+ iter0 = 0):
+ assert (self.parameters['niter_todo'] % self.parameters['niter_stat'] == 0)
+ assert (self.parameters['niter_todo'] % self.parameters['niter_out'] == 0)
+ assert (self.parameters['niter_out'] % self.parameters['niter_stat'] == 0)
+ if self.dns_type in self.list_of_particle_codes:
+ assert (self.parameters['niter_todo'] % self.parameters['niter_part'] == 0)
+ assert (self.parameters['niter_out'] % self.parameters['niter_part'] == 0)
+ _code.write_par(self, iter0 = iter0)
+ with h5py.File(self.get_data_file_name(), 'r+') as ofile:
+ ofile['code_info/exec_name'] = self.name
+ kspace = self.get_kspace()
+ for k in kspace.keys():
+ ofile['kspace/' + k] = kspace[k]
+ nshells = kspace['nshell'].shape[0]
+ kspace = self.get_kspace()
+ nshells = kspace['nshell'].shape[0]
+ vec_stat_datasets = ['velocity', 'vorticity']
+ scal_stat_datasets = []
+ for k in vec_stat_datasets:
+ time_chunk = 2**20//(8*3*3*nshells)
+ time_chunk = max(time_chunk, 1)
+ ofile.create_dataset('statistics/spectra/' + k + '_' + k,
+ (1, nshells, 3, 3),
+ chunks = (time_chunk, nshells, 3, 3),
+ maxshape = (None, nshells, 3, 3),
+ dtype = np.float64)
+ time_chunk = 2**20//(8*4*10)
+ time_chunk = max(time_chunk, 1)
+ a = ofile.create_dataset('statistics/moments/' + k,
+ (1, 10, 4),
+ chunks = (time_chunk, 10, 4),
+ maxshape = (None, 10, 4),
+ dtype = np.float64)
+ time_chunk = 2**20//(8*4*self.parameters['histogram_bins'])
+ time_chunk = max(time_chunk, 1)
+ ofile.create_dataset('statistics/histograms/' + k,
+ (1,
+ self.parameters['histogram_bins'],
+ 4),
+ chunks = (time_chunk,
+ self.parameters['histogram_bins'],
+ 4),
+ maxshape = (None,
+ self.parameters['histogram_bins'],
+ 4),
+ dtype = np.int64)
+ ofile['checkpoint'] = int(0)
+ if self.dns_type in ['static_field_with_ghost_collisions']:
+ ofile.create_group('statistics/collisions')
+ if (self.dns_type in ['NSE', 'NSVE', 'NSVE_no_output']):
+ return None
+ return None
+ def simulation_parser_arguments(
+ self,
+ parser):
+ parser.add_argument(
+ '--simname',
+ type = str, dest = 'simname',
+ default = 'test')
+ parser.add_argument(
+ '-n', '--grid-size',
+ type = int,
+ dest = 'n',
+ default = 32,
+ metavar = 'N',
+ help = 'code is run by default in a grid of NxNxN')
+ for coord in ['x', 'y', 'z']:
+ parser.add_argument(
+ '--L{0}'.format(coord), '--box-length-{0}'.format(coord),
+ type = float,
+ dest = 'L{0}'.format(coord),
+ default = 2.0,
+ metavar = 'length{0}'.format(coord),
+ help = 'length of the box in the {0} direction will be `length{0} x pi`'.format(coord))
+ parser.add_argument(
+ '--wd',
+ type = str, dest = 'work_dir',
+ default = './')
+ parser.add_argument(
+ '--precision',
+ choices = ['single', 'double'],
+ type = str,
+ default = 'single')
+ parser.add_argument(
+ '--src-wd',
+ type = str,
+ dest = 'src_work_dir',
+ default = '')
+ parser.add_argument(
+ '--src-simname',
+ type = str,
+ dest = 'src_simname',
+ default = '')
+ parser.add_argument(
+ '--src-iteration',
+ type = int,
+ dest = 'src_iteration',
+ default = 0)
+ parser.add_argument(
+ '--overwrite-src-parameters',
+ action = 'store_true',
+ dest = 'overwrite_src_parameters',
+ help = 'False by default. If using a source simulation, do we keep its physical parameters? Note: if this option is switched on, default values are used for parameters that are not set explicitly by the command --- NOT the values of the source simulation.')
+ parser.add_argument(
+ '--kMeta',
+ type = float,
+ dest = 'kMeta',
+ default = 2.0)
+ parser.add_argument(
+ '--dtfactor',
+ type = float,
+ dest = 'dtfactor',
+ default = 0.5,
+ help = 'dt is computed as DTFACTOR / N')
+ return None
+ def particle_parser_arguments(
+ self,
+ parser):
+ parser.add_argument(
+ '--particle-rand-seed',
+ type = int,
+ dest = 'particle_rand_seed',
+ default = None)
+ parser.add_argument(
+ '--pclouds',
+ type = int,
+ dest = 'pclouds',
+ default = 1,
+ help = ('number of particle clouds. Particle "clouds" '
+ 'consist of particles distributed according to '
+ 'pcloud-type.'))
+ parser.add_argument(
+ '--pcloud-type',
+ choices = ['random-cube',
+ 'regular-cube'],
+ dest = 'pcloud_type',
+ default = 'random-cube')
+ parser.add_argument(
+ '--particle-cloud-size',
+ type = float,
+ dest = 'particle_cloud_size',
+ default = 2*np.pi)
+ return None
+ def add_parser_arguments(
+ self,
+ parser):
+ """Creates subparsers for all simulation types and supplies them with parameters"""
+ subparsers = parser.add_subparsers(
+ dest = 'DNS_class',
+ help = 'type of simulation to run')
+ subparsers.required = True
+ parser_list = {}
+
+ parser_list['NSE'] = subparsers.add_parser(
+ 'NSE',
+ help = 'plain Navier-Stokes equation')
+
+ parser_list['NSVE'] = subparsers.add_parser(
+ 'NSVE',
+ help = 'plain Navier-Stokes vorticity equation')
+
+ parser_list['NSVE_no_output'] = subparsers.add_parser(
+ 'NSVE_no_output',
+ help = 'plain Navier-Stokes vorticity formulation, checkpoints are NOT SAVED')
+
+ parser_list['NSVEparticles_no_output'] = subparsers.add_parser(
+ 'NSVEparticles_no_output',
+ help = 'plain Navier-Stokes vorticity formulation, with basic fluid tracers, checkpoints are NOT SAVED')
+
+ parser_list['static_field'] = subparsers.add_parser(
+ 'static_field',
+ help = 'static field with basic fluid tracers')
+
+ parser_list['static_field_with_ghost_collisions'] = subparsers.add_parser(
+ 'static_field_with_ghost_collisions',
+ help = 'static field with basic fluid tracers and ghost collisions')
+
+ parser_list['kraichnan_field'] = subparsers.add_parser(
+ 'kraichnan_field',
+ help = 'Kraichnan field with basic fluid tracers')
+
+ parser_list['NSVEparticles'] = subparsers.add_parser(
+ 'NSVEparticles',
+ help = 'plain Navier-Stokes vorticity formulation, with basic fluid tracers')
+
+ parser_list['NSVE_Stokes_particles'] = subparsers.add_parser(
+ 'NSVE_Stokes_particles',
+ help = 'plain Navier-Stokes vorticity formulation, with passive Stokes drag particles')
+
+ parser_list['NSVEcomplex_particles'] = subparsers.add_parser(
+ 'NSVEcomplex_particles',
+ help = 'plain Navier-Stokes vorticity formulation, with oriented active particles')
+ parser_list['NSVEp_extra_sampling'] = subparsers.add_parser(
+ 'NSVEp_extra_sampling',
+ help = 'plain Navier-Stokes vorticity formulation, with basic fluid tracers, that sample velocity gradient, as well as pressure and its derivatives.')
+
+ for extra_dns_type in self.extra_parameters.keys():
+ if extra_dns_type not in parser_list.keys():
+ parser_list[extra_dns_type] = subparsers.add_parser(
+ extra_dns_type,
+ help = 'Custom DNS class of type ' + extra_dns_type)
+
+ for pp in parser_list:
+ self.simulation_parser_arguments(parser_list[pp])
+ self.job_parser_arguments(parser_list[pp])
+ self.parameters_to_parser_arguments(parser_list[pp])
+ if pp in self.extra_parameters:
+ self.parameters_to_parser_arguments(
+ parser_list[pp],
+ self.extra_parameters[pp])
+
+ for pp in self.list_of_particle_codes:
+ self.particle_parser_arguments(parser_list[pp])
+ #self.parameters_to_parser_arguments(parser_list[pp],
+ # self.NSVEp_extra_parameters)
+ return None
+
+ def custom_parameter_update(self):
+ """To be reimplemented by child classes.
+
+ Children of `DNS` should modify the default values of
+ parameters in reimplementations of this method.
+ The method is called *before* parameters are read from
+ the command line.
+ """
+ return None
+
+ def prepare_launch(
+ self,
+ args = [],
+ extra_parameters = None):
+ """Set up reasonable parameters.
+
+ With the default Lundgren forcing applied in the band [2, 4],
+ we can estimate the dissipation, therefore we can estimate
+ :math:`k_M \\eta_K` and constrain the viscosity.
+
+ In brief, the command line parameter :math:`k_M \\eta_K` is
+ used in the following formula for :math:`\\nu` (:math:`N` is the
+ number of real space grid points per coordinate):
+
+ .. math::
+
+ \\nu = \\left(\\frac{2 k_M \\eta_K}{N} \\right)^{4/3}
+
+ With this choice, the average dissipation :math:`\\varepsilon`
+ will be close to 0.4, and the integral scale velocity will be
+ close to 0.77, yielding the approximate value for the Taylor
+ microscale and corresponding Reynolds number:
+
+ .. math::
+
+ \\lambda \\approx 4.75\\left(\\frac{2 k_M \\eta_K}{N} \\right)^{4/6}, \\hskip .5in
+ R_\\lambda \\approx 3.7 \\left(\\frac{N}{2 k_M \\eta_K} \\right)^{4/6}
+
+ """
+ self.generate_default_parameters()
+ self.generate_extra_parameters()
+ for key in self.list_of_particle_codes:
+ assert(key in self.extra_parameters.keys())
+ self.extra_parameters[key].update(self.NSVEp_extra_parameters)
+ self.custom_parameter_update()
+
+ opt = _code.prepare_launch(self, args = args)
+ self.set_precision(opt.precision)
+ self.dns_type = opt.DNS_class
+ self.name = self.dns_type + '-' + self.fluid_precision + '-v' + TurTLE.__version__
+
+ # now add parameters for the chosen dns_type to self.parameters
+ # extra parameters from prepare launch argument
+ if type(extra_parameters) != type(None):
+ if self.dns_type in extra_parameters.keys():
+ for k in extra_parameters[self.dns_type].keys():
+ self.parameters[k] = extra_parameters[self.dns_type][k]
+ # extra parameters from `self.extra_parameters`
+ if self.dns_type in self.extra_parameters.keys():
+ for k in self.extra_parameters[self.dns_type].keys():
+ self.parameters[k] = self.extra_parameters[self.dns_type][k]
+
+ if ((self.parameters['niter_todo'] % self.parameters['niter_out']) != 0):
+ self.parameters['niter_out'] = self.parameters['niter_todo']
+ if len(opt.src_work_dir) == 0:
+ opt.src_work_dir = os.path.realpath(opt.work_dir)
+ if type(opt.dkx) == type(None):
+ opt.dkx = 2. / opt.Lx
+ if type(opt.dky) == type(None):
+ opt.dky = 2. / opt.Ly
+ if type(opt.dkz) == type(None):
+ opt.dkz = 2. / opt.Lz
+ if type(opt.nx) == type(None):
+ opt.nx = opt.n
+ if type(opt.ny) == type(None):
+ opt.ny = opt.n
+ if type(opt.nz) == type(None):
+ opt.nz = opt.n
+ if type(opt.fk0) == type(None):
+ opt.fk0 = self.parameters['fk0']
+ if type(opt.fk1) == type(None):
+ opt.fk1 = self.parameters['fk1']
+ if type(opt.forcing_type) == type(None):
+ opt.forcing_type = self.parameters['forcing_type']
+ if type(opt.injection_rate) == type(None):
+ opt.injection_rate = self.parameters['injection_rate']
+ if type(opt.dealias_type) == type(None):
+ opt.dealias_type = self.parameters['dealias_type']
+ if (opt.nx > opt.n or
+ opt.ny > opt.n or
+ opt.nz > opt.n):
+ opt.n = min(opt.nx, opt.ny, opt.nz)
+ print("Warning: '-n' parameter changed to minimum of nx, ny, nz. This affects the computation of nu.")
+ if type(opt.dt) == type(None):
+ if self.dns_type in ['kraichnan_field']:
+ opt.dt = opt.dtfactor * 0.5 / opt.n**2
+ else:
+ opt.dt = (opt.dtfactor / opt.n)
+ self.parameters['dt'] = opt.dt
+ # check value of kMax
+ kM = opt.n * 0.5
+ if opt.dealias_type == 1:
+ kM *= 0.8
+ # tweak forcing/viscosity based on forcing type
+ if opt.forcing_type == 'linear':
+ # custom famplitude for 288 and 576
+ if opt.n == 288:
+ self.parameters['famplitude'] = 0.45
+ elif opt.n == 576:
+ self.parameters['famplitude'] = 0.47
+ elif opt.forcing_type == 'fixed_energy_injection_rate':
+ # use the fact that mean dissipation rate is equal to injection rate
+ if type(opt.nu) == type(None):
+ opt.nu = (
+ opt.injection_rate *
+ (opt.kMeta / kM)**4)**(1./3)
+ if opt.injection_rate == 0:
+ opt.nu = (
+ (opt.kMeta / kM)**(4./3) *
+ (np.pi / 1.5)**(1./3) *
+ (2*self.parameters['energy'] / 3)**0.5)
+ elif opt.forcing_type == 'fixed_energy':
+ kf = 1. / (1./opt.fk0 +
+ 1./opt.fk1)
+ if type(opt.nu) == type(None):
+ opt.nu = (
+ (opt.kMeta / kM)**(4./3) *
+ (np.pi / kf)**(1./3) *
+ (2*self.parameters['energy'] / 3)**0.5)
+ if type(opt.nu) == type(None):
+ opt.nu = (opt.kMeta * 2 / opt.n)**(4./3)
+ self.parameters['nu'] = opt.nu
+ if type(opt.checkpoints_per_file) == type(None):
+ # hardcoded FFTW complex representation size
+ field_size = 3*(opt.nx+2)*opt.ny*opt.nz*self.fluid_dtype.itemsize
+ checkpoint_size = field_size
+ if self.dns_type in self.list_of_particle_codes:
+ rhs_size = self.parameters['tracers0_integration_steps']
+ if type(opt.tracers0_integration_steps) != type(None):
+ rhs_size = opt.tracers0_integration_steps
+ nparticles = opt.nparticles
+ if type(nparticles) == type(None):
+ nparticles = self.NSVEp_extra_parameters['nparticles']
+ particle_size = (1+rhs_size)*3*nparticles*8
+ checkpoint_size += particle_size
+ if checkpoint_size < 1e9:
+ opt.checkpoints_per_file = int(1e9 / checkpoint_size)
+ self.pars_from_namespace(opt)
+ return opt
+ def launch(
+ self,
+ args = [],
+ **kwargs):
+ opt = self.prepare_launch(args = args)
+ self.launch_jobs(opt = opt, **kwargs)
+ return None
+ def get_checkpoint_0_fname(self):
+ return os.path.join(
+ self.work_dir,
+ self.simname + '_checkpoint_0.h5')
+ def get_checkpoint_fname(self, iteration = 0):
+ checkpoint = (iteration // self.parameters['niter_out']) // self.parameters['checkpoints_per_file']
+ return os.path.join(
+ self.work_dir,
+ self.simname + '_checkpoint_{0}.h5'.format(checkpoint))
+ def generate_tracer_state(
+ self,
+ rseed = None,
+ species = 0,
+ integration_steps = None,
+ ncomponents = 3):
+ try:
+ if type(integration_steps) == type(None):
+ integration_steps = self.NSVEp_extra_parameters['tracers0_integration_steps']
+ if 'tracers{0}_integration_steps'.format(species) in self.parameters.keys():
+ integration_steps = self.parameters['tracers{0}_integration_steps'.format(species)]
+ if self.dns_type in [
+ 'NSVEcomplex_particles',
+ 'NSVE_Stokes_particles'] and species == 0:
+ ncomponents = 6
+ with h5py.File(self.get_checkpoint_0_fname(), 'a') as data_file:
+ nn = self.parameters['nparticles']
+ if not 'tracers{0}'.format(species) in data_file.keys():
+ data_file.create_group('tracers{0}'.format(species))
+ data_file.create_group('tracers{0}/rhs'.format(species))
+ data_file.create_group('tracers{0}/state'.format(species))
+ data_file['tracers{0}/rhs'.format(species)].create_dataset(
+ '0',
+ shape = (integration_steps, nn, ncomponents,),
+ dtype = np.float)
+ dset = data_file['tracers{0}/state'.format(species)].create_dataset(
+ '0',
+ shape = (nn, ncomponents,),
+ dtype = np.float)
+ if not type(rseed) == type(None):
+ np.random.seed(rseed)
+ cc = int(0)
+ batch_size = int(1e6)
+ def get_random_phases(npoints):
+ return np.random.random(
+ (npoints, 3))*2*np.pi
+ def get_random_versors(npoints):
+ bla = np.random.normal(
+ size = (npoints, 3))
+ bla /= np.sum(bla**2, axis = 1)[:, None]**.5
+ return bla
+ while nn > 0:
+ if nn > batch_size:
+ dset[cc*batch_size:(cc+1)*batch_size, :3] = get_random_phases(batch_size)
+ if dset.shape[1] == 6:
+ if self.dns_type == 'NSVE_Stokes_particles':
+ dset[cc*batch_size:(cc+1)*batch_size, 3:] = self.parameters['initial_particle_vel']*get_random_versors(batch_size)
+ else:
+ dset[cc*batch_size:(cc+1)*batch_size, 3:] = get_random_versors(batch_size)
+ nn -= batch_size
+ else:
+ dset[cc*batch_size:cc*batch_size+nn, :3] = get_random_phases(nn)
+ if dset.shape[1] == 6:
+ if self.dns_type == 'NSVE_Stokes_particles':
+ dset[cc*batch_size:cc*batch_size+nn, 3:] = self.parameters['initial_particle_vel']*get_random_versors(nn)
+ else:
+ dset[cc*batch_size:cc*batch_size+nn, 3:] = get_random_versors(nn)
+ nn = 0
+ cc += 1
+ except Exception as e:
+ print('!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!WARNING')
+ print('WARNING: when generating particles, caught the exception ', e)
+ print('!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!WARNING')
+ return None
+ def generate_vector_field(
+ self,
+ rseed = 7547,
+ spectra_slope = 1.,
+ amplitude = 1.,
+ iteration = 0,
+ field_name = 'vorticity',
+ write_to_file = False,
+ # to switch to constant field, use generate_data_3D_uniform
+ # for scalar_generator
+ scalar_generator = tools.generate_data_3D):
+ """generate vector field.
+
+ The generated field is not divergence free, but it has the proper
+ shape.
+
+ :param rseed: seed for random number generator
+ :param spectra_slope: spectrum of field will look like k^(-p)
+ :param amplitude: all amplitudes are multiplied with this value
+ :param iteration: the field is written at this iteration
+ :param field_name: the name of the field being generated
+ :param write_to_file: should we write the field to file?
+ :param scalar_generator: which function to use for generating the
+ individual components.
+ Possible values: TurTLE.tools.generate_data_3D,
+ TurTLE.tools.generate_data_3D_uniform
+ :type rseed: int
+ :type spectra_slope: float
+ :type amplitude: float
+ :type iteration: int
+ :type field_name: str
+ :type write_to_file: bool
+ :type scalar_generator: function
+
+ :returns: ``Kdata``, a complex valued 4D ``numpy.array`` that uses the
+ transposed FFTW layout.
+ Kdata[ky, kz, kx, i] is the amplitude of mode (kx, ky, kz) for
+ the i-th component of the field.
+ (i.e. x is the fastest index and z the slowest index in the
+ real-space representation).
+ """
+ np.random.seed(rseed)
+ Kdata00 = scalar_generator(
+ self.parameters['nz'],
+ self.parameters['ny'],
+ self.parameters['nx'],
+ p = spectra_slope,
+ amplitude = amplitude).astype(self.ctype)
+ Kdata01 = scalar_generator(
+ self.parameters['nz'],
+ self.parameters['ny'],
+ self.parameters['nx'],
+ p = spectra_slope,
+ amplitude = amplitude).astype(self.ctype)
+ Kdata02 = scalar_generator(
+ self.parameters['nz'],
+ self.parameters['ny'],
+ self.parameters['nx'],
+ p = spectra_slope,
+ amplitude = amplitude).astype(self.ctype)
+ Kdata0 = np.zeros(
+ Kdata00.shape + (3,),
+ Kdata00.dtype)
+ Kdata0[..., 0] = Kdata00
+ Kdata0[..., 1] = Kdata01
+ Kdata0[..., 2] = Kdata02
+ Kdata1 = tools.padd_with_zeros(
+ Kdata0,
+ self.parameters['nz'],
+ self.parameters['ny'],
+ self.parameters['nx'])
+ if write_to_file:
+ Kdata1.tofile(
+ os.path.join(self.work_dir,
+ self.simname + "_c{0}_i{1:0>5x}".format(field_name, iteration)))
+ return Kdata1
+ def copy_complex_field(
+ self,
+ src_file_name,
+ src_dset_name,
+ dst_file,
+ dst_dset_name,
+ make_link = True):
+ # I define a min_shape thingie, but for now I only trust this method for
+ # the case of increasing/decreasing by the same factor in all directions.
+ # in principle we could write something more generic, but i'm not sure
+ # how complicated that would be
+ dst_shape = (self.parameters['ny'],
+ self.parameters['nz'],
+ (self.parameters['nx']+2) // 2,
+ 3)
+ src_file = h5py.File(src_file_name, 'r')
+ if (src_file[src_dset_name].shape == dst_shape):
+ dst_file[dst_dset_name] = h5py.ExternalLink(
+ src_file_name,
+ src_dset_name)
+ else:
+ min_shape = (min(dst_shape[0], src_file[src_dset_name].shape[0]),
+ min(dst_shape[1], src_file[src_dset_name].shape[1]),
+ min(dst_shape[2], src_file[src_dset_name].shape[2]),
+ 3)
+ src_shape = src_file[src_dset_name].shape
+ dst_file.create_dataset(
+ dst_dset_name,
+ shape = dst_shape,
+ dtype = np.dtype(self.ctype),
+ fillvalue = complex(0))
+ for kz in range(min_shape[0]//2):
+ dst_file[dst_dset_name][kz,:min_shape[1]//2, :min_shape[2]] = \
+ src_file[src_dset_name][kz, :min_shape[1]//2, :min_shape[2]]
+ dst_file[dst_dset_name][kz,
+ dst_shape[1] - min_shape[1]//2+1:,
+ :min_shape[2]] = \
+ src_file[src_dset_name][kz,
+ src_shape[1] - min_shape[1]//2+1,
+ :min_shape[2]]
+ if kz > 0:
+ dst_file[dst_dset_name][-kz,:min_shape[1]//2, :min_shape[2]] = \
+ src_file[src_dset_name][-kz, :min_shape[1]//2, :min_shape[2]]
+ dst_file[dst_dset_name][-kz,
+ dst_shape[1] - min_shape[1]//2+1:,
+ :min_shape[2]] = \
+ src_file[src_dset_name][-kz,
+ src_shape[1] - min_shape[1]//2+1,
+ :min_shape[2]]
+ return None
+ def generate_particle_data(
+ self,
+ opt = None):
+ if (self.parameters['nparticles'] > 0):
+ if (self.parameters['cpp_random_particles'] == 0):
+ self.generate_tracer_state(
+ species = 0,
+ rseed = opt.particle_rand_seed)
+ if not os.path.exists(self.get_particle_file_name()):
+ with h5py.File(self.get_particle_file_name(), 'w') as particle_file:
+ particle_file.create_group('tracers0/position')
+ particle_file.create_group('tracers0/velocity')
+ particle_file.create_group('tracers0/acceleration')
+ if self.dns_type in ['NSVEcomplex_particles']:
+ particle_file.create_group('tracers0/orientation')
+ particle_file.create_group('tracers0/velocity_gradient')
+ if self.dns_type in ['NSVE_Stokes_particles']:
+ particle_file.create_group('tracers0/momentum')
+ particle_file.create_group('tracers0/vorticity')
+ if self.dns_type in ['NSVEp_extra_sampling']:
+ particle_file.create_group('tracers0/velocity_gradient')
+ particle_file.create_group('tracers0/pressure')
+ particle_file.create_group('tracers0/pressure_gradient')
+ particle_file.create_group('tracers0/pressure_Hessian')
+ return None
+ def generate_initial_condition(
+ self,
+ opt = None):
+ # take care of fields' initial condition
+ # first, check if initial field exists
+ need_field = False
+ if self.check_current_vorticity_exists:
+ need_field = True
+ if self.dns_type == 'NSE':
+ checkpoint_field = 'velocity'
+ else:
+ checkpoint_field = 'vorticity'
+ if self.dns_type in [
+ 'NSE',
+ 'NSVE',
+ 'NSVE_no_output',
+ 'static_field',
+ 'NSVEparticles',
+ 'NSVEcomplex_particles',
+ 'NSVE_Stokes_particles',
+ 'NSVEparticles_no_output',
+ 'NSVEp_extra_sampling']:
+ if not os.path.exists(self.get_checkpoint_0_fname()):
+ need_field = True
+ else:
+ f = h5py.File(self.get_checkpoint_0_fname(), 'r')
+ try:
+ dset = f[checkpoint_field + '/complex/0']
+ need_field = (dset.shape != (self.parameters['ny'],
+ self.parameters['nz'],
+ self.parameters['nx']//2+1,
+ 3))
+ except:
+ need_field = True
+ f.close()
+ if need_field:
+ f = h5py.File(self.get_checkpoint_0_fname(), 'a')
+ if len(opt.src_simname) > 0:
+ source_cp = 0
+ src_file = 'not_a_file'
+ while True:
+ src_file = os.path.join(
+ os.path.realpath(opt.src_work_dir),
+ opt.src_simname + '_checkpoint_{0}.h5'.format(source_cp))
+ f0 = h5py.File(src_file, 'r')
+ if '{0}'.format(opt.src_iteration) in f0[checkpoint_field + '/complex'].keys():
+ f0.close()
+ break
+ source_cp += 1
+ self.copy_complex_field(
+ src_file,
+ checkpoint_field + '/complex/{0}'.format(opt.src_iteration),
+ f,
+ checkpoint_field + '/complex/{0}'.format(0))
+ if not opt.overwrite_src_parameters:
+ print('DNS will use physical parameters from source simulation.\n'
+ 'This means `dealias_type`, `energy`, `famplitude`, `fk0`, `fk1`, `fmode`, `friction_coefficient`, `injection_rate`, `nu` and `forcing_type`.')
+ # timestep left out on purpose
+ src_file = os.path.join(
+ os.path.realpath(opt.src_work_dir),
+ opt.src_simname + '.h5')
+ if os.path.exists(src_file):
+ f0 = h5py.File(src_file, 'r')
+ for kk in ['dealias_type',
+ 'energy',
+ 'famplitude',
+ 'fk0',
+ 'fk1',
+ 'fmode',
+ 'friction_coefficient',
+ 'injection_rate',
+ 'nu']:
+ if (kk in f0['parameters'].keys()) and (kk in self.parameters.keys()):
+ self.parameters[kk] = type(self.parameters[kk])(f0['parameters/' + kk][()])
+ self.parameters['forcing_type'] = str(f0['parameters/forcing_type'][()], 'ascii')
+ f0.close()
+ else:
+ print('TurTLE WARNING: unable to copy parameters of source simulation. File {0} is missing.'.format(src_file))
+ else:
+ print('TurTLE WARNING: DNS will use default values for physical parameters.\n'
+ 'This means `dealias_type`, `energy`, `famplitude`, `fk0`, `fk1`, `fmode`, `friction_coefficient`, `injection_rate`, `nu` and `forcing_type` may differ from values used by the source DNS.')
+ else:
+ if self.dns_type == 'NSVE_Stokes_particles':
+ data = self.generate_vector_field(
+ write_to_file = False,
+ spectra_slope = 2.0,
+ amplitude = self.parameters['initial_field_amplitude'])
+ else:
+ data = self.generate_vector_field(
+ write_to_file = False,
+ spectra_slope = 2.0,
+ amplitude = 0.05)
+ f[checkpoint_field + '/complex/{0}'.format(0)] = data
+ f.close()
+ if self.dns_type == 'kraichnan_field':
+ if not os.path.exists(self.get_checkpoint_0_fname()):
+ f = h5py.File(self.get_checkpoint_0_fname(), 'a')
+ f.create_group('velocity/real')
+ f.close()
+ # now take care of particles' initial condition
+ if self.dns_type in self.list_of_particle_codes:
+ self.generate_particle_data(opt = opt)
+ return None
+ def launch_jobs(
+ self,
+ opt = None,
+ iter0 = 0):
+ """Launches jobs. Prepares initial condition and writes parameters to file if necessary."""
+ if not os.path.exists(self.get_data_file_name()):
+ if not os.path.isdir(self.work_dir):
+ os.makedirs(self.work_dir)
+ self.generate_initial_condition(opt = opt)
+ self.write_par(iter0 = iter0)
+ if (('test' in self.dns_type) or
+ (self.dns_type in ['kraichnan_field'])):
+ self.check_current_vorticity_exists = False
+ if self.dns_type == 'NSE':
+ self.check_current_vorticity_exists = False
+ self.check_current_velocity_exists = True
+ self.run(
+ nb_processes = opt.nb_processes,
+ nb_threads_per_process = opt.nb_threads_per_process,
+ njobs = opt.njobs,
+ hours = opt.minutes // 60,
+ minutes = opt.minutes % 60,
+ no_submit = opt.no_submit,
+ no_debug = opt.no_debug)
+ return None
diff --git a/bfps/PP.py b/TurTLE/PP.py
similarity index 82%
rename from bfps/PP.py
rename to TurTLE/PP.py
index 6e02f2aefd5db2e9790f3a16cbc2bfa3c85ab37b..4699b52d8cd9f662f380bf0eff337e3595f9572c 100644
--- a/bfps/PP.py
+++ b/TurTLE/PP.py
@@ -1,26 +1,25 @@
-#######################################################################
-# #
-# Copyright 2015 Max Planck Institute #
-# for Dynamics and Self-Organization #
-# #
-# This file is part of bfps. #
-# #
-# bfps is free software: you can redistribute it and/or modify #
-# it under the terms of the GNU General Public License as published #
-# by the Free Software Foundation, either version 3 of the License, #
-# or (at your option) any later version. #
-# #
-# bfps is distributed in the hope that it will be useful, #
-# but WITHOUT ANY WARRANTY; without even the implied warranty of #
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the #
-# GNU General Public License for more details. #
-# #
-# You should have received a copy of the GNU General Public License #
-# along with bfps. If not, see <http://www.gnu.org/licenses/> #
-# #
-# Contact: Cristian.Lalescu@ds.mpg.de #
-# #
-#######################################################################
+################################################################################
+# #
+# Copyright 2015-2019 Max Planck Institute for Dynamics and Self-Organization #
+# #
+# This file is part of TurTLE. #
+# #
+# TurTLE is free software: you can redistribute it and/or modify #
+# it under the terms of the GNU General Public License as published #
+# by the Free Software Foundation, either version 3 of the License, #
+# or (at your option) any later version. #
+# #
+# TurTLE is distributed in the hope that it will be useful, #
+# but WITHOUT ANY WARRANTY; without even the implied warranty of #
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the #
+# GNU General Public License for more details. #
+# #
+# You should have received a copy of the GNU General Public License #
+# along with TurTLE. If not, see <http://www.gnu.org/licenses/> #
+# #
+# Contact: Cristian.Lalescu@ds.mpg.de #
+# #
+################################################################################
@@ -33,10 +32,11 @@ import h5py
import math
import numpy as np
import warnings
+import glob
-import bfps
+import TurTLE
from ._code import _code
-from bfps import tools
+from TurTLE import tools
class PP(_code):
"""This class is meant to stitch together the C++ code into a final source file,
@@ -50,13 +50,8 @@ class PP(_code):
self,
work_dir = work_dir,
simname = simname)
- self.host_info = {'type' : 'cluster',
- 'environment' : None,
- 'deltanprocs' : 1,
- 'queue' : '',
- 'mail_address': '',
- 'mail_events' : None}
self.generate_default_parameters()
+ self.check_current_vorticity_exists = False
return None
def set_precision(
self,
@@ -81,8 +76,8 @@ class PP(_code):
def write_src(self):
self.version_message = (
'/***********************************************************************\n' +
- '* this code automatically generated by bfps\n' +
- '* version {0}\n'.format(bfps.__version__) +
+ '* this code automatically generated by TurTLE\n' +
+ '* version {0}\n'.format(TurTLE.__version__) +
'***********************************************************************/\n\n\n')
self.include_list = [
'"base.hpp"',
@@ -103,8 +98,8 @@ class PP(_code):
int main(int argc, char *argv[])
{{
bool fpe = (
- (getenv("BFPS_FPE_OFF") == nullptr) ||
- (getenv("BFPS_FPE_OFF") != std::string("TRUE")));
+ (getenv("TURTLE_FPE_OFF") == nullptr) ||
+ (getenv("TURTLE_FPE_OFF") != std::string("TRUE")));
return main_code< {0} >(argc, argv, fpe);
}}
""".format(self.dns_type + '<{0}>'.format(self.C_field_dtype))
@@ -118,6 +113,7 @@ class PP(_code):
return None
def generate_default_parameters(self):
# these parameters are relevant for all PP classes
+ self.parameters['fftw_plan_rigor'] = 'FFTW_ESTIMATE'
self.parameters['dealias_type'] = int(1)
self.parameters['dkx'] = float(1.0)
self.parameters['dky'] = float(1.0)
@@ -135,10 +131,26 @@ class PP(_code):
self,
dns_type = 'joint_acc_vel_stats'):
pars = {}
+ if dns_type == 'pressure_stats':
+ pars['max_pressure_estimate'] = float(1)
+ pars['histogram_bins'] = int(129)
if dns_type == 'joint_acc_vel_stats':
pars['max_acceleration_estimate'] = float(10)
pars['max_velocity_estimate'] = float(1)
pars['histogram_bins'] = int(129)
+ elif dns_type == 'resize':
+ pars['new_nx'] = int(32)
+ pars['new_ny'] = int(32)
+ pars['new_nz'] = int(32)
+ pars['new_simname'] = 'test_resized'
+ elif dns_type == 'bandpass_stats':
+ pars['k0list'] = np.linspace(4, 10, 3)
+ pars['k1list'] = np.linspace(5, 11, 3)
+ pars['filter_type'] = 'Gauss'
+ pars['max_velocity_estimate'] = float(10)
+ pars['histogram_bins'] = int(129)
+ elif dns_type == 'get_rfields':
+ pars['TrS2_on'] = int(0)
return pars
def get_data_file_name(self):
return os.path.join(self.work_dir, self.simname + '.h5')
@@ -177,19 +189,19 @@ class PP(_code):
self.parameters['niter_stat']-1),
iter0)
if type(iter1) == type(None):
- iter1 = data_file['iteration'].value
+ iter1 = data_file['iteration'][...]
else:
- iter1 = min(data_file['iteration'].value, iter1)
+ iter1 = min(data_file['iteration'][...], iter1)
ii0 = iter0 // self.parameters['niter_stat']
ii1 = iter1 // self.parameters['niter_stat']
- self.statistics['kshell'] = data_file['kspace/kshell'].value
- self.statistics['kM'] = data_file['kspace/kM'].value
- self.statistics['dk'] = data_file['kspace/dk'].value
+ self.statistics['kshell'] = data_file['kspace/kshell'][...]
+ self.statistics['kM'] = data_file['kspace/kM'][...]
+ self.statistics['dk'] = data_file['kspace/dk'][...]
computation_needed = True
pp_file = h5py.File(self.get_postprocess_file_name(), 'a')
if 'ii0' in pp_file.keys():
- computation_needed = not (ii0 == pp_file['ii0'].value and
- ii1 == pp_file['ii1'].value)
+ computation_needed = not (ii0 == pp_file['ii0'][...] and
+ ii1 == pp_file['ii1'][...])
if computation_needed:
for k in pp_file.keys():
del pp_file[k]
@@ -217,7 +229,7 @@ class PP(_code):
'vel_max(t)',
'renergy(t)']:
if k in pp_file.keys():
- self.statistics[k] = pp_file[k].value
+ self.statistics[k] = pp_file[k][...]
self.compute_time_averages()
return None
def compute_time_averages(self):
@@ -226,7 +238,7 @@ class PP(_code):
Further computation of statistics based on the contents of
``simname_postprocess.h5``.
Standard quantities are as follows
- (consistent with [Ishihara]_):
+ (consistent with [ishihara2007jfm]_):
.. math::
@@ -242,13 +254,6 @@ class PP(_code):
Re = \\frac{U_{\\textrm{int}} L_{\\textrm{int}}}{\\nu}, \\hskip
.5cm
R_{\\lambda} = \\frac{U_{\\textrm{int}} \\lambda}{\\nu}
-
- .. [Ishihara] T. Ishihara et al,
- *Small-scale statistics in high-resolution direct numerical
- simulation of turbulence: Reynolds number dependence of
- one-point velocity gradient statistics*.
- J. Fluid Mech.,
- **592**, 335-366, 2007
"""
for key in ['energy', 'enstrophy']:
self.statistics[key + '(t)'] = (self.statistics['dk'] *
@@ -319,48 +324,6 @@ class PP(_code):
f[field_name + '/complex/{0}'.format(iteration)] = data
f.close()
return None
- def job_parser_arguments(
- self,
- parser):
- parser.add_argument(
- '--ncpu',
- type = int,
- dest = 'ncpu',
- default = -1)
- parser.add_argument(
- '--np', '--nprocesses',
- metavar = 'NPROCESSES',
- help = 'number of mpi processes to use',
- type = int,
- dest = 'nb_processes',
- default = 4)
- parser.add_argument(
- '--ntpp', '--nthreads-per-process',
- type = int,
- dest = 'nb_threads_per_process',
- metavar = 'NTHREADS_PER_PROCESS',
- help = 'number of threads to use per MPI process',
- default = 1)
- parser.add_argument(
- '--no-submit',
- action = 'store_true',
- dest = 'no_submit')
- parser.add_argument(
- '--environment',
- type = str,
- dest = 'environment',
- default = None)
- parser.add_argument(
- '--minutes',
- type = int,
- dest = 'minutes',
- default = 5,
- help = 'If environment supports it, this is the requested wall-clock-limit.')
- parser.add_argument(
- '--njobs',
- type = int, dest = 'njobs',
- default = 1)
- return None
def simulation_parser_arguments(
self,
parser):
@@ -426,24 +389,52 @@ class PP(_code):
parser_native_binary_to_hdf5 = subparsers.add_parser(
'native_binary_to_hdf5',
help = 'convert native binary to hdf5')
- self.simulation_parser_arguments(parser_native_binary_to_hdf5)
- self.job_parser_arguments(parser_native_binary_to_hdf5)
- self.parameters_to_parser_arguments(parser_native_binary_to_hdf5)
+ parser_field_single_to_double = subparsers.add_parser(
+ 'field_single_to_double',
+ help = 'convert complex vorticity from single to double')
parser_get_rfields = subparsers.add_parser(
'get_rfields',
help = 'get real space velocity field')
- self.simulation_parser_arguments(parser_get_rfields)
- self.job_parser_arguments(parser_get_rfields)
- self.parameters_to_parser_arguments(parser_get_rfields)
+ parser_get_velocity = subparsers.add_parser(
+ 'get_velocity',
+ help = 'get velocity field')
+ parser_get_3D_correlations = subparsers.add_parser(
+ 'get_3D_correlations',
+ help = 'get Rij full space field')
+ parser_bandpass_stats = subparsers.add_parser(
+ 'bandpass_stats',
+ help = 'get stats of bandpassed velocity')
parser_joint_acc_vel_stats = subparsers.add_parser(
+ 'pressure_stats',
+ help = 'get pressure statistics')
+ parser_pressure_stats = subparsers.add_parser(
'joint_acc_vel_stats',
help = 'get joint acceleration and velocity statistics')
- self.simulation_parser_arguments(parser_joint_acc_vel_stats)
- self.job_parser_arguments(parser_joint_acc_vel_stats)
- self.parameters_to_parser_arguments(parser_joint_acc_vel_stats)
- self.parameters_to_parser_arguments(
- parser_joint_acc_vel_stats,
- parameters = self.extra_postprocessing_parameters('joint_acc_vel_stats'))
+ parser_resize = subparsers.add_parser(
+ 'resize',
+ help = 'resize field')
+ parser_write_rpressure = subparsers.add_parser(
+ 'write_rpressure',
+ help = 'write real pressure field to binary')
+ for pp_type in [
+ 'resize',
+ 'joint_acc_vel_stats',
+ 'pressure_stats',
+ 'bandpass_stats',
+ 'get_rfields',
+ 'get_velocity',
+ 'get_3D_correlations',
+ 'field_single_to_double',
+ 'native_binary_to_hdf5',
+ 'write_rpressure']:
+ eval('self.simulation_parser_arguments(parser_' + pp_type + ')')
+ eval('self.job_parser_arguments(parser_' + pp_type + ')')
+ eval('self.parameters_to_parser_arguments(parser_' + pp_type + ')')
+ eval('self.parameters_to_parser_arguments(parser_' +
+ pp_type +
+ ', parameters = self.extra_postprocessing_parameters("' +
+ pp_type +
+ '"))')
return None
def prepare_launch(
self,
@@ -476,12 +467,12 @@ class PP(_code):
opt = _code.prepare_launch(self, args = args)
self.set_precision(opt.precision)
self.dns_type = opt.DNS_class
- self.name = self.dns_type + '-' + self.fluid_precision + '-v' + bfps.__version__
+ self.name = self.dns_type + '-' + self.fluid_precision + '-v' + TurTLE.__version__
# merge parameters if needed
for k in self.pp_parameters.keys():
self.parameters[k] = self.pp_parameters[k]
self.pars_from_namespace(opt)
- niter_out = self.get_data_file()['parameters/niter_out'].value
+ niter_out = self.get_data_file()['parameters/niter_out'][...]
assert(opt.iter0 % niter_out == 0)
self.pp_parameters['iteration_list'] = np.arange(
opt.iter0, opt.iter1+niter_out, niter_out, dtype = np.int)
@@ -544,8 +535,8 @@ class PP(_code):
:param write_to_file: should we write the field to file?
:param scalar_generator: which function to use for generating the
individual components.
- Possible values: bfps.tools.generate_data_3D,
- bfps.tools.generate_data_3D_uniform
+ Possible values: TurTLE.tools.generate_data_3D,
+ TurTLE.tools.generate_data_3D_uniform
:type rseed: int
:type spectra_slope: float
:type amplitude: float
@@ -626,7 +617,6 @@ class PP(_code):
for kz in range(src_file[src_dset_name].shape[0]):
dst_file[dst_dset_name][kz] = src_file[src_dset_name][kz]
else:
- print('aloha')
min_shape = (min(dst_shape[0], src_file[src_dset_name].shape[0]),
min(dst_shape[1], src_file[src_dset_name].shape[1]),
min(dst_shape[2], src_file[src_dset_name].shape[2]),
@@ -651,7 +641,7 @@ class PP(_code):
parameters = self.pp_parameters,
get_sim_info = False)
for kk in ['nx', 'ny', 'nz']:
- self.parameters[kk] = self.get_data_file()['parameters/' + kk].value
+ self.parameters[kk] = self.get_data_file()['parameters/' + kk][...]
n = self.parameters['nx']
if self.dns_type in ['filtered_slices',
'filtered_acceleration']:
@@ -659,7 +649,7 @@ class PP(_code):
opt.klist_kmax = n / 3.
if type(opt.klist_kmin) == type(None):
opt.klist_kmin = 6.
- kvals = bfps_addons.tools.power_space_array(
+ kvals = TurTLE_addons.tools.power_space_array(
power = opt.klist_power,
size = opt.klist_size,
vmin = opt.klist_kmin,
@@ -674,10 +664,11 @@ class PP(_code):
group = self.dns_type + '/parameters',
parameters = self.pp_parameters,
file_name = os.path.join(self.work_dir, self.simname + '_post.h5'))
- histogram_bins = opt.histogram_bins
- if (type(histogram_bins) == type(None) and
- 'histogram_bins' in self.pp_parameters.keys()):
- histogram_bins = self.pp_parameters['histogram_bins']
+ if 'histogram_bins' in opt.__dict__.keys():
+ histogram_bins = opt.histogram_bins
+ if (type(histogram_bins) == type(None) and
+ 'histogram_bins' in self.pp_parameters.keys()):
+ histogram_bins = self.pp_parameters['histogram_bins']
with h5py.File(os.path.join(self.work_dir, self.simname + '_post.h5'), 'r+') as ofile:
group = ofile[self.dns_type]
group.require_group('histograms')
@@ -686,11 +677,19 @@ class PP(_code):
vec_spectra_stats = []
vec4_rspace_stats = []
scal_rspace_stats = []
+ if self.dns_type == 'bandpass_stats':
+ vec4_rspace_stats.append('velocity')
+ for kindex in range(len(self.pp_parameters['k0list'])):
+ vec4_rspace_stats.append('velocity_band{0}'.format(kindex))
if self.dns_type == 'joint_acc_vel_stats':
vec_spectra_stats.append('velocity')
vec4_rspace_stats.append('velocity')
vec_spectra_stats.append('acceleration')
vec4_rspace_stats.append('acceleration')
+ if self.dns_type == 'pressure_stats':
+ scal_rspace_stats.append('kk0_scalar')
+ scal_rspace_stats.append('kk3_scalar')
+ scal_rspace_stats.append('kk4_scalar')
for quantity in scal_rspace_stats:
if quantity not in group['histograms'].keys():
time_chunk = 2**20 // (8*histogram_bins)
@@ -772,39 +771,49 @@ class PP(_code):
dtype = np.float64)
df.close()
return None
- def prepare_field_file(self):
+ def prepare_field_file(self, iter0 = 0):
df = self.get_data_file()
if 'field_dtype' in df.keys():
# we don't need to do anything, raw binary files are used
return None
- last_iteration = df['iteration'].value
- cppf = df['parameters/checkpoints_per_file'].value
- niter_out = df['parameters/niter_out'].value
+ last_iteration = df['iteration'][()]
+ cppf = df['parameters/checkpoints_per_file'][()]
+ niter_out = df['parameters/niter_out'][()]
with h5py.File(os.path.join(self.work_dir, self.simname + '_fields.h5'), 'a') as ff:
ff.require_group('vorticity')
ff.require_group('vorticity/complex')
- checkpoint = 0
- while True:
- cpf_name = os.path.join(
- self.work_dir,
- self.simname + '_checkpoint_{0}.h5'.format(checkpoint))
+ checkpoint_file_list = [self.simname + '_checkpoint_{0}.h5'.format(cp)
+ for cp in range(df['checkpoint'][()]+1)]
+ for cpf_name in checkpoint_file_list:
if os.path.exists(cpf_name):
cpf = h5py.File(cpf_name, 'r')
- for iter_name in cpf['vorticity/complex'].keys():
- if iter_name not in ff['vorticity/complex'].keys():
- ff['vorticity/complex/' + iter_name] = h5py.ExternalLink(
- cpf_name,
- 'vorticity/complex/' + iter_name)
- checkpoint += 1
- else:
- break
+ if 'vorticity' not in cpf.keys():
+ print('file ', cpf_name, ' does not have vorticity group')
+ continue
+ else:
+ for iter_name in cpf['vorticity/complex'].keys():
+ if iter_name not in ff['vorticity/complex'].keys():
+ ff['vorticity/complex/' + iter_name] = h5py.ExternalLink(
+ cpf_name,
+ 'vorticity/complex/' + iter_name)
+ cpf.close()
+ if self.dns_type == 'resize':
+ with h5py.File(os.path.join(self.work_dir, self.pp_parameters['new_simname'] + '.h5'), 'a') as ff:
+ for kk in df['parameters'].keys():
+ ff['parameters/' + kk] = df['parameters/' + kk][()]
+ ff['parameters/nx'][()] = self.pp_parameters['new_nx']
+ ff['parameters/ny'][()] = self.pp_parameters['new_ny']
+ ff['parameters/nz'][()] = self.pp_parameters['new_nz']
+ if 'iteration' not in ff.keys():
+ ff['iteration'] = last_iteration
+ ff['checkpoint'] = df['checkpoint'][()]
return None
def launch_jobs(
self,
opt = None,
particle_initial_condition = None):
self.prepare_post_file(opt)
- self.prepare_field_file()
+ self.prepare_field_file(iter0 = opt.iter0)
self.run(
nb_processes = opt.nb_processes,
nb_threads_per_process = opt.nb_threads_per_process,
diff --git a/TurTLE/TEST.py b/TurTLE/TEST.py
new file mode 100644
index 0000000000000000000000000000000000000000..925f71f65fec8e1e9474aa78dc5ce48bd3dc440a
--- /dev/null
+++ b/TurTLE/TEST.py
@@ -0,0 +1,508 @@
+################################################################################
+# #
+# Copyright 2015-2019 Max Planck Institute for Dynamics and Self-Organization #
+# #
+# This file is part of TurTLE. #
+# #
+# TurTLE is free software: you can redistribute it and/or modify #
+# it under the terms of the GNU General Public License as published #
+# by the Free Software Foundation, either version 3 of the License, #
+# or (at your option) any later version. #
+# #
+# TurTLE is distributed in the hope that it will be useful, #
+# but WITHOUT ANY WARRANTY; without even the implied warranty of #
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the #
+# GNU General Public License for more details. #
+# #
+# You should have received a copy of the GNU General Public License #
+# along with TurTLE. If not, see <http://www.gnu.org/licenses/> #
+# #
+# Contact: Cristian.Lalescu@ds.mpg.de #
+# #
+################################################################################
+
+
+
+import os
+import sys
+import shutil
+import subprocess
+import argparse
+import h5py
+import math
+import numpy as np
+import warnings
+
+import TurTLE
+from ._code import _code
+from TurTLE import tools
+from TurTLE import DNS
+
+class TEST(_code):
+ """This class is meant to stitch together the C++ code into a final source file,
+ compile it, and handle all job launching.
+ """
+ def __init__(
+ self,
+ work_dir = './',
+ simname = 'test'):
+ _code.__init__(
+ self,
+ work_dir = work_dir,
+ simname = simname)
+ self.generate_default_parameters()
+ self.check_current_vorticity_exists = False
+ return None
+ def set_precision(
+ self,
+ fluid_dtype):
+ if fluid_dtype in [np.float32, np.float64]:
+ self.fluid_dtype = fluid_dtype
+ elif fluid_dtype in ['single', 'double']:
+ if fluid_dtype == 'single':
+ self.fluid_dtype = np.dtype(np.float32)
+ elif fluid_dtype == 'double':
+ self.fluid_dtype = np.dtype(np.float64)
+ self.rtype = self.fluid_dtype
+ if self.rtype == np.float32:
+ self.ctype = np.dtype(np.complex64)
+ self.C_field_dtype = 'float'
+ self.fluid_precision = 'single'
+ elif self.rtype == np.float64:
+ self.ctype = np.dtype(np.complex128)
+ self.C_field_dtype = 'double'
+ self.fluid_precision = 'double'
+ return None
+ def write_src(self):
+ self.version_message = (
+ '/***********************************************************************\n' +
+ '* this code automatically generated by TurTLE\n' +
+ '* version {0}\n'.format(TurTLE.__version__) +
+ '***********************************************************************/\n\n\n')
+ self.include_list = [
+ '"base.hpp"',
+ '"scope_timer.hpp"',
+ '"fftw_interface.hpp"',
+ '"full_code/main_code.hpp"',
+ '<cmath>',
+ '<iostream>',
+ '<hdf5.h>',
+ '<string>',
+ '<cstring>',
+ '<fftw3-mpi.h>',
+ '<omp.h>',
+ '<cfenv>',
+ '<cstdlib>',
+ '"full_code/{0}.hpp"\n'.format(self.dns_type)]
+ self.main = """
+ int main(int argc, char *argv[])
+ {{
+ bool fpe = (
+ (getenv("TURTLE_FPE_OFF") == nullptr) ||
+ (getenv("TURTLE_FPE_OFF") != std::string("TRUE")));
+ return main_code< {0} >(argc, argv, fpe);
+ }}
+ """.format(self.dns_type + '<{0}>'.format(self.C_field_dtype))
+ self.includes = '\n'.join(
+ ['#include ' + hh
+ for hh in self.include_list])
+ with open(self.name + '.cpp', 'w') as outfile:
+ outfile.write(self.version_message + '\n\n')
+ outfile.write(self.includes + '\n\n')
+ outfile.write(self.main + '\n')
+ return None
+ def generate_default_parameters(self):
+ # these parameters are relevant for all TEST classes
+ self.parameters['fftw_plan_rigor'] = 'FFTW_ESTIMATE'
+ self.parameters['dealias_type'] = int(1)
+ self.parameters['dkx'] = float(1.0)
+ self.parameters['dky'] = float(1.0)
+ self.parameters['dkz'] = float(1.0)
+ self.parameters['filter_length'] = float(1.0)
+ self.parameters['field_random_seed'] = int(1)
+ return None
+ def generate_extra_parameters(
+ self,
+ dns_type = None):
+ pars = {}
+ if dns_type == 'test_interpolation':
+ pars['nparticles'] = 3
+ pars['tracers0_integration_steps'] = int(4)
+ pars['tracers0_neighbours'] = int(1)
+ pars['tracers0_smoothness'] = int(1)
+ if dns_type == 'test_interpolation_methods':
+ pars['nxparticles'] = 160
+ pars['nzparticles'] = 180
+ if dns_type == 'test_particle_integration':
+ pars['nxparticles'] = 13
+ pars['nzparticles'] = 11
+ pars['dt'] = 0.125
+ pars['niter_todo'] = 128
+ if dns_type == 'phase_shift_test':
+ pars['random_phase_seed'] = 1
+ if dns_type in ['dealias_test', 'Gauss_field_test']:
+ pars['output_incompressible_field'] = int(0)
+ pars['histogram_bins'] = int(129)
+ pars['max_velocity_estimate'] = float(8)
+ pars['spectrum_dissipation'] = float(0.027)
+ pars['spectrum_Lint'] = float(1.3)
+ pars['spectrum_etaK'] = float(0.3)
+ pars['spectrum_large_scale_const'] = float(6.78)
+ pars['spectrum_small_scale_const'] = float(0.40)
+ return pars
+ def get_kspace(self):
+ kspace = {}
+ if self.parameters['dealias_type'] == 1:
+ kMx = self.parameters['dkx']*(self.parameters['nx']//2 - 1)
+ kMy = self.parameters['dky']*(self.parameters['ny']//2 - 1)
+ kMz = self.parameters['dkz']*(self.parameters['nz']//2 - 1)
+ else:
+ kMx = self.parameters['dkx']*(self.parameters['nx']//3 - 1)
+ kMy = self.parameters['dky']*(self.parameters['ny']//3 - 1)
+ kMz = self.parameters['dkz']*(self.parameters['nz']//3 - 1)
+ kspace['kM'] = max(kMx, kMy, kMz)
+ kspace['dk'] = min(self.parameters['dkx'],
+ self.parameters['dky'],
+ self.parameters['dkz'])
+ nshells = int(kspace['kM'] / kspace['dk']) + 2
+ kspace['nshell'] = np.zeros(nshells, dtype = np.int64)
+ kspace['kshell'] = np.zeros(nshells, dtype = np.float64)
+ kspace['kx'] = np.arange( 0,
+ self.parameters['nx']//2 + 1).astype(np.float64)*self.parameters['dkx']
+ kspace['ky'] = np.arange(-self.parameters['ny']//2 + 1,
+ self.parameters['ny']//2 + 1).astype(np.float64)*self.parameters['dky']
+ kspace['ky'] = np.roll(kspace['ky'], self.parameters['ny']//2+1)
+ kspace['kz'] = np.arange(-self.parameters['nz']//2 + 1,
+ self.parameters['nz']//2 + 1).astype(np.float64)*self.parameters['dkz']
+ kspace['kz'] = np.roll(kspace['kz'], self.parameters['nz']//2+1)
+ return kspace
+ def get_data_file_name(self):
+ return os.path.join(self.work_dir, self.simname + '.h5')
+ def get_data_file(self):
+ return h5py.File(self.get_data_file_name(), 'r')
+ def write_par(
+ self,
+ iter0 = 0,
+ particle_ic = None):
+ assert (iter0 == 0)
+ _code.write_par(self, iter0 = iter0)
+ with h5py.File(self.get_data_file_name(), 'r+') as ofile:
+ ofile['code_info/exec_name'] = self.name
+ kspace = self.get_kspace()
+ for k in kspace.keys():
+ ofile['kspace/' + k] = kspace[k]
+ nshells = kspace['nshell'].shape[0]
+ kspace = self.get_kspace()
+ nshells = kspace['nshell'].shape[0]
+ ofile['checkpoint'] = int(0)
+ vec_spectra_stats = []
+ tens_rspace_stats = []
+ vec4_rspace_stats = []
+ scal_rspace_stats = []
+ if self.dns_type in ['dealias_test']:
+ vec_spectra_stats.append('field3')
+ vec_spectra_stats.append('field4')
+ vec4_rspace_stats.append('field3')
+ vec4_rspace_stats.append('field4')
+ if self.dns_type in ['Gauss_field_test']:
+ vec_spectra_stats.append('velocity')
+ vec_spectra_stats.append('k*velocity')
+ vec_spectra_stats.append('k2*velocity')
+ vec4_rspace_stats.append('velocity')
+ tens_rspace_stats.append('velocity_gradient')
+ scal_rspace_stats.append('velocity_divergence')
+ for k in vec_spectra_stats:
+ time_chunk = 2**20//(8*3*3*nshells)
+ time_chunk = max(time_chunk, 1)
+ ofile.create_dataset('statistics/spectra/' + k + '_' + k,
+ (1, nshells, 3, 3),
+ chunks = (time_chunk, nshells, 3, 3),
+ maxshape = (None, nshells, 3, 3),
+ dtype = np.float64)
+ for k in scal_rspace_stats:
+ time_chunk = 2**20//(8*10)
+ time_chunk = max(time_chunk, 1)
+ a = ofile.create_dataset('statistics/moments/' + k,
+ (1, 10),
+ chunks = (time_chunk, 10),
+ maxshape = (None, 10),
+ dtype = np.float64)
+ time_chunk = 2**20//(8*self.parameters['histogram_bins'])
+ time_chunk = max(time_chunk, 1)
+ ofile.create_dataset('statistics/histograms/' + k,
+ (1,
+ self.parameters['histogram_bins']),
+ chunks = (time_chunk,
+ self.parameters['histogram_bins']),
+ maxshape = (None,
+ self.parameters['histogram_bins']),
+ dtype = np.int64)
+ for k in vec4_rspace_stats:
+ time_chunk = 2**20//(8*4*10)
+ time_chunk = max(time_chunk, 1)
+ a = ofile.create_dataset('statistics/moments/' + k,
+ (1, 10, 4),
+ chunks = (time_chunk, 10, 4),
+ maxshape = (None, 10, 4),
+ dtype = np.float64)
+ time_chunk = 2**20//(8*4*self.parameters['histogram_bins'])
+ time_chunk = max(time_chunk, 1)
+ ofile.create_dataset('statistics/histograms/' + k,
+ (1,
+ self.parameters['histogram_bins'],
+ 4),
+ chunks = (time_chunk,
+ self.parameters['histogram_bins'],
+ 4),
+ maxshape = (None,
+ self.parameters['histogram_bins'],
+ 4),
+ dtype = np.int64)
+ for k in tens_rspace_stats:
+ time_chunk = 2**20//(8*9*10)
+ time_chunk = max(time_chunk, 1)
+ a = ofile.create_dataset('statistics/moments/' + k,
+ (1, 10, 3, 3),
+ chunks = (time_chunk, 10, 3, 3),
+ maxshape = (None, 10, 3, 3),
+ dtype = np.float64)
+ time_chunk = 2**20//(8*9*self.parameters['histogram_bins'])
+ time_chunk = max(time_chunk, 1)
+ ofile.create_dataset('statistics/histograms/' + k,
+ (1,
+ self.parameters['histogram_bins'],
+ 3, 3),
+ chunks = (time_chunk,
+ self.parameters['histogram_bins'],
+ 3, 3),
+ maxshape = (None,
+ self.parameters['histogram_bins'],
+ 3, 3),
+ dtype = np.int64)
+ return None
+ def simulation_parser_arguments(
+ self,
+ parser):
+ parser.add_argument(
+ '--simname',
+ type = str, dest = 'simname',
+ default = 'test')
+ parser.add_argument(
+ '-n', '--grid-size',
+ type = int,
+ dest = 'n',
+ default = 32,
+ metavar = 'N',
+ help = 'code is run by default in a grid of NxNxN')
+ for coord in ['x', 'y', 'z']:
+ parser.add_argument(
+ '--L{0}'.format(coord), '--box-length-{0}'.format(coord),
+ type = float,
+ dest = 'L{0}'.format(coord),
+ default = 2.0,
+ metavar = 'length{0}'.format(coord),
+ help = 'length of the box in the {0} direction will be `length{0} x pi`'.format(coord))
+ parser.add_argument(
+ '--wd',
+ type = str, dest = 'work_dir',
+ default = './')
+ parser.add_argument(
+ '--precision',
+ choices = ['single', 'double'],
+ type = str,
+ default = 'single')
+ return None
+ def add_parser_arguments(
+ self,
+ parser):
+ subparsers = parser.add_subparsers(
+ dest = 'TEST_class',
+ help = 'type of simulation to run')
+ subparsers.required = True
+ parser_dealias_test = subparsers.add_parser(
+ 'dealias_test',
+ help = 'generate spectra for two thirds and one half dealiasing')
+ parser_phase_shift_test = subparsers.add_parser(
+ 'phase_shift_test',
+ help = 'test phase shift functionality')
+ parser_Gauss_field_test = subparsers.add_parser(
+ 'Gauss_field_test',
+ help = 'test incompressible Gaussian random fields')
+ parser_filter_test = subparsers.add_parser(
+ 'filter_test',
+ help = 'plain filter test')
+ parser_write_filtered_test = subparsers.add_parser(
+ 'write_filtered_test',
+ help = 'test of write_filtered functionality')
+ parser_field_test = subparsers.add_parser(
+ 'field_test',
+ help = 'plain field test')
+ parser_symmetrize_test = subparsers.add_parser(
+ 'symmetrize_test',
+ help = 'plain symmetrize test')
+ parser_field_output_test = subparsers.add_parser(
+ 'field_output_test',
+ help = 'plain field output test')
+ parser_test_interpolation = subparsers.add_parser(
+ 'test_interpolation',
+ help = 'test velocity gradient interpolation')
+ parser_test_interpolation_methods = subparsers.add_parser(
+ 'test_interpolation_methods',
+ help = 'test interpolation methods')
+ parser_test_particle_integration = subparsers.add_parser(
+ 'test_particle_integration',
+ help = 'test interpolation methods')
+ parser_ornstein_uhlenbeck_test = subparsers.add_parser(
+ 'ornstein_uhlenbeck_test',
+ help = 'test ornstein uhlenbeck process')
+ parser_test_tracer_set = subparsers.add_parser(
+ 'test_tracer_set',
+ help = 'basic test of tracer set solver')
+
+ for parser in ['dealias_test',
+ 'phase_shift_test',
+ 'Gauss_field_test',
+ 'filter_test',
+ 'write_filtered_test',
+ 'field_test',
+ 'symmetrize_test',
+ 'field_output_test',
+ 'test_interpolation',
+ 'test_interpolation_methods',
+ 'test_particle_integration',
+ 'ornstein_uhlenbeck_test',
+ 'test_tracer_set']:
+ eval('self.simulation_parser_arguments(parser_' + parser + ')')
+ eval('self.job_parser_arguments(parser_' + parser + ')')
+ eval('self.parameters_to_parser_arguments(parser_' + parser + ')')
+ eval('self.parameters_to_parser_arguments(parser_' + parser + ', self.generate_extra_parameters(dns_type = \'' + parser + '\'))')
+ return None
+ def prepare_launch(
+ self,
+ args = []):
+ opt = _code.prepare_launch(self, args = args)
+ self.set_precision(opt.precision)
+ self.dns_type = opt.TEST_class
+ self.name = self.dns_type + '-' + self.fluid_precision + '-v' + TurTLE.__version__
+ self.parameters.update(
+ self.generate_extra_parameters(dns_type = self.dns_type))
+ # merge parameters if needed
+ self.pars_from_namespace(opt)
+ return opt
+ def launch(
+ self,
+ args = [],
+ **kwargs):
+ opt = self.prepare_launch(args = args)
+ self.launch_jobs(opt = opt, **kwargs)
+ return None
+ def launch_jobs(
+ self,
+ opt = None,
+ particle_initial_condition = None):
+ if not os.path.exists(os.path.join(self.work_dir, self.simname + '.h5')):
+ # the write_par call creates the work dir fi needed
+ self.write_par(
+ particle_ic = particle_initial_condition)
+ if self.dns_type == 'test_tracer_set':
+ ofile = h5py.File(os.path.join(self.work_dir, 'test_particle_sample.h5'), 'w')
+ ofile.create_group('tracers0')
+ ofile.create_group('tracers0/position')
+ ofile.create_group('tracers0/velocity')
+ ofile.close()
+ if self.dns_type == 'test_interpolation_methods':
+ ofile = h5py.File(os.path.join(self.work_dir, self.simname + '_particles.h5'), 'w')
+ ofile.create_group('particle')
+ ofile.create_group('particle/position')
+ for n, m in [(0, 0),
+ (1, 1),
+ (2, 1), (2, 2),
+ (3, 1), (3, 2),
+ (4, 1), (4, 2),
+ (5, 1), (5, 2)]:
+ ofile.create_group('particle/phi_{0}{1}'.format(n, m))
+ ofile.close()
+ if self.dns_type == 'test_particle_integration':
+ ofile = h5py.File(os.path.join(self.work_dir, self.simname + '_particles.h5'), 'w')
+ ofile.create_group('particle')
+ ofile.create_group('particle/position')
+ for n, m in [(0, 0),
+ (1, 1),
+ (2, 1), (2, 2),
+ (3, 1), (3, 2),
+ (4, 1), (4, 2),
+ (5, 1), (5, 2)]:
+ for integration_order in [1, 2, 4]:
+ for factor in [1, 2, 4, 8]:
+ ofile.create_group(
+ 'particle_o{0}_n{1}_m{2}_f{3}/position'.format(
+ integration_order, n, m, factor))
+ ofile.create_group(
+ 'particle_o{0}_n{1}_m{2}_f{3}/velocity'.format(
+ integration_order, n, m, factor))
+ ofile.close()
+ if self.dns_type == 'test_interpolation':
+ if type(particle_initial_condition) == type(None):
+ pbase_shape = (self.parameters['nparticles'],)
+ number_of_particles = self.parameters['nparticles']
+ else:
+ pbase_shape = particle_initial_condition.shape[:-1]
+ assert(particle_initial_condition.shape[-1] == 3)
+ number_of_particles = 1
+ for val in pbase_shape[1:]:
+ number_of_particles *= val
+ ncomponents = 3
+ with h5py.File(os.path.join(self.work_dir, self.simname + '_input.h5'), 'a') as ofile:
+ s = 0
+ ofile.create_group('tracers{0}'.format(s))
+ ofile.create_group('tracers{0}/rhs'.format(s))
+ ofile.create_group('tracers{0}/state'.format(s))
+ ofile['tracers{0}/rhs'.format(s)].create_dataset(
+ '0',
+ shape = (
+ (self.parameters['tracers{0}_integration_steps'.format(s)],) +
+ pbase_shape +
+ (ncomponents,)),
+ dtype = np.float)
+ ofile['tracers{0}/state'.format(s)].create_dataset(
+ '0',
+ shape = (
+ pbase_shape +
+ (ncomponents,)),
+ dtype = np.float)
+ if type(particle_initial_condition) == type(None):
+ ofile['tracers0/state/0'][:] = np.random.random(pbase_shape + (ncomponents,))*2*np.pi
+ else:
+ ofile['tracers0/state/0'][:] = particle_initial_condition
+ with h5py.File(os.path.join(self.work_dir, self.simname + '_input.h5'), 'a') as ofile:
+ data = DNS.generate_vector_field(self,
+ write_to_file = False,
+ spectra_slope = 1.0,
+ amplitude = 0.05)
+ #data[:] = 0.0
+ ## ABC
+ #data[0, 0, 1, 1] = -0.5*(1j)
+ #data[0, 0, 1, 2] = 0.5*(1j)
+ #data[0, 1, 0, 0] = -0.5*(1j)
+ #data[0, self.parameters['nz'] - 1, 0, 0] = 0.5*(1j)
+ #data[0, 1, 0, 1] = 0.5
+ #data[0, self.parameters['nz'] - 1, 0, 1] = 0.5
+ #data[1, 0, 0, 0] = 0.5
+ #data[self.parameters['ny'] - 1, 0, 0, 0] = 0.5
+ #data[1, 0, 0, 2] = -0.5*(1j)
+ #data[self.parameters['ny'] - 1, 0, 0, 2] = 0.5*(1j)
+ ofile['vorticity/complex/{0}'.format(0)] = data
+ with h5py.File(os.path.join(self.work_dir, self.simname + '_output.h5'), 'a') as ofile:
+ ofile.require_group('tracers0')
+ for kk in ['position', 'velocity', 'vorticity', 'velocity_gradient']:
+ ofile['tracers0'].require_group(kk)
+
+
+ self.run(
+ nb_processes = opt.nb_processes,
+ nb_threads_per_process = opt.nb_threads_per_process,
+ njobs = opt.njobs,
+ hours = opt.minutes // 60,
+ minutes = opt.minutes % 60,
+ no_submit = opt.no_submit)
+ return None
diff --git a/TurTLE/__init__.py.in b/TurTLE/__init__.py.in
new file mode 100644
index 0000000000000000000000000000000000000000..4c08dcecdbbf90e24cc693e3a2577e8d2b1bfbbd
--- /dev/null
+++ b/TurTLE/__init__.py.in
@@ -0,0 +1,38 @@
+################################################################################
+# #
+# Copyright 2015-2019 Max Planck Institute for Dynamics and Self-Organization #
+# #
+# This file is part of TurTLE. #
+# #
+# TurTLE is free software: you can redistribute it and/or modify #
+# it under the terms of the GNU General Public License as published #
+# by the Free Software Foundation, either version 3 of the License, #
+# or (at your option) any later version. #
+# #
+# TurTLE is distributed in the hope that it will be useful, #
+# but WITHOUT ANY WARRANTY; without even the implied warranty of #
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the #
+# GNU General Public License for more details. #
+# #
+# You should have received a copy of the GNU General Public License #
+# along with TurTLE. If not, see <http://www.gnu.org/licenses/> #
+# #
+# Contact: Cristian.Lalescu@ds.mpg.de #
+# #
+################################################################################
+
+
+
+import os
+
+__version__ = '@TURTLE_VERSION_LONG@'
+
+data_dir = os.path.join('@CMAKE_INSTALL_PREFIX@', 'share/TurTLE-@TURTLE_VERSION_LONG@')
+
+from .host_info import host_info
+
+from .DNS import DNS
+from .PP import PP
+from .TEST import TEST
+
+#import test
diff --git a/TurTLE/__main__.py b/TurTLE/__main__.py
new file mode 100644
index 0000000000000000000000000000000000000000..4f96bad4ef3c1a2f9712e3c53cbf7f718ab47bd7
--- /dev/null
+++ b/TurTLE/__main__.py
@@ -0,0 +1,60 @@
+################################################################################
+# #
+# Copyright 2015-2019 Max Planck Institute for Dynamics and Self-Organization #
+# #
+# This file is part of TurTLE. #
+# #
+# TurTLE is free software: you can redistribute it and/or modify #
+# it under the terms of the GNU General Public License as published #
+# by the Free Software Foundation, either version 3 of the License, #
+# or (at your option) any later version. #
+# #
+# TurTLE is distributed in the hope that it will be useful, #
+# but WITHOUT ANY WARRANTY; without even the implied warranty of #
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the #
+# GNU General Public License for more details. #
+# #
+# You should have received a copy of the GNU General Public License #
+# along with TurTLE. If not, see <http://www.gnu.org/licenses/> #
+# #
+# Contact: Cristian.Lalescu@ds.mpg.de #
+# #
+################################################################################
+
+
+
+import sys
+import argparse
+
+import TurTLE
+from .DNS import DNS
+from .PP import PP
+from .TEST import TEST
+
+def main():
+ parser = argparse.ArgumentParser(prog = 'turtle', conflict_handler = 'resolve')
+ parser.add_argument(
+ '-v', '--version',
+ action = 'version',
+ version = '%(prog)s ' + TurTLE.__version__)
+ parser.add_argument(
+ 'base_class',
+ choices = ['DNS', 'PP', 'TEST'],
+ type = str)
+ # first option is the choice of base class or -h or -v
+ # all other options are passed on to the base_class instance
+ opt = parser.parse_args(sys.argv[1:2])
+ # error is thrown if first option is not a base class, so launch
+ # cannot be executed by mistake.
+ if opt.base_class == 'DNS':
+ c = DNS()
+ if opt.base_class == 'PP':
+ c = PP()
+ if opt.base_class == 'TEST':
+ c = TEST()
+ c.launch(args = sys.argv[2:])
+ return None
+
+if __name__ == '__main__':
+ main()
+
diff --git a/bfps/_base.py b/TurTLE/_base.py
similarity index 70%
rename from bfps/_base.py
rename to TurTLE/_base.py
index 037261d3f1c6ea7af7fc58b79484ed461f84a28b..2b882dd0bb0004640901bef83cac7d228dc3bb4a 100644
--- a/bfps/_base.py
+++ b/TurTLE/_base.py
@@ -1,26 +1,25 @@
-#######################################################################
-# #
-# Copyright 2015 Max Planck Institute #
-# for Dynamics and Self-Organization #
-# #
-# This file is part of bfps. #
-# #
-# bfps is free software: you can redistribute it and/or modify #
-# it under the terms of the GNU General Public License as published #
-# by the Free Software Foundation, either version 3 of the License, #
-# or (at your option) any later version. #
-# #
-# bfps is distributed in the hope that it will be useful, #
-# but WITHOUT ANY WARRANTY; without even the implied warranty of #
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the #
-# GNU General Public License for more details. #
-# #
-# You should have received a copy of the GNU General Public License #
-# along with bfps. If not, see <http://www.gnu.org/licenses/> #
-# #
-# Contact: Cristian.Lalescu@ds.mpg.de #
-# #
-#######################################################################
+################################################################################
+# #
+# Copyright 2015-2019 Max Planck Institute for Dynamics and Self-Organization #
+# #
+# This file is part of bfps. #
+# #
+# bfps is free software: you can redistribute it and/or modify #
+# it under the terms of the GNU General Public License as published #
+# by the Free Software Foundation, either version 3 of the License, #
+# or (at your option) any later version. #
+# #
+# bfps is distributed in the hope that it will be useful, #
+# but WITHOUT ANY WARRANTY; without even the implied warranty of #
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the #
+# GNU General Public License for more details. #
+# #
+# You should have received a copy of the GNU General Public License #
+# along with bfps. If not, see <http://www.gnu.org/licenses/> #
+# #
+# Contact: Cristian.Lalescu@ds.mpg.de #
+# #
+################################################################################
@@ -28,8 +27,7 @@ import os
import sys
import numpy as np
import h5py
-from bfps import install_info
-from bfps import __version__
+from TurTLE import __version__
class _base(object):
"""This class contains simulation parameters, and handles parameter related
@@ -44,6 +42,11 @@ class _base(object):
self.parameters = {'nx' : 32,
'ny' : 32,
'nz' : 32}
+ self.parameter_description = {
+ 'nx' : 'Number of real-space grid nodes in the x direction.',
+ 'ny' : 'Number of real-space grid nodes in the y direction.',
+ 'nz' : 'Number of real-space grid nodes in the z direction.',
+ }
self.string_length = 512
self.work_dir = os.path.realpath(work_dir)
self.simname = simname
@@ -97,7 +100,7 @@ class _base(object):
'char fname[256];\n' +
'hsize_t dims[1];\n' +
'char *string_data;\n' +
- 'sprintf(fname, "%s.h5", {0});\n'.format(simname_variable) +
+ 'snprintf(fname, 255, "%s.h5", {0});\n'.format(simname_variable) +
'parameter_file = H5Fopen(fname, H5F_ACC_RDONLY, H5P_DEFAULT);\n')
key_prefix = ''
if prepend_this:
@@ -106,18 +109,24 @@ class _base(object):
src_txt += 'dset = H5Dopen(parameter_file, "/{0}/{1}", H5P_DEFAULT);\n'.format(
file_group, key[i])
if (type(parameters[key[i]]) == int and parameters[key[i]] >= 1<<30):
- src_txt += 'H5Dread(dset, H5T_NATIVE_LLONG, H5S_ALL, H5S_ALL, H5P_DEFAULT, &{0});\n'.format(key_prefix + key[i])
+ src_txt += ('if (dset > 0) H5Dread(dset, H5T_NATIVE_LLONG, H5S_ALL, H5S_ALL, H5P_DEFAULT, &{0});\n'
+ + 'else {0} = 0;\n').format(key_prefix + key[i])
elif type(parameters[key[i]]) == int:
- src_txt += 'H5Dread(dset, H5T_NATIVE_INT, H5S_ALL, H5S_ALL, H5P_DEFAULT, &{0});\n'.format(key_prefix + key[i])
+ src_txt += ('if (dset > 0) H5Dread(dset, H5T_NATIVE_INT, H5S_ALL, H5S_ALL, H5P_DEFAULT, &{0});\n'
+ + 'else {0} = 0;\n').format(key_prefix + key[i])
elif type(parameters[key[i]]) == str:
- src_txt += ('space = H5Dget_space(dset);\n' +
+ src_txt += ('if (dset > 0)\n' +
+ '{\n'
+ 'space = H5Dget_space(dset);\n' +
'memtype = H5Dget_type(dset);\n' +
'string_data = (char*)malloc(256);\n' +
'H5Dread(dset, memtype, H5S_ALL, H5S_ALL, H5P_DEFAULT, &string_data);\n' +
- 'sprintf({0}, "%s", string_data);\n'.format(key_prefix + key[i]) +
+ 'snprintf({0}, 255, "%s", string_data);\n'.format(key_prefix + key[i]) +
'free(string_data);\n'
'H5Sclose(space);\n' +
- 'H5Tclose(memtype);\n')
+ 'H5Tclose(memtype);\n' +
+ '}\n' +
+ 'else printf({0}, "NULL");\n'.format(key_prefix + key[i]))
elif type(parameters[key[i]]) == np.ndarray:
if parameters[key[i]].dtype in [np.int, np.int64, np.int32]:
template_par = 'int'
@@ -126,10 +135,11 @@ class _base(object):
src_txt += '{0} = hdf5_tools::read_vector<{1}>(parameter_file, "/{2}/{0}");\n'.format(
key_prefix + key[i], template_par, file_group)
else:
- src_txt += 'H5Dread(dset, H5T_NATIVE_DOUBLE, H5S_ALL, H5S_ALL, H5P_DEFAULT, &{0});\n'.format(key_prefix + key[i])
+ src_txt += ('if (dset > 0) H5Dread(dset, H5T_NATIVE_DOUBLE, H5S_ALL, H5S_ALL, H5P_DEFAULT, &{0});\n' +
+ 'else {0} = 0.0;\n').format(key_prefix + key[i])
src_txt += 'H5Dclose(dset);\n'
src_txt += 'H5Fclose(parameter_file);\n'
- src_txt += 'return 0;\n}\n' # finishing read_parameters
+ src_txt += 'return EXIT_SUCCESS;\n}\n' # finishing read_parameters
return src_txt
def cprint_pars(self):
key = sorted(list(self.parameters.keys()))
@@ -157,20 +167,23 @@ class _base(object):
return src_txt
def write_par(
self,
- iter0 = 0):
+ iter0 = 0,
+ force = False):
if not os.path.isdir(self.work_dir):
os.makedirs(self.work_dir)
- ofile = h5py.File(os.path.join(self.work_dir, self.simname + '.h5'), 'w-')
- for k in self.parameters.keys():
- if (type(self.parameters[k]) == str) and (sys.version_info[0] == 3):
- ofile['parameters/' + k] = bytes(self.parameters[k], 'ascii')
- else:
- ofile['parameters/' + k] = self.parameters[k]
- ofile['iteration'] = int(iter0)
- ofile['bfps_info/solver_class'] = type(self).__name__
- for k in install_info.keys():
- ofile['bfps_info/' + k] = str(install_info[k])
- ofile.close()
+ if not os.path.exists(os.path.join(self.work_dir, self.simname + '.h5')) or force:
+ ofile = h5py.File(os.path.join(self.work_dir, self.simname + '.h5'), 'w-')
+ for k in self.parameters.keys():
+ if (type(self.parameters[k]) == str) and (sys.version_info[0] == 3):
+ ofile['parameters/' + k] = bytes(self.parameters[k], 'ascii')
+ else:
+ ofile['parameters/' + k] = self.parameters[k]
+ ofile['iteration'] = int(iter0)
+ ofile['code_info/solver_class'] = type(self).__name__
+ ofile['code_info/VERSION'] = __version__
+ return ofile.close()
+ else:
+ print('Warning: you are attempting to overwrite simulation parameters, but did not specify "force = True". I will not overwrite --- you should be using "rewrite", in any case.')
return None
def rewrite_par(
self,
@@ -217,14 +230,32 @@ class _base(object):
ofile[group + '/' + k][...] = parameters[k]
ofile.close()
return None
- def read_parameters(self):
- with h5py.File(os.path.join(self.work_dir, self.simname + '.h5'), 'r') as data_file:
+ def read_parameters(self, fname = None):
+ if type(fname) == type(None):
+ fname = os.path.join(self.work_dir, self.simname + '.h5')
+ with h5py.File(fname, 'r') as data_file:
for k in data_file['parameters'].keys():
if k in self.parameters.keys():
- if type(self.parameters[k]) in [int, str, float]:
- self.parameters[k] = type(self.parameters[k])(data_file['parameters/' + k].value)
+ if type(self.parameters[k]) in [int, float]:
+ self.parameters[k] = type(self.parameters[k])(data_file['parameters/' + k][...])
+ if data_file['parameters/' + k].size == 1:
+ self.parameters[k] = type(self.parameters[k])(data_file['parameters/' + k][()])
+ else:
+ self.parameters[k] = np.array(data_file['parameters/' + k][()]).astype(type(self.parameters[k]))
+ elif type(self.parameters[k]) == str:
+ assert(type(data_file['parameters/' + k][()]) == bytes)
+ self.parameters[k] = data_file['parameters/' + k][()].decode('ascii')
+ else:
+ self.parameters[k] = data_file['parameters/' + k][()]
+ else:
+ values = data_file['parameters/' + k][()]
+ if type(values) == bytes:
+ self.parameters[k] = str(values.decode('ascii'))
else:
- self.parameters[k] = data_file['parameters/' + k].value
+ if values.size == 1:
+ self.parameters[k] = values.dtype.type(values)
+ else:
+ self.parameters[k] = values
return None
def pars_from_namespace(
self,
@@ -252,23 +283,16 @@ class _base(object):
def add_parser_arguments(
self,
parser):
- self.specific_parser_arguments(parser)
+ self.job_parser_arguments(parser)
self.parameters_to_parser_arguments(parser)
return None
- def specific_parser_arguments(
+ def job_parser_arguments(
self,
parser):
parser.add_argument(
'-v', '--version',
action = 'version',
version = '%(prog)s ' + __version__)
- parser.add_argument(
- '-n', '--cube-size',
- type = int,
- dest = 'n',
- default = 32,
- metavar = 'N',
- help = 'code is run by default in a grid of NxNxN')
parser.add_argument(
'--ncpu',
type = int,
@@ -293,37 +317,53 @@ class _base(object):
action = 'store_true',
dest = 'no_submit')
parser.add_argument(
- '--simname',
- type = str, dest = 'simname',
- default = 'test')
+ '--no-debug',
+ action = 'store_true',
+ dest = 'no_debug')
parser.add_argument(
'--environment',
type = str,
dest = 'environment',
default = None)
- parser.add_argument(
- '--wd',
- type = str, dest = 'work_dir',
- default = './')
parser.add_argument(
'--minutes',
type = int,
dest = 'minutes',
default = 5,
help = 'If environment supports it, this is the requested wall-clock-limit.')
-
+ parser.add_argument(
+ '--njobs',
+ type = int, dest = 'njobs',
+ default = 1)
+ parser.add_argument(
+ '--profile-with-vtune',
+ action = 'store_true',
+ dest = 'use_vtune')
+ parser.add_argument(
+ '--profile-with-aps',
+ action = 'store_true',
+ dest = 'use_aps')
return None
def parameters_to_parser_arguments(
self,
parser,
- parameters = None):
+ parameters = None,
+ parameter_description = None):
+ """Adds dict of parameters to parser arguments."""
if type(parameters) == type(None):
parameters = self.parameters
+ if type(parameter_description) == type(None):
+ parameter_description = self.parameter_description
for k in sorted(parameters.keys()):
+ if k in parameter_description.keys():
+ description = parameter_description[k]
+ else:
+ description = 'No description available.'
parser.add_argument(
'--{0}'.format(k),
type = type(parameters[k]),
dest = k,
+ help = description,
default = None)
return None
diff --git a/bfps/_code.py b/TurTLE/_code.py
similarity index 57%
rename from bfps/_code.py
rename to TurTLE/_code.py
index 22bcd9101ff6591e00f0455c1de1af2698c5f842..24481fbca5913a103f762b042e8076c744fe672c 100644
--- a/bfps/_code.py
+++ b/TurTLE/_code.py
@@ -1,26 +1,25 @@
-#######################################################################
-# #
-# Copyright 2015 Max Planck Institute #
-# for Dynamics and Self-Organization #
-# #
-# This file is part of bfps. #
-# #
-# bfps is free software: you can redistribute it and/or modify #
-# it under the terms of the GNU General Public License as published #
-# by the Free Software Foundation, either version 3 of the License, #
-# or (at your option) any later version. #
-# #
-# bfps is distributed in the hope that it will be useful, #
-# but WITHOUT ANY WARRANTY; without even the implied warranty of #
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the #
-# GNU General Public License for more details. #
-# #
-# You should have received a copy of the GNU General Public License #
-# along with bfps. If not, see <http://www.gnu.org/licenses/> #
-# #
-# Contact: Cristian.Lalescu@ds.mpg.de #
-# #
-#######################################################################
+################################################################################
+# #
+# Copyright 2015-2019 Max Planck Institute for Dynamics and Self-Organization #
+# #
+# This file is part of bfps. #
+# #
+# TurTLE is free software: you can redistribute it and/or modify #
+# it under the terms of the GNU General Public License as published #
+# by the Free Software Foundation, either version 3 of the License, #
+# or (at your option) any later version. #
+# #
+# TurTLE is distributed in the hope that it will be useful, #
+# but WITHOUT ANY WARRANTY; without even the implied warranty of #
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the #
+# GNU General Public License for more details. #
+# #
+# You should have received a copy of the GNU General Public License #
+# along with TurTLE. If not, see <http://www.gnu.org/licenses/> #
+# #
+# Contact: Cristian.Lalescu@ds.mpg.de #
+# #
+################################################################################
@@ -34,7 +33,9 @@ from datetime import datetime
import math
import warnings
-import bfps
+import TurTLE
+import TurTLE.tools
+from TurTLE.host_info import host_info
from ._base import _base
class _code(_base):
@@ -48,8 +49,8 @@ class _code(_base):
_base.__init__(self, work_dir = work_dir, simname = simname)
self.version_message = (
'/***********************************************************************\n' +
- '* this code automatically generated by bfps\n' +
- '* version {0}\n'.format(bfps.__version__) +
+ '* this code automatically generated by TurTLE\n' +
+ '* version {0}\n'.format(TurTLE.__version__) +
'***********************************************************************/\n\n\n')
self.includes = """
//begincpp
@@ -62,7 +63,7 @@ class _code(_base):
#include <string>
#include <cstring>
#include <fftw3-mpi.h>
- #include <omp.h>
+ #include <omp.h>
#include <fenv.h>
#include <cstdlib>
//endcpp
@@ -76,7 +77,7 @@ class _code(_base):
//begincpp
int main(int argc, char *argv[])
{
- if(getenv("BFPS_FPE_OFF") == nullptr || getenv("BFPS_FPE_OFF") != std::string("TRUE")){
+ if(getenv("TURTLE_FPE_OFF") == nullptr || getenv("TURTLE_FPE_OFF") != std::string("TRUE")){
feenableexcept(FE_INVALID | FE_OVERFLOW);
}
else{
@@ -116,7 +117,7 @@ class _code(_base):
}
#endif
strcpy(simname, argv[1]);
- sprintf(fname, "%s.h5", simname);
+ snprintf(fname, 255, "%s.h5", simname);
parameter_file = H5Fopen(fname, H5F_ACC_RDONLY, H5P_DEFAULT);
Cdset = H5Dopen(parameter_file, "iteration", H5P_DEFAULT);
H5Dread(
@@ -174,13 +175,10 @@ class _code(_base):
}
//endcpp
"""
- self.host_info = {'type' : 'cluster',
- 'environment' : None,
- 'deltanprocs' : 1,
- 'queue' : '',
- 'mail_address': '',
- 'mail_events' : None}
+ self.host_info = host_info
self.main = ''
+ self.check_current_vorticity_exists = True
+ self.check_current_velocity_exists = False
return None
def write_src(self):
with open(self.name + '.cpp', 'w') as outfile:
@@ -194,33 +192,64 @@ class _code(_base):
outfile.write(self.main)
outfile.write(self.main_end)
return None
- def compile_code(self):
+ def compile_code(
+ self,
+ no_debug = True):
+ if os.path.exists(os.path.join(self.work_dir, self.name)):
+ return 0
# compile code
- if not os.path.isfile(os.path.join(bfps.header_dir, 'base.hpp')):
- raise IOError('header not there:\n' +
- '{0}\n'.format(os.path.join(bfps.header_dir, 'base.hpp')) +
- '{0}\n'.format(bfps.dist_loc))
- libraries = ['bfps']
- libraries += bfps.install_info['libraries']
-
- command_strings = [bfps.install_info['compiler']]
- command_strings += [self.name + '.cpp', '-o', self.name]
- command_strings += bfps.install_info['extra_compile_args']
- command_strings += ['-I' + idir for idir in bfps.install_info['include_dirs']]
- command_strings.append('-I' + bfps.header_dir)
- command_strings += ['-L' + ldir for ldir in bfps.install_info['library_dirs']]
- command_strings += ['-Wl,-rpath=' + ldir for ldir in bfps.install_info['library_dirs']]
- command_strings.append('-L' + bfps.lib_dir)
- command_strings.append('-Wl,-rpath=' + bfps.lib_dir)
-
- for libname in libraries:
- command_strings += ['-l' + libname]
-
- command_strings += ['-fopenmp']
-
+ build_dir = 'TurTLE_build_' + self.name
+ os.makedirs(build_dir, exist_ok = True)
+ os.chdir(build_dir)
self.write_src()
- print('compiling code with command\n' + ' '.join(command_strings))
- return subprocess.call(command_strings)
+ with open('CMakeLists.txt', 'w') as outfile:
+ outfile.write('cmake_minimum_required(VERSION 3.10)\n')
+ outfile.write('cmake_policy(VERSION 3.12)\n')
+ outfile.write('project(project_{0} LANGUAGES C CXX)\n'.format(self.name))
+ outfile.write('set(CMAKE_CXX_STANDARD 17)\n')
+ outfile.write('set(CMAKE_CXX_STANDARD_REQUIRED ON)\n')
+ outfile.write('find_package(OpenMP REQUIRED)\n')
+ outfile.write('set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${OpenMP_CXX_FLAGS}")\n')
+ outfile.write('find_package(MPI REQUIRED)\n')
+ outfile.write('set(CMAKE_CXX_COMPILE_FLAGS "${CMAKE_CXX_COMPILE_FLAGS} ${MPI_CXX_COMPILE_OPTIONS}")\n')
+ outfile.write('include_directories(${MPI_CXX_INCLUDE_DIRS})\n')
+ outfile.write('add_definitions(${MPI_CXX_COMPILE_DEFINITIONS})\n')
+ outfile.write('list(APPEND TURTLE_LIBS "${MPI_CXX_LIBRARIES}")\n')
+ outfile.write('list(APPEND TURTLE_LIBS "${OpenMP_CXX_LIB_NAMES}")\n')
+ #ideally we should use something like the following 2 lines
+ #outfile.write('set(CMAKE_CXX_COMPILER ${TURTLE_CXX_COMPILER})\n')
+ #outfile.write('set(CMAKE_C_COMPILER ${TURTLE_C_COMPILER})\n')
+ outfile.write('find_package(TurTLE REQUIRED)\n')
+ outfile.write('set(CMAKE_CXX_COMPILE_FLAGS "${CMAKE_CXX_COMPILE_FLAGS} ${TURTLE_CXX_COMPILE_FLAGS}")\n')
+ outfile.write('set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${CMAKE_CXX_COMPILE_FLAGS}")\n')
+ outfile.write('set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} ${TURTLE_EXE_LINKER_FLAGS}")\n')
+ outfile.write('set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} ${MPI_CXX_LINK_FLAGS}")\n')
+ outfile.write('set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} ${OpenMP_EXE_LINKER_FLAGS}")\n')
+ outfile.write('if(NDEBUG)\n')
+ outfile.write(' add_definitions(-DNDEBUG)\n')
+ outfile.write('endif()\n')
+ outfile.write('if(TIMING_OUTPUT)\n')
+ outfile.write(' add_definitions(-DUSE_TIMINGOUTPUT)\n')
+ outfile.write('endif()\n')
+ outfile.write('include_directories(${TURTLE_INCLUDE_DIRECTORIES} ${TURTLE_INCLUDE_DIR}/TurTLE)\n')
+ outfile.write('link_directories(${TURTLE_LINK_DIRECTORIES} ${TURTLE_LIBRARIES_DIR})\n')
+ outfile.write('find_library(TURTLE_STATIC_LIBRARY TurTLE HINTS ${TURTLE_LIBRARIES_DIR})\n')
+ outfile.write('add_executable({0} {0}.cpp)\n'.format(self.name))
+ outfile.write('target_link_libraries(' + self.name + ' ${TURTLE_STATIC_LIBRARY})\n')
+ outfile.write('target_link_libraries(' + self.name + ' ${TURTLE_LIBS})\n')
+ fname = self.dns_type + '_extra_cmake.txt'
+ if os.path.exists(fname):
+ with open(fname, 'r') as ifile:
+ for text_line in ifile:
+ outfile.write(text_line.replace(self.dns_type, self.name))
+ subprocess.check_call(['cmake', '.'])
+ current_environment = os.environ
+ if not no_debug:
+ current_environment['VERBOSE'] = '1'
+ make_result = subprocess.check_call(['make'], env = current_environment)
+ os.chdir('..')
+ shutil.copy2(os.path.join(build_dir, self.name), os.path.join(self.work_dir, self.name))
+ return make_result
def set_host_info(
self,
host_info = {}):
@@ -234,31 +263,57 @@ class _code(_base):
hours = 0,
minutes = 10,
njobs = 1,
- no_submit = False):
+ no_submit = False,
+ no_debug = True):
self.read_parameters()
- with h5py.File(os.path.join(self.work_dir, self.simname + '.h5'), 'r') as data_file:
- iter0 = data_file['iteration'].value
+ with h5py.File(os.path.join(self.work_dir, self.simname + '.h5'), 'r') as ifile:
+ # if we are restarting, check that we're using the same precision
+ # I consider it an error to run a different precision code at different points,
+ # although technically some might just call it "bad practice".
+ # control the behavior here if you want to
+ if 'code_info' in ifile.keys():
+ if 'exec_name' in ifile['code_info'].keys():
+ if not (str(self.fluid_precision) in str(ifile['code_info/exec_name'][()])):
+ raise AssertionError('Desired precision does not match existing DNS precision.')
+ # check if stop_<simname> exists, remove if it does
+ if os.path.exists(os.path.join(self.work_dir, 'stop_' + self.simname)):
+ warnings.warn("Found stop_<simname> file, I will remove it.")
+ os.remove(os.path.join(self.work_dir, 'stop_' + self.simname))
+ if self.check_current_vorticity_exists:
+ with h5py.File(os.path.join(self.work_dir, self.simname + '.h5'), 'r') as data_file:
+ iter0 = data_file['iteration'][...]
+ checkpoint0 = data_file['checkpoint'][()]
+ cp_fname = os.path.join(self.work_dir, self.simname + '_checkpoint_{0}.h5'.format(checkpoint0))
+ assert os.path.exists(cp_fname)
+ with h5py.File(cp_fname, 'r') as checkpoint_file:
+ assert '{0}'.format(iter0) in checkpoint_file['vorticity/complex'].keys()
+ elif self.check_current_velocity_exists:
+ with h5py.File(os.path.join(self.work_dir, self.simname + '.h5'), 'r') as data_file:
+ iter0 = data_file['iteration'][...]
+ checkpoint0 = data_file['checkpoint'][()]
+ cp_fname = os.path.join(self.work_dir, self.simname + '_checkpoint_{0}.h5'.format(checkpoint0))
+ assert os.path.exists(cp_fname)
+ with h5py.File(cp_fname, 'r') as checkpoint_file:
+ assert '{0}'.format(iter0) in checkpoint_file['velocity/complex'].keys()
+ else:
+ iter0 = 0
if not os.path.isdir(self.work_dir):
os.makedirs(self.work_dir)
- if not os.path.exists(os.path.join(self.work_dir, self.name)):
- need_to_compile = True
- else:
- need_to_compile = (datetime.fromtimestamp(os.path.getctime(os.path.join(self.work_dir, self.name))) <
- bfps.install_info['install_date'])
- if need_to_compile:
- assert(self.compile_code() == 0)
- if self.work_dir != os.path.realpath(os.getcwd()):
- shutil.copy(self.name, self.work_dir)
+ assert (self.compile_code(no_debug = no_debug) == 0)
if 'niter_todo' not in self.parameters.keys():
self.parameters['niter_todo'] = 1
current_dir = os.getcwd()
os.chdir(self.work_dir)
os.chdir(current_dir)
+ if not 'MPI' in self.host_info.keys():
+ self.host_info['MPI'] = 'mpich'
+ if self.host_info['MPI'] == 'openmpi':
+ mpirun_environment_set = 'x'
+ else:
+ mpirun_environment_set = 'env'
command_atoms = ['mpirun',
'-np',
'{0}'.format(nb_processes),
- '-x',
- 'OMP_NUM_THREADS={0}'.format(nb_threads_per_process),
'./' + self.name,
self.simname]
if self.host_info['type'] == 'cluster':
@@ -268,20 +323,22 @@ class _code(_base):
qsub_script_name = 'run_' + suffix + '.sh'
self.write_sge_file(
file_name = os.path.join(self.work_dir, qsub_script_name),
- nprocesses = nb_processes*nb_threads_per_process,
+ nprocesses = nb_processes,
name_of_run = suffix,
- command_atoms = command_atoms[5:],
+ command_atoms = command_atoms[3:],
hours = hours,
minutes = minutes,
- out_file = out_file + '_' + suffix,
- err_file = err_file + '_' + suffix)
+ out_file = out_file + '_' + suffix + '.%j',
+ err_file = err_file + '_' + suffix + '.%j',
+ nb_threads_per_process = nb_threads_per_process)
os.chdir(self.work_dir)
qsub_atoms = ['qsub']
- if len(job_name_list) >= 1:
- qsub_atoms += ['-hold_jid', job_name_list[-1]]
- subprocess.call(qsub_atoms + [qsub_script_name])
- os.chdir(current_dir)
- job_name_list.append(suffix)
+ if not no_submit:
+ if len(job_name_list) >= 1:
+ qsub_atoms += ['-hold_jid', job_name_list[-1]]
+ subprocess.check_call(qsub_atoms + [qsub_script_name])
+ os.chdir(current_dir)
+ job_name_list.append(suffix)
if self.host_info['type'] == 'SLURM':
job_id_list = []
for j in range(njobs):
@@ -290,13 +347,13 @@ class _code(_base):
self.write_slurm_file(
file_name = os.path.join(self.work_dir, qsub_script_name),
name_of_run = suffix,
- command_atoms = command_atoms[5:],
+ command_atoms = command_atoms[3:],
hours = hours,
minutes = minutes,
out_file = out_file + '_' + suffix,
err_file = err_file + '_' + suffix,
nb_mpi_processes = nb_processes,
- nb_threads_per_process = nb_threads_per_process)
+ nb_threads_per_process = nb_threads_per_process)
os.chdir(self.work_dir)
qsub_atoms = ['sbatch']
@@ -313,46 +370,75 @@ class _code(_base):
elif self.host_info['type'] == 'IBMLoadLeveler':
suffix = self.simname + '_{0}'.format(iter0)
job_script_name = 'run_' + suffix + '.sh'
+ energy_policy_tag = (
+ 'TurTLE'
+ + '_np{0}_ntpp{1}'.format(
+ nb_processes, nb_threads_per_process)
+ + '_Nx{0}_Ny{1}_Nz{2}'.format(
+ self.parameters['nx'], self.parameters['ny'], self.parameters['nz']))
+ if 'nparticles' in self.parameters.keys():
+ energy_policy_tag += '_nparticles{0}'.format(self.parameters['nparticles'])
if (njobs == 1):
self.write_IBMLoadLeveler_file_single_job(
file_name = os.path.join(self.work_dir, job_script_name),
name_of_run = suffix,
- command_atoms = command_atoms[5:],
+ command_atoms = command_atoms[3:],
hours = hours,
minutes = minutes,
out_file = out_file + '_' + suffix,
err_file = err_file + '_' + suffix,
nb_mpi_processes = nb_processes,
- nb_threads_per_process = nb_threads_per_process)
+ nb_threads_per_process = nb_threads_per_process,
+ energy_policy_tag = energy_policy_tag)
else:
self.write_IBMLoadLeveler_file_many_job(
file_name = os.path.join(self.work_dir, job_script_name),
name_of_run = suffix,
- command_atoms = command_atoms[5:],
+ command_atoms = command_atoms[3:],
hours = hours,
minutes = minutes,
out_file = out_file + '_' + suffix,
err_file = err_file + '_' + suffix,
njobs = njobs,
nb_mpi_processes = nb_processes,
- nb_threads_per_process = nb_threads_per_process)
+ nb_threads_per_process = nb_threads_per_process,
+ energy_policy_tag = energy_policy_tag)
submit_atoms = ['llsubmit']
if not no_submit:
- subprocess.call(submit_atoms + [os.path.join(self.work_dir, job_script_name)])
+ subprocess.check_call(submit_atoms + [os.path.join(self.work_dir, job_script_name)])
elif self.host_info['type'] == 'pc':
- os.chdir(self.work_dir)
- if os.getenv('LD_LIBRARY_PATH') != None:
- os.environ['LD_LIBRARY_PATH'] += ':{0}'.format(bfps.lib_dir)
- print('added to LD_LIBRARY_PATH the location {0}'.format(bfps.lib_dir))
- for j in range(njobs):
- suffix = self.simname + '_{0}'.format(iter0 + j*self.parameters['niter_todo'])
- print('running code with command\n' + ' '.join(command_atoms))
- subprocess.call(command_atoms,
- stdout = open(out_file + '_' + suffix, 'w'),
- stderr = open(err_file + '_' + suffix, 'w'))
- os.chdir(current_dir)
+ if 'executable_launcher' in self.host_info.keys():
+ command_atoms = [self.host_info['executable_launcher']]
+ if self.host_info['executable_launcher'] == 'srun':
+ command_atoms += ['-p', 'interactive']
+ if 'executable_parameters' in self.host_info.keys():
+ command_atoms += self.host_info['executable_parameters']
+ command_atoms += ['-n',
+ '{0}'.format(nb_processes),
+ './' + self.name,
+ self.simname]
+ if not no_submit:
+ os.chdir(self.work_dir)
+ custom_env = os.environ
+ custom_env['OMP_NUM_THREADS'] = str(nb_threads_per_process)
+ for j in range(njobs):
+ suffix = self.simname + '_{0}'.format(iter0 + j*self.parameters['niter_todo'])
+ print('running code with command\n' + ' '.join(command_atoms))
+ try:
+ subprocess.check_call(
+ command_atoms,
+ stdout = open(os.path.join(self.work_dir, out_file + '_' + suffix), 'w'),
+ stderr = open(os.path.join(self.work_dir, err_file + '_' + suffix), 'w'),
+ env = custom_env)
+ except subprocess.CalledProcessError:
+ print('!!!! TurTLE-generated executable failed to run, output follows:')
+ print(open(out_file + '_' + suffix, 'r').read())
+ print('!!!! TurTLE-generated executable failed to run, error log follows:')
+ print(open(err_file + '_' + suffix, 'r').read())
+ raise SystemExit('TurTLE-generated executable failed to run, see above for details.')
+ os.chdir(current_dir)
return None
def write_IBMLoadLeveler_file_single_job(
self,
@@ -364,8 +450,9 @@ class _code(_base):
minutes = None,
out_file = None,
err_file = None,
- nb_mpi_processes = None,
- nb_threads_per_process = None):
+ nb_mpi_processes = None,
+ nb_threads_per_process = None,
+ energy_policy_tag = 'TurTLE'):
script_file = open(file_name, 'w')
script_file.write('# @ shell=/bin/bash\n')
@@ -380,19 +467,22 @@ class _code(_base):
# If Ibm is used should be : script_file.write('# @ job_type = parallel\n')
script_file.write('# @ job_type = MPICH\n')
+ assert(type(self.host_info['environment']) != type(None))
+ script_file.write('# @ class = {0}\n'.format(self.host_info['environment']))
script_file.write('# @ node_usage = not_shared\n')
script_file.write('# @ notification = complete\n')
- script_file.write('# @ notify_user = $(user)@rzg.mpg.de\n')
+ script_file.write('# @ notify_user = {0}\n'.format(self.host_info['mail_address']))
nb_cpus_per_node = self.host_info['deltanprocs']
- assert(isinstance(nb_cpus_per_node, int) and nb_cpus_per_node >= 1,
- 'nb_cpus_per_node is {}'.format(nb_cpus_per_node))
+ assert isinstance(nb_cpus_per_node, int) and \
+ nb_cpus_per_node >= 1, \
+ 'nb_cpus_per_node is {}'.format(nb_cpus_per_node)
# No more threads than the number of cores
- assert(nb_threads_per_process <= nb_cpus_per_node,
+ assert nb_threads_per_process <= nb_cpus_per_node, \
"Cannot use more threads ({} asked) than the number of cores ({})".format(
- nb_threads_per_process, nb_cpus_per_node))
+ nb_threads_per_process, nb_cpus_per_node)
# Warn if some core will not be ued
if nb_cpus_per_node%nb_threads_per_process != 0:
warnings.warn("The number of threads is smaller than the number of cores (machine will be underused)",
@@ -410,10 +500,11 @@ class _code(_base):
nb_processes_per_node = int(nb_cpus_per_node // nb_threads_per_process)
first_node_tasks = int(nb_mpi_processes - (nb_nodes-1)*nb_processes_per_node)
+ script_file.write('# @ energy_policy_tag = {0}\n'.format(energy_policy_tag))
+ script_file.write('# @ minimize_time_to_solution = yes\n')
script_file.write('# @ resources = ConsumableCpus({})\n'.format(nb_threads_per_process))
script_file.write('# @ network.MPI = sn_all,not_shared,us\n')
script_file.write('# @ wall_clock_limit = {0}:{1:0>2d}:00\n'.format(hours, minutes))
- assert(type(self.host_info['environment']) != type(None))
script_file.write('# @ node = {0}\n'.format(nb_nodes))
script_file.write('# @ tasks_per_node = {0}\n'.format(nb_processes_per_node))
if (first_node_tasks > 0):
@@ -421,13 +512,10 @@ class _code(_base):
script_file.write('# @ queue\n')
- script_file.write('source ~/.config/bfps/bashrc\n')
+ script_file.write('source ~/.config/TurTLE/bashrc\n')
script_file.write('module li\n')
script_file.write('export OMP_NUM_THREADS={}\n'.format(nb_threads_per_process))
- script_file.write('LD_LIBRARY_PATH=' +
- ':'.join([bfps.lib_dir] + bfps.install_info['library_dirs']) +
- ':${LD_LIBRARY_PATH}\n')
script_file.write('echo "Start time is `date`"\n')
script_file.write('export HTMLOUTPUT={}.html\n'.format(command_atoms[-1]))
script_file.write('cd ' + self.work_dir + '\n')
@@ -437,7 +525,7 @@ class _code(_base):
script_file.write('mpiexec.hydra '
+ ' -np {} '.format(nb_mpi_processes)
+ ' -ppn {} '.format(nb_processes_per_node)
- + ' -ordered-output -prepend-rank '
+ #+ ' -ordered-output -prepend-rank '
+ os.path.join(
self.work_dir,
command_atoms[0]) +
@@ -460,8 +548,9 @@ class _code(_base):
out_file = None,
err_file = None,
njobs = 2,
- nb_mpi_processes = None,
- nb_threads_per_process = None):
+ nb_mpi_processes = None,
+ nb_threads_per_process = None,
+ energy_policy_tag = 'TurTLE'):
assert(type(self.host_info['environment']) != type(None))
script_file = open(file_name, 'w')
script_file.write('# @ shell=/bin/bash\n')
@@ -475,16 +564,23 @@ class _code(_base):
script_file.write('# @ output = ' + os.path.join(self.work_dir, out_file) + '\n')
# If Ibm is used should be : script_file.write('# @ job_type = parallel\n')
script_file.write('# @ job_type = MPICH\n')
+ assert(type(self.host_info['environment']) != type(None))
+ script_file.write('# @ class = {0}\n'.format(self.host_info['environment']))
script_file.write('# @ node_usage = not_shared\n')
+
+ script_file.write('# @ notification = error\n')
+ script_file.write('# @ notify_user = {0}\n'.format(self.host_info['mail_address']))
script_file.write('#\n')
nb_cpus_per_node = self.host_info['deltanprocs']
- assert(isinstance(nb_cpus_per_node, int) and nb_cpus_per_node >= 1, 'nb_cpus_per_node is {}'.format(nb_cpus_per_node))
+ assert isinstance(nb_cpus_per_node, int) and \
+ nb_cpus_per_node >= 1, \
+ 'nb_cpus_per_node is {}'.format(nb_cpus_per_node)
# No more threads than the number of cores
- assert(nb_threads_per_process <= nb_cpus_per_node,
+ assert nb_threads_per_process <= nb_cpus_per_node, \
"Cannot use more threads ({} asked) than the number of cores ({})".format(
- nb_threads_per_process, nb_cpus_per_node))
+ nb_threads_per_process, nb_cpus_per_node)
# Warn if some core will not be ued
if nb_cpus_per_node%nb_threads_per_process != 0:
warnings.warn("The number of threads is smaller than the number of cores (machine will be underused)",
@@ -503,24 +599,25 @@ class _code(_base):
first_node_tasks = int(nb_mpi_processes - (nb_nodes-1)*nb_processes_per_node)
for job in range(njobs):
- script_file.write('# @ step_name = {0}.$(stepid)\n'.format(self.simname))
+ script_file.write('# @ step_name = {0}.{1}\n'.format(self.simname, job))
+ if job > 0:
+ script_file.write('# @ dependency = {0}.{1} == 0\n'.format(self.simname, job - 1))
script_file.write('# @ resources = ConsumableCpus({})\n'.format(nb_threads_per_process))
script_file.write('# @ network.MPI = sn_all,not_shared,us\n')
script_file.write('# @ wall_clock_limit = {0}:{1:0>2d}:00\n'.format(hours, minutes))
- assert(type(self.host_info['environment']) != type(None))
+ script_file.write('# @ energy_policy_tag = {0}\n'.format(energy_policy_tag))
+ script_file.write('# @ minimize_time_to_solution = yes\n')
+ assert type(self.host_info['environment']) != type(None)
script_file.write('# @ node = {0}\n'.format(nb_nodes))
script_file.write('# @ tasks_per_node = {0}\n'.format(nb_processes_per_node))
if (first_node_tasks > 0):
script_file.write('# @ first_node_tasks = {0}\n'.format(first_node_tasks))
script_file.write('# @ queue\n')
- script_file.write('source ~/.config/bfps/bashrc\n')
+ script_file.write('source ~/.config/TurTLE/bashrc\n')
script_file.write('module li\n')
script_file.write('export OMP_NUM_THREADS={}\n'.format(nb_threads_per_process))
- script_file.write('LD_LIBRARY_PATH=' +
- ':'.join([bfps.lib_dir] + bfps.install_info['library_dirs']) +
- ':${LD_LIBRARY_PATH}\n')
script_file.write('echo "Start time is `date`"\n')
script_file.write('export HTMLOUTPUT={}.html\n'.format(command_atoms[-1]))
script_file.write('cd ' + self.work_dir + '\n')
@@ -552,7 +649,8 @@ class _code(_base):
hours = None,
minutes = None,
out_file = None,
- err_file = None):
+ err_file = None,
+ nb_threads_per_process = 1):
script_file = open(file_name, 'w')
script_file.write('#!/bin/bash\n')
# export all environment variables
@@ -563,23 +661,22 @@ class _code(_base):
script_file.write('#$ -cwd\n')
# error file
if not type(err_file) == type(None):
- script_file.write('#$ -e ' + err_file + '\n')
+ script_file.write('#$ -e ' + os.path.join(self.work_dir, err_file) + '\n')
# output file
if not type(out_file) == type(None):
- script_file.write('#$ -o ' + out_file + '\n')
+ script_file.write('#$ -o ' + os.path.join(self.work_dir, out_file) + '\n')
if not type(self.host_info['environment']) == type(None):
- envprocs = self.host_info['deltanprocs'] * int(math.ceil((nprocesses *1.0/ self.host_info['deltanprocs'])))
+ envprocs = nb_threads_per_process * nprocesses
script_file.write('#$ -pe {0} {1}\n'.format(
self.host_info['environment'],
envprocs))
script_file.write('echo "got $NSLOTS slots."\n')
script_file.write('echo "Start time is `date`"\n')
- script_file.write('mpiexec -machinefile $TMPDIR/machines ' +
- '-genv LD_LIBRARY_PATH ' +
- '"' +
- ':'.join([bfps.lib_dir] + bfps.install_info['library_dirs']) +
- '" ' +
- '-n {0} {1}\n'.format(nprocesses, ' '.join(command_atoms)))
+ script_file.write('mpiexec \\\n' +
+ '\t-machinefile $TMPDIR/machines \\\n' +
+ '\t-genv OMP_NUM_THREADS={0} \\\n'.format(nb_threads_per_process) +
+ '\t-genv OMP_PLACES=cores \\\n' +
+ '\t-n {0} \\\n\t{1}\n'.format(nprocesses, ' '.join(command_atoms)))
script_file.write('echo "End time is `date`"\n')
script_file.write('exit 0\n')
script_file.close()
@@ -593,8 +690,8 @@ class _code(_base):
minutes = None,
out_file = None,
err_file = None,
- nb_mpi_processes = None,
- nb_threads_per_process = None):
+ nb_mpi_processes = None,
+ nb_threads_per_process = None):
script_file = open(file_name, 'w')
script_file.write('#!/bin/bash -l\n')
# job name
@@ -603,21 +700,28 @@ class _code(_base):
script_file.write('#SBATCH -D ./\n')
# error file
if not type(err_file) == type(None):
- script_file.write('#SBATCH -e ' + err_file + '\n')
+ script_file.write('#SBATCH -e ' + os.path.join(self.work_dir, err_file) + '\n')
# output file
if not type(out_file) == type(None):
- script_file.write('#SBATCH -o ' + out_file + '\n')
- script_file.write('#SBATCH --partition={0}\n'.format(
- self.host_info['environment']))
+ script_file.write('#SBATCH -o ' + os.path.join(self.work_dir, out_file) + '\n')
+
+ # set up environment
+ if self.host_info['explicit_slurm_environment']:
+ script_file.write('#SBATCH --partition={0}\n'.format(
+ self.host_info['environment']))
+ if 'account' in self.host_info.keys():
+ script_file.write('#SBATCH --account={0}\n'.format(
+ self.host_info['account']))
nb_cpus_per_node = self.host_info['deltanprocs']
- assert(isinstance(nb_cpus_per_node, int) and nb_cpus_per_node >= 1,
- 'nb_cpus_per_node is {}'.format(nb_cpus_per_node))
+ assert isinstance(nb_cpus_per_node, int) \
+ and nb_cpus_per_node >= 1, \
+ 'nb_cpus_per_node is {}'.format(nb_cpus_per_node)
# No more threads than the number of cores
- assert(nb_threads_per_process <= nb_cpus_per_node,
+ assert nb_threads_per_process <= nb_cpus_per_node, \
"Cannot use more threads ({} asked) than the number of cores ({})".format(
- nb_threads_per_process, nb_cpus_per_node))
+ nb_threads_per_process, nb_cpus_per_node)
# Warn if some core will not be ued
if nb_cpus_per_node%nb_threads_per_process != 0:
warnings.warn(
@@ -641,18 +745,60 @@ class _code(_base):
script_file.write('#SBATCH --mail-type=none\n')
script_file.write('#SBATCH --time={0}:{1:0>2d}:00\n'.format(hours, minutes))
- script_file.write('source ~/.config/bfps/bashrc\n')
+ if 'extra_slurm_lines' in self.host_info.keys():
+ for line in self.host_info['extra_slurm_lines']:
+ script_file.write(line + '\n')
+ ## following cleans up environment for job.
+ ## put these in the 'extra_slurm_lines' list if you need them
+ ## make sure that "~/.config/TurTLE/bashrc" exists and builds desired job environment
+ #script_file.write('#SBATCH --export=NONE\n')
+ #script_file.write('#SBATCH --get-user-env\n')
+ #script_file.write('export OMP_PLACES=cores\n') # or threads, as appropriate
+ # also look up OMP_PROC_BIND and SLURM_HINT
+ #script_file.write('source ~/.config/TurTLE/bashrc\n')
if nb_threads_per_process > 1:
- script_file.write('export OMP_NUM_THREADS={0}\n'.format(nb_threads_per_process))
- script_file.write('export OMP_PLACES=cores\n')
+ script_file.write('export OMP_NUM_THREADS=${SLURM_CPUS_PER_TASK}\n')
+
+ # explicit binding options that can be used further on if needed
+ if 'use_TurTLE_core_distribution' not in host_info.keys():
+ host_info['use_TurTLE_core_distribution'] = False
+ if host_info['use_TurTLE_core_distribution']:
+ core_masks = TurTLE.tools.distribute_cores_evenly(
+ nprocesses = nb_processes_per_node,
+ nthreads_per_process = nb_threads_per_process,
+ total_number_of_cores = nb_cpus_per_node)
+ script_file.write('export SLURM_CPU_BIND_OPTION="--cpu-bind=verbose,mask_cpu:' +
+ ','.join(['0x{0:x}'.format(mm) for mm in core_masks]) + '"\n')
- script_file.write('LD_LIBRARY_PATH=' +
- ':'.join([bfps.lib_dir] + bfps.install_info['library_dirs']) +
- ':${LD_LIBRARY_PATH}\n')
script_file.write('echo "Start time is `date`"\n')
script_file.write('cd ' + self.work_dir + '\n')
script_file.write('export HTMLOUTPUT={}.html\n'.format(command_atoms[-1]))
- script_file.write('srun {0}\n'.format(' '.join(command_atoms)))
+ if not 'use_vtune' in host_info.keys():
+ host_info['use_vtune'] = False
+ if not 'use_aps' in host_info.keys():
+ host_info['use_aps'] = False
+ if host_info['use_vtune']:
+ script_file.write('module load vtune\n')
+ if host_info['use_aps']:
+ if 'aps_setup' not in host_info.keys():
+ host_info['aps_setup'] = 'module load vtune'
+ script_file.write(host_info['aps_setup'] + '\n')
+ if 'executable_launcher' in self.host_info.keys():
+ executable_launcher = self.host_info['executable_launcher']
+ else:
+ executable_launcher = 'srun'
+ if host_info['use_TurTLE_core_distribution']:
+ executable_launcher += ' ${SLURM_CPU_BIND_OPTION}'
+ if host_info['use_vtune']:
+ if 'vtune_executable' not in host_info.keys():
+ host_info['vtune_executable'] = 'vtune'
+ executable_launcher += ' ' + host_info['vtune_executable'] + ' -collect hpc-performance -trace-mpi -quiet -result-dir=vtune_${SLURM_JOB_NAME}'
+ if host_info['use_aps']:
+ if 'aps_executable' not in host_info.keys():
+ host_info['aps_executable'] = 'aps'
+
+ executable_launcher += ' ' + host_info['aps_executable'] + ' --result-dir=aps_${SLURM_JOB_NAME} --collection-mode=all'
+ script_file.write(executable_launcher + ' {0}\n'.format(' '.join(command_atoms)))
script_file.write('echo "End time is `date`"\n')
script_file.write('exit 0\n')
script_file.close()
@@ -661,7 +807,7 @@ class _code(_base):
self,
args = [],
**kwargs):
- parser = argparse.ArgumentParser('bfps ' + type(self).__name__)
+ parser = argparse.ArgumentParser('turtle ' + type(self).__name__)
self.add_parser_arguments(parser)
opt = parser.parse_args(args)
@@ -672,8 +818,12 @@ class _code(_base):
opt.nb_processes = opt.ncpu
opt.nb_threads_per_process = 1
- self.set_host_info(bfps.host_info)
+ self.set_host_info(TurTLE.host_info)
if type(opt.environment) != type(None):
self.host_info['environment'] = opt.environment
+ # we cannot use both vtune and aps for profiling at the same time
+ assert(not (opt.use_vtune and opt.use_aps))
+ self.host_info['use_vtune'] = opt.use_vtune
+ self.host_info['use_aps'] = opt.use_aps
return opt
diff --git a/TurTLE/test/B32p1e4_checkpoint_0.h5 b/TurTLE/test/B32p1e4_checkpoint_0.h5
new file mode 100644
index 0000000000000000000000000000000000000000..90e89e13e101973660cf16f51838b8b67ef1fc6e
Binary files /dev/null and b/TurTLE/test/B32p1e4_checkpoint_0.h5 differ
diff --git a/bfps/test/__init__.py b/TurTLE/test/__init__.py
similarity index 100%
rename from bfps/test/__init__.py
rename to TurTLE/test/__init__.py
diff --git a/TurTLE/test/collisions/NSVE_Stokes_growing_subset.cpp b/TurTLE/test/collisions/NSVE_Stokes_growing_subset.cpp
new file mode 100644
index 0000000000000000000000000000000000000000..485078bc36a839cdb42668a5574ea1b3871e27c8
--- /dev/null
+++ b/TurTLE/test/collisions/NSVE_Stokes_growing_subset.cpp
@@ -0,0 +1,211 @@
+/**********************************************************************
+* *
+* Copyright 2022 Max Planck Institute *
+* for Dynamics and Self-Organization *
+* *
+* This file is part of TurTLE. *
+* *
+* TurTLE is free software: you can redistribute it and/or modify *
+* it under the terms of the GNU General Public License as published *
+* by the Free Software Foundation, either version 3 of the License, *
+* or (at your option) any later version. *
+* *
+* TurTLE is distributed in the hope that it will be useful, *
+* but WITHOUT ANY WARRANTY; without even the implied warranty of *
+* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the *
+* GNU General Public License for more details. *
+* *
+* You should have received a copy of the GNU General Public License *
+* along with TurTLE. If not, see <http://www.gnu.org/licenses/> *
+* *
+* Contact: Cristian.Lalescu@mpcdf.mpg.de *
+* *
+**********************************************************************/
+
+
+
+#include "scope_timer.hpp"
+#include "particles/particles_output_hdf5.hpp"
+#include <string>
+#include <cmath>
+
+template <typename rnumber>
+int NSVE_Stokes_growing_subset<rnumber>::initialize(void)
+{
+ TIMEZONE("NSVE_Stokes_growing_subset::intialize");
+ this->NSVE<rnumber>::initialize();
+ this->fti = new field_tinterpolator<rnumber, FFTW, THREE, NONE>();
+ this->srhs = new stokes_collisions_with_background_rhs<rnumber, FFTW, NONE>();
+ this->srhs->setDragCoefficient(0.1);
+ this->srhs->setStateSize(7);
+ this->srhs->getCollisionCounter().set_density_ratio(0.001);
+ this->srhs->getCollisionCounter().set_nu(this->fs->nu);
+ this->srhs->getCollisionCounter().set_dt(this->dt);
+ this->srhs->getCollisionCounter().set_nb_growing_particles(this->nb_growing_particles);
+ // We're not using Adams-Bashforth in this code.
+ // Euler integration for now, but this assert is there to ensure
+ // user knows what's happening.
+ assert(tracers0_integration_steps == 0);
+ // neighbours and smoothness are second and third template parameters of particle_set
+ assert(tracers0_neighbours == 3);
+ assert(tracers0_smoothness == 2);
+ this->pset = new particle_set<7, 3, 2>(
+ this->fs->cvelocity->rlayout,
+ this->dkx,
+ this->dky,
+ this->dkz,
+ this->tracers0_cutoff);
+
+ this->fti->set_field(this->fs->cvelocity);
+ this->srhs->setVelocity(this->fti);
+
+ particles_input_hdf5<long long int, double, 7, 7> particle_reader(
+ this->comm,
+ this->fs->get_current_fname(),
+ std::string("/tracers0/state/") + std::to_string(this->fs->iteration), // dataset name for initial input
+ std::string("/tracers0/rhs/") + std::to_string(this->fs->iteration), // dataset name for initial input
+ this->pset->getSpatialLowLimitZ(),
+ this->pset->getSpatialUpLimitZ());
+ this->pset->init(particle_reader);
+ DEBUG_MSG("local number of particles after input %d\n",this->pset->getLocalNumberOfParticles());
+
+ this->psolver = new particle_solver(*(this->pset), 0);
+
+ this->particles_output_writer_mpi = new particles_output_hdf5<
+ long long int, double, 7>(
+ MPI_COMM_WORLD,
+ "tracers0",
+ nparticles,
+ tracers0_integration_steps);
+ this->particles_sample_writer_mpi = new particles_output_sampling_hdf5<
+ long long int, double, double, 3>(
+ MPI_COMM_WORLD,
+ this->pset->getTotalNumberOfParticles(),
+ (this->simname + "_particles.h5"),
+ "tracers0",
+ "position/0");
+ return EXIT_SUCCESS;
+}
+
+template <typename rnumber>
+int NSVE_Stokes_growing_subset<rnumber>::step(void)
+{
+ TIMEZONE("NSVE_Stokes_growing_subset::step");
+ DEBUG_MSG("step() is called\n");
+ this->fs->compute_velocity(this->fs->cvorticity);
+ this->fs->cvelocity->ift();
+ DEBUG_MSG("real velocity field is computed\n");
+ this->psolver->Euler(this->dt, *(this->srhs));
+ DEBUG_MSG("particle timestep is performed\n");
+ DEBUG_MSG("local_number of collision pairs: %d\n", (this->srhs->getCollisionCounter()).get_local_collision_nb());
+ this->NSVE<rnumber>::step();
+ DEBUG_MSG("field step() is called\n");
+ this->psolver->setIteration(this->fs->iteration);
+ DEBUG_MSG("step is cpompleted\n");
+ return EXIT_SUCCESS;
+}
+
+template <typename rnumber>
+int NSVE_Stokes_growing_subset<rnumber>::write_checkpoint(void)
+{
+ TIMEZONE("NSVE_Stokes_growing_subset::write_checkpoint");
+ this->NSVE<rnumber>::write_checkpoint();
+ this->psolver->setIteration(this->fs->iteration);
+ this->particles_output_writer_mpi->open_file(this->fs->get_current_fname());
+ this->psolver->template writeCheckpoint<7>(this->particles_output_writer_mpi);
+ this->particles_output_writer_mpi->close_file();
+ return EXIT_SUCCESS;
+}
+
+template <typename rnumber>
+int NSVE_Stokes_growing_subset<rnumber>::finalize(void)
+{
+ TIMEZONE("NSVE_Stokes_growing_subset::finalize");
+ delete this->particles_output_writer_mpi;
+ delete this->particles_sample_writer_mpi;
+ delete this->psolver;
+ delete this->pset;
+ delete this->srhs;
+ delete this->fti;
+ this->NSVE<rnumber>::finalize();
+ return EXIT_SUCCESS;
+}
+
+/** \brief Compute fluid stats and sample fields at particle locations.
+ */
+
+template <typename rnumber>
+int NSVE_Stokes_growing_subset<rnumber>::do_stats()
+{
+ TIMEZONE("NSVE_Stokes_growing_subset::do_stats");
+ /// fluid stats go here
+ this->NSVE<rnumber>::do_stats();
+ this->srhs->getCollisionCounter().MPI_merge(
+ this->comm,
+ this->myrank,
+ this->nprocs);
+ if (this->myrank == 0)
+ {
+ // this->stat_file is only defined for rank 0,
+ // so only rank 0 can write the pairs.
+ // this is also a reason for calling MPI_merge beforehand.
+ hdf5_tools::write_particle_ID_pairs_with_single_rank<long long>(
+ this->srhs->getCollisionCounter().get_collision_pairs(
+ this->comm,
+ this->myrank,
+ this->nprocs),
+ this->stat_file,
+ std::string("/statistics/collisions/pair_list_") + std::to_string(psolver->getIteration()).c_str());
+ }
+
+ /// either one of two conditions suffices to compute statistics:
+ /// 1) current iteration is a multiple of niter_part
+ /// 2) we are within niter_part_fine_duration/2 of a multiple of niter_part_fine_period
+ if (!(this->iteration % this->niter_part == 0))
+ return EXIT_SUCCESS;
+ // sample position
+ this->pset->writeStateTriplet(
+ 0,
+ this->particles_sample_writer_mpi,
+ "tracers0",
+ "position",
+ psolver->getIteration());
+ if (!(this->iteration % this->niter_stat == 0))
+ {
+ // we need to compute velocity field manually, because it didn't happen in NSVE::do_stats()
+ this->fs->compute_velocity(this->fs->cvorticity);
+ *this->tmp_vec_field = this->fs->cvelocity->get_cdata();
+ this->tmp_vec_field->ift();
+ }
+ this->pset->writeSample(
+ this->tmp_vec_field,
+ this->particles_sample_writer_mpi,
+ "tracers0",
+ "velocity",
+ psolver->getIteration());
+ return EXIT_SUCCESS;
+}
+
+template <typename rnumber>
+int NSVE_Stokes_growing_subset<rnumber>::read_parameters(void)
+{
+ TIMEZONE("NSVE_Stokes_growing_subset::read_parameters");
+ this->NSVE<rnumber>::read_parameters();
+ hid_t parameter_file = H5Fopen((this->simname + ".h5").c_str(), H5F_ACC_RDONLY, H5P_DEFAULT);
+ this->niter_part = hdf5_tools::read_value<int>(parameter_file, "parameters/niter_part");
+ this->niter_part_fine_period = hdf5_tools::read_value<int>(parameter_file, "parameters/niter_part_fine_period");
+ this->niter_part_fine_duration = hdf5_tools::read_value<int>(parameter_file, "parameters/niter_part_fine_duration");
+ this->nparticles = hdf5_tools::read_value<int>(parameter_file, "parameters/nparticles");
+ this->tracers0_integration_steps = hdf5_tools::read_value<int>(parameter_file, "parameters/tracers0_integration_steps");
+ this->tracers0_neighbours = hdf5_tools::read_value<int>(parameter_file, "parameters/tracers0_neighbours");
+ this->tracers0_cutoff = hdf5_tools::read_value<double>(parameter_file, "parameters/tracers0_cutoff");
+ this->tracers0_smoothness = hdf5_tools::read_value<int>(parameter_file, "parameters/tracers0_smoothness");
+ this->nb_growing_particles = hdf5_tools::read_value<int>(parameter_file, "parameters/nb_growing_particles");
+ H5Fclose(parameter_file);
+ MPI_Barrier(this->comm);
+ return EXIT_SUCCESS;
+}
+
+template class NSVE_Stokes_growing_subset<float>;
+template class NSVE_Stokes_growing_subset<double>;
+
diff --git a/TurTLE/test/collisions/NSVE_Stokes_growing_subset.hpp b/TurTLE/test/collisions/NSVE_Stokes_growing_subset.hpp
new file mode 100644
index 0000000000000000000000000000000000000000..cd98d358f5340ddd109a2aa8e4d0778c397b8e90
--- /dev/null
+++ b/TurTLE/test/collisions/NSVE_Stokes_growing_subset.hpp
@@ -0,0 +1,103 @@
+/**********************************************************************
+* *
+* Copyright 2022 Max Planck Institute *
+* for Dynamics and Self-Organization *
+* *
+* This file is part of TurTLE. *
+* *
+* TurTLE is free software: you can redistribute it and/or modify *
+* it under the terms of the GNU General Public License as published *
+* by the Free Software Foundation, either version 3 of the License, *
+* or (at your option) any later version. *
+* *
+* TurTLE is distributed in the hope that it will be useful, *
+* but WITHOUT ANY WARRANTY; without even the implied warranty of *
+* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the *
+* GNU General Public License for more details. *
+* *
+* You should have received a copy of the GNU General Public License *
+* along with TurTLE. If not, see <http://www.gnu.org/licenses/> *
+* *
+* Contact: Cristian.Lalescu@mpcdf.mpg.de *
+* *
+**********************************************************************/
+
+
+
+#ifndef NSVE_STOKES_GROWING_SUBSET_HPP
+#define NSVE_STOKES_GROWING_SUBSET_HPP
+
+
+
+#include "base.hpp"
+#include "vorticity_equation.hpp"
+#include "full_code/NSVE.hpp"
+#include "particles/particles_system_builder.hpp"
+#include "particles/particles_output_hdf5.hpp"
+#include "particles/particles_sampling.hpp"
+#include "particles/interpolation/field_tinterpolator.hpp"
+#include "particles/interpolation/particle_set.hpp"
+#include "particles/particle_solver.hpp"
+//#include "stokes_collision_counter_rhs.hpp"
+
+#include <cstdlib>
+
+/** \brief Navier-Stokes solver that includes simple Lagrangian tracers.
+ *
+ * Child of Navier Stokes vorticity equation solver, this class calls all the
+ * methods from `NSVE`, and in addition integrates simple Lagrangian tracers
+ * in the resulting velocity field.
+ */
+
+template <typename rnumber>
+class NSVE_Stokes_growing_subset : public NSVE<rnumber>
+{
+ public:
+
+ /* parameters that are read in read_parameters */
+ int niter_part;
+ int niter_part_fine_period;
+ int niter_part_fine_duration;
+ int nparticles;
+ int tracers0_integration_steps;
+ int tracers0_neighbours;
+ int tracers0_smoothness;
+ double tracers0_cutoff;
+
+ /* specific parameters */
+ int sample_pressure;
+ int sample_pressure_gradient;
+ int sample_pressure_Hessian;
+ int sample_velocity_gradient;
+ long long int nb_growing_particles;
+
+
+ /* other stuff */
+ field_tinterpolator<rnumber, FFTW, THREE, NONE> *fti;
+ stokes_collisions_with_background_rhs<rnumber, FFTW, NONE> *srhs;
+ particle_set<7, 3, 2> *pset;
+ particle_solver *psolver;
+
+
+ particles_output_hdf5<long long int, double, 7> *particles_output_writer_mpi;
+ particles_output_sampling_hdf5<long long int, double, double, 3> *particles_sample_writer_mpi;
+
+ NSVE_Stokes_growing_subset(
+ const MPI_Comm COMMUNICATOR,
+ const std::string &simulation_name):
+ NSVE<rnumber>(
+ COMMUNICATOR,
+ simulation_name){}
+ ~NSVE_Stokes_growing_subset(){}
+
+ int initialize(void);
+ int step(void);
+ int finalize(void);
+
+ int read_parameters(void);
+ int write_checkpoint(void);
+ int do_stats(void);
+};
+
+#endif//NSVE_STOKES_GROWING_SUBSET_HPP
+
diff --git a/TurTLE/test/collisions/p2p_stokes_collisions_growing_subset.hpp b/TurTLE/test/collisions/p2p_stokes_collisions_growing_subset.hpp
new file mode 100644
index 0000000000000000000000000000000000000000..ba82a08ef82378688f2461972e81017a788212cb
--- /dev/null
+++ b/TurTLE/test/collisions/p2p_stokes_collisions_growing_subset.hpp
@@ -0,0 +1,228 @@
+/******************************************************************************
+* *
+* Copyright 2022 Max Planck Institute for Dynamics and Self-Organization *
+* *
+* This file is part of TurTLE. *
+* *
+* TurTLE is free software: you can redistribute it and/or modify *
+* it under the terms of the GNU General Public License as published *
+* by the Free Software Foundation, either version 3 of the License, *
+* or (at your option) any later version. *
+* *
+* TurTLE is distributed in the hope that it will be useful, *
+* but WITHOUT ANY WARRANTY; without even the implied warranty of *
+* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the *
+* GNU General Public License for more details. *
+* *
+* You should have received a copy of the GNU General Public License *
+* along with TurTLE. If not, see <http://www.gnu.org/licenses/> *
+* *
+* Contact: Cristian.Lalescu@mpcdf.mpg.de *
+* *
+******************************************************************************/
+#ifndef P2P_STOKES_COLLISIONS_GROWING_SUBSET_HPP
+#define P2P_STOKES_COLLISIONS_GROWING_SUBSET_HPP
+
+#include "particles/p2p/p2p_ghost_collisions.hpp"
+
+#include <cstring>
+#include <cmath> /* for sqrt, abs */
+
+#include <set>
+#include <utility>
+#include <vector>
+
+// This class computes ghost collisions for Stokes particles where the 7th degree of freedom is the drag coeffcient
+template <class real_number, class partsize_t>
+class p2p_stokes_collisions_growing_subset: public p2p_ghost_collisions<real_number, partsize_t>
+{
+ private:
+ // Following doubles are needed for the collision computation
+ double dt;
+ double dt_inverse;
+ double nu;
+ double density_ratio;
+ int order;
+ const double pi2 = atan(double(1))*double(8);
+ const double pi = atan(double(1))*double(4);
+ long long int nb_growing_particles;
+public:
+ p2p_stokes_collisions_growing_subset() : dt(0.01), nu(0.1), density_ratio(0.001), order(1)
+ {
+ TIMEZONE("p2p_stokes_collisions_growing_subset::p2p_stokes_collisions_growing_subset");
+ dt_inverse = 1/dt;
+ this->set_sphere();
+ DEBUG_MSG(" nu: %f\n", nu);
+ }
+
+ // Copy constructor use a counter set to zero
+ p2p_stokes_collisions_growing_subset(const p2p_stokes_collisions_growing_subset& p2p_sc)
+ {
+ TIMEZONE("p2p_stokes_collisions_growing_subset::p2p_stokes_collisions copy constructor");
+ DEBUG_MSG("copy constructor is called\n");
+ dt = p2p_sc.dt;
+ dt_inverse = p2p_sc.dt_inverse;
+ nu = p2p_sc.nu;
+ density_ratio = p2p_sc.density_ratio;
+ order = p2p_sc.order;
+ }
+
+ template <int size_particle_positions, int size_particle_rhs>
+ void compute_interaction(const partsize_t idx_part1,
+ real_number pos_part1[],
+ real_number rhs_part1[],
+ const partsize_t idx_part2,
+ real_number pos_part2[],
+ real_number rhs_part2[],
+ const real_number rr,
+ const real_number /*cutoff*/,
+ const real_number x,
+ const real_number y,
+ const real_number z){
+ static_assert(size_particle_positions == 7, "This kernel works only with 7 values for one particle's position+orientation");
+ static_assert(size_particle_rhs == 7, "This kernel works only with 7 values per particle's rhs");
+ if( pos_part1[6]<2 or pos_part2[6]<2){
+ DEBUG_MSG(" drag1: %f\n", pos_part1[6]);
+ DEBUG_MSG(" drag2: %f\n", pos_part2[6]);
+ }
+ double t_min, r_min2;
+
+ const double v_x = pos_part2[3] - pos_part1[3];
+ const double v_y = pos_part2[4] - pos_part1[4];
+ const double v_z = pos_part2[5] - pos_part1[5];
+
+ const double rv = x*v_x + y*v_y + z*v_z;
+ const double vv = v_x*v_x + v_y*v_y + v_z*v_z;
+ const double a1 = sqrt((double(9)/double(2))*density_ratio*nu/pos_part1[6]);
+ const double a2 = sqrt((double(9)/double(2))*density_ratio*nu/pos_part2[6]);
+ const double r_col = a1 + a2;
+ if(rr>r_col*r_col)
+ {
+ if(vv!=0)
+ {
+ t_min = rv/vv;
+ }
+ else
+ {
+ t_min = 0;
+ }
+ assert(pos_part1[6] > 0);
+ if(t_min <= 0)
+ {
+ r_min2 = rr;
+ DEBUG_MSG("r_min2 test1 %f", r_min2);
+ }
+ else if(t_min > dt)
+ {
+ const double xtmp2 = x - v_x*dt;
+ const double ytmp2 = y - v_y*dt;
+ const double ztmp2 = z - v_z*dt;
+ r_min2 = xtmp2*xtmp2 + ytmp2*ytmp2 + ztmp2*ztmp2;
+ DEBUG_MSG("r_min2 test2 %f", r_min2);
+
+ }
+ else
+ {
+ r_min2 = rr - rv*rv/vv;
+ DEBUG_MSG("r_min2 test3 %f", r_min2);
+ }
+ if(r_min2 < r_col*r_col)
+ {
+ DEBUG_MSG("nu: %f collision radius: %f drag1: %f\n",nu,r_col,pos_part1[6]);
+ DEBUG_MSG("Collision happens, %d %d\n", idx_part1, idx_part2);
+ this->add_colliding_pair(idx_part1, idx_part2);
+ DEBUG_MSG("local_number of collision pairs: %d\n", this->collision_pairs_local.size());
+ assert(idx_part1!=idx_part2);
+ const bool condition1 = (idx_part1 < this->nb_growing_particles and idx_part2 >= this->nb_growing_particles);
+ const bool condition2 = (idx_part1 >= this->nb_growing_particles and idx_part2 < this->nb_growing_particles);
+ if (condition1 or condition2)
+ {
+ const double tmp_m1 = pow(pos_part1[6], -double(3)/double(2));
+ const double tmp_m2 = pow(pos_part2[6], -double(3)/double(2));
+ const double tmp = tmp_m1 + tmp_m2;
+ const double new_size = 1./std::cbrt(tmp*tmp);
+ if (condition1)
+ {
+ DEBUG_MSG("rhs started to be adapted\n");
+ rhs_part1[6] = dt_inverse*(new_size - pos_part1[6]);
+ DEBUG_MSG("drag_coeff rhs: %f\n", rhs_part1[6]);
+ rhs_part1[3] = dt_inverse*((tmp_m1*pos_part1[3] + tmp_m2*pos_part2[3])/tmp - pos_part1[3]);
+ DEBUG_MSG(" rhs 3: %f\n", rhs_part1[3]);
+
+ rhs_part1[4] = dt_inverse*((tmp_m1*pos_part1[4] + tmp_m2*pos_part2[4])/tmp - pos_part1[4]);
+ rhs_part1[5] = dt_inverse*((tmp_m1*pos_part1[5] + tmp_m2*pos_part2[5])/tmp - pos_part1[5]);
+ DEBUG_MSG("rhs should be adapdted\n");
+ }
+ else
+ {
+ DEBUG_MSG("rhs started to be adapted\n");
+ rhs_part2[6] = dt_inverse*(new_size - pos_part2[6]);
+ DEBUG_MSG("drag_coeff rhs: %f\n", rhs_part2[6]);
+
+ rhs_part2[3] = dt_inverse*((tmp_m1*pos_part1[3] + tmp_m2*pos_part2[3])/tmp - pos_part2[3]);
+ rhs_part2[4] = dt_inverse*((tmp_m1*pos_part1[4] + tmp_m2*pos_part2[4])/tmp - pos_part2[4]);
+ rhs_part2[5] = dt_inverse*((tmp_m1*pos_part1[5] + tmp_m2*pos_part2[5])/tmp - pos_part2[5]);
+ DEBUG_MSG("rhs should be adapdted\n");
+ }
+ DEBUG_MSG("growth should have happend\n");
+ }
+ }
+ }
+
+ }
+
+ void set_density_ratio(const double DENSITY_RATIO)
+ {
+ this->density_ratio = DENSITY_RATIO;
+ }
+
+ double get_density_ratio()
+ {
+ return this->density_ratio;
+ }
+
+ void set_nu(const double NU)
+ {
+ this->nu = NU;
+ }
+
+ void set_order(const int ORDER)
+ {
+ this->order = ORDER;
+ }
+
+ double get_nu()
+ {
+ return this->nu;
+ }
+
+ void set_dt(const double DT)
+ {
+ this->dt = DT;
+ this->dt_inverse = 1/DT;
+ }
+
+ void set_nb_growing_particles(const long long int NB_GROWING_PARTICLES)
+ {
+ this->nb_growing_particles = NB_GROWING_PARTICLES;
+ }
+
+ double get_dt()
+ {
+ return this->dt;
+ }
+
+ int get_local_collision_nb()
+ {
+ return this->collision_pairs_local.size();
+ }
+
+ std::set<std::pair <partsize_t, partsize_t>> getCollisionPairsLocal() const
+ {
+ return this->collision_pairs_local;
+ }
+};
+
+
+#endif // P2P_STOKES_COLLISIONS_GROWING_SUBSET_HPP
+
diff --git a/TurTLE/test/collisions/stokes_collisions_with_background_rhs.cpp b/TurTLE/test/collisions/stokes_collisions_with_background_rhs.cpp
new file mode 100644
index 0000000000000000000000000000000000000000..7b6d1e378e7b10615147cf5e2db63961df20b33c
--- /dev/null
+++ b/TurTLE/test/collisions/stokes_collisions_with_background_rhs.cpp
@@ -0,0 +1,30 @@
+/******************************************************************************
+* *
+* Copyright 2022 Max Planck Institute for Dynamics and Self-Organization *
+* *
+* This file is part of TurTLE. *
+* *
+* TurTLE is free software: you can redistribute it and/or modify *
+* it under the terms of the GNU General Public License as published *
+* by the Free Software Foundation, either version 3 of the License, *
+* or (at your option) any later version. *
+* *
+* TurTLE is distributed in the hope that it will be useful, *
+* but WITHOUT ANY WARRANTY; without even the implied warranty of *
+* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the *
+* GNU General Public License for more details. *
+* *
+* You should have received a copy of the GNU General Public License *
+* along with TurTLE. If not, see <http://www.gnu.org/licenses/> *
+* *
+* Contact: Cristian.Lalescu@mpcdf.mpg.de *
+* *
+******************************************************************************/
+
+
+
+//#include "stokes_collisions_with_background_rhs.hpp"
+
+
+template class stokes_collisions_with_background_rhs<float, FFTW, NONE>;
+template class stokes_collisions_with_background_rhs<double, FFTW, NONE>;
diff --git a/TurTLE/test/collisions/stokes_collisions_with_background_rhs.hpp b/TurTLE/test/collisions/stokes_collisions_with_background_rhs.hpp
new file mode 100644
index 0000000000000000000000000000000000000000..efda9bfef72516515e6c8f98cff8d4b2a48da797
--- /dev/null
+++ b/TurTLE/test/collisions/stokes_collisions_with_background_rhs.hpp
@@ -0,0 +1,133 @@
+/******************************************************************************
+* *
+* Copyright 2022 Max Planck Institute for Dynamics and Self-Organization *
+* *
+* This file is part of TurTLE. *
+* *
+* TurTLE is free software: you can redistribute it and/or modify *
+* it under the terms of the GNU General Public License as published *
+* by the Free Software Foundation, either version 3 of the License, *
+* or (at your option) any later version. *
+* *
+* TurTLE is distributed in the hope that it will be useful, *
+* but WITHOUT ANY WARRANTY; without even the implied warranty of *
+* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the *
+* GNU General Public License for more details. *
+* *
+* You should have received a copy of the GNU General Public License *
+* along with TurTLE. If not, see <http://www.gnu.org/licenses/> *
+* *
+* Contact: Cristian.Lalescu@mpcdf.mpg.de *
+* *
+******************************************************************************/
+
+
+
+#ifndef STOKES_COLLISIONS_WITH_BACKGROUND_RHS_HPP
+#define STOKES_COLLISIONS_WITH_BACKGROUND_RHS_HPP
+
+#include "particles/interpolation/field_tinterpolator.hpp"
+#include "particles/abstract_particle_rhs.hpp"
+//#include "stokes_rhs.hpp"
+//#include "p2p_stokes_collisions.hpp"
+//#include "stokes_rhs.hpp"
+
+template <typename rnumber,
+ field_backend be,
+ temporal_interpolation_type tt>
+class stokes_collisions_with_background_rhs: public stokes_rhs<rnumber, be, tt>
+{
+ protected:
+ p2p_stokes_collisions_growing_subset<abstract_particle_set::particle_rnumber, abstract_particle_set::partsize_t> p2p_sc;
+ public:
+ stokes_collisions_with_background_rhs(){}
+ ~stokes_collisions_with_background_rhs(){}
+
+ int operator()(
+ double t,
+ abstract_particle_set &pset,
+ abstract_particle_set::particle_rnumber *result,
+ std::vector<std::unique_ptr<abstract_particle_set::particle_rnumber[]>> &additional_data)
+ {
+ auto *sampled_velocity = new abstract_particle_set::particle_rnumber[pset.getLocalNumberOfParticles()*3];
+ // interpolation adds on top of existing values, so result must be cleared.
+ std::fill_n(sampled_velocity, pset.getLocalNumberOfParticles()*3, 0);
+ (*(this->velocity))(t, pset, sampled_velocity);
+ additional_data.push_back(std::unique_ptr<abstract_particle_set::particle_rnumber[]>(
+ new abstract_particle_set::particle_rnumber[pset.getLocalNumberOfParticles()*pset.getStateSize()]));
+
+
+ switch (pset.getStateSize())
+ {
+ case 6:
+ for (long long int idx_part = 0; idx_part < pset.getLocalNumberOfParticles(); idx_part++)
+ {
+ result[idx_part*pset.getStateSize() + IDXC_X] = pset.getParticleState()[idx_part*pset.getStateSize() + 3 + IDXC_X];
+ result[idx_part*pset.getStateSize() + IDXC_Y] = pset.getParticleState()[idx_part*pset.getStateSize() + 3 + IDXC_Y];
+ result[idx_part*pset.getStateSize() + IDXC_Z] = pset.getParticleState()[idx_part*pset.getStateSize() + 3 + IDXC_Z];
+ result[idx_part*pset.getStateSize() + 3 + IDXC_X] = - this->drag_coefficient
+ *(pset.getParticleState()[idx_part*pset.getStateSize() + 3 + IDXC_X]
+ - sampled_velocity[idx_part*3 + IDXC_X]);
+ result[idx_part*pset.getStateSize() + 3 + IDXC_Y] = - this->drag_coefficient
+ *(pset.getParticleState()[idx_part*pset.getStateSize() + 3 + IDXC_Y]
+ - sampled_velocity[idx_part*3 + IDXC_Y]);
+ result[idx_part*pset.getStateSize() + 3 + IDXC_Z] = - this->drag_coefficient
+ *(pset.getParticleState()[idx_part*pset.getStateSize() + 3 + IDXC_Z]
+ - sampled_velocity[idx_part*3 + IDXC_Z]);
+ }
+ break;
+ case 7:
+ for (long long int idx_part = 0; idx_part < pset.getLocalNumberOfParticles(); idx_part++)
+ {
+ result[idx_part*pset.getStateSize() + IDXC_X] = pset.getParticleState()[idx_part*pset.getStateSize() + 3 + IDXC_X];
+ result[idx_part*pset.getStateSize() + IDXC_Y] = pset.getParticleState()[idx_part*pset.getStateSize() + 3 + IDXC_Y];
+ result[idx_part*pset.getStateSize() + IDXC_Z] = pset.getParticleState()[idx_part*pset.getStateSize() + 3 + IDXC_Z];
+ result[idx_part*pset.getStateSize() + 3 + IDXC_X] = - pset.getParticleState()[idx_part*pset.getStateSize() +6]
+ *(pset.getParticleState()[idx_part*pset.getStateSize() + 3 + IDXC_X]
+ - sampled_velocity[idx_part*3 + IDXC_X]);
+ result[idx_part*pset.getStateSize() + 3 + IDXC_Y] = - pset.getParticleState()[idx_part*pset.getStateSize() +6]
+ *(pset.getParticleState()[idx_part*pset.getStateSize() + 3 + IDXC_Y]
+ - sampled_velocity[idx_part*3 + IDXC_Y]);
+ result[idx_part*pset.getStateSize() + 3 + IDXC_Z] = - pset.getParticleState()[idx_part*pset.getStateSize() +6]
+ *(pset.getParticleState()[idx_part*pset.getStateSize() + 3 + IDXC_Z]
+ - sampled_velocity[idx_part*3 + IDXC_Z]);
+ result[idx_part*pset.getStateSize() + 6] = 0;
+ }
+ break;
+ default:
+ break;
+ }
+
+ // applying P2P kernel typically implies a reordering of the particle data
+ // we must apply the reordering to the previously computed rhs values
+ // create array where to store rhs values
+ // copy rhs values to temporary array
+ pset.copy_state_tofrom(
+ additional_data[additional_data.size()-1].get(),
+ result);
+
+ // clear list of colliding particles
+ this->p2p_sc.reset_collision_pairs();
+ // update list of colliding particles
+ pset.template applyP2PKernel<7, p2p_stokes_collisions_growing_subset<abstract_particle_set::particle_rnumber, abstract_particle_set::partsize_t>>(
+ this->p2p_sc,
+ additional_data);
+ // copy shuffled rhs values
+ pset.copy_state_tofrom(
+ result,
+ additional_data[additional_data.size()-1].get());
+ // clear temporary array
+ additional_data.pop_back();
+ delete[] sampled_velocity;
+ return EXIT_SUCCESS;
+ }
+
+ p2p_stokes_collisions_growing_subset<abstract_particle_set::particle_rnumber, abstract_particle_set::partsize_t> &getCollisionCounter()
+ {
+ return this->p2p_sc;
+ }
+
+};
+
+#endif//STOKES_COLLISIONS_WITH_BACKGROUND_RHS_HPP
+
diff --git a/TurTLE/test/collisions/stokes_rhs.cpp b/TurTLE/test/collisions/stokes_rhs.cpp
new file mode 100644
index 0000000000000000000000000000000000000000..d99577166437fdc9400f9bd81dd0906ead728472
--- /dev/null
+++ b/TurTLE/test/collisions/stokes_rhs.cpp
@@ -0,0 +1,31 @@
+/******************************************************************************
+* *
+* Copyright 2022 Max Planck Institute for Dynamics and Self-Organization *
+* *
+* This file is part of TurTLE. *
+* *
+* TurTLE is free software: you can redistribute it and/or modify *
+* it under the terms of the GNU General Public License as published *
+* by the Free Software Foundation, either version 3 of the License, *
+* or (at your option) any later version. *
+* *
+* TurTLE is distributed in the hope that it will be useful, *
+* but WITHOUT ANY WARRANTY; without even the implied warranty of *
+* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the *
+* GNU General Public License for more details. *
+* *
+* You should have received a copy of the GNU General Public License *
+* along with TurTLE. If not, see <http://www.gnu.org/licenses/> *
+* *
+* Contact: Cristian.Lalescu@mpcdf.mpg.de *
+* *
+******************************************************************************/
+
+
+
+//#include "particles/rhs/stokes_rhs.cpp"
+
+
+template class stokes_rhs<float, FFTW, NONE>;
+template class stokes_rhs<double, FFTW, NONE>;
+
diff --git a/TurTLE/test/collisions/stokes_rhs.hpp b/TurTLE/test/collisions/stokes_rhs.hpp
new file mode 100644
index 0000000000000000000000000000000000000000..582f7089f3ed0759329b172363ac14823e500fdd
--- /dev/null
+++ b/TurTLE/test/collisions/stokes_rhs.hpp
@@ -0,0 +1,132 @@
+/******************************************************************************
+* *
+* Copyright 2022 Max Planck Institute for Dynamics and Self-Organization *
+* *
+* This file is part of TurTLE. *
+* *
+* TurTLE is free software: you can redistribute it and/or modify *
+* it under the terms of the GNU General Public License as published *
+* by the Free Software Foundation, either version 3 of the License, *
+* or (at your option) any later version. *
+* *
+* TurTLE is distributed in the hope that it will be useful, *
+* but WITHOUT ANY WARRANTY; without even the implied warranty of *
+* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the *
+* GNU General Public License for more details. *
+* *
+* You should have received a copy of the GNU General Public License *
+* along with TurTLE. If not, see <http://www.gnu.org/licenses/> *
+* *
+* Contact: Cristian.Lalescu@mpcdf.mpg.de *
+* *
+******************************************************************************/
+
+
+
+#ifndef STOKES_RHS_HPP
+#define STOKES_RHS_HPP
+
+#include "particles/interpolation/field_tinterpolator.hpp"
+#include "particles/abstract_particle_rhs.hpp"
+
+
+
+template <typename rnumber,
+ field_backend be,
+ temporal_interpolation_type tt>
+class stokes_rhs: public abstract_particle_rhs
+{
+ protected:
+ field_tinterpolator<rnumber, be, THREE, tt> *velocity;
+ particle_rnumber drag_coefficient;
+ int state_size;
+ public:
+ stokes_rhs(){}
+ ~stokes_rhs(){}
+ const int getStateSize() const
+ {
+ // 3 numbers for position
+ // // 9 numbers for components of deformation tensor // 3 numbers to store logarithmic factors
+ return state_size;
+ }
+
+ int setVelocity(field_tinterpolator<rnumber, be, THREE, tt> *in_velocity)
+ {
+ this->velocity = in_velocity;
+ return EXIT_SUCCESS;
+ }
+
+ int operator()(
+ double t,
+ abstract_particle_set &pset,
+ particle_rnumber *result,
+ std::vector<std::unique_ptr<abstract_particle_set::particle_rnumber[]>> &additional_data)
+ {
+ TIMEZONE("stokes_rhs::operator()");
+ auto *sampled_velocity = new particle_rnumber[pset.getLocalNumberOfParticles()*3];
+ // interpolation adds on top of existing values, so result must be cleared.
+ std::fill_n(sampled_velocity, pset.getLocalNumberOfParticles()*3, 0);
+ (*(this->velocity))(t, pset, sampled_velocity);
+ for (long long int idx_part = 0; idx_part < pset.getLocalNumberOfParticles(); idx_part++)
+ {
+ if(pset.getStateSize()==6)
+ {
+ result[idx_part*pset.getStateSize() + IDXC_X] = pset.getParticleState()[idx_part*pset.getStateSize() + 3 + IDXC_X];
+ result[idx_part*pset.getStateSize() + IDXC_Y] = pset.getParticleState()[idx_part*pset.getStateSize() + 3 + IDXC_Y];
+ result[idx_part*pset.getStateSize() + IDXC_Z] = pset.getParticleState()[idx_part*pset.getStateSize() + 3 + IDXC_Z];
+ result[idx_part*pset.getStateSize() + 3 + IDXC_X] = - this->drag_coefficient
+ *(pset.getParticleState()[idx_part*pset.getStateSize() + 3 + IDXC_X]
+ - sampled_velocity[idx_part*3 + IDXC_X]);
+ result[idx_part*pset.getStateSize() + 3 + IDXC_Y] = - this->drag_coefficient
+ *(pset.getParticleState()[idx_part*pset.getStateSize() + 3 + IDXC_Y]
+ - sampled_velocity[idx_part*3 + IDXC_Y]);
+ result[idx_part*pset.getStateSize() + 3 + IDXC_Z] = - this->drag_coefficient
+ *(pset.getParticleState()[idx_part*pset.getStateSize() + 3 + IDXC_Z]
+ - sampled_velocity[idx_part*3 + IDXC_Z]);
+ }
+
+
+ if(pset.getStateSize()==7)
+ {
+ result[idx_part*pset.getStateSize() + IDXC_X] = pset.getParticleState()[idx_part*pset.getStateSize() + 3 + IDXC_X];
+ result[idx_part*pset.getStateSize() + IDXC_Y] = pset.getParticleState()[idx_part*pset.getStateSize() + 3 + IDXC_Y];
+ result[idx_part*pset.getStateSize() + IDXC_Z] = pset.getParticleState()[idx_part*pset.getStateSize() + 3 + IDXC_Z];
+ result[idx_part*pset.getStateSize() + 3 + IDXC_X] = - pset.getParticleState()[idx_part*pset.getStateSize() +6]
+ *(pset.getParticleState()[idx_part*pset.getStateSize() + 3 + IDXC_X]
+ - sampled_velocity[idx_part*3 + IDXC_X]);
+ result[idx_part*pset.getStateSize() + 3 + IDXC_Y] = - pset.getParticleState()[idx_part*pset.getStateSize() +6]
+ *(pset.getParticleState()[idx_part*pset.getStateSize() + 3 + IDXC_Y]
+ - sampled_velocity[idx_part*3 + IDXC_Y]);
+ result[idx_part*pset.getStateSize() + 3 + IDXC_Z] = - pset.getParticleState()[idx_part*pset.getStateSize() +6]
+ *(pset.getParticleState()[idx_part*pset.getStateSize() + 3 + IDXC_Z]
+ - sampled_velocity[idx_part*3 + IDXC_Z]);
+ result[idx_part*pset.getStateSize() + 6] = 0;
+ }
+
+ }
+ return EXIT_SUCCESS;
+ }
+
+ int imposeModelConstraints(abstract_particle_set &pset) const
+ {
+ return EXIT_SUCCESS;
+ }
+
+ void setDragCoefficient(particle_rnumber in_drag_coefficient)
+ {
+ this->drag_coefficient = in_drag_coefficient;
+ }
+
+ void setStateSize(int in_state_size)
+ {
+ this->state_size = in_state_size;
+ }
+
+ particle_rnumber getDragCoefficient()
+ {
+ return this->drag_coefficient;
+ }
+};
+
+#endif//STOKES_RHS_HPP
+
diff --git a/TurTLE/test/particle_set/NSVEparticle_set.cpp b/TurTLE/test/particle_set/NSVEparticle_set.cpp
new file mode 100644
index 0000000000000000000000000000000000000000..da986349dbe84e1db8e4945a4cf4e475ec894ea3
--- /dev/null
+++ b/TurTLE/test/particle_set/NSVEparticle_set.cpp
@@ -0,0 +1,214 @@
+/**********************************************************************
+* *
+* Copyright 2019 Max Planck Institute *
+* for Dynamics and Self-Organization *
+* *
+* This file is part of TurTLE. *
+* *
+* TurTLE is free software: you can redistribute it and/or modify *
+* it under the terms of the GNU General Public License as published *
+* by the Free Software Foundation, either version 3 of the License, *
+* or (at your option) any later version. *
+* *
+* TurTLE is distributed in the hope that it will be useful, *
+* but WITHOUT ANY WARRANTY; without even the implied warranty of *
+* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the *
+* GNU General Public License for more details. *
+* *
+* You should have received a copy of the GNU General Public License *
+* along with TurTLE. If not, see <http://www.gnu.org/licenses/> *
+* *
+* Contact: Cristian.Lalescu@ds.mpg.de *
+* *
+**********************************************************************/
+
+
+
+#include <string>
+#include <cmath>
+//#include "NSVEparticle_set.hpp"
+#include "scope_timer.hpp"
+
+template <typename rnumber>
+int NSVEparticle_set<rnumber>::initialize(void)
+{
+ TIMEZONE("NSVEparticle_set::intialize");
+ this->NSVE<rnumber>::initialize();
+
+
+ // allocate previous velocity
+ this->previous_velocity = new field<rnumber, FFTW, THREE>(
+ this->fs->cvelocity->rlayout->sizes[2],
+ this->fs->cvelocity->rlayout->sizes[1],
+ this->fs->cvelocity->rlayout->sizes[0],
+ this->fs->cvelocity->rlayout->comm,
+ this->fs->cvelocity->fftw_plan_rigor);
+ this->fs->compute_velocity(this->fs->cvorticity);
+ *this->previous_velocity = *this->fs->cvelocity;
+ this->previous_velocity->ift();
+
+ this->fti = new field_tinterpolator<rnumber, FFTW, THREE, LINEAR>();
+ this->trhs = new tracer_with_collision_counter_rhs<rnumber, FFTW, LINEAR>();
+
+ // We're not using Adams-Bashforth in this code.
+ // This assert is there to ensure
+ // user knows what's happening.
+ assert(tracers0_integration_steps == 0);
+ // neighbours and smoothness are second and third template parameters of particle_set
+ assert(tracers0_neighbours == 3);
+ assert(tracers0_smoothness == 2);
+ this->pset = new particle_set<3, 3, 2>(
+ this->fs->cvelocity->rlayout,
+ this->dkx,
+ this->dky,
+ this->dkz,
+ 0.5);
+
+ // set two fields for the temporal interpolation
+ this->fti->set_field(this->previous_velocity, 0);
+ this->fti->set_field(this->fs->cvelocity, 1);
+ this->trhs->setVelocity(this->fti);
+
+ particles_input_hdf5<long long int, double, 3, 3> particle_reader(
+ this->comm,
+ this->fs->get_current_fname(),
+ std::string("/tracers0/state/") + std::to_string(this->fs->iteration), // dataset name for initial input
+ std::string("/tracers0/rhs/") + std::to_string(this->fs->iteration), // dataset name for initial input
+ this->pset->getSpatialLowLimitZ(),
+ this->pset->getSpatialUpLimitZ());
+ this->pset->init(particle_reader);
+
+ this->psolver = new particle_solver(*(this->pset), 0);
+
+ this->particles_output_writer_mpi = new particles_output_hdf5<
+ long long int, double, 3>(
+ MPI_COMM_WORLD,
+ "tracers0",
+ nparticles,
+ tracers0_integration_steps);
+ this->particles_sample_writer_mpi = new particles_output_sampling_hdf5<
+ long long int, double, double, 3>(
+ MPI_COMM_WORLD,
+ this->pset->getTotalNumberOfParticles(),
+ (this->simname + "_particles.h5"),
+ "tracers0",
+ "position/0");
+ return EXIT_SUCCESS;
+}
+
+template <typename rnumber>
+int NSVEparticle_set<rnumber>::step(void)
+{
+ TIMEZONE("NSVEparticle_set::step");
+ // compute next step Navier-Stokes
+ this->NSVE<rnumber>::step();
+ // update velocity 1
+ this->fs->compute_velocity(this->fs->cvorticity);
+ this->fs->cvelocity->ift();
+ // compute particle step using velocity 0 and velocity 1
+ this->psolver->Heun(this->dt, *(this->trhs));
+ // update velocity 0
+ *this->previous_velocity = *this->fs->cvelocity;
+ this->psolver->setIteration(this->fs->iteration);
+ return EXIT_SUCCESS;
+}
+
+template <typename rnumber>
+int NSVEparticle_set<rnumber>::write_checkpoint(void)
+{
+ TIMEZONE("NSVEparticle_set::write_checkpoint");
+ this->NSVE<rnumber>::write_checkpoint();
+ this->psolver->setIteration(this->fs->iteration);
+ this->particles_output_writer_mpi->open_file(this->fs->get_current_fname());
+ this->psolver->template writeCheckpoint<3>(this->particles_output_writer_mpi);
+ this->particles_output_writer_mpi->close_file();
+ return EXIT_SUCCESS;
+}
+
+template <typename rnumber>
+int NSVEparticle_set<rnumber>::finalize(void)
+{
+ TIMEZONE("NSVEparticle_set::finalize");
+ delete this->particles_sample_writer_mpi;
+ delete this->particles_output_writer_mpi;
+ delete this->psolver;
+ delete this->pset;
+ delete this->trhs;
+ delete this->fti;
+ delete this->previous_velocity;
+ this->NSVE<rnumber>::finalize();
+ return EXIT_SUCCESS;
+}
+
+/** \brief Compute fluid stats and sample fields at particle locations.
+ */
+
+template <typename rnumber>
+int NSVEparticle_set<rnumber>::do_stats()
+{
+ TIMEZONE("NSVEparticle_set::do_stats");
+ /// fluid stats go here
+ this->NSVE<rnumber>::do_stats();
+
+
+ /// either one of two conditions suffices to compute statistics:
+ /// 1) current iteration is a multiple of niter_part
+ /// 2) we are within niter_part_fine_duration/2 of a multiple of niter_part_fine_period
+ if (!(this->iteration % this->niter_part == 0 ||
+ ((this->iteration + this->niter_part_fine_duration/2) % this->niter_part_fine_period <=
+ this->niter_part_fine_duration)))
+ return EXIT_SUCCESS;
+
+ // sample position
+ this->pset->writeStateTriplet(
+ 0,
+ this->particles_sample_writer_mpi,
+ "tracers0",
+ "position",
+ psolver->getIteration());
+
+ MPI_Barrier(this->comm);
+
+ // sample velocity
+ // first ensure velocity field is computed, and is in real space
+ DEBUG_MSG("test 00\n");
+ if (!(this->iteration % this->niter_stat == 0))
+ {
+ // we need to compute velocity field manually, because it didn't happen in NSVE::do_stats()
+ this->fs->compute_velocity(this->fs->cvorticity);
+ *this->tmp_vec_field = *(this->fs->cvelocity);
+ this->tmp_vec_field->ift();
+ }
+ DEBUG_MSG("test 01\n");
+ MPI_Barrier(this->comm);
+ this->pset->writeSample(
+ this->tmp_vec_field,
+ this->particles_sample_writer_mpi,
+ "tracers0",
+ "velocity",
+ psolver->getIteration());
+
+ return EXIT_SUCCESS;
+}
+
+template <typename rnumber>
+int NSVEparticle_set<rnumber>::read_parameters(void)
+{
+ TIMEZONE("NSVEparticle_set::read_parameters");
+ this->NSVE<rnumber>::read_parameters();
+ hid_t parameter_file = H5Fopen((this->simname + ".h5").c_str(), H5F_ACC_RDONLY, H5P_DEFAULT);
+ this->niter_part = hdf5_tools::read_value<int>(parameter_file, "parameters/niter_part");
+ this->niter_part_fine_period = hdf5_tools::read_value<int>(parameter_file, "parameters/niter_part_fine_period");
+ this->niter_part_fine_duration = hdf5_tools::read_value<int>(parameter_file, "parameters/niter_part_fine_duration");
+ this->nparticles = hdf5_tools::read_value<int>(parameter_file, "parameters/nparticles");
+ this->tracers0_integration_steps = hdf5_tools::read_value<int>(parameter_file, "parameters/tracers0_integration_steps");
+ this->tracers0_neighbours = hdf5_tools::read_value<int>(parameter_file, "parameters/tracers0_neighbours");
+ this->tracers0_smoothness = hdf5_tools::read_value<int>(parameter_file, "parameters/tracers0_smoothness");
+ H5Fclose(parameter_file);
+ MPI_Barrier(this->comm);
+ return EXIT_SUCCESS;
+}
+
+template class NSVEparticle_set<float>;
+template class NSVEparticle_set<double>;
+
diff --git a/TurTLE/test/particle_set/NSVEparticle_set.hpp b/TurTLE/test/particle_set/NSVEparticle_set.hpp
new file mode 100644
index 0000000000000000000000000000000000000000..57d7a584fe34d6c9a271a0754712d8cb97657344
--- /dev/null
+++ b/TurTLE/test/particle_set/NSVEparticle_set.hpp
@@ -0,0 +1,99 @@
+/**********************************************************************
+* *
+* Copyright 2017 Max Planck Institute *
+* for Dynamics and Self-Organization *
+* *
+* This file is part of TurTLE. *
+* *
+* TurTLE is free software: you can redistribute it and/or modify *
+* it under the terms of the GNU General Public License as published *
+* by the Free Software Foundation, either version 3 of the License, *
+* or (at your option) any later version. *
+* *
+* TurTLE is distributed in the hope that it will be useful, *
+* but WITHOUT ANY WARRANTY; without even the implied warranty of *
+* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the *
+* GNU General Public License for more details. *
+* *
+* You should have received a copy of the GNU General Public License *
+* along with TurTLE. If not, see <http://www.gnu.org/licenses/> *
+* *
+* Contact: Cristian.Lalescu@ds.mpg.de *
+* *
+**********************************************************************/
+
+
+
+#ifndef NSVEPARTICLE_SET_HPP
+#define NSVEPARTICLE_SET_HPP
+
+
+
+#include <cstdlib>
+#include "base.hpp"
+#include "vorticity_equation.hpp"
+#include "full_code/NSVE.hpp"
+#include "particles/particles_system_builder.hpp"
+#include "particles/particles_output_hdf5.hpp"
+#include "particles/particles_sampling.hpp"
+#include "particles/interpolation/field_tinterpolator.hpp"
+#include "particles/interpolation/particle_set.hpp"
+#include "particles/particle_solver.hpp"
+#include "particles/rhs/tracer_with_collision_counter_rhs.hpp"
+
+/** \brief Test of particle set class hierarchy.
+ *
+ * Child of Navier Stokes vorticity equation solver, this class calls all the
+ * methods from `NSVE`, and in addition integrates simple Lagrangian tracers
+ * in the resulting velocity field.
+ * It also executes a p2p kernel that doesn't affect the particle trajectories
+ * in any way.
+ *
+ * The solver is meant to test that the particle set class hierarchy is working
+ * as desired.
+ */
+
+template <typename rnumber>
+class NSVEparticle_set: public NSVE<rnumber>
+{
+ public:
+
+ /* parameters that are read in read_parameters */
+ int niter_part;
+ int niter_part_fine_period;
+ int niter_part_fine_duration;
+ long long int nparticles;
+ int tracers0_integration_steps;
+ int tracers0_neighbours;
+ int tracers0_smoothness;
+
+ /* other stuff */
+ field_tinterpolator<rnumber, FFTW, THREE, LINEAR> *fti;
+ tracer_with_collision_counter_rhs<rnumber, FFTW, LINEAR> *trhs;
+ particle_set<3, 3, 2> *pset;
+ particle_solver *psolver;
+
+ field<rnumber, FFTW, THREE> *previous_velocity;
+
+ particles_output_hdf5<long long int, double, 3> *particles_output_writer_mpi;
+ particles_output_sampling_hdf5<long long int, double, double, 3> *particles_sample_writer_mpi;
+
+ NSVEparticle_set(
+ const MPI_Comm COMMUNICATOR,
+ const std::string &simulation_name):
+ NSVE<rnumber>(
+ COMMUNICATOR,
+ simulation_name){}
+ ~NSVEparticle_set(){}
+
+ int initialize(void);
+ int step(void);
+ int finalize(void);
+
+ int read_parameters(void);
+ int write_checkpoint(void);
+ int do_stats(void);
+};
+
+#endif//NSVEPARTICLE_SET_HPP
+
diff --git a/TurTLE/test/particle_set/particle_deleter.cpp b/TurTLE/test/particle_set/particle_deleter.cpp
new file mode 100644
index 0000000000000000000000000000000000000000..884517cc8ff891fbc19cf9785153245dd019bc5a
--- /dev/null
+++ b/TurTLE/test/particle_set/particle_deleter.cpp
@@ -0,0 +1,296 @@
+/**********************************************************************
+* *
+* Copyright 2021 Max Planck Institute *
+* for Dynamics and Self-Organization *
+* *
+* This file is part of TurTLE. *
+* *
+* TurTLE is free software: you can redistribute it and/or modify *
+* it under the terms of the GNU General Public License as published *
+* by the Free Software Foundation, either version 3 of the License, *
+* or (at your option) any later version. *
+* *
+* TurTLE is distributed in the hope that it will be useful, *
+* but WITHOUT ANY WARRANTY; without even the implied warranty of *
+* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the *
+* GNU General Public License for more details. *
+* *
+* You should have received a copy of the GNU General Public License *
+* along with TurTLE. If not, see <http://www.gnu.org/licenses/> *
+* *
+* Contact: Cristian.Lalescu@ds.mpg.de *
+* *
+**********************************************************************/
+
+
+
+#include <string>
+#include <cmath>
+//#include "particle_deleter.hpp"
+#include "scope_timer.hpp"
+
+template <typename rnumber>
+int particle_deleter<rnumber>::initialize(void)
+{
+ TIMEZONE("particle_deleter::intialize");
+ this->NSVE<rnumber>::initialize();
+
+
+ // allocate previous velocity
+ this->previous_velocity = new field<rnumber, FFTW, THREE>(
+ this->fs->cvelocity->rlayout->sizes[2],
+ this->fs->cvelocity->rlayout->sizes[1],
+ this->fs->cvelocity->rlayout->sizes[0],
+ this->fs->cvelocity->rlayout->comm,
+ this->fs->cvelocity->fftw_plan_rigor);
+ this->fs->compute_velocity(this->fs->cvorticity);
+ *this->previous_velocity = *this->fs->cvelocity;
+ this->previous_velocity->ift();
+
+ this->fti = new field_tinterpolator<rnumber, FFTW, THREE, LINEAR>();
+ this->trhs = new tracer_with_collision_counter_rhs<rnumber, FFTW, LINEAR>();
+
+ // We're not using Adams-Bashforth in this code.
+ // This assert is there to ensure
+ // user knows what's happening.
+ assert(tracers0_integration_steps == 0);
+ // neighbours and smoothness are second and third template parameters of particle_set
+ assert(tracers0_neighbours == 3);
+ assert(tracers0_smoothness == 2);
+ this->pset = new particle_set<3, 3, 2>(
+ this->fs->cvelocity->rlayout,
+ this->dkx,
+ this->dky,
+ this->dkz,
+ 0.5);
+
+ // set two fields for the temporal interpolation
+ this->fti->set_field(this->previous_velocity, 0);
+ this->fti->set_field(this->fs->cvelocity, 1);
+ this->trhs->setVelocity(this->fti);
+
+ particles_input_hdf5<long long int, double, 3, 3> particle_reader(
+ this->comm,
+ this->fs->get_current_fname(),
+ std::string("/tracers0/state/") + std::to_string(this->fs->iteration), // dataset name for initial input
+ std::string("/tracers0/rhs/") + std::to_string(this->fs->iteration), // dataset name for initial input
+ this->pset->getSpatialLowLimitZ(),
+ this->pset->getSpatialUpLimitZ());
+ this->pset->init(particle_reader);
+
+ this->psolver = new particle_solver(*(this->pset), 0);
+
+ this->particles_output_writer_mpi = new particles_output_hdf5<
+ long long int, double, 3>(
+ MPI_COMM_WORLD,
+ "tracers0",
+ nparticles,
+ tracers0_integration_steps);
+ this->particles_sample_writer_mpi = new particles_output_sampling_hdf5<
+ long long int, double, double, 3>(
+ MPI_COMM_WORLD,
+ this->pset->getTotalNumberOfParticles(),
+ (this->simname + "_particles.h5"),
+ "tracers0",
+ "position/0");
+ return EXIT_SUCCESS;
+}
+
+template <typename rnumber>
+int particle_deleter<rnumber>::step(void)
+{
+ TIMEZONE("particle_deleter::step");
+ // compute next step Navier-Stokes
+ this->NSVE<rnumber>::step();
+ // update velocity 1
+ this->fs->compute_velocity(this->fs->cvorticity);
+ this->fs->cvelocity->ift();
+ // compute particle step using velocity 0 and velocity 1
+ this->psolver->Heun(this->dt, *(this->trhs));
+ // update velocity 0
+ *this->previous_velocity = *this->fs->cvelocity;
+ this->psolver->setIteration(this->fs->iteration);
+ return EXIT_SUCCESS;
+}
+
+template <typename rnumber>
+int particle_deleter<rnumber>::write_checkpoint(void)
+{
+ TIMEZONE("particle_deleter::write_checkpoint");
+ this->NSVE<rnumber>::write_checkpoint();
+ this->psolver->setIteration(this->fs->iteration);
+ this->particles_output_writer_mpi->open_file(this->fs->get_current_fname());
+ this->psolver->template writeCheckpoint<3>(this->particles_output_writer_mpi);
+ this->particles_output_writer_mpi->close_file();
+ return EXIT_SUCCESS;
+}
+
+template <typename rnumber>
+int particle_deleter<rnumber>::finalize(void)
+{
+ TIMEZONE("particle_deleter::finalize");
+ delete this->particles_sample_writer_mpi;
+ delete this->particles_output_writer_mpi;
+ delete this->psolver;
+ delete this->pset;
+ delete this->trhs;
+ delete this->fti;
+ delete this->previous_velocity;
+ this->NSVE<rnumber>::finalize();
+ return EXIT_SUCCESS;
+}
+
+/** \brief Compute fluid stats and sample fields at particle locations.
+ */
+
+template <typename rnumber>
+int particle_deleter<rnumber>::do_stats()
+{
+ TIMEZONE("particle_deleter::do_stats");
+ /// fluid stats go here
+ this->NSVE<rnumber>::do_stats();
+
+
+ /// either one of two conditions suffices to compute statistics:
+ /// 1) current iteration is a multiple of niter_part
+ /// 2) we are within niter_part_fine_duration/2 of a multiple of niter_part_fine_period
+ if (!(this->iteration % this->niter_part == 0 ||
+ ((this->iteration + this->niter_part_fine_duration/2) % this->niter_part_fine_period <=
+ this->niter_part_fine_duration)))
+ return EXIT_SUCCESS;
+
+ // sample position
+ this->pset->writeStateTriplet(
+ 0,
+ this->particles_sample_writer_mpi,
+ "tracers0",
+ "position",
+ psolver->getIteration());
+
+ // sample velocity
+ // first ensure velocity field is computed, and is in real space
+ if (!(this->iteration % this->niter_stat == 0))
+ {
+ // we need to compute velocity field manually, because it didn't happen in NSVE::do_stats()
+ this->fs->compute_velocity(this->fs->cvorticity);
+ *this->tmp_vec_field = this->fs->cvelocity->get_cdata();
+ this->tmp_vec_field->ift();
+ }
+ this->pset->writeSample(
+ this->tmp_vec_field,
+ this->particles_sample_writer_mpi,
+ "tracers0",
+ "velocity",
+ psolver->getIteration());
+
+ if (this->pset->getTotalNumberOfParticles() > 3)
+ {
+ this->pset->writeParticleLabels(
+ "particle_index_before.h5",
+ "tracers0",
+ "index",
+ psolver->getIteration());
+ std::vector<long long int> indices_to_delete(2);
+ indices_to_delete[0] = 1;
+ indices_to_delete[1] = (this->iteration*3)%this->pset->getTotalNumberOfParticles();
+ if (indices_to_delete[1] == indices_to_delete[0])
+ indices_to_delete[1] = 3;
+ std::sort(indices_to_delete.begin(), indices_to_delete.end());
+
+ DEBUG_MSG("computed indices_to_delete:\n");
+ DEBUG_MSG("indices_to_delete[0] = %d\n",
+ indices_to_delete[0]);
+ DEBUG_MSG("indices_to_delete[1] = %d\n",
+ indices_to_delete[1]);
+ DEBUG_MSG("before delete particles, iteration %d\n",
+ this->iteration);
+ this->pset->_print_debug_info();
+ particle_set<3, 3, 2> *tmp_pset = new particle_set<3, 3, 2>(
+ this->fs->cvelocity->rlayout,
+ this->dkx,
+ this->dky,
+ this->dkz,
+ 0.5);
+ tmp_pset->init_as_subset_of(
+ *(this->pset),
+ indices_to_delete,
+ true);
+ *(this->pset) = tmp_pset;
+ delete tmp_pset;
+
+ DEBUG_MSG("after delete particles\n");
+ this->pset->_print_debug_info();
+
+ delete this->particles_output_writer_mpi;
+ delete this->particles_sample_writer_mpi;
+ this->particles_output_writer_mpi = new particles_output_hdf5<
+ long long int, double, 3>(
+ MPI_COMM_WORLD,
+ "tracers0",
+ this->pset->getTotalNumberOfParticles(),
+ tracers0_integration_steps);
+ this->particles_sample_writer_mpi = new particles_output_sampling_hdf5<
+ long long int, double, double, 3>(
+ MPI_COMM_WORLD,
+ this->pset->getTotalNumberOfParticles(),
+ (this->simname + "_particles.h5"),
+ "tracers0",
+ "position/0");
+
+ DEBUG_MSG("before writeParticleIndex\n");
+ this->pset->writeParticleLabels(
+ "particle_index_after.h5",
+ "tracers0",
+ "index",
+ psolver->getIteration());
+ DEBUG_MSG("after writeParticleIndex\n");
+ }
+
+ // test particle subset extraction
+ if (this->pset->getTotalNumberOfParticles() > 4)
+ {
+ particle_set<3, 1, 0> temp_pset(
+ this->fs->cvelocity->rlayout,
+ this->dkx,
+ this->dky,
+ this->dkz);
+ std::vector<long long int> indices_to_extract(2);
+ // TODO: possibly use a more complicated pattern, but remember to keep
+ // indices smaller than current total number of particles
+ indices_to_extract[0] = 2;
+ indices_to_extract[1] = 4;
+ temp_pset.init_as_subset_of(*(this->pset), indices_to_extract, false);
+ DEBUG_MSG("before writeParticleLabels for extracted subset\n");
+ temp_pset.writeParticleLabels(
+ "particle_subset_index.h5",
+ "tracers_subset",
+ "index",
+ psolver->getIteration());
+ DEBUG_MSG("after writeParticleLabels for extracted subset\n");
+ }
+
+ return EXIT_SUCCESS;
+}
+
+template <typename rnumber>
+int particle_deleter<rnumber>::read_parameters(void)
+{
+ TIMEZONE("particle_deleter::read_parameters");
+ this->NSVE<rnumber>::read_parameters();
+ hid_t parameter_file = H5Fopen((this->simname + ".h5").c_str(), H5F_ACC_RDONLY, H5P_DEFAULT);
+ this->niter_part = hdf5_tools::read_value<int>(parameter_file, "parameters/niter_part");
+ this->niter_part_fine_period = hdf5_tools::read_value<int>(parameter_file, "parameters/niter_part_fine_period");
+ this->niter_part_fine_duration = hdf5_tools::read_value<int>(parameter_file, "parameters/niter_part_fine_duration");
+ this->nparticles = hdf5_tools::read_value<int>(parameter_file, "parameters/nparticles");
+ this->tracers0_integration_steps = hdf5_tools::read_value<int>(parameter_file, "parameters/tracers0_integration_steps");
+ this->tracers0_neighbours = hdf5_tools::read_value<int>(parameter_file, "parameters/tracers0_neighbours");
+ this->tracers0_smoothness = hdf5_tools::read_value<int>(parameter_file, "parameters/tracers0_smoothness");
+ H5Fclose(parameter_file);
+ MPI_Barrier(this->comm);
+ return EXIT_SUCCESS;
+}
+
+template class particle_deleter<float>;
+template class particle_deleter<double>;
+
+
diff --git a/TurTLE/test/particle_set/particle_deleter.hpp b/TurTLE/test/particle_set/particle_deleter.hpp
new file mode 100644
index 0000000000000000000000000000000000000000..0561b56d9cb9c2915ac9a8e133449e50ebd239b3
--- /dev/null
+++ b/TurTLE/test/particle_set/particle_deleter.hpp
@@ -0,0 +1,98 @@
+/**********************************************************************
+* *
+* Copyright 2021 Max Planck Institute *
+* for Dynamics and Self-Organization *
+* *
+* This file is part of TurTLE. *
+* *
+* TurTLE is free software: you can redistribute it and/or modify *
+* it under the terms of the GNU General Public License as published *
+* by the Free Software Foundation, either version 3 of the License, *
+* or (at your option) any later version. *
+* *
+* TurTLE is distributed in the hope that it will be useful, *
+* but WITHOUT ANY WARRANTY; without even the implied warranty of *
+* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the *
+* GNU General Public License for more details. *
+* *
+* You should have received a copy of the GNU General Public License *
+* along with TurTLE. If not, see <http://www.gnu.org/licenses/> *
+* *
+* Contact: Cristian.Lalescu@ds.mpg.de *
+* *
+**********************************************************************/
+
+
+
+#ifndef PARTICLE_DELETER_HPP
+#define PARTICLE_DELETER_HPP
+
+
+
+#include <cstdlib>
+#include "base.hpp"
+#include "vorticity_equation.hpp"
+#include "full_code/NSVE.hpp"
+#include "particles/particles_system_builder.hpp"
+#include "particles/particles_output_hdf5.hpp"
+#include "particles/particles_sampling.hpp"
+#include "particles/interpolation/field_tinterpolator.hpp"
+#include "particles/interpolation/particle_set.hpp"
+#include "particles/particle_solver.hpp"
+#include "particles/rhs/tracer_with_collision_counter_rhs.hpp"
+
+/** \brief Test of particle deletion functionality.
+ *
+ * Child of Navier Stokes vorticity equation solver, this class calls all the
+ * methods from `NSVE`, and in addition:
+ * --- integrates simple Lagrangian tracers in the resulting velocity field.
+ * --- removes arbitrary particles from the set of tracers at each time step.
+ * It also executes a p2p kernel that doesn't affect the particle trajectories
+ * in any way.
+ *
+ */
+
+template <typename rnumber>
+class particle_deleter: public NSVE<rnumber>
+{
+ public:
+
+ /* parameters that are read in read_parameters */
+ int niter_part;
+ int niter_part_fine_period;
+ int niter_part_fine_duration;
+ int nparticles;
+ int tracers0_integration_steps;
+ int tracers0_neighbours;
+ int tracers0_smoothness;
+
+ /* other stuff */
+ field_tinterpolator<rnumber, FFTW, THREE, LINEAR> *fti;
+ tracer_with_collision_counter_rhs<rnumber, FFTW, LINEAR> *trhs;
+ particle_set<3, 3, 2> *pset;
+ particle_solver *psolver;
+
+ field<rnumber, FFTW, THREE> *previous_velocity;
+
+ particles_output_hdf5<long long int, double, 3> *particles_output_writer_mpi;
+ particles_output_sampling_hdf5<long long int, double, double, 3> *particles_sample_writer_mpi;
+
+ particle_deleter(
+ const MPI_Comm COMMUNICATOR,
+ const std::string &simulation_name):
+ NSVE<rnumber>(
+ COMMUNICATOR,
+ simulation_name){}
+ ~particle_deleter(){}
+
+ int initialize(void);
+ int step(void);
+ int finalize(void);
+
+ int read_parameters(void);
+ int write_checkpoint(void);
+ int do_stats(void);
+};
+
+#endif//PARTICLE_DELETER_HPP
+
diff --git a/TurTLE/test/profile_scalar.py b/TurTLE/test/profile_scalar.py
new file mode 100644
index 0000000000000000000000000000000000000000..a1ff24090415108d06cf13e460a108ec1518bec3
--- /dev/null
+++ b/TurTLE/test/profile_scalar.py
@@ -0,0 +1,224 @@
+import os
+import sys
+import argparse
+import getpass
+
+import numpy as np
+import h5py
+
+import TurTLE
+import TurTLE._base
+import TurTLE.DNS
+from TurTLE._code import _code
+
+cpp_location = os.path.join(
+ TurTLE.data_dir, 'profiler')
+
+
+class ADNS(TurTLE.DNS):
+ def __init__(
+ self,
+ dns_type = 'scalar_evolution',
+ **kwargs):
+ self.dns_type = dns_type
+ TurTLE.DNS.__init__(self, **kwargs)
+ return None
+ def write_src(
+ self):
+ self.version_message = (
+ '/***********************************************************************\n' +
+ '* this code automatically generated by TurTLE\n' +
+ '* version {0}\n'.format(TurTLE.__version__) +
+ '***********************************************************************/\n\n\n')
+ self.include_list = [
+ '"base.hpp"',
+ '"scope_timer.hpp"',
+ '"fftw_interface.hpp"',
+ '"full_code/main_code.hpp"',
+ '"full_code/NSVEparticles.hpp"',
+ '<cmath>',
+ '<iostream>',
+ '<hdf5.h>',
+ '<string>',
+ '<cstring>',
+ '<fftw3-mpi.h>',
+ '<omp.h>',
+ '<cfenv>',
+ '<cstdlib>']
+ self.main = """
+ int main(int argc, char *argv[])
+ {{
+ bool fpe = (
+ (getenv("BFPS_FPE_OFF") == nullptr) ||
+ (getenv("BFPS_FPE_OFF") != std::string("TRUE")));
+ return main_code< {0} >(argc, argv, fpe);
+ }}
+ """.format(self.dns_type + '<{0}>'.format(self.C_field_dtype))
+ self.includes = '\n'.join(
+ ['#include ' + hh
+ for hh in self.include_list])
+ self.name = 'scalar_evolution'
+ self.definitions += open(
+ os.path.join(
+ cpp_location, self.dns_type + '.hpp'), 'r').read()
+ self.definitions += open(
+ os.path.join(
+ cpp_location, self.dns_type + '.cpp'), 'r').read()
+ with open(self.name + '.cpp', 'w') as outfile:
+ outfile.write(self.version_message + '\n\n')
+ outfile.write(self.includes + '\n\n')
+ outfile.write(self.definitions + '\n\n')
+ outfile.write(self.main + '\n')
+ self.check_current_vorticity_exists = True
+ return None
+ def generate_default_parameters(self):
+ # these parameters are relevant for all DNS classes
+ self.parameters['fftw_plan_rigor'] = 'FFTW_ESTIMATE'
+ self.parameters['dealias_type'] = int(1)
+ self.parameters['dkx'] = float(1.0)
+ self.parameters['dky'] = float(1.0)
+ self.parameters['dkz'] = float(1.0)
+ self.parameters['niter_todo'] = int(12)
+ self.parameters['niter_stat'] = int(4)
+ self.parameters['niter_out'] = int(12)
+ self.parameters['checkpoints_per_file'] = int(1)
+ self.parameters['dt'] = float(0.0001)
+ self.parameters['histogram_bins'] = int(64)
+ self.parameters['max_value_estimate'] = float(1)
+ self.parameters['field_random_seed'] = int(1)
+
+ self.parameters['nu'] = float(0.1)
+ if self.dns_type == 'scalar_evolution':
+ self.parameters['mu'] = float(0.2)
+ self.parameters['injection_rate'] = float(0.4)
+ self.parameters['fk0'] = float(2.0)
+ self.parameters['fk1'] = float(4.0)
+
+ self.parameters['spectrum_dissipation'] = float(0.027)
+ self.parameters['spectrum_Lint'] = float(0.5)
+ self.parameters['spectrum_etaK'] = float(3.0)
+ self.parameters['spectrum_large_scale_const'] = float(6.78)
+ self.parameters['spectrum_small_scale_const'] = float(0.40)
+ return None
+ def prepare_launch(
+ self,
+ args = [],
+ **kwargs):
+ parser = argparse.ArgumentParser('turtle ' + type(self).__name__)
+ self.job_parser_arguments(parser)
+ self.simulation_parser_arguments(parser)
+ self.parameters_to_parser_arguments(parser)
+ opt = parser.parse_args(args)
+ self.simname=opt.simname
+ self.set_precision(opt.precision)
+ self.name = self.dns_type + '-' + self.fluid_precision + '-v' + TurTLE.__version__
+ if ((self.parameters['niter_todo'] % self.parameters['niter_out']) != 0):
+ self.parameters['niter_out'] = self.parameters['niter_todo']
+ if len(opt.src_work_dir) == 0:
+ opt.src_work_dir = os.path.realpath(opt.work_dir)
+ if type(opt.dkx) == type(None):
+ opt.dkx = 2. / opt.Lx
+ if type(opt.dky) == type(None):
+ opt.dky = 2. / opt.Ly
+ if type(opt.dkz) == type(None):
+ opt.dkz = 2. / opt.Lz
+ if type(opt.nx) == type(None):
+ opt.nx = opt.n
+ if type(opt.ny) == type(None):
+ opt.ny = opt.n
+ if type(opt.nz) == type(None):
+ opt.nz = opt.n
+ if self.dns_type == 'scalar_evolution':
+ if type(opt.fk0) == type(None):
+ opt.fk0 = self.parameters['fk0']
+ if type(opt.fk1) == type(None):
+ opt.fk1 = self.parameters['fk1']
+ if type(opt.injection_rate) == type(None):
+ opt.injection_rate = self.parameters['injection_rate']
+ if type(opt.dealias_type) == type(None):
+ opt.dealias_type = self.parameters['dealias_type']
+ if (opt.nx > opt.n or
+ opt.ny > opt.n or
+ opt.nz > opt.n):
+ opt.n = min(opt.nx, opt.ny, opt.nz)
+ print("Warning: '-n' parameter changed to minimum of nx, ny, nz. This affects the computation of nu.")
+ self.parameters['nu'] = (opt.kMeta * 2 / opt.n)**(4./3)
+ # check value of kMax
+ kM = opt.n * 0.5
+ if opt.dealias_type == 1:
+ kM *= 0.8
+ self.set_host_info(TurTLE.host_info)
+ if type(opt.environment) != type(None):
+ self.host_info['environment'] = opt.environment
+ self.pars_from_namespace(opt)
+ return opt
+
+ def write_par(
+ self,
+ iter0 = 0):
+ assert (self.parameters['niter_todo'] % self.parameters['niter_stat'] == 0)
+ assert (self.parameters['niter_todo'] % self.parameters['niter_out'] == 0)
+ assert (self.parameters['niter_out'] % self.parameters['niter_stat'] == 0)
+ _code.write_par(self, iter0 = iter0)
+ with h5py.File(self.get_data_file_name(), 'r+') as ofile:
+ ofile['code_info/exec_name'] = self.name
+ kspace = self.get_kspace()
+ for k in kspace.keys():
+ ofile['kspace/' + k] = kspace[k]
+ nshells = kspace['nshell'].shape[0]
+ kspace = self.get_kspace()
+ nshells = kspace['nshell'].shape[0]
+ scal_stat_datasets = ['scalar']
+ for k in scal_stat_datasets:
+ time_chunk = 2**20//(8*nshells)
+ time_chunk = max(time_chunk, 1)
+ ofile.create_dataset('statistics/spectra/' + k + '_' + k,
+ (1, nshells),
+ chunks = (time_chunk, nshells),
+ maxshape = (None, nshells),
+ dtype = np.float64)
+ time_chunk = 2**20//(8*10)
+ time_chunk = max(time_chunk, 1)
+ a = ofile.create_dataset('statistics/moments/' + k,
+ (1, 10),
+ chunks = (time_chunk, 10),
+ maxshape = (None, 10),
+ dtype = np.float64)
+ time_chunk = 2**20//(8*self.parameters['histogram_bins'])
+ time_chunk = max(time_chunk, 1)
+ ofile.create_dataset('statistics/histograms/' + k,
+ (1,
+ self.parameters['histogram_bins']),
+ chunks = (time_chunk,
+ self.parameters['histogram_bins']),
+ maxshape = (None,
+ self.parameters['histogram_bins']),
+ dtype = np.int64)
+ ofile['checkpoint'] = int(0)
+ return None
+
+ def launch(
+ self,
+ args = [],
+ **kwargs):
+ self.check_current_vorticity_exists = False
+ opt = self.prepare_launch(args = args)
+ if not os.path.exists(self.get_data_file_name()):
+ self.write_par()
+ self.run(
+ nb_processes = opt.nb_processes,
+ nb_threads_per_process = opt.nb_threads_per_process,
+ njobs = opt.njobs,
+ hours = opt.minutes // 60,
+ minutes = opt.minutes % 60,
+ no_submit = opt.no_submit,
+ no_debug = opt.no_debug)
+ return None
+
+def main():
+ bla = ADNS('Kraichnan_scalar_v1')
+ bla.launch(sys.argv[1:])
+ return None
+
+if __name__ == '__main__':
+ main()
diff --git a/TurTLE/test/profiler/Kraichnan_scalar_v1.cpp b/TurTLE/test/profiler/Kraichnan_scalar_v1.cpp
new file mode 100644
index 0000000000000000000000000000000000000000..7e9a647da18ccf034b4a1ff050aedf575ea9abe4
--- /dev/null
+++ b/TurTLE/test/profiler/Kraichnan_scalar_v1.cpp
@@ -0,0 +1,323 @@
+/******************************************************************************
+* *
+* Copyright 2019 Max Planck Institute for Dynamics and Self-Organization *
+* *
+* This file is part of TurTLE. *
+* *
+* TurTLE is free software: you can redistribute it and/or modify *
+* it under the terms of the GNU General Public License as published *
+* by the Free Software Foundation, either version 3 of the License, *
+* or (at your option) any later version. *
+* *
+* TurTLE is distributed in the hope that it will be useful, *
+* but WITHOUT ANY WARRANTY; without even the implied warranty of *
+* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the *
+* GNU General Public License for more details. *
+* *
+* You should have received a copy of the GNU General Public License *
+* along with TurTLE. If not, see <http://www.gnu.org/licenses/> *
+* *
+* Contact: Cristian.Lalescu@ds.mpg.de *
+* *
+******************************************************************************/
+
+
+
+#include <string>
+#include <cmath>
+//#include "Kraichnan_scalar_v1.hpp"
+#include "scope_timer.hpp"
+#include "fftw_tools.hpp"
+
+
+template <typename rnumber>
+int Kraichnan_scalar_v1<rnumber>::initialize(void)
+{
+ TIMEZONE("Kraichnan_scalar_v1::initialize");
+ this->read_iteration();
+ this->read_parameters();
+ if (this->myrank == 0)
+ {
+ // set caching parameters
+ hid_t fapl = H5Pcreate(H5P_FILE_ACCESS);
+ herr_t cache_err = H5Pset_cache(fapl, 0, 521, 134217728, 1.0);
+ variable_used_only_in_assert(cache_err);
+ DEBUG_MSG("when setting stat_file cache I got %d\n", cache_err);
+ this->stat_file = H5Fopen(
+ (this->simname + ".h5").c_str(),
+ H5F_ACC_RDWR,
+ fapl);
+ }
+ int data_file_problem;
+ if (this->myrank == 0)
+ data_file_problem = this->grow_file_datasets();
+ MPI_Bcast(&data_file_problem, 1, MPI_INT, 0, this->comm);
+ if (data_file_problem > 0)
+ {
+ std::cerr <<
+ data_file_problem <<
+ " problems growing file datasets.\ntrying to exit now." <<
+ std::endl;
+ return EXIT_FAILURE;
+ }
+
+ this->scalar = new field<rnumber, FFTW, ONE>(
+ this->nx, this->ny, this->nz,
+ this->comm,
+ fftw_planner_string_to_flag[this->fftw_plan_rigor]);
+
+ this->tscal0 = new field<rnumber, FFTW, ONE>(
+ this->nx, this->ny, this->nz,
+ this->comm,
+ fftw_planner_string_to_flag[this->fftw_plan_rigor]);
+
+ this->ux = new field<rnumber, FFTW, ONE>(
+ this->nx, this->ny, this->nz,
+ this->comm,
+ fftw_planner_string_to_flag[this->fftw_plan_rigor]);
+ this->uy = new field<rnumber, FFTW, ONE>(
+ this->nx, this->ny, this->nz,
+ this->comm,
+ fftw_planner_string_to_flag[this->fftw_plan_rigor]);
+ this->uz = new field<rnumber, FFTW, ONE>(
+ this->nx, this->ny, this->nz,
+ this->comm,
+ fftw_planner_string_to_flag[this->fftw_plan_rigor]);
+
+ this->scalar->print_plan("scalar");
+
+
+ this->kk = new kspace<FFTW, SMOOTH>(
+ this->scalar->clayout,
+ this->dkx, this->dky, this->dkz);
+
+ make_gaussian_random_field<rnumber, FFTW, ONE, SMOOTH>(
+ this->kk,
+ this->scalar,
+ this->random_seed,
+ this->spectrum_dissipation,
+ this->spectrum_Lint,
+ this->spectrum_etaK,
+ this->spectrum_large_scale_const,
+ this->spectrum_small_scale_const);
+
+ this->scalar->symmetrize();
+ this->kk->template dealias<rnumber, ONE>(this->scalar->get_cdata());
+
+ if (this->myrank == 0 && this->iteration == 0)
+ this->kk->store(stat_file);
+ return EXIT_SUCCESS;
+}
+
+template <typename rnumber>
+int Kraichnan_scalar_v1<rnumber>::step(void)
+{
+ TIMEZONE("Kraichnan_scalar_v1::step");
+ this->iteration++;
+
+ // store scalar
+ *(this->tscal0) = *(this->scalar);
+
+ // generate random velocity field
+ make_gaussian_random_field<rnumber, FFTW, ONE, SMOOTH>(
+ this->kk,
+ this->ux,
+ 3*(this->random_seed+this->iteration),
+ this->spectrum_dissipation,
+ this->spectrum_Lint,
+ this->spectrum_etaK,
+ this->spectrum_large_scale_const,
+ this->spectrum_small_scale_const,
+ double(3)/2); // normalization required because we will impose incompressibility
+ make_gaussian_random_field<rnumber, FFTW, ONE, SMOOTH>(
+ this->kk,
+ this->uy,
+ 3*(this->random_seed+this->iteration)+1,
+ this->spectrum_dissipation,
+ this->spectrum_Lint,
+ this->spectrum_etaK,
+ this->spectrum_large_scale_const,
+ this->spectrum_small_scale_const,
+ double(3)/2);
+ make_gaussian_random_field<rnumber, FFTW, ONE, SMOOTH>(
+ this->kk,
+ this->uz,
+ 3*(this->random_seed+this->iteration)+2,
+ this->spectrum_dissipation,
+ this->spectrum_Lint,
+ this->spectrum_etaK,
+ this->spectrum_large_scale_const,
+ this->spectrum_small_scale_const,
+ double(3)/2);
+
+ // ensure velocity field is solenoidal
+ {
+ TIMEZONE("Kraichnan_scalar_v1::divfree");
+ this->kk->CLOOP_K2(
+ [&](const ptrdiff_t cindex,
+ const ptrdiff_t xindex,
+ const ptrdiff_t yindex,
+ const ptrdiff_t zindex,
+ const double k2){
+ if (k2 > 0)
+ {
+ const typename fftw_interface<rnumber>::complex tval = {
+ rnumber((this->kk->kx[xindex]*this->ux->cval(cindex, 0) +
+ this->kk->ky[yindex]*this->uy->cval(cindex, 0) +
+ this->kk->kz[zindex]*this->uz->cval(cindex, 0) ) / k2),
+ rnumber((this->kk->kx[xindex]*this->ux->cval(cindex, 1) +
+ this->kk->ky[yindex]*this->uy->cval(cindex, 1) +
+ this->kk->kz[zindex]*this->uz->cval(cindex, 1) ) / k2)};
+ for (unsigned int imag_part=0; imag_part<2; imag_part++)
+ {
+ this->ux->cval(cindex, imag_part) -= tval[imag_part]*this->kk->kx[xindex];
+ this->uy->cval(cindex, imag_part) -= tval[imag_part]*this->kk->ky[yindex];
+ this->uz->cval(cindex, imag_part) -= tval[imag_part]*this->kk->kz[zindex];
+ }
+ }
+ });
+ }
+
+ // take fields to real space
+ this->ux->ift();
+ this->uy->ift();
+ this->uz->ift();
+ this->tscal0->ift();
+
+ // compute nonlinear term
+ this->tscal0->RLOOP (
+ [&](const ptrdiff_t rindex,
+ const ptrdiff_t xindex,
+ const ptrdiff_t yindex,
+ const ptrdiff_t zindex){
+ this->ux->rval(rindex) *= this->tscal0->rval(rindex) / this->scalar->npoints;
+ this->uy->rval(rindex) *= this->tscal0->rval(rindex) / this->scalar->npoints;
+ this->uz->rval(rindex) *= this->tscal0->rval(rindex) / this->scalar->npoints;
+ }
+ );
+
+ // take fields back to Fourier space
+ this->ux->dft();
+ this->uy->dft();
+ this->uz->dft();
+
+ this->kk->template dealias<rnumber, ONE>(this->ux->get_cdata());
+ this->kk->template dealias<rnumber, ONE>(this->uy->get_cdata());
+ this->kk->template dealias<rnumber, ONE>(this->uz->get_cdata());
+
+ // Euler step
+ this->kk->CLOOP_K2(
+ [&](const ptrdiff_t cindex,
+ const ptrdiff_t xindex,
+ const ptrdiff_t yindex,
+ const ptrdiff_t zindex,
+ const double k2){
+ if (k2 <= this->kk->kM2)
+ {
+ this->scalar->cval(cindex,0) -= this->dt*(
+ this->nu*k2 * this->scalar->cval(cindex, 0) + // linear term
+ this->kk->kx[xindex] * this->ux->cval(cindex, 1) +
+ this->kk->ky[yindex] * this->uy->cval(cindex, 1) +
+ this->kk->kz[zindex] * this->uz->cval(cindex, 1)
+ );
+ this->scalar->cval(cindex,1) += this->dt*(
+ -this->nu*k2 * this->scalar->cval(cindex, 1) + // linear term
+ this->kk->kx[xindex] * this->ux->cval(cindex, 0) +
+ this->kk->ky[yindex] * this->uy->cval(cindex, 0) +
+ this->kk->kz[zindex] * this->uz->cval(cindex, 0)
+ );
+ }
+ else
+ std::fill_n((rnumber*)(this->scalar->get_cdata()+cindex), 2, 0.0);
+ });
+
+ // symmetrize
+ this->scalar->symmetrize();
+
+ return EXIT_SUCCESS;
+}
+
+template <typename rnumber>
+int Kraichnan_scalar_v1<rnumber>::write_checkpoint(void)
+{
+ TIMEZONE("Kraichnan_scalar_v1::write_checkpoint");
+ this->update_checkpoint();
+
+ this->scalar->io(
+ this->get_current_fname(),
+ "scalar",
+ this->iteration,
+ false);
+
+ this->write_iteration();
+ return EXIT_SUCCESS;
+}
+
+template <typename rnumber>
+int Kraichnan_scalar_v1<rnumber>::finalize(void)
+{
+ TIMEZONE("Kraichnan_scalar_v1::finalize");
+ if (this->myrank == 0)
+ H5Fclose(this->stat_file);
+ delete this->tscal0;
+ delete this->ux;
+ delete this->uy;
+ delete this->uz;
+ delete this->scalar;
+ return EXIT_SUCCESS;
+}
+
+template <typename rnumber>
+int Kraichnan_scalar_v1<rnumber>::do_stats()
+{
+ TIMEZONE("Kraichnan_scalar_v1::do_stats");
+ if (!(this->iteration % this->niter_stat == 0))
+ return EXIT_SUCCESS;
+ hid_t stat_group;
+ if (this->myrank == 0)
+ stat_group = H5Gopen(
+ this->stat_file,
+ "statistics",
+ H5P_DEFAULT);
+ else
+ stat_group = 0;
+
+ *(this->tscal0) = *(this->scalar);
+
+ this->tscal0->compute_stats(
+ this->kk,
+ stat_group,
+ "scalar",
+ this->iteration / niter_stat,
+ max_value_estimate);
+
+ if (this->myrank == 0)
+ H5Gclose(stat_group);
+ return EXIT_SUCCESS;
+}
+
+template <typename rnumber>
+int Kraichnan_scalar_v1<rnumber>::read_parameters(void)
+{
+ TIMEZONE("Kraichnan_scalar_v1::read_parameters");
+ this->direct_numerical_simulation::read_parameters();
+ hid_t parameter_file = H5Fopen((this->simname + ".h5").c_str(), H5F_ACC_RDONLY, H5P_DEFAULT);
+ this->nu = hdf5_tools::read_value<double>(parameter_file, "parameters/nu");
+ this->dt = hdf5_tools::read_value<double>(parameter_file, "parameters/dt");
+ this->histogram_bins = hdf5_tools::read_value<int>(parameter_file, "parameters/histogram_bins");
+ this->max_value_estimate = hdf5_tools::read_value<double>(parameter_file, "parameters/max_value_estimate");
+ this->fftw_plan_rigor = hdf5_tools::read_string(parameter_file, "parameters/fftw_plan_rigor");
+
+ this->spectrum_dissipation = hdf5_tools::read_value<double>(parameter_file, "parameters/spectrum_dissipation");
+ this->spectrum_Lint = hdf5_tools::read_value<double>(parameter_file, "parameters/spectrum_Lint");
+ this->spectrum_etaK = hdf5_tools::read_value<double>(parameter_file, "parameters/spectrum_etaK");
+ this->spectrum_large_scale_const = hdf5_tools::read_value<double>(parameter_file, "parameters/spectrum_large_scale_const");
+ this->spectrum_small_scale_const = hdf5_tools::read_value<double>(parameter_file, "parameters/spectrum_small_scale_const");
+ this->random_seed = hdf5_tools::read_value<int>(parameter_file, "/parameters/field_random_seed");
+ H5Fclose(parameter_file);
+ return EXIT_SUCCESS;
+}
+
+template class Kraichnan_scalar_v1<float>;
+template class Kraichnan_scalar_v1<double>;
+
diff --git a/TurTLE/test/profiler/Kraichnan_scalar_v1.hpp b/TurTLE/test/profiler/Kraichnan_scalar_v1.hpp
new file mode 100644
index 0000000000000000000000000000000000000000..97aa9cdb6e37c49dc6ae446dc3ea88984894eb31
--- /dev/null
+++ b/TurTLE/test/profiler/Kraichnan_scalar_v1.hpp
@@ -0,0 +1,80 @@
+/**********************************************************************
+* *
+* Copyright 2017 Max Planck Institute *
+* for Dynamics and Self-Organization *
+* *
+* This file is part of TurTLE. *
+* *
+* TurTLE is free software: you can redistribute it and/or modify *
+* it under the terms of the GNU General Public License as published *
+* by the Free Software Foundation, either version 3 of the License, *
+* or (at your option) any later version. *
+* *
+* TurTLE is distributed in the hope that it will be useful, *
+* but WITHOUT ANY WARRANTY; without even the implied warranty of *
+* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the *
+* GNU General Public License for more details. *
+* *
+* You should have received a copy of the GNU General Public License *
+* along with TurTLE. If not, see <http://www.gnu.org/licenses/> *
+* *
+* Contact: Cristian.Lalescu@ds.mpg.de *
+* *
+**********************************************************************/
+
+
+
+#ifndef KRAICHNAN_SCALAR_V1
+#define KRAICHNAN_SCALAR_V1
+
+
+
+#include <cstdlib>
+#include "base.hpp"
+#include "full_code/direct_numerical_simulation.hpp"
+
+template <typename rnumber>
+class Kraichnan_scalar_v1: public direct_numerical_simulation
+{
+ public:
+
+ /* parameters that are read in read_parameters */
+ double dt;
+ int histogram_bins;
+ double max_value_estimate;
+ double nu;
+ std::string fftw_plan_rigor;
+
+ int random_seed;
+ double spectrum_dissipation;
+ double spectrum_Lint;
+ double spectrum_etaK;
+ double spectrum_large_scale_const;
+ double spectrum_small_scale_const;
+
+ /* other stuff */
+ field<rnumber, FFTW, ONE> *scalar;
+ field<rnumber, FFTW, ONE> *tscal0;
+ field<rnumber, FFTW, ONE> *ux, *uy, *uz;
+ kspace<FFTW, SMOOTH> *kk;
+
+
+ Kraichnan_scalar_v1(
+ const MPI_Comm COMMUNICATOR,
+ const std::string &simulation_name):
+ direct_numerical_simulation(
+ COMMUNICATOR,
+ simulation_name){}
+ ~Kraichnan_scalar_v1(){}
+
+ int initialize(void);
+ int step(void);
+ int finalize(void);
+
+ virtual int read_parameters(void);
+ int write_checkpoint(void);
+ int do_stats(void);
+};
+
+#endif//KRAICHNAN_SCALAR_V1
+
diff --git a/TurTLE/test/profiler/scalar_evolution.cpp b/TurTLE/test/profiler/scalar_evolution.cpp
new file mode 100644
index 0000000000000000000000000000000000000000..5e112ea31ecc86aabbfd67f38f326ec5a0627d37
--- /dev/null
+++ b/TurTLE/test/profiler/scalar_evolution.cpp
@@ -0,0 +1,272 @@
+/******************************************************************************
+* *
+* Copyright 2019 Max Planck Institute for Dynamics and Self-Organization *
+* *
+* This file is part of TurTLE. *
+* *
+* TurTLE is free software: you can redistribute it and/or modify *
+* it under the terms of the GNU General Public License as published *
+* by the Free Software Foundation, either version 3 of the License, *
+* or (at your option) any later version. *
+* *
+* TurTLE is distributed in the hope that it will be useful, *
+* but WITHOUT ANY WARRANTY; without even the implied warranty of *
+* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the *
+* GNU General Public License for more details. *
+* *
+* You should have received a copy of the GNU General Public License *
+* along with TurTLE. If not, see <http://www.gnu.org/licenses/> *
+* *
+* Contact: Cristian.Lalescu@ds.mpg.de *
+* *
+******************************************************************************/
+
+
+
+#include <string>
+#include <cmath>
+//#include "scalar_evolution.hpp"
+#include "scope_timer.hpp"
+#include "fftw_tools.hpp"
+
+
+template <typename rnumber>
+int scalar_evolution<rnumber>::initialize(void)
+{
+ TIMEZONE("scalar_evolution::initialize");
+ this->read_iteration();
+ this->read_parameters();
+ if (this->myrank == 0)
+ {
+ // set caching parameters
+ hid_t fapl = H5Pcreate(H5P_FILE_ACCESS);
+ herr_t cache_err = H5Pset_cache(fapl, 0, 521, 134217728, 1.0);
+ variable_used_only_in_assert(cache_err);
+ DEBUG_MSG("when setting stat_file cache I got %d\n", cache_err);
+ this->stat_file = H5Fopen(
+ (this->simname + ".h5").c_str(),
+ H5F_ACC_RDWR,
+ fapl);
+ }
+ int data_file_problem;
+ if (this->myrank == 0)
+ data_file_problem = this->grow_file_datasets();
+ MPI_Bcast(&data_file_problem, 1, MPI_INT, 0, this->comm);
+ if (data_file_problem > 0)
+ {
+ std::cerr <<
+ data_file_problem <<
+ " problems growing file datasets.\ntrying to exit now." <<
+ std::endl;
+ return EXIT_FAILURE;
+ }
+
+ this->scalar = new field<rnumber, FFTW, ONE>(
+ this->nx, this->ny, this->nz,
+ this->comm,
+ fftw_planner_string_to_flag[this->fftw_plan_rigor]);
+
+ this->tscal0 = new field<rnumber, FFTW, ONE>(
+ this->nx, this->ny, this->nz,
+ this->comm,
+ fftw_planner_string_to_flag[this->fftw_plan_rigor]);
+ this->tscal1 = new field<rnumber, FFTW, ONE>(
+ this->nx, this->ny, this->nz,
+ this->comm,
+ fftw_planner_string_to_flag[this->fftw_plan_rigor]);
+
+ this->kk = new kspace<FFTW, SMOOTH>(
+ this->scalar->clayout,
+ this->dkx, this->dky, this->dkz);
+
+ make_gaussian_random_field<rnumber, FFTW, ONE, SMOOTH>(
+ this->kk,
+ this->scalar,
+ this->random_seed,
+ this->spectrum_dissipation,
+ this->spectrum_Lint,
+ this->spectrum_etaK,
+ this->spectrum_large_scale_const,
+ this->spectrum_small_scale_const);
+
+ this->scalar->symmetrize();
+ this->kk->template dealias<rnumber, ONE>(this->scalar->get_cdata());
+ this->scalar->ift();
+
+ if (this->myrank == 0 && this->iteration == 0)
+ this->kk->store(stat_file);
+ return EXIT_SUCCESS;
+}
+
+template <typename rnumber>
+int scalar_evolution<rnumber>::step(void)
+{
+ TIMEZONE("scalar_evolution::step");
+ this->iteration++;
+
+ // store real space field
+ *(this->tscal0) = *(this->scalar);
+
+ // go to Fourier space
+ this->scalar->dft();
+
+ make_gaussian_random_field<rnumber, FFTW, ONE, SMOOTH>(
+ this->kk,
+ this->tscal1,
+ this->random_seed+this->iteration,
+ this->spectrum_dissipation,
+ this->spectrum_Lint,
+ this->spectrum_etaK,
+ this->spectrum_large_scale_const,
+ this->spectrum_small_scale_const);
+
+ // some nontrivial Fourier space evolution
+ this->kk->CLOOP_K2(
+ [&](const ptrdiff_t cindex,
+ const ptrdiff_t xindex,
+ const ptrdiff_t yindex,
+ const ptrdiff_t zindex,
+ const double k2){
+ const double kval = sqrt(k2);
+ if (k2 <= this->kk->kM2)
+ {
+ this->scalar->cval(cindex,0) += this->dt*(
+ (-this->nu*k2 + this->mu*kval) * this->scalar->cval(cindex, 0) + // linear term
+ (kval / this->kk->kM2) * this->tscal1->cval(cindex, 0) // random change, larger for larger wavenumbers
+ );
+ this->scalar->cval(cindex,1) += this->dt*(
+ (-this->nu*k2 + this->mu*kval) * this->scalar->cval(cindex, 1) + // linear term
+ (kval / this->kk->kM2) * this->tscal1->cval(cindex, 1) // random change, larger for larger wavenumbers
+ );
+ }
+ else
+ std::fill_n((rnumber*)(this->scalar->get_cdata()+cindex), 2, 0.0);
+ });
+
+ // take random field to real space
+ this->tscal1->ift();
+
+ this->tscal0->RLOOP (
+ [&](const ptrdiff_t rindex,
+ const ptrdiff_t xindex,
+ const ptrdiff_t yindex,
+ const ptrdiff_t zindex){
+ this->tscal0->rval(rindex) *= this->tscal1->rval(rindex) / this->scalar->npoints;
+ }
+ );
+
+ this->tscal0->dft();
+ // dealias
+ this->kk->template dealias<rnumber, ONE>(this->tscal0->get_cdata());
+ // another nontrivial Fourier space evolution
+ this->kk->CLOOP_K2(
+ [&](const ptrdiff_t cindex,
+ const ptrdiff_t xindex,
+ const ptrdiff_t yindex,
+ const ptrdiff_t zindex,
+ const double k2){
+ const double kval = 1 + sqrt(k2);
+ const double scal_val = sqrt(
+ this->scalar->cval(cindex, 0)*this->scalar->cval(cindex, 0) +
+ this->scalar->cval(cindex, 1)*this->scalar->cval(cindex, 1));
+ if (k2 <= this->kk->kM2)
+ {
+ this->scalar->cval(cindex,0) /= this->scalar->npoints;
+ this->scalar->cval(cindex,0) -= this->dt*(
+ this->tscal0->cval(cindex, 1)*scal_val / kval
+ );
+ this->scalar->cval(cindex,1) /= this->scalar->npoints;
+ this->scalar->cval(cindex,1) += this->dt*(
+ this->tscal0->cval(cindex, 0)*scal_val / kval
+ );
+ }
+ else
+ std::fill_n((rnumber*)(this->scalar->get_cdata()+cindex), 2, 0.0);
+ });
+
+ // symmetrize
+ this->scalar->symmetrize();
+
+ // go back to real space
+ this->scalar->ift();
+
+ return EXIT_SUCCESS;
+}
+
+template <typename rnumber>
+int scalar_evolution<rnumber>::write_checkpoint(void)
+{
+ TIMEZONE("scalar_evolution::write_checkpoint");
+ return EXIT_SUCCESS;
+}
+
+template <typename rnumber>
+int scalar_evolution<rnumber>::finalize(void)
+{
+ TIMEZONE("scalar_evolution::finalize");
+ if (this->myrank == 0)
+ H5Fclose(this->stat_file);
+ delete this->tscal0;
+ delete this->tscal1;
+ delete this->scalar;
+ return EXIT_SUCCESS;
+}
+
+template <typename rnumber>
+int scalar_evolution<rnumber>::do_stats()
+{
+ TIMEZONE("scalar_evolution::do_stats");
+ if (!(this->iteration % this->niter_stat == 0))
+ return EXIT_SUCCESS;
+ hid_t stat_group;
+ if (this->myrank == 0)
+ stat_group = H5Gopen(
+ this->stat_file,
+ "statistics",
+ H5P_DEFAULT);
+ else
+ stat_group = 0;
+
+ *(this->tscal0) = *(this->scalar);
+
+ this->tscal0->compute_stats(
+ this->kk,
+ stat_group,
+ "scalar",
+ this->iteration / niter_stat,
+ max_value_estimate);
+
+ if (this->myrank == 0)
+ H5Gclose(stat_group);
+ return EXIT_SUCCESS;
+}
+
+template <typename rnumber>
+int scalar_evolution<rnumber>::read_parameters(void)
+{
+ TIMEZONE("scalar_evolution::read_parameters");
+ this->direct_numerical_simulation::read_parameters();
+ hid_t parameter_file = H5Fopen((this->simname + ".h5").c_str(), H5F_ACC_RDONLY, H5P_DEFAULT);
+ this->nu = hdf5_tools::read_value<double>(parameter_file, "parameters/nu");
+ this->mu = hdf5_tools::read_value<double>(parameter_file, "parameters/mu");
+ this->dt = hdf5_tools::read_value<double>(parameter_file, "parameters/dt");
+ this->injection_rate = hdf5_tools::read_value<double>(parameter_file, "parameters/injection_rate");
+ this->fk0 = hdf5_tools::read_value<double>(parameter_file, "parameters/fk0");
+ this->fk1 = hdf5_tools::read_value<double>(parameter_file, "parameters/fk1");
+ this->histogram_bins = hdf5_tools::read_value<int>(parameter_file, "parameters/histogram_bins");
+ this->max_value_estimate = hdf5_tools::read_value<double>(parameter_file, "parameters/max_value_estimate");
+ this->fftw_plan_rigor = hdf5_tools::read_string(parameter_file, "parameters/fftw_plan_rigor");
+
+ this->spectrum_dissipation = hdf5_tools::read_value<double>(parameter_file, "parameters/spectrum_dissipation");
+ this->spectrum_Lint = hdf5_tools::read_value<double>(parameter_file, "parameters/spectrum_Lint");
+ this->spectrum_etaK = hdf5_tools::read_value<double>(parameter_file, "parameters/spectrum_etaK");
+ this->spectrum_large_scale_const = hdf5_tools::read_value<double>(parameter_file, "parameters/spectrum_large_scale_const");
+ this->spectrum_small_scale_const = hdf5_tools::read_value<double>(parameter_file, "parameters/spectrum_small_scale_const");
+ this->random_seed = hdf5_tools::read_value<int>(parameter_file, "/parameters/field_random_seed");
+ H5Fclose(parameter_file);
+ return EXIT_SUCCESS;
+}
+
+template class scalar_evolution<float>;
+template class scalar_evolution<double>;
+
diff --git a/TurTLE/test/profiler/scalar_evolution.hpp b/TurTLE/test/profiler/scalar_evolution.hpp
new file mode 100644
index 0000000000000000000000000000000000000000..0eea813183e52d1f489e502f4205b00fcef48833
--- /dev/null
+++ b/TurTLE/test/profiler/scalar_evolution.hpp
@@ -0,0 +1,83 @@
+/**********************************************************************
+* *
+* Copyright 2017 Max Planck Institute *
+* for Dynamics and Self-Organization *
+* *
+* This file is part of TurTLE. *
+* *
+* TurTLE is free software: you can redistribute it and/or modify *
+* it under the terms of the GNU General Public License as published *
+* by the Free Software Foundation, either version 3 of the License, *
+* or (at your option) any later version. *
+* *
+* TurTLE is distributed in the hope that it will be useful, *
+* but WITHOUT ANY WARRANTY; without even the implied warranty of *
+* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the *
+* GNU General Public License for more details. *
+* *
+* You should have received a copy of the GNU General Public License *
+* along with TurTLE. If not, see <http://www.gnu.org/licenses/> *
+* *
+* Contact: Cristian.Lalescu@ds.mpg.de *
+* *
+**********************************************************************/
+
+
+
+#ifndef SCALAR_EVOLUTION_HPP
+#define SCALAR_EVOLUTION_HPP
+
+
+
+#include <cstdlib>
+#include "base.hpp"
+#include "full_code/direct_numerical_simulation.hpp"
+
+template <typename rnumber>
+class scalar_evolution: public direct_numerical_simulation
+{
+ public:
+
+ /* parameters that are read in read_parameters */
+ double dt;
+ double fk0;
+ double fk1;
+ double injection_rate;
+ int histogram_bins;
+ double max_value_estimate;
+ double nu;
+ double mu;
+ std::string fftw_plan_rigor;
+
+ int random_seed;
+ double spectrum_dissipation;
+ double spectrum_Lint;
+ double spectrum_etaK;
+ double spectrum_large_scale_const;
+ double spectrum_small_scale_const;
+
+ /* other stuff */
+ field<rnumber, FFTW, ONE> *scalar;
+ field<rnumber, FFTW, ONE> *tscal0, *tscal1;
+ kspace<FFTW, SMOOTH> *kk;
+
+
+ scalar_evolution(
+ const MPI_Comm COMMUNICATOR,
+ const std::string &simulation_name):
+ direct_numerical_simulation(
+ COMMUNICATOR,
+ simulation_name){}
+ ~scalar_evolution(){}
+
+ int initialize(void);
+ int step(void);
+ int finalize(void);
+
+ virtual int read_parameters(void);
+ int write_checkpoint(void);
+ int do_stats(void);
+};
+
+#endif//SCALAR_EVOLUTION_HPP
+
diff --git a/TurTLE/test/test_Gaussian_field.py b/TurTLE/test/test_Gaussian_field.py
new file mode 100644
index 0000000000000000000000000000000000000000..31f3813eb20624fc552ea8ba99168b435636d22f
--- /dev/null
+++ b/TurTLE/test/test_Gaussian_field.py
@@ -0,0 +1,307 @@
+#! /usr/bin/env python
+
+import numpy as np
+from scipy import trapz
+from scipy.stats import norm
+from scipy.integrate import quad
+from scipy.optimize import root
+import h5py
+import sys, os
+import time
+
+import TurTLE
+from TurTLE import TEST
+
+try:
+ import matplotlib.pyplot as plt
+except:
+ plt = None
+
+def main():
+ # size of grid
+ n = 256
+ # from a 1024^3 simulation
+ dissipation = 0.37127850066981344
+ Lint = 0.9029294339535114 # probably the wrong (inconsistent with Pope) definition of Lint
+ etaK = 0.003730966576882254
+
+ c_L, c_eta = calculate_constants(epsilon=dissipation, Lint=Lint, etaK=etaK)
+
+ bin_no = 100
+ rseed = int(time.time())
+ simname = 'Gaussianity_test'
+
+ if not os.path.exists(simname+'.h5'):
+ c = TEST()
+ opt = c.launch(
+ ['Gauss_field_test',
+ '--nx', str(n),
+ '--ny', str(n),
+ '--nz', str(n),
+ '--simname', simname,
+ '--np', '4',
+ '--environment', 'short',
+ '--minutes', '60',
+ '--ntpp', '1',
+ '--wd', './',
+ '--histogram_bins', str(bin_no),
+ '--max_velocity_estimate', '8.',
+ '--spectrum_dissipation', str(dissipation),
+ '--spectrum_Lint', str(Lint),
+ '--spectrum_etaK', str(etaK),
+ '--spectrum_large_scale_const', str(c_L),
+ '--spectrum_small_scale_const', str(c_eta),
+ '--field_random_seed', str(rseed)] +
+ sys.argv[1:])
+ plot_stuff(simname)
+ return None
+
+def main_premultiplied_spectra_test():
+ n_arr = [64, 128, 256, 512]
+ #n_arr = [64]
+ run_no = 3
+ dk = 1.
+ diss = 0.007036634455033392
+ energy = 0.0417604575121591
+ etaK = 0.2157937040813811
+ Lint = energy**(3./2.)/diss
+
+ c_L, c_eta = calculate_constants(epsilon=diss, Lint=Lint, etaK=etaK)
+
+ bin_no = 100
+
+ i = 1
+ if not os.path.exists('conv_test_n64_run0.h5'):
+ for n in n_arr:
+ for run in range(run_no):
+ simname = 'conv_test_n{0}_run{1}'.format(n, run)
+
+ rseed = i
+ i += 1
+ c = TEST()
+ opt = c.launch(
+ ['Gauss_field_test',
+ '--nx', str(n),
+ '--ny', str(n),
+ '--nz', str(n),
+ '--dkx', str(dk),
+ '--dky', str(dk),
+ '--dkz', str(dk),
+ '--simname', simname,
+ '--np', '4',
+ '--environment', 'short',
+ '--minutes', '60',
+ '--ntpp', '1',
+ '--wd', './',
+ '--histogram_bins', str(bin_no),
+ '--max_velocity_estimate', '8.',
+ '--spectrum_dissipation', str(diss),
+ '--spectrum_Lint', str(Lint),
+ '--spectrum_etaK', str(etaK),
+ '--spectrum_large_scale_const', str(c_L),
+ '--spectrum_small_scale_const', str(c_eta),
+ '--field_random_seed', str(rseed)] +
+ sys.argv[1:])
+ else:
+ print('Energy = {}'.format(energy))
+
+ Q = quad(lambda k : k**2*E(k, epsilon=diss, etaK=etaK, Lint=Lint, c_L=c_L, c_eta=c_eta),
+ 0., np.inf)[0]/15.
+ print('Q = {}'.format(Q))
+ P = quad(lambda k : k**4*E(k, epsilon=diss, etaK=etaK, Lint=Lint, c_L=c_L, c_eta=c_eta),
+ 0., np.inf)[0]/105.
+ print('P = {}'.format(P))
+
+ for n in n_arr:
+ for run in range(run_no):
+ simname = 'conv_test_n{0}_run{1}'.format(n, run)
+ df = h5py.File(simname + '.h5', 'r')
+
+ kk = df['kspace/kshell'][...]
+
+ phi_ij = df['statistics/spectra/velocity_velocity'][0]
+ energy_spec = (phi_ij[..., 0, 0] + phi_ij[..., 1, 1] + phi_ij[..., 2, 2])/2
+ k2spec_trace = (
+ df['statistics/spectra/k*velocity_k*velocity'][..., 0, 0]
+ + df['statistics/spectra/k*velocity_k*velocity'][..., 1, 1]
+ + df['statistics/spectra/k*velocity_k*velocity'][..., 2, 2])/2
+ k4spec_trace = (
+ df['statistics/spectra/k2*velocity_k2*velocity'][..., 0, 0]
+ + df['statistics/spectra/k2*velocity_k2*velocity'][..., 1, 1]
+ + df['statistics/spectra/k2*velocity_k2*velocity'][..., 2, 2])/2
+ print('----------------------')
+ print('n = {}, run = {}'.format(n, run))
+ print('Energy = {}'.format(np.sum(energy_spec)))
+ print('Q = {}'.format(np.sum(k2spec_trace)/15.))
+ print('P = {}'.format(np.sum(k4spec_trace)/105.))
+
+
+
+def main_deltak_test():
+ n_arr = [64, 128, 256, 512]
+ dk_arr = [1., 0.5, 0.25, 0.125]
+ diss = 0.007036634455033392
+ energy = 0.0417604575121591
+ etaK = 0.2157937040813811
+ Lint = energy**(3./2.)/diss
+
+ c_L, c_eta = calculate_constants(epsilon=diss, Lint=Lint, etaK=etaK)
+
+ bin_no = 100
+
+ if not os.path.exists('conv_test_n64_run0.h5'):
+ for n, dk in zip(n_arr, dk_arr):
+ for run in range(3):
+ simname = 'conv_test_n{0}_run{1}'.format(n, run)
+
+ rseed = int(time.time())
+ c = TEST()
+ opt = c.launch(
+ ['Gauss_field_test',
+ '--nx', str(n),
+ '--ny', str(n),
+ '--nz', str(n),
+ '--dkx', str(dk),
+ '--dky', str(dk),
+ '--dkz', str(dk),
+ '--simname', simname,
+ '--np', '4',
+ '--environment', 'short',
+ '--minutes', '60',
+ '--ntpp', '1',
+ '--wd', './',
+ '--histogram_bins', str(bin_no),
+ '--max_velocity_estimate', '8.',
+ '--spectrum_dissipation', str(diss),
+ '--spectrum_Lint', str(Lint),
+ '--spectrum_etaK', str(etaK),
+ '--spectrum_large_scale_const', str(c_L),
+ '--spectrum_small_scale_const', str(c_eta),
+ '--field_random_seed', str(rseed)] +
+ sys.argv[1:])
+ else:
+ plt.figure()
+ k = np.geomspace(1e-1, 4e1, 1000)
+ plt.plot(k, E(k, epsilon=diss, Lint=Lint, etaK=etaK, c_L=c_L, c_eta=c_eta))
+ for n in n_arr:
+ run = 0
+ simname = 'conv_test_n{0}_run{1}'.format(n, run)
+ df = h5py.File(simname + '.h5', 'r')
+
+ kk = df['kspace/kshell'][...]
+ phi_ij = df['statistics/spectra/velocity_velocity'][0] / df['kspace/dk'][()]
+ energy_spec = (phi_ij[..., 0, 0] + phi_ij[..., 1, 1] + phi_ij[..., 2, 2])/2
+ plt.scatter(kk[1:-2], energy_spec[1:-2], label='n={0}, r={1}'.format(n, run))
+ plt.xscale('log')
+ plt.yscale('log')
+ plt.legend()
+ plt.savefig('spectra_convergence.pdf')
+
+
+def calculate_constants(*, epsilon, Lint, etaK):
+ sol = root(optimize_func,
+ x0 = (6.78, 0.40),
+ args = (epsilon, Lint, etaK),
+ method='hybr',
+ options={'eps':0.001, 'factor':0.1})
+ assert sol.success
+ return sol.x
+
+def optimize_func(constants, diss, Lint, etaK):
+ energy = (Lint*diss)**(2./3.)
+ energy_model = quad(
+ lambda k : E(k, epsilon=diss, Lint=Lint, etaK=etaK, c_L=constants[0], c_eta=constants[1]),
+ 0, np.inf)[0]
+ nu = (etaK**4*diss)**(1./3.)
+ diss_model = quad(
+ lambda k : 2*nu*k**2*E(k, epsilon=diss, Lint=Lint, etaK=etaK, c_L=constants[0], c_eta=constants[1]),
+ 0, np.inf)[0]
+ return (energy_model - energy, diss_model - diss)
+
+def E(k, *, epsilon, C=1.5, Lint, etaK, c_L=6.78, c_eta=0.4, beta=5.2):
+ return C*epsilon**(2./3.)*k**(-5./3.)*f_L(k*Lint, c_L=c_L)*f_eta(k*etaK, beta=beta, c_eta=c_eta)
+
+def f_L(x, *, c_L):
+ return (x/(x**2 + c_L)**(1./2.))**(11./3.)
+
+def f_eta(x, *, beta, c_eta):
+ return np.exp(-beta*((x**4 + c_eta**4)**(1./4.) - c_eta))
+
+def plot_stuff(simname):
+ df = h5py.File(simname + '.h5', 'r')
+ for kk in ['spectrum_dissipation',
+ 'spectrum_Lint',
+ 'spectrum_etaK',
+ 'spectrum_large_scale_const',
+ 'spectrum_small_scale_const',
+ 'field_random_seed',
+ 'histogram_bins']:
+ print(kk, df['parameters/' + kk][...])
+ dissipation = df['parameters/spectrum_dissipation'][()]
+ Lint = df['parameters/spectrum_Lint'][()]
+ etaK = df['parameters/spectrum_etaK'][()]
+ c_L = df['parameters/spectrum_large_scale_const'][()]
+ c_eta = df['parameters/spectrum_small_scale_const'][()]
+ bin_no = df['parameters/histogram_bins'][()]
+
+ f = plt.figure()
+ # test spectrum
+ a = f.add_subplot(121)
+ kk = df['kspace/kshell'][...]
+ print('dk: {}'.format(df['kspace/dk'][()]))
+ phi_ij = df['statistics/spectra/velocity_velocity'][0] / df['kspace/dk'][()]
+ energy = (phi_ij[..., 0, 0] + phi_ij[..., 1, 1] + phi_ij[..., 2, 2])/2
+
+ a.scatter(kk[1:-2], energy[1:-2])
+ a.plot(kk[1:-2],
+ E(kk[1:-2],
+ epsilon=dissipation, Lint=Lint, etaK=etaK, c_eta=c_eta, c_L=c_L),
+ ls='--', c='C0')
+ a.set_xscale('log')
+ a.set_yscale('log')
+ # test isotropy
+ a = f.add_subplot(122)
+
+ max_vel_estimate = df['parameters/max_velocity_estimate'][()]
+ velbinsize = 2*max_vel_estimate/bin_no
+ vel = np.linspace(-max_vel_estimate+velbinsize/2, max_vel_estimate-velbinsize/2, bin_no)
+ hist_vel = df['statistics/histograms/velocity'][0, :, :3]
+ f_vel = hist_vel / np.sum(hist_vel, axis=0, keepdims=True).astype(float) / velbinsize
+
+ print(np.sum(energy*np.arange(len(energy))**2))
+ print('Energy sum: {}'.format(np.sum(energy*df['kspace/dk'][()])))
+ print('Moment sum: {}'.format(df['statistics/moments/velocity'][0,2,3]/2))
+ print('Velocity variances: {}'.format(trapz(vel[:,None]**2*f_vel, vel[:,None], axis=0)))
+
+ vel_variance = df['statistics/moments/velocity'][0,2,3]/3.
+ a.plot(vel[:,None]/np.sqrt(vel_variance), f_vel*np.sqrt(vel_variance))
+ a.plot(vel/np.sqrt(vel_variance), norm.pdf(vel/np.sqrt(vel_variance)), ls='--')
+ a.set_yscale('log')
+ a.set_xlim(-5,5)
+ a.set_ylim(1e-5,1)
+ f.tight_layout()
+ f.savefig('spectrum_isotropy_test.pdf')
+ plt.close(f)
+
+ ### check divergence
+ print('Divergence second moment is: {0}'.format(
+ df['statistics/moments/velocity_divergence'][0, 2]))
+ print('Gradient second moment is: {0}'.format(
+ df['statistics/moments/velocity_gradient'][0, 2].mean()))
+
+ print('----------- k2-premultiplied spectrum -----------')
+ #k2func = lambda k, k_c=k_cutoff, s=slope : k**(2+s)*np.exp(-k/k_c)
+ #k2sum_analytic = quad(k2func, 0, k_cutoff*20)[0]
+ #print('Analytically: {}'.format(k2sum_analytic))
+ k2spec_trace = (
+ df['statistics/spectra/k*velocity_k*velocity'][..., 0, 0]
+ + df['statistics/spectra/k*velocity_k*velocity'][..., 1, 1]
+ + df['statistics/spectra/k*velocity_k*velocity'][..., 2, 2])
+ #print('Energy sum: {}'.format(np.sum(k2spec_trace*df['kspace/dk'][()])/2./coeff))
+
+ df.close()
+ return None
+
+if __name__ == '__main__':
+ main_premultiplied_spectra_test()
+
diff --git a/TurTLE/test/test_Heun_p2p.py b/TurTLE/test/test_Heun_p2p.py
new file mode 100644
index 0000000000000000000000000000000000000000..7947ded4da7e908e6db037bedbd719518786ddd7
--- /dev/null
+++ b/TurTLE/test/test_Heun_p2p.py
@@ -0,0 +1,162 @@
+import os
+import sys
+import argparse
+import getpass
+
+import numpy as np
+import h5py
+
+import TurTLE
+import TurTLE._base
+import TurTLE.DNS
+from TurTLE._code import _code
+
+cpp_location = os.path.join(
+ TurTLE.data_dir, 'particle_set')
+
+
+class ADNS(TurTLE.DNS):
+ def __init__(
+ self,
+ **kwargs):
+ TurTLE.DNS.__init__(self, **kwargs)
+ self.list_of_particle_codes.append('NSVEparticle_set')
+ self.extra_parameters['NSVEparticle_set'] = {}
+ return None
+ def write_src(
+ self):
+ self.version_message = (
+ '/***********************************************************************\n' +
+ '* this code automatically generated by TurTLE\n' +
+ '* version {0}\n'.format(TurTLE.__version__) +
+ '***********************************************************************/\n\n\n')
+ self.include_list = [
+ '"base.hpp"',
+ '"scope_timer.hpp"',
+ '"fftw_interface.hpp"',
+ '"full_code/main_code.hpp"',
+ '"full_code/NSVEparticles.hpp"',
+ '<cmath>',
+ '<iostream>',
+ '<hdf5.h>',
+ '<string>',
+ '<cstring>',
+ '<fftw3-mpi.h>',
+ '<omp.h>',
+ '<cfenv>',
+ '<cstdlib>']
+ self.main = """
+ int main(int argc, char *argv[])
+ {{
+ bool fpe = (
+ (getenv("BFPS_FPE_OFF") == nullptr) ||
+ (getenv("BFPS_FPE_OFF") != std::string("TRUE")));
+ return main_code< {0} >(argc, argv, fpe);
+ }}
+ """.format(self.dns_type + '<{0}>'.format(self.C_field_dtype))
+ self.includes = '\n'.join(
+ ['#include ' + hh
+ for hh in self.include_list])
+ self.name = 'NSVEparticle_set'
+ self.dns_type = 'NSVEparticle_set'
+ self.definitions += open(
+ os.path.join(
+ cpp_location, 'NSVEparticle_set.hpp'), 'r').read()
+ self.definitions += open(
+ os.path.join(
+ cpp_location, 'NSVEparticle_set.cpp'), 'r').read()
+ with open(self.name + '.cpp', 'w') as outfile:
+ outfile.write(self.version_message + '\n\n')
+ outfile.write(self.includes + '\n\n')
+ outfile.write(self.definitions + '\n\n')
+ outfile.write(self.main + '\n')
+ self.check_current_vorticity_exists = True
+ return None
+
+ def write_par(
+ self,
+ iter0 = 0):
+ TurTLE.DNS.write_par(self, iter0 = iter0)
+ with h5py.File(self.get_data_file_name(), 'r+') as ofile:
+ #TODO move collision stats to particle stats
+ for i in range(self.parameters['niter_out']):
+ ofile.create_dataset('statistics/collisions/'+str(i),
+ shape=(1,),
+ dtype = np.int64)
+ return None
+
+ def launch(
+ self,
+ args = [],
+ **kwargs):
+ opt = self.prepare_launch(args = args)
+ if not os.path.exists(self.get_data_file_name()):
+ self.write_par()
+ self.generate_initial_condition(opt = opt)
+ with h5py.File(self.get_particle_file_name(), 'w') as particle_file:
+ particle_file.create_group('tracers0/position')
+ particle_file.create_group('tracers0/velocity')
+ particle_file.create_group('collisions0')
+ self.generate_tracer_state(integration_steps = self.parameters['tracers0_integration_steps'],
+ rseed = opt.particle_rand_seed,
+ ncomponents = 3)
+ self.run(
+ nb_processes = opt.nb_processes,
+ nb_threads_per_process = opt.nb_threads_per_process,
+ njobs = opt.njobs,
+ hours = opt.minutes // 60,
+ minutes = opt.minutes % 60,
+ no_submit = opt.no_submit,
+ no_debug = opt.no_debug)
+ return None
+
+def main():
+ bla = ADNS()
+ bla.launch([
+ 'NSVEparticle_set',
+ '--niter_todo', '200',
+ '--niter_stat', '20',
+ '--niter_out', '200',
+ '--dt', '0.0001',
+ '--nparticles', '1000',
+ '--tracers0_integration_steps', '0',
+ '--tracers0_neighbours', '3',
+ '--tracers0_smoothness', '2',
+ '--tracers0_cutoff', '0.920272'
+ ] + sys.argv[1:])
+ assert(bla.parameters['niter_part'] == 1)
+ df = bla.get_particle_file()
+ x = []
+ v = []
+ for ii in range(bla.parameters['niter_out']):
+ x.append(df['tracers0/position/{0}'.format(ii)][()])
+ v.append(df['tracers0/velocity/{0}'.format(ii)][()])
+ df.close()
+ x = np.array(x)
+ v = np.array(v)
+ d1x = (x[2:] - x[:-2]) / (2*bla.parameters['dt'])
+ vv = v[1:-1]
+ diff = vv - d1x
+ relative_error = (np.sum(diff**2, axis = 2) / np.sum(vv**2, axis = 2))**0.5
+ tindex, trajindex = np.unravel_index(
+ np.argmax(relative_error),
+ relative_error.shape)
+ try:
+ import matplotlib.pyplot as plt
+ f = plt.figure()
+ a = f.add_subplot(211)
+ a.plot(vv[:, trajindex])
+ a.plot(d1x[:, trajindex], dashes = (2, 2))
+ a = f.add_subplot(212)
+ a.plot(x[:, trajindex])
+ a.set_title('maximum error for (tindex, trajindex) = ({0}, {1})'.format(tindex, trajindex))
+ f.tight_layout()
+ f.savefig('traj.pdf')
+ except ImportError:
+ print('couldn\'t import matplotlib, no figure')
+ print(np.max(relative_error))
+ assert(np.max(relative_error) < 1e-4)
+ return None
+
+if __name__ == '__main__':
+ main()
diff --git a/TurTLE/test/test_Parseval.py b/TurTLE/test/test_Parseval.py
new file mode 100644
index 0000000000000000000000000000000000000000..54dfe2f1b9f4b55d52b4e7887058693970d471c0
--- /dev/null
+++ b/TurTLE/test/test_Parseval.py
@@ -0,0 +1,38 @@
+#! /usr/bin/env python
+
+import numpy as np
+import sys
+
+import TurTLE
+from TurTLE import DNS
+
+def main():
+ niterations = 10
+ nlist = [16, 32, 48, 24, 64, 12]
+ for ii in range(len(nlist)):
+ c = DNS()
+ c.launch(
+ ['NSVE',
+ '--nx', str(nlist[ii]),
+ '--ny', str(nlist[(ii+1)%(len(nlist))]),
+ '--nz', str(nlist[(ii+2)%(len(nlist))]),
+ '--Lx', str(2+np.random.random()),
+ '--Ly', str(2+np.random.random()),
+ '--Lz', str(2+np.random.random()),
+ '--simname', 'test_Parseval_{0}'.format(ii),
+ '--np', '4',
+ '--ntpp', '1',
+ '--niter_todo', '{0}'.format(niterations),
+ '--niter_out', '{0}'.format(niterations),
+ '--niter_stat', '1',
+ '--wd', './'] +
+ sys.argv[1:])
+ c.compute_statistics()
+ Parseval_error = np.abs((c.statistics['energy(t)'] - c.statistics['renergy(t)']) / c.statistics['renergy(t)'])
+ assert(np.max(Parseval_error) < 1e-6)
+ print('SUCCESS!!! Parseval test passed for unequal nx, ny, nz and random Lx, Ly, Lz')
+ return None
+
+if __name__ == '__main__':
+ main()
+
diff --git a/TurTLE/test/test_bfps_resize.py b/TurTLE/test/test_bfps_resize.py
new file mode 100644
index 0000000000000000000000000000000000000000..c9d07eb7293e72ac442666fdd76195d8e84793e1
--- /dev/null
+++ b/TurTLE/test/test_bfps_resize.py
@@ -0,0 +1,113 @@
+#! /usr/bin/env python
+
+import os
+import numpy as np
+import h5py
+import sys
+
+import TurTLE
+from TurTLE import DNS
+from TurTLE import PP
+
+import matplotlib.pyplot as plt
+import pyfftw
+
+
+def main():
+ niterations = 2
+ c = DNS()
+ c.launch(
+ ['NSVE',
+ '-n', '32',
+ '--src-simname', 'B32p1e4',
+ '--src-wd', TurTLE.data_dir,
+ '--src-iteration', '0',
+ '--simname', 'dns_test',
+ '--np', '4',
+ '--ntpp', '1',
+ '--niter_todo', '{0}'.format(niterations),
+ '--niter_out', '{0}'.format(niterations),
+ '--niter_stat', '1',
+ '--wd', './'] +
+ sys.argv[1:])
+ rr = PP()
+ rr.launch(
+ ['resize',
+ '--simname', 'dns_test',
+ '--new_nx', '64',
+ '--new_ny', '64',
+ '--new_nz', '64',
+ '--new_simname', 'pp_resize_test',
+ '--np', '4',
+ '--ntpp', '1',
+ '--iter0', '0',
+ '--iter1', '{0}'.format(niterations),
+ '--wd', './'] +
+ sys.argv[1:])
+ f0 = h5py.File(c.get_checkpoint_0_fname(), 'r')
+ f1 = h5py.File('pp_resize_test_fields.h5', 'r')
+ d0 = f0['vorticity/complex/0'][...]
+ d1 = f1['vorticity/complex/0'][...]
+ small_kdata = pyfftw.n_byte_align_empty(
+ (32, 32, 17, 3),
+ pyfftw.simd_alignment,
+ dtype = c.ctype)
+ small_rdata = pyfftw.n_byte_align_empty(
+ (32, 32, 32, 3),
+ pyfftw.simd_alignment,
+ dtype = c.rtype)
+ small_plan = pyfftw.FFTW(
+ small_kdata.transpose((1, 0, 2, 3)),
+ small_rdata,
+ axes = (0, 1, 2),
+ direction = 'FFTW_BACKWARD',
+ threads = 4)
+ big_kdata = pyfftw.n_byte_align_empty(
+ (64, 64, 33, 3),
+ pyfftw.simd_alignment,
+ dtype = c.ctype)
+ big_rdata = pyfftw.n_byte_align_empty(
+ (64, 64, 64, 3),
+ pyfftw.simd_alignment,
+ dtype = c.rtype)
+ big_plan = pyfftw.FFTW(
+ big_kdata.transpose((1, 0, 2, 3)),
+ big_rdata,
+ axes = (0, 1, 2),
+ direction = 'FFTW_BACKWARD',
+ threads = 4)
+ small_kdata[:] = d0
+ big_kdata[:] = d1
+ small_plan.execute()
+ big_plan.execute()
+
+ se = np.mean(small_rdata**2, axis = 3)**.5
+ be = np.mean(big_rdata**2, axis = 3)**.5
+
+ f = plt.figure(figsize = (6, 4))
+ a = f.add_subplot(231)
+ a.set_axis_off()
+ a.imshow(se[0])
+ a = f.add_subplot(234)
+ a.set_axis_off()
+ a.imshow(be[0])
+ a = f.add_subplot(232)
+ a.set_axis_off()
+ a.imshow(se[:, 0])
+ a = f.add_subplot(235)
+ a.set_axis_off()
+ a.imshow(be[:, 0])
+ a = f.add_subplot(233)
+ a.set_axis_off()
+ a.imshow(se[:, :, 0])
+ a = f.add_subplot(236)
+ a.set_axis_off()
+ a.imshow(be[:, :, 0])
+ f.tight_layout()
+ f.savefig('resize_test.pdf')
+ plt.close(f)
+ return None
+
+if __name__ == '__main__':
+ main()
+
diff --git a/TurTLE/test/test_collisions.py b/TurTLE/test/test_collisions.py
new file mode 100644
index 0000000000000000000000000000000000000000..cf3afd9c608e7b24a9cf7e498be209e387fcba52
--- /dev/null
+++ b/TurTLE/test/test_collisions.py
@@ -0,0 +1,310 @@
+import os
+import sys
+import argparse
+import getpass
+from math import ceil
+
+import numpy as np
+import h5py
+
+import TurTLE
+import TurTLE._base
+import TurTLE.DNS
+from TurTLE._code import _code
+
+cpp_location = os.path.join(
+ TurTLE.data_dir, 'collisions')
+
+
+
+class ADNS_Stokes_test(TurTLE.DNS):
+ def __init__(
+ self,
+ cylinders_on = False,
+ NSVEcylinders_on = False,
+ **kwargs):
+ self.cylinders_on = cylinders_on
+ self.NSVEcylinders_on = NSVEcylinders_on
+ TurTLE.DNS.__init__(self, **kwargs)
+ self.list_of_particle_codes.append('NSVE_Stokes_growing_subset')
+ self.extra_parameters['NSVE_Stokes_growing_subset'] = {
+ 'stokes_number' : float(0.1),
+ 'stokes_number_evolving' : float(0.1),
+ 'relative_stokes_distr_width' : float(0),
+ 'nb_growing_particles' : int(0)}
+ return None
+ def custom_parameter_update(self):
+ self.extra_parameters['NSVE_Stokes_growing_subset']['tracers0_integration_steps'] = int(0)
+ self.extra_parameters['NSVE_Stokes_growing_subset']['tracers0_neighbours'] = int(3)
+ self.extra_parameters['NSVE_Stokes_growing_subset']['tracers0_smoothness'] = int(2)
+ return None
+ def write_src(
+ self):
+ self.version_message = (
+ '/***********************************************************************\n' +
+ '* this code automatically generated by TurTLE\n' +
+ '* version {0}\n'.format(TurTLE.__version__) +
+ '***********************************************************************/\n\n\n')
+ self.include_list = [
+ '"base.hpp"',
+ '"scope_timer.hpp"',
+ '"fftw_interface.hpp"',
+ '"full_code/main_code.hpp"',
+ '"full_code/NSVEparticles.hpp"',
+ '<cmath>',
+ '<iostream>',
+ '<hdf5.h>',
+ '<string>',
+ '<cstring>',
+ '<fftw3-mpi.h>',
+ '<omp.h>',
+ '<cfenv>',
+ '<cstdlib>']
+ self.main = """
+ int main(int argc, char *argv[])
+ {{
+ bool fpe = (
+ (getenv("BFPS_FPE_OFF") == nullptr) ||
+ (getenv("BFPS_FPE_OFF") != std::string("TRUE")));
+ return main_code< {0} >(argc, argv, fpe);
+ }}
+ """.format(self.dns_type + '<{0}>'.format(self.C_field_dtype))
+ self.includes = '\n'.join(
+ ['#include ' + hh
+ for hh in self.include_list])
+ self.name = 'NSVEparticles_stokes_set'
+ self.dns_type = 'NSVE_Stokes_growing_subset'
+ self.definitions += open(
+ os.path.join(
+ cpp_location, 'p2p_stokes_collisions_growing_subset.hpp'), 'r').read()
+ self.definitions += open(
+ os.path.join(
+ cpp_location, 'stokes_rhs.hpp'), 'r').read()
+ self.definitions += open(
+ os.path.join(
+ cpp_location, 'stokes_collisions_with_background_rhs.hpp'), 'r').read()
+ self.definitions += open(
+ os.path.join(
+ cpp_location, 'NSVE_Stokes_growing_subset.hpp'), 'r').read()
+ self.definitions += open(
+ os.path.join(
+ cpp_location, 'stokes_rhs.cpp'), 'r').read()
+ self.definitions += open(
+ os.path.join(
+ cpp_location, 'stokes_collisions_with_background_rhs.cpp'), 'r').read()
+ self.definitions += open(
+ os.path.join(
+ cpp_location, 'NSVE_Stokes_growing_subset.cpp'), 'r').read()
+ with open(self.name + '.cpp', 'w') as outfile:
+ outfile.write(self.version_message + '\n\n')
+ outfile.write(self.includes + '\n\n')
+ outfile.write(self.definitions + '\n\n')
+ outfile.write(self.main + '\n')
+ self.check_current_vorticity_exists = True
+ return None
+ def generate_tracer_state(
+ self,
+ rseed = None,
+ species = 0,
+ integration_steps = None,
+ ncomponents = 3,
+ opt = None):
+ need_tracer_state = False
+
+ try:
+ source_file = 'not_a_file'
+ source_cp = 0
+ while True:
+ src_file = os.path.join(
+ os.path.realpath(opt.src_work_dir),
+ opt.src_simname + '_checkpoint_{0}.h5'.format(source_cp))
+ f0 = h5py.File(src_file, 'r')
+ if '{0}'.format(opt.src_iteration) in f0['tracers{0}/state'.format(species)].keys():
+ f0.close()
+ break
+ source_cp += 1
+ with h5py.File(self.get_checkpoint_0_fname(), 'a') as data_file:
+
+ self.copy_tracer_state(
+ src_file,
+ 'tracer{0}/state/'.format(species)+'{0}'.format(opt.src_iteration),
+ data_file,
+ 'tracer{0}/state/0'.format(species),
+ ncomponents = ncomponents)
+ print('copy was successful')
+ except: need_tracer_state = True
+
+ if need_tracer_state:
+ try:
+ print('generate tracer state')
+ if type(integration_steps) == type(None):
+ integration_steps = self.NSVEp_extra_parameters['tracers0_integration_steps']
+ if 'tracers{0}_integration_steps'.format(species) in self.parameters.keys():
+ integration_steps = self.parameters['tracers{0}_integration_steps'.format(species)]
+ with h5py.File(self.get_checkpoint_0_fname(), 'a') as data_file:
+ nn = self.parameters['nparticles']
+ if not 'tracers{0}'.format(species) in data_file.keys():
+ data_file.create_group('tracers{0}'.format(species))
+ data_file.create_group('tracers{0}/rhs'.format(species))
+ data_file.create_group('tracers{0}/state'.format(species))
+ data_file['tracers{0}/rhs'.format(species)].create_dataset(
+ '0',
+ shape = (integration_steps, nn, ncomponents,),
+ dtype = np.float)
+ dset = data_file['tracers{0}/state'.format(species)].create_dataset(
+ '0',
+ shape = (nn, ncomponents,),
+ dtype = np.float)
+ if not type(rseed) == type(None):
+ np.random.seed(rseed)
+ cc = int(0)
+ batch_size = int(1e6)
+ def get_random_phases(npoints):
+ return np.random.random(
+ (npoints, 3))*2*np.pi
+ def get_random_versors(npoints):
+ bla = np.random.normal(
+ size = (npoints, 3))
+ bla /= np.sum(bla**2, axis = 1)[:, None]**.5
+ return bla
+ position1 = np.zeros(shape=(nn//2,3))
+ print(position1, nn)
+ position1[:,2]= np.linspace(0, 2*np.pi, nn//2, endpoint= False)
+ position2= np.copy(position1)
+ position2[:,2]+= np.linspace(0, 0.01, nn//2, endpoint= False)
+ position2[:,0]+= 0.05
+ print(position2)
+ positions= np.zeros((nn,3))
+ positions[:nn//2]=position1
+ positions[nn//2:]=position2
+ print(positions)
+ vel1= np.zeros((nn//2,3))
+ vel1[:,0] = 0.05/self.parameters['dt']
+ vel2=-vel1
+ vel= np.zeros((nn,3))
+ vel[:nn//2]=vel1
+ vel[nn//2:]=vel2
+
+
+
+ while nn > 0:
+ if nn>batch_size:
+ dset[cc*batch_size:(cc+1)*batch_size, :3] = positions[cc*batch_size:(cc+1)*batch_size]
+ dset[cc*batch_size:(cc+1)*batch_size, 3:6] = vel[cc*batch_size:(cc+1)*batch_size]
+ if (cc+1)*batch_size<self.parameters['nb_growing_particles']:
+ dset[cc*batch_size:(cc+1)*batch_size, 6] = np.sqrt(self.parameters['injection_rate']/self.parameters['nu'])/self.parameters['stokes_number_evolving']
+ elif (cc)*batch_size<self.parameters['nb_growing_particles']:
+ dset[cc*batch_size:self.parameters['nb_growing_particles'], 6] = np.sqrt(self.parameters['injection_rate']/self.parameters['nu'])/self.parameters['stokes_number_evolving']
+ dset[self.parameters['nb_growing_particles']:(cc+1)*batch_size, 6] = np.sqrt(self.parameters['injection_rate']/self.parameters['nu'])/self.parameters['stokes_number']
+ else:
+ dset[cc*batch_size:(cc+1)*batch_size, 6] = np.sqrt(self.parameters['injection_rate']/self.parameters['nu'])/self.parameters['stokes_number']
+
+ nn -= batch_size
+ else:
+ dset[cc*batch_size:cc*batch_size+nn, :3] = positions[cc*batch_size:cc*batch_size+nn]
+ dset[cc*batch_size:cc*batch_size+nn, 3:6] = vel[cc*batch_size:cc*batch_size+nn]
+ print(positions)
+ if (cc)*batch_size<self.parameters['nb_growing_particles']:
+ dset[cc*batch_size:self.parameters['nb_growing_particles'], 6] = np.sqrt(self.parameters['injection_rate']/self.parameters['nu'])/self.parameters['stokes_number_evolving']
+ dset[self.parameters['nb_growing_particles']:cc*batch_size + nn, 6] = np.sqrt(self.parameters['injection_rate']/self.parameters['nu'])/self.parameters['stokes_number']
+ print(np.sqrt(self.parameters['injection_rate']/self.parameters['nu'])/self.parameters['stokes_number'])
+ else:
+ dset[cc*batch_size:(cc+1)*batch_size, 6] = np.sqrt(self.parameters['injection_rate']/self.parameters['nu'])/self.parameters['stokes_number']
+
+
+ nn = 0
+ cc += 1
+ except Exception as e:
+ print('!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!WARNING')
+ print(e)
+ print('!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!WARNING')
+ return None
+
+ def launch(
+ self,
+ args = [],
+ **kwargs):
+ opt = self.prepare_launch(
+ args = args,
+ extra_parameters = self.extra_parameters['NSVE_Stokes_growing_subset'])
+ if not os.path.exists(self.get_data_file_name()):
+ self.write_par()
+ self.generate_initial_condition(opt = opt)
+ with h5py.File(self.get_particle_file_name(), 'w') as particle_file:
+ particle_file.create_group('tracers0/position')
+ particle_file.create_group('tracers0/velocity')
+ particle_file.create_group('tracers0/velocity_gradient')
+ particle_file.create_group('tracers0/pressure')
+ particle_file.create_group('tracers0/pressure_gradient')
+ particle_file.create_group('tracers0/pressure_Hessian')
+ particle_file.create_group('tracers0/drag_coeff')
+ particle_file.create_group('collisions0')
+ self.generate_tracer_state(integration_steps = self.parameters['tracers0_integration_steps'],
+ rseed = opt.particle_rand_seed,
+ ncomponents = 7)
+ self.run(
+ nb_processes = opt.nb_processes,
+ nb_threads_per_process = opt.nb_threads_per_process,
+ njobs = opt.njobs,
+ hours = opt.minutes // 60,
+ minutes = opt.minutes % 60,
+ no_submit = opt.no_submit,
+ no_debug = opt.no_debug)
+ return None
+
+
+niterations = 1
+niter_out = 1
+nparticles = 10
+nb_growing_particles = int(nparticles//2)
+n = 32
+checkpoints_per_file = 1
+stokes_number=0.1
+niterations_eq = 10
+def main():
+ ####################################################################################
+ ## run c++ test
+ bla2 = ADNS_Stokes_test()
+ bla2.launch([
+ 'NSVE_Stokes_growing_subset',
+ '--simname', 'test_stokes',
+ '-n', '{0}'.format(n),
+ '--stokes_number', '{0}'.format(stokes_number),
+ '--stokes_number_evolving', '{0}'.format(stokes_number),
+ '--np', '4',
+ '--ntpp', '2',
+ '--kMeta', '3.0',
+ '--niter_todo', '{0}'.format(niterations),
+ '--niter_out', '{0}'.format(niter_out),
+ '--niter_stat', '{0}'.format(niter_out),
+ '--nparticles', '{0}'.format(nparticles),
+ '--nb_growing_particles', '{0}'.format(nb_growing_particles),
+ '--tracers0_cutoff', '0.5'])
+ ####################################################################################
+ return None
+
+def check_results():
+ ####################################################################################
+ ## perform sanity check
+ #reading files
+ f = h5py.File('Stokes_alt.h5','r')
+ checkpoint = h5py.File('Stokes_alt_checkpoint_0.h5', 'r')
+
+ #computing droplet radius a
+ drag_coeff = np.sqrt(np.array(f['parameters/injection_rate'])/np.array(f['parameters/nu']))/np.array(f['parameters/stokes_number'])
+ a = np.sqrt((9/2)*0.001*np.array(f['parameters/nu'])/drag_coeff)
+
+ #expected number of collisions where 0.01 is the max offset
+ number_of_collisions = ceil(nparticles*a/0.01)
+ #recorded number of colllisions
+ actual_number_collisions = np.array(f['statistics/collisions/pair_list_1']).shape[0]
+ #number of droplets growing according to collisions
+ number_of_grown_droplets = np.sum(np.array(checkpoint['/tracers0/state/1'])[:,6]<drag_coeff)
+
+ #checks that expected, recorded collisions and number of grown droplets coincides
+ assert(number_of_collisions==actual_number_collisions and actual_number_collisions==number_of_grown_droplets)
+ ####################################################################################
+ return None
+if __name__ == '__main__':
+ main()
+ check_results()
diff --git a/TurTLE/test/test_fftw.py b/TurTLE/test/test_fftw.py
new file mode 100644
index 0000000000000000000000000000000000000000..bb35935a4bad1660fd15893a162efd24ef7e3867
--- /dev/null
+++ b/TurTLE/test/test_fftw.py
@@ -0,0 +1,114 @@
+#! /usr/bin/env python
+
+import numpy as np
+import h5py
+import sys
+
+import TurTLE
+from TurTLE import TEST
+
+try:
+ import matplotlib.pyplot as plt
+except:
+ plt = None
+
+try:
+ import cmocean
+except:
+ cmocean = None
+
+def main():
+ niterations = 10
+ nlist = [16, 32, 48, 24, 64, 12]
+ for ii in range(len(nlist)):
+ c = TEST()
+ c.launch(
+ ['symmetrize_test',
+ '--nx', str(nlist[ii]),
+ '--ny', str(nlist[(ii+1)%(len(nlist))]),
+ '--nz', str(nlist[(ii+2)%(len(nlist))]),
+ '--Lx', str(2+np.random.random()),
+ '--Ly', str(2+np.random.random()),
+ '--Lz', str(2+np.random.random()),
+ '--simname', 'fftw_vs_numpy_{0}'.format(ii),
+ '--np', '4',
+ '--ntpp', '1',
+ '--wd', './'] +
+ sys.argv[1:])
+ df = h5py.File(c.simname + '.h5', 'r')
+ df = h5py.File(c.simname + '_fields.h5', 'r')
+ field0_complex = df['field0/complex/0'][...]
+ field1_complex = df['field1/complex/0'][...]
+ field1_real = df['field1/real/0'][...]
+ npoints = field1_real.size//3
+
+ # first test symmetrize
+ # field0 is subject to TurTLE symmetrize
+ # field1 has additionally been through ift+dft+normalize
+ # ideally they should be identical. otherwise it means
+ # the TurTLE symmetrize is broken.
+ f0 = field0_complex[:, :, 0, :]
+ f1 = field1_complex[:, :, 0, :]
+ diff = np.abs(f0 - f1)
+ ii = np.unravel_index(np.argmax(diff), diff.shape)
+ print('found maximum difference {0} between field0 and field1 at {1}'.format(diff[ii[0], ii[1], ii[2]], ii))
+ assert(np.max(np.abs(diff)) < 1e-5)
+
+ # now compare numpy with fftw
+ np_field1_real = np.fft.irfftn(field1_complex, axes = (0, 1, 2)).transpose(1, 0, 2, 3)
+ L2normr = np.sqrt(np.mean(np.sum(field1_real**2, axis = 3)))
+ np_L2normr = np.sqrt(np.mean(np.sum(np_field1_real**2, axis = 3)))
+ err = np.max(np.abs(field1_real - np_field1_real*npoints)) / L2normr
+ print('found maximum difference {0} between field1_real and np_field1_real'.format(err))
+ assert(err < 1e-5)
+
+ np_field1_complex = np.fft.rfftn(field1_real.transpose(1, 0, 2, 3), axes = (0, 1, 2)) / npoints
+
+ L2norm0 = np.sqrt(np.sum(np.abs(field1_complex[:, :, 0])**2) + 2*np.sum(np.abs(field1_complex[:, :, 1:])**2))
+ L2norm1 = np.sqrt(np.sum(np.abs(np_field1_complex[:, :, 0])**2) + 2*np.sum(np.abs(np_field1_complex[:, :, 1:])**2))
+ err = np.max(np.abs(np_field1_complex - field1_complex)) / L2norm0
+ assert(err < 1e-5)
+
+ err = abs(L2normr - L2norm0) / L2norm0
+ assert(err < 1e-5)
+
+ if not type(plt) == type(None):
+ f = plt.figure()
+ a = f.add_subplot(221)
+ a.imshow(np.log(np.abs(np_field1_complex[:, :, 0, 0])), interpolation = 'nearest')
+ a.set_title('numpy')
+ a = f.add_subplot(222)
+ a.imshow(np.log(np.abs(field1_complex[:, :, 0, 0])), interpolation = 'nearest')
+ a.set_title('TurTLE')
+ if not type(cmocean) == type(None):
+ a = f.add_subplot(223)
+ a.imshow(
+ np.log(np.angle(np_field1_complex[:, :, 0, 0])),
+ interpolation = 'nearest',
+ cmap = cmocean.cm.phase)
+ a = f.add_subplot(224)
+ a.imshow(
+ np.log(np.angle(field1_complex[:, :, 0, 0])),
+ interpolation = 'nearest',
+ cmap = cmocean.cm.phase)
+ f.tight_layout()
+ f.savefig(c.simname + '_complex_slice_kx0.pdf')
+
+ # hard test.
+ # generate random rseed2_field
+ # field0 is rseed2_field through TurTLE symmetrize
+ # field1 is rseed2_field through ift+dft+normalize
+ # compare field0 with field1
+ field0_complex = df['rseed2_field0/complex/0'][...]
+ field1_complex = df['rseed2_field1/complex/0'][...]
+ f0 = field0_complex[:, :, 0, :]
+ f1 = field1_complex[:, :, 0, :]
+ diff = np.abs(f0 - f1)
+ ii = np.unravel_index(np.argmax(diff), diff.shape)
+ print('found maximum difference {0} between rseed2_field0 and rseed2_field1 at {1}'.format(diff[ii[0], ii[1], ii[2]], ii))
+ assert(np.max(np.abs(diff)) < 1e-5)
+ return None
+
+if __name__ == '__main__':
+ main()
+
diff --git a/TurTLE/test/test_interpolation.py b/TurTLE/test/test_interpolation.py
new file mode 100644
index 0000000000000000000000000000000000000000..97bc4bd7a7f349f4edf767e60632b123c756265f
--- /dev/null
+++ b/TurTLE/test/test_interpolation.py
@@ -0,0 +1,54 @@
+#! /usr/bin/env python
+
+import os
+import numpy as np
+import h5py
+import sys
+
+import TurTLE
+from TurTLE import TEST
+
+try:
+ import matplotlib.pyplot as plt
+ matplotlib_on = True
+except ImportError:
+ matplotlib_on = False
+
+
+def main():
+ nparticles = 100
+ c = TEST()
+ c.launch(
+ ['test_interpolation',
+ '-n', '32',
+ '--np', '4',
+ '--ntpp', '1',
+ #'--nparticles', '{0}'.format(nparticles),
+ '--wd', './'] +
+ sys.argv[3:])
+ ifile = h5py.File(
+ 'test_input.h5',
+ 'r')
+ ofile = h5py.File(
+ 'test_output.h5',
+ 'r')
+ pos0 = ifile['tracers0/state/0'][...]
+ pos1 = ofile['tracers0/position/0'][...]
+ assert(np.max(np.abs(pos0-pos1) / np.abs(pos0)) <= 1e-5)
+ vort0 = ofile['tracers0/vorticity/0'][...]
+ vel_gradient = ofile['tracers0/velocity_gradient/0'][...]
+ vort1 = vort0.copy()
+ vort1[:, 0] = vel_gradient[:, 5] - vel_gradient[:, 7]
+ vort1[:, 1] = vel_gradient[:, 6] - vel_gradient[:, 2]
+ vort1[:, 2] = vel_gradient[:, 1] - vel_gradient[:, 3]
+ assert(np.max(np.abs(vort0-vort1) / np.abs(vort0)) <= 1e-5)
+ divergence = vel_gradient[:, 0] + vel_gradient[:, 4] + vel_gradient[:, 8]
+ divergence_error = np.abs(divergence) / (vel_gradient[:, 0]**2 + vel_gradient[:, 1]**2 + vel_gradient[:, 2]**2)**.5
+ print('mean divergence error is ', np.mean(divergence_error))
+ print('maximum divergence error is ', np.max(divergence_error))
+ print('SUCCESS! Interpolated vorticity agrees with vorticity from interpolated velocity gradient.')
+ return None
+
+if __name__ == '__main__':
+ main()
+
diff --git a/TurTLE/test/test_interpolation_methods.py b/TurTLE/test/test_interpolation_methods.py
new file mode 100644
index 0000000000000000000000000000000000000000..50a58328ae3037b053a69a851bac3c6de2b6d428
--- /dev/null
+++ b/TurTLE/test/test_interpolation_methods.py
@@ -0,0 +1,83 @@
+import numpy as np
+import h5py
+import matplotlib.pyplot as plt
+
+from TurTLE import TEST
+
+def main():
+ c = TEST()
+ c.launch([
+ 'test_interpolation_methods'])
+
+ data_file = h5py.File(c.simname + '_particles.h5', 'r')
+
+
+ f = plt.figure(figsize = (9, 9))
+
+ xx = data_file['particle/position/0'][()]
+
+ counter = 1
+ for n, m in [(0, 0),
+ (1, 1),
+ (2, 1),
+ (2, 2),
+ (3, 1),
+ (3, 2),
+ (4, 1),
+ (4, 2),
+ (5, 2)]:
+ phi = data_file['particle/phi_{0}{1}/0'.format(n, m)][()]
+ a = f.add_subplot(330 + counter)
+ a.contour(xx[..., 0], xx[..., 2], phi[..., 0], levels = 50)
+ a.set_title('{0}{1}'.format(n, m))
+ counter += 1
+
+ f.tight_layout()
+ f.savefig('test_interpolation_methods.pdf')
+ plt.close(f)
+
+ f = plt.figure(figsize = (6, 6))
+ a = f.add_subplot(111)
+ err_mean_1 = []
+ err_max_1 = []
+ n = 1
+ phi0 = data_file['particle/phi_{0}{1}/0'.format(n , 1)][()]
+ phi1 = data_file['particle/phi_{0}{1}/0'.format(n+1, 1)][()]
+ err = np.abs(phi0 - phi1)
+ err_mean_1.append(err.mean())
+ err_max_1.append(err.max())
+ err_mean_2 = []
+ err_max_2 = []
+ for n in [2, 3, 4]:
+ phi0 = data_file['particle/phi_{0}{1}/0'.format(n , 1)][()]
+ phi1 = data_file['particle/phi_{0}{1}/0'.format(n+1, 1)][()]
+ err = np.abs(phi0 - phi1)
+ err_mean_1.append(err.mean())
+ err_max_1.append(err.max())
+ phi0 = data_file['particle/phi_{0}{1}/0'.format(n , 2)][()]
+ phi1 = data_file['particle/phi_{0}{1}/0'.format(n+1, 2)][()]
+ err = np.abs(phi0 - phi1)
+ err_mean_2.append(err.mean())
+ err_max_2.append(err.max())
+ neighbour_list = np.array([1, 2, 3, 4]).astype(np.float)
+ a.plot(neighbour_list, err_mean_1, marker = '.', label = 'm=1, mean')
+ a.plot(neighbour_list, err_max_1, marker = '.', label = 'm=1, max')
+ a.plot(neighbour_list, 1e-4*neighbour_list**(-4), color = 'black', dashes = (2, 2))
+ neighbour_list = np.array([2, 3, 4]).astype(np.float)
+ a.plot(neighbour_list, err_mean_2, marker = '.', label = 'm=2, mean')
+ a.plot(neighbour_list, err_max_2, marker = '.', label = 'm=2, max')
+
+ a.set_xscale('log')
+ a.set_yscale('log')
+ a.legend(loc = 'best')
+
+ f.tight_layout()
+ f.savefig('test_interpolation_methods_err.pdf')
+ plt.close(f)
+
+ data_file.close()
+ return None
+
+if __name__ == '__main__':
+ main()
+
diff --git a/TurTLE/test/test_parameter_copy.py b/TurTLE/test/test_parameter_copy.py
new file mode 100644
index 0000000000000000000000000000000000000000..4fdf2eba44fa8451b281f064a91bb76a465f139d
--- /dev/null
+++ b/TurTLE/test/test_parameter_copy.py
@@ -0,0 +1,69 @@
+import os
+import h5py
+import random
+
+from TurTLE import DNS
+
+def main():
+ for simulation_type in ['NSVE', 'NSE']:
+ # generate random parameters
+ # this list should correspond to the list output by the message "DNS will use physical parameters from source simulation."
+ # as of right now, line 1158 of TurTLE/DNS.py
+ random_parameters = [
+ '--energy', '{0}'.format(random.random()),
+ '--famplitude', '{0}'.format(random.random()),
+ '--fk0', '{0}'.format(random.random()*4),
+ '--fk1', '{0}'.format(4 + random.random()*4),
+ '--fmode', '{0}'.format(1+int(random.random()*3)),
+ '--friction_coefficient', '{0}'.format(random.random()),
+ '--injection_rate', '{0}'.format(random.random()),
+ '--nu', '{0}'.format(random.random()),
+ '--forcing_type', 'fixed_energy']
+
+ # generate parent simulation with the above parameters
+ c = DNS()
+ c.launch([
+ simulation_type,
+ '-n', '48',
+ '--wd', 'test_parameter_creation',
+ '--simname', 'test_' + simulation_type + '_source',
+ '--niter_todo', '1',
+ '--niter_stat', '1',
+ '--niter_out', '1',
+ '--no-submit',
+ ] + random_parameters)
+
+ # generate child simulation
+ c = DNS()
+ c.launch([
+ simulation_type,
+ '-n', '48',
+ '--wd', 'test_parameter_creation',
+ '--simname', 'test_' + simulation_type + '_destination',
+ '--niter_todo', '1',
+ '--niter_stat', '1',
+ '--niter_out', '1',
+ '--src-simname', 'test_' + simulation_type + '_source',
+ '--src-iteration', '0',
+ '--no-submit',
+ ])
+
+ # compare parameters between the two simulations
+ f1 = h5py.File(os.path.join(
+ 'test_parameter_creation',
+ 'test_' + simulation_type + '_source.h5'), 'r')
+ f2 = h5py.File(os.path.join(
+ 'test_parameter_creation',
+ 'test_' + simulation_type + '_destination.h5'), 'r')
+
+ # list MUST agree with parameters from "random_parameters"
+ for kk in ['energy', 'famplitude', 'fk0', 'fk1', 'fmode', 'friction_coefficient', 'injection_rate', 'nu']:
+ v1 = f1['parameters/' + kk][()]
+ v2 = f2['parameters/' + kk][()]
+ print(kk, v1, v2)
+ assert(v1 == v2)
+ return None
+
+if __name__ == '__main__':
+ main()
+
diff --git a/TurTLE/test/test_particle_clouds.py b/TurTLE/test/test_particle_clouds.py
new file mode 100644
index 0000000000000000000000000000000000000000..17e63175298d79c698cf7afcd0442e35297c72f1
--- /dev/null
+++ b/TurTLE/test/test_particle_clouds.py
@@ -0,0 +1,143 @@
+#! /usr/bin/env python
+#######################################################################
+# #
+# Copyright 2019 Max Planck Institute #
+# for Dynamics and Self-Organization #
+# #
+# This file is part of TurTLE. #
+# #
+# TurTLE is free software: you can redistribute it and/or modify #
+# it under the terms of the GNU General Public License as published #
+# by the Free Software Foundation, either version 3 of the License, #
+# or (at your option) any later version. #
+# #
+# TurTLE is distributed in the hope that it will be useful, #
+# but WITHOUT ANY WARRANTY; without even the implied warranty of #
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the #
+# GNU General Public License for more details. #
+# #
+# You should have received a copy of the GNU General Public License #
+# along with TurTLE. If not, see <http://www.gnu.org/licenses/> #
+# #
+# Contact: Cristian.Lalescu@ds.mpg.de #
+# #
+#######################################################################
+
+
+
+import os
+import numpy as np
+import h5py
+import sys
+
+import TurTLE
+from TurTLE import DNS
+
+def basic_test():
+ nclouds = 10
+ nparticles_per_cloud = 1000
+ nparticles = nclouds*nparticles_per_cloud
+ niterations = 32
+ c = DNS()
+ c.dns_type = 'NSVEparticles'
+ c.simname = 'basic_cloud_test'
+ f0 = h5py.File(
+ os.path.join(
+ TurTLE.data_dir,
+ 'B32p1e4_checkpoint_0.h5'),
+ 'r')
+ ic_file = h5py.File(c.get_checkpoint_0_fname(), 'a')
+ ic_file['tracers0/state/0'] = f0['tracers0/state/0'][...].reshape(nclouds, nparticles_per_cloud, 3)
+ ic_file['tracers0/rhs/0'] = f0['tracers0/rhs/0'][...].reshape(4, nclouds, nparticles_per_cloud, 3)
+ ic_file.close()
+ c.launch(
+ ['NSVEparticles',
+ '-n', '32',
+ '--src-simname', 'B32p1e4',
+ '--forcing_type', 'linear',
+ '--src-wd', TurTLE.data_dir,
+ '--src-iteration', '0',
+ '--simname', c.simname,
+ '--np', '4',
+ '--ntpp', '1',
+ '--fftw_plan_rigor', 'FFTW_PATIENT',
+ '--niter_todo', '{0}'.format(niterations),
+ '--niter_out', '{0}'.format(niterations),
+ '--niter_stat', '1',
+ '--checkpoints_per_file', '{0}'.format(3),
+ '--nparticles', '{0}'.format(nparticles),
+ '--njobs', '2',
+ '--wd', './'])
+ f0 = h5py.File(
+ os.path.join(
+ TurTLE.data_dir,
+ 'B32p1e4_checkpoint_0.h5'),
+ 'r')
+ f1 = h5py.File(c.get_checkpoint_0_fname(), 'r')
+ for iteration in [0, 32, 64]:
+ field0 = f0['vorticity/complex/{0}'.format(iteration)][...]
+ field1 = f1['vorticity/complex/{0}'.format(iteration)][...]
+ field_error = np.max(np.abs(field0 - field1))
+ x0 = f0['tracers0/state/{0}'.format(iteration)][...]
+ x1 = f1['tracers0/state/{0}'.format(iteration)][...].reshape(x0.shape)
+ traj_error = np.max(np.abs(x0 - x1))
+ print(traj_error)
+ y0 = f0['tracers0/rhs/{0}'.format(iteration)][...]
+ y1 = f1['tracers0/rhs/{0}'.format(iteration)][...].reshape(y0.shape)
+ rhs_error = np.max(np.abs(y0 - y1))
+ assert(field_error < 1e-5)
+ assert(traj_error < 1e-5)
+ assert(rhs_error < 1e-5)
+ print('SUCCESS! Basic test passed.')
+ return None
+
+def nasty_test():
+ nclouds = 10
+ nparticles_per_cloud = 1000000
+ nparticles = nclouds*nparticles_per_cloud
+ niterations = 8
+ c = DNS()
+ c.dns_type = 'NSVEparticles'
+ c.simname = 'nasty_cloud_test'
+ c.parameters['nparticles'] = nparticles
+ c.parameters['tracers1_integration_steps'] = 4
+ c.generate_tracer_state(rseed = 2, species = 1)
+ del c.parameters['nparticles']
+ del c.parameters['tracers1_integration_steps']
+ ic_file = h5py.File(c.get_checkpoint_0_fname(), 'a')
+ ic_file['tracers0/state/0'] = ic_file['tracers1/state/0'][...].reshape(nclouds, nparticles_per_cloud, 3)
+ ic_file['tracers0/rhs/0'] = ic_file['tracers1/rhs/0'][...].reshape(4, nclouds, nparticles_per_cloud, 3)
+ # put all particles in single z cell
+ ic_file['tracers0/state/0'][..., 2] = 0.0001
+ # put one cloud on another process
+ ic_file['tracers0/state/0'][1, :, 2] = np.pi + 0.0001
+ ic_file.close()
+ c.launch(
+ ['NSVEparticles',
+ '-n', '8',
+ '--src-simname', 'B32p1e4',
+ '--simname', c.simname,
+ '--forcing_type', 'linear',
+ '--src-wd', TurTLE.data_dir,
+ '--src-iteration', '0',
+ '--np', '4',
+ '--ntpp', '2',
+ '--fftw_plan_rigor', 'FFTW_PATIENT',
+ '--niter_todo', '{0}'.format(niterations),
+ '--niter_out', '{0}'.format(niterations),
+ '--niter_stat', '1',
+ '--nparticles', '{0}'.format(nparticles),
+ '--njobs', '1',
+ '--wd', './'])
+ print('SUCCESS! Nasty test passed.')
+ return None
+
+
+def main():
+ basic_test()
+ nasty_test()
+ return None
+
+if __name__ == '__main__':
+ main()
+
diff --git a/TurTLE/test/test_particle_deleter.py b/TurTLE/test/test_particle_deleter.py
new file mode 100644
index 0000000000000000000000000000000000000000..af8fb85170bb6ca76aa856ab09dc1ff7a22c59a8
--- /dev/null
+++ b/TurTLE/test/test_particle_deleter.py
@@ -0,0 +1,176 @@
+import os
+import sys
+import argparse
+import getpass
+
+import numpy as np
+import h5py
+
+import TurTLE
+import TurTLE._base
+import TurTLE.DNS
+from TurTLE._code import _code
+
+cpp_location = os.path.join(
+ TurTLE.data_dir, 'particle_set')
+
+
+class ADNS(TurTLE.DNS):
+ def __init__(
+ self,
+ **kwargs):
+ TurTLE.DNS.__init__(self, **kwargs)
+ self.list_of_particle_codes.append('particle_deleter')
+ self.extra_parameters['particle_deleter'] = {}
+ return None
+ def custom_parameter_update(self):
+ self.parameters['niter_out'] = int(1)
+ self.extra_parameters['particle_deleter']['nparticles'] = int(100)
+ self.extra_parameters['particle_deleter']['tracers0_integration_steps'] = int(0)
+ self.extra_parameters['particle_deleter']['tracers0_neighbours'] = int(3)
+ self.extra_parameters['particle_deleter']['tracers0_smoothness'] = int(2)
+ return None
+ def write_src(
+ self):
+ self.version_message = (
+ '/***********************************************************************\n' +
+ '* this code automatically generated by TurTLE\n' +
+ '* version {0}\n'.format(TurTLE.__version__) +
+ '***********************************************************************/\n\n\n')
+ self.include_list = [
+ '"base.hpp"',
+ '"scope_timer.hpp"',
+ '"fftw_interface.hpp"',
+ '"full_code/main_code.hpp"',
+ '"full_code/NSVEparticles.hpp"',
+ '<cmath>',
+ '<iostream>',
+ '<hdf5.h>',
+ '<string>',
+ '<cstring>',
+ '<fftw3-mpi.h>',
+ '<omp.h>',
+ '<cfenv>',
+ '<cstdlib>']
+ self.main = """
+ int main(int argc, char *argv[])
+ {{
+ bool fpe = (
+ (getenv("BFPS_FPE_OFF") == nullptr) ||
+ (getenv("BFPS_FPE_OFF") != std::string("TRUE")));
+ return main_code< {0} >(argc, argv, fpe);
+ }}
+ """.format(self.dns_type + '<{0}>'.format(self.C_field_dtype))
+ self.includes = '\n'.join(
+ ['#include ' + hh
+ for hh in self.include_list])
+ self.name = 'particle_deleter'
+ self.dns_type = 'particle_deleter'
+ self.definitions += open(
+ os.path.join(
+ cpp_location, self.dns_type + '.hpp'), 'r').read()
+ self.definitions += open(
+ os.path.join(
+ cpp_location, self.dns_type + '.cpp'), 'r').read()
+ with open(self.name + '.cpp', 'w') as outfile:
+ outfile.write(self.version_message + '\n\n')
+ outfile.write(self.includes + '\n\n')
+ outfile.write(self.definitions + '\n\n')
+ outfile.write(self.main + '\n')
+ self.check_current_vorticity_exists = True
+ return None
+
+ def write_par(
+ self,
+ iter0 = 0):
+ TurTLE.DNS.write_par(self, iter0 = iter0)
+ with h5py.File(self.get_data_file_name(), 'r+') as ofile:
+ #TODO move collision stats to particle stats
+ for i in range(self.parameters['niter_out']):
+ ofile.create_dataset('statistics/collisions/'+str(i),
+ shape=(1,),
+ dtype = np.int64)
+ return None
+
+ def launch(
+ self,
+ args = [],
+ **kwargs):
+ opt = self.prepare_launch(args = args)
+ if not os.path.exists(self.get_data_file_name()):
+ self.write_par()
+ self.generate_initial_condition(opt = opt)
+ with h5py.File(self.get_particle_file_name(), 'w') as particle_file:
+ particle_file.create_group('tracers0/position')
+ particle_file.create_group('tracers0/velocity')
+ particle_file.create_group('collisions0')
+ with h5py.File('particle_index_before.h5', 'w') as particle_file:
+ particle_file.create_group('tracers0/index')
+ with h5py.File('particle_index_after.h5', 'w') as particle_file:
+ particle_file.create_group('tracers0/index')
+ with h5py.File('particle_subset_index.h5', 'w') as particle_file:
+ particle_file.create_group('tracers_subset/index')
+ self.generate_tracer_state(integration_steps = self.parameters['tracers0_integration_steps'],
+ rseed = opt.particle_rand_seed,
+ ncomponents = 3)
+ self.run(
+ nb_processes = opt.nb_processes,
+ nb_threads_per_process = opt.nb_threads_per_process,
+ njobs = opt.njobs,
+ hours = opt.minutes // 60,
+ minutes = opt.minutes % 60,
+ no_submit = opt.no_submit,
+ no_debug = opt.no_debug)
+ return None
+
+def main():
+ #####################################################################
+ ## run c++ test
+ bla = ADNS()
+ bla.launch(sys.argv[1:])
+ #####################################################################
+
+ #####################################################################
+ ## python routine replicating deleting pattern of the test in c++
+ def delete_index(before_py, iteration):
+ index2 = (iteration*3)%len(before_py)
+ index_to_delete = np.array([1, index2])
+ if index2==1:
+ index_to_delete = np.array([1, 3])
+ after_py = np.delete(before_py, index_to_delete)
+ return before_py, after_py
+ #####################################################################
+
+ #####################################################################
+ ## perform sanity check
+ f = h5py.File(bla.simname + '.h5', 'r')
+ p = f['parameters']
+ #initialising list of indeces
+ before_py = np.arange(0,np.array(p['nparticles']))
+ #comparing list of particles for each iteration
+ for i in range(np.array(p['niter_todo'])):
+ #apply python routine
+ before_py, after_py = delete_index(before_py, i)
+ #load turtle indeces
+ before_turtle = h5py.File('particle_index_before.h5','r')
+ after_turtle = h5py.File('particle_index_after.h5','r')
+ subset_turtle = h5py.File('particle_subset_index.h5', 'r')
+ index_before_turtle = np.array(before_turtle['tracers0/index/'+str(i)])
+ index_after_turtle = np.array(after_turtle['tracers0/index/'+str(i)])
+ index_subset = np.array(subset_turtle['tracers_subset/index/'+str(i)])
+ index_before_turtle = index_before_turtle.flatten()
+ index_after_turtle = index_after_turtle.flatten()
+ index_subset = index_subset.flatten()
+ print('testing labels iteration {0}'.format(i))
+ assert(np.all(before_py==index_before_turtle) and
+ np.all(after_py==index_after_turtle))
+ print('testing subset labels iteration {0}'.format(i))
+ assert(np.all(after_py[[2, 4]]==index_subset))
+ if(len(after_py)==2):
+ break
+ before_py = after_py
+ #####################################################################
+ return None
+
+if __name__ == '__main__':
+ main()
diff --git a/TurTLE/test/test_particle_integration.py b/TurTLE/test/test_particle_integration.py
new file mode 100644
index 0000000000000000000000000000000000000000..93bb0f4af35a29174e53e60cbc95490554c1d355
--- /dev/null
+++ b/TurTLE/test/test_particle_integration.py
@@ -0,0 +1,132 @@
+import os
+import numpy as np
+import h5py
+import matplotlib.pyplot as plt
+
+from TurTLE import TEST
+
+def main():
+ c = TEST()
+ if not os.path.exists('test.h5'):
+ c.launch([
+ 'test_particle_integration',
+ '--np', '4',
+ '--ntpp', '3'])
+
+ data_file = h5py.File(c.simname + '_particles.h5', 'r')
+
+ # to figure out reasonable timestep to satisfy "particle shall not move more than one grid cell"
+ #vv = data_file['particle_o2_n1_m1_f1/velocity/0'][()]
+ #print(np.min(vv), np.max(vv))
+
+
+ f = plt.figure(figsize = (9, 9))
+ a = f.add_subplot(111)
+
+ for ff in [1, 2, 4, 8]:
+ gg = data_file['particle_o2_n2_m1_f{0}/position'.format(ff)]
+ xx = read_trajectory(gg, 5, 7)
+ a.plot(xx[:, 0], xx[:, 2], marker = '.')
+ f.tight_layout()
+ f.savefig('test_particle_integration_trajectory.pdf')
+ plt.close(f)
+
+ f = plt.figure(figsize = (9, 9))
+ a = f.add_subplot(111)
+ dt0 = c.get_data_file()['parameters/dt'][()]
+ dtlist = [dt0, dt0/2, dt0/4]
+ for n, m in [(0, 0),
+ (1, 1),
+ (2, 1),
+ (3, 2)]:
+ for oo in [1, 2, 4]:
+ err_list = []
+ for factor in [1, 2, 4]:
+ xx1 = data_file['particle_o{0}_n{1}_m{2}_f{3}/position/{4}'.format(oo, n, m, factor, factor*8)][()]
+ xx2 = data_file['particle_o{0}_n{1}_m{2}_f{3}/position/{4}'.format(oo, n, m, factor*2, factor*16)][()]
+ diff = np.sqrt(np.sum((xx2 - xx1)**2, axis = -1))
+ err_list.append(np.mean(diff))
+ a.plot(dtlist,
+ err_list,
+ label = 'o{0}_n{1}_m{2}'.format(oo, n, m),
+ dashes = (oo, oo),
+ marker = '.')
+ a.plot(dtlist,
+ np.array(dtlist)**2 * (1e-2),
+ color = (0, 0, 0),
+ dashes = (1, 1),
+ label = '$\\propto \\Delta t^2$')
+ a.plot(dtlist,
+ np.array(dtlist)**3 * (1e-2),
+ color = (0, 0, 0),
+ dashes = (2, 1, 2),
+ label = '$\\propto \\Delta t^3$')
+ a.plot(dtlist,
+ np.array(dtlist)**4 * (1e-2),
+ color = (0, 0, 0),
+ dashes = (3, 5, 3),
+ label = '$\\propto \\Delta t^4$')
+ a.legend(loc = 'upper left')
+ a.set_xlabel('$\\Delta t$')
+ a.set_ylabel('mean trajectory error')
+ a.set_xscale('log')
+ a.set_yscale('log')
+ f.tight_layout()
+ f.savefig('test_particle_integration_evdt.pdf')
+ f.savefig('test_particle_integration_evdt.svg')
+ plt.close(f)
+
+ #f = plt.figure(figsize = (6, 6))
+ #a = f.add_subplot(111)
+ #err_mean_1 = []
+ #err_max_1 = []
+ #n = 1
+ #phi0 = data_file['particle/phi_{0}{1}/0'.format(n , 1)][()]
+ #phi1 = data_file['particle/phi_{0}{1}/0'.format(n+1, 1)][()]
+ #err = np.abs(phi0 - phi1)
+ #err_mean_1.append(err.mean())
+ #err_max_1.append(err.max())
+ #err_mean_2 = []
+ #err_max_2 = []
+ #for n in [2, 3, 4]:
+ # phi0 = data_file['particle/phi_{0}{1}/0'.format(n , 1)][()]
+ # phi1 = data_file['particle/phi_{0}{1}/0'.format(n+1, 1)][()]
+ # err = np.abs(phi0 - phi1)
+ # err_mean_1.append(err.mean())
+ # err_max_1.append(err.max())
+ # phi0 = data_file['particle/phi_{0}{1}/0'.format(n , 2)][()]
+ # phi1 = data_file['particle/phi_{0}{1}/0'.format(n+1, 2)][()]
+ # err = np.abs(phi0 - phi1)
+ # err_mean_2.append(err.mean())
+ # err_max_2.append(err.max())
+ #neighbour_list = np.array([1, 2, 3, 4]).astype(np.float)
+ #a.plot(neighbour_list, err_mean_1, marker = '.', label = 'm=1, mean')
+ #a.plot(neighbour_list, err_max_1, marker = '.', label = 'm=1, max')
+ #a.plot(neighbour_list, 1e-4*neighbour_list**(-4), color = 'black', dashes = (2, 2))
+ #neighbour_list = np.array([2, 3, 4]).astype(np.float)
+ #a.plot(neighbour_list, err_mean_2, marker = '.', label = 'm=2, mean')
+ #a.plot(neighbour_list, err_max_2, marker = '.', label = 'm=2, max')
+
+ #a.set_xscale('log')
+ #a.set_yscale('log')
+ #a.legend(loc = 'best')
+
+ #f.tight_layout()
+ #f.savefig('test_interpolation_methods_err.pdf')
+ #plt.close(f)
+
+ #data_file.close()
+ return None
+
+def read_trajectory(group, ix, iz):
+ iter_names = group.keys()
+ iterations = np.sort([int(ii) for ii in iter_names])
+
+ xx = []
+ for ii in iterations:
+ xx.append(group['{0}'.format(ii)][iz, ix])
+ return np.array(xx)
+
+if __name__ == '__main__':
+ main()
+
diff --git a/TurTLE/test/test_particles.py b/TurTLE/test/test_particles.py
new file mode 100644
index 0000000000000000000000000000000000000000..f3afb7f7cbd659311845a6d0c32f00fa9653b022
--- /dev/null
+++ b/TurTLE/test/test_particles.py
@@ -0,0 +1,134 @@
+#! /usr/bin/env python
+
+import os
+import numpy as np
+import h5py
+import sys
+
+import TurTLE
+from TurTLE import DNS
+
+try:
+ import matplotlib.pyplot as plt
+ matplotlib_on = True
+except ImportError:
+ matplotlib_on = False
+
+
+def main():
+ assert(sys.argv[1] in ['p2p_sampling'])
+ assert(sys.argv[2] in ['on', 'off'])
+ niterations = 32
+ nparticles = 1000
+ njobs = 1
+ if sys.argv[2] == 'on':
+ c = DNS()
+ c.launch(
+ ['NSVEcomplex_particles',
+ '-n', '32',
+ '--src-simname', 'B32p1e4',
+ '--src-wd', TurTLE.data_dir,
+ '--src-iteration', '0',
+ '--np', '4',
+ '--ntpp', '1',
+ '--niter_todo', '{0}'.format(niterations),
+ '--niter_out', '{0}'.format(niterations),
+ '--niter_stat', '1',
+ '--checkpoints_per_file', '{0}'.format(3),
+ '--nparticles', '{0}'.format(nparticles),
+ '--particle-rand-seed', '2',
+ '--cpp_random_particles', '0',
+ '--njobs', '{0}'.format(njobs),
+ '--wd', './'] +
+ sys.argv[3:])
+ if sys.argv[1] == 'p2p_sampling':
+ cf = h5py.File(
+ 'test_checkpoint_0.h5',
+ 'r')
+ pf = h5py.File(
+ 'test_particles.h5',
+ 'r')
+ if matplotlib_on:
+ # initial condition:
+ # show a histogram of the orientations
+ f = plt.figure()
+ a = f.add_subplot(111)
+ for iteration in range(1):
+ x = cf['tracers0/state/{0}'.format(iteration)][:, 3:]
+ hist, bins = np.histogram(
+ np.sum(x**2, axis = -1).flatten()**.5,
+ bins = np.linspace(0, 2, 40))
+ bb = (bins[:-1] + bins[1:])/2
+ pp = hist.astype(np.float) / (np.sum(hist) * (bb[1] - bb[0]))
+ a.plot(bb, pp, label = '{0}'.format(iteration))
+ a.legend(loc = 'best')
+ f.tight_layout()
+ f.savefig('orientation_histogram.pdf')
+ plt.close(f)
+ # show a histogram of the positions
+ f = plt.figure()
+ a = f.add_subplot(111)
+ for iteration in range(0, niterations*njobs+1, niterations//2):
+ x = pf['tracers0/position/{0}'.format(iteration)][...]
+ hist, bins = np.histogram(
+ np.sum(x**2, axis = -1).flatten()**.5,
+ bins = 40)
+ bb = (bins[:-1] + bins[1:])/2
+ pp = hist.astype(np.float) / (np.sum(hist) * (bb[1] - bb[0]))
+ a.plot(bb, pp, label = '{0}'.format(iteration))
+ a.legend(loc = 'best')
+ f.tight_layout()
+ f.savefig('position_histogram.pdf')
+ plt.close(f)
+ # show a histogram of the orientations
+ f = plt.figure()
+ a = f.add_subplot(111)
+ for iteration in range(0, niterations*njobs+1, niterations//2):
+ x = pf['tracers0/orientation/{0}'.format(iteration)][...]
+ hist, bins = np.histogram(
+ np.sum(x**2, axis = -1).flatten()**.5,
+ bins = np.linspace(0, 2, 40))
+ bb = (bins[:-1] + bins[1:])/2
+ pp = hist.astype(np.float) / (np.sum(hist) * (bb[1] - bb[0]))
+ a.plot(bb, pp, label = '{0}'.format(iteration))
+ a.legend(loc = 'best')
+ f.tight_layout()
+ f.savefig('orientation_histogram.pdf')
+ plt.close(f)
+ # compared sampled positions with checkpoint positions
+ for iteration in range(0, niterations*njobs+1, niterations):
+ x = pf['tracers0/position/{0}'.format(iteration)][...]
+ s = cf['tracers0/state/{0}'.format(iteration)][...]
+ distance = (np.max(np.abs(x - s[..., :3]) /
+ np.maximum(np.ones(x.shape),
+ np.maximum(np.abs(x),
+ np.abs(s[..., :3])))))
+ assert(distance < 1e-14)
+ x = pf['tracers0/orientation/{0}'.format(iteration)][...]
+ distance = (np.max(np.abs(x - s[..., 3:]) /
+ np.maximum(np.ones(x.shape),
+ np.maximum(np.abs(x),
+ np.abs(s[..., 3:])))))
+ assert(distance < 1e-14)
+ # code relevant when velocity field is 0 everywhere.
+ # we check to see what happens to the orientation of the particles
+ # show a histogram of the orientations
+ f = plt.figure()
+ a = f.add_subplot(111)
+ for iteration in range(0, niterations*njobs+1, niterations//4):
+ x = pf['tracers0/orientation/{0}'.format(iteration)][...]
+ hist, bins = np.histogram(
+ x.flatten(),
+ bins = 100)
+ bb = (bins[:-1] + bins[1:])/2
+ pp = hist.astype(np.float) / (np.sum(hist) * (bb[1] - bb[0]))
+ a.plot(bb, pp, label = '{0}'.format(iteration))
+ a.legend(loc = 'best')
+ f.tight_layout()
+ f.savefig('full_orientation_histogram.pdf')
+ plt.close(f)
+ return None
+
+if __name__ == '__main__':
+ main()
+
diff --git a/TurTLE/test/test_phase_shift.py b/TurTLE/test/test_phase_shift.py
new file mode 100644
index 0000000000000000000000000000000000000000..33bfe6fb16cf326147905ae7593b99611fc9e1aa
--- /dev/null
+++ b/TurTLE/test/test_phase_shift.py
@@ -0,0 +1,53 @@
+#! /usr/bin/env python
+
+import os
+import sys
+import numpy as np
+import h5py
+
+import TurTLE
+from TurTLE import TEST
+
+try:
+ import matplotlib.pyplot as plt
+ import cmocean
+except:
+ plt = None
+
+def main():
+ simname = 'test'
+ if not os.path.exists(simname + '.h5'):
+ c = TEST()
+ opt = c.launch(
+ ['phase_shift_test',
+ '--simname', str(simname),
+ '--random_phase_seed', str(1)] +
+ sys.argv[1:])
+ if type(plt) != type(None):
+ df = h5py.File(simname + '_phase_field.h5', 'r')
+ phi0 = df['random_phase_field/complex/0'][()]
+ phi1 = df['random_phase_field/complex/1'][()]
+ div = df['divergence/real/1'][()]
+ df.close()
+ print('max divergence is', np.max(np.abs(div)))
+ f = plt.figure(figsize = (6, 6))
+ a = f.add_subplot(111)
+ XX = 0
+ #a.imshow(, interpolation = 'none')
+ diff = np.abs(phi0 - phi1)
+ ii = np.unravel_index(np.argmax(diff), diff.shape)
+ print(phi0[15, 12, 16])
+ print(phi1[15, 12, 16])
+ print(np.max(diff), ii, diff[ii], phi0[ii], phi1[ii])
+ a.imshow(np.abs(phi0[:, :, XX]),
+ interpolation = 'none',
+ cmap = cmocean.cm.balance,
+ vmin = -1,
+ vmax = 1)
+ f.tight_layout()
+ f.savefig(simname + '_symmetry_error.pdf')
+ return None
+
+if __name__ == '__main__':
+ main()
+
diff --git a/TurTLE/test/test_turtle_NSE.py b/TurTLE/test/test_turtle_NSE.py
new file mode 100644
index 0000000000000000000000000000000000000000..6fe5633dfa35e2a5ed1acf4ebc5b4000bb8792d2
--- /dev/null
+++ b/TurTLE/test/test_turtle_NSE.py
@@ -0,0 +1,205 @@
+#! /usr/bin/env python
+#######################################################################
+# #
+# Copyright 2021 Max Planck Institute #
+# for Dynamics and Self-Organization #
+# #
+# This file is part of TurTLE. #
+# #
+# TurTLE is free software: you can redistribute it and/or modify #
+# it under the terms of the GNU General Public License as published #
+# by the Free Software Foundation, either version 3 of the License, #
+# or (at your option) any later version. #
+# #
+# TurTLE is distributed in the hope that it will be useful, #
+# but WITHOUT ANY WARRANTY; without even the implied warranty of #
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the #
+# GNU General Public License for more details. #
+# #
+# You should have received a copy of the GNU General Public License #
+# along with TurTLE. If not, see <http://www.gnu.org/licenses/> #
+# #
+# Contact: Cristian.Lalescu@mpcdf.mpg.de #
+# #
+#######################################################################
+
+
+
+import os
+import numpy as np
+import h5py
+import sys
+
+import TurTLE
+from TurTLE import DNS, PP
+
+from TurTLE_addons import NSReader
+
+try:
+ import matplotlib.pyplot as plt
+except:
+ plt = None
+
+
+def main_basic():
+ niterations = 32
+ njobs = 2
+ c0 = DNS()
+ c0.launch(
+ ['NSVE',
+ '-n', '32',
+ '--src-simname', 'B32p1e4',
+ '--src-wd', TurTLE.data_dir,
+ '--src-iteration', '0',
+ '--simname', 'nsve_for_comparison',
+ '--np', '4',
+ '--ntpp', '2',
+ '--fftw_plan_rigor', 'FFTW_PATIENT',
+ '--niter_todo', '{0}'.format(niterations),
+ '--niter_out', '{0}'.format(niterations),
+ '--forcing_type', 'linear',
+ '--niter_stat', '1',
+ '--checkpoints_per_file', '{0}'.format(3),
+ '--njobs', '{0}'.format(njobs),
+ '--wd', './'] +
+ sys.argv[1:])
+ cc = PP()
+ cc.launch(
+ ['get_velocity',
+ '--simname', 'nsve_for_comparison',
+ '--np', '4',
+ '--ntpp', '2',
+ '--iter0', '0',
+ '--iter1', '64'] +
+ sys.argv[1:])
+ c1 = DNS()
+ c1.launch(
+ ['NSE',
+ '-n', '32',
+ '--src-simname', 'nsve_for_comparison',
+ '--src-wd', './',
+ '--src-iteration', '0',
+ '--simname', 'nse_for_comparison',
+ '--np', '4',
+ '--ntpp', '2',
+ '--fftw_plan_rigor', 'FFTW_PATIENT',
+ '--niter_todo', '{0}'.format(niterations),
+ '--niter_out', '{0}'.format(niterations),
+ '--niter_stat', '1',
+ '--forcing_type', 'linear',
+ '--checkpoints_per_file', '{0}'.format(3),
+ '--njobs', '{0}'.format(njobs),
+ '--wd', './'] +
+ sys.argv[1:])
+ f0 = h5py.File(
+ 'nsve_for_comparison.h5', 'r')
+ f1 = h5py.File(
+ 'nse_for_comparison.h5', 'r')
+ if not type(plt) == type(None):
+ e0 = 0.5*f0['statistics/moments/velocity'][:, 2, 3]
+ e1 = 0.5*f1['statistics/moments/velocity'][:, 2, 3]
+ fig = plt.figure()
+ a = fig.add_subplot(111)
+ a.plot(e0)
+ a.plot(e1)
+ fig.tight_layout()
+ fig.savefig('comparison_energy.pdf')
+ plt.close(fig)
+ f0.close()
+ f1.close()
+ f0 = h5py.File(
+ 'nsve_for_comparison_checkpoint_0.h5', 'r')
+ f1 = h5py.File(
+ 'nse_for_comparison_checkpoint_0.h5', 'r')
+ for iteration in [0, 32, 64]:
+ field0 = f0['velocity/complex/{0}'.format(iteration)][...]
+ field1 = f1['velocity/complex/{0}'.format(iteration)][...]
+ field_error = np.max(np.abs(field0 - field1))
+ print(field_error)
+ assert(field_error < 1e-1)
+ f0.close()
+ f1.close()
+ print('SUCCESS! Basic test passed.')
+ return None
+
+def main_long():
+ niterations = 2048
+ njobs = 2
+ c0 = DNS()
+ c0.launch(
+ ['NSVE',
+ '-n', '64',
+ '--src-simname', 'B32p1e4',
+ '--src-wd', TurTLE.data_dir,
+ '--src-iteration', '0',
+ '--simname', 'nsve_for_long_comparison',
+ '--np', '4',
+ '--ntpp', '2',
+ '--fftw_plan_rigor', 'FFTW_PATIENT',
+ '--niter_todo', '{0}'.format(niterations),
+ '--niter_out', '{0}'.format(niterations),
+ '--forcing_type', 'linear',
+ '--niter_stat', '1',
+ '--checkpoints_per_file', '{0}'.format(3),
+ '--njobs', '{0}'.format(njobs),
+ '--wd', './'] +
+ sys.argv[1:])
+ cc = PP()
+ cc.launch(
+ ['get_velocity',
+ '--simname', 'nsve_for_long_comparison',
+ '--np', '4',
+ '--ntpp', '2',
+ '--iter0', '0',
+ '--iter1', '64'] +
+ sys.argv[1:])
+ c1 = DNS()
+ c1.launch(
+ ['NSE',
+ '-n', '64',
+ '--src-simname', 'nsve_for_long_comparison',
+ '--src-wd', './',
+ '--src-iteration', '0',
+ '--simname', 'nse_for_long_comparison',
+ '--np', '4',
+ '--ntpp', '2',
+ '--fftw_plan_rigor', 'FFTW_PATIENT',
+ '--niter_todo', '{0}'.format(niterations),
+ '--niter_out', '{0}'.format(niterations),
+ '--niter_stat', '1',
+ '--forcing_type', 'linear',
+ '--checkpoints_per_file', '{0}'.format(3),
+ '--njobs', '{0}'.format(njobs),
+ '--wd', './'] +
+ sys.argv[1:])
+ c0 = NSReader(simname = 'nsve_for_long_comparison')
+ c1 = NSReader(simname = 'nse_for_long_comparison')
+ c0.do_plots()
+ c1.do_plots()
+ if not type(plt) == type(None):
+ fig = plt.figure()
+ a = fig.add_subplot(211)
+ a.plot(c0.statistics['energy(t)'],
+ label = 'NSVE')
+ a.plot(c1.statistics['energy(t)'],
+ label = 'NSVE')
+ a.legend(loc = 'best')
+ a = fig.add_subplot(212)
+ a.plot(c0.statistics['kshell'],
+ c0.statistics['energy(k)'],
+ label = 'NSVE')
+ a.plot(c1.statistics['kshell'],
+ c1.statistics['energy(k)'],
+ label = 'NSE')
+ a.set_xscale('log')
+ a.set_yscale('log')
+ a.set_ylim(1e-7, 10)
+ fig.tight_layout()
+ fig.savefig('comparison_energy_long.pdf')
+ plt.close(fig)
+ return None
+
+if __name__ == '__main__':
+ main_basic()
+ #main_long()
+
diff --git a/TurTLE/test/test_turtle_NSVE_Stokes_particles.py b/TurTLE/test/test_turtle_NSVE_Stokes_particles.py
new file mode 100644
index 0000000000000000000000000000000000000000..400a0cdb12ba0790343489a8570f9383a16daa80
--- /dev/null
+++ b/TurTLE/test/test_turtle_NSVE_Stokes_particles.py
@@ -0,0 +1,100 @@
+#! /usr/bin/env python
+#######################################################################
+# #
+# Copyright 2019 Max Planck Institute #
+# for Dynamics and Self-Organization #
+# #
+# This file is part of TurTLE. #
+# #
+# TurTLE is free software: you can redistribute it and/or modify #
+# it under the terms of the GNU General Public License as published #
+# by the Free Software Foundation, either version 3 of the License, #
+# or (at your option) any later version. #
+# #
+# TurTLE is distributed in the hope that it will be useful, #
+# but WITHOUT ANY WARRANTY; without even the implied warranty of #
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the #
+# GNU General Public License for more details. #
+# #
+# You should have received a copy of the GNU General Public License #
+# along with TurTLE. If not, see <http://www.gnu.org/licenses/> #
+# #
+# Contact: Cristian.Lalescu@ds.mpg.de #
+# #
+#######################################################################
+
+
+
+import os
+import numpy as np
+import h5py
+import sys
+import matplotlib.pyplot as plt
+
+
+import TurTLE
+from TurTLE import DNS
+
+def theory_exponential(t,initial_value,drag_coeff):
+ return initial_value*np.exp(-t*drag_coeff)
+
+def main():
+ """
+ Run test where the trajectory of initially moving particles with varying drag coefficient in a
+ quiescent flow is compared to the analytically expected exponential. The generated plot shows the accuracy of the corresponding particles for a certain fixed timestep.
+ """
+ drag_coefficients = ['30','20','10','1']
+ niterations = 100
+ nparticles = 1
+ njobs = 1
+ dt=0.01
+ fig = plt.figure()
+ plt.ylabel('$v$')
+ plt.xlabel('$t$')
+ plt.yscale('log')
+ j=0
+ for drag_coeff in drag_coefficients:
+ c = DNS()
+ c.launch(
+ ['NSVE_Stokes_particles',
+ '-n', '32',
+ '--simname', 'quiescent_nsve_stokes_particles_drag'+str(j),
+ '--np', '4',
+ '--ntpp', '1',
+ '--dt', '{0}'.format(dt),
+ '--fftw_plan_rigor', 'FFTW_PATIENT',
+ '--niter_todo', '{0}'.format(niterations),
+ '--niter_out', '{0}'.format(1),
+ '--niter_stat', '1',
+ '--nparticles', '{0}'.format(nparticles),
+ '--initial_field_amplitude', '0',
+ '--initial_particle_vel', '0.05',
+ '--drag_coefficient', drag_coeff,
+ '--famplitude', '0',
+ '--njobs', '{0}'.format(njobs),
+ '--wd', './'] +
+ sys.argv[1:])
+ f = h5py.File('quiescent_nsve_stokes_particles_drag'+str(j)+'_particles.h5', 'r')
+ f2 = h5py.File('quiescent_nsve_stokes_particles_drag'+str(j)+'.h5', 'r')
+ particle_momentum = [np.sqrt(f['tracers0/momentum/'+'{}'.format(i)][0][0]**2
+ +f['tracers0/momentum/'+'{}'.format(i)][0][1]**2
+ +f['tracers0/momentum/'+'{}'.format(i)][0][2]**2)
+ for i in range(niterations)]
+ time_values = np.arange(0, niterations*dt, dt)
+ numerics, = plt.plot(time_values, particle_momentum, alpha = 1, lw=2)
+ plt.plot(time_values,
+ theory_exponential(time_values, f2['parameters/initial_particle_vel'],np.float(drag_coeff)),
+ color='black', ls = '--')
+ plt.plot(0,0,color=numerics.get_color(),label=r'drag coefficient '+drag_coeff)
+ j+=1
+ plt.plot(0,
+ 0,
+ color='black', ls = '--',
+ label='analytic sol.')
+ plt.legend(loc='lower left')
+ plt.savefig('Stokes_quiescent_test.pdf')
+ return None
+
+if __name__ == '__main__':
+ main()
+
diff --git a/TurTLE/test/test_turtle_NSVEparticles.py b/TurTLE/test/test_turtle_NSVEparticles.py
new file mode 100644
index 0000000000000000000000000000000000000000..8d5d614da4d5ebe529f61953bc87d0651221600a
--- /dev/null
+++ b/TurTLE/test/test_turtle_NSVEparticles.py
@@ -0,0 +1,101 @@
+#! /usr/bin/env python
+#######################################################################
+# #
+# Copyright 2019 Max Planck Institute #
+# for Dynamics and Self-Organization #
+# #
+# This file is part of TurTLE. #
+# #
+# TurTLE is free software: you can redistribute it and/or modify #
+# it under the terms of the GNU General Public License as published #
+# by the Free Software Foundation, either version 3 of the License, #
+# or (at your option) any later version. #
+# #
+# TurTLE is distributed in the hope that it will be useful, #
+# but WITHOUT ANY WARRANTY; without even the implied warranty of #
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the #
+# GNU General Public License for more details. #
+# #
+# You should have received a copy of the GNU General Public License #
+# along with TurTLE. If not, see <http://www.gnu.org/licenses/> #
+# #
+# Contact: Cristian.Lalescu@ds.mpg.de #
+# #
+#######################################################################
+
+
+
+import os
+import numpy as np
+import h5py
+import sys
+
+import TurTLE
+from TurTLE import DNS
+
+
+def main():
+ """
+ Run short DNS, test against archived data.
+
+ This test runs two MPI jobs, generating a short DNS from initial
+ conditions stored as python package data.
+ This is a full DNS of Navier-Stokes with particles, and we check that
+ the new fields, trajectories and velocity sampled along trajectories
+ agree to within numerical precision (single precision).
+
+ Implicitly, this checks that the entire compilation mechanism is set
+ up properly, as well as the correctness of the code.
+ 4 MPI processes with one OpenMP thread per process are used by default,
+ but this may be changed with command line parameters as desired.
+ """
+ niterations = 32
+ nparticles = 10000
+ njobs = 2
+ c = DNS()
+ c.launch(
+ ['NSVEparticles',
+ '-n', '32',
+ '--src-simname', 'B32p1e4',
+ '--src-wd', TurTLE.data_dir,
+ '--src-iteration', '0',
+ '--simname', 'dns_nsveparticles',
+ '--np', '4',
+ '--ntpp', '1',
+ '--fftw_plan_rigor', 'FFTW_PATIENT',
+ '--niter_todo', '{0}'.format(niterations),
+ '--niter_out', '{0}'.format(niterations),
+ '--niter_stat', '1',
+ '--checkpoints_per_file', '{0}'.format(3),
+ '--nparticles', '{0}'.format(nparticles),
+ '--particle-rand-seed', '2',
+ '--cpp_random_particles', '0',
+ '--njobs', '{0}'.format(njobs),
+ '--wd', './'] +
+ sys.argv[1:])
+ f0 = h5py.File(
+ os.path.join(
+ TurTLE.data_dir,
+ 'B32p1e4_checkpoint_0.h5'),
+ 'r')
+ f1 = h5py.File(c.get_checkpoint_0_fname(), 'r')
+ for iteration in [0, 32, 64]:
+ field0 = f0['vorticity/complex/{0}'.format(iteration)][...]
+ field1 = f1['vorticity/complex/{0}'.format(iteration)][...]
+ field_error = np.max(np.abs(field0 - field1))
+ x0 = f0['tracers0/state/{0}'.format(iteration)][...]
+ x1 = f1['tracers0/state/{0}'.format(iteration)][...]
+ traj_error = np.max(np.abs(x0 - x1))
+ y0 = f0['tracers0/rhs/{0}'.format(iteration)][...]
+ y1 = f1['tracers0/rhs/{0}'.format(iteration)][...]
+ rhs_error = np.max(np.abs(y0 - y1))
+ print(field_error, traj_error, rhs_error)
+ assert(field_error < 1e-5)
+ assert(traj_error < 1e-5)
+ assert(rhs_error < 1e-5)
+ print('SUCCESS! Basic test passed.')
+ return None
+
+if __name__ == '__main__':
+ main()
+
diff --git a/TurTLE/test/test_write_filtered.py b/TurTLE/test/test_write_filtered.py
new file mode 100644
index 0000000000000000000000000000000000000000..b43fda3222f8ab1a6fe608bc9a81c9e369b66841
--- /dev/null
+++ b/TurTLE/test/test_write_filtered.py
@@ -0,0 +1,45 @@
+#! /usr/bin/env python
+
+import numpy as np
+import h5py
+import sys
+
+import TurTLE
+from TurTLE import TEST
+
+try:
+ import matplotlib.pyplot as plt
+except:
+ plt = None
+
+def main():
+ c = TEST()
+ c.launch(
+ ['write_filtered_test',
+ '--simname', 'write_filtered_test',
+ '--np', '4',
+ '--ntpp', '1',
+ '--wd', './'] +
+ sys.argv[1:])
+ df = h5py.File('data_for_write_filtered.h5', 'r')
+ bla0 = df['scal_field_full/complex/0'][:, 0]
+ bla1 = df['scal_field_z0slice/complex/0'][...]
+ max_dist = np.max(np.abs(bla1 - bla0))
+ print('maximum distance for scalar is ', max_dist)
+ assert(max_dist < 1e-5)
+ bla0 = df['vec_field_full/complex/0'][:, 0]
+ bla1 = df['vec_field_z0slice/complex/0'][...]
+ max_dist = np.max(np.abs(bla1 - bla0))
+ print('maximum distance for vector is ', max_dist)
+ assert(max_dist < 1e-5)
+ bla0 = df['tens_field_full/complex/0'][:, 0]
+ bla1 = df['tens_field_z0slice/complex/0'][...]
+ max_dist = np.max(np.abs(bla1 - bla0))
+ print('maximum distance for tensor is ', max_dist)
+ assert(max_dist < 1e-5)
+ print('SUCCESS!!! z=0 slice agrees between different datasets')
+ return None
+
+if __name__ == '__main__':
+ main()
+
diff --git a/bfps/tools.py b/TurTLE/tools.py
similarity index 84%
rename from bfps/tools.py
rename to TurTLE/tools.py
index 69756ec648409ab52d57930d26b1ab1ca8b942c1..6d3869854a9070eaeca2c1bc8149e38637730df7 100644
--- a/bfps/tools.py
+++ b/TurTLE/tools.py
@@ -1,26 +1,25 @@
-#######################################################################
-# #
-# Copyright 2015 Max Planck Institute #
-# for Dynamics and Self-Organization #
-# #
-# This file is part of bfps. #
-# #
-# bfps is free software: you can redistribute it and/or modify #
-# it under the terms of the GNU General Public License as published #
-# by the Free Software Foundation, either version 3 of the License, #
-# or (at your option) any later version. #
-# #
-# bfps is distributed in the hope that it will be useful, #
-# but WITHOUT ANY WARRANTY; without even the implied warranty of #
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the #
-# GNU General Public License for more details. #
-# #
-# You should have received a copy of the GNU General Public License #
-# along with bfps. If not, see <http://www.gnu.org/licenses/> #
-# #
-# Contact: Cristian.Lalescu@ds.mpg.de #
-# #
-#######################################################################
+################################################################################
+# #
+# Copyright 2015-2019 Max Planck Institute for Dynamics and Self-Organization #
+# #
+# This file is part of TurTLE. #
+# #
+# TurTLE is free software: you can redistribute it and/or modify #
+# it under the terms of the GNU General Public License as published #
+# by the Free Software Foundation, either version 3 of the License, #
+# or (at your option) any later version. #
+# #
+# TurTLE is distributed in the hope that it will be useful, #
+# but WITHOUT ANY WARRANTY; without even the implied warranty of #
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the #
+# GNU General Public License for more details. #
+# #
+# You should have received a copy of the GNU General Public License #
+# along with TurTLE. If not, see <http://www.gnu.org/licenses/> #
+# #
+# Contact: Cristian.Lalescu@ds.mpg.de #
+# #
+################################################################################
@@ -143,6 +142,19 @@ def generate_data_3D(
a[ii] = 0
return a
+
+def generate_random_discontinuous_data_3D(
+ n0, n1, n2,
+ dtype = np.complex128,
+ p = 1.5,
+ amplitude = 0.5):
+ """returns the Fourier representation of a random field.
+ """
+ assert(n0 % 2 == 0 and n1 % 2 == 0 and n2 % 2 == 0)
+ a = np.random.randn(n1, n0, n2)
+ b = np.fft.rfftn(a).astype(dtype)
+ return b
+
def randomize_phases(v):
"""randomize the phases of an FFTW complex field.
@@ -156,6 +168,7 @@ def randomize_phases(v):
:returns: ``v`` with randomized phases (i.e. a Gaussian random field).
"""
phi = np.random.random(v.shape[:3])*(2*np.pi)
+ # TODO should manually set n/2 to 0
phi[0, 0, 0] = 0.0
for ky in range(1, phi.shape[0]//2):
phi[phi.shape[0] - ky, 0, 0] = - phi[ky, 0, 0]
@@ -190,10 +203,10 @@ def padd_with_zeros(
"""
if (type(odtype) == type(None)):
odtype = a.dtype
- assert(a.shape[0] <= n0 and
- a.shape[1] <= n1 and
+ assert(a.shape[0] <= n1 and
+ a.shape[1] <= n0 and
a.shape[2] <= n2//2+1)
- b = np.zeros((n0, n1, n2//2 + 1) + a.shape[3:], dtype = odtype)
+ b = np.zeros((n1, n0, n2//2 + 1) + a.shape[3:], dtype = odtype)
m0 = a.shape[1]
m1 = a.shape[0]
m2 = a.shape[2]
@@ -217,13 +230,8 @@ def get_fornberg_coeffs(
"""compute finite difference coefficients
Compute finite difference coefficients for a generic grid specified
- by ``x``, according to [Fornberg]_.
+ by ``x``, according to [fornberg1988mc]_.
The function is used by :func:`particle_finite_diff_test`.
-
- .. [Fornberg] B. Fornberg,
- *Generation of Finite Difference Formulas on Arbitrarily Spaced Grids*.
- Mathematics of Computation,
- **51:184**, 699-706, 1988
"""
N = len(a) - 1
d = []
@@ -271,10 +279,10 @@ def particle_finite_diff_test(
pf = c.get_particle_file()
group = pf['tracers{0}'.format(species)]
acc_on = 'acceleration' in group.keys()
- pos = group['state'].value
- vel = group['velocity'].value
+ pos = group['state'][...]
+ vel = group['velocity'][...]
if acc_on:
- acc = group['acceleration'].value
+ acc = group['acceleration'][...]
n = m
fc = get_fornberg_coeffs(0, range(-n, n+1))
dt = c.parameters['dt']*c.parameters['niter_part']
@@ -298,7 +306,7 @@ def particle_finite_diff_test(
if interp_name not in pars.keys():
# old format
interp_name = 'tracers{0}_field'.format(species)
- interp_name = pars[interp_name].value
+ interp_name = pars[interp_name][...]
if type(interp_name) == bytes:
if sys.version_info[0] == 3:
interp_name = str(interp_name, 'ASCII')
@@ -306,11 +314,11 @@ def particle_finite_diff_test(
interp_name = str(interp_name)
to_print = (
'steps={0}, interp={1}, neighbours={2}, '.format(
- pars['tracers{0}_integration_steps'.format(species)].value,
- pars[interp_name + '_type'].value,
- pars[interp_name + '_neighbours'].value))
+ pars['tracers{0}_integration_steps'.format(species)][...],
+ pars[interp_name + '_type'][...],
+ pars[interp_name + '_neighbours'][...]))
if 'spline' in interp_name:
- to_print += 'smoothness = {0}, '.format(pars[interp_name + '_smoothness'].value)
+ to_print += 'smoothness = {0}, '.format(pars[interp_name + '_smoothness'][...])
to_print += (
'SNR d1p-vel={0:.3f}'.format(np.mean(snr_vel1)))
if acc_on:
@@ -376,3 +384,30 @@ def particle_finite_diff_test(
plt.close(fig)
return pid
+
+def distribute_cores_evenly(
+ nprocesses = 4,
+ nthreads_per_process = 3,
+ total_number_of_cores = 16):
+ assert(nprocesses*nthreads_per_process <= total_number_of_cores)
+
+ # first, determine how many total cores we can allocate per process
+ max_cores_per_process = total_number_of_cores // nprocesses
+
+ # then spread useful cores evenly throughout the total cores per process
+ # start with no cores allocated
+ single_process_mask = np.zeros(max_cores_per_process, np.bool)
+ # allocate cores evenly
+ skip = max_cores_per_process // nthreads_per_process
+ for t in range(nthreads_per_process):
+ single_process_mask[t*skip] = 1
+
+ single_process_mask = sum(int(single_process_mask[i])*(2**i) for i in range(max_cores_per_process))
+
+ # now create full node mask:
+ all_masks = []
+ for p in range(nprocesses):
+ all_masks.append(single_process_mask*(2**(max_cores_per_process*p)))
+ return all_masks
+
+
diff --git a/bash_setup_template.sh b/bash_setup_template.sh
new file mode 100644
index 0000000000000000000000000000000000000000..863d3ee3aa9e19b468079dacb24d47e5c7115001
--- /dev/null
+++ b/bash_setup_template.sh
@@ -0,0 +1,59 @@
+#! /bin/bash
+
+# Load dependencies here if necessary, e.g. `module load cmake`
+
+# module-dependent variables
+export CC=<CC> # Preferred C compiler, e.g. `gcc`
+export CXX=<CXX> # Preferred C++ compiler, e.g. `g++`
+export MPI_HOME=<MPI_HOME> # MPI installation directory, e.g. ${I_MPI_ROOT}
+export FFTW_ROOT=<FFTW_DIR> # Base directory of FFTW
+export FFTW_DIR=<FFTW_DIR> # Base directory of FFTW
+export HDF5_ROOT=<HDF5_DIR> # Base directory of HDF5
+
+########################################################################
+# OPTIONAL
+#
+# export PINCHECK_ROOT=<PINCHECK_ROOT>
+#
+# If this variable is exported, you can check whether TurTLE
+# MPI processes and OpenMP threads are pinned properly to hardware threads.
+# It is available from https://gitlab.mpcdf.mpg.de/khr/pincheck.
+# The pincheck headers need to be placed under `<PINCHECK_ROOT>/include`.
+########################################################################
+
+########################################################################
+# OPTIONAL
+#
+# export TURTLE_COMPILATION_FLAGS=<COMPILATION_FLAGS>
+#
+# We also recommend setting this variable to optimize compilation
+# For clusters of unknown architecture it helps to log into
+# individual nodes and run the following command:
+# gcc -march=native -Q --help=target
+########################################################################
+
+########################################################################
+# OPTIONAL
+#
+# export FFTW_INCDIR=<FFTW_INCDIR> # Directory containing FFTW header files
+# export FFTW_LIBDIR=<FFTW_LIBDIR> # Directory containing FFTW library files
+# export FFTW_OPENMP_LIBDIR=<FFTW_OMP_LIBDIR> # Directory containing OpenMP FFTW library files (please define if different from FFTW_LIBDIR)
+# export FFTW_MPI_LIBDIR=<FFTW_MPI_LIBDIR> # Directory containing MPI FFTW library files (please define if different from FFTW_LIBDIR)
+#
+# There is at least one recorded case of a cluster where different
+# FFTW libraries (serial, MPI and OpenMP) were located in different
+# folders, hence the options of specifying the details.
+# See also lines 264 through 282 of `CMakeLists.txt`.
+########################################################################
+
+# Turtle installation directory
+export TURTLE_ROOT=<TURTLE_DIR>
+# (library will go under ${TURTLE_ROOT}/lib, headers under ${TURTLE_ROOT}/include, etc.)
+
+# location-dependent variables
+export CUSTOM_INSTALL_PATH=${TURTLE_ROOT}
+export PATH=${CUSTOM_INSTALL_PATH}/bin:${PATH}
+export CMAKE_PREFIX_PATH=${CUSTOM_INSTALL_PATH}/lib
+
+# activate python virtual environment
+source ${TURTLE_ROOT}/bin/activate
diff --git a/bfps/DNS.py b/bfps/DNS.py
deleted file mode 100644
index 4f26b86c5d4739e1bb3989f2e4a7d9a70ad3f009..0000000000000000000000000000000000000000
--- a/bfps/DNS.py
+++ /dev/null
@@ -1,930 +0,0 @@
-#######################################################################
-# #
-# Copyright 2015 Max Planck Institute #
-# for Dynamics and Self-Organization #
-# #
-# This file is part of bfps. #
-# #
-# bfps is free software: you can redistribute it and/or modify #
-# it under the terms of the GNU General Public License as published #
-# by the Free Software Foundation, either version 3 of the License, #
-# or (at your option) any later version. #
-# #
-# bfps is distributed in the hope that it will be useful, #
-# but WITHOUT ANY WARRANTY; without even the implied warranty of #
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the #
-# GNU General Public License for more details. #
-# #
-# You should have received a copy of the GNU General Public License #
-# along with bfps. If not, see <http://www.gnu.org/licenses/> #
-# #
-# Contact: Cristian.Lalescu@ds.mpg.de #
-# #
-#######################################################################
-
-
-
-import os
-import sys
-import shutil
-import subprocess
-import argparse
-import h5py
-import math
-import numpy as np
-import warnings
-
-import bfps
-from ._code import _code
-from bfps import tools
-
-class DNS(_code):
- """This class is meant to stitch together the C++ code into a final source file,
- compile it, and handle all job launching.
- """
- def __init__(
- self,
- work_dir = './',
- simname = 'test'):
- _code.__init__(
- self,
- work_dir = work_dir,
- simname = simname)
- self.host_info = {'type' : 'cluster',
- 'environment' : None,
- 'deltanprocs' : 1,
- 'queue' : '',
- 'mail_address': '',
- 'mail_events' : None}
- self.generate_default_parameters()
- return None
- def set_precision(
- self,
- fluid_dtype):
- if fluid_dtype in [np.float32, np.float64]:
- self.fluid_dtype = fluid_dtype
- elif fluid_dtype in ['single', 'double']:
- if fluid_dtype == 'single':
- self.fluid_dtype = np.dtype(np.float32)
- elif fluid_dtype == 'double':
- self.fluid_dtype = np.dtype(np.float64)
- self.rtype = self.fluid_dtype
- if self.rtype == np.float32:
- self.ctype = np.dtype(np.complex64)
- self.C_field_dtype = 'float'
- self.fluid_precision = 'single'
- elif self.rtype == np.float64:
- self.ctype = np.dtype(np.complex128)
- self.C_field_dtype = 'double'
- self.fluid_precision = 'double'
- return None
- def write_src(self):
- self.version_message = (
- '/***********************************************************************\n' +
- '* this code automatically generated by bfps\n' +
- '* version {0}\n'.format(bfps.__version__) +
- '***********************************************************************/\n\n\n')
- self.include_list = [
- '"base.hpp"',
- '"scope_timer.hpp"',
- '"fftw_interface.hpp"',
- '"full_code/main_code.hpp"',
- '<cmath>',
- '<iostream>',
- '<hdf5.h>',
- '<string>',
- '<cstring>',
- '<fftw3-mpi.h>',
- '<omp.h>',
- '<cfenv>',
- '<cstdlib>',
- '"full_code/{0}.hpp"\n'.format(self.dns_type)]
- self.main = """
- int main(int argc, char *argv[])
- {{
- bool fpe = (
- (getenv("BFPS_FPE_OFF") == nullptr) ||
- (getenv("BFPS_FPE_OFF") != std::string("TRUE")));
- return main_code< {0} >(argc, argv, fpe);
- }}
- """.format(self.dns_type + '<{0}>'.format(self.C_field_dtype))
- self.includes = '\n'.join(
- ['#include ' + hh
- for hh in self.include_list])
- with open(self.name + '.cpp', 'w') as outfile:
- outfile.write(self.version_message + '\n\n')
- outfile.write(self.includes + '\n\n')
- outfile.write(
- self.cread_pars(
- template_class = '{0}<rnumber>::'.format(self.dns_type),
- template_prefix = 'template <typename rnumber> ',
- simname_variable = 'this->simname.c_str()',
- prepend_this = True) +
- '\n\n')
- for rnumber in ['float', 'double']:
- outfile.write(self.cread_pars(
- template_class = '{0}<{1}>::'.format(self.dns_type, rnumber),
- template_prefix = 'template '.format(rnumber),
- just_declaration = True) + '\n\n')
- if self.dns_type in ['NSVEparticles', 'NSVE_no_output', 'NSVEparticles_no_output']:
- outfile.write('template <typename rnumber> int NSVE<rnumber>::read_parameters(){return EXIT_SUCCESS;}\n')
- outfile.write('template int NSVE<float>::read_parameters();\n')
- outfile.write('template int NSVE<double>::read_parameters();\n\n')
- if self.dns_type in ['NSVEparticles_no_output']:
- outfile.write('template <typename rnumber> int NSVEparticles<rnumber>::read_parameters(){return EXIT_SUCCESS;}\n')
- outfile.write('template int NSVEparticles<float>::read_parameters();\n')
- outfile.write('template int NSVEparticles<double>::read_parameters();\n\n')
- outfile.write(self.main + '\n')
- return None
- def generate_default_parameters(self):
- # these parameters are relevant for all DNS classes
- self.parameters['dealias_type'] = int(1)
- self.parameters['dkx'] = float(1.0)
- self.parameters['dky'] = float(1.0)
- self.parameters['dkz'] = float(1.0)
- self.parameters['niter_todo'] = int(8)
- self.parameters['niter_stat'] = int(1)
- self.parameters['niter_out'] = int(8)
- self.parameters['checkpoints_per_file'] = int(1)
- self.parameters['dt'] = float(0.01)
- self.parameters['nu'] = float(0.1)
- self.parameters['fmode'] = int(1)
- self.parameters['famplitude'] = float(0.5)
- self.parameters['fk0'] = float(2.0)
- self.parameters['fk1'] = float(4.0)
- self.parameters['forcing_type'] = 'linear'
- self.parameters['histogram_bins'] = int(256)
- self.parameters['max_velocity_estimate'] = float(1)
- self.parameters['max_vorticity_estimate'] = float(1)
- # parameters specific to particle version
- self.NSVEp_extra_parameters = {}
- self.NSVEp_extra_parameters['niter_part'] = int(1)
- self.NSVEp_extra_parameters['nparticles'] = int(10)
- self.NSVEp_extra_parameters['tracers0_integration_steps'] = int(4)
- self.NSVEp_extra_parameters['tracers0_neighbours'] = int(1)
- self.NSVEp_extra_parameters['tracers0_smoothness'] = int(1)
- return None
- def get_kspace(self):
- kspace = {}
- if self.parameters['dealias_type'] == 1:
- kMx = self.parameters['dkx']*(self.parameters['nx']//2 - 1)
- kMy = self.parameters['dky']*(self.parameters['ny']//2 - 1)
- kMz = self.parameters['dkz']*(self.parameters['nz']//2 - 1)
- else:
- kMx = self.parameters['dkx']*(self.parameters['nx']//3 - 1)
- kMy = self.parameters['dky']*(self.parameters['ny']//3 - 1)
- kMz = self.parameters['dkz']*(self.parameters['nz']//3 - 1)
- kspace['kM'] = max(kMx, kMy, kMz)
- kspace['dk'] = min(self.parameters['dkx'],
- self.parameters['dky'],
- self.parameters['dkz'])
- nshells = int(kspace['kM'] / kspace['dk']) + 2
- kspace['nshell'] = np.zeros(nshells, dtype = np.int64)
- kspace['kshell'] = np.zeros(nshells, dtype = np.float64)
- kspace['kx'] = np.arange( 0,
- self.parameters['nx']//2 + 1).astype(np.float64)*self.parameters['dkx']
- kspace['ky'] = np.arange(-self.parameters['ny']//2 + 1,
- self.parameters['ny']//2 + 1).astype(np.float64)*self.parameters['dky']
- kspace['ky'] = np.roll(kspace['ky'], self.parameters['ny']//2+1)
- kspace['kz'] = np.arange(-self.parameters['nz']//2 + 1,
- self.parameters['nz']//2 + 1).astype(np.float64)*self.parameters['dkz']
- kspace['kz'] = np.roll(kspace['kz'], self.parameters['nz']//2+1)
- return kspace
- def get_data_file_name(self):
- return os.path.join(self.work_dir, self.simname + '.h5')
- def get_data_file(self):
- return h5py.File(self.get_data_file_name(), 'r')
- def get_particle_file_name(self):
- return os.path.join(self.work_dir, self.simname + '_particles.h5')
- def get_particle_file(self):
- return h5py.File(self.get_particle_file_name(), 'r')
- def get_postprocess_file_name(self):
- return os.path.join(self.work_dir, self.simname + '_postprocess.h5')
- def get_postprocess_file(self):
- return h5py.File(self.get_postprocess_file_name(), 'r')
- def compute_statistics(self, iter0 = 0, iter1 = None):
- """Run basic postprocessing on raw data.
- The energy spectrum :math:`E(t, k)` and the enstrophy spectrum
- :math:`\\frac{1}{2}\omega^2(t, k)` are computed from the
-
- .. math::
-
- \sum_{k \\leq \\|\\mathbf{k}\\| \\leq k+dk}\\hat{u_i} \\hat{u_j}^*, \\hskip .5cm
- \sum_{k \\leq \\|\\mathbf{k}\\| \\leq k+dk}\\hat{\omega_i} \\hat{\\omega_j}^*
-
- tensors, and the enstrophy spectrum is also used to
- compute the dissipation :math:`\\varepsilon(t)`.
- These basic quantities are stored in a newly created HDF5 file,
- ``simname_postprocess.h5``.
- """
- if len(list(self.statistics.keys())) > 0:
- return None
- self.read_parameters()
- with self.get_data_file() as data_file:
- if 'moments' not in data_file['statistics'].keys():
- return None
- iter0 = min((data_file['statistics/moments/velocity'].shape[0] *
- self.parameters['niter_stat']-1),
- iter0)
- if type(iter1) == type(None):
- iter1 = data_file['iteration'].value
- else:
- iter1 = min(data_file['iteration'].value, iter1)
- ii0 = iter0 // self.parameters['niter_stat']
- ii1 = iter1 // self.parameters['niter_stat']
- self.statistics['kshell'] = data_file['kspace/kshell'].value
- self.statistics['kM'] = data_file['kspace/kM'].value
- self.statistics['dk'] = data_file['kspace/dk'].value
- computation_needed = True
- pp_file = h5py.File(self.get_postprocess_file_name(), 'a')
- if 'ii0' in pp_file.keys():
- computation_needed = not (ii0 == pp_file['ii0'].value and
- ii1 == pp_file['ii1'].value)
- if computation_needed:
- for k in pp_file.keys():
- del pp_file[k]
- if computation_needed:
- pp_file['iter0'] = iter0
- pp_file['iter1'] = iter1
- pp_file['ii0'] = ii0
- pp_file['ii1'] = ii1
- pp_file['t'] = (self.parameters['dt']*
- self.parameters['niter_stat']*
- (np.arange(ii0, ii1+1).astype(np.float)))
- pp_file['energy(t, k)'] = (
- data_file['statistics/spectra/velocity_velocity'][ii0:ii1+1, :, 0, 0] +
- data_file['statistics/spectra/velocity_velocity'][ii0:ii1+1, :, 1, 1] +
- data_file['statistics/spectra/velocity_velocity'][ii0:ii1+1, :, 2, 2])/2
- pp_file['enstrophy(t, k)'] = (
- data_file['statistics/spectra/vorticity_vorticity'][ii0:ii1+1, :, 0, 0] +
- data_file['statistics/spectra/vorticity_vorticity'][ii0:ii1+1, :, 1, 1] +
- data_file['statistics/spectra/vorticity_vorticity'][ii0:ii1+1, :, 2, 2])/2
- pp_file['vel_max(t)'] = data_file['statistics/moments/velocity'] [ii0:ii1+1, 9, 3]
- pp_file['renergy(t)'] = data_file['statistics/moments/velocity'][ii0:ii1+1, 2, 3]/2
- for k in ['t',
- 'energy(t, k)',
- 'enstrophy(t, k)',
- 'vel_max(t)',
- 'renergy(t)']:
- if k in pp_file.keys():
- self.statistics[k] = pp_file[k].value
- self.compute_time_averages()
- return None
- def compute_time_averages(self):
- """Compute easy stats.
-
- Further computation of statistics based on the contents of
- ``simname_postprocess.h5``.
- Standard quantities are as follows
- (consistent with [Ishihara]_):
-
- .. math::
-
- U_{\\textrm{int}}(t) = \\sqrt{\\frac{2E(t)}{3}}, \\hskip .5cm
- L_{\\textrm{int}}(t) = \\frac{\pi}{2U_{int}^2(t)} \\int \\frac{dk}{k} E(t, k), \\hskip .5cm
- T_{\\textrm{int}}(t) =
- \\frac{L_{\\textrm{int}}(t)}{U_{\\textrm{int}}(t)}
-
- \\eta_K = \\left(\\frac{\\nu^3}{\\varepsilon}\\right)^{1/4}, \\hskip .5cm
- \\tau_K = \\left(\\frac{\\nu}{\\varepsilon}\\right)^{1/2}, \\hskip .5cm
- \\lambda = \\sqrt{\\frac{15 \\nu U_{\\textrm{int}}^2}{\\varepsilon}}
-
- Re = \\frac{U_{\\textrm{int}} L_{\\textrm{int}}}{\\nu}, \\hskip
- .5cm
- R_{\\lambda} = \\frac{U_{\\textrm{int}} \\lambda}{\\nu}
-
- .. [Ishihara] T. Ishihara et al,
- *Small-scale statistics in high-resolution direct numerical
- simulation of turbulence: Reynolds number dependence of
- one-point velocity gradient statistics*.
- J. Fluid Mech.,
- **592**, 335-366, 2007
- """
- for key in ['energy', 'enstrophy']:
- self.statistics[key + '(t)'] = (self.statistics['dk'] *
- np.sum(self.statistics[key + '(t, k)'], axis = 1))
- self.statistics['Uint(t)'] = np.sqrt(2*self.statistics['energy(t)'] / 3)
- self.statistics['Lint(t)'] = ((self.statistics['dk']*np.pi /
- (2*self.statistics['Uint(t)']**2)) *
- np.nansum(self.statistics['energy(t, k)'] /
- self.statistics['kshell'][None, :], axis = 1))
- for key in ['energy',
- 'enstrophy',
- 'vel_max',
- 'Uint',
- 'Lint']:
- if key + '(t)' in self.statistics.keys():
- self.statistics[key] = np.average(self.statistics[key + '(t)'], axis = 0)
- for suffix in ['', '(t)']:
- self.statistics['diss' + suffix] = (self.parameters['nu'] *
- self.statistics['enstrophy' + suffix]*2)
- self.statistics['etaK' + suffix] = (self.parameters['nu']**3 /
- self.statistics['diss' + suffix])**.25
- self.statistics['tauK' + suffix] = (self.parameters['nu'] /
- self.statistics['diss' + suffix])**.5
- self.statistics['Re' + suffix] = (self.statistics['Uint' + suffix] *
- self.statistics['Lint' + suffix] /
- self.parameters['nu'])
- self.statistics['lambda' + suffix] = (15 * self.parameters['nu'] *
- self.statistics['Uint' + suffix]**2 /
- self.statistics['diss' + suffix])**.5
- self.statistics['Rlambda' + suffix] = (self.statistics['Uint' + suffix] *
- self.statistics['lambda' + suffix] /
- self.parameters['nu'])
- self.statistics['kMeta' + suffix] = (self.statistics['kM'] *
- self.statistics['etaK' + suffix])
- if self.parameters['dealias_type'] == 1:
- self.statistics['kMeta' + suffix] *= 0.8
- self.statistics['Tint'] = self.statistics['Lint'] / self.statistics['Uint']
- self.statistics['Taylor_microscale'] = self.statistics['lambda']
- return None
- def set_plt_style(
- self,
- style = {'dashes' : (None, None)}):
- self.style.update(style)
- return None
- def convert_complex_from_binary(
- self,
- field_name = 'vorticity',
- iteration = 0,
- file_name = None):
- """read the Fourier representation of a vector field.
-
- Read the binary file containing iteration ``iteration`` of the
- field ``field_name``, and write it in a ``.h5`` file.
- """
- data = np.memmap(
- os.path.join(self.work_dir,
- self.simname + '_{0}_i{1:0>5x}'.format('c' + field_name, iteration)),
- dtype = self.ctype,
- mode = 'r',
- shape = (self.parameters['ny'],
- self.parameters['nz'],
- self.parameters['nx']//2+1,
- 3))
- if type(file_name) == type(None):
- file_name = self.simname + '_{0}_i{1:0>5x}.h5'.format('c' + field_name, iteration)
- file_name = os.path.join(self.work_dir, file_name)
- f = h5py.File(file_name, 'a')
- f[field_name + '/complex/{0}'.format(iteration)] = data
- f.close()
- return None
- def write_par(
- self,
- iter0 = 0,
- particle_ic = None):
- assert (self.parameters['niter_todo'] % self.parameters['niter_stat'] == 0)
- assert (self.parameters['niter_todo'] % self.parameters['niter_out'] == 0)
- assert (self.parameters['niter_out'] % self.parameters['niter_stat'] == 0)
- if self.dns_type in ['NSVEparticles_no_output', 'NSVEparticles']:
- assert (self.parameters['niter_todo'] % self.parameters['niter_part'] == 0)
- assert (self.parameters['niter_out'] % self.parameters['niter_part'] == 0)
- _code.write_par(self, iter0 = iter0)
- with h5py.File(self.get_data_file_name(), 'r+') as ofile:
- ofile['bfps_info/exec_name'] = self.name
- kspace = self.get_kspace()
- for k in kspace.keys():
- ofile['kspace/' + k] = kspace[k]
- nshells = kspace['nshell'].shape[0]
- kspace = self.get_kspace()
- nshells = kspace['nshell'].shape[0]
- vec_stat_datasets = ['velocity', 'vorticity']
- scal_stat_datasets = []
- for k in vec_stat_datasets:
- time_chunk = 2**20//(8*3*3*nshells)
- time_chunk = max(time_chunk, 1)
- ofile.create_dataset('statistics/spectra/' + k + '_' + k,
- (1, nshells, 3, 3),
- chunks = (time_chunk, nshells, 3, 3),
- maxshape = (None, nshells, 3, 3),
- dtype = np.float64)
- time_chunk = 2**20//(8*4*10)
- time_chunk = max(time_chunk, 1)
- a = ofile.create_dataset('statistics/moments/' + k,
- (1, 10, 4),
- chunks = (time_chunk, 10, 4),
- maxshape = (None, 10, 4),
- dtype = np.float64)
- time_chunk = 2**20//(8*4*self.parameters['histogram_bins'])
- time_chunk = max(time_chunk, 1)
- ofile.create_dataset('statistics/histograms/' + k,
- (1,
- self.parameters['histogram_bins'],
- 4),
- chunks = (time_chunk,
- self.parameters['histogram_bins'],
- 4),
- maxshape = (None,
- self.parameters['histogram_bins'],
- 4),
- dtype = np.int64)
- ofile['checkpoint'] = int(0)
- if self.dns_type in ['NSVE', 'NSVE_no_output']:
- return None
-
- if type(particle_ic) == type(None):
- pbase_shape = (self.parameters['nparticles'],)
- number_of_particles = self.parameters['nparticles']
- else:
- pbase_shape = particle_ic.shape[:-1]
- assert(particle_ic.shape[-1] == 3)
- number_of_particles = 1
- for val in pbase_shape[1:]:
- number_of_particles *= val
- with h5py.File(self.get_checkpoint_0_fname(), 'a') as ofile:
- s = 0
- ofile.create_group('tracers{0}'.format(s))
- ofile.create_group('tracers{0}/rhs'.format(s))
- ofile.create_group('tracers{0}/state'.format(s))
- ofile['tracers{0}/rhs'.format(s)].create_dataset(
- '0',
- shape = (
- (self.parameters['tracers{0}_integration_steps'.format(s)],) +
- pbase_shape +
- (3,)),
- dtype = np.float)
- ofile['tracers{0}/state'.format(s)].create_dataset(
- '0',
- shape = (
- pbase_shape +
- (3,)),
- dtype = np.float)
- return None
- def job_parser_arguments(
- self,
- parser):
- parser.add_argument(
- '--ncpu',
- type = int,
- dest = 'ncpu',
- default = -1)
- parser.add_argument(
- '--np', '--nprocesses',
- metavar = 'NPROCESSES',
- help = 'number of mpi processes to use',
- type = int,
- dest = 'nb_processes',
- default = 4)
- parser.add_argument(
- '--ntpp', '--nthreads-per-process',
- type = int,
- dest = 'nb_threads_per_process',
- metavar = 'NTHREADS_PER_PROCESS',
- help = 'number of threads to use per MPI process',
- default = 1)
- parser.add_argument(
- '--no-submit',
- action = 'store_true',
- dest = 'no_submit')
- parser.add_argument(
- '--environment',
- type = str,
- dest = 'environment',
- default = None)
- parser.add_argument(
- '--minutes',
- type = int,
- dest = 'minutes',
- default = 5,
- help = 'If environment supports it, this is the requested wall-clock-limit.')
- parser.add_argument(
- '--njobs',
- type = int, dest = 'njobs',
- default = 1)
- return None
- def simulation_parser_arguments(
- self,
- parser):
- parser.add_argument(
- '--simname',
- type = str, dest = 'simname',
- default = 'test')
- parser.add_argument(
- '-n', '--grid-size',
- type = int,
- dest = 'n',
- default = 32,
- metavar = 'N',
- help = 'code is run by default in a grid of NxNxN')
- for coord in ['x', 'y', 'z']:
- parser.add_argument(
- '--L{0}'.format(coord), '--box-length-{0}'.format(coord),
- type = float,
- dest = 'L{0}'.format(coord),
- default = 2.0,
- metavar = 'length{0}'.format(coord),
- help = 'length of the box in the {0} direction will be `length{0} x pi`'.format(coord))
- parser.add_argument(
- '--wd',
- type = str, dest = 'work_dir',
- default = './')
- parser.add_argument(
- '--precision',
- choices = ['single', 'double'],
- type = str,
- default = 'single')
- parser.add_argument(
- '--src-wd',
- type = str,
- dest = 'src_work_dir',
- default = '')
- parser.add_argument(
- '--src-simname',
- type = str,
- dest = 'src_simname',
- default = '')
- parser.add_argument(
- '--src-iteration',
- type = int,
- dest = 'src_iteration',
- default = 0)
- parser.add_argument(
- '--kMeta',
- type = float,
- dest = 'kMeta',
- default = 2.0)
- parser.add_argument(
- '--dtfactor',
- type = float,
- dest = 'dtfactor',
- default = 0.5,
- help = 'dt is computed as DTFACTOR / N')
- return None
- def particle_parser_arguments(
- self,
- parser):
- parser.add_argument(
- '--particle-rand-seed',
- type = int,
- dest = 'particle_rand_seed',
- default = None)
- parser.add_argument(
- '--pclouds',
- type = int,
- dest = 'pclouds',
- default = 1,
- help = ('number of particle clouds. Particle "clouds" '
- 'consist of particles distributed according to '
- 'pcloud-type.'))
- parser.add_argument(
- '--pcloud-type',
- choices = ['random-cube',
- 'regular-cube'],
- dest = 'pcloud_type',
- default = 'random-cube')
- parser.add_argument(
- '--particle-cloud-size',
- type = float,
- dest = 'particle_cloud_size',
- default = 2*np.pi)
- return None
- def add_parser_arguments(
- self,
- parser):
- subparsers = parser.add_subparsers(
- dest = 'DNS_class',
- help = 'type of simulation to run')
- subparsers.required = True
- parser_NSVE = subparsers.add_parser(
- 'NSVE',
- help = 'plain Navier-Stokes vorticity formulation')
- self.simulation_parser_arguments(parser_NSVE)
- self.job_parser_arguments(parser_NSVE)
- self.parameters_to_parser_arguments(parser_NSVE)
-
- parser_NSVE_no_output = subparsers.add_parser(
- 'NSVE_no_output',
- help = 'plain Navier-Stokes vorticity formulation, checkpoints are NOT SAVED')
- self.simulation_parser_arguments(parser_NSVE_no_output)
- self.job_parser_arguments(parser_NSVE_no_output)
- self.parameters_to_parser_arguments(parser_NSVE_no_output)
-
- parser_NSVEparticles_no_output = subparsers.add_parser(
- 'NSVEparticles_no_output',
- help = 'plain Navier-Stokes vorticity formulation, with basic fluid tracers, checkpoints are NOT SAVED')
- self.simulation_parser_arguments(parser_NSVEparticles_no_output)
- self.job_parser_arguments(parser_NSVEparticles_no_output)
- self.particle_parser_arguments(parser_NSVEparticles_no_output)
- self.parameters_to_parser_arguments(parser_NSVEparticles_no_output)
- self.parameters_to_parser_arguments(
- parser_NSVEparticles_no_output,
- self.NSVEp_extra_parameters)
-
- parser_NSVEp2 = subparsers.add_parser(
- 'NSVEparticles',
- help = 'plain Navier-Stokes vorticity formulation, with basic fluid tracers')
- self.simulation_parser_arguments(parser_NSVEp2)
- self.job_parser_arguments(parser_NSVEp2)
- self.particle_parser_arguments(parser_NSVEp2)
- self.parameters_to_parser_arguments(parser_NSVEp2)
- self.parameters_to_parser_arguments(
- parser_NSVEp2,
- self.NSVEp_extra_parameters)
- return None
- def prepare_launch(
- self,
- args = []):
- """Set up reasonable parameters.
-
- With the default Lundgren forcing applied in the band [2, 4],
- we can estimate the dissipation, therefore we can estimate
- :math:`k_M \\eta_K` and constrain the viscosity.
-
- In brief, the command line parameter :math:`k_M \\eta_K` is
- used in the following formula for :math:`\\nu` (:math:`N` is the
- number of real space grid points per coordinate):
-
- .. math::
-
- \\nu = \\left(\\frac{2 k_M \\eta_K}{N} \\right)^{4/3}
-
- With this choice, the average dissipation :math:`\\varepsilon`
- will be close to 0.4, and the integral scale velocity will be
- close to 0.77, yielding the approximate value for the Taylor
- microscale and corresponding Reynolds number:
-
- .. math::
-
- \\lambda \\approx 4.75\\left(\\frac{2 k_M \\eta_K}{N} \\right)^{4/6}, \\hskip .5in
- R_\\lambda \\approx 3.7 \\left(\\frac{N}{2 k_M \\eta_K} \\right)^{4/6}
-
- """
- opt = _code.prepare_launch(self, args = args)
- self.set_precision(opt.precision)
- self.dns_type = opt.DNS_class
- self.name = self.dns_type + '-' + self.fluid_precision + '-v' + bfps.__version__
- # merge parameters if needed
- if self.dns_type in ['NSVEparticles', 'NSVEparticles_no_output']:
- for k in self.NSVEp_extra_parameters.keys():
- self.parameters[k] = self.NSVEp_extra_parameters[k]
- self.parameters['nu'] = (opt.kMeta * 2 / opt.n)**(4./3)
- self.parameters['dt'] = (opt.dtfactor / opt.n)
- # custom famplitude for 288 and 576
- if opt.n == 288:
- self.parameters['famplitude'] = 0.45
- elif opt.n == 576:
- self.parameters['famplitude'] = 0.47
- if ((self.parameters['niter_todo'] % self.parameters['niter_out']) != 0):
- self.parameters['niter_out'] = self.parameters['niter_todo']
- if len(opt.src_work_dir) == 0:
- opt.src_work_dir = os.path.realpath(opt.work_dir)
- if type(opt.dkx) == type(None):
- opt.dkx = 2. / opt.Lx
- if type(opt.dky) == type(None):
- opt.dky = 2. / opt.Ly
- if type(opt.dkx) == type(None):
- opt.dkz = 2. / opt.Lz
- if type(opt.nx) == type(None):
- opt.nx = opt.n
- if type(opt.ny) == type(None):
- opt.ny = opt.n
- if type(opt.nz) == type(None):
- opt.nz = opt.n
- if type(opt.checkpoints_per_file) == type(None):
- # hardcoded FFTW complex representation size
- field_size = 3*(opt.nx+2)*opt.ny*opt.nz*self.fluid_dtype.itemsize
- checkpoint_size = field_size
- if self.dns_type in ['NSVEparticles', 'NSVEparticles_no_output']:
- rhs_size = self.parameters['tracers0_integration_steps']
- if type(opt.tracers0_integration_steps) != type(None):
- rhs_size = opt.tracers0_integration_steps
- nparticles = opt.nparticles
- if type(nparticles) == type(None):
- nparticles = self.NSVEp_extra_parameters['nparticles']
- particle_size = (1+rhs_size)*3*nparticles*8
- checkpoint_size += particle_size
- if checkpoint_size < 1e9:
- opt.checkpoints_per_file = int(1e9 / checkpoint_size)
- self.pars_from_namespace(opt)
- return opt
- def launch(
- self,
- args = [],
- **kwargs):
- opt = self.prepare_launch(args = args)
- self.launch_jobs(opt = opt, **kwargs)
- return None
- def get_checkpoint_0_fname(self):
- return os.path.join(
- self.work_dir,
- self.simname + '_checkpoint_0.h5')
- def generate_tracer_state(
- self,
- rseed = None,
- species = 0):
- with h5py.File(self.get_checkpoint_0_fname(), 'a') as data_file:
- dset = data_file[
- 'tracers{0}/state/0'.format(species)]
- if not type(rseed) == type(None):
- np.random.seed(rseed)
- nn = self.parameters['nparticles']
- cc = int(0)
- batch_size = int(1e6)
- while nn > 0:
- if nn > batch_size:
- dset[cc*batch_size:(cc+1)*batch_size] = np.random.random(
- (batch_size, 3))*2*np.pi
- nn -= batch_size
- else:
- dset[cc*batch_size:cc*batch_size+nn] = np.random.random(
- (nn, 3))*2*np.pi
- nn = 0
- cc += 1
- return None
- def generate_vector_field(
- self,
- rseed = 7547,
- spectra_slope = 1.,
- amplitude = 1.,
- iteration = 0,
- field_name = 'vorticity',
- write_to_file = False,
- # to switch to constant field, use generate_data_3D_uniform
- # for scalar_generator
- scalar_generator = tools.generate_data_3D):
- """generate vector field.
-
- The generated field is not divergence free, but it has the proper
- shape.
-
- :param rseed: seed for random number generator
- :param spectra_slope: spectrum of field will look like k^(-p)
- :param amplitude: all amplitudes are multiplied with this value
- :param iteration: the field is written at this iteration
- :param field_name: the name of the field being generated
- :param write_to_file: should we write the field to file?
- :param scalar_generator: which function to use for generating the
- individual components.
- Possible values: bfps.tools.generate_data_3D,
- bfps.tools.generate_data_3D_uniform
- :type rseed: int
- :type spectra_slope: float
- :type amplitude: float
- :type iteration: int
- :type field_name: str
- :type write_to_file: bool
- :type scalar_generator: function
-
- :returns: ``Kdata``, a complex valued 4D ``numpy.array`` that uses the
- transposed FFTW layout.
- Kdata[ky, kz, kx, i] is the amplitude of mode (kx, ky, kz) for
- the i-th component of the field.
- (i.e. x is the fastest index and z the slowest index in the
- real-space representation).
- """
- np.random.seed(rseed)
- Kdata00 = scalar_generator(
- self.parameters['nz']//2,
- self.parameters['ny']//2,
- self.parameters['nx']//2,
- p = spectra_slope,
- amplitude = amplitude).astype(self.ctype)
- Kdata01 = scalar_generator(
- self.parameters['nz']//2,
- self.parameters['ny']//2,
- self.parameters['nx']//2,
- p = spectra_slope,
- amplitude = amplitude).astype(self.ctype)
- Kdata02 = scalar_generator(
- self.parameters['nz']//2,
- self.parameters['ny']//2,
- self.parameters['nx']//2,
- p = spectra_slope,
- amplitude = amplitude).astype(self.ctype)
- Kdata0 = np.zeros(
- Kdata00.shape + (3,),
- Kdata00.dtype)
- Kdata0[..., 0] = Kdata00
- Kdata0[..., 1] = Kdata01
- Kdata0[..., 2] = Kdata02
- Kdata1 = tools.padd_with_zeros(
- Kdata0,
- self.parameters['nz'],
- self.parameters['ny'],
- self.parameters['nx'])
- if write_to_file:
- Kdata1.tofile(
- os.path.join(self.work_dir,
- self.simname + "_c{0}_i{1:0>5x}".format(field_name, iteration)))
- return Kdata1
- def copy_complex_field(
- self,
- src_file_name,
- src_dset_name,
- dst_file,
- dst_dset_name,
- make_link = True):
- # I define a min_shape thingie, but for now I only trust this method for
- # the case of increasing/decreasing by the same factor in all directions.
- # in principle we could write something more generic, but i'm not sure
- # how complicated that would be
- dst_shape = (self.parameters['nz'],
- self.parameters['ny'],
- (self.parameters['nx']+2) // 2,
- 3)
- src_file = h5py.File(src_file_name, 'r')
- if (src_file[src_dset_name].shape == dst_shape):
- if make_link and (src_file[src_dset_name].dtype == self.ctype):
- dst_file[dst_dset_name] = h5py.ExternalLink(
- src_file_name,
- src_dset_name)
- else:
- dst_file.create_dataset(
- dst_dset_name,
- shape = dst_shape,
- dtype = self.ctype,
- fillvalue = 0.0)
- for kz in range(src_file[src_dset_name].shape[0]):
- dst_file[dst_dset_name][kz] = src_file[src_dset_name][kz]
- else:
- print('aloha')
- min_shape = (min(dst_shape[0], src_file[src_dset_name].shape[0]),
- min(dst_shape[1], src_file[src_dset_name].shape[1]),
- min(dst_shape[2], src_file[src_dset_name].shape[2]),
- 3)
- print(self.ctype)
- dst_file.create_dataset(
- dst_dset_name,
- shape = dst_shape,
- dtype = np.dtype(self.ctype),
- fillvalue = complex(0))
- for kz in range(min_shape[0]):
- dst_file[dst_dset_name][kz,:min_shape[1], :min_shape[2]] = \
- src_file[src_dset_name][kz, :min_shape[1], :min_shape[2]]
- return None
- def launch_jobs(
- self,
- opt = None,
- particle_initial_condition = None):
- if not os.path.exists(os.path.join(self.work_dir, self.simname + '.h5')):
- # take care of fields' initial condition
- if not os.path.exists(self.get_checkpoint_0_fname()):
- f = h5py.File(self.get_checkpoint_0_fname(), 'w')
- if len(opt.src_simname) > 0:
- source_cp = 0
- src_file = 'not_a_file'
- while True:
- src_file = os.path.join(
- os.path.realpath(opt.src_work_dir),
- opt.src_simname + '_checkpoint_{0}.h5'.format(source_cp))
- f0 = h5py.File(src_file, 'r')
- if '{0}'.format(opt.src_iteration) in f0['vorticity/complex'].keys():
- f0.close()
- break
- source_cp += 1
- self.copy_complex_field(
- src_file,
- 'vorticity/complex/{0}'.format(opt.src_iteration),
- f,
- 'vorticity/complex/{0}'.format(0))
- else:
- data = self.generate_vector_field(
- write_to_file = False,
- spectra_slope = 2.0,
- amplitude = 0.05)
- f['vorticity/complex/{0}'.format(0)] = data
- f.close()
- ## take care of particles' initial condition
- #if self.dns_type in ['NSVEparticles', 'NSVEparticles_no_output']:
- # if opt.pclouds > 1:
- # np.random.seed(opt.particle_rand_seed)
- # if opt.pcloud_type == 'random-cube':
- # particle_initial_condition = (
- # np.random.random((opt.pclouds, 1, 3))*2*np.pi +
- # np.random.random((1, self.parameters['nparticles'], 3))*opt.particle_cloud_size)
- # elif opt.pcloud_type == 'regular-cube':
- # onedarray = np.linspace(
- # -opt.particle_cloud_size/2,
- # opt.particle_cloud_size/2,
- # self.parameters['nparticles'])
- # particle_initial_condition = np.zeros(
- # (opt.pclouds,
- # self.parameters['nparticles'],
- # self.parameters['nparticles'],
- # self.parameters['nparticles'], 3),
- # dtype = np.float64)
- # particle_initial_condition[:] = \
- # np.random.random((opt.pclouds, 1, 1, 1, 3))*2*np.pi
- # particle_initial_condition[..., 0] += onedarray[None, None, None, :]
- # particle_initial_condition[..., 1] += onedarray[None, None, :, None]
- # particle_initial_condition[..., 2] += onedarray[None, :, None, None]
- self.write_par(
- particle_ic = None)
- if self.dns_type in ['NSVEparticles', 'NSVEparticles_no_output']:
- if self.parameters['nparticles'] > 0:
- self.generate_tracer_state(
- species = 0,
- rseed = opt.particle_rand_seed)
- if not os.path.exists(self.get_particle_file_name()):
- with h5py.File(self.get_particle_file_name(), 'w') as particle_file:
- particle_file.create_group('tracers0/velocity')
- particle_file.create_group('tracers0/acceleration')
- self.run(
- nb_processes = opt.nb_processes,
- nb_threads_per_process = opt.nb_threads_per_process,
- njobs = opt.njobs,
- hours = opt.minutes // 60,
- minutes = opt.minutes % 60,
- no_submit = opt.no_submit)
- return None
-
diff --git a/bfps/FluidConvert.py b/bfps/FluidConvert.py
deleted file mode 100644
index 58d19116bfb8ab386ef9783babb2ad8da79760e4..0000000000000000000000000000000000000000
--- a/bfps/FluidConvert.py
+++ /dev/null
@@ -1,140 +0,0 @@
-#######################################################################
-# #
-# Copyright 2015 Max Planck Institute #
-# for Dynamics and Self-Organization #
-# #
-# This file is part of bfps. #
-# #
-# bfps is free software: you can redistribute it and/or modify #
-# it under the terms of the GNU General Public License as published #
-# by the Free Software Foundation, either version 3 of the License, #
-# or (at your option) any later version. #
-# #
-# bfps is distributed in the hope that it will be useful, #
-# but WITHOUT ANY WARRANTY; without even the implied warranty of #
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the #
-# GNU General Public License for more details. #
-# #
-# You should have received a copy of the GNU General Public License #
-# along with bfps. If not, see <http://www.gnu.org/licenses/> #
-# #
-# Contact: Cristian.Lalescu@ds.mpg.de #
-# #
-#######################################################################
-
-
-
-import numpy as np
-import pickle
-import os
-from ._fluid_base import _fluid_particle_base
-from ._base import _base
-import bfps
-
-class FluidConvert(_fluid_particle_base):
- """This class is meant to be used for conversion of native DNS field
- representations to real-space representations of velocity/vorticity
- fields.
- It may be superseeded by streamlined functionality in the future...
- """
- def __init__(
- self,
- name = 'FluidConvert-v' + bfps.__version__,
- work_dir = './',
- simname = 'test',
- fluid_precision = 'single',
- use_fftw_wisdom = False):
- _fluid_particle_base.__init__(
- self,
- name = name + '-' + fluid_precision,
- work_dir = work_dir,
- simname = simname,
- dtype = fluid_precision,
- use_fftw_wisdom = use_fftw_wisdom)
- self.spec_parameters = {}
- self.spec_parameters['write_rvelocity'] = 1
- self.spec_parameters['write_rvorticity'] = 1
- self.spec_parameters['write_rTrS2'] = 1
- self.spec_parameters['write_renstrophy'] = 1
- self.spec_parameters['write_rpressure'] = 1
- self.spec_parameters['iter0'] = 0
- self.spec_parameters['iter1'] = -1
- self.fill_up_fluid_code()
- self.finalize_code(postprocess_mode = True)
- return None
- def fill_up_fluid_code(self):
- self.definitions += self.cread_pars(
- parameters = self.spec_parameters,
- function_suffix = '_specific',
- file_group = 'conversion_parameters')
- self.variables += self.cdef_pars(
- parameters = self.spec_parameters)
- self.main_start += 'read_parameters_specific();\n'
- self.fluid_includes += '#include <cstring>\n'
- self.fluid_variables += ('double t;\n' +
- 'fluid_solver<{0}> *fs;\n').format(self.C_dtype)
- self.fluid_definitions += """
- //begincpp
- void do_conversion(fluid_solver<{0}> *bla)
- {{
- bla->read('v', 'c');
- if (write_rvelocity)
- bla->write('u', 'r');
- if (write_rvorticity)
- bla->write('v', 'r');
- if (write_rTrS2)
- bla->write_rTrS2();
- if (write_renstrophy)
- bla->write_renstrophy();
- if (write_rpressure)
- bla->write_rpressure();
- }}
- //endcpp
- """.format(self.C_dtype)
- self.fluid_start += """
- //begincpp
- fs = new fluid_solver<{0}>(
- simname,
- nx, ny, nz,
- dkx, dky, dkz,
- dealias_type,
- DEFAULT_FFTW_FLAG);
- //endcpp
- """.format(self.C_dtype)
- self.fluid_loop += """
- //begincpp
- fs->iteration = frame_index;
- do_conversion(fs);
- //endcpp
- """
- self.fluid_end += 'delete fs;\n'
- return None
- def specific_parser_arguments(
- self,
- parser):
- _fluid_particle_base.specific_parser_arguments(self, parser)
- self.parameters_to_parser_arguments(
- parser,
- parameters = self.spec_parameters)
- return None
- def launch(
- self,
- args = [],
- **kwargs):
- opt = self.prepare_launch(args)
- if opt.iter1 == -1:
- opt.iter1 = self.get_data_file()['iteration'].value
- self.pars_from_namespace(
- opt,
- parameters = self.spec_parameters)
- self.rewrite_par(
- group = 'conversion_parameters',
- parameters = self.spec_parameters)
- self.run(opt.nb_processes,
- 1,
- hours = opt.minutes // 60,
- minutes = opt.minutes % 60,
- err_file = 'err_convert',
- out_file = 'out_convert')
- return None
-
diff --git a/bfps/FluidResize.py b/bfps/FluidResize.py
deleted file mode 100644
index fb5e26208f6960d447bc927bd9e207354620d188..0000000000000000000000000000000000000000
--- a/bfps/FluidResize.py
+++ /dev/null
@@ -1,156 +0,0 @@
-#######################################################################
-# #
-# Copyright 2015 Max Planck Institute #
-# for Dynamics and Self-Organization #
-# #
-# This file is part of bfps. #
-# #
-# bfps is free software: you can redistribute it and/or modify #
-# it under the terms of the GNU General Public License as published #
-# by the Free Software Foundation, either version 3 of the License, #
-# or (at your option) any later version. #
-# #
-# bfps is distributed in the hope that it will be useful, #
-# but WITHOUT ANY WARRANTY; without even the implied warranty of #
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the #
-# GNU General Public License for more details. #
-# #
-# You should have received a copy of the GNU General Public License #
-# along with bfps. If not, see <http://www.gnu.org/licenses/> #
-# #
-# Contact: Cristian.Lalescu@ds.mpg.de #
-# #
-#######################################################################
-
-
-
-import os
-
-import bfps
-from ._fluid_base import _fluid_particle_base
-
-class FluidResize(_fluid_particle_base):
- """This class is meant to resize snapshots of DNS states to new grids.
- Typical stuff for DNS of turbulence.
- It will become superfluous when HDF5 is used for field I/O.
- """
- def __init__(
- self,
- name = 'FluidResize-v' + bfps.__version__,
- work_dir = './',
- simname = 'test',
- fluid_precision = 'single',
- use_fftw_wisdom = False):
- _fluid_particle_base.__init__(
- self,
- name = name + '-' + fluid_precision,
- work_dir = work_dir,
- simname = simname,
- dtype = fluid_precision,
- use_fftw_wisdom = use_fftw_wisdom)
- self.parameters['src_simname'] = 'test'
- self.parameters['dst_iter'] = 0
- self.parameters['dst_nx'] = 32
- self.parameters['dst_ny'] = 32
- self.parameters['dst_nz'] = 32
- self.parameters['dst_simname'] = 'new_test'
- self.parameters['dst_dkx'] = 1.0
- self.parameters['dst_dky'] = 1.0
- self.parameters['dst_dkz'] = 1.0
- self.fill_up_fluid_code()
- self.finalize_code()
- return None
- def fill_up_fluid_code(self):
- self.fluid_includes += '#include <cstring>\n'
- self.fluid_includes += '#include "fftw_tools.hpp"\n'
- self.fluid_variables += ('double t;\n' +
- 'fluid_solver<' + self.C_dtype + '> *fs0, *fs1;\n')
- self.fluid_start += """
- //begincpp
- char fname[512];
- fs0 = new fluid_solver<{0}>(
- src_simname,
- nx, ny, nz,
- dkx, dky, dkz);
- fs1 = new fluid_solver<{0}>(
- dst_simname,
- dst_nx, dst_ny, dst_nz,
- dst_dkx, dst_dky, dst_dkz);
- fs0->iteration = iteration;
- fs1->iteration = 0;
- DEBUG_MSG("about to read field\\n");
- fs0->read('v', 'c');
- DEBUG_MSG("field read, about to copy data\\n");
- double a, b;
- fs0->compute_velocity(fs0->cvorticity);
- a = 0.5*fs0->autocorrel(fs0->cvelocity);
- b = 0.5*fs0->autocorrel(fs0->cvorticity);
- DEBUG_MSG("old field %d %g %g\\n", fs0->iteration, a, b);
- copy_complex_array<{0}>(fs0->cd, fs0->cvorticity,
- fs1->cd, fs1->cvorticity,
- 3);
- DEBUG_MSG("data copied, about to write new field\\n");
- fs1->write('v', 'c');
- DEBUG_MSG("finished writing\\n");
- fs1->compute_velocity(fs1->cvorticity);
- a = 0.5*fs1->autocorrel(fs1->cvelocity);
- b = 0.5*fs1->autocorrel(fs1->cvorticity);
- DEBUG_MSG("new field %d %g %g\\n", fs1->iteration, a, b);
- //endcpp
- """.format(self.C_dtype)
- self.fluid_end += """
- //begincpp
- delete fs0;
- delete fs1;
- //endcpp
- """
- return None
- def specific_parser_arguments(
- self,
- parser):
- _fluid_particle_base.specific_parser_arguments(self, parser)
- parser.add_argument(
- '-m',
- type = int,
- dest = 'm',
- default = 32,
- metavar = 'M',
- help = 'resize from N to M')
- parser.add_argument(
- '--src_wd',
- type = str,
- dest = 'src_work_dir',
- required = True)
- parser.add_argument(
- '--src_iteration',
- type = int,
- dest = 'src_iteration',
- required = True)
- return None
- def launch(
- self,
- args = [],
- **kwargs):
- opt = self.prepare_launch(args)
- cmd_line_pars = vars(opt)
- for k in ['dst_nx', 'dst_ny', 'dst_nz']:
- if type(cmd_line_pars[k]) == type(None):
- cmd_line_pars[k] = opt.m
- # the 3 dst_ni have been updated in opt itself at this point
- # I'm not sure if this code is future-proof...
- self.parameters['niter_todo'] = 0
- self.pars_from_namespace(opt)
- src_file = os.path.join(
- os.path.realpath(opt.src_work_dir),
- opt.src_simname + '_cvorticity_i{0:0>5x}'.format(opt.src_iteration))
- read_file = os.path.join(
- self.work_dir,
- opt.src_simname + '_cvorticity_i{0:0>5x}'.format(opt.src_iteration))
- self.write_par(iter0 = opt.src_iteration)
- if not os.path.exists(read_file):
- os.symlink(src_file, read_file)
- self.run(ncpu = opt.ncpu,
- hours = opt.minutes // 60,
- minutes = opt.minutes % 60)
- return None
-
diff --git a/bfps/NSManyParticles.py b/bfps/NSManyParticles.py
deleted file mode 100644
index 03f7345f61b27299bd2da60ea0c4d44924112837..0000000000000000000000000000000000000000
--- a/bfps/NSManyParticles.py
+++ /dev/null
@@ -1,92 +0,0 @@
-#######################################################################
-# #
-# Copyright 2015 Max Planck Institute #
-# for Dynamics and Self-Organization #
-# #
-# This file is part of bfps. #
-# #
-# bfps is free software: you can redistribute it and/or modify #
-# it under the terms of the GNU General Public License as published #
-# by the Free Software Foundation, either version 3 of the License, #
-# or (at your option) any later version. #
-# #
-# bfps is distributed in the hope that it will be useful, #
-# but WITHOUT ANY WARRANTY; without even the implied warranty of #
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the #
-# GNU General Public License for more details. #
-# #
-# You should have received a copy of the GNU General Public License #
-# along with bfps. If not, see <http://www.gnu.org/licenses/> #
-# #
-# Contact: Cristian.Lalescu@ds.mpg.de #
-# #
-#######################################################################
-
-
-
-import bfps
-
-class NSManyParticles(bfps.NavierStokes):
- def specific_parser_arguments(
- self,
- parser):
- bfps.NavierStokes.specific_parser_arguments(self, parser)
- parser.add_argument(
- '--particle-class',
- default = 'rFFTW_distributed_particles',
- dest = 'particle_class',
- type = str)
- parser.add_argument(
- '--interpolator-class',
- default = 'rFFTW_interpolator',
- dest = 'interpolator_class',
- type = str)
- parser.add_argument('--neighbours',
- type = int,
- dest = 'neighbours',
- default = 3)
- parser.add_argument('--smoothness',
- type = int,
- dest = 'smoothness',
- default = 2)
- return None
- def launch(
- self,
- args = [],
- **kwargs):
- opt = self.prepare_launch(args = args)
- self.fill_up_fluid_code()
- if type(opt.nparticles) == int:
- if opt.nparticles > 0:
- self.add_3D_rFFTW_field(
- name = 'rFFTW_acc')
- interp_list = []
- for n in range(1, opt.neighbours):
- interp_list.append('Lagrange_n{0}'.format(n))
- self.add_interpolator(
- interp_type = 'Lagrange',
- name = interp_list[-1],
- neighbours = n,
- class_name = opt.interpolator_class)
- for m in range(1, opt.smoothness):
- interp_list.append('spline_n{0}m{1}'.format(n, m))
- self.add_interpolator(
- interp_type = 'spline',
- name = interp_list[-1],
- neighbours = n,
- smoothness = m,
- class_name = opt.interpolator_class)
- self.add_particles(
- integration_steps = 2,
- interpolator = interp_list,
- acc_name = 'rFFTW_acc',
- class_name = opt.particle_class)
- self.add_particles(
- integration_steps = 4,
- interpolator = interp_list,
- acc_name = 'rFFTW_acc',
- class_name = opt.particle_class)
- self.finalize_code()
- self.launch_jobs(opt = opt)
- return None
-
diff --git a/bfps/NSVorticityEquation.py b/bfps/NSVorticityEquation.py
deleted file mode 100644
index 5f87097fefbb56f731a75597395d42423fc17ba6..0000000000000000000000000000000000000000
--- a/bfps/NSVorticityEquation.py
+++ /dev/null
@@ -1,864 +0,0 @@
-#######################################################################
-# #
-# Copyright 2015 Max Planck Institute #
-# for Dynamics and Self-Organization #
-# #
-# This file is part of bfps. #
-# #
-# bfps is free software: you can redistribute it and/or modify #
-# it under the terms of the GNU General Public License as published #
-# by the Free Software Foundation, either version 3 of the License, #
-# or (at your option) any later version. #
-# #
-# bfps is distributed in the hope that it will be useful, #
-# but WITHOUT ANY WARRANTY; without even the implied warranty of #
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the #
-# GNU General Public License for more details. #
-# #
-# You should have received a copy of the GNU General Public License #
-# along with bfps. If not, see <http://www.gnu.org/licenses/> #
-# #
-# Contact: Cristian.Lalescu@ds.mpg.de #
-# #
-#######################################################################
-
-
-
-import sys
-import os
-import numpy as np
-import h5py
-import argparse
-
-import bfps
-import bfps.tools
-from bfps._code import _code
-from bfps._fluid_base import _fluid_particle_base
-
-class NSVorticityEquation(_fluid_particle_base):
- def __init__(
- self,
- name = 'NSVorticityEquation-v' + bfps.__version__,
- work_dir = './',
- simname = 'test',
- fluid_precision = 'single',
- fftw_plan_rigor = 'FFTW_MEASURE',
- use_fftw_wisdom = True):
- """
- This code uses checkpoints for DNS restarts, and it can be stopped
- by creating the file "stop_<simname>" in the working directory.
- For postprocessing of field snapshots, consider creating a separate
- HDF5 file (from the python wrapper) which contains links to all the
- different snapshots.
- """
- self.fftw_plan_rigor = fftw_plan_rigor
- _fluid_particle_base.__init__(
- self,
- name = name + '-' + fluid_precision,
- work_dir = work_dir,
- simname = simname,
- dtype = fluid_precision,
- use_fftw_wisdom = use_fftw_wisdom)
- self.parameters['nu'] = float(0.1)
- self.parameters['fmode'] = 1
- self.parameters['famplitude'] = float(0.5)
- self.parameters['fk0'] = float(2.0)
- self.parameters['fk1'] = float(4.0)
- self.parameters['forcing_type'] = 'linear'
- self.parameters['histogram_bins'] = int(256)
- self.parameters['max_velocity_estimate'] = float(1)
- self.parameters['max_vorticity_estimate'] = float(1)
- self.parameters['checkpoints_per_file'] = int(1)
- self.file_datasets_grow = """
- //begincpp
- hid_t group;
- group = H5Gopen(stat_file, "/statistics", H5P_DEFAULT);
- H5Ovisit(group, H5_INDEX_NAME, H5_ITER_NATIVE, grow_statistics_dataset, NULL);
- H5Gclose(group);
- //endcpp
- """
- self.style = {}
- self.statistics = {}
- self.fluid_output = """
- fs->io_checkpoint(false);
- """
- # vorticity_equation specific things
- self.includes += '#include "vorticity_equation.hpp"\n'
- self.store_kspace = """
- //begincpp
- if (myrank == 0 && iteration == 0)
- {
- TIMEZONE("fluid_base::store_kspace");
- hsize_t dims[4];
- hid_t space, dset;
- // store kspace information
- dset = H5Dopen(stat_file, "/kspace/kshell", H5P_DEFAULT);
- space = H5Dget_space(dset);
- H5Sget_simple_extent_dims(space, dims, NULL);
- H5Sclose(space);
- if (fs->kk->nshells != dims[0])
- {
- DEBUG_MSG(
- "ERROR: computed nshells %d not equal to data file nshells %d\\n",
- fs->kk->nshells, dims[0]);
- }
- H5Dwrite(dset, H5T_NATIVE_DOUBLE, H5S_ALL, H5S_ALL, H5P_DEFAULT, &fs->kk->kshell.front());
- H5Dclose(dset);
- dset = H5Dopen(stat_file, "/kspace/nshell", H5P_DEFAULT);
- H5Dwrite(dset, H5T_NATIVE_INT64, H5S_ALL, H5S_ALL, H5P_DEFAULT, &fs->kk->nshell.front());
- H5Dclose(dset);
- dset = H5Dopen(stat_file, "/kspace/kM", H5P_DEFAULT);
- H5Dwrite(dset, H5T_NATIVE_DOUBLE, H5S_ALL, H5S_ALL, H5P_DEFAULT, &fs->kk->kM);
- H5Dclose(dset);
- dset = H5Dopen(stat_file, "/kspace/dk", H5P_DEFAULT);
- H5Dwrite(dset, H5T_NATIVE_DOUBLE, H5S_ALL, H5S_ALL, H5P_DEFAULT, &fs->kk->dk);
- H5Dclose(dset);
- }
- //endcpp
- """
- return None
- def add_particles(
- self,
- integration_steps = 2,
- neighbours = 1,
- smoothness = 1):
- assert(integration_steps > 0 and integration_steps < 6)
- self.particle_species = 1
- self.parameters['tracers0_integration_steps'] = int(integration_steps)
- self.parameters['tracers0_neighbours'] = int(neighbours)
- self.parameters['tracers0_smoothness'] = int(smoothness)
- self.parameters['tracers0_interpolator'] = 'spline'
- self.particle_includes += """
- #include "particles/particles_system_builder.hpp"
- #include "particles/particles_output_hdf5.hpp"
- """
- ## initialize
- self.particle_start += """
- DEBUG_MSG(
- "current fname is %s\\n and iteration is %d",
- fs->get_current_fname().c_str(),
- fs->iteration);
- std::unique_ptr<abstract_particles_system<long long int, double>> ps = particles_system_builder(
- fs->cvelocity, // (field object)
- fs->kk, // (kspace object, contains dkx, dky, dkz)
- tracers0_integration_steps, // to check coherency between parameters and hdf input file (nb rhs)
- (long long int)nparticles, // to check coherency between parameters and hdf input file
- fs->get_current_fname(), // particles input filename
- std::string("/tracers0/state/") + std::to_string(fs->iteration), // dataset name for initial input
- std::string("/tracers0/rhs/") + std::to_string(fs->iteration), // dataset name for initial input
- tracers0_neighbours, // parameter (interpolation no neighbours)
- tracers0_smoothness, // parameter
- MPI_COMM_WORLD,
- fs->iteration+1);
- particles_output_hdf5<long long int, double,3,3> particles_output_writer_mpi(
- MPI_COMM_WORLD,
- "tracers0",
- nparticles,
- tracers0_integration_steps);
- """
- self.particle_loop += """
- fs->compute_velocity(fs->cvorticity);
- fs->cvelocity->ift();
- ps->completeLoop(dt);
- """
- self.particle_output = """
- {
- particles_output_writer_mpi.open_file(fs->get_current_fname());
- particles_output_writer_mpi.save(ps->getParticlesPositions(),
- ps->getParticlesRhs(),
- ps->getParticlesIndexes(),
- ps->getLocalNbParticles(),
- fs->iteration);
- particles_output_writer_mpi.close_file();
- }
- """
- self.particle_end += 'ps.release();\n'
- return None
- def create_stat_output(
- self,
- dset_name,
- data_buffer,
- data_type = 'H5T_NATIVE_DOUBLE',
- size_setup = None,
- close_spaces = True):
- new_stat_output_txt = 'Cdset = H5Dopen(stat_file, "{0}", H5P_DEFAULT);\n'.format(dset_name)
- if not type(size_setup) == type(None):
- new_stat_output_txt += (
- size_setup +
- 'wspace = H5Dget_space(Cdset);\n' +
- 'ndims = H5Sget_simple_extent_dims(wspace, dims, NULL);\n' +
- 'mspace = H5Screate_simple(ndims, count, NULL);\n' +
- 'H5Sselect_hyperslab(wspace, H5S_SELECT_SET, offset, NULL, count, NULL);\n')
- new_stat_output_txt += ('H5Dwrite(Cdset, {0}, mspace, wspace, H5P_DEFAULT, {1});\n' +
- 'H5Dclose(Cdset);\n').format(data_type, data_buffer)
- if close_spaces:
- new_stat_output_txt += ('H5Sclose(mspace);\n' +
- 'H5Sclose(wspace);\n')
- return new_stat_output_txt
- def write_fluid_stats(self):
- self.fluid_includes += '#include <cmath>\n'
- self.fluid_includes += '#include "fftw_tools.hpp"\n'
- self.stat_src += """
- //begincpp
- hid_t stat_group;
- if (myrank == 0)
- stat_group = H5Gopen(stat_file, "statistics", H5P_DEFAULT);
- fs->compute_velocity(fs->cvorticity);
- *tmp_vec_field = fs->cvelocity->get_cdata();
- tmp_vec_field->compute_stats(
- fs->kk,
- stat_group,
- "velocity",
- fs->iteration / niter_stat,
- max_velocity_estimate/sqrt(3));
- //endcpp
- """
- self.stat_src += """
- //begincpp
- *tmp_vec_field = fs->cvorticity->get_cdata();
- tmp_vec_field->compute_stats(
- fs->kk,
- stat_group,
- "vorticity",
- fs->iteration / niter_stat,
- max_vorticity_estimate/sqrt(3));
- //endcpp
- """
- self.stat_src += """
- //begincpp
- if (myrank == 0)
- H5Gclose(stat_group);
- if (myrank == 0)
- {{
- hid_t Cdset, wspace, mspace;
- int ndims;
- hsize_t count[4], offset[4], dims[4];
- offset[0] = fs->iteration/niter_stat;
- offset[1] = 0;
- offset[2] = 0;
- offset[3] = 0;
- //endcpp
- """.format(self.C_dtype)
- if self.dtype == np.float32:
- field_H5T = 'H5T_NATIVE_FLOAT'
- elif self.dtype == np.float64:
- field_H5T = 'H5T_NATIVE_DOUBLE'
- self.stat_src += self.create_stat_output(
- '/statistics/xlines/velocity',
- 'fs->rvelocity->get_rdata()',
- data_type = field_H5T,
- size_setup = """
- count[0] = 1;
- count[1] = nx;
- count[2] = 3;
- """,
- close_spaces = False)
- self.stat_src += self.create_stat_output(
- '/statistics/xlines/vorticity',
- 'fs->rvorticity->get_rdata()',
- data_type = field_H5T)
- self.stat_src += '}\n'
- ## checkpoint
- self.stat_src += """
- //begincpp
- if (myrank == 0)
- {
- std::string fname = (
- std::string("stop_") +
- std::string(simname));
- {
- struct stat file_buffer;
- stop_code_now = (stat(fname.c_str(), &file_buffer) == 0);
- }
- }
- MPI_Bcast(&stop_code_now, 1, MPI_C_BOOL, 0, MPI_COMM_WORLD);
- //endcpp
- """
- return None
- def fill_up_fluid_code(self):
- self.fluid_includes += '#include <cstring>\n'
- self.fluid_variables += (
- 'vorticity_equation<{0}, FFTW> *fs;\n'.format(self.C_dtype) +
- 'field<{0}, FFTW, THREE> *tmp_vec_field;\n'.format(self.C_dtype) +
- 'field<{0}, FFTW, ONE> *tmp_scal_field;\n'.format(self.C_dtype))
- self.fluid_definitions += """
- typedef struct {{
- {0} re;
- {0} im;
- }} tmp_complex_type;
- """.format(self.C_dtype)
- self.write_fluid_stats()
- if self.dtype == np.float32:
- field_H5T = 'H5T_NATIVE_FLOAT'
- elif self.dtype == np.float64:
- field_H5T = 'H5T_NATIVE_DOUBLE'
- self.variables += 'int checkpoint;\n'
- self.variables += 'bool stop_code_now;\n'
- self.read_checkpoint = """
- //begincpp
- if (myrank == 0)
- {
- hid_t dset = H5Dopen(stat_file, "checkpoint", H5P_DEFAULT);
- H5Dread(
- dset,
- H5T_NATIVE_INT,
- H5S_ALL,
- H5S_ALL,
- H5P_DEFAULT,
- &checkpoint);
- H5Dclose(dset);
- }
- MPI_Bcast(&checkpoint, 1, MPI_INT, 0, MPI_COMM_WORLD);
- fs->checkpoint = checkpoint;
- //endcpp
- """
- self.store_checkpoint = """
- //begincpp
- checkpoint = fs->checkpoint;
- if (myrank == 0)
- {
- hid_t dset = H5Dopen(stat_file, "checkpoint", H5P_DEFAULT);
- H5Dwrite(
- dset,
- H5T_NATIVE_INT,
- H5S_ALL,
- H5S_ALL,
- H5P_DEFAULT,
- &checkpoint);
- H5Dclose(dset);
- }
- //endcpp
- """
- self.fluid_start += """
- //begincpp
- char fname[512];
- fs = new vorticity_equation<{0}, FFTW>(
- simname,
- nx, ny, nz,
- dkx, dky, dkz,
- {1});
- tmp_vec_field = new field<{0}, FFTW, THREE>(
- nx, ny, nz,
- MPI_COMM_WORLD,
- {1});
- tmp_scal_field = new field<{0}, FFTW, ONE>(
- nx, ny, nz,
- MPI_COMM_WORLD,
- {1});
- fs->checkpoints_per_file = checkpoints_per_file;
- fs->nu = nu;
- fs->fmode = fmode;
- fs->famplitude = famplitude;
- fs->fk0 = fk0;
- fs->fk1 = fk1;
- strncpy(fs->forcing_type, forcing_type, 128);
- fs->iteration = iteration;
- {2}
- fs->cvorticity->real_space_representation = false;
- fs->io_checkpoint();
- //endcpp
- """.format(
- self.C_dtype,
- self.fftw_plan_rigor,
- self.read_checkpoint)
- self.fluid_start += self.store_kspace
- self.fluid_start += 'stop_code_now = false;\n'
- self.fluid_loop = 'fs->step(dt);\n'
- self.fluid_loop += ('if (fs->iteration % niter_out == 0)\n{\n' +
- self.fluid_output +
- self.particle_output +
- self.store_checkpoint +
- '\n}\n' +
- 'if (stop_code_now){\n' +
- 'iteration = fs->iteration;\n' +
- 'break;\n}\n')
- self.fluid_end = ('if (fs->iteration % niter_out != 0)\n{\n' +
- self.fluid_output +
- self.particle_output +
- self.store_checkpoint +
- 'DEBUG_MSG("checkpoint value is %d\\n", checkpoint);\n' +
- '\n}\n' +
- 'delete fs;\n' +
- 'delete tmp_vec_field;\n' +
- 'delete tmp_scal_field;\n')
- return None
- def get_postprocess_file_name(self):
- return os.path.join(self.work_dir, self.simname + '_postprocess.h5')
- def get_postprocess_file(self):
- return h5py.File(self.get_postprocess_file_name(), 'r')
- def compute_statistics(self, iter0 = 0, iter1 = None):
- """Run basic postprocessing on raw data.
- The energy spectrum :math:`E(t, k)` and the enstrophy spectrum
- :math:`\\frac{1}{2}\omega^2(t, k)` are computed from the
-
- .. math::
-
- \sum_{k \\leq \\|\\mathbf{k}\\| \\leq k+dk}\\hat{u_i} \\hat{u_j}^*, \\hskip .5cm
- \sum_{k \\leq \\|\\mathbf{k}\\| \\leq k+dk}\\hat{\omega_i} \\hat{\\omega_j}^*
-
- tensors, and the enstrophy spectrum is also used to
- compute the dissipation :math:`\\varepsilon(t)`.
- These basic quantities are stored in a newly created HDF5 file,
- ``simname_postprocess.h5``.
- """
- if len(list(self.statistics.keys())) > 0:
- return None
- self.read_parameters()
- with self.get_data_file() as data_file:
- if 'moments' not in data_file['statistics'].keys():
- return None
- iter0 = min((data_file['statistics/moments/velocity'].shape[0] *
- self.parameters['niter_stat']-1),
- iter0)
- if type(iter1) == type(None):
- iter1 = data_file['iteration'].value
- else:
- iter1 = min(data_file['iteration'].value, iter1)
- ii0 = iter0 // self.parameters['niter_stat']
- ii1 = iter1 // self.parameters['niter_stat']
- self.statistics['kshell'] = data_file['kspace/kshell'].value
- self.statistics['kM'] = data_file['kspace/kM'].value
- self.statistics['dk'] = data_file['kspace/dk'].value
- computation_needed = True
- pp_file = h5py.File(self.get_postprocess_file_name(), 'a')
- if 'ii0' in pp_file.keys():
- computation_needed = not (ii0 == pp_file['ii0'].value and
- ii1 == pp_file['ii1'].value)
- if computation_needed:
- for k in pp_file.keys():
- del pp_file[k]
- if computation_needed:
- pp_file['iter0'] = iter0
- pp_file['iter1'] = iter1
- pp_file['ii0'] = ii0
- pp_file['ii1'] = ii1
- pp_file['t'] = (self.parameters['dt']*
- self.parameters['niter_stat']*
- (np.arange(ii0, ii1+1).astype(np.float)))
- pp_file['energy(t, k)'] = (
- data_file['statistics/spectra/velocity_velocity'][ii0:ii1+1, :, 0, 0] +
- data_file['statistics/spectra/velocity_velocity'][ii0:ii1+1, :, 1, 1] +
- data_file['statistics/spectra/velocity_velocity'][ii0:ii1+1, :, 2, 2])/2
- pp_file['enstrophy(t, k)'] = (
- data_file['statistics/spectra/vorticity_vorticity'][ii0:ii1+1, :, 0, 0] +
- data_file['statistics/spectra/vorticity_vorticity'][ii0:ii1+1, :, 1, 1] +
- data_file['statistics/spectra/vorticity_vorticity'][ii0:ii1+1, :, 2, 2])/2
- pp_file['vel_max(t)'] = data_file['statistics/moments/velocity'] [ii0:ii1+1, 9, 3]
- pp_file['renergy(t)'] = data_file['statistics/moments/velocity'][ii0:ii1+1, 2, 3]/2
- for k in ['t',
- 'energy(t, k)',
- 'enstrophy(t, k)',
- 'vel_max(t)',
- 'renergy(t)']:
- if k in pp_file.keys():
- self.statistics[k] = pp_file[k].value
- self.compute_time_averages()
- return None
- def compute_time_averages(self):
- """Compute easy stats.
-
- Further computation of statistics based on the contents of
- ``simname_postprocess.h5``.
- Standard quantities are as follows
- (consistent with [Ishihara]_):
-
- .. math::
-
- U_{\\textrm{int}}(t) = \\sqrt{\\frac{2E(t)}{3}}, \\hskip .5cm
- L_{\\textrm{int}}(t) = \\frac{\pi}{2U_{int}^2(t)} \\int \\frac{dk}{k} E(t, k), \\hskip .5cm
- T_{\\textrm{int}}(t) =
- \\frac{L_{\\textrm{int}}(t)}{U_{\\textrm{int}}(t)}
-
- \\eta_K = \\left(\\frac{\\nu^3}{\\varepsilon}\\right)^{1/4}, \\hskip .5cm
- \\tau_K = \\left(\\frac{\\nu}{\\varepsilon}\\right)^{1/2}, \\hskip .5cm
- \\lambda = \\sqrt{\\frac{15 \\nu U_{\\textrm{int}}^2}{\\varepsilon}}
-
- Re = \\frac{U_{\\textrm{int}} L_{\\textrm{int}}}{\\nu}, \\hskip
- .5cm
- R_{\\lambda} = \\frac{U_{\\textrm{int}} \\lambda}{\\nu}
-
- .. [Ishihara] T. Ishihara et al,
- *Small-scale statistics in high-resolution direct numerical
- simulation of turbulence: Reynolds number dependence of
- one-point velocity gradient statistics*.
- J. Fluid Mech.,
- **592**, 335-366, 2007
- """
- for key in ['energy', 'enstrophy']:
- self.statistics[key + '(t)'] = (self.statistics['dk'] *
- np.sum(self.statistics[key + '(t, k)'], axis = 1))
- self.statistics['Uint(t)'] = np.sqrt(2*self.statistics['energy(t)'] / 3)
- self.statistics['Lint(t)'] = ((self.statistics['dk']*np.pi /
- (2*self.statistics['Uint(t)']**2)) *
- np.nansum(self.statistics['energy(t, k)'] /
- self.statistics['kshell'][None, :], axis = 1))
- for key in ['energy',
- 'enstrophy',
- 'vel_max',
- 'Uint',
- 'Lint']:
- if key + '(t)' in self.statistics.keys():
- self.statistics[key] = np.average(self.statistics[key + '(t)'], axis = 0)
- for suffix in ['', '(t)']:
- self.statistics['diss' + suffix] = (self.parameters['nu'] *
- self.statistics['enstrophy' + suffix]*2)
- self.statistics['etaK' + suffix] = (self.parameters['nu']**3 /
- self.statistics['diss' + suffix])**.25
- self.statistics['tauK' + suffix] = (self.parameters['nu'] /
- self.statistics['diss' + suffix])**.5
- self.statistics['Re' + suffix] = (self.statistics['Uint' + suffix] *
- self.statistics['Lint' + suffix] /
- self.parameters['nu'])
- self.statistics['lambda' + suffix] = (15 * self.parameters['nu'] *
- self.statistics['Uint' + suffix]**2 /
- self.statistics['diss' + suffix])**.5
- self.statistics['Rlambda' + suffix] = (self.statistics['Uint' + suffix] *
- self.statistics['lambda' + suffix] /
- self.parameters['nu'])
- self.statistics['kMeta' + suffix] = (self.statistics['kM'] *
- self.statistics['etaK' + suffix])
- if self.parameters['dealias_type'] == 1:
- self.statistics['kMeta' + suffix] *= 0.8
- self.statistics['Tint'] = self.statistics['Lint'] / self.statistics['Uint']
- self.statistics['Taylor_microscale'] = self.statistics['lambda']
- return None
- def set_plt_style(
- self,
- style = {'dashes' : (None, None)}):
- self.style.update(style)
- return None
- def convert_complex_from_binary(
- self,
- field_name = 'vorticity',
- iteration = 0,
- file_name = None):
- """read the Fourier representation of a vector field.
-
- Read the binary file containing iteration ``iteration`` of the
- field ``field_name``, and write it in a ``.h5`` file.
- """
- data = np.memmap(
- os.path.join(self.work_dir,
- self.simname + '_{0}_i{1:0>5x}'.format('c' + field_name, iteration)),
- dtype = self.ctype,
- mode = 'r',
- shape = (self.parameters['ny'],
- self.parameters['nz'],
- self.parameters['nx']//2+1,
- 3))
- if type(file_name) == type(None):
- file_name = self.simname + '_{0}_i{1:0>5x}.h5'.format('c' + field_name, iteration)
- file_name = os.path.join(self.work_dir, file_name)
- f = h5py.File(file_name, 'a')
- f[field_name + '/complex/{0}'.format(iteration)] = data
- f.close()
- return None
- def write_par(
- self,
- iter0 = 0,
- particle_ic = None):
- _fluid_particle_base.write_par(self, iter0 = iter0)
- with h5py.File(self.get_data_file_name(), 'r+') as ofile:
- kspace = self.get_kspace()
- nshells = kspace['nshell'].shape[0]
- vec_stat_datasets = ['velocity', 'vorticity']
- scal_stat_datasets = []
- for k in vec_stat_datasets:
- time_chunk = 2**20//(8*3*self.parameters['nx']) # FIXME: use proper size of self.dtype
- time_chunk = max(time_chunk, 1)
- ofile.create_dataset('statistics/xlines/' + k,
- (1, self.parameters['nx'], 3),
- chunks = (time_chunk, self.parameters['nx'], 3),
- maxshape = (None, self.parameters['nx'], 3),
- dtype = self.dtype)
- for k in vec_stat_datasets:
- time_chunk = 2**20//(8*3*3*nshells)
- time_chunk = max(time_chunk, 1)
- ofile.create_dataset('statistics/spectra/' + k + '_' + k,
- (1, nshells, 3, 3),
- chunks = (time_chunk, nshells, 3, 3),
- maxshape = (None, nshells, 3, 3),
- dtype = np.float64)
- time_chunk = 2**20//(8*4*10)
- time_chunk = max(time_chunk, 1)
- a = ofile.create_dataset('statistics/moments/' + k,
- (1, 10, 4),
- chunks = (time_chunk, 10, 4),
- maxshape = (None, 10, 4),
- dtype = np.float64)
- time_chunk = 2**20//(8*4*self.parameters['histogram_bins'])
- time_chunk = max(time_chunk, 1)
- ofile.create_dataset('statistics/histograms/' + k,
- (1,
- self.parameters['histogram_bins'],
- 4),
- chunks = (time_chunk,
- self.parameters['histogram_bins'],
- 4),
- maxshape = (None,
- self.parameters['histogram_bins'],
- 4),
- dtype = np.int64)
- ofile['checkpoint'] = int(0)
- if self.particle_species == 0:
- return None
-
- if type(particle_ic) == type(None):
- pbase_shape = (self.parameters['nparticles'],)
- number_of_particles = self.parameters['nparticles']
- else:
- pbase_shape = particle_ic.shape[:-1]
- assert(particle_ic.shape[-1] == 3)
- number_of_particles = 1
- for val in pbase_shape[1:]:
- number_of_particles *= val
- with h5py.File(self.get_checkpoint_0_fname(), 'a') as ofile:
- s = 0
- ofile.create_group('tracers{0}'.format(s))
- ofile.create_group('tracers{0}/rhs'.format(s))
- ofile.create_group('tracers{0}/state'.format(s))
- ofile['tracers{0}/rhs'.format(s)].create_dataset(
- '0',
- shape = (
- (self.parameters['tracers{0}_integration_steps'.format(s)],) +
- pbase_shape +
- (3,)),
- dtype = np.float)
- ofile['tracers{0}/state'.format(s)].create_dataset(
- '0',
- shape = (
- pbase_shape +
- (3,)),
- dtype = np.float)
- return None
- def specific_parser_arguments(
- self,
- parser):
- _fluid_particle_base.specific_parser_arguments(self, parser)
- parser.add_argument(
- '--src-wd',
- type = str,
- dest = 'src_work_dir',
- default = '')
- parser.add_argument(
- '--src-simname',
- type = str,
- dest = 'src_simname',
- default = '')
- parser.add_argument(
- '--src-iteration',
- type = int,
- dest = 'src_iteration',
- default = 0)
- parser.add_argument(
- '--njobs',
- type = int, dest = 'njobs',
- default = 1)
- parser.add_argument(
- '--kMeta',
- type = float,
- dest = 'kMeta',
- default = 2.0)
- parser.add_argument(
- '--dtfactor',
- type = float,
- dest = 'dtfactor',
- default = 0.5,
- help = 'dt is computed as DTFACTOR / N')
- parser.add_argument(
- '--particle-rand-seed',
- type = int,
- dest = 'particle_rand_seed',
- default = None)
- parser.add_argument(
- '--pclouds',
- type = int,
- dest = 'pclouds',
- default = 1,
- help = ('number of particle clouds. Particle "clouds" '
- 'consist of particles distributed according to '
- 'pcloud-type.'))
- parser.add_argument(
- '--pcloud-type',
- choices = ['random-cube',
- 'regular-cube'],
- dest = 'pcloud_type',
- default = 'random-cube')
- parser.add_argument(
- '--particle-cloud-size',
- type = float,
- dest = 'particle_cloud_size',
- default = 2*np.pi)
- parser.add_argument(
- '--neighbours',
- type = int,
- dest = 'neighbours',
- default = 1)
- parser.add_argument(
- '--smoothness',
- type = int,
- dest = 'smoothness',
- default = 1)
- return None
- def prepare_launch(
- self,
- args = []):
- """Set up reasonable parameters.
-
- With the default Lundgren forcing applied in the band [2, 4],
- we can estimate the dissipation, therefore we can estimate
- :math:`k_M \\eta_K` and constrain the viscosity.
-
- In brief, the command line parameter :math:`k_M \\eta_K` is
- used in the following formula for :math:`\\nu` (:math:`N` is the
- number of real space grid points per coordinate):
-
- .. math::
-
- \\nu = \\left(\\frac{2 k_M \\eta_K}{N} \\right)^{4/3}
-
- With this choice, the average dissipation :math:`\\varepsilon`
- will be close to 0.4, and the integral scale velocity will be
- close to 0.77, yielding the approximate value for the Taylor
- microscale and corresponding Reynolds number:
-
- .. math::
-
- \\lambda \\approx 4.75\\left(\\frac{2 k_M \\eta_K}{N} \\right)^{4/6}, \\hskip .5in
- R_\\lambda \\approx 3.7 \\left(\\frac{N}{2 k_M \\eta_K} \\right)^{4/6}
-
- """
- opt = _code.prepare_launch(self, args = args)
- self.parameters['nu'] = (opt.kMeta * 2 / opt.n)**(4./3)
- self.parameters['dt'] = (opt.dtfactor / opt.n)
- # custom famplitude for 288 and 576
- if opt.n == 288:
- self.parameters['famplitude'] = 0.45
- elif opt.n == 576:
- self.parameters['famplitude'] = 0.47
- if ((self.parameters['niter_todo'] % self.parameters['niter_out']) != 0):
- self.parameters['niter_out'] = self.parameters['niter_todo']
- if len(opt.src_work_dir) == 0:
- opt.src_work_dir = os.path.realpath(opt.work_dir)
- self.pars_from_namespace(opt)
- return opt
- def launch(
- self,
- args = [],
- **kwargs):
- opt = self.prepare_launch(args = args)
- if type(opt.nparticles) != type(None):
- if opt.nparticles > 0:
- self.name += '-particles'
- self.add_particles(
- integration_steps = 4,
- neighbours = opt.neighbours,
- smoothness = opt.smoothness)
- self.fill_up_fluid_code()
- self.finalize_code()
- self.launch_jobs(opt = opt, **kwargs)
- return None
- def get_checkpoint_0_fname(self):
- return os.path.join(
- self.work_dir,
- self.simname + '_checkpoint_0.h5')
- def generate_tracer_state(
- self,
- rseed = None,
- iteration = 0,
- species = 0,
- write_to_file = False,
- ncomponents = 3,
- testing = False,
- data = None):
- if (type(data) == type(None)):
- if not type(rseed) == type(None):
- np.random.seed(rseed)
- #point with problems: 5.37632864e+00, 6.10414710e+00, 6.25256493e+00]
- data = np.zeros(self.parameters['nparticles']*ncomponents).reshape(-1, ncomponents)
- data[:, :3] = np.random.random((self.parameters['nparticles'], 3))*2*np.pi
- if testing:
- #data[0] = np.array([3.26434, 4.24418, 3.12157])
- data[:] = np.array([ 0.72086101, 2.59043666, 6.27501953])
- with h5py.File(self.get_checkpoint_0_fname(), 'a') as data_file:
- data_file['tracers{0}/state/0'.format(species)][:] = data
- if write_to_file:
- data.tofile(
- os.path.join(
- self.work_dir,
- "tracers{0}_state_i{1:0>5x}".format(species, iteration)))
- return data
- def launch_jobs(
- self,
- opt = None,
- particle_initial_condition = None):
- if not os.path.exists(os.path.join(self.work_dir, self.simname + '.h5')):
- # take care of fields' initial condition
- if not os.path.exists(self.get_checkpoint_0_fname()):
- f = h5py.File(self.get_checkpoint_0_fname(), 'w')
- if len(opt.src_simname) > 0:
- source_cp = 0
- src_file = 'not_a_file'
- while True:
- src_file = os.path.join(
- os.path.realpath(opt.src_work_dir),
- opt.src_simname + '_checkpoint_{0}.h5'.format(source_cp))
- f0 = h5py.File(src_file, 'r')
- if '{0}'.format(opt.src_iteration) in f0['vorticity/complex'].keys():
- f0.close()
- break
- source_cp += 1
- f['vorticity/complex/{0}'.format(0)] = h5py.ExternalLink(
- src_file,
- 'vorticity/complex/{0}'.format(opt.src_iteration))
- else:
- data = self.generate_vector_field(
- write_to_file = False,
- spectra_slope = 2.0,
- amplitude = 0.05)
- f['vorticity/complex/{0}'.format(0)] = data
- f.close()
- # take care of particles' initial condition
- if opt.pclouds > 1:
- np.random.seed(opt.particle_rand_seed)
- if opt.pcloud_type == 'random-cube':
- particle_initial_condition = (
- np.random.random((opt.pclouds, 1, 3))*2*np.pi +
- np.random.random((1, self.parameters['nparticles'], 3))*opt.particle_cloud_size)
- elif opt.pcloud_type == 'regular-cube':
- onedarray = np.linspace(
- -opt.particle_cloud_size/2,
- opt.particle_cloud_size/2,
- self.parameters['nparticles'])
- particle_initial_condition = np.zeros(
- (opt.pclouds,
- self.parameters['nparticles'],
- self.parameters['nparticles'],
- self.parameters['nparticles'], 3),
- dtype = np.float64)
- particle_initial_condition[:] = \
- np.random.random((opt.pclouds, 1, 1, 1, 3))*2*np.pi
- particle_initial_condition[..., 0] += onedarray[None, None, None, :]
- particle_initial_condition[..., 1] += onedarray[None, None, :, None]
- particle_initial_condition[..., 2] += onedarray[None, :, None, None]
- self.write_par(
- particle_ic = particle_initial_condition)
- if self.parameters['nparticles'] > 0:
- data = self.generate_tracer_state(
- species = 0,
- rseed = opt.particle_rand_seed,
- data = particle_initial_condition)
- for s in range(1, self.particle_species):
- self.generate_tracer_state(species = s, data = data)
- self.run(
- nb_processes = opt.nb_processes,
- nb_threads_per_process = opt.nb_threads_per_process,
- njobs = opt.njobs,
- hours = opt.minutes // 60,
- minutes = opt.minutes % 60,
- no_submit = opt.no_submit)
- return None
-
-if __name__ == '__main__':
- pass
-
diff --git a/bfps/NavierStokes.py b/bfps/NavierStokes.py
deleted file mode 100644
index 59fb907c4a79f73dec5b6a8cfcb06d99b0b584bb..0000000000000000000000000000000000000000
--- a/bfps/NavierStokes.py
+++ /dev/null
@@ -1,1213 +0,0 @@
-#######################################################################
-# #
-# Copyright 2015 Max Planck Institute #
-# for Dynamics and Self-Organization #
-# #
-# This file is part of bfps. #
-# #
-# bfps is free software: you can redistribute it and/or modify #
-# it under the terms of the GNU General Public License as published #
-# by the Free Software Foundation, either version 3 of the License, #
-# or (at your option) any later version. #
-# #
-# bfps is distributed in the hope that it will be useful, #
-# but WITHOUT ANY WARRANTY; without even the implied warranty of #
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the #
-# GNU General Public License for more details. #
-# #
-# You should have received a copy of the GNU General Public License #
-# along with bfps. If not, see <http://www.gnu.org/licenses/> #
-# #
-# Contact: Cristian.Lalescu@ds.mpg.de #
-# #
-#######################################################################
-
-
-
-import sys
-import os
-import numpy as np
-import h5py
-import argparse
-
-import bfps
-import bfps.tools
-from ._code import _code
-from ._fluid_base import _fluid_particle_base
-
-class NavierStokes(_fluid_particle_base):
- """Objects of this class can be used to generate production DNS codes.
- Any functionality that users require should be available through this class,
- in the sense that they can implement whatever they need by simply inheriting
- this class.
- """
- def __init__(
- self,
- name = 'NavierStokes-v' + bfps.__version__,
- work_dir = './',
- simname = 'test',
- fluid_precision = 'single',
- fftw_plan_rigor = 'FFTW_MEASURE',
- frozen_fields = False,
- use_fftw_wisdom = True,
- QR_stats_on = False,
- Lag_acc_stats_on = False):
- self.QR_stats_on = QR_stats_on
- self.Lag_acc_stats_on = Lag_acc_stats_on
- self.frozen_fields = frozen_fields
- self.fftw_plan_rigor = fftw_plan_rigor
- _fluid_particle_base.__init__(
- self,
- name = name + '-' + fluid_precision,
- work_dir = work_dir,
- simname = simname,
- dtype = fluid_precision,
- use_fftw_wisdom = use_fftw_wisdom)
- self.parameters['nu'] = 0.1
- self.parameters['fmode'] = 1
- self.parameters['famplitude'] = 0.5
- self.parameters['fk0'] = 2.0
- self.parameters['fk1'] = 4.0
- self.parameters['forcing_type'] = 'linear'
- self.parameters['histogram_bins'] = 256
- self.parameters['max_velocity_estimate'] = 1.0
- self.parameters['max_vorticity_estimate'] = 1.0
- self.parameters['max_Lag_acc_estimate'] = 1.0
- self.parameters['max_pressure_estimate'] = 1.0
- self.parameters['QR2D_histogram_bins'] = 64
- self.parameters['max_trS2_estimate'] = 1.0
- self.parameters['max_Q_estimate'] = 1.0
- self.parameters['max_R_estimate'] = 1.0
- self.file_datasets_grow = """
- //begincpp
- hid_t group;
- group = H5Gopen(stat_file, "/statistics", H5P_DEFAULT);
- H5Ovisit(group, H5_INDEX_NAME, H5_ITER_NATIVE, grow_statistics_dataset, NULL);
- H5Gclose(group);
- //endcpp
- """
- self.style = {}
- self.statistics = {}
- self.fluid_output = 'fs->write(\'v\', \'c\');\n'
- return None
- def create_stat_output(
- self,
- dset_name,
- data_buffer,
- data_type = 'H5T_NATIVE_DOUBLE',
- size_setup = None,
- close_spaces = True):
- new_stat_output_txt = 'Cdset = H5Dopen(stat_file, "{0}", H5P_DEFAULT);\n'.format(dset_name)
- if not type(size_setup) == type(None):
- new_stat_output_txt += (
- size_setup +
- 'wspace = H5Dget_space(Cdset);\n' +
- 'ndims = H5Sget_simple_extent_dims(wspace, dims, NULL);\n' +
- 'mspace = H5Screate_simple(ndims, count, NULL);\n' +
- 'H5Sselect_hyperslab(wspace, H5S_SELECT_SET, offset, NULL, count, NULL);\n')
- new_stat_output_txt += ('H5Dwrite(Cdset, {0}, mspace, wspace, H5P_DEFAULT, {1});\n' +
- 'H5Dclose(Cdset);\n').format(data_type, data_buffer)
- if close_spaces:
- new_stat_output_txt += ('H5Sclose(mspace);\n' +
- 'H5Sclose(wspace);\n')
- return new_stat_output_txt
- def write_fluid_stats(self):
- self.fluid_includes += '#include <cmath>\n'
- self.fluid_includes += '#include "fftw_tools.hpp"\n'
- self.stat_src += """
- //begincpp
- hid_t stat_group;
- if (myrank == 0)
- stat_group = H5Gopen(stat_file, "statistics", H5P_DEFAULT);
- fs->compute_velocity(fs->cvorticity);
- std::vector<double> max_estimate_vector;
- max_estimate_vector.resize(4);
- *tmp_vec_field = fs->cvelocity;
- switch(fs->dealias_type)
- {
- case 0:
- tmp_vec_field->compute_stats(
- kk_two_thirds,
- stat_group,
- "velocity",
- fs->iteration / niter_stat,
- max_velocity_estimate/sqrt(3));
- break;
- case 1:
- tmp_vec_field->compute_stats(
- kk_smooth,
- stat_group,
- "velocity",
- fs->iteration / niter_stat,
- max_velocity_estimate/sqrt(3));
- break;
- }
- //endcpp
- """
- if self.Lag_acc_stats_on:
- self.stat_src += """
- //begincpp
- tmp_vec_field->real_space_representation = false;
- fs->compute_Lagrangian_acceleration(tmp_vec_field->get_cdata());
- switch(fs->dealias_type)
- {
- case 0:
- tmp_vec_field->compute_stats(
- kk_two_thirds,
- stat_group,
- "Lagrangian_acceleration",
- fs->iteration / niter_stat,
- max_Lag_acc_estimate);
- break;
- case 1:
- tmp_vec_field->compute_stats(
- kk_smooth,
- stat_group,
- "Lagrangian_acceleration",
- fs->iteration / niter_stat,
- max_Lag_acc_estimate);
- break;
- }
- tmp_scal_field->real_space_representation = false;
- fs->compute_velocity(fs->cvorticity);
- fs->ift_velocity();
- fs->compute_pressure(tmp_scal_field->get_cdata());
- switch(fs->dealias_type)
- {
- case 0:
- tmp_scal_field->compute_stats(
- kk_two_thirds,
- stat_group,
- "pressure",
- fs->iteration / niter_stat,
- max_pressure_estimate);
- break;
- case 1:
- tmp_scal_field->compute_stats(
- kk_smooth,
- stat_group,
- "pressure",
- fs->iteration / niter_stat,
- max_pressure_estimate);
- break;
- }
- //endcpp
- """
- self.stat_src += """
- //begincpp
- *tmp_vec_field = fs->cvorticity;
- switch(fs->dealias_type)
- {
- case 0:
- tmp_vec_field->compute_stats(
- kk_two_thirds,
- stat_group,
- "vorticity",
- fs->iteration / niter_stat,
- max_vorticity_estimate/sqrt(3));
- break;
- case 1:
- tmp_vec_field->compute_stats(
- kk_smooth,
- stat_group,
- "vorticity",
- fs->iteration / niter_stat,
- max_vorticity_estimate/sqrt(3));
- break;
- }
- //endcpp
- """
- if self.QR_stats_on:
- self.stat_src += """
- //begincpp
- double *trS2_Q_R_moments = new double[10*3];
- double *gradu_moments = new double[10*9];
- ptrdiff_t *hist_trS2_Q_R = new ptrdiff_t[histogram_bins*3];
- ptrdiff_t *hist_gradu = new ptrdiff_t[histogram_bins*9];
- ptrdiff_t *hist_QR2D = new ptrdiff_t[QR2D_histogram_bins*QR2D_histogram_bins];
- double trS2QR_max_estimates[3];
- double gradu_max_estimates[9];
- trS2QR_max_estimates[0] = max_trS2_estimate;
- trS2QR_max_estimates[1] = max_Q_estimate;
- trS2QR_max_estimates[2] = max_R_estimate;
- std::fill_n(gradu_max_estimates, 9, sqrt(3*max_trS2_estimate));
- fs->compute_gradient_statistics(
- fs->cvelocity,
- gradu_moments,
- trS2_Q_R_moments,
- hist_gradu,
- hist_trS2_Q_R,
- hist_QR2D,
- trS2QR_max_estimates,
- gradu_max_estimates,
- histogram_bins,
- QR2D_histogram_bins);
- //endcpp
- """
- self.stat_src += """
- //begincpp
- if (myrank == 0)
- H5Gclose(stat_group);
- if (fs->cd->myrank == 0)
- {{
- hid_t Cdset, wspace, mspace;
- int ndims;
- hsize_t count[4], offset[4], dims[4];
- offset[0] = fs->iteration/niter_stat;
- offset[1] = 0;
- offset[2] = 0;
- offset[3] = 0;
- //endcpp
- """.format(self.C_dtype)
- if self.dtype == np.float32:
- field_H5T = 'H5T_NATIVE_FLOAT'
- elif self.dtype == np.float64:
- field_H5T = 'H5T_NATIVE_DOUBLE'
- if self.QR_stats_on:
- self.stat_src += self.create_stat_output(
- '/statistics/moments/trS2_Q_R',
- 'trS2_Q_R_moments',
- size_setup ="""
- count[0] = 1;
- count[1] = 10;
- count[2] = 3;
- """)
- self.stat_src += self.create_stat_output(
- '/statistics/moments/velocity_gradient',
- 'gradu_moments',
- size_setup ="""
- count[0] = 1;
- count[1] = 10;
- count[2] = 3;
- count[3] = 3;
- """)
- self.stat_src += self.create_stat_output(
- '/statistics/histograms/trS2_Q_R',
- 'hist_trS2_Q_R',
- data_type = 'H5T_NATIVE_INT64',
- size_setup = """
- count[0] = 1;
- count[1] = histogram_bins;
- count[2] = 3;
- """)
- self.stat_src += self.create_stat_output(
- '/statistics/histograms/velocity_gradient',
- 'hist_gradu',
- data_type = 'H5T_NATIVE_INT64',
- size_setup = """
- count[0] = 1;
- count[1] = histogram_bins;
- count[2] = 3;
- count[3] = 3;
- """)
- self.stat_src += self.create_stat_output(
- '/statistics/histograms/QR2D',
- 'hist_QR2D',
- data_type = 'H5T_NATIVE_INT64',
- size_setup = """
- count[0] = 1;
- count[1] = QR2D_histogram_bins;
- count[2] = QR2D_histogram_bins;
- """)
- self.stat_src += '}\n'
- if self.QR_stats_on:
- self.stat_src += """
- //begincpp
- delete[] trS2_Q_R_moments;
- delete[] gradu_moments;
- delete[] hist_trS2_Q_R;
- delete[] hist_gradu;
- delete[] hist_QR2D;
- //endcpp
- """
- return None
- def fill_up_fluid_code(self):
- self.fluid_includes += '#include <cstring>\n'
- self.fluid_variables += (
- 'fluid_solver<{0}> *fs;\n'.format(self.C_dtype) +
- 'field<{0}, FFTW, THREE> *tmp_vec_field;\n'.format(self.C_dtype) +
- 'field<{0}, FFTW, ONE> *tmp_scal_field;\n'.format(self.C_dtype) +
- 'kspace<FFTW, SMOOTH> *kk_smooth;\n' +
- 'kspace<FFTW, TWO_THIRDS> *kk_two_thirds;\n')
- self.fluid_definitions += """
- typedef struct {{
- {0} re;
- {0} im;
- }} tmp_complex_type;
- """.format(self.C_dtype)
- self.write_fluid_stats()
- if self.dtype == np.float32:
- field_H5T = 'H5T_NATIVE_FLOAT'
- elif self.dtype == np.float64:
- field_H5T = 'H5T_NATIVE_DOUBLE'
- self.fluid_start += """
- //begincpp
- char fname[512];
- fs = new fluid_solver<{0}>(
- simname,
- nx, ny, nz,
- dkx, dky, dkz,
- dealias_type,
- {1});
- tmp_vec_field = new field<{0}, FFTW, THREE>(
- nx, ny, nz,
- MPI_COMM_WORLD,
- {1});
- tmp_scal_field = new field<{0}, FFTW, ONE>(
- nx, ny, nz,
- MPI_COMM_WORLD,
- {1});
- kk_smooth = new kspace<FFTW, SMOOTH>(
- tmp_vec_field->clayout,
- fs->dkx, fs->dky, fs->dkz);
- kk_two_thirds = new kspace<FFTW, TWO_THIRDS>(
- tmp_vec_field->clayout,
- fs->dkx, fs->dky, fs->dkz);
- fs->nu = nu;
- fs->fmode = fmode;
- fs->famplitude = famplitude;
- fs->fk0 = fk0;
- fs->fk1 = fk1;
- strncpy(fs->forcing_type, forcing_type, 128);
- fs->iteration = iteration;
- fs->read('v', 'c');
- //endcpp
- """.format(self.C_dtype, self.fftw_plan_rigor, field_H5T)
- self.fluid_start += self.store_kspace
- if not self.frozen_fields:
- self.fluid_loop = 'fs->step(dt);\n'
- else:
- self.fluid_loop = ''
- self.fluid_loop += ('if (fs->iteration % niter_out == 0)\n{\n' +
- self.fluid_output + '\n}\n')
- self.fluid_end = ('if (fs->iteration % niter_out != 0)\n{\n' +
- self.fluid_output + '\n}\n' +
- 'delete fs;\n' +
- 'delete tmp_vec_field;\n' +
- 'delete tmp_scal_field;\n' +
- 'delete kk_smooth;\n' +
- 'delete kk_two_thirds;\n')
- return None
- def add_3D_rFFTW_field(
- self,
- name = 'rFFTW_acc'):
- if self.dtype == np.float32:
- FFTW = 'fftwf'
- elif self.dtype == np.float64:
- FFTW = 'fftw'
- self.fluid_variables += '{0} *{1};\n'.format(self.C_dtype, name)
- self.fluid_start += '{0} = {1}_alloc_real(2*fs->cd->local_size);\n'.format(name, FFTW)
- self.fluid_end += '{0}_free({1});\n'.format(FFTW, name)
- return None
- def add_interpolator(
- self,
- interp_type = 'spline',
- neighbours = 1,
- smoothness = 1,
- name = 'field_interpolator',
- field_name = 'fs->rvelocity',
- class_name = 'rFFTW_interpolator'):
- self.fluid_includes += '#include "{0}.hpp"\n'.format(class_name)
- self.fluid_variables += '{0} <{1}, {2}> *{3};\n'.format(
- class_name, self.C_dtype, neighbours, name)
- self.parameters[name + '_type'] = interp_type
- self.parameters[name + '_neighbours'] = neighbours
- if interp_type == 'spline':
- self.parameters[name + '_smoothness'] = smoothness
- beta_name = 'beta_n{0}_m{1}'.format(neighbours, smoothness)
- elif interp_type == 'Lagrange':
- beta_name = 'beta_Lagrange_n{0}'.format(neighbours)
- self.fluid_start += '{0} = new {1}<{2}, {3}>(fs, {4}, {5});\n'.format(
- name,
- class_name,
- self.C_dtype,
- neighbours,
- beta_name,
- field_name)
- self.fluid_end += 'delete {0};\n'.format(name)
- return None
- def add_particles(
- self,
- integration_steps = 2,
- kcut = None,
- interpolator = 'field_interpolator',
- frozen_particles = False,
- acc_name = None,
- class_name = 'particles'):
- """Adds code for tracking a series of particle species, each
- consisting of `nparticles` particles.
-
- :type integration_steps: int, list of int
- :type kcut: None (default), str, list of str
- :type interpolator: str, list of str
- :type frozen_particles: bool
- :type acc_name: str
-
- .. warning:: if not None, kcut must be a list of decreasing
- wavenumbers, since filtering is done sequentially
- on the same complex FFTW field.
- """
- if self.dtype == np.float32:
- FFTW = 'fftwf'
- elif self.dtype == np.float64:
- FFTW = 'fftw'
- s0 = self.particle_species
- if type(integration_steps) == int:
- integration_steps = [integration_steps]
- if type(kcut) == str:
- kcut = [kcut]
- if type(interpolator) == str:
- interpolator = [interpolator]
- nspecies = max(len(integration_steps), len(interpolator))
- if type(kcut) == list:
- nspecies = max(nspecies, len(kcut))
- if len(integration_steps) == 1:
- integration_steps = [integration_steps[0] for s in range(nspecies)]
- if len(interpolator) == 1:
- interpolator = [interpolator[0] for s in range(nspecies)]
- if type(kcut) == list:
- if len(kcut) == 1:
- kcut = [kcut[0] for s in range(nspecies)]
- assert(len(integration_steps) == nspecies)
- assert(len(interpolator) == nspecies)
- if type(kcut) == list:
- assert(len(kcut) == nspecies)
- for s in range(nspecies):
- neighbours = self.parameters[interpolator[s] + '_neighbours']
- if type(kcut) == list:
- self.parameters['tracers{0}_kcut'.format(s0 + s)] = kcut[s]
- self.parameters['tracers{0}_interpolator'.format(s0 + s)] = interpolator[s]
- self.parameters['tracers{0}_acc_on'.format(s0 + s)] = int(not type(acc_name) == type(None))
- self.parameters['tracers{0}_integration_steps'.format(s0 + s)] = integration_steps[s]
- self.file_datasets_grow += """
- //begincpp
- group = H5Gopen(particle_file, "/tracers{0}", H5P_DEFAULT);
- grow_particle_datasets(group, "", NULL, NULL);
- H5Gclose(group);
- //endcpp
- """.format(s0 + s)
-
- #### code that outputs statistics
- output_vel_acc = '{\n'
- # array for putting sampled velocity in
- # must compute velocity, just in case it was messed up by some
- # other particle species before the stats
- output_vel_acc += 'fs->compute_velocity(fs->cvorticity);\n'
- if not type(kcut) == list:
- output_vel_acc += 'fs->ift_velocity();\n'
- if not type(acc_name) == type(None):
- # array for putting sampled acceleration in
- # must compute acceleration
- output_vel_acc += 'fs->compute_Lagrangian_acceleration({0});\n'.format(acc_name)
- for s in range(nspecies):
- if type(kcut) == list:
- output_vel_acc += 'fs->low_pass_Fourier(fs->cvelocity, 3, {0});\n'.format(kcut[s])
- output_vel_acc += 'fs->ift_velocity();\n'
- output_vel_acc += """
- {0}->read_rFFTW(fs->rvelocity);
- ps{1}->sample({0}, "velocity");
- """.format(interpolator[s], s0 + s)
- if not type(acc_name) == type(None):
- output_vel_acc += """
- {0}->read_rFFTW({1});
- ps{2}->sample({0}, "acceleration");
- """.format(interpolator[s], acc_name, s0 + s)
- output_vel_acc += '}\n'
-
- #### initialize, stepping and finalize code
- if not type(kcut) == list:
- update_fields = ('fs->compute_velocity(fs->cvorticity);\n' +
- 'fs->ift_velocity();\n')
- self.particle_start += update_fields
- self.particle_loop += update_fields
- else:
- self.particle_loop += 'fs->compute_velocity(fs->cvorticity);\n'
- self.particle_includes += '#include "{0}.hpp"\n'.format(class_name)
- self.particle_stat_src += (
- 'if (ps0->iteration % niter_part == 0)\n' +
- '{\n')
- for s in range(nspecies):
- neighbours = self.parameters[interpolator[s] + '_neighbours']
- self.particle_start += 'sprintf(fname, "tracers{0}");\n'.format(s0 + s)
- self.particle_end += ('ps{0}->write();\n' +
- 'delete ps{0};\n').format(s0 + s)
- self.particle_variables += '{0}<VELOCITY_TRACER, {1}, {2}> *ps{3};\n'.format(
- class_name,
- self.C_dtype,
- neighbours,
- s0 + s)
- self.particle_start += ('ps{0} = new {1}<VELOCITY_TRACER, {2}, {3}>(\n' +
- 'fname, particle_file, {4},\n' +
- 'niter_part, tracers{0}_integration_steps);\n').format(
- s0 + s,
- class_name,
- self.C_dtype,
- neighbours,
- interpolator[s])
- self.particle_start += ('ps{0}->dt = dt;\n' +
- 'ps{0}->iteration = iteration;\n' +
- 'ps{0}->read();\n').format(s0 + s)
- if not frozen_particles:
- if type(kcut) == list:
- update_field = ('fs->low_pass_Fourier(fs->cvelocity, 3, {0});\n'.format(kcut[s]) +
- 'fs->ift_velocity();\n')
- self.particle_loop += update_field
- self.particle_loop += '{0}->read_rFFTW(fs->rvelocity);\n'.format(interpolator[s])
- self.particle_loop += 'ps{0}->step();\n'.format(s0 + s)
- self.particle_stat_src += 'ps{0}->write(false);\n'.format(s0 + s)
- self.particle_stat_src += output_vel_acc
- self.particle_stat_src += '}\n'
- self.particle_species += nspecies
- return None
- def get_cache_file_name(self):
- return os.path.join(self.work_dir, self.simname + '_cache.h5')
- def get_cache_file(self):
- return h5py.File(self.get_postprocess_file_name(), 'r')
- def get_postprocess_file_name(self):
- return self.get_cache_file_name()
- def get_postprocess_file(self):
- return h5py.File(self.get_postprocess_file_name(), 'r')
- def compute_statistics(self, iter0 = 0, iter1 = None):
- """Run basic postprocessing on raw data.
- The energy spectrum :math:`E(t, k)` and the enstrophy spectrum
- :math:`\\frac{1}{2}\omega^2(t, k)` are computed from the
-
- .. math::
-
- \sum_{k \\leq \\|\\mathbf{k}\\| \\leq k+dk}\\hat{u_i} \\hat{u_j}^*, \\hskip .5cm
- \sum_{k \\leq \\|\\mathbf{k}\\| \\leq k+dk}\\hat{\omega_i} \\hat{\\omega_j}^*
-
- tensors, and the enstrophy spectrum is also used to
- compute the dissipation :math:`\\varepsilon(t)`.
- These basic quantities are stored in a newly created HDF5 file,
- ``simname_cache.h5``.
- """
- if len(list(self.statistics.keys())) > 0:
- return None
- self.read_parameters()
- with self.get_data_file() as data_file:
- if 'moments' not in data_file['statistics'].keys():
- return None
- iter0 = min((data_file['statistics/moments/velocity'].shape[0] *
- self.parameters['niter_stat']-1),
- iter0)
- if type(iter1) == type(None):
- iter1 = data_file['iteration'].value
- else:
- iter1 = min(data_file['iteration'].value, iter1)
- ii0 = iter0 // self.parameters['niter_stat']
- ii1 = iter1 // self.parameters['niter_stat']
- self.statistics['kshell'] = data_file['kspace/kshell'].value
- self.statistics['kM'] = data_file['kspace/kM'].value
- self.statistics['dk'] = data_file['kspace/dk'].value
- computation_needed = True
- pp_file = h5py.File(self.get_postprocess_file_name(), 'a')
- if 'ii0' in pp_file.keys():
- computation_needed = not (ii0 == pp_file['ii0'].value and
- ii1 == pp_file['ii1'].value)
- if computation_needed:
- for k in ['t', 'vel_max(t)', 'renergy(t)',
- 'energy(t, k)', 'enstrophy(t, k)',
- 'ii0', 'ii1', 'iter0', 'iter1']:
- del pp_file[k]
- if computation_needed:
- pp_file['iter0'] = iter0
- pp_file['iter1'] = iter1
- pp_file['ii0'] = ii0
- pp_file['ii1'] = ii1
- pp_file['t'] = (self.parameters['dt']*
- self.parameters['niter_stat']*
- (np.arange(ii0, ii1+1).astype(np.float)))
- pp_file['energy(t, k)'] = (
- data_file['statistics/spectra/velocity_velocity'][ii0:ii1+1, :, 0, 0] +
- data_file['statistics/spectra/velocity_velocity'][ii0:ii1+1, :, 1, 1] +
- data_file['statistics/spectra/velocity_velocity'][ii0:ii1+1, :, 2, 2])/2
- pp_file['enstrophy(t, k)'] = (
- data_file['statistics/spectra/vorticity_vorticity'][ii0:ii1+1, :, 0, 0] +
- data_file['statistics/spectra/vorticity_vorticity'][ii0:ii1+1, :, 1, 1] +
- data_file['statistics/spectra/vorticity_vorticity'][ii0:ii1+1, :, 2, 2])/2
- pp_file['vel_max(t)'] = data_file['statistics/moments/velocity'] [ii0:ii1+1, 9, 3]
- pp_file['renergy(t)'] = data_file['statistics/moments/velocity'][ii0:ii1+1, 2, 3]/2
- if 'trS2_Q_R' in data_file['statistics/moments'].keys():
- pp_file['mean_trS2(t)'] = data_file['statistics/moments/trS2_Q_R'][:, 1, 0]
- for k in ['t',
- 'energy(t, k)',
- 'enstrophy(t, k)',
- 'vel_max(t)',
- 'renergy(t)',
- 'mean_trS2(t)']:
- if k in pp_file.keys():
- self.statistics[k] = pp_file[k].value
- self.compute_time_averages()
- return None
- def compute_time_averages(self):
- """Compute easy stats.
-
- Further computation of statistics based on the contents of
- ``simname_cache.h5``.
- Standard quantities are as follows
- (consistent with [Ishihara]_):
-
- .. math::
-
- U_{\\textrm{int}}(t) = \\sqrt{\\frac{2E(t)}{3}}, \\hskip .5cm
- L_{\\textrm{int}}(t) = \\frac{\pi}{2U_{int}^2(t)} \\int \\frac{dk}{k} E(t, k), \\hskip .5cm
- T_{\\textrm{int}}(t) =
- \\frac{L_{\\textrm{int}}(t)}{U_{\\textrm{int}}(t)}
-
- \\eta_K = \\left(\\frac{\\nu^3}{\\varepsilon}\\right)^{1/4}, \\hskip .5cm
- \\tau_K = \\left(\\frac{\\nu}{\\varepsilon}\\right)^{1/2}, \\hskip .5cm
- \\lambda = \\sqrt{\\frac{15 \\nu U_{\\textrm{int}}^2}{\\varepsilon}}
-
- Re = \\frac{U_{\\textrm{int}} L_{\\textrm{int}}}{\\nu}, \\hskip
- .5cm
- R_{\\lambda} = \\frac{U_{\\textrm{int}} \\lambda}{\\nu}
-
- .. [Ishihara] T. Ishihara et al,
- *Small-scale statistics in high-resolution direct numerical
- simulation of turbulence: Reynolds number dependence of
- one-point velocity gradient statistics*.
- J. Fluid Mech.,
- **592**, 335-366, 2007
- """
- for key in ['energy', 'enstrophy']:
- self.statistics[key + '(t)'] = (self.statistics['dk'] *
- np.sum(self.statistics[key + '(t, k)'], axis = 1))
- self.statistics['Uint(t)'] = np.sqrt(2*self.statistics['energy(t)'] / 3)
- self.statistics['Lint(t)'] = ((self.statistics['dk']*np.pi /
- (2*self.statistics['Uint(t)']**2)) *
- np.nansum(self.statistics['energy(t, k)'] /
- self.statistics['kshell'][None, :], axis = 1))
- for key in ['energy',
- 'enstrophy',
- 'vel_max',
- 'mean_trS2',
- 'Uint',
- 'Lint']:
- if key + '(t)' in self.statistics.keys():
- self.statistics[key] = np.average(self.statistics[key + '(t)'], axis = 0)
- for suffix in ['', '(t)']:
- self.statistics['diss' + suffix] = (self.parameters['nu'] *
- self.statistics['enstrophy' + suffix]*2)
- self.statistics['etaK' + suffix] = (self.parameters['nu']**3 /
- self.statistics['diss' + suffix])**.25
- self.statistics['tauK' + suffix] = (self.parameters['nu'] /
- self.statistics['diss' + suffix])**.5
- self.statistics['Re' + suffix] = (self.statistics['Uint' + suffix] *
- self.statistics['Lint' + suffix] /
- self.parameters['nu'])
- self.statistics['lambda' + suffix] = (15 * self.parameters['nu'] *
- self.statistics['Uint' + suffix]**2 /
- self.statistics['diss' + suffix])**.5
- self.statistics['Rlambda' + suffix] = (self.statistics['Uint' + suffix] *
- self.statistics['lambda' + suffix] /
- self.parameters['nu'])
- self.statistics['kMeta' + suffix] = (self.statistics['kM'] *
- self.statistics['etaK' + suffix])
- if self.parameters['dealias_type'] == 1:
- self.statistics['kMeta' + suffix] *= 0.8
- self.statistics['Tint'] = self.statistics['Lint'] / self.statistics['Uint']
- self.statistics['Taylor_microscale'] = self.statistics['lambda']
- return None
- def set_plt_style(
- self,
- style = {'dashes' : (None, None)}):
- self.style.update(style)
- return None
- def read_cfield(
- self,
- field_name = 'vorticity',
- iteration = 0):
- """read the Fourier representation of a vector field.
-
- Read the binary file containing iteration ``iteration`` of the
- field ``field_name``, and return it as a properly shaped
- ``numpy.memmap`` object.
- """
- return np.memmap(
- os.path.join(self.work_dir,
- self.simname + '_{0}_i{1:0>5x}'.format('c' + field_name, iteration)),
- dtype = self.ctype,
- mode = 'r',
- shape = (self.parameters['ny'],
- self.parameters['nz'],
- self.parameters['nx']//2+1,
- 3))
- def write_par(
- self,
- iter0 = 0,
- particle_ic = None):
- _fluid_particle_base.write_par(self, iter0 = iter0)
- with h5py.File(self.get_data_file_name(), 'r+') as ofile:
- kspace = self.get_kspace()
- nshells = kspace['nshell'].shape[0]
- vec_stat_datasets = ['velocity', 'vorticity']
- scal_stat_datasets = []
- for k in vec_stat_datasets:
- time_chunk = 2**20 // (
- self.dtype.itemsize*3*
- self.parameters['nx']*self.parameters['ny'])
- time_chunk = max(time_chunk, 1)
- ofile.create_dataset('statistics/0slices/' + k + '/real',
- (1, self.parameters['ny'], self.parameters['nx'], 3),
- chunks = (time_chunk, self.parameters['ny'], self.parameters['nx'], 3),
- maxshape = (None, self.parameters['ny'], self.parameters['nx'], 3),
- dtype = self.dtype)
- if self.Lag_acc_stats_on:
- vec_stat_datasets += ['Lagrangian_acceleration']
- scal_stat_datasets += ['pressure']
- for k in vec_stat_datasets:
- time_chunk = 2**20//(8*3*3*nshells)
- time_chunk = max(time_chunk, 1)
- ofile.create_dataset('statistics/spectra/' + k + '_' + k,
- (1, nshells, 3, 3),
- chunks = (time_chunk, nshells, 3, 3),
- maxshape = (None, nshells, 3, 3),
- dtype = np.float64)
- time_chunk = 2**20//(8*4*10)
- time_chunk = max(time_chunk, 1)
- a = ofile.create_dataset('statistics/moments/' + k,
- (1, 10, 4),
- chunks = (time_chunk, 10, 4),
- maxshape = (None, 10, 4),
- dtype = np.float64)
- time_chunk = 2**20//(8*4*self.parameters['histogram_bins'])
- time_chunk = max(time_chunk, 1)
- ofile.create_dataset('statistics/histograms/' + k,
- (1,
- self.parameters['histogram_bins'],
- 4),
- chunks = (time_chunk,
- self.parameters['histogram_bins'],
- 4),
- maxshape = (None,
- self.parameters['histogram_bins'],
- 4),
- dtype = np.int64)
- for k in scal_stat_datasets:
- time_chunk = 2**20//(8*nshells)
- time_chunk = max(time_chunk, 1)
- ofile.create_dataset('statistics/spectra/' + k + '_' + k,
- (1, nshells),
- chunks = (time_chunk, nshells),
- maxshape = (None, nshells),
- dtype = np.float64)
- time_chunk = 2**20//(8*10)
- time_chunk = max(time_chunk, 1)
- a = ofile.create_dataset('statistics/moments/' + k,
- (1, 10),
- chunks = (time_chunk, 10),
- maxshape = (None, 10),
- dtype = np.float64)
- time_chunk = 2**20//(8*self.parameters['histogram_bins'])
- time_chunk = max(time_chunk, 1)
- ofile.create_dataset('statistics/histograms/' + k,
- (1,
- self.parameters['histogram_bins']),
- chunks = (time_chunk,
- self.parameters['histogram_bins']),
- maxshape = (None,
- self.parameters['histogram_bins']),
- dtype = np.int64)
- if self.QR_stats_on:
- time_chunk = 2**20//(8*3*self.parameters['histogram_bins'])
- time_chunk = max(time_chunk, 1)
- ofile.create_dataset('statistics/histograms/trS2_Q_R',
- (1,
- self.parameters['histogram_bins'],
- 3),
- chunks = (time_chunk,
- self.parameters['histogram_bins'],
- 3),
- maxshape = (None,
- self.parameters['histogram_bins'],
- 3),
- dtype = np.int64)
- time_chunk = 2**20//(8*9*self.parameters['histogram_bins'])
- time_chunk = max(time_chunk, 1)
- ofile.create_dataset('statistics/histograms/velocity_gradient',
- (1,
- self.parameters['histogram_bins'],
- 3,
- 3),
- chunks = (time_chunk,
- self.parameters['histogram_bins'],
- 3,
- 3),
- maxshape = (None,
- self.parameters['histogram_bins'],
- 3,
- 3),
- dtype = np.int64)
- time_chunk = 2**20//(8*3*10)
- time_chunk = max(time_chunk, 1)
- a = ofile.create_dataset('statistics/moments/trS2_Q_R',
- (1, 10, 3),
- chunks = (time_chunk, 10, 3),
- maxshape = (None, 10, 3),
- dtype = np.float64)
- time_chunk = 2**20//(8*9*10)
- time_chunk = max(time_chunk, 1)
- a = ofile.create_dataset('statistics/moments/velocity_gradient',
- (1, 10, 3, 3),
- chunks = (time_chunk, 10, 3, 3),
- maxshape = (None, 10, 3, 3),
- dtype = np.float64)
- time_chunk = 2**20//(8*self.parameters['QR2D_histogram_bins']**2)
- time_chunk = max(time_chunk, 1)
- ofile.create_dataset('statistics/histograms/QR2D',
- (1,
- self.parameters['QR2D_histogram_bins'],
- self.parameters['QR2D_histogram_bins']),
- chunks = (time_chunk,
- self.parameters['QR2D_histogram_bins'],
- self.parameters['QR2D_histogram_bins']),
- maxshape = (None,
- self.parameters['QR2D_histogram_bins'],
- self.parameters['QR2D_histogram_bins']),
- dtype = np.int64)
- if self.particle_species == 0:
- return None
-
- if type(particle_ic) == type(None):
- pbase_shape = (self.parameters['nparticles'],)
- number_of_particles = self.parameters['nparticles']
- else:
- pbase_shape = particle_ic.shape[:-1]
- assert(particle_ic.shape[-1] == 3)
- if len(pbase_shape) == 1:
- number_of_particles = pbase_shape[0]
- else:
- number_of_particles = 1
- for val in pbase_shape[1:]:
- number_of_particles *= val
-
- with h5py.File(self.get_particle_file_name(), 'a') as ofile:
- for s in range(self.particle_species):
- ofile.create_group('tracers{0}'.format(s))
- time_chunk = 2**20 // (8*3*number_of_particles)
- time_chunk = max(time_chunk, 1)
- dims = ((1,
- self.parameters['tracers{0}_integration_steps'.format(s)]) +
- pbase_shape + (3,))
- maxshape = (h5py.h5s.UNLIMITED,) + dims[1:]
- if len(pbase_shape) > 1:
- chunks = (time_chunk, 1, 1) + dims[3:]
- else:
- chunks = (time_chunk, 1) + dims[2:]
- bfps.tools.create_alloc_early_dataset(
- ofile,
- '/tracers{0}/rhs'.format(s),
- dims, maxshape, chunks)
- if len(pbase_shape) > 1:
- chunks = (time_chunk, 1) + pbase_shape[1:] + (3,)
- else:
- chunks = (time_chunk, pbase_shape[0], 3)
- bfps.tools.create_alloc_early_dataset(
- ofile,
- '/tracers{0}/state'.format(s),
- (1,) + pbase_shape + (3,),
- (h5py.h5s.UNLIMITED,) + pbase_shape + (3,),
- chunks)
- # "velocity" is sampled, single precision is enough
- # for the results we are interested in.
- bfps.tools.create_alloc_early_dataset(
- ofile,
- '/tracers{0}/velocity'.format(s),
- (1,) + pbase_shape + (3,),
- (h5py.h5s.UNLIMITED,) + pbase_shape + (3,),
- chunks,
- dset_dtype = h5py.h5t.IEEE_F32LE)
- if self.parameters['tracers{0}_acc_on'.format(s)]:
- bfps.tools.create_alloc_early_dataset(
- ofile,
- '/tracers{0}/acceleration'.format(s),
- (1,) + pbase_shape + (3,),
- (h5py.h5s.UNLIMITED,) + pbase_shape + (3,),
- chunks,
- dset_dtype = h5py.h5t.IEEE_F32LE)
- return None
- def add_particle_fields(
- self,
- interp_type = 'spline',
- kcut = None,
- neighbours = 1,
- smoothness = 1,
- name = 'particle_field',
- field_class = 'rFFTW_interpolator',
- acc_field_name = 'rFFTW_acc'):
- self.fluid_includes += '#include "{0}.hpp"\n'.format(field_class)
- self.fluid_variables += field_class + '<{0}, {1}> *vel_{2}, *acc_{2};\n'.format(
- self.C_dtype, neighbours, name)
- self.parameters[name + '_type'] = interp_type
- self.parameters[name + '_neighbours'] = neighbours
- if interp_type == 'spline':
- self.parameters[name + '_smoothness'] = smoothness
- beta_name = 'beta_n{0}_m{1}'.format(neighbours, smoothness)
- elif interp_type == 'Lagrange':
- beta_name = 'beta_Lagrange_n{0}'.format(neighbours)
- if field_class == 'rFFTW_interpolator':
- self.fluid_start += ('vel_{0} = new {1}<{2}, {3}>(fs, {4}, fs->rvelocity);\n' +
- 'acc_{0} = new {1}<{2}, {3}>(fs, {4}, {5});\n').format(name,
- field_class,
- self.C_dtype,
- neighbours,
- beta_name,
- acc_field_name)
- elif field_class == 'interpolator':
- self.fluid_start += ('vel_{0} = new {1}<{2}, {3}>(fs, {4});\n' +
- 'acc_{0} = new {1}<{2}, {3}>(fs, {4});\n').format(name,
- field_class,
- self.C_dtype,
- neighbours,
- beta_name,
- acc_field_name)
- self.fluid_end += ('delete vel_{0};\n' +
- 'delete acc_{0};\n').format(name)
- update_fields = 'fs->compute_velocity(fs->cvorticity);\n'
- if not type(kcut) == type(None):
- update_fields += 'fs->low_pass_Fourier(fs->cvelocity, 3, {0});\n'.format(kcut)
- update_fields += ('fs->ift_velocity();\n' +
- 'fs->compute_Lagrangian_acceleration(acc_{0}->field);\n').format(name)
- self.fluid_start += update_fields
- self.fluid_loop += update_fields
- return None
- def specific_parser_arguments(
- self,
- parser):
- _fluid_particle_base.specific_parser_arguments(self, parser)
- parser.add_argument(
- '--src-wd',
- type = str,
- dest = 'src_work_dir',
- default = '')
- parser.add_argument(
- '--src-simname',
- type = str,
- dest = 'src_simname',
- default = '')
- parser.add_argument(
- '--src-iteration',
- type = int,
- dest = 'src_iteration',
- default = 0)
- parser.add_argument(
- '--njobs',
- type = int, dest = 'njobs',
- default = 1)
- parser.add_argument(
- '--QR-stats',
- action = 'store_true',
- dest = 'QR_stats',
- help = 'add this option if you want to compute velocity gradient and QR stats')
- parser.add_argument(
- '--Lag-acc-stats',
- action = 'store_true',
- dest = 'Lag_acc_stats',
- help = 'add this option if you want to compute Lagrangian acceleration statistics')
- parser.add_argument(
- '--kMeta',
- type = float,
- dest = 'kMeta',
- default = 2.0)
- parser.add_argument(
- '--dtfactor',
- type = float,
- dest = 'dtfactor',
- default = 0.5,
- help = 'dt is computed as DTFACTOR / N')
- parser.add_argument(
- '--particle-rand-seed',
- type = int,
- dest = 'particle_rand_seed',
- default = None)
- parser.add_argument(
- '--pclouds',
- type = int,
- dest = 'pclouds',
- default = 1,
- help = ('number of particle clouds. Particle "clouds" '
- 'consist of particles distributed according to '
- 'pcloud-type.'))
- parser.add_argument(
- '--pcloud-type',
- choices = ['random-cube',
- 'regular-cube'],
- dest = 'pcloud_type',
- default = 'random-cube')
- parser.add_argument(
- '--particle-cloud-size',
- type = float,
- dest = 'particle_cloud_size',
- default = 2*np.pi)
- parser.add_argument(
- '--neighbours',
- type = int,
- dest = 'neighbours',
- default = 1)
- parser.add_argument(
- '--smoothness',
- type = int,
- dest = 'smoothness',
- default = 1)
- return None
- def prepare_launch(
- self,
- args = []):
- """Set up reasonable parameters.
-
- With the default Lundgren forcing applied in the band [2, 4],
- we can estimate the dissipation, therefore we can estimate
- :math:`k_M \\eta_K` and constrain the viscosity.
- Also, if velocity gradient statistics are computed, the
- dissipation is used for estimating the bins of the QR histogram.
-
- In brief, the command line parameter :math:`k_M \\eta_K` is
- used in the following formula for :math:`\\nu` (:math:`N` is the
- number of real space grid points per coordinate):
-
- .. math::
-
- \\nu = \\left(\\frac{2 k_M \\eta_K}{N} \\right)^{4/3}
-
- With this choice, the average dissipation :math:`\\varepsilon`
- will be close to 0.4, and the integral scale velocity will be
- close to 0.77, yielding the approximate value for the Taylor
- microscale and corresponding Reynolds number:
-
- .. math::
-
- \\lambda \\approx 4.75\\left(\\frac{2 k_M \\eta_K}{N} \\right)^{4/6}, \\hskip .5in
- R_\\lambda \\approx 3.7 \\left(\\frac{N}{2 k_M \\eta_K} \\right)^{4/6}
-
- """
- opt = _code.prepare_launch(self, args = args)
- self.QR_stats_on = opt.QR_stats
- self.Lag_acc_stats_on = opt.Lag_acc_stats
- self.parameters['nu'] = (opt.kMeta * 2 / opt.n)**(4./3)
- self.parameters['dt'] = (opt.dtfactor / opt.n)
- # custom famplitude for 288 and 576
- if opt.n == 288:
- self.parameters['famplitude'] = 0.45
- elif opt.n == 576:
- self.parameters['famplitude'] = 0.47
- if ((self.parameters['niter_todo'] % self.parameters['niter_out']) != 0):
- self.parameters['niter_out'] = self.parameters['niter_todo']
- if self.QR_stats_on:
- # max_Q_estimate and max_R_estimate are just used for the 2D pdf
- # therefore I just want them to be small multiples of mean trS2
- # I'm already estimating the dissipation with kMeta...
- meantrS2 = (opt.n//2 / opt.kMeta)**4 * self.parameters['nu']**2
- self.parameters['max_Q_estimate'] = meantrS2
- self.parameters['max_R_estimate'] = .4*meantrS2**1.5
- # add QR suffix to code name, since we now expect additional
- # datasets in the .h5 file
- self.name += '-QR'
- if self.Lag_acc_stats_on:
- self.name += '-Lag_acc'
- if len(opt.src_work_dir) == 0:
- opt.src_work_dir = os.path.realpath(opt.work_dir)
- self.pars_from_namespace(opt)
- return opt
- def launch(
- self,
- args = [],
- noparticles = False,
- **kwargs):
- opt = self.prepare_launch(args = args)
- self.fill_up_fluid_code()
- if noparticles:
- opt.nparticles = 0
- elif type(opt.nparticles) == int:
- if opt.nparticles > 0:
- self.name += '-particles'
- self.add_3D_rFFTW_field(
- name = 'rFFTW_acc')
- self.add_interpolator(
- name = 'cubic_spline',
- neighbours = opt.neighbours,
- smoothness = opt.smoothness,
- class_name = 'rFFTW_interpolator')
- self.add_particles(
- integration_steps = [4],
- interpolator = 'cubic_spline',
- acc_name = 'rFFTW_acc',
- class_name = 'rFFTW_distributed_particles')
- self.variables += 'hid_t particle_file;\n'
- self.main_start += """
- if (myrank == 0)
- {
- // set caching parameters
- hid_t fapl = H5Pcreate(H5P_FILE_ACCESS);
- herr_t cache_err = H5Pset_cache(fapl, 0, 521, 134217728, 1.0);
- DEBUG_MSG("when setting cache for particles I got %d\\n", cache_err);
- sprintf(fname, "%s_particles.h5", simname);
- particle_file = H5Fopen(fname, H5F_ACC_RDWR, fapl);
- }
- """
- self.main_end = ('if (myrank == 0)\n' +
- '{\n' +
- 'H5Fclose(particle_file);\n' +
- '}\n') + self.main_end
- self.finalize_code()
- self.launch_jobs(opt = opt, **kwargs)
- return None
- def launch_jobs(
- self,
- opt = None,
- particle_initial_condition = None):
- if not os.path.exists(os.path.join(self.work_dir, self.simname + '.h5')):
- if opt.pclouds > 1:
- np.random.seed(opt.particle_rand_seed)
- if opt.pcloud_type == 'random-cube':
- particle_initial_condition = (
- np.random.random((opt.pclouds, 1, 3))*2*np.pi +
- np.random.random((1, self.parameters['nparticles'], 3))*opt.particle_cloud_size)
- elif opt.pcloud_type == 'regular-cube':
- onedarray = np.linspace(
- -opt.particle_cloud_size/2,
- opt.particle_cloud_size/2,
- self.parameters['nparticles'])
- particle_initial_condition = np.zeros(
- (opt.pclouds,
- self.parameters['nparticles'],
- self.parameters['nparticles'],
- self.parameters['nparticles'], 3),
- dtype = np.float64)
- particle_initial_condition[:] = \
- np.random.random((opt.pclouds, 1, 1, 1, 3))*2*np.pi
- particle_initial_condition[..., 0] += onedarray[None, None, None, :]
- particle_initial_condition[..., 1] += onedarray[None, None, :, None]
- particle_initial_condition[..., 2] += onedarray[None, :, None, None]
- self.write_par(
- particle_ic = particle_initial_condition)
- if self.parameters['nparticles'] > 0:
- data = self.generate_tracer_state(
- species = 0,
- rseed = opt.particle_rand_seed,
- data = particle_initial_condition)
- for s in range(1, self.particle_species):
- self.generate_tracer_state(species = s, data = data)
- init_condition_file = os.path.join(
- self.work_dir,
- self.simname + '_cvorticity_i{0:0>5x}'.format(0))
- if not os.path.exists(init_condition_file):
- if len(opt.src_simname) > 0:
- src_file = os.path.join(
- os.path.realpath(opt.src_work_dir),
- opt.src_simname + '_cvorticity_i{0:0>5x}'.format(opt.src_iteration))
- os.symlink(src_file, init_condition_file)
- else:
- self.generate_vector_field(
- write_to_file = True,
- spectra_slope = 2.0,
- amplitude = 0.05)
- self.run(
- nb_processes = opt.nb_processes,
- nb_threads_per_process = opt.nb_threads_per_process,
- njobs = opt.njobs,
- hours = opt.minutes // 60,
- minutes = opt.minutes % 60,
- no_submit = opt.no_submit)
- return None
-
diff --git a/bfps/TEST.py b/bfps/TEST.py
deleted file mode 100644
index 5f5734030344f15c7b23d7849fede80105e11fc6..0000000000000000000000000000000000000000
--- a/bfps/TEST.py
+++ /dev/null
@@ -1,293 +0,0 @@
-#######################################################################
-# #
-# Copyright 2015 Max Planck Institute #
-# for Dynamics and Self-Organization #
-# #
-# This file is part of bfps. #
-# #
-# bfps is free software: you can redistribute it and/or modify #
-# it under the terms of the GNU General Public License as published #
-# by the Free Software Foundation, either version 3 of the License, #
-# or (at your option) any later version. #
-# #
-# bfps is distributed in the hope that it will be useful, #
-# but WITHOUT ANY WARRANTY; without even the implied warranty of #
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the #
-# GNU General Public License for more details. #
-# #
-# You should have received a copy of the GNU General Public License #
-# along with bfps. If not, see <http://www.gnu.org/licenses/> #
-# #
-# Contact: Cristian.Lalescu@ds.mpg.de #
-# #
-#######################################################################
-
-
-
-import os
-import sys
-import shutil
-import subprocess
-import argparse
-import h5py
-import math
-import numpy as np
-import warnings
-
-import bfps
-from ._code import _code
-from bfps import tools
-
-class TEST(_code):
- """This class is meant to stitch together the C++ code into a final source file,
- compile it, and handle all job launching.
- """
- def __init__(
- self,
- work_dir = './',
- simname = 'test'):
- _code.__init__(
- self,
- work_dir = work_dir,
- simname = simname)
- self.host_info = {'type' : 'cluster',
- 'environment' : None,
- 'deltanprocs' : 1,
- 'queue' : '',
- 'mail_address': '',
- 'mail_events' : None}
- self.generate_default_parameters()
- return None
- def set_precision(
- self,
- fluid_dtype):
- if fluid_dtype in [np.float32, np.float64]:
- self.fluid_dtype = fluid_dtype
- elif fluid_dtype in ['single', 'double']:
- if fluid_dtype == 'single':
- self.fluid_dtype = np.dtype(np.float32)
- elif fluid_dtype == 'double':
- self.fluid_dtype = np.dtype(np.float64)
- self.rtype = self.fluid_dtype
- if self.rtype == np.float32:
- self.ctype = np.dtype(np.complex64)
- self.C_field_dtype = 'float'
- self.fluid_precision = 'single'
- elif self.rtype == np.float64:
- self.ctype = np.dtype(np.complex128)
- self.C_field_dtype = 'double'
- self.fluid_precision = 'double'
- return None
- def write_src(self):
- self.version_message = (
- '/***********************************************************************\n' +
- '* this code automatically generated by bfps\n' +
- '* version {0}\n'.format(bfps.__version__) +
- '***********************************************************************/\n\n\n')
- self.include_list = [
- '"base.hpp"',
- '"scope_timer.hpp"',
- '"fftw_interface.hpp"',
- '"full_code/main_code.hpp"',
- '<cmath>',
- '<iostream>',
- '<hdf5.h>',
- '<string>',
- '<cstring>',
- '<fftw3-mpi.h>',
- '<omp.h>',
- '<cfenv>',
- '<cstdlib>',
- '"full_code/{0}.hpp"\n'.format(self.dns_type)]
- self.main = """
- int main(int argc, char *argv[])
- {{
- bool fpe = (
- (getenv("BFPS_FPE_OFF") == nullptr) ||
- (getenv("BFPS_FPE_OFF") != std::string("TRUE")));
- return main_code< {0} >(argc, argv, fpe);
- }}
- """.format(self.dns_type + '<{0}>'.format(self.C_field_dtype))
- self.includes = '\n'.join(
- ['#include ' + hh
- for hh in self.include_list])
- with open(self.name + '.cpp', 'w') as outfile:
- outfile.write(self.version_message + '\n\n')
- outfile.write(self.includes + '\n\n')
- outfile.write(self.main + '\n')
- return None
- def generate_default_parameters(self):
- # these parameters are relevant for all TEST classes
- self.parameters['dealias_type'] = int(1)
- self.parameters['dkx'] = float(1.0)
- self.parameters['dky'] = float(1.0)
- self.parameters['dkz'] = float(1.0)
- self.parameters['filter_length'] = float(1.0)
- return None
- def get_kspace(self):
- kspace = {}
- if self.parameters['dealias_type'] == 1:
- kMx = self.parameters['dkx']*(self.parameters['nx']//2 - 1)
- kMy = self.parameters['dky']*(self.parameters['ny']//2 - 1)
- kMz = self.parameters['dkz']*(self.parameters['nz']//2 - 1)
- else:
- kMx = self.parameters['dkx']*(self.parameters['nx']//3 - 1)
- kMy = self.parameters['dky']*(self.parameters['ny']//3 - 1)
- kMz = self.parameters['dkz']*(self.parameters['nz']//3 - 1)
- kspace['kM'] = max(kMx, kMy, kMz)
- kspace['dk'] = min(self.parameters['dkx'],
- self.parameters['dky'],
- self.parameters['dkz'])
- nshells = int(kspace['kM'] / kspace['dk']) + 2
- kspace['nshell'] = np.zeros(nshells, dtype = np.int64)
- kspace['kshell'] = np.zeros(nshells, dtype = np.float64)
- kspace['kx'] = np.arange( 0,
- self.parameters['nx']//2 + 1).astype(np.float64)*self.parameters['dkx']
- kspace['ky'] = np.arange(-self.parameters['ny']//2 + 1,
- self.parameters['ny']//2 + 1).astype(np.float64)*self.parameters['dky']
- kspace['ky'] = np.roll(kspace['ky'], self.parameters['ny']//2+1)
- kspace['kz'] = np.arange(-self.parameters['nz']//2 + 1,
- self.parameters['nz']//2 + 1).astype(np.float64)*self.parameters['dkz']
- kspace['kz'] = np.roll(kspace['kz'], self.parameters['nz']//2+1)
- return kspace
- def get_data_file_name(self):
- return os.path.join(self.work_dir, self.simname + '.h5')
- def get_data_file(self):
- return h5py.File(self.get_data_file_name(), 'r')
- def write_par(
- self,
- iter0 = 0,
- particle_ic = None):
- assert (iter0 == 0)
- _code.write_par(self, iter0 = iter0)
- with h5py.File(self.get_data_file_name(), 'r+') as ofile:
- ofile['bfps_info/exec_name'] = self.name
- kspace = self.get_kspace()
- for k in kspace.keys():
- ofile['kspace/' + k] = kspace[k]
- nshells = kspace['nshell'].shape[0]
- kspace = self.get_kspace()
- nshells = kspace['nshell'].shape[0]
- ofile['checkpoint'] = int(0)
- return None
- def job_parser_arguments(
- self,
- parser):
- parser.add_argument(
- '--ncpu',
- type = int,
- dest = 'ncpu',
- default = -1)
- parser.add_argument(
- '--np', '--nprocesses',
- metavar = 'NPROCESSES',
- help = 'number of mpi processes to use',
- type = int,
- dest = 'nb_processes',
- default = 4)
- parser.add_argument(
- '--ntpp', '--nthreads-per-process',
- type = int,
- dest = 'nb_threads_per_process',
- metavar = 'NTHREADS_PER_PROCESS',
- help = 'number of threads to use per MPI process',
- default = 1)
- parser.add_argument(
- '--no-submit',
- action = 'store_true',
- dest = 'no_submit')
- parser.add_argument(
- '--environment',
- type = str,
- dest = 'environment',
- default = None)
- parser.add_argument(
- '--minutes',
- type = int,
- dest = 'minutes',
- default = 5,
- help = 'If environment supports it, this is the requested wall-clock-limit.')
- parser.add_argument(
- '--njobs',
- type = int, dest = 'njobs',
- default = 1)
- return None
- def simulation_parser_arguments(
- self,
- parser):
- parser.add_argument(
- '--simname',
- type = str, dest = 'simname',
- default = 'test')
- parser.add_argument(
- '-n', '--grid-size',
- type = int,
- dest = 'n',
- default = 32,
- metavar = 'N',
- help = 'code is run by default in a grid of NxNxN')
- for coord in ['x', 'y', 'z']:
- parser.add_argument(
- '--L{0}'.format(coord), '--box-length-{0}'.format(coord),
- type = float,
- dest = 'L{0}'.format(coord),
- default = 2.0,
- metavar = 'length{0}'.format(coord),
- help = 'length of the box in the {0} direction will be `length{0} x pi`'.format(coord))
- parser.add_argument(
- '--wd',
- type = str, dest = 'work_dir',
- default = './')
- parser.add_argument(
- '--precision',
- choices = ['single', 'double'],
- type = str,
- default = 'single')
- return None
- def add_parser_arguments(
- self,
- parser):
- subparsers = parser.add_subparsers(
- dest = 'TEST_class',
- help = 'type of simulation to run')
- subparsers.required = True
- parser_filter_test = subparsers.add_parser(
- 'filter_test',
- help = 'plain filter test')
- self.simulation_parser_arguments(parser_filter_test)
- self.job_parser_arguments(parser_filter_test)
- self.parameters_to_parser_arguments(parser_filter_test)
- return None
- def prepare_launch(
- self,
- args = []):
- opt = _code.prepare_launch(self, args = args)
- self.set_precision(opt.precision)
- self.dns_type = opt.TEST_class
- self.name = self.dns_type + '-' + self.fluid_precision + '-v' + bfps.__version__
- # merge parameters if needed
- self.pars_from_namespace(opt)
- return opt
- def launch(
- self,
- args = [],
- **kwargs):
- opt = self.prepare_launch(args = args)
- self.launch_jobs(opt = opt, **kwargs)
- return None
- def launch_jobs(
- self,
- opt = None,
- particle_initial_condition = None):
- if not os.path.exists(os.path.join(self.work_dir, self.simname + '.h5')):
- self.write_par(
- particle_ic = None)
- self.run(
- nb_processes = opt.nb_processes,
- nb_threads_per_process = opt.nb_threads_per_process,
- njobs = opt.njobs,
- hours = opt.minutes // 60,
- minutes = opt.minutes % 60,
- no_submit = opt.no_submit)
- return None
-
diff --git a/bfps/__main__.py b/bfps/__main__.py
deleted file mode 100644
index c41a6ffb67f91983f7969f40bc048a2e36e23afe..0000000000000000000000000000000000000000
--- a/bfps/__main__.py
+++ /dev/null
@@ -1,114 +0,0 @@
-#######################################################################
-# #
-# Copyright 2015 Max Planck Institute #
-# for Dynamics and Self-Organization #
-# #
-# This file is part of bfps. #
-# #
-# bfps is free software: you can redistribute it and/or modify #
-# it under the terms of the GNU General Public License as published #
-# by the Free Software Foundation, either version 3 of the License, #
-# or (at your option) any later version. #
-# #
-# bfps is distributed in the hope that it will be useful, #
-# but WITHOUT ANY WARRANTY; without even the implied warranty of #
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the #
-# GNU General Public License for more details. #
-# #
-# You should have received a copy of the GNU General Public License #
-# along with bfps. If not, see <http://www.gnu.org/licenses/> #
-# #
-# Contact: Cristian.Lalescu@ds.mpg.de #
-# #
-#######################################################################
-
-
-
-import sys
-import argparse
-
-import bfps
-from .DNS import DNS
-from .PP import PP
-from .TEST import TEST
-from .NavierStokes import NavierStokes
-from .NSVorticityEquation import NSVorticityEquation
-from .FluidResize import FluidResize
-from .FluidConvert import FluidConvert
-from .NSManyParticles import NSManyParticles
-
-def main():
- parser = argparse.ArgumentParser(prog = 'bfps')
- parser.add_argument(
- '-v', '--version',
- action = 'version',
- version = '%(prog)s ' + bfps.__version__)
- NSoptions = ['NavierStokes',
- 'NavierStokes-single',
- 'NavierStokes-double',
- 'NS',
- 'NS-single',
- 'NS-double']
- NSVEoptions = ['NSVorticityEquation',
- 'NSVorticityEquation-single',
- 'NSVorticityEquation-double',
- 'NSVE',
- 'NSVE-single',
- 'NSVE-double']
- FRoptions = ['FluidResize',
- 'FluidResize-single',
- 'FluidResize-double',
- 'FR',
- 'FR-single',
- 'FR-double']
- FCoptions = ['FluidConvert']
- NSMPopt = ['NSManyParticles',
- 'NSManyParticles-single',
- 'NSManyParticles-double']
- parser.add_argument(
- 'base_class',
- choices = ['DNS', 'PP', 'TEST'] +
- NSoptions +
- NSVEoptions +
- FRoptions +
- FCoptions +
- NSMPopt,
- type = str)
- # first option is the choice of base class or -h or -v
- # all other options are passed on to the base_class instance
- opt = parser.parse_args(sys.argv[1:2])
- # error is thrown if first option is not a base class, so launch
- # cannot be executed by mistake.
- if opt.base_class == 'DNS':
- c = DNS()
- c.launch(args = sys.argv[2:])
- return None
- if opt.base_class == 'PP':
- c = PP()
- c.launch(args = sys.argv[2:])
- return None
- if opt.base_class == 'TEST':
- c = TEST()
- c.launch(args = sys.argv[2:])
- return None
- if 'double' in opt.base_class:
- precision = 'double'
- else:
- precision = 'single'
- if opt.base_class in NSoptions:
- base_class = NavierStokes
- if opt.base_class in NSVEoptions:
- base_class = NSVorticityEquation
- elif opt.base_class in FRoptions:
- base_class = FluidResize
- elif opt.base_class in FCoptions:
- base_class = FluidConvert
- elif opt.base_class in NSMPopt:
- base_class = NSManyParticles
- c = base_class(fluid_precision = precision)
- c.launch(args = sys.argv[2:])
- return None
-
-if __name__ == '__main__':
- main()
-
diff --git a/bfps/_fluid_base.py b/bfps/_fluid_base.py
deleted file mode 100644
index 757e6cb81e6c605cbcb3c2e9d19bd7487add115f..0000000000000000000000000000000000000000
--- a/bfps/_fluid_base.py
+++ /dev/null
@@ -1,503 +0,0 @@
-#######################################################################
-# #
-# Copyright 2015 Max Planck Institute #
-# for Dynamics and Self-Organization #
-# #
-# This file is part of bfps. #
-# #
-# bfps is free software: you can redistribute it and/or modify #
-# it under the terms of the GNU General Public License as published #
-# by the Free Software Foundation, either version 3 of the License, #
-# or (at your option) any later version. #
-# #
-# bfps is distributed in the hope that it will be useful, #
-# but WITHOUT ANY WARRANTY; without even the implied warranty of #
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the #
-# GNU General Public License for more details. #
-# #
-# You should have received a copy of the GNU General Public License #
-# along with bfps. If not, see <http://www.gnu.org/licenses/> #
-# #
-# Contact: Cristian.Lalescu@ds.mpg.de #
-# #
-#######################################################################
-
-
-
-from ._code import _code
-from bfps import tools
-
-import os
-import numpy as np
-import h5py
-
-class _fluid_particle_base(_code):
- """This class is meant to put together all common code between the
- different C++ solvers/postprocessing tools, so that development of
- specific functionalities is not overwhelming.
- """
- def __init__(
- self,
- name = 'solver',
- work_dir = './',
- simname = 'test',
- dtype = np.float32,
- use_fftw_wisdom = True):
- _code.__init__(
- self,
- work_dir = work_dir,
- simname = simname)
- self.use_fftw_wisdom = use_fftw_wisdom
- self.name = name
- self.particle_species = 0
- if dtype in [np.float32, np.float64]:
- self.dtype = dtype
- elif dtype in ['single', 'double']:
- if dtype == 'single':
- self.dtype = np.dtype(np.float32)
- elif dtype == 'double':
- self.dtype = np.dtype(np.float64)
- self.rtype = self.dtype
- if self.rtype == np.float32:
- self.ctype = np.dtype(np.complex64)
- self.C_dtype = 'float'
- elif self.rtype == np.float64:
- self.ctype = np.dtype(np.complex128)
- self.C_dtype = 'double'
- self.parameters['dealias_type'] = 1
- self.parameters['dkx'] = 1.0
- self.parameters['dky'] = 1.0
- self.parameters['dkz'] = 1.0
- self.parameters['niter_todo'] = 8
- self.parameters['niter_part'] = 1
- self.parameters['niter_stat'] = 1
- self.parameters['niter_out'] = 1024
- self.parameters['nparticles'] = 0
- self.parameters['dt'] = 0.01
- self.fluid_includes = '#include "fluid_solver.hpp"\n'
- self.fluid_includes = '#include "field.hpp"\n'
- self.fluid_variables = ''
- self.fluid_definitions = ''
- self.fluid_start = ''
- self.fluid_loop = ''
- self.fluid_end = ''
- self.fluid_output = ''
- self.stat_src = ''
- self.particle_includes = ''
- self.particle_variables = ''
- self.particle_definitions = ''
- self.particle_start = ''
- self.particle_loop = ''
- self.particle_output = ''
- self.particle_end = ''
- self.particle_stat_src = ''
- self.file_datasets_grow = ''
- self.store_kspace = """
- //begincpp
- if (myrank == 0 && iteration == 0)
- {
- TIMEZONE("fuild_base::store_kspace");
- hsize_t dims[4];
- hid_t space, dset;
- // store kspace information
- hid_t parameter_file = stat_file;
- //char fname[256];
- //sprintf(fname, "%s.h5", simname);
- //parameter_file = H5Fopen(fname, H5F_ACC_RDWR, H5P_DEFAULT);
- dset = H5Dopen(parameter_file, "/kspace/kshell", H5P_DEFAULT);
- space = H5Dget_space(dset);
- H5Sget_simple_extent_dims(space, dims, NULL);
- H5Sclose(space);
- if (fs->nshells != dims[0])
- {
- DEBUG_MSG(
- "ERROR: computed nshells %d not equal to data file nshells %d\\n",
- fs->nshells, dims[0]);
- }
- H5Dwrite(dset, H5T_NATIVE_DOUBLE, H5S_ALL, H5S_ALL, H5P_DEFAULT, fs->kshell);
- H5Dclose(dset);
- dset = H5Dopen(parameter_file, "/kspace/nshell", H5P_DEFAULT);
- H5Dwrite(dset, H5T_NATIVE_INT64, H5S_ALL, H5S_ALL, H5P_DEFAULT, fs->nshell);
- H5Dclose(dset);
- dset = H5Dopen(parameter_file, "/kspace/kM", H5P_DEFAULT);
- H5Dwrite(dset, H5T_NATIVE_DOUBLE, H5S_ALL, H5S_ALL, H5P_DEFAULT, &fs->kMspec);
- H5Dclose(dset);
- dset = H5Dopen(parameter_file, "/kspace/dk", H5P_DEFAULT);
- H5Dwrite(dset, H5T_NATIVE_DOUBLE, H5S_ALL, H5S_ALL, H5P_DEFAULT, &fs->dk);
- H5Dclose(dset);
- //H5Fclose(parameter_file);
- }
- //endcpp
- """
- return None
- def get_data_file_name(self):
- return os.path.join(self.work_dir, self.simname + '.h5')
- def get_data_file(self):
- return h5py.File(self.get_data_file_name(), 'r')
- def get_particle_file_name(self):
- return os.path.join(self.work_dir, self.simname + '_particles.h5')
- def get_particle_file(self):
- return h5py.File(self.get_particle_file_name(), 'r')
- def finalize_code(
- self,
- postprocess_mode = False):
- self.includes += self.fluid_includes
- self.includes += '#include <ctime>\n'
- self.variables += self.fluid_variables
- self.definitions += ('int grow_single_dataset(hid_t dset, int tincrement)\n{\n' +
- 'int ndims;\n' +
- 'hsize_t space;\n' +
- 'space = H5Dget_space(dset);\n' +
- 'ndims = H5Sget_simple_extent_ndims(space);\n' +
- 'hsize_t *dims = new hsize_t[ndims];\n' +
- 'H5Sget_simple_extent_dims(space, dims, NULL);\n' +
- 'dims[0] += tincrement;\n' +
- 'H5Dset_extent(dset, dims);\n' +
- 'H5Sclose(space);\n' +
- 'delete[] dims;\n' +
- 'return EXIT_SUCCESS;\n}\n')
- self.definitions+= self.fluid_definitions
- if self.particle_species > 0:
- self.includes += self.particle_includes
- self.variables += self.particle_variables
- self.definitions += self.particle_definitions
- self.definitions += ('herr_t grow_statistics_dataset(hid_t o_id, const char *name, const H5O_info_t *info, void *op_data)\n{\n' +
- 'if (info->type == H5O_TYPE_DATASET)\n{\n' +
- 'hsize_t dset = H5Dopen(o_id, name, H5P_DEFAULT);\n' +
- 'grow_single_dataset(dset, niter_todo/niter_stat);\n'
- 'H5Dclose(dset);\n}\n' +
- 'return 0;\n}\n')
- self.definitions += ('herr_t grow_particle_datasets(hid_t g_id, const char *name, const H5L_info_t *info, void *op_data)\n{\n' +
- 'hsize_t dset;\n')
- for key in ['state', 'velocity', 'acceleration']:
- self.definitions += ('if (H5Lexists(g_id, "{0}", H5P_DEFAULT))\n'.format(key) +
- '{\n' +
- 'dset = H5Dopen(g_id, "{0}", H5P_DEFAULT);\n'.format(key) +
- 'grow_single_dataset(dset, niter_todo/niter_part);\n' +
- 'H5Dclose(dset);\n}\n')
- self.definitions += ('if (H5Lexists(g_id, "rhs", H5P_DEFAULT))\n{\n' +
- 'dset = H5Dopen(g_id, "rhs", H5P_DEFAULT);\n' +
- 'grow_single_dataset(dset, 1);\n' +
- 'H5Dclose(dset);\n}\n' +
- 'return 0;\n}\n')
- self.definitions += ('int grow_file_datasets()\n{\n' +
- 'int file_problems = 0;\n' +
- self.file_datasets_grow +
- 'return file_problems;\n'
- '}\n')
- self.definitions += 'void do_stats()\n{\n' + self.stat_src + '}\n'
- self.definitions += 'void do_particle_stats()\n{\n' + self.particle_stat_src + '}\n'
- # take care of wisdom
- if self.use_fftw_wisdom:
- if self.dtype == np.float32:
- fftw_prefix = 'fftwf_'
- elif self.dtype == np.float64:
- fftw_prefix = 'fftw_'
- self.main_start += """
- //begincpp
- if (myrank == 0)
- {{
- char fname[256];
- sprintf(fname, "%s_fftw_wisdom.txt", simname);
- {0}import_wisdom_from_filename(fname);
- }}
- {0}mpi_broadcast_wisdom(MPI_COMM_WORLD);
- //endcpp
- """.format(fftw_prefix)
- self.main_end = """
- //begincpp
- {0}mpi_gather_wisdom(MPI_COMM_WORLD);
- MPI_Barrier(MPI_COMM_WORLD);
- if (myrank == 0)
- {{
- char fname[256];
- sprintf(fname, "%s_fftw_wisdom.txt", simname);
- {0}export_wisdom_to_filename(fname);
- }}
- //endcpp
- """.format(fftw_prefix) + self.main_end
- self.main = """
- //begincpp
- int data_file_problem;
- clock_t time0, time1;
- double time_difference, local_time_difference;
- time0 = clock();
- if (myrank == 0) data_file_problem = grow_file_datasets();
- MPI_Bcast(&data_file_problem, 1, MPI_INT, 0, MPI_COMM_WORLD);
- if (data_file_problem > 0)
- {
- std::cerr << data_file_problem << " problems growing file datasets.\\ntrying to exit now." << std::endl;
- MPI_Finalize();
- return EXIT_SUCCESS;
- }
- //endcpp
- """
- self.main += self.fluid_start
- if self.particle_species > 0:
- self.main += self.particle_start
- output_time_difference = ('time1 = clock();\n' +
- 'local_time_difference = ((unsigned int)(time1 - time0))/((double)CLOCKS_PER_SEC);\n' +
- 'time_difference = 0.0;\n' +
- 'MPI_Allreduce(&local_time_difference, &time_difference, ' +
- '1, MPI_DOUBLE, MPI_SUM, MPI_COMM_WORLD);\n' +
- 'if (myrank == 0) std::cout << "iteration " ' +
- '<< {0} << " took " ' +
- '<< time_difference/nprocs << " seconds" << std::endl;\n' +
- 'if (myrank == 0) std::cerr << "iteration " ' +
- '<< {0} << " took " ' +
- '<< time_difference/nprocs << " seconds" << std::endl;\n' +
- 'time0 = time1;\n')
- if not postprocess_mode:
- self.main += 'for (int max_iter = iteration+niter_todo-iteration%niter_todo; iteration < max_iter; iteration++)\n'
- self.main += '{\n'
-
- self.main += """
- #ifdef USE_TIMINGOUTPUT
- const std::string loopLabel = "code::main_start::loop-" + std::to_string(iteration);
- TIMEZONE(loopLabel.c_str());
- #endif
- """
- self.main += 'if (iteration % niter_stat == 0) do_stats();\n'
- if self.particle_species > 0:
- self.main += 'if (iteration % niter_part == 0) do_particle_stats();\n'
- self.main += self.particle_loop
- self.main += self.fluid_loop
- self.main += output_time_difference.format('iteration')
- self.main += '}\n'
- self.main += 'do_stats();\n'
- self.main += 'do_particle_stats();\n'
- self.main += output_time_difference.format('iteration')
- else:
- self.main += 'for (int frame_index = iter0; frame_index <= iter1; frame_index += niter_out)\n'
- self.main += '{\n'
- self.main += """
- #ifdef USE_TIMINGOUTPUT
- const std::string loopLabel = "code::main_start::loop-" + std::to_string(frame_index);
- TIMEZONE(loopLabel.c_str());
- #endif
- """
- if self.particle_species > 0:
- self.main += self.particle_loop
- self.main += self.fluid_loop
- self.main += output_time_difference.format('frame_index')
- self.main += '}\n'
- self.main += self.fluid_end
- if self.particle_species > 0:
- self.main += self.particle_end
- return None
- def read_rfield(
- self,
- field = 'velocity',
- iteration = 0,
- filename = None):
- """
- :note: assumes field is a vector field
- """
- if type(filename) == type(None):
- filename = os.path.join(
- self.work_dir,
- self.simname + '_r' + field + '_i{0:0>5x}'.format(iteration))
- return np.memmap(
- filename,
- dtype = self.dtype,
- mode = 'r',
- shape = (self.parameters['nz'],
- self.parameters['ny'],
- self.parameters['nx'], 3))
- def transpose_frame(
- self,
- field = 'velocity',
- iteration = 0,
- filename = None,
- ofile = None):
- Rdata = self.read_rfield(
- field = field,
- iteration = iteration,
- filename = filename)
- new_data = np.zeros(
- (3,
- self.parameters['nz'],
- self.parameters['ny'],
- self.parameters['nx']),
- dtype = self.dtype)
- for i in range(3):
- new_data[i] = Rdata[..., i]
- if type(ofile) == type(None):
- ofile = os.path.join(
- self.work_dir,
- self.simname + '_r' + field + '_i{0:0>5x}_3xNZxNYxNX'.format(iteration))
- else:
- new_data.tofile(ofile)
- return new_data
- def plot_vel_cut(
- self,
- axis,
- field = 'velocity',
- iteration = 0,
- yval = 13,
- filename = None):
- axis.set_axis_off()
- Rdata0 = self.read_rfield(field = field, iteration = iteration, filename = filename)
- energy = np.sum(Rdata0[:, yval, :, :]**2, axis = 2)*.5
- axis.imshow(energy, interpolation='none')
- axis.set_title('{0}'.format(np.average(Rdata0[..., 0]**2 +
- Rdata0[..., 1]**2 +
- Rdata0[..., 2]**2)*.5))
- return Rdata0
- def generate_vector_field(
- self,
- rseed = 7547,
- spectra_slope = 1.,
- amplitude = 1.,
- iteration = 0,
- field_name = 'vorticity',
- write_to_file = False,
- # to switch to constant field, use generate_data_3D_uniform
- # for scalar_generator
- scalar_generator = tools.generate_data_3D):
- """generate vector field.
-
- The generated field is not divergence free, but it has the proper
- shape.
-
- :param rseed: seed for random number generator
- :param spectra_slope: spectrum of field will look like k^(-p)
- :param amplitude: all amplitudes are multiplied with this value
- :param iteration: the field is written at this iteration
- :param field_name: the name of the field being generated
- :param write_to_file: should we write the field to file?
- :param scalar_generator: which function to use for generating the
- individual components.
- Possible values: bfps.tools.generate_data_3D,
- bfps.tools.generate_data_3D_uniform
- :type rseed: int
- :type spectra_slope: float
- :type amplitude: float
- :type iteration: int
- :type field_name: str
- :type write_to_file: bool
- :type scalar_generator: function
-
- :returns: ``Kdata``, a complex valued 4D ``numpy.array`` that uses the
- transposed FFTW layout.
- Kdata[ky, kz, kx, i] is the amplitude of mode (kx, ky, kz) for
- the i-th component of the field.
- (i.e. x is the fastest index and z the slowest index in the
- real-space representation).
- """
- np.random.seed(rseed)
- Kdata00 = scalar_generator(
- self.parameters['nz']//2,
- self.parameters['ny']//2,
- self.parameters['nx']//2,
- p = spectra_slope,
- amplitude = amplitude).astype(self.ctype)
- Kdata01 = scalar_generator(
- self.parameters['nz']//2,
- self.parameters['ny']//2,
- self.parameters['nx']//2,
- p = spectra_slope,
- amplitude = amplitude).astype(self.ctype)
- Kdata02 = scalar_generator(
- self.parameters['nz']//2,
- self.parameters['ny']//2,
- self.parameters['nx']//2,
- p = spectra_slope,
- amplitude = amplitude).astype(self.ctype)
- Kdata0 = np.zeros(
- Kdata00.shape + (3,),
- Kdata00.dtype)
- Kdata0[..., 0] = Kdata00
- Kdata0[..., 1] = Kdata01
- Kdata0[..., 2] = Kdata02
- Kdata1 = tools.padd_with_zeros(
- Kdata0,
- self.parameters['nz'],
- self.parameters['ny'],
- self.parameters['nx'])
- if write_to_file:
- Kdata1.tofile(
- os.path.join(self.work_dir,
- self.simname + "_c{0}_i{1:0>5x}".format(field_name, iteration)))
- return Kdata1
- def generate_tracer_state(
- self,
- rseed = None,
- iteration = 0,
- species = 0,
- write_to_file = False,
- ncomponents = 3,
- testing = False,
- data = None):
- if (type(data) == type(None)):
- if not type(rseed) == type(None):
- np.random.seed(rseed)
- #point with problems: 5.37632864e+00, 6.10414710e+00, 6.25256493e+00]
- data = np.zeros(self.parameters['nparticles']*ncomponents).reshape(-1, ncomponents)
- data[:, :3] = np.random.random((self.parameters['nparticles'], 3))*2*np.pi
- if testing:
- #data[0] = np.array([3.26434, 4.24418, 3.12157])
- data[0] = np.array([ 0.72086101, 2.59043666, 6.27501953])
- with h5py.File(self.get_particle_file_name(), 'r+') as data_file:
- data_file['tracers{0}/state'.format(species)][0] = data
- if write_to_file:
- data.tofile(
- os.path.join(
- self.work_dir,
- "tracers{0}_state_i{1:0>5x}".format(species, iteration)))
- return data
- def generate_initial_condition(self):
- self.generate_vector_field(write_to_file = True)
- for species in range(self.particle_species):
- self.generate_tracer_state(
- species = species,
- write_to_file = False)
- return None
- def get_kspace(self):
- kspace = {}
- if self.parameters['dealias_type'] == 1:
- kMx = self.parameters['dkx']*(self.parameters['nx']//2 - 1)
- kMy = self.parameters['dky']*(self.parameters['ny']//2 - 1)
- kMz = self.parameters['dkz']*(self.parameters['nz']//2 - 1)
- else:
- kMx = self.parameters['dkx']*(self.parameters['nx']//3 - 1)
- kMy = self.parameters['dky']*(self.parameters['ny']//3 - 1)
- kMz = self.parameters['dkz']*(self.parameters['nz']//3 - 1)
- kspace['kM'] = max(kMx, kMy, kMz)
- kspace['dk'] = min(self.parameters['dkx'],
- self.parameters['dky'],
- self.parameters['dkz'])
- nshells = int(kspace['kM'] / kspace['dk']) + 2
- kspace['nshell'] = np.zeros(nshells, dtype = np.int64)
- kspace['kshell'] = np.zeros(nshells, dtype = np.float64)
- kspace['kx'] = np.arange( 0,
- self.parameters['nx']//2 + 1).astype(np.float64)*self.parameters['dkx']
- kspace['ky'] = np.arange(-self.parameters['ny']//2 + 1,
- self.parameters['ny']//2 + 1).astype(np.float64)*self.parameters['dky']
- kspace['ky'] = np.roll(kspace['ky'], self.parameters['ny']//2+1)
- kspace['kz'] = np.arange(-self.parameters['nz']//2 + 1,
- self.parameters['nz']//2 + 1).astype(np.float64)*self.parameters['dkz']
- kspace['kz'] = np.roll(kspace['kz'], self.parameters['nz']//2+1)
- return kspace
- def write_par(self, iter0 = 0):
- assert (self.parameters['niter_todo'] % self.parameters['niter_stat'] == 0)
- assert (self.parameters['niter_todo'] % self.parameters['niter_out'] == 0)
- assert (self.parameters['niter_todo'] % self.parameters['niter_part'] == 0)
- assert (self.parameters['niter_out'] % self.parameters['niter_stat'] == 0)
- assert (self.parameters['niter_out'] % self.parameters['niter_part'] == 0)
- _code.write_par(self, iter0 = iter0)
- with h5py.File(os.path.join(self.work_dir, self.simname + '.h5'), 'r+') as ofile:
- ofile['bfps_info/exec_name'] = self.name
- ofile['field_dtype'] = np.dtype(self.dtype).str
- kspace = self.get_kspace()
- for k in kspace.keys():
- ofile['kspace/' + k] = kspace[k]
- nshells = kspace['nshell'].shape[0]
- ofile.close()
- return None
- def specific_parser_arguments(
- self,
- parser):
- _code.specific_parser_arguments(self, parser)
- return None
-
diff --git a/bfps/cpp/distributed_particles.cpp b/bfps/cpp/distributed_particles.cpp
deleted file mode 100644
index 73fd0275d8138d41bb4ee7fbc28e2d41e8017661..0000000000000000000000000000000000000000
--- a/bfps/cpp/distributed_particles.cpp
+++ /dev/null
@@ -1,472 +0,0 @@
-/**********************************************************************
-* *
-* Copyright 2015 Max Planck Institute *
-* for Dynamics and Self-Organization *
-* *
-* This file is part of bfps. *
-* *
-* bfps is free software: you can redistribute it and/or modify *
-* it under the terms of the GNU General Public License as published *
-* by the Free Software Foundation, either version 3 of the License, *
-* or (at your option) any later version. *
-* *
-* bfps is distributed in the hope that it will be useful, *
-* but WITHOUT ANY WARRANTY; without even the implied warranty of *
-* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the *
-* GNU General Public License for more details. *
-* *
-* You should have received a copy of the GNU General Public License *
-* along with bfps. If not, see <http://www.gnu.org/licenses/> *
-* *
-* Contact: Cristian.Lalescu@ds.mpg.de *
-* *
-**********************************************************************/
-
-
-
-//#define NDEBUG
-
-#include <cmath>
-#include <cassert>
-#include <cstring>
-#include <string>
-#include <sstream>
-#include <array>
-
-#include "base.hpp"
-#include "distributed_particles.hpp"
-#include "fftw_tools.hpp"
-#include "scope_timer.hpp"
-
-
-extern int myrank, nprocs;
-
-template <particle_types particle_type, class rnumber, int interp_neighbours>
-distributed_particles<particle_type, rnumber, interp_neighbours>::distributed_particles(
- const char *NAME,
- const hid_t data_file_id,
- interpolator<rnumber, interp_neighbours> *VEL,
- const int TRAJ_SKIP,
- const int INTEGRATION_STEPS) : particles_io_base<particle_type>(
- NAME,
- TRAJ_SKIP,
- data_file_id,
- VEL->descriptor->comm)
-{
- assert((INTEGRATION_STEPS <= 6) &&
- (INTEGRATION_STEPS >= 1));
- this->vel = VEL;
- this->rhs.resize(INTEGRATION_STEPS);
- this->integration_steps = INTEGRATION_STEPS;
- this->state.reserve(2*this->nparticles / this->nprocs);
- for (unsigned int i=0; i<this->rhs.size(); i++)
- this->rhs[i].reserve(2*this->nparticles / this->nprocs);
-}
-
-template <particle_types particle_type, class rnumber, int interp_neighbours>
-distributed_particles<particle_type, rnumber, interp_neighbours>::~distributed_particles()
-{
-}
-
-template <particle_types particle_type, class rnumber, int interp_neighbours>
-void distributed_particles<particle_type, rnumber, interp_neighbours>::sample(
- interpolator<rnumber, interp_neighbours> *field,
- const std::unordered_map<int, single_particle_state<particle_type>> &x,
- std::unordered_map<int, single_particle_state<POINT3D>> &y)
-{
- std::array<double, 3> yy;
- y.clear();
- for (auto &pp: x)
- {
- (*field)(pp.second.data, &yy.front());
- y[pp.first] = &yy.front();
- }
-}
-
-template <particle_types particle_type, class rnumber, int interp_neighbours>
-void distributed_particles<particle_type, rnumber, interp_neighbours>::get_rhs(
- const std::unordered_map<int, single_particle_state<particle_type>> &x,
- std::unordered_map<int, single_particle_state<particle_type>> &y)
-{
- std::unordered_map<int, single_particle_state<POINT3D>> yy;
- switch(particle_type)
- {
- case VELOCITY_TRACER:
- this->sample(this->vel, this->state, yy);
- y.clear();
- for (auto &pp: x)
- y[pp.first] = yy[pp.first].data;
- break;
- }
-}
-
-template <particle_types particle_type, class rnumber, int interp_neighbours>
-void distributed_particles<particle_type, rnumber, interp_neighbours>::sample(
- interpolator<rnumber, interp_neighbours> *field,
- const char *dset_name)
-{
- std::unordered_map<int, single_particle_state<POINT3D>> y;
- this->sample(field, this->state, y);
- this->write(dset_name, y);
-}
-
-template <particle_types particle_type, class rnumber, int interp_neighbours>
-void distributed_particles<particle_type, rnumber, interp_neighbours>::roll_rhs()
-{
- for (int i=this->integration_steps-2; i>=0; i--)
- rhs[i+1] = rhs[i];
-}
-
-template <particle_types particle_type, class rnumber, int interp_neighbours>
-void distributed_particles<particle_type, rnumber, interp_neighbours>::redistribute(
- std::unordered_map<int, single_particle_state<particle_type>> &x,
- std::vector<std::unordered_map<int, single_particle_state<particle_type>>> &vals)
-{
- TIMEZONE("distributed_particles::redistribute");
- //DEBUG_MSG("entered redistribute\n");
- /* neighbouring rank offsets */
- int ro[2];
- ro[0] = -1;
- ro[1] = 1;
- /* neighbouring ranks */
- int nr[2];
- nr[0] = MOD(this->myrank+ro[0], this->nprocs);
- nr[1] = MOD(this->myrank+ro[1], this->nprocs);
- /* particles to send, particles to receive */
- std::vector<int> ps[2], pr[2];
- /* number of particles to send, number of particles to receive */
- int nps[2], npr[2];
- int rsrc, rdst;
- /* get list of id-s to send */
- for (auto &pp: x)
- for (unsigned int i=0; i<2; i++)
- if (this->vel->get_rank(pp.second.data[2]) == nr[i])
- ps[i].push_back(pp.first);
- /* prepare data for send recv */
- for (unsigned int i=0; i<2; i++)
- nps[i] = ps[i].size();
- for (rsrc = 0; rsrc<this->nprocs; rsrc++)
- for (unsigned int i=0; i<2; i++)
- {
- rdst = MOD(rsrc+ro[i], this->nprocs);
- if (this->myrank == rsrc)
- MPI_Send(
- nps+i,
- 1,
- MPI_INTEGER,
- rdst,
- 2*(rsrc*this->nprocs + rdst)+i,
- this->comm);
- if (this->myrank == rdst)
- MPI_Recv(
- npr+1-i,
- 1,
- MPI_INTEGER,
- rsrc,
- 2*(rsrc*this->nprocs + rdst)+i,
- this->comm,
- MPI_STATUS_IGNORE);
- }
- //DEBUG_MSG("I have to send %d %d particles\n", nps[0], nps[1]);
- //DEBUG_MSG("I have to recv %d %d particles\n", npr[0], npr[1]);
- for (unsigned int i=0; i<2; i++)
- pr[i].resize(npr[i]);
-
- int buffer_size = (nps[0] > nps[1]) ? nps[0] : nps[1];
- buffer_size = (buffer_size > npr[0])? buffer_size : npr[0];
- buffer_size = (buffer_size > npr[1])? buffer_size : npr[1];
- //DEBUG_MSG("buffer size is %d\n", buffer_size);
- double *buffer = new double[buffer_size*state_dimension(particle_type)*(1+vals.size())];
- for (rsrc = 0; rsrc<this->nprocs; rsrc++)
- for (unsigned int i=0; i<2; i++)
- {
- rdst = MOD(rsrc+ro[i], this->nprocs);
- if (this->myrank == rsrc && nps[i] > 0)
- {
- MPI_Send(
- &ps[i].front(),
- nps[i],
- MPI_INTEGER,
- rdst,
- 2*(rsrc*this->nprocs + rdst),
- this->comm);
- int pcounter = 0;
- for (int p: ps[i])
- {
- std::copy(x[p].data,
- x[p].data + state_dimension(particle_type),
- buffer + pcounter*(1+vals.size())*state_dimension(particle_type));
- x.erase(p);
- for (unsigned int tindex=0; tindex<vals.size(); tindex++)
- {
- std::copy(vals[tindex][p].data,
- vals[tindex][p].data + state_dimension(particle_type),
- buffer + (pcounter*(1+vals.size()) + tindex+1)*state_dimension(particle_type));
- vals[tindex].erase(p);
- }
- pcounter++;
- }
- MPI_Send(
- buffer,
- nps[i]*(1+vals.size())*state_dimension(particle_type),
- MPI_DOUBLE,
- rdst,
- 2*(rsrc*this->nprocs + rdst)+1,
- this->comm);
- }
- if (this->myrank == rdst && npr[1-i] > 0)
- {
- MPI_Recv(
- &pr[1-i].front(),
- npr[1-i],
- MPI_INTEGER,
- rsrc,
- 2*(rsrc*this->nprocs + rdst),
- this->comm,
- MPI_STATUS_IGNORE);
- MPI_Recv(
- buffer,
- npr[1-i]*(1+vals.size())*state_dimension(particle_type),
- MPI_DOUBLE,
- rsrc,
- 2*(rsrc*this->nprocs + rdst)+1,
- this->comm,
- MPI_STATUS_IGNORE);
- unsigned int pcounter = 0;
- for (int p: pr[1-i])
- {
- x[p] = buffer + (pcounter*(1+vals.size()))*state_dimension(particle_type);
- for (unsigned int tindex=0; tindex<vals.size(); tindex++)
- {
- vals[tindex][p] = buffer + (pcounter*(1+vals.size()) + tindex+1)*state_dimension(particle_type);
- }
- pcounter++;
- }
- }
- }
- delete[] buffer;
-
-
-#ifndef NDEBUG
- /* check that all particles at x are local */
- for (auto &pp: x)
- if (this->vel->get_rank(pp.second.data[2]) != this->myrank)
- {
- DEBUG_MSG("found particle %d with rank %d\n",
- pp.first,
- this->vel->get_rank(pp.second.data[2]));
- assert(false);
- }
-#endif
- //DEBUG_MSG("exiting redistribute\n");
-}
-
-
-
-template <particle_types particle_type, class rnumber, int interp_neighbours>
-void distributed_particles<particle_type, rnumber, interp_neighbours>::AdamsBashforth(
- const int nsteps)
-{
- this->get_rhs(this->state, this->rhs[0]);
- for (auto &pp: this->state)
- for (unsigned int i=0; i<state_dimension(particle_type); i++)
- switch(nsteps)
- {
- case 1:
- pp.second[i] += this->dt*this->rhs[0][pp.first][i];
- break;
- case 2:
- pp.second[i] += this->dt*(3*this->rhs[0][pp.first][i]
- - this->rhs[1][pp.first][i])/2;
- break;
- case 3:
- pp.second[i] += this->dt*(23*this->rhs[0][pp.first][i]
- - 16*this->rhs[1][pp.first][i]
- + 5*this->rhs[2][pp.first][i])/12;
- break;
- case 4:
- pp.second[i] += this->dt*(55*this->rhs[0][pp.first][i]
- - 59*this->rhs[1][pp.first][i]
- + 37*this->rhs[2][pp.first][i]
- - 9*this->rhs[3][pp.first][i])/24;
- break;
- case 5:
- pp.second[i] += this->dt*(1901*this->rhs[0][pp.first][i]
- - 2774*this->rhs[1][pp.first][i]
- + 2616*this->rhs[2][pp.first][i]
- - 1274*this->rhs[3][pp.first][i]
- + 251*this->rhs[4][pp.first][i])/720;
- break;
- case 6:
- pp.second[i] += this->dt*(4277*this->rhs[0][pp.first][i]
- - 7923*this->rhs[1][pp.first][i]
- + 9982*this->rhs[2][pp.first][i]
- - 7298*this->rhs[3][pp.first][i]
- + 2877*this->rhs[4][pp.first][i]
- - 475*this->rhs[5][pp.first][i])/1440;
- break;
- }
- this->redistribute(this->state, this->rhs);
- this->roll_rhs();
-}
-
-
-template <particle_types particle_type, class rnumber, int interp_neighbours>
-void distributed_particles<particle_type, rnumber, interp_neighbours>::step()
-{
- TIMEZONE("distributed_particles::step");
- this->AdamsBashforth((this->iteration < this->integration_steps) ?
- this->iteration+1 :
- this->integration_steps);
- this->iteration++;
-}
-
-
-template <particle_types particle_type, class rnumber, int interp_neighbours>
-void distributed_particles<particle_type, rnumber, interp_neighbours>::read()
-{
- double *temp = new double[this->chunk_size*state_dimension(particle_type)];
- for (unsigned int cindex=0; cindex<this->get_number_of_chunks(); cindex++)
- {
- //read state
- if (this->myrank == 0)
- this->read_state_chunk(cindex, temp);
- MPI_Bcast(
- temp,
- this->chunk_size*state_dimension(particle_type),
- MPI_DOUBLE,
- 0,
- this->comm);
- for (unsigned int p=0; p<this->chunk_size; p++)
- {
- if (this->vel->get_rank(temp[state_dimension(particle_type)*p+2]) == this->myrank)
- this->state[p+cindex*this->chunk_size] = temp + state_dimension(particle_type)*p;
- }
- //read rhs
- if (this->iteration > 0)
- for (int i=0; i<this->integration_steps; i++)
- {
- if (this->myrank == 0)
- this->read_rhs_chunk(cindex, i, temp);
- MPI_Bcast(
- temp,
- this->chunk_size*state_dimension(particle_type),
- MPI_DOUBLE,
- 0,
- this->comm);
- for (unsigned int p=0; p<this->chunk_size; p++)
- {
- auto pp = this->state.find(p+cindex*this->chunk_size);
- if (pp != this->state.end())
- this->rhs[i][p+cindex*this->chunk_size] = temp + state_dimension(particle_type)*p;
- }
- }
- }
- DEBUG_MSG("%s->state.size = %ld\n", this->name.c_str(), this->state.size());
- delete[] temp;
-}
-
-template <particle_types particle_type, class rnumber, int interp_neighbours>
-void distributed_particles<particle_type, rnumber, interp_neighbours>::write(
- const char *dset_name,
- std::unordered_map<int, single_particle_state<POINT3D>> &y)
-{
- TIMEZONE("distributed_particles::write");
- double *data = new double[this->nparticles*3];
- double *yy = new double[this->nparticles*3];
- for (unsigned int cindex=0; cindex<this->get_number_of_chunks(); cindex++)
- {
- std::fill_n(yy, this->chunk_size*3, 0);
- for (unsigned int p=0; p<this->chunk_size; p++)
- {
- auto pp = y.find(p+cindex*this->chunk_size);
- if (pp != y.end())
- std::copy(pp->second.data,
- pp->second.data + 3,
- yy + pp->first*3);
- }
- MPI_Allreduce(
- yy,
- data,
- 3*this->nparticles,
- MPI_DOUBLE,
- MPI_SUM,
- this->comm);
- if (this->myrank == 0)
- this->write_point3D_chunk(dset_name, cindex, data);
- }
- delete[] yy;
- delete[] data;
-}
-
-template <particle_types particle_type, class rnumber, int interp_neighbours>
-void distributed_particles<particle_type, rnumber, interp_neighbours>::write(
- const bool write_rhs)
-{
- TIMEZONE("distributed_particles::write2");
- double *temp0 = new double[this->chunk_size*state_dimension(particle_type)];
- double *temp1 = new double[this->chunk_size*state_dimension(particle_type)];
- for (unsigned int cindex=0; cindex<this->get_number_of_chunks(); cindex++)
- {
- //write state
- std::fill_n(temp0, state_dimension(particle_type)*this->chunk_size, 0);
- for (unsigned int p=0; p<this->chunk_size; p++)
- {
- auto pp = this->state.find(p + cindex*this->chunk_size);
- if (pp != this->state.end())
- std::copy(pp->second.data,
- pp->second.data + state_dimension(particle_type),
- temp0 + p*state_dimension(particle_type));
- }
- MPI_Allreduce(
- temp0,
- temp1,
- state_dimension(particle_type)*this->chunk_size,
- MPI_DOUBLE,
- MPI_SUM,
- this->comm);
- if (this->myrank == 0)
- this->write_state_chunk(cindex, temp1);
- //write rhs
- if (write_rhs)
- for (int i=0; i<this->integration_steps; i++)
- {
- std::fill_n(temp0, state_dimension(particle_type)*this->chunk_size, 0);
- for (unsigned int p=0; p<this->chunk_size; p++)
- {
- auto pp = this->rhs[i].find(p + cindex*this->chunk_size);
- if (pp != this->rhs[i].end())
- std::copy(pp->second.data,
- pp->second.data + state_dimension(particle_type),
- temp0 + p*state_dimension(particle_type));
- }
- MPI_Allreduce(
- temp0,
- temp1,
- state_dimension(particle_type)*this->chunk_size,
- MPI_DOUBLE,
- MPI_SUM,
- this->comm);
- if (this->myrank == 0)
- this->write_rhs_chunk(cindex, i, temp1);
- }
- }
- delete[] temp0;
- delete[] temp1;
-}
-
-
-/*****************************************************************************/
-template class distributed_particles<VELOCITY_TRACER, float, 1>;
-template class distributed_particles<VELOCITY_TRACER, float, 2>;
-template class distributed_particles<VELOCITY_TRACER, float, 3>;
-template class distributed_particles<VELOCITY_TRACER, float, 4>;
-template class distributed_particles<VELOCITY_TRACER, float, 5>;
-template class distributed_particles<VELOCITY_TRACER, float, 6>;
-template class distributed_particles<VELOCITY_TRACER, double, 1>;
-template class distributed_particles<VELOCITY_TRACER, double, 2>;
-template class distributed_particles<VELOCITY_TRACER, double, 3>;
-template class distributed_particles<VELOCITY_TRACER, double, 4>;
-template class distributed_particles<VELOCITY_TRACER, double, 5>;
-template class distributed_particles<VELOCITY_TRACER, double, 6>;
-/*****************************************************************************/
diff --git a/bfps/cpp/distributed_particles.hpp b/bfps/cpp/distributed_particles.hpp
deleted file mode 100644
index cf6e124a7744c049b6fcf0c84c1618a0a214c30e..0000000000000000000000000000000000000000
--- a/bfps/cpp/distributed_particles.hpp
+++ /dev/null
@@ -1,105 +0,0 @@
-/**********************************************************************
-* *
-* Copyright 2015 Max Planck Institute *
-* for Dynamics and Self-Organization *
-* *
-* This file is part of bfps. *
-* *
-* bfps is free software: you can redistribute it and/or modify *
-* it under the terms of the GNU General Public License as published *
-* by the Free Software Foundation, either version 3 of the License, *
-* or (at your option) any later version. *
-* *
-* bfps is distributed in the hope that it will be useful, *
-* but WITHOUT ANY WARRANTY; without even the implied warranty of *
-* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the *
-* GNU General Public License for more details. *
-* *
-* You should have received a copy of the GNU General Public License *
-* along with bfps. If not, see <http://www.gnu.org/licenses/> *
-* *
-* Contact: Cristian.Lalescu@ds.mpg.de *
-* *
-**********************************************************************/
-
-
-
-#include <stdio.h>
-#include <stdlib.h>
-#include <iostream>
-#include <unordered_map>
-#include <vector>
-#include <hdf5.h>
-#include "base.hpp"
-#include "particles_base.hpp"
-#include "fluid_solver_base.hpp"
-#include "interpolator.hpp"
-
-#ifndef DISTRIBUTED_PARTICLES
-
-#define DISTRIBUTED_PARTICLES
-
-template <particle_types particle_type, class rnumber, int interp_neighbours>
-class distributed_particles: public particles_io_base<particle_type>
-{
- private:
- std::unordered_map<int, single_particle_state<particle_type> > state;
- std::vector<std::unordered_map<int, single_particle_state<particle_type>>> rhs;
-
- public:
- int integration_steps;
- // this class only works with buffered interpolator
- interpolator<rnumber, interp_neighbours> *vel;
-
- /* simulation parameters */
- double dt;
-
- /* methods */
-
- /* constructor and destructor.
- * allocate and deallocate:
- * this->state
- * this->rhs
- * */
- distributed_particles(
- const char *NAME,
- const hid_t data_file_id,
- interpolator<rnumber, interp_neighbours> *FIELD,
- const int TRAJ_SKIP,
- const int INTEGRATION_STEPS = 2);
- ~distributed_particles();
-
- void sample(
- interpolator<rnumber, interp_neighbours> *field,
- const char *dset_name);
- void sample(
- interpolator<rnumber, interp_neighbours> *field,
- const std::unordered_map<int, single_particle_state<particle_type>> &x,
- std::unordered_map<int, single_particle_state<POINT3D>> &y);
- void get_rhs(
- const std::unordered_map<int, single_particle_state<particle_type>> &x,
- std::unordered_map<int, single_particle_state<particle_type>> &y);
-
- void redistribute(
- std::unordered_map<int, single_particle_state<particle_type>> &x,
- std::vector<std::unordered_map<int, single_particle_state<particle_type>>> &vals);
-
-
- /* input/output */
- void read();
- void write(
- const char *dset_name,
- std::unordered_map<int, single_particle_state<POINT3D>> &y);
- void write(
- const char *dset_name,
- std::unordered_map<int, single_particle_state<particle_type>> &y);
- void write(const bool write_rhs = true);
-
- /* solvers */
- void step();
- void roll_rhs();
- void AdamsBashforth(const int nsteps);
-};
-
-#endif//DISTRIBUTED_PARTICLES
-
diff --git a/bfps/cpp/fftw_interface.hpp b/bfps/cpp/fftw_interface.hpp
deleted file mode 100644
index 495ec9fa3712153df4d31faf7dfb3046637b5483..0000000000000000000000000000000000000000
--- a/bfps/cpp/fftw_interface.hpp
+++ /dev/null
@@ -1,173 +0,0 @@
-/**********************************************************************
-* *
-* Copyright 2015 Max Planck Institute *
-* for Dynamics and Self-Organization *
-* *
-* This file is part of bfps. *
-* *
-* bfps is free software: you can redistribute it and/or modify *
-* it under the terms of the GNU General Public License as published *
-* by the Free Software Foundation, either version 3 of the License, *
-* or (at your option) any later version. *
-* *
-* bfps is distributed in the hope that it will be useful, *
-* but WITHOUT ANY WARRANTY; without even the implied warranty of *
-* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the *
-* GNU General Public License for more details. *
-* *
-* You should have received a copy of the GNU General Public License *
-* along with bfps. If not, see <http://www.gnu.org/licenses/> *
-* *
-* Contact: Cristian.Lalescu@ds.mpg.de *
-* *
-**********************************************************************/
-
-#ifndef FFTW_INTERFACE_HPP
-#define FFTW_INTERFACE_HPP
-
-#include <fftw3-mpi.h>
-
-#ifdef USE_FFTWESTIMATE
-#define DEFAULT_FFTW_FLAG FFTW_ESTIMATE
-#warning You are using FFTW estimate
-#else
-#define DEFAULT_FFTW_FLAG FFTW_PATIENT
-#endif
-
-template <class realtype>
-class fftw_interface;
-
-template <>
-class fftw_interface<float>
-{
-public:
- using real = float;
- using complex = fftwf_complex;
- using plan = fftwf_plan;
- using iodim = fftwf_iodim;
-
- static complex* alloc_complex(const size_t in_size){
- return fftwf_alloc_complex(in_size);
- }
-
- static real* alloc_real(const size_t in_size){
- return fftwf_alloc_real(in_size);
- }
-
- static void free(void* ptr){
- fftwf_free(ptr);
- }
-
- static void execute(plan in_plan){
- fftwf_execute(in_plan);
- }
-
- static void destroy_plan(plan in_plan){
- fftwf_destroy_plan(in_plan);
- }
-
- template <class ... Params>
- static plan mpi_plan_transpose(Params ... params){
- return fftwf_mpi_plan_transpose(params...);
- }
-
- template <class ... Params>
- static plan mpi_plan_many_transpose(Params ... params){
- return fftwf_mpi_plan_many_transpose(params...);
- }
-
- template <class ... Params>
- static plan plan_guru_r2r(Params ... params){
- return fftwf_plan_guru_r2r(params...);
- }
-
- template <class ... Params>
- static plan plan_guru_dft(Params ... params){
- return fftwf_plan_guru_dft(params...);
- }
-
- template <class ... Params>
- static plan mpi_plan_many_dft_c2r(Params ... params){
- return fftwf_mpi_plan_many_dft_c2r(params...);
- }
-
- template <class ... Params>
- static plan mpi_plan_many_dft_r2c(Params ... params){
- return fftwf_mpi_plan_many_dft_r2c(params...);
- }
-
- template <class ... Params>
- static plan mpi_plan_dft_c2r_3d(Params ... params){
- return fftwf_mpi_plan_dft_c2r_3d(params...);
- }
-};
-
-template <>
-class fftw_interface<double>
-{
-public:
- using real = double;
- using complex = fftw_complex;
- using plan = fftw_plan;
- using iodim = fftw_iodim;
-
- static complex* alloc_complex(const size_t in_size){
- return fftw_alloc_complex(in_size);
- }
-
- static real* alloc_real(const size_t in_size){
- return fftw_alloc_real(in_size);
- }
-
- static void free(void* ptr){
- fftw_free(ptr);
- }
-
- static void execute(plan in_plan){
- fftw_execute(in_plan);
- }
-
- static void destroy_plan(plan in_plan){
- fftw_destroy_plan(in_plan);
- }
-
- template <class ... Params>
- static plan mpi_plan_transpose(Params ... params){
- return fftw_mpi_plan_transpose(params...);
- }
-
- template <class ... Params>
- static plan mpi_plan_many_transpose(Params ... params){
- return fftw_mpi_plan_many_transpose(params...);
- }
-
- template <class ... Params>
- static plan plan_guru_r2r(Params ... params){
- return fftw_plan_guru_r2r(params...);
- }
-
- template <class ... Params>
- static plan plan_guru_dft(Params ... params){
- return fftw_plan_guru_dft(params...);
- }
-
- template <class ... Params>
- static plan mpi_plan_many_dft_c2r(Params ... params){
- return fftw_mpi_plan_many_dft_c2r(params...);
- }
-
- template <class ... Params>
- static plan mpi_plan_many_dft_r2c(Params ... params){
- return fftw_mpi_plan_many_dft_r2c(params...);
- }
-
- template <class ... Params>
- static plan mpi_plan_dft_c2r_3d(Params ... params){
- return fftw_mpi_plan_dft_c2r_3d(params...);
- }
-};
-
-
-
-#endif // FFTW_INTERFACE_HPP
-
diff --git a/bfps/cpp/fftw_tools.cpp b/bfps/cpp/fftw_tools.cpp
deleted file mode 100644
index 61e03d292f81aed1fa4b2dfcab880fb7105b676e..0000000000000000000000000000000000000000
--- a/bfps/cpp/fftw_tools.cpp
+++ /dev/null
@@ -1,222 +0,0 @@
-/**********************************************************************
-* *
-* Copyright 2015 Max Planck Institute *
-* for Dynamics and Self-Organization *
-* *
-* This file is part of bfps. *
-* *
-* bfps is free software: you can redistribute it and/or modify *
-* it under the terms of the GNU General Public License as published *
-* by the Free Software Foundation, either version 3 of the License, *
-* or (at your option) any later version. *
-* *
-* bfps is distributed in the hope that it will be useful, *
-* but WITHOUT ANY WARRANTY; without even the implied warranty of *
-* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the *
-* GNU General Public License for more details. *
-* *
-* You should have received a copy of the GNU General Public License *
-* along with bfps. If not, see <http://www.gnu.org/licenses/> *
-* *
-* Contact: Cristian.Lalescu@ds.mpg.de *
-* *
-**********************************************************************/
-
-#include <stdlib.h>
-#include <algorithm>
-#include <iostream>
-#include "base.hpp"
-#include "fftw_tools.hpp"
-#include "fftw_interface.hpp"
-
-#define NDEBUG
-
-template <class rnumber>
-int clip_zero_padding(
- field_descriptor<rnumber> *f,
- rnumber *a,
- int howmany)
-{
- if (f->ndims < 3)
- return EXIT_FAILURE;
- rnumber *b = a;
- ptrdiff_t copy_size = f->sizes[2] * howmany;
- ptrdiff_t skip_size = copy_size + 2*howmany;
- for (int i0 = 0; i0 < f->subsizes[0]; i0++)
- for (int i1 = 0; i1 < f->sizes[1]; i1++)
- {
- std::copy(a, a + copy_size, b);
- a += skip_size;
- b += copy_size;
- }
- return EXIT_SUCCESS;
-}
-
-template
-int clip_zero_padding<float>(
- field_descriptor<float> *f,
- float *a,
- int howmany);
-
-template
-int clip_zero_padding<double>(
- field_descriptor<double> *f,
- double *a,
- int howmany);
-
-
-
-template <class rnumber>
-int copy_complex_array(
- field_descriptor<rnumber> *fi,
- rnumber (*ai)[2],
-field_descriptor<rnumber> *fo,
-rnumber (*ao)[2],
-int howmany)
-{
- DEBUG_MSG("entered copy_complex_array\n");
- typename fftw_interface<rnumber>::complex *buffer;
- buffer = fftw_interface<rnumber>::alloc_complex(fi->slice_size*howmany);
-
- int min_fast_dim;
- min_fast_dim =
- (fi->sizes[2] > fo->sizes[2]) ?
- fo->sizes[2] : fi->sizes[2];
-
- /* clean up destination, in case we're padding with zeros
- (even if only for one dimension) */
- std::fill_n((rnumber*)ao, fo->local_size*2, 0.0);
-
- int64_t ii0, ii1;
- int64_t oi0, oi1;
- int64_t delta1, delta0;
- int irank, orank;
- delta0 = (fo->sizes[0] - fi->sizes[0]);
- delta1 = (fo->sizes[1] - fi->sizes[1]);
- for (ii0=0; ii0 < fi->sizes[0]; ii0++)
- {
- if (ii0 <= fi->sizes[0]/2)
- {
- oi0 = ii0;
- if (oi0 > fo->sizes[0]/2)
- continue;
- }
- else
- {
- oi0 = ii0 + delta0;
- if ((oi0 < 0) || ((fo->sizes[0] - oi0) >= fo->sizes[0]/2))
- continue;
- }
- irank = fi->rank[ii0];
- orank = fo->rank[oi0];
- if ((irank == orank) &&
- (irank == fi->myrank))
- {
- std::copy(
- (rnumber*)(ai + (ii0 - fi->starts[0] )*fi->slice_size),
- (rnumber*)(ai + (ii0 - fi->starts[0] + 1)*fi->slice_size),
- (rnumber*)buffer);
- }
- else
- {
- if (fi->myrank == irank)
- {
- MPI_Send(
- (void*)(ai + (ii0-fi->starts[0])*fi->slice_size),
- fi->slice_size,
- mpi_real_type<rnumber>::complex(),
- orank,
- ii0,
- fi->comm);
- }
- if (fi->myrank == orank)
- {
- MPI_Recv(
- (void*)(buffer),
- fi->slice_size,
- mpi_real_type<rnumber>::complex(),
- irank,
- ii0,
- fi->comm,
- MPI_STATUS_IGNORE);
- }
- }
- if (fi->myrank == orank)
- {
- for (ii1 = 0; ii1 < fi->sizes[1]; ii1++)
- {
- if (ii1 <= fi->sizes[1]/2)
- {
- oi1 = ii1;
- if (oi1 > fo->sizes[1]/2)
- continue;
- }
- else
- {
- oi1 = ii1 + delta1;
- if ((oi1 < 0) || ((fo->sizes[1] - oi1) >= fo->sizes[1]/2))
- continue;
- }
- std::copy(
- (rnumber*)(buffer + (ii1*fi->sizes[2]*howmany)),
- (rnumber*)(buffer + (ii1*fi->sizes[2] + min_fast_dim)*howmany),
- (rnumber*)(ao +
- ((oi0 - fo->starts[0])*fo->sizes[1] +
- oi1)*fo->sizes[2]*howmany));
- }
- }
- }
- fftw_interface<rnumber>::free(buffer);
- MPI_Barrier(fi->comm);
-
- DEBUG_MSG("exiting copy_complex_array\n");
- return EXIT_SUCCESS;
-}
-
-template
-int copy_complex_array<float>(
- field_descriptor<float> *fi,
- float (*ai)[2],
- field_descriptor<float> *fo,
- float (*ao)[2],
- int howmany);
-
-template
-int copy_complex_array<double>(
- field_descriptor<double> *fi,
- double (*ai)[2],
- field_descriptor<double> *fo,
- double (*ao)[2],
- int howmany);
-
-
-template <class rnumber>
-int get_descriptors_3D(
- int n0, int n1, int n2,
- field_descriptor<rnumber> **fr,
- field_descriptor<rnumber> **fc)
-{
- int ntmp[3];
- ntmp[0] = n0;
- ntmp[1] = n1;
- ntmp[2] = n2;
- *fr = new field_descriptor<rnumber>(3, ntmp, mpi_real_type<rnumber>::real(), MPI_COMM_WORLD);
- ntmp[0] = n0;
- ntmp[1] = n1;
- ntmp[2] = n2/2+1;
- *fc = new field_descriptor<rnumber>(3, ntmp, mpi_real_type<rnumber>::complex(), MPI_COMM_WORLD);
- return EXIT_SUCCESS;
-}
-
-template
-int get_descriptors_3D<float>(
- int n0, int n1, int n2,
- field_descriptor<float> **fr,
- field_descriptor<float> **fc);
-
-template
-int get_descriptors_3D<double>(
- int n0, int n1, int n2,
- field_descriptor<double> **fr,
- field_descriptor<double> **fc);
-
diff --git a/bfps/cpp/field.cpp b/bfps/cpp/field.cpp
deleted file mode 100644
index 197ccb5da26dabf9f35d84bdc627a31f20ee49ad..0000000000000000000000000000000000000000
--- a/bfps/cpp/field.cpp
+++ /dev/null
@@ -1,1470 +0,0 @@
-/**********************************************************************
-* *
-* Copyright 2015 Max Planck Institute *
-* for Dynamics and Self-Organization *
-* *
-* This file is part of bfps. *
-* *
-* bfps is free software: you can redistribute it and/or modify *
-* it under the terms of the GNU General Public License as published *
-* by the Free Software Foundation, either version 3 of the License, *
-* or (at your option) any later version. *
-* *
-* bfps is distributed in the hope that it will be useful, *
-* but WITHOUT ANY WARRANTY; without even the implied warranty of *
-* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the *
-* GNU General Public License for more details. *
-* *
-* You should have received a copy of the GNU General Public License *
-* along with bfps. If not, see <http://www.gnu.org/licenses/> *
-* *
-* Contact: Cristian.Lalescu@ds.mpg.de *
-* *
-**********************************************************************/
-
-
-#include <sys/stat.h>
-#include <cmath>
-#include <cstdlib>
-#include <algorithm>
-#include <cassert>
-#include "field.hpp"
-#include "scope_timer.hpp"
-#include "shared_array.hpp"
-
-
-
-template <typename rnumber,
- field_backend be,
- field_components fc>
-field<rnumber, be, fc>::field(
- const int nx,
- const int ny,
- const int nz,
- const MPI_Comm COMM_TO_USE,
- const unsigned FFTW_PLAN_RIGOR)
-{
- TIMEZONE("field::field");
- this->comm = COMM_TO_USE;
- MPI_Comm_rank(this->comm, &this->myrank);
- MPI_Comm_size(this->comm, &this->nprocs);
-
- this->fftw_plan_rigor = FFTW_PLAN_RIGOR;
- this->real_space_representation = true;
-
- /* generate HDF5 data types */
- if (typeid(rnumber) == typeid(float))
- this->rnumber_H5T = H5Tcopy(H5T_NATIVE_FLOAT);
- else if (typeid(rnumber) == typeid(double))
- this->rnumber_H5T = H5Tcopy(H5T_NATIVE_DOUBLE);
- typedef struct {
- rnumber re; /*real part*/
- rnumber im; /*imaginary part*/
- } tmp_complex_type;
- this->cnumber_H5T = H5Tcreate(H5T_COMPOUND, sizeof(tmp_complex_type));
- H5Tinsert(this->cnumber_H5T, "r", HOFFSET(tmp_complex_type, re), this->rnumber_H5T);
- H5Tinsert(this->cnumber_H5T, "i", HOFFSET(tmp_complex_type, im), this->rnumber_H5T);
-
- /* switch on backend */
- switch(be)
- {
- case FFTW:
- ptrdiff_t nfftw[3];
- nfftw[0] = nz;
- nfftw[1] = ny;
- nfftw[2] = nx;
- //ptrdiff_t tmp_local_size;
- ptrdiff_t local_n0, local_0_start;
- ptrdiff_t local_n1, local_1_start;
- //tmp_local_size = fftw_mpi_local_size_many_transposed(
- fftw_mpi_local_size_many_transposed(
- 3, nfftw, ncomp(fc),
- FFTW_MPI_DEFAULT_BLOCK, FFTW_MPI_DEFAULT_BLOCK, this->comm,
- &local_n0, &local_0_start,
- &local_n1, &local_1_start);
- hsize_t sizes[3], subsizes[3], starts[3];
- sizes[0] = nz; sizes[1] = ny; sizes[2] = nx;
- subsizes[0] = local_n0; subsizes[1] = ny; subsizes[2] = nx;
- starts[0] = local_0_start; starts[1] = 0; starts[2] = 0;
- this->rlayout = new field_layout<fc>(
- sizes, subsizes, starts, this->comm);
- this->npoints = this->rlayout->full_size / ncomp(fc);
- sizes[0] = nz; sizes[1] = ny; sizes[2] = nx+2;
- subsizes[0] = local_n0; subsizes[1] = ny; subsizes[2] = nx+2;
- starts[0] = local_0_start; starts[1] = 0; starts[2] = 0;
- this->rmemlayout = new field_layout<fc>(
- sizes, subsizes, starts, this->comm);
- sizes[0] = ny; sizes[1] = nz; sizes[2] = nx/2+1;
- subsizes[0] = local_n1; subsizes[1] = nz; subsizes[2] = nx/2+1;
- starts[0] = local_1_start; starts[1] = 0; starts[2] = 0;
- this->clayout = new field_layout<fc>(
- sizes, subsizes, starts, this->comm);
- this->data = fftw_interface<rnumber>::alloc_real(
- this->rmemlayout->local_size);
- memset(this->data, 0, sizeof(rnumber)*this->rmemlayout->local_size);
- this->c2r_plan = fftw_interface<rnumber>::mpi_plan_many_dft_c2r(
- 3, nfftw, ncomp(fc),
- FFTW_MPI_DEFAULT_BLOCK, FFTW_MPI_DEFAULT_BLOCK,
- (typename fftw_interface<rnumber>::complex*)this->data,
- this->data,
- this->comm,
- this->fftw_plan_rigor | FFTW_MPI_TRANSPOSED_IN);
- this->r2c_plan = fftw_interface<rnumber>::mpi_plan_many_dft_r2c(
- 3, nfftw, ncomp(fc),
- FFTW_MPI_DEFAULT_BLOCK, FFTW_MPI_DEFAULT_BLOCK,
- this->data,
- (typename fftw_interface<rnumber>::complex*)this->data,
- this->comm,
- this->fftw_plan_rigor | FFTW_MPI_TRANSPOSED_OUT);
- break;
- }
-}
-
-template <typename rnumber,
- field_backend be,
- field_components fc>
-field<rnumber, be, fc>::~field()
-{
- /* close data types */
- H5Tclose(this->rnumber_H5T);
- H5Tclose(this->cnumber_H5T);
- switch(be)
- {
- case FFTW:
- delete this->rlayout;
- delete this->rmemlayout;
- delete this->clayout;
- fftw_interface<rnumber>::free(this->data);
- fftw_interface<rnumber>::destroy_plan(this->c2r_plan);
- fftw_interface<rnumber>::destroy_plan(this->r2c_plan);
- break;
- }
-}
-
-template <typename rnumber,
- field_backend be,
- field_components fc>
-void field<rnumber, be, fc>::ift()
-{
- TIMEZONE("field::ift");
- fftw_interface<rnumber>::execute(this->c2r_plan);
- this->real_space_representation = true;
-}
-
-template <typename rnumber,
- field_backend be,
- field_components fc>
-void field<rnumber, be, fc>::dft()
-{
- TIMEZONE("field::dft");
- fftw_interface<rnumber>::execute(this->r2c_plan);
- this->real_space_representation = false;
-}
-
-template <typename rnumber,
- field_backend be,
- field_components fc>
-int field<rnumber, be, fc>::io(
- const std::string fname,
- const std::string field_name,
- const int iteration,
- const bool read)
-{
- /* file dataset has same dimensions as field */
- TIMEZONE("field::io");
- hid_t file_id, dset_id, plist_id;
- dset_id = H5I_BADID;
- std::string representation = std::string(
- this->real_space_representation ?
- "real" : "complex");
- std::string dset_name = (
- "/" + field_name +
- "/" + representation +
- "/" + std::to_string(iteration));
-
- /* open/create file */
- plist_id = H5Pcreate(H5P_FILE_ACCESS);
- H5Pset_fapl_mpio(plist_id, this->comm, MPI_INFO_NULL);
- bool file_exists = false;
- {
- struct stat file_buffer;
- file_exists = (stat(fname.c_str(), &file_buffer) == 0);
- }
- if (read)
- {
- assert(file_exists);
- file_id = H5Fopen(fname.c_str(), H5F_ACC_RDONLY, plist_id);
- }
- else
- {
- if (file_exists)
- file_id = H5Fopen(fname.c_str(), H5F_ACC_RDWR, plist_id);
- else
- file_id = H5Fcreate(fname.c_str(), H5F_ACC_EXCL, H5P_DEFAULT, plist_id);
- }
- assert(file_id >= 0);
- H5Pclose(plist_id);
-
- /* check what kind of representation is being used */
- if (read)
- {
- dset_id = H5Dopen(
- file_id,
- dset_name.c_str(),
- H5P_DEFAULT);
- assert(dset_id >= 0);
- hid_t dset_type = H5Dget_type(dset_id);
- assert(dset_type >= 0);
- bool io_for_real = (
- H5Tequal(dset_type, H5T_IEEE_F32BE) ||
- H5Tequal(dset_type, H5T_IEEE_F32LE) ||
- H5Tequal(dset_type, H5T_INTEL_F32) ||
- H5Tequal(dset_type, H5T_NATIVE_FLOAT) ||
- H5Tequal(dset_type, H5T_IEEE_F64BE) ||
- H5Tequal(dset_type, H5T_IEEE_F64LE) ||
- H5Tequal(dset_type, H5T_INTEL_F64) ||
- H5Tequal(dset_type, H5T_NATIVE_DOUBLE));
- H5Tclose(dset_type);
- assert(this->real_space_representation == io_for_real);
- }
-
- /* generic space initialization */
- hid_t fspace, mspace;
- hsize_t count[ndim(fc)], offset[ndim(fc)], dims[ndim(fc)];
- hsize_t memoffset[ndim(fc)], memshape[ndim(fc)];
-
- if (this->real_space_representation)
- {
- for (unsigned int i=0; i<ndim(fc); i++)
- {
- count[i] = this->rlayout->subsizes[i];
- offset[i] = this->rlayout->starts[i];
- dims[i] = this->rlayout->sizes[i];
- memshape[i] = this->rmemlayout->subsizes[i];
- memoffset[i] = 0;
- }
- }
- else
- {
- for (unsigned int i=0; i<ndim(fc); i++)
- {
- count [i] = this->clayout->subsizes[i];
- offset[i] = this->clayout->starts[i];
- dims [i] = this->clayout->sizes[i];
- memshape [i] = count[i];
- memoffset[i] = 0;
- }
- }
- mspace = H5Screate_simple(ndim(fc), memshape, NULL);
- H5Sselect_hyperslab(mspace, H5S_SELECT_SET, memoffset, NULL, count, NULL);
-
- /* open/create data set */
- if (read)
- fspace = H5Dget_space(dset_id);
- else
- {
- if (!H5Lexists(file_id, field_name.c_str(), H5P_DEFAULT))
- {
- hid_t gid_tmp = H5Gcreate(
- file_id, field_name.c_str(),
- H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT);
- H5Gclose(gid_tmp);
- }
-
- if (!H5Lexists(file_id, (field_name + "/" + representation).c_str(), H5P_DEFAULT))
- {
- hid_t gid_tmp = H5Gcreate(
- file_id, ("/" + field_name + "/" + representation).c_str(),
- H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT);
- H5Gclose(gid_tmp);
- }
- if (H5Lexists(file_id, dset_name.c_str(), H5P_DEFAULT))
- {
- dset_id = H5Dopen(file_id, dset_name.c_str(), H5P_DEFAULT);
- fspace = H5Dget_space(dset_id);
- }
- else
- {
- fspace = H5Screate_simple(
- ndim(fc),
- dims,
- NULL);
- /* chunking needs to go in here */
- dset_id = H5Dcreate(
- file_id,
- dset_name.c_str(),
- (this->real_space_representation ? this->rnumber_H5T : this->cnumber_H5T),
- fspace,
- H5P_DEFAULT,
- H5P_DEFAULT,
- H5P_DEFAULT);
- }
- }
- /* both dset_id and fspace should now have sane values */
-
- /* check file space */
- int ndims_fspace = H5Sget_simple_extent_dims(fspace, dims, NULL);
- assert(((unsigned int)(ndims_fspace)) == ndim(fc));
- if (this->real_space_representation)
- {
- for (unsigned int i=0; i<ndim(fc); i++)
- {
- offset[i] = this->rlayout->starts[i];
- assert(dims[i] == this->rlayout->sizes[i]);
- }
- H5Sselect_hyperslab(fspace, H5S_SELECT_SET, offset, NULL, count, NULL);
- if (read)
- {
- std::fill_n(this->data, this->rmemlayout->local_size, 0);
- H5Dread(dset_id, this->rnumber_H5T, mspace, fspace, H5P_DEFAULT, this->data);
- }
- else
- {
- assert(this->real_space_representation);
- H5Dwrite(dset_id, this->rnumber_H5T, mspace, fspace, H5P_DEFAULT, this->data);
- }
- H5Sclose(mspace);
- }
- else
- {
- for (unsigned int i=0; i<ndim(fc); i++)
- {
- offset[i] = this->clayout->starts[i];
- assert(dims[i] == this->clayout->sizes[i]);
- }
- H5Sselect_hyperslab(fspace, H5S_SELECT_SET, offset, NULL, count, NULL);
- if (read)
- {
- std::fill_n(this->data, this->clayout->local_size*2, 0);
- H5Dread(dset_id, this->cnumber_H5T, mspace, fspace, H5P_DEFAULT, this->data);
- this->symmetrize();
- }
- else
- {
- assert(!this->real_space_representation);
- H5Dwrite(dset_id, this->cnumber_H5T, mspace, fspace, H5P_DEFAULT, this->data);
- }
- H5Sclose(mspace);
- }
-
- H5Sclose(fspace);
- /* close data set */
- H5Dclose(dset_id);
- /* close file */
- H5Fclose(file_id);
- return EXIT_SUCCESS;
-}
-
-template <typename rnumber,
- field_backend be,
- field_components fc>
-int field<rnumber, be, fc>::io_database(
- const std::string fname,
- const std::string field_name,
- const int toffset,
- const bool read)
-{
- /* file dataset is has a time dimension as well */
- TIMEZONE("field::io_database");
- hid_t file_id, dset_id, plist_id;
- dset_id = H5I_BADID;
- std::string representation = std::string(
- this->real_space_representation ?
- "real" : "complex");
- std::string dset_name = (
- "/" + field_name +
- "/" + representation);
-
- /* open/create file */
- plist_id = H5Pcreate(H5P_FILE_ACCESS);
- H5Pset_fapl_mpio(plist_id, this->comm, MPI_INFO_NULL);
- bool file_exists = false;
- {
- struct stat file_buffer;
- file_exists = (stat(fname.c_str(), &file_buffer) == 0);
- }
- if (read)
- {
- assert(file_exists);
- file_id = H5Fopen(fname.c_str(), H5F_ACC_RDONLY, plist_id);
- }
- else
- {
- if (file_exists)
- file_id = H5Fopen(fname.c_str(), H5F_ACC_RDWR, plist_id);
- else
- file_id = H5Fcreate(fname.c_str(), H5F_ACC_EXCL, H5P_DEFAULT, plist_id);
- }
- H5Pclose(plist_id);
-
- /* check what kind of representation is being used */
- if (read)
- {
- dset_id = H5Dopen(
- file_id,
- dset_name.c_str(),
- H5P_DEFAULT);
- hid_t dset_type = H5Dget_type(dset_id);
- bool io_for_real = (
- H5Tequal(dset_type, H5T_IEEE_F32BE) ||
- H5Tequal(dset_type, H5T_IEEE_F32LE) ||
- H5Tequal(dset_type, H5T_INTEL_F32) ||
- H5Tequal(dset_type, H5T_NATIVE_FLOAT) ||
- H5Tequal(dset_type, H5T_IEEE_F64BE) ||
- H5Tequal(dset_type, H5T_IEEE_F64LE) ||
- H5Tequal(dset_type, H5T_INTEL_F64) ||
- H5Tequal(dset_type, H5T_NATIVE_DOUBLE));
- H5Tclose(dset_type);
- assert(this->real_space_representation == io_for_real);
- }
-
- /* generic space initialization */
- hid_t fspace, mspace;
- hsize_t count[ndim(fc)+1], offset[ndim(fc)+1], dims[ndim(fc)+1];
- hsize_t memoffset[ndim(fc)+1], memshape[ndim(fc)+1];
-
- int dim_counter_offset = 1;
- dim_counter_offset = 1;
- count[0] = 1;
- memshape[0] = 1;
- memoffset[0] = 0;
- if (this->real_space_representation)
- {
- for (unsigned int i=0; i<ndim(fc); i++)
- {
- count[i+dim_counter_offset] = this->rlayout->subsizes[i];
- offset[i+dim_counter_offset] = this->rlayout->starts[i];
- dims[i+dim_counter_offset] = this->rlayout->sizes[i];
- memshape[i+dim_counter_offset] = this->rmemlayout->subsizes[i];
- memoffset[i+dim_counter_offset] = 0;
- }
- mspace = H5Screate_simple(dim_counter_offset + ndim(fc), memshape, NULL);
- H5Sselect_hyperslab(mspace, H5S_SELECT_SET, memoffset, NULL, count, NULL);
- }
- else
- {
- for (unsigned int i=0; i<ndim(fc); i++)
- {
- count[i+dim_counter_offset] = this->clayout->subsizes[i];
- offset[i+dim_counter_offset] = this->clayout->starts[i];
- dims[i+dim_counter_offset] = this->clayout->sizes[i];
- memshape[i+dim_counter_offset] = count[i+dim_counter_offset];
- memoffset[i+dim_counter_offset] = 0;
- }
- mspace = H5Screate_simple(dim_counter_offset + ndim(fc), memshape, NULL);
- H5Sselect_hyperslab(mspace, H5S_SELECT_SET, memoffset, NULL, count, NULL);
- }
-
- /* open/create data set */
- if (read)
- fspace = H5Dget_space(dset_id);
- else
- {
- if (!H5Lexists(file_id, field_name.c_str(), H5P_DEFAULT))
- H5Gcreate(
- file_id, field_name.c_str(),
- H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT);
- if (H5Lexists(file_id, dset_name.c_str(), H5P_DEFAULT))
- {
- dset_id = H5Dopen(file_id, dset_name.c_str(), H5P_DEFAULT);
- fspace = H5Dget_space(dset_id);
- }
- else
- {
- fspace = H5Screate_simple(
- ndim(fc),
- dims,
- NULL);
- /* chunking needs to go in here */
- dset_id = H5Dcreate(
- file_id,
- dset_name.c_str(),
- (this->real_space_representation ? this->rnumber_H5T : this->cnumber_H5T),
- fspace,
- H5P_DEFAULT,
- H5P_DEFAULT,
- H5P_DEFAULT);
- }
- }
- /* both dset_id and fspace should now have sane values */
-
- /* check file space */
- int ndims_fspace = H5Sget_simple_extent_dims(fspace, dims, NULL);
- assert(ndims_fspace == int(ndim(fc) + 1));
- offset[0] = toffset;
- if (this->real_space_representation)
- {
- for (unsigned int i=0; i<ndim(fc); i++)
- {
- offset[i+dim_counter_offset] = this->rlayout->starts[i];
- assert(dims[i+dim_counter_offset] == this->rlayout->sizes[i]);
- }
- H5Sselect_hyperslab(fspace, H5S_SELECT_SET, offset, NULL, count, NULL);
- if (read)
- {
- std::fill_n(this->data, this->rmemlayout->local_size, 0);
- H5Dread(dset_id, this->rnumber_H5T, mspace, fspace, H5P_DEFAULT, this->data);
- this->real_space_representation = true;
- }
- else
- {
- assert(this->real_space_representation);
- H5Dwrite(dset_id, this->rnumber_H5T, mspace, fspace, H5P_DEFAULT, this->data);
- }
- H5Sclose(mspace);
- }
- else
- {
- for (unsigned int i=0; i<ndim(fc); i++)
- {
- offset[i+dim_counter_offset] = this->clayout->starts[i];
- assert(dims[i+dim_counter_offset] == this->clayout->sizes[i]);
- }
- H5Sselect_hyperslab(fspace, H5S_SELECT_SET, offset, NULL, count, NULL);
- if (read)
- {
- H5Dread(dset_id, this->cnumber_H5T, mspace, fspace, H5P_DEFAULT, this->data);
- this->real_space_representation = false;
- this->symmetrize();
- }
- else
- {
- assert(!this->real_space_representation);
- H5Dwrite(dset_id, this->cnumber_H5T, mspace, fspace, H5P_DEFAULT, this->data);
- }
- H5Sclose(mspace);
- }
-
- H5Sclose(fspace);
- /* close data set */
- H5Dclose(dset_id);
- /* close file */
- H5Fclose(file_id);
- return EXIT_SUCCESS;
-}
-
-
-template <typename rnumber,
- field_backend be,
- field_components fc>
-int field<rnumber, be, fc>::write_0slice(
- const hid_t group,
- const std::string field_name,
- const int iteration)
-{
- // this should in principle work for any fc
- TIMEZONE("field::write_0slice");
- assert(this->real_space_representation);
- if (this->myrank == 0)
- {
- hid_t dset, wspace, mspace;
- int ndims;
- hsize_t count[5], offset[5], dims[5];
- offset[0] = iteration;
- offset[1] = 0;
- offset[2] = 0;
- offset[3] = 0;
- offset[4] = 0;
- dset = H5Dopen(
- group,
- ("0slices/" + field_name + "/real").c_str(),
- H5P_DEFAULT);
- wspace = H5Dget_space(dset);
- ndims = H5Sget_simple_extent_dims(wspace, dims, NULL);
- // array in memory has 2 extra x points, because FFTW
- count[0] = 1;
- count[1] = this->rmemlayout->sizes[1];
- count[2] = this->rmemlayout->sizes[2];
- count[3] = 3;
- count[3] = 3;
- mspace = H5Screate_simple(ndims, count, NULL);
- // array in file should not have the extra 2 points
- count[1] = this->rlayout->sizes[1];
- count[2] = this->rlayout->sizes[2];
- // select right slice in file
- H5Sselect_hyperslab(
- wspace,
- H5S_SELECT_SET,
- offset,
- NULL,
- count,
- NULL);
- offset[0] = 0;
- // select proper regions of memory
- H5Sselect_hyperslab(
- mspace,
- H5S_SELECT_SET,
- offset,
- NULL,
- count,
- NULL);
- H5Dwrite(
- dset,
- this->rnumber_H5T,
- mspace,
- wspace,
- H5P_DEFAULT,
- this->data);
- H5Dclose(dset);
- H5Sclose(mspace);
- H5Sclose(wspace);
- }
- return EXIT_SUCCESS;
-}
-
-
-template <typename rnumber,
- field_backend be,
- field_components fc>
-void field<rnumber, be, fc>::compute_rspace_xincrement_stats(
- const int xcells,
- const hid_t group,
- const std::string dset_name,
- const hsize_t toffset,
- const std::vector<double> max_estimate)
-{
- TIMEZONE("field::compute_rspace_xincrement_stats");
- assert(this->real_space_representation);
- assert(fc == ONE || fc == THREE);
- field<rnumber, be, fc> *tmp_field = new field<rnumber, be, fc>(
- this->rlayout->sizes[2],
- this->rlayout->sizes[1],
- this->rlayout->sizes[0],
- this->rlayout->comm);
- tmp_field->real_space_representation = true;
- this->RLOOP(
- [&](ptrdiff_t rindex,
- ptrdiff_t xindex,
- ptrdiff_t yindex,
- ptrdiff_t zindex){
- hsize_t rrindex = (xindex + xcells)%this->rlayout->sizes[2] + (
- zindex * this->rlayout->subsizes[1] + yindex)*(
- this->rmemlayout->subsizes[2]);
- for (unsigned int component=0; component < ncomp(fc); component++)
- tmp_field->data[rindex*ncomp(fc) + component] =
- this->data[rrindex*ncomp(fc) + component] -
- this->data[rindex*ncomp(fc) + component];
- });
- tmp_field->compute_rspace_stats(
- group,
- dset_name,
- toffset,
- max_estimate);
- delete tmp_field;
-}
-
-
-
-template <typename rnumber,
- field_backend be,
- field_components fc>
-void field<rnumber, be, fc>::compute_rspace_stats(
- const hid_t group,
- const std::string dset_name,
- const hsize_t toffset,
- const std::vector<double> max_estimate)
-{
- TIMEZONE("field::compute_rspace_stats");
- assert(this->real_space_representation);
- const unsigned int nmoments = 10;
- int nvals, nbins;
- if (this->myrank == 0)
- {
- hid_t dset, wspace;
- hsize_t dims[ndim(fc)-1];
- int ndims;
- dset = H5Dopen(group, ("moments/" + dset_name).c_str(), H5P_DEFAULT);
- wspace = H5Dget_space(dset);
- ndims = H5Sget_simple_extent_dims(wspace, dims, NULL);
- assert(ndims == int(ndim(fc))-1);
- assert(dims[1] == nmoments);
- switch(ndims)
- {
- case 2:
- nvals = 1;
- break;
- case 3:
- nvals = dims[2];
- break;
- case 4:
- nvals = dims[2]*dims[3];
- break;
- }
- H5Sclose(wspace);
- H5Dclose(dset);
- dset = H5Dopen(group, ("histograms/" + dset_name).c_str(), H5P_DEFAULT);
- wspace = H5Dget_space(dset);
- ndims = H5Sget_simple_extent_dims(wspace, dims, NULL);
- assert(ndims == int(ndim(fc))-1);
- nbins = dims[1];
- if (ndims == 3)
- assert(nvals == int(dims[2]));
- else if (ndims == 4)
- assert(nvals == int(dims[2]*dims[3]));
- H5Sclose(wspace);
- H5Dclose(dset);
- }
- {
- TIMEZONE("MPI_Bcast");
- MPI_Bcast(&nvals, 1, MPI_INT, 0, this->comm);
- MPI_Bcast(&nbins, 1, MPI_INT, 0, this->comm);
- }
- assert(nvals == int(max_estimate.size()));
-
- shared_array<double> local_moments_threaded(nmoments*nvals, [&](double* local_moments){
- std::fill_n(local_moments, nmoments*nvals, 0);
- if (nvals == 4) local_moments[3] = max_estimate[3];
- });
-
- shared_array<double> val_tmp_threaded(nvals,[&](double *val_tmp){
- std::fill_n(val_tmp, nvals, 0);
- });
-
- shared_array<ptrdiff_t> local_hist_threaded(nbins*nvals,[&](ptrdiff_t* local_hist){
- std::fill_n(local_hist, nbins*nvals, 0);
- });
-
- shared_array<double> local_pow_tmp(nvals);
-
- double *binsize = new double[nvals];
- for (int i=0; i<nvals; i++)
- binsize[i] = 2*max_estimate[i] / nbins;
-
- {
- TIMEZONE("field::RLOOP");
- this->RLOOP(
- [&](ptrdiff_t rindex,
- ptrdiff_t xindex,
- ptrdiff_t yindex,
- ptrdiff_t zindex){
- double* pow_tmp = local_pow_tmp.getMine();
- std::fill_n(pow_tmp, nvals, 1);
-
- double *local_moments = local_moments_threaded.getMine();
- double *val_tmp = val_tmp_threaded.getMine();
- ptrdiff_t *local_hist = local_hist_threaded.getMine();
-
- if (nvals == int(4)) val_tmp[3] = 0.0;
- for (unsigned int i=0; i<ncomp(fc); i++)
- {
- val_tmp[i] = this->data[rindex*ncomp(fc)+i];
- if (nvals == int(4)) val_tmp[3] += val_tmp[i]*val_tmp[i];
- }
- if (nvals == int(4))
- {
- val_tmp[3] = sqrt(val_tmp[3]);
- if (val_tmp[3] < local_moments[0*nvals+3])
- local_moments[0*nvals+3] = val_tmp[3];
- if (val_tmp[3] > local_moments[9*nvals+3])
- local_moments[9*nvals+3] = val_tmp[3];
- int bin = int(floor(val_tmp[3]*2/binsize[3]));
- if (bin >= 0 && bin < nbins){
- local_hist[bin*nvals+3]++;
- }
- }
- for (unsigned int i=0; i<ncomp(fc); i++)
- {
- if (val_tmp[i] < local_moments[0*nvals+i])
- local_moments[0*nvals+i] = val_tmp[i];
- if (val_tmp[i] > local_moments[(nmoments-1)*nvals+i])
- local_moments[(nmoments-1)*nvals+i] = val_tmp[i];
- int bin = int(floor((val_tmp[i] + max_estimate[i]) / binsize[i]));
- if (bin >= 0 && bin < nbins)
- local_hist[bin*nvals+i]++;
- }
- for (int n=1; n < int(nmoments)-1; n++){
- for (int i=0; i<nvals; i++){
- local_moments[n*nvals + i] += (pow_tmp[i] = val_tmp[i]*pow_tmp[i]);
- }
- }
- });
-
- TIMEZONE("FIELD_RLOOP::Merge");
- local_moments_threaded.mergeParallel([&](const int idx, const double& v1, const double& v2) -> double {
- if(nvals == int(4) && idx == 0*nvals+3){
- return std::min(v1, v2);
- }
- if(nvals == int(4) && idx == 9*nvals+3){
- return std::max(v1, v2);
- }
- if(idx < int(ncomp(fc))){
- return std::min(v1, v2);
- }
- if(int(nmoments-1)*nvals <= idx && idx < int(int(nmoments-1)*nvals+ncomp(fc))){
- return std::max(v1, v2);
- }
- return v1 + v2;
- });
-
- local_hist_threaded.mergeParallel();
- }
- ptrdiff_t *hist = new ptrdiff_t[nbins*nvals];
- double *moments = new double[nmoments*nvals];
- {
- TIMEZONE("MPI_Allreduce");
- MPI_Allreduce(
- (void*)local_moments_threaded.getMasterData(),
- (void*)moments,
- nvals,
- MPI_DOUBLE, MPI_MIN, this->comm);
- MPI_Allreduce(
- (void*)(local_moments_threaded.getMasterData() + nvals),
- (void*)(moments+nvals),
- (nmoments-2)*nvals,
- MPI_DOUBLE, MPI_SUM, this->comm);
- MPI_Allreduce(
- (void*)(local_moments_threaded.getMasterData() + (nmoments-1)*nvals),
- (void*)(moments+(nmoments-1)*nvals),
- nvals,
- MPI_DOUBLE, MPI_MAX, this->comm);
- MPI_Allreduce(
- (void*)local_hist_threaded.getMasterData(),
- (void*)hist,
- nbins*nvals,
- MPI_INT64_T, MPI_SUM, this->comm);
- }
- for (int n=1; n < int(nmoments)-1; n++)
- for (int i=0; i<nvals; i++)
- moments[n*nvals + i] /= this->npoints;
-
- delete[] binsize;
- if (this->myrank == 0)
- {
- TIMEZONE("root-work");
- hid_t dset, wspace, mspace;
- hsize_t count[ndim(fc)-1], offset[ndim(fc)-1], dims[ndim(fc)-1];
- dset = H5Dopen(group, ("moments/" + dset_name).c_str(), H5P_DEFAULT);
- assert(dset>0);
- wspace = H5Dget_space(dset);
- H5Sget_simple_extent_dims(wspace, dims, NULL);
- offset[0] = toffset;
- offset[1] = 0;
- count[0] = 1;
- count[1] = nmoments;
- if (fc == THREE)
- {
- offset[2] = 0;
- count[2] = nvals;
- }
- if (fc == THREExTHREE)
- {
- offset[2] = 0;
- count[2] = 3;
- offset[3] = 0;
- count[3] = 3;
- }
- mspace = H5Screate_simple(ndim(fc)-1, count, NULL);
- H5Sselect_hyperslab(wspace, H5S_SELECT_SET, offset, NULL, count, NULL);
- H5Dwrite(dset, H5T_NATIVE_DOUBLE, mspace, wspace, H5P_DEFAULT, moments);
- H5Sclose(wspace);
- H5Sclose(mspace);
- H5Dclose(dset);
- dset = H5Dopen(group, ("histograms/" + dset_name).c_str(), H5P_DEFAULT);
- assert(dset > 0);
- wspace = H5Dget_space(dset);
- count[1] = nbins;
- mspace = H5Screate_simple(ndim(fc)-1, count, NULL);
- H5Sselect_hyperslab(wspace, H5S_SELECT_SET, offset, NULL, count, NULL);
- H5Dwrite(dset, H5T_NATIVE_INT64, mspace, wspace, H5P_DEFAULT, hist);
- H5Sclose(wspace);
- H5Sclose(mspace);
- H5Dclose(dset);
- if (H5Lexists(
- group,
- "0slices",
- H5P_DEFAULT))
- {
- if (H5Lexists(
- group,
- (std::string("0slices/") + dset_name).c_str(),
- H5P_DEFAULT))
- this->write_0slice(
- group,
- dset_name,
- toffset);
- }
- }
- delete[] moments;
- delete[] hist;
-}
-
-template <typename rnumber,
- field_backend be,
- field_components fc>
-void field<rnumber, be, fc>::normalize()
-{
- for (hsize_t tmp_index=0; tmp_index<this->rmemlayout->local_size; tmp_index++)
- this->data[tmp_index] /= this->npoints;
-}
-
-template <typename rnumber,
- field_backend be,
- field_components fc>
-void field<rnumber, be, fc>::symmetrize()
-{
- TIMEZONE("field::symmetrize");
- assert(!this->real_space_representation);
- ptrdiff_t ii, cc;
- typename fftw_interface<rnumber>::complex *data = this->get_cdata();
- MPI_Status *mpistatus = new MPI_Status;
- if (this->myrank == this->clayout->rank[0][0])
- {
- for (cc = 0; cc < ncomp(fc); cc++)
- data[cc][1] = 0.0;
- for (ii = 1; ii < ptrdiff_t(this->clayout->sizes[1]/2); ii++)
- for (cc = 0; cc < ncomp(fc); cc++) {
- ( *(data + cc + ncomp(fc)*(this->clayout->sizes[1] - ii)*this->clayout->sizes[2]))[0] =
- (*(data + cc + ncomp(fc)*( ii)*this->clayout->sizes[2]))[0];
- ( *(data + cc + ncomp(fc)*(this->clayout->sizes[1] - ii)*this->clayout->sizes[2]))[1] =
- -(*(data + cc + ncomp(fc)*( ii)*this->clayout->sizes[2]))[1];
- }
- }
- typename fftw_interface<rnumber>::complex *buffer;
- buffer = fftw_interface<rnumber>::alloc_complex(ncomp(fc)*this->clayout->sizes[1]);
- ptrdiff_t yy;
- /*ptrdiff_t tindex;*/
- int ranksrc, rankdst;
- for (yy = 1; yy < ptrdiff_t(this->clayout->sizes[0]/2); yy++) {
- ranksrc = this->clayout->rank[0][yy];
- rankdst = this->clayout->rank[0][this->clayout->sizes[0] - yy];
- if (this->clayout->myrank == ranksrc)
- for (ii = 0; ii < ptrdiff_t(this->clayout->sizes[1]); ii++)
- for (cc = 0; cc < ncomp(fc); cc++)
- for (int imag_comp=0; imag_comp<2; imag_comp++)
- (*(buffer + ncomp(fc)*ii+cc))[imag_comp] =
- (*(data + ncomp(fc)*((yy - this->clayout->starts[0])*this->clayout->sizes[1] + ii)*this->clayout->sizes[2] + cc))[imag_comp];
- if (ranksrc != rankdst)
- {
- if (this->clayout->myrank == ranksrc)
- MPI_Send((void*)buffer,
- ncomp(fc)*this->clayout->sizes[1], mpi_real_type<rnumber>::complex(), rankdst, yy,
- this->clayout->comm);
- if (this->clayout->myrank == rankdst)
- MPI_Recv((void*)buffer,
- ncomp(fc)*this->clayout->sizes[1], mpi_real_type<rnumber>::complex(), ranksrc, yy,
- this->clayout->comm, mpistatus);
- }
- if (this->clayout->myrank == rankdst)
- {
- for (ii = 1; ii < ptrdiff_t(this->clayout->sizes[1]); ii++)
- for (cc = 0; cc < ncomp(fc); cc++)
- {
- (*(data + ncomp(fc)*((this->clayout->sizes[0] - yy - this->clayout->starts[0])*this->clayout->sizes[1] + ii)*this->clayout->sizes[2] + cc))[0] =
- (*(buffer + ncomp(fc)*(this->clayout->sizes[1]-ii)+cc))[0];
- (*(data + ncomp(fc)*((this->clayout->sizes[0] - yy - this->clayout->starts[0])*this->clayout->sizes[1] + ii)*this->clayout->sizes[2] + cc))[1] =
- -(*(buffer + ncomp(fc)*(this->clayout->sizes[1]-ii)+cc))[1];
- }
- for (cc = 0; cc < ncomp(fc); cc++)
- {
- (*((data + cc + ncomp(fc)*(this->clayout->sizes[0] - yy - this->clayout->starts[0])*this->clayout->sizes[1]*this->clayout->sizes[2])))[0] = (*(buffer + cc))[0];
- (*((data + cc + ncomp(fc)*(this->clayout->sizes[0] - yy - this->clayout->starts[0])*this->clayout->sizes[1]*this->clayout->sizes[2])))[1] = -(*(buffer + cc))[1];
- }
- }
- }
- fftw_interface<rnumber>::free(buffer);
- delete mpistatus;
- /* put asymmetric data to 0 */
- /*if (this->clayout->myrank == this->clayout->rank[0][this->clayout->sizes[0]/2])
- {
- tindex = ncomp(fc)*(this->clayout->sizes[0]/2 - this->clayout->starts[0])*this->clayout->sizes[1]*this->clayout->sizes[2];
- for (ii = 0; ii < this->clayout->sizes[1]; ii++)
- {
- std::fill_n((rnumber*)(data + tindex), ncomp(fc)*2*this->clayout->sizes[2], 0.0);
- tindex += ncomp(fc)*this->clayout->sizes[2];
- }
- }
- tindex = ncomp(fc)*();
- std::fill_n((rnumber*)(data + tindex), ncomp(fc)*2, 0.0);*/
-}
-
-template <typename rnumber,
- field_backend be,
- field_components fc>
-template <kspace_dealias_type dt>
-void field<rnumber, be, fc>::compute_stats(
- kspace<be, dt> *kk,
- const hid_t group,
- const std::string dset_name,
- const hsize_t toffset,
- const double max_estimate)
-{
- TIMEZONE("field::compute_stats");
- std::vector<double> max_estimate_vector;
- bool did_rspace = false;
- switch(fc)
- {
- case ONE:
- max_estimate_vector.resize(1, max_estimate);
- break;
- case THREE:
- max_estimate_vector.resize(4, max_estimate);
- max_estimate_vector[3] *= sqrt(3);
- break;
- case THREExTHREE:
- max_estimate_vector.resize(9, max_estimate);
- break;
- }
- if (this->real_space_representation)
- {
- TIMEZONE("field::compute_stats::compute_rspace_stats");
- this->compute_rspace_stats(
- group,
- dset_name,
- toffset,
- max_estimate_vector);
- did_rspace = true;
- this->dft();
- // normalize
- TIMEZONE("field::normalize");
- for (hsize_t tmp_index=0; tmp_index<this->rmemlayout->local_size; tmp_index++)
- this->data[tmp_index] /= this->npoints;
- }
- // what follows gave me a headache until I found this link:
- // http://stackoverflow.com/questions/8256636/expected-primary-expression-error-on-template-method-using
- kk->template cospectrum<rnumber, fc>(
- (typename fftw_interface<rnumber>::complex*)this->data,
- (typename fftw_interface<rnumber>::complex*)this->data,
- group,
- dset_name + "_" + dset_name,
- toffset);
- if (!did_rspace)
- {
- this->ift();
- // normalization not required
- this->compute_rspace_stats(
- group,
- dset_name,
- toffset,
- max_estimate_vector);
- }
-}
-
-template <typename rnumber,
- field_backend be,
- field_components fc1,
- field_components fc2,
- kspace_dealias_type dt>
-int compute_gradient(
- kspace<be, dt> *kk,
- field<rnumber, be, fc1> *src,
- field<rnumber, be, fc2> *dst)
-{
- TIMEZONE("compute_gradient");
- assert(!src->real_space_representation);
- assert((fc1 == ONE && fc2 == THREE) ||
- (fc1 == THREE && fc2 == THREExTHREE));
- std::fill_n(dst->get_rdata(), dst->rmemlayout->local_size, 0);
- dst->real_space_representation = false;
- switch(fc1)
- {
- case ONE:
- kk->CLOOP_K2(
- [&](ptrdiff_t cindex,
- ptrdiff_t xindex,
- ptrdiff_t yindex,
- ptrdiff_t zindex,
- double k2){
- if (k2 < kk->kM2)
- {
- dst->cval(cindex, 0, 0) = -kk->kx[xindex]*src->cval(cindex, 1);
- dst->cval(cindex, 0, 1) = kk->kx[xindex]*src->cval(cindex, 0);
- dst->cval(cindex, 1, 0) = -kk->ky[yindex]*src->cval(cindex, 1);
- dst->cval(cindex, 1, 1) = kk->ky[yindex]*src->cval(cindex, 0);
- dst->cval(cindex, 2, 0) = -kk->kz[zindex]*src->cval(cindex, 1);
- dst->cval(cindex, 2, 1) = kk->kz[zindex]*src->cval(cindex, 0);
- }});
- break;
- case THREE:
- kk->CLOOP_K2(
- [&](ptrdiff_t cindex,
- ptrdiff_t xindex,
- ptrdiff_t yindex,
- ptrdiff_t zindex,
- double k2){
- if (k2 < kk->kM2)
- {
- for (unsigned int field_component = 0;
- field_component < ncomp(fc1);
- field_component++)
- {
- dst->cval(cindex, 0, field_component, 0) = -kk->kx[xindex]*src->cval(cindex, field_component, 1);
- dst->cval(cindex, 0, field_component, 1) = kk->kx[xindex]*src->cval(cindex, field_component, 0);
- dst->cval(cindex, 1, field_component, 0) = -kk->ky[yindex]*src->cval(cindex, field_component, 1);
- dst->cval(cindex, 1, field_component, 1) = kk->ky[yindex]*src->cval(cindex, field_component, 0);
- dst->cval(cindex, 2, field_component, 0) = -kk->kz[zindex]*src->cval(cindex, field_component, 1);
- dst->cval(cindex, 2, field_component, 1) = kk->kz[zindex]*src->cval(cindex, field_component, 0);
- }
- }});
- break;
- }
- return EXIT_SUCCESS;
-}
-
-template <typename rnumber,
- field_backend be,
- kspace_dealias_type dt>
-int invert_curl(
- kspace<be, dt> *kk,
- field<rnumber, be, THREE> *src,
- field<rnumber, be, THREE> *dst)
-{
- TIMEZONE("invert_curl");
- assert(!src->real_space_representation);
- std::fill_n(dst->get_rdata(), dst->rmemlayout->local_size, 0);
- dst->real_space_representation = false;
- kk->CLOOP_K2(
- [&](ptrdiff_t cindex,
- ptrdiff_t xindex,
- ptrdiff_t yindex,
- ptrdiff_t zindex,
- double k2){
- if (k2 <= kk->kM2 && k2 > 0)
- {
- dst->cval(cindex,0,0) = -(kk->ky[yindex]*src->cval(cindex,2,1) - kk->kz[zindex]*src->cval(cindex,1,1)) / k2;
- dst->cval(cindex,0,1) = (kk->ky[yindex]*src->cval(cindex,2,0) - kk->kz[zindex]*src->cval(cindex,1,0)) / k2;
- dst->cval(cindex,1,0) = -(kk->kz[zindex]*src->cval(cindex,0,1) - kk->kx[xindex]*src->cval(cindex,2,1)) / k2;
- dst->cval(cindex,1,1) = (kk->kz[zindex]*src->cval(cindex,0,0) - kk->kx[xindex]*src->cval(cindex,2,0)) / k2;
- dst->cval(cindex,2,0) = -(kk->kx[xindex]*src->cval(cindex,1,1) - kk->ky[yindex]*src->cval(cindex,0,1)) / k2;
- dst->cval(cindex,2,1) = (kk->kx[xindex]*src->cval(cindex,1,0) - kk->ky[yindex]*src->cval(cindex,0,0)) / k2;
- }
- else
- std::fill_n((rnumber*)(dst->get_cdata()+3*cindex), 6, 0.0);
- }
- );
- dst->symmetrize();
- return EXIT_SUCCESS;
-}
-
-template <typename rnumber,
- field_backend be,
- field_components fc>
-int joint_rspace_PDF(
- field<rnumber, be, fc> *f1,
- field<rnumber, be, fc> *f2,
- const hid_t group,
- const std::string dset_name,
- const hsize_t toffset,
- const std::vector<double> max_f1_estimate,
- const std::vector<double> max_f2_estimate)
-{
- TIMEZONE("joint_rspace_PDF");
- assert(f1->real_space_representation);
- assert(f2->real_space_representation);
- if (fc == THREE)
- {
- assert(max_f1_estimate.size() == 4);
- assert(max_f2_estimate.size() == 4);
- }
- else if (fc == ONE)
- {
- assert(max_f1_estimate.size() == 1);
- assert(max_f2_estimate.size() == 1);
- }
- int nbins;
- std::string dsetc, dsetm;
- dsetc = "histograms/" + dset_name + "_components";
- if (fc == THREE)
- dsetm = "histograms/" + dset_name + "_magnitudes";
- else
- dsetm = "histograms/" + dset_name;
- if (f1->myrank == 0)
- {
- hid_t dset, wspace;
- hsize_t dims[5];
- int ndims;
- if (fc == THREE)
- {
- dset = H5Dopen(
- group,
- dsetc.c_str(),
- H5P_DEFAULT);
- wspace = H5Dget_space(dset);
- ndims = H5Sget_simple_extent_dims(wspace, dims, NULL);
- DEBUG_MSG("number of dimensions is %d\n", ndims);
- assert(ndims == 5);
- assert(dims[3] == 3);
- assert(dims[4] == 3);
- H5Sclose(wspace);
- H5Dclose(dset);
- }
- dset = H5Dopen(
- group,
- dsetm.c_str(),
- H5P_DEFAULT);
- wspace = H5Dget_space(dset);
- ndims = H5Sget_simple_extent_dims(wspace, dims, NULL);
- assert(ndims == 3);
- H5Sclose(wspace);
- H5Dclose(dset);
- nbins = dims[1];
- }
- {
- TIMEZONE("MPI_Bcast");
- MPI_Bcast(&nbins, 1, MPI_INT, 0, f1->comm);
- }
-
- /// histogram components
- shared_array<ptrdiff_t> local_histc_threaded(
- nbins*nbins*9,
- [&](ptrdiff_t* local_hist){
- std::fill_n(local_hist, nbins*nbins*9, 0);
- });
-
- /// histogram magnitudes
- shared_array<ptrdiff_t> local_histm_threaded(
- nbins*nbins,
- [&](ptrdiff_t* local_hist){
- std::fill_n(local_hist, nbins*nbins, 0);
- });
-
- /// set up bin sizes
- std::vector<double> bin1size, bin2size;
- bin1size.resize(4);
- bin2size.resize(4);
- if (fc == THREE)
- {
- for (unsigned int i=0; i<3; i++)
- {
- bin1size[i] = 2*max_f1_estimate[i] / nbins;
- bin2size[i] = 2*max_f2_estimate[i] / nbins;
- }
- bin1size[3] = max_f1_estimate[3] / nbins;
- bin2size[3] = max_f2_estimate[3] / nbins;
- }
- else if (fc == ONE)
- {
- for (unsigned int i=0; i<4; i++)
- {
- bin1size[i] = max_f1_estimate[0] / nbins;
- bin2size[i] = max_f2_estimate[0] / nbins;
- }
- }
-
-
- {
- TIMEZONE("field::RLOOP");
- f1->RLOOP(
- [&](ptrdiff_t rindex,
- ptrdiff_t xindex,
- ptrdiff_t yindex,
- ptrdiff_t zindex){
- ptrdiff_t *local_histm = local_histm_threaded.getMine();
- int bin1 = 0;
- int bin2 = 0;
- if (fc == THREE)
- {
- ptrdiff_t *local_histc = local_histc_threaded.getMine();
-
- double mag1, mag2;
- mag1 = 0.0;
- mag2 = 0.0;
- for (unsigned int i=0; i<3; i++)
- {
- double val1 = f1->rval(rindex, i);
- mag1 += val1*val1;
- int bin1 = int(floor((val1 + max_f1_estimate[i])/bin1size[i]));
- mag2 = 0.0;
- for (unsigned int j=0; j<3; j++)
- {
- double val2 = f2->rval(rindex, j);
- mag2 += val2*val2;
- int bin2 = int(floor((val2 + max_f2_estimate[j])/bin2size[j]));
- if ((bin1 >= 0 && bin1 < nbins) &&
- (bin2 >= 0 && bin2 < nbins))
- local_histc[(bin1*nbins + bin2)*9 + i*3 + j]++;
- }
- }
- bin1 = int(floor(sqrt(mag1)/bin1size[3]));
- bin2 = int(floor(sqrt(mag2)/bin2size[3]));
- }
- else if (fc == ONE)
- {
- bin1 = int(floor(f1->rval(rindex)/bin1size[3]));
- bin2 = int(floor(f2->rval(rindex)/bin2size[3]));
- }
- if ((bin1 >= 0 && bin1 < nbins) &&
- (bin2 >= 0 && bin2 < nbins))
- local_histm[bin1*nbins + bin2]++;
- });
- }
- local_histm_threaded.mergeParallel();
- ptrdiff_t *histm = new ptrdiff_t[nbins*nbins];
- ptrdiff_t *histc = NULL;
- if (fc == THREE)
- {
- local_histc_threaded.mergeParallel();
- histc = new ptrdiff_t[nbins*nbins*9];
- }
- {
- MPI_Allreduce(
- (void*)local_histm_threaded.getMasterData(),
- (void*)histm,
- nbins*nbins,
- MPI_INT64_T, MPI_SUM, f1->comm);
- if (fc == THREE)
- MPI_Allreduce(
- (void*)local_histc_threaded.getMasterData(),
- (void*)histc,
- nbins*nbins*9,
- MPI_INT64_T, MPI_SUM, f1->comm);
- }
-
- if (f1->myrank == 0)
- {
- TIMEZONE("root-work");
- hid_t dset, wspace, mspace;
- hsize_t count[5], offset[5];
- if (fc == THREE)
- {
- dset = H5Dopen(group, dsetc.c_str(), H5P_DEFAULT);
- assert(dset > 0);
- wspace = H5Dget_space(dset);
- offset[0] = toffset;
- offset[1] = 0;
- offset[2] = 0;
- offset[3] = 0;
- offset[4] = 0;
- count[0] = 1;
- count[1] = nbins;
- count[2] = nbins;
- count[3] = 3;
- count[4] = 3;
- mspace = H5Screate_simple(5, count, NULL);
- H5Sselect_hyperslab(wspace, H5S_SELECT_SET, offset, NULL, count, NULL);
- H5Dwrite(dset, H5T_NATIVE_INT64, mspace, wspace, H5P_DEFAULT, histc);
- H5Sclose(wspace);
- H5Sclose(mspace);
- H5Dclose(dset);
- }
- dset = H5Dopen(group, dsetm.c_str(), H5P_DEFAULT);
- assert(dset > 0);
- offset[0] = toffset;
- offset[1] = 0;
- offset[2] = 0;
- count[0] = 1;
- count[1] = nbins;
- count[2] = nbins;
- mspace = H5Screate_simple(3, count, NULL);
- wspace = H5Dget_space(dset);
- H5Sselect_hyperslab(wspace, H5S_SELECT_SET, offset, NULL, count, NULL);
- H5Dwrite(dset, H5T_NATIVE_INT64, mspace, wspace, H5P_DEFAULT, histm);
- H5Sclose(wspace);
- H5Sclose(mspace);
- H5Dclose(dset);
- }
-
- delete[] histm;
- if (fc == THREE)
- delete[] histc;
-
- return EXIT_SUCCESS;
-}
-
-template class field<float, FFTW, ONE>;
-template class field<float, FFTW, THREE>;
-template class field<float, FFTW, THREExTHREE>;
-template class field<double, FFTW, ONE>;
-template class field<double, FFTW, THREE>;
-template class field<double, FFTW, THREExTHREE>;
-
-template void field<float, FFTW, ONE>::compute_stats<TWO_THIRDS>(
- kspace<FFTW, TWO_THIRDS> *,
- const hid_t, const std::string, const hsize_t, const double);
-template void field<float, FFTW, THREE>::compute_stats<TWO_THIRDS>(
- kspace<FFTW, TWO_THIRDS> *,
- const hid_t, const std::string, const hsize_t, const double);
-template void field<float, FFTW, THREExTHREE>::compute_stats<TWO_THIRDS>(
- kspace<FFTW, TWO_THIRDS> *,
- const hid_t, const std::string, const hsize_t, const double);
-
-template void field<double, FFTW, ONE>::compute_stats<TWO_THIRDS>(
- kspace<FFTW, TWO_THIRDS> *,
- const hid_t, const std::string, const hsize_t, const double);
-template void field<double, FFTW, THREE>::compute_stats<TWO_THIRDS>(
- kspace<FFTW, TWO_THIRDS> *,
- const hid_t, const std::string, const hsize_t, const double);
-template void field<double, FFTW, THREExTHREE>::compute_stats<TWO_THIRDS>(
- kspace<FFTW, TWO_THIRDS> *,
- const hid_t, const std::string, const hsize_t, const double);
-
-template void field<float, FFTW, ONE>::compute_stats<SMOOTH>(
- kspace<FFTW, SMOOTH> *,
- const hid_t, const std::string, const hsize_t, const double);
-template void field<float, FFTW, THREE>::compute_stats<SMOOTH>(
- kspace<FFTW, SMOOTH> *,
- const hid_t, const std::string, const hsize_t, const double);
-template void field<float, FFTW, THREExTHREE>::compute_stats<SMOOTH>(
- kspace<FFTW, SMOOTH> *,
- const hid_t, const std::string, const hsize_t, const double);
-
-template void field<double, FFTW, ONE>::compute_stats<SMOOTH>(
- kspace<FFTW, SMOOTH> *,
- const hid_t, const std::string, const hsize_t, const double);
-template void field<double, FFTW, THREE>::compute_stats<SMOOTH>(
- kspace<FFTW, SMOOTH> *,
- const hid_t, const std::string, const hsize_t, const double);
-template void field<double, FFTW, THREExTHREE>::compute_stats<SMOOTH>(
- kspace<FFTW, SMOOTH> *,
- const hid_t, const std::string, const hsize_t, const double);
-
-template int compute_gradient<float, FFTW, THREE, THREExTHREE, SMOOTH>(
- kspace<FFTW, SMOOTH> *,
- field<float, FFTW, THREE> *,
- field<float, FFTW, THREExTHREE> *);
-template int compute_gradient<double, FFTW, THREE, THREExTHREE, SMOOTH>(
- kspace<FFTW, SMOOTH> *,
- field<double, FFTW, THREE> *,
- field<double, FFTW, THREExTHREE> *);
-
-template int compute_gradient<float, FFTW, ONE, THREE, SMOOTH>(
- kspace<FFTW, SMOOTH> *,
- field<float, FFTW, ONE> *,
- field<float, FFTW, THREE> *);
-template int compute_gradient<double, FFTW, ONE, THREE, SMOOTH>(
- kspace<FFTW, SMOOTH> *,
- field<double, FFTW, ONE> *,
- field<double, FFTW, THREE> *);
-
-template int invert_curl<float, FFTW, SMOOTH>(
- kspace<FFTW, SMOOTH> *,
- field<float, FFTW, THREE> *,
- field<float, FFTW, THREE> *);
-template int invert_curl<double, FFTW, SMOOTH>(
- kspace<FFTW, SMOOTH> *,
- field<double, FFTW, THREE> *,
- field<double, FFTW, THREE> *);
-
-template int joint_rspace_PDF<float, FFTW, THREE>(
- field<float, FFTW, THREE> *,
- field<float, FFTW, THREE> *,
- const hid_t,
- const std::string,
- const hsize_t,
- const std::vector<double>,
- const std::vector<double>);
-template int joint_rspace_PDF<double, FFTW, THREE>(
- field<double, FFTW, THREE> *,
- field<double, FFTW, THREE> *,
- const hid_t,
- const std::string,
- const hsize_t,
- const std::vector<double>,
- const std::vector<double>);
-
-template int joint_rspace_PDF<float, FFTW, ONE>(
- field<float, FFTW, ONE> *,
- field<float, FFTW, ONE> *,
- const hid_t,
- const std::string,
- const hsize_t,
- const std::vector<double>,
- const std::vector<double>);
-template int joint_rspace_PDF<double, FFTW, ONE>(
- field<double, FFTW, ONE> *,
- field<double, FFTW, ONE> *,
- const hid_t,
- const std::string,
- const hsize_t,
- const std::vector<double>,
- const std::vector<double>);
-
diff --git a/bfps/cpp/field_descriptor.cpp b/bfps/cpp/field_descriptor.cpp
deleted file mode 100644
index 20c634262dbb45ad4c2bb5a1b5640b6df23d4d2c..0000000000000000000000000000000000000000
--- a/bfps/cpp/field_descriptor.cpp
+++ /dev/null
@@ -1,543 +0,0 @@
-/**********************************************************************
-* *
-* Copyright 2015 Max Planck Institute *
-* for Dynamics and Self-Organization *
-* *
-* This file is part of bfps. *
-* *
-* bfps is free software: you can redistribute it and/or modify *
-* it under the terms of the GNU General Public License as published *
-* by the Free Software Foundation, either version 3 of the License, *
-* or (at your option) any later version. *
-* *
-* bfps is distributed in the hope that it will be useful, *
-* but WITHOUT ANY WARRANTY; without even the implied warranty of *
-* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the *
-* GNU General Public License for more details. *
-* *
-* You should have received a copy of the GNU General Public License *
-* along with bfps. If not, see <http://www.gnu.org/licenses/> *
-* *
-* Contact: Cristian.Lalescu@ds.mpg.de *
-* *
-**********************************************************************/
-
-
-
-#define NDEBUG
-
-#include <stdlib.h>
-#include <algorithm>
-#include <iostream>
-#include "base.hpp"
-#include "field_descriptor.hpp"
-#include "fftw_interface.hpp"
-#include "scope_timer.hpp"
-
-/*****************************************************************************/
-/* macro for specializations to numeric types compatible with FFTW */
-
-
-template <class rnumber>
-field_descriptor<rnumber>::field_descriptor(
- int ndims,
- int *n,
- MPI_Datatype element_type,
- MPI_Comm COMM_TO_USE)
-{
- TIMEZONE("field_descriptor");
- DEBUG_MSG("entered field_descriptor::field_descriptor\n");
- this->comm = COMM_TO_USE;
- MPI_Comm_rank(this->comm, &this->myrank);
- MPI_Comm_size(this->comm, &this->nprocs);
- this->ndims = ndims;
- this->sizes = new int[ndims];
- this->subsizes = new int[ndims];
- this->starts = new int[ndims];
- int tsizes [ndims];
- int tsubsizes[ndims];
- int tstarts [ndims];
- std::vector<ptrdiff_t> nfftw;
- nfftw.resize(ndims);
- ptrdiff_t local_n0, local_0_start;
- for (int i = 0; i < this->ndims; i++)
- nfftw[i] = n[i];
- this->local_size = fftw_mpi_local_size_many(
- this->ndims,
- &nfftw.front(),
- 1,
- FFTW_MPI_DEFAULT_BLOCK,
- this->comm,
- &local_n0,
- &local_0_start);
- this->sizes[0] = n[0];
- this->subsizes[0] = (int)local_n0;
- this->starts[0] = (int)local_0_start;
- DEBUG_MSG_WAIT(
- this->comm,
- "first subsizes[0] = %d %d %d\n",
- this->subsizes[0],
- tsubsizes[0],
- (int)local_n0);
- tsizes[0] = n[0];
- tsubsizes[0] = (int)local_n0;
- tstarts[0] = (int)local_0_start;
- DEBUG_MSG_WAIT(
- this->comm,
- "second subsizes[0] = %d %d %d\n",
- this->subsizes[0],
- tsubsizes[0],
- (int)local_n0);
- this->mpi_dtype = element_type;
- this->slice_size = 1;
- this->full_size = this->sizes[0];
- for (int i = 1; i < this->ndims; i++)
- {
- this->sizes[i] = n[i];
- this->subsizes[i] = n[i];
- this->starts[i] = 0;
- this->slice_size *= this->subsizes[i];
- this->full_size *= this->sizes[i];
- tsizes[i] = this->sizes[i];
- tsubsizes[i] = this->subsizes[i];
- tstarts[i] = this->starts[i];
- }
- tsizes[ndims-1] *= sizeof(rnumber);
- tsubsizes[ndims-1] *= sizeof(rnumber);
- tstarts[ndims-1] *= sizeof(rnumber);
- if (this->mpi_dtype == mpi_real_type<rnumber>::complex())
- {
- tsizes[ndims-1] *= 2;
- tsubsizes[ndims-1] *= 2;
- tstarts[ndims-1] *= 2;
- }
- int local_zero_array[this->nprocs], zero_array[this->nprocs];
- for (int i=0; i<this->nprocs; i++)
- local_zero_array[i] = 0;
- local_zero_array[this->myrank] = (this->subsizes[0] == 0) ? 1 : 0;
- MPI_Allreduce(
- local_zero_array,
- zero_array,
- this->nprocs,
- MPI_INT,
- MPI_SUM,
- this->comm);
- int no_of_excluded_ranks = 0;
- for (int i = 0; i<this->nprocs; i++)
- no_of_excluded_ranks += zero_array[i];
- DEBUG_MSG_WAIT(
- this->comm,
- "subsizes[0] = %d %d\n",
- this->subsizes[0],
- tsubsizes[0]);
- if (no_of_excluded_ranks == 0)
- {
- this->io_comm = this->comm;
- this->io_nprocs = this->nprocs;
- this->io_myrank = this->myrank;
- }
- else
- {
- int excluded_rank[no_of_excluded_ranks];
- for (int i=0, j=0; i<this->nprocs; i++)
- if (zero_array[i])
- {
- excluded_rank[j] = i;
- j++;
- }
- MPI_Group tgroup0, tgroup;
- MPI_Comm_group(this->comm, &tgroup0);
- MPI_Group_excl(tgroup0, no_of_excluded_ranks, excluded_rank, &tgroup);
- MPI_Comm_create(this->comm, tgroup, &this->io_comm);
- MPI_Group_free(&tgroup0);
- MPI_Group_free(&tgroup);
- if (this->subsizes[0] > 0)
- {
- MPI_Comm_rank(this->io_comm, &this->io_myrank);
- MPI_Comm_size(this->io_comm, &this->io_nprocs);
- }
- else
- {
- this->io_myrank = MPI_PROC_NULL;
- this->io_nprocs = -1;
- }
- }
- DEBUG_MSG_WAIT(
- this->comm,
- "inside field_descriptor constructor, about to call "
- "MPI_Type_create_subarray "
- "%d %d %d\n",
- this->sizes[0],
- this->subsizes[0],
- this->starts[0]);
- for (int i=0; i<this->ndims; i++)
- DEBUG_MSG_WAIT(
- this->comm,
- "tsizes "
- "%d %d %d\n",
- tsizes[i],
- tsubsizes[i],
- tstarts[i]);
- if (this->subsizes[0] > 0)
- {
- DEBUG_MSG("creating subarray\n");
- MPI_Type_create_subarray(
- ndims,
- tsizes,
- tsubsizes,
- tstarts,
- MPI_ORDER_C,
- MPI_UNSIGNED_CHAR,
- &this->mpi_array_dtype);
- MPI_Type_commit(&this->mpi_array_dtype);
- }
- this->rank = new int[this->sizes[0]];
- int *local_rank = new int[this->sizes[0]];
- std::fill_n(local_rank, this->sizes[0], 0);
- for (int i = 0; i < this->sizes[0]; i++)
- if (i >= this->starts[0] && i < this->starts[0] + this->subsizes[0])
- local_rank[i] = this->myrank;
- MPI_Allreduce(
- local_rank,
- this->rank,
- this->sizes[0],
- MPI_INT,
- MPI_SUM,
- this->comm);
- delete[] local_rank;
- this->all_start0 = new int[this->nprocs];
- int *local_start0 = new int[this->nprocs];
- std::fill_n(local_start0, this->nprocs, 0);
- for (int i = 0; i < this->nprocs; i++)
- if (this->myrank == i)
- local_start0[i] = this->starts[0];
- MPI_Allreduce(
- local_start0,
- this->all_start0,
- this->nprocs,
- MPI_INT,
- MPI_SUM,
- this->comm);
- delete[] local_start0;
- this->all_size0 = new int[this->nprocs];
- int *local_size0 = new int[this->nprocs];
- std::fill_n(local_size0, this->nprocs, 0);
- for (int i = 0; i < this->nprocs; i++)
- if (this->myrank == i)
- local_size0[i] = this->subsizes[0];
- MPI_Allreduce(
- local_size0,
- this->all_size0,
- this->nprocs,
- MPI_INT,
- MPI_SUM,
- this->comm);
- delete[] local_size0;
- DEBUG_MSG("exiting field_descriptor constructor\n");
-}
-
-template <class rnumber>
-int field_descriptor<rnumber>::read(
- const char *fname,
- void *buffer)
-{
- TIMEZONE("field_descriptor::read");
- DEBUG_MSG("entered field_descriptor::read\n");
- char representation[] = "native";
- if (this->subsizes[0] > 0)
- {
- MPI_Info info;
- MPI_Info_create(&info);
- MPI_File f;
- ptrdiff_t read_size = this->local_size*sizeof(rnumber);
- DEBUG_MSG("read size is %ld\n", read_size);
- char ffname[200];
- if (this->mpi_dtype == mpi_real_type<rnumber>::complex())
- read_size *= 2;
- DEBUG_MSG("read size is %ld\n", read_size);
- sprintf(ffname, "%s", fname);
-
- MPI_File_open(
- this->io_comm,
- ffname,
- MPI_MODE_RDONLY,
- info,
- &f);
- DEBUG_MSG("opened file\n");
- MPI_File_set_view(
- f,
- 0,
- MPI_UNSIGNED_CHAR,
- this->mpi_array_dtype,
- representation,
- info);
- DEBUG_MSG("view is set\n");
- MPI_File_read_all(
- f,
- buffer,
- read_size,
- MPI_UNSIGNED_CHAR,
- MPI_STATUS_IGNORE);
- DEBUG_MSG("info is read\n");
- MPI_File_close(&f);
- }
- DEBUG_MSG("finished with field_descriptor::read\n");
- return EXIT_SUCCESS;
-}
-
-template <class rnumber>
-int field_descriptor<rnumber>::write(
- const char *fname,
- void *buffer)
-{
- TIMEZONE("field_descriptor::write");
- char representation[] = "native";
- if (this->subsizes[0] > 0)
- {
- MPI_Info info;
- MPI_Info_create(&info);
- MPI_File f;
- ptrdiff_t read_size = this->local_size*sizeof(rnumber);
- char ffname[200];
- if (this->mpi_dtype == mpi_real_type<rnumber>::complex())
- read_size *= 2;
- sprintf(ffname, "%s", fname);
-
- MPI_File_open(
- this->io_comm,
- ffname,
- MPI_MODE_CREATE | MPI_MODE_WRONLY,
- info,
- &f);
- MPI_File_set_view(
- f,
- 0,
- MPI_UNSIGNED_CHAR,
- this->mpi_array_dtype,
- representation,
- info);
- MPI_File_write_all(
- f,
- buffer,
- read_size,
- MPI_UNSIGNED_CHAR,
- MPI_STATUS_IGNORE);
- MPI_File_close(&f);
- }
-
- return EXIT_SUCCESS;
-}
-
-template <class rnumber>
-int field_descriptor<rnumber>::transpose(
- rnumber *input,
- rnumber *output)
-{
- TIMEZONE("field_descriptor::transpose");
- /* IMPORTANT NOTE:
- for 3D transposition, the input data is messed up */
- typename fftw_interface<rnumber>::plan tplan;
- if (this->ndims == 3)
- {
- /* transpose the two local dimensions 1 and 2 */
- rnumber *atmp;
- atmp = fftw_interface<rnumber>::alloc_real(this->slice_size);
- for (int k = 0; k < this->subsizes[0]; k++)
- {
- /* put transposed slice in atmp */
- for (int j = 0; j < this->sizes[1]; j++)
- for (int i = 0; i < this->sizes[2]; i++)
- atmp[i*this->sizes[1] + j] =
- input[(k*this->sizes[1] + j)*this->sizes[2] + i];
- /* copy back transposed slice */
- std::copy(
- atmp,
- atmp + this->slice_size,
- input + k*this->slice_size);
- }
- fftw_interface<rnumber>::free(atmp);
- }
- tplan = fftw_interface<rnumber>::mpi_plan_transpose(
- this->sizes[0], this->slice_size,
- input, output,
- this->comm,
- DEFAULT_FFTW_FLAG);
- fftw_interface<rnumber>::execute(tplan);
- fftw_interface<rnumber>::destroy_plan(tplan);
- return EXIT_SUCCESS;
-}
-
-template <class rnumber>
-int field_descriptor<rnumber>::transpose(
- typename fftw_interface<rnumber>::complex *input,
- typename fftw_interface<rnumber>::complex *output)
-{
- TIMEZONE("field_descriptor::transpose2");
- switch (this->ndims)
- {
- case 2:
- /* do a global transpose over the 2 dimensions */
- if (output == NULL)
- {
- std::cerr << "bad arguments for transpose.\n" << std::endl;
- return EXIT_FAILURE;
- }
- typename fftw_interface<rnumber>::plan tplan;
- tplan = fftw_interface<rnumber>::mpi_plan_many_transpose(
- this->sizes[0], this->sizes[1], 2,
- FFTW_MPI_DEFAULT_BLOCK,
- FFTW_MPI_DEFAULT_BLOCK,
- (rnumber*)input, (rnumber*)output,
- this->comm,
- DEFAULT_FFTW_FLAG);
- fftw_interface<rnumber>::execute(tplan);
- fftw_interface<rnumber>::destroy_plan(tplan);
- break;
- case 3:
- /* transpose the two local dimensions 1 and 2 */
- typename fftw_interface<rnumber>::complex *atmp;
- atmp = fftw_interface<rnumber>::alloc_complex(this->slice_size);
- for (int k = 0; k < this->subsizes[0]; k++)
- {
- /* put transposed slice in atmp */
- for (int j = 0; j < this->sizes[1]; j++)
- for (int i = 0; i < this->sizes[2]; i++)
- {
- atmp[i*this->sizes[1] + j][0] =
- input[(k*this->sizes[1] + j)*this->sizes[2] + i][0];
- atmp[i*this->sizes[1] + j][1] =
- input[(k*this->sizes[1] + j)*this->sizes[2] + i][1];
- }
- /* copy back transposed slice */
- std::copy(
- (rnumber*)(atmp),
- (rnumber*)(atmp + this->slice_size),
- (rnumber*)(input + k*this->slice_size));
- }
- fftw_interface<rnumber>::free(atmp);
- break;
- default:
- return EXIT_FAILURE;
- break;
- }
- return EXIT_SUCCESS;
-}
-
-template <class rnumber>
-int field_descriptor<rnumber>::interleave(
- rnumber *a,
- int dim)
-{
- TIMEZONE("field_descriptor::interleav");
- /* the following is copied from
- * http://agentzlerich.blogspot.com/2010/01/using-fftw-for-in-place-matrix.html
- * */
- typename fftw_interface<rnumber>::iodim howmany_dims[2];
- howmany_dims[0].n = dim;
- howmany_dims[0].is = this->local_size;
- howmany_dims[0].os = 1;
- howmany_dims[1].n = this->local_size;
- howmany_dims[1].is = 1;
- howmany_dims[1].os = dim;
- const int howmany_rank = sizeof(howmany_dims)/sizeof(howmany_dims[0]);
-
- typename fftw_interface<rnumber>::plan tmp = fftw_interface<rnumber>::plan_guru_r2r(
- /*rank*/0,
- /*dims*/nullptr,
- howmany_rank,
- howmany_dims,
- a,
- a,
- /*kind*/nullptr,
- DEFAULT_FFTW_FLAG);
- fftw_interface<rnumber>::execute(tmp);
- fftw_interface<rnumber>::destroy_plan(tmp);
- return EXIT_SUCCESS;
-}
-
-template <class rnumber>
-int field_descriptor<rnumber>::interleave(
- typename fftw_interface<rnumber>::complex *a,
- int dim)
-{
- TIMEZONE("field_descriptor::interleave2");
- typename fftw_interface<rnumber>::iodim howmany_dims[2];
- howmany_dims[0].n = dim;
- howmany_dims[0].is = this->local_size;
- howmany_dims[0].os = 1;
- howmany_dims[1].n = this->local_size;
- howmany_dims[1].is = 1;
- howmany_dims[1].os = dim;
- const int howmany_rank = sizeof(howmany_dims)/sizeof(howmany_dims[0]);
-
- typename fftw_interface<rnumber>::plan tmp = fftw_interface<rnumber>::plan_guru_dft(
- /*rank*/0,
- /*dims*/nullptr,
- howmany_rank,
- howmany_dims,
- a,
- a,
- +1,
- DEFAULT_FFTW_FLAG);
- fftw_interface<rnumber>::execute(tmp);
- fftw_interface<rnumber>::destroy_plan(tmp);
- return EXIT_SUCCESS;
-}
-
-template <class rnumber>
-field_descriptor<rnumber>* field_descriptor<rnumber>::get_transpose()
-{
- TIMEZONE("field_descriptor::get_transpose");
- int n[this->ndims];
- for (int i=0; i<this->ndims; i++)
- n[i] = this->sizes[this->ndims - i - 1];
- return new field_descriptor<rnumber>(this->ndims, n, this->mpi_dtype, this->comm);
-}
-
-/*****************************************************************************/
-/*****************************************************************************/
-
-
-
-/*****************************************************************************/
-/* destructor looks the same for both float and double */
-template <class rnumber>
-field_descriptor<rnumber>::~field_descriptor()
-{
- DEBUG_MSG_WAIT(
- MPI_COMM_WORLD,
- this->io_comm == MPI_COMM_NULL ? "null\n" : "not null\n");
- DEBUG_MSG_WAIT(
- MPI_COMM_WORLD,
- "subsizes[0] = %d \n", this->subsizes[0]);
- if (this->subsizes[0] > 0)
- {
- DEBUG_MSG_WAIT(
- this->io_comm,
- "deallocating mpi_array_dtype\n");
- MPI_Type_free(&this->mpi_array_dtype);
- }
- if (this->nprocs != this->io_nprocs && this->io_myrank != MPI_PROC_NULL)
- {
- DEBUG_MSG_WAIT(
- this->io_comm,
- "freeing io_comm\n");
- MPI_Comm_free(&this->io_comm);
- }
- delete[] this->sizes;
- delete[] this->subsizes;
- delete[] this->starts;
- delete[] this->rank;
- delete[] this->all_start0;
- delete[] this->all_size0;
-}
-/*****************************************************************************/
-
-
-
-/*****************************************************************************/
-/* finally, force generation of code */
-template class field_descriptor<float>;
-template class field_descriptor<double>;
-/*****************************************************************************/
-
diff --git a/bfps/cpp/field_descriptor.hpp b/bfps/cpp/field_descriptor.hpp
deleted file mode 100644
index 2fb491bca7c130704fc5de5d22c3393cb196eec7..0000000000000000000000000000000000000000
--- a/bfps/cpp/field_descriptor.hpp
+++ /dev/null
@@ -1,114 +0,0 @@
-/**********************************************************************
-* *
-* Copyright 2015 Max Planck Institute *
-* for Dynamics and Self-Organization *
-* *
-* This file is part of bfps. *
-* *
-* bfps is free software: you can redistribute it and/or modify *
-* it under the terms of the GNU General Public License as published *
-* by the Free Software Foundation, either version 3 of the License, *
-* or (at your option) any later version. *
-* *
-* bfps is distributed in the hope that it will be useful, *
-* but WITHOUT ANY WARRANTY; without even the implied warranty of *
-* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the *
-* GNU General Public License for more details. *
-* *
-* You should have received a copy of the GNU General Public License *
-* along with bfps. If not, see <http://www.gnu.org/licenses/> *
-* *
-* Contact: Cristian.Lalescu@ds.mpg.de *
-* *
-**********************************************************************/
-
-
-
-#include <mpi.h>
-#include <fftw3-mpi.h>
-#include "fftw_interface.hpp"
-
-#ifndef FIELD_DESCRIPTOR
-
-#define FIELD_DESCRIPTOR
-
-extern int myrank, nprocs;
-
-template <class rnumber>
-class field_descriptor
-{
- private:
- typedef rnumber cnumber[2];
- public:
-
- /* data */
- int *sizes;
- int *subsizes;
- int *starts;
- int ndims;
- int *rank;
- int *all_start0;
- int *all_size0;
- ptrdiff_t slice_size, local_size, full_size;
- MPI_Datatype mpi_array_dtype, mpi_dtype;
- int myrank, nprocs, io_myrank, io_nprocs;
- MPI_Comm comm, io_comm;
-
-
- /* methods */
- field_descriptor(
- int ndims,
- int *n,
- MPI_Datatype element_type,
- MPI_Comm COMM_TO_USE);
- ~field_descriptor();
-
- /* io is performed using MPI_File stuff, and our
- * own mpi_array_dtype that was defined in the constructor.
- * */
- int read(
- const char *fname,
- void *buffer);
- int write(
- const char *fname,
- void *buffer);
-
- /* a function that generates the transposed descriptor.
- * don't forget to delete the result once you're done with it.
- * the transposed descriptor is useful for io operations.
- * */
- field_descriptor<rnumber> *get_transpose();
-
- /* we don't actually need the transposed descriptor to perform
- * the transpose operation: we only need the in/out fields.
- * */
- int transpose(
- rnumber *input,
- rnumber *output);
- int transpose(
- typename fftw_interface<rnumber>::complex *input,
- typename fftw_interface<rnumber>::complex *output = NULL);
-
- int interleave(
- rnumber *input,
- int dim);
- int interleave(
- typename fftw_interface<rnumber>::complex *input,
- int dim);
-};
-
-
-inline float btle(const float be)
- {
- float le;
- char *befloat = (char *) & be;
- char *lefloat = (char *) & le;
- lefloat[0] = befloat[3];
- lefloat[1] = befloat[2];
- lefloat[2] = befloat[1];
- lefloat[3] = befloat[0];
- return le;
- }
-
-#endif//FIELD_DESCRIPTOR
-
diff --git a/bfps/cpp/fluid_solver.cpp b/bfps/cpp/fluid_solver.cpp
deleted file mode 100644
index 319186103797f8135d4d3e2244ed5e3a8f271b00..0000000000000000000000000000000000000000
--- a/bfps/cpp/fluid_solver.cpp
+++ /dev/null
@@ -1,1057 +0,0 @@
-/**********************************************************************
-* *
-* Copyright 2015 Max Planck Institute *
-* for Dynamics and Self-Organization *
-* *
-* This file is part of bfps. *
-* *
-* bfps is free software: you can redistribute it and/or modify *
-* it under the terms of the GNU General Public License as published *
-* by the Free Software Foundation, either version 3 of the License, *
-* or (at your option) any later version. *
-* *
-* bfps is distributed in the hope that it will be useful, *
-* but WITHOUT ANY WARRANTY; without even the implied warranty of *
-* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the *
-* GNU General Public License for more details. *
-* *
-* You should have received a copy of the GNU General Public License *
-* along with bfps. If not, see <http://www.gnu.org/licenses/> *
-* *
-* Contact: Cristian.Lalescu@ds.mpg.de *
-* *
-**********************************************************************/
-
-
-
-//#define NDEBUG
-
-#include <cassert>
-#include <cmath>
-#include <cstring>
-#include "fluid_solver.hpp"
-#include "fftw_tools.hpp"
-#include "scope_timer.hpp"
-#include "shared_array.hpp"
-
-
-template <class rnumber>
-void fluid_solver<rnumber>::impose_zero_modes()
-{
- if (this->cd->myrank == this->cd->rank[0])
- {
- std::fill_n((rnumber*)(this->cu), 6, 0.0);
- std::fill_n((rnumber*)(this->cv[0]), 6, 0.0);
- std::fill_n((rnumber*)(this->cv[1]), 6, 0.0);
- std::fill_n((rnumber*)(this->cv[2]), 6, 0.0);
- }
-}
-/*****************************************************************************/
-/* macro for specializations to numeric types compatible with FFTW */
-
-template <class rnumber>
-fluid_solver<rnumber>::fluid_solver(
- const char *NAME,
- int nx,
- int ny,
- int nz,
- double DKX,
- double DKY,
- double DKZ,
- int DEALIAS_TYPE,
- unsigned FFTW_PLAN_RIGOR) : fluid_solver_base<rnumber>(
- NAME,
- nx , ny , nz,
- DKX, DKY, DKZ,
- DEALIAS_TYPE,
- FFTW_PLAN_RIGOR)
-{
- TIMEZONE("fluid_solver::fluid_solver");
- this->cvorticity = fftw_interface<rnumber>::alloc_complex(this->cd->local_size);
- this->cvelocity = fftw_interface<rnumber>::alloc_complex(this->cd->local_size);
- this->rvorticity = fftw_interface<rnumber>::alloc_real(this->cd->local_size*2);
- /*this->rvelocity = (rnumber*)(this->cvelocity);*/
- this->rvelocity = fftw_interface<rnumber>::alloc_real(this->cd->local_size*2);
-
- this->ru = this->rvelocity;
- this->cu = this->cvelocity;
-
- this->rv[0] = this->rvorticity;
- this->rv[3] = this->rvorticity;
- this->cv[0] = this->cvorticity;
- this->cv[3] = this->cvorticity;
-
- this->cv[1] = fftw_interface<rnumber>::alloc_complex(this->cd->local_size);
- this->cv[2] = this->cv[1];
- this->rv[1] = fftw_interface<rnumber>::alloc_real(this->cd->local_size*2);
- this->rv[2] = this->rv[1];
-
- this->c2r_vorticity = new typename fftw_interface<rnumber>::plan;
- this->r2c_vorticity = new typename fftw_interface<rnumber>::plan;
- this->c2r_velocity = new typename fftw_interface<rnumber>::plan;
- this->r2c_velocity = new typename fftw_interface<rnumber>::plan;
-
- ptrdiff_t sizes[] = {nz,
- ny,
- nx};
-
- *this->c2r_vorticity = fftw_interface<rnumber>::mpi_plan_many_dft_c2r(
- 3, sizes, 3, FFTW_MPI_DEFAULT_BLOCK, FFTW_MPI_DEFAULT_BLOCK,
- this->cvorticity, this->rvorticity,
- MPI_COMM_WORLD, this->fftw_plan_rigor | FFTW_MPI_TRANSPOSED_IN);
-
- *this->r2c_vorticity = fftw_interface<rnumber>::mpi_plan_many_dft_r2c(
- 3, sizes, 3, FFTW_MPI_DEFAULT_BLOCK, FFTW_MPI_DEFAULT_BLOCK,
- this->rvorticity, this->cvorticity,
- MPI_COMM_WORLD, this->fftw_plan_rigor | FFTW_MPI_TRANSPOSED_OUT);
-
- *this->c2r_velocity = fftw_interface<rnumber>::mpi_plan_many_dft_c2r(
- 3, sizes, 3, FFTW_MPI_DEFAULT_BLOCK, FFTW_MPI_DEFAULT_BLOCK,
- this->cvelocity, this->rvelocity,
- MPI_COMM_WORLD, this->fftw_plan_rigor | FFTW_MPI_TRANSPOSED_IN);
-
- *this->r2c_velocity = fftw_interface<rnumber>::mpi_plan_many_dft_r2c(
- 3, sizes, 3, FFTW_MPI_DEFAULT_BLOCK, FFTW_MPI_DEFAULT_BLOCK,
- this->rvelocity, this->cvelocity,
- MPI_COMM_WORLD, this->fftw_plan_rigor | FFTW_MPI_TRANSPOSED_OUT);
-
- this->uc2r = this->c2r_velocity;
- this->ur2c = this->r2c_velocity;
- this->vc2r[0] = this->c2r_vorticity;
- this->vr2c[0] = this->r2c_vorticity;
-
- this->vc2r[1] = new typename fftw_interface<rnumber>::plan;
- this->vr2c[1] = new typename fftw_interface<rnumber>::plan;
- this->vc2r[2] = new typename fftw_interface<rnumber>::plan;
- this->vr2c[2] = new typename fftw_interface<rnumber>::plan;
-
- *(this->vc2r[1]) = fftw_interface<rnumber>::mpi_plan_many_dft_c2r(
- 3, sizes, 3, FFTW_MPI_DEFAULT_BLOCK, FFTW_MPI_DEFAULT_BLOCK,
- this->cv[1], this->rv[1],
- MPI_COMM_WORLD, this->fftw_plan_rigor | FFTW_MPI_TRANSPOSED_IN);
-
- *this->vc2r[2] = fftw_interface<rnumber>::mpi_plan_many_dft_c2r(
- 3, sizes, 3, FFTW_MPI_DEFAULT_BLOCK, FFTW_MPI_DEFAULT_BLOCK,
- this->cv[2], this->rv[2],
- MPI_COMM_WORLD, this->fftw_plan_rigor | FFTW_MPI_TRANSPOSED_IN);
-
- *this->vr2c[1] = fftw_interface<rnumber>::mpi_plan_many_dft_r2c(
- 3, sizes, 3, FFTW_MPI_DEFAULT_BLOCK, FFTW_MPI_DEFAULT_BLOCK,
- this->rv[1], this->cv[1],
- MPI_COMM_WORLD, this->fftw_plan_rigor | FFTW_MPI_TRANSPOSED_OUT);
-
- *this->vr2c[2] = fftw_interface<rnumber>::mpi_plan_many_dft_r2c(
- 3, sizes, 3, FFTW_MPI_DEFAULT_BLOCK, FFTW_MPI_DEFAULT_BLOCK,
- this->rv[2], this->cv[2],
- MPI_COMM_WORLD, this->fftw_plan_rigor | FFTW_MPI_TRANSPOSED_OUT);
-
- /* ``physical'' parameters etc, initialized here just in case */
-
- this->nu = 0.1;
- this->fmode = 1;
- this->famplitude = 1.0;
- this->fk0 = 0;
- this->fk1 = 3.0;
- /* initialization of fields must be done AFTER planning */
- std::fill_n((rnumber*)this->cvorticity, this->cd->local_size*2, 0.0);
- std::fill_n((rnumber*)this->cvelocity, this->cd->local_size*2, 0.0);
- std::fill_n(this->rvelocity, this->cd->local_size*2, 0.0);
- std::fill_n(this->rvorticity, this->cd->local_size*2, 0.0);
- std::fill_n((rnumber*)this->cv[1], this->cd->local_size*2, 0.0);
- std::fill_n(this->rv[1], this->cd->local_size*2, 0.0);
- std::fill_n(this->rv[2], this->cd->local_size*2, 0.0);
-}
-
-template <class rnumber>
-fluid_solver<rnumber>::~fluid_solver()
-{
- fftw_interface<rnumber>::destroy_plan(*this->c2r_vorticity);
- fftw_interface<rnumber>::destroy_plan(*this->r2c_vorticity);
- fftw_interface<rnumber>::destroy_plan(*this->c2r_velocity );
- fftw_interface<rnumber>::destroy_plan(*this->r2c_velocity );
- fftw_interface<rnumber>::destroy_plan(*this->vc2r[1]);
- fftw_interface<rnumber>::destroy_plan(*this->vr2c[1]);
- fftw_interface<rnumber>::destroy_plan(*this->vc2r[2]);
- fftw_interface<rnumber>::destroy_plan(*this->vr2c[2]);
-
- delete this->c2r_vorticity;
- delete this->r2c_vorticity;
- delete this->c2r_velocity ;
- delete this->r2c_velocity ;
- delete this->vc2r[1];
- delete this->vr2c[1];
- delete this->vc2r[2];
- delete this->vr2c[2];
-
- fftw_interface<rnumber>::free(this->cv[1]);
- fftw_interface<rnumber>::free(this->rv[1]);
- fftw_interface<rnumber>::free(this->cvorticity);
- fftw_interface<rnumber>::free(this->rvorticity);
- fftw_interface<rnumber>::free(this->cvelocity);
- fftw_interface<rnumber>::free(this->rvelocity);
-}
-
-template <class rnumber>
-void fluid_solver<rnumber>::compute_vorticity()
-{
- TIMEZONE("fluid_solver::compute_vorticity");
- CLOOP_K2(
- this,
- [&](ptrdiff_t cindex, ptrdiff_t xindex, ptrdiff_t yindex, ptrdiff_t zindex, double k2){
- // cindex indexing is thread safe (and tindex too) + it is a write
- ptrdiff_t tindex = 3*cindex;
- if (k2 <= this->kM2)
- {
- this->cvorticity[tindex+0][0] = -(this->ky[yindex]*this->cu[tindex+2][1] - this->kz[zindex]*this->cu[tindex+1][1]);
- this->cvorticity[tindex+1][0] = -(this->kz[zindex]*this->cu[tindex+0][1] - this->kx[xindex]*this->cu[tindex+2][1]);
- this->cvorticity[tindex+2][0] = -(this->kx[xindex]*this->cu[tindex+1][1] - this->ky[yindex]*this->cu[tindex+0][1]);
- this->cvorticity[tindex+0][1] = (this->ky[yindex]*this->cu[tindex+2][0] - this->kz[zindex]*this->cu[tindex+1][0]);
- this->cvorticity[tindex+1][1] = (this->kz[zindex]*this->cu[tindex+0][0] - this->kx[xindex]*this->cu[tindex+2][0]);
- this->cvorticity[tindex+2][1] = (this->kx[xindex]*this->cu[tindex+1][0] - this->ky[yindex]*this->cu[tindex+0][0]);
- }
- else{
- std::fill_n((rnumber*)(this->cvorticity+tindex), 6, 0.0);
- }
- }
- );
- this->symmetrize(this->cvorticity, 3);
-}
-
-template <class rnumber>
-void fluid_solver<rnumber>::compute_velocity(rnumber (*__restrict__ vorticity)[2])
-{
- TIMEZONE("fluid_solver::compute_velocity");
- CLOOP_K2(
- this,
- [&](ptrdiff_t cindex, ptrdiff_t xindex, ptrdiff_t yindex, ptrdiff_t zindex, double k2){
- // cindex indexing is thread safe (and tindex too) + it is a write
- ptrdiff_t tindex = 3*cindex;
- if (k2 <= this->kM2 && k2 > 0)
- {
- this->cu[tindex+0][0] = -(this->ky[yindex]*vorticity[tindex+2][1] - this->kz[zindex]*vorticity[tindex+1][1]) / k2;
- this->cu[tindex+1][0] = -(this->kz[zindex]*vorticity[tindex+0][1] - this->kx[xindex]*vorticity[tindex+2][1]) / k2;
- this->cu[tindex+2][0] = -(this->kx[xindex]*vorticity[tindex+1][1] - this->ky[yindex]*vorticity[tindex+0][1]) / k2;
- this->cu[tindex+0][1] = (this->ky[yindex]*vorticity[tindex+2][0] - this->kz[zindex]*vorticity[tindex+1][0]) / k2;
- this->cu[tindex+1][1] = (this->kz[zindex]*vorticity[tindex+0][0] - this->kx[xindex]*vorticity[tindex+2][0]) / k2;
- this->cu[tindex+2][1] = (this->kx[xindex]*vorticity[tindex+1][0] - this->ky[yindex]*vorticity[tindex+0][0]) / k2;
- }
- else
- std::fill_n((rnumber*)(this->cu+tindex), 6, 0.0);
- }
- );
- /*this->symmetrize(this->cu, 3);*/
-}
-
-template <class rnumber>
-void fluid_solver<rnumber>::ift_velocity()
-{
- TIMEZONE("fluid_solver::ift_velocity");
- fftw_interface<rnumber>::execute(*(this->c2r_velocity ));
-}
-
-template <class rnumber>
-void fluid_solver<rnumber>::ift_vorticity()
-{
- TIMEZONE("fluid_solver::ift_vorticity");
- std::fill_n(this->rvorticity, this->cd->local_size*2, 0.0);
- fftw_interface<rnumber>::execute(*(this->c2r_vorticity ));
-}
-
-template <class rnumber>
-void fluid_solver<rnumber>::dft_velocity()
-{
- TIMEZONE("fluid_solver::dft_velocity");
- fftw_interface<rnumber>::execute(*(this->r2c_velocity ));
-}
-
-template <class rnumber>
-void fluid_solver<rnumber>::dft_vorticity()
-{
- TIMEZONE("fluid_solver::dft_vorticity");
- std::fill_n((rnumber*)this->cvorticity, this->cd->local_size*2, 0.0);
- fftw_interface<rnumber>::execute(*(this->r2c_vorticity ));
-}
-
-template <class rnumber>
-void fluid_solver<rnumber>::add_forcing(
- rnumber (*__restrict__ acc_field)[2], rnumber (*__restrict__ vort_field)[2], rnumber factor)
-{
- TIMEZONE("fluid_solver::add_forcing");
- if (strcmp(this->forcing_type, "none") == 0)
- return;
- if (strcmp(this->forcing_type, "Kolmogorov") == 0)
- {
- ptrdiff_t cindex;
- if (this->cd->myrank == this->cd->rank[this->fmode])
- {
- cindex = ((this->fmode - this->cd->starts[0]) * this->cd->sizes[1])*this->cd->sizes[2]*3;
- acc_field[cindex+2][0] -= this->famplitude*factor/2;
- }
- if (this->cd->myrank == this->cd->rank[this->cd->sizes[0] - this->fmode])
- {
- cindex = ((this->cd->sizes[0] - this->fmode - this->cd->starts[0]) * this->cd->sizes[1])*this->cd->sizes[2]*3;
- acc_field[cindex+2][0] -= this->famplitude*factor/2;
- }
- return;
- }
- if (strcmp(this->forcing_type, "linear") == 0)
- {
- CLOOP(
- this,
- [&](ptrdiff_t cindex, ptrdiff_t xindex, ptrdiff_t yindex, ptrdiff_t zindex){
- // cindex indexing is thread safe (and cindex*3+c too)
- double knorm = sqrt(this->kx[xindex]*this->kx[xindex] +
- this->ky[yindex]*this->ky[yindex] +
- this->kz[zindex]*this->kz[zindex]);
- if ((this->fk0 <= knorm) && (this->fk1 >= knorm))
- for (int c=0; c<3; c++)
- for (int i=0; i<2; i++)
- acc_field[cindex*3+c][i] += this->famplitude*vort_field[cindex*3+c][i]*factor;
- }
- );
- return;
- }
-}
-
-template <class rnumber>
-void fluid_solver<rnumber>::omega_nonlin(
- int src)
-{
- TIMEZONE("fluid_solver::omega_nonlin");
- assert(src >= 0 && src < 3);
- this->compute_velocity(this->cv[src]);
- /* get fields from Fourier space to real space */
- {
- TIMEZONE("fluid_solver::omega_nonlin::fftw");
- fftw_interface<rnumber>::execute(*(this->c2r_velocity ));
- fftw_interface<rnumber>::execute(*(this->vc2r[src]));
- }
- /* compute cross product $u \times \omega$, and normalize */
- {
- TIMEZONE("fluid_solver::omega_nonlin::RLOOP");
- RLOOP (
- this,
- [&](ptrdiff_t rindex, ptrdiff_t /*xindex*/, ptrdiff_t /*yindex*/, ptrdiff_t /*zindex*/){
- ptrdiff_t tindex = 3*rindex;
- rnumber tmp[3][2];
- for (int cc=0; cc<3; cc++)
- tmp[cc][0] = (this->ru[tindex+(cc+1)%3]*this->rv[src][tindex+(cc+2)%3] -
- this->ru[tindex+(cc+2)%3]*this->rv[src][tindex+(cc+1)%3]);
- // Access to rindex is thread safe so there is no overlap between threads
- for (int cc=0; cc<3; cc++)
- this->ru[(3*rindex)+cc] = tmp[cc][0] / this->normalization_factor;
- }
- );
- }
- /* go back to Fourier space */
- this->clean_up_real_space(this->ru, 3);
- {
- TIMEZONE("fluid_solver::omega_nonlin::fftw-2");
- fftw_interface<rnumber>::execute(*(this->r2c_velocity ));
- }
- this->dealias(this->cu, 3);
- /* $\imath k \times Fourier(u \times \omega)$ */
- {
- TIMEZONE("fluid_solver::omega_nonlin::CLOOP");
- CLOOP(
- this,
- [&](ptrdiff_t cindex, ptrdiff_t xindex, ptrdiff_t yindex, ptrdiff_t zindex){
- rnumber tmp[3][2];
- ptrdiff_t tindex = 3*cindex;
- {
- tmp[0][0] = -(this->ky[yindex]*this->cu[tindex+2][1] - this->kz[zindex]*this->cu[tindex+1][1]);
- tmp[1][0] = -(this->kz[zindex]*this->cu[tindex+0][1] - this->kx[xindex]*this->cu[tindex+2][1]);
- tmp[2][0] = -(this->kx[xindex]*this->cu[tindex+1][1] - this->ky[yindex]*this->cu[tindex+0][1]);
- tmp[0][1] = (this->ky[yindex]*this->cu[tindex+2][0] - this->kz[zindex]*this->cu[tindex+1][0]);
- tmp[1][1] = (this->kz[zindex]*this->cu[tindex+0][0] - this->kx[xindex]*this->cu[tindex+2][0]);
- tmp[2][1] = (this->kx[xindex]*this->cu[tindex+1][0] - this->ky[yindex]*this->cu[tindex+0][0]);
- }
- // cindex indexing is thread safe so it is 3*cindex so there is no overlap between threads
- for (int cc=0; cc<3; cc++)
- for (int i=0; i<2; i++)
- this->cu[tindex+cc][i] = tmp[cc][i];
- }
- );
- }
- {
- TIMEZONE("fluid_solver::omega_nonlin::add_forcing");
- this->add_forcing(this->cu, this->cv[src], 1.0);
- }
- {
- TIMEZONE("fluid_solver::omega_nonlin::force_divfree");
- this->force_divfree(this->cu);
- }
-}
-
-template <class rnumber>
-void fluid_solver<rnumber>::step(double dt)
-{
- TIMEZONE("fluid_solver::step");
- std::fill_n((rnumber*)this->cv[1], this->cd->local_size*2, 0.0);
- this->omega_nonlin(0);
- CLOOP_K2(
- this,
- [&](ptrdiff_t cindex, ptrdiff_t /*xindex*/, ptrdiff_t /*yindex*/, ptrdiff_t /*zindex*/, double k2){
- if (k2 <= this->kM2)
- {
- double factor0 = exp(-this->nu * k2 * dt);
- // cindex indexing is thread safe so there is no overlap between threads
- for (int cc=0; cc<3; cc++) for (int i=0; i<2; i++)
- this->cv[1][3*cindex+cc][i] = (this->cv[0][3*cindex+cc][i] +
- dt*this->cu[3*cindex+cc][i])*factor0;
- }
- }
- );
-
- this->omega_nonlin(1);
- CLOOP_K2(
- this,
- [&](ptrdiff_t cindex, ptrdiff_t /*xindex*/, ptrdiff_t /*yindex*/, ptrdiff_t /*zindex*/, double k2){
- if (k2 <= this->kM2)
- {
- double factor0 = exp(-this->nu * k2 * dt/2);
- double factor1 = exp( this->nu * k2 * dt/2);
- // cindex indexing is thread safe so there is no overlap between threads
- for (int cc=0; cc<3; cc++) for (int i=0; i<2; i++)
- this->cv[2][3*cindex+cc][i] = (3*this->cv[0][3*cindex+cc][i]*factor0 +
- (this->cv[1][3*cindex+cc][i] +
- dt*this->cu[3*cindex+cc][i])*factor1)*0.25;
- }
- }
- );
-
- this->omega_nonlin(2);
- CLOOP_K2(
- this,
- [&](ptrdiff_t cindex, ptrdiff_t /*xindex*/, ptrdiff_t /*yindex*/, ptrdiff_t /*zindex*/, double k2){
- if (k2 <= this->kM2)
- {
- double factor0 = exp(-this->nu * k2 * dt * 0.5);
- // cindex indexing is thread safe so there is no overlap between threads
- for (int cc=0; cc<3; cc++) for (int i=0; i<2; i++)
- this->cv[3][3*cindex+cc][i] = (this->cv[0][3*cindex+cc][i]*factor0 +
- 2*(this->cv[2][3*cindex+cc][i] +
- dt*this->cu[3*cindex+cc][i]))*factor0/3;
- }
- }
- );
-
- this->force_divfree(this->cvorticity);
- this->symmetrize(this->cvorticity, 3);
- this->iteration++;
-}
-
-template <class rnumber>
-int fluid_solver<rnumber>::read(char field, char representation)
-{
- TIMEZONE("fluid_solver::read");
- char fname[512];
- int read_result;
- if (field == 'v')
- {
- if (representation == 'c')
- {
- this->fill_up_filename("cvorticity", fname);
- read_result = this->cd->read(fname, (void*)this->cvorticity);
- if (read_result != EXIT_SUCCESS)
- return read_result;
- }
- if (representation == 'r')
- {
- read_result = this->read_base("rvorticity", this->rvorticity);
- if (read_result != EXIT_SUCCESS)
- return read_result;
- else
- fftw_interface<rnumber>::execute(*(this->r2c_vorticity ));
- }
- this->low_pass_Fourier(this->cvorticity, 3, this->kM);
- this->force_divfree(this->cvorticity);
- this->symmetrize(this->cvorticity, 3);
- return EXIT_SUCCESS;
- }
- if ((field == 'u') && (representation == 'c'))
- {
- read_result = this->read_base("cvelocity", this->cvelocity);
- this->low_pass_Fourier(this->cvelocity, 3, this->kM);
- this->force_divfree(this->cvorticity);
- this->symmetrize(this->cvorticity, 3);
- return read_result;
- }
- if ((field == 'u') && (representation == 'r'))
- return this->read_base("rvelocity", this->rvelocity);
- return EXIT_FAILURE;
-}
-
-template <class rnumber>
-int fluid_solver<rnumber>::write(char field, char representation)
-{
- TIMEZONE("fluid_solver::write");
- char fname[512];
- if ((field == 'v') && (representation == 'c'))
- {
- this->fill_up_filename("cvorticity", fname);
- return this->cd->write(fname, (void*)this->cvorticity);
- }
- if ((field == 'v') && (representation == 'r'))
- {
- fftw_interface<rnumber>::execute(*(this->c2r_vorticity ));
- clip_zero_padding<rnumber>(this->rd, this->rvorticity, 3);
- this->fill_up_filename("rvorticity", fname);
- return this->rd->write(fname, this->rvorticity);
- }
- this->compute_velocity(this->cvorticity);
- if ((field == 'u') && (representation == 'c'))
- {
- this->fill_up_filename("cvelocity", fname);
- return this->cd->write(fname, this->cvelocity);
- }
- if ((field == 'u') && (representation == 'r'))
- {
- this->ift_velocity();
- clip_zero_padding<rnumber>(this->rd, this->rvelocity, 3);
- this->fill_up_filename("rvelocity", fname);
- return this->rd->write(fname, this->rvelocity);
- }
- return EXIT_FAILURE;
-}
-
-template <class rnumber>
-int fluid_solver<rnumber>::write_rTrS2()
-{
- TIMEZONE("fluid_solver::write_rTrS2");
- char fname[512];
- this->fill_up_filename("rTrS2", fname);
- typename fftw_interface<rnumber>::complex *ca;
- rnumber *ra;
- ca = fftw_interface<rnumber>::alloc_complex(this->cd->local_size*3);
- ra = (rnumber*)(ca);
- this->compute_velocity(this->cvorticity);
- this->compute_vector_gradient(ca, this->cvelocity);
- for (int cc=0; cc<3; cc++)
- {
- std::copy(
- (rnumber*)(ca + cc*this->cd->local_size),
- (rnumber*)(ca + (cc+1)*this->cd->local_size),
- (rnumber*)this->cv[1]);
- fftw_interface<rnumber>::execute(*(this->vc2r[1]));
- std::copy(
- this->rv[1],
- this->rv[1] + this->cd->local_size*2,
- ra + cc*this->cd->local_size*2);
- }
- /* velocity gradient is now stored, in real space, in ra */
- rnumber *dx_u, *dy_u, *dz_u;
- dx_u = ra;
- dy_u = ra + 2*this->cd->local_size;
- dz_u = ra + 4*this->cd->local_size;
- rnumber *trS2 = fftw_interface<rnumber>::alloc_real((this->cd->local_size/3)*2);
- shared_array<double> average_local(1, [&](double* data){
- data[0] = 0;
- });
-
- RLOOP(
- this,
- [&](ptrdiff_t rindex, ptrdiff_t /*xindex*/, ptrdiff_t /*yindex*/, ptrdiff_t /*zindex*/){
- rnumber AxxAxx;
- rnumber AyyAyy;
- rnumber AzzAzz;
- rnumber Sxy;
- rnumber Syz;
- rnumber Szx;
- ptrdiff_t tindex = 3*rindex;
- AxxAxx = dx_u[tindex+0]*dx_u[tindex+0];
- AyyAyy = dy_u[tindex+1]*dy_u[tindex+1];
- AzzAzz = dz_u[tindex+2]*dz_u[tindex+2];
- Sxy = dx_u[tindex+1]+dy_u[tindex+0];
- Syz = dy_u[tindex+2]+dz_u[tindex+1];
- Szx = dz_u[tindex+0]+dx_u[tindex+2];
- // rindex is thread safe + No overlap between thread it is a write
- trS2[rindex] = (AxxAxx + AyyAyy + AzzAzz +
- (Sxy*Sxy + Syz*Syz + Szx*Szx)/2);
- average_local.getMine()[0] += trS2[rindex];
- }
- );
- average_local.mergeParallel();
- double average;
- MPI_Allreduce(
- average_local.getMasterData(),
- &average,
- 1,
- MPI_DOUBLE, MPI_SUM, this->cd->comm);
- DEBUG_MSG("average TrS2 is %g\n", average);
- fftw_interface<rnumber>::free(ca);
- /* output goes here */
- int ntmp[3];
- ntmp[0] = this->rd->sizes[0];
- ntmp[1] = this->rd->sizes[1];
- ntmp[2] = this->rd->sizes[2];
- field_descriptor<rnumber> *scalar_descriptor = new field_descriptor<rnumber>(3, ntmp, mpi_real_type<rnumber>::real(), this->cd->comm);
- clip_zero_padding<rnumber>(scalar_descriptor, trS2, 1);
- int return_value = scalar_descriptor->write(fname, trS2);
- delete scalar_descriptor;
- fftw_interface<rnumber>::free(trS2);
- return return_value;
-}
-
-template <class rnumber>
-int fluid_solver<rnumber>::write_renstrophy()
-{
- TIMEZONE("fluid_solver::write_renstrophy");
- char fname[512];
- this->fill_up_filename("renstrophy", fname);
- rnumber *enstrophy = fftw_interface<rnumber>::alloc_real((this->cd->local_size/3)*2);
- this->ift_vorticity();
- shared_array<double> average_local(1, [&](double* data){
- data[0] = 0;
- });
-
- RLOOP(
- this,
- [&](ptrdiff_t rindex, ptrdiff_t /*xindex*/, ptrdiff_t /*yindex*/, ptrdiff_t /*zindex*/){
- ptrdiff_t tindex = 3*rindex;
- // rindex indexing is thread safe so there is no overlap between threads
- enstrophy[rindex] = (
- this->rvorticity[tindex+0]*this->rvorticity[tindex+0] +
- this->rvorticity[tindex+1]*this->rvorticity[tindex+1] +
- this->rvorticity[tindex+2]*this->rvorticity[tindex+2]
- )/2;
- average_local.getMine()[0] += enstrophy[rindex];
- }
- );
- average_local.mergeParallel();
- double average;
- MPI_Allreduce(
- average_local.getMasterData(),
- &average,
- 1,
- MPI_DOUBLE, MPI_SUM, this->cd->comm);
- DEBUG_MSG("average enstrophy is %g\n", average);
- /* output goes here */
- int ntmp[3];
- ntmp[0] = this->rd->sizes[0];
- ntmp[1] = this->rd->sizes[1];
- ntmp[2] = this->rd->sizes[2];
- field_descriptor<rnumber> *scalar_descriptor = new field_descriptor<rnumber>(3, ntmp, mpi_real_type<rnumber>::real(), this->cd->comm);
- clip_zero_padding<rnumber>(scalar_descriptor, enstrophy, 1);
- int return_value = scalar_descriptor->write(fname, enstrophy);
- delete scalar_descriptor;
- fftw_interface<rnumber>::free(enstrophy);
- return return_value;
-}
-
-template <class rnumber>
-void fluid_solver<rnumber>::compute_pressure(rnumber (*__restrict__ pressure)[2])
-{
- TIMEZONE("fluid_solver::compute_pressure");
- /* assume velocity is already in real space representation */
- /* diagonal terms 11 22 33 */
- RLOOP (
- this,
- [&](ptrdiff_t rindex, ptrdiff_t /*xindex*/, ptrdiff_t /*yindex*/, ptrdiff_t /*zindex*/){
- // rindex indexing is thread safe so there is no overlap between threads
- ptrdiff_t tindex = 3*rindex;
- for (int cc=0; cc<3; cc++)
- this->rv[1][tindex+cc] = this->ru[tindex+cc]*this->ru[tindex+cc];
- }
- );
- this->clean_up_real_space(this->rv[1], 3);
- {
- TIMEZONE("fftw_interface<rnumber>::execute");
- fftw_interface<rnumber>::execute(*(this->vr2c[1]));
- }
- this->dealias(this->cv[1], 3);
- CLOOP_K2(
- this,
- [&](ptrdiff_t cindex, ptrdiff_t xindex, ptrdiff_t yindex, ptrdiff_t zindex, double k2){
- if (k2 <= this->kM2 && k2 > 0)
- {
- // cindex indexing is thread safe so there is no overlap between threads
- ptrdiff_t tindex = 3*cindex;
- for (int i=0; i<2; i++)
- {
- pressure[cindex][i] = -(this->kx[xindex]*this->kx[xindex]*this->cv[1][tindex+0][i] +
- this->ky[yindex]*this->ky[yindex]*this->cv[1][tindex+1][i] +
- this->kz[zindex]*this->kz[zindex]*this->cv[1][tindex+2][i]);
- }
- }
- else
- std::fill_n((rnumber*)(pressure+cindex), 2, 0.0);
- }
- );
- /* off-diagonal terms 12 23 31 */
- RLOOP (
- this,
- [&](ptrdiff_t rindex, ptrdiff_t /*xindex*/, ptrdiff_t /*yindex*/, ptrdiff_t /*zindex*/){
- // rindex indexing is thread safe so there is no overlap between threads
- ptrdiff_t tindex = 3*rindex;
- for (int cc=0; cc<3; cc++)
- this->rv[1][tindex+cc] = this->ru[tindex+cc]*this->ru[tindex+(cc+1)%3];
- }
- );
- this->clean_up_real_space(this->rv[1], 3);
- {
- TIMEZONE("fftw_interface<rnumber>::execute");
- fftw_interface<rnumber>::execute(*(this->vr2c[1]));
- }
- this->dealias(this->cv[1], 3);
- CLOOP_K2(
- this,
- [&](ptrdiff_t cindex, ptrdiff_t xindex, ptrdiff_t yindex, ptrdiff_t zindex, double k2){
- if (k2 <= this->kM2 && k2 > 0)
- {
- // cindex indexing is thread safe so there is no overlap between threads
- ptrdiff_t tindex = 3*cindex;
- for (int i=0; i<2; i++)
- {
- pressure[cindex][i] -= 2*(this->kx[xindex]*this->ky[yindex]*this->cv[1][tindex+0][i] +
- this->ky[yindex]*this->kz[zindex]*this->cv[1][tindex+1][i] +
- this->kz[zindex]*this->kx[xindex]*this->cv[1][tindex+2][i]);
- pressure[cindex][i] /= this->normalization_factor*k2;
- }
- }
- }
- );
-}
-
-template <class rnumber>
-void fluid_solver<rnumber>::compute_gradient_statistics(
- rnumber (*__restrict__ vec)[2],
-double *gradu_moments,
-double *trS2QR_moments,
-ptrdiff_t *gradu_hist,
-ptrdiff_t *trS2QR_hist,
-ptrdiff_t *QR2D_hist,
-double trS2QR_max_estimates[],
-double gradu_max_estimates[],
-int nbins,
-int QR2D_nbins)
-{
- TIMEZONE("fluid_solver::compute_gradient_statistics");
- typename fftw_interface<rnumber>::complex *ca;
- rnumber *ra;
- ca = fftw_interface<rnumber>::alloc_complex(this->cd->local_size*3);
- ra = (rnumber*)(ca);
- this->compute_vector_gradient(ca, vec);
- for (int cc=0; cc<3; cc++)
- {
- std::copy(
- (rnumber*)(ca + cc*this->cd->local_size),
- (rnumber*)(ca + (cc+1)*this->cd->local_size),
- (rnumber*)this->cv[1]);
- fftw_interface<rnumber>::execute(*(this->vc2r[1]));
- std::copy(
- this->rv[1],
- this->rv[1] + this->cd->local_size*2,
- ra + cc*this->cd->local_size*2);
- }
- /* velocity gradient is now stored, in real space, in ra */
- std::fill_n(this->rv[1], 2*this->cd->local_size, 0.0);
- rnumber *dx_u, *dy_u, *dz_u;
- dx_u = ra;
- dy_u = ra + 2*this->cd->local_size;
- dz_u = ra + 4*this->cd->local_size;
- double binsize[2];
- double tmp_max_estimate[3];
- tmp_max_estimate[0] = trS2QR_max_estimates[0];
- tmp_max_estimate[1] = trS2QR_max_estimates[1];
- tmp_max_estimate[2] = trS2QR_max_estimates[2];
- binsize[0] = 2*tmp_max_estimate[2] / QR2D_nbins;
- binsize[1] = 2*tmp_max_estimate[1] / QR2D_nbins;
- ptrdiff_t *local_hist = new ptrdiff_t[QR2D_nbins*QR2D_nbins];
- std::fill_n(local_hist, QR2D_nbins*QR2D_nbins, 0);
- RLOOP(
- this,
- [&](ptrdiff_t rindex, ptrdiff_t /*xindex*/, ptrdiff_t /*yindex*/, ptrdiff_t /*zindex*/){
- rnumber AxxAxx;
- rnumber AyyAyy;
- rnumber AzzAzz;
- rnumber AxyAyx;
- rnumber AyzAzy;
- rnumber AzxAxz;
- rnumber Sxy;
- rnumber Syz;
- rnumber Szx;
- // rindex indexing is thread safe so there is no overlap between threads
- // tindex[0:2] is thread safe too
- ptrdiff_t tindex = 3*rindex;
- AxxAxx = dx_u[tindex+0]*dx_u[tindex+0];
- AyyAyy = dy_u[tindex+1]*dy_u[tindex+1];
- AzzAzz = dz_u[tindex+2]*dz_u[tindex+2];
- AxyAyx = dx_u[tindex+1]*dy_u[tindex+0];
- AyzAzy = dy_u[tindex+2]*dz_u[tindex+1];
- AzxAxz = dz_u[tindex+0]*dx_u[tindex+2];
- this->rv[1][tindex+1] = - (AxxAxx + AyyAyy + AzzAzz)/2 - AxyAyx - AyzAzy - AzxAxz;
- this->rv[1][tindex+2] = - (dx_u[tindex+0]*(AxxAxx/3 + AxyAyx + AzxAxz) +
- dy_u[tindex+1]*(AyyAyy/3 + AxyAyx + AyzAzy) +
- dz_u[tindex+2]*(AzzAzz/3 + AzxAxz + AyzAzy) +
- dx_u[tindex+1]*dy_u[tindex+2]*dz_u[tindex+0] +
- dx_u[tindex+2]*dy_u[tindex+0]*dz_u[tindex+1]);
- int bin0 = int(floor((this->rv[1][tindex+2] + tmp_max_estimate[2]) / binsize[0]));
- int bin1 = int(floor((this->rv[1][tindex+1] + tmp_max_estimate[1]) / binsize[1]));
- if ((bin0 >= 0 && bin0 < QR2D_nbins) &&
- (bin1 >= 0 && bin1 < QR2D_nbins))
- local_hist[bin1*QR2D_nbins + bin0]++;
- Sxy = dx_u[tindex+1]+dy_u[tindex+0];
- Syz = dy_u[tindex+2]+dz_u[tindex+1];
- Szx = dz_u[tindex+0]+dx_u[tindex+2];
- this->rv[1][tindex] = (AxxAxx + AyyAyy + AzzAzz +
- (Sxy*Sxy + Syz*Syz + Szx*Szx)/2);
- }
- );
- MPI_Allreduce(
- local_hist,
- QR2D_hist,
- QR2D_nbins * QR2D_nbins,
- MPI_INT64_T, MPI_SUM, this->cd->comm);
- delete[] local_hist;
- this->compute_rspace_stats3(
- this->rv[1],
- trS2QR_moments,
- trS2QR_hist,
- tmp_max_estimate,
- nbins);
- double *tmp_moments = new double[10*3];
- ptrdiff_t *tmp_hist = new ptrdiff_t[nbins*3];
- for (int cc=0; cc<3; cc++)
- {
- tmp_max_estimate[0] = gradu_max_estimates[cc*3 + 0];
- tmp_max_estimate[1] = gradu_max_estimates[cc*3 + 1];
- tmp_max_estimate[2] = gradu_max_estimates[cc*3 + 2];
- this->compute_rspace_stats3(
- dx_u + cc*2*this->cd->local_size,
- tmp_moments,
- tmp_hist,
- tmp_max_estimate,
- nbins);
- for (int n = 0; n < 10; n++)
- for (int i = 0; i < 3 ; i++)
- {
- gradu_moments[(n*3 + cc)*3 + i] = tmp_moments[n*3 + i];
- }
- for (int n = 0; n < nbins; n++)
- for (int i = 0; i < 3; i++)
- {
- gradu_hist[(n*3 + cc)*3 + i] = tmp_hist[n*3 + i];
- }
- }
- delete[] tmp_moments;
- delete[] tmp_hist;
- fftw_interface<rnumber>::free(ca);
-}
-
-template <class rnumber>
-void fluid_solver<rnumber>::compute_Lagrangian_acceleration(rnumber (*acceleration)[2])
-{
- TIMEZONE("fluid_solver::compute_Lagrangian_acceleration");
- typename fftw_interface<rnumber>::complex *pressure;
- pressure = fftw_interface<rnumber>::alloc_complex(this->cd->local_size/3);
- this->compute_velocity(this->cvorticity);
- this->ift_velocity();
- this->compute_pressure(pressure);
- this->compute_velocity(this->cvorticity);
- std::fill_n((rnumber*)this->cv[1], 2*this->cd->local_size, 0.0);
- CLOOP_K2(
- this,
- [&](ptrdiff_t cindex, ptrdiff_t xindex, ptrdiff_t yindex, ptrdiff_t zindex, double k2){
- if (k2 <= this->kM2)
- {
- // cindex indexing is thread safe so there is no overlap between threads
- ptrdiff_t tindex = 3*cindex;
- for (int cc=0; cc<3; cc++)
- for (int i=0; i<2; i++)
- this->cv[1][tindex+cc][i] = - this->nu*k2*this->cu[tindex+cc][i];
- if (strcmp(this->forcing_type, "linear") == 0)
- {
- double knorm = sqrt(k2);
- if ((this->fk0 <= knorm) &&
- (this->fk1 >= knorm))
- for (int c=0; c<3; c++)
- for (int i=0; i<2; i++)
- this->cv[1][tindex+c][i] += this->famplitude*this->cu[tindex+c][i];
- }
- this->cv[1][tindex+0][0] += this->kx[xindex]*pressure[cindex][1];
- this->cv[1][tindex+1][0] += this->ky[yindex]*pressure[cindex][1];
- this->cv[1][tindex+2][0] += this->kz[zindex]*pressure[cindex][1];
- this->cv[1][tindex+0][1] -= this->kx[xindex]*pressure[cindex][0];
- this->cv[1][tindex+1][1] -= this->ky[yindex]*pressure[cindex][0];
- this->cv[1][tindex+2][1] -= this->kz[zindex]*pressure[cindex][0];
- }
- }
- );
- std::copy(
- (rnumber*)this->cv[1],
- (rnumber*)(this->cv[1] + this->cd->local_size),
- (rnumber*)acceleration);
- fftw_interface<rnumber>::free(pressure);
-}
-
-template <class rnumber>
-void fluid_solver<rnumber>::compute_Eulerian_acceleration(rnumber (*__restrict__ acceleration)[2])
-{
- TIMEZONE("fluid_solver::compute_Eulerian_acceleration");
- std::fill_n((rnumber*)(acceleration), 2*this->cd->local_size, 0.0);
- this->compute_velocity(this->cvorticity);
- /* put in linear terms */
- CLOOP_K2(
- this,
- [&](ptrdiff_t cindex, ptrdiff_t /*xindex*/, ptrdiff_t /*yindex*/, ptrdiff_t /*zindex*/, double k2){
- if (k2 <= this->kM2)
- {
- // cindex indexing is thread safe so there is no overlap between threads
- ptrdiff_t tindex = 3*cindex;
- for (int cc=0; cc<3; cc++)
- for (int i=0; i<2; i++)
- acceleration[tindex+cc][i] = - this->nu*k2*this->cu[tindex+cc][i];
- if (strcmp(this->forcing_type, "linear") == 0)
- {
- double knorm = sqrt(k2);
- if ((this->fk0 <= knorm) &&
- (this->fk1 >= knorm))
- {
- for (int c=0; c<3; c++)
- for (int i=0; i<2; i++)
- acceleration[tindex+c][i] += this->famplitude*this->cu[tindex+c][i];
- }
- }
- }
- }
- );
- this->ift_velocity();
- /* compute uu */
- /* 11 22 33 */
- RLOOP (
- this,
- [&](ptrdiff_t rindex, ptrdiff_t /*xindex*/, ptrdiff_t /*yindex*/, ptrdiff_t /*zindex*/){
- // cindex indexing is thread safe so there is no overlap between threads
- ptrdiff_t tindex = 3*rindex;
- for (int cc=0; cc<3; cc++)
- this->rv[1][tindex+cc] = this->ru[tindex+cc]*this->ru[tindex+cc] / this->normalization_factor;
- }
- );
- this->clean_up_real_space(this->rv[1], 3);
- fftw_interface<rnumber>::execute(*(this->vr2c[1]));
- this->dealias(this->cv[1], 3);
- CLOOP_K2(
- this,
- [&](ptrdiff_t cindex, ptrdiff_t xindex, ptrdiff_t yindex, ptrdiff_t zindex, double k2){
- if (k2 <= this->kM2)
- {
- // cindex indexing is thread safe so there is no overlap between threads
- ptrdiff_t tindex = 3*cindex;
- acceleration[tindex+0][0] +=
- this->kx[xindex]*this->cv[1][tindex+0][1];
- acceleration[tindex+0][1] +=
- -this->kx[xindex]*this->cv[1][tindex+0][0];
- acceleration[tindex+1][0] +=
- this->ky[yindex]*this->cv[1][tindex+1][1];
- acceleration[tindex+1][1] +=
- -this->ky[yindex]*this->cv[1][tindex+1][0];
- acceleration[tindex+2][0] +=
- this->kz[zindex]*this->cv[1][tindex+2][1];
- acceleration[tindex+2][1] +=
- -this->kz[zindex]*this->cv[1][tindex+2][0];
- }
- }
- );
- /* 12 23 31 */
- RLOOP (
- this,
- [&](ptrdiff_t rindex, ptrdiff_t /*xindex*/, ptrdiff_t /*yindex*/, ptrdiff_t /*zindex*/){
- // cindex indexing is thread safe so there is no overlap between threads
- ptrdiff_t tindex = 3*rindex;
- for (int cc=0; cc<3; cc++)
- this->rv[1][tindex+cc] = this->ru[tindex+cc]*this->ru[tindex+(cc+1)%3] / this->normalization_factor;
- }
- );
- this->clean_up_real_space(this->rv[1], 3);
- fftw_interface<rnumber>::execute(*(this->vr2c[1]));
- this->dealias(this->cv[1], 3);
- CLOOP_K2(
- this,
- [&](ptrdiff_t cindex, ptrdiff_t xindex, ptrdiff_t yindex, ptrdiff_t zindex, double k2){
- if (k2 <= this->kM2)
- {
- // cindex indexing is thread safe so there is no overlap between threads
- ptrdiff_t tindex = 3*cindex;
- acceleration[tindex+0][0] +=
- (this->ky[yindex]*this->cv[1][tindex+0][1] +
- this->kz[zindex]*this->cv[1][tindex+2][1]);
- acceleration[tindex+0][1] +=
- - (this->ky[yindex]*this->cv[1][tindex+0][0] +
- this->kz[zindex]*this->cv[1][tindex+2][0]);
- acceleration[tindex+1][0] +=
- (this->kz[zindex]*this->cv[1][tindex+1][1] +
- this->kx[xindex]*this->cv[1][tindex+0][1]);
- acceleration[tindex+1][1] +=
- - (this->kz[zindex]*this->cv[1][tindex+1][0] +
- this->kx[xindex]*this->cv[1][tindex+0][0]);
- acceleration[tindex+2][0] +=
- (this->kx[xindex]*this->cv[1][tindex+2][1] +
- this->ky[yindex]*this->cv[1][tindex+1][1]);
- acceleration[tindex+2][1] +=
- - (this->kx[xindex]*this->cv[1][tindex+2][0] +
- this->ky[yindex]*this->cv[1][tindex+1][0]);
- }
- }
- );
- if (this->cd->myrank == this->cd->rank[0])
- std::fill_n((rnumber*)(acceleration), 6, 0.0);
- this->force_divfree(acceleration);
-}
-
-template <class rnumber>
-void fluid_solver<rnumber>::compute_Lagrangian_acceleration(rnumber *__restrict__ acceleration)
-{
- TIMEZONE("fluid_solver::compute_Lagrangian_acceleration");
- this->compute_Lagrangian_acceleration((typename fftw_interface<rnumber>::complex*)acceleration);
- fftw_interface<rnumber>::execute(*(this->vc2r[1]));
- std::copy(
- this->rv[1],
- this->rv[1] + 2*this->cd->local_size,
- acceleration);
-}
-
-template <class rnumber>
-int fluid_solver<rnumber>::write_rpressure()
-{
- TIMEZONE("fluid_solver::write_rpressure");
- char fname[512];
- typename fftw_interface<rnumber>::complex *pressure;
- pressure = fftw_interface<rnumber>::alloc_complex(this->cd->local_size/3);
- this->compute_velocity(this->cvorticity);
- this->ift_velocity();
- this->compute_pressure(pressure);
- this->fill_up_filename("rpressure", fname);
- rnumber *rpressure = fftw_interface<rnumber>::alloc_real((this->cd->local_size/3)*2);
- typename fftw_interface<rnumber>::plan c2r;
- c2r = fftw_interface<rnumber>::mpi_plan_dft_c2r_3d(
- this->rd->sizes[0], this->rd->sizes[1], this->rd->sizes[2],
- pressure, rpressure, this->cd->comm,
- this->fftw_plan_rigor | FFTW_MPI_TRANSPOSED_IN);
- fftw_interface<rnumber>::execute(c2r);
- /* output goes here */
- int ntmp[3];
- ntmp[0] = this->rd->sizes[0];
- ntmp[1] = this->rd->sizes[1];
- ntmp[2] = this->rd->sizes[2];
- field_descriptor<rnumber> *scalar_descriptor = new field_descriptor<rnumber>(3, ntmp, mpi_real_type<rnumber>::real(), this->cd->comm);
- clip_zero_padding<rnumber>(scalar_descriptor, rpressure, 1);
- int return_value = scalar_descriptor->write(fname, rpressure);
- delete scalar_descriptor;
- fftw_interface<rnumber>::destroy_plan(c2r);
- fftw_interface<rnumber>::free(pressure);
- fftw_interface<rnumber>::free(rpressure);
- return return_value;
-}
-
-/*****************************************************************************/
-
-
-
-
-/*****************************************************************************/
-/* finally, force generation of code for single precision */
-template class fluid_solver<float>;
-template class fluid_solver<double>;
-/*****************************************************************************/
-
diff --git a/bfps/cpp/fluid_solver.hpp b/bfps/cpp/fluid_solver.hpp
deleted file mode 100644
index 4cc75cee4385353f64dc9bc9e7d34c6efba9ad48..0000000000000000000000000000000000000000
--- a/bfps/cpp/fluid_solver.hpp
+++ /dev/null
@@ -1,120 +0,0 @@
-/**********************************************************************
-* *
-* Copyright 2015 Max Planck Institute *
-* for Dynamics and Self-Organization *
-* *
-* This file is part of bfps. *
-* *
-* bfps is free software: you can redistribute it and/or modify *
-* it under the terms of the GNU General Public License as published *
-* by the Free Software Foundation, either version 3 of the License, *
-* or (at your option) any later version. *
-* *
-* bfps is distributed in the hope that it will be useful, *
-* but WITHOUT ANY WARRANTY; without even the implied warranty of *
-* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the *
-* GNU General Public License for more details. *
-* *
-* You should have received a copy of the GNU General Public License *
-* along with bfps. If not, see <http://www.gnu.org/licenses/> *
-* *
-* Contact: Cristian.Lalescu@ds.mpg.de *
-* *
-**********************************************************************/
-
-#include <stdio.h>
-#include <stdlib.h>
-#include <iostream>
-#include "field_descriptor.hpp"
-#include "fluid_solver_base.hpp"
-
-#ifndef FLUID_SOLVER
-
-#define FLUID_SOLVER
-
-extern int myrank, nprocs;
-
-
-/* container for field descriptor, fields themselves, parameters, etc
- * using the same big macro idea that they're using in fftw3.h
- * I feel like I should quote: Ugh.
- * */
-
-template <class rnumber>
-class fluid_solver:public fluid_solver_base<rnumber>
-{
- public:
- /* fields */
- rnumber *rvorticity;
- rnumber *rvelocity ;
- typename fluid_solver_base<rnumber>::cnumber *cvorticity;
- typename fluid_solver_base<rnumber>::cnumber *cvelocity ;
-
- /* short names for velocity, and 4 vorticity fields */
- rnumber *ru, *rv[4];
- typename fluid_solver_base<rnumber>::cnumber *cu, *cv[4];
-
- /* plans */
- typename fftw_interface<rnumber>::plan *c2r_vorticity;
- typename fftw_interface<rnumber>::plan *r2c_vorticity;
- typename fftw_interface<rnumber>::plan *c2r_velocity;
- typename fftw_interface<rnumber>::plan *r2c_velocity;
- typename fftw_interface<rnumber>::plan *uc2r, *ur2c;
- typename fftw_interface<rnumber>::plan *vr2c[3], *vc2r[3];
-
- /* physical parameters */
- double nu;
- int fmode; // for Kolmogorov flow
- double famplitude; // both for Kflow and band forcing
- double fk0, fk1; // for band forcing
- char forcing_type[128];
-
- /* methods */
- fluid_solver(
- const char *NAME,
- int nx,
- int ny,
- int nz,
- double DKX = 1.0,
- double DKY = 1.0,
- double DKZ = 1.0,
- int DEALIAS_TYPE = 1,
- unsigned FFTW_PLAN_RIGOR = FFTW_MEASURE);
- ~fluid_solver(void);
-
- void compute_gradient_statistics(
- rnumber (*__restrict__ vec)[2],
- double *__restrict__ gradu_moments,
- double *__restrict__ trS2_Q_R_moments,
- ptrdiff_t *__restrict__ gradu_histograms,
- ptrdiff_t *__restrict__ trS2_Q_R_histograms,
- ptrdiff_t *__restrict__ QR2D_histogram,
- double trS2_Q_R_max_estimates[3],
- double gradu_max_estimates[9],
- const int nbins_1D = 256,
- const int nbins_2D = 64);
-
- void compute_vorticity(void);
- void compute_velocity(rnumber (*__restrict__ vorticity)[2]);
- void compute_pressure(rnumber (*__restrict__ pressure)[2]);
- void compute_Eulerian_acceleration(rnumber (*__restrict__ dst)[2]);
- void compute_Lagrangian_acceleration(rnumber (*__restrict__ dst)[2]);
- void compute_Lagrangian_acceleration(rnumber *__restrict__ dst);
- void ift_velocity();
- void dft_velocity();
- void ift_vorticity();
- void dft_vorticity();
- void omega_nonlin(int src);
- void step(double dt);
- void impose_zero_modes(void);
- void add_forcing(rnumber (*__restrict__ acc_field)[2], rnumber (*__restrict__ vort_field)[2], rnumber factor);
-
- int read(char field, char representation);
- int write(char field, char representation);
- int write_rTrS2();
- int write_renstrophy();
- int write_rpressure();
-};
-
-#endif//FLUID_SOLVER
-
diff --git a/bfps/cpp/fluid_solver_base.cpp b/bfps/cpp/fluid_solver_base.cpp
deleted file mode 100644
index 6e4fd3335238218bad0b78462d3506ca9b48c721..0000000000000000000000000000000000000000
--- a/bfps/cpp/fluid_solver_base.cpp
+++ /dev/null
@@ -1,834 +0,0 @@
-/**********************************************************************
-* *
-* Copyright 2015 Max Planck Institute *
-* for Dynamics and Self-Organization *
-* *
-* This file is part of bfps. *
-* *
-* bfps is free software: you can redistribute it and/or modify *
-* it under the terms of the GNU General Public License as published *
-* by the Free Software Foundation, either version 3 of the License, *
-* or (at your option) any later version. *
-* *
-* bfps is distributed in the hope that it will be useful, *
-* but WITHOUT ANY WARRANTY; without even the implied warranty of *
-* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the *
-* GNU General Public License for more details. *
-* *
-* You should have received a copy of the GNU General Public License *
-* along with bfps. If not, see <http://www.gnu.org/licenses/> *
-* *
-* Contact: Cristian.Lalescu@ds.mpg.de *
-* *
-**********************************************************************/
-
-
-
-#define NDEBUG
-
-#include <cassert>
-#include <cmath>
-#include <cstring>
-#include "base.hpp"
-#include "fluid_solver_base.hpp"
-#include "fftw_tools.hpp"
-#include "scope_timer.hpp"
-#include "shared_array.hpp"
-
-template <class rnumber>
-void fluid_solver_base<rnumber>::fill_up_filename(const char *base_name, char *destination)
-{
- sprintf(destination, "%s_%s_i%.5x", this->name, base_name, this->iteration);
-}
-
-template <class rnumber>
-void fluid_solver_base<rnumber>::clean_up_real_space(rnumber *a, int howmany)
-{
- TIMEZONE("fluid_solver_base::clean_up_real_space");
- for (ptrdiff_t rindex = 0; rindex < this->cd->local_size*2; rindex += howmany*(this->rd->subsizes[2]+2))
- std::fill_n(a+rindex+this->rd->subsizes[2]*howmany, 2*howmany, 0.0);
-}
-
-template <class rnumber>
-double fluid_solver_base<rnumber>::autocorrel(cnumber *a)
-{
- double *spec = fftw_alloc_real(this->nshells*9);
- double sum_local;
- this->cospectrum(a, a, spec);
- sum_local = 0.0;
- for (unsigned int n = 0; n < this->nshells; n++)
- {
- sum_local += spec[n*9] + spec[n*9 + 4] + spec[n*9 + 8];
- }
- fftw_free(spec);
- return sum_local;
-}
-
-template <class rnumber>
-void fluid_solver_base<rnumber>::cospectrum(cnumber *a, cnumber *b, double *spec)
-{
- TIMEZONE("fluid_solver_base::cospectrum");
- shared_array<double> cospec_local_thread(this->nshells*9,[&](double* cospec_local){
- std::fill_n(cospec_local, this->nshells*9, 0);
- });
-
- CLOOP_K2_NXMODES(
- this,
-
- [&](ptrdiff_t cindex, ptrdiff_t /*xindex*/, ptrdiff_t /*yindex*/,
- ptrdiff_t /*zindex*/, double k2, int nxmodes){
- if (k2 <= this->kMspec2)
- {
- int tmp_int = int(sqrt(k2)/this->dk)*9;
- double* cospec_local = cospec_local_thread.getMine();
- for (int i=0; i<3; i++)
- for (int j=0; j<3; j++)
- {
- cospec_local[tmp_int+i*3+j] += nxmodes * (
- (*(a + 3*cindex+i))[0] * (*(b + 3*cindex+j))[0] +
- (*(a + 3*cindex+i))[1] * (*(b + 3*cindex+j))[1]);
- }
- }}
- );
- cospec_local_thread.mergeParallel();
- MPI_Allreduce(
- cospec_local_thread.getMasterData(),
- (void*)spec,
- this->nshells*9,
- MPI_DOUBLE, MPI_SUM, this->cd->comm);
-}
-
-template <class rnumber>
-void fluid_solver_base<rnumber>::cospectrum(cnumber *a, cnumber *b, double *spec, const double k2exponent)
-{
- TIMEZONE("fluid_solver_base::cospectrum2");
- shared_array<double> cospec_local_thread(this->nshells*9,[&](double* cospec_local){
- std::fill_n(cospec_local, this->nshells*9, 0);
- });
-
- CLOOP_K2_NXMODES(
- this,
-
- [&](ptrdiff_t cindex, ptrdiff_t /*xindex*/, ptrdiff_t /*yindex*/,
- ptrdiff_t /*zindex*/, double k2, int nxmodes){
- if (k2 <= this->kMspec2)
- {
- double factor = nxmodes*pow(k2, k2exponent);
- int tmp_int = int(sqrt(k2)/this->dk)*9;
- double* cospec_local = cospec_local_thread.getMine();
- for (int i=0; i<3; i++)
- for (int j=0; j<3; j++)
- {
- cospec_local[tmp_int+i*3+j] += factor * (
- (*(a + 3*cindex+i))[0] * (*(b + 3*cindex+j))[0] +
- (*(a + 3*cindex+i))[1] * (*(b + 3*cindex+j))[1]);
- }
- }}
- );
- cospec_local_thread.mergeParallel();
- MPI_Allreduce(
- cospec_local_thread.getMasterData(),
- (void*)spec,
- this->nshells*9,
- MPI_DOUBLE, MPI_SUM, this->cd->comm);
- //for (int n=0; n<this->nshells; n++)
- //{
- // spec[n] *= 12.5663706144*pow(this->kshell[n], 2) / this->nshell[n];
- // /*is normalization needed?
- // * spec[n] /= this->normalization_factor*/
- //}
-}
-
-template <class rnumber>
-void fluid_solver_base<rnumber>::compute_rspace_stats(
- const rnumber *a,
- const hid_t group,
- const std::string dset_name,
- const hsize_t toffset,
- const std::vector<double> max_estimate)
-{
- TIMEZONE("fluid_solver_base::compute_rspace_stats");
- const int nmoments = 10;
- int nvals, nbins;
- if (this->rd->myrank == 0)
- {
- hid_t dset, wspace;
- hsize_t dims[3];
- int ndims;
- dset = H5Dopen(group, ("moments/" + dset_name).c_str(), H5P_DEFAULT);
- wspace = H5Dget_space(dset);
- ndims = H5Sget_simple_extent_dims(wspace, dims, NULL);
- assert(ndims == 3);
- variable_used_only_in_assert(ndims);
- assert(dims[1] == nmoments);
- nvals = dims[2];
- H5Sclose(wspace);
- H5Dclose(dset);
- dset = H5Dopen(group, ("histograms/" + dset_name).c_str(), H5P_DEFAULT);
- wspace = H5Dget_space(dset);
- ndims = H5Sget_simple_extent_dims(wspace, dims, NULL);
- assert(ndims == 3);
- nbins = dims[1];
- assert(nvals == dims[2]);
- H5Sclose(wspace);
- H5Dclose(dset);
- }
- MPI_Bcast(&nvals, 1, MPI_INT, 0, this->rd->comm);
- MPI_Bcast(&nbins, 1, MPI_INT, 0, this->rd->comm);
- assert(nvals == max_estimate.size());
- shared_array<double> threaded_local_moments(nmoments*nvals, [&](double* local_moments){
- std::fill_n(local_moments, nmoments*nvals, 0);
- if (nvals == 4) local_moments[3] = max_estimate[3];
- });
-
- shared_array<double> threaded_val_tmp(nvals);
-
- shared_array<ptrdiff_t> threaded_local_hist(nbins*nvals, [&](ptrdiff_t* local_hist){
- std::fill_n(local_hist, nbins*nvals, 0);
- });
-
- // Not written by threads
- double *binsize = new double[nvals];
- for (int i=0; i<nvals; i++)
- binsize[i] = 2*max_estimate[i] / nbins;
-
- RLOOP(
- this,
- [&](ptrdiff_t rindex, ptrdiff_t /*xindex*/, ptrdiff_t /*yindex*/, ptrdiff_t /*zindex*/){
- double *val_tmp = threaded_val_tmp.getMine();
- ptrdiff_t* local_hist = threaded_local_hist.getMine();
- double *local_moments = threaded_local_moments.getMine();
-
- if (nvals == 4) val_tmp[3] = 0.0;
- for (int i=0; i<3; i++)
- {
- val_tmp[i] = a[rindex*3+i];
- if (nvals == 4) val_tmp[3] += val_tmp[i]*val_tmp[i];
- }
- if (nvals == 4)
- {
- val_tmp[3] = sqrt(val_tmp[3]);
- if (val_tmp[3] < local_moments[0*nvals+3])
- local_moments[0*nvals+3] = val_tmp[3];
- if (val_tmp[3] > local_moments[9*nvals+3])
- local_moments[9*nvals+3] = val_tmp[3];
- int bin = int(floor(val_tmp[3]*2/binsize[3]));
- if (bin >= 0 && bin < nbins)
- local_hist[bin*nvals+3]++;
- }
- for (int i=0; i<3; i++)
- {
- if (val_tmp[i] < local_moments[0*nvals+i])
- local_moments[0*nvals+i] = val_tmp[i];
- if (val_tmp[i] > local_moments[(nmoments-1)*nvals+i])
- local_moments[(nmoments-1)*nvals+i] = val_tmp[i];
- int bin = int(floor((val_tmp[i] + max_estimate[i]) / binsize[i]));
- if (bin >= 0 && bin < nbins)
- local_hist[bin*nvals+i]++;
- }
- for (int n=1; n < nmoments-1; n++){
- double pow_tmp = 1.;
- for (int i=0; i<nvals; i++){
- local_moments[n*nvals + i] += (pow_tmp = val_tmp[i]*pow_tmp);
- }
- }
- }
- );
-
- threaded_local_hist.mergeParallel();
- threaded_local_moments.mergeParallel([&](const int idx, const double& v1, const double& v2) -> double {
- if(nvals == int(4) && idx == 0*nvals+3){
- return std::min(v1, v2);
- }
- if(nvals == int(4) && idx == 9*nvals+3){
- return std::max(v1, v2);
- }
- if(idx < 3){
- return std::min(v1, v2);
- }
- if((nmoments-1)*nvals <= idx && idx < (nmoments-1)*nvals+3){
- return std::max(v1, v2);
- }
- return v1 + v2;
- });
-
-
- double *moments = new double[nmoments*nvals];
- MPI_Allreduce(
- threaded_local_moments.getMasterData(),
- (void*)moments,
- nvals,
- MPI_DOUBLE, MPI_MIN, this->cd->comm);
- MPI_Allreduce(
- (threaded_local_moments.getMasterData() + nvals),
- (void*)(moments+nvals),
- (nmoments-2)*nvals,
- MPI_DOUBLE, MPI_SUM, this->cd->comm);
- MPI_Allreduce(
- (threaded_local_moments.getMasterData() + (nmoments-1)*nvals),
- (void*)(moments+(nmoments-1)*nvals),
- nvals,
- MPI_DOUBLE, MPI_MAX, this->cd->comm);
- ptrdiff_t *hist = new ptrdiff_t[nbins*nvals];
- MPI_Allreduce(
- threaded_local_hist.getMasterData(),
- (void*)hist,
- nbins*nvals,
- MPI_INT64_T, MPI_SUM, this->cd->comm);
- for (int n=1; n < nmoments-1; n++)
- for (int i=0; i<nvals; i++)
- moments[n*nvals + i] /= this->normalization_factor;
- delete[] binsize;
- if (this->rd->myrank == 0)
- {
- hid_t dset, wspace, mspace;
- hsize_t count[3], offset[3], dims[3];
- dset = H5Dopen(group, ("moments/" + dset_name).c_str(), H5P_DEFAULT);
- wspace = H5Dget_space(dset);
- H5Sget_simple_extent_dims(wspace, dims, NULL);
- offset[0] = toffset;
- offset[1] = 0;
- offset[2] = 0;
- count[0] = 1;
- count[1] = nmoments;
- count[2] = nvals;
- mspace = H5Screate_simple(3, count, NULL);
- H5Sselect_hyperslab(wspace, H5S_SELECT_SET, offset, NULL, count, NULL);
- H5Dwrite(dset, H5T_NATIVE_DOUBLE, mspace, wspace, H5P_DEFAULT, moments);
- H5Sclose(wspace);
- H5Sclose(mspace);
- H5Dclose(dset);
- dset = H5Dopen(group, ("histograms/" + dset_name).c_str(), H5P_DEFAULT);
- wspace = H5Dget_space(dset);
- count[1] = nbins;
- mspace = H5Screate_simple(3, count, NULL);
- H5Sselect_hyperslab(wspace, H5S_SELECT_SET, offset, NULL, count, NULL);
- H5Dwrite(dset, H5T_NATIVE_INT64, mspace, wspace, H5P_DEFAULT, hist);
- H5Sclose(wspace);
- H5Sclose(mspace);
- H5Dclose(dset);
- }
- delete[] moments;
- delete[] hist;
-}
-
-
-
-template <class rnumber>
-template<int nvals>
-void fluid_solver_base<rnumber>::compute_rspace_stats(
- rnumber *a,
- double *moments,
- ptrdiff_t *hist,
- double max_estimate[],
- const int nbins)
-{
- TIMEZONE("fluid_solver_base::compute_rspace_stats");
- shared_array<double> threaded_local_moments(10*nvals,[&](double* local_moments){
- std::fill_n(local_moments, 10*nvals, 0);
- if (nvals == 4) local_moments[3] = max_estimate[3];
- });
-
- shared_array<ptrdiff_t> threaded_local_hist(nbins*nvals, [&](ptrdiff_t* local_hist){
- std::fill_n(local_hist, nbins*nvals, 0);
- });
-
- // Will not be modified by the threads
- double binsize[nvals];
- for (int i=0; i<nvals; i++)
- binsize[i] = 2*max_estimate[i] / nbins;
-
- RLOOP(
- this,
- [&](ptrdiff_t rindex, ptrdiff_t /*xindex*/, ptrdiff_t /*yindex*/, ptrdiff_t /*zindex*/){
- ptrdiff_t *local_hist = threaded_local_hist.getMine();
- double *local_moments = threaded_local_moments.getMine();
-
- double val_tmp[nvals];
- if (nvals == 4) val_tmp[3] = 0.0;
- for (int i=0; i<3; i++)
- {
- val_tmp[i] = a[rindex*3+i];
- if (nvals == 4) val_tmp[3] += val_tmp[i]*val_tmp[i];
- }
- if (nvals == 4)
- {
- val_tmp[3] = sqrt(val_tmp[3]);
- if (val_tmp[3] < local_moments[0*nvals+3])
- local_moments[0*nvals+3] = val_tmp[3];
- if (val_tmp[3] > local_moments[9*nvals+3])
- local_moments[9*nvals+3] = val_tmp[3];
- int bin = int(floor(val_tmp[3]*2/binsize[3]));
- if (bin >= 0 && bin < nbins)
- local_hist[bin*nvals+3]++;
- }
- for (int i=0; i<3; i++)
- {
- if (val_tmp[i] < local_moments[0*nvals+i])
- local_moments[0*nvals+i] = val_tmp[i];
- if (val_tmp[i] > local_moments[9*nvals+i])
- local_moments[9*nvals+i] = val_tmp[i];
- int bin = int(floor((val_tmp[i] + max_estimate[i]) / binsize[i]));
- if (bin >= 0 && bin < nbins)
- local_hist[bin*nvals+i]++;
- }
- for (int n=1; n<9; n++){
- double pow_tmp = 1;
- for (int i=0; i<nvals; i++){
- local_moments[n*nvals + i] += (pow_tmp = val_tmp[i]*pow_tmp);
- }
- }
- }
- );
-
- threaded_local_moments.mergeParallel([&](const int idx, const double& v1, const double& v2) -> double {
- if(nvals == int(4) && idx == 0*nvals+3){
- return std::min(v1, v2);
- }
- if(nvals == int(4) && idx == 9*nvals+3){
- return std::max(v1, v2);
- }
- if(idx < 3){
- return std::min(v1, v2);
- }
- if(9*nvals <= idx && idx < 9*nvals+3){
- return std::max(v1, v2);
- }
- return v1 + v2;
- });
- threaded_local_hist.mergeParallel();
-
- MPI_Allreduce(
- threaded_local_moments.getMasterData(),
- (void*)moments,
- nvals,
- MPI_DOUBLE, MPI_MIN, this->cd->comm);
- MPI_Allreduce(
- (threaded_local_moments.getMasterData() + nvals),
- (void*)(moments+nvals),
- 8*nvals,
- MPI_DOUBLE, MPI_SUM, this->cd->comm);
- MPI_Allreduce(
- (threaded_local_moments.getMasterData() + 9*nvals),
- (void*)(moments+9*nvals),
- nvals,
- MPI_DOUBLE, MPI_MAX, this->cd->comm);
- MPI_Allreduce(
- (void*)threaded_local_hist.getMasterData(),
- (void*)hist,
- nbins*nvals,
- MPI_INT64_T, MPI_SUM, this->cd->comm);
- for (int n=1; n<9; n++)
- for (int i=0; i<nvals; i++)
- moments[n*nvals + i] /= this->normalization_factor;
-}
-
-template <class rnumber>
-void fluid_solver_base<rnumber>::write_spectrum(const char *fname, cnumber *a, const double k2exponent)
-{
- TIMEZONE("fluid_solver_base::write_spectrum");
- double *spec = fftw_alloc_real(this->nshells);
- this->cospectrum(a, a, spec, k2exponent);
- if (this->cd->myrank == 0)
- {
- FILE *spec_file;
- char full_name[512];
- sprintf(full_name, "%s_%s_spec", this->name, fname);
- spec_file = fopen(full_name, "ab");
- fwrite((void*)&this->iteration, sizeof(int), 1, spec_file);
- fwrite((void*)spec, sizeof(double), this->nshells, spec_file);
- fclose(spec_file);
- }
- fftw_free(spec);
-}
-
-/*****************************************************************************/
-/* macro for specializations to numeric types compatible with FFTW */
-
-template <class rnumber>
-fluid_solver_base<rnumber>::fluid_solver_base(
- const char *NAME,
- int nx,
- int ny,
- int nz,
- double DKX,
- double DKY,
- double DKZ,
- int DEALIAS_TYPE,
- unsigned FFTW_PLAN_RIGOR)
-{
- TIMEZONE("fluid_solver_base::fluid_solver_base");
- strncpy(this->name, NAME, 256);
- this->name[255] = '\0';
- this->iteration = 0;
- this->fftw_plan_rigor = FFTW_PLAN_RIGOR;
-
- int ntmp[4];
- ntmp[0] = nz;
- ntmp[1] = ny;
- ntmp[2] = nx;
- ntmp[3] = 3;
- this->rd = new field_descriptor<rnumber>(
- 4, ntmp, mpi_real_type<rnumber>::real(), MPI_COMM_WORLD);
- this->normalization_factor = (this->rd->full_size/3);
- ntmp[0] = ny;
- ntmp[1] = nz;
- ntmp[2] = nx/2 + 1;
- ntmp[3] = 3;
- this->cd = new field_descriptor<rnumber>(
- 4, ntmp, mpi_real_type<rnumber>::complex(), this->rd->comm);
-
- this->dkx = DKX;
- this->dky = DKY;
- this->dkz = DKZ;
- this->kx = new double[this->cd->sizes[2]];
- this->ky = new double[this->cd->subsizes[0]];
- this->kz = new double[this->cd->sizes[1]];
- this->dealias_type = DEALIAS_TYPE;
- switch(this->dealias_type)
- {
- /* HL07 smooth filter */
- case 1:
- this->kMx = this->dkx*(int(this->rd->sizes[2] / 2)-1);
- this->kMy = this->dky*(int(this->rd->sizes[1] / 2)-1);
- this->kMz = this->dkz*(int(this->rd->sizes[0] / 2)-1);
- break;
- default:
- this->kMx = this->dkx*(int(this->rd->sizes[2] / 3)-1);
- this->kMy = this->dky*(int(this->rd->sizes[1] / 3)-1);
- this->kMz = this->dkz*(int(this->rd->sizes[0] / 3)-1);
- }
- int i, ii;
- for (i = 0; i<this->cd->sizes[2]; i++)
- this->kx[i] = i*this->dkx;
- for (i = 0; i<this->cd->subsizes[0]; i++)
- {
- ii = i + this->cd->starts[0];
- if (ii <= this->rd->sizes[1]/2)
- this->ky[i] = this->dky*ii;
- else
- this->ky[i] = this->dky*(ii - this->rd->sizes[1]);
- }
- for (i = 0; i<this->cd->sizes[1]; i++)
- {
- if (i <= this->rd->sizes[0]/2)
- this->kz[i] = this->dkz*i;
- else
- this->kz[i] = this->dkz*(i - this->rd->sizes[0]);
- }
- this->kM = this->kMx;
- if (this->kM < this->kMy) this->kM = this->kMy;
- if (this->kM < this->kMz) this->kM = this->kMz;
- this->kM2 = this->kM * this->kM;
- this->kMspec = this->kM;
- this->kMspec2 = this->kM2;
- this->dk = this->dkx;
- if (this->dk > this->dky) this->dk = this->dky;
- if (this->dk > this->dkz) this->dk = this->dkz;
- this->dk2 = this->dk*this->dk;
- DEBUG_MSG(
- "kM = %g, kM2 = %g, dk = %g, dk2 = %g\n",
- this->kM, this->kM2, this->dk, this->dk2);
- /* spectra stuff */
- this->nshells = int(this->kMspec / this->dk) + 2;
- DEBUG_MSG(
- "kMspec = %g, kMspec2 = %g, nshells = %ld\n",
- this->kMspec, this->kMspec2, this->nshells);
- this->kshell = new double[this->nshells];
- std::fill_n(this->kshell, this->nshells, 0.0);
- this->nshell = new int64_t[this->nshells];
- std::fill_n(this->nshell, this->nshells, 0);
- DEBUG_MSG("fluid_solver_base::fluid_solver_base before declaring shared_array\n");
-
- shared_array<double> kshell_local_threaded(this->nshells,[&](double* kshell_local){
- std::fill_n(kshell_local, this->nshells, 0.0);
- });
- DEBUG_MSG("fluid_solver_base::fluid_solver_base before declaring shared_array\n");
- shared_array<int64_t> nshell_local_threaded(this->nshells,[&](int64_t* nshell_local){
- std::fill_n(nshell_local, this->nshells, 0);
- });
-
- std::vector<std::unordered_map<int, double>> Fourier_filter_threaded(omp_get_max_threads());
-
- DEBUG_MSG("fluid_solver_base::fluid_solver_base before cloop_k2_nxmodes\n");
- CLOOP_K2_NXMODES(
- this,
-
- [&](ptrdiff_t /*cindex*/, ptrdiff_t /*xindex*/, ptrdiff_t /*yindex*/,
- ptrdiff_t /*zindex*/, double k2, int nxmodes){
- if (k2 < this->kM2)
- {
- double knorm = sqrt(k2);
- nshell_local_threaded.getMine()[int(knorm/this->dk)] += nxmodes;
- kshell_local_threaded.getMine()[int(knorm/this->dk)] += nxmodes*knorm;
- }
- Fourier_filter_threaded[omp_get_thread_num()][int(round(k2 / this->dk2))] = exp(-36.0 * pow(k2/this->kM2, 18.));}
- );
-
- // Merge results
- nshell_local_threaded.mergeParallel();
- kshell_local_threaded.mergeParallel();
- for(int idxMerge = 0 ; idxMerge < int(Fourier_filter_threaded.size()) ; ++idxMerge){
- for(const auto kv : Fourier_filter_threaded[idxMerge]){
- this->Fourier_filter[kv.first] = kv.second;
- }
- }
-
- MPI_Allreduce(
- (void*)(nshell_local_threaded.getMasterData()),
- (void*)(this->nshell),
- this->nshells,
- MPI_INT64_T, MPI_SUM, this->cd->comm);
- MPI_Allreduce(
- (void*)(kshell_local_threaded.getMasterData()),
- (void*)(this->kshell),
- this->nshells,
- MPI_DOUBLE, MPI_SUM, this->cd->comm);
- for (unsigned int n=0; n<this->nshells; n++)
- {
- if (this->nshell[n] != 0)
- this->kshell[n] /= this->nshell[n];
- else
- this->kshell[n] = -1;
- }
- DEBUG_MSG("exiting fluid_solver_base::fluid_solver_base\n");
-}
-
-template <class rnumber>
-fluid_solver_base<rnumber>::~fluid_solver_base()
-{
- delete[] this->kshell;
- delete[] this->nshell;
-
- delete[] this->kx;
- delete[] this->ky;
- delete[] this->kz;
-
- delete this->cd;
- delete this->rd;
-}
-
-template <class rnumber>
-void fluid_solver_base<rnumber>::low_pass_Fourier(cnumber *a, const int howmany, const double kmax)
-{
- TIMEZONE("fluid_solver_base::low_pass_Fourier");
- const double km2 = kmax*kmax;
- const int howmany2 = 2*howmany;
- /*DEBUG_MSG("entered low_pass_Fourier, kmax=%lg km2=%lg howmany2=%d\n", kmax, km2, howmany2);*/
- CLOOP_K2(
- this,
- /*DEBUG_MSG("kx=%lg ky=%lg kz=%lg k2=%lg\n",
- this->kx[xindex],
- this->ky[yindex],
- this->kz[zindex],
- k2);*/
-
- [&](ptrdiff_t cindex, ptrdiff_t xindex, ptrdiff_t yindex,
- ptrdiff_t zindex, double k2){
- if (k2 >= km2)
- std::fill_n((rnumber*)(a + howmany*cindex), howmany2, 0.0);}
- );
-}
-
-template <class rnumber>
-void fluid_solver_base<rnumber>::dealias(cnumber *a, const int howmany)
-{
- TIMEZONE("fluid_solver_base::dealias");
- if (this->dealias_type == 0)
- {
- this->low_pass_Fourier(a, howmany, this->kM);
- return;
- }
-
- CLOOP_K2(
- this,
- [&](ptrdiff_t cindex, ptrdiff_t /*xindex*/, ptrdiff_t /*yindex*/,
- ptrdiff_t /*zindex*/, double k2){
- double tval = this->Fourier_filter[int(round(k2/this->dk2))];
- // It is thread safe on the index cindex
- for (int tcounter = 0; tcounter < howmany; tcounter++)
- for (int i=0; i<2; i++)
- a[howmany*cindex+tcounter][i] *= tval;
- }
- );
-}
-
-template <class rnumber>
-void fluid_solver_base<rnumber>::force_divfree(cnumber *a)
-{
- TIMEZONE("fluid_solver_base::force_divfree");
- CLOOP_K2(
- this,
-
- [&](ptrdiff_t cindex, ptrdiff_t xindex, ptrdiff_t yindex,
- ptrdiff_t zindex, double k2){
- if (k2 > 0)
- {
- // It is thread safe on index cindex
- cnumber tval;
- tval[0] = (this->kx[xindex]*((*(a + cindex*3 ))[0]) +
- this->ky[yindex]*((*(a + cindex*3+1))[0]) +
- this->kz[zindex]*((*(a + cindex*3+2))[0]) ) / k2;
- tval[1] = (this->kx[xindex]*((*(a + cindex*3 ))[1]) +
- this->ky[yindex]*((*(a + cindex*3+1))[1]) +
- this->kz[zindex]*((*(a + cindex*3+2))[1]) ) / k2;
- for (int imag_part=0; imag_part<2; imag_part++)
- {
- a[cindex*3 ][imag_part] -= tval[imag_part]*this->kx[xindex];
- a[cindex*3+1][imag_part] -= tval[imag_part]*this->ky[yindex];
- a[cindex*3+2][imag_part] -= tval[imag_part]*this->kz[zindex];
- }
- }}
- );
- if (this->cd->myrank == this->cd->rank[0])
- std::fill_n((rnumber*)(a), 6, 0.0);
-}
-
-template <class rnumber>
-void fluid_solver_base<rnumber>::compute_vector_gradient(cnumber *A, cnumber *cvec)
-{
- TIMEZONE("fluid_solver_base::compute_vector_gradient");
- std::fill_n((rnumber*)A, 3*2*this->cd->local_size, 0.0);
- cnumber *dx_u, *dy_u, *dz_u;
- dx_u = A;
- dy_u = A + this->cd->local_size;
- dz_u = A + 2*this->cd->local_size;
- CLOOP_K2(
- this,
-
- [&](ptrdiff_t cindex, ptrdiff_t xindex, ptrdiff_t yindex,
- ptrdiff_t zindex, double k2){
- if (k2 <= this->kM2)
- {
- // It is thread safe on cindex
- ptrdiff_t tindex = 3*cindex;
- for (int cc=0; cc<3; cc++)
- {
- dx_u[tindex + cc][0] = -this->kx[xindex]*cvec[tindex+cc][1];
- dx_u[tindex + cc][1] = this->kx[xindex]*cvec[tindex+cc][0];
- dy_u[tindex + cc][0] = -this->ky[yindex]*cvec[tindex+cc][1];
- dy_u[tindex + cc][1] = this->ky[yindex]*cvec[tindex+cc][0];
- dz_u[tindex + cc][0] = -this->kz[zindex]*cvec[tindex+cc][1];
- dz_u[tindex + cc][1] = this->kz[zindex]*cvec[tindex+cc][0];
- }
- }}
- );
-}
-
-template <class rnumber>
-void fluid_solver_base<rnumber>::symmetrize(cnumber *data, const int howmany)
-{
- TIMEZONE("fluid_solver_base::symmetrize");
- ptrdiff_t ii, cc;
- MPI_Status *mpistatus = new MPI_Status;
- if (this->cd->myrank == this->cd->rank[0])
- {
- for (cc = 0; cc < howmany; cc++)
- data[cc][1] = 0.0;
- for (ii = 1; ii < this->cd->sizes[1]/2; ii++)
- for (cc = 0; cc < howmany; cc++) {
- ( *(data + cc + howmany*(this->cd->sizes[1] - ii)*this->cd->sizes[2]))[0] =
- (*(data + cc + howmany*( ii)*this->cd->sizes[2]))[0];
- ( *(data + cc + howmany*(this->cd->sizes[1] - ii)*this->cd->sizes[2]))[1] =
- -(*(data + cc + howmany*( ii)*this->cd->sizes[2]))[1];
- }
- }
- cnumber *buffer;
- buffer = fftw_interface<rnumber>::alloc_complex(howmany*this->cd->sizes[1]);
- ptrdiff_t yy;
- /*ptrdiff_t tindex;*/
- int ranksrc, rankdst;
- for (yy = 1; yy < this->cd->sizes[0]/2; yy++) {
- ranksrc = this->cd->rank[yy];
- rankdst = this->cd->rank[this->cd->sizes[0] - yy];
- if (this->cd->myrank == ranksrc)
- for (ii = 0; ii < this->cd->sizes[1]; ii++)
- for (cc = 0; cc < howmany; cc++)
- for (int imag_comp=0; imag_comp<2; imag_comp++)
- (*(buffer + howmany*ii+cc))[imag_comp] =
- (*(data + howmany*((yy - this->cd->starts[0])*this->cd->sizes[1] + ii)*this->cd->sizes[2] + cc))[imag_comp];
- if (ranksrc != rankdst)
- {
- if (this->cd->myrank == ranksrc)
- MPI_Send((void*)buffer,
- howmany*this->cd->sizes[1], mpi_real_type<rnumber>::complex(), rankdst, yy,
- this->cd->comm);
- if (this->cd->myrank == rankdst)
- MPI_Recv((void*)buffer,
- howmany*this->cd->sizes[1], mpi_real_type<rnumber>::complex(), ranksrc, yy,
- this->cd->comm, mpistatus);
- }
- if (this->cd->myrank == rankdst)
- {
- for (ii = 1; ii < this->cd->sizes[1]; ii++)
- for (cc = 0; cc < howmany; cc++)
- {
- (*(data + howmany*((this->cd->sizes[0] - yy - this->cd->starts[0])*this->cd->sizes[1] + ii)*this->cd->sizes[2] + cc))[0] =
- (*(buffer + howmany*(this->cd->sizes[1]-ii)+cc))[0];
- (*(data + howmany*((this->cd->sizes[0] - yy - this->cd->starts[0])*this->cd->sizes[1] + ii)*this->cd->sizes[2] + cc))[1] =
- -(*(buffer + howmany*(this->cd->sizes[1]-ii)+cc))[1];
- }
- for (cc = 0; cc < howmany; cc++)
- {
- (*((data + cc + howmany*(this->cd->sizes[0] - yy - this->cd->starts[0])*this->cd->sizes[1]*this->cd->sizes[2])))[0] = (*(buffer + cc))[0];
- (*((data + cc + howmany*(this->cd->sizes[0] - yy - this->cd->starts[0])*this->cd->sizes[1]*this->cd->sizes[2])))[1] = -(*(buffer + cc))[1];
- }
- }
- }
- fftw_interface<rnumber>::free(buffer);
- delete mpistatus;
- /* put asymmetric data to 0 */
- /*if (this->cd->myrank == this->cd->rank[this->cd->sizes[0]/2])
- {
- tindex = howmany*(this->cd->sizes[0]/2 - this->cd->starts[0])*this->cd->sizes[1]*this->cd->sizes[2];
- for (ii = 0; ii < this->cd->sizes[1]; ii++)
- {
- std::fill_n((rnumber*)(data + tindex), howmany*2*this->cd->sizes[2], 0.0);
- tindex += howmany*this->cd->sizes[2];
- }
- }
- tindex = howmany*();
- std::fill_n((rnumber*)(data + tindex), howmany*2, 0.0);*/
-}
-
-template <class rnumber>
-int fluid_solver_base<rnumber>::read_base(const char *fname, rnumber *data)
-{
- char full_name[512];
- sprintf(full_name, "%s_%s_i%.5x", this->name, fname, this->iteration);
- return this->rd->read(full_name, (void*)data);
-}
-
-template <class rnumber>
-int fluid_solver_base<rnumber>::read_base(const char *fname, cnumber *data)
-{
- char full_name[512];
- sprintf(full_name, "%s_%s_i%.5x", this->name, fname, this->iteration);
- return this->cd->read(full_name, (void*)data);
-}
-
-template <class rnumber>
-int fluid_solver_base<rnumber>::write_base(const char *fname, rnumber *data)
-{
- char full_name[512];
- sprintf(full_name, "%s_%s_i%.5x", this->name, fname, this->iteration);
- return this->rd->write(full_name, (void*)data);
-}
-
-template <class rnumber>
-int fluid_solver_base<rnumber>::write_base(const char *fname, cnumber *data)
-{
- char full_name[512];
- sprintf(full_name, "%s_%s_i%.5x", this->name, fname, this->iteration);
- return this->cd->write(full_name, (void*)data);
-}
-
-/* finally, force generation of code */
-template class fluid_solver_base<float>;
-template class fluid_solver_base<double>;
-
-/*****************************************************************************/
-
-
-
-
diff --git a/bfps/cpp/fluid_solver_base.hpp b/bfps/cpp/fluid_solver_base.hpp
deleted file mode 100644
index e446956001a08fdbf0d3b11da8552e1cb6c61a45..0000000000000000000000000000000000000000
--- a/bfps/cpp/fluid_solver_base.hpp
+++ /dev/null
@@ -1,272 +0,0 @@
-/**********************************************************************
-* *
-* Copyright 2015 Max Planck Institute *
-* for Dynamics and Self-Organization *
-* *
-* This file is part of bfps. *
-* *
-* bfps is free software: you can redistribute it and/or modify *
-* it under the terms of the GNU General Public License as published *
-* by the Free Software Foundation, either version 3 of the License, *
-* or (at your option) any later version. *
-* *
-* bfps is distributed in the hope that it will be useful, *
-* but WITHOUT ANY WARRANTY; without even the implied warranty of *
-* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the *
-* GNU General Public License for more details. *
-* *
-* You should have received a copy of the GNU General Public License *
-* along with bfps. If not, see <http://www.gnu.org/licenses/> *
-* *
-* Contact: Cristian.Lalescu@ds.mpg.de *
-* *
-**********************************************************************/
-
-#include <stdio.h>
-#include <stdlib.h>
-#include <hdf5.h>
-#include <iostream>
-#include <unordered_map>
-#include <vector>
-#include "base.hpp"
-#include "field_descriptor.hpp"
-#include "scope_timer.hpp"
-#include "omputils.hpp"
-
-#ifndef FLUID_SOLVER_BASE
-
-#define FLUID_SOLVER_BASE
-
-extern int myrank, nprocs;
-
-
-/* container for field descriptor, fields themselves, parameters, etc
- * using the same big macro idea that they're using in fftw3.h
- * I feel like I should quote: Ugh.
- * */
-
-template <class rnumber>
-class fluid_solver_base
-{
- protected:
- typedef rnumber cnumber[2];
- public:
- field_descriptor<rnumber> *cd, *rd;
- ptrdiff_t normalization_factor;
- unsigned fftw_plan_rigor;
-
- /* simulation parameters */
- char name[256];
- int iteration;
-
- /* physical parameters */
- double dkx, dky, dkz, dk, dk2;
-
- /* mode and dealiasing information */
- int dealias_type;
- double kMx, kMy, kMz, kM, kM2;
- double kMspec, kMspec2;
- double *kx, *ky, *kz;
- std::unordered_map<int, double> Fourier_filter;
- double *kshell;
- int64_t *nshell;
- unsigned int nshells;
-
-
- /* methods */
- fluid_solver_base(
- const char *NAME,
- int nx,
- int ny,
- int nz,
- double DKX = 1.0,
- double DKY = 1.0,
- double DKZ = 1.0,
- int DEALIAS_TYPE = 0,
- unsigned FFTW_PLAN_RIGOR = DEFAULT_FFTW_FLAG);
- ~fluid_solver_base();
-
- void low_pass_Fourier(cnumber *__restrict__ a, int howmany, double kmax);
- void dealias(cnumber *__restrict__ a, int howmany);
- void force_divfree(cnumber *__restrict__ a);
- void symmetrize(cnumber *__restrict__ a, int howmany);
- void clean_up_real_space(rnumber *__restrict__ a, int howmany);
- void cospectrum(cnumber *__restrict__ a, cnumber *__restrict__ b, double *__restrict__ spec);
- void cospectrum(cnumber *__restrict__ a, cnumber *__restrict__ b, double *__restrict__ spec, const double k2exponent);
- double autocorrel(cnumber *__restrict__ a);
- void compute_rspace_stats(
- const rnumber *__restrict__ a,
- const hid_t group,
- const std::string dset_name,
- const hsize_t toffset,
- const std::vector<double> max_estimate);
- template <int nvals>
- void compute_rspace_stats(rnumber *__restrict__ a,
- double *__restrict__ moments,
- ptrdiff_t *__restrict__ hist,
- double max_estimate[nvals],
- const int nbins = 256);
- inline void compute_rspace_stats3(rnumber *__restrict__ a,
- double *__restrict__ moments,
- ptrdiff_t *__restrict__ hist,
- double max_estimate[3],
- const int nbins = 256)
- {
- this->compute_rspace_stats<3>(a, moments, hist, max_estimate, nbins);
- }
- inline void compute_rspace_stats4(rnumber *__restrict__ a,
- double *__restrict__ moments,
- ptrdiff_t *__restrict__ hist,
- double max_estimate[4],
- const int nbins = 256)
- {
- this->compute_rspace_stats<4>(a, moments, hist, max_estimate, nbins);
- }
- void compute_vector_gradient(rnumber (*__restrict__ A)[2], rnumber(*__restrict__ source)[2]);
- void write_spectrum(const char *fname, cnumber *a, const double k2exponent = 0.0);
- void fill_up_filename(const char *base_name, char *full_name);
- int read_base(const char *fname, rnumber *data);
- int read_base(const char *fname, cnumber *data);
- int write_base(const char *fname, rnumber *data);
- int write_base(const char *fname, cnumber *data);
-};
-
-
-
-/*****************************************************************************/
-/* macros for loops */
-
-/* Fourier space loop */
-template <class ObjectType, class FuncType>
-void CLOOP(ObjectType* obj, FuncType expression)
-{
- TIMEZONE("CLOOP");
- #pragma omp parallel
- {
- const hsize_t start = OmpUtils::ForIntervalStart(obj->cd->subsizes[0]);
- const hsize_t end = OmpUtils::ForIntervalEnd(obj->cd->subsizes[0]);
- for (ptrdiff_t yindex = start; yindex < ptrdiff_t(end); yindex++){
- ptrdiff_t cindex = yindex*obj->cd->subsizes[1]*obj->cd->subsizes[2];
- for (ptrdiff_t zindex = 0; zindex < obj->cd->subsizes[1]; zindex++)
- for (ptrdiff_t xindex = 0; xindex < obj->cd->subsizes[2]; xindex++)
- {
- expression(cindex, xindex, yindex, zindex);
- cindex++;
- }
- }
- }
-}
-
-template <class ObjectType, class FuncType>
-void CLOOP_NXMODES(ObjectType* obj, FuncType expression)
-{
- TIMEZONE("CLOOP_NXMODES");
- #pragma omp parallel
- {
- const hsize_t start = OmpUtils::ForIntervalStart(obj->cd->subsizes[1]);
- const hsize_t end = OmpUtils::ForIntervalEnd(obj->cd->subsizes[1]);
- for (ptrdiff_t yindex = 0; yindex < obj->cd->subsizes[0]; yindex++){
- for (ptrdiff_t zindex = start; zindex < ptrdiff_t(end); zindex++)
- {
- ptrdiff_t cindex = yindex*obj->cd->subsizes[1]*obj->cd->subsizes[2]
- + zindex*obj->cd->subsizes[2];
- int nxmodes = 1;
- ptrdiff_t xindex = 0;
- expression();
- cindex++;
- nxmodes = 2;
- for (xindex = 1; xindex < obj->cd->subsizes[2]; xindex++)
- {
- expression();
- cindex++;
- }
- }
- }
- }
-}
-
-
-template <class ObjectType, class FuncType>
-void CLOOP_K2(ObjectType* obj, FuncType expression)
-{
- TIMEZONE("CLOOP_K2");
- #pragma omp parallel
- {
- const hsize_t start = OmpUtils::ForIntervalStart(obj->cd->subsizes[1]);
- const hsize_t end = OmpUtils::ForIntervalEnd(obj->cd->subsizes[1]);
- for (ptrdiff_t yindex = 0; yindex < obj->cd->subsizes[0]; yindex++){
- for (ptrdiff_t zindex = start; zindex < ptrdiff_t(end); zindex++){
- ptrdiff_t cindex = yindex*obj->cd->subsizes[1]*obj->cd->subsizes[2]
- + zindex*obj->cd->subsizes[2];
- for (ptrdiff_t xindex = 0; xindex < obj->cd->subsizes[2]; xindex++)
- {
- double k2 = (obj->kx[xindex]*obj->kx[xindex] +
- obj->ky[yindex]*obj->ky[yindex] +
- obj->kz[zindex]*obj->kz[zindex]);
- expression(cindex, xindex, yindex, zindex, k2);
- cindex++;
- }
- }
- }
- }
-}
-
-
-template <class ObjectType, class FuncType>
-void CLOOP_K2_NXMODES(ObjectType* obj, FuncType expression)
-{
- #pragma omp parallel
- {
- const hsize_t start = OmpUtils::ForIntervalStart(obj->cd->subsizes[1]);
- const hsize_t end = OmpUtils::ForIntervalEnd(obj->cd->subsizes[1]);
- for (ptrdiff_t yindex = 0; yindex < obj->cd->subsizes[0]; yindex++){
- for (ptrdiff_t zindex = start; zindex < ptrdiff_t(end); zindex++)
- {
- ptrdiff_t cindex = yindex*obj->cd->subsizes[1]*obj->cd->subsizes[2]
- + zindex*obj->cd->subsizes[2];
- int nxmodes = 1;
- ptrdiff_t xindex = 0;
- double k2 = (obj->kx[xindex]*obj->kx[xindex] +
- obj->ky[yindex]*obj->ky[yindex] +
- obj->kz[zindex]*obj->kz[zindex]);
- expression(cindex, xindex, yindex, zindex, k2, nxmodes);
- cindex++;
- nxmodes = 2;
- for (xindex = 1; xindex < obj->cd->subsizes[2]; xindex++)
- {
- double k2 = (obj->kx[xindex]*obj->kx[xindex] +
- obj->ky[yindex]*obj->ky[yindex] +
- obj->kz[zindex]*obj->kz[zindex]);
- expression(cindex, xindex, yindex, zindex, k2, nxmodes);
- cindex++;
- }
- }
- }
- }
-}
-
-
-template <class ObjectType, class FuncType>
-void RLOOP(ObjectType* obj, FuncType expression)
-{
- #pragma omp parallel
- {
- const hsize_t start = OmpUtils::ForIntervalStart(obj->rd->subsizes[1]);
- const hsize_t end = OmpUtils::ForIntervalEnd(obj->rd->subsizes[1]);
- for (int zindex = 0; zindex < obj->rd->subsizes[0] ; zindex++)
- for (int yindex = start; yindex < ptrdiff_t(end); yindex++)
- {
- ptrdiff_t rindex = (zindex * obj->rd->subsizes[1] + yindex)*(obj->rd->subsizes[2]+2);
- for (int xindex = 0; xindex < obj->rd->subsizes[2]; xindex++)
- {
- expression(rindex, xindex, yindex, zindex);
- rindex++;
- }
- }
- }
-}
-
-/*****************************************************************************/
-
-#endif//FLUID_SOLVER_BASE
-
diff --git a/bfps/cpp/full_code/NSVE.cpp b/bfps/cpp/full_code/NSVE.cpp
deleted file mode 100644
index 1e24c7af531e7184f75b1f14257d42b822db7a9c..0000000000000000000000000000000000000000
--- a/bfps/cpp/full_code/NSVE.cpp
+++ /dev/null
@@ -1,139 +0,0 @@
-#include <string>
-#include <cmath>
-#include "NSVE.hpp"
-#include "scope_timer.hpp"
-
-
-template <typename rnumber>
-int NSVE<rnumber>::initialize(void)
-{
- this->read_iteration();
- this->read_parameters();
- if (this->myrank == 0)
- {
- // set caching parameters
- hid_t fapl = H5Pcreate(H5P_FILE_ACCESS);
- herr_t cache_err = H5Pset_cache(fapl, 0, 521, 134217728, 1.0);
- DEBUG_MSG("when setting stat_file cache I got %d\n", cache_err);
- this->stat_file = H5Fopen(
- (this->simname + ".h5").c_str(),
- H5F_ACC_RDWR,
- fapl);
- }
- int data_file_problem;
- if (this->myrank == 0)
- data_file_problem = this->grow_file_datasets();
- MPI_Bcast(&data_file_problem, 1, MPI_INT, 0, this->comm);
- if (data_file_problem > 0)
- {
- std::cerr <<
- data_file_problem <<
- " problems growing file datasets.\ntrying to exit now." <<
- std::endl;
- return EXIT_FAILURE;
- }
- this->fs = new vorticity_equation<rnumber, FFTW>(
- simname.c_str(),
- nx, ny, nz,
- dkx, dky, dkz,
- DEFAULT_FFTW_FLAG);
- this->tmp_vec_field = new field<rnumber, FFTW, THREE>(
- nx, ny, nz,
- this->comm,
- DEFAULT_FFTW_FLAG);
-
-
- this->fs->checkpoints_per_file = checkpoints_per_file;
- this->fs->nu = nu;
- this->fs->fmode = fmode;
- this->fs->famplitude = famplitude;
- this->fs->fk0 = fk0;
- this->fs->fk1 = fk1;
- strncpy(this->fs->forcing_type, forcing_type, 128);
- this->fs->iteration = this->iteration;
- this->fs->checkpoint = this->checkpoint;
-
- this->fs->cvorticity->real_space_representation = false;
- this->fs->io_checkpoint();
-
- if (this->myrank == 0 && this->iteration == 0)
- this->fs->kk->store(stat_file);
- return EXIT_SUCCESS;
-}
-
-template <typename rnumber>
-int NSVE<rnumber>::step(void)
-{
- this->fs->step(this->dt);
- this->iteration = this->fs->iteration;
- return EXIT_SUCCESS;
-}
-
-template <typename rnumber>
-int NSVE<rnumber>::write_checkpoint(void)
-{
- this->fs->io_checkpoint(false);
- this->checkpoint = this->fs->checkpoint;
- this->write_iteration();
- return EXIT_SUCCESS;
-}
-
-template <typename rnumber>
-int NSVE<rnumber>::finalize(void)
-{
- if (this->myrank == 0)
- H5Fclose(this->stat_file);
- delete this->fs;
- delete this->tmp_vec_field;
- return EXIT_SUCCESS;
-}
-
-/** \brief Compute standard statistics for velocity and vorticity fields.
- *
- * IMPORTANT: at the end of this subroutine, `this->fs->cvelocity` contains
- * the Fourier space representation of the velocity field, and
- * `this->tmp_vec_field` contains the real space representation of the
- * velocity field.
- * This behavior is relied upon in the `NSVEparticles` class, so please
- * don't break it.
- */
-
-template <typename rnumber>
-int NSVE<rnumber>::do_stats()
-{
- if (!(this->iteration % this->niter_stat == 0))
- return EXIT_SUCCESS;
- hid_t stat_group;
- if (this->myrank == 0)
- stat_group = H5Gopen(
- this->stat_file,
- "statistics",
- H5P_DEFAULT);
- else
- stat_group = 0;
-
- *tmp_vec_field = fs->cvorticity->get_cdata();
- tmp_vec_field->compute_stats(
- fs->kk,
- stat_group,
- "vorticity",
- fs->iteration / niter_stat,
- max_vorticity_estimate/sqrt(3));
-
- fs->compute_velocity(fs->cvorticity);
- *tmp_vec_field = fs->cvelocity->get_cdata();
- tmp_vec_field->compute_stats(
- fs->kk,
- stat_group,
- "velocity",
- fs->iteration / niter_stat,
- max_velocity_estimate/sqrt(3));
-
- if (this->myrank == 0)
- H5Gclose(stat_group);
- return EXIT_SUCCESS;
-}
-
-template class NSVE<float>;
-template class NSVE<double>;
-
diff --git a/bfps/cpp/full_code/NSVE_field_stats.cpp b/bfps/cpp/full_code/NSVE_field_stats.cpp
deleted file mode 100644
index 7e33acf93644208d292c5d8df66653f4bb7b806f..0000000000000000000000000000000000000000
--- a/bfps/cpp/full_code/NSVE_field_stats.cpp
+++ /dev/null
@@ -1,93 +0,0 @@
-#include <string>
-#include <cmath>
-#include "NSVE_field_stats.hpp"
-#include "scope_timer.hpp"
-
-
-template <typename rnumber>
-int NSVE_field_stats<rnumber>::initialize(void)
-{
- this->postprocess::read_parameters();
- this->vorticity = new field<rnumber, FFTW, THREE>(
- nx, ny, nz,
- this->comm,
- DEFAULT_FFTW_FLAG);
- this->vorticity->real_space_representation = false;
- hid_t parameter_file = H5Fopen(
- (this->simname + std::string(".h5")).c_str(),
- H5F_ACC_RDONLY,
- H5P_DEFAULT);
- if (!H5Lexists(parameter_file, "field_dtype", H5P_DEFAULT))
- this->bin_IO = NULL;
- else
- {
- hid_t dset = H5Dopen(parameter_file, "field_dtype", H5P_DEFAULT);
- hid_t space = H5Dget_space(dset);
- hid_t memtype = H5Dget_type(dset);
- char *string_data = (char*)malloc(256);
- H5Dread(dset, memtype, H5S_ALL, H5S_ALL, H5P_DEFAULT, &string_data);
- // check that we're using the correct data type
- // field_dtype SHOULD be something like "<f4", "<f8", ">f4", ">f8"
- // first character is ordering, which is machine specific
- // for the other two I am checking that they have the correct values
- assert(string_data[1] == 'f');
- assert(string_data[2] == '0' + sizeof(rnumber));
- free(string_data);
- H5Sclose(space);
- H5Tclose(memtype);
- H5Dclose(dset);
- this->bin_IO = new field_binary_IO<rnumber, COMPLEX, THREE>(
- this->vorticity->clayout->sizes,
- this->vorticity->clayout->subsizes,
- this->vorticity->clayout->starts,
- this->vorticity->clayout->comm);
- }
- H5Fclose(parameter_file);
- return EXIT_SUCCESS;
-}
-
-template <typename rnumber>
-int NSVE_field_stats<rnumber>::read_current_cvorticity(void)
-{
- this->vorticity->real_space_representation = false;
- if (this->bin_IO != NULL)
- {
- char itername[16];
- sprintf(itername, "i%.5x", this->iteration);
- std::string native_binary_fname = (
- this->simname +
- std::string("_cvorticity_") +
- std::string(itername));
- this->bin_IO->read(
- native_binary_fname,
- this->vorticity->get_cdata());
- }
- else
- {
- this->vorticity->io(
- this->simname + std::string("_fields.h5"),
- "vorticity",
- this->iteration,
- true);
- }
- return EXIT_SUCCESS;
-}
-
-template <typename rnumber>
-int NSVE_field_stats<rnumber>::finalize(void)
-{
- if (this->bin_IO != NULL)
- delete this->bin_IO;
- delete this->vorticity;
- return EXIT_SUCCESS;
-}
-
-template <typename rnumber>
-int NSVE_field_stats<rnumber>::work_on_current_iteration(void)
-{
- return EXIT_SUCCESS;
-}
-
-template class NSVE_field_stats<float>;
-template class NSVE_field_stats<double>;
-
diff --git a/bfps/cpp/full_code/NSVE_no_output.hpp b/bfps/cpp/full_code/NSVE_no_output.hpp
deleted file mode 100644
index 0047a45a02dd58ae8934f78fdd8d804424ae817c..0000000000000000000000000000000000000000
--- a/bfps/cpp/full_code/NSVE_no_output.hpp
+++ /dev/null
@@ -1,25 +0,0 @@
-#ifndef NSVE_NO_OUTPUT_HPP
-#define NSVE_NO_OUTPUT_HPP
-
-#include "full_code/NSVE.hpp"
-
-template <typename rnumber>
-class NSVE_no_output: public NSVE<rnumber>
-{
- public:
- NSVE_no_output(
- const MPI_Comm COMMUNICATOR,
- const std::string &simulation_name):
- NSVE<rnumber>(
- COMMUNICATOR,
- simulation_name){}
- ~NSVE_no_output(){}
- int write_checkpoint(void)
- {
- return 0;
- }
- int read_parameters(void);
-};
-
-#endif//NSVE_NO_OUTPUT_HPP
-
diff --git a/bfps/cpp/full_code/NSVEparticles.cpp b/bfps/cpp/full_code/NSVEparticles.cpp
deleted file mode 100644
index ba84b3943d579965836f05af2447722e273f2dc3..0000000000000000000000000000000000000000
--- a/bfps/cpp/full_code/NSVEparticles.cpp
+++ /dev/null
@@ -1,102 +0,0 @@
-#include <string>
-#include <cmath>
-#include "NSVEparticles.hpp"
-#include "scope_timer.hpp"
-#include "particles/particles_sampling.hpp"
-
-template <typename rnumber>
-int NSVEparticles<rnumber>::initialize(void)
-{
- this->NSVE<rnumber>::initialize();
-
- this->ps = particles_system_builder(
- this->fs->cvelocity, // (field object)
- this->fs->kk, // (kspace object, contains dkx, dky, dkz)
- tracers0_integration_steps, // to check coherency between parameters and hdf input file (nb rhs)
- (long long int)nparticles, // to check coherency between parameters and hdf input file
- this->fs->get_current_fname(), // particles input filename
- std::string("/tracers0/state/") + std::to_string(this->fs->iteration), // dataset name for initial input
- std::string("/tracers0/rhs/") + std::to_string(this->fs->iteration), // dataset name for initial input
- tracers0_neighbours, // parameter (interpolation no neighbours)
- tracers0_smoothness, // parameter
- this->comm,
- this->fs->iteration+1);
- this->particles_output_writer_mpi = new particles_output_hdf5<
- long long int, double, 3, 3>(
- MPI_COMM_WORLD,
- "tracers0",
- nparticles,
- tracers0_integration_steps);
- return EXIT_SUCCESS;
-}
-
-template <typename rnumber>
-int NSVEparticles<rnumber>::step(void)
-{
- this->fs->compute_velocity(this->fs->cvorticity);
- this->fs->cvelocity->ift();
- this->ps->completeLoop(this->dt);
- this->NSVE<rnumber>::step();
- return EXIT_SUCCESS;
-}
-
-template <typename rnumber>
-int NSVEparticles<rnumber>::write_checkpoint(void)
-{
- this->NSVE<rnumber>::write_checkpoint();
- this->particles_output_writer_mpi->open_file(this->fs->get_current_fname());
- this->particles_output_writer_mpi->save(
- this->ps->getParticlesPositions(),
- this->ps->getParticlesRhs(),
- this->ps->getParticlesIndexes(),
- this->ps->getLocalNbParticles(),
- this->fs->iteration);
- this->particles_output_writer_mpi->close_file();
- return EXIT_SUCCESS;
-}
-
-template <typename rnumber>
-int NSVEparticles<rnumber>::finalize(void)
-{
- this->NSVE<rnumber>::finalize();
- this->ps.release();
- delete this->particles_output_writer_mpi;
- return EXIT_SUCCESS;
-}
-
-/** \brief Compute fluid stats and sample fields at particle locations.
- */
-
-template <typename rnumber>
-int NSVEparticles<rnumber>::do_stats()
-{
- /// fluid stats go here
- this->NSVE<rnumber>::do_stats();
-
-
- if (!(this->iteration % this->niter_part == 0))
- return EXIT_SUCCESS;
-
- /// sample velocity
- sample_from_particles_system(*this->tmp_vec_field, // field to save
- this->ps,
- (this->simname + "_particles.h5"), // filename
- "tracers0", // hdf5 parent group
- "velocity" // dataset basename TODO
- );
-
- /// compute acceleration and sample it
- this->fs->compute_Lagrangian_acceleration(this->tmp_vec_field);
- this->tmp_vec_field->ift();
- sample_from_particles_system(*this->tmp_vec_field,
- this->ps,
- (this->simname + "_particles.h5"),
- "tracers0",
- "acceleration");
-
- return EXIT_SUCCESS;
-}
-
-template class NSVEparticles<float>;
-template class NSVEparticles<double>;
-
diff --git a/bfps/cpp/full_code/NSVEparticles_no_output.hpp b/bfps/cpp/full_code/NSVEparticles_no_output.hpp
deleted file mode 100644
index 264fd75ac9b0628aff167d018d888030b7029a35..0000000000000000000000000000000000000000
--- a/bfps/cpp/full_code/NSVEparticles_no_output.hpp
+++ /dev/null
@@ -1,25 +0,0 @@
-#ifndef NSVEPARTICLES_NO_OUTPUT_HPP
-#define NSVEPARTICLES_NO_OUTPUT_HPP
-
-#include "full_code/NSVEparticles.hpp"
-
-template <typename rnumber>
-class NSVEparticles_no_output: public NSVEparticles<rnumber>
-{
- public:
- NSVEparticles_no_output(
- const MPI_Comm COMMUNICATOR,
- const std::string &simulation_name):
- NSVEparticles<rnumber>(
- COMMUNICATOR,
- simulation_name){}
- ~NSVEparticles_no_output(){}
- int write_checkpoint(void)
- {
- return 0;
- }
- int read_parameters(void);
-};
-
-#endif//NSVEPARTICLES_NO_OUTPUT_HPP
-
diff --git a/bfps/cpp/full_code/code_base.cpp b/bfps/cpp/full_code/code_base.cpp
deleted file mode 100644
index 1b06fe8e66a4180034b9f6a494a1a432ae5ea3f9..0000000000000000000000000000000000000000
--- a/bfps/cpp/full_code/code_base.cpp
+++ /dev/null
@@ -1,36 +0,0 @@
-#include "code_base.hpp"
-#include "scope_timer.hpp"
-
-code_base::code_base(
- const MPI_Comm COMMUNICATOR,
- const std::string &simulation_name):
- comm(COMMUNICATOR),
- simname(simulation_name)
-{
- MPI_Comm_rank(this->comm, &this->myrank);
- MPI_Comm_size(this->comm, &this->nprocs);
- this->stop_code_now = false;
-}
-
-int code_base::check_stopping_condition(void)
-{
- if (myrank == 0)
- {
- std::string fname = (
- std::string("stop_") +
- std::string(this->simname));
- {
- struct stat file_buffer;
- this->stop_code_now = (
- stat(fname.c_str(), &file_buffer) == 0);
- }
- }
- MPI_Bcast(
- &this->stop_code_now,
- 1,
- MPI_C_BOOL,
- 0,
- MPI_COMM_WORLD);
- return EXIT_SUCCESS;
-}
-
diff --git a/bfps/cpp/full_code/codes_with_no_output.hpp b/bfps/cpp/full_code/codes_with_no_output.hpp
deleted file mode 100644
index f4cd3b5495ecb432653a7027bcaa330954865d21..0000000000000000000000000000000000000000
--- a/bfps/cpp/full_code/codes_with_no_output.hpp
+++ /dev/null
@@ -1,9 +0,0 @@
-#ifndef CODES_WITH_NO_OUTPUT_HPP
-#define CODES_WITH_NO_OUTPUT_HPP
-
-#include "full_code/NSVE_no_output.hpp"
-#include "full_code/NSVEparticles_no_output.hpp"
-
-
-#endif//CODES_WITH_NO_OUTPUT_HPP
-
diff --git a/bfps/cpp/full_code/get_rfields.cpp b/bfps/cpp/full_code/get_rfields.cpp
deleted file mode 100644
index 0df8b564a61fba11118ef3f551b0a2db6cbfec1d..0000000000000000000000000000000000000000
--- a/bfps/cpp/full_code/get_rfields.cpp
+++ /dev/null
@@ -1,93 +0,0 @@
-#include <string>
-#include <cmath>
-#include "get_rfields.hpp"
-#include "scope_timer.hpp"
-
-
-template <typename rnumber>
-int get_rfields<rnumber>::initialize(void)
-{
- this->NSVE_field_stats<rnumber>::initialize();
- this->kk = new kspace<FFTW, SMOOTH>(
- this->vorticity->clayout, this->dkx, this->dky, this->dkz);
- hid_t parameter_file = H5Fopen(
- (this->simname + std::string(".h5")).c_str(),
- H5F_ACC_RDONLY,
- H5P_DEFAULT);
- hid_t dset = H5Dopen(parameter_file, "/parameters/niter_out", H5P_DEFAULT);
- H5Dread(dset, H5T_NATIVE_INT, H5S_ALL, H5S_ALL, H5P_DEFAULT, &this->niter_out);
- H5Dclose(dset);
- if (H5Lexists(parameter_file, "/parameters/checkpoints_per_file", H5P_DEFAULT))
- {
- dset = H5Dopen(parameter_file, "/parameters/checkpoints_per_file", H5P_DEFAULT);
- H5Dread(dset, H5T_NATIVE_INT, H5S_ALL, H5S_ALL, H5P_DEFAULT, &this->checkpoints_per_file);
- H5Dclose(dset);
- }
- else
- this->checkpoints_per_file = 1;
- this->iteration_list = hdf5_tools::read_vector<int>(
- parameter_file,
- "/get_rfields/iteration_list");
- H5Fclose(parameter_file);
- return EXIT_SUCCESS;
-}
-
-template <typename rnumber>
-int get_rfields<rnumber>::work_on_current_iteration(void)
-{
- DEBUG_MSG("entered get_rfields::work_on_current_iteration\n");
- this->read_current_cvorticity();
- field<rnumber, FFTW, THREE> *vel = new field<rnumber, FFTW, THREE>(
- this->nx, this->ny, this->nz,
- this->comm,
- this->vorticity->fftw_plan_rigor);
-
- vel->real_space_representation = false;
- this->kk->CLOOP_K2(
- [&](ptrdiff_t cindex,
- ptrdiff_t xindex,
- ptrdiff_t yindex,
- ptrdiff_t zindex,
- double k2){
- if (k2 <= this->kk->kM2 && k2 > 0)
- {
- vel->cval(cindex,0,0) = -(this->kk->ky[yindex]*this->vorticity->cval(cindex,2,1) - this->kk->kz[zindex]*this->vorticity->cval(cindex,1,1)) / k2;
- vel->cval(cindex,0,1) = (this->kk->ky[yindex]*this->vorticity->cval(cindex,2,0) - this->kk->kz[zindex]*this->vorticity->cval(cindex,1,0)) / k2;
- vel->cval(cindex,1,0) = -(this->kk->kz[zindex]*this->vorticity->cval(cindex,0,1) - this->kk->kx[xindex]*this->vorticity->cval(cindex,2,1)) / k2;
- vel->cval(cindex,1,1) = (this->kk->kz[zindex]*this->vorticity->cval(cindex,0,0) - this->kk->kx[xindex]*this->vorticity->cval(cindex,2,0)) / k2;
- vel->cval(cindex,2,0) = -(this->kk->kx[xindex]*this->vorticity->cval(cindex,1,1) - this->kk->ky[yindex]*this->vorticity->cval(cindex,0,1)) / k2;
- vel->cval(cindex,2,1) = (this->kk->kx[xindex]*this->vorticity->cval(cindex,1,0) - this->kk->ky[yindex]*this->vorticity->cval(cindex,0,0)) / k2;
- }
- else
- std::fill_n((rnumber*)(vel->get_cdata()+3*cindex), 6, 0.0);
- }
- );
- vel->symmetrize();
- vel->ift();
-
- std::string fname = (
- this->simname +
- std::string("_checkpoint_") +
- std::to_string(this->iteration / (this->niter_out*this->checkpoints_per_file)) +
- std::string(".h5"));
- vel->io(
- fname,
- "velocity",
- this->iteration,
- false);
-
- delete vel;
- return EXIT_SUCCESS;
-}
-
-template <typename rnumber>
-int get_rfields<rnumber>::finalize(void)
-{
- delete this->kk;
- this->NSVE_field_stats<rnumber>::finalize();
- return EXIT_SUCCESS;
-}
-
-template class get_rfields<float>;
-template class get_rfields<double>;
-
diff --git a/bfps/cpp/full_code/native_binary_to_hdf5.cpp b/bfps/cpp/full_code/native_binary_to_hdf5.cpp
deleted file mode 100644
index 7774e2dea9012394c389858038e8ca82674256d7..0000000000000000000000000000000000000000
--- a/bfps/cpp/full_code/native_binary_to_hdf5.cpp
+++ /dev/null
@@ -1,70 +0,0 @@
-#include <string>
-#include <cmath>
-#include "native_binary_to_hdf5.hpp"
-#include "scope_timer.hpp"
-
-
-template <typename rnumber>
-int native_binary_to_hdf5<rnumber>::initialize(void)
-{
- this->read_parameters();
- this->vec_field = new field<rnumber, FFTW, THREE>(
- nx, ny, nz,
- this->comm,
- DEFAULT_FFTW_FLAG);
- this->vec_field->real_space_representation = false;
- this->bin_IO = new field_binary_IO<rnumber, COMPLEX, THREE>(
- this->vec_field->clayout->sizes,
- this->vec_field->clayout->subsizes,
- this->vec_field->clayout->starts,
- this->vec_field->clayout->comm);
- return EXIT_SUCCESS;
-}
-
-template <typename rnumber>
-int native_binary_to_hdf5<rnumber>::work_on_current_iteration(void)
-{
- char itername[16];
- sprintf(itername, "i%.5x", this->iteration);
- std::string native_binary_fname = (
- this->simname +
- std::string("_cvorticity_") +
- std::string(itername));
- this->bin_IO->read(
- native_binary_fname,
- this->vec_field->get_cdata());
- this->vec_field->io(
- (native_binary_fname +
- std::string(".h5")),
- "vorticity",
- this->iteration,
- false);
- return EXIT_SUCCESS;
-}
-
-template <typename rnumber>
-int native_binary_to_hdf5<rnumber>::finalize(void)
-{
- delete this->bin_IO;
- delete this->vec_field;
- return EXIT_SUCCESS;
-}
-
-template <typename rnumber>
-int native_binary_to_hdf5<rnumber>::read_parameters(void)
-{
- this->postprocess::read_parameters();
- hid_t parameter_file = H5Fopen(
- (this->simname + std::string(".h5")).c_str(),
- H5F_ACC_RDONLY,
- H5P_DEFAULT);
- this->iteration_list = hdf5_tools::read_vector<int>(
- parameter_file,
- "/native_binary_to_hdf5/iteration_list");
- H5Fclose(parameter_file);
- return EXIT_SUCCESS;
-}
-
-template class native_binary_to_hdf5<float>;
-template class native_binary_to_hdf5<double>;
-
diff --git a/bfps/cpp/full_code/postprocess.cpp b/bfps/cpp/full_code/postprocess.cpp
deleted file mode 100644
index edb5929f72c5197c123f8f4e20d426ca1ad9eb6f..0000000000000000000000000000000000000000
--- a/bfps/cpp/full_code/postprocess.cpp
+++ /dev/null
@@ -1,94 +0,0 @@
-#include <cstdlib>
-#include <sys/types.h>
-#include <sys/stat.h>
-#include "scope_timer.hpp"
-#include "hdf5_tools.hpp"
-#include "full_code/postprocess.hpp"
-
-
-int postprocess::main_loop(void)
-{
- this->start_simple_timer();
- for (unsigned int iteration_counter = 0;
- iteration_counter < iteration_list.size();
- iteration_counter++)
- {
- this->iteration = iteration_list[iteration_counter];
- #ifdef USE_TIMINGOUTPUT
- const std::string loopLabel = ("postprocess::main_loop-" +
- std::to_string(this->iteration));
- TIMEZONE(loopLabel.c_str());
- #endif
- this->work_on_current_iteration();
- this->print_simple_timer(
- "iteration " + std::to_string(this->iteration));
-
- this->check_stopping_condition();
- if (this->stop_code_now)
- break;
- }
- return EXIT_SUCCESS;
-}
-
-
-int postprocess::read_parameters()
-{
- hid_t parameter_file;
- hid_t dset, memtype, space;
- char fname[256];
- char *string_data;
- sprintf(fname, "%s.h5", this->simname.c_str());
- parameter_file = H5Fopen(fname, H5F_ACC_RDONLY, H5P_DEFAULT);
- dset = H5Dopen(parameter_file, "/parameters/dealias_type", H5P_DEFAULT);
- H5Dread(dset, H5T_NATIVE_INT, H5S_ALL, H5S_ALL, H5P_DEFAULT, &this->dealias_type);
- H5Dclose(dset);
- dset = H5Dopen(parameter_file, "/parameters/dkx", H5P_DEFAULT);
- H5Dread(dset, H5T_NATIVE_DOUBLE, H5S_ALL, H5S_ALL, H5P_DEFAULT, &this->dkx);
- H5Dclose(dset);
- dset = H5Dopen(parameter_file, "/parameters/dky", H5P_DEFAULT);
- H5Dread(dset, H5T_NATIVE_DOUBLE, H5S_ALL, H5S_ALL, H5P_DEFAULT, &this->dky);
- H5Dclose(dset);
- dset = H5Dopen(parameter_file, "/parameters/dkz", H5P_DEFAULT);
- H5Dread(dset, H5T_NATIVE_DOUBLE, H5S_ALL, H5S_ALL, H5P_DEFAULT, &this->dkz);
- H5Dclose(dset);
- dset = H5Dopen(parameter_file, "/parameters/dt", H5P_DEFAULT);
- H5Dread(dset, H5T_NATIVE_DOUBLE, H5S_ALL, H5S_ALL, H5P_DEFAULT, &this->dt);
- H5Dclose(dset);
- dset = H5Dopen(parameter_file, "/parameters/famplitude", H5P_DEFAULT);
- H5Dread(dset, H5T_NATIVE_DOUBLE, H5S_ALL, H5S_ALL, H5P_DEFAULT, &this->famplitude);
- H5Dclose(dset);
- dset = H5Dopen(parameter_file, "/parameters/fk0", H5P_DEFAULT);
- H5Dread(dset, H5T_NATIVE_DOUBLE, H5S_ALL, H5S_ALL, H5P_DEFAULT, &this->fk0);
- H5Dclose(dset);
- dset = H5Dopen(parameter_file, "/parameters/fk1", H5P_DEFAULT);
- H5Dread(dset, H5T_NATIVE_DOUBLE, H5S_ALL, H5S_ALL, H5P_DEFAULT, &this->fk1);
- H5Dclose(dset);
- dset = H5Dopen(parameter_file, "/parameters/fmode", H5P_DEFAULT);
- H5Dread(dset, H5T_NATIVE_INT, H5S_ALL, H5S_ALL, H5P_DEFAULT, &this->fmode);
- H5Dclose(dset);
- dset = H5Dopen(parameter_file, "/parameters/forcing_type", H5P_DEFAULT);
- space = H5Dget_space(dset);
- memtype = H5Dget_type(dset);
- string_data = (char*)malloc(256);
- H5Dread(dset, memtype, H5S_ALL, H5S_ALL, H5P_DEFAULT, &string_data);
- sprintf(this->forcing_type, "%s", string_data);
- free(string_data);
- H5Sclose(space);
- H5Tclose(memtype);
- H5Dclose(dset);
- dset = H5Dopen(parameter_file, "/parameters/nu", H5P_DEFAULT);
- H5Dread(dset, H5T_NATIVE_DOUBLE, H5S_ALL, H5S_ALL, H5P_DEFAULT, &this->nu);
- H5Dclose(dset);
- dset = H5Dopen(parameter_file, "/parameters/nx", H5P_DEFAULT);
- H5Dread(dset, H5T_NATIVE_INT, H5S_ALL, H5S_ALL, H5P_DEFAULT, &this->nx);
- H5Dclose(dset);
- dset = H5Dopen(parameter_file, "/parameters/ny", H5P_DEFAULT);
- H5Dread(dset, H5T_NATIVE_INT, H5S_ALL, H5S_ALL, H5P_DEFAULT, &this->ny);
- H5Dclose(dset);
- dset = H5Dopen(parameter_file, "/parameters/nz", H5P_DEFAULT);
- H5Dread(dset, H5T_NATIVE_INT, H5S_ALL, H5S_ALL, H5P_DEFAULT, &this->nz);
- H5Dclose(dset);
- H5Fclose(parameter_file);
- return 0;
-}
-
diff --git a/bfps/cpp/full_code/test.cpp b/bfps/cpp/full_code/test.cpp
deleted file mode 100644
index 4f7a402c44c2a2999975881929c2582107897c5c..0000000000000000000000000000000000000000
--- a/bfps/cpp/full_code/test.cpp
+++ /dev/null
@@ -1,54 +0,0 @@
-#include <cstdlib>
-#include <sys/types.h>
-#include <sys/stat.h>
-#include "scope_timer.hpp"
-#include "hdf5_tools.hpp"
-#include "full_code/test.hpp"
-
-
-int test::main_loop(void)
-{
- #ifdef USE_TIMINGOUTPUT
- TIMEZONE("test::main_loop");
- #endif
- this->start_simple_timer();
- this->do_work();
- this->print_simple_timer(
- "do_work required " + std::to_string(this->iteration));
- return EXIT_SUCCESS;
-}
-
-
-int test::read_parameters()
-{
- hid_t parameter_file;
- hid_t dset, memtype, space;
- char fname[256];
- char *string_data;
- sprintf(fname, "%s.h5", this->simname.c_str());
- parameter_file = H5Fopen(fname, H5F_ACC_RDONLY, H5P_DEFAULT);
- dset = H5Dopen(parameter_file, "/parameters/dealias_type", H5P_DEFAULT);
- H5Dread(dset, H5T_NATIVE_INT, H5S_ALL, H5S_ALL, H5P_DEFAULT, &this->dealias_type);
- H5Dclose(dset);
- dset = H5Dopen(parameter_file, "/parameters/dkx", H5P_DEFAULT);
- H5Dread(dset, H5T_NATIVE_DOUBLE, H5S_ALL, H5S_ALL, H5P_DEFAULT, &this->dkx);
- H5Dclose(dset);
- dset = H5Dopen(parameter_file, "/parameters/dky", H5P_DEFAULT);
- H5Dread(dset, H5T_NATIVE_DOUBLE, H5S_ALL, H5S_ALL, H5P_DEFAULT, &this->dky);
- H5Dclose(dset);
- dset = H5Dopen(parameter_file, "/parameters/dkz", H5P_DEFAULT);
- H5Dread(dset, H5T_NATIVE_DOUBLE, H5S_ALL, H5S_ALL, H5P_DEFAULT, &this->dkz);
- H5Dclose(dset);
- dset = H5Dopen(parameter_file, "/parameters/nx", H5P_DEFAULT);
- H5Dread(dset, H5T_NATIVE_INT, H5S_ALL, H5S_ALL, H5P_DEFAULT, &this->nx);
- H5Dclose(dset);
- dset = H5Dopen(parameter_file, "/parameters/ny", H5P_DEFAULT);
- H5Dread(dset, H5T_NATIVE_INT, H5S_ALL, H5S_ALL, H5P_DEFAULT, &this->ny);
- H5Dclose(dset);
- dset = H5Dopen(parameter_file, "/parameters/nz", H5P_DEFAULT);
- H5Dread(dset, H5T_NATIVE_INT, H5S_ALL, H5S_ALL, H5P_DEFAULT, &this->nz);
- H5Dclose(dset);
- H5Fclose(parameter_file);
- return 0;
-}
-
diff --git a/bfps/cpp/hdf5_tools.cpp b/bfps/cpp/hdf5_tools.cpp
deleted file mode 100644
index 4328b28703ac60de7e82e4e3e729134ee3ff1520..0000000000000000000000000000000000000000
--- a/bfps/cpp/hdf5_tools.cpp
+++ /dev/null
@@ -1,216 +0,0 @@
-#include "hdf5_tools.hpp"
-
-int hdf5_tools::require_size_single_dataset(hid_t dset, int tsize)
-{
- int ndims;
- hsize_t space;
- space = H5Dget_space(dset);
- ndims = H5Sget_simple_extent_ndims(space);
- hsize_t *dims = new hsize_t[ndims];
- hsize_t *maxdims = new hsize_t[ndims];
- H5Sget_simple_extent_dims(space, dims, maxdims);
- if (dims[0] < hsize_t(tsize) && maxdims[0] == H5S_UNLIMITED)
- {
- dims[0] = tsize;
- H5Dset_extent(dset, dims);
- }
- H5Sclose(space);
- delete[] maxdims;
- delete[] dims;
- return EXIT_SUCCESS;
-}
-
-int hdf5_tools::grow_single_dataset(hid_t dset, int tincrement)
-{
- int ndims;
- hsize_t space;
- space = H5Dget_space(dset);
- ndims = H5Sget_simple_extent_ndims(space);
- hsize_t *dims = new hsize_t[ndims];
- hsize_t *maxdims = new hsize_t[ndims];
- H5Sget_simple_extent_dims(space, dims, maxdims);
- if (maxdims[0] == H5S_UNLIMITED)
- {
- dims[0] += tincrement;
- H5Dset_extent(dset, dims);
- }
- H5Sclose(space);
- delete[] maxdims;
- delete[] dims;
- return EXIT_SUCCESS;
-}
-
-herr_t hdf5_tools::require_size_dataset_visitor(
- hid_t o_id,
- const char *name,
- const H5O_info_t *info,
- void *op_data)
-{
- if (info->type == H5O_TYPE_DATASET)
- {
- hsize_t dset = H5Dopen(o_id, name, H5P_DEFAULT);
- require_size_single_dataset(dset, *((int*)(op_data)));
- H5Dclose(dset);
- }
- return EXIT_SUCCESS;
-}
-
-herr_t hdf5_tools::grow_dataset_visitor(
- hid_t o_id,
- const char *name,
- const H5O_info_t *info,
- void *op_data)
-{
- if (info->type == H5O_TYPE_DATASET)
- {
- hsize_t dset = H5Dopen(o_id, name, H5P_DEFAULT);
- grow_single_dataset(dset, *((int*)(op_data)));
- H5Dclose(dset);
- }
- return EXIT_SUCCESS;
-}
-
-
-int hdf5_tools::grow_file_datasets(
- const hid_t stat_file,
- const std::string group_name,
- int tincrement)
-{
- int file_problems = 0;
-
- hid_t group;
- group = H5Gopen(stat_file, group_name.c_str(), H5P_DEFAULT);
- H5Ovisit(
- group,
- H5_INDEX_NAME,
- H5_ITER_NATIVE,
- grow_dataset_visitor,
- &tincrement);
- H5Gclose(group);
- return file_problems;
-}
-
-
-int hdf5_tools::require_size_file_datasets(
- const hid_t stat_file,
- const std::string group_name,
- int tsize)
-{
- int file_problems = 0;
-
- hid_t group;
- group = H5Gopen(stat_file, group_name.c_str(), H5P_DEFAULT);
- H5Ovisit(
- group,
- H5_INDEX_NAME,
- H5_ITER_NATIVE,
- require_size_dataset_visitor,
- &tsize);
- H5Gclose(group);
- return file_problems;
-}
-
-template <typename number>
-std::vector<number> hdf5_tools::read_vector(
- const hid_t group,
- const std::string dset_name)
-{
- std::vector<number> result;
- hsize_t vector_length;
- // first, read size of array
- hid_t dset, dspace;
- hid_t mem_dtype;
- if (typeid(number) == typeid(int))
- mem_dtype = H5Tcopy(H5T_NATIVE_INT);
- else if (typeid(number) == typeid(double))
- mem_dtype = H5Tcopy(H5T_NATIVE_DOUBLE);
- dset = H5Dopen(group, dset_name.c_str(), H5P_DEFAULT);
- dspace = H5Dget_space(dset);
- assert(H5Sget_simple_extent_ndims(dspace) == 1);
- H5Sget_simple_extent_dims(dspace, &vector_length, NULL);
- result.resize(vector_length);
- H5Dread(dset, mem_dtype, H5S_ALL, H5S_ALL, H5P_DEFAULT, &result.front());
- H5Sclose(dspace);
- H5Dclose(dset);
- H5Tclose(mem_dtype);
- return result;
-}
-
-template <typename dtype>
-std::vector<dtype> hdf5_tools::read_vector_with_single_rank(
- const int myrank,
- const int rank_to_use,
- const MPI_Comm COMM,
- const hid_t file_id,
- const std::string dset_name)
-{
- std::vector<dtype> data;
- int vector_size;
- if (myrank == rank_to_use)
- {
- data = hdf5_tools::read_vector<dtype>(
- file_id,
- dset_name);
- vector_size = data.size();
- }
- MPI_Bcast(
- &vector_size,
- 1,
- MPI_INT,
- rank_to_use,
- COMM);
-
- if (myrank != rank_to_use)
- data.resize(vector_size);
- MPI_Bcast(
- &data.front(),
- vector_size,
- (typeid(dtype) == typeid(int)) ? MPI_INT : MPI_DOUBLE,
- rank_to_use,
- COMM);
- return data;
-}
-
-std::string hdf5_tools::read_string(
- const hid_t group,
- const std::string dset_name)
-{
- hid_t dset = H5Dopen(group, dset_name.c_str(), H5P_DEFAULT);
- hid_t space = H5Dget_space(dset);
- hid_t memtype = H5Dget_type(dset);
- char *string_data = (char*)malloc(256);
- H5Dread(dset, memtype, H5S_ALL, H5S_ALL, H5P_DEFAULT, &string_data);
- std::string std_string_data = std::string(string_data);
- free(string_data);
- H5Sclose(space);
- H5Tclose(memtype);
- H5Dclose(dset);
- return std_string_data;
-}
-
-template
-std::vector<int> hdf5_tools::read_vector<int>(
- const hid_t,
- const std::string);
-
-template
-std::vector<double> hdf5_tools::read_vector<double>(
- const hid_t,
- const std::string);
-
-template
-std::vector<int> hdf5_tools::read_vector_with_single_rank<int>(
- const int myrank,
- const int rank_to_use,
- const MPI_Comm COMM,
- const hid_t file_id,
- const std::string dset_name);
-
-template
-std::vector<double> hdf5_tools::read_vector_with_single_rank<double>(
- const int myrank,
- const int rank_to_use,
- const MPI_Comm COMM,
- const hid_t file_id,
- const std::string dset_name);
-
diff --git a/bfps/cpp/interpolator.cpp b/bfps/cpp/interpolator.cpp
deleted file mode 100644
index a0b38c4059585cc7fd58ab830b792be4f8bc193d..0000000000000000000000000000000000000000
--- a/bfps/cpp/interpolator.cpp
+++ /dev/null
@@ -1,214 +0,0 @@
-/**********************************************************************
-* *
-* Copyright 2015 Max Planck Institute *
-* for Dynamics and Self-Organization *
-* *
-* This file is part of bfps. *
-* *
-* bfps is free software: you can redistribute it and/or modify *
-* it under the terms of the GNU General Public License as published *
-* by the Free Software Foundation, either version 3 of the License, *
-* or (at your option) any later version. *
-* *
-* bfps is distributed in the hope that it will be useful, *
-* but WITHOUT ANY WARRANTY; without even the implied warranty of *
-* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the *
-* GNU General Public License for more details. *
-* *
-* You should have received a copy of the GNU General Public License *
-* along with bfps. If not, see <http://www.gnu.org/licenses/> *
-* *
-* Contact: Cristian.Lalescu@ds.mpg.de *
-* *
-**********************************************************************/
-
-
-
-#define NDEBUG
-
-#include "interpolator.hpp"
-
-template <class rnumber, int interp_neighbours>
-interpolator<rnumber, interp_neighbours>::interpolator(
- fluid_solver_base<rnumber> *fs,
- base_polynomial_values BETA_POLYS,
- ...) : interpolator_base<rnumber, interp_neighbours>(fs, BETA_POLYS)
-{
- int tdims[4];
- this->compute_beta = BETA_POLYS;
- tdims[0] = (interp_neighbours+1)*2*this->descriptor->nprocs + this->descriptor->sizes[0];
- tdims[1] = this->descriptor->sizes[1];
- tdims[2] = this->descriptor->sizes[2]+2;
- tdims[3] = this->descriptor->sizes[3];
- this->buffered_descriptor = new field_descriptor<rnumber>(
- 4, tdims,
- this->descriptor->mpi_dtype,
- this->descriptor->comm);
- this->buffer_size = (interp_neighbours+1)*this->buffered_descriptor->slice_size;
- this->field = new rnumber[this->buffered_descriptor->local_size];
-}
-
-template <class rnumber, int interp_neighbours>
-interpolator<rnumber, interp_neighbours>::~interpolator()
-{
- delete[] this->field;
- delete this->buffered_descriptor;
-}
-
-template <class rnumber, int interp_neighbours>
-int interpolator<rnumber, interp_neighbours>::read_rFFTW(const void *void_src)
-{
- rnumber *src = (rnumber*)void_src;
- rnumber *dst = this->field;
- /* do big copy of middle stuff */
- std::copy(src,
- src + this->buffered_descriptor->slice_size*this->descriptor->subsizes[0],
- dst + this->buffer_size);
- MPI_Datatype MPI_RNUM = (sizeof(rnumber) == 4) ? MPI_FLOAT : MPI_DOUBLE;
- int rsrc;
- /* get upper slices */
- for (int rdst = 0; rdst < this->descriptor->nprocs; rdst++)
- {
- rsrc = this->descriptor->rank[(this->descriptor->all_start0[rdst] +
- this->descriptor->all_size0[rdst]) %
- this->descriptor->sizes[0]];
- if (this->descriptor->myrank == rsrc)
- MPI_Send(
- src,
- this->buffer_size,
- MPI_RNUM,
- rdst,
- 2*(rsrc*this->descriptor->nprocs + rdst),
- this->buffered_descriptor->comm);
- if (this->descriptor->myrank == rdst)
- MPI_Recv(
- dst + this->buffer_size + this->buffered_descriptor->slice_size*this->descriptor->subsizes[0],
- this->buffer_size,
- MPI_RNUM,
- rsrc,
- 2*(rsrc*this->descriptor->nprocs + rdst),
- this->buffered_descriptor->comm,
- MPI_STATUS_IGNORE);
- }
- /* get lower slices */
- for (int rdst = 0; rdst < this->descriptor->nprocs; rdst++)
- {
- rsrc = this->descriptor->rank[MOD(this->descriptor->all_start0[rdst] - 1,
- this->descriptor->sizes[0])];
- if (this->descriptor->myrank == rsrc)
- MPI_Send(
- src + this->buffered_descriptor->slice_size*this->descriptor->subsizes[0] - this->buffer_size,
- this->buffer_size,
- MPI_RNUM,
- rdst,
- 2*(rsrc*this->descriptor->nprocs + rdst)+1,
- this->descriptor->comm);
- if (this->descriptor->myrank == rdst)
- MPI_Recv(
- dst,
- this->buffer_size,
- MPI_RNUM,
- rsrc,
- 2*(rsrc*this->descriptor->nprocs + rdst)+1,
- this->descriptor->comm,
- MPI_STATUS_IGNORE);
- }
- return EXIT_SUCCESS;
-}
-
-template <class rnumber, int interp_neighbours>
-void interpolator<rnumber, interp_neighbours>::sample(
- const int nparticles,
- const int pdimension,
- const double *__restrict__ x,
- double *__restrict__ y,
- const int *deriv)
-{
- /* get grid coordinates */
- int *xg = new int[3*nparticles];
- double *xx = new double[3*nparticles];
- double *yy = new double[3*nparticles];
- std::fill_n(yy, 3*nparticles, 0.0);
- this->get_grid_coordinates(nparticles, pdimension, x, xg, xx);
- /* perform interpolation */
- for (int p=0; p<nparticles; p++)
- if (this->descriptor->rank[MOD(xg[p*3+2], this->descriptor->sizes[0])] == this->descriptor->myrank)
- this->operator()(xg + p*3, xx + p*3, yy + p*3, deriv);
- MPI_Allreduce(
- yy,
- y,
- 3*nparticles,
- MPI_DOUBLE,
- MPI_SUM,
- this->descriptor->comm);
- delete[] yy;
- delete[] xg;
- delete[] xx;
-}
-
-template <class rnumber, int interp_neighbours>
-void interpolator<rnumber, interp_neighbours>::operator()(
- const int *xg,
- const double *xx,
- double *__restrict__ dest,
- const int *deriv)
-{
- double bx[interp_neighbours*2+2], by[interp_neighbours*2+2], bz[interp_neighbours*2+2];
- if (deriv == NULL)
- {
- this->compute_beta(0, xx[0], bx);
- this->compute_beta(0, xx[1], by);
- this->compute_beta(0, xx[2], bz);
- }
- else
- {
- this->compute_beta(deriv[0], xx[0], bx);
- this->compute_beta(deriv[1], xx[1], by);
- this->compute_beta(deriv[2], xx[2], bz);
- }
- std::fill_n(dest, 3, 0);
- ptrdiff_t bigiz, bigiy, bigix;
- for (int iz = -interp_neighbours; iz <= interp_neighbours+1; iz++)
- {
- bigiz = ptrdiff_t(xg[2]+iz)-this->descriptor->starts[0];
- for (int iy = -interp_neighbours; iy <= interp_neighbours+1; iy++)
- {
- bigiy = ptrdiff_t(MOD(xg[1]+iy, this->descriptor->sizes[1]));
- for (int ix = -interp_neighbours; ix <= interp_neighbours+1; ix++)
- {
- bigix = ptrdiff_t(MOD(xg[0]+ix, this->descriptor->sizes[2]));
- ptrdiff_t tindex = ((bigiz *this->buffered_descriptor->sizes[1] +
- bigiy)*this->buffered_descriptor->sizes[2] +
- bigix)*3 + this->buffer_size;
- for (int c=0; c<3; c++)
- {
- dest[c] += this->field[tindex+c]*(bz[iz+interp_neighbours]*
- by[iy+interp_neighbours]*
- bx[ix+interp_neighbours]);
- }
- }
- }
- }
-}
-
-template class interpolator<float, 1>;
-template class interpolator<float, 2>;
-template class interpolator<float, 3>;
-template class interpolator<float, 4>;
-template class interpolator<float, 5>;
-template class interpolator<float, 6>;
-template class interpolator<float, 7>;
-template class interpolator<float, 8>;
-template class interpolator<float, 9>;
-template class interpolator<float, 10>;
-template class interpolator<double, 1>;
-template class interpolator<double, 2>;
-template class interpolator<double, 3>;
-template class interpolator<double, 4>;
-template class interpolator<double, 5>;
-template class interpolator<double, 6>;
-template class interpolator<double, 7>;
-template class interpolator<double, 8>;
-template class interpolator<double, 9>;
-template class interpolator<double, 10>;
-
diff --git a/bfps/cpp/interpolator.hpp b/bfps/cpp/interpolator.hpp
deleted file mode 100644
index 7e56ebe159fd24ed7cf623f0a869e1d262d4aadb..0000000000000000000000000000000000000000
--- a/bfps/cpp/interpolator.hpp
+++ /dev/null
@@ -1,79 +0,0 @@
-/**********************************************************************
-* *
-* Copyright 2015 Max Planck Institute *
-* for Dynamics and Self-Organization *
-* *
-* This file is part of bfps. *
-* *
-* bfps is free software: you can redistribute it and/or modify *
-* it under the terms of the GNU General Public License as published *
-* by the Free Software Foundation, either version 3 of the License, *
-* or (at your option) any later version. *
-* *
-* bfps is distributed in the hope that it will be useful, *
-* but WITHOUT ANY WARRANTY; without even the implied warranty of *
-* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the *
-* GNU General Public License for more details. *
-* *
-* You should have received a copy of the GNU General Public License *
-* along with bfps. If not, see <http://www.gnu.org/licenses/> *
-* *
-* Contact: Cristian.Lalescu@ds.mpg.de *
-* *
-**********************************************************************/
-
-
-
-#include <cmath>
-#include "field_descriptor.hpp"
-#include "fftw_tools.hpp"
-#include "fluid_solver_base.hpp"
-#include "interpolator_base.hpp"
-
-#ifndef INTERPOLATOR
-
-#define INTERPOLATOR
-
-template <class rnumber, int interp_neighbours>
-class interpolator:public interpolator_base<rnumber, interp_neighbours>
-{
- private:
- /* pointer to buffered field */
- rnumber *field;
-
- public:
- using interpolator_base<rnumber, interp_neighbours>::operator();
- ptrdiff_t buffer_size;
-
- /* descriptor for buffered field */
- field_descriptor<rnumber> *buffered_descriptor;
-
- interpolator(
- fluid_solver_base<rnumber> *FSOLVER,
- base_polynomial_values BETA_POLYS,
- ...);
- ~interpolator();
-
- int read_rFFTW(const void *src);
-
- inline int get_rank(double z)
- {
- return this->descriptor->rank[MOD(int(floor(z/this->dz)), this->descriptor->sizes[0])];
- }
-
- /* interpolate field at an array of locations */
- void sample(
- const int nparticles,
- const int pdimension,
- const double *__restrict__ x,
- double *__restrict__ y,
- const int *deriv = NULL);
- void operator()(
- const int *__restrict__ xg,
- const double *__restrict__ xx,
- double *__restrict__ dest,
- const int *deriv = NULL);
-};
-
-#endif//INTERPOLATOR
-
diff --git a/bfps/cpp/interpolator_base.cpp b/bfps/cpp/interpolator_base.cpp
deleted file mode 100644
index 668a965c65744ac5aae31afb6bee05711a433657..0000000000000000000000000000000000000000
--- a/bfps/cpp/interpolator_base.cpp
+++ /dev/null
@@ -1,113 +0,0 @@
-/**********************************************************************
-* *
-* Copyright 2015 Max Planck Institute *
-* for Dynamics and Self-Organization *
-* *
-* This file is part of bfps. *
-* *
-* bfps is free software: you can redistribute it and/or modify *
-* it under the terms of the GNU General Public License as published *
-* by the Free Software Foundation, either version 3 of the License, *
-* or (at your option) any later version. *
-* *
-* bfps is distributed in the hope that it will be useful, *
-* but WITHOUT ANY WARRANTY; without even the implied warranty of *
-* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the *
-* GNU General Public License for more details. *
-* *
-* You should have received a copy of the GNU General Public License *
-* along with bfps. If not, see <http://www.gnu.org/licenses/> *
-* *
-* Contact: Cristian.Lalescu@ds.mpg.de *
-* *
-**********************************************************************/
-
-
-
-#define NDEBUG
-
-#include <cmath>
-#include "interpolator_base.hpp"
-
-template <class rnumber, int interp_neighbours>
-interpolator_base<rnumber, interp_neighbours>::interpolator_base(
- fluid_solver_base<rnumber> *fs,
- base_polynomial_values BETA_POLYS)
-{
- this->descriptor = fs->rd;
- this->compute_beta = BETA_POLYS;
-
- // compute dx, dy, dz;
- this->dx = 4*acos(0) / (fs->dkx*this->descriptor->sizes[2]);
- this->dy = 4*acos(0) / (fs->dky*this->descriptor->sizes[1]);
- this->dz = 4*acos(0) / (fs->dkz*this->descriptor->sizes[0]);
-}
-
-template <class rnumber, int interp_neighbours>
-interpolator_base<rnumber, interp_neighbours>::interpolator_base(
- vorticity_equation<rnumber, FFTW> *fs,
- base_polynomial_values BETA_POLYS)
-{
-// this->descriptor = fs->rd;
-// this->compute_beta = BETA_POLYS;
-//
-// // compute dx, dy, dz;
-// this->dx = 4*acos(0) / (fs->kk->dkx*this->descriptor->sizes[2]);
-// this->dy = 4*acos(0) / (fs->kk->dky*this->descriptor->sizes[1]);
-// this->dz = 4*acos(0) / (fs->kk->dkz*this->descriptor->sizes[0]);
-}
-
-template <class rnumber, int interp_neighbours>
-void interpolator_base<rnumber, interp_neighbours>::get_grid_coordinates(
- const int nparticles,
- const int pdimension,
- const double *x,
- int *xg,
- double *xx)
-{
- for (int p=0; p<nparticles; p++)
- this->get_grid_coordinates(
- x + p*pdimension,
- xg + p*3,
- xx + p*3);
-}
-
-template <class rnumber, int interp_neighbours>
-void interpolator_base<rnumber, interp_neighbours>::get_grid_coordinates(
- const double *x,
- int *xg,
- double *xx)
-{
- static double grid_size[] = {this->dx, this->dy, this->dz};
- double tval;
- for (int c=0; c<3; c++)
- {
- tval = floor(x[c]/grid_size[c]);
- xg[c] = MOD(int(tval), this->descriptor->sizes[2-c]);
- xx[c] = (x[c] - tval*grid_size[c]) / grid_size[c];
- }
-}
-
-
-
-template class interpolator_base<float, 1>;
-template class interpolator_base<float, 2>;
-template class interpolator_base<float, 3>;
-template class interpolator_base<float, 4>;
-template class interpolator_base<float, 5>;
-template class interpolator_base<float, 6>;
-template class interpolator_base<float, 7>;
-template class interpolator_base<float, 8>;
-template class interpolator_base<float, 9>;
-template class interpolator_base<float, 10>;
-template class interpolator_base<double, 1>;
-template class interpolator_base<double, 2>;
-template class interpolator_base<double, 3>;
-template class interpolator_base<double, 4>;
-template class interpolator_base<double, 5>;
-template class interpolator_base<double, 6>;
-template class interpolator_base<double, 7>;
-template class interpolator_base<double, 8>;
-template class interpolator_base<double, 9>;
-template class interpolator_base<double, 10>;
-
diff --git a/bfps/cpp/interpolator_base.hpp b/bfps/cpp/interpolator_base.hpp
deleted file mode 100644
index f4c28db7b9de632e8ec4977dd67f929f06080e19..0000000000000000000000000000000000000000
--- a/bfps/cpp/interpolator_base.hpp
+++ /dev/null
@@ -1,114 +0,0 @@
-/**********************************************************************
-* *
-* Copyright 2015 Max Planck Institute *
-* for Dynamics and Self-Organization *
-* *
-* This file is part of bfps. *
-* *
-* bfps is free software: you can redistribute it and/or modify *
-* it under the terms of the GNU General Public License as published *
-* by the Free Software Foundation, either version 3 of the License, *
-* or (at your option) any later version. *
-* *
-* bfps is distributed in the hope that it will be useful, *
-* but WITHOUT ANY WARRANTY; without even the implied warranty of *
-* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the *
-* GNU General Public License for more details. *
-* *
-* You should have received a copy of the GNU General Public License *
-* along with bfps. If not, see <http://www.gnu.org/licenses/> *
-* *
-* Contact: Cristian.Lalescu@ds.mpg.de *
-* *
-**********************************************************************/
-
-
-
-#include "fluid_solver_base.hpp"
-#include "vorticity_equation.hpp"
-#include "spline_n1.hpp"
-#include "spline_n2.hpp"
-#include "spline_n3.hpp"
-#include "spline_n4.hpp"
-#include "spline_n5.hpp"
-#include "spline_n6.hpp"
-#include "spline_n7.hpp"
-#include "spline_n8.hpp"
-#include "spline_n9.hpp"
-#include "spline_n10.hpp"
-#include "Lagrange_polys.hpp"
-
-#ifndef INTERPOLATOR_BASE
-
-#define INTERPOLATOR_BASE
-
-typedef void (*base_polynomial_values)(
- const int derivative,
- const double fraction,
- double *__restrict__ destination);
-
-template <class rnumber, int interp_neighbours>
-class interpolator_base
-{
- public:
- /* pointer to polynomial function */
- base_polynomial_values compute_beta;
-
- /* descriptor of field to interpolate */
- field_descriptor<rnumber> *descriptor;
-
- /* physical parameters of field */
- double dx, dy, dz;
-
- interpolator_base(
- fluid_solver_base<rnumber> *FSOLVER,
- base_polynomial_values BETA_POLYS);
-
- interpolator_base(
- vorticity_equation<rnumber, FFTW> *FSOLVER,
- base_polynomial_values BETA_POLYS);
- virtual ~interpolator_base(){}
-
- /* may not destroy input */
- virtual int read_rFFTW(const void *src) = 0;
-
- /* map real locations to grid coordinates */
- void get_grid_coordinates(
- const int nparticles,
- const int pdimension,
- const double *__restrict__ x,
- int *__restrict__ xg,
- double *__restrict__ xx);
- void get_grid_coordinates(
- const double *__restrict__ x,
- int *__restrict__ xg,
- double *__restrict__ xx);
- /* interpolate field at an array of locations */
- virtual void sample(
- const int nparticles,
- const int pdimension,
- const double *__restrict__ x,
- double *__restrict__ y,
- const int *deriv = NULL) = 0;
- /* interpolate 1 point */
- virtual void operator()(
- const int *__restrict__ xg,
- const double *__restrict__ xx,
- double *__restrict__ dest,
- const int *deriv = NULL) = 0;
-
- /* interpolate 1 point */
- inline void operator()(
- const double *__restrict__ x,
- double *__restrict__ dest,
- const int *deriv = NULL)
- {
- int xg[3];
- double xx[3];
- this->get_grid_coordinates(x, xg, xx);
- (*this)(xg, xx, dest, deriv);
- }
-};
-
-#endif//INTERPOLATOR_BASE
-
diff --git a/bfps/cpp/kspace.cpp b/bfps/cpp/kspace.cpp
deleted file mode 100644
index 01accf4dc1f24d7fe92ee622dd4faf9eaf12a485..0000000000000000000000000000000000000000
--- a/bfps/cpp/kspace.cpp
+++ /dev/null
@@ -1,808 +0,0 @@
-/**********************************************************************
-* *
-* Copyright 2015 Max Planck Institute *
-* for Dynamics and Self-Organization *
-* *
-* This file is part of bfps. *
-* *
-* bfps is free software: you can redistribute it and/or modify *
-* it under the terms of the GNU General Public License as published *
-* by the Free Software Foundation, either version 3 of the License, *
-* or (at your option) any later version. *
-* *
-* bfps is distributed in the hope that it will be useful, *
-* but WITHOUT ANY WARRANTY; without even the implied warranty of *
-* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the *
-* GNU General Public License for more details. *
-* *
-* You should have received a copy of the GNU General Public License *
-* along with bfps. If not, see <http://www.gnu.org/licenses/> *
-* *
-* Contact: Cristian.Lalescu@ds.mpg.de *
-* *
-**********************************************************************/
-
-
-#include <cmath>
-#include <cstdlib>
-#include <algorithm>
-#include <cassert>
-#include "kspace.hpp"
-#include "scope_timer.hpp"
-#include "shared_array.hpp"
-
-template <field_backend be,
- kspace_dealias_type dt>
-template <field_components fc>
-kspace<be, dt>::kspace(
- const field_layout<fc> *source_layout,
- const double DKX,
- const double DKY,
- const double DKZ)
-{
- TIMEZONE("kspace::kspace");
- /* get layout */
- this->layout = new field_layout<ONE>(
- source_layout->sizes,
- source_layout->subsizes,
- source_layout->starts,
- source_layout->comm);
-
- /* store dk values */
- this->dkx = DKX;
- this->dky = DKY;
- this->dkz = DKZ;
-
- /* compute kx, ky, kz and compute kM values */
- switch(be)
- {
- case FFTW:
- this->kx.resize(this->layout->sizes[2]);
- this->ky.resize(this->layout->subsizes[0]);
- this->kz.resize(this->layout->sizes[1]);
- int i, ii;
- for (i = 0; i<int(this->layout->sizes[2]); i++)
- this->kx[i] = i*this->dkx;
- for (i = 0; i<int(this->layout->subsizes[0]); i++)
- {
- ii = i + this->layout->starts[0];
- if (ii <= int(this->layout->sizes[1]/2))
- this->ky[i] = this->dky*ii;
- else
- this->ky[i] = this->dky*(ii - int(this->layout->sizes[1]));
- }
- for (i = 0; i<int(this->layout->sizes[1]); i++)
- {
- if (i <= int(this->layout->sizes[0]/2))
- this->kz[i] = this->dkz*i;
- else
- this->kz[i] = this->dkz*(i - int(this->layout->sizes[0]));
- }
- switch(dt)
- {
- case TWO_THIRDS:
- this->kMx = this->dkx*(int(2*(int(this->layout->sizes[2])-1)/3)-1);
- this->kMy = this->dky*(int(this->layout->sizes[0] / 3)-1);
- this->kMz = this->dkz*(int(this->layout->sizes[1] / 3)-1);
- break;
- case SMOOTH:
- this->kMx = this->dkx*(int(this->layout->sizes[2])-2);
- this->kMy = this->dky*(int(this->layout->sizes[0] / 2)-1);
- this->kMz = this->dkz*(int(this->layout->sizes[1] / 2)-1);
- break;
- }
- break;
- }
-
- /* get global kM and dk */
- this->kM = this->kMx;
- if (this->kM < this->kMy) this->kM = this->kMy;
- if (this->kM < this->kMz) this->kM = this->kMz;
- this->kM2 = this->kM * this->kM;
- this->dk = this->dkx;
- if (this->dk > this->dky) this->dk = this->dky;
- if (this->dk > this->dkz) this->dk = this->dkz;
- this->dk2 = this->dk*this->dk;
-
- /* spectra stuff */
- this->nshells = int(this->kM / this->dk) + 2;
- this->kshell.resize(this->nshells, 0);
- this->nshell.resize(this->nshells, 0);
-
- shared_array<double> kshell_local_thread(this->nshells,[&](double* kshell_local){
- std::fill_n(kshell_local, this->nshells, 0);
- });
- shared_array<int64_t> nshell_local_thread(this->nshells,[&](int64_t* nshell_local){
- std::fill_n(nshell_local, this->nshells, 0);
- });
-
- std::vector<std::unordered_map<int, double>> dealias_filter_threaded(omp_get_max_threads());
-
- this->CLOOP_K2_NXMODES(
- [&](ptrdiff_t cindex,
- ptrdiff_t xindex,
- ptrdiff_t yindex,
- ptrdiff_t zindex,
- double k2,
- int nxmodes){
- if (k2 < this->kM2)
- {
- double knorm = sqrt(k2);
- kshell_local_thread.getMine()[int(knorm/this->dk)] += nxmodes*knorm;
- nshell_local_thread.getMine()[int(knorm/this->dk)] += nxmodes;
- }
- if (dt == SMOOTH){
- dealias_filter_threaded[omp_get_thread_num()][int(round(k2 / this->dk2))] = exp(-36.0 * pow(k2/this->kM2, 18.));
- }
- });
-
- // Merge results
-
- kshell_local_thread.mergeParallel();
- nshell_local_thread.mergeParallel();
-
- if (dt == SMOOTH){
- for(int idxMerge = 0 ; idxMerge < int(dealias_filter_threaded.size()) ; ++idxMerge){
- for(const auto kv : dealias_filter_threaded[idxMerge]){
- this->dealias_filter[kv.first] = kv.second;
- }
- }
- }
-
- MPI_Allreduce(
- nshell_local_thread.getMasterData(),
- &this->nshell.front(),
- this->nshells,
- MPI_INT64_T, MPI_SUM, this->layout->comm);
- MPI_Allreduce(
- kshell_local_thread.getMasterData(),
- &this->kshell.front(),
- this->nshells,
- MPI_DOUBLE, MPI_SUM, this->layout->comm);
- for (int n=0; n<this->nshells; n++){
- if(this->nshell[n] != 0){
- this->kshell[n] /= this->nshell[n];
- }
- }
-}
-
-template <field_backend be,
- kspace_dealias_type dt>
-kspace<be, dt>::~kspace()
-{
- delete this->layout;
-}
-
-template <field_backend be,
- kspace_dealias_type dt>
-int kspace<be, dt>::store(hid_t stat_file)
-{
- TIMEZONE("kspace::store");
- assert(this->layout->myrank == 0);
- hsize_t dims[4];
- hid_t space, dset;
- // store kspace information
- dset = H5Dopen(stat_file, "/kspace/kshell", H5P_DEFAULT);
- space = H5Dget_space(dset);
- H5Sget_simple_extent_dims(space, dims, NULL);
- H5Sclose(space);
- if (this->nshells != int(dims[0]))
- {
- DEBUG_MSG(
- "ERROR: computed nshells %d not equal to data file nshells %d\n",
- this->nshells, dims[0]);
- }
- H5Dwrite(
- dset,
- H5T_NATIVE_DOUBLE,
- H5S_ALL,
- H5S_ALL,
- H5P_DEFAULT,
- &this->kshell.front());
- H5Dclose(dset);
- dset = H5Dopen(
- stat_file,
- "/kspace/nshell",
- H5P_DEFAULT);
- H5Dwrite(
- dset,
- H5T_NATIVE_INT64,
- H5S_ALL,
- H5S_ALL,
- H5P_DEFAULT,
- &this->nshell.front());
- H5Dclose(dset);
- dset = H5Dopen(stat_file, "/kspace/kM", H5P_DEFAULT);
- H5Dwrite(
- dset,
- H5T_NATIVE_DOUBLE,
- H5S_ALL,
- H5S_ALL,
- H5P_DEFAULT,
- &this->kM);
- H5Dclose(dset);
- dset = H5Dopen(stat_file, "/kspace/dk", H5P_DEFAULT);
- H5Dwrite(dset,
- H5T_NATIVE_DOUBLE,
- H5S_ALL,
- H5S_ALL,
- H5P_DEFAULT,
- &this->dk);
- H5Dclose(dset);
- return EXIT_SUCCESS;
-}
-
-template <field_backend be,
- kspace_dealias_type dt>
-template <typename rnumber,
- field_components fc>
-void kspace<be, dt>::low_pass(
- typename fftw_interface<rnumber>::complex *__restrict__ a,
- const double kmax)
-{
- const double km2 = kmax*kmax;
- this->CLOOP_K2(
- [&](ptrdiff_t cindex,
- ptrdiff_t xindex,
- ptrdiff_t yindex,
- ptrdiff_t zindex,
- double k2){
- if (k2 >= km2)
- std::fill_n((rnumber*)(a + ncomp(fc)*cindex), 2*ncomp(fc), 0);
- });
-}
-
-/** \brief Filter a field using a ball shaped top hat filter.
- *
- * Filter's mathematical expression in real space is as follows:
- * \f[
- * \phi^b_\ell(r) =
- * \frac{1}{\ell^3}\frac{6}{\pi} H(\ell/2 - r)
- * \f]
- * with the corresponding Fourier space expression:
- * \f[
- * \hat{\phi^b_\ell}(k) =
- * \frac{3}{2(k\ell/2)^3}
- * \left(2\sin (k \ell/2) - k \ell \cos (k \ell/2)\right)
- * \f]
- */
-template <field_backend be,
- kspace_dealias_type dt>
-template <typename rnumber,
- field_components fc>
-void kspace<be, dt>::ball_filter(
- typename fftw_interface<rnumber>::complex *__restrict__ a,
- const double ell)
-{
- const double prefactor0 = double(3) / pow(ell/2, 3);
- this->CLOOP_K2(
- [&](ptrdiff_t cindex,
- ptrdiff_t xindex,
- ptrdiff_t yindex,
- ptrdiff_t zindex,
- double k2){
- if (k2 > 0)
- {
- double argument = sqrt(k2)*ell / 2;
- double prefactor = prefactor0 / pow(k2, 1.5);
- for (unsigned int tcounter=0; tcounter<2*ncomp(fc); tcounter++)
- ((rnumber*)a)[2*ncomp(fc)*cindex + tcounter] *= (
- prefactor *
- (sin(argument) - argument * cos(argument)));
- }
- });
-}
-
-/** \brief Filter a field using a Gaussian kernel.
- *
- * Filter's mathematical expression in Fourier space is as follows:
- * \f[
- * \hat{g}_\ell(\mathbf{k}) = \exp(-k^2 \sigma^2 / 2)
- * \f]
- */
-template <field_backend be,
- kspace_dealias_type dt>
-template <typename rnumber,
- field_components fc>
-void kspace<be, dt>::Gauss_filter(
- typename fftw_interface<rnumber>::complex *__restrict__ a,
- const double sigma)
-{
- const double prefactor = - sigma*sigma/2;
- this->CLOOP_K2(
- [&](ptrdiff_t cindex,
- ptrdiff_t xindex,
- ptrdiff_t yindex,
- ptrdiff_t zindex,
- double k2){
- {
- for (unsigned int tcounter=0; tcounter<2*ncomp(fc); tcounter++)
- ((rnumber*)a)[2*ncomp(fc)*cindex + tcounter] *= exp(prefactor*k2);
- }
- });
-}
-
-/** \brief Filter a field.
- *
- * This is a wrapper that can choose between a sharp Fourier spherical filter,
- * a Gaussian filter and a sharp real space spherical filter.
- *
- * Filter expressions in real space are as follows:
- * \f{eqnarray*}{
- * \phi^b_\ell(r) &=&
- * \frac{1}{\ell^3}\frac{6}{\pi} H(\ell/2 - r) \\
- * \phi^g_\ell(r) &=&
- * \frac{1}{\sigma_\ell^3}\frac{1}{(2\pi)^{3/2}}
- * \exp\left(-\frac{1}{2}\left(\frac{r}{\sigma_\ell}\right)^2\right) \\
- * \phi^s_\ell(r) &=&
- * \frac{1}{2 \pi^2 r^3}
- * \left(\sin k_\ell r - k_\ell r \cos k_\ell r\right)
- * \f}
- * and the corresponding expressions in Fourier space are:
- * \f{eqnarray*}{
- * \hat{\phi^b_\ell}(k) &=&
- * \frac{3}{2(k\ell/2)^3}
- * \left(2\sin (k \ell/2) - k \ell \cos (k \ell/2)\right) \\
- * \hat{\phi^g_\ell}(k) &=&
- * \exp\left(-\frac{1}{2}k^2 \sigma_\ell^2\right) \\
- * \hat{\phi^s_\ell}(k) &=& H(k_\ell - k)
- * \f}
- *
- * \f$ k_\ell \f$ is given as a parameter, and then we use
- * \f[
- * \ell = \pi / k_\ell,
- * \sigma_\ell = \pi / k_\ell
- * \f]
- *
- * For the Gaussian filter this is the same convention used in
- * \cite Buzzicotti2017 .
- *
- * See also `filter_calibrated_ell`.
- */
-template <field_backend be,
- kspace_dealias_type dt>
-template <typename rnumber,
- field_components fc>
-int kspace<be, dt>::filter(
- typename fftw_interface<rnumber>::complex *__restrict__ a,
- const double wavenumber,
- std::string filter_type)
-{
- if (filter_type == std::string("sharp_Fourier_sphere"))
- {
- this->template low_pass<rnumber, fc>(
- a,
- wavenumber);
- }
- else if (filter_type == std::string("Gauss"))
- {
- this->template Gauss_filter<rnumber, fc>(
- a,
- 2*acos(0.)/wavenumber);
- }
- else if (filter_type == std::string("ball"))
- {
- this->template ball_filter<rnumber, fc>(
- a,
- 2*acos(0.)/wavenumber);
- }
- return EXIT_SUCCESS;
-}
-
-/** \brief Filter a field.
- *
- * This is a wrapper that can choose between a sharp Fourier spherical filter,
- * a Gaussian filter and a sharp real space spherical filter.
- *
- * Filter expressions in real space are as follows:
- * \f{eqnarray*}{
- * \phi^b_\ell(r) &=&
- * \frac{1}{\ell^3}\frac{6}{\pi} H(\ell/2 - r) \\
- * \phi^g_\ell(r) &=&
- * \frac{1}{\sigma_\ell^3}\frac{1}{(2\pi)^{3/2}}
- * \exp\left(-\frac{1}{2}\left(\frac{r}{\sigma_\ell}\right)^2\right) \\
- * \phi^s_\ell(r) &=&
- * \frac{1}{2 \pi^2 r^3}
- * \left(\sin k_\ell r - k_\ell r \cos k_\ell r\right)
- * \f}
- * and the corresponding expressions in Fourier space are:
- * \f{eqnarray*}{
- * \hat{\phi^b_\ell}(k) &=&
- * \frac{3}{2(k\ell/2)^3}
- * \left(2\sin (k \ell/2) - k \ell \cos (k \ell/2)\right) \\
- * \hat{\phi^g_\ell}(k) &=&
- * \exp\left(-\frac{1}{2}k^2 \sigma_\ell^2\right) \\
- * \hat{\phi^s_\ell}(k) &=& H(k_\ell - k)
- * \f}
- *
- * \f$\sigma_\ell\f$ and \f$k_\ell\f$ are calibrated such that the energy of
- * the large scales is approximately the same (within the inertial range)
- * independently of the shape of the filter.
- *
- * This was done by hand, see [INSERT CITATION HERE] for details, with the
- * results:
- *
- * \f[
- * \sigma_\ell = 0.23 \ell,
- * k_\ell = 2.8 / \ell
- * \f]
- *
- */
-template <field_backend be,
- kspace_dealias_type dt>
-template <typename rnumber,
- field_components fc>
-int kspace<be, dt>::filter_calibrated_ell(
- typename fftw_interface<rnumber>::complex *__restrict__ a,
- const double ell,
- std::string filter_type)
-{
- if (filter_type == std::string("sharp_Fourier_sphere"))
- {
- this->template low_pass<rnumber, fc>(
- a,
- 2.8 / ell);
- }
- else if (filter_type == std::string("Gauss"))
- {
- this->template Gauss_filter<rnumber, fc>(
- a,
- 0.23*ell);
- }
- else if (filter_type == std::string("ball"))
- {
- this->template ball_filter<rnumber, fc>(
- a,
- ell);
- }
- return EXIT_SUCCESS;
-}
-
-template <field_backend be,
- kspace_dealias_type dt>
-template <typename rnumber,
- field_components fc>
-void kspace<be, dt>::dealias(typename fftw_interface<rnumber>::complex *__restrict__ a)
-{
- switch(dt)
- {
- case TWO_THIRDS:
- this->low_pass<rnumber, fc>(a, this->kM);
- break;
- case SMOOTH:
- this->CLOOP_K2(
- [&](ptrdiff_t cindex,
- ptrdiff_t xindex,
- ptrdiff_t yindex,
- ptrdiff_t zindex,
- double k2){
- double tval = this->dealias_filter[int(round(k2 / this->dk2))];
- for (unsigned int tcounter=0; tcounter<2*ncomp(fc); tcounter++)
- ((rnumber*)a)[2*ncomp(fc)*cindex + tcounter] *= tval;
- });
- break;
- }
-}
-
-template <field_backend be,
- kspace_dealias_type dt>
-template <typename rnumber>
-void kspace<be, dt>::force_divfree(typename fftw_interface<rnumber>::complex *__restrict__ a)
-{
- TIMEZONE("kspace::force_divfree");
- this->CLOOP_K2(
- [&](ptrdiff_t cindex,
- ptrdiff_t xindex,
- ptrdiff_t yindex,
- ptrdiff_t zindex,
- double k2){
- if (k2 > 0)
- {
- typename fftw_interface<rnumber>::complex tval;
- tval[0] = (this->kx[xindex]*((*(a + cindex*3 ))[0]) +
- this->ky[yindex]*((*(a + cindex*3+1))[0]) +
- this->kz[zindex]*((*(a + cindex*3+2))[0]) ) / k2;
- tval[1] = (this->kx[xindex]*((*(a + cindex*3 ))[1]) +
- this->ky[yindex]*((*(a + cindex*3+1))[1]) +
- this->kz[zindex]*((*(a + cindex*3+2))[1]) ) / k2;
- for (int imag_part=0; imag_part<2; imag_part++)
- {
- a[cindex*3 ][imag_part] -= tval[imag_part]*this->kx[xindex];
- a[cindex*3+1][imag_part] -= tval[imag_part]*this->ky[yindex];
- a[cindex*3+2][imag_part] -= tval[imag_part]*this->kz[zindex];
- }
- }
- }
- );
- if (this->layout->myrank == this->layout->rank[0][0])
- std::fill_n((rnumber*)(a), 6, 0.0);
-}
-
-template <field_backend be,
- kspace_dealias_type dt>
-template <typename rnumber,
- field_components fc>
-void kspace<be, dt>::cospectrum(
- const rnumber(* __restrict a)[2],
- const rnumber(* __restrict b)[2],
- const hid_t group,
- const std::string dset_name,
- const hsize_t toffset)
-{
- TIMEZONE("field::cospectrum");
- shared_array<double> spec_local_thread(this->nshells*ncomp(fc)*ncomp(fc),[&](double* spec_local){
- std::fill_n(spec_local, this->nshells*ncomp(fc)*ncomp(fc), 0);
- });
-
- this->CLOOP_K2_NXMODES(
- [&](ptrdiff_t cindex,
- ptrdiff_t xindex,
- ptrdiff_t yindex,
- ptrdiff_t zindex,
- double k2,
- int nxmodes){
- if (k2 <= this->kM2)
- {
- double* spec_local = spec_local_thread.getMine();
- int tmp_int = int(sqrt(k2) / this->dk)*ncomp(fc)*ncomp(fc);
- for (hsize_t i=0; i<ncomp(fc); i++)
- for (hsize_t j=0; j<ncomp(fc); j++){
- spec_local[tmp_int + i*ncomp(fc)+j] += nxmodes * (
- (a[ncomp(fc)*cindex + i][0] * b[ncomp(fc)*cindex + j][0]) +
- (a[ncomp(fc)*cindex + i][1] * b[ncomp(fc)*cindex + j][1]));
- }
- }
- });
-
- spec_local_thread.mergeParallel();
-
- std::vector<double> spec;
- spec.resize(this->nshells*ncomp(fc)*ncomp(fc), 0);
- MPI_Allreduce(
- spec_local_thread.getMasterData(),
- &spec.front(),
- spec.size(),
- MPI_DOUBLE, MPI_SUM, this->layout->comm);
- if (this->layout->myrank == 0)
- {
- hid_t dset, wspace, mspace;
- hsize_t count[(ndim(fc)-2)*2], offset[(ndim(fc)-2)*2], dims[(ndim(fc)-2)*2];
- dset = H5Dopen(group, ("spectra/" + dset_name).c_str(), H5P_DEFAULT);
- wspace = H5Dget_space(dset);
- H5Sget_simple_extent_dims(wspace, dims, NULL);
- switch (fc)
- {
- case THREExTHREE:
- offset[4] = 0;
- offset[5] = 0;
- count[4] = ncomp(fc);
- count[5] = ncomp(fc);
- case THREE:
- offset[2] = 0;
- offset[3] = 0;
- count[2] = ncomp(fc);
- count[3] = ncomp(fc);
- default:
- offset[0] = toffset;
- offset[1] = 0;
- count[0] = 1;
- count[1] = this->nshells;
- }
- mspace = H5Screate_simple((ndim(fc)-2)*2, count, NULL);
- H5Sselect_hyperslab(wspace, H5S_SELECT_SET, offset, NULL, count, NULL);
- H5Dwrite(dset, H5T_NATIVE_DOUBLE, mspace, wspace, H5P_DEFAULT, &spec.front());
- H5Sclose(wspace);
- H5Sclose(mspace);
- H5Dclose(dset);
- }
-}
-
-
-template class kspace<FFTW, TWO_THIRDS>;
-template class kspace<FFTW, SMOOTH>;
-
-template kspace<FFTW, TWO_THIRDS>::kspace<>(
- const field_layout<ONE> *,
- const double, const double, const double);
-template kspace<FFTW, TWO_THIRDS>::kspace<>(
- const field_layout<THREE> *,
- const double, const double, const double);
-template kspace<FFTW, TWO_THIRDS>::kspace<>(
- const field_layout<THREExTHREE> *,
- const double, const double, const double);
-
-template kspace<FFTW, SMOOTH>::kspace<>(
- const field_layout<ONE> *,
- const double, const double, const double);
-template kspace<FFTW, SMOOTH>::kspace<>(
- const field_layout<THREE> *,
- const double, const double, const double);
-template kspace<FFTW, SMOOTH>::kspace<>(
- const field_layout<THREExTHREE> *,
- const double, const double, const double);
-
-template void kspace<FFTW, SMOOTH>::low_pass<float, ONE>(
- typename fftw_interface<float>::complex *__restrict__ a,
- const double kmax);
-template void kspace<FFTW, SMOOTH>::low_pass<float, THREE>(
- typename fftw_interface<float>::complex *__restrict__ a,
- const double kmax);
-template void kspace<FFTW, SMOOTH>::low_pass<float, THREExTHREE>(
- typename fftw_interface<float>::complex *__restrict__ a,
- const double kmax);
-
-template void kspace<FFTW, SMOOTH>::low_pass<double, ONE>(
- typename fftw_interface<double>::complex *__restrict__ a,
- const double kmax);
-template void kspace<FFTW, SMOOTH>::low_pass<double, THREE>(
- typename fftw_interface<double>::complex *__restrict__ a,
- const double kmax);
-template void kspace<FFTW, SMOOTH>::low_pass<double, THREExTHREE>(
- typename fftw_interface<double>::complex *__restrict__ a,
- const double kmax);
-
-template void kspace<FFTW, SMOOTH>::Gauss_filter<float, ONE>(
- typename fftw_interface<float>::complex *__restrict__ a,
- const double kmax);
-template void kspace<FFTW, SMOOTH>::Gauss_filter<float, THREE>(
- typename fftw_interface<float>::complex *__restrict__ a,
- const double kmax);
-template void kspace<FFTW, SMOOTH>::Gauss_filter<float, THREExTHREE>(
- typename fftw_interface<float>::complex *__restrict__ a,
- const double kmax);
-
-template void kspace<FFTW, SMOOTH>::Gauss_filter<double, ONE>(
- typename fftw_interface<double>::complex *__restrict__ a,
- const double kmax);
-template void kspace<FFTW, SMOOTH>::Gauss_filter<double, THREE>(
- typename fftw_interface<double>::complex *__restrict__ a,
- const double kmax);
-template void kspace<FFTW, SMOOTH>::Gauss_filter<double, THREExTHREE>(
- typename fftw_interface<double>::complex *__restrict__ a,
- const double kmax);
-
-template int kspace<FFTW, SMOOTH>::filter<float, ONE>(
- typename fftw_interface<float>::complex *__restrict__ a,
- const double kmax,
- std::string filter_type);
-template int kspace<FFTW, SMOOTH>::filter<float, THREE>(
- typename fftw_interface<float>::complex *__restrict__ a,
- const double kmax,
- std::string filter_type);
-template int kspace<FFTW, SMOOTH>::filter<float, THREExTHREE>(
- typename fftw_interface<float>::complex *__restrict__ a,
- const double kmax,
- std::string filter_type);
-
-template int kspace<FFTW, SMOOTH>::filter<double, ONE>(
- typename fftw_interface<double>::complex *__restrict__ a,
- const double kmax,
- std::string filter_type);
-template int kspace<FFTW, SMOOTH>::filter<double, THREE>(
- typename fftw_interface<double>::complex *__restrict__ a,
- const double kmax,
- std::string filter_type);
-template int kspace<FFTW, SMOOTH>::filter<double, THREExTHREE>(
- typename fftw_interface<double>::complex *__restrict__ a,
- const double kmax,
- std::string filter_type);
-
-template int kspace<FFTW, SMOOTH>::filter_calibrated_ell<float, ONE>(
- typename fftw_interface<float>::complex *__restrict__ a,
- const double kmax,
- std::string filter_type);
-template int kspace<FFTW, SMOOTH>::filter_calibrated_ell<float, THREE>(
- typename fftw_interface<float>::complex *__restrict__ a,
- const double kmax,
- std::string filter_type);
-template int kspace<FFTW, SMOOTH>::filter_calibrated_ell<float, THREExTHREE>(
- typename fftw_interface<float>::complex *__restrict__ a,
- const double kmax,
- std::string filter_type);
-
-template int kspace<FFTW, SMOOTH>::filter_calibrated_ell<double, ONE>(
- typename fftw_interface<double>::complex *__restrict__ a,
- const double kmax,
- std::string filter_type);
-template int kspace<FFTW, SMOOTH>::filter_calibrated_ell<double, THREE>(
- typename fftw_interface<double>::complex *__restrict__ a,
- const double kmax,
- std::string filter_type);
-template int kspace<FFTW, SMOOTH>::filter_calibrated_ell<double, THREExTHREE>(
- typename fftw_interface<double>::complex *__restrict__ a,
- const double kmax,
- std::string filter_type);
-
-template void kspace<FFTW, SMOOTH>::dealias<float, ONE>(
- typename fftw_interface<float>::complex *__restrict__ a);
-template void kspace<FFTW, SMOOTH>::dealias<float, THREE>(
- typename fftw_interface<float>::complex *__restrict__ a);
-template void kspace<FFTW, SMOOTH>::dealias<float, THREExTHREE>(
- typename fftw_interface<float>::complex *__restrict__ a);
-
-template void kspace<FFTW, SMOOTH>::dealias<double, ONE>(
- typename fftw_interface<double>::complex *__restrict__ a);
-template void kspace<FFTW, SMOOTH>::dealias<double, THREE>(
- typename fftw_interface<double>::complex *__restrict__ a);
-template void kspace<FFTW, SMOOTH>::dealias<double, THREExTHREE>(
- typename fftw_interface<double>::complex *__restrict__ a);
-
-template void kspace<FFTW, TWO_THIRDS>::cospectrum<float, ONE>(
- const typename fftw_interface<float>::complex *__restrict__ a,
- const typename fftw_interface<float>::complex *__restrict__ b,
- const hid_t group,
- const std::string dset_name,
- const hsize_t toffset);
-template void kspace<FFTW, TWO_THIRDS>::cospectrum<float, THREE>(
- const typename fftw_interface<float>::complex *__restrict__ a,
- const typename fftw_interface<float>::complex *__restrict__ b,
- const hid_t group,
- const std::string dset_name,
- const hsize_t toffset);
-template void kspace<FFTW, TWO_THIRDS>::cospectrum<float, THREExTHREE>(
- const typename fftw_interface<float>::complex *__restrict__ a,
- const typename fftw_interface<float>::complex *__restrict__ b,
- const hid_t group,
- const std::string dset_name,
- const hsize_t toffset);
-template void kspace<FFTW, TWO_THIRDS>::cospectrum<double, ONE>(
- const typename fftw_interface<double>::complex *__restrict__ a,
- const typename fftw_interface<double>::complex *__restrict__ b,
- const hid_t group,
- const std::string dset_name,
- const hsize_t toffset);
-template void kspace<FFTW, TWO_THIRDS>::cospectrum<double, THREE>(
- const typename fftw_interface<double>::complex *__restrict__ a,
- const typename fftw_interface<double>::complex *__restrict__ b,
- const hid_t group,
- const std::string dset_name,
- const hsize_t toffset);
-template void kspace<FFTW, TWO_THIRDS>::cospectrum<double, THREExTHREE>(
- const typename fftw_interface<double>::complex *__restrict__ a,
- const typename fftw_interface<double>::complex *__restrict__ b,
- const hid_t group,
- const std::string dset_name,
- const hsize_t toffset);
-
-template void kspace<FFTW, SMOOTH>::cospectrum<float, ONE>(
- const typename fftw_interface<float>::complex *__restrict__ a,
- const typename fftw_interface<float>::complex *__restrict__ b,
- const hid_t group,
- const std::string dset_name,
- const hsize_t toffset);
-template void kspace<FFTW, SMOOTH>::cospectrum<float, THREE>(
- const typename fftw_interface<float>::complex *__restrict__ a,
- const typename fftw_interface<float>::complex *__restrict__ b,
- const hid_t group,
- const std::string dset_name,
- const hsize_t toffset);
-template void kspace<FFTW, SMOOTH>::cospectrum<float, THREExTHREE>(
- const typename fftw_interface<float>::complex *__restrict__ a,
- const typename fftw_interface<float>::complex *__restrict__ b,
- const hid_t group,
- const std::string dset_name,
- const hsize_t toffset);
-template void kspace<FFTW, SMOOTH>::cospectrum<double, ONE>(
- const typename fftw_interface<double>::complex *__restrict__ a,
- const typename fftw_interface<double>::complex *__restrict__ b,
- const hid_t group,
- const std::string dset_name,
- const hsize_t toffset);
-template void kspace<FFTW, SMOOTH>::cospectrum<double, THREE>(
- const typename fftw_interface<double>::complex *__restrict__ a,
- const typename fftw_interface<double>::complex *__restrict__ b,
- const hid_t group,
- const std::string dset_name,
- const hsize_t toffset);
-template void kspace<FFTW, SMOOTH>::cospectrum<double, THREExTHREE>(
- const typename fftw_interface<double>::complex *__restrict__ a,
- const typename fftw_interface<double>::complex *__restrict__ b,
- const hid_t group,
- const std::string dset_name,
- const hsize_t toffset);
-
-template void kspace<FFTW, SMOOTH>::force_divfree<float>(
- typename fftw_interface<float>::complex *__restrict__ a);
-template void kspace<FFTW, SMOOTH>::force_divfree<double>(
- typename fftw_interface<double>::complex *__restrict__ a);
-
diff --git a/bfps/cpp/kspace.hpp b/bfps/cpp/kspace.hpp
deleted file mode 100644
index d8bc008daade0704c5f8c1981c4a4f24400a5868..0000000000000000000000000000000000000000
--- a/bfps/cpp/kspace.hpp
+++ /dev/null
@@ -1,198 +0,0 @@
-/**********************************************************************
-* *
-* Copyright 2015 Max Planck Institute *
-* for Dynamics and Self-Organization *
-* *
-* This file is part of bfps. *
-* *
-* bfps is free software: you can redistribute it and/or modify *
-* it under the terms of the GNU General Public License as published *
-* by the Free Software Foundation, either version 3 of the License, *
-* or (at your option) any later version. *
-* *
-* bfps is distributed in the hope that it will be useful, *
-* but WITHOUT ANY WARRANTY; without even the implied warranty of *
-* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the *
-* GNU General Public License for more details. *
-* *
-* You should have received a copy of the GNU General Public License *
-* along with bfps. If not, see <http://www.gnu.org/licenses/> *
-* *
-* Contact: Cristian.Lalescu@ds.mpg.de *
-* *
-**********************************************************************/
-
-
-
-#include <hdf5.h>
-#include <unordered_map>
-#include <vector>
-#include <string>
-#include "omputils.hpp"
-#include "fftw_interface.hpp"
-#include "field_layout.hpp"
-
-#ifndef KSPACE_HPP
-
-#define KSPACE_HPP
-
-enum field_backend {FFTW};
-enum kspace_dealias_type {TWO_THIRDS, SMOOTH};
-
-
-template <field_backend be,
- kspace_dealias_type dt>
-class kspace
-{
- public:
- /* relevant field layout */
- field_layout<ONE> *layout;
-
- /* physical parameters */
- double dkx, dky, dkz, dk, dk2;
-
- /* mode and dealiasing information */
- double kMx, kMy, kMz, kM, kM2;
- std::vector<double> kx, ky, kz;
- std::unordered_map<int, double> dealias_filter;
- std::vector<double> kshell;
- std::vector<int64_t> nshell;
- int nshells;
-
- /* methods */
- template <field_components fc>
- kspace(
- const field_layout<fc> *source_layout,
- const double DKX = 1.0,
- const double DKY = 1.0,
- const double DKZ = 1.0);
- ~kspace();
-
- int store(hid_t stat_file);
-
- template <typename rnumber,
- field_components fc>
- void low_pass(
- typename fftw_interface<rnumber>::complex *__restrict__ a,
- const double kmax);
-
- template <typename rnumber,
- field_components fc>
- void Gauss_filter(
- typename fftw_interface<rnumber>::complex *__restrict__ a,
- const double sigma);
-
- template <typename rnumber,
- field_components fc>
- void ball_filter(
- typename fftw_interface<rnumber>::complex *__restrict__ a,
- const double sigma);
-
- template <typename rnumber,
- field_components fc>
- int filter(
- typename fftw_interface<rnumber>::complex *__restrict__ a,
- const double wavenumber,
- std::string filter_type = std::string("Gauss"));
-
- template <typename rnumber,
- field_components fc>
- int filter_calibrated_ell(
- typename fftw_interface<rnumber>::complex *__restrict__ a,
- const double wavenumber,
- std::string filter_type = std::string("Gauss"));
-
- template <typename rnumber,
- field_components fc>
- void dealias(typename fftw_interface<rnumber>::complex *__restrict__ a);
-
- template <typename rnumber,
- field_components fc>
- void cospectrum(
- const rnumber(* __restrict__ a)[2],
- const rnumber(* __restrict__ b)[2],
- const hid_t group,
- const std::string dset_name,
- const hsize_t toffset);
- template <class func_type>
- void CLOOP(func_type expression)
- {
- #pragma omp parallel
- {
- const hsize_t start = OmpUtils::ForIntervalStart(this->layout->subsizes[1]);
- const hsize_t end = OmpUtils::ForIntervalEnd(this->layout->subsizes[1]);
-
- for (hsize_t yindex = 0; yindex < this->layout->subsizes[0]; yindex++){
- for (hsize_t zindex = start; zindex < end; zindex++){
- ptrdiff_t cindex = yindex*this->layout->subsizes[1]*this->layout->subsizes[2]
- + zindex*this->layout->subsizes[2];
- for (hsize_t xindex = 0; xindex < this->layout->subsizes[2]; xindex++)
- {
- expression(cindex, xindex, yindex, zindex);
- cindex++;
- }
- }
- }
- }
- }
- template <class func_type>
- void CLOOP_K2(func_type expression)
- {
- #pragma omp parallel
- {
- const hsize_t start = OmpUtils::ForIntervalStart(this->layout->subsizes[1]);
- const hsize_t end = OmpUtils::ForIntervalEnd(this->layout->subsizes[1]);
-
- for (hsize_t yindex = 0; yindex < this->layout->subsizes[0]; yindex++){
- for (hsize_t zindex = start; zindex < end; zindex++){
- ptrdiff_t cindex = yindex*this->layout->subsizes[1]*this->layout->subsizes[2]
- + zindex*this->layout->subsizes[2];
- for (hsize_t xindex = 0; xindex < this->layout->subsizes[2]; xindex++)
- {
- double k2 = (this->kx[xindex]*this->kx[xindex] +
- this->ky[yindex]*this->ky[yindex] +
- this->kz[zindex]*this->kz[zindex]);
- expression(cindex, xindex, yindex, zindex, k2);
- cindex++;
- }
- }
- }
- }
- }
- template <class func_type>
- void CLOOP_K2_NXMODES(func_type expression)
- {
- #pragma omp parallel
- {
- const hsize_t start = OmpUtils::ForIntervalStart(this->layout->subsizes[1]);
- const hsize_t end = OmpUtils::ForIntervalEnd(this->layout->subsizes[1]);
-
- for (hsize_t yindex = 0; yindex < this->layout->subsizes[0]; yindex++){
- for (hsize_t zindex = start; zindex < end; zindex++){
- ptrdiff_t cindex = yindex*this->layout->subsizes[1]*this->layout->subsizes[2]
- + zindex*this->layout->subsizes[2];
- hsize_t xindex = 0;
- double k2 = (
- this->kx[xindex]*this->kx[xindex] +
- this->ky[yindex]*this->ky[yindex] +
- this->kz[zindex]*this->kz[zindex]);
- expression(cindex, xindex, yindex, zindex, k2, 1);
- cindex++;
- for (xindex = 1; xindex < this->layout->subsizes[2]; xindex++)
- {
- k2 = (this->kx[xindex]*this->kx[xindex] +
- this->ky[yindex]*this->ky[yindex] +
- this->kz[zindex]*this->kz[zindex]);
- expression(cindex, xindex, yindex, zindex, k2, 2);
- cindex++;
- }
- }
- }
- }
- }
- template <typename rnumber>
- void force_divfree(typename fftw_interface<rnumber>::complex *__restrict__ a);
-};
-
-#endif//KSPACE_HPP
-
diff --git a/bfps/cpp/particles.cpp b/bfps/cpp/particles.cpp
deleted file mode 100644
index cdaf157cb912c3074faf84bfecf1d9b3752c78a7..0000000000000000000000000000000000000000
--- a/bfps/cpp/particles.cpp
+++ /dev/null
@@ -1,254 +0,0 @@
-/**********************************************************************
-* *
-* Copyright 2015 Max Planck Institute *
-* for Dynamics and Self-Organization *
-* *
-* This file is part of bfps. *
-* *
-* bfps is free software: you can redistribute it and/or modify *
-* it under the terms of the GNU General Public License as published *
-* by the Free Software Foundation, either version 3 of the License, *
-* or (at your option) any later version. *
-* *
-* bfps is distributed in the hope that it will be useful, *
-* but WITHOUT ANY WARRANTY; without even the implied warranty of *
-* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the *
-* GNU General Public License for more details. *
-* *
-* You should have received a copy of the GNU General Public License *
-* along with bfps. If not, see <http://www.gnu.org/licenses/> *
-* *
-* Contact: Cristian.Lalescu@ds.mpg.de *
-* *
-**********************************************************************/
-
-
-
-#define NDEBUG
-
-#include <cmath>
-#include <cassert>
-#include <cstring>
-#include <string>
-#include <sstream>
-
-#include "base.hpp"
-#include "particles.hpp"
-#include "fftw_tools.hpp"
-
-
-extern int myrank, nprocs;
-
-template <particle_types particle_type, class rnumber, int interp_neighbours>
-particles<particle_type, rnumber, interp_neighbours>::particles(
- const char *NAME,
- const hid_t data_file_id,
- interpolator_base<rnumber, interp_neighbours> *VEL,
- const int TRAJ_SKIP,
- const int INTEGRATION_STEPS) : particles_io_base<particle_type>(
- NAME,
- TRAJ_SKIP,
- data_file_id,
- VEL->descriptor->comm)
-{
- assert((INTEGRATION_STEPS <= 6) &&
- (INTEGRATION_STEPS >= 1));
- this->vel = VEL;
- this->integration_steps = INTEGRATION_STEPS;
- this->array_size = this->nparticles * state_dimension(particle_type);
- this->state = new double[this->array_size];
- std::fill_n(this->state, this->array_size, 0.0);
- for (int i=0; i < this->integration_steps; i++)
- {
- this->rhs[i] = new double[this->array_size];
- std::fill_n(this->rhs[i], this->array_size, 0.0);
- }
-}
-
-template <particle_types particle_type, class rnumber, int interp_neighbours>
-particles<particle_type, rnumber, interp_neighbours>::~particles()
-{
- delete[] this->state;
- for (int i=0; i < this->integration_steps; i++)
- {
- delete[] this->rhs[i];
- }
-}
-
-template <particle_types particle_type, class rnumber, int interp_neighbours>
-void particles<particle_type, rnumber, interp_neighbours>::get_rhs(double *x, double *y)
-{
- switch(particle_type)
- {
- case VELOCITY_TRACER:
- this->vel->sample(this->nparticles, state_dimension(particle_type), x, y);
- break;
- }
-}
-
-template <particle_types particle_type, class rnumber, int interp_neighbours>
-void particles<particle_type, rnumber, interp_neighbours>::roll_rhs()
-{
- for (int i=this->integration_steps-2; i>=0; i--)
- std::copy(this->rhs[i],
- this->rhs[i] + this->array_size,
- this->rhs[i+1]);
-}
-
-
-
-template <particle_types particle_type, class rnumber, int interp_neighbours>
-void particles<particle_type, rnumber, interp_neighbours>::AdamsBashforth(
- const int nsteps)
-{
- ptrdiff_t ii;
- this->get_rhs(this->state, this->rhs[0]);
- switch(nsteps)
- {
- case 1:
- for (unsigned int p=0; p<this->nparticles; p++)
- for (unsigned int i=0; i<state_dimension(particle_type); i++)
- {
- ii = p*state_dimension(particle_type)+i;
- this->state[ii] += this->dt*this->rhs[0][ii];
- }
- break;
- case 2:
- for (unsigned int p=0; p<this->nparticles; p++)
- for (unsigned int i=0; i<state_dimension(particle_type); i++)
- {
- ii = p*state_dimension(particle_type)+i;
- this->state[ii] += this->dt*(3*this->rhs[0][ii]
- - this->rhs[1][ii])/2;
- }
- break;
- case 3:
- for (unsigned int p=0; p<this->nparticles; p++)
- for (unsigned int i=0; i<state_dimension(particle_type); i++)
- {
- ii = p*state_dimension(particle_type)+i;
- this->state[ii] += this->dt*(23*this->rhs[0][ii]
- - 16*this->rhs[1][ii]
- + 5*this->rhs[2][ii])/12;
- }
- break;
- case 4:
- for (unsigned int p=0; p<this->nparticles; p++)
- for (unsigned int i=0; i<state_dimension(particle_type); i++)
- {
- ii = p*state_dimension(particle_type)+i;
- this->state[ii] += this->dt*(55*this->rhs[0][ii]
- - 59*this->rhs[1][ii]
- + 37*this->rhs[2][ii]
- - 9*this->rhs[3][ii])/24;
- }
- break;
- case 5:
- for (unsigned int p=0; p<this->nparticles; p++)
- for (unsigned int i=0; i<state_dimension(particle_type); i++)
- {
- ii = p*state_dimension(particle_type)+i;
- this->state[ii] += this->dt*(1901*this->rhs[0][ii]
- - 2774*this->rhs[1][ii]
- + 2616*this->rhs[2][ii]
- - 1274*this->rhs[3][ii]
- + 251*this->rhs[4][ii])/720;
- }
- break;
- case 6:
- for (unsigned int p=0; p<this->nparticles; p++)
- for (unsigned int i=0; i<state_dimension(particle_type); i++)
- {
- ii = p*state_dimension(particle_type)+i;
- this->state[ii] += this->dt*(4277*this->rhs[0][ii]
- - 7923*this->rhs[1][ii]
- + 9982*this->rhs[2][ii]
- - 7298*this->rhs[3][ii]
- + 2877*this->rhs[4][ii]
- - 475*this->rhs[5][ii])/1440;
- }
- break;
- }
- this->roll_rhs();
-}
-
-
-template <particle_types particle_type, class rnumber, int interp_neighbours>
-void particles<particle_type, rnumber, interp_neighbours>::step()
-{
- this->AdamsBashforth((this->iteration < this->integration_steps) ?
- this->iteration+1 :
- this->integration_steps);
- this->iteration++;
-}
-
-
-template <particle_types particle_type, class rnumber, int interp_neighbours>
-void particles<particle_type, rnumber, interp_neighbours>::read()
-{
- if (this->myrank == 0)
- for (unsigned int cindex=0; cindex<this->get_number_of_chunks(); cindex++)
- {
- this->read_state_chunk(cindex, this->state+cindex*this->chunk_size*state_dimension(particle_type));
- if (this->iteration > 0)
- for (int i=0; i<this->integration_steps; i++)
- this->read_rhs_chunk(cindex, i, this->rhs[i]+cindex*this->chunk_size*state_dimension(particle_type));
- }
- MPI_Bcast(
- this->state,
- this->array_size,
- MPI_DOUBLE,
- 0,
- this->comm);
- if (this->iteration > 0)
- for (int i = 0; i<this->integration_steps; i++)
- MPI_Bcast(
- this->rhs[i],
- this->array_size,
- MPI_DOUBLE,
- 0,
- this->comm);
-}
-
-template <particle_types particle_type, class rnumber, int interp_neighbours>
-void particles<particle_type, rnumber, interp_neighbours>::write(
- const bool write_rhs)
-{
- if (this->myrank == 0)
- for (unsigned int cindex=0; cindex<this->get_number_of_chunks(); cindex++)
- {
- this->write_state_chunk(cindex, this->state+cindex*this->chunk_size*state_dimension(particle_type));
- if (write_rhs)
- for (int i=0; i<this->integration_steps; i++)
- this->write_rhs_chunk(cindex, i, this->rhs[i]+cindex*this->chunk_size*state_dimension(particle_type));
- }
-}
-
-template <particle_types particle_type, class rnumber, int interp_neighbours>
-void particles<particle_type, rnumber, interp_neighbours>::sample(
- interpolator_base<rnumber, interp_neighbours> *field,
- const char *dset_name)
-{
- double *y = new double[this->nparticles*3];
- field->sample(this->nparticles, state_dimension(particle_type), this->state, y);
- if (this->myrank == 0)
- for (unsigned int cindex=0; cindex<this->get_number_of_chunks(); cindex++)
- this->write_point3D_chunk(dset_name, cindex, y+cindex*this->chunk_size*3);
- delete[] y;
-}
-
-
-/*****************************************************************************/
-template class particles<VELOCITY_TRACER, float, 1>;
-template class particles<VELOCITY_TRACER, float, 2>;
-template class particles<VELOCITY_TRACER, float, 3>;
-template class particles<VELOCITY_TRACER, float, 4>;
-template class particles<VELOCITY_TRACER, float, 5>;
-template class particles<VELOCITY_TRACER, float, 6>;
-template class particles<VELOCITY_TRACER, double, 1>;
-template class particles<VELOCITY_TRACER, double, 2>;
-template class particles<VELOCITY_TRACER, double, 3>;
-template class particles<VELOCITY_TRACER, double, 4>;
-template class particles<VELOCITY_TRACER, double, 5>;
-template class particles<VELOCITY_TRACER, double, 6>;
-/*****************************************************************************/
diff --git a/bfps/cpp/particles.hpp b/bfps/cpp/particles.hpp
deleted file mode 100644
index 03daf3e3fc866ac485b3649a28dfb13cf1b50ff1..0000000000000000000000000000000000000000
--- a/bfps/cpp/particles.hpp
+++ /dev/null
@@ -1,99 +0,0 @@
-/**********************************************************************
-* *
-* Copyright 2015 Max Planck Institute *
-* for Dynamics and Self-Organization *
-* *
-* This file is part of bfps. *
-* *
-* bfps is free software: you can redistribute it and/or modify *
-* it under the terms of the GNU General Public License as published *
-* by the Free Software Foundation, either version 3 of the License, *
-* or (at your option) any later version. *
-* *
-* bfps is distributed in the hope that it will be useful, *
-* but WITHOUT ANY WARRANTY; without even the implied warranty of *
-* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the *
-* GNU General Public License for more details. *
-* *
-* You should have received a copy of the GNU General Public License *
-* along with bfps. If not, see <http://www.gnu.org/licenses/> *
-* *
-* Contact: Cristian.Lalescu@ds.mpg.de *
-* *
-**********************************************************************/
-
-
-
-#include <stdio.h>
-#include <stdlib.h>
-#include <iostream>
-#include <hdf5.h>
-#include "base.hpp"
-#include "particles_base.hpp"
-#include "fluid_solver_base.hpp"
-#include "interpolator_base.hpp"
-
-#ifndef PARTICLES
-
-#define PARTICLES
-
-template <particle_types particle_type, class rnumber, int interp_neighbours>
-class particles: public particles_io_base<particle_type>
-{
- private:
- double *state;
- double *rhs[6];
-
- public:
- int array_size;
- int integration_steps;
- interpolator_base<rnumber, interp_neighbours> *vel;
-
- /* simulation parameters */
- double dt;
-
- /* methods */
-
- /* constructor and destructor.
- * allocate and deallocate:
- * this->state
- * this->rhs
- * */
- particles(
- const char *NAME,
- const hid_t data_file_id,
- interpolator_base<rnumber, interp_neighbours> *FIELD,
- const int TRAJ_SKIP,
- const int INTEGRATION_STEPS = 2);
- ~particles();
-
- void sample(
- interpolator_base<rnumber, interp_neighbours> *field,
- const char *dset_name);
-
- inline void sample(
- interpolator_base<rnumber, interp_neighbours> *field,
- double *y)
- {
- field->sample(this->nparticles, state_dimension(particle_type), this->state, y);
- }
-
- void get_rhs(
- double *__restrict__ x,
- double *__restrict__ rhs);
-
- /* input/output */
- void read();
- void write(
- const char *dset_name,
- const double *data);
- void write(const bool write_rhs = true);
-
- /* solvers */
- void step();
- void roll_rhs();
- void AdamsBashforth(const int nsteps);
-};
-
-#endif//PARTICLES
-
diff --git a/bfps/cpp/particles/abstract_particles_input.hpp b/bfps/cpp/particles/abstract_particles_input.hpp
deleted file mode 100644
index 77dcbc638903a668ce6e2a0084815832b0580495..0000000000000000000000000000000000000000
--- a/bfps/cpp/particles/abstract_particles_input.hpp
+++ /dev/null
@@ -1,21 +0,0 @@
-#ifndef ABSTRACT_PARTICLES_INPUT_HPP
-#define ABSTRACT_PARTICLES_INPUT_HPP
-
-#include <tuple>
-
-template <class partsize_t, class real_number>
-class abstract_particles_input {
-public:
- virtual ~abstract_particles_input(){}
-
- virtual partsize_t getTotalNbParticles() = 0;
- virtual partsize_t getLocalNbParticles() = 0;
- virtual int getNbRhs() = 0;
-
- virtual std::unique_ptr<real_number[]> getMyParticles() = 0;
- virtual std::unique_ptr<partsize_t[]> getMyParticlesIndexes() = 0;
- virtual std::vector<std::unique_ptr<real_number[]>> getMyRhs() = 0;
-};
-
-
-#endif
diff --git a/bfps/cpp/particles/abstract_particles_system.hpp b/bfps/cpp/particles/abstract_particles_system.hpp
deleted file mode 100644
index 1c8592f37536e5c6c6b4df8f45cc855b3f21eb3f..0000000000000000000000000000000000000000
--- a/bfps/cpp/particles/abstract_particles_system.hpp
+++ /dev/null
@@ -1,57 +0,0 @@
-#ifndef ABSTRACT_PARTICLES_SYSTEM_HPP
-#define ABSTRACT_PARTICLES_SYSTEM_HPP
-
-#include <memory>
-
-//- Not generic to enable sampling begin
-#include "field.hpp"
-#include "kspace.hpp"
-//- Not generic to enable sampling end
-
-
-template <class partsize_t, class real_number>
-class abstract_particles_system {
-public:
- virtual void compute() = 0;
-
- virtual void move(const real_number dt) = 0;
-
- virtual void redistribute() = 0;
-
- virtual void inc_step_idx() = 0;
-
- virtual void shift_rhs_vectors() = 0;
-
- virtual void completeLoop(const real_number dt) = 0;
-
- virtual const real_number* getParticlesPositions() const = 0;
-
- virtual const std::unique_ptr<real_number[]>* getParticlesRhs() const = 0;
-
- virtual const partsize_t* getParticlesIndexes() const = 0;
-
- virtual partsize_t getLocalNbParticles() const = 0;
-
- virtual partsize_t getGlobalNbParticles() const = 0;
-
- virtual int getNbRhs() const = 0;
-
- virtual int get_step_idx() const = 0;
-
- //- Not generic to enable sampling begin
- virtual void sample_compute_field(const field<float, FFTW, ONE>& sample_field,
- real_number sample_rhs[]) = 0;
- virtual void sample_compute_field(const field<float, FFTW, THREE>& sample_field,
- real_number sample_rhs[]) = 0;
- virtual void sample_compute_field(const field<float, FFTW, THREExTHREE>& sample_field,
- real_number sample_rhs[]) = 0;
- virtual void sample_compute_field(const field<double, FFTW, ONE>& sample_field,
- real_number sample_rhs[]) = 0;
- virtual void sample_compute_field(const field<double, FFTW, THREE>& sample_field,
- real_number sample_rhs[]) = 0;
- virtual void sample_compute_field(const field<double, FFTW, THREExTHREE>& sample_field,
- real_number sample_rhs[]) = 0;
- //- Not generic to enable sampling end
-};
-
-#endif
diff --git a/bfps/cpp/particles/particles_generic_interp.hpp b/bfps/cpp/particles/particles_generic_interp.hpp
deleted file mode 100644
index 98d0363d4fcfae8c05b6ceabef620e17c1263eee..0000000000000000000000000000000000000000
--- a/bfps/cpp/particles/particles_generic_interp.hpp
+++ /dev/null
@@ -1,316 +0,0 @@
-#ifndef PARTICLES_GENERIC_INTERP_HPP
-#define PARTICLES_GENERIC_INTERP_HPP
-
-template <class real_number, int interp_neighbours, int mode>
-class particles_generic_interp;
-
-#include "Lagrange_polys.hpp"
-#include "spline.hpp"
-
-template <>
-class particles_generic_interp<double, 1,0>{
-public:
- using real_number = double;
-
- void compute_beta(const int in_derivative, const double in_part_val, double poly_val[]) const {
- beta_Lagrange_n1(in_derivative, in_part_val, poly_val);
- }
-};
-
-template <>
-class particles_generic_interp<double, 1,1>{
-public:
- using real_number = double;
-
- void compute_beta(const int in_derivative, const double in_part_val, double poly_val[]) const {
- beta_n1_m1(in_derivative, in_part_val, poly_val);
- }
-};
-
-template <>
-class particles_generic_interp<double, 1,2>{
-public:
- using real_number = double;
-
- void compute_beta(const int in_derivative, const double in_part_val, double poly_val[]) const {
- beta_n1_m2(in_derivative, in_part_val, poly_val);
- }
-};
-
-template <>
-class particles_generic_interp<double, 2,0>{
-public:
- using real_number = double;
-
- void compute_beta(const int in_derivative, const double in_part_val, double poly_val[]) const {
- beta_Lagrange_n2(in_derivative, in_part_val, poly_val);
- }
-};
-
-template <>
-class particles_generic_interp<double, 2,1>{
-public:
- using real_number = double;
-
- void compute_beta(const int in_derivative, const double in_part_val, double poly_val[]) const {
- beta_n2_m1(in_derivative, in_part_val, poly_val);
- }
-};
-
-template <>
-class particles_generic_interp<double, 2,2>{
-public:
- using real_number = double;
-
- void compute_beta(const int in_derivative, const double in_part_val, double poly_val[]) const {
- beta_n2_m2(in_derivative, in_part_val, poly_val);
- }
-};
-
-template <>
-class particles_generic_interp<double, 3,0>{
-public:
- using real_number = double;
-
- void compute_beta(const int in_derivative, const double in_part_val, double poly_val[]) const {
- beta_Lagrange_n3(in_derivative, in_part_val, poly_val);
- }
-};
-
-template <>
-class particles_generic_interp<double, 3,1>{
-public:
- using real_number = double;
-
- void compute_beta(const int in_derivative, const double in_part_val, double poly_val[]) const {
- beta_n3_m1(in_derivative, in_part_val, poly_val);
- }
-};
-
-template <>
-class particles_generic_interp<double, 3,2>{
-public:
- using real_number = double;
-
- void compute_beta(const int in_derivative, const double in_part_val, double poly_val[]) const {
- beta_n3_m2(in_derivative, in_part_val, poly_val);
- }
-};
-
-template <>
-class particles_generic_interp<double, 4,0>{
-public:
- using real_number = double;
-
- void compute_beta(const int in_derivative, const double in_part_val, double poly_val[]) const {
- beta_Lagrange_n4(in_derivative, in_part_val, poly_val);
- }
-};
-
-template <>
-class particles_generic_interp<double, 4,1>{
-public:
- using real_number = double;
-
- void compute_beta(const int in_derivative, const double in_part_val, double poly_val[]) const {
- beta_n4_m1(in_derivative, in_part_val, poly_val);
- }
-};
-
-template <>
-class particles_generic_interp<double, 4,2>{
-public:
- using real_number = double;
-
- void compute_beta(const int in_derivative, const double in_part_val, double poly_val[]) const {
- beta_n4_m2(in_derivative, in_part_val, poly_val);
- }
-};
-
-template <>
-class particles_generic_interp<double, 5,0>{
-public:
- using real_number = double;
-
- void compute_beta(const int in_derivative, const double in_part_val, double poly_val[]) const {
- beta_Lagrange_n5(in_derivative, in_part_val, poly_val);
- }
-};
-
-template <>
-class particles_generic_interp<double, 5,1>{
-public:
- using real_number = double;
-
- void compute_beta(const int in_derivative, const double in_part_val, double poly_val[]) const {
- beta_n5_m1(in_derivative, in_part_val, poly_val);
- }
-};
-
-template <>
-class particles_generic_interp<double, 5,2>{
-public:
- using real_number = double;
-
- void compute_beta(const int in_derivative, const double in_part_val, double poly_val[]) const {
- beta_n5_m2(in_derivative, in_part_val, poly_val);
- }
-};
-
-
-template <>
-class particles_generic_interp<double, 6,0>{
-public:
- using real_number = double;
-
- void compute_beta(const int in_derivative, const double in_part_val, double poly_val[]) const {
- beta_Lagrange_n6(in_derivative, in_part_val, poly_val);
- }
-};
-
-template <>
-class particles_generic_interp<double, 6,1>{
-public:
- using real_number = double;
-
- void compute_beta(const int in_derivative, const double in_part_val, double poly_val[]) const {
- beta_n6_m1(in_derivative, in_part_val, poly_val);
- }
-};
-
-template <>
-class particles_generic_interp<double, 6,2>{
-public:
- using real_number = double;
-
- void compute_beta(const int in_derivative, const double in_part_val, double poly_val[]) const {
- beta_n6_m2(in_derivative, in_part_val, poly_val);
- }
-};
-
-
-template <>
-class particles_generic_interp<double, 7,0>{
-public:
- using real_number = double;
-
- void compute_beta(const int in_derivative, const double in_part_val, double poly_val[]) const {
- beta_Lagrange_n7(in_derivative, in_part_val, poly_val);
- }
-};
-
-template <>
-class particles_generic_interp<double, 7,1>{
-public:
- using real_number = double;
-
- void compute_beta(const int in_derivative, const double in_part_val, double poly_val[]) const {
- beta_n7_m1(in_derivative, in_part_val, poly_val);
- }
-};
-
-template <>
-class particles_generic_interp<double, 7,2>{
-public:
- using real_number = double;
-
- void compute_beta(const int in_derivative, const double in_part_val, double poly_val[]) const {
- beta_n7_m2(in_derivative, in_part_val, poly_val);
- }
-};
-
-
-template <>
-class particles_generic_interp<double, 8,0>{
-public:
- using real_number = double;
-
- void compute_beta(const int in_derivative, const double in_part_val, double poly_val[]) const {
- beta_Lagrange_n8(in_derivative, in_part_val, poly_val);
- }
-};
-
-template <>
-class particles_generic_interp<double, 8,1>{
-public:
- using real_number = double;
-
- void compute_beta(const int in_derivative, const double in_part_val, double poly_val[]) const {
- beta_n8_m1(in_derivative, in_part_val, poly_val);
- }
-};
-
-template <>
-class particles_generic_interp<double, 8,2>{
-public:
- using real_number = double;
-
- void compute_beta(const int in_derivative, const double in_part_val, double poly_val[]) const {
- beta_n8_m2(in_derivative, in_part_val, poly_val);
- }
-};
-
-
-template <>
-class particles_generic_interp<double, 9, 0>{
-public:
- using real_number = double;
-
- void compute_beta(const int in_derivative, const double in_part_val, double poly_val[]) const {
- beta_Lagrange_n9(in_derivative, in_part_val, poly_val);
- }
-};
-
-template <>
-class particles_generic_interp<double, 9,1>{
-public:
- using real_number = double;
-
- void compute_beta(const int in_derivative, const double in_part_val, double poly_val[]) const {
- beta_n9_m1(in_derivative, in_part_val, poly_val);
- }
-};
-
-template <>
-class particles_generic_interp<double, 9,2>{
-public:
- using real_number = double;
-
- void compute_beta(const int in_derivative, const double in_part_val, double poly_val[]) const {
- beta_n9_m2(in_derivative, in_part_val, poly_val);
- }
-};
-
-
-template <>
-class particles_generic_interp<double, 10,0>{
-public:
- using real_number = double;
-
- void compute_beta(const int in_derivative, const double in_part_val, double poly_val[]) const {
- beta_Lagrange_n10(in_derivative, in_part_val, poly_val);
- }
-};
-
-template <>
-class particles_generic_interp<double, 10,1>{
-public:
- using real_number = double;
-
- void compute_beta(const int in_derivative, const double in_part_val, double poly_val[]) const {
- beta_n10_m1(in_derivative, in_part_val, poly_val);
- }
-};
-
-template <>
-class particles_generic_interp<double, 10,2>{
-public:
- using real_number = double;
-
- void compute_beta(const int in_derivative, const double in_part_val, double poly_val[]) const {
- beta_n10_m2(in_derivative, in_part_val, poly_val);
- }
-};
-
-#endif//PARTICLES_INTERP_SPLINE_HPP
-
diff --git a/bfps/cpp/particles/particles_output_sampling_hdf5.hpp b/bfps/cpp/particles/particles_output_sampling_hdf5.hpp
deleted file mode 100644
index 238c9acf9a16db9c36b81d3c6eb6dc2388bbf117..0000000000000000000000000000000000000000
--- a/bfps/cpp/particles/particles_output_sampling_hdf5.hpp
+++ /dev/null
@@ -1,188 +0,0 @@
-#ifndef PARTICLES_OUTPUT_SAMPLING_HDF5_HPP
-#define PARTICLES_OUTPUT_SAMPLING_HDF5_HPP
-
-#include "abstract_particles_output.hpp"
-
-#include <hdf5.h>
-
-template <class partsize_t,
- class real_number,
- int size_particle_positions,
- int size_particle_rhs>
-class particles_output_sampling_hdf5 : public abstract_particles_output<partsize_t,
- real_number,
- size_particle_positions,
- size_particle_rhs>{
- using Parent = abstract_particles_output<partsize_t,
- real_number,
- size_particle_positions,
- size_particle_rhs>;
-
- hid_t file_id, pgroup_id;
-
- const std::string dataset_name;
- const bool use_collective_io;
-
-public:
- static bool DatasetExistsCol(MPI_Comm in_mpi_com,
- const std::string& in_filename,
- const std::string& in_groupname,
- const std::string& in_dataset_name){
- int my_rank;
- AssertMpi(MPI_Comm_rank(in_mpi_com, &my_rank));
-
- int dataset_exists = -1;
-
- if(my_rank == 0){
- // Parallel HDF5 write
- hid_t file_id = H5Fopen(
- in_filename.c_str(),
- H5F_ACC_RDWR | H5F_ACC_DEBUG,
- H5P_DEFAULT);
- assert(file_id >= 0);
-
- dataset_exists = H5Lexists(
- file_id,
- (in_groupname + "/" + in_dataset_name).c_str(),
- H5P_DEFAULT);
-
- int retTest = H5Fclose(file_id);
- assert(retTest >= 0);
- }
-
- AssertMpi(MPI_Bcast( &dataset_exists, 1, MPI_INT, 0, in_mpi_com ));
- return dataset_exists;
- }
-
- particles_output_sampling_hdf5(MPI_Comm in_mpi_com,
- const partsize_t inTotalNbParticles,
- const std::string& in_filename,
- const std::string& in_groupname,
- const std::string& in_dataset_name,
- const bool in_use_collective_io = false)
- : Parent(in_mpi_com, inTotalNbParticles, 1),
- dataset_name(in_dataset_name),
- use_collective_io(in_use_collective_io){
- if(Parent::isInvolved()){
- hid_t plist_id_par = H5Pcreate(H5P_FILE_ACCESS);
- assert(plist_id_par >= 0);
- int retTest = H5Pset_fapl_mpio(
- plist_id_par,
- Parent::getComWriter(),
- MPI_INFO_NULL);
- assert(retTest >= 0);
-
- // Parallel HDF5 write
- file_id = H5Fopen(
- in_filename.c_str(),
- H5F_ACC_RDWR | H5F_ACC_DEBUG,
- plist_id_par);
- assert(file_id >= 0);
- retTest = H5Pclose(plist_id_par);
- assert(retTest >= 0);
-
- pgroup_id = H5Gopen(
- file_id,
- in_groupname.c_str(),
- H5P_DEFAULT);
- assert(pgroup_id >= 0);
- }
- }
-
- ~particles_output_sampling_hdf5(){
- if(Parent::isInvolved()){
- int retTest = H5Gclose(pgroup_id);
- assert(retTest >= 0);
- retTest = H5Fclose(file_id);
- assert(retTest >= 0);
- }
- }
-
- void write(
- const int /*idx_time_step*/,
- const real_number* /*particles_positions*/,
- const std::unique_ptr<real_number[]>* particles_rhs,
- const partsize_t nb_particles,
- const partsize_t particles_idx_offset) final{
- assert(Parent::isInvolved());
-
- TIMEZONE("particles_output_hdf5::write");
-
- assert(particles_idx_offset < Parent::getTotalNbParticles() || (particles_idx_offset == Parent::getTotalNbParticles() && nb_particles == 0));
- assert(particles_idx_offset+nb_particles <= Parent::getTotalNbParticles());
-
- static_assert(std::is_same<real_number, double>::value ||
- std::is_same<real_number, float>::value,
- "real_number must be double or float");
- const hid_t type_id = (sizeof(real_number) == 8 ? H5T_NATIVE_DOUBLE : H5T_NATIVE_FLOAT);
-
- hid_t plist_id = H5Pcreate(H5P_DATASET_XFER);
- assert(plist_id >= 0);
- {
- int rethdf = H5Pset_dxpl_mpio(plist_id, use_collective_io ? H5FD_MPIO_COLLECTIVE : H5FD_MPIO_INDEPENDENT);
- assert(rethdf >= 0);
- }
- {
- assert(size_particle_rhs >= 0);
- const hsize_t datacount[3] = {hsize_t(Parent::getNbRhs()),
- hsize_t(Parent::getTotalNbParticles()),
- hsize_t(size_particle_rhs)};
- hid_t dataspace = H5Screate_simple(3, datacount, NULL);
- assert(dataspace >= 0);
-
- hid_t dataset_id = H5Dcreate( pgroup_id,
- dataset_name.c_str(),
- type_id,
- dataspace,
- H5P_DEFAULT,
- H5P_DEFAULT,
- H5P_DEFAULT);
- assert(dataset_id >= 0);
-
- assert(particles_idx_offset >= 0);
- const hsize_t count[3] = {
- 1,
- hsize_t(nb_particles),
- hsize_t(size_particle_rhs)};
- const hsize_t offset[3] = {
- 0,
- hsize_t(particles_idx_offset),
- 0};
- hid_t memspace = H5Screate_simple(3, count, NULL);
- assert(memspace >= 0);
-
- hid_t filespace = H5Dget_space(dataset_id);
- assert(filespace >= 0);
- int rethdf = H5Sselect_hyperslab(
- filespace,
- H5S_SELECT_SET,
- offset,
- NULL,
- count,
- NULL);
- assert(rethdf >= 0);
-
- herr_t status = H5Dwrite(
- dataset_id,
- type_id,
- memspace,
- filespace,
- plist_id,
- particles_rhs[0].get());
- assert(status >= 0);
- rethdf = H5Sclose(filespace);
- assert(rethdf >= 0);
- rethdf = H5Sclose(memspace);
- assert(rethdf >= 0);
- rethdf = H5Dclose(dataset_id);
- assert(rethdf >= 0);
- }
-
- {
- int rethdf = H5Pclose(plist_id);
- assert(rethdf >= 0);
- }
- }
-};
-
-#endif
diff --git a/bfps/cpp/particles/particles_sampling.hpp b/bfps/cpp/particles/particles_sampling.hpp
deleted file mode 100644
index 3adc255341f3ca879d5cae1445124091f31b4394..0000000000000000000000000000000000000000
--- a/bfps/cpp/particles/particles_sampling.hpp
+++ /dev/null
@@ -1,52 +0,0 @@
-#ifndef PARTICLES_SAMPLING_HPP
-#define PARTICLES_SAMPLING_HPP
-
-#include <memory>
-#include <string>
-
-#include "abstract_particles_system.hpp"
-#include "particles_output_sampling_hdf5.hpp"
-
-#include "field.hpp"
-#include "kspace.hpp"
-
-
-template <class partsize_t, class particles_rnumber, class rnumber, field_backend be, field_components fc>
-void sample_from_particles_system(const field<rnumber, be, fc>& in_field, // a pointer to a field<rnumber, FFTW, fc>
- std::unique_ptr<abstract_particles_system<partsize_t, particles_rnumber>>& ps, // a pointer to an particles_system<double>
- const std::string& filename,
- const std::string& parent_groupname,
- const std::string& fname){
- const std::string datasetname = fname + std::string("/") + std::to_string(ps->get_step_idx());
- const int size_particle_rhs = ncomp(fc);
-
- // Stop here if already exists
- if(particles_output_sampling_hdf5<partsize_t, particles_rnumber, 3, size_particle_rhs>::DatasetExistsCol(MPI_COMM_WORLD,
- filename,
- parent_groupname,
- datasetname)){
- return;
- }
-
- const partsize_t nb_particles = ps->getLocalNbParticles();
- std::unique_ptr<particles_rnumber[]> sample_rhs(new particles_rnumber[size_particle_rhs*nb_particles]);
- std::fill_n(sample_rhs.get(), size_particle_rhs*nb_particles, 0);
-
- ps->sample_compute_field(in_field, sample_rhs.get());
-
-
-
- particles_output_sampling_hdf5<partsize_t, particles_rnumber, 3, size_particle_rhs> outputclass(MPI_COMM_WORLD,
- ps->getGlobalNbParticles(),
- filename,
- parent_groupname,
- datasetname);
- outputclass.save(ps->getParticlesPositions(),
- &sample_rhs,
- ps->getParticlesIndexes(),
- ps->getLocalNbParticles(),
- ps->get_step_idx());
-}
-
-#endif
-
diff --git a/bfps/cpp/particles/particles_system.hpp b/bfps/cpp/particles/particles_system.hpp
deleted file mode 100644
index 02767a8b433ecb8365f4a0577d1c0d6508c2bed1..0000000000000000000000000000000000000000
--- a/bfps/cpp/particles/particles_system.hpp
+++ /dev/null
@@ -1,259 +0,0 @@
-#ifndef PARTICLES_SYSTEM_HPP
-#define PARTICLES_SYSTEM_HPP
-
-#include <array>
-
-#include "abstract_particles_system.hpp"
-#include "particles_distr_mpi.hpp"
-#include "particles_output_hdf5.hpp"
-#include "particles_output_mpiio.hpp"
-#include "particles_field_computer.hpp"
-#include "abstract_particles_input.hpp"
-#include "particles_adams_bashforth.hpp"
-#include "scope_timer.hpp"
-
-template <class partsize_t, class real_number, class field_rnumber, class field_class, class interpolator_class, int interp_neighbours,
- int size_particle_rhs>
-class particles_system : public abstract_particles_system<partsize_t, real_number> {
- MPI_Comm mpi_com;
-
- const std::pair<int,int> current_partition_interval;
- const int partition_interval_size;
-
- interpolator_class interpolator;
-
- particles_distr_mpi<partsize_t, real_number> particles_distr;
-
- particles_adams_bashforth<partsize_t, real_number, 3, size_particle_rhs> positions_updater;
-
- using computer_class = particles_field_computer<partsize_t, real_number, interpolator_class, interp_neighbours>;
- computer_class computer;
-
- field_class default_field;
-
- std::unique_ptr<partsize_t[]> current_my_nb_particles_per_partition;
- std::unique_ptr<partsize_t[]> current_offset_particles_for_partition;
-
- const std::array<real_number,3> spatial_box_width;
- const std::array<real_number,3> spatial_partition_width;
- const real_number my_spatial_low_limit;
- const real_number my_spatial_up_limit;
-
- std::unique_ptr<real_number[]> my_particles_positions;
- std::unique_ptr<partsize_t[]> my_particles_positions_indexes;
- partsize_t my_nb_particles;
- const partsize_t total_nb_particles;
- std::vector<std::unique_ptr<real_number[]>> my_particles_rhs;
-
- int step_idx;
-
-public:
- particles_system(const std::array<size_t,3>& field_grid_dim, const std::array<real_number,3>& in_spatial_box_width,
- const std::array<real_number,3>& in_spatial_box_offset,
- const std::array<real_number,3>& in_spatial_partition_width,
- const real_number in_my_spatial_low_limit, const real_number in_my_spatial_up_limit,
- const std::array<size_t,3>& in_local_field_dims,
- const std::array<size_t,3>& in_local_field_offset,
- const field_class& in_field,
- MPI_Comm in_mpi_com,
- const partsize_t in_total_nb_particles,
- const int in_current_iteration = 1)
- : mpi_com(in_mpi_com),
- current_partition_interval({in_local_field_offset[IDX_Z], in_local_field_offset[IDX_Z] + in_local_field_dims[IDX_Z]}),
- partition_interval_size(current_partition_interval.second - current_partition_interval.first),
- interpolator(),
- particles_distr(in_mpi_com, current_partition_interval,field_grid_dim),
- positions_updater(),
- computer(field_grid_dim, current_partition_interval,
- interpolator, in_spatial_box_width, in_spatial_box_offset, in_spatial_partition_width),
- default_field(in_field),
- spatial_box_width(in_spatial_box_width), spatial_partition_width(in_spatial_partition_width),
- my_spatial_low_limit(in_my_spatial_low_limit), my_spatial_up_limit(in_my_spatial_up_limit),
- my_nb_particles(0), total_nb_particles(in_total_nb_particles), step_idx(in_current_iteration){
-
- current_my_nb_particles_per_partition.reset(new partsize_t[partition_interval_size]);
- current_offset_particles_for_partition.reset(new partsize_t[partition_interval_size+1]);
- }
-
- ~particles_system(){
- }
-
- void init(abstract_particles_input<partsize_t, real_number>& particles_input) {
- TIMEZONE("particles_system::init");
-
- my_particles_positions = particles_input.getMyParticles();
- my_particles_positions_indexes = particles_input.getMyParticlesIndexes();
- my_particles_rhs = particles_input.getMyRhs();
- my_nb_particles = particles_input.getLocalNbParticles();
-
- for(partsize_t idx_part = 0 ; idx_part < my_nb_particles ; ++idx_part){ // TODO remove me
- const int partition_level = computer.pbc_field_layer(my_particles_positions[idx_part*3+IDX_Z], IDX_Z);
- assert(partition_level >= current_partition_interval.first);
- assert(partition_level < current_partition_interval.second);
- }
-
- particles_utils::partition_extra_z<partsize_t, 3>(&my_particles_positions[0], my_nb_particles, partition_interval_size,
- current_my_nb_particles_per_partition.get(), current_offset_particles_for_partition.get(),
- [&](const real_number& z_pos){
- const int partition_level = computer.pbc_field_layer(z_pos, IDX_Z);
- assert(current_partition_interval.first <= partition_level && partition_level < current_partition_interval.second);
- return partition_level - current_partition_interval.first;
- },
- [&](const partsize_t idx1, const partsize_t idx2){
- std::swap(my_particles_positions_indexes[idx1], my_particles_positions_indexes[idx2]);
- for(int idx_rhs = 0 ; idx_rhs < int(my_particles_rhs.size()) ; ++idx_rhs){
- for(int idx_val = 0 ; idx_val < size_particle_rhs ; ++idx_val){
- std::swap(my_particles_rhs[idx_rhs][idx1*size_particle_rhs + idx_val],
- my_particles_rhs[idx_rhs][idx2*size_particle_rhs + idx_val]);
- }
- }
- });
-
- {// TODO remove
- for(int idxPartition = 0 ; idxPartition < partition_interval_size ; ++idxPartition){
- assert(current_my_nb_particles_per_partition[idxPartition] ==
- current_offset_particles_for_partition[idxPartition+1] - current_offset_particles_for_partition[idxPartition]);
- for(partsize_t idx = current_offset_particles_for_partition[idxPartition] ; idx < current_offset_particles_for_partition[idxPartition+1] ; ++idx){
- assert(computer.pbc_field_layer(my_particles_positions[idx*3+IDX_Z], IDX_Z)-current_partition_interval.first == idxPartition);
- }
- }
- }
- }
-
-
- void compute() final {
- TIMEZONE("particles_system::compute");
- particles_distr.template compute_distr<computer_class, field_class, 3, size_particle_rhs>(
- computer, default_field,
- current_my_nb_particles_per_partition.get(),
- my_particles_positions.get(),
- my_particles_rhs.front().get(),
- interp_neighbours);
- }
-
- template <class sample_field_class, int sample_size_particle_rhs>
- void sample_compute(const sample_field_class& sample_field,
- real_number sample_rhs[]) {
- TIMEZONE("particles_system::compute");
- particles_distr.template compute_distr<computer_class, sample_field_class, 3, sample_size_particle_rhs>(
- computer, sample_field,
- current_my_nb_particles_per_partition.get(),
- my_particles_positions.get(),
- sample_rhs,
- interp_neighbours);
- }
-
- //- Not generic to enable sampling begin
- void sample_compute_field(const field<float, FFTW, ONE>& sample_field,
- real_number sample_rhs[]) final {
- // sample_compute<decltype(sample_field), 1>(sample_field, sample_rhs);
- }
- void sample_compute_field(const field<float, FFTW, THREE>& sample_field,
- real_number sample_rhs[]) final {
- sample_compute<decltype(sample_field), 3>(sample_field, sample_rhs);
- }
- void sample_compute_field(const field<float, FFTW, THREExTHREE>& sample_field,
- real_number sample_rhs[]) final {
- sample_compute<decltype(sample_field), 9>(sample_field, sample_rhs);
- }
- void sample_compute_field(const field<double, FFTW, ONE>& sample_field,
- real_number sample_rhs[]) final {
- sample_compute<decltype(sample_field), 1>(sample_field, sample_rhs);
- }
- void sample_compute_field(const field<double, FFTW, THREE>& sample_field,
- real_number sample_rhs[]) final {
- sample_compute<decltype(sample_field), 3>(sample_field, sample_rhs);
- }
- void sample_compute_field(const field<double, FFTW, THREExTHREE>& sample_field,
- real_number sample_rhs[]) final {
- sample_compute<decltype(sample_field), 9>(sample_field, sample_rhs);
- }
- //- Not generic to enable sampling end
-
- void move(const real_number dt) final {
- TIMEZONE("particles_system::move");
- positions_updater.move_particles(my_particles_positions.get(), my_nb_particles,
- my_particles_rhs.data(), std::min(step_idx,int(my_particles_rhs.size())),
- dt);
- }
-
- void redistribute() final {
- TIMEZONE("particles_system::redistribute");
- particles_distr.template redistribute<computer_class, 3, size_particle_rhs, 1>(
- computer,
- current_my_nb_particles_per_partition.get(),
- &my_nb_particles,
- &my_particles_positions,
- my_particles_rhs.data(), int(my_particles_rhs.size()),
- &my_particles_positions_indexes);
- }
-
- void inc_step_idx() final {
- step_idx += 1;
- }
-
- int get_step_idx() const final {
- return step_idx;
- }
-
- void shift_rhs_vectors() final {
- if(my_particles_rhs.size()){
- std::unique_ptr<real_number[]> next_current(std::move(my_particles_rhs.back()));
- for(int idx_rhs = int(my_particles_rhs.size())-1 ; idx_rhs > 0 ; --idx_rhs){
- my_particles_rhs[idx_rhs] = std::move(my_particles_rhs[idx_rhs-1]);
- }
- my_particles_rhs[0] = std::move(next_current);
- particles_utils::memzero(my_particles_rhs[0], size_particle_rhs*my_nb_particles);
- }
- }
-
- void completeLoop(const real_number dt) final {
- TIMEZONE("particles_system::completeLoop");
- compute();
- move(dt);
- redistribute();
- inc_step_idx();
- shift_rhs_vectors();
- }
-
- const real_number* getParticlesPositions() const final {
- return my_particles_positions.get();
- }
-
- const std::unique_ptr<real_number[]>* getParticlesRhs() const final {
- return my_particles_rhs.data();
- }
-
- const partsize_t* getParticlesIndexes() const final {
- return my_particles_positions_indexes.get();
- }
-
- partsize_t getLocalNbParticles() const final {
- return my_nb_particles;
- }
-
- partsize_t getGlobalNbParticles() const final {
- return total_nb_particles;
- }
-
- int getNbRhs() const final {
- return int(my_particles_rhs.size());
- }
-
- void checkNan() const { // TODO remove
- for(partsize_t idx_part = 0 ; idx_part < my_nb_particles ; ++idx_part){ // TODO remove me
- assert(std::isnan(my_particles_positions[idx_part*3+IDX_X]) == false);
- assert(std::isnan(my_particles_positions[idx_part*3+IDX_Y]) == false);
- assert(std::isnan(my_particles_positions[idx_part*3+IDX_Z]) == false);
-
- for(int idx_rhs = 0 ; idx_rhs < my_particles_rhs.size() ; ++idx_rhs){
- for(int idx_rhs_val = 0 ; idx_rhs_val < size_particle_rhs ; ++idx_rhs_val){
- assert(std::isnan(my_particles_rhs[idx_rhs][idx_part*size_particle_rhs+idx_rhs_val]) == false);
- }
- }
- }
- }
-};
-
-
-#endif
diff --git a/bfps/cpp/particles_base.cpp b/bfps/cpp/particles_base.cpp
deleted file mode 100644
index 1410488410a429ff463a1751e86f78cc2157679b..0000000000000000000000000000000000000000
--- a/bfps/cpp/particles_base.cpp
+++ /dev/null
@@ -1,424 +0,0 @@
-/**********************************************************************
-* *
-* Copyright 2015 Max Planck Institute *
-* for Dynamics and Self-Organization *
-* *
-* This file is part of bfps. *
-* *
-* bfps is free software: you can redistribute it and/or modify *
-* it under the terms of the GNU General Public License as published *
-* by the Free Software Foundation, either version 3 of the License, *
-* or (at your option) any later version. *
-* *
-* bfps is distributed in the hope that it will be useful, *
-* but WITHOUT ANY WARRANTY; without even the implied warranty of *
-* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the *
-* GNU General Public License for more details. *
-* *
-* You should have received a copy of the GNU General Public License *
-* along with bfps. If not, see <http://www.gnu.org/licenses/> *
-* *
-* Contact: Cristian.Lalescu@ds.mpg.de *
-* *
-**********************************************************************/
-
-
-
-#define NDEBUG
-
-#include <algorithm>
-#include <cassert>
-#include "particles_base.hpp"
-#include "scope_timer.hpp"
-
-template <particle_types particle_type>
-single_particle_state<particle_type>::single_particle_state()
-{
- std::fill_n(this->data, state_dimension(particle_type), 0);
-}
-
-template <particle_types particle_type>
-single_particle_state<particle_type>::single_particle_state(
- const single_particle_state<particle_type> &src)
-{
- std::copy(
- src.data,
- src.data + state_dimension(particle_type),
- this->data);
-}
-
-template <particle_types particle_type>
-single_particle_state<particle_type>::single_particle_state(
- const double *src)
-{
- std::copy(
- src,
- src + state_dimension(particle_type),
- this->data);
-}
-
-template <particle_types particle_type>
-single_particle_state<particle_type>::~single_particle_state()
-{
-}
-
-template <particle_types particle_type>
-single_particle_state<particle_type> &single_particle_state<particle_type>::operator=(
- const single_particle_state &src)
-{
- std::copy(
- src.data,
- src.data + state_dimension(particle_type),
- this->data);
- return *this;
-}
-
-template <particle_types particle_type>
-single_particle_state<particle_type> &single_particle_state<particle_type>::operator=(
- const double *src)
-{
- std::copy(
- src,
- src + state_dimension(particle_type),
- this->data);
- return *this;
-}
-
-int get_chunk_offsets(
- std::vector<hsize_t> data_dims,
- std::vector<hsize_t> chnk_dims,
- std::vector<std::vector<hsize_t>> &co)
-{
- TIMEZONE("get_chunk_offsets");
- std::vector<hsize_t> nchunks(data_dims);
- int total_number_of_chunks = 1;
- for (unsigned i=0; i<nchunks.size(); i++)
- {
- DEBUG_MSG("get_chunk_offset nchunks[%d] = %ld, chnk_dims[%d] = %ld\n",
- i, nchunks[i], i, chnk_dims[i]);
- nchunks[i] = data_dims[i] / chnk_dims[i];
- total_number_of_chunks *= nchunks[i];
- }
- co.resize(total_number_of_chunks);
- DEBUG_MSG("total number of chunks is %d\n", total_number_of_chunks);
- for (int cindex=0; cindex < total_number_of_chunks; cindex++)
- {
- int cc = cindex;
- for (unsigned i=0; i<nchunks.size(); i++)
- {
- int ii = nchunks.size()-1-i;
- co[cindex].resize(nchunks.size());
- co[cindex][ii] = cc % nchunks[ii];
- cc = (cc - co[cindex][ii]) / nchunks[ii];
- co[cindex][ii] *= chnk_dims[ii];
- }
- }
- return EXIT_SUCCESS;
-}
-
-template <particle_types particle_type>
-particles_io_base<particle_type>::particles_io_base(
- const char *NAME,
- const int TRAJ_SKIP,
- const hid_t data_file_id,
- MPI_Comm COMM)
-{
- TIMEZONE("particles_io_base::particles_io_base");
- this->name = std::string(NAME);
- this->traj_skip = TRAJ_SKIP;
- this->comm = COMM;
- MPI_Comm_rank(COMM, &this->myrank);
- MPI_Comm_size(COMM, &this->nprocs);
-
- if (this->myrank == 0)
- {
- hid_t dset, prop_list, dspace;
- this->hdf5_group_id = H5Gopen(data_file_id, this->name.c_str(), H5P_DEFAULT);
- dset = H5Dopen(this->hdf5_group_id, "state", H5P_DEFAULT);
- dspace = H5Dget_space(dset);
- this->hdf5_state_dims.resize(H5Sget_simple_extent_ndims(dspace));
- H5Sget_simple_extent_dims(dspace, &this->hdf5_state_dims.front(), NULL);
- assert(this->hdf5_state_dims[this->hdf5_state_dims.size()-1] == state_dimension(particle_type));
- this->nparticles = 1;
- for (unsigned int i=1; i<this->hdf5_state_dims.size()-1; i++)
- this->nparticles *= this->hdf5_state_dims[i];
- prop_list = H5Dget_create_plist(dset);
- this->hdf5_state_chunks.resize(this->hdf5_state_dims.size());
- H5Pget_chunk(prop_list, this->hdf5_state_dims.size(), &this->hdf5_state_chunks.front());
- H5Pclose(prop_list);
- H5Sclose(dspace);
- H5Dclose(dset);
- this->chunk_size = 1;
- for (unsigned int i=1; i<this->hdf5_state_dims.size()-1; i++)
- this->chunk_size *= this->hdf5_state_chunks[i];
- dset = H5Dopen(this->hdf5_group_id, "rhs", H5P_DEFAULT);
- dspace = H5Dget_space(dset);
- this->hdf5_rhs_dims.resize(H5Sget_simple_extent_ndims(dspace));
- H5Sget_simple_extent_dims(dspace, &this->hdf5_rhs_dims.front(), NULL);
- prop_list = H5Dget_create_plist(dset);
- this->hdf5_rhs_chunks.resize(this->hdf5_rhs_dims.size());
- H5Pget_chunk(prop_list, this->hdf5_rhs_dims.size(), &this->hdf5_rhs_chunks.front());
- H5Pclose(prop_list);
- H5Sclose(dspace);
- H5Dclose(dset);
- }
- DEBUG_MSG("hello, rank 0 just read particle thingie\n");
-
- int tmp;
- tmp = this->hdf5_state_dims.size();
- MPI_Bcast(
- &tmp,
- 1,
- MPI_INTEGER,
- 0,
- this->comm);
- if (this->myrank != 0)
- {
- this->hdf5_state_dims.resize(tmp);
- this->hdf5_state_chunks.resize(tmp);
- }
- DEBUG_MSG("successfully resized state_dims and state_chunks\n");
- MPI_Bcast(
- &this->hdf5_state_dims.front(),
- this->hdf5_state_dims.size(),
- // hsize_t is in fact unsigned long long. Will this ever change...?
- MPI_UNSIGNED_LONG_LONG,
- 0,
- this->comm);
- MPI_Bcast(
- &this->hdf5_state_chunks.front(),
- this->hdf5_state_chunks.size(),
- MPI_UNSIGNED_LONG_LONG,
- 0,
- this->comm);
- DEBUG_MSG("successfully broadcasted state_dims and state_chunks\n");
- for (unsigned i=0; i<this->hdf5_state_chunks.size(); i++)
- DEBUG_MSG(
- "hdf5_state_dims[%d] = %ld, hdf5_state_chunks[%d] = %ld\n",
- i, this->hdf5_state_dims[i],
- i, this->hdf5_state_chunks[i]
- );
- std::vector<hsize_t> tdims(this->hdf5_state_dims), tchnk(this->hdf5_state_chunks);
- tdims.erase(tdims.begin()+0);
- tchnk.erase(tchnk.begin()+0);
- tdims.erase(tdims.end()-1);
- tchnk.erase(tchnk.end()-1);
- DEBUG_MSG("before get_chunk_offsets\n");
- get_chunk_offsets(tdims, tchnk, this->chunk_offsets);
- DEBUG_MSG("after get_chunk_offsets\n");
- MPI_Bcast(
- &this->chunk_size,
- 1,
- MPI_UNSIGNED,
- 0,
- this->comm);
- MPI_Bcast(
- &this->nparticles,
- 1,
- MPI_UNSIGNED,
- 0,
- this->comm);
- DEBUG_MSG("nparticles = %d, chunk_size = %d\n",
- this->nparticles,
- this->chunk_size);
- DEBUG_MSG("exiting particles_io_base constructor\n");
-}
-
-template <particle_types particle_type>
-particles_io_base<particle_type>::~particles_io_base()
-{
- if(this->myrank == 0)
- H5Gclose(this->hdf5_group_id);
-}
-
-template <particle_types particle_type>
-void particles_io_base<particle_type>::read_state_chunk(
- const int cindex,
- double *data)
-{
- TIMEZONE("particles_io_base::read_state_chunk");
- DEBUG_MSG("entered read_state_chunk\n");
- hid_t dset = H5Dopen(this->hdf5_group_id, "state", H5P_DEFAULT);
- hid_t rspace = H5Dget_space(dset);
- std::vector<hsize_t> mem_dims(this->hdf5_state_chunks);
- mem_dims[0] = 1;
- hid_t mspace = H5Screate_simple(
- this->hdf5_state_dims.size(),
- &mem_dims.front(),
- NULL);
- hsize_t *offset = new hsize_t[this->hdf5_state_dims.size()];
- offset[0] = this->iteration / this->traj_skip;
- for (unsigned int i=1; i<this->hdf5_state_dims.size()-1; i++)
- offset[i] = this->chunk_offsets[cindex][i-1];
- offset[this->hdf5_state_dims.size()-1] = 0;
- H5Sselect_hyperslab(
- rspace,
- H5S_SELECT_SET,
- offset,
- NULL,
- &mem_dims.front(),
- NULL);
- H5Dread(dset, H5T_NATIVE_DOUBLE, mspace, rspace, H5P_DEFAULT, data);
- H5Sclose(mspace);
- H5Sclose(rspace);
- H5Dclose(dset);
- delete[] offset;
- DEBUG_MSG("exiting read_state_chunk\n");
-}
-
-template <particle_types particle_type>
-void particles_io_base<particle_type>::write_state_chunk(
- const int cindex,
- const double *data)
-{
- TIMEZONE("particles_io_base::write_state_chunk");
- hid_t dset = H5Dopen(this->hdf5_group_id, "state", H5P_DEFAULT);
- hid_t rspace = H5Dget_space(dset);
- std::vector<hsize_t> mem_dims(this->hdf5_state_chunks);
- mem_dims[0] = 1;
- hid_t mspace = H5Screate_simple(
- this->hdf5_state_dims.size(),
- &mem_dims.front(),
- NULL);
- hsize_t *offset = new hsize_t[this->hdf5_state_dims.size()];
- offset[0] = this->iteration / this->traj_skip;
- for (unsigned int i=1; i<this->hdf5_state_dims.size()-1; i++)
- offset[i] = this->chunk_offsets[cindex][i-1];
- offset[this->hdf5_state_dims.size()-1] = 0;
- H5Sselect_hyperslab(
- rspace,
- H5S_SELECT_SET,
- offset,
- NULL,
- &mem_dims.front(),
- NULL);
- H5Dwrite(dset, H5T_NATIVE_DOUBLE, mspace, rspace, H5P_DEFAULT, data);
- H5Sclose(mspace);
- H5Sclose(rspace);
- H5Dclose(dset);
- delete[] offset;
-}
-
-template <particle_types particle_type>
-void particles_io_base<particle_type>::read_rhs_chunk(
- const int cindex,
- const int rhsindex,
- double *data)
-{
- TIMEZONE("particles_io_base::read_rhs_chunk");
- //DEBUG_MSG("entered read_rhs_chunk\n");
- hid_t dset = H5Dopen(this->hdf5_group_id, "rhs", H5P_DEFAULT);
- hid_t rspace = H5Dget_space(dset);
- std::vector<hsize_t> mem_dims(this->hdf5_rhs_chunks);
- mem_dims[0] = 1;
- mem_dims[1] = 1;
- hid_t mspace = H5Screate_simple(
- this->hdf5_rhs_dims.size(),
- &mem_dims.front(),
- NULL);
- hsize_t *offset = new hsize_t[this->hdf5_rhs_dims.size()];
- offset[0] = this->hdf5_rhs_dims[0]-2;
- offset[1] = rhsindex;
- for (unsigned int i=2; i<this->hdf5_rhs_dims.size()-1; i++)
- offset[i] = this->chunk_offsets[cindex][i-2];
- offset[this->hdf5_rhs_dims.size()-1] = 0;
- //for (int i=0; i<this->hdf5_rhs_dims.size(); i++)
- // DEBUG_MSG("rhs dim %d: size=%d chunk=%d offset=%d\n",
- // i, this->hdf5_rhs_dims[i], this->hdf5_rhs_chunks[i], offset[i]);
- H5Sselect_hyperslab(
- rspace,
- H5S_SELECT_SET,
- offset,
- NULL,
- &mem_dims.front(),
- NULL);
- //DEBUG_MSG("selected hyperslab\n");
- H5Dread(dset, H5T_NATIVE_DOUBLE, mspace, rspace, H5P_DEFAULT, data);
- //DEBUG_MSG("data has been read\n");
- H5Sclose(mspace);
- H5Sclose(rspace);
- H5Dclose(dset);
- delete[] offset;
- //DEBUG_MSG("exiting read_rhs_chunk\n");
-}
-
-template <particle_types particle_type>
-void particles_io_base<particle_type>::write_rhs_chunk(
- const int cindex,
- const int rhsindex,
- const double *data)
-{
- TIMEZONE("particles_io_base::write_rhs_chunk");
- hid_t dset = H5Dopen(this->hdf5_group_id, "rhs", H5P_DEFAULT);
- hid_t rspace = H5Dget_space(dset);
- std::vector<hsize_t> mem_dims(this->hdf5_rhs_chunks);
- mem_dims[0] = 1;
- mem_dims[1] = 1;
- hid_t mspace = H5Screate_simple(
- this->hdf5_rhs_dims.size(),
- &mem_dims.front(),
- NULL);
- hsize_t *offset = new hsize_t[this->hdf5_rhs_dims.size()];
- offset[0] = this->hdf5_rhs_dims[0]-1;
- offset[1] = rhsindex;
- for (unsigned int i=2; i<this->hdf5_rhs_dims.size()-1; i++)
- offset[i] = this->chunk_offsets[cindex][i-2];
- offset[this->hdf5_rhs_dims.size()-1] = 0;
- DEBUG_MSG("rhs write offsets are %d %d %d %d\n",
- offset[0], offset[1], offset[2], offset[3]);
- H5Sselect_hyperslab(
- rspace,
- H5S_SELECT_SET,
- offset,
- NULL,
- &mem_dims.front(),
- NULL);
- H5Dwrite(dset, H5T_NATIVE_DOUBLE, mspace, rspace, H5P_DEFAULT, data);
- H5Sclose(mspace);
- H5Sclose(rspace);
- H5Dclose(dset);
- delete[] offset;
-}
-
-template <particle_types particle_type>
-void particles_io_base<particle_type>::write_point3D_chunk(
- const std::string dset_name,
- const int cindex,
- const double *data)
-{
- TIMEZONE("particles_io_base::write_point3D_chunk");
- hid_t dset = H5Dopen(this->hdf5_group_id, dset_name.c_str(), H5P_DEFAULT);
- hid_t rspace = H5Dget_space(dset);
- std::vector<hsize_t> mem_dims(this->hdf5_state_chunks);
- mem_dims[0] = 1;
- mem_dims[mem_dims.size()-1] = 3;
- hid_t mspace = H5Screate_simple(
- this->hdf5_state_dims.size(),
- &mem_dims.front(),
- NULL);
- hsize_t *offset = new hsize_t[this->hdf5_state_dims.size()];
- offset[0] = this->iteration / this->traj_skip;
- for (unsigned int i=1; i<this->hdf5_state_dims.size()-1; i++)
- offset[i] = this->chunk_offsets[cindex][i-1];
- offset[this->hdf5_state_dims.size()-1] = 0;
- H5Sselect_hyperslab(
- rspace,
- H5S_SELECT_SET,
- offset,
- NULL,
- &mem_dims.front(),
- NULL);
- H5Dwrite(dset, H5T_NATIVE_DOUBLE, mspace, rspace, H5P_DEFAULT, data);
- H5Sclose(mspace);
- H5Sclose(rspace);
- H5Dclose(dset);
- delete[] offset;
-}
-
-/*****************************************************************************/
-template class single_particle_state<POINT3D>;
-template class single_particle_state<VELOCITY_TRACER>;
-
-template class particles_io_base<VELOCITY_TRACER>;
-/*****************************************************************************/
-
diff --git a/bfps/cpp/particles_base.hpp b/bfps/cpp/particles_base.hpp
deleted file mode 100644
index 8afd5d439cdc121982868b5eadc991cdc1c5abdb..0000000000000000000000000000000000000000
--- a/bfps/cpp/particles_base.hpp
+++ /dev/null
@@ -1,136 +0,0 @@
-/**********************************************************************
-* *
-* Copyright 2015 Max Planck Institute *
-* for Dynamics and Self-Organization *
-* *
-* This file is part of bfps. *
-* *
-* bfps is free software: you can redistribute it and/or modify *
-* it under the terms of the GNU General Public License as published *
-* by the Free Software Foundation, either version 3 of the License, *
-* or (at your option) any later version. *
-* *
-* bfps is distributed in the hope that it will be useful, *
-* but WITHOUT ANY WARRANTY; without even the implied warranty of *
-* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the *
-* GNU General Public License for more details. *
-* *
-* You should have received a copy of the GNU General Public License *
-* along with bfps. If not, see <http://www.gnu.org/licenses/> *
-* *
-* Contact: Cristian.Lalescu@ds.mpg.de *
-* *
-**********************************************************************/
-
-
-
-#include <vector>
-#include <hdf5.h>
-#include <unordered_map>
-#include "interpolator_base.hpp"
-
-#ifndef PARTICLES_BASE
-
-#define PARTICLES_BASE
-
-/* particle types */
-enum particle_types {POINT3D, VELOCITY_TRACER};
-
-/* space dimension */
-constexpr unsigned int state_dimension(particle_types particle_type)
-{
- return ((particle_type == POINT3D) ? 3 : (
- (particle_type == VELOCITY_TRACER) ? 3 :
- 3));
-}
-
-/* 1 particle state type */
-
-template <particle_types particle_type>
-class single_particle_state
-{
- public:
- double data[state_dimension(particle_type)];
-
- single_particle_state();
- single_particle_state(const single_particle_state &src);
- single_particle_state(const double *src);
- ~single_particle_state();
-
- single_particle_state<particle_type> &operator=(const single_particle_state &src);
- single_particle_state<particle_type> &operator=(const double *src);
-
- inline double &operator[](const int i)
- {
- return this->data[i];
- }
-};
-
-std::vector<std::vector<hsize_t>> get_chunk_offsets(
- std::vector<hsize_t> data_dims,
- std::vector<hsize_t> chnk_dims);
-
-template <particle_types particle_type>
-class particles_io_base
-{
- protected:
- int myrank, nprocs;
- MPI_Comm comm;
-
- unsigned int nparticles;
-
- std::string name;
- unsigned int chunk_size;
- int traj_skip;
-
- hid_t hdf5_group_id;
- std::vector<hsize_t> hdf5_state_dims, hdf5_state_chunks;
- std::vector<hsize_t> hdf5_rhs_dims, hdf5_rhs_chunks;
-
- std::vector<std::vector<hsize_t>> chunk_offsets;
-
- particles_io_base(
- const char *NAME,
- const int TRAJ_SKIP,
- const hid_t data_file_id,
- MPI_Comm COMM);
- virtual ~particles_io_base();
-
- void read_state_chunk(
- const int cindex,
- double *__restrict__ data);
- void write_state_chunk(
- const int cindex,
- const double *data);
- void read_rhs_chunk(
- const int cindex,
- const int rhsindex,
- double *__restrict__ data);
- void write_rhs_chunk(
- const int cindex,
- const int rhsindex,
- const double *data);
-
- void write_point3D_chunk(
- const std::string dset_name,
- const int cindex,
- const double *data);
-
- public:
- int iteration;
-
- inline const char *get_name()
- {
- return this->name.c_str();
- }
- inline const unsigned int get_number_of_chunks()
- {
- return this->chunk_offsets.size();
- }
- inline const unsigned int get_number_of_rhs_chunks();
- virtual void read() = 0;
- virtual void write(const bool write_rhs = true) = 0;
-};
-
-#endif//PARTICLES_BASE
-
diff --git a/bfps/cpp/rFFTW_distributed_particles.cpp b/bfps/cpp/rFFTW_distributed_particles.cpp
deleted file mode 100644
index 265975f8c817a1b40942e076bd016c2921618bbc..0000000000000000000000000000000000000000
--- a/bfps/cpp/rFFTW_distributed_particles.cpp
+++ /dev/null
@@ -1,804 +0,0 @@
-/**********************************************************************
-* *
-* Copyright 2015 Max Planck Institute *
-* for Dynamics and Self-Organization *
-* *
-* This file is part of bfps. *
-* *
-* bfps is free software: you can redistribute it and/or modify *
-* it under the terms of the GNU General Public License as published *
-* by the Free Software Foundation, either version 3 of the License, *
-* or (at your option) any later version. *
-* *
-* bfps is distributed in the hope that it will be useful, *
-* but WITHOUT ANY WARRANTY; without even the implied warranty of *
-* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the *
-* GNU General Public License for more details. *
-* *
-* You should have received a copy of the GNU General Public License *
-* along with bfps. If not, see <http://www.gnu.org/licenses/> *
-* *
-* Contact: Cristian.Lalescu@ds.mpg.de *
-* *
-**********************************************************************/
-
-
-
-#define NDEBUG
-
-#include <cmath>
-#include <cassert>
-#include <cstring>
-#include <string>
-#include <sstream>
-#include <set>
-#include <algorithm>
-#include <ctime>
-
-#include "base.hpp"
-#include "rFFTW_distributed_particles.hpp"
-#include "fftw_tools.hpp"
-#include "scope_timer.hpp"
-
-
-extern int myrank, nprocs;
-
-template <particle_types particle_type, class rnumber, int interp_neighbours>
-rFFTW_distributed_particles<particle_type, rnumber, interp_neighbours>::rFFTW_distributed_particles(
- const char *NAME,
- const hid_t data_file_id,
- rFFTW_interpolator<rnumber, interp_neighbours> *VEL,
- const int TRAJ_SKIP,
- const int INTEGRATION_STEPS) : particles_io_base<particle_type>(
- NAME,
- TRAJ_SKIP,
- data_file_id,
- VEL->descriptor->comm)
-{
- TIMEZONE("rFFTW_distributed_particles::rFFTW_distributed_particles");
- /* check that integration_steps has a valid value.
- * If NDEBUG is defined, "assert" doesn't do anything.
- * With NDEBUG defined, and an invalid INTEGRATION_STEPS,
- * the particles will simply sit still.
- * */
- assert((INTEGRATION_STEPS <= 6) &&
- (INTEGRATION_STEPS >= 1));
- /* check that the field layout is compatible with this class.
- * if it's not, the code will fail in bad ways, most likely ending up
- * with various CPUs locked in some MPI send/receive.
- * therefore I prefer to just kill the code at this point,
- * no matter whether or not NDEBUG is present.
- * */
- if (interp_neighbours*2+2 > VEL->descriptor->subsizes[0])
- {
- DEBUG_MSG("parameters incompatible with rFFTW_distributed_particles.\n"
- "interp kernel size is %d, local_z_size is %d\n",
- interp_neighbours*2+2, VEL->descriptor->subsizes[0]);
- if (VEL->descriptor->myrank == 0)
- std::cerr << "parameters incompatible with rFFTW_distributed_particles." << std::endl;
- exit(0);
- }
- this->vel = VEL;
- this->rhs.resize(INTEGRATION_STEPS);
- this->integration_steps = INTEGRATION_STEPS;
- /* the particles are expected to be evenly distributed among processes.
- * therefore allocating twice that amount of memory seems enough.
- * */
- this->state.reserve(2*this->nparticles / this->nprocs);
- for (unsigned int i=0; i<this->rhs.size(); i++)
- this->rhs[i].reserve(2*this->nparticles / this->nprocs);
-
- /* build communicators and stuff for interpolation */
-
- /* number of processors per domain */
- this->domain_nprocs[-1] = 2; // domain in common with lower z CPU
- this->domain_nprocs[ 0] = 1; // local domain
- this->domain_nprocs[ 1] = 2; // domain in common with higher z CPU
-
- /* initialize domain bins */
- this->domain_particles[-1] = std::unordered_set<int>();
- this->domain_particles[ 0] = std::unordered_set<int>();
- this->domain_particles[ 1] = std::unordered_set<int>();
- this->domain_particles[-1].reserve(unsigned(
- 1.5*(interp_neighbours*2+2)*
- float(this->nparticles) /
- this->nprocs));
- this->domain_particles[ 1].reserve(unsigned(
- 1.5*(interp_neighbours*2+2)*
- float(this->nparticles) /
- this->nprocs));
- this->domain_particles[ 0].reserve(unsigned(
- 1.5*(this->vel->descriptor->subsizes[0] - interp_neighbours*2-2)*
- float(this->nparticles) /
- this->nprocs));
-
- int color, key;
- MPI_Comm tmpcomm;
- for (int rank=0; rank<this->nprocs; rank++)
- {
- color = MPI_UNDEFINED;
- key = MPI_UNDEFINED;
- if (this->myrank == rank)
- {
- color = rank;
- key = 0;
- }
- if (this->myrank == MOD(rank + 1, this->nprocs))
- {
- color = rank;
- key = 1;
- }
- MPI_Comm_split(this->comm, color, key, &tmpcomm);
- if (this->myrank == rank)
- this->domain_comm[ 1] = tmpcomm;
- if (this->myrank == MOD(rank+1, this->nprocs))
- this->domain_comm[-1] = tmpcomm;
-
- }
-
- /* following code may be useful in the future for the general case */
- //this->interp_comm.resize(this->vel->descriptor->sizes[0]);
- //this->interp_nprocs.resize(this->vel->descriptor->sizes[0]);
- //for (int zg=0; zg<this->vel->descriptor->sizes[0]; zg++)
- //{
- // color = (this->vel->get_rank_info(
- // (zg+.5)*this->vel->dz, rminz, rmaxz) ? zg : MPI_UNDEFINED);
- // key = zg - this->vel->descriptor->starts[0] + interp_neighbours;
- // MPI_Comm_split(this->comm, color, key, &this->interp_comm[zg]);
- // if (this->interp_comm[zg] != MPI_COMM_NULL)
- // MPI_Comm_size(this->interp_comm[zg], &this->interp_nprocs[zg]);
- // else
- // this->interp_nprocs[zg] = 0;
- //}
-}
-
-template <particle_types particle_type, class rnumber, int interp_neighbours>
-rFFTW_distributed_particles<particle_type, rnumber, interp_neighbours>::~rFFTW_distributed_particles()
-{
-}
-
-template <particle_types particle_type, class rnumber, int interp_neighbours>
-void rFFTW_distributed_particles<particle_type, rnumber, interp_neighbours>::sample(
- rFFTW_interpolator<rnumber, interp_neighbours> *field,
- const std::unordered_map<int, single_particle_state<particle_type>> &x,
- const std::unordered_map<int, std::unordered_set<int>> &dp,
- std::unordered_map<int, single_particle_state<POINT3D>> &y)
-{
- TIMEZONE("rFFTW_distributed_particles::sample");
- double *yyy;
- double *yy;
- y.clear();
- /* local z domain */
- yy = new double[3];
- for (auto p: dp.at(0))
- {
- (*field)(x.find(p)->second.data, yy);
- y[p] = yy;
- }
- delete[] yy;
- /* boundary z domains */
- int domain_index;
- for (int rankpair = 0; rankpair < this->nprocs; rankpair++)
- {
- if (this->myrank == rankpair)
- domain_index = 1;
- if (this->myrank == MOD(rankpair+1, this->nprocs))
- domain_index = -1;
- if (this->myrank == rankpair ||
- this->myrank == MOD(rankpair+1, this->nprocs))
- {
- yy = new double[3*dp.at(domain_index).size()];
- yyy = new double[3*dp.at(domain_index).size()];
- int tindex;
- tindex = 0;
- // can this sorting be done more efficiently?
- std::vector<int> ordered_dp;
- {
- TIMEZONE("rFFTW_distributed_particles::sample::ordered_dp");
- ordered_dp.reserve(dp.at(domain_index).size());
- for (auto p: dp.at(domain_index))
- ordered_dp.push_back(p);
- //std::set<int> ordered_dp(dp.at(domain_index));
- std::sort(ordered_dp.begin(), ordered_dp.end());
- }
-
- for (auto p: ordered_dp)
- //for (auto p: dp.at(domain_index))
- {
- (*field)(x.at(p).data, yy + tindex*3);
- tindex++;
- }
- {
- TIMEZONE("rFFTW_distributed_particles::sample::MPI_Allreduce");
- MPI_Allreduce(
- yy,
- yyy,
- 3*dp.at(domain_index).size(),
- MPI_DOUBLE,
- MPI_SUM,
- this->domain_comm[domain_index]);
- }
- tindex = 0;
- for (auto p: ordered_dp)
- //for (auto p: dp.at(domain_index))
- {
- y[p] = yyy + tindex*3;
- tindex++;
- }
- delete[] yy;
- delete[] yyy;
- }
- }
-}
-
-template <particle_types particle_type, class rnumber, int interp_neighbours>
-void rFFTW_distributed_particles<particle_type, rnumber, interp_neighbours>::get_rhs(
- const std::unordered_map<int, single_particle_state<particle_type>> &x,
- const std::unordered_map<int, std::unordered_set<int>> &dp,
- std::unordered_map<int, single_particle_state<particle_type>> &y)
-{
- std::unordered_map<int, single_particle_state<POINT3D>> yy;
- switch(particle_type)
- {
- case VELOCITY_TRACER:
- this->sample(this->vel, x, dp, yy);
- y.clear();
- y.reserve(yy.size());
- y.rehash(this->nparticles);
- for (auto &pp: yy)
- y[pp.first] = pp.second.data;
- break;
- }
-}
-
-template <particle_types particle_type, class rnumber, int interp_neighbours>
-void rFFTW_distributed_particles<particle_type, rnumber, interp_neighbours>::sample(
- rFFTW_interpolator<rnumber, interp_neighbours> *field,
- const char *dset_name)
-{
- std::unordered_map<int, single_particle_state<POINT3D>> y;
- this->sample(field, this->state, this->domain_particles, y);
- this->write(dset_name, y);
-}
-
-template <particle_types particle_type, class rnumber, int interp_neighbours>
-void rFFTW_distributed_particles<particle_type, rnumber, interp_neighbours>::roll_rhs()
-{
- for (int i=this->integration_steps-2; i>=0; i--)
- rhs[i+1] = rhs[i];
-}
-
-template <particle_types particle_type, class rnumber, int interp_neighbours>
-void rFFTW_distributed_particles<particle_type, rnumber, interp_neighbours>::redistribute(
- std::unordered_map<int, single_particle_state<particle_type>> &x,
- std::vector<std::unordered_map<int, single_particle_state<particle_type>>> &vals,
- std::unordered_map<int, std::unordered_set<int>> &dp)
-{
- TIMEZONE("rFFTW_distributed_particles::redistribute");
- //DEBUG_MSG("entered redistribute\n");
- /* get new distribution of particles */
- std::unordered_map<int, std::unordered_set<int>> newdp;
- {
- TIMEZONE("sort_into_domains");
- this->sort_into_domains(x, newdp);
- }
- /* take care of particles that are leaving the shared domains */
- int dindex[2] = {-1, 1};
- // for each D of the 2 shared domains
- {
- TIMEZONE("Loop1");
- for (int di=0; di<2; di++)
- // for all particles previously in D
- for (auto p: dp[dindex[di]])
- {
- // if the particle is no longer in D
- if (newdp[dindex[di]].find(p) == newdp[dindex[di]].end())
- {
- // and the particle is not in the local domain
- if (newdp[0].find(p) == newdp[0].end())
- {
- // remove the particle from the local list
- x.erase(p);
- for (unsigned int i=0; i<vals.size(); i++)
- vals[i].erase(p);
- }
- // if the particle is in the local domain, do nothing
- }
- }
- }
- /* take care of particles that are entering the shared domains */
- /* neighbouring rank offsets */
- int ro[2];
- ro[0] = -1;
- ro[1] = 1;
- /* particles to send, particles to receive */
- std::vector<int> ps[2], pr[2];
- for (int tcounter = 0; tcounter < 2; tcounter++)
- {
- ps[tcounter].reserve(newdp[dindex[tcounter]].size());
- }
- /* number of particles to send, number of particles to receive */
- int nps[2], npr[2];
- int rsrc, rdst;
- /* get list of id-s to send */
- {
- TIMEZONE("Loop2");
- for (auto &p: dp[0])
- {
- for (int di=0; di<2; di++)
- {
- if (newdp[dindex[di]].find(p) != newdp[dindex[di]].end())
- ps[di].push_back(p);
- }
- }
- }
- /* prepare data for send recv */
- for (int i=0; i<2; i++)
- nps[i] = ps[i].size();
- for (rsrc = 0; rsrc<this->nprocs; rsrc++)
- for (int i=0; i<2; i++)
- {
- rdst = MOD(rsrc+ro[i], this->nprocs);
- if (this->myrank == rsrc){
- TIMEZONE("MPI_Send");
- MPI_Send(
- nps+i,
- 1,
- MPI_INTEGER,
- rdst,
- 2*(rsrc*this->nprocs + rdst)+i,
- this->comm);
- }
- if (this->myrank == rdst){
- TIMEZONE("MPI_Recv");
- MPI_Recv(
- npr+1-i,
- 1,
- MPI_INTEGER,
- rsrc,
- 2*(rsrc*this->nprocs + rdst)+i,
- this->comm,
- MPI_STATUS_IGNORE);
- }
- }
- //DEBUG_MSG("I have to send %d %d particles\n", nps[0], nps[1]);
- //DEBUG_MSG("I have to recv %d %d particles\n", npr[0], npr[1]);
- for (int i=0; i<2; i++)
- pr[i].resize(npr[i]);
-
- int buffer_size = (nps[0] > nps[1]) ? nps[0] : nps[1];
- buffer_size = (buffer_size > npr[0])? buffer_size : npr[0];
- buffer_size = (buffer_size > npr[1])? buffer_size : npr[1];
- //DEBUG_MSG("buffer size is %d\n", buffer_size);
- double *buffer = new double[buffer_size*state_dimension(particle_type)*(1+vals.size())];
- for (rsrc = 0; rsrc<this->nprocs; rsrc++)
- for (int i=0; i<2; i++)
- {
- rdst = MOD(rsrc+ro[i], this->nprocs);
- if (this->myrank == rsrc && nps[i] > 0)
- {
- TIMEZONE("this->myrank == rsrc && nps[i] > 0");
- MPI_Send(
- &ps[i].front(),
- nps[i],
- MPI_INTEGER,
- rdst,
- 2*(rsrc*this->nprocs + rdst),
- this->comm);
- int pcounter = 0;
- for (int p: ps[i])
- {
- std::copy(x[p].data,
- x[p].data + state_dimension(particle_type),
- buffer + pcounter*(1+vals.size())*state_dimension(particle_type));
- for (unsigned int tindex=0; tindex<vals.size(); tindex++)
- {
- std::copy(vals[tindex][p].data,
- vals[tindex][p].data + state_dimension(particle_type),
- buffer + (pcounter*(1+vals.size()) + tindex+1)*state_dimension(particle_type));
- }
- pcounter++;
- }
- MPI_Send(
- buffer,
- nps[i]*(1+vals.size())*state_dimension(particle_type),
- MPI_DOUBLE,
- rdst,
- 2*(rsrc*this->nprocs + rdst)+1,
- this->comm);
- }
- if (this->myrank == rdst && npr[1-i] > 0)
- {
- TIMEZONE("this->myrank == rdst && npr[1-i] > 0");
- MPI_Recv(
- &pr[1-i].front(),
- npr[1-i],
- MPI_INTEGER,
- rsrc,
- 2*(rsrc*this->nprocs + rdst),
- this->comm,
- MPI_STATUS_IGNORE);
- MPI_Recv(
- buffer,
- npr[1-i]*(1+vals.size())*state_dimension(particle_type),
- MPI_DOUBLE,
- rsrc,
- 2*(rsrc*this->nprocs + rdst)+1,
- this->comm,
- MPI_STATUS_IGNORE);
- int pcounter = 0;
- for (int p: pr[1-i])
- {
- x[p] = buffer + (pcounter*(1+vals.size()))*state_dimension(particle_type);
- newdp[1-i].insert(p);
- for (unsigned int tindex=0; tindex<vals.size(); tindex++)
- {
- vals[tindex][p] = buffer + (pcounter*(1+vals.size()) + tindex+1)*state_dimension(particle_type);
- }
- pcounter++;
- }
- }
- }
- delete[] buffer;
- // x has been changed, so newdp is obsolete
- // we need to sort into domains again
- {
- TIMEZONE("sort_into_domains2");
- this->sort_into_domains(x, dp);
- }
-
-#ifndef NDEBUG
- /* check that all particles at x are local */
- //for (auto &pp: x)
- // if (this->vel->get_rank(pp.second.data[2]) != this->myrank)
- // {
- // DEBUG_MSG("found particle %d with rank %d\n",
- // pp.first,
- // this->vel->get_rank(pp.second.data[2]));
- // assert(false);
- // }
-#endif
- //DEBUG_MSG("exiting redistribute\n");
-}
-
-
-
-template <particle_types particle_type, class rnumber, int interp_neighbours>
-void rFFTW_distributed_particles<particle_type, rnumber, interp_neighbours>::AdamsBashforth(
- const int nsteps)
-{
- this->get_rhs(this->state, this->domain_particles, this->rhs[0]);
-#define AdamsBashforth_LOOP_PREAMBLE \
- for (auto &pp: this->state) \
- for (unsigned int i=0; i<state_dimension(particle_type); i++)
- switch(nsteps)
- {
- case 1:
- AdamsBashforth_LOOP_PREAMBLE
- pp.second[i] += this->dt*this->rhs[0][pp.first][i];
- break;
- case 2:
- AdamsBashforth_LOOP_PREAMBLE
- pp.second[i] += this->dt*(3*this->rhs[0][pp.first][i]
- - this->rhs[1][pp.first][i])/2;
- break;
- case 3:
- AdamsBashforth_LOOP_PREAMBLE
- pp.second[i] += this->dt*(23*this->rhs[0][pp.first][i]
- - 16*this->rhs[1][pp.first][i]
- + 5*this->rhs[2][pp.first][i])/12;
- break;
- case 4:
- AdamsBashforth_LOOP_PREAMBLE
- pp.second[i] += this->dt*(55*this->rhs[0][pp.first][i]
- - 59*this->rhs[1][pp.first][i]
- + 37*this->rhs[2][pp.first][i]
- - 9*this->rhs[3][pp.first][i])/24;
- break;
- case 5:
- AdamsBashforth_LOOP_PREAMBLE
- pp.second[i] += this->dt*(1901*this->rhs[0][pp.first][i]
- - 2774*this->rhs[1][pp.first][i]
- + 2616*this->rhs[2][pp.first][i]
- - 1274*this->rhs[3][pp.first][i]
- + 251*this->rhs[4][pp.first][i])/720;
- break;
- case 6:
- AdamsBashforth_LOOP_PREAMBLE
- pp.second[i] += this->dt*(4277*this->rhs[0][pp.first][i]
- - 7923*this->rhs[1][pp.first][i]
- + 9982*this->rhs[2][pp.first][i]
- - 7298*this->rhs[3][pp.first][i]
- + 2877*this->rhs[4][pp.first][i]
- - 475*this->rhs[5][pp.first][i])/1440;
- break;
- }
- this->redistribute(this->state, this->rhs, this->domain_particles);
- this->roll_rhs();
-}
-
-
-template <particle_types particle_type, class rnumber, int interp_neighbours>
-void rFFTW_distributed_particles<particle_type, rnumber, interp_neighbours>::step()
-{
- TIMEZONE("rFFTW_distributed_particles::step");
- this->AdamsBashforth((this->iteration < this->integration_steps) ?
- this->iteration+1 :
- this->integration_steps);
- this->iteration++;
-}
-
-
-template <particle_types particle_type, class rnumber, int interp_neighbours>
-void rFFTW_distributed_particles<particle_type, rnumber, interp_neighbours>::sort_into_domains(
- const std::unordered_map<int, single_particle_state<particle_type>> &x,
- std::unordered_map<int, std::unordered_set<int>> &dp)
-{
- TIMEZONE("rFFTW_distributed_particles::sort_into_domains");
- int tmpint1, tmpint2;
- dp.clear();
- dp[-1] = std::unordered_set<int>();
- dp[ 0] = std::unordered_set<int>();
- dp[ 1] = std::unordered_set<int>();
- dp[-1].reserve(unsigned(
- 1.5*(interp_neighbours*2+2)*
- float(this->nparticles) /
- this->nprocs));
- dp[ 1].reserve(unsigned(
- 1.5*(interp_neighbours*2+2)*
- float(this->nparticles) /
- this->nprocs));
- dp[ 0].reserve(unsigned(
- 1.5*(this->vel->descriptor->subsizes[0] - interp_neighbours*2-2)*
- float(this->nparticles) /
- this->nprocs));
- for (auto &xx: x)
- {
- if (this->vel->get_rank_info(xx.second.data[2], tmpint1, tmpint2))
- {
- if (tmpint1 == tmpint2)
- dp[0].insert(xx.first);
- else
- {
- if (this->myrank == tmpint1)
- dp[-1].insert(xx.first);
- else
- dp[ 1].insert(xx.first);
- }
- }
- }
-}
-
-
-template <particle_types particle_type, class rnumber, int interp_neighbours>
-void rFFTW_distributed_particles<particle_type, rnumber, interp_neighbours>::read()
-{
- TIMEZONE("rFFTW_distributed_particles::read");
- double *temp = new double[this->chunk_size*state_dimension(particle_type)];
- int tmpint1, tmpint2;
- for (unsigned int cindex=0; cindex<this->get_number_of_chunks(); cindex++)
- {
- //read state
- if (this->myrank == 0){
- TIMEZONE("read_state_chunk");
- this->read_state_chunk(cindex, temp);
- }
- {
- TIMEZONE("MPI_Bcast");
- MPI_Bcast(
- temp,
- this->chunk_size*state_dimension(particle_type),
- MPI_DOUBLE,
- 0,
- this->comm);
- }
- for (unsigned int p=0; p<this->chunk_size; p++)
- {
- if (this->vel->get_rank_info(temp[state_dimension(particle_type)*p+2], tmpint1, tmpint2))
- {
- this->state[p+cindex*this->chunk_size] = temp + state_dimension(particle_type)*p;
- }
- }
- //read rhs
- if (this->iteration > 0){
- TIMEZONE("this->iteration > 0");
- for (int i=0; i<this->integration_steps; i++)
- {
- if (this->myrank == 0){
- TIMEZONE("read_rhs_chunk");
- this->read_rhs_chunk(cindex, i, temp);
- }
- {
- TIMEZONE("MPI_Bcast");
- MPI_Bcast(
- temp,
- this->chunk_size*state_dimension(particle_type),
- MPI_DOUBLE,
- 0,
- this->comm);
- }
- for (unsigned int p=0; p<this->chunk_size; p++)
- {
- auto pp = this->state.find(p+cindex*this->chunk_size);
- if (pp != this->state.end())
- this->rhs[i][p+cindex*this->chunk_size] = temp + state_dimension(particle_type)*p;
- }
- }
- }
- }
- this->sort_into_domains(this->state, this->domain_particles);
- DEBUG_MSG("%s->state.size = %ld\n", this->name.c_str(), this->state.size());
- for (int domain=-1; domain<=1; domain++)
- {
- DEBUG_MSG("domain %d nparticles = %ld\n", domain, this->domain_particles[domain].size());
- }
- delete[] temp;
-}
-
-template <particle_types particle_type, class rnumber, int interp_neighbours>
-void rFFTW_distributed_particles<particle_type, rnumber, interp_neighbours>::write(
- const char *dset_name,
- std::unordered_map<int, single_particle_state<POINT3D>> &y)
-{
- TIMEZONE("rFFTW_distributed_particles::write");
- double *data = new double[this->chunk_size*3];
- double *yy = new double[this->chunk_size*3];
- //int pindex = 0;
- for (unsigned int cindex=0; cindex<this->get_number_of_chunks(); cindex++)
- {
- std::fill_n(yy, this->chunk_size*3, 0);
- //for (unsigned int p=0; p<this->chunk_size; p++, pindex++)
- //{
- // if (this->domain_particles[-1].find(pindex) != this->domain_particles[-1].end() ||
- // this->domain_particles[ 0].find(pindex) != this->domain_particles[ 0].end())
- // {
- // std::copy(y[pindex].data,
- // y[pindex].data + 3,
- // yy + p*3);
- // }
- //}
- for (int s = -1; s <= 0; s++)
- for (auto &pp: this->domain_particles[s])
- {
- if (pp >= int(cindex*this->chunk_size) &&
- pp < int((cindex+1)*this->chunk_size))
- {
- std::copy(y[pp].data,
- y[pp].data + 3,
- yy + (pp-cindex*this->chunk_size)*3);
- }
- }
- {
- TIMEZONE("MPI_Allreduce");
- MPI_Allreduce(
- yy,
- data,
- 3*this->chunk_size,
- MPI_DOUBLE,
- MPI_SUM,
- this->comm);
- }
- if (this->myrank == 0){
- TIMEZONE("write_point3D_chunk");
- this->write_point3D_chunk(dset_name, cindex, data);
- }
- }
- delete[] yy;
- delete[] data;
-}
-
-template <particle_types particle_type, class rnumber, int interp_neighbours>
-void rFFTW_distributed_particles<particle_type, rnumber, interp_neighbours>::write(
- const bool write_rhs)
-{
- TIMEZONE("rFFTW_distributed_particles::write2");
- double *temp0 = new double[this->chunk_size*state_dimension(particle_type)];
- double *temp1 = new double[this->chunk_size*state_dimension(particle_type)];
- //int pindex = 0;
- for (unsigned int cindex=0; cindex<this->get_number_of_chunks(); cindex++)
- {
- //write state
- std::fill_n(temp0, state_dimension(particle_type)*this->chunk_size, 0);
- //pindex = cindex*this->chunk_size;
- //for (unsigned int p=0; p<this->chunk_size; p++, pindex++)
- //{
- // if (this->domain_particles[-1].find(pindex) != this->domain_particles[-1].end() ||
- // this->domain_particles[ 0].find(pindex) != this->domain_particles[ 0].end())
- // {
- // TIMEZONE("std::copy");
- // std::copy(this->state[pindex].data,
- // this->state[pindex].data + state_dimension(particle_type),
- // temp0 + p*state_dimension(particle_type));
- // }
- //}
- for (int s = -1; s <= 0; s++)
- for (auto &pp: this->domain_particles[s])
- {
- if (pp >= int(cindex*this->chunk_size) &&
- pp < int((cindex+1)*this->chunk_size))
- {
- std::copy(this->state[pp].data,
- this->state[pp].data + state_dimension(particle_type),
- temp0 + (pp-cindex*this->chunk_size)*state_dimension(particle_type));
- }
- }
- {
- TIMEZONE("MPI_Allreduce");
- MPI_Allreduce(
- temp0,
- temp1,
- state_dimension(particle_type)*this->chunk_size,
- MPI_DOUBLE,
- MPI_SUM,
- this->comm);
- }
- if (this->myrank == 0){
- TIMEZONE("write_state_chunk");
- this->write_state_chunk(cindex, temp1);
- }
- //write rhs
- if (write_rhs){
- TIMEZONE("write_rhs");
- for (int i=0; i<this->integration_steps; i++)
- {
- std::fill_n(temp0, state_dimension(particle_type)*this->chunk_size, 0);
- //pindex = cindex*this->chunk_size;
- //for (unsigned int p=0; p<this->chunk_size; p++, pindex++)
- //{
- // if (this->domain_particles[-1].find(pindex) != this->domain_particles[-1].end() ||
- // this->domain_particles[ 0].find(pindex) != this->domain_particles[ 0].end())
- // {
- // TIMEZONE("std::copy");
- // std::copy(this->rhs[i][pindex].data,
- // this->rhs[i][pindex].data + state_dimension(particle_type),
- // temp0 + p*state_dimension(particle_type));
- // }
- //}
- for (int s = -1; s <= 0; s++)
- for (auto &pp: this->domain_particles[s])
- {
- if (pp >= int(cindex*this->chunk_size) &&
- pp < int((cindex+1)*this->chunk_size))
- {
- std::copy(this->rhs[i][pp].data,
- this->rhs[i][pp].data + state_dimension(particle_type),
- temp0 + (pp-cindex*this->chunk_size)*state_dimension(particle_type));
- }
- }
- {
- TIMEZONE("MPI_Allreduce");
- MPI_Allreduce(
- temp0,
- temp1,
- state_dimension(particle_type)*this->chunk_size,
- MPI_DOUBLE,
- MPI_SUM,
- this->comm);
- }
- if (this->myrank == 0){
- TIMEZONE("write_rhs_chunk");
- this->write_rhs_chunk(cindex, i, temp1);
- }
- }
- }
- }
- delete[] temp0;
- delete[] temp1;
-}
-
-
-/*****************************************************************************/
-template class rFFTW_distributed_particles<VELOCITY_TRACER, float, 1>;
-template class rFFTW_distributed_particles<VELOCITY_TRACER, float, 2>;
-template class rFFTW_distributed_particles<VELOCITY_TRACER, float, 3>;
-template class rFFTW_distributed_particles<VELOCITY_TRACER, float, 4>;
-template class rFFTW_distributed_particles<VELOCITY_TRACER, float, 5>;
-template class rFFTW_distributed_particles<VELOCITY_TRACER, float, 6>;
-template class rFFTW_distributed_particles<VELOCITY_TRACER, double, 1>;
-template class rFFTW_distributed_particles<VELOCITY_TRACER, double, 2>;
-template class rFFTW_distributed_particles<VELOCITY_TRACER, double, 3>;
-template class rFFTW_distributed_particles<VELOCITY_TRACER, double, 4>;
-template class rFFTW_distributed_particles<VELOCITY_TRACER, double, 5>;
-template class rFFTW_distributed_particles<VELOCITY_TRACER, double, 6>;
-/*****************************************************************************/
-
diff --git a/bfps/cpp/rFFTW_distributed_particles.hpp b/bfps/cpp/rFFTW_distributed_particles.hpp
deleted file mode 100644
index 400411d5f1fd6e597714be494a72272a76e01206..0000000000000000000000000000000000000000
--- a/bfps/cpp/rFFTW_distributed_particles.hpp
+++ /dev/null
@@ -1,144 +0,0 @@
-/**********************************************************************
-* *
-* Copyright 2015 Max Planck Institute *
-* for Dynamics and Self-Organization *
-* *
-* This file is part of bfps. *
-* *
-* bfps is free software: you can redistribute it and/or modify *
-* it under the terms of the GNU General Public License as published *
-* by the Free Software Foundation, either version 3 of the License, *
-* or (at your option) any later version. *
-* *
-* bfps is distributed in the hope that it will be useful, *
-* but WITHOUT ANY WARRANTY; without even the implied warranty of *
-* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the *
-* GNU General Public License for more details. *
-* *
-* You should have received a copy of the GNU General Public License *
-* along with bfps. If not, see <http://www.gnu.org/licenses/> *
-* *
-* Contact: Cristian.Lalescu@ds.mpg.de *
-* *
-**********************************************************************/
-
-
-
-#include <stdio.h>
-#include <stdlib.h>
-#include <iostream>
-#include <unordered_map>
-#include <unordered_set>
-#include <vector>
-#include <hdf5.h>
-#include "base.hpp"
-#include "particles_base.hpp"
-#include "fluid_solver_base.hpp"
-#include "rFFTW_interpolator.hpp"
-
-#ifndef RFFTW_DISTRIBUTED_PARTICLES
-
-#define RFFTW_DISTRIBUTED_PARTICLES
-
-template <particle_types particle_type, class rnumber, int interp_neighbours>
-class rFFTW_distributed_particles: public particles_io_base<particle_type>
-{
- private:
- // a "domain" corresponds to a region in 3D real space where a fixed set
- // of MPI processes are required to participate in the interpolation
- // formula (i.e. they all contain required information).
- // we need to know how many processes there are for each of the domains
- // to which the local process belongs.
- std::unordered_map<int, int> domain_nprocs;
- // each domain has an associated communicator, and we keep a list of the
- // communicators to which the local process belongs
- std::unordered_map<int, MPI_Comm> domain_comm;
- // for each domain, we need a list of the IDs of the particles located
- // in that domain
- std::unordered_map<int, std::unordered_set<int>> domain_particles;
-
- // for each domain, we need the state of each particle
- std::unordered_map<int, single_particle_state<particle_type>> state;
- // for each domain, we also need the last few values of the right hand
- // side of the ODE, since we use Adams-Bashforth integration
- std::vector<std::unordered_map<int, single_particle_state<particle_type>>> rhs;
-
- public:
- int integration_steps;
- // this class only works with rFFTW interpolator
- rFFTW_interpolator<rnumber, interp_neighbours> *vel;
-
- /* simulation parameters */
- double dt;
-
- /* methods */
-
- /* constructor and destructor.
- * allocate and deallocate:
- * this->state
- * this->rhs
- * */
- rFFTW_distributed_particles(
- const char *NAME,
- const hid_t data_file_id,
- rFFTW_interpolator<rnumber, interp_neighbours> *FIELD,
- const int TRAJ_SKIP,
- const int INTEGRATION_STEPS = 2);
- ~rFFTW_distributed_particles();
-
- void sample(
- rFFTW_interpolator<rnumber, interp_neighbours> *field,
- const char *dset_name);
- void sample(
- rFFTW_interpolator<rnumber, interp_neighbours> *field,
- const std::unordered_map<int, single_particle_state<particle_type>> &x,
- const std::unordered_map<int, std::unordered_set<int>> &dp,
- std::unordered_map<int, single_particle_state<POINT3D>> &y);
- void get_rhs(
- const std::unordered_map<int, single_particle_state<particle_type>> &x,
- const std::unordered_map<int, std::unordered_set<int>> &dp,
- std::unordered_map<int, single_particle_state<particle_type>> &y);
-
-
- /* given a list of particle positions,
- * figure out which go into what local domain, and construct the relevant
- * map of ID lists "dp" (for domain particles).
- * */
- void sort_into_domains(
- const std::unordered_map<int, single_particle_state<particle_type>> &x,
- std::unordered_map<int, std::unordered_set<int>> &dp);
- /* suppose the particles are currently badly distributed, and some
- * arbitrary quantities (stored in "vals") are associated to the particles,
- * and we need to properly distribute them among processes.
- * that's what this function does.
- * In practice it's only used to redistribute the rhs values (and it
- * automatically redistributes the state x being passed).
- * Some more comments are present in the .cpp file, but, in brief: the
- * particles are simply moved from one domain to another.
- * If it turns out that the new domain contains a process which does not
- * know about a particle, that information is sent from the closest process.
- * */
- void redistribute(
- std::unordered_map<int, single_particle_state<particle_type>> &x,
- std::vector<std::unordered_map<int, single_particle_state<particle_type>>> &vals,
- std::unordered_map<int, std::unordered_set<int>> &dp);
-
-
- /* input/output */
- void read();
- void write(
- const char *dset_name,
- std::unordered_map<int, single_particle_state<POINT3D>> &y);
- void write(
- const char *dset_name,
- std::unordered_map<int, single_particle_state<particle_type>> &y);
- void write(const bool write_rhs = true);
-
- /* solvers */
- void step();
- void roll_rhs();
- void AdamsBashforth(const int nsteps);
-};
-
-#endif//RFFTW_DISTRIBUTED_PARTICLES
-
diff --git a/bfps/cpp/rFFTW_interpolator.cpp b/bfps/cpp/rFFTW_interpolator.cpp
deleted file mode 100644
index b8b21e8811d7f5286dc4edd00833c205539ea89c..0000000000000000000000000000000000000000
--- a/bfps/cpp/rFFTW_interpolator.cpp
+++ /dev/null
@@ -1,210 +0,0 @@
-/**********************************************************************
-* *
-* Copyright 2015 Max Planck Institute *
-* for Dynamics and Self-Organization *
-* *
-* This file is part of bfps. *
-* *
-* bfps is free software: you can redistribute it and/or modify *
-* it under the terms of the GNU General Public License as published *
-* by the Free Software Foundation, either version 3 of the License, *
-* or (at your option) any later version. *
-* *
-* bfps is distributed in the hope that it will be useful, *
-* but WITHOUT ANY WARRANTY; without even the implied warranty of *
-* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the *
-* GNU General Public License for more details. *
-* *
-* You should have received a copy of the GNU General Public License *
-* along with bfps. If not, see <http://www.gnu.org/licenses/> *
-* *
-* Contact: Cristian.Lalescu@ds.mpg.de *
-* *
-**********************************************************************/
-
-
-
-#define NDEBUG
-
-#include <cmath>
-#include "rFFTW_interpolator.hpp"
-#include "scope_timer.hpp"
-
-template <class rnumber, int interp_neighbours>
-rFFTW_interpolator<rnumber, interp_neighbours>::rFFTW_interpolator(
- fluid_solver_base<rnumber> *fs,
- base_polynomial_values BETA_POLYS,
- rnumber *FIELD_POINTER) : interpolator_base<rnumber, interp_neighbours>(fs, BETA_POLYS)
-{
- this->field = FIELD_POINTER;
-
-
- // generate compute array
- this->compute = new bool[this->descriptor->sizes[0]];
- std::fill_n(this->compute, this->descriptor->sizes[0], false);
- for (int iz = this->descriptor->starts[0]-interp_neighbours-1;
- iz <= this->descriptor->starts[0]+this->descriptor->subsizes[0]+interp_neighbours;
- iz++)
- this->compute[((iz + this->descriptor->sizes[0]) % this->descriptor->sizes[0])] = true;
-}
-
-template <class rnumber, int interp_neighbours>
-rFFTW_interpolator<rnumber, interp_neighbours>::rFFTW_interpolator(
- vorticity_equation<rnumber, FFTW> *fs,
- base_polynomial_values BETA_POLYS,
- rnumber *FIELD_POINTER) : interpolator_base<rnumber, interp_neighbours>(fs, BETA_POLYS)
-{
-// this->field = FIELD_POINTER;
-//
-//
-// // generate compute array
-// this->compute = new bool[this->descriptor->sizes[0]];
-// std::fill_n(this->compute, this->descriptor->sizes[0], false);
-// for (int iz = this->descriptor->starts[0]-interp_neighbours-1;
-// iz <= this->descriptor->starts[0]+this->descriptor->subsizes[0]+interp_neighbours;
-// iz++)
-// this->compute[((iz + this->descriptor->sizes[0]) % this->descriptor->sizes[0])] = true;
-}
-
-template <class rnumber, int interp_neighbours>
-rFFTW_interpolator<rnumber, interp_neighbours>::~rFFTW_interpolator()
-{
- delete[] this->compute;
-}
-
-template <class rnumber, int interp_neighbours>
-bool rFFTW_interpolator<rnumber, interp_neighbours>::get_rank_info(double z, int &maxz_rank, int &minz_rank)
-{
- int zg = int(floor(z/this->dz));
- minz_rank = this->descriptor->rank[MOD(
- zg - interp_neighbours,
- this->descriptor->sizes[0])];
- maxz_rank = this->descriptor->rank[MOD(
- zg + 1 + interp_neighbours,
- this->descriptor->sizes[0])];
- bool is_here = false;
- for (int iz = -interp_neighbours; iz <= interp_neighbours+1; iz++)
- is_here = (is_here ||
- (this->descriptor->myrank ==
- this->descriptor->rank[MOD(zg+iz, this->descriptor->sizes[0])]));
- return is_here;
-}
-
-template <class rnumber, int interp_neighbours>
-void rFFTW_interpolator<rnumber, interp_neighbours>::sample(
- const int nparticles,
- const int pdimension,
- const double *__restrict__ x,
- double *__restrict__ y,
- const int *deriv)
-{
- TIMEZONE("rFFTW_interpolator::sample");
- /* get grid coordinates */
- int *xg = new int[3*nparticles];
- double *xx = new double[3*nparticles];
- double *yy = new double[3*nparticles];
- std::fill_n(yy, 3*nparticles, 0.0);
- this->get_grid_coordinates(nparticles, pdimension, x, xg, xx);
- /* perform interpolation */
- for (int p=0; p<nparticles; p++)
- if (this->compute[xg[p*3+2]])
- this->operator()(xg + p*3, xx + p*3, yy + p*3, deriv);
- MPI_Allreduce(
- yy,
- y,
- 3*nparticles,
- MPI_DOUBLE,
- MPI_SUM,
- this->descriptor->comm);
- delete[] yy;
- delete[] xg;
- delete[] xx;
-}
-
-template <class rnumber, int interp_neighbours>
-void rFFTW_interpolator<rnumber, interp_neighbours>::operator()(
- const int *xg,
- const double *xx,
- double *dest,
- const int *deriv)
-{
- TIMEZONE("rFFTW_interpolator::operator()");
- double bx[interp_neighbours*2+2], by[interp_neighbours*2+2], bz[interp_neighbours*2+2];
- /* please note that the polynomials in z are computed for all the different
- * iz values, independently of whether or not "myrank" will perform the
- * computation for all the different iz slices.
- * I don't know how big a deal this really is, but it is something that we can
- * optimize.
- * */
- if (deriv == NULL)
- {
- this->compute_beta(0, xx[0], bx);
- this->compute_beta(0, xx[1], by);
- this->compute_beta(0, xx[2], bz);
- }
- else
- {
- this->compute_beta(deriv[0], xx[0], bx);
- this->compute_beta(deriv[1], xx[1], by);
- this->compute_beta(deriv[2], xx[2], bz);
- }
- std::fill_n(dest, 3, 0);
- ptrdiff_t bigiz, bigiy, bigix;
- // loop over the 2*interp_neighbours + 2 z slices
- for (int iz = -interp_neighbours; iz <= interp_neighbours+1; iz++)
- {
- // bigiz is the z index of the cell containing the particles
- // this->descriptor->sizes[0] is added before taking the modulo
- // because we want to be sure that "bigiz" is a positive number.
- // I'm no longer sure why I don't use the MOD function here.
- bigiz = ptrdiff_t(((xg[2]+iz) + this->descriptor->sizes[0]) % this->descriptor->sizes[0]);
- // once we know bigiz, we know whether "myrank" has the relevant slice.
- // if not, go to next value of bigiz
- if (this->descriptor->myrank == this->descriptor->rank[bigiz])
- {
- for (int iy = -interp_neighbours; iy <= interp_neighbours+1; iy++)
- {
- // bigiy is the y index of the cell
- // since we have all the y indices in myrank, we can safely use the
- // modulo value
- bigiy = ptrdiff_t(MOD(xg[1]+iy, this->descriptor->sizes[1]));
- for (int ix = -interp_neighbours; ix <= interp_neighbours+1; ix++)
- {
- // bigix is the x index of the cell
- bigix = ptrdiff_t(MOD(xg[0]+ix, this->descriptor->sizes[2]));
- // here we create the index to the current grid node
- // note the removal of local_z_start from bigiz.
- ptrdiff_t tindex = (((bigiz-this->descriptor->starts[0])*this->descriptor->sizes[1] +
- bigiy)*(this->descriptor->sizes[2]+2) +
- bigix)*3;
- for (int c=0; c<3; c++)
- dest[c] += this->field[tindex+c]*(bz[iz+interp_neighbours]*
- by[iy+interp_neighbours]*
- bx[ix+interp_neighbours]);
- }
- }
- }
- }
-}
-
-template class rFFTW_interpolator<float, 1>;
-template class rFFTW_interpolator<float, 2>;
-template class rFFTW_interpolator<float, 3>;
-template class rFFTW_interpolator<float, 4>;
-template class rFFTW_interpolator<float, 5>;
-template class rFFTW_interpolator<float, 6>;
-template class rFFTW_interpolator<float, 7>;
-template class rFFTW_interpolator<float, 8>;
-template class rFFTW_interpolator<float, 9>;
-template class rFFTW_interpolator<float, 10>;
-template class rFFTW_interpolator<double, 1>;
-template class rFFTW_interpolator<double, 2>;
-template class rFFTW_interpolator<double, 3>;
-template class rFFTW_interpolator<double, 4>;
-template class rFFTW_interpolator<double, 5>;
-template class rFFTW_interpolator<double, 6>;
-template class rFFTW_interpolator<double, 7>;
-template class rFFTW_interpolator<double, 8>;
-template class rFFTW_interpolator<double, 9>;
-template class rFFTW_interpolator<double, 10>;
-
diff --git a/bfps/cpp/rFFTW_interpolator.hpp b/bfps/cpp/rFFTW_interpolator.hpp
deleted file mode 100644
index 5088be8b2f3094fd96332af0c923d7cc905e4f3f..0000000000000000000000000000000000000000
--- a/bfps/cpp/rFFTW_interpolator.hpp
+++ /dev/null
@@ -1,118 +0,0 @@
-/**********************************************************************
-* *
-* Copyright 2015 Max Planck Institute *
-* for Dynamics and Self-Organization *
-* *
-* This file is part of bfps. *
-* *
-* bfps is free software: you can redistribute it and/or modify *
-* it under the terms of the GNU General Public License as published *
-* by the Free Software Foundation, either version 3 of the License, *
-* or (at your option) any later version. *
-* *
-* bfps is distributed in the hope that it will be useful, *
-* but WITHOUT ANY WARRANTY; without even the implied warranty of *
-* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the *
-* GNU General Public License for more details. *
-* *
-* You should have received a copy of the GNU General Public License *
-* along with bfps. If not, see <http://www.gnu.org/licenses/> *
-* *
-* Contact: Cristian.Lalescu@ds.mpg.de *
-* *
-**********************************************************************/
-
-
-
-#include "field_descriptor.hpp"
-#include "fftw_tools.hpp"
-#include "fluid_solver_base.hpp"
-#include "vorticity_equation.hpp"
-#include "interpolator_base.hpp"
-
-#ifndef RFFTW_INTERPOLATOR
-
-#define RFFTW_INTERPOLATOR
-
-template <class rnumber, int interp_neighbours>
-class rFFTW_interpolator:public interpolator_base<rnumber, interp_neighbours>
-{
- public:
- using interpolator_base<rnumber, interp_neighbours>::operator();
-
- /* pointer to field that has to be interpolated
- * The reason this is a member variable is because I want this class
- * to be consistent with the "interpolator" class, where a member
- * variable is absolutely required (since that class uses padding).
- * */
- rnumber *field;
-
- /* compute[iz] is an array that says whether or not the current MPI
- * process is involved in the interpolation formula for a particle
- * located in cell "iz".
- * It is mostly used in the formula itself.
- * This translates as the following condition:
- * local_zstart - neighbours <= iz <= local_zend + 1 + neighbours
- * I think it's cleaner to keep things in an array, especially since
- * "local_zend" is shorthand for another arithmetic operation anyway.
- * */
- bool *compute;
-
-
- /* Constructors */
- rFFTW_interpolator(
- fluid_solver_base<rnumber> *FSOLVER,
- base_polynomial_values BETA_POLYS,
- rnumber *FIELD_DATA);
-
- /* this constructor is empty, I just needed for a quick hack of the
- * "vorticity_equation" class.
- * It should be removed soon.
- * */
- rFFTW_interpolator(
- vorticity_equation<rnumber, FFTW> *FSOLVER,
- base_polynomial_values BETA_POLYS,
- rnumber *FIELD_DATA);
- ~rFFTW_interpolator();
-
- /* This method is provided for consistency with "interpolator", and it
- * does not destroy input */
- inline int read_rFFTW(const void *src)
- {
- this->field = (rnumber*)src;
- return EXIT_SUCCESS;
- }
-
- /* This is used when "compute" is not enough.
- * For a given z location, it gives the outermost ranks that are relevant
- * for the interpolation formula.
- * */
- bool get_rank_info(double z, int &maxz_rank, int &minz_rank);
-
- /* interpolate field at an array of locations.
- * After interpolation is performed, call Allreduce for "y", over
- * this->descriptor->comm --- generally MPI_COMM_WORLD.
- * This is useful for the simple "particles" class, where particle
- * information is synchronized across all processes.
- * */
- void sample(
- const int nparticles,
- const int pdimension,
- const double *__restrict__ x,
- double *__restrict__ y,
- const int *deriv = NULL);
- /* interpolate 1 point.
- * Result is kept local.
- * This is used in the "rFFTW_distributed_particles" class, with the
- * result being synchronized across the relevant "local particle
- * communicator".
- * */
- void operator()(
- const int *__restrict__ xg,
- const double *__restrict__ xx,
- double *__restrict__ dest,
- const int *deriv = NULL);
-};
-
-#endif//RFFTW_INTERPOLATOR
-
diff --git a/bfps/cpp/slab_field_particles.cpp b/bfps/cpp/slab_field_particles.cpp
deleted file mode 100644
index 15fa363f6d277d34c6081fd545c4578e1f735929..0000000000000000000000000000000000000000
--- a/bfps/cpp/slab_field_particles.cpp
+++ /dev/null
@@ -1,799 +0,0 @@
-/**********************************************************************
-* *
-* Copyright 2015 Max Planck Institute *
-* for Dynamics and Self-Organization *
-* *
-* This file is part of bfps. *
-* *
-* bfps is free software: you can redistribute it and/or modify *
-* it under the terms of the GNU General Public License as published *
-* by the Free Software Foundation, either version 3 of the License, *
-* or (at your option) any later version. *
-* *
-* bfps is distributed in the hope that it will be useful, *
-* but WITHOUT ANY WARRANTY; without even the implied warranty of *
-* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the *
-* GNU General Public License for more details. *
-* *
-* You should have received a copy of the GNU General Public License *
-* along with bfps. If not, see <http://www.gnu.org/licenses/> *
-* *
-* Contact: Cristian.Lalescu@ds.mpg.de *
-* *
-**********************************************************************/
-
-
-
-#define NDEBUG
-
-
-#include <cmath>
-#include <cassert>
-#include <cstring>
-#include <string>
-#include <sstream>
-
-#include "base.hpp"
-#include "slab_field_particles.hpp"
-#include "fftw_tools.hpp"
-
-
-extern int myrank, nprocs;
-
-template <class rnumber>
-slab_field_particles<rnumber>::slab_field_particles(
- const char *NAME,
- fluid_solver_base<rnumber> *FSOLVER,
- const int NPARTICLES,
- const int NCOMPONENTS,
- base_polynomial_values BETA_POLYS,
- const int INTERP_NEIGHBOURS,
- const int TRAJ_SKIP,
- const int INTEGRATION_STEPS)
-{
- assert((NCOMPONENTS % 3) == 0);
- assert((INTERP_NEIGHBOURS >= 1) ||
- (INTERP_NEIGHBOURS <= 8));
- assert((INTEGRATION_STEPS <= 6) &&
- (INTEGRATION_STEPS >= 1));
- strncpy(this->name, NAME, 256);
- this->fs = FSOLVER;
- this->nparticles = NPARTICLES;
- this->ncomponents = NCOMPONENTS;
- this->integration_steps = INTEGRATION_STEPS;
- this->interp_neighbours = INTERP_NEIGHBOURS;
- this->traj_skip = TRAJ_SKIP;
- this->compute_beta = BETA_POLYS;
- // in principle only the buffer width at the top needs the +1,
- // but things are simpler if buffer_width is the same
- this->buffer_width = this->interp_neighbours+1;
- this->buffer_size = this->buffer_width*this->fs->rd->slice_size;
- this->array_size = this->nparticles * this->ncomponents;
- this->state = fftw_alloc_real(this->array_size);
- std::fill_n(this->state, this->array_size, 0.0);
- for (int i=0; i < this->integration_steps; i++)
- {
- this->rhs[i] = fftw_alloc_real(this->array_size);
- std::fill_n(this->rhs[i], this->array_size, 0.0);
- }
- this->watching = new bool[this->fs->rd->nprocs*nparticles];
- std::fill_n(this->watching, this->fs->rd->nprocs*this->nparticles, false);
- this->computing = new int[nparticles];
-
- int tdims[4];
- tdims[0] = this->buffer_width*2*this->fs->rd->nprocs + this->fs->rd->sizes[0];
- tdims[1] = this->fs->rd->sizes[1];
- tdims[2] = this->fs->rd->sizes[2];
- tdims[3] = this->fs->rd->sizes[3];
- this->buffered_field_descriptor = new field_descriptor<rnumber>(
- 4, tdims,
- this->fs->rd->mpi_dtype,
- this->fs->rd->comm);
-
- // compute dx, dy, dz;
- this->dx = 4*acos(0) / (this->fs->dkx*this->fs->rd->sizes[2]);
- this->dy = 4*acos(0) / (this->fs->dky*this->fs->rd->sizes[1]);
- this->dz = 4*acos(0) / (this->fs->dkz*this->fs->rd->sizes[0]);
-
- // compute lower and upper bounds
- this->lbound = new double[nprocs];
- this->ubound = new double[nprocs];
- double *tbound = new double[nprocs];
- std::fill_n(tbound, nprocs, 0.0);
- tbound[this->fs->rd->myrank] = this->fs->rd->starts[0]*this->dz;
- MPI_Allreduce(
- tbound,
- this->lbound,
- nprocs,
- MPI_DOUBLE,
- MPI_SUM,
- this->fs->rd->comm);
- std::fill_n(tbound, nprocs, 0.0);
- tbound[this->fs->rd->myrank] = (this->fs->rd->starts[0] + this->fs->rd->subsizes[0])*this->dz;
- MPI_Allreduce(
- tbound,
- this->ubound,
- nprocs,
- MPI_DOUBLE,
- MPI_SUM,
- this->fs->rd->comm);
- delete[] tbound;
- //for (int r = 0; r<nprocs; r++)
- // DEBUG_MSG(
- // "lbound[%d] = %lg, ubound[%d] = %lg\n",
- // r, this->lbound[r],
- // r, this->ubound[r]
- // );
-}
-
-template <class rnumber>
-slab_field_particles<rnumber>::~slab_field_particles()
-{
- delete[] this->computing;
- delete[] this->watching;
- fftw_free(this->state);
- for (int i=0; i < this->integration_steps; i++)
- {
- fftw_free(this->rhs[i]);
- }
- delete[] this->lbound;
- delete[] this->ubound;
- delete this->buffered_field_descriptor;
-}
-
-template <class rnumber>
-void slab_field_particles<rnumber>::get_rhs(double *x, double *y)
-{
- std::fill_n(y, this->array_size, 0.0);
-}
-
-template <class rnumber>
-void slab_field_particles<rnumber>::jump_estimate(double *dest)
-{
- std::fill_n(dest, this->nparticles, 0.0);
-}
-
-template <class rnumber>
-int slab_field_particles<rnumber>::get_rank(double z)
-{
- int tmp = this->fs->rd->rank[MOD(int(floor(z/this->dz)), this->fs->rd->sizes[0])];
- assert(tmp >= 0 && tmp < this->fs->rd->nprocs);
- return tmp;
-}
-
-template <class rnumber>
-void slab_field_particles<rnumber>::synchronize_single_particle_state(int p, double *x, int source)
-{
- if (source == -1) source = this->computing[p];
- if (this->watching[this->fs->rd->myrank*this->nparticles+p]) for (int r=0; r<this->fs->rd->nprocs; r++)
- if (r != source &&
- this->watching[r*this->nparticles+p])
- {
- //DEBUG_MSG("synchronizing state %d from %d to %d\n", p, this->computing[p], r);
- if (this->fs->rd->myrank == source)
- MPI_Send(
- x+p*this->ncomponents,
- this->ncomponents,
- MPI_DOUBLE,
- r,
- p+this->computing[p]*this->nparticles,
- this->fs->rd->comm);
- if (this->fs->rd->myrank == r)
- MPI_Recv(
- x+p*this->ncomponents,
- this->ncomponents,
- MPI_DOUBLE,
- source,
- p+this->computing[p]*this->nparticles,
- this->fs->rd->comm,
- MPI_STATUS_IGNORE);
- }
-}
-
-template <class rnumber>
-void slab_field_particles<rnumber>::synchronize()
-{
- double *tstate = fftw_alloc_real(this->array_size);
- // first, synchronize state and jump across CPUs
- std::fill_n(tstate, this->array_size, 0.0);
- for (int p=0; p<this->nparticles; p++)
- {
- //if (this->watching[this->fs->rd->myrank*this->nparticles + p])
- //DEBUG_MSG(
- // "in synchronize, position for particle %d is %g %g %g\n",
- // p,
- // this->state[p*this->ncomponents],
- // this->state[p*this->ncomponents+1],
- // this->state[p*this->ncomponents+2]);
- if (this->fs->rd->myrank == this->computing[p])
- std::copy(this->state + p*this->ncomponents,
- this->state + (p+1)*this->ncomponents,
- tstate + p*this->ncomponents);
- }
- MPI_Allreduce(
- tstate,
- this->state,
- this->array_size,
- MPI_DOUBLE,
- MPI_SUM,
- this->fs->rd->comm);
- if (this->integration_steps >= 1)
- {
- for (int i=0; i<this->integration_steps; i++)
- {
- std::fill_n(tstate, this->array_size, 0.0);
- for (int p=0; p<this->nparticles; p++) if (this->fs->rd->myrank == this->computing[p])
- std::copy(this->rhs[i] + p*this->ncomponents,
- this->rhs[i] + (p+1)*this->ncomponents,
- tstate + p*this->ncomponents);
- std::fill_n(this->rhs[i], this->array_size, 0.0);
- MPI_Allreduce(
- tstate,
- this->rhs[i],
- this->array_size,
- MPI_DOUBLE,
- MPI_SUM,
- this->fs->rd->comm);
- }
- }
- fftw_free(tstate);
- // assignment of particles
- for (int p=0; p<this->nparticles; p++)
- {
- this->computing[p] = this->get_rank(this->state[p*this->ncomponents + 2]);
- //DEBUG_MSG("synchronizing particles, particle %d computing is %d\n", p, this->computing[p]);
- }
- double *jump = fftw_alloc_real(this->nparticles);
- this->jump_estimate(jump);
- // now, see who needs to watch
- bool *local_watching = new bool[this->fs->rd->nprocs*this->nparticles];
- std::fill_n(local_watching, this->fs->rd->nprocs*this->nparticles, false);
- for (int p=0; p<this->nparticles; p++)
- if (this->fs->rd->myrank == this->computing[p])
- {
- local_watching[this->get_rank(this->state[this->ncomponents*p+2] )*this->nparticles+p] = true;
- local_watching[this->get_rank(this->state[this->ncomponents*p+2]-jump[p])*this->nparticles+p] = true;
- local_watching[this->get_rank(this->state[this->ncomponents*p+2]+jump[p])*this->nparticles+p] = true;
- }
- fftw_free(jump);
- MPI_Allreduce(
- local_watching,
- this->watching,
- this->nparticles*this->fs->rd->nprocs,
- MPI_C_BOOL,
- MPI_LOR,
- this->fs->rd->comm);
- delete[] local_watching;
- for (int p=0; p<this->nparticles; p++)
- DEBUG_MSG("watching = %d for particle %d\n", this->watching[this->fs->rd->myrank*nparticles+p], p);
-}
-
-
-
-template <class rnumber>
-void slab_field_particles<rnumber>::roll_rhs()
-{
- for (int i=this->integration_steps-2; i>=0; i--)
- std::copy(this->rhs[i],
- this->rhs[i] + this->array_size,
- this->rhs[i+1]);
-}
-
-
-
-template <class rnumber>
-void slab_field_particles<rnumber>::AdamsBashforth(int nsteps)
-{
- ptrdiff_t ii;
- this->get_rhs(this->state, this->rhs[0]);
- //if (myrank == 0)
- //{
- // DEBUG_MSG(
- // "in AdamsBashforth for particles %s, integration_steps = %d, nsteps = %d, iteration = %d\n",
- // this->name,
- // this->integration_steps,
- // nsteps,
- // this->iteration);
- // std::stringstream tstring;
- // for (int p=0; p<this->nparticles; p++)
- // tstring << " " << this->computing[p];
- // DEBUG_MSG("%s\n", tstring.str().c_str());
- // for (int i=0; i<this->integration_steps; i++)
- // {
- // std::stringstream tstring;
- // for (int p=0; p<this->nparticles; p++)
- // tstring << " " << this->rhs[i][p*3];
- // DEBUG_MSG("%s\n", tstring.str().c_str());
- // }
- //}
- switch(nsteps)
- {
- case 1:
- for (int p=0; p<this->nparticles; p++) if (this->fs->rd->myrank == this->computing[p])
- for (int i=0; i<this->ncomponents; i++)
- {
- ii = p*this->ncomponents+i;
- this->state[ii] += this->dt*this->rhs[0][ii];
- }
- break;
- case 2:
- for (int p=0; p<this->nparticles; p++) if (this->fs->rd->myrank == this->computing[p])
- for (int i=0; i<this->ncomponents; i++)
- {
- ii = p*this->ncomponents+i;
- this->state[ii] += this->dt*(3*this->rhs[0][ii]
- - this->rhs[1][ii])/2;
- }
- break;
- case 3:
- for (int p=0; p<this->nparticles; p++) if (this->fs->rd->myrank == this->computing[p])
- for (int i=0; i<this->ncomponents; i++)
- {
- ii = p*this->ncomponents+i;
- this->state[ii] += this->dt*(23*this->rhs[0][ii]
- - 16*this->rhs[1][ii]
- + 5*this->rhs[2][ii])/12;
- }
- break;
- case 4:
- for (int p=0; p<this->nparticles; p++) if (this->fs->rd->myrank == this->computing[p])
- for (int i=0; i<this->ncomponents; i++)
- {
- ii = p*this->ncomponents+i;
- this->state[ii] += this->dt*(55*this->rhs[0][ii]
- - 59*this->rhs[1][ii]
- + 37*this->rhs[2][ii]
- - 9*this->rhs[3][ii])/24;
- }
- break;
- case 5:
- for (int p=0; p<this->nparticles; p++) if (this->fs->rd->myrank == this->computing[p])
- for (int i=0; i<this->ncomponents; i++)
- {
- ii = p*this->ncomponents+i;
- this->state[ii] += this->dt*(1901*this->rhs[0][ii]
- - 2774*this->rhs[1][ii]
- + 2616*this->rhs[2][ii]
- - 1274*this->rhs[3][ii]
- + 251*this->rhs[4][ii])/720;
- }
- break;
- case 6:
- for (int p=0; p<this->nparticles; p++) if (this->fs->rd->myrank == this->computing[p])
- for (int i=0; i<this->ncomponents; i++)
- {
- ii = p*this->ncomponents+i;
- this->state[ii] += this->dt*(4277*this->rhs[0][ii]
- - 7923*this->rhs[1][ii]
- + 9982*this->rhs[2][ii]
- - 7298*this->rhs[3][ii]
- + 2877*this->rhs[4][ii]
- - 475*this->rhs[5][ii])/1440;
- }
- break;
- }
- this->roll_rhs();
-}
-
-
-template <class rnumber>
-void slab_field_particles<rnumber>::step()
-{
- this->AdamsBashforth((this->iteration < this->integration_steps) ? this->iteration+1 : this->integration_steps);
- //this->cRK4();
- this->iteration++;
- this->synchronize();
-}
-
-
-template <class rnumber>
-void slab_field_particles<rnumber>::Euler()
-{
- double *y = fftw_alloc_real(this->array_size);
- this->get_rhs(this->state, y);
- for (int p=0; p<this->nparticles; p++) if (this->fs->rd->myrank == this->computing[p])
- {
- for (int i=0; i<this->ncomponents; i++)
- this->state[p*this->ncomponents+i] += this->dt*y[p*this->ncomponents+i];
- //DEBUG_MSG(
- // "particle %d state is %lg %lg %lg\n",
- // p, this->state[p*this->ncomponents], this->state[p*this->ncomponents+1], this->state[p*this->ncomponents+2]);
- }
- fftw_free(y);
-}
-
-
-template <class rnumber>
-void slab_field_particles<rnumber>::Heun()
-{
- double *y = new double[this->array_size];
- double dtfactor[] = {0.0, this->dt};
- this->get_rhs(this->state, this->rhs[0]);
- for (int p=0; p<this->nparticles; p++)
- {
- this->synchronize_single_particle_state(p, this->rhs[0]);
- //int crank = this->get_rank(this->state[p*3 + 2]);
- //DEBUG_MSG(
- // "k 0 iteration %d particle is %d, crank is %d, computing rank is %d, position is %g %g %g, rhs is %g %g %g\n",
- // this->iteration, p,
- // crank, this->computing[p],
- // this->state[p*3], this->state[p*3+1], this->state[p*3+2],
- // this->rhs[0][p*3], this->rhs[0][p*3+1], this->rhs[0][p*3+2]);
- }
- for (int kindex = 1; kindex < 2; kindex++)
- {
- for (int p=0; p<this->nparticles; p++)
- {
- if (this->watching[this->fs->rd->myrank*this->nparticles+p])
- for (int i=0; i<this->ncomponents; i++)
- {
- ptrdiff_t tindex = ptrdiff_t(p)*this->ncomponents + i;
- y[tindex] = this->state[tindex] + dtfactor[kindex]*this->rhs[kindex-1][tindex];
- }
- }
- for (int p=0; p<this->nparticles; p++)
- this->synchronize_single_particle_state(p, y);
- this->get_rhs(y, this->rhs[kindex]);
- for (int p=0; p<this->nparticles; p++)
- {
- this->synchronize_single_particle_state(p, this->rhs[kindex]);
- DEBUG_MSG(
- "k %d iteration %d particle is %d, position is %g %g %g, rhs is %g %g %g\n",
- kindex, this->iteration, p,
- y[p*3], y[p*3+1], y[p*3+2],
- this->rhs[kindex][p*3], this->rhs[kindex][p*3+1], this->rhs[kindex][p*3+2]);
- }
- }
- for (int p=0; p<this->nparticles; p++)
- {
- if (this->watching[this->fs->rd->myrank*this->nparticles+p])
- {
- for (int i=0; i<this->ncomponents; i++)
- {
- ptrdiff_t tindex = ptrdiff_t(p)*this->ncomponents + i;
- this->state[tindex] += this->dt*(this->rhs[0][tindex] + this->rhs[1][tindex])/2;
- }
- //int crank = this->get_rank(this->state[p*3 + 2]);
- //if (crank != this->computing[p])
- // DEBUG_MSG(
- // "k _ iteration %d particle is %d, crank is %d, computing rank is %d, position is %g %g %g\n",
- // this->iteration, p,
- // crank, this->computing[p],
- // this->state[p*3], this->state[p*3+1], this->state[p*3+2]);
- }
- }
- delete[] y;
- DEBUG_MSG("exiting Heun\n");
-}
-
-
-template <class rnumber>
-void slab_field_particles<rnumber>::cRK4()
-{
- double *y = new double[this->array_size];
- double dtfactor[] = {0.0, this->dt/2, this->dt/2, this->dt};
- this->get_rhs(this->state, this->rhs[0]);
- for (int p=0; p<this->nparticles; p++)
- this->synchronize_single_particle_state(p, this->rhs[0]);
- for (int kindex = 1; kindex < 4; kindex++)
- {
- for (int p=0; p<this->nparticles; p++)
- {
- if (this->watching[this->fs->rd->myrank*this->nparticles+p])
- for (int i=0; i<this->ncomponents; i++)
- {
- ptrdiff_t tindex = ptrdiff_t(p)*this->ncomponents + i;
- y[tindex] = this->state[tindex] + dtfactor[kindex]*this->rhs[kindex-1][tindex];
- }
- }
- for (int p=0; p<this->nparticles; p++)
- this->synchronize_single_particle_state(p, y);
- this->get_rhs(y, this->rhs[kindex]);
- for (int p=0; p<this->nparticles; p++)
- this->synchronize_single_particle_state(p, this->rhs[kindex]);
- }
- for (int p=0; p<this->nparticles; p++)
- {
- if (this->watching[this->fs->rd->myrank*this->nparticles+p])
- for (int i=0; i<this->ncomponents; i++)
- {
- ptrdiff_t tindex = ptrdiff_t(p)*this->ncomponents + i;
- this->state[tindex] += this->dt*(this->rhs[0][tindex] +
- 2*(this->rhs[1][tindex] + this->rhs[2][tindex]) +
- this->rhs[3][tindex])/6;
- }
- }
- delete[] y;
-}
-
-template <class rnumber>
-void slab_field_particles<rnumber>::get_grid_coordinates(double *x, int *xg, double *xx)
-{
- static double grid_size[] = {this->dx, this->dy, this->dz};
- double tval;
- std::fill_n(xg, this->nparticles*3, 0);
- std::fill_n(xx, this->nparticles*3, 0.0);
- for (int p=0; p<this->nparticles; p++) if (this->watching[this->fs->rd->myrank*this->nparticles+p])
- {
- for (int c=0; c<3; c++)
- {
- tval = floor(x[p*this->ncomponents+c]/grid_size[c]);
- xg[p*3+c] = MOD(int(tval), this->fs->rd->sizes[2-c]);
- xx[p*3+c] = (x[p*this->ncomponents+c] - tval*grid_size[c]) / grid_size[c];
- }
- xg[p*3+2] -= this->fs->rd->starts[0];
- if (this->fs->rd->myrank == this->fs->rd->rank[0] &&
- xg[p*3+2] > this->fs->rd->subsizes[0])
- xg[p*3+2] -= this->fs->rd->sizes[0];
- //DEBUG_MSG(
- // "particle %d x is %lg %lg %lg xx is %lg %lg %lg xg is %d %d %d\n",
- // p,
- // x[p*3], x[p*3+1], x[p*3+2],
- // xx[p*3], xx[p*3+1], xx[p*3+2],
- // xg[p*3], xg[p*3+1], xg[p*3+2]);
- }
-}
-
-template <class rnumber>
-void slab_field_particles<rnumber>::interpolation_formula(rnumber *field, int *xg, double *xx, double *dest, int *deriv)
-{
- double bx[this->interp_neighbours*2+2], by[this->interp_neighbours*2+2], bz[this->interp_neighbours*2+2];
- this->compute_beta(deriv[0], xx[0], bx);
- this->compute_beta(deriv[1], xx[1], by);
- this->compute_beta(deriv[2], xx[2], bz);
- //DEBUG_MSG("computed beta polynomials\n");
- std::fill_n(dest, 3, 0);
- for (int iz = -this->interp_neighbours; iz <= this->interp_neighbours+1; iz++)
- for (int iy = -this->interp_neighbours; iy <= this->interp_neighbours+1; iy++)
- for (int ix = -this->interp_neighbours; ix <= this->interp_neighbours+1; ix++)
- for (int c=0; c<3; c++)
- {
- //DEBUG_MSG(
- // "%d %d %d %d %d %d %d %ld %ld\n",
- // xg[2], xg[1], xg[0], iz, iy, ix, c,
- // ((ptrdiff_t(xg[2]+iz) *this->fs->rd->subsizes[1] +
- // ptrdiff_t(xg[1]+iy))*this->fs->rd->subsizes[2] +
- // ptrdiff_t(xg[0]+ix))*3+c,
- // this->buffered_field_descriptor->local_size
- // );
- dest[c] += field[((ptrdiff_t( xg[2]+iz ) *this->fs->rd->subsizes[1] +
- ptrdiff_t(MOD(xg[1]+iy, this->fs->rd->sizes[1])))*this->fs->rd->subsizes[2] +
- ptrdiff_t(MOD(xg[0]+ix, this->fs->rd->sizes[2])))*3+c]*(bz[iz+this->interp_neighbours]*
- by[iy+this->interp_neighbours]*
- bx[ix+this->interp_neighbours]);
- }
-}
-
-template <class rnumber>
-void slab_field_particles<rnumber>::linear_interpolation(rnumber *field, int *xg, double *xx, double *dest, int *deriv)
-{
- //ptrdiff_t tindex, tmp;
- //tindex = ((ptrdiff_t(xg[2] )*this->fs->rd->subsizes[1]+xg[1] )*this->fs->rd->subsizes[2]+xg[0] )*3;
- //tmp = ptrdiff_t(xg[2]);
- //DEBUG_MSG(
- // "linear interpolation xx is %lg %lg %lg xg is %d %d %d,"
- // " corner index is ((%ld*%d+%d)*%d+%d)*3 = %ld\n",
- // xx[0], xx[1], xx[2],
- // xg[0], xg[1], xg[2],
- // tmp, this->fs->rd->subsizes[1], xg[1], this->fs->rd->subsizes[2], xg[0],
- // tindex);
- for (int c=0; c<3; c++)
- dest[c] = (field[((ptrdiff_t(xg[2] )*this->fs->rd->subsizes[1]+xg[1] )*this->fs->rd->subsizes[2]+xg[0] )*3+c]*((1-xx[0])*(1-xx[1])*(1-xx[2])) +
- field[((ptrdiff_t(xg[2] )*this->fs->rd->subsizes[1]+xg[1] )*this->fs->rd->subsizes[2]+xg[0]+1)*3+c]*(( xx[0])*(1-xx[1])*(1-xx[2])) +
- field[((ptrdiff_t(xg[2] )*this->fs->rd->subsizes[1]+xg[1]+1)*this->fs->rd->subsizes[2]+xg[0] )*3+c]*((1-xx[0])*( xx[1])*(1-xx[2])) +
- field[((ptrdiff_t(xg[2] )*this->fs->rd->subsizes[1]+xg[1]+1)*this->fs->rd->subsizes[2]+xg[0]+1)*3+c]*(( xx[0])*( xx[1])*(1-xx[2])) +
- field[((ptrdiff_t(xg[2]+1)*this->fs->rd->subsizes[1]+xg[1] )*this->fs->rd->subsizes[2]+xg[0] )*3+c]*((1-xx[0])*(1-xx[1])*( xx[2])) +
- field[((ptrdiff_t(xg[2]+1)*this->fs->rd->subsizes[1]+xg[1] )*this->fs->rd->subsizes[2]+xg[0]+1)*3+c]*(( xx[0])*(1-xx[1])*( xx[2])) +
- field[((ptrdiff_t(xg[2]+1)*this->fs->rd->subsizes[1]+xg[1]+1)*this->fs->rd->subsizes[2]+xg[0] )*3+c]*((1-xx[0])*( xx[1])*( xx[2])) +
- field[((ptrdiff_t(xg[2]+1)*this->fs->rd->subsizes[1]+xg[1]+1)*this->fs->rd->subsizes[2]+xg[0]+1)*3+c]*(( xx[0])*( xx[1])*( xx[2])));
-}
-
-template <class rnumber>
-void slab_field_particles<rnumber>::read(hid_t data_file_id)
-{
- //DEBUG_MSG("aloha\n");
- if (this->fs->rd->myrank == 0)
- {
- std::string temp_string = (std::string("/particles/") +
- std::string(this->name) +
- std::string("/state"));
- hid_t Cdset = H5Dopen(data_file_id, temp_string.c_str(), H5P_DEFAULT);
- hid_t mspace, rspace;
- hsize_t count[4], offset[4];
- rspace = H5Dget_space(Cdset);
- H5Sget_simple_extent_dims(rspace, count, NULL);
- count[0] = 1;
- offset[0] = this->iteration / this->traj_skip;
- offset[1] = 0;
- offset[2] = 0;
- mspace = H5Screate_simple(3, count, NULL);
- H5Sselect_hyperslab(rspace, H5S_SELECT_SET, offset, NULL, count, NULL);
- H5Dread(Cdset, H5T_NATIVE_DOUBLE, mspace, rspace, H5P_DEFAULT, this->state);
- H5Sclose(mspace);
- H5Sclose(rspace);
- H5Dclose(Cdset);
- if (this->iteration > 0)
- {
- temp_string = (std::string("/particles/") +
- std::string(this->name) +
- std::string("/rhs"));
- Cdset = H5Dopen(data_file_id, temp_string.c_str(), H5P_DEFAULT);
- rspace = H5Dget_space(Cdset);
- H5Sget_simple_extent_dims(rspace, count, NULL);
- //reading from last available position
- offset[0] = count[0] - 1;
- offset[3] = 0;
- count[0] = 1;
- count[1] = 1;
- mspace = H5Screate_simple(4, count, NULL);
- for (int i=0; i<this->integration_steps; i++)
- {
- offset[1] = i;
- H5Sselect_hyperslab(rspace, H5S_SELECT_SET, offset, NULL, count, NULL);
- H5Dread(Cdset, H5T_NATIVE_DOUBLE, mspace, rspace, H5P_DEFAULT, this->rhs[i]);
- }
- H5Sclose(mspace);
- H5Sclose(rspace);
- H5Dclose(Cdset);
- }
- }
- MPI_Bcast(
- this->state,
- this->array_size,
- MPI_DOUBLE,
- 0,
- this->fs->rd->comm);
- for (int i = 0; i<this->integration_steps; i++)
- {
- MPI_Bcast(
- this->rhs[i],
- this->array_size,
- MPI_DOUBLE,
- 0,
- this->fs->rd->comm);
- }
- // initial assignment of particles
- for (int p=0; p<this->nparticles; p++)
- {
- this->computing[p] = this->get_rank(this->state[p*this->ncomponents + 2]);
- //DEBUG_MSG("reading particles, particle %d computing is %d\n", p, this->computing[p]);
- }
- // now actual synchronization
- this->synchronize();
-}
-
-template <class rnumber>
-void slab_field_particles<rnumber>::write(hid_t data_file_id, bool write_rhs)
-{
- if (this->fs->rd->myrank == 0)
- {
- std::string temp_string = (std::string("/particles/") +
- std::string(this->name) +
- std::string("/state"));
- hid_t Cdset = H5Dopen(data_file_id, temp_string.c_str(), H5P_DEFAULT);
- hid_t mspace, wspace;
- hsize_t count[4], offset[4];
- wspace = H5Dget_space(Cdset);
- H5Sget_simple_extent_dims(wspace, count, NULL);
- count[0] = 1;
- offset[0] = this->iteration / this->traj_skip;
- offset[1] = 0;
- offset[2] = 0;
- mspace = H5Screate_simple(3, count, NULL);
- H5Sselect_hyperslab(wspace, H5S_SELECT_SET, offset, NULL, count, NULL);
- H5Dwrite(Cdset, H5T_NATIVE_DOUBLE, mspace, wspace, H5P_DEFAULT, this->state);
- H5Sclose(mspace);
- H5Sclose(wspace);
- H5Dclose(Cdset);
- if (write_rhs)
- {
- temp_string = (std::string("/particles/") +
- std::string(this->name) +
- std::string("/rhs"));
- Cdset = H5Dopen(data_file_id, temp_string.c_str(), H5P_DEFAULT);
- wspace = H5Dget_space(Cdset);
- H5Sget_simple_extent_dims(wspace, count, NULL);
- //writing to last available position
- offset[0] = count[0] - 1;
- count[0] = 1;
- count[1] = 1;
- offset[3] = 0;
- mspace = H5Screate_simple(4, count, NULL);
- for (int i=0; i<this->integration_steps; i++)
- {
- offset[1] = i;
- H5Sselect_hyperslab(wspace, H5S_SELECT_SET, offset, NULL, count, NULL);
- H5Dwrite(Cdset, H5T_NATIVE_DOUBLE, mspace, wspace, H5P_DEFAULT, this->rhs[i]);
- }
- H5Sclose(mspace);
- H5Sclose(wspace);
- H5Dclose(Cdset);
- }
- }
-}
-
-
-
-/*****************************************************************************/
-/* macro for specializations to numeric types compatible with FFTW */
-#define SLAB_FIELD_PARTICLES_DEFINITIONS(FFTW, R, MPI_RNUM) \
- \
-template <> \
-void slab_field_particles<R>::rFFTW_to_buffered(R *src, R *dst) \
-{ \
- /* do big copy of middle stuff */ \
- std::copy(src, \
- src + this->fs->rd->local_size, \
- dst + this->buffer_size); \
- int rsrc; \
- /* get upper slices */ \
- for (int rdst = 0; rdst < this->fs->rd->nprocs; rdst++) \
- { \
- rsrc = this->fs->rd->rank[(this->fs->rd->all_start0[rdst] + \
- this->fs->rd->all_size0[rdst]) % \
- this->fs->rd->sizes[0]]; \
- if (this->fs->rd->myrank == rsrc) \
- MPI_Send( \
- (void*)(src), \
- this->buffer_size, \
- MPI_RNUM, \
- rdst, \
- 2*(rsrc*this->fs->rd->nprocs + rdst), \
- this->fs->rd->comm); \
- if (this->fs->rd->myrank == rdst) \
- MPI_Recv( \
- (void*)(dst + this->buffer_size + this->fs->rd->local_size), \
- this->buffer_size, \
- MPI_RNUM, \
- rsrc, \
- 2*(rsrc*this->fs->rd->nprocs + rdst), \
- this->fs->rd->comm, \
- MPI_STATUS_IGNORE); \
- } \
- /* get lower slices */ \
- for (int rdst = 0; rdst < this->fs->rd->nprocs; rdst++) \
- { \
- rsrc = this->fs->rd->rank[MOD(this->fs->rd->all_start0[rdst] - 1, \
- this->fs->rd->sizes[0])]; \
- if (this->fs->rd->myrank == rsrc) \
- MPI_Send( \
- (void*)(src + this->fs->rd->local_size - this->buffer_size), \
- this->buffer_size, \
- MPI_RNUM, \
- rdst, \
- 2*(rsrc*this->fs->rd->nprocs + rdst)+1, \
- this->fs->rd->comm); \
- if (this->fs->rd->myrank == rdst) \
- MPI_Recv( \
- (void*)(dst), \
- this->buffer_size, \
- MPI_RNUM, \
- rsrc, \
- 2*(rsrc*this->fs->rd->nprocs + rdst)+1, \
- this->fs->rd->comm, \
- MPI_STATUS_IGNORE); \
- } \
-} \
-/*****************************************************************************/
-
-
-
-/*****************************************************************************/
-/* now actually use the macro defined above */
-SLAB_FIELD_PARTICLES_DEFINITIONS(
- FFTW_MANGLE_FLOAT,
- float,
- MPI_FLOAT)
-SLAB_FIELD_PARTICLES_DEFINITIONS(
- FFTW_MANGLE_DOUBLE,
- double,
- MPI_DOUBLE)
-/*****************************************************************************/
-
-
-
-/*****************************************************************************/
-/* finally, force generation of code for single precision */
-template class slab_field_particles<float>;
-template class slab_field_particles<double>;
-/*****************************************************************************/
diff --git a/bfps/cpp/slab_field_particles.hpp b/bfps/cpp/slab_field_particles.hpp
deleted file mode 100644
index 15f9477bbfb680be17390447ce88bc40cd7471e2..0000000000000000000000000000000000000000
--- a/bfps/cpp/slab_field_particles.hpp
+++ /dev/null
@@ -1,149 +0,0 @@
-/**********************************************************************
-* *
-* Copyright 2015 Max Planck Institute *
-* for Dynamics and Self-Organization *
-* *
-* This file is part of bfps. *
-* *
-* bfps is free software: you can redistribute it and/or modify *
-* it under the terms of the GNU General Public License as published *
-* by the Free Software Foundation, either version 3 of the License, *
-* or (at your option) any later version. *
-* *
-* bfps is distributed in the hope that it will be useful, *
-* but WITHOUT ANY WARRANTY; without even the implied warranty of *
-* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the *
-* GNU General Public License for more details. *
-* *
-* You should have received a copy of the GNU General Public License *
-* along with bfps. If not, see <http://www.gnu.org/licenses/> *
-* *
-* Contact: Cristian.Lalescu@ds.mpg.de *
-* *
-**********************************************************************/
-
-
-
-#include <stdio.h>
-#include <stdlib.h>
-#include <iostream>
-#include <hdf5.h>
-#include "base.hpp"
-#include "fluid_solver_base.hpp"
-#include "interpolator.hpp"
-
-#ifndef SLAB_FIELD_PARTICLES
-
-#define SLAB_FIELD_PARTICLES
-
-extern int myrank, nprocs;
-
-template <class rnumber>
-class slab_field_particles
-{
- protected:
- //typedef void (slab_field_particles<rnumber>::*tensor_product_interpolation_formula)(
- // rnumber *field,
- // int *xg,
- // double *xx,
- // double *dest,
- // int *deriv);
- public:
- fluid_solver_base<rnumber> *fs;
- field_descriptor<rnumber> *buffered_field_descriptor;
-
- /* watching is an array of shape [nparticles], with
- * watching[p] being true if particle p is in the domain of myrank
- * or in the buffer regions.
- * watching is not really being used right now, since I don't do partial
- * synchronizations of particles.
- * we may do this at some point in the future, if it seems needed...
- * */
- bool *watching;
- /* computing is an array of shape [nparticles], with
- * computing[p] being the rank that is currently working on particle p
- * */
- int *computing;
-
- /* state will generally hold all the information about the particles.
- * in the beginning, we will only need to solve 3D ODEs, but I figured
- * a general ncomponents is better, since we may change our minds.
- * */
- double *state;
- double *rhs[6];
- int nparticles;
- int ncomponents;
- int array_size;
- int interp_neighbours;
- int buffer_width;
- int integration_steps;
- int traj_skip;
- ptrdiff_t buffer_size;
- double *lbound;
- double *ubound;
- //tensor_product_interpolation_formula spline_formula;
- base_polynomial_values compute_beta;
-
- /* simulation parameters */
- char name[256];
- int iteration;
- double dt;
-
- /* physical parameters of field */
- rnumber dx, dy, dz;
-
- /* methods */
-
- /* constructor and destructor.
- * allocate and deallocate:
- * this->state
- * this->lbound
- * this->ubound
- * this->watching
- * */
- slab_field_particles(
- const char *NAME,
- fluid_solver_base<rnumber> *FSOLVER,
- const int NPARTICLES,
- const int NCOMPONENTS,
- base_polynomial_values BETA_POLYS,
- const int INTERP_NEIGHBOURS,
- const int TRAJ_SKIP,
- const int INTEGRATION_STEPS = 2);
- ~slab_field_particles();
-
- /* an Euler step is needed to compute an estimate of future positions,
- * which is needed for synchronization.
- * */
- virtual void jump_estimate(double *jump_length);
- /* function get_rhs is virtual since we want children to do different things,
- * depending on the type of particle.
- * */
- virtual void get_rhs(double *x, double *rhs);
-
- /* generic methods, should work for all children of this class */
- int get_rank(double z); // get rank for given value of z
- void synchronize();
- void synchronize_single_particle_state(int p, double *x, int source_id = -1);
- void get_grid_coordinates(double *x, int *xg, double *xx);
- void linear_interpolation(rnumber *field, int *xg, double *xx, double *dest, int *deriv);
- void interpolation_formula(rnumber *field, int *xg, double *xx, double *dest, int *deriv);
-
- void rFFTW_to_buffered(rnumber *src, rnumber *dst);
-
- /* generic methods, should work for all children of this class */
- void read(hid_t data_file_id);
- void write(hid_t data_file_id, bool write_rhs = true);
-
- /* solver stuff */
- void step();
- void roll_rhs();
- void AdamsBashforth(int nsteps);
- void Euler();
- void Heun();
- void cRK4();
-};
-
-
-#endif//SLAB_FIELD_PARTICLES
-
diff --git a/bfps/cpp/spline.hpp b/bfps/cpp/spline.hpp
deleted file mode 100644
index d66d2b1eb42278b987072ffff24d0123c86a1e2f..0000000000000000000000000000000000000000
--- a/bfps/cpp/spline.hpp
+++ /dev/null
@@ -1,15 +0,0 @@
-#ifndef SPLINE_HPP
-#define SPLINE_HPP
-
-#include "spline_n1.hpp"
-#include "spline_n2.hpp"
-#include "spline_n3.hpp"
-#include "spline_n4.hpp"
-#include "spline_n5.hpp"
-#include "spline_n6.hpp"
-#include "spline_n7.hpp"
-#include "spline_n8.hpp"
-#include "spline_n9.hpp"
-#include "spline_n10.hpp"
-
-#endif
diff --git a/bfps/cpp/tracers.cpp b/bfps/cpp/tracers.cpp
deleted file mode 100644
index 3d9fbfb6a1e357d70452466b6cc901659444539d..0000000000000000000000000000000000000000
--- a/bfps/cpp/tracers.cpp
+++ /dev/null
@@ -1,204 +0,0 @@
-/**********************************************************************
-* *
-* Copyright 2015 Max Planck Institute *
-* for Dynamics and Self-Organization *
-* *
-* This file is part of bfps. *
-* *
-* bfps is free software: you can redistribute it and/or modify *
-* it under the terms of the GNU General Public License as published *
-* by the Free Software Foundation, either version 3 of the License, *
-* or (at your option) any later version. *
-* *
-* bfps is distributed in the hope that it will be useful, *
-* but WITHOUT ANY WARRANTY; without even the implied warranty of *
-* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the *
-* GNU General Public License for more details. *
-* *
-* You should have received a copy of the GNU General Public License *
-* along with bfps. If not, see <http://www.gnu.org/licenses/> *
-* *
-* Contact: Cristian.Lalescu@ds.mpg.de *
-* *
-**********************************************************************/
-
-
-
-#define NDEBUG
-
-
-#include <cmath>
-#include "base.hpp"
-#include "fftw_tools.hpp"
-#include "tracers.hpp"
-
-template <class rnumber>
-void tracers<rnumber>::jump_estimate(double *jump)
-{
- int deriv[] = {0, 0, 0};
- int *xg = new int[this->array_size];
- double *xx = new double[this->array_size];
- rnumber *vel = this->data + this->buffer_size;
- double tmp[3];
- /* get grid coordinates */
- this->get_grid_coordinates(this->state, xg, xx);
-
- /* perform interpolation */
- for (int p=0; p<this->nparticles; p++) if (this->fs->rd->myrank == this->computing[p])
- {
- this->interpolation_formula(vel, xg + p*3, xx + p*3, tmp, deriv);
- jump[p] = fabs(3*this->dt * tmp[2]);
- if (jump[p] < this->dz*1.01)
- jump[p] = this->dz*1.01;
- }
- delete[] xg;
- delete[] xx;
-}
-
-template <class rnumber>
-void tracers<rnumber>::get_rhs(double *x, double *y)
-{
- std::fill_n(y, this->array_size, 0.0);
- int deriv[] = {0, 0, 0};
- /* get grid coordinates */
- int *xg = new int[this->array_size];
- double *xx = new double[this->array_size];
- rnumber *vel = this->data + this->buffer_size;
- this->get_grid_coordinates(x, xg, xx);
- //DEBUG_MSG(
- // "position is %g %g %g, grid_coords are %d %d %d %g %g %g\n",
- // x[0], x[1], x[2],
- // xg[0], xg[1], xg[2],
- // xx[0], xx[1], xx[2]);
- /* perform interpolation */
- for (int p=0; p<this->nparticles; p++)
- {
- if (this->watching[this->fs->rd->myrank*this->nparticles+p])
- {
- int crank = this->get_rank(x[p*3 + 2]);
- if (this->fs->rd->myrank == crank)
- {
- this->interpolation_formula(vel, xg + p*3, xx + p*3, y + p*3, deriv);
- DEBUG_MSG(
- "position is %g %g %g %d %d %d %g %g %g, result is %g %g %g\n",
- x[p*3], x[p*3+1], x[p*3+2],
- xg[p*3], xg[p*3+1], xg[p*3+2],
- xx[p*3], xx[p*3+1], xx[p*3+2],
- y[p*3], y[p*3+1], y[p*3+2]);
- }
- if (crank != this->computing[p])
- {
- this->synchronize_single_particle_state(p, y, crank);
- }
- //DEBUG_MSG(
- // "after synch crank is %d, computing rank is %d, position is %g %g %g, result is %g %g %g\n",
- // this->iteration, p,
- // crank, this->computing[p],
- // x[p*3], x[p*3+1], x[p*3+2],
- // y[p*3], y[p*3+1], y[p*3+2]);
- }
- }
- delete[] xg;
- delete[] xx;
-}
-
-template<class rnumber>
-void tracers<rnumber>::update_field(bool clip_on)
-{
- if (clip_on)
- clip_zero_padding<rnumber>(this->fs->rd, this->source_data, 3);
- this->rFFTW_to_buffered(this->source_data, this->data);
-}
-
-/*****************************************************************************/
-/* macro for specializations to numeric types compatible with FFTW */
-
-#define TRACERS_DEFINITIONS(FFTW, R, MPI_RNUM, MPI_CNUM) \
- \
-template <> \
-tracers<R>::tracers( \
- const char *NAME, \
- fluid_solver_base<R> *FSOLVER, \
- const int NPARTICLES, \
- base_polynomial_values BETA_POLYS, \
- const int NEIGHBOURS, \
- const int TRAJ_SKIP, \
- const int INTEGRATION_STEPS, \
- R *SOURCE_DATA) : slab_field_particles<R>( \
- NAME, \
- FSOLVER, \
- NPARTICLES, \
- 3, \
- BETA_POLYS, \
- NEIGHBOURS, \
- TRAJ_SKIP, \
- INTEGRATION_STEPS) \
-{ \
- this->source_data = SOURCE_DATA; \
- this->data = FFTW(alloc_real)(this->buffered_field_descriptor->local_size); \
-} \
- \
-template<> \
-tracers<R>::~tracers() \
-{ \
- FFTW(free)(this->data); \
-} \
- \
-template <> \
-void tracers<R>::sample_vec_field(R *vec_field, double *vec_values) \
-{ \
- vec_field += this->buffer_size; \
- double *vec_local = new double[this->array_size]; \
- std::fill_n(vec_local, this->array_size, 0.0); \
- int deriv[] = {0, 0, 0}; \
- /* get grid coordinates */ \
- int *xg = new int[this->array_size]; \
- double *xx = new double[this->array_size]; \
- this->get_grid_coordinates(this->state, xg, xx); \
- /* perform interpolation */ \
- for (int p=0; p<this->nparticles; p++) \
- if (this->fs->rd->myrank == this->computing[p]) \
- this->interpolation_formula( \
- vec_field, \
- xg + p*3, \
- xx + p*3, \
- vec_local + p*3, \
- deriv); \
- MPI_Allreduce( \
- vec_local, \
- vec_values, \
- this->array_size, \
- MPI_DOUBLE, \
- MPI_SUM, \
- this->fs->rd->comm); \
- delete[] xg; \
- delete[] xx; \
- delete[] vec_local; \
-} \
-
-/*****************************************************************************/
-
-
-
-/*****************************************************************************/
-/* now actually use the macro defined above */
-TRACERS_DEFINITIONS(
- FFTW_MANGLE_FLOAT,
- float,
- MPI_FLOAT,
- MPI_COMPLEX)
-TRACERS_DEFINITIONS(
- FFTW_MANGLE_DOUBLE,
- double,
- MPI_DOUBLE,
- BFPS_MPICXX_DOUBLE_COMPLEX)
-/*****************************************************************************/
-
-
-
-/*****************************************************************************/
-/* finally, force generation of code */
-template class tracers<float>;
-template class tracers<double>;
-/*****************************************************************************/
-
diff --git a/bfps/test/B32p1e4_checkpoint_0.h5 b/bfps/test/B32p1e4_checkpoint_0.h5
deleted file mode 100644
index 236da62d4dffeb15b056ea3869f6343c74298e94..0000000000000000000000000000000000000000
Binary files a/bfps/test/B32p1e4_checkpoint_0.h5 and /dev/null differ
diff --git a/bfps/test/test_bfps_NSVEparticles.py b/bfps/test/test_bfps_NSVEparticles.py
deleted file mode 100644
index ab77e2103ccda7685cebe759f8e11cfe2a5b5ec9..0000000000000000000000000000000000000000
--- a/bfps/test/test_bfps_NSVEparticles.py
+++ /dev/null
@@ -1,56 +0,0 @@
-#! /usr/bin/env python
-
-import os
-import numpy as np
-import h5py
-import sys
-
-import bfps
-from bfps import DNS
-
-
-def main():
- niterations = 32
- nparticles = 10000
- njobs = 2
- c = DNS()
- c.launch(
- ['NSVEparticles',
- '-n', '32',
- '--src-simname', 'B32p1e4',
- '--src-wd', bfps.lib_dir + '/test',
- '--src-iteration', '0',
- '--simname', 'dns_nsveparticles',
- '--np', '4',
- '--ntpp', '1',
- '--niter_todo', '{0}'.format(niterations),
- '--niter_out', '{0}'.format(niterations),
- '--niter_stat', '1',
- '--checkpoints_per_file', '{0}'.format(3),
- '--nparticles', '{0}'.format(nparticles),
- '--particle-rand-seed', '2',
- '--njobs', '{0}'.format(njobs),
- '--wd', './'] +
- sys.argv[1:])
- f0 = h5py.File(
- os.path.join(
- os.path.join(bfps.lib_dir, 'test'),
- 'B32p1e4_checkpoint_0.h5'),
- 'r')
- f1 = h5py.File(c.get_checkpoint_0_fname(), 'r')
- for iteration in [0, 32, 64]:
- field0 = f0['vorticity/complex/{0}'.format(iteration)].value
- field1 = f1['vorticity/complex/{0}'.format(iteration)].value
- assert(np.max(np.abs(field0 - field1)) < 1e-5)
- x0 = f0['tracers0/state/{0}'.format(iteration)].value
- x1 = f1['tracers0/state/{0}'.format(iteration)].value
- assert(np.max(np.abs(x0 - x1)) < 1e-5)
- y0 = f0['tracers0/rhs/{0}'.format(iteration)].value
- y1 = f1['tracers0/rhs/{0}'.format(iteration)].value
- assert(np.max(np.abs(y0 - y1)) < 1e-5)
- print('SUCCESS! Basic test passed.')
- return None
-
-if __name__ == '__main__':
- main()
-
diff --git a/cmake/FindSphinx.cmake b/cmake/FindSphinx.cmake
new file mode 100644
index 0000000000000000000000000000000000000000..6e9efe350e5860f5fabea2d6695046313c85b593
--- /dev/null
+++ b/cmake/FindSphinx.cmake
@@ -0,0 +1,17 @@
+# taken from https://eb2.co/blog/2012/03/sphinx-and-cmake-beautiful-documentation-for-c---projects/
+
+find_program(SPHINX_EXECUTABLE NAMES sphinx-build
+ HINTS
+ $ENV{SPHINX_DIR}
+ PATH_SUFFIXES bin
+ DOC "Sphinx documentation generator"
+)
+
+include(FindPackageHandleStandardArgs)
+
+find_package_handle_standard_args(Sphinx DEFAULT_MSG
+ SPHINX_EXECUTABLE
+)
+
+mark_as_advanced(SPHINX_EXECUTABLE)
+
diff --git a/bfps/__init__.py b/cmake/TurTLEConfig.cmake.in
similarity index 53%
rename from bfps/__init__.py
rename to cmake/TurTLEConfig.cmake.in
index 6c220e69d877670206e411c5a0f1f1ae78c04d33..1b4a7057a30fe453211fff0eb4d73b1996af32d7 100644
--- a/bfps/__init__.py
+++ b/cmake/TurTLEConfig.cmake.in
@@ -1,6 +1,6 @@
#######################################################################
# #
-# Copyright 2015 Max Planck Institute #
+# Copyright 2019 Max Planck Institute #
# for Dynamics and Self-Organization #
# #
# This file is part of bfps. #
@@ -23,33 +23,47 @@
#######################################################################
+#-----------------------------------------------------------------------------
+#
+# TurTLEConfig.cmake - TurTLE CMake configuration file for external projects.
+#
+# This file is configured by TurTLE and used by the TurTLE.cmake module
+# to load TurTLE's settings for an external project.
+#
+@TURTLE_CONFIG_INSTALL_ONLY@
-import os
-import sys
-import pickle
+#
+SET(TURTLE_VERSION "@TURTLE_VERSION@")
-import pkg_resources
+#
+SET(HAVE_TURTLE TRUE)
+SET(TURTLE_PREFIX "@CMAKE_INSTALL_PREFIX@")
+SET(TURTLE_INCLUDE_DIR "@CMAKE_INSTALL_PREFIX@/include")
+SET(TURTLE_LIBRARIES_DIR "@CMAKE_INSTALL_PREFIX@/lib")
-__version__ = pkg_resources.require('bfps')[0].version
+SET(TURTLE_LINK_DIRECTORIES "@ALL_LINK_DIRS@")
+SET(TURTLE_INCLUDE_DIRECTORIES "@ALL_INCLUDE_DIRS@")
-_dist = pkg_resources.get_distribution('bfps')
-dist_loc = os.path.realpath(_dist.location)
-here = os.path.normcase(__file__)
-header_dir = os.path.join(os.path.join(dist_loc, 'bfps'), 'cpp')
-lib_dir = os.path.join(dist_loc, 'bfps')
+SET(TURTLE_CXX_COMPILE_FLAGS "@CMAKE_CXX_COMPILE_FLAGS@")
+SET(TURTLE_CXX_COMPILER "@CMAKE_CXX_COMPILER@")
+SET(TURTLE_C_COMPILER "@CMAKE_C_COMPILER@")
+SET(TURTLE_EXE_LINKER_FLAGS "@CMAKE_EXE_LINKER_FLAGS@")
+SET(TURTLE_LIBS "@TURTLE_LIBS@")
+set(TURTLE_DEFINITIONS @COMPILE_DEFINITIONS@)
+set(NDEBUG "@NDEBUG@")
+set(TIMING_OUTPUT "@TIMING_OUTPUT@")
-install_info = pickle.load(
- open(os.path.join(os.path.dirname(here), 'install_info.pickle'), 'rb'))
+#
+SET(TURTLE_SOURCE_DIR "@TURTLE_SOURCE_DIR@")
-homefolder = os.path.expanduser('~')
-bfpsfolder = os.path.join(homefolder, '.config/', 'bfps')
-sys.path.append(bfpsfolder)
-from host_information import host_info
+#
+SET(TURTLE_BUILD_TYPE "@CMAKE_BUILD_TYPE@")
-from .DNS import DNS
-from .FluidConvert import FluidConvert
-from .FluidResize import FluidResize
-from .NavierStokes import NavierStokes
-from .NSVorticityEquation import NSVorticityEquation
+#
+SET(TURTLE_HDF5_USE_SZIP "@TURTLE_HDF5_USE_SZIP@")
+SET(TURTLE_HDF5_SZIP_LIB_PATH "@TURTLE_HDF5_SZIP_LIB_PATH@")
+
+#
+set(TURTLE_SRC_INCLUDE_DIRS "@TURTLE_INCLUDE_DIRS@")
+set(TURTLE_BUILD_LIBRARY_DIRS "@TURTLE_LIB_DIR@")
-#import test
diff --git a/cmake/morse/FindCommon.cmake b/cmake/morse/FindCommon.cmake
new file mode 100644
index 0000000000000000000000000000000000000000..95d8c1f5404c0d7ea2384d84bd12c2e4a3cc3418
--- /dev/null
+++ b/cmake/morse/FindCommon.cmake
@@ -0,0 +1,47 @@
+###
+#
+# @copyright (c) 2018 Inria. All rights reserved.
+#
+###
+#
+# @file FindCommon.cmake
+#
+# @project MORSE
+# MORSE is a software package provided by:
+# Inria Bordeaux - Sud-Ouest,
+# Univ. of Tennessee,
+# King Abdullah Univesity of Science and Technology
+# Univ. of California Berkeley,
+# Univ. of Colorado Denver.
+#
+# @version 1.0.0
+# @author Florent Pruvost
+# @date 13-04-2018
+#
+###
+
+# clean these variables before using them in CMAKE_REQUIRED_* variables in
+# check_function_exists
+macro(finds_remove_duplicates)
+ if (REQUIRED_DEFINITIONS)
+ list(REMOVE_DUPLICATES REQUIRED_DEFINITIONS)
+ endif()
+ if (REQUIRED_INCDIRS)
+ list(REMOVE_DUPLICATES REQUIRED_INCDIRS)
+ endif()
+ if (REQUIRED_FLAGS)
+ list(REMOVE_DUPLICATES REQUIRED_FLAGS)
+ endif()
+ if (REQUIRED_LDFLAGS)
+ list(REMOVE_DUPLICATES REQUIRED_LDFLAGS)
+ endif()
+ if (REQUIRED_LIBS)
+ list(REVERSE REQUIRED_LIBS)
+ list(REMOVE_DUPLICATES REQUIRED_LIBS)
+ list(REVERSE REQUIRED_LIBS)
+ endif()
+endmacro()
+
+##
+## @end file FindCommon
+##
diff --git a/cmake/morse/FindFFTW.cmake b/cmake/morse/FindFFTW.cmake
new file mode 100644
index 0000000000000000000000000000000000000000..37450baea9f52a9a4e8a1236d6234d3c3840ba79
--- /dev/null
+++ b/cmake/morse/FindFFTW.cmake
@@ -0,0 +1,832 @@
+###
+#
+# @copyright (c) 2009-2014 The University of Tennessee and The University
+# of Tennessee Research Foundation.
+# All rights reserved.
+# @copyright (c) 2012-2018 Inria. All rights reserved.
+# @copyright (c) 2012-2014 Bordeaux INP, CNRS (LaBRI UMR 5800), Inria, Univ. Bordeaux. All rights reserved.
+#
+###
+#
+# - Find FFTW Version 3 include dirs and libraries
+# Default configuration will find the real double precision fftw library version
+# without THREADS|OMP.
+# Use this module by invoking find_package with the form:
+# find_package(FFTW
+# [REQUIRED] # Fail with error if fftw is not found
+# [COMPONENTS MKL]
+#
+# COMPONENTS can be some of the following:
+# - MKL: to detect the FFTW from Intel MKL
+# - ESSL: to detect the FFTW from IBM ESSL
+# - THREADS: to detect the Threads version of FFTW
+# - OMP: to detect the OpenMP version of FFTW
+# - SIMPLE: to detect the FFTW simple precision fftw3f
+# - LONG: to detect the FFTW long double precision fftw3l
+# - QUAD: to detect the FFTW quadruple precision fftw3q
+#
+# This module finds headers and fftw library.
+# Results are reported in variables:
+# FFTW_FOUND - True if headers and requested libraries were found
+# FFTW_CFLAGS_OTHER - fftw compiler flags without headers paths
+# FFTW_LDFLAGS_OTHER - fftw linker flags without libraries
+# FFTW_INCLUDE_DIRS - fftw include directories
+# FFTW_LIBRARY_DIRS - fftw link directories
+# FFTW_LIBRARIES - fftw libraries to be linked (absolute path)
+# FFTW_CFLAGS_OTHER_DEP - fftw + dependencies compiler flags without headers paths
+# FFTW_LDFLAGS_OTHER_DEP - fftw + dependencies linker flags without libraries
+# FFTW_INCLUDE_DIRS_DEP - fftw + dependencies include directories
+# FFTW_LIBRARY_DIRS_DEP - fftw + dependencies link directories
+# FFTW_LIBRARIES_DEP - fftw + dependencies libraries
+#
+# FFTW_FOUND_WITH_PKGCONFIG - True if found with pkg-config
+# if found with pkg-config the following variables are set
+# <PREFIX> = FFTW3F or FFTW3 or FFTW3L or FFTW3Q
+# <XPREFIX> = <PREFIX> for common case
+# <XPREFIX> = <PREFIX>_STATIC for static linking
+# <XPREFIX>_FOUND ... set to 1 if module(s) exist
+# <XPREFIX>_LIBRARIES ... only the libraries (w/o the '-l')
+# <XPREFIX>_LIBRARY_DIRS ... the paths of the libraries (w/o the '-L')
+# <XPREFIX>_LDFLAGS ... all required linker flags
+# <XPREFIX>_LDFLAGS_OTHER ... all other linker flags
+# <XPREFIX>_INCLUDE_DIRS ... the '-I' preprocessor flags (w/o the '-I')
+# <XPREFIX>_CFLAGS ... all required cflags
+# <XPREFIX>_CFLAGS_OTHER ... the other compiler flags
+#
+# The user can give specific paths where to find the libraries adding cmake
+# options at configure (ex: cmake path/to/project -DFFTW_DIR=path/to/fftw):
+# FFTW_DIR - Where to find the base directory of fftw
+# FFTW_INCDIR - Where to find the header files
+# FFTW_LIBDIR - Where to find the library files
+# The module can also look for the following environment variables if paths
+# are not given as cmake variable: FFTW_DIR, FFTW_INCDIR, FFTW_LIBDIR
+# For MKL case and if no paths are given as hints, we will try to use the MKLROOT
+# environment variable
+
+#=============================================================================
+# Copyright 2012-2018 Inria
+# Copyright 2012-2013 Emmanuel Agullo
+# Copyright 2012-2013 Mathieu Faverge
+# Copyright 2012 Cedric Castagnede
+# Copyright 2013-2018 Florent Pruvost
+#
+# Distributed under the OSI-approved BSD License (the "License");
+# see accompanying file MORSE-Copyright.txt for details.
+#
+# This software is distributed WITHOUT ANY WARRANTY; without even the
+# implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
+# See the License for more information.
+#=============================================================================
+# (To distribute this file outside of Morse, substitute the full
+# License text for the above reference.)
+
+# Common macros to use in finds
+include(FindInit)
+
+if (NOT FFTW_FOUND)
+ set(FFTW_DIR "" CACHE PATH "Installation directory of FFTW library given by user")
+ if (NOT FFTW_FIND_QUIETLY)
+ message(STATUS "A cache variable, namely FFTW_DIR, has been set to specify the install directory of FFTW")
+ endif()
+endif()
+
+# Set the version to find
+set(FFTW_LOOK_FOR_MKL OFF)
+set(FFTW_LOOK_FOR_ESSL OFF)
+set(FFTW_LOOK_FOR_THREADS OFF)
+set(FFTW_LOOK_FOR_OMP OFF)
+set(FFTW_LOOK_FOR_FFTW_SIMPLE OFF)
+set(FFTW_LOOK_FOR_FFTW_LONG OFF)
+set(FFTW_LOOK_FOR_FFTW_QUAD OFF)
+
+if( FFTW_FIND_COMPONENTS )
+ foreach( component ${FFTW_FIND_COMPONENTS} )
+ if (${component} STREQUAL "THREADS")
+ # means we look for the Threads version of FFTW
+ set(FFTW_LOOK_FOR_THREADS ON)
+ endif()
+ if (${component} STREQUAL "OMP")
+ # means we look for the OpenMP version of FFTW
+ set(FFTW_LOOK_FOR_OMP ON)
+ endif()
+ if (${component} STREQUAL "SIMPLE")
+ # means we look for FFTW simple precision (fftw3f)
+ set(FFTW_LOOK_FOR_FFTW_SIMPLE ON)
+ set(FFTW_LOOK_FOR_FFTW_LONG OFF)
+ set(FFTW_LOOK_FOR_FFTW_QUAD OFF)
+ endif()
+ if (${component} STREQUAL "LONG")
+ # means we look for FFTW long double precision (fftw3l)
+ set(FFTW_LOOK_FOR_FFTW_SIMPLE OFF)
+ set(FFTW_LOOK_FOR_FFTW_LONG ON)
+ set(FFTW_LOOK_FOR_FFTW_QUAD OFF)
+ endif()
+ if (${component} STREQUAL "QUAD")
+ # means we look for FFTW quad precision (fftw3q)
+ set(FFTW_LOOK_FOR_FFTW_SIMPLE OFF)
+ set(FFTW_LOOK_FOR_FFTW_LONG OFF)
+ set(FFTW_LOOK_FOR_FFTW_QUAD ON)
+ endif()
+ if (${component} STREQUAL "MKL")
+ # means we look for the Intel MKL version of FFTW
+ set(FFTW_LOOK_FOR_MKL ON)
+ if (FFTW_LOOK_FOR_FFTW_LONG)
+ if (NOT FFTW_FIND_QUIETLY)
+ message(WARNING "Looking for FFTW -- long precision functions do not exist in MKL FFTW")
+ endif()
+ set(FFTW_LOOK_FOR_FFTW_LONG OFF)
+ endif()
+ if (FFTW_LOOK_FOR_FFTW_QUAD)
+ if (NOT FFTW_FIND_QUIETLY)
+ message(WARNING "Looking for FFTW -- quadruple functions do not exist in MKL FFTW")
+ endif()
+ set(FFTW_LOOK_FOR_FFTW_QUAD OFF)
+ endif()
+ endif()
+ if (${component} STREQUAL "ESSL")
+ # means we look for the Intel MKL version of FFTW
+ set(FFTW_LOOK_FOR_ESSL ON)
+ if (FFTW_LOOK_FOR_FFTW_LONG)
+ if (NOT FFTW_FIND_QUIETLY)
+ message(WARNING "Looking for FFTW -- long precision functions do not exist in FFTW_ESSL")
+ endif()
+ set(FFTW_LOOK_FOR_FFTW_LONG OFF)
+ endif()
+ if (FFTW_LOOK_FOR_FFTW_QUAD)
+ if (NOT FFTW_FIND_QUIETLY)
+ message(WARNING "Looking for FFTW -- quadruple functions do not exist in FFTW_ESSL")
+ endif()
+ set(FFTW_LOOK_FOR_FFTW_QUAD OFF)
+ endif()
+ if (FFTW_LOOK_FOR_OMP)
+ if (NOT FFTW_FIND_QUIETLY)
+ message(WARNING "Looking for FFTW -- FFTW_ESSL does not use OpenMP")
+ endif()
+ set(FFTW_LOOK_FOR_OMP OFF)
+ endif()
+ endif()
+ endforeach()
+endif()
+
+if (FFTW_LOOK_FOR_THREADS)
+ if (NOT FFTW_FIND_QUIETLY)
+ message(STATUS "FFTW looks for threads")
+ endif()
+ if (FFTW_FIND_REQUIRED AND FFTW_FIND_REQUIRED_THREADS)
+ find_package(Threads REQUIRED)
+ else()
+ find_package(Threads)
+ endif()
+endif()
+
+if (FFTW_LOOK_FOR_OMP)
+ if (NOT FFTW_FIND_QUIETLY)
+ message(STATUS "FFTW looks for openmp")
+ endif()
+ if (FFTW_FIND_REQUIRED AND FFTW_FIND_REQUIRED_OMP)
+ find_package(OpenMP REQUIRED)
+ else()
+ find_package(OpenMP)
+ endif()
+endif()
+
+if (FFTW_LOOK_FOR_MKL)
+ if (NOT FFTW_FIND_QUIETLY)
+ message(STATUS "FFTW looks for threads and Intel MKL")
+ endif()
+ if (FFTW_LOOK_FOR_THREADS)
+ set(BLA_VENDOR "Intel10_64lp")
+ else()
+ set(BLA_VENDOR "Intel10_64lp_seq")
+ endif()
+ if (FFTW_FIND_REQUIRED AND FFTW_FIND_REQUIRED_MKL)
+ find_package(Threads REQUIRED)
+ find_package(BLAS REQUIRED)
+ else()
+ find_package(Threads)
+ find_package(BLAS)
+ endif()
+endif()
+
+if (FFTW_LOOK_FOR_ESSL)
+ if (NOT FFTW_FIND_QUIETLY)
+ message(STATUS "FFTW looks for IBM ESSL")
+ endif()
+ if (FFTW_LOOK_FOR_THREADS)
+ set(BLA_VENDOR "IBMESSLMT")
+ else()
+ set(BLA_VENDOR "IBMESSL")
+ endif()
+ if (FFTW_FIND_REQUIRED AND FFTW_FIND_REQUIRED_ESSL)
+ find_package(BLAS REQUIRED)
+ else()
+ find_package(BLAS)
+ endif()
+endif()
+
+
+if( THREADS_FOUND )
+ libraries_absolute_path(CMAKE_THREAD_LIBS_INIT "")
+endif ()
+
+set(ENV_FFTW_DIR "$ENV{FFTW_DIR}")
+set(ENV_FFTW_INCDIR "$ENV{FFTW_INCDIR}")
+set(ENV_FFTW_LIBDIR "$ENV{FFTW_LIBDIR}")
+set(FFTW_GIVEN_BY_USER "FALSE")
+if ( FFTW_DIR OR ( FFTW_INCDIR AND FFTW_LIBDIR) OR ENV_FFTW_DIR OR (ENV_FFTW_INCDIR AND ENV_FFTW_LIBDIR) )
+ set(FFTW_GIVEN_BY_USER "TRUE")
+endif()
+
+
+# Optionally use pkg-config to detect include/library dirs (if pkg-config is available)
+# -------------------------------------------------------------------------------------
+if (NOT FFTW_LOOK_FOR_MKL AND NOT FFTW_LOOK_FOR_ESSL)
+ include(FindPkgConfig)
+ find_package(PkgConfig QUIET)
+ if( PKG_CONFIG_EXECUTABLE AND NOT FFTW_GIVEN_BY_USER )
+
+ set(FFTW_INCLUDE_DIRS)
+ set(FFTW_LIBRARY_DIRS)
+ set(FFTW_LIBRARIES)
+
+ if(FFTW_LOOK_FOR_FFTW_SIMPLE)
+ pkg_search_module(FFTW3F fftw3f)
+ pkg_search_module(FFTW3 fftw3)
+ if (FFTW3F_FOUND)
+ if (NOT FFTW_FIND_QUIETLY)
+ message(STATUS "Looking for FFTW3F - found using PkgConfig")
+ endif()
+ if (FFTW3F_LIBRARIES)
+ find_pkgconfig_libraries_absolute_path(FFTW3F)
+ list(APPEND FFTW_LIBRARIES "${FFTW3F_LIBRARIES}")
+ endif()
+ if(FFTW3F_INCLUDE_DIRS)
+ list(APPEND FFTW_INCLUDE_DIRS "${FFTW3F_INCLUDE_DIRS}")
+ else()
+ if (NOT FFTW_FIND_QUIETLY)
+ message(WARNING "FFTW3F_INCLUDE_DIRS is empty using PkgConfig."
+ "Perhaps the path to fftw3f headers is already present in your"
+ "CPATH/C(PLUS)_INCLUDE_PATH environment variables.")
+ endif()
+ endif()
+ if(FFTW3F_LIBRARY_DIRS)
+ list(APPEND FFTW_LIBRARY_DIRS "${FFTW3F_LIBRARY_DIRS}")
+ endif()
+ else(FFTW3F_FOUND)
+ if (NOT FFTW_FIND_QUIETLY)
+ message(STATUS "Looking for FFTW3F - not found using PkgConfig."
+ "\n Perhaps you should add the directory containing fftw3f.pc to"
+ "\n the PKG_CONFIG_PATH environment variable.")
+ endif()
+ endif(FFTW3F_FOUND)
+ elseif(FFTW_LOOK_FOR_FFTW_LONG)
+ pkg_search_module(FFTW3L fftw3l)
+ pkg_search_module(FFTW3 fftw3)
+ if (FFTW3L_FOUND)
+ if (NOT FFTW_FIND_QUIETLY)
+ message(STATUS "Looking for FFTW3L - found using PkgConfig")
+ endif()
+ if (FFTW3L_LIBRARIES)
+ find_pkgconfig_libraries_absolute_path(FFTW3L)
+ list(APPEND FFTW_LIBRARIES "${FFTW3L_LIBRARIES}")
+ endif()
+ if(FFTW3L_INCLUDE_DIRS)
+ list(APPEND FFTW_INCLUDE_DIRS "${FFTW3L_INCLUDE_DIRS}")
+ else()
+ if (NOT FFTW_FIND_QUIETLY)
+ message(WARNING "FFTW3L_INCLUDE_DIRS is empty using PkgConfig."
+ "Perhaps the path to fftw3l headers is already present in your"
+ "CPATH/C(PLUS)_INCLUDE_PATH environment variables.")
+ endif()
+ endif()
+ if(FFTW3L_LIBRARY_DIRS)
+ list(APPEND FFTW_LIBRARY_DIRS "${FFTW3L_LIBRARY_DIRS}")
+ endif()
+ else(FFTW3L_FOUND)
+ if (NOT FFTW_FIND_QUIETLY)
+ message(STATUS "Looking for FFTW3L - not found using PkgConfig."
+ "\n Perhaps you should add the directory containing fftw3l.pc to"
+ "\n the PKG_CONFIG_PATH environment variable.")
+ endif()
+ endif(FFTW3L_FOUND)
+ elseif(FFTW_LOOK_FOR_FFTW_QUAD)
+ pkg_search_module(FFTW3Q fftw3q)
+ pkg_search_module(FFTW3 fftw3)
+ if (FFTW3Q_FOUND)
+ if (NOT FFTW_FIND_QUIETLY)
+ message(STATUS "Looking for FFTW3Q - found using PkgConfig")
+ endif()
+ if (FFTW3Q_LIBRARIES)
+ find_pkgconfig_libraries_absolute_path(FFTW3Q)
+ list(APPEND FFTW_LIBRARIES "${FFTW3Q_LIBRARIES}")
+ endif()
+ if(FFTW3Q_INCLUDE_DIRS)
+ list(APPEND FFTW_INCLUDE_DIRS "${FFTW3Q_INCLUDE_DIRS}")
+ else()
+ if (NOT FFTW_FIND_QUIETLY)
+ message(WARNING "FFTW3Q_INCLUDE_DIRS is empty using PkgConfig."
+ "Perhaps the path to fftw3q headers is already present in your"
+ "CPATH/C(PLUS)_INCLUDE_PATH environment variables.")
+ endif()
+ endif()
+ if(FFTW3Q_LIBRARY_DIRS)
+ list(APPEND FFTW_LIBRARY_DIRS "${FFTW3Q_LIBRARY_DIRS}")
+ endif()
+ else(FFTW3Q_FOUND)
+ if (NOT FFTW_FIND_QUIETLY)
+ message(STATUS "Looking for FFTW3Q - not found using PkgConfig."
+ "\n Perhaps you should add the directory containing fftw3q.pc to"
+ "\n the PKG_CONFIG_PATH environment variable.")
+ endif()
+ endif(FFTW3Q_FOUND)
+ else()
+ pkg_search_module(FFTW3 fftw3)
+ if (FFTW3_FOUND AND FFTW3_LIBRARIES)
+ find_pkgconfig_libraries_absolute_path(FFTW3)
+ endif()
+ endif()
+ if (FFTW3_FOUND)
+ if (NOT FFTW_FIND_QUIETLY)
+ message(STATUS "Looking for FFTW3 - found using PkgConfig")
+ endif()
+ if (FFTW3_LIBRARIES)
+ find_pkgconfig_libraries_absolute_path(FFTW3)
+ list(APPEND FFTW_LIBRARIES "${FFTW3_LIBRARIES}")
+ endif()
+ if(FFTW3_INCLUDE_DIRS)
+ list(APPEND FFTW_INCLUDE_DIRS "${FFTW3_INCLUDE_DIRS}")
+ else()
+ if (NOT FFTW_FIND_QUIETLY)
+ message(WARNING "FFTW3_INCLUDE_DIRS is empty using PkgConfig."
+ "Perhaps the path to fftw3 headers is already present in your"
+ "CPATH/C(PLUS)_INCLUDE_PATH environment variables.")
+ endif()
+ endif()
+ if(FFTW3_LIBRARY_DIRS)
+ list(APPEND FFTW_LIBRARY_DIRS "${FFTW3_LIBRARY_DIRS}")
+ endif()
+ else(FFTW3_FOUND)
+ if (NOT FFTW_FIND_QUIETLY)
+ message(STATUS "Looking for FFTW3 - not found using PkgConfig."
+ "\n Perhaps you should add the directory containing fftw3.pc to"
+ "\n the PKG_CONFIG_PATH environment variable.")
+ endif()
+ endif(FFTW3_FOUND)
+
+ if (FFTW_FOUND AND FFTW_LIBRARIES)
+ set(FFTW_FOUND_WITH_PKGCONFIG "TRUE")
+ else()
+ set(FFTW_FOUND_WITH_PKGCONFIG "FALSE")
+ endif()
+
+ endif( PKG_CONFIG_EXECUTABLE AND NOT FFTW_GIVEN_BY_USER )
+
+endif(NOT FFTW_LOOK_FOR_MKL AND NOT FFTW_LOOK_FOR_ESSL)
+
+if( (NOT PKG_CONFIG_EXECUTABLE) OR
+ (PKG_CONFIG_EXECUTABLE AND NOT FFTW_FOUND) OR
+ FFTW_GIVEN_BY_USER OR
+ FFTW_LOOK_FOR_MKL OR
+ FFTW_LOOK_FOR_ESSL
+ )
+
+ # Looking for include
+ # -------------------
+
+ # Add system include paths to search include
+ # ------------------------------------------
+ unset(_inc_env)
+ set(ENV_MKLROOT "$ENV{MKLROOT}")
+ set(ENV_FFTW_DIR "$ENV{FFTW_DIR}")
+ set(ENV_FFTW_INCDIR "$ENV{FFTW_INCDIR}")
+ if(ENV_FFTW_INCDIR)
+ list(APPEND _inc_env "${ENV_FFTW_INCDIR}")
+ elseif(ENV_FFTW_DIR)
+ list(APPEND _inc_env "${ENV_FFTW_DIR}")
+ list(APPEND _inc_env "${ENV_FFTW_DIR}/include")
+ list(APPEND _inc_env "${ENV_FFTW_DIR}/include/fftw")
+ else()
+ if (ENV_MKLROOT)
+ list(APPEND _inc_env "${ENV_MKLROOT}/include/fftw")
+ endif()
+ # system variables
+ if(WIN32)
+ string(REPLACE ":" ";" _path_env "$ENV{INCLUDE}")
+ list(APPEND _inc_env "${_path_env}")
+ else()
+ string(REPLACE ":" ";" _path_env "$ENV{INCLUDE}")
+ list(APPEND _inc_env "${_path_env}")
+ string(REPLACE ":" ";" _path_env "$ENV{C_INCLUDE_PATH}")
+ list(APPEND _inc_env "${_path_env}")
+ string(REPLACE ":" ";" _path_env "$ENV{CPATH}")
+ list(APPEND _inc_env "${_path_env}")
+ string(REPLACE ":" ";" _path_env "$ENV{INCLUDE_PATH}")
+ list(APPEND _inc_env "${_path_env}")
+ endif()
+ endif()
+ list(APPEND _inc_env "${CMAKE_C_IMPLICIT_INCLUDE_DIRECTORIES}")
+ list(REMOVE_DUPLICATES _inc_env)
+
+ # set paths where to look for
+ set(PATH_TO_LOOK_FOR "${_inc_env}")
+
+ if (FFTW_LOOK_FOR_ESSL)
+ set(FFTW3_HEADER_TO_FIND "fftw3_essl.h")
+ else()
+ set(FFTW3_HEADER_TO_FIND "fftw3.h")
+ endif()
+
+ # Try to find the fftw header in the given paths
+ # -------------------------------------------------
+ # call cmake macro to find the header path
+ if(FFTW_INCDIR)
+ set(FFTW_${FFTW3_HEADER_TO_FIND}_DIRS "FFTW_${FFTW3_HEADER_TO_FIND}_DIRS-NOTFOUND")
+ find_path(FFTW_${FFTW3_HEADER_TO_FIND}_DIRS
+ NAMES ${FFTW3_HEADER_TO_FIND}
+ HINTS ${FFTW_INCDIR})
+ else()
+ if(FFTW_DIR)
+ set(FFTW_${FFTW3_HEADER_TO_FIND}_DIRS "FFTW_${FFTW3_HEADER_TO_FIND}_DIRS-NOTFOUND")
+ find_path(FFTW_${FFTW3_HEADER_TO_FIND}_DIRS
+ NAMES ${FFTW3_HEADER_TO_FIND}
+ HINTS ${FFTW_DIR}
+ PATH_SUFFIXES "include" "include/fftw")
+ else()
+ set(FFTW_${FFTW3_HEADER_TO_FIND}_DIRS "FFTW_${FFTW3_HEADER_TO_FIND}_DIRS-NOTFOUND")
+ find_path(FFTW_${FFTW3_HEADER_TO_FIND}_DIRS
+ NAMES ${FFTW3_HEADER_TO_FIND}
+ HINTS ${PATH_TO_LOOK_FOR}
+ PATH_SUFFIXES "fftw")
+ endif()
+ endif()
+ mark_as_advanced(FFTW_${FFTW3_HEADER_TO_FIND}_DIRS)
+
+ # Add path to cmake variable
+ # ------------------------------------
+ if (FFTW_${FFTW3_HEADER_TO_FIND}_DIRS)
+ set(FFTW_INCLUDE_DIRS "${FFTW_${FFTW3_HEADER_TO_FIND}_DIRS}")
+ else ()
+ set(FFTW_INCLUDE_DIRS "FFTW_INCLUDE_DIRS-NOTFOUND")
+ if(NOT FFTW_FIND_QUIETLY)
+ message(STATUS "Looking for FFTW -- ${FFTW3_HEADER_TO_FIND} not found")
+ endif()
+ endif ()
+
+
+ # Looking for lib
+ # ---------------
+
+ # Add system library paths to search lib
+ # --------------------------------------
+ unset(_lib_env)
+ set(ENV_FFTW_LIBDIR "$ENV{FFTW_LIBDIR}")
+ if(ENV_FFTW_LIBDIR)
+ list(APPEND _lib_env "${ENV_FFTW_LIBDIR}")
+ elseif(ENV_FFTW_DIR)
+ list(APPEND _lib_env "${ENV_FFTW_DIR}")
+ list(APPEND _lib_env "${ENV_FFTW_DIR}/lib")
+ if("${CMAKE_SIZEOF_VOID_P}" EQUAL "8")
+ list(APPEND _lib_env "${ENV_FFTW_DIR}/lib64")
+ list(APPEND _lib_env "${ENV_FFTW_DIR}/lib/intel64")
+ else()
+ list(APPEND _lib_env "${ENV_FFTW_DIR}/lib32")
+ list(APPEND _lib_env "${ENV_FFTW_DIR}/lib/ia32")
+ endif()
+ else()
+ if (ENV_MKLROOT)
+ list(APPEND _lib_env "${ENV_MKLROOT}/lib")
+ if("${CMAKE_SIZEOF_VOID_P}" EQUAL "8")
+ list(APPEND _lib_env "${ENV_MKLROOT}/lib64")
+ list(APPEND _lib_env "${ENV_MKLROOT}/lib/intel64")
+ else()
+ list(APPEND _lib_env "${ENV_MKLROOT}/lib32")
+ list(APPEND _lib_env "${ENV_MKLROOT}/lib/ia32")
+ endif()
+ endif()
+ list(APPEND _lib_env "$ENV{LIBRARY_PATH}")
+ if(WIN32)
+ string(REPLACE ":" ";" _lib_env2 "$ENV{LIB}")
+ elseif(APPLE)
+ string(REPLACE ":" ";" _lib_env2 "$ENV{DYLD_LIBRARY_PATH}")
+ else()
+ string(REPLACE ":" ";" _lib_env2 "$ENV{LD_LIBRARY_PATH}")
+ endif()
+ list(APPEND _lib_env "${_lib_env2}")
+ list(APPEND _lib_env "${CMAKE_C_IMPLICIT_LINK_DIRECTORIES}")
+ endif()
+ list(REMOVE_DUPLICATES _lib_env)
+
+ # set paths where to look for
+ set(PATH_TO_LOOK_FOR "${_lib_env}")
+
+ if(FFTW_LOOK_FOR_FFTW_SIMPLE)
+ set(FFTW_PREC "f")
+ set(FFTW_PREC_TESTFUNC "s")
+ elseif(FFTW_LOOK_FOR_FFTW_LONG)
+ set(FFTW_PREC "l")
+ set(FFTW_PREC_TESTFUNC "l")
+ elseif(FFTW_LOOK_FOR_FFTW_QUAD)
+ set(FFTW_PREC "q")
+ set(FFTW_PREC_TESTFUNC "q")
+ else()
+ set(FFTW_PREC "")
+ set(FFTW_PREC_TESTFUNC "d")
+ endif()
+
+ set(FFTW_LIBRARIES "")
+ set(FFTW_LIBRARY_DIRS "")
+
+ if(NOT FFTW_LOOK_FOR_MKL)
+
+ if (FFTW_LOOK_FOR_THREADS)
+ set(FFTW_libs_to_find "fftw3${FFTW_PREC}_threads;fftw3${FFTW_PREC};fftw3")
+ elseif (FFTW_LOOK_FOR_OMP)
+ set(FFTW_libs_to_find "fftw3${FFTW_PREC}_omp;fftw3${FFTW_PREC};fftw3")
+ else()
+ set(FFTW_libs_to_find "fftw3${FFTW_PREC};fftw3")
+ endif()
+ if (FFTW_LOOK_FOR_FFTW_QUAD)
+ if (NOT FFTW_LOOK_FOR_MKL AND NOT FFTW_LOOK_FOR_ESSL)
+ list(APPEND FFTW_libs_to_find "quadmath")
+ endif()
+ endif()
+
+ if (FFTW_LOOK_FOR_ESSL)
+ set(FFTW_libs_to_find "fftw3_essl")
+ endif()
+
+ # Try to find the fftw lib in the given paths
+ # ----------------------------------------------
+
+ # call cmake macro to find the lib path
+ if(FFTW_LIBDIR)
+ foreach(fftw_lib ${FFTW_libs_to_find})
+ set(FFTW_${fftw_lib}_LIBRARY "FFTW_${fftw_lib}_LIBRARY-NOTFOUND")
+ find_library(FFTW_${fftw_lib}_LIBRARY
+ NAMES ${fftw_lib}
+ HINTS ${FFTW_LIBDIR})
+ endforeach()
+ else()
+ if(FFTW_DIR)
+ foreach(fftw_lib ${FFTW_libs_to_find})
+ set(FFTW_${fftw_lib}_LIBRARY "FFTW_${fftw_lib}_LIBRARY-NOTFOUND")
+ find_library(FFTW_${fftw_lib}_LIBRARY
+ NAMES ${fftw_lib}
+ HINTS ${FFTW_DIR}
+ PATH_SUFFIXES lib lib32 lib64)
+ endforeach()
+ else()
+ foreach(fftw_lib ${FFTW_libs_to_find})
+ set(FFTW_${fftw_lib}_LIBRARY "FFTW_${fftw_lib}_LIBRARY-NOTFOUND")
+ find_library(FFTW_${fftw_lib}_LIBRARY
+ NAMES ${fftw_lib}
+ HINTS ${PATH_TO_LOOK_FOR})
+ endforeach()
+ endif()
+ endif()
+
+ # If found, add path to cmake variable
+ # ------------------------------------
+ foreach(fftw_lib ${FFTW_libs_to_find})
+
+ if (FFTW_${fftw_lib}_LIBRARY)
+ get_filename_component(${fftw_lib}_lib_path "${FFTW_${fftw_lib}_LIBRARY}" PATH)
+ # set cmake variables
+ list(APPEND FFTW_LIBRARIES "${FFTW_${fftw_lib}_LIBRARY}")
+ list(APPEND FFTW_LIBRARY_DIRS "${${fftw_lib}_lib_path}")
+ else ()
+ list(APPEND FFTW_LIBRARIES "${FFTW_${fftw_lib}_LIBRARY}")
+ if (NOT FFTW_FIND_QUIETLY)
+ message(STATUS "Looking for FFTW -- lib ${fftw_lib} not found")
+ endif()
+ endif ()
+ mark_as_advanced(FFTW_${fftw_lib}_LIBRARY)
+
+ endforeach()
+
+ # check if one lib is NOTFOUND
+ foreach(lib ${FFTW_LIBRARIES})
+ if (NOT lib)
+ set(FFTW_LIBRARIES "FFTW_LIBRARIES-NOTFOUND")
+ endif()
+ endforeach()
+
+ endif(NOT FFTW_LOOK_FOR_MKL)
+
+ if (FFTW_LOOK_FOR_MKL OR FFTW_LOOK_FOR_ESSL)
+
+ # FFTW relies on blas libs
+ if (FFTW_LOOK_FOR_THREADS)
+ if (FFTW_LOOK_FOR_MKL)
+ if (BLAS_LIBRARIES_PAR)
+ list(APPEND FFTW_LIBRARIES "${BLAS_LIBRARIES_PAR}")
+ if (NOT FFTW_FIND_QUIETLY)
+ message(STATUS "Multithreaded FFTW has been found: ${FFTW_LIBRARIES}")
+ endif()
+ else()
+ if (NOT FFTW_FIND_QUIETLY)
+ if (FFTW_FIND_REQUIRED AND FFTW_FIND_REQUIRED_MKL)
+ message(FATAL_ERROR "FFTW is required but not found.")
+ else()
+ message(STATUS "Multithreaded FFTW not found.")
+ endif()
+ endif()
+ endif(BLAS_LIBRARIES_PAR)
+ elseif (FFTW_LOOK_FOR_ESSL)
+ if (FFTW_LIBRARIES AND BLAS_LIBRARIES_PAR)
+ list(APPEND FFTW_LIBRARIES "${BLAS_LIBRARIES_PAR}")
+ if (NOT FFTW_FIND_QUIETLY)
+ message(STATUS "Multithreaded FFTW has been found: ${FFTW_LIBRARIES}")
+ endif()
+ else()
+ if (NOT FFTW_FIND_QUIETLY)
+ if (FFTW_FIND_REQUIRED AND FFTW_FIND_REQUIRED_MKL)
+ message(FATAL_ERROR "FFTW is required but not found.")
+ else()
+ message(STATUS "Multithreaded FFTW not found.")
+ endif()
+ endif()
+ endif(FFTW_LIBRARIES AND BLAS_LIBRARIES_PAR)
+ endif()
+ else(FFTW_LOOK_FOR_THREADS)
+ if (FFTW_LOOK_FOR_MKL)
+ if (BLAS_LIBRARIES_SEQ)
+ list(APPEND FFTW_LIBRARIES "${BLAS_LIBRARIES_SEQ}")
+ if (NOT FFTW_FIND_QUIETLY)
+ message(STATUS "FFTW has been found: ${FFTW_LIBRARIES}")
+ endif()
+ else()
+ if (NOT FFTW_FIND_QUIETLY)
+ if (FFTW_FIND_REQUIRED AND FFTW_FIND_REQUIRED_MKL)
+ message(FATAL_ERROR "FFTW is required but not found.")
+ else()
+ message(STATUS "FFTW not found.")
+ endif()
+ endif()
+ endif(BLAS_LIBRARIES_SEQ)
+ elseif (FFTW_LOOK_FOR_ESSL)
+ if (FFTW_LIBRARIES AND BLAS_LIBRARIES_SEQ)
+ list(APPEND FFTW_LIBRARIES "${BLAS_LIBRARIES_SEQ}")
+ if (NOT FFTW_FIND_QUIETLY)
+ message(STATUS "FFTW has been found: ${FFTW_LIBRARIES}")
+ endif()
+ else()
+ if (NOT FFTW_FIND_QUIETLY)
+ if (FFTW_FIND_REQUIRED AND FFTW_FIND_REQUIRED_MKL)
+ message(FATAL_ERROR "FFTW is required but not found.")
+ else()
+ message(STATUS "FFTW not found.")
+ endif()
+ endif()
+ endif(FFTW_LIBRARIES AND BLAS_LIBRARIES_SEQ)
+ endif()
+ endif(FFTW_LOOK_FOR_THREADS)
+
+ if (BLAS_LIBRARY_DIRS)
+ list(APPEND FFTW_LIBRARY_DIRS "${BLAS_LIBRARY_DIRS}")
+ else()
+ if (NOT FFTW_FIND_QUIETLY)
+ message(WARNING "FFTW_LIBRARY_DIRS may not be complete because BLAS_LIBRARY_DIRS is empty.")
+ endif()
+ endif()
+
+ endif(FFTW_LOOK_FOR_MKL OR FFTW_LOOK_FOR_ESSL)
+
+ list(REMOVE_DUPLICATES FFTW_INCLUDE_DIRS)
+ list(REMOVE_DUPLICATES FFTW_LIBRARY_DIRS)
+
+ # check if one lib is NOTFOUND
+ foreach(lib ${FFTW_LIBRARIES})
+ if (NOT lib)
+ set(FFTW_LIBRARIES "FFTW_LIBRARIES-NOTFOUND")
+ endif()
+ endforeach()
+
+endif( (NOT PKG_CONFIG_EXECUTABLE) OR
+ (PKG_CONFIG_EXECUTABLE AND NOT FFTW_FOUND) OR
+ FFTW_GIVEN_BY_USER OR
+ FFTW_LOOK_FOR_MKL OR
+ FFTW_LOOK_FOR_ESSL
+ )
+
+# check a function to validate the find
+if(FFTW_LIBRARIES)
+
+ set(REQUIRED_FLAGS)
+ set(REQUIRED_LDFLAGS)
+ set(REQUIRED_INCDIRS)
+ set(REQUIRED_LIBDIRS)
+ set(REQUIRED_LIBS)
+
+ # FFTW
+ if (FFTW_INCLUDE_DIRS)
+ set(REQUIRED_INCDIRS "${FFTW_INCLUDE_DIRS}")
+ endif()
+ if (FFTW_CFLAGS_OTHER)
+ set(REQUIRED_FLAGS "${FFTW_CFLAGS_OTHER}")
+ endif()
+ if (FFTW_LDFLAGS_OTHER)
+ set(REQUIRED_LDFLAGS "${FFTW_LDFLAGS_OTHER}")
+ endif()
+ if (FFTW_LIBRARY_DIRS)
+ set(REQUIRED_LIBDIRS "${FFTW_LIBRARY_DIRS}")
+ endif()
+ set(REQUIRED_LIBS "${FFTW_LIBRARIES}")
+ # THREADS
+ if (FFTW_LOOK_FOR_THREADS)
+ list(APPEND REQUIRED_LIBS "${CMAKE_THREAD_LIBS_INIT}")
+ endif()
+ # OMP
+ if(FFTW_LOOK_FOR_OMP)
+ list(APPEND REQUIRED_FLAGS "${OPENMP_C_FLAGS}")
+ endif()
+ # MKL
+ if(FFTW_LOOK_FOR_MKL)
+ list(APPEND REQUIRED_LIBS "${CMAKE_THREAD_LIBS_INIT}")
+ if (CMAKE_C_COMPILER_ID STREQUAL "GNU" AND CMAKE_SYSTEM_NAME STREQUAL "Linux")
+ list(APPEND REQUIRED_LDFLAGS "-Wl,--no-as-needed")
+ endif()
+ endif()
+ # m
+ find_library(M_LIBRARY NAMES m)
+ mark_as_advanced(M_LIBRARY)
+ if(M_LIBRARY)
+ list(APPEND REQUIRED_LIBS "-lm")
+ endif()
+
+ # set required libraries for link
+ set(CMAKE_REQUIRED_INCLUDES "${REQUIRED_INCDIRS}")
+ if (REQUIRED_FLAGS)
+ set(REQUIRED_FLAGS_COPY "${REQUIRED_FLAGS}")
+ set(REQUIRED_FLAGS)
+ set(REQUIRED_DEFINITIONS)
+ foreach(_flag ${REQUIRED_FLAGS_COPY})
+ if (_flag MATCHES "^-D")
+ list(APPEND REQUIRED_DEFINITIONS "${_flag}")
+ endif()
+ string(REGEX REPLACE "^-D.*" "" _flag "${_flag}")
+ list(APPEND REQUIRED_FLAGS "${_flag}")
+ endforeach()
+ endif()
+ finds_remove_duplicates()
+ set(CMAKE_REQUIRED_DEFINITIONS "${REQUIRED_DEFINITIONS}")
+ set(CMAKE_REQUIRED_FLAGS "${REQUIRED_FLAGS}")
+ set(CMAKE_REQUIRED_LIBRARIES)
+ list(APPEND CMAKE_REQUIRED_LIBRARIES "${REQUIRED_LDFLAGS}")
+ list(APPEND CMAKE_REQUIRED_LIBRARIES "${REQUIRED_LIBS}")
+ list(APPEND CMAKE_REQUIRED_FLAGS "${REQUIRED_FLAGS}")
+ string(REGEX REPLACE "^ -" "-" CMAKE_REQUIRED_LIBRARIES "${CMAKE_REQUIRED_LIBRARIES}")
+
+ # test link
+ unset(FFTW_WORKS CACHE)
+ include(CheckFunctionExists)
+ if (FFTW_LOOK_FOR_ESSL)
+ check_function_exists(${FFTW_PREC_TESTFUNC}fftw_execute FFTW_WORKS)
+ else()
+ check_function_exists(${FFTW_PREC_TESTFUNC}fftw_execute_ FFTW_WORKS)
+ endif()
+ mark_as_advanced(FFTW_WORKS)
+
+ if(FFTW_WORKS)
+ # save link with dependencies
+ set(FFTW_LIBRARIES_DEP "${REQUIRED_LIBS}")
+ set(FFTW_LIBRARY_DIRS_DEP "${REQUIRED_LIBDIRS}")
+ set(FFTW_INCLUDE_DIRS_DEP "${REQUIRED_INCDIRS}")
+ set(FFTW_CFLAGS_OTHER_DEP "${REQUIRED_FLAGS}")
+ set(FFTW_LDFLAGS_OTHER_DEP "${REQUIRED_LDFLAGS}")
+ else()
+ if(NOT FFTW_FIND_QUIETLY)
+ message(STATUS "Looking for FFTW : test of ${FFTW_PREC_TESTFUNC}fftw_execute_ with fftw library fails")
+ message(STATUS "CMAKE_REQUIRED_LIBRARIES: ${CMAKE_REQUIRED_LIBRARIES}")
+ message(STATUS "CMAKE_REQUIRED_INCLUDES: ${CMAKE_REQUIRED_INCLUDES}")
+ message(STATUS "CMAKE_REQUIRED_FLAGS: ${CMAKE_REQUIRED_FLAGS}")
+ message(STATUS "Check in CMakeFiles/CMakeError.log to figure out why it fails")
+ endif()
+ endif()
+ set(CMAKE_REQUIRED_INCLUDES)
+ set(CMAKE_REQUIRED_FLAGS)
+ set(CMAKE_REQUIRED_LIBRARIES)
+endif(FFTW_LIBRARIES)
+
+if (FFTW_LIBRARIES)
+ list(GET FFTW_LIBRARIES 0 first_lib)
+ get_filename_component(first_lib_path "${first_lib}" PATH)
+ if (NOT FFTW_LIBRARY_DIRS)
+ set(FFTW_LIBRARY_DIRS "${first_lib_path}")
+ endif()
+ if (${first_lib_path} MATCHES "(/lib(32|64)?$)|(/lib/intel64$|/lib/ia32$)")
+ string(REGEX REPLACE "(/lib(32|64)?$)|(/lib/intel64$|/lib/ia32$)" "" not_cached_dir "${first_lib_path}")
+ set(FFTW_DIR_FOUND "${not_cached_dir}" CACHE PATH "Installation directory of FFTW library" FORCE)
+ else()
+ set(FFTW_DIR_FOUND "${first_lib_path}" CACHE PATH "Installation directory of FFTW library" FORCE)
+ endif()
+endif()
+mark_as_advanced(FFTW_DIR)
+mark_as_advanced(FFTW_DIR_FOUND)
+
+# check that FFTW has been found
+# -------------------------------
+include(FindPackageHandleStandardArgs)
+find_package_handle_standard_args(FFTW DEFAULT_MSG
+ FFTW_LIBRARIES
+ FFTW_WORKS)
diff --git a/cmake/morse/FindHeadersAndLibs.cmake b/cmake/morse/FindHeadersAndLibs.cmake
new file mode 100644
index 0000000000000000000000000000000000000000..64144bdbf8a35f966f1ac802e5765e6ad81abf7c
--- /dev/null
+++ b/cmake/morse/FindHeadersAndLibs.cmake
@@ -0,0 +1,94 @@
+###
+#
+# @copyright (c) 2009-2014 The University of Tennessee and The University
+# of Tennessee Research Foundation.
+# All rights reserved.
+# @copyright (c) 2012-2014 Inria. All rights reserved.
+# @copyright (c) 2012-2014 Bordeaux INP, CNRS (LaBRI UMR 5800), Inria, Univ. Bordeaux. All rights reserved.
+#
+###
+#
+# @file FindHeadersAndLibs.cmake
+#
+# @project MORSE
+# MORSE is a software package provided by:
+# Inria Bordeaux - Sud-Ouest,
+# Univ. of Tennessee,
+# King Abdullah Univesity of Science and Technology
+# Univ. of California Berkeley,
+# Univ. of Colorado Denver.
+#
+# @version 0.9.0
+# @author Cedric Castagnede
+# @author Emmanuel Agullo
+# @author Mathieu Faverge
+# @author Florent Pruvost
+# @date 13-07-2012
+#
+###
+
+# Some macros to print status when search for headers and libs
+include(PrintFindStatus)
+
+function(FindHeader _libname _header_to_find)
+
+ # save _libname upper and lower case
+ string(TOUPPER ${_libname} LIBNAME)
+ string(TOLOWER ${_libname} libname)
+
+ # Looking for include
+ # -------------------
+
+ # Add system include paths to search include
+ # ------------------------------------------
+ unset(_inc_env)
+ if(WIN32)
+ string(REPLACE ":" ";" _inc_env "$ENV{INCLUDE}")
+ else()
+ string(REPLACE ":" ";" _path_env "$ENV{INCLUDE}")
+ list(APPEND _inc_env "${_path_env}")
+ string(REPLACE ":" ";" _path_env "$ENV{C_INCLUDE_PATH}")
+ list(APPEND _inc_env "${_path_env}")
+ string(REPLACE ":" ";" _path_env "$ENV{CPATH}")
+ list(APPEND _inc_env "${_path_env}")
+ string(REPLACE ":" ";" _path_env "$ENV{INCLUDE_PATH}")
+ list(APPEND _inc_env "${_path_env}")
+ endif()
+ list(APPEND _inc_env "${CMAKE_C_IMPLICIT_INCLUDE_DIRECTORIES}")
+ list(REMOVE_DUPLICATES _inc_env)
+
+
+ # Try to find the _header_to_find in the given paths
+ # --------------------------------------------------
+ # call cmake macro to find the header path
+ if(${LIBNAME}_INCDIR)
+ set(${LIBNAME}_${_header_to_find}_DIRS "${LIBNAME}_${_header_to_find}_DIRS-NOTFOUND")
+ find_path(${LIBNAME}_${_header_to_find}_DIRS
+ NAMES ${_header_to_find}
+ HINTS ${${LIBNAME}_INCDIR})
+ elseif(${LIBNAME}_DIR)
+ set(${LIBNAME}_${_header_to_find}_DIRS "${LIBNAME}_${_header_to_find}_DIRS-NOTFOUND")
+ find_path(${LIBNAME}_${_header_to_find}_DIRS
+ NAMES ${_header_to_find}
+ HINTS ${${LIBNAME}_DIR}
+ PATH_SUFFIXES include)
+ else()
+ set(${LIBNAME}_${_header_to_find}_DIRS "${LIBNAME}_${_header_to_find}_DIRS-NOTFOUND")
+ find_path(${LIBNAME}_${_header_to_find}_DIRS
+ NAMES ${_header_to_find}
+ HINTS ${_inc_env})
+ endif()
+ mark_as_advanced(${LIBNAME}_${_header_to_find}_DIRS)
+
+ # Print status if not found
+ # -------------------------
+ if (NOT ${LIBNAME}_${_header_to_find}_DIRS)
+ Print_Find_Header_Status(${libname} ${_header_to_find})
+ endif ()
+
+endfunction(FindHeader)
+
+
+##
+## @end file FindHeadersAndLibs.cmake
+##
diff --git a/cmake/morse/FindInit.cmake b/cmake/morse/FindInit.cmake
new file mode 100644
index 0000000000000000000000000000000000000000..e59d41a077848029e04065d5f46bba57bcf0277d
--- /dev/null
+++ b/cmake/morse/FindInit.cmake
@@ -0,0 +1,45 @@
+###
+#
+# @copyright (c) 2018 Inria. All rights reserved.
+#
+###
+#
+# @file FindInit.cmake
+#
+# @project MORSE
+# MORSE is a software package provided by:
+# Inria Bordeaux - Sud-Ouest,
+# Univ. of Tennessee,
+# King Abdullah Univesity of Science and Technology
+# Univ. of California Berkeley,
+# Univ. of Colorado Denver.
+#
+# @version 1.0.0
+# @author Florent Pruvost
+# @date 24-04-2018
+#
+###
+
+
+# This include is required to check symbols of libs
+include(CheckFunctionExists)
+
+# This include is required to check defines in headers
+include(CheckIncludeFiles)
+
+# Factorize some piece of code
+include(FindCommon)
+
+# To find headers and libs
+include(FindHeadersAndLibs)
+
+# To transform relative path into absolute for a list of libraries
+include(LibrariesAbsolutePath)
+include(FindPkgconfigLibrariesAbsolutePath)
+
+# Some macros to print status when search for headers and libs
+include(PrintFindStatus)
+
+##
+## @end file FindInit.cmake
+##
diff --git a/cmake/morse/FindPkgconfigLibrariesAbsolutePath.cmake b/cmake/morse/FindPkgconfigLibrariesAbsolutePath.cmake
new file mode 100644
index 0000000000000000000000000000000000000000..51b08ce59853459f493a0892874f71678467c392
--- /dev/null
+++ b/cmake/morse/FindPkgconfigLibrariesAbsolutePath.cmake
@@ -0,0 +1,99 @@
+###
+#
+# @copyright (c) 2018 Inria. All rights reserved.
+#
+###
+#
+# @file FindPkgconfigLibrariesAbsolutePath.cmake
+#
+# @project MORSE
+# MORSE is a software package provided by:
+# Inria Bordeaux - Sud-Ouest,
+# Univ. of Tennessee,
+# King Abdullah Univesity of Science and Technology
+# Univ. of California Berkeley,
+# Univ. of Colorado Denver.
+#
+# @version 1.0.0
+# @author Florent Pruvost
+# @date 06-04-2018
+#
+###
+
+# Transform relative path into absolute path for libraries found with the
+# pkg_search_module cmake macro
+# _prefix: the name of the CMake variable used when pkg_search_module was called
+# e.g. for pkg_search_module(BLAS blas) _prefix would be BLAS
+macro(FIND_PKGCONFIG_LIBRARIES_ABSOLUTE_PATH _prefix)
+ list(APPEND _lib_env "$ENV{LIBRARY_PATH}")
+ if(WIN32)
+ string(REPLACE ":" ";" _lib_env2 "$ENV{LIB}")
+ elseif(APPLE)
+ string(REPLACE ":" ";" _lib_env2 "$ENV{DYLD_LIBRARY_PATH}")
+ else()
+ string(REPLACE ":" ";" _lib_env2 "$ENV{LD_LIBRARY_PATH}")
+ endif()
+ list(APPEND _lib_env "${_lib_env2}")
+ list(APPEND _lib_env "${CMAKE_C_IMPLICIT_LINK_DIRECTORIES}")
+ # non static case
+ set(${_prefix}_LIBRARIES_COPY "${${_prefix}_LIBRARIES}")
+ set(${_prefix}_LIBRARIES "")
+ foreach(_library ${${_prefix}_LIBRARIES_COPY})
+ if(EXISTS "${_library}")
+ list(APPEND ${_prefix}_LIBRARIES ${_library})
+ else()
+ get_filename_component(_ext "${_library}" EXT)
+ set(_lib_extensions ".so" ".a" ".dyld" ".dll")
+ list(FIND _lib_extensions "${_ext}" _index)
+ if (${_index} GREATER -1)
+ get_filename_component(_library "${_library}" NAME_WE)
+ endif()
+ find_library(_library_path NAMES ${_library}
+ HINTS ${${_prefix}_LIBDIR} ${${_prefix}_LIBRARY_DIRS} ${_lib_env})
+ if (_library_path)
+ list(APPEND ${_prefix}_LIBRARIES ${_library_path})
+ else()
+ message(FATAL_ERROR "Dependency of ${_prefix} '${_library}' NOT FOUND")
+ endif()
+ unset(_library_path CACHE)
+ endif()
+ endforeach()
+ set (${_prefix}_LIBRARIES "${${_prefix}_LIBRARIES}" CACHE INTERNAL "" FORCE)
+ ## static case
+ #set(${_prefix}_STATIC_LIBRARIES_COPY "${${_prefix}_STATIC_LIBRARIES}")
+ #set(${_prefix}_STATIC_LIBRARIES "")
+ #foreach(_library ${${_prefix}_STATIC_LIBRARIES_COPY})
+ # if(EXISTS "${_library}")
+ # list(APPEND ${_prefix}_STATIC_LIBRARIES ${_library})
+ # else()
+ # get_filename_component(_ext "${_library}" EXT)
+ # set(_lib_extensions ".so" ".a" ".dyld" ".dll")
+ # list(FIND _lib_extensions "${_ext}" _index)
+ # if (${_index} GREATER -1)
+ # get_filename_component(_library "${_library}" NAME_WE)
+ # endif()
+ # # try static first
+ # set (default_find_library_suffixes ${CMAKE_FIND_LIBRARY_SUFFIXES})
+ # set (CMAKE_FIND_LIBRARY_SUFFIXES ${CMAKE_STATIC_LIBRARY_SUFFIX})
+ # find_library(_library_path NAMES ${_library}
+ # HINTS ${${_prefix}_STATIC_LIBDIR} ${${_prefix}_STATIC_LIBRARY_DIRS} ${_lib_env})
+ # set (CMAKE_FIND_LIBRARY_SUFFIXES ${default_find_library_suffixes})
+ # # if not found try dynamic
+ # if (NOT _library_path)
+ # find_library(_library_path NAMES ${_library}
+ # HINTS ${${_prefix}_STATIC_LIBDIR} ${${_prefix}_STATIC_LIBRARY_DIRS} ${_lib_env})
+ # endif()
+ # if (_library_path)
+ # list(APPEND ${_prefix}_STATIC_LIBRARIES ${_library_path})
+ # else()
+ # message(FATAL_ERROR "Dependency of ${_prefix} '${_library}' NOT FOUND")
+ # endif()
+ # unset(_library_path CACHE)
+ # endif()
+ #endforeach()
+ #set (${_prefix}_STATIC_LIBRARIES "${${_prefix}_STATIC_LIBRARIES}" CACHE INTERNAL "" FORCE)
+endmacro()
+
+##
+## @end file FindPkgconfigLibrariesAbsolutePath.cmake
+##
diff --git a/cmake/morse/LICENCE.txt b/cmake/morse/LICENCE.txt
new file mode 100644
index 0000000000000000000000000000000000000000..b95821f36afa3579a5f1de4fe840aec43b7a4b96
--- /dev/null
+++ b/cmake/morse/LICENCE.txt
@@ -0,0 +1,42 @@
+###
+#
+# @copyright (c) 2009-2014 The University of Tennessee and The University
+# of Tennessee Research Foundation.
+# All rights reserved.
+# @copyright (c) 2012-2016 Bordeaux INP, CNRS (LaBRI UMR 5800), Inria,
+# Univ. Bordeaux. All rights reserved.
+# @copyright (c) 2016 KAUST. All rights reserved.
+#
+###
+#
+# This software is a computer program whose purpose is to process
+# Matrices Over Runtime Systems @ Exascale (MORSE). More information
+# can be found on the following website: http://www.inria.fr/en/teams/morse.
+#
+# This software is governed by the CeCILL-C license under French law and
+# abiding by the rules of distribution of free software. You can use,
+# modify and/ or redistribute the software under the terms of the CeCILL-C
+# license as circulated by CEA, CNRS and INRIA at the following URL
+# "http://www.cecill.info".
+#
+# As a counterpart to the access to the source code and rights to copy,
+# modify and redistribute granted by the license, users are provided only
+# with a limited warranty and the software's author, the holder of the
+# economic rights, and the successive licensors have only limited
+# liability.
+#
+# In this respect, the user's attention is drawn to the risks associated
+# with loading, using, modifying and/or developing or reproducing the
+# software by the user in light of its specific status of free software,
+# that may mean that it is complicated to manipulate, and that also
+# therefore means that it is reserved for developers and experienced
+# professionals having in-depth computer knowledge. Users are therefore
+# encouraged to load and test the software's suitability as regards their
+# requirements in conditions enabling the security of their systems and/or
+# data to be ensured and, more generally, to use and operate it in the
+# same conditions as regards security.
+#
+# The fact that you are presently reading this means that you have had
+# knowledge of the CeCILL-C license and that you accept its terms.
+#
+###
diff --git a/cmake/morse/LibrariesAbsolutePath.cmake b/cmake/morse/LibrariesAbsolutePath.cmake
new file mode 100644
index 0000000000000000000000000000000000000000..7aaab504d7348090e36c502755020b0b1439f123
--- /dev/null
+++ b/cmake/morse/LibrariesAbsolutePath.cmake
@@ -0,0 +1,70 @@
+###
+#
+# @copyright (c) 2018 Inria. All rights reserved.
+#
+###
+#
+# @file LibrariesAbsolutePath.cmake
+#
+# @project MORSE
+# MORSE is a software package provided by:
+# Inria Bordeaux - Sud-Ouest,
+# Univ. of Tennessee,
+# King Abdullah Univesity of Science and Technology
+# Univ. of California Berkeley,
+# Univ. of Colorado Denver.
+#
+# @version 1.0.0
+# @author Florent Pruvost
+# @date 13-04-2018
+#
+###
+
+# Transform relative path into absolute path for libraries
+# lib_list (input/output): the name of the CMake variable containing libraries, e.g. BLAS_LIBRARIES
+# hints_paths (input): additional paths to add when looking for libraries
+macro(LIBRARIES_ABSOLUTE_PATH lib_list hints_paths)
+ # collect environment paths to dig
+ list(APPEND _lib_env "$ENV{LIBRARY_PATH}")
+ if(WIN32)
+ string(REPLACE ":" ";" _lib_env2 "$ENV{LIB}")
+ elseif(APPLE)
+ string(REPLACE ":" ";" _lib_env2 "$ENV{DYLD_LIBRARY_PATH}")
+ else()
+ string(REPLACE ":" ";" _lib_env2 "$ENV{LD_LIBRARY_PATH}")
+ endif()
+ list(APPEND _lib_env "${_lib_env2}")
+ list(APPEND _lib_env "${CMAKE_C_IMPLICIT_LINK_DIRECTORIES}")
+ # copy the lib list
+ set (${lib_list}_COPY "${${lib_list}}")
+ # reset the lib list to populate
+ set(${lib_list} "")
+ foreach(_library ${${lib_list}_COPY})
+ if(EXISTS "${_library}")
+ # if already an absolute path, nothing special to do
+ list(APPEND ${lib_list} ${_library})
+ else()
+ # replace pattern -lfoo -> foo
+ string(REGEX REPLACE "^-l" "" _library "${_library}")
+ # remove extensions if exist
+ get_filename_component(_ext "${_library}" EXT)
+ set(_lib_extensions ".so" ".a" ".dyld" ".dll")
+ list(FIND _lib_extensions "${_ext}" _index)
+ if (${_index} GREATER -1)
+ get_filename_component(_library "${_library}" NAME_WE)
+ endif()
+ # try to find the lib
+ find_library(_library_path NAMES ${_library} HINTS ${hints_paths} ${_lib_env})
+ if (_library_path)
+ list(APPEND ${lib_list} ${_library_path})
+ else()
+ message(FATAL_ERROR "Dependency of ${lib_list} '${_library}' NOT FOUND")
+ endif()
+ unset(_library_path CACHE)
+ endif()
+ endforeach()
+endmacro()
+
+##
+## @end file LibrariesAbsolutePath.cmake
+##
diff --git a/cmake/morse/MorseInit.cmake b/cmake/morse/MorseInit.cmake
new file mode 100644
index 0000000000000000000000000000000000000000..fc51170401cc17045854de4ee51f477eff1b66c1
--- /dev/null
+++ b/cmake/morse/MorseInit.cmake
@@ -0,0 +1,67 @@
+###
+#
+# @copyright (c) 2009-2014 The University of Tennessee and The University
+# of Tennessee Research Foundation.
+# All rights reserved.
+# @copyright (c) 2012-2018 Inria. All rights reserved.
+# @copyright (c) 2012-2014 Bordeaux INP, CNRS (LaBRI UMR 5800), Inria, Univ. Bordeaux. All rights reserved.
+#
+###
+#
+# @file MorseInit.cmake
+#
+# @project MORSE
+# MORSE is a software package provided by:
+# Inria Bordeaux - Sud-Ouest,
+# Univ. of Tennessee,
+# King Abdullah Univesity of Science and Technology
+# Univ. of California Berkeley,
+# Univ. of Colorado Denver.
+#
+# @version 1.0.0
+# @author Cedric Castagnede
+# @author Emmanuel Agullo
+# @author Mathieu Faverge
+# @author Florent Pruvost
+# @date 13-07-2012
+#
+###
+
+# Path to Morse modules
+get_filename_component(MORSE_CMAKE_MODULE_PATH ${CMAKE_CURRENT_LIST_FILE} DIRECTORY CACHE)
+
+# Global Morse options
+option(MORSE_ENABLE_WARNING "Enable warning messages" OFF)
+option(MORSE_ENABLE_COVERAGE "Enable flags for coverage test" OFF)
+option(MORSE_ENABLE_COLOR_MESSAGE "Enable colors in messages" OFF)
+#option(MORSE_VERBOSE_FIND_PACKAGE "Add additional messages concerning packages not found" OFF)
+#message(STATUS "MORSE_VERBOSE_FIND_PACKAGE is set to OFF, turn it ON to get"
+# " information about packages not found")
+
+
+# This include is required to check symbols of libs in the main CMakeLists.txt
+include(CheckFunctionExists)
+
+# This include is required to check defines in headers
+include(CheckIncludeFiles)
+
+if (MORSE_ENABLE_COLOR_MESSAGE)
+ # colorize messages
+ include(ColorizeMessage)
+endif()
+
+# Define some auxilary flags
+include(AuxilaryFlags)
+
+# Define some variables to et info about ressources
+include(Ressources)
+
+# Add the path where we handle our FindFOO.cmake to seek for liraries
+list(APPEND CMAKE_MODULE_PATH ${MORSE_CMAKE_MODULE_PATH}/find)
+
+# To load some macros used in Finds (could be useful for other projects)
+include(FindInit)
+
+##
+## @end file MorseInit.cmake
+##
diff --git a/cmake/morse/PrintFindStatus.cmake b/cmake/morse/PrintFindStatus.cmake
new file mode 100644
index 0000000000000000000000000000000000000000..1fdd403b7de11a8b946b178c5aada2da5e6fe33e
--- /dev/null
+++ b/cmake/morse/PrintFindStatus.cmake
@@ -0,0 +1,207 @@
+###
+#
+# @copyright (c) 2009-2014 The University of Tennessee and The University
+# of Tennessee Research Foundation.
+# All rights reserved.
+# @copyright (c) 2012-2014 Inria. All rights reserved.
+# @copyright (c) 2012-2014 Bordeaux INP, CNRS (LaBRI UMR 5800), Inria, Univ. Bordeaux. All rights reserved.
+#
+###
+#
+# - Some macros to print status when search for headers and libs
+# Main parameters of macros
+# _libname: name of the lib you seek, foo for example
+# _header_to_find: name of the header you seek, foo.h for example
+# _lib_to_find: name of the library you seek, libfoo for example
+# _pc_to_find: name of the pkg-config file zyou seek, foo.pc for example
+
+
+#=============================================================================
+# Copyright 2012-2013 Inria
+# Copyright 2012-2013 Emmanuel Agullo
+# Copyright 2012-2013 Mathieu Faverge
+# Copyright 2012 Cedric Castagnede
+# Copyright 2013 Florent Pruvost
+#
+# Distributed under the OSI-approved BSD License (the "License");
+# see accompanying file MORSE-Copyright.txt for details.
+#
+# This software is distributed WITHOUT ANY WARRANTY; without even the
+# implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
+# See the License for more information.
+
+#=============================================================================
+# (To distribute this file outside of Morse, substitute the full
+# License text for the above reference.)
+
+
+# Set some colors
+#if(NOT WIN32)
+# string(ASCII 27 Esc)
+# set(ColourReset "${Esc}[m")
+# set(ColourBold "${Esc}[1m")
+# set(Red "${Esc}[31m")
+# set(Green "${Esc}[32m")
+# set(Yellow "${Esc}[33m")
+# set(Blue "${Esc}[34m")
+# set(Magenta "${Esc}[35m")
+# set(Cyan "${Esc}[36m")
+# set(White "${Esc}[37m")
+# set(BoldRed "${Esc}[1;31m")
+# set(BoldGreen "${Esc}[1;32m")
+# set(BoldYellow "${Esc}[1;33m")
+# set(BoldBlue "${Esc}[1;34m")
+# set(BoldMagenta "${Esc}[1;35m")
+# set(BoldCyan "${Esc}[1;36m")
+# set(BoldWhite "${Esc}[1;37m")
+#endif()
+
+
+# This macro informs why the _header_to_find file has not been found
+macro(Print_Find_Header_Status _libname _header_to_find)
+
+ # save _libname upper and lower case
+ string(TOUPPER ${_libname} LIBNAME)
+ string(TOLOWER ${_libname} libname)
+
+ # print status
+ #message(" ")
+ if(${LIBNAME}_INCDIR)
+ message("${Blue}${LIBNAME}_INCDIR is defined but ${_header_to_find}"
+ "has not been found in ${${LIBNAME}_INCDIR}${ColourReset}")
+ else()
+ if(${LIBNAME}_DIR)
+ message("${Blue}${LIBNAME}_DIR is defined but"
+ "${_header_to_find} has not been found in"
+ "${${LIBNAME}_DIR}/include${ColourReset}")
+ else()
+ message("${Blue}${_header_to_find} not found."
+ "Nor ${LIBNAME}_DIR neither ${LIBNAME}_INCDIR"
+ "are defined so that we looked for ${_header_to_find} in"
+ "system paths (INCLUDE, CPATH, C_INCLUDE_PATH,"
+ "INCLUDE_PATH, CMAKE_C_IMPLICIT_INCLUDE_DIRECTORIES)${ColourReset}")
+ if(_inc_env)
+ message("${Blue}${_header_to_find} has not been found in"
+ "${_inc_env}${ColourReset}")
+ endif()
+ endif()
+ endif()
+ message("${BoldBlue}Please indicate where to find ${_header_to_find}. You have three options:\n"
+ "- Option 1: Provide the root directory of the library with cmake option: -D${LIBNAME}_DIR=your/path/to/${libname}/\n"
+ "- Option 2: Provide the directory where to find the headers with cmake option: -D${LIBNAME}_INCDIR=your/path/to/${libname}/include/\n"
+ "- Option 3: Update your environment variable (INCLUDE or CPATH)\n"
+ "- Option 4: If your library provides a PkgConfig file, make sure pkg-config finds your library${ColourReset}")
+ #message(" ")
+
+endmacro()
+
+# This macro informs why the _lib_to_find file has not been found
+macro(Print_Find_Library_Status _libname _lib_to_find)
+
+ # save _libname upper/lower case
+ string(TOUPPER ${_libname} LIBNAME)
+ string(TOLOWER ${_libname} libname)
+
+ # print status
+ #message(" ")
+ if(${LIBNAME}_LIBDIR)
+ message("${Yellow}${LIBNAME}_LIBDIR is defined but ${_lib_to_find}"
+ "has not been found in ${${LIBNAME}_LIBDIR}${ColourReset}")
+ else()
+ if(${LIBNAME}_DIR)
+ message("${Yellow}${LIBNAME}_DIR is defined but ${_lib_to_find}"
+ "has not been found in ${${LIBNAME}_DIR}/lib(or /lib32 or"
+ "/lib64)${ColourReset}")
+ else()
+ message("${Yellow}${_lib_to_find} not found."
+ "Nor ${LIBNAME}_DIR neither ${LIBNAME}_LIBDIR"
+ "are defined so that we looked for ${_lib_to_find} in"
+ "system paths (Linux: LD_LIBRARY_PATH, Windows: LIB,"
+ "Mac: DYLD_LIBRARY_PATH,"
+ "CMAKE_C_IMPLICIT_LINK_DIRECTORIES)${ColourReset}")
+ if(_lib_env)
+ message("${Yellow}${_lib_to_find} has not been found in"
+ "${_lib_env}${ColourReset}")
+ endif()
+ endif()
+ endif()
+ message("${BoldYellow}Please indicate where to find ${_lib_to_find}. You have three options:\n"
+ "- Option 1: Provide the root directory of the library with cmake option: -D${LIBNAME}_DIR=your/path/to/${libname}/\n"
+ "- Option 2: Provide the directory where to find the library with cmake option: -D${LIBNAME}_LIBDIR=your/path/to/${libname}/lib/\n"
+ "- Option 3: Update your environment variable (Linux: LD_LIBRARY_PATH, Windows: LIB, Mac: DYLD_LIBRARY_PATH)\n"
+ "- Option 4: If your library provides a PkgConfig file, make sure pkg-config finds your library${ColourReset}")
+
+endmacro()
+
+# This macro informs why the _lib_to_find file has not been found
+macro(Print_Find_Library_Blas_Status _libname _lib_to_find)
+
+ # save _libname upper/lower case
+ string(TOUPPER ${_libname} LIBNAME)
+ string(TOLOWER ${_libname} libname)
+
+ # print status
+ #message(" ")
+ if(${LIBNAME}_LIBDIR)
+ message("${Yellow}${LIBNAME}_LIBDIR is defined but ${_lib_to_find}"
+ "has not been found in ${ARGN}${ColourReset}")
+ else()
+ if(${LIBNAME}_DIR)
+ message("${Yellow}${LIBNAME}_DIR is defined but ${_lib_to_find}"
+ "has not been found in ${ARGN}${ColourReset}")
+ else()
+ message("${Yellow}${_lib_to_find} not found."
+ "Nor ${LIBNAME}_DIR neither ${LIBNAME}_LIBDIR"
+ "are defined so that we look for ${_lib_to_find} in"
+ "system paths (Linux: LD_LIBRARY_PATH, Windows: LIB,"
+ "Mac: DYLD_LIBRARY_PATH,"
+ "CMAKE_C_IMPLICIT_LINK_DIRECTORIES)${ColourReset}")
+ if(_lib_env)
+ message("${Yellow}${_lib_to_find} has not been found in"
+ "${_lib_env}${ColourReset}")
+ endif()
+ endif()
+ endif()
+ message("${BoldYellow}Please indicate where to find ${_lib_to_find}. You have three options:\n"
+ "- Option 1: Provide the root directory of the library with cmake option: -D${LIBNAME}_DIR=your/path/to/${libname}/\n"
+ "- Option 2: Provide the directory where to find the library with cmake option: -D${LIBNAME}_LIBDIR=your/path/to/${libname}/lib/\n"
+ "- Option 3: Update your environment variable (Linux: LD_LIBRARY_PATH, Windows: LIB, Mac: DYLD_LIBRARY_PATH)\n"
+ "- Option 4: If your library provides a PkgConfig file, make sure pkg-config finds your library${ColourReset}")
+
+endmacro()
+
+# This macro informs why the _lib_to_find file has not been found
+macro(Print_Find_Library_Blas_CheckFunc_Status _name)
+
+ # save _libname upper/lower case
+ string(TOUPPER ${_name} FUNCNAME)
+ string(TOLOWER ${_name} funcname)
+
+ # print status
+ #message(" ")
+ message("${Red}Libs have been found but check of symbol ${_name} failed "
+ "with following libraries ${ARGN}${ColourReset}")
+ message("${BoldRed}Please open your error file CMakeFiles/CMakeError.log"
+ "to figure out why it fails${ColourReset}")
+ #message(" ")
+
+endmacro()
+
+# This macro informs that _pc_to_find file has not been found in the list
+# path you give as last argument (read in ${ARGN})
+# ex: Print_Find_Pkgconfig_Status(foo foo.pc ${PATHLIST}
+macro(Print_Find_Pkgconfig_Status _libname _pc_to_find)
+
+ # save _libname lower case
+ string(TOLOWER ${_libname} libname)
+
+ # print status
+ #message(" ")
+ message("${Magenta}${_pc_to_find} has not been found in"
+ "${ARGN}${ColourReset}")
+ message("${BoldMagenta}If you really want to use the pkg-config file of"
+ "${libname}, please update your PKG_CONFIG_PATH with the path"
+ "where ${_pc_to_find} states${ColourReset}")
+ #message(" ")
+
+endmacro()
diff --git a/configure_python.sh b/configure_python.sh
new file mode 100644
index 0000000000000000000000000000000000000000..bc38e9eef17722eec5ea71fd86838ccd7ced2404
--- /dev/null
+++ b/configure_python.sh
@@ -0,0 +1,10 @@
+#! /usr/bin/env bash
+
+set -e
+
+cp host_info.py TurTLE/
+sed "s/@TURTLE_VERSION_LONG@/3.15/" setup.py.in > setup.py
+sed "s/@TURTLE_VERSION_LONG@/3.15/" TurTLE/__init__.py.in > TurTLE/__init__.py.tmp
+sed "s/@CMAKE_INSTALL_DIR@/UNAVAILABLE_FOR_PURE_PYTHON_INSTALLS/" TurTLE/__init__.py.tmp > TurTLE/__init__.py
+rm TurTLE/__init__.py.tmp
+
diff --git a/bfps/cpp/base.hpp b/cpp/base.hpp
similarity index 55%
rename from bfps/cpp/base.hpp
rename to cpp/base.hpp
index 61029c2641b37aab5e47ebc4bb511f38b893eeb1..55562ebcefd600fbeb6087f60552bede030ec64f 100644
--- a/bfps/cpp/base.hpp
+++ b/cpp/base.hpp
@@ -3,39 +3,42 @@
* Copyright 2015 Max Planck Institute *
* for Dynamics and Self-Organization *
* *
-* This file is part of bfps. *
+* This file is part of TurTLE. *
* *
-* bfps is free software: you can redistribute it and/or modify *
+* TurTLE is free software: you can redistribute it and/or modify *
* it under the terms of the GNU General Public License as published *
* by the Free Software Foundation, either version 3 of the License, *
* or (at your option) any later version. *
* *
-* bfps is distributed in the hope that it will be useful, *
+* TurTLE is distributed in the hope that it will be useful, *
* but WITHOUT ANY WARRANTY; without even the implied warranty of *
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the *
* GNU General Public License for more details. *
* *
* You should have received a copy of the GNU General Public License *
-* along with bfps. If not, see <http://www.gnu.org/licenses/> *
+* along with TurTLE. If not, see <http://www.gnu.org/licenses/> *
* *
* Contact: Cristian.Lalescu@ds.mpg.de *
* *
**********************************************************************/
+#ifndef BASE_HPP
+
+#define BASE_HPP
+
#include <cassert>
#include <mpi.h>
#include <stdarg.h>
#include <iostream>
#include <typeinfo>
-#include "hdf5_tools.hpp"
-#ifndef BASE
-#define BASE
+/////////////////////////////////////////////////////////////
+/////////////////////////////////////////////////////////////
-static const int message_buffer_length = 2048;
+static const int message_buffer_length = 32768;
extern int myrank, nprocs;
inline int MOD(int a, int n)
@@ -43,16 +46,59 @@ inline int MOD(int a, int n)
return ((a%n) + n) % n;
}
+template <typename T> int sgn(T val)
+{
+ return (T(0) < val) - (val < T(0));
+}
+
+/////////////////////////////////////////////////////////////
+/////////////////////////////////////////////////////////////
+
+namespace turtle_mpi_pcontrol{
+ const int FFTW = 5;
+ const int FIELD = 6;
+ const int PARTICLES = 7;
+ const int HDF5 = 8;
+}
+
+#ifdef USE_TIMINGOUTPUT
+inline int start_mpi_profiling_zone(const int zone_name)
+{
+ assert((zone_name == turtle_mpi_pcontrol::FFTW) ||
+ (zone_name == turtle_mpi_pcontrol::FIELD) ||
+ (zone_name == turtle_mpi_pcontrol::PARTICLES) ||
+ (zone_name == turtle_mpi_pcontrol::HDF5));
+ return MPI_Pcontrol(zone_name);
+}
+inline int finish_mpi_profiling_zone(const int zone_name)
+{
+ assert((zone_name == turtle_mpi_pcontrol::FFTW) ||
+ (zone_name == turtle_mpi_pcontrol::FIELD) ||
+ (zone_name == turtle_mpi_pcontrol::PARTICLES) ||
+ (zone_name == turtle_mpi_pcontrol::HDF5));
+ return MPI_Pcontrol(-zone_name);
+}
+#else
+inline int start_mpi_profiling_zone(const int zone_name)
+{
+ return EXIT_SUCCESS;
+}
+inline int finish_mpi_profiling_zone(const int zone_name)
+{
+ return EXIT_SUCCESS;
+}
+#endif//USE_TIMINGOUTPUT
+
/////////////////////////////////////////////////////////////
/////////////////////////////////////////////////////////////
#ifdef OMPI_MPI_H
-#define BFPS_MPICXX_DOUBLE_COMPLEX MPI_DOUBLE_COMPLEX
+#define TURTLE_MPICXX_DOUBLE_COMPLEX MPI_DOUBLE_COMPLEX
#else
-#define BFPS_MPICXX_DOUBLE_COMPLEX MPI_C_DOUBLE_COMPLEX
+#define TURTLE_MPICXX_DOUBLE_COMPLEX MPI_C_DOUBLE_COMPLEX
#endif//OMPI_MPI_H
@@ -81,10 +127,37 @@ public:
}
static constexpr MPI_Datatype complex(){
- return BFPS_MPICXX_DOUBLE_COMPLEX;
+ return TURTLE_MPICXX_DOUBLE_COMPLEX;
}
};
+template <>
+class mpi_real_type<int>
+{
+ public:
+ static constexpr MPI_Datatype real(){
+ return MPI_INT;
+ }
+};
+
+template <>
+class mpi_real_type<long int>
+{
+ public:
+ static constexpr MPI_Datatype real(){
+ return MPI_LONG;
+ }
+};
+
+template <>
+class mpi_real_type<long long int>
+{
+ public:
+ static constexpr MPI_Datatype real(){
+ return MPI_LONG_LONG;
+ }
+};
+
/////////////////////////////////////////////////////////////
/////////////////////////////////////////////////////////////
@@ -98,11 +171,11 @@ inline void DEBUG_MSG(const char * format, ...)
va_start(argptr, format);
sprintf(
debug_message_buffer,
- "cpu%.4d ",
+ "MPIrank%.4d ",
myrank);
vsnprintf(
- debug_message_buffer + 8,
- message_buffer_length - 8,
+ debug_message_buffer + 12,
+ message_buffer_length - 12,
format,
argptr);
va_end(argptr);
@@ -115,11 +188,11 @@ inline void DEBUG_MSG_WAIT(MPI_Comm communicator, const char * format, ...)
va_start(argptr, format);
sprintf(
debug_message_buffer,
- "cpu%.4d ",
+ "MPIrank%.4d ",
myrank);
vsnprintf(
- debug_message_buffer + 8,
- message_buffer_length - 8,
+ debug_message_buffer + 12,
+ message_buffer_length - 12,
format,
argptr);
va_end(argptr);
@@ -136,5 +209,5 @@ inline void DEBUG_MSG_WAIT(MPI_Comm communicator, const char * format, ...)
#define variable_used_only_in_assert(x) ((void)(x))
-#endif//BASE
+#endif//BASE_HPP
diff --git a/bfps/cpp/particles/env_utils.hpp b/cpp/env_utils.hpp
similarity index 58%
rename from bfps/cpp/particles/env_utils.hpp
rename to cpp/env_utils.hpp
index cd6fb3026ac19397fb525235f3d4f87e2cc2bb94..f030c7c8a4fec908fff0b2f7514ead49d151c416 100644
--- a/bfps/cpp/particles/env_utils.hpp
+++ b/cpp/env_utils.hpp
@@ -1,3 +1,28 @@
+/******************************************************************************
+* *
+* Copyright 2019 Max Planck Institute for Dynamics and Self-Organization *
+* *
+* This file is part of TurTLE. *
+* *
+* TurTLE is free software: you can redistribute it and/or modify *
+* it under the terms of the GNU General Public License as published *
+* by the Free Software Foundation, either version 3 of the License, *
+* or (at your option) any later version. *
+* *
+* TurTLE is distributed in the hope that it will be useful, *
+* but WITHOUT ANY WARRANTY; without even the implied warranty of *
+* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the *
+* GNU General Public License for more details. *
+* *
+* You should have received a copy of the GNU General Public License *
+* along with TurTLE. If not, see <http://www.gnu.org/licenses/> *
+* *
+* Contact: Cristian.Lalescu@ds.mpg.de *
+* *
+******************************************************************************/
+
+
+
#ifndef ENV_UTILS_HPP
#define ENV_UTILS_HPP
@@ -9,7 +34,9 @@
#include <cstring>
#include <array>
-
+/** \class env_utils
+ * \brief utilities
+ */
class env_utils {
template <class VariableType>
static const VariableType StrToOther(const char* const str, const VariableType& defaultValue = VariableType()){
diff --git a/cpp/fftw_interface.hpp b/cpp/fftw_interface.hpp
new file mode 100644
index 0000000000000000000000000000000000000000..8528aba39dbe56855181ae26be9fc1b984171aa5
--- /dev/null
+++ b/cpp/fftw_interface.hpp
@@ -0,0 +1,799 @@
+/**********************************************************************
+* *
+* Copyright 2015 Max Planck Institute *
+* for Dynamics and Self-Organization *
+* *
+* This file is part of TurTLE. *
+* *
+* TurTLE is free software: you can redistribute it and/or modify *
+* it under the terms of the GNU General Public License as published *
+* by the Free Software Foundation, either version 3 of the License, *
+* or (at your option) any later version. *
+* *
+* TurTLE is distributed in the hope that it will be useful, *
+* but WITHOUT ANY WARRANTY; without even the implied warranty of *
+* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the *
+* GNU General Public License for more details. *
+* *
+* You should have received a copy of the GNU General Public License *
+* along with TurTLE. If not, see <http://www.gnu.org/licenses/> *
+* *
+* Contact: Cristian.Lalescu@ds.mpg.de *
+* *
+**********************************************************************/
+
+#ifndef FFTW_INTERFACE_HPP
+#define FFTW_INTERFACE_HPP
+
+#include <fftw3-mpi.h>
+#include <map>
+#include <string>
+
+#ifdef USE_FFTWESTIMATE
+#define DEFAULT_FFTW_FLAG FFTW_ESTIMATE
+#warning You are using FFTW estimate
+#else
+#define DEFAULT_FFTW_FLAG FFTW_PATIENT
+#endif
+
+// To have multiple calls to c2r/r2c
+// you must define SPLIT_FFTW_MANY
+// by calling setup.py --split-fftw-many
+#ifdef SPLIT_FFTW_MANY
+#include <vector>
+#include <memory>
+#include <algorithm>
+#include <cassert>
+#include <cstring>
+#include <type_traits>
+
+// To mix unique ptr with allocation from fftw
+struct fftw_free_deleter{
+ template <typename T>
+ void operator()(T *p) const {
+ fftwf_free(const_cast<typename std::remove_const<T>::type*>(p));
+ }
+};
+
+#endif
+
+template <class realtype>
+class fftw_interface;
+
+template <>
+class fftw_interface<float>
+{
+public:
+ using real = float;
+ using complex = fftwf_complex;
+ using plan = fftwf_plan;
+ using iodim = fftwf_iodim;
+#ifdef SPLIT_FFTW_MANY
+ struct many_plan_container{
+ int rnk;
+ std::vector<ptrdiff_t> n;
+ int howmany;
+ ptrdiff_t iblock;
+ ptrdiff_t oblock;
+ std::unique_ptr<real[], fftw_free_deleter> buffer;
+ plan plan_to_use;
+
+ ptrdiff_t local_n0, local_0_start;
+ ptrdiff_t local_n1, local_1_start;
+
+ bool is_r2c;
+ void* in;
+ void* out;
+
+ ptrdiff_t nb_sections_real;
+ ptrdiff_t size_real_section;
+ ptrdiff_t nb_sections_complex;
+ ptrdiff_t size_complex_section;
+
+ ptrdiff_t sizeBuffer;
+ };
+
+ using many_plan = many_plan_container;
+
+ static char* sprint(const many_plan mp)
+ {
+ return fftwf_sprint_plan(mp.plan_to_use);
+ }
+#else
+ using many_plan = fftwf_plan;
+
+ static char* sprint(many_plan mp)
+ {
+ return fftwf_sprint_plan(mp);
+ }
+#endif
+
+ static complex* alloc_complex(const size_t in_size){
+ return fftwf_alloc_complex(in_size);
+ }
+
+ static real* alloc_real(const size_t in_size){
+ return fftwf_alloc_real(in_size);
+ }
+
+ static void free(void* ptr){
+ fftwf_free(ptr);
+ }
+
+ static void execute(plan in_plan){
+ fftwf_execute(in_plan);
+ }
+
+ static void destroy_plan(plan in_plan){
+ fftwf_destroy_plan(in_plan);
+ }
+
+ template <class ... Params>
+ static ptrdiff_t mpi_local_size_many(Params ... params){
+ return fftwf_mpi_local_size_many(params...);
+ }
+
+ template <class ... Params>
+ static plan mpi_plan_transpose(Params ... params){
+ return fftwf_mpi_plan_transpose(params...);
+ }
+
+ template <class ... Params>
+ static plan mpi_plan_many_transpose(Params ... params){
+ return fftwf_mpi_plan_many_transpose(params...);
+ }
+
+ template <class ... Params>
+ static plan plan_guru_r2r(Params ... params){
+ return fftwf_plan_guru_r2r(params...);
+ }
+
+ template <class ... Params>
+ static plan plan_guru_dft(Params ... params){
+ return fftwf_plan_guru_dft(params...);
+ }
+
+ template <class ... Params>
+ static ptrdiff_t mpi_local_size_transposed(Params ... params){
+ return fftwf_mpi_local_size_transposed(params...);
+ }
+
+#ifdef SPLIT_FFTW_MANY
+ static ptrdiff_t mpi_local_size_many_transposed(int rnk, const ptrdiff_t *n, ptrdiff_t howmany,
+ ptrdiff_t block0, ptrdiff_t block1, MPI_Comm comm,
+ ptrdiff_t *local_n0, ptrdiff_t *local_0_start,
+ ptrdiff_t *local_n1, ptrdiff_t *local_1_start){
+ assert(block0 == FFTW_MPI_DEFAULT_BLOCK);
+ assert(block1 == FFTW_MPI_DEFAULT_BLOCK);
+ return howmany*mpi_local_size_transposed(rnk, n, comm,
+ local_n0, local_0_start,
+ local_n1, local_1_start);
+ }
+
+ static many_plan mpi_plan_many_dft_c2r(int rnk, const ptrdiff_t *n, ptrdiff_t howmany,
+ ptrdiff_t iblock, ptrdiff_t oblock,
+ complex *in, real *out,
+ MPI_Comm comm, unsigned flags){
+ assert(iblock == FFTW_MPI_DEFAULT_BLOCK);
+ assert(oblock == FFTW_MPI_DEFAULT_BLOCK);
+
+ many_plan c2r_plan;
+ c2r_plan.rnk = rnk;
+ c2r_plan.n.insert(c2r_plan.n.end(), n, n+rnk);
+ c2r_plan.howmany = howmany;
+ c2r_plan.iblock = iblock;
+ c2r_plan.oblock = oblock;
+ c2r_plan.is_r2c = false;
+ c2r_plan.in = in;
+ c2r_plan.out = out;
+ c2r_plan.sizeBuffer = 0;
+
+ // If 1 then use default without copy
+ if(howmany == 1){
+ c2r_plan.plan_to_use = mpi_plan_dft_c2r(rnk, n,
+ (complex*)in,
+ out,
+ comm, flags);
+ return c2r_plan;
+ }
+
+ // We need to find out the size of the buffer to allocate
+ mpi_local_size_transposed(
+ rnk, n, comm,
+ &c2r_plan.local_n0, &c2r_plan.local_0_start,
+ &c2r_plan.local_n1, &c2r_plan.local_1_start);
+
+ ptrdiff_t sizeBuffer = c2r_plan.local_n0;
+ for(int idxrnk = 1 ; idxrnk < rnk-1 ; ++idxrnk){
+ sizeBuffer *= n[idxrnk];
+ }
+ sizeBuffer *= n[rnk-1]+2;
+
+ c2r_plan.buffer.reset(alloc_real(sizeBuffer));
+ memset(c2r_plan.buffer.get(), 0, sizeof(real)*sizeBuffer);
+ c2r_plan.sizeBuffer = sizeBuffer;
+ // Init the plan
+ c2r_plan.plan_to_use = mpi_plan_dft_c2r(rnk, n,
+ (complex*)c2r_plan.buffer.get(),
+ c2r_plan.buffer.get(),
+ comm, flags);
+
+ c2r_plan.nb_sections_real = c2r_plan.local_n0;
+ for(int idxrnk = 1 ; idxrnk < rnk-1 ; ++idxrnk){
+ c2r_plan.nb_sections_real *= n[idxrnk];
+ c2r_plan.nb_sections_complex *= n[idxrnk];
+ }
+ c2r_plan.size_real_section = (n[rnk-1] + 2);
+
+ c2r_plan.nb_sections_complex = c2r_plan.local_n1;
+ for(int idxrnk = 1 ; idxrnk < rnk-1 ; ++idxrnk){
+ if(idxrnk == 1){
+ c2r_plan.nb_sections_complex *= n[0];
+ }
+ else{
+ c2r_plan.nb_sections_complex *= n[idxrnk];
+ }
+ }
+ c2r_plan.size_complex_section = (n[rnk-1]/2 + 1);
+
+ return c2r_plan;
+ }
+
+ static many_plan mpi_plan_many_dft_r2c(int rnk, const ptrdiff_t *n, ptrdiff_t howmany,
+ ptrdiff_t iblock, ptrdiff_t oblock,
+ real *in, complex *out,
+ MPI_Comm comm, unsigned flags){
+ assert(iblock == FFTW_MPI_DEFAULT_BLOCK);
+ assert(oblock == FFTW_MPI_DEFAULT_BLOCK);
+
+ many_plan r2c_plan;
+ r2c_plan.rnk = rnk;
+ r2c_plan.n.insert(r2c_plan.n.end(), n, n+rnk);
+ r2c_plan.howmany = howmany;
+ r2c_plan.iblock = iblock;
+ r2c_plan.oblock = oblock;
+ r2c_plan.is_r2c = true;
+ r2c_plan.in = in;
+ r2c_plan.out = out;
+ r2c_plan.sizeBuffer = 0;
+
+ // If 1 then use default without copy
+ if(howmany == 1){
+ r2c_plan.plan_to_use = mpi_plan_dft_r2c(rnk, n,
+ in,
+ (complex*)out,
+ comm, flags);
+ return r2c_plan;
+ }
+
+ // We need to find out the size of the buffer to allocate
+ mpi_local_size_transposed(
+ rnk, n, comm,
+ &r2c_plan.local_n0, &r2c_plan.local_0_start,
+ &r2c_plan.local_n1, &r2c_plan.local_1_start);
+
+ ptrdiff_t sizeBuffer = r2c_plan.local_n0;
+ for(int idxrnk = 1 ; idxrnk < rnk-1 ; ++idxrnk){
+ sizeBuffer *= n[idxrnk];
+ }
+ sizeBuffer *= n[rnk-1]+2;
+
+ r2c_plan.buffer.reset(alloc_real(sizeBuffer));
+ memset(r2c_plan.buffer.get(), 0, sizeof(real)*sizeBuffer);
+ r2c_plan.sizeBuffer = sizeBuffer;
+ // Init the plan
+ r2c_plan.plan_to_use = mpi_plan_dft_r2c(rnk, n,
+ r2c_plan.buffer.get(),
+ (complex*)r2c_plan.buffer.get(),
+ comm, flags);
+
+ r2c_plan.nb_sections_real = r2c_plan.local_n0;
+ for(int idxrnk = 1 ; idxrnk < rnk-1 ; ++idxrnk){
+ r2c_plan.nb_sections_real *= n[idxrnk];
+ r2c_plan.nb_sections_complex *= n[idxrnk];
+ }
+ r2c_plan.size_real_section = (n[rnk-1] + 2);
+
+ r2c_plan.nb_sections_complex = r2c_plan.local_n1;
+ for(int idxrnk = 1 ; idxrnk < rnk-1 ; ++idxrnk){
+ if(idxrnk == 1){
+ r2c_plan.nb_sections_complex *= n[0];
+ }
+ else{
+ r2c_plan.nb_sections_complex *= n[idxrnk];
+ }
+ }
+ r2c_plan.size_complex_section = (n[rnk-1]/2 + 1);
+
+ return r2c_plan;
+ }
+
+ static void execute(many_plan& in_plan){
+ if(in_plan.howmany == 1){
+ execute(in_plan.plan_to_use);
+ return;
+ }
+
+ std::unique_ptr<real[]> in_copy;
+ if(in_plan.is_r2c){
+ in_copy.reset(new real[in_plan.nb_sections_real * in_plan.size_real_section * in_plan.howmany]);
+
+ for(int idx_section = 0 ; idx_section < in_plan.nb_sections_real ; ++idx_section){
+ for(ptrdiff_t idx_copy = 0 ; idx_copy < in_plan.n[in_plan.rnk-1] ; ++idx_copy){
+ for(int idx_howmany = 0 ; idx_howmany < in_plan.howmany ; ++idx_howmany){
+ in_copy[idx_howmany + idx_copy*in_plan.howmany + idx_section*in_plan.size_real_section*in_plan.howmany] =
+ ((const real*)in_plan.in)[idx_howmany + idx_copy*in_plan.howmany + idx_section*in_plan.size_real_section*in_plan.howmany];
+ }
+ }
+ }
+ }
+ else{
+ in_copy.reset((real*)new complex[in_plan.nb_sections_complex * in_plan.size_complex_section * in_plan.howmany]);
+
+ for(int idx_section = 0 ; idx_section < in_plan.nb_sections_complex ; ++idx_section){
+ for(ptrdiff_t idx_copy = 0 ; idx_copy < in_plan.n[in_plan.rnk-1]/2+1 ; ++idx_copy){
+ for(int idx_howmany = 0 ; idx_howmany < in_plan.howmany ; ++idx_howmany){
+ ((complex*)in_copy.get())[idx_howmany + idx_copy*in_plan.howmany + idx_section*in_plan.size_complex_section*in_plan.howmany][0] =
+ ((const complex*)in_plan.in)[idx_howmany + idx_copy*in_plan.howmany + idx_section*in_plan.size_complex_section*in_plan.howmany][0];
+ ((complex*)in_copy.get())[idx_howmany + idx_copy*in_plan.howmany + idx_section*in_plan.size_complex_section*in_plan.howmany][1] =
+ ((const complex*)in_plan.in)[idx_howmany + idx_copy*in_plan.howmany + idx_section*in_plan.size_complex_section*in_plan.howmany][1];
+ }
+ }
+ }
+ }
+
+ for(int idx_howmany = 0 ; idx_howmany < in_plan.howmany ; ++idx_howmany){
+ // Copy to buffer
+ if(in_plan.is_r2c){
+ for(int idx_section = 0 ; idx_section < in_plan.nb_sections_real ; ++idx_section){
+ real* dest = in_plan.buffer.get() + idx_section*in_plan.size_real_section;
+ const real* src = in_copy.get()+idx_howmany + idx_section*in_plan.size_real_section*in_plan.howmany;
+
+ for(ptrdiff_t idx_copy = 0 ; idx_copy < in_plan.n[in_plan.rnk-1] ; ++idx_copy){
+ dest[idx_copy] = src[idx_copy*in_plan.howmany];
+ }
+ }
+ }
+ else{
+ for(int idx_section = 0 ; idx_section < in_plan.nb_sections_complex ; ++idx_section){
+ complex* dest = ((complex*)in_plan.buffer.get()) + idx_section*in_plan.size_complex_section;
+ const complex* src = ((const complex*)in_copy.get()) + idx_howmany + idx_section*in_plan.size_complex_section*in_plan.howmany;
+ for(ptrdiff_t idx_copy = 0 ; idx_copy < in_plan.n[in_plan.rnk-1]/2+1 ; ++idx_copy){
+ dest[idx_copy][0] = src[idx_copy*in_plan.howmany][0];
+ dest[idx_copy][1] = src[idx_copy*in_plan.howmany][1];
+ }
+ }
+ }
+
+ execute(in_plan.plan_to_use);
+ // Copy result from buffer
+ if(in_plan.is_r2c){
+ for(int idx_section = 0 ; idx_section < in_plan.nb_sections_complex ; ++idx_section){
+ complex* dest = ((complex*)in_plan.out) + idx_howmany + idx_section*in_plan.size_complex_section*in_plan.howmany;
+ const complex* src = ((const complex*)in_plan.buffer.get()) + idx_section*in_plan.size_complex_section;
+ for(ptrdiff_t idx_copy = 0 ; idx_copy < in_plan.n[in_plan.rnk-1]/2+1 ; ++idx_copy){
+ dest[idx_copy*in_plan.howmany][0] = src[idx_copy][0];
+ dest[idx_copy*in_plan.howmany][1] = src[idx_copy][1];
+ }
+ }
+ }
+ else{
+ for(int idx_section = 0 ; idx_section < in_plan.nb_sections_real ; ++idx_section){
+ real* dest = ((real*)in_plan.out)+idx_howmany + idx_section*in_plan.size_real_section*in_plan.howmany;
+ const real* src = in_plan.buffer.get() + idx_section*in_plan.size_real_section;
+
+ for(ptrdiff_t idx_copy = 0 ; idx_copy < in_plan.n[in_plan.rnk-1] ; ++idx_copy){
+ dest[idx_copy*in_plan.howmany] = src[idx_copy];
+ }
+ }
+ }
+ }
+ }
+
+ static void destroy_plan(many_plan& in_plan){
+ destroy_plan(in_plan.plan_to_use);
+ }
+#else
+
+ template <class ... Params>
+ static ptrdiff_t mpi_local_size_many_transposed(Params ... params){
+ return fftwf_mpi_local_size_many_transposed(params...);
+ }
+
+ template <class ... Params>
+ static plan mpi_plan_many_dft_c2r(Params ... params){
+ return fftwf_mpi_plan_many_dft_c2r(params...);
+ }
+
+ template <class ... Params>
+ static plan mpi_plan_many_dft_r2c(Params ... params){
+ return fftwf_mpi_plan_many_dft_r2c(params...);
+ }
+#endif
+
+ template <class ... Params>
+ static plan mpi_plan_dft_c2r(Params ... params){
+ return fftwf_mpi_plan_dft_c2r(params...);
+ }
+
+ template <class ... Params>
+ static plan mpi_plan_dft_r2c(Params ... params){
+ return fftwf_mpi_plan_dft_r2c(params...);
+ }
+
+ template <class ... Params>
+ static plan mpi_plan_dft_c2r_3d(Params ... params){
+ return fftwf_mpi_plan_dft_c2r_3d(params...);
+ }
+};
+
+template <>
+class fftw_interface<double>
+{
+public:
+ using real = double;
+ using complex = fftw_complex;
+ using plan = fftw_plan;
+ using iodim = fftw_iodim;
+#ifdef SPLIT_FFTW_MANY
+ struct many_plan_container{
+ int rnk;
+ std::vector<ptrdiff_t> n;
+ int howmany;
+ ptrdiff_t iblock;
+ ptrdiff_t oblock;
+ std::unique_ptr<real[], fftw_free_deleter> buffer;
+ plan plan_to_use;
+
+ ptrdiff_t local_n0, local_0_start;
+ ptrdiff_t local_n1, local_1_start;
+
+ bool is_r2c;
+ void* in;
+ void* out;
+
+ ptrdiff_t nb_sections_real;
+ ptrdiff_t size_real_section;
+ ptrdiff_t nb_sections_complex;
+ ptrdiff_t size_complex_section;
+
+ ptrdiff_t sizeBuffer;
+ };
+
+ using many_plan = many_plan_container;
+
+ static char* sprint(const many_plan mp)
+ {
+ return fftw_sprint_plan(mp.plan_to_use);
+ }
+#else
+ using many_plan = fftw_plan;
+
+ static char* sprint(const many_plan mp)
+ {
+ return fftw_sprint_plan(mp);
+ }
+#endif
+
+ static complex* alloc_complex(const size_t in_size){
+ return fftw_alloc_complex(in_size);
+ }
+
+ static real* alloc_real(const size_t in_size){
+ return fftw_alloc_real(in_size);
+ }
+
+ static void free(void* ptr){
+ fftw_free(ptr);
+ }
+
+ static void execute(plan in_plan){
+ fftw_execute(in_plan);
+ }
+
+ static void destroy_plan(plan in_plan){
+ fftw_destroy_plan(in_plan);
+ }
+
+ template <class ... Params>
+ static ptrdiff_t mpi_local_size_many(Params ... params){
+ return fftw_mpi_local_size_many(params...);
+ }
+
+ template <class ... Params>
+ static plan mpi_plan_transpose(Params ... params){
+ return fftw_mpi_plan_transpose(params...);
+ }
+
+ template <class ... Params>
+ static plan mpi_plan_many_transpose(Params ... params){
+ return fftw_mpi_plan_many_transpose(params...);
+ }
+
+ template <class ... Params>
+ static plan plan_guru_r2r(Params ... params){
+ return fftw_plan_guru_r2r(params...);
+ }
+
+ template <class ... Params>
+ static plan plan_guru_dft(Params ... params){
+ return fftw_plan_guru_dft(params...);
+ }
+
+ template <class ... Params>
+ static ptrdiff_t mpi_local_size_transposed(Params ... params){
+ return fftw_mpi_local_size_transposed(params...);
+ }
+
+#ifdef SPLIT_FFTW_MANY
+ static ptrdiff_t mpi_local_size_many_transposed(int rnk, const ptrdiff_t *n, ptrdiff_t howmany,
+ ptrdiff_t block0, ptrdiff_t block1, MPI_Comm comm,
+ ptrdiff_t *local_n0, ptrdiff_t *local_0_start,
+ ptrdiff_t *local_n1, ptrdiff_t *local_1_start){
+ assert(block0 == FFTW_MPI_DEFAULT_BLOCK);
+ assert(block1 == FFTW_MPI_DEFAULT_BLOCK);
+ return howmany*mpi_local_size_transposed(rnk, n, comm,
+ local_n0, local_0_start,
+ local_n1, local_1_start);
+ }
+
+ static many_plan mpi_plan_many_dft_c2r(int rnk, const ptrdiff_t *n, ptrdiff_t howmany,
+ ptrdiff_t iblock, ptrdiff_t oblock,
+ complex *in, real *out,
+ MPI_Comm comm, unsigned flags){
+ assert(iblock == FFTW_MPI_DEFAULT_BLOCK);
+ assert(oblock == FFTW_MPI_DEFAULT_BLOCK);
+
+ many_plan c2r_plan;
+ c2r_plan.rnk = rnk;
+ c2r_plan.n.insert(c2r_plan.n.end(), n, n+rnk);
+ c2r_plan.howmany = howmany;
+ c2r_plan.iblock = iblock;
+ c2r_plan.oblock = oblock;
+ c2r_plan.is_r2c = false;
+ c2r_plan.in = in;
+ c2r_plan.out = out;
+ c2r_plan.sizeBuffer = 0;
+
+ // If 1 then use default without copy
+ if(howmany == 1){
+ c2r_plan.plan_to_use = mpi_plan_dft_c2r(rnk, n,
+ (complex*)in,
+ out,
+ comm, flags);
+ return c2r_plan;
+ }
+
+ // We need to find out the size of the buffer to allocate
+ mpi_local_size_transposed(
+ rnk, n, comm,
+ &c2r_plan.local_n0, &c2r_plan.local_0_start,
+ &c2r_plan.local_n1, &c2r_plan.local_1_start);
+
+ ptrdiff_t sizeBuffer = c2r_plan.local_n0;
+ for(int idxrnk = 1 ; idxrnk < rnk-1 ; ++idxrnk){
+ sizeBuffer *= n[idxrnk];
+ }
+ sizeBuffer *= n[rnk-1]+2;
+
+ c2r_plan.buffer.reset(alloc_real(sizeBuffer));
+ memset(c2r_plan.buffer.get(), 0, sizeof(real)*sizeBuffer);
+ c2r_plan.sizeBuffer = sizeBuffer;
+ // Init the plan
+ c2r_plan.plan_to_use = mpi_plan_dft_c2r(rnk, n,
+ (complex*)c2r_plan.buffer.get(),
+ c2r_plan.buffer.get(),
+ comm, flags);
+
+ c2r_plan.nb_sections_real = c2r_plan.local_n0;
+ for(int idxrnk = 1 ; idxrnk < rnk-1 ; ++idxrnk){
+ c2r_plan.nb_sections_real *= n[idxrnk];
+ c2r_plan.nb_sections_complex *= n[idxrnk];
+ }
+ c2r_plan.size_real_section = (n[rnk-1] + 2);
+
+ c2r_plan.nb_sections_complex = c2r_plan.local_n1;
+ for(int idxrnk = 1 ; idxrnk < rnk-1 ; ++idxrnk){
+ if(idxrnk == 1){
+ c2r_plan.nb_sections_complex *= n[0];
+ }
+ else{
+ c2r_plan.nb_sections_complex *= n[idxrnk];
+ }
+ }
+ c2r_plan.size_complex_section = (n[rnk-1]/2 + 1);
+
+ return c2r_plan;
+ }
+
+ static many_plan mpi_plan_many_dft_r2c(int rnk, const ptrdiff_t *n, ptrdiff_t howmany,
+ ptrdiff_t iblock, ptrdiff_t oblock,
+ real *in, complex *out,
+ MPI_Comm comm, unsigned flags){
+ assert(iblock == FFTW_MPI_DEFAULT_BLOCK);
+ assert(oblock == FFTW_MPI_DEFAULT_BLOCK);
+
+ many_plan r2c_plan;
+ r2c_plan.rnk = rnk;
+ r2c_plan.n.insert(r2c_plan.n.end(), n, n+rnk);
+ r2c_plan.howmany = howmany;
+ r2c_plan.iblock = iblock;
+ r2c_plan.oblock = oblock;
+ r2c_plan.is_r2c = true;
+ r2c_plan.in = in;
+ r2c_plan.out = out;
+ r2c_plan.sizeBuffer = 0;
+
+ // If 1 then use default without copy
+ if(howmany == 1){
+ r2c_plan.plan_to_use = mpi_plan_dft_r2c(rnk, n,
+ in,
+ (complex*)out,
+ comm, flags);
+ return r2c_plan;
+ }
+
+ // We need to find out the size of the buffer to allocate
+ mpi_local_size_transposed(
+ rnk, n, comm,
+ &r2c_plan.local_n0, &r2c_plan.local_0_start,
+ &r2c_plan.local_n1, &r2c_plan.local_1_start);
+
+ ptrdiff_t sizeBuffer = r2c_plan.local_n0;
+ for(int idxrnk = 1 ; idxrnk < rnk-1 ; ++idxrnk){
+ sizeBuffer *= n[idxrnk];
+ }
+ sizeBuffer *= n[rnk-1]+2;
+
+ r2c_plan.buffer.reset(alloc_real(sizeBuffer));
+ memset(r2c_plan.buffer.get(), 0, sizeof(real)*sizeBuffer);
+ r2c_plan.sizeBuffer = sizeBuffer;
+ // Init the plan
+ r2c_plan.plan_to_use = mpi_plan_dft_r2c(rnk, n,
+ r2c_plan.buffer.get(),
+ (complex*)r2c_plan.buffer.get(),
+ comm, flags);
+
+ r2c_plan.nb_sections_real = r2c_plan.local_n0;
+ for(int idxrnk = 1 ; idxrnk < rnk-1 ; ++idxrnk){
+ r2c_plan.nb_sections_real *= n[idxrnk];
+ r2c_plan.nb_sections_complex *= n[idxrnk];
+ }
+ r2c_plan.size_real_section = (n[rnk-1] + 2);
+
+ r2c_plan.nb_sections_complex = r2c_plan.local_n1;
+ for(int idxrnk = 1 ; idxrnk < rnk-1 ; ++idxrnk){
+ if(idxrnk == 1){
+ r2c_plan.nb_sections_complex *= n[0];
+ }
+ else{
+ r2c_plan.nb_sections_complex *= n[idxrnk];
+ }
+ }
+ r2c_plan.size_complex_section = (n[rnk-1]/2 + 1);
+
+ return r2c_plan;
+ }
+
+ static void execute(many_plan& in_plan){
+ if(in_plan.howmany == 1){
+ execute(in_plan.plan_to_use);
+ return;
+ }
+
+ std::unique_ptr<real[]> in_copy;
+ if(in_plan.is_r2c){
+ in_copy.reset(new real[in_plan.nb_sections_real * in_plan.size_real_section * in_plan.howmany]);
+
+ for(int idx_section = 0 ; idx_section < in_plan.nb_sections_real ; ++idx_section){
+ for(ptrdiff_t idx_copy = 0 ; idx_copy < in_plan.n[in_plan.rnk-1] ; ++idx_copy){
+ for(int idx_howmany = 0 ; idx_howmany < in_plan.howmany ; ++idx_howmany){
+ in_copy[idx_howmany + idx_copy*in_plan.howmany + idx_section*in_plan.size_real_section*in_plan.howmany] =
+ ((const real*)in_plan.in)[idx_howmany + idx_copy*in_plan.howmany + idx_section*in_plan.size_real_section*in_plan.howmany];
+ }
+ }
+ }
+ }
+ else{
+ in_copy.reset((real*)new complex[in_plan.nb_sections_complex * in_plan.size_complex_section * in_plan.howmany]);
+
+ for(int idx_section = 0 ; idx_section < in_plan.nb_sections_complex ; ++idx_section){
+ for(ptrdiff_t idx_copy = 0 ; idx_copy < in_plan.n[in_plan.rnk-1]/2+1 ; ++idx_copy){
+ for(int idx_howmany = 0 ; idx_howmany < in_plan.howmany ; ++idx_howmany){
+ ((complex*)in_copy.get())[idx_howmany + idx_copy*in_plan.howmany + idx_section*in_plan.size_complex_section*in_plan.howmany][0] =
+ ((const complex*)in_plan.in)[idx_howmany + idx_copy*in_plan.howmany + idx_section*in_plan.size_complex_section*in_plan.howmany][0];
+ ((complex*)in_copy.get())[idx_howmany + idx_copy*in_plan.howmany + idx_section*in_plan.size_complex_section*in_plan.howmany][1] =
+ ((const complex*)in_plan.in)[idx_howmany + idx_copy*in_plan.howmany + idx_section*in_plan.size_complex_section*in_plan.howmany][1];
+ }
+ }
+ }
+ }
+
+ for(int idx_howmany = 0 ; idx_howmany < in_plan.howmany ; ++idx_howmany){
+ // Copy to buffer
+ if(in_plan.is_r2c){
+ for(int idx_section = 0 ; idx_section < in_plan.nb_sections_real ; ++idx_section){
+ real* dest = in_plan.buffer.get() + idx_section*in_plan.size_real_section;
+ const real* src = in_copy.get()+idx_howmany + idx_section*in_plan.size_real_section*in_plan.howmany;
+
+ for(ptrdiff_t idx_copy = 0 ; idx_copy < in_plan.n[in_plan.rnk-1] ; ++idx_copy){
+ dest[idx_copy] = src[idx_copy*in_plan.howmany];
+ }
+ }
+ }
+ else{
+ for(int idx_section = 0 ; idx_section < in_plan.nb_sections_complex ; ++idx_section){
+ complex* dest = ((complex*)in_plan.buffer.get()) + idx_section*in_plan.size_complex_section;
+ const complex* src = ((const complex*)in_copy.get()) + idx_howmany + idx_section*in_plan.size_complex_section*in_plan.howmany;
+ for(ptrdiff_t idx_copy = 0 ; idx_copy < in_plan.n[in_plan.rnk-1]/2+1 ; ++idx_copy){
+ dest[idx_copy][0] = src[idx_copy*in_plan.howmany][0];
+ dest[idx_copy][1] = src[idx_copy*in_plan.howmany][1];
+ }
+ }
+ }
+
+ execute(in_plan.plan_to_use);
+ // Copy result from buffer
+ if(in_plan.is_r2c){
+ for(int idx_section = 0 ; idx_section < in_plan.nb_sections_complex ; ++idx_section){
+ complex* dest = ((complex*)in_plan.out) + idx_howmany + idx_section*in_plan.size_complex_section*in_plan.howmany;
+ const complex* src = ((const complex*)in_plan.buffer.get()) + idx_section*in_plan.size_complex_section;
+ for(ptrdiff_t idx_copy = 0 ; idx_copy < in_plan.n[in_plan.rnk-1]/2+1 ; ++idx_copy){
+ dest[idx_copy*in_plan.howmany][0] = src[idx_copy][0];
+ dest[idx_copy*in_plan.howmany][1] = src[idx_copy][1];
+ }
+ }
+ }
+ else{
+ for(int idx_section = 0 ; idx_section < in_plan.nb_sections_real ; ++idx_section){
+ real* dest = ((real*)in_plan.out)+idx_howmany + idx_section*in_plan.size_real_section*in_plan.howmany;
+ const real* src = in_plan.buffer.get() + idx_section*in_plan.size_real_section;
+
+ for(ptrdiff_t idx_copy = 0 ; idx_copy < in_plan.n[in_plan.rnk-1] ; ++idx_copy){
+ dest[idx_copy*in_plan.howmany] = src[idx_copy];
+ }
+ }
+ }
+ }
+ }
+
+ static void destroy_plan(many_plan& in_plan){
+ destroy_plan(in_plan.plan_to_use);
+ }
+#else
+ template <class ... Params>
+ static ptrdiff_t mpi_local_size_many_transposed(Params ... params){
+ return fftw_mpi_local_size_many_transposed(params...);
+ }
+
+ template <class ... Params>
+ static plan mpi_plan_many_dft_c2r(Params ... params){
+ return fftw_mpi_plan_many_dft_c2r(params...);
+ }
+
+ template <class ... Params>
+ static plan mpi_plan_many_dft_r2c(Params ... params){
+ return fftw_mpi_plan_many_dft_r2c(params...);
+ }
+#endif
+
+ template <class ... Params>
+ static plan mpi_plan_dft_c2r(Params ... params){
+ return fftw_mpi_plan_dft_c2r(params...);
+ }
+
+ template <class ... Params>
+ static plan mpi_plan_dft_r2c(Params ... params){
+ return fftw_mpi_plan_dft_r2c(params...);
+ }
+
+ template <class ... Params>
+ static plan mpi_plan_dft_c2r_3d(Params ... params){
+ return fftw_mpi_plan_dft_c2r_3d(params...);
+ }
+};
+
+
+
+#endif // FFTW_INTERFACE_HPP
+
diff --git a/bfps/cpp/tracers.hpp b/cpp/fftw_tools.cpp
similarity index 54%
rename from bfps/cpp/tracers.hpp
rename to cpp/fftw_tools.cpp
index 1a063e026578dd71b9a223ee46b55d2c86d4399f..ae03966d686baab544d865cf1eab253faea6584d 100644
--- a/bfps/cpp/tracers.hpp
+++ b/cpp/fftw_tools.cpp
@@ -3,61 +3,36 @@
* Copyright 2015 Max Planck Institute *
* for Dynamics and Self-Organization *
* *
-* This file is part of bfps. *
+* This file is part of TurTLE. *
* *
-* bfps is free software: you can redistribute it and/or modify *
+* TurTLE is free software: you can redistribute it and/or modify *
* it under the terms of the GNU General Public License as published *
* by the Free Software Foundation, either version 3 of the License, *
* or (at your option) any later version. *
* *
-* bfps is distributed in the hope that it will be useful, *
+* TurTLE is distributed in the hope that it will be useful, *
* but WITHOUT ANY WARRANTY; without even the implied warranty of *
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the *
* GNU General Public License for more details. *
* *
* You should have received a copy of the GNU General Public License *
-* along with bfps. If not, see <http://www.gnu.org/licenses/> *
+* along with TurTLE. If not, see <http://www.gnu.org/licenses/> *
* *
* Contact: Cristian.Lalescu@ds.mpg.de *
* *
**********************************************************************/
-
-
-#include "slab_field_particles.hpp"
-
-#ifndef TRACERS
-
-#define TRACERS
-
-extern int myrank, nprocs;
-
-template <class rnumber>
-class tracers final:public slab_field_particles<rnumber>
-{
- public:
- rnumber *source_data;
- rnumber *data;
-
- /* methods */
- tracers(
- const char *NAME,
- fluid_solver_base<rnumber> *FSOLVER,
- const int NPARTICLES,
- base_polynomial_values BETA_POLYS,
- const int NEIGHBOURS,
- const int TRAJ_SKIP,
- const int INTEGRATION_STEPS,
- rnumber *SOURCE_DATA);
- ~tracers();
-
- void update_field(bool clip_on = true);
- virtual void get_rhs(double *x, double *rhs);
- virtual void jump_estimate(double *jump_length);
-
- void sample_vec_field(rnumber *vec_field, double *vec_values);
+#include <stdlib.h>
+#include <algorithm>
+#include <iostream>
+#include "base.hpp"
+#include "fftw_tools.hpp"
+#include "fftw_interface.hpp"
+
+std::map<std::string, unsigned> fftw_planner_string_to_flag = {
+ {"FFTW_ESTIMATE", FFTW_ESTIMATE},
+ {"FFTW_MEASURE", FFTW_MEASURE},
+ {"FFTW_PATIENT", FFTW_PATIENT},
+ {"parameter does not exist", DEFAULT_FFTW_FLAG},
};
-
-#endif//TRACERS
-
diff --git a/bfps/cpp/fftw_tools.hpp b/cpp/fftw_tools.hpp
similarity index 52%
rename from bfps/cpp/fftw_tools.hpp
rename to cpp/fftw_tools.hpp
index d0f3dbf30df3ee95f3d7934f0dd7fca633858b44..5fbcdf81bb3e3e82def8e897ed80a8c803584282 100644
--- a/bfps/cpp/fftw_tools.hpp
+++ b/cpp/fftw_tools.hpp
@@ -3,20 +3,20 @@
* Copyright 2015 Max Planck Institute *
* for Dynamics and Self-Organization *
* *
-* This file is part of bfps. *
+* This file is part of TurTLE. *
* *
-* bfps is free software: you can redistribute it and/or modify *
+* TurTLE is free software: you can redistribute it and/or modify *
* it under the terms of the GNU General Public License as published *
* by the Free Software Foundation, either version 3 of the License, *
* or (at your option) any later version. *
* *
-* bfps is distributed in the hope that it will be useful, *
+* TurTLE is distributed in the hope that it will be useful, *
* but WITHOUT ANY WARRANTY; without even the implied warranty of *
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the *
* GNU General Public License for more details. *
* *
* You should have received a copy of the GNU General Public License *
-* along with bfps. If not, see <http://www.gnu.org/licenses/> *
+* along with TurTLE. If not, see <http://www.gnu.org/licenses/> *
* *
* Contact: Cristian.Lalescu@ds.mpg.de *
* *
@@ -24,47 +24,51 @@
+#ifndef FFTW_TOOLS
+#define FFTW_TOOLS
+
+
#include <mpi.h>
#include <fftw3-mpi.h>
-#include "field_descriptor.hpp"
+#include <string>
+#include <map>
+#include <cmath>
-#ifndef FFTW_TOOLS
-#define FFTW_TOOLS
+// helper function
+inline void complex_number_phase_shifter(double &xx, double &yy, const double cos_val, const double sin_val)
+{
+ const double new_real = double(xx*cos_val - yy*sin_val);
+ const double new_imag = double(yy*cos_val + xx*sin_val);
+ xx = new_real;
+ yy = new_imag;
+}
-extern int myrank, nprocs;
+inline void complex_number_phase_shifter(double &xx, double &yy, const double phase)
+{
+ const double cval = cos(phase);
+ const double sval = sin(phase);
+ complex_number_phase_shifter(xx, yy, cval, sval);
+}
-/* given two arrays of the same dimension, we do a simple resize in
- * Fourier space: either chop off high modes, or pad with zeros.
- * the arrays are assumed to use 3D mpi fftw layout.
- * */
-template <class rnumber>
-int copy_complex_array(
- field_descriptor<rnumber> *fi,
- rnumber (*ai)[2],
- field_descriptor<rnumber> *fo,
- rnumber (*ao)[2],
- int howmany=1);
+inline void complex_number_phase_shifter(float &xx, float &yy, const double cos_val, const double sin_val)
+{
+ const float new_real = float(xx*cos_val - yy*sin_val);
+ const float new_imag = float(yy*cos_val + xx*sin_val);
+ xx = new_real;
+ yy = new_imag;
+}
-template <class rnumber>
-int clip_zero_padding(
- field_descriptor<rnumber> *f,
- rnumber *a,
- int howmany=1);
+inline void complex_number_phase_shifter(float &xx, float &yy, const double phase)
+{
+ const double cval = cos(phase);
+ const double sval = sin(phase);
+ complex_number_phase_shifter(xx, yy, cval, sval);
+}
+
+extern int myrank, nprocs;
-/* function to get pair of descriptors for real and Fourier space
- * arrays used with fftw.
- * the n0, n1, n2 correspond to the real space data WITHOUT the zero
- * padding that FFTW needs.
- * IMPORTANT: the real space array must be allocated with
- * 2*fc->local_size, and then the zeros cleaned up before trying
- * to write data.
- * */
-template <class rnumber>
-int get_descriptors_3D(
- int n0, int n1, int n2,
- field_descriptor<rnumber> **fr,
- field_descriptor<rnumber> **fc);
+extern std::map<std::string, unsigned> fftw_planner_string_to_flag;
#endif//FFTW_TOOLS
diff --git a/cpp/field.cpp b/cpp/field.cpp
new file mode 100644
index 0000000000000000000000000000000000000000..513c4b9c99d467faca85d399bdcd620728dfcff3
--- /dev/null
+++ b/cpp/field.cpp
@@ -0,0 +1,3478 @@
+/**********************************************************************
+* *
+* Copyright 2015 Max Planck Institute *
+* for Dynamics and Self-Organization *
+* *
+* This file is part of TurTLE. *
+* *
+* TurTLE is free software: you can redistribute it and/or modify *
+* it under the terms of the GNU General Public License as published *
+* by the Free Software Foundation, either version 3 of the License, *
+* or (at your option) any later version. *
+* *
+* TurTLE is distributed in the hope that it will be useful, *
+* but WITHOUT ANY WARRANTY; without even the implied warranty of *
+* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the *
+* GNU General Public License for more details. *
+* *
+* You should have received a copy of the GNU General Public License *
+* along with TurTLE. If not, see <http://www.gnu.org/licenses/> *
+* *
+* Contact: Cristian.Lalescu@ds.mpg.de *
+* *
+**********************************************************************/
+
+
+
+#include <sys/stat.h>
+#include <cmath>
+#include <cstdlib>
+#include <algorithm>
+#include <cassert>
+#include <random>
+#include "fftw_tools.hpp"
+#include "field.hpp"
+#include "scope_timer.hpp"
+#include "shared_array.hpp"
+
+template <typename rnumber,
+ field_backend be,
+ field_components fc>
+field<rnumber, be, fc>::field(
+ const int nx,
+ const int ny,
+ const int nz,
+ const MPI_Comm COMM_TO_USE,
+ const unsigned FFTW_PLAN_RIGOR)
+{
+ TIMEZONE("field::field");
+ this->comm = COMM_TO_USE;
+ MPI_Comm_rank(this->comm, &this->myrank);
+ MPI_Comm_size(this->comm, &this->nprocs);
+
+ this->fftw_plan_rigor = FFTW_PLAN_RIGOR;
+ this->real_space_representation = true;
+
+ /* generate HDF5 data types */
+ if (typeid(rnumber) == typeid(float))
+ this->rnumber_H5T = H5Tcopy(H5T_NATIVE_FLOAT);
+ else if (typeid(rnumber) == typeid(double))
+ this->rnumber_H5T = H5Tcopy(H5T_NATIVE_DOUBLE);
+ typedef struct {
+ rnumber re; /*real part*/
+ rnumber im; /*imaginary part*/
+ } tmp_complex_type;
+ this->cnumber_H5T = H5Tcreate(H5T_COMPOUND, sizeof(tmp_complex_type));
+ H5Tinsert(this->cnumber_H5T, "r", HOFFSET(tmp_complex_type, re), this->rnumber_H5T);
+ H5Tinsert(this->cnumber_H5T, "i", HOFFSET(tmp_complex_type, im), this->rnumber_H5T);
+
+ /* switch on backend */
+ switch(be)
+ {
+ case FFTW:
+ ptrdiff_t nfftw[3];
+ nfftw[0] = nz;
+ nfftw[1] = ny;
+ nfftw[2] = nx;
+ hsize_t tmp_local_size;
+ ptrdiff_t local_n0, local_0_start;
+ ptrdiff_t local_n1, local_1_start;
+ variable_used_only_in_assert(tmp_local_size);
+ tmp_local_size = fftw_interface<rnumber>::mpi_local_size_many_transposed(
+ 3, nfftw, ncomp(fc),
+ FFTW_MPI_DEFAULT_BLOCK, FFTW_MPI_DEFAULT_BLOCK, this->comm,
+ &local_n0, &local_0_start,
+ &local_n1, &local_1_start);
+ hsize_t sizes[3], subsizes[3], starts[3];
+ sizes[0] = nz; sizes[1] = ny; sizes[2] = nx;
+ subsizes[0] = local_n0; subsizes[1] = ny; subsizes[2] = nx;
+ starts[0] = local_0_start; starts[1] = 0; starts[2] = 0;
+ this->rlayout = new field_layout<fc>(
+ sizes, subsizes, starts, this->comm);
+ assert(tmp_local_size == this->rlayout->local_size);
+ this->npoints = this->rlayout->full_size / ncomp(fc);
+ sizes[0] = nz; sizes[1] = ny; sizes[2] = nx+2;
+ subsizes[0] = local_n0; subsizes[1] = ny; subsizes[2] = nx+2;
+ starts[0] = local_0_start; starts[1] = 0; starts[2] = 0;
+ this->rmemlayout = new field_layout<fc>(
+ sizes, subsizes, starts, this->comm);
+ sizes[0] = ny; sizes[1] = nz; sizes[2] = nx/2+1;
+ subsizes[0] = local_n1; subsizes[1] = nz; subsizes[2] = nx/2+1;
+ starts[0] = local_1_start; starts[1] = 0; starts[2] = 0;
+ this->clayout = new field_layout<fc>(
+ sizes, subsizes, starts, this->comm);
+ this->data = fftw_interface<rnumber>::alloc_real(
+ this->rmemlayout->local_size);
+ //char *plan_information = NULL;
+ this->c2r_plan = fftw_interface<rnumber>::mpi_plan_many_dft_c2r(
+ 3, nfftw, ncomp(fc),
+ FFTW_MPI_DEFAULT_BLOCK, FFTW_MPI_DEFAULT_BLOCK,
+ (typename fftw_interface<rnumber>::complex*)this->data,
+ this->data,
+ this->comm,
+ this->fftw_plan_rigor | FFTW_MPI_TRANSPOSED_IN);
+ //plan_information = fftw_interface<rnumber>::sprint(this->c2r_plan);
+ //DEBUG_MSG("field::field c2r plan representation is\n\%s\n", plan_information);
+ //free(plan_information);
+ this->r2c_plan = fftw_interface<rnumber>::mpi_plan_many_dft_r2c(
+ 3, nfftw, ncomp(fc),
+ FFTW_MPI_DEFAULT_BLOCK, FFTW_MPI_DEFAULT_BLOCK,
+ this->data,
+ (typename fftw_interface<rnumber>::complex*)this->data,
+ this->comm,
+ this->fftw_plan_rigor | FFTW_MPI_TRANSPOSED_OUT);
+ //plan_information = fftw_interface<rnumber>::sprint(this->r2c_plan);
+ //DEBUG_MSG("field::field r2c plan representation is\n\%s\n", plan_information);
+ //free(plan_information);
+ break;
+ }
+ // use Fourier representation for setting field to 0,
+ // since this actually touches all data
+ this->real_space_representation = false;
+ this->operator=(rnumber(0));
+ this->real_space_representation = true;
+}
+
+template <typename rnumber,
+ field_backend be,
+ field_components fc>
+field<rnumber, be, fc>::~field()
+{
+ /* close data types */
+ H5Tclose(this->rnumber_H5T);
+ H5Tclose(this->cnumber_H5T);
+ switch(be)
+ {
+ case FFTW:
+ delete this->rlayout;
+ delete this->rmemlayout;
+ delete this->clayout;
+ fftw_interface<rnumber>::free(this->data);
+ fftw_interface<rnumber>::destroy_plan(this->c2r_plan);
+ fftw_interface<rnumber>::destroy_plan(this->r2c_plan);
+ break;
+ }
+}
+
+template <typename rnumber,
+ field_backend be,
+ field_components fc>
+void field<rnumber, be, fc>::ift()
+{
+ TIMEZONE("field::ift");
+ start_mpi_profiling_zone(turtle_mpi_pcontrol::FFTW);
+ fftw_interface<rnumber>::execute(this->c2r_plan);
+ finish_mpi_profiling_zone(turtle_mpi_pcontrol::FFTW);
+ this->real_space_representation = true;
+}
+
+template <typename rnumber,
+ field_backend be,
+ field_components fc>
+void field<rnumber, be, fc>::dft()
+{
+ TIMEZONE("field::dft");
+ start_mpi_profiling_zone(turtle_mpi_pcontrol::FFTW);
+ fftw_interface<rnumber>::execute(this->r2c_plan);
+ finish_mpi_profiling_zone(turtle_mpi_pcontrol::FFTW);
+ this->real_space_representation = false;
+}
+
+template <typename rnumber,
+ field_backend be,
+ field_components fc>
+int field<rnumber, be, fc>::io(
+ const std::string fname,
+ const std::string field_name,
+ const int iteration,
+ const bool read)
+{
+ /* file dataset has same dimensions as field */
+ TIMEZONE("field::io");
+ hid_t file_id, dset_id, plist_id;
+ dset_id = H5I_BADID;
+ std::string representation = std::string(
+ this->real_space_representation ?
+ "real" : "complex");
+ std::string dset_name = (
+ "/" + field_name +
+ "/" + representation +
+ "/" + std::to_string(iteration));
+
+ /* open/create file */
+ start_mpi_profiling_zone(turtle_mpi_pcontrol::HDF5);
+ plist_id = H5Pcreate(H5P_FILE_ACCESS);
+ H5Pset_fapl_mpio(plist_id, this->comm, MPI_INFO_NULL);
+ bool file_exists = false;
+ {
+ struct stat file_buffer;
+ file_exists = (stat(fname.c_str(), &file_buffer) == 0);
+ }
+ if (read)
+ {
+ DEBUG_MSG("field::io trying to read field %s from file %s\n", dset_name.c_str(), fname.c_str());
+ assert(file_exists);
+ file_id = H5Fopen(fname.c_str(), H5F_ACC_RDONLY, plist_id);
+ }
+ else
+ {
+ if (file_exists)
+ file_id = H5Fopen(fname.c_str(), H5F_ACC_RDWR, plist_id);
+ else
+ file_id = H5Fcreate(fname.c_str(), H5F_ACC_EXCL, H5P_DEFAULT, plist_id);
+ }
+ assert(file_id >= 0);
+ H5Pclose(plist_id);
+
+ /* check what kind of representation is being used */
+ if (read)
+ {
+ dset_id = H5Dopen(
+ file_id,
+ dset_name.c_str(),
+ H5P_DEFAULT);
+ assert(dset_id >= 0);
+ hid_t dset_type = H5Dget_type(dset_id);
+ assert(dset_type >= 0);
+ bool io_for_real = (
+ H5Tequal(dset_type, H5T_IEEE_F32BE) ||
+ H5Tequal(dset_type, H5T_IEEE_F32LE) ||
+ H5Tequal(dset_type, H5T_INTEL_F32) ||
+ H5Tequal(dset_type, H5T_NATIVE_FLOAT) ||
+ H5Tequal(dset_type, H5T_IEEE_F64BE) ||
+ H5Tequal(dset_type, H5T_IEEE_F64LE) ||
+ H5Tequal(dset_type, H5T_INTEL_F64) ||
+ H5Tequal(dset_type, H5T_NATIVE_DOUBLE));
+ variable_used_only_in_assert(io_for_real);
+ H5Tclose(dset_type);
+ assert(this->real_space_representation == io_for_real);
+ }
+ finish_mpi_profiling_zone(turtle_mpi_pcontrol::HDF5);
+
+ /* generic space initialization */
+ hid_t fspace, mspace;
+ hsize_t count[ndim(fc)], offset[ndim(fc)], dims[ndim(fc)];
+ hsize_t memoffset[ndim(fc)], memshape[ndim(fc)];
+
+ if (this->real_space_representation)
+ {
+ for (unsigned int i=0; i<ndim(fc); i++)
+ {
+ count[i] = this->rlayout->subsizes[i];
+ offset[i] = this->rlayout->starts[i];
+ dims[i] = this->rlayout->sizes[i];
+ memshape[i] = this->rmemlayout->subsizes[i];
+ memoffset[i] = 0;
+ }
+ }
+ else
+ {
+ for (unsigned int i=0; i<ndim(fc); i++)
+ {
+ count [i] = this->clayout->subsizes[i];
+ offset[i] = this->clayout->starts[i];
+ dims [i] = this->clayout->sizes[i];
+ memshape [i] = count[i];
+ memoffset[i] = 0;
+ }
+ }
+ start_mpi_profiling_zone(turtle_mpi_pcontrol::HDF5);
+ mspace = H5Screate_simple(ndim(fc), memshape, NULL);
+ H5Sselect_hyperslab(mspace, H5S_SELECT_SET, memoffset, NULL, count, NULL);
+
+ /* open/create data set */
+ if (read)
+ fspace = H5Dget_space(dset_id);
+ else
+ {
+ if (!H5Lexists(file_id, field_name.c_str(), H5P_DEFAULT))
+ {
+ hid_t gid_tmp = H5Gcreate(
+ file_id, field_name.c_str(),
+ H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT);
+ H5Gclose(gid_tmp);
+ }
+
+ if (!H5Lexists(file_id, (field_name + "/" + representation).c_str(), H5P_DEFAULT))
+ {
+ hid_t gid_tmp = H5Gcreate(
+ file_id, ("/" + field_name + "/" + representation).c_str(),
+ H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT);
+ H5Gclose(gid_tmp);
+ }
+ if (H5Lexists(file_id, dset_name.c_str(), H5P_DEFAULT))
+ {
+ dset_id = H5Dopen(file_id, dset_name.c_str(), H5P_DEFAULT);
+ fspace = H5Dget_space(dset_id);
+ }
+ else
+ {
+ fspace = H5Screate_simple(
+ ndim(fc),
+ dims,
+ NULL);
+ /* chunking needs to go in here */
+ dset_id = H5Dcreate(
+ file_id,
+ dset_name.c_str(),
+ (this->real_space_representation ? this->rnumber_H5T : this->cnumber_H5T),
+ fspace,
+ H5P_DEFAULT,
+ H5P_DEFAULT,
+ H5P_DEFAULT);
+ assert(dset_id > 0);
+ }
+ }
+ finish_mpi_profiling_zone(turtle_mpi_pcontrol::HDF5);
+ /* both dset_id and fspace should now have sane values */
+
+ /* if reading, first set local data to 0 */
+ if (read)
+ *this = rnumber(0.0);
+
+ start_mpi_profiling_zone(turtle_mpi_pcontrol::HDF5);
+ /* both dset_id and fspace should now have sane values */
+ /* check file space */
+ int ndims_fspace = H5Sget_simple_extent_dims(fspace, dims, NULL);
+ variable_used_only_in_assert(ndims_fspace);
+ assert(((unsigned int)(ndims_fspace)) == ndim(fc));
+ if (this->real_space_representation)
+ {
+ for (unsigned int i=0; i<ndim(fc); i++)
+ {
+ offset[i] = this->rlayout->starts[i];
+ assert(dims[i] == this->rlayout->sizes[i]);
+ }
+ H5Sselect_hyperslab(fspace, H5S_SELECT_SET, offset, NULL, count, NULL);
+ if (read)
+ {
+ H5Dread(dset_id, this->rnumber_H5T, mspace, fspace, H5P_DEFAULT, this->data);
+ }
+ else
+ {
+ assert(this->real_space_representation);
+ H5Dwrite(dset_id, this->rnumber_H5T, mspace, fspace, H5P_DEFAULT, this->data);
+ }
+ H5Sclose(mspace);
+ }
+ else
+ {
+ for (unsigned int i=0; i<ndim(fc); i++)
+ {
+ offset[i] = this->clayout->starts[i];
+ assert(dims[i] == this->clayout->sizes[i]);
+ }
+ H5Sselect_hyperslab(fspace, H5S_SELECT_SET, offset, NULL, count, NULL);
+ if (read)
+ {
+ H5Dread(dset_id, this->cnumber_H5T, mspace, fspace, H5P_DEFAULT, this->data);
+ this->symmetrize();
+ }
+ else
+ {
+ assert(!this->real_space_representation);
+ H5Dwrite(dset_id, this->cnumber_H5T, mspace, fspace, H5P_DEFAULT, this->data);
+ }
+ H5Sclose(mspace);
+ }
+
+ H5Sclose(fspace);
+ /* close data set */
+ H5Dclose(dset_id);
+ /* close file */
+ H5Fclose(file_id);
+ finish_mpi_profiling_zone(turtle_mpi_pcontrol::HDF5);
+ return EXIT_SUCCESS;
+}
+
+template <typename rnumber,
+ field_backend be,
+ field_components fc>
+int field<rnumber, be, fc>::io_database(
+ const std::string fname,
+ const std::string field_name,
+ const int toffset,
+ const bool read)
+{
+ /* file dataset is has a time dimension as well */
+ TIMEZONE("field::io_database");
+ hid_t file_id, dset_id, plist_id;
+ dset_id = H5I_BADID;
+ std::string representation = std::string(
+ this->real_space_representation ?
+ "real" : "complex");
+ std::string dset_name = (
+ "/" + field_name +
+ "/" + representation);
+
+ /* open/create file */
+ plist_id = H5Pcreate(H5P_FILE_ACCESS);
+ H5Pset_fapl_mpio(plist_id, this->comm, MPI_INFO_NULL);
+ bool file_exists = false;
+ {
+ struct stat file_buffer;
+ file_exists = (stat(fname.c_str(), &file_buffer) == 0);
+ }
+ if (read)
+ {
+ DEBUG_MSG("field::io trying to read field from file %s\n", fname.c_str());
+ assert(file_exists);
+ file_id = H5Fopen(fname.c_str(), H5F_ACC_RDONLY, plist_id);
+ }
+ else
+ {
+ if (file_exists)
+ file_id = H5Fopen(fname.c_str(), H5F_ACC_RDWR, plist_id);
+ else
+ file_id = H5Fcreate(fname.c_str(), H5F_ACC_EXCL, H5P_DEFAULT, plist_id);
+ }
+ H5Pclose(plist_id);
+
+ /* check what kind of representation is being used */
+ if (read)
+ {
+ dset_id = H5Dopen(
+ file_id,
+ dset_name.c_str(),
+ H5P_DEFAULT);
+ hid_t dset_type = H5Dget_type(dset_id);
+ bool io_for_real = (
+ H5Tequal(dset_type, H5T_IEEE_F32BE) ||
+ H5Tequal(dset_type, H5T_IEEE_F32LE) ||
+ H5Tequal(dset_type, H5T_INTEL_F32) ||
+ H5Tequal(dset_type, H5T_NATIVE_FLOAT) ||
+ H5Tequal(dset_type, H5T_IEEE_F64BE) ||
+ H5Tequal(dset_type, H5T_IEEE_F64LE) ||
+ H5Tequal(dset_type, H5T_INTEL_F64) ||
+ H5Tequal(dset_type, H5T_NATIVE_DOUBLE));
+ variable_used_only_in_assert(io_for_real);
+ H5Tclose(dset_type);
+ assert(this->real_space_representation == io_for_real);
+ }
+
+ /* generic space initialization */
+ hid_t fspace, mspace;
+ hsize_t count[ndim(fc)+1], offset[ndim(fc)+1], dims[ndim(fc)+1];
+ hsize_t memoffset[ndim(fc)+1], memshape[ndim(fc)+1];
+
+ int dim_counter_offset = 1;
+ dim_counter_offset = 1;
+ count[0] = 1;
+ memshape[0] = 1;
+ memoffset[0] = 0;
+ if (this->real_space_representation)
+ {
+ for (unsigned int i=0; i<ndim(fc); i++)
+ {
+ count[i+dim_counter_offset] = this->rlayout->subsizes[i];
+ offset[i+dim_counter_offset] = this->rlayout->starts[i];
+ dims[i+dim_counter_offset] = this->rlayout->sizes[i];
+ memshape[i+dim_counter_offset] = this->rmemlayout->subsizes[i];
+ memoffset[i+dim_counter_offset] = 0;
+ }
+ mspace = H5Screate_simple(dim_counter_offset + ndim(fc), memshape, NULL);
+ H5Sselect_hyperslab(mspace, H5S_SELECT_SET, memoffset, NULL, count, NULL);
+ }
+ else
+ {
+ for (unsigned int i=0; i<ndim(fc); i++)
+ {
+ count[i+dim_counter_offset] = this->clayout->subsizes[i];
+ offset[i+dim_counter_offset] = this->clayout->starts[i];
+ dims[i+dim_counter_offset] = this->clayout->sizes[i];
+ memshape[i+dim_counter_offset] = count[i+dim_counter_offset];
+ memoffset[i+dim_counter_offset] = 0;
+ }
+ mspace = H5Screate_simple(dim_counter_offset + ndim(fc), memshape, NULL);
+ H5Sselect_hyperslab(mspace, H5S_SELECT_SET, memoffset, NULL, count, NULL);
+ }
+
+ /* open/create data set */
+ if (read)
+ fspace = H5Dget_space(dset_id);
+ else
+ {
+ if (!H5Lexists(file_id, field_name.c_str(), H5P_DEFAULT))
+ H5Gcreate(
+ file_id, field_name.c_str(),
+ H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT);
+ if (H5Lexists(file_id, dset_name.c_str(), H5P_DEFAULT))
+ {
+ dset_id = H5Dopen(file_id, dset_name.c_str(), H5P_DEFAULT);
+ fspace = H5Dget_space(dset_id);
+ }
+ else
+ {
+ fspace = H5Screate_simple(
+ ndim(fc),
+ dims,
+ NULL);
+ /* chunking needs to go in here */
+ dset_id = H5Dcreate(
+ file_id,
+ dset_name.c_str(),
+ (this->real_space_representation ? this->rnumber_H5T : this->cnumber_H5T),
+ fspace,
+ H5P_DEFAULT,
+ H5P_DEFAULT,
+ H5P_DEFAULT);
+ assert(dset_id > 0);
+ }
+ }
+ /* both dset_id and fspace should now have sane values */
+
+ if (read)
+ *this = rnumber(0.0);
+
+ /* check file space */
+ int ndims_fspace = H5Sget_simple_extent_dims(fspace, dims, NULL);
+ variable_used_only_in_assert(ndims_fspace);
+ assert(ndims_fspace == int(ndim(fc) + 1));
+ offset[0] = toffset;
+ if (this->real_space_representation)
+ {
+ for (unsigned int i=0; i<ndim(fc); i++)
+ {
+ offset[i+dim_counter_offset] = this->rlayout->starts[i];
+ assert(dims[i+dim_counter_offset] == this->rlayout->sizes[i]);
+ }
+ H5Sselect_hyperslab(fspace, H5S_SELECT_SET, offset, NULL, count, NULL);
+ if (read)
+ {
+ H5Dread(dset_id, this->rnumber_H5T, mspace, fspace, H5P_DEFAULT, this->data);
+ this->real_space_representation = true;
+ }
+ else
+ {
+ assert(this->real_space_representation);
+ H5Dwrite(dset_id, this->rnumber_H5T, mspace, fspace, H5P_DEFAULT, this->data);
+ }
+ H5Sclose(mspace);
+ }
+ else
+ {
+ for (unsigned int i=0; i<ndim(fc); i++)
+ {
+ offset[i+dim_counter_offset] = this->clayout->starts[i];
+ assert(dims[i+dim_counter_offset] == this->clayout->sizes[i]);
+ }
+ H5Sselect_hyperslab(fspace, H5S_SELECT_SET, offset, NULL, count, NULL);
+ if (read)
+ {
+ H5Dread(dset_id, this->cnumber_H5T, mspace, fspace, H5P_DEFAULT, this->data);
+ this->real_space_representation = false;
+ this->symmetrize();
+ }
+ else
+ {
+ assert(!this->real_space_representation);
+ H5Dwrite(dset_id, this->cnumber_H5T, mspace, fspace, H5P_DEFAULT, this->data);
+ }
+ H5Sclose(mspace);
+ }
+
+ H5Sclose(fspace);
+ /* close data set */
+ H5Dclose(dset_id);
+ /* close file */
+ H5Fclose(file_id);
+ return EXIT_SUCCESS;
+}
+
+
+template <typename rnumber,
+ field_backend be,
+ field_components fc>
+int field<rnumber, be, fc>::write_0slice(
+ const hid_t group,
+ const std::string field_name,
+ const int iteration)
+{
+ start_mpi_profiling_zone(turtle_mpi_pcontrol::HDF5);
+ // this should in principle work for any fc
+ TIMEZONE("field::write_0slice");
+ assert(this->real_space_representation);
+ if (this->myrank == 0)
+ {
+ hid_t dset, wspace, mspace;
+ int ndims;
+ hsize_t count[5], offset[5], dims[5];
+ offset[0] = iteration;
+ offset[1] = 0;
+ offset[2] = 0;
+ offset[3] = 0;
+ offset[4] = 0;
+ dset = H5Dopen(
+ group,
+ ("0slices/" + field_name + "/real").c_str(),
+ H5P_DEFAULT);
+ wspace = H5Dget_space(dset);
+ ndims = H5Sget_simple_extent_dims(wspace, dims, NULL);
+ // array in memory has 2 extra x points, because FFTW
+ count[0] = 1;
+ count[1] = this->rmemlayout->sizes[1];
+ count[2] = this->rmemlayout->sizes[2];
+ count[3] = 3;
+ count[4] = 3;
+ mspace = H5Screate_simple(ndims, count, NULL);
+ // array in file should not have the extra 2 points
+ count[1] = this->rlayout->sizes[1];
+ count[2] = this->rlayout->sizes[2];
+ // select right slice in file
+ H5Sselect_hyperslab(
+ wspace,
+ H5S_SELECT_SET,
+ offset,
+ NULL,
+ count,
+ NULL);
+ offset[0] = 0;
+ // select proper regions of memory
+ H5Sselect_hyperslab(
+ mspace,
+ H5S_SELECT_SET,
+ offset,
+ NULL,
+ count,
+ NULL);
+ H5Dwrite(
+ dset,
+ this->rnumber_H5T,
+ mspace,
+ wspace,
+ H5P_DEFAULT,
+ this->data);
+ H5Dclose(dset);
+ H5Sclose(mspace);
+ H5Sclose(wspace);
+ }
+ finish_mpi_profiling_zone(turtle_mpi_pcontrol::HDF5);
+ return EXIT_SUCCESS;
+}
+
+template <typename rnumber,
+ field_backend be,
+ field_components fc>
+int field<rnumber, be, fc>::write_filtered(
+ const std::string fname,
+ const std::string field_name,
+ const int iteration,
+ int nx,
+ int ny,
+ int nz)
+{
+ /* file dataset has same dimensions as field */
+ TIMEZONE("field::write_filtered");
+ // only works in Fourier representation
+ assert(!this->real_space_representation);
+ assert(hsize_t(nx) <= this->rlayout->sizes[2]);
+ assert(hsize_t(ny) <= this->rlayout->sizes[1]);
+ assert(hsize_t(nz) <= this->rlayout->sizes[0]);
+ // current algorithm only works for more than one process
+ assert(this->nprocs >= 2);
+ hid_t file_id, dset_id, plist_id;
+ dset_id = H5I_BADID;
+ std::string dset_name = (
+ "/" + field_name +
+ "/complex" +
+ "/" + std::to_string(iteration));
+
+ /* open/create file */
+ plist_id = H5Pcreate(H5P_FILE_ACCESS);
+ H5Pset_fapl_mpio(plist_id, this->comm, MPI_INFO_NULL);
+ bool file_exists = false;
+ struct stat file_buffer;
+ file_exists = (stat(fname.c_str(), &file_buffer) == 0);
+ if (file_exists)
+ file_id = H5Fopen(fname.c_str(), H5F_ACC_RDWR, plist_id);
+ else
+ file_id = H5Fcreate(fname.c_str(), H5F_ACC_EXCL, H5P_DEFAULT, plist_id);
+ assert(file_id >= 0);
+ H5Pclose(plist_id);
+
+ /* generic space initialization */
+ hid_t fspace, mspace;
+ hsize_t count[ndim(fc)], offset[ndim(fc)], dims[ndim(fc)], fdims[ndim(fc)];
+ hsize_t memoffset[ndim(fc)], memshape[ndim(fc)];
+
+ // set up dimensions
+ for (unsigned int i=3; i<ndim(fc); i++)
+ {
+ count [i] = this->clayout->subsizes[i];
+ offset[i] = this->clayout->starts[i];
+ dims [i] = this->clayout->sizes[i];
+ memshape [i] = count[i];
+ memoffset[i] = 0;
+ }
+ // these are dimensions of dataset, needed
+ // to create dataset
+ dims[0] = ny;
+ dims[1] = nz;
+ dims[2] = nx/2+1;
+
+ /* open/create data set */
+ if (!H5Lexists(file_id, field_name.c_str(), H5P_DEFAULT))
+ {
+ hid_t gid_tmp = H5Gcreate(
+ file_id, field_name.c_str(),
+ H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT);
+ H5Gclose(gid_tmp);
+ }
+ if (!H5Lexists(file_id, (field_name + "/complex").c_str(), H5P_DEFAULT))
+ {
+ hid_t gid_tmp = H5Gcreate(
+ file_id, ("/" + field_name + "/complex").c_str(),
+ H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT);
+ H5Gclose(gid_tmp);
+ }
+ if (H5Lexists(file_id, dset_name.c_str(), H5P_DEFAULT))
+ {
+ dset_id = H5Dopen(file_id, dset_name.c_str(), H5P_DEFAULT);
+ fspace = H5Dget_space(dset_id);
+ }
+ else
+ {
+ if (nz == 1)
+ {
+ hsize_t temp_dims[ndim(fc)-1];
+ temp_dims[0] = dims[0];
+ for (unsigned int i=1; i<ndim(fc)-1; i++)
+ temp_dims[i] = dims[i+1];
+ fspace = H5Screate_simple(
+ ndim(fc)-1,
+ temp_dims,
+ NULL);
+ }
+ else
+ fspace = H5Screate_simple(
+ ndim(fc),
+ dims,
+ NULL);
+ /* chunking needs to go in here */
+ dset_id = H5Dcreate(
+ file_id,
+ dset_name.c_str(),
+ this->cnumber_H5T,
+ fspace,
+ H5P_DEFAULT,
+ H5P_DEFAULT,
+ H5P_DEFAULT);
+ assert(dset_id > 0);
+ }
+ /* check file space */
+ int ndims_fspace = H5Sget_simple_extent_dims(fspace, fdims, NULL);
+ variable_used_only_in_assert(ndims_fspace);
+ if (nz == 1)
+ {
+ assert(((unsigned int)(ndims_fspace)) == ndim(fc)-1);
+ assert(dims[0] == fdims[0]);
+ for (unsigned int i=1; i<ndim(fc)-1; i++)
+ assert(dims[i+1] == fdims[i]);
+ }
+ else
+ {
+ assert(((unsigned int)(ndims_fspace)) == ndim(fc));
+ for (unsigned int i=0; i<ndim(fc); i++)
+ {
+ assert(dims[i] == fdims[i]);
+ }
+ }
+ /* both dset_id and fspace now have sane values */
+
+ /// set up counts and offsets
+ /// x is easy, since only positive modes are present
+ count [2] = nx/2+1;
+ offset[2] = 0;
+ memshape [2] = this->clayout->subsizes[2];
+ memoffset[2] = 0;
+
+ /// three options for y:
+ /// this->starts[0] <= ny/2
+ /// ny / 2 < this->starts[0] +this->clayout->subsizes[0] < this->sizes[0] - ny/2
+ /// this->starts[0] >= this->sizes[0] - ny/2
+ /// we don't care about saving the ny/2 mode, because of symmetry
+ hsize_t y0 = this->clayout->starts[0];
+ hsize_t y1 = this->clayout->starts[0] + this->clayout->subsizes[0];
+ memshape[0] = this->clayout->subsizes[0];
+ if (y1 <= hsize_t(ny/2))
+ {
+ count[0] = this->clayout->subsizes[0];
+ offset[0] = y0;
+ memoffset[0] = 0;
+ }
+ else
+ {
+ if (y0 < hsize_t(ny)/2)
+ {
+ count[0] = ny/2 - y0;
+ offset[0] = y0;
+ memoffset[0] = 0;
+ }
+ else
+ {
+ if (y1 <= hsize_t(this->clayout->sizes[0] - ny/2 + 1))
+ { // y0 < y1 therefore y0 <= this->clayout->sizes[0] - ny/2
+ count[0] = 0;
+ offset[0] = ny/2;
+ memoffset[0] = 0;
+ }
+ else
+ {
+ if (y0 <= hsize_t(this->clayout->sizes[0] - ny/2))
+ {
+ count[0] = y1 - (this->clayout->sizes[0] - ny/2);
+ offset[0] = ny - (this->clayout->sizes[0] - y0);
+ memoffset[0] = this->clayout->subsizes[0] - count[0];
+ }
+ else
+ {
+ count[0] = this->clayout->subsizes[0];
+ offset[0] = ny - (this->clayout->sizes[0] - y0);
+ memoffset[0] = 0;
+ }
+ }
+ }
+ }
+ //DEBUG_MSG("count[0] = %ld, offset[0] = %ld\n",
+ // count[0], offset[0]);
+ if (nz>=2)
+ {
+ assert(nz%2==0);
+ /// for z, we need to take into account that there are
+ /// both positive and negative modes
+ for (int cz = 0; cz < 2; cz++)
+ {
+ count [1] = nz/2;
+ offset[1] = cz*nz/2;
+ memshape [1] = this->clayout->sizes[1];
+ memoffset[1] = cz*(this->clayout->sizes[1] - nz/2);
+ //DEBUG_MSG("cz = %d, count[1] + offset[1] = %ld\n",
+ // cz, count[1] + offset[1]);
+
+ //now write data
+ mspace = H5Screate_simple(ndim(fc), memshape, NULL);
+ H5Sselect_hyperslab(mspace, H5S_SELECT_SET, memoffset, NULL, count, NULL);
+ H5Sselect_hyperslab(fspace, H5S_SELECT_SET, offset, NULL, count, NULL);
+ H5Dwrite(dset_id, this->cnumber_H5T, mspace, fspace, H5P_DEFAULT, this->data);
+ H5Sclose(mspace);
+ }
+ }
+ else
+ {
+ assert(nz == 1);
+ count [1] = 1;
+ offset[1] = 0;
+ memshape [1] = this->clayout->sizes[1];
+ memoffset[1] = 0;
+ //DEBUG_MSG("filtered write nz=1\n");
+
+ //now write data
+ mspace = H5Screate_simple(ndim(fc), memshape, NULL);
+ H5Sselect_hyperslab(mspace, H5S_SELECT_SET, memoffset, NULL, count, NULL);
+ for (unsigned int i=1; i<ndim(fc)-1; i++)
+ count[i] = count[i+1];
+ H5Sselect_hyperslab(fspace, H5S_SELECT_SET, offset, NULL, count, NULL);
+ H5Dwrite(dset_id, this->cnumber_H5T, mspace, fspace, H5P_DEFAULT, this->data);
+ H5Sclose(mspace);
+ }
+
+
+ /* close file data space */
+ H5Sclose(fspace);
+ /* close data set */
+ H5Dclose(dset_id);
+ /* close file */
+ H5Fclose(file_id);
+ return EXIT_SUCCESS;
+}
+
+
+template <typename rnumber,
+ field_backend be,
+ field_components fc>
+void field<rnumber, be, fc>::compute_rspace_xincrement_stats(
+ const int xcells,
+ const hid_t group,
+ const std::string dset_name,
+ const hsize_t toffset,
+ const std::vector<double> max_estimate,
+ field<rnumber, be, fc> *tmp_field)
+{
+ TIMEZONE("field::compute_rspace_xincrement_stats");
+ assert(this->real_space_representation);
+ assert(fc == ONE || fc == THREE);
+ bool own_field = false;
+ own_field = (tmp_field == NULL);
+ if (own_field)
+ tmp_field = new field<rnumber, be, fc>(
+ this->rlayout->sizes[2],
+ this->rlayout->sizes[1],
+ this->rlayout->sizes[0],
+ this->rlayout->comm);
+ tmp_field->real_space_representation = true;
+ this->RLOOP_simd(
+ [&](const ptrdiff_t rindex,
+ const ptrdiff_t xindex,
+ const ptrdiff_t yindex,
+ const ptrdiff_t zindex){
+ const hsize_t rrindex = (xindex + xcells)%this->rlayout->sizes[2] + (
+ zindex * this->rlayout->subsizes[1] + yindex)*(
+ this->rmemlayout->subsizes[2]);
+ for (unsigned int component=0; component < ncomp(fc); component++)
+ tmp_field->data[rindex*ncomp(fc) + component] =
+ this->data[rrindex*ncomp(fc) + component] -
+ this->data[rindex*ncomp(fc) + component];
+ });
+ tmp_field->compute_rspace_stats(
+ group,
+ dset_name,
+ toffset,
+ max_estimate);
+ if (own_field)
+ delete tmp_field;
+}
+
+
+
+template <typename rnumber,
+ field_backend be,
+ field_components fc>
+void field<rnumber, be, fc>::compute_rspace_stats(
+ const hid_t group,
+ const std::string dset_name,
+ const hsize_t toffset,
+ const std::vector<double> max_estimate)
+{
+ TIMEZONE("field::compute_rspace_stats");
+ assert(this->real_space_representation);
+ const unsigned int nmoments = 10;
+ int nvals, nbins;
+ if (this->myrank == 0)
+ {
+ hid_t dset, wspace;
+ hsize_t dims[ndim(fc)-1];
+ int ndims;
+ dset = H5Dopen(group, ("moments/" + dset_name).c_str(), H5P_DEFAULT);
+ wspace = H5Dget_space(dset);
+ ndims = H5Sget_simple_extent_dims(wspace, dims, NULL);
+ // the following -1 comes from -3+2
+ // field data has 3 space array indices that are averaged over,
+ // and possible component indices that are NOT averaged over
+ // HDF5 dataset has 2 extra indices for time and order of moment,
+ // and possible component indices
+ //DEBUG_MSG("ndims = %d, ndim(fc) = %d, dsetname = %s\n",
+ // ndims, ndim(fc),
+ // dset_name.c_str());
+ assert(ndims == int(ndim(fc))-1);
+ assert(dims[1] == nmoments);
+ switch(ndims)
+ {
+ case 2:
+ nvals = 1;
+ break;
+ case 3:
+ nvals = dims[2];
+ break;
+ case 4:
+ nvals = dims[2]*dims[3];
+ break;
+ }
+ H5Sclose(wspace);
+ H5Dclose(dset);
+ dset = H5Dopen(group, ("histograms/" + dset_name).c_str(), H5P_DEFAULT);
+ wspace = H5Dget_space(dset);
+ ndims = H5Sget_simple_extent_dims(wspace, dims, NULL);
+ assert(ndims == int(ndim(fc))-1);
+ nbins = dims[1];
+ if (ndims == 3)
+ assert(nvals == int(dims[2]));
+ else if (ndims == 4)
+ assert(nvals == int(dims[2]*dims[3]));
+ H5Sclose(wspace);
+ H5Dclose(dset);
+ }
+ {
+ TIMEZONE("MPI_Bcast");
+ MPI_Bcast(&nvals, 1, MPI_INT, 0, this->comm);
+ MPI_Bcast(&nbins, 1, MPI_INT, 0, this->comm);
+ }
+ assert(nvals == int(max_estimate.size()));
+
+ start_mpi_profiling_zone(turtle_mpi_pcontrol::FIELD);
+ shared_array<double> local_moments_threaded(nmoments*nvals, [&](double* local_moments){
+ std::fill_n(local_moments, nmoments*nvals, 0);
+ if (nvals == 4) local_moments[3] = max_estimate[3];
+ });
+
+ shared_array<double> val_tmp_threaded(nvals,[&](double *val_tmp){
+ std::fill_n(val_tmp, nvals, 0);
+ });
+
+ shared_array<ptrdiff_t> local_hist_threaded(nbins*nvals,[&](ptrdiff_t* local_hist){
+ std::fill_n(local_hist, nbins*nvals, 0);
+ });
+
+ shared_array<double> local_pow_tmp(nvals);
+
+ double *binsize = new double[nvals];
+ for (int i=0; i<nvals; i++)
+ binsize[i] = 2*max_estimate[i] / nbins;
+
+ {
+ {
+ TIMEZONE("field::RLOOP");
+ this->RLOOP(
+ [&](const ptrdiff_t rindex,
+ const ptrdiff_t xindex,
+ const ptrdiff_t yindex,
+ const ptrdiff_t zindex){
+ double* pow_tmp = local_pow_tmp.getMine();
+ std::fill_n(pow_tmp, nvals, 1);
+
+ double *local_moments = local_moments_threaded.getMine();
+ double *val_tmp = val_tmp_threaded.getMine();
+ ptrdiff_t *local_hist = local_hist_threaded.getMine();
+
+ if (nvals == int(4)) val_tmp[3] = 0.0;
+ for (unsigned int i=0; i<ncomp(fc); i++)
+ {
+ val_tmp[i] = this->data[rindex*ncomp(fc)+i];
+ if (nvals == int(4)) val_tmp[3] += val_tmp[i]*val_tmp[i];
+ }
+ if (nvals == int(4))
+ {
+ val_tmp[3] = sqrt(val_tmp[3]);
+ if (val_tmp[3] < local_moments[0*nvals+3])
+ local_moments[0*nvals+3] = val_tmp[3];
+ if (val_tmp[3] > local_moments[9*nvals+3])
+ local_moments[9*nvals+3] = val_tmp[3];
+ const int bin = int(floor(val_tmp[3]*2/binsize[3]));
+ if (bin >= 0 && bin < nbins){
+ local_hist[bin*nvals+3]++;
+ }
+ }
+ for (unsigned int i=0; i<ncomp(fc); i++)
+ {
+ if (val_tmp[i] < local_moments[0*nvals+i])
+ local_moments[0*nvals+i] = val_tmp[i];
+ if (val_tmp[i] > local_moments[(nmoments-1)*nvals+i])
+ local_moments[(nmoments-1)*nvals+i] = val_tmp[i];
+ const int bin = int(floor((val_tmp[i] + max_estimate[i]) / binsize[i]));
+ if (bin >= 0 && bin < nbins)
+ local_hist[bin*nvals+i]++;
+ }
+ for (int n=1; n < int(nmoments)-1; n++){
+ for (int i=0; i<nvals; i++){
+ local_moments[n*nvals + i] += (pow_tmp[i] = val_tmp[i]*pow_tmp[i]);
+ }
+ }
+ });
+#ifdef USE_TIMINGOUTPUT
+ MPI_Barrier(this->comm);
+#endif
+ }
+
+ {
+ TIMEZONE("FIELD_RLOOP::Merge");
+ local_moments_threaded.mergeParallel([&](const int idx, const double& v1, const double& v2) -> double {
+ if(nvals == int(4) && idx == 0*nvals+3){
+ return std::min(v1, v2);
+ }
+ if(nvals == int(4) && idx == 9*nvals+3){
+ return std::max(v1, v2);
+ }
+ if(idx < int(ncomp(fc))){
+ return std::min(v1, v2);
+ }
+ if(int(nmoments-1)*nvals <= idx && idx < int(int(nmoments-1)*nvals+ncomp(fc))){
+ return std::max(v1, v2);
+ }
+ return v1 + v2;
+ });
+
+ local_hist_threaded.mergeParallel();
+#ifdef USE_TIMINGOUTPUT
+ MPI_Barrier(this->comm);
+#endif
+ }
+ }
+ ptrdiff_t *hist = new ptrdiff_t[nbins*nvals];
+ double *moments = new double[nmoments*nvals];
+ {
+ TIMEZONE("MPI_Allreduce");
+ MPI_Allreduce(
+ (void*)local_moments_threaded.getMasterData(),
+ (void*)moments,
+ nvals,
+ MPI_DOUBLE, MPI_MIN, this->comm);
+ MPI_Allreduce(
+ (void*)(local_moments_threaded.getMasterData() + nvals),
+ (void*)(moments+nvals),
+ (nmoments-2)*nvals,
+ MPI_DOUBLE, MPI_SUM, this->comm);
+ MPI_Allreduce(
+ (void*)(local_moments_threaded.getMasterData() + (nmoments-1)*nvals),
+ (void*)(moments+(nmoments-1)*nvals),
+ nvals,
+ MPI_DOUBLE, MPI_MAX, this->comm);
+ MPI_Allreduce(
+ (void*)local_hist_threaded.getMasterData(),
+ (void*)hist,
+ nbins*nvals,
+ MPI_INT64_T, MPI_SUM, this->comm);
+ }
+ for (int n=1; n < int(nmoments)-1; n++)
+ for (int i=0; i<nvals; i++)
+ moments[n*nvals + i] /= this->npoints;
+
+ delete[] binsize;
+ finish_mpi_profiling_zone(turtle_mpi_pcontrol::FIELD);
+ if (this->myrank == 0)
+ {
+ TIMEZONE("root-work");
+ hid_t dset, wspace, mspace;
+ hsize_t count[ndim(fc)-1], offset[ndim(fc)-1], dims[ndim(fc)-1];
+ dset = H5Dopen(group, ("moments/" + dset_name).c_str(), H5P_DEFAULT);
+ assert(dset>0);
+ wspace = H5Dget_space(dset);
+ H5Sget_simple_extent_dims(wspace, dims, NULL);
+ offset[0] = toffset;
+ offset[1] = 0;
+ count[0] = 1;
+ count[1] = nmoments;
+ if (fc == THREE)
+ {
+ offset[2] = 0;
+ count[2] = nvals;
+ }
+ if (fc == THREExTHREE)
+ {
+ offset[2] = 0;
+ count[2] = 3;
+ offset[3] = 0;
+ count[3] = 3;
+ }
+ mspace = H5Screate_simple(ndim(fc)-1, count, NULL);
+ H5Sselect_hyperslab(wspace, H5S_SELECT_SET, offset, NULL, count, NULL);
+ H5Dwrite(dset, H5T_NATIVE_DOUBLE, mspace, wspace, H5P_DEFAULT, moments);
+ H5Sclose(wspace);
+ H5Sclose(mspace);
+ H5Dclose(dset);
+ dset = H5Dopen(group, ("histograms/" + dset_name).c_str(), H5P_DEFAULT);
+ assert(dset > 0);
+ wspace = H5Dget_space(dset);
+ count[1] = nbins;
+ mspace = H5Screate_simple(ndim(fc)-1, count, NULL);
+ H5Sselect_hyperslab(wspace, H5S_SELECT_SET, offset, NULL, count, NULL);
+ H5Dwrite(dset, H5T_NATIVE_INT64, mspace, wspace, H5P_DEFAULT, hist);
+ H5Sclose(wspace);
+ H5Sclose(mspace);
+ H5Dclose(dset);
+ }
+ delete[] moments;
+ delete[] hist;
+}
+
+
+
+template <typename rnumber,
+ field_backend be,
+ field_components fc>
+void field<rnumber, be, fc>::compute_rspace_zaverage(
+ const hid_t group,
+ const std::string dset_name,
+ const hsize_t toffset)
+{
+ TIMEZONE("field::compute_rspace_zaverage");
+ assert(this->real_space_representation);
+ const hsize_t slice_size = this->rlayout->local_size / this->rlayout->subsizes[0];
+
+ // initial arrays MUST be 0, because I'm just adding to them afterwards.
+ shared_array<double> local_zaverage_threaded(
+ slice_size, [&](double* local_zaverage){
+ std::fill_n(local_zaverage, slice_size, 0);
+ });
+
+ // sum along z direction
+ {
+ TIMEZONE("field::RLOOP");
+ this->RLOOP(
+ [&](const ptrdiff_t rindex,
+ const ptrdiff_t xindex,
+ const ptrdiff_t yindex,
+ const ptrdiff_t zindex){
+
+ double *local_zaverage = local_zaverage_threaded.getMine();
+ ptrdiff_t zaverage_index = (yindex*this->rlayout->subsizes[2]+xindex)*ncomp(fc);
+
+ switch(fc)
+ {
+ case ONE:
+ local_zaverage[zaverage_index] += this->rval(rindex);
+ break;
+ case THREE:
+ local_zaverage[zaverage_index+0] += this->rval(rindex, 0);
+ local_zaverage[zaverage_index+1] += this->rval(rindex, 1);
+ local_zaverage[zaverage_index+2] += this->rval(rindex, 2);
+ break;
+ case THREExTHREE:
+ local_zaverage[zaverage_index+0 + 0] += this->rval(rindex, 0, 0);
+ local_zaverage[zaverage_index+0 + 1] += this->rval(rindex, 0, 1);
+ local_zaverage[zaverage_index+0 + 2] += this->rval(rindex, 0, 2);
+ local_zaverage[zaverage_index+3 + 0] += this->rval(rindex, 1, 0);
+ local_zaverage[zaverage_index+3 + 1] += this->rval(rindex, 1, 1);
+ local_zaverage[zaverage_index+3 + 2] += this->rval(rindex, 1, 2);
+ local_zaverage[zaverage_index+6 + 0] += this->rval(rindex, 2, 0);
+ local_zaverage[zaverage_index+6 + 1] += this->rval(rindex, 2, 1);
+ local_zaverage[zaverage_index+6 + 2] += this->rval(rindex, 2, 2);
+ break;
+ }
+ });
+
+ TIMEZONE("FIELD_RLOOP::Merge");
+ local_zaverage_threaded.mergeParallel();
+ }
+ // sum along MPI processes
+ double *zaverage = new double[slice_size];
+ {
+ TIMEZONE("MPI_Allreduce");
+ MPI_Allreduce(
+ (void*)local_zaverage_threaded.getMasterData(),
+ (void*)zaverage,
+ slice_size,
+ MPI_DOUBLE, MPI_SUM, this->comm);
+ }
+ // divide by total number of slices
+ for (ptrdiff_t n=0; n < ptrdiff_t(slice_size); n++)
+ zaverage[n] /= this->rlayout->sizes[0];
+
+ if (this->myrank == 0)
+ {
+ TIMEZONE("root-work");
+ hid_t dset, wspace, mspace;
+ int ndims;
+ hsize_t count[5], offset[5], dims[5];
+ offset[0] = toffset;
+ offset[1] = 0;
+ offset[2] = 0;
+ offset[3] = 0;
+ offset[4] = 0;
+ dset = H5Dopen(
+ group,
+ ("zaverage/" + dset_name).c_str(),
+ H5P_DEFAULT);
+ wspace = H5Dget_space(dset);
+ ndims = H5Sget_simple_extent_dims(wspace, dims, NULL);
+ count[0] = 1;
+ count[1] = this->rlayout->sizes[1];
+ count[2] = this->rlayout->sizes[2];
+ count[3] = 3;
+ count[4] = 3;
+ // select right slice in file
+ H5Sselect_hyperslab(
+ wspace,
+ H5S_SELECT_SET,
+ offset,
+ NULL,
+ count,
+ NULL);
+ offset[0] = 0;
+ // select proper regions of memory
+ mspace = H5Screate_simple(ndims-1, count+1, NULL);
+ H5Sselect_hyperslab(
+ mspace,
+ H5S_SELECT_SET,
+ offset+1,
+ NULL,
+ count+1,
+ NULL);
+ H5Dwrite(
+ dset,
+ H5T_NATIVE_DOUBLE,
+ mspace,
+ wspace,
+ H5P_DEFAULT,
+ zaverage);
+ H5Dclose(dset);
+ H5Sclose(mspace);
+ H5Sclose(wspace);
+ }
+ delete[] zaverage;
+}
+
+template <typename rnumber,
+ field_backend be,
+ field_components fc>
+void field<rnumber, be, fc>::normalize()
+{
+ this->RLOOP(
+ [&](const ptrdiff_t rindex,
+ const ptrdiff_t xindex,
+ const ptrdiff_t yindex,
+ const ptrdiff_t zindex){
+ for (unsigned int i=0; i<ncomp(fc); i++)
+ this->data[rindex*ncomp(fc) + i] /= this->npoints;
+ });
+}
+
+template <typename rnumber,
+ field_backend be,
+ field_components fc>
+void field<rnumber, be, fc>::Hermitian_reflect()
+{
+ TIMEZONE("field::Hermitian_reflect");
+ start_mpi_profiling_zone(turtle_mpi_pcontrol::FIELD);
+ assert(!this->real_space_representation);
+ typename fftw_interface<rnumber>::complex *cdata = this->get_cdata();
+ // reflect kx = 0 plane, line by line, for ky != 0
+ MPI_Status *mpistatus = new MPI_Status;
+ // bufferp will hold data from "plus", i.e. iy
+ // bufferm will hold data from "minus", i.e. ny - iy
+ typename fftw_interface<rnumber>::complex *bufferp = new typename fftw_interface<rnumber>::complex[ncomp(fc)*this->clayout->sizes[1]];
+ typename fftw_interface<rnumber>::complex *bufferm = new typename fftw_interface<rnumber>::complex[ncomp(fc)*this->clayout->sizes[1]];
+ int rankp, rankm;
+ // for each ky slice except ky=0
+ for (ptrdiff_t iy = 1; iy < ptrdiff_t(this->clayout->sizes[0]/2); iy++)
+ {
+ // read rank plus and rank minus
+ rankp = this->clayout->rank[0][iy];
+ rankm = this->clayout->rank[0][this->clayout->sizes[0] - iy];
+ // if my rank is rank plus, I should fill out the plus buffer
+ if (this->clayout->myrank == rankp)
+ {
+ ptrdiff_t iyy = iy - this->clayout->starts[0];
+ for (ptrdiff_t iz = 0; iz < ptrdiff_t(this->clayout->sizes[1]); iz++)
+ {
+ ptrdiff_t cindex = this->get_cindex(0, iyy, iz);
+ for (int cc = 0; cc < int(ncomp(fc)); cc++)
+ for (int imag_comp=0; imag_comp<2; imag_comp++)
+ (*(bufferp + ncomp(fc)*iz+cc))[imag_comp] =
+ (*(cdata + ncomp(fc)*cindex + cc))[imag_comp];
+ }
+ }
+ // if my rank is rank minus, I should fill out the minus buffer
+ if (this->clayout->myrank == rankm)
+ {
+ ptrdiff_t iyy = (this->clayout->sizes[0] - iy) - this->clayout->starts[0];
+ for (ptrdiff_t iz = 0; iz < ptrdiff_t(this->clayout->sizes[1]); iz++)
+ {
+ ptrdiff_t cindex = this->get_cindex(0, iyy, iz);
+ for (int cc = 0; cc < int(ncomp(fc)); cc++)
+ for (int imag_comp=0; imag_comp<2; imag_comp++)
+ (*(bufferm + ncomp(fc)*iz+cc))[imag_comp] =
+ (*(cdata + ncomp(fc)*cindex + cc))[imag_comp];
+ }
+ }
+ // if ranks are different, send and receive
+ if (rankp != rankm)
+ {
+ // if my rank is rank plus, send buffer plus, receive buffer minus
+ if (this->clayout->myrank == rankp)
+ {
+ MPI_Send((void*)bufferp,
+ ncomp(fc)*this->clayout->sizes[1],
+ mpi_real_type<rnumber>::complex(),
+ rankm, 2*iy+0,
+ this->clayout->comm);
+ MPI_Recv((void*)bufferm,
+ ncomp(fc)*this->clayout->sizes[1],
+ mpi_real_type<rnumber>::complex(),
+ rankm, 2*iy+1,
+ this->clayout->comm,
+ mpistatus);
+ }
+ // if my rank is rank minus, receive buffer plus, send buffer minus
+ if (this->clayout->myrank == rankm)
+ {
+ MPI_Recv((void*)bufferp,
+ ncomp(fc)*this->clayout->sizes[1],
+ mpi_real_type<rnumber>::complex(),
+ rankp, 2*iy+0,
+ this->clayout->comm,
+ mpistatus);
+ MPI_Send((void*)bufferm,
+ ncomp(fc)*this->clayout->sizes[1],
+ mpi_real_type<rnumber>::complex(),
+ rankp, 2*iy+1,
+ this->clayout->comm);
+ }
+ }
+ // if I my rank is either plus or minus, I should update my slice
+ if (this->clayout->myrank == rankp)
+ {
+ ptrdiff_t iyy = iy - this->clayout->starts[0];
+ for (ptrdiff_t iz = 1; iz < ptrdiff_t(this->clayout->sizes[1]); iz++)
+ {
+ ptrdiff_t izz = (this->clayout->sizes[1] - iz);
+ ptrdiff_t cindex = this->get_cindex(0, iyy, iz);
+ for (int cc = 0; cc < int(ncomp(fc)); cc++)
+ {
+ (*(cdata + ncomp(fc)*cindex + cc))[0] = (*(bufferm + ncomp(fc)*izz+cc))[0];
+ (*(cdata + ncomp(fc)*cindex + cc))[1] = -(*(bufferm + ncomp(fc)*izz+cc))[1];
+ }
+ }
+ ptrdiff_t cindex = this->get_cindex(0, iyy, 0);
+ for (int cc = 0; cc < int(ncomp(fc)); cc++)
+ {
+ (*(cdata + cc + ncomp(fc)*cindex))[0] = (*(bufferm + cc))[0];
+ (*(cdata + cc + ncomp(fc)*cindex))[1] = -(*(bufferm + cc))[1];
+ }
+ }
+ // if I my rank is either plus or minus, I should update my slice
+ if (this->clayout->myrank == rankm)
+ {
+ ptrdiff_t iyy = (this->clayout->sizes[0] - iy) - this->clayout->starts[0];
+ for (ptrdiff_t iz = 1; iz < ptrdiff_t(this->clayout->sizes[1]); iz++)
+ {
+ ptrdiff_t izz = (this->clayout->sizes[1] - iz);
+ ptrdiff_t cindex = this->get_cindex(0, iyy, izz);
+ for (int cc = 0; cc < int(ncomp(fc)); cc++)
+ {
+ (*(cdata + ncomp(fc)*cindex + cc))[0] = (*(bufferp + ncomp(fc)*iz+cc))[0];
+ (*(cdata + ncomp(fc)*cindex + cc))[1] = -(*(bufferp + ncomp(fc)*iz+cc))[1];
+ }
+ }
+ ptrdiff_t cindex = this->get_cindex(0, iyy, 0);
+ for (int cc = 0; cc < int(ncomp(fc)); cc++)
+ {
+ (*(cdata + cc + ncomp(fc)*cindex))[0] = (*(bufferp + cc))[0];
+ (*(cdata + cc + ncomp(fc)*cindex))[1] = -(*(bufferp + cc))[1];
+ }
+ }
+ }
+ //fftw_interface<rnumber>::free(buffer);
+ delete[] bufferp;
+ delete[] bufferm;
+ delete mpistatus;
+ // reflect kx = 0, ky = 0 line
+ if (this->clayout->myrank == this->clayout->rank[0][0])
+ {
+ for (ptrdiff_t iz = 1; iz < ptrdiff_t(this->clayout->sizes[1]/2); iz++)
+ {
+ ptrdiff_t cindex0 = this->get_cindex(0, 0, iz);
+ ptrdiff_t cindex1 = this->get_cindex(0, 0, this->clayout->sizes[1] - iz);
+ for (int cc = 0; cc < int(ncomp(fc)); cc++)
+ {
+ double rep = (*(cdata + cc + ncomp(fc)*cindex0))[0];
+ double imp = (*(cdata + cc + ncomp(fc)*cindex0))[1];
+ double rem = (*(cdata + cc + ncomp(fc)*cindex1))[0];
+ double imm = (*(cdata + cc + ncomp(fc)*cindex1))[1];
+ (*(cdata + cc + ncomp(fc)*cindex0))[0] = rem;
+ (*(cdata + cc + ncomp(fc)*cindex0))[1] = -imm;
+ (*(cdata + cc + ncomp(fc)*cindex1))[0] = rep;
+ (*(cdata + cc + ncomp(fc)*cindex1))[1] = -imp;
+ }
+ }
+ }
+ // make 0 mode real
+ if (this->myrank == this->clayout->rank[0][0])
+ {
+ for (ptrdiff_t cc = 0; cc < ncomp(fc); cc++)
+ cdata[cc][1] = 0.0;
+ }
+ // put kx = nx/2 modes to 0
+ for (ptrdiff_t iy = 0; iy < ptrdiff_t(this->clayout->subsizes[0]); iy++)
+ for (ptrdiff_t iz = 0; iz < ptrdiff_t(this->clayout->subsizes[1]); iz++)
+ {
+ ptrdiff_t cindex = this->get_cindex(this->clayout->sizes[2]-1, iy, iz);
+ for (int cc = 0; cc < int(ncomp(fc)); cc++) {
+ (*(cdata + cc + ncomp(fc)*cindex))[0] = 0.0;
+ (*(cdata + cc + ncomp(fc)*cindex))[1] = 0.0;
+ }
+ }
+ // put ky = ny/2 modes to 0
+ if (this->clayout->myrank == this->clayout->rank[0][this->clayout->sizes[0]/2])
+ {
+ for (ptrdiff_t iz = 0; iz < ptrdiff_t(this->clayout->subsizes[1]); iz++)
+ for (ptrdiff_t ix = 0; ix < ptrdiff_t(this->clayout->subsizes[2]); ix++)
+ {
+ ptrdiff_t cindex = this->get_cindex(ix, this->clayout->sizes[0]/2-this->clayout->starts[0], iz);
+ for (int cc = 0; cc < int(ncomp(fc)); cc++) {
+ (*(cdata + cc + ncomp(fc)*cindex))[0] = 0.0;
+ (*(cdata + cc + ncomp(fc)*cindex))[1] = 0.0;
+ }
+ }
+ }
+ // put kz = nz/2 modes to 0
+ for (ptrdiff_t iy = 0; iy < ptrdiff_t(this->clayout->subsizes[0]); iy++)
+ for (ptrdiff_t ix = 0; ix < ptrdiff_t(this->clayout->subsizes[2]); ix++)
+ {
+ ptrdiff_t cindex = this->get_cindex(ix, iy, this->clayout->sizes[1]/2);
+ for (int cc = 0; cc < int(ncomp(fc)); cc++) {
+ (*(cdata + cc + ncomp(fc)*cindex))[0] = 0.0;
+ (*(cdata + cc + ncomp(fc)*cindex))[1] = 0.0;
+ }
+ }
+ finish_mpi_profiling_zone(turtle_mpi_pcontrol::FIELD);
+}
+
+
+/** \brief Enforce Hermitian symmetry (slow, reference)
+ *
+ * TurTLE uses real-to-complex and complex-to-real FFTW transforms, because the
+ * equations are PDEs of real-valued fields.
+ * Hermitian symmetry means that Fourier-transformed real valued fields must
+ * respect the equation \f$\hat f(-\mathbf{k}) = {\hat f}^*(\mathbf{k})\f$.
+ *
+ * FFTW enforces this property mainly by only storing the positive half of the
+ * Fourier grid for the fastest array component. In TurTLE's case, this means
+ * \f$ k_x \f$.
+ * For the \f$ k_x = 0 \f$ plane, the symmetry must be enforced.
+ *
+ * This method uses a pair of backwards-forwards FFTs, which leads to FFTW
+ * effectively imposing Hermitian symmetry. It should be used as a reference
+ * implementation to calibrate against in the general case.
+ *
+ * */
+template <typename rnumber,
+ field_backend be,
+ field_components fc>
+void field<rnumber, be, fc>::symmetrize_FFT()
+{
+ TIMEZONE("field::symmetrize_FFT");
+ assert(!this->real_space_representation);
+ typename fftw_interface<rnumber>::complex *cdata = this->get_cdata();
+
+ // make 0 mode real
+ if (this->myrank == this->clayout->rank[0][0])
+ {
+ for (ptrdiff_t cc = 0; cc < ncomp(fc); cc++)
+ cdata[cc][1] = 0.0;
+ }
+ // put kx = nx/2 modes to 0
+ for (ptrdiff_t iy = 0; iy < ptrdiff_t(this->clayout->subsizes[0]); iy++)
+ for (ptrdiff_t iz = 0; iz < ptrdiff_t(this->clayout->subsizes[1]); iz++)
+ {
+ ptrdiff_t cindex = this->get_cindex(this->clayout->sizes[2]-1, iy, iz);
+ for (int cc = 0; cc < int(ncomp(fc)); cc++) {
+ (*(cdata + cc + ncomp(fc)*cindex))[0] = 0.0;
+ (*(cdata + cc + ncomp(fc)*cindex))[1] = 0.0;
+ }
+ }
+ // put ky = ny/2 modes to 0
+ if (this->clayout->myrank == this->clayout->rank[0][this->clayout->sizes[0]/2])
+ {
+ for (ptrdiff_t iz = 0; iz < ptrdiff_t(this->clayout->subsizes[1]); iz++)
+ for (ptrdiff_t ix = 0; ix < ptrdiff_t(this->clayout->subsizes[2]); ix++)
+ {
+ ptrdiff_t cindex = this->get_cindex(ix, this->clayout->sizes[0]/2-this->clayout->starts[0], iz);
+ for (int cc = 0; cc < int(ncomp(fc)); cc++) {
+ (*(cdata + cc + ncomp(fc)*cindex))[0] = 0.0;
+ (*(cdata + cc + ncomp(fc)*cindex))[1] = 0.0;
+ }
+ }
+ }
+ // put kz = nz/2 modes to 0
+ for (ptrdiff_t iy = 0; iy < ptrdiff_t(this->clayout->subsizes[0]); iy++)
+ for (ptrdiff_t ix = 0; ix < ptrdiff_t(this->clayout->subsizes[2]); ix++)
+ {
+ ptrdiff_t cindex = this->get_cindex(ix, iy, this->clayout->sizes[1]/2);
+ for (int cc = 0; cc < int(ncomp(fc)); cc++) {
+ (*(cdata + cc + ncomp(fc)*cindex))[0] = 0.0;
+ (*(cdata + cc + ncomp(fc)*cindex))[1] = 0.0;
+ }
+ }
+
+ this->ift();
+ this->dft();
+ this->normalize();
+ return;
+}
+
+/** \brief Enforce Hermitian symmetry (unoptimized, amplitude-aware)
+ *
+ * TurTLE uses real-to-complex and complex-to-real FFTW transforms, because the
+ * equations are PDEs of real-valued fields.
+ * Hermitian symmetry means that Fourier-transformed real valued fields must
+ * respect the equation \f$\hat f(-\mathbf{k}) = {\hat f}^*(\mathbf{k})\f$.
+ *
+ * FFTW enforces this property mainly by only storing the positive half of the
+ * Fourier grid for the fastest array component. In TurTLE's case, this means
+ * \f$ k_x \f$.
+ * For the \f$ k_x = 0 \f$ plane, the symmetry must be enforced.
+ *
+ * This method is an alternative to the default arithmetic mean method, meant
+ * to be used in special circumstances where it is important to retain the
+ * exact amplitude of modes.
+ * Rather than an arithmetic mean, here we compute the amplitudes and phases
+ * for the \f$(0, k_y, k_z)\f$ and \f$(0, -k_y, -k_z)\f$ modes. We then compute
+ * a mean amplitude as the square root of the product of the two amplitudes,
+ * and a mean phase as the arithmetic mean of the two phases.
+ * When this method is applied to a field with fixed amplitudes, but random
+ * phases, it should preserve the spectrum of the initial field exactly.
+ *
+ * */
+template <typename rnumber,
+ field_backend be,
+ field_components fc>
+void field<rnumber, be, fc>::symmetrize_alternate()
+{
+ TIMEZONE("field::symmetrize");
+ MPI_Barrier(this->clayout->comm); // TODO: figure out if this can be taken out by careful MPI tag generation
+ start_mpi_profiling_zone(turtle_mpi_pcontrol::FIELD);
+ assert(!this->real_space_representation);
+ typename fftw_interface<rnumber>::complex *cdata = this->get_cdata();
+
+ // make 0 mode real
+ if (this->myrank == this->clayout->rank[0][0])
+ {
+ for (ptrdiff_t cc = 0; cc < ncomp(fc); cc++)
+ cdata[cc][1] = 0.0;
+ }
+ // put kx = nx/2 modes to 0
+ for (ptrdiff_t iy = 0; iy < ptrdiff_t(this->clayout->subsizes[0]); iy++)
+ for (ptrdiff_t iz = 0; iz < ptrdiff_t(this->clayout->subsizes[1]); iz++)
+ {
+ ptrdiff_t cindex = this->get_cindex(this->clayout->sizes[2]-1, iy, iz);
+ for (int cc = 0; cc < int(ncomp(fc)); cc++) {
+ (*(cdata + cc + ncomp(fc)*cindex))[0] = 0.0;
+ (*(cdata + cc + ncomp(fc)*cindex))[1] = 0.0;
+ }
+ }
+ // put ky = ny/2 modes to 0
+ if (this->clayout->myrank == this->clayout->rank[0][this->clayout->sizes[0]/2])
+ {
+ for (ptrdiff_t iz = 0; iz < ptrdiff_t(this->clayout->subsizes[1]); iz++)
+ for (ptrdiff_t ix = 0; ix < ptrdiff_t(this->clayout->subsizes[2]); ix++)
+ {
+ ptrdiff_t cindex = this->get_cindex(ix, this->clayout->sizes[0]/2-this->clayout->starts[0], iz);
+ for (int cc = 0; cc < int(ncomp(fc)); cc++) {
+ (*(cdata + cc + ncomp(fc)*cindex))[0] = 0.0;
+ (*(cdata + cc + ncomp(fc)*cindex))[1] = 0.0;
+ }
+ }
+ }
+ // put kz = nz/2 modes to 0
+ for (ptrdiff_t iy = 0; iy < ptrdiff_t(this->clayout->subsizes[0]); iy++)
+ for (ptrdiff_t ix = 0; ix < ptrdiff_t(this->clayout->subsizes[2]); ix++)
+ {
+ ptrdiff_t cindex = this->get_cindex(ix, iy, this->clayout->sizes[1]/2);
+ for (int cc = 0; cc < int(ncomp(fc)); cc++) {
+ (*(cdata + cc + ncomp(fc)*cindex))[0] = 0.0;
+ (*(cdata + cc + ncomp(fc)*cindex))[1] = 0.0;
+ }
+ }
+
+ // symmetrize kx = 0 plane, line by line, for ky != 0
+ MPI_Status *mpistatus = new MPI_Status;
+ // bufferp will hold data from "plus", i.e. iy
+ // bufferm will hold data from "minus", i.e. ny - iy
+ typename fftw_interface<rnumber>::complex *bufferp = new typename fftw_interface<rnumber>::complex[ncomp(fc)*this->clayout->sizes[1]];
+ typename fftw_interface<rnumber>::complex *bufferm = new typename fftw_interface<rnumber>::complex[ncomp(fc)*this->clayout->sizes[1]];
+ int rankp, rankm;
+ // for each ky slice except ky=0
+ for (ptrdiff_t iy = 1; iy < ptrdiff_t(this->clayout->sizes[0]/2); iy++)
+ {
+ // read rank plus and rank minus
+ rankp = this->clayout->rank[0][iy];
+ rankm = this->clayout->rank[0][this->clayout->sizes[0] - iy];
+ // if my rank is plus or minus, then I should do actual work
+ if (this->clayout->myrank == rankp ||
+ this->clayout->myrank == rankm)
+ {
+ #pragma omp parallel
+ {
+ const ptrdiff_t zstart = ptrdiff_t(OmpUtils::ForIntervalStart(this->clayout->sizes[1]));
+ const ptrdiff_t zend = ptrdiff_t(OmpUtils::ForIntervalEnd(this->clayout->sizes[1]));
+ // if my rank is rank plus, I should fill out the plus buffer
+ if (this->clayout->myrank == rankp)
+ {
+ ptrdiff_t iyy = iy - this->clayout->starts[0];
+ for (ptrdiff_t iz = zstart; iz < zend; iz++)
+ {
+ ptrdiff_t cindex = this->get_cindex(0, iyy, iz);
+ for (int cc = 0; cc < int(ncomp(fc)); cc++)
+ for (int imag_comp=0; imag_comp<2; imag_comp++)
+ (*(bufferp + ncomp(fc)*iz+cc))[imag_comp] =
+ (*(cdata + ncomp(fc)*cindex + cc))[imag_comp];
+ }
+ }
+ // if my rank is rank minus, I should fill out the minus buffer
+ if (this->clayout->myrank == rankm)
+ {
+ ptrdiff_t iyy = (this->clayout->sizes[0] - iy) - this->clayout->starts[0];
+ for (ptrdiff_t iz = zstart; iz < zend; iz++)
+ {
+ ptrdiff_t cindex = this->get_cindex(0, iyy, iz);
+ for (int cc = 0; cc < int(ncomp(fc)); cc++)
+ for (int imag_comp=0; imag_comp<2; imag_comp++)
+ (*(bufferm + ncomp(fc)*iz+cc))[imag_comp] =
+ (*(cdata + ncomp(fc)*cindex + cc))[imag_comp];
+ }
+ }
+ // after filling out buffers, synchronize threads
+ #pragma omp barrier
+
+ // if ranks are different, send and receive
+ if (rankp != rankm)
+ {
+ // if my rank is rank plus, send buffer plus, receive buffer minus
+ if (this->clayout->myrank == rankp && omp_get_thread_num() == 0)
+ {
+ MPI_Send((void*)bufferp,
+ ncomp(fc)*this->clayout->sizes[1],
+ mpi_real_type<rnumber>::complex(),
+ rankm, 2*iy+0,
+ this->clayout->comm);
+ MPI_Recv((void*)bufferm,
+ ncomp(fc)*this->clayout->sizes[1],
+ mpi_real_type<rnumber>::complex(),
+ rankm, 2*iy+1,
+ this->clayout->comm,
+ mpistatus);
+ }
+ // if my rank is rank minus, receive buffer plus, send buffer minus
+ if (this->clayout->myrank == rankm && omp_get_thread_num() == 0)
+ {
+ MPI_Recv((void*)bufferp,
+ ncomp(fc)*this->clayout->sizes[1],
+ mpi_real_type<rnumber>::complex(),
+ rankp, 2*iy+0,
+ this->clayout->comm,
+ mpistatus);
+ MPI_Send((void*)bufferm,
+ ncomp(fc)*this->clayout->sizes[1],
+ mpi_real_type<rnumber>::complex(),
+ rankp, 2*iy+1,
+ this->clayout->comm);
+ }
+ }
+ // ensure buffers are updated by MPI transfer before threads
+ // start working again
+ #pragma omp barrier
+
+ // if I my rank is either plus or minus, I should update my slice
+ if (this->clayout->myrank == rankp)
+ {
+ ptrdiff_t iyy = iy - this->clayout->starts[0];
+ for (ptrdiff_t iz = zstart; iz < zend; iz++)
+ {
+ // modulo operation makes sense only for slab decomposition
+ ptrdiff_t izz = (this->clayout->sizes[1] - iz) % this->clayout->sizes[1];
+ ptrdiff_t cindex = this->get_cindex(0, iyy, iz);
+ for (int cc = 0; cc < int(ncomp(fc)); cc++)
+ {
+ const double ampp = sqrt(
+ (*(bufferp + ncomp(fc)*iz+cc))[0]*(*(bufferp + ncomp(fc)*iz+cc))[0] +
+ (*(bufferp + ncomp(fc)*iz+cc))[1]*(*(bufferp + ncomp(fc)*iz+cc))[1]);
+ const double phip = atan2(
+ (*(bufferp + ncomp(fc)*iz+cc))[1],
+ (*(bufferp + ncomp(fc)*iz+cc))[0]);
+ const double ampm = sqrt(
+ (*(bufferm + ncomp(fc)*izz+cc))[0]*(*(bufferm + ncomp(fc)*izz+cc))[0] +
+ (*(bufferm + ncomp(fc)*izz+cc))[1]*(*(bufferm + ncomp(fc)*izz+cc))[1]);
+ const double phim = atan2(
+ (*(bufferm + ncomp(fc)*izz+cc))[1],
+ (*(bufferm + ncomp(fc)*izz+cc))[0]);
+ const double amp = sqrt(ampp*ampm);
+ const double phi = (phip - phim)/2;
+ (*(cdata + ncomp(fc)*cindex + cc))[0] = amp*cos(phi);
+ (*(cdata + ncomp(fc)*cindex + cc))[1] = amp*sin(phi);
+ }
+ }
+ }
+ // if I my rank is either plus or minus, I should update my slice
+ if (this->clayout->myrank == rankm)
+ {
+ ptrdiff_t iyy = (this->clayout->sizes[0] - iy) - this->clayout->starts[0];
+ for (ptrdiff_t iz = zstart; iz < zend; iz++)
+ {
+ // modulo operation makes sense only for slab decomposition
+ ptrdiff_t izz = (this->clayout->sizes[1] - iz) % this->clayout->sizes[1];
+ ptrdiff_t cindex = this->get_cindex(0, iyy, izz);
+ for (int cc = 0; cc < int(ncomp(fc)); cc++)
+ {
+ const double ampp = sqrt(
+ (*(bufferp + ncomp(fc)*iz+cc))[0]*(*(bufferp + ncomp(fc)*iz+cc))[0] +
+ (*(bufferp + ncomp(fc)*iz+cc))[1]*(*(bufferp + ncomp(fc)*iz+cc))[1]);
+ const double phip = atan2(
+ (*(bufferp + ncomp(fc)*iz+cc))[1],
+ (*(bufferp + ncomp(fc)*iz+cc))[0]);
+ const double ampm = sqrt(
+ (*(bufferm + ncomp(fc)*izz+cc))[0]*(*(bufferm + ncomp(fc)*izz+cc))[0] +
+ (*(bufferm + ncomp(fc)*izz+cc))[1]*(*(bufferm + ncomp(fc)*izz+cc))[1]);
+ const double phim = atan2(
+ (*(bufferm + ncomp(fc)*izz+cc))[1],
+ (*(bufferm + ncomp(fc)*izz+cc))[0]);
+ const double amp = sqrt(ampp*ampm);
+ const double phi = (phip - phim)/2;
+ (*(cdata + ncomp(fc)*cindex + cc))[0] = amp*cos(phi);
+ (*(cdata + ncomp(fc)*cindex + cc))[1] = -amp*sin(phi);
+ }
+ }
+ }
+ }// end omp parallel region
+ }// end if
+ }
+ //fftw_interface<rnumber>::free(buffer);
+ delete[] bufferp;
+ delete[] bufferm;
+ delete mpistatus;
+ // symmetrize kx = 0, ky = 0 line
+ if (this->clayout->myrank == this->clayout->rank[0][0])
+ {
+ for (ptrdiff_t iz = 1; iz < ptrdiff_t(this->clayout->sizes[1]/2); iz++)
+ {
+ ptrdiff_t cindex0 = this->get_cindex(0, 0, iz);
+ ptrdiff_t cindex1 = this->get_cindex(0, 0, this->clayout->sizes[1] - iz);
+ for (int cc = 0; cc < int(ncomp(fc)); cc++)
+ {
+ const double ampp = sqrt(
+ (*(cdata + cc + ncomp(fc)*cindex0))[0]*(*(cdata + cc + ncomp(fc)*cindex0))[0] +
+ (*(cdata + cc + ncomp(fc)*cindex0))[1]*(*(cdata + cc + ncomp(fc)*cindex0))[1]);
+ const double phip = atan2(
+ (*(cdata + cc + ncomp(fc)*cindex0))[1],
+ (*(cdata + cc + ncomp(fc)*cindex0))[0]);
+ const double ampm = sqrt(
+ (*(cdata + cc + ncomp(fc)*cindex1))[0]*(*(cdata + cc + ncomp(fc)*cindex1))[0] +
+ (*(cdata + cc + ncomp(fc)*cindex1))[1]*(*(cdata + cc + ncomp(fc)*cindex1))[1]);
+ const double phim = atan2(
+ (*(cdata + cc + ncomp(fc)*cindex1))[1],
+ (*(cdata + cc + ncomp(fc)*cindex1))[0]);
+ const double amp = sqrt(ampp*ampm);
+ const double phi = (phip - phim)/2;
+ (*(cdata + cc + ncomp(fc)*cindex0))[0] = amp*cos(phi);
+ (*(cdata + cc + ncomp(fc)*cindex0))[1] = amp*sin(phi);
+ (*(cdata + cc + ncomp(fc)*cindex1))[0] = amp*cos(phi);
+ (*(cdata + cc + ncomp(fc)*cindex1))[1] = -amp*sin(phi);
+ }
+ }
+ }
+ finish_mpi_profiling_zone(turtle_mpi_pcontrol::FIELD);
+ MPI_Barrier(this->clayout->comm); // TODO: figure out if this can be taken out by careful MPI tag generation
+}
+
+/** \brief Enforce Hermitian symmetry (fast and reasonable)
+ *
+ * TurTLE uses real-to-complex and complex-to-real FFTW transforms, because the
+ * equations are PDEs of real-valued fields.
+ * Hermitian symmetry means that Fourier-transformed real valued fields must
+ * respect the equation \f$\hat f(-\mathbf{k}) = {\hat f}^*(\mathbf{k})\f$.
+ *
+ * FFTW enforces this property mainly by only storing the positive half of the
+ * Fourier grid for the fastest array component. In TurTLE's case, this means
+ * \f$ k_x \f$.
+ * For the \f$ k_x = 0 \f$ plane, the symmetry must be enforced.
+ *
+ * This method uses an arithmetic mean of the \f$ (0, k_y, k_z)\f$ mode and
+ * the conjugate of the \f$(0, -k_y, -k_z) \f$ mode to generate the desired
+ * values. The method is fast (other than the required MPI communications).
+ *
+ * Note: the method is adequate either in cases where deviations from
+ * Hermitian symmetry are small, or in cases where deviations from correct
+ * physics is irrelevant.
+ * In practice: initial condition fields may be strongly perturbed by the
+ * application of this method, but they are unphysical anyway; during
+ * the quasistationary regime of some simulation, the method is applied
+ * regularly to all relevant fields, and deviations are expected to be small,
+ * i.e. effect on PDE approximations is negligible.
+ *
+ * */
+
+template <typename rnumber,
+ field_backend be,
+ field_components fc>
+void field<rnumber, be, fc>::symmetrize()
+{
+ TIMEZONE("field::symmetrize");
+ start_mpi_profiling_zone(turtle_mpi_pcontrol::FIELD);
+ assert(!this->real_space_representation);
+ typename fftw_interface<rnumber>::complex *cdata = this->get_cdata();
+
+ // make 0 mode real
+ if (this->myrank == this->clayout->rank[0][0])
+ {
+ for (ptrdiff_t cc = 0; cc < ncomp(fc); cc++)
+ cdata[cc][1] = 0.0;
+ }
+ // put kx = nx/2 modes to 0
+ for (ptrdiff_t iy = 0; iy < ptrdiff_t(this->clayout->subsizes[0]); iy++)
+ for (ptrdiff_t iz = 0; iz < ptrdiff_t(this->clayout->subsizes[1]); iz++)
+ {
+ ptrdiff_t cindex = this->get_cindex(this->clayout->sizes[2]-1, iy, iz);
+ for (int cc = 0; cc < int(ncomp(fc)); cc++) {
+ (*(cdata + cc + ncomp(fc)*cindex))[0] = 0.0;
+ (*(cdata + cc + ncomp(fc)*cindex))[1] = 0.0;
+ }
+ }
+ // put ky = ny/2 modes to 0
+ if (this->clayout->myrank == this->clayout->rank[0][this->clayout->sizes[0]/2])
+ {
+ for (ptrdiff_t iz = 0; iz < ptrdiff_t(this->clayout->subsizes[1]); iz++)
+ for (ptrdiff_t ix = 0; ix < ptrdiff_t(this->clayout->subsizes[2]); ix++)
+ {
+ ptrdiff_t cindex = this->get_cindex(ix, this->clayout->sizes[0]/2-this->clayout->starts[0], iz);
+ for (int cc = 0; cc < int(ncomp(fc)); cc++) {
+ (*(cdata + cc + ncomp(fc)*cindex))[0] = 0.0;
+ (*(cdata + cc + ncomp(fc)*cindex))[1] = 0.0;
+ }
+ }
+ }
+ // put kz = nz/2 modes to 0
+ for (ptrdiff_t iy = 0; iy < ptrdiff_t(this->clayout->subsizes[0]); iy++)
+ for (ptrdiff_t ix = 0; ix < ptrdiff_t(this->clayout->subsizes[2]); ix++)
+ {
+ ptrdiff_t cindex = this->get_cindex(ix, iy, this->clayout->sizes[1]/2);
+ for (int cc = 0; cc < int(ncomp(fc)); cc++) {
+ (*(cdata + cc + ncomp(fc)*cindex))[0] = 0.0;
+ (*(cdata + cc + ncomp(fc)*cindex))[1] = 0.0;
+ }
+ }
+
+ // symmetrize kx = 0 plane, line by line, for ky != 0
+ MPI_Status *mpistatus = new MPI_Status;
+ // bufferp will hold data from "plus", i.e. iy
+ // bufferm will hold data from "minus", i.e. ny - iy
+ typename fftw_interface<rnumber>::complex *bufferp = new typename fftw_interface<rnumber>::complex[ncomp(fc)*this->clayout->sizes[1]];
+ typename fftw_interface<rnumber>::complex *bufferm = new typename fftw_interface<rnumber>::complex[ncomp(fc)*this->clayout->sizes[1]];
+ int rankp, rankm;
+ // for each ky slice except ky=0
+ for (ptrdiff_t iy = 1; iy < ptrdiff_t(this->clayout->sizes[0]/2); iy++)
+ {
+ // read rank plus and rank minus
+ rankp = this->clayout->rank[0][iy];
+ rankm = this->clayout->rank[0][this->clayout->sizes[0] - iy];
+ // if my rank is plus or minus, then I should do actual work
+ if (this->clayout->myrank == rankp ||
+ this->clayout->myrank == rankm)
+ {
+ #pragma omp parallel
+ {
+ const ptrdiff_t zstart = ptrdiff_t(OmpUtils::ForIntervalStart(this->clayout->sizes[1]));
+ const ptrdiff_t zend = ptrdiff_t(OmpUtils::ForIntervalEnd(this->clayout->sizes[1]));
+ // if my rank is rank plus, I should fill out the plus buffer
+ if (this->clayout->myrank == rankp)
+ {
+ ptrdiff_t iyy = iy - this->clayout->starts[0];
+ for (ptrdiff_t iz = zstart; iz < zend; iz++)
+ {
+ ptrdiff_t cindex = this->get_cindex(0, iyy, iz);
+ for (int cc = 0; cc < int(ncomp(fc)); cc++)
+ for (int imag_comp=0; imag_comp<2; imag_comp++)
+ (*(bufferp + ncomp(fc)*iz+cc))[imag_comp] =
+ (*(cdata + ncomp(fc)*cindex + cc))[imag_comp];
+ }
+ }
+ // if my rank is rank minus, I should fill out the minus buffer
+ if (this->clayout->myrank == rankm)
+ {
+ ptrdiff_t iyy = (this->clayout->sizes[0] - iy) - this->clayout->starts[0];
+ for (ptrdiff_t iz = zstart; iz < zend; iz++)
+ {
+ ptrdiff_t cindex = this->get_cindex(0, iyy, iz);
+ for (int cc = 0; cc < int(ncomp(fc)); cc++)
+ for (int imag_comp=0; imag_comp<2; imag_comp++)
+ (*(bufferm + ncomp(fc)*iz+cc))[imag_comp] =
+ (*(cdata + ncomp(fc)*cindex + cc))[imag_comp];
+ }
+ }
+ // after filling out buffers, synchronize threads
+ #pragma omp barrier
+
+ // if ranks are different, send and receive
+ if (rankp != rankm)
+ {
+ // if my rank is rank plus, send buffer plus, receive buffer minus
+ if (this->clayout->myrank == rankp && omp_get_thread_num() == 0)
+ {
+ MPI_Send((void*)bufferp,
+ ncomp(fc)*this->clayout->sizes[1],
+ mpi_real_type<rnumber>::complex(),
+ rankm, 2*iy+0,
+ this->clayout->comm);
+ MPI_Recv((void*)bufferm,
+ ncomp(fc)*this->clayout->sizes[1],
+ mpi_real_type<rnumber>::complex(),
+ rankm, 2*iy+1,
+ this->clayout->comm,
+ mpistatus);
+ }
+ // if my rank is rank minus, receive buffer plus, send buffer minus
+ if (this->clayout->myrank == rankm && omp_get_thread_num() == 0)
+ {
+ MPI_Recv((void*)bufferp,
+ ncomp(fc)*this->clayout->sizes[1],
+ mpi_real_type<rnumber>::complex(),
+ rankp, 2*iy+0,
+ this->clayout->comm,
+ mpistatus);
+ MPI_Send((void*)bufferm,
+ ncomp(fc)*this->clayout->sizes[1],
+ mpi_real_type<rnumber>::complex(),
+ rankp, 2*iy+1,
+ this->clayout->comm);
+ }
+ }
+ // ensure buffers are updated by MPI transfer before threads
+ // start working again
+ #pragma omp barrier
+
+ // if I my rank is either plus or minus, I should update my slice
+ if (this->clayout->myrank == rankp)
+ {
+ ptrdiff_t iyy = iy - this->clayout->starts[0];
+ for (ptrdiff_t iz = zstart; iz < zend; iz++)
+ {
+ // modulo operation makes sense only for slab decomposition
+ ptrdiff_t izz = (this->clayout->sizes[1] - iz) % this->clayout->sizes[1];
+ ptrdiff_t cindex = this->get_cindex(0, iyy, iz);
+ for (int cc = 0; cc < int(ncomp(fc)); cc++)
+ {
+ (*(cdata + ncomp(fc)*cindex + cc))[0] = ((*(bufferp + ncomp(fc)*iz+cc))[0] + (*(bufferm + ncomp(fc)*izz+cc))[0])/2;
+ (*(cdata + ncomp(fc)*cindex + cc))[1] = ((*(bufferp + ncomp(fc)*iz+cc))[1] - (*(bufferm + ncomp(fc)*izz+cc))[1])/2;
+ }
+ }
+ }
+ // if I my rank is either plus or minus, I should update my slice
+ if (this->clayout->myrank == rankm)
+ {
+ ptrdiff_t iyy = (this->clayout->sizes[0] - iy) - this->clayout->starts[0];
+ for (ptrdiff_t iz = zstart; iz < zend; iz++)
+ {
+ // modulo operation makes sense only for slab decomposition
+ ptrdiff_t izz = (this->clayout->sizes[1] - iz) % this->clayout->sizes[1];
+ ptrdiff_t cindex = this->get_cindex(0, iyy, izz);
+ for (int cc = 0; cc < int(ncomp(fc)); cc++)
+ {
+ (*(cdata + ncomp(fc)*cindex + cc))[0] = ((*(bufferp + ncomp(fc)*iz+cc))[0] + (*(bufferm + ncomp(fc)*izz+cc))[0])/2;
+ (*(cdata + ncomp(fc)*cindex + cc))[1] = -((*(bufferp + ncomp(fc)*iz+cc))[1] - (*(bufferm + ncomp(fc)*izz+cc))[1])/2;
+ }
+ }
+ }
+ }// end omp parallel region
+ }// end if
+ }
+ //fftw_interface<rnumber>::free(buffer);
+ delete[] bufferp;
+ delete[] bufferm;
+ delete mpistatus;
+ // symmetrize kx = 0, ky = 0 line
+ if (this->clayout->myrank == this->clayout->rank[0][0])
+ {
+ for (ptrdiff_t iz = 1; iz < ptrdiff_t(this->clayout->sizes[1]/2); iz++)
+ {
+ ptrdiff_t cindex0 = this->get_cindex(0, 0, iz);
+ ptrdiff_t cindex1 = this->get_cindex(0, 0, this->clayout->sizes[1] - iz);
+ for (int cc = 0; cc < int(ncomp(fc)); cc++)
+ {
+ double re = ((*(cdata + cc + ncomp(fc)*cindex0))[0] + (*(cdata + cc + ncomp(fc)*cindex1))[0])/2;
+ double im = ((*(cdata + cc + ncomp(fc)*cindex0))[1] - (*(cdata + cc + ncomp(fc)*cindex1))[1])/2;
+ (*(cdata + cc + ncomp(fc)*cindex0))[0] = re;
+ (*(cdata + cc + ncomp(fc)*cindex0))[1] = im;
+ (*(cdata + cc + ncomp(fc)*cindex1))[0] = re;
+ (*(cdata + cc + ncomp(fc)*cindex1))[1] = -im;
+ }
+ }
+ }
+ finish_mpi_profiling_zone(turtle_mpi_pcontrol::FIELD);
+}
+
+template <typename rnumber,
+ field_backend be,
+ field_components fc>
+template <kspace_dealias_type dt>
+void field<rnumber, be, fc>::compute_stats(
+ kspace<be, dt> *kk,
+ const hid_t group,
+ const std::string dset_name,
+ const hsize_t toffset,
+ const double max_estimate)
+{
+ TIMEZONE("field::compute_stats");
+ std::vector<double> max_estimate_vector;
+ bool did_rspace = false;
+ switch(fc)
+ {
+ case ONE:
+ max_estimate_vector.resize(1, max_estimate);
+ break;
+ case THREE:
+ max_estimate_vector.resize(4, max_estimate);
+ max_estimate_vector[3] *= sqrt(3);
+ break;
+ case THREExTHREE:
+ max_estimate_vector.resize(9, max_estimate);
+ break;
+ }
+ if (this->real_space_representation)
+ {
+ TIMEZONE("field::compute_stats::compute_rspace_stats");
+ this->compute_rspace_stats(
+ group,
+ dset_name,
+ toffset,
+ max_estimate_vector);
+ did_rspace = true;
+ this->dft();
+ // normalize
+ TIMEZONE("field::normalize");
+ for (hsize_t tmp_index=0; tmp_index<this->rmemlayout->local_size; tmp_index++)
+ this->data[tmp_index] /= this->npoints;
+ }
+ // what follows gave me a headache until I found this link:
+ // http://stackoverflow.com/questions/8256636/expected-primary-expression-error-on-template-method-using
+ kk->template cospectrum<rnumber, fc>(
+ (typename fftw_interface<rnumber>::complex*)this->data,
+ group,
+ dset_name + "_" + dset_name,
+ toffset);
+ if (!did_rspace)
+ {
+ this->ift();
+ // normalization not required
+ this->compute_rspace_stats(
+ group,
+ dset_name,
+ toffset,
+ max_estimate_vector);
+ }
+}
+
+template <typename rnumber,
+ field_backend be,
+ field_components fc>
+template <kspace_dealias_type dt>
+double field<rnumber, be, fc>::L2norm(
+ kspace<be, dt> *kk)
+{
+ TIMEZONE("field::L2norm");
+ if (!this->real_space_representation)
+ return kk->template L2norm<rnumber, fc>(this->get_cdata());
+ else
+ {
+ shared_array<double> local_m2_threaded(1, [&](double* local_moment){
+ std::fill_n(local_moment, 1, 0);});
+
+ this->RLOOP(
+ [&](const ptrdiff_t rindex,
+ const ptrdiff_t xindex,
+ const ptrdiff_t yindex,
+ const ptrdiff_t zindex){
+ double *local_m2 = local_m2_threaded.getMine();
+ for (unsigned int i=0; i<ncomp(fc); i++)
+ local_m2[0] += this->data[rindex*ncomp(fc)+i]*this->data[rindex*ncomp(fc)+i];
+ });
+
+ local_m2_threaded.mergeParallel();
+ double m2;
+ MPI_Allreduce(
+ (void*)local_m2_threaded.getMasterData(),
+ &m2,
+ 1,
+ MPI_DOUBLE, MPI_SUM, this->comm);
+ return sqrt(m2 / this->npoints);
+ }
+}
+
+/** \brief Compute field gradient
+ *
+ * Scalar fields are turned into vector fields
+ *
+ * Vector fields are turned into tensor fields, with
+ * - fastest dimension corresponding to vector component
+ * - second fastest dimension corresponding to derivative direction
+ * i.e. values should be accessed with something like
+ * val(index, nabla_component, vector_component)
+ * (either cval or rval).
+ * */
+
+template <typename rnumber,
+ field_backend be,
+ field_components fc1,
+ field_components fc2,
+ kspace_dealias_type dt>
+int compute_gradient(
+ kspace<be, dt> *kk,
+ field<rnumber, be, fc1> *src,
+ field<rnumber, be, fc2> *dst)
+{
+ TIMEZONE("compute_gradient");
+ assert(!src->real_space_representation);
+ assert((fc1 == ONE && fc2 == THREE) ||
+ (fc1 == THREE && fc2 == THREExTHREE));
+ *dst = 0.0;
+ dst->real_space_representation = false;
+ switch(fc1)
+ {
+ case ONE:
+ kk->CLOOP_K2(
+ [&](const ptrdiff_t cindex,
+ const ptrdiff_t xindex,
+ const ptrdiff_t yindex,
+ const ptrdiff_t zindex,
+ const double k2){
+ if (k2 < kk->kM2)
+ {
+ dst->cval(cindex, 0, 0) = -kk->kx[xindex]*src->cval(cindex, 1);
+ dst->cval(cindex, 0, 1) = kk->kx[xindex]*src->cval(cindex, 0);
+ dst->cval(cindex, 1, 0) = -kk->ky[yindex]*src->cval(cindex, 1);
+ dst->cval(cindex, 1, 1) = kk->ky[yindex]*src->cval(cindex, 0);
+ dst->cval(cindex, 2, 0) = -kk->kz[zindex]*src->cval(cindex, 1);
+ dst->cval(cindex, 2, 1) = kk->kz[zindex]*src->cval(cindex, 0);
+ }});
+ break;
+ case THREE:
+ kk->CLOOP_K2(
+ [&](const ptrdiff_t cindex,
+ const ptrdiff_t xindex,
+ const ptrdiff_t yindex,
+ const ptrdiff_t zindex,
+ const double k2){
+ if (k2 < kk->kM2)
+ {
+ for (unsigned int field_component = 0;
+ field_component < ncomp(fc1);
+ field_component++)
+ {
+ dst->cval(cindex, 0, field_component, 0) = -kk->kx[xindex]*src->cval(cindex, field_component, 1);
+ dst->cval(cindex, 0, field_component, 1) = kk->kx[xindex]*src->cval(cindex, field_component, 0);
+ dst->cval(cindex, 1, field_component, 0) = -kk->ky[yindex]*src->cval(cindex, field_component, 1);
+ dst->cval(cindex, 1, field_component, 1) = kk->ky[yindex]*src->cval(cindex, field_component, 0);
+ dst->cval(cindex, 2, field_component, 0) = -kk->kz[zindex]*src->cval(cindex, field_component, 1);
+ dst->cval(cindex, 2, field_component, 1) = kk->kz[zindex]*src->cval(cindex, field_component, 0);
+ }
+ }});
+ break;
+ }
+ return EXIT_SUCCESS;
+}
+
+template <typename rnumber,
+ field_backend be,
+ kspace_dealias_type dt>
+int compute_divergence(
+ kspace<be, dt> *kk,
+ field<rnumber, be, THREE> *src,
+ field<rnumber, be, ONE> *dst)
+{
+ TIMEZONE("compute_divergence");
+ assert(!src->real_space_representation);
+ *dst = 0.0;
+ dst->real_space_representation = false;
+ kk->CLOOP_K2(
+ [&](const ptrdiff_t cindex,
+ const ptrdiff_t xindex,
+ const ptrdiff_t yindex,
+ const ptrdiff_t zindex,
+ const double k2){
+ dst->cval(cindex, 0) = -(kk->kx[xindex]*src->cval(cindex, 0, 1) +
+ kk->ky[yindex]*src->cval(cindex, 1, 1) +
+ kk->kz[zindex]*src->cval(cindex, 2, 1));
+ dst->cval(cindex, 1) = (kk->kx[xindex]*src->cval(cindex, 0, 0) +
+ kk->ky[yindex]*src->cval(cindex, 1, 0) +
+ kk->kz[zindex]*src->cval(cindex, 2, 0));
+ });
+ return EXIT_SUCCESS;
+}
+
+template <typename rnumber,
+ field_backend be,
+ kspace_dealias_type dt>
+int compute_curl(
+ kspace<be, dt> *kk,
+ field<rnumber, be, THREE> *src,
+ field<rnumber, be, THREE> *dst)
+{
+ TIMEZONE("compute_curl");
+ assert(!src->real_space_representation);
+ *dst = 0.0;
+ dst->real_space_representation = false;
+ kk->CLOOP_K2(
+ [&](const ptrdiff_t cindex,
+ const ptrdiff_t xindex,
+ const ptrdiff_t yindex,
+ const ptrdiff_t zindex,
+ const double k2){
+ if (k2 <= kk->kM2)
+ {
+ dst->cval(cindex,0,0) = -(kk->ky[yindex]*src->cval(cindex,2,1) - kk->kz[zindex]*src->cval(cindex,1,1));
+ dst->cval(cindex,0,1) = (kk->ky[yindex]*src->cval(cindex,2,0) - kk->kz[zindex]*src->cval(cindex,1,0));
+ dst->cval(cindex,1,0) = -(kk->kz[zindex]*src->cval(cindex,0,1) - kk->kx[xindex]*src->cval(cindex,2,1));
+ dst->cval(cindex,1,1) = (kk->kz[zindex]*src->cval(cindex,0,0) - kk->kx[xindex]*src->cval(cindex,2,0));
+ dst->cval(cindex,2,0) = -(kk->kx[xindex]*src->cval(cindex,1,1) - kk->ky[yindex]*src->cval(cindex,0,1));
+ dst->cval(cindex,2,1) = (kk->kx[xindex]*src->cval(cindex,1,0) - kk->ky[yindex]*src->cval(cindex,0,0));
+ }
+ else
+ std::fill_n((rnumber*)(dst->get_cdata()+3*cindex), 6, 0.0);
+ }
+ );
+ if (kk->layout->myrank == 0)
+ std::fill_n((rnumber*)(dst->get_cdata()), 6, 0.0);
+ dst->symmetrize();
+ return EXIT_SUCCESS;
+}
+
+template <typename rnumber,
+ field_backend be,
+ kspace_dealias_type dt>
+int invert_curl(
+ kspace<be, dt> *kk,
+ field<rnumber, be, THREE> *src,
+ field<rnumber, be, THREE> *dst)
+{
+ TIMEZONE("invert_curl");
+ assert(!src->real_space_representation);
+ *dst = 0.0;
+ dst->real_space_representation = false;
+ kk->CLOOP_K2(
+ [&](const ptrdiff_t cindex,
+ const ptrdiff_t xindex,
+ const ptrdiff_t yindex,
+ const ptrdiff_t zindex,
+ const double k2){
+ const double kk2 = (k2 > 0) ? k2 : 1.0;
+ if (k2 <= kk->kM2)
+ {
+ dst->cval(cindex,0,0) = -(kk->ky[yindex]*src->cval(cindex,2,1) - kk->kz[zindex]*src->cval(cindex,1,1)) / kk2;
+ dst->cval(cindex,0,1) = (kk->ky[yindex]*src->cval(cindex,2,0) - kk->kz[zindex]*src->cval(cindex,1,0)) / kk2;
+ dst->cval(cindex,1,0) = -(kk->kz[zindex]*src->cval(cindex,0,1) - kk->kx[xindex]*src->cval(cindex,2,1)) / kk2;
+ dst->cval(cindex,1,1) = (kk->kz[zindex]*src->cval(cindex,0,0) - kk->kx[xindex]*src->cval(cindex,2,0)) / kk2;
+ dst->cval(cindex,2,0) = -(kk->kx[xindex]*src->cval(cindex,1,1) - kk->ky[yindex]*src->cval(cindex,0,1)) / kk2;
+ dst->cval(cindex,2,1) = (kk->kx[xindex]*src->cval(cindex,1,0) - kk->ky[yindex]*src->cval(cindex,0,0)) / kk2;
+ }
+ else
+ std::fill_n((rnumber*)(dst->get_cdata()+3*cindex), 6, 0.0);
+ }
+ );
+ if (kk->layout->myrank == 0)
+ std::fill_n((rnumber*)(dst->get_cdata()), 6, 0.0);
+ dst->symmetrize();
+ return EXIT_SUCCESS;
+}
+
+template <typename rnumber,
+ field_backend be,
+ field_components fc>
+int joint_rspace_PDF(
+ field<rnumber, be, fc> *f1,
+ field<rnumber, be, fc> *f2,
+ const hid_t group,
+ const std::string dset_name,
+ const hsize_t toffset,
+ const std::vector<double> max_f1_estimate,
+ const std::vector<double> max_f2_estimate)
+{
+ TIMEZONE("joint_rspace_PDF");
+ assert(f1->real_space_representation);
+ assert(f2->real_space_representation);
+ if (fc == THREE)
+ {
+ assert(max_f1_estimate.size() == 4);
+ assert(max_f2_estimate.size() == 4);
+ }
+ else if (fc == ONE)
+ {
+ assert(max_f1_estimate.size() == 1);
+ assert(max_f2_estimate.size() == 1);
+ }
+ int nbins;
+ std::string dsetc, dsetm;
+ dsetc = "histograms/" + dset_name + "_components";
+ if (fc == THREE)
+ dsetm = "histograms/" + dset_name + "_magnitudes";
+ else
+ dsetm = "histograms/" + dset_name;
+ if (f1->myrank == 0)
+ {
+ hid_t dset, wspace;
+ hsize_t dims[5];
+ int ndims;
+ variable_used_only_in_assert(ndims);
+ if (fc == THREE)
+ {
+ dset = H5Dopen(
+ group,
+ dsetc.c_str(),
+ H5P_DEFAULT);
+ wspace = H5Dget_space(dset);
+ ndims = H5Sget_simple_extent_dims(wspace, dims, NULL);
+ assert(ndims == 5);
+ assert(dims[3] == 3);
+ assert(dims[4] == 3);
+ H5Sclose(wspace);
+ H5Dclose(dset);
+ }
+ dset = H5Dopen(
+ group,
+ dsetm.c_str(),
+ H5P_DEFAULT);
+ wspace = H5Dget_space(dset);
+ ndims = H5Sget_simple_extent_dims(wspace, dims, NULL);
+ assert(ndims == 3);
+ H5Sclose(wspace);
+ H5Dclose(dset);
+ nbins = dims[1];
+ }
+ {
+ TIMEZONE("MPI_Bcast");
+ MPI_Bcast(&nbins, 1, MPI_INT, 0, f1->comm);
+ }
+
+ /// histogram components
+ shared_array<ptrdiff_t> local_histc_threaded(
+ nbins*nbins*9,
+ [&](ptrdiff_t* local_hist){
+ std::fill_n(local_hist, nbins*nbins*9, 0);
+ });
+
+ /// histogram magnitudes
+ shared_array<ptrdiff_t> local_histm_threaded(
+ nbins*nbins,
+ [&](ptrdiff_t* local_hist){
+ std::fill_n(local_hist, nbins*nbins, 0);
+ });
+
+ /// set up bin sizes
+ std::vector<double> bin1size, bin2size;
+ bin1size.resize(4);
+ bin2size.resize(4);
+ if (fc == THREE)
+ {
+ for (unsigned int i=0; i<3; i++)
+ {
+ bin1size[i] = 2*max_f1_estimate[i] / nbins;
+ bin2size[i] = 2*max_f2_estimate[i] / nbins;
+ }
+ bin1size[3] = max_f1_estimate[3] / nbins;
+ bin2size[3] = max_f2_estimate[3] / nbins;
+ }
+ else if (fc == ONE)
+ {
+ for (unsigned int i=0; i<4; i++)
+ {
+ bin1size[i] = 2*max_f1_estimate[0] / nbins;
+ bin2size[i] = 2*max_f2_estimate[0] / nbins;
+ }
+ }
+
+
+ {
+ TIMEZONE("field::RLOOP");
+ f1->RLOOP(
+ [&](const ptrdiff_t rindex,
+ const ptrdiff_t xindex,
+ const ptrdiff_t yindex,
+ const ptrdiff_t zindex){
+ ptrdiff_t *local_histm = local_histm_threaded.getMine();
+ int bin1 = 0;
+ int bin2 = 0;
+ if (fc == THREE)
+ {
+ ptrdiff_t *local_histc = local_histc_threaded.getMine();
+
+ double mag1, mag2;
+ mag1 = 0.0;
+ mag2 = 0.0;
+ for (unsigned int i=0; i<3; i++)
+ {
+ double val1 = f1->rval(rindex, i);
+ mag1 += val1*val1;
+ int bin1 = int(floor((val1 + max_f1_estimate[i])/bin1size[i]));
+ mag2 = 0.0;
+ for (unsigned int j=0; j<3; j++)
+ {
+ double val2 = f2->rval(rindex, j);
+ mag2 += val2*val2;
+ int bin2 = int(floor((val2 + max_f2_estimate[j])/bin2size[j]));
+ if ((bin1 >= 0 && bin1 < nbins) &&
+ (bin2 >= 0 && bin2 < nbins))
+ local_histc[(bin1*nbins + bin2)*9 + i*3 + j]++;
+ }
+ }
+ bin1 = int(floor(sqrt(mag1)/bin1size[3]));
+ bin2 = int(floor(sqrt(mag2)/bin2size[3]));
+ }
+ else if (fc == ONE)
+ {
+ bin1 = int(floor((f1->rval(rindex) + max_f1_estimate[0])/bin1size[3]));
+ bin2 = int(floor((f2->rval(rindex) + max_f2_estimate[0])/bin2size[3]));
+ }
+ if ((bin1 >= 0 && bin1 < nbins) &&
+ (bin2 >= 0 && bin2 < nbins))
+ local_histm[bin1*nbins + bin2]++;
+ });
+ }
+ local_histm_threaded.mergeParallel();
+ ptrdiff_t *histm = new ptrdiff_t[nbins*nbins];
+ ptrdiff_t *histc = NULL;
+ if (fc == THREE)
+ {
+ local_histc_threaded.mergeParallel();
+ histc = new ptrdiff_t[nbins*nbins*9];
+ }
+ {
+ MPI_Allreduce(
+ (void*)local_histm_threaded.getMasterData(),
+ (void*)histm,
+ nbins*nbins,
+ MPI_INT64_T, MPI_SUM, f1->comm);
+ if (fc == THREE)
+ MPI_Allreduce(
+ (void*)local_histc_threaded.getMasterData(),
+ (void*)histc,
+ nbins*nbins*9,
+ MPI_INT64_T, MPI_SUM, f1->comm);
+ }
+
+ if (f1->myrank == 0)
+ {
+ TIMEZONE("root-work");
+ hid_t dset, wspace, mspace;
+ hsize_t count[5], offset[5];
+ if (fc == THREE)
+ {
+ dset = H5Dopen(group, dsetc.c_str(), H5P_DEFAULT);
+ assert(dset > 0);
+ wspace = H5Dget_space(dset);
+ offset[0] = toffset;
+ offset[1] = 0;
+ offset[2] = 0;
+ offset[3] = 0;
+ offset[4] = 0;
+ count[0] = 1;
+ count[1] = nbins;
+ count[2] = nbins;
+ count[3] = 3;
+ count[4] = 3;
+ mspace = H5Screate_simple(5, count, NULL);
+ H5Sselect_hyperslab(wspace, H5S_SELECT_SET, offset, NULL, count, NULL);
+ H5Dwrite(dset, H5T_NATIVE_INT64, mspace, wspace, H5P_DEFAULT, histc);
+ H5Sclose(wspace);
+ H5Sclose(mspace);
+ H5Dclose(dset);
+ }
+ dset = H5Dopen(group, dsetm.c_str(), H5P_DEFAULT);
+ assert(dset > 0);
+ offset[0] = toffset;
+ offset[1] = 0;
+ offset[2] = 0;
+ count[0] = 1;
+ count[1] = nbins;
+ count[2] = nbins;
+ mspace = H5Screate_simple(3, count, NULL);
+ wspace = H5Dget_space(dset);
+ H5Sselect_hyperslab(wspace, H5S_SELECT_SET, offset, NULL, count, NULL);
+ H5Dwrite(dset, H5T_NATIVE_INT64, mspace, wspace, H5P_DEFAULT, histm);
+ H5Sclose(wspace);
+ H5Sclose(mspace);
+ H5Dclose(dset);
+ }
+
+ delete[] histm;
+ if (fc == THREE)
+ delete[] histc;
+
+ return EXIT_SUCCESS;
+}
+
+// Debarghya edit for 3 scale PDFs //
+
+template <typename rnumber,
+ field_backend be>
+int joint_rspace_3PDF(
+ field<rnumber, be, ONE> *f1,
+ field<rnumber, be, ONE> *f2,
+ field<rnumber, be, ONE> *f3,
+ const hid_t group,
+ const std::string dset_name,
+ const hsize_t toffset,
+ const std::vector<double> max_f1_estimate,
+ const std::vector<double> max_f2_estimate,
+ const std::vector<double> max_f3_estimate)
+{
+ TIMEZONE("joint_rspace_3PDF");
+ assert(f1->real_space_representation);
+ assert(f2->real_space_representation);
+ assert(f3->real_space_representation);
+
+ assert(max_f1_estimate.size() == 1);
+ assert(max_f2_estimate.size() == 1);
+ assert(max_f3_estimate.size() == 1);
+
+ int nbins;
+ std::string dsetc, dsetm;
+ dsetc = "histograms/" + dset_name + "_components";
+ dsetm = "histograms/" + dset_name;
+ if (f1->myrank == 0)
+ {
+ hid_t dset, wspace;
+ hsize_t dims[5];
+ int ndims;
+ dset = H5Dopen(
+ group,
+ dsetm.c_str(),
+ H5P_DEFAULT);
+ wspace = H5Dget_space(dset);
+ ndims = H5Sget_simple_extent_dims(wspace, dims, NULL);
+ variable_used_only_in_assert(ndims);
+ assert(ndims == 4);
+ H5Sclose(wspace);
+ H5Dclose(dset);
+ nbins = dims[1];
+ }
+ {
+ TIMEZONE("MPI_Bcast");
+ MPI_Bcast(&nbins, 1, MPI_INT, 0, f1->comm);
+ }
+
+
+ /// histogram magnitudes
+ shared_array<ptrdiff_t> local_histm_threaded(
+ nbins*nbins*nbins,
+ [&](ptrdiff_t* local_hist){
+ std::fill_n(local_hist, nbins*nbins*nbins, 0);
+ });
+
+ /// set up bin sizes
+ std::vector<double> bin1size, bin2size, bin3size;
+ bin1size.resize(1);
+ bin2size.resize(1);
+ bin3size.resize(1);
+
+ bin1size[0] = 2*max_f1_estimate[0] / nbins;
+ bin2size[0] = 2*max_f2_estimate[0] / nbins;
+ bin3size[0] = 2*max_f3_estimate[0] / nbins;
+
+
+ {
+ TIMEZONE("field::RLOOP");
+ f1->RLOOP(
+ [&](const ptrdiff_t rindex,
+ const ptrdiff_t xindex,
+ const ptrdiff_t yindex,
+ const ptrdiff_t zindex){
+ ptrdiff_t *local_histm = local_histm_threaded.getMine();
+ int bin1 = 0;
+ int bin2 = 0;
+ int bin3 = 0;
+
+ bin1 = int(floor((f1->rval(rindex) + max_f1_estimate[0])/bin1size[0]));
+ bin2 = int(floor((f2->rval(rindex) + max_f2_estimate[0])/bin2size[0]));
+ bin3 = int(floor((f3->rval(rindex) + max_f3_estimate[0])/bin3size[0]));
+ if ((bin1 >= 0 && bin1 < nbins) &&
+ (bin2 >= 0 && bin2 < nbins) &&
+ (bin3 >= 0 && bin3 < nbins))
+ local_histm[bin1*nbins*nbins + bin2*nbins + bin3]++;
+ });
+ }
+ local_histm_threaded.mergeParallel();
+ ptrdiff_t *histm = new ptrdiff_t[nbins*nbins*nbins];
+ {
+ MPI_Allreduce(
+ (void*)local_histm_threaded.getMasterData(),
+ (void*)histm,
+ nbins*nbins*nbins,
+ MPI_INT64_T, MPI_SUM, f1->comm);
+ }
+
+ if (f1->myrank == 0)
+ {
+ TIMEZONE("root-work");
+ hid_t dset, wspace, mspace;
+ hsize_t count[5], offset[5];
+
+ dset = H5Dopen(group, dsetm.c_str(), H5P_DEFAULT);
+ assert(dset > 0);
+ offset[0] = toffset;
+ offset[1] = 0;
+ offset[2] = 0;
+ offset[3] = 0;
+ count[0] = 1;
+ count[1] = nbins;
+ count[2] = nbins;
+ count[3] = nbins;
+ mspace = H5Screate_simple(4, count, NULL);
+ wspace = H5Dget_space(dset);
+ H5Sselect_hyperslab(wspace, H5S_SELECT_SET, offset, NULL, count, NULL);
+ H5Dwrite(dset, H5T_NATIVE_INT64, mspace, wspace, H5P_DEFAULT, histm);
+ H5Sclose(wspace);
+ H5Sclose(mspace);
+ H5Dclose(dset);
+ }
+
+ delete[] histm;
+
+ return EXIT_SUCCESS;
+}
+
+
+template <typename rnumber,
+ field_backend be,
+ field_components fc>
+field<rnumber, be, fc>& field<rnumber, be, fc>::operator=(const typename fftw_interface<rnumber>::complex *__restrict__ source)
+{
+ // use CLOOP pattern, because we want the array to be arranged in memory
+ // for optimal access by FFTW
+ #pragma omp parallel
+ {
+ const hsize_t start = OmpUtils::ForIntervalStart(this->clayout->subsizes[1]);
+ const hsize_t end = OmpUtils::ForIntervalEnd(this->clayout->subsizes[1]);
+
+ for (hsize_t yindex = 0; yindex < this->clayout->subsizes[0]; yindex++){
+ for (hsize_t zindex = start; zindex < end; zindex++){
+ const ptrdiff_t cindex = (
+ yindex*this->clayout->subsizes[1]*this->clayout->subsizes[2] +
+ zindex*this->clayout->subsizes[2]);
+ std::copy((rnumber*)(source + cindex*ncomp(fc)),
+ (rnumber*)(source + (cindex+this->clayout->subsizes[2])*ncomp(fc)),
+ this->data+(cindex*ncomp(fc))*2);
+ }
+ }
+ }
+ this->real_space_representation = false;
+ return *this;
+}
+
+
+template <typename rnumber,
+ field_backend be,
+ field_components fc>
+field<rnumber, be, fc>& field<rnumber, be, fc>::operator=(const rnumber *__restrict__ source)
+{
+ TIMEZONE("field::operator=(const rnumber *source)");
+ // use RLOOP, such that memory caching per thread stuff is not messed up
+ this->RLOOP(
+ [&](const ptrdiff_t rindex,
+ const ptrdiff_t xindex,
+ const ptrdiff_t yindex,
+ const ptrdiff_t zindex)
+ {
+ std::copy(source + rindex*ncomp(fc),
+ source + (rindex+1)*ncomp(fc),
+ this->data + rindex*ncomp(fc));
+ });
+ this->real_space_representation = true;
+ return *this;
+}
+
+template <typename rnumber,
+ field_backend be,
+ field_components fc>
+field<rnumber, be, fc> &field<rnumber, be, fc>::operator=(
+ const rnumber value)
+{
+ TIMEZONE("field::operator=(const rnumber value)");
+ if (this->real_space_representation)
+ // use RLOOP, such that memory caching per thread stuff is not messed up
+ this->RLOOP(
+ [&](const ptrdiff_t rindex,
+ const ptrdiff_t xindex,
+ const ptrdiff_t yindex,
+ const ptrdiff_t zindex)
+ {
+ std::fill_n(this->data + rindex*ncomp(fc),
+ ncomp(fc),
+ value);
+ });
+ else
+ {
+ // use CLOOP, such that memory caching per thread stuff is not messed up
+ #pragma omp parallel
+ {
+ hsize_t npoints = 0;
+ const hsize_t start = OmpUtils::ForIntervalStart(this->clayout->subsizes[1]);
+ const hsize_t end = OmpUtils::ForIntervalEnd(this->clayout->subsizes[1]);
+
+ for (hsize_t yindex = 0; yindex < this->clayout->subsizes[0]; yindex++){
+ for (hsize_t zindex = start; zindex < end; zindex++){
+ const ptrdiff_t cindex = (
+ yindex*this->clayout->subsizes[1]*this->clayout->subsizes[2] +
+ zindex*this->clayout->subsizes[2]);
+ for (hsize_t xindex = 0; xindex < this->clayout->subsizes[2]; xindex++)
+ {
+ npoints++;
+ for (unsigned int cc=0; cc < ncomp(fc); cc++)
+ {
+ *(this->data + 2*((cindex+xindex)*ncomp(fc) + cc)) = value;
+ *(this->data + 2*((cindex+xindex)*ncomp(fc) + cc)+1) = 0.0;
+ }
+ }
+ }
+ }
+ }
+ }
+ return *this;
+}
+
+
+
+template <typename rnumber,
+ field_backend be,
+ field_components fc>
+field<rnumber, be, fc> &field<rnumber, be, fc>::operator=(
+ const field<rnumber, be, fc> &src)
+{
+ TIMEZONE("field::operator=(const field<rnumber, be, fc> &src)");
+ if (src.real_space_representation)
+ {
+ assert(this->get_nx() == src.get_nx());
+ assert(this->get_ny() == src.get_ny());
+ assert(this->get_nz() == src.get_nz());
+ this->operator=(src.data);
+ }
+ else
+ {
+ // simple copy
+ if (this->get_nx() == src.get_nx() &&
+ this->get_ny() == src.get_ny() &&
+ this->get_nz() == src.get_nz())
+ {
+ this->operator=(src.get_cdata());
+ }
+ // complicated resize
+ else
+ {
+ this->real_space_representation = false;
+ int64_t slice_size = src.clayout->local_size / src.clayout->subsizes[0];
+ // clean up
+ *this = 0.0;
+ typename fftw_interface<rnumber>::complex *buffer;
+ buffer = fftw_interface<rnumber>::alloc_complex(slice_size*ncomp(fc));
+
+ int min_fast_dim =
+ (src.clayout->sizes[2] > this->clayout->sizes[2]) ?
+ this->clayout->sizes[2] : src.clayout->sizes[2];
+
+ int64_t ii0, ii1;
+ int64_t oi0, oi1;
+ int64_t delta1, delta0;
+ int irank, orank;
+ delta0 = (this->clayout->sizes[0] - src.clayout->sizes[0]);
+ delta1 = (this->clayout->sizes[1] - src.clayout->sizes[1]);
+ for (ii0=0; ii0 < int64_t(src.clayout->sizes[0]); ii0++)
+ {
+ if (ii0 <= int64_t(src.clayout->sizes[0]/2))
+ {
+ oi0 = ii0;
+ if (oi0 > int64_t(this->clayout->sizes[0]/2))
+ continue;
+ }
+ else
+ {
+ oi0 = ii0 + delta0;
+ if ((oi0 < 0) || ((int64_t(this->clayout->sizes[0]) - oi0) >= int64_t(this->clayout->sizes[0]/2)))
+ continue;
+ }
+ if (be == FFTW)
+ {
+ irank = src.clayout->rank[0][ii0];
+ orank = this->clayout->rank[0][oi0];
+ }
+ else
+ {// TODO: handle 2D layout here
+ }
+ if ((irank == orank) &&
+ (irank == src.clayout->myrank))
+ {
+ std::copy(
+ (rnumber*)(src.get_cdata() + (ii0 - src.clayout->starts[0] )*slice_size),
+ (rnumber*)(src.get_cdata() + (ii0 - src.clayout->starts[0] + 1)*slice_size),
+ (rnumber*)buffer);
+ }
+ else
+ {
+ if (src.clayout->myrank == irank)
+ {
+ MPI_Send(
+ (void*)(src.get_cdata() + (ii0-src.clayout->starts[0])*slice_size),
+ slice_size,
+ mpi_real_type<rnumber>::complex(),
+ orank,
+ ii0,
+ src.clayout->comm);
+ }
+ if (src.clayout->myrank == orank)
+ {
+ MPI_Recv(
+ (void*)(buffer),
+ slice_size,
+ mpi_real_type<rnumber>::complex(),
+ irank,
+ ii0,
+ src.clayout->comm,
+ MPI_STATUS_IGNORE);
+ }
+ }
+ if (src.clayout->myrank == orank)
+ {
+ for (ii1 = 0; ii1 < int64_t(src.clayout->sizes[1]); ii1++)
+ {
+ if (ii1 <= int64_t(src.clayout->sizes[1]/2))
+ {
+ oi1 = ii1;
+ if (oi1 > int64_t(this->clayout->sizes[1]/2))
+ continue;
+ }
+ else
+ {
+ oi1 = ii1 + delta1;
+ if ((oi1 < 0) || ((int64_t(this->clayout->sizes[1]) - oi1) >= int64_t(this->clayout->sizes[1]/2)))
+ continue;
+ }
+ std::copy(
+ (rnumber*)(buffer + (ii1*src.clayout->sizes[2]*ncomp(fc))),
+ (rnumber*)(buffer + (ii1*src.clayout->sizes[2] + min_fast_dim)*ncomp(fc)),
+ (rnumber*)(this->get_cdata() +
+ ((oi0 - this->clayout->starts[0])*this->clayout->sizes[1] +
+ oi1)*this->clayout->sizes[2]*ncomp(fc)));
+ }
+ }
+ }
+ fftw_interface<rnumber>::free(buffer);
+ MPI_Barrier(src.clayout->comm);
+ }
+ }
+ return *this;
+}
+
+template <typename rnumber,
+ field_backend be,
+ field_components fc,
+ kspace_dealias_type dt>
+int make_gaussian_random_field(
+ kspace<be, dt> *kk,
+ field<rnumber, be, fc> *output_field,
+ const int rseed,
+ const double dissipation,
+ const double Lint,
+ const double etaK,
+ const double c_L,
+ const double c_eta,
+ const double coefficient)
+{
+ TIMEZONE("make_gaussian_random_field");
+ // initialize a separate random number generator for each thread
+ std::vector<std::mt19937_64> rgen;
+ std::normal_distribution<rnumber> rdist;
+ rgen.resize(omp_get_max_threads());
+ // seed random number generators such that no seed is ever repeated if we change the value of rseed.
+ // basically use a multi-dimensional array indexing technique to come up with actual seed.
+ // Note: this method IS NOT MPI/OpenMP-invariant!
+ for (int thread_id=0; thread_id < omp_get_max_threads(); thread_id++)
+ {
+ int current_seed = (
+ rseed*omp_get_max_threads()*output_field->clayout->nprocs +
+ output_field->clayout->myrank*omp_get_max_threads() +
+ thread_id);
+ //DEBUG_MSG("in make_gaussian_random_field, thread_id = %d, current_seed = %d\n", thread_id, current_seed);
+ rgen[thread_id].seed(current_seed);
+ }
+ output_field->real_space_representation = false;
+ *output_field = 0.0;
+ //DEBUG_MSG("slope: %g\n", slope);
+ // inside loop use only thread-local random number generator
+ kk->CLOOP_K2([&](
+ const ptrdiff_t cindex,
+ const ptrdiff_t xindex,
+ const ptrdiff_t yindex,
+ const ptrdiff_t zindex,
+ const double k2)
+ {
+ if (k2 > 0){
+ // Simple spectrum, diverges at zero
+ // double spectrum = coefficient * pow(k2, slope/2.) * exp(-sqrt(k2)/k_cutoff);
+
+ // Pope spectrum, Pope (2000), ''Turbulent flows'', p. 232
+ double C = 1.5;
+ double beta = 5.2;
+ double f_L = pow(sqrt(k2)*Lint/sqrt(k2*pow(Lint, 2) + c_L), 11./3.);
+ double f_eta = exp(-beta*(pow(pow(k2, 2)*pow(etaK, 4) + pow(c_eta, 4), 1./4.) - c_eta));
+ double spectrum = coefficient*C*pow(dissipation, 2./3.)*pow(k2, -5./6.)*f_L*f_eta;
+ // TODO: What about Delta k?
+ switch(fc)
+ {
+ case ONE:
+ {
+ output_field->cval(cindex,0) = rdist(rgen[omp_get_thread_num()]) * sqrt(spectrum * kk->dkx*kk->dky*kk->dkz / (4.*M_PI*k2));
+ output_field->cval(cindex,1) = rdist(rgen[omp_get_thread_num()]) * sqrt(spectrum * kk->dkx*kk->dky*kk->dkz / (4.*M_PI*k2));
+ break;
+ }
+ case THREE:
+ for (int cc = 0; cc<3; cc++)
+ {
+ // factor 3 compensates the trace between the spectral tensor and the energy spectrum
+ output_field->cval(cindex,cc,0) = rdist(rgen[omp_get_thread_num()]) * sqrt(spectrum * kk->dkx*kk->dky*kk->dkz / 3. / (4.*M_PI*k2));
+ output_field->cval(cindex,cc,1) = rdist(rgen[omp_get_thread_num()]) * sqrt(spectrum * kk->dkx*kk->dky*kk->dkz / 3. / (4.*M_PI*k2));
+ }
+ break;
+ case THREExTHREE:
+ for (int cc = 0; cc<3; cc++)
+ for (int ccc = 0; ccc<3; ccc++)
+ {
+ // factor 9 compensates the trace between the spectral tensor and the energy spectrum
+ output_field->cval(cindex,cc,ccc,0) = rdist(rgen[omp_get_thread_num()]) * sqrt(spectrum * kk->dkx*kk->dky*kk->dkz / 9. / (4.*M_PI*k2));
+ output_field->cval(cindex,cc,ccc,1) = rdist(rgen[omp_get_thread_num()]) * sqrt(spectrum * kk->dkx*kk->dky*kk->dkz / 9. / (4.*M_PI*k2));
+ }
+ break;
+ }
+ }
+ });
+ output_field->symmetrize();
+
+ return EXIT_SUCCESS;
+}
+
+template <typename rnumber,
+ field_backend be,
+ kspace_dealias_type dt>
+int generate_random_phase_field(
+ kspace<be, dt> *kk,
+ field<rnumber, be, ONE> *output_field,
+ const int rseed)
+{
+ TIMEZONE("generate_random_phase_field");
+ assert(output_field->real_space_representation == false);
+ const double twopi = 4*acos(0);
+
+ // initialize a separate random number generator for each thread
+ std::vector<std::mt19937_64> rgen;
+ std::uniform_real_distribution<rnumber> rdist(0, twopi);
+ rgen.resize(omp_get_max_threads());
+ // seed random number generators such that no seed is ever repeated if we change the value of rseed.
+ // basically use a multi-dimensional array indexing technique to come up with actual seed.
+ // Note: this method IS NOT invariant to the MPI/OpenMP configuration.
+ for (int thread_id=0; thread_id < omp_get_max_threads(); thread_id++)
+ {
+ int current_seed = (
+ rseed*omp_get_max_threads()*output_field->clayout->nprocs +
+ output_field->clayout->myrank*omp_get_max_threads() +
+ thread_id);
+ rgen[thread_id].seed(current_seed);
+ }
+
+ // inside loop use only thread-local random number generator
+ kk->CLOOP_K2([&](
+ const ptrdiff_t cindex,
+ const ptrdiff_t xindex,
+ const ptrdiff_t yindex,
+ const ptrdiff_t zindex,
+ const double k2)
+ {
+ if (k2 > 0 && k2 < kk->kM2)
+ {
+ // single phase shift for all components
+ const double phi = rdist(rgen[omp_get_thread_num()]);
+ output_field->cval(cindex, 0) = cos(phi);
+ output_field->cval(cindex, 1) = sin(phi);
+ }
+ else
+ {
+ output_field->cval(cindex, 0) = rnumber(0);
+ output_field->cval(cindex, 1) = rnumber(0);
+ }
+ });
+ output_field->symmetrize();
+ // ensure absolute value is 1 for all modes
+ {
+ kk->CLOOP_K2([&](
+ const ptrdiff_t cindex,
+ const ptrdiff_t xindex,
+ const ptrdiff_t yindex,
+ const ptrdiff_t zindex,
+ const double k2)
+ {
+ if (k2 > 0 && k2 < kk->kM2)
+ {
+ {
+ const double amplitude = sqrt(
+ output_field->cval(cindex, 0)*output_field->cval(cindex, 0)+
+ output_field->cval(cindex, 1)*output_field->cval(cindex, 1));
+ if (amplitude > 0)
+ {
+ output_field->cval(cindex, 0) /= amplitude;
+ output_field->cval(cindex, 1) /= amplitude;
+ }
+ else
+ {
+ output_field->cval(cindex, 0) = 1;
+ output_field->cval(cindex, 1) = 0;
+ }
+ }
+ }
+ if (xindex == ptrdiff_t(output_field->clayout->sizes[2])-ptrdiff_t(1))
+ {
+ output_field->cval(cindex, 0) = 0;
+ output_field->cval(cindex, 1) = 0;
+ }
+ //if ((xindex == output_field->clayout->sizes[2]-1) &&
+ // (zindex == output_field->clayout->sizes[1]-1))
+ //{
+ // DEBUG_MSG("yindex = %ld, phase_field = %g + j%g\n",
+ // yindex,
+ // output_field->cval(cindex, 0),
+ // output_field->cval(cindex, 1)
+ // );
+ //}
+ });
+ }
+
+ return EXIT_SUCCESS;
+}
+
+template <typename rnumber,
+ field_backend be,
+ field_components fc,
+ kspace_dealias_type dt>
+int apply_phase_field_shift(
+ kspace<be, dt> *kk,
+ field<rnumber, be, fc> *output_field,
+ field<rnumber, be, ONE> *phase_field)
+{
+ TIMEZONE("apply_phase_field_shift");
+ assert(output_field->real_space_representation == false);
+ assert(phase_field->real_space_representation == false);
+
+ // inside loop use only thread-local random number generator
+ kk->CLOOP_K2([&](
+ const ptrdiff_t cindex,
+ const ptrdiff_t xindex,
+ const ptrdiff_t yindex,
+ const ptrdiff_t zindex,
+ const double k2)
+ {
+ if (k2 > 0)
+ {
+ // single phase shift for all components
+ switch(fc)
+ {
+ case ONE:
+ {
+ complex_number_phase_shifter(
+ output_field->cval(cindex, 0),
+ output_field->cval(cindex, 1),
+ phase_field->cval(cindex, 0),
+ phase_field->cval(cindex, 1));
+ break;
+ }
+ case THREE:
+ for (int cc = 0; cc<3; cc++)
+ {
+ complex_number_phase_shifter(
+ output_field->cval(cindex, cc, 0),
+ output_field->cval(cindex, cc, 1),
+ phase_field->cval(cindex, 0),
+ phase_field->cval(cindex, 1));
+ }
+ break;
+ case THREExTHREE:
+ for (int cc = 0; cc<3; cc++)
+ for (int ccc = 0; ccc<3; ccc++)
+ {
+ complex_number_phase_shifter(
+ output_field->cval(cindex, cc, ccc, 0),
+ output_field->cval(cindex, cc, ccc, 1),
+ phase_field->cval(cindex, 0),
+ phase_field->cval(cindex, 1));
+ }
+ break;
+ }
+ }
+ });
+ output_field->symmetrize();
+
+ return EXIT_SUCCESS;
+}
+
+template class field<float, FFTW, ONE>;
+template class field<float, FFTW, THREE>;
+template class field<float, FFTW, THREExTHREE>;
+template class field<double, FFTW, ONE>;
+template class field<double, FFTW, THREE>;
+template class field<double, FFTW, THREExTHREE>;
+
+template void field<float, FFTW, ONE>::compute_stats<TWO_THIRDS>(
+ kspace<FFTW, TWO_THIRDS> *,
+ const hid_t, const std::string, const hsize_t, const double);
+template void field<float, FFTW, THREE>::compute_stats<TWO_THIRDS>(
+ kspace<FFTW, TWO_THIRDS> *,
+ const hid_t, const std::string, const hsize_t, const double);
+template void field<float, FFTW, THREExTHREE>::compute_stats<TWO_THIRDS>(
+ kspace<FFTW, TWO_THIRDS> *,
+ const hid_t, const std::string, const hsize_t, const double);
+
+template void field<double, FFTW, ONE>::compute_stats<TWO_THIRDS>(
+ kspace<FFTW, TWO_THIRDS> *,
+ const hid_t, const std::string, const hsize_t, const double);
+template void field<double, FFTW, THREE>::compute_stats<TWO_THIRDS>(
+ kspace<FFTW, TWO_THIRDS> *,
+ const hid_t, const std::string, const hsize_t, const double);
+template void field<double, FFTW, THREExTHREE>::compute_stats<TWO_THIRDS>(
+ kspace<FFTW, TWO_THIRDS> *,
+ const hid_t, const std::string, const hsize_t, const double);
+
+template void field<float, FFTW, ONE>::compute_stats<SMOOTH>(
+ kspace<FFTW, SMOOTH> *,
+ const hid_t, const std::string, const hsize_t, const double);
+template void field<float, FFTW, THREE>::compute_stats<SMOOTH>(
+ kspace<FFTW, SMOOTH> *,
+ const hid_t, const std::string, const hsize_t, const double);
+template void field<float, FFTW, THREExTHREE>::compute_stats<SMOOTH>(
+ kspace<FFTW, SMOOTH> *,
+ const hid_t, const std::string, const hsize_t, const double);
+
+template void field<double, FFTW, ONE>::compute_stats<SMOOTH>(
+ kspace<FFTW, SMOOTH> *,
+ const hid_t, const std::string, const hsize_t, const double);
+template void field<double, FFTW, THREE>::compute_stats<SMOOTH>(
+ kspace<FFTW, SMOOTH> *,
+ const hid_t, const std::string, const hsize_t, const double);
+template void field<double, FFTW, THREExTHREE>::compute_stats<SMOOTH>(
+ kspace<FFTW, SMOOTH> *,
+ const hid_t, const std::string, const hsize_t, const double);
+
+template double field<float, FFTW, ONE>::L2norm<TWO_THIRDS>(
+ kspace<FFTW, TWO_THIRDS> *);
+template double field<float, FFTW, THREE>::L2norm<TWO_THIRDS>(
+ kspace<FFTW, TWO_THIRDS> *);
+template double field<float, FFTW, THREExTHREE>::L2norm<TWO_THIRDS>(
+ kspace<FFTW, TWO_THIRDS> *);
+
+template double field<double, FFTW, ONE>::L2norm<TWO_THIRDS>(
+ kspace<FFTW, TWO_THIRDS> *);
+template double field<double, FFTW, THREE>::L2norm<TWO_THIRDS>(
+ kspace<FFTW, TWO_THIRDS> *);
+template double field<double, FFTW, THREExTHREE>::L2norm<TWO_THIRDS>(
+ kspace<FFTW, TWO_THIRDS> *);
+
+template double field<float, FFTW, ONE>::L2norm<SMOOTH>(
+ kspace<FFTW, SMOOTH> *);
+template double field<float, FFTW, THREE>::L2norm<SMOOTH>(
+ kspace<FFTW, SMOOTH> *);
+template double field<float, FFTW, THREExTHREE>::L2norm<SMOOTH>(
+ kspace<FFTW, SMOOTH> *);
+
+template double field<double, FFTW, ONE>::L2norm<SMOOTH>(
+ kspace<FFTW, SMOOTH> *);
+template double field<double, FFTW, THREE>::L2norm<SMOOTH>(
+ kspace<FFTW, SMOOTH> *);
+template double field<double, FFTW, THREExTHREE>::L2norm<SMOOTH>(
+ kspace<FFTW, SMOOTH> *);
+
+template int compute_gradient<float, FFTW, THREE, THREExTHREE, SMOOTH>(
+ kspace<FFTW, SMOOTH> *,
+ field<float, FFTW, THREE> *,
+ field<float, FFTW, THREExTHREE> *);
+template int compute_gradient<double, FFTW, THREE, THREExTHREE, SMOOTH>(
+ kspace<FFTW, SMOOTH> *,
+ field<double, FFTW, THREE> *,
+ field<double, FFTW, THREExTHREE> *);
+
+template int compute_gradient<float, FFTW, ONE, THREE, SMOOTH>(
+ kspace<FFTW, SMOOTH> *,
+ field<float, FFTW, ONE> *,
+ field<float, FFTW, THREE> *);
+template int compute_gradient<double, FFTW, ONE, THREE, SMOOTH>(
+ kspace<FFTW, SMOOTH> *,
+ field<double, FFTW, ONE> *,
+ field<double, FFTW, THREE> *);
+
+template int compute_divergence(
+ kspace<FFTW, SMOOTH> *,
+ field<float, FFTW, THREE> *,
+ field<float, FFTW, ONE> *);
+template int compute_divergence(
+ kspace<FFTW, SMOOTH> *,
+ field<double, FFTW, THREE> *,
+ field<double, FFTW, ONE> *);
+
+template int compute_curl<float, FFTW, SMOOTH>(
+ kspace<FFTW, SMOOTH> *,
+ field<float, FFTW, THREE> *,
+ field<float, FFTW, THREE> *);
+template int compute_curl<double, FFTW, SMOOTH>(
+ kspace<FFTW, SMOOTH> *,
+ field<double, FFTW, THREE> *,
+ field<double, FFTW, THREE> *);
+
+template int invert_curl<float, FFTW, SMOOTH>(
+ kspace<FFTW, SMOOTH> *,
+ field<float, FFTW, THREE> *,
+ field<float, FFTW, THREE> *);
+template int invert_curl<double, FFTW, SMOOTH>(
+ kspace<FFTW, SMOOTH> *,
+ field<double, FFTW, THREE> *,
+ field<double, FFTW, THREE> *);
+
+template int joint_rspace_PDF<float, FFTW, THREE>(
+ field<float, FFTW, THREE> *,
+ field<float, FFTW, THREE> *,
+ const hid_t,
+ const std::string,
+ const hsize_t,
+ const std::vector<double>,
+ const std::vector<double>);
+template int joint_rspace_PDF<double, FFTW, THREE>(
+ field<double, FFTW, THREE> *,
+ field<double, FFTW, THREE> *,
+ const hid_t,
+ const std::string,
+ const hsize_t,
+ const std::vector<double>,
+ const std::vector<double>);
+
+template int joint_rspace_PDF<float, FFTW, ONE>(
+ field<float, FFTW, ONE> *,
+ field<float, FFTW, ONE> *,
+ const hid_t,
+ const std::string,
+ const hsize_t,
+ const std::vector<double>,
+ const std::vector<double>);
+template int joint_rspace_PDF<double, FFTW, ONE>(
+ field<double, FFTW, ONE> *,
+ field<double, FFTW, ONE> *,
+ const hid_t,
+ const std::string,
+ const hsize_t,
+ const std::vector<double>,
+ const std::vector<double>);
+
+template int joint_rspace_3PDF<float, FFTW>(
+ field<float, FFTW, ONE> *,
+ field<float, FFTW, ONE> *,
+ field<float, FFTW, ONE> *,
+ const hid_t,
+ const std::string,
+ const hsize_t,
+ const std::vector<double>,
+ const std::vector<double>,
+ const std::vector<double>);
+template int joint_rspace_3PDF<double, FFTW>(
+ field<double, FFTW, ONE> *,
+ field<double, FFTW, ONE> *,
+ field<double, FFTW, ONE> *,
+ const hid_t,
+ const std::string,
+ const hsize_t,
+ const std::vector<double>,
+ const std::vector<double>,
+ const std::vector<double>);
+
+
+template int make_gaussian_random_field<
+ float,
+ FFTW,
+ ONE,
+ SMOOTH>(
+ kspace<FFTW, SMOOTH> *kk,
+ field<float, FFTW, ONE> *output_field,
+ const int rseed,
+ const double dissipation,
+ const double Lint,
+ const double etaK,
+ const double c_L,
+ const double c_eta,
+ const double coefficient);
+template int make_gaussian_random_field<
+ double,
+ FFTW,
+ ONE,
+ SMOOTH>(
+ kspace<FFTW, SMOOTH> *kk,
+ field<double, FFTW, ONE> *output_field,
+ const int rseed,
+ const double dissipation,
+ const double Lint,
+ const double etaK,
+ const double c_L,
+ const double c_eta,
+ const double coefficient);
+template int make_gaussian_random_field<
+ float,
+ FFTW,
+ THREE,
+ SMOOTH>(
+ kspace<FFTW, SMOOTH> *kk,
+ field<float, FFTW, THREE> *output_field,
+ const int rseed,
+ const double dissipation,
+ const double Lint,
+ const double etaK,
+ const double c_L,
+ const double c_eta,
+ const double coefficient);
+template int make_gaussian_random_field<
+ double,
+ FFTW,
+ THREE,
+ SMOOTH>(
+ kspace<FFTW, SMOOTH> *kk,
+ field<double, FFTW, THREE> *output_field,
+ const int rseed,
+ const double dissipation,
+ const double Lint,
+ const double etaK,
+ const double c_L,
+ const double c_eta,
+ const double coefficient);
+
+template int apply_phase_field_shift<float,
+ FFTW,
+ ONE,
+ SMOOTH>
+(
+ kspace<FFTW, SMOOTH> *kk,
+ field<float, FFTW, ONE> *output_field,
+ field<float, FFTW, ONE> *phase_field);
+
+template int apply_phase_field_shift<float,
+ FFTW,
+ THREE,
+ SMOOTH>
+(
+ kspace<FFTW, SMOOTH> *kk,
+ field<float, FFTW, THREE> *output_field,
+ field<float, FFTW, ONE> *phase_field);
+
+template int apply_phase_field_shift<float,
+ FFTW,
+ THREExTHREE,
+ SMOOTH>
+(
+ kspace<FFTW, SMOOTH> *kk,
+ field<float, FFTW, THREExTHREE> *output_field,
+ field<float, FFTW, ONE> *phase_field);
+
+template int apply_phase_field_shift<double,
+ FFTW,
+ ONE,
+ SMOOTH>
+(
+ kspace<FFTW, SMOOTH> *kk,
+ field<double, FFTW, ONE> *output_field,
+ field<double, FFTW, ONE> *phase_field);
+
+template int apply_phase_field_shift<double,
+ FFTW,
+ THREE,
+ SMOOTH>
+(
+ kspace<FFTW, SMOOTH> *kk,
+ field<double, FFTW, THREE> *output_field,
+ field<double, FFTW, ONE> *phase_field);
+
+template int apply_phase_field_shift<double,
+ FFTW,
+ THREExTHREE,
+ SMOOTH>
+(
+ kspace<FFTW, SMOOTH> *kk,
+ field<double, FFTW, THREExTHREE> *output_field,
+ field<double, FFTW, ONE> *phase_field);
+
+
+template int generate_random_phase_field<float,
+ FFTW,
+ SMOOTH>
+(
+ kspace<FFTW, SMOOTH> *kk,
+ field<float, FFTW, ONE> *output_field,
+ const int rseed);
+
+template int generate_random_phase_field<double,
+ FFTW,
+ SMOOTH>
+(
+ kspace<FFTW, SMOOTH> *kk,
+ field<double, FFTW, ONE> *output_field,
+ const int rseed);
diff --git a/bfps/cpp/field.hpp b/cpp/field.hpp
similarity index 58%
rename from bfps/cpp/field.hpp
rename to cpp/field.hpp
index 52a936320974a9076a419d4b081d0ee9ab5d4ae5..f6c69a7d1e10d6751e0be0055518b1fa4c2d2e6c 100644
--- a/bfps/cpp/field.hpp
+++ b/cpp/field.hpp
@@ -3,20 +3,20 @@
* Copyright 2015 Max Planck Institute *
* for Dynamics and Self-Organization *
* *
-* This file is part of bfps. *
+* This file is part of TurTLE. *
* *
-* bfps is free software: you can redistribute it and/or modify *
+* TurTLE is free software: you can redistribute it and/or modify *
* it under the terms of the GNU General Public License as published *
* by the Free Software Foundation, either version 3 of the License, *
* or (at your option) any later version. *
* *
-* bfps is distributed in the hope that it will be useful, *
+* TurTLE is distributed in the hope that it will be useful, *
* but WITHOUT ANY WARRANTY; without even the implied warranty of *
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the *
* GNU General Public License for more details. *
* *
* You should have received a copy of the GNU General Public License *
-* along with bfps. If not, see <http://www.gnu.org/licenses/> *
+* along with TurTLE. If not, see <http://www.gnu.org/licenses/> *
* *
* Contact: Cristian.Lalescu@ds.mpg.de *
* *
@@ -24,6 +24,10 @@
+#ifndef FIELD_HPP
+
+#define FIELD_HPP
+
#include <hdf5.h>
#include <unordered_map>
#include <vector>
@@ -31,10 +35,6 @@
#include "kspace.hpp"
#include "omputils.hpp"
-#ifndef FIELD_HPP
-
-#define FIELD_HPP
-
/** \class field
* \brief Holds field data, performs FFTs and HDF5 I/O operations.
*
@@ -58,6 +58,7 @@ class field
private:
rnumber *__restrict__ data; /**< data array */
public:
+ static const int number_of_components = ncomp(fc);
hsize_t npoints; /**< total number of grid points. Useful for normalization. */
bool real_space_representation; /**< `true` if field is in real space representation. */
@@ -72,8 +73,8 @@ class field
field_layout<fc> *clayout, *rlayout, *rmemlayout;
/* FFT plans */
- typename fftw_interface<rnumber>::plan c2r_plan;
- typename fftw_interface<rnumber>::plan r2c_plan;
+ typename fftw_interface<rnumber>::many_plan c2r_plan;
+ typename fftw_interface<rnumber>::many_plan r2c_plan;
unsigned fftw_plan_rigor;
/* HDF5 data types for arrays */
@@ -86,7 +87,7 @@ class field
const int nz,
const MPI_Comm COMM_TO_USE,
const unsigned FFTW_PLAN_RIGOR = DEFAULT_FFTW_FLAG);
- ~field();
+ ~field() noexcept(false);
int io(
const std::string fname,
@@ -103,6 +104,13 @@ class field
const hid_t group,
const std::string field_name,
const int iteration);
+ int write_filtered(
+ const std::string fname,
+ const std::string field_name,
+ const int iteration,
+ const int nx,
+ const int ny,
+ const int nz);
int io_binary(
const std::string fname,
@@ -114,6 +122,9 @@ class field
void ift();
void normalize();
void symmetrize();
+ void symmetrize_FFT();
+ void symmetrize_alternate();
+ void Hermitian_reflect();
/* stats */
void compute_rspace_xincrement_stats(
@@ -121,7 +132,8 @@ class field
const hid_t group,
const std::string dset_name,
const hsize_t toffset,
- const std::vector<double> max_estimate);
+ const std::vector<double> max_estimate,
+ field<rnumber, be, fc> *tmp_field = NULL);
void compute_rspace_stats(
const hid_t group,
@@ -129,6 +141,25 @@ class field
const hsize_t toffset,
const std::vector<double> max_estimate);
+ void compute_rspace_zaverage(
+ const hid_t group,
+ const std::string dset_name,
+ const hsize_t toffset);
+
+ /* access sizes */
+ inline int get_nx() const
+ {
+ return this->rlayout->sizes[2];
+ }
+ inline int get_ny() const
+ {
+ return this->rlayout->sizes[1];
+ }
+ inline int get_nz() const
+ {
+ return this->rlayout->sizes[0];
+ }
+
/* acess data */
inline rnumber *__restrict__ get_rdata()
{
@@ -145,6 +176,11 @@ class field
return (typename fftw_interface<rnumber>::complex*__restrict__)this->data;
}
+ inline typename fftw_interface<rnumber>::complex *__restrict__ get_cdata() const
+ {
+ return (typename fftw_interface<rnumber>::complex*__restrict__)this->data;
+ }
+
inline rnumber &rval(ptrdiff_t rindex, unsigned int component = 0)
{
assert(fc == ONE || fc == THREE);
@@ -154,7 +190,7 @@ class field
inline const rnumber& rval(ptrdiff_t rindex, unsigned int component = 0) const
{
- assert(fc == ONE || fc == THREE);
+ //assert(fc == ONE || fc == THREE);
assert(component >= 0 && component < ncomp(fc));
return *(this->data + rindex*ncomp(fc) + component);
}
@@ -190,31 +226,10 @@ class field
return *(this->data + ((cindex*3 + comp1)*3+comp0)*2 + imag);
}
- inline field<rnumber, be, fc>& operator=(const typename fftw_interface<rnumber>::complex *__restrict__ source)
- {
- std::copy((rnumber*)source,
- (rnumber*)(source + this->clayout->local_size),
- this->data);
- this->real_space_representation = false;
- return *this;
- }
-
- inline field<rnumber, be, fc>& operator=(const rnumber *__restrict__ source)
- {
- std::copy(source,
- source + this->rmemlayout->local_size,
- this->data);
- this->real_space_representation = true;
- return *this;
- }
-
- inline field<rnumber, be, fc>& operator=(const rnumber value)
- {
- std::fill_n(this->data,
- this->rmemlayout->local_size,
- value);
- return *this;
- }
+ field<rnumber, be, fc>& operator=(const typename fftw_interface<rnumber>::complex *__restrict__ source);
+ field<rnumber, be, fc>& operator=(const rnumber *__restrict__ source);
+ field<rnumber, be, fc>& operator=(const rnumber value);
+ field<rnumber, be, fc>& operator=(const field<rnumber, be, fc> &src);
template <kspace_dealias_type dt>
void compute_stats(
@@ -223,6 +238,9 @@ class field
const std::string dset_name,
const hsize_t toffset,
const double max_estimate);
+ template <kspace_dealias_type dt>
+ double L2norm(
+ kspace<be, dt> *kk);
inline void impose_zero_mode()
{
if (this->clayout->myrank == this->clayout->rank[0][0] &&
@@ -232,8 +250,38 @@ class field
}
}
template <class func_type>
+ void RLOOP_simd(func_type expression)
+ {
+ start_mpi_profiling_zone(turtle_mpi_pcontrol::FIELD);
+ switch(be)
+ {
+ case FFTW:
+ #pragma omp parallel
+ {
+ const hsize_t start = OmpUtils::ForIntervalStart(this->rlayout->subsizes[1]);
+ const hsize_t end = OmpUtils::ForIntervalEnd(this->rlayout->subsizes[1]);
+
+ #pragma omp simd
+ for (hsize_t zindex = 0; zindex < this->rlayout->subsizes[0]; zindex++)
+ for (hsize_t yindex = start; yindex < end; yindex++)
+ {
+ const ptrdiff_t rindex = (
+ zindex * this->rlayout->subsizes[1] + yindex)*(
+ this->rmemlayout->subsizes[2]);
+ for (hsize_t xindex = 0; xindex < this->rlayout->subsizes[2]; xindex++)
+ {
+ expression(rindex + xindex, xindex, yindex, zindex);
+ }
+ }
+ }
+ break;
+ }
+ finish_mpi_profiling_zone(turtle_mpi_pcontrol::FIELD);
+ }
+ template <class func_type>
void RLOOP(func_type expression)
{
+ start_mpi_profiling_zone(turtle_mpi_pcontrol::FIELD);
switch(be)
{
case FFTW:
@@ -243,20 +291,21 @@ class field
const hsize_t end = OmpUtils::ForIntervalEnd(this->rlayout->subsizes[1]);
for (hsize_t zindex = 0; zindex < this->rlayout->subsizes[0]; zindex++)
+ //#pragma omp simd
for (hsize_t yindex = start; yindex < end; yindex++)
{
- ptrdiff_t rindex = (
+ const ptrdiff_t rindex = (
zindex * this->rlayout->subsizes[1] + yindex)*(
this->rmemlayout->subsizes[2]);
for (hsize_t xindex = 0; xindex < this->rlayout->subsizes[2]; xindex++)
{
- expression(rindex, xindex, yindex, zindex);
- rindex++;
+ expression(rindex + xindex, xindex, yindex, zindex);
}
}
}
break;
}
+ finish_mpi_profiling_zone(turtle_mpi_pcontrol::FIELD);
}
ptrdiff_t get_cindex(
ptrdiff_t xindex,
@@ -286,8 +335,45 @@ class field
in_global_y - this->rlayout->starts[1],
in_global_z - this->rlayout->starts[0]);
}
+
+ int print_plan(const std::string preamble)
+ {
+ char *c2r_plan_information = fftw_interface<rnumber>::sprint(this->c2r_plan);
+ char *r2c_plan_information = fftw_interface<rnumber>::sprint(this->r2c_plan);
+ if (this->myrank == 0)
+ {
+ std::cout << preamble <<
+ std::endl <<
+ "----c2r plan is:\n" <<
+ c2r_plan_information <<
+ std::endl <<
+ "----r2c plan is:\n" <<
+ r2c_plan_information <<
+ std::endl;
+ }
+ std::string err_message = (
+ std::string("MPI rank ") +
+ preamble +
+ std::to_string(this->myrank) +
+ std::string("\n----c2r plan is:\n") +
+ std::string(c2r_plan_information) +
+ std::string("\n----r2c plan is:\n") +
+ std::string(r2c_plan_information) +
+ std::string("\n"));
+ std::cerr << err_message;
+ free(c2r_plan_information);
+ free(r2c_plan_information);
+ return EXIT_SUCCESS;
+ }
};
+/** \brief Compute gradient of a field
+ *
+ * if $A_{ij} = \partial_j u_i$
+ * then `compute_gradient` for a vector field stores the data in the resulting
+ * THREExTHREE field with $i$ the fast counter and $j$ the slow counter.
+ *
+ * */
template <typename rnumber,
field_backend be,
field_components fc1,
@@ -298,6 +384,17 @@ int compute_gradient(
field<rnumber, be, fc1> *source,
field<rnumber, be, fc2> *destination);
+/** \brief Compute divergence of a vector field
+ *
+ * */
+template <typename rnumber,
+ field_backend be,
+ kspace_dealias_type dt>
+int compute_divergence(
+ kspace<be, dt> *kk,
+ field<rnumber, be, THREE> *source,
+ field<rnumber, be, ONE> *destination);
+
template <typename rnumber,
field_backend be,
kspace_dealias_type dt>
@@ -306,6 +403,14 @@ int invert_curl(
field<rnumber, be, THREE> *source,
field<rnumber, be, THREE> *destination);
+template <typename rnumber,
+ field_backend be,
+ kspace_dealias_type dt>
+int compute_curl(
+ kspace<be, dt> *kk,
+ field<rnumber, be, THREE> *source,
+ field<rnumber, be, THREE> *destination);
+
template <typename rnumber,
field_backend be,
field_components fc>
@@ -318,5 +423,61 @@ int joint_rspace_PDF(
const std::vector<double> max_f1_estimate,
const std::vector<double> max_f2_estimate);
+template <typename rnumber,
+ field_backend be>
+int joint_rspace_3PDF(
+ field<rnumber, be, ONE> *f1,
+ field<rnumber, be, ONE> *f2,
+ field<rnumber, be, ONE> *f3,
+ const hid_t group,
+ const std::string dset_name,
+ const hsize_t toffset,
+ const std::vector<double> max_f1_estimate,
+ const std::vector<double> max_f2_estimate,
+ const std::vector<double> max_f3_estimate);
+
+/** \brief Generate a Gaussian random field
+ *
+ * \note This method IS NOT invariant to the MPI/OpenMP configuration!
+ * */
+template <typename rnumber,
+ field_backend be,
+ field_components fc,
+ kspace_dealias_type dt>
+int make_gaussian_random_field(
+ kspace<be, dt> *kk,
+ field<rnumber, be, fc> *output_field,
+ const int rseed = 0,
+ const double dissipation = 0.3,
+ const double Lint = 1.,
+ const double etaK = 0.01,
+ const double c_L = 6.78,
+ const double c_eta = 0.40,
+ const double coefficient = 1.);
+
+/** \brief Apply random phase shift
+ *
+ * \note This method IS NOT invariant to the MPI/OpenMP configuration!
+ * */
+template <typename rnumber,
+ field_backend be,
+ kspace_dealias_type dt>
+int generate_random_phase_field(
+ kspace<be, dt> *kk,
+ field<rnumber, be, ONE> *output_field,
+ const int rseed = 0);
+
+/** \brief Apply random phase shift
+ *
+ * */
+template <typename rnumber,
+ field_backend be,
+ field_components fc,
+ kspace_dealias_type dt>
+int apply_phase_field_shift(
+ kspace<be, dt> *kk,
+ field<rnumber, be, fc> *output_field,
+ field<rnumber, be, ONE> *phase_field);
+
#endif//FIELD_HPP
diff --git a/bfps/cpp/field_binary_IO.cpp b/cpp/field_binary_IO.cpp
similarity index 96%
rename from bfps/cpp/field_binary_IO.cpp
rename to cpp/field_binary_IO.cpp
index 52ca21f5947689408265423e0c4f075a670fa578..288ef26a9cdb490895e908c5c3f377ed13c756de 100644
--- a/bfps/cpp/field_binary_IO.cpp
+++ b/cpp/field_binary_IO.cpp
@@ -3,20 +3,20 @@
* Copyright 2015 Max Planck Institute *
* for Dynamics and Self-Organization *
* *
-* This file is part of bfps. *
+* This file is part of TurTLE. *
* *
-* bfps is free software: you can redistribute it and/or modify *
+* TurTLE is free software: you can redistribute it and/or modify *
* it under the terms of the GNU General Public License as published *
* by the Free Software Foundation, either version 3 of the License, *
* or (at your option) any later version. *
* *
-* bfps is distributed in the hope that it will be useful, *
+* TurTLE is distributed in the hope that it will be useful, *
* but WITHOUT ANY WARRANTY; without even the implied warranty of *
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the *
* GNU General Public License for more details. *
* *
* You should have received a copy of the GNU General Public License *
-* along with bfps. If not, see <http://www.gnu.org/licenses/> *
+* along with TurTLE. If not, see <http://www.gnu.org/licenses/> *
* *
* Contact: Cristian.Lalescu@ds.mpg.de *
* *
diff --git a/bfps/cpp/field_binary_IO.hpp b/cpp/field_binary_IO.hpp
similarity index 83%
rename from bfps/cpp/field_binary_IO.hpp
rename to cpp/field_binary_IO.hpp
index b06e901c27acd99855a314dc7b5afb8b43f190b7..c615f9e266fa7de5816c0cd98b5a830dc81d6e48 100644
--- a/bfps/cpp/field_binary_IO.hpp
+++ b/cpp/field_binary_IO.hpp
@@ -3,20 +3,20 @@
* Copyright 2015 Max Planck Institute *
* for Dynamics and Self-Organization *
* *
-* This file is part of bfps. *
+* This file is part of TurTLE. *
* *
-* bfps is free software: you can redistribute it and/or modify *
+* TurTLE is free software: you can redistribute it and/or modify *
* it under the terms of the GNU General Public License as published *
* by the Free Software Foundation, either version 3 of the License, *
* or (at your option) any later version. *
* *
-* bfps is distributed in the hope that it will be useful, *
+* TurTLE is distributed in the hope that it will be useful, *
* but WITHOUT ANY WARRANTY; without even the implied warranty of *
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the *
* GNU General Public License for more details. *
* *
* You should have received a copy of the GNU General Public License *
-* along with bfps. If not, see <http://www.gnu.org/licenses/> *
+* along with TurTLE. If not, see <http://www.gnu.org/licenses/> *
* *
* Contact: Cristian.Lalescu@ds.mpg.de *
* *
@@ -24,6 +24,10 @@
+#ifndef FIELD_BINARY_IO_HPP
+
+#define FIELD_BINARY_IO_HPP
+
#include <vector>
#include <string>
#include "base.hpp"
@@ -31,10 +35,6 @@
#include "field_layout.hpp"
#include "field.hpp"
-#ifndef FIELD_BINARY_IO_HPP
-
-#define FIELD_BINARY_IO_HPP
-
/* could this be a boolean somehow?*/
enum field_representation: bool {
REAL = true,
@@ -49,6 +49,14 @@ constexpr MPI_Datatype mpi_type(
mpi_real_type<rnumber>::complex());
}
+/** \class field_binary_IO
+ * \brief A class to handle binary field IO
+ *
+ * \tparam rnumber field data type
+ * \tparam fr field representation (REAL or COMPLEX)
+ * \tparam fc number of field components
+ */
+
template <typename rnumber, field_representation fr, field_components fc>
class field_binary_IO:public field_layout<fc>
{
@@ -64,7 +72,7 @@ class field_binary_IO:public field_layout<fc>
const hsize_t *SUBSIZES,
const hsize_t *STARTS,
const MPI_Comm COMM_TO_USE);
- ~field_binary_IO();
+ ~field_binary_IO() noexcept(false);
int read(
const std::string fname,
diff --git a/bfps/cpp/field_layout.cpp b/cpp/field_layout.cpp
similarity index 93%
rename from bfps/cpp/field_layout.cpp
rename to cpp/field_layout.cpp
index 908904991d5d95b0c89ba679b402d8d5727b8c85..0f19a6cd98a5dbe6d2be54d80f887d77a33f67ea 100644
--- a/bfps/cpp/field_layout.cpp
+++ b/cpp/field_layout.cpp
@@ -3,30 +3,33 @@
* Copyright 2015 Max Planck Institute *
* for Dynamics and Self-Organization *
* *
-* This file is part of bfps. *
+* This file is part of TurTLE. *
* *
-* bfps is free software: you can redistribute it and/or modify *
+* TurTLE is free software: you can redistribute it and/or modify *
* it under the terms of the GNU General Public License as published *
* by the Free Software Foundation, either version 3 of the License, *
* or (at your option) any later version. *
* *
-* bfps is distributed in the hope that it will be useful, *
+* TurTLE is distributed in the hope that it will be useful, *
* but WITHOUT ANY WARRANTY; without even the implied warranty of *
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the *
* GNU General Public License for more details. *
* *
* You should have received a copy of the GNU General Public License *
-* along with bfps. If not, see <http://www.gnu.org/licenses/> *
+* along with TurTLE. If not, see <http://www.gnu.org/licenses/> *
* *
* Contact: Cristian.Lalescu@ds.mpg.de *
* *
**********************************************************************/
+
#include <cassert>
#include "field_layout.hpp"
#include "scope_timer.hpp"
+
+
template <field_components fc>
field_layout<fc>::field_layout(
const hsize_t *SIZES,
diff --git a/bfps/cpp/field_layout.hpp b/cpp/field_layout.hpp
similarity index 64%
rename from bfps/cpp/field_layout.hpp
rename to cpp/field_layout.hpp
index 770119c2dcb05017d495b62559f050646872dc84..e7ba578cbeb5f7e990e2c21f95e0cb9cc1054985 100644
--- a/bfps/cpp/field_layout.hpp
+++ b/cpp/field_layout.hpp
@@ -3,20 +3,20 @@
* Copyright 2015 Max Planck Institute *
* for Dynamics and Self-Organization *
* *
-* This file is part of bfps. *
+* This file is part of TurTLE. *
* *
-* bfps is free software: you can redistribute it and/or modify *
+* TurTLE is free software: you can redistribute it and/or modify *
* it under the terms of the GNU General Public License as published *
* by the Free Software Foundation, either version 3 of the License, *
* or (at your option) any later version. *
* *
-* bfps is distributed in the hope that it will be useful, *
+* TurTLE is distributed in the hope that it will be useful, *
* but WITHOUT ANY WARRANTY; without even the implied warranty of *
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the *
* GNU General Public License for more details. *
* *
* You should have received a copy of the GNU General Public License *
-* along with bfps. If not, see <http://www.gnu.org/licenses/> *
+* along with TurTLE. If not, see <http://www.gnu.org/licenses/> *
* *
* Contact: Cristian.Lalescu@ds.mpg.de *
* *
@@ -24,13 +24,14 @@
-#include <vector>
-#include "base.hpp"
-
#ifndef FIELD_LAYOUT_HPP
#define FIELD_LAYOUT_HPP
+#include <vector>
+#include <hdf5.h>
+#include "base.hpp"
+
enum field_components {ONE, THREE, THREExTHREE};
constexpr unsigned int ncomp(
@@ -49,8 +50,21 @@ constexpr unsigned int ndim(
(fc == THREExTHREE) ? 5 : 3));
}
+class abstract_field_layout
+{
+ public:
+ virtual ~abstract_field_layout() noexcept(false){}
+
+ virtual MPI_Comm getMPIComm() const = 0;
+ virtual int getMPIRank() const = 0;
+
+ virtual hsize_t getSize(int component) const = 0;
+ virtual hsize_t getSubSize(int component) const = 0;
+ virtual hsize_t getStart(int component) const = 0;
+};
+
template <field_components fc>
-class field_layout
+class field_layout: public abstract_field_layout
{
public:
/* description */
@@ -72,7 +86,31 @@ class field_layout
const hsize_t *SUBSIZES,
const hsize_t *STARTS,
const MPI_Comm COMM_TO_USE);
- ~field_layout(){}
+ ~field_layout() noexcept(false){}
+
+ MPI_Comm getMPIComm() const
+ {
+ return this->comm;
+ }
+ int getMPIRank() const
+ {
+ return this->myrank;
+ }
+ hsize_t getSize(int component) const
+ {
+ assert(component >= 0 && component < int(ndim(fc)));
+ return this->sizes[component];
+ }
+ hsize_t getSubSize(int component) const
+ {
+ assert(component >= 0 && component < int(ndim(fc)));
+ return this->subsizes[component];
+ }
+ hsize_t getStart(int component) const
+ {
+ assert(component >= 0 && component < int(ndim(fc)));
+ return this->starts[component];
+ }
};
#endif//FIELD_LAYOUT_HPP
diff --git a/cpp/full_code/Gauss_field_test.cpp b/cpp/full_code/Gauss_field_test.cpp
new file mode 100644
index 0000000000000000000000000000000000000000..6f7cdfd16877f3d207dd8f1a30c5058bf2a03dfb
--- /dev/null
+++ b/cpp/full_code/Gauss_field_test.cpp
@@ -0,0 +1,219 @@
+/**********************************************************************
+* *
+* Copyright 2019 Max Planck Institute *
+* for Dynamics and Self-Organization *
+* *
+* This file is part of TurTLE. *
+* *
+* TurTLE is free software: you can redistribute it and/or modify *
+* it under the terms of the GNU General Public License as published *
+* by the Free Software Foundation, either version 3 of the License, *
+* or (at your option) any later version. *
+* *
+* TurTLE is distributed in the hope that it will be useful, *
+* but WITHOUT ANY WARRANTY; without even the implied warranty of *
+* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the *
+* GNU General Public License for more details. *
+* *
+* You should have received a copy of the GNU General Public License *
+* along with TurTLE. If not, see <http://www.gnu.org/licenses/> *
+* *
+* Contact: Cristian.Lalescu@ds.mpg.de *
+* *
+**********************************************************************/
+
+
+
+#include <string>
+#include <cmath>
+#include <random>
+#include "Gauss_field_test.hpp"
+#include "scope_timer.hpp"
+#include "hdf5_tools.hpp"
+
+
+
+
+
+template <typename rnumber>
+int Gauss_field_test<rnumber>::initialize(void)
+{
+ TIMEZONE("Gauss_field_test::initialize");
+ this->read_parameters();
+ this->scalar_field = new field<rnumber, FFTW, ONE>(
+ nx, ny, nz,
+ this->comm,
+ FFTW_ESTIMATE);
+ this->vector_field = new field<rnumber, FFTW, THREE>(
+ nx, ny, nz,
+ this->comm,
+ FFTW_ESTIMATE);
+ this->kk = new kspace<FFTW, SMOOTH>(
+ this->scalar_field->clayout, this->dkx, this->dky, this->dkz);
+
+ if (this->myrank == 0)
+ {
+ hid_t stat_file = H5Fopen(
+ (this->simname + std::string(".h5")).c_str(),
+ H5F_ACC_RDWR,
+ H5P_DEFAULT);
+ this->kk->store(stat_file);
+ H5Fclose(stat_file);
+ }
+ return EXIT_SUCCESS;
+}
+
+template <typename rnumber>
+int Gauss_field_test<rnumber>::finalize(void)
+{
+ TIMEZONE("Gauss_field_test::finalize");
+ delete this->vector_field;
+ delete this->scalar_field;
+ delete this->kk;
+ return EXIT_SUCCESS;
+}
+
+template <typename rnumber>
+int Gauss_field_test<rnumber>::read_parameters()
+{
+ TIMEZONE("Gauss_field_test::read_parameters");
+ this->test::read_parameters();
+ hid_t parameter_file = H5Fopen(
+ (this->simname + std::string(".h5")).c_str(),
+ H5F_ACC_RDONLY,
+ H5P_DEFAULT);
+ this->max_velocity_estimate = hdf5_tools::read_value<double>(parameter_file, "/parameters/max_velocity_estimate");
+ this->spectrum_dissipation = hdf5_tools::read_value<double>(parameter_file, "parameters/spectrum_dissipation");
+ this->spectrum_Lint = hdf5_tools::read_value<double>(parameter_file, "parameters/spectrum_Lint");
+ this->spectrum_etaK = hdf5_tools::read_value<double>(parameter_file, "parameters/spectrum_etaK");
+ this->spectrum_large_scale_const = hdf5_tools::read_value<double>(parameter_file, "parameters/spectrum_large_scale_const");
+ this->spectrum_small_scale_const = hdf5_tools::read_value<double>(parameter_file, "parameters/spectrum_small_scale_const");
+ this->random_seed = hdf5_tools::read_value<int>(parameter_file, "/parameters/field_random_seed");
+ this->output_incompressible_field = hdf5_tools::read_value<int>(parameter_file, "/parameters/output_incompressible_field");
+ H5Fclose(parameter_file);
+ MPI_Barrier(this->comm);
+ return EXIT_SUCCESS;
+}
+
+template <typename rnumber>
+int Gauss_field_test<rnumber>::do_work(void)
+{
+ TIMEZONE("Gauss_field_test::do_work");
+
+ /// initialize vector Gaussian random field
+ make_gaussian_random_field(
+ this->kk,
+ this->vector_field,
+ this->random_seed,
+ this->spectrum_dissipation,
+ this->spectrum_Lint,
+ this->spectrum_etaK,
+ this->spectrum_large_scale_const,
+ this->spectrum_small_scale_const,
+ 3./2. //incompressibility projection factor
+ );
+
+ /// impose divergence free condition while maintaining the energy of the field
+ this->kk->template project_divfree<rnumber>(this->vector_field->get_cdata());
+
+ if (this->output_incompressible_field == 1)
+ {
+ this->vector_field->io(
+ (std::string(this->simname) +
+ std::string("_checkpoint_") +
+ std::to_string(0) +
+ std::string(".h5")),
+ "vorticity",
+ 0,
+ false);
+ }
+
+ /// initialize statistics file.
+ hid_t stat_group, stat_file;
+ if (this->myrank == 0)
+ {
+ // set caching parameters
+ hid_t fapl = H5Pcreate(H5P_FILE_ACCESS);
+ herr_t cache_err = H5Pset_cache(fapl, 0, 521, 134217728, 1.0);
+ variable_used_only_in_assert(cache_err);
+ DEBUG_MSG("when setting stat_file cache I got %d\n", cache_err);
+ stat_file = H5Fopen(
+ (this->simname + ".h5").c_str(),
+ H5F_ACC_RDWR,
+ fapl);
+ stat_group = H5Gopen(
+ stat_file,
+ "statistics",
+ H5P_DEFAULT);
+ }
+ else
+ {
+ stat_file = 0;
+ stat_group = 0;
+ }
+
+ /// compute gradient and divergence of field
+ field<rnumber, FFTW, THREExTHREE> *tensor_field = new field<rnumber, FFTW, THREExTHREE>(
+ nx, ny, nz,
+ this->comm,
+ FFTW_ESTIMATE);
+ compute_gradient(
+ this->kk,
+ this->vector_field,
+ tensor_field);
+ compute_divergence(
+ this->kk,
+ this->vector_field,
+ this->scalar_field);
+
+
+ /// compute integral of premultiplied energy spectrum
+ this->kk->template cospectrum<rnumber, THREE>(
+ this->vector_field->get_cdata(),
+ stat_group,
+ "k*velocity_k*velocity",
+ 0,
+ 2.);
+ this->kk->template cospectrum<rnumber, THREE>(
+ this->vector_field->get_cdata(),
+ stat_group,
+ "k2*velocity_k2*velocity",
+ 0,
+ 4.);
+
+ /// compute basic statistics of Gaussian field
+ this->vector_field->compute_stats(
+ this->kk,
+ stat_group,
+ "velocity",
+ 0,
+ this->max_velocity_estimate);
+
+ /// verify divergence free method
+ tensor_field->ift();
+ /// for the stats max value doesn't really matter, I just want the moments
+ tensor_field->compute_rspace_stats(
+ stat_group,
+ "velocity_gradient",
+ 0,
+ std::vector<double>({1, 1, 1, 1, 1, 1, 1, 1, 1}));
+ this->scalar_field->ift();
+ this->scalar_field->compute_rspace_stats(
+ stat_group,
+ "velocity_divergence",
+ 0,
+ std::vector<double>({1}));
+ delete tensor_field;
+
+ /// close stat file
+ if (this->myrank == 0)
+ {
+ H5Gclose(stat_group);
+ H5Fclose(stat_file);
+ }
+ return EXIT_SUCCESS;
+}
+
+template class Gauss_field_test<float>;
+template class Gauss_field_test<double>;
+
diff --git a/cpp/full_code/Gauss_field_test.hpp b/cpp/full_code/Gauss_field_test.hpp
new file mode 100644
index 0000000000000000000000000000000000000000..311e7a8c9f31f8172114d85242fdb519155f7856
--- /dev/null
+++ b/cpp/full_code/Gauss_field_test.hpp
@@ -0,0 +1,77 @@
+/**********************************************************************
+* *
+* Copyright 2019 Max Planck Institute *
+* for Dynamics and Self-Organization *
+* *
+* This file is part of TurTLE. *
+* *
+* TurTLE is free software: you can redistribute it and/or modify *
+* it under the terms of the GNU General Public License as published *
+* by the Free Software Foundation, either version 3 of the License, *
+* or (at your option) any later version. *
+* *
+* TurTLE is distributed in the hope that it will be useful, *
+* but WITHOUT ANY WARRANTY; without even the implied warranty of *
+* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the *
+* GNU General Public License for more details. *
+* *
+* You should have received a copy of the GNU General Public License *
+* along with TurTLE. If not, see <http://www.gnu.org/licenses/> *
+* *
+* Contact: Cristian.Lalescu@ds.mpg.de *
+* *
+**********************************************************************/
+
+
+
+#ifndef GAUSS_FIELD_TEST_HPP
+#define GAUSS_FIELD_TEST_HPP
+
+
+
+#include <cstdlib>
+#include "base.hpp"
+#include "kspace.hpp"
+#include "field.hpp"
+#include "full_code/test.hpp"
+
+/** \brief A test of incompressible Gaussian random fields.
+ *
+ */
+
+template <typename rnumber>
+class Gauss_field_test: public test
+{
+ public:
+
+ /* parameters that are read in read_parameters */
+ double max_velocity_estimate;
+ double spectrum_dissipation;
+ double spectrum_Lint;
+ double spectrum_etaK;
+ double spectrum_large_scale_const;
+ double spectrum_small_scale_const;
+ int output_incompressible_field;
+ int random_seed;
+
+ /* other stuff */
+ kspace<FFTW, SMOOTH> *kk;
+ field<rnumber, FFTW, ONE> *scalar_field;
+ field<rnumber, FFTW, THREE> *vector_field;
+
+ Gauss_field_test(
+ const MPI_Comm COMMUNICATOR,
+ const std::string &simulation_name):
+ test(
+ COMMUNICATOR,
+ simulation_name){}
+ ~Gauss_field_test(){}
+
+ int initialize(void);
+ int do_work(void);
+ int finalize(void);
+ int read_parameters(void);
+};
+
+#endif//GAUSS_FIELD_TEST_HPP
+
diff --git a/cpp/full_code/NSE.cpp b/cpp/full_code/NSE.cpp
new file mode 100644
index 0000000000000000000000000000000000000000..2e1c0b68e0b36b328692a6c2155bc17a840c5929
--- /dev/null
+++ b/cpp/full_code/NSE.cpp
@@ -0,0 +1,612 @@
+/******************************************************************************
+* *
+* Copyright 2019 Max Planck Institute for Dynamics and Self-Organization *
+* *
+* This file is part of bfps. *
+* *
+* bfps is free software: you can redistribute it and/or modify *
+* it under the terms of the GNU General Public License as published *
+* by the Free Software Foundation, either version 3 of the License, *
+* or (at your option) any later version. *
+* *
+* bfps is distributed in the hope that it will be useful, *
+* but WITHOUT ANY WARRANTY; without even the implied warranty of *
+* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the *
+* GNU General Public License for more details. *
+* *
+* You should have received a copy of the GNU General Public License *
+* along with bfps. If not, see <http://www.gnu.org/licenses/> *
+* *
+* Contact: Cristian.Lalescu@ds.mpg.de *
+* *
+******************************************************************************/
+
+
+
+#include <string>
+#include <cmath>
+#include "NSE.hpp"
+#include "scope_timer.hpp"
+#include "fftw_tools.hpp"
+#include "hdf5_tools.hpp"
+#include "shared_array.hpp"
+
+
+template <typename rnumber>
+int NSE<rnumber>::initialize(void)
+{
+ TIMEZONE("NSE::initialize");
+ this->read_iteration();
+ this->read_parameters();
+ if (this->myrank == 0)
+ {
+ // set caching parameters
+ hid_t fapl = H5Pcreate(H5P_FILE_ACCESS);
+ herr_t cache_err = H5Pset_cache(fapl, 0, 521, 134217728, 1.0);
+ variable_used_only_in_assert(cache_err);
+ DEBUG_MSG("when setting stat_file cache I got %d\n", cache_err);
+ this->stat_file = H5Fopen(
+ (this->simname + ".h5").c_str(),
+ H5F_ACC_RDWR,
+ fapl);
+ }
+ int data_file_problem;
+ if (this->myrank == 0)
+ data_file_problem = this->grow_file_datasets();
+ MPI_Bcast(&data_file_problem, 1, MPI_INT, 0, this->comm);
+ if (data_file_problem > 0)
+ {
+ std::cerr <<
+ data_file_problem <<
+ " problems growing file datasets.\ntrying to exit now." <<
+ std::endl;
+ return EXIT_FAILURE;
+ }
+
+ /* allocate vector fields */
+ this->velocity = new field<rnumber, FFTW, THREE>(
+ this->nx, this->ny, this->nz,
+ this->comm,
+ fftw_planner_string_to_flag[this->fftw_plan_rigor]);
+
+ this->tmp_velocity1 = new field<rnumber, FFTW, THREE>(
+ this->nx, this->ny, this->nz,
+ this->comm,
+ fftw_planner_string_to_flag[this->fftw_plan_rigor]);
+
+ this->tmp_velocity2 = new field<rnumber, FFTW, THREE>(
+ this->nx, this->ny, this->nz,
+ this->comm,
+ fftw_planner_string_to_flag[this->fftw_plan_rigor]);
+
+ this->tmp_vec_field0 = new field<rnumber, FFTW, THREE>(
+ this->nx, this->ny, this->nz,
+ this->comm,
+ fftw_planner_string_to_flag[this->fftw_plan_rigor]);
+
+ this->tmp_vec_field1 = new field<rnumber, FFTW, THREE>(
+ this->nx, this->ny, this->nz,
+ this->comm,
+ fftw_planner_string_to_flag[this->fftw_plan_rigor]);
+
+ /* initialize kspace */
+ this->kk = new kspace<FFTW, SMOOTH>(
+ this->velocity->clayout, this->dkx, this->dky, this->dkz);
+
+ {
+ this->velocity->real_space_representation = false;
+ this->velocity->io(
+ this->get_current_fname(),
+ "velocity",
+ this->iteration,
+ true);
+ TIMEZONE("NSE::initialize::read_initial_condition");
+ #if (__GNUC__ <= 4 && __GNUC_MINOR__ < 7)
+ this->kk->low_pass<rnumber, THREE>(
+ this->velocity->get_cdata(),
+ this->kk->kM);
+ this->kk->force_divfree<rnumber>(
+ this->velocity->get_cdata());
+ #else
+ this->kk->template low_pass<rnumber, THREE>(
+ this->velocity->get_cdata(),
+ this->kk->kM);
+ this->kk->template force_divfree<rnumber>(
+ this->velocity->get_cdata());
+ #endif
+ this->velocity->symmetrize();
+ MPI_Barrier(this->comm);
+ }
+
+ if (this->myrank == 0 && this->iteration == 0)
+ this->kk->store(stat_file);
+ return EXIT_SUCCESS;
+}
+
+template <typename rnumber>
+int NSE<rnumber>::step(void)
+{
+ TIMEZONE("NSE::step");
+ this->iteration++;
+ //return this->Euler_step();
+ return this->ShuOsher();
+ return EXIT_SUCCESS;
+}
+
+template <typename rnumber>
+int NSE<rnumber>::add_field_band(
+ field<rnumber, FFTW, THREE> *dst,
+ field<rnumber, FFTW, THREE> *src,
+ const double k0,
+ const double k1,
+ const double prefactor)
+{
+ TIMEZONE("NSE::add_field_band");
+ this->kk->CLOOP(
+ [&](const ptrdiff_t cindex,
+ const ptrdiff_t xindex,
+ const ptrdiff_t yindex,
+ const ptrdiff_t zindex){
+ const double knorm = sqrt(this->kk->kx[xindex]*this->kk->kx[xindex] +
+ this->kk->ky[yindex]*this->kk->ky[yindex] +
+ this->kk->kz[zindex]*this->kk->kz[zindex]);
+ if ((k0 <= knorm) &&
+ (k1 >= knorm))
+ for (int c=0; c<3; c++)
+ for (int i=0; i<2; i++)
+ dst->cval(cindex,c,i) += prefactor*src->cval(cindex,c,i);
+ }
+ );
+ return EXIT_SUCCESS;
+}
+
+template <typename rnumber>
+int NSE<rnumber>::add_forcing_term(
+ field<rnumber, FFTW, THREE> *vel,
+ field<rnumber, FFTW, THREE> *dst)
+{
+ TIMEZONE("NSE::add_forcing_term");
+ if (this->forcing_type == std::string("linear"))
+ {
+ return this->add_field_band(
+ dst, this->velocity,
+ this->fk0, this->fk1,
+ this->famplitude);
+ }
+ if ((this->forcing_type == std::string("fixed_energy_injection_rate")) ||
+ (this->forcing_type == std::string("fixed_energy_injection_rate_and_drag")))
+ {
+ // first, compute energy in shell
+ shared_array<double> local_energy_in_shell(1);
+ double energy_in_shell = 0;
+ this->kk->CLOOP_K2_NXMODES(
+ [&](const ptrdiff_t cindex,
+ const ptrdiff_t xindex,
+ const ptrdiff_t yindex,
+ const ptrdiff_t zindex,
+ const double k2,
+ const int nxmodes){
+ const double knorm = sqrt(k2);
+ if ((k2 > 0) &&
+ (this->fk0 <= knorm) &&
+ (this->fk1 >= knorm))
+ *local_energy_in_shell.getMine() += nxmodes*(
+ vel->cval(cindex, 0, 0)*vel->cval(cindex, 0, 0) + vel->cval(cindex, 0, 1)*vel->cval(cindex, 0, 1) +
+ vel->cval(cindex, 1, 0)*vel->cval(cindex, 1, 0) + vel->cval(cindex, 1, 1)*vel->cval(cindex, 1, 1) +
+ vel->cval(cindex, 2, 0)*vel->cval(cindex, 2, 0) + vel->cval(cindex, 2, 1)*vel->cval(cindex, 2, 1)
+ );
+ }
+ );
+ local_energy_in_shell.mergeParallel();
+ MPI_Allreduce(
+ local_energy_in_shell.getMasterData(),
+ &energy_in_shell,
+ 1,
+ MPI_DOUBLE,
+ MPI_SUM,
+ vel->comm);
+ // we should divide by 2, if we wanted energy;
+ // but then we would need to multiply the amplitude by 2 anyway,
+ // because what we really care about is force dotted into velocity,
+ // without the division by 2.
+
+ // now, modify amplitudes
+ if (energy_in_shell < 10*std::numeric_limits<rnumber>::epsilon())
+ energy_in_shell = 1;
+ double temp_famplitude = this->injection_rate / energy_in_shell;
+ this->add_field_band(
+ dst, vel,
+ this->fk0, this->fk1,
+ temp_famplitude);
+ // and add drag if desired
+ if (this->forcing_type == std::string("fixed_energy_injection_rate_and_drag"))
+ this->add_field_band(
+ dst, vel,
+ this->fmode, this->fmode + (this->fk1 - this->fk0),
+ -this->friction_coefficient);
+ return EXIT_SUCCESS;
+ }
+ if (this->forcing_type == std::string("fixed_energy"))
+ return EXIT_SUCCESS;
+ else
+ {
+ DEBUG_MSG("unknown forcing type printed on next line\n%s", this->forcing_type.c_str());
+ throw std::invalid_argument("unknown forcing type");
+ }
+ return EXIT_SUCCESS;
+}
+
+template <typename rnumber>
+int NSE<rnumber>::impose_forcing(
+ field<rnumber, FFTW, THREE> *dst)
+{
+ TIMEZONE("NSE::impose_forcing");
+ return EXIT_SUCCESS;
+}
+
+template <typename rnumber>
+int NSE<rnumber>::add_nonlinear_term(
+ const field<rnumber, FFTW, THREE> *vel,
+ field<rnumber, FFTW, THREE> *dst)
+{
+ TIMEZONE("NSE::add_nonlinear_term");
+ dst->real_space_representation = false;
+
+ /* This computes the nonlinear term coming from vel, for the Navier Stokes
+ * equations. Using vel=\f$u\f$ and dst=\f$a^{nl}\f$, we compute
+ * \f[
+ * a^{nl}_i = - u_j \partial_j u_i
+ * \f]
+ * The result is given in Fourier representation.
+ * */
+ const int sign_array[2] = {1, -1};
+
+ /* copy velocity */
+ this->tmp_vec_field0->real_space_representation = false;
+ *this->tmp_vec_field0 = *vel;
+
+ // put velocity in real space
+ this->tmp_vec_field0->ift();
+
+ /* compute uu */
+ /* store 11 22 33 components */
+ this->tmp_vec_field1->real_space_representation = true;
+ this->tmp_vec_field1->RLOOP(
+ [&](const ptrdiff_t rindex,
+ const ptrdiff_t xindex,
+ const ptrdiff_t yindex,
+ const ptrdiff_t zindex){
+ for (int cc=0; cc<3; cc++)
+ this->tmp_vec_field1->rval(rindex,cc) = (
+ this->tmp_vec_field0->rval(rindex,cc) *
+ this->tmp_vec_field0->rval(rindex,cc) ) / this->tmp_vec_field1->npoints;
+ });
+ /* take 11 22 33 components to Fourier representation */
+ this->tmp_vec_field1->dft();
+ this->kk->template dealias<rnumber, THREE>(this->tmp_vec_field1->get_cdata());
+
+ this->kk->CLOOP(
+ [&](const ptrdiff_t cindex,
+ const ptrdiff_t xindex,
+ const ptrdiff_t yindex,
+ const ptrdiff_t zindex){
+ for (int ccc = 0; ccc < 2; ccc++)
+ {
+ dst->cval(cindex, 0, ccc) += (
+ sign_array[ccc]*this->kk->kx[xindex] * this->tmp_vec_field1->cval(cindex, 0, (ccc+1)%2));
+ dst->cval(cindex, 1, ccc) += (
+ sign_array[ccc]*this->kk->ky[yindex] * this->tmp_vec_field1->cval(cindex, 1, (ccc+1)%2));
+ dst->cval(cindex, 2, ccc) += (
+ sign_array[ccc]*this->kk->kz[zindex] * this->tmp_vec_field1->cval(cindex, 2, (ccc+1)%2));
+ }
+ });
+
+ /* store 12 23 31 components */
+ this->tmp_vec_field1->real_space_representation = true;
+ this->tmp_vec_field1->RLOOP(
+ [&](const ptrdiff_t rindex,
+ const ptrdiff_t xindex,
+ const ptrdiff_t yindex,
+ const ptrdiff_t zindex){
+ for (int cc=0; cc<3; cc++)
+ this->tmp_vec_field1->rval(rindex,cc) = (
+ this->tmp_vec_field0->rval(rindex,cc) *
+ this->tmp_vec_field0->rval(rindex,(cc+1)%3) ) / this->tmp_vec_field1->npoints;
+ });
+ /* take 12 23 31 components to Fourier representation */
+ this->tmp_vec_field1->dft();
+ this->kk->template dealias<rnumber, THREE>(this->tmp_vec_field1->get_cdata());
+
+ this->kk->CLOOP(
+ [&](const ptrdiff_t cindex,
+ const ptrdiff_t xindex,
+ const ptrdiff_t yindex,
+ const ptrdiff_t zindex){
+ for (int ccc = 0; ccc < 2; ccc++)
+ {
+ dst->cval(cindex, 0, ccc) += (
+ sign_array[ccc]*this->kk->ky[yindex] * this->tmp_vec_field1->cval(cindex, 0, (ccc+1)%2) +
+ sign_array[ccc]*this->kk->kz[zindex] * this->tmp_vec_field1->cval(cindex, 2, (ccc+1)%2));
+ dst->cval(cindex, 1, ccc) += (
+ sign_array[ccc]*this->kk->kz[zindex] * this->tmp_vec_field1->cval(cindex, 1, (ccc+1)%2) +
+ sign_array[ccc]*this->kk->kx[xindex] * this->tmp_vec_field1->cval(cindex, 0, (ccc+1)%2));
+ dst->cval(cindex, 2, ccc) += (
+ sign_array[ccc]*this->kk->kx[xindex] * this->tmp_vec_field1->cval(cindex, 2, (ccc+1)%2) +
+ sign_array[ccc]*this->kk->ky[yindex] * this->tmp_vec_field1->cval(cindex, 1, (ccc+1)%2));
+ }
+ });
+
+ dst->symmetrize();
+ /* done */
+
+ return EXIT_SUCCESS;
+}
+
+template <typename rnumber>
+int NSE<rnumber>::Euler_step(void)
+{
+ // technically this is not a pure Euler method,
+ // because I use an exponential factor for the diffusion term
+ TIMEZONE("NSE::Euler_step");
+
+ // temporary field
+ field<rnumber, FFTW, THREE> *acc = new field<rnumber, FFTW, THREE>(
+ this->nx, this->ny, this->nz,
+ this->comm,
+ fftw_planner_string_to_flag[this->fftw_plan_rigor]);
+
+ *acc = rnumber(0);
+
+ // add nonlinear term
+ this->add_nonlinear_term(
+ this->velocity,
+ acc);
+
+ // add forcing
+ this->add_forcing_term(this->velocity, acc);
+
+ // perform Euler step
+ this->kk->CLOOP(
+ [&](const ptrdiff_t cindex,
+ const ptrdiff_t xindex,
+ const ptrdiff_t yindex,
+ const ptrdiff_t zindex){
+ for (int cc = 0; cc < 3; cc++)
+ for (int ccc = 0; ccc < 2; ccc++)
+ {
+ this->velocity->cval(cindex, cc, ccc) += this->dt*acc->cval(cindex, cc, ccc);
+ }
+ });
+
+ // enforce incompressibility
+ this->kk->template force_divfree<rnumber>(this->velocity->get_cdata());
+
+ // apply diffusion exponential
+ this->kk->CLOOP_K2(
+ [&](const ptrdiff_t cindex,
+ const ptrdiff_t xindex,
+ const ptrdiff_t yindex,
+ const ptrdiff_t zindex,
+ const double k2){
+ const double factor = exp(-this->nu*k2 * this->dt);
+ for (int cc = 0; cc < 3; cc++)
+ for (int ccc = 0; ccc < 2; ccc++)
+ {
+ this->velocity->cval(cindex, cc, ccc) *= factor;
+ }
+ });
+
+ // enforce Hermitian symmetry
+ this->velocity->symmetrize();
+
+ // delete temporary field
+ delete acc;
+ return EXIT_SUCCESS;
+}
+
+/** \brief Time step with Shu Osher method
+ *
+ * The Navier Stokes equations are integrated with a
+ * third-order Runge-Kutta method [shu1988jcp]_, which is an explicit
+ * Runge-Kutta method with the Butcher tableau
+ *
+ * | | | | |
+ * |:--|:--|:--|:--|
+ * | 0 | | | |
+ * | 1 | 1 | | |
+ * | 1/2 | 1/4 | 1/4 | |
+ * | | 1/6 | 1/6 | 2/3 |
+ *
+ * In addition to the stability properties described in [shu1988jcp]_, this method has the advantage that it is memory-efficient, requiring only two additional field allocations, as can be seen from
+ * \f{eqnarray*}{
+ * \hat{\boldsymbol w}_1(\boldsymbol k) &= {\hat{\boldsymbol u}}(\boldsymbol k, t)e^{-\nu k^2 h} + h \textrm{NL}[\hat{\boldsymbol u}(\boldsymbol k, t)]e^{-\nu k^2 h}, \\
+ * \hat{\boldsymbol w}_2(\boldsymbol k) &= \frac{3}{4}\hat{\boldsymbol u}( \boldsymbol k, t)e^{-\nu k^2 h/2}
+ * +
+ * \frac{1}{4}(\hat{\boldsymbol w}_1(\boldsymbol k) + h \textrm{NL}[\hat{\boldsymbol w}_1(\boldsymbol k)])e^{\nu k^2 h/2}, \\
+ * \hat{\boldsymbol u}(\boldsymbol k, t+h) &= \frac{1}{3}\hat{\boldsymbol u}(\boldsymbol k, t)e^{-\nu k^2 h} +
+ * \frac{2}{3}(\hat{\boldsymbol w}_2(\boldsymbol k) + h \textrm{NL}[\hat{\boldsymbol w}_2(\boldsymbol k)])e^{-\nu k^2 h/2},
+ * \f}
+ */
+template <typename rnumber>
+int NSE<rnumber>::ShuOsher(void)
+{
+ *this->tmp_velocity1 = rnumber(0);
+ this->add_nonlinear_term(this->velocity, this->tmp_velocity1);
+ this->add_forcing_term(this->velocity, this->tmp_velocity1);
+ this->kk->CLOOP_K2(
+ [&](const ptrdiff_t cindex,
+ const ptrdiff_t xindex,
+ const ptrdiff_t yindex,
+ const ptrdiff_t zindex,
+ const double k2){
+ const double factor = exp(-this->nu*k2*this->dt);
+ for (int cc = 0; cc < 3; cc++)
+ for (int ccc = 0; ccc < 2; ccc++)
+ {
+ this->tmp_velocity1->cval(cindex, cc, ccc) = factor*(
+ this->velocity->cval(cindex, cc, ccc) +
+ this->dt*this->tmp_velocity1->cval(cindex, cc, ccc));
+ }});
+ this->impose_forcing(this->tmp_velocity1);
+ this->kk->template force_divfree<rnumber>(this->tmp_velocity1->get_cdata());
+
+ *this->tmp_velocity2 = rnumber(0);
+ this->add_nonlinear_term(this->tmp_velocity1, this->tmp_velocity2);
+ this->add_forcing_term(this->tmp_velocity1, this->tmp_velocity2);
+ this->kk->CLOOP_K2(
+ [&](const ptrdiff_t cindex,
+ const ptrdiff_t xindex,
+ const ptrdiff_t yindex,
+ const ptrdiff_t zindex,
+ const double k2){
+ const double factor0 = exp(-0.5*this->nu*k2*this->dt)*0.75;
+ const double factor1 = exp( 0.5*this->nu*k2*this->dt)*0.25;
+ for (int cc = 0; cc < 3; cc++)
+ for (int ccc = 0; ccc < 2; ccc++)
+ {
+ this->tmp_velocity2->cval(cindex, cc, ccc) = (
+ this->velocity->cval(cindex, cc, ccc)*factor0 +
+ (this->tmp_velocity1->cval(cindex, cc, ccc) +
+ this->dt*this->tmp_velocity2->cval(cindex, cc, ccc))*factor1);
+ }});
+ this->impose_forcing(this->tmp_velocity2);
+ this->kk->template force_divfree<rnumber>(this->tmp_velocity2->get_cdata());
+
+ *this->tmp_velocity1 = rnumber(0);
+ this->add_nonlinear_term(this->tmp_velocity2, this->tmp_velocity1);
+ this->add_forcing_term(this->tmp_velocity2, this->tmp_velocity1);
+ this->kk->CLOOP_K2(
+ [&](const ptrdiff_t cindex,
+ const ptrdiff_t xindex,
+ const ptrdiff_t yindex,
+ const ptrdiff_t zindex,
+ const double k2){
+ const double factor0 = exp(- this->nu*k2*this->dt) / 3;
+ const double factor1 = 2*exp(-0.5*this->nu*k2*this->dt) / 3;
+ for (int cc = 0; cc < 3; cc++)
+ for (int ccc = 0; ccc < 2; ccc++)
+ {
+ this->velocity->cval(cindex, cc, ccc) = (
+ this->velocity->cval(cindex, cc, ccc)*factor0 +
+ (this->tmp_velocity2->cval(cindex, cc, ccc) +
+ this->dt*this->tmp_velocity1->cval(cindex, cc, ccc))*factor1);
+ }});
+ this->impose_forcing(this->velocity);
+ this->kk->template force_divfree<rnumber>(this->velocity->get_cdata());
+
+ this->velocity->symmetrize();
+ return EXIT_SUCCESS;
+}
+
+template <typename rnumber>
+int NSE<rnumber>::write_checkpoint(void)
+{
+ TIMEZONE("NSE::write_checkpoint");
+ this->update_checkpoint();
+
+ assert(!this->velocity->real_space_representation);
+ std::string fname = this->get_current_fname();
+ this->velocity->io(
+ fname,
+ "velocity",
+ this->iteration,
+ false);
+
+ this->write_iteration();
+ return EXIT_SUCCESS;
+}
+
+template <typename rnumber>
+int NSE<rnumber>::finalize(void)
+{
+ TIMEZONE("NSE::finalize");
+ if (this->myrank == 0)
+ H5Fclose(this->stat_file);
+ delete this->kk;
+ delete this->tmp_velocity1;
+ delete this->tmp_velocity2;
+ delete this->tmp_vec_field1;
+ delete this->tmp_vec_field0;
+ delete this->velocity;
+ return EXIT_SUCCESS;
+}
+
+/** \brief Compute standard statistics for velocity and vorticity fields.
+ *
+ * IMPORTANT: at the end of this subroutine, `this->fs->cvelocity` contains
+ * the Fourier space representation of the velocity field, and
+ * `this->tmp_vec_field` contains the real space representation of the
+ * velocity field.
+ * This behavior is relied upon in the `NSEparticles` class, so please
+ * don't break it.
+ */
+
+template <typename rnumber>
+int NSE<rnumber>::do_stats()
+{
+ TIMEZONE("NSE::do_stats");
+ if (!(this->iteration % this->niter_stat == 0))
+ return EXIT_SUCCESS;
+ hid_t stat_group;
+ if (this->myrank == 0)
+ stat_group = H5Gopen(
+ this->stat_file,
+ "statistics",
+ H5P_DEFAULT);
+ else
+ stat_group = 0;
+
+ this->tmp_vec_field0->real_space_representation = false;
+ *this->tmp_vec_field0 = *this->velocity;
+
+ this->tmp_vec_field0->compute_stats(
+ this->kk,
+ stat_group,
+ "velocity",
+ this->iteration / niter_stat,
+ this->max_velocity_estimate/sqrt(3));
+
+ compute_curl(this->kk,
+ this->velocity,
+ this->tmp_vec_field0);
+
+ this->tmp_vec_field0->compute_stats(
+ this->kk,
+ stat_group,
+ "vorticity",
+ this->iteration / niter_stat,
+ this->max_vorticity_estimate/sqrt(3));
+
+ if (this->myrank == 0)
+ H5Gclose(stat_group);
+ return EXIT_SUCCESS;
+}
+
+template <typename rnumber>
+int NSE<rnumber>::read_parameters(void)
+{
+ TIMEZONE("NSE::read_parameters");
+ this->direct_numerical_simulation::read_parameters();
+ hid_t parameter_file = H5Fopen((this->simname + ".h5").c_str(), H5F_ACC_RDONLY, H5P_DEFAULT);
+ this->nu = hdf5_tools::read_value<double>(parameter_file, "parameters/nu");
+ this->dt = hdf5_tools::read_value<double>(parameter_file, "parameters/dt");
+ this->fmode = hdf5_tools::read_value<int>(parameter_file, "parameters/fmode");
+ this->famplitude = hdf5_tools::read_value<double>(parameter_file, "parameters/famplitude");
+ this->friction_coefficient = hdf5_tools::read_value<double>(parameter_file, "parameters/friction_coefficient");
+ this->injection_rate = hdf5_tools::read_value<double>(parameter_file, "parameters/injection_rate");
+ this->fk0 = hdf5_tools::read_value<double>(parameter_file, "parameters/fk0");
+ this->fk1 = hdf5_tools::read_value<double>(parameter_file, "parameters/fk1");
+ this->energy = hdf5_tools::read_value<double>(parameter_file, "parameters/energy");
+ this->histogram_bins = hdf5_tools::read_value<int>(parameter_file, "parameters/histogram_bins");
+ this->max_velocity_estimate = hdf5_tools::read_value<double>(parameter_file, "parameters/max_velocity_estimate");
+ this->max_vorticity_estimate = hdf5_tools::read_value<double>(parameter_file, "parameters/max_vorticity_estimate");
+ this->forcing_type = hdf5_tools::read_string(parameter_file, "parameters/forcing_type");
+ this->fftw_plan_rigor = hdf5_tools::read_string(parameter_file, "parameters/fftw_plan_rigor");
+ H5Fclose(parameter_file);
+ MPI_Barrier(this->comm);
+ return EXIT_SUCCESS;
+}
+
+template class NSE<float>;
+template class NSE<double>;
+
diff --git a/cpp/full_code/NSE.hpp b/cpp/full_code/NSE.hpp
new file mode 100644
index 0000000000000000000000000000000000000000..0ab0d3d6c6ea72ac4d92b41d648f14b861003bab
--- /dev/null
+++ b/cpp/full_code/NSE.hpp
@@ -0,0 +1,111 @@
+/**********************************************************************
+* *
+* Copyright 2017 Max Planck Institute *
+* for Dynamics and Self-Organization *
+* *
+* This file is part of bfps. *
+* *
+* bfps is free software: you can redistribute it and/or modify *
+* it under the terms of the GNU General Public License as published *
+* by the Free Software Foundation, either version 3 of the License, *
+* or (at your option) any later version. *
+* *
+* bfps is distributed in the hope that it will be useful, *
+* but WITHOUT ANY WARRANTY; without even the implied warranty of *
+* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the *
+* GNU General Public License for more details. *
+* *
+* You should have received a copy of the GNU General Public License *
+* along with bfps. If not, see <http://www.gnu.org/licenses/> *
+* *
+* Contact: Cristian.Lalescu@ds.mpg.de *
+* *
+**********************************************************************/
+
+
+
+#ifndef NSE_HPP
+#define NSE_HPP
+
+
+
+#include <cstdlib>
+#include "base.hpp"
+#include "vorticity_equation.hpp"
+#include "full_code/direct_numerical_simulation.hpp"
+
+/** \brief Navier-Stokes solver.
+ *
+ * Solves the Navier Stokes equation.
+ *
+ * Allocates fields and a kspace object.
+ */
+
+
+template <typename rnumber>
+class NSE: public direct_numerical_simulation
+{
+ public:
+ std::string name;
+
+ /* parameters that are read in read_parameters */
+ double dt;
+ double famplitude;
+ double friction_coefficient;
+ double fk0;
+ double fk1;
+ double energy;
+ double injection_rate;
+ int fmode;
+ std::string forcing_type;
+ int histogram_bins;
+ double max_velocity_estimate;
+ double max_vorticity_estimate;
+ double nu;
+ std::string fftw_plan_rigor;
+
+ /* other stuff */
+ kspace<FFTW, SMOOTH> *kk;
+ field<rnumber, FFTW, THREE> *velocity;
+ field<rnumber, FFTW, THREE> *tmp_velocity1; // for timestep
+ field<rnumber, FFTW, THREE> *tmp_velocity2; // for timestep
+ field<rnumber, FFTW, THREE> *tmp_vec_field0; // can be changed during methods
+ field<rnumber, FFTW, THREE> *tmp_vec_field1; // can be changed during methods
+
+ NSE(
+ const MPI_Comm COMMUNICATOR,
+ const std::string &simulation_name):
+ direct_numerical_simulation(
+ COMMUNICATOR,
+ simulation_name){}
+ ~NSE(){}
+
+ int initialize(void);
+ int step(void);
+ int finalize(void);
+
+ int Euler_step(void);
+ int ShuOsher(void);
+
+ int add_field_band(
+ field<rnumber, FFTW, THREE> *dst,
+ field<rnumber, FFTW, THREE> *src,
+ const double k0,
+ const double k1,
+ const double prefactor);
+ virtual int add_nonlinear_term(
+ const field<rnumber, FFTW, THREE> *vel,
+ field<rnumber, FFTW, THREE> *dst);
+ int add_forcing_term(
+ field<rnumber, FFTW, THREE> *vel,
+ field<rnumber, FFTW, THREE> *dst);
+ int impose_forcing(
+ field<rnumber, FFTW, THREE> *vel);
+
+ virtual int read_parameters(void);
+ int write_checkpoint(void);
+ int do_stats(void);
+};
+
+#endif//NSE_HPP
+
diff --git a/cpp/full_code/NSVE.cpp b/cpp/full_code/NSVE.cpp
new file mode 100644
index 0000000000000000000000000000000000000000..09ca7f286be9d1d17ba3bce5125187bf88fb8024
--- /dev/null
+++ b/cpp/full_code/NSVE.cpp
@@ -0,0 +1,202 @@
+/******************************************************************************
+* *
+* Copyright 2019 Max Planck Institute for Dynamics and Self-Organization *
+* *
+* This file is part of TurTLE. *
+* *
+* TurTLE is free software: you can redistribute it and/or modify *
+* it under the terms of the GNU General Public License as published *
+* by the Free Software Foundation, either version 3 of the License, *
+* or (at your option) any later version. *
+* *
+* TurTLE is distributed in the hope that it will be useful, *
+* but WITHOUT ANY WARRANTY; without even the implied warranty of *
+* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the *
+* GNU General Public License for more details. *
+* *
+* You should have received a copy of the GNU General Public License *
+* along with TurTLE. If not, see <http://www.gnu.org/licenses/> *
+* *
+* Contact: Cristian.Lalescu@ds.mpg.de *
+* *
+******************************************************************************/
+
+
+
+#include <string>
+#include <cmath>
+#include "NSVE.hpp"
+#include "scope_timer.hpp"
+#include "fftw_tools.hpp"
+#include "hdf5_tools.hpp"
+
+
+template <typename rnumber>
+int NSVE<rnumber>::initialize(void)
+{
+ TIMEZONE("NSVE::initialize");
+ this->read_iteration();
+ this->read_parameters();
+ if (this->myrank == 0)
+ {
+ // set caching parameters
+ hid_t fapl = H5Pcreate(H5P_FILE_ACCESS);
+ herr_t cache_err = H5Pset_cache(fapl, 0, 521, 134217728, 1.0);
+ variable_used_only_in_assert(cache_err);
+ DEBUG_MSG("when setting stat_file cache I got %d\n", cache_err);
+ this->stat_file = H5Fopen(
+ (this->simname + ".h5").c_str(),
+ H5F_ACC_RDWR,
+ fapl);
+ }
+ int data_file_problem;
+ if (this->myrank == 0)
+ data_file_problem = this->grow_file_datasets();
+ MPI_Bcast(&data_file_problem, 1, MPI_INT, 0, this->comm);
+ if (data_file_problem > 0)
+ {
+ std::cerr <<
+ data_file_problem <<
+ " problems growing file datasets.\ntrying to exit now." <<
+ std::endl;
+ return EXIT_FAILURE;
+ }
+ this->fs = new vorticity_equation<rnumber, FFTW>(
+ simname.c_str(),
+ this->nx, this->ny, this->nz,
+ this->dkx, this->dky, this->dkz,
+ fftw_planner_string_to_flag[this->fftw_plan_rigor]);
+ this->tmp_vec_field = new field<rnumber, FFTW, THREE>(
+ this->nx, this->ny, this->nz,
+ this->comm,
+ fftw_planner_string_to_flag[this->fftw_plan_rigor]);
+
+
+ this->fs->checkpoints_per_file = checkpoints_per_file;
+ this->fs->nu = nu;
+ this->fs->fmode = fmode;
+ this->fs->famplitude = famplitude;
+ this->fs->friction_coefficient = friction_coefficient;
+ this->fs->energy = energy;
+ this->fs->injection_rate = injection_rate;
+ this->fs->fk0 = fk0;
+ this->fs->fk1 = fk1;
+ strncpy(this->fs->forcing_type, forcing_type, 128);
+ this->fs->iteration = this->iteration;
+ this->fs->checkpoint = this->checkpoint;
+
+ this->fs->cvorticity->real_space_representation = false;
+ this->fs->io_checkpoint();
+
+ if (this->myrank == 0 && this->iteration == 0)
+ this->fs->kk->store(stat_file);
+ return EXIT_SUCCESS;
+}
+
+template <typename rnumber>
+int NSVE<rnumber>::step(void)
+{
+ TIMEZONE("NSVE::step");
+ this->fs->step(this->dt);
+ this->iteration = this->fs->iteration;
+ return EXIT_SUCCESS;
+}
+
+template <typename rnumber>
+int NSVE<rnumber>::write_checkpoint(void)
+{
+ TIMEZONE("NSVE::write_checkpoint");
+ this->fs->io_checkpoint(false);
+ this->checkpoint = this->fs->checkpoint;
+ this->write_iteration();
+ return EXIT_SUCCESS;
+}
+
+template <typename rnumber>
+int NSVE<rnumber>::finalize(void)
+{
+ TIMEZONE("NSVE::finalize");
+ if (this->myrank == 0)
+ H5Fclose(this->stat_file);
+ delete this->fs;
+ delete this->tmp_vec_field;
+ return EXIT_SUCCESS;
+}
+
+/** \brief Compute standard statistics for velocity and vorticity fields.
+ *
+ * IMPORTANT: at the end of this subroutine, `this->fs->cvelocity` contains
+ * the Fourier space representation of the velocity field, and
+ * `this->tmp_vec_field` contains the real space representation of the
+ * velocity field.
+ * This behavior is relied upon in the `NSVEparticles` class, so please
+ * don't break it.
+ */
+
+template <typename rnumber>
+int NSVE<rnumber>::do_stats()
+{
+ TIMEZONE("NSVE::do_stats");
+ if (!(this->iteration % this->niter_stat == 0))
+ return EXIT_SUCCESS;
+ hid_t stat_group;
+ if (this->myrank == 0)
+ stat_group = H5Gopen(
+ this->stat_file,
+ "statistics",
+ H5P_DEFAULT);
+ else
+ stat_group = 0;
+
+ *tmp_vec_field = fs->cvorticity->get_cdata();
+ tmp_vec_field->compute_stats(
+ fs->kk,
+ stat_group,
+ "vorticity",
+ fs->iteration / niter_stat,
+ max_vorticity_estimate/sqrt(3));
+
+ fs->compute_velocity(fs->cvorticity);
+ *tmp_vec_field = fs->cvelocity->get_cdata();
+ tmp_vec_field->compute_stats(
+ fs->kk,
+ stat_group,
+ "velocity",
+ fs->iteration / niter_stat,
+ max_velocity_estimate/sqrt(3));
+
+ if (this->myrank == 0)
+ H5Gclose(stat_group);
+ return EXIT_SUCCESS;
+}
+
+template <typename rnumber>
+int NSVE<rnumber>::read_parameters(void)
+{
+ TIMEZONE("NSVE::read_parameters");
+ this->direct_numerical_simulation::read_parameters();
+ hid_t parameter_file = H5Fopen((this->simname + ".h5").c_str(), H5F_ACC_RDONLY, H5P_DEFAULT);
+ this->nu = hdf5_tools::read_value<double>(parameter_file, "parameters/nu");
+ this->dt = hdf5_tools::read_value<double>(parameter_file, "parameters/dt");
+ this->fmode = hdf5_tools::read_value<int>(parameter_file, "parameters/fmode");
+ this->famplitude = hdf5_tools::read_value<double>(parameter_file, "parameters/famplitude");
+ this->friction_coefficient = hdf5_tools::read_value<double>(parameter_file, "parameters/friction_coefficient");
+ this->injection_rate = hdf5_tools::read_value<double>(parameter_file, "parameters/injection_rate");
+ this->fk0 = hdf5_tools::read_value<double>(parameter_file, "parameters/fk0");
+ this->fk1 = hdf5_tools::read_value<double>(parameter_file, "parameters/fk1");
+ this->energy = hdf5_tools::read_value<double>(parameter_file, "parameters/energy");
+ this->histogram_bins = hdf5_tools::read_value<int>(parameter_file, "parameters/histogram_bins");
+ this->max_velocity_estimate = hdf5_tools::read_value<double>(parameter_file, "parameters/max_velocity_estimate");
+ this->max_vorticity_estimate = hdf5_tools::read_value<double>(parameter_file, "parameters/max_vorticity_estimate");
+ std::string tmp = hdf5_tools::read_string(parameter_file, "parameters/forcing_type");
+ snprintf(this->forcing_type, 511, "%s", tmp.c_str());
+ this->fftw_plan_rigor = hdf5_tools::read_string(parameter_file, "parameters/fftw_plan_rigor");
+ H5Fclose(parameter_file);
+ // the following ensures the file is free after exiting this method
+ MPI_Barrier(this->comm);
+ return EXIT_SUCCESS;
+}
+
+template class NSVE<float>;
+template class NSVE<double>;
+
diff --git a/bfps/cpp/full_code/NSVE.hpp b/cpp/full_code/NSVE.hpp
similarity index 80%
rename from bfps/cpp/full_code/NSVE.hpp
rename to cpp/full_code/NSVE.hpp
index d444b71ceb48ea19dc292a57cc91ac81157e15ed..2f29f91fef5c990f1676dc547e30458c0ff97a0b 100644
--- a/bfps/cpp/full_code/NSVE.hpp
+++ b/cpp/full_code/NSVE.hpp
@@ -3,20 +3,20 @@
* Copyright 2017 Max Planck Institute *
* for Dynamics and Self-Organization *
* *
-* This file is part of bfps. *
+* This file is part of TurTLE. *
* *
-* bfps is free software: you can redistribute it and/or modify *
+* TurTLE is free software: you can redistribute it and/or modify *
* it under the terms of the GNU General Public License as published *
* by the Free Software Foundation, either version 3 of the License, *
* or (at your option) any later version. *
* *
-* bfps is distributed in the hope that it will be useful, *
+* TurTLE is distributed in the hope that it will be useful, *
* but WITHOUT ANY WARRANTY; without even the implied warranty of *
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the *
* GNU General Public License for more details. *
* *
* You should have received a copy of the GNU General Public License *
-* along with bfps. If not, see <http://www.gnu.org/licenses/> *
+* along with TurTLE. If not, see <http://www.gnu.org/licenses/> *
* *
* Contact: Cristian.Lalescu@ds.mpg.de *
* *
@@ -34,6 +34,14 @@
#include "vorticity_equation.hpp"
#include "full_code/direct_numerical_simulation.hpp"
+/** \brief Navier-Stokes solver.
+ *
+ * Solves the vorticity equation for a Navier-Stokes fluid.
+ *
+ * Allocates 1 vorticity equation and 1 vector field.
+ */
+
+
template <typename rnumber>
class NSVE: public direct_numerical_simulation
{
@@ -42,14 +50,18 @@ class NSVE: public direct_numerical_simulation
/* parameters that are read in read_parameters */
double dt;
double famplitude;
+ double friction_coefficient;
double fk0;
double fk1;
+ double energy;
+ double injection_rate;
int fmode;
char forcing_type[512];
int histogram_bins;
double max_velocity_estimate;
double max_vorticity_estimate;
double nu;
+ std::string fftw_plan_rigor;
/* other stuff */
vorticity_equation<rnumber, FFTW> *fs;
@@ -63,7 +75,7 @@ class NSVE: public direct_numerical_simulation
direct_numerical_simulation(
COMMUNICATOR,
simulation_name){}
- ~NSVE(){}
+ ~NSVE() noexcept(false){}
int initialize(void);
int step(void);
diff --git a/cpp/full_code/NSVE_Stokes_particles.cpp b/cpp/full_code/NSVE_Stokes_particles.cpp
new file mode 100644
index 0000000000000000000000000000000000000000..8da3bcd0ac5dd8091d05b7c938856858dd8ffef8
--- /dev/null
+++ b/cpp/full_code/NSVE_Stokes_particles.cpp
@@ -0,0 +1,229 @@
+/**********************************************************************
+* *
+* Copyright 2019 Max Planck Institute *
+* for Dynamics and Self-Organization *
+* *
+* This file is part of TurTLE. *
+* *
+* TurTLE is free software: you can redistribute it and/or modify *
+* it under the terms of the GNU General Public License as published *
+* by the Free Software Foundation, either version 3 of the License, *
+* or (at your option) any later version. *
+* *
+* TurTLE is distributed in the hope that it will be useful, *
+* but WITHOUT ANY WARRANTY; without even the implied warranty of *
+* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the *
+* GNU General Public License for more details. *
+* *
+* You should have received a copy of the GNU General Public License *
+* along with TurTLE. If not, see <http://www.gnu.org/licenses/> *
+* *
+* Contact: Cristian.Lalescu@ds.mpg.de *
+* *
+**********************************************************************/
+
+
+
+#include <string>
+#include <cmath>
+#include "full_code/NSVE_Stokes_particles.hpp"
+#include "scope_timer.hpp"
+#include "particles/particles_sampling.hpp"
+#include "particles/p2p/p2p_ghost_collisions.hpp"
+#include "particles/inner/particles_inner_computer_2nd_order.hpp"
+
+template <typename rnumber>
+int NSVE_Stokes_particles<rnumber>::initialize(void)
+{
+ TIMEZONE("NSVE_Stokes_particles::intialize");
+ this->NSVE<rnumber>::initialize();
+ this->pressure = new field<rnumber, FFTW, ONE>(
+ this->fs->cvelocity->rlayout->sizes[2],
+ this->fs->cvelocity->rlayout->sizes[1],
+ this->fs->cvelocity->rlayout->sizes[0],
+ this->fs->cvelocity->rlayout->comm,
+ this->fs->cvelocity->fftw_plan_rigor);
+
+ particles_inner_computer_2nd_order_Stokes<double, long long int> current_particles_inner_computer;
+ current_particles_inner_computer.set_drag_coefficient(this->drag_coefficient);
+ this->ps = particles_system_builder_with_p2p(
+ this->fs->cvelocity, // (field object)
+ this->fs->kk, // (kspace object, contains dkx, dky, dkz)
+ tracers0_integration_steps, // to check coherency between parameters and hdf input file (nb rhs)
+ (long long int)nparticles, // to check coherency between parameters and hdf input file
+ this->fs->get_current_fname(), // particles input filename
+ std::string("/tracers0/state/") + std::to_string(this->fs->iteration), // dataset name for initial input
+ std::string("/tracers0/rhs/") + std::to_string(this->fs->iteration), // dataset name for initial input
+ tracers0_neighbours, // parameter (interpolation no neighbours)
+ tracers0_smoothness, // parameter
+ this->comm,
+ this->fs->iteration+1,
+ std::move(p2p_ghost_collisions<double, long long int>()),
+ std::move(current_particles_inner_computer),
+ this->tracers0_cutoff);
+ //DEBUG_MSG_WAIT(MPI_COMM_WORLD, "after call to particles_system_builder\n");
+ this->particles_output_writer_mpi = new particles_output_hdf5<
+ long long int, double, 6>(
+ MPI_COMM_WORLD,
+ "tracers0",
+ nparticles,
+ tracers0_integration_steps);
+ this->particles_output_writer_mpi->setParticleFileLayout(this->ps->getParticleFileLayout());
+ this->particles_sample_writer_mpi = new particles_output_sampling_hdf5<
+ long long int, double, double, 3>(
+ MPI_COMM_WORLD,
+ this->ps->getGlobalNbParticles(),
+ (this->simname + "_particles.h5"),
+ "tracers0",
+ "position/0");
+ this->particles_sample_writer_mpi->setParticleFileLayout(this->ps->getParticleFileLayout());
+ return EXIT_SUCCESS;
+}
+
+template <typename rnumber>
+int NSVE_Stokes_particles<rnumber>::step(void)
+{
+ TIMEZONE("NSVE_Stokes_particles::step");
+ this->fs->compute_velocity(this->fs->cvorticity);
+ this->fs->cvelocity->ift();
+ this->ps->complete2ndOrderLoop(this->dt);
+ this->NSVE<rnumber>::step();
+ return EXIT_SUCCESS;
+}
+
+template <typename rnumber>
+int NSVE_Stokes_particles<rnumber>::write_checkpoint(void)
+{
+ TIMEZONE("NSVE_Stokes_particles::write_checkpoint");
+ this->NSVE<rnumber>::write_checkpoint();
+ this->particles_output_writer_mpi->open_file(this->fs->get_current_fname());
+ this->particles_output_writer_mpi->template save<6>(
+ this->ps->getParticlesState(),
+ this->ps->getParticlesRhs(),
+ this->ps->getParticlesIndexes(),
+ this->ps->getLocalNbParticles(),
+ this->fs->iteration);
+ this->particles_output_writer_mpi->close_file();
+ return EXIT_SUCCESS;
+}
+
+template <typename rnumber>
+int NSVE_Stokes_particles<rnumber>::finalize(void)
+{
+ TIMEZONE("NSVE_Stokes_particles::finalize");
+ delete this->pressure;
+ delete this->ps.release();
+ delete this->particles_output_writer_mpi;
+ delete this->particles_sample_writer_mpi;
+ this->NSVE<rnumber>::finalize();
+ return EXIT_SUCCESS;
+}
+
+/** \brief Compute fluid stats and sample fields at particle locations.
+ */
+
+template <typename rnumber>
+int NSVE_Stokes_particles<rnumber>::do_stats()
+{
+ TIMEZONE("NSVE_Stokes_particles::do_stats");
+ /// fluid stats go here
+ this->NSVE<rnumber>::do_stats();
+
+
+ /// either one of two conditions suffices to compute statistics:
+ /// 1) current iteration is a multiple of niter_part
+ /// 2) we are within niter_part_fine_duration/2 of a multiple of niter_part_fine_period
+ if (!(this->iteration % this->niter_part == 0 ||
+ ((this->iteration + this->niter_part_fine_duration/2) % this->niter_part_fine_period <=
+ this->niter_part_fine_duration)))
+ return EXIT_SUCCESS;
+
+ /// allocate temporary data array
+ /// initialize pdata0 with the positions, and pdata1 with the orientations
+ std::unique_ptr<double[]> pdata0 = this->ps->extractParticlesState(0, 3);
+ std::unique_ptr<double[]> pdata1 = this->ps->extractParticlesState(3, 6);
+ std::unique_ptr<double[]> pdata2(new double[9*this->ps->getLocalNbParticles()]);
+
+ /// sample position
+ this->particles_sample_writer_mpi->template save_dataset<3>(
+ "tracers0",
+ "position",
+ pdata0.get(), // we need to use pdata0.get() here, because it's 3D, whereas getParticlesState may give something else
+ &pdata0,
+ this->ps->getParticlesIndexes(),
+ this->ps->getLocalNbParticles(),
+ this->ps->get_step_idx()-1);
+
+ /// sample particle momentum
+ this->particles_sample_writer_mpi->template save_dataset<3>(
+ "tracers0",
+ "momentum",
+ pdata0.get(),
+ &pdata1,
+ this->ps->getParticlesIndexes(),
+ this->ps->getLocalNbParticles(),
+ this->ps->get_step_idx()-1);
+
+ /// sample velocity
+ std::fill_n(pdata1.get(), 3*this->ps->getLocalNbParticles(), 0);
+ if (!(this->iteration % this->niter_stat == 0))
+ {
+ // we need to compute velocity field manually, because it didn't happen in NSVE::do_stats()
+ this->fs->compute_velocity(this->fs->cvorticity);
+ *this->tmp_vec_field = this->fs->cvelocity->get_cdata();
+ this->tmp_vec_field->ift();
+ }
+ this->ps->sample_compute_field(*this->tmp_vec_field, pdata1.get());
+ this->particles_sample_writer_mpi->template save_dataset<3>(
+ "tracers0",
+ "velocity",
+ pdata0.get(),
+ &pdata1,
+ this->ps->getParticlesIndexes(),
+ this->ps->getLocalNbParticles(),
+ this->ps->get_step_idx()-1);
+
+ /// sample vorticity
+ std::fill_n(pdata1.get(), 3*this->ps->getLocalNbParticles(), 0);
+ *this->tmp_vec_field = this->fs->cvorticity->get_cdata();
+ this->tmp_vec_field->ift();
+ this->ps->sample_compute_field(*this->tmp_vec_field, pdata1.get());
+ this->particles_sample_writer_mpi->template save_dataset<3>(
+ "tracers0",
+ "vorticity",
+ pdata0.get(),
+ &pdata1,
+ this->ps->getParticlesIndexes(),
+ this->ps->getLocalNbParticles(),
+ this->ps->get_step_idx()-1);
+
+ // deallocate temporary data array
+ delete[] pdata0.release();
+ delete[] pdata1.release();
+
+ return EXIT_SUCCESS;
+}
+
+template <typename rnumber>
+int NSVE_Stokes_particles<rnumber>::read_parameters(void)
+{
+ TIMEZONE("NSVE_Stokes_particles::read_parameters");
+ this->NSVE<rnumber>::read_parameters();
+ hid_t parameter_file = H5Fopen((this->simname + ".h5").c_str(), H5F_ACC_RDONLY, H5P_DEFAULT);
+ this->niter_part = hdf5_tools::read_value<int>(parameter_file, "parameters/niter_part");
+ this->niter_part_fine_period = hdf5_tools::read_value<int>(parameter_file, "parameters/niter_part_fine_period");
+ this->niter_part_fine_duration = hdf5_tools::read_value<int>(parameter_file, "parameters/niter_part_fine_duration");
+ this->nparticles = hdf5_tools::read_value<long long int>(parameter_file, "parameters/nparticles");
+ this->tracers0_integration_steps = hdf5_tools::read_value<int>(parameter_file, "parameters/tracers0_integration_steps");
+ this->tracers0_neighbours = hdf5_tools::read_value<int>(parameter_file, "parameters/tracers0_neighbours");
+ this->tracers0_smoothness = hdf5_tools::read_value<int>(parameter_file, "parameters/tracers0_smoothness");
+ this->tracers0_cutoff = hdf5_tools::read_value<double>(parameter_file, "parameters/tracers0_cutoff");
+ this->drag_coefficient = hdf5_tools::read_value<double>(parameter_file, "parameters/drag_coefficient");
+ H5Fclose(parameter_file);
+ MPI_Barrier(this->comm);
+ return EXIT_SUCCESS;
+}
+
+template class NSVE_Stokes_particles<float>;
+template class NSVE_Stokes_particles<double>;
+
diff --git a/cpp/full_code/NSVE_Stokes_particles.hpp b/cpp/full_code/NSVE_Stokes_particles.hpp
new file mode 100644
index 0000000000000000000000000000000000000000..fb8497c99074e52c246ac83ba9a071e53d3023c6
--- /dev/null
+++ b/cpp/full_code/NSVE_Stokes_particles.hpp
@@ -0,0 +1,89 @@
+/**********************************************************************
+* *
+* Copyright 2017 Max Planck Institute *
+* for Dynamics and Self-Organization *
+* *
+* This file is part of TurTLE. *
+* *
+* TurTLE is free software: you can redistribute it and/or modify *
+* it under the terms of the GNU General Public License as published *
+* by the Free Software Foundation, either version 3 of the License, *
+* or (at your option) any later version. *
+* *
+* TurTLE is distributed in the hope that it will be useful, *
+* but WITHOUT ANY WARRANTY; without even the implied warranty of *
+* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the *
+* GNU General Public License for more details. *
+* *
+* You should have received a copy of the GNU General Public License *
+* along with TurTLE. If not, see <http://www.gnu.org/licenses/> *
+* *
+* Contact: Cristian.Lalescu@ds.mpg.de *
+* *
+**********************************************************************/
+
+
+
+#ifndef NSVE_STOKES_PARTICLES_HPP
+#define NSVE_STOKES_PARTICLES_HPP
+
+
+
+#include <cstdlib>
+#include "base.hpp"
+#include "vorticity_equation.hpp"
+#include "full_code/NSVE.hpp"
+#include "particles/particles_system_builder.hpp"
+#include "particles/particles_output_hdf5.hpp"
+#include "particles/particles_sampling.hpp"
+
+/** \brief Navier-Stokes solver that includes simple Lagrangian tracers.
+ *
+ * Child of Navier Stokes vorticity equation solver, this class calls all the
+ * methods from `NSVE`, and in addition integrates simple Lagrangian tracers
+ * in the resulting velocity field.
+ */
+
+template <typename rnumber>
+class NSVE_Stokes_particles: public NSVE<rnumber>
+{
+ public:
+
+ /* parameters that are read in read_parameters */
+ int niter_part;
+ int niter_part_fine_period;
+ int niter_part_fine_duration;
+ long long int nparticles;
+ int tracers0_integration_steps;
+ int tracers0_neighbours;
+ int tracers0_smoothness;
+ double tracers0_cutoff;
+ double drag_coefficient;
+
+ /* other stuff */
+ std::unique_ptr<abstract_particles_system<long long int, double>> ps;
+ field<rnumber, FFTW, ONE> *pressure;
+
+ particles_output_hdf5<long long int, double,6> *particles_output_writer_mpi;
+ particles_output_sampling_hdf5<long long int, double, double, 3> *particles_sample_writer_mpi;
+
+
+ NSVE_Stokes_particles(
+ const MPI_Comm COMMUNICATOR,
+ const std::string &simulation_name):
+ NSVE<rnumber>(
+ COMMUNICATOR,
+ simulation_name){}
+ ~NSVE_Stokes_particles(){}
+
+ int initialize(void);
+ int step(void);
+ int finalize(void);
+
+ int read_parameters(void);
+ int write_checkpoint(void);
+ int do_stats(void);
+};
+
+#endif//NSVE_STOKES_PARTICLES_HPP
+
diff --git a/cpp/full_code/NSVE_field_stats.cpp b/cpp/full_code/NSVE_field_stats.cpp
new file mode 100644
index 0000000000000000000000000000000000000000..4ee109b88463436a9448a421dad31b5833af76b0
--- /dev/null
+++ b/cpp/full_code/NSVE_field_stats.cpp
@@ -0,0 +1,152 @@
+/******************************************************************************
+* *
+* Copyright 2019 Max Planck Institute for Dynamics and Self-Organization *
+* *
+* This file is part of TurTLE. *
+* *
+* TurTLE is free software: you can redistribute it and/or modify *
+* it under the terms of the GNU General Public License as published *
+* by the Free Software Foundation, either version 3 of the License, *
+* or (at your option) any later version. *
+* *
+* TurTLE is distributed in the hope that it will be useful, *
+* but WITHOUT ANY WARRANTY; without even the implied warranty of *
+* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the *
+* GNU General Public License for more details. *
+* *
+* You should have received a copy of the GNU General Public License *
+* along with TurTLE. If not, see <http://www.gnu.org/licenses/> *
+* *
+* Contact: Cristian.Lalescu@ds.mpg.de *
+* *
+******************************************************************************/
+
+
+
+#include <string>
+#include <cmath>
+#include "NSVE_field_stats.hpp"
+#include "fftw_tools.hpp"
+#include "scope_timer.hpp"
+#include "hdf5_tools.hpp"
+
+
+template <typename rnumber>
+int NSVE_field_stats<rnumber>::initialize(void)
+{
+ TIMEZONE("NSVE_field_stats::initialize");
+ this->postprocess::read_parameters();
+ this->vorticity = new field<rnumber, FFTW, THREE>(
+ nx, ny, nz,
+ this->comm,
+ fftw_planner_string_to_flag[this->fftw_plan_rigor]);
+ this->vorticity->real_space_representation = false;
+ hid_t parameter_file = H5Fopen(
+ (this->simname + std::string(".h5")).c_str(),
+ H5F_ACC_RDONLY,
+ H5P_DEFAULT);
+ if (!H5Lexists(parameter_file, "field_dtype", H5P_DEFAULT))
+ this->bin_IO = NULL;
+ else
+ {
+ hid_t dset = H5Dopen(parameter_file, "field_dtype", H5P_DEFAULT);
+ hid_t space = H5Dget_space(dset);
+ hid_t memtype = H5Dget_type(dset);
+ char *string_data = (char*)malloc(256);
+ H5Dread(dset, memtype, H5S_ALL, H5S_ALL, H5P_DEFAULT, &string_data);
+ // check that we're using the correct data type
+ // field_dtype SHOULD be something like "<f4", "<f8", ">f4", ">f8"
+ // first character is ordering, which is machine specific
+ // for the other two I am checking that they have the correct values
+ assert(string_data[1] == 'f');
+ assert(string_data[2] == '0' + sizeof(rnumber));
+ free(string_data);
+ H5Sclose(space);
+ H5Tclose(memtype);
+ H5Dclose(dset);
+ this->bin_IO = new field_binary_IO<rnumber, COMPLEX, THREE>(
+ this->vorticity->clayout->sizes,
+ this->vorticity->clayout->subsizes,
+ this->vorticity->clayout->starts,
+ this->vorticity->clayout->comm);
+ }
+ this->fftw_plan_rigor = hdf5_tools::read_string(parameter_file, "parameters/fftw_plan_rigor");
+ H5Fclose(parameter_file);
+ return EXIT_SUCCESS;
+}
+
+template <typename rnumber>
+int NSVE_field_stats<rnumber>::read_current_cvorticity(void)
+{
+ TIMEZONE("NSVE_field_stats::read_current_cvorticity");
+ this->vorticity->real_space_representation = false;
+ if (this->bin_IO != NULL)
+ {
+ char itername[16];
+ sprintf(itername, "i%.5x", this->iteration);
+ std::string native_binary_fname = (
+ this->simname +
+ std::string("_cvorticity_") +
+ std::string(itername));
+ this->bin_IO->read(
+ native_binary_fname,
+ this->vorticity->get_cdata());
+ }
+ else
+ {
+ this->vorticity->io(
+ this->simname + std::string("_fields.h5"),
+ "vorticity",
+ this->iteration,
+ true);
+ }
+ return EXIT_SUCCESS;
+}
+template <typename rnumber>
+int NSVE_field_stats<rnumber>::read_arbitrary_cvorticity(int arbitrary_iteration)
+{
+ TIMEZONE("NSVE_field_stats::read_arbitrary_cvorticity");
+ this->vorticity->real_space_representation = false;
+ if (this->bin_IO != NULL)
+ {
+ char itername[16];
+ sprintf(itername, "i%.5x", arbitrary_iteration);
+ std::string native_binary_fname = (
+ this->simname +
+ std::string("_cvorticity_") +
+ std::string(itername));
+ this->bin_IO->read(
+ native_binary_fname,
+ this->vorticity->get_cdata());
+ }
+ else
+ {
+ this->vorticity->io(
+ this->simname + std::string("_fields.h5"),
+ "vorticity",
+ arbitrary_iteration,
+ true);
+ }
+ return EXIT_SUCCESS;
+}
+
+template <typename rnumber>
+int NSVE_field_stats<rnumber>::finalize(void)
+{
+ TIMEZONE("NSVE_field_stats::finalize");
+ if (this->bin_IO != NULL)
+ delete this->bin_IO;
+ delete this->vorticity;
+ return EXIT_SUCCESS;
+}
+
+template <typename rnumber>
+int NSVE_field_stats<rnumber>::work_on_current_iteration(void)
+{
+ TIMEZONE("NSVE_field_stats::work_on_current_iteration");
+ return EXIT_SUCCESS;
+}
+
+template class NSVE_field_stats<float>;
+template class NSVE_field_stats<double>;
+
diff --git a/bfps/cpp/full_code/NSVE_field_stats.hpp b/cpp/full_code/NSVE_field_stats.hpp
similarity index 83%
rename from bfps/cpp/full_code/NSVE_field_stats.hpp
rename to cpp/full_code/NSVE_field_stats.hpp
index d544c0c7d5f4c75559e63ea3e59bf9457d4730c5..a04215389a7773f8ecbdf052d4a2a6fd4119b1b0 100644
--- a/bfps/cpp/full_code/NSVE_field_stats.hpp
+++ b/cpp/full_code/NSVE_field_stats.hpp
@@ -3,20 +3,20 @@
* Copyright 2017 Max Planck Institute *
* for Dynamics and Self-Organization *
* *
-* This file is part of bfps. *
+* This file is part of TurTLE. *
* *
-* bfps is free software: you can redistribute it and/or modify *
+* TurTLE is free software: you can redistribute it and/or modify *
* it under the terms of the GNU General Public License as published *
* by the Free Software Foundation, either version 3 of the License, *
* or (at your option) any later version. *
* *
-* bfps is distributed in the hope that it will be useful, *
+* TurTLE is distributed in the hope that it will be useful, *
* but WITHOUT ANY WARRANTY; without even the implied warranty of *
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the *
* GNU General Public License for more details. *
* *
* You should have received a copy of the GNU General Public License *
-* along with bfps. If not, see <http://www.gnu.org/licenses/> *
+* along with TurTLE. If not, see <http://www.gnu.org/licenses/> *
* *
* Contact: Cristian.Lalescu@ds.mpg.de *
* *
@@ -42,6 +42,8 @@ class NSVE_field_stats: public postprocess
private:
field_binary_IO<rnumber, COMPLEX, THREE> *bin_IO;
public:
+ std::string fftw_plan_rigor;
+
field<rnumber, FFTW, THREE> *vorticity;
NSVE_field_stats(
@@ -50,13 +52,14 @@ class NSVE_field_stats: public postprocess
postprocess(
COMMUNICATOR,
simulation_name){}
- virtual ~NSVE_field_stats(){}
+ virtual ~NSVE_field_stats() noexcept(false){}
virtual int initialize(void);
virtual int work_on_current_iteration(void);
virtual int finalize(void);
int read_current_cvorticity(void);
+ int read_arbitrary_cvorticity(int iter);
};
#endif//NSVE_FIELD_STATS_HPP
diff --git a/cpp/full_code/NSVE_no_output.hpp b/cpp/full_code/NSVE_no_output.hpp
new file mode 100644
index 0000000000000000000000000000000000000000..d739745cff1df3fe98ad555b0c1c919834cd64da
--- /dev/null
+++ b/cpp/full_code/NSVE_no_output.hpp
@@ -0,0 +1,51 @@
+/**********************************************************************
+* *
+* Copyright 2017 Max Planck Institute *
+* for Dynamics and Self-Organization *
+* *
+* This file is part of TurTLE. *
+* *
+* TurTLE is free software: you can redistribute it and/or modify *
+* it under the terms of the GNU General Public License as published *
+* by the Free Software Foundation, either version 3 of the License, *
+* or (at your option) any later version. *
+* *
+* TurTLE is distributed in the hope that it will be useful, *
+* but WITHOUT ANY WARRANTY; without even the implied warranty of *
+* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the *
+* GNU General Public License for more details. *
+* *
+* You should have received a copy of the GNU General Public License *
+* along with TurTLE. If not, see <http://www.gnu.org/licenses/> *
+* *
+* Contact: Cristian.Lalescu@ds.mpg.de *
+* *
+**********************************************************************/
+
+
+
+#ifndef NSVE_NO_OUTPUT_HPP
+#define NSVE_NO_OUTPUT_HPP
+
+#include "full_code/NSVE.hpp"
+
+template <typename rnumber>
+class NSVE_no_output: public NSVE<rnumber>
+{
+ public:
+ NSVE_no_output(
+ const MPI_Comm COMMUNICATOR,
+ const std::string &simulation_name):
+ NSVE<rnumber>(
+ COMMUNICATOR,
+ simulation_name){}
+ ~NSVE_no_output() noexcept(false){}
+ int write_checkpoint(void)
+ {
+ TIMEZONE("NSVE_no_output::write_checkpoint");
+ return EXIT_SUCCESS;
+ }
+};
+
+#endif//NSVE_NO_OUTPUT_HPP
+
diff --git a/cpp/full_code/NSVEcomplex_particles.cpp b/cpp/full_code/NSVEcomplex_particles.cpp
new file mode 100644
index 0000000000000000000000000000000000000000..0c71cf723ca18e982a9b684a76b9f9c9f8834ae9
--- /dev/null
+++ b/cpp/full_code/NSVEcomplex_particles.cpp
@@ -0,0 +1,266 @@
+/**********************************************************************
+* *
+* Copyright 2017 Max Planck Institute *
+* for Dynamics and Self-Organization *
+* *
+* This file is part of TurTLE. *
+* *
+* TurTLE is free software: you can redistribute it and/or modify *
+* it under the terms of the GNU General Public License as published *
+* by the Free Software Foundation, either version 3 of the License, *
+* or (at your option) any later version. *
+* *
+* TurTLE is distributed in the hope that it will be useful, *
+* but WITHOUT ANY WARRANTY; without even the implied warranty of *
+* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the *
+* GNU General Public License for more details. *
+* *
+* You should have received a copy of the GNU General Public License *
+* along with TurTLE. If not, see <http://www.gnu.org/licenses/> *
+* *
+* Contact: Cristian.Lalescu@ds.mpg.de *
+* *
+**********************************************************************/
+
+
+
+#include <string>
+#include <cmath>
+#include "full_code/NSVEcomplex_particles.hpp"
+#include "scope_timer.hpp"
+#include "particles/particles_sampling.hpp"
+#include "particles/p2p/p2p_computer.hpp"
+#include "particles/inner/particles_inner_computer.hpp"
+
+template <typename rnumber>
+int NSVEcomplex_particles<rnumber>::initialize(void)
+{
+ TIMEZONE("NSVEcomplex_particles::initialize");
+ this->NSVE<rnumber>::initialize();
+
+ p2p_computer<double, long long int> current_p2p_computer;
+ current_p2p_computer.setEnable(this->enable_p2p);
+
+ particles_inner_computer<double, long long int> current_particles_inner_computer(inner_v0, this->lambda);
+ current_particles_inner_computer.setEnable(enable_inner);
+
+
+ this->ps = particles_system_builder_with_p2p(
+ this->fs->cvelocity, // (field object)
+ this->fs->kk, // (kspace object, contains dkx, dky, dkz)
+ tracers0_integration_steps, // to check coherency between parameters and hdf input file (nb rhs)
+ (long long int)nparticles, // to check coherency between parameters and hdf input file
+ this->fs->get_current_fname(), // particles input filename
+ std::string("/tracers0/state/") + std::to_string(this->fs->iteration), // dataset name for initial input
+ std::string("/tracers0/rhs/") + std::to_string(this->fs->iteration), // dataset name for initial input
+ tracers0_neighbours, // parameter (interpolation no neighbours)
+ tracers0_smoothness, // parameter (how many continuous derivatives)
+ this->comm,
+ this->fs->iteration+1,
+ std::move(current_p2p_computer),
+ std::move(current_particles_inner_computer),
+ cutoff);
+
+ this->particles_output_writer_mpi = new particles_output_hdf5<
+ long long int, double, 6>(
+ MPI_COMM_WORLD,
+ "tracers0",
+ nparticles,
+ tracers0_integration_steps);
+ this->particles_sample_writer_mpi = new particles_output_sampling_hdf5<
+ long long int, double, double, 3>(
+ MPI_COMM_WORLD,
+ this->ps->getGlobalNbParticles(),
+ (this->simname + "_particles.h5"),
+ "tracers0",
+ "position/0");
+ this->particles_output_writer_mpi->setParticleFileLayout(this->ps->getParticleFileLayout());
+ this->particles_sample_writer_mpi->setParticleFileLayout(this->ps->getParticleFileLayout());
+
+
+ /// allocate grad vel field
+ this->nabla_u = new field<rnumber, FFTW, THREExTHREE>(
+ this->nx, this->ny, this->nz,
+ this->comm,
+ this->fs->cvorticity->fftw_plan_rigor);
+ return EXIT_SUCCESS;
+}
+
+template <typename rnumber>
+int NSVEcomplex_particles<rnumber>::step(void)
+{
+ TIMEZONE("NSVEcomplex_particles::step");
+ this->fs->compute_velocity(this->fs->cvorticity);
+ if(this->enable_vorticity_omega){
+ compute_gradient(
+ this->fs->kk,
+ this->fs->cvelocity,
+ this->nabla_u);
+ this->nabla_u->ift();
+ this->fs->cvelocity->ift(); // needed before completeloop
+ //std::unique_ptr<double[]> sampled_vorticity(new double[9*this->ps->getLocalNbParticles()]);
+ //std::fill_n(sampled_vorticity.get(), 9*this->ps->getLocalNbParticles(), 0);
+ //this->ps->sample_compute_field(*this->nabla_u, sampled_vorticity.get());
+ //*this->tmp_vec_field = this->fs->cvorticity->get_cdata();
+ //this->tmp_vec_field->ift();
+ this->ps->completeLoopWithExtraField(this->dt, *this->nabla_u);
+ }
+ else{
+ this->fs->cvelocity->ift();
+ this->ps->completeLoop(this->dt);
+ }
+ this->NSVE<rnumber>::step();
+ return EXIT_SUCCESS;
+}
+
+template <typename rnumber>
+int NSVEcomplex_particles<rnumber>::write_checkpoint(void)
+{
+ TIMEZONE("NSVEcomplex_particles::write_checkpoint");
+ this->NSVE<rnumber>::write_checkpoint();
+ this->particles_output_writer_mpi->open_file(this->fs->get_current_fname());
+ // TODO P2P write particle data too
+ this->particles_output_writer_mpi->template save<6>(
+ this->ps->getParticlesState(),
+ this->ps->getParticlesRhs(),
+ this->ps->getParticlesIndexes(),
+ this->ps->getLocalNbParticles(),
+ this->fs->iteration);
+ this->particles_output_writer_mpi->close_file();
+ return EXIT_SUCCESS;
+}
+
+template <typename rnumber>
+int NSVEcomplex_particles<rnumber>::finalize(void)
+{
+ TIMEZONE("NSVEcomplex_particles::finalize");
+ delete this->nabla_u;
+ delete this->particles_output_writer_mpi;
+ delete this->particles_sample_writer_mpi;
+ this->NSVE<rnumber>::finalize();
+ return EXIT_SUCCESS;
+}
+
+/** \brief Compute fluid stats and sample particle data.
+ */
+
+template <typename rnumber>
+int NSVEcomplex_particles<rnumber>::do_stats()
+{
+ TIMEZONE("NSVEcomplex_particles::do_stats");
+ /// perform fluid stats
+ this->NSVE<rnumber>::do_stats();
+
+ /// check if particle stats should be performed now;
+ /// if not, exit method.
+ if (!(this->iteration % this->niter_part == 0))
+ return EXIT_SUCCESS;
+
+ /// allocate temporary data array
+ /// initialize pdata0 with the positions, and pdata1 with the orientations
+ std::unique_ptr<double[]> pdata0 = this->ps->extractParticlesState(0, 3);
+ std::unique_ptr<double[]> pdata1 = this->ps->extractParticlesState(3, 6);
+ std::unique_ptr<double[]> pdata2(new double[9*this->ps->getLocalNbParticles()]);
+
+ /// sample position
+ this->particles_sample_writer_mpi->template save_dataset<3>(
+ "tracers0",
+ "position",
+ pdata0.get(), // we need to use pdata0.get() here, because it's 3D, whereas getParticlesState may give something else
+ &pdata0,
+ this->ps->getParticlesIndexes(),
+ this->ps->getLocalNbParticles(),
+ this->ps->get_step_idx()-1);
+
+ /// sample orientation
+ this->particles_sample_writer_mpi->template save_dataset<3>(
+ "tracers0",
+ "orientation",
+ pdata0.get(),
+ &pdata1,
+ this->ps->getParticlesIndexes(),
+ this->ps->getLocalNbParticles(),
+ this->ps->get_step_idx()-1);
+
+ /// sample velocity
+ /// from now on, we need to clean up data arrays before interpolation
+ std::fill_n(pdata1.get(), 3*this->ps->getLocalNbParticles(), 0);
+ this->ps->sample_compute_field(*this->tmp_vec_field, pdata1.get());
+ this->particles_sample_writer_mpi->template save_dataset<3>(
+ "tracers0",
+ "velocity",
+ pdata0.get(),
+ &pdata1,
+ this->ps->getParticlesIndexes(),
+ this->ps->getLocalNbParticles(),
+ this->ps->get_step_idx()-1);
+
+ /// sample velocity gradient
+ /// fs->cvelocity should contain the velocity in Fourier space
+ this->fs->compute_velocity(this->fs->cvorticity);
+ compute_gradient(
+ this->fs->kk,
+ this->fs->cvelocity,
+ this->nabla_u);
+ this->nabla_u->ift();
+ std::fill_n(pdata2.get(), 9*this->ps->getLocalNbParticles(), 0);
+ this->ps->sample_compute_field(*this->nabla_u, pdata2.get());
+ this->particles_sample_writer_mpi->template save_dataset<9>(
+ "tracers0",
+ "velocity_gradient",
+ pdata0.get(),
+ &pdata2,
+ this->ps->getParticlesIndexes(),
+ this->ps->getLocalNbParticles(),
+ this->ps->get_step_idx()-1);
+
+ /// compute acceleration and sample it
+ this->fs->compute_Lagrangian_acceleration(this->tmp_vec_field);
+ this->tmp_vec_field->ift();
+ std::fill_n(pdata1.get(), 3*this->ps->getLocalNbParticles(), 0);
+ this->ps->sample_compute_field(*this->tmp_vec_field, pdata1.get());
+ this->particles_sample_writer_mpi->template save_dataset<3>(
+ "tracers0",
+ "acceleration",
+ pdata0.get(),
+ &pdata1,
+ this->ps->getParticlesIndexes(),
+ this->ps->getLocalNbParticles(),
+ this->ps->get_step_idx()-1);
+
+ // deallocate temporary data array
+ delete[] pdata0.release();
+ delete[] pdata1.release();
+ delete[] pdata2.release();
+
+ return EXIT_SUCCESS;
+}
+
+template <typename rnumber>
+int NSVEcomplex_particles<rnumber>::read_parameters(void)
+{
+ TIMEZONE("NSVEcomplex_particles::read_parameters");
+ this->NSVE<rnumber>::read_parameters();
+ hid_t parameter_file = H5Fopen((this->simname + ".h5").c_str(), H5F_ACC_RDONLY, H5P_DEFAULT);
+ this->niter_part = hdf5_tools::read_value<int>(parameter_file, "parameters/niter_part");
+ this->nparticles = hdf5_tools::read_value<long long int>(parameter_file, "parameters/nparticles");
+ this->tracers0_integration_steps = hdf5_tools::read_value<int>(parameter_file, "parameters/tracers0_integration_steps");
+ this->tracers0_neighbours = hdf5_tools::read_value<int>(parameter_file, "parameters/tracers0_neighbours");
+ this->tracers0_smoothness = hdf5_tools::read_value<int>(parameter_file, "parameters/tracers0_smoothness");
+ this->enable_p2p = hdf5_tools::read_value<int>(parameter_file, "parameters/tracers0_enable_p2p");
+ this->enable_inner = hdf5_tools::read_value<int>(parameter_file, "parameters/tracers0_enable_inner");
+ int tval = hdf5_tools::read_value<int>(parameter_file, "parameters/tracers0_enable_vorticity_omega");
+ this->enable_vorticity_omega = tval;
+ DEBUG_MSG("tracers0_enable_vorticity_omega = %d, this->enable_vorticity_omega = %d\n",
+ tval, this->enable_vorticity_omega);
+ this->cutoff = hdf5_tools::read_value<double>(parameter_file, "parameters/tracers0_cutoff");
+ this->inner_v0 = hdf5_tools::read_value<double>(parameter_file, "parameters/tracers0_inner_v0");
+ this->lambda = hdf5_tools::read_value<double>(parameter_file, "parameters/tracers0_lambda");
+ H5Fclose(parameter_file);
+ MPI_Barrier(this->comm);
+ return EXIT_SUCCESS;
+}
+
+template class NSVEcomplex_particles<float>;
+template class NSVEcomplex_particles<double>;
+
diff --git a/cpp/full_code/NSVEcomplex_particles.hpp b/cpp/full_code/NSVEcomplex_particles.hpp
new file mode 100644
index 0000000000000000000000000000000000000000..3f6adaac45e79594e1e15a1e1fde737e4a1ea27f
--- /dev/null
+++ b/cpp/full_code/NSVEcomplex_particles.hpp
@@ -0,0 +1,97 @@
+/**********************************************************************
+* *
+* Copyright 2017 Max Planck Institute *
+* for Dynamics and Self-Organization *
+* *
+* This file is part of TurTLE. *
+* *
+* TurTLE is free software: you can redistribute it and/or modify *
+* it under the terms of the GNU General Public License as published *
+* by the Free Software Foundation, either version 3 of the License, *
+* or (at your option) any later version. *
+* *
+* TurTLE is distributed in the hope that it will be useful, *
+* but WITHOUT ANY WARRANTY; without even the implied warranty of *
+* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the *
+* GNU General Public License for more details. *
+* *
+* You should have received a copy of the GNU General Public License *
+* along with TurTLE. If not, see <http://www.gnu.org/licenses/> *
+* *
+* Contact: Cristian.Lalescu@ds.mpg.de *
+* *
+**********************************************************************/
+
+
+
+#ifndef NSVECOMPLEX_PARTICLES_HPP
+#define NSVECOMPLEX_PARTICLES_HPP
+
+
+
+#include <cstdlib>
+#include "base.hpp"
+#include "vorticity_equation.hpp"
+#include "full_code/NSVE.hpp"
+#include "particles/particles_system_builder.hpp"
+#include "particles/particles_output_hdf5.hpp"
+#include "particles/particles_sampling.hpp"
+
+/** \brief Navier-Stokes solver that includes complex particles.
+ *
+ * Child of Navier Stokes vorticity equation solver, this class calls all the
+ * methods from `NSVE`, and in addition integrates `complex particles`
+ * in the resulting velocity field.
+ * By `complex particles` we mean neutrally buoyant, very small particles,
+ * which have an orientation and actively swim in that direction, and they may
+ * also interact with each other, trying to reorient to a common orientation.
+ */
+
+template <typename rnumber>
+class NSVEcomplex_particles: public NSVE<rnumber>
+{
+ public:
+
+ /* parameters that are read in read_parameters */
+ int niter_part;
+ long long int nparticles;
+ int tracers0_integration_steps;
+ int tracers0_neighbours;
+ int tracers0_smoothness;
+
+ double cutoff;
+ double inner_v0;
+ double lambda;
+ bool enable_p2p;
+ bool enable_inner;
+ bool enable_vorticity_omega;
+
+ /* other stuff */
+ std::unique_ptr<abstract_particles_system<long long int, double>> ps;
+ // TODO P2P use a reader with particle data
+ particles_output_hdf5<long long int, double,6> *particles_output_writer_mpi;
+ particles_output_sampling_hdf5<long long int, double, double, 3> *particles_sample_writer_mpi;
+ // field for sampling velocity gradient
+ field<rnumber, FFTW, THREExTHREE> *nabla_u;
+
+
+ NSVEcomplex_particles(
+ const MPI_Comm COMMUNICATOR,
+ const std::string &simulation_name):
+ NSVE<rnumber>(
+ COMMUNICATOR,
+ simulation_name),
+ cutoff(10), inner_v0(1), lambda(1.0), enable_p2p(true), enable_inner(true), enable_vorticity_omega(true){}
+ ~NSVEcomplex_particles() noexcept(false){}
+
+ int initialize(void);
+ int step(void);
+ int finalize(void);
+
+ int read_parameters(void);
+ int write_checkpoint(void);
+ int do_stats(void);
+};
+
+#endif//NSVECOMPLEX_PARTICLES_HPP
+
diff --git a/cpp/full_code/NSVEp_extra_sampling.cpp b/cpp/full_code/NSVEp_extra_sampling.cpp
new file mode 100644
index 0000000000000000000000000000000000000000..0b96e259add4a1b6eca00eecd1ee430052ea8755
--- /dev/null
+++ b/cpp/full_code/NSVEp_extra_sampling.cpp
@@ -0,0 +1,195 @@
+/******************************************************************************
+* *
+* Copyright 2019 Max Planck Institute for Dynamics and Self-Organization *
+* *
+* This file is part of TurTLE. *
+* *
+* TurTLE is free software: you can redistribute it and/or modify *
+* it under the terms of the GNU General Public License as published *
+* by the Free Software Foundation, either version 3 of the License, *
+* or (at your option) any later version. *
+* *
+* TurTLE is distributed in the hope that it will be useful, *
+* but WITHOUT ANY WARRANTY; without even the implied warranty of *
+* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the *
+* GNU General Public License for more details. *
+* *
+* You should have received a copy of the GNU General Public License *
+* along with TurTLE. If not, see <http://www.gnu.org/licenses/> *
+* *
+* Contact: Cristian.Lalescu@ds.mpg.de *
+* *
+******************************************************************************/
+
+
+
+#include "full_code/NSVEp_extra_sampling.hpp"
+
+
+
+template <typename rnumber>
+int NSVEp_extra_sampling<rnumber>::initialize(void)
+{
+ TIMEZONE("NSVEp_extra_sampling::initialize");
+ this->NSVEparticles<rnumber>::initialize();
+
+ /// allocate grad vel field
+ this->nabla_u = new field<rnumber, FFTW, THREExTHREE>(
+ this->nx, this->ny, this->nz,
+ this->comm,
+ this->fs->cvorticity->fftw_plan_rigor);
+ this->pressure = new field<rnumber, FFTW, ONE>(
+ this->nx, this->ny, this->nz,
+ this->comm,
+ this->fs->cvorticity->fftw_plan_rigor);
+ this->nabla_p = new field<rnumber, FFTW, THREE>(
+ this->nx, this->ny, this->nz,
+ this->comm,
+ this->fs->cvorticity->fftw_plan_rigor);
+ this->Hessian_p = new field<rnumber, FFTW, THREExTHREE>(
+ this->nx, this->ny, this->nz,
+ this->comm,
+ this->fs->cvorticity->fftw_plan_rigor);
+ return EXIT_SUCCESS;
+}
+
+template <typename rnumber>
+int NSVEp_extra_sampling<rnumber>::finalize(void)
+{
+ TIMEZONE("NSVEp_extra_sampling::finalize");
+ delete this->nabla_u;
+ delete this->pressure;
+ delete this->nabla_p;
+ delete this->Hessian_p;
+ this->NSVEparticles<rnumber>::finalize();
+ return EXIT_SUCCESS;
+}
+
+template <typename rnumber>
+int NSVEp_extra_sampling<rnumber>::do_stats()
+{
+ TIMEZONE("NSVEp_extra_sampling::do_stats");
+ this->NSVEparticles<rnumber>::do_stats();
+ if (!(this->iteration % this->niter_part == 0))
+ return EXIT_SUCCESS;
+
+ // allocate temporary array
+ std::unique_ptr<double[]> pdata(new double[9*this->ps->getLocalNbParticles()]);
+
+ if (this->sample_velocity_gradient == 1)
+ {
+ /// fs->cvelocity should contain the velocity in Fourier space
+ this->fs->compute_velocity(this->fs->cvorticity);
+ compute_gradient(
+ this->fs->kk,
+ this->fs->cvelocity,
+ this->nabla_u);
+ this->nabla_u->ift();
+
+ // clean up and sample
+ std::fill_n(pdata.get(), 9*this->ps->getLocalNbParticles(), 0);
+ this->ps->sample_compute_field(*this->nabla_u, pdata.get());
+
+ // write to file
+ this->particles_sample_writer_mpi->template save_dataset<9>(
+ "tracers0",
+ "velocity_gradient",
+ this->ps->getParticlesState(),
+ &pdata,
+ this->ps->getParticlesIndexes(),
+ this->ps->getLocalNbParticles(),
+ this->ps->get_step_idx()-1);
+ }
+
+ if ((this->sample_pressure == 1) ||
+ (this->sample_pressure_gradient == 1) ||
+ (this->sample_pressure_Hessian == 1))
+ {
+ this->fs->compute_pressure(this->pressure);
+
+ if ((this->sample_pressure_gradient == 1) ||
+ (this->sample_pressure_Hessian == 1))
+ {
+ compute_gradient(
+ this->fs->kk,
+ this->pressure,
+ this->nabla_p);
+
+ if (this->sample_pressure_Hessian == 1)
+ {
+ compute_gradient(
+ this->fs->kk,
+ this->nabla_p,
+ this->Hessian_p);
+ }
+ }
+
+ if (this->sample_pressure == 1)
+ {
+ // sample pressure
+ this->pressure->ift();
+ std::fill_n(pdata.get(), this->ps->getLocalNbParticles(), 0);
+ this->ps->sample_compute_field(*this->pressure, pdata.get());
+ this->particles_sample_writer_mpi->template save_dataset<1>(
+ "tracers0",
+ "pressure",
+ this->ps->getParticlesState(),
+ &pdata,
+ this->ps->getParticlesIndexes(),
+ this->ps->getLocalNbParticles(),
+ this->ps->get_step_idx()-1);
+ }
+
+ if (this->sample_pressure_gradient == 1)
+ {
+ // sample pressure gradient
+ this->nabla_p->ift();
+ std::fill_n(pdata.get(), 3*this->ps->getLocalNbParticles(), 0);
+ this->ps->sample_compute_field(*this->nabla_p, pdata.get());
+ this->particles_sample_writer_mpi->template save_dataset<3>(
+ "tracers0",
+ "pressure_gradient",
+ this->ps->getParticlesState(),
+ &pdata,
+ this->ps->getParticlesIndexes(),
+ this->ps->getLocalNbParticles(),
+ this->ps->get_step_idx()-1);
+ }
+
+ if (this->sample_pressure_Hessian == 1)
+ {
+ // sample pressure Hessian
+ this->Hessian_p->ift();
+ std::fill_n(pdata.get(), 9*this->ps->getLocalNbParticles(), 0);
+ this->ps->sample_compute_field(*this->Hessian_p, pdata.get());
+ this->particles_sample_writer_mpi->template save_dataset<9>(
+ "tracers0",
+ "pressure_Hessian",
+ this->ps->getParticlesState(),
+ &pdata,
+ this->ps->getParticlesIndexes(),
+ this->ps->getLocalNbParticles(),
+ this->ps->get_step_idx()-1);
+ }
+ }
+ return EXIT_SUCCESS;
+}
+
+template <typename rnumber>
+int NSVEp_extra_sampling<rnumber>::read_parameters(void)
+{
+ TIMEZONE("NSVEp_extra_sampling::read_parameters");
+ this->NSVEparticles<rnumber>::read_parameters();
+ hid_t parameter_file = H5Fopen((this->simname + ".h5").c_str(), H5F_ACC_RDONLY, H5P_DEFAULT);
+ this->sample_pressure = hdf5_tools::read_value<int>(parameter_file, "parameters/sample_pressure");
+ this->sample_pressure_gradient = hdf5_tools::read_value<int>(parameter_file, "parameters/sample_pressure_gradient");
+ this->sample_pressure_Hessian = hdf5_tools::read_value<int>(parameter_file, "parameters/sample_pressure_Hessian");
+ this->sample_velocity_gradient = hdf5_tools::read_value<int>(parameter_file, "parameters/sample_velocity_gradient");
+ H5Fclose(parameter_file);
+ MPI_Barrier(this->comm);
+ return EXIT_SUCCESS;
+}
+
+template class NSVEp_extra_sampling<float>;
+template class NSVEp_extra_sampling<double>;
+
diff --git a/cpp/full_code/NSVEp_extra_sampling.hpp b/cpp/full_code/NSVEp_extra_sampling.hpp
new file mode 100644
index 0000000000000000000000000000000000000000..a1f5a75a07e83c75291b7b699e4a518f36b18047
--- /dev/null
+++ b/cpp/full_code/NSVEp_extra_sampling.hpp
@@ -0,0 +1,77 @@
+/******************************************************************************
+* *
+* Copyright 2019 Max Planck Institute for Dynamics and Self-Organization *
+* *
+* This file is part of TurTLE. *
+* *
+* TurTLE is free software: you can redistribute it and/or modify *
+* it under the terms of the GNU General Public License as published *
+* by the Free Software Foundation, either version 3 of the License, *
+* or (at your option) any later version. *
+* *
+* TurTLE is distributed in the hope that it will be useful, *
+* but WITHOUT ANY WARRANTY; without even the implied warranty of *
+* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the *
+* GNU General Public License for more details. *
+* *
+* You should have received a copy of the GNU General Public License *
+* along with TurTLE. If not, see <http://www.gnu.org/licenses/> *
+* *
+* Contact: Cristian.Lalescu@ds.mpg.de *
+* *
+******************************************************************************/
+
+
+
+#ifndef NSVEP_EXTRA_SAMPLING_HPP
+#define NSVEP_EXTRA_SAMPLING_HPP
+
+
+
+#include <cstdlib>
+#include "base.hpp"
+#include "vorticity_equation.hpp"
+#include "full_code/NSVEparticles.hpp"
+#include "particles/particles_system_builder.hpp"
+#include "particles/particles_output_hdf5.hpp"
+#include "particles/particles_sampling.hpp"
+
+/** \brief Navier-Stokes solver with tracers that sample velocity gradient
+ * and pressure Hessian.
+ *
+ */
+
+template <typename rnumber>
+class NSVEp_extra_sampling: public NSVEparticles<rnumber>
+{
+ public:
+ /* specific parameters */
+ int sample_pressure;
+ int sample_pressure_gradient;
+ int sample_pressure_Hessian;
+ int sample_velocity_gradient;
+
+ /* other stuff */
+ field<rnumber, FFTW, ONE> *pressure;
+ field<rnumber, FFTW, THREE> *nabla_p;
+ field<rnumber, FFTW, THREExTHREE> *nabla_u;
+ field<rnumber, FFTW, THREExTHREE> *Hessian_p;
+
+ NSVEp_extra_sampling(
+ const MPI_Comm COMMUNICATOR,
+ const std::string &simulation_name):
+ NSVEparticles<rnumber>(
+ COMMUNICATOR,
+ simulation_name){}
+ ~NSVEp_extra_sampling(){}
+
+ int initialize(void);
+ int finalize(void);
+
+ int read_parameters(void);
+ int do_stats(void);
+};
+
+#endif//NSVEP_EXTRA_SAMPLING_HPP
+
+
diff --git a/cpp/full_code/NSVEparticles.cpp b/cpp/full_code/NSVEparticles.cpp
new file mode 100644
index 0000000000000000000000000000000000000000..3f24be4060ac03b45bdd55f13a81d46482327037
--- /dev/null
+++ b/cpp/full_code/NSVEparticles.cpp
@@ -0,0 +1,285 @@
+/**********************************************************************
+* *
+* Copyright 2019 Max Planck Institute *
+* for Dynamics and Self-Organization *
+* *
+* This file is part of TurTLE. *
+* *
+* TurTLE is free software: you can redistribute it and/or modify *
+* it under the terms of the GNU General Public License as published *
+* by the Free Software Foundation, either version 3 of the License, *
+* or (at your option) any later version. *
+* *
+* TurTLE is distributed in the hope that it will be useful, *
+* but WITHOUT ANY WARRANTY; without even the implied warranty of *
+* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the *
+* GNU General Public License for more details. *
+* *
+* You should have received a copy of the GNU General Public License *
+* along with TurTLE. If not, see <http://www.gnu.org/licenses/> *
+* *
+* Contact: Cristian.Lalescu@ds.mpg.de *
+* *
+**********************************************************************/
+
+
+
+#include <string>
+#include <cmath>
+#include "NSVEparticles.hpp"
+#include "scope_timer.hpp"
+#include "particles/interpolation/particle_set.hpp"
+#include "particles/particles_input_random.hpp"
+
+template <typename rnumber>
+int NSVEparticles<rnumber>::initialize(void)
+{
+ TIMEZONE("NSVEparticles::intialize");
+ this->NSVE<rnumber>::initialize();
+
+ // initialize particle output object
+ // this may be needed before the particle system,
+ // if cpp_random_particles is true
+ this->particles_output_writer_mpi = new particles_output_hdf5<
+ long long int, double, 3>(
+ MPI_COMM_WORLD,
+ "tracers0",
+ nparticles,
+ tracers0_integration_steps);
+
+ // if this is the first iteration, and we are supposed to generate our own random particles,
+ // we do that here, and then we write them to file for future reference.
+ if ((this->cpp_random_particles > 0) &&
+ this->fs->iteration == 0)
+ {
+ // temporary particle set
+ // interpolation setup is irrelevant for output
+ particle_set<3, 1, 1> pset(
+ this->fs->cvelocity->rlayout,
+ this->fs->kk->dkx,
+ this->fs->kk->dky,
+ this->fs->kk->dkz);
+ particles_input_random<long long int, double, 3> pinput(
+ this->comm,
+ this->nparticles,
+ this->cpp_random_particles, // seed
+ pset.getSpatialLowLimitZ(),
+ pset.getSpatialUpLimitZ());
+ // initialize particle set
+ pset.init(pinput);
+ // allocate dummy rhs data
+ // we must fill these arrays with 0
+ // otherwise we get floating point exceptions because data is being written/read
+ std::vector<std::unique_ptr<double[]>> rhs_data;
+ rhs_data.resize(tracers0_integration_steps);
+ for (int counter = 0; counter < tracers0_integration_steps; counter++)
+ {
+ rhs_data[counter].reset(new double[pset.getLocalNumberOfParticles()*3]); // memory allocated here will be freed outside of the "if" block
+ std::fill_n(rhs_data[counter].get(), pset.getLocalNumberOfParticles()*3, 0);
+ }
+ // create dummy file_layout object
+ std::vector<hsize_t> file_layout;
+ file_layout.resize(1);
+ file_layout[0] = this->nparticles;
+ // write data
+ this->particles_output_writer_mpi->open_file(this->fs->get_current_fname());
+ this->particles_output_writer_mpi->update_particle_species_name("tracers0");
+ this->particles_output_writer_mpi->setParticleFileLayout(file_layout);
+#ifdef USE_TIMINGOUTPUT
+ // barrier so that timings within save are more accurate.
+ MPI_Barrier(this->comm);
+#endif
+ this->particles_output_writer_mpi->template save<3>(
+ pset.getParticleState(),
+ rhs_data.data(),
+ pset.getParticleIndices(),
+ pset.getLocalNumberOfParticles(),
+ 0);
+ this->particles_output_writer_mpi->close_file();
+ }
+
+ DEBUG_MSG("fs->iteration = %d\n", this->fs->iteration);
+ DEBUG_MSG_WAIT(MPI_COMM_WORLD, "about to call particles_system_builder\n");
+ this->ps = particles_system_builder(
+ this->fs->cvelocity, // (field object)
+ this->fs->kk, // (kspace object, contains dkx, dky, dkz)
+ tracers0_integration_steps, // to check coherency between parameters and hdf input file (nb rhs)
+ (long long int)nparticles, // to check coherency between parameters and hdf input file
+ this->fs->get_current_fname(), // particles input filename
+ std::string("/tracers0/state/") + std::to_string(this->fs->iteration), // dataset name for initial input
+ std::string("/tracers0/rhs/") + std::to_string(this->fs->iteration), // dataset name for initial input
+ tracers0_neighbours, // parameter (interpolation no neighbours)
+ tracers0_smoothness, // parameter
+ this->comm,
+ this->fs->iteration+1);
+ DEBUG_MSG_WAIT(MPI_COMM_WORLD, "after call to particles_system_builder\n");
+
+ // initialize sample object
+ this->particles_sample_writer_mpi = new particles_output_sampling_hdf5<
+ long long int, double, double, 3>(
+ MPI_COMM_WORLD,
+ this->ps->getGlobalNbParticles(),
+ (this->simname + "_particles.h5"),
+ "tracers0",
+ "position/0");
+
+ // set particle file layout for output objects
+ this->particles_output_writer_mpi->setParticleFileLayout(this->ps->getParticleFileLayout());
+ this->particles_sample_writer_mpi->setParticleFileLayout(this->ps->getParticleFileLayout());
+ return EXIT_SUCCESS;
+}
+
+template <typename rnumber>
+int NSVEparticles<rnumber>::step(void)
+{
+ TIMEZONE("NSVEparticles::step");
+ this->fs->compute_velocity(this->fs->cvorticity);
+ this->fs->cvelocity->ift();
+#ifdef USE_TIMINGOUTPUT
+ // barrier so that timings for completeLoop are more accurate
+ MPI_Barrier(this->comm);
+#endif
+ this->ps->completeLoop(this->dt);
+#ifdef USE_TIMINGOUTPUT
+ // barrier so that timings for step are more accurate
+ MPI_Barrier(this->comm);
+#endif
+ this->NSVE<rnumber>::step();
+ return EXIT_SUCCESS;
+}
+
+template <typename rnumber>
+int NSVEparticles<rnumber>::write_checkpoint(void)
+{
+ TIMEZONE("NSVEparticles::write_checkpoint");
+ this->NSVE<rnumber>::write_checkpoint();
+ this->particles_output_writer_mpi->open_file(this->fs->get_current_fname());
+ this->particles_output_writer_mpi->template save<3>(
+ this->ps->getParticlesState(),
+ this->ps->getParticlesRhs(),
+ this->ps->getParticlesIndexes(),
+ this->ps->getLocalNbParticles(),
+ this->fs->iteration);
+ this->particles_output_writer_mpi->close_file();
+ return EXIT_SUCCESS;
+}
+
+template <typename rnumber>
+int NSVEparticles<rnumber>::finalize(void)
+{
+ TIMEZONE("NSVEparticles::finalize");
+ delete this->ps.release();
+ delete this->particles_output_writer_mpi;
+ delete this->particles_sample_writer_mpi;
+ this->NSVE<rnumber>::finalize();
+ return EXIT_SUCCESS;
+}
+
+/** \brief Compute fluid stats and sample fields at particle locations.
+ */
+
+template <typename rnumber>
+int NSVEparticles<rnumber>::do_stats()
+{
+ TIMEZONE("NSVEparticles::do_stats");
+ /// fluid stats go here
+ this->NSVE<rnumber>::do_stats();
+
+
+ start_mpi_profiling_zone(turtle_mpi_pcontrol::PARTICLES);
+ /// either one of two conditions suffices to compute statistics:
+ /// 1) current iteration is a multiple of niter_part
+ /// 2) we are within niter_part_fine_duration/2 of a multiple of niter_part_fine_period
+ if (!(this->iteration % this->niter_part == 0 ||
+ ((this->iteration + this->niter_part_fine_duration/2) % this->niter_part_fine_period <=
+ this->niter_part_fine_duration)))
+ return EXIT_SUCCESS;
+
+ // allocate temporary data array
+ std::unique_ptr<double[]> pdata(new double[3*this->ps->getLocalNbParticles()]);
+
+ /// copy position data
+
+ /// sample position
+ std::copy(this->ps->getParticlesState(),
+ this->ps->getParticlesState()+3*this->ps->getLocalNbParticles(),
+ pdata.get());
+ this->particles_sample_writer_mpi->template save_dataset<3>(
+ "tracers0",
+ "position",
+ this->ps->getParticlesState(),
+ &pdata,
+ this->ps->getParticlesIndexes(),
+ this->ps->getLocalNbParticles(),
+ this->ps->get_step_idx()-1);
+
+ /// sample velocity
+ std::fill_n(pdata.get(), 3*this->ps->getLocalNbParticles(), 0);
+ finish_mpi_profiling_zone(turtle_mpi_pcontrol::PARTICLES);
+ if (!(this->iteration % this->niter_stat == 0))
+ {
+ // we need to compute velocity field manually, because it didn't happen in NSVE::do_stats()
+ this->fs->compute_velocity(this->fs->cvorticity);
+ *this->tmp_vec_field = this->fs->cvelocity->get_cdata();
+ this->tmp_vec_field->ift();
+ }
+ start_mpi_profiling_zone(turtle_mpi_pcontrol::PARTICLES);
+ this->ps->sample_compute_field(*this->tmp_vec_field, pdata.get());
+ this->particles_sample_writer_mpi->template save_dataset<3>(
+ "tracers0",
+ "velocity",
+ this->ps->getParticlesState(),
+ &pdata,
+ this->ps->getParticlesIndexes(),
+ this->ps->getLocalNbParticles(),
+ this->ps->get_step_idx()-1);
+ finish_mpi_profiling_zone(turtle_mpi_pcontrol::PARTICLES);
+
+ /// compute acceleration and sample it
+ if (this->sample_acceleration == 1)
+ {
+ this->fs->compute_Lagrangian_acceleration(this->tmp_vec_field);
+ this->tmp_vec_field->ift();
+ start_mpi_profiling_zone(turtle_mpi_pcontrol::PARTICLES);
+ std::fill_n(pdata.get(), 3*this->ps->getLocalNbParticles(), 0);
+ this->ps->sample_compute_field(*this->tmp_vec_field, pdata.get());
+ this->particles_sample_writer_mpi->template save_dataset<3>(
+ "tracers0",
+ "acceleration",
+ this->ps->getParticlesState(),
+ &pdata,
+ this->ps->getParticlesIndexes(),
+ this->ps->getLocalNbParticles(),
+ this->ps->get_step_idx()-1);
+ }
+
+ // deallocate temporary data array
+ delete[] pdata.release();
+ finish_mpi_profiling_zone(turtle_mpi_pcontrol::PARTICLES);
+
+ return EXIT_SUCCESS;
+}
+
+template <typename rnumber>
+int NSVEparticles<rnumber>::read_parameters(void)
+{
+ TIMEZONE("NSVEparticles::read_parameters");
+ this->NSVE<rnumber>::read_parameters();
+ hid_t parameter_file = H5Fopen((this->simname + ".h5").c_str(), H5F_ACC_RDONLY, H5P_DEFAULT);
+ this->niter_part = hdf5_tools::read_value<int>(parameter_file, "parameters/niter_part");
+ this->niter_part_fine_period = hdf5_tools::read_value<int>(parameter_file, "parameters/niter_part_fine_period");
+ this->niter_part_fine_duration = hdf5_tools::read_value<int>(parameter_file, "parameters/niter_part_fine_duration");
+ this->nparticles = hdf5_tools::read_value<long long int>(parameter_file, "parameters/nparticles");
+ this->cpp_random_particles = hdf5_tools::read_value<int>(parameter_file, "parameters/cpp_random_particles");
+ this->tracers0_integration_steps = hdf5_tools::read_value<int>(parameter_file, "parameters/tracers0_integration_steps");
+ this->tracers0_neighbours = hdf5_tools::read_value<int>(parameter_file, "parameters/tracers0_neighbours");
+ this->tracers0_smoothness = hdf5_tools::read_value<int>(parameter_file, "parameters/tracers0_smoothness");
+ this->sample_acceleration = hdf5_tools::read_value<int>(parameter_file, "parameters/sample_acceleration");
+ H5Fclose(parameter_file);
+ MPI_Barrier(this->comm);
+ return EXIT_SUCCESS;
+}
+
+template class NSVEparticles<float>;
+template class NSVEparticles<double>;
+
diff --git a/bfps/cpp/full_code/NSVEparticles.hpp b/cpp/full_code/NSVEparticles.hpp
similarity index 77%
rename from bfps/cpp/full_code/NSVEparticles.hpp
rename to cpp/full_code/NSVEparticles.hpp
index ccafe6eeb09d27a6b211cfd75ecfba4fc5abe92b..dfa5598a911c3a6f7071312a8c22a43e9d0095e7 100644
--- a/bfps/cpp/full_code/NSVEparticles.hpp
+++ b/cpp/full_code/NSVEparticles.hpp
@@ -3,20 +3,20 @@
* Copyright 2017 Max Planck Institute *
* for Dynamics and Self-Organization *
* *
-* This file is part of bfps. *
+* This file is part of TurTLE. *
* *
-* bfps is free software: you can redistribute it and/or modify *
+* TurTLE is free software: you can redistribute it and/or modify *
* it under the terms of the GNU General Public License as published *
* by the Free Software Foundation, either version 3 of the License, *
* or (at your option) any later version. *
* *
-* bfps is distributed in the hope that it will be useful, *
+* TurTLE is distributed in the hope that it will be useful, *
* but WITHOUT ANY WARRANTY; without even the implied warranty of *
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the *
* GNU General Public License for more details. *
* *
* You should have received a copy of the GNU General Public License *
-* along with bfps. If not, see <http://www.gnu.org/licenses/> *
+* along with TurTLE. If not, see <http://www.gnu.org/licenses/> *
* *
* Contact: Cristian.Lalescu@ds.mpg.de *
* *
@@ -35,6 +35,7 @@
#include "full_code/NSVE.hpp"
#include "particles/particles_system_builder.hpp"
#include "particles/particles_output_hdf5.hpp"
+#include "particles/particles_sampling.hpp"
/** \brief Navier-Stokes solver that includes simple Lagrangian tracers.
*
@@ -50,14 +51,21 @@ class NSVEparticles: public NSVE<rnumber>
/* parameters that are read in read_parameters */
int niter_part;
- int nparticles;
+ int niter_part_fine_period;
+ int niter_part_fine_duration;
+ long long int nparticles;
int tracers0_integration_steps;
int tracers0_neighbours;
int tracers0_smoothness;
+ int cpp_random_particles;
+ int sample_acceleration;
/* other stuff */
std::unique_ptr<abstract_particles_system<long long int, double>> ps;
- particles_output_hdf5<long long int, double,3,3> *particles_output_writer_mpi;
+ field<rnumber, FFTW, ONE> *pressure;
+
+ particles_output_hdf5<long long int, double,3> *particles_output_writer_mpi;
+ particles_output_sampling_hdf5<long long int, double, double, 3> *particles_sample_writer_mpi;
NSVEparticles(
@@ -66,7 +74,7 @@ class NSVEparticles: public NSVE<rnumber>
NSVE<rnumber>(
COMMUNICATOR,
simulation_name){}
- ~NSVEparticles(){}
+ ~NSVEparticles() noexcept(false){}
int initialize(void);
int step(void);
diff --git a/cpp/full_code/NSVEparticles_no_output.hpp b/cpp/full_code/NSVEparticles_no_output.hpp
new file mode 100644
index 0000000000000000000000000000000000000000..bac0d886184d85295ede68f71474a5b6261c2f10
--- /dev/null
+++ b/cpp/full_code/NSVEparticles_no_output.hpp
@@ -0,0 +1,50 @@
+/******************************************************************************
+* *
+* Copyright 2019 Max Planck Institute for Dynamics and Self-Organization *
+* *
+* This file is part of TurTLE. *
+* *
+* TurTLE is free software: you can redistribute it and/or modify *
+* it under the terms of the GNU General Public License as published *
+* by the Free Software Foundation, either version 3 of the License, *
+* or (at your option) any later version. *
+* *
+* TurTLE is distributed in the hope that it will be useful, *
+* but WITHOUT ANY WARRANTY; without even the implied warranty of *
+* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the *
+* GNU General Public License for more details. *
+* *
+* You should have received a copy of the GNU General Public License *
+* along with TurTLE. If not, see <http://www.gnu.org/licenses/> *
+* *
+* Contact: Cristian.Lalescu@ds.mpg.de *
+* *
+******************************************************************************/
+
+
+
+#ifndef NSVEPARTICLES_NO_OUTPUT_HPP
+#define NSVEPARTICLES_NO_OUTPUT_HPP
+
+#include "full_code/NSVEparticles.hpp"
+
+template <typename rnumber>
+class NSVEparticles_no_output: public NSVEparticles<rnumber>
+{
+ public:
+ NSVEparticles_no_output(
+ const MPI_Comm COMMUNICATOR,
+ const std::string &simulation_name):
+ NSVEparticles<rnumber>(
+ COMMUNICATOR,
+ simulation_name){}
+ ~NSVEparticles_no_output(){}
+ int write_checkpoint(void)
+ {
+ TIMEZONE("NSVEparticles_no_output::write_checkpoint");
+ return EXIT_SUCCESS;
+ }
+};
+
+#endif//NSVEPARTICLES_NO_OUTPUT_HPP
+
diff --git a/cpp/full_code/bandpass_stats.cpp b/cpp/full_code/bandpass_stats.cpp
new file mode 100644
index 0000000000000000000000000000000000000000..7985a26af9be291915a0fcc088d4ccc084a6280f
--- /dev/null
+++ b/cpp/full_code/bandpass_stats.cpp
@@ -0,0 +1,206 @@
+/******************************************************************************
+* *
+* Copyright 2019 Max Planck Institute for Dynamics and Self-Organization *
+* *
+* This file is part of TurTLE. *
+* *
+* TurTLE is free software: you can redistribute it and/or modify *
+* it under the terms of the GNU General Public License as published *
+* by the Free Software Foundation, either version 3 of the License, *
+* or (at your option) any later version. *
+* *
+* TurTLE is distributed in the hope that it will be useful, *
+* but WITHOUT ANY WARRANTY; without even the implied warranty of *
+* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the *
+* GNU General Public License for more details. *
+* *
+* You should have received a copy of the GNU General Public License *
+* along with TurTLE. If not, see <http://www.gnu.org/licenses/> *
+* *
+* Contact: Cristian.Lalescu@ds.mpg.de *
+* *
+******************************************************************************/
+
+
+
+#include <string>
+#include <cmath>
+#include "bandpass_stats.hpp"
+#include "scope_timer.hpp"
+#include "hdf5_tools.hpp"
+
+
+template <typename rnumber>
+int bandpass_stats<rnumber>::initialize(void)
+{
+ DEBUG_MSG("entered bandpass_stats::initialize\n");
+ TIMEZONE("bandpass_stats::initialize");
+ this->NSVE_field_stats<rnumber>::initialize();
+ DEBUG_MSG("after NSVE_field_stats::initialize\n");
+ this->kk = new kspace<FFTW, SMOOTH>(
+ this->vorticity->clayout, this->dkx, this->dky, this->dkz);
+ hid_t parameter_file = H5Fopen(
+ (this->simname + std::string(".h5")).c_str(),
+ H5F_ACC_RDONLY,
+ H5P_DEFAULT);
+ this->niter_out = hdf5_tools::read_value<int>(parameter_file, "/parameters/niter_out");
+ this->checkpoints_per_file = hdf5_tools::read_value<int>(parameter_file, "/parameters/checkpoints_per_file");
+ if (this->checkpoints_per_file == INT_MAX) // value returned if dataset does not exist
+ this->checkpoints_per_file = 1;
+ H5Fclose(parameter_file);
+ // the following ensures the file is free for rank 0 to open in read/write mode
+ MPI_Barrier(this->comm);
+ parameter_file = H5Fopen(
+ (this->simname + std::string("_post.h5")).c_str(),
+ H5F_ACC_RDONLY,
+ H5P_DEFAULT);
+ this->iteration_list = hdf5_tools::read_vector<int>(
+ parameter_file,
+ "/bandpass_stats/parameters/iteration_list");
+ this->max_velocity_estimate = hdf5_tools::read_value<double>(
+ parameter_file,
+ "/bandpass_stats/parameters/max_velocity_estimate");
+ this->k0list = hdf5_tools::read_vector<double>(
+ parameter_file,
+ "/bandpass_stats/parameters/k0list");
+ this->k1list = hdf5_tools::read_vector<double>(
+ parameter_file,
+ "/bandpass_stats/parameters/k1list");
+ this->filter_type = hdf5_tools::read_string(
+ parameter_file,
+ "/bandpass_stats/parameters/filter_type");
+ H5Fclose(parameter_file);
+ // the following ensures the file is free for rank 0 to open in read/write mode
+ MPI_Barrier(this->comm);
+ assert(this->k0list.size() == this->k1list.size());
+ if (this->myrank == 0)
+ {
+ // set caching parameters
+ hid_t fapl = H5Pcreate(H5P_FILE_ACCESS);
+ herr_t cache_err = H5Pset_cache(fapl, 0, 521, 134217728, 1.0);
+ variable_used_only_in_assert(cache_err);
+ DEBUG_MSG("when setting stat_file cache I got %d\n", cache_err);
+ this->stat_file = H5Fopen(
+ (this->simname + "_post.h5").c_str(),
+ H5F_ACC_RDWR,
+ fapl);
+ }
+ else
+ {
+ this->stat_file = 0;
+ }
+ int data_file_problem;
+ if (this->myrank == 0)
+ data_file_problem = hdf5_tools::require_size_file_datasets(
+ this->stat_file,
+ "bandpass_stats",
+ (this->iteration_list.back() / this->niter_out) + 1);
+ MPI_Bcast(&data_file_problem, 1, MPI_INT, 0, this->comm);
+ if (data_file_problem > 0)
+ {
+ std::cerr <<
+ data_file_problem <<
+ " problems setting sizes of file datasets.\ntrying to exit now." <<
+ std::endl;
+ return EXIT_FAILURE;
+ }
+ return EXIT_SUCCESS;
+}
+
+template <typename rnumber>
+int bandpass_stats<rnumber>::work_on_current_iteration(void)
+{
+ TIMEZONE("bandpass_stats::work_on_current_iteration");
+ this->read_current_cvorticity();
+ field<rnumber, FFTW, THREE> *vel = new field<rnumber, FFTW, THREE>(
+ this->nx, this->ny, this->nz,
+ this->comm,
+ this->vorticity->fftw_plan_rigor);
+
+
+ /// compute the velocity, store in vel
+ invert_curl(
+ this->kk,
+ this->vorticity,
+ vel);
+
+ /// allocate temporary filtered field container
+ field<rnumber, FFTW, THREE> *filtered_vel0 = new field<rnumber, FFTW, THREE>(
+ this->nx, this->ny, this->nz,
+ this->comm,
+ this->vorticity->fftw_plan_rigor);
+ field<rnumber, FFTW, THREE> *filtered_vel1 = new field<rnumber, FFTW, THREE>(
+ this->nx, this->ny, this->nz,
+ this->comm,
+ this->vorticity->fftw_plan_rigor);
+
+ /// make max vel estimate std::vector, required by compute_rspace_stats
+ std::vector<double> max_vel_estimate;
+ max_vel_estimate.resize(4, max_velocity_estimate / sqrt(3));
+ max_vel_estimate[3] = max_velocity_estimate;
+
+ /// initialize `stat_group`.
+ /// i.e. open HDF5 file for writing stats
+ hid_t stat_group;
+ if (this->myrank == 0)
+ stat_group = H5Gopen(
+ this->stat_file,
+ "bandpass_stats",
+ H5P_DEFAULT);
+ else
+ stat_group = 0;
+
+ /// loop over all bands
+ for (int band_counter = 0; band_counter < int(this->k0list.size()); band_counter++)
+ {
+ *filtered_vel0 = *vel;
+ *filtered_vel1 = *vel;
+ this->kk->template filter_calibrated_ell<rnumber, THREE>(
+ filtered_vel0->get_cdata(),
+ 4*acos(0) / this->k0list[band_counter],
+ this->filter_type);
+ this->kk->template filter_calibrated_ell<rnumber, THREE>(
+ filtered_vel1->get_cdata(),
+ 4*acos(0) / this->k1list[band_counter],
+ this->filter_type);
+ this->kk->CLOOP(
+ [&](
+ ptrdiff_t cindex,
+ ptrdiff_t xindex,
+ ptrdiff_t yindex,
+ ptrdiff_t zindex){
+ for (unsigned int component=0; component < 3; component++)
+ for (unsigned int cc=0; cc < 2; cc++)
+ filtered_vel1->cval(cindex, component, cc) -= filtered_vel0->cval(cindex, component, cc);
+ });
+ filtered_vel1->ift();
+ filtered_vel1->compute_rspace_stats(
+ stat_group,
+ "velocity_band" + std::to_string(band_counter),
+ this->iteration / this->niter_out,
+ max_vel_estimate);
+ }
+
+ if (this->myrank == 0)
+ H5Gclose(stat_group);
+
+ delete filtered_vel1;
+ delete filtered_vel0;
+ delete vel;
+ return EXIT_SUCCESS;
+}
+
+template <typename rnumber>
+int bandpass_stats<rnumber>::finalize(void)
+{
+ TIMEZONE("bandpass_stats::finalize");
+ delete this->kk;
+ this->NSVE_field_stats<rnumber>::finalize();
+ if (this->myrank == 0)
+ H5Fclose(this->stat_file);
+ return EXIT_SUCCESS;
+}
+
+template class bandpass_stats<float>;
+template class bandpass_stats<double>;
+
diff --git a/cpp/full_code/bandpass_stats.hpp b/cpp/full_code/bandpass_stats.hpp
new file mode 100644
index 0000000000000000000000000000000000000000..2210fd3efe5c80fd5410da34decdc00b13a931d3
--- /dev/null
+++ b/cpp/full_code/bandpass_stats.hpp
@@ -0,0 +1,78 @@
+/**********************************************************************
+* *
+* Copyright 2017 Max Planck Institute *
+* for Dynamics and Self-Organization *
+* *
+* This file is part of TurTLE. *
+* *
+* TurTLE is free software: you can redistribute it and/or modify *
+* it under the terms of the GNU General Public License as published *
+* by the Free Software Foundation, either version 3 of the License, *
+* or (at your option) any later version. *
+* *
+* TurTLE is distributed in the hope that it will be useful, *
+* but WITHOUT ANY WARRANTY; without even the implied warranty of *
+* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the *
+* GNU General Public License for more details. *
+* *
+* You should have received a copy of the GNU General Public License *
+* along with TurTLE. If not, see <http://www.gnu.org/licenses/> *
+* *
+* Contact: Cristian.Lalescu@ds.mpg.de *
+* *
+**********************************************************************/
+
+
+
+#ifndef BANDPASS_STATS_HPP
+#define BANDPASS_STATS_HPP
+
+#include <cstdlib>
+#include <sys/types.h>
+#include <sys/stat.h>
+#include <vector>
+#include "base.hpp"
+#include "field.hpp"
+#include "field_binary_IO.hpp"
+#include "full_code/NSVE_field_stats.hpp"
+
+/** \brief A class for running bandpass statistics.
+ *
+ * This class computes statistics for a particular type of filter.
+ * Given a list of pairs of length-scales, the velocity field is filtered
+ * with a band-pass filter for each pair of length scales, and standard
+ * statistics of the band-passed field are computed.
+ * Relevant publication:
+ * Drivas, T. D. and Johnson, P. L. and Lalescu, C. C. and Wilczek, M. Phys Rev Fluids 2 104603 (2017)
+ * https://dx.doi.org/10.1103/PhysRevFluids.2.104603
+ * Not all statistics discussed in the paper are performed here, but the
+ * existing code can be expanded if needed.
+ */
+
+template <typename rnumber>
+class bandpass_stats: public NSVE_field_stats<rnumber>
+{
+ public:
+ int checkpoints_per_file;
+ int niter_out;
+ double max_velocity_estimate;
+ kspace<FFTW, SMOOTH> *kk;
+ /* parameters that are read in read_parameters */
+ std::vector<double> k0list, k1list;
+ std::string filter_type;
+
+ bandpass_stats(
+ const MPI_Comm COMMUNICATOR,
+ const std::string &simulation_name):
+ NSVE_field_stats<rnumber>(
+ COMMUNICATOR,
+ simulation_name){}
+ virtual ~bandpass_stats(){}
+
+ int initialize(void);
+ int work_on_current_iteration(void);
+ int finalize(void);
+};
+
+#endif//BANDPASS_STATS_HPP
+
diff --git a/cpp/full_code/code_base.cpp b/cpp/full_code/code_base.cpp
new file mode 100644
index 0000000000000000000000000000000000000000..b62f50150fe815fc554c5bf63eab8568fa0218e6
--- /dev/null
+++ b/cpp/full_code/code_base.cpp
@@ -0,0 +1,82 @@
+/**********************************************************************
+* *
+* Copyright 2017 Max Planck Institute *
+* for Dynamics and Self-Organization *
+* *
+* This file is part of TurTLE. *
+* *
+* TurTLE is free software: you can redistribute it and/or modify *
+* it under the terms of the GNU General Public License as published *
+* by the Free Software Foundation, either version 3 of the License, *
+* or (at your option) any later version. *
+* *
+* TurTLE is distributed in the hope that it will be useful, *
+* but WITHOUT ANY WARRANTY; without even the implied warranty of *
+* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the *
+* GNU General Public License for more details. *
+* *
+* You should have received a copy of the GNU General Public License *
+* along with TurTLE. If not, see <http://www.gnu.org/licenses/> *
+* *
+* Contact: Cristian.Lalescu@ds.mpg.de *
+* *
+**********************************************************************/
+
+
+
+#include "code_base.hpp"
+#include "scope_timer.hpp"
+#include "hdf5_tools.hpp"
+
+code_base::code_base(
+ const MPI_Comm COMMUNICATOR,
+ const std::string &simulation_name):
+ comm(COMMUNICATOR),
+ simname(simulation_name)
+{
+ TIMEZONE("code_base::code_base");
+ MPI_Comm_rank(this->comm, &this->myrank);
+ MPI_Comm_size(this->comm, &this->nprocs);
+ this->stop_code_now = false;
+ this->iteration = 0;
+}
+
+int code_base::check_stopping_condition(void)
+{
+ TIMEZONE("code_base::check_stopping_condition");
+ if (myrank == 0)
+ {
+ std::string fname = (
+ std::string("stop_") +
+ std::string(this->simname));
+ {
+ struct stat file_buffer;
+ this->stop_code_now = (
+ stat(fname.c_str(), &file_buffer) == 0);
+ }
+ }
+ MPI_Bcast(
+ &this->stop_code_now,
+ 1,
+ MPI_C_BOOL,
+ 0,
+ MPI_COMM_WORLD);
+ return EXIT_SUCCESS;
+}
+
+int code_base::read_parameters(void)
+{
+ TIMEZONE("code_base::read_parameters");
+ hid_t parameter_file = H5Fopen((this->simname + ".h5").c_str(), H5F_ACC_RDONLY, H5P_DEFAULT);
+ this->dkx = hdf5_tools::read_value<double>(parameter_file, "parameters/dkx");
+ this->dky = hdf5_tools::read_value<double>(parameter_file, "parameters/dky");
+ this->dkz = hdf5_tools::read_value<double>(parameter_file, "parameters/dkz");
+ this->nx = hdf5_tools::read_value<int>(parameter_file, "parameters/nx");
+ this->ny = hdf5_tools::read_value<int>(parameter_file, "parameters/ny");
+ this->nz = hdf5_tools::read_value<int>(parameter_file, "parameters/nz");
+ this->dealias_type = hdf5_tools::read_value<int>(parameter_file, "parameters/dealias_type");
+ H5Fclose(parameter_file);
+ MPI_Barrier(this->comm);
+ return EXIT_SUCCESS;
+}
+
diff --git a/bfps/cpp/full_code/code_base.hpp b/cpp/full_code/code_base.hpp
similarity index 85%
rename from bfps/cpp/full_code/code_base.hpp
rename to cpp/full_code/code_base.hpp
index cf0521e2b7383edf925e1129d4fa4a931a55efe4..017785d4a3e52cd92c06c2fa356a6c1c76763eaa 100644
--- a/bfps/cpp/full_code/code_base.hpp
+++ b/cpp/full_code/code_base.hpp
@@ -3,20 +3,20 @@
* Copyright 2017 Max Planck Institute *
* for Dynamics and Self-Organization *
* *
-* This file is part of bfps. *
+* This file is part of TurTLE. *
* *
-* bfps is free software: you can redistribute it and/or modify *
+* TurTLE is free software: you can redistribute it and/or modify *
* it under the terms of the GNU General Public License as published *
* by the Free Software Foundation, either version 3 of the License, *
* or (at your option) any later version. *
* *
-* bfps is distributed in the hope that it will be useful, *
+* TurTLE is distributed in the hope that it will be useful, *
* but WITHOUT ANY WARRANTY; without even the implied warranty of *
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the *
* GNU General Public License for more details. *
* *
* You should have received a copy of the GNU General Public License *
-* along with bfps. If not, see <http://www.gnu.org/licenses/> *
+* along with TurTLE. If not, see <http://www.gnu.org/licenses/> *
* *
* Contact: Cristian.Lalescu@ds.mpg.de *
* *
@@ -71,7 +71,7 @@ class code_base
code_base(
const MPI_Comm COMMUNICATOR,
const std::string &simulation_name);
- virtual ~code_base(){}
+ virtual ~code_base() noexcept(false){}
int check_stopping_condition(void);
@@ -108,6 +108,13 @@ class code_base
return EXIT_SUCCESS;
}
+ /** Reads parameters
+ * \warning This method should ensure the parameter file is closed by
+ * all MPI processes when finished.
+ * One solution is to open the file in sequential mode, read-only, and
+ * then call `MPI_Barrier`.
+ */
+ virtual int read_parameters(void);
virtual int initialize(void) = 0;
virtual int main_loop(void) = 0;
virtual int finalize(void) = 0;
diff --git a/cpp/full_code/codes_with_no_output.hpp b/cpp/full_code/codes_with_no_output.hpp
new file mode 100644
index 0000000000000000000000000000000000000000..fd04f6e0bb98e720eddda9ab155e3e2034b32e0e
--- /dev/null
+++ b/cpp/full_code/codes_with_no_output.hpp
@@ -0,0 +1,34 @@
+/******************************************************************************
+* *
+* Copyright 2019 Max Planck Institute for Dynamics and Self-Organization *
+* *
+* This file is part of TurTLE. *
+* *
+* TurTLE is free software: you can redistribute it and/or modify *
+* it under the terms of the GNU General Public License as published *
+* by the Free Software Foundation, either version 3 of the License, *
+* or (at your option) any later version. *
+* *
+* TurTLE is distributed in the hope that it will be useful, *
+* but WITHOUT ANY WARRANTY; without even the implied warranty of *
+* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the *
+* GNU General Public License for more details. *
+* *
+* You should have received a copy of the GNU General Public License *
+* along with TurTLE. If not, see <http://www.gnu.org/licenses/> *
+* *
+* Contact: Cristian.Lalescu@ds.mpg.de *
+* *
+******************************************************************************/
+
+
+
+#ifndef CODES_WITH_NO_OUTPUT_HPP
+#define CODES_WITH_NO_OUTPUT_HPP
+
+#include "full_code/NSVE_no_output.hpp"
+#include "full_code/NSVEparticles_no_output.hpp"
+
+
+#endif//CODES_WITH_NO_OUTPUT_HPP
+
diff --git a/cpp/full_code/dealias_test.cpp b/cpp/full_code/dealias_test.cpp
new file mode 100644
index 0000000000000000000000000000000000000000..56980b54a9bbb8c7e35574e7c26908cd948a831f
--- /dev/null
+++ b/cpp/full_code/dealias_test.cpp
@@ -0,0 +1,187 @@
+/**********************************************************************
+* *
+* Copyright 2019 Max Planck Institute *
+* for Dynamics and Self-Organization *
+* *
+* This file is part of TurTLE. *
+* *
+* TurTLE is free software: you can redistribute it and/or modify *
+* it under the terms of the GNU General Public License as published *
+* by the Free Software Foundation, either version 3 of the License, *
+* or (at your option) any later version. *
+* *
+* TurTLE is distributed in the hope that it will be useful, *
+* but WITHOUT ANY WARRANTY; without even the implied warranty of *
+* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the *
+* GNU General Public License for more details. *
+* *
+* You should have received a copy of the GNU General Public License *
+* along with TurTLE. If not, see <http://www.gnu.org/licenses/> *
+* *
+* Contact: Cristian.Lalescu@ds.mpg.de *
+* *
+**********************************************************************/
+
+
+
+#include <string>
+#include <cmath>
+#include <random>
+#include "dealias_test.hpp"
+#include "scope_timer.hpp"
+#include "hdf5_tools.hpp"
+
+
+
+
+
+template <typename rnumber>
+int dealias_test<rnumber>::initialize(void)
+{
+ TIMEZONE("dealias_test::initialize");
+ this->read_parameters();
+ this->vector_field0 = new field<rnumber, FFTW, THREE>(
+ nx, ny, nz,
+ this->comm,
+ FFTW_ESTIMATE);
+ this->vector_field1 = new field<rnumber, FFTW, THREE>(
+ nx, ny, nz,
+ this->comm,
+ FFTW_ESTIMATE);
+ this->kk = new kspace<FFTW, SMOOTH>(
+ this->vector_field0->clayout, this->dkx, this->dky, this->dkz);
+ this->kk4 = new kspace<FFTW, ONE_HALF>(
+ this->vector_field0->clayout, this->dkx, this->dky, this->dkz);
+ this->kk3 = new kspace<FFTW, TWO_THIRDS>(
+ this->vector_field0->clayout, this->dkx, this->dky, this->dkz);
+
+ if (this->myrank == 0)
+ {
+ hid_t stat_file = H5Fopen(
+ (this->simname + std::string(".h5")).c_str(),
+ H5F_ACC_RDWR,
+ H5P_DEFAULT);
+ this->kk->store(stat_file);
+ H5Fclose(stat_file);
+ }
+ return EXIT_SUCCESS;
+}
+
+template <typename rnumber>
+int dealias_test<rnumber>::finalize(void)
+{
+ TIMEZONE("dealias_test::finalize");
+ delete this->vector_field1;
+ delete this->vector_field0;
+ delete this->kk3;
+ delete this->kk4;
+ delete this->kk;
+ return EXIT_SUCCESS;
+}
+
+template <typename rnumber>
+int dealias_test<rnumber>::read_parameters()
+{
+ TIMEZONE("dealias_test::read_parameters");
+ this->test::read_parameters();
+ hid_t parameter_file = H5Fopen(
+ (this->simname + std::string(".h5")).c_str(),
+ H5F_ACC_RDONLY,
+ H5P_DEFAULT);
+ this->max_velocity_estimate = hdf5_tools::read_value<double>(parameter_file, "/parameters/max_velocity_estimate");
+ this->spectrum_dissipation = hdf5_tools::read_value<double>(parameter_file, "parameters/spectrum_dissipation");
+ this->spectrum_Lint = hdf5_tools::read_value<double>(parameter_file, "parameters/spectrum_Lint");
+ this->spectrum_etaK = hdf5_tools::read_value<double>(parameter_file, "parameters/spectrum_etaK");
+ this->spectrum_large_scale_const = hdf5_tools::read_value<double>(parameter_file, "parameters/spectrum_large_scale_const");
+ this->spectrum_small_scale_const = hdf5_tools::read_value<double>(parameter_file, "parameters/spectrum_small_scale_const");
+ this->random_seed = hdf5_tools::read_value<int>(parameter_file, "/parameters/field_random_seed");
+ H5Fclose(parameter_file);
+ MPI_Barrier(this->comm);
+ return EXIT_SUCCESS;
+}
+
+template <typename rnumber>
+int dealias_test<rnumber>::do_work(void)
+{
+ TIMEZONE("dealias_test::do_work");
+
+ /// initialize vector Gaussian random field
+ make_gaussian_random_field(
+ this->kk,
+ this->vector_field0,
+ this->random_seed,
+ this->spectrum_dissipation,
+ this->spectrum_Lint,
+ this->spectrum_etaK,
+ this->spectrum_large_scale_const,
+ this->spectrum_small_scale_const,
+ 3./2. //incompressibility projection factor
+ );
+
+ /// impose divergence free condition while maintaining the energy of the field
+ this->kk->template project_divfree<rnumber>(this->vector_field0->get_cdata());
+
+ /// initialize statistics file.
+ hid_t stat_group, stat_file;
+ if (this->myrank == 0)
+ {
+ // set caching parameters
+ hid_t fapl = H5Pcreate(H5P_FILE_ACCESS);
+ herr_t cache_err = H5Pset_cache(fapl, 0, 521, 134217728, 1.0);
+ variable_used_only_in_assert(cache_err);
+ DEBUG_MSG("when setting stat_file cache I got %d\n", cache_err);
+ stat_file = H5Fopen(
+ (this->simname + ".h5").c_str(),
+ H5F_ACC_RDWR,
+ fapl);
+ stat_group = H5Gopen(
+ stat_file,
+ "statistics",
+ H5P_DEFAULT);
+ }
+ else
+ {
+ stat_file = 0;
+ stat_group = 0;
+ }
+
+ /// copy field to temporary field
+ *this->vector_field1 = *this->vector_field0;
+
+ /// dealias temporary field with two thirds sharp cutoff
+ this->kk3->template dealias<rnumber, THREE>(this->vector_field1->get_cdata());
+
+ /// compute basic statistics of temporary field
+ this->vector_field1->compute_stats(
+ this->kk,
+ stat_group,
+ "field3",
+ 0,
+ this->max_velocity_estimate);
+
+ /// copy field to temporary field
+ *this->vector_field1 = *this->vector_field0;
+
+ /// dealias temporary field with one half sharp cutoff
+ this->kk4->template dealias<rnumber, THREE>(this->vector_field1->get_cdata());
+
+ /// compute basic statistics of temporary field
+ this->vector_field1->compute_stats(
+ this->kk,
+ stat_group,
+ "field4",
+ 0,
+ this->max_velocity_estimate);
+
+ /// close stat file
+ if (this->myrank == 0)
+ {
+ H5Gclose(stat_group);
+ H5Fclose(stat_file);
+ }
+ return EXIT_SUCCESS;
+}
+
+template class dealias_test<float>;
+template class dealias_test<double>;
+
diff --git a/cpp/full_code/dealias_test.hpp b/cpp/full_code/dealias_test.hpp
new file mode 100644
index 0000000000000000000000000000000000000000..ba765c45086760910cd8bbc7270b55189e58b48e
--- /dev/null
+++ b/cpp/full_code/dealias_test.hpp
@@ -0,0 +1,77 @@
+/**********************************************************************
+* *
+* Copyright 2019 Max Planck Institute *
+* for Dynamics and Self-Organization *
+* *
+* This file is part of TurTLE. *
+* *
+* TurTLE is free software: you can redistribute it and/or modify *
+* it under the terms of the GNU General Public License as published *
+* by the Free Software Foundation, either version 3 of the License, *
+* or (at your option) any later version. *
+* *
+* TurTLE is distributed in the hope that it will be useful, *
+* but WITHOUT ANY WARRANTY; without even the implied warranty of *
+* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the *
+* GNU General Public License for more details. *
+* *
+* You should have received a copy of the GNU General Public License *
+* along with TurTLE. If not, see <http://www.gnu.org/licenses/> *
+* *
+* Contact: Cristian.Lalescu@ds.mpg.de *
+* *
+**********************************************************************/
+
+
+
+#ifndef DEALIAS_TEST_HPP
+#define DEALIAS_TEST_HPP
+
+
+
+#include <cstdlib>
+#include "base.hpp"
+#include "kspace.hpp"
+#include "field.hpp"
+#include "full_code/test.hpp"
+
+/** \brief Basic sanity check for dealiasing.
+ *
+ */
+
+template <typename rnumber>
+class dealias_test: public test
+{
+ public:
+
+ /* parameters that are read in read_parameters */
+ double max_velocity_estimate;
+ double spectrum_dissipation;
+ double spectrum_Lint;
+ double spectrum_etaK;
+ double spectrum_large_scale_const;
+ double spectrum_small_scale_const;
+ int random_seed;
+
+ /* other stuff */
+ kspace<FFTW, SMOOTH> *kk;
+ kspace<FFTW, ONE_HALF> *kk4;
+ kspace<FFTW, TWO_THIRDS> *kk3;
+ field<rnumber, FFTW, THREE> *vector_field0, *vector_field1;
+
+ dealias_test(
+ const MPI_Comm COMMUNICATOR,
+ const std::string &simulation_name):
+ test(
+ COMMUNICATOR,
+ simulation_name){}
+ ~dealias_test() noexcept(false){}
+
+ int initialize(void);
+ int do_work(void);
+ int finalize(void);
+ int read_parameters(void);
+};
+
+#endif//DEALIAS_TEST_HPP
+
diff --git a/bfps/cpp/full_code/direct_numerical_simulation.cpp b/cpp/full_code/direct_numerical_simulation.cpp
similarity index 51%
rename from bfps/cpp/full_code/direct_numerical_simulation.cpp
rename to cpp/full_code/direct_numerical_simulation.cpp
index edc2f99497a21368c63348167190dc6c64b44712..a12a79f8c7016d5c344134e054a61f6d509d5e3c 100644
--- a/bfps/cpp/full_code/direct_numerical_simulation.cpp
+++ b/cpp/full_code/direct_numerical_simulation.cpp
@@ -1,3 +1,28 @@
+/******************************************************************************
+* *
+* Copyright 2019 Max Planck Institute for Dynamics and Self-Organization *
+* *
+* This file is part of TurTLE. *
+* *
+* TurTLE is free software: you can redistribute it and/or modify *
+* it under the terms of the GNU General Public License as published *
+* by the Free Software Foundation, either version 3 of the License, *
+* or (at your option) any later version. *
+* *
+* TurTLE is distributed in the hope that it will be useful, *
+* but WITHOUT ANY WARRANTY; without even the implied warranty of *
+* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the *
+* GNU General Public License for more details. *
+* *
+* You should have received a copy of the GNU General Public License *
+* along with TurTLE. If not, see <http://www.gnu.org/licenses/> *
+* *
+* Contact: Cristian.Lalescu@ds.mpg.de *
+* *
+******************************************************************************/
+
+
+
#include <cstdlib>
#include <sys/types.h>
#include <sys/stat.h>
@@ -5,9 +30,9 @@
#include "scope_timer.hpp"
#include "hdf5_tools.hpp"
-
int direct_numerical_simulation::grow_file_datasets()
{
+ TIMEZONE("direct_numerical_simulation::grow_file_datasets");
return hdf5_tools::grow_file_datasets(
this->stat_file,
"statistics",
@@ -16,6 +41,7 @@ int direct_numerical_simulation::grow_file_datasets()
int direct_numerical_simulation::read_iteration(void)
{
+ TIMEZONE("direct_numerical_simulation::read_iteration");
/* read iteration */
hid_t dset;
hid_t iteration_file = H5Fopen(
@@ -56,6 +82,7 @@ int direct_numerical_simulation::read_iteration(void)
int direct_numerical_simulation::write_iteration(void)
{
+ TIMEZONE("direct_numerical_simulation::write_iteration");
if (this->myrank == 0)
{
hid_t dset = H5Dopen(
@@ -86,8 +113,16 @@ int direct_numerical_simulation::write_iteration(void)
return EXIT_SUCCESS;
}
+int direct_numerical_simulation::update_checkpoint(void)
+{
+ assert(this->checkpoints_per_file > 0);
+ this->checkpoint = this->iteration / (this->niter_out*this->checkpoints_per_file);
+ return EXIT_SUCCESS;
+}
+
int direct_numerical_simulation::main_loop(void)
{
+ TIMEZONE("direct_numerical_simulation::main_loop");
this->start_simple_timer();
int max_iter = (this->iteration + this->niter_todo -
(this->iteration % this->niter_todo));
@@ -117,3 +152,16 @@ int direct_numerical_simulation::main_loop(void)
return EXIT_SUCCESS;
}
+int direct_numerical_simulation::read_parameters(void)
+{
+ TIMEZONE("direct_numerical_simulation::read_parameters");
+ this->code_base::read_parameters();
+ hid_t parameter_file = H5Fopen((this->simname + ".h5").c_str(), H5F_ACC_RDONLY, H5P_DEFAULT);
+ this->checkpoints_per_file = hdf5_tools::read_value<int>(parameter_file, "parameters/checkpoints_per_file");
+ this->niter_out = hdf5_tools::read_value<int>(parameter_file, "parameters/niter_out");
+ this->niter_stat = hdf5_tools::read_value<int>(parameter_file, "parameters/niter_stat");
+ this->niter_todo = hdf5_tools::read_value<int>(parameter_file, "parameters/niter_todo");
+ H5Fclose(parameter_file);
+ MPI_Barrier(this->comm);
+ return EXIT_SUCCESS;
+}
diff --git a/bfps/cpp/full_code/direct_numerical_simulation.hpp b/cpp/full_code/direct_numerical_simulation.hpp
similarity index 70%
rename from bfps/cpp/full_code/direct_numerical_simulation.hpp
rename to cpp/full_code/direct_numerical_simulation.hpp
index 8050bb045b29acf29d655273f7dff310dd10d0fa..be5d498866372c2a7843fbac0768add479dcc271 100644
--- a/bfps/cpp/full_code/direct_numerical_simulation.hpp
+++ b/cpp/full_code/direct_numerical_simulation.hpp
@@ -3,20 +3,20 @@
* Copyright 2017 Max Planck Institute *
* for Dynamics and Self-Organization *
* *
-* This file is part of bfps. *
+* This file is part of TurTLE. *
* *
-* bfps is free software: you can redistribute it and/or modify *
+* TurTLE is free software: you can redistribute it and/or modify *
* it under the terms of the GNU General Public License as published *
* by the Free Software Foundation, either version 3 of the License, *
* or (at your option) any later version. *
* *
-* bfps is distributed in the hope that it will be useful, *
+* TurTLE is distributed in the hope that it will be useful, *
* but WITHOUT ANY WARRANTY; without even the implied warranty of *
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the *
* GNU General Public License for more details. *
* *
* You should have received a copy of the GNU General Public License *
-* along with bfps. If not, see <http://www.gnu.org/licenses/> *
+* along with TurTLE. If not, see <http://www.gnu.org/licenses/> *
* *
* Contact: Cristian.Lalescu@ds.mpg.de *
* *
@@ -31,11 +31,13 @@
#include <sys/types.h>
#include <sys/stat.h>
#include "base.hpp"
+#include "hdf5_tools.hpp"
#include "full_code/code_base.hpp"
class direct_numerical_simulation: public code_base
{
public:
+ std::string name;
int checkpoint;
int checkpoints_per_file;
int niter_out;
@@ -49,8 +51,9 @@ class direct_numerical_simulation: public code_base
code_base(
COMMUNICATOR,
simulation_name){}
- virtual ~direct_numerical_simulation(){}
+ virtual ~direct_numerical_simulation() noexcept(false){}
+ virtual int read_parameters(void);
virtual int write_checkpoint(void) = 0;
virtual int initialize(void) = 0;
virtual int step(void) = 0;
@@ -61,6 +64,22 @@ class direct_numerical_simulation: public code_base
int read_iteration(void);
int write_iteration(void);
int grow_file_datasets(void);
+
+ /** \brief Basic computation of current checkpoint
+ *
+ * This will fail if the values of `niter_out`, `niter_todo`, or
+ * `checkpoints_per_file` are changed.
+ */
+ int update_checkpoint(void);
+
+ const inline std::string get_current_fname()
+ {
+ return (
+ std::string(this->simname) +
+ std::string("_checkpoint_") +
+ std::to_string(this->checkpoint) +
+ std::string(".h5"));
+ }
};
#endif//DIRECT_NUMERICAL_SIMULATION_HPP
diff --git a/cpp/full_code/field_output_test.cpp b/cpp/full_code/field_output_test.cpp
new file mode 100644
index 0000000000000000000000000000000000000000..89c2883a7cc48d84755004a6ba8b2a486b09dcde
--- /dev/null
+++ b/cpp/full_code/field_output_test.cpp
@@ -0,0 +1,93 @@
+/******************************************************************************
+* *
+* Copyright 2019 Max Planck Institute for Dynamics and Self-Organization *
+* *
+* This file is part of TurTLE. *
+* *
+* TurTLE is free software: you can redistribute it and/or modify *
+* it under the terms of the GNU General Public License as published *
+* by the Free Software Foundation, either version 3 of the License, *
+* or (at your option) any later version. *
+* *
+* TurTLE is distributed in the hope that it will be useful, *
+* but WITHOUT ANY WARRANTY; without even the implied warranty of *
+* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the *
+* GNU General Public License for more details. *
+* *
+* You should have received a copy of the GNU General Public License *
+* along with TurTLE. If not, see <http://www.gnu.org/licenses/> *
+* *
+* Contact: Cristian.Lalescu@ds.mpg.de *
+* *
+******************************************************************************/
+
+
+
+#include <string>
+#include <cmath>
+#include <random>
+#include "field_output_test.hpp"
+#include "scope_timer.hpp"
+
+
+template <typename rnumber>
+int field_output_test<rnumber>::initialize(void)
+{
+ TIMEZONE("field_output_test::initialize");
+ this->read_parameters();
+ return EXIT_SUCCESS;
+}
+
+template <typename rnumber>
+int field_output_test<rnumber>::finalize(void)
+{
+ TIMEZONE("field_output_test::finalize");
+ return EXIT_SUCCESS;
+}
+
+template <typename rnumber>
+int field_output_test<rnumber>::read_parameters()
+{
+ TIMEZONE("field_output_test::read_parameters");
+ this->test::read_parameters();
+ MPI_Barrier(this->comm);
+ return EXIT_SUCCESS;
+}
+
+template <typename rnumber>
+int field_output_test<rnumber>::do_work(void)
+{
+ TIMEZONE("field_output_test::do_work");
+ // allocate
+ field<rnumber, FFTW, ONE> *scal_field = new field<rnumber, FFTW, ONE>(
+ this->nx, this->ny, this->nz,
+ this->comm,
+ FFTW_ESTIMATE);
+ std::default_random_engine rgen;
+ std::normal_distribution<rnumber> rdist;
+ rgen.seed(1);
+
+ // fill up scal_field
+ scal_field->real_space_representation = true;
+ scal_field->RLOOP(
+ [&](ptrdiff_t rindex,
+ ptrdiff_t xindex,
+ ptrdiff_t yindex,
+ ptrdiff_t zindex){
+ scal_field->rval(rindex) = rdist(rgen);
+ });
+
+ scal_field->io(
+ this->simname + std::string("_fields.h5"),
+ "scal_field",
+ 0,
+ false);
+
+ // deallocate
+ delete scal_field;
+ return EXIT_SUCCESS;
+}
+
+template class field_output_test<float>;
+template class field_output_test<double>;
+
diff --git a/cpp/full_code/field_output_test.hpp b/cpp/full_code/field_output_test.hpp
new file mode 100644
index 0000000000000000000000000000000000000000..72c832450de7d921700cf11df36670f857ba365d
--- /dev/null
+++ b/cpp/full_code/field_output_test.hpp
@@ -0,0 +1,60 @@
+/**********************************************************************
+* *
+* Copyright 2017 Max Planck Institute *
+* for Dynamics and Self-Organization *
+* *
+* This file is part of TurTLE. *
+* *
+* TurTLE is free software: you can redistribute it and/or modify *
+* it under the terms of the GNU General Public License as published *
+* by the Free Software Foundation, either version 3 of the License, *
+* or (at your option) any later version. *
+* *
+* TurTLE is distributed in the hope that it will be useful, *
+* but WITHOUT ANY WARRANTY; without even the implied warranty of *
+* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the *
+* GNU General Public License for more details. *
+* *
+* You should have received a copy of the GNU General Public License *
+* along with TurTLE. If not, see <http://www.gnu.org/licenses/> *
+* *
+* Contact: Cristian.Lalescu@ds.mpg.de *
+* *
+**********************************************************************/
+
+
+
+#ifndef FILTER_OUTPUT_TEST_HPP
+#define FILTER_OUTPUT_TEST_HPP
+
+
+
+#include <cstdlib>
+#include "base.hpp"
+#include "kspace.hpp"
+#include "field.hpp"
+#include "full_code/test.hpp"
+
+/** \brief A class for testing basic field class functionality.
+ */
+
+template <typename rnumber>
+class field_output_test: public test
+{
+ public:
+ field_output_test(
+ const MPI_Comm COMMUNICATOR,
+ const std::string &simulation_name):
+ test(
+ COMMUNICATOR,
+ simulation_name){}
+ ~field_output_test(){}
+
+ int initialize(void);
+ int do_work(void);
+ int finalize(void);
+ int read_parameters(void);
+};
+
+#endif//FILTER_OUTPUT_TEST_HPP
+
diff --git a/cpp/full_code/field_single_to_double.cpp b/cpp/full_code/field_single_to_double.cpp
new file mode 100644
index 0000000000000000000000000000000000000000..9f2082e63613e7c2d64943c01d8c782f57e4a1cf
--- /dev/null
+++ b/cpp/full_code/field_single_to_double.cpp
@@ -0,0 +1,126 @@
+/******************************************************************************
+* *
+* Copyright 2019 Max Planck Institute for Dynamics and Self-Organization *
+* *
+* This file is part of TurTLE. *
+* *
+* TurTLE is free software: you can redistribute it and/or modify *
+* it under the terms of the GNU General Public License as published *
+* by the Free Software Foundation, either version 3 of the License, *
+* or (at your option) any later version. *
+* *
+* TurTLE is distributed in the hope that it will be useful, *
+* but WITHOUT ANY WARRANTY; without even the implied warranty of *
+* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the *
+* GNU General Public License for more details. *
+* *
+* You should have received a copy of the GNU General Public License *
+* along with TurTLE. If not, see <http://www.gnu.org/licenses/> *
+* *
+* Contact: Cristian.Lalescu@ds.mpg.de *
+* *
+******************************************************************************/
+
+
+
+#include <string>
+#include <cmath>
+#include "field_single_to_double.hpp"
+#include "scope_timer.hpp"
+#include "hdf5_tools.hpp"
+
+
+template <typename rnumber>
+int field_single_to_double<rnumber>::initialize(void)
+{
+ TIMEZONE("field_single_to_double::intialize");
+ this->NSVE_field_stats<rnumber>::initialize();
+ DEBUG_MSG("after NSVE_field_stats::initialize\n");
+ this->kk = new kspace<FFTW, SMOOTH>(
+ this->vorticity->clayout, this->dkx, this->dky, this->dkz);
+ this->vec_field_double = new field<double, FFTW, THREE>(
+ this->nx, this->ny, this->nz,
+ this->comm,
+ this->vorticity->fftw_plan_rigor);
+ this->vec_field_double->real_space_representation = false;
+ hid_t parameter_file = H5Fopen(
+ (this->simname + std::string(".h5")).c_str(),
+ H5F_ACC_RDONLY,
+ H5P_DEFAULT);
+ hid_t dset = H5Dopen(parameter_file, "/parameters/niter_out", H5P_DEFAULT);
+ H5Dread(dset, H5T_NATIVE_INT, H5S_ALL, H5S_ALL, H5P_DEFAULT, &this->niter_out);
+ H5Dclose(dset);
+ if (H5Lexists(parameter_file, "/parameters/checkpoints_per_file", H5P_DEFAULT))
+ {
+ dset = H5Dopen(parameter_file, "/parameters/checkpoints_per_file", H5P_DEFAULT);
+ H5Dread(dset, H5T_NATIVE_INT, H5S_ALL, H5S_ALL, H5P_DEFAULT, &this->checkpoints_per_file);
+ H5Dclose(dset);
+ }
+ else
+ this->checkpoints_per_file = 1;
+ H5Fclose(parameter_file);
+ // the following ensures the file is free for rank 0 to open in read/write mode
+ MPI_Barrier(this->comm);
+ parameter_file = H5Fopen(
+ (this->simname + std::string("_post.h5")).c_str(),
+ H5F_ACC_RDONLY,
+ H5P_DEFAULT);
+ DEBUG_MSG("before read_vector\n");
+ this->iteration_list = hdf5_tools::read_vector<int>(
+ parameter_file,
+ "/field_single_to_double/parameters/iteration_list");
+ H5Fclose(parameter_file);
+ // the following ensures the file is free for rank 0 to open in read/write mode
+ MPI_Barrier(this->comm);
+ return EXIT_SUCCESS;
+}
+
+template <typename rnumber>
+int field_single_to_double<rnumber>::work_on_current_iteration(void)
+{
+ TIMEZONE("field_single_to_double::work_on_current_iteration");
+ DEBUG_MSG("before read_vorticity at iteration %d\n", this->iteration);
+ this->read_current_cvorticity();
+ DEBUG_MSG("after read_vorticity at iteration %d\n", this->iteration);
+
+ // using CLOOP as opposed to a global std::copy because CLOOP
+ // is openmp parallelized.
+ this->kk->CLOOP(
+ [&](ptrdiff_t cindex,
+ ptrdiff_t xindex,
+ ptrdiff_t yindex,
+ ptrdiff_t zindex){
+ {
+ std::copy(
+ (rnumber*)(this->vorticity->get_cdata() + cindex*3),
+ (rnumber*)(this->vorticity->get_cdata() + cindex*3) + 6,
+ (double*)(this->vec_field_double->get_cdata() + cindex*3));
+ }
+ }
+ );
+
+ std::string fname = (
+ this->simname +
+ std::string("_checkpoint_double_") +
+ std::to_string(this->iteration / (this->niter_out*this->checkpoints_per_file)) +
+ std::string(".h5"));
+ this->vec_field_double->io(
+ fname,
+ "vorticity",
+ this->iteration,
+ false);
+
+ return EXIT_SUCCESS;
+}
+
+template <typename rnumber>
+int field_single_to_double<rnumber>::finalize(void)
+{
+ TIMEZONE("field_single_to_double::finalize");
+ delete this->vec_field_double;
+ delete this->kk;
+ return EXIT_SUCCESS;
+}
+
+template class field_single_to_double<float>;
+
diff --git a/cpp/full_code/field_single_to_double.hpp b/cpp/full_code/field_single_to_double.hpp
new file mode 100644
index 0000000000000000000000000000000000000000..de98e63ed5fda3fa9e6ffadfbb076c38436d414f
--- /dev/null
+++ b/cpp/full_code/field_single_to_double.hpp
@@ -0,0 +1,63 @@
+/**********************************************************************
+* *
+* Copyright 2017 Max Planck Institute *
+* for Dynamics and Self-Organization *
+* *
+* This file is part of TurTLE. *
+* *
+* TurTLE is free software: you can redistribute it and/or modify *
+* it under the terms of the GNU General Public License as published *
+* by the Free Software Foundation, either version 3 of the License, *
+* or (at your option) any later version. *
+* *
+* TurTLE is distributed in the hope that it will be useful, *
+* but WITHOUT ANY WARRANTY; without even the implied warranty of *
+* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the *
+* GNU General Public License for more details. *
+* *
+* You should have received a copy of the GNU General Public License *
+* along with TurTLE. If not, see <http://www.gnu.org/licenses/> *
+* *
+* Contact: Cristian.Lalescu@ds.mpg.de *
+* *
+**********************************************************************/
+
+
+
+#ifndef FIELD_SINGLE_TO_DOUBLE_HPP
+#define FIELD_SINGLE_TO_DOUBLE_HPP
+
+#include <cstdlib>
+#include <sys/types.h>
+#include <sys/stat.h>
+#include <vector>
+#include "base.hpp"
+#include "field.hpp"
+#include "field_binary_IO.hpp"
+#include "full_code/NSVE_field_stats.hpp"
+
+template <typename rnumber>
+class field_single_to_double: public NSVE_field_stats<rnumber>
+{
+ public:
+ int checkpoints_per_file;
+ int niter_out;
+ kspace<FFTW, SMOOTH> *kk;
+
+ field<double, FFTW, THREE> *vec_field_double;
+
+ field_single_to_double(
+ const MPI_Comm COMMUNICATOR,
+ const std::string &simulation_name):
+ NSVE_field_stats<rnumber>(
+ COMMUNICATOR,
+ simulation_name){}
+ virtual ~field_single_to_double() noexcept(false){}
+
+ int initialize(void);
+ int work_on_current_iteration(void);
+ int finalize(void);
+};
+
+#endif//FIELD_SINGLE_TO_DOUBLE_HPP
+
diff --git a/cpp/full_code/field_test.cpp b/cpp/full_code/field_test.cpp
new file mode 100644
index 0000000000000000000000000000000000000000..0aa2b940fe69a5919d7ce6a7c38bc2d1ffb07f93
--- /dev/null
+++ b/cpp/full_code/field_test.cpp
@@ -0,0 +1,149 @@
+/******************************************************************************
+* *
+* Copyright 2019 Max Planck Institute for Dynamics and Self-Organization *
+* *
+* This file is part of TurTLE. *
+* *
+* TurTLE is free software: you can redistribute it and/or modify *
+* it under the terms of the GNU General Public License as published *
+* by the Free Software Foundation, either version 3 of the License, *
+* or (at your option) any later version. *
+* *
+* TurTLE is distributed in the hope that it will be useful, *
+* but WITHOUT ANY WARRANTY; without even the implied warranty of *
+* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the *
+* GNU General Public License for more details. *
+* *
+* You should have received a copy of the GNU General Public License *
+* along with TurTLE. If not, see <http://www.gnu.org/licenses/> *
+* *
+* Contact: Cristian.Lalescu@ds.mpg.de *
+* *
+******************************************************************************/
+
+
+
+#include <string>
+#include <cmath>
+#include <random>
+#include "field_test.hpp"
+#include "scope_timer.hpp"
+#include "hdf5_tools.hpp"
+
+
+template <typename rnumber>
+int field_test<rnumber>::initialize(void)
+{
+ TIMEZONE("field_test::initialize");
+ this->read_parameters();
+ return EXIT_SUCCESS;
+}
+
+template <typename rnumber>
+int field_test<rnumber>::finalize(void)
+{
+ TIMEZONE("field_test::finalize");
+ this->read_parameters();
+ return EXIT_SUCCESS;
+}
+
+template <typename rnumber>
+int field_test<rnumber>::read_parameters()
+{
+ TIMEZONE("field_test::read_parameters");
+ this->test::read_parameters();
+ // in case any parameters are needed, this is where they should be read
+ hid_t parameter_file = H5Fopen(
+ (this->simname + std::string(".h5")).c_str(),
+ H5F_ACC_RDONLY,
+ H5P_DEFAULT);
+ this->filter_length = hdf5_tools::read_value<double>(parameter_file, "/parameters/filter_length");
+ H5Fclose(parameter_file);
+ MPI_Barrier(this->comm);
+ return EXIT_SUCCESS;
+}
+
+template <typename rnumber>
+int field_test<rnumber>::do_work(void)
+{
+ TIMEZONE("field_test::do_work");
+ // allocate
+ field<rnumber, FFTW, ONE> *scal_field = new field<rnumber, FFTW, ONE>(
+ this->nx, this->ny, this->nz,
+ this->comm,
+ FFTW_ESTIMATE);
+ field<rnumber, FFTW, ONE> *scal_field_alt = new field<rnumber, FFTW, ONE>(
+ this->nx, this->ny, this->nz,
+ this->comm,
+ FFTW_ESTIMATE);
+ std::default_random_engine rgen;
+ std::normal_distribution<rnumber> rdist;
+ rgen.seed(2);
+ //auto gaussian = std::bind(rgen, rdist);
+ kspace<FFTW,SMOOTH> *kk = new kspace<FFTW, SMOOTH>(
+ scal_field->clayout, this->dkx, this->dky, this->dkz);
+
+ if (this->myrank == 0)
+ {
+ hid_t stat_file = H5Fopen(
+ (this->simname + std::string(".h5")).c_str(),
+ H5F_ACC_RDWR,
+ H5P_DEFAULT);
+ kk->store(stat_file);
+ H5Fclose(stat_file);
+ }
+
+ // fill up scal_field
+ scal_field->real_space_representation = true;
+ scal_field->RLOOP(
+ [&](ptrdiff_t rindex,
+ ptrdiff_t xindex,
+ ptrdiff_t yindex,
+ ptrdiff_t zindex){
+ scal_field->rval(rindex) = rdist(rgen);
+ });
+
+ *scal_field_alt = scal_field->get_rdata();
+ double L2r = scal_field->L2norm(kk);
+ variable_used_only_in_assert(L2r);
+ scal_field->dft();
+ double L2c = scal_field->L2norm(kk);
+ variable_used_only_in_assert(L2c);
+ scal_field->ift();
+ scal_field->normalize();
+ DEBUG_MSG("L2r = %g, L2c = %g\n",
+ L2r, L2c / scal_field->npoints);
+
+ double max_error = 0;
+ scal_field->RLOOP(
+ [&](ptrdiff_t rindex,
+ ptrdiff_t xindex,
+ ptrdiff_t yindex,
+ ptrdiff_t zindex){
+ double tval = fabs(scal_field->rval(rindex) - scal_field_alt->rval(rindex));
+ if (max_error < tval)
+ max_error = tval;
+ });
+
+ DEBUG_MSG("maximum error is %g\n", max_error);
+
+ scal_field->dft();
+ kk->template dealias<rnumber, ONE>(scal_field->get_cdata());
+ scal_field->symmetrize();
+ scal_field->normalize();
+ L2c = scal_field->L2norm(kk);
+ scal_field->ift();
+ L2r = scal_field->L2norm(kk);
+ DEBUG_MSG("L2r = %g, L2c = %g\n",
+ L2r, L2c);
+
+ // deallocate
+ delete kk;
+ delete scal_field;
+ delete scal_field_alt;
+ return EXIT_SUCCESS;
+}
+
+template class field_test<float>;
+template class field_test<double>;
+
diff --git a/cpp/full_code/field_test.hpp b/cpp/full_code/field_test.hpp
new file mode 100644
index 0000000000000000000000000000000000000000..5a57c1d2e6f5b0f92ddeb8a01f238521cf3e5bfe
--- /dev/null
+++ b/cpp/full_code/field_test.hpp
@@ -0,0 +1,63 @@
+/**********************************************************************
+* *
+* Copyright 2017 Max Planck Institute *
+* for Dynamics and Self-Organization *
+* *
+* This file is part of TurTLE. *
+* *
+* TurTLE is free software: you can redistribute it and/or modify *
+* it under the terms of the GNU General Public License as published *
+* by the Free Software Foundation, either version 3 of the License, *
+* or (at your option) any later version. *
+* *
+* TurTLE is distributed in the hope that it will be useful, *
+* but WITHOUT ANY WARRANTY; without even the implied warranty of *
+* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the *
+* GNU General Public License for more details. *
+* *
+* You should have received a copy of the GNU General Public License *
+* along with TurTLE. If not, see <http://www.gnu.org/licenses/> *
+* *
+* Contact: Cristian.Lalescu@ds.mpg.de *
+* *
+**********************************************************************/
+
+
+
+#ifndef FILTER_TEST_HPP
+#define FILTER_TEST_HPP
+
+
+
+#include <cstdlib>
+#include "base.hpp"
+#include "kspace.hpp"
+#include "field.hpp"
+#include "full_code/test.hpp"
+
+/** \brief A class for testing basic field class functionality.
+ */
+
+template <typename rnumber>
+class field_test: public test
+{
+ public:
+ double filter_length;
+ // kspace, in case we want to compute spectra or smth
+
+ field_test(
+ const MPI_Comm COMMUNICATOR,
+ const std::string &simulation_name):
+ test(
+ COMMUNICATOR,
+ simulation_name){}
+ ~field_test(){}
+
+ int initialize(void);
+ int do_work(void);
+ int finalize(void);
+ int read_parameters(void);
+};
+
+#endif//FILTER_TEST_HPP
+
diff --git a/bfps/cpp/full_code/filter_test.cpp b/cpp/full_code/filter_test.cpp
similarity index 65%
rename from bfps/cpp/full_code/filter_test.cpp
rename to cpp/full_code/filter_test.cpp
index aeedfbe74806adcff53a97d6c227b8fdcd30195f..e70cc61c7ce8d40b606262a451e99f3829958832 100644
--- a/bfps/cpp/full_code/filter_test.cpp
+++ b/cpp/full_code/filter_test.cpp
@@ -1,17 +1,44 @@
+/******************************************************************************
+* *
+* Copyright 2019 Max Planck Institute for Dynamics and Self-Organization *
+* *
+* This file is part of TurTLE. *
+* *
+* TurTLE is free software: you can redistribute it and/or modify *
+* it under the terms of the GNU General Public License as published *
+* by the Free Software Foundation, either version 3 of the License, *
+* or (at your option) any later version. *
+* *
+* TurTLE is distributed in the hope that it will be useful, *
+* but WITHOUT ANY WARRANTY; without even the implied warranty of *
+* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the *
+* GNU General Public License for more details. *
+* *
+* You should have received a copy of the GNU General Public License *
+* along with TurTLE. If not, see <http://www.gnu.org/licenses/> *
+* *
+* Contact: Cristian.Lalescu@ds.mpg.de *
+* *
+******************************************************************************/
+
+
+
#include <string>
#include <cmath>
#include "filter_test.hpp"
#include "scope_timer.hpp"
+#include "hdf5_tools.hpp"
template <typename rnumber>
int filter_test<rnumber>::initialize(void)
{
+ TIMEZONE("filter_test::initialize");
this->read_parameters();
this->scal_field = new field<rnumber, FFTW, ONE>(
nx, ny, nz,
this->comm,
- DEFAULT_FFTW_FLAG);
+ FFTW_ESTIMATE);
this->kk = new kspace<FFTW, SMOOTH>(
this->scal_field->clayout, this->dkx, this->dky, this->dkz);
@@ -30,6 +57,7 @@ int filter_test<rnumber>::initialize(void)
template <typename rnumber>
int filter_test<rnumber>::finalize(void)
{
+ TIMEZONE("filter_test::finalize");
delete this->scal_field;
delete this->kk;
return EXIT_SUCCESS;
@@ -38,17 +66,15 @@ int filter_test<rnumber>::finalize(void)
template <typename rnumber>
int filter_test<rnumber>::read_parameters()
{
+ TIMEZONE("filter_test::read_parameters");
this->test::read_parameters();
- hid_t parameter_file;
- hid_t dset, memtype, space;
- parameter_file = H5Fopen(
+ hid_t parameter_file = H5Fopen(
(this->simname + std::string(".h5")).c_str(),
H5F_ACC_RDONLY,
H5P_DEFAULT);
- dset = H5Dopen(parameter_file, "/parameters/filter_length", H5P_DEFAULT);
- H5Dread(dset, H5T_NATIVE_DOUBLE, H5S_ALL, H5S_ALL, H5P_DEFAULT, &this->filter_length);
- H5Dclose(dset);
+ this->filter_length = hdf5_tools::read_value<double>(parameter_file, "/parameters/filter_length");
H5Fclose(parameter_file);
+ MPI_Barrier(this->comm);
return EXIT_SUCCESS;
}
@@ -56,6 +82,7 @@ template <typename rnumber>
int filter_test<rnumber>::reset_field(
int dimension)
{
+ TIMEZONE("filter_test::reset_field");
this->scal_field->real_space_representation = true;
*this->scal_field = 0.0;
if (this->scal_field->rlayout->starts[0] == 0)
@@ -95,6 +122,7 @@ int filter_test<rnumber>::reset_field(
template <typename rnumber>
int filter_test<rnumber>::do_work(void)
{
+ TIMEZONE("filter_test::do_work");
std::string filename = this->simname + std::string("_fields.h5");
for (int dimension = 0; dimension < 3; dimension++)
{
diff --git a/bfps/cpp/full_code/filter_test.hpp b/cpp/full_code/filter_test.hpp
similarity index 90%
rename from bfps/cpp/full_code/filter_test.hpp
rename to cpp/full_code/filter_test.hpp
index adb8d470198d49ee93fa7e46d67e4f12cdf9dd2d..c7df600f7b638c34617be03e68691f2305f5087a 100644
--- a/bfps/cpp/full_code/filter_test.hpp
+++ b/cpp/full_code/filter_test.hpp
@@ -3,20 +3,20 @@
* Copyright 2017 Max Planck Institute *
* for Dynamics and Self-Organization *
* *
-* This file is part of bfps. *
+* This file is part of TurTLE. *
* *
-* bfps is free software: you can redistribute it and/or modify *
+* TurTLE is free software: you can redistribute it and/or modify *
* it under the terms of the GNU General Public License as published *
* by the Free Software Foundation, either version 3 of the License, *
* or (at your option) any later version. *
* *
-* bfps is distributed in the hope that it will be useful, *
+* TurTLE is distributed in the hope that it will be useful, *
* but WITHOUT ANY WARRANTY; without even the implied warranty of *
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the *
* GNU General Public License for more details. *
* *
* You should have received a copy of the GNU General Public License *
-* along with bfps. If not, see <http://www.gnu.org/licenses/> *
+* along with TurTLE. If not, see <http://www.gnu.org/licenses/> *
* *
* Contact: Cristian.Lalescu@ds.mpg.de *
* *
diff --git a/cpp/full_code/get_3D_correlations.cpp b/cpp/full_code/get_3D_correlations.cpp
new file mode 100644
index 0000000000000000000000000000000000000000..6970e7af4a10e2f97d0a9fca542a0ee65989d800
--- /dev/null
+++ b/cpp/full_code/get_3D_correlations.cpp
@@ -0,0 +1,147 @@
+/******************************************************************************
+* *
+* Copyright 2019 Max Planck Institute for Dynamics and Self-Organization *
+* *
+* This file is part of TurTLE. *
+* *
+* TurTLE is free software: you can redistribute it and/or modify *
+* it under the terms of the GNU General Public License as published *
+* by the Free Software Foundation, either version 3 of the License, *
+* or (at your option) any later version. *
+* *
+* TurTLE is distributed in the hope that it will be useful, *
+* but WITHOUT ANY WARRANTY; without even the implied warranty of *
+* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the *
+* GNU General Public License for more details. *
+* *
+* You should have received a copy of the GNU General Public License *
+* along with TurTLE. If not, see <http://www.gnu.org/licenses/> *
+* *
+* Contact: Cristian.Lalescu@ds.mpg.de *
+* *
+******************************************************************************/
+
+
+
+#include <string>
+#include <cmath>
+#include "get_3D_correlations.hpp"
+#include "scope_timer.hpp"
+#include "hdf5_tools.hpp"
+
+template <typename rnumber>
+int get_3D_correlations<rnumber>::initialize(void)
+{
+ TIMEZONE("get_3D_correlations::initialize");
+ this->NSVE_field_stats<rnumber>::initialize();
+ DEBUG_MSG("after NSVE_field_stats::initialize\n");
+
+ // allocate kspace
+ this->kk = new kspace<FFTW, SMOOTH>(
+ this->vorticity->clayout, this->dkx, this->dky, this->dkz);
+ // allocate field for velocity
+ this->vel = new field<rnumber, FFTW, THREE>(
+ this->nx, this->ny, this->nz,
+ this->comm,
+ this->vorticity->fftw_plan_rigor);
+ this->vel->real_space_representation = false;
+ // allocate field for velocity correlations
+ this->Rij = new field<rnumber, FFTW, THREExTHREE>(
+ this->nx, this->ny, this->nz,
+ this->comm,
+ this->vorticity->fftw_plan_rigor);
+ this->Rij->real_space_representation = false;
+
+ // open parameter file
+ hid_t parameter_file = H5Fopen(
+ (this->simname + std::string(".h5")).c_str(),
+ H5F_ACC_RDONLY,
+ H5P_DEFAULT);
+ this->niter_out = hdf5_tools::read_value<int>(parameter_file, "/parameters/niter_out");
+ this->checkpoints_per_file = hdf5_tools::read_value<int>(parameter_file, "/parameters/checkpoints_per_file");
+ if (this->checkpoints_per_file == INT_MAX) // value returned if dataset does not exist
+ this->checkpoints_per_file = 1;
+ H5Fclose(parameter_file);
+ // the following ensures the file is free for rank 0 to open in read/write mode
+ MPI_Barrier(this->comm);
+ parameter_file = H5Fopen(
+ (this->simname + std::string("_post.h5")).c_str(),
+ H5F_ACC_RDONLY,
+ H5P_DEFAULT);
+ this->iteration_list = hdf5_tools::read_vector<int>(
+ parameter_file,
+ "/get_3D_correlations/parameters/iteration_list");
+ H5Fclose(parameter_file);
+ // the following ensures the file is free for rank 0 to open in read/write mode
+ MPI_Barrier(this->comm);
+ return EXIT_SUCCESS;
+}
+
+template <typename rnumber>
+int get_3D_correlations<rnumber>::work_on_current_iteration(void)
+{
+ TIMEZONE("get_3D_correlations::work_on_current_iteration");
+ this->read_current_cvorticity();
+
+ invert_curl(
+ this->kk,
+ this->vorticity,
+ this->vel);
+ *(this->Rij) = 0.0;
+ this->Rij->real_space_representation = false;
+
+
+ const double dkvolume = this->kk->dkx*this->kk->dky*this->kk->dkz;
+ // compute Rij
+ this->kk->CLOOP_K2(
+ [&](ptrdiff_t cindex,
+ ptrdiff_t xindex,
+ ptrdiff_t yindex,
+ ptrdiff_t zindex,
+ const double k2){
+ // compute v_i[k] v_j[-k]
+ for (unsigned int i=0; i<3; i++)
+ for (unsigned int j=0; j<3; j++)
+ {
+ this->Rij->cval(cindex, i, j, 0) = (
+ this->vel->cval(cindex, i, 0)*this->vel->cval(cindex, j, 0) +
+ this->vel->cval(cindex, i, 1)*this->vel->cval(cindex, j, 1)) / dkvolume;
+ this->Rij->cval(cindex, i, j, 1) = (
+ this->vel->cval(cindex, i, 0)*this->vel->cval(cindex, j, 1) -
+ this->vel->cval(cindex, i, 1)*this->vel->cval(cindex, j, 0)) / dkvolume;
+ }
+ });
+
+ // go to real space
+ this->Rij->ift();
+
+ const std::string fname = (
+ this->simname +
+ std::string("_checkpoint_Rij_") +
+ std::to_string(this->iteration / (this->niter_out*this->checkpoints_per_file)) +
+ std::string(".h5"));
+
+ /// output Rij field
+ this->Rij->io(
+ fname,
+ "Rij",
+ this->iteration,
+ false);
+
+ return EXIT_SUCCESS;
+}
+
+template <typename rnumber>
+int get_3D_correlations<rnumber>::finalize(void)
+{
+ TIMEZONE("get_3D_correlations::finalize");
+ delete this->kk;
+ delete this->vel;
+ delete this->Rij;
+ this->NSVE_field_stats<rnumber>::finalize();
+ return EXIT_SUCCESS;
+}
+
+template class get_3D_correlations<float>;
+template class get_3D_correlations<double>;
+
diff --git a/cpp/full_code/get_3D_correlations.hpp b/cpp/full_code/get_3D_correlations.hpp
new file mode 100644
index 0000000000000000000000000000000000000000..e328d83cea5921de74a83474baf363a37540c2f4
--- /dev/null
+++ b/cpp/full_code/get_3D_correlations.hpp
@@ -0,0 +1,63 @@
+/**********************************************************************
+* *
+* Copyright 2017 Max Planck Institute *
+* for Dynamics and Self-Organization *
+* *
+* This file is part of TurTLE. *
+* *
+* TurTLE is free software: you can redistribute it and/or modify *
+* it under the terms of the GNU General Public License as published *
+* by the Free Software Foundation, either version 3 of the License, *
+* or (at your option) any later version. *
+* *
+* TurTLE is distributed in the hope that it will be useful, *
+* but WITHOUT ANY WARRANTY; without even the implied warranty of *
+* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the *
+* GNU General Public License for more details. *
+* *
+* You should have received a copy of the GNU General Public License *
+* along with TurTLE. If not, see <http://www.gnu.org/licenses/> *
+* *
+* Contact: Cristian.Lalescu@ds.mpg.de *
+* *
+**********************************************************************/
+
+
+
+#ifndef GET_3D_CORRELATIONS_HPP
+#define GET_3D_CORRELATIONS_HPP
+
+#include <cstdlib>
+#include <sys/types.h>
+#include <sys/stat.h>
+#include <vector>
+#include "base.hpp"
+#include "field.hpp"
+#include "field_binary_IO.hpp"
+#include "full_code/NSVE_field_stats.hpp"
+
+template <typename rnumber>
+class get_3D_correlations: public NSVE_field_stats<rnumber>
+{
+ public:
+ int checkpoints_per_file;
+ int niter_out;
+ kspace<FFTW, SMOOTH> *kk;
+ field<rnumber, FFTW, THREE> *vel;
+ field<rnumber, FFTW, THREExTHREE> *Rij;
+
+ get_3D_correlations(
+ const MPI_Comm COMMUNICATOR,
+ const std::string &simulation_name):
+ NSVE_field_stats<rnumber>(
+ COMMUNICATOR,
+ simulation_name){}
+ virtual ~get_3D_correlations(){}
+
+ int initialize(void);
+ int work_on_current_iteration(void);
+ int finalize(void);
+};
+
+#endif//GET_3D_CORRELATIONS_HPP
+
diff --git a/cpp/full_code/get_rfields.cpp b/cpp/full_code/get_rfields.cpp
new file mode 100644
index 0000000000000000000000000000000000000000..85864ee4d8f4fda468539a94cd43512a6aea5573
--- /dev/null
+++ b/cpp/full_code/get_rfields.cpp
@@ -0,0 +1,169 @@
+/******************************************************************************
+* *
+* Copyright 2019 Max Planck Institute for Dynamics and Self-Organization *
+* *
+* This file is part of TurTLE. *
+* *
+* TurTLE is free software: you can redistribute it and/or modify *
+* it under the terms of the GNU General Public License as published *
+* by the Free Software Foundation, either version 3 of the License, *
+* or (at your option) any later version. *
+* *
+* TurTLE is distributed in the hope that it will be useful, *
+* but WITHOUT ANY WARRANTY; without even the implied warranty of *
+* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the *
+* GNU General Public License for more details. *
+* *
+* You should have received a copy of the GNU General Public License *
+* along with TurTLE. If not, see <http://www.gnu.org/licenses/> *
+* *
+* Contact: Cristian.Lalescu@ds.mpg.de *
+* *
+******************************************************************************/
+
+
+
+#include <string>
+#include <cmath>
+#include "get_rfields.hpp"
+#include "scope_timer.hpp"
+#include "hdf5_tools.hpp"
+
+
+template <typename rnumber>
+int get_rfields<rnumber>::initialize(void)
+{
+ TIMEZONE("get_rfields::initialize");
+ this->NSVE_field_stats<rnumber>::initialize();
+ DEBUG_MSG("after NSVE_field_stats::initialize\n");
+
+ // allocate kspace
+ this->kk = new kspace<FFTW, SMOOTH>(
+ this->vorticity->clayout, this->dkx, this->dky, this->dkz);
+ // allocate field for TrS2
+ this->traceS2 = new field<rnumber, FFTW, ONE>(
+ this->nx, this->ny, this->nz,
+ this->comm,
+ this->vorticity->fftw_plan_rigor);
+ this->traceS2->real_space_representation = true;
+ // allocate field for velocity
+ this->vel = new field<rnumber, FFTW, THREE>(
+ this->nx, this->ny, this->nz,
+ this->comm,
+ this->vorticity->fftw_plan_rigor);
+ this->vel->real_space_representation = false;
+ // allocate field for velocity gradient
+ this->grad_vel = new field<rnumber, FFTW, THREExTHREE>(
+ this->nx, this->ny, this->nz,
+ this->comm,
+ this->vorticity->fftw_plan_rigor);
+ hid_t parameter_file = H5Fopen(
+ (this->simname + std::string(".h5")).c_str(),
+ H5F_ACC_RDONLY,
+ H5P_DEFAULT);
+ this->niter_out = hdf5_tools::read_value<int>(parameter_file, "/parameters/niter_out");
+ this->checkpoints_per_file = hdf5_tools::read_value<int>(parameter_file, "/parameters/checkpoints_per_file");
+ if (this->checkpoints_per_file == INT_MAX) // value returned if dataset does not exist
+ this->checkpoints_per_file = 1;
+ H5Fclose(parameter_file);
+ // the following ensures the file is free for rank 0 to open in read/write mode
+ MPI_Barrier(this->comm);
+ parameter_file = H5Fopen(
+ (this->simname + std::string("_post.h5")).c_str(),
+ H5F_ACC_RDONLY,
+ H5P_DEFAULT);
+ this->iteration_list = hdf5_tools::read_vector<int>(
+ parameter_file,
+ "/get_rfields/parameters/iteration_list");
+ this->TrS2_on = hdf5_tools::read_value<int>(
+ parameter_file,
+ "/get_rfields/parameters/TrS2_on");
+ H5Fclose(parameter_file);
+ // the following ensures the file is free for rank 0 to open in read/write mode
+ MPI_Barrier(this->comm);
+ return EXIT_SUCCESS;
+}
+
+template <typename rnumber>
+int get_rfields<rnumber>::work_on_current_iteration(void)
+{
+ TIMEZONE("get_rfields::work_on_current_iteration");
+ this->read_current_cvorticity();
+
+ invert_curl(
+ this->kk,
+ this->vorticity,
+ this->vel);
+
+ /// compute velocity gradient
+ compute_gradient<rnumber, FFTW, THREE, THREExTHREE>(
+ this->kk,
+ this->vel,
+ this->grad_vel);
+
+ // compute TrS2
+ this->grad_vel->ift();
+ this->traceS2->RLOOP(
+ [&](ptrdiff_t rindex,
+ ptrdiff_t xindex,
+ ptrdiff_t yindex,
+ ptrdiff_t zindex){
+ rnumber AxxAxx = this->grad_vel->rval(rindex, 0, 0)*this->grad_vel->rval(rindex, 0, 0);
+ rnumber AyyAyy = this->grad_vel->rval(rindex, 1, 1)*this->grad_vel->rval(rindex, 1, 1);
+ rnumber AzzAzz = this->grad_vel->rval(rindex, 2, 2)*this->grad_vel->rval(rindex, 2, 2);
+ rnumber Sxy = this->grad_vel->rval(rindex, 0, 1) + this->grad_vel->rval(rindex, 1, 0);
+ rnumber Syz = this->grad_vel->rval(rindex, 1, 2) + this->grad_vel->rval(rindex, 2, 1);
+ rnumber Szx = this->grad_vel->rval(rindex, 2, 0) + this->grad_vel->rval(rindex, 0, 2);
+ this->traceS2->rval(rindex) = (
+ AxxAxx + AyyAyy + AzzAzz +
+ (Sxy*Sxy + Syz*Syz + Szx*Szx)/2);
+ });
+
+ std::string fname = (
+ this->simname +
+ std::string("_checkpoint_") +
+ std::to_string(this->iteration / (this->niter_out*this->checkpoints_per_file)) +
+ std::string(".h5"));
+
+ /// output velocity field
+ this->vel->ift();
+ this->vel->io(
+ fname,
+ "velocity",
+ this->iteration,
+ false);
+
+ /// output vorticity field
+ this->vorticity->ift();
+ this->vorticity->io(
+ fname,
+ "vorticity",
+ this->iteration,
+ false);
+
+ if (this->TrS2_on)
+ {
+ this->traceS2->io(
+ fname,
+ "TrS2",
+ this->iteration,
+ false);
+ }
+ return EXIT_SUCCESS;
+}
+
+template <typename rnumber>
+int get_rfields<rnumber>::finalize(void)
+{
+ TIMEZONE("get_rfields::finalize");
+ delete this->kk;
+ delete this->traceS2;
+ delete this->vel;
+ delete this->grad_vel;
+ this->NSVE_field_stats<rnumber>::finalize();
+ return EXIT_SUCCESS;
+}
+
+template class get_rfields<float>;
+template class get_rfields<double>;
+
diff --git a/bfps/cpp/full_code/get_rfields.hpp b/cpp/full_code/get_rfields.hpp
similarity index 83%
rename from bfps/cpp/full_code/get_rfields.hpp
rename to cpp/full_code/get_rfields.hpp
index ae669aa3012ce492835a9737ca443cdc846e00ba..2d266d390322fbf3c2ecb631c95e074aef298491 100644
--- a/bfps/cpp/full_code/get_rfields.hpp
+++ b/cpp/full_code/get_rfields.hpp
@@ -3,20 +3,20 @@
* Copyright 2017 Max Planck Institute *
* for Dynamics and Self-Organization *
* *
-* This file is part of bfps. *
+* This file is part of TurTLE. *
* *
-* bfps is free software: you can redistribute it and/or modify *
+* TurTLE is free software: you can redistribute it and/or modify *
* it under the terms of the GNU General Public License as published *
* by the Free Software Foundation, either version 3 of the License, *
* or (at your option) any later version. *
* *
-* bfps is distributed in the hope that it will be useful, *
+* TurTLE is distributed in the hope that it will be useful, *
* but WITHOUT ANY WARRANTY; without even the implied warranty of *
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the *
* GNU General Public License for more details. *
* *
* You should have received a copy of the GNU General Public License *
-* along with bfps. If not, see <http://www.gnu.org/licenses/> *
+* along with TurTLE. If not, see <http://www.gnu.org/licenses/> *
* *
* Contact: Cristian.Lalescu@ds.mpg.de *
* *
@@ -42,7 +42,11 @@ class get_rfields: public NSVE_field_stats<rnumber>
public:
int checkpoints_per_file;
int niter_out;
+ int TrS2_on;
kspace<FFTW, SMOOTH> *kk;
+ field<rnumber, FFTW, ONE> *traceS2;
+ field<rnumber, FFTW, THREE> *vel;
+ field<rnumber, FFTW, THREExTHREE> *grad_vel;
get_rfields(
const MPI_Comm COMMUNICATOR,
diff --git a/cpp/full_code/get_velocity.cpp b/cpp/full_code/get_velocity.cpp
new file mode 100644
index 0000000000000000000000000000000000000000..a29cc660c72fd6e07cfcd41ad9f03482be7f1c1b
--- /dev/null
+++ b/cpp/full_code/get_velocity.cpp
@@ -0,0 +1,111 @@
+/******************************************************************************
+* *
+* Copyright 2019 Max Planck Institute for Dynamics and Self-Organization *
+* *
+* This file is part of TurTLE. *
+* *
+* TurTLE is free software: you can redistribute it and/or modify *
+* it under the terms of the GNU General Public License as published *
+* by the Free Software Foundation, either version 3 of the License, *
+* or (at your option) any later version. *
+* *
+* TurTLE is distributed in the hope that it will be useful, *
+* but WITHOUT ANY WARRANTY; without even the implied warranty of *
+* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the *
+* GNU General Public License for more details. *
+* *
+* You should have received a copy of the GNU General Public License *
+* along with TurTLE. If not, see <http://www.gnu.org/licenses/> *
+* *
+* Contact: Cristian.Lalescu@ds.mpg.de *
+* *
+******************************************************************************/
+
+
+
+#include <string>
+#include <cmath>
+#include "get_velocity.hpp"
+#include "scope_timer.hpp"
+#include "hdf5_tools.hpp"
+
+
+template <typename rnumber>
+int get_velocity<rnumber>::initialize(void)
+{
+ TIMEZONE("get_velocity::initialize");
+ this->NSVE_field_stats<rnumber>::initialize();
+ DEBUG_MSG("after NSVE_field_stats::initialize\n");
+
+ // allocate kspace
+ this->kk = new kspace<FFTW, SMOOTH>(
+ this->vorticity->clayout, this->dkx, this->dky, this->dkz);
+ // allocate field for velocity
+ this->vel = new field<rnumber, FFTW, THREE>(
+ this->nx, this->ny, this->nz,
+ this->comm,
+ this->vorticity->fftw_plan_rigor);
+ hid_t parameter_file = H5Fopen(
+ (this->simname + std::string(".h5")).c_str(),
+ H5F_ACC_RDONLY,
+ H5P_DEFAULT);
+ this->niter_out = hdf5_tools::read_value<int>(parameter_file, "/parameters/niter_out");
+ this->checkpoints_per_file = hdf5_tools::read_value<int>(parameter_file, "/parameters/checkpoints_per_file");
+ if (this->checkpoints_per_file == INT_MAX) // value returned if dataset does not exist
+ this->checkpoints_per_file = 1;
+ H5Fclose(parameter_file);
+ // the following ensures the file is free for rank 0 to open in read/write mode
+ MPI_Barrier(this->comm);
+ parameter_file = H5Fopen(
+ (this->simname + std::string("_post.h5")).c_str(),
+ H5F_ACC_RDONLY,
+ H5P_DEFAULT);
+ this->iteration_list = hdf5_tools::read_vector<int>(
+ parameter_file,
+ "/get_velocity/parameters/iteration_list");
+ H5Fclose(parameter_file);
+ // the following ensures the file is free for rank 0 to open in read/write mode
+ MPI_Barrier(this->comm);
+ return EXIT_SUCCESS;
+}
+
+template <typename rnumber>
+int get_velocity<rnumber>::work_on_current_iteration(void)
+{
+ TIMEZONE("get_velocity::work_on_current_iteration");
+ this->read_current_cvorticity();
+
+ invert_curl(
+ this->kk,
+ this->vorticity,
+ this->vel);
+
+ std::string fname = (
+ this->simname +
+ std::string("_checkpoint_") +
+ std::to_string(this->iteration / (this->niter_out*this->checkpoints_per_file)) +
+ std::string(".h5"));
+
+ /// output velocity field
+ this->vel->io(
+ fname,
+ "velocity",
+ this->iteration,
+ false);
+
+ return EXIT_SUCCESS;
+}
+
+template <typename rnumber>
+int get_velocity<rnumber>::finalize(void)
+{
+ TIMEZONE("get_velocity::finalize");
+ delete this->kk;
+ delete this->vel;
+ this->NSVE_field_stats<rnumber>::finalize();
+ return EXIT_SUCCESS;
+}
+
+template class get_velocity<float>;
+template class get_velocity<double>;
+
diff --git a/cpp/full_code/get_velocity.hpp b/cpp/full_code/get_velocity.hpp
new file mode 100644
index 0000000000000000000000000000000000000000..dababf2c16c290b2e6aa60d76f151026bc49c167
--- /dev/null
+++ b/cpp/full_code/get_velocity.hpp
@@ -0,0 +1,62 @@
+/**********************************************************************
+* *
+* Copyright 2021 Max Planck Institute *
+* for Dynamics and Self-Organization *
+* *
+* This file is part of TurTLE. *
+* *
+* TurTLE is free software: you can redistribute it and/or modify *
+* it under the terms of the GNU General Public License as published *
+* by the Free Software Foundation, either version 3 of the License, *
+* or (at your option) any later version. *
+* *
+* TurTLE is distributed in the hope that it will be useful, *
+* but WITHOUT ANY WARRANTY; without even the implied warranty of *
+* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the *
+* GNU General Public License for more details. *
+* *
+* You should have received a copy of the GNU General Public License *
+* along with TurTLE. If not, see <http://www.gnu.org/licenses/> *
+* *
+* Contact: Cristian.Lalescu@mpcdf.mpg.de *
+* *
+**********************************************************************/
+
+
+
+#ifndef GET_VELOCITY_HPP
+#define GET_VELOCITY_HPP
+
+#include <cstdlib>
+#include <sys/types.h>
+#include <sys/stat.h>
+#include <vector>
+#include "base.hpp"
+#include "field.hpp"
+#include "field_binary_IO.hpp"
+#include "full_code/NSVE_field_stats.hpp"
+
+template <typename rnumber>
+class get_velocity: public NSVE_field_stats<rnumber>
+{
+ public:
+ int checkpoints_per_file;
+ int niter_out;
+ kspace<FFTW, SMOOTH> *kk;
+ field<rnumber, FFTW, THREE> *vel;
+
+ get_velocity(
+ const MPI_Comm COMMUNICATOR,
+ const std::string &simulation_name):
+ NSVE_field_stats<rnumber>(
+ COMMUNICATOR,
+ simulation_name){}
+ virtual ~get_velocity(){}
+
+ int initialize(void);
+ int work_on_current_iteration(void);
+ int finalize(void);
+};
+
+#endif//GET_VELOCITY_HPP
+
diff --git a/bfps/cpp/full_code/joint_acc_vel_stats.cpp b/cpp/full_code/joint_acc_vel_stats.cpp
similarity index 67%
rename from bfps/cpp/full_code/joint_acc_vel_stats.cpp
rename to cpp/full_code/joint_acc_vel_stats.cpp
index e4f4d5d40772292f44c7e776dcd4d1b82c4ce222..8fe55d2025e1cf7c0c1b91144db2aab56ba6b4bc 100644
--- a/bfps/cpp/full_code/joint_acc_vel_stats.cpp
+++ b/cpp/full_code/joint_acc_vel_stats.cpp
@@ -1,12 +1,39 @@
+/******************************************************************************
+* *
+* Copyright 2019 Max Planck Institute for Dynamics and Self-Organization *
+* *
+* This file is part of TurTLE. *
+* *
+* TurTLE is free software: you can redistribute it and/or modify *
+* it under the terms of the GNU General Public License as published *
+* by the Free Software Foundation, either version 3 of the License, *
+* or (at your option) any later version. *
+* *
+* TurTLE is distributed in the hope that it will be useful, *
+* but WITHOUT ANY WARRANTY; without even the implied warranty of *
+* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the *
+* GNU General Public License for more details. *
+* *
+* You should have received a copy of the GNU General Public License *
+* along with TurTLE. If not, see <http://www.gnu.org/licenses/> *
+* *
+* Contact: Cristian.Lalescu@ds.mpg.de *
+* *
+******************************************************************************/
+
+
+
#include <string>
#include <cmath>
#include "joint_acc_vel_stats.hpp"
#include "scope_timer.hpp"
+#include "hdf5_tools.hpp"
template <typename rnumber>
int joint_acc_vel_stats<rnumber>::initialize(void)
{
+ TIMEZONE("joint_acc_vel_stats::initialize");
this->NSVE_field_stats<rnumber>::initialize();
this->kk = new kspace<FFTW, SMOOTH>(
this->vorticity->clayout, this->dkx, this->dky, this->dkz);
@@ -19,6 +46,7 @@ int joint_acc_vel_stats<rnumber>::initialize(void)
this->dky,
this->dkz,
this->vorticity->fftw_plan_rigor);
+ this->template copy_parameters_into<rnumber, FFTW>(this->ve);
hid_t parameter_file = H5Fopen(
(this->simname + std::string(".h5")).c_str(),
H5F_ACC_RDONLY,
@@ -35,6 +63,8 @@ int joint_acc_vel_stats<rnumber>::initialize(void)
else
this->checkpoints_per_file = 1;
H5Fclose(parameter_file);
+ // the following ensures the file is free for rank 0 to open in read/write mode
+ MPI_Barrier(this->comm);
parameter_file = H5Fopen(
(this->simname + std::string("_post.h5")).c_str(),
H5F_ACC_RDONLY,
@@ -49,11 +79,14 @@ int joint_acc_vel_stats<rnumber>::initialize(void)
H5Dread(dset, H5T_NATIVE_DOUBLE, H5S_ALL, H5S_ALL, H5P_DEFAULT, &this->max_velocity_estimate);
H5Dclose(dset);
H5Fclose(parameter_file);
+ // the following ensures the file is free for rank 0 to open in read/write mode
+ MPI_Barrier(this->comm);
if (this->myrank == 0)
{
// set caching parameters
hid_t fapl = H5Pcreate(H5P_FILE_ACCESS);
herr_t cache_err = H5Pset_cache(fapl, 0, 521, 134217728, 1.0);
+ variable_used_only_in_assert(cache_err);
DEBUG_MSG("when setting stat_file cache I got %d\n", cache_err);
this->stat_file = H5Fopen(
(this->simname + "_post.h5").c_str(),
@@ -85,7 +118,7 @@ int joint_acc_vel_stats<rnumber>::initialize(void)
template <typename rnumber>
int joint_acc_vel_stats<rnumber>::work_on_current_iteration(void)
{
- DEBUG_MSG("entered joint_acc_vel_stats::work_on_current_iteration\n");
+ TIMEZONE("joint_acc_vel_stats::work_on_current_iteration");
/// read current vorticity, place it in this->ve->cvorticity
this->read_current_cvorticity();
*this->ve->cvorticity = this->vorticity->get_cdata();
@@ -109,7 +142,7 @@ int joint_acc_vel_stats<rnumber>::work_on_current_iteration(void)
vel = new field<rnumber, FFTW, THREE>(
this->nx, this->ny, this->nz,
this->comm,
- DEFAULT_FFTW_FLAG);
+ this->vorticity->fftw_plan_rigor);
invert_curl(kk, this->ve->cvorticity, vel);
vel->ift();
@@ -128,17 +161,6 @@ int joint_acc_vel_stats<rnumber>::work_on_current_iteration(void)
max_acc_estimate[3] = max_acceleration_estimate;
max_vel_estimate[3] = max_velocity_estimate;
- acc->compute_rspace_stats(
- stat_group,
- "acceleration",
- this->iteration / this->niter_out,
- max_acc_estimate);
- vel->compute_rspace_stats(
- stat_group,
- "velocity",
- this->iteration / this->niter_out,
- max_vel_estimate);
-
/// compute joint PDF
joint_rspace_PDF(
acc, vel,
@@ -148,6 +170,24 @@ int joint_acc_vel_stats<rnumber>::work_on_current_iteration(void)
max_acc_estimate,
max_vel_estimate);
+ /// compute real space stats and spectra
+ acc->compute_stats(
+ kk,
+ stat_group,
+ "acceleration",
+ this->iteration / this->niter_out,
+ max_acceleration_estimate / sqrt(3));
+ vel->compute_stats(
+ kk,
+ stat_group,
+ "velocity",
+ this->iteration / this->niter_out,
+ max_velocity_estimate / sqrt(3));
+
+ /// close stat group
+ if (this->myrank == 0)
+ H5Gclose(stat_group);
+
delete vel;
return EXIT_SUCCESS;
@@ -156,6 +196,7 @@ int joint_acc_vel_stats<rnumber>::work_on_current_iteration(void)
template <typename rnumber>
int joint_acc_vel_stats<rnumber>::finalize(void)
{
+ DEBUG_MSG("entered joint_acc_vel_stats::finalize\n");
delete this->ve;
delete this->kk;
if (this->myrank == 0)
diff --git a/bfps/cpp/full_code/joint_acc_vel_stats.hpp b/cpp/full_code/joint_acc_vel_stats.hpp
similarity index 89%
rename from bfps/cpp/full_code/joint_acc_vel_stats.hpp
rename to cpp/full_code/joint_acc_vel_stats.hpp
index 721698b7620dbe0ae282da28bd38835b475d50ca..814ddfd1b0b0c24af6e41c2397c862c35d4c17b1 100644
--- a/bfps/cpp/full_code/joint_acc_vel_stats.hpp
+++ b/cpp/full_code/joint_acc_vel_stats.hpp
@@ -3,20 +3,20 @@
* Copyright 2017 Max Planck Institute *
* for Dynamics and Self-Organization *
* *
-* This file is part of bfps. *
+* This file is part of TurTLE. *
* *
-* bfps is free software: you can redistribute it and/or modify *
+* TurTLE is free software: you can redistribute it and/or modify *
* it under the terms of the GNU General Public License as published *
* by the Free Software Foundation, either version 3 of the License, *
* or (at your option) any later version. *
* *
-* bfps is distributed in the hope that it will be useful, *
+* TurTLE is distributed in the hope that it will be useful, *
* but WITHOUT ANY WARRANTY; without even the implied warranty of *
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the *
* GNU General Public License for more details. *
* *
* You should have received a copy of the GNU General Public License *
-* along with bfps. If not, see <http://www.gnu.org/licenses/> *
+* along with TurTLE. If not, see <http://www.gnu.org/licenses/> *
* *
* Contact: Cristian.Lalescu@ds.mpg.de *
* *
diff --git a/cpp/full_code/kraichnan_field.cpp b/cpp/full_code/kraichnan_field.cpp
new file mode 100644
index 0000000000000000000000000000000000000000..bb63f5def0bef1b470801aff19bbd2d916e16235
--- /dev/null
+++ b/cpp/full_code/kraichnan_field.cpp
@@ -0,0 +1,388 @@
+/******************************************************************************
+* *
+* Copyright 2019 Max Planck Institute for Dynamics and Self-Organization *
+* *
+* This file is part of TurTLE. *
+* *
+* TurTLE is free software: you can redistribute it and/or modify *
+* it under the terms of the GNU General Public License as published *
+* by the Free Software Foundation, either version 3 of the License, *
+* or (at your option) any later version. *
+* *
+* TurTLE is distributed in the hope that it will be useful, *
+* but WITHOUT ANY WARRANTY; without even the implied warranty of *
+* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the *
+* GNU General Public License for more details. *
+* *
+* You should have received a copy of the GNU General Public License *
+* along with TurTLE. If not, see <http://www.gnu.org/licenses/> *
+* *
+* Contact: Cristian.Lalescu@ds.mpg.de *
+* *
+******************************************************************************/
+
+
+
+#include <string>
+#include <cmath>
+#include "kraichnan_field.hpp"
+#include "scope_timer.hpp"
+#include "fftw_tools.hpp"
+
+
+template <typename rnumber>
+int kraichnan_field<rnumber>::initialize(void)
+{
+ TIMEZONE("kraichnan_file::initialize");
+ this->read_iteration();
+ this->read_parameters();
+ if (this->myrank == 0)
+ {
+ // set caching parameters
+ hid_t fapl = H5Pcreate(H5P_FILE_ACCESS);
+ herr_t cache_err = H5Pset_cache(fapl, 0, 521, 134217728, 1.0);
+ variable_used_only_in_assert(cache_err);
+ DEBUG_MSG("when setting stat_file cache I got %d\n", cache_err);
+ this->stat_file = H5Fopen(
+ (this->simname + ".h5").c_str(),
+ H5F_ACC_RDWR,
+ fapl);
+ }
+ int data_file_problem;
+ if (this->myrank == 0)
+ data_file_problem = this->grow_file_datasets();
+ MPI_Bcast(&data_file_problem, 1, MPI_INT, 0, this->comm);
+ if (data_file_problem > 0)
+ {
+ std::cerr <<
+ data_file_problem <<
+ " problems growing file datasets.\ntrying to exit now." <<
+ std::endl;
+ return EXIT_FAILURE;
+ }
+
+ this->velocity = new field<rnumber, FFTW, THREE>(
+ nx, ny, nz,
+ this->comm,
+ fftw_planner_string_to_flag[this->fftw_plan_rigor]);
+ this->velocity->real_space_representation = true;
+
+ // the velocity layout should be the same as the vorticity one
+ this->kk = new kspace<FFTW, SMOOTH>(
+ this->velocity->clayout, this->dkx, this->dky, this->dkz);
+
+ // initialize particles
+ this->ps = particles_system_builder(
+ this->velocity, // (field object)
+ this->kk, // (kspace object, contains dkx, dky, dkz)
+ tracers0_integration_steps, // to check coherency between parameters and hdf input file (nb rhs)
+ (long long int)nparticles, // to check coherency between parameters and hdf input file
+ this->get_current_fname(), // particles input filename
+ std::string("/tracers0/state/") + std::to_string(this->iteration), // dataset name for initial input
+ std::string("/tracers0/rhs/") + std::to_string(this->iteration), // dataset name for initial input
+ tracers0_neighbours, // parameter (interpolation no neighbours)
+ tracers0_smoothness, // parameter
+ this->comm,
+ this->iteration+1);
+ // initialize output objects
+ this->particles_output_writer_mpi = new particles_output_hdf5<
+ long long int, double, 3>(
+ MPI_COMM_WORLD,
+ "tracers0",
+ nparticles,
+ tracers0_integration_steps);
+ this->particles_output_writer_mpi->setParticleFileLayout(this->ps->getParticleFileLayout());
+ this->particles_sample_writer_mpi = new particles_output_sampling_hdf5<
+ long long int, double, double, 3>(
+ MPI_COMM_WORLD,
+ this->ps->getGlobalNbParticles(),
+ (this->simname + "_particles.h5"),
+ "tracers0",
+ "position/0");
+ this->particles_sample_writer_mpi->setParticleFileLayout(this->ps->getParticleFileLayout());
+
+ this->generate_random_velocity();
+ // is this the first iteration?
+ if ((this->iteration == 0) and (this->output_velocity == 1))
+ {
+ // if yes, generate random field and save it
+ this->velocity->io(
+ this->get_current_fname(),
+ "velocity",
+ this->iteration,
+ false);
+ }
+
+ return EXIT_SUCCESS;
+}
+
+template <typename rnumber>
+int kraichnan_field<rnumber>::generate_random_velocity(void)
+{
+ TIMEZONE("kraichnan_file::make_random_field");
+ // generate Gaussian random field
+ make_gaussian_random_field(
+ this->kk,
+ this->velocity,
+ this->field_random_seed + 10*this->iteration,
+ this->spectrum_dissipation,
+ this->spectrum_Lint,
+ this->spectrum_etaK,
+ this->spectrum_large_scale_const,
+ this->spectrum_small_scale_const,
+ 3./2.); // incompressibility projection factor
+ // not an ideal choice because resulting field sequence depends on MPI/OpenMP configuration. See note below
+ // this->velocity is now in Fourier space
+ // project divfree, requires field in Fourier space
+ // Note on the choice of random seed:
+ // If in the future the simulation will continue with a smaller number of total threads (number of processes times number of threads per process),
+ // then during that run some of the threads will be seeded with a seed that has already been used for a previous iteration.
+ // So some sequences of Fourier modes will be identical to sequences of Fourier modes that occured in the past.
+ // Also see implementation of "make_gaussian_random_field".
+ // One work-around would be to multiply "this->iteration" with 10 or so ---
+ // it is unlikely the simulation will be continued with less than 0.1 of the initial total number of threads.
+ DEBUG_MSG("L2Norm before: %g\n",
+ this->velocity->L2norm(this->kk));
+ this->kk->template project_divfree<rnumber>(this->velocity->get_cdata());
+ DEBUG_MSG("L2Norm after: %g\n",
+ this->velocity->L2norm(this->kk));
+ // transform velocity to real space representation
+ this->velocity->ift();
+ return EXIT_SUCCESS;
+}
+
+template <typename rnumber>
+int kraichnan_field<rnumber>::step(void)
+{
+ TIMEZONE("kraichnan_file::step");
+ this->ps->completeLoop(sqrt(this->dt));
+ this->generate_random_velocity();
+ this->iteration++;
+ return EXIT_SUCCESS;
+}
+
+template <typename rnumber>
+int kraichnan_field<rnumber>::write_checkpoint(void)
+{
+ TIMEZONE("kraichnan_file::write_checkpoint");
+ this->update_checkpoint();
+ // output field
+ if (this->output_velocity == 1)
+ {
+ assert(this->velocity->real_space_representation);
+ std::string fname = this->get_current_fname();
+ this->velocity->io(
+ fname,
+ "velocity",
+ this->iteration,
+ false);
+ }
+ // output particles
+ this->particles_output_writer_mpi->open_file(this->get_current_fname());
+ this->particles_output_writer_mpi->template save<3>(
+ this->ps->getParticlesState(),
+ this->ps->getParticlesRhs(),
+ this->ps->getParticlesIndexes(),
+ this->ps->getLocalNbParticles(),
+ this->iteration);
+ this->particles_output_writer_mpi->close_file();
+ this->write_iteration();
+ return EXIT_SUCCESS;
+}
+
+template <typename rnumber>
+int kraichnan_field<rnumber>::finalize(void)
+{
+ TIMEZONE("kraichnan_field::finalize");
+ if (this->myrank == 0)
+ H5Fclose(this->stat_file);
+ // good practice rule number n+1: always delete in inverse order of allocation
+ delete this->ps.release();
+ delete this->particles_output_writer_mpi;
+ delete this->particles_sample_writer_mpi;
+ delete this->kk;
+ delete this->velocity;
+ return EXIT_SUCCESS;
+}
+
+/** \brief Compute statistics.
+ *
+ */
+
+template <typename rnumber>
+int kraichnan_field<rnumber>::do_stats()
+{
+ TIMEZONE("kraichnan_field::do_stats");
+ /// compute field statistics
+ if (this->iteration % this->niter_stat == 0)
+ {
+ hid_t stat_group;
+ if (this->myrank == 0)
+ stat_group = H5Gopen(
+ this->stat_file,
+ "statistics",
+ H5P_DEFAULT);
+ else
+ stat_group = 0;
+ field<rnumber, FFTW, THREE> *tvelocity = new field<rnumber, FFTW, THREE>(
+ this->nx, this->ny, this->nz,
+ this->comm,
+ fftw_planner_string_to_flag[this->fftw_plan_rigor]);
+ *tvelocity = *this->velocity;
+ tvelocity->compute_stats(
+ this->kk,
+ stat_group,
+ "velocity",
+ this->iteration / this->niter_stat,
+ this->max_velocity_estimate/sqrt(3));
+ if (this->myrank == 0)
+ H5Gclose(stat_group);
+ delete tvelocity;
+ }
+ /// either one of two conditions suffices to compute statistics:
+ /// 1) current iteration is a multiple of niter_part
+ /// 2) we are within niter_part_fine_duration/2 of a multiple of niter_part_fine_period
+ if (!(this->iteration % this->niter_part == 0 ||
+ ((this->iteration + this->niter_part_fine_duration/2) % this->niter_part_fine_period <=
+ this->niter_part_fine_duration)))
+ return EXIT_SUCCESS;
+
+ // allocate temporary data array
+ std::unique_ptr<double[]> pdata(new double[3*this->ps->getLocalNbParticles()]);
+
+ /// copy position data
+
+ /// sample position
+ std::copy(this->ps->getParticlesState(),
+ this->ps->getParticlesState()+3*this->ps->getLocalNbParticles(),
+ pdata.get());
+ this->particles_sample_writer_mpi->template save_dataset<3>(
+ "tracers0",
+ "position",
+ this->ps->getParticlesState(),
+ &pdata,
+ this->ps->getParticlesIndexes(),
+ this->ps->getLocalNbParticles(),
+ this->ps->get_step_idx()-1);
+
+ /// sample velocity
+ std::fill_n(pdata.get(), 3*this->ps->getLocalNbParticles(), 0);
+ this->ps->sample_compute_field(*this->velocity, pdata.get());
+ this->particles_sample_writer_mpi->template save_dataset<3>(
+ "tracers0",
+ "velocity",
+ this->ps->getParticlesState(),
+ &pdata,
+ this->ps->getParticlesIndexes(),
+ this->ps->getLocalNbParticles(),
+ this->ps->get_step_idx()-1);
+
+ // deallocate temporary data array
+ delete[] pdata.release();
+
+ return EXIT_SUCCESS;
+}
+
+template <typename rnumber>
+int kraichnan_field<rnumber>::read_parameters(void)
+{
+ TIMEZONE("kraichnan_field::read_parameters");
+ this->direct_numerical_simulation::read_parameters();
+ hid_t parameter_file = H5Fopen((this->simname + ".h5").c_str(), H5F_ACC_RDONLY, H5P_DEFAULT);
+ this->fftw_plan_rigor = hdf5_tools::read_string(parameter_file, "parameters/fftw_plan_rigor");
+ this->dt = hdf5_tools::read_value<double>(parameter_file, "parameters/dt");
+ this->niter_part = hdf5_tools::read_value<int>(parameter_file, "parameters/niter_part");
+ this->niter_part_fine_period = hdf5_tools::read_value<int>(parameter_file, "parameters/niter_part_fine_period");
+ this->niter_part_fine_duration = hdf5_tools::read_value<int>(parameter_file, "parameters/niter_part_fine_duration");
+ this->nparticles = hdf5_tools::read_value<long long int>(parameter_file, "parameters/nparticles");
+ this->tracers0_integration_steps = hdf5_tools::read_value<int>(parameter_file, "parameters/tracers0_integration_steps");
+ this->tracers0_neighbours = hdf5_tools::read_value<int>(parameter_file, "parameters/tracers0_neighbours");
+ this->tracers0_smoothness = hdf5_tools::read_value<int>(parameter_file, "parameters/tracers0_smoothness");
+ this->spectrum_dissipation = hdf5_tools::read_value<double>(parameter_file, "parameters/spectrum_dissipation");
+ this->spectrum_Lint = hdf5_tools::read_value<double>(parameter_file, "parameters/spectrum_Lint");
+ this->spectrum_etaK = hdf5_tools::read_value<double>(parameter_file, "parameters/spectrum_etaK");
+ this->spectrum_large_scale_const = hdf5_tools::read_value<double>(parameter_file, "parameters/spectrum_large_scale_const");
+ this->spectrum_small_scale_const = hdf5_tools::read_value<double>(parameter_file, "parameters/spectrum_small_scale_const");
+ this->field_random_seed = hdf5_tools::read_value<int>(parameter_file, "parameters/field_random_seed");
+ this->output_velocity = hdf5_tools::read_value<int>(parameter_file, "parameters/output_velocity");
+ this->max_velocity_estimate = hdf5_tools::read_value<double>(parameter_file, "parameters/max_velocity_estimate");
+ H5Fclose(parameter_file);
+ MPI_Barrier(this->comm);
+ return EXIT_SUCCESS;
+}
+
+template <class rnumber>
+void kraichnan_field<rnumber>::update_checkpoint()
+{
+ TIMEZONE("kraichnan_field::update_checkpoint");
+ DEBUG_MSG("entered kraichan_field::update_checkpoint\n");
+ std::string fname = this->get_current_fname();
+ if (this->kk->layout->myrank == 0)
+ {
+ bool file_exists = false;
+ {
+ struct stat file_buffer;
+ file_exists = (stat(fname.c_str(), &file_buffer) == 0);
+ }
+ if (file_exists)
+ {
+ // check how many fields there are in the checkpoint file
+ // increment checkpoint if needed
+ hsize_t fields_stored;
+ hid_t fid, group_id;
+ fid = H5Fopen(fname.c_str(), H5F_ACC_RDONLY, H5P_DEFAULT);
+ group_id = H5Gopen(fid, "tracers0/state", H5P_DEFAULT);
+ bool dset_exists;
+ if (group_id > 0)
+ {
+ H5Gget_num_objs(
+ group_id,
+ &fields_stored);
+ dset_exists = H5Lexists(
+ group_id,
+ std::to_string(this->iteration).c_str(),
+ H5P_DEFAULT);
+ }
+ else
+ {
+ fields_stored = 0;
+ dset_exists = false;
+ }
+ H5Gclose(group_id);
+ H5Fclose(fid);
+ if ((int(fields_stored) >= this->checkpoints_per_file) &&
+ !dset_exists)
+ this->checkpoint++;
+ }
+ else
+ {
+ // create file, create fields_stored dset
+ hid_t fid = H5Fcreate(
+ fname.c_str(),
+ H5F_ACC_EXCL,
+ H5P_DEFAULT,
+ H5P_DEFAULT);
+ hid_t gg = H5Gcreate(
+ fid,
+ "tracers0",
+ H5P_DEFAULT,
+ H5P_DEFAULT,
+ H5P_DEFAULT);
+ hid_t ggg = H5Gcreate(
+ gg,
+ "state",
+ H5P_DEFAULT,
+ H5P_DEFAULT,
+ H5P_DEFAULT);
+ H5Gclose(ggg);
+ H5Gclose(gg);
+ H5Fclose(fid);
+ }
+ }
+ MPI_Bcast(&this->checkpoint, 1, MPI_INT, 0, this->kk->layout->comm);
+ DEBUG_MSG("exiting kraichan_field::update_checkpoint\n");
+}
+
+template class kraichnan_field<float>;
+template class kraichnan_field<double>;
+
diff --git a/cpp/full_code/kraichnan_field.hpp b/cpp/full_code/kraichnan_field.hpp
new file mode 100644
index 0000000000000000000000000000000000000000..020cd0cdc7b2c5b8e35114bc7d78106e9a8c1fb2
--- /dev/null
+++ b/cpp/full_code/kraichnan_field.hpp
@@ -0,0 +1,99 @@
+/**********************************************************************
+* *
+* Copyright 2017 Max Planck Institute *
+* for Dynamics and Self-Organization *
+* *
+* This file is part of TurTLE. *
+* *
+* TurTLE is free software: you can redistribute it and/or modify *
+* it under the terms of the GNU General Public License as published *
+* by the Free Software Foundation, either version 3 of the License, *
+* or (at your option) any later version. *
+* *
+* TurTLE is distributed in the hope that it will be useful, *
+* but WITHOUT ANY WARRANTY; without even the implied warranty of *
+* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the *
+* GNU General Public License for more details. *
+* *
+* You should have received a copy of the GNU General Public License *
+* along with TurTLE. If not, see <http://www.gnu.org/licenses/> *
+* *
+* Contact: Cristian.Lalescu@ds.mpg.de *
+* *
+**********************************************************************/
+
+
+
+#ifndef KRAICHNAN_FIELD_HPP
+#define KRAICHNAN_FIELD_HPP
+
+
+
+#include <cstdlib>
+#include "base.hpp"
+#include "kspace.hpp"
+#include "field.hpp"
+#include "full_code/direct_numerical_simulation.hpp"
+#include "particles/particles_system_builder.hpp"
+#include "particles/particles_output_hdf5.hpp"
+#include "particles/particles_sampling.hpp"
+
+template <typename rnumber>
+class kraichnan_field: public direct_numerical_simulation
+{
+ public:
+
+ /* parameters that are read in read_parameters */
+ double dt;
+ std::string fftw_plan_rigor;
+
+ /* particle parameters */
+ int niter_part;
+ int niter_part_fine_period;
+ int niter_part_fine_duration;
+ long long int nparticles;
+ int tracers0_integration_steps;
+ int tracers0_neighbours;
+ int tracers0_smoothness;
+ int output_velocity;
+
+ /* other stuff */
+ kspace<FFTW, SMOOTH> *kk;
+ field<rnumber, FFTW, THREE> *velocity;
+ double spectrum_dissipation;
+ double spectrum_Lint;
+ double spectrum_etaK;
+ double spectrum_large_scale_const;
+ double spectrum_small_scale_const;
+ double max_velocity_estimate;
+ int field_random_seed;
+
+ /* other stuff */
+ std::unique_ptr<abstract_particles_system<long long int, double>> ps;
+
+ particles_output_hdf5<long long int, double,3> *particles_output_writer_mpi;
+ particles_output_sampling_hdf5<long long int, double, double, 3> *particles_sample_writer_mpi;
+
+
+ kraichnan_field(
+ const MPI_Comm COMMUNICATOR,
+ const std::string &simulation_name):
+ direct_numerical_simulation(
+ COMMUNICATOR,
+ simulation_name){}
+ ~kraichnan_field(){}
+
+ int initialize(void);
+ int step(void);
+ int finalize(void);
+
+ virtual int read_parameters(void);
+ int write_checkpoint(void);
+ int do_stats(void);
+
+ int generate_random_velocity(void);
+ void update_checkpoint(void);
+};
+
+#endif//KRAICHNAN_FIELD_HPP
+
diff --git a/bfps/cpp/full_code/main_code.hpp b/cpp/full_code/main_code.hpp
similarity index 76%
rename from bfps/cpp/full_code/main_code.hpp
rename to cpp/full_code/main_code.hpp
index cae34b69b18b4f2550f5b6e5f28b48a659ea75f1..a9a26a57dbf85a36454de01314366cc356ffc097 100644
--- a/bfps/cpp/full_code/main_code.hpp
+++ b/cpp/full_code/main_code.hpp
@@ -3,20 +3,20 @@
* Copyright 2017 Max Planck Institute *
* for Dynamics and Self-Organization *
* *
-* This file is part of bfps. *
+* This file is part of TurTLE. *
* *
-* bfps is free software: you can redistribute it and/or modify *
+* TurTLE is free software: you can redistribute it and/or modify *
* it under the terms of the GNU General Public License as published *
* by the Free Software Foundation, either version 3 of the License, *
* or (at your option) any later version. *
* *
-* bfps is distributed in the hope that it will be useful, *
+* TurTLE is distributed in the hope that it will be useful, *
* but WITHOUT ANY WARRANTY; without even the implied warranty of *
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the *
* GNU General Public License for more details. *
* *
* You should have received a copy of the GNU General Public License *
-* along with bfps. If not, see <http://www.gnu.org/licenses/> *
+* along with TurTLE. If not, see <http://www.gnu.org/licenses/> *
* *
* Contact: Cristian.Lalescu@ds.mpg.de *
* *
@@ -38,6 +38,31 @@
int myrank, nprocs;
+#ifdef PINCHECK_FOUND
+#include <pincheck.hpp>
+
+void print_pinning_info(void)
+{
+ // obtain string with pinning information on rank 0,
+ // ranks >0 get an empty string
+ const std::string pinning_info = pincheck::pincheck();
+ if (myrank == 0)
+ {
+ std::cerr << "### pinning info begin" << std::endl;
+ std::cerr << pinning_info;
+ std::cerr << "### pinning info end" << std::endl;
+ std::cout << "### pinning info begin" << std::endl;
+ std::cout << pinning_info;
+ std::cout << "### pinning info end" << std::endl;
+ }
+}
+
+#else
+
+#define print_pinning_info(...)
+
+#endif
+
template <class DNS>
int main_code(
int argc,
@@ -66,17 +91,26 @@ int main_code(
/* initialize MPI environment */
#ifdef NO_FFTWOMP
MPI_Init(&argc, &argv);
+ // turn off MPI profiling for initialization
+ MPI_Pcontrol(0);
MPI_Comm_rank(MPI_COMM_WORLD, &myrank);
MPI_Comm_size(MPI_COMM_WORLD, &nprocs);
+
+ print_pinning_info();
+
fftw_mpi_init();
fftwf_mpi_init();
DEBUG_MSG("There are %d processes\n", nprocs);
#else
int mpiprovided;
MPI_Init_thread(&argc, &argv, MPI_THREAD_FUNNELED, &mpiprovided);
+ MPI_Pcontrol(0);
assert(mpiprovided >= MPI_THREAD_FUNNELED);
MPI_Comm_rank(MPI_COMM_WORLD, &myrank);
MPI_Comm_size(MPI_COMM_WORLD, &nprocs);
+
+ print_pinning_info();
+
const int nThreads = omp_get_max_threads();
DEBUG_MSG("Number of threads for the FFTW = %d\n",
nThreads);
@@ -99,9 +133,17 @@ int main_code(
/* import fftw wisdom */
if (myrank == 0)
+ {
fftwf_import_wisdom_from_filename(
+ (simname + std::string("_fftwf_wisdom.txt")).c_str());
+ fftw_import_wisdom_from_filename(
(simname + std::string("_fftw_wisdom.txt")).c_str());
+ }
fftwf_mpi_broadcast_wisdom(MPI_COMM_WORLD);
+ fftw_mpi_broadcast_wisdom(MPI_COMM_WORLD);
+
+ fftwf_set_timelimit(300);
+ fftw_set_timelimit(300);
@@ -124,7 +166,13 @@ int main_code(
int return_value;
return_value = dns->initialize();
if (return_value == EXIT_SUCCESS)
+ {
+ // turn on MPI profiling for main loop
+ MPI_Pcontrol(1);
return_value = dns->main_loop();
+ // turn off MPI profiling
+ MPI_Pcontrol(0);
+ }
else
DEBUG_MSG("problem calling dns->initialize(), return value is %d",
return_value);
@@ -143,10 +191,15 @@ int main_code(
/* export fftw wisdom */
fftwf_mpi_gather_wisdom(MPI_COMM_WORLD);
+ fftw_mpi_gather_wisdom(MPI_COMM_WORLD);
MPI_Barrier(MPI_COMM_WORLD);
if (myrank == 0)
+ {
fftwf_export_wisdom_to_filename(
+ (simname + std::string("_fftwf_wisdom.txt")).c_str());
+ fftw_export_wisdom_to_filename(
(simname + std::string("_fftw_wisdom.txt")).c_str());
+ }
@@ -161,6 +214,7 @@ int main_code(
#endif
#ifdef USE_TIMINGOUTPUT
global_timer_manager.show(MPI_COMM_WORLD);
+ global_timer_manager.showMpi(MPI_COMM_WORLD);
global_timer_manager.showHtml(MPI_COMM_WORLD);
#endif
diff --git a/cpp/full_code/native_binary_to_hdf5.cpp b/cpp/full_code/native_binary_to_hdf5.cpp
new file mode 100644
index 0000000000000000000000000000000000000000..ac0973d61144e0e7bdd5bb08533ece0779d7d757
--- /dev/null
+++ b/cpp/full_code/native_binary_to_hdf5.cpp
@@ -0,0 +1,101 @@
+/******************************************************************************
+* *
+* Copyright 2019 Max Planck Institute for Dynamics and Self-Organization *
+* *
+* This file is part of TurTLE. *
+* *
+* TurTLE is free software: you can redistribute it and/or modify *
+* it under the terms of the GNU General Public License as published *
+* by the Free Software Foundation, either version 3 of the License, *
+* or (at your option) any later version. *
+* *
+* TurTLE is distributed in the hope that it will be useful, *
+* but WITHOUT ANY WARRANTY; without even the implied warranty of *
+* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the *
+* GNU General Public License for more details. *
+* *
+* You should have received a copy of the GNU General Public License *
+* along with TurTLE. If not, see <http://www.gnu.org/licenses/> *
+* *
+* Contact: Cristian.Lalescu@ds.mpg.de *
+* *
+******************************************************************************/
+
+
+
+#include <string>
+#include <cmath>
+#include "native_binary_to_hdf5.hpp"
+#include "scope_timer.hpp"
+#include "hdf5_tools.hpp"
+
+
+template <typename rnumber>
+int native_binary_to_hdf5<rnumber>::initialize(void)
+{
+ TIMEZONE("native_binary_to_hdf5::initialize");
+ this->read_parameters();
+ this->vec_field = new field<rnumber, FFTW, THREE>(
+ nx, ny, nz,
+ this->comm,
+ FFTW_ESTIMATE);
+ this->vec_field->real_space_representation = false;
+ this->bin_IO = new field_binary_IO<rnumber, COMPLEX, THREE>(
+ this->vec_field->clayout->sizes,
+ this->vec_field->clayout->subsizes,
+ this->vec_field->clayout->starts,
+ this->vec_field->clayout->comm);
+ return EXIT_SUCCESS;
+}
+
+template <typename rnumber>
+int native_binary_to_hdf5<rnumber>::work_on_current_iteration(void)
+{
+ TIMEZONE("native_binary_to_hdf5::work_on_current_iteration");
+ char itername[16];
+ sprintf(itername, "i%.5x", this->iteration);
+ std::string native_binary_fname = (
+ this->simname +
+ std::string("_cvorticity_") +
+ std::string(itername));
+ this->bin_IO->read(
+ native_binary_fname,
+ this->vec_field->get_cdata());
+ this->vec_field->io(
+ (native_binary_fname +
+ std::string(".h5")),
+ "vorticity",
+ this->iteration,
+ false);
+ return EXIT_SUCCESS;
+}
+
+template <typename rnumber>
+int native_binary_to_hdf5<rnumber>::finalize(void)
+{
+ TIMEZONE("native_binary_to_hdf5::finalize");
+ delete this->bin_IO;
+ delete this->vec_field;
+ return EXIT_SUCCESS;
+}
+
+template <typename rnumber>
+int native_binary_to_hdf5<rnumber>::read_parameters(void)
+{
+ TIMEZONE("native_binary_to_hdf5::read_parameters");
+ this->postprocess::read_parameters();
+ hid_t parameter_file = H5Fopen(
+ (this->simname + std::string(".h5")).c_str(),
+ H5F_ACC_RDONLY,
+ H5P_DEFAULT);
+ this->iteration_list = hdf5_tools::read_vector<int>(
+ parameter_file,
+ "/native_binary_to_hdf5/iteration_list");
+ H5Fclose(parameter_file);
+ MPI_Barrier(this->comm);
+ return EXIT_SUCCESS;
+}
+
+template class native_binary_to_hdf5<float>;
+template class native_binary_to_hdf5<double>;
+
diff --git a/bfps/cpp/full_code/native_binary_to_hdf5.hpp b/cpp/full_code/native_binary_to_hdf5.hpp
similarity index 86%
rename from bfps/cpp/full_code/native_binary_to_hdf5.hpp
rename to cpp/full_code/native_binary_to_hdf5.hpp
index 35619952a754ec9680b7681fd51d7bff862ac36a..3baaf609bfa4bba093dd097dce3be3d3aa590a4e 100644
--- a/bfps/cpp/full_code/native_binary_to_hdf5.hpp
+++ b/cpp/full_code/native_binary_to_hdf5.hpp
@@ -3,20 +3,20 @@
* Copyright 2017 Max Planck Institute *
* for Dynamics and Self-Organization *
* *
-* This file is part of bfps. *
+* This file is part of TurTLE. *
* *
-* bfps is free software: you can redistribute it and/or modify *
+* TurTLE is free software: you can redistribute it and/or modify *
* it under the terms of the GNU General Public License as published *
* by the Free Software Foundation, either version 3 of the License, *
* or (at your option) any later version. *
* *
-* bfps is distributed in the hope that it will be useful, *
+* TurTLE is distributed in the hope that it will be useful, *
* but WITHOUT ANY WARRANTY; without even the implied warranty of *
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the *
* GNU General Public License for more details. *
* *
* You should have received a copy of the GNU General Public License *
-* along with bfps. If not, see <http://www.gnu.org/licenses/> *
+* along with TurTLE. If not, see <http://www.gnu.org/licenses/> *
* *
* Contact: Cristian.Lalescu@ds.mpg.de *
* *
@@ -50,7 +50,7 @@ class native_binary_to_hdf5: public postprocess
postprocess(
COMMUNICATOR,
simulation_name){}
- virtual ~native_binary_to_hdf5(){}
+ virtual ~native_binary_to_hdf5() noexcept(false){}
int initialize(void);
int work_on_current_iteration(void);
diff --git a/cpp/full_code/ornstein_uhlenbeck_process.cpp b/cpp/full_code/ornstein_uhlenbeck_process.cpp
new file mode 100644
index 0000000000000000000000000000000000000000..653c2540f01ee18661346b04945c6cade09450d2
--- /dev/null
+++ b/cpp/full_code/ornstein_uhlenbeck_process.cpp
@@ -0,0 +1,292 @@
+#include "ornstein_uhlenbeck_process.hpp"
+#include <cmath>
+#include <cstring>
+#include <cassert>
+#include "scope_timer.hpp"
+#include <algorithm>
+#include <chrono>
+
+
+template <class rnumber,field_backend be>
+ornstein_uhlenbeck_process<rnumber,be>::ornstein_uhlenbeck_process(
+ const char *NAME,
+ int nx,
+ int ny,
+ int nz,
+ double ou_kmin,
+ double ou_kmax,
+ double ou_energy_amplitude,
+ double ou_gamma_factor,
+ double DKX,
+ double DKY,
+ double DKZ,
+ unsigned FFTW_PLAN_RIGOR)
+{
+ TIMEZONE("ornstein_uhlenbeck_process::ornstein_uhlenbeck_process");
+ strncpy(this->name, NAME, 256);
+ this->name[255] = '\0';
+ this->iteration = 0;
+ this->ou_field = new field<rnumber,be,THREE>(
+ nx,ny,nz, MPI_COMM_WORLD, FFTW_PLAN_RIGOR);
+ *this->ou_field = 0.0;
+ this->ou_field->dft();
+
+ this->ou_field_vort = new field<rnumber,be,THREE>(
+ nx,ny,nz, MPI_COMM_WORLD, FFTW_PLAN_RIGOR);
+ *this->ou_field_vort = 0.0;
+ this->ou_field_vort->dft();
+
+ this->B = new field<rnumber,be,THREExTHREE>(
+ nx,ny,nz, MPI_COMM_WORLD, FFTW_PLAN_RIGOR);
+
+ this->kk = new kspace<be,SMOOTH>(
+ this->ou_field->clayout, DKX, DKY, DKZ);
+
+ this->ou_kmin_squ = pow(ou_kmin,2);
+ this->ou_kmax_squ = pow(ou_kmax,2);
+ this->ou_energy_amplitude = ou_energy_amplitude;
+ this->ou_gamma_factor = ou_gamma_factor;
+ this->epsilon = pow((this->ou_energy_amplitude/this->kolmogorov_constant), 3./2.);
+
+ assert(this->kk->kM2 >= this->ou_kmax_squ);
+
+ gen.resize(omp_get_max_threads());
+
+ long now;
+
+ if (this->ou_field->clayout->myrank == 0){
+ now = std::chrono::duration_cast<std::chrono::milliseconds>(std::chrono::system_clock::now().time_since_epoch()).count();
+ }
+ MPI_Bcast(&now,1,MPI_LONG,0,this->ou_field->comm);
+
+ for(int thread=0;thread<omp_get_max_threads();thread++)
+ {
+ long current_seed =
+ this->ou_field->clayout->myrank*omp_get_max_threads() +
+ thread+now;
+ gen[thread].seed(current_seed);
+ }
+
+
+ this->initialize_B();
+
+}
+
+template <class rnumber,field_backend be>
+ornstein_uhlenbeck_process<rnumber,be>::~ornstein_uhlenbeck_process()
+{
+ TIMEZONE("ornstein_uhlenbeck_process::~ornstein_uhlenbeck_process");
+ delete this->kk;
+ delete this->ou_field;
+ delete this->ou_field_vort;
+ delete this->B;
+}
+
+template <class rnumber,field_backend be>
+void ornstein_uhlenbeck_process<rnumber,be>::step(
+ double dt)
+{
+ // put in "CFL"-criterium TODO!!!
+ TIMEZONE("ornstein_uhlenbeck_process::step");
+ this->kk->CLOOP_K2(
+ [&](ptrdiff_t cindex,
+ ptrdiff_t xindex,
+ ptrdiff_t yindex,
+ ptrdiff_t zindex,
+ double k2){
+ if (k2 <= this->ou_kmax_squ && k2 >= this->ou_kmin_squ)
+ {
+ double g = this->gamma(sqrt(k2));
+ int tr = omp_get_thread_num();
+ double random[6] =
+ {
+ this->d(this->gen[tr]),this->d(this->gen[tr]),this->d(this->gen[tr]),
+ this->d(this->gen[tr]),this->d(this->gen[tr]),this->d(this->gen[tr])
+ };
+ for (int cc=0; cc<3; cc++) for (int i=0; i<2; i++)
+ this->ou_field->cval(cindex,cc,i) =
+ (1-dt*g) * this->ou_field->cval(cindex,cc,i) +
+ sqrt(dt) * (
+ this->B->cval(cindex,0,cc,0) * random[0+3*i] +
+ this->B->cval(cindex,1,cc,0) * random[1+3*i] +
+ this->B->cval(cindex,2,cc,0) * random[2+3*i] );
+
+ }
+ });
+
+ this->ou_field->symmetrize();
+ this->calc_ou_vorticity();
+
+}
+
+
+template <class rnumber, field_backend be>
+void ornstein_uhlenbeck_process<rnumber,be>::initialize_B()
+{
+ TIMEZONE("ornstein_uhlenbeck_process::initialize_B");
+ *this->B = 0.0;
+ this->kk->CLOOP(
+ [&](ptrdiff_t cindex,
+ ptrdiff_t xindex,
+ ptrdiff_t yindex,
+ ptrdiff_t zindex){
+ double ksqu = pow(this->kk->kx[xindex],2)+
+ pow(this->kk->ky[yindex],2)+
+ pow(this->kk->kz[zindex],2);
+ if (ksqu <= this->ou_kmax_squ && ksqu >= this->ou_kmin_squ)
+ {
+ double kabs = sqrt(ksqu);
+ double sigma;
+ if (ksqu == 0 || ksqu == this->ou_kmax_squ)
+ {
+ ksqu = 1; kabs = 1; sigma = 0;
+ }
+ else
+ {
+ sigma =
+ sqrt(4*this->gamma(kabs)*this->energy(kabs)
+ /(4*M_PI*ksqu));
+ }
+
+ for(int i=0;i<3;i++) {
+ for(int j=0;j<3;j++) {
+
+ if (i+j == 0)
+ this->B->cval(cindex,i,j,0) =
+ sigma/2. * (1-pow(this->kk->kx[xindex],2)/ksqu);
+
+ if (i+j == 4)
+ this->B->cval(cindex,i,j,0) =
+ sigma/2. * (1-pow(this->kk->kz[zindex],2)/ksqu);
+
+ if (i+j == 1)
+ this->B->cval(cindex,i,j,0) =
+ sigma/2. * (0-this->kk->kx[xindex]*this->kk->ky[yindex]/ksqu);
+
+ if (i+j == 3)
+ this->B->cval(cindex,i,j,0) =
+ sigma/2. * (0-this->kk->kz[zindex]*this->kk->ky[yindex]/ksqu);
+
+ if (i+j == 2) {
+
+ if(i==j)
+ this->B->cval(cindex,i,j,0) =
+ sigma/2. * (1-pow(this->kk->ky[yindex],2)/ksqu);
+
+ if(i!=j)
+ this->B->cval(cindex,i,j,0) =
+ sigma/2. * (0-this->kk->kx[xindex]*this->kk->kz[zindex]/ksqu);
+ }
+ }
+ }
+ }
+
+ });
+}
+
+
+template <class rnumber, field_backend be>
+void ornstein_uhlenbeck_process<rnumber, be>::let_converge(void)
+{
+ double ou_kmin = sqrt(this->ou_kmin_squ);
+ double tau = 1.0/this->gamma(ou_kmin);
+ double dt = tau/1000.;
+
+ for(int i=0; i<2000; i++)
+ {
+ this->step(dt);
+ }
+}
+
+template <class rnumber, field_backend be>
+void ornstein_uhlenbeck_process<rnumber, be>::strip_from_field(
+ field<rnumber, be, THREE> *src)
+{
+ assert(src->real_space_representation==false);
+
+ this->kk->CLOOP_K2(
+ [&](ptrdiff_t cindex,
+ ptrdiff_t xindex,
+ ptrdiff_t yindex,
+ ptrdiff_t zindex,
+ double k2){
+
+ if (k2 <= this->ou_kmax_squ && k2 >= this->ou_kmin_squ){
+
+ for(int cc=0; cc < 3; cc++){
+ for(int imag=0; imag < 2; imag++){
+ src->cval(cindex,cc,imag) = 0;
+ }
+ }
+ }
+
+ }
+
+ );
+
+}
+
+template <class rnumber, field_backend be>
+void ornstein_uhlenbeck_process<rnumber,be>::add_to_field_replace(
+ field<rnumber, be, THREE> *src, std::string uv)
+{
+ assert(src->real_space_representation==false);
+ assert((uv == "vorticity") || (uv == "velocity"));
+
+ field<rnumber, be, THREE> *field_to_replace;
+
+ if (uv == "vorticity") field_to_replace = this->ou_field_vort;
+ else field_to_replace = this->ou_field;
+
+ this->kk->CLOOP_K2(
+ [&](ptrdiff_t cindex,
+ ptrdiff_t xindex,
+ ptrdiff_t yindex,
+ ptrdiff_t zindex,
+ double k2){
+
+ if (k2 <= this->ou_kmax_squ && k2 >= this->ou_kmin_squ){
+
+ rnumber tmp;
+
+ for(int cc=0; cc < 3; cc++){
+ for(int imag=0; imag < 2; imag++){
+ tmp = field_to_replace->cval(cindex,cc,imag);
+ src->cval(cindex,cc,imag) = tmp;
+ }
+ }
+ }
+
+ }
+
+ );
+}
+
+
+template <class rnumber, field_backend be>
+void ornstein_uhlenbeck_process<rnumber,be>::calc_ou_vorticity(void)
+{
+ this->kk->CLOOP_K2(
+ [&](ptrdiff_t cindex,
+ ptrdiff_t xindex,
+ ptrdiff_t yindex,
+ ptrdiff_t zindex,
+ double k2){
+ if (k2 <= this->kk->kM2)
+ {
+ this->ou_field_vort->cval(cindex,0,0) = -(this->kk->ky[yindex]*this->ou_field->cval(cindex,2,1) - this->kk->kz[zindex]*this->ou_field->cval(cindex,1,1));
+ this->ou_field_vort->cval(cindex,0,1) = (this->kk->ky[yindex]*this->ou_field->cval(cindex,2,0) - this->kk->kz[zindex]*this->ou_field->cval(cindex,1,0));
+ this->ou_field_vort->cval(cindex,1,0) = -(this->kk->kz[zindex]*this->ou_field->cval(cindex,0,1) - this->kk->kx[xindex]*this->ou_field->cval(cindex,2,1));
+ this->ou_field_vort->cval(cindex,1,1) = (this->kk->kz[zindex]*this->ou_field->cval(cindex,0,0) - this->kk->kx[xindex]*this->ou_field->cval(cindex,2,0));
+ this->ou_field_vort->cval(cindex,2,0) = -(this->kk->kx[xindex]*this->ou_field->cval(cindex,1,1) - this->kk->ky[yindex]*this->ou_field->cval(cindex,0,1));
+ this->ou_field_vort->cval(cindex,2,1) = (this->kk->kx[xindex]*this->ou_field->cval(cindex,1,0) - this->kk->ky[yindex]*this->ou_field->cval(cindex,0,0));
+ }
+ else
+ std::fill_n((rnumber*)(this->ou_field_vort->get_cdata()+3*cindex), 6, 0.0);
+ }
+ );
+ this->ou_field_vort->symmetrize();
+}
+
+template class ornstein_uhlenbeck_process<float,FFTW>;
+template class ornstein_uhlenbeck_process<double,FFTW>;
diff --git a/cpp/full_code/ornstein_uhlenbeck_process.hpp b/cpp/full_code/ornstein_uhlenbeck_process.hpp
new file mode 100644
index 0000000000000000000000000000000000000000..5a1288c929afac4cad4a3a6f36ff5b2a0436b1ca
--- /dev/null
+++ b/cpp/full_code/ornstein_uhlenbeck_process.hpp
@@ -0,0 +1,77 @@
+#ifndef ORNSTEIN_UHLENBECK_PROCESS_HPP
+#define ORNSTEIN_UHLENBECK_PROCESS_HPP
+
+#include <sys/stat.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <iostream>
+#include "field.hpp"
+#include <random>
+
+extern int myrank, nprocs;
+
+template <typename rnumber, field_backend be>
+class ornstein_uhlenbeck_process{
+ public:
+ char name[256];
+
+ int iteration;
+ double ou_kmin_squ;
+ double ou_kmax_squ;
+ double ou_energy_amplitude;
+ double ou_gamma_factor;
+ double kolmogorov_constant = 2;
+ double epsilon;
+
+ field<rnumber,be,THREE> *ou_field;
+ field<rnumber,be,THREE> *ou_field_vort;
+ field<rnumber,be,THREExTHREE> *B;
+ kspace<be,SMOOTH> *kk;
+
+ std::vector<std::mt19937_64> gen;
+ std::normal_distribution<double> d{0,1};
+
+ ornstein_uhlenbeck_process(
+ const char *NAME,
+ int nx,
+ int ny,
+ int nz,
+ double ou_kmin,
+ double ou_kmax,
+ double ou_energy_amplitude,
+ double ou_gamma_factor,
+ double DKX = 1.0,
+ double DKY = 1.0,
+ double DKZ = 1.0,
+ unsigned FFTW_PLAN_RIGOR = FFTW_MEASURE);
+ ~ornstein_uhlenbeck_process(void);
+
+ void step(double dt);
+ inline double energy(double kabs)
+ {
+ return this->ou_energy_amplitude * pow(kabs,-5./3);
+ }
+
+ inline double gamma(double kabs)
+ {
+ return this->ou_gamma_factor*pow(kabs,2./3.)*sqrt(this->ou_energy_amplitude);
+ }
+
+
+ void initialize_B(void);
+
+ void let_converge(void);
+
+ void add_to_field_replace(
+ field<rnumber,be,THREE> *src, std::string uv);
+
+ void strip_from_field(
+ field<rnumber, be, THREE> *src);
+
+ void calc_ou_vorticity(void);
+
+
+};
+
+
+#endif
diff --git a/cpp/full_code/ou_vorticity_equation.cpp b/cpp/full_code/ou_vorticity_equation.cpp
new file mode 100644
index 0000000000000000000000000000000000000000..561b70201e8167c08b3b4b14d457a20e5b583539
--- /dev/null
+++ b/cpp/full_code/ou_vorticity_equation.cpp
@@ -0,0 +1,163 @@
+#include "ou_vorticity_equation.hpp"
+#include <cmath>
+#include <cstring>
+#include <cassert>
+#include "scope_timer.hpp"
+
+
+template <class rnumber, field_backend be>
+ou_vorticity_equation<rnumber,be>::~ou_vorticity_equation()
+{
+ delete this->ou;
+}
+
+template <class rnumber,
+ field_backend be>
+void ou_vorticity_equation<rnumber, be>::step(double dt)
+{
+ DEBUG_MSG("vorticity_equation::step\n");
+ TIMEZONE("vorticity_equation::step");
+ this->ou->add_to_field_replace(this->cvorticity, "vorticity");
+ *this->v[1] = 0.0;
+ this->omega_nonlin(0);
+ this->kk->CLOOP_K2(
+ [&](ptrdiff_t cindex,
+ ptrdiff_t xindex,
+ ptrdiff_t yindex,
+ ptrdiff_t zindex,
+ double k2){
+ if (k2 <= this->kk->kM2)
+ {
+ double factor0;
+ factor0 = exp(-this->nu * k2 * dt);
+ for (int cc=0; cc<3; cc++) for (int i=0; i<2; i++)
+ this->v[1]->cval(cindex,cc,i) = (
+ this->v[0]->cval(cindex,cc,i) +
+ dt*this->u->cval(cindex,cc,i))*factor0;
+ }
+ }
+ );
+
+ this->omega_nonlin(1);
+ this->kk->CLOOP_K2(
+ [&](ptrdiff_t cindex,
+ ptrdiff_t xindex,
+ ptrdiff_t yindex,
+ ptrdiff_t zindex,
+ double k2){
+ if (k2 <= this->kk->kM2)
+ {
+ double factor0, factor1;
+ factor0 = exp(-this->nu * k2 * dt/2);
+ factor1 = exp( this->nu * k2 * dt/2);
+ for (int cc=0; cc<3; cc++) for (int i=0; i<2; i++)
+ this->v[2]->cval(cindex, cc, i) = (
+ 3*this->v[0]->cval(cindex,cc,i)*factor0 +
+ ( this->v[1]->cval(cindex,cc,i) +
+ dt*this->u->cval(cindex,cc,i))*factor1)*0.25;
+ }
+ }
+ );
+
+ this->omega_nonlin(2);
+ // store old vorticity
+ *this->v[1] = *this->v[0];
+ this->kk->CLOOP_K2(
+ [&](ptrdiff_t cindex,
+ ptrdiff_t xindex,
+ ptrdiff_t yindex,
+ ptrdiff_t zindex,
+ double k2){
+ if (k2 <= this->kk->kM2)
+ {
+ double factor0;
+ factor0 = exp(-this->nu * k2 * dt * 0.5);
+ for (int cc=0; cc<3; cc++) for (int i=0; i<2; i++)
+ this->v[3]->cval(cindex,cc,i) = (
+ this->v[0]->cval(cindex,cc,i)*factor0 +
+ 2*(this->v[2]->cval(cindex,cc,i) +
+ dt*this->u->cval(cindex,cc,i)))*factor0/3;
+ }
+ }
+ );
+
+ this->kk->template force_divfree<rnumber>(this->cvorticity->get_cdata());
+ this->cvorticity->symmetrize();
+ this->iteration++;
+}
+
+
+template <class rnumber, field_backend be>
+void ou_vorticity_equation<rnumber, be>::omega_nonlin(int src)
+{
+ DEBUG_MSG("ou_vorticity_equation::omega_nonlin(%d)\n", src);
+ assert(src >= 0 && src < 3);
+ this->compute_velocity(this->v[src]);
+
+ this->add_ou_forcing_velocity();
+ // TIME STEP TODO
+
+ /* get fields from Fourier space to real space */
+ this->u->ift();
+ this->rvorticity->real_space_representation = false;
+ *this->rvorticity = this->v[src]->get_cdata();
+ this->rvorticity->ift();
+ /* compute cross product $u \times \omega$, and normalize */
+ this->u->RLOOP(
+ [&](ptrdiff_t rindex,
+ ptrdiff_t xindex,
+ ptrdiff_t yindex,
+ ptrdiff_t zindex){
+ rnumber tmp[3];
+ for (int cc=0; cc<3; cc++)
+ tmp[cc] = (this->u->rval(rindex,(cc+1)%3)*this->rvorticity->rval(rindex,(cc+2)%3) -
+ this->u->rval(rindex,(cc+2)%3)*this->rvorticity->rval(rindex,(cc+1)%3));
+ for (int cc=0; cc<3; cc++)
+ this->u->rval(rindex,cc) = tmp[cc] / this->u->npoints;
+ }
+ );
+ /* go back to Fourier space */
+ //this->clean_up_real_space(this->ru, 3);
+ this->u->dft();
+ this->kk->template dealias<rnumber, THREE>(this->u->get_cdata());
+ /* $\imath k \times Fourier(u \times \omega)$ */
+ this->kk->CLOOP(
+ [&](ptrdiff_t cindex,
+ ptrdiff_t xindex,
+ ptrdiff_t yindex,
+ ptrdiff_t zindex){
+ rnumber tmp[3][2];
+ {
+ tmp[0][0] = -(this->kk->ky[yindex]*this->u->cval(cindex,2,1) - this->kk->kz[zindex]*this->u->cval(cindex,1,1));
+ tmp[1][0] = -(this->kk->kz[zindex]*this->u->cval(cindex,0,1) - this->kk->kx[xindex]*this->u->cval(cindex,2,1));
+ tmp[2][0] = -(this->kk->kx[xindex]*this->u->cval(cindex,1,1) - this->kk->ky[yindex]*this->u->cval(cindex,0,1));
+ tmp[0][1] = (this->kk->ky[yindex]*this->u->cval(cindex,2,0) - this->kk->kz[zindex]*this->u->cval(cindex,1,0));
+ tmp[1][1] = (this->kk->kz[zindex]*this->u->cval(cindex,0,0) - this->kk->kx[xindex]*this->u->cval(cindex,2,0));
+ tmp[2][1] = (this->kk->kx[xindex]*this->u->cval(cindex,1,0) - this->kk->ky[yindex]*this->u->cval(cindex,0,0));
+ }
+ for (int cc=0; cc<3; cc++) for (int i=0; i<2; i++)
+ this->u->cval(cindex, cc, i) = tmp[cc][i];
+ }
+ );
+ this->kk->template force_divfree<rnumber>(this->u->get_cdata());
+ this->u->symmetrize();
+}
+
+template <class rnumber, field_backend be>
+void ou_vorticity_equation<rnumber, be>::add_ou_forcing_velocity()
+{
+ if (this->ou_forcing_type == "replace"){
+ this->ou->add_to_field_replace(this->u,"velocity");
+ }
+}
+
+template <class rnumber, field_backend be>
+void ou_vorticity_equation<rnumber, be>::add_ou_forcing_vorticity()
+{
+ if (this->ou_forcing_type == "replace"){
+ this->ou->add_to_field_replace(this->cvorticity,"vorticity");
+ }
+}
+
+template class ou_vorticity_equation<float, FFTW>;
+template class ou_vorticity_equation<double, FFTW>;
diff --git a/cpp/full_code/ou_vorticity_equation.hpp b/cpp/full_code/ou_vorticity_equation.hpp
new file mode 100644
index 0000000000000000000000000000000000000000..8f8e110c64ca29bd1a5039a96f7cc8e632b5fc7b
--- /dev/null
+++ b/cpp/full_code/ou_vorticity_equation.hpp
@@ -0,0 +1,54 @@
+#ifndef OU_VORTICITY_EQUATION_HPP
+#define OU_VORTICITY_EQUATION_HPP
+
+#include <cstdlib>
+#include "vorticity_equation.hpp"
+#include "ornstein_uhlenbeck_process.hpp"
+
+
+// vorticity_equation<double, FFTW> *test;
+
+template <typename rnumber, field_backend be>
+class ou_vorticity_equation : public vorticity_equation<rnumber, be>
+{
+ public:
+
+ std::string ou_forcing_type;
+
+ ornstein_uhlenbeck_process<rnumber,be> *ou;
+ ou_vorticity_equation(
+ const char *NAME,
+ int nx,
+ int ny,
+ int nz,
+ double ou_kmin,
+ double ou_kmax,
+ double ou_energy_amplitude,
+ double ou_gamma_factor,
+ double DKX = 1.0,
+ double DKY = 1.0,
+ double DKZ = 1.0,
+ unsigned FFTW_PLAN_RIGOR = FFTW_MEASURE):
+ vorticity_equation<rnumber,be>(
+ NAME, nx, ny, nz,
+ DKX,DKY,DKZ, FFTW_PLAN_RIGOR){
+ this->ou = new ornstein_uhlenbeck_process<rnumber,be>(
+ NAME,
+ nx, ny, nz,
+ ou_kmin, ou_kmax, ou_energy_amplitude,ou_gamma_factor,
+ DKX, DKY, DKZ, FFTW_PLAN_RIGOR);
+ this->ou_forcing_type = "replace";
+ }
+
+ ~ou_vorticity_equation();
+
+ void omega_nonlin(int src);
+ void step(double dt);
+
+ void add_ou_forcing_velocity(void);
+ void add_ou_forcing_vorticity(void);
+
+};
+
+
+#endif
diff --git a/cpp/full_code/phase_shift_test.cpp b/cpp/full_code/phase_shift_test.cpp
new file mode 100644
index 0000000000000000000000000000000000000000..7e6c2ad402ca3becf99c497eceefd511ae61f113
--- /dev/null
+++ b/cpp/full_code/phase_shift_test.cpp
@@ -0,0 +1,139 @@
+/******************************************************************************
+* *
+* Copyright 2019 Max Planck Institute for Dynamics and Self-Organization *
+* *
+* This file is part of TurTLE. *
+* *
+* TurTLE is free software: you can redistribute it and/or modify *
+* it under the terms of the GNU General Public License as published *
+* by the Free Software Foundation, either version 3 of the License, *
+* or (at your option) any later version. *
+* *
+* TurTLE is distributed in the hope that it will be useful, *
+* but WITHOUT ANY WARRANTY; without even the implied warranty of *
+* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the *
+* GNU General Public License for more details. *
+* *
+* You should have received a copy of the GNU General Public License *
+* along with TurTLE. If not, see <http://www.gnu.org/licenses/> *
+* *
+* Contact: Cristian.Lalescu@ds.mpg.de *
+* *
+******************************************************************************/
+
+
+
+#include <string>
+#include <cmath>
+#include <random>
+#include "phase_shift_test.hpp"
+#include "scope_timer.hpp"
+
+
+template <typename rnumber>
+int phase_shift_test<rnumber>::initialize(void)
+{
+ TIMEZONE("phase_shift_test::initialize");
+ this->read_parameters();
+ this->random_phase_seed = 1;
+ return EXIT_SUCCESS;
+}
+
+template <typename rnumber>
+int phase_shift_test<rnumber>::finalize(void)
+{
+ TIMEZONE("phase_shift_test::finalize");
+ return EXIT_SUCCESS;
+}
+
+template <typename rnumber>
+int phase_shift_test<rnumber>::read_parameters()
+{
+ TIMEZONE("phase_shift_test::read_parameters");
+ this->test::read_parameters();
+ MPI_Barrier(this->comm);
+ return EXIT_SUCCESS;
+}
+
+template <typename rnumber>
+int phase_shift_test<rnumber>::do_work(void)
+{
+ TIMEZONE("phase_shift_test::do_work");
+ // allocate
+ this->phase_field = new field<rnumber, FFTW, ONE>(
+ this->nx, this->ny, this->nz,
+ this->comm,
+ FFTW_ESTIMATE);
+ this->phase_field->real_space_representation = false;
+ this->kk = new kspace<FFTW, SMOOTH>(
+ this->phase_field->clayout, this->dkx, this->dky, this->dkz);
+ // phase field must be in Fourier space
+ // then the following initializes real parts with 1, and imaginary parts with 0
+ *this->phase_field = 1.0;
+ // now apply random phase shift
+ generate_random_phase_field(
+ this->kk,
+ this->phase_field,
+ this->random_phase_seed);
+
+ this->phase_field->io(
+ this->simname + "_phase_field.h5",
+ "random_phase_field",
+ 0,
+ false);
+
+ this->phase_field->ift();
+ this->phase_field->dft();
+ this->phase_field->normalize();
+
+ this->phase_field->io(
+ this->simname + "_phase_field.h5",
+ "random_phase_field",
+ 1,
+ false);
+
+ field<rnumber, FFTW, THREE> *vf = new field<rnumber, FFTW, THREE>(
+ this->nx, this->ny, this->nz,
+ this->comm,
+ FFTW_ESTIMATE);
+ make_gaussian_random_field(
+ this->kk,
+ vf,
+ 1,
+ 0.4,
+ 1,
+ 0.01,
+ 1,
+ 1,
+ 3./2. //incompressibility projection factor
+ );
+ this->kk->template project_divfree<rnumber>(vf->get_cdata());
+
+ apply_phase_field_shift(
+ this->kk,
+ vf,
+ this->phase_field);
+
+ compute_divergence(
+ this->kk,
+ vf,
+ this->phase_field);
+
+ this->phase_field->ift();
+
+ this->phase_field->io(
+ this->simname + "_phase_field.h5",
+ "divergence",
+ 1,
+ false);
+
+ // deallocate
+ delete vf;
+ delete this->phase_field;
+ delete this->kk;
+ return EXIT_SUCCESS;
+}
+
+template class phase_shift_test<float>;
+template class phase_shift_test<double>;
+
diff --git a/cpp/full_code/phase_shift_test.hpp b/cpp/full_code/phase_shift_test.hpp
new file mode 100644
index 0000000000000000000000000000000000000000..e98f76cd568239a88f15da18c4fa72bc255fae9b
--- /dev/null
+++ b/cpp/full_code/phase_shift_test.hpp
@@ -0,0 +1,63 @@
+/**********************************************************************
+* *
+* Copyright 2021 Max Planck Computing and Data Facility *
+* *
+* This file is part of TurTLE. *
+* *
+* TurTLE is free software: you can redistribute it and/or modify *
+* it under the terms of the GNU General Public License as published *
+* by the Free Software Foundation, either version 3 of the License, *
+* or (at your option) any later version. *
+* *
+* TurTLE is distributed in the hope that it will be useful, *
+* but WITHOUT ANY WARRANTY; without even the implied warranty of *
+* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the *
+* GNU General Public License for more details. *
+* *
+* You should have received a copy of the GNU General Public License *
+* along with TurTLE. If not, see <http://www.gnu.org/licenses/> *
+* *
+* Contact: Cristian.Lalescu@mpcdf.mpg.de *
+* *
+**********************************************************************/
+
+
+
+#ifndef PHASE_SHIFT_TEST_HPP
+#define PHASE_SHIFT_TEST_HPP
+
+
+
+#include <cstdlib>
+#include "base.hpp"
+#include "kspace.hpp"
+#include "field.hpp"
+#include "full_code/test.hpp"
+
+/** \brief A class for testing phase shift functionality.
+ */
+
+template <typename rnumber>
+class phase_shift_test: public test
+{
+ private:
+ kspace<FFTW, SMOOTH> *kk;
+ field<rnumber, FFTW, ONE> *phase_field;
+ int random_phase_seed;
+ public:
+ phase_shift_test(
+ const MPI_Comm COMMUNICATOR,
+ const std::string &simulation_name):
+ test(
+ COMMUNICATOR,
+ simulation_name){}
+ ~phase_shift_test() noexcept(false){}
+
+ int initialize(void);
+ int do_work(void);
+ int finalize(void);
+ int read_parameters(void);
+};
+
+#endif// PHASE_SHIFT_TEST_HPP
+
diff --git a/cpp/full_code/postprocess.cpp b/cpp/full_code/postprocess.cpp
new file mode 100644
index 0000000000000000000000000000000000000000..0f80890c144ba6a64ffcfbb72e9b24ede476aea2
--- /dev/null
+++ b/cpp/full_code/postprocess.cpp
@@ -0,0 +1,103 @@
+/******************************************************************************
+* *
+* Copyright 2019 Max Planck Institute for Dynamics and Self-Organization *
+* *
+* This file is part of TurTLE. *
+* *
+* TurTLE is free software: you can redistribute it and/or modify *
+* it under the terms of the GNU General Public License as published *
+* by the Free Software Foundation, either version 3 of the License, *
+* or (at your option) any later version. *
+* *
+* TurTLE is distributed in the hope that it will be useful, *
+* but WITHOUT ANY WARRANTY; without even the implied warranty of *
+* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the *
+* GNU General Public License for more details. *
+* *
+* You should have received a copy of the GNU General Public License *
+* along with TurTLE. If not, see <http://www.gnu.org/licenses/> *
+* *
+* Contact: Cristian.Lalescu@ds.mpg.de *
+* *
+******************************************************************************/
+
+
+
+#include <cstdlib>
+#include <sys/types.h>
+#include <sys/stat.h>
+#include "scope_timer.hpp"
+#include "hdf5_tools.hpp"
+#include "full_code/postprocess.hpp"
+
+
+int postprocess::main_loop(void)
+{
+ TIMEZONE("postprocess::main_loop");
+ this->start_simple_timer();
+ for (unsigned int iteration_counter = 0;
+ iteration_counter < iteration_list.size();
+ iteration_counter++)
+ {
+ this->iteration = iteration_list[iteration_counter];
+ #ifdef USE_TIMINGOUTPUT
+ const std::string loopLabel = ("postprocess::main_loop-" +
+ std::to_string(this->iteration));
+ TIMEZONE(loopLabel.c_str());
+ #endif
+ this->work_on_current_iteration();
+ this->print_simple_timer(
+ "iteration " + std::to_string(this->iteration));
+
+ this->check_stopping_condition();
+ if (this->stop_code_now)
+ break;
+ }
+ return EXIT_SUCCESS;
+}
+
+
+int postprocess::read_parameters()
+{
+ TIMEZONE("postprocess::read_parameters");
+ this->code_base::read_parameters();
+ hid_t parameter_file = H5Fopen((this->simname + ".h5").c_str(), H5F_ACC_RDONLY, H5P_DEFAULT);
+ this->nu = hdf5_tools::read_value<double>(parameter_file, "parameters/nu");
+ this->dt = hdf5_tools::read_value<double>(parameter_file, "parameters/dt");
+ this->fmode = hdf5_tools::read_value<int>(parameter_file, "parameters/fmode");
+ this->famplitude = hdf5_tools::read_value<double>(parameter_file, "parameters/famplitude");
+ this->friction_coefficient = hdf5_tools::read_value<double>(parameter_file, "parameters/friction_coefficient");
+ this->fk0 = hdf5_tools::read_value<double>(parameter_file, "parameters/fk0");
+ this->fk1 = hdf5_tools::read_value<double>(parameter_file, "parameters/fk1");
+ this->energy = hdf5_tools::read_value<double>(parameter_file, "parameters/energy");
+ this->injection_rate = hdf5_tools::read_value<double>(parameter_file, "parameters/injection_rate");
+ std::string tmp = hdf5_tools::read_string(parameter_file, "parameters/forcing_type");
+ snprintf(this->forcing_type, 511, "%s", tmp.c_str());
+ H5Fclose(parameter_file);
+ MPI_Barrier(this->comm);
+ return EXIT_SUCCESS;
+}
+
+template <typename rnumber,
+ field_backend be>
+int postprocess::copy_parameters_into(
+ vorticity_equation<rnumber, be> *dst)
+{
+ dst->nu = this->nu;
+ dst->fmode = this->fmode;
+ dst->famplitude = this->famplitude;
+ dst->fk0 = this->fk0;
+ dst->fk1 = this->fk1;
+ dst->injection_rate = this->injection_rate;
+ dst->energy = this->energy;
+ dst->friction_coefficient = this->friction_coefficient;
+ strncpy(this->forcing_type, dst->forcing_type, 128);
+ return EXIT_SUCCESS;
+}
+
+template int postprocess::copy_parameters_into<float, FFTW>(
+ vorticity_equation<float, FFTW> *dst);
+
+template int postprocess::copy_parameters_into<double, FFTW>(
+ vorticity_equation<double, FFTW> *dst);
+
diff --git a/bfps/cpp/full_code/postprocess.hpp b/cpp/full_code/postprocess.hpp
similarity index 68%
rename from bfps/cpp/full_code/postprocess.hpp
rename to cpp/full_code/postprocess.hpp
index c80fc3f2dfdc35691d9e69442fa3ad7b6e592891..da22c91e9f7ca187cf7b3038994c0852e255924b 100644
--- a/bfps/cpp/full_code/postprocess.hpp
+++ b/cpp/full_code/postprocess.hpp
@@ -3,20 +3,20 @@
* Copyright 2017 Max Planck Institute *
* for Dynamics and Self-Organization *
* *
-* This file is part of bfps. *
+* This file is part of TurTLE. *
* *
-* bfps is free software: you can redistribute it and/or modify *
+* TurTLE is free software: you can redistribute it and/or modify *
* it under the terms of the GNU General Public License as published *
* by the Free Software Foundation, either version 3 of the License, *
* or (at your option) any later version. *
* *
-* bfps is distributed in the hope that it will be useful, *
+* TurTLE is distributed in the hope that it will be useful, *
* but WITHOUT ANY WARRANTY; without even the implied warranty of *
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the *
* GNU General Public License for more details. *
* *
* You should have received a copy of the GNU General Public License *
-* along with bfps. If not, see <http://www.gnu.org/licenses/> *
+* along with TurTLE. If not, see <http://www.gnu.org/licenses/> *
* *
* Contact: Cristian.Lalescu@ds.mpg.de *
* *
@@ -32,6 +32,7 @@
#include <sys/stat.h>
#include <vector>
#include "base.hpp"
+#include "vorticity_equation.hpp"
#include "full_code/code_base.hpp"
class postprocess: public code_base
@@ -40,11 +41,18 @@ class postprocess: public code_base
std::vector<int> iteration_list;
hid_t stat_file;
- /* parameters that are read in read_parameters */
+ /* parameters that are read in read_parameters.
+ * This includes physical parameters, such that children of `postprocess`
+ * do not need to read them (but they do need to be copied with
+ * the `copy_parameters_into` method or similar.
+ * */
double dt;
double famplitude;
+ double friction_coefficient;
double fk0;
double fk1;
+ double energy;
+ double injection_rate;
int fmode;
char forcing_type[512];
double nu;
@@ -55,7 +63,7 @@ class postprocess: public code_base
code_base(
COMMUNICATOR,
simulation_name){}
- virtual ~postprocess(){}
+ virtual ~postprocess() noexcept(false){}
virtual int initialize(void) = 0;
virtual int work_on_current_iteration(void) = 0;
@@ -63,6 +71,14 @@ class postprocess: public code_base
int main_loop(void);
virtual int read_parameters(void);
+
+ /* Method to copy physical simulation parameters into arbitrary
+ * `vorticity_equation` object (defined by children classes).
+ * */
+ template <typename rnumber,
+ field_backend be>
+ int copy_parameters_into(
+ vorticity_equation<rnumber, be> *dst);
};
#endif//POSTPROCESS_HPP
diff --git a/cpp/full_code/pressure_stats.cpp b/cpp/full_code/pressure_stats.cpp
new file mode 100644
index 0000000000000000000000000000000000000000..abeccf5cacca445032bd1e1f8898522f280c745f
--- /dev/null
+++ b/cpp/full_code/pressure_stats.cpp
@@ -0,0 +1,266 @@
+/******************************************************************************
+* *
+* Copyright 2019 Max Planck Institute for Dynamics and Self-Organization *
+* *
+* This file is part of TurTLE. *
+* *
+* TurTLE is free software: you can redistribute it and/or modify *
+* it under the terms of the GNU General Public License as published *
+* by the Free Software Foundation, either version 3 of the License, *
+* or (at your option) any later version. *
+* *
+* TurTLE is distributed in the hope that it will be useful, *
+* but WITHOUT ANY WARRANTY; without even the implied warranty of *
+* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the *
+* GNU General Public License for more details. *
+* *
+* You should have received a copy of the GNU General Public License *
+* along with TurTLE. If not, see <http://www.gnu.org/licenses/> *
+* *
+* Contact: Cristian.Lalescu@ds.mpg.de *
+* *
+******************************************************************************/
+
+
+
+#include <string>
+#include <cmath>
+#include "pressure_stats.hpp"
+#include "scope_timer.hpp"
+#include "hdf5_tools.hpp"
+
+
+template <typename rnumber>
+int pressure_stats<rnumber>::initialize(void)
+{
+ TIMEZONE("pressure_stats::initialize");
+ this->NSVE_field_stats<rnumber>::initialize();
+
+ /// allocate kspace objects
+ this->kk4 = new kspace<FFTW, ONE_HALF>(
+ this->vorticity->clayout, this->dkx, this->dky, this->dkz);
+ this->kk3 = new kspace<FFTW, TWO_THIRDS>(
+ this->vorticity->clayout, this->dkx, this->dky, this->dkz);
+
+ this->ve = new vorticity_equation<rnumber, FFTW>(
+ this->simname.c_str(),
+ this->nx,
+ this->ny,
+ this->nz,
+ this->dkx,
+ this->dky,
+ this->dkz,
+ this->vorticity->fftw_plan_rigor);
+ this->template copy_parameters_into<rnumber, FFTW>(this->ve);
+
+ /// allocate extra fields
+ this->nabla_u = new field<rnumber, FFTW, THREExTHREE>(
+ this->nx, this->ny, this->nz,
+ this->comm,
+ this->ve->cvorticity->fftw_plan_rigor);
+ this->pressure = new field<rnumber, FFTW, ONE>(
+ this->nx, this->ny, this->nz,
+ this->comm,
+ this->ve->cvorticity->fftw_plan_rigor);
+ this->nabla_p = new field<rnumber, FFTW, THREE>(
+ this->nx, this->ny, this->nz,
+ this->comm,
+ this->ve->cvorticity->fftw_plan_rigor);
+ this->Hessian_p = new field<rnumber, FFTW, THREExTHREE>(
+ this->nx, this->ny, this->nz,
+ this->comm,
+ this->ve->cvorticity->fftw_plan_rigor);
+ this->scalar = new field<rnumber, FFTW, ONE>(
+ this->nx, this->ny, this->nz,
+ this->comm,
+ this->ve->cvorticity->fftw_plan_rigor);
+
+ hid_t parameter_file = H5Fopen(
+ (this->simname + std::string(".h5")).c_str(),
+ H5F_ACC_RDONLY,
+ H5P_DEFAULT);
+ this->niter_out = hdf5_tools::read_value<int>(parameter_file, "/parameters/niter_out");
+ this->checkpoints_per_file = hdf5_tools::read_value<int>(parameter_file, "/parameters/checkpoints_per_file");
+ if (this->checkpoints_per_file == INT_MAX) // value returned if dataset does not exist
+ this->checkpoints_per_file = 1;
+ H5Fclose(parameter_file);
+ // the following ensures the file is free for rank 0 to open in read/write mode
+ MPI_Barrier(this->comm);
+ parameter_file = H5Fopen(
+ (this->simname + std::string("_post.h5")).c_str(),
+ H5F_ACC_RDONLY,
+ H5P_DEFAULT);
+ this->iteration_list = hdf5_tools::read_vector<int>(
+ parameter_file,
+ "/pressure_stats/parameters/iteration_list");
+ this->max_pressure_estimate = hdf5_tools::read_value<double>(
+ parameter_file,
+ "/pressure_stats/parameters/max_pressure_estimate");
+ H5Fclose(parameter_file);
+ // the following ensures the file is free for rank 0 to open in read/write mode
+ MPI_Barrier(this->comm);
+ if (this->myrank == 0)
+ {
+ // set caching parameters
+ hid_t fapl = H5Pcreate(H5P_FILE_ACCESS);
+ herr_t cache_err = H5Pset_cache(fapl, 0, 521, 134217728, 1.0);
+ variable_used_only_in_assert(cache_err);
+ DEBUG_MSG("when setting stat_file cache I got %d\n", cache_err);
+ this->stat_file = H5Fopen(
+ (this->simname + "_post.h5").c_str(),
+ H5F_ACC_RDWR,
+ fapl);
+ }
+ else
+ {
+ this->stat_file = 0;
+ }
+ int data_file_problem;
+ if (this->myrank == 0)
+ data_file_problem = hdf5_tools::require_size_file_datasets(
+ this->stat_file,
+ "pressure_stats",
+ (this->iteration_list.back() / this->niter_out) + 1);
+ MPI_Bcast(&data_file_problem, 1, MPI_INT, 0, this->comm);
+ if (data_file_problem > 0)
+ {
+ std::cerr <<
+ data_file_problem <<
+ " problems setting sizes of file datasets.\ntrying to exit now." <<
+ std::endl;
+ return EXIT_FAILURE;
+ }
+ return EXIT_SUCCESS;
+}
+
+template <typename rnumber>
+int pressure_stats<rnumber>::work_on_current_iteration(void)
+{
+ TIMEZONE("pressure_stats::work_on_current_iteration");
+ /// read current vorticity, place it in this->ve->cvorticity
+ this->read_current_cvorticity();
+ *this->ve->cvorticity = this->vorticity->get_cdata();
+ /// after the previous instruction, we are free to use this->vorticity
+ /// for any other purpose
+
+ /// initialize `stat_group`.
+ hid_t stat_group;
+ if (this->myrank == 0)
+ stat_group = H5Gopen(
+ this->stat_file,
+ "pressure_stats",
+ H5P_DEFAULT);
+ else
+ stat_group = 0;
+
+ /// define local "compute scalar stats" lambda function
+ /// it can't be a simple function, C++ doesn't allow that.
+ /// however, we want a simple function here because this whole block
+ /// consists of code that must be identical when applied to different
+ /// versions of the velocity field (i.e. vel field differently filtered).
+ /// luckily, C++11 allows lambda functions.
+ /// we must explicitly pass the "this" and "stat_group" names to the lambda function.
+ auto compute_stats_for_cvelocity = [this, stat_group](std::string stat_prefix)
+ {
+ /// set scalar to 0
+ this->scalar->real_space_representation = true;
+ *this->scalar = 0.0;
+
+ /// compute velocity
+ this->ve->compute_velocity(this->ve->cvorticity);
+ /// compute velocity gradient
+ compute_gradient(
+ this->ve->kk,
+ this->ve->cvelocity,
+ this->nabla_u);
+ /// put velocity gradient in real space
+ this->nabla_u->ift();
+ /// add Aij Aji to scalar
+ this->scalar->RLOOP(
+ [&](ptrdiff_t rindex,
+ ptrdiff_t xindex,
+ ptrdiff_t yindex,
+ ptrdiff_t zindex){
+ for (int c0 = 0; c0 < 3; c0++)
+ for (int c1 = 0; c1 < 3; c1++)
+ {
+ scalar->rval(rindex) += this->nabla_u->rval(rindex, c0, c1)*this->nabla_u->rval(rindex, c1, c0);
+ }
+ });
+
+ /// compute pressure. velocity should already be in Fourier space.
+ this->ve->compute_pressure(this->pressure);
+ /// compute pressure gradient
+ compute_gradient(
+ this->ve->kk,
+ this->pressure,
+ this->nabla_p);
+ /// compute pressure Hessian
+ compute_gradient(
+ this->ve->kk,
+ this->nabla_p,
+ this->Hessian_p);
+ this->Hessian_p->ift();
+
+ /// add Hii to scalar
+ this->scalar->RLOOP(
+ [&](ptrdiff_t rindex,
+ ptrdiff_t xindex,
+ ptrdiff_t yindex,
+ ptrdiff_t zindex){
+ for (int c0 = 0; c0 < 3; c0++)
+ {
+ scalar->rval(rindex) += this->Hessian_p->rval(rindex, c0, c0);
+ }
+ });
+
+ /// compute statistics of scalar
+ std::vector<double> max_scal_estimate;
+ max_scal_estimate.resize(1, max_pressure_estimate);
+ this->scalar->compute_rspace_stats(
+ stat_group,
+ stat_prefix + std::string("_scalar"),
+ this->iteration / this->niter_out,
+ max_scal_estimate);
+ };
+
+ /// 1. "unfiltered" velocity
+ compute_stats_for_cvelocity("kk0");
+
+ /// 2. "2/3" velocity
+ this->kk3->template dealias<rnumber, THREE>(this->ve->cvorticity->get_cdata());
+ compute_stats_for_cvelocity("kk3");
+
+ /// 3. "1/2" velocity. the filtering is a simple band pass,
+ /// so applying it to the 2/3 filtered data is not a problem.
+ this->kk4->template dealias<rnumber, THREE>(this->ve->cvorticity->get_cdata());
+ compute_stats_for_cvelocity("kk4");
+
+ /// close stat group
+ if (this->myrank == 0)
+ H5Gclose(stat_group);
+
+ return EXIT_SUCCESS;
+}
+
+template <typename rnumber>
+int pressure_stats<rnumber>::finalize(void)
+{
+ DEBUG_MSG("entered pressure_stats::finalize\n");
+ delete this->scalar;
+ delete this->nabla_u;
+ delete this->pressure;
+ delete this->nabla_p;
+ delete this->Hessian_p;
+ delete this->ve;
+ delete this->kk3;
+ delete this->kk4;
+ if (this->myrank == 0)
+ H5Fclose(this->stat_file);
+ this->NSVE_field_stats<rnumber>::finalize();
+ return EXIT_SUCCESS;
+}
+
+template class pressure_stats<float>;
+template class pressure_stats<double>;
+
diff --git a/cpp/full_code/pressure_stats.hpp b/cpp/full_code/pressure_stats.hpp
new file mode 100644
index 0000000000000000000000000000000000000000..f20ba08ee767fcd44bf6017670f3a3a41d7ba96e
--- /dev/null
+++ b/cpp/full_code/pressure_stats.hpp
@@ -0,0 +1,75 @@
+/**********************************************************************
+* *
+* Copyright 2017 Max Planck Institute *
+* for Dynamics and Self-Organization *
+* *
+* This file is part of TurTLE. *
+* *
+* TurTLE is free software: you can redistribute it and/or modify *
+* it under the terms of the GNU General Public License as published *
+* by the Free Software Foundation, either version 3 of the License, *
+* or (at your option) any later version. *
+* *
+* TurTLE is distributed in the hope that it will be useful, *
+* but WITHOUT ANY WARRANTY; without even the implied warranty of *
+* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the *
+* GNU General Public License for more details. *
+* *
+* You should have received a copy of the GNU General Public License *
+* along with TurTLE. If not, see <http://www.gnu.org/licenses/> *
+* *
+* Contact: Cristian.Lalescu@ds.mpg.de *
+* *
+**********************************************************************/
+
+
+
+#ifndef PRESSURE_STATS_HPP
+#define PRESSURE_STATS_HPP
+
+#include <cstdlib>
+#include <sys/types.h>
+#include <sys/stat.h>
+#include <vector>
+#include "base.hpp"
+#include "field.hpp"
+#include "field_binary_IO.hpp"
+#include "vorticity_equation.hpp"
+#include "full_code/NSVE_field_stats.hpp"
+
+template <typename rnumber>
+class pressure_stats: public NSVE_field_stats<rnumber>
+{
+ public:
+ hid_t stat_file;
+ kspace<FFTW, SMOOTH> *kk;
+ vorticity_equation<rnumber, FFTW> *ve;
+
+ kspace<FFTW, ONE_HALF> *kk4;
+ kspace<FFTW, TWO_THIRDS> *kk3;
+
+ field<rnumber, FFTW, ONE> *pressure;
+ field<rnumber, FFTW, THREE> *nabla_p;
+ field<rnumber, FFTW, THREExTHREE> *nabla_u;
+ field<rnumber, FFTW, THREExTHREE> *Hessian_p;
+ field<rnumber, FFTW, ONE> *scalar;
+
+ int checkpoints_per_file;
+ int niter_out;
+ double max_pressure_estimate;
+
+ pressure_stats(
+ const MPI_Comm COMMUNICATOR,
+ const std::string &simulation_name):
+ NSVE_field_stats<rnumber>(
+ COMMUNICATOR,
+ simulation_name){}
+ virtual ~pressure_stats() noexcept(false){}
+
+ int initialize(void);
+ int work_on_current_iteration(void);
+ int finalize(void);
+};
+
+#endif//PRESSURE_STATS_HPP
+
diff --git a/cpp/full_code/resize.cpp b/cpp/full_code/resize.cpp
new file mode 100644
index 0000000000000000000000000000000000000000..12f125e026e9faf864b5d510d59371390d5874e9
--- /dev/null
+++ b/cpp/full_code/resize.cpp
@@ -0,0 +1,106 @@
+/******************************************************************************
+* *
+* Copyright 2019 Max Planck Institute for Dynamics and Self-Organization *
+* *
+* This file is part of TurTLE. *
+* *
+* TurTLE is free software: you can redistribute it and/or modify *
+* it under the terms of the GNU General Public License as published *
+* by the Free Software Foundation, either version 3 of the License, *
+* or (at your option) any later version. *
+* *
+* TurTLE is distributed in the hope that it will be useful, *
+* but WITHOUT ANY WARRANTY; without even the implied warranty of *
+* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the *
+* GNU General Public License for more details. *
+* *
+* You should have received a copy of the GNU General Public License *
+* along with TurTLE. If not, see <http://www.gnu.org/licenses/> *
+* *
+* Contact: Cristian.Lalescu@ds.mpg.de *
+* *
+******************************************************************************/
+
+
+
+#include <string>
+#include <cmath>
+#include "resize.hpp"
+#include "scope_timer.hpp"
+#include "hdf5_tools.hpp"
+
+
+template <typename rnumber>
+int resize<rnumber>::initialize(void)
+{
+ TIMEZONE("resize::initialize");
+ this->NSVE_field_stats<rnumber>::initialize();
+ DEBUG_MSG("after NSVE_field_stats::initialize\n");
+ hid_t parameter_file = H5Fopen(
+ (this->simname + std::string(".h5")).c_str(),
+ H5F_ACC_RDONLY,
+ H5P_DEFAULT);
+
+ this->niter_out = hdf5_tools::read_value<int>(
+ parameter_file, "/parameters/niter_out");
+ H5Fclose(parameter_file);
+ // the following ensures the file is free for rank 0 to open in read/write mode
+ MPI_Barrier(this->comm);
+ parameter_file = H5Fopen(
+ (this->simname + std::string("_post.h5")).c_str(),
+ H5F_ACC_RDONLY,
+ H5P_DEFAULT);
+ DEBUG_MSG("before read_vector\n");
+ this->iteration_list = hdf5_tools::read_vector<int>(
+ parameter_file,
+ "/resize/parameters/iteration_list");
+
+ this->new_nx = hdf5_tools::read_value<int>(
+ parameter_file, "/resize/parameters/new_nx");
+ this->new_ny = hdf5_tools::read_value<int>(
+ parameter_file, "/resize/parameters/new_ny");
+ this->new_nz = hdf5_tools::read_value<int>(
+ parameter_file, "/resize/parameters/new_nz");
+ this->new_simname = hdf5_tools::read_string(
+ parameter_file, "/resize/parameters/new_simname");
+ H5Fclose(parameter_file);
+ // the following ensures the file is free for rank 0 to open in read/write mode
+ MPI_Barrier(this->comm);
+
+ this->new_field = new field<rnumber, FFTW, THREE>(
+ this->new_nx, this->new_ny, this->new_nz,
+ this->comm,
+ this->vorticity->fftw_plan_rigor);
+ return EXIT_SUCCESS;
+}
+
+template <typename rnumber>
+int resize<rnumber>::work_on_current_iteration(void)
+{
+ TIMEZONE("resize::work_on_current_iteration");
+ this->read_current_cvorticity();
+
+ std::string fname = (
+ this->new_simname +
+ std::string("_fields.h5"));
+ *this->new_field = *this->vorticity;
+ this->new_field->io(
+ fname,
+ "vorticity",
+ this->iteration,
+ false);
+ return EXIT_SUCCESS;
+}
+
+template <typename rnumber>
+int resize<rnumber>::finalize(void)
+{
+ TIMEZONE("resize::finalize");
+ delete this->new_field;
+ this->NSVE_field_stats<rnumber>::finalize();
+ return EXIT_SUCCESS;
+}
+
+template class resize<float>;
+template class resize<double>;
+
diff --git a/cpp/full_code/resize.hpp b/cpp/full_code/resize.hpp
new file mode 100644
index 0000000000000000000000000000000000000000..9123dd9c85dbc767222b03ef9c9291b3ff7d9896
--- /dev/null
+++ b/cpp/full_code/resize.hpp
@@ -0,0 +1,67 @@
+/**********************************************************************
+* *
+* Copyright 2017 Max Planck Institute *
+* for Dynamics and Self-Organization *
+* *
+* This file is part of TurTLE. *
+* *
+* TurTLE is free software: you can redistribute it and/or modify *
+* it under the terms of the GNU General Public License as published *
+* by the Free Software Foundation, either version 3 of the License, *
+* or (at your option) any later version. *
+* *
+* TurTLE is distributed in the hope that it will be useful, *
+* but WITHOUT ANY WARRANTY; without even the implied warranty of *
+* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the *
+* GNU General Public License for more details. *
+* *
+* You should have received a copy of the GNU General Public License *
+* along with TurTLE. If not, see <http://www.gnu.org/licenses/> *
+* *
+* Contact: Cristian.Lalescu@ds.mpg.de *
+* *
+**********************************************************************/
+
+
+
+#ifndef RESIZE_HPP
+#define RESIZE_HPP
+
+#include <cstdlib>
+#include <sys/types.h>
+#include <sys/stat.h>
+#include <vector>
+#include "base.hpp"
+#include "field.hpp"
+#include "field_binary_IO.hpp"
+#include "full_code/NSVE_field_stats.hpp"
+
+template <typename rnumber>
+class resize: public NSVE_field_stats<rnumber>
+{
+ public:
+ std::string new_simname;
+
+ int new_nx;
+ int new_ny;
+ int new_nz;
+
+ int niter_out;
+
+ field<rnumber, FFTW, THREE> *new_field;
+
+ resize(
+ const MPI_Comm COMMUNICATOR,
+ const std::string &simulation_name):
+ NSVE_field_stats<rnumber>(
+ COMMUNICATOR,
+ simulation_name){}
+ virtual ~resize(){}
+
+ int initialize(void);
+ int work_on_current_iteration(void);
+ int finalize(void);
+};
+
+#endif//RESIZE_HPP
+
diff --git a/cpp/full_code/static_field.cpp b/cpp/full_code/static_field.cpp
new file mode 100644
index 0000000000000000000000000000000000000000..1fa52b646deb48f53b2066440221e10b01c3ab55
--- /dev/null
+++ b/cpp/full_code/static_field.cpp
@@ -0,0 +1,246 @@
+/******************************************************************************
+* *
+* Copyright 2019 Max Planck Institute for Dynamics and Self-Organization *
+* *
+* This file is part of TurTLE. *
+* *
+* TurTLE is free software: you can redistribute it and/or modify *
+* it under the terms of the GNU General Public License as published *
+* by the Free Software Foundation, either version 3 of the License, *
+* or (at your option) any later version. *
+* *
+* TurTLE is distributed in the hope that it will be useful, *
+* but WITHOUT ANY WARRANTY; without even the implied warranty of *
+* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the *
+* GNU General Public License for more details. *
+* *
+* You should have received a copy of the GNU General Public License *
+* along with TurTLE. If not, see <http://www.gnu.org/licenses/> *
+* *
+* Contact: Cristian.Lalescu@ds.mpg.de *
+* *
+******************************************************************************/
+
+
+
+#include <string>
+#include <cmath>
+#include "static_field.hpp"
+#include "scope_timer.hpp"
+#include "fftw_tools.hpp"
+
+
+template <typename rnumber>
+int static_field<rnumber>::initialize(void)
+{
+ TIMEZONE("static_file::initialize");
+ this->read_iteration();
+ this->read_parameters();
+ if (this->myrank == 0)
+ {
+ // set caching parameters
+ hid_t fapl = H5Pcreate(H5P_FILE_ACCESS);
+ herr_t cache_err = H5Pset_cache(fapl, 0, 521, 134217728, 1.0);
+ variable_used_only_in_assert(cache_err);
+ DEBUG_MSG("when setting stat_file cache I got %d\n", cache_err);
+ this->stat_file = H5Fopen(
+ (this->simname + ".h5").c_str(),
+ H5F_ACC_RDWR,
+ fapl);
+ }
+ int data_file_problem;
+ if (this->myrank == 0)
+ data_file_problem = this->grow_file_datasets();
+ MPI_Bcast(&data_file_problem, 1, MPI_INT, 0, this->comm);
+ if (data_file_problem > 0)
+ {
+ std::cerr <<
+ data_file_problem <<
+ " problems growing file datasets.\ntrying to exit now." <<
+ std::endl;
+ return EXIT_FAILURE;
+ }
+
+ this->vorticity = new field<rnumber, FFTW, THREE>(
+ nx, ny, nz,
+ this->comm,
+ fftw_planner_string_to_flag[this->fftw_plan_rigor]);
+ this->vorticity->real_space_representation = false;
+
+ this->velocity = new field<rnumber, FFTW, THREE>(
+ nx, ny, nz,
+ this->comm,
+ fftw_planner_string_to_flag[this->fftw_plan_rigor]);
+ this->velocity->real_space_representation = false;
+ //read static vorticity field from iteration 0
+ std::string checkpoint_fname = (
+ std::string(this->simname) +
+ std::string("_checkpoint_") +
+ std::to_string(0) +
+ std::string(".h5"));
+ this->vorticity->io(
+ checkpoint_fname,
+ "vorticity",
+ 0,
+ true);
+ this->kk = new kspace<FFTW, SMOOTH>(
+ this->vorticity->clayout, this->dkx, this->dky, this->dkz);
+
+ // compute the velocity field and store
+ invert_curl(
+ this->kk,
+ this->vorticity,
+ velocity);
+ // transform velocity and vorticity fields to real space
+ this->velocity->ift();
+ this->vorticity->ift();
+
+ // initialize particles
+ this->ps = particles_system_builder(
+ this->velocity, // (field object)
+ this->kk, // (kspace object, contains dkx, dky, dkz)
+ tracers0_integration_steps, // to check coherency between parameters and hdf input file (nb rhs)
+ (long long int)nparticles, // to check coherency between parameters and hdf input file
+ this->get_current_fname(), // particles input filename
+ std::string("/tracers0/state/") + std::to_string(this->iteration), // dataset name for initial input
+ std::string("/tracers0/rhs/") + std::to_string(this->iteration), // dataset name for initial input
+ tracers0_neighbours, // parameter (interpolation no neighbours)
+ tracers0_smoothness, // parameter
+ this->comm,
+ this->iteration+1);
+ // initialize output objects
+ this->particles_output_writer_mpi = new particles_output_hdf5<
+ long long int, double, 3>(
+ MPI_COMM_WORLD,
+ "tracers0",
+ nparticles,
+ tracers0_integration_steps);
+ this->particles_output_writer_mpi->setParticleFileLayout(this->ps->getParticleFileLayout());
+ this->particles_sample_writer_mpi = new particles_output_sampling_hdf5<
+ long long int, double, double, 3>(
+ MPI_COMM_WORLD,
+ this->ps->getGlobalNbParticles(),
+ (this->simname + "_particles.h5"),
+ "tracers0",
+ "position/0");
+ this->particles_sample_writer_mpi->setParticleFileLayout(this->ps->getParticleFileLayout());
+
+ return EXIT_SUCCESS;
+}
+
+template <typename rnumber>
+int static_field<rnumber>::step(void)
+{
+ TIMEZONE("static_file::step");
+ this->ps->completeLoop(this->dt);
+ this->iteration++;
+ return EXIT_SUCCESS;
+}
+
+template <typename rnumber>
+int static_field<rnumber>::write_checkpoint(void)
+{
+ TIMEZONE("static_file::write_checkpoint");
+ this->particles_output_writer_mpi->open_file(this->get_current_fname());
+ this->particles_output_writer_mpi->template save<3>(
+ this->ps->getParticlesState(),
+ this->ps->getParticlesRhs(),
+ this->ps->getParticlesIndexes(),
+ this->ps->getLocalNbParticles(),
+ this->iteration);
+ this->particles_output_writer_mpi->close_file();
+ this->write_iteration();
+ return EXIT_SUCCESS;
+}
+
+template <typename rnumber>
+int static_field<rnumber>::finalize(void)
+{
+ TIMEZONE("static_field::finalize");
+ if (this->myrank == 0)
+ H5Fclose(this->stat_file);
+ // good practice rule number n+1: always delete in inverse order of allocation
+ delete this->ps.release();
+ delete this->particles_output_writer_mpi;
+ delete this->particles_sample_writer_mpi;
+ delete this->kk;
+ delete this->velocity;
+ delete this->vorticity;
+ return EXIT_SUCCESS;
+}
+
+/** \brief Compute statistics.
+ *
+ */
+
+template <typename rnumber>
+int static_field<rnumber>::do_stats()
+{
+ TIMEZONE("static_field::do_stats");
+ /// either one of two conditions suffices to compute statistics:
+ /// 1) current iteration is a multiple of niter_part
+ /// 2) we are within niter_part_fine_duration/2 of a multiple of niter_part_fine_period
+ if (!(this->iteration % this->niter_part == 0 ||
+ ((this->iteration + this->niter_part_fine_duration/2) % this->niter_part_fine_period <=
+ this->niter_part_fine_duration)))
+ return EXIT_SUCCESS;
+
+ // allocate temporary data array
+ std::unique_ptr<double[]> pdata(new double[3*this->ps->getLocalNbParticles()]);
+
+ /// copy position data
+
+ /// sample position
+ std::copy(this->ps->getParticlesState(),
+ this->ps->getParticlesState()+3*this->ps->getLocalNbParticles(),
+ pdata.get());
+ this->particles_sample_writer_mpi->template save_dataset<3>(
+ "tracers0",
+ "position",
+ this->ps->getParticlesState(),
+ &pdata,
+ this->ps->getParticlesIndexes(),
+ this->ps->getLocalNbParticles(),
+ this->ps->get_step_idx()-1);
+
+ /// sample velocity
+ std::fill_n(pdata.get(), 3*this->ps->getLocalNbParticles(), 0);
+ this->ps->sample_compute_field(*this->velocity, pdata.get());
+ this->particles_sample_writer_mpi->template save_dataset<3>(
+ "tracers0",
+ "velocity",
+ this->ps->getParticlesState(),
+ &pdata,
+ this->ps->getParticlesIndexes(),
+ this->ps->getLocalNbParticles(),
+ this->ps->get_step_idx()-1);
+
+ // deallocate temporary data array
+ delete[] pdata.release();
+
+ return EXIT_SUCCESS;
+}
+
+template <typename rnumber>
+int static_field<rnumber>::read_parameters(void)
+{
+ TIMEZONE("static_field::read_parameters");
+ this->direct_numerical_simulation::read_parameters();
+ hid_t parameter_file = H5Fopen((this->simname + ".h5").c_str(), H5F_ACC_RDONLY, H5P_DEFAULT);
+ this->fftw_plan_rigor = hdf5_tools::read_string(parameter_file, "parameters/fftw_plan_rigor");
+ this->dt = hdf5_tools::read_value<double>(parameter_file, "parameters/dt");
+ this->niter_part = hdf5_tools::read_value<int>(parameter_file, "parameters/niter_part");
+ this->niter_part_fine_period = hdf5_tools::read_value<int>(parameter_file, "parameters/niter_part_fine_period");
+ this->niter_part_fine_duration = hdf5_tools::read_value<int>(parameter_file, "parameters/niter_part_fine_duration");
+ this->nparticles = hdf5_tools::read_value<long long int>(parameter_file, "parameters/nparticles");
+ this->tracers0_integration_steps = hdf5_tools::read_value<int>(parameter_file, "parameters/tracers0_integration_steps");
+ this->tracers0_neighbours = hdf5_tools::read_value<int>(parameter_file, "parameters/tracers0_neighbours");
+ this->tracers0_smoothness = hdf5_tools::read_value<int>(parameter_file, "parameters/tracers0_smoothness");
+ H5Fclose(parameter_file);
+ MPI_Barrier(this->comm);
+ return EXIT_SUCCESS;
+}
+
+template class static_field<float>;
+template class static_field<double>;
+
diff --git a/cpp/full_code/static_field.hpp b/cpp/full_code/static_field.hpp
new file mode 100644
index 0000000000000000000000000000000000000000..90b563d1cd777768a63591e78ef121c9a594fc27
--- /dev/null
+++ b/cpp/full_code/static_field.hpp
@@ -0,0 +1,89 @@
+/**********************************************************************
+* *
+* Copyright 2017 Max Planck Institute *
+* for Dynamics and Self-Organization *
+* *
+* This file is part of TurTLE. *
+* *
+* TurTLE is free software: you can redistribute it and/or modify *
+* it under the terms of the GNU General Public License as published *
+* by the Free Software Foundation, either version 3 of the License, *
+* or (at your option) any later version. *
+* *
+* TurTLE is distributed in the hope that it will be useful, *
+* but WITHOUT ANY WARRANTY; without even the implied warranty of *
+* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the *
+* GNU General Public License for more details. *
+* *
+* You should have received a copy of the GNU General Public License *
+* along with TurTLE. If not, see <http://www.gnu.org/licenses/> *
+* *
+* Contact: Cristian.Lalescu@ds.mpg.de *
+* *
+**********************************************************************/
+
+
+
+#ifndef STATIC_FIELD_HPP
+#define STATIC_FIELD_HPP
+
+
+
+#include <cstdlib>
+#include "base.hpp"
+#include "kspace.hpp"
+#include "field.hpp"
+#include "full_code/direct_numerical_simulation.hpp"
+#include "particles/particles_system_builder.hpp"
+#include "particles/particles_output_hdf5.hpp"
+#include "particles/particles_sampling.hpp"
+
+template <typename rnumber>
+class static_field: public direct_numerical_simulation
+{
+ public:
+
+ /* parameters that are read in read_parameters */
+ double dt;
+ std::string fftw_plan_rigor;
+
+ /* particle parameters */
+ int niter_part;
+ int niter_part_fine_period;
+ int niter_part_fine_duration;
+ long long int nparticles;
+ int tracers0_integration_steps;
+ int tracers0_neighbours;
+ int tracers0_smoothness;
+
+ /* other stuff */
+ kspace<FFTW, SMOOTH> *kk;
+ field<rnumber, FFTW, THREE> *vorticity;
+ field<rnumber, FFTW, THREE> *velocity;
+
+ /* other stuff */
+ std::unique_ptr<abstract_particles_system<long long int, double>> ps;
+
+ particles_output_hdf5<long long int, double,3> *particles_output_writer_mpi;
+ particles_output_sampling_hdf5<long long int, double, double, 3> *particles_sample_writer_mpi;
+
+
+ static_field(
+ const MPI_Comm COMMUNICATOR,
+ const std::string &simulation_name):
+ direct_numerical_simulation(
+ COMMUNICATOR,
+ simulation_name){}
+ ~static_field() noexcept(false){}
+
+ int initialize(void);
+ int step(void);
+ int finalize(void);
+
+ virtual int read_parameters(void);
+ int write_checkpoint(void);
+ int do_stats(void);
+};
+
+#endif//STATIC_FIELD_HPP
+
diff --git a/cpp/full_code/symmetrize_test.cpp b/cpp/full_code/symmetrize_test.cpp
new file mode 100644
index 0000000000000000000000000000000000000000..904b57ae2e32004c97f0ca1046e450822f853809
--- /dev/null
+++ b/cpp/full_code/symmetrize_test.cpp
@@ -0,0 +1,297 @@
+/******************************************************************************
+* *
+* Copyright 2019 Max Planck Institute for Dynamics and Self-Organization *
+* *
+* This file is part of TurTLE. *
+* *
+* TurTLE is free software: you can redistribute it and/or modify *
+* it under the terms of the GNU General Public License as published *
+* by the Free Software Foundation, either version 3 of the License, *
+* or (at your option) any later version. *
+* *
+* TurTLE is distributed in the hope that it will be useful, *
+* but WITHOUT ANY WARRANTY; without even the implied warranty of *
+* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the *
+* GNU General Public License for more details. *
+* *
+* You should have received a copy of the GNU General Public License *
+* along with TurTLE. If not, see <http://www.gnu.org/licenses/> *
+* *
+* Contact: Cristian.Lalescu@ds.mpg.de *
+* *
+******************************************************************************/
+
+
+
+#include <string>
+#include <cmath>
+#include "symmetrize_test.hpp"
+#include "fftw_tools.hpp"
+#include "scope_timer.hpp"
+#include "hdf5_tools.hpp"
+
+
+template <typename rnumber>
+int symmetrize_test<rnumber>::initialize(void)
+{
+ TIMEZONE("symmetrize_test::initialize");
+ this->read_parameters();
+ return EXIT_SUCCESS;
+}
+
+template <typename rnumber>
+int symmetrize_test<rnumber>::finalize(void)
+{
+ TIMEZONE("symmetrize_test::finalize");
+ return EXIT_SUCCESS;
+}
+
+template <typename rnumber>
+int symmetrize_test<rnumber>::read_parameters()
+{
+ TIMEZONE("symmetrize_test::read_parameters");
+ this->test::read_parameters();
+ hid_t parameter_file = H5Fopen(
+ (this->simname + std::string(".h5")).c_str(),
+ H5F_ACC_RDONLY,
+ H5P_DEFAULT);
+ this->random_seed = hdf5_tools::read_value<int>(
+ parameter_file, "/parameters/random_seed");
+ this->fftw_plan_rigor = hdf5_tools::read_string(parameter_file, "parameters/fftw_plan_rigor");
+ H5Fclose(parameter_file);
+ MPI_Barrier(this->comm);
+ return EXIT_SUCCESS;
+}
+
+/** \brief Sanity check for symmetrize
+ *
+ * Generate a random field, apply symmetrize to it, call the result f0.
+ * Make a copy, and perform an ift+dft+normalize, call the result f1.
+ * Compare f0 with f1, they should be fairly close. Since we know that
+ * ift+dft+normalize only changes the field within numerical precision
+ * if it's properly Hermitian, then any difference between f0 and f1 is
+ * due to our own symmetrize being broken somehow.
+ * */
+
+template <typename rnumber>
+int symmetrize_test<rnumber>::do_work(void)
+{
+ TIMEZONE("symmetrize_test::do_work");
+ // allocate
+ DEBUG_MSG("about to allocate field0\n");
+ field<rnumber, FFTW, THREE> *test_field0 = new field<rnumber, FFTW, THREE>(
+ this->nx, this->ny, this->nz,
+ this->comm,
+ fftw_planner_string_to_flag[this->fftw_plan_rigor]);
+ DEBUG_MSG("finished allocating field0\n");
+ DEBUG_MSG("about to allocate field1\n");
+ field<rnumber, FFTW, THREE> *test_field1 = new field<rnumber, FFTW, THREE>(
+ this->nx, this->ny, this->nz,
+ this->comm,
+ fftw_planner_string_to_flag[this->fftw_plan_rigor]);
+ DEBUG_MSG("finished allocating field1\n");
+ kspace<FFTW,SMOOTH> *kk = new kspace<FFTW, SMOOTH>(
+ test_field0->clayout, this->dkx, this->dky, this->dkz);
+
+ if (this->myrank == 0)
+ {
+ hid_t stat_file = H5Fopen(
+ (this->simname + std::string(".h5")).c_str(),
+ H5F_ACC_RDWR,
+ H5P_DEFAULT);
+ kk->store(stat_file);
+ H5Fclose(stat_file);
+ }
+
+ // compute minimum available lengthscale
+ int max_n = this->nx;
+ if (this->ny > max_n)
+ max_n = this->ny;
+ if (this->nz > max_n)
+ max_n = this->nz;
+ double min_lengthscale = 8*acos(0) / max_n;
+ /*************************************************
+ * here we test that after applying symmetrize
+ * field is actually Hermitian-symmetric
+ *************************************************/
+ // fill up test_field0
+ *test_field0 = rnumber(0.0);
+ make_gaussian_random_field<rnumber, FFTW, THREE, SMOOTH>(
+ kk,
+ test_field0,
+ 1,
+ 0.4,
+ 1.0,
+ min_lengthscale);
+ // make the field divergence free
+ kk->template force_divfree<rnumber>(test_field0->get_cdata());
+ // apply symmetrize to test_field0
+ test_field0->symmetrize();
+
+
+ // make copy in test_field1
+ // this MUST be made after symmetrizing test_field0
+ // (alternatively, we may symmetrize test_field1 as well before the ift-dft cycle
+ *test_field1 = test_field0->get_cdata();
+
+ // go back and forth with test_field1, to enforce symmetry
+ test_field1->symmetrize_FFT();
+
+ // temporary variables for field comparison
+ double max_diff;
+ ptrdiff_t ix, iy, iz;
+ double k_at_max_diff;
+ double a0, a1;
+
+ // now compare the two fields
+ max_diff = 0;
+ k_at_max_diff = 0;
+ kk->CLOOP_K2(
+ [&](ptrdiff_t cindex,
+ ptrdiff_t xindex,
+ ptrdiff_t yindex,
+ ptrdiff_t zindex,
+ double k2){
+ double diff_re0 = test_field0->cval(cindex, 0, 0) - test_field1->cval(cindex, 0, 0);
+ double diff_re1 = test_field0->cval(cindex, 1, 0) - test_field1->cval(cindex, 1, 0);
+ double diff_re2 = test_field0->cval(cindex, 2, 0) - test_field1->cval(cindex, 2, 0);
+ double diff_im0 = test_field0->cval(cindex, 0, 1) - test_field1->cval(cindex, 0, 1);
+ double diff_im1 = test_field0->cval(cindex, 1, 1) - test_field1->cval(cindex, 1, 1);
+ double diff_im2 = test_field0->cval(cindex, 2, 1) - test_field1->cval(cindex, 2, 1);
+ double diff = sqrt(diff_re0*diff_re0 + diff_re1*diff_re1 + diff_re2*diff_re2 +
+ diff_im0*diff_im0 + diff_im1*diff_im1 + diff_im2*diff_im2);
+ double amplitude0 = (test_field0->cval(cindex, 0, 0)*test_field0->cval(cindex, 0, 0) +
+ test_field0->cval(cindex, 1, 0)*test_field0->cval(cindex, 1, 0) +
+ test_field0->cval(cindex, 2, 0)*test_field0->cval(cindex, 2, 0) +
+ test_field0->cval(cindex, 0, 1)*test_field0->cval(cindex, 0, 1) +
+ test_field0->cval(cindex, 1, 1)*test_field0->cval(cindex, 1, 1) +
+ test_field0->cval(cindex, 2, 1)*test_field0->cval(cindex, 2, 1));
+ double amplitude1 = (test_field1->cval(cindex, 0, 0)*test_field1->cval(cindex, 0, 0) +
+ test_field1->cval(cindex, 1, 0)*test_field1->cval(cindex, 1, 0) +
+ test_field1->cval(cindex, 2, 0)*test_field1->cval(cindex, 2, 0) +
+ test_field1->cval(cindex, 0, 1)*test_field1->cval(cindex, 0, 1) +
+ test_field1->cval(cindex, 1, 1)*test_field1->cval(cindex, 1, 1) +
+ test_field1->cval(cindex, 2, 1)*test_field1->cval(cindex, 2, 1));
+ double amplitude = sqrt((amplitude0 + amplitude1)/2);
+ if (amplitude > 0)
+ if (diff/amplitude > max_diff)
+ {
+ max_diff = diff / amplitude;
+ ix = xindex;
+ iy = yindex + test_field0->clayout->starts[0];
+ iz = zindex;
+ k_at_max_diff = sqrt(k2);
+ a0 = sqrt(amplitude0);
+ a1 = sqrt(amplitude1);
+ }
+ });
+ DEBUG_MSG("rseed1 found maximum relative difference %g at ix = %ld, iy = %ld, iz = %ld, wavenumber = %g, amplitudes %g %g\n",
+ max_diff, ix, iy, iz, k_at_max_diff, a0, a1);
+
+ test_field0->io(
+ this->simname + "_fields.h5",
+ "field0",
+ 0,
+ false);
+ test_field1->io(
+ this->simname + "_fields.h5",
+ "field1",
+ 0,
+ false);
+ test_field1->ift();
+ test_field1->io(
+ this->simname + "_fields.h5",
+ "field1",
+ 0,
+ false);
+
+ /*************************************************
+ * here we test that applying our symmetrize, or
+ * just using fft, leads to the same result.
+ * there's no guarantee that this will work though,
+ * since FFT documentation doesn't say IFT+DFT+normalize leads to
+ * arithmetic mean-based Hermitian symmetry.
+ *************************************************/
+
+ make_gaussian_random_field<rnumber, FFTW, THREE, SMOOTH>(
+ kk,
+ test_field0,
+ 2,
+ 0.4,
+ 1.0,
+ min_lengthscale);
+
+ // copy unmodified data
+ *test_field1 = test_field0->get_cdata();
+
+ // apply different symmetrize methods
+ test_field0->symmetrize();
+ test_field1->symmetrize_FFT();
+
+ // now compare the two fields
+ max_diff = 0;
+ k_at_max_diff = 0;
+ kk->CLOOP_K2(
+ [&](ptrdiff_t cindex,
+ ptrdiff_t xindex,
+ ptrdiff_t yindex,
+ ptrdiff_t zindex,
+ double k2){
+ double diff_re0 = test_field0->cval(cindex, 0, 0) - test_field1->cval(cindex, 0, 0);
+ double diff_re1 = test_field0->cval(cindex, 1, 0) - test_field1->cval(cindex, 1, 0);
+ double diff_re2 = test_field0->cval(cindex, 2, 0) - test_field1->cval(cindex, 2, 0);
+ double diff_im0 = test_field0->cval(cindex, 0, 1) - test_field1->cval(cindex, 0, 1);
+ double diff_im1 = test_field0->cval(cindex, 1, 1) - test_field1->cval(cindex, 1, 1);
+ double diff_im2 = test_field0->cval(cindex, 2, 1) - test_field1->cval(cindex, 2, 1);
+ double diff = sqrt(diff_re0*diff_re0 + diff_re1*diff_re1 + diff_re2*diff_re2 +
+ diff_im0*diff_im0 + diff_im1*diff_im1 + diff_im2*diff_im2);
+ double amplitude0 = (test_field0->cval(cindex, 0, 0)*test_field0->cval(cindex, 0, 0) +
+ test_field0->cval(cindex, 1, 0)*test_field0->cval(cindex, 1, 0) +
+ test_field0->cval(cindex, 2, 0)*test_field0->cval(cindex, 2, 0) +
+ test_field0->cval(cindex, 0, 1)*test_field0->cval(cindex, 0, 1) +
+ test_field0->cval(cindex, 1, 1)*test_field0->cval(cindex, 1, 1) +
+ test_field0->cval(cindex, 2, 1)*test_field0->cval(cindex, 2, 1));
+ double amplitude1 = (test_field1->cval(cindex, 0, 0)*test_field1->cval(cindex, 0, 0) +
+ test_field1->cval(cindex, 1, 0)*test_field1->cval(cindex, 1, 0) +
+ test_field1->cval(cindex, 2, 0)*test_field1->cval(cindex, 2, 0) +
+ test_field1->cval(cindex, 0, 1)*test_field1->cval(cindex, 0, 1) +
+ test_field1->cval(cindex, 1, 1)*test_field1->cval(cindex, 1, 1) +
+ test_field1->cval(cindex, 2, 1)*test_field1->cval(cindex, 2, 1));
+ double amplitude = sqrt((amplitude0 + amplitude1)/2);
+ if (amplitude > 0)
+ if (diff/amplitude > max_diff)
+ {
+ max_diff = diff / amplitude;
+ ix = xindex;
+ iy = yindex + test_field0->clayout->starts[0];
+ iz = zindex;
+ k_at_max_diff = sqrt(k2);
+ a0 = sqrt(amplitude0);
+ a1 = sqrt(amplitude1);
+ }
+ });
+ DEBUG_MSG("rseed2 found maximum relative difference %g at ix = %ld, iy = %ld, iz = %ld, wavenumber = %g, amplitudes %g %g\n",
+ max_diff, ix, iy, iz, k_at_max_diff, a0, a1);
+
+ // output fields for possible subsequent comparison
+ test_field0->io(
+ this->simname + "_fields.h5",
+ "rseed2_field0",
+ 0,
+ false);
+ test_field1->io(
+ this->simname + "_fields.h5",
+ "rseed2_field1",
+ 0,
+ false);
+
+ // deallocate
+ delete kk;
+ delete test_field1;
+ delete test_field0;
+ return EXIT_SUCCESS;
+}
+
+template class symmetrize_test<float>;
+template class symmetrize_test<double>;
+
diff --git a/cpp/full_code/symmetrize_test.hpp b/cpp/full_code/symmetrize_test.hpp
new file mode 100644
index 0000000000000000000000000000000000000000..69443d1a8dbeee28d035c5c2364f2d79dce8f8af
--- /dev/null
+++ b/cpp/full_code/symmetrize_test.hpp
@@ -0,0 +1,63 @@
+/**********************************************************************
+* *
+* Copyright 2018 Max Planck Institute *
+* for Dynamics and Self-Organization *
+* *
+* This file is part of TurTLE. *
+* *
+* TurTLE is free software: you can redistribute it and/or modify *
+* it under the terms of the GNU General Public License as published *
+* by the Free Software Foundation, either version 3 of the License, *
+* or (at your option) any later version. *
+* *
+* TurTLE is distributed in the hope that it will be useful, *
+* but WITHOUT ANY WARRANTY; without even the implied warranty of *
+* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the *
+* GNU General Public License for more details. *
+* *
+* You should have received a copy of the GNU General Public License *
+* along with TurTLE. If not, see <http://www.gnu.org/licenses/> *
+* *
+* Contact: Cristian.Lalescu@ds.mpg.de *
+* *
+**********************************************************************/
+
+
+
+#ifndef SYMMETRIZE_TEST_HPP
+#define SYMMETRIZE_TEST_HPP
+
+
+
+#include <cstdlib>
+#include "base.hpp"
+#include "kspace.hpp"
+#include "field.hpp"
+#include "full_code/test.hpp"
+
+/** \brief A class for testing basic field class functionality.
+ */
+
+template <typename rnumber>
+class symmetrize_test: public test
+{
+ public:
+ std::string fftw_plan_rigor;
+ int random_seed;
+
+ symmetrize_test(
+ const MPI_Comm COMMUNICATOR,
+ const std::string &simulation_name):
+ test(
+ COMMUNICATOR,
+ simulation_name){}
+ ~symmetrize_test() noexcept(false){}
+
+ int initialize(void);
+ int do_work(void);
+ int finalize(void);
+ int read_parameters(void);
+};
+
+#endif//SYMMETRIZE_TEST_HPP
+
diff --git a/cpp/full_code/test.cpp b/cpp/full_code/test.cpp
new file mode 100644
index 0000000000000000000000000000000000000000..2724bc7215244841082dfb562a4caba3301fe132
--- /dev/null
+++ b/cpp/full_code/test.cpp
@@ -0,0 +1,43 @@
+/******************************************************************************
+* *
+* Copyright 2019 Max Planck Institute for Dynamics and Self-Organization *
+* *
+* This file is part of TurTLE. *
+* *
+* TurTLE is free software: you can redistribute it and/or modify *
+* it under the terms of the GNU General Public License as published *
+* by the Free Software Foundation, either version 3 of the License, *
+* or (at your option) any later version. *
+* *
+* TurTLE is distributed in the hope that it will be useful, *
+* but WITHOUT ANY WARRANTY; without even the implied warranty of *
+* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the *
+* GNU General Public License for more details. *
+* *
+* You should have received a copy of the GNU General Public License *
+* along with TurTLE. If not, see <http://www.gnu.org/licenses/> *
+* *
+* Contact: Cristian.Lalescu@ds.mpg.de *
+* *
+******************************************************************************/
+
+
+
+#include <cstdlib>
+#include <sys/types.h>
+#include <sys/stat.h>
+#include "scope_timer.hpp"
+#include "hdf5_tools.hpp"
+#include "full_code/test.hpp"
+
+
+int test::main_loop(void)
+{
+ TIMEZONE("test::main_loop");
+ this->start_simple_timer();
+ this->do_work();
+ this->print_simple_timer(
+ "do_work required ");
+ return EXIT_SUCCESS;
+}
+
diff --git a/bfps/cpp/full_code/test.hpp b/cpp/full_code/test.hpp
similarity index 86%
rename from bfps/cpp/full_code/test.hpp
rename to cpp/full_code/test.hpp
index 134a01512b3fd836a8ac4d40068b3954752c4844..f7f9d8eeae30246d993edc75d5d5196b9ee5dabd 100644
--- a/bfps/cpp/full_code/test.hpp
+++ b/cpp/full_code/test.hpp
@@ -3,20 +3,20 @@
* Copyright 2017 Max Planck Institute *
* for Dynamics and Self-Organization *
* *
-* This file is part of bfps. *
+* This file is part of TurTLE. *
* *
-* bfps is free software: you can redistribute it and/or modify *
+* TurTLE is free software: you can redistribute it and/or modify *
* it under the terms of the GNU General Public License as published *
* by the Free Software Foundation, either version 3 of the License, *
* or (at your option) any later version. *
* *
-* bfps is distributed in the hope that it will be useful, *
+* TurTLE is distributed in the hope that it will be useful, *
* but WITHOUT ANY WARRANTY; without even the implied warranty of *
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the *
* GNU General Public License for more details. *
* *
* You should have received a copy of the GNU General Public License *
-* along with bfps. If not, see <http://www.gnu.org/licenses/> *
+* along with TurTLE. If not, see <http://www.gnu.org/licenses/> *
* *
* Contact: Cristian.Lalescu@ds.mpg.de *
* *
@@ -56,7 +56,6 @@ class test: public code_base
virtual int finalize(void) = 0;
int main_loop(void);
- virtual int read_parameters(void);
};
#endif//TEST_HPP
diff --git a/cpp/full_code/test_interpolation.cpp b/cpp/full_code/test_interpolation.cpp
new file mode 100644
index 0000000000000000000000000000000000000000..9b9f2f50b53d05205eef6ece4287f2e8f2b136a2
--- /dev/null
+++ b/cpp/full_code/test_interpolation.cpp
@@ -0,0 +1,235 @@
+/******************************************************************************
+* *
+* Copyright 2019 Max Planck Institute for Dynamics and Self-Organization *
+* *
+* This file is part of TurTLE. *
+* *
+* TurTLE is free software: you can redistribute it and/or modify *
+* it under the terms of the GNU General Public License as published *
+* by the Free Software Foundation, either version 3 of the License, *
+* or (at your option) any later version. *
+* *
+* TurTLE is distributed in the hope that it will be useful, *
+* but WITHOUT ANY WARRANTY; without even the implied warranty of *
+* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the *
+* GNU General Public License for more details. *
+* *
+* You should have received a copy of the GNU General Public License *
+* along with TurTLE. If not, see <http://www.gnu.org/licenses/> *
+* *
+* Contact: Cristian.Lalescu@ds.mpg.de *
+* *
+******************************************************************************/
+
+
+
+#include "full_code/test_interpolation.hpp"
+
+
+template <typename rnumber>
+int test_interpolation<rnumber>::read_parameters(void)
+{
+ TIMEZONE("test_interpolation::read_parameters");
+ this->test::read_parameters();
+ hid_t parameter_file = H5Fopen(
+ (this->simname + std::string(".h5")).c_str(),
+ H5F_ACC_RDONLY,
+ H5P_DEFAULT);
+ this->nparticles = hdf5_tools::read_value<long long int>(
+ parameter_file, "/parameters/nparticles");
+ this->tracers0_integration_steps = hdf5_tools::read_value<int>(
+ parameter_file, "/parameters/tracers0_integration_steps");
+ this->tracers0_neighbours = hdf5_tools::read_value<int>(
+ parameter_file, "/parameters/tracers0_neighbours");
+ this->tracers0_smoothness = hdf5_tools::read_value<int>(
+ parameter_file, "/parameters/tracers0_smoothness");
+ H5Fclose(parameter_file);
+ MPI_Barrier(this->comm);
+ return EXIT_SUCCESS;
+}
+
+template <typename rnumber>
+int test_interpolation<rnumber>::initialize(void)
+{
+ TIMEZONE("test_interpolation::initialize");
+ this->read_parameters();
+ this->vorticity = new field<rnumber, FFTW, THREE>(
+ this->nx, this->ny, this->nz,
+ this->comm,
+ FFTW_ESTIMATE);
+ this->vorticity->real_space_representation = false;
+
+ this->velocity = new field<rnumber, FFTW, THREE>(
+ this->nx, this->ny, this->nz,
+ this->comm,
+ FFTW_ESTIMATE);
+
+ this->nabla_u = new field<rnumber, FFTW, THREExTHREE>(
+ this->nx, this->ny, this->nz,
+ this->comm,
+ FFTW_ESTIMATE);
+
+ this->kk = new kspace<FFTW, SMOOTH>(
+ this->vorticity->clayout, this->dkx, this->dky, this->dkz);
+
+ if (this->myrank == 0)
+ {
+ hid_t stat_file = H5Fopen(
+ (this->simname + std::string(".h5")).c_str(),
+ H5F_ACC_RDWR,
+ H5P_DEFAULT);
+ this->kk->store(stat_file);
+ H5Fclose(stat_file);
+ }
+
+ this->ps = particles_system_builder(
+ this->velocity, // (field object)
+ this->kk, // (kspace object, contains dkx, dky, dkz)
+ this->tracers0_integration_steps, // to check coherency between parameters and hdf input file (nb rhs)
+ (long long int)nparticles, // to check coherency between parameters and hdf input file
+ this->simname + "_input.h5", // particles input filename
+ std::string("/tracers0/state/0"), // dataset name for initial input
+ std::string("/tracers0/rhs/0") , // dataset name for initial input
+ this->tracers0_neighbours, // parameter (interpolation no neighbours)
+ this->tracers0_smoothness, // parameter
+ this->comm,
+ 1);
+ this->particles_output_writer_mpi = new particles_output_hdf5<
+ long long int, double, 3>(
+ MPI_COMM_WORLD,
+ "tracers0",
+ nparticles,
+ this->tracers0_integration_steps);
+ this->particles_sample_writer_mpi = new particles_output_sampling_hdf5<
+ long long int, double, double, 3>(
+ MPI_COMM_WORLD,
+ this->ps->getGlobalNbParticles(),
+ (this->simname + "_output.h5"),
+ "tracers0",
+ "position/0");
+
+ return EXIT_SUCCESS;
+}
+
+template <typename rnumber>
+int test_interpolation<rnumber>::finalize(void)
+{
+ TIMEZONE("test_interpolation::finalize");
+ delete this->nabla_u;
+ delete this->velocity;
+ delete this->vorticity;
+ delete this->ps.release();
+ delete this->kk;
+ delete particles_output_writer_mpi;
+ delete particles_sample_writer_mpi;
+ return EXIT_SUCCESS;
+}
+
+template <typename rnumber>
+int test_interpolation<rnumber>::do_work()
+{
+ TIMEZONE("test_interpolation::do_work");
+ *this->nabla_u = 0.0;
+ this->velocity->real_space_representation = false;
+ this->vorticity->real_space_representation = false;
+ this->nabla_u->real_space_representation = false;
+ // read vorticity field
+ this->vorticity->io(
+ this->simname + std::string("_input.h5"),
+ "vorticity",
+ 0, true);
+ this->kk->template force_divfree<rnumber>(this->vorticity->get_cdata());
+
+ // compute velocity
+ invert_curl(this->kk, this->vorticity, this->velocity);
+
+ // compute velocity gradient
+ compute_gradient(this->kk, this->velocity, this->nabla_u);
+
+ // go to real space
+ this->vorticity->ift();
+ this->velocity->ift();
+ this->nabla_u->ift();
+
+ DEBUG_MSG("some vorticity values: %g %g %g\n",
+ this->vorticity->rval(20, 1),
+ this->vorticity->rval(200, 2),
+ this->vorticity->rval(741, 0));
+ DEBUG_MSG("corresponding velocity gradient to vorticity values: %g %g %g\n",
+ this->nabla_u->rval( 20, 2, 0) - this->nabla_u->rval( 20, 0, 2),
+ this->nabla_u->rval(200, 1, 0) - this->nabla_u->rval(200, 0, 1),
+ this->nabla_u->rval(741, 1, 2) - this->nabla_u->rval(741, 2, 1));
+
+ // allocate interpolation arrays
+ std::unique_ptr<double[]> p3data;
+ std::unique_ptr<double[]> p9data;
+ if(this->ps->getLocalNbParticles()){
+ p3data.reset(new double[3*this->ps->getLocalNbParticles()]);
+ p9data.reset(new double[9*this->ps->getLocalNbParticles()]);
+ }
+
+ /// sample position
+ std::copy(this->ps->getParticlesState(),
+ this->ps->getParticlesState()+3*this->ps->getLocalNbParticles(),
+ p3data.get());
+ this->particles_sample_writer_mpi->template save_dataset<3>(
+ "tracers0",
+ "position",
+ this->ps->getParticlesState(),
+ &p3data,
+ this->ps->getParticlesIndexes(),
+ this->ps->getLocalNbParticles(),
+ this->ps->get_step_idx()-1);
+
+ /// sample velocity at particles' position
+ std::fill_n(p3data.get(), 3*this->ps->getLocalNbParticles(), 0);
+ this->ps->sample_compute_field(*this->velocity, p3data.get());
+ if(p3data){
+ DEBUG_MSG("first vel value is %g\n", p3data.get()[0]);
+ }
+ this->particles_sample_writer_mpi->template save_dataset<3>(
+ "tracers0",
+ "velocity",
+ this->ps->getParticlesState(),
+ &p3data,
+ this->ps->getParticlesIndexes(),
+ this->ps->getLocalNbParticles(),
+ this->ps->get_step_idx()-1);
+ /// sample vorticity at particles' position
+ std::fill_n(p3data.get(), 3*this->ps->getLocalNbParticles(), 0);
+ this->ps->sample_compute_field(*this->vorticity, p3data.get());
+ if(p3data){
+ DEBUG_MSG("first vort value is %g\n", p3data.get()[0]);
+ }
+ this->particles_sample_writer_mpi->template save_dataset<3>(
+ "tracers0",
+ "vorticity",
+ this->ps->getParticlesState(),
+ &p3data,
+ this->ps->getParticlesIndexes(),
+ this->ps->getLocalNbParticles(),
+ this->ps->get_step_idx()-1);
+ /// sample velocity gradient at particles' position
+ std::fill_n(p9data.get(), 9*this->ps->getLocalNbParticles(), 0);
+ this->ps->sample_compute_field(*this->nabla_u, p9data.get());
+ if(p9data){
+ DEBUG_MSG("first vel gradient value is %g\n", p9data.get()[0]);
+ }
+ this->particles_sample_writer_mpi->template save_dataset<9>(
+ "tracers0",
+ "velocity_gradient",
+ this->ps->getParticlesState(),
+ &p9data,
+ this->ps->getParticlesIndexes(),
+ this->ps->getLocalNbParticles(),
+ this->ps->get_step_idx()-1);
+
+ // deallocate temporary arrays
+ delete[] p3data.release();
+ delete[] p9data.release();
+ return EXIT_SUCCESS;
+}
+
+template class test_interpolation<float>;
+template class test_interpolation<double>;
+
diff --git a/cpp/full_code/test_interpolation.hpp b/cpp/full_code/test_interpolation.hpp
new file mode 100644
index 0000000000000000000000000000000000000000..7c9e0fff1f1d286865fd1f29b62005063f141e75
--- /dev/null
+++ b/cpp/full_code/test_interpolation.hpp
@@ -0,0 +1,84 @@
+/******************************************************************************
+* *
+* Copyright 2019 Max Planck Institute for Dynamics and Self-Organization *
+* *
+* This file is part of TurTLE. *
+* *
+* TurTLE is free software: you can redistribute it and/or modify *
+* it under the terms of the GNU General Public License as published *
+* by the Free Software Foundation, either version 3 of the License, *
+* or (at your option) any later version. *
+* *
+* TurTLE is distributed in the hope that it will be useful, *
+* but WITHOUT ANY WARRANTY; without even the implied warranty of *
+* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the *
+* GNU General Public License for more details. *
+* *
+* You should have received a copy of the GNU General Public License *
+* along with TurTLE. If not, see <http://www.gnu.org/licenses/> *
+* *
+* Contact: Cristian.Lalescu@ds.mpg.de *
+* *
+******************************************************************************/
+
+
+
+#ifndef TEST_INTERPOLATION_HPP
+#define TEST_INTERPOLATION_HPP
+
+
+
+#include <cstdlib>
+#include "base.hpp"
+#include "kspace.hpp"
+#include "full_code/test.hpp"
+#include "particles/particles_system_builder.hpp"
+#include "particles/particles_output_hdf5.hpp"
+#include "particles/particles_sampling.hpp"
+
+/** \brief Interpolation tester.
+ *
+ */
+
+template <typename rnumber>
+class test_interpolation: public test
+{
+ public:
+ long long int nparticles;
+ int tracers0_integration_steps;
+ int tracers0_neighbours;
+ int tracers0_smoothness;
+
+ std::unique_ptr<abstract_particles_system<long long int, double>> ps;
+
+ particles_output_hdf5<long long int, double,3> *particles_output_writer_mpi;
+ particles_output_sampling_hdf5<long long int, double, double, 3> *particles_sample_writer_mpi;
+
+ field<rnumber, FFTW, THREE> *velocity, *vorticity;
+ field<rnumber, FFTW, THREExTHREE> *nabla_u;
+
+ kspace<FFTW, SMOOTH> *kk;
+
+ test_interpolation(
+ const MPI_Comm COMMUNICATOR,
+ const std::string &simulation_name):
+ test(
+ COMMUNICATOR,
+ simulation_name),
+ particles_output_writer_mpi(nullptr),
+ particles_sample_writer_mpi(nullptr),
+ velocity(nullptr),
+ vorticity(nullptr),
+ nabla_u(nullptr),
+ kk(nullptr) {}
+ ~test_interpolation(){}
+
+ int initialize(void);
+ int do_work(void);
+ int finalize(void);
+
+ int read_parameters(void);
+};
+
+#endif//TEST_INTERPOLATION_HPP
+
diff --git a/cpp/full_code/test_interpolation_methods.cpp b/cpp/full_code/test_interpolation_methods.cpp
new file mode 100644
index 0000000000000000000000000000000000000000..17cdfaf0ad75a02f78859bdedf61edf8536d9929
--- /dev/null
+++ b/cpp/full_code/test_interpolation_methods.cpp
@@ -0,0 +1,191 @@
+/******************************************************************************
+* *
+* Copyright 2022 Max Planck Computing and Data Facility *
+* *
+* This file is part of TurTLE. *
+* *
+* TurTLE is free software: you can redistribute it and/or modify *
+* it under the terms of the GNU General Public License as published *
+* by the Free Software Foundation, either version 3 of the License, *
+* or (at your option) any later version. *
+* *
+* TurTLE is distributed in the hope that it will be useful, *
+* but WITHOUT ANY WARRANTY; without even the implied warranty of *
+* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the *
+* GNU General Public License for more details. *
+* *
+* You should have received a copy of the GNU General Public License *
+* along with TurTLE. If not, see <http://www.gnu.org/licenses/> *
+* *
+* Contact: Cristian.Lalescu@mpcdf.mpg.de *
+* *
+******************************************************************************/
+
+
+
+#include "test_interpolation_methods.hpp"
+#include "particles/particles_input_grid.hpp"
+
+template <typename rnumber>
+int test_interpolation_methods<rnumber>::initialize()
+{
+ TIMEZONE("test_interpolation_methods::initialize");
+ this->read_parameters();
+
+ this->phi = new field<rnumber, FFTW, ONE>(
+ nx, ny, nz,
+ this->comm,
+ FFTW_ESTIMATE);
+
+ this->kk = new kspace<FFTW, SMOOTH>(
+ this->phi->clayout,
+ this->dkx, this->dky, this->dkz);
+
+ if (this->myrank == 0)
+ {
+ hid_t stat_file = H5Fopen(
+ (this->simname + std::string(".h5")).c_str(),
+ H5F_ACC_RDWR,
+ H5P_DEFAULT);
+ this->kk->store(stat_file);
+ H5Fclose(stat_file);
+ }
+
+ return EXIT_SUCCESS;
+}
+
+
+template <typename rnumber>
+template <int neighbours, int smoothness>
+int test_interpolation_methods<rnumber>::interpolate()
+{
+ // create a particle object
+ particle_set<3, neighbours, smoothness> pset(
+ this->phi->rlayout,
+ this->kk->dkx,
+ this->kk->dky,
+ this->kk->dkz);
+
+ // initialize particles
+ particles_input_grid<long long int, double, 3> pinput(
+ this->comm,
+ this->nxparticles,
+ this->nzparticles,
+ pset.getSpatialLowLimitZ(),
+ pset.getSpatialUpLimitZ());
+
+ pset.init(pinput);
+
+ assert(this->phi->real_space_representation);
+
+ pset.writeSample(
+ this->phi,
+ this->particles_sample_writer_mpi,
+ "particle",
+ "phi_" + std::to_string(neighbours) + std::to_string(smoothness),
+ 0);
+
+ return EXIT_SUCCESS;
+}
+
+
+template <typename rnumber>
+int test_interpolation_methods<rnumber>::do_work()
+{
+ TIMEZONE("test_interpolation_methods::do_work");
+
+ // create field
+ make_gaussian_random_field(
+ this->kk,
+ this->phi,
+ this->random_seed,
+ 0.4, // dissipation
+ 1., // Lint
+ 2*4*acos(0) / this->nx); // etaK
+
+ // take field to real space
+ this->phi->ift();
+
+ // create a particle object
+ particle_set<3, 1, 1> pset(
+ this->phi->rlayout,
+ this->kk->dkx,
+ this->kk->dky,
+ this->kk->dkz);
+ particles_input_grid<long long int, double, 3> pinput(
+ this->comm,
+ this->nxparticles,
+ this->nzparticles,
+ pset.getSpatialLowLimitZ(),
+ pset.getSpatialUpLimitZ());
+
+ pset.init(pinput);
+
+ // initialize writer
+ this->particles_sample_writer_mpi = new particles_output_sampling_hdf5<
+ long long int, double, double, 3>(
+ this->comm,
+ pset.getTotalNumberOfParticles(),
+ (this->simname + "_particles.h5"),
+ "particle",
+ "position/0");
+ this->particles_sample_writer_mpi->setParticleFileLayout(pset.getParticleFileLayout());
+
+ // sample position
+ pset.writeStateTriplet(
+ 0,
+ this->particles_sample_writer_mpi,
+ "particle",
+ "position",
+ 0);
+
+ // sample field
+ this->template interpolate<0, 0>();
+ this->template interpolate<1, 1>();
+ this->template interpolate<2, 1>();
+ this->template interpolate<2, 2>();
+ this->template interpolate<3, 1>();
+ this->template interpolate<3, 2>();
+ this->template interpolate<4, 1>();
+ this->template interpolate<4, 2>();
+ this->template interpolate<5, 1>();
+ this->template interpolate<5, 2>();
+
+ delete this->particles_sample_writer_mpi;
+
+ return EXIT_SUCCESS;
+}
+
+
+template <typename rnumber>
+int test_interpolation_methods<rnumber>::finalize()
+{
+ TIMEZONE("test_interpolation_methods::finalize");
+ delete this->kk;
+ delete this->phi;
+ return EXIT_SUCCESS;
+}
+
+
+template <typename rnumber>
+int test_interpolation_methods<rnumber>::read_parameters()
+{
+ TIMEZONE("test_interpolation_methods::read_parameters");
+ this->test::read_parameters();
+
+ hid_t parameter_file = H5Fopen(
+ (this->simname + std::string(".h5")).c_str(),
+ H5F_ACC_RDONLY,
+ H5P_DEFAULT);
+ this->nxparticles = hdf5_tools::read_value<int>(parameter_file, "/parameters/nxparticles");
+ this->nzparticles = hdf5_tools::read_value<int>(parameter_file, "/parameters/nzparticles");
+ this->random_seed = hdf5_tools::read_value<int>(parameter_file, "/parameters/field_random_seed");
+ H5Fclose(parameter_file);
+ MPI_Barrier(this->comm);
+ return EXIT_SUCCESS;
+}
+
+
+template class test_interpolation_methods<float>;
+template class test_interpolation_methods<double>;
+
diff --git a/cpp/full_code/test_interpolation_methods.hpp b/cpp/full_code/test_interpolation_methods.hpp
new file mode 100644
index 0000000000000000000000000000000000000000..62f661729cfbec51a25e6bb73819c8daac6dec66
--- /dev/null
+++ b/cpp/full_code/test_interpolation_methods.hpp
@@ -0,0 +1,79 @@
+/******************************************************************************
+* *
+* Copyright 2022 Max Planck Computing and Data Facility *
+* *
+* This file is part of TurTLE. *
+* *
+* TurTLE is free software: you can redistribute it and/or modify *
+* it under the terms of the GNU General Public License as published *
+* by the Free Software Foundation, either version 3 of the License, *
+* or (at your option) any later version. *
+* *
+* TurTLE is distributed in the hope that it will be useful, *
+* but WITHOUT ANY WARRANTY; without even the implied warranty of *
+* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the *
+* GNU General Public License for more details. *
+* *
+* You should have received a copy of the GNU General Public License *
+* along with TurTLE. If not, see <http://www.gnu.org/licenses/> *
+* *
+* Contact: Cristian.Lalescu@mpcdf.mpg.de *
+* *
+******************************************************************************/
+
+
+
+#ifndef TEST_INTERPOLATION_METHODS_HPP
+#define TEST_INTERPOLATION_METHODS_HPP
+
+
+
+#include <cstdlib>
+#include "base.hpp"
+#include "kspace.hpp"
+#include "full_code/test.hpp"
+#include "particles/interpolation/field_tinterpolator.hpp"
+#include "particles/interpolation/particle_set.hpp"
+
+/** \brief Test of interpolation methods.
+ *
+ */
+
+template <typename rnumber>
+class test_interpolation_methods: public test
+{
+ public:
+ int nxparticles;
+ int nzparticles;
+ int random_seed;
+
+ particles_output_sampling_hdf5<long long int, double, double, 3> *particles_sample_writer_mpi;
+
+ field<rnumber, FFTW, ONE> *phi;
+
+ kspace<FFTW, SMOOTH> *kk;
+
+ test_interpolation_methods(
+ const MPI_Comm COMMUNICATOR,
+ const std::string &simulation_name):
+ test(
+ COMMUNICATOR,
+ simulation_name),
+ random_seed(0),
+ particles_sample_writer_mpi(nullptr),
+ phi(nullptr),
+ kk(nullptr){}
+ ~test_interpolation_methods(){}
+
+ int initialize(void);
+ int do_work(void);
+ int finalize(void);
+
+ int read_parameters(void);
+
+ template <int neighbours, int smoothness>
+ int interpolate(void);
+};
+
+#endif//TEST_INTERPOLATION_METHODS_HPP
+
diff --git a/cpp/full_code/test_particle_integration.cpp b/cpp/full_code/test_particle_integration.cpp
new file mode 100644
index 0000000000000000000000000000000000000000..a1d3cbaf957ffb6dab9070746ac70d3d22f57695
--- /dev/null
+++ b/cpp/full_code/test_particle_integration.cpp
@@ -0,0 +1,263 @@
+/******************************************************************************
+* *
+* Copyright 2022 Max Planck Computing and Data Facility *
+* *
+* This file is part of TurTLE. *
+* *
+* TurTLE is free software: you can redistribute it and/or modify *
+* it under the terms of the GNU General Public License as published *
+* by the Free Software Foundation, either version 3 of the License, *
+* or (at your option) any later version. *
+* *
+* TurTLE is distributed in the hope that it will be useful, *
+* but WITHOUT ANY WARRANTY; without even the implied warranty of *
+* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the *
+* GNU General Public License for more details. *
+* *
+* You should have received a copy of the GNU General Public License *
+* along with TurTLE. If not, see <http://www.gnu.org/licenses/> *
+* *
+* Contact: Cristian.Lalescu@mpcdf.mpg.de *
+* *
+******************************************************************************/
+
+
+
+#include "test_particle_integration.hpp"
+#include "particles/particles_input_grid.hpp"
+#include "particles/rhs/tracer_rhs.hpp"
+
+template <typename rnumber>
+int test_particle_integration<rnumber>::initialize()
+{
+ TIMEZONE("test_particle_integration::initialize");
+ this->read_parameters();
+
+ this->velocity_back = new field<rnumber, FFTW, THREE>(
+ nx, ny, nz,
+ this->comm,
+ FFTW_ESTIMATE);
+
+ this->kk = new kspace<FFTW, SMOOTH>(
+ this->velocity_back->clayout,
+ this->dkx, this->dky, this->dkz);
+
+ this->velocity_front = new field_tinterpolator<rnumber, FFTW, THREE, NONE>();
+ this->velocity_front->set_field(this->velocity_back);
+
+ if (this->myrank == 0)
+ {
+ hid_t stat_file = H5Fopen(
+ (this->simname + std::string(".h5")).c_str(),
+ H5F_ACC_RDWR,
+ H5P_DEFAULT);
+ this->kk->store(stat_file);
+ H5Fclose(stat_file);
+ }
+
+ return EXIT_SUCCESS;
+}
+
+
+template <typename rnumber>
+template <int neighbours, int smoothness>
+int test_particle_integration<rnumber>::integrate(
+ const int order,
+ const int factor,
+ const int total_iterations)
+{
+ TIMEZONE("test_particle_integration::integrate");
+ DEBUG_MSG("entered integrate<%d, %d>(order=%d, factor=%d, iterations=%d)\n",
+ neighbours,
+ smoothness,
+ order,
+ factor,
+ total_iterations);
+ // create a particle object
+ particle_set<3, neighbours, smoothness> pset(
+ this->velocity_back->rlayout,
+ this->kk->dkx,
+ this->kk->dky,
+ this->kk->dkz);
+
+ // initialize particles
+ particles_input_grid<long long int, double, 3> pinput(
+ this->comm,
+ this->nxparticles,
+ this->nzparticles,
+ pset.getSpatialLowLimitZ(),
+ pset.getSpatialUpLimitZ());
+
+ pset.init(pinput);
+
+ tracer_rhs<rnumber, FFTW, NONE> rhs;
+ rhs.setVelocity(this->velocity_front);
+
+ particle_solver psolver(pset, 0);
+
+ const std::string species_name = (
+ "particle_o" + std::to_string(order) +
+ "_n" + std::to_string(neighbours) +
+ "_m" + std::to_string(smoothness) +
+ "_f" + std::to_string(factor));
+
+ for (int tt = 0; tt < total_iterations*factor; tt++)
+ {
+ // output
+ if (tt % 4 == 0)
+ {
+ pset.writeStateTriplet(
+ 0,
+ this->particles_sample_writer_mpi,
+ species_name,
+ "position",
+ tt);
+ pset.writeSample(
+ this->velocity_back,
+ this->particles_sample_writer_mpi,
+ species_name,
+ "velocity",
+ tt);
+ }
+ // time step
+ switch(order)
+ {
+ case 1:
+ psolver.Euler(this->dt / factor, rhs);
+ break;
+ case 2:
+ psolver.Heun(this->dt / factor, rhs);
+ break;
+ case 4:
+ psolver.cRK4(this->dt / factor, rhs);
+ break;
+ default:
+ assert(false);
+ break;
+ }
+ }
+ // output final state
+ {
+ int tt = total_iterations*factor;
+ pset.writeStateTriplet(
+ 0,
+ this->particles_sample_writer_mpi,
+ species_name,
+ "position",
+ tt);
+ pset.writeSample(
+ this->velocity_back,
+ this->particles_sample_writer_mpi,
+ species_name,
+ "velocity",
+ tt);
+ }
+
+ return EXIT_SUCCESS;
+}
+
+
+template <typename rnumber>
+int test_particle_integration<rnumber>::do_work()
+{
+ TIMEZONE("test_particle_integration::do_work");
+
+ // create field
+ make_gaussian_random_field(
+ this->kk,
+ this->velocity_back,
+ this->random_seed,
+ 0.4, // dissipation
+ 1., // Lint
+ 4*acos(0) / this->nx, // etaK
+ 6.78, // default
+ 0.40, // default
+ 3*3./2); // 3/2 to account for divfree call below
+ // 3 because I want a larger amplitude
+ this->kk->force_divfree<rnumber>(this->velocity_back->get_cdata());
+
+ // take field to real space
+ this->velocity_back->ift();
+
+ // create a particle object
+ particle_set<3, 1, 1> pset(
+ this->velocity_back->rlayout,
+ this->kk->dkx,
+ this->kk->dky,
+ this->kk->dkz);
+ particles_input_grid<long long int, double, 3> pinput(
+ this->comm,
+ this->nxparticles,
+ this->nzparticles,
+ pset.getSpatialLowLimitZ(),
+ pset.getSpatialUpLimitZ());
+
+ pset.init(pinput);
+
+ // initialize writer
+ this->particles_sample_writer_mpi = new particles_output_sampling_hdf5<
+ long long int, double, double, 3>(
+ this->comm,
+ pset.getTotalNumberOfParticles(),
+ (this->simname + "_particles.h5"),
+ "particle",
+ "position/0");
+ this->particles_sample_writer_mpi->setParticleFileLayout(pset.getParticleFileLayout());
+
+ // test integration
+ for (int oo = 1; oo < 5; oo *= 2)
+ for (int ff = 1; ff < 9; ff *= 2)
+ {
+ this->template integrate<0, 0>(oo, ff, this->niter_todo);
+ this->template integrate<1, 1>(oo, ff, this->niter_todo);
+ this->template integrate<2, 1>(oo, ff, this->niter_todo);
+ this->template integrate<3, 1>(oo, ff, this->niter_todo);
+ this->template integrate<4, 1>(oo, ff, this->niter_todo);
+ this->template integrate<5, 1>(oo, ff, this->niter_todo);
+ this->template integrate<2, 2>(oo, ff, this->niter_todo);
+ this->template integrate<3, 2>(oo, ff, this->niter_todo);
+ this->template integrate<4, 2>(oo, ff, this->niter_todo);
+ this->template integrate<5, 2>(oo, ff, this->niter_todo);
+ }
+
+ delete this->particles_sample_writer_mpi;
+
+ return EXIT_SUCCESS;
+}
+
+
+template <typename rnumber>
+int test_particle_integration<rnumber>::finalize()
+{
+ TIMEZONE("test_particle_integration::finalize");
+ delete this->velocity_front;
+ delete this->velocity_back;
+ delete this->kk;
+ return EXIT_SUCCESS;
+}
+
+
+template <typename rnumber>
+int test_particle_integration<rnumber>::read_parameters()
+{
+ TIMEZONE("test_particle_integration::read_parameters");
+ this->test::read_parameters();
+
+ hid_t parameter_file = H5Fopen(
+ (this->simname + std::string(".h5")).c_str(),
+ H5F_ACC_RDONLY,
+ H5P_DEFAULT);
+ this->nxparticles = hdf5_tools::read_value<int>(parameter_file, "/parameters/nxparticles");
+ this->nzparticles = hdf5_tools::read_value<int>(parameter_file, "/parameters/nzparticles");
+ this->dt = hdf5_tools::read_value<double>(parameter_file, "/parameters/dt");
+ this->niter_todo = hdf5_tools::read_value<int>(parameter_file, "/parameters/niter_todo");
+ this->random_seed = hdf5_tools::read_value<int>(parameter_file, "/parameters/field_random_seed");
+ H5Fclose(parameter_file);
+ MPI_Barrier(this->comm);
+ return EXIT_SUCCESS;
+}
+
+
+template class test_particle_integration<float>;
+template class test_particle_integration<double>;
+
diff --git a/cpp/full_code/test_particle_integration.hpp b/cpp/full_code/test_particle_integration.hpp
new file mode 100644
index 0000000000000000000000000000000000000000..88a32dc19216887d619832fddcf6c5dc6ec18118
--- /dev/null
+++ b/cpp/full_code/test_particle_integration.hpp
@@ -0,0 +1,89 @@
+/******************************************************************************
+* *
+* Copyright 2022 Max Planck Computing and Data Facility *
+* *
+* This file is part of TurTLE. *
+* *
+* TurTLE is free software: you can redistribute it and/or modify *
+* it under the terms of the GNU General Public License as published *
+* by the Free Software Foundation, either version 3 of the License, *
+* or (at your option) any later version. *
+* *
+* TurTLE is distributed in the hope that it will be useful, *
+* but WITHOUT ANY WARRANTY; without even the implied warranty of *
+* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the *
+* GNU General Public License for more details. *
+* *
+* You should have received a copy of the GNU General Public License *
+* along with TurTLE. If not, see <http://www.gnu.org/licenses/> *
+* *
+* Contact: Cristian.Lalescu@mpcdf.mpg.de *
+* *
+******************************************************************************/
+
+
+
+#ifndef TEST_PARTICLE_INTEGRATION_HPP
+#define TEST_PARTICLE_INTEGRATION_HPP
+
+
+
+#include <cstdlib>
+#include "base.hpp"
+#include "kspace.hpp"
+#include "full_code/test.hpp"
+#include "particles/interpolation/field_tinterpolator.hpp"
+#include "particles/interpolation/particle_set.hpp"
+#include "particles/particle_solver.hpp"
+
+/** \brief Test of interpolation methods.
+ *
+ */
+
+template <typename rnumber>
+class test_particle_integration: public test
+{
+ public:
+ int nxparticles;
+ int nzparticles;
+ double dt;
+ int niter_todo;
+ int random_seed;
+
+
+ field_tinterpolator<rnumber, FFTW, THREE, NONE> *velocity_front;
+
+ particles_output_sampling_hdf5<long long int, double, double, 3> *particles_sample_writer_mpi;
+
+ field<rnumber, FFTW, THREE> *velocity_back;
+
+ kspace<FFTW, SMOOTH> *kk;
+
+ test_particle_integration(
+ const MPI_Comm COMMUNICATOR,
+ const std::string &simulation_name):
+ test(
+ COMMUNICATOR,
+ simulation_name),
+ random_seed(0),
+ velocity_front(nullptr),
+ particles_sample_writer_mpi(nullptr),
+ velocity_back(nullptr),
+ kk(nullptr){}
+ ~test_particle_integration(){}
+
+ int initialize(void);
+ int do_work(void);
+ int finalize(void);
+
+ int read_parameters(void);
+
+ template <int neighbours, int smoothness>
+ int integrate(
+ const int order,
+ const int factor,
+ const int total_iterations);
+};
+
+#endif//TEST_PARTICLE_INTEGRATION_HPP
+
diff --git a/cpp/full_code/test_tracer_set.cpp b/cpp/full_code/test_tracer_set.cpp
new file mode 100644
index 0000000000000000000000000000000000000000..ec8893577686cab0410ec49b396133535868b586
--- /dev/null
+++ b/cpp/full_code/test_tracer_set.cpp
@@ -0,0 +1,168 @@
+/******************************************************************************
+* *
+* Copyright 2019 Max Planck Institute for Dynamics and Self-Organization *
+* *
+* This file is part of TurTLE. *
+* *
+* TurTLE is free software: you can redistribute it and/or modify *
+* it under the terms of the GNU General Public License as published *
+* by the Free Software Foundation, either version 3 of the License, *
+* or (at your option) any later version. *
+* *
+* TurTLE is distributed in the hope that it will be useful, *
+* but WITHOUT ANY WARRANTY; without even the implied warranty of *
+* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the *
+* GNU General Public License for more details. *
+* *
+* You should have received a copy of the GNU General Public License *
+* along with TurTLE. If not, see <http://www.gnu.org/licenses/> *
+* *
+* Contact: Cristian.Lalescu@ds.mpg.de *
+* *
+******************************************************************************/
+
+
+
+#include <string>
+#include <cmath>
+#include <random>
+#include "test_tracer_set.hpp"
+#include "particles/rhs/tracer_rhs.hpp"
+#include "particles/rhs/tracer_with_collision_counter_rhs.hpp"
+#include "particles/interpolation/particle_set.hpp"
+#include "particles/particle_solver.hpp"
+#include "particles/particles_input_random.hpp"
+#include "scope_timer.hpp"
+
+
+template <typename rnumber>
+int test_tracer_set<rnumber>::initialize(void)
+{
+ TIMEZONE("test_tracer_set::initialize");
+ this->read_parameters();
+ return EXIT_SUCCESS;
+}
+
+template <typename rnumber>
+int test_tracer_set<rnumber>::finalize(void)
+{
+ TIMEZONE("test_tracer_set::finalize");
+ return EXIT_SUCCESS;
+}
+
+template <typename rnumber>
+int test_tracer_set<rnumber>::read_parameters()
+{
+ TIMEZONE("test_tracer_set::read_parameters");
+ this->test::read_parameters();
+ MPI_Barrier(this->comm);
+ return EXIT_SUCCESS;
+}
+
+template <typename rnumber>
+int test_tracer_set<rnumber>::do_work(void)
+{
+ TIMEZONE("test_tracer_set::do_work");
+ const int nparticles = 1001;
+ // allocate
+ field<rnumber, FFTW, THREE> *vec_field = new field<rnumber, FFTW, THREE>(
+ this->nx, this->ny, this->nz,
+ this->comm,
+ FFTW_ESTIMATE);
+
+ kspace<FFTW, SMOOTH> *kk = new kspace<FFTW, SMOOTH>(
+ vec_field->clayout,
+ this->dkx, this->dky, this->dkz);
+
+ // initialize field
+ vec_field->real_space_representation = false;
+ make_gaussian_random_field(
+ kk,
+ vec_field,
+ 1 // rseed
+ );
+ vec_field->ift();
+
+ field_tinterpolator<rnumber, FFTW, THREE, NONE> *fti = new field_tinterpolator<rnumber, FFTW, THREE, NONE>();
+ tracer_with_collision_counter_rhs<rnumber, FFTW, NONE> *trhs = new tracer_with_collision_counter_rhs<rnumber, FFTW, NONE>();
+
+ particle_set<3, 2, 1> pset(
+ vec_field->rlayout,
+ this->dkx,
+ this->dky,
+ this->dkz,
+ 0.5);
+
+ // initialize particles
+ particles_input_random<long long int, double, 3> bla(
+ this->comm,
+ nparticles,
+ 1,
+ pset.getSpatialLowLimitZ(),
+ pset.getSpatialUpLimitZ());
+ pset.init(bla);
+
+ // initialize particle output object
+ particles_output_hdf5<long long int, double, 3> *particle_output_writer = new particles_output_hdf5<
+ long long int, double, 3>(
+ MPI_COMM_WORLD,
+ "tracers0",
+ pset.getTotalNumberOfParticles(),
+ 0);
+ particles_output_sampling_hdf5<long long int, double, double, 3> *particle_sample_writer = new particles_output_sampling_hdf5<
+ long long int, double, double, 3>(
+ MPI_COMM_WORLD,
+ pset.getTotalNumberOfParticles(),
+ "test_particle_sample.h5",
+ "tracers0",
+ "position/0");
+
+ fti->set_field(vec_field);
+ trhs->setVelocity(fti);
+
+ particle_solver psolver(pset, 0);
+
+ particle_output_writer->open_file("test_particle_checkpoint.h5");
+ psolver.template writeCheckpoint<3>(particle_output_writer);
+ pset.writeSample(
+ vec_field,
+ particle_sample_writer,
+ "tracers0",
+ "velocity",
+ psolver.getIteration());
+ pset.writeStateTriplet(
+ 0,
+ particle_sample_writer,
+ "tracers0",
+ "position",
+ psolver.getIteration());
+ psolver.Heun(0.001, *trhs);
+ psolver.setIteration(1);
+ psolver.template writeCheckpoint<3>(particle_output_writer);
+ pset.writeSample(
+ vec_field,
+ particle_sample_writer,
+ "tracers0",
+ "velocity",
+ psolver.getIteration());
+ pset.writeStateTriplet(
+ 0,
+ particle_sample_writer,
+ "tracers0",
+ "position",
+ psolver.getIteration());
+ particle_output_writer->close_file();
+
+ // deallocate
+ delete particle_sample_writer;
+ delete particle_output_writer;
+ delete trhs;
+ delete fti;
+ delete kk;
+ delete vec_field;
+ return EXIT_SUCCESS;
+}
+
+template class test_tracer_set<float>;
+template class test_tracer_set<double>;
+
diff --git a/cpp/full_code/test_tracer_set.hpp b/cpp/full_code/test_tracer_set.hpp
new file mode 100644
index 0000000000000000000000000000000000000000..b6d0ad4a0a68500639fea1f5b1e214a2fa61aad4
--- /dev/null
+++ b/cpp/full_code/test_tracer_set.hpp
@@ -0,0 +1,60 @@
+/**********************************************************************
+* *
+* Copyright 2017 Max Planck Institute *
+* for Dynamics and Self-Organization *
+* *
+* This file is part of TurTLE. *
+* *
+* TurTLE is free software: you can redistribute it and/or modify *
+* it under the terms of the GNU General Public License as published *
+* by the Free Software Foundation, either version 3 of the License, *
+* or (at your option) any later version. *
+* *
+* TurTLE is distributed in the hope that it will be useful, *
+* but WITHOUT ANY WARRANTY; without even the implied warranty of *
+* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the *
+* GNU General Public License for more details. *
+* *
+* You should have received a copy of the GNU General Public License *
+* along with TurTLE. If not, see <http://www.gnu.org/licenses/> *
+* *
+* Contact: Cristian.Lalescu@ds.mpg.de *
+* *
+**********************************************************************/
+
+
+
+#ifndef TEST_TRACER_SET_HPP
+#define TEST_TRACER_SET_HPP
+
+
+
+#include <cstdlib>
+#include "base.hpp"
+#include "kspace.hpp"
+#include "field.hpp"
+#include "full_code/test.hpp"
+
+/** \brief A class for testing basic `particle_set` functionality.
+ */
+
+template <typename rnumber>
+class test_tracer_set: public test
+{
+ public:
+ test_tracer_set(
+ const MPI_Comm COMMUNICATOR,
+ const std::string &simulation_name):
+ test(
+ COMMUNICATOR,
+ simulation_name){}
+ ~test_tracer_set(){}
+
+ int initialize(void);
+ int do_work(void);
+ int finalize(void);
+ int read_parameters(void);
+};
+
+#endif//TEST_TRACER_SET_HPP
+
diff --git a/cpp/full_code/write_filtered_test.cpp b/cpp/full_code/write_filtered_test.cpp
new file mode 100644
index 0000000000000000000000000000000000000000..ead8916e26d0b3b3c0be3e3e98985ca6dfc237b0
--- /dev/null
+++ b/cpp/full_code/write_filtered_test.cpp
@@ -0,0 +1,211 @@
+/******************************************************************************
+* *
+* Copyright 2019 Max Planck Institute for Dynamics and Self-Organization *
+* *
+* This file is part of TurTLE. *
+* *
+* TurTLE is free software: you can redistribute it and/or modify *
+* it under the terms of the GNU General Public License as published *
+* by the Free Software Foundation, either version 3 of the License, *
+* or (at your option) any later version. *
+* *
+* TurTLE is distributed in the hope that it will be useful, *
+* but WITHOUT ANY WARRANTY; without even the implied warranty of *
+* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the *
+* GNU General Public License for more details. *
+* *
+* You should have received a copy of the GNU General Public License *
+* along with TurTLE. If not, see <http://www.gnu.org/licenses/> *
+* *
+* Contact: Cristian.Lalescu@ds.mpg.de *
+* *
+******************************************************************************/
+
+
+
+#include <string>
+#include <cmath>
+#include <random>
+#include "write_filtered_test.hpp"
+#include "scope_timer.hpp"
+
+
+template <typename rnumber>
+int write_filtered_test<rnumber>::initialize(void)
+{
+ TIMEZONE("write_filtered_test::initialize");
+ this->read_parameters();
+ return EXIT_SUCCESS;
+}
+
+template <typename rnumber>
+int write_filtered_test<rnumber>::finalize(void)
+{
+ TIMEZONE("write_filtered_test::finalize");
+ this->read_parameters();
+ return EXIT_SUCCESS;
+}
+
+template <typename rnumber>
+int write_filtered_test<rnumber>::read_parameters()
+{
+ TIMEZONE("write_filtered_test::read_parameters");
+ this->test::read_parameters();
+ // in case any parameters are needed, this is where they should be read
+ hid_t parameter_file = H5Fopen(
+ (this->simname + std::string(".h5")).c_str(),
+ H5F_ACC_RDONLY,
+ H5P_DEFAULT);
+ H5Fclose(parameter_file);
+ MPI_Barrier(this->comm);
+ return EXIT_SUCCESS;
+}
+
+template <typename rnumber>
+int write_filtered_test<rnumber>::do_work(void)
+{
+ TIMEZONE("write_filtered_test::do_work");
+ // allocate
+ field<rnumber, FFTW, ONE> *scal_field = new field<rnumber, FFTW, ONE>(
+ this->nx, this->ny, this->nz,
+ this->comm,
+ FFTW_ESTIMATE);
+ std::default_random_engine rgen;
+ std::normal_distribution<rnumber> rdist;
+ rgen.seed(2);
+
+ // fill up scal_field
+ scal_field->real_space_representation = true;
+ scal_field->RLOOP(
+ [&](ptrdiff_t rindex,
+ ptrdiff_t xindex,
+ ptrdiff_t yindex,
+ ptrdiff_t zindex){
+ scal_field->rval(rindex) = rdist(rgen);
+ });
+ scal_field->dft();
+
+ scal_field->write_filtered(
+ "data_for_write_filtered.h5",
+ "scal_field_z0slice",
+ this->iteration,
+ this->nx,
+ this->ny,
+ 1);
+
+ scal_field->write_filtered(
+ "data_for_write_filtered.h5",
+ "scal_field_full",
+ this->iteration,
+ this->nx,
+ this->ny,
+ this->nz);
+
+ scal_field->write_filtered(
+ "data_for_write_filtered.h5",
+ "scal_field_half",
+ this->iteration,
+ this->nx/2,
+ this->ny/2,
+ this->nz/2);
+
+ // deallocate
+ delete scal_field;
+
+ // allocate
+ field<rnumber, FFTW, THREE> *vec_field = new field<rnumber, FFTW, THREE>(
+ this->nx, this->ny, this->nz,
+ this->comm,
+ FFTW_ESTIMATE);
+
+ // fill up vec_field
+ vec_field->real_space_representation = true;
+ vec_field->RLOOP(
+ [&](ptrdiff_t rindex,
+ ptrdiff_t xindex,
+ ptrdiff_t yindex,
+ ptrdiff_t zindex){
+ vec_field->rval(rindex, 0) = rdist(rgen);
+ vec_field->rval(rindex, 1) = rdist(rgen);
+ vec_field->rval(rindex, 2) = rdist(rgen);
+ });
+ vec_field->dft();
+
+ vec_field->write_filtered(
+ "data_for_write_filtered.h5",
+ "vec_field_z0slice",
+ this->iteration,
+ this->nx,
+ this->ny,
+ 1);
+
+ vec_field->write_filtered(
+ "data_for_write_filtered.h5",
+ "vec_field_full",
+ this->iteration,
+ this->nx,
+ this->ny,
+ this->nz);
+
+ vec_field->write_filtered(
+ "data_for_write_filtered.h5",
+ "vec_field_half",
+ this->iteration,
+ this->nx/2,
+ this->ny/2,
+ this->nz/2);
+
+ // deallocate
+ delete vec_field;
+
+ // allocate
+ field<rnumber, FFTW, THREExTHREE> *tens_field = new field<rnumber, FFTW, THREExTHREE>(
+ this->nx, this->ny, this->nz,
+ this->comm,
+ FFTW_ESTIMATE);
+
+ // fill up tens_field
+ tens_field->real_space_representation = true;
+ tens_field->RLOOP(
+ [&](ptrdiff_t rindex,
+ ptrdiff_t xindex,
+ ptrdiff_t yindex,
+ ptrdiff_t zindex){
+ for (int cc = 0; cc<3; cc++)
+ for (int ccc = 0; ccc<3; ccc++)
+ tens_field->rval(rindex, cc, ccc) = rdist(rgen);
+ });
+ tens_field->dft();
+
+ tens_field->write_filtered(
+ "data_for_write_filtered.h5",
+ "tens_field_z0slice",
+ this->iteration,
+ this->nx,
+ this->ny,
+ 1);
+
+ tens_field->write_filtered(
+ "data_for_write_filtered.h5",
+ "tens_field_full",
+ this->iteration,
+ this->nx,
+ this->ny,
+ this->nz);
+
+ tens_field->write_filtered(
+ "data_for_write_filtered.h5",
+ "tens_field_half",
+ this->iteration,
+ this->nx/2,
+ this->ny/2,
+ this->nz/2);
+
+ // deallocate
+ delete tens_field;
+ return EXIT_SUCCESS;
+}
+
+template class write_filtered_test<float>;
+template class write_filtered_test<double>;
+
diff --git a/cpp/full_code/write_filtered_test.hpp b/cpp/full_code/write_filtered_test.hpp
new file mode 100644
index 0000000000000000000000000000000000000000..34cab4de7b1027b7520980365ea1f03a9df2f12e
--- /dev/null
+++ b/cpp/full_code/write_filtered_test.hpp
@@ -0,0 +1,60 @@
+/**********************************************************************
+* *
+* Copyright 2019 Max Planck Institute *
+* for Dynamics and Self-Organization *
+* *
+* This file is part of TurTLE. *
+* *
+* TurTLE is free software: you can redistribute it and/or modify *
+* it under the terms of the GNU General Public License as published *
+* by the Free Software Foundation, either version 3 of the License, *
+* or (at your option) any later version. *
+* *
+* TurTLE is distributed in the hope that it will be useful, *
+* but WITHOUT ANY WARRANTY; without even the implied warranty of *
+* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the *
+* GNU General Public License for more details. *
+* *
+* You should have received a copy of the GNU General Public License *
+* along with TurTLE. If not, see <http://www.gnu.org/licenses/> *
+* *
+* Contact: Cristian.Lalescu@ds.mpg.de *
+* *
+**********************************************************************/
+
+
+
+#ifndef WRITE_FILTERED_TEST_HPP
+#define WRITE_FILTERED_TEST_HPP
+
+
+
+#include <cstdlib>
+#include "base.hpp"
+#include "kspace.hpp"
+#include "field.hpp"
+#include "full_code/test.hpp"
+
+/** \brief A class for testing basic field class functionality.
+ */
+
+template <typename rnumber>
+class write_filtered_test: public test
+{
+ public:
+ write_filtered_test(
+ const MPI_Comm COMMUNICATOR,
+ const std::string &simulation_name):
+ test(
+ COMMUNICATOR,
+ simulation_name){}
+ ~write_filtered_test(){}
+
+ int initialize(void);
+ int do_work(void);
+ int finalize(void);
+ int read_parameters(void);
+};
+
+#endif//WRITE_FILTERED_TEST_HPP
+
diff --git a/cpp/full_code/write_rpressure.cpp b/cpp/full_code/write_rpressure.cpp
new file mode 100644
index 0000000000000000000000000000000000000000..a9e5e4ffa6972030c597befdb7e1020464af3f73
--- /dev/null
+++ b/cpp/full_code/write_rpressure.cpp
@@ -0,0 +1,141 @@
+#include <string>
+#include <cmath>
+#include <hdf5.h>
+#include "write_rpressure.hpp"
+#include "scope_timer.hpp"
+#include "hdf5_tools.hpp"
+
+template <typename rnumber>
+int write_rpressure<rnumber>::initialize(void)
+{
+
+ // initialize
+ this->NSVE_field_stats<rnumber>::initialize();
+
+ //iteration list needed by postprocess.cpp for the main loop
+ hid_t parameter_file = H5Fopen(
+ (this->simname + std::string("_post.h5")).c_str(),
+ H5F_ACC_RDONLY,
+ H5P_DEFAULT);
+ this->iteration_list = hdf5_tools::read_vector<int>(
+ parameter_file,
+ "/write_rpressure/parameters/iteration_list");
+ H5Fclose(parameter_file);
+ // the following ensures the file is free for rank 0 to open in read/write mode
+ MPI_Barrier(this->comm);
+ if (this->myrank==0) DEBUG_MSG("Iteration list[0] = %d \n", this->iteration_list[0]);
+
+ //get the pressure from here
+ this->ve = new vorticity_equation<rnumber, FFTW>(
+ this->simname.c_str(),
+ this->nx,
+ this->ny,
+ this->nz,
+ this->dkx,
+ this->dky,
+ this->dkz,
+ this->vorticity->fftw_plan_rigor);
+
+ // output field
+ this->pressure = new field<rnumber, FFTW, ONE>(
+ this->nx, this->ny, this->nz,
+ this->comm,
+ this->vorticity->fftw_plan_rigor);
+
+ // write to binary
+ this->bin_IO_scalar = new field_binary_IO<rnumber, REAL, ONE>(
+ this->pressure->rlayout->sizes,
+ this->pressure->rlayout->subsizes,
+ this->pressure->rlayout->starts,
+ this->pressure->rlayout->comm);
+
+ if (this->myrank==0) DEBUG_MSG("Initialize end\n");
+ return EXIT_SUCCESS;
+}
+
+template <typename rnumber>
+int write_rpressure<rnumber>::clip_zero_padding(
+ field <rnumber, FFTW, ONE>* f)
+{
+ rnumber *a = f->get_rdata();
+ rnumber *b = f->get_rdata();
+ ptrdiff_t copy_size = f->rlayout->sizes[2] * ncomp(ONE);
+ ptrdiff_t skip_size = copy_size + 2*ncomp(ONE);
+ for (int i0 = 0; i0 < int(f->rlayout->subsizes[0]); i0++)
+ for (int i1 = 0; i1 < int(f->rlayout->sizes[1]); i1++)
+ {
+ std::copy(a, a + copy_size, b);
+ a += skip_size;
+ b += copy_size;
+ }
+ return EXIT_SUCCESS;
+}
+
+
+template <typename rnumber>
+int write_rpressure<rnumber>::work_on_current_iteration(void)
+{
+ if (this->myrank==0) DEBUG_MSG("\nComputing the pressure\n");
+ //compute_pressure
+ this->read_current_cvorticity();
+ *this->ve->cvorticity = this->vorticity->get_cdata();
+
+ this->ve->compute_velocity(this->ve->cvorticity);
+ this->ve->cvelocity->ift();
+ this->ve->compute_pressure(this->pressure);
+
+
+ if (this->myrank==0) DEBUG_MSG("Computed the pressure\n");
+
+ if (this->myrank==0) DEBUG_MSG("vorticity in ve fourier = [%lG, %lG; ...; %lG, %lG; %lG, %lG;.....; %lG, %lG;] \n", this->vorticity->get_cdata()[0][0], this->vorticity->get_cdata()[0][1], this->vorticity->get_cdata()[1][0], this->vorticity->get_cdata()[1][1], this->vorticity->get_cdata()[2][0], this->vorticity->get_cdata()[2][1],this->vorticity->get_cdata()[65][0], this->vorticity->get_cdata()[65][1]);
+
+ if (this->myrank==0) DEBUG_MSG("vorticity in ve fourier = [%lG, %lG; ...; %lG, %lG; %lG, %lG;.....; %lG, %lG;] \n", this->ve->cvorticity->get_cdata()[0][0], this->ve->cvorticity->get_cdata()[0][1], this->ve->cvorticity->get_cdata()[1][0], this->ve->cvorticity->get_cdata()[1][1], this->ve->cvorticity->get_cdata()[2][0], this->ve->cvorticity->get_cdata()[2][1],this->ve->cvorticity->get_cdata()[65][0], this->ve->cvorticity->get_cdata()[65][1]);
+ this->ve->compute_velocity(this->vorticity);
+
+ if (this->myrank==0) DEBUG_MSG("velocity in ve fourier = [%lG, %lG; ...; %lG, %lG; %lG, %lG;.....; %lG, %lG;] \n", this->ve->u->get_cdata()[0][0], this->ve->u->get_cdata()[0][1], this->ve->u->get_cdata()[1][0], this->ve->u->get_cdata()[1][1], this->ve->u->get_cdata()[2][0], this->ve->u->get_cdata()[2][1],this->ve->u->get_cdata()[65][0], this->ve->u->get_cdata()[65][1]);
+
+ if (this->myrank==0) DEBUG_MSG("pressure fourier = [%lG, %lG; ...; %lG, %lG; %lG, %lG;.....; %lG, %lG;] \n", this->pressure->get_cdata()[0][0], this->pressure->get_cdata()[0][1], this->pressure->get_cdata()[1][0], this->pressure->get_cdata()[1][1], this->pressure->get_cdata()[2][0], this->pressure->get_cdata()[2][1],this->pressure->get_cdata()[65][0], this->pressure->get_cdata()[65][1]);
+
+ //transform to real space
+ if (this->pressure->real_space_representation == false)
+ this->pressure->ift();
+
+ if (this->myrank==0) DEBUG_MSG("Real space\n");
+
+ if (this->myrank==0) DEBUG_MSG("pressure before clipping = [%lG; ...; %lG;.....; %lG;] \n", this->pressure->get_rdata()[0], this->pressure->get_rdata()[2], this->pressure->get_rdata()[13549]);
+
+ //write out to file
+ this->clip_zero_padding(pressure);
+ if (this->myrank==0) DEBUG_MSG("pressure after clipping = [%lG; ...; %lG;.....; %lG;] \n", this->pressure->get_rdata()[0], this->pressure->get_rdata()[2], this->pressure->get_rdata()[13549]);
+
+ //write to file -- THIS IS OK
+ char itername[16];
+ sprintf(itername, "i%.5x", this->iteration);
+ std::string native_binary_fname;
+
+ native_binary_fname = (
+ this->simname +
+ std::string("_rpressure_") +
+ std::string(itername));
+ if (this->myrank ==0) DEBUG_MSG("%s\n", native_binary_fname.c_str());
+
+ this->bin_IO_scalar->write(
+ native_binary_fname,
+ this->pressure->get_rdata());
+ return EXIT_SUCCESS;
+}
+
+template <typename rnumber>
+int write_rpressure<rnumber>::finalize(void)
+{
+
+ delete this->bin_IO_scalar;
+ delete this->pressure;
+ delete this->ve;
+ this->NSVE_field_stats<rnumber>::finalize();
+ return EXIT_SUCCESS;
+}
+
+template class write_rpressure<float>;
+template class write_rpressure<double>;
+
diff --git a/cpp/full_code/write_rpressure.hpp b/cpp/full_code/write_rpressure.hpp
new file mode 100644
index 0000000000000000000000000000000000000000..2a14dcb6d58188cb5ad7b6d9333cff76ef963282
--- /dev/null
+++ b/cpp/full_code/write_rpressure.hpp
@@ -0,0 +1,68 @@
+/**********************************************************************
+* *
+* Copyright 2017 Max Planck Institute *
+* for Dynamics and Self-Organization *
+* *
+* This file is part of TurTLE. *
+* *
+* TurTLE is free software: you can redistribute it and/or modify *
+* it under the terms of the GNU General Public License as published *
+* by the Free Software Foundation, either version 3 of the License, *
+* or (at your option) any later version. *
+* *
+* TurTLE is distributed in the hope that it will be useful, *
+* but WITHOUT ANY WARRANTY; without even the implied warranty of *
+* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the *
+* GNU General Public License for more details. *
+* *
+* You should have received a copy of the GNU General Public License *
+* along with TurTLE. If not, see <http://www.gnu.org/licenses/> *
+* *
+* Contact: Cristian.Lalescu@ds.mpg.de *
+* *
+**********************************************************************/
+
+
+
+#ifndef WRITE_RPRESSURE
+#define WRITE_RPRESSURE
+
+#include <cstdlib>
+#include <sys/types.h>
+#include <sys/stat.h>
+#include <vector>
+#include "base.hpp"
+#include "field.hpp"
+#include "field_binary_IO.hpp"
+#include "vorticity_equation.hpp"
+#include "full_code/NSVE_field_stats.hpp"
+
+/** generate and write out pressure field in real space **/
+
+template <typename rnumber>
+class write_rpressure: public NSVE_field_stats<rnumber>
+{
+ public:
+ int niter_out;
+
+ vorticity_equation<rnumber, FFTW> *ve;
+ field<rnumber, FFTW, ONE> *pressure;
+ field_binary_IO <rnumber, REAL, ONE> *bin_IO_scalar;
+ int clip_zero_padding(field <rnumber, FFTW, ONE>* );
+
+ write_rpressure(
+ const MPI_Comm COMMUNICATOR,
+ const std::string &simulation_name):
+ NSVE_field_stats<rnumber>(
+ COMMUNICATOR,
+ simulation_name){}
+ virtual ~write_rpressure() noexcept(false){}
+
+ int initialize(void);
+ int work_on_current_iteration(void);
+ int finalize(void);
+};
+
+
+#endif//WRITE_PRESSURE
+
diff --git a/cpp/hdf5_tools.cpp b/cpp/hdf5_tools.cpp
new file mode 100644
index 0000000000000000000000000000000000000000..9a92c459cf368de3a66b456bffef502fcc9478cb
--- /dev/null
+++ b/cpp/hdf5_tools.cpp
@@ -0,0 +1,596 @@
+#include "hdf5_tools.hpp"
+#include "scope_timer.hpp"
+#include <cfloat>
+#include <climits>
+
+template <> hid_t hdf5_tools::hdf5_type_id<char>()
+{
+ return H5T_NATIVE_CHAR;
+}
+template <> hid_t hdf5_tools::hdf5_type_id<signed char>()
+{
+ return H5T_NATIVE_SCHAR;
+}
+template <> hid_t hdf5_tools::hdf5_type_id<unsigned char>()
+{
+ return H5T_NATIVE_UCHAR;
+}
+template <> hid_t hdf5_tools::hdf5_type_id<short>()
+{
+ return H5T_NATIVE_SHORT;
+}
+template <> hid_t hdf5_tools::hdf5_type_id<unsigned short>()
+{
+ return H5T_NATIVE_USHORT;
+}
+template <> hid_t hdf5_tools::hdf5_type_id<int>()
+{
+ return H5T_NATIVE_INT;
+}
+template <> hid_t hdf5_tools::hdf5_type_id<unsigned>()
+{
+ return H5T_NATIVE_UINT;
+}
+template <> hid_t hdf5_tools::hdf5_type_id<long>()
+{
+ return H5T_NATIVE_LONG;
+}
+template <> hid_t hdf5_tools::hdf5_type_id<unsigned long>()
+{
+ return H5T_NATIVE_ULONG;
+}
+template <> hid_t hdf5_tools::hdf5_type_id<long long>()
+{
+ return H5T_NATIVE_LLONG;
+}
+template <> hid_t hdf5_tools::hdf5_type_id<unsigned long long>()
+{
+ return H5T_NATIVE_ULLONG;
+}
+template <> hid_t hdf5_tools::hdf5_type_id<float>()
+{
+ return H5T_NATIVE_FLOAT;
+}
+template <> hid_t hdf5_tools::hdf5_type_id<double>()
+{
+ return H5T_NATIVE_DOUBLE;
+}
+template <> hid_t hdf5_tools::hdf5_type_id<long double>()
+{
+ return H5T_NATIVE_LDOUBLE;
+}
+
+int hdf5_tools::require_size_single_dataset(hid_t dset, int tsize)
+{
+ TIMEZONE("hdf5_tools::require_size_single_dataset");
+ int ndims;
+ hsize_t space;
+ space = H5Dget_space(dset);
+ ndims = H5Sget_simple_extent_ndims(space);
+ hsize_t *dims = new hsize_t[ndims];
+ hsize_t *maxdims = new hsize_t[ndims];
+ H5Sget_simple_extent_dims(space, dims, maxdims);
+ if (dims[0] < hsize_t(tsize) && maxdims[0] == H5S_UNLIMITED)
+ {
+ dims[0] = tsize;
+ H5Dset_extent(dset, dims);
+ }
+ H5Sclose(space);
+ delete[] maxdims;
+ delete[] dims;
+ return EXIT_SUCCESS;
+}
+
+int hdf5_tools::grow_single_dataset(hid_t dset, int tincrement)
+{
+ TIMEZONE("hdf5_tools::grow_single_dataset");
+ int ndims;
+ hsize_t space;
+ space = H5Dget_space(dset);
+ ndims = H5Sget_simple_extent_ndims(space);
+ hsize_t *dims = new hsize_t[ndims];
+ hsize_t *maxdims = new hsize_t[ndims];
+ H5Sget_simple_extent_dims(space, dims, maxdims);
+ if (maxdims[0] == H5S_UNLIMITED)
+ {
+ dims[0] += tincrement;
+ H5Dset_extent(dset, dims);
+ }
+ H5Sclose(space);
+ delete[] maxdims;
+ delete[] dims;
+ return EXIT_SUCCESS;
+}
+
+herr_t hdf5_tools::require_size_dataset_visitor(
+ hid_t o_id,
+ const char *name,
+ const H5O_info_t *info,
+ void *op_data)
+{
+ TIMEZONE("hdf5_tools::require_size_dataset_visitor");
+ if (info->type == H5O_TYPE_DATASET)
+ {
+ hsize_t dset = H5Dopen(o_id, name, H5P_DEFAULT);
+ require_size_single_dataset(dset, *((int*)(op_data)));
+ H5Dclose(dset);
+ }
+ return EXIT_SUCCESS;
+}
+
+herr_t hdf5_tools::grow_dataset_visitor(
+ hid_t o_id,
+ const char *name,
+ const H5O_info_t *info,
+ void *op_data)
+{
+ TIMEZONE("hdf5_tools::grow_dataset_visitor");
+ if (info->type == H5O_TYPE_DATASET)
+ {
+ hsize_t dset = H5Dopen(o_id, name, H5P_DEFAULT);
+ grow_single_dataset(dset, *((int*)(op_data)));
+ H5Dclose(dset);
+ }
+ return EXIT_SUCCESS;
+}
+
+
+int hdf5_tools::grow_file_datasets(
+ const hid_t stat_file,
+ const std::string group_name,
+ int tincrement)
+{
+ TIMEZONE("hdf5_tools::grow_file_datasets");
+ int file_problems = 0;
+
+ hid_t group;
+ group = H5Gopen(stat_file, group_name.c_str(), H5P_DEFAULT);
+ // in the following there's ugly code because of the HDF group
+ // see https://portal.hdfgroup.org/display/HDF5/H5O_VISIT (valid link on 2021-12-12)
+#if H5_VERSION_GE(1, 12, 0) // https://portal.hdfgroup.org/display/HDF5/H5_VERSION_GE
+ H5Ovisit(
+ group,
+ H5_INDEX_NAME,
+ H5_ITER_NATIVE,
+ grow_dataset_visitor,
+ &tincrement,
+ H5O_INFO_ALL);
+#else
+ H5Ovisit(
+ group,
+ H5_INDEX_NAME,
+ H5_ITER_NATIVE,
+ grow_dataset_visitor,
+ &tincrement);
+#endif//H5_VERSION_GE
+ H5Gclose(group);
+ return file_problems;
+}
+
+
+int hdf5_tools::require_size_file_datasets(
+ const hid_t stat_file,
+ const std::string group_name,
+ int tsize)
+{
+ TIMEZONE("hdf5_tools::require_size_file_datasets");
+ int file_problems = 0;
+
+ hid_t group;
+ group = H5Gopen(stat_file, group_name.c_str(), H5P_DEFAULT);
+#if H5_VERSION_GE(1, 12, 0)
+ H5Ovisit(
+ group,
+ H5_INDEX_NAME,
+ H5_ITER_NATIVE,
+ require_size_dataset_visitor,
+ &tsize,
+ H5O_INFO_ALL);
+#else
+ H5Ovisit(
+ group,
+ H5_INDEX_NAME,
+ H5_ITER_NATIVE,
+ require_size_dataset_visitor,
+ &tsize);
+#endif//H5_VERSION_GE
+ H5Gclose(group);
+ return file_problems;
+}
+
+template <typename number>
+std::vector<number> hdf5_tools::read_vector(
+ const hid_t group,
+ const std::string dset_name)
+{
+ TIMEZONE("hdf5_tools::read_vector");
+ std::vector<number> result;
+ hsize_t vector_length;
+ // first, read size of array
+ hid_t dset, dspace;
+ hid_t mem_dtype;
+ if (typeid(number) == typeid(int))
+ mem_dtype = H5Tcopy(H5T_NATIVE_INT);
+ else if (typeid(number) == typeid(double))
+ mem_dtype = H5Tcopy(H5T_NATIVE_DOUBLE);
+ else
+ return result;
+ dset = H5Dopen(group, dset_name.c_str(), H5P_DEFAULT);
+ dspace = H5Dget_space(dset);
+ assert(H5Sget_simple_extent_ndims(dspace) == 1);
+ H5Sget_simple_extent_dims(dspace, &vector_length, NULL);
+ result.resize(vector_length);
+ H5Dread(dset, mem_dtype, H5S_ALL, H5S_ALL, H5P_DEFAULT, &result.front());
+ H5Sclose(dspace);
+ H5Dclose(dset);
+ H5Tclose(mem_dtype);
+ return result;
+}
+
+template <typename number>
+number hdf5_tools::read_value(
+ const hid_t group,
+ const std::string dset_name)
+{
+ TIMEZONE("hdf5_tools::read_value");
+ number result;
+ hid_t dset;
+ hid_t mem_dtype;
+ if (typeid(number) == typeid(int))
+ mem_dtype = H5Tcopy(H5T_NATIVE_INT);
+ else if (typeid(number) == typeid(long long int))
+ mem_dtype = H5Tcopy(H5T_NATIVE_LLONG);
+ else if (typeid(number) == typeid(double))
+ mem_dtype = H5Tcopy(H5T_NATIVE_DOUBLE);
+ else
+ return result;
+ if (H5Lexists(group, dset_name.c_str(), H5P_DEFAULT))
+ {
+ dset = H5Dopen(group, dset_name.c_str(), H5P_DEFAULT);
+ herr_t status = H5Dread(dset, mem_dtype, H5S_ALL, H5S_ALL, H5P_DEFAULT, &result);
+ if (status < 0)
+ {
+ DEBUG_MSG("had an error trying to read %s.\n H5Dread returned status %ld\n", dset_name.c_str(), status);
+ }
+ H5Dclose(dset);
+ }
+ else
+ {
+ DEBUG_MSG("attempted to read dataset %s which does not exist.\n",
+ dset_name.c_str());
+ if (typeid(number) == typeid(int))
+ result = INT_MAX;
+ else if (typeid(number) == typeid(long long int))
+ result = -1;
+ else if (typeid(number) == typeid(double))
+ result = number(DBL_MAX);
+ }
+ H5Tclose(mem_dtype);
+ return result;
+}
+
+template <typename number>
+int hdf5_tools::write_value_with_single_rank(
+ const hid_t group,
+ const std::string dset_name,
+ const number value)
+{
+ hid_t dset, mem_dtype;
+ if (typeid(number) == typeid(int))
+ mem_dtype = H5Tcopy(H5T_NATIVE_INT);
+ else if (typeid(number) == typeid(double))
+ mem_dtype = H5Tcopy(H5T_NATIVE_DOUBLE);
+ else
+ return EXIT_FAILURE;
+ if (H5Lexists(group, dset_name.c_str(), H5P_DEFAULT))
+ {
+ dset = H5Dopen(group, dset_name.c_str(), H5P_DEFAULT);
+ assert(dset > 0);
+ }
+ else
+ {
+ hid_t fspace;
+ hsize_t count[1];
+ count[0] = 1;
+ fspace = H5Screate_simple(1, count, NULL);
+ assert(fspace > 0);
+ dset = H5Dcreate(group, dset_name.c_str(), mem_dtype, fspace, H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT);
+ assert(dset > 0);
+ H5Sclose(fspace);
+ }
+ H5Dwrite(dset, mem_dtype, H5S_ALL, H5S_ALL, H5P_DEFAULT, &value);
+ H5Dclose(dset);
+ H5Tclose(mem_dtype);
+ return EXIT_SUCCESS;
+}
+
+template <typename dtype>
+std::vector<dtype> hdf5_tools::read_vector_with_single_rank(
+ const int myrank,
+ const int rank_to_use,
+ const MPI_Comm COMM,
+ const hid_t file_id,
+ const std::string dset_name)
+{
+ TIMEZONE("hdf5_tools::read_vector_with_single_rank");
+ std::vector<dtype> data;
+ int vector_size;
+ if (myrank == rank_to_use)
+ {
+ data = hdf5_tools::read_vector<dtype>(
+ file_id,
+ dset_name);
+ vector_size = data.size();
+ }
+ MPI_Bcast(
+ &vector_size,
+ 1,
+ MPI_INT,
+ rank_to_use,
+ COMM);
+
+ if (myrank != rank_to_use)
+ data.resize(vector_size);
+ MPI_Bcast(
+ &data.front(),
+ vector_size,
+ (typeid(dtype) == typeid(int)) ? MPI_INT : MPI_DOUBLE,
+ rank_to_use,
+ COMM);
+ return data;
+}
+
+std::string hdf5_tools::read_string(
+ const hid_t group,
+ const std::string dset_name)
+{
+ TIMEZONE("hdf5_tools::read_string");
+ if (H5Lexists(group, dset_name.c_str(), H5P_DEFAULT))
+ {
+ hid_t dset = H5Dopen(group, dset_name.c_str(), H5P_DEFAULT);
+ hid_t space = H5Dget_space(dset);
+ hid_t memtype = H5Dget_type(dset);
+ // fsanitize complains unless I have a static array here
+ // but that doesn't actually work (data is read incorrectly).
+ // this is caught by bfps.test_NSVEparticles
+ char *string_data = (char*)malloc(256);
+ H5Dread(dset, memtype, H5S_ALL, H5S_ALL, H5P_DEFAULT, &string_data);
+ std::string std_string_data = std::string(string_data);
+ free(string_data);
+ H5Sclose(space);
+ H5Tclose(memtype);
+ H5Dclose(dset);
+ return std_string_data;
+ }
+ else
+ {
+ DEBUG_MSG("attempted to read dataset %s which does not exist.\n",
+ dset_name.c_str());
+ return std::string("parameter does not exist");
+ }
+}
+
+template <class partsize_t>
+int hdf5_tools::write_particle_ID_pairs_with_single_rank(
+ const std::vector<partsize_t> v,
+ const hid_t group,
+ const std::string dset_name)
+{
+ TIMEZONE("hdf5_tools::write_particle_ID_pairs_with_single_rank");
+ // the vector contains pair information, so its size must be a multiple of 2
+ assert((v.size() % 2) == 0);
+ // file space creation
+ hid_t fspace;
+ hsize_t dims[2];
+ dims[0] = v.size()/2;
+ dims[1] = 2;
+ fspace = H5Screate_simple(2, dims, NULL);
+ assert(fspace > 0);
+ // create dataset
+ hsize_t dset_id = H5Dcreate(
+ group,
+ dset_name.c_str(),
+ hdf5_tools::hdf5_type_id<partsize_t>(),
+ fspace,
+ H5P_DEFAULT,
+ H5P_DEFAULT,
+ H5P_DEFAULT);
+ assert(dset_id > 0);
+ // write data
+ H5Dwrite(
+ dset_id,
+ hdf5_tools::hdf5_type_id<partsize_t>(),
+ H5S_ALL,
+ H5S_ALL,
+ H5P_DEFAULT,
+ &v.front());
+ // clean up
+ H5Dclose(dset_id);
+ H5Sclose(fspace);
+ return EXIT_SUCCESS;
+}
+
+template <class real_number>
+int hdf5_tools::write_vector_with_single_rank(
+ const std::vector<real_number> v,
+ const hid_t group,
+ const std::string dset_name)
+{
+ TIMEZONE("hdf5_tools::write_vector_with_single_rank");
+ // file space creation
+ hid_t fspace;
+ hsize_t dim[1];
+ dim[0] = v.size();
+ fspace = H5Screate_simple(1, dim, NULL);
+ // create dataset
+ hsize_t dset_id = H5Dcreate(
+ group,
+ dset_name.c_str(),
+ hdf5_tools::hdf5_type_id<real_number>(),
+ fspace,
+ H5P_DEFAULT,
+ H5P_DEFAULT,
+ H5P_DEFAULT);
+ assert(dset_id > 0);
+ // write data
+ H5Dwrite(
+ dset_id,
+ hdf5_tools::hdf5_type_id<real_number>(),
+ H5S_ALL,
+ H5S_ALL,
+ H5P_DEFAULT,
+ &v.front());
+ // clean up
+ H5Dclose(dset_id);
+ H5Sclose(fspace);
+ return EXIT_SUCCESS;
+}
+
+template <typename number>
+int hdf5_tools::gather_and_write_with_single_rank(
+ const int myrank,
+ const int rank_to_use,
+ const MPI_Comm COMM,
+ const number *data,
+ const int number_of_items,
+ const hid_t group,
+ const std::string dset_name)
+{
+ TIMEZONE("hdf5_tools::gather_and_write_with_single_rank");
+ int nprocs;
+ MPI_Comm_size(COMM, &nprocs);
+ // first get number of items for each rank:
+ int *recvcounts = NULL;
+ if (myrank == rank_to_use)
+ recvcounts = new int[nprocs];
+ MPI_Gather(
+ &number_of_items,
+ 1,
+ MPI_INT,
+ recvcounts,
+ 1,
+ MPI_INT,
+ rank_to_use,
+ COMM);
+ // now prepare to gather data on rank_to_use
+ int total_number_of_items = 0;
+ int *displacement = NULL;
+ if (myrank == rank_to_use)
+ {
+ displacement = new int[nprocs];
+ displacement[0] = 0;
+ for(int ii = 1; ii < nprocs; ii++)
+ {
+ displacement[ii] = displacement[ii-1] + recvcounts[ii-1];
+ }
+ total_number_of_items = displacement[nprocs-1] + recvcounts[nprocs-1];
+ }
+ std::vector<number> data_to_write;
+ if (myrank == rank_to_use)
+ data_to_write.resize(total_number_of_items);
+ MPI_Gatherv(
+ data,
+ number_of_items,
+ mpi_real_type<number>::real(),
+ &data_to_write.front(),
+ recvcounts,
+ displacement,
+ mpi_real_type<number>::real(),
+ rank_to_use,
+ COMM);
+ if (myrank == rank_to_use)
+ {
+ write_vector_with_single_rank<number>(
+ data_to_write,
+ group,
+ dset_name);
+ delete[] recvcounts;
+ }
+ return EXIT_SUCCESS;
+}
+
+template
+std::vector<int> hdf5_tools::read_vector<int>(
+ const hid_t,
+ const std::string);
+
+template
+std::vector<double> hdf5_tools::read_vector<double>(
+ const hid_t,
+ const std::string);
+
+template
+std::vector<int> hdf5_tools::read_vector_with_single_rank<int>(
+ const int myrank,
+ const int rank_to_use,
+ const MPI_Comm COMM,
+ const hid_t file_id,
+ const std::string dset_name);
+
+template
+std::vector<double> hdf5_tools::read_vector_with_single_rank<double>(
+ const int myrank,
+ const int rank_to_use,
+ const MPI_Comm COMM,
+ const hid_t file_id,
+ const std::string dset_name);
+
+template
+int hdf5_tools::read_value<int>(
+ const hid_t,
+ const std::string);
+
+template
+long long int hdf5_tools::read_value<long long int>(
+ const hid_t,
+ const std::string);
+
+template
+double hdf5_tools::read_value<double>(
+ const hid_t,
+ const std::string);
+
+template
+int hdf5_tools::write_value_with_single_rank<int>(
+ const hid_t group,
+ const std::string dset_name,
+ const int value);
+
+template
+int hdf5_tools::write_value_with_single_rank<double>(
+ const hid_t group,
+ const std::string dset_name,
+ const double value);
+
+template
+int hdf5_tools::write_particle_ID_pairs_with_single_rank(
+ const std::vector<long long> v,
+ const hid_t group,
+ const std::string dset_name);
+
+template
+int hdf5_tools::write_vector_with_single_rank(
+ const std::vector<double> v,
+ const hid_t group,
+ const std::string dset_name);
+
+template
+int hdf5_tools::gather_and_write_with_single_rank<double>(
+ const int myrank,
+ const int rank_to_use,
+ const MPI_Comm COMM,
+ const double *data,
+ const int number_of_items,
+ const hid_t group,
+ const std::string dset_name);
+
+template
+int hdf5_tools::gather_and_write_with_single_rank<long long int>(
+ const int myrank,
+ const int rank_to_use,
+ const MPI_Comm COMM,
+ const long long int *data,
+ const int number_of_items,
+ const hid_t group,
+ const std::string dset_name);
+
diff --git a/bfps/cpp/hdf5_tools.hpp b/cpp/hdf5_tools.hpp
similarity index 66%
rename from bfps/cpp/hdf5_tools.hpp
rename to cpp/hdf5_tools.hpp
index 456beefe362c5d0871f8014c7a1cc468614e6374..b6068de6e4686e3fb478e28080f8235e4169d956 100644
--- a/bfps/cpp/hdf5_tools.hpp
+++ b/cpp/hdf5_tools.hpp
@@ -3,20 +3,20 @@
* Copyright 2017 Max Planck Institute *
* for Dynamics and Self-Organization *
* *
-* This file is part of bfps. *
+* This file is part of TurTLE. *
* *
-* bfps is free software: you can redistribute it and/or modify *
+* TurTLE is free software: you can redistribute it and/or modify *
* it under the terms of the GNU General Public License as published *
* by the Free Software Foundation, either version 3 of the License, *
* or (at your option) any later version. *
* *
-* bfps is distributed in the hope that it will be useful, *
+* TurTLE is distributed in the hope that it will be useful, *
* but WITHOUT ANY WARRANTY; without even the implied warranty of *
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the *
* GNU General Public License for more details. *
* *
* You should have received a copy of the GNU General Public License *
-* along with bfps. If not, see <http://www.gnu.org/licenses/> *
+* along with TurTLE. If not, see <http://www.gnu.org/licenses/> *
* *
* Contact: Cristian.Lalescu@ds.mpg.de *
* *
@@ -33,6 +33,9 @@
namespace hdf5_tools
{
+ // see https://support.hdfgroup.org/HDF5/doc/H5.user/Datatypes.html
+ template <typename data_type> hid_t hdf5_type_id();
+
int grow_single_dataset(
hid_t dset,
int tincrement);
@@ -79,6 +82,39 @@ namespace hdf5_tools
std::string read_string(
const hid_t group,
const std::string dset_name);
+
+ template <typename number>
+ number read_value(
+ const hid_t group,
+ const std::string dset_name);
+
+ template <typename number>
+ int write_value_with_single_rank(
+ const hid_t group,
+ const std::string dset_name,
+ const number value);
+
+ template <class partsize_t>
+ int write_particle_ID_pairs_with_single_rank(
+ const std::vector<partsize_t> v,
+ const hid_t group,
+ const std::string dset_name);
+
+ template <class real_number>
+ int write_vector_with_single_rank(
+ const std::vector<real_number> v,
+ const hid_t group,
+ const std::string dset_name);
+
+ template <typename number>
+ int gather_and_write_with_single_rank(
+ const int myrank,
+ const int rank_to_use,
+ const MPI_Comm COMM,
+ const number *data,
+ const int number_of_items,
+ const hid_t group,
+ const std::string dset_name);
}
#endif//HDF5_TOOLS_HPP
diff --git a/cpp/kspace.cpp b/cpp/kspace.cpp
new file mode 100644
index 0000000000000000000000000000000000000000..b1f40ae446e469888ae645d0dd751aae4635817d
--- /dev/null
+++ b/cpp/kspace.cpp
@@ -0,0 +1,1429 @@
+/**********************************************************************
+* *
+* Copyright 2015 Max Planck Institute *
+* for Dynamics and Self-Organization *
+* *
+* This file is part of TurTLE. *
+* *
+* TurTLE is free software: you can redistribute it and/or modify *
+* it under the terms of the GNU General Public License as published *
+* by the Free Software Foundation, either version 3 of the License, *
+* or (at your option) any later version. *
+* *
+* TurTLE is distributed in the hope that it will be useful, *
+* but WITHOUT ANY WARRANTY; without even the implied warranty of *
+* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the *
+* GNU General Public License for more details. *
+* *
+* You should have received a copy of the GNU General Public License *
+* along with TurTLE. If not, see <http://www.gnu.org/licenses/> *
+* *
+* Contact: Cristian.Lalescu@ds.mpg.de *
+* *
+**********************************************************************/
+
+
+
+#include <cmath>
+#include <cstdlib>
+#include <algorithm>
+#include <cassert>
+#include <stdexcept>
+#include "kspace.hpp"
+#include "scope_timer.hpp"
+#include "shared_array.hpp"
+
+template <field_backend be,
+ kspace_dealias_type dt>
+template <field_components fc>
+kspace<be, dt>::kspace(
+ const field_layout<fc> *source_layout,
+ const double DKX,
+ const double DKY,
+ const double DKZ)
+{
+ TIMEZONE("kspace::kspace");
+ /* get layout */
+ this->layout = new field_layout<ONE>(
+ source_layout->sizes,
+ source_layout->subsizes,
+ source_layout->starts,
+ source_layout->comm);
+
+ /* store dk values */
+ this->dkx = DKX;
+ this->dky = DKY;
+ this->dkz = DKZ;
+
+ /* compute kx, ky, kz and compute kM values */
+ switch(be)
+ {
+ case FFTW:
+ this->kx.resize(this->layout->sizes[2]);
+ this->ky.resize(this->layout->subsizes[0]);
+ this->kz.resize(this->layout->sizes[1]);
+ int i, ii;
+ for (i = 0; i<int(this->layout->sizes[2]); i++)
+ this->kx[i] = i*this->dkx;
+ for (i = 0; i<int(this->layout->subsizes[0]); i++)
+ {
+ ii = i + this->layout->starts[0];
+ if (ii <= int(this->layout->sizes[0]/2))
+ this->ky[i] = this->dky*ii;
+ else
+ this->ky[i] = this->dky*(ii - int(this->layout->sizes[0]));
+ }
+ for (i = 0; i<int(this->layout->sizes[1]); i++)
+ {
+ if (i <= int(this->layout->sizes[1]/2))
+ this->kz[i] = this->dkz*i;
+ else
+ this->kz[i] = this->dkz*(i - int(this->layout->sizes[1]));
+ }
+ switch(dt)
+ {
+ case ONE_HALF:
+ this->kMx = this->dkx*(int(2*(int(this->layout->sizes[2])-1)/4)-1);
+ this->kMy = this->dky*(int(this->layout->sizes[0] / 4)-1);
+ this->kMz = this->dkz*(int(this->layout->sizes[1] / 4)-1);
+ break;
+ case TWO_THIRDS:
+ this->kMx = this->dkx*(int(2*(int(this->layout->sizes[2])-1)/3)-1);
+ this->kMy = this->dky*(int(this->layout->sizes[0] / 3)-1);
+ this->kMz = this->dkz*(int(this->layout->sizes[1] / 3)-1);
+ break;
+ case SMOOTH:
+ this->kMx = this->dkx*(int(this->layout->sizes[2])-2);
+ this->kMy = this->dky*(int(this->layout->sizes[0] / 2)-1);
+ this->kMz = this->dkz*(int(this->layout->sizes[1] / 2)-1);
+ break;
+ }
+ break;
+ }
+
+ /* get global kM and dk */
+ this->kM = this->kMx;
+ if (this->kM < this->kMy) this->kM = this->kMy;
+ if (this->kM < this->kMz) this->kM = this->kMz;
+ this->kM2 = this->kM * this->kM;
+ this->dk = this->dkx;
+ if (this->dk > this->dky) this->dk = this->dky;
+ if (this->dk > this->dkz) this->dk = this->dkz;
+ this->dk2 = this->dk*this->dk;
+
+ /* spectra stuff */
+ this->nshells = int(this->kM / this->dk) + 2;
+ this->kshell.resize(this->nshells, 0);
+ this->nshell.resize(this->nshells, 0);
+
+ shared_array<double> kshell_local_thread(this->nshells,[&](double* kshell_local){
+ std::fill_n(kshell_local, this->nshells, 0);
+ });
+ shared_array<int64_t> nshell_local_thread(this->nshells,[&](int64_t* nshell_local){
+ std::fill_n(nshell_local, this->nshells, 0);
+ });
+
+ this->CLOOP_K2_NXMODES(
+ [&](const ptrdiff_t cindex,
+ const ptrdiff_t xindex,
+ const ptrdiff_t yindex,
+ const ptrdiff_t zindex,
+ const double k2,
+ const int nxmodes){
+ if (k2 < this->kM2)
+ {
+ const double knorm = sqrt(k2);
+ kshell_local_thread.getMine()[int(knorm/this->dk)] += nxmodes*knorm;
+ nshell_local_thread.getMine()[int(knorm/this->dk)] += nxmodes;
+ }
+ });
+
+ // Merge results
+
+ kshell_local_thread.mergeParallel();
+ nshell_local_thread.mergeParallel();
+
+ MPI_Allreduce(
+ nshell_local_thread.getMasterData(),
+ &this->nshell.front(),
+ this->nshells,
+ MPI_INT64_T, MPI_SUM, this->layout->comm);
+ MPI_Allreduce(
+ kshell_local_thread.getMasterData(),
+ &this->kshell.front(),
+ this->nshells,
+ MPI_DOUBLE, MPI_SUM, this->layout->comm);
+
+ // 2020-08-07
+ // following loop generated some problems with intel compiler
+ // at highest optimization level, the intel compiler performs aggressive
+ // vectorization of loops, including this one.
+ // it also turns on speculative execution, i.e. it computes both branches
+ // of the if clause in parallel, and then it picks out the useful result.
+ // the problem is that one of the branches is a division by 0, hence a
+ // floating point exception is raised.
+ // there are several possible solutions:
+ // * instead of dividing by `this->nshell[n]`, create a double `nnshell`
+ // that takes the maximum value between 1.0 and `this->nshell[n]`, and
+ // then divide by `nnshell`
+ // * use the '-fp-speculation=safe' intel compiler option
+ // * tell the compiler that we do not want to vectorize this particular
+ // loop by using `#pragma novector`.
+ // I chose the last option because there's no reason to optimize this
+ // loop. Furthermore, it seems like the solution that's most readable,
+ // and with the least amount of side effects.
+ # pragma novector
+ for (int n=0; n<this->nshells; n++){
+ if (this->nshell[n] > 0)
+ this->kshell[n] /= this->nshell[n];
+ }
+}
+
+template <field_backend be,
+ kspace_dealias_type dt>
+kspace<be, dt>::~kspace()
+{
+ delete this->layout;
+}
+
+template <field_backend be,
+ kspace_dealias_type dt>
+int kspace<be, dt>::store(hid_t stat_file)
+{
+ TIMEZONE("kspace::store");
+ assert(this->layout->myrank == 0);
+ hsize_t dims[4];
+ hid_t space, dset;
+ // store kspace information
+ dset = H5Dopen(stat_file, "/kspace/kshell", H5P_DEFAULT);
+ space = H5Dget_space(dset);
+ H5Sget_simple_extent_dims(space, dims, NULL);
+ H5Sclose(space);
+ if (this->nshells != int(dims[0]))
+ {
+ DEBUG_MSG(
+ "ERROR: computed nshells %d not equal to data file nshells %d for dealiasing %d\n",
+ this->nshells, dims[0], dt);
+ throw std::runtime_error("Computed nshells not equal to data file nshells. Something is probably wrong with the dealiasing option.\n");
+ }
+ H5Dwrite(
+ dset,
+ H5T_NATIVE_DOUBLE,
+ H5S_ALL,
+ H5S_ALL,
+ H5P_DEFAULT,
+ &this->kshell.front());
+ H5Dclose(dset);
+ dset = H5Dopen(
+ stat_file,
+ "/kspace/nshell",
+ H5P_DEFAULT);
+ H5Dwrite(
+ dset,
+ H5T_NATIVE_INT64,
+ H5S_ALL,
+ H5S_ALL,
+ H5P_DEFAULT,
+ &this->nshell.front());
+ H5Dclose(dset);
+ dset = H5Dopen(stat_file, "/kspace/kM", H5P_DEFAULT);
+ H5Dwrite(
+ dset,
+ H5T_NATIVE_DOUBLE,
+ H5S_ALL,
+ H5S_ALL,
+ H5P_DEFAULT,
+ &this->kM);
+ H5Dclose(dset);
+ dset = H5Dopen(stat_file, "/kspace/dk", H5P_DEFAULT);
+ H5Dwrite(dset,
+ H5T_NATIVE_DOUBLE,
+ H5S_ALL,
+ H5S_ALL,
+ H5P_DEFAULT,
+ &this->dk);
+ H5Dclose(dset);
+ return EXIT_SUCCESS;
+}
+
+template <field_backend be,
+ kspace_dealias_type dt>
+template <typename rnumber,
+ field_components fc>
+void kspace<be, dt>::low_pass(
+ typename fftw_interface<rnumber>::complex *__restrict__ a,
+ const double kmax)
+{
+ const double km2 = kmax*kmax;
+ this->CLOOP_K2(
+ [&](const ptrdiff_t cindex,
+ const ptrdiff_t xindex,
+ const ptrdiff_t yindex,
+ const ptrdiff_t zindex,
+ const double k2){
+ if (k2 >= km2)
+ std::fill_n((rnumber*)(a + ncomp(fc)*cindex), 2*ncomp(fc), 0);
+ });
+}
+
+/** \brief Filter a field using a ball shaped top hat filter.
+ *
+ * Filter's mathematical expression in real space is as follows:
+ * \f[
+ * \phi^b_\ell(r) =
+ * \frac{1}{\ell^3}\frac{6}{\pi} H(\ell/2 - r)
+ * \f]
+ * with the corresponding Fourier space expression:
+ * \f[
+ * \hat{\phi^b_\ell}(k) =
+ * \frac{3}{2(k\ell/2)^3}
+ * \left(2\sin (k \ell/2) - k \ell \cos (k \ell/2)\right)
+ * \f]
+ */
+template <field_backend be,
+ kspace_dealias_type dt>
+template <typename rnumber,
+ field_components fc>
+void kspace<be, dt>::ball_filter(
+ typename fftw_interface<rnumber>::complex *__restrict__ a,
+ const double ell)
+{
+ const double prefactor0 = double(3) / pow(ell/2, 3);
+ this->CLOOP_K2(
+ [&](const ptrdiff_t cindex,
+ const ptrdiff_t xindex,
+ const ptrdiff_t yindex,
+ const ptrdiff_t zindex,
+ const double k2){
+ if (k2 > 0)
+ {
+ const double argument = sqrt(k2)*ell / 2;
+ const double prefactor = prefactor0 / pow(k2, 1.5);
+ for (unsigned int tcounter=0; tcounter<2*ncomp(fc); tcounter++)
+ ((rnumber*)a)[2*ncomp(fc)*cindex + tcounter] *= (
+ prefactor *
+ (sin(argument) - argument * cos(argument)));
+ }
+ });
+}
+
+/** \brief Filter a field using a M-filter to reproduce dissipation range.
+ *
+ * Filter's Fourier space expression:
+ * \f[
+ * \hat{\phi^M_\ell}(k) =
+ * \exp(-\frac{(3.54 k \ell)^{122 \ell^{0.0836}}}{2})
+ * \left( 1 + \frac{(k \eta/0.0636)^{3.44}}{1 + (k \eta/ 0.0621)^{3.44}} \right)^{1/2}
+ * \f]
+ */
+template <field_backend be,
+ kspace_dealias_type dt>
+template <typename rnumber,
+ field_components fc>
+void kspace<be, dt>::general_M_filter(
+ typename fftw_interface<rnumber>::complex *__restrict__ a,
+ const double ell)
+{
+ const double prefactor0 = 1.0;
+ this->CLOOP_K2(
+ [&](const ptrdiff_t cindex,
+ const ptrdiff_t xindex,
+ const ptrdiff_t yindex,
+ const ptrdiff_t zindex,
+ const double k2){
+ if (k2 > 0)
+ {
+ const double argument = sqrt(k2)*ell;
+ const double prefactor = prefactor0;
+ for (unsigned int tcounter=0; tcounter<2*ncomp(fc); tcounter++)
+ ((rnumber*)a)[2*ncomp(fc)*cindex + tcounter] *= (
+// prefactor * (exp(-0.5*pow((2.9*argument),(68.0*(pow(ell,0.74))))) * sqrt(1.0 + (pow((argument/0.06),3.8))/(1.0 + (pow((argument/0.057),3.8))))));
+ prefactor * (exp(-1.7*argument)) * sqrt(1.0 + 0.68*(pow((argument/0.065),4.6))/(1.0 + (pow((argument/0.065),4.6)))));
+ }
+ });
+}
+
+
+/** \brief Filter a field using a Gaussian kernel.
+ *
+ * \tparam rnumber type of real number, float or double.
+ * \tparam fc field components, ONE, THREE or THREExTHREE.
+ * \return nothing
+ *
+ * Filter's mathematical expression in Fourier space is as follows:
+ * \f[
+ * \hat{g}_\ell(\mathbf{k}) = \exp(-k^2 \sigma^2 / 2)
+ * \f]
+ */
+template <field_backend be,
+ kspace_dealias_type dt>
+template <typename rnumber,
+ field_components fc>
+void kspace<be, dt>::Gauss_filter(
+ typename fftw_interface<rnumber>::complex *__restrict__ a,
+ const double sigma)
+{
+ const double prefactor = - sigma*sigma/2;
+ this->CLOOP_K2(
+ [&](const ptrdiff_t cindex,
+ const ptrdiff_t xindex,
+ const ptrdiff_t yindex,
+ const ptrdiff_t zindex,
+ const double k2){
+ {
+ for (unsigned int tcounter=0; tcounter<2*ncomp(fc); tcounter++)
+ ((rnumber*)a)[2*ncomp(fc)*cindex + tcounter] *= exp(prefactor*k2);
+ }
+ });
+}
+
+/** \brief Filter a field.
+ *
+ * This is a wrapper that can choose between a sharp Fourier spherical filter,
+ * a Gaussian filter and a sharp real space spherical filter.
+ *
+ * Filter expressions in real space are as follows:
+ * \f[
+ * \phi^b_\ell(r) =
+ * \frac{1}{\ell^3}\frac{6}{\pi} H(\ell/2 - r)
+ * \f]
+ * \f[
+ * \phi^g_\ell(r) =
+ * \frac{1}{\sigma_\ell^3}\frac{1}{(2\pi)^{3/2}}
+ * \exp\left(-\frac{1}{2}\left(\frac{r}{\sigma_\ell}\right)^2\right)
+ * \f]
+ * \f[
+ * \phi^s_\ell(r) =
+ * \frac{1}{2 \pi^2 r^3}
+ * \left(\sin k_\ell r - k_\ell r \cos k_\ell r\right)
+ * \f]
+ * and the corresponding expressions in Fourier space are:
+ * \f[
+ * \hat{\phi^b_\ell}(k) =
+ * \frac{3}{2(k\ell/2)^3}
+ * \left(2\sin (k \ell/2) - k \ell \cos (k \ell/2)\right)
+ * \f]
+ * \f[
+ * \hat{\phi^g_\ell}(k) =
+ * \exp\left(-\frac{1}{2}k^2 \sigma_\ell^2\right)
+ * \f]
+ * \f[
+ * \hat{\phi^s_\ell}(k) = H(k_\ell - k)
+ * \f]
+ *
+ * \f$ k_\ell \f$ is given as a parameter, and then we use
+ * \f[
+ * \ell = \pi / k_\ell,
+ * \sigma_\ell = \pi / k_\ell
+ * \f]
+ *
+ * For the Gaussian filter this is the same convention used in
+ * \cite Buzzicotti2017 .
+ *
+ * See also `filter_calibrated_ell`.
+ */
+template <field_backend be,
+ kspace_dealias_type dt>
+template <typename rnumber,
+ field_components fc>
+int kspace<be, dt>::filter(
+ typename fftw_interface<rnumber>::complex *__restrict__ a,
+ const double wavenumber,
+ std::string filter_type)
+{
+ if (filter_type == std::string("sharp_Fourier_sphere"))
+ {
+ this->template low_pass<rnumber, fc>(
+ a,
+ wavenumber);
+ }
+ else if (filter_type == std::string("Gauss"))
+ {
+ this->template Gauss_filter<rnumber, fc>(
+ a,
+ 2*acos(0.)/wavenumber);
+ }
+ else if (filter_type == std::string("ball"))
+ {
+ this->template ball_filter<rnumber, fc>(
+ a,
+ 2*acos(0.)/wavenumber);
+ }
+ else if (filter_type == std::string("general_M"))
+ {
+ this->template general_M_filter<rnumber, fc>(
+ a,
+ 2*acos(0.)/wavenumber);
+ }
+ return EXIT_SUCCESS;
+}
+
+/** \brief Filter a field.
+ *
+ * This is a wrapper that can choose between a sharp Fourier spherical filter,
+ * a Gaussian filter and a sharp real space spherical filter.
+ *
+ * Filter expressions in real space are as follows:
+ * \rst
+ * .. math::
+ * :nowrap:
+ *
+ * \begin{eqnarray*}
+ * \phi^b_\ell(r) &=&
+ * \frac{1}{\ell^3}\frac{6}{\pi} H(\ell/2 - r) \\
+ * \phi^g_\ell(r) &=&
+ * \frac{1}{\sigma_\ell^3}\frac{1}{(2\pi)^{3/2}}
+ * \exp\left(-\frac{1}{2}\left(\frac{r}{\sigma_\ell}\right)^2\right) \\
+ * \phi^s_\ell(r) &=&
+ * \frac{1}{2 \pi^2 r^3}
+ * \left(\sin k_\ell r - k_\ell r \cos k_\ell r\right)
+ * \end{eqnarray*}
+ *
+ * \endrst
+ *
+ * and the corresponding expressions in Fourier space are:
+ * \rst
+ * .. math::
+ * :nowrap:
+ *
+ * \begin{eqnarray*}
+ * \hat{\phi^b_\ell}(k) &=&
+ * \frac{3}{2(k\ell/2)^3}
+ * \left(2\sin (k \ell/2) - k \ell \cos (k \ell/2)\right) \\
+ * \hat{\phi^g_\ell}(k) &=&
+ * \exp\left(-\frac{1}{2}k^2 \sigma_\ell^2\right) \\
+ * \hat{\phi^s_\ell}(k) &=& H(k_\ell - k)
+ * \end{eqnarray*}
+ *
+ * \endrst
+ *
+ * \f$\sigma_\ell\f$ and \f$k_\ell\f$ are calibrated such that the energy of
+ * the large scales is approximately the same (within the inertial range)
+ * independently of the shape of the filter.
+ *
+ * This was done by hand, see [lalescu2018jfm]_ for details, with the
+ * results:
+ *
+ * \f[
+ * \sigma_\ell = 0.23 \ell,
+ * k_\ell = 2.8 / \ell
+ * \f]
+ *
+ */
+template <field_backend be,
+ kspace_dealias_type dt>
+template <typename rnumber,
+ field_components fc>
+int kspace<be, dt>::filter_calibrated_ell(
+ typename fftw_interface<rnumber>::complex *__restrict__ a,
+ const double ell,
+ std::string filter_type)
+{
+ TIMEZONE("kspace::filter_calibrated_ell");
+ if (filter_type == std::string("sharp_Fourier_sphere"))
+ {
+ this->template low_pass<rnumber, fc>(
+ a,
+ 2.8 / ell);
+ }
+ else if (filter_type == std::string("Gauss"))
+ {
+ this->template Gauss_filter<rnumber, fc>(
+ a,
+ 0.23*ell);
+ }
+ else if (filter_type == std::string("ball"))
+ {
+ this->template ball_filter<rnumber, fc>(
+ a,
+ ell);
+ }
+ else if (filter_type == std::string("general_M"))
+ {
+ this->template general_M_filter<rnumber, fc>(
+ a,
+ ell);
+ }
+ return EXIT_SUCCESS;
+}
+
+template <field_backend be,
+ kspace_dealias_type dt>
+template <typename rnumber,
+ field_components fc>
+void kspace<be, dt>::dealias(typename fftw_interface<rnumber>::complex *__restrict__ a)
+{
+ TIMEZONE("kspace::dealias");
+ switch(dt)
+ {
+ case ONE_HALF:
+ this->low_pass<rnumber, fc>(a, this->kM);
+ break;
+ case TWO_THIRDS:
+ this->low_pass<rnumber, fc>(a, this->kM);
+ break;
+ case SMOOTH:
+ this->CLOOP_simd(
+ [&](const ptrdiff_t cindex,
+ const ptrdiff_t xindex,
+ const ptrdiff_t yindex,
+ const ptrdiff_t zindex){
+ const double kk2 = (pow(this->kx[xindex]/this->kMx, 2) +
+ pow(this->ky[yindex]/this->kMy, 2) +
+ pow(this->kz[zindex]/this->kMz, 2));
+ const double tval = exp(-36.0 * (pow(kk2, 18)));
+ for (unsigned int tcounter=0; tcounter<2*ncomp(fc); tcounter++)
+ ((rnumber*)a)[2*ncomp(fc)*cindex + tcounter] *= tval;
+ });
+ break;
+ }
+}
+
+template <field_backend be,
+ kspace_dealias_type dt>
+template <typename rnumber>
+void kspace<be, dt>::project_divfree(
+ typename fftw_interface<rnumber>::complex *__restrict__ a,
+ const bool maintain_energy)
+{
+ TIMEZONE("kspace::project_divfree");
+ this->CLOOP_K2(
+ [&](const ptrdiff_t cindex,
+ const ptrdiff_t xindex,
+ const ptrdiff_t yindex,
+ const ptrdiff_t zindex,
+ const double k2){
+ if (k2 > 0)
+ {
+ const typename fftw_interface<rnumber>::complex tval = {
+ rnumber((this->kx[xindex]*((*(a + cindex*3 ))[0]) +
+ this->ky[yindex]*((*(a + cindex*3+1))[0]) +
+ this->kz[zindex]*((*(a + cindex*3+2))[0]) ) / k2),
+ rnumber((this->kx[xindex]*((*(a + cindex*3 ))[1]) +
+ this->ky[yindex]*((*(a + cindex*3+1))[1]) +
+ this->kz[zindex]*((*(a + cindex*3+2))[1]) ) / k2)};
+ double initial_size = 1;
+ double projected_size = 1;
+ if (maintain_energy)
+ {
+ initial_size = sqrt(
+ ((*(a + cindex*3 ))[0])*((*(a + cindex*3 ))[0]) +
+ ((*(a + cindex*3+1))[0])*((*(a + cindex*3+1))[0]) +
+ ((*(a + cindex*3+2))[0])*((*(a + cindex*3+2))[0]) +
+ ((*(a + cindex*3 ))[1])*((*(a + cindex*3 ))[1]) +
+ ((*(a + cindex*3+1))[1])*((*(a + cindex*3+1))[1]) +
+ ((*(a + cindex*3+2))[1])*((*(a + cindex*3+2))[1]));
+ }
+ for (int imag_part=0; imag_part<2; imag_part++)
+ {
+ a[cindex*3 ][imag_part] -= tval[imag_part]*this->kx[xindex];
+ a[cindex*3+1][imag_part] -= tval[imag_part]*this->ky[yindex];
+ a[cindex*3+2][imag_part] -= tval[imag_part]*this->kz[zindex];
+ }
+ if (maintain_energy)
+ {
+ projected_size = sqrt(
+ ((*(a + cindex*3 ))[0])*((*(a + cindex*3 ))[0]) +
+ ((*(a + cindex*3+1))[0])*((*(a + cindex*3+1))[0]) +
+ ((*(a + cindex*3+2))[0])*((*(a + cindex*3+2))[0]) +
+ ((*(a + cindex*3 ))[1])*((*(a + cindex*3 ))[1]) +
+ ((*(a + cindex*3+1))[1])*((*(a + cindex*3+1))[1]) +
+ ((*(a + cindex*3+2))[1])*((*(a + cindex*3+2))[1]));
+ if (projected_size > 0)
+ #pragma omp simd
+ for (int component=0; component<3; component++)
+ for (int imag_part=0; imag_part<2; imag_part++)
+ {
+ (*(a + cindex*3+component))[imag_part] *= initial_size / projected_size;
+ }
+ }
+ }
+ }
+ );
+ if (this->layout->myrank == this->layout->rank[0][0])
+ std::fill_n((rnumber*)(a), 6, 0.0);
+}
+
+/** \brief Rotate vector modes perpendicular to wavenumber
+ * This is different from project because it maintains the energy of the field,
+ * I want it in order to be able to generate random fields with prescribed
+ * spectra.
+ */
+template <field_backend be,
+ kspace_dealias_type dt>
+template <typename rnumber>
+void kspace<be, dt>::rotate_divfree(typename fftw_interface<rnumber>::complex *__restrict__ a)
+{
+ TIMEZONE("kspace::rotate_divfree");
+ this->CLOOP_K2_simd(
+ [&](const ptrdiff_t cindex,
+ const ptrdiff_t xindex,
+ const ptrdiff_t yindex,
+ const ptrdiff_t zindex,
+ const double k2){
+ if (k2 > 0)
+ {
+ double cosTheta;
+ double usize;
+ for (int cc=0; cc<2; cc++)
+ {
+ // compute dot product
+ cosTheta = (this->kx[xindex]*((*(a + cindex*3 ))[cc]) +
+ this->ky[yindex]*((*(a + cindex*3+1))[cc]) +
+ this->kz[zindex]*((*(a + cindex*3+2))[cc]) );
+ // now compute size of initial velocity vector
+ usize = sqrt(pow((*(a + cindex*3 ))[cc], 2) +
+ pow((*(a + cindex*3+1))[cc], 2) +
+ pow((*(a + cindex*3+2))[cc], 2));
+ // finalize computation of cos Theta
+ if (usize != 0)
+ {
+ cosTheta /= (usize * sqrt(k2));
+ // now compute cross product
+ // cross product for complex vectors is complex conjugate of regular cross product
+ double cp[3];
+ cp[0] = (*(a + cindex*3+1))[cc]*this->kz[zindex] - (*(a + cindex*3+2))[cc]*this->ky[yindex];
+ cp[1] = (*(a + cindex*3+2))[cc]*this->kx[xindex] - (*(a + cindex*3+0))[cc]*this->kz[zindex];
+ cp[2] = (*(a + cindex*3+0))[cc]*this->ky[yindex] - (*(a + cindex*3+1))[cc]*this->kx[xindex];
+ double cpsize = sqrt(cp[0]*cp[0] + cp[1]*cp[1] + cp[2]*cp[2]);
+ cp[0] /= cpsize;
+ cp[1] /= cpsize;
+ cp[2] /= cpsize;
+ double sinTheta = sqrt(1 - cosTheta*cosTheta);
+ // we are actually interested in rotating the vector with pi/2 - Theta
+ double tmpdouble = cosTheta;
+ cosTheta = sinTheta;
+ sinTheta = -tmpdouble;
+ // store initial velocity
+ double u[3];
+ u[0] = (*(a + cindex*3+0))[cc];
+ u[1] = (*(a + cindex*3+1))[cc];
+ u[2] = (*(a + cindex*3+2))[cc];
+ // store final result
+ (*(a + cindex*3+0))[cc] = u[0]*cosTheta + (cp[1]*u[2] - cp[2]*u[1])*sinTheta;
+ (*(a + cindex*3+1))[cc] = u[1]*cosTheta + (cp[2]*u[0] - cp[0]*u[2])*sinTheta;
+ (*(a + cindex*3+2))[cc] = u[2]*cosTheta + (cp[0]*u[1] - cp[1]*u[0])*sinTheta;
+ }
+ }}
+ });
+ if (this->layout->myrank == this->layout->rank[0][0])
+ std::fill_n((rnumber*)(a), 6, 0.0);
+}
+
+template <field_backend be,
+ kspace_dealias_type dt>
+template <typename rnumber,
+ field_components fc>
+void kspace<be, dt>::cospectrum(
+ const rnumber(* __restrict a)[2],
+ const rnumber(* __restrict b)[2],
+ const hid_t group,
+ const std::string dset_name,
+ const hsize_t toffset,
+ const double wavenumber_exp)
+{
+ TIMEZONE("field::cospectrum2");
+ shared_array<double> spec_local_thread(this->nshells*ncomp(fc)*ncomp(fc),[&](double* spec_local){
+ std::fill_n(spec_local, this->nshells*ncomp(fc)*ncomp(fc), 0);
+ });
+
+ this->CLOOP_K2_NXMODES(
+ [&](const ptrdiff_t cindex,
+ const ptrdiff_t xindex,
+ const ptrdiff_t yindex,
+ const ptrdiff_t zindex,
+ const double k2,
+ const int nxmodes){
+ if (k2 <= this->kM2)
+ {
+ double* spec_local = spec_local_thread.getMine();
+ const int tmp_int = int(sqrt(k2) / this->dk)*ncomp(fc)*ncomp(fc);
+ for (hsize_t i=0; i<ncomp(fc); i++)
+ for (hsize_t j=0; j<ncomp(fc); j++){
+ spec_local[tmp_int + i*ncomp(fc)+j] += pow(k2,wavenumber_exp/2.)*
+ nxmodes * (
+ (a[ncomp(fc)*cindex + i][0] * b[ncomp(fc)*cindex + j][0]) +
+ (a[ncomp(fc)*cindex + i][1] * b[ncomp(fc)*cindex + j][1]));
+ }
+ }
+ });
+
+ spec_local_thread.mergeParallel();
+
+ std::vector<double> spec;
+ spec.resize(this->nshells*ncomp(fc)*ncomp(fc), 0);
+ MPI_Allreduce(
+ spec_local_thread.getMasterData(),
+ &spec.front(),
+ spec.size(),
+ MPI_DOUBLE, MPI_SUM, this->layout->comm);
+ if (this->layout->myrank == 0)
+ {
+ hid_t dset, wspace, mspace;
+ hsize_t count[(ndim(fc)-2)*2], offset[(ndim(fc)-2)*2], dims[(ndim(fc)-2)*2];
+ dset = H5Dopen(group, ("spectra/" + dset_name).c_str(), H5P_DEFAULT);
+ wspace = H5Dget_space(dset);
+ H5Sget_simple_extent_dims(wspace, dims, NULL);
+ switch (fc)
+ {
+ case THREExTHREE:
+ offset[4] = 0;
+ offset[5] = 0;
+ count[4] = 3;
+ count[5] = 3;
+ case THREE:
+ offset[2] = 0;
+ offset[3] = 0;
+ count[2] = 3;
+ count[3] = 3;
+ default:
+ offset[0] = toffset;
+ offset[1] = 0;
+ count[0] = 1;
+ count[1] = this->nshells;
+ }
+ mspace = H5Screate_simple((ndim(fc)-2)*2, count, NULL);
+ H5Sselect_hyperslab(wspace, H5S_SELECT_SET, offset, NULL, count, NULL);
+ H5Dwrite(dset, H5T_NATIVE_DOUBLE, mspace, wspace, H5P_DEFAULT, &spec.front());
+ H5Sclose(wspace);
+ H5Sclose(mspace);
+ H5Dclose(dset);
+ }
+}
+
+template <field_backend be,
+ kspace_dealias_type dt>
+template <typename rnumber,
+ field_components fc>
+void kspace<be, dt>::cospectrum(
+ const rnumber(* __restrict a)[2],
+ const hid_t group,
+ const std::string dset_name,
+ const hsize_t toffset,
+ const double wavenumber_exp)
+{
+ TIMEZONE("field::cospectrum1");
+
+ std::vector<double> spec;
+ this->template cospectrum<rnumber, fc>(a, spec, wavenumber_exp);
+
+ if (this->layout->myrank == 0)
+ {
+ hid_t dset, wspace, mspace;
+ hsize_t count[(ndim(fc)-2)*2], offset[(ndim(fc)-2)*2], dims[(ndim(fc)-2)*2];
+ dset = H5Dopen(group, ("spectra/" + dset_name).c_str(), H5P_DEFAULT);
+ wspace = H5Dget_space(dset);
+ H5Sget_simple_extent_dims(wspace, dims, NULL);
+ switch (fc)
+ {
+ case THREExTHREE:
+ offset[4] = 0;
+ offset[5] = 0;
+ count[4] = 3;
+ count[5] = 3;
+ case THREE:
+ offset[2] = 0;
+ offset[3] = 0;
+ count[2] = 3;
+ count[3] = 3;
+ default:
+ offset[0] = toffset;
+ offset[1] = 0;
+ count[0] = 1;
+ count[1] = this->nshells;
+ }
+ mspace = H5Screate_simple((ndim(fc)-2)*2, count, NULL);
+ H5Sselect_hyperslab(wspace, H5S_SELECT_SET, offset, NULL, count, NULL);
+ H5Dwrite(dset, H5T_NATIVE_DOUBLE, mspace, wspace, H5P_DEFAULT, &spec.front());
+ H5Sclose(wspace);
+ H5Sclose(mspace);
+ H5Dclose(dset);
+ }
+}
+
+template <field_backend be,
+ kspace_dealias_type dt>
+template <typename rnumber,
+ field_components fc>
+void kspace<be, dt>::cospectrum(
+ const rnumber(* __restrict a)[2],
+ std::vector<double> &spec,
+ const double wavenumber_exp)
+{
+ TIMEZONE("field::cospectrum1_in_memory");
+ shared_array<double> spec_local_thread(this->nshells*ncomp(fc)*ncomp(fc),[&](double* spec_local){
+ std::fill_n(spec_local, this->nshells*ncomp(fc)*ncomp(fc), 0);
+ });
+
+ this->CLOOP_K2_NXMODES(
+ [&](const ptrdiff_t cindex,
+ const ptrdiff_t xindex,
+ const ptrdiff_t yindex,
+ const ptrdiff_t zindex,
+ const double k2,
+ const int nxmodes){
+ if (k2 <= this->kM2)
+ {
+ double* spec_local = spec_local_thread.getMine();
+ const int tmp_int = int(sqrt(k2) / this->dk)*ncomp(fc)*ncomp(fc);
+
+ for (hsize_t i=0; i<ncomp(fc); i++)
+ for (hsize_t j=0; j<ncomp(fc); j++){
+ spec_local[tmp_int + i*ncomp(fc)+j] += pow(k2, wavenumber_exp/2.)*
+ nxmodes * (
+ (a[ncomp(fc)*cindex + i][0] * a[ncomp(fc)*cindex + j][0]) +
+ (a[ncomp(fc)*cindex + i][1] * a[ncomp(fc)*cindex + j][1]));
+ }
+ }
+ });
+
+ spec_local_thread.mergeParallel();
+
+ spec.resize(this->nshells*ncomp(fc)*ncomp(fc), 0);
+ MPI_Allreduce(
+ spec_local_thread.getMasterData(),
+ &spec.front(),
+ spec.size(),
+ MPI_DOUBLE, MPI_SUM, this->layout->comm);
+}
+
+template <field_backend be,
+ kspace_dealias_type dt>
+template <typename rnumber,
+ field_components fc>
+double kspace<be, dt>::L2norm(
+ const rnumber(* __restrict a)[2])
+{
+ TIMEZONE("field::L2norm");
+ shared_array<double> L2_local_thread(1,[&](double* spec_local){
+ std::fill_n(spec_local, 1, 0);
+ });
+
+ this->CLOOP_K2_NXMODES(
+ [&](const ptrdiff_t cindex,
+ const ptrdiff_t xindex,
+ const ptrdiff_t yindex,
+ const ptrdiff_t zindex,
+ const double k2,
+ const int nxmodes){
+ {
+ double* L2_local = L2_local_thread.getMine();
+ for (hsize_t i=0; i<ncomp(fc); i++){
+ L2_local[0] += nxmodes * (
+ (a[ncomp(fc)*cindex + i][0] * a[ncomp(fc)*cindex + i][0]) +
+ (a[ncomp(fc)*cindex + i][1] * a[ncomp(fc)*cindex + i][1]));
+ }
+ }
+ });
+
+ L2_local_thread.mergeParallel();
+
+ double L2;
+ MPI_Allreduce(
+ L2_local_thread.getMasterData(),
+ &L2,
+ 1,
+ MPI_DOUBLE, MPI_SUM, this->layout->comm);
+ return sqrt(L2 * this->dkx * this->dky * this->dkz);
+}
+
+
+template class kspace<FFTW, ONE_HALF>;
+template class kspace<FFTW, TWO_THIRDS>;
+template class kspace<FFTW, SMOOTH>;
+
+template kspace<FFTW, ONE_HALF>::kspace<>(
+ const field_layout<ONE> *,
+ const double, const double, const double);
+template kspace<FFTW, ONE_HALF>::kspace<>(
+ const field_layout<THREE> *,
+ const double, const double, const double);
+template kspace<FFTW, ONE_HALF>::kspace<>(
+ const field_layout<THREExTHREE> *,
+ const double, const double, const double);
+
+template kspace<FFTW, TWO_THIRDS>::kspace<>(
+ const field_layout<ONE> *,
+ const double, const double, const double);
+template kspace<FFTW, TWO_THIRDS>::kspace<>(
+ const field_layout<THREE> *,
+ const double, const double, const double);
+template kspace<FFTW, TWO_THIRDS>::kspace<>(
+ const field_layout<THREExTHREE> *,
+ const double, const double, const double);
+
+template kspace<FFTW, SMOOTH>::kspace<>(
+ const field_layout<ONE> *,
+ const double, const double, const double);
+template kspace<FFTW, SMOOTH>::kspace<>(
+ const field_layout<THREE> *,
+ const double, const double, const double);
+template kspace<FFTW, SMOOTH>::kspace<>(
+ const field_layout<THREExTHREE> *,
+ const double, const double, const double);
+
+template void kspace<FFTW, SMOOTH>::low_pass<float, ONE>(
+ typename fftw_interface<float>::complex *__restrict__ a,
+ const double kmax);
+template void kspace<FFTW, SMOOTH>::low_pass<float, THREE>(
+ typename fftw_interface<float>::complex *__restrict__ a,
+ const double kmax);
+template void kspace<FFTW, SMOOTH>::low_pass<float, THREExTHREE>(
+ typename fftw_interface<float>::complex *__restrict__ a,
+ const double kmax);
+
+template void kspace<FFTW, SMOOTH>::low_pass<double, ONE>(
+ typename fftw_interface<double>::complex *__restrict__ a,
+ const double kmax);
+template void kspace<FFTW, SMOOTH>::low_pass<double, THREE>(
+ typename fftw_interface<double>::complex *__restrict__ a,
+ const double kmax);
+template void kspace<FFTW, SMOOTH>::low_pass<double, THREExTHREE>(
+ typename fftw_interface<double>::complex *__restrict__ a,
+ const double kmax);
+
+template void kspace<FFTW, SMOOTH>::Gauss_filter<float, ONE>(
+ typename fftw_interface<float>::complex *__restrict__ a,
+ const double kmax);
+template void kspace<FFTW, SMOOTH>::Gauss_filter<float, THREE>(
+ typename fftw_interface<float>::complex *__restrict__ a,
+ const double kmax);
+template void kspace<FFTW, SMOOTH>::Gauss_filter<float, THREExTHREE>(
+ typename fftw_interface<float>::complex *__restrict__ a,
+ const double kmax);
+
+template void kspace<FFTW, SMOOTH>::Gauss_filter<double, ONE>(
+ typename fftw_interface<double>::complex *__restrict__ a,
+ const double kmax);
+template void kspace<FFTW, SMOOTH>::Gauss_filter<double, THREE>(
+ typename fftw_interface<double>::complex *__restrict__ a,
+ const double kmax);
+template void kspace<FFTW, SMOOTH>::Gauss_filter<double, THREExTHREE>(
+ typename fftw_interface<double>::complex *__restrict__ a,
+ const double kmax);
+
+template int kspace<FFTW, SMOOTH>::filter<float, ONE>(
+ typename fftw_interface<float>::complex *__restrict__ a,
+ const double kmax,
+ std::string filter_type);
+template int kspace<FFTW, SMOOTH>::filter<float, THREE>(
+ typename fftw_interface<float>::complex *__restrict__ a,
+ const double kmax,
+ std::string filter_type);
+template int kspace<FFTW, SMOOTH>::filter<float, THREExTHREE>(
+ typename fftw_interface<float>::complex *__restrict__ a,
+ const double kmax,
+ std::string filter_type);
+
+template int kspace<FFTW, SMOOTH>::filter<double, ONE>(
+ typename fftw_interface<double>::complex *__restrict__ a,
+ const double kmax,
+ std::string filter_type);
+template int kspace<FFTW, SMOOTH>::filter<double, THREE>(
+ typename fftw_interface<double>::complex *__restrict__ a,
+ const double kmax,
+ std::string filter_type);
+template int kspace<FFTW, SMOOTH>::filter<double, THREExTHREE>(
+ typename fftw_interface<double>::complex *__restrict__ a,
+ const double kmax,
+ std::string filter_type);
+
+template int kspace<FFTW, SMOOTH>::filter_calibrated_ell<float, ONE>(
+ typename fftw_interface<float>::complex *__restrict__ a,
+ const double kmax,
+ std::string filter_type);
+template int kspace<FFTW, SMOOTH>::filter_calibrated_ell<float, THREE>(
+ typename fftw_interface<float>::complex *__restrict__ a,
+ const double kmax,
+ std::string filter_type);
+template int kspace<FFTW, SMOOTH>::filter_calibrated_ell<float, THREExTHREE>(
+ typename fftw_interface<float>::complex *__restrict__ a,
+ const double kmax,
+ std::string filter_type);
+
+template int kspace<FFTW, SMOOTH>::filter_calibrated_ell<double, ONE>(
+ typename fftw_interface<double>::complex *__restrict__ a,
+ const double kmax,
+ std::string filter_type);
+template int kspace<FFTW, SMOOTH>::filter_calibrated_ell<double, THREE>(
+ typename fftw_interface<double>::complex *__restrict__ a,
+ const double kmax,
+ std::string filter_type);
+template int kspace<FFTW, SMOOTH>::filter_calibrated_ell<double, THREExTHREE>(
+ typename fftw_interface<double>::complex *__restrict__ a,
+ const double kmax,
+ std::string filter_type);
+
+template void kspace<FFTW, ONE_HALF>::dealias<float, ONE>(
+ typename fftw_interface<float>::complex *__restrict__ a);
+template void kspace<FFTW, ONE_HALF>::dealias<float, THREE>(
+ typename fftw_interface<float>::complex *__restrict__ a);
+template void kspace<FFTW, ONE_HALF>::dealias<float, THREExTHREE>(
+ typename fftw_interface<float>::complex *__restrict__ a);
+
+template void kspace<FFTW, ONE_HALF>::dealias<double, ONE>(
+ typename fftw_interface<double>::complex *__restrict__ a);
+template void kspace<FFTW, ONE_HALF>::dealias<double, THREE>(
+ typename fftw_interface<double>::complex *__restrict__ a);
+template void kspace<FFTW, ONE_HALF>::dealias<double, THREExTHREE>(
+ typename fftw_interface<double>::complex *__restrict__ a);
+
+template void kspace<FFTW, TWO_THIRDS>::dealias<float, ONE>(
+ typename fftw_interface<float>::complex *__restrict__ a);
+template void kspace<FFTW, TWO_THIRDS>::dealias<float, THREE>(
+ typename fftw_interface<float>::complex *__restrict__ a);
+template void kspace<FFTW, TWO_THIRDS>::dealias<float, THREExTHREE>(
+ typename fftw_interface<float>::complex *__restrict__ a);
+
+template void kspace<FFTW, TWO_THIRDS>::dealias<double, ONE>(
+ typename fftw_interface<double>::complex *__restrict__ a);
+template void kspace<FFTW, TWO_THIRDS>::dealias<double, THREE>(
+ typename fftw_interface<double>::complex *__restrict__ a);
+template void kspace<FFTW, TWO_THIRDS>::dealias<double, THREExTHREE>(
+ typename fftw_interface<double>::complex *__restrict__ a);
+
+template void kspace<FFTW, SMOOTH>::dealias<float, ONE>(
+ typename fftw_interface<float>::complex *__restrict__ a);
+template void kspace<FFTW, SMOOTH>::dealias<float, THREE>(
+ typename fftw_interface<float>::complex *__restrict__ a);
+template void kspace<FFTW, SMOOTH>::dealias<float, THREExTHREE>(
+ typename fftw_interface<float>::complex *__restrict__ a);
+
+template void kspace<FFTW, SMOOTH>::dealias<double, ONE>(
+ typename fftw_interface<double>::complex *__restrict__ a);
+template void kspace<FFTW, SMOOTH>::dealias<double, THREE>(
+ typename fftw_interface<double>::complex *__restrict__ a);
+template void kspace<FFTW, SMOOTH>::dealias<double, THREExTHREE>(
+ typename fftw_interface<double>::complex *__restrict__ a);
+
+template void kspace<FFTW, ONE_HALF>::cospectrum<float, ONE>(
+ const typename fftw_interface<float>::complex *__restrict__ a,
+ const typename fftw_interface<float>::complex *__restrict__ b,
+ const hid_t group,
+ const std::string dset_name,
+ const hsize_t toffset,
+ const double wavenumber_exp);
+template void kspace<FFTW, ONE_HALF>::cospectrum<float, THREE>(
+ const typename fftw_interface<float>::complex *__restrict__ a,
+ const typename fftw_interface<float>::complex *__restrict__ b,
+ const hid_t group,
+ const std::string dset_name,
+ const hsize_t toffset,
+ const double wavenumber_exp);
+template void kspace<FFTW, ONE_HALF>::cospectrum<float, THREExTHREE>(
+ const typename fftw_interface<float>::complex *__restrict__ a,
+ const typename fftw_interface<float>::complex *__restrict__ b,
+ const hid_t group,
+ const std::string dset_name,
+ const hsize_t toffset,
+ const double wavenumber_exp);
+template void kspace<FFTW, ONE_HALF>::cospectrum<double, ONE>(
+ const typename fftw_interface<double>::complex *__restrict__ a,
+ const typename fftw_interface<double>::complex *__restrict__ b,
+ const hid_t group,
+ const std::string dset_name,
+ const hsize_t toffset,
+ const double wavenumber_exp);
+template void kspace<FFTW, ONE_HALF>::cospectrum<double, THREE>(
+ const typename fftw_interface<double>::complex *__restrict__ a,
+ const typename fftw_interface<double>::complex *__restrict__ b,
+ const hid_t group,
+ const std::string dset_name,
+ const hsize_t toffset,
+ const double wavenumber_exp);
+template void kspace<FFTW, ONE_HALF>::cospectrum<double, THREExTHREE>(
+ const typename fftw_interface<double>::complex *__restrict__ a,
+ const typename fftw_interface<double>::complex *__restrict__ b,
+ const hid_t group,
+ const std::string dset_name,
+ const hsize_t toffset,
+ const double wavenumber_exp);
+
+
+template void kspace<FFTW, TWO_THIRDS>::cospectrum<float, ONE>(
+ const typename fftw_interface<float>::complex *__restrict__ a,
+ const typename fftw_interface<float>::complex *__restrict__ b,
+ const hid_t group,
+ const std::string dset_name,
+ const hsize_t toffset,
+ const double wavenumber_exp);
+template void kspace<FFTW, TWO_THIRDS>::cospectrum<float, THREE>(
+ const typename fftw_interface<float>::complex *__restrict__ a,
+ const typename fftw_interface<float>::complex *__restrict__ b,
+ const hid_t group,
+ const std::string dset_name,
+ const hsize_t toffset,
+ const double wavenumber_exp);
+template void kspace<FFTW, TWO_THIRDS>::cospectrum<float, THREExTHREE>(
+ const typename fftw_interface<float>::complex *__restrict__ a,
+ const typename fftw_interface<float>::complex *__restrict__ b,
+ const hid_t group,
+ const std::string dset_name,
+ const hsize_t toffset,
+ const double wavenumber_exp);
+template void kspace<FFTW, TWO_THIRDS>::cospectrum<double, ONE>(
+ const typename fftw_interface<double>::complex *__restrict__ a,
+ const typename fftw_interface<double>::complex *__restrict__ b,
+ const hid_t group,
+ const std::string dset_name,
+ const hsize_t toffset,
+ const double wavenumber_exp);
+template void kspace<FFTW, TWO_THIRDS>::cospectrum<double, THREE>(
+ const typename fftw_interface<double>::complex *__restrict__ a,
+ const typename fftw_interface<double>::complex *__restrict__ b,
+ const hid_t group,
+ const std::string dset_name,
+ const hsize_t toffset,
+ const double wavenumber_exp);
+template void kspace<FFTW, TWO_THIRDS>::cospectrum<double, THREExTHREE>(
+ const typename fftw_interface<double>::complex *__restrict__ a,
+ const typename fftw_interface<double>::complex *__restrict__ b,
+ const hid_t group,
+ const std::string dset_name,
+ const hsize_t toffset,
+ const double wavenumber_exp);
+
+template void kspace<FFTW, SMOOTH>::cospectrum<float, ONE>(
+ const typename fftw_interface<float>::complex *__restrict__ a,
+ const typename fftw_interface<float>::complex *__restrict__ b,
+ const hid_t group,
+ const std::string dset_name,
+ const hsize_t toffset,
+ const double wavenumber_exp);
+template void kspace<FFTW, SMOOTH>::cospectrum<float, THREE>(
+ const typename fftw_interface<float>::complex *__restrict__ a,
+ const typename fftw_interface<float>::complex *__restrict__ b,
+ const hid_t group,
+ const std::string dset_name,
+ const hsize_t toffset,
+ const double wavenumber_exp);
+template void kspace<FFTW, SMOOTH>::cospectrum<float, THREExTHREE>(
+ const typename fftw_interface<float>::complex *__restrict__ a,
+ const typename fftw_interface<float>::complex *__restrict__ b,
+ const hid_t group,
+ const std::string dset_name,
+ const hsize_t toffset,
+ const double wavenumber_exp);
+template void kspace<FFTW, SMOOTH>::cospectrum<double, ONE>(
+ const typename fftw_interface<double>::complex *__restrict__ a,
+ const typename fftw_interface<double>::complex *__restrict__ b,
+ const hid_t group,
+ const std::string dset_name,
+ const hsize_t toffset,
+ const double wavenumber_exp);
+template void kspace<FFTW, SMOOTH>::cospectrum<double, THREE>(
+ const typename fftw_interface<double>::complex *__restrict__ a,
+ const typename fftw_interface<double>::complex *__restrict__ b,
+ const hid_t group,
+ const std::string dset_name,
+ const hsize_t toffset,
+ const double wavenumber_exp);
+template void kspace<FFTW, SMOOTH>::cospectrum<double, THREExTHREE>(
+ const typename fftw_interface<double>::complex *__restrict__ a,
+ const typename fftw_interface<double>::complex *__restrict__ b,
+ const hid_t group,
+ const std::string dset_name,
+ const hsize_t toffset,
+ const double wavenumber_exp);
+
+template void kspace<FFTW, ONE_HALF>::cospectrum<float, ONE>(
+ const typename fftw_interface<float>::complex *__restrict__ a,
+ const hid_t group,
+ const std::string dset_name,
+ const hsize_t toffset,
+ const double wavenumber_exp);
+template void kspace<FFTW, ONE_HALF>::cospectrum<float, THREE>(
+ const typename fftw_interface<float>::complex *__restrict__ a,
+ const hid_t group,
+ const std::string dset_name,
+ const hsize_t toffset,
+ const double wavenumber_exp);
+template void kspace<FFTW, ONE_HALF>::cospectrum<float, THREExTHREE>(
+ const typename fftw_interface<float>::complex *__restrict__ a,
+ const hid_t group,
+ const std::string dset_name,
+ const hsize_t toffset,
+ const double wavenumber_exp);
+template void kspace<FFTW, ONE_HALF>::cospectrum<double, ONE>(
+ const typename fftw_interface<double>::complex *__restrict__ a,
+ const hid_t group,
+ const std::string dset_name,
+ const hsize_t toffset,
+ const double wavenumber_exp);
+template void kspace<FFTW, ONE_HALF>::cospectrum<double, THREE>(
+ const typename fftw_interface<double>::complex *__restrict__ a,
+ const hid_t group,
+ const std::string dset_name,
+ const hsize_t toffset,
+ const double wavenumber_exp);
+template void kspace<FFTW, ONE_HALF>::cospectrum<double, THREExTHREE>(
+ const typename fftw_interface<double>::complex *__restrict__ a,
+ const hid_t group,
+ const std::string dset_name,
+ const hsize_t toffset,
+ const double wavenumber_exp);
+
+template void kspace<FFTW, TWO_THIRDS>::cospectrum<float, ONE>(
+ const typename fftw_interface<float>::complex *__restrict__ a,
+ const hid_t group,
+ const std::string dset_name,
+ const hsize_t toffset,
+ const double wavenumber_exp);
+template void kspace<FFTW, TWO_THIRDS>::cospectrum<float, THREE>(
+ const typename fftw_interface<float>::complex *__restrict__ a,
+ const hid_t group,
+ const std::string dset_name,
+ const hsize_t toffset,
+ const double wavenumber_exp);
+template void kspace<FFTW, TWO_THIRDS>::cospectrum<float, THREExTHREE>(
+ const typename fftw_interface<float>::complex *__restrict__ a,
+ const hid_t group,
+ const std::string dset_name,
+ const hsize_t toffset,
+ const double wavenumber_exp);
+template void kspace<FFTW, TWO_THIRDS>::cospectrum<double, ONE>(
+ const typename fftw_interface<double>::complex *__restrict__ a,
+ const hid_t group,
+ const std::string dset_name,
+ const hsize_t toffset,
+ const double wavenumber_exp);
+template void kspace<FFTW, TWO_THIRDS>::cospectrum<double, THREE>(
+ const typename fftw_interface<double>::complex *__restrict__ a,
+ const hid_t group,
+ const std::string dset_name,
+ const hsize_t toffset,
+ const double wavenumber_exp);
+template void kspace<FFTW, TWO_THIRDS>::cospectrum<double, THREExTHREE>(
+ const typename fftw_interface<double>::complex *__restrict__ a,
+ const hid_t group,
+ const std::string dset_name,
+ const hsize_t toffset,
+ const double wavenumber_exp);
+
+template void kspace<FFTW, SMOOTH>::cospectrum<float, ONE>(
+ const typename fftw_interface<float>::complex *__restrict__ a,
+ const hid_t group,
+ const std::string dset_name,
+ const hsize_t toffset,
+ const double wavenumber_exp);
+template void kspace<FFTW, SMOOTH>::cospectrum<float, THREE>(
+ const typename fftw_interface<float>::complex *__restrict__ a,
+ const hid_t group,
+ const std::string dset_name,
+ const hsize_t toffset,
+ const double wavenumber_exp);
+template void kspace<FFTW, SMOOTH>::cospectrum<float, THREExTHREE>(
+ const typename fftw_interface<float>::complex *__restrict__ a,
+ const hid_t group,
+ const std::string dset_name,
+ const hsize_t toffset,
+ const double wavenumber_exp);
+template void kspace<FFTW, SMOOTH>::cospectrum<double, ONE>(
+ const typename fftw_interface<double>::complex *__restrict__ a,
+ const hid_t group,
+ const std::string dset_name,
+ const hsize_t toffset,
+ const double wavenumber_exp);
+template void kspace<FFTW, SMOOTH>::cospectrum<double, THREE>(
+ const typename fftw_interface<double>::complex *__restrict__ a,
+ const hid_t group,
+ const std::string dset_name,
+ const hsize_t toffset,
+ const double wavenumber_exp);
+template void kspace<FFTW, SMOOTH>::cospectrum<double, THREExTHREE>(
+ const typename fftw_interface<double>::complex *__restrict__ a,
+ const hid_t group,
+ const std::string dset_name,
+ const hsize_t toffset,
+ const double wavenumber_exp);
+
+template double kspace<FFTW, ONE_HALF>::L2norm<float, ONE>(
+ const typename fftw_interface<float>::complex *__restrict__ a);
+template double kspace<FFTW, ONE_HALF>::L2norm<float, THREE>(
+ const typename fftw_interface<float>::complex *__restrict__ a);
+template double kspace<FFTW, ONE_HALF>::L2norm<float, THREExTHREE>(
+ const typename fftw_interface<float>::complex *__restrict__ a);
+template double kspace<FFTW, ONE_HALF>::L2norm<double, ONE>(
+ const typename fftw_interface<double>::complex *__restrict__ a);
+template double kspace<FFTW, ONE_HALF>::L2norm<double, THREE>(
+ const typename fftw_interface<double>::complex *__restrict__ a);
+template double kspace<FFTW, ONE_HALF>::L2norm<double, THREExTHREE>(
+ const typename fftw_interface<double>::complex *__restrict__ a);
+
+template double kspace<FFTW, TWO_THIRDS>::L2norm<float, ONE>(
+ const typename fftw_interface<float>::complex *__restrict__ a);
+template double kspace<FFTW, TWO_THIRDS>::L2norm<float, THREE>(
+ const typename fftw_interface<float>::complex *__restrict__ a);
+template double kspace<FFTW, TWO_THIRDS>::L2norm<float, THREExTHREE>(
+ const typename fftw_interface<float>::complex *__restrict__ a);
+template double kspace<FFTW, TWO_THIRDS>::L2norm<double, ONE>(
+ const typename fftw_interface<double>::complex *__restrict__ a);
+template double kspace<FFTW, TWO_THIRDS>::L2norm<double, THREE>(
+ const typename fftw_interface<double>::complex *__restrict__ a);
+template double kspace<FFTW, TWO_THIRDS>::L2norm<double, THREExTHREE>(
+ const typename fftw_interface<double>::complex *__restrict__ a);
+
+template double kspace<FFTW, SMOOTH>::L2norm<float, ONE>(
+ const typename fftw_interface<float>::complex *__restrict__ a);
+template double kspace<FFTW, SMOOTH>::L2norm<float, THREE>(
+ const typename fftw_interface<float>::complex *__restrict__ a);
+template double kspace<FFTW, SMOOTH>::L2norm<float, THREExTHREE>(
+ const typename fftw_interface<float>::complex *__restrict__ a);
+template double kspace<FFTW, SMOOTH>::L2norm<double, ONE>(
+ const typename fftw_interface<double>::complex *__restrict__ a);
+template double kspace<FFTW, SMOOTH>::L2norm<double, THREE>(
+ const typename fftw_interface<double>::complex *__restrict__ a);
+template double kspace<FFTW, SMOOTH>::L2norm<double, THREExTHREE>(
+ const typename fftw_interface<double>::complex *__restrict__ a);
+
+template void kspace<FFTW, ONE_HALF>::project_divfree<float>(
+ typename fftw_interface<float>::complex *__restrict__ a,
+ const bool);
+template void kspace<FFTW, ONE_HALF>::project_divfree<double>(
+ typename fftw_interface<double>::complex *__restrict__ a,
+ const bool);
+
+template void kspace<FFTW, ONE_HALF>::force_divfree<float>(
+ typename fftw_interface<float>::complex *__restrict__ a);
+template void kspace<FFTW, ONE_HALF>::force_divfree<double>(
+ typename fftw_interface<double>::complex *__restrict__ a);
+
+template void kspace<FFTW, ONE_HALF>::rotate_divfree<float>(
+ typename fftw_interface<float>::complex *__restrict__ a);
+template void kspace<FFTW, ONE_HALF>::rotate_divfree<double>(
+ typename fftw_interface<double>::complex *__restrict__ a);
+
+template void kspace<FFTW, TWO_THIRDS>::project_divfree<float>(
+ typename fftw_interface<float>::complex *__restrict__ a,
+ const bool);
+template void kspace<FFTW, TWO_THIRDS>::project_divfree<double>(
+ typename fftw_interface<double>::complex *__restrict__ a,
+ const bool);
+
+template void kspace<FFTW, TWO_THIRDS>::force_divfree<float>(
+ typename fftw_interface<float>::complex *__restrict__ a);
+template void kspace<FFTW, TWO_THIRDS>::force_divfree<double>(
+ typename fftw_interface<double>::complex *__restrict__ a);
+
+template void kspace<FFTW, TWO_THIRDS>::rotate_divfree<float>(
+ typename fftw_interface<float>::complex *__restrict__ a);
+template void kspace<FFTW, TWO_THIRDS>::rotate_divfree<double>(
+ typename fftw_interface<double>::complex *__restrict__ a);
+
+template void kspace<FFTW, SMOOTH>::project_divfree<float>(
+ typename fftw_interface<float>::complex *__restrict__ a,
+ const bool);
+template void kspace<FFTW, SMOOTH>::project_divfree<double>(
+ typename fftw_interface<double>::complex *__restrict__ a,
+ const bool);
+
+template void kspace<FFTW, SMOOTH>::force_divfree<float>(
+ typename fftw_interface<float>::complex *__restrict__ a);
+template void kspace<FFTW, SMOOTH>::force_divfree<double>(
+ typename fftw_interface<double>::complex *__restrict__ a);
+
+template void kspace<FFTW, SMOOTH>::rotate_divfree<float>(
+ typename fftw_interface<float>::complex *__restrict__ a);
+template void kspace<FFTW, SMOOTH>::rotate_divfree<double>(
+ typename fftw_interface<double>::complex *__restrict__ a);
+
diff --git a/cpp/kspace.hpp b/cpp/kspace.hpp
new file mode 100644
index 0000000000000000000000000000000000000000..6457c8bd0a65d229d154984c05341f69a7728f7b
--- /dev/null
+++ b/cpp/kspace.hpp
@@ -0,0 +1,310 @@
+/**********************************************************************
+* *
+* Copyright 2015 Max Planck Institute *
+* for Dynamics and Self-Organization *
+* *
+* This file is part of TurTLE. *
+* *
+* TurTLE is free software: you can redistribute it and/or modify *
+* it under the terms of the GNU General Public License as published *
+* by the Free Software Foundation, either version 3 of the License, *
+* or (at your option) any later version. *
+* *
+* TurTLE is distributed in the hope that it will be useful, *
+* but WITHOUT ANY WARRANTY; without even the implied warranty of *
+* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the *
+* GNU General Public License for more details. *
+* *
+* You should have received a copy of the GNU General Public License *
+* along with TurTLE. If not, see <http://www.gnu.org/licenses/> *
+* *
+* Contact: Cristian.Lalescu@ds.mpg.de *
+* *
+**********************************************************************/
+
+
+
+#ifndef KSPACE_HPP
+
+#define KSPACE_HPP
+
+#include <hdf5.h>
+#include <vector>
+#include <string>
+#include "omputils.hpp"
+#include "fftw_interface.hpp"
+#include "field_layout.hpp"
+
+enum field_backend {FFTW};
+enum kspace_dealias_type {ONE_HALF, TWO_THIRDS, SMOOTH};
+
+/** \class kspace
+ * \brief A class for handling Fourier representation tasks.
+ *
+ * \tparam be field backend, currently only FFTW is possible.
+ * \tparam dt dealiasing mode, either ONE_HALF, TWO_THIRDS or SMOOTH.
+ *
+ * Contains wavenumber information (specific to each MPI process).
+ * This includes values of kx, ky, kz, including lowest modes dkx etc,
+ * as well as number of modes within sferical shells and mean wavenumber
+ * within shells.
+ * - has methods for spectrum computation and similar.
+ * - has methods for filtering.
+ * - has CLOOP methods, useful for computing arbitrary formulas over the
+ * Fourier space grid (i.e. use lambda expressions).
+ */
+template <field_backend be,
+ kspace_dealias_type dt>
+class kspace
+{
+ public:
+ /* relevant field layout */
+ field_layout<ONE> *layout;
+
+ /* physical parameters */
+ double dkx, dky, dkz, dk, dk2;
+
+ /* mode and dealiasing information */
+ double kMx, kMy, kMz, kM, kM2;
+ std::vector<double> kx, ky, kz;
+ std::vector<double> kshell;
+ std::vector<int64_t> nshell;
+ int nshells;
+
+ /* methods */
+ template <field_components fc>
+ kspace(
+ const field_layout<fc> *source_layout,
+ const double DKX = 1.0,
+ const double DKY = 1.0,
+ const double DKZ = 1.0);
+ ~kspace() noexcept(false);
+
+ int store(hid_t stat_file);
+
+ template <typename rnumber,
+ field_components fc>
+ void low_pass(
+ typename fftw_interface<rnumber>::complex *__restrict__ a,
+ const double kmax);
+
+
+ template <typename rnumber,
+ field_components fc>
+ void Gauss_filter(
+ typename fftw_interface<rnumber>::complex *__restrict__ a,
+ const double sigma);
+
+ template <typename rnumber,
+ field_components fc>
+ void ball_filter(
+ typename fftw_interface<rnumber>::complex *__restrict__ a,
+ const double sigma);
+
+ template <typename rnumber,
+ field_components fc>
+ void general_M_filter(
+ typename fftw_interface<rnumber>::complex *__restrict__ a,
+ const double sigma);
+
+ /**
+ * \tparam rnumber type of real number, float or double.
+ * \tparam fc field components, ONE, THREE or THREExTHREE.
+ * \return exit mode (integer), EXIT_SUCCESS or arbitrary value for failure.
+ */
+ template <typename rnumber,
+ field_components fc>
+ int filter(
+ typename fftw_interface<rnumber>::complex *__restrict__ a,
+ const double wavenumber,
+ std::string filter_type = std::string("Gauss"));
+
+ template <typename rnumber,
+ field_components fc>
+ int filter_calibrated_ell(
+ typename fftw_interface<rnumber>::complex *__restrict__ a,
+ const double wavenumber,
+ std::string filter_type = std::string("Gauss"));
+
+ template <typename rnumber,
+ field_components fc>
+ void dealias(typename fftw_interface<rnumber>::complex *__restrict__ a);
+
+ template <typename rnumber,
+ field_components fc>
+ void cospectrum(
+ const rnumber(* __restrict__ a)[2],
+ const rnumber(* __restrict__ b)[2],
+ const hid_t group,
+ const std::string dset_name,
+ const hsize_t toffset,
+ const double wavenumber_exp = 0);
+
+ template <typename rnumber,
+ field_components fc>
+ void cospectrum(
+ const rnumber(* __restrict__ a)[2],
+ const hid_t group,
+ const std::string dset_name,
+ const hsize_t toffset,
+ const double wavenumber_exp = 0);
+
+ template <typename rnumber,
+ field_components fc>
+ void cospectrum(
+ const rnumber(* __restrict__ a)[2],
+ std::vector<double> &spec,
+ const double wavenumber_exp = 0);
+
+ template <typename rnumber,
+ field_components fc>
+ double L2norm(
+ const rnumber(* __restrict__ a)[2]);
+
+ template <class func_type>
+ void CLOOP(func_type expression)
+ {
+ start_mpi_profiling_zone(turtle_mpi_pcontrol::FIELD);
+ #pragma omp parallel
+ {
+ const hsize_t start = OmpUtils::ForIntervalStart(this->layout->subsizes[1]);
+ const hsize_t end = OmpUtils::ForIntervalEnd(this->layout->subsizes[1]);
+
+ for (hsize_t yindex = 0; yindex < this->layout->subsizes[0]; yindex++){
+ for (hsize_t zindex = start; zindex < end; zindex++){
+ const ptrdiff_t cindex = (
+ yindex*this->layout->subsizes[1]*this->layout->subsizes[2] +
+ zindex*this->layout->subsizes[2]);
+ for (hsize_t xindex = 0; xindex < this->layout->subsizes[2]; xindex++)
+ {
+ expression(cindex + xindex, xindex, yindex, zindex);
+ }
+ }
+ }
+ }
+ finish_mpi_profiling_zone(turtle_mpi_pcontrol::FIELD);
+ }
+ template <class func_type>
+ void CLOOP_simd(func_type expression)
+ {
+ start_mpi_profiling_zone(turtle_mpi_pcontrol::FIELD);
+ #pragma omp parallel
+ {
+ const hsize_t start = OmpUtils::ForIntervalStart(this->layout->subsizes[1]);
+ const hsize_t end = OmpUtils::ForIntervalEnd(this->layout->subsizes[1]);
+
+ for (hsize_t yindex = 0; yindex < this->layout->subsizes[0]; yindex++){
+ #pragma omp simd
+ for (hsize_t zindex = start; zindex < end; zindex++){
+ const ptrdiff_t cindex = (
+ yindex*this->layout->subsizes[1]*this->layout->subsizes[2] +
+ zindex*this->layout->subsizes[2]);
+ for (hsize_t xindex = 0; xindex < this->layout->subsizes[2]; xindex++)
+ {
+ expression(cindex + xindex, xindex, yindex, zindex);
+ }
+ }
+ }
+ }
+ finish_mpi_profiling_zone(turtle_mpi_pcontrol::FIELD);
+ }
+ template <class func_type>
+ void CLOOP_K2(func_type expression)
+ {
+ start_mpi_profiling_zone(turtle_mpi_pcontrol::FIELD);
+ #pragma omp parallel
+ {
+ const hsize_t start = OmpUtils::ForIntervalStart(this->layout->subsizes[1]);
+ const hsize_t end = OmpUtils::ForIntervalEnd(this->layout->subsizes[1]);
+
+ for (hsize_t yindex = 0; yindex < this->layout->subsizes[0]; yindex++){
+ for (hsize_t zindex = start; zindex < end; zindex++){
+ const ptrdiff_t cindex = yindex*this->layout->subsizes[1]*this->layout->subsizes[2]
+ + zindex*this->layout->subsizes[2];
+ for (hsize_t xindex = 0; xindex < this->layout->subsizes[2]; xindex++)
+ {
+ expression(
+ cindex+xindex,
+ xindex,
+ yindex,
+ zindex,
+ (this->kx[xindex]*this->kx[xindex] +
+ this->ky[yindex]*this->ky[yindex] +
+ this->kz[zindex]*this->kz[zindex]));
+ }
+ }
+ }
+ }
+ finish_mpi_profiling_zone(turtle_mpi_pcontrol::FIELD);
+ }
+ template <class func_type>
+ void CLOOP_K2_simd(func_type expression)
+ {
+ start_mpi_profiling_zone(turtle_mpi_pcontrol::FIELD);
+ #pragma omp parallel
+ {
+ const hsize_t start = OmpUtils::ForIntervalStart(this->layout->subsizes[1]);
+ const hsize_t end = OmpUtils::ForIntervalEnd(this->layout->subsizes[1]);
+
+ for (hsize_t yindex = 0; yindex < this->layout->subsizes[0]; yindex++){
+ #pragma omp simd
+ for (hsize_t zindex = start; zindex < end; zindex++){
+ const ptrdiff_t cindex = yindex*this->layout->subsizes[1]*this->layout->subsizes[2]
+ + zindex*this->layout->subsizes[2];
+ for (hsize_t xindex = 0; xindex < this->layout->subsizes[2]; xindex++)
+ {
+ expression(
+ cindex+xindex,
+ xindex,
+ yindex,
+ zindex,
+ (this->kx[xindex]*this->kx[xindex] +
+ this->ky[yindex]*this->ky[yindex] +
+ this->kz[zindex]*this->kz[zindex]));
+ }
+ }
+ }
+ }
+ finish_mpi_profiling_zone(turtle_mpi_pcontrol::FIELD);
+ }
+ template <class func_type>
+ void CLOOP_K2_NXMODES(func_type expression)
+ {
+ start_mpi_profiling_zone(turtle_mpi_pcontrol::FIELD);
+ #pragma omp parallel
+ {
+ const hsize_t start = OmpUtils::ForIntervalStart(this->layout->subsizes[1]);
+ const hsize_t end = OmpUtils::ForIntervalEnd(this->layout->subsizes[1]);
+
+ for (hsize_t yindex = 0; yindex < this->layout->subsizes[0]; yindex++){
+ for (hsize_t zindex = start; zindex < end; zindex++){
+ const ptrdiff_t cindex = yindex*this->layout->subsizes[1]*this->layout->subsizes[2]
+ + zindex*this->layout->subsizes[2];
+ const double k2 = (
+ this->ky[yindex]*this->ky[yindex] +
+ this->kz[zindex]*this->kz[zindex]);
+ expression(cindex, 0, yindex, zindex, k2, 1);
+ for (hsize_t xindex = 1; xindex < this->layout->subsizes[2]; xindex++)
+ {
+ expression(cindex+xindex, xindex, yindex, zindex, k2 + this->kx[xindex]*this->kx[xindex], 2);
+ }
+ }
+ }
+ }
+ finish_mpi_profiling_zone(turtle_mpi_pcontrol::FIELD);
+ }
+ template <typename rnumber>
+ void project_divfree(
+ typename fftw_interface<rnumber>::complex *__restrict__ a,
+ const bool maintain_energy = false);
+ // TODO: can the following be done in a cleaner way?
+ template <typename rnumber>
+ void force_divfree(typename fftw_interface<rnumber>::complex *__restrict__ a){
+ this->template project_divfree<rnumber>(a, false);
+ }
+ template <typename rnumber>
+ void rotate_divfree(typename fftw_interface<rnumber>::complex *__restrict__ a);
+};
+
+#endif//KSPACE_HPP
+
diff --git a/cpp/mainpage.dox b/cpp/mainpage.dox
new file mode 100644
index 0000000000000000000000000000000000000000..63151e345be74ea2a1e9acc4b283685b1155057d
--- /dev/null
+++ b/cpp/mainpage.dox
@@ -0,0 +1,32 @@
+/** \mainpage
+ *
+ * Turbulence Tools: Lagrangian and Eulerian
+ * =========================================
+ *
+ * In brief, this code runs pseudospectral direct numerical simulations
+ * (DNS) of the incompressible Navier-Stokes equations, using FFTW 3, and
+ * it can integrate particle trajectories in the resulting fields.
+ *
+ * The Navier-Stokes solver has been extensively tested (tests are included
+ * in the repository), and it is working as expected. Parameters and
+ * statistics are stored in HDF5 format, together with code information, so
+ * simulation data should be "future proof" --- suggestions of possible
+ * improvements to the current approach are always welcome.
+ *
+ * The primary aim of TurTLE is to reduce the time spent on setting up and
+ * baby sitting DNS, as well as simplify the analysis of the generated
+ * data. The wish is that this Python package provides an easy and general
+ * way of constructing efficient specialized DNS C++ codes for different
+ * turbulence problems encountered in research. At the same time, the
+ * package should provide a unified way of postprocessing, and accessing
+ * the postprocessing results. The code therefore consists of two main
+ * parts: the pure C++ code, a set of loosely related "building blocks",
+ * and the Python code, which can generate C++ code using the pure classes,
+ * but with a significant degree of flexibility.
+ *
+ * The code user is expected to write a small python script that will
+ * properly define the DNS they are interested in running. That code will
+ * generate an executable that can then be run directly on the user's
+ * machine, or submitted to a queue on a cluster.
+ *
+ */
diff --git a/bfps/cpp/omputils.hpp b/cpp/omputils.hpp
similarity index 100%
rename from bfps/cpp/omputils.hpp
rename to cpp/omputils.hpp
diff --git a/cpp/particles/.tocompile b/cpp/particles/.tocompile
new file mode 100644
index 0000000000000000000000000000000000000000..02874ed792f4eedb859e1b779facd3d2c775ec08
--- /dev/null
+++ b/cpp/particles/.tocompile
@@ -0,0 +1,2 @@
+mpicxx -g main_tocompile.cpp -o /tmp/main_test_part.exe -I/home/bbramas/Projects/bfps/bfps/cpp/ -I/home/bbramas/Downloads/hdf5install/include -L/home/bbramas/Downloads/hdf5install/lib -lhdf5 -lsz -lz
+mpicxx -fPIC -rdynamic -g NSVE-v2.0.1-single.cpp -o /tmp/NSVE-v2.0.1-single.exe -I/home/bbramas/Projects/bfps/bfps/cpp/ -I/home/bbramas/Downloads/hdf5install/include -I/home/bbramas/Downloads/fftw-3.3.4/install/include/ -L/home/bbramas/Downloads/hdf5install/lib -lhdf5 -lsz -lz -L/home/bbramas/.local/lib/python2.7/site-packages/bfps-2.0.1.post31+g12693ea-py2.7.egg/bfps/ -lbfps -fopenmp -lgomp -L/home/bbramas/Downloads/fftw-3.3.4/install/lib/ -lfftw3_mpi -lfftw3f_mpi -lfftw3_omp -lfftw3f_omp -lfftw3 -lfftw3f
diff --git a/cpp/particles/abstract_particle_rhs.hpp b/cpp/particles/abstract_particle_rhs.hpp
new file mode 100644
index 0000000000000000000000000000000000000000..947e152430cb14bc439021d83e48860411c2c175
--- /dev/null
+++ b/cpp/particles/abstract_particle_rhs.hpp
@@ -0,0 +1,103 @@
+/******************************************************************************
+* *
+* Copyright 2020 Max Planck Institute for Dynamics and Self-Organization *
+* *
+* This file is part of TurTLE. *
+* *
+* TurTLE is free software: you can redistribute it and/or modify *
+* it under the terms of the GNU General Public License as published *
+* by the Free Software Foundation, either version 3 of the License, *
+* or (at your option) any later version. *
+* *
+* TurTLE is distributed in the hope that it will be useful, *
+* but WITHOUT ANY WARRANTY; without even the implied warranty of *
+* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the *
+* GNU General Public License for more details. *
+* *
+* You should have received a copy of the GNU General Public License *
+* along with TurTLE. If not, see <http://www.gnu.org/licenses/> *
+* *
+* Contact: Cristian.Lalescu@ds.mpg.de *
+* *
+******************************************************************************/
+
+
+
+#ifndef ABSTRACT_PARTICLE_RHS_HPP
+#define ABSTRACT_PARTICLE_RHS_HPP
+
+#include "particles/interpolation/abstract_particle_set.hpp"
+
+/** Particle model definition, i.e. provides right-hand-side of ODEs
+ *
+ * This abstract class prescribes the way that particle models shall be
+ * implemented in TurTLE.
+ * Children of this class must implement a "function call" operator,
+ * where the right-hand-side of the corresponding ODE shall be computed,
+ * as well as an "imposeModelConstraints" method where miscellaneous properties
+ * can be enforced as needed (an example would be orientation vectors being
+ * reset to unit lenght).
+ *
+ */
+class abstract_particle_rhs
+{
+ protected:
+ using particle_rnumber = abstract_particle_set::particle_rnumber;
+ public:
+ // destructor
+ virtual ~abstract_particle_rhs() noexcept(false){}
+
+ /** Probe the dimension of the ODE
+ */
+ virtual const int getStateSize() const = 0;
+
+
+ /** Compute right-hand-side of the ODE
+ *
+ * Notes
+ * -----
+ * 1. This method may shuffle particle data in-memory, but
+ * it may not redistribute particles between MPI processes.
+ * The `additional_data` parameter allows for other particle data to be
+ * shuffled in the same fashion.
+ * In particular, you may use `additional_data` when computing substeps
+ * for Runge Kutta methods to keep the different temporary arrays in
+ * the same order.
+ * Please see implementation of the Heun method for an example.
+ *
+ * For completeness: shuffling of data will at least take place during
+ * the computation of particle-to-particle interaction terms.
+ *
+ * 2. This method may not modify the current state of the particles.
+ * I cannot mark the parameter `abstract_particle_set &pset` as
+ * `const` because the arrays may need to be shuffled, but separate
+ * parts of the code rely on the fact that this method does not change
+ * the state of the particle set.
+ * If your particle model has strict constraints that affect the
+ * computation of the right-hand side, you must ensure that those
+ * constraints are respected within this method *without* modifying
+ * the current particle state.
+ * Otherwise the ODE may not be integrated correctly.
+ */
+ virtual int operator()(
+ double t,
+ abstract_particle_set &pset,
+ particle_rnumber *result,
+ std::vector<std::unique_ptr<
+ abstract_particle_set::particle_rnumber[]>> &additional_data) = 0;
+ /** Enforce model constraints on a set of particles
+ *
+ * Note:
+ * This is meant to be used as a safe-guard against growth of small
+ * errors.
+ * As an example, shaped particles for which orientation must be tracked
+ * through time may slowly develop non-unit-sized orientation vectors.
+ * This method will change the particle states such that any such
+ * constraints are respected.
+ */
+ virtual int imposeModelConstraints(
+ abstract_particle_set &pset) const = 0;
+};
+
+#endif//ABSTRACT_PARTICLE_RHS_HPP
+
diff --git a/cpp/particles/abstract_particles_input.hpp b/cpp/particles/abstract_particles_input.hpp
new file mode 100644
index 0000000000000000000000000000000000000000..20884283491a8f316277ef78be20026080a34e9a
--- /dev/null
+++ b/cpp/particles/abstract_particles_input.hpp
@@ -0,0 +1,48 @@
+/******************************************************************************
+* *
+* Copyright 2019 Max Planck Institute for Dynamics and Self-Organization *
+* *
+* This file is part of TurTLE. *
+* *
+* TurTLE is free software: you can redistribute it and/or modify *
+* it under the terms of the GNU General Public License as published *
+* by the Free Software Foundation, either version 3 of the License, *
+* or (at your option) any later version. *
+* *
+* TurTLE is distributed in the hope that it will be useful, *
+* but WITHOUT ANY WARRANTY; without even the implied warranty of *
+* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the *
+* GNU General Public License for more details. *
+* *
+* You should have received a copy of the GNU General Public License *
+* along with TurTLE. If not, see <http://www.gnu.org/licenses/> *
+* *
+* Contact: Cristian.Lalescu@ds.mpg.de *
+* *
+******************************************************************************/
+
+
+
+#ifndef ABSTRACT_PARTICLES_INPUT_HPP
+#define ABSTRACT_PARTICLES_INPUT_HPP
+
+#include <tuple>
+
+template <class partsize_t, class real_number>
+class abstract_particles_input {
+public:
+ virtual ~abstract_particles_input() noexcept(false){}
+
+ virtual partsize_t getTotalNbParticles() = 0;
+ virtual partsize_t getLocalNbParticles() = 0;
+ virtual int getNbRhs() = 0;
+
+ virtual std::unique_ptr<real_number[]> getMyParticles() = 0;
+ virtual std::unique_ptr<partsize_t[]> getMyParticlesIndexes() = 0;
+ virtual std::unique_ptr<partsize_t[]> getMyParticlesLabels() = 0;
+ virtual std::vector<std::unique_ptr<real_number[]>> getMyRhs() = 0;
+ virtual std::vector<hsize_t> getParticleFileLayout() = 0;
+};
+
+
+#endif
diff --git a/bfps/cpp/particles/abstract_particles_output.hpp b/cpp/particles/abstract_particles_output.hpp
similarity index 56%
rename from bfps/cpp/particles/abstract_particles_output.hpp
rename to cpp/particles/abstract_particles_output.hpp
index a6eccaea003618b8acbf1a9252c1e6c5bedb3378..af690dcbd72ca06cf756d1d639a6efaebfd6b677 100644
--- a/bfps/cpp/particles/abstract_particles_output.hpp
+++ b/cpp/particles/abstract_particles_output.hpp
@@ -1,3 +1,28 @@
+/******************************************************************************
+* *
+* Copyright 2019 Max Planck Institute for Dynamics and Self-Organization *
+* *
+* This file is part of TurTLE. *
+* *
+* TurTLE is free software: you can redistribute it and/or modify *
+* it under the terms of the GNU General Public License as published *
+* by the Free Software Foundation, either version 3 of the License, *
+* or (at your option) any later version. *
+* *
+* TurTLE is distributed in the hope that it will be useful, *
+* but WITHOUT ANY WARRANTY; without even the implied warranty of *
+* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the *
+* GNU General Public License for more details. *
+* *
+* You should have received a copy of the GNU General Public License *
+* along with TurTLE. If not, see <http://www.gnu.org/licenses/> *
+* *
+* Contact: Cristian.Lalescu@ds.mpg.de *
+* *
+******************************************************************************/
+
+
+
#ifndef ABSTRACT_PARTICLES_OUTPUT
#define ABSTRACT_PARTICLES_OUTPUT
@@ -13,7 +38,15 @@
#include "scope_timer.hpp"
#include "env_utils.hpp"
-template <class partsize_t, class real_number, int size_particle_positions, int size_particle_rhs>
+/** \brief Class to handle distributed output of particle data
+ *
+ * \tparam partsize_t data type of particle index
+ * \tparam position_type data type of particle positions
+ * \tparam output_type data type of output (typically same as position_type, but it may be different)
+ * \tparam size_particle_positions int, size of particle state
+ *
+ */
+template <class partsize_t, class position_type, class output_type, int size_particle_positions>
class abstract_particles_output {
MPI_Comm mpi_com;
MPI_Comm mpi_com_writer;
@@ -25,14 +58,16 @@ class abstract_particles_output {
const int nb_rhs;
std::unique_ptr<std::pair<partsize_t,partsize_t>[]> buffer_indexes_send;
- std::unique_ptr<real_number[]> buffer_particles_positions_send;
- std::vector<std::unique_ptr<real_number[]>> buffer_particles_rhs_send;
+ std::unique_ptr<position_type[]> buffer_particles_positions_send;
+ std::vector<std::unique_ptr<output_type[]>> buffer_particles_rhs_send;
partsize_t size_buffers_send;
+ int buffers_size_particle_rhs_send;
- std::unique_ptr<real_number[]> buffer_particles_positions_recv;
- std::vector<std::unique_ptr<real_number[]>> buffer_particles_rhs_recv;
+ std::unique_ptr<position_type[]> buffer_particles_positions_recv;
+ std::vector<std::unique_ptr<output_type[]>> buffer_particles_rhs_recv;
std::unique_ptr<partsize_t[]> buffer_indexes_recv;
partsize_t size_buffers_recv;
+ int buffers_size_particle_rhs_recv;
int nb_processes_involved;
bool current_is_involved;
@@ -41,6 +76,10 @@ class abstract_particles_output {
partsize_t particles_chunk_current_offset;
protected:
+ MPI_Comm& getCom(){
+ return mpi_com;
+ }
+
MPI_Comm& getComWriter(){
return mpi_com_writer;
}
@@ -58,11 +97,13 @@ protected:
}
public:
- abstract_particles_output(MPI_Comm in_mpi_com, const partsize_t inTotalNbParticles, const int in_nb_rhs) throw()
+ abstract_particles_output(MPI_Comm in_mpi_com, const partsize_t inTotalNbParticles, const int in_nb_rhs) noexcept(false)
: mpi_com(in_mpi_com), my_rank(-1), nb_processes(-1),
total_nb_particles(inTotalNbParticles), nb_rhs(in_nb_rhs),
- buffer_particles_rhs_send(in_nb_rhs), size_buffers_send(-1),
- buffer_particles_rhs_recv(in_nb_rhs), size_buffers_recv(-1),
+ buffer_particles_rhs_send(in_nb_rhs), size_buffers_send(0),
+ buffers_size_particle_rhs_send(0),
+ buffer_particles_rhs_recv(in_nb_rhs), size_buffers_recv(0),
+ buffers_size_particle_rhs_recv(0),
nb_processes_involved(0), current_is_involved(true), particles_chunk_per_process(0),
particles_chunk_current_size(0), particles_chunk_current_offset(0) {
@@ -74,7 +115,7 @@ public:
const int MaxProcessesInvolved = std::min(nb_processes, env_utils::GetValue<int>("BFPS_PO_MAX_PROCESSES", 128));
// We split the processes using positions size only
- const size_t totalBytesForPositions = total_nb_particles*size_particle_positions*sizeof(real_number);
+ const size_t totalBytesForPositions = total_nb_particles*size_particle_positions*sizeof(position_type);
if(MinBytesPerProcess*MaxProcessesInvolved < totalBytesForPositions){
@@ -83,13 +124,13 @@ public:
extraChunkBytes += 1;
}
const size_t bytesPerProcess = (MinBytesPerProcess+extraChunkBytes*ChunkBytes);
- particles_chunk_per_process = partsize_t((bytesPerProcess+sizeof(real_number)*size_particle_positions-1)/(sizeof(real_number)*size_particle_positions));
+ particles_chunk_per_process = partsize_t((bytesPerProcess+sizeof(position_type)*size_particle_positions-1)/(sizeof(position_type)*size_particle_positions));
nb_processes_involved = int((total_nb_particles+particles_chunk_per_process-1)/particles_chunk_per_process);
}
// else limit based on minBytesPerProcess
else{
nb_processes_involved = std::max(1,std::min(MaxProcessesInvolved,int((totalBytesForPositions+MinBytesPerProcess-1)/MinBytesPerProcess)));
- particles_chunk_per_process = partsize_t((MinBytesPerProcess+sizeof(real_number)*size_particle_positions-1)/(sizeof(real_number)*size_particle_positions));
+ particles_chunk_per_process = partsize_t((MinBytesPerProcess+sizeof(position_type)*size_particle_positions-1)/(sizeof(position_type)*size_particle_positions));
}
// Print out
@@ -118,7 +159,7 @@ public:
my_rank, &mpi_com_writer) );
}
- virtual ~abstract_particles_output(){
+ virtual ~abstract_particles_output() noexcept(false){
if(current_is_involved){
AssertMpi( MPI_Comm_free(&mpi_com_writer) );
}
@@ -129,21 +170,24 @@ public:
}
void releaseMemory(){
- buffer_indexes_send.release();
- buffer_particles_positions_send.release();
- size_buffers_send = -1;
- buffer_indexes_recv.release();
- buffer_particles_positions_recv.release();
- size_buffers_recv = -1;
+ delete[] buffer_indexes_send.release();
+ delete[] buffer_particles_positions_send.release();
+ size_buffers_send = 0;
+ delete[] buffer_indexes_recv.release();
+ delete[] buffer_particles_positions_recv.release();
+ size_buffers_recv = 0;
for(int idx_rhs = 0 ; idx_rhs < nb_rhs ; ++idx_rhs){
- buffer_particles_rhs_send[idx_rhs].release();
- buffer_particles_rhs_recv[idx_rhs].release();
+ delete[] buffer_particles_rhs_send[idx_rhs].release();
+ delete[] buffer_particles_rhs_recv[idx_rhs].release();
}
+ buffers_size_particle_rhs_send = 0;
+ buffers_size_particle_rhs_recv = 0;
}
+ template <int size_particle_rhs>
void save(
- const real_number input_particles_positions[],
- const std::unique_ptr<real_number[]> input_particles_rhs[],
+ const position_type input_particles_positions[],
+ const std::unique_ptr<output_type[]> input_particles_rhs[],
const partsize_t index_particles[],
const partsize_t nb_particles,
const int idx_time_step){
@@ -153,13 +197,25 @@ public:
{
TIMEZONE("sort-to-distribute");
- if(size_buffers_send < nb_particles && nb_particles){
- buffer_indexes_send.reset(new std::pair<partsize_t,partsize_t>[nb_particles]);
- buffer_particles_positions_send.reset(new real_number[nb_particles*size_particle_positions]);
+ if(size_buffers_send < nb_particles){
+ size_buffers_send = nb_particles;
+ buffer_indexes_send.reset(new std::pair<partsize_t,partsize_t>[size_buffers_send]);
+ buffer_particles_positions_send.reset(new position_type[size_buffers_send*size_particle_positions]);
+
+ if(buffers_size_particle_rhs_send < size_particle_rhs){
+ buffers_size_particle_rhs_send = size_particle_rhs;
+ }
for(int idx_rhs = 0 ; idx_rhs < nb_rhs ; ++idx_rhs){
- buffer_particles_rhs_send[idx_rhs].reset(new real_number[nb_particles*size_particle_rhs]);
+ buffer_particles_rhs_send[idx_rhs].reset(new output_type[size_buffers_send*buffers_size_particle_rhs_send]);
+ }
+ }
+ else if(buffers_size_particle_rhs_send < size_particle_rhs){
+ buffers_size_particle_rhs_send = size_particle_rhs;
+ if(size_buffers_send > 0){
+ for(int idx_rhs = 0 ; idx_rhs < nb_rhs ; ++idx_rhs){
+ buffer_particles_rhs_send[idx_rhs].reset(new output_type[size_buffers_send*buffers_size_particle_rhs_send]);
+ }
}
- size_buffers_send = nb_particles;
}
for(partsize_t idx_part = 0 ; idx_part < nb_particles ; ++idx_part){
@@ -167,10 +223,14 @@ public:
buffer_indexes_send[idx_part].second = index_particles[idx_part];
}
- std::sort(&buffer_indexes_send[0], &buffer_indexes_send[nb_particles], [](const std::pair<partsize_t,partsize_t>& p1,
- const std::pair<partsize_t,partsize_t>& p2){
- return p1.second < p2.second;
- });
+ std::sort(
+ &buffer_indexes_send[0],
+ &buffer_indexes_send[nb_particles],
+ [](const std::pair<partsize_t,partsize_t>& p1,
+ const std::pair<partsize_t,partsize_t>& p2)
+ {
+ return p1.second < p2.second;
+ });
for(partsize_t idx_part = 0 ; idx_part < nb_particles ; ++idx_part){
const partsize_t src_idx = buffer_indexes_send[idx_part].first;
@@ -202,24 +262,36 @@ public:
// nb_particles_to_send is invalid after here
const int nb_to_receive = exchanger.getTotalToRecv();
+ //DEBUG_MSG("nb_to_receive = %d, particles_chunk_current_size = %d\n",
+ // nb_to_receive, particles_chunk_current_size);
assert(nb_to_receive == particles_chunk_current_size);
- if(size_buffers_recv < nb_to_receive && nb_to_receive){
- buffer_indexes_recv.reset(new partsize_t[nb_to_receive]);
- buffer_particles_positions_recv.reset(new real_number[nb_to_receive*size_particle_positions]);
+ if(size_buffers_recv < nb_to_receive){
+ size_buffers_recv = nb_to_receive;
+ buffer_indexes_recv.reset(new partsize_t[size_buffers_recv]);
+ buffer_particles_positions_recv.reset(new position_type[size_buffers_recv*size_particle_positions]);
+
+ buffers_size_particle_rhs_recv = size_particle_rhs;
for(int idx_rhs = 0 ; idx_rhs < nb_rhs ; ++idx_rhs){
- buffer_particles_rhs_recv[idx_rhs].reset(new real_number[nb_to_receive*size_particle_rhs]);
+ buffer_particles_rhs_recv[idx_rhs].reset(new output_type[size_buffers_recv*buffers_size_particle_rhs_recv]);
+ }
+ }
+ else if(buffers_size_particle_rhs_recv < size_particle_rhs){
+ buffers_size_particle_rhs_recv = size_particle_rhs;
+ if(size_buffers_recv > 0){
+ for(int idx_rhs = 0 ; idx_rhs < nb_rhs ; ++idx_rhs){
+ buffer_particles_rhs_recv[idx_rhs].reset(new output_type[size_buffers_recv*buffers_size_particle_rhs_recv]);
+ }
}
- size_buffers_recv = nb_to_receive;
}
{
TIMEZONE("exchange");
// Could be done with multiple asynchronous coms
exchanger.alltoallv<partsize_t>(buffer_indexes_send_tmp, buffer_indexes_recv.get());
- exchanger.alltoallv<real_number>(buffer_particles_positions_send.get(), buffer_particles_positions_recv.get(), size_particle_positions);
+ exchanger.alltoallv<position_type>(buffer_particles_positions_send.get(), buffer_particles_positions_recv.get(), size_particle_positions);
for(int idx_rhs = 0 ; idx_rhs < nb_rhs ; ++idx_rhs){
- exchanger.alltoallv<real_number>(buffer_particles_rhs_send[idx_rhs].get(), buffer_particles_rhs_recv[idx_rhs].get(), size_particle_rhs);
+ exchanger.alltoallv<output_type>(buffer_particles_rhs_send[idx_rhs].get(), buffer_particles_rhs_recv[idx_rhs].get(), size_particle_rhs);
}
}
@@ -229,20 +301,32 @@ public:
return;
}
- if(size_buffers_send < nb_to_receive && nb_to_receive){
- buffer_indexes_send.reset(new std::pair<partsize_t,partsize_t>[nb_to_receive]);
- buffer_particles_positions_send.reset(new real_number[nb_to_receive*size_particle_positions]);
+ if(size_buffers_send < nb_to_receive){
+ size_buffers_send = nb_to_receive;
+ buffer_indexes_send.reset(new std::pair<partsize_t,partsize_t>[size_buffers_send]);
+ buffer_particles_positions_send.reset(new position_type[size_buffers_send*size_particle_positions]);
+ buffers_size_particle_rhs_send = size_particle_rhs;
for(int idx_rhs = 0 ; idx_rhs < nb_rhs ; ++idx_rhs){
- buffer_particles_rhs_send[idx_rhs].reset(new real_number[nb_to_receive*size_particle_rhs]);
+ buffer_particles_rhs_send[idx_rhs].reset(new output_type[size_buffers_send*buffers_size_particle_rhs_send]);
}
- size_buffers_send = nb_to_receive;
}
{
+ // Local process has received particles in `buffer_particles_positions_recv`.
+ // It will now place this data (which is shuffled), in order, in the `buffer_particles_positions_send` array.
+ // However, to place the data it uses the particle indices as memory addresses.
+ // This is fine in general, but if we use particle indices as labels, and we sometimes delete particles, the particle indices are no longer valid memory addresses.
+ // So we need to fix this.
TIMEZONE("copy-local-order");
for(partsize_t idx_part = 0 ; idx_part < nb_to_receive ; ++idx_part){
const partsize_t src_idx = idx_part;
+ //DEBUG_MSG("idx_part = %d, buffer_indexes_recv[idx_part] = %d, particles_chunk_current_offset = %d\n",
+ // idx_part,
+ // buffer_indexes_recv[idx_part],
+ // particles_chunk_current_size);
const partsize_t dst_idx = buffer_indexes_recv[idx_part]-particles_chunk_current_offset;
+ //DEBUG_MSG("dst_idx is %d, particles_chunk_current_size is %d\n",
+ // dst_idx, particles_chunk_current_size);
assert(0 <= dst_idx);
assert(dst_idx < particles_chunk_current_size);
@@ -260,11 +344,11 @@ public:
}
write(idx_time_step, buffer_particles_positions_send.get(), buffer_particles_rhs_send.data(),
- nb_to_receive, particles_chunk_current_offset);
+ nb_to_receive, particles_chunk_current_offset, size_particle_rhs);
}
- virtual void write(const int idx_time_step, const real_number* positions, const std::unique_ptr<real_number[]>* rhs,
- const partsize_t nb_particles, const partsize_t particles_idx_offset) = 0;
+ virtual void write(const int idx_time_step, const position_type* positions, const std::unique_ptr<output_type[]>* rhs,
+ const partsize_t nb_particles, const partsize_t particles_idx_offset, const int size_particle_rhs) = 0;
};
#endif
diff --git a/cpp/particles/abstract_particles_system.hpp b/cpp/particles/abstract_particles_system.hpp
new file mode 100644
index 0000000000000000000000000000000000000000..4edf20c7ca2d34fb023c6a0ea1e370c29e288df0
--- /dev/null
+++ b/cpp/particles/abstract_particles_system.hpp
@@ -0,0 +1,133 @@
+/******************************************************************************
+* *
+* Copyright 2019 Max Planck Institute for Dynamics and Self-Organization *
+* *
+* This file is part of TurTLE. *
+* *
+* TurTLE is free software: you can redistribute it and/or modify *
+* it under the terms of the GNU General Public License as published *
+* by the Free Software Foundation, either version 3 of the License, *
+* or (at your option) any later version. *
+* *
+* TurTLE is distributed in the hope that it will be useful, *
+* but WITHOUT ANY WARRANTY; without even the implied warranty of *
+* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the *
+* GNU General Public License for more details. *
+* *
+* You should have received a copy of the GNU General Public License *
+* along with TurTLE. If not, see <http://www.gnu.org/licenses/> *
+* *
+* Contact: Cristian.Lalescu@ds.mpg.de *
+* *
+******************************************************************************/
+
+
+
+#ifndef ABSTRACT_PARTICLES_SYSTEM_HPP
+#define ABSTRACT_PARTICLES_SYSTEM_HPP
+
+#include <memory>
+#include <vector>
+
+//- Not generic to enable sampling begin
+#include "field.hpp"
+#include "kspace.hpp"
+//- Not generic to enable sampling end
+
+
+template <class partsize_t, class real_number>
+class abstract_particles_system {
+public:
+ virtual ~abstract_particles_system() noexcept(false){}
+
+ virtual void delete_particles_from_indexes(const std::vector<partsize_t>& inIndexToDelete) = 0;
+
+ virtual void compute() = 0;
+
+ virtual void compute_p2p() = 0;
+
+ virtual void compute_particles_inner() = 0;
+
+ virtual void enforce_unit_orientation() = 0;
+
+ virtual void add_Lagrange_multipliers() = 0;
+
+ virtual void compute_sphere_particles_inner(const real_number particle_extra_rhs[]) = 0;
+ virtual void compute_ellipsoid_particles_inner(const real_number particle_extra_rhs[]) = 0;
+
+ virtual void move(const real_number dt) = 0;
+
+ virtual void redistribute() = 0;
+
+ virtual void inc_step_idx() = 0;
+
+ virtual void shift_rhs_vectors() = 0;
+
+ virtual void completeLoop(const real_number dt) = 0;
+ virtual void complete2ndOrderLoop(const real_number dt) = 0;
+
+ virtual void completeLoopWithVorticity(
+ const real_number dt,
+ const real_number sampled_vorticity[]) = 0;
+
+ virtual void completeLoopWithVelocityGradient(
+ const real_number dt,
+ const real_number sampled_velocity_gradient[]) = 0;
+
+ virtual const real_number* getParticlesState() const = 0;
+
+ virtual std::unique_ptr<real_number[]> extractParticlesState(const int firstState, const int lastState) const = 0;
+
+ virtual const std::unique_ptr<real_number[]>* getParticlesRhs() const = 0;
+
+ virtual const partsize_t* getParticlesIndexes() const = 0;
+
+ virtual partsize_t getLocalNbParticles() const = 0;
+
+ virtual partsize_t getGlobalNbParticles() const = 0;
+
+ virtual int getNbRhs() const = 0;
+
+ virtual int get_step_idx() const = 0;
+
+ //- Not generic to enable sampling begin
+ virtual void sample_compute_field(const field<float, FFTW, ONE>& sample_field,
+ real_number sample_rhs[]) = 0;
+ virtual void sample_compute_field(const field<float, FFTW, THREE>& sample_field,
+ real_number sample_rhs[]) = 0;
+ virtual void sample_compute_field(const field<float, FFTW, THREExTHREE>& sample_field,
+ real_number sample_rhs[]) = 0;
+ virtual void sample_compute_field(const field<double, FFTW, ONE>& sample_field,
+ real_number sample_rhs[]) = 0;
+ virtual void sample_compute_field(const field<double, FFTW, THREE>& sample_field,
+ real_number sample_rhs[]) = 0;
+ virtual void sample_compute_field(const field<double, FFTW, THREExTHREE>& sample_field,
+ real_number sample_rhs[]) = 0;
+ //- Not generic to enable sampling end
+
+ template <typename rnumber, field_backend be, field_components fc>
+ void completeLoopWithExtraField(
+ const real_number dt,
+ const field<rnumber, be, fc>& in_field) {
+ static_assert((fc == THREE) || (fc == THREExTHREE), "only THREE or THREExTHREE is supported for now");
+ if (fc == THREE)
+ {
+ std::unique_ptr<real_number[]> extra_rhs(new real_number[getLocalNbParticles()*3]());
+ std::fill_n(extra_rhs.get(), 3*getLocalNbParticles(), 0);
+ sample_compute_field(in_field, extra_rhs.get());
+ completeLoopWithVorticity(dt, extra_rhs.get());
+ }
+ else if (fc == THREExTHREE)
+ {
+ std::unique_ptr<real_number[]> extra_rhs(new real_number[getLocalNbParticles()*9]());
+ std::fill_n(extra_rhs.get(), 9*getLocalNbParticles(), 0);
+ sample_compute_field(in_field, extra_rhs.get());
+ completeLoopWithVelocityGradient(dt, extra_rhs.get());
+ }
+ }
+
+ virtual int setParticleFileLayout(std::vector<hsize_t>) = 0;
+ virtual std::vector<hsize_t> getParticleFileLayout() = 0;
+};
+
+#endif
diff --git a/cpp/particles/abstract_particles_system_with_p2p.hpp b/cpp/particles/abstract_particles_system_with_p2p.hpp
new file mode 100644
index 0000000000000000000000000000000000000000..8ce95b0c35cd36db153176b6f2ec6de6c0099839
--- /dev/null
+++ b/cpp/particles/abstract_particles_system_with_p2p.hpp
@@ -0,0 +1,38 @@
+/******************************************************************************
+* *
+* Copyright 2019 Max Planck Institute for Dynamics and Self-Organization *
+* *
+* This file is part of TurTLE. *
+* *
+* TurTLE is free software: you can redistribute it and/or modify *
+* it under the terms of the GNU General Public License as published *
+* by the Free Software Foundation, either version 3 of the License, *
+* or (at your option) any later version. *
+* *
+* TurTLE is distributed in the hope that it will be useful, *
+* but WITHOUT ANY WARRANTY; without even the implied warranty of *
+* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the *
+* GNU General Public License for more details. *
+* *
+* You should have received a copy of the GNU General Public License *
+* along with TurTLE. If not, see <http://www.gnu.org/licenses/> *
+* *
+* Contact: Cristian.Lalescu@ds.mpg.de *
+* *
+******************************************************************************/
+
+
+
+#ifndef ABSTRACT_PARTICLES_SYSTEM_WITH_P2P_HPP
+#define ABSTRACT_PARTICLES_SYSTEM_WITH_P2P_HPP
+
+#include "abstract_particles_system.hpp"
+
+template <class partsize_t, class real_number, class p2p_computer_class>
+class abstract_particles_system_with_p2p : public abstract_particles_system<partsize_t,real_number> {
+public:
+ virtual p2p_computer_class& getP2PComputer() = 0;
+ virtual const p2p_computer_class& getP2PComputer() const = 0;
+};
+
+#endif // ABSTRACT_PARTICLES_SYSTEM_WITH_P2P_HPP
diff --git a/bfps/cpp/particles/alltoall_exchanger.hpp b/cpp/particles/alltoall_exchanger.hpp
similarity index 79%
rename from bfps/cpp/particles/alltoall_exchanger.hpp
rename to cpp/particles/alltoall_exchanger.hpp
index 2beaf092e8e6c7a801efd492270d29c2d4dba398..288ff47e3b68e9fc5c759ce3547ec530b76e74d8 100644
--- a/bfps/cpp/particles/alltoall_exchanger.hpp
+++ b/cpp/particles/alltoall_exchanger.hpp
@@ -1,3 +1,28 @@
+/******************************************************************************
+* *
+* Copyright 2019 Max Planck Institute for Dynamics and Self-Organization *
+* *
+* This file is part of TurTLE. *
+* *
+* TurTLE is free software: you can redistribute it and/or modify *
+* it under the terms of the GNU General Public License as published *
+* by the Free Software Foundation, either version 3 of the License, *
+* or (at your option) any later version. *
+* *
+* TurTLE is distributed in the hope that it will be useful, *
+* but WITHOUT ANY WARRANTY; without even the implied warranty of *
+* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the *
+* GNU General Public License for more details. *
+* *
+* You should have received a copy of the GNU General Public License *
+* along with TurTLE. If not, see <http://www.gnu.org/licenses/> *
+* *
+* Contact: Cristian.Lalescu@ds.mpg.de *
+* *
+******************************************************************************/
+
+
+
#ifndef ALLTOALL_EXCHANGER_HPP
#define ALLTOALL_EXCHANGER_HPP
diff --git a/cpp/particles/inner/particles_inner_computer.cpp b/cpp/particles/inner/particles_inner_computer.cpp
new file mode 100644
index 0000000000000000000000000000000000000000..2a8ac8da1ab7978a123569b9a052f9821423e9da
--- /dev/null
+++ b/cpp/particles/inner/particles_inner_computer.cpp
@@ -0,0 +1,193 @@
+/******************************************************************************
+* *
+* Copyright 2019 Max Planck Institute for Dynamics and Self-Organization *
+* *
+* This file is part of TurTLE. *
+* *
+* TurTLE is free software: you can redistribute it and/or modify *
+* it under the terms of the GNU General Public License as published *
+* by the Free Software Foundation, either version 3 of the License, *
+* or (at your option) any later version. *
+* *
+* TurTLE is distributed in the hope that it will be useful, *
+* but WITHOUT ANY WARRANTY; without even the implied warranty of *
+* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the *
+* GNU General Public License for more details. *
+* *
+* You should have received a copy of the GNU General Public License *
+* along with TurTLE. If not, see <http://www.gnu.org/licenses/> *
+* *
+* Contact: Cristian.Lalescu@ds.mpg.de *
+* *
+******************************************************************************/
+
+
+
+#include "base.hpp"
+#include "particles/particles_utils.hpp"
+#include "particles/inner/particles_inner_computer.hpp"
+
+#include <cmath>
+
+template <class real_number, class partsize_t>
+template <int size_particle_positions, int size_particle_rhs>
+void particles_inner_computer<real_number, partsize_t>::compute_interaction(
+ const partsize_t nb_particles,
+ const real_number pos_part[],
+ real_number rhs_part[]) const{
+ static_assert(size_particle_positions == 6, "This kernel works only with 6 values for one particle's position");
+ static_assert(size_particle_rhs == 6, "This kernel works only with 6 values per particle's rhs");
+
+ #pragma omp parallel for
+ for(partsize_t idx_part = 0 ; idx_part < nb_particles ; ++idx_part){
+ // Add attr × V0 to the field interpolation
+ rhs_part[idx_part*size_particle_rhs + IDXC_X] += pos_part[idx_part*size_particle_positions + 3+IDXC_X]*v0;
+ rhs_part[idx_part*size_particle_rhs + IDXC_Y] += pos_part[idx_part*size_particle_positions + 3+IDXC_Y]*v0;
+ rhs_part[idx_part*size_particle_rhs + IDXC_Z] += pos_part[idx_part*size_particle_positions + 3+IDXC_Z]*v0;
+ }
+}
+
+ // for given orientation and right-hand-side, recompute right-hand-side such
+ // that it is perpendicular to the current orientation.
+ // this is the job of the Lagrange multiplier terms, hence the
+ // "add_Lagrange_multipliers" name of the method.
+template <>
+template <>
+void particles_inner_computer<double, long long>::add_Lagrange_multipliers<6,6>(
+ const long long nb_particles,
+ const double pos_part[],
+ double rhs_part[]) const{
+
+ #pragma omp parallel for
+ for(long long idx_part = 0 ; idx_part < nb_particles ; ++idx_part){
+ const long long idx0 = idx_part*6 + 3;
+ const long long idx1 = idx_part*6 + 3;
+ // check that orientation is unit vector:
+ double orientation_size = sqrt(
+ pos_part[idx0+IDXC_X]*pos_part[idx0+IDXC_X] +
+ pos_part[idx0+IDXC_Y]*pos_part[idx0+IDXC_Y] +
+ pos_part[idx0+IDXC_Z]*pos_part[idx0+IDXC_Z]);
+ variable_used_only_in_assert(orientation_size);
+ assert(orientation_size > 0.99);
+ assert(orientation_size < 1.01);
+ // I call "rotation" to be the right hand side of the orientation part of the ODE
+ // project rotation on orientation:
+ double projection = (
+ pos_part[idx0+IDXC_X]*rhs_part[idx1+IDXC_X] +
+ pos_part[idx0+IDXC_Y]*rhs_part[idx1+IDXC_Y] +
+ pos_part[idx0+IDXC_Z]*rhs_part[idx1+IDXC_Z]);
+
+ // now remove parallel bit.
+ rhs_part[idx1+IDXC_X] -= pos_part[idx0+IDXC_X]*projection;
+ rhs_part[idx1+IDXC_Y] -= pos_part[idx0+IDXC_Y]*projection;
+ rhs_part[idx1+IDXC_Z] -= pos_part[idx0+IDXC_Z]*projection;
+
+ // DEBUG
+ // sanity check, for debugging purposes
+ // compute dot product between orientation and orientation change
+ //double dotproduct = (
+ // rhs_part[idx1 + IDXC_X]*pos_part[idx0 + IDXC_X] +
+ // rhs_part[idx1 + IDXC_Y]*pos_part[idx0 + IDXC_Y] +
+ // rhs_part[idx1 + IDXC_Z]*pos_part[idx0 + IDXC_Z]);
+ //if (dotproduct > 0.1)
+ //{
+ // DEBUG_MSG("dotproduct = %g, projection = %g\n"
+ // "pos_part[%d] = %g, pos_part[%d] = %g, pos_part[%d] = %g\n"
+ // "rhs_part[%d] = %g, rhs_part[%d] = %g, rhs_part[%d] = %g\n",
+ // dotproduct,
+ // projection,
+ // IDXC_X, pos_part[idx0 + IDXC_X],
+ // IDXC_Y, pos_part[idx0 + IDXC_Y],
+ // IDXC_Z, pos_part[idx0 + IDXC_Z],
+ // IDXC_X, rhs_part[idx1 + IDXC_X],
+ // IDXC_Y, rhs_part[idx1 + IDXC_Y],
+ // IDXC_Z, rhs_part[idx1 + IDXC_Z]);
+ // assert(false);
+ //}
+ //assert(dotproduct <= 0.1);
+ }
+ }
+
+template <>
+template <>
+void particles_inner_computer<double, long long>::compute_interaction_with_extra<6,6,3>(
+ const long long nb_particles,
+ const double pos_part[],
+ double rhs_part[],
+ const double rhs_part_extra[]) const{
+ // call plain compute_interaction first
+ compute_interaction<6, 6>(nb_particles, pos_part, rhs_part);
+
+ // now add vorticity term
+ #pragma omp parallel for
+ for(long long idx_part = 0 ; idx_part < nb_particles ; ++idx_part){
+ // Cross product vorticity/orientation
+ rhs_part[idx_part*6 + 3+IDXC_X] += 0.5*(rhs_part_extra[idx_part*3 + IDXC_Y]*pos_part[idx_part*6 + 3+IDXC_Z] -
+ rhs_part_extra[idx_part*3 + IDXC_Z]*pos_part[idx_part*6 + 3+IDXC_Y]);
+ rhs_part[idx_part*6 + 3+IDXC_Y] += 0.5*(rhs_part_extra[idx_part*3 + IDXC_Z]*pos_part[idx_part*6 + 3+IDXC_X] -
+ rhs_part_extra[idx_part*3 + IDXC_X]*pos_part[idx_part*6 + 3+IDXC_Z]);
+ rhs_part[idx_part*6 + 3+IDXC_Z] += 0.5*(rhs_part_extra[idx_part*3 + IDXC_X]*pos_part[idx_part*6 + 3+IDXC_Y] -
+ rhs_part_extra[idx_part*3 + IDXC_Y]*pos_part[idx_part*6 + 3+IDXC_X]);
+ }
+}
+
+template <> //Work here
+template <>
+void particles_inner_computer<double, long long>::compute_interaction_with_extra<6,6,9>(
+ const long long nb_particles,
+ const double pos_part[],
+ double rhs_part[],
+ const double rhs_part_extra[]) const{
+ // call plain compute_interaction first
+ compute_interaction<6, 6>(nb_particles, pos_part, rhs_part);
+ const double ll2 = lambda*lambda;
+
+ // now add vorticity term
+ #pragma omp parallel for
+ for(long long idx_part = 0 ; idx_part < nb_particles ; ++idx_part){
+ long long idx_part6 = idx_part*6 + 3;
+ long long idx_part9 = idx_part*9;
+ rhs_part[idx_part6+IDXC_X] += (
+ pos_part[idx_part6+IDXC_Z]*(ll2*rhs_part_extra[idx_part9 + IDXC_DZ_X]-rhs_part_extra[idx_part9 + IDXC_DX_Z])
+ + pos_part[idx_part6+IDXC_Y]*(ll2*rhs_part_extra[idx_part9 + IDXC_DY_X]-rhs_part_extra[idx_part9 + IDXC_DX_Y])
+ + pos_part[idx_part6+IDXC_X]*(ll2-1)*rhs_part_extra[idx_part9 + IDXC_DX_X]) / (ll2+1);
+ rhs_part[idx_part6+IDXC_Y] += (
+ pos_part[idx_part6+IDXC_X]*(ll2*rhs_part_extra[idx_part9 + IDXC_DX_Y]-rhs_part_extra[idx_part9 + IDXC_DY_X])
+ + pos_part[idx_part6+IDXC_Z]*(ll2*rhs_part_extra[idx_part9 + IDXC_DZ_Y]-rhs_part_extra[idx_part9 + IDXC_DY_Z])
+ + pos_part[idx_part6+IDXC_Y]*(ll2-1)*rhs_part_extra[idx_part9 + IDXC_DY_Y]) / (ll2+1);
+ rhs_part[idx_part6+IDXC_Z] += (
+ pos_part[idx_part6+IDXC_Y]*(ll2*rhs_part_extra[idx_part9 + IDXC_DY_Z]-rhs_part_extra[idx_part9 + IDXC_DZ_Y])
+ + pos_part[idx_part6+IDXC_X]*(ll2*rhs_part_extra[idx_part9 + IDXC_DX_Z]-rhs_part_extra[idx_part9 + IDXC_DZ_X])
+ + pos_part[idx_part6+IDXC_Z]*(ll2-1)*rhs_part_extra[idx_part9 + IDXC_DZ_Z]) / (ll2+1);
+ }
+}
+
+
+// meant to be called AFTER executing the time-stepping operation.
+// once the particles have been moved, ensure that the orientation is a unit vector.
+template <>
+template <>
+void particles_inner_computer<double, long long>::enforce_unit_orientation<6>(
+ const long long nb_particles,
+ double pos_part[]) const{
+ #pragma omp parallel for
+ for(long long idx_part = 0 ; idx_part < nb_particles ; ++idx_part){
+ const long long idx0 = idx_part*6 + 3;
+ // compute orientation size:
+ double orientation_size = sqrt(
+ pos_part[idx0+IDXC_X]*pos_part[idx0+IDXC_X] +
+ pos_part[idx0+IDXC_Y]*pos_part[idx0+IDXC_Y] +
+ pos_part[idx0+IDXC_Z]*pos_part[idx0+IDXC_Z]);
+ // now renormalize
+ pos_part[idx0 + IDXC_X] /= orientation_size;
+ pos_part[idx0 + IDXC_Y] /= orientation_size;
+ pos_part[idx0 + IDXC_Z] /= orientation_size;
+ }
+}
+
+template
+void particles_inner_computer<double, long long>::compute_interaction<6, 6>(
+ const long long nb_particles,
+ const double pos_part[],
+ double rhs_part[]) const;
+
diff --git a/cpp/particles/inner/particles_inner_computer.hpp b/cpp/particles/inner/particles_inner_computer.hpp
new file mode 100644
index 0000000000000000000000000000000000000000..fd9dd678c27c2f1f878a40cecd7b2a1df4a0c727
--- /dev/null
+++ b/cpp/particles/inner/particles_inner_computer.hpp
@@ -0,0 +1,109 @@
+/******************************************************************************
+* *
+* Copyright 2019 Max Planck Institute for Dynamics and Self-Organization *
+* *
+* This file is part of TurTLE. *
+* *
+* TurTLE is free software: you can redistribute it and/or modify *
+* it under the terms of the GNU General Public License as published *
+* by the Free Software Foundation, either version 3 of the License, *
+* or (at your option) any later version. *
+* *
+* TurTLE is distributed in the hope that it will be useful, *
+* but WITHOUT ANY WARRANTY; without even the implied warranty of *
+* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the *
+* GNU General Public License for more details. *
+* *
+* You should have received a copy of the GNU General Public License *
+* along with TurTLE. If not, see <http://www.gnu.org/licenses/> *
+* *
+* Contact: Cristian.Lalescu@ds.mpg.de *
+* *
+******************************************************************************/
+
+
+
+#ifndef PARTICLES_INNER_COMPUTER_HPP
+#define PARTICLES_INNER_COMPUTER_HPP
+
+#include <cstring>
+#include <cassert>
+#include <iostream>
+
+template <class real_number, class partsize_t>
+class particles_inner_computer{
+ bool isActive;
+ const real_number v0;
+ const real_number lambda;
+ const real_number lambda1;
+ const real_number lambda2;
+ const real_number lambda3;
+
+public:
+ explicit particles_inner_computer(const real_number inV0):
+ isActive(true),
+ v0(inV0),
+ lambda(0),
+ lambda1(0),
+ lambda2(0),
+ lambda3(0)
+ {}
+ explicit particles_inner_computer(const real_number inV0, const real_number inLambda):
+ isActive(true),
+ v0(inV0),
+ lambda(inLambda),
+ lambda1(0),
+ lambda2(0),
+ lambda3(0)
+ {}
+ explicit particles_inner_computer(
+ const real_number inV0,
+ const real_number inLambda1,
+ const real_number inLambda2,
+ const real_number inLambda3):
+ isActive(true),
+ v0(inV0),
+ lambda(0),
+ lambda1(inLambda1),
+ lambda2(inLambda2),
+ lambda3(inLambda3)
+ {}
+
+ template <int size_particle_positions, int size_particle_rhs>
+ void compute_interaction(
+ const partsize_t nb_particles,
+ const real_number pos_part[],
+ real_number rhs_part[]) const;
+ // for given orientation and right-hand-side, recompute right-hand-side such
+ // that it is perpendicular to the current orientation.
+ // this is the job of the Lagrange multiplier terms, hence the
+ // "add_Lagrange_multipliers" name of the method.
+ template <int size_particle_positions, int size_particle_rhs>
+ void add_Lagrange_multipliers(
+ const partsize_t nb_particles,
+ const real_number pos_part[],
+ real_number rhs_part[]) const;
+ template <int size_particle_positions, int size_particle_rhs, int size_particle_rhs_extra>
+ void compute_interaction_with_extra(
+ const partsize_t nb_particles,
+ const real_number pos_part[],
+ real_number rhs_part[],
+ const real_number rhs_part_extra[]) const;
+ // meant to be called AFTER executing the time-stepping operation.
+ // once the particles have been moved, ensure that the orientation is a unit vector.
+ template <int size_particle_positions>
+ void enforce_unit_orientation(
+ const partsize_t nb_particles,
+ real_number pos_part[]) const;
+
+ bool isEnable() const {
+ return isActive;
+ }
+
+ void setEnable(const bool inIsActive) {
+ isActive = inIsActive;
+ }
+};
+
+#endif
+
diff --git a/cpp/particles/inner/particles_inner_computer_2nd_order.hpp b/cpp/particles/inner/particles_inner_computer_2nd_order.hpp
new file mode 100644
index 0000000000000000000000000000000000000000..eebe162e627709bafd73b067d8660b6cdc2fd6fd
--- /dev/null
+++ b/cpp/particles/inner/particles_inner_computer_2nd_order.hpp
@@ -0,0 +1,108 @@
+/******************************************************************************
+* *
+* Copyright 2019 Max Planck Institute for Dynamics and Self-Organization *
+* *
+* This file is part of TurTLE. *
+* *
+* TurTLE is free software: you can redistribute it and/or modify *
+* it under the terms of the GNU General Public License as published *
+* by the Free Software Foundation, either version 3 of the License, *
+* or (at your option) any later version. *
+* *
+* TurTLE is distributed in the hope that it will be useful, *
+* but WITHOUT ANY WARRANTY; without even the implied warranty of *
+* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the *
+* GNU General Public License for more details. *
+* *
+* You should have received a copy of the GNU General Public License *
+* along with TurTLE. If not, see <http://www.gnu.org/licenses/> *
+* *
+* Contact: Cristian.Lalescu@ds.mpg.de *
+* *
+******************************************************************************/
+
+
+
+#ifndef PARTICLES_INNER_COMPUTER_2ND_ORDER_HPP
+#define PARTICLES_INNER_COMPUTER_2ND_ORDER_HPP
+
+#include <cstring>
+#include <cassert>
+
+/** \brief Computes action of Stokes drag on particles.
+ *
+ * Either 6 state variables (positions + momenta) or 7 state variables (positions + momenta + value of drag coefficient).
+ * The case of particles with individual values of the drag coefficient is relevant for the case of plastic collisions,
+ * when we do not have a priori knowledge of what the mass/size/drag coefficient of the particle is.
+ *
+ * `compute_interaction_with_extra` is currently only defined for 6 or 7 state variables.
+ *
+ */
+
+template <class real_number, class partsize_t>
+class particles_inner_computer_2nd_order_Stokes{
+ double drag_coefficient;
+public:
+ template <int size_particle_state, int size_particle_rhs>
+ void compute_interaction(const partsize_t number_of_particles, real_number particle_state[], real_number particle_rhs[]) const{
+ }
+
+ template <int size_particle_state>
+ void enforce_unit_orientation(const partsize_t /*nb_particles*/, real_number /*pos_part*/[]) const{
+ }
+
+ template <int size_particle_state, int size_particle_rhs>
+ void add_Lagrange_multipliers(const partsize_t /*nb_particles*/, real_number /*pos_part*/[], real_number /*rhs_part*/[]) const{
+ }
+
+ template <int size_particle_state, int size_particle_rhs, int size_particle_rhs_extra>
+ void compute_interaction_with_extra(
+ const partsize_t number_of_particles,
+ real_number particle_state[],
+ real_number particle_rhs[],
+ const real_number sampled_velocity[]) const{
+ assert(size_particle_rhs_extra == 3);
+ assert(size_particle_state == size_particle_rhs);
+ assert((size_particle_state == 6) || (size_particle_state == 7));
+ if (size_particle_state == 6){
+ #pragma omp parallel for
+ for(partsize_t idx_part = 0 ; idx_part < number_of_particles ; ++idx_part){
+ particle_rhs[idx_part*size_particle_rhs + IDXC_X] = particle_state[idx_part*size_particle_state + 3 + IDXC_X];
+ particle_rhs[idx_part*size_particle_rhs + IDXC_Y] = particle_state[idx_part*size_particle_state + 3 + IDXC_Y];
+ particle_rhs[idx_part*size_particle_rhs + IDXC_Z] = particle_state[idx_part*size_particle_state + 3 + IDXC_Z];
+ particle_rhs[idx_part*size_particle_rhs + 3 + IDXC_X] = - this->drag_coefficient * (particle_state[idx_part*size_particle_state + 3 + IDXC_X] - sampled_velocity[idx_part*size_particle_rhs_extra + IDXC_X]);
+ particle_rhs[idx_part*size_particle_rhs + 3 + IDXC_Y] = - this->drag_coefficient * (particle_state[idx_part*size_particle_state + 3 + IDXC_Y] - sampled_velocity[idx_part*size_particle_rhs_extra + IDXC_Y]);
+ particle_rhs[idx_part*size_particle_rhs + 3 + IDXC_Z] = - this->drag_coefficient * (particle_state[idx_part*size_particle_state + 3 + IDXC_Z] - sampled_velocity[idx_part*size_particle_rhs_extra + IDXC_Z]);
+ }
+ }
+ else if (size_particle_state == 7){
+ // particle stores its own drag coefficient in the 7th state variable
+ #pragma omp parallel for
+ for(partsize_t idx_part = 0 ; idx_part < number_of_particles ; ++idx_part){
+ particle_rhs[idx_part*size_particle_rhs + IDXC_X] = particle_state[idx_part*size_particle_state + 3 + IDXC_X];
+ particle_rhs[idx_part*size_particle_rhs + IDXC_Y] = particle_state[idx_part*size_particle_state + 3 + IDXC_Y];
+ particle_rhs[idx_part*size_particle_rhs + IDXC_Z] = particle_state[idx_part*size_particle_state + 3 + IDXC_Z];
+ particle_rhs[idx_part*size_particle_rhs + 3 + IDXC_X] = - particle_state[idx_part*size_particle_state + 6] * (particle_state[idx_part*size_particle_state + 3 + IDXC_X] - sampled_velocity[idx_part*size_particle_rhs_extra + IDXC_X]);
+ particle_rhs[idx_part*size_particle_rhs + 3 + IDXC_Y] = - particle_state[idx_part*size_particle_state + 6] * (particle_state[idx_part*size_particle_state + 3 + IDXC_Y] - sampled_velocity[idx_part*size_particle_rhs_extra + IDXC_Y]);
+ particle_rhs[idx_part*size_particle_rhs + 3 + IDXC_Z] = - particle_state[idx_part*size_particle_state + 6] * (particle_state[idx_part*size_particle_state + 3 + IDXC_Z] - sampled_velocity[idx_part*size_particle_rhs_extra + IDXC_Z]);
+ }
+ }
+ }
+
+ constexpr static bool isEnable() {
+ return true;
+ }
+
+ void set_drag_coefficient(double mu)
+ {
+ this->drag_coefficient = mu;
+ }
+
+ double get_drag_coefficient()
+ {
+ return this->drag_coefficient;
+ }
+};
+
+#endif//PARTICLES_INNER_COMPUTER_2ND_ORDER_HPP
+
diff --git a/cpp/particles/inner/particles_inner_computer_empty.hpp b/cpp/particles/inner/particles_inner_computer_empty.hpp
new file mode 100644
index 0000000000000000000000000000000000000000..aeff11c4e1c87256f216b3c70eda6fe78ea39f65
--- /dev/null
+++ b/cpp/particles/inner/particles_inner_computer_empty.hpp
@@ -0,0 +1,57 @@
+/******************************************************************************
+* *
+* Copyright 2019 Max Planck Institute for Dynamics and Self-Organization *
+* *
+* This file is part of TurTLE. *
+* *
+* TurTLE is free software: you can redistribute it and/or modify *
+* it under the terms of the GNU General Public License as published *
+* by the Free Software Foundation, either version 3 of the License, *
+* or (at your option) any later version. *
+* *
+* TurTLE is distributed in the hope that it will be useful, *
+* but WITHOUT ANY WARRANTY; without even the implied warranty of *
+* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the *
+* GNU General Public License for more details. *
+* *
+* You should have received a copy of the GNU General Public License *
+* along with TurTLE. If not, see <http://www.gnu.org/licenses/> *
+* *
+* Contact: Cristian.Lalescu@ds.mpg.de *
+* *
+******************************************************************************/
+
+
+
+#ifndef PARTICLES_INNER_COMPUTER_EMPTY_HPP
+#define PARTICLES_INNER_COMPUTER_EMPTY_HPP
+
+#include <cstring>
+#include <cassert>
+
+template <class real_number, class partsize_t>
+class particles_inner_computer_empty{
+public:
+ template <int size_particle_positions, int size_particle_rhs>
+ void compute_interaction(const partsize_t /*nb_particles*/, real_number /*pos_part*/[], real_number /*rhs_part*/[]) const{
+ }
+
+ template <int size_particle_positions>
+ void enforce_unit_orientation(const partsize_t /*nb_particles*/, real_number /*pos_part*/[]) const{
+ }
+
+ template <int size_particle_positions, int size_particle_rhs>
+ void add_Lagrange_multipliers(const partsize_t /*nb_particles*/, real_number /*pos_part*/[], real_number /*rhs_part*/[]) const{
+ }
+
+ template <int size_particle_positions, int size_particle_rhs, int size_particle_rhs_extra>
+ void compute_interaction_with_extra(const partsize_t /*nb_particles*/, real_number /*pos_part*/[], real_number /*rhs_part*/[],
+ const real_number /*rhs_part_extra*/[]) const{
+ }
+
+ constexpr static bool isEnable() {
+ return false;
+ }
+};
+
+#endif
diff --git a/bfps/cpp/Lagrange_polys.cpp b/cpp/particles/interpolation/Lagrange_polys.cpp
similarity index 99%
rename from bfps/cpp/Lagrange_polys.cpp
rename to cpp/particles/interpolation/Lagrange_polys.cpp
index bbc8997f896aebba486e4fa601c18eaf2aacfcc1..eff66cd40cb5c8dd70a6813a4b5882e52893a467 100644
--- a/bfps/cpp/Lagrange_polys.cpp
+++ b/cpp/particles/interpolation/Lagrange_polys.cpp
@@ -3,20 +3,20 @@
* Copyright 2017 Max Planck Institute *
* for Dynamics and Self-Organization *
* *
-* This file is part of bfps. *
+* This file is part of TurTLE. *
* *
-* bfps is free software: you can redistribute it and/or modify *
+* TurTLE is free software: you can redistribute it and/or modify *
* it under the terms of the GNU General Public License as published *
* by the Free Software Foundation, either version 3 of the License, *
* or (at your option) any later version. *
* *
-* bfps is distributed in the hope that it will be useful, *
+* TurTLE is distributed in the hope that it will be useful, *
* but WITHOUT ANY WARRANTY; without even the implied warranty of *
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the *
* GNU General Public License for more details. *
* *
* You should have received a copy of the GNU General Public License *
-* along with bfps. If not, see <http://www.gnu.org/licenses/> *
+* along with TurTLE. If not, see <http://www.gnu.org/licenses/> *
* *
* Contact: Cristian.Lalescu@ds.mpg.de *
* *
diff --git a/bfps/cpp/Lagrange_polys.hpp b/cpp/particles/interpolation/Lagrange_polys.hpp
similarity index 89%
rename from bfps/cpp/Lagrange_polys.hpp
rename to cpp/particles/interpolation/Lagrange_polys.hpp
index 9f4742a3303d9586cc7832e568162465767a4922..7d4d27f8ac2868c0acab48e07e9af23d09b2423d 100644
--- a/bfps/cpp/Lagrange_polys.hpp
+++ b/cpp/particles/interpolation/Lagrange_polys.hpp
@@ -3,20 +3,20 @@
* Copyright 2015 Max Planck Institute *
* for Dynamics and Self-Organization *
* *
-* This file is part of bfps. *
+* This file is part of TurTLE. *
* *
-* bfps is free software: you can redistribute it and/or modify *
+* TurTLE is free software: you can redistribute it and/or modify *
* it under the terms of the GNU General Public License as published *
* by the Free Software Foundation, either version 3 of the License, *
* or (at your option) any later version. *
* *
-* bfps is distributed in the hope that it will be useful, *
+* TurTLE is distributed in the hope that it will be useful, *
* but WITHOUT ANY WARRANTY; without even the implied warranty of *
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the *
* GNU General Public License for more details. *
* *
* You should have received a copy of the GNU General Public License *
-* along with bfps. If not, see <http://www.gnu.org/licenses/> *
+* along with TurTLE. If not, see <http://www.gnu.org/licenses/> *
* *
* Contact: Cristian.Lalescu@ds.mpg.de *
* *
diff --git a/cpp/particles/interpolation/abstract_particle_set.hpp b/cpp/particles/interpolation/abstract_particle_set.hpp
new file mode 100644
index 0000000000000000000000000000000000000000..95d94fa1798a806cc97fd92197e96757ba174564
--- /dev/null
+++ b/cpp/particles/interpolation/abstract_particle_set.hpp
@@ -0,0 +1,418 @@
+/******************************************************************************
+* *
+* Copyright 2020 Max Planck Institute for Dynamics and Self-Organization *
+* *
+* This file is part of TurTLE. *
+* *
+* TurTLE is free software: you can redistribute it and/or modify *
+* it under the terms of the GNU General Public License as published *
+* by the Free Software Foundation, either version 3 of the License, *
+* or (at your option) any later version. *
+* *
+* TurTLE is distributed in the hope that it will be useful, *
+* but WITHOUT ANY WARRANTY; without even the implied warranty of *
+* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the *
+* GNU General Public License for more details. *
+* *
+* You should have received a copy of the GNU General Public License *
+* along with TurTLE. If not, see <http://www.gnu.org/licenses/> *
+* *
+* Contact: Cristian.Lalescu@ds.mpg.de *
+* *
+******************************************************************************/
+
+
+
+#ifndef ABSTRACT_PARTICLE_SET_HPP
+#define ABSTRACT_PARTICLE_SET_HPP
+
+#include <array>
+#include <cassert>
+#include <vector>
+#include <memory>
+#include <string>
+#include "field.hpp"
+#include "particles/p2p/p2p_distr_mpi.hpp"
+#include "particles/particles_sampling.hpp"
+
+
+
+/** \brief Brings together particle information with interpolation functionality.
+ *
+ * This is an abstract class that defines the functionality required by the
+ * particle solver code.
+ * We define methods for:
+ * - accessing particle set data and miscelaneous information.
+ * - updating particle information and redistributing data among MPI processes
+ * accordingly.
+ * - approximating fields at particle locations ("sampling").
+ * - applying a generic particle-to-particle interaction object (P2PKernel)
+ *
+ */
+
+class abstract_particle_set
+{
+ public:
+ using partsize_t = long long int;
+ using particle_rnumber = double;
+
+ // virtual destructor
+ virtual ~abstract_particle_set() noexcept(false){}
+
+ // extract particle information
+ virtual particle_rnumber* getParticleState() const = 0;
+ virtual partsize_t* getParticleIndices() const = 0;
+ virtual partsize_t* getParticleLabels() const = 0;
+ virtual int setParticleState(particle_rnumber *) = 0;
+ virtual int getParticleState(particle_rnumber *) = 0;
+
+ virtual std::unique_ptr<particle_rnumber[]> extractFromParticleState(
+ const int firstComponent,
+ const int lastComponent) const = 0;
+
+ virtual partsize_t getLocalNumberOfParticles() const = 0;
+ virtual partsize_t getTotalNumberOfParticles() const = 0;
+ virtual partsize_t* getParticlesPerPartition() const = 0;
+ virtual int getStateSize() const = 0;
+
+ virtual std::vector<hsize_t> getParticleFileLayout() = 0;
+
+ // get p2p computer
+ virtual p2p_distr_mpi<partsize_t, particle_rnumber>* getP2PComputer() = 0;
+
+ // generic loop over data
+ template <class func_type>
+ int LOOP(func_type expression)
+ {
+ TIMEZONE("abstract_particle_set::LOOP");
+ for (partsize_t idx = 0; idx < this->getLocalNumberOfParticles()*this->getStateSize(); idx++)
+ expression(idx);
+ return EXIT_SUCCESS;
+ }
+ template <class func_type>
+ int LOOP_state(func_type expression)
+ {
+ TIMEZONE("abstract_particle_set::LOOP_state");
+ for (partsize_t idx_part = 0; idx_part < this->getLocalNumberOfParticles(); idx_part++)
+ for (unsigned int cc = 0; cc < this->getStateSize(); cc++)
+ expression(idx_part, cc);
+ return EXIT_SUCCESS;
+ }
+
+ int copy_state_tofrom(
+ particle_rnumber *dst,
+ const particle_rnumber *src)
+ {
+ return this->LOOP(
+ [&](const partsize_t idx)
+ {
+ dst[idx] = src[idx];
+ });
+ }
+
+ int copy_state_tofrom(
+ std::unique_ptr<particle_rnumber[]> &dst,
+ const std::unique_ptr<particle_rnumber[]> &src)
+ {
+ return this->LOOP(
+ [&](const partsize_t idx)
+ {
+ dst[idx] = src[idx];
+ });
+ }
+
+ /** Reorganize particles within MPI domain.
+ *
+ * Based on current particle locations, redistribute the full state
+ * data among the MPI processes.
+ * Optional: also redistribute a list of arrays of the same shape.
+ */
+ virtual int redistribute(
+ std::vector<std::unique_ptr<particle_rnumber[]>> &additional_data) = 0;
+
+ // sample field values at particle location
+ virtual int sample(
+ const field<float,
+ FFTW,
+ ONE> &field_to_sample,
+ particle_rnumber *result) = 0;
+ virtual int sample(
+ const field<float,
+ FFTW,
+ THREE> &field_to_sample,
+ particle_rnumber *result) = 0;
+ virtual int sample(
+ const field<float,
+ FFTW,
+ THREExTHREE> &field_to_sample,
+ particle_rnumber *result) = 0;
+ virtual int sample(
+ const field<double,
+ FFTW,
+ ONE> &field_to_sample,
+ particle_rnumber *result) = 0;
+ virtual int sample(
+ const field<double,
+ FFTW,
+ THREE> &field_to_sample,
+ particle_rnumber *result) = 0;
+ virtual int sample(
+ const field<double,
+ FFTW,
+ THREExTHREE> &field_to_sample,
+ particle_rnumber *result) = 0;
+
+ // apply p2p kernel
+ // IMPORTANT: this method shuffles particle arrays.
+ // If you need some data to be shuffled according to the same permutation,
+ // please place it in `additional_data`.
+ // This applies for instance to rhs data during time steps.
+ template <int state_size,
+ class p2p_kernel_class>
+ int applyP2PKernel(
+ p2p_kernel_class &p2p_kernel,
+ std::vector<std::unique_ptr<particle_rnumber[]>> &additional_data)
+ {
+ TIMEZONE("abstract_particle_set::applyP2PKernel");
+ // there must be at least one array with additional data,
+ // since the p2p kernel expects an array where to store the result of the computation
+ assert(int(additional_data.size()) > int(0));
+ this->getP2PComputer()->template compute_distr<p2p_kernel_class,
+ state_size, // I'd prefer to use this->getStateSize() here. not possible because virtual function is not constexpr...
+ state_size>(
+ p2p_kernel,
+ this->getParticlesPerPartition(),
+ this->getParticleState(),
+ additional_data.data(),
+ int(additional_data.size()),
+ this->getParticleIndices(),
+ this->getParticleLabels());
+ return EXIT_SUCCESS;
+ }
+
+ int writeParticleLabels(
+ const std::string file_name,
+ const std::string species_name,
+ const std::string field_name,
+ const int iteration)
+ {
+ TIMEZONE("abstract_particle_set::writeParticleLabels");
+ MPI_Barrier(MPI_COMM_WORLD);
+ particles_output_sampling_hdf5<partsize_t, particle_rnumber, partsize_t, 3> *particle_sample_writer = new particles_output_sampling_hdf5<partsize_t, double, partsize_t, 3>(
+ MPI_COMM_WORLD,
+ this->getTotalNumberOfParticles(),
+ file_name,
+ species_name,
+ field_name + std::string("/") + std::to_string(iteration));
+ // set file layout
+ particle_sample_writer->setParticleFileLayout(this->getParticleFileLayout());
+ // allocate position array
+ std::unique_ptr<particle_rnumber[]> xx = this->extractFromParticleState(0, 3);
+ // allocate temporary array
+ std::unique_ptr<partsize_t[]> pdata(new partsize_t[this->getLocalNumberOfParticles()]);
+ // clean up temporary array
+ std::copy(this->getParticleLabels(),
+ this->getParticleLabels() + this->getLocalNumberOfParticles(),
+ pdata.get());
+ particle_sample_writer->template save_dataset<1>(
+ species_name,
+ field_name,
+ xx.get(),
+ &pdata,
+ this->getParticleIndices(),
+ this->getLocalNumberOfParticles(),
+ iteration);
+ // deallocate sample writer
+ delete particle_sample_writer;
+ // deallocate temporary array
+ delete[] pdata.release();
+ // deallocate position array
+ delete[] xx.release();
+ MPI_Barrier(MPI_COMM_WORLD);
+ return EXIT_SUCCESS;
+ }
+
+ template <typename field_rnumber,
+ field_backend be,
+ field_components fc>
+ int writeSample(
+ field<field_rnumber, be, fc> *field_to_sample,
+ particles_output_sampling_hdf5<partsize_t,
+ particle_rnumber,
+ particle_rnumber,
+ 3> *particle_sample_writer,
+ const std::string species_name,
+ const std::string field_name,
+ const int iteration)
+ {
+ TIMEZONE("abstract_particle_set::writeSample");
+ // set file layout
+ particle_sample_writer->setParticleFileLayout(this->getParticleFileLayout());
+ // allocate position array
+ std::unique_ptr<particle_rnumber[]> xx = this->extractFromParticleState(0, 3);
+ // allocate temporary array
+ std::unique_ptr<particle_rnumber[]> pdata(new particle_rnumber[ncomp(fc)*this->getLocalNumberOfParticles()]);
+ // clean up temporary array
+ std::fill_n(pdata.get(), ncomp(fc)*this->getLocalNumberOfParticles(), 0);
+ this->sample(*field_to_sample, pdata.get());
+ particle_sample_writer->template save_dataset<ncomp(fc)>(
+ species_name,
+ field_name,
+ xx.get(),
+ &pdata,
+ this->getParticleIndices(),
+ this->getLocalNumberOfParticles(),
+ iteration);
+ // deallocate temporary array
+ delete[] pdata.release();
+ // deallocate position array
+ delete[] xx.release();
+ return EXIT_SUCCESS;
+ }
+
+ template <typename field_rnumber,
+ field_backend be,
+ field_components fc>
+ int writeSample(
+ field<field_rnumber, be, fc> *field_to_sample,
+ const std::string file_name,
+ const std::string species_name,
+ const std::string field_name,
+ const int iteration)
+ {
+ TIMEZONE("abstract_particle_set::writeSample_direct_to_file");
+ MPI_Barrier(field_to_sample->comm);
+ // allocate temporary array
+ std::unique_ptr<particle_rnumber[]> pdata(new particle_rnumber[ncomp(fc)*this->getLocalNumberOfParticles()]);
+ // clean up temporary array
+ set_particle_data_to_zero<partsize_t, particle_rnumber, ncomp(fc)>(
+ pdata.get(), this->getLocalNumberOfParticles());
+ this->sample(*field_to_sample, pdata.get());
+ hid_t file_id = -1;
+ if (field_to_sample->myrank == 0)
+ {
+ file_id = H5Fopen(file_name.c_str(), H5F_ACC_RDWR, H5P_DEFAULT);
+ assert(file_id > 0);
+ }
+ hdf5_tools::gather_and_write_with_single_rank(
+ field_to_sample->myrank,
+ 0,
+ field_to_sample->comm,
+ pdata.get(),
+ this->getLocalNumberOfParticles()*ncomp(fc),
+ file_id,
+ species_name + std::string("/") + field_name + std::string("/") + std::to_string(iteration));
+ if (field_to_sample->myrank == 0)
+ H5Fclose(file_id);
+ // deallocate temporary array
+ delete[] pdata.release();
+ MPI_Barrier(field_to_sample->comm);
+ return EXIT_SUCCESS;
+ }
+
+ int writeStateChunk(
+ const int i0,
+ const int contiguous_state_chunk,
+ particles_output_sampling_hdf5<partsize_t,
+ particle_rnumber,
+ particle_rnumber,
+ 3> *particle_sample_writer,
+ const std::string species_name,
+ const std::string field_name,
+ const int iteration)
+ {
+ TIMEZONE("abstract_particle_set::writeStateChunk");
+ MPI_Barrier(MPI_COMM_WORLD);
+ assert(i0 >= 0);
+ assert(i0 <= this->getStateSize()-contiguous_state_chunk);
+ // set file layout
+ particle_sample_writer->setParticleFileLayout(this->getParticleFileLayout());
+ // allocate position array
+ std::unique_ptr<particle_rnumber[]> xx = this->extractFromParticleState(0, 3);
+ // allocate temporary array
+ std::unique_ptr<particle_rnumber[]> yy = this->extractFromParticleState(i0, i0+contiguous_state_chunk);
+ switch(contiguous_state_chunk)
+ {
+ case 1:
+ particle_sample_writer->template save_dataset<1>(
+ species_name,
+ field_name,
+ xx.get(),
+ &yy,
+ this->getParticleIndices(),
+ this->getLocalNumberOfParticles(),
+ iteration);
+ break;
+ case 3:
+ particle_sample_writer->template save_dataset<3>(
+ species_name,
+ field_name,
+ xx.get(),
+ &yy,
+ this->getParticleIndices(),
+ this->getLocalNumberOfParticles(),
+ iteration);
+ break;
+ case 9:
+ particle_sample_writer->template save_dataset<9>(
+ species_name,
+ field_name,
+ xx.get(),
+ &yy,
+ this->getParticleIndices(),
+ this->getLocalNumberOfParticles(),
+ iteration);
+ break;
+ default:
+ DEBUG_MSG("code not specialized for this value of contiguous_state_chunk in abstract_particle_set::writeStateChunk\n");
+ std::cerr << "error in abstract_particle_set::writeStateChunk. please contact maintainer.\n" << std::endl;
+ }
+ // deallocate temporary array
+ delete[] yy.release();
+ // deallocate position array
+ delete[] xx.release();
+ MPI_Barrier(MPI_COMM_WORLD);
+ return EXIT_SUCCESS;
+ }
+
+ int writeStateTriplet(
+ const int i0,
+ particles_output_sampling_hdf5<partsize_t,
+ particle_rnumber,
+ particle_rnumber,
+ 3> *particle_sample_writer,
+ const std::string species_name,
+ const std::string field_name,
+ const int iteration)
+ {
+ TIMEZONE("abstract_particle_set::writeStateTriplet");
+ return this->writeStateChunk(
+ i0, 3,
+ particle_sample_writer,
+ species_name,
+ field_name,
+ iteration);
+ }
+
+ int writeStateComponent(
+ const int i0,
+ particles_output_sampling_hdf5<partsize_t,
+ particle_rnumber,
+ particle_rnumber,
+ 3> *particle_sample_writer,
+ const std::string species_name,
+ const std::string field_name,
+ const int iteration)
+ {
+ TIMEZONE("abstract_particle_set::writeStateComponent");
+ return this->writeStateChunk(
+ i0, 1,
+ particle_sample_writer,
+ species_name,
+ field_name,
+ iteration);
+ }
+};
+
+#endif//ABSTRACT_PARTICLE_SET_HPP
+
diff --git a/cpp/particles/interpolation/field_tinterpolator.cpp b/cpp/particles/interpolation/field_tinterpolator.cpp
new file mode 100644
index 0000000000000000000000000000000000000000..968829b98702f82c7f53a9d954e3abada249ac35
--- /dev/null
+++ b/cpp/particles/interpolation/field_tinterpolator.cpp
@@ -0,0 +1,41 @@
+/******************************************************************************
+* *
+* Copyright 2020 Max Planck Institute for Dynamics and Self-Organization *
+* *
+* This file is part of TurTLE. *
+* *
+* TurTLE is free software: you can redistribute it and/or modify *
+* it under the terms of the GNU General Public License as published *
+* by the Free Software Foundation, either version 3 of the License, *
+* or (at your option) any later version. *
+* *
+* TurTLE is distributed in the hope that it will be useful, *
+* but WITHOUT ANY WARRANTY; without even the implied warranty of *
+* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the *
+* GNU General Public License for more details. *
+* *
+* You should have received a copy of the GNU General Public License *
+* along with TurTLE. If not, see <http://www.gnu.org/licenses/> *
+* *
+* Contact: Cristian.Lalescu@ds.mpg.de *
+* *
+******************************************************************************/
+
+#include "particles/interpolation/field_tinterpolator.hpp"
+
+template class field_tinterpolator<float, FFTW, ONE, NONE>;
+template class field_tinterpolator<float, FFTW, THREE, NONE>;
+template class field_tinterpolator<float, FFTW, THREExTHREE, NONE>;
+
+template class field_tinterpolator<double, FFTW, ONE, NONE>;
+template class field_tinterpolator<double, FFTW, THREE, NONE>;
+template class field_tinterpolator<double, FFTW, THREExTHREE, NONE>;
+
+template class field_tinterpolator<float, FFTW, ONE, LINEAR>;
+template class field_tinterpolator<float, FFTW, THREE, LINEAR>;
+template class field_tinterpolator<float, FFTW, THREExTHREE, LINEAR>;
+
+template class field_tinterpolator<double, FFTW, ONE, LINEAR>;
+template class field_tinterpolator<double, FFTW, THREE, LINEAR>;
+template class field_tinterpolator<double, FFTW, THREExTHREE, LINEAR>;
+
diff --git a/cpp/particles/interpolation/field_tinterpolator.hpp b/cpp/particles/interpolation/field_tinterpolator.hpp
new file mode 100644
index 0000000000000000000000000000000000000000..2dc27b56a69d5462b131347b23c8fce1903864af
--- /dev/null
+++ b/cpp/particles/interpolation/field_tinterpolator.hpp
@@ -0,0 +1,133 @@
+/******************************************************************************
+* *
+* Copyright 2020 Max Planck Institute for Dynamics and Self-Organization *
+* *
+* This file is part of TurTLE. *
+* *
+* TurTLE is free software: you can redistribute it and/or modify *
+* it under the terms of the GNU General Public License as published *
+* by the Free Software Foundation, either version 3 of the License, *
+* or (at your option) any later version. *
+* *
+* TurTLE is distributed in the hope that it will be useful, *
+* but WITHOUT ANY WARRANTY; without even the implied warranty of *
+* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the *
+* GNU General Public License for more details. *
+* *
+* You should have received a copy of the GNU General Public License *
+* along with TurTLE. If not, see <http://www.gnu.org/licenses/> *
+* *
+* Contact: Cristian.Lalescu@ds.mpg.de *
+* *
+******************************************************************************/
+
+
+
+#ifndef FIELD_TINTERPOLATOR_HPP
+#define FIELD_TINTERPOLATOR_HPP
+
+#include <array>
+#include <cassert>
+#include <vector>
+#include "field.hpp"
+#include "particles/particles_distr_mpi.hpp"
+#include "particles/particles_output_hdf5.hpp"
+#include "particles/interpolation/abstract_particle_set.hpp"
+
+enum temporal_interpolation_type {NONE, LINEAR, CUBIC};
+
+template <typename rnumber,
+ field_backend be,
+ field_components fc,
+ temporal_interpolation_type tt>
+class field_tinterpolator
+{
+ private:
+ using particle_rnumber = double;
+ using partsize_t = long long int;
+
+ std::array<field<rnumber, be, fc>*, 4> field_list; // list of fields for temporal interpolation
+
+ public:
+ field_tinterpolator(){
+ this->field_list[0] = NULL;
+ this->field_list[1] = NULL;
+ this->field_list[2] = NULL;
+ this->field_list[3] = NULL;
+ }
+
+ ~field_tinterpolator() noexcept(false){}
+
+ int set_field(
+ field<rnumber, be, fc> *field_src = NULL,
+ const int tindex = 0)
+ {
+ switch(tt)
+ {
+ case NONE:
+ assert(tindex == 0);
+ this->field_list[0] = field_src;
+ break;
+ case LINEAR:
+ assert(tindex == 0 || tindex == 1);
+ this->field_list[tindex] = field_src;
+ break;
+ case CUBIC:
+ // TODO: add cubic spline interpolation in time here
+ assert(false);
+ break;
+ }
+ return EXIT_SUCCESS;
+ }
+
+ // t is a fraction between 0 and 1.
+ int operator()(
+ double t,
+ abstract_particle_set &pset,
+ particle_rnumber *result)
+ {
+ switch(tt)
+ {
+ case NONE:
+ {
+ pset.sample(*(this->field_list[0]), result);
+ break;
+ }
+ case LINEAR:
+ {
+ particle_rnumber *result0, *result1;
+ result0 = new particle_rnumber[pset.getLocalNumberOfParticles()*ncomp(fc)];
+ result1 = new particle_rnumber[pset.getLocalNumberOfParticles()*ncomp(fc)];
+ // interpolation adds on top of existing values, so result must be cleared.
+ std::fill_n(result0, pset.getLocalNumberOfParticles()*ncomp(fc), 0);
+ pset.sample(*(this->field_list[0]), result0);
+ // interpolation adds on top of existing values, so result must be cleared.
+ std::fill_n(result1, pset.getLocalNumberOfParticles()*ncomp(fc), 0);
+ pset.sample(*(this->field_list[1]), result1);
+ // not using LOOP method because we want the inner loop over ncomp,
+ // not over pset.getStateSize()
+ DEBUG_MSG("t value in linear interpolation is %g\n", t);
+ for (partsize_t idx_part = 0; idx_part < pset.getLocalNumberOfParticles(); idx_part++)
+ {
+ for (unsigned int cc = 0; cc < ncomp(fc); cc++)
+ {
+ result[idx_part*ncomp(fc) + cc] = (
+ (1-t)*result0[idx_part*ncomp(fc) + cc] +
+ t *result1[idx_part*ncomp(fc) + cc]);
+ }
+ }
+ delete[] result0;
+ delete[] result1;
+ break;
+ }
+ case CUBIC:
+ // TODO: add cubic spline interpolation in time here
+ assert(false);
+ break;
+ }
+ return EXIT_SUCCESS;
+ }
+};
+
+#endif//FIELD_TINTERPOLATOR_HPP
+
diff --git a/cpp/particles/interpolation/particle_set.cpp b/cpp/particles/interpolation/particle_set.cpp
new file mode 100644
index 0000000000000000000000000000000000000000..3d0284a338138045bfebca6e8a11fee7c10c840b
--- /dev/null
+++ b/cpp/particles/interpolation/particle_set.cpp
@@ -0,0 +1,37 @@
+/******************************************************************************
+* *
+* Copyright 2020 Max Planck Institute for Dynamics and Self-Organization *
+* *
+* This file is part of TurTLE. *
+* *
+* TurTLE is free software: you can redistribute it and/or modify *
+* it under the terms of the GNU General Public License as published *
+* by the Free Software Foundation, either version 3 of the License, *
+* or (at your option) any later version. *
+* *
+* TurTLE is distributed in the hope that it will be useful, *
+* but WITHOUT ANY WARRANTY; without even the implied warranty of *
+* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the *
+* GNU General Public License for more details. *
+* *
+* You should have received a copy of the GNU General Public License *
+* along with TurTLE. If not, see <http://www.gnu.org/licenses/> *
+* *
+* Contact: Cristian.Lalescu@ds.mpg.de *
+* *
+******************************************************************************/
+
+#include "particles/interpolation/particle_set.hpp"
+
+template class particle_set<3, 1, 0>;
+template class particle_set<3, 1, 1>;
+template class particle_set<3, 2, 0>;
+template class particle_set<3, 2, 1>;
+template class particle_set<3, 2, 2>;
+
+template class particle_set<15, 1, 0>;
+template class particle_set<15, 1, 1>;
+template class particle_set<15, 2, 0>;
+template class particle_set<15, 2, 1>;
+template class particle_set<15, 2, 2>;
+
diff --git a/cpp/particles/interpolation/particle_set.hpp b/cpp/particles/interpolation/particle_set.hpp
new file mode 100644
index 0000000000000000000000000000000000000000..071614ad52369d596565a50fa2e8ac34d5c22e99
--- /dev/null
+++ b/cpp/particles/interpolation/particle_set.hpp
@@ -0,0 +1,742 @@
+/******************************************************************************
+* *
+* Copyright 2020 Max Planck Institute for Dynamics and Self-Organization *
+* *
+* This file is part of TurTLE. *
+* *
+* TurTLE is free software: you can redistribute it and/or modify *
+* it under the terms of the GNU General Public License as published *
+* by the Free Software Foundation, either version 3 of the License, *
+* or (at your option) any later version. *
+* *
+* TurTLE is distributed in the hope that it will be useful, *
+* but WITHOUT ANY WARRANTY; without even the implied warranty of *
+* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the *
+* GNU General Public License for more details. *
+* *
+* You should have received a copy of the GNU General Public License *
+* along with TurTLE. If not, see <http://www.gnu.org/licenses/> *
+* *
+* Contact: Cristian.Lalescu@ds.mpg.de *
+* *
+******************************************************************************/
+
+
+
+#ifndef PARTICLE_SET_HPP
+#define PARTICLE_SET_HPP
+
+#include <array>
+#include <cassert>
+#include <vector>
+#include <set>
+#include <memory>
+#include "field_layout.hpp"
+#include "field.hpp"
+#include "particles/particles_distr_mpi.hpp"
+#include "particles/abstract_particles_output.hpp"
+#include "particles/abstract_particles_input.hpp"
+#include "particles/interpolation/particles_field_computer.hpp"
+#include "particles/interpolation/particles_generic_interp.hpp"
+#include "particles/interpolation/abstract_particle_set.hpp"
+#include "particles/p2p/p2p_distr_mpi.hpp"
+
+
+
+/** Practical implementation of particle sets.
+ *
+ * Child of `abstract_particle_set`. Brings together the functionality of
+ * `particles_field_computer` and `particles_distr_mpi`.
+ *
+ */
+
+template <int state_size,
+ int neighbours,
+ int smoothness>
+class particle_set: public abstract_particle_set
+{
+ private:
+ // custom names for types
+ using interpolator_class = particles_field_computer<partsize_t,
+ particle_rnumber,
+ particles_generic_interp<particle_rnumber, neighbours, smoothness>,
+ neighbours>;
+ using distributor_class = particles_distr_mpi<partsize_t, particle_rnumber>;
+
+ // MPI
+ MPI_Comm mpi_comm;
+
+ // data
+ const std::pair<int,int> current_partition_interval;
+ const int partition_interval_size;
+
+ std::unique_ptr<partsize_t[]> number_particles_per_partition;
+ std::unique_ptr<partsize_t[]> offset_particles_for_partition;
+
+ std::unique_ptr<particle_rnumber[]> local_state;
+ std::unique_ptr<partsize_t[]> local_indices;
+ std::unique_ptr<partsize_t[]> local_labels;
+ partsize_t local_number_of_particles;
+ partsize_t total_number_of_particles;
+
+ // for I/O it helps to have custom array shape, we store it here
+ std::vector<hsize_t> particle_file_layout;
+
+ // raw objects for distributing data in MPI, and for interpolating fields
+ distributor_class pDistributor;
+ particles_generic_interp<particle_rnumber, neighbours, smoothness> interpolation_formula;
+ interpolator_class pInterpolator;
+ p2p_distr_mpi<partsize_t, particle_rnumber> p2pComputer;
+
+ public:
+ particle_set(
+ const abstract_field_layout *fl,
+ const double dkx,
+ const double dky,
+ const double dkz,
+ const double p2p_cutoff = 1.0):
+ mpi_comm(fl->getMPIComm()),
+ current_partition_interval(
+ {fl->getStart(IDXV_Z),
+ fl->getStart(IDXV_Z) + fl->getSubSize(IDXV_Z)}),
+ partition_interval_size(current_partition_interval.second - current_partition_interval.first),
+ pDistributor(
+ mpi_comm,
+ current_partition_interval,
+ std::array<size_t, 3>({ //field_grid_dim from particles_system_builder
+ fl->getSize(IDXV_X),
+ fl->getSize(IDXV_Y),
+ fl->getSize(IDXV_Z),
+ })),
+ interpolation_formula(),
+ pInterpolator(
+ std::array<size_t, 3>({ //field_grid_dim from particles_system_builder
+ fl->getSize(IDXV_X),
+ fl->getSize(IDXV_Y),
+ fl->getSize(IDXV_Z),
+ }),
+ current_partition_interval,
+ interpolation_formula,
+ std::array<particle_rnumber, 3>({ // spatial_box_width from particles_system_builder
+ 4*acos(0) / dkx,
+ 4*acos(0) / dky,
+ 4*acos(0) / dkz,
+ }),
+ std::array<particle_rnumber, 3>({ // spatial_box_offset from particles_system_builder
+ 0,
+ 0,
+ 0,
+ }),
+ std::array<particle_rnumber, 3>({ // spatial_box_width divided by field_grid_dim
+ (4*acos(0) / dkx) / fl->getSize(IDXV_X),
+ (4*acos(0) / dky) / fl->getSize(IDXV_Y),
+ (4*acos(0) / dkz) / fl->getSize(IDXV_Z)
+ })),
+ p2pComputer(
+ mpi_comm,
+ current_partition_interval,
+ std::array<size_t, 3>({ //field_grid_dim from particles_system_builder
+ fl->getSize(IDXV_X),
+ fl->getSize(IDXV_Y),
+ fl->getSize(IDXV_Z),
+ }),
+ std::array<particle_rnumber, 3>({ // spatial_box_width from particles_system_builder
+ 4*acos(0) / dkx,
+ 4*acos(0) / dky,
+ 4*acos(0) / dkz,
+ }),
+ std::array<particle_rnumber, 3>({ // spatial_box_offset from particles_system_builder
+ 0,
+ 0,
+ 0,
+ }),
+ p2p_cutoff)
+
+ {
+ TIMEZONE("particle_set::particle_set");
+ // if these assertions fail,
+ // it means the constructor prototype is broken
+ // because the arrays above are wrong.
+ assert(IDXC_X == 0);
+ assert(IDXC_Y == 1);
+ assert(IDXC_Z == 2);
+
+ //DEBUG_MSG("current partition interval %d %d\n",
+ // current_partition_interval.first,
+ // current_partition_interval.second);
+
+ this->number_particles_per_partition.reset(new partsize_t[partition_interval_size]);
+ this->offset_particles_for_partition.reset(new partsize_t[partition_interval_size+1]);
+ }
+
+ int _print_debug_info()
+ {
+ int myrank;
+ AssertMpi(MPI_Comm_rank(mpi_comm, &myrank));
+ int nprocs;
+ AssertMpi(MPI_Comm_size(mpi_comm, &nprocs));
+
+ for (int rr = 0; rr < nprocs; rr++)
+ {
+ if (myrank == rr)
+ {
+ DEBUG_MSG("##########################################\n");
+ DEBUG_MSG("# particle_set debug info follows\n");
+
+ DEBUG_MSG("current partition interval %d %d\n",
+ this->current_partition_interval.first,
+ this->current_partition_interval.second);
+ DEBUG_MSG("partition_interval_size %d\n",
+ this->partition_interval_size);
+
+ for (int i=0; i<this->partition_interval_size; i++)
+ {
+ DEBUG_MSG("number_particles_per_partition[%d] = %d\n",
+ i,
+ number_particles_per_partition[i]);
+ }
+
+ for (int i=0; i<this->partition_interval_size+1; i++)
+ {
+ DEBUG_MSG("offset_particles_for_partition[%d] = %d\n",
+ i,
+ offset_particles_for_partition[i]);
+ }
+
+ DEBUG_MSG("local_number_of_particles = %d\n",
+ this->local_number_of_particles);
+ DEBUG_MSG("total_number_of_particles = %d\n",
+ this->total_number_of_particles);
+
+ for (int i=0; i<int(this->particle_file_layout.size()); i++)
+ DEBUG_MSG("particle_file_layout[%d] = %d\n",
+ i,
+ this->particle_file_layout[i]);
+ if (this->total_number_of_particles < 100)
+ {
+ for (partsize_t ii=0; ii < this->local_number_of_particles; ii++)
+ {
+ DEBUG_MSG("local_counter = %d, index = %d, label = %d\n",
+ ii,
+ this->local_indices[ii],
+ this->local_labels[ii]);
+ }
+ }
+
+ DEBUG_MSG("# particle_set debug info ends\n");
+ DEBUG_MSG("##########################################\n");
+ }
+ MPI_Barrier(this->mpi_comm);
+ }
+ return EXIT_SUCCESS;
+ }
+
+ ~particle_set() noexcept(false){}
+
+ particle_rnumber* getParticleState() const
+ {
+ return this->local_state.get();
+ }
+
+ int setParticleState(particle_rnumber *src_local_state)
+ {
+ this->LOOP(
+ [&](const partsize_t idx){
+ this->local_state[idx] = src_local_state[idx];
+ });
+ return EXIT_SUCCESS;
+ }
+
+ int getParticleState(particle_rnumber *dst_local_state)
+ {
+ this->LOOP(
+ [&](const partsize_t idx){
+ dst_local_state[idx] = this->local_state[idx];
+ });
+ return EXIT_SUCCESS;
+ }
+
+ partsize_t* getParticleIndices() const
+ {
+ return this->local_indices.get();
+ }
+
+ partsize_t* getParticleLabels() const
+ {
+ return this->local_labels.get();
+ }
+
+ partsize_t getLocalNumberOfParticles() const
+ {
+ return this->local_number_of_particles;
+ }
+
+ partsize_t getTotalNumberOfParticles() const
+ {
+ return this->total_number_of_particles;
+ }
+
+ partsize_t* getParticlesPerPartition() const
+ {
+ return this->number_particles_per_partition.get();
+ }
+
+ int getStateSize() const
+ {
+ return state_size;
+ }
+
+ int setParticleFileLayout(std::vector<hsize_t> input_layout)
+ {
+ this->particle_file_layout.resize(input_layout.size());
+ for (unsigned int i=0; i<this->particle_file_layout.size(); i++)
+ this->particle_file_layout[i] = input_layout[i];
+ return EXIT_SUCCESS;
+ }
+
+ std::vector<hsize_t> getParticleFileLayout(void)
+ {
+ return std::vector<hsize_t>(this->particle_file_layout);
+ }
+
+ std::unique_ptr<particle_rnumber[]> extractFromParticleState(
+ const int firstState,
+ const int lastState) const
+ {
+ TIMEZONE("particle_set::extractFromParticleState");
+ const int numberOfStates = std::max(0,(std::min(lastState, state_size)-firstState));
+
+ std::unique_ptr<particle_rnumber[]> stateExtract(new particle_rnumber[local_number_of_particles*numberOfStates]);
+
+ for(partsize_t idx_part = 0 ; idx_part < this->local_number_of_particles ; ++idx_part){
+ for(int idxState = 0 ; idxState < numberOfStates ; ++idxState){
+ stateExtract[idx_part*numberOfStates + idxState] = this->local_state[idx_part*state_size + idxState+firstState];
+ }
+ }
+
+ return stateExtract;
+ }
+
+ template <typename field_rnumber,
+ field_backend be,
+ field_components fc>
+ int template_sample(const field<field_rnumber, be, fc> &field_to_sample,
+ particle_rnumber *result)
+ {
+ TIMEZONE("particle_set::template_sample");
+ // attention: compute_distr adds result on top of existing values.
+ // please clean up result as appropriate before call.
+ this->pDistributor.template compute_distr<interpolator_class,
+ field<field_rnumber, be, fc>,
+ state_size,
+ ncomp(fc)>(
+ this->pInterpolator,
+ field_to_sample,
+ this->number_particles_per_partition.get(),
+ this->local_state.get(),
+ result,
+ neighbours);
+ return EXIT_SUCCESS;
+ }
+
+ int sample(const field<float, FFTW, ONE> &field_to_sample,
+ particle_rnumber *result)
+ {
+ return template_sample<float, FFTW, ONE>(field_to_sample, result);
+ }
+ int sample(const field<float, FFTW, THREE> &field_to_sample,
+ particle_rnumber *result)
+ {
+ return template_sample<float, FFTW, THREE>(field_to_sample, result);
+ }
+ int sample(const field<float, FFTW, THREExTHREE> &field_to_sample,
+ particle_rnumber *result)
+ {
+ return template_sample<float, FFTW, THREExTHREE>(field_to_sample, result);
+ }
+ int sample(const field<double, FFTW, ONE> &field_to_sample,
+ particle_rnumber *result)
+ {
+ return template_sample<double, FFTW, ONE>(field_to_sample, result);
+ }
+ int sample(const field<double, FFTW, THREE> &field_to_sample,
+ particle_rnumber *result)
+ {
+ return template_sample<double, FFTW, THREE>(field_to_sample, result);
+ }
+ int sample(const field<double, FFTW, THREExTHREE> &field_to_sample,
+ particle_rnumber *result)
+ {
+ return template_sample<double, FFTW, THREExTHREE>(field_to_sample, result);
+ }
+
+ int redistribute(std::vector<std::unique_ptr<particle_rnumber[]>> &additional_data)
+ {
+ TIMEZONE("particle_set::redistribute");
+ this->pDistributor.template redistribute<interpolator_class,
+ state_size,
+ state_size,
+ 1>(
+ this->pInterpolator,
+ this->number_particles_per_partition.get(),
+ &this->local_number_of_particles,
+ &this->local_state,
+ additional_data.data(),
+ int(additional_data.size()),
+ &this->local_indices,
+ &this->local_labels);
+ return EXIT_SUCCESS;
+ }
+
+ int init(abstract_particles_input<partsize_t, particle_rnumber>& particles_input)
+ {
+ TIMEZONE("particle_set::init");
+
+ this->local_state = particles_input.getMyParticles();
+ this->local_indices = particles_input.getMyParticlesIndexes();
+ this->local_labels = particles_input.getMyParticlesLabels();
+ this->local_number_of_particles = particles_input.getLocalNbParticles();
+ this->total_number_of_particles = particles_input.getTotalNbParticles();
+
+
+ particles_utils::partition_extra_z<partsize_t, state_size>(
+ &this->local_state[0],
+ this->local_number_of_particles,
+ partition_interval_size,
+ this->number_particles_per_partition.get(),
+ this->offset_particles_for_partition.get(),
+ [&](const particle_rnumber& z_pos){
+ const int partition_level = this->pInterpolator.pbc_field_layer(z_pos, IDXC_Z);
+ assert(current_partition_interval.first <= partition_level && partition_level < current_partition_interval.second);
+ return partition_level - current_partition_interval.first;
+ },
+ [&](const partsize_t idx1, const partsize_t idx2){
+ std::swap(this->local_indices[idx1], this->local_indices[idx2]);
+ std::swap(this->local_labels[idx1], this->local_labels[idx2]);
+ });
+
+ int file_layout_result = this->setParticleFileLayout(particles_input.getParticleFileLayout());
+ variable_used_only_in_assert(file_layout_result);
+ assert(file_layout_result == EXIT_SUCCESS);
+
+ return EXIT_SUCCESS;
+ }
+
+ int init_as_subset_of(
+ abstract_particles_system<partsize_t, particle_rnumber> &src,
+ const std::vector<partsize_t> &indices_to_copy)
+ {
+ TIMEZONE("particle_set::init_as_subset_of");
+ assert(indices_to_copy.size() > 0);
+ particle_rnumber *tmp_local_state = new particle_rnumber[state_size * src.getLocalNbParticles()];
+ partsize_t *tmp_local_indices = new partsize_t[src.getLocalNbParticles()];
+ partsize_t *tmp_local_labels = new partsize_t[src.getLocalNbParticles()];
+ this->local_number_of_particles = 0;
+ this->total_number_of_particles = indices_to_copy.size();
+
+ //DEBUG_MSG("particle_set::init_as_subset_of, desired subset_size = %ld, src_local_number = %ld, src_total_number = %ld\n",
+ // this->total_number_of_particles,
+ // src.getLocalNbParticles(),
+ // src.getGlobalNbParticles());
+
+ // dumb selection of interesting particles
+ for (partsize_t ii = 0; ii < partsize_t(src.getLocalNbParticles()); ii++)
+ {
+ partsize_t src_label = src.getParticlesIndexes()[ii];
+ for (partsize_t iii=0; iii < partsize_t(indices_to_copy.size()); iii++)
+ {
+ if (src_label == indices_to_copy[iii])
+ {
+ tmp_local_indices[this->local_number_of_particles] = iii;
+ tmp_local_labels[this->local_number_of_particles] = src_label;
+ std::copy(src.getParticlesState()+state_size*ii,
+ src.getParticlesState()+state_size*(ii+1),
+ tmp_local_state + state_size*this->local_number_of_particles);
+ this->local_number_of_particles++;
+ break;
+ }
+ }
+ }
+
+ // now we actually put the data "here"
+ if (this->local_number_of_particles > 0)
+ {
+ this->local_state.reset(new particle_rnumber[state_size*this->local_number_of_particles]);
+ this->local_indices.reset(new partsize_t[this->local_number_of_particles]);
+ this->local_labels.reset(new partsize_t[this->local_number_of_particles]);
+ std::copy(tmp_local_state,
+ tmp_local_state + state_size*this->local_number_of_particles,
+ this->local_state.get());
+ std::copy(tmp_local_indices,
+ tmp_local_indices + this->local_number_of_particles,
+ this->local_indices.get());
+ std::copy(tmp_local_labels,
+ tmp_local_labels + this->local_number_of_particles,
+ this->local_labels.get());
+ }
+
+ particles_utils::partition_extra_z<partsize_t, state_size>(
+ &this->local_state[0],
+ this->local_number_of_particles,
+ partition_interval_size,
+ this->number_particles_per_partition.get(),
+ this->offset_particles_for_partition.get(),
+ [&](const particle_rnumber& z_pos){
+ const int partition_level = this->pInterpolator.pbc_field_layer(z_pos, IDXC_Z);
+ assert(current_partition_interval.first <= partition_level && partition_level < current_partition_interval.second);
+ return partition_level - current_partition_interval.first;
+ },
+ [&](const partsize_t idx1, const partsize_t idx2){
+ std::swap(this->local_indices[idx1], this->local_indices[idx2]);
+ std::swap(this->local_labels[idx1], this->local_labels[idx2]);
+ });
+
+ delete[] tmp_local_state;
+ delete[] tmp_local_indices;
+ delete[] tmp_local_labels;
+
+ std::vector<hsize_t> tmp_file_layout(1);
+ tmp_file_layout[0] = hsize_t(this->total_number_of_particles);
+
+ int file_layout_result = this->setParticleFileLayout(tmp_file_layout);
+ variable_used_only_in_assert(file_layout_result);
+ assert(file_layout_result == EXIT_SUCCESS);
+ return EXIT_SUCCESS;
+ }
+
+ int operator=(
+ abstract_particle_set *src)
+ {
+ TIMEZONE("particle_set::operator=");
+ assert(src->getStateSize() == state_size);
+ particle_rnumber *tmp_local_state = new particle_rnumber[state_size * src->getLocalNumberOfParticles()];
+ partsize_t *tmp_local_indices = new partsize_t[src->getLocalNumberOfParticles()];
+ partsize_t *tmp_local_labels = new partsize_t[src->getLocalNumberOfParticles()];
+ this->local_number_of_particles = 0;
+
+ this->total_number_of_particles = src->getTotalNumberOfParticles();
+
+ for (partsize_t ii = 0; ii < partsize_t(src->getLocalNumberOfParticles()); ii++)
+ {
+ partsize_t src_label = src->getParticleLabels()[ii];
+ partsize_t src_index = src->getParticleIndices()[ii];
+
+ {
+ // set new index
+ tmp_local_indices[this->local_number_of_particles] = src_index;
+ tmp_local_labels[this->local_number_of_particles] = src_label;
+ std::copy(src->getParticleState()+state_size*ii,
+ src->getParticleState()+state_size*(ii+1),
+ tmp_local_state + state_size*this->local_number_of_particles);
+ this->local_number_of_particles++;
+ }
+ }
+
+ // now we actually put the data "here"
+ if (this->local_number_of_particles > 0)
+ {
+ this->local_state.reset(new particle_rnumber[state_size*this->local_number_of_particles]);
+ this->local_indices.reset(new partsize_t[this->local_number_of_particles]);
+ this->local_labels.reset(new partsize_t[this->local_number_of_particles]);
+ std::copy(tmp_local_state,
+ tmp_local_state + state_size*this->local_number_of_particles,
+ this->local_state.get());
+ std::copy(tmp_local_indices,
+ tmp_local_indices + this->local_number_of_particles,
+ this->local_indices.get());
+ std::copy(tmp_local_labels,
+ tmp_local_labels + this->local_number_of_particles,
+ this->local_labels.get());
+ }
+
+ particles_utils::partition_extra_z<partsize_t, state_size>(
+ &this->local_state[0],
+ this->local_number_of_particles,
+ partition_interval_size,
+ this->number_particles_per_partition.get(),
+ this->offset_particles_for_partition.get(),
+ [&](const particle_rnumber& z_pos){
+ const int partition_level = this->pInterpolator.pbc_field_layer(z_pos, IDXC_Z);
+ assert(current_partition_interval.first <= partition_level && partition_level < current_partition_interval.second);
+ return partition_level - current_partition_interval.first;
+ },
+ [&](const partsize_t idx1, const partsize_t idx2){
+ std::swap(this->local_indices[idx1], this->local_indices[idx2]);
+ std::swap(this->local_labels[idx1], this->local_labels[idx2]);
+ });
+
+ delete[] tmp_local_state;
+ delete[] tmp_local_indices;
+ delete[] tmp_local_labels;
+
+ int file_layout_result = this->setParticleFileLayout(src->getParticleFileLayout());
+ variable_used_only_in_assert(file_layout_result);
+ assert(file_layout_result == EXIT_SUCCESS);
+ return EXIT_SUCCESS;
+ }
+
+ int reset_labels()
+ {
+ std::copy(
+ this->local_indices.get(),
+ this->local_indices.get() + this->getLocalNumberOfParticles(),
+ this->local_labels.get());
+ return EXIT_SUCCESS;
+ }
+
+ /** \brief Filter particle set versus small list of particle indices.
+ *
+ * Using this method, we can select particles from a `src` particle set into the current object.
+ * The code will work reasonably as long as a small list of "indices of interest" is used.
+ *
+ * \warning The indices of interest must be existing indices, otherwise the object will be broken.
+ * In principle we could have a general algorithm that could handle anything, but it would be a messy code,
+ * which would need multiple passes of the data to figure out the new indices.
+ * It's easier to just clean up the data before feeding it to this method.
+ *
+ * \warning `indices_of_interest` must be in increasing order.
+ *
+ */
+ int init_as_subset_of(
+ abstract_particle_set &src,
+ const std::vector<partsize_t> indices_of_interest,
+ bool use_set_complement)
+ {
+ TIMEZONE("particle_set::init_as_subset_of version 2");
+ assert(indices_of_interest.size() > 0);
+ assert(partsize_t(indices_of_interest.size()) < src.getTotalNumberOfParticles());
+ // ensure that indices of interest are sorted
+ particle_rnumber *tmp_local_state = new particle_rnumber[state_size * src.getLocalNumberOfParticles()];
+ partsize_t *tmp_local_indices = new partsize_t[src.getLocalNumberOfParticles()];
+ partsize_t *tmp_local_labels = new partsize_t[src.getLocalNumberOfParticles()];
+ this->local_number_of_particles = 0;
+
+ // set new number of particles
+ if (use_set_complement)
+ this->total_number_of_particles = src.getTotalNumberOfParticles() - indices_of_interest.size();
+ else
+ this->total_number_of_particles = indices_of_interest.size();
+
+ // select interesting particles
+ for (partsize_t ii = 0; ii < partsize_t(src.getLocalNumberOfParticles()); ii++)
+ {
+ partsize_t src_label = src.getParticleLabels()[ii];
+ partsize_t src_index = src.getParticleIndices()[ii];
+
+ bool index_is_interesting = false;
+ // find out if index is interesting
+ partsize_t iii = 0;
+ for (; iii < partsize_t(indices_of_interest.size()); iii++)
+ {
+ index_is_interesting = (indices_of_interest[iii] == src_index);
+ if (indices_of_interest[iii] >= src_index)
+ break;
+ }
+ bool copy_data = false;
+
+ if (use_set_complement && (!index_is_interesting)) // we are effectively deleting particles present in "indices_of_interest"
+ copy_data = true;
+ if ((!use_set_complement) && index_is_interesting) // we are only keeping particles present in "indices_of_interest"
+ copy_data = true;
+ if (copy_data)
+ {
+ // set new index
+ tmp_local_indices[this->local_number_of_particles] = src_index;
+ if (use_set_complement)
+ tmp_local_indices[this->local_number_of_particles] -= iii;
+ else
+ tmp_local_indices[this->local_number_of_particles] = iii;
+ // copy particle with label src_label
+ tmp_local_labels[this->local_number_of_particles] = src_label;
+ std::copy(src.getParticleState()+state_size*ii,
+ src.getParticleState()+state_size*(ii+1),
+ tmp_local_state + state_size*this->local_number_of_particles);
+ this->local_number_of_particles++;
+ }
+ }
+ //DEBUG_MSG("particle_set::init_as_subset_of --- after particle selection local number of particles is %d\n",
+ // this->local_number_of_particles);
+ //for (partsize_t ii = 0; ii < this->local_number_of_particles; ii++)
+ // DEBUG_MSG(" --- ii = %d, tmp_local_labels[ii] = %d, tmp_local_indices[ii] = %d\n",
+ // ii,
+ // tmp_local_labels[ii],
+ // tmp_local_indices[ii]);
+
+ // now we actually put the data "here"
+ if (this->local_number_of_particles > 0)
+ {
+ this->local_state.reset(new particle_rnumber[state_size*this->local_number_of_particles]);
+ this->local_indices.reset(new partsize_t[this->local_number_of_particles]);
+ this->local_labels.reset(new partsize_t[this->local_number_of_particles]);
+ std::copy(tmp_local_state,
+ tmp_local_state + state_size*this->local_number_of_particles,
+ this->local_state.get());
+ std::copy(tmp_local_indices,
+ tmp_local_indices + this->local_number_of_particles,
+ this->local_indices.get());
+ std::copy(tmp_local_labels,
+ tmp_local_labels + this->local_number_of_particles,
+ this->local_labels.get());
+ }
+
+ particles_utils::partition_extra_z<partsize_t, state_size>(
+ &this->local_state[0],
+ this->local_number_of_particles,
+ partition_interval_size,
+ this->number_particles_per_partition.get(),
+ this->offset_particles_for_partition.get(),
+ [&](const particle_rnumber& z_pos){
+ const int partition_level = this->pInterpolator.pbc_field_layer(z_pos, IDXC_Z);
+ assert(current_partition_interval.first <= partition_level && partition_level < current_partition_interval.second);
+ return partition_level - current_partition_interval.first;
+ },
+ [&](const partsize_t idx1, const partsize_t idx2){
+ std::swap(this->local_indices[idx1], this->local_indices[idx2]);
+ std::swap(this->local_labels[idx1], this->local_labels[idx2]);
+ });
+
+ delete[] tmp_local_state;
+ delete[] tmp_local_indices;
+ delete[] tmp_local_labels;
+
+ // update particle file layout
+ std::vector<hsize_t> tmp_file_layout(1);
+ tmp_file_layout[0] = hsize_t(this->total_number_of_particles);
+
+ int file_layout_result = this->setParticleFileLayout(tmp_file_layout);
+ variable_used_only_in_assert(file_layout_result);
+ assert(file_layout_result == EXIT_SUCCESS);
+ return EXIT_SUCCESS;
+ }
+
+ particle_rnumber getSpatialLowLimitZ()
+ {
+ return this->pInterpolator.getSpatialLowLimitZ();
+ }
+
+ particle_rnumber getSpatialUpLimitZ()
+ {
+ return this->pInterpolator.getSpatialUpLimitZ();
+ }
+
+ p2p_distr_mpi<partsize_t, particle_rnumber> *getP2PComputer()
+ {
+ return &(this->p2pComputer);
+ }
+
+ std::vector<partsize_t> selectLocalParticleFromFewIndices(
+ const std::vector<partsize_t> &inGlobalIndicesToDelete)
+ {
+ std::vector<partsize_t> local_indices;
+ for (partsize_t i=0; i<this->getLocalNumberOfParticles(); i++)
+ for (partsize_t ii=0; ii<partsize_t(inGlobalIndicesToDelete.size()); ii++)
+ {
+ if (this->getParticleIndices()[i] == inGlobalIndicesToDelete[ii])
+ local_indices.push_back(inGlobalIndicesToDelete[ii]);
+ }
+ return local_indices;
+ }
+};
+
+#endif//PARTICLE_SET_HPP
+
diff --git a/bfps/cpp/particles/particles_field_computer.hpp b/cpp/particles/interpolation/particles_field_computer.hpp
similarity index 55%
rename from bfps/cpp/particles/particles_field_computer.hpp
rename to cpp/particles/interpolation/particles_field_computer.hpp
index f68f2fc02b4ee40aa9583385c0bd18195b92b6dc..172ca37a0b3a303d9a25a966b682920f40c5822b 100644
--- a/bfps/cpp/particles/particles_field_computer.hpp
+++ b/cpp/particles/interpolation/particles_field_computer.hpp
@@ -1,11 +1,43 @@
+/******************************************************************************
+* *
+* Copyright 2019 Max Planck Institute for Dynamics and Self-Organization *
+* *
+* This file is part of TurTLE. *
+* *
+* TurTLE is free software: you can redistribute it and/or modify *
+* it under the terms of the GNU General Public License as published *
+* by the Free Software Foundation, either version 3 of the License, *
+* or (at your option) any later version. *
+* *
+* TurTLE is distributed in the hope that it will be useful, *
+* but WITHOUT ANY WARRANTY; without even the implied warranty of *
+* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the *
+* GNU General Public License for more details. *
+* *
+* You should have received a copy of the GNU General Public License *
+* along with TurTLE. If not, see <http://www.gnu.org/licenses/> *
+* *
+* Contact: Cristian.Lalescu@ds.mpg.de *
+* *
+******************************************************************************/
+
+
+
#ifndef PARTICLES_FIELD_COMPUTER_HPP
#define PARTICLES_FIELD_COMPUTER_HPP
#include <array>
#include <utility>
+#include <cmath>
#include "scope_timer.hpp"
-#include "particles_utils.hpp"
+#include "particles/particles_utils.hpp"
+
+
+
+/** \brief Computes interpolation kernel sum operation.
+ *
+ * */
template <class partsize_t,
class real_number,
@@ -29,66 +61,101 @@ public:
particles_field_computer(const std::array<size_t,3>& in_field_grid_dim,
const std::pair<int,int>& in_current_partitions,
const interpolator_class& in_interpolator,
- const std::array<real_number,3>& in_spatial_box_width, const std::array<real_number,3>& in_spatial_box_offset,
+ const std::array<real_number,3>& in_spatial_box_width,
+ const std::array<real_number,3>& in_spatial_box_offset,
const std::array<real_number,3>& in_box_step_width)
- : field_grid_dim({{int(in_field_grid_dim[0]),int(in_field_grid_dim[1]),int(in_field_grid_dim[2])}}), current_partition_interval(in_current_partitions),
+ : field_grid_dim({{int(in_field_grid_dim[0]),
+ int(in_field_grid_dim[1]),
+ int(in_field_grid_dim[2])}}),
+ current_partition_interval(in_current_partitions),
interpolator(in_interpolator),
- spatial_box_width(in_spatial_box_width), spatial_box_offset(in_spatial_box_offset), box_step_width(in_box_step_width){
- deriv[IDX_X] = 0;
- deriv[IDX_Y] = 0;
- deriv[IDX_Z] = 0;
+ spatial_box_width(in_spatial_box_width),
+ spatial_box_offset(in_spatial_box_offset),
+ box_step_width(in_box_step_width){
+ deriv[IDXC_X] = 0;
+ deriv[IDXC_Y] = 0;
+ deriv[IDXC_Z] = 0;
+ DEBUG_MSG("particles_field_computer::particles_field_computer, spatial_box_offset is %g %g %g\n",
+ spatial_box_offset[0], spatial_box_offset[1], spatial_box_offset[2]);
+ DEBUG_MSG("particles_field_computer::particles_field_computer, spatial_box_width is %g %g %g\n",
+ spatial_box_width[0], spatial_box_width[1], spatial_box_width[2]);
+ DEBUG_MSG("particles_field_computer::particles_field_computer, box_step_width is %g %g %g\n",
+ box_step_width[0], box_step_width[1], box_step_width[2]);
+ DEBUG_MSG("particles_field_computer::particles_field_computer, field_grid_dim is %ld %ld %ld\n",
+ field_grid_dim[0], field_grid_dim[1], field_grid_dim[2]);
}
////////////////////////////////////////////////////////////////////////
/// Computation related
////////////////////////////////////////////////////////////////////////
- template <int size_particle_rhs>
- void init_result_array(real_number particles_current_rhs[],
- const partsize_t nb_particles) const {
- // Set values to zero initialy
- std::fill(particles_current_rhs,
- particles_current_rhs+nb_particles*size_particle_rhs,
- 0);
+ const std::array<real_number, 3> getSpatialBoxWidth() const
+ {
+ return this->spatial_box_width;
+ }
+ const std::array<real_number, 3> getSpatialBoxOffset() const
+ {
+ return this->spatial_box_offset;
+ }
+ const std::array<real_number, 3> getBoxStepWidth() const
+ {
+ return this->box_step_width;
+ }
+
+ const real_number getSpatialLowLimitZ() const
+ {
+ return this->current_partition_interval.first*this->box_step_width[IDXC_Z];
+ }
+
+ const real_number getSpatialUpLimitZ() const
+ {
+ return this->current_partition_interval.second*this->box_step_width[IDXC_Z];
}
real_number get_norm_pos_in_cell(const real_number in_pos, const int idx_pos) const {
const real_number shifted_pos = in_pos - spatial_box_offset[idx_pos];
- const real_number nb_box_repeat = floor(shifted_pos/spatial_box_width[idx_pos]);
+ //DEBUG_MSG("particles_field_computer::get_norm_pos_in_cell, shifted_pos = %g\n", shifted_pos);
+ const int nb_box_repeat = int(floor(shifted_pos/spatial_box_width[idx_pos]));
+ //DEBUG_MSG("particles_field_computer::get_norm_pos_in_cell, nb_box_repeat = %d\n", nb_box_repeat);
const real_number pos_in_box = shifted_pos - nb_box_repeat*spatial_box_width[idx_pos];
- const real_number cell_idx = floor(pos_in_box/box_step_width[idx_pos]);
+ //DEBUG_MSG("particles_field_computer::get_norm_pos_in_cell, pos_in_box = %g\n", pos_in_box);
+ const int cell_idx = int(floor(pos_in_box/box_step_width[idx_pos]));
+ //DEBUG_MSG("particles_field_computer::get_norm_pos_in_cell, cell_idx = %d\n", cell_idx);
const real_number pos_in_cell = (pos_in_box - cell_idx*box_step_width[idx_pos]) / box_step_width[idx_pos];
+ //DEBUG_MSG("particles_field_computer::get_norm_pos_in_cell, pos_in_cell = %g\n", pos_in_cell);
assert(0 <= pos_in_cell && pos_in_cell < 1);
return pos_in_cell;
}
- template <class field_class, int size_particle_rhs>
+ template <class field_class, int size_particle_positions, int size_particle_rhs>
void apply_computation(const field_class& field,
const real_number particles_positions[],
real_number particles_current_rhs[],
const partsize_t nb_particles) const {
+ constexpr int nb_components_in_field = field.number_of_components;
+ static_assert(nb_components_in_field <= size_particle_rhs, "Cannot store all the component in the given array");
TIMEZONE("particles_field_computer::apply_computation");
- //DEBUG_MSG("just entered particles_field_computer::apply_computation\n");
+
for(partsize_t idxPart = 0 ; idxPart < nb_particles ; ++idxPart){
- const real_number reltv_x = get_norm_pos_in_cell(particles_positions[idxPart*3+IDX_X], IDX_X);
- const real_number reltv_y = get_norm_pos_in_cell(particles_positions[idxPart*3+IDX_Y], IDX_Y);
- const real_number reltv_z = get_norm_pos_in_cell(particles_positions[idxPart*3+IDX_Z], IDX_Z);
+ const real_number reltv_x = get_norm_pos_in_cell(particles_positions[idxPart*size_particle_positions+IDXC_X], IDXC_X);
+ const real_number reltv_y = get_norm_pos_in_cell(particles_positions[idxPart*size_particle_positions+IDXC_Y], IDXC_Y);
+ const real_number reltv_z = get_norm_pos_in_cell(particles_positions[idxPart*size_particle_positions+IDXC_Z], IDXC_Z);
typename interpolator_class::real_number
bx[interp_neighbours*2+2],
by[interp_neighbours*2+2],
bz[interp_neighbours*2+2];
- interpolator.compute_beta(deriv[IDX_X], reltv_x, bx);
- interpolator.compute_beta(deriv[IDX_Y], reltv_y, by);
- interpolator.compute_beta(deriv[IDX_Z], reltv_z, bz);
+ interpolator.compute_beta(deriv[IDXC_X], reltv_x, bx);
+ interpolator.compute_beta(deriv[IDXC_Y], reltv_y, by);
+ interpolator.compute_beta(deriv[IDXC_Z], reltv_z, bz);
- const int partGridIdx_x = pbc_field_layer(particles_positions[idxPart*3+IDX_X], IDX_X);
- const int partGridIdx_y = pbc_field_layer(particles_positions[idxPart*3+IDX_Y], IDX_Y);
- const int partGridIdx_z = pbc_field_layer(particles_positions[idxPart*3+IDX_Z], IDX_Z);
+ const int partGridIdx_x = pbc_field_layer(particles_positions[idxPart*size_particle_positions+IDXC_X], IDXC_X);
+ const int partGridIdx_y = pbc_field_layer(particles_positions[idxPart*size_particle_positions+IDXC_Y], IDXC_Y);
+ const int partGridIdx_z = pbc_field_layer(particles_positions[idxPart*size_particle_positions+IDXC_Z], IDXC_Z);
- assert(0 <= partGridIdx_x && partGridIdx_x < int(field_grid_dim[IDX_X]));
- assert(0 <= partGridIdx_y && partGridIdx_y < int(field_grid_dim[IDX_Y]));
- assert(0 <= partGridIdx_z && partGridIdx_z < int(field_grid_dim[IDX_Z]));
+ assert(0 <= partGridIdx_x && partGridIdx_x < int(field_grid_dim[IDXC_X]));
+ assert(0 <= partGridIdx_y && partGridIdx_y < int(field_grid_dim[IDXC_Y]));
+ assert(0 <= partGridIdx_z && partGridIdx_z < int(field_grid_dim[IDXC_Z]));
const int interp_limit_mx = partGridIdx_x-interp_neighbours;
const int interp_limit_x = partGridIdx_x+interp_neighbours+1;
@@ -101,8 +168,8 @@ public:
int nb_z_intervals;
if((partGridIdx_z-interp_neighbours) < 0){
- assert(partGridIdx_z+interp_neighbours+1 < int(field_grid_dim[IDX_Z]));
- interp_limit_mz[0] = std::max(current_partition_interval.first, partGridIdx_z-interp_neighbours+int(field_grid_dim[IDX_Z]));
+ assert(partGridIdx_z+interp_neighbours+1 < int(field_grid_dim[IDXC_Z]));
+ interp_limit_mz[0] = std::max(current_partition_interval.first, partGridIdx_z-interp_neighbours+int(field_grid_dim[IDXC_Z]));
interp_limit_z[0] = current_partition_interval.second-1;
interp_limit_mz[1] = std::max(0, current_partition_interval.first);
@@ -110,12 +177,12 @@ public:
nb_z_intervals = 2;
}
- else if(int(field_grid_dim[IDX_Z]) <= (partGridIdx_z+interp_neighbours+1)){
+ else if(int(field_grid_dim[IDXC_Z]) <= (partGridIdx_z+interp_neighbours+1)){
interp_limit_mz[0] = std::max(current_partition_interval.first, partGridIdx_z-interp_neighbours);
- interp_limit_z[0] = std::min(int(field_grid_dim[IDX_Z])-1,current_partition_interval.second-1);
+ interp_limit_z[0] = std::min(int(field_grid_dim[IDXC_Z])-1,current_partition_interval.second-1);
interp_limit_mz[1] = std::max(0, current_partition_interval.first);
- interp_limit_z[1] = std::min(partGridIdx_z+interp_neighbours+1-int(field_grid_dim[IDX_Z]), current_partition_interval.second-1);
+ interp_limit_z[1] = std::min(partGridIdx_z+interp_neighbours+1-int(field_grid_dim[IDXC_Z]), current_partition_interval.second-1);
nb_z_intervals = 2;
}
@@ -127,26 +194,27 @@ public:
for(int idx_inter = 0 ; idx_inter < nb_z_intervals ; ++idx_inter){
for(int idx_z = interp_limit_mz[idx_inter] ; idx_z <= interp_limit_z[idx_inter] ; ++idx_z ){
- const int idx_z_pbc = (idx_z + field_grid_dim[IDX_Z])%field_grid_dim[IDX_Z];
+ const int idx_z_pbc = (idx_z + field_grid_dim[IDXC_Z])%field_grid_dim[IDXC_Z];
assert(current_partition_interval.first <= idx_z_pbc && idx_z_pbc < current_partition_interval.second);
- assert(((idx_z+field_grid_dim[IDX_Z]-interp_limit_mz_bz)%field_grid_dim[IDX_Z]) < interp_neighbours*2+2);
+ assert(((idx_z+field_grid_dim[IDXC_Z]-interp_limit_mz_bz)%field_grid_dim[IDXC_Z]) < interp_neighbours*2+2);
for(int idx_x = interp_limit_mx ; idx_x <= interp_limit_x ; ++idx_x ){
- const int idx_x_pbc = (idx_x + field_grid_dim[IDX_X])%field_grid_dim[IDX_X];
+ const int idx_x_pbc = (idx_x + field_grid_dim[IDXC_X])%field_grid_dim[IDXC_X];
assert(idx_x-interp_limit_mx < interp_neighbours*2+2);
for(int idx_y = interp_limit_my ; idx_y <= interp_limit_y ; ++idx_y ){
- const int idx_y_pbc = (idx_y + field_grid_dim[IDX_Y])%field_grid_dim[IDX_Y];
+ const int idx_y_pbc = (idx_y + field_grid_dim[IDXC_Y])%field_grid_dim[IDXC_Y];
assert(idx_y-interp_limit_my < interp_neighbours*2+2);
- const real_number coef = (bz[((idx_z+field_grid_dim[IDX_Z]-interp_limit_mz_bz)%field_grid_dim[IDX_Z])]
+ const real_number coef = (bz[((idx_z+field_grid_dim[IDXC_Z]-interp_limit_mz_bz)%field_grid_dim[IDXC_Z])]
* by[idx_y-interp_limit_my]
* bx[idx_x-interp_limit_mx]);
const ptrdiff_t tindex = field.get_rindex_from_global(idx_x_pbc, idx_y_pbc, idx_z_pbc);
// getValue does not necessary return real_number
- for(int idx_rhs_val = 0 ; idx_rhs_val < size_particle_rhs ; ++idx_rhs_val){
+ // size_particle_rhs is just for the leading dimension of the array
+ for(int idx_rhs_val = 0 ; idx_rhs_val < nb_components_in_field ; ++idx_rhs_val){
particles_current_rhs[idxPart*size_particle_rhs+idx_rhs_val] += real_number(field.rval(tindex,idx_rhs_val))*coef;
}
}
@@ -154,6 +222,7 @@ public:
}
}
}
+ //DEBUG_MSG("exiting apply_computation\n");
}
template <int size_particle_rhs>
diff --git a/cpp/particles/interpolation/particles_generic_interp.hpp b/cpp/particles/interpolation/particles_generic_interp.hpp
new file mode 100644
index 0000000000000000000000000000000000000000..37ddc5ca94da03084c16164bba6b9b744492f273
--- /dev/null
+++ b/cpp/particles/interpolation/particles_generic_interp.hpp
@@ -0,0 +1,583 @@
+/******************************************************************************
+* *
+* Copyright 2019 Max Planck Institute for Dynamics and Self-Organization *
+* *
+* This file is part of TurTLE. *
+* *
+* TurTLE is free software: you can redistribute it and/or modify *
+* it under the terms of the GNU General Public License as published *
+* by the Free Software Foundation, either version 3 of the License, *
+* or (at your option) any later version. *
+* *
+* TurTLE is distributed in the hope that it will be useful, *
+* but WITHOUT ANY WARRANTY; without even the implied warranty of *
+* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the *
+* GNU General Public License for more details. *
+* *
+* You should have received a copy of the GNU General Public License *
+* along with TurTLE. If not, see <http://www.gnu.org/licenses/> *
+* *
+* Contact: Cristian.Lalescu@ds.mpg.de *
+* *
+******************************************************************************/
+
+
+
+#ifndef PARTICLES_GENERIC_INTERP_HPP
+#define PARTICLES_GENERIC_INTERP_HPP
+
+/** \brief Computes weights for interpolation formulas.
+ *
+ * */
+
+template <class real_number, int interp_neighbours, int smoothness>
+class particles_generic_interp;
+
+#include "Lagrange_polys.hpp"
+#include "spline.hpp"
+
+/*****************************************************************************/
+/* Linear interpolation */
+
+template <>
+class particles_generic_interp<double, 0,0>{
+public:
+ using real_number = double;
+
+ void compute_beta(const int in_derivative, const double in_part_val, double poly_val[]) const {
+ beta_n0_m0(in_derivative, in_part_val, poly_val);
+ }
+};
+/*****************************************************************************/
+
+/*****************************************************************************/
+/* Lagrange interpolation */
+
+template <>
+class particles_generic_interp<double, 1,0>{
+public:
+ using real_number = double;
+
+ void compute_beta(const int in_derivative, const double in_part_val, double poly_val[]) const {
+ beta_Lagrange_n1(in_derivative, in_part_val, poly_val);
+ }
+};
+
+template <>
+class particles_generic_interp<double, 2,0>{
+public:
+ using real_number = double;
+
+ void compute_beta(const int in_derivative, const double in_part_val, double poly_val[]) const {
+ beta_Lagrange_n2(in_derivative, in_part_val, poly_val);
+ }
+};
+
+template <>
+class particles_generic_interp<double, 3,0>{
+public:
+ using real_number = double;
+
+ void compute_beta(const int in_derivative, const double in_part_val, double poly_val[]) const {
+ beta_Lagrange_n3(in_derivative, in_part_val, poly_val);
+ }
+};
+
+template <>
+class particles_generic_interp<double, 4,0>{
+public:
+ using real_number = double;
+
+ void compute_beta(const int in_derivative, const double in_part_val, double poly_val[]) const {
+ beta_Lagrange_n4(in_derivative, in_part_val, poly_val);
+ }
+};
+
+template <>
+class particles_generic_interp<double, 5,0>{
+public:
+ using real_number = double;
+
+ void compute_beta(const int in_derivative, const double in_part_val, double poly_val[]) const {
+ beta_Lagrange_n5(in_derivative, in_part_val, poly_val);
+ }
+};
+
+template <>
+class particles_generic_interp<double, 6,0>{
+public:
+ using real_number = double;
+
+ void compute_beta(const int in_derivative, const double in_part_val, double poly_val[]) const {
+ beta_Lagrange_n6(in_derivative, in_part_val, poly_val);
+ }
+};
+
+template <>
+class particles_generic_interp<double, 7,0>{
+public:
+ using real_number = double;
+
+ void compute_beta(const int in_derivative, const double in_part_val, double poly_val[]) const {
+ beta_Lagrange_n7(in_derivative, in_part_val, poly_val);
+ }
+};
+
+template <>
+class particles_generic_interp<double, 8,0>{
+public:
+ using real_number = double;
+
+ void compute_beta(const int in_derivative, const double in_part_val, double poly_val[]) const {
+ beta_Lagrange_n8(in_derivative, in_part_val, poly_val);
+ }
+};
+
+template <>
+class particles_generic_interp<double, 9,0>{
+public:
+ using real_number = double;
+
+ void compute_beta(const int in_derivative, const double in_part_val, double poly_val[]) const {
+ beta_Lagrange_n9(in_derivative, in_part_val, poly_val);
+ }
+};
+
+template <>
+class particles_generic_interp<double, 10,0>{
+public:
+ using real_number = double;
+
+ void compute_beta(const int in_derivative, const double in_part_val, double poly_val[]) const {
+ beta_Lagrange_n10(in_derivative, in_part_val, poly_val);
+ }
+};
+
+/*****************************************************************************/
+
+/*****************************************************************************/
+/* spline C1 */
+
+template <>
+class particles_generic_interp<double, 1,1>{
+public:
+ using real_number = double;
+
+ void compute_beta(const int in_derivative, const double in_part_val, double poly_val[]) const {
+ beta_n1_m1(in_derivative, in_part_val, poly_val);
+ }
+};
+
+template <>
+class particles_generic_interp<double, 2,1>{
+public:
+ using real_number = double;
+
+ void compute_beta(const int in_derivative, const double in_part_val, double poly_val[]) const {
+ beta_n2_m1(in_derivative, in_part_val, poly_val);
+ }
+};
+
+template <>
+class particles_generic_interp<double, 3,1>{
+public:
+ using real_number = double;
+
+ void compute_beta(const int in_derivative, const double in_part_val, double poly_val[]) const {
+ beta_n3_m1(in_derivative, in_part_val, poly_val);
+ }
+};
+
+template <>
+class particles_generic_interp<double, 4,1>{
+public:
+ using real_number = double;
+
+ void compute_beta(const int in_derivative, const double in_part_val, double poly_val[]) const {
+ beta_n4_m1(in_derivative, in_part_val, poly_val);
+ }
+};
+
+template <>
+class particles_generic_interp<double, 5,1>{
+public:
+ using real_number = double;
+
+ void compute_beta(const int in_derivative, const double in_part_val, double poly_val[]) const {
+ beta_n5_m1(in_derivative, in_part_val, poly_val);
+ }
+};
+
+template <>
+class particles_generic_interp<double, 6,1>{
+public:
+ using real_number = double;
+
+ void compute_beta(const int in_derivative, const double in_part_val, double poly_val[]) const {
+ beta_n6_m1(in_derivative, in_part_val, poly_val);
+ }
+};
+
+template <>
+class particles_generic_interp<double, 7,1>{
+public:
+ using real_number = double;
+
+ void compute_beta(const int in_derivative, const double in_part_val, double poly_val[]) const {
+ beta_n7_m1(in_derivative, in_part_val, poly_val);
+ }
+};
+
+template <>
+class particles_generic_interp<double, 8,1>{
+public:
+ using real_number = double;
+
+ void compute_beta(const int in_derivative, const double in_part_val, double poly_val[]) const {
+ beta_n8_m1(in_derivative, in_part_val, poly_val);
+ }
+};
+
+template <>
+class particles_generic_interp<double, 9,1>{
+public:
+ using real_number = double;
+
+ void compute_beta(const int in_derivative, const double in_part_val, double poly_val[]) const {
+ beta_n9_m1(in_derivative, in_part_val, poly_val);
+ }
+};
+
+template <>
+class particles_generic_interp<double, 10,1>{
+public:
+ using real_number = double;
+
+ void compute_beta(const int in_derivative, const double in_part_val, double poly_val[]) const {
+ beta_n10_m1(in_derivative, in_part_val, poly_val);
+ }
+};
+
+/*****************************************************************************/
+
+/*****************************************************************************/
+/* spline C2 */
+
+template <>
+class particles_generic_interp<double, 1,2>{
+public:
+ using real_number = double;
+
+ void compute_beta(const int in_derivative, const double in_part_val, double poly_val[]) const {
+ beta_n1_m2(in_derivative, in_part_val, poly_val);
+ }
+};
+
+template <>
+class particles_generic_interp<double, 2,2>{
+public:
+ using real_number = double;
+
+ void compute_beta(const int in_derivative, const double in_part_val, double poly_val[]) const {
+ beta_n2_m2(in_derivative, in_part_val, poly_val);
+ }
+};
+
+template <>
+class particles_generic_interp<double, 3,2>{
+public:
+ using real_number = double;
+
+ void compute_beta(const int in_derivative, const double in_part_val, double poly_val[]) const {
+ beta_n3_m2(in_derivative, in_part_val, poly_val);
+ }
+};
+
+template <>
+class particles_generic_interp<double, 4,2>{
+public:
+ using real_number = double;
+
+ void compute_beta(const int in_derivative, const double in_part_val, double poly_val[]) const {
+ beta_n4_m2(in_derivative, in_part_val, poly_val);
+ }
+};
+
+template <>
+class particles_generic_interp<double, 5,2>{
+public:
+ using real_number = double;
+
+ void compute_beta(const int in_derivative, const double in_part_val, double poly_val[]) const {
+ beta_n5_m2(in_derivative, in_part_val, poly_val);
+ }
+};
+
+template <>
+class particles_generic_interp<double, 6,2>{
+public:
+ using real_number = double;
+
+ void compute_beta(const int in_derivative, const double in_part_val, double poly_val[]) const {
+ beta_n6_m2(in_derivative, in_part_val, poly_val);
+ }
+};
+
+template <>
+class particles_generic_interp<double, 7,2>{
+public:
+ using real_number = double;
+
+ void compute_beta(const int in_derivative, const double in_part_val, double poly_val[]) const {
+ beta_n7_m2(in_derivative, in_part_val, poly_val);
+ }
+};
+
+template <>
+class particles_generic_interp<double, 8,2>{
+public:
+ using real_number = double;
+
+ void compute_beta(const int in_derivative, const double in_part_val, double poly_val[]) const {
+ beta_n8_m2(in_derivative, in_part_val, poly_val);
+ }
+};
+
+template <>
+class particles_generic_interp<double, 9,2>{
+public:
+ using real_number = double;
+
+ void compute_beta(const int in_derivative, const double in_part_val, double poly_val[]) const {
+ beta_n9_m2(in_derivative, in_part_val, poly_val);
+ }
+};
+
+template <>
+class particles_generic_interp<double, 10,2>{
+public:
+ using real_number = double;
+
+ void compute_beta(const int in_derivative, const double in_part_val, double poly_val[]) const {
+ beta_n10_m2(in_derivative, in_part_val, poly_val);
+ }
+};
+
+/*****************************************************************************/
+
+/*****************************************************************************/
+/* spline C3 */
+
+template <>
+class particles_generic_interp<double, 1,3>{
+ // no such thing as C3 interpolant with just a 4-point kernel, so we just use C2
+ // this specialisation must exist, otherwise the template_double_for_if will fail
+public:
+ using real_number = double;
+
+ void compute_beta(const int in_derivative, const double in_part_val, double poly_val[]) const {
+ beta_n1_m2(in_derivative, in_part_val, poly_val);
+ }
+};
+
+template <>
+class particles_generic_interp<double, 2,3>{
+public:
+ using real_number = double;
+
+ void compute_beta(const int in_derivative, const double in_part_val, double poly_val[]) const {
+ beta_n2_m3(in_derivative, in_part_val, poly_val);
+ }
+};
+
+template <>
+class particles_generic_interp<double, 3,3>{
+public:
+ using real_number = double;
+
+ void compute_beta(const int in_derivative, const double in_part_val, double poly_val[]) const {
+ beta_n3_m3(in_derivative, in_part_val, poly_val);
+ }
+};
+
+template <>
+class particles_generic_interp<double, 4,3>{
+public:
+ using real_number = double;
+
+ void compute_beta(const int in_derivative, const double in_part_val, double poly_val[]) const {
+ beta_n4_m3(in_derivative, in_part_val, poly_val);
+ }
+};
+
+template <>
+class particles_generic_interp<double, 5,3>{
+public:
+ using real_number = double;
+
+ void compute_beta(const int in_derivative, const double in_part_val, double poly_val[]) const {
+ beta_n5_m3(in_derivative, in_part_val, poly_val);
+ }
+};
+
+template <>
+class particles_generic_interp<double, 6,3>{
+public:
+ using real_number = double;
+
+ void compute_beta(const int in_derivative, const double in_part_val, double poly_val[]) const {
+ beta_n6_m3(in_derivative, in_part_val, poly_val);
+ }
+};
+
+template <>
+class particles_generic_interp<double, 7,3>{
+public:
+ using real_number = double;
+
+ void compute_beta(const int in_derivative, const double in_part_val, double poly_val[]) const {
+ beta_n7_m3(in_derivative, in_part_val, poly_val);
+ }
+};
+
+template <>
+class particles_generic_interp<double, 8,3>{
+public:
+ using real_number = double;
+
+ void compute_beta(const int in_derivative, const double in_part_val, double poly_val[]) const {
+ beta_n8_m3(in_derivative, in_part_val, poly_val);
+ }
+};
+
+template <>
+class particles_generic_interp<double, 9,3>{
+public:
+ using real_number = double;
+
+ void compute_beta(const int in_derivative, const double in_part_val, double poly_val[]) const {
+ beta_n9_m3(in_derivative, in_part_val, poly_val);
+ }
+};
+
+template <>
+class particles_generic_interp<double, 10,3>{
+public:
+ using real_number = double;
+
+ void compute_beta(const int in_derivative, const double in_part_val, double poly_val[]) const {
+ beta_n10_m3(in_derivative, in_part_val, poly_val);
+ }
+};
+
+/*****************************************************************************/
+
+/*****************************************************************************/
+/* spline C4 */
+
+template <>
+class particles_generic_interp<double, 1,4>{
+ // no such thing as C4 interpolant with just a 4-point kernel, so we just use C2
+ // this specialisation must exist, otherwise the template_double_for_if will fail
+public:
+ using real_number = double;
+
+ void compute_beta(const int in_derivative, const double in_part_val, double poly_val[]) const {
+ beta_n1_m2(in_derivative, in_part_val, poly_val);
+ }
+};
+
+template <>
+class particles_generic_interp<double, 2,4>{
+public:
+ using real_number = double;
+
+ void compute_beta(const int in_derivative, const double in_part_val, double poly_val[]) const {
+ beta_n2_m4(in_derivative, in_part_val, poly_val);
+ }
+};
+
+template <>
+class particles_generic_interp<double, 3,4>{
+public:
+ using real_number = double;
+
+ void compute_beta(const int in_derivative, const double in_part_val, double poly_val[]) const {
+ beta_n3_m4(in_derivative, in_part_val, poly_val);
+ }
+};
+
+template <>
+class particles_generic_interp<double, 4,4>{
+public:
+ using real_number = double;
+
+ void compute_beta(const int in_derivative, const double in_part_val, double poly_val[]) const {
+ beta_n4_m4(in_derivative, in_part_val, poly_val);
+ }
+};
+
+template <>
+class particles_generic_interp<double, 5,4>{
+public:
+ using real_number = double;
+
+ void compute_beta(const int in_derivative, const double in_part_val, double poly_val[]) const {
+ beta_n5_m4(in_derivative, in_part_val, poly_val);
+ }
+};
+
+template <>
+class particles_generic_interp<double, 6,4>{
+public:
+ using real_number = double;
+
+ void compute_beta(const int in_derivative, const double in_part_val, double poly_val[]) const {
+ beta_n6_m4(in_derivative, in_part_val, poly_val);
+ }
+};
+
+template <>
+class particles_generic_interp<double, 7,4>{
+public:
+ using real_number = double;
+
+ void compute_beta(const int in_derivative, const double in_part_val, double poly_val[]) const {
+ beta_n7_m4(in_derivative, in_part_val, poly_val);
+ }
+};
+
+template <>
+class particles_generic_interp<double, 8,4>{
+public:
+ using real_number = double;
+
+ void compute_beta(const int in_derivative, const double in_part_val, double poly_val[]) const {
+ beta_n8_m4(in_derivative, in_part_val, poly_val);
+ }
+};
+
+template <>
+class particles_generic_interp<double, 9,4>{
+public:
+ using real_number = double;
+
+ void compute_beta(const int in_derivative, const double in_part_val, double poly_val[]) const {
+ beta_n9_m4(in_derivative, in_part_val, poly_val);
+ }
+};
+
+template <>
+class particles_generic_interp<double, 10,4>{
+public:
+ using real_number = double;
+
+ void compute_beta(const int in_derivative, const double in_part_val, double poly_val[]) const {
+ beta_n10_m4(in_derivative, in_part_val, poly_val);
+ }
+};
+
+/*****************************************************************************/
+
+#endif//PARTICLES_INTERP_SPLINE_HPP
+
diff --git a/cpp/particles/interpolation/spline.hpp b/cpp/particles/interpolation/spline.hpp
new file mode 100644
index 0000000000000000000000000000000000000000..f7101ab3537ae9fd31c6391c2dcceb58ea6cb3c4
--- /dev/null
+++ b/cpp/particles/interpolation/spline.hpp
@@ -0,0 +1,41 @@
+/******************************************************************************
+* *
+* Copyright 2017 Max Planck Institute for Dynamics and Self-Organization *
+* *
+* This file is part of TurTLE. *
+* *
+* TurTLE is free software: you can redistribute it and/or modify *
+* it under the terms of the GNU General Public License as published *
+* by the Free Software Foundation, either version 3 of the License, *
+* or (at your option) any later version. *
+* *
+* TurTLE is distributed in the hope that it will be useful, *
+* but WITHOUT ANY WARRANTY; without even the implied warranty of *
+* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the *
+* GNU General Public License for more details. *
+* *
+* You should have received a copy of the GNU General Public License *
+* along with TurTLE. If not, see <http://www.gnu.org/licenses/> *
+* *
+* Contact: Cristian.Lalescu@ds.mpg.de *
+* *
+******************************************************************************/
+
+
+
+#ifndef SPLINE_HPP
+#define SPLINE_HPP
+
+#include "particles/interpolation/spline_n0.hpp"
+#include "particles/interpolation/spline_n1.hpp"
+#include "particles/interpolation/spline_n2.hpp"
+#include "particles/interpolation/spline_n3.hpp"
+#include "particles/interpolation/spline_n4.hpp"
+#include "particles/interpolation/spline_n5.hpp"
+#include "particles/interpolation/spline_n6.hpp"
+#include "particles/interpolation/spline_n7.hpp"
+#include "particles/interpolation/spline_n8.hpp"
+#include "particles/interpolation/spline_n9.hpp"
+#include "particles/interpolation/spline_n10.hpp"
+
+#endif
diff --git a/cpp/particles/interpolation/spline_n0.cpp b/cpp/particles/interpolation/spline_n0.cpp
new file mode 100644
index 0000000000000000000000000000000000000000..8009fb6962174d5030953561633a7ad6e7183817
--- /dev/null
+++ b/cpp/particles/interpolation/spline_n0.cpp
@@ -0,0 +1,47 @@
+/**********************************************************************
+* *
+* Copyright 2022 Max Planck Computing and Data Facility *
+* *
+* This file is part of TurTLE. *
+* *
+* TurTLE is free software: you can redistribute it and/or modify *
+* it under the terms of the GNU General Public License as published *
+* by the Free Software Foundation, either version 3 of the License, *
+* or (at your option) any later version. *
+* *
+* TurTLE is distributed in the hope that it will be useful, *
+* but WITHOUT ANY WARRANTY; without even the implied warranty of *
+* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the *
+* GNU General Public License for more details. *
+* *
+* You should have received a copy of the GNU General Public License *
+* along with TurTLE. If not, see <http://www.gnu.org/licenses/> *
+* *
+* Contact: Cristian.Lalescu@mpcdf.mpg.de *
+* *
+**********************************************************************/
+
+
+
+#include "spline_n0.hpp"
+#include <cmath>
+
+void beta_n0_m0(int deriv, double x, double *poly_val)
+{
+ switch(deriv)
+ {
+ case 0:
+ poly_val[0] = -x + 1;
+ poly_val[1] = x;
+ break;
+ case 1:
+ poly_val[0] = -1;
+ poly_val[1] = 1;
+ break;
+ case 2:
+ poly_val[0] = 0;
+ poly_val[1] = 0;
+ break;
+ }
+}
+
diff --git a/cpp/particles/interpolation/spline_n0.hpp b/cpp/particles/interpolation/spline_n0.hpp
new file mode 100644
index 0000000000000000000000000000000000000000..e1fe6378009f0d8de5cc863ee721ac9f8bfac298
--- /dev/null
+++ b/cpp/particles/interpolation/spline_n0.hpp
@@ -0,0 +1,33 @@
+/**********************************************************************
+* *
+* Copyright 2022 Max Planck Computing and Data Facility *
+* *
+* This file is part of TurTLE. *
+* *
+* TurTLE is free software: you can redistribute it and/or modify *
+* it under the terms of the GNU General Public License as published *
+* by the Free Software Foundation, either version 3 of the License, *
+* or (at your option) any later version. *
+* *
+* TurTLE is distributed in the hope that it will be useful, *
+* but WITHOUT ANY WARRANTY; without even the implied warranty of *
+* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the *
+* GNU General Public License for more details. *
+* *
+* You should have received a copy of the GNU General Public License *
+* along with TurTLE. If not, see <http://www.gnu.org/licenses/> *
+* *
+* Contact: Cristian.Lalescu@mpcdf.mpg.de *
+* *
+**********************************************************************/
+
+
+
+#ifndef SPLINE_N0
+
+#define SPLINE_N0
+
+void beta_n0_m0(const int deriv, const double x, double *__restrict__ poly_val);
+
+#endif//SPLINE_N0
+
diff --git a/bfps/cpp/spline_n1.cpp b/cpp/particles/interpolation/spline_n1.cpp
similarity index 93%
rename from bfps/cpp/spline_n1.cpp
rename to cpp/particles/interpolation/spline_n1.cpp
index 62b2928001438876cce92ff9523ee0a9d57ad469..f311f9e6b14761f20ec859b14e3bfe6de1b95ea6 100644
--- a/bfps/cpp/spline_n1.cpp
+++ b/cpp/particles/interpolation/spline_n1.cpp
@@ -3,20 +3,20 @@
* Copyright 2015 Max Planck Institute *
* for Dynamics and Self-Organization *
* *
-* This file is part of bfps. *
+* This file is part of TurTLE. *
* *
-* bfps is free software: you can redistribute it and/or modify *
+* TurTLE is free software: you can redistribute it and/or modify *
* it under the terms of the GNU General Public License as published *
* by the Free Software Foundation, either version 3 of the License, *
* or (at your option) any later version. *
* *
-* bfps is distributed in the hope that it will be useful, *
+* TurTLE is distributed in the hope that it will be useful, *
* but WITHOUT ANY WARRANTY; without even the implied warranty of *
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the *
* GNU General Public License for more details. *
* *
* You should have received a copy of the GNU General Public License *
-* along with bfps. If not, see <http://www.gnu.org/licenses/> *
+* along with TurTLE. If not, see <http://www.gnu.org/licenses/> *
* *
* Contact: Cristian.Lalescu@ds.mpg.de *
* *
diff --git a/bfps/cpp/spline_n1.hpp b/cpp/particles/interpolation/spline_n1.hpp
similarity index 86%
rename from bfps/cpp/spline_n1.hpp
rename to cpp/particles/interpolation/spline_n1.hpp
index 707e9fbcd3a1772c07ba59e6494260f0d8740a7d..f090e07a8a0ee074ea394171de3de9142aca87d8 100644
--- a/bfps/cpp/spline_n1.hpp
+++ b/cpp/particles/interpolation/spline_n1.hpp
@@ -3,20 +3,20 @@
* Copyright 2015 Max Planck Institute *
* for Dynamics and Self-Organization *
* *
-* This file is part of bfps. *
+* This file is part of TurTLE. *
* *
-* bfps is free software: you can redistribute it and/or modify *
+* TurTLE is free software: you can redistribute it and/or modify *
* it under the terms of the GNU General Public License as published *
* by the Free Software Foundation, either version 3 of the License, *
* or (at your option) any later version. *
* *
-* bfps is distributed in the hope that it will be useful, *
+* TurTLE is distributed in the hope that it will be useful, *
* but WITHOUT ANY WARRANTY; without even the implied warranty of *
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the *
* GNU General Public License for more details. *
* *
* You should have received a copy of the GNU General Public License *
-* along with bfps. If not, see <http://www.gnu.org/licenses/> *
+* along with TurTLE. If not, see <http://www.gnu.org/licenses/> *
* *
* Contact: Cristian.Lalescu@ds.mpg.de *
* *
diff --git a/bfps/cpp/spline_n10.cpp b/cpp/particles/interpolation/spline_n10.cpp
similarity index 99%
rename from bfps/cpp/spline_n10.cpp
rename to cpp/particles/interpolation/spline_n10.cpp
index 1712663933addeecba17955f8632d1a20e8c32c6..b40be0f557ceb972faaf57cb656e596bcc5973de 100644
--- a/bfps/cpp/spline_n10.cpp
+++ b/cpp/particles/interpolation/spline_n10.cpp
@@ -3,20 +3,20 @@
* Copyright 2015 Max Planck Institute *
* for Dynamics and Self-Organization *
* *
-* This file is part of bfps. *
+* This file is part of TurTLE. *
* *
-* bfps is free software: you can redistribute it and/or modify *
+* TurTLE is free software: you can redistribute it and/or modify *
* it under the terms of the GNU General Public License as published *
* by the Free Software Foundation, either version 3 of the License, *
* or (at your option) any later version. *
* *
-* bfps is distributed in the hope that it will be useful, *
+* TurTLE is distributed in the hope that it will be useful, *
* but WITHOUT ANY WARRANTY; without even the implied warranty of *
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the *
* GNU General Public License for more details. *
* *
* You should have received a copy of the GNU General Public License *
-* along with bfps. If not, see <http://www.gnu.org/licenses/> *
+* along with TurTLE. If not, see <http://www.gnu.org/licenses/> *
* *
* Contact: Cristian.Lalescu@ds.mpg.de *
* *
diff --git a/bfps/cpp/spline_n10.hpp b/cpp/particles/interpolation/spline_n10.hpp
similarity index 87%
rename from bfps/cpp/spline_n10.hpp
rename to cpp/particles/interpolation/spline_n10.hpp
index 4853f1975e8759fbcaf4c499df02a5b3bc934a91..c0fa6d0a3f8d64caa4ed734726811f3dd8213baa 100644
--- a/bfps/cpp/spline_n10.hpp
+++ b/cpp/particles/interpolation/spline_n10.hpp
@@ -3,20 +3,20 @@
* Copyright 2015 Max Planck Institute *
* for Dynamics and Self-Organization *
* *
-* This file is part of bfps. *
+* This file is part of TurTLE. *
* *
-* bfps is free software: you can redistribute it and/or modify *
+* TurTLE is free software: you can redistribute it and/or modify *
* it under the terms of the GNU General Public License as published *
* by the Free Software Foundation, either version 3 of the License, *
* or (at your option) any later version. *
* *
-* bfps is distributed in the hope that it will be useful, *
+* TurTLE is distributed in the hope that it will be useful, *
* but WITHOUT ANY WARRANTY; without even the implied warranty of *
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the *
* GNU General Public License for more details. *
* *
* You should have received a copy of the GNU General Public License *
-* along with bfps. If not, see <http://www.gnu.org/licenses/> *
+* along with TurTLE. If not, see <http://www.gnu.org/licenses/> *
* *
* Contact: Cristian.Lalescu@ds.mpg.de *
* *
diff --git a/bfps/cpp/spline_n2.cpp b/cpp/particles/interpolation/spline_n2.cpp
similarity index 97%
rename from bfps/cpp/spline_n2.cpp
rename to cpp/particles/interpolation/spline_n2.cpp
index b2fc17f4c6be63883bbc4346d43bb59f238d1bad..be015b15b45e9e1944d52fd8405ecd21cae93025 100644
--- a/bfps/cpp/spline_n2.cpp
+++ b/cpp/particles/interpolation/spline_n2.cpp
@@ -3,20 +3,20 @@
* Copyright 2015 Max Planck Institute *
* for Dynamics and Self-Organization *
* *
-* This file is part of bfps. *
+* This file is part of TurTLE. *
* *
-* bfps is free software: you can redistribute it and/or modify *
+* TurTLE is free software: you can redistribute it and/or modify *
* it under the terms of the GNU General Public License as published *
* by the Free Software Foundation, either version 3 of the License, *
* or (at your option) any later version. *
* *
-* bfps is distributed in the hope that it will be useful, *
+* TurTLE is distributed in the hope that it will be useful, *
* but WITHOUT ANY WARRANTY; without even the implied warranty of *
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the *
* GNU General Public License for more details. *
* *
* You should have received a copy of the GNU General Public License *
-* along with bfps. If not, see <http://www.gnu.org/licenses/> *
+* along with TurTLE. If not, see <http://www.gnu.org/licenses/> *
* *
* Contact: Cristian.Lalescu@ds.mpg.de *
* *
diff --git a/bfps/cpp/spline_n2.hpp b/cpp/particles/interpolation/spline_n2.hpp
similarity index 87%
rename from bfps/cpp/spline_n2.hpp
rename to cpp/particles/interpolation/spline_n2.hpp
index b29e7be13f663d9bdbca310b91b7e3a1db92a205..033d4a42429ddb947054512326c88bab97a9dfa4 100644
--- a/bfps/cpp/spline_n2.hpp
+++ b/cpp/particles/interpolation/spline_n2.hpp
@@ -3,20 +3,20 @@
* Copyright 2015 Max Planck Institute *
* for Dynamics and Self-Organization *
* *
-* This file is part of bfps. *
+* This file is part of TurTLE. *
* *
-* bfps is free software: you can redistribute it and/or modify *
+* TurTLE is free software: you can redistribute it and/or modify *
* it under the terms of the GNU General Public License as published *
* by the Free Software Foundation, either version 3 of the License, *
* or (at your option) any later version. *
* *
-* bfps is distributed in the hope that it will be useful, *
+* TurTLE is distributed in the hope that it will be useful, *
* but WITHOUT ANY WARRANTY; without even the implied warranty of *
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the *
* GNU General Public License for more details. *
* *
* You should have received a copy of the GNU General Public License *
-* along with bfps. If not, see <http://www.gnu.org/licenses/> *
+* along with TurTLE. If not, see <http://www.gnu.org/licenses/> *
* *
* Contact: Cristian.Lalescu@ds.mpg.de *
* *
diff --git a/bfps/cpp/spline_n3.cpp b/cpp/particles/interpolation/spline_n3.cpp
similarity index 98%
rename from bfps/cpp/spline_n3.cpp
rename to cpp/particles/interpolation/spline_n3.cpp
index ef83a28e31da17d204693b217affc493dd5e632a..b1ac72d768b537365392ee5a00e2f07606b6d48e 100644
--- a/bfps/cpp/spline_n3.cpp
+++ b/cpp/particles/interpolation/spline_n3.cpp
@@ -3,20 +3,20 @@
* Copyright 2015 Max Planck Institute *
* for Dynamics and Self-Organization *
* *
-* This file is part of bfps. *
+* This file is part of TurTLE. *
* *
-* bfps is free software: you can redistribute it and/or modify *
+* TurTLE is free software: you can redistribute it and/or modify *
* it under the terms of the GNU General Public License as published *
* by the Free Software Foundation, either version 3 of the License, *
* or (at your option) any later version. *
* *
-* bfps is distributed in the hope that it will be useful, *
+* TurTLE is distributed in the hope that it will be useful, *
* but WITHOUT ANY WARRANTY; without even the implied warranty of *
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the *
* GNU General Public License for more details. *
* *
* You should have received a copy of the GNU General Public License *
-* along with bfps. If not, see <http://www.gnu.org/licenses/> *
+* along with TurTLE. If not, see <http://www.gnu.org/licenses/> *
* *
* Contact: Cristian.Lalescu@ds.mpg.de *
* *
diff --git a/bfps/cpp/spline_n3.hpp b/cpp/particles/interpolation/spline_n3.hpp
similarity index 88%
rename from bfps/cpp/spline_n3.hpp
rename to cpp/particles/interpolation/spline_n3.hpp
index 159f8b970947b507205f40fcfb5140ca683770ce..57206746240b948eeaf7d32db8c04bbbbf6f4937 100644
--- a/bfps/cpp/spline_n3.hpp
+++ b/cpp/particles/interpolation/spline_n3.hpp
@@ -3,20 +3,20 @@
* Copyright 2015 Max Planck Institute *
* for Dynamics and Self-Organization *
* *
-* This file is part of bfps. *
+* This file is part of TurTLE. *
* *
-* bfps is free software: you can redistribute it and/or modify *
+* TurTLE is free software: you can redistribute it and/or modify *
* it under the terms of the GNU General Public License as published *
* by the Free Software Foundation, either version 3 of the License, *
* or (at your option) any later version. *
* *
-* bfps is distributed in the hope that it will be useful, *
+* TurTLE is distributed in the hope that it will be useful, *
* but WITHOUT ANY WARRANTY; without even the implied warranty of *
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the *
* GNU General Public License for more details. *
* *
* You should have received a copy of the GNU General Public License *
-* along with bfps. If not, see <http://www.gnu.org/licenses/> *
+* along with TurTLE. If not, see <http://www.gnu.org/licenses/> *
* *
* Contact: Cristian.Lalescu@ds.mpg.de *
* *
diff --git a/bfps/cpp/spline_n4.cpp b/cpp/particles/interpolation/spline_n4.cpp
similarity index 99%
rename from bfps/cpp/spline_n4.cpp
rename to cpp/particles/interpolation/spline_n4.cpp
index 1e42c7838d5bbc0c3fe9a78ebc45993112d72568..ccb73c4534333c76c9e6e29ed55a380d0a99737a 100644
--- a/bfps/cpp/spline_n4.cpp
+++ b/cpp/particles/interpolation/spline_n4.cpp
@@ -3,20 +3,20 @@
* Copyright 2015 Max Planck Institute *
* for Dynamics and Self-Organization *
* *
-* This file is part of bfps. *
+* This file is part of TurTLE. *
* *
-* bfps is free software: you can redistribute it and/or modify *
+* TurTLE is free software: you can redistribute it and/or modify *
* it under the terms of the GNU General Public License as published *
* by the Free Software Foundation, either version 3 of the License, *
* or (at your option) any later version. *
* *
-* bfps is distributed in the hope that it will be useful, *
+* TurTLE is distributed in the hope that it will be useful, *
* but WITHOUT ANY WARRANTY; without even the implied warranty of *
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the *
* GNU General Public License for more details. *
* *
* You should have received a copy of the GNU General Public License *
-* along with bfps. If not, see <http://www.gnu.org/licenses/> *
+* along with TurTLE. If not, see <http://www.gnu.org/licenses/> *
* *
* Contact: Cristian.Lalescu@ds.mpg.de *
* *
diff --git a/bfps/cpp/spline_n4.hpp b/cpp/particles/interpolation/spline_n4.hpp
similarity index 88%
rename from bfps/cpp/spline_n4.hpp
rename to cpp/particles/interpolation/spline_n4.hpp
index 4b80d16ea110fe1738f1eecca5f2f9cbe6e43e84..13417a3d4ae49d99608c47a45c9ae49bc4c53dac 100644
--- a/bfps/cpp/spline_n4.hpp
+++ b/cpp/particles/interpolation/spline_n4.hpp
@@ -3,20 +3,20 @@
* Copyright 2015 Max Planck Institute *
* for Dynamics and Self-Organization *
* *
-* This file is part of bfps. *
+* This file is part of TurTLE. *
* *
-* bfps is free software: you can redistribute it and/or modify *
+* TurTLE is free software: you can redistribute it and/or modify *
* it under the terms of the GNU General Public License as published *
* by the Free Software Foundation, either version 3 of the License, *
* or (at your option) any later version. *
* *
-* bfps is distributed in the hope that it will be useful, *
+* TurTLE is distributed in the hope that it will be useful, *
* but WITHOUT ANY WARRANTY; without even the implied warranty of *
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the *
* GNU General Public License for more details. *
* *
* You should have received a copy of the GNU General Public License *
-* along with bfps. If not, see <http://www.gnu.org/licenses/> *
+* along with TurTLE. If not, see <http://www.gnu.org/licenses/> *
* *
* Contact: Cristian.Lalescu@ds.mpg.de *
* *
diff --git a/bfps/cpp/spline_n5.cpp b/cpp/particles/interpolation/spline_n5.cpp
similarity index 99%
rename from bfps/cpp/spline_n5.cpp
rename to cpp/particles/interpolation/spline_n5.cpp
index 90e3a6e46424eca376ed85abf4f67a559e3cb910..dee1a574427a606777ba6d55ceb9c238cfcac9a1 100644
--- a/bfps/cpp/spline_n5.cpp
+++ b/cpp/particles/interpolation/spline_n5.cpp
@@ -3,20 +3,20 @@
* Copyright 2015 Max Planck Institute *
* for Dynamics and Self-Organization *
* *
-* This file is part of bfps. *
+* This file is part of TurTLE. *
* *
-* bfps is free software: you can redistribute it and/or modify *
+* TurTLE is free software: you can redistribute it and/or modify *
* it under the terms of the GNU General Public License as published *
* by the Free Software Foundation, either version 3 of the License, *
* or (at your option) any later version. *
* *
-* bfps is distributed in the hope that it will be useful, *
+* TurTLE is distributed in the hope that it will be useful, *
* but WITHOUT ANY WARRANTY; without even the implied warranty of *
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the *
* GNU General Public License for more details. *
* *
* You should have received a copy of the GNU General Public License *
-* along with bfps. If not, see <http://www.gnu.org/licenses/> *
+* along with TurTLE. If not, see <http://www.gnu.org/licenses/> *
* *
* Contact: Cristian.Lalescu@ds.mpg.de *
* *
diff --git a/bfps/cpp/spline_n5.hpp b/cpp/particles/interpolation/spline_n5.hpp
similarity index 89%
rename from bfps/cpp/spline_n5.hpp
rename to cpp/particles/interpolation/spline_n5.hpp
index fccd512076dc571fdded3fe3ccf44f7e4132ec48..fa4371acc2c9f41da1200f7d5096e4d84413f918 100644
--- a/bfps/cpp/spline_n5.hpp
+++ b/cpp/particles/interpolation/spline_n5.hpp
@@ -3,20 +3,20 @@
* Copyright 2015 Max Planck Institute *
* for Dynamics and Self-Organization *
* *
-* This file is part of bfps. *
+* This file is part of TurTLE. *
* *
-* bfps is free software: you can redistribute it and/or modify *
+* TurTLE is free software: you can redistribute it and/or modify *
* it under the terms of the GNU General Public License as published *
* by the Free Software Foundation, either version 3 of the License, *
* or (at your option) any later version. *
* *
-* bfps is distributed in the hope that it will be useful, *
+* TurTLE is distributed in the hope that it will be useful, *
* but WITHOUT ANY WARRANTY; without even the implied warranty of *
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the *
* GNU General Public License for more details. *
* *
* You should have received a copy of the GNU General Public License *
-* along with bfps. If not, see <http://www.gnu.org/licenses/> *
+* along with TurTLE. If not, see <http://www.gnu.org/licenses/> *
* *
* Contact: Cristian.Lalescu@ds.mpg.de *
* *
diff --git a/bfps/cpp/spline_n6.cpp b/cpp/particles/interpolation/spline_n6.cpp
similarity index 99%
rename from bfps/cpp/spline_n6.cpp
rename to cpp/particles/interpolation/spline_n6.cpp
index 4fe0101d171a53519ec4ae05cda2cce8f7da9b0b..3e9285470c60970f6404db0fb3fd40e517f2b374 100644
--- a/bfps/cpp/spline_n6.cpp
+++ b/cpp/particles/interpolation/spline_n6.cpp
@@ -3,20 +3,20 @@
* Copyright 2015 Max Planck Institute *
* for Dynamics and Self-Organization *
* *
-* This file is part of bfps. *
+* This file is part of TurTLE. *
* *
-* bfps is free software: you can redistribute it and/or modify *
+* TurTLE is free software: you can redistribute it and/or modify *
* it under the terms of the GNU General Public License as published *
* by the Free Software Foundation, either version 3 of the License, *
* or (at your option) any later version. *
* *
-* bfps is distributed in the hope that it will be useful, *
+* TurTLE is distributed in the hope that it will be useful, *
* but WITHOUT ANY WARRANTY; without even the implied warranty of *
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the *
* GNU General Public License for more details. *
* *
* You should have received a copy of the GNU General Public License *
-* along with bfps. If not, see <http://www.gnu.org/licenses/> *
+* along with TurTLE. If not, see <http://www.gnu.org/licenses/> *
* *
* Contact: Cristian.Lalescu@ds.mpg.de *
* *
diff --git a/bfps/cpp/spline_n6.hpp b/cpp/particles/interpolation/spline_n6.hpp
similarity index 90%
rename from bfps/cpp/spline_n6.hpp
rename to cpp/particles/interpolation/spline_n6.hpp
index 8eb99092eafd0f1f7c783a023eae49d714479ce1..2a0a0194892df1982089d0ab94fa67d37d1a647e 100644
--- a/bfps/cpp/spline_n6.hpp
+++ b/cpp/particles/interpolation/spline_n6.hpp
@@ -3,20 +3,20 @@
* Copyright 2015 Max Planck Institute *
* for Dynamics and Self-Organization *
* *
-* This file is part of bfps. *
+* This file is part of TurTLE. *
* *
-* bfps is free software: you can redistribute it and/or modify *
+* TurTLE is free software: you can redistribute it and/or modify *
* it under the terms of the GNU General Public License as published *
* by the Free Software Foundation, either version 3 of the License, *
* or (at your option) any later version. *
* *
-* bfps is distributed in the hope that it will be useful, *
+* TurTLE is distributed in the hope that it will be useful, *
* but WITHOUT ANY WARRANTY; without even the implied warranty of *
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the *
* GNU General Public License for more details. *
* *
* You should have received a copy of the GNU General Public License *
-* along with bfps. If not, see <http://www.gnu.org/licenses/> *
+* along with TurTLE. If not, see <http://www.gnu.org/licenses/> *
* *
* Contact: Cristian.Lalescu@ds.mpg.de *
* *
diff --git a/bfps/cpp/spline_n7.cpp b/cpp/particles/interpolation/spline_n7.cpp
similarity index 99%
rename from bfps/cpp/spline_n7.cpp
rename to cpp/particles/interpolation/spline_n7.cpp
index 84a3fdbc99a96cd3f5a13501414cbaa0a908096d..01407629782e07ba9b473da9226f51c594c1c9be 100644
--- a/bfps/cpp/spline_n7.cpp
+++ b/cpp/particles/interpolation/spline_n7.cpp
@@ -3,20 +3,20 @@
* Copyright 2015 Max Planck Institute *
* for Dynamics and Self-Organization *
* *
-* This file is part of bfps. *
+* This file is part of TurTLE. *
* *
-* bfps is free software: you can redistribute it and/or modify *
+* TurTLE is free software: you can redistribute it and/or modify *
* it under the terms of the GNU General Public License as published *
* by the Free Software Foundation, either version 3 of the License, *
* or (at your option) any later version. *
* *
-* bfps is distributed in the hope that it will be useful, *
+* TurTLE is distributed in the hope that it will be useful, *
* but WITHOUT ANY WARRANTY; without even the implied warranty of *
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the *
* GNU General Public License for more details. *
* *
* You should have received a copy of the GNU General Public License *
-* along with bfps. If not, see <http://www.gnu.org/licenses/> *
+* along with TurTLE. If not, see <http://www.gnu.org/licenses/> *
* *
* Contact: Cristian.Lalescu@ds.mpg.de *
* *
diff --git a/bfps/cpp/spline_n7.hpp b/cpp/particles/interpolation/spline_n7.hpp
similarity index 87%
rename from bfps/cpp/spline_n7.hpp
rename to cpp/particles/interpolation/spline_n7.hpp
index 2c0b86f6ff8bd722fd6bd038d8dfee39727d5fc1..cceab8ffe259534ae39799082e21b0a6bf56c249 100644
--- a/bfps/cpp/spline_n7.hpp
+++ b/cpp/particles/interpolation/spline_n7.hpp
@@ -3,20 +3,20 @@
* Copyright 2017 Max Planck Institute *
* for Dynamics and Self-Organization *
* *
-* This file is part of bfps. *
+* This file is part of TurTLE. *
* *
-* bfps is free software: you can redistribute it and/or modify *
+* TurTLE is free software: you can redistribute it and/or modify *
* it under the terms of the GNU General Public License as published *
* by the Free Software Foundation, either version 3 of the License, *
* or (at your option) any later version. *
* *
-* bfps is distributed in the hope that it will be useful, *
+* TurTLE is distributed in the hope that it will be useful, *
* but WITHOUT ANY WARRANTY; without even the implied warranty of *
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the *
* GNU General Public License for more details. *
* *
* You should have received a copy of the GNU General Public License *
-* along with bfps. If not, see <http://www.gnu.org/licenses/> *
+* along with TurTLE. If not, see <http://www.gnu.org/licenses/> *
* *
* Contact: Cristian.Lalescu@ds.mpg.de *
* *
diff --git a/bfps/cpp/spline_n8.cpp b/cpp/particles/interpolation/spline_n8.cpp
similarity index 99%
rename from bfps/cpp/spline_n8.cpp
rename to cpp/particles/interpolation/spline_n8.cpp
index d6cea769acbc3a74eee1dfd53933a45441318016..53841835a32c1f1ebb14a4a3a1cc11f6c70c8719 100644
--- a/bfps/cpp/spline_n8.cpp
+++ b/cpp/particles/interpolation/spline_n8.cpp
@@ -3,20 +3,20 @@
* Copyright 2015 Max Planck Institute *
* for Dynamics and Self-Organization *
* *
-* This file is part of bfps. *
+* This file is part of TurTLE. *
* *
-* bfps is free software: you can redistribute it and/or modify *
+* TurTLE is free software: you can redistribute it and/or modify *
* it under the terms of the GNU General Public License as published *
* by the Free Software Foundation, either version 3 of the License, *
* or (at your option) any later version. *
* *
-* bfps is distributed in the hope that it will be useful, *
+* TurTLE is distributed in the hope that it will be useful, *
* but WITHOUT ANY WARRANTY; without even the implied warranty of *
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the *
* GNU General Public License for more details. *
* *
* You should have received a copy of the GNU General Public License *
-* along with bfps. If not, see <http://www.gnu.org/licenses/> *
+* along with TurTLE. If not, see <http://www.gnu.org/licenses/> *
* *
* Contact: Cristian.Lalescu@ds.mpg.de *
* *
diff --git a/bfps/cpp/spline_n8.hpp b/cpp/particles/interpolation/spline_n8.hpp
similarity index 87%
rename from bfps/cpp/spline_n8.hpp
rename to cpp/particles/interpolation/spline_n8.hpp
index 39b2e03ad321b495a1da04f1fae8526daa4d3536..5377f5402a086778f8ad04571b19507da6d86e01 100644
--- a/bfps/cpp/spline_n8.hpp
+++ b/cpp/particles/interpolation/spline_n8.hpp
@@ -3,20 +3,20 @@
* Copyright 2017 Max Planck Institute *
* for Dynamics and Self-Organization *
* *
-* This file is part of bfps. *
+* This file is part of TurTLE. *
* *
-* bfps is free software: you can redistribute it and/or modify *
+* TurTLE is free software: you can redistribute it and/or modify *
* it under the terms of the GNU General Public License as published *
* by the Free Software Foundation, either version 3 of the License, *
* or (at your option) any later version. *
* *
-* bfps is distributed in the hope that it will be useful, *
+* TurTLE is distributed in the hope that it will be useful, *
* but WITHOUT ANY WARRANTY; without even the implied warranty of *
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the *
* GNU General Public License for more details. *
* *
* You should have received a copy of the GNU General Public License *
-* along with bfps. If not, see <http://www.gnu.org/licenses/> *
+* along with TurTLE. If not, see <http://www.gnu.org/licenses/> *
* *
* Contact: Cristian.Lalescu@ds.mpg.de *
* *
diff --git a/bfps/cpp/spline_n9.cpp b/cpp/particles/interpolation/spline_n9.cpp
similarity index 99%
rename from bfps/cpp/spline_n9.cpp
rename to cpp/particles/interpolation/spline_n9.cpp
index 4587a3f91c27937f553430463d19ea2221a173dc..ed92aeb14a0120001c39bbafebef2bb91437a3fc 100644
--- a/bfps/cpp/spline_n9.cpp
+++ b/cpp/particles/interpolation/spline_n9.cpp
@@ -3,20 +3,20 @@
* Copyright 2015 Max Planck Institute *
* for Dynamics and Self-Organization *
* *
-* This file is part of bfps. *
+* This file is part of TurTLE. *
* *
-* bfps is free software: you can redistribute it and/or modify *
+* TurTLE is free software: you can redistribute it and/or modify *
* it under the terms of the GNU General Public License as published *
* by the Free Software Foundation, either version 3 of the License, *
* or (at your option) any later version. *
* *
-* bfps is distributed in the hope that it will be useful, *
+* TurTLE is distributed in the hope that it will be useful, *
* but WITHOUT ANY WARRANTY; without even the implied warranty of *
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the *
* GNU General Public License for more details. *
* *
* You should have received a copy of the GNU General Public License *
-* along with bfps. If not, see <http://www.gnu.org/licenses/> *
+* along with TurTLE. If not, see <http://www.gnu.org/licenses/> *
* *
* Contact: Cristian.Lalescu@ds.mpg.de *
* *
diff --git a/bfps/cpp/spline_n9.hpp b/cpp/particles/interpolation/spline_n9.hpp
similarity index 87%
rename from bfps/cpp/spline_n9.hpp
rename to cpp/particles/interpolation/spline_n9.hpp
index b3770a824b2faec1f9196a2b366ed729e6ad21c7..35882791ceb3a486a5eff3d3f5e62ebf83a33287 100644
--- a/bfps/cpp/spline_n9.hpp
+++ b/cpp/particles/interpolation/spline_n9.hpp
@@ -3,20 +3,20 @@
* Copyright 2017 Max Planck Institute *
* for Dynamics and Self-Organization *
* *
-* This file is part of bfps. *
+* This file is part of TurTLE. *
* *
-* bfps is free software: you can redistribute it and/or modify *
+* TurTLE is free software: you can redistribute it and/or modify *
* it under the terms of the GNU General Public License as published *
* by the Free Software Foundation, either version 3 of the License, *
* or (at your option) any later version. *
* *
-* bfps is distributed in the hope that it will be useful, *
+* TurTLE is distributed in the hope that it will be useful, *
* but WITHOUT ANY WARRANTY; without even the implied warranty of *
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the *
* GNU General Public License for more details. *
* *
* You should have received a copy of the GNU General Public License *
-* along with bfps. If not, see <http://www.gnu.org/licenses/> *
+* along with TurTLE. If not, see <http://www.gnu.org/licenses/> *
* *
* Contact: Cristian.Lalescu@ds.mpg.de *
* *
diff --git a/cpp/particles/lock_free_bool_array.hpp b/cpp/particles/lock_free_bool_array.hpp
new file mode 100644
index 0000000000000000000000000000000000000000..985960c34facc49c669e33d08757a813e51a78a4
--- /dev/null
+++ b/cpp/particles/lock_free_bool_array.hpp
@@ -0,0 +1,132 @@
+/******************************************************************************
+* *
+* Copyright 2019 Max Planck Institute for Dynamics and Self-Organization *
+* *
+* This file is part of TurTLE. *
+* *
+* TurTLE is free software: you can redistribute it and/or modify *
+* it under the terms of the GNU General Public License as published *
+* by the Free Software Foundation, either version 3 of the License, *
+* or (at your option) any later version. *
+* *
+* TurTLE is distributed in the hope that it will be useful, *
+* but WITHOUT ANY WARRANTY; without even the implied warranty of *
+* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the *
+* GNU General Public License for more details. *
+* *
+* You should have received a copy of the GNU General Public License *
+* along with TurTLE. If not, see <http://www.gnu.org/licenses/> *
+* *
+* Contact: Cristian.Lalescu@ds.mpg.de *
+* *
+******************************************************************************/
+
+
+
+#ifndef LOCK_FREE_BOOL_ARRAY_HPP
+#define LOCK_FREE_BOOL_ARRAY_HPP
+
+#include <vector>
+#include <atomic>
+#include <unistd.h>
+#include <cstdio>
+#include <omp.h>
+
+class lock_free_bool_array{
+ static const int Available = 0;
+ static const int Busy = 1;
+ static const int NoOwner = -1;
+
+#ifdef __INTEL_COMPILER
+ struct Locker {
+ Locker(){
+ omp_init_nest_lock(&lock);
+ }
+ ~Locker() noexcept(false){
+ omp_destroy_nest_lock(&lock);
+ }
+ omp_nest_lock_t lock;
+ };
+#else
+ struct Locker {
+ Locker() : lock(Available), ownerId(NoOwner), counter(0) {
+ }
+ std::atomic_int lock;
+ std::atomic_int ownerId;
+ int counter;
+ };
+#endif
+
+ std::vector<std::unique_ptr<Locker>> keys;
+
+
+public:
+ explicit lock_free_bool_array(const long int inNbKeys = 1024){
+ keys.resize(inNbKeys);
+ for(auto& k : keys){
+ k.reset(new Locker());
+ }
+ }
+#ifndef NDEBUG
+ ~lock_free_bool_array() noexcept(false){
+ for(auto& k : keys){
+#ifdef __INTEL_COMPILER
+#else
+ assert(k->lock.load() == Available);
+ assert(k->ownerId.load() == NoOwner);
+#endif
+ }
+ }
+#endif
+
+#ifdef __INTEL_COMPILER
+ void lock(const long int inKey){
+ Locker* k = keys[inKey%keys.size()].get();
+ omp_set_nest_lock(&k->lock);
+ }
+
+ void unlock(const long int inKey){
+ Locker* k = keys[inKey%keys.size()].get();
+ omp_unset_nest_lock(&k->lock);
+ }
+#else
+ void lock(const long int inKey){
+ Locker* k = keys[inKey%keys.size()].get();
+ if(k->ownerId.load() != omp_get_thread_num()){
+ int localBusy = Busy;// Intel complains if we pass a const as last param
+ int expected = Available;
+#ifdef __INTEL_COMPILER
+ while(!__sync_val_compare_and_swap((int*)&k->lock, expected, localBusy)){
+#else
+ while(!std::atomic_compare_exchange_strong(&k->lock, &expected, localBusy)){
+#endif
+ usleep(1);
+ expected = Available;
+ }
+#ifdef __INTEL_COMPILER
+ assert(k->ownerId.load() == NoOwner);
+#endif
+ k->ownerId.store(omp_get_thread_num());
+ k->counter = 0; // must remain
+ }
+ k->counter += 1;
+ assert(k->lock.load() == Busy);
+ assert(k->counter >= 1);
+ assert(k->ownerId.load() == omp_get_thread_num());
+ }
+
+ void unlock(const long int inKey){
+ Locker* k = keys[inKey%keys.size()].get();
+ assert(k->lock.load() == Busy);
+ assert(k->counter >= 1);
+ assert(k->ownerId.load() == omp_get_thread_num());
+ k->counter -= 1;
+ if(k->counter == 0){
+ k->ownerId.store(NoOwner);
+ k->lock.store(Available);
+ }
+ }
+#endif
+};
+
+#endif
diff --git a/cpp/particles/p2p/p2p_computer.hpp b/cpp/particles/p2p/p2p_computer.hpp
new file mode 100644
index 0000000000000000000000000000000000000000..5a399e515da2bf59bdfcb3874c7591daf0af1b4c
--- /dev/null
+++ b/cpp/particles/p2p/p2p_computer.hpp
@@ -0,0 +1,118 @@
+/******************************************************************************
+* *
+* Copyright 2019 Max Planck Institute for Dynamics and Self-Organization *
+* *
+* This file is part of TurTLE. *
+* *
+* TurTLE is free software: you can redistribute it and/or modify *
+* it under the terms of the GNU General Public License as published *
+* by the Free Software Foundation, either version 3 of the License, *
+* or (at your option) any later version. *
+* *
+* TurTLE is distributed in the hope that it will be useful, *
+* but WITHOUT ANY WARRANTY; without even the implied warranty of *
+* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the *
+* GNU General Public License for more details. *
+* *
+* You should have received a copy of the GNU General Public License *
+* along with TurTLE. If not, see <http://www.gnu.org/licenses/> *
+* *
+* Contact: Cristian.Lalescu@ds.mpg.de *
+* *
+******************************************************************************/
+
+
+
+#ifndef P2P_COMPUTER_HPP
+#define P2P_COMPUTER_HPP
+
+#include <cstring>
+#include <cassert>
+
+template <class real_number, class partsize_t>
+class p2p_computer{
+
+ bool isActive;
+
+ /** \brief A simple distance weighting function.
+ *
+ * This function returns 1 if a distance is smaller than a cut-off length,
+ * i.e. particle 1 interacts with particle 2 if particle 2 is inside a
+ * sphere of radius `cutoff' centered on particle 1.
+ */
+ static double dumb_distance_weight(
+ const double dist_pow2,
+ const double cutoff){
+ // this function should only be called for interacting particles,
+ // and particles interact if they are closer than cutoff.
+ assert(dist_pow2 < cutoff*cutoff);
+ return 1.0;
+ }
+
+
+public:
+ p2p_computer() : isActive(true){}
+
+ template <int size_particle_rhs>
+ void reduce_particles_rhs(real_number rhs_dst[], const real_number rhs_src[], const partsize_t nbParticles) const{
+ static_assert(size_particle_rhs == 6, "This kernel works only with 6 values per particle's rhs");
+ for(int idx_part = 0 ; idx_part < nbParticles ; ++idx_part){
+ // We merge only the values modified by the current kernel (3-5)
+ for(int idx_rhs = 3 ; idx_rhs < size_particle_rhs ; ++idx_rhs){
+ rhs_dst[idx_part*size_particle_rhs+idx_rhs] += rhs_src[idx_part*size_particle_rhs+idx_rhs];
+ }
+ }
+ }
+
+ /**
+ * NOTE: this is called only ONCE for each pair of interacting particles.
+ */
+ template <int size_particle_positions, int size_particle_rhs>
+ void compute_interaction(const partsize_t /*idx_part1*/,
+ const real_number pos_part1[],
+ real_number rhs_part1[],
+ const partsize_t /*idx_part2*/,
+ const real_number pos_part2[],
+ real_number rhs_part2[],
+ const real_number dist_pow2,
+ const real_number cutoff,
+ const real_number /*xseparation*/,
+ const real_number /*yseparation*/,
+ const real_number /*zseparation*/) const{
+ static_assert(size_particle_positions == 6, "This kernel works only with 6 values for one particle's position+orientation");
+ static_assert(size_particle_rhs == 6, "This kernel works only with 6 values per particle's rhs");
+
+ // TODO: a reasonable way of choosing between different distance_weight functions should be thought of.
+ // We need to ask Michael about how flexible this distance_weight needs to be.
+ const double ww = dumb_distance_weight(dist_pow2, cutoff);
+ ///
+ /// term in equation is:
+ ///
+ /// \f[
+ /// (4 / \tau) \sum_j W_\ell ( | x^i - x^j | ) (p^i \cdot p^j)p^j
+ /// \f]
+ ///
+ const double dot_product = (pos_part1[3+IDXC_X]*pos_part2[3+IDXC_X] +
+ pos_part1[3+IDXC_Y]*pos_part2[3+IDXC_Y] +
+ pos_part1[3+IDXC_Z]*pos_part2[3+IDXC_Z]);
+ rhs_part1[3+IDXC_X] += pos_part2[3+IDXC_X] * 4 * ww * dot_product;
+ rhs_part1[3+IDXC_Y] += pos_part2[3+IDXC_Y] * 4 * ww * dot_product;
+ rhs_part1[3+IDXC_Z] += pos_part2[3+IDXC_Z] * 4 * ww * dot_product;
+ rhs_part2[3+IDXC_X] += pos_part1[3+IDXC_X] * 4 * ww * dot_product;
+ rhs_part2[3+IDXC_Y] += pos_part1[3+IDXC_Y] * 4 * ww * dot_product;
+ rhs_part2[3+IDXC_Z] += pos_part1[3+IDXC_Z] * 4 * ww * dot_product;
+ }
+
+
+ void merge(const p2p_computer&){}
+
+ bool isEnable() const {
+ return isActive;
+ }
+
+ void setEnable(const bool inIsActive) {
+ isActive = inIsActive;
+ }
+};
+
+#endif
diff --git a/cpp/particles/p2p/p2p_computer_empty.hpp b/cpp/particles/p2p/p2p_computer_empty.hpp
new file mode 100644
index 0000000000000000000000000000000000000000..b13a95e1f2bb8fd4f51f51c4cb3b6f74e00b079b
--- /dev/null
+++ b/cpp/particles/p2p/p2p_computer_empty.hpp
@@ -0,0 +1,61 @@
+/******************************************************************************
+* *
+* Copyright 2019 Max Planck Institute for Dynamics and Self-Organization *
+* *
+* This file is part of TurTLE. *
+* *
+* TurTLE is free software: you can redistribute it and/or modify *
+* it under the terms of the GNU General Public License as published *
+* by the Free Software Foundation, either version 3 of the License, *
+* or (at your option) any later version. *
+* *
+* TurTLE is distributed in the hope that it will be useful, *
+* but WITHOUT ANY WARRANTY; without even the implied warranty of *
+* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the *
+* GNU General Public License for more details. *
+* *
+* You should have received a copy of the GNU General Public License *
+* along with TurTLE. If not, see <http://www.gnu.org/licenses/> *
+* *
+* Contact: Cristian.Lalescu@ds.mpg.de *
+* *
+******************************************************************************/
+
+
+
+#ifndef P2P_COMPUTER_EMPTY_HPP
+#define P2P_COMPUTER_EMPTY_HPP
+
+#include <cstring>
+
+template <class real_number, class partsize_t>
+class p2p_computer_empty{
+public:
+
+ template <int size_particle_rhs>
+ void reduce_particles_rhs(real_number /*rhs_dst*/[], const real_number /*rhs_src*/[], const partsize_t /*nbParticles*/) const{
+ }
+
+ /**
+ * NOTE: this is called only ONCE for each pair of interacting particles.
+ */
+ template <int size_particle_positions, int size_particle_rhs>
+ void compute_interaction(const partsize_t /*idx_part1*/,
+ const real_number /*pos_part1*/[], real_number /*rhs_part1*/[],
+ const partsize_t /*idx_part2*/,
+ const real_number /*pos_part2*/[], real_number /*rhs_part2*/[],
+ const real_number /*dist_pow2*/,
+ const real_number /*cutoff*/,
+ const real_number /*xseparation*/,
+ const real_number /*yseparation*/,
+ const real_number /*zseparation*/) const{
+ }
+
+ void merge(const p2p_computer_empty&){}
+
+ constexpr static bool isEnable() {
+ return false;
+ }
+};
+
+#endif
diff --git a/cpp/particles/p2p/p2p_distr_mpi.hpp b/cpp/particles/p2p/p2p_distr_mpi.hpp
new file mode 100644
index 0000000000000000000000000000000000000000..3051bcee05b5d6078abdbac571cc4500ff6af475
--- /dev/null
+++ b/cpp/particles/p2p/p2p_distr_mpi.hpp
@@ -0,0 +1,1030 @@
+/******************************************************************************
+* *
+* Copyright 2019 Max Planck Institute for Dynamics and Self-Organization *
+* *
+* This file is part of TurTLE. *
+* *
+* TurTLE is free software: you can redistribute it and/or modify *
+* it under the terms of the GNU General Public License as published *
+* by the Free Software Foundation, either version 3 of the License, *
+* or (at your option) any later version. *
+* *
+* TurTLE is distributed in the hope that it will be useful, *
+* but WITHOUT ANY WARRANTY; without even the implied warranty of *
+* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the *
+* GNU General Public License for more details. *
+* *
+* You should have received a copy of the GNU General Public License *
+* along with TurTLE. If not, see <http://www.gnu.org/licenses/> *
+* *
+* Contact: Cristian.Lalescu@ds.mpg.de *
+* *
+******************************************************************************/
+
+
+
+#ifndef P2P_DISTR_MPI_HPP
+#define P2P_DISTR_MPI_HPP
+
+#include <mpi.h>
+
+#include <vector>
+#include <memory>
+#include <cassert>
+
+#include <type_traits>
+#include <omp.h>
+#include <algorithm>
+
+#include "scope_timer.hpp"
+#include "particles/particles_utils.hpp"
+#include "particles/p2p/p2p_tree.hpp"
+#include "particles/lock_free_bool_array.hpp"
+
+template <class partsize_t, class real_number>
+class p2p_distr_mpi {
+protected:
+ static const int MaxNbRhs = 10;
+
+ enum MpiTag{
+ TAG_NB_PARTICLES,
+ TAG_POSITION_PARTICLES,
+ TAG_INDEX_PARTICLES,
+ TAG_LABEL_PARTICLES,
+ TAG_RESULT_PARTICLES,
+ };
+
+ struct NeighborDescriptor{
+ partsize_t nbParticlesToExchange;
+ int destProc;
+ int nbLevelsToExchange;
+ bool isRecv;
+ int nbReceived;
+
+ std::unique_ptr<real_number[]> toRecvAndMerge;
+ std::unique_ptr<real_number[]> toCompute;
+ std::unique_ptr<real_number[]> results;
+ std::unique_ptr<partsize_t[]> indexes;
+ std::unique_ptr<partsize_t[]> labels;
+ };
+
+ enum Action{
+ NOTHING_TODO = 512,
+ RECV_PARTICLES,
+ COMPUTE_PARTICLES,
+ RELEASE_BUFFER_PARTICLES,
+ MERGE_PARTICLES
+ };
+
+ MPI_Comm current_com;
+
+ int my_rank;
+ int nb_processes;
+ int nb_processes_involved;
+
+ const std::pair<int,int> current_partition_interval;
+ const int current_partition_size;
+ const std::array<size_t,3> field_grid_dim;
+
+ std::unique_ptr<int[]> partition_interval_size_per_proc;
+ std::unique_ptr<int[]> partition_interval_offset_per_proc;
+
+ std::unique_ptr<partsize_t[]> current_offset_particles_for_partition;
+
+ std::vector<std::pair<Action,int>> whatNext;
+ std::vector<MPI_Request> mpiRequests;
+ std::vector<NeighborDescriptor> neigDescriptors;
+
+ std::array<real_number,3> spatial_box_width;
+ std::array<real_number,3> spatial_box_offset;
+
+ const long double cutoff_radius_compute;
+ const int nb_cells_factor;
+ const long double cutoff_radius;
+ std::array<long int,3> nb_cell_levels;
+
+ template <class DataType, int sizeElement>
+ static void permute_copy(const partsize_t offsetIdx,
+ const partsize_t nbElements,
+ const std::pair<long int, partsize_t> permutation[],
+ DataType data[],
+ std::vector<unsigned char>* buffer){
+ buffer->resize(nbElements*sizeof(DataType)*sizeElement);
+ DataType* dataBuffer = reinterpret_cast<DataType*>(buffer->data());
+
+ // Permute
+ for(partsize_t idxPart = 0 ; idxPart < nbElements ; ++idxPart){
+ const partsize_t srcData = permutation[idxPart].second;
+ const partsize_t destData = idxPart;
+ for(int idxVal = 0 ; idxVal < sizeElement ; ++idxVal){
+ dataBuffer[destData*sizeElement + idxVal]
+ = data[srcData*sizeElement + idxVal];
+ }
+ }
+
+ // Copy back
+ for(partsize_t idxPart = 0 ; idxPart < nbElements ; ++idxPart){
+ const partsize_t srcData = idxPart;
+ const partsize_t destData = idxPart+offsetIdx;
+ for(int idxVal = 0 ; idxVal < sizeElement ; ++idxVal){
+ data[destData*sizeElement + idxVal]
+ = dataBuffer[srcData*sizeElement + idxVal];
+ }
+
+ // Clean up memory
+ buffer->resize(0);
+ }
+
+ buffer->resize(0);
+ }
+
+ static int foundGridFactor(const real_number in_cutoff_radius, const std::array<real_number,3>& in_spatial_box_width){
+ int idx_factor = 1;
+ DEBUG_MSG("in_cutoff_radius is %g, in_spatial_box_width[IDXC_Z] = %g\n", in_cutoff_radius, in_spatial_box_width[IDXC_Z]);
+ while(in_cutoff_radius <= in_spatial_box_width[IDXC_Z]/(long double)(idx_factor+1)){
+ idx_factor += 1;
+ }
+ return idx_factor;
+ }
+
+public:
+ ////////////////////////////////////////////////////////////////////////////
+
+ p2p_distr_mpi(MPI_Comm in_current_com,
+ const std::pair<int,int>& in_current_partitions,
+ const std::array<size_t,3>& in_field_grid_dim,
+ const std::array<real_number,3>& in_spatial_box_width,
+ const std::array<real_number,3>& in_spatial_box_offset,
+ const real_number in_cutoff_radius)
+ : current_com(in_current_com),
+ my_rank(-1), nb_processes(-1),nb_processes_involved(-1),
+ current_partition_interval(in_current_partitions),
+ current_partition_size(current_partition_interval.second-current_partition_interval.first),
+ field_grid_dim(in_field_grid_dim),
+ spatial_box_width(in_spatial_box_width), spatial_box_offset(in_spatial_box_offset),
+ cutoff_radius_compute(in_cutoff_radius),
+ nb_cells_factor(foundGridFactor(in_cutoff_radius, in_spatial_box_width)),
+ cutoff_radius(in_spatial_box_width[IDXC_Z]/real_number(nb_cells_factor)){
+
+ AssertMpi(MPI_Comm_rank(current_com, &my_rank));
+ AssertMpi(MPI_Comm_size(current_com, &nb_processes));
+
+ partition_interval_size_per_proc.reset(new int[nb_processes]);
+ AssertMpi( MPI_Allgather( const_cast<int*>(¤t_partition_size), 1, MPI_INT,
+ partition_interval_size_per_proc.get(), 1, MPI_INT,
+ current_com) );
+ assert(partition_interval_size_per_proc[my_rank] == current_partition_size);
+
+ partition_interval_offset_per_proc.reset(new int[nb_processes+1]);
+ partition_interval_offset_per_proc[0] = 0;
+ for(int idxProc = 0 ; idxProc < nb_processes ; ++idxProc){
+ partition_interval_offset_per_proc[idxProc+1] = partition_interval_offset_per_proc[idxProc] + partition_interval_size_per_proc[idxProc];
+ }
+
+ current_offset_particles_for_partition.reset(new partsize_t[current_partition_size+1]);
+
+ nb_processes_involved = nb_processes;
+ while(nb_processes_involved != 0 && partition_interval_size_per_proc[nb_processes_involved-1] == 0){
+ nb_processes_involved -= 1;
+ }
+ assert(nb_processes_involved != 0);
+ for(int idx_proc_involved = 0 ; idx_proc_involved < nb_processes_involved ; ++idx_proc_involved){
+ assert(partition_interval_size_per_proc[idx_proc_involved] != 0);
+ }
+
+ assert(int(field_grid_dim[IDXC_Z]) == partition_interval_offset_per_proc[nb_processes_involved]);
+
+ nb_cell_levels[IDXC_X] = nb_cells_factor;
+ nb_cell_levels[IDXC_Y] = nb_cells_factor;
+ nb_cell_levels[IDXC_Z] = nb_cells_factor;
+ }
+
+ virtual ~p2p_distr_mpi() noexcept(false){}
+
+ ////////////////////////////////////////////////////////////////////////////
+
+ int getGridFactor() const{
+ return nb_cells_factor;
+ }
+
+ real_number getGridCutoff() const{
+ return cutoff_radius;
+ }
+
+ long int get_cell_coord_x_from_index(const long int index) const{
+ return index % nb_cell_levels[IDXC_X];
+ }
+
+ long int get_cell_coord_y_from_index(const long int index) const{
+ return (index % (nb_cell_levels[IDXC_X]*nb_cell_levels[IDXC_Y]))
+ / nb_cell_levels[IDXC_X];
+ }
+
+ long int get_cell_coord_z_from_index(const long int index) const{
+ return index / (nb_cell_levels[IDXC_X]*nb_cell_levels[IDXC_Y]);
+ }
+
+ long int first_cell_level_proc(const int dest_proc) const{
+ const real_number field_section_width_z = spatial_box_width[IDXC_Z]/real_number(field_grid_dim[IDXC_Z]);
+ return static_cast<long int>((field_section_width_z*real_number(partition_interval_offset_per_proc[dest_proc]))/cutoff_radius);
+ }
+
+ long int last_cell_level_proc(const int dest_proc) const{
+ const long double field_section_width_z = spatial_box_width[IDXC_Z]/(long double)(field_grid_dim[IDXC_Z]);
+ const long int limite = static_cast<long int>((field_section_width_z*(long double)(partition_interval_offset_per_proc[dest_proc+1])
+ - std::numeric_limits<long double>::epsilon())/cutoff_radius);
+ if(static_cast<long double>(limite)*cutoff_radius
+ == field_section_width_z*(long double)(partition_interval_offset_per_proc[dest_proc+1])
+ || limite == nb_cell_levels[IDXC_Z]){
+ return limite-1;
+ }
+ return limite;
+ }
+
+ real_number apply_pbc(real_number pos, IDX_COMPONENT_3D dim) const{
+ while( pos < spatial_box_offset[dim] ){
+ pos += spatial_box_width[dim];
+ }
+ while( spatial_box_width[dim]+spatial_box_offset[dim] <= pos){
+ pos -= spatial_box_width[dim];
+ }
+ return pos;
+ }
+
+ std::array<long int,3> get_cell_coordinate(const real_number pos_x, const real_number pos_y,
+ const real_number pos_z) const {
+ const real_number diff_x = apply_pbc(pos_x,IDXC_X) - spatial_box_offset[IDXC_X];
+ const real_number diff_y = apply_pbc(pos_y,IDXC_Y) - spatial_box_offset[IDXC_Y];
+ const real_number diff_z = apply_pbc(pos_z,IDXC_Z) - spatial_box_offset[IDXC_Z];
+ std::array<long int,3> coord;
+ coord[IDXC_X] = static_cast<long int>(diff_x/cutoff_radius);
+ coord[IDXC_Y] = static_cast<long int>(diff_y/cutoff_radius);
+ coord[IDXC_Z] = static_cast<long int>(diff_z/cutoff_radius);
+ return coord;
+ }
+
+ long int get_cell_idx(const real_number pos_x, const real_number pos_y,
+ const real_number pos_z) const {
+ std::array<long int,3> coord = get_cell_coordinate(pos_x, pos_y, pos_z);
+ return ((coord[IDXC_Z]*nb_cell_levels[IDXC_Y])+coord[IDXC_Y])*nb_cell_levels[IDXC_X]+coord[IDXC_X];
+ }
+
+ real_number compute_distance_r2(const real_number x1, const real_number y1, const real_number z1,
+ const real_number x2, const real_number y2, const real_number z2,
+ const real_number xshift_coef, const real_number yshift_coef, const real_number zshift_coef,
+ real_number &diff_x, real_number &diff_y, real_number &diff_z) const {
+ diff_x = apply_pbc(x1,IDXC_X)-apply_pbc(x2,IDXC_X)+xshift_coef*spatial_box_width[IDXC_X];
+ assert(std::abs(diff_x) <= 2*cutoff_radius);
+
+ diff_y = apply_pbc(y1,IDXC_X)-apply_pbc(y2,IDXC_X)+yshift_coef*spatial_box_width[IDXC_Y];
+ assert(std::abs(diff_y) <= 2*cutoff_radius);
+
+ diff_z = apply_pbc(z1,IDXC_X)-apply_pbc(z2,IDXC_X)+zshift_coef*spatial_box_width[IDXC_Z];
+ assert(std::abs(diff_z) <= 2*cutoff_radius);
+
+ return (diff_x*diff_x) + (diff_y*diff_y) + (diff_z*diff_z);
+ }
+
+ template <class computer_class, int size_particle_positions, int size_particle_rhs>
+ void compute_distr(computer_class& in_computer,
+ const partsize_t current_my_nb_particles_per_partition[],
+ real_number particles_positions[],
+ std::unique_ptr<real_number[]> particles_current_rhs[], const int nb_rhs,
+ partsize_t inout_index_particles[],
+ partsize_t inout_label_particles[] = nullptr){
+ TIMEZONE("p2p_distr_mpi::compute_distr");
+
+ // Some processes might not be involved
+ if(nb_processes_involved <= my_rank){
+ DEBUG_MSG("warning: nb_processes_involved <= my_rank, and this process is exiting p2p_distr_mpi::compute_distr now.\nHowever, there is a check below which calls an MPI_Gather over MPI_COMM_WORLD.\n");
+ return;
+ }
+
+ const bool labels_present = (inout_label_particles != nullptr);
+
+ const long int my_top_z_cell_level = last_cell_level_proc(my_rank);
+ assert(my_top_z_cell_level < nb_cell_levels[IDXC_Z]);
+ const long int my_down_z_cell_level = first_cell_level_proc(my_rank);
+ assert(0 <= my_down_z_cell_level);
+ const long int my_nb_cell_levels = 1+my_top_z_cell_level-my_down_z_cell_level;
+
+ current_offset_particles_for_partition[0] = 0;
+ partsize_t myTotalNbParticles = 0;
+ for(int idxPartition = 0 ; idxPartition < current_partition_size ; ++idxPartition){
+ myTotalNbParticles += current_my_nb_particles_per_partition[idxPartition];
+ current_offset_particles_for_partition[idxPartition+1] = current_offset_particles_for_partition[idxPartition] + current_my_nb_particles_per_partition[idxPartition];
+ }
+
+ // Compute box idx for each particle
+ std::unique_ptr<long int[]> particles_coord(new long int[current_offset_particles_for_partition[current_partition_size]]);
+
+ {
+ for(int idxPartition = 0 ; idxPartition < current_partition_size ; ++idxPartition){
+ #pragma omp parallel for schedule(static)
+ for(partsize_t idxPart = current_offset_particles_for_partition[idxPartition] ; idxPart < current_offset_particles_for_partition[idxPartition+1] ; ++idxPart ){
+ particles_coord[idxPart] = get_cell_idx(particles_positions[(idxPart)*size_particle_positions + IDXC_X],
+ particles_positions[(idxPart)*size_particle_positions + IDXC_Y],
+ particles_positions[(idxPart)*size_particle_positions + IDXC_Z]);
+ assert(my_down_z_cell_level <= get_cell_coord_z_from_index(particles_coord[idxPart]));
+ assert(get_cell_coord_z_from_index(particles_coord[idxPart]) <= my_top_z_cell_level);
+ }
+ }
+
+ std::vector<std::pair<long int,partsize_t>> part_to_sort;
+
+ // Sort each partition in cells
+ for(int idxPartition = 0 ; idxPartition < current_partition_size ; ++idxPartition){
+ part_to_sort.clear();
+
+ for(partsize_t idxPart = current_offset_particles_for_partition[idxPartition] ; idxPart < current_offset_particles_for_partition[idxPartition+1] ; ++idxPart ){
+ part_to_sort.emplace_back();
+ part_to_sort.back().first = particles_coord[idxPart];
+ part_to_sort.back().second = idxPart;
+ }
+
+ assert(partsize_t(part_to_sort.size()) == (current_my_nb_particles_per_partition[idxPartition]));
+
+ std::sort(part_to_sort.begin(), part_to_sort.end(),
+ [](const std::pair<long int,partsize_t>& p1,
+ const std::pair<long int,partsize_t>& p2){
+ return p1.first < p2.first;
+ });
+
+ // Permute array using buffer
+ // permute 4th function parameter using buffer, based on information in first 3 parameters
+ std::vector<unsigned char> buffer;
+ permute_copy<real_number, size_particle_positions>(
+ current_offset_particles_for_partition[idxPartition],
+ current_my_nb_particles_per_partition[idxPartition],
+ part_to_sort.data(),
+ particles_positions,
+ &buffer);
+ for(int idx_rhs = 0 ; idx_rhs < nb_rhs ; ++idx_rhs)
+ {
+ permute_copy<real_number, size_particle_rhs>(
+ current_offset_particles_for_partition[idxPartition],
+ current_my_nb_particles_per_partition[idxPartition],
+ part_to_sort.data(),
+ particles_current_rhs[idx_rhs].get(),
+ &buffer);
+ }
+ permute_copy<partsize_t, 1>(
+ current_offset_particles_for_partition[idxPartition],
+ current_my_nb_particles_per_partition[idxPartition],
+ part_to_sort.data(),
+ inout_index_particles,
+ &buffer);
+ if (labels_present)
+ permute_copy<partsize_t, 1>(
+ current_offset_particles_for_partition[idxPartition],
+ current_my_nb_particles_per_partition[idxPartition],
+ part_to_sort.data(),
+ inout_label_particles,
+ &buffer);
+ permute_copy<long int, 1>(
+ current_offset_particles_for_partition[idxPartition],
+ current_my_nb_particles_per_partition[idxPartition],
+ part_to_sort.data(),
+ particles_coord.get(),
+ &buffer);
+ }
+ }
+
+ // Build the tree
+ p2p_tree<std::vector<std::pair<partsize_t,partsize_t>>> my_tree(nb_cell_levels);
+
+ for(int idxPartition = 0 ; idxPartition < current_partition_size ; ++idxPartition){
+ long int current_cell_idx = -1;
+ partsize_t current_nb_particles_in_cell = 0;
+ partsize_t current_cell_offset = 0;
+
+ for(partsize_t idx_part = current_offset_particles_for_partition[idxPartition] ;
+ idx_part != current_offset_particles_for_partition[idxPartition+1]; ++idx_part){
+ if(particles_coord[idx_part] != current_cell_idx){
+ if(current_nb_particles_in_cell){
+ my_tree.getCell(current_cell_idx).emplace_back(current_cell_offset,current_nb_particles_in_cell);
+ }
+ current_cell_idx = particles_coord[idx_part];
+ current_nb_particles_in_cell = 1;
+ current_cell_offset = idx_part;
+ }
+ else{
+ current_nb_particles_in_cell += 1;
+ }
+ }
+ if(current_nb_particles_in_cell){
+ my_tree.getCell(current_cell_idx).emplace_back(current_cell_offset,current_nb_particles_in_cell);
+
+ }
+ }
+
+ // Offset per cell layers
+ long int previous_index = 0;
+ variable_used_only_in_assert(previous_index);
+ std::unique_ptr<partsize_t[]> particles_offset_layers(new partsize_t[my_nb_cell_levels+1]());
+ for(int idxPartition = 0 ; idxPartition < current_partition_size ; ++idxPartition){
+ for(partsize_t idx_part = current_offset_particles_for_partition[idxPartition] ;
+ idx_part != current_offset_particles_for_partition[idxPartition+1]; ++idx_part){
+ const long int part_box_z_index = get_cell_coord_z_from_index(particles_coord[idx_part]);
+ assert(my_down_z_cell_level <= part_box_z_index);
+ assert(part_box_z_index <= my_top_z_cell_level);
+ particles_offset_layers[part_box_z_index+1-my_down_z_cell_level] += 1;
+ assert(previous_index <= part_box_z_index);
+ previous_index = part_box_z_index;
+ }
+ }
+ for(long int idx_layer = 0 ; idx_layer < my_nb_cell_levels ; ++idx_layer){
+ particles_offset_layers[idx_layer+1] += particles_offset_layers[idx_layer];
+ }
+
+ // Reset vectors
+ assert(whatNext.size() == 0);
+ assert(mpiRequests.size() == 0);
+ neigDescriptors.clear();
+
+ DEBUG_MSG("my_top_z_cell_level = %d\n", int(my_top_z_cell_level));
+ DEBUG_MSG("my_down_z_cell_level = %d\n", int(my_down_z_cell_level));
+ DEBUG_MSG("first_cell_level_proc(0) = %d\n", int(first_cell_level_proc(0)));
+ DEBUG_MSG("first_cell_level_proc(1) = %d\n", int(first_cell_level_proc(1)));
+ if (nb_processes > 1)
+ DEBUG_MSG("first_cell_level_proc(2) = %d\n", int(first_cell_level_proc(2)));
+ if (nb_processes > 2)
+ DEBUG_MSG("first_cell_level_proc(3) = %d\n", int(first_cell_level_proc(3)));
+ DEBUG_MSG("nb_cell_levels[IDXC_Z] = %d\n", int(nb_cell_levels[IDXC_Z]));
+ DEBUG_MSG("last_cell_level_proc(0) = %d\n", int(last_cell_level_proc(0)));
+ if (nb_processes > 1)
+ DEBUG_MSG("last_cell_level_proc(1) = %d\n", int(last_cell_level_proc(1)));
+ if (nb_processes > 2)
+ DEBUG_MSG("last_cell_level_proc(2) = %d\n", int(last_cell_level_proc(2)));
+ if (nb_processes > 3)
+ DEBUG_MSG("last_cell_level_proc(3) = %d\n", int(last_cell_level_proc(3)));
+
+ // Find process with at least one neighbor
+ {
+ int dest_proc = (my_rank+1)%nb_processes_involved;
+ while(dest_proc != my_rank
+ && (my_top_z_cell_level == first_cell_level_proc(dest_proc)
+ || (my_top_z_cell_level+1)%nb_cell_levels[IDXC_Z] == first_cell_level_proc(dest_proc))){
+ // Find if we have to send 1 or 2 cell levels
+ int nb_levels_to_send = 1;
+ if(my_nb_cell_levels > 1 // I have more than one level
+ && (my_top_z_cell_level-1+2)%nb_cell_levels[IDXC_Z] <= last_cell_level_proc(dest_proc)){
+ nb_levels_to_send += 1;
+ }
+ DEBUG_MSG("looking at dest_proc = %d ; nb_levels_to_send = %d\n", dest_proc, nb_levels_to_send);
+
+ NeighborDescriptor descriptor;
+ descriptor.destProc = dest_proc;
+ descriptor.nbLevelsToExchange = nb_levels_to_send;
+ descriptor.nbParticlesToExchange = particles_offset_layers[my_nb_cell_levels] - particles_offset_layers[my_nb_cell_levels-nb_levels_to_send];
+ descriptor.isRecv = false;
+ descriptor.nbReceived = 0;
+
+ neigDescriptors.emplace_back(std::move(descriptor));
+
+ dest_proc = (dest_proc+1)%nb_processes_involved;
+ }
+
+ int src_proc = (my_rank-1+nb_processes_involved)%nb_processes_involved;
+ while(src_proc != my_rank
+ && (last_cell_level_proc(src_proc) == my_down_z_cell_level
+ || (last_cell_level_proc(src_proc)+1)%nb_cell_levels[IDXC_Z] == my_down_z_cell_level)){
+ // Find if we have to send 1 or 2 cell levels
+ int nb_levels_to_recv = 1;
+ if(my_nb_cell_levels > 1 // I have more than one level
+ && first_cell_level_proc(src_proc) <= (my_down_z_cell_level-1+2)%nb_cell_levels[IDXC_Z]){
+ nb_levels_to_recv += 1;
+ }
+ DEBUG_MSG("looking at src_proc = %d ; nb_levels_to_recv = %d\n", src_proc, nb_levels_to_recv);
+
+ NeighborDescriptor descriptor;
+ descriptor.destProc = src_proc;
+ descriptor.nbLevelsToExchange = nb_levels_to_recv;
+ descriptor.nbParticlesToExchange = -1;
+ descriptor.isRecv = true;
+ descriptor.nbReceived = 0;
+
+ neigDescriptors.emplace_back(std::move(descriptor));
+
+ src_proc = (src_proc-1+nb_processes_involved)%nb_processes_involved;
+ }
+ }
+
+ DEBUG_MSG_WAIT(current_com, "hello, found processes with neighbours.\n");
+
+ //////////////////////////////////////////////////////////////////////
+ /// Exchange the number of particles in each partition
+ /// Could involve only here but I do not think it will be a problem
+ //////////////////////////////////////////////////////////////////////
+
+ assert(whatNext.size() == 0);
+ assert(mpiRequests.size() == 0);
+#ifndef NDEBUG // Just for assertion
+ std::vector<int> willsend(nb_processes_involved, 0);
+ std::vector<int> willrecv(nb_processes_involved, 0);
+#endif
+
+ for(int idxDescr = 0 ; idxDescr < int(neigDescriptors.size()) ; ++idxDescr){
+ NeighborDescriptor& descriptor = neigDescriptors[idxDescr];
+
+ if(descriptor.isRecv == false){
+ whatNext.emplace_back(std::pair<Action,int>{NOTHING_TODO, -1});
+ mpiRequests.emplace_back();
+ AssertMpi(MPI_Isend(
+ const_cast<partsize_t*>(&descriptor.nbParticlesToExchange),
+ 1,
+ particles_utils::GetMpiType(partsize_t()),
+ descriptor.destProc,
+ TAG_NB_PARTICLES,
+ current_com,
+ &mpiRequests.back()));
+#ifndef NDEBUG // Just for assertion
+ willsend[descriptor.destProc] += 1;
+#endif
+ if(descriptor.nbParticlesToExchange){
+ whatNext.emplace_back(std::pair<Action,int>{NOTHING_TODO, -1});
+ mpiRequests.emplace_back();
+ assert(descriptor.nbParticlesToExchange*size_particle_positions < std::numeric_limits<int>::max());
+ AssertMpi(MPI_Isend(
+ const_cast<real_number*>(&particles_positions[particles_offset_layers[my_nb_cell_levels-descriptor.nbLevelsToExchange]*size_particle_positions]),
+ int(descriptor.nbParticlesToExchange*size_particle_positions),
+ particles_utils::GetMpiType(real_number()),
+ descriptor.destProc,
+ TAG_POSITION_PARTICLES,
+ current_com,
+ &mpiRequests.back()));
+
+ whatNext.emplace_back(std::pair<Action,int>{NOTHING_TODO, -1});
+ mpiRequests.emplace_back();
+ assert(descriptor.nbParticlesToExchange*size_particle_positions < std::numeric_limits<int>::max());
+ AssertMpi(MPI_Isend(
+ const_cast<partsize_t*>(&inout_index_particles[particles_offset_layers[my_nb_cell_levels-descriptor.nbLevelsToExchange]]),
+ int(descriptor.nbParticlesToExchange),
+ particles_utils::GetMpiType(partsize_t()),
+ descriptor.destProc,
+ TAG_INDEX_PARTICLES,
+ current_com,
+ &mpiRequests.back()));
+
+ if (labels_present)
+ {
+ whatNext.emplace_back(std::pair<Action,int>{NOTHING_TODO, -1});
+ mpiRequests.emplace_back();
+ assert(descriptor.nbParticlesToExchange*size_particle_positions < std::numeric_limits<int>::max());
+ AssertMpi(MPI_Isend(
+ const_cast<partsize_t*>(&inout_label_particles[particles_offset_layers[my_nb_cell_levels-descriptor.nbLevelsToExchange]]),
+ int(descriptor.nbParticlesToExchange),
+ particles_utils::GetMpiType(partsize_t()),
+ descriptor.destProc,
+ TAG_LABEL_PARTICLES,
+ current_com,
+ &mpiRequests.back()));
+ }
+
+ assert(descriptor.toRecvAndMerge == nullptr);
+ descriptor.toRecvAndMerge.reset(new real_number[descriptor.nbParticlesToExchange*size_particle_rhs]);
+ whatNext.emplace_back(std::pair<Action,int>{MERGE_PARTICLES, idxDescr});
+ mpiRequests.emplace_back();
+ assert(descriptor.nbParticlesToExchange*size_particle_rhs < std::numeric_limits<int>::max());
+ AssertMpi(MPI_Irecv(
+ descriptor.toRecvAndMerge.get(),
+ int(descriptor.nbParticlesToExchange*size_particle_rhs),
+ particles_utils::GetMpiType(real_number()),
+ descriptor.destProc,
+ TAG_RESULT_PARTICLES,
+ current_com,
+ &mpiRequests.back()));
+ }
+ }
+ else{
+#ifndef NDEBUG // Just for assertion
+ willrecv[descriptor.destProc] += 1;
+#endif
+ whatNext.emplace_back(std::pair<Action,int>{RECV_PARTICLES, idxDescr});
+ mpiRequests.emplace_back();
+ AssertMpi(MPI_Irecv(
+ &descriptor.nbParticlesToExchange,
+ 1,
+ particles_utils::GetMpiType(partsize_t()),
+ descriptor.destProc,
+ TAG_NB_PARTICLES,
+ current_com,
+ &mpiRequests.back()));
+ }
+ }
+
+#ifndef NDEBUG // Just for assertion
+ {
+ if(myrank == 0){
+ std::vector<int> willsendall(nb_processes_involved*nb_processes_involved, 0);// TODO debug
+ std::vector<int> willrecvall(nb_processes_involved*nb_processes_involved, 0);// TODO debug
+
+ MPI_Gather(willrecv.data(),
+ nb_processes_involved,
+ MPI_INT,
+ willrecvall.data(),
+ nb_processes_involved,
+ MPI_INT,
+ 0,
+ MPI_COMM_WORLD);
+ MPI_Gather(willsend.data(),
+ nb_processes_involved,
+ MPI_INT,
+ willsendall.data(),
+ nb_processes_involved,
+ MPI_INT,
+ 0,
+ MPI_COMM_WORLD);
+
+ for(int idxproc = 0 ; idxproc < nb_processes_involved ; ++idxproc){
+ for(int idxtest = 0 ; idxtest < nb_processes_involved ; ++idxtest){
+ DEBUG_MSG("p2p_distr_mpi::compute_distribution, comparing willsendall(%d, %d)=%d with willrecvall(%d, %d) = %d\n",
+ idxproc, idxtest, willsendall[idxproc*nb_processes_involved + idxtest],
+ idxtest, idxproc, willrecvall[idxtest*nb_processes_involved + idxproc]);
+ assert(willsendall[idxproc*nb_processes_involved + idxtest]
+ == willrecvall[idxtest*nb_processes_involved + idxproc]);
+ }
+ }
+ }
+ else{
+ MPI_Gather(willrecv.data(),
+ nb_processes_involved,
+ MPI_INT,
+ nullptr,
+ 0,
+ MPI_INT,
+ 0,
+ MPI_COMM_WORLD);
+ MPI_Gather(willsend.data(),
+ nb_processes_involved,
+ MPI_INT,
+ nullptr,
+ 0,
+ MPI_INT,
+ 0,
+ MPI_COMM_WORLD);
+ }
+ }
+#endif
+
+ lock_free_bool_array cells_locker(512);
+
+ std::vector<std::unique_ptr<computer_class>> computer_for_all_threads(omp_get_max_threads()-1);
+ for(int idxThread = 1 ; idxThread < omp_get_max_threads() ; ++idxThread){
+ computer_for_all_threads[idxThread-1].reset(new computer_class(in_computer));
+ }
+
+ TIMEZONE_OMP_INIT_PREPARALLEL(omp_get_max_threads())
+ #pragma omp parallel default(shared)
+ {
+ computer_class& computer_thread = (omp_get_thread_num() == 0 ? in_computer : *computer_for_all_threads[omp_get_thread_num()-1]);
+ #pragma omp master
+ {
+ while(mpiRequests.size()){
+ TIMEZONE("wait-loop");
+ assert(mpiRequests.size() == whatNext.size());
+
+ int idxDone = int(mpiRequests.size());
+ {
+ TIMEZONE("wait");
+ AssertMpi(MPI_Waitany(int(mpiRequests.size()), mpiRequests.data(), &idxDone, MPI_STATUSES_IGNORE));
+ }
+ const std::pair<Action, int> releasedAction = whatNext[idxDone];
+ std::swap(mpiRequests[idxDone], mpiRequests[mpiRequests.size()-1]);
+ std::swap(whatNext[idxDone], whatNext[mpiRequests.size()-1]);
+ mpiRequests.pop_back();
+ whatNext.pop_back();
+
+ //////////////////////////////////////////////////////////////////////
+ /// Data to exchange particles
+ //////////////////////////////////////////////////////////////////////
+ if(releasedAction.first == RECV_PARTICLES){
+ TIMEZONE("post-recv-particles");
+ NeighborDescriptor& descriptor = neigDescriptors[releasedAction.second];
+ assert(descriptor.isRecv);
+ const int destProc = descriptor.destProc;
+ const partsize_t NbParticlesToReceive = descriptor.nbParticlesToExchange;
+ assert(NbParticlesToReceive != -1);
+ assert(descriptor.toCompute == nullptr);
+ assert(descriptor.indexes == nullptr);
+ if (labels_present)
+ assert(descriptor.labels == nullptr);
+
+ if(NbParticlesToReceive){
+ descriptor.toCompute.reset(new real_number[NbParticlesToReceive*size_particle_positions]);
+ whatNext.emplace_back(std::pair<Action,int>{COMPUTE_PARTICLES, releasedAction.second});
+ mpiRequests.emplace_back();
+ assert(NbParticlesToReceive*size_particle_positions < std::numeric_limits<int>::max());
+ AssertMpi(MPI_Irecv(
+ descriptor.toCompute.get(),
+ int(NbParticlesToReceive*size_particle_positions),
+ particles_utils::GetMpiType(real_number()),
+ destProc,
+ TAG_POSITION_PARTICLES,
+ current_com,
+ &mpiRequests.back()));
+
+ descriptor.indexes.reset(new partsize_t[NbParticlesToReceive]);
+ whatNext.emplace_back(std::pair<Action,int>{COMPUTE_PARTICLES, releasedAction.second});
+ mpiRequests.emplace_back();
+ assert(NbParticlesToReceive*size_particle_positions < std::numeric_limits<int>::max());
+ AssertMpi(MPI_Irecv(
+ descriptor.indexes.get(),
+ int(NbParticlesToReceive),
+ particles_utils::GetMpiType(partsize_t()),
+ destProc,
+ TAG_INDEX_PARTICLES,
+ current_com,
+ &mpiRequests.back()));
+
+ if (labels_present)
+ {
+ descriptor.labels.reset(new partsize_t[NbParticlesToReceive]);
+ whatNext.emplace_back(std::pair<Action,int>{COMPUTE_PARTICLES, releasedAction.second});
+ mpiRequests.emplace_back();
+ assert(NbParticlesToReceive*size_particle_positions < std::numeric_limits<int>::max());
+ AssertMpi(MPI_Irecv(
+ descriptor.labels.get(),
+ int(NbParticlesToReceive),
+ particles_utils::GetMpiType(partsize_t()),
+ destProc,
+ TAG_LABEL_PARTICLES,
+ current_com,
+ &mpiRequests.back()));
+ }
+ }
+ }
+
+ //////////////////////////////////////////////////////////////////////
+ /// Computation
+ //////////////////////////////////////////////////////////////////////
+ if(releasedAction.first == COMPUTE_PARTICLES){
+ NeighborDescriptor& descriptor = neigDescriptors[releasedAction.second];
+ descriptor.nbReceived += 1;
+
+ int nbReceived_desired_value = 2;
+ if (labels_present)
+ nbReceived_desired_value = 3;
+
+ if(descriptor.nbReceived == nbReceived_desired_value){
+ TIMEZONE("compute-particles");
+ NeighborDescriptor& descriptor = neigDescriptors[releasedAction.second];
+ assert(descriptor.isRecv);
+ const partsize_t NbParticlesToReceive = descriptor.nbParticlesToExchange;
+
+ assert(descriptor.toCompute != nullptr);
+ assert(descriptor.indexes != nullptr);
+ if (labels_present)
+ assert(descriptor.labels != nullptr);
+ // allocate rhs buffer
+ descriptor.results.reset(new real_number[NbParticlesToReceive*size_particle_rhs]);
+ // clean up rhs buffer
+ set_particle_data_to_zero<partsize_t, real_number, size_particle_rhs>(
+ descriptor.results.get(),
+ NbParticlesToReceive);
+
+ // Compute
+ partsize_t idxPart = 0;
+ while(idxPart != NbParticlesToReceive){
+ const long int current_cell_idx = get_cell_idx(descriptor.toCompute[idxPart*size_particle_positions + IDXC_X],
+ descriptor.toCompute[idxPart*size_particle_positions + IDXC_Y],
+ descriptor.toCompute[idxPart*size_particle_positions + IDXC_Z]);
+ partsize_t nb_parts_in_cell = 1;
+ while(idxPart+nb_parts_in_cell != NbParticlesToReceive
+ && current_cell_idx == get_cell_idx(descriptor.toCompute[(idxPart+nb_parts_in_cell)*size_particle_positions + IDXC_X],
+ descriptor.toCompute[(idxPart+nb_parts_in_cell)*size_particle_positions + IDXC_Y],
+ descriptor.toCompute[(idxPart+nb_parts_in_cell)*size_particle_positions + IDXC_Z])){
+ nb_parts_in_cell += 1;
+ }
+
+ #pragma omp task default(shared) firstprivate(idxPart, nb_parts_in_cell, current_cell_idx)
+ {
+ computer_class& computer_thread_task = (omp_get_thread_num() == 0 ? in_computer : *computer_for_all_threads[omp_get_thread_num()-1]);
+ const std::vector<std::pair<partsize_t,partsize_t>>* neighbors[27];
+ long int neighbors_indexes[27];
+ std::array<real_number,3> shift[27];
+ real_number diff_x, diff_y, diff_z;
+ const int nbNeighbors = my_tree.getNeighbors(
+ current_cell_idx,
+ neighbors,
+ neighbors_indexes,
+ shift,
+ true);
+
+ // with other interval
+ for(int idx_neighbor = 0 ; idx_neighbor < nbNeighbors ; ++idx_neighbor){
+ cells_locker.lock(neighbors_indexes[idx_neighbor]);
+
+ for(size_t idx_2 = 0 ; idx_2 < (*neighbors[idx_neighbor]).size() ; ++idx_2){
+ for(partsize_t idx_p1 = 0 ; idx_p1 < nb_parts_in_cell ; ++idx_p1){
+ for(partsize_t idx_p2 = 0 ; idx_p2 < (*neighbors[idx_neighbor])[idx_2].second ; ++idx_p2){
+ const real_number dist_r2 = compute_distance_r2(
+ descriptor.toCompute[(idxPart+idx_p1)*size_particle_positions + IDXC_X],
+ descriptor.toCompute[(idxPart+idx_p1)*size_particle_positions + IDXC_Y],
+ descriptor.toCompute[(idxPart+idx_p1)*size_particle_positions + IDXC_Z],
+ particles_positions[((*neighbors[idx_neighbor])[idx_2].first+idx_p2)*size_particle_positions + IDXC_X],
+ particles_positions[((*neighbors[idx_neighbor])[idx_2].first+idx_p2)*size_particle_positions + IDXC_Y],
+ particles_positions[((*neighbors[idx_neighbor])[idx_2].first+idx_p2)*size_particle_positions + IDXC_Z],
+ shift[idx_neighbor][IDXC_X],
+ shift[idx_neighbor][IDXC_Y],
+ shift[idx_neighbor][IDXC_Z],
+ diff_x,
+ diff_y,
+ diff_z);
+ if(dist_r2 < cutoff_radius_compute*cutoff_radius_compute){
+ computer_thread_task.template compute_interaction<size_particle_positions, size_particle_rhs>(
+ descriptor.indexes[(idxPart+idx_p1)],
+ &descriptor.toCompute[(idxPart+idx_p1)*size_particle_positions],
+ &descriptor.results[(idxPart+idx_p1)*size_particle_rhs],
+ inout_index_particles[((*neighbors[idx_neighbor])[idx_2].first+idx_p2)],
+ &particles_positions[((*neighbors[idx_neighbor])[idx_2].first+idx_p2)*size_particle_positions],
+ &particles_current_rhs[0][((*neighbors[idx_neighbor])[idx_2].first+idx_p2)*size_particle_rhs],
+ dist_r2,
+ cutoff_radius_compute,
+ diff_x,
+ diff_y,
+ diff_z);
+ }
+ }
+ }
+ }
+
+ cells_locker.unlock(neighbors_indexes[idx_neighbor]);
+ }
+ }
+
+ idxPart += nb_parts_in_cell;
+ }
+
+ #pragma omp taskwait
+
+ // Send back
+ const int destProc = descriptor.destProc;
+ whatNext.emplace_back(std::pair<Action,int>{RELEASE_BUFFER_PARTICLES, releasedAction.second});
+ mpiRequests.emplace_back();
+ assert(NbParticlesToReceive*size_particle_rhs < std::numeric_limits<int>::max());
+ AssertMpi(MPI_Isend(
+ descriptor.results.get(),
+ int(NbParticlesToReceive*size_particle_rhs),
+ particles_utils::GetMpiType(real_number()),
+ destProc,
+ TAG_RESULT_PARTICLES,
+ current_com,
+ &mpiRequests.back()));
+ delete[] descriptor.toCompute.release();
+ delete[] descriptor.indexes.release();
+ delete[] descriptor.labels.release();
+ }
+ }
+ //////////////////////////////////////////////////////////////////////
+ /// Release memory that was sent back
+ //////////////////////////////////////////////////////////////////////
+ if(releasedAction.first == RELEASE_BUFFER_PARTICLES){
+ NeighborDescriptor& descriptor = neigDescriptors[releasedAction.second];
+ assert(descriptor.results != nullptr);
+ assert(descriptor.isRecv);
+ delete[] descriptor.results.release();
+ }
+ //////////////////////////////////////////////////////////////////////
+ /// Merge
+ //////////////////////////////////////////////////////////////////////
+ if(releasedAction.first == MERGE_PARTICLES){
+ TIMEZONE("merge");
+ NeighborDescriptor& descriptor = neigDescriptors[releasedAction.second];
+ assert(descriptor.isRecv == false);
+ assert(descriptor.toRecvAndMerge != nullptr);
+ computer_thread.template reduce_particles_rhs<size_particle_rhs>(
+ &particles_current_rhs[0][particles_offset_layers[my_nb_cell_levels-descriptor.nbLevelsToExchange]*size_particle_rhs],
+ descriptor.toRecvAndMerge.get(),
+ descriptor.nbParticlesToExchange);
+ delete[] descriptor.toRecvAndMerge.release();
+ }
+ }
+ }
+ }
+
+ assert(whatNext.size() == 0);
+ assert(mpiRequests.size() == 0);
+
+ {
+ computer_class& computer_thread = (omp_get_thread_num() == 0 ? in_computer : *computer_for_all_threads[omp_get_thread_num()-1]);
+ // Compute self data
+ for(const auto& iter_cell : my_tree){
+ TIMEZONE("proceed-leaf");
+ const long int currenct_cell_idx = iter_cell.first;
+ const std::vector<std::pair<partsize_t,partsize_t>>* intervals_ptr = &iter_cell.second;
+
+ #pragma omp task default(shared) firstprivate(currenct_cell_idx, intervals_ptr)
+ {
+ const std::vector<std::pair<partsize_t,partsize_t>>& intervals = (*intervals_ptr);
+
+ cells_locker.lock(currenct_cell_idx);
+
+ for(size_t idx_1 = 0 ; idx_1 < intervals.size() ; ++idx_1){
+ // self interval
+ for(partsize_t idx_p1 = 0 ; idx_p1 < intervals[idx_1].second ; ++idx_p1){
+ for(partsize_t idx_p2 = idx_p1+1 ; idx_p2 < intervals[idx_1].second ; ++idx_p2){
+ real_number diff_x, diff_y, diff_z;
+ const real_number dist_r2 = compute_distance_r2(particles_positions[(intervals[idx_1].first+idx_p1)*size_particle_positions + IDXC_X],
+ particles_positions[(intervals[idx_1].first+idx_p1)*size_particle_positions + IDXC_Y],
+ particles_positions[(intervals[idx_1].first+idx_p1)*size_particle_positions + IDXC_Z],
+ particles_positions[(intervals[idx_1].first+idx_p2)*size_particle_positions + IDXC_X],
+ particles_positions[(intervals[idx_1].first+idx_p2)*size_particle_positions + IDXC_Y],
+ particles_positions[(intervals[idx_1].first+idx_p2)*size_particle_positions + IDXC_Z],
+ 0, 0, 0,
+ diff_x, diff_y, diff_z);
+ if(dist_r2 < cutoff_radius_compute*cutoff_radius_compute){
+ computer_thread.template compute_interaction<size_particle_positions,size_particle_rhs>(
+ inout_index_particles[(intervals[idx_1].first+idx_p1)],
+ &particles_positions[(intervals[idx_1].first+idx_p1)*size_particle_positions],
+ &particles_current_rhs[0][(intervals[idx_1].first+idx_p1)*size_particle_rhs],
+ inout_index_particles[(intervals[idx_1].first+idx_p2)],
+ &particles_positions[(intervals[idx_1].first+idx_p2)*size_particle_positions],
+ &particles_current_rhs[0][(intervals[idx_1].first+idx_p2)*size_particle_rhs],
+ dist_r2, cutoff_radius_compute, diff_x, diff_y, diff_z);
+ }
+ }
+ }
+
+ // with other interval
+ for(size_t idx_2 = idx_1+1 ; idx_2 < intervals.size() ; ++idx_2){
+ for(partsize_t idx_p1 = 0 ; idx_p1 < intervals[idx_1].second ; ++idx_p1){
+ for(partsize_t idx_p2 = 0 ; idx_p2 < intervals[idx_2].second ; ++idx_p2){
+ real_number diff_x, diff_y, diff_z;
+ const real_number dist_r2 = compute_distance_r2(particles_positions[(intervals[idx_1].first+idx_p1)*size_particle_positions + IDXC_X],
+ particles_positions[(intervals[idx_1].first+idx_p1)*size_particle_positions + IDXC_Y],
+ particles_positions[(intervals[idx_1].first+idx_p1)*size_particle_positions + IDXC_Z],
+ particles_positions[(intervals[idx_2].first+idx_p2)*size_particle_positions + IDXC_X],
+ particles_positions[(intervals[idx_2].first+idx_p2)*size_particle_positions + IDXC_Y],
+ particles_positions[(intervals[idx_2].first+idx_p2)*size_particle_positions + IDXC_Z],
+ 0, 0, 0,
+ diff_x, diff_y, diff_z);
+ if(dist_r2 < cutoff_radius_compute*cutoff_radius_compute){
+ computer_thread.template compute_interaction<size_particle_positions,size_particle_rhs>(
+ inout_index_particles[(intervals[idx_1].first+idx_p1)],
+ &particles_positions[(intervals[idx_1].first+idx_p1)*size_particle_positions],
+ &particles_current_rhs[0][(intervals[idx_1].first+idx_p1)*size_particle_rhs],
+ inout_index_particles[(intervals[idx_2].first+idx_p2)],
+ &particles_positions[(intervals[idx_2].first+idx_p2)*size_particle_positions],
+ &particles_current_rhs[0][(intervals[idx_2].first+idx_p2)*size_particle_rhs],
+ dist_r2, cutoff_radius_compute, diff_x, diff_y, diff_z);
+ }
+ }
+ }
+ }
+ }
+
+ const std::vector<std::pair<partsize_t,partsize_t>>* neighbors[27];
+ long int neighbors_indexes[27];
+ std::array<real_number,3> shift[27];
+ const int nbNeighbors = my_tree.getNeighbors(currenct_cell_idx, neighbors, neighbors_indexes, shift, false);
+
+ for(size_t idx_1 = 0 ; idx_1 < intervals.size() ; ++idx_1){
+ // with other interval
+ for(int idx_neighbor = 0 ; idx_neighbor < nbNeighbors ; ++idx_neighbor){
+ if(currenct_cell_idx < neighbors_indexes[idx_neighbor]){
+ cells_locker.lock(neighbors_indexes[idx_neighbor]);
+
+ for(size_t idx_2 = 0 ; idx_2 < (*neighbors[idx_neighbor]).size() ; ++idx_2){
+ for(partsize_t idx_p1 = 0 ; idx_p1 < intervals[idx_1].second ; ++idx_p1){
+ for(partsize_t idx_p2 = 0 ; idx_p2 < (*neighbors[idx_neighbor])[idx_2].second ; ++idx_p2){
+ real_number diff_x, diff_y, diff_z;
+ const real_number dist_r2 = compute_distance_r2(particles_positions[(intervals[idx_1].first+idx_p1)*size_particle_positions + IDXC_X],
+ particles_positions[(intervals[idx_1].first+idx_p1)*size_particle_positions + IDXC_Y],
+ particles_positions[(intervals[idx_1].first+idx_p1)*size_particle_positions + IDXC_Z],
+ particles_positions[((*neighbors[idx_neighbor])[idx_2].first+idx_p2)*size_particle_positions + IDXC_X],
+ particles_positions[((*neighbors[idx_neighbor])[idx_2].first+idx_p2)*size_particle_positions + IDXC_Y],
+ particles_positions[((*neighbors[idx_neighbor])[idx_2].first+idx_p2)*size_particle_positions + IDXC_Z],
+ shift[idx_neighbor][IDXC_X], shift[idx_neighbor][IDXC_Y], shift[idx_neighbor][IDXC_Z],
+ diff_x, diff_y, diff_z);
+ if(dist_r2 < cutoff_radius_compute*cutoff_radius_compute){
+ computer_thread.template compute_interaction<size_particle_positions,size_particle_rhs>(
+ inout_index_particles[(intervals[idx_1].first+idx_p1)],
+ &particles_positions[(intervals[idx_1].first+idx_p1)*size_particle_positions],
+ &particles_current_rhs[0][(intervals[idx_1].first+idx_p1)*size_particle_rhs],
+ inout_index_particles[((*neighbors[idx_neighbor])[idx_2].first+idx_p2)],
+ &particles_positions[((*neighbors[idx_neighbor])[idx_2].first+idx_p2)*size_particle_positions],
+ &particles_current_rhs[0][((*neighbors[idx_neighbor])[idx_2].first+idx_p2)*size_particle_rhs],
+ dist_r2, cutoff_radius_compute,
+ diff_x, diff_y, diff_z);
+ }
+ }
+ }
+ }
+ cells_locker.unlock(neighbors_indexes[idx_neighbor]);
+ }
+ }
+ }
+
+ cells_locker.unlock(currenct_cell_idx);
+ }
+ }
+ }
+
+ for(int idxThread = 1 ; idxThread < omp_get_num_threads() ; ++idxThread){
+ in_computer.merge(*computer_for_all_threads[idxThread-1]);
+ }
+ }
+};
+
+#endif
diff --git a/cpp/particles/p2p/p2p_ghost_collisions.hpp b/cpp/particles/p2p/p2p_ghost_collisions.hpp
new file mode 100644
index 0000000000000000000000000000000000000000..36a17d3de920f33c60577c376ab907412ac5acce
--- /dev/null
+++ b/cpp/particles/p2p/p2p_ghost_collisions.hpp
@@ -0,0 +1,449 @@
+/******************************************************************************
+* *
+* Copyright 2019 Max Planck Institute for Dynamics and Self-Organization *
+* *
+* This file is part of bfps. *
+* *
+* bfps is free software: you can redistribute it and/or modify *
+* it under the terms of the GNU General Public License as published *
+* by the Free Software Foundation, either version 3 of the License, *
+* or (at your option) any later version. *
+* *
+* bfps is distributed in the hope that it will be useful, *
+* but WITHOUT ANY WARRANTY; without even the implied warranty of *
+* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the *
+* GNU General Public License for more details. *
+* *
+* You should have received a copy of the GNU General Public License *
+* along with bfps. If not, see <http://www.gnu.org/licenses/> *
+* *
+* Contact: Cristian.Lalescu@ds.mpg.de *
+* *
+******************************************************************************/
+#ifndef P2P_GHOST_COLLISIONS_HPP
+#define P2P_GHOST_COLLISIONS_HPP
+
+#include <cstring>
+#include <utility>
+#include <vector>
+#include <cmath>
+
+
+template <class partsize_t>
+void print_pair_vec(std::vector<partsize_t> vec)
+{
+ assert(vec.size() % 2 == 0);
+ for (int i = 0; i < int(vec.size())/2; i++)
+ DEBUG_MSG("in print_pair_vec, pair %d is (%ld,%ld)\n", i, vec[2*i], vec[2*i+1]);
+}
+
+
+template <class real_number, class partsize_t>
+class p2p_ghost_collisions
+{
+ protected:
+ const double pi = atan(1.0)*4;
+ const double pi2 = atan(1.0)*8;
+
+ enum particle_shape {CYLINDER, SPHERE, DISK};
+
+ private:
+ bool isActive;
+
+ bool synchronisation;
+
+ // description for cylinders:
+ double cylinder_width;
+ double cylinder_length;
+
+ // description for disks:
+ double disk_width;
+
+ // depending on the particle shape, we will be deciding whether particles intersect in different ways
+ particle_shape current_particle_shape;
+
+ protected:
+ /** \brief Adds pair of colliding particle to list
+ *
+ * Given a pair of particle IDs, add them as an *ordered pair* to `collision_pairs_local`.
+ *
+ */
+ void add_colliding_pair(partsize_t idx_part1, partsize_t idx_part2)
+ {
+ // store colliding particle ids in order, to be able to identify pairs more easily
+ assert(idx_part1!=idx_part2);
+ partsize_t idx_part_small = idx_part1;
+ partsize_t idx_part_big = idx_part2;
+ if (idx_part1 > idx_part2)
+ {
+ idx_part_small = idx_part2;
+ idx_part_big = idx_part1;
+ }
+ this->collision_pairs_local.push_back(idx_part_small);
+ this->collision_pairs_local.push_back(idx_part_big);
+ }
+
+ std::vector <partsize_t> collision_pairs_local;
+ std::vector <partsize_t> collision_pairs_global;
+
+public:
+ p2p_ghost_collisions():
+ isActive(true),
+ synchronisation(false),
+ cylinder_width(1.0),
+ cylinder_length(1.0),
+ disk_width(1.0),
+ current_particle_shape(SPHERE) {}
+
+
+ void copy_from_p2p_ghost_collisions(const p2p_ghost_collisions<real_number, partsize_t>& src)
+ {
+ this->current_particle_shape = src.current_particle_shape;
+ this->cylinder_width = src.cylinder_width;
+ this->cylinder_length = src.cylinder_length;
+ this->disk_width = src.disk_width;
+ this->isActive = src.isActive;
+ this->synchronisation = src.synchronisation;
+ this->collision_pairs_local.reserve(src.collision_pairs_local.capacity());
+ }
+
+ // Copy constructor use a counter set to zero
+ p2p_ghost_collisions(const p2p_ghost_collisions<real_number, partsize_t>& src)
+ {
+ this->copy_from_p2p_ghost_collisions(src);
+ }
+
+ double rod_distance(double dist_pow2, double xp, double xq, double pq, double t, double s){
+ double min_distance;
+ min_distance = dist_pow2 + 0.25*this->get_cylinder_length()*this->get_cylinder_length() * ( t*t + s*s - 2.0*t*s*pq ) + this->get_cylinder_length() * ( t*xp - s*xq );
+ min_distance = sqrt( min_distance );
+ return min_distance;
+ }
+
+ template <int size_particle_rhs>
+ void reduce_particles_rhs(real_number /*rhs_dst*/[], const real_number /*rhs_src*/[], const partsize_t /*nbParticles*/) const{
+ }
+
+ void merge(const p2p_ghost_collisions& other){
+ //std::vector <std::array <partsize_t, 2>> new_collision_pairs;
+ //std::set_union(collision_pairs_local.begin(), collision_pairs_local.end(),
+ // other.collision_pairs_local.begin(), other.collision_pairs_local.end(),
+ // std::inserter(new_collision_pairs, new_collision_pairs.begin()));
+ //collision_pairs_local = new_collision_pairs;
+ collision_pairs_local.insert(
+ collision_pairs_local.end(),
+ other.collision_pairs_local.begin(),
+ other.collision_pairs_local.end());
+ }
+
+ void MPI_merge(MPI_Comm comm, int myrank, int nprocs) {
+ ///collect collision pairs
+ DEBUG_MSG("p2p_ghost_collisions::MPI_merge\n");
+ int * recvcounts = NULL;
+ int vec_len_local = collision_pairs_local.size();
+ int * displ = NULL;
+ if (myrank == 0)
+ {
+ recvcounts = new int[nprocs];
+ }
+ MPI_Gather(&vec_len_local, 1, MPI_INT,
+ recvcounts, 1, MPI_INT,
+ 0, comm);
+ DEBUG_MSG("after gather\n");
+ if (myrank == 0)
+ {
+ int nbElements;
+ displ = new int[nprocs];
+ displ[0]=0;
+ for (int i=1; i<nprocs; i++)
+ {
+ displ[i]=displ[i-1]+recvcounts[i-1];
+ }
+ nbElements = displ[nprocs-1] + recvcounts[nprocs-1];
+ this->collision_pairs_global.resize(nbElements);
+ DEBUG_MSG(("nbElements: "+std::to_string(nbElements)+"\n").c_str());
+ }
+ MPI_Gatherv(&collision_pairs_local.front(),
+ vec_len_local,
+ MPI_LONG_LONG_INT,
+ &this->collision_pairs_global.front(),
+ recvcounts,
+ displ,
+ MPI_LONG_LONG_INT,
+ 0,
+ comm);
+ if(myrank == 0)
+ {
+ print_pair_vec(this->collision_pairs_global);
+ }
+ this->synchronisation = true;
+ // free root rank memory
+ if (myrank == 0)
+ {
+ delete[] recvcounts;
+ delete[] displ;
+ }
+ }
+
+ //get_collision_counter() will only give the correct result on rank 0
+ long int get_collision_counter(MPI_Comm comm, int myrank, int nprocs) {
+ if(synchronisation==false)
+ {
+ MPI_merge(comm, myrank, nprocs);
+ }
+ return this->collision_pairs_global.size();
+ }
+
+ //get_collision_pairs() will only give global pairs on rank 0
+ std::vector <partsize_t> get_collision_pairs(MPI_Comm comm, int myrank, int nprocs) {
+ if(synchronisation==false)
+ {
+ MPI_merge(comm, myrank, nprocs);
+ }
+ return this->collision_pairs_global;
+ }
+
+
+ void reset_collision_pairs(){
+ this->collision_pairs_local.resize(0);
+ this->collision_pairs_global.resize(0);
+ this->synchronisation = false;
+ }
+
+
+ /**
+ * NOTE: this is called only ONCE for each pair of interacting particles.
+ */
+ template <int size_particle_positions, int size_particle_rhs>
+ void compute_interaction(const partsize_t idx_part1,
+ const real_number pos_part1[],
+ real_number /*rhs_part1*/[],
+ const partsize_t idx_part2,
+ const real_number pos_part2[],
+ real_number /*rhs_part2*/[],
+ const real_number dist_pow2,
+ const real_number /*cutoff*/,
+ const real_number xseparation, /* This separation is x1-x2 */
+ const real_number yseparation,
+ const real_number zseparation){
+ switch(this->current_particle_shape)
+ {
+ case SPHERE:
+ {
+ this->add_colliding_pair(idx_part1, idx_part2);
+ }
+ break;
+ case CYLINDER:
+ {
+ double pq, xp, xq, t, s, min_distance, min_distance_current;
+ double px,py, pz, qx, qy, qz;
+
+ const double x = xseparation;
+ const double y = yseparation;
+ const double z = zseparation;
+ /* const double r = sqrt(dist_pow2); This variable is not needed. */
+
+ /* p and q are the orientation vectors of the first and second particles. */
+ px = pos_part1[IDXC_X+3];
+ py = pos_part1[IDXC_Y+3];
+ pz = pos_part1[IDXC_Z+3];
+ qx = pos_part2[IDXC_X+3];
+ qy = pos_part2[IDXC_Y+3];
+ qz = pos_part2[IDXC_Z+3];
+ /* pq, xp, xq are scalar products of these vectors with x, relative position */
+ pq = px * qx + py * qy + pz * qz;
+ xp = x * px + y * py + z * pz;
+ xq = x * qx + y * qy + z * qz;
+ /* t and s parametrize the two rods. Find min distance: */
+ if( pq == 1.0 ){
+ min_distance = sqrt(dist_pow2-xp*xp);
+ }
+ else{
+ t = 2.0/(this->get_cylinder_length()*(pq*pq-1.0))*(xp-pq*xq);
+ s = 2.0/(this->get_cylinder_length()*(pq*pq-1.0))*(pq*xp-xq);
+ /* Test if -1<s<1 and -1<t<1 */
+ if( fabs(t)<=1.0 and fabs(s)<=1.0 )
+ {
+ /* Get minimal distance in case of both t and s in {-1,1}. Else: check edges */
+ min_distance = this->rod_distance(dist_pow2,xp,xq,pq,t,s);
+ }
+ else
+ {
+ /* t fixed at 1, find min along s */
+ t = 1.0;
+ s = t*pq+2.0/this->get_cylinder_length()*xq;
+ if( fabs(s)>1.0 ) { s = s / fabs(s) ;}
+ min_distance_current = this->rod_distance(dist_pow2,xp,xq,pq,t,s);
+ min_distance = min_distance_current;
+ /* t fixed at -1, find min along s */
+ t = -1.0;
+ s = t*pq+2.0/this->get_cylinder_length()*xq;
+ if( fabs(s)>1.0 ) { s = s / fabs(s) ;}
+ min_distance_current = this->rod_distance(dist_pow2,xp,xq,pq,t,s);
+ min_distance = fmin( min_distance_current, min_distance );
+ /* s fixed at 1, find min along t */
+ s = 1.0;
+ t = s*pq-2.0/this->get_cylinder_length()*xp;
+ if( fabs(t)>1.0 ) { t = t / fabs(t) ;}
+ min_distance_current = this->rod_distance(dist_pow2,xp,xq,pq,t,s);
+ min_distance = fmin( min_distance_current, min_distance );
+ /* s fixed at -1, find min along t */
+ s = -1.0;
+ t = s*pq-2.0/this->get_cylinder_length()*xp;
+ if( fabs(t)>1.0 ) { t = t / fabs(t) ;}
+ min_distance_current = this->rod_distance(dist_pow2,xp,xq,pq,t,s);
+ min_distance = fmin( min_distance_current, min_distance );
+ }
+ }
+ /* If cylinders overlap count it as a collision */
+ if( min_distance<=this->get_cylinder_width() ){
+ this->add_colliding_pair(idx_part1, idx_part2);
+ }
+ }
+ break;
+ case DISK:
+ {
+ double p1p2, x1p1, x2p2, x0_p2_comp, p0_norm, x1_x0_2, x2_x0_2, x1p0, x2p0, det;
+ std::array<double, 3> x0, x1, x2, p0, p1, p2;
+ std::array<double, 2> t1, t2;
+ /* Particle 2 is in the origin. Use xseperation etc for particle 1. */
+ /* Particle 1 data. */
+ x1[IDXC_X] = xseparation;
+ x1[IDXC_Y] = yseparation;
+ x1[IDXC_Z] = zseparation;
+ p1[IDXC_X] = pos_part1[IDXC_X+3];
+ p1[IDXC_Y] = pos_part1[IDXC_Y+3];
+ p1[IDXC_Z] = pos_part1[IDXC_Z+3];
+ /* Particle 2 data. */
+ x2[IDXC_X] = 0.0;
+ x2[IDXC_Y] = 0.0;
+ x2[IDXC_Z] = 0.0;
+ p2[IDXC_X] = pos_part2[IDXC_X+3];
+ p2[IDXC_Y] = pos_part2[IDXC_Y+3];
+ p2[IDXC_Z] = pos_part2[IDXC_Z+3];
+ /* Exit if the disk planes parallel. */
+ p1p2 = p1[IDXC_X]*p2[IDXC_X] + p1[IDXC_Y]*p2[IDXC_Y] + p1[IDXC_Z]*p2[IDXC_Z];
+ if ( p1p2 == 1.0 ) {
+ return;
+ }
+ x1p1 = x1[IDXC_X]*p1[IDXC_X] + x1[IDXC_Y]*p1[IDXC_Y] + x1[IDXC_Z]*p1[IDXC_Z];
+ x2p2 = x2[IDXC_X]*p2[IDXC_X] + x2[IDXC_Y]*p2[IDXC_Y] + x2[IDXC_Z]*p2[IDXC_Z];
+ x0_p2_comp = (x2p2-p1p2*x1p1)/(1.0-p1p2*p1p2);
+ /* Get x0 and p0. These define the intersection of the two planes of the disks as x0+p0*t */
+ /* Get x0. */
+ x0[IDXC_X] = x1p1*p1[IDXC_X] + x0_p2_comp*( p2[IDXC_X]-p1p2*p1[IDXC_X] );
+ x0[IDXC_Y] = x1p1*p1[IDXC_Y] + x0_p2_comp*( p2[IDXC_Y]-p1p2*p1[IDXC_Y] );
+ x0[IDXC_Z] = x1p1*p1[IDXC_Z] + x0_p2_comp*( p2[IDXC_Z]-p1p2*p1[IDXC_Z] );
+ /* Get p0. */
+ p0[IDXC_X] = p1[IDXC_Y]*p2[IDXC_Z] - p1[IDXC_Z]*p2[IDXC_Y];
+ p0[IDXC_Y] = p1[IDXC_Z]*p2[IDXC_X] - p1[IDXC_X]*p2[IDXC_Z];
+ p0[IDXC_Z] = p1[IDXC_X]*p2[IDXC_Y] - p1[IDXC_Y]*p2[IDXC_X];
+ p0_norm = sqrt(p0[IDXC_X]*p0[IDXC_X]+p0[IDXC_Y]*p0[IDXC_Y]+p0[IDXC_Z]*p0[IDXC_Z]);
+ p0[IDXC_X] = p0[IDXC_X] / p0_norm;
+ p0[IDXC_Y] = p0[IDXC_Y] / p0_norm;
+ p0[IDXC_Z] = p0[IDXC_Z] / p0_norm;
+
+ /* Scalar products needed to solve for minimal distance in t. */
+ x1_x0_2 = (x1[IDXC_X]-x0[IDXC_X])*(x1[IDXC_X]-x0[IDXC_X]) + (x1[IDXC_Y]-x0[IDXC_Y])*(x1[IDXC_Y]-x0[IDXC_Y]) + (x1[IDXC_Z]-x0[IDXC_Z])*(x1[IDXC_Z]-x0[IDXC_Z]);
+ x2_x0_2 = (x2[IDXC_X]-x0[IDXC_X])*(x2[IDXC_X]-x0[IDXC_X]) + (x2[IDXC_Y]-x0[IDXC_Y])*(x2[IDXC_Y]-x0[IDXC_Y]) + (x2[IDXC_Z]-x0[IDXC_Z])*(x2[IDXC_Z]-x0[IDXC_Z]);
+ x1p0 = x1[IDXC_X]*p0[IDXC_X] + x1[IDXC_Y]*p0[IDXC_Y] + x1[IDXC_Z]*p0[IDXC_Z];
+ x2p0 = x2[IDXC_X]*p0[IDXC_X] + x2[IDXC_Y]*p0[IDXC_Y] + x2[IDXC_Z]*p0[IDXC_Z];
+
+ /*Check for collision. Check if segments of disk along common line between planes p1 and p2 intersect. */
+ det = x1p0 * x1p0 + 0.25 * this->get_disk_width() * this->get_disk_width() - x1_x0_2;
+ if (det == 0.0){
+ t1[0] = x1p0;
+ t1[1] = x1p0;
+ } else if (det > 0.0) {
+ t1[0] = x1p0 - sqrt(det);
+ t1[1] = x1p0 + sqrt(det);
+ } else {return;}
+
+ det = x2p0 * x2p0 + 0.25 * this->get_disk_width() * this->get_disk_width() - x2_x0_2;
+ if (det == 0.0){
+ t2[0] = x2p0;
+ t2[1] = x2p0;
+ } else if (det > 0.0) {
+ t2[0] = x2p0 - sqrt(det);
+ t2[1] = x2p0 + sqrt(det);
+ } else {return;}
+
+ if (( t1[1]>=t2[0] ) and
+ ( t2[1]>=t1[0] ))
+ {
+ this->add_colliding_pair(idx_part1, idx_part2);
+ return;
+ }
+ }
+ break;
+ }
+ }
+
+ void set_sphere()
+ {
+ this->current_particle_shape = SPHERE;
+ }
+
+ void set_cylinder()
+ {
+ this->current_particle_shape = CYLINDER;
+ }
+
+ void set_cylinder_width(const double WIDTH)
+ {
+ this->cylinder_width = WIDTH;
+ }
+
+ void set_cylinder_length(const double LENGTH)
+ {
+ this->cylinder_length = LENGTH;
+ }
+
+ double get_cylinder_width() const
+ {
+ return this->cylinder_width;
+ }
+
+ double get_cylinder_length() const
+ {
+ return this->cylinder_length;
+ }
+
+ void set_disk()
+ {
+ this->current_particle_shape = DISK;
+ }
+
+ particle_shape get_current_particle_shape() const
+ {
+ return this->current_particle_shape;
+ }
+
+ void set_disk_width(const double WIDTH)
+ {
+ this->disk_width = WIDTH;
+ }
+
+ double get_disk_width() const
+ {
+ return this->disk_width;
+ }
+
+ bool isEnable() const {
+ return isActive;
+ }
+
+ void setEnable(const bool inIsActive)
+ {
+ isActive = inIsActive;
+ }
+
+ bool isSynchronized() const
+ {
+ return this->synchronisation;
+ }
+};
+
+
+#endif // P2P_GHOST_COLLISIONS_HPP
+
diff --git a/cpp/particles/p2p/p2p_merge_collisions.hpp b/cpp/particles/p2p/p2p_merge_collisions.hpp
new file mode 100644
index 0000000000000000000000000000000000000000..4b728c8f3c3db1c26e41947fe1a23325fbe688e8
--- /dev/null
+++ b/cpp/particles/p2p/p2p_merge_collisions.hpp
@@ -0,0 +1,77 @@
+/******************************************************************************
+* *
+* Copyright 2019 Max Planck Institute for Dynamics and Self-Organization *
+* *
+* This file is part of bfps. *
+* *
+* bfps is free software: you can redistribute it and/or modify *
+* it under the terms of the GNU General Public License as published *
+* by the Free Software Foundation, either version 3 of the License, *
+* or (at your option) any later version. *
+* *
+* bfps is distributed in the hope that it will be useful, *
+* but WITHOUT ANY WARRANTY; without even the implied warranty of *
+* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the *
+* GNU General Public License for more details. *
+* *
+* You should have received a copy of the GNU General Public License *
+* along with bfps. If not, see <http://www.gnu.org/licenses/> *
+* *
+* Contact: Cristian.Lalescu@ds.mpg.de *
+* *
+******************************************************************************/
+#ifndef P2P_MERGE_COLLISIONS_HPP
+#define P2P_MERGE_COLLISIONS_HPP
+
+#include <cstring>
+
+template <class real_number, class partsize_t>
+class p2p_merge_collisions{
+ long int collision_counter;
+
+ std::vector<partsize_t> mergedParticles;
+
+public:
+ p2p_merge_collisions() {}
+
+ // Copy constructor use a counter set to zero
+ p2p_merge_collisions(const p2p_merge_collisions&){}
+
+ template <int size_particle_rhs>
+ void reduce_particles_rhs(real_number /*rhs_dst*/[], const real_number /*rhs_src*/[], const partsize_t /*nbParticles*/) const{
+ }
+
+ template <int size_particle_positions, int size_particle_rhs>
+ void compute_interaction(const partsize_t idx_part1,
+ const real_number /*pos_part1*/[],
+ real_number /*rhs_part1*/[],
+ const partsize_t idx_part2,
+ const real_number /*pos_part2*/[],
+ real_number /*rhs_part2*/[],
+ const real_number /*dist_pow2*/,
+ const real_number /*cutoff*/,
+ const real_number /*xseparation*/,
+ const real_number /*yseparation*/,
+ const real_number /*zseparation*/){
+ mergedParticles.emplace_back(std::max(idx_part1,idx_part2));
+ }
+
+ void merge(const p2p_merge_collisions& other){
+ collision_counter.insert(collision_counter.end(), other.collision_counter.begin(), other.collision_counter.end());
+ }
+
+ constexpr static bool isEnable() {
+ return true;
+ }
+
+ auto& get_merge_list() const{
+ return collision_counter;
+ }
+
+ void reset_merge_list(){
+ mergedParticles.clear();
+ }
+};
+
+
+#endif // P2P_GHOST_COLLISIONS_HPP
diff --git a/cpp/particles/p2p/p2p_tree.hpp b/cpp/particles/p2p/p2p_tree.hpp
new file mode 100644
index 0000000000000000000000000000000000000000..928957c02eb51fb02b9ad526ae05d61cd3052389
--- /dev/null
+++ b/cpp/particles/p2p/p2p_tree.hpp
@@ -0,0 +1,153 @@
+/******************************************************************************
+* *
+* Copyright 2019 Max Planck Institute for Dynamics and Self-Organization *
+* *
+* This file is part of TurTLE. *
+* *
+* TurTLE is free software: you can redistribute it and/or modify *
+* it under the terms of the GNU General Public License as published *
+* by the Free Software Foundation, either version 3 of the License, *
+* or (at your option) any later version. *
+* *
+* TurTLE is distributed in the hope that it will be useful, *
+* but WITHOUT ANY WARRANTY; without even the implied warranty of *
+* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the *
+* GNU General Public License for more details. *
+* *
+* You should have received a copy of the GNU General Public License *
+* along with TurTLE. If not, see <http://www.gnu.org/licenses/> *
+* *
+* Contact: Cristian.Lalescu@ds.mpg.de *
+* *
+******************************************************************************/
+
+
+
+#ifndef P2P_TREE_HPP
+#define P2P_TREE_HPP
+
+#include <unordered_map>
+#include <vector>
+
+template <class CellClass>
+class p2p_tree{
+ std::unordered_map<long int, CellClass> data;
+ CellClass emptyCell;
+ std::array<long int,3> nb_cell_levels;
+
+ long int get_cell_coord_x_from_index(const long int index) const{
+ return index % nb_cell_levels[IDXC_X];
+ }
+
+ long int get_cell_coord_y_from_index(const long int index) const{
+ return (index % (nb_cell_levels[IDXC_X]*nb_cell_levels[IDXC_Y]))
+ / nb_cell_levels[IDXC_X];
+ }
+
+ long int get_cell_coord_z_from_index(const long int index) const{
+ return index / (nb_cell_levels[IDXC_X]*nb_cell_levels[IDXC_Y]);
+ }
+
+ long int get_cell_idx(const long int idx_x, const long int idx_y,
+ const long int idx_z) const {
+ return (((idx_z*nb_cell_levels[IDXC_Y])+idx_y)*nb_cell_levels[IDXC_X])+idx_x;
+ }
+
+public:
+ explicit p2p_tree(std::array<long int,3> in_nb_cell_levels)
+ : nb_cell_levels(in_nb_cell_levels){
+ }
+
+ CellClass& getCell(const long int idx){
+ return data[idx];
+ }
+
+
+ const CellClass& getCell(const long int idx) const {
+ const auto& iter = data.find(idx);
+ if(iter != data.end()){
+ return iter->second;
+ }
+ return emptyCell;
+ }
+
+ template <class ShiftType>
+ int getNeighbors(const long int idx, const CellClass* output[27], long int output_indexes[27],
+ std::array<ShiftType,3> shift[27], const bool include_target) const{
+ int nbNeighbors = 0;
+
+ std::fill_n(output, 27, nullptr);
+
+ const long int idx_x = get_cell_coord_x_from_index(idx);
+ const long int idx_y = get_cell_coord_y_from_index(idx);
+ const long int idx_z = get_cell_coord_z_from_index(idx);
+
+ for(long int neigh_x = -1 ; neigh_x <= 1 ; ++neigh_x){
+ long int neigh_x_pbc = neigh_x+idx_x;
+ ShiftType shift_x = 0;
+ if(neigh_x_pbc < 0){
+ neigh_x_pbc += nb_cell_levels[IDXC_X];
+ shift_x = 1;
+ }
+ else if(nb_cell_levels[IDXC_X] <= neigh_x_pbc){
+ neigh_x_pbc -= nb_cell_levels[IDXC_X];
+ shift_x = -1;
+ }
+
+ for(long int neigh_y = -1 ; neigh_y <= 1 ; ++neigh_y){
+ long int neigh_y_pbc = neigh_y+idx_y;
+ ShiftType shift_y = 0;
+ if(neigh_y_pbc < 0){
+ neigh_y_pbc += nb_cell_levels[IDXC_Y];
+ shift_y = 1;
+ }
+ else if(nb_cell_levels[IDXC_Y] <= neigh_y_pbc){
+ neigh_y_pbc -= nb_cell_levels[IDXC_Y];
+ shift_y = -1;
+ }
+
+ for(long int neigh_z = -1 ; neigh_z <= 1 ; ++neigh_z){
+ long int neigh_z_pbc = neigh_z+idx_z;
+ ShiftType shift_z = 0;
+ if(neigh_z_pbc < 0){
+ neigh_z_pbc += nb_cell_levels[IDXC_Z];
+ shift_z = 1;
+ }
+ else if(nb_cell_levels[IDXC_Z] <= neigh_z_pbc){
+ neigh_z_pbc -= nb_cell_levels[IDXC_Z];
+ shift_z = -1;
+ }
+
+ if(include_target || neigh_x_pbc != idx_x || neigh_y_pbc != idx_y || neigh_z_pbc != idx_z){
+ const long int idx_neigh = get_cell_idx(neigh_x_pbc,
+ neigh_y_pbc,
+ neigh_z_pbc);
+ const auto& iter = data.find(idx_neigh);
+ if(iter != data.end()){
+ output[nbNeighbors] = &(iter->second);
+ output_indexes[nbNeighbors] = idx_neigh;
+
+ shift[nbNeighbors][IDXC_X] = shift_x;
+ shift[nbNeighbors][IDXC_Y] = shift_y;
+ shift[nbNeighbors][IDXC_Z] = shift_z;
+
+ nbNeighbors += 1;
+ }
+ }
+ }
+ }
+ }
+
+ return nbNeighbors;
+ }
+
+ typename std::unordered_map<long int, CellClass>::iterator begin(){
+ return data.begin();
+ }
+
+ typename std::unordered_map<long int, CellClass>::iterator end(){
+ return data.end();
+ }
+};
+
+#endif
diff --git a/cpp/particles/particle_solver.cpp b/cpp/particles/particle_solver.cpp
new file mode 100644
index 0000000000000000000000000000000000000000..e22290e0a815f9e072b5fdc4a524aa31ef089d9c
--- /dev/null
+++ b/cpp/particles/particle_solver.cpp
@@ -0,0 +1,329 @@
+/******************************************************************************
+* *
+* Copyright 2020 Max Planck Institute for Dynamics and Self-Organization *
+* *
+* This file is part of TurTLE. *
+* *
+* TurTLE is free software: you can redistribute it and/or modify *
+* it under the terms of the GNU General Public License as published *
+* by the Free Software Foundation, either version 3 of the License, *
+* or (at your option) any later version. *
+* *
+* TurTLE is distributed in the hope that it will be useful, *
+* but WITHOUT ANY WARRANTY; without even the implied warranty of *
+* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the *
+* GNU General Public License for more details. *
+* *
+* You should have received a copy of the GNU General Public License *
+* along with TurTLE. If not, see <http://www.gnu.org/licenses/> *
+* *
+* Contact: Cristian.Lalescu@ds.mpg.de *
+* *
+******************************************************************************/
+
+
+
+#include "particles/particle_solver.hpp"
+
+int particle_solver::Euler(
+ double dt,
+ abstract_particle_rhs &rhs)
+ {
+ TIMEZONE("particle_solver::Euler");
+ assert(this->pset->getStateSize() == rhs.getStateSize());
+ // temporary array for storing right-hand-side
+ auto *rhs_val = new particle_rnumber[this->pset->getLocalNumberOfParticles()*(this->pset->getStateSize())];
+ // temporary array for new particle state
+ auto *xx = new particle_rnumber[this->pset->getLocalNumberOfParticles()*(this->pset->getStateSize())];
+ // empty array required by rhs call
+ std::vector<std::unique_ptr<particle_rnumber[]>> tvalues;
+
+ // compute right hand side
+ rhs(0, *(this->pset), rhs_val, tvalues);
+
+
+ // compute new particle state
+ this->pset->LOOP(
+ [&](const partsize_t idx){
+ xx[idx] = this->pset->getParticleState()[idx] + dt * rhs_val[idx];
+ });
+
+ // update particle set
+ //
+
+ //std::vector<std::unique_ptr<particle_rnumber[]>> temporary;
+ this->pset->setParticleState(xx);
+ this->pset->redistribute(tvalues);
+
+ // ensure new state is consistent with physical model
+ rhs.imposeModelConstraints(*(this->pset));
+
+ // deallocate temporary arrays
+ delete[] rhs_val;
+ delete[] xx;
+ return EXIT_SUCCESS;
+ }
+
+/** \brief Implementation of the Heun integration method
+ *
+ * This is a second order Runge-Kutta time-stepping algorithm.
+ * We integrate the following first oder ODE:
+ * \f[
+ * x'(t) = f(t, x(t)).
+ * \f]
+ * The initial condition is \f$ x_n \f$ (for time \f$ t_n \f$), and we would
+ * like to integrate the equation forward for an interval \f$ h \f$.
+ * Then we must apply the following formulas:
+ * \f{eqnarray*}{
+ * k_1 &=& f(t_n, x_n) \\
+ * k_2 &=& f(t_n+h, x_n + h k_1) \\
+ * x_{n+1} &=& x_n + \frac{h}{2}(k_1 + k_2)
+ * \f}
+ *
+ */
+int particle_solver::Heun(
+ double dt,
+ abstract_particle_rhs &rhs)
+{
+ TIMEZONE("particle_solver::Euler");
+ assert(this->pset->getStateSize() == rhs.getStateSize());
+ // temporary arrays for storing the two right-hand-side values
+ // We need to be able to redistribute the initial condition for computations to make sense,
+ // so we must put everything in the same vector.
+ std::vector<std::unique_ptr<particle_rnumber[]>> tvalues;
+ std::unique_ptr<particle_rnumber[]> xx, x0, k1, k2;
+
+ // allocate x0
+ x0.reset(new particle_rnumber[this->pset->getLocalNumberOfParticles()*(this->pset->getStateSize())]);
+ // store initial condition
+ this->pset->getParticleState(x0.get());
+
+ tvalues.resize(1);
+ tvalues[0].reset(new particle_rnumber[
+ this->pset->getLocalNumberOfParticles()*
+ this->pset->getStateSize()]);
+ this->pset->copy_state_tofrom(tvalues[0], x0);
+ // allocate k1
+ k1.reset(new particle_rnumber[this->pset->getLocalNumberOfParticles()*(this->pset->getStateSize())]);
+ // compute k1
+ // possibly change local ordering, but size of particle state remains the same
+ rhs(0, *(this->pset), k1.get(), tvalues);
+ // copy back initial data to x0
+ this->pset->copy_state_tofrom(x0, tvalues[0]);
+
+ // allocate xx
+ xx.reset(new particle_rnumber[this->pset->getLocalNumberOfParticles()*(this->pset->getStateSize())]);
+ // compute xn+h*k1 in particle set
+ this->pset->LOOP(
+ [&](const partsize_t idx){
+ xx[idx] = x0[idx] + dt*k1[idx];
+ });
+ // copy the temporary state to the particle set, and redistribute
+ // we MUST redistribute, because we want to interpolate.
+ // we MUST shuffle values according to the same redistribution, otherwise
+ // the subsequent computations are meaningless (and can lead to segfaults).
+ this->pset->setParticleState(xx.get());
+ tvalues.resize(2);
+ tvalues[0].reset(new particle_rnumber[
+ this->pset->getLocalNumberOfParticles()*
+ this->pset->getStateSize()]);
+ tvalues[1].reset(new particle_rnumber[
+ this->pset->getLocalNumberOfParticles()*
+ this->pset->getStateSize()]);
+ this->pset->copy_state_tofrom(tvalues[0], x0);
+ this->pset->copy_state_tofrom(tvalues[1], k1);
+ this->pset->redistribute(tvalues);
+ // allocate k2
+ k2.reset(new particle_rnumber[this->pset->getLocalNumberOfParticles()*(this->pset->getStateSize())]);
+ // compute k2
+ // note the temporal interpolation is at `1`, i.e. end of time-step
+ rhs(1, *(this->pset), k2.get(), tvalues);
+ // reallocate k0
+ x0.reset(new particle_rnumber[this->pset->getLocalNumberOfParticles()*(this->pset->getStateSize())]);
+ // reallocate k1
+ k1.reset(new particle_rnumber[this->pset->getLocalNumberOfParticles()*(this->pset->getStateSize())]);
+ this->pset->copy_state_tofrom(x0, tvalues[0]);
+ this->pset->copy_state_tofrom(k1, tvalues[1]);
+ // tvalues no longer required
+ tvalues.resize(0);
+
+ // local number of particles may have changed
+ xx.reset(new particle_rnumber[this->pset->getLocalNumberOfParticles()*(this->pset->getStateSize())]);
+ // compute final state
+ this->pset->LOOP(
+ [&](const partsize_t idx){
+ xx[idx] = x0[idx] + 0.5*dt*(k1[idx] + k2[idx]);
+ });
+ this->pset->setParticleState(xx.get());
+ // redistribute, impose model constraints
+ this->pset->redistribute(tvalues);
+ rhs.imposeModelConstraints(*(this->pset));
+
+ // and we are done
+ return EXIT_SUCCESS;
+}
+
+/** \brief Implementation of the classic 4th order Runge Kutta integration method
+ *
+ * This is a fourth order Runge-Kutta time-stepping algorithm.
+ * We integrate the following first oder ODE:
+ * \f[
+ * x'(t) = f(t, x(t)).
+ * \f]
+ * The initial condition is \f$ x_n \f$ (for time \f$ t_n \f$), and we would
+ * like to integrate the equation forward for an interval \f$ h \f$.
+ * Then we must apply the following formulas:
+ * \f{eqnarray*}{
+ * k_1 &=& f(t_n, x_n) \\
+ * k_2 &=& f(t_n+frac{h}{2}, x_n + \frac{h}{2} k_1) \\
+ * k_3 &=& f(t_n+frac{h}{2}, x_n + \frac{h}{2} k_2) \\
+ * k_4 &=& f(t_n+h, x_n + h k_3) \\
+ * x_{n+1} &=& x_n + \frac{h}{6}(k_1 + 2 k_2 + 2 k_3 + k_4)
+ * \f}
+ *
+ */
+int particle_solver::cRK4(
+ double dt,
+ abstract_particle_rhs &rhs)
+{
+ TIMEZONE("particle_solver::Euler");
+ assert(this->pset->getStateSize() == rhs.getStateSize());
+ // temporary arrays for storing the two right-hand-side values
+ // We need to be able to redistribute the initial condition for computations to make sense,
+ // so we must put everything in the same vector.
+ std::vector<std::unique_ptr<particle_rnumber[]>> extra_values;
+ std::unique_ptr<particle_rnumber[]> xx, kcurrent;
+
+ // allocate extra_values
+ // I use the following convention:
+ // extra_values[0] = x0
+ // extra_values[1] = k1
+ // extra_values[2] = k2
+ // extra_values[3] = k3
+ // extra_values[4] = k4
+ extra_values.resize(5);
+ for (int temp_counter = 0; temp_counter < 5; temp_counter++)
+ {
+ extra_values[temp_counter].reset(new particle_rnumber[
+ this->pset->getLocalNumberOfParticles()*
+ this->pset->getStateSize()]);
+ }
+
+ // store initial condition
+ this->pset->getParticleState(extra_values[0].get());
+
+ /*** COMPUTE K1 ***/
+ kcurrent.reset(new particle_rnumber[
+ this->pset->getLocalNumberOfParticles()*
+ this->pset->getStateSize()]);
+ // call to rhs will possibly change local ordering, but local number of particles remains the same
+ rhs(0, *(this->pset), kcurrent.get(), extra_values);
+ // store k1 values
+ this->pset->copy_state_tofrom(extra_values[1], kcurrent);
+
+ /*** COMPUTE xn+h*k1/2 ***/
+ // allocate xx
+ xx.reset(new particle_rnumber[
+ this->pset->getLocalNumberOfParticles()*
+ this->pset->getStateSize()]);
+ this->pset->LOOP(
+ [&](const partsize_t idx){
+ xx[idx] = extra_values[0][idx] + 0.5*dt*extra_values[1][idx];
+ });
+ // copy the temporary state to the particle set, and redistribute
+ // we MUST redistribute, because we want to interpolate.
+ // we MUST shuffle values according to the same redistribution, otherwise
+ // the subsequent computations are meaningless.
+ // this will possibly change local ordering AND/OR local number of particles
+ this->pset->setParticleState(xx.get());
+ this->pset->redistribute(extra_values);
+
+ /*** COMPUTE K2 ***/
+ kcurrent.reset(new particle_rnumber[
+ this->pset->getLocalNumberOfParticles()*
+ this->pset->getStateSize()]);
+ rhs(0.5, *(this->pset), kcurrent.get(), extra_values);
+ this->pset->copy_state_tofrom(extra_values[2], kcurrent);
+
+ /*** COMPUTE xn+h*k2/2 ***/
+ xx.reset(new particle_rnumber[
+ this->pset->getLocalNumberOfParticles()*
+ this->pset->getStateSize()]);
+ this->pset->LOOP(
+ [&](const partsize_t idx){
+ xx[idx] = extra_values[0][idx] + 0.5*dt*extra_values[2][idx];
+ });
+ this->pset->setParticleState(xx.get());
+ this->pset->redistribute(extra_values);
+
+ /*** COMPUTE K3 ***/
+ kcurrent.reset(new particle_rnumber[
+ this->pset->getLocalNumberOfParticles()*
+ this->pset->getStateSize()]);
+ rhs(0.5, *(this->pset), kcurrent.get(), extra_values);
+ this->pset->copy_state_tofrom(extra_values[3], kcurrent);
+
+ /*** COMPUTE xn+h*k3 ***/
+ xx.reset(new particle_rnumber[
+ this->pset->getLocalNumberOfParticles()*
+ this->pset->getStateSize()]);
+ this->pset->LOOP(
+ [&](const partsize_t idx){
+ xx[idx] = extra_values[0][idx] + dt*extra_values[3][idx];
+ });
+ this->pset->setParticleState(xx.get());
+ this->pset->redistribute(extra_values);
+
+ /*** COMPUTE K4 ***/
+ kcurrent.reset(new particle_rnumber[
+ this->pset->getLocalNumberOfParticles()*
+ this->pset->getStateSize()]);
+ rhs(0.5, *(this->pset), kcurrent.get(), extra_values);
+ this->pset->copy_state_tofrom(extra_values[4], kcurrent);
+
+ /*** COMPUTE xn+h*(k1 + 2*k2 + 2*k3 + k4)/6 ***/
+ xx.reset(new particle_rnumber[
+ this->pset->getLocalNumberOfParticles()*
+ this->pset->getStateSize()]);
+ this->pset->LOOP(
+ [&](const partsize_t idx){
+ xx[idx] = extra_values[0][idx] +
+ dt*(extra_values[1][idx] +
+ 2*extra_values[2][idx] +
+ 2*extra_values[3][idx] +
+ extra_values[4][idx]) / 6;
+ });
+ this->pset->setParticleState(xx.get());
+ extra_values.resize(0);
+ this->pset->redistribute(extra_values);
+
+ // impose model constraints
+ rhs.imposeModelConstraints(*(this->pset));
+
+ // and we are done
+ return EXIT_SUCCESS;
+}
+
+
+template <int state_size>
+int particle_solver::writeCheckpoint(
+ particles_output_hdf5<partsize_t, particle_rnumber, state_size> *particles_output_writer)
+{
+ TIMEZONE("particle_solver::writeCheckpoint");
+ particles_output_writer->update_particle_species_name(this->particle_species_name);
+ particles_output_writer->setParticleFileLayout(pset->getParticleFileLayout());
+ particles_output_writer->template save<state_size>(
+ this->pset->getParticleState(),
+ this->additional_states.data(),
+ this->pset->getParticleIndices(),
+ this->pset->getLocalNumberOfParticles(),
+ this->iteration);
+ return EXIT_SUCCESS;
+}
+
+template int particle_solver::writeCheckpoint<3>(particles_output_hdf5<partsize_t, particle_rnumber, 3> *);
+template int particle_solver::writeCheckpoint<6>(particles_output_hdf5<partsize_t, particle_rnumber, 6> *);
+template int particle_solver::writeCheckpoint<7>(particles_output_hdf5<partsize_t, particle_rnumber, 7> *);
+template int particle_solver::writeCheckpoint<8>(particles_output_hdf5<partsize_t, particle_rnumber, 8> *);
+template int particle_solver::writeCheckpoint<15>(particles_output_hdf5<partsize_t, particle_rnumber, 15> *);
+
diff --git a/cpp/particles/particle_solver.hpp b/cpp/particles/particle_solver.hpp
new file mode 100644
index 0000000000000000000000000000000000000000..54bbc3c846022c00f220d9f427e121241fd10223
--- /dev/null
+++ b/cpp/particles/particle_solver.hpp
@@ -0,0 +1,119 @@
+/******************************************************************************
+* *
+* Copyright 2020 Max Planck Institute for Dynamics and Self-Organization *
+* *
+* This file is part of TurTLE. *
+* *
+* TurTLE is free software: you can redistribute it and/or modify *
+* it under the terms of the GNU General Public License as published *
+* by the Free Software Foundation, either version 3 of the License, *
+* or (at your option) any later version. *
+* *
+* TurTLE is distributed in the hope that it will be useful, *
+* but WITHOUT ANY WARRANTY; without even the implied warranty of *
+* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the *
+* GNU General Public License for more details. *
+* *
+* You should have received a copy of the GNU General Public License *
+* along with TurTLE. If not, see <http://www.gnu.org/licenses/> *
+* *
+* Contact: Cristian.Lalescu@ds.mpg.de *
+* *
+******************************************************************************/
+
+
+
+#ifndef PARTICLE_SOLVER_HPP
+#define PARTICLE_SOLVER_HPP
+
+#include <cassert>
+#include "particles/interpolation/abstract_particle_set.hpp"
+#include "particles/abstract_particle_rhs.hpp"
+#include "particles/particles_output_hdf5.hpp"
+
+
+
+/** Time-stepping and checkpointing functionality for particle systems.
+ *
+ * This class implements the universal functionality of a particle solver:
+ * time-stepping methods and checkpointing, together with the associated
+ * bookkeeping.
+ *
+ * We rely on predefined functionality of particle set objects, as well as
+ * "particle right-hand-side computers".
+ *
+ * Development note: the `additional_states` member is to be used for
+ * Adams-Bashforth time-stepping, not for Runge-Kutta k values.
+ * In particular values stored in `additional_states` are used for
+ * check-pointing, so there's no sense to allocate this array unless strictly
+ * required as for the Adams-Bashforth scheme.
+ */
+
+class particle_solver
+{
+ protected:
+ using particle_rnumber = abstract_particle_set::particle_rnumber;
+ using partsize_t = abstract_particle_set::partsize_t;
+
+ abstract_particle_set *pset;
+
+ std::vector<std::unique_ptr<particle_rnumber[]>> additional_states;
+ const int number_of_additional_states;
+
+ std::string particle_species_name;
+ int iteration;
+
+ public:
+ particle_solver(
+ abstract_particle_set &in_pset,
+ const int in_number_of_additional_states):
+ pset(&in_pset),
+ number_of_additional_states(in_number_of_additional_states)
+ {
+ this->iteration = 0;
+ this->additional_states.resize(number_of_additional_states);
+ this->particle_species_name = "tracers0";
+ }
+ ~particle_solver() noexcept(false){}
+
+ int setIteration(const int i)
+ {
+ this->iteration = i;
+ return EXIT_SUCCESS;
+ }
+ int getIteration(void)
+ {
+ return this->iteration;
+ }
+
+ std::string getParticleSpeciesName(
+ const std::string new_name)
+ {
+ return this->particle_species_name;
+ }
+ int setParticleSpeciesName(
+ const std::string new_name)
+ {
+ this->particle_species_name.assign(new_name);
+ return EXIT_SUCCESS;
+ }
+
+ template <int state_size>
+ int writeCheckpoint(
+ particles_output_hdf5<partsize_t, particle_rnumber, state_size> *particles_output_writer);
+
+ int Euler(
+ double dt,
+ abstract_particle_rhs &rhs);
+
+ int Heun(
+ double dt,
+ abstract_particle_rhs &rhs);
+
+ int cRK4(
+ double dt,
+ abstract_particle_rhs &rhs);
+};
+
+#endif//PARTICLE_SOLVER_HPP
+
diff --git a/bfps/cpp/particles/particles_adams_bashforth.hpp b/cpp/particles/particles_adams_bashforth.hpp
similarity index 75%
rename from bfps/cpp/particles/particles_adams_bashforth.hpp
rename to cpp/particles/particles_adams_bashforth.hpp
index 2fb61462f7970d823acd6dc3405799e362fa15af..f760d473890d8210a5b9927e524bad0a6d1df761 100644
--- a/bfps/cpp/particles/particles_adams_bashforth.hpp
+++ b/cpp/particles/particles_adams_bashforth.hpp
@@ -1,3 +1,28 @@
+/******************************************************************************
+* *
+* Copyright 2019 Max Planck Institute for Dynamics and Self-Organization *
+* *
+* This file is part of TurTLE. *
+* *
+* TurTLE is free software: you can redistribute it and/or modify *
+* it under the terms of the GNU General Public License as published *
+* by the Free Software Foundation, either version 3 of the License, *
+* or (at your option) any later version. *
+* *
+* TurTLE is distributed in the hope that it will be useful, *
+* but WITHOUT ANY WARRANTY; without even the implied warranty of *
+* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the *
+* GNU General Public License for more details. *
+* *
+* You should have received a copy of the GNU General Public License *
+* along with TurTLE. If not, see <http://www.gnu.org/licenses/> *
+* *
+* Contact: Cristian.Lalescu@ds.mpg.de *
+* *
+******************************************************************************/
+
+
+
#ifndef PARTICLES_ADAMS_BASHFORTH_HPP
#define PARTICLES_ADAMS_BASHFORTH_HPP
@@ -7,11 +32,10 @@
#include "scope_timer.hpp"
#include "particles_utils.hpp"
-template <class partsize_t, class real_number, int size_particle_positions = 3, int size_particle_rhs = 3>
-class particles_adams_bashforth {
- static_assert(size_particle_positions == size_particle_rhs,
- "Not having the same dimension for positions and rhs looks like a bug,"
- "otherwise comment this assertion.");
+template <class partsize_t, class real_number, int size_particle_positions, int size_particle_rhs>
+class particles_adams_bashforth{
+ static_assert(size_particle_positions == size_particle_rhs, "This class is designed for the same number of values in positions and rhs");
+
public:
static const int Max_steps = 6;
@@ -22,7 +46,7 @@ public:
TIMEZONE("particles_adams_bashforth::move_particles");
if(Max_steps < nb_rhs){
- throw std::runtime_error("Error, in bfps particles_adams_bashforth.\n"
+ throw std::runtime_error("Error, in TurTLE particles_adams_bashforth.\n"
"Step in particles_adams_bashforth is too large,"
"you must add formulation up this number or limit the number of steps.");
}
diff --git a/bfps/cpp/particles/particles_distr_mpi.hpp b/cpp/particles/particles_distr_mpi.hpp
similarity index 75%
rename from bfps/cpp/particles/particles_distr_mpi.hpp
rename to cpp/particles/particles_distr_mpi.hpp
index 485595181f69b9fe1cf204b06df550a9ca74215d..0a468253dc4d38d362d2dec189445de55df3ec7e 100644
--- a/bfps/cpp/particles/particles_distr_mpi.hpp
+++ b/cpp/particles/particles_distr_mpi.hpp
@@ -1,3 +1,28 @@
+/******************************************************************************
+* *
+* Copyright 2019 Max Planck Institute for Dynamics and Self-Organization *
+* *
+* This file is part of TurTLE. *
+* *
+* TurTLE is free software: you can redistribute it and/or modify *
+* it under the terms of the GNU General Public License as published *
+* by the Free Software Foundation, either version 3 of the License, *
+* or (at your option) any later version. *
+* *
+* TurTLE is distributed in the hope that it will be useful, *
+* but WITHOUT ANY WARRANTY; without even the implied warranty of *
+* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the *
+* GNU General Public License for more details. *
+* *
+* You should have received a copy of the GNU General Public License *
+* along with TurTLE. If not, see <http://www.gnu.org/licenses/> *
+* *
+* Contact: Cristian.Lalescu@ds.mpg.de *
+* *
+******************************************************************************/
+
+
+
#ifndef PARTICLES_DISTR_MPI_HPP
#define PARTICLES_DISTR_MPI_HPP
@@ -17,10 +42,10 @@
template <class partsize_t, class real_number>
class particles_distr_mpi {
protected:
- static const int MaxNbRhs = 100;
+ static const int MaxNbRhs = 10;
enum MpiTag{
- TAG_LOW_UP_NB_PARTICLES,
+ TAG_LOW_UP_NB_PARTICLES = 999,
TAG_UP_LOW_NB_PARTICLES,
TAG_LOW_UP_PARTICLES,
TAG_UP_LOW_PARTICLES,
@@ -35,11 +60,16 @@ protected:
TAG_LOW_UP_MOVED_PARTICLES_INDEXES,
TAG_UP_LOW_MOVED_PARTICLES_INDEXES,
+ TAG_LOW_UP_MOVED_PARTICLES_LABELS,
+ TAG_UP_LOW_MOVED_PARTICLES_LABELS,
+
TAG_LOW_UP_MOVED_PARTICLES_RHS,
TAG_LOW_UP_MOVED_PARTICLES_RHS_MAX = TAG_LOW_UP_MOVED_PARTICLES_RHS+MaxNbRhs,
TAG_UP_LOW_MOVED_PARTICLES_RHS = TAG_LOW_UP_MOVED_PARTICLES_RHS_MAX,
TAG_UP_LOW_MOVED_PARTICLES_RHS_MAX = TAG_UP_LOW_MOVED_PARTICLES_RHS+MaxNbRhs,
+
+ TAG_SHIFT_OFFSET
};
struct NeighborDescriptor{
@@ -89,6 +119,8 @@ protected:
std::vector<MPI_Request> mpiRequests;
std::vector<NeighborDescriptor> neigDescriptors;
+ int counter_shift_tags;
+
public:
////////////////////////////////////////////////////////////////////////////
@@ -99,7 +131,8 @@ public:
my_rank(-1), nb_processes(-1),nb_processes_involved(-1),
current_partition_interval(in_current_partitions),
current_partition_size(current_partition_interval.second-current_partition_interval.first),
- field_grid_dim(in_field_grid_dim){
+ field_grid_dim(in_field_grid_dim),
+ counter_shift_tags(0){
AssertMpi(MPI_Comm_rank(current_com, &my_rank));
AssertMpi(MPI_Comm_size(current_com, &nb_processes));
@@ -127,21 +160,21 @@ public:
assert(partition_interval_size_per_proc[idx_proc_involved] != 0);
}
- assert(int(field_grid_dim[IDX_Z]) == partition_interval_offset_per_proc[nb_processes_involved]);
+ assert(int(field_grid_dim[IDXC_Z]) == partition_interval_offset_per_proc[nb_processes_involved]);
}
- virtual ~particles_distr_mpi(){}
+ virtual ~particles_distr_mpi() noexcept(false){}
////////////////////////////////////////////////////////////////////////////
template <class computer_class, class field_class, int size_particle_positions, int size_particle_rhs>
void compute_distr(computer_class& in_computer,
- field_class& in_field,
+ const field_class& in_field,
const partsize_t current_my_nb_particles_per_partition[],
const real_number particles_positions[],
real_number particles_current_rhs[],
const int interpolation_size){
- TIMEZONE("compute_distr");
+ TIMEZONE("particle_distr_mpi::compute_distr");
// Some processes might not be involved
if(nb_processes_involved <= my_rank){
@@ -182,6 +215,7 @@ public:
const int nbPartitionsToSend = std::min(current_partition_size, interpolation_size-(idxLower-1));
assert(nbPartitionsToSend >= 0);
const partsize_t nbParticlesToSend = current_offset_particles_for_partition[nbPartitionsToSend] - current_offset_particles_for_partition[0];
+ assert(nbParticlesToSend >= 0);
const int nbPartitionsToRecv = std::min(partition_interval_size_per_proc[destProc], (interpolation_size+1)-(idxLower-1));
assert(nbPartitionsToRecv > 0);
@@ -235,6 +269,7 @@ public:
}
const int nbProcToRecvUpper = int(neigDescriptors.size())-nbProcToRecvLower;
const int nbProcToRecv = nbProcToRecvUpper + nbProcToRecvLower;
+ variable_used_only_in_assert(nbProcToRecv);
assert(int(neigDescriptors.size()) == nbProcToRecv);
for(int idxDescr = 0 ; idxDescr < int(neigDescriptors.size()) ; ++idxDescr){
@@ -245,14 +280,18 @@ public:
whatNext.emplace_back(std::pair<Action,int>{NOTHING_TODO, -1});
mpiRequests.emplace_back();
AssertMpi(MPI_Isend(const_cast<partsize_t*>(&descriptor.nbParticlesToSend), 1, particles_utils::GetMpiType(partsize_t()),
- descriptor.destProc, TAG_LOW_UP_NB_PARTICLES,
+ descriptor.destProc, TAG_SHIFT_OFFSET*counter_shift_tags + TAG_LOW_UP_NB_PARTICLES,
current_com, &mpiRequests.back()));
if(descriptor.nbParticlesToSend){
whatNext.emplace_back(std::pair<Action,int>{NOTHING_TODO, -1});
mpiRequests.emplace_back();
+ //DEBUG_MSG("descriptor.nparticlestosend = %d, size_particle_positions = %d, std::numeric_limits = %d\n",
+ // descriptor.nbParticlesToSend,
+ // size_particle_positions,
+ // std::numeric_limits<int>::max());
assert(descriptor.nbParticlesToSend*size_particle_positions < std::numeric_limits<int>::max());
- AssertMpi(MPI_Isend(const_cast<real_number*>(&particles_positions[0]), int(descriptor.nbParticlesToSend*size_particle_positions), particles_utils::GetMpiType(real_number()), descriptor.destProc, TAG_LOW_UP_PARTICLES,
+ AssertMpi(MPI_Isend(const_cast<real_number*>(&particles_positions[0]), int(descriptor.nbParticlesToSend*size_particle_positions), particles_utils::GetMpiType(real_number()), descriptor.destProc, TAG_SHIFT_OFFSET*counter_shift_tags + TAG_LOW_UP_PARTICLES,
current_com, &mpiRequests.back()));
assert(descriptor.toRecvAndMerge == nullptr);
@@ -260,7 +299,7 @@ public:
whatNext.emplace_back(std::pair<Action,int>{MERGE_PARTICLES, idxDescr});
mpiRequests.emplace_back();
assert(descriptor.nbParticlesToSend*size_particle_rhs < std::numeric_limits<int>::max());
- AssertMpi(MPI_Irecv(descriptor.toRecvAndMerge.get(), int(descriptor.nbParticlesToSend*size_particle_rhs), particles_utils::GetMpiType(real_number()), descriptor.destProc, TAG_UP_LOW_RESULTS,
+ AssertMpi(MPI_Irecv(descriptor.toRecvAndMerge.get(), int(descriptor.nbParticlesToSend*size_particle_rhs), particles_utils::GetMpiType(real_number()), descriptor.destProc, TAG_SHIFT_OFFSET*counter_shift_tags + TAG_UP_LOW_RESULTS,
current_com, &mpiRequests.back()));
}
}
@@ -269,7 +308,7 @@ public:
whatNext.emplace_back(std::pair<Action,int>{RECV_PARTICLES, idxDescr});
mpiRequests.emplace_back();
AssertMpi(MPI_Irecv(&descriptor.nbParticlesToRecv,
- 1, particles_utils::GetMpiType(partsize_t()), descriptor.destProc, TAG_UP_LOW_NB_PARTICLES,
+ 1, particles_utils::GetMpiType(partsize_t()), descriptor.destProc, TAG_SHIFT_OFFSET*counter_shift_tags + TAG_UP_LOW_NB_PARTICLES,
current_com, &mpiRequests.back()));
}
else{
@@ -277,16 +316,16 @@ public:
whatNext.emplace_back(std::pair<Action,int>{NOTHING_TODO, -1});
mpiRequests.emplace_back();
AssertMpi(MPI_Isend(const_cast<partsize_t*>(&descriptor.nbParticlesToSend), 1, particles_utils::GetMpiType(partsize_t()),
- descriptor.destProc, TAG_UP_LOW_NB_PARTICLES,
+ descriptor.destProc, TAG_SHIFT_OFFSET*counter_shift_tags + TAG_UP_LOW_NB_PARTICLES,
current_com, &mpiRequests.back()));
if(descriptor.nbParticlesToSend){
whatNext.emplace_back(std::pair<Action,int>{NOTHING_TODO, -1});
- mpiRequests.emplace_back();
+ mpiRequests.emplace_back();
assert(descriptor.nbParticlesToSend*size_particle_positions < std::numeric_limits<int>::max());
AssertMpi(MPI_Isend(const_cast<real_number*>(&particles_positions[(current_offset_particles_for_partition[current_partition_size-descriptor.nbPartitionsToSend])*size_particle_positions]),
int(descriptor.nbParticlesToSend*size_particle_positions), particles_utils::GetMpiType(real_number()),
- descriptor.destProc, TAG_UP_LOW_PARTICLES,
+ descriptor.destProc, TAG_SHIFT_OFFSET*counter_shift_tags + TAG_UP_LOW_PARTICLES,
current_com, &mpiRequests.back()));
assert(descriptor.toRecvAndMerge == nullptr);
@@ -294,7 +333,7 @@ public:
whatNext.emplace_back(std::pair<Action,int>{MERGE_PARTICLES, idxDescr});
mpiRequests.emplace_back();
assert(descriptor.nbParticlesToSend*size_particle_rhs < std::numeric_limits<int>::max());
- AssertMpi(MPI_Irecv(descriptor.toRecvAndMerge.get(), int(descriptor.nbParticlesToSend*size_particle_rhs), particles_utils::GetMpiType(real_number()), descriptor.destProc, TAG_LOW_UP_RESULTS,
+ AssertMpi(MPI_Irecv(descriptor.toRecvAndMerge.get(), int(descriptor.nbParticlesToSend*size_particle_rhs), particles_utils::GetMpiType(real_number()), descriptor.destProc, TAG_SHIFT_OFFSET*counter_shift_tags + TAG_LOW_UP_RESULTS,
current_com, &mpiRequests.back()));
}
@@ -302,13 +341,15 @@ public:
whatNext.emplace_back(std::pair<Action,int>{RECV_PARTICLES, idxDescr});
mpiRequests.emplace_back();
AssertMpi(MPI_Irecv(&descriptor.nbParticlesToRecv,
- 1, particles_utils::GetMpiType(partsize_t()), descriptor.destProc, TAG_LOW_UP_NB_PARTICLES,
+ 1, particles_utils::GetMpiType(partsize_t()), descriptor.destProc, TAG_SHIFT_OFFSET*counter_shift_tags + TAG_LOW_UP_NB_PARTICLES,
current_com, &mpiRequests.back()));
}
}
}
const bool more_than_one_thread = (omp_get_max_threads() > 1);
+ /// MPI_Barrier(MPI_COMM_WORLD);
+ /// DEBUG_MSG_WAIT(MPI_COMM_WORLD, "line 338 of particles_distr_mpi.hpp\n");
TIMEZONE_OMP_INIT_PREPARALLEL(omp_get_max_threads())
#pragma omp parallel default(shared)
@@ -348,7 +389,7 @@ public:
mpiRequests.emplace_back();
assert(NbParticlesToReceive*size_particle_positions < std::numeric_limits<int>::max());
AssertMpi(MPI_Irecv(descriptor.toCompute.get(), int(NbParticlesToReceive*size_particle_positions),
- particles_utils::GetMpiType(real_number()), destProc, TAG_UP_LOW_PARTICLES,
+ particles_utils::GetMpiType(real_number()), destProc, TAG_SHIFT_OFFSET*counter_shift_tags + TAG_UP_LOW_PARTICLES,
current_com, &mpiRequests.back()));
}
}
@@ -365,7 +406,7 @@ public:
mpiRequests.emplace_back();
assert(NbParticlesToReceive*size_particle_positions < std::numeric_limits<int>::max());
AssertMpi(MPI_Irecv(descriptor.toCompute.get(), int(NbParticlesToReceive*size_particle_positions),
- particles_utils::GetMpiType(real_number()), destProc, TAG_LOW_UP_PARTICLES,
+ particles_utils::GetMpiType(real_number()), destProc, TAG_SHIFT_OFFSET*counter_shift_tags + TAG_LOW_UP_PARTICLES,
current_com, &mpiRequests.back()));
}
}
@@ -379,11 +420,19 @@ public:
const partsize_t NbParticlesToReceive = descriptor.nbParticlesToRecv;
assert(descriptor.toCompute != nullptr);
+ // allocate buffer
descriptor.results.reset(new real_number[NbParticlesToReceive*size_particle_rhs]);
- in_computer.template init_result_array<size_particle_rhs>(descriptor.results.get(), NbParticlesToReceive);
+ // clean up buffer
+ set_particle_data_to_zero<partsize_t, real_number, size_particle_rhs>(
+ descriptor.results.get(),
+ NbParticlesToReceive);
if(more_than_one_thread == false){
- in_computer.template apply_computation<field_class, size_particle_rhs>(in_field, descriptor.toCompute.get(), descriptor.results.get(), NbParticlesToReceive);
+ in_computer.template apply_computation<field_class, size_particle_positions, size_particle_rhs>(
+ in_field,
+ descriptor.toCompute.get(),
+ descriptor.results.get(),
+ NbParticlesToReceive);
}
else{
TIMEZONE_OMP_INIT_PRETASK(timeZoneTaskKey)
@@ -396,7 +445,7 @@ public:
TIMEZONE_OMP_PRAGMA_TASK_KEY(timeZoneTaskKey)
{
TIMEZONE_OMP_TASK("in_computer.apply_computation", timeZoneTaskKey);
- in_computer.template apply_computation<field_class, size_particle_rhs>(in_field, &ptr_descriptor->toCompute[idxPart*size_particle_positions],
+ in_computer.template apply_computation<field_class, size_particle_positions, size_particle_rhs>(in_field, &ptr_descriptor->toCompute[idxPart*size_particle_positions],
&ptr_descriptor->results[idxPart*size_particle_rhs], sizeToDo);
}
}
@@ -406,7 +455,7 @@ public:
const int destProc = descriptor.destProc;
whatNext.emplace_back(std::pair<Action,int>{RELEASE_BUFFER_PARTICLES, releasedAction.second});
mpiRequests.emplace_back();
- const int tag = descriptor.isLower? TAG_LOW_UP_RESULTS : TAG_UP_LOW_RESULTS;
+ const int tag = descriptor.isLower? TAG_SHIFT_OFFSET*counter_shift_tags + TAG_LOW_UP_RESULTS : TAG_SHIFT_OFFSET*counter_shift_tags + TAG_UP_LOW_RESULTS;
assert(NbParticlesToReceive*size_particle_rhs < std::numeric_limits<int>::max());
AssertMpi(MPI_Isend(descriptor.results.get(), int(NbParticlesToReceive*size_particle_rhs), particles_utils::GetMpiType(real_number()), destProc, tag,
current_com, &mpiRequests.back()));
@@ -417,7 +466,7 @@ public:
if(releasedAction.first == RELEASE_BUFFER_PARTICLES){
NeighborDescriptor& descriptor = neigDescriptors[releasedAction.second];
assert(descriptor.toCompute != nullptr);
- descriptor.toCompute.release();
+ delete[] descriptor.toCompute.release();
}
//////////////////////////////////////////////////////////////////////
/// Merge
@@ -429,14 +478,14 @@ public:
TIMEZONE("reduce");
assert(descriptor.toRecvAndMerge != nullptr);
in_computer.template reduce_particles_rhs<size_particle_rhs>(&particles_current_rhs[0], descriptor.toRecvAndMerge.get(), descriptor.nbParticlesToSend);
- descriptor.toRecvAndMerge.release();
+ delete[] descriptor.toRecvAndMerge.release();
}
else {
TIMEZONE("reduce");
assert(descriptor.toRecvAndMerge != nullptr);
in_computer.template reduce_particles_rhs<size_particle_rhs>(&particles_current_rhs[(current_offset_particles_for_partition[current_partition_size]-descriptor.nbParticlesToSend)*size_particle_rhs],
descriptor.toRecvAndMerge.get(), descriptor.nbParticlesToSend);
- descriptor.toRecvAndMerge.release();
+ delete[] descriptor.toRecvAndMerge.release();
}
}
}
@@ -456,7 +505,7 @@ public:
#pragma omp task default(shared) firstprivate(idxPart, sizeToDo) priority(0) TIMEZONE_OMP_PRAGMA_TASK_KEY(timeZoneTaskKey)
{
TIMEZONE_OMP_TASK("in_computer.apply_computation", timeZoneTaskKey);
- in_computer.template apply_computation<field_class, size_particle_rhs>(in_field, &particles_positions[idxPart*size_particle_positions],
+ in_computer.template apply_computation<field_class, size_particle_positions, size_particle_rhs>(in_field, &particles_positions[idxPart*size_particle_positions],
&particles_current_rhs[idxPart*size_particle_rhs],
sizeToDo);
}
@@ -474,14 +523,14 @@ public:
TIMEZONE("reduce_later");
assert(descriptor.toRecvAndMerge != nullptr);
in_computer.template reduce_particles_rhs<size_particle_rhs>(&particles_current_rhs[0], descriptor.toRecvAndMerge.get(), descriptor.nbParticlesToSend);
- descriptor.toRecvAndMerge.release();
+ delete[] descriptor.toRecvAndMerge.release();
}
else {
TIMEZONE("reduce_later");
assert(descriptor.toRecvAndMerge != nullptr);
in_computer.template reduce_particles_rhs<size_particle_rhs>(&particles_current_rhs[(current_offset_particles_for_partition[current_partition_size]-descriptor.nbParticlesToSend)*size_particle_rhs],
descriptor.toRecvAndMerge.get(), descriptor.nbParticlesToSend);
- descriptor.toRecvAndMerge.release();
+ delete[] descriptor.toRecvAndMerge.release();
}
}
}
@@ -492,12 +541,14 @@ public:
TIMEZONE("compute-my_compute");
// Compute my particles
if(myTotalNbParticles){
- in_computer.template apply_computation<field_class, size_particle_rhs>(in_field, particles_positions, particles_current_rhs, myTotalNbParticles);
+ in_computer.template apply_computation<field_class, size_particle_positions, size_particle_rhs>(in_field, particles_positions, particles_current_rhs, myTotalNbParticles);
}
}
assert(whatNext.size() == 0);
assert(mpiRequests.size() == 0);
+
+ counter_shift_tags += 1;
}
@@ -509,7 +560,8 @@ public:
partsize_t* nb_particles,
std::unique_ptr<real_number[]>* inout_positions_particles,
std::unique_ptr<real_number[]> inout_rhs_particles[], const int in_nb_rhs,
- std::unique_ptr<partsize_t[]>* inout_index_particles){
+ std::unique_ptr<partsize_t[]>* inout_index_particles,
+ std::unique_ptr<partsize_t[]>* inout_label_particles = nullptr){
TIMEZONE("redistribute");
// Some latest processes might not be involved
@@ -517,6 +569,22 @@ public:
return;
}
+ const bool labels_present = (inout_label_particles != nullptr);
+
+ {// this fails when particles move more than one cell during a single time-step
+ partsize_t partOffset = 0;
+ for(int idxPartition = 0 ; idxPartition < current_partition_size ; ++idxPartition){
+ for(partsize_t idx = 0 ; idx < current_my_nb_particles_per_partition[idxPartition] ; ++idx){
+ const int partition_level = in_computer.pbc_field_layer((*inout_positions_particles)[(idx+partOffset)*size_particle_positions+IDXC_Z], IDXC_Z);
+ variable_used_only_in_assert(partition_level);
+ assert(partition_level == current_partition_interval.first + idxPartition
+ || partition_level == (current_partition_interval.first + idxPartition-1+int(field_grid_dim[IDXC_Z]))%int(field_grid_dim[IDXC_Z])
+ || partition_level == (current_partition_interval.first + idxPartition+1)%int(field_grid_dim[IDXC_Z]));
+ }
+ partOffset += current_my_nb_particles_per_partition[idxPartition];
+ }
+ }
+
current_offset_particles_for_partition[0] = 0;
partsize_t myTotalNbParticles = 0;
for(int idxPartition = 0 ; idxPartition < current_partition_size ; ++idxPartition){
@@ -528,16 +596,26 @@ public:
// Find particles outside my interval
const partsize_t nbOutLower = particles_utils::partition_extra<partsize_t, size_particle_positions>(&(*inout_positions_particles)[0], current_my_nb_particles_per_partition[0],
[&](const real_number val[]){
- const int partition_level = in_computer.pbc_field_layer(val[IDX_Z], IDX_Z);
+ const int partition_level = in_computer.pbc_field_layer(val[IDXC_Z], IDXC_Z);
assert(partition_level == current_partition_interval.first
- || partition_level == (current_partition_interval.first-1+int(field_grid_dim[IDX_Z]))%int(field_grid_dim[IDX_Z])
- || partition_level == (current_partition_interval.first+1)%int(field_grid_dim[IDX_Z]));
- const bool isLower = partition_level == (current_partition_interval.first-1+int(field_grid_dim[IDX_Z]))%int(field_grid_dim[IDX_Z]);
+ || partition_level == (current_partition_interval.first-1+int(field_grid_dim[IDXC_Z]))%int(field_grid_dim[IDXC_Z])
+ || partition_level == (current_partition_interval.first+1)%int(field_grid_dim[IDXC_Z]));
+ const bool isLower = partition_level == (current_partition_interval.first-1+int(field_grid_dim[IDXC_Z]))%int(field_grid_dim[IDXC_Z]);
return isLower;
},
[&](const partsize_t idx1, const partsize_t idx2){
for(int idx_val = 0 ; idx_val < size_particle_index ; ++idx_val){
- std::swap((*inout_index_particles)[idx1], (*inout_index_particles)[idx2]);
+ std::swap((*inout_index_particles)[size_particle_index*idx1+idx_val],
+ (*inout_index_particles)[size_particle_index*idx2+idx_val]);
+ }
+
+ if (labels_present)
+ {
+ for(int idx_val = 0 ; idx_val < size_particle_index ; ++idx_val)
+ {
+ std::swap((*inout_label_particles)[size_particle_index*idx1+idx_val],
+ (*inout_label_particles)[size_particle_index*idx2+idx_val]);
+ }
}
for(int idx_rhs = 0 ; idx_rhs < in_nb_rhs ; ++idx_rhs){
@@ -553,16 +631,26 @@ public:
&(*inout_positions_particles)[(current_offset_particles_for_partition[current_partition_size-1]+offesetOutLow)*size_particle_positions],
myTotalNbParticles - (current_offset_particles_for_partition[current_partition_size-1]+offesetOutLow),
[&](const real_number val[]){
- const int partition_level = in_computer.pbc_field_layer(val[IDX_Z], IDX_Z);
+ const int partition_level = in_computer.pbc_field_layer(val[IDXC_Z], IDXC_Z);
assert(partition_level == (current_partition_interval.second-1)
- || partition_level == ((current_partition_interval.second-1)-1+int(field_grid_dim[IDX_Z]))%int(field_grid_dim[IDX_Z])
- || partition_level == ((current_partition_interval.second-1)+1)%int(field_grid_dim[IDX_Z]));
- const bool isUpper = (partition_level == ((current_partition_interval.second-1)+1)%int(field_grid_dim[IDX_Z]));
+ || partition_level == ((current_partition_interval.second-1)-1+int(field_grid_dim[IDXC_Z]))%int(field_grid_dim[IDXC_Z])
+ || partition_level == ((current_partition_interval.second-1)+1)%int(field_grid_dim[IDXC_Z]));
+ const bool isUpper = (partition_level == ((current_partition_interval.second-1)+1)%int(field_grid_dim[IDXC_Z]));
return !isUpper;
},
[&](const partsize_t idx1, const partsize_t idx2){
for(int idx_val = 0 ; idx_val < size_particle_index ; ++idx_val){
- std::swap((*inout_index_particles)[idx1], (*inout_index_particles)[idx2]);
+ std::swap((*inout_index_particles)[size_particle_index*idx1+idx_val],
+ (*inout_index_particles)[size_particle_index*idx2+idx_val]);
+ }
+
+ if (labels_present)
+ {
+ for(int idx_val = 0 ; idx_val < size_particle_index ; ++idx_val)
+ {
+ std::swap((*inout_label_particles)[size_particle_index*idx1+idx_val],
+ (*inout_label_particles)[size_particle_index*idx2+idx_val]);
+ }
}
for(int idx_rhs = 0 ; idx_rhs < in_nb_rhs ; ++idx_rhs){
@@ -581,6 +669,8 @@ public:
std::unique_ptr<real_number[]> newParticlesUp;
std::unique_ptr<partsize_t[]> newParticlesLowIndexes;
std::unique_ptr<partsize_t[]> newParticlesUpIndexes;
+ std::unique_ptr<partsize_t[]> newParticlesLowLabels;
+ std::unique_ptr<partsize_t[]> newParticlesUpLabels;
std::vector<std::unique_ptr<real_number[]>> newParticlesLowRhs(in_nb_rhs);
std::vector<std::unique_ptr<real_number[]>> newParticlesUpRhs(in_nb_rhs);
@@ -591,34 +681,42 @@ public:
whatNext.emplace_back(std::pair<Action,int>{RECV_MOVE_NB_LOW, -1});
mpiRequests.emplace_back();
AssertMpi(MPI_Irecv(&nbNewFromLow, 1, particles_utils::GetMpiType(partsize_t()),
- (my_rank-1+nb_processes_involved)%nb_processes_involved, TAG_UP_LOW_MOVED_NB_PARTICLES,
+ (my_rank-1+nb_processes_involved)%nb_processes_involved, TAG_SHIFT_OFFSET*counter_shift_tags + TAG_UP_LOW_MOVED_NB_PARTICLES,
MPI_COMM_WORLD, &mpiRequests.back()));
eventsBeforeWaitall += 1;
whatNext.emplace_back(std::pair<Action,int>{NOTHING_TODO, -1});
mpiRequests.emplace_back();
AssertMpi(MPI_Isend(const_cast<partsize_t*>(&nbOutLower), 1, particles_utils::GetMpiType(partsize_t()),
- (my_rank-1+nb_processes_involved)%nb_processes_involved, TAG_LOW_UP_MOVED_NB_PARTICLES,
+ (my_rank-1+nb_processes_involved)%nb_processes_involved, TAG_SHIFT_OFFSET*counter_shift_tags + TAG_LOW_UP_MOVED_NB_PARTICLES,
MPI_COMM_WORLD, &mpiRequests.back()));
if(nbOutLower){
whatNext.emplace_back(std::pair<Action,int>{NOTHING_TODO, -1});
- mpiRequests.emplace_back();
+ mpiRequests.emplace_back();
assert(nbOutLower*size_particle_positions < std::numeric_limits<int>::max());
- AssertMpi(MPI_Isend(&(*inout_positions_particles)[0], int(nbOutLower*size_particle_positions), particles_utils::GetMpiType(real_number()), (my_rank-1+nb_processes_involved)%nb_processes_involved, TAG_LOW_UP_MOVED_PARTICLES,
+ AssertMpi(MPI_Isend(&(*inout_positions_particles)[0], int(nbOutLower*size_particle_positions), particles_utils::GetMpiType(real_number()), (my_rank-1+nb_processes_involved)%nb_processes_involved, TAG_SHIFT_OFFSET*counter_shift_tags + TAG_LOW_UP_MOVED_PARTICLES,
MPI_COMM_WORLD, &mpiRequests.back()));
+
whatNext.emplace_back(std::pair<Action,int>{NOTHING_TODO, -1});
mpiRequests.emplace_back();
- assert(nbOutLower < std::numeric_limits<int>::max());
- AssertMpi(MPI_Isend(&(*inout_index_particles)[0], int(nbOutLower), particles_utils::GetMpiType(partsize_t()),
- (my_rank-1+nb_processes_involved)%nb_processes_involved, TAG_LOW_UP_MOVED_PARTICLES_INDEXES,
+ assert(nbOutLower*size_particle_index < std::numeric_limits<int>::max());
+ AssertMpi(MPI_Isend(&(*inout_index_particles)[0], int(nbOutLower*size_particle_index), particles_utils::GetMpiType(partsize_t()),
+ (my_rank-1+nb_processes_involved)%nb_processes_involved, TAG_SHIFT_OFFSET*counter_shift_tags + TAG_LOW_UP_MOVED_PARTICLES_INDEXES,
MPI_COMM_WORLD, &mpiRequests.back()));
+ if (labels_present)
+ {
+ AssertMpi(MPI_Isend(&(*inout_label_particles)[0], int(nbOutLower*size_particle_index), particles_utils::GetMpiType(partsize_t()),
+ (my_rank-1+nb_processes_involved)%nb_processes_involved, TAG_SHIFT_OFFSET*counter_shift_tags + TAG_LOW_UP_MOVED_PARTICLES_LABELS,
+ MPI_COMM_WORLD, &mpiRequests.back()));
+ }
+
for(int idx_rhs = 0 ; idx_rhs < in_nb_rhs ; ++idx_rhs){
whatNext.emplace_back(std::pair<Action,int>{NOTHING_TODO, -1});
mpiRequests.emplace_back();
assert(nbOutLower*size_particle_rhs < std::numeric_limits<int>::max());
- AssertMpi(MPI_Isend(&inout_rhs_particles[idx_rhs][0], int(nbOutLower*size_particle_rhs), particles_utils::GetMpiType(real_number()), (my_rank-1+nb_processes_involved)%nb_processes_involved, TAG_LOW_UP_MOVED_PARTICLES_RHS+idx_rhs,
+ AssertMpi(MPI_Isend(&inout_rhs_particles[idx_rhs][0], int(nbOutLower*size_particle_rhs), particles_utils::GetMpiType(real_number()), (my_rank-1+nb_processes_involved)%nb_processes_involved, TAG_SHIFT_OFFSET*counter_shift_tags + TAG_LOW_UP_MOVED_PARTICLES_RHS+idx_rhs,
MPI_COMM_WORLD, &mpiRequests.back()));
}
}
@@ -626,14 +724,14 @@ public:
whatNext.emplace_back(std::pair<Action,int>{RECV_MOVE_NB_UP, -1});
mpiRequests.emplace_back();
AssertMpi(MPI_Irecv(&nbNewFromUp, 1, particles_utils::GetMpiType(partsize_t()), (my_rank+1)%nb_processes_involved,
- TAG_LOW_UP_MOVED_NB_PARTICLES,
+ TAG_SHIFT_OFFSET*counter_shift_tags + TAG_LOW_UP_MOVED_NB_PARTICLES,
MPI_COMM_WORLD, &mpiRequests.back()));
eventsBeforeWaitall += 1;
whatNext.emplace_back(std::pair<Action,int>{NOTHING_TODO, -1});
mpiRequests.emplace_back();
AssertMpi(MPI_Isend(const_cast<partsize_t*>(&nbOutUpper), 1, particles_utils::GetMpiType(partsize_t()),
- (my_rank+1)%nb_processes_involved, TAG_UP_LOW_MOVED_NB_PARTICLES,
+ (my_rank+1)%nb_processes_involved, TAG_SHIFT_OFFSET*counter_shift_tags + TAG_UP_LOW_MOVED_NB_PARTICLES,
MPI_COMM_WORLD, &mpiRequests.back()));
if(nbOutUpper){
@@ -641,22 +739,28 @@ public:
mpiRequests.emplace_back();
assert(nbOutUpper*size_particle_positions < std::numeric_limits<int>::max());
AssertMpi(MPI_Isend(&(*inout_positions_particles)[(myTotalNbParticles-nbOutUpper)*size_particle_positions],
- int(nbOutUpper*size_particle_positions), particles_utils::GetMpiType(real_number()), (my_rank+1)%nb_processes_involved, TAG_UP_LOW_MOVED_PARTICLES,
+ int(nbOutUpper*size_particle_positions), particles_utils::GetMpiType(real_number()), (my_rank+1)%nb_processes_involved, TAG_SHIFT_OFFSET*counter_shift_tags + TAG_UP_LOW_MOVED_PARTICLES,
MPI_COMM_WORLD, &mpiRequests.back()));
+
whatNext.emplace_back(std::pair<Action,int>{NOTHING_TODO, -1});
mpiRequests.emplace_back();
- assert(nbOutUpper < std::numeric_limits<int>::max());
- AssertMpi(MPI_Isend(&(*inout_index_particles)[(myTotalNbParticles-nbOutUpper)], int(nbOutUpper),
- particles_utils::GetMpiType(partsize_t()), (my_rank+1)%nb_processes_involved, TAG_UP_LOW_MOVED_PARTICLES_INDEXES,
+ assert(nbOutUpper*size_particle_index < std::numeric_limits<int>::max());
+ AssertMpi(MPI_Isend(&(*inout_index_particles)[(myTotalNbParticles-nbOutUpper)*size_particle_index], int(nbOutUpper*size_particle_index),
+ particles_utils::GetMpiType(partsize_t()), (my_rank+1)%nb_processes_involved, TAG_SHIFT_OFFSET*counter_shift_tags + TAG_UP_LOW_MOVED_PARTICLES_INDEXES,
MPI_COMM_WORLD, &mpiRequests.back()));
-
+ if (labels_present)
+ {
+ AssertMpi(MPI_Isend(&(*inout_label_particles)[(myTotalNbParticles-nbOutUpper)*size_particle_index], int(nbOutUpper*size_particle_index),
+ particles_utils::GetMpiType(partsize_t()), (my_rank+1)%nb_processes_involved, TAG_SHIFT_OFFSET*counter_shift_tags + TAG_UP_LOW_MOVED_PARTICLES_LABELS,
+ MPI_COMM_WORLD, &mpiRequests.back()));
+ }
for(int idx_rhs = 0 ; idx_rhs < in_nb_rhs ; ++idx_rhs){
whatNext.emplace_back(std::pair<Action,int>{NOTHING_TODO, -1});
mpiRequests.emplace_back();
assert(nbOutUpper*size_particle_rhs < std::numeric_limits<int>::max());
AssertMpi(MPI_Isend(&inout_rhs_particles[idx_rhs][(myTotalNbParticles-nbOutUpper)*size_particle_rhs],
- int(nbOutUpper*size_particle_rhs), particles_utils::GetMpiType(real_number()), (my_rank+1)%nb_processes_involved, TAG_UP_LOW_MOVED_PARTICLES_RHS+idx_rhs,
+ int(nbOutUpper*size_particle_rhs), particles_utils::GetMpiType(real_number()), (my_rank+1)%nb_processes_involved, TAG_SHIFT_OFFSET*counter_shift_tags + TAG_UP_LOW_MOVED_PARTICLES_RHS+idx_rhs,
MPI_COMM_WORLD, &mpiRequests.back()));
}
}
@@ -681,23 +785,36 @@ public:
mpiRequests.emplace_back();
assert(nbNewFromLow*size_particle_positions < std::numeric_limits<int>::max());
AssertMpi(MPI_Irecv(&newParticlesLow[0], int(nbNewFromLow*size_particle_positions), particles_utils::GetMpiType(real_number()),
- (my_rank-1+nb_processes_involved)%nb_processes_involved, TAG_UP_LOW_MOVED_PARTICLES,
+ (my_rank-1+nb_processes_involved)%nb_processes_involved, TAG_SHIFT_OFFSET*counter_shift_tags + TAG_UP_LOW_MOVED_PARTICLES,
MPI_COMM_WORLD, &mpiRequests.back()));
- newParticlesLowIndexes.reset(new partsize_t[nbNewFromLow]);
+ newParticlesLowIndexes.reset(new partsize_t[nbNewFromLow*size_particle_index]);
whatNext.emplace_back(std::pair<Action,int>{NOTHING_TODO, -1});
mpiRequests.emplace_back();
- assert(nbNewFromLow < std::numeric_limits<int>::max());
- AssertMpi(MPI_Irecv(&newParticlesLowIndexes[0], int(nbNewFromLow), particles_utils::GetMpiType(partsize_t()),
- (my_rank-1+nb_processes_involved)%nb_processes_involved, TAG_UP_LOW_MOVED_PARTICLES_INDEXES,
+ assert(nbNewFromLow*size_particle_index < std::numeric_limits<int>::max());
+ AssertMpi(MPI_Irecv(&newParticlesLowIndexes[0], int(nbNewFromLow*size_particle_index),
+ particles_utils::GetMpiType(partsize_t()),
+ (my_rank-1+nb_processes_involved)%nb_processes_involved, TAG_SHIFT_OFFSET*counter_shift_tags + TAG_UP_LOW_MOVED_PARTICLES_INDEXES,
MPI_COMM_WORLD, &mpiRequests.back()));
+ if (labels_present)
+ {
+ newParticlesLowLabels.reset(new partsize_t[nbNewFromLow*size_particle_index]);
+ whatNext.emplace_back(std::pair<Action,int>{NOTHING_TODO, -1});
+ mpiRequests.emplace_back();
+ assert(nbNewFromLow*size_particle_index < std::numeric_limits<int>::max());
+ AssertMpi(MPI_Irecv(&newParticlesLowLabels[0], int(nbNewFromLow*size_particle_index),
+ particles_utils::GetMpiType(partsize_t()),
+ (my_rank-1+nb_processes_involved)%nb_processes_involved, TAG_SHIFT_OFFSET*counter_shift_tags + TAG_UP_LOW_MOVED_PARTICLES_LABELS,
+ MPI_COMM_WORLD, &mpiRequests.back()));
+ }
+
for(int idx_rhs = 0 ; idx_rhs < in_nb_rhs ; ++idx_rhs){
newParticlesLowRhs[idx_rhs].reset(new real_number[nbNewFromLow*size_particle_rhs]);
whatNext.emplace_back(std::pair<Action,int>{NOTHING_TODO, -1});
mpiRequests.emplace_back();
assert(nbNewFromLow*size_particle_rhs < std::numeric_limits<int>::max());
- AssertMpi(MPI_Irecv(&newParticlesLowRhs[idx_rhs][0], int(nbNewFromLow*size_particle_rhs), particles_utils::GetMpiType(real_number()), (my_rank-1+nb_processes_involved)%nb_processes_involved, TAG_UP_LOW_MOVED_PARTICLES_RHS+idx_rhs,
+ AssertMpi(MPI_Irecv(&newParticlesLowRhs[idx_rhs][0], int(nbNewFromLow*size_particle_rhs), particles_utils::GetMpiType(real_number()), (my_rank-1+nb_processes_involved)%nb_processes_involved, TAG_SHIFT_OFFSET*counter_shift_tags + TAG_UP_LOW_MOVED_PARTICLES_RHS+idx_rhs,
MPI_COMM_WORLD, &mpiRequests.back()));
}
}
@@ -710,23 +827,36 @@ public:
whatNext.emplace_back(std::pair<Action,int>{RECV_MOVE_UP, -1});
mpiRequests.emplace_back();
assert(nbNewFromUp*size_particle_positions < std::numeric_limits<int>::max());
- AssertMpi(MPI_Irecv(&newParticlesUp[0], int(nbNewFromUp*size_particle_positions), particles_utils::GetMpiType(real_number()), (my_rank+1)%nb_processes_involved, TAG_LOW_UP_MOVED_PARTICLES,
+ AssertMpi(MPI_Irecv(&newParticlesUp[0], int(nbNewFromUp*size_particle_positions), particles_utils::GetMpiType(real_number()), (my_rank+1)%nb_processes_involved, TAG_SHIFT_OFFSET*counter_shift_tags + TAG_LOW_UP_MOVED_PARTICLES,
MPI_COMM_WORLD, &mpiRequests.back()));
- newParticlesUpIndexes.reset(new partsize_t[nbNewFromUp]);
+ newParticlesUpIndexes.reset(new partsize_t[nbNewFromUp*size_particle_index]);
whatNext.emplace_back(std::pair<Action,int>{NOTHING_TODO, -1});
mpiRequests.emplace_back();
- assert(nbNewFromUp < std::numeric_limits<int>::max());
- AssertMpi(MPI_Irecv(&newParticlesUpIndexes[0], int(nbNewFromUp), particles_utils::GetMpiType(partsize_t()),
- (my_rank+1)%nb_processes_involved, TAG_LOW_UP_MOVED_PARTICLES_INDEXES,
+ assert(nbNewFromUp*size_particle_index < std::numeric_limits<int>::max());
+ AssertMpi(MPI_Irecv(&newParticlesUpIndexes[0], int(nbNewFromUp*size_particle_index),
+ particles_utils::GetMpiType(partsize_t()),
+ (my_rank+1)%nb_processes_involved, TAG_SHIFT_OFFSET*counter_shift_tags + TAG_LOW_UP_MOVED_PARTICLES_INDEXES,
MPI_COMM_WORLD, &mpiRequests.back()));
+ if (labels_present)
+ {
+ newParticlesUpLabels.reset(new partsize_t[nbNewFromUp*size_particle_index]);
+ whatNext.emplace_back(std::pair<Action,int>{NOTHING_TODO, -1});
+ mpiRequests.emplace_back();
+ assert(nbNewFromUp*size_particle_index < std::numeric_limits<int>::max());
+ AssertMpi(MPI_Irecv(&newParticlesUpLabels[0], int(nbNewFromUp*size_particle_index),
+ particles_utils::GetMpiType(partsize_t()),
+ (my_rank+1)%nb_processes_involved, TAG_SHIFT_OFFSET*counter_shift_tags + TAG_LOW_UP_MOVED_PARTICLES_LABELS,
+ MPI_COMM_WORLD, &mpiRequests.back()));
+ }
+
for(int idx_rhs = 0 ; idx_rhs < in_nb_rhs ; ++idx_rhs){
newParticlesUpRhs[idx_rhs].reset(new real_number[nbNewFromUp*size_particle_rhs]);
whatNext.emplace_back(std::pair<Action,int>{NOTHING_TODO, -1});
mpiRequests.emplace_back();
assert(nbNewFromUp*size_particle_rhs < std::numeric_limits<int>::max());
- AssertMpi(MPI_Irecv(&newParticlesUpRhs[idx_rhs][0], int(nbNewFromUp*size_particle_rhs), particles_utils::GetMpiType(real_number()), (my_rank+1)%nb_processes_involved, TAG_LOW_UP_MOVED_PARTICLES_RHS+idx_rhs,
+ AssertMpi(MPI_Irecv(&newParticlesUpRhs[idx_rhs][0], int(nbNewFromUp*size_particle_rhs), particles_utils::GetMpiType(real_number()), (my_rank+1)%nb_processes_involved, TAG_SHIFT_OFFSET*counter_shift_tags + TAG_LOW_UP_MOVED_PARTICLES_RHS+idx_rhs,
MPI_COMM_WORLD, &mpiRequests.back()));
}
}
@@ -750,7 +880,8 @@ public:
const partsize_t myTotalNewNbParticles = nbOldParticlesInside + nbNewFromLow + nbNewFromUp;
std::unique_ptr<real_number[]> newArray(new real_number[myTotalNewNbParticles*size_particle_positions]);
- std::unique_ptr<partsize_t[]> newArrayIndexes(new partsize_t[myTotalNewNbParticles]);
+ std::unique_ptr<partsize_t[]> newArrayIndexes(new partsize_t[myTotalNewNbParticles*size_particle_index]);
+ std::unique_ptr<partsize_t[]> newArrayLabels(new partsize_t[myTotalNewNbParticles*size_particle_index]);
std::vector<std::unique_ptr<real_number[]>> newArrayRhs(in_nb_rhs);
for(int idx_rhs = 0 ; idx_rhs < in_nb_rhs ; ++idx_rhs){
newArrayRhs[idx_rhs].reset(new real_number[myTotalNewNbParticles*size_particle_rhs]);
@@ -760,7 +891,9 @@ public:
if(nbNewFromLow){
const particles_utils::fixed_copy fcp(0, 0, nbNewFromLow);
fcp.copy(newArray, newParticlesLow, size_particle_positions);
- fcp.copy(newArrayIndexes, newParticlesLowIndexes);
+ fcp.copy(newArrayIndexes, newParticlesLowIndexes, size_particle_index);
+ if (labels_present)
+ fcp.copy(newArrayLabels, newParticlesLowLabels, size_particle_index);
for(int idx_rhs = 0 ; idx_rhs < in_nb_rhs ; ++idx_rhs){
fcp.copy(newArrayRhs[idx_rhs], newParticlesLowRhs[idx_rhs], size_particle_rhs);
}
@@ -770,7 +903,11 @@ public:
{
const particles_utils::fixed_copy fcp(nbNewFromLow, nbOutLower, nbOldParticlesInside);
fcp.copy(newArray, (*inout_positions_particles), size_particle_positions);
- fcp.copy(newArrayIndexes, (*inout_index_particles));
+ fcp.copy(newArrayIndexes, (*inout_index_particles), size_particle_index);
+ if (labels_present)
+ {
+ fcp.copy(newArrayLabels, (*inout_label_particles), size_particle_index);
+ }
for(int idx_rhs = 0 ; idx_rhs < in_nb_rhs ; ++idx_rhs){
fcp.copy(newArrayRhs[idx_rhs], inout_rhs_particles[idx_rhs], size_particle_rhs);
}
@@ -780,7 +917,9 @@ public:
if(nbNewFromUp){
const particles_utils::fixed_copy fcp(nbNewFromLow+nbOldParticlesInside, 0, nbNewFromUp);
fcp.copy(newArray, newParticlesUp, size_particle_positions);
- fcp.copy(newArrayIndexes, newParticlesUpIndexes);
+ fcp.copy(newArrayIndexes, newParticlesUpIndexes, size_particle_index);
+ if (labels_present)
+ fcp.copy(newArrayLabels, newParticlesUpLabels, size_particle_index);
for(int idx_rhs = 0 ; idx_rhs < in_nb_rhs ; ++idx_rhs){
fcp.copy(newArrayRhs[idx_rhs], newParticlesUpRhs[idx_rhs], size_particle_rhs);
}
@@ -788,6 +927,10 @@ public:
(*inout_positions_particles) = std::move(newArray);
(*inout_index_particles) = std::move(newArrayIndexes);
+ if (labels_present)
+ {
+ (*inout_label_particles) = std::move(newArrayLabels);
+ }
for(int idx_rhs = 0 ; idx_rhs < in_nb_rhs ; ++idx_rhs){
inout_rhs_particles[idx_rhs] = std::move(newArrayRhs[idx_rhs]);
}
@@ -802,13 +945,21 @@ public:
myTotalNbParticles,current_partition_size,
current_my_nb_particles_per_partition, current_offset_particles_for_partition.get(),
[&](const real_number& z_pos){
- const int partition_level = in_computer.pbc_field_layer(z_pos, IDX_Z);
+ const int partition_level = in_computer.pbc_field_layer(z_pos, IDXC_Z);
assert(current_partition_interval.first <= partition_level && partition_level < current_partition_interval.second);
return partition_level - current_partition_interval.first;
},
[&](const partsize_t idx1, const partsize_t idx2){
for(int idx_val = 0 ; idx_val < size_particle_index ; ++idx_val){
- std::swap((*inout_index_particles)[idx1], (*inout_index_particles)[idx2]);
+ std::swap((*inout_index_particles)[size_particle_index*idx1 + idx_val],
+ (*inout_index_particles)[size_particle_index*idx2 + idx_val]);
+ }
+ if (labels_present)
+ {
+ for(int idx_val = 0 ; idx_val < size_particle_index ; ++idx_val){
+ std::swap((*inout_label_particles)[size_particle_index*idx1 + idx_val],
+ (*inout_label_particles)[size_particle_index*idx2 + idx_val]);
+ }
}
for(int idx_rhs = 0 ; idx_rhs < in_nb_rhs ; ++idx_rhs){
@@ -824,7 +975,7 @@ public:
assert(current_my_nb_particles_per_partition[idxPartition] ==
current_offset_particles_for_partition[idxPartition+1] - current_offset_particles_for_partition[idxPartition]);
for(partsize_t idx = current_offset_particles_for_partition[idxPartition] ; idx < current_offset_particles_for_partition[idxPartition+1] ; ++idx){
- assert(in_computer.pbc_field_layer((*inout_positions_particles)[idx*3+IDX_Z], IDX_Z)-current_partition_interval.first == idxPartition);
+ assert(in_computer.pbc_field_layer((*inout_positions_particles)[idx*size_particle_positions+IDXC_Z], IDXC_Z)-current_partition_interval.first == idxPartition);
}
}
}
@@ -832,6 +983,8 @@ public:
(*nb_particles) = myTotalNbParticles;
assert(mpiRequests.size() == 0);
+
+ counter_shift_tags += 1;
}
};
diff --git a/cpp/particles/particles_input_grid.hpp b/cpp/particles/particles_input_grid.hpp
new file mode 100644
index 0000000000000000000000000000000000000000..784c08e0623b602dc6e81a7b4bd13ed3c8a8dfe0
--- /dev/null
+++ b/cpp/particles/particles_input_grid.hpp
@@ -0,0 +1,238 @@
+/******************************************************************************
+* *
+* Copyright 2020 Max Planck Institute for Dynamics and Self-Organization *
+* *
+* This file is part of TurTLE. *
+* *
+* TurTLE is free software: you can redistribute it and/or modify *
+* it under the terms of the GNU General Public License as published *
+* by the Free Software Foundation, either version 3 of the License, *
+* or (at your option) any later version. *
+* *
+* TurTLE is distributed in the hope that it will be useful, *
+* but WITHOUT ANY WARRANTY; without even the implied warranty of *
+* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the *
+* GNU General Public License for more details. *
+* *
+* You should have received a copy of the GNU General Public License *
+* along with TurTLE. If not, see <http://www.gnu.org/licenses/> *
+* *
+* Contact: Cristian.Lalescu@ds.mpg.de *
+* *
+******************************************************************************/
+
+
+
+#ifndef PARTICLES_INPUT_GRID_HPP
+#define PARTICLES_INPUT_GRID_HPP
+
+#include <mpi.h>
+#include <cassert>
+#include "particles/abstract_particles_input.hpp"
+
+template <class partsize_t, class particle_rnumber, int state_size>
+class particles_input_grid: public abstract_particles_input<partsize_t, particle_rnumber>
+{
+ private:
+ MPI_Comm comm;
+ int myrank, nprocs;
+
+ hsize_t total_number_of_particles;
+ hsize_t number_rhs;
+ const hsize_t nxparticles;
+ const hsize_t nzparticles;
+ partsize_t local_number_of_particles;
+
+ std::unique_ptr<particle_rnumber[]> local_particle_state;
+ std::unique_ptr<partsize_t[]> local_particle_index;
+ std::unique_ptr<partsize_t[]> local_particle_label;
+ std::vector<std::unique_ptr<particle_rnumber[]>> local_particle_rhs;
+ public:
+ ~particles_input_grid() noexcept(false){}
+ particles_input_grid(
+ const MPI_Comm in_mpi_comm,
+ const partsize_t NXPARTICLES,
+ const partsize_t NZPARTICLES,
+ const particle_rnumber my_spatial_low_limit,
+ const particle_rnumber my_spatial_up_limit):
+ comm(in_mpi_comm),
+ nxparticles(NXPARTICLES),
+ nzparticles(NZPARTICLES)
+ {
+ TIMEZONE("particles_input_grid::particles_input_grid");
+ assert(state_size >= 3);
+ static_assert((std::is_same<particle_rnumber, double>::value ||
+ std::is_same<particle_rnumber, float>::value),
+ "real_number must be double or float");
+ AssertMpi(MPI_Comm_rank(comm, &myrank));
+ AssertMpi(MPI_Comm_size(comm, &nprocs));
+ std::vector<particle_rnumber> in_spatial_limit_per_prod = BuildLimitsAllProcesses<particle_rnumber>(
+ comm, my_spatial_low_limit, my_spatial_up_limit);
+ assert(int(in_spatial_limit_per_prod.size()) == nprocs+1);
+
+ const double twopi = 4*acos(0);
+ const double grid_distancex = twopi / this->nxparticles;
+ const double grid_distancez = twopi / this->nzparticles;
+
+ this->total_number_of_particles = this->nxparticles*this->nzparticles;
+
+ // we know the total number of particles, but we want to generate particle locations in parallel.
+ // so we need a preliminary distributor of particles, location-agnostic:
+ particles_utils::IntervalSplitter<hsize_t> load_splitter(
+ this->total_number_of_particles, this->nprocs, this->myrank);
+
+ // allocate array for preliminary particle data
+ std::unique_ptr<particle_rnumber[]> split_particle_state;
+ if(load_splitter.getMySize())
+ {
+ split_particle_state.reset(new particle_rnumber[load_splitter.getMySize()*state_size]);
+ }
+
+ // allocate and populate array for preliminary particle indices
+ std::unique_ptr<partsize_t[]> split_particle_index;
+ if(load_splitter.getMySize())
+ {
+ split_particle_index.reset(new partsize_t[load_splitter.getMySize()]);
+ }
+ for(partsize_t idx = 0 ; idx < partsize_t(load_splitter.getMySize()) ; idx++){
+ split_particle_index[idx] = idx + partsize_t(load_splitter.getMyOffset());
+ }
+
+ {
+ TIMEZONE("particles_input_grid::compute particle states");
+ for (partsize_t idx=0; idx < partsize_t(load_splitter.getMySize()); idx++)
+ {
+ // compute position within grid
+ const int ix = split_particle_index[idx] % this->nxparticles;
+ const int iz = split_particle_index[idx] / this->nxparticles;
+ split_particle_state[idx*state_size + IDXC_X] = grid_distancex * ix;
+ split_particle_state[idx*state_size + IDXC_Y] = twopi / 7;
+ split_particle_state[idx*state_size + IDXC_Z] = grid_distancez * iz;
+ }
+ }
+
+
+ /*************************************************/
+ // meta: permutation and echange regions are copy/pasted from particles_input_hdf5
+ // the two blocks should agree (other than type/variable renaming, and the removal of rhs stuff).
+ // Permute
+ std::vector<particle_rnumber> in_spatial_limit_per_proc = BuildLimitsAllProcesses<particle_rnumber>(
+ this->comm,
+ my_spatial_low_limit,
+ my_spatial_up_limit);
+ std::vector<partsize_t> nb_particles_per_proc(this->nprocs);
+ {
+ TIMEZONE("particles_input_grid::partition");
+
+ const particle_rnumber spatial_box_offset = in_spatial_limit_per_proc[0];
+ const particle_rnumber spatial_box_width =
+ in_spatial_limit_per_proc[this->nprocs] - in_spatial_limit_per_proc[0];
+
+ partsize_t previousOffset = 0;
+ for(int idx_proc = 0 ; idx_proc < this->nprocs-1 ; ++idx_proc){
+ const particle_rnumber limitPartitionShifted =
+ in_spatial_limit_per_proc[idx_proc+1] - spatial_box_offset;
+ const partsize_t localOffset = particles_utils::partition_extra<partsize_t, state_size>(
+ &split_particle_state[previousOffset*state_size],
+ partsize_t(load_splitter.getMySize())-previousOffset,
+ [&](const particle_rnumber val[]){
+ const particle_rnumber shiftPos = val[IDXC_Z] - spatial_box_offset;
+ const particle_rnumber nbRepeat = floor(shiftPos/spatial_box_width);
+ const particle_rnumber posInBox = shiftPos - (spatial_box_width*nbRepeat);
+ return posInBox < limitPartitionShifted;
+ },
+ [&](const partsize_t idx1, const partsize_t idx2){
+ std::swap(split_particle_index[idx1],
+ split_particle_index[idx2]);
+ },
+ previousOffset);
+
+ nb_particles_per_proc[idx_proc] = localOffset;
+ previousOffset += localOffset;
+ }
+ nb_particles_per_proc[this->nprocs-1] = partsize_t(load_splitter.getMySize()) - previousOffset;
+ }
+
+ {
+ TIMEZONE("particles_input_grid::exchanger");
+ alltoall_exchanger exchanger(
+ this->comm,
+ std::move(nb_particles_per_proc));
+ // nb_particles_per_processes cannot be used after due to move
+
+ this->local_number_of_particles = exchanger.getTotalToRecv();
+
+ if(this->local_number_of_particles){
+ this->local_particle_state.reset(new particle_rnumber[exchanger.getTotalToRecv()*state_size]);
+ }
+ exchanger.alltoallv<particle_rnumber>(
+ split_particle_state.get(),
+ this->local_particle_state.get(),
+ state_size);
+ delete[] split_particle_state.release();
+
+ if(this->local_number_of_particles){
+ this->local_particle_index.reset(new partsize_t[exchanger.getTotalToRecv()]);
+ }
+ exchanger.alltoallv<partsize_t>(
+ split_particle_index.get(),
+ this->local_particle_index.get());
+ delete[] split_particle_index.release();
+ }
+ /*************************************************/
+
+
+ // clone indices in labels:
+ local_particle_label.reset(new partsize_t[this->getLocalNbParticles()]);
+ std::copy(local_particle_index.get(),
+ local_particle_index.get()+this->getLocalNbParticles(),
+ local_particle_label.get());
+ }
+
+ partsize_t getTotalNbParticles()
+ {
+ return this->total_number_of_particles;
+ }
+ partsize_t getLocalNbParticles()
+ {
+ return this->local_number_of_particles;
+ }
+ int getNbRhs()
+ {
+ return 0;
+ }
+
+ std::unique_ptr<particle_rnumber[]> getMyParticles()
+ {
+ assert(this->local_particle_state != nullptr || this->local_number_of_particles == 0);
+ return std::move(this->local_particle_state);
+ }
+
+ std::unique_ptr<partsize_t[]> getMyParticlesIndexes()
+ {
+ assert(this->local_particle_index != nullptr || this->local_number_of_particles == 0);
+ return std::move(this->local_particle_index);
+ }
+
+ std::unique_ptr<partsize_t[]> getMyParticlesLabels()
+ {
+ assert(this->local_particle_label != nullptr || this->local_number_of_particles == 0);
+ return std::move(this->local_particle_label);
+ }
+
+ std::vector<std::unique_ptr<particle_rnumber[]>> getMyRhs()
+ {
+ return std::move(std::vector<std::unique_ptr<particle_rnumber[]>>());
+ }
+
+ std::vector<hsize_t> getParticleFileLayout()
+ {
+ return std::vector<hsize_t>({
+ this->nzparticles,
+ this->nxparticles,
+ });
+ }
+};
+
+
+#endif
diff --git a/bfps/cpp/particles/particles_input_hdf5.hpp b/cpp/particles/particles_input_hdf5.hpp
similarity index 59%
rename from bfps/cpp/particles/particles_input_hdf5.hpp
rename to cpp/particles/particles_input_hdf5.hpp
index 32cfec05ad854cd7f3ffd88d771418d0552237d8..b1ea662ed14eaf4e725c532429eb631ade4ef8b7 100644
--- a/bfps/cpp/particles/particles_input_hdf5.hpp
+++ b/cpp/particles/particles_input_hdf5.hpp
@@ -1,3 +1,28 @@
+/******************************************************************************
+* *
+* Copyright 2019 Max Planck Institute for Dynamics and Self-Organization *
+* *
+* This file is part of TurTLE. *
+* *
+* TurTLE is free software: you can redistribute it and/or modify *
+* it under the terms of the GNU General Public License as published *
+* by the Free Software Foundation, either version 3 of the License, *
+* or (at your option) any later version. *
+* *
+* TurTLE is distributed in the hope that it will be useful, *
+* but WITHOUT ANY WARRANTY; without even the implied warranty of *
+* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the *
+* GNU General Public License for more details. *
+* *
+* You should have received a copy of the GNU General Public License *
+* along with TurTLE. If not, see <http://www.gnu.org/licenses/> *
+* *
+* Contact: Cristian.Lalescu@ds.mpg.de *
+* *
+******************************************************************************/
+
+
+
#ifndef PARTICLES_INPUT_HDF5_HPP
#define PARTICLES_INPUT_HDF5_HPP
@@ -10,12 +35,11 @@
#include "abstract_particles_input.hpp"
#include "base.hpp"
#include "alltoall_exchanger.hpp"
-#include "particles_utils.hpp"
+#include "particles/particles_utils.hpp"
#include "scope_timer.hpp"
+#include "hdf5_tools.hpp"
-// why is "size_particle_rhs" a template parameter?
-// I think it's safe to assume this will always be 3.
template <class partsize_t, class real_number, int size_particle_positions, int size_particle_rhs>
class particles_input_hdf5 : public abstract_particles_input<partsize_t, real_number> {
const std::string filename;
@@ -24,52 +48,46 @@ class particles_input_hdf5 : public abstract_particles_input<partsize_t, real_nu
int my_rank;
int nb_processes;
- hsize_t nb_total_particles;
+ hsize_t total_number_of_particles;
hsize_t nb_rhs;
partsize_t nb_particles_for_me;
+ std::vector<hsize_t> particle_file_layout; // to hold the shape of initial condition array
std::unique_ptr<real_number[]> my_particles_positions;
std::unique_ptr<partsize_t[]> my_particles_indexes;
+ std::unique_ptr<partsize_t[]> my_particles_labels;
std::vector<std::unique_ptr<real_number[]>> my_particles_rhs;
- static std::vector<real_number> BuildLimitsAllProcesses(MPI_Comm mpi_comm,
- const real_number my_spatial_low_limit, const real_number my_spatial_up_limit){
- int my_rank;
- int nb_processes;
-
- AssertMpi(MPI_Comm_rank(mpi_comm, &my_rank));
- AssertMpi(MPI_Comm_size(mpi_comm, &nb_processes));
-
- std::vector<real_number> spatial_limit_per_proc(nb_processes*2);
-
- real_number intervalToSend[2] = {my_spatial_low_limit, my_spatial_up_limit};
- AssertMpi(MPI_Allgather(intervalToSend, 2, particles_utils::GetMpiType(real_number()),
- spatial_limit_per_proc.data(), 2, particles_utils::GetMpiType(real_number()), mpi_comm));
-
- for(int idx_proc = 0; idx_proc < nb_processes-1 ; ++idx_proc){
- assert(spatial_limit_per_proc[idx_proc*2] <= spatial_limit_per_proc[idx_proc*2+1]);
- assert(spatial_limit_per_proc[idx_proc*2+1] == spatial_limit_per_proc[(idx_proc+1)*2]);
- spatial_limit_per_proc[idx_proc+1] = spatial_limit_per_proc[idx_proc*2+1];
- }
- spatial_limit_per_proc[nb_processes] = spatial_limit_per_proc[(nb_processes-1)*2+1];
- spatial_limit_per_proc.resize(nb_processes+1);
-
- return spatial_limit_per_proc;
- }
-
public:
- particles_input_hdf5(const MPI_Comm in_mpi_comm,const std::string& inFilename,
- const std::string& inDatanameState, const std::string& inDatanameRhs,
- const real_number my_spatial_low_limit, const real_number my_spatial_up_limit)
- : particles_input_hdf5(in_mpi_comm, inFilename, inDatanameState, inDatanameRhs,
- BuildLimitsAllProcesses(in_mpi_comm, my_spatial_low_limit, my_spatial_up_limit)){
+ particles_input_hdf5(
+ const MPI_Comm in_mpi_comm,
+ const std::string& inFilename,
+ const std::string& inDatanameState,
+ const std::string& inDatanameRhs,
+ const real_number my_spatial_low_limit,
+ const real_number my_spatial_up_limit)
+ : particles_input_hdf5(
+ in_mpi_comm,
+ inFilename,
+ inDatanameState,
+ inDatanameRhs,
+ BuildLimitsAllProcesses<real_number>(
+ in_mpi_comm,
+ my_spatial_low_limit,
+ my_spatial_up_limit)){
}
- particles_input_hdf5(const MPI_Comm in_mpi_comm,const std::string& inFilename,
- const std::string& inDatanameState, const std::string& inDatanameRhs,
- const std::vector<real_number>& in_spatial_limit_per_proc)
+ particles_input_hdf5(
+ const MPI_Comm in_mpi_comm,
+ const std::string& inFilename,
+ const std::string& inDatanameState,
+ const std::string& inDatanameRhs,
+ const std::vector<real_number>& in_spatial_limit_per_proc)
: filename(inFilename),
- mpi_comm(in_mpi_comm), my_rank(-1), nb_processes(-1), nb_total_particles(0),
+ mpi_comm(in_mpi_comm),
+ my_rank(-1),
+ nb_processes(-1),
+ total_number_of_particles(0),
nb_particles_for_me(0){
TIMEZONE("particles_input_hdf5");
@@ -81,6 +99,7 @@ public:
assert(plist_id_par >= 0);
{
int retTest = H5Pset_fapl_mpio(plist_id_par, mpi_comm, MPI_INFO_NULL);
+ variable_used_only_in_assert(retTest);
assert(retTest >= 0);
}
@@ -100,13 +119,17 @@ public:
std::vector<hsize_t> state_dim_array(space_dim);
int hdfret = H5Sget_simple_extent_dims(dspace, &state_dim_array[0], NULL);
+ variable_used_only_in_assert(hdfret);
assert(hdfret >= 0);
// Last value is the position dim of the particles
assert(state_dim_array.back() == size_particle_positions);
- nb_total_particles = 1;
+ // compute total number of particles, store initial condition array shape
+ total_number_of_particles = 1;
+ particle_file_layout.resize(state_dim_array.size()-1);
for (size_t idx_dim = 0; idx_dim < state_dim_array.size()-1; ++idx_dim){
- nb_total_particles *= state_dim_array[idx_dim];
+ total_number_of_particles *= state_dim_array[idx_dim];
+ particle_file_layout[idx_dim] = state_dim_array[idx_dim];
}
hdfret = H5Sclose(dspace);
@@ -128,6 +151,7 @@ public:
// Chichi comment: wouldn't &rhs_dim_array.front() be safer?
int hdfret = H5Sget_simple_extent_dims(dspace, &rhs_dim_array[0], NULL);
+ variable_used_only_in_assert(hdfret);
assert(hdfret >= 0);
assert(rhs_dim_array.back() == size_particle_rhs);
// Chichi comment: this assertion will fail in general
@@ -140,30 +164,36 @@ public:
assert(hdfret >= 0);
}
- particles_utils::IntervalSplitter<hsize_t> load_splitter(nb_total_particles, nb_processes, my_rank);
+ particles_utils::IntervalSplitter<hsize_t> load_splitter(total_number_of_particles, nb_processes, my_rank);
static_assert(std::is_same<real_number, double>::value
|| std::is_same<real_number, float>::value, "real_number must be double or float");
- const hid_t type_id = (sizeof(real_number) == 8?H5T_NATIVE_DOUBLE:H5T_NATIVE_FLOAT);
+ const hid_t type_id = hdf5_tools::hdf5_type_id<real_number>();
/// Load the data
- std::unique_ptr<real_number[]> split_particles_positions(new real_number[load_splitter.getMySize()*size_particle_positions]);
+ std::unique_ptr<real_number[]> split_particles_positions;
+ if(load_splitter.getMySize()){
+ split_particles_positions.reset(new real_number[load_splitter.getMySize()*size_particle_positions]);
+ }
+
{
TIMEZONE("state-read");
hid_t dset = H5Dopen(particle_file, inDatanameState.c_str(), H5P_DEFAULT);
assert(dset >= 0);
- hid_t rspace = H5Dget_space(dset);
+ hsize_t file_space_dims[2] = {total_number_of_particles, size_particle_positions};
+ hid_t rspace = H5Screate_simple(2, file_space_dims, NULL);
assert(rspace >= 0);
hsize_t offset[2] = {load_splitter.getMyOffset(), 0};
- hsize_t mem_dims[2] = {load_splitter.getMySize(), 3};
+ hsize_t mem_dims[2] = {load_splitter.getMySize(), size_particle_positions};
hid_t mspace = H5Screate_simple(2, &mem_dims[0], NULL);
assert(mspace >= 0);
int rethdf = H5Sselect_hyperslab(rspace, H5S_SELECT_SET, offset,
NULL, mem_dims, NULL);
+ variable_used_only_in_assert(rethdf);
assert(rethdf >= 0);
rethdf = H5Dread(dset, type_id, mspace, rspace, H5P_DEFAULT, split_particles_positions.get());
assert(rethdf >= 0);
@@ -178,12 +208,14 @@ public:
TIMEZONE("rhs-read");
hid_t dset = H5Dopen(particle_file, inDatanameRhs.c_str(), H5P_DEFAULT);
assert(dset >= 0);
+ hsize_t file_space_dims[3] = {nb_rhs, total_number_of_particles, size_particle_rhs};
+ hid_t rspace = H5Screate_simple(3, file_space_dims, NULL);
+ assert(rspace >= 0);
for(hsize_t idx_rhs = 0 ; idx_rhs < nb_rhs ; ++idx_rhs){
- hid_t rspace = H5Dget_space(dset);
- assert(rspace >= 0);
-
- split_particles_rhs[idx_rhs].reset(new real_number[load_splitter.getMySize()*size_particle_rhs]);
+ if(load_splitter.getMySize()){
+ split_particles_rhs[idx_rhs].reset(new real_number[load_splitter.getMySize()*size_particle_rhs]);
+ }
hsize_t offset[3] = {idx_rhs, load_splitter.getMyOffset(), 0};
hsize_t mem_dims[3] = {1, load_splitter.getMySize(), size_particle_rhs};
@@ -193,21 +225,26 @@ public:
int rethdf = H5Sselect_hyperslab( rspace, H5S_SELECT_SET, offset,
NULL, mem_dims, NULL);
+ variable_used_only_in_assert(rethdf);
assert(rethdf >= 0);
rethdf = H5Dread(dset, type_id, mspace, rspace, H5P_DEFAULT, split_particles_rhs[idx_rhs].get());
assert(rethdf >= 0);
rethdf = H5Sclose(mspace);
assert(rethdf >= 0);
-
- rethdf = H5Sclose(rspace);
- assert(rethdf >= 0);
}
- int rethdf = H5Dclose(dset);
+
+ int rethdf = H5Sclose(rspace);
+ assert(rethdf >= 0);
+ rethdf = H5Dclose(dset);
+ variable_used_only_in_assert(rethdf);
assert(rethdf >= 0);
}
- std::unique_ptr<partsize_t[]> split_particles_indexes(new partsize_t[load_splitter.getMySize()]);
+ std::unique_ptr<partsize_t[]> split_particles_indexes;
+ if(load_splitter.getMySize()){
+ split_particles_indexes.reset(new partsize_t[load_splitter.getMySize()]);
+ }
for(partsize_t idx_part = 0 ; idx_part < partsize_t(load_splitter.getMySize()) ; ++idx_part){
split_particles_indexes[idx_part] = idx_part + partsize_t(load_splitter.getMyOffset());
}
@@ -227,7 +264,7 @@ public:
&split_particles_positions[previousOffset*size_particle_positions],
partsize_t(load_splitter.getMySize())-previousOffset,
[&](const real_number val[]){
- const real_number shiftPos = val[IDX_Z]-spatial_box_offset;
+ const real_number shiftPos = val[IDXC_Z]-spatial_box_offset;
const real_number nbRepeat = floor(shiftPos/spatial_box_width);
const real_number posInBox = shiftPos - (spatial_box_width*nbRepeat);
return posInBox < limitPartitionShifted;
@@ -254,17 +291,23 @@ public:
// nb_particles_per_processes cannot be used after due to move
nb_particles_for_me = exchanger.getTotalToRecv();
- my_particles_positions.reset(new real_number[exchanger.getTotalToRecv()*size_particle_positions]);
+ if(nb_particles_for_me){
+ my_particles_positions.reset(new real_number[exchanger.getTotalToRecv()*size_particle_positions]);
+ }
exchanger.alltoallv<real_number>(split_particles_positions.get(), my_particles_positions.get(), size_particle_positions);
- split_particles_positions.release();
+ delete[] split_particles_positions.release();
- my_particles_indexes.reset(new partsize_t[exchanger.getTotalToRecv()]);
+ if(nb_particles_for_me){
+ my_particles_indexes.reset(new partsize_t[exchanger.getTotalToRecv()]);
+ }
exchanger.alltoallv<partsize_t>(split_particles_indexes.get(), my_particles_indexes.get());
- split_particles_indexes.release();
+ delete[] split_particles_indexes.release();
my_particles_rhs.resize(nb_rhs);
for(int idx_rhs = 0 ; idx_rhs < int(nb_rhs) ; ++idx_rhs){
- my_particles_rhs[idx_rhs].reset(new real_number[exchanger.getTotalToRecv()*size_particle_rhs]);
+ if(nb_particles_for_me){
+ my_particles_rhs[idx_rhs].reset(new real_number[exchanger.getTotalToRecv()*size_particle_rhs]);
+ }
exchanger.alltoallv<real_number>(split_particles_rhs[idx_rhs].get(), my_particles_rhs[idx_rhs].get(), size_particle_rhs);
}
}
@@ -272,17 +315,24 @@ public:
{
TIMEZONE("close");
int hdfret = H5Fclose(particle_file);
+ variable_used_only_in_assert(hdfret);
assert(hdfret >= 0);
hdfret = H5Pclose(plist_id_par);
assert(hdfret >= 0);
}
+
+ // clone indices in labels:
+ my_particles_labels.reset(new partsize_t[this->getLocalNbParticles()]);
+ std::copy(my_particles_indexes.get(),
+ my_particles_indexes.get()+this->getLocalNbParticles(),
+ my_particles_labels.get());
}
- ~particles_input_hdf5(){
+ ~particles_input_hdf5() noexcept(false){
}
partsize_t getTotalNbParticles() final{
- return partsize_t(nb_total_particles);
+ return partsize_t(total_number_of_particles);
}
partsize_t getLocalNbParticles() final{
@@ -294,7 +344,7 @@ public:
}
std::unique_ptr<real_number[]> getMyParticles() final {
- assert(my_particles_positions != nullptr);
+ assert(my_particles_positions != nullptr || nb_particles_for_me == 0);
return std::move(my_particles_positions);
}
@@ -304,9 +354,18 @@ public:
}
std::unique_ptr<partsize_t[]> getMyParticlesIndexes() final {
- assert(my_particles_indexes != nullptr);
+ assert(my_particles_indexes != nullptr || nb_particles_for_me == 0);
return std::move(my_particles_indexes);
}
+
+ std::unique_ptr<partsize_t[]> getMyParticlesLabels() final {
+ assert(my_particles_labels != nullptr || nb_particles_for_me == 0);
+ return std::move(my_particles_labels);
+ }
+
+ std::vector<hsize_t> getParticleFileLayout(){
+ return std::vector<hsize_t>(this->particle_file_layout);
+ }
};
#endif
diff --git a/cpp/particles/particles_input_random.hpp b/cpp/particles/particles_input_random.hpp
new file mode 100644
index 0000000000000000000000000000000000000000..0b40ae57f70729b82984c4c5b4591568e4078485
--- /dev/null
+++ b/cpp/particles/particles_input_random.hpp
@@ -0,0 +1,275 @@
+/******************************************************************************
+* *
+* Copyright 2020 Max Planck Institute for Dynamics and Self-Organization *
+* *
+* This file is part of TurTLE. *
+* *
+* TurTLE is free software: you can redistribute it and/or modify *
+* it under the terms of the GNU General Public License as published *
+* by the Free Software Foundation, either version 3 of the License, *
+* or (at your option) any later version. *
+* *
+* TurTLE is distributed in the hope that it will be useful, *
+* but WITHOUT ANY WARRANTY; without even the implied warranty of *
+* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the *
+* GNU General Public License for more details. *
+* *
+* You should have received a copy of the GNU General Public License *
+* along with TurTLE. If not, see <http://www.gnu.org/licenses/> *
+* *
+* Contact: Cristian.Lalescu@ds.mpg.de *
+* *
+******************************************************************************/
+
+
+
+#ifndef PARTICLES_INPUT_RANDOM_HPP
+#define PARTICLES_INPUT_RANDOM_HPP
+
+#include <mpi.h>
+#include <cassert>
+#include <random>
+#include "particles/abstract_particles_input.hpp"
+
+template <class partsize_t, class particle_rnumber, int state_size>
+class particles_input_random: public abstract_particles_input<partsize_t, particle_rnumber>
+{
+ private:
+ MPI_Comm comm;
+ int myrank, nprocs;
+
+ hsize_t total_number_of_particles;
+ hsize_t number_rhs;
+ partsize_t local_number_of_particles;
+
+ std::unique_ptr<particle_rnumber[]> local_particle_state;
+ std::unique_ptr<partsize_t[]> local_particle_index;
+ std::unique_ptr<partsize_t[]> local_particle_label;
+ std::vector<std::unique_ptr<particle_rnumber[]>> local_particle_rhs;
+ public:
+ particles_input_random(
+ const MPI_Comm in_mpi_comm,
+ const partsize_t NPARTICLES,
+ const int rseed,
+ const particle_rnumber my_spatial_low_limit,
+ const particle_rnumber my_spatial_up_limit):
+ comm(in_mpi_comm)
+ {
+ TIMEZONE("particles_input_random::particles_input_random");
+ assert(state_size >= 3);
+ static_assert((std::is_same<particle_rnumber, double>::value ||
+ std::is_same<particle_rnumber, float>::value),
+ "real_number must be double or float");
+ AssertMpi(MPI_Comm_rank(comm, &myrank));
+ AssertMpi(MPI_Comm_size(comm, &nprocs));
+ std::vector<particle_rnumber> in_spatial_limit_per_prod = BuildLimitsAllProcesses<particle_rnumber>(
+ comm, my_spatial_low_limit, my_spatial_up_limit);
+ assert(int(in_spatial_limit_per_prod.size()) == nprocs+1);
+
+ // initialize a separate random number generator for each MPI process
+ std::mt19937_64 rgen;
+ const double twopi = 4*acos(0);
+ // positions are uniformly distributed within 2pi cube
+ // TODO: provide possibilities for different rectangle sizes
+ std::uniform_real_distribution<particle_rnumber> udist(0.0, twopi);
+
+ // other state components are normally distributed (presumably they're velocities)
+ std::normal_distribution<particle_rnumber> gdist;
+
+ // seed random number generators such that no seed is ever repeated if we change the value of rseed.
+ // basically use a multi-dimensional array indexing technique to come up with actual seed.
+ // Note: this method is not invariant to the number of MPI processes!
+ int current_seed = (
+ rseed * (this->nprocs) +
+ this->myrank);
+ rgen.seed(current_seed);
+
+ this->total_number_of_particles = NPARTICLES;
+
+ // we know the total number of particles, but we want to generate particle locations in parallel.
+ // so we need a preliminary distributor of particles, location-agnostic:
+ particles_utils::IntervalSplitter<hsize_t> load_splitter(
+ this->total_number_of_particles, this->nprocs, this->myrank);
+
+ // allocate array for preliminary particle data
+ std::unique_ptr<particle_rnumber[]> split_particle_state;
+ if(load_splitter.getMySize())
+ {
+ split_particle_state.reset(new particle_rnumber[load_splitter.getMySize()*state_size]);
+ }
+
+ // allocate and populate array for preliminary particle indices
+ std::unique_ptr<partsize_t[]> split_particle_index;
+ if(load_splitter.getMySize())
+ {
+ split_particle_index.reset(new partsize_t[load_splitter.getMySize()]);
+ }
+ for(partsize_t idx = 0 ; idx < partsize_t(load_splitter.getMySize()) ; idx++){
+ split_particle_index[idx] = idx + partsize_t(load_splitter.getMyOffset());
+ }
+
+ // generate random particle states
+ {
+ TIMEZONE("particles_input_random::generate random numbers");
+ if (state_size == 15)
+ // deformation tensor
+ {
+ for (partsize_t idx=0; idx < partsize_t(load_splitter.getMySize()); idx++)
+ {
+ // positions are uniformly distributed within cube
+ for (int cc=0; cc < 3; cc++)
+ split_particle_state[idx*state_size + cc] = udist(rgen);
+ // Q matrix is initially identity
+ split_particle_state[idx*state_size + 3] = particle_rnumber(1.0);
+ split_particle_state[idx*state_size + 4] = particle_rnumber(0.0);
+ split_particle_state[idx*state_size + 5] = particle_rnumber(0.0);
+ split_particle_state[idx*state_size + 6] = particle_rnumber(0.0);
+ split_particle_state[idx*state_size + 7] = particle_rnumber(1.0);
+ split_particle_state[idx*state_size + 8] = particle_rnumber(0.0);
+ split_particle_state[idx*state_size + 9] = particle_rnumber(0.0);
+ split_particle_state[idx*state_size + 10] = particle_rnumber(0.0);
+ split_particle_state[idx*state_size + 11] = particle_rnumber(1.0);
+ // log-stretching factors are initially 0
+ split_particle_state[idx*state_size + 12] = particle_rnumber(0.0);
+ split_particle_state[idx*state_size + 13] = particle_rnumber(0.0);
+ split_particle_state[idx*state_size + 14] = particle_rnumber(0.0);
+ }
+ }
+ else
+ {
+ for (partsize_t idx=0; idx < partsize_t(load_splitter.getMySize()); idx++)
+ {
+ // positions are uniformly distributed within cube
+ for (int cc=0; cc < 3; cc++)
+ split_particle_state[idx*state_size + cc] = udist(rgen);
+ // velocities/orientations are normally distributed
+ // TODO: add option for normalization of orientation vectors etc
+ for (int cc = 3; cc < state_size; cc++)
+ split_particle_state[idx*state_size + cc] = gdist(rgen);
+ }
+ }
+ }
+
+
+ /*************************************************/
+ // meta: permutation and echange regions are copy/pasted from particles_input_hdf5
+ // the two blocks should agree (other than type/variable renaming, and the removal of rhs stuff).
+ // Permute
+ std::vector<particle_rnumber> in_spatial_limit_per_proc = BuildLimitsAllProcesses<particle_rnumber>(
+ this->comm,
+ my_spatial_low_limit,
+ my_spatial_up_limit);
+ std::vector<partsize_t> nb_particles_per_proc(this->nprocs);
+ {
+ TIMEZONE("particles_input_random::partition");
+
+ const particle_rnumber spatial_box_offset = in_spatial_limit_per_proc[0];
+ const particle_rnumber spatial_box_width =
+ in_spatial_limit_per_proc[this->nprocs] - in_spatial_limit_per_proc[0];
+
+ partsize_t previousOffset = 0;
+ for(int idx_proc = 0 ; idx_proc < this->nprocs-1 ; ++idx_proc){
+ const particle_rnumber limitPartitionShifted =
+ in_spatial_limit_per_proc[idx_proc+1] - spatial_box_offset;
+ const partsize_t localOffset = particles_utils::partition_extra<partsize_t, state_size>(
+ &split_particle_state[previousOffset*state_size],
+ partsize_t(load_splitter.getMySize())-previousOffset,
+ [&](const particle_rnumber val[]){
+ const particle_rnumber shiftPos = val[IDXC_Z] - spatial_box_offset;
+ const particle_rnumber nbRepeat = floor(shiftPos/spatial_box_width);
+ const particle_rnumber posInBox = shiftPos - (spatial_box_width*nbRepeat);
+ return posInBox < limitPartitionShifted;
+ },
+ [&](const partsize_t idx1, const partsize_t idx2){
+ std::swap(split_particle_index[idx1],
+ split_particle_index[idx2]);
+ },
+ previousOffset);
+
+ nb_particles_per_proc[idx_proc] = localOffset;
+ previousOffset += localOffset;
+ }
+ nb_particles_per_proc[this->nprocs-1] = partsize_t(load_splitter.getMySize()) - previousOffset;
+ }
+
+ {
+ TIMEZONE("particles_input_random::exchanger");
+ alltoall_exchanger exchanger(
+ this->comm,
+ std::move(nb_particles_per_proc));
+ // nb_particles_per_processes cannot be used after due to move
+
+ this->local_number_of_particles = exchanger.getTotalToRecv();
+
+ if(this->local_number_of_particles){
+ this->local_particle_state.reset(new particle_rnumber[exchanger.getTotalToRecv()*state_size]);
+ }
+ exchanger.alltoallv<particle_rnumber>(
+ split_particle_state.get(),
+ this->local_particle_state.get(),
+ state_size);
+ delete[] split_particle_state.release();
+
+ if(this->local_number_of_particles){
+ this->local_particle_index.reset(new partsize_t[exchanger.getTotalToRecv()]);
+ }
+ exchanger.alltoallv<partsize_t>(
+ split_particle_index.get(),
+ this->local_particle_index.get());
+ delete[] split_particle_index.release();
+ }
+ /*************************************************/
+
+
+ // clone indices in labels:
+ local_particle_label.reset(new partsize_t[this->getLocalNbParticles()]);
+ std::copy(local_particle_index.get(),
+ local_particle_index.get()+this->getLocalNbParticles(),
+ local_particle_label.get());
+ }
+
+ partsize_t getTotalNbParticles()
+ {
+ return this->total_number_of_particles;
+ }
+ partsize_t getLocalNbParticles()
+ {
+ return this->local_number_of_particles;
+ }
+ int getNbRhs()
+ {
+ return 0;
+ }
+
+ std::unique_ptr<particle_rnumber[]> getMyParticles()
+ {
+ assert(this->local_particle_state != nullptr || this->local_number_of_particles == 0);
+ return std::move(this->local_particle_state);
+ }
+
+ std::unique_ptr<partsize_t[]> getMyParticlesIndexes()
+ {
+ assert(this->local_particle_index != nullptr || this->local_number_of_particles == 0);
+ return std::move(this->local_particle_index);
+ }
+
+ std::unique_ptr<partsize_t[]> getMyParticlesLabels()
+ {
+ assert(this->local_particle_label != nullptr || this->local_number_of_particles == 0);
+ return std::move(this->local_particle_label);
+ }
+
+ std::vector<std::unique_ptr<particle_rnumber[]>> getMyRhs()
+ {
+ return std::move(std::vector<std::unique_ptr<particle_rnumber[]>>());
+ }
+
+ std::vector<hsize_t> getParticleFileLayout()
+ {
+ return std::vector<hsize_t>({
+ hsize_t(this->getTotalNbParticles())});
+ }
+};
+
+
+#endif
diff --git a/bfps/cpp/particles/particles_output_hdf5.hpp b/cpp/particles/particles_output_hdf5.hpp
similarity index 67%
rename from bfps/cpp/particles/particles_output_hdf5.hpp
rename to cpp/particles/particles_output_hdf5.hpp
index bc0a03690293668203dd78978680fdea03ab3a28..0ef809e63114948509c96f5cbccc5cdd615b3599 100644
--- a/bfps/cpp/particles/particles_output_hdf5.hpp
+++ b/cpp/particles/particles_output_hdf5.hpp
@@ -1,30 +1,56 @@
+/******************************************************************************
+* *
+* Copyright 2019 Max Planck Institute for Dynamics and Self-Organization *
+* *
+* This file is part of TurTLE. *
+* *
+* TurTLE is free software: you can redistribute it and/or modify *
+* it under the terms of the GNU General Public License as published *
+* by the Free Software Foundation, either version 3 of the License, *
+* or (at your option) any later version. *
+* *
+* TurTLE is distributed in the hope that it will be useful, *
+* but WITHOUT ANY WARRANTY; without even the implied warranty of *
+* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the *
+* GNU General Public License for more details. *
+* *
+* You should have received a copy of the GNU General Public License *
+* along with TurTLE. If not, see <http://www.gnu.org/licenses/> *
+* *
+* Contact: Cristian.Lalescu@ds.mpg.de *
+* *
+******************************************************************************/
+
+
+
#ifndef PARTICLES_OUTPUT_HDF5_HPP
#define PARTICLES_OUTPUT_HDF5_HPP
#include <memory>
#include <vector>
#include <hdf5.h>
+#include <sys/stat.h>
#include "abstract_particles_output.hpp"
#include "scope_timer.hpp"
template <class partsize_t,
class real_number,
- int size_particle_positions,
- int size_particle_rhs>
+ int size_particle_positions>
class particles_output_hdf5 : public abstract_particles_output<partsize_t,
real_number,
- size_particle_positions,
- size_particle_rhs>{
+ real_number,
+ size_particle_positions>{
using Parent = abstract_particles_output<partsize_t,
real_number,
- size_particle_positions,
- size_particle_rhs>;
+ real_number,
+ size_particle_positions>;
- const std::string particle_species_name;
+ std::string particle_species_name;
hid_t file_id;
const partsize_t total_nb_particles;
+ std::vector<hsize_t> particle_file_layout; // to hold the shape of initial condition array
hid_t dset_id_state;
hid_t dset_id_rhs;
@@ -39,8 +65,8 @@ public:
const bool in_use_collective_io = false)
: abstract_particles_output<partsize_t,
real_number,
- size_particle_positions,
- size_particle_rhs>(
+ real_number,
+ size_particle_positions>(
in_mpi_com,
inTotalNbParticles,
in_nb_rhs),
@@ -63,6 +89,7 @@ public:
plist_id_par,
Parent::getComWriter(),
MPI_INFO_NULL);
+ variable_used_only_in_assert(retTest);
assert(retTest >= 0);
// Parallel HDF5 write
@@ -88,13 +115,20 @@ public:
return EXIT_SUCCESS;
}
- ~particles_output_hdf5(){}
+ ~particles_output_hdf5() noexcept(false){}
+
+ void update_particle_species_name(
+ const std::string new_name)
+ {
+ this->particle_species_name.assign(new_name);
+ }
int close_file(void){
if(Parent::isInvolved()){
TIMEZONE("particles_output_hdf5::close_file");
int rethdf = H5Gclose(dset_id_state);
+ variable_used_only_in_assert(rethdf);
assert(rethdf >= 0);
rethdf = H5Gclose(dset_id_rhs);
@@ -110,10 +144,23 @@ public:
if(Parent::isInvolved()){
if (Parent::getMyRank() == 0)
{
- hid_t file_id = H5Fopen(
+ bool file_exists = false;
+ {
+ struct stat file_buffer;
+ file_exists = (stat(filename.c_str(), &file_buffer) == 0);
+ }
+ hid_t file_id;
+ if (file_exists)
+ file_id = H5Fopen(
filename.c_str(),
H5F_ACC_RDWR | H5F_ACC_DEBUG,
H5P_DEFAULT);
+ else
+ file_id = H5Fcreate(
+ filename.c_str(),
+ H5F_ACC_EXCL | H5F_ACC_DEBUG,
+ H5P_DEFAULT,
+ H5P_DEFAULT);
assert(file_id >= 0);
bool group_exists = H5Lexists(
file_id,
@@ -177,7 +224,8 @@ public:
const real_number* particles_positions,
const std::unique_ptr<real_number[]>* particles_rhs,
const partsize_t nb_particles,
- const partsize_t particles_idx_offset) final{
+ const partsize_t particles_idx_offset,
+ const int size_particle_rhs) final{
assert(Parent::isInvolved());
TIMEZONE("particles_output_hdf5::write");
@@ -194,16 +242,14 @@ public:
assert(plist_id >= 0);
{
int rethdf = H5Pset_dxpl_mpio(plist_id, use_collective_io ? H5FD_MPIO_COLLECTIVE : H5FD_MPIO_INDEPENDENT);
+ variable_used_only_in_assert(rethdf);
assert(rethdf >= 0);
}
{
- assert(total_nb_particles >= 0);
- assert(size_particle_positions >= 0);
- const hsize_t datacount[2] = {
- hsize_t(total_nb_particles),
- hsize_t(size_particle_positions)};
- hid_t dataspace = H5Screate_simple(2, datacount, NULL);
+ std::vector<hsize_t> datacount = std::vector<hsize_t>(this->particle_file_layout);
+ datacount.push_back(size_particle_positions);
+ hid_t dataspace = H5Screate_simple(datacount.size(), &datacount.front(), NULL);
assert(dataspace >= 0);
hid_t dataset_id = H5Dcreate( dset_id_state,
@@ -222,7 +268,13 @@ public:
hid_t memspace = H5Screate_simple(2, count, NULL);
assert(memspace >= 0);
- hid_t filespace = H5Dget_space(dataset_id);
+
+ assert(total_nb_particles >= 0);
+ assert(size_particle_positions >= 0);
+ const hsize_t file_count[2] = {hsize_t(total_nb_particles), size_particle_positions};
+ hid_t filespace = H5Screate_simple(2, file_count, NULL);
+ assert(filespace >= 0);
+
int rethdf = H5Sselect_hyperslab(
filespace,
H5S_SELECT_SET,
@@ -230,6 +282,7 @@ public:
NULL,
count,
NULL);
+ variable_used_only_in_assert(rethdf);
assert(rethdf >= 0);
herr_t status = H5Dwrite(
@@ -239,6 +292,7 @@ public:
filespace,
plist_id,
particles_positions);
+ variable_used_only_in_assert(status);
assert(status >= 0);
rethdf = H5Sclose(memspace);
assert(rethdf >= 0);
@@ -249,10 +303,10 @@ public:
}
{
assert(size_particle_rhs >= 0);
- const hsize_t datacount[3] = {hsize_t(Parent::getNbRhs()),
- hsize_t(total_nb_particles),
- hsize_t(size_particle_rhs)};
- hid_t dataspace = H5Screate_simple(3, datacount, NULL);
+ std::vector<hsize_t> datacount = std::vector<hsize_t>(this->particle_file_layout);
+ datacount.insert(datacount.begin(), hsize_t(Parent::getNbRhs()));
+ datacount.push_back(size_particle_rhs);
+ hid_t dataspace = H5Screate_simple(datacount.size(), &datacount.front(), NULL);
assert(dataspace >= 0);
hid_t dataset_id = H5Dcreate( dset_id_rhs,
@@ -277,8 +331,12 @@ public:
hid_t memspace = H5Screate_simple(3, count, NULL);
assert(memspace >= 0);
- hid_t filespace = H5Dget_space(dataset_id);
+ assert(total_nb_particles >= 0);
+ assert(size_particle_positions >= 0);
+ const hsize_t file_count[3] = {hsize_t(Parent::getNbRhs()), hsize_t(total_nb_particles), size_particle_positions};
+ hid_t filespace = H5Screate_simple(3, file_count, NULL);
assert(filespace >= 0);
+
int rethdf = H5Sselect_hyperslab(
filespace,
H5S_SELECT_SET,
@@ -286,6 +344,7 @@ public:
NULL,
count,
NULL);
+ variable_used_only_in_assert(rethdf);
assert(rethdf >= 0);
herr_t status = H5Dwrite(
@@ -295,6 +354,7 @@ public:
filespace,
plist_id,
particles_rhs[idx_rhs].get());
+ variable_used_only_in_assert(status);
assert(status >= 0);
rethdf = H5Sclose(filespace);
assert(rethdf >= 0);
@@ -302,14 +362,27 @@ public:
assert(rethdf >= 0);
}
int rethdf = H5Dclose(dataset_id);
+ variable_used_only_in_assert(rethdf);
assert(rethdf >= 0);
}
{
int rethdf = H5Pclose(plist_id);
+ variable_used_only_in_assert(rethdf);
assert(rethdf >= 0);
}
}
+
+ int setParticleFileLayout(std::vector<hsize_t> input_layout){
+ this->particle_file_layout.resize(input_layout.size());
+ for (unsigned int i=0; i<this->particle_file_layout.size(); i++)
+ this->particle_file_layout[i] = input_layout[i];
+ return EXIT_SUCCESS;
+ }
+
+ std::vector<hsize_t> getParticleFileLayout(void){
+ return std::vector<hsize_t>(this->particle_file_layout);
+ }
};
#endif//PARTICLES_OUTPUT_HDF5_HPP
diff --git a/bfps/cpp/particles/particles_output_mpiio.hpp b/cpp/particles/particles_output_mpiio.hpp
similarity index 67%
rename from bfps/cpp/particles/particles_output_mpiio.hpp
rename to cpp/particles/particles_output_mpiio.hpp
index 77dae6ca2f9441948ccf04f8a72e4a53d249894b..ee9bd9cee42d3973e1e9e1bb93eb67a6329836d3 100644
--- a/bfps/cpp/particles/particles_output_mpiio.hpp
+++ b/cpp/particles/particles_output_mpiio.hpp
@@ -1,3 +1,28 @@
+/******************************************************************************
+* *
+* Copyright 2019 Max Planck Institute for Dynamics and Self-Organization *
+* *
+* This file is part of TurTLE. *
+* *
+* TurTLE is free software: you can redistribute it and/or modify *
+* it under the terms of the GNU General Public License as published *
+* by the Free Software Foundation, either version 3 of the License, *
+* or (at your option) any later version. *
+* *
+* TurTLE is distributed in the hope that it will be useful, *
+* but WITHOUT ANY WARRANTY; without even the implied warranty of *
+* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the *
+* GNU General Public License for more details. *
+* *
+* You should have received a copy of the GNU General Public License *
+* along with TurTLE. If not, see <http://www.gnu.org/licenses/> *
+* *
+* Contact: Cristian.Lalescu@ds.mpg.de *
+* *
+******************************************************************************/
+
+
+
#ifndef PARTICLES_OUTPUT_MPIIO
#define PARTICLES_OUTPUT_MPIIO
@@ -11,8 +36,8 @@
#include "particles_utils.hpp"
template <class partsize_t, class real_number, int size_particle_positions, int size_particle_rhs>
-class particles_output_mpiio : public abstract_particles_output<partsize_t, real_number, size_particle_positions, size_particle_rhs>{
- using Parent = abstract_particles_output<partsize_t, real_number, size_particle_positions, size_particle_rhs>;
+class particles_output_mpiio : public abstract_particles_output<partsize_t, real_number, real_number, size_particle_positions>{
+ using Parent = abstract_particles_output<partsize_t, real_number, real_number, size_particle_positions>;
const std::string filename;
const int nb_step_prealloc;
@@ -24,7 +49,7 @@ class particles_output_mpiio : public abstract_particles_output<partsize_t, real
public:
particles_output_mpiio(MPI_Comm in_mpi_com, const std::string in_filename, const partsize_t inTotalNbParticles,
const int in_nb_rhs, const int in_nb_step_prealloc = -1)
- : abstract_particles_output<partsize_t, real_number, size_particle_positions, size_particle_rhs>(in_mpi_com, inTotalNbParticles, in_nb_rhs),
+ : abstract_particles_output<partsize_t, real_number, real_number, size_particle_positions>(in_mpi_com, inTotalNbParticles, in_nb_rhs),
filename(in_filename), nb_step_prealloc(in_nb_step_prealloc), current_step_in_file(0){
if(Parent::isInvolved()){
{
@@ -40,7 +65,7 @@ public:
}
}
- ~particles_output_mpiio(){
+ ~particles_output_mpiio() noexcept(false){
if(Parent::isInvolved()){
TIMEZONE("particles_output_mpiio::MPI_File_close");
AssertMpi(MPI_File_close(&mpi_file));
diff --git a/cpp/particles/particles_output_sampling_hdf5.hpp b/cpp/particles/particles_output_sampling_hdf5.hpp
new file mode 100644
index 0000000000000000000000000000000000000000..2d85d8ea781339b5d572968d50d09ae078c657eb
--- /dev/null
+++ b/cpp/particles/particles_output_sampling_hdf5.hpp
@@ -0,0 +1,295 @@
+/******************************************************************************
+* *
+* Copyright 2019 Max Planck Institute for Dynamics and Self-Organization *
+* *
+* This file is part of TurTLE. *
+* *
+* TurTLE is free software: you can redistribute it and/or modify *
+* it under the terms of the GNU General Public License as published *
+* by the Free Software Foundation, either version 3 of the License, *
+* or (at your option) any later version. *
+* *
+* TurTLE is distributed in the hope that it will be useful, *
+* but WITHOUT ANY WARRANTY; without even the implied warranty of *
+* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the *
+* GNU General Public License for more details. *
+* *
+* You should have received a copy of the GNU General Public License *
+* along with TurTLE. If not, see <http://www.gnu.org/licenses/> *
+* *
+* Contact: Cristian.Lalescu@ds.mpg.de *
+* *
+******************************************************************************/
+
+
+
+#ifndef PARTICLES_OUTPUT_SAMPLING_HDF5_HPP
+#define PARTICLES_OUTPUT_SAMPLING_HDF5_HPP
+
+#include "abstract_particles_output.hpp"
+#include "hdf5_tools.hpp"
+
+
+template <class partsize_t,
+ class position_type,
+ class output_type,
+ int size_particle_positions>
+class particles_output_sampling_hdf5 : public abstract_particles_output<
+ partsize_t,
+ position_type,
+ output_type,
+ size_particle_positions>{
+ using Parent = abstract_particles_output<partsize_t,
+ position_type,
+ output_type,
+ size_particle_positions>;
+
+ hid_t file_id, pgroup_id;
+
+ std::string dataset_name;
+ std::vector<hsize_t> particle_file_layout; // to hold the shape of initial condition array
+ const bool use_collective_io;
+
+public:
+ static bool DatasetExistsCol(MPI_Comm in_mpi_com,
+ const std::string& in_filename,
+ const std::string& in_groupname,
+ const std::string& in_dataset_name){
+ int my_rank;
+ AssertMpi(MPI_Comm_rank(in_mpi_com, &my_rank));
+
+ int dataset_exists = -1;
+
+ if(my_rank == 0){
+ hid_t file_id = H5Fopen(
+ in_filename.c_str(),
+ H5F_ACC_RDONLY| H5F_ACC_DEBUG,
+ H5P_DEFAULT);
+ assert(file_id >= 0);
+
+ dataset_exists = H5Lexists(
+ file_id,
+ (in_groupname + "/" + in_dataset_name).c_str(),
+ H5P_DEFAULT);
+
+ int retTest = H5Fclose(file_id);
+ assert(retTest >= 0);
+ }
+
+ AssertMpi(MPI_Bcast( &dataset_exists, 1, MPI_INT, 0, in_mpi_com ));
+ return dataset_exists;
+ }
+
+ particles_output_sampling_hdf5(
+ MPI_Comm in_mpi_com,
+ const partsize_t inTotalNbParticles,
+ const std::string& in_filename,
+ const std::string& in_groupname,
+ const std::string& in_dataset_name,
+ const bool in_use_collective_io = false)
+ : Parent(in_mpi_com, inTotalNbParticles, 1),
+ dataset_name(in_dataset_name),
+ use_collective_io(in_use_collective_io){
+ if(Parent::isInvolved()){
+ // prepare parallel MPI access property list
+ hid_t plist_id_par = H5Pcreate(H5P_FILE_ACCESS);
+ assert(plist_id_par >= 0);
+ int retTest = H5Pset_fapl_mpio(
+ plist_id_par,
+ Parent::getComWriter(),
+ MPI_INFO_NULL);
+ variable_used_only_in_assert(retTest);
+ assert(retTest >= 0);
+
+ // open file for parallel HDF5 access
+ file_id = H5Fopen(
+ in_filename.c_str(),
+ H5F_ACC_RDWR | H5F_ACC_DEBUG,
+ plist_id_par);
+ assert(file_id >= 0);
+ retTest = H5Pclose(plist_id_par);
+ assert(retTest >= 0);
+
+ // open group
+ pgroup_id = H5Gopen(
+ file_id,
+ in_groupname.c_str(),
+ H5P_DEFAULT);
+ assert(pgroup_id >= 0);
+ }
+ }
+
+ ~particles_output_sampling_hdf5() noexcept(false){
+ if(Parent::isInvolved()){
+ // close group
+ int retTest = H5Gclose(pgroup_id);
+ variable_used_only_in_assert(retTest);
+ assert(retTest >= 0);
+ // close file
+ retTest = H5Fclose(file_id);
+ assert(retTest >= 0);
+ }
+ }
+
+ int switch_to_group(
+ const std::string &in_groupname)
+ {
+ if(Parent::isInvolved()){
+ // close old group
+ int retTest = H5Gclose(pgroup_id);
+ variable_used_only_in_assert(retTest);
+ assert(retTest >= 0);
+
+ // open new group
+ pgroup_id = H5Gopen(
+ file_id,
+ in_groupname.c_str(),
+ H5P_DEFAULT);
+ assert(pgroup_id >= 0);
+ }
+ return EXIT_SUCCESS;
+ }
+
+ template <int size_particle_rhs>
+ int save_dataset(
+ const std::string& in_groupname,
+ const std::string& in_dataset_name,
+ const position_type input_particles_positions[],
+ const std::unique_ptr<output_type[]> input_particles_rhs[],
+ const partsize_t index_particles[],
+ const partsize_t nb_particles,
+ const int idx_time_step)
+ {
+ // update group
+ int retTest = this->switch_to_group(
+ in_groupname);
+ variable_used_only_in_assert(retTest);
+ assert(retTest == EXIT_SUCCESS);
+ // update dataset name
+ dataset_name = in_dataset_name + "/" + std::to_string(idx_time_step);
+ int dataset_exists;
+ if (this->getMyRank() == 0)
+ dataset_exists = H5Lexists(
+ pgroup_id,
+ dataset_name.c_str(),
+ H5P_DEFAULT);
+ AssertMpi(MPI_Bcast(&dataset_exists, 1, MPI_INT, 0, this->getCom()));
+ if (dataset_exists == 0)
+ this->template save<size_particle_rhs>(
+ input_particles_positions,
+ input_particles_rhs,
+ index_particles,
+ nb_particles,
+ idx_time_step);
+ return EXIT_SUCCESS;
+ }
+
+ void write(
+ const int /*idx_time_step*/,
+ const position_type* /*particles_positions*/,
+ const std::unique_ptr<output_type[]>* particles_rhs,
+ const partsize_t nb_particles,
+ const partsize_t particles_idx_offset,
+ const int size_particle_rhs) final{
+ assert(Parent::isInvolved());
+
+ TIMEZONE("particles_output_sampling_hdf5::write");
+
+ assert(particles_idx_offset < Parent::getTotalNbParticles() ||
+ (particles_idx_offset == Parent::getTotalNbParticles() &&
+ nb_particles == 0));
+ assert(particles_idx_offset+nb_particles <= Parent::getTotalNbParticles());
+
+ static_assert(std::is_same<output_type, double>::value ||
+ std::is_same<output_type, float>::value ||
+ std::is_same<output_type, long long int>::value,
+ "output_type must be double or float or long long int");
+ const hid_t type_id = hdf5_tools::hdf5_type_id<output_type>();
+
+ hid_t plist_id = H5Pcreate(H5P_DATASET_XFER);
+ assert(plist_id >= 0);
+ {
+ int rethdf = H5Pset_dxpl_mpio(
+ plist_id,
+ (use_collective_io ?
+ H5FD_MPIO_COLLECTIVE :
+ H5FD_MPIO_INDEPENDENT));
+ variable_used_only_in_assert(rethdf);
+ assert(rethdf >= 0);
+ }
+ {
+ assert(size_particle_rhs >= 0);
+ std::vector<hsize_t> datacount = std::vector<hsize_t>(this->particle_file_layout);
+ datacount.push_back(size_particle_rhs);
+ hid_t dataspace = H5Screate_simple(datacount.size(), &datacount.front(), NULL);
+ assert(dataspace >= 0);
+
+ hid_t dataset_id = H5Dcreate( pgroup_id,
+ dataset_name.c_str(),
+ type_id,
+ dataspace,
+ H5P_DEFAULT,
+ H5P_DEFAULT,
+ H5P_DEFAULT);
+ assert(dataset_id >= 0);
+
+ assert(particles_idx_offset >= 0);
+ const hsize_t count[2] = {
+ hsize_t(nb_particles),
+ hsize_t(size_particle_rhs)};
+ const hsize_t offset[2] = {
+ hsize_t(particles_idx_offset),
+ 0};
+ hid_t memspace = H5Screate_simple(2, count, NULL);
+ assert(memspace >= 0);
+
+ const hsize_t file_count[2] = {hsize_t(Parent::getTotalNbParticles()), hsize_t(size_particle_rhs)};
+ hid_t filespace = H5Screate_simple(2, file_count, NULL);
+ assert(filespace >= 0);
+ int rethdf = H5Sselect_hyperslab(
+ filespace,
+ H5S_SELECT_SET,
+ offset,
+ NULL,
+ count,
+ NULL);
+ variable_used_only_in_assert(rethdf);
+ assert(rethdf >= 0);
+
+ herr_t status = H5Dwrite(
+ dataset_id,
+ type_id,
+ memspace,
+ filespace,
+ plist_id,
+ particles_rhs[0].get());
+ variable_used_only_in_assert(status);
+ assert(status >= 0);
+ rethdf = H5Sclose(filespace);
+ assert(rethdf >= 0);
+ rethdf = H5Sclose(memspace);
+ assert(rethdf >= 0);
+ rethdf = H5Dclose(dataset_id);
+ assert(rethdf >= 0);
+ }
+
+ {
+ int rethdf = H5Pclose(plist_id);
+ variable_used_only_in_assert(rethdf);
+ assert(rethdf >= 0);
+ }
+ }
+
+ int setParticleFileLayout(const std::vector<hsize_t> input_layout){
+ this->particle_file_layout.resize(input_layout.size());
+ for (unsigned int i=0; i<this->particle_file_layout.size(); i++)
+ this->particle_file_layout[i] = input_layout[i];
+ return EXIT_SUCCESS;
+ }
+
+ std::vector<hsize_t> getParticleFileLayout(void){
+ return std::vector<hsize_t>(this->particle_file_layout);
+ }
+};
+
+#endif
diff --git a/cpp/particles/particles_sampling.hpp b/cpp/particles/particles_sampling.hpp
new file mode 100644
index 0000000000000000000000000000000000000000..8e83a5a53690b6cb3abaddaafe2245b1f448d164
--- /dev/null
+++ b/cpp/particles/particles_sampling.hpp
@@ -0,0 +1,109 @@
+/******************************************************************************
+* *
+* Copyright 2019 Max Planck Institute for Dynamics and Self-Organization *
+* *
+* This file is part of TurTLE. *
+* *
+* TurTLE is free software: you can redistribute it and/or modify *
+* it under the terms of the GNU General Public License as published *
+* by the Free Software Foundation, either version 3 of the License, *
+* or (at your option) any later version. *
+* *
+* TurTLE is distributed in the hope that it will be useful, *
+* but WITHOUT ANY WARRANTY; without even the implied warranty of *
+* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the *
+* GNU General Public License for more details. *
+* *
+* You should have received a copy of the GNU General Public License *
+* along with TurTLE. If not, see <http://www.gnu.org/licenses/> *
+* *
+* Contact: Cristian.Lalescu@ds.mpg.de *
+* *
+******************************************************************************/
+
+
+
+#ifndef PARTICLES_SAMPLING_HPP
+#define PARTICLES_SAMPLING_HPP
+
+#include <memory>
+#include <string>
+
+#include "abstract_particles_system.hpp"
+#include "particles_output_sampling_hdf5.hpp"
+
+#include "field.hpp"
+#include "kspace.hpp"
+
+
+template <class partsize_t, class particles_rnumber, class rnumber, field_backend be, field_components fc>
+void sample_from_particles_system(const field<rnumber, be, fc>& in_field, // a pointer to a field<rnumber, FFTW, fc>
+ std::unique_ptr<abstract_particles_system<partsize_t, particles_rnumber>>& ps, // a pointer to an particles_system<double>
+ const std::string& filename,
+ const std::string& parent_groupname,
+ const std::string& fname){
+ const std::string datasetname = fname + std::string("/") + std::to_string(ps->get_step_idx());
+ const int size_particle_rhs = ncomp(fc);
+
+ // Stop here if already exists
+ if(particles_output_sampling_hdf5<partsize_t, particles_rnumber, particles_rnumber, 3>::DatasetExistsCol(MPI_COMM_WORLD,
+ filename,
+ parent_groupname,
+ datasetname)){
+ return;
+ }
+
+ const partsize_t nb_particles = ps->getLocalNbParticles();
+ std::unique_ptr<particles_rnumber[]> sample_rhs(new particles_rnumber[size_particle_rhs*nb_particles]);
+ std::fill_n(sample_rhs.get(), size_particle_rhs*nb_particles, 0);
+
+ ps->sample_compute_field(in_field, sample_rhs.get());
+
+
+
+ particles_output_sampling_hdf5<partsize_t, particles_rnumber, particles_rnumber, 3> outputclass(MPI_COMM_WORLD,
+ ps->getGlobalNbParticles(),
+ filename,
+ parent_groupname,
+ datasetname);
+ outputclass.template save<size_particle_rhs>(ps->getParticlesState(),
+ &sample_rhs,
+ ps->getParticlesIndexes(),
+ ps->getLocalNbParticles(),
+ ps->get_step_idx());
+}
+
+template <class partsize_t, class particles_rnumber>
+void sample_particles_system_position(
+ std::unique_ptr<abstract_particles_system<partsize_t, particles_rnumber>>& ps, // a pointer to an particles_system<double>
+ const std::string& filename,
+ const std::string& parent_groupname,
+ const std::string& fname){
+ const std::string datasetname = fname + std::string("/") + std::to_string(ps->get_step_idx());
+
+ // Stop here if already exists
+ if(particles_output_sampling_hdf5<partsize_t, particles_rnumber, particles_rnumber, 3>::DatasetExistsCol(MPI_COMM_WORLD,
+ filename,
+ parent_groupname,
+ datasetname)){
+ return;
+ }
+
+ const partsize_t nb_particles = ps->getLocalNbParticles();
+ std::unique_ptr<particles_rnumber[]> sample_rhs(new particles_rnumber[3*nb_particles]);
+ std::copy(ps->getParticlesState(), ps->getParticlesState() + 3*nb_particles, sample_rhs.get());
+
+ particles_output_sampling_hdf5<partsize_t, particles_rnumber, particles_rnumber, 3> outputclass(MPI_COMM_WORLD,
+ ps->getGlobalNbParticles(),
+ filename,
+ parent_groupname,
+ datasetname);
+ outputclass.template save<3>(ps->getParticlesState(),
+ &sample_rhs,
+ ps->getParticlesIndexes(),
+ ps->getLocalNbParticles(),
+ ps->get_step_idx());
+}
+
+#endif//PARTICLES_SAMPLING_HPP
+
diff --git a/cpp/particles/particles_system.hpp b/cpp/particles/particles_system.hpp
new file mode 100644
index 0000000000000000000000000000000000000000..9589e4dcc2738c554f7ff364235b7bc92d8a155a
--- /dev/null
+++ b/cpp/particles/particles_system.hpp
@@ -0,0 +1,583 @@
+/******************************************************************************
+* *
+* Copyright 2019 Max Planck Institute for Dynamics and Self-Organization *
+* *
+* This file is part of TurTLE. *
+* *
+* TurTLE is free software: you can redistribute it and/or modify *
+* it under the terms of the GNU General Public License as published *
+* by the Free Software Foundation, either version 3 of the License, *
+* or (at your option) any later version. *
+* *
+* TurTLE is distributed in the hope that it will be useful, *
+* but WITHOUT ANY WARRANTY; without even the implied warranty of *
+* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the *
+* GNU General Public License for more details. *
+* *
+* You should have received a copy of the GNU General Public License *
+* along with TurTLE. If not, see <http://www.gnu.org/licenses/> *
+* *
+* Contact: Cristian.Lalescu@ds.mpg.de *
+* *
+******************************************************************************/
+
+
+
+#ifndef PARTICLES_SYSTEM_HPP
+#define PARTICLES_SYSTEM_HPP
+
+#include <array>
+#include <set>
+#include <algorithm>
+#include <cmath>
+
+#include "particles/abstract_particles_system.hpp"
+#include "particles/abstract_particles_system_with_p2p.hpp"
+#include "particles/particles_distr_mpi.hpp"
+#include "particles/particles_output_hdf5.hpp"
+#include "particles/particles_output_mpiio.hpp"
+#include "particles/interpolation/particles_field_computer.hpp"
+#include "particles/abstract_particles_input.hpp"
+#include "particles/particles_adams_bashforth.hpp"
+#include "scope_timer.hpp"
+
+#include "particles/p2p/p2p_distr_mpi.hpp"
+
+template <class partsize_t, class real_number, class field_rnumber, class field_class, class interpolator_class, int interp_neighbours,
+ int size_particle_positions, int size_particle_rhs, class p2p_computer_class, class particles_inner_computer_class>
+class particles_system : public abstract_particles_system_with_p2p<partsize_t, real_number, p2p_computer_class> {
+ static_assert(size_particle_positions >= 3, "There should be at least the positions X,Y,Z in the state");
+
+ MPI_Comm mpi_com;
+
+ const std::pair<int,int> current_partition_interval;
+ const int partition_interval_size;
+
+ interpolator_class interpolator;
+
+ particles_distr_mpi<partsize_t, real_number> particles_distr;
+
+ particles_adams_bashforth<partsize_t, real_number, size_particle_positions, size_particle_rhs> positions_updater;
+
+ using computer_class = particles_field_computer<partsize_t, real_number, interpolator_class, interp_neighbours>;
+ computer_class computer;
+
+ const field_class& default_field;
+
+ std::unique_ptr<partsize_t[]> current_my_nb_particles_per_partition;
+ std::unique_ptr<partsize_t[]> current_offset_particles_for_partition;
+
+ const std::array<real_number,3> spatial_box_width;
+ const std::array<real_number,3> spatial_partition_width;
+ const real_number my_spatial_low_limit;
+ const real_number my_spatial_up_limit;
+
+ std::unique_ptr<real_number[]> my_particles_positions;
+ std::unique_ptr<partsize_t[]> my_particles_positions_indexes;
+ partsize_t my_nb_particles;
+ partsize_t total_nb_particles;
+ std::vector<std::unique_ptr<real_number[]>> my_particles_rhs;
+ std::vector<hsize_t> particle_file_layout;
+
+ int step_idx;
+
+ p2p_distr_mpi<partsize_t, real_number> distr_p2p;
+ p2p_computer_class computer_p2p;
+ particles_inner_computer_class computer_particules_inner;
+
+public:
+ particles_system(const std::array<size_t,3>& field_grid_dim,
+ const std::array<real_number,3>& in_spatial_box_width,
+ const std::array<real_number,3>& in_spatial_box_offset,
+ const std::array<real_number,3>& in_spatial_partition_width,
+ const real_number in_my_spatial_low_limit,
+ const real_number in_my_spatial_up_limit,
+ const std::array<size_t,3>& in_local_field_dims,
+ const std::array<size_t,3>& in_local_field_offset,
+ const field_class& in_field,
+ MPI_Comm in_mpi_com,
+ const partsize_t in_total_nb_particles,
+ const real_number in_cutoff,
+ p2p_computer_class in_computer_p2p,
+ particles_inner_computer_class in_computer_particules_inner,
+ const int in_current_iteration = 1)
+ : mpi_com(in_mpi_com),
+ current_partition_interval({in_local_field_offset[IDXC_Z],
+ in_local_field_offset[IDXC_Z] + in_local_field_dims[IDXC_Z]}),
+ partition_interval_size(current_partition_interval.second - current_partition_interval.first),
+ interpolator(),
+ particles_distr(in_mpi_com,
+ current_partition_interval,
+ field_grid_dim),
+ positions_updater(),
+ computer(field_grid_dim,
+ current_partition_interval,
+ interpolator,
+ in_spatial_box_width,
+ in_spatial_box_offset,
+ in_spatial_partition_width),
+ default_field(in_field),
+ spatial_box_width(in_spatial_box_width),
+ spatial_partition_width(in_spatial_partition_width),
+ my_spatial_low_limit(in_my_spatial_low_limit),
+ my_spatial_up_limit(in_my_spatial_up_limit),
+ my_nb_particles(0),
+ total_nb_particles(in_total_nb_particles),
+ step_idx(in_current_iteration),
+ distr_p2p(in_mpi_com,
+ current_partition_interval,
+ field_grid_dim,
+ spatial_box_width,
+ in_spatial_box_offset,
+ in_cutoff),
+ computer_p2p(std::move(in_computer_p2p)),
+ computer_particules_inner(std::move(in_computer_particules_inner)){
+
+ current_my_nb_particles_per_partition.reset(new partsize_t[partition_interval_size]);
+ current_offset_particles_for_partition.reset(new partsize_t[partition_interval_size+1]);
+ }
+
+ ~particles_system() noexcept(false){
+ }
+
+ void init(abstract_particles_input<partsize_t, real_number>& particles_input) {
+ TIMEZONE("particles_system::init");
+
+ my_particles_positions = particles_input.getMyParticles();
+ my_particles_positions_indexes = particles_input.getMyParticlesIndexes();
+ my_particles_rhs = particles_input.getMyRhs();
+ my_nb_particles = particles_input.getLocalNbParticles();
+
+ for(partsize_t idx_part = 0 ; idx_part < my_nb_particles ; ++idx_part){ // TODO remove me
+ const int partition_level = computer.pbc_field_layer(my_particles_positions[idx_part*size_particle_positions+IDXC_Z], IDXC_Z);
+ variable_used_only_in_assert(partition_level);
+ assert(partition_level >= current_partition_interval.first);
+ assert(partition_level < current_partition_interval.second);
+ }
+
+ particles_utils::partition_extra_z<partsize_t, size_particle_positions>(&my_particles_positions[0], my_nb_particles, partition_interval_size,
+ current_my_nb_particles_per_partition.get(), current_offset_particles_for_partition.get(),
+ [&](const real_number& z_pos){
+ const int partition_level = computer.pbc_field_layer(z_pos, IDXC_Z);
+ assert(current_partition_interval.first <= partition_level && partition_level < current_partition_interval.second);
+ return partition_level - current_partition_interval.first;
+ },
+ [&](const partsize_t idx1, const partsize_t idx2){
+ std::swap(my_particles_positions_indexes[idx1], my_particles_positions_indexes[idx2]);
+ for(int idx_rhs = 0 ; idx_rhs < int(my_particles_rhs.size()) ; ++idx_rhs){
+ for(int idx_val = 0 ; idx_val < size_particle_rhs ; ++idx_val){
+ std::swap(my_particles_rhs[idx_rhs][idx1*size_particle_rhs + idx_val],
+ my_particles_rhs[idx_rhs][idx2*size_particle_rhs + idx_val]);
+ }
+ }
+ });
+
+ {// TODO remove
+ for(int idxPartition = 0 ; idxPartition < partition_interval_size ; ++idxPartition){
+ assert(current_my_nb_particles_per_partition[idxPartition] ==
+ current_offset_particles_for_partition[idxPartition+1] - current_offset_particles_for_partition[idxPartition]);
+ for(partsize_t idx = current_offset_particles_for_partition[idxPartition] ; idx < current_offset_particles_for_partition[idxPartition+1] ; ++idx){
+ assert(computer.pbc_field_layer(my_particles_positions[idx*size_particle_positions+IDXC_Z], IDXC_Z)-current_partition_interval.first == idxPartition);
+ }
+ }
+ }
+ }
+
+ void delete_particles_from_indexes(const std::vector<partsize_t>& inIndexToDelete) final {
+ // We consider that the given indexes are here or in our neighbors,
+ // so we exchange them
+ int my_rank;
+ AssertMpi(MPI_Comm_rank(mpi_com, &my_rank));
+ int nb_processes;
+ AssertMpi(MPI_Comm_size(mpi_com, &nb_processes));
+
+ std::set<partsize_t> setIndexToDelete(inIndexToDelete.begin(), inIndexToDelete.end());
+
+ if(nb_processes > 1){
+ const int TopToBottom = 0;
+ const int BottomToTop = 1;
+
+ partsize_t nbIndexesFromTop = 0;
+ partsize_t nbIndexesFromBottom = 0;
+ partsize_t myNbIndexes = inIndexToDelete.size();
+ {
+ MPI_Request requests[4];
+ AssertMpi(MPI_Isend(&myNbIndexes, 1, particles_utils::GetMpiType(partsize_t()),
+ (my_rank+1)%nb_processes, BottomToTop, mpi_com, &requests[0]));
+ AssertMpi(MPI_Isend(&myNbIndexes, 1, particles_utils::GetMpiType(partsize_t()),
+ (my_rank+nb_processes-1)%nb_processes, TopToBottom, mpi_com, &requests[1]));
+
+ AssertMpi(MPI_Irecv(&nbIndexesFromTop, 1, particles_utils::GetMpiType(partsize_t()),
+ (my_rank+1)%nb_processes, TopToBottom, mpi_com, &requests[2]));
+ AssertMpi(MPI_Irecv(&nbIndexesFromBottom, 1, particles_utils::GetMpiType(partsize_t()),
+ (my_rank+nb_processes-1)%nb_processes, BottomToTop, mpi_com, &requests[3]));
+
+ AssertMpi(MPI_Waitall(4, requests, MPI_STATUSES_IGNORE));
+ }
+ {
+ MPI_Request requests[4];
+ int nbRequests = 0;
+
+ if(myNbIndexes){
+ AssertMpi(MPI_Isend(&inIndexToDelete[0], myNbIndexes, particles_utils::GetMpiType(partsize_t()),
+ (my_rank+1)%nb_processes, BottomToTop, mpi_com, &requests[nbRequests++]));
+ AssertMpi(MPI_Isend(&inIndexToDelete[0], myNbIndexes, particles_utils::GetMpiType(partsize_t()),
+ (my_rank+nb_processes-1)%nb_processes, TopToBottom, mpi_com, &requests[nbRequests++]));
+ }
+
+ std::vector<partsize_t> indexesFromTop(nbIndexesFromTop);
+ std::vector<partsize_t> indexesFromBottom(nbIndexesFromTop);
+
+ if(nbIndexesFromTop){
+ AssertMpi(MPI_Irecv(&indexesFromTop[0], int(nbIndexesFromTop), particles_utils::GetMpiType(partsize_t()),
+ (my_rank+1)%nb_processes, TopToBottom, mpi_com, &requests[nbRequests++]));
+ }
+ if(nbIndexesFromBottom){
+ AssertMpi(MPI_Irecv(&indexesFromBottom[0], int(nbIndexesFromTop), particles_utils::GetMpiType(partsize_t()),
+ (my_rank+nb_processes-1)%nb_processes, BottomToTop, mpi_com, &requests[nbRequests++]));
+ }
+
+ AssertMpi(MPI_Waitall(nbRequests, requests, MPI_STATUSES_IGNORE));
+
+ std::copy( indexesFromTop.begin(), indexesFromTop.end(), std::inserter( setIndexToDelete, setIndexToDelete.end() ) );
+ std::copy( indexesFromBottom.begin(), indexesFromBottom.end(), std::inserter( setIndexToDelete, setIndexToDelete.end() ) );
+ }
+ }
+
+ if(setIndexToDelete.size()){
+ partsize_t nbDeletedParticles = 0;
+
+ for(int idxPartition = 0 ; idxPartition < partition_interval_size ; ++idxPartition){
+ const partsize_t nbDeletedParticlesBefore = nbDeletedParticles;
+ for(partsize_t idx = current_offset_particles_for_partition[idxPartition] ; idx < current_offset_particles_for_partition[idxPartition+1] ; ++idx){
+ if(setIndexToDelete.find(my_particles_positions_indexes[idx]) != setIndexToDelete.end()){
+ nbDeletedParticles += 1;
+ }
+ else if(nbDeletedParticles){
+ my_particles_positions_indexes[idx-nbDeletedParticles] = my_particles_positions_indexes[idx];
+
+ for(int idx_pos = 0 ; idx_pos < size_particle_positions ; ++idx_pos){
+ my_particles_positions[(idx-nbDeletedParticles)*size_particle_positions+idx_pos] =
+ my_particles_positions[idx*size_particle_positions+idx_pos];
+ }
+
+ for(int idx_rhs = 0 ; idx_rhs < int(my_particles_rhs.size()) ; ++idx_rhs){
+ for(int idx_val = 0 ; idx_val < size_particle_rhs ; ++idx_val){
+ my_particles_rhs[idx_rhs][(idx-nbDeletedParticles)*size_particle_rhs + idx_val] =
+ my_particles_rhs[idx_rhs][idx*size_particle_rhs + idx_val];
+ }
+ }
+ }
+ }
+
+ current_offset_particles_for_partition[idxPartition] -= nbDeletedParticlesBefore;
+ }
+
+ current_offset_particles_for_partition[partition_interval_size] -= nbDeletedParticles;
+
+ my_nb_particles -= nbDeletedParticles;
+ assert(my_nb_particles == current_offset_particles_for_partition[partition_interval_size]);
+ }
+
+ AssertMpi(MPI_Allreduce(const_cast<partsize_t*>(&my_nb_particles), &total_nb_particles, 1,
+ particles_utils::GetMpiType(partsize_t()), MPI_SUM, mpi_com));
+ }
+
+ void compute() final {
+ TIMEZONE("particles_system::compute");
+ particles_distr.template compute_distr<computer_class, field_class, size_particle_positions, size_particle_rhs>(
+ computer, default_field,
+ current_my_nb_particles_per_partition.get(),
+ my_particles_positions.get(),
+ my_particles_rhs.front().get(),
+ interp_neighbours);
+ }
+
+ void compute_p2p() final {
+ // TODO P2P
+ if(computer_p2p.isEnable() == true){
+ TIMEZONE("particles_system::compute_p2p");
+ distr_p2p.template compute_distr<p2p_computer_class, size_particle_positions, size_particle_rhs>(
+ computer_p2p,
+ current_my_nb_particles_per_partition.get(),
+ my_particles_positions.get(),
+ my_particles_rhs.data(),
+ int(my_particles_rhs.size()),
+ my_particles_positions_indexes.get());
+ }
+ }
+
+ void compute_particles_inner() final {
+ if(computer_particules_inner.isEnable() == true){
+ TIMEZONE("particles_system::compute_particles_inner");
+ computer_particules_inner.template compute_interaction<size_particle_positions, size_particle_rhs>(
+ my_nb_particles, my_particles_positions.get(), my_particles_rhs.front().get());
+ }
+ }
+
+ void add_Lagrange_multipliers() final {
+ if(computer_particules_inner.isEnable() == true){
+ TIMEZONE("particles_system::add_Lagrange_multipliers");
+ computer_particules_inner.template add_Lagrange_multipliers<size_particle_positions, size_particle_rhs>(
+ my_nb_particles, my_particles_positions.get(), my_particles_rhs.front().get());
+ }
+ }
+
+ void enforce_unit_orientation() final {
+ if(computer_particules_inner.isEnable() == true){
+ TIMEZONE("particles_system::enforce_unit_orientation");
+ computer_particules_inner.template enforce_unit_orientation<size_particle_positions>(
+ my_nb_particles, my_particles_positions.get());
+ }
+ }
+
+ void compute_sphere_particles_inner(const real_number particle_extra_field[]) final {
+ if(computer_particules_inner.isEnable() == true){
+ TIMEZONE("particles_system::compute_sphere_particles_inner");
+ computer_particules_inner.template compute_interaction_with_extra<size_particle_positions, size_particle_rhs, 3>(
+ my_nb_particles, my_particles_positions.get(), my_particles_rhs.front().get(),
+ particle_extra_field);
+ }
+ }
+
+ void compute_ellipsoid_particles_inner(const real_number particle_extra_field[]) final {
+ if(computer_particules_inner.isEnable() == true){
+ TIMEZONE("particles_system::compute_ellipsoid_particles_inner");
+ computer_particules_inner.template compute_interaction_with_extra<size_particle_positions, size_particle_rhs, 9>(
+ my_nb_particles, my_particles_positions.get(), my_particles_rhs.front().get(),
+ particle_extra_field);
+ }
+ }
+
+ template <class sample_field_class, int sample_size_particle_rhs>
+ void sample_compute(const sample_field_class& sample_field,
+ real_number sample_rhs[]) {
+ TIMEZONE("particles_system::compute");
+ particles_distr.template compute_distr<computer_class, sample_field_class, size_particle_positions, sample_size_particle_rhs>(
+ computer, sample_field,
+ current_my_nb_particles_per_partition.get(),
+ my_particles_positions.get(),
+ sample_rhs,
+ interp_neighbours);
+ }
+
+ //- Not generic to enable sampling begin
+ void sample_compute_field(const field<float, FFTW, ONE>& sample_field,
+ real_number sample_rhs[]) final {
+ sample_compute<decltype(sample_field), 1>(sample_field, sample_rhs);
+ }
+ void sample_compute_field(const field<float, FFTW, THREE>& sample_field,
+ real_number sample_rhs[]) final {
+ sample_compute<decltype(sample_field), 3>(sample_field, sample_rhs);
+ }
+ void sample_compute_field(const field<float, FFTW, THREExTHREE>& sample_field,
+ real_number sample_rhs[]) final {
+ sample_compute<decltype(sample_field), 9>(sample_field, sample_rhs);
+ }
+ void sample_compute_field(const field<double, FFTW, ONE>& sample_field,
+ real_number sample_rhs[]) final {
+ sample_compute<decltype(sample_field), 1>(sample_field, sample_rhs);
+ }
+ void sample_compute_field(const field<double, FFTW, THREE>& sample_field,
+ real_number sample_rhs[]) final {
+ sample_compute<decltype(sample_field), 3>(sample_field, sample_rhs);
+ }
+ void sample_compute_field(const field<double, FFTW, THREExTHREE>& sample_field,
+ real_number sample_rhs[]) final {
+ sample_compute<decltype(sample_field), 9>(sample_field, sample_rhs);
+ }
+ //- Not generic to enable sampling end
+
+ void move(const real_number dt) final {
+ TIMEZONE("particles_system::move");
+ positions_updater.move_particles(my_particles_positions.get(), my_nb_particles,
+ my_particles_rhs.data(), std::min(step_idx, int(my_particles_rhs.size())),
+ dt);
+ }
+
+ void redistribute() final {
+ TIMEZONE("particles_system::redistribute");
+ //DEBUG_MSG("step index is %d\n", step_idx);
+ particles_distr.template redistribute<computer_class, size_particle_positions, size_particle_rhs, 1>(
+ computer,
+ current_my_nb_particles_per_partition.get(),
+ &my_nb_particles,
+ &my_particles_positions,
+ my_particles_rhs.data(), int(my_particles_rhs.size()),
+ &my_particles_positions_indexes);
+ }
+
+ void inc_step_idx() final {
+ step_idx += 1;
+ }
+
+ int get_step_idx() const final {
+ return step_idx;
+ }
+
+ void shift_rhs_vectors() final {
+ if(my_particles_rhs.size()){
+ std::unique_ptr<real_number[]> next_current(std::move(my_particles_rhs.back()));
+ for(int idx_rhs = int(my_particles_rhs.size())-1 ; idx_rhs > 0 ; --idx_rhs){
+ my_particles_rhs[idx_rhs] = std::move(my_particles_rhs[idx_rhs-1]);
+ }
+ my_particles_rhs[0] = std::move(next_current);
+ particles_utils::memzero(my_particles_rhs[0], size_particle_rhs*my_nb_particles);
+ }
+ }
+
+ void completeLoop(const real_number dt) final {
+ start_mpi_profiling_zone(turtle_mpi_pcontrol::PARTICLES);
+ TIMEZONE("particles_system::completeLoop");
+ compute();
+ compute_particles_inner();
+ compute_p2p();
+ move(dt);
+ enforce_unit_orientation();
+ redistribute();
+ inc_step_idx();
+ shift_rhs_vectors();
+ finish_mpi_profiling_zone(turtle_mpi_pcontrol::PARTICLES);
+ }
+
+ void complete2ndOrderLoop(const real_number dt) final {
+ start_mpi_profiling_zone(turtle_mpi_pcontrol::PARTICLES);
+ assert((size_particle_positions == 6) || (size_particle_positions == 7));
+ assert(size_particle_rhs == size_particle_positions);
+ std::unique_ptr<real_number[]> sampled_velocity(new real_number[getLocalNbParticles()*3]());
+ std::fill_n(sampled_velocity.get(), 3*getLocalNbParticles(), 0);
+ sample_compute_field(default_field, sampled_velocity.get());
+ this->computer_particules_inner.template compute_interaction_with_extra<size_particle_positions, size_particle_rhs, 3>(
+ my_nb_particles, my_particles_positions.get(), my_particles_rhs.front().get(),
+ sampled_velocity.get());
+ move(dt);
+ redistribute();
+ inc_step_idx();
+ shift_rhs_vectors();
+ finish_mpi_profiling_zone(turtle_mpi_pcontrol::PARTICLES);
+ }
+
+ void completeLoopWithVorticity(
+ const real_number dt,
+ const real_number particle_extra_field[]) final {
+ start_mpi_profiling_zone(turtle_mpi_pcontrol::PARTICLES);
+ TIMEZONE("particles_system::completeLoopWithVorticity");
+ compute();
+ compute_sphere_particles_inner(particle_extra_field);
+ // p2p must be placed after compute_inner
+ // because compute inner is using samples.
+ // p2p may, in principle, reorder the particle state (and labels).
+ // by enforcing this calling order, we enforce that the samples
+ // are sorted consistently with the particle data/labels.
+ compute_p2p();
+ move(dt);
+ enforce_unit_orientation();
+ redistribute();
+ inc_step_idx();
+ shift_rhs_vectors();
+ finish_mpi_profiling_zone(turtle_mpi_pcontrol::PARTICLES);
+ }
+
+ void completeLoopWithVelocityGradient(
+ const real_number dt,
+ const real_number particle_extra_field[]) final {
+ start_mpi_profiling_zone(turtle_mpi_pcontrol::PARTICLES);
+ TIMEZONE("particles_system::completeLoopWithVelocityGradient");
+ compute();
+ compute_ellipsoid_particles_inner(particle_extra_field);
+ // p2p must be placed after compute_inner
+ // because compute inner is using samples.
+ // p2p may, in principle, reorder the particle state (and labels).
+ // by enforcing this calling order, we enforce that the samples
+ // are sorted consistently with the particle data/labels.
+ compute_p2p();
+ move(dt);
+ enforce_unit_orientation();
+ redistribute();
+ inc_step_idx();
+ shift_rhs_vectors();
+ finish_mpi_profiling_zone(turtle_mpi_pcontrol::PARTICLES);
+ }
+
+ const real_number* getParticlesState() const final {
+ return my_particles_positions.get();
+ }
+
+ std::unique_ptr<real_number[]> extractParticlesState(const int firstState, const int lastState) const final {
+ const int nbStates = std::max(0,(std::min(lastState,size_particle_positions)-firstState));
+
+ std::unique_ptr<real_number[]> stateExtract(new real_number[my_nb_particles*nbStates]);
+
+ for(partsize_t idx_part = 0 ; idx_part < my_nb_particles ; ++idx_part){
+ for(int idxState = 0 ; idxState < nbStates ; ++idxState){
+ stateExtract[idx_part*nbStates + idxState] = my_particles_positions[idx_part*size_particle_positions + idxState+firstState];
+ }
+ }
+
+ return stateExtract;
+ }
+
+ const std::unique_ptr<real_number[]>* getParticlesRhs() const final {
+ return my_particles_rhs.data();
+ }
+
+ const partsize_t* getParticlesIndexes() const final {
+ return my_particles_positions_indexes.get();
+ }
+
+ partsize_t getLocalNbParticles() const final {
+ return my_nb_particles;
+ }
+
+ partsize_t getGlobalNbParticles() const final {
+ return total_nb_particles;
+ }
+
+ int getNbRhs() const final {
+ return int(my_particles_rhs.size());
+ }
+
+ int setParticleFileLayout(std::vector<hsize_t> input_layout) final{
+ this->particle_file_layout.resize(input_layout.size());
+ for (unsigned int i=0; i<this->particle_file_layout.size(); i++)
+ this->particle_file_layout[i] = input_layout[i];
+ return EXIT_SUCCESS;
+ }
+
+ std::vector<hsize_t> getParticleFileLayout(void) final{
+ return std::vector<hsize_t>(this->particle_file_layout);
+ }
+
+ void checkNan() const { // TODO remove
+ for(partsize_t idx_part = 0 ; idx_part < my_nb_particles ; ++idx_part){ // TODO remove me
+ assert(std::isnan(my_particles_positions[idx_part*size_particle_positions+IDXC_X]) == false);
+ assert(std::isnan(my_particles_positions[idx_part*size_particle_positions+IDXC_Y]) == false);
+ assert(std::isnan(my_particles_positions[idx_part*size_particle_positions+IDXC_Z]) == false);
+
+ for(int idx_rhs = 0 ; idx_rhs < my_particles_rhs.size() ; ++idx_rhs){
+ for(int idx_rhs_val = 0 ; idx_rhs_val < size_particle_rhs ; ++idx_rhs_val){
+ assert(std::isnan(my_particles_rhs[idx_rhs][idx_part*size_particle_rhs+idx_rhs_val]) == false);
+ }
+ }
+ }
+ }
+
+ const p2p_computer_class& getP2PComputer() const final{
+ return computer_p2p;
+ }
+
+ p2p_computer_class& getP2PComputer() final{
+ return computer_p2p;
+ }
+
+ const particles_inner_computer_class& getInnerComputer() const{
+ return computer_particules_inner;
+ }
+
+ particles_inner_computer_class& getInnnerComputer(){
+ return computer_particules_inner;
+ }
+};
+
+
+#endif
diff --git a/bfps/cpp/particles/particles_system_builder.hpp b/cpp/particles/particles_system_builder.hpp
similarity index 51%
rename from bfps/cpp/particles/particles_system_builder.hpp
rename to cpp/particles/particles_system_builder.hpp
index 7a2d49c07c3a6de21fb93d83b338609be858f0dc..dd2b775032fa5ffe8e3439a500419e83d219de1e 100644
--- a/bfps/cpp/particles/particles_system_builder.hpp
+++ b/cpp/particles/particles_system_builder.hpp
@@ -1,22 +1,58 @@
+/******************************************************************************
+* *
+* Copyright 2019 Max Planck Institute for Dynamics and Self-Organization *
+* *
+* This file is part of TurTLE. *
+* *
+* TurTLE is free software: you can redistribute it and/or modify *
+* it under the terms of the GNU General Public License as published *
+* by the Free Software Foundation, either version 3 of the License, *
+* or (at your option) any later version. *
+* *
+* TurTLE is distributed in the hope that it will be useful, *
+* but WITHOUT ANY WARRANTY; without even the implied warranty of *
+* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the *
+* GNU General Public License for more details. *
+* *
+* You should have received a copy of the GNU General Public License *
+* along with TurTLE. If not, see <http://www.gnu.org/licenses/> *
+* *
+* Contact: Cristian.Lalescu@ds.mpg.de *
+* *
+******************************************************************************/
+
+
+
#ifndef PARTICLES_SYSTEM_BUILDER_HPP
#define PARTICLES_SYSTEM_BUILDER_HPP
#include <string>
-#include "abstract_particles_system.hpp"
-#include "particles_system.hpp"
-#include "particles_input_hdf5.hpp"
-#include "particles_generic_interp.hpp"
+#include <cmath>
+#include "particles/abstract_particles_system.hpp"
+#include "particles/particles_system.hpp"
+#include "particles/particles_input_hdf5.hpp"
+#include "particles/interpolation/particles_generic_interp.hpp"
+#include "particles/p2p/p2p_computer_empty.hpp"
+#include "particles/inner/particles_inner_computer_empty.hpp"
#include "field.hpp"
#include "kspace.hpp"
-
+const int MAX_INTERPOLATION_NEIGHBOURS=7;
+const int MAX_INTERPOLATION_SMOOTHNESS=4;
//////////////////////////////////////////////////////////////////////////////
///
/// Double template "for"
///
+/// "Template_double_for_if" is used to systematically generate specialized
+/// templates for two ranges of template parameters.
+/// For the interpolation we have `n` and `m` that designate "number of
+/// neighbours" and "smoothness of interpolant".
+/// Calling Template_double_for_if is essentially equivalent to having a couple
+/// of nested "switch" statements, but without all the repetitive code lines.
+///
//////////////////////////////////////////////////////////////////////////////
namespace Template_double_for_if{
@@ -108,10 +144,11 @@ inline RetType evaluate(IterType1 value1, IterType2 value2, Args... args){
///
//////////////////////////////////////////////////////////////////////////////
-template <class partsize_t, class field_rnumber, field_backend be, field_components fc, class particles_rnumber>
+template <class partsize_t, class field_rnumber, field_backend be, field_components fc, class particles_rnumber, class p2p_computer_class,
+ class particles_inner_computer_class, int size_particle_positions, int size_particle_rhs>
struct particles_system_build_container {
template <const int interpolation_size, const int spline_mode>
- static std::unique_ptr<abstract_particles_system<partsize_t, particles_rnumber>> instanciate(
+ static std::unique_ptr<abstract_particles_system_with_p2p<partsize_t, particles_rnumber, p2p_computer_class>> instanciate(
const field<field_rnumber, be, fc>* fs_field, // (field object)
const kspace<be, SMOOTH>* fs_kk, // (kspace object, contains dkx, dky, dkz)
const int nsteps, // to check coherency between parameters and hdf input file (nb rhs)
@@ -119,25 +156,28 @@ struct particles_system_build_container {
const std::string& fname_input, // particles input filename
const std::string& inDatanameState, const std::string& inDatanameRhs, // input dataset names
MPI_Comm mpi_comm,
- const int in_current_iteration){
+ const int in_current_iteration,
+ p2p_computer_class p2p_computer,
+ particles_inner_computer_class inner_computer,
+ const particles_rnumber cutoff = std::numeric_limits<particles_rnumber>::max()){
// The size of the field grid (global size) all_size seems
std::array<size_t,3> field_grid_dim;
- field_grid_dim[IDX_X] = fs_field->rlayout->sizes[FIELD_IDX_X];// nx
- field_grid_dim[IDX_Y] = fs_field->rlayout->sizes[FIELD_IDX_Y];// nx
- field_grid_dim[IDX_Z] = fs_field->rlayout->sizes[FIELD_IDX_Z];// nz
+ field_grid_dim[IDXC_X] = fs_field->rlayout->sizes[IDXV_X];// nx
+ field_grid_dim[IDXC_Y] = fs_field->rlayout->sizes[IDXV_Y];// nx
+ field_grid_dim[IDXC_Z] = fs_field->rlayout->sizes[IDXV_Z];// nz
// The size of the local field grid (the field nodes that belong to current process)
std::array<size_t,3> local_field_dims;
- local_field_dims[IDX_X] = fs_field->rlayout->subsizes[FIELD_IDX_X];
- local_field_dims[IDX_Y] = fs_field->rlayout->subsizes[FIELD_IDX_Y];
- local_field_dims[IDX_Z] = fs_field->rlayout->subsizes[FIELD_IDX_Z];
+ local_field_dims[IDXC_X] = fs_field->rlayout->subsizes[IDXV_X];
+ local_field_dims[IDXC_Y] = fs_field->rlayout->subsizes[IDXV_Y];
+ local_field_dims[IDXC_Z] = fs_field->rlayout->subsizes[IDXV_Z];
// The offset of the local field grid
std::array<size_t,3> local_field_offset;
- local_field_offset[IDX_X] = fs_field->rlayout->starts[FIELD_IDX_X];
- local_field_offset[IDX_Y] = fs_field->rlayout->starts[FIELD_IDX_Y];
- local_field_offset[IDX_Z] = fs_field->rlayout->starts[FIELD_IDX_Z];
+ local_field_offset[IDXC_X] = fs_field->rlayout->starts[IDXV_X];
+ local_field_offset[IDXC_Y] = fs_field->rlayout->starts[IDXV_Y];
+ local_field_offset[IDXC_Z] = fs_field->rlayout->starts[IDXV_Z];
// Retreive split from fftw to know processes that have no work
@@ -145,57 +185,60 @@ struct particles_system_build_container {
AssertMpi(MPI_Comm_rank(mpi_comm, &my_rank));
AssertMpi(MPI_Comm_size(mpi_comm, &nb_processes));
- const int split_step = (int(field_grid_dim[IDX_Z])+nb_processes-1)/nb_processes;
- const int nb_processes_involved = (int(field_grid_dim[IDX_Z])+split_step-1)/split_step;
+ const int split_step = (int(field_grid_dim[IDXC_Z])+nb_processes-1)/nb_processes;
+ const int nb_processes_involved = (int(field_grid_dim[IDXC_Z])+split_step-1)/split_step;
- assert((my_rank < nb_processes_involved && local_field_dims[IDX_Z] != 0)
- || (nb_processes_involved <= my_rank && local_field_dims[IDX_Z] == 0));
- assert(nb_processes_involved <= int(field_grid_dim[IDX_Z]));
+ assert((my_rank < nb_processes_involved && local_field_dims[IDXC_Z] != 0)
+ || (nb_processes_involved <= my_rank && local_field_dims[IDXC_Z] == 0));
+ assert(nb_processes_involved <= int(field_grid_dim[IDXC_Z]));
// Make the idle processes starting from the limit (and not 0 as set by fftw)
if(nb_processes_involved <= my_rank){
- local_field_offset[IDX_Z] = field_grid_dim[IDX_Z];
+ local_field_offset[IDXC_Z] = field_grid_dim[IDXC_Z];
}
// Ensure that 1D partitioning is used
{
- assert(local_field_offset[IDX_X] == 0);
- assert(local_field_offset[IDX_Y] == 0);
- assert(local_field_dims[IDX_X] == field_grid_dim[IDX_X]);
- assert(local_field_dims[IDX_Y] == field_grid_dim[IDX_Y]);
-
- assert(my_rank >= nb_processes_involved || ((my_rank == 0 && local_field_offset[IDX_Z] == 0)
- || (my_rank != 0 && local_field_offset[IDX_Z] != 0)));
- assert(my_rank >= nb_processes_involved || ((my_rank == nb_processes_involved-1 && local_field_offset[IDX_Z]+local_field_dims[IDX_Z] == field_grid_dim[IDX_Z])
- || (my_rank != nb_processes_involved-1 && local_field_offset[IDX_Z]+local_field_dims[IDX_Z] != field_grid_dim[IDX_Z])));
+ assert(local_field_offset[IDXC_X] == 0);
+ assert(local_field_offset[IDXC_Y] == 0);
+ assert(local_field_dims[IDXC_X] == field_grid_dim[IDXC_X]);
+ assert(local_field_dims[IDXC_Y] == field_grid_dim[IDXC_Y]);
+
+ assert(my_rank >= nb_processes_involved || ((my_rank == 0 && local_field_offset[IDXC_Z] == 0)
+ || (my_rank != 0 && local_field_offset[IDXC_Z] != 0)));
+ assert(my_rank >= nb_processes_involved || ((my_rank == nb_processes_involved-1 && local_field_offset[IDXC_Z]+local_field_dims[IDXC_Z] == field_grid_dim[IDXC_Z])
+ || (my_rank != nb_processes_involved-1 && local_field_offset[IDXC_Z]+local_field_dims[IDXC_Z] != field_grid_dim[IDXC_Z])));
}
// The spatial box size (all particles should be included inside)
std::array<particles_rnumber,3> spatial_box_width;
- spatial_box_width[IDX_X] = 4 * acos(0) / (fs_kk->dkx);
- spatial_box_width[IDX_Y] = 4 * acos(0) / (fs_kk->dky);
- spatial_box_width[IDX_Z] = 4 * acos(0) / (fs_kk->dkz);
+ spatial_box_width[IDXC_X] = 4 * acos(0) / (fs_kk->dkx);
+ spatial_box_width[IDXC_Y] = 4 * acos(0) / (fs_kk->dky);
+ spatial_box_width[IDXC_Z] = 4 * acos(0) / (fs_kk->dkz);
// Box is in the corner
std::array<particles_rnumber,3> spatial_box_offset;
- spatial_box_offset[IDX_X] = 0;
- spatial_box_offset[IDX_Y] = 0;
- spatial_box_offset[IDX_Z] = 0;
+ spatial_box_offset[IDXC_X] = 0;
+ spatial_box_offset[IDXC_Y] = 0;
+ spatial_box_offset[IDXC_Z] = 0;
// The distance between two field nodes in z
std::array<particles_rnumber,3> spatial_partition_width;
- spatial_partition_width[IDX_X] = spatial_box_width[IDX_X]/particles_rnumber(field_grid_dim[IDX_X]);
- spatial_partition_width[IDX_Y] = spatial_box_width[IDX_Y]/particles_rnumber(field_grid_dim[IDX_Y]);
- spatial_partition_width[IDX_Z] = spatial_box_width[IDX_Z]/particles_rnumber(field_grid_dim[IDX_Z]);
+ spatial_partition_width[IDXC_X] = spatial_box_width[IDXC_X]/particles_rnumber(field_grid_dim[IDXC_X]);
+ spatial_partition_width[IDXC_Y] = spatial_box_width[IDXC_Y]/particles_rnumber(field_grid_dim[IDXC_Y]);
+ spatial_partition_width[IDXC_Z] = spatial_box_width[IDXC_Z]/particles_rnumber(field_grid_dim[IDXC_Z]);
// The spatial interval of the current process
- const particles_rnumber my_spatial_low_limit_z = particles_rnumber(local_field_offset[IDX_Z])*spatial_partition_width[IDX_Z];
- const particles_rnumber my_spatial_up_limit_z = particles_rnumber(local_field_offset[IDX_Z]+local_field_dims[IDX_Z])*spatial_partition_width[IDX_Z];
+ const particles_rnumber my_spatial_low_limit_z = particles_rnumber(local_field_offset[IDXC_Z])*spatial_partition_width[IDXC_Z];
+ const particles_rnumber my_spatial_up_limit_z = particles_rnumber(local_field_offset[IDXC_Z]+local_field_dims[IDXC_Z])*spatial_partition_width[IDXC_Z];
// Create the particles system
using particles_system_type = particles_system<partsize_t, particles_rnumber, field_rnumber,
field<field_rnumber, be, fc>,
particles_generic_interp<particles_rnumber, interpolation_size,spline_mode>,
- interpolation_size, ncomp(fc)>;
+ interpolation_size,
+ size_particle_positions, size_particle_rhs,
+ p2p_computer_class,
+ particles_inner_computer_class>;
particles_system_type* part_sys = new particles_system_type(field_grid_dim,
spatial_box_width,
spatial_box_offset,
@@ -207,11 +250,23 @@ struct particles_system_build_container {
(*fs_field),
mpi_comm,
nparticles,
+ cutoff,
+ p2p_computer,
+ inner_computer,
in_current_iteration);
+ // TODO P2P load particle data too
// Load particles from hdf5
- particles_input_hdf5<partsize_t, particles_rnumber, 3,3> generator(mpi_comm, fname_input,
- inDatanameState, inDatanameRhs, my_spatial_low_limit_z, my_spatial_up_limit_z);
+ particles_input_hdf5<partsize_t,
+ particles_rnumber,
+ size_particle_positions,
+ size_particle_rhs> generator(
+ mpi_comm,
+ fname_input,
+ inDatanameState,
+ inDatanameRhs,
+ my_spatial_low_limit_z,
+ my_spatial_up_limit_z);
// Ensure parameters match the input file
if(generator.getNbRhs() != nsteps){
@@ -229,8 +284,11 @@ struct particles_system_build_container {
assert(part_sys->getNbRhs() == nsteps);
+ // store particle file layout
+ part_sys->setParticleFileLayout(generator.getParticleFileLayout());
+
// Return the created particles system
- return std::unique_ptr<abstract_particles_system<partsize_t, particles_rnumber>>(part_sys);
+ return std::unique_ptr<abstract_particles_system_with_p2p<partsize_t, particles_rnumber, p2p_computer_class>>(part_sys);
}
};
@@ -248,12 +306,46 @@ inline std::unique_ptr<abstract_particles_system<partsize_t, particles_rnumber>>
MPI_Comm mpi_comm,
const int in_current_iteration){
return Template_double_for_if::evaluate<std::unique_ptr<abstract_particles_system<partsize_t, particles_rnumber>>,
- int, 1, 11, 1, // interpolation_size
- int, 0, 3, 1, // spline_mode
- particles_system_build_container<partsize_t, field_rnumber,be,fc,particles_rnumber>>(
+ int, 1, MAX_INTERPOLATION_NEIGHBOURS, 1, // interpolation_size
+ int, 0, MAX_INTERPOLATION_SMOOTHNESS, 1, // spline_mode
+ particles_system_build_container<partsize_t, field_rnumber,be,fc,particles_rnumber,
+ p2p_computer_empty<particles_rnumber,partsize_t>,
+ particles_inner_computer_empty<particles_rnumber,partsize_t>,
+ 3,3>>(
+ interpolation_size, // template iterator 1
+ spline_mode, // template iterator 2
+ fs_field,fs_kk, nsteps, nparticles, fname_input, inDatanameState, inDatanameRhs, mpi_comm, in_current_iteration,
+ p2p_computer_empty<particles_rnumber,partsize_t>(), particles_inner_computer_empty<particles_rnumber,partsize_t>());
+}
+
+template <class partsize_t, class field_rnumber, field_backend be, field_components fc,
+ class p2p_computer_class, class particles_inner_computer_class,
+ class particles_rnumber = double>
+inline std::unique_ptr<abstract_particles_system_with_p2p<partsize_t, particles_rnumber, p2p_computer_class>> particles_system_builder_with_p2p(
+ const field<field_rnumber, be, fc>* fs_field, // (field object)
+ const kspace<be, SMOOTH>* fs_kk, // (kspace object, contains dkx, dky, dkz)
+ const int nsteps, // to check coherency between parameters and hdf input file (nb rhs)
+ const partsize_t nparticles, // to check coherency between parameters and hdf input file
+ const std::string& fname_input, // particles input filename
+ const std::string& inDatanameState, const std::string& inDatanameRhs, // input dataset names
+ const int interpolation_size,
+ const int spline_mode,
+ MPI_Comm mpi_comm,
+ const int in_current_iteration,
+ p2p_computer_class p2p_computer,
+ particles_inner_computer_class inner_computer,
+ const particles_rnumber cutoff){
+ return Template_double_for_if::evaluate<std::unique_ptr<abstract_particles_system_with_p2p<partsize_t, particles_rnumber, p2p_computer_class>>,
+ int, 1, MAX_INTERPOLATION_NEIGHBOURS, 1, // interpolation_size
+ int, 0, MAX_INTERPOLATION_SMOOTHNESS, 1, // spline_mode
+ particles_system_build_container<partsize_t, field_rnumber,be,fc,particles_rnumber,
+ p2p_computer_class,
+ particles_inner_computer_class,
+ 6,6>>(
interpolation_size, // template iterator 1
spline_mode, // template iterator 2
- fs_field,fs_kk, nsteps, nparticles, fname_input, inDatanameState, inDatanameRhs, mpi_comm, in_current_iteration);
+ fs_field,fs_kk, nsteps, nparticles, fname_input, inDatanameState, inDatanameRhs, mpi_comm, in_current_iteration,
+ std::move(p2p_computer), std::move(inner_computer), cutoff);
}
diff --git a/bfps/cpp/particles/particles_utils.hpp b/cpp/particles/particles_utils.hpp
similarity index 70%
rename from bfps/cpp/particles/particles_utils.hpp
rename to cpp/particles/particles_utils.hpp
index 146dc4399477b72c30329edff587d35d7b44d69d..6ceb8490b88ec783ba9feb4fd2104c1626ddb0a4 100644
--- a/bfps/cpp/particles/particles_utils.hpp
+++ b/cpp/particles/particles_utils.hpp
@@ -1,3 +1,28 @@
+/******************************************************************************
+* *
+* Copyright 2019 Max Planck Institute for Dynamics and Self-Organization *
+* *
+* This file is part of TurTLE. *
+* *
+* TurTLE is free software: you can redistribute it and/or modify *
+* it under the terms of the GNU General Public License as published *
+* by the Free Software Foundation, either version 3 of the License, *
+* or (at your option) any later version. *
+* *
+* TurTLE is distributed in the hope that it will be useful, *
+* but WITHOUT ANY WARRANTY; without even the implied warranty of *
+* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the *
+* GNU General Public License for more details. *
+* *
+* You should have received a copy of the GNU General Public License *
+* along with TurTLE. If not, see <http://www.gnu.org/licenses/> *
+* *
+* Contact: Cristian.Lalescu@ds.mpg.de *
+* *
+******************************************************************************/
+
+
+
#ifndef PARTICLES_UTILS_HPP
#define PARTICLES_UTILS_HPP
@@ -8,6 +33,7 @@
#include <vector>
#include <memory>
#include <cstring>
+#include <stdexcept>
#if _OPENMP < 201511
#warning Openmp priority is not supported here
@@ -16,19 +42,44 @@
#ifndef AssertMpi
-#define AssertMpi(X) if(MPI_SUCCESS != (X)) { printf("MPI Error at line %d\n",__LINE__); fflush(stdout) ; throw std::runtime_error("Stop from from mpi erro"); }
+#define AssertMpi(X) if(MPI_SUCCESS != (X)) { printf("MPI Error at line %d\n",__LINE__); fflush(stdout) ; throw std::runtime_error("Stop from from mpi error"); }
#endif
-enum IDXS_3D {
- IDX_X = 0,
- IDX_Y = 1,
- IDX_Z = 2
+enum IDX_COMPONENT_3D {
+ IDXC_X = 0,
+ IDXC_Y = 1,
+ IDXC_Z = 2
+};
+
+enum IDX_COMPONENT_DEL_3D {
+ // Di_j is (derivative with respect to i direction) of (component j)
+ IDXC_DX_X = 0,
+ IDXC_DX_Y = 1,
+ IDXC_DX_Z = 2,
+ IDXC_DY_X = 3,
+ IDXC_DY_Y = 4,
+ IDXC_DY_Z = 5,
+ IDXC_DZ_X = 6,
+ IDXC_DZ_Y = 7,
+ IDXC_DZ_Z = 8,
+};
+
+enum IDX_COMPONENT_GSL_3x3D {
+ IDXC_GSL_XX = 0,
+ IDXC_GSL_XY = 1,
+ IDXC_GSL_XZ = 2,
+ IDXC_GSL_YX = 3,
+ IDXC_GSL_YY = 4,
+ IDXC_GSL_YZ = 5,
+ IDXC_GSL_ZX = 6,
+ IDXC_GSL_ZY = 7,
+ IDXC_GSL_ZZ = 8,
};
-enum FIELD_IDXS_3D {
- FIELD_IDX_X = 2,
- FIELD_IDX_Y = 1,
- FIELD_IDX_Z = 0
+enum IDX_VARIABLE_3D {
+ IDXV_X = 2,
+ IDXV_Y = 1,
+ IDXV_Z = 0
};
namespace particles_utils {
@@ -123,7 +174,7 @@ inline void partition_extra_z(real_number* array, const partsize_t size, const i
if(nb_partitions == 2){
const partsize_t size_current = partition_extra<partsize_t, nb_values>(array, size,
[&](const real_number inval[]){
- return partitions_levels(inval[IDX_Z]) == 0;
+ return partitions_levels(inval[IDXC_Z]) == 0;
}, pdcswap);
partitions_size[0] = size_current;
partitions_size[1] = size-size_current;
@@ -152,7 +203,7 @@ inline void partition_extra_z(real_number* array, const partsize_t size, const i
const partsize_t size_current = partition_extra<partsize_t, nb_values>(&array[partitions_offset[current_part.first]*nb_values],
size_unpart,
[&](const real_number inval[]){
- return partitions_levels(inval[IDX_Z]) <= idx_middle;
+ return partitions_levels(inval[IDXC_Z]) <= idx_middle;
}, pdcswap, partitions_offset[current_part.first]);
partitions_offset[idx_middle+1] = size_current + partitions_offset[current_part.first];
@@ -311,7 +362,56 @@ public:
}
};
+}
+template <class real_number>
+std::vector<real_number> BuildLimitsAllProcesses(
+ MPI_Comm mpi_comm,
+ const real_number my_spatial_low_limit,
+ const real_number my_spatial_up_limit){
+ int my_rank;
+ int nb_processes;
+
+ AssertMpi(MPI_Comm_rank(mpi_comm, &my_rank));
+ AssertMpi(MPI_Comm_size(mpi_comm, &nb_processes));
+
+ std::vector<real_number> spatial_limit_per_proc(nb_processes*2);
+
+ real_number intervalToSend[2] = {my_spatial_low_limit, my_spatial_up_limit};
+ AssertMpi(
+ MPI_Allgather(
+ intervalToSend,
+ 2,
+ particles_utils::GetMpiType(real_number()),
+ spatial_limit_per_proc.data(),
+ 2,
+ particles_utils::GetMpiType(real_number()),
+ mpi_comm));
+
+ for(int idx_proc = 0; idx_proc < nb_processes-1 ; ++idx_proc){
+ assert(spatial_limit_per_proc[idx_proc*2] <= spatial_limit_per_proc[idx_proc*2+1]);
+ assert(spatial_limit_per_proc[idx_proc*2+1] == spatial_limit_per_proc[(idx_proc+1)*2]);
+ spatial_limit_per_proc[idx_proc+1] = spatial_limit_per_proc[idx_proc*2+1];
+ }
+ spatial_limit_per_proc[nb_processes] = spatial_limit_per_proc[(nb_processes-1)*2+1];
+ spatial_limit_per_proc.resize(nb_processes+1);
+
+ return spatial_limit_per_proc;
}
-#endif
+template <typename partsize_t, typename rnumber, int size_of_particle>
+int set_particle_data_to_zero(
+ rnumber *data, // TODO: this should be "restrict", but intel can't handle it.
+ const partsize_t numberParticles)
+{
+ // TODO: ensure simd.
+ // don't use openmp here, as this function WILL be called from within openmp parallel regions
+ std::fill_n(
+ data,
+ numberParticles*size_of_particle,
+ 0);
+ return EXIT_SUCCESS;
+}
+
+#endif//PARTICLES_UTILS_HPP
+
diff --git a/cpp/particles/rhs/deformation_tensor_rhs.cpp b/cpp/particles/rhs/deformation_tensor_rhs.cpp
new file mode 100644
index 0000000000000000000000000000000000000000..256b1a4d79bcab0eb44c80b887ed226b06713a12
--- /dev/null
+++ b/cpp/particles/rhs/deformation_tensor_rhs.cpp
@@ -0,0 +1,31 @@
+/******************************************************************************
+* *
+* Copyright 2020 Max Planck Institute for Dynamics and Self-Organization *
+* *
+* This file is part of TurTLE. *
+* *
+* TurTLE is free software: you can redistribute it and/or modify *
+* it under the terms of the GNU General Public License as published *
+* by the Free Software Foundation, either version 3 of the License, *
+* or (at your option) any later version. *
+* *
+* TurTLE is distributed in the hope that it will be useful, *
+* but WITHOUT ANY WARRANTY; without even the implied warranty of *
+* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the *
+* GNU General Public License for more details. *
+* *
+* You should have received a copy of the GNU General Public License *
+* along with TurTLE. If not, see <http://www.gnu.org/licenses/> *
+* *
+* Contact: Cristian.Lalescu@ds.mpg.de *
+* *
+******************************************************************************/
+
+
+
+#include "particles/rhs/deformation_tensor_rhs.hpp"
+
+
+template class deformation_tensor_rhs<float, FFTW, NONE>;
+template class deformation_tensor_rhs<double, FFTW, NONE>;
+
diff --git a/cpp/particles/rhs/deformation_tensor_rhs.hpp b/cpp/particles/rhs/deformation_tensor_rhs.hpp
new file mode 100644
index 0000000000000000000000000000000000000000..0835f9869645d14eacedea5990562dffbd0203e4
--- /dev/null
+++ b/cpp/particles/rhs/deformation_tensor_rhs.hpp
@@ -0,0 +1,175 @@
+/******************************************************************************
+* *
+* Copyright 2020 Max Planck Institute for Dynamics and Self-Organization *
+* *
+* This file is part of TurTLE. *
+* *
+* TurTLE is free software: you can redistribute it and/or modify *
+* it under the terms of the GNU General Public License as published *
+* by the Free Software Foundation, either version 3 of the License, *
+* or (at your option) any later version. *
+* *
+* TurTLE is distributed in the hope that it will be useful, *
+* but WITHOUT ANY WARRANTY; without even the implied warranty of *
+* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the *
+* GNU General Public License for more details. *
+* *
+* You should have received a copy of the GNU General Public License *
+* along with TurTLE. If not, see <http://www.gnu.org/licenses/> *
+* *
+* Contact: Cristian.Lalescu@ds.mpg.de *
+* *
+******************************************************************************/
+
+
+
+#ifndef DEFORMATION_TENSOR_RHS_HPP
+#define DEFORMATION_TENSOR_RHS_HPP
+
+#include "particles/particles_utils.hpp"
+#include "particles/interpolation/field_tinterpolator.hpp"
+#include "particles/abstract_particle_rhs.hpp"
+#include <gsl/gsl_linalg.h>
+
+template <typename rnumber,
+ field_backend be,
+ temporal_interpolation_type tt>
+class deformation_tensor_rhs: public abstract_particle_rhs
+{
+ private:
+ using particle_rnumber = abstract_particle_set::particle_rnumber;
+ using partsize_t = abstract_particle_set::partsize_t;
+
+ protected:
+ field_tinterpolator<rnumber, be, THREE, tt> *velocity;
+ field_tinterpolator<rnumber, be, THREExTHREE, tt> *velocity_gradient;
+
+ public:
+ deformation_tensor_rhs(){}
+ ~deformation_tensor_rhs() noexcept(false){}
+
+ const int getStateSize() const
+ {
+ // 3 numbers for position
+ // 9 numbers for components of deformation tensor
+ // 3 numbers to store logarithmic factors
+ return 15;
+ }
+
+ int setVelocity(field_tinterpolator<rnumber, be, THREE, tt> *in_velocity)
+ {
+ this->velocity = in_velocity;
+ return EXIT_SUCCESS;
+ }
+
+ int setVelocityGradient(field_tinterpolator<rnumber, be, THREExTHREE, tt> *in_velocity_gradient)
+ {
+ this->velocity_gradient = in_velocity_gradient;
+ return EXIT_SUCCESS;
+ }
+
+ int operator()(
+ double t,
+ abstract_particle_set &pset,
+ particle_rnumber *result,
+ std::vector<std::unique_ptr<abstract_particle_set::particle_rnumber[]>> &additional_data)
+ {
+ std::unique_ptr<double[]> pdata3(new double[3*pset.getLocalNumberOfParticles()]);
+ std::unique_ptr<double[]> pdata9(new double[9*pset.getLocalNumberOfParticles()]);
+ // reference to current particle state
+ particle_rnumber *current_state = pset.getParticleState();
+ // interpolation adds on top of existing values, so result arrays must be cleared.
+ for (long long int idx_part = 0; idx_part < pset.getLocalNumberOfParticles(); idx_part++)
+ {
+ std::fill_n(pdata3.get()+3*idx_part, 3, 0);
+ std::fill_n(pdata9.get()+9*idx_part, 9, 0);
+ }
+ // interpolate velocity field
+ (*(this->velocity))(t, pset, pdata3.get());
+ (*(this->velocity_gradient))(t, pset, pdata9.get());
+
+ // now store right hand side
+ for (partsize_t idx = 0; idx < pset.getLocalNumberOfParticles(); idx++)
+ {
+ // for position right hand side is velocity
+ const partsize_t idx15 = 15*idx;
+ const partsize_t idx9 = 9*idx;
+ const partsize_t idx3 = 3*idx;
+ result[idx15 + IDXC_X] = pdata3[idx3 + IDXC_X];
+ result[idx15 + IDXC_Y] = pdata3[idx3 + IDXC_Y];
+ result[idx15 + IDXC_Z] = pdata3[idx3 + IDXC_Z];
+
+ // ODE of the deformation tensor: (dt F_{ij}) = F_{kj} (du_i/dx_k)
+ // We save the matrix F_{ij} = dX_i/dx_j in row-major
+ // to be compatible with GSL
+ result[idx15 + 3 + IDXC_GSL_XX] = pdata9[idx9+IDXC_DX_X]*current_state[idx15 + 3 + IDXC_GSL_XX]
+ + pdata9[idx9+IDXC_DY_X]*current_state[idx15 + 3 + IDXC_GSL_YX]
+ + pdata9[idx9+IDXC_DZ_X]*current_state[idx15 + 3 + IDXC_GSL_ZX];
+ result[idx15 + 3 + IDXC_GSL_YX] = pdata9[idx9+IDXC_DX_Y]*current_state[idx15 + 3 + IDXC_GSL_XX]
+ + pdata9[idx9+IDXC_DY_Y]*current_state[idx15 + 3 + IDXC_GSL_YX]
+ + pdata9[idx9+IDXC_DZ_Y]*current_state[idx15 + 3 + IDXC_GSL_ZX];
+ result[idx15 + 3 + IDXC_GSL_ZX] = pdata9[idx9+IDXC_DX_Z]*current_state[idx15 + 3 + IDXC_GSL_XX]
+ + pdata9[idx9+IDXC_DY_Z]*current_state[idx15 + 3 + IDXC_GSL_YX]
+ + pdata9[idx9+IDXC_DZ_Z]*current_state[idx15 + 3 + IDXC_GSL_ZX];
+
+ result[idx15 + 3 + IDXC_GSL_XY] = pdata9[idx9+IDXC_DX_X]*current_state[idx15 + 3 + IDXC_GSL_XY]
+ + pdata9[idx9+IDXC_DY_X]*current_state[idx15 + 3 + IDXC_GSL_YY]
+ + pdata9[idx9+IDXC_DZ_X]*current_state[idx15 + 3 + IDXC_GSL_ZY];
+ result[idx15 + 3 + IDXC_GSL_YY] = pdata9[idx9+IDXC_DX_Y]*current_state[idx15 + 3 + IDXC_GSL_XY]
+ + pdata9[idx9+IDXC_DY_Y]*current_state[idx15 + 3 + IDXC_GSL_YY]
+ + pdata9[idx9+IDXC_DZ_Y]*current_state[idx15 + 3 + IDXC_GSL_ZY];
+ result[idx15 + 3 + IDXC_GSL_ZY] = pdata9[idx9+IDXC_DX_Z]*current_state[idx15 + 3 + IDXC_GSL_XY]
+ + pdata9[idx9+IDXC_DY_Z]*current_state[idx15 + 3 + IDXC_GSL_YY]
+ + pdata9[idx9+IDXC_DZ_Z]*current_state[idx15 + 3 + IDXC_GSL_ZY];
+
+ result[idx15 + 3 + IDXC_GSL_XZ] = pdata9[idx9+IDXC_DX_X]*current_state[idx15 + 3 + IDXC_GSL_XZ]
+ + pdata9[idx9+IDXC_DY_X]*current_state[idx15 + 3 + IDXC_GSL_YZ]
+ + pdata9[idx9+IDXC_DZ_X]*current_state[idx15 + 3 + IDXC_GSL_ZZ];
+ result[idx15 + 3 + IDXC_GSL_YZ] = pdata9[idx9+IDXC_DX_Y]*current_state[idx15 + 3 + IDXC_GSL_XZ]
+ + pdata9[idx9+IDXC_DY_Y]*current_state[idx15 + 3 + IDXC_GSL_YZ]
+ + pdata9[idx9+IDXC_DZ_Y]*current_state[idx15 + 3 + IDXC_GSL_ZZ];
+ result[idx15 + 3 + IDXC_GSL_ZZ] = pdata9[idx9+IDXC_DX_Z]*current_state[idx15 + 3 + IDXC_GSL_XZ]
+ + pdata9[idx9+IDXC_DY_Z]*current_state[idx15 + 3 + IDXC_GSL_YZ]
+ + pdata9[idx9+IDXC_DZ_Z]*current_state[idx15 + 3 + IDXC_GSL_ZZ];
+
+ // The last three components contain the logarithmic stretching factors.
+ // They are updated by the orthogonalization procedure
+ result[idx15 + 12 + IDXC_X] = 0;
+ result[idx15 + 12 + IDXC_Y] = 0;
+ result[idx15 + 12 + IDXC_Z] = 0;
+ }
+ // we use `unique_ptr`, so we don't need to explicitly free the memory for pdata3 and pdata9
+ return EXIT_SUCCESS;
+ }
+
+ int imposeModelConstraints(
+ abstract_particle_set &pset) const
+ {
+ particle_rnumber *current_state = pset.getParticleState();
+
+ gsl_vector * tmp_vector = gsl_vector_alloc(3);
+ gsl_matrix_view Q;
+ gsl_matrix * Q_tmp = gsl_matrix_alloc(3, 3);
+ gsl_matrix * R_tmp = gsl_matrix_alloc(3, 3);
+
+ for (partsize_t idx = 0; idx < pset.getLocalNumberOfParticles(); idx++)
+ {
+ const partsize_t idx15 = 15*idx;
+
+ // QR decomposition
+ Q = gsl_matrix_view_array(current_state+idx15 + 3, 3, 3);
+ gsl_linalg_QR_decomp(&Q.matrix, tmp_vector);
+ gsl_linalg_QR_unpack(&Q.matrix, tmp_vector, Q_tmp, R_tmp);
+
+ // save results
+ gsl_matrix_memcpy(&Q.matrix, Q_tmp);
+ current_state[idx15 + 12 + IDXC_X] += log(abs(gsl_matrix_get(R_tmp, 0, 0)));
+ current_state[idx15 + 12 + IDXC_Y] += log(abs(gsl_matrix_get(R_tmp, 1, 1)));
+ current_state[idx15 + 12 + IDXC_Z] += log(abs(gsl_matrix_get(R_tmp, 2, 2)));
+ }
+ return EXIT_SUCCESS;
+ }
+};
+
+#endif//DEFORMATION_TENSOR_RHS_HPP
+
diff --git a/cpp/particles/rhs/tracer_rhs.cpp b/cpp/particles/rhs/tracer_rhs.cpp
new file mode 100644
index 0000000000000000000000000000000000000000..d9cebafd48abef29e60c56ea3381420d8eeadced
--- /dev/null
+++ b/cpp/particles/rhs/tracer_rhs.cpp
@@ -0,0 +1,31 @@
+/******************************************************************************
+* *
+* Copyright 2020 Max Planck Institute for Dynamics and Self-Organization *
+* *
+* This file is part of TurTLE. *
+* *
+* TurTLE is free software: you can redistribute it and/or modify *
+* it under the terms of the GNU General Public License as published *
+* by the Free Software Foundation, either version 3 of the License, *
+* or (at your option) any later version. *
+* *
+* TurTLE is distributed in the hope that it will be useful, *
+* but WITHOUT ANY WARRANTY; without even the implied warranty of *
+* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the *
+* GNU General Public License for more details. *
+* *
+* You should have received a copy of the GNU General Public License *
+* along with TurTLE. If not, see <http://www.gnu.org/licenses/> *
+* *
+* Contact: Cristian.Lalescu@ds.mpg.de *
+* *
+******************************************************************************/
+
+
+
+#include "particles/rhs/tracer_rhs.hpp"
+
+
+template class tracer_rhs<float, FFTW, NONE>;
+template class tracer_rhs<double, FFTW, NONE>;
+
diff --git a/cpp/particles/rhs/tracer_rhs.hpp b/cpp/particles/rhs/tracer_rhs.hpp
new file mode 100644
index 0000000000000000000000000000000000000000..680b46fdbc330d3bd892a1343094dd066c1f83a5
--- /dev/null
+++ b/cpp/particles/rhs/tracer_rhs.hpp
@@ -0,0 +1,76 @@
+/******************************************************************************
+* *
+* Copyright 2020 Max Planck Institute for Dynamics and Self-Organization *
+* *
+* This file is part of TurTLE. *
+* *
+* TurTLE is free software: you can redistribute it and/or modify *
+* it under the terms of the GNU General Public License as published *
+* by the Free Software Foundation, either version 3 of the License, *
+* or (at your option) any later version. *
+* *
+* TurTLE is distributed in the hope that it will be useful, *
+* but WITHOUT ANY WARRANTY; without even the implied warranty of *
+* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the *
+* GNU General Public License for more details. *
+* *
+* You should have received a copy of the GNU General Public License *
+* along with TurTLE. If not, see <http://www.gnu.org/licenses/> *
+* *
+* Contact: Cristian.Lalescu@ds.mpg.de *
+* *
+******************************************************************************/
+
+
+
+#ifndef TRACER_RHS_HPP
+#define TRACER_RHS_HPP
+
+#include "particles/interpolation/field_tinterpolator.hpp"
+#include "particles/abstract_particle_rhs.hpp"
+
+
+
+template <typename rnumber,
+ field_backend be,
+ temporal_interpolation_type tt>
+class tracer_rhs: public abstract_particle_rhs
+{
+ protected:
+ field_tinterpolator<rnumber, be, THREE, tt> *velocity;
+
+ public:
+ tracer_rhs(){}
+ ~tracer_rhs() noexcept(false){}
+
+ const int getStateSize() const
+ {
+ return 3;
+ }
+
+ int setVelocity(field_tinterpolator<rnumber, be, THREE, tt> *in_velocity)
+ {
+ this->velocity = in_velocity;
+ return EXIT_SUCCESS;
+ }
+
+ int operator()(
+ double t,
+ abstract_particle_set &pset,
+ particle_rnumber *result,
+ std::vector<std::unique_ptr<abstract_particle_set::particle_rnumber[]>> &additional_data)
+ {
+ // interpolation adds on top of existing values, so result must be cleared.
+ std::fill_n(result, pset.getLocalNumberOfParticles()*3, 0);
+ return (*(this->velocity))(t, pset, result);
+ }
+
+ int imposeModelConstraints(
+ abstract_particle_set &pset) const
+ {
+ return EXIT_SUCCESS;
+ }
+};
+
+#endif//TRACER_RHS_HPP
+
diff --git a/cpp/particles/rhs/tracer_with_collision_counter_rhs.cpp b/cpp/particles/rhs/tracer_with_collision_counter_rhs.cpp
new file mode 100644
index 0000000000000000000000000000000000000000..3f9efd293aae9da1048c856418a31cf5b2e1f82c
--- /dev/null
+++ b/cpp/particles/rhs/tracer_with_collision_counter_rhs.cpp
@@ -0,0 +1,31 @@
+/******************************************************************************
+* *
+* Copyright 2020 Max Planck Institute for Dynamics and Self-Organization *
+* *
+* This file is part of TurTLE. *
+* *
+* TurTLE is free software: you can redistribute it and/or modify *
+* it under the terms of the GNU General Public License as published *
+* by the Free Software Foundation, either version 3 of the License, *
+* or (at your option) any later version. *
+* *
+* TurTLE is distributed in the hope that it will be useful, *
+* but WITHOUT ANY WARRANTY; without even the implied warranty of *
+* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the *
+* GNU General Public License for more details. *
+* *
+* You should have received a copy of the GNU General Public License *
+* along with TurTLE. If not, see <http://www.gnu.org/licenses/> *
+* *
+* Contact: Cristian.Lalescu@ds.mpg.de *
+* *
+******************************************************************************/
+
+
+
+#include "particles/rhs/tracer_with_collision_counter_rhs.hpp"
+
+
+template class tracer_with_collision_counter_rhs<float, FFTW, NONE>;
+template class tracer_with_collision_counter_rhs<double, FFTW, NONE>;
+
diff --git a/cpp/particles/rhs/tracer_with_collision_counter_rhs.hpp b/cpp/particles/rhs/tracer_with_collision_counter_rhs.hpp
new file mode 100644
index 0000000000000000000000000000000000000000..5aa15213d02702eb96c198fab0e333584efb1598
--- /dev/null
+++ b/cpp/particles/rhs/tracer_with_collision_counter_rhs.hpp
@@ -0,0 +1,92 @@
+/******************************************************************************
+* *
+* Copyright 2020 Max Planck Institute for Dynamics and Self-Organization *
+* *
+* This file is part of TurTLE. *
+* *
+* TurTLE is free software: you can redistribute it and/or modify *
+* it under the terms of the GNU General Public License as published *
+* by the Free Software Foundation, either version 3 of the License, *
+* or (at your option) any later version. *
+* *
+* TurTLE is distributed in the hope that it will be useful, *
+* but WITHOUT ANY WARRANTY; without even the implied warranty of *
+* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the *
+* GNU General Public License for more details. *
+* *
+* You should have received a copy of the GNU General Public License *
+* along with TurTLE. If not, see <http://www.gnu.org/licenses/> *
+* *
+* Contact: Cristian.Lalescu@ds.mpg.de *
+* *
+******************************************************************************/
+
+
+
+#ifndef TRACER_WITH_COLLISION_COUNTER_RHS_HPP
+#define TRACER_WITH_COLLISION_COUNTER_RHS_HPP
+
+#include "particles/interpolation/field_tinterpolator.hpp"
+#include "particles/abstract_particle_rhs.hpp"
+#include "particles/rhs/tracer_rhs.hpp"
+#include "particles/p2p/p2p_ghost_collisions.hpp"
+
+
+template <typename rnumber,
+ field_backend be,
+ temporal_interpolation_type tt>
+class tracer_with_collision_counter_rhs: public tracer_rhs<rnumber, be, tt>
+{
+ protected:
+ p2p_ghost_collisions<abstract_particle_set::particle_rnumber, abstract_particle_set::partsize_t> p2p_gc;
+
+ public:
+ tracer_with_collision_counter_rhs(){}
+ ~tracer_with_collision_counter_rhs() noexcept(false){}
+
+ virtual const int getStateSize() const
+ {
+ return 3;
+ }
+
+ int operator()(
+ double t,
+ abstract_particle_set &pset,
+ abstract_particle_set::particle_rnumber *result,
+ std::vector<std::unique_ptr<abstract_particle_set::particle_rnumber[]>> &additional_data)
+ {
+ // interpolation adds on top of existing values, so result must be cleared.
+ std::fill_n(result, pset.getLocalNumberOfParticles()*3, 0);
+ int interpolation_result = (*(this->velocity))(t, pset, result);
+ additional_data.insert(
+ additional_data.begin(),
+ std::unique_ptr<abstract_particle_set::particle_rnumber[]>(
+ new abstract_particle_set::particle_rnumber[pset.getLocalNumberOfParticles()*pset.getStateSize()]));
+ // copy rhs values to temporary array
+ pset.copy_state_tofrom(
+ additional_data[0].get(),
+ result);
+ this->p2p_gc.reset_collision_pairs();
+ pset.template applyP2PKernel<
+ 3,
+ p2p_ghost_collisions<abstract_particle_set::particle_rnumber,
+ abstract_particle_set::partsize_t>>(
+ this->p2p_gc,
+ additional_data);
+ // copy shuffled rhs values
+ pset.copy_state_tofrom(
+ result,
+ additional_data[0].get());
+ // clear temporary array
+ additional_data.erase(additional_data.begin());
+ return interpolation_result;
+ }
+
+ p2p_ghost_collisions<abstract_particle_set::particle_rnumber, abstract_particle_set::partsize_t> &getCollisionCounter()
+ {
+ return this->p2p_gc;
+ }
+};
+
+#endif//TRACER_WITH_COLLISION_COUNTER_RHS_HPP
+
diff --git a/bfps/cpp/scope_timer.cpp b/cpp/scope_timer.cpp
similarity index 52%
rename from bfps/cpp/scope_timer.cpp
rename to cpp/scope_timer.cpp
index 61ddd89583fe8d53cee328c4267df603e128d417..2e10f9a8356c995bf9244b50fcbf8a9d4accb410 100644
--- a/bfps/cpp/scope_timer.cpp
+++ b/cpp/scope_timer.cpp
@@ -4,5 +4,5 @@
#ifdef USE_TIMINGOUTPUT
-EventManager global_timer_manager("BFPS", std::cout);
+EventManager global_timer_manager("TurTLE", std::cout);
#endif
diff --git a/bfps/cpp/scope_timer.hpp b/cpp/scope_timer.hpp
similarity index 97%
rename from bfps/cpp/scope_timer.hpp
rename to cpp/scope_timer.hpp
index 2c48e2eda06ded74e668825181f0444eef22f647..211cccd49cabd63b3f874150406b285417a8f72d 100644
--- a/bfps/cpp/scope_timer.hpp
+++ b/cpp/scope_timer.hpp
@@ -3,20 +3,20 @@
* Copyright 2015 Max Planck Institute *
* for Dynamics and Self-Organization *
* *
-* This file is part of bfps. *
+* This file is part of TurTLE. *
* *
-* bfps is free software: you can redistribute it and/or modify *
+* TurTLE is free software: you can redistribute it and/or modify *
* it under the terms of the GNU General Public License as published *
* by the Free Software Foundation, either version 3 of the License, *
* or (at your option) any later version. *
* *
-* bfps is distributed in the hope that it will be useful, *
+* TurTLE is distributed in the hope that it will be useful, *
* but WITHOUT ANY WARRANTY; without even the implied warranty of *
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the *
* GNU General Public License for more details. *
* *
* You should have received a copy of the GNU General Public License *
-* along with bfps. If not, see <http://www.gnu.org/licenses/> *
+* along with TurTLE. If not, see <http://www.gnu.org/licenses/> *
* *
* Contact: Cristian.Lalescu@ds.mpg.de *
* *
@@ -42,7 +42,7 @@
#include <fstream>
#include "base.hpp"
-#include "bfps_timer.hpp"
+#include "turtle_timer.hpp"
//< To add it as friend of EventManager
class ScopeEvent;
@@ -94,7 +94,7 @@ protected:
omp_destroy_lock(&m_updateLock);
}
- /** Add a record */
+ /** \brief Add a record */
void addRecord(const double inDuration, const bool isTask) {
#pragma omp atomic update
m_totalTime += inDuration;
@@ -123,7 +123,8 @@ protected:
omp_unset_lock(&m_childrenLock);
}
- //! Must not be called during a paralle execution
+ /** \note Must not be called during a paralle execution
+ */
const std::vector<CoreEvent*>& getChildren() const {
assert(omp_in_parallel() == 0);
return m_children;
@@ -243,7 +244,7 @@ public:
omp_init_lock(&m_recordsLock);
}
- ~EventManager() throw() {
+ ~EventManager() noexcept(false) {
assert(m_currentEventsStackPerThread[0].size() == 1);
assert(m_currentEventsStackPerThread[0].top().size() == 1);
@@ -374,7 +375,7 @@ public:
if(myrank != 0){
const std::string strOutput = myResults.str();
- assert(strOutput.length() <= std::numeric_limits<int>::max());
+ assert(strOutput.length() <= std::numeric_limits<decltype(strOutput.length())>::max());
int sizeOutput = int(strOutput.length());
retMpi = MPI_Send(&sizeOutput, 1, MPI_INT, 0, 99, inComm);
assert(retMpi == MPI_SUCCESS);
@@ -415,7 +416,7 @@ public:
std::vector<SerializedEvent> myEvents;
myEvents.reserve(m_records.size());
- for(const std::pair<std::string, const CoreEvent*>& event : m_records){
+ for(const std::pair<const std::string, CoreEvent*>& event : m_records){
myEvents.emplace_back();
SerializedEvent& current_event = myEvents.back();
@@ -639,7 +640,7 @@ public:
if(myRank != 0){
const std::string strOutput = myResults.str();
- assert(strOutput.length() <= std::numeric_limits<int>::max());
+ assert(strOutput.length() <= std::numeric_limits<decltype(strOutput.length())>::max());
int sizeOutput = int(strOutput.length());
retMpi = MPI_Send(&sizeOutput, 1, MPI_INT, 0, 99, inComm);
assert(retMpi == MPI_SUCCESS);
@@ -737,7 +738,7 @@ protected:
//< The core event
EventManager::CoreEvent* m_event;
//< Time to get elapsed time
- bfps_timer m_timer;
+ turtle_timer m_timer;
//< Is true if it has been created for task
bool m_isTask;
@@ -791,7 +792,8 @@ extern EventManager global_timer_manager;
#define TIMEZONE(NAME) \
ScopeEvent TIMEZONE_Core_Pre_Merge(____TIMEZONE_AUTO_ID, __LINE__)( \
- NAME, global_timer_manager, ScopeEventUniqueKey);
+ NAME, global_timer_manager, ScopeEventUniqueKey); \
+ DEBUG_MSG((NAME + std::string("\n")).c_str());
#define TIMEZONE_MULTI_REF(NAME) \
ScopeEvent TIMEZONE_Core_Pre_Merge(____TIMEZONE_AUTO_ID, __LINE__)( \
NAME, global_timer_manager, ScopeEventMultiRefKey);
diff --git a/bfps/cpp/shared_array.hpp b/cpp/shared_array.hpp
similarity index 62%
rename from bfps/cpp/shared_array.hpp
rename to cpp/shared_array.hpp
index 1951e2f9838ccf37367d859206453d3db91e8e19..21851000beedb8d296461f3e7c804b25fc8e92c4 100644
--- a/bfps/cpp/shared_array.hpp
+++ b/cpp/shared_array.hpp
@@ -1,3 +1,28 @@
+/******************************************************************************
+* *
+* Copyright 2016 Max Planck Institute for Dynamics and Self-Organization *
+* *
+* This file is part of TurTLE. *
+* *
+* TurTLE is free software: you can redistribute it and/or modify *
+* it under the terms of the GNU General Public License as published *
+* by the Free Software Foundation, either version 3 of the License, *
+* or (at your option) any later version. *
+* *
+* TurTLE is distributed in the hope that it will be useful, *
+* but WITHOUT ANY WARRANTY; without even the implied warranty of *
+* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the *
+* GNU General Public License for more details. *
+* *
+* You should have received a copy of the GNU General Public License *
+* along with TurTLE. If not, see <http://www.gnu.org/licenses/> *
+* *
+* Contact: Cristian.Lalescu@ds.mpg.de *
+* *
+******************************************************************************/
+
+
+
#ifndef SHAREDARRAY_HPP
#define SHAREDARRAY_HPP
diff --git a/cpp/spectrum_function.hpp b/cpp/spectrum_function.hpp
new file mode 100644
index 0000000000000000000000000000000000000000..7be4474c0a6151e4eed81248140ad95fc2cdec51
--- /dev/null
+++ b/cpp/spectrum_function.hpp
@@ -0,0 +1,38 @@
+#ifndef SPECTRUM_FUNCTION_CPP
+#define SPECTRUM_FUNCTION_CPP
+
+#include<cmath>
+#include<vector>
+
+#include "kspace.hpp"
+
+template <field_backend be,
+ kspace_dealias_type dt>
+class spectrum_function
+{
+ private:
+ const kspace<be, dt> *kk;
+ const std::vector<double> values;
+
+ public:
+ spectrum_function(
+ const kspace<be, dt> *KK,
+ const std::vector<double> &source_values):
+ kk(KK),
+ values(source_values)
+ {
+ assert(this->values.size() == this->kk->nshells);
+ }
+ ~spectrum_function() noexcept(false){}
+
+ double operator()(double kvalue)
+ {
+ assert(kvalue >= double(0));
+ int index = floor(kvalue / this->kk->dk);
+ assert(index < this->values.size());
+ return this->values[index];
+ }
+};
+
+#endif//SPECTRUM_FUNCTION_CPP
+
diff --git a/bfps/cpp/bfps_timer.hpp b/cpp/turtle_timer.hpp
similarity index 70%
rename from bfps/cpp/bfps_timer.hpp
rename to cpp/turtle_timer.hpp
index b299dc115555bd3952e22ce5984f27eb9bfb1914..60478d58d3f9d6411402b7d675673d1208c47cca 100644
--- a/bfps/cpp/bfps_timer.hpp
+++ b/cpp/turtle_timer.hpp
@@ -3,47 +3,52 @@
* Copyright 2015 Max Planck Institute *
* for Dynamics and Self-Organization *
* *
-* This file is part of bfps. *
+* This file is part of TurTLE. *
* *
-* bfps is free software: you can redistribute it and/or modify *
+* TurTLE is free software: you can redistribute it and/or modify *
* it under the terms of the GNU General Public License as published *
* by the Free Software Foundation, either version 3 of the License, *
* or (at your option) any later version. *
* *
-* bfps is distributed in the hope that it will be useful, *
+* TurTLE is distributed in the hope that it will be useful, *
* but WITHOUT ANY WARRANTY; without even the implied warranty of *
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the *
* GNU General Public License for more details. *
* *
* You should have received a copy of the GNU General Public License *
-* along with bfps. If not, see <http://www.gnu.org/licenses/> *
+* along with TurTLE. If not, see <http://www.gnu.org/licenses/> *
* *
* Contact: Cristian.Lalescu@ds.mpg.de *
* *
**********************************************************************/
-#ifndef BFPS_TIMER_HPP
-#define BFPS_TIMER_HPP
+#ifndef TURTLE_TIMER_HPP
+#define TURTLE_TIMER_HPP
#include <chrono>
/**
- * @file
+ * @file
*
* Each section to measure should be embraced by start/stop.
- * The measured time is given by "getElapsed".
- * The total time measured by a timer is given by "getCumulated".
+ * The measured time is given by `getElapsed`.
+ * The total time measured by a timer is given by `getCumulated`.
* Example :
- * @code bfps_timer tm; // Implicit start
- * @code ...
- * @code tm.stop(); // stop the timer
- * @code tm.getElapsed(); // return the duration in s [A]
- * @code tm.start(); // restart the timer
- * @code ...
- * @code tm.stopAndGetElapsed(); // stop the timer and return the duraction in s
- * [B]
- * @code tm.getCumulated(); // Equal [A] + [B]
+ *
+ * ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~{.cpp}
+ * turtle_timer tm; // Implicit start
+ * ...
+ * tm.stop(); // stop the timer
+ * tm.getElapsed(); // return the duration in s [A]
+ * tm.start(); // restart the timer
+ * ...
+ * tm.stopAndGetElapsed(); // stop the timer and return the duraction in s [B]
+ * tm.getCumulated(); // Equal [A] + [B]
+ * ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+ */
+
+/** \class turtle_timer
*/
-class bfps_timer {
+class turtle_timer {
using double_second_time = std::chrono::duration<double, std::ratio<1, 1>>;
std::chrono::high_resolution_clock::time_point
@@ -53,16 +58,16 @@ class bfps_timer {
public:
/// Constructor
- bfps_timer() : m_cumulate(std::chrono::nanoseconds::zero()) { start(); }
+ turtle_timer() : m_cumulate(std::chrono::nanoseconds::zero()) { start(); }
/// Copy constructor
- bfps_timer(const bfps_timer& other) = delete;
+ turtle_timer(const turtle_timer& other) = delete;
/// Copies an other timer
- bfps_timer& operator=(const bfps_timer& other) = delete;
+ turtle_timer& operator=(const turtle_timer& other) = delete;
/// Move constructor
- bfps_timer(bfps_timer&& other) = delete;
+ turtle_timer(turtle_timer&& other) = delete;
/// Copies an other timer
- bfps_timer& operator=(bfps_timer&& other) = delete;
+ turtle_timer& operator=(turtle_timer&& other) = delete;
/** Rest all the values, and apply start */
void reset() {
@@ -101,4 +106,5 @@ public:
}
};
-#endif
+#endif//TURTLE_TIMER_HPP
+
diff --git a/bfps/cpp/vorticity_equation.cpp b/cpp/vorticity_equation.cpp
similarity index 58%
rename from bfps/cpp/vorticity_equation.cpp
rename to cpp/vorticity_equation.cpp
index 737db2c47e89624065f3d29a1657575bac5ea786..e6c500b90277f2d586bf92ef5aa3c645f2949c3b 100644
--- a/bfps/cpp/vorticity_equation.cpp
+++ b/cpp/vorticity_equation.cpp
@@ -3,20 +3,20 @@
* Copyright 2015 Max Planck Institute *
* for Dynamics and Self-Organization *
* *
-* This file is part of bfps. *
+* This file is part of TurTLE. *
* *
-* bfps is free software: you can redistribute it and/or modify *
+* TurTLE is free software: you can redistribute it and/or modify *
* it under the terms of the GNU General Public License as published *
* by the Free Software Foundation, either version 3 of the License, *
* or (at your option) any later version. *
* *
-* bfps is distributed in the hope that it will be useful, *
+* TurTLE is distributed in the hope that it will be useful, *
* but WITHOUT ANY WARRANTY; without even the implied warranty of *
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the *
* GNU General Public License for more details. *
* *
* You should have received a copy of the GNU General Public License *
-* along with bfps. If not, see <http://www.gnu.org/licenses/> *
+* along with TurTLE. If not, see <http://www.gnu.org/licenses/> *
* *
* Contact: Cristian.Lalescu@ds.mpg.de *
* *
@@ -24,14 +24,15 @@
-#define NDEBUG
-
+#include <limits>
#include <cassert>
#include <cmath>
#include <cstring>
+#include <stdexcept>
#include "fftw_tools.hpp"
#include "vorticity_equation.hpp"
#include "scope_timer.hpp"
+#include "shared_array.hpp"
@@ -142,6 +143,10 @@ vorticity_equation<rnumber, be>::vorticity_equation(
nx, ny, nz, MPI_COMM_WORLD, FFTW_PLAN_RIGOR);
this->u = this->cvelocity;
+ /* print fftw plan information for relevant fields */
+ this->rvorticity->print_plan("rvorticity");
+ this->cvelocity->print_plan("cvelocity");
+
/* initialize kspace */
this->kk = new kspace<be, SMOOTH>(
this->cvorticity->clayout, DKX, DKY, DKZ);
@@ -151,6 +156,7 @@ vorticity_equation<rnumber, be>::vorticity_equation(
this->nu = 0.1;
this->fmode = 1;
this->famplitude = 1.0;
+ this->friction_coefficient = 1.0;
this->fk0 = 2.0;
this->fk1 = 4.0;
}
@@ -175,11 +181,11 @@ void vorticity_equation<rnumber, be>::compute_vorticity()
TIMEZONE("vorticity_equation::compute_vorticity");
this->cvorticity->real_space_representation = false;
this->kk->CLOOP_K2(
- [&](ptrdiff_t cindex,
- ptrdiff_t xindex,
- ptrdiff_t yindex,
- ptrdiff_t zindex,
- double k2){
+ [&](const ptrdiff_t cindex,
+ const ptrdiff_t xindex,
+ const ptrdiff_t yindex,
+ const ptrdiff_t zindex,
+ const double k2){
if (k2 <= this->kk->kM2)
{
this->cvorticity->cval(cindex,0,0) = -(this->kk->ky[yindex]*this->u->cval(cindex,2,1) - this->kk->kz[zindex]*this->u->cval(cindex,1,1));
@@ -188,13 +194,6 @@ void vorticity_equation<rnumber, be>::compute_vorticity()
this->cvorticity->cval(cindex,1,1) = (this->kk->kz[zindex]*this->u->cval(cindex,0,0) - this->kk->kx[xindex]*this->u->cval(cindex,2,0));
this->cvorticity->cval(cindex,2,0) = -(this->kk->kx[xindex]*this->u->cval(cindex,1,1) - this->kk->ky[yindex]*this->u->cval(cindex,0,1));
this->cvorticity->cval(cindex,2,1) = (this->kk->kx[xindex]*this->u->cval(cindex,1,0) - this->kk->ky[yindex]*this->u->cval(cindex,0,0));
- //ptrdiff_t tindex = 3*cindex;
- //this->cvorticity->get_cdata()[tindex+0][0] = -(this->kk->ky[yindex]*this->u->get_cdata()[tindex+2][1] - this->kk->kz[zindex]*this->u->get_cdata()[tindex+1][1]);
- //this->cvorticity->get_cdata()[tindex+1][0] = -(this->kk->kz[zindex]*this->u->get_cdata()[tindex+0][1] - this->kk->kx[xindex]*this->u->get_cdata()[tindex+2][1]);
- //this->cvorticity->get_cdata()[tindex+2][0] = -(this->kk->kx[xindex]*this->u->get_cdata()[tindex+1][1] - this->kk->ky[yindex]*this->u->get_cdata()[tindex+0][1]);
- //this->cvorticity->get_cdata()[tindex+0][1] = (this->kk->ky[yindex]*this->u->get_cdata()[tindex+2][0] - this->kk->kz[zindex]*this->u->get_cdata()[tindex+1][0]);
- //this->cvorticity->get_cdata()[tindex+1][1] = (this->kk->kz[zindex]*this->u->get_cdata()[tindex+0][0] - this->kk->kx[xindex]*this->u->get_cdata()[tindex+2][0]);
- //this->cvorticity->get_cdata()[tindex+2][1] = (this->kk->kx[xindex]*this->u->get_cdata()[tindex+1][0] - this->kk->ky[yindex]*this->u->get_cdata()[tindex+0][0]);
}
else
std::fill_n((rnumber*)(this->cvorticity->get_cdata()+3*cindex), 6, 0.0);
@@ -210,77 +209,260 @@ void vorticity_equation<rnumber, be>::compute_velocity(field<rnumber, be, THREE>
TIMEZONE("vorticity_equation::compute_velocity");
this->u->real_space_representation = false;
this->kk->CLOOP_K2(
- [&](ptrdiff_t cindex,
- ptrdiff_t xindex,
- ptrdiff_t yindex,
- ptrdiff_t zindex,
- double k2){
- if (k2 <= this->kk->kM2 && k2 > 0)
+ [&](const ptrdiff_t cindex,
+ const ptrdiff_t xindex,
+ const ptrdiff_t yindex,
+ const ptrdiff_t zindex,
+ const double k2){
+ const double kk2 = (k2 > 0) ? k2 : 1.;
+ if (k2 <= this->kk->kM2)
{
- this->u->cval(cindex,0,0) = -(this->kk->ky[yindex]*vorticity->cval(cindex,2,1) - this->kk->kz[zindex]*vorticity->cval(cindex,1,1)) / k2;
- this->u->cval(cindex,0,1) = (this->kk->ky[yindex]*vorticity->cval(cindex,2,0) - this->kk->kz[zindex]*vorticity->cval(cindex,1,0)) / k2;
- this->u->cval(cindex,1,0) = -(this->kk->kz[zindex]*vorticity->cval(cindex,0,1) - this->kk->kx[xindex]*vorticity->cval(cindex,2,1)) / k2;
- this->u->cval(cindex,1,1) = (this->kk->kz[zindex]*vorticity->cval(cindex,0,0) - this->kk->kx[xindex]*vorticity->cval(cindex,2,0)) / k2;
- this->u->cval(cindex,2,0) = -(this->kk->kx[xindex]*vorticity->cval(cindex,1,1) - this->kk->ky[yindex]*vorticity->cval(cindex,0,1)) / k2;
- this->u->cval(cindex,2,1) = (this->kk->kx[xindex]*vorticity->cval(cindex,1,0) - this->kk->ky[yindex]*vorticity->cval(cindex,0,0)) / k2;
- //ptrdiff_t tindex = 3*cindex;
- //this->u->get_cdata()[tindex+0][0] = -(this->kk->ky[yindex]*vorticity->get_cdata()[tindex+2][1] - this->kk->kz[zindex]*vorticity->get_cdata()[tindex+1][1]) / k2;
- //this->u->get_cdata()[tindex+0][1] = (this->kk->ky[yindex]*vorticity->get_cdata()[tindex+2][0] - this->kk->kz[zindex]*vorticity->get_cdata()[tindex+1][0]) / k2;
- //this->u->get_cdata()[tindex+1][0] = -(this->kk->kz[zindex]*vorticity->get_cdata()[tindex+0][1] - this->kk->kx[xindex]*vorticity->get_cdata()[tindex+2][1]) / k2;
- //this->u->get_cdata()[tindex+1][1] = (this->kk->kz[zindex]*vorticity->get_cdata()[tindex+0][0] - this->kk->kx[xindex]*vorticity->get_cdata()[tindex+2][0]) / k2;
- //this->u->get_cdata()[tindex+2][0] = -(this->kk->kx[xindex]*vorticity->get_cdata()[tindex+1][1] - this->kk->ky[yindex]*vorticity->get_cdata()[tindex+0][1]) / k2;
- //this->u->get_cdata()[tindex+2][1] = (this->kk->kx[xindex]*vorticity->get_cdata()[tindex+1][0] - this->kk->ky[yindex]*vorticity->get_cdata()[tindex+0][0]) / k2;
+ this->u->cval(cindex,0,0) = -(this->kk->ky[yindex]*vorticity->cval(cindex,2,1) - this->kk->kz[zindex]*vorticity->cval(cindex,1,1)) / kk2;
+ this->u->cval(cindex,0,1) = (this->kk->ky[yindex]*vorticity->cval(cindex,2,0) - this->kk->kz[zindex]*vorticity->cval(cindex,1,0)) / kk2;
+ this->u->cval(cindex,1,0) = -(this->kk->kz[zindex]*vorticity->cval(cindex,0,1) - this->kk->kx[xindex]*vorticity->cval(cindex,2,1)) / kk2;
+ this->u->cval(cindex,1,1) = (this->kk->kz[zindex]*vorticity->cval(cindex,0,0) - this->kk->kx[xindex]*vorticity->cval(cindex,2,0)) / kk2;
+ this->u->cval(cindex,2,0) = -(this->kk->kx[xindex]*vorticity->cval(cindex,1,1) - this->kk->ky[yindex]*vorticity->cval(cindex,0,1)) / kk2;
+ this->u->cval(cindex,2,1) = (this->kk->kx[xindex]*vorticity->cval(cindex,1,0) - this->kk->ky[yindex]*vorticity->cval(cindex,0,0)) / kk2;
}
else
std::fill_n((rnumber*)(this->u->get_cdata()+3*cindex), 6, 0.0);
}
);
+ if (this->kk->layout->myrank == 0)
+ std::fill_n((rnumber*)this->u->get_cdata(), 6, 0.0);
this->u->symmetrize();
}
+template <class rnumber,
+ field_backend be>
+void vorticity_equation<rnumber, be>::add_Kolmogorov_forcing(
+ field<rnumber, be, THREE> *dst,
+ const int fmode,
+ const double famplitude)
+{
+ TIMEZONE("vorticity_equation::add_Kolmogorov_forcing");
+ ptrdiff_t cindex;
+ if (dst->clayout->myrank == dst->clayout->rank[0][fmode])
+ {
+ cindex = dst->get_cindex(0, (fmode - dst->clayout->starts[0]), 0);
+ dst->cval(cindex,2, 0) -= famplitude/2;
+ }
+ if (dst->clayout->myrank == dst->clayout->rank[0][dst->clayout->sizes[0] - fmode])
+ {
+ cindex = dst->get_cindex(0, (dst->clayout->sizes[0] - fmode - dst->clayout->starts[0]), 0);
+ dst->cval(cindex, 2, 0) -= famplitude/2;
+ }
+}
+
+template <class rnumber,
+ field_backend be>
+void vorticity_equation<rnumber, be>::add_field_band(
+ field<rnumber, be, THREE> *dst,
+ field<rnumber, be, THREE> *src,
+ const double k0, const double k1,
+ const double prefactor)
+{
+ TIMEZONE("vorticity_equation::add_field_band");
+ this->kk->CLOOP(
+ [&](const ptrdiff_t cindex,
+ const ptrdiff_t xindex,
+ const ptrdiff_t yindex,
+ const ptrdiff_t zindex){
+ const double knorm = sqrt(this->kk->kx[xindex]*this->kk->kx[xindex] +
+ this->kk->ky[yindex]*this->kk->ky[yindex] +
+ this->kk->kz[zindex]*this->kk->kz[zindex]);
+ if ((k0 <= knorm) &&
+ (k1 >= knorm))
+ for (int c=0; c<3; c++)
+ for (int i=0; i<2; i++)
+ dst->cval(cindex,c,i) += prefactor*src->cval(cindex,c,i);
+ }
+ );
+}
+
template <class rnumber,
field_backend be>
void vorticity_equation<rnumber, be>::add_forcing(
field<rnumber, be, THREE> *dst,
- field<rnumber, be, THREE> *vort_field,
- rnumber factor)
+ field<rnumber, be, THREE> *vort_field)
{
TIMEZONE("vorticity_equation::add_forcing");
- if (strcmp(this->forcing_type, "none") == 0)
- return;
if (strcmp(this->forcing_type, "Kolmogorov") == 0)
{
- ptrdiff_t cindex;
- if (this->cvorticity->clayout->myrank == this->cvorticity->clayout->rank[0][this->fmode])
- {
- cindex = ((this->fmode - this->cvorticity->clayout->starts[0]) * this->cvorticity->clayout->sizes[1])*this->cvorticity->clayout->sizes[2];
- dst->cval(cindex,2, 0) -= this->famplitude*factor/2;
- //dst->get_cdata()[cindex*3+2][0] -= this->famplitude*factor/2;
- }
- if (this->cvorticity->clayout->myrank == this->cvorticity->clayout->rank[0][this->cvorticity->clayout->sizes[0] - this->fmode])
- {
- cindex = ((this->cvorticity->clayout->sizes[0] - this->fmode - this->cvorticity->clayout->starts[0]) * this->cvorticity->clayout->sizes[1])*this->cvorticity->clayout->sizes[2];
- dst->cval(cindex, 2, 0) -= this->famplitude*factor/2;
- //dst->get_cdata()[cindex*3+2][0] -= this->famplitude*factor/2;
- }
+ this->add_Kolmogorov_forcing(dst, this->fmode, this->famplitude);
+ return;
+ }
+ if (strcmp(this->forcing_type, "2Kolmogorov") == 0)
+ {
+ // 2 Kolmogorov forces
+ // first one wavenumber fk0, amplitude 1 - A
+ double amplitude = 1 - this->famplitude;
+ int fmode = int(this->fk0 / this->kk->dky);
+ this->add_Kolmogorov_forcing(dst, fmode, amplitude);
+ // second one wavenumber fk1, amplitude A
+ amplitude = this->famplitude * pow(int(this->fk1) / double(int(this->fk0)), 3);
+ fmode = int(this->fk1 / this->kk->dky);
+ this->add_Kolmogorov_forcing(dst, fmode, amplitude);
+ return;
+ }
+ if (strcmp(this->forcing_type, "Kolmogorov_and_drag") == 0)
+ {
+ this->add_Kolmogorov_forcing(dst, this->fmode, this->famplitude);
+ this->add_field_band(
+ dst, vort_field,
+ this->fk0, this->fk1,
+ -this->friction_coefficient);
+ return;
+ }
+ if (strcmp(this->forcing_type, "Kolmogorov_and_compensated_drag") == 0)
+ {
+ double amplitude = this->famplitude * (
+ 1 + this->friction_coefficient / sqrt(this->fmode * this->famplitude));
+ this->add_Kolmogorov_forcing(dst, this->fmode, amplitude);
+ this->add_field_band(
+ dst, vort_field,
+ this->fk0, this->fk1,
+ -this->friction_coefficient);
return;
}
if (strcmp(this->forcing_type, "linear") == 0)
{
- this->kk->CLOOP(
- [&](ptrdiff_t cindex,
- ptrdiff_t xindex,
- ptrdiff_t yindex,
- ptrdiff_t zindex){
- double knorm = sqrt(this->kk->kx[xindex]*this->kk->kx[xindex] +
- this->kk->ky[yindex]*this->kk->ky[yindex] +
- this->kk->kz[zindex]*this->kk->kz[zindex]);
+ this->add_field_band(
+ dst, vort_field,
+ this->fk0, this->fk1,
+ this->famplitude);
+ return;
+ }
+ if ((strcmp(this->forcing_type, "fixed_energy_injection_rate") == 0) ||
+ (strcmp(this->forcing_type, "fixed_energy_injection_rate_and_drag") == 0))
+ {
+ // first, compute energy in shell
+ shared_array<double> local_energy_in_shell(1);
+ double energy_in_shell = 0;
+ this->kk->CLOOP_K2_NXMODES(
+ [&](const ptrdiff_t cindex,
+ const ptrdiff_t xindex,
+ const ptrdiff_t yindex,
+ const ptrdiff_t zindex,
+ const double k2,
+ const int nxmodes){
+ const double knorm = sqrt(k2);
+ if ((k2 > 0) &&
+ (this->fk0 <= knorm) &&
+ (this->fk1 >= knorm))
+ *local_energy_in_shell.getMine() += nxmodes*(
+ vort_field->cval(cindex, 0, 0)*vort_field->cval(cindex, 0, 0) + vort_field->cval(cindex, 0, 1)*vort_field->cval(cindex, 0, 1) +
+ vort_field->cval(cindex, 1, 0)*vort_field->cval(cindex, 1, 0) + vort_field->cval(cindex, 1, 1)*vort_field->cval(cindex, 1, 1) +
+ vort_field->cval(cindex, 2, 0)*vort_field->cval(cindex, 2, 0) + vort_field->cval(cindex, 2, 1)*vort_field->cval(cindex, 2, 1)
+ ) / k2;
+ }
+ );
+ local_energy_in_shell.mergeParallel();
+ MPI_Allreduce(
+ local_energy_in_shell.getMasterData(),
+ &energy_in_shell,
+ 1,
+ MPI_DOUBLE,
+ MPI_SUM,
+ vort_field->comm);
+ // we should divide by 2, if we wanted energy;
+ // but then we would need to multiply the amplitude by 2 anyway,
+ // because what we really care about is force dotted into velocity,
+ // without the division by 2.
+
+ // now, modify amplitudes
+ if (energy_in_shell < 10*std::numeric_limits<rnumber>::epsilon())
+ energy_in_shell = 1;
+ double temp_famplitude = this->injection_rate / energy_in_shell;
+ this->add_field_band(
+ dst, vort_field,
+ this->fk0, this->fk1,
+ temp_famplitude);
+ // and add drag if desired
+ if (strcmp(this->forcing_type, "fixed_energy_injection_rate_and_drag") == 0)
+ this->add_field_band(
+ dst, vort_field,
+ this->fmode, this->fmode + (this->fk1 - this->fk0),
+ -this->friction_coefficient);
+ return;
+ }
+ if (strcmp(this->forcing_type, "fixed_energy") == 0)
+ return;
+ else
+ {
+ DEBUG_MSG("unknown forcing type printed on next line\n%s", this->forcing_type);
+ throw std::invalid_argument("unknown forcing type");
+ }
+}
+
+template <class rnumber,
+ field_backend be>
+void vorticity_equation<rnumber, be>::impose_forcing(
+ field<rnumber, be, THREE> *onew,
+ field<rnumber, be, THREE> *oold)
+{
+ TIMEZONE("vorticity_equation::impose_forcing");
+ if (strcmp(this->forcing_type, "fixed_energy") == 0)
+ {
+ // first, compute energy in shell
+ shared_array<double> local_energy_in_shell(1);
+ shared_array<double> local_total_energy(1);
+ double energy_in_shell, total_energy;
+ this->kk->CLOOP_K2_NXMODES(
+ [&](const ptrdiff_t cindex,
+ const ptrdiff_t xindex,
+ const ptrdiff_t yindex,
+ const ptrdiff_t zindex,
+ const double k2,
+ const int nxmodes){
+ if (k2 > 0)
+ {
+ const double mode_energy = nxmodes*(
+ onew->cval(cindex, 0, 0)*onew->cval(cindex, 0, 0) + onew->cval(cindex, 0, 1)*onew->cval(cindex, 0, 1) +
+ onew->cval(cindex, 1, 0)*onew->cval(cindex, 1, 0) + onew->cval(cindex, 1, 1)*onew->cval(cindex, 1, 1) +
+ onew->cval(cindex, 2, 0)*onew->cval(cindex, 2, 0) + onew->cval(cindex, 2, 1)*onew->cval(cindex, 2, 1)
+ ) / k2;
+ *local_total_energy.getMine() += mode_energy;
+ double knorm = sqrt(k2);
+ if ((this->fk0 <= knorm) && (this->fk1 >= knorm))
+ *local_energy_in_shell.getMine() += mode_energy;
+ }
+ }
+ );
+ local_total_energy.mergeParallel();
+ local_energy_in_shell.mergeParallel();
+ MPI_Allreduce(
+ local_energy_in_shell.getMasterData(),
+ &energy_in_shell,
+ 1,
+ MPI_DOUBLE,
+ MPI_SUM,
+ onew->comm);
+ MPI_Allreduce(
+ local_total_energy.getMasterData(),
+ &total_energy,
+ 1,
+ MPI_DOUBLE,
+ MPI_SUM,
+ onew->comm);
+ // divide by 2, because we want energy
+ total_energy /= 2;
+ energy_in_shell /= 2;
+ // now, add forcing term
+ // see Michael's thesis, page 38
+ double temp_famplitude = sqrt((this->energy - total_energy + energy_in_shell) / energy_in_shell);
+ this->kk->CLOOP_K2(
+ [&](const ptrdiff_t cindex,
+ const ptrdiff_t xindex,
+ const ptrdiff_t yindex,
+ const ptrdiff_t zindex,
+ const double k2){
+ const double knorm = sqrt(k2);
if ((this->fk0 <= knorm) &&
- (this->fk1 >= knorm))
+ (this->fk1 >= knorm))
for (int c=0; c<3; c++)
for (int i=0; i<2; i++)
- dst->cval(cindex,c,i) += this->famplitude*vort_field->cval(cindex,c,i)*factor;
- //dst->get_cdata()[cindex*3+c][i] += this->famplitude*vort_field->get_cdata()[cindex*3+c][i]*factor;
+ onew->cval(cindex,c,i) *= temp_famplitude;
}
);
return;
@@ -292,7 +474,7 @@ template <class rnumber,
void vorticity_equation<rnumber, be>::omega_nonlin(
int src)
{
- DEBUG_MSG("vorticity_equation::omega_nonlin(%d)\n", src);
+ //DEBUG_MSG("vorticity_equation::omega_nonlin(%d)\n", src);
assert(src >= 0 && src < 3);
this->compute_velocity(this->v[src]);
/* get fields from Fourier space to real space */
@@ -302,20 +484,16 @@ void vorticity_equation<rnumber, be>::omega_nonlin(
this->rvorticity->ift();
/* compute cross product $u \times \omega$, and normalize */
this->u->RLOOP(
- [&](ptrdiff_t rindex,
- ptrdiff_t xindex,
- ptrdiff_t yindex,
- ptrdiff_t zindex){
- //ptrdiff_t tindex = 3*rindex;
+ [&](const ptrdiff_t rindex,
+ const ptrdiff_t xindex,
+ const ptrdiff_t yindex,
+ const ptrdiff_t zindex){
rnumber tmp[3];
for (int cc=0; cc<3; cc++)
tmp[cc] = (this->u->rval(rindex,(cc+1)%3)*this->rvorticity->rval(rindex,(cc+2)%3) -
this->u->rval(rindex,(cc+2)%3)*this->rvorticity->rval(rindex,(cc+1)%3));
- //tmp[cc][0] = (this->u->get_rdata()[tindex+(cc+1)%3]*this->rvorticity->get_rdata()[tindex+(cc+2)%3] -
- // this->u->get_rdata()[tindex+(cc+2)%3]*this->rvorticity->get_rdata()[tindex+(cc+1)%3]);
for (int cc=0; cc<3; cc++)
this->u->rval(rindex,cc) = tmp[cc] / this->u->npoints;
- //this->u->get_rdata()[(3*rindex)+cc] = tmp[cc][0] / this->u->npoints;
}
);
/* go back to Fourier space */
@@ -324,114 +502,106 @@ void vorticity_equation<rnumber, be>::omega_nonlin(
this->kk->template dealias<rnumber, THREE>(this->u->get_cdata());
/* $\imath k \times Fourier(u \times \omega)$ */
this->kk->CLOOP(
- [&](ptrdiff_t cindex,
- ptrdiff_t xindex,
- ptrdiff_t yindex,
- ptrdiff_t zindex){
- rnumber tmp[3][2];
- {
- tmp[0][0] = -(this->kk->ky[yindex]*this->u->cval(cindex,2,1) - this->kk->kz[zindex]*this->u->cval(cindex,1,1));
- tmp[1][0] = -(this->kk->kz[zindex]*this->u->cval(cindex,0,1) - this->kk->kx[xindex]*this->u->cval(cindex,2,1));
- tmp[2][0] = -(this->kk->kx[xindex]*this->u->cval(cindex,1,1) - this->kk->ky[yindex]*this->u->cval(cindex,0,1));
- tmp[0][1] = (this->kk->ky[yindex]*this->u->cval(cindex,2,0) - this->kk->kz[zindex]*this->u->cval(cindex,1,0));
- tmp[1][1] = (this->kk->kz[zindex]*this->u->cval(cindex,0,0) - this->kk->kx[xindex]*this->u->cval(cindex,2,0));
- tmp[2][1] = (this->kk->kx[xindex]*this->u->cval(cindex,1,0) - this->kk->ky[yindex]*this->u->cval(cindex,0,0));
- }
- //ptrdiff_t tindex = 3*cindex;
+ [&](const ptrdiff_t cindex,
+ const ptrdiff_t xindex,
+ const ptrdiff_t yindex,
+ const ptrdiff_t zindex){
+ // this tmp needs to be defined as const
+ const rnumber tmp[3][2] = {
+ {(rnumber)(-(this->kk->ky[yindex]*this->u->cval(cindex,2,1) - this->kk->kz[zindex]*this->u->cval(cindex,1,1))),
+ (rnumber)( (this->kk->ky[yindex]*this->u->cval(cindex,2,0) - this->kk->kz[zindex]*this->u->cval(cindex,1,0)))},
+ {(rnumber)(-(this->kk->kz[zindex]*this->u->cval(cindex,0,1) - this->kk->kx[xindex]*this->u->cval(cindex,2,1))),
+ (rnumber)( (this->kk->kz[zindex]*this->u->cval(cindex,0,0) - this->kk->kx[xindex]*this->u->cval(cindex,2,0)))},
+ {(rnumber)(-(this->kk->kx[xindex]*this->u->cval(cindex,1,1) - this->kk->ky[yindex]*this->u->cval(cindex,0,1))),
+ (rnumber)( (this->kk->kx[xindex]*this->u->cval(cindex,1,0) - this->kk->ky[yindex]*this->u->cval(cindex,0,0)))}
+ };
+ //rnumber tmp[3][2];
//{
- // tmp[0][0] = -(this->kk->ky[yindex]*this->u->get_cdata()[tindex+2][1] - this->kk->kz[zindex]*this->u->get_cdata()[tindex+1][1]);
- // tmp[1][0] = -(this->kk->kz[zindex]*this->u->get_cdata()[tindex+0][1] - this->kk->kx[xindex]*this->u->get_cdata()[tindex+2][1]);
- // tmp[2][0] = -(this->kk->kx[xindex]*this->u->get_cdata()[tindex+1][1] - this->kk->ky[yindex]*this->u->get_cdata()[tindex+0][1]);
- // tmp[0][1] = (this->kk->ky[yindex]*this->u->get_cdata()[tindex+2][0] - this->kk->kz[zindex]*this->u->get_cdata()[tindex+1][0]);
- // tmp[1][1] = (this->kk->kz[zindex]*this->u->get_cdata()[tindex+0][0] - this->kk->kx[xindex]*this->u->get_cdata()[tindex+2][0]);
- // tmp[2][1] = (this->kk->kx[xindex]*this->u->get_cdata()[tindex+1][0] - this->kk->ky[yindex]*this->u->get_cdata()[tindex+0][0]);
+ // tmp[0][0] = -(this->kk->ky[yindex]*this->u->cval(cindex,2,1) - this->kk->kz[zindex]*this->u->cval(cindex,1,1));
+ // tmp[1][0] = -(this->kk->kz[zindex]*this->u->cval(cindex,0,1) - this->kk->kx[xindex]*this->u->cval(cindex,2,1));
+ // tmp[2][0] = -(this->kk->kx[xindex]*this->u->cval(cindex,1,1) - this->kk->ky[yindex]*this->u->cval(cindex,0,1));
+ // tmp[0][1] = (this->kk->ky[yindex]*this->u->cval(cindex,2,0) - this->kk->kz[zindex]*this->u->cval(cindex,1,0));
+ // tmp[1][1] = (this->kk->kz[zindex]*this->u->cval(cindex,0,0) - this->kk->kx[xindex]*this->u->cval(cindex,2,0));
+ // tmp[2][1] = (this->kk->kx[xindex]*this->u->cval(cindex,1,0) - this->kk->ky[yindex]*this->u->cval(cindex,0,0));
//}
+ //#pragma omp simd
for (int cc=0; cc<3; cc++) for (int i=0; i<2; i++)
this->u->cval(cindex, cc, i) = tmp[cc][i];
- //this->u->get_cdata()[3*cindex+cc][i] = tmp[cc][i];
}
);
- this->add_forcing(this->u, this->v[src], 1.0);
+ this->add_forcing(this->u, this->v[src]);
this->kk->template force_divfree<rnumber>(this->u->get_cdata());
+ this->u->symmetrize();
}
template <class rnumber,
field_backend be>
void vorticity_equation<rnumber, be>::step(double dt)
{
- DEBUG_MSG("vorticity_equation::step\n");
+ //DEBUG_MSG("vorticity_equation::step\n");
TIMEZONE("vorticity_equation::step");
*this->v[1] = 0.0;
this->omega_nonlin(0);
this->kk->CLOOP_K2(
- [&](ptrdiff_t cindex,
- ptrdiff_t xindex,
- ptrdiff_t yindex,
- ptrdiff_t zindex,
- double k2){
+ [&](const ptrdiff_t cindex,
+ const ptrdiff_t xindex,
+ const ptrdiff_t yindex,
+ const ptrdiff_t zindex,
+ const double k2){
if (k2 <= this->kk->kM2)
{
- double factor0;
- factor0 = exp(-this->nu * k2 * dt);
+ const double factor0 = exp(-this->nu * k2 * dt);
for (int cc=0; cc<3; cc++) for (int i=0; i<2; i++)
this->v[1]->cval(cindex,cc,i) = (
this->v[0]->cval(cindex,cc,i) +
dt*this->u->cval(cindex,cc,i))*factor0;
- //this->v[1]->get_cdata()[3*cindex+cc][i] = (
- // this->v[0]->get_cdata()[3*cindex+cc][i] +
- // dt*this->u->get_cdata()[3*cindex+cc][i])*factor0;
}
}
);
+ this->impose_forcing(this->v[1], this->v[0]);
this->omega_nonlin(1);
this->kk->CLOOP_K2(
- [&](ptrdiff_t cindex,
- ptrdiff_t xindex,
- ptrdiff_t yindex,
- ptrdiff_t zindex,
- double k2){
+ [&](const ptrdiff_t cindex,
+ const ptrdiff_t xindex,
+ const ptrdiff_t yindex,
+ const ptrdiff_t zindex,
+ const double k2){
if (k2 <= this->kk->kM2)
{
- double factor0, factor1;
- factor0 = exp(-this->nu * k2 * dt/2);
- factor1 = exp( this->nu * k2 * dt/2);
+ const double factor0 = exp(-this->nu * k2 * dt/2);
+ const double factor1 = exp( this->nu * k2 * dt/2);
for (int cc=0; cc<3; cc++) for (int i=0; i<2; i++)
this->v[2]->cval(cindex, cc, i) = (
3*this->v[0]->cval(cindex,cc,i)*factor0 +
( this->v[1]->cval(cindex,cc,i) +
dt*this->u->cval(cindex,cc,i))*factor1)*0.25;
- //this->v[2]->get_cdata()[3*cindex+cc][i] = (
- // 3*this->v[0]->get_cdata()[3*cindex+cc][i]*factor0 +
- // (this->v[1]->get_cdata()[3*cindex+cc][i] +
- // dt*this->u->get_cdata()[3*cindex+cc][i])*factor1)*0.25;
}
}
);
+ this->impose_forcing(this->v[2], this->v[0]);
this->omega_nonlin(2);
+ // store old vorticity
+ *this->v[1] = *this->v[0];
this->kk->CLOOP_K2(
- [&](ptrdiff_t cindex,
- ptrdiff_t xindex,
- ptrdiff_t yindex,
- ptrdiff_t zindex,
- double k2){
+ [&](const ptrdiff_t cindex,
+ const ptrdiff_t xindex,
+ const ptrdiff_t yindex,
+ const ptrdiff_t zindex,
+ const double k2){
if (k2 <= this->kk->kM2)
{
- double factor0;
- factor0 = exp(-this->nu * k2 * dt * 0.5);
+ const double factor0 = exp(-this->nu * k2 * dt * 0.5);
for (int cc=0; cc<3; cc++) for (int i=0; i<2; i++)
this->v[3]->cval(cindex,cc,i) = (
this->v[0]->cval(cindex,cc,i)*factor0 +
2*(this->v[2]->cval(cindex,cc,i) +
dt*this->u->cval(cindex,cc,i)))*factor0/3;
- //this->v[3]->get_cdata()[3*cindex+cc][i] = (
- // this->v[0]->get_cdata()[3*cindex+cc][i]*factor0 +
- // 2*(this->v[2]->get_cdata()[3*cindex+cc][i] +
- // dt*this->u->get_cdata()[3*cindex+cc][i]))*factor0/3;
}
}
);
+ this->impose_forcing(this->v[0], this->v[1]);
this->kk->template force_divfree<rnumber>(this->cvorticity->get_cdata());
this->cvorticity->symmetrize();
@@ -445,29 +615,30 @@ void vorticity_equation<rnumber, be>::compute_pressure(field<rnumber, be, ONE> *
TIMEZONE("vorticity_equation::compute_pressure");
pressure->real_space_representation = false;
/* assume velocity is already in real space representation */
-
+ /* set the pressure representation to Fourier space */
+ pressure->real_space_representation = false;
+
this->v[1]->real_space_representation = true;
/* diagonal terms 11 22 33 */
this->v[1]->RLOOP (
- [&](ptrdiff_t rindex,
- ptrdiff_t xindex,
- ptrdiff_t yindex,
- ptrdiff_t zindex){
+ [&](const ptrdiff_t rindex,
+ const ptrdiff_t xindex,
+ const ptrdiff_t yindex,
+ const ptrdiff_t zindex){
//ptrdiff_t tindex = 3*rindex;
for (int cc=0; cc<3; cc++)
this->v[1]->rval(rindex,cc) = this->u->rval(rindex,cc)*this->u->rval(rindex,cc);
- //this->v[1]->get_rdata()[tindex+cc] = this->u->get_rdata()[tindex+cc]*this->u->get_rdata()[tindex+cc];
}
);
//this->clean_up_real_space(this->rv[1], 3);
this->v[1]->dft();
this->kk->template dealias<rnumber, THREE>(this->v[1]->get_cdata());
this->kk->CLOOP_K2(
- [&](ptrdiff_t cindex,
- ptrdiff_t xindex,
- ptrdiff_t yindex,
- ptrdiff_t zindex,
- double k2){
+ [&](const ptrdiff_t cindex,
+ const ptrdiff_t xindex,
+ const ptrdiff_t yindex,
+ const ptrdiff_t zindex,
+ const double k2){
if (k2 <= this->kk->kM2 && k2 > 0)
{
ptrdiff_t tindex = 3*cindex;
@@ -486,25 +657,24 @@ void vorticity_equation<rnumber, be>::compute_pressure(field<rnumber, be, ONE> *
/* off-diagonal terms 12 23 31 */
this->v[1]->real_space_representation = true;
this->v[1]->RLOOP (
- [&](ptrdiff_t rindex,
- ptrdiff_t xindex,
- ptrdiff_t yindex,
- ptrdiff_t zindex){
+ [&](const ptrdiff_t rindex,
+ const ptrdiff_t xindex,
+ const ptrdiff_t yindex,
+ const ptrdiff_t zindex){
//ptrdiff_t tindex = 3*rindex;
for (int cc=0; cc<3; cc++)
this->v[1]->rval(rindex,cc) = this->u->rval(rindex,cc)*this->u->rval(rindex,(cc+1)%3);
- //this->v[1]->get_rdata()[tindex+cc] = this->u->get_rdata()[tindex+cc]*this->u->get_rdata()[tindex+(cc+1)%3];
}
);
//this->clean_up_real_space(this->rv[1], 3);
this->v[1]->dft();
this->kk->template dealias<rnumber, THREE>(this->v[1]->get_cdata());
this->kk->CLOOP_K2(
- [&](ptrdiff_t cindex,
- ptrdiff_t xindex,
- ptrdiff_t yindex,
- ptrdiff_t zindex,
- double k2){
+ [&](const ptrdiff_t cindex,
+ const ptrdiff_t xindex,
+ const ptrdiff_t yindex,
+ const ptrdiff_t zindex,
+ const double k2){
if (k2 <= this->kk->kM2 && k2 > 0)
{
ptrdiff_t tindex = 3*cindex;
@@ -529,14 +699,20 @@ void vorticity_equation<rnumber, be>::compute_pressure(field<rnumber, be, ONE> *
template <class rnumber,
field_backend be>
void vorticity_equation<rnumber, be>::compute_Lagrangian_acceleration(
- field<rnumber, be, THREE> *acceleration)
+ field<rnumber, be, THREE> *acceleration,
+ field<rnumber, be, ONE> *pressure)
{
- field<rnumber, be, ONE> *pressure = new field<rnumber, be, ONE>(
+ bool own_pressure = false;
+ if (pressure == NULL)
+ {
+ pressure = new field<rnumber, be, ONE>(
this->cvelocity->rlayout->sizes[2],
this->cvelocity->rlayout->sizes[1],
this->cvelocity->rlayout->sizes[0],
this->cvelocity->rlayout->comm,
this->cvelocity->fftw_plan_rigor);
+ own_pressure = true;
+ }
this->compute_velocity(this->cvorticity);
this->cvelocity->ift();
this->compute_pressure(pressure);
@@ -544,11 +720,11 @@ void vorticity_equation<rnumber, be>::compute_Lagrangian_acceleration(
acceleration->real_space_representation = false;
*acceleration = 0.0;
this->kk->CLOOP_K2(
- [&](ptrdiff_t cindex,
- ptrdiff_t xindex,
- ptrdiff_t yindex,
- ptrdiff_t zindex,
- double k2){
+ [&](const ptrdiff_t cindex,
+ const ptrdiff_t xindex,
+ const ptrdiff_t yindex,
+ const ptrdiff_t zindex,
+ const double k2){
if (k2 <= this->kk->kM2)
{
ptrdiff_t tindex = 3*cindex;
@@ -574,7 +750,8 @@ void vorticity_equation<rnumber, be>::compute_Lagrangian_acceleration(
acceleration->get_cdata()[tindex+2][1] -= this->kk->kz[zindex]*pressure->get_cdata()[cindex][0];
}
});
- delete pressure;
+ if (own_pressure)
+ delete pressure;
}
template <class rnumber,
@@ -586,11 +763,11 @@ void vorticity_equation<rnumber, be>::compute_Eulerian_acceleration(
acceleration->real_space_representation = false;
/* put in linear terms */
this->kk->CLOOP_K2(
- [&](ptrdiff_t cindex,
- ptrdiff_t xindex,
- ptrdiff_t yindex,
- ptrdiff_t zindex,
- double k2){
+ [&](const ptrdiff_t cindex,
+ const ptrdiff_t xindex,
+ const ptrdiff_t yindex,
+ const ptrdiff_t zindex,
+ const double k2){
if (k2 <= this->kk->kM2)
{
ptrdiff_t tindex = 3*cindex;
@@ -626,17 +803,16 @@ void vorticity_equation<rnumber, be>::compute_Eulerian_acceleration(
for (int cc=0; cc<3; cc++)
this->v[1]->rval(rindex,cc) = \
this->cvelocity->rval(rindex,cc)*this->cvelocity->rval(rindex,cc) / this->cvelocity->npoints;
- //this->v[1]->get_rdata()[tindex+cc] = this->cvelocity->get_rdata()[tindex+cc]*this->cvelocity->get_rdata()[tindex+cc] / this->cvelocity->npoints;
}
);
this->v[1]->dft();
this->kk->template dealias<rnumber, THREE>(this->v[1]->get_cdata());
this->kk->CLOOP_K2(
- [&](ptrdiff_t cindex,
- ptrdiff_t xindex,
- ptrdiff_t yindex,
- ptrdiff_t zindex,
- double k2){
+ [&](const ptrdiff_t cindex,
+ const ptrdiff_t xindex,
+ const ptrdiff_t yindex,
+ const ptrdiff_t zindex,
+ const double k2){
if (k2 <= this->kk->kM2)
{
ptrdiff_t tindex = 3*cindex;
@@ -666,17 +842,16 @@ void vorticity_equation<rnumber, be>::compute_Eulerian_acceleration(
for (int cc=0; cc<3; cc++)
this->v[1]->rval(rindex,cc) = \
this->cvelocity->rval(rindex,cc)*this->cvelocity->rval(rindex,(cc+1)%3) / this->cvelocity->npoints;
- //this->v[1]->get_rdata()[tindex+cc] = this->cvelocity->get_rdata()[tindex+cc]*this->cvelocity->get_rdata()[tindex+(cc+1)%3] / this->cvelocity->npoints;
}
);
this->v[1]->dft();
this->kk->template dealias<rnumber, THREE>(this->v[1]->get_cdata());
this->kk->CLOOP_K2(
- [&](ptrdiff_t cindex,
- ptrdiff_t xindex,
- ptrdiff_t yindex,
- ptrdiff_t zindex,
- double k2){
+ [&](const ptrdiff_t cindex,
+ const ptrdiff_t xindex,
+ const ptrdiff_t yindex,
+ const ptrdiff_t zindex,
+ const double k2){
if (k2 <= this->kk->kM2)
{
ptrdiff_t tindex = 3*cindex;
diff --git a/bfps/cpp/vorticity_equation.hpp b/cpp/vorticity_equation.hpp
similarity index 64%
rename from bfps/cpp/vorticity_equation.hpp
rename to cpp/vorticity_equation.hpp
index e8bd1d843f730d39439bc99703956dc623ca4e42..32c6547b9781d23bf180d2b28a13d5cfdcd4e69a 100644
--- a/bfps/cpp/vorticity_equation.hpp
+++ b/cpp/vorticity_equation.hpp
@@ -3,44 +3,47 @@
* Copyright 2015 Max Planck Institute *
* for Dynamics and Self-Organization *
* *
-* This file is part of bfps. *
+* This file is part of TurTLE. *
* *
-* bfps is free software: you can redistribute it and/or modify *
+* TurTLE is free software: you can redistribute it and/or modify *
* it under the terms of the GNU General Public License as published *
* by the Free Software Foundation, either version 3 of the License, *
* or (at your option) any later version. *
* *
-* bfps is distributed in the hope that it will be useful, *
+* TurTLE is distributed in the hope that it will be useful, *
* but WITHOUT ANY WARRANTY; without even the implied warranty of *
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the *
* GNU General Public License for more details. *
* *
* You should have received a copy of the GNU General Public License *
-* along with bfps. If not, see <http://www.gnu.org/licenses/> *
+* along with TurTLE. If not, see <http://www.gnu.org/licenses/> *
* *
* Contact: Cristian.Lalescu@ds.mpg.de *
* *
**********************************************************************/
+
+
+#ifndef VORTICITY_EQUATION
+#define VORTICITY_EQUATION
+
#include <sys/stat.h>
#include <stdio.h>
#include <stdlib.h>
#include <iostream>
#include "field.hpp"
-#include "field_descriptor.hpp"
-
-#ifndef VORTICITY_EQUATION
-
-#define VORTICITY_EQUATION
extern int myrank, nprocs;
-/* container for field descriptor, fields themselves, parameters, etc
+/** container for field descriptor, fields themselves, parameters, etc
+ *
* This particular class is only meant as a stepping stone to a proper solver
* that only uses the field class (and related layout and kspace classes), and
* HDF5 for I/O.
+ *
+ * Allocates 5 vector fields, and 1 kspace object.
* */
template <typename rnumber,
@@ -67,9 +70,12 @@ class vorticity_equation
/* physical parameters */
double nu;
- int fmode; // for Kolmogorov flow
- double famplitude; // both for Kflow and band forcing
- double fk0, fk1; // for band forcing
+ int fmode; // for Kolmogorov flow
+ double famplitude; // both for Kflow and band forcing
+ double fk0, fk1; // for band forcing
+ double injection_rate; // for fixed energy injection rate
+ double energy; // for fixed energy
+ double friction_coefficient; // for Kolmogorov_and_drag
char forcing_type[128];
/* constructor, destructor */
@@ -82,15 +88,42 @@ class vorticity_equation
double DKY = 1.0,
double DKZ = 1.0,
unsigned FFTW_PLAN_RIGOR = FFTW_MEASURE);
- ~vorticity_equation(void);
+ virtual ~vorticity_equation(void) noexcept(false);
/* solver essential methods */
- void omega_nonlin(int src);
- void step(double dt);
+ virtual void omega_nonlin(int src);
+ virtual void step(double dt);
void impose_zero_modes(void);
+
+ /** \brief Method that computes force and adds it to the right hand side of the NS equations.
+ *
+ * If the force has an explicit expression, as for instance in the case of Kolmogorov forcing,
+ * the term should be added to the nonlinear term for the purposes of time-stepping, since
+ * otherwise a custom time-stepping scheme would need to be implemented for each forcing type.
+ *
+ */
void add_forcing(field<rnumber, be, THREE> *dst,
- field<rnumber, be, THREE> *src_vorticity,
- rnumber factor);
+ field<rnumber, be, THREE> *src_vorticity);
+
+ void add_Kolmogorov_forcing(field<rnumber, be, THREE> *dst,
+ const int fmode,
+ const double famplitude);
+ void add_field_band(
+ field<rnumber, be, THREE> *dst,
+ field<rnumber, be, THREE> *src,
+ const double k0, const double k1,
+ const double prefactor);
+
+ /** \brief Method that imposes action of forcing on new vorticity field.
+ *
+ * If the force is implicit, in the sense that kinetic energy must be
+ * preserved or something similar, then the action must be imposed
+ * after the non-linear term has been added.
+ *
+ */
+ void impose_forcing(
+ field<rnumber, be, THREE> *omega_new,
+ field<rnumber, be, THREE> *omega_old);
void compute_vorticity(void);
void compute_velocity(field<rnumber, be, THREE> *vorticity);
@@ -124,14 +157,16 @@ class vorticity_equation
this->kk->template low_pass<rnumber, THREE>(this->cvorticity->get_cdata(), this->kk->kM);
this->kk->template force_divfree<rnumber>(this->cvorticity->get_cdata());
#endif
+ this->cvorticity->symmetrize();
}
}
/* statistics and general postprocessing */
void compute_pressure(field<rnumber, be, ONE> *pressure);
void compute_Eulerian_acceleration(field<rnumber, be, THREE> *acceleration);
- void compute_Lagrangian_acceleration(field<rnumber, be, THREE> *acceleration);
+ void compute_Lagrangian_acceleration(
+ field<rnumber, be, THREE> *acceleration,
+ field<rnumber, be, ONE> *pressure = NULL);
};
#endif//VORTICITY_EQUATION
-
diff --git a/documentation/Makefile b/documentation/Makefile
deleted file mode 100644
index 6da2f7a60d704280c87a27e38572b387de2b6347..0000000000000000000000000000000000000000
--- a/documentation/Makefile
+++ /dev/null
@@ -1,192 +0,0 @@
-# Makefile for Sphinx documentation
-#
-
-# You can set these variables from the command line.
-SPHINXOPTS =
-SPHINXBUILD = sphinx-build
-PAPER =
-BUILDDIR = _build
-
-# User-friendly check for sphinx-build
-ifeq ($(shell which $(SPHINXBUILD) >/dev/null 2>&1; echo $$?), 1)
-$(error The '$(SPHINXBUILD)' command was not found. Make sure you have Sphinx installed, then set the SPHINXBUILD environment variable to point to the full path of the '$(SPHINXBUILD)' executable. Alternatively you can add the directory with the executable to your PATH. If you don't have Sphinx installed, grab it from http://sphinx-doc.org/)
-endif
-
-# Internal variables.
-PAPEROPT_a4 = -D latex_paper_size=a4
-PAPEROPT_letter = -D latex_paper_size=letter
-ALLSPHINXOPTS = -d $(BUILDDIR)/doctrees $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) .
-# the i18n builder cannot share the environment and doctrees with the others
-I18NSPHINXOPTS = $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) .
-
-.PHONY: help clean html dirhtml singlehtml pickle json htmlhelp qthelp devhelp epub latex latexpdf text man changes linkcheck doctest coverage gettext
-
-help:
- @echo "Please use \`make <target>' where <target> is one of"
- @echo " html to make standalone HTML files"
- @echo " dirhtml to make HTML files named index.html in directories"
- @echo " singlehtml to make a single large HTML file"
- @echo " pickle to make pickle files"
- @echo " json to make JSON files"
- @echo " htmlhelp to make HTML files and a HTML help project"
- @echo " qthelp to make HTML files and a qthelp project"
- @echo " applehelp to make an Apple Help Book"
- @echo " devhelp to make HTML files and a Devhelp project"
- @echo " epub to make an epub"
- @echo " latex to make LaTeX files, you can set PAPER=a4 or PAPER=letter"
- @echo " latexpdf to make LaTeX files and run them through pdflatex"
- @echo " latexpdfja to make LaTeX files and run them through platex/dvipdfmx"
- @echo " text to make text files"
- @echo " man to make manual pages"
- @echo " texinfo to make Texinfo files"
- @echo " info to make Texinfo files and run them through makeinfo"
- @echo " gettext to make PO message catalogs"
- @echo " changes to make an overview of all changed/added/deprecated items"
- @echo " xml to make Docutils-native XML files"
- @echo " pseudoxml to make pseudoxml-XML files for display purposes"
- @echo " linkcheck to check all external links for integrity"
- @echo " doctest to run all doctests embedded in the documentation (if enabled)"
- @echo " coverage to run coverage check of the documentation (if enabled)"
-
-clean:
- rm -rf $(BUILDDIR)/*
-
-html:
- $(SPHINXBUILD) -b html $(ALLSPHINXOPTS) $(BUILDDIR)/html
- @echo
- @echo "Build finished. The HTML pages are in $(BUILDDIR)/html."
-
-dirhtml:
- $(SPHINXBUILD) -b dirhtml $(ALLSPHINXOPTS) $(BUILDDIR)/dirhtml
- @echo
- @echo "Build finished. The HTML pages are in $(BUILDDIR)/dirhtml."
-
-singlehtml:
- $(SPHINXBUILD) -b singlehtml $(ALLSPHINXOPTS) $(BUILDDIR)/singlehtml
- @echo
- @echo "Build finished. The HTML page is in $(BUILDDIR)/singlehtml."
-
-pickle:
- $(SPHINXBUILD) -b pickle $(ALLSPHINXOPTS) $(BUILDDIR)/pickle
- @echo
- @echo "Build finished; now you can process the pickle files."
-
-json:
- $(SPHINXBUILD) -b json $(ALLSPHINXOPTS) $(BUILDDIR)/json
- @echo
- @echo "Build finished; now you can process the JSON files."
-
-htmlhelp:
- $(SPHINXBUILD) -b htmlhelp $(ALLSPHINXOPTS) $(BUILDDIR)/htmlhelp
- @echo
- @echo "Build finished; now you can run HTML Help Workshop with the" \
- ".hhp project file in $(BUILDDIR)/htmlhelp."
-
-qthelp:
- $(SPHINXBUILD) -b qthelp $(ALLSPHINXOPTS) $(BUILDDIR)/qthelp
- @echo
- @echo "Build finished; now you can run "qcollectiongenerator" with the" \
- ".qhcp project file in $(BUILDDIR)/qthelp, like this:"
- @echo "# qcollectiongenerator $(BUILDDIR)/qthelp/bfps.qhcp"
- @echo "To view the help file:"
- @echo "# assistant -collectionFile $(BUILDDIR)/qthelp/bfps.qhc"
-
-applehelp:
- $(SPHINXBUILD) -b applehelp $(ALLSPHINXOPTS) $(BUILDDIR)/applehelp
- @echo
- @echo "Build finished. The help book is in $(BUILDDIR)/applehelp."
- @echo "N.B. You won't be able to view it unless you put it in" \
- "~/Library/Documentation/Help or install it in your application" \
- "bundle."
-
-devhelp:
- $(SPHINXBUILD) -b devhelp $(ALLSPHINXOPTS) $(BUILDDIR)/devhelp
- @echo
- @echo "Build finished."
- @echo "To view the help file:"
- @echo "# mkdir -p $$HOME/.local/share/devhelp/bfps"
- @echo "# ln -s $(BUILDDIR)/devhelp $$HOME/.local/share/devhelp/bfps"
- @echo "# devhelp"
-
-epub:
- $(SPHINXBUILD) -b epub $(ALLSPHINXOPTS) $(BUILDDIR)/epub
- @echo
- @echo "Build finished. The epub file is in $(BUILDDIR)/epub."
-
-latex:
- $(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex
- @echo
- @echo "Build finished; the LaTeX files are in $(BUILDDIR)/latex."
- @echo "Run \`make' in that directory to run these through (pdf)latex" \
- "(use \`make latexpdf' here to do that automatically)."
-
-latexpdf:
- $(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex
- @echo "Running LaTeX files through pdflatex..."
- $(MAKE) -C $(BUILDDIR)/latex all-pdf
- @echo "pdflatex finished; the PDF files are in $(BUILDDIR)/latex."
-
-latexpdfja:
- $(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex
- @echo "Running LaTeX files through platex and dvipdfmx..."
- $(MAKE) -C $(BUILDDIR)/latex all-pdf-ja
- @echo "pdflatex finished; the PDF files are in $(BUILDDIR)/latex."
-
-text:
- $(SPHINXBUILD) -b text $(ALLSPHINXOPTS) $(BUILDDIR)/text
- @echo
- @echo "Build finished. The text files are in $(BUILDDIR)/text."
-
-man:
- $(SPHINXBUILD) -b man $(ALLSPHINXOPTS) $(BUILDDIR)/man
- @echo
- @echo "Build finished. The manual pages are in $(BUILDDIR)/man."
-
-texinfo:
- $(SPHINXBUILD) -b texinfo $(ALLSPHINXOPTS) $(BUILDDIR)/texinfo
- @echo
- @echo "Build finished. The Texinfo files are in $(BUILDDIR)/texinfo."
- @echo "Run \`make' in that directory to run these through makeinfo" \
- "(use \`make info' here to do that automatically)."
-
-info:
- $(SPHINXBUILD) -b texinfo $(ALLSPHINXOPTS) $(BUILDDIR)/texinfo
- @echo "Running Texinfo files through makeinfo..."
- make -C $(BUILDDIR)/texinfo info
- @echo "makeinfo finished; the Info files are in $(BUILDDIR)/texinfo."
-
-gettext:
- $(SPHINXBUILD) -b gettext $(I18NSPHINXOPTS) $(BUILDDIR)/locale
- @echo
- @echo "Build finished. The message catalogs are in $(BUILDDIR)/locale."
-
-changes:
- $(SPHINXBUILD) -b changes $(ALLSPHINXOPTS) $(BUILDDIR)/changes
- @echo
- @echo "The overview file is in $(BUILDDIR)/changes."
-
-linkcheck:
- $(SPHINXBUILD) -b linkcheck $(ALLSPHINXOPTS) $(BUILDDIR)/linkcheck
- @echo
- @echo "Link check complete; look for any errors in the above output " \
- "or in $(BUILDDIR)/linkcheck/output.txt."
-
-doctest:
- $(SPHINXBUILD) -b doctest $(ALLSPHINXOPTS) $(BUILDDIR)/doctest
- @echo "Testing of doctests in the sources finished, look at the " \
- "results in $(BUILDDIR)/doctest/output.txt."
-
-coverage:
- $(SPHINXBUILD) -b coverage $(ALLSPHINXOPTS) $(BUILDDIR)/coverage
- @echo "Testing of coverage in the sources finished, look at the " \
- "results in $(BUILDDIR)/coverage/python.txt."
-
-xml:
- $(SPHINXBUILD) -b xml $(ALLSPHINXOPTS) $(BUILDDIR)/xml
- @echo
- @echo "Build finished. The XML files are in $(BUILDDIR)/xml."
-
-pseudoxml:
- $(SPHINXBUILD) -b pseudoxml $(ALLSPHINXOPTS) $(BUILDDIR)/pseudoxml
- @echo
- @echo "Build finished. The pseudo-XML files are in $(BUILDDIR)/pseudoxml."
diff --git a/documentation/TurTLE_logo.svg b/documentation/TurTLE_logo.svg
new file mode 100644
index 0000000000000000000000000000000000000000..420b8b6f9477151c5c281a49534bc252c82519a1
--- /dev/null
+++ b/documentation/TurTLE_logo.svg
@@ -0,0 +1,165 @@
+<?xml version="1.0" encoding="utf-8" standalone="no"?>
+<!DOCTYPE svg PUBLIC "-//W3C//DTD SVG 1.1//EN"
+ "http://www.w3.org/Graphics/SVG/1.1/DTD/svg11.dtd">
+<!-- Created with matplotlib (https://matplotlib.org/) -->
+<svg height="162pt" version="1.1" viewBox="0 0 288 162" width="288pt" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink">
+ <metadata>
+ <rdf:RDF xmlns:cc="http://creativecommons.org/ns#" xmlns:dc="http://purl.org/dc/elements/1.1/" xmlns:rdf="http://www.w3.org/1999/02/22-rdf-syntax-ns#">
+ <cc:Work>
+ <dc:type rdf:resource="http://purl.org/dc/dcmitype/StillImage"/>
+ <dc:date>2022-02-02T13:16:27.242777</dc:date>
+ <dc:format>image/svg+xml</dc:format>
+ <dc:creator>
+ <cc:Agent>
+ <dc:title>Matplotlib v3.3.4, https://matplotlib.org/</dc:title>
+ </cc:Agent>
+ </dc:creator>
+ </cc:Work>
+ </rdf:RDF>
+ </metadata>
+ <defs>
+ <style type="text/css">*{stroke-linecap:butt;stroke-linejoin:round;}</style>
+ </defs>
+ <g id="figure_1">
+ <g id="patch_1">
+ <path d="M 0 162
+L 288 162
+L 288 0
+L 0 0
+z
+" style="fill:#ffffff;"/>
+ </g>
+ <g id="axes_1">
+ <g id="patch_2">
+ <path clip-path="url(#pa9854da833)" d="M 78.101695 63.915254
+C 68.338983 63.915254 29.288136 146.898305 53.694915 146.898305
+C 87.864407 146.898305 87.864407 63.915254 78.101695 63.915254
+z
+" style="fill:#007d7a;stroke:#007d7a;stroke-linejoin:miter;"/>
+ </g>
+ <g id="patch_3">
+ <path clip-path="url(#pa9854da833)" d="M 4.881356 112.728814
+C 29.288136 106.627119 29.288136 10.220339 109.586441 10.220339
+C 119.53258 10.158945 119.53258 10.158945 124.53597 15.040301
+C 129.53936 19.921657 129.53936 19.921657 129.6 29.745763
+C 129.53936 39.569868 129.53936 39.569868 124.53597 44.451224
+C 119.53258 49.33258 119.53258 49.33258 109.586441 49.271186
+C 29.288136 49.271186 53.694915 112.728814 4.881356 112.728814
+z
+" style="fill:#007d7a;stroke:#007d7a;stroke-linejoin:miter;"/>
+ </g>
+ <g id="patch_4">
+ <path clip-path="url(#pa9854da833)" d="M 175.728814 63.915254
+C 185.491525 63.915254 224.542373 146.898305 200.135593 146.898305
+C 165.966102 146.898305 165.966102 63.915254 175.728814 63.915254
+z
+" style="fill:#007d7a;stroke:#007d7a;stroke-linejoin:miter;"/>
+ </g>
+ <g id="patch_5">
+ <path clip-path="url(#pa9854da833)" d="M 152.054237 10.220339
+C 142.291525 10.220339 142.291525 10.220339 137.410169 15.101695
+C 132.528814 19.983051 132.528814 19.983051 132.528814 29.745763
+C 132.528814 39.508475 132.528814 39.508475 137.410169 44.389831
+C 142.291525 49.271186 142.291525 49.271186 152.054237 49.271186
+C 170.847458 49.271186 178.169492 59.033898 189.152542 59.033898
+C 190.372881 59.033898 192.813559 59.033898 197.694915 54.152542
+C 202.576271 49.271186 202.576271 49.271186 202.576271 44.389831
+C 202.576271 39.508475 202.576271 39.508475 197.694915 34.627119
+C 195.254237 32.186441 175.728814 10.220339 147.172881 10.220339
+z
+" style="fill:#007d7a;stroke:#007d7a;stroke-linejoin:miter;"/>
+ </g>
+ <g id="patch_6">
+ <path clip-path="url(#pa9854da833)" d="M 200.135593 56.59322
+C 203.79661 52.932203 203.79661 52.932203 207.457627 52.932203
+C 211.118644 52.932203 211.118644 52.932203 214.779661 56.59322
+C 214.779661 56.59322 231.864407 76.118644 231.864407 100.525424
+L 246.508475 100.525424
+C 247.484746 102.966102 253.830508 111.752542 261.152542 112.728814
+L 219.661017 112.728814
+C 219.661017 100.525424 214.779661 76.118644 200.135593 66.355932
+C 197.694915 63.915254 197.694915 63.915254 197.694915 61.474576
+C 197.694915 59.033898 197.694915 59.033898 200.135593 56.59322
+z
+" style="fill:#007d7a;stroke:#007d7a;stroke-linejoin:miter;"/>
+ </g>
+ <g id="patch_7">
+ <path clip-path="url(#pa9854da833)" d="M 248.949153 93.20339
+C 248.949153 19.983051 283.118644 49.271186 283.118644 76.118644
+C 258.711864 54.152542 253.830508 83.440678 278.237288 83.440678
+C 258.711864 83.440678 253.830508 102.966102 283.118644 93.20339
+C 283.118644 122.491525 248.949153 112.728814 248.949153 93.20339
+z
+" style="fill:#007d7a;stroke:#007d7a;stroke-linejoin:miter;"/>
+ </g>
+ <g id="text_1">
+ <!-- ur -->
+ <g style="fill:#007d7a;" transform="translate(90.487754 112.728814)scale(0.72 -0.72)">
+ <defs>
+ <path d="M 37.203125 6.40625
+L 37.40625 6.40625
+L 37.90625 0
+C 37.90625 -0.203125 38.203125 -0.296875 38.796875 -0.296875
+C 40.09375 -0.203125 43.09375 0 44.5 0
+C 45.796875 0 50.59375 -0.09375 51.90625 -0.296875
+L 52.09375 0
+C 51.296875 4.296875 50.40625 11.703125 50.40625 19.203125
+L 50.40625 24.90625
+C 50.40625 32.40625 50.40625 37.59375 51.296875 42.90625
+L 51.203125 43.203125
+C 51.203125 43.203125 47 42.90625 43.5 42.90625
+C 40.09375 42.90625 36.703125 43.203125 36.703125 43.203125
+L 36.5 42.90625
+C 37.203125 37.296875 37.203125 32.5 37.203125 24.90625
+L 37.203125 16.90625
+C 37.203125 11.09375 30.40625 6.703125 26 6.703125
+C 23 6.703125 20.09375 7.703125 20.296875 15.90625
+L 20.5 24.90625
+C 20.59375 32.40625 20.796875 37.703125 21.296875 42.90625
+L 21.203125 43.203125
+C 21.203125 43.203125 17.203125 42.90625 13.703125 42.90625
+C 10.296875 42.90625 6.703125 43.203125 6.703125 43.203125
+L 6.5 42.90625
+C 7.09375 37.203125 7.40625 32.09375 7.296875 24.90625
+L 7.09375 12.59375
+C 7 5.703125 10.5 -1 20.40625 -1
+C 24.5 -1 31.59375 0.40625 37.203125 6.40625
+z
+" id="LinBiolinumOB-117"/>
+ <path d="M 22.5 34.90625
+L 22 35.203125
+C 22 37.40625 21.90625 41.59375 21.796875 42.5
+C 21.59375 43.09375 21.5 43.5 20.90625 43.5
+C 18 43 11.5 43 8 43.203125
+L 7.796875 42.90625
+C 8.5 38.59375 8.90625 30.90625 8.90625 23.5
+L 8.90625 18
+C 8.90625 10.5 8.703125 5.40625 7.90625 0
+L 8 -0.296875
+C 8 -0.296875 11.203125 0 14.703125 0
+C 18.09375 0 22.90625 -0.296875 22.90625 -0.296875
+L 23.09375 0
+C 22.296875 5.703125 22.09375 10.40625 22.09375 18
+L 22.09375 25.296875
+C 22.09375 33 28.90625 34.203125 30.5 34.203125
+C 32.40625 34.203125 34.5 33.703125 36.296875 32.296875
+L 38.09375 32.5
+L 40.09375 42.796875
+L 39.703125 43.203125
+C 38 43.703125 36.40625 43.90625 34.59375 43.90625
+C 29.90625 43.90625 25.09375 38.796875 22.5 34.90625
+z
+" id="LinBiolinumOB-114"/>
+ </defs>
+ <use xlink:href="#LinBiolinumOB-117"/>
+ <use x="59.099991" xlink:href="#LinBiolinumOB-114"/>
+ </g>
+ </g>
+ </g>
+ </g>
+ <defs>
+ <clipPath id="pa9854da833">
+ <rect height="161.084746" width="288" x="0" y="0.457627"/>
+ </clipPath>
+ </defs>
+</svg>
diff --git a/documentation/_static/api.rst b/documentation/_static/api.rst
deleted file mode 100644
index a1288bad8a875cff458e68ce13f7bcb85debed9c..0000000000000000000000000000000000000000
--- a/documentation/_static/api.rst
+++ /dev/null
@@ -1,44 +0,0 @@
-===
-API
-===
-
-
-----------
-bfps.tools
-----------
-
-.. automodule:: bfps.tools
- :members:
- :undoc-members:
- :show-inheritance:
-
-
------------------
-bfps.NavierStokes
------------------
-
-.. automodule:: bfps.NavierStokes
- :members:
- :undoc-members:
- :show-inheritance:
-
-
------------------
-bfps.FluidConvert
------------------
-
-.. automodule:: bfps.FluidConvert
- :members:
- :undoc-members:
- :show-inheritance:
-
-
-----------------
-bfps.FluidResize
-----------------
-
-.. automodule:: bfps.FluidResize
- :members:
- :undoc-members:
- :show-inheritance:
-
diff --git a/documentation/bandpass/notes0.txt b/documentation/bandpass/notes0.txt
new file mode 100644
index 0000000000000000000000000000000000000000..6608629e2f2c0d83bbff8c885fafa9a0b62839e1
--- /dev/null
+++ b/documentation/bandpass/notes0.txt
@@ -0,0 +1,33 @@
+-------------------------
+TurTLE: Bandpass tutorial
+-------------------------
+
+Exercise objective:
+-------------------
+
+We sill learn how to develop with TurTLE using and expanding the existing infrastructure.
+
+Problem:
+--------
+
+Band-filtering: fluctuations of a turbulent field happened on different scales. Filter these scales using filters in Fourier space, then transform to real space and take moments, histograms, in general calculate statistics.
+
+Tools:
+------
+
+TurTLE can rum simulations (DNS), do post-processing (PP), or run simple tests (test).
+
+Overview
+--------
+
+Steps taken to solve the above problem:
+
+ * Get code. In documentation/cpp use the command doxygen cpp-config to produce a readable documentation file of the code. There data structure and hierarchies can be overseen. For the task at hand the post-processing (PP) class is the most suitable to expand and adapt.
+ * Create a git branch to work on, feature/bandpass_tutorial.
+ * Look for the post-processing help (PP) and get acquainted with methods, classes, etc. We will eventually need to define a non-linear filter, and the modes we want to calculate. Additionally, the bands have to be defined, e.g. by a list of lower and upper wave-numbers k0list and k1list. The best way to proceed is to start from the preexisting get_rfields and modify it acordingly. In order for it to be compiled and included in the library we must modify the PP.py and CmakeLists.txt files. Statistics may be called from the field class.
+ * Based on get_rfields write a new bandpass_stats.cpp and include the necessary parameters. Make sure the python wrapper PP.py is writing the parameters you need.
+ * Tip: use gitg to visualize git history and changes.
+ * Write a test to check not only that compilation is successful, but also that the new parameters are being written and read from the hdf5 parameters file. For data you can easily use the TurTLE test functionality.
+ * Calculate the filtered fields using the given filtering options and obtain the desired band-passed field by subtracting two filtered fields. This defines the non-linear filter.
+ * Look in the code for examples on how to use the CLOOP subroutine. This is the most efficient way to carry out the subtraction operation.
+
diff --git a/documentation/bandpass/notes1.txt b/documentation/bandpass/notes1.txt
new file mode 100644
index 0000000000000000000000000000000000000000..3326d5d81bf2156f79a542939cddef653c878712
--- /dev/null
+++ b/documentation/bandpass/notes1.txt
@@ -0,0 +1,52 @@
+
+
+git clone turtle
+
+- cd documnetation/cpp
+- doxygen cpp_config
+- firefox html/index.html
+
+- see the class hierarchy
+
+- get_rfields<rnumber>
+- hdf5_tools
+
+----------------------------
+
+
+We use TurTLE to write a test code that bandpasses turbulent velocity field
+and calculates statistics:
+
+TurTLE has :-
+
+1/ DNS
+2/ Post Processing
+3/ Test
+
+DNS :-
+
+initialise --> time marching --> finalise
+
+PP:-
+
+loop for every checkpoint
+
+Test:-
+
+initialise --> task --> finalise
+
+
+For the bandpass tutorial
+
+PP :- read --> filter --> statistics --> output
+
+--------------------------
+
+While debugging make sure -DNDEBUG is disabled
+debug messages are written in err files.
+
+Usage:
+DEBUG_MSG("....%d ...", n);
+------------------------
+
+
diff --git a/documentation/chapters/AUTHORS b/documentation/chapters/AUTHORS
new file mode 120000
index 0000000000000000000000000000000000000000..f04b7e8a2af221a97d050fdf3e89cb77fe4407ef
--- /dev/null
+++ b/documentation/chapters/AUTHORS
@@ -0,0 +1 @@
+../../AUTHORS
\ No newline at end of file
diff --git a/documentation/_static/README.rst b/documentation/chapters/README.rst
similarity index 100%
rename from documentation/_static/README.rst
rename to documentation/chapters/README.rst
diff --git a/documentation/chapters/api.rst b/documentation/chapters/api.rst
new file mode 100644
index 0000000000000000000000000000000000000000..16b3b3b592d54fc204ea124c45b5e097799f9142
--- /dev/null
+++ b/documentation/chapters/api.rst
@@ -0,0 +1,53 @@
+==========
+Python API
+==========
+
+
+------------
+TurTLE
+------------
+
+.. automodule:: TurTLE
+ :members:
+ :undoc-members:
+ :show-inheritance:
+
+------------
+TurTLE.tools
+------------
+
+.. automodule:: TurTLE.tools
+ :members:
+ :undoc-members:
+ :show-inheritance:
+
+
+-----------
+TurTLE.TEST
+-----------
+
+.. automodule:: TurTLE.TEST
+ :members:
+ :undoc-members:
+ :show-inheritance:
+
+
+----------
+TurTLE.DNS
+----------
+
+.. automodule:: TurTLE.DNS
+ :members:
+ :undoc-members:
+ :show-inheritance:
+
+
+---------
+TurTLE.PP
+---------
+
+.. automodule:: TurTLE.PP
+ :members:
+ :undoc-members:
+ :show-inheritance:
+
diff --git a/documentation/chapters/cpp_doxygen.rst b/documentation/chapters/cpp_doxygen.rst
new file mode 100644
index 0000000000000000000000000000000000000000..6823569063ce9e7b58976d412d280e2f3eacd807
--- /dev/null
+++ b/documentation/chapters/cpp_doxygen.rst
@@ -0,0 +1,128 @@
+---
+C++
+---
+
+kspace
+------
+
+`Full doxygen documentation link. <../../doxygen_html/classkspace.html>`_
+
+.. doxygenclass:: kspace
+ :project: TurTLE
+ :members:
+
+
+
+field
+-----
+
+`Full doxygen documentation link. <../../doxygen_html/classfield.html>`_
+
+.. doxygenclass:: field
+ :project: TurTLE
+ :members:
+
+
+code_base
+---------
+
+`Full doxygen documentation link. <../../doxygen_html/classcode__base.html>`_
+
+.. doxygenclass:: code_base
+ :project: TurTLE
+ :members:
+
+
+direct_numerical_simulation
+---------------------------
+
+`Full doxygen documentation link. <../../doxygen_html/classdirect__numerical__simulation.html>`_
+
+.. doxygenclass:: direct_numerical_simulation
+ :project: TurTLE
+ :members:
+
+
+NSE
+----
+
+`Full doxygen documentation link. <../../doxygen_html/classNSE.html>`_
+
+.. doxygenclass:: NSE
+ :project: TurTLE
+ :members:
+
+
+NSVE
+----
+
+`Full doxygen documentation link. <../../doxygen_html/classNSVE.html>`_
+
+.. doxygenclass:: NSVE
+ :project: TurTLE
+ :members:
+
+
+abstract_particle_set
+---------------------
+
+`Full doxygen documentation link. <../../doxygen_html/classabstract__particle__set.html>`_
+
+ .. doxygenclass:: abstract_particle_set
+ :project: TurTLE
+ :path: ...
+ :members: [...]
+ :protected-members:
+ :private-members:
+ :undoc-members:
+ :outline:
+ :no-link:
+
+
+abstract_particle_rhs
+---------------------
+
+`Full doxygen documentation link. <../../doxygen_html/classabstract__particle__rhs.html>`_
+
+ .. doxygenclass:: abstract_particle_rhs
+ :project: TurTLE
+ :path: ...
+ :members: [...]
+ :protected-members:
+ :private-members:
+ :undoc-members:
+ :outline:
+ :no-link:
+
+
+field_tinterpolator
+-------------------
+
+`Full doxygen documentation link. <../../doxygen_html/classfield__tinterpolator.html>`_
+
+ .. doxygenclass:: field_tinterpolator
+ :project: TurTLE
+ :path: ...
+ :members: [...]
+ :protected-members:
+ :private-members:
+ :undoc-members:
+ :outline:
+ :no-link:
+
+
+particle_solver
+---------------
+
+`Full doxygen documentation link. <../../doxygen_html/classparticle__solver.html>`_
+
+ .. doxygenclass:: particle_solver
+ :project: TurTLE
+ :path: ...
+ :members: [...]
+ :protected-members:
+ :private-members:
+ :undoc-members:
+ :outline:
+ :no-link:
+
diff --git a/documentation/conf.py b/documentation/conf.py.in
similarity index 50%
rename from documentation/conf.py
rename to documentation/conf.py.in
index 5fc2f86699b6f446001efb8dda00fab733c434a4..89e40fedc21df0f888c8ac31c19f653007db471b 100644
--- a/documentation/conf.py
+++ b/documentation/conf.py.in
@@ -1,178 +1,73 @@
-#!/usr/bin/env python3
-# -*- coding: utf-8 -*-
+# Configuration file for the Sphinx documentation builder.
#
-# bfps documentation build configuration file, created by
-# sphinx-quickstart on Thu Jan 21 20:53:27 2016.
-#
-# This file is execfile()d with the current directory set to its
-# containing dir.
-#
-# Note that not all possible configuration values are present in this
-# autogenerated file.
-#
-# All configuration values have a default; values that are commented out
-# serve to show the default.
+# This file only contains a selection of the most common options. For a full
+# list see the documentation:
+# https://www.sphinx-doc.org/en/master/usage/configuration.html
-import sys
-import os
-import shlex
+# -- Path setup --------------------------------------------------------------
# If extensions (or modules to document with autodoc) are in another directory,
# add these directories to sys.path here. If the directory is relative to the
# documentation root, use os.path.abspath to make it absolute, like shown here.
-#sys.path.insert(0, os.path.abspath('.'))
-sys.path.insert(0, os.path.abspath('..'))
+#
+import os
+import sys
+sys.path.insert(0, os.path.abspath('@PROJECT_BINARY_DIR@/python'))
+
+breathe_projects = {
+ 'TurTLE' : os.path.abspath('@CMAKE_CURRENT_BINARY_DIR@/xml/'),
+ }
+
-# -- General configuration ------------------------------------------------
+# -- Project information -----------------------------------------------------
-# If your documentation needs a minimal Sphinx version, state it here.
-#needs_sphinx = '1.0'
+project = 'TurTLE'
+copyright = '2022, TurTLE team'
+author = 'TurTLE team'
+
+# The full version, including alpha/beta/rc tags
+VERSION = '@TURTLE_VERSION@'
+version = VERSION
+release = VERSION
+
+
+# -- General configuration ---------------------------------------------------
# Add any Sphinx extension module names here, as strings. They can be
# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom
# ones.
extensions = [
+ 'breathe',
'sphinx.ext.autodoc',
'sphinx.ext.mathjax',
'sphinx.ext.viewcode',
+ 'sphinx.ext.imgconverter',
]
# Add any paths that contain templates here, relative to this directory.
-templates_path = ['_templates']
+templates_path = ['sphinx_templates']
-# The suffix(es) of source filenames.
-# You can specify multiple suffix as a list of string:
-# source_suffix = ['.rst', '.md']
source_suffix = '.rst'
-
-# The encoding of source files.
-#source_encoding = 'utf-8-sig'
-
-# The master toctree document.
master_doc = 'index'
-# General information about the project.
-project = 'bfps'
-copyright = '2016, Cristian C Lalescu'
-author = 'Cristian C Lalescu'
-
-### package versioning
-# get current time
-import datetime
-import shutil
-import subprocess
-now = datetime.datetime.now()
-# obtain version
-try:
- git_branch = subprocess.check_output(['git',
- 'rev-parse',
- '--abbrev-ref',
- 'HEAD']).strip().split()[-1].decode()
- git_revision = subprocess.check_output(['git', 'rev-parse', 'HEAD']).strip()
- git_date = datetime.datetime.fromtimestamp(int(subprocess.check_output(['git', 'log', '-1', '--format=%ct']).strip()))
-except:
- git_revision = ''
- git_branch = ''
- git_date = now
-if git_branch == '':
- # there's no git available or something
- release = '{0:0>4}{1:0>2}{2:0>2}.{3:0>2}{4:0>2}{5:0>2}'.format(
- git_date.year, git_date.month, git_date.day,
- git_date.hour, git_date.minute, git_date.second)
-else:
- if (('develop' in git_branch) or
- ('feature' in git_branch) or
- ('bugfix' in git_branch)):
- release = subprocess.check_output(['git', 'describe', '--tags']).strip().decode()
- else:
- release = subprocess.check_output(['git', 'describe', '--tags']).strip().decode().split('-')[0]
-# The version info for the project you're documenting, acts as replacement for
-# |version| and |release|, also used in various other places throughout the
-# built documents.
-#
-# release is the full version, including alpha/beta/rc tags.
-# version is supposed to be the short X.Y version.
-version = release
-
-# The language for content autogenerated by Sphinx. Refer to documentation
-# for a list of supported languages.
-#
-# This is also used if you do content translation via gettext catalogs.
-# Usually you set "language" from the command line for these cases.
-language = None
-
-# There are two options for replacing |today|: either, you set today to some
-# non-false value, then it is used:
-#today = ''
-# Else, today_fmt is used as the format for a strftime call.
-#today_fmt = '%B %d, %Y'
-
# List of patterns, relative to source directory, that match files and
# directories to ignore when looking for source files.
-exclude_patterns = ['_build']
-
-# The reST default role (used for this markup: `text`) to use for all
-# documents.
-#default_role = None
-
-# If true, '()' will be appended to :func: etc. cross-reference text.
-#add_function_parentheses = True
-
-# If true, the current module name will be prepended to all description
-# unit titles (such as .. function::).
-#add_module_names = True
-
-# If true, sectionauthor and moduleauthor directives will be shown in the
-# output. They are ignored by default.
-#show_authors = False
-
-# The name of the Pygments (syntax highlighting) style to use.
-pygments_style = 'sphinx'
-
-# A list of ignored prefixes for module index sorting.
-#modindex_common_prefix = []
-
-# If true, keep warnings as "system message" paragraphs in the built documents.
-#keep_warnings = False
+# This pattern also affects html_static_path and html_extra_path.
+exclude_patterns = ['sphinx_build']
-# If true, `todo` and `todoList` produce output, else they produce nothing.
-todo_include_todos = False
-
-# -- Options for HTML output ----------------------------------------------
+# -- Options for HTML output -------------------------------------------------
# The theme to use for HTML and HTML Help pages. See the documentation for
# a list of builtin themes.
+#
html_theme = 'alabaster'
-
-# Theme options are theme-specific and customize the look and feel of a theme
-# further. For a list of options available for each theme, see the
-# documentation.
-#html_theme_options = {}
-
-# Add any paths that contain custom themes here, relative to this directory.
-#html_theme_path = []
-
-# The name for this set of Sphinx documents. If None, it defaults to
-# "<project> v<release> documentation".
-#html_title = None
-
-# A shorter title for the navigation bar. Default is the same as html_title.
-#html_short_title = None
-
-# The name of an image file (relative to this directory) to place at the top
-# of the sidebar.
-#html_logo = None
-
-# The name of an image file (within the static path) to use as favicon of the
-# docs. This file should be a Windows icon file (.ico) being 16x16 or 32x32
-# pixels large.
-#html_favicon = None
+pygments_style = 'sphinx'
# Add any paths that contain custom static files (such as style sheets) here,
# relative to this directory. They are copied after the builtin static files,
# so a file named "default.css" will overwrite the builtin "default.css".
-html_static_path = ['_static']
+html_static_path = ['@PROJECT_SOURCE_DIR@/documentation/sphinx_static']
# Add any extra paths that contain custom files (such as robots.txt or
# .htaccess) here, relative to this directory. These files are copied
@@ -222,8 +117,8 @@ html_static_path = ['_static']
# Language to be used for generating the HTML full-text search index.
# Sphinx supports the following languages:
-# 'da', 'de', 'en', 'es', 'fi', 'fr', 'h', 'it', 'ja'
-# 'nl', 'no', 'pt', 'ro', 'r', 'sv', 'tr'
+# 'da', 'de', 'en', 'es', 'fi', 'fr', 'hu', 'it', 'ja'
+# 'nl', 'no', 'pt', 'ro', 'ru', 'sv', 'tr'
#html_search_language = 'en'
# A dictionary with options for the search language support, empty by default.
@@ -235,7 +130,7 @@ html_static_path = ['_static']
#html_search_scorer = 'scorer.js'
# Output file base name for HTML help builder.
-htmlhelp_basename = 'bfpsdoc'
+htmlhelp_basename = 'TurTLEdoc'
# -- Options for LaTeX output ---------------------------------------------
@@ -247,7 +142,7 @@ latex_elements = {
#'pointsize': '10pt',
# Additional stuff for the LaTeX preamble.
-#'preamble': '',
+'preamble': '\\usepackage{amssymb,amsmath,amsfonts}',
# Latex figure (float) alignment
#'figure_align': 'htbp',
@@ -257,8 +152,8 @@ latex_elements = {
# (source start file, target name, title,
# author, documentclass [howto, manual, or own class]).
latex_documents = [
- (master_doc, 'bfps.tex', 'bfps Documentation',
- 'Cristian C Lalescu', 'manual'),
+ (master_doc, 'TurTLE.tex', 'TurTLE Documentation',
+ 'TurTLE team', 'manual'),
]
# The name of an image file (relative to this directory) to place at the top of
@@ -287,7 +182,7 @@ latex_documents = [
# One entry per manual page. List of tuples
# (source start file, name, description, authors, manual section).
man_pages = [
- (master_doc, 'bfps', 'bfps Documentation',
+ (master_doc, 'turtle', 'TurTLE Documentation',
[author], 1)
]
@@ -301,9 +196,9 @@ man_pages = [
# (source start file, target name, title, author,
# dir menu entry, description, category)
texinfo_documents = [
- (master_doc, 'bfps', 'bfps Documentation',
- author, 'bfps', 'One line description of project.',
- 'Miscellaneous'),
+ (master_doc, 'TurTLE', u'TurTLE Documentation',
+ author, 'TurTLE', 'One line description of project.',
+ 'Miscellaneous'),
]
# Documents to append as an appendix to all manuals.
diff --git a/documentation/cpp/cpp_config b/documentation/cpp/cpp_config
index 02cf369779ec1370da3303b018d39e11be9613d4..e5fa2a6d5b6ab0e54cba96d3cb8e1f94805342af 100644
--- a/documentation/cpp/cpp_config
+++ b/documentation/cpp/cpp_config
@@ -32,7 +32,7 @@ DOXYFILE_ENCODING = UTF-8
# title of most generated pages and in a few other places.
# The default value is: My Project.
-PROJECT_NAME = "BFPS"
+PROJECT_NAME = "TURTLE"
# The PROJECT_NUMBER tag can be used to enter a project or revision number. This
# could be handy for archiving the generated documentation or if some version
@@ -238,7 +238,10 @@ TAB_SIZE = 4
# "Side Effects:". You can put \n's in the value part of an alias to insert
# newlines.
-ALIASES =
+# added rst and endrst as per https://exhale.readthedocs.io/en/latest/mastering_doxygen.html (retrieved 2019-11-16)
+# this is such that we can use reStructuredText within the documentation
+ALIASES = "rst=\verbatim embed:rst:leading-asterisk"
+ALIASES += "endrst=\endverbatim"
# This tag can be used to specify a number of word-keyword mappings (TCL only).
# A mapping has the form "name=value". For example adding "class=itcl::class"
@@ -435,25 +438,25 @@ LOOKUP_CACHE_SIZE = 0
# normally produced when WARNINGS is set to YES.
# The default value is: NO.
-EXTRACT_ALL = NO
+EXTRACT_ALL = YES
# If the EXTRACT_PRIVATE tag is set to YES, all private members of a class will
# be included in the documentation.
# The default value is: NO.
-EXTRACT_PRIVATE = NO
+EXTRACT_PRIVATE = YES
# If the EXTRACT_PACKAGE tag is set to YES, all members with package or internal
# scope will be included in the documentation.
# The default value is: NO.
-EXTRACT_PACKAGE = NO
+EXTRACT_PACKAGE = YES
# If the EXTRACT_STATIC tag is set to YES, all static members of a file will be
# included in the documentation.
# The default value is: NO.
-EXTRACT_STATIC = NO
+EXTRACT_STATIC = YES
# If the EXTRACT_LOCAL_CLASSES tag is set to YES, classes (and structs) defined
# locally in source files will be included in the documentation. If set to NO,
@@ -790,7 +793,7 @@ WARN_LOGFILE =
# spaces. See also FILE_PATTERNS and EXTENSION_MAPPING
# Note: If this tag is empty the current directory is searched.
-INPUT = ../../bfps/cpp
+INPUT = ../cpp
# This tag can be used to specify the character encoding of the source files
# that doxygen parses. Internally doxygen uses the UTF-8 encoding. Doxygen uses
@@ -1504,6 +1507,10 @@ TREEVIEW_WIDTH = 250
EXT_LINKS_IN_WINDOW = NO
+# file with custom latex macros
+
+FORMULA_MACROFILE = ../documentation/cpp/latex_defs.tex
+
# Use this tag to change the font size of LaTeX formulas included as images in
# the HTML documentation. When you change the font size after a successful
# doxygen run you need to manually remove any form_*.png images from the HTML
@@ -1666,7 +1673,7 @@ EXTRA_SEARCH_MAPPINGS =
# If the GENERATE_LATEX tag is set to YES, doxygen will generate LaTeX output.
# The default value is: YES.
-GENERATE_LATEX = YES
+GENERATE_LATEX = NO
# The LATEX_OUTPUT tag is used to specify where the LaTeX docs will be put. If a
# relative path is entered the value of OUTPUT_DIRECTORY will be put in front of
@@ -1946,7 +1953,7 @@ MAN_LINKS = NO
# captures the structure of the code including all documentation.
# The default value is: NO.
-GENERATE_XML = NO
+GENERATE_XML = YES
# The XML_OUTPUT tag is used to specify where the XML pages will be put. If a
# relative path is entered the value of OUTPUT_DIRECTORY will be put in front of
@@ -2099,7 +2106,7 @@ INCLUDE_FILE_PATTERNS =
# recursively expanded use the := operator instead of the = operator.
# This tag requires that the tag ENABLE_PREPROCESSING is set to YES.
-PREDEFINED =
+PREDEFINED = IN_DOXYGEN
# If the MACRO_EXPANSION and EXPAND_ONLY_PREDEF tags are set to YES then this
# tag can be used to specify a list of macro names that should be expanded. The
@@ -2301,7 +2308,7 @@ UML_LIMIT_NUM_FIELDS = 10
# The default value is: NO.
# This tag requires that the tag HAVE_DOT is set to YES.
-TEMPLATE_RELATIONS = NO
+TEMPLATE_RELATIONS = YES
# If the INCLUDE_GRAPH, ENABLE_PREPROCESSING and SEARCH_INCLUDES tags are set to
# YES then doxygen will generate a graph for each documented file showing the
diff --git a/documentation/cpp/latex_defs.tex b/documentation/cpp/latex_defs.tex
new file mode 100644
index 0000000000000000000000000000000000000000..ed93a7b638ec486d67df79cbeff2632fe6de38b1
--- /dev/null
+++ b/documentation/cpp/latex_defs.tex
@@ -0,0 +1,2 @@
+\newcommand{\bs}[1]{{\mathbf{#1}}}
+\newcommand{\NL}[1]{{\mathcal{N}[#1]}}
diff --git a/documentation/cpp/references.bib b/documentation/cpp/references.bib
index c8bdd9453c54609cd8eac207a9d4369e275baf8e..fc25c646662f98ca1b9f8388eb79696286a03994 100644
--- a/documentation/cpp/references.bib
+++ b/documentation/cpp/references.bib
@@ -12,3 +12,16 @@ archivePrefix = "arXiv",
adsurl = {http://adsabs.harvard.edu/abs/2017arXiv170603219B},
adsnote = {Provided by the SAO/NASA Astrophysics Data System}
}
+
+@article{shu1988jcp,
+author={Shu, C.-W. and Osher, S.},
+title={Efficient implementation of essentially non-oscillatory shock-capturing schemes},
+journal={J. Comput. Phys.},
+year={1988},
+volume={77},
+number={2},
+pages={439-471},
+doi={10.1016/0021-9991(88)90177-5},
+}
+
+
diff --git a/documentation/figs/interpolation.py b/documentation/figs/interpolation.py
index 302efcc157971b8b0407bb76bd3e7be6437f1206..95597b7dd5e2dfefd8f62f86db8859db116037c9 100644
--- a/documentation/figs/interpolation.py
+++ b/documentation/figs/interpolation.py
@@ -4,30 +4,44 @@ import math
def main():
slab = 2
- nproc = 5
- f = plt.figure(figsize = (6, 4.5))
+ nproc = 6
+ f = plt.figure(figsize = (4., 4.))
a = f.add_subplot(111)
+ for p in range(nproc):
+ for y in range((nproc)*slab):
+ a.plot([y, y],
+ range(p*slab, (p+1)*slab),
+ marker = '.',
+ linestyle = 'none',
+ color = 'brown')
+ a.set_ylim(bottom = -1, top = 12)
+ a.set_xlim(left = -1)
+ a.set_ylabel('$z$')
+ a.set_xlabel('$x,y$')
+ a.set_aspect('equal')
+ f.tight_layout(pad = 0.5)
+ f.savefig('interp_problem_0.pdf')
for p in range(nproc):
color = plt.get_cmap('plasma')(p*1./nproc)
a.add_patch(
mpatches.Rectangle(
[0, p*slab],
- slab*(nproc+2)-1, 1,
+ slab*(nproc)-1, 1,
color = color,
alpha = .2))
- a.text(-.5, p*slab+.5, '$p_{0}$'.format(p),
+ a.text(-.6, p*slab+.5, '$p_{0}$'.format(p),
verticalalignment = 'center')
- for y in range((nproc+2)*slab):
- a.plot([y, y],
- range(p*slab, (p+1)*slab),
- marker = '.',
- linestyle = 'none',
- color = color)
- for X, Y in [(9.9, 6.3),
- (3.3, 3.7)]:
+ f.savefig('interp_problem_1.pdf')
+ for X, Y in [(8.9, 8.3),
+ (2.3, 3.7)]:
a.plot([X], [Y],
color = 'black',
- marker = 'x')
+ marker = 'x',
+ markersize = 10,
+ markeredgewidth=3)
+ f.savefig('interp_problem_2.pdf')
+ for X, Y in [(8.9, 8.3),
+ (2.3, 3.7)]:
for n in [1, 2]:
a.add_patch(
mpatches.Rectangle(
@@ -35,16 +49,11 @@ def main():
2*n+1, 2*n+1,
color = 'green',
alpha = .2))
- a.text(math.floor(X)+.5, math.floor(Y - n)-.3,
+ a.text(math.floor(X)+.5, math.floor(Y - n)-.6,
'$n = {0}$'.format(n),
horizontalalignment = 'center')
- a.set_ylim(bottom = -1, top = 10)
- a.set_xlim(left = -1)
- a.set_ylabel('$z$')
- a.set_xlabel('$x,y$')
- a.set_aspect('equal')
- f.tight_layout()
f.savefig('interp_problem.pdf')
+ f.savefig('interp_problem_3.pdf')
return None
if __name__ == '__main__':
diff --git a/documentation/index.html.in b/documentation/index.html.in
new file mode 100644
index 0000000000000000000000000000000000000000..8614030531341f1e033c295771b5e1ccbc7793d7
--- /dev/null
+++ b/documentation/index.html.in
@@ -0,0 +1,44 @@
+<html>
+ <title>TurTLE v@TURTLE_VERSION@</title>
+ <body>
+ <h1>TurTLE v@TURTLE_VERSION@</h1>
+ <figure>
+ <a href="https://gitlab.mpcdf.mpg.de/TurTLE/turtle">
+ <img
+ src="TurTLE_logo.svg"
+ alt="TurTLE"></a>
+ <figcaption>Thank you for your interest in TurTLE.</figcaption>
+ </figure>
+ <p>
+ TurTLE stands for "Turbulence Tools: Lagrangian and Eulerian" and it is
+ developed and maintained by the Wilczek group at the
+ <a href="https://www.ds.mpg.de/wilczek">Max Planck
+ Institute for Dynamics and Self-Organization</a>
+ and the
+ <a href="https://www.wilczek.physik.uni-bayreuth.de/en/team/index.html">
+ University of Bayreuth</a>
+ in collaboration with the Application Support Group of the Max Planck
+ Computing and Data Facility.
+ </p>
+ <p>
+ TurTLE is a framework for
+ pseudo-spectral simulations of turbulence, with a highly optimized particle tracking method.
+ It consists of a C++ core library, as well as a Python "wrapper" library.
+ </p>
+ <p>
+ The documentation consists of two parts:
+ </p>
+ <ul>
+ <li>
+ <a href="doxygen_html/index.html">HTML C++</a> documentation.
+ Full Doxygen output, including, e.g., class index and inheritance graphs.</li>
+ <li>
+ <a href="sphinx_html/index.html">HTML manual</a> generated with Sphinx.
+ This contains tutorials, the Python library documentation, and part of the C++ Doxygen output.
+ </li>
+ </ul>
+ </body>
+ <footer>
+ Last updated @BUILD_DATE@.
+ </footer>
+</html>
diff --git a/documentation/index.rst b/documentation/index.rst
index 9b0abc23648a08a9bd6692d2bc7651323c1b09d8..36c60cd39a51e02fe717312b31da22ef1107032a 100644
--- a/documentation/index.rst
+++ b/documentation/index.rst
@@ -1,20 +1,25 @@
-.. bfps documentation master file, created by
- sphinx-quickstart on Thu Jan 21 20:53:27 2016.
+.. TurTLE documentation master file, created by
+ sphinx-quickstart on Sat Nov 16 10:17:05 2019.
You can adapt this file completely to your liking, but it should at least
contain the root `toctree` directive.
+Welcome to TurTLE's documentation!
+==================================
-Welcome to bfps's documentation!
-================================
+Contents:
.. toctree::
- :maxdepth: 2
+ :maxdepth: 4
- /_static/README
- /_static/overview
- /_static/development
- /_static/api
- /_static/cpp
+ chapters/README
+ sphinx_static/overview
+ sphinx_static/convergence
+ sphinx_static/bandpass
+ sphinx_static/development
+ chapters/api
+ chapters/cpp_doxygen
+
+ sphinx_static/bibliography
Indices and tables
diff --git a/documentation/merge_doc_html.sh.in b/documentation/merge_doc_html.sh.in
new file mode 100644
index 0000000000000000000000000000000000000000..0844d2e7ede25cb140159e132579315300beae93
--- /dev/null
+++ b/documentation/merge_doc_html.sh.in
@@ -0,0 +1,15 @@
+#! /usr/bin/env bash
+
+mkdir -p public/doxygen_html
+rsync --archive \
+ @CMAKE_CURRENT_BINARY_DIR@/html/ \
+ @CMAKE_CURRENT_BINARY_DIR@/public/doxygen_html
+rsync --archive \
+ @CMAKE_CURRENT_BINARY_DIR@/sphinx_html \
+ @CMAKE_CURRENT_BINARY_DIR@/public/
+rsync --archive \
+ @PROJECT_SOURCE_DIR@/documentation/TurTLE_logo.svg \
+ @CMAKE_CURRENT_BINARY_DIR@/public/
+cp \
+ @CMAKE_CURRENT_BINARY_DIR@/doc_full_index.html \
+ @CMAKE_CURRENT_BINARY_DIR@/public/index.html
diff --git a/documentation/sphinx_static/bandpass.rst b/documentation/sphinx_static/bandpass.rst
new file mode 100644
index 0000000000000000000000000000000000000000..8a3c12f06c7f358e9c72243ae93ae65782287612
--- /dev/null
+++ b/documentation/sphinx_static/bandpass.rst
@@ -0,0 +1,39 @@
+---------------------------------
+Tutorial: post-processing example
+---------------------------------
+
+Example problem:
+----------------
+
+Statistics of band-passed fields.
+Please see [drivas2017prf]_.
+
+The relevant files are :code:`cpp/full_code/bandpass_stats.?pp`,
+:code:`TurTLE/PP.py` and :code:`CMakeLists.txt` (the list of headers should be
+updated).
+
+Instructions in brief
+---------------------
+
+These are minimal instructions on how to create a generic post-processing tool,
+and they are roughly the procedure followed when writing the
+:code:`cpp/full_code/bandpass_stats.?pp` files.
+
+1. Get code, install (please see README).
+2. Have a look at the Doxygen generated documentation for the
+ ``postprocess`` class.
+3. Write down the explicit numerical task.
+ Identify algorithms, parameters, check whether required functionality
+ is available.
+4. Write down simple outline of new code to implement within
+ ``initialize``, ``work_on_current_iteration`` and ``finalize``.
+5. Create a new :code:`git` branch to work on, :code:`feature/<mypptool>`.
+6. One option is to start from preexisting tools such as :code:`get_rfields`
+ and modify them accordingly.
+ In particular simple standard modifications must be made to the :code:`PP.py`
+ and :code:`CmakeLists.txt` files.
+7. Design a test of the new functionality (possibly just a modification
+ of an existing test as needed).
+ Test should begin as a parameter I/O sanity check, and then grow as
+ the required functionality is implemented.
+
diff --git a/documentation/sphinx_static/bibliography.rst b/documentation/sphinx_static/bibliography.rst
new file mode 100644
index 0000000000000000000000000000000000000000..c648c9ac4e4da6bfc8dafba4d4ab24ff312745c1
--- /dev/null
+++ b/documentation/sphinx_static/bibliography.rst
@@ -0,0 +1,39 @@
+Bibliography
+============
+
+.. [fornberg1988mc] B. Fornberg,
+ *Generation of Finite Difference Formulas on Arbitrarily Spaced Grids*.
+ Mathematics of Computation,
+ **51:184**, 699-706, 1988
+
+.. [ishihara2007jfm] T. Ishihara et al,
+ *Small-scale statistics in high-resolution direct numerical
+ simulation of turbulence: Reynolds number dependence of
+ one-point velocity gradient statistics*.
+ J. Fluid Mech.,
+ **592**, 335-366, 2007
+
+.. [lalescu2010jcp] C. C. Lalescu, B. Teaca and D. Carati
+ *Implementation of high order spline interpolations for tracking test
+ particles in discretized fields*.
+ J. Comput. Phys.
+ **229**, 5862-5869, 2010
+
+.. [lalescu2021arXiv] C. C. Lalescu, B. Bramas, M. Rampp and M. Wilczek
+ *An Efficient Particle Tracking Algorithm for Large-Scale Parallel
+ Pseudo-Spectral Simulations of Turbulence*.
+ `arXiv 2107.01104 (2021) <https://arxiv.org/abs/2107.01104>`_
+
+.. [lalescu2018jfm] C. C. Lalescu and M. Wilczek
+ *Acceleration statistics of tracer particles in filtered turbulent fields*.
+ `J. Fluid Mech. 847, R2 (2018)
+ <https://doi.org/10.1017/jfm.2018.381>`_
+
+.. [drivas2017prf] Drivas, T. D. and Johnson, P. L. and Lalescu, C. C. and Wilczek, M.
+ *Large-scale sweeping of small-scale eddies in turbulence: A filtering approach*.
+ `Phys Rev Fluids 2 104603 (2017) <https://dx.doi.org/10.1103/PhysRevFluids.2.104603>`_
+
+.. [shu1988jcp] C.-W. Shu and S. Osher,
+ *Efficient implementation of essentially non-oscillatory shock-capturing schemes*.
+ `J. Comput. Phys. 77, 439-471 (1988) <http://dx.doi.org/10.1016/0021-9991(88)90177-5>`_
+
diff --git a/documentation/sphinx_static/convergence.rst b/documentation/sphinx_static/convergence.rst
new file mode 100644
index 0000000000000000000000000000000000000000..5d0b9574ed83d8efb1465766e800f98b13de628e
--- /dev/null
+++ b/documentation/sphinx_static/convergence.rst
@@ -0,0 +1,114 @@
+Tutorial: convergence test
+==========================
+
+An essential property of numerical methods is their accuracy. With
+standard approaches to differential equations, one may use
+generic convergence tests to characterize algorithms and/or
+implementations.
+
+As an example usage of TurTLE, we show here how to characterize
+convergence for the field solver, or the AdamsBashforth
+particle solver.
+The purpose of this tutorial is to introduce TurTLE users to nontrivial
+workflows that involve keeping track of a number of simulations and
+prescribing their initial conditions.
+
+The field solver test is a fairly simple pursuit, showing basic
+operations with the `turtle` launcher.
+
+The particle solver test is also minimal within the methodological
+constraints imposed by the use of the Adams Bashforth integration
+method, leading to a relatively increased complexity.
+The example shows how to account for a number of technical details: full
+control over multi-step method initialization and full control of fluid
+solver forcing parameters.
+
+
+Navier-Stokes solution
+----------------------
+
+TurTLE implements two distinct solvers of the Navier-Stokes equations:
+one for their vorticity formulation, and one for the more common velocity
+formulation.
+While there are inescapable differences in the two implementations, they
+both use the same time-stepping algorithm (a 3rd order RungeKutta
+method).
+
+We test convergence properties for a "statistically stationary initial
+condition", since the smoothness properties of the field will influence
+convergence properties in general.
+As an example of the variety of questions that we can address with
+TurTLE, we also perform a direct comparison of the NSVE (vorticity
+solver) and NSE (velocity solver) results.
+
+A basic Python script that performs this convergence test is provided
+under `examples/convergence/fields_temporal.py`, and it is discussed
+below.
+In brief, simulations are performed using the same initial conditions,
+but different time steps.
+Error estimates are computed as differences between the different
+solutions.
+The figure shows the L2 norm of these differences, normalized by
+the root-mean-squared velocity magnitude.
+Note that the actual value of this "relative error" is irrelevant
+(it depends on total integration time, spatial resolution, etc), but the
+plot shows that (1) solutions converge with the 3rd order of the
+timestep and (2) the systematic difference between NSE and NSVE behaves
+quite differently from the two individual error estimates.
+
+.. image:: err_vs_dt_field.svg
+
+Particle tracking errors
+------------------------
+
+There is a more complex mechanism that generates numerical errors for
+particle tracking in this setting.
+In particular, the accuracy of the field itself is relevant, as well as
+the accuracy of the interpolation method coupled to the ODE solver.
+For instance the default particle tracking solver is a 4th order
+Adams Bashforth method in TurTLE, which seems at first unnecessary since the
+fluid only provides a 3rd order solution.
+The default interpolation method is that of cubic splines, which should
+in principle limit ODE error convergence to 2nd order (because convergence
+relies on existence of relevant Taylor expansion terms, and interpolated
+fields only have a finite number of continuous derivatives).
+This discussion is briefly continued below, but we will not do it justice
+in this tutorial.
+
+We present below a Python script that performs a particle tracking
+convergence test, provided as
+`examples/convergence/particles_temporal.py`.
+The figure shows trajectory integration errors, in a fairly simple
+setting, for two interpolation methods: cubic splines (I4O3) and 5th
+order splines on a kernel of 8 points (I8O5).
+Furthermore, we show both the maximum and the mean errors for the same
+set of trajectories.
+While both mean errors seem to converge with the 3rd order of the
+timestep (consistent with the error of the velocity field), the maximum
+error obtained for cubic splines converges with the second order ---
+which is the expected behavior for particle tracking in a field that
+only has one continuous derivative.
+
+.. image:: err_vs_dt_particle.svg
+
+The results would merit a longer and more careful analysis outside the
+scope of this particular tutorial, so we limit ourselves to noting that
+(1) different ODEs will have different numerical properties (i.e.
+rotating particles, or inertial particles) and (2) interpolation errors
+may in general grow with the Reynolds number, since interpolation
+errors depend on values of gradients (however the distribution of
+extreme gradient values also changes with Reynolds number, so the
+overall behavior is complex).
+
+
+Python code for fields
+----------------------
+
+.. include:: convergence_fields.rst
+
+
+Python code for particles
+-------------------------
+
+.. include:: convergence_particles.rst
+
diff --git a/documentation/sphinx_static/convergence_fields.rst b/documentation/sphinx_static/convergence_fields.rst
new file mode 100644
index 0000000000000000000000000000000000000000..5830e4e7dff3e479d4e050678870cd68649d7e0d
--- /dev/null
+++ b/documentation/sphinx_static/convergence_fields.rst
@@ -0,0 +1,281 @@
+We present here a Python script that performs simple convergence tests
+with TurTLE. The script itself is provided as the file
+`examples/convergence/fields_temporal.py`.
+
+The preamble imports standard packages, TurTLE itself, and a few simple
+tools from `examples/convergence/fields_base.py` (not discussed here):
+
+.. code:: python
+
+ import numpy as np
+ import h5py
+ import matplotlib.pyplot as plt
+
+ import TurTLE
+ from TurTLE import DNS, PP
+
+ from fields_base import *
+
+As explained above, we should test convergence for a statistically
+stationary regime.
+The base simulation will use a real-space grid of :math:`128 \times 128 \times 128` points,
+and experience shows that reasonable quasistationarity is reached for this
+problem-size in a few hundred iterations (here `base_niterations`).
+For the convergence test itself, we use a relatively small number
+of iterations (i.e. `test_niterations`), otherwise deterministic chaos will dominate the
+difference between different solutions (rather than numerical integration
+errors).
+
+.. code:: python
+
+ base_niterations = 512
+ test_niterations = 32
+ grid_size = 128
+
+We distinguish between three steps of this script.
+First, we generate the statistically quasistationary regime.
+Second, we run the fluid solver.
+Finally, we "analyze" the results (i.e. we generate simple error plots).
+
+.. code:: python
+
+ def main():
+ generate_initial_conditions()
+
+ run_simulations_field(
+ err_vs_t = False)
+ plot_error_field(
+ err_vs_t = False)
+ return None
+
+The `err_vs_t` parameter may be used if one is interested in looking at
+the difference between the NSE and NSVE solutions as a function of time.
+It turns on a fine-grained output of the fields in
+`run_simulations_field` and it turns on the corresponding plot in
+`plot_error_field`.
+
+`generate_initial_conditions` performs a few calls to C++ executable
+codes generated with TurTLE. In brief, it
+
+ * generates a vorticity field in a quasistationary Navier-Stokes
+ regime (using the NSVE solver).
+ * generates the corresponding velocity field needed for the NSE solver
+
+We use the `launch` method of the `DNS` and `PP` Python
+classes --- the wrappers around the C++ `direct_numerical_simulation`
+and `postprocess` functionality.
+This means that the current Python code may be translated to a shell
+script directly, only by writing out the arguments (rather than placing
+them in a list of Python strings).
+The full function is given below:
+
+.. code:: python
+
+ def generate_initial_conditions():
+ # change these two values as needed.
+ # number of MPI processes to use
+ nprocesses = 8
+ # number of OpenMP threads per MPI process to use
+ nthreads_per_process = 1
+
+ # 1. Generate quasistationary state to use for initial conditions.
+ # create a dns object
+ c0 = DNS()
+ # launch the simulation
+ c0.launch([
+ 'NSVE',
+ '-n', '{0}'.format(grid_size),
+ '--np', '{0}'.format(nprocesses),
+ '--ntpp', '{0}'.format(nthreads_per_process),
+ '--precision', 'double',
+ '--src-simname', 'B32p1e4',
+ '--src-wd', TurTLE.data_dir,
+ '--src-iteration', '0',
+ '--simname', 'base',
+ '--niter_todo', '{0}'.format(base_niterations),
+ '--niter_out', '{0}'.format(base_niterations),
+ '--overwrite-src-parameters',
+ '--kMeta', '1.5',
+ '--fk0', '2',
+ '--fk1', '3',
+ '--niter_stat', '16',
+ '--checkpoints_per_file', '{0}'.format(64)])
+
+ # 3. Generate initial conditions for NSE runs.
+ # create postprocess object
+ cc = PP()
+ # launches code to compute velocity field at last iteration
+ cc.launch([
+ 'get_velocity',
+ '--np', '{0}'.format(nprocesses),
+ '--ntpp', '{0}'.format(nthreads_per_process),
+ '--simname', 'base',
+ '--precision', 'double',
+ '--iter0', '{0}'.format(base_niterations),
+ '--iter1', '{0}'.format(base_niterations)])
+ return None
+
+`run_simulations_field` runs six DNS (3 resolutions for each of NSE and
+NSVE solvers):
+
+.. code:: python
+
+ def run_simulations_field(err_vs_t = False):
+ # change these two values as needed.
+ # number of MPI processes to use
+ nprocesses = 8
+ # number of OpenMP threads per MPI process to use
+ nthreads_per_process = 1
+
+ if err_vs_t:
+ niter_out_factor = 1
+ else:
+ niter_out_factor = test_niterations
+
+ # 1. Run NSVE for the three resolutions.
+ for factor in [1, 2, 4]:
+ # create dns object
+ cc = DNS()
+ # launch simulation
+ cc.launch([
+ 'NSVE',
+ '-n', '{0}'.format(grid_size),
+ '--np', '{0}'.format(nprocesses),
+ '--ntpp', '{0}'.format(nthreads_per_process),
+ '--src-simname', 'base',
+ '--src-iteration', '{0}'.format(base_niterations),
+ '--simname', 'nsve_{0}x'.format(factor),
+ '--precision', 'double',
+ '--dtfactor', '{0}'.format(0.5 / factor),
+ '--niter_todo', '{0}'.format(test_niterations*factor),
+ '--niter_out', '{0}'.format(niter_out_factor*factor),
+ '--niter_stat', '{0}'.format(factor)])
+
+ # 2. Run NSE for the three resolutions.
+ for factor in [1, 2, 4]:
+ # create dns object
+ cc = DNS()
+ # launch simulation
+ cc.launch([
+ 'NSE',
+ '-n', '{0}'.format(grid_size),
+ '--np', '{0}'.format(nprocesses),
+ '--ntpp', '{0}'.format(nthreads_per_process),
+ '--src-simname', 'base',
+ '--src-iteration', '{0}'.format(base_niterations),
+ '--simname', 'nse_{0}x'.format(factor),
+ '--precision', 'double',
+ '--dtfactor', '{0}'.format(0.5 / factor),
+ '--niter_todo', '{0}'.format(test_niterations*factor),
+ '--niter_out', '{0}'.format(niter_out_factor*factor),
+ '--niter_out', '{0}'.format(test_niterations*factor),
+ '--niter_stat', '{0}'.format(factor)])
+ return None
+
+Once the runs are finished successfully, the code calls
+`plot_error_field`.
+Here we first read the output of the six different runs, and we compute
+their statistics, since the call to `compute_statistics` will also
+perform some sanity checks on the output.
+We then compute the error analysis for each of the two solvers, and also
+compare the two solutions.
+The `get_velocity` and `compute_vector_field_distance` methods are defined in
+`examples/convergence/fields_base.py`.
+
+.. code:: python
+
+ def plot_error_field(err_vs_t = False):
+ factor_list = [1, 2, 4]
+ c0_list = [DNS(simname = 'nsve_{0}x'.format(factor))
+ for factor in factor_list]
+ c1_list = [DNS(simname = 'nse_{0}x'.format(factor))
+ for factor in factor_list]
+ cl = [c0_list, c1_list]
+ # sanity checks
+ for cc in c0_list + c1_list:
+ cc.compute_statistics()
+
+ # errors for individual solvers
+ error_list = [[], [], []]
+ for ii in range(len(factor_list)-1):
+ factor = factor_list[ii]
+ for jj in [0, 1]:
+ vel1 = get_velocity(cl[jj][ii ], iteration = test_niterations*factor)
+ vel2 = get_velocity(cl[jj][ii+1], iteration = 2*test_niterations*factor)
+ dd = compute_vector_field_distance(vel1, vel2, figname = cl[jj][ii].simname + '_vs_' + cl[jj][ii+1].simname)
+ error_list[jj].append(dd['L2_rel'])
+
+ # comparisons of two solutions
+ for ii in range(len(factor_list)):
+ factor = factor_list[ii]
+ vel1 = get_velocity(cl[0][ii], iteration = test_niterations*factor)
+ vel2 = get_velocity(cl[1][ii], iteration = test_niterations*factor)
+ dd = compute_vector_field_distance(vel1, vel2)
+ error_list[2].append(dd['L2_rel'])
+
+ f = plt.figure(figsize = (4, 4))
+ a = f.add_subplot(111)
+ a.plot(factor_list[:len(error_list[0])],
+ error_list[0],
+ marker = '.',
+ dashes = (2, 3),
+ label = 'NSVE')
+ a.plot(factor_list[:len(error_list[1])],
+ error_list[1],
+ marker = '.',
+ dashes = (3, 5),
+ label = 'NSE')
+ a.plot(factor_list,
+ error_list[2],
+ marker = '.',
+ label = 'NSVE vs NSE')
+ fl = np.array(factor_list).astype(np.float)
+ for ee in [2, 3]:
+ a.plot(fl[:2], error_list[0][0] * fl[:2]**(-ee),
+ dashes = (ee, ee),
+ color = 'black',
+ label = '$\propto f^{{-{0}}}$'.format(ee),
+ zorder = -ee)
+ a.set_ylabel('relative error')
+ a.set_xlabel('resolution factor $f$')
+ a.set_xscale('log')
+ a.set_yscale('log')
+ a.legend(loc = 'best', fontsize = 6)
+ f.tight_layout()
+ f.savefig('err_vs_dt_field.pdf')
+ f.savefig('err_vs_dt_field.svg')
+ plt.close(f)
+
+ if err_vs_t:
+ t = range(test_niterations+1)
+ err_vs_t = [[], [], []]
+ # comparisons of two solvers
+ for ii in range(len(factor_list)):
+ factor = factor_list[ii]
+ for tt in t:
+ vel1 = get_velocity(cl[0][ii], iteration = tt*factor)
+ vel2 = get_velocity(cl[1][ii], iteration = tt*factor)
+ dd = compute_vector_field_distance(vel1, vel2)
+ err_vs_t[ii].append(dd['L2_rel'])
+ # distance between NSVE and NSE as a function of time
+ f = plt.figure(figsize = (4, 4))
+ a = f.add_subplot(111)
+ a.plot(t, err_vs_t[0], label = 'f = 1')
+ a.plot(t, err_vs_t[1], label = 'f = 2')
+ a.plot(t, err_vs_t[2], label = 'f = 4')
+ a.set_yscale('log')
+ a.legend(loc = 'best', fontsize = 6)
+ f.tight_layout()
+ f.savefig('err_vs_t_field.pdf')
+ f.savefig('err_vs_t_field.svg')
+ plt.close(f)
+ return None
+
+
+Finally, the script ends with a standard call to `main`:
+
+.. code:: python
+
+ if __name__ == '__main__':
+ main()
+
diff --git a/documentation/sphinx_static/convergence_particles.rst b/documentation/sphinx_static/convergence_particles.rst
new file mode 100644
index 0000000000000000000000000000000000000000..45fd4dab443a3dd3e51c911090ad0cfd1c01e316
--- /dev/null
+++ b/documentation/sphinx_static/convergence_particles.rst
@@ -0,0 +1,405 @@
+We present here a Python script that performs a particle tracking
+convergence test with TurTLE, in the current default configuration (a
+`particles_system` object coupled with an NSVE fluid solver).
+
+The preamble imports standard packages, and TurTLE itself:
+
+.. code:: python
+
+ import numpy as np
+ import h5py
+ import matplotlib.pyplot as plt
+
+ import TurTLE
+ from TurTLE import DNS
+
+We then set a few basic parameters, including a list of interpolation
+parameters.
+In brief, the spline interpolations used in TurTLE are parametrized by
+the size :math:`I` of the interpolation kernel and the order :math:`O` of
+the polynomial.
+In this example code we emphasize two equivalences:
+
+* :math:`I = 2 n + 2` where :math:`n` is the number of neighbouring
+ points (in one orientation) required to interpolate the field
+ within one cell.
+* :math:`O = 2 m + 1` where :math:`m` is the order of highest
+ continuous derivative of the interpolating polynomial.
+
+More details on the interpolation method can be found in
+[lalescu2010jcp]_.
+
+.. code:: python
+
+ base_niterations = 256
+
+ nparticles = 1000
+ particle_random_seed = 15
+ base_particle_dt = 0.5
+ test_niterations_particles = 128
+
+ neighbours_smoothness_list = [
+ (1, 1),
+ (3, 2)]
+
+We distinguish between three main parts of this script: generation of
+initial conditions with statistically stationary regime, computation of
+numerical solutions, analysis of results (plot commands):
+
+.. code:: python
+
+ def main():
+ generate_initial_conditions()
+
+ run_simulations_particles()
+
+ plot_error_particles()
+ plot_traj_particles()
+ return None
+
+`generate_initial_conditions` consists of a single launch of an NSVE
+run:
+
+.. code:: python
+
+ def generate_initial_conditions():
+ # change these two values as needed.
+ # number of MPI processes to use
+ nprocesses = 8
+ # number of OpenMP threads per MPI process to use
+ nthreads_per_process = 1
+
+ # 1. Generate quasistationary state to use for initial conditions.
+ # create a dns object
+ c0 = DNS()
+ # launch the simulation
+ c0.launch([
+ 'NSVE',
+ '-n', '32',
+ '--np', '{0}'.format(nprocesses),
+ '--ntpp', '{0}'.format(nthreads_per_process),
+ '--precision', 'double',
+ '--src-simname', 'B32p1e4',
+ '--src-wd', TurTLE.data_dir,
+ '--src-iteration', '0',
+ '--simname', 'base',
+ '--niter_todo', '{0}'.format(base_niterations),
+ '--niter_out', '{0}'.format(base_niterations),
+ '--overwrite-src-parameters',
+ '--kMeta', '1.',
+ '--niter_stat', '16',
+ '--checkpoints_per_file', '{0}'.format(16)])
+ return None
+
+We then define the function that runs the particle simulations,
+`run_simulations_particles`.
+Because we use an Adams-Bashforth method, we need to know the values of
+the particle velocity (i.e. the right-hand-side of the ODE) at previous
+steps.
+By default TurTLE accomplishes this by using the Euler integration scheme
+for "time step 0", then using a 2nd order Adams-Bashforth, etc, until
+the desired number of right-hand-side values is stored.
+Thus the code first integrates trajectories for 8 iterations, and the
+output will contain the additional information.
+This happens because such information is crucial for checkpointing ---
+i.e. stopping/restarting the simulation at arbitrary iterations without
+affecting the numerical solution.
+
+To put it simply, for the simulation "nsvep_base" the dataset
+"tracers0/rhs/0" contains only zeros, and TurTLE ignores them.
+However, the "tracers0/rhs/8" dataset for simulation "nsvep_1x" contains
+meaningful values (that **are** used because we request that `iter0` is
+different from 0).
+In order to generate consistent values of the "rhs" datasets, we first
+run a simulation at the highest resolution (smallest time-step), and
+then we extract the appropriate snapshots --- see also the second part
+of the trajectory plot function.
+
+Finally, the `launch` method is called in an unusual way for TurTLE.
+Firstly, we demand a DNS that starts at an iteration different from 0.
+Secondly, we manually copy the forcing parameters from the source
+simulation; since the source cannot be specified explicitly, the `DNS`
+class cannot search for the source parameters itself.
+Thirdly, I will note that one may in fact break this call by modifying
+the `iter0` and `checkpoints_per_file` values, since we have explicitly
+placed `iter0` in the `checkpoint` 0 file.
+While a full discussion of the `iter0` and `checkpoint` subtleties is
+beyond the scope of this tutorial, I will mention that it was a
+conscious choice to use a simple `iteration` to `checkpoint`
+correspondence, since we don't actually need anything smarter outside of
+extreme scenarios that need to be addressed individually anyway
+due to other complexities.
+
+.. code:: python
+
+ def run_simulations_particles():
+ # change these two values as needed.
+ # number of MPI processes to use
+ nprocesses = 8
+ # number of OpenMP threads per MPI process to use
+ nthreads_per_process = 1
+
+ # 1. Run NSVEparticles for a few iterations, to build up rhs values, consistent
+ # with the relevant velocity field.
+ factor = 1
+ for neighbours, smoothness in neighbours_smoothness_list:
+ interp_name = 'I{0}O{1}'.format(2*neighbours+2, 2*smoothness+1)
+ # create dns object
+ cc = DNS()
+ # launch simulation
+ cc.launch([
+ 'NSVEparticles',
+ '-n', '{0}'.format(factor*32),
+ '--np', '{0}'.format(nprocesses),
+ '--ntpp', '{0}'.format(nthreads_per_process),
+ '--src-simname', 'base',
+ '--src-iteration', '{0}'.format(base_niterations),
+ '--simname', 'nsvep_base_' + interp_name,
+ '--precision', 'double',
+ '--dtfactor', '{0}'.format(base_particle_dt/4),
+ '--kMeta', '{0}'.format(factor*1.0),
+ '--niter_todo', '{0}'.format(20),
+ '--niter_out', '{0}'.format(1),
+ '--niter_stat', '{0}'.format(1),
+ '--nparticles', '{0}'.format(nparticles),
+ '--niter_part', '{0}'.format(1),
+ '--tracers0_neighbours', '{0}'.format(neighbours),
+ '--tracers0_smoothness', '{0}'.format(smoothness),
+ '--tracers0_integration_steps', '4',
+ '--cpp_random_particles', '{0}'.format(particle_random_seed),
+ '--checkpoints_per_file', '{0}'.format(16)])
+
+ # 2. Prepare initial conditions
+ for neighbours, smoothness in neighbours_smoothness_list:
+ interp_name = 'I{0}O{1}'.format(2*neighbours+2, 2*smoothness+1)
+ for factor in [1, 2, 4]:
+ df = h5py.File('nsvep_{0}x_{1}_checkpoint_0.h5'.format(factor, interp_name), 'w')
+ # field
+ field_iteration = 16
+ df['vorticity/complex/{0}'.format(8*factor)] = h5py.ExternalLink(
+ 'nsvep_base_{0}_checkpoint_0.h5'.format(interp_name),
+ 'vorticity/complex/{0}'.format(field_iteration))
+ # particles
+ df['tracers0/state/{0}'.format(8*factor)] = h5py.ExternalLink(
+ 'nsvep_base_{0}_checkpoint_0.h5'.format(interp_name),
+ 'tracers0/state/16')
+ # copy rhs
+ source_file = h5py.File(
+ 'nsvep_base_{0}_checkpoint_0.h5'.format(interp_name), 'r')
+ rhs = source_file['tracers0/rhs/16'][()]
+ for tt in range(1, rhs.shape[0]):
+ ii = 16 - tt*4//factor + 1
+ #print(factor, tt, ii)
+ rhs[tt] = source_file['tracers0/rhs/{0}'.format(16 - tt*4//factor + 1)][1]
+ df['tracers0/rhs/{0}'.format(8*factor)] = rhs
+ df.close()
+
+ # 3. Run NSVEparticles
+ for factor in [1, 2, 4]:
+ # Test different interpolations.
+ # The loop iterates over number of neighbours and smoothness.
+ # The simulation names are defined based on "kernel size" and "order
+ # of polynomial".
+ # We do this to emphasize the one-to-one correspondence between
+ # neighbours and kernel size, and smoothness and order of polynomial.
+ for neighbours, smoothness in neighbours_smoothness_list:
+ interp_name = 'I{0}O{1}'.format(2*neighbours+2, 2*smoothness+1)
+ source_file = h5py.File('nsvep_base_{0}.h5'.format(interp_name), 'r')
+ # create dns object
+ cc = DNS()
+ # launch simulation
+ cc.launch([
+ 'NSVEparticles',
+ '-n', '{0}'.format(source_file['parameters/nx'][()]),
+ '--np', '{0}'.format(nprocesses),
+ '--ntpp', '{0}'.format(nthreads_per_process),
+ '--simname', 'nsvep_{0}x_{1}'.format(factor, interp_name),
+ '--precision', 'double',
+ '--dtfactor', '{0}'.format(base_particle_dt/factor),
+ '--niter_todo', '{0}'.format(test_niterations_particles*factor+8*factor),
+ '--niter_out', '{0}'.format(test_niterations_particles*factor+8*factor),
+ '--niter_stat', '{0}'.format(1),
+ ## ensure fluid physics is the same
+ '--dealias_type', '{0}'.format(source_file['parameters/dealias_type'][()]),
+ '--energy', '{0}'.format(source_file['parameters/energy'][()]),
+ '--famplitude', '{0}'.format(source_file['parameters/famplitude'][()]),
+ '--fk0', '{0}'.format(source_file['parameters/fk0'][()]),
+ '--fk1', '{0}'.format(source_file['parameters/fk1'][()]),
+ '--fmode', '{0}'.format(source_file['parameters/fmode'][()]),
+ '--friction_coefficient', '{0}'.format(source_file['parameters/friction_coefficient'][()]),
+ '--injection_rate', '{0}'.format(source_file['parameters/injection_rate'][()]),
+ '--nu', '{0}'.format(source_file['parameters/nu'][()]),
+ '--forcing_type', '{0}'.format(str(source_file['parameters/forcing_type'][()], 'ascii')),
+ ## particle params
+ '--nparticles', '{0}'.format(nparticles),
+ '--niter_part', '{0}'.format(1),
+ '--tracers0_neighbours', '{0}'.format(neighbours),
+ '--tracers0_smoothness', '{0}'.format(smoothness),
+ '--tracers0_integration_steps', '4',
+ '--cpp_random_particles', '0', # turn off C++ particle initialization
+ '--checkpoints_per_file', '{0}'.format(16)],
+ iter0 = 8*factor)
+ source_file.close()
+ return None
+
+To compute the trajectory integration error, we proceed as explained
+above, and compare each DNS with its corresponding double-the-resolution DNS:
+
+.. code:: python
+
+ def plot_error_particles():
+ f = plt.figure()
+ a = f.add_subplot(111)
+ factor_list = [1, 2]
+ factor_list = np.array(factor_list).astype(np.float)
+ for neighbours, smoothness in neighbours_smoothness_list:
+ interp_name = 'I{0}O{1}'.format(2*neighbours+2, 2*smoothness+1)
+ err_max_list = []
+ err_mean_list = []
+ for factor in factor_list:
+ factor = int(factor)
+ c1 = DNS(simname = 'nsvep_{0}x_'.format(int(factor)) + interp_name)
+ c2 = DNS(simname = 'nsvep_{0}x_'.format(int(factor)*2) + interp_name)
+ for cc in [c1, c2]:
+ cc.parameters['niter_part'] = cc.get_data_file()['parameters/niter_part'][()]
+ c1.compute_statistics(iter0 = 8*factor)
+ c2.compute_statistics(iter0 = 8*factor*2)
+ final_iter_x1 = c1.get_data_file()['iteration'][()]
+ final_iter_x2 = c2.get_data_file()['iteration'][()]
+ f1 = h5py.File(c1.simname + '_checkpoint_0.h5', 'r')
+ f2 = h5py.File(c2.simname + '_checkpoint_0.h5', 'r')
+ x1 = f1['tracers0/state/{0}'.format(final_iter_x1)][()]
+ x2 = f2['tracers0/state/{0}'.format(final_iter_x2)][()]
+ diff = np.sqrt(np.sum((x2-x1)**2, axis = -1))
+ err_max_list.append(np.max(diff))
+ err_mean_list.append(np.mean(diff))
+ f1.close()
+ f2.close()
+ err_max_list = np.array(err_max_list)
+ err_mean_list = np.array(err_mean_list)
+ a.plot(factor_list, err_max_list, marker = '.', label = 'max error ' + interp_name)
+ a.plot(factor_list, err_mean_list, marker = '.', label = 'mean error ' + interp_name)
+ for ee in [2, 3, 4]:
+ a.plot(factor_list,
+ (1e-4)*factor_list**(-ee),
+ color = 'black',
+ dashes = (ee, ee),
+ label = '$\propto f^{{-{0}}}$'.format(ee))
+ a.set_xscale('log')
+ a.set_yscale('log')
+ a.set_xlabel('resolution factor f')
+ a.set_ylabel('absolute trajectory error [DNS units]')
+ a.legend(loc = 'best', fontsize = 7)
+ f.tight_layout()
+ f.savefig('err_vs_dt_particle.pdf')
+ f.savefig('err_vs_dt_particle.svg')
+ plt.close(f)
+ return None
+
+For the beginner's convenience, we also provide a method to plot
+individual particle trajectories.
+Note that the second part of this function is in fact a sanity check for
+the initial conditions: the plot confirms that the data required for the
+multi-step method (explicitly placed in the checkpoint file) is
+consistent with the simulation parameters.
+
+.. code:: python
+
+ def plot_traj_particles():
+ ###########################################################################
+ f = plt.figure()
+ a = f.add_subplot(111)
+ for neighbours, smoothness in neighbours_smoothness_list:
+ interp_name = 'I{0}O{1}'.format(2*neighbours+2, 2*smoothness+1)
+ for factor in [1, 2, 4]:
+ cc = DNS(simname = 'nsvep_{0}x_'.format(factor) + interp_name)
+ cc.compute_statistics(iter0 = 8*factor)
+ tt = 3
+ xx = read_trajectory(cc, tt, iter0 = 8*factor)
+ a.plot(xx[:, 0],
+ xx[:, 1],
+ label = interp_name + '$f = {0}$'.format(factor),
+ marker = 'x',
+ dashes = (4/factor, 3, 4/factor),
+ alpha = 0.2)
+ #a.plot(xx[:, 0], xx[:, 1], dashes = (4/factor, 3, 4/factor), alpha = 0.2)
+ a.legend(loc = 'best', fontsize = 7)
+ a.set_xlabel('$x$ [code units]')
+ a.set_ylabel('$y$ [code units]')
+ f.tight_layout()
+ f.savefig('trajectories.pdf')
+ f.savefig('trajectories.svg')
+ plt.close(f)
+ ###########################################################################
+ ###########################################################################
+ traj_index = 3
+ for neighbours, smoothness in neighbours_smoothness_list:
+ interp_name = 'I{0}O{1}'.format(2*neighbours+2, 2*smoothness+1)
+ cc = DNS(simname = 'nsvep_base_' + interp_name)
+ cc.compute_statistics()
+ xx_base = read_trajectory(cc, traj_index, iter0 = 0, dset_name = 'velocity')
+ iter0 = 16
+ tt_base = np.array(range(iter0, iter0 + xx_base.shape[0]))*cc.parameters['dt']
+ f = plt.figure()
+ ax_counter = 1
+ for factor in [1, 2, 4]:
+ a = f.add_subplot(310 + ax_counter)
+ ax_counter += 1
+ a.plot(tt_base,
+ xx_base[:, 2],
+ label = 'base_' + interp_name,
+ marker = '+',
+ linewidth = 0.5,
+ alpha = 0.2)
+ cc = DNS(simname = 'nsvep_{0}x_'.format(factor) + interp_name)
+ cc.compute_statistics(iter0 = 8*factor)
+ xx = read_trajectory(cc, traj_index, iter0 = 8*factor, dset_name = 'velocity')
+ tt = np.array(range(8*factor, xx.shape[0]+8*factor))*cc.parameters['dt']
+ a.plot(tt,
+ xx[:, 2],
+ label = interp_name + '$f = {0}$'.format(factor),
+ marker = 'x',
+ dashes = (4/factor, 3, 4/factor),
+ alpha = 0.2)
+ df = h5py.File(cc.simname + '_checkpoint_0.h5', 'r')
+ xx = df['tracers0/rhs/{0}'.format(8*factor)][()]
+ for rhs_index in [1, 2, 3]:
+ a.scatter((8*factor - rhs_index)*cc.parameters['dt'],
+ xx[rhs_index, traj_index, 2],
+ marker = '*',
+ zorder = -10,
+ alpha = 0.5,
+ color = 'black')
+ df.close()
+ a.legend(loc = 'best', fontsize = 7)
+ a.set_xlabel('$t$ [code units]')
+ a.set_ylabel('$v_y$ [code units]')
+ f.tight_layout()
+ f.savefig('velocities_{0}.pdf'.format(interp_name))
+ plt.close(f)
+ ###########################################################################
+ return None
+
+ def read_trajectory(
+ cc,
+ traj_index,
+ iter0,
+ dset_name = 'position'):
+ cc.parameters['niter_part'] = cc.get_data_file()['parameters/niter_part'][()]
+ df = cc.get_particle_file()
+ xx = []
+ for ii in range(iter0, cc.get_data_file()['iteration'][()]+1, cc.parameters['niter_part']):
+ xx.append(df['tracers0/' + dset_name + '/{0}'.format(ii)][traj_index])
+ df.close()
+ return np.array(xx)
+
+.. image:: trajectories.svg
+
+Finally, the script ends with a standard call to `main`:
+
+.. code:: python
+
+ if __name__ == '__main__':
+ main()
+
+
diff --git a/documentation/_static/cpp.rst b/documentation/sphinx_static/cpp.rst
similarity index 100%
rename from documentation/_static/cpp.rst
rename to documentation/sphinx_static/cpp.rst
diff --git a/documentation/_static/development.rst b/documentation/sphinx_static/development.rst
similarity index 70%
rename from documentation/_static/development.rst
rename to documentation/sphinx_static/development.rst
index 2d9b9e525817d06c41ff15675a81f22e780702c5..7dc9836b820d1067bbf42d4fd3f7f6fc94530a45 100644
--- a/documentation/_static/development.rst
+++ b/documentation/sphinx_static/development.rst
@@ -7,7 +7,7 @@ Development
Versioning guidelines
---------------------
-Version tracking for :mod:`bfps` is done with ``git`` (see https://git-scm.com/
+Version tracking for :mod:`TurTLE` is done with ``git`` (see https://git-scm.com/
for description).
The branching model described at
http://nvie.com/posts/a-successful-git-branching-model/ should be
@@ -15,12 +15,12 @@ adhered to as strictly as possible.
The usable ``VERSION`` number will be constructed according to semantic
versioning rules (see http://semver.org/), ``MAJOR.MINOR.PATCH``.
-In principle, if you use :mod:`bfps` and you've created children of the
-:class:`NavierStokes <bfps.NavierStokes.NavierStokes>` class, you should not need to rewrite your code unless
+In principle, if you use :mod:`TurTLE` and you've created children of the
+:class:`DNS <TurTLE.DNS>` class, you should not need to rewrite your code unless
the ``MAJOR`` version changes.
There are 2 main branches, ``master`` and ``develop``.
-``setup.py`` will call ``git`` to read in the ``VERSION``: it will get the
+``CMake`` will essentially call ``git`` to read in the ``VERSION``: it will get the
latest available tag.
If the active branch name contains either of the strings ``develop``,
``feature`` or ``bugfix``, then the full output of ``git describe --tags``
@@ -56,22 +56,6 @@ projects, these may change.
Code testing
------------
-Testing for :mod:`bfps` is done with ``tox``.
+We use CTest for testing `TurTLE`, and the ``.gitlab-ci.yml`` script
+calls CTest.
-----------------
-Work in progress
-----------------
-
-HDF5 field I/O
---------------
-
-As you can tell from the ``todo.txt`` file, in the future the code will
-use HDF5 for input/output of fields.
-For now, field I/O is done with binary files, and the field data type is
-stored in the parameter file.
-
-Code flexibility
-----------------
-
-Version 2.0 will be designed so that new fluid equations can be coded in
-from python classes, rather than as new C++ files.
diff --git a/documentation/sphinx_static/err_vs_dt_field.svg b/documentation/sphinx_static/err_vs_dt_field.svg
new file mode 100644
index 0000000000000000000000000000000000000000..96260bb1503d7d03103c838a9acd0d082c23ab1c
--- /dev/null
+++ b/documentation/sphinx_static/err_vs_dt_field.svg
@@ -0,0 +1,1157 @@
+<?xml version="1.0" encoding="utf-8" standalone="no"?>
+<!DOCTYPE svg PUBLIC "-//W3C//DTD SVG 1.1//EN"
+ "http://www.w3.org/Graphics/SVG/1.1/DTD/svg11.dtd">
+<!-- Created with matplotlib (https://matplotlib.org/) -->
+<svg height="288pt" version="1.1" viewBox="0 0 288 288" width="288pt" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink">
+ <metadata>
+ <rdf:RDF xmlns:cc="http://creativecommons.org/ns#" xmlns:dc="http://purl.org/dc/elements/1.1/" xmlns:rdf="http://www.w3.org/1999/02/22-rdf-syntax-ns#">
+ <cc:Work>
+ <dc:type rdf:resource="http://purl.org/dc/dcmitype/StillImage"/>
+ <dc:date>2022-01-19T15:33:10.015620</dc:date>
+ <dc:format>image/svg+xml</dc:format>
+ <dc:creator>
+ <cc:Agent>
+ <dc:title>Matplotlib v3.3.4, https://matplotlib.org/</dc:title>
+ </cc:Agent>
+ </dc:creator>
+ </cc:Work>
+ </rdf:RDF>
+ </metadata>
+ <defs>
+ <style type="text/css">*{stroke-linecap:butt;stroke-linejoin:round;}</style>
+ </defs>
+ <g id="figure_1">
+ <g id="patch_1">
+ <path d="M 0 288
+L 288 288
+L 288 0
+L 0 0
+z
+" style="fill:#ffffff;"/>
+ </g>
+ <g id="axes_1">
+ <g id="patch_2">
+ <path d="M 55.970346 242.008
+L 268.914154 242.008
+L 268.914154 10.8
+L 55.970346 10.8
+z
+" style="fill:#ffffff;"/>
+ </g>
+ <g id="line2d_1">
+ <path clip-path="url(#p697d71626a)" d="M 65.64961 153.279364
+L 162.44225 231.498545
+" style="fill:none;stroke:#000000;stroke-dasharray:4.5,4.5;stroke-dashoffset:0;stroke-width:1.5;"/>
+ </g>
+ <g id="line2d_2">
+ <path clip-path="url(#p697d71626a)" d="M 65.64961 153.279364
+L 162.44225 205.425485
+" style="fill:none;stroke:#000000;stroke-dasharray:3,3;stroke-dashoffset:0;stroke-width:1.5;"/>
+ </g>
+ <g id="matplotlib.axis_1">
+ <g id="xtick_1">
+ <g id="line2d_3">
+ <defs>
+ <path d="M 0 0
+L 0 3.5
+" id="mc0115b44e6" style="stroke:#000000;stroke-width:0.8;"/>
+ </defs>
+ <g>
+ <use style="stroke:#000000;stroke-width:0.8;" x="65.64961" xlink:href="#mc0115b44e6" y="242.008"/>
+ </g>
+ </g>
+ <g id="text_1">
+ <!-- $\mathdefault{10^{0}}$ -->
+ <g transform="translate(56.84961 256.606437)scale(0.1 -0.1)">
+ <defs>
+ <path d="M 12.40625 8.296875
+L 28.515625 8.296875
+L 28.515625 63.921875
+L 10.984375 60.40625
+L 10.984375 69.390625
+L 28.421875 72.90625
+L 38.28125 72.90625
+L 38.28125 8.296875
+L 54.390625 8.296875
+L 54.390625 0
+L 12.40625 0
+z
+" id="DejaVuSans-49"/>
+ <path d="M 31.78125 66.40625
+Q 24.171875 66.40625 20.328125 58.90625
+Q 16.5 51.421875 16.5 36.375
+Q 16.5 21.390625 20.328125 13.890625
+Q 24.171875 6.390625 31.78125 6.390625
+Q 39.453125 6.390625 43.28125 13.890625
+Q 47.125 21.390625 47.125 36.375
+Q 47.125 51.421875 43.28125 58.90625
+Q 39.453125 66.40625 31.78125 66.40625
+z
+M 31.78125 74.21875
+Q 44.046875 74.21875 50.515625 64.515625
+Q 56.984375 54.828125 56.984375 36.375
+Q 56.984375 17.96875 50.515625 8.265625
+Q 44.046875 -1.421875 31.78125 -1.421875
+Q 19.53125 -1.421875 13.0625 8.265625
+Q 6.59375 17.96875 6.59375 36.375
+Q 6.59375 54.828125 13.0625 64.515625
+Q 19.53125 74.21875 31.78125 74.21875
+z
+" id="DejaVuSans-48"/>
+ </defs>
+ <use transform="translate(0 0.765625)" xlink:href="#DejaVuSans-49"/>
+ <use transform="translate(63.623047 0.765625)" xlink:href="#DejaVuSans-48"/>
+ <use transform="translate(128.203125 39.046875)scale(0.7)" xlink:href="#DejaVuSans-48"/>
+ </g>
+ </g>
+ </g>
+ <g id="xtick_2">
+ <g id="line2d_4">
+ <defs>
+ <path d="M 0 0
+L 0 2
+" id="ma73562a9e4" style="stroke:#000000;stroke-width:0.6;"/>
+ </defs>
+ <g>
+ <use style="stroke:#000000;stroke-width:0.6;" x="162.44225" xlink:href="#ma73562a9e4" y="242.008"/>
+ </g>
+ </g>
+ <g id="text_2">
+ <!-- $\mathdefault{2\times10^{0}}$ -->
+ <g transform="translate(144.34225 255.006438)scale(0.1 -0.1)">
+ <defs>
+ <path d="M 19.1875 8.296875
+L 53.609375 8.296875
+L 53.609375 0
+L 7.328125 0
+L 7.328125 8.296875
+Q 12.9375 14.109375 22.625 23.890625
+Q 32.328125 33.6875 34.8125 36.53125
+Q 39.546875 41.84375 41.421875 45.53125
+Q 43.3125 49.21875 43.3125 52.78125
+Q 43.3125 58.59375 39.234375 62.25
+Q 35.15625 65.921875 28.609375 65.921875
+Q 23.96875 65.921875 18.8125 64.3125
+Q 13.671875 62.703125 7.8125 59.421875
+L 7.8125 69.390625
+Q 13.765625 71.78125 18.9375 73
+Q 24.125 74.21875 28.421875 74.21875
+Q 39.75 74.21875 46.484375 68.546875
+Q 53.21875 62.890625 53.21875 53.421875
+Q 53.21875 48.921875 51.53125 44.890625
+Q 49.859375 40.875 45.40625 35.40625
+Q 44.1875 33.984375 37.640625 27.21875
+Q 31.109375 20.453125 19.1875 8.296875
+z
+" id="DejaVuSans-50"/>
+ <path d="M 70.125 53.71875
+L 47.796875 31.296875
+L 70.125 8.984375
+L 64.3125 3.078125
+L 41.890625 25.484375
+L 19.484375 3.078125
+L 13.71875 8.984375
+L 35.984375 31.296875
+L 13.71875 53.71875
+L 19.484375 59.625
+L 41.890625 37.203125
+L 64.3125 59.625
+z
+" id="DejaVuSans-215"/>
+ </defs>
+ <use transform="translate(0 0.765625)" xlink:href="#DejaVuSans-50"/>
+ <use transform="translate(83.105469 0.765625)" xlink:href="#DejaVuSans-215"/>
+ <use transform="translate(186.376953 0.765625)" xlink:href="#DejaVuSans-49"/>
+ <use transform="translate(250 0.765625)" xlink:href="#DejaVuSans-48"/>
+ <use transform="translate(314.580078 39.046875)scale(0.7)" xlink:href="#DejaVuSans-48"/>
+ </g>
+ </g>
+ </g>
+ <g id="xtick_3">
+ <g id="line2d_5">
+ <g>
+ <use style="stroke:#000000;stroke-width:0.6;" x="219.062315" xlink:href="#ma73562a9e4" y="242.008"/>
+ </g>
+ </g>
+ <g id="text_3">
+ <!-- $\mathdefault{3\times10^{0}}$ -->
+ <g transform="translate(200.962315 255.006438)scale(0.1 -0.1)">
+ <defs>
+ <path d="M 40.578125 39.3125
+Q 47.65625 37.796875 51.625 33
+Q 55.609375 28.21875 55.609375 21.1875
+Q 55.609375 10.40625 48.1875 4.484375
+Q 40.765625 -1.421875 27.09375 -1.421875
+Q 22.515625 -1.421875 17.65625 -0.515625
+Q 12.796875 0.390625 7.625 2.203125
+L 7.625 11.71875
+Q 11.71875 9.328125 16.59375 8.109375
+Q 21.484375 6.890625 26.8125 6.890625
+Q 36.078125 6.890625 40.9375 10.546875
+Q 45.796875 14.203125 45.796875 21.1875
+Q 45.796875 27.640625 41.28125 31.265625
+Q 36.765625 34.90625 28.71875 34.90625
+L 20.21875 34.90625
+L 20.21875 43.015625
+L 29.109375 43.015625
+Q 36.375 43.015625 40.234375 45.921875
+Q 44.09375 48.828125 44.09375 54.296875
+Q 44.09375 59.90625 40.109375 62.90625
+Q 36.140625 65.921875 28.71875 65.921875
+Q 24.65625 65.921875 20.015625 65.03125
+Q 15.375 64.15625 9.8125 62.3125
+L 9.8125 71.09375
+Q 15.4375 72.65625 20.34375 73.4375
+Q 25.25 74.21875 29.59375 74.21875
+Q 40.828125 74.21875 47.359375 69.109375
+Q 53.90625 64.015625 53.90625 55.328125
+Q 53.90625 49.265625 50.4375 45.09375
+Q 46.96875 40.921875 40.578125 39.3125
+z
+" id="DejaVuSans-51"/>
+ </defs>
+ <use transform="translate(0 0.765625)" xlink:href="#DejaVuSans-51"/>
+ <use transform="translate(83.105469 0.765625)" xlink:href="#DejaVuSans-215"/>
+ <use transform="translate(186.376953 0.765625)" xlink:href="#DejaVuSans-49"/>
+ <use transform="translate(250 0.765625)" xlink:href="#DejaVuSans-48"/>
+ <use transform="translate(314.580078 39.046875)scale(0.7)" xlink:href="#DejaVuSans-48"/>
+ </g>
+ </g>
+ </g>
+ <g id="xtick_4">
+ <g id="line2d_6">
+ <g>
+ <use style="stroke:#000000;stroke-width:0.6;" x="259.23489" xlink:href="#ma73562a9e4" y="242.008"/>
+ </g>
+ </g>
+ <g id="text_4">
+ <!-- $\mathdefault{4\times10^{0}}$ -->
+ <g transform="translate(241.13489 255.006438)scale(0.1 -0.1)">
+ <defs>
+ <path d="M 37.796875 64.3125
+L 12.890625 25.390625
+L 37.796875 25.390625
+z
+M 35.203125 72.90625
+L 47.609375 72.90625
+L 47.609375 25.390625
+L 58.015625 25.390625
+L 58.015625 17.1875
+L 47.609375 17.1875
+L 47.609375 0
+L 37.796875 0
+L 37.796875 17.1875
+L 4.890625 17.1875
+L 4.890625 26.703125
+z
+" id="DejaVuSans-52"/>
+ </defs>
+ <use transform="translate(0 0.765625)" xlink:href="#DejaVuSans-52"/>
+ <use transform="translate(83.105469 0.765625)" xlink:href="#DejaVuSans-215"/>
+ <use transform="translate(186.376953 0.765625)" xlink:href="#DejaVuSans-49"/>
+ <use transform="translate(250 0.765625)" xlink:href="#DejaVuSans-48"/>
+ <use transform="translate(314.580078 39.046875)scale(0.7)" xlink:href="#DejaVuSans-48"/>
+ </g>
+ </g>
+ </g>
+ <g id="text_5">
+ <!-- resolution factor $f$ -->
+ <g transform="translate(117.89225 270.284562)scale(0.1 -0.1)">
+ <defs>
+ <path d="M 41.109375 46.296875
+Q 39.59375 47.171875 37.8125 47.578125
+Q 36.03125 48 33.890625 48
+Q 26.265625 48 22.1875 43.046875
+Q 18.109375 38.09375 18.109375 28.8125
+L 18.109375 0
+L 9.078125 0
+L 9.078125 54.6875
+L 18.109375 54.6875
+L 18.109375 46.1875
+Q 20.953125 51.171875 25.484375 53.578125
+Q 30.03125 56 36.53125 56
+Q 37.453125 56 38.578125 55.875
+Q 39.703125 55.765625 41.0625 55.515625
+z
+" id="DejaVuSans-114"/>
+ <path d="M 56.203125 29.59375
+L 56.203125 25.203125
+L 14.890625 25.203125
+Q 15.484375 15.921875 20.484375 11.0625
+Q 25.484375 6.203125 34.421875 6.203125
+Q 39.59375 6.203125 44.453125 7.46875
+Q 49.3125 8.734375 54.109375 11.28125
+L 54.109375 2.78125
+Q 49.265625 0.734375 44.1875 -0.34375
+Q 39.109375 -1.421875 33.890625 -1.421875
+Q 20.796875 -1.421875 13.15625 6.1875
+Q 5.515625 13.8125 5.515625 26.8125
+Q 5.515625 40.234375 12.765625 48.109375
+Q 20.015625 56 32.328125 56
+Q 43.359375 56 49.78125 48.890625
+Q 56.203125 41.796875 56.203125 29.59375
+z
+M 47.21875 32.234375
+Q 47.125 39.59375 43.09375 43.984375
+Q 39.0625 48.390625 32.421875 48.390625
+Q 24.90625 48.390625 20.390625 44.140625
+Q 15.875 39.890625 15.1875 32.171875
+z
+" id="DejaVuSans-101"/>
+ <path d="M 44.28125 53.078125
+L 44.28125 44.578125
+Q 40.484375 46.53125 36.375 47.5
+Q 32.28125 48.484375 27.875 48.484375
+Q 21.1875 48.484375 17.84375 46.4375
+Q 14.5 44.390625 14.5 40.28125
+Q 14.5 37.15625 16.890625 35.375
+Q 19.28125 33.59375 26.515625 31.984375
+L 29.59375 31.296875
+Q 39.15625 29.25 43.1875 25.515625
+Q 47.21875 21.78125 47.21875 15.09375
+Q 47.21875 7.46875 41.1875 3.015625
+Q 35.15625 -1.421875 24.609375 -1.421875
+Q 20.21875 -1.421875 15.453125 -0.5625
+Q 10.6875 0.296875 5.421875 2
+L 5.421875 11.28125
+Q 10.40625 8.6875 15.234375 7.390625
+Q 20.0625 6.109375 24.8125 6.109375
+Q 31.15625 6.109375 34.5625 8.28125
+Q 37.984375 10.453125 37.984375 14.40625
+Q 37.984375 18.0625 35.515625 20.015625
+Q 33.0625 21.96875 24.703125 23.78125
+L 21.578125 24.515625
+Q 13.234375 26.265625 9.515625 29.90625
+Q 5.8125 33.546875 5.8125 39.890625
+Q 5.8125 47.609375 11.28125 51.796875
+Q 16.75 56 26.8125 56
+Q 31.78125 56 36.171875 55.265625
+Q 40.578125 54.546875 44.28125 53.078125
+z
+" id="DejaVuSans-115"/>
+ <path d="M 30.609375 48.390625
+Q 23.390625 48.390625 19.1875 42.75
+Q 14.984375 37.109375 14.984375 27.296875
+Q 14.984375 17.484375 19.15625 11.84375
+Q 23.34375 6.203125 30.609375 6.203125
+Q 37.796875 6.203125 41.984375 11.859375
+Q 46.1875 17.53125 46.1875 27.296875
+Q 46.1875 37.015625 41.984375 42.703125
+Q 37.796875 48.390625 30.609375 48.390625
+z
+M 30.609375 56
+Q 42.328125 56 49.015625 48.375
+Q 55.71875 40.765625 55.71875 27.296875
+Q 55.71875 13.875 49.015625 6.21875
+Q 42.328125 -1.421875 30.609375 -1.421875
+Q 18.84375 -1.421875 12.171875 6.21875
+Q 5.515625 13.875 5.515625 27.296875
+Q 5.515625 40.765625 12.171875 48.375
+Q 18.84375 56 30.609375 56
+z
+" id="DejaVuSans-111"/>
+ <path d="M 9.421875 75.984375
+L 18.40625 75.984375
+L 18.40625 0
+L 9.421875 0
+z
+" id="DejaVuSans-108"/>
+ <path d="M 8.5 21.578125
+L 8.5 54.6875
+L 17.484375 54.6875
+L 17.484375 21.921875
+Q 17.484375 14.15625 20.5 10.265625
+Q 23.53125 6.390625 29.59375 6.390625
+Q 36.859375 6.390625 41.078125 11.03125
+Q 45.3125 15.671875 45.3125 23.6875
+L 45.3125 54.6875
+L 54.296875 54.6875
+L 54.296875 0
+L 45.3125 0
+L 45.3125 8.40625
+Q 42.046875 3.421875 37.71875 1
+Q 33.40625 -1.421875 27.6875 -1.421875
+Q 18.265625 -1.421875 13.375 4.4375
+Q 8.5 10.296875 8.5 21.578125
+z
+M 31.109375 56
+z
+" id="DejaVuSans-117"/>
+ <path d="M 18.3125 70.21875
+L 18.3125 54.6875
+L 36.8125 54.6875
+L 36.8125 47.703125
+L 18.3125 47.703125
+L 18.3125 18.015625
+Q 18.3125 11.328125 20.140625 9.421875
+Q 21.96875 7.515625 27.59375 7.515625
+L 36.8125 7.515625
+L 36.8125 0
+L 27.59375 0
+Q 17.1875 0 13.234375 3.875
+Q 9.28125 7.765625 9.28125 18.015625
+L 9.28125 47.703125
+L 2.6875 47.703125
+L 2.6875 54.6875
+L 9.28125 54.6875
+L 9.28125 70.21875
+z
+" id="DejaVuSans-116"/>
+ <path d="M 9.421875 54.6875
+L 18.40625 54.6875
+L 18.40625 0
+L 9.421875 0
+z
+M 9.421875 75.984375
+L 18.40625 75.984375
+L 18.40625 64.59375
+L 9.421875 64.59375
+z
+" id="DejaVuSans-105"/>
+ <path d="M 54.890625 33.015625
+L 54.890625 0
+L 45.90625 0
+L 45.90625 32.71875
+Q 45.90625 40.484375 42.875 44.328125
+Q 39.84375 48.1875 33.796875 48.1875
+Q 26.515625 48.1875 22.3125 43.546875
+Q 18.109375 38.921875 18.109375 30.90625
+L 18.109375 0
+L 9.078125 0
+L 9.078125 54.6875
+L 18.109375 54.6875
+L 18.109375 46.1875
+Q 21.34375 51.125 25.703125 53.5625
+Q 30.078125 56 35.796875 56
+Q 45.21875 56 50.046875 50.171875
+Q 54.890625 44.34375 54.890625 33.015625
+z
+" id="DejaVuSans-110"/>
+ <path id="DejaVuSans-32"/>
+ <path d="M 37.109375 75.984375
+L 37.109375 68.5
+L 28.515625 68.5
+Q 23.6875 68.5 21.796875 66.546875
+Q 19.921875 64.59375 19.921875 59.515625
+L 19.921875 54.6875
+L 34.71875 54.6875
+L 34.71875 47.703125
+L 19.921875 47.703125
+L 19.921875 0
+L 10.890625 0
+L 10.890625 47.703125
+L 2.296875 47.703125
+L 2.296875 54.6875
+L 10.890625 54.6875
+L 10.890625 58.5
+Q 10.890625 67.625 15.140625 71.796875
+Q 19.390625 75.984375 28.609375 75.984375
+z
+" id="DejaVuSans-102"/>
+ <path d="M 34.28125 27.484375
+Q 23.390625 27.484375 19.1875 25
+Q 14.984375 22.515625 14.984375 16.5
+Q 14.984375 11.71875 18.140625 8.90625
+Q 21.296875 6.109375 26.703125 6.109375
+Q 34.1875 6.109375 38.703125 11.40625
+Q 43.21875 16.703125 43.21875 25.484375
+L 43.21875 27.484375
+z
+M 52.203125 31.203125
+L 52.203125 0
+L 43.21875 0
+L 43.21875 8.296875
+Q 40.140625 3.328125 35.546875 0.953125
+Q 30.953125 -1.421875 24.3125 -1.421875
+Q 15.921875 -1.421875 10.953125 3.296875
+Q 6 8.015625 6 15.921875
+Q 6 25.140625 12.171875 29.828125
+Q 18.359375 34.515625 30.609375 34.515625
+L 43.21875 34.515625
+L 43.21875 35.40625
+Q 43.21875 41.609375 39.140625 45
+Q 35.0625 48.390625 27.6875 48.390625
+Q 23 48.390625 18.546875 47.265625
+Q 14.109375 46.140625 10.015625 43.890625
+L 10.015625 52.203125
+Q 14.9375 54.109375 19.578125 55.046875
+Q 24.21875 56 28.609375 56
+Q 40.484375 56 46.34375 49.84375
+Q 52.203125 43.703125 52.203125 31.203125
+z
+" id="DejaVuSans-97"/>
+ <path d="M 48.78125 52.59375
+L 48.78125 44.1875
+Q 44.96875 46.296875 41.140625 47.34375
+Q 37.3125 48.390625 33.40625 48.390625
+Q 24.65625 48.390625 19.8125 42.84375
+Q 14.984375 37.3125 14.984375 27.296875
+Q 14.984375 17.28125 19.8125 11.734375
+Q 24.65625 6.203125 33.40625 6.203125
+Q 37.3125 6.203125 41.140625 7.25
+Q 44.96875 8.296875 48.78125 10.40625
+L 48.78125 2.09375
+Q 45.015625 0.34375 40.984375 -0.53125
+Q 36.96875 -1.421875 32.421875 -1.421875
+Q 20.0625 -1.421875 12.78125 6.34375
+Q 5.515625 14.109375 5.515625 27.296875
+Q 5.515625 40.671875 12.859375 48.328125
+Q 20.21875 56 33.015625 56
+Q 37.15625 56 41.109375 55.140625
+Q 45.0625 54.296875 48.78125 52.59375
+z
+" id="DejaVuSans-99"/>
+ <path d="M 47.796875 75.984375
+L 46.390625 68.5
+L 37.796875 68.5
+Q 32.90625 68.5 30.6875 66.578125
+Q 28.46875 64.65625 27.390625 59.515625
+L 26.421875 54.6875
+L 41.21875 54.6875
+L 39.890625 47.703125
+L 25.09375 47.703125
+L 15.828125 0
+L 6.78125 0
+L 16.109375 47.703125
+L 7.515625 47.703125
+L 8.796875 54.6875
+L 17.390625 54.6875
+L 18.109375 58.5
+Q 19.96875 68.171875 24.625 72.078125
+Q 29.296875 75.984375 39.3125 75.984375
+z
+" id="DejaVuSans-Oblique-102"/>
+ </defs>
+ <use transform="translate(0 0.015625)" xlink:href="#DejaVuSans-114"/>
+ <use transform="translate(41.113281 0.015625)" xlink:href="#DejaVuSans-101"/>
+ <use transform="translate(102.636719 0.015625)" xlink:href="#DejaVuSans-115"/>
+ <use transform="translate(154.736328 0.015625)" xlink:href="#DejaVuSans-111"/>
+ <use transform="translate(215.917969 0.015625)" xlink:href="#DejaVuSans-108"/>
+ <use transform="translate(243.701172 0.015625)" xlink:href="#DejaVuSans-117"/>
+ <use transform="translate(307.080078 0.015625)" xlink:href="#DejaVuSans-116"/>
+ <use transform="translate(346.289062 0.015625)" xlink:href="#DejaVuSans-105"/>
+ <use transform="translate(374.072266 0.015625)" xlink:href="#DejaVuSans-111"/>
+ <use transform="translate(435.253906 0.015625)" xlink:href="#DejaVuSans-110"/>
+ <use transform="translate(498.632812 0.015625)" xlink:href="#DejaVuSans-32"/>
+ <use transform="translate(530.419922 0.015625)" xlink:href="#DejaVuSans-102"/>
+ <use transform="translate(565.625 0.015625)" xlink:href="#DejaVuSans-97"/>
+ <use transform="translate(626.904297 0.015625)" xlink:href="#DejaVuSans-99"/>
+ <use transform="translate(681.884766 0.015625)" xlink:href="#DejaVuSans-116"/>
+ <use transform="translate(721.09375 0.015625)" xlink:href="#DejaVuSans-111"/>
+ <use transform="translate(782.275391 0.015625)" xlink:href="#DejaVuSans-114"/>
+ <use transform="translate(823.388672 0.015625)" xlink:href="#DejaVuSans-32"/>
+ <use transform="translate(855.175781 0.015625)" xlink:href="#DejaVuSans-Oblique-102"/>
+ </g>
+ </g>
+ </g>
+ <g id="matplotlib.axis_2">
+ <g id="ytick_1">
+ <g id="line2d_7">
+ <defs>
+ <path d="M 0 0
+L -3.5 0
+" id="mf67fbfa108" style="stroke:#000000;stroke-width:0.8;"/>
+ </defs>
+ <g>
+ <use style="stroke:#000000;stroke-width:0.8;" x="55.970346" xlink:href="#mf67fbfa108" y="237.429866"/>
+ </g>
+ </g>
+ <g id="text_6">
+ <!-- $\mathdefault{10^{-7}}$ -->
+ <g transform="translate(25.470346 241.229085)scale(0.1 -0.1)">
+ <defs>
+ <path d="M 10.59375 35.5
+L 73.1875 35.5
+L 73.1875 27.203125
+L 10.59375 27.203125
+z
+" id="DejaVuSans-8722"/>
+ <path d="M 8.203125 72.90625
+L 55.078125 72.90625
+L 55.078125 68.703125
+L 28.609375 0
+L 18.3125 0
+L 43.21875 64.59375
+L 8.203125 64.59375
+z
+" id="DejaVuSans-55"/>
+ </defs>
+ <use transform="translate(0 0.684375)" xlink:href="#DejaVuSans-49"/>
+ <use transform="translate(63.623047 0.684375)" xlink:href="#DejaVuSans-48"/>
+ <use transform="translate(128.203125 38.965625)scale(0.7)" xlink:href="#DejaVuSans-8722"/>
+ <use transform="translate(186.855469 38.965625)scale(0.7)" xlink:href="#DejaVuSans-55"/>
+ </g>
+ </g>
+ </g>
+ <g id="ytick_2">
+ <g id="line2d_8">
+ <g>
+ <use style="stroke:#000000;stroke-width:0.8;" x="55.970346" xlink:href="#mf67fbfa108" y="150.817035"/>
+ </g>
+ </g>
+ <g id="text_7">
+ <!-- $\mathdefault{10^{-6}}$ -->
+ <g transform="translate(25.470346 154.616254)scale(0.1 -0.1)">
+ <defs>
+ <path d="M 33.015625 40.375
+Q 26.375 40.375 22.484375 35.828125
+Q 18.609375 31.296875 18.609375 23.390625
+Q 18.609375 15.53125 22.484375 10.953125
+Q 26.375 6.390625 33.015625 6.390625
+Q 39.65625 6.390625 43.53125 10.953125
+Q 47.40625 15.53125 47.40625 23.390625
+Q 47.40625 31.296875 43.53125 35.828125
+Q 39.65625 40.375 33.015625 40.375
+z
+M 52.59375 71.296875
+L 52.59375 62.3125
+Q 48.875 64.0625 45.09375 64.984375
+Q 41.3125 65.921875 37.59375 65.921875
+Q 27.828125 65.921875 22.671875 59.328125
+Q 17.53125 52.734375 16.796875 39.40625
+Q 19.671875 43.65625 24.015625 45.921875
+Q 28.375 48.1875 33.59375 48.1875
+Q 44.578125 48.1875 50.953125 41.515625
+Q 57.328125 34.859375 57.328125 23.390625
+Q 57.328125 12.15625 50.6875 5.359375
+Q 44.046875 -1.421875 33.015625 -1.421875
+Q 20.359375 -1.421875 13.671875 8.265625
+Q 6.984375 17.96875 6.984375 36.375
+Q 6.984375 53.65625 15.1875 63.9375
+Q 23.390625 74.21875 37.203125 74.21875
+Q 40.921875 74.21875 44.703125 73.484375
+Q 48.484375 72.75 52.59375 71.296875
+z
+" id="DejaVuSans-54"/>
+ </defs>
+ <use transform="translate(0 0.765625)" xlink:href="#DejaVuSans-49"/>
+ <use transform="translate(63.623047 0.765625)" xlink:href="#DejaVuSans-48"/>
+ <use transform="translate(128.203125 39.046875)scale(0.7)" xlink:href="#DejaVuSans-8722"/>
+ <use transform="translate(186.855469 39.046875)scale(0.7)" xlink:href="#DejaVuSans-54"/>
+ </g>
+ </g>
+ </g>
+ <g id="ytick_3">
+ <g id="line2d_9">
+ <g>
+ <use style="stroke:#000000;stroke-width:0.8;" x="55.970346" xlink:href="#mf67fbfa108" y="64.204203"/>
+ </g>
+ </g>
+ <g id="text_8">
+ <!-- $\mathdefault{10^{-5}}$ -->
+ <g transform="translate(25.470346 68.003422)scale(0.1 -0.1)">
+ <defs>
+ <path d="M 10.796875 72.90625
+L 49.515625 72.90625
+L 49.515625 64.59375
+L 19.828125 64.59375
+L 19.828125 46.734375
+Q 21.96875 47.46875 24.109375 47.828125
+Q 26.265625 48.1875 28.421875 48.1875
+Q 40.625 48.1875 47.75 41.5
+Q 54.890625 34.8125 54.890625 23.390625
+Q 54.890625 11.625 47.5625 5.09375
+Q 40.234375 -1.421875 26.90625 -1.421875
+Q 22.3125 -1.421875 17.546875 -0.640625
+Q 12.796875 0.140625 7.71875 1.703125
+L 7.71875 11.625
+Q 12.109375 9.234375 16.796875 8.0625
+Q 21.484375 6.890625 26.703125 6.890625
+Q 35.15625 6.890625 40.078125 11.328125
+Q 45.015625 15.765625 45.015625 23.390625
+Q 45.015625 31 40.078125 35.4375
+Q 35.15625 39.890625 26.703125 39.890625
+Q 22.75 39.890625 18.8125 39.015625
+Q 14.890625 38.140625 10.796875 36.28125
+z
+" id="DejaVuSans-53"/>
+ </defs>
+ <use transform="translate(0 0.684375)" xlink:href="#DejaVuSans-49"/>
+ <use transform="translate(63.623047 0.684375)" xlink:href="#DejaVuSans-48"/>
+ <use transform="translate(128.203125 38.965625)scale(0.7)" xlink:href="#DejaVuSans-8722"/>
+ <use transform="translate(186.855469 38.965625)scale(0.7)" xlink:href="#DejaVuSans-53"/>
+ </g>
+ </g>
+ </g>
+ <g id="ytick_4">
+ <g id="line2d_10">
+ <defs>
+ <path d="M 0 0
+L -2 0
+" id="me29b2c3ebe" style="stroke:#000000;stroke-width:0.6;"/>
+ </defs>
+ <g>
+ <use style="stroke:#000000;stroke-width:0.6;" x="55.970346" xlink:href="#me29b2c3ebe" y="241.393052"/>
+ </g>
+ </g>
+ </g>
+ <g id="ytick_5">
+ <g id="line2d_11">
+ <g>
+ <use style="stroke:#000000;stroke-width:0.6;" x="55.970346" xlink:href="#me29b2c3ebe" y="211.356806"/>
+ </g>
+ </g>
+ </g>
+ <g id="ytick_6">
+ <g id="line2d_12">
+ <g>
+ <use style="stroke:#000000;stroke-width:0.6;" x="55.970346" xlink:href="#me29b2c3ebe" y="196.105044"/>
+ </g>
+ </g>
+ </g>
+ <g id="ytick_7">
+ <g id="line2d_13">
+ <g>
+ <use style="stroke:#000000;stroke-width:0.6;" x="55.970346" xlink:href="#me29b2c3ebe" y="185.283746"/>
+ </g>
+ </g>
+ </g>
+ <g id="ytick_8">
+ <g id="line2d_14">
+ <g>
+ <use style="stroke:#000000;stroke-width:0.6;" x="55.970346" xlink:href="#me29b2c3ebe" y="176.890095"/>
+ </g>
+ </g>
+ </g>
+ <g id="ytick_9">
+ <g id="line2d_15">
+ <g>
+ <use style="stroke:#000000;stroke-width:0.6;" x="55.970346" xlink:href="#me29b2c3ebe" y="170.031983"/>
+ </g>
+ </g>
+ </g>
+ <g id="ytick_10">
+ <g id="line2d_16">
+ <g>
+ <use style="stroke:#000000;stroke-width:0.6;" x="55.970346" xlink:href="#me29b2c3ebe" y="164.233532"/>
+ </g>
+ </g>
+ </g>
+ <g id="ytick_11">
+ <g id="line2d_17">
+ <g>
+ <use style="stroke:#000000;stroke-width:0.6;" x="55.970346" xlink:href="#me29b2c3ebe" y="159.210685"/>
+ </g>
+ </g>
+ </g>
+ <g id="ytick_12">
+ <g id="line2d_18">
+ <g>
+ <use style="stroke:#000000;stroke-width:0.6;" x="55.970346" xlink:href="#me29b2c3ebe" y="154.780221"/>
+ </g>
+ </g>
+ </g>
+ <g id="ytick_13">
+ <g id="line2d_19">
+ <g>
+ <use style="stroke:#000000;stroke-width:0.6;" x="55.970346" xlink:href="#me29b2c3ebe" y="124.743974"/>
+ </g>
+ </g>
+ </g>
+ <g id="ytick_14">
+ <g id="line2d_20">
+ <g>
+ <use style="stroke:#000000;stroke-width:0.6;" x="55.970346" xlink:href="#me29b2c3ebe" y="109.492212"/>
+ </g>
+ </g>
+ </g>
+ <g id="ytick_15">
+ <g id="line2d_21">
+ <g>
+ <use style="stroke:#000000;stroke-width:0.6;" x="55.970346" xlink:href="#me29b2c3ebe" y="98.670914"/>
+ </g>
+ </g>
+ </g>
+ <g id="ytick_16">
+ <g id="line2d_22">
+ <g>
+ <use style="stroke:#000000;stroke-width:0.6;" x="55.970346" xlink:href="#me29b2c3ebe" y="90.277263"/>
+ </g>
+ </g>
+ </g>
+ <g id="ytick_17">
+ <g id="line2d_23">
+ <g>
+ <use style="stroke:#000000;stroke-width:0.6;" x="55.970346" xlink:href="#me29b2c3ebe" y="83.419152"/>
+ </g>
+ </g>
+ </g>
+ <g id="ytick_18">
+ <g id="line2d_24">
+ <g>
+ <use style="stroke:#000000;stroke-width:0.6;" x="55.970346" xlink:href="#me29b2c3ebe" y="77.6207"/>
+ </g>
+ </g>
+ </g>
+ <g id="ytick_19">
+ <g id="line2d_25">
+ <g>
+ <use style="stroke:#000000;stroke-width:0.6;" x="55.970346" xlink:href="#me29b2c3ebe" y="72.597854"/>
+ </g>
+ </g>
+ </g>
+ <g id="ytick_20">
+ <g id="line2d_26">
+ <g>
+ <use style="stroke:#000000;stroke-width:0.6;" x="55.970346" xlink:href="#me29b2c3ebe" y="68.167389"/>
+ </g>
+ </g>
+ </g>
+ <g id="ytick_21">
+ <g id="line2d_27">
+ <g>
+ <use style="stroke:#000000;stroke-width:0.6;" x="55.970346" xlink:href="#me29b2c3ebe" y="38.131143"/>
+ </g>
+ </g>
+ </g>
+ <g id="ytick_22">
+ <g id="line2d_28">
+ <g>
+ <use style="stroke:#000000;stroke-width:0.6;" x="55.970346" xlink:href="#me29b2c3ebe" y="22.87938"/>
+ </g>
+ </g>
+ </g>
+ <g id="ytick_23">
+ <g id="line2d_29">
+ <g>
+ <use style="stroke:#000000;stroke-width:0.6;" x="55.970346" xlink:href="#me29b2c3ebe" y="12.058082"/>
+ </g>
+ </g>
+ </g>
+ <g id="text_9">
+ <!-- relative error -->
+ <g transform="translate(19.390659 158.953219)rotate(-90)scale(0.1 -0.1)">
+ <defs>
+ <path d="M 2.984375 54.6875
+L 12.5 54.6875
+L 29.59375 8.796875
+L 46.6875 54.6875
+L 56.203125 54.6875
+L 35.6875 0
+L 23.484375 0
+z
+" id="DejaVuSans-118"/>
+ </defs>
+ <use xlink:href="#DejaVuSans-114"/>
+ <use x="38.863281" xlink:href="#DejaVuSans-101"/>
+ <use x="100.386719" xlink:href="#DejaVuSans-108"/>
+ <use x="128.169922" xlink:href="#DejaVuSans-97"/>
+ <use x="189.449219" xlink:href="#DejaVuSans-116"/>
+ <use x="228.658203" xlink:href="#DejaVuSans-105"/>
+ <use x="256.441406" xlink:href="#DejaVuSans-118"/>
+ <use x="315.621094" xlink:href="#DejaVuSans-101"/>
+ <use x="377.144531" xlink:href="#DejaVuSans-32"/>
+ <use x="408.931641" xlink:href="#DejaVuSans-101"/>
+ <use x="470.455078" xlink:href="#DejaVuSans-114"/>
+ <use x="509.818359" xlink:href="#DejaVuSans-114"/>
+ <use x="548.681641" xlink:href="#DejaVuSans-111"/>
+ <use x="609.863281" xlink:href="#DejaVuSans-114"/>
+ </g>
+ </g>
+ </g>
+ <g id="line2d_30">
+ <path clip-path="url(#p697d71626a)" d="M 65.64961 153.279364
+L 162.44225 231.485234
+" style="fill:none;stroke:#1f77b4;stroke-dasharray:3,4.5;stroke-dashoffset:0;stroke-width:1.5;"/>
+ <defs>
+ <path d="M 0 1.5
+C 0.397805 1.5 0.77937 1.341951 1.06066 1.06066
+C 1.341951 0.77937 1.5 0.397805 1.5 0
+C 1.5 -0.397805 1.341951 -0.77937 1.06066 -1.06066
+C 0.77937 -1.341951 0.397805 -1.5 0 -1.5
+C -0.397805 -1.5 -0.77937 -1.341951 -1.06066 -1.06066
+C -1.341951 -0.77937 -1.5 -0.397805 -1.5 0
+C -1.5 0.397805 -1.341951 0.77937 -1.06066 1.06066
+C -0.77937 1.341951 -0.397805 1.5 0 1.5
+z
+" id="m82b2a5bbca" style="stroke:#1f77b4;"/>
+ </defs>
+ <g clip-path="url(#p697d71626a)">
+ <use style="fill:#1f77b4;stroke:#1f77b4;" x="65.64961" xlink:href="#m82b2a5bbca" y="153.279364"/>
+ <use style="fill:#1f77b4;stroke:#1f77b4;" x="162.44225" xlink:href="#m82b2a5bbca" y="231.485234"/>
+ </g>
+ </g>
+ <g id="line2d_31">
+ <path clip-path="url(#p697d71626a)" d="M 65.64961 153.279119
+L 162.44225 231.484979
+" style="fill:none;stroke:#ff7f0e;stroke-dasharray:4.5,7.5;stroke-dashoffset:0;stroke-width:1.5;"/>
+ <defs>
+ <path d="M 0 1.5
+C 0.397805 1.5 0.77937 1.341951 1.06066 1.06066
+C 1.341951 0.77937 1.5 0.397805 1.5 0
+C 1.5 -0.397805 1.341951 -0.77937 1.06066 -1.06066
+C 0.77937 -1.341951 0.397805 -1.5 0 -1.5
+C -0.397805 -1.5 -0.77937 -1.341951 -1.06066 -1.06066
+C -1.341951 -0.77937 -1.5 -0.397805 -1.5 0
+C -1.5 0.397805 -1.341951 0.77937 -1.06066 1.06066
+C -0.77937 1.341951 -0.397805 1.5 0 1.5
+z
+" id="m4d39af77bf" style="stroke:#ff7f0e;"/>
+ </defs>
+ <g clip-path="url(#p697d71626a)">
+ <use style="fill:#ff7f0e;stroke:#ff7f0e;" x="65.64961" xlink:href="#m4d39af77bf" y="153.279119"/>
+ <use style="fill:#ff7f0e;stroke:#ff7f0e;" x="162.44225" xlink:href="#m4d39af77bf" y="231.484979"/>
+ </g>
+ </g>
+ <g id="line2d_32">
+ <path clip-path="url(#p697d71626a)" d="M 65.64961 21.309455
+L 162.44225 21.310486
+L 259.23489 21.310609
+" style="fill:none;stroke:#2ca02c;stroke-linecap:square;stroke-width:1.5;"/>
+ <defs>
+ <path d="M 0 1.5
+C 0.397805 1.5 0.77937 1.341951 1.06066 1.06066
+C 1.341951 0.77937 1.5 0.397805 1.5 0
+C 1.5 -0.397805 1.341951 -0.77937 1.06066 -1.06066
+C 0.77937 -1.341951 0.397805 -1.5 0 -1.5
+C -0.397805 -1.5 -0.77937 -1.341951 -1.06066 -1.06066
+C -1.341951 -0.77937 -1.5 -0.397805 -1.5 0
+C -1.5 0.397805 -1.341951 0.77937 -1.06066 1.06066
+C -0.77937 1.341951 -0.397805 1.5 0 1.5
+z
+" id="m66f4880a7e" style="stroke:#2ca02c;"/>
+ </defs>
+ <g clip-path="url(#p697d71626a)">
+ <use style="fill:#2ca02c;stroke:#2ca02c;" x="65.64961" xlink:href="#m66f4880a7e" y="21.309455"/>
+ <use style="fill:#2ca02c;stroke:#2ca02c;" x="162.44225" xlink:href="#m66f4880a7e" y="21.310486"/>
+ <use style="fill:#2ca02c;stroke:#2ca02c;" x="259.23489" xlink:href="#m66f4880a7e" y="21.310609"/>
+ </g>
+ </g>
+ <g id="patch_3">
+ <path d="M 55.970346 242.008
+L 55.970346 10.8
+" style="fill:none;stroke:#000000;stroke-linecap:square;stroke-linejoin:miter;stroke-width:0.8;"/>
+ </g>
+ <g id="patch_4">
+ <path d="M 268.914154 242.008
+L 268.914154 10.8
+" style="fill:none;stroke:#000000;stroke-linecap:square;stroke-linejoin:miter;stroke-width:0.8;"/>
+ </g>
+ <g id="patch_5">
+ <path d="M 55.970346 242.008
+L 268.914154 242.008
+" style="fill:none;stroke:#000000;stroke-linecap:square;stroke-linejoin:miter;stroke-width:0.8;"/>
+ </g>
+ <g id="patch_6">
+ <path d="M 55.970346 10.8
+L 268.914154 10.8
+" style="fill:none;stroke:#000000;stroke-linecap:square;stroke-linejoin:miter;stroke-width:0.8;"/>
+ </g>
+ <g id="legend_1">
+ <g id="patch_7">
+ <path d="M 206.741029 239.008
+L 264.714154 239.008
+Q 265.914154 239.008 265.914154 237.808
+L 265.914154 194.373625
+Q 265.914154 193.173625 264.714154 193.173625
+L 206.741029 193.173625
+Q 205.541029 193.173625 205.541029 194.373625
+L 205.541029 237.808
+Q 205.541029 239.008 206.741029 239.008
+z
+" style="fill:#ffffff;opacity:0.8;stroke:#cccccc;stroke-linejoin:miter;"/>
+ </g>
+ <g id="line2d_33">
+ <path d="M 207.941029 198.032688
+L 219.941029 198.032688
+" style="fill:none;stroke:#1f77b4;stroke-dasharray:3,4.5;stroke-dashoffset:0;stroke-width:1.5;"/>
+ </g>
+ <g id="line2d_34">
+ <g>
+ <use style="fill:#1f77b4;stroke:#1f77b4;" x="213.941029" xlink:href="#m82b2a5bbca" y="198.032688"/>
+ </g>
+ </g>
+ <g id="text_10">
+ <!-- NSVE -->
+ <g transform="translate(224.741029 200.132688)scale(0.06 -0.06)">
+ <defs>
+ <path d="M 9.8125 72.90625
+L 23.09375 72.90625
+L 55.421875 11.921875
+L 55.421875 72.90625
+L 64.984375 72.90625
+L 64.984375 0
+L 51.703125 0
+L 19.390625 60.984375
+L 19.390625 0
+L 9.8125 0
+z
+" id="DejaVuSans-78"/>
+ <path d="M 53.515625 70.515625
+L 53.515625 60.890625
+Q 47.90625 63.578125 42.921875 64.890625
+Q 37.9375 66.21875 33.296875 66.21875
+Q 25.25 66.21875 20.875 63.09375
+Q 16.5 59.96875 16.5 54.203125
+Q 16.5 49.359375 19.40625 46.890625
+Q 22.3125 44.4375 30.421875 42.921875
+L 36.375 41.703125
+Q 47.40625 39.59375 52.65625 34.296875
+Q 57.90625 29 57.90625 20.125
+Q 57.90625 9.515625 50.796875 4.046875
+Q 43.703125 -1.421875 29.984375 -1.421875
+Q 24.8125 -1.421875 18.96875 -0.25
+Q 13.140625 0.921875 6.890625 3.21875
+L 6.890625 13.375
+Q 12.890625 10.015625 18.65625 8.296875
+Q 24.421875 6.59375 29.984375 6.59375
+Q 38.421875 6.59375 43.015625 9.90625
+Q 47.609375 13.234375 47.609375 19.390625
+Q 47.609375 24.75 44.3125 27.78125
+Q 41.015625 30.8125 33.5 32.328125
+L 27.484375 33.5
+Q 16.453125 35.6875 11.515625 40.375
+Q 6.59375 45.0625 6.59375 53.421875
+Q 6.59375 63.09375 13.40625 68.65625
+Q 20.21875 74.21875 32.171875 74.21875
+Q 37.3125 74.21875 42.625 73.28125
+Q 47.953125 72.359375 53.515625 70.515625
+z
+" id="DejaVuSans-83"/>
+ <path d="M 28.609375 0
+L 0.78125 72.90625
+L 11.078125 72.90625
+L 34.1875 11.53125
+L 57.328125 72.90625
+L 67.578125 72.90625
+L 39.796875 0
+z
+" id="DejaVuSans-86"/>
+ <path d="M 9.8125 72.90625
+L 55.90625 72.90625
+L 55.90625 64.59375
+L 19.671875 64.59375
+L 19.671875 43.015625
+L 54.390625 43.015625
+L 54.390625 34.71875
+L 19.671875 34.71875
+L 19.671875 8.296875
+L 56.78125 8.296875
+L 56.78125 0
+L 9.8125 0
+z
+" id="DejaVuSans-69"/>
+ </defs>
+ <use xlink:href="#DejaVuSans-78"/>
+ <use x="74.804688" xlink:href="#DejaVuSans-83"/>
+ <use x="138.28125" xlink:href="#DejaVuSans-86"/>
+ <use x="206.689453" xlink:href="#DejaVuSans-69"/>
+ </g>
+ </g>
+ <g id="line2d_35">
+ <path d="M 207.941029 206.839563
+L 219.941029 206.839563
+" style="fill:none;stroke:#ff7f0e;stroke-dasharray:4.5,7.5;stroke-dashoffset:0;stroke-width:1.5;"/>
+ </g>
+ <g id="line2d_36">
+ <g>
+ <use style="fill:#ff7f0e;stroke:#ff7f0e;" x="213.941029" xlink:href="#m4d39af77bf" y="206.839563"/>
+ </g>
+ </g>
+ <g id="text_11">
+ <!-- NSE -->
+ <g transform="translate(224.741029 208.939563)scale(0.06 -0.06)">
+ <use xlink:href="#DejaVuSans-78"/>
+ <use x="74.804688" xlink:href="#DejaVuSans-83"/>
+ <use x="138.28125" xlink:href="#DejaVuSans-69"/>
+ </g>
+ </g>
+ <g id="line2d_37">
+ <path d="M 207.941029 215.646438
+L 219.941029 215.646438
+" style="fill:none;stroke:#2ca02c;stroke-linecap:square;stroke-width:1.5;"/>
+ </g>
+ <g id="line2d_38">
+ <g>
+ <use style="fill:#2ca02c;stroke:#2ca02c;" x="213.941029" xlink:href="#m66f4880a7e" y="215.646438"/>
+ </g>
+ </g>
+ <g id="text_12">
+ <!-- NSVE vs NSE -->
+ <g transform="translate(224.741029 217.746438)scale(0.06 -0.06)">
+ <use xlink:href="#DejaVuSans-78"/>
+ <use x="74.804688" xlink:href="#DejaVuSans-83"/>
+ <use x="138.28125" xlink:href="#DejaVuSans-86"/>
+ <use x="206.689453" xlink:href="#DejaVuSans-69"/>
+ <use x="269.873047" xlink:href="#DejaVuSans-32"/>
+ <use x="301.660156" xlink:href="#DejaVuSans-118"/>
+ <use x="360.839844" xlink:href="#DejaVuSans-115"/>
+ <use x="412.939453" xlink:href="#DejaVuSans-32"/>
+ <use x="444.726562" xlink:href="#DejaVuSans-78"/>
+ <use x="519.53125" xlink:href="#DejaVuSans-83"/>
+ <use x="583.007812" xlink:href="#DejaVuSans-69"/>
+ </g>
+ </g>
+ <g id="line2d_39">
+ <path d="M 207.941029 224.453313
+L 219.941029 224.453313
+" style="fill:none;stroke:#000000;stroke-dasharray:3,3;stroke-dashoffset:0;stroke-width:1.5;"/>
+ </g>
+ <g id="line2d_40"/>
+ <g id="text_13">
+ <!-- $\propto f^{-2}$ -->
+ <g transform="translate(224.741029 226.553313)scale(0.06 -0.06)">
+ <defs>
+ <path d="M 25.734375 17.71875
+Q 33.796875 17.71875 38.625 29.4375
+Q 32.421875 42.234375 25.734375 42.234375
+Q 20.84375 42.234375 18.359375 38.71875
+Q 15.71875 34.96875 15.71875 29.984375
+Q 15.71875 24.515625 18.359375 21.140625
+Q 21.09375 17.71875 25.734375 17.71875
+z
+M 60.6875 17.625
+L 60.6875 11.1875
+L 56.9375 11.1875
+Q 53.375 11.1875 50.734375 12.640625
+Q 48.09375 13.96875 45.5625 17.046875
+Q 44.046875 18.796875 41.65625 22.90625
+Q 38.375 17.046875 35.5 14.453125
+Q 31.9375 11.328125 26.265625 11.328125
+Q 19.671875 11.328125 15.328125 16.265625
+Q 10.75 21.4375 10.75 29.984375
+Q 10.75 38.1875 15.328125 43.75
+Q 19.390625 48.6875 26.375 48.6875
+Q 29.9375 48.6875 32.5625 47.21875
+Q 35.640625 45.609375 37.75 42.828125
+Q 39.703125 40.328125 41.65625 36.96875
+Q 44.921875 42.828125 47.796875 45.40625
+Q 51.375 48.53125 57.03125 48.53125
+L 60.6875 48.53125
+L 60.6875 42.140625
+L 57.5625 42.140625
+Q 50.34375 42.140625 44.671875 30.421875
+Q 50.875 17.625 57.5625 17.625
+z
+" id="DejaVuSans-8733"/>
+ </defs>
+ <use transform="translate(19.482422 0.765625)" xlink:href="#DejaVuSans-8733"/>
+ <use transform="translate(110.400391 0.765625)" xlink:href="#DejaVuSans-Oblique-102"/>
+ <use transform="translate(153.053177 39.046875)scale(0.7)" xlink:href="#DejaVuSans-8722"/>
+ <use transform="translate(211.705521 39.046875)scale(0.7)" xlink:href="#DejaVuSans-50"/>
+ </g>
+ </g>
+ <g id="line2d_41">
+ <path d="M 207.941029 233.260188
+L 219.941029 233.260188
+" style="fill:none;stroke:#000000;stroke-dasharray:4.5,4.5;stroke-dashoffset:0;stroke-width:1.5;"/>
+ </g>
+ <g id="line2d_42"/>
+ <g id="text_14">
+ <!-- $\propto f^{-3}$ -->
+ <g transform="translate(224.741029 235.360187)scale(0.06 -0.06)">
+ <use transform="translate(19.482422 0.765625)" xlink:href="#DejaVuSans-8733"/>
+ <use transform="translate(110.400391 0.765625)" xlink:href="#DejaVuSans-Oblique-102"/>
+ <use transform="translate(153.053177 39.046875)scale(0.7)" xlink:href="#DejaVuSans-8722"/>
+ <use transform="translate(211.705521 39.046875)scale(0.7)" xlink:href="#DejaVuSans-51"/>
+ </g>
+ </g>
+ </g>
+ </g>
+ </g>
+ <defs>
+ <clipPath id="p697d71626a">
+ <rect height="231.208" width="212.943808" x="55.970346" y="10.8"/>
+ </clipPath>
+ </defs>
+</svg>
diff --git a/documentation/sphinx_static/err_vs_dt_particle.svg b/documentation/sphinx_static/err_vs_dt_particle.svg
new file mode 100644
index 0000000000000000000000000000000000000000..354d1447133274e33f51be754d5169c7311c4ac7
--- /dev/null
+++ b/documentation/sphinx_static/err_vs_dt_particle.svg
@@ -0,0 +1,1508 @@
+<?xml version="1.0" encoding="utf-8" standalone="no"?>
+<!DOCTYPE svg PUBLIC "-//W3C//DTD SVG 1.1//EN"
+ "http://www.w3.org/Graphics/SVG/1.1/DTD/svg11.dtd">
+<!-- Created with matplotlib (https://matplotlib.org/) -->
+<svg height="345.6pt" version="1.1" viewBox="0 0 460.8 345.6" width="460.8pt" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink">
+ <metadata>
+ <rdf:RDF xmlns:cc="http://creativecommons.org/ns#" xmlns:dc="http://purl.org/dc/elements/1.1/" xmlns:rdf="http://www.w3.org/1999/02/22-rdf-syntax-ns#">
+ <cc:Work>
+ <dc:type rdf:resource="http://purl.org/dc/dcmitype/StillImage"/>
+ <dc:date>2022-01-20T07:20:03.747523</dc:date>
+ <dc:format>image/svg+xml</dc:format>
+ <dc:creator>
+ <cc:Agent>
+ <dc:title>Matplotlib v3.3.4, https://matplotlib.org/</dc:title>
+ </cc:Agent>
+ </dc:creator>
+ </cc:Work>
+ </rdf:RDF>
+ </metadata>
+ <defs>
+ <style type="text/css">*{stroke-linecap:butt;stroke-linejoin:round;}</style>
+ </defs>
+ <g id="figure_1">
+ <g id="patch_1">
+ <path d="M 0 345.6
+L 460.8 345.6
+L 460.8 0
+L 0 0
+z
+" style="fill:#ffffff;"/>
+ </g>
+ <g id="axes_1">
+ <g id="patch_2">
+ <path d="M 56.015346 300.328
+L 447.801427 300.328
+L 447.801427 10.8
+L 56.015346 10.8
+z
+" style="fill:#ffffff;"/>
+ </g>
+ <g id="matplotlib.axis_1">
+ <g id="xtick_1">
+ <g id="line2d_1">
+ <defs>
+ <path d="M 0 0
+L 0 3.5
+" id="m726d39d34e" style="stroke:#000000;stroke-width:0.8;"/>
+ </defs>
+ <g>
+ <use style="stroke:#000000;stroke-width:0.8;" x="73.823804" xlink:href="#m726d39d34e" y="300.328"/>
+ </g>
+ </g>
+ <g id="text_1">
+ <!-- $\mathdefault{10^{0}}$ -->
+ <g transform="translate(65.023804 314.926437)scale(0.1 -0.1)">
+ <defs>
+ <path d="M 12.40625 8.296875
+L 28.515625 8.296875
+L 28.515625 63.921875
+L 10.984375 60.40625
+L 10.984375 69.390625
+L 28.421875 72.90625
+L 38.28125 72.90625
+L 38.28125 8.296875
+L 54.390625 8.296875
+L 54.390625 0
+L 12.40625 0
+z
+" id="DejaVuSans-49"/>
+ <path d="M 31.78125 66.40625
+Q 24.171875 66.40625 20.328125 58.90625
+Q 16.5 51.421875 16.5 36.375
+Q 16.5 21.390625 20.328125 13.890625
+Q 24.171875 6.390625 31.78125 6.390625
+Q 39.453125 6.390625 43.28125 13.890625
+Q 47.125 21.390625 47.125 36.375
+Q 47.125 51.421875 43.28125 58.90625
+Q 39.453125 66.40625 31.78125 66.40625
+z
+M 31.78125 74.21875
+Q 44.046875 74.21875 50.515625 64.515625
+Q 56.984375 54.828125 56.984375 36.375
+Q 56.984375 17.96875 50.515625 8.265625
+Q 44.046875 -1.421875 31.78125 -1.421875
+Q 19.53125 -1.421875 13.0625 8.265625
+Q 6.59375 17.96875 6.59375 36.375
+Q 6.59375 54.828125 13.0625 64.515625
+Q 19.53125 74.21875 31.78125 74.21875
+z
+" id="DejaVuSans-48"/>
+ </defs>
+ <use transform="translate(0 0.765625)" xlink:href="#DejaVuSans-49"/>
+ <use transform="translate(63.623047 0.765625)" xlink:href="#DejaVuSans-48"/>
+ <use transform="translate(128.203125 39.046875)scale(0.7)" xlink:href="#DejaVuSans-48"/>
+ </g>
+ </g>
+ </g>
+ <g id="xtick_2">
+ <g id="line2d_2">
+ <defs>
+ <path d="M 0 0
+L 0 2
+" id="maf1e5af406" style="stroke:#000000;stroke-width:0.6;"/>
+ </defs>
+ <g>
+ <use style="stroke:#000000;stroke-width:0.6;" x="167.508549" xlink:href="#maf1e5af406" y="300.328"/>
+ </g>
+ </g>
+ <g id="text_2">
+ <!-- $\mathdefault{1.2\times10^{0}}$ -->
+ <g transform="translate(144.908549 313.326437)scale(0.1 -0.1)">
+ <defs>
+ <path d="M 10.6875 12.40625
+L 21 12.40625
+L 21 0
+L 10.6875 0
+z
+" id="DejaVuSans-46"/>
+ <path d="M 19.1875 8.296875
+L 53.609375 8.296875
+L 53.609375 0
+L 7.328125 0
+L 7.328125 8.296875
+Q 12.9375 14.109375 22.625 23.890625
+Q 32.328125 33.6875 34.8125 36.53125
+Q 39.546875 41.84375 41.421875 45.53125
+Q 43.3125 49.21875 43.3125 52.78125
+Q 43.3125 58.59375 39.234375 62.25
+Q 35.15625 65.921875 28.609375 65.921875
+Q 23.96875 65.921875 18.8125 64.3125
+Q 13.671875 62.703125 7.8125 59.421875
+L 7.8125 69.390625
+Q 13.765625 71.78125 18.9375 73
+Q 24.125 74.21875 28.421875 74.21875
+Q 39.75 74.21875 46.484375 68.546875
+Q 53.21875 62.890625 53.21875 53.421875
+Q 53.21875 48.921875 51.53125 44.890625
+Q 49.859375 40.875 45.40625 35.40625
+Q 44.1875 33.984375 37.640625 27.21875
+Q 31.109375 20.453125 19.1875 8.296875
+z
+" id="DejaVuSans-50"/>
+ <path d="M 70.125 53.71875
+L 47.796875 31.296875
+L 70.125 8.984375
+L 64.3125 3.078125
+L 41.890625 25.484375
+L 19.484375 3.078125
+L 13.71875 8.984375
+L 35.984375 31.296875
+L 13.71875 53.71875
+L 19.484375 59.625
+L 41.890625 37.203125
+L 64.3125 59.625
+z
+" id="DejaVuSans-215"/>
+ </defs>
+ <use transform="translate(0 0.765625)" xlink:href="#DejaVuSans-49"/>
+ <use transform="translate(63.623047 0.765625)" xlink:href="#DejaVuSans-46"/>
+ <use transform="translate(89.910156 0.765625)" xlink:href="#DejaVuSans-50"/>
+ <use transform="translate(173.015625 0.765625)" xlink:href="#DejaVuSans-215"/>
+ <use transform="translate(276.287109 0.765625)" xlink:href="#DejaVuSans-49"/>
+ <use transform="translate(339.910156 0.765625)" xlink:href="#DejaVuSans-48"/>
+ <use transform="translate(404.490234 39.046875)scale(0.7)" xlink:href="#DejaVuSans-48"/>
+ </g>
+ </g>
+ </g>
+ <g id="xtick_3">
+ <g id="line2d_3">
+ <g>
+ <use style="stroke:#000000;stroke-width:0.6;" x="246.717872" xlink:href="#maf1e5af406" y="300.328"/>
+ </g>
+ </g>
+ <g id="text_3">
+ <!-- $\mathdefault{1.4\times10^{0}}$ -->
+ <g transform="translate(223.817872 313.326437)scale(0.1 -0.1)">
+ <defs>
+ <path d="M 37.796875 64.3125
+L 12.890625 25.390625
+L 37.796875 25.390625
+z
+M 35.203125 72.90625
+L 47.609375 72.90625
+L 47.609375 25.390625
+L 58.015625 25.390625
+L 58.015625 17.1875
+L 47.609375 17.1875
+L 47.609375 0
+L 37.796875 0
+L 37.796875 17.1875
+L 4.890625 17.1875
+L 4.890625 26.703125
+z
+" id="DejaVuSans-52"/>
+ </defs>
+ <use transform="translate(0 0.765625)" xlink:href="#DejaVuSans-49"/>
+ <use transform="translate(63.623047 0.765625)" xlink:href="#DejaVuSans-46"/>
+ <use transform="translate(95.410156 0.765625)" xlink:href="#DejaVuSans-52"/>
+ <use transform="translate(178.515625 0.765625)" xlink:href="#DejaVuSans-215"/>
+ <use transform="translate(281.787109 0.765625)" xlink:href="#DejaVuSans-49"/>
+ <use transform="translate(345.410156 0.765625)" xlink:href="#DejaVuSans-48"/>
+ <use transform="translate(409.990234 39.046875)scale(0.7)" xlink:href="#DejaVuSans-48"/>
+ </g>
+ </g>
+ </g>
+ <g id="xtick_4">
+ <g id="line2d_4">
+ <g>
+ <use style="stroke:#000000;stroke-width:0.6;" x="315.332108" xlink:href="#maf1e5af406" y="300.328"/>
+ </g>
+ </g>
+ <g id="text_4">
+ <!-- $\mathdefault{1.6\times10^{0}}$ -->
+ <g transform="translate(292.432108 313.326437)scale(0.1 -0.1)">
+ <defs>
+ <path d="M 33.015625 40.375
+Q 26.375 40.375 22.484375 35.828125
+Q 18.609375 31.296875 18.609375 23.390625
+Q 18.609375 15.53125 22.484375 10.953125
+Q 26.375 6.390625 33.015625 6.390625
+Q 39.65625 6.390625 43.53125 10.953125
+Q 47.40625 15.53125 47.40625 23.390625
+Q 47.40625 31.296875 43.53125 35.828125
+Q 39.65625 40.375 33.015625 40.375
+z
+M 52.59375 71.296875
+L 52.59375 62.3125
+Q 48.875 64.0625 45.09375 64.984375
+Q 41.3125 65.921875 37.59375 65.921875
+Q 27.828125 65.921875 22.671875 59.328125
+Q 17.53125 52.734375 16.796875 39.40625
+Q 19.671875 43.65625 24.015625 45.921875
+Q 28.375 48.1875 33.59375 48.1875
+Q 44.578125 48.1875 50.953125 41.515625
+Q 57.328125 34.859375 57.328125 23.390625
+Q 57.328125 12.15625 50.6875 5.359375
+Q 44.046875 -1.421875 33.015625 -1.421875
+Q 20.359375 -1.421875 13.671875 8.265625
+Q 6.984375 17.96875 6.984375 36.375
+Q 6.984375 53.65625 15.1875 63.9375
+Q 23.390625 74.21875 37.203125 74.21875
+Q 40.921875 74.21875 44.703125 73.484375
+Q 48.484375 72.75 52.59375 71.296875
+z
+" id="DejaVuSans-54"/>
+ </defs>
+ <use transform="translate(0 0.765625)" xlink:href="#DejaVuSans-49"/>
+ <use transform="translate(63.623047 0.765625)" xlink:href="#DejaVuSans-46"/>
+ <use transform="translate(95.410156 0.765625)" xlink:href="#DejaVuSans-54"/>
+ <use transform="translate(178.515625 0.765625)" xlink:href="#DejaVuSans-215"/>
+ <use transform="translate(281.787109 0.765625)" xlink:href="#DejaVuSans-49"/>
+ <use transform="translate(345.410156 0.765625)" xlink:href="#DejaVuSans-48"/>
+ <use transform="translate(409.990234 39.046875)scale(0.7)" xlink:href="#DejaVuSans-48"/>
+ </g>
+ </g>
+ </g>
+ <g id="xtick_5">
+ <g id="line2d_5">
+ <g>
+ <use style="stroke:#000000;stroke-width:0.6;" x="375.854154" xlink:href="#maf1e5af406" y="300.328"/>
+ </g>
+ </g>
+ <g id="text_5">
+ <!-- $\mathdefault{1.8\times10^{0}}$ -->
+ <g transform="translate(353.104154 313.326437)scale(0.1 -0.1)">
+ <defs>
+ <path d="M 31.78125 34.625
+Q 24.75 34.625 20.71875 30.859375
+Q 16.703125 27.09375 16.703125 20.515625
+Q 16.703125 13.921875 20.71875 10.15625
+Q 24.75 6.390625 31.78125 6.390625
+Q 38.8125 6.390625 42.859375 10.171875
+Q 46.921875 13.96875 46.921875 20.515625
+Q 46.921875 27.09375 42.890625 30.859375
+Q 38.875 34.625 31.78125 34.625
+z
+M 21.921875 38.8125
+Q 15.578125 40.375 12.03125 44.71875
+Q 8.5 49.078125 8.5 55.328125
+Q 8.5 64.0625 14.71875 69.140625
+Q 20.953125 74.21875 31.78125 74.21875
+Q 42.671875 74.21875 48.875 69.140625
+Q 55.078125 64.0625 55.078125 55.328125
+Q 55.078125 49.078125 51.53125 44.71875
+Q 48 40.375 41.703125 38.8125
+Q 48.828125 37.15625 52.796875 32.3125
+Q 56.78125 27.484375 56.78125 20.515625
+Q 56.78125 9.90625 50.3125 4.234375
+Q 43.84375 -1.421875 31.78125 -1.421875
+Q 19.734375 -1.421875 13.25 4.234375
+Q 6.78125 9.90625 6.78125 20.515625
+Q 6.78125 27.484375 10.78125 32.3125
+Q 14.796875 37.15625 21.921875 38.8125
+z
+M 18.3125 54.390625
+Q 18.3125 48.734375 21.84375 45.5625
+Q 25.390625 42.390625 31.78125 42.390625
+Q 38.140625 42.390625 41.71875 45.5625
+Q 45.3125 48.734375 45.3125 54.390625
+Q 45.3125 60.0625 41.71875 63.234375
+Q 38.140625 66.40625 31.78125 66.40625
+Q 25.390625 66.40625 21.84375 63.234375
+Q 18.3125 60.0625 18.3125 54.390625
+z
+" id="DejaVuSans-56"/>
+ </defs>
+ <use transform="translate(0 0.765625)" xlink:href="#DejaVuSans-49"/>
+ <use transform="translate(63.623047 0.765625)" xlink:href="#DejaVuSans-46"/>
+ <use transform="translate(92.785156 0.765625)" xlink:href="#DejaVuSans-56"/>
+ <use transform="translate(175.890625 0.765625)" xlink:href="#DejaVuSans-215"/>
+ <use transform="translate(279.162109 0.765625)" xlink:href="#DejaVuSans-49"/>
+ <use transform="translate(342.785156 0.765625)" xlink:href="#DejaVuSans-48"/>
+ <use transform="translate(407.365234 39.046875)scale(0.7)" xlink:href="#DejaVuSans-48"/>
+ </g>
+ </g>
+ </g>
+ <g id="xtick_6">
+ <g id="line2d_6">
+ <g>
+ <use style="stroke:#000000;stroke-width:0.6;" x="429.992969" xlink:href="#maf1e5af406" y="300.328"/>
+ </g>
+ </g>
+ <g id="text_6">
+ <!-- $\mathdefault{2\times10^{0}}$ -->
+ <g transform="translate(411.892969 313.326437)scale(0.1 -0.1)">
+ <use transform="translate(0 0.765625)" xlink:href="#DejaVuSans-50"/>
+ <use transform="translate(83.105469 0.765625)" xlink:href="#DejaVuSans-215"/>
+ <use transform="translate(186.376953 0.765625)" xlink:href="#DejaVuSans-49"/>
+ <use transform="translate(250 0.765625)" xlink:href="#DejaVuSans-48"/>
+ <use transform="translate(314.580078 39.046875)scale(0.7)" xlink:href="#DejaVuSans-48"/>
+ </g>
+ </g>
+ </g>
+ <g id="text_7">
+ <!-- resolution factor f -->
+ <g transform="translate(207.502918 328.604562)scale(0.1 -0.1)">
+ <defs>
+ <path d="M 41.109375 46.296875
+Q 39.59375 47.171875 37.8125 47.578125
+Q 36.03125 48 33.890625 48
+Q 26.265625 48 22.1875 43.046875
+Q 18.109375 38.09375 18.109375 28.8125
+L 18.109375 0
+L 9.078125 0
+L 9.078125 54.6875
+L 18.109375 54.6875
+L 18.109375 46.1875
+Q 20.953125 51.171875 25.484375 53.578125
+Q 30.03125 56 36.53125 56
+Q 37.453125 56 38.578125 55.875
+Q 39.703125 55.765625 41.0625 55.515625
+z
+" id="DejaVuSans-114"/>
+ <path d="M 56.203125 29.59375
+L 56.203125 25.203125
+L 14.890625 25.203125
+Q 15.484375 15.921875 20.484375 11.0625
+Q 25.484375 6.203125 34.421875 6.203125
+Q 39.59375 6.203125 44.453125 7.46875
+Q 49.3125 8.734375 54.109375 11.28125
+L 54.109375 2.78125
+Q 49.265625 0.734375 44.1875 -0.34375
+Q 39.109375 -1.421875 33.890625 -1.421875
+Q 20.796875 -1.421875 13.15625 6.1875
+Q 5.515625 13.8125 5.515625 26.8125
+Q 5.515625 40.234375 12.765625 48.109375
+Q 20.015625 56 32.328125 56
+Q 43.359375 56 49.78125 48.890625
+Q 56.203125 41.796875 56.203125 29.59375
+z
+M 47.21875 32.234375
+Q 47.125 39.59375 43.09375 43.984375
+Q 39.0625 48.390625 32.421875 48.390625
+Q 24.90625 48.390625 20.390625 44.140625
+Q 15.875 39.890625 15.1875 32.171875
+z
+" id="DejaVuSans-101"/>
+ <path d="M 44.28125 53.078125
+L 44.28125 44.578125
+Q 40.484375 46.53125 36.375 47.5
+Q 32.28125 48.484375 27.875 48.484375
+Q 21.1875 48.484375 17.84375 46.4375
+Q 14.5 44.390625 14.5 40.28125
+Q 14.5 37.15625 16.890625 35.375
+Q 19.28125 33.59375 26.515625 31.984375
+L 29.59375 31.296875
+Q 39.15625 29.25 43.1875 25.515625
+Q 47.21875 21.78125 47.21875 15.09375
+Q 47.21875 7.46875 41.1875 3.015625
+Q 35.15625 -1.421875 24.609375 -1.421875
+Q 20.21875 -1.421875 15.453125 -0.5625
+Q 10.6875 0.296875 5.421875 2
+L 5.421875 11.28125
+Q 10.40625 8.6875 15.234375 7.390625
+Q 20.0625 6.109375 24.8125 6.109375
+Q 31.15625 6.109375 34.5625 8.28125
+Q 37.984375 10.453125 37.984375 14.40625
+Q 37.984375 18.0625 35.515625 20.015625
+Q 33.0625 21.96875 24.703125 23.78125
+L 21.578125 24.515625
+Q 13.234375 26.265625 9.515625 29.90625
+Q 5.8125 33.546875 5.8125 39.890625
+Q 5.8125 47.609375 11.28125 51.796875
+Q 16.75 56 26.8125 56
+Q 31.78125 56 36.171875 55.265625
+Q 40.578125 54.546875 44.28125 53.078125
+z
+" id="DejaVuSans-115"/>
+ <path d="M 30.609375 48.390625
+Q 23.390625 48.390625 19.1875 42.75
+Q 14.984375 37.109375 14.984375 27.296875
+Q 14.984375 17.484375 19.15625 11.84375
+Q 23.34375 6.203125 30.609375 6.203125
+Q 37.796875 6.203125 41.984375 11.859375
+Q 46.1875 17.53125 46.1875 27.296875
+Q 46.1875 37.015625 41.984375 42.703125
+Q 37.796875 48.390625 30.609375 48.390625
+z
+M 30.609375 56
+Q 42.328125 56 49.015625 48.375
+Q 55.71875 40.765625 55.71875 27.296875
+Q 55.71875 13.875 49.015625 6.21875
+Q 42.328125 -1.421875 30.609375 -1.421875
+Q 18.84375 -1.421875 12.171875 6.21875
+Q 5.515625 13.875 5.515625 27.296875
+Q 5.515625 40.765625 12.171875 48.375
+Q 18.84375 56 30.609375 56
+z
+" id="DejaVuSans-111"/>
+ <path d="M 9.421875 75.984375
+L 18.40625 75.984375
+L 18.40625 0
+L 9.421875 0
+z
+" id="DejaVuSans-108"/>
+ <path d="M 8.5 21.578125
+L 8.5 54.6875
+L 17.484375 54.6875
+L 17.484375 21.921875
+Q 17.484375 14.15625 20.5 10.265625
+Q 23.53125 6.390625 29.59375 6.390625
+Q 36.859375 6.390625 41.078125 11.03125
+Q 45.3125 15.671875 45.3125 23.6875
+L 45.3125 54.6875
+L 54.296875 54.6875
+L 54.296875 0
+L 45.3125 0
+L 45.3125 8.40625
+Q 42.046875 3.421875 37.71875 1
+Q 33.40625 -1.421875 27.6875 -1.421875
+Q 18.265625 -1.421875 13.375 4.4375
+Q 8.5 10.296875 8.5 21.578125
+z
+M 31.109375 56
+z
+" id="DejaVuSans-117"/>
+ <path d="M 18.3125 70.21875
+L 18.3125 54.6875
+L 36.8125 54.6875
+L 36.8125 47.703125
+L 18.3125 47.703125
+L 18.3125 18.015625
+Q 18.3125 11.328125 20.140625 9.421875
+Q 21.96875 7.515625 27.59375 7.515625
+L 36.8125 7.515625
+L 36.8125 0
+L 27.59375 0
+Q 17.1875 0 13.234375 3.875
+Q 9.28125 7.765625 9.28125 18.015625
+L 9.28125 47.703125
+L 2.6875 47.703125
+L 2.6875 54.6875
+L 9.28125 54.6875
+L 9.28125 70.21875
+z
+" id="DejaVuSans-116"/>
+ <path d="M 9.421875 54.6875
+L 18.40625 54.6875
+L 18.40625 0
+L 9.421875 0
+z
+M 9.421875 75.984375
+L 18.40625 75.984375
+L 18.40625 64.59375
+L 9.421875 64.59375
+z
+" id="DejaVuSans-105"/>
+ <path d="M 54.890625 33.015625
+L 54.890625 0
+L 45.90625 0
+L 45.90625 32.71875
+Q 45.90625 40.484375 42.875 44.328125
+Q 39.84375 48.1875 33.796875 48.1875
+Q 26.515625 48.1875 22.3125 43.546875
+Q 18.109375 38.921875 18.109375 30.90625
+L 18.109375 0
+L 9.078125 0
+L 9.078125 54.6875
+L 18.109375 54.6875
+L 18.109375 46.1875
+Q 21.34375 51.125 25.703125 53.5625
+Q 30.078125 56 35.796875 56
+Q 45.21875 56 50.046875 50.171875
+Q 54.890625 44.34375 54.890625 33.015625
+z
+" id="DejaVuSans-110"/>
+ <path id="DejaVuSans-32"/>
+ <path d="M 37.109375 75.984375
+L 37.109375 68.5
+L 28.515625 68.5
+Q 23.6875 68.5 21.796875 66.546875
+Q 19.921875 64.59375 19.921875 59.515625
+L 19.921875 54.6875
+L 34.71875 54.6875
+L 34.71875 47.703125
+L 19.921875 47.703125
+L 19.921875 0
+L 10.890625 0
+L 10.890625 47.703125
+L 2.296875 47.703125
+L 2.296875 54.6875
+L 10.890625 54.6875
+L 10.890625 58.5
+Q 10.890625 67.625 15.140625 71.796875
+Q 19.390625 75.984375 28.609375 75.984375
+z
+" id="DejaVuSans-102"/>
+ <path d="M 34.28125 27.484375
+Q 23.390625 27.484375 19.1875 25
+Q 14.984375 22.515625 14.984375 16.5
+Q 14.984375 11.71875 18.140625 8.90625
+Q 21.296875 6.109375 26.703125 6.109375
+Q 34.1875 6.109375 38.703125 11.40625
+Q 43.21875 16.703125 43.21875 25.484375
+L 43.21875 27.484375
+z
+M 52.203125 31.203125
+L 52.203125 0
+L 43.21875 0
+L 43.21875 8.296875
+Q 40.140625 3.328125 35.546875 0.953125
+Q 30.953125 -1.421875 24.3125 -1.421875
+Q 15.921875 -1.421875 10.953125 3.296875
+Q 6 8.015625 6 15.921875
+Q 6 25.140625 12.171875 29.828125
+Q 18.359375 34.515625 30.609375 34.515625
+L 43.21875 34.515625
+L 43.21875 35.40625
+Q 43.21875 41.609375 39.140625 45
+Q 35.0625 48.390625 27.6875 48.390625
+Q 23 48.390625 18.546875 47.265625
+Q 14.109375 46.140625 10.015625 43.890625
+L 10.015625 52.203125
+Q 14.9375 54.109375 19.578125 55.046875
+Q 24.21875 56 28.609375 56
+Q 40.484375 56 46.34375 49.84375
+Q 52.203125 43.703125 52.203125 31.203125
+z
+" id="DejaVuSans-97"/>
+ <path d="M 48.78125 52.59375
+L 48.78125 44.1875
+Q 44.96875 46.296875 41.140625 47.34375
+Q 37.3125 48.390625 33.40625 48.390625
+Q 24.65625 48.390625 19.8125 42.84375
+Q 14.984375 37.3125 14.984375 27.296875
+Q 14.984375 17.28125 19.8125 11.734375
+Q 24.65625 6.203125 33.40625 6.203125
+Q 37.3125 6.203125 41.140625 7.25
+Q 44.96875 8.296875 48.78125 10.40625
+L 48.78125 2.09375
+Q 45.015625 0.34375 40.984375 -0.53125
+Q 36.96875 -1.421875 32.421875 -1.421875
+Q 20.0625 -1.421875 12.78125 6.34375
+Q 5.515625 14.109375 5.515625 27.296875
+Q 5.515625 40.671875 12.859375 48.328125
+Q 20.21875 56 33.015625 56
+Q 37.15625 56 41.109375 55.140625
+Q 45.0625 54.296875 48.78125 52.59375
+z
+" id="DejaVuSans-99"/>
+ </defs>
+ <use xlink:href="#DejaVuSans-114"/>
+ <use x="38.863281" xlink:href="#DejaVuSans-101"/>
+ <use x="100.386719" xlink:href="#DejaVuSans-115"/>
+ <use x="152.486328" xlink:href="#DejaVuSans-111"/>
+ <use x="213.667969" xlink:href="#DejaVuSans-108"/>
+ <use x="241.451172" xlink:href="#DejaVuSans-117"/>
+ <use x="304.830078" xlink:href="#DejaVuSans-116"/>
+ <use x="344.039062" xlink:href="#DejaVuSans-105"/>
+ <use x="371.822266" xlink:href="#DejaVuSans-111"/>
+ <use x="433.003906" xlink:href="#DejaVuSans-110"/>
+ <use x="496.382812" xlink:href="#DejaVuSans-32"/>
+ <use x="528.169922" xlink:href="#DejaVuSans-102"/>
+ <use x="563.375" xlink:href="#DejaVuSans-97"/>
+ <use x="624.654297" xlink:href="#DejaVuSans-99"/>
+ <use x="679.634766" xlink:href="#DejaVuSans-116"/>
+ <use x="718.84375" xlink:href="#DejaVuSans-111"/>
+ <use x="780.025391" xlink:href="#DejaVuSans-114"/>
+ <use x="821.138672" xlink:href="#DejaVuSans-32"/>
+ <use x="852.925781" xlink:href="#DejaVuSans-102"/>
+ </g>
+ </g>
+ </g>
+ <g id="matplotlib.axis_2">
+ <g id="ytick_1">
+ <g id="line2d_7">
+ <defs>
+ <path d="M 0 0
+L -3.5 0
+" id="m29c7ff5d91" style="stroke:#000000;stroke-width:0.8;"/>
+ </defs>
+ <g>
+ <use style="stroke:#000000;stroke-width:0.8;" x="56.015346" xlink:href="#m29c7ff5d91" y="240.31258"/>
+ </g>
+ </g>
+ <g id="text_8">
+ <!-- $\mathdefault{10^{-5}}$ -->
+ <g transform="translate(25.515346 244.111798)scale(0.1 -0.1)">
+ <defs>
+ <path d="M 10.59375 35.5
+L 73.1875 35.5
+L 73.1875 27.203125
+L 10.59375 27.203125
+z
+" id="DejaVuSans-8722"/>
+ <path d="M 10.796875 72.90625
+L 49.515625 72.90625
+L 49.515625 64.59375
+L 19.828125 64.59375
+L 19.828125 46.734375
+Q 21.96875 47.46875 24.109375 47.828125
+Q 26.265625 48.1875 28.421875 48.1875
+Q 40.625 48.1875 47.75 41.5
+Q 54.890625 34.8125 54.890625 23.390625
+Q 54.890625 11.625 47.5625 5.09375
+Q 40.234375 -1.421875 26.90625 -1.421875
+Q 22.3125 -1.421875 17.546875 -0.640625
+Q 12.796875 0.140625 7.71875 1.703125
+L 7.71875 11.625
+Q 12.109375 9.234375 16.796875 8.0625
+Q 21.484375 6.890625 26.703125 6.890625
+Q 35.15625 6.890625 40.078125 11.328125
+Q 45.015625 15.765625 45.015625 23.390625
+Q 45.015625 31 40.078125 35.4375
+Q 35.15625 39.890625 26.703125 39.890625
+Q 22.75 39.890625 18.8125 39.015625
+Q 14.890625 38.140625 10.796875 36.28125
+z
+" id="DejaVuSans-53"/>
+ </defs>
+ <use transform="translate(0 0.684375)" xlink:href="#DejaVuSans-49"/>
+ <use transform="translate(63.623047 0.684375)" xlink:href="#DejaVuSans-48"/>
+ <use transform="translate(128.203125 38.965625)scale(0.7)" xlink:href="#DejaVuSans-8722"/>
+ <use transform="translate(186.855469 38.965625)scale(0.7)" xlink:href="#DejaVuSans-53"/>
+ </g>
+ </g>
+ </g>
+ <g id="ytick_2">
+ <g id="line2d_8">
+ <g>
+ <use style="stroke:#000000;stroke-width:0.8;" x="56.015346" xlink:href="#m29c7ff5d91" y="133.142634"/>
+ </g>
+ </g>
+ <g id="text_9">
+ <!-- $\mathdefault{10^{-4}}$ -->
+ <g transform="translate(25.515346 136.941853)scale(0.1 -0.1)">
+ <use transform="translate(0 0.684375)" xlink:href="#DejaVuSans-49"/>
+ <use transform="translate(63.623047 0.684375)" xlink:href="#DejaVuSans-48"/>
+ <use transform="translate(128.203125 38.965625)scale(0.7)" xlink:href="#DejaVuSans-8722"/>
+ <use transform="translate(186.855469 38.965625)scale(0.7)" xlink:href="#DejaVuSans-52"/>
+ </g>
+ </g>
+ </g>
+ <g id="ytick_3">
+ <g id="line2d_9">
+ <g>
+ <use style="stroke:#000000;stroke-width:0.8;" x="56.015346" xlink:href="#m29c7ff5d91" y="25.972689"/>
+ </g>
+ </g>
+ <g id="text_10">
+ <!-- $\mathdefault{10^{-3}}$ -->
+ <g transform="translate(25.515346 29.771908)scale(0.1 -0.1)">
+ <defs>
+ <path d="M 40.578125 39.3125
+Q 47.65625 37.796875 51.625 33
+Q 55.609375 28.21875 55.609375 21.1875
+Q 55.609375 10.40625 48.1875 4.484375
+Q 40.765625 -1.421875 27.09375 -1.421875
+Q 22.515625 -1.421875 17.65625 -0.515625
+Q 12.796875 0.390625 7.625 2.203125
+L 7.625 11.71875
+Q 11.71875 9.328125 16.59375 8.109375
+Q 21.484375 6.890625 26.8125 6.890625
+Q 36.078125 6.890625 40.9375 10.546875
+Q 45.796875 14.203125 45.796875 21.1875
+Q 45.796875 27.640625 41.28125 31.265625
+Q 36.765625 34.90625 28.71875 34.90625
+L 20.21875 34.90625
+L 20.21875 43.015625
+L 29.109375 43.015625
+Q 36.375 43.015625 40.234375 45.921875
+Q 44.09375 48.828125 44.09375 54.296875
+Q 44.09375 59.90625 40.109375 62.90625
+Q 36.140625 65.921875 28.71875 65.921875
+Q 24.65625 65.921875 20.015625 65.03125
+Q 15.375 64.15625 9.8125 62.3125
+L 9.8125 71.09375
+Q 15.4375 72.65625 20.34375 73.4375
+Q 25.25 74.21875 29.59375 74.21875
+Q 40.828125 74.21875 47.359375 69.109375
+Q 53.90625 64.015625 53.90625 55.328125
+Q 53.90625 49.265625 50.4375 45.09375
+Q 46.96875 40.921875 40.578125 39.3125
+z
+" id="DejaVuSans-51"/>
+ </defs>
+ <use transform="translate(0 0.765625)" xlink:href="#DejaVuSans-49"/>
+ <use transform="translate(63.623047 0.765625)" xlink:href="#DejaVuSans-48"/>
+ <use transform="translate(128.203125 39.046875)scale(0.7)" xlink:href="#DejaVuSans-8722"/>
+ <use transform="translate(186.855469 39.046875)scale(0.7)" xlink:href="#DejaVuSans-51"/>
+ </g>
+ </g>
+ </g>
+ <g id="ytick_4">
+ <g id="line2d_10">
+ <defs>
+ <path d="M 0 0
+L -2 0
+" id="m7feeaf6abe" style="stroke:#000000;stroke-width:0.6;"/>
+ </defs>
+ <g>
+ <use style="stroke:#000000;stroke-width:0.6;" x="56.015346" xlink:href="#m7feeaf6abe" y="296.349466"/>
+ </g>
+ </g>
+ </g>
+ <g id="ytick_5">
+ <g id="line2d_11">
+ <g>
+ <use style="stroke:#000000;stroke-width:0.6;" x="56.015346" xlink:href="#m7feeaf6abe" y="282.959789"/>
+ </g>
+ </g>
+ </g>
+ <g id="ytick_6">
+ <g id="line2d_12">
+ <g>
+ <use style="stroke:#000000;stroke-width:0.6;" x="56.015346" xlink:href="#m7feeaf6abe" y="272.573948"/>
+ </g>
+ </g>
+ </g>
+ <g id="ytick_7">
+ <g id="line2d_13">
+ <g>
+ <use style="stroke:#000000;stroke-width:0.6;" x="56.015346" xlink:href="#m7feeaf6abe" y="264.088098"/>
+ </g>
+ </g>
+ </g>
+ <g id="ytick_8">
+ <g id="line2d_14">
+ <g>
+ <use style="stroke:#000000;stroke-width:0.6;" x="56.015346" xlink:href="#m7feeaf6abe" y="256.913414"/>
+ </g>
+ </g>
+ </g>
+ <g id="ytick_9">
+ <g id="line2d_15">
+ <g>
+ <use style="stroke:#000000;stroke-width:0.6;" x="56.015346" xlink:href="#m7feeaf6abe" y="250.69842"/>
+ </g>
+ </g>
+ </g>
+ <g id="ytick_10">
+ <g id="line2d_16">
+ <g>
+ <use style="stroke:#000000;stroke-width:0.6;" x="56.015346" xlink:href="#m7feeaf6abe" y="245.216407"/>
+ </g>
+ </g>
+ </g>
+ <g id="ytick_11">
+ <g id="line2d_17">
+ <g>
+ <use style="stroke:#000000;stroke-width:0.6;" x="56.015346" xlink:href="#m7feeaf6abe" y="208.051212"/>
+ </g>
+ </g>
+ </g>
+ <g id="ytick_12">
+ <g id="line2d_18">
+ <g>
+ <use style="stroke:#000000;stroke-width:0.6;" x="56.015346" xlink:href="#m7feeaf6abe" y="189.179521"/>
+ </g>
+ </g>
+ </g>
+ <g id="ytick_13">
+ <g id="line2d_19">
+ <g>
+ <use style="stroke:#000000;stroke-width:0.6;" x="56.015346" xlink:href="#m7feeaf6abe" y="175.789843"/>
+ </g>
+ </g>
+ </g>
+ <g id="ytick_14">
+ <g id="line2d_20">
+ <g>
+ <use style="stroke:#000000;stroke-width:0.6;" x="56.015346" xlink:href="#m7feeaf6abe" y="165.404003"/>
+ </g>
+ </g>
+ </g>
+ <g id="ytick_15">
+ <g id="line2d_21">
+ <g>
+ <use style="stroke:#000000;stroke-width:0.6;" x="56.015346" xlink:href="#m7feeaf6abe" y="156.918153"/>
+ </g>
+ </g>
+ </g>
+ <g id="ytick_16">
+ <g id="line2d_22">
+ <g>
+ <use style="stroke:#000000;stroke-width:0.6;" x="56.015346" xlink:href="#m7feeaf6abe" y="149.743469"/>
+ </g>
+ </g>
+ </g>
+ <g id="ytick_17">
+ <g id="line2d_23">
+ <g>
+ <use style="stroke:#000000;stroke-width:0.6;" x="56.015346" xlink:href="#m7feeaf6abe" y="143.528475"/>
+ </g>
+ </g>
+ </g>
+ <g id="ytick_18">
+ <g id="line2d_24">
+ <g>
+ <use style="stroke:#000000;stroke-width:0.6;" x="56.015346" xlink:href="#m7feeaf6abe" y="138.046462"/>
+ </g>
+ </g>
+ </g>
+ <g id="ytick_19">
+ <g id="line2d_25">
+ <g>
+ <use style="stroke:#000000;stroke-width:0.6;" x="56.015346" xlink:href="#m7feeaf6abe" y="100.881266"/>
+ </g>
+ </g>
+ </g>
+ <g id="ytick_20">
+ <g id="line2d_26">
+ <g>
+ <use style="stroke:#000000;stroke-width:0.6;" x="56.015346" xlink:href="#m7feeaf6abe" y="82.009576"/>
+ </g>
+ </g>
+ </g>
+ <g id="ytick_21">
+ <g id="line2d_27">
+ <g>
+ <use style="stroke:#000000;stroke-width:0.6;" x="56.015346" xlink:href="#m7feeaf6abe" y="68.619898"/>
+ </g>
+ </g>
+ </g>
+ <g id="ytick_22">
+ <g id="line2d_28">
+ <g>
+ <use style="stroke:#000000;stroke-width:0.6;" x="56.015346" xlink:href="#m7feeaf6abe" y="58.234057"/>
+ </g>
+ </g>
+ </g>
+ <g id="ytick_23">
+ <g id="line2d_29">
+ <g>
+ <use style="stroke:#000000;stroke-width:0.6;" x="56.015346" xlink:href="#m7feeaf6abe" y="49.748208"/>
+ </g>
+ </g>
+ </g>
+ <g id="ytick_24">
+ <g id="line2d_30">
+ <g>
+ <use style="stroke:#000000;stroke-width:0.6;" x="56.015346" xlink:href="#m7feeaf6abe" y="42.573524"/>
+ </g>
+ </g>
+ </g>
+ <g id="ytick_25">
+ <g id="line2d_31">
+ <g>
+ <use style="stroke:#000000;stroke-width:0.6;" x="56.015346" xlink:href="#m7feeaf6abe" y="36.35853"/>
+ </g>
+ </g>
+ </g>
+ <g id="ytick_26">
+ <g id="line2d_32">
+ <g>
+ <use style="stroke:#000000;stroke-width:0.6;" x="56.015346" xlink:href="#m7feeaf6abe" y="30.876517"/>
+ </g>
+ </g>
+ </g>
+ <g id="text_11">
+ <!-- absolute trajectory error [DNS units] -->
+ <g transform="translate(19.435659 246.806187)rotate(-90)scale(0.1 -0.1)">
+ <defs>
+ <path d="M 48.6875 27.296875
+Q 48.6875 37.203125 44.609375 42.84375
+Q 40.53125 48.484375 33.40625 48.484375
+Q 26.265625 48.484375 22.1875 42.84375
+Q 18.109375 37.203125 18.109375 27.296875
+Q 18.109375 17.390625 22.1875 11.75
+Q 26.265625 6.109375 33.40625 6.109375
+Q 40.53125 6.109375 44.609375 11.75
+Q 48.6875 17.390625 48.6875 27.296875
+z
+M 18.109375 46.390625
+Q 20.953125 51.265625 25.265625 53.625
+Q 29.59375 56 35.59375 56
+Q 45.5625 56 51.78125 48.09375
+Q 58.015625 40.1875 58.015625 27.296875
+Q 58.015625 14.40625 51.78125 6.484375
+Q 45.5625 -1.421875 35.59375 -1.421875
+Q 29.59375 -1.421875 25.265625 0.953125
+Q 20.953125 3.328125 18.109375 8.203125
+L 18.109375 0
+L 9.078125 0
+L 9.078125 75.984375
+L 18.109375 75.984375
+z
+" id="DejaVuSans-98"/>
+ <path d="M 9.421875 54.6875
+L 18.40625 54.6875
+L 18.40625 -0.984375
+Q 18.40625 -11.421875 14.421875 -16.109375
+Q 10.453125 -20.796875 1.609375 -20.796875
+L -1.8125 -20.796875
+L -1.8125 -13.1875
+L 0.59375 -13.1875
+Q 5.71875 -13.1875 7.5625 -10.8125
+Q 9.421875 -8.453125 9.421875 -0.984375
+z
+M 9.421875 75.984375
+L 18.40625 75.984375
+L 18.40625 64.59375
+L 9.421875 64.59375
+z
+" id="DejaVuSans-106"/>
+ <path d="M 32.171875 -5.078125
+Q 28.375 -14.84375 24.75 -17.8125
+Q 21.140625 -20.796875 15.09375 -20.796875
+L 7.90625 -20.796875
+L 7.90625 -13.28125
+L 13.1875 -13.28125
+Q 16.890625 -13.28125 18.9375 -11.515625
+Q 21 -9.765625 23.484375 -3.21875
+L 25.09375 0.875
+L 2.984375 54.6875
+L 12.5 54.6875
+L 29.59375 11.921875
+L 46.6875 54.6875
+L 56.203125 54.6875
+z
+" id="DejaVuSans-121"/>
+ <path d="M 8.59375 75.984375
+L 29.296875 75.984375
+L 29.296875 69
+L 17.578125 69
+L 17.578125 -6.203125
+L 29.296875 -6.203125
+L 29.296875 -13.1875
+L 8.59375 -13.1875
+z
+" id="DejaVuSans-91"/>
+ <path d="M 19.671875 64.796875
+L 19.671875 8.109375
+L 31.59375 8.109375
+Q 46.6875 8.109375 53.6875 14.9375
+Q 60.6875 21.78125 60.6875 36.53125
+Q 60.6875 51.171875 53.6875 57.984375
+Q 46.6875 64.796875 31.59375 64.796875
+z
+M 9.8125 72.90625
+L 30.078125 72.90625
+Q 51.265625 72.90625 61.171875 64.09375
+Q 71.09375 55.28125 71.09375 36.53125
+Q 71.09375 17.671875 61.125 8.828125
+Q 51.171875 0 30.078125 0
+L 9.8125 0
+z
+" id="DejaVuSans-68"/>
+ <path d="M 9.8125 72.90625
+L 23.09375 72.90625
+L 55.421875 11.921875
+L 55.421875 72.90625
+L 64.984375 72.90625
+L 64.984375 0
+L 51.703125 0
+L 19.390625 60.984375
+L 19.390625 0
+L 9.8125 0
+z
+" id="DejaVuSans-78"/>
+ <path d="M 53.515625 70.515625
+L 53.515625 60.890625
+Q 47.90625 63.578125 42.921875 64.890625
+Q 37.9375 66.21875 33.296875 66.21875
+Q 25.25 66.21875 20.875 63.09375
+Q 16.5 59.96875 16.5 54.203125
+Q 16.5 49.359375 19.40625 46.890625
+Q 22.3125 44.4375 30.421875 42.921875
+L 36.375 41.703125
+Q 47.40625 39.59375 52.65625 34.296875
+Q 57.90625 29 57.90625 20.125
+Q 57.90625 9.515625 50.796875 4.046875
+Q 43.703125 -1.421875 29.984375 -1.421875
+Q 24.8125 -1.421875 18.96875 -0.25
+Q 13.140625 0.921875 6.890625 3.21875
+L 6.890625 13.375
+Q 12.890625 10.015625 18.65625 8.296875
+Q 24.421875 6.59375 29.984375 6.59375
+Q 38.421875 6.59375 43.015625 9.90625
+Q 47.609375 13.234375 47.609375 19.390625
+Q 47.609375 24.75 44.3125 27.78125
+Q 41.015625 30.8125 33.5 32.328125
+L 27.484375 33.5
+Q 16.453125 35.6875 11.515625 40.375
+Q 6.59375 45.0625 6.59375 53.421875
+Q 6.59375 63.09375 13.40625 68.65625
+Q 20.21875 74.21875 32.171875 74.21875
+Q 37.3125 74.21875 42.625 73.28125
+Q 47.953125 72.359375 53.515625 70.515625
+z
+" id="DejaVuSans-83"/>
+ <path d="M 30.421875 75.984375
+L 30.421875 -13.1875
+L 9.71875 -13.1875
+L 9.71875 -6.203125
+L 21.390625 -6.203125
+L 21.390625 69
+L 9.71875 69
+L 9.71875 75.984375
+z
+" id="DejaVuSans-93"/>
+ </defs>
+ <use xlink:href="#DejaVuSans-97"/>
+ <use x="61.279297" xlink:href="#DejaVuSans-98"/>
+ <use x="124.755859" xlink:href="#DejaVuSans-115"/>
+ <use x="176.855469" xlink:href="#DejaVuSans-111"/>
+ <use x="238.037109" xlink:href="#DejaVuSans-108"/>
+ <use x="265.820312" xlink:href="#DejaVuSans-117"/>
+ <use x="329.199219" xlink:href="#DejaVuSans-116"/>
+ <use x="368.408203" xlink:href="#DejaVuSans-101"/>
+ <use x="429.931641" xlink:href="#DejaVuSans-32"/>
+ <use x="461.71875" xlink:href="#DejaVuSans-116"/>
+ <use x="500.927734" xlink:href="#DejaVuSans-114"/>
+ <use x="542.041016" xlink:href="#DejaVuSans-97"/>
+ <use x="603.320312" xlink:href="#DejaVuSans-106"/>
+ <use x="631.103516" xlink:href="#DejaVuSans-101"/>
+ <use x="692.626953" xlink:href="#DejaVuSans-99"/>
+ <use x="747.607422" xlink:href="#DejaVuSans-116"/>
+ <use x="786.816406" xlink:href="#DejaVuSans-111"/>
+ <use x="847.998047" xlink:href="#DejaVuSans-114"/>
+ <use x="889.111328" xlink:href="#DejaVuSans-121"/>
+ <use x="948.291016" xlink:href="#DejaVuSans-32"/>
+ <use x="980.078125" xlink:href="#DejaVuSans-101"/>
+ <use x="1041.601562" xlink:href="#DejaVuSans-114"/>
+ <use x="1080.964844" xlink:href="#DejaVuSans-114"/>
+ <use x="1119.828125" xlink:href="#DejaVuSans-111"/>
+ <use x="1181.009766" xlink:href="#DejaVuSans-114"/>
+ <use x="1222.123047" xlink:href="#DejaVuSans-32"/>
+ <use x="1253.910156" xlink:href="#DejaVuSans-91"/>
+ <use x="1292.923828" xlink:href="#DejaVuSans-68"/>
+ <use x="1369.925781" xlink:href="#DejaVuSans-78"/>
+ <use x="1444.730469" xlink:href="#DejaVuSans-83"/>
+ <use x="1508.207031" xlink:href="#DejaVuSans-32"/>
+ <use x="1539.994141" xlink:href="#DejaVuSans-117"/>
+ <use x="1603.373047" xlink:href="#DejaVuSans-110"/>
+ <use x="1666.751953" xlink:href="#DejaVuSans-105"/>
+ <use x="1694.535156" xlink:href="#DejaVuSans-116"/>
+ <use x="1733.744141" xlink:href="#DejaVuSans-115"/>
+ <use x="1785.84375" xlink:href="#DejaVuSans-93"/>
+ </g>
+ </g>
+ </g>
+ <g id="line2d_33">
+ <path clip-path="url(#p4f155a8e51)" d="M 73.823804 23.960364
+L 429.992969 86.137037
+" style="fill:none;stroke:#1f77b4;stroke-linecap:square;stroke-width:1.5;"/>
+ <defs>
+ <path d="M 0 1.5
+C 0.397805 1.5 0.77937 1.341951 1.06066 1.06066
+C 1.341951 0.77937 1.5 0.397805 1.5 0
+C 1.5 -0.397805 1.341951 -0.77937 1.06066 -1.06066
+C 0.77937 -1.341951 0.397805 -1.5 0 -1.5
+C -0.397805 -1.5 -0.77937 -1.341951 -1.06066 -1.06066
+C -1.341951 -0.77937 -1.5 -0.397805 -1.5 0
+C -1.5 0.397805 -1.341951 0.77937 -1.06066 1.06066
+C -0.77937 1.341951 -0.397805 1.5 0 1.5
+z
+" id="m677678fb2c" style="stroke:#1f77b4;"/>
+ </defs>
+ <g clip-path="url(#p4f155a8e51)">
+ <use style="fill:#1f77b4;stroke:#1f77b4;" x="73.823804" xlink:href="#m677678fb2c" y="23.960364"/>
+ <use style="fill:#1f77b4;stroke:#1f77b4;" x="429.992969" xlink:href="#m677678fb2c" y="86.137037"/>
+ </g>
+ </g>
+ <g id="line2d_34">
+ <path clip-path="url(#p4f155a8e51)" d="M 73.823804 179.842942
+L 429.992969 281.091827
+" style="fill:none;stroke:#ff7f0e;stroke-linecap:square;stroke-width:1.5;"/>
+ <defs>
+ <path d="M 0 1.5
+C 0.397805 1.5 0.77937 1.341951 1.06066 1.06066
+C 1.341951 0.77937 1.5 0.397805 1.5 0
+C 1.5 -0.397805 1.341951 -0.77937 1.06066 -1.06066
+C 0.77937 -1.341951 0.397805 -1.5 0 -1.5
+C -0.397805 -1.5 -0.77937 -1.341951 -1.06066 -1.06066
+C -1.341951 -0.77937 -1.5 -0.397805 -1.5 0
+C -1.5 0.397805 -1.341951 0.77937 -1.06066 1.06066
+C -0.77937 1.341951 -0.397805 1.5 0 1.5
+z
+" id="md9d020bb03" style="stroke:#ff7f0e;"/>
+ </defs>
+ <g clip-path="url(#p4f155a8e51)">
+ <use style="fill:#ff7f0e;stroke:#ff7f0e;" x="73.823804" xlink:href="#md9d020bb03" y="179.842942"/>
+ <use style="fill:#ff7f0e;stroke:#ff7f0e;" x="429.992969" xlink:href="#md9d020bb03" y="281.091827"/>
+ </g>
+ </g>
+ <g id="line2d_35">
+ <path clip-path="url(#p4f155a8e51)" d="M 73.823804 45.980935
+L 429.992969 141.908924
+" style="fill:none;stroke:#2ca02c;stroke-linecap:square;stroke-width:1.5;"/>
+ <defs>
+ <path d="M 0 1.5
+C 0.397805 1.5 0.77937 1.341951 1.06066 1.06066
+C 1.341951 0.77937 1.5 0.397805 1.5 0
+C 1.5 -0.397805 1.341951 -0.77937 1.06066 -1.06066
+C 0.77937 -1.341951 0.397805 -1.5 0 -1.5
+C -0.397805 -1.5 -0.77937 -1.341951 -1.06066 -1.06066
+C -1.341951 -0.77937 -1.5 -0.397805 -1.5 0
+C -1.5 0.397805 -1.341951 0.77937 -1.06066 1.06066
+C -0.77937 1.341951 -0.397805 1.5 0 1.5
+z
+" id="mfae7bf72f6" style="stroke:#2ca02c;"/>
+ </defs>
+ <g clip-path="url(#p4f155a8e51)">
+ <use style="fill:#2ca02c;stroke:#2ca02c;" x="73.823804" xlink:href="#mfae7bf72f6" y="45.980935"/>
+ <use style="fill:#2ca02c;stroke:#2ca02c;" x="429.992969" xlink:href="#mfae7bf72f6" y="141.908924"/>
+ </g>
+ </g>
+ <g id="line2d_36">
+ <path clip-path="url(#p4f155a8e51)" d="M 73.823804 188.351626
+L 429.992969 287.167636
+" style="fill:none;stroke:#d62728;stroke-linecap:square;stroke-width:1.5;"/>
+ <defs>
+ <path d="M 0 1.5
+C 0.397805 1.5 0.77937 1.341951 1.06066 1.06066
+C 1.341951 0.77937 1.5 0.397805 1.5 0
+C 1.5 -0.397805 1.341951 -0.77937 1.06066 -1.06066
+C 0.77937 -1.341951 0.397805 -1.5 0 -1.5
+C -0.397805 -1.5 -0.77937 -1.341951 -1.06066 -1.06066
+C -1.341951 -0.77937 -1.5 -0.397805 -1.5 0
+C -1.5 0.397805 -1.341951 0.77937 -1.06066 1.06066
+C -0.77937 1.341951 -0.397805 1.5 0 1.5
+z
+" id="me87b41afb3" style="stroke:#d62728;"/>
+ </defs>
+ <g clip-path="url(#p4f155a8e51)">
+ <use style="fill:#d62728;stroke:#d62728;" x="73.823804" xlink:href="#me87b41afb3" y="188.351626"/>
+ <use style="fill:#d62728;stroke:#d62728;" x="429.992969" xlink:href="#me87b41afb3" y="287.167636"/>
+ </g>
+ </g>
+ <g id="line2d_37">
+ <path clip-path="url(#p4f155a8e51)" d="M 73.823804 133.142634
+L 429.992969 197.665371
+" style="fill:none;stroke:#000000;stroke-dasharray:3,3;stroke-dashoffset:0;stroke-width:1.5;"/>
+ </g>
+ <g id="line2d_38">
+ <path clip-path="url(#p4f155a8e51)" d="M 73.823804 133.142634
+L 429.992969 229.926739
+" style="fill:none;stroke:#000000;stroke-dasharray:4.5,4.5;stroke-dashoffset:0;stroke-width:1.5;"/>
+ </g>
+ <g id="line2d_39">
+ <path clip-path="url(#p4f155a8e51)" d="M 73.823804 133.142634
+L 429.992969 262.188107
+" style="fill:none;stroke:#000000;stroke-dasharray:6,6;stroke-dashoffset:0;stroke-width:1.5;"/>
+ </g>
+ <g id="patch_3">
+ <path d="M 56.015346 300.328
+L 56.015346 10.8
+" style="fill:none;stroke:#000000;stroke-linecap:square;stroke-linejoin:miter;stroke-width:0.8;"/>
+ </g>
+ <g id="patch_4">
+ <path d="M 447.801427 300.328
+L 447.801427 10.8
+" style="fill:none;stroke:#000000;stroke-linecap:square;stroke-linejoin:miter;stroke-width:0.8;"/>
+ </g>
+ <g id="patch_5">
+ <path d="M 56.015346 300.328
+L 447.801427 300.328
+" style="fill:none;stroke:#000000;stroke-linecap:square;stroke-linejoin:miter;stroke-width:0.8;"/>
+ </g>
+ <g id="patch_6">
+ <path d="M 56.015346 10.8
+L 447.801427 10.8
+" style="fill:none;stroke:#000000;stroke-linecap:square;stroke-linejoin:miter;stroke-width:0.8;"/>
+ </g>
+ <g id="legend_1">
+ <g id="patch_7">
+ <path d="M 60.915346 296.828
+L 141.042377 296.828
+Q 142.442377 296.828 142.442377 295.428
+L 142.442377 224.205187
+Q 142.442377 222.805187 141.042377 222.805187
+L 60.915346 222.805187
+Q 59.515346 222.805187 59.515346 224.205187
+L 59.515346 295.428
+Q 59.515346 296.828 60.915346 296.828
+z
+" style="fill:#ffffff;opacity:0.8;stroke:#cccccc;stroke-linejoin:miter;"/>
+ </g>
+ <g id="line2d_40">
+ <path d="M 62.315346 228.474094
+L 76.315346 228.474094
+" style="fill:none;stroke:#1f77b4;stroke-linecap:square;stroke-width:1.5;"/>
+ </g>
+ <g id="line2d_41">
+ <g>
+ <use style="fill:#1f77b4;stroke:#1f77b4;" x="69.315346" xlink:href="#m677678fb2c" y="228.474094"/>
+ </g>
+ </g>
+ <g id="text_12">
+ <!-- max error I4O3 -->
+ <g transform="translate(81.915346 230.924094)scale(0.07 -0.07)">
+ <defs>
+ <path d="M 52 44.1875
+Q 55.375 50.25 60.0625 53.125
+Q 64.75 56 71.09375 56
+Q 79.640625 56 84.28125 50.015625
+Q 88.921875 44.046875 88.921875 33.015625
+L 88.921875 0
+L 79.890625 0
+L 79.890625 32.71875
+Q 79.890625 40.578125 77.09375 44.375
+Q 74.3125 48.1875 68.609375 48.1875
+Q 61.625 48.1875 57.5625 43.546875
+Q 53.515625 38.921875 53.515625 30.90625
+L 53.515625 0
+L 44.484375 0
+L 44.484375 32.71875
+Q 44.484375 40.625 41.703125 44.40625
+Q 38.921875 48.1875 33.109375 48.1875
+Q 26.21875 48.1875 22.15625 43.53125
+Q 18.109375 38.875 18.109375 30.90625
+L 18.109375 0
+L 9.078125 0
+L 9.078125 54.6875
+L 18.109375 54.6875
+L 18.109375 46.1875
+Q 21.1875 51.21875 25.484375 53.609375
+Q 29.78125 56 35.6875 56
+Q 41.65625 56 45.828125 52.96875
+Q 50 49.953125 52 44.1875
+z
+" id="DejaVuSans-109"/>
+ <path d="M 54.890625 54.6875
+L 35.109375 28.078125
+L 55.90625 0
+L 45.3125 0
+L 29.390625 21.484375
+L 13.484375 0
+L 2.875 0
+L 24.125 28.609375
+L 4.6875 54.6875
+L 15.28125 54.6875
+L 29.78125 35.203125
+L 44.28125 54.6875
+z
+" id="DejaVuSans-120"/>
+ <path d="M 9.8125 72.90625
+L 19.671875 72.90625
+L 19.671875 0
+L 9.8125 0
+z
+" id="DejaVuSans-73"/>
+ <path d="M 39.40625 66.21875
+Q 28.65625 66.21875 22.328125 58.203125
+Q 16.015625 50.203125 16.015625 36.375
+Q 16.015625 22.609375 22.328125 14.59375
+Q 28.65625 6.59375 39.40625 6.59375
+Q 50.140625 6.59375 56.421875 14.59375
+Q 62.703125 22.609375 62.703125 36.375
+Q 62.703125 50.203125 56.421875 58.203125
+Q 50.140625 66.21875 39.40625 66.21875
+z
+M 39.40625 74.21875
+Q 54.734375 74.21875 63.90625 63.9375
+Q 73.09375 53.65625 73.09375 36.375
+Q 73.09375 19.140625 63.90625 8.859375
+Q 54.734375 -1.421875 39.40625 -1.421875
+Q 24.03125 -1.421875 14.8125 8.828125
+Q 5.609375 19.09375 5.609375 36.375
+Q 5.609375 53.65625 14.8125 63.9375
+Q 24.03125 74.21875 39.40625 74.21875
+z
+" id="DejaVuSans-79"/>
+ </defs>
+ <use xlink:href="#DejaVuSans-109"/>
+ <use x="97.412109" xlink:href="#DejaVuSans-97"/>
+ <use x="158.691406" xlink:href="#DejaVuSans-120"/>
+ <use x="217.871094" xlink:href="#DejaVuSans-32"/>
+ <use x="249.658203" xlink:href="#DejaVuSans-101"/>
+ <use x="311.181641" xlink:href="#DejaVuSans-114"/>
+ <use x="350.544922" xlink:href="#DejaVuSans-114"/>
+ <use x="389.408203" xlink:href="#DejaVuSans-111"/>
+ <use x="450.589844" xlink:href="#DejaVuSans-114"/>
+ <use x="491.703125" xlink:href="#DejaVuSans-32"/>
+ <use x="523.490234" xlink:href="#DejaVuSans-73"/>
+ <use x="552.982422" xlink:href="#DejaVuSans-52"/>
+ <use x="616.605469" xlink:href="#DejaVuSans-79"/>
+ <use x="695.316406" xlink:href="#DejaVuSans-51"/>
+ </g>
+ </g>
+ <g id="line2d_42">
+ <path d="M 62.315346 238.748781
+L 76.315346 238.748781
+" style="fill:none;stroke:#ff7f0e;stroke-linecap:square;stroke-width:1.5;"/>
+ </g>
+ <g id="line2d_43">
+ <g>
+ <use style="fill:#ff7f0e;stroke:#ff7f0e;" x="69.315346" xlink:href="#md9d020bb03" y="238.748781"/>
+ </g>
+ </g>
+ <g id="text_13">
+ <!-- mean error I4O3 -->
+ <g transform="translate(81.915346 241.198781)scale(0.07 -0.07)">
+ <use xlink:href="#DejaVuSans-109"/>
+ <use x="97.412109" xlink:href="#DejaVuSans-101"/>
+ <use x="158.935547" xlink:href="#DejaVuSans-97"/>
+ <use x="220.214844" xlink:href="#DejaVuSans-110"/>
+ <use x="283.59375" xlink:href="#DejaVuSans-32"/>
+ <use x="315.380859" xlink:href="#DejaVuSans-101"/>
+ <use x="376.904297" xlink:href="#DejaVuSans-114"/>
+ <use x="416.267578" xlink:href="#DejaVuSans-114"/>
+ <use x="455.130859" xlink:href="#DejaVuSans-111"/>
+ <use x="516.3125" xlink:href="#DejaVuSans-114"/>
+ <use x="557.425781" xlink:href="#DejaVuSans-32"/>
+ <use x="589.212891" xlink:href="#DejaVuSans-73"/>
+ <use x="618.705078" xlink:href="#DejaVuSans-52"/>
+ <use x="682.328125" xlink:href="#DejaVuSans-79"/>
+ <use x="761.039062" xlink:href="#DejaVuSans-51"/>
+ </g>
+ </g>
+ <g id="line2d_44">
+ <path d="M 62.315346 249.023469
+L 76.315346 249.023469
+" style="fill:none;stroke:#2ca02c;stroke-linecap:square;stroke-width:1.5;"/>
+ </g>
+ <g id="line2d_45">
+ <g>
+ <use style="fill:#2ca02c;stroke:#2ca02c;" x="69.315346" xlink:href="#mfae7bf72f6" y="249.023469"/>
+ </g>
+ </g>
+ <g id="text_14">
+ <!-- max error I8O5 -->
+ <g transform="translate(81.915346 251.473469)scale(0.07 -0.07)">
+ <use xlink:href="#DejaVuSans-109"/>
+ <use x="97.412109" xlink:href="#DejaVuSans-97"/>
+ <use x="158.691406" xlink:href="#DejaVuSans-120"/>
+ <use x="217.871094" xlink:href="#DejaVuSans-32"/>
+ <use x="249.658203" xlink:href="#DejaVuSans-101"/>
+ <use x="311.181641" xlink:href="#DejaVuSans-114"/>
+ <use x="350.544922" xlink:href="#DejaVuSans-114"/>
+ <use x="389.408203" xlink:href="#DejaVuSans-111"/>
+ <use x="450.589844" xlink:href="#DejaVuSans-114"/>
+ <use x="491.703125" xlink:href="#DejaVuSans-32"/>
+ <use x="523.490234" xlink:href="#DejaVuSans-73"/>
+ <use x="552.982422" xlink:href="#DejaVuSans-56"/>
+ <use x="616.605469" xlink:href="#DejaVuSans-79"/>
+ <use x="695.316406" xlink:href="#DejaVuSans-53"/>
+ </g>
+ </g>
+ <g id="line2d_46">
+ <path d="M 62.315346 259.298156
+L 76.315346 259.298156
+" style="fill:none;stroke:#d62728;stroke-linecap:square;stroke-width:1.5;"/>
+ </g>
+ <g id="line2d_47">
+ <g>
+ <use style="fill:#d62728;stroke:#d62728;" x="69.315346" xlink:href="#me87b41afb3" y="259.298156"/>
+ </g>
+ </g>
+ <g id="text_15">
+ <!-- mean error I8O5 -->
+ <g transform="translate(81.915346 261.748156)scale(0.07 -0.07)">
+ <use xlink:href="#DejaVuSans-109"/>
+ <use x="97.412109" xlink:href="#DejaVuSans-101"/>
+ <use x="158.935547" xlink:href="#DejaVuSans-97"/>
+ <use x="220.214844" xlink:href="#DejaVuSans-110"/>
+ <use x="283.59375" xlink:href="#DejaVuSans-32"/>
+ <use x="315.380859" xlink:href="#DejaVuSans-101"/>
+ <use x="376.904297" xlink:href="#DejaVuSans-114"/>
+ <use x="416.267578" xlink:href="#DejaVuSans-114"/>
+ <use x="455.130859" xlink:href="#DejaVuSans-111"/>
+ <use x="516.3125" xlink:href="#DejaVuSans-114"/>
+ <use x="557.425781" xlink:href="#DejaVuSans-32"/>
+ <use x="589.212891" xlink:href="#DejaVuSans-73"/>
+ <use x="618.705078" xlink:href="#DejaVuSans-56"/>
+ <use x="682.328125" xlink:href="#DejaVuSans-79"/>
+ <use x="761.039062" xlink:href="#DejaVuSans-53"/>
+ </g>
+ </g>
+ <g id="line2d_48">
+ <path d="M 62.315346 269.572844
+L 76.315346 269.572844
+" style="fill:none;stroke:#000000;stroke-dasharray:3,3;stroke-dashoffset:0;stroke-width:1.5;"/>
+ </g>
+ <g id="line2d_49"/>
+ <g id="text_16">
+ <!-- $\propto f^{-2}$ -->
+ <g transform="translate(81.915346 272.022844)scale(0.07 -0.07)">
+ <defs>
+ <path d="M 25.734375 17.71875
+Q 33.796875 17.71875 38.625 29.4375
+Q 32.421875 42.234375 25.734375 42.234375
+Q 20.84375 42.234375 18.359375 38.71875
+Q 15.71875 34.96875 15.71875 29.984375
+Q 15.71875 24.515625 18.359375 21.140625
+Q 21.09375 17.71875 25.734375 17.71875
+z
+M 60.6875 17.625
+L 60.6875 11.1875
+L 56.9375 11.1875
+Q 53.375 11.1875 50.734375 12.640625
+Q 48.09375 13.96875 45.5625 17.046875
+Q 44.046875 18.796875 41.65625 22.90625
+Q 38.375 17.046875 35.5 14.453125
+Q 31.9375 11.328125 26.265625 11.328125
+Q 19.671875 11.328125 15.328125 16.265625
+Q 10.75 21.4375 10.75 29.984375
+Q 10.75 38.1875 15.328125 43.75
+Q 19.390625 48.6875 26.375 48.6875
+Q 29.9375 48.6875 32.5625 47.21875
+Q 35.640625 45.609375 37.75 42.828125
+Q 39.703125 40.328125 41.65625 36.96875
+Q 44.921875 42.828125 47.796875 45.40625
+Q 51.375 48.53125 57.03125 48.53125
+L 60.6875 48.53125
+L 60.6875 42.140625
+L 57.5625 42.140625
+Q 50.34375 42.140625 44.671875 30.421875
+Q 50.875 17.625 57.5625 17.625
+z
+" id="DejaVuSans-8733"/>
+ <path d="M 47.796875 75.984375
+L 46.390625 68.5
+L 37.796875 68.5
+Q 32.90625 68.5 30.6875 66.578125
+Q 28.46875 64.65625 27.390625 59.515625
+L 26.421875 54.6875
+L 41.21875 54.6875
+L 39.890625 47.703125
+L 25.09375 47.703125
+L 15.828125 0
+L 6.78125 0
+L 16.109375 47.703125
+L 7.515625 47.703125
+L 8.796875 54.6875
+L 17.390625 54.6875
+L 18.109375 58.5
+Q 19.96875 68.171875 24.625 72.078125
+Q 29.296875 75.984375 39.3125 75.984375
+z
+" id="DejaVuSans-Oblique-102"/>
+ </defs>
+ <use transform="translate(19.482422 0.765625)" xlink:href="#DejaVuSans-8733"/>
+ <use transform="translate(110.400391 0.765625)" xlink:href="#DejaVuSans-Oblique-102"/>
+ <use transform="translate(153.053177 39.046875)scale(0.7)" xlink:href="#DejaVuSans-8722"/>
+ <use transform="translate(211.705521 39.046875)scale(0.7)" xlink:href="#DejaVuSans-50"/>
+ </g>
+ </g>
+ <g id="line2d_50">
+ <path d="M 62.315346 279.847531
+L 76.315346 279.847531
+" style="fill:none;stroke:#000000;stroke-dasharray:4.5,4.5;stroke-dashoffset:0;stroke-width:1.5;"/>
+ </g>
+ <g id="line2d_51"/>
+ <g id="text_17">
+ <!-- $\propto f^{-3}$ -->
+ <g transform="translate(81.915346 282.297531)scale(0.07 -0.07)">
+ <use transform="translate(19.482422 0.765625)" xlink:href="#DejaVuSans-8733"/>
+ <use transform="translate(110.400391 0.765625)" xlink:href="#DejaVuSans-Oblique-102"/>
+ <use transform="translate(153.053177 39.046875)scale(0.7)" xlink:href="#DejaVuSans-8722"/>
+ <use transform="translate(211.705521 39.046875)scale(0.7)" xlink:href="#DejaVuSans-51"/>
+ </g>
+ </g>
+ <g id="line2d_52">
+ <path d="M 62.315346 290.122219
+L 76.315346 290.122219
+" style="fill:none;stroke:#000000;stroke-dasharray:6,6;stroke-dashoffset:0;stroke-width:1.5;"/>
+ </g>
+ <g id="line2d_53"/>
+ <g id="text_18">
+ <!-- $\propto f^{-4}$ -->
+ <g transform="translate(81.915346 292.572219)scale(0.07 -0.07)">
+ <use transform="translate(19.482422 0.684375)" xlink:href="#DejaVuSans-8733"/>
+ <use transform="translate(110.400391 0.684375)" xlink:href="#DejaVuSans-Oblique-102"/>
+ <use transform="translate(153.053177 38.965625)scale(0.7)" xlink:href="#DejaVuSans-8722"/>
+ <use transform="translate(211.705521 38.965625)scale(0.7)" xlink:href="#DejaVuSans-52"/>
+ </g>
+ </g>
+ </g>
+ </g>
+ </g>
+ <defs>
+ <clipPath id="p4f155a8e51">
+ <rect height="289.528" width="391.786081" x="56.015346" y="10.8"/>
+ </clipPath>
+ </defs>
+</svg>
diff --git a/documentation/sphinx_static/hs.rst b/documentation/sphinx_static/hs.rst
new file mode 100644
index 0000000000000000000000000000000000000000..3978ebb7c9951c1b136a09f865af8f54e70a0e68
--- /dev/null
+++ b/documentation/sphinx_static/hs.rst
@@ -0,0 +1,43 @@
+Tutorial: hydrodynamic similarity
+---------------------------------
+
+run turtle twice with different values of parameters.
+
+
+first run with default parameters
+
+.. code:: bash
+
+ turtle NSE --simname test_1 \
+ --dkx 1.0 \
+ --dky 1.0 \
+ --dkz 1.0 \
+ --nu 0.1 \
+ --injection_rate 0.4 \
+
+
+second, scale parameters by 2
+
+.. code:: bash
+
+ turtle NSE --simname test_2 \
+ --dkx 0.5 \
+ --dky 0.5 \
+ --dkz 0.5 \
+ --nu 0.1*factor \ #FIXME
+ --injection_rate 0.4*forcing_factor #FIXME
+
+
+afterwards, execute the following python code:
+
+.. code:: python
+
+ c1 = NSReader(simname = 'test_1')
+ c1 = NSReader(simname = 'test_2')
+ u1 = c1.read_cvelocity(iter1)
+ u2 = c2.read_cvelocity(iter1)
+
+ diff = np.abs(u1 - u2*vel_factor) / np.abs(u1) # take care of 0s
+
+ assert(diff.max() < 1e-5)
+
diff --git a/documentation/_static/overview.rst b/documentation/sphinx_static/overview.rst
similarity index 53%
rename from documentation/_static/overview.rst
rename to documentation/sphinx_static/overview.rst
index afe7a753666e6ea5911ce1266d0803aa25ea5c45..2c640529f078bb56646e220cec24a7098991a078 100644
--- a/documentation/_static/overview.rst
+++ b/documentation/sphinx_static/overview.rst
@@ -1,6 +1,6 @@
-=====================
-Overview and Tutorial
-=====================
+========
+Overview
+========
----------------
General comments
@@ -8,48 +8,31 @@ General comments
The purpose of this code is to run pseudo-spectral DNS of turbulence,
and integrate particle trajectories in the resulting fields.
-In brief, the main aim of the code is to simplify the launching of
+An important aim of the code is to simplify the launching of
compute jobs and postprocessing, up to and including the generation of
publication-ready figures.
For research, people routinely write code from scratch because research
goals change to a point where modifying the previous code is too
expensive.
-With bfps, the desire is to identify core functionality that should be
+With TurTLE, the desire is to identify core functionality that should be
implemented in a library.
The core library can then be used by many problem-specific codes.
-In this sense, the structuring of the code-base is non-standard.
-The core functionality is implemented in C++ (classes useful for
-describing working with fields or sets of particles), while a python
-wrapper is used for generating "main" programmes to be linked against
-the core library.
-The core library uses MPI for parallelization, and the python wrapper
-compiles this core library when being installed.
-The compilation environment can be configured for different
-machines as required.
+The core functionality is implemented in C++, while a Python3
+wrapper is typically used for generating initial conditions and
+light-weight post-processing.
+The core library uses a hybrid MPI/OpenMP approach for parallelization.
Python3 "wrapper"
-----------------
-In principle, users of the code should only need to use python3 for
+In principle, users of the code should only need to use Python3 for
launching jobs and postprocessing data.
-While python2 compatibility should not be too hard to maintain, the
-usage of strings makes it a bit cumbersome ---
-the code makes extensive usage of strings for `HDF5` I/O.
-Classes defined in the python package can be used to generate executable
+Classes defined in the Python3 package can be used to generate executable
codes, compile/launch them, and then for accessing and postprocessing
-data.
-Obviously, postprocessing methods can be optimized with C extensions or
-otherwise, as needed.
-
-Code generation is quite straightforward, with C++ code snippets handled
-as strings in the python code, such that they can be combined in
-different ways.
-
-Once a "main" file has been written, it is compiled and linked against
-the core library.
+data with a full Python3 environment.
Depending on machine-specific settings, the code can then be launched
directly, or job scripts appropriate for queueing systems are generated
and submitted.
@@ -57,9 +40,22 @@ and submitted.
C++ core library
----------------
-A small set of base classes are implemented.
-
-[ some details to be added here ]
+As already stated, TurTLE's main goal is to facilitate novel research
+avenues.
+To this end, there are two types of simple objects.
+Firstly, children of an abstract class that encapsulates three elements: generic
+*initialization*, *do work* and *finalization* functionality.
+Secondly, essential data structures (i.e. fields, sets of particles) and
+associated functionality (i.e. I/O) are provided by "building block"-classes.
+Each TurTLE "solver" then consists of a specific arrangement of the building
+blocks.
+
+The heterogeneous TurTLE development team benefits from the separation of
+generic functionality from building blocks:
+TurTLE is naturally well-suited to the distribution of conceptually distinct
+work, in particular fully isolating projects such as, e.g., "implement new
+numerical method" from "optimize the computation of field statistics with
+OpenMP".
---------
Equations
@@ -67,15 +63,16 @@ Equations
The code uses a fairly standard pseudo-spectral algorithm to solve fluid
equations.
-The incompressible Navier Stokes equations in velocity form are as
-follows:
+The incompressible Navier Stokes equations in velocity form (see the
+`NSE` C++ class) are as follows:
.. math::
\partial_t \mathbf{u} + \mathbf{u} \cdot \nabla \mathbf{u} =
- \nabla p + \nu \Delta \mathbf{u} + \mathbf{f}
-In fact, the code solves the vorticity formulation of these equations:
+TurTLE also solves the vorticity formulation of these equations (see the
+`NSVE` C++ class):
.. math::
\partial_t \mathbf{\omega} +
@@ -97,24 +94,15 @@ following:
\varepsilon_{\textrm{inj}} = \sum_{\mathbf{k}} \hat{\mathbf{u}} \cdot \hat{\mathbf{f}}^*
-In fact, C++ code generated by
-:class:`NavierStokes <bfps.NavierStokes.NavierStokes>`
-computes and stores the velocity
-and vorticity cospectra (9 components each):
+In fact we store a proxy for the energy spectrum tensor:
.. math::
\sum_{k \leq \|\mathbf{k}\| \leq k+dk}\hat{u_i} \cdot \hat{u_j}^*, \hskip .5cm
\sum_{k \leq \|\mathbf{k}\| \leq k+dk}\hat{\omega_i} \cdot \hat{\omega_j}^*
-In all honesty, this is overkill for homogenous and isotropic flows, but
-in principle we will look at more complicated flows.
-
-See :func:`compute_statistics <bfps.NavierStokes.NavierStokes.compute_statistics>`
-and
-:func:`compute_time_averages <bfps.NavierStokes.NavierStokes.compute_time_averages>`
-for quantities
-computed in postprocessing by the python code.
+The word "proxy" is used because technically there are normalization
+factors (specifically a Dirac delta) that need to be accounted for.
-----------
Conventions
@@ -139,65 +127,57 @@ memory (both in the C++ backend and in Python postprocessing):
components, as ``FFTW`` requires for the correspondence with the
real space representations.
-:func:`read_cfield <bfps.NavierStokes.NavierStokes.read_cfield>` will return
-a properly shaped ``numpy.array`` containing a snapshot of the Fourier
-representation of a 3D field.
-
-If you'd like to construct the corresponding wave numbers, you can
-follow this procedure:
+We recommend the following procedure to read the corresponding
+wavenumbers:
.. code:: python
import numpy as np
- from bfps import NavierStokes
+ from TurTLE.DNS import DNS
- c = NavierStokes(
+ c = DNS(
work_dir = '/location/of/simulation/data',
simname = 'simulation_name_goes_here')
+
df = c.get_data_file()
- kx = df['kspace/kx'].value
- ky = df['kspace/ky'].value
- kz = df['kspace/kz'].value
+ kx = df['kspace/kx'][()]
+ ky = df['kspace/ky'][()]
+ kz = df['kspace/kz'][()]
df.close()
- kval = np.zeros(kz.shape + ky.shape + kx.shape + (3,),
- dtype = kx.dtype)
+ # optional: build full 3D field of k vector
+ kval = np.zeros(
+ kz.shape + ky.shape + kx.shape + (3,),
+ dtype = kx.dtype)
kval[..., 0] = kx[None, None, :]
kval[..., 1] = ky[:, None, None]
kval[..., 2] = kz[None, :, None]
-``kval`` will have the same shape as the result of
-:func:`read_cfield <NavierStokes.NavierStokes.read_cfield>`.
-Obviously, the machine being used should have enough RAM to hold the
-field...
-
---------
-Tutorial
---------
+----------------
+Tutorial: basics
+----------------
First DNS
---------
-Installing ``bfps`` is not trivial, and the instructions are in
-:ref:`sec-installation`.
-After installing, you should have a new executable script
-available, called ``bfps``, that you can execute.
-Just executing it will run a small test DNS on a real space grid of size
-:math:`32 \times 32 \times 32`, in the current
-folder, with the simulation name ``test``.
-So, open a console, and type ``bfps NavierStokes``:
+As a first step, we will run a small :math:`32^3` simulation starting
+from random initial conditions, and have a look at the results.
.. code:: bash
# depending on how curious you are, you may have a look at the
# options first:
- bfps --help
- bfps NavierStokes --help
+ turtle --help
+ turtle DNS --help
+ turtle DNS NSVE --help
# or you may just run it:
- bfps NavierStokes
+ turtle DNS NSVE -n 32
+This launches a simulation in the current folder, with the simulation
+name "test".
The simulation itself should not take more than a few seconds, since
this is just a :math:`32^3` simulation run for 8 iterations.
-First thing you can do afterwards is open up a python console, and type
+The prior compilation step may take a bit longer.
+First thing you can do afterwards is open a python console, and type
the following:
.. _sec-first-postprocessing:
@@ -205,11 +185,11 @@ the following:
.. code:: python
import numpy as np
- from bfps import NavierStokes
+ from TurTLE.DNS import DNS
- c = NavierStokes(
- work_dir = '/location/of/simulation/data',
- simname = 'simulation_name_goes_here')
+ c = DNS(
+ work_dir = './',
+ simname = 'test')
c.compute_statistics()
print ('Rlambda = {0:.0f}, kMeta = {1:.4f}, CFL = {2:.4f}'.format(
c.statistics['Rlambda'],
@@ -223,19 +203,19 @@ the following:
data_file['iteration'].value*c.parameters['dt'] / c.statistics['Tint'],
data_file['iteration'].value*c.parameters['dt'] / c.statistics['tauK']))
-:func:`compute_statistics <bfps.NavierStokes.NavierStokes.compute_statistics>`
+:func:`compute_statistics <TurTLE.DNS.DNS.compute_statistics>`
will read the data
-file generated by the DNS, compute a bunch of basic statistics, for
+file, and it will compute a bunch of basic statistics, for
example the Taylor scale Reynolds number :math:`R_\lambda` that we're
printing in the example code.
What happens is that the DNS will have generated an ``HDF5`` file
-containing a bunch of specific datasets (spectra, moments of real space
+containing specific datasets (spectra, moments of real space
representations, etc).
The function
-:func:`compute_statistics <bfps.NavierStokes.NavierStokes.compute_statistics>`
+:func:`compute_statistics <TurTLE.DNS.DNS.compute_statistics>`
performs simple postprocessing that may however be expensive, therefore
-it also saves some data into a ``<simname>_postprocess.h5`` file, and
+it also saves some data into a ``<simname>_cache.h5`` file, and
then it also performs some time averages, yielding the ``statistics``
dictionary that is used in the above code.
@@ -245,24 +225,58 @@ Behind the scenes
In brief the following takes place:
1. An instance ``c`` of
- :class:`NavierStokes <bfps.NavierStokes.NavierStokes>` is created.
+ :class:`DNS <TurTLE.DNS.DNS>` is created.
It is used to generate an :class:`argparse.ArgumentParser`, and
- it processes command line arguments given to the ``bfps
- NavierStokes`` command.
- 2. reasonable DNS parameters are constructed from the command line
+ it processes command line arguments given to the ``turtle
+ DNS NSVE`` command.
+ 2. Reasonable DNS parameters are constructed from the command line
arguments.
- 4. ``c`` generates a parameter file ``<simname>.h5``, into which the
+ 3. ``c`` generates a parameter file ``<simname>.h5``, into which the
various parameters are written.
``c`` also generates the various datasets that the backend code
- will write into (statistics and other stuff).
- 3. ``c`` writes a C++ file that is compiled and linked against
- ``libbfps``.
- 4. ``c`` executes the C++ code using ``mpirun``.
- 5. the C++ code actually performs the DNS, and outputs various
- results into the ``<simname>.h5`` file.
+ will write into (statistics).
+ 4. ``c`` writes a simple C++ file that is compiled and linked against
+ ``libTurTLE`` (to be specific, it calls the :code:`main_code`
+ function defined in :code:`cpp/full_code/main_code.hpp`).
+ 5. ``c`` executes the C++ code using the appropriate launcher (e.g. ``mpiexec``).
+ 6. the C++ code actually performs the DNS, and outputs various
+ results into the ``<simname>.h5`` file. It also outputs the final
+ state of the vorticity field into an appropriate checkpoint file
+ (also in HDF5 format).
After the simulation is done, things are simpler.
In fact, any ``HDF5`` capable software can be used to read the data
file, and the dataset names should be reasonably easy to interpret, so
custom postprocessing codes can easily be generated.
+
+-------------
+Scaling tests
+-------------
+
+Our own scaling tests are reported in [lalescu2021arXiv]_.
+
+Initial scaling data is available at (url coming soon).
+For now you need to run a preliminary DNS of 8192 iterations using a
+grid of :math:`128^3` points.
+Please copy the files to the location `TURTLE_FIELD_DATABASE`.
+
+Separately, please recompile TurTLE with the `TIMING_OUTPUT` cmake
+option switched to `ON`.
+
+Afterwards, please run variations of the following command:
+
+ .. code:: bash
+
+ python ${TURTLE_REPOSITORY}/tests/DNS/test_scaling.py D \
+ -n 128 \
+ --nprocesses 4 \
+ --ncores 1 \
+ --src-wd ${TURTLE_FIELD_DATABASE} \
+ --src-iteration 8192
+
+..
+ Available iterations for
+
+ * n = 128: 8192
+
diff --git a/documentation/sphinx_static/trajectories.svg b/documentation/sphinx_static/trajectories.svg
new file mode 100644
index 0000000000000000000000000000000000000000..5c6ec9a6652a090271b127425c87958f09d77888
--- /dev/null
+++ b/documentation/sphinx_static/trajectories.svg
@@ -0,0 +1,3262 @@
+<?xml version="1.0" encoding="utf-8" standalone="no"?>
+<!DOCTYPE svg PUBLIC "-//W3C//DTD SVG 1.1//EN"
+ "http://www.w3.org/Graphics/SVG/1.1/DTD/svg11.dtd">
+<!-- Created with matplotlib (https://matplotlib.org/) -->
+<svg height="345.6pt" version="1.1" viewBox="0 0 460.8 345.6" width="460.8pt" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink">
+ <metadata>
+ <rdf:RDF xmlns:cc="http://creativecommons.org/ns#" xmlns:dc="http://purl.org/dc/elements/1.1/" xmlns:rdf="http://www.w3.org/1999/02/22-rdf-syntax-ns#">
+ <cc:Work>
+ <dc:type rdf:resource="http://purl.org/dc/dcmitype/StillImage"/>
+ <dc:date>2022-01-20T07:20:04.265273</dc:date>
+ <dc:format>image/svg+xml</dc:format>
+ <dc:creator>
+ <cc:Agent>
+ <dc:title>Matplotlib v3.3.4, https://matplotlib.org/</dc:title>
+ </cc:Agent>
+ </dc:creator>
+ </cc:Work>
+ </rdf:RDF>
+ </metadata>
+ <defs>
+ <style type="text/css">*{stroke-linecap:butt;stroke-linejoin:round;}</style>
+ </defs>
+ <g id="figure_1">
+ <g id="patch_1">
+ <path d="M 0 345.6
+L 460.8 345.6
+L 460.8 0
+L 0 0
+z
+" style="fill:#ffffff;"/>
+ </g>
+ <g id="axes_1">
+ <g id="patch_2">
+ <path d="M 50.69 301.9975
+L 450 301.9975
+L 450 10.8
+L 50.69 10.8
+z
+" style="fill:#ffffff;"/>
+ </g>
+ <g id="matplotlib.axis_1">
+ <g id="xtick_1">
+ <g id="line2d_1">
+ <defs>
+ <path d="M 0 0
+L 0 3.5
+" id="m78350b8f3e" style="stroke:#000000;stroke-width:0.8;"/>
+ </defs>
+ <g>
+ <use style="stroke:#000000;stroke-width:0.8;" x="92.807052" xlink:href="#m78350b8f3e" y="301.9975"/>
+ </g>
+ </g>
+ <g id="text_1">
+ <!-- 1.3 -->
+ <g transform="translate(84.85549 316.595937)scale(0.1 -0.1)">
+ <defs>
+ <path d="M 12.40625 8.296875
+L 28.515625 8.296875
+L 28.515625 63.921875
+L 10.984375 60.40625
+L 10.984375 69.390625
+L 28.421875 72.90625
+L 38.28125 72.90625
+L 38.28125 8.296875
+L 54.390625 8.296875
+L 54.390625 0
+L 12.40625 0
+z
+" id="DejaVuSans-49"/>
+ <path d="M 10.6875 12.40625
+L 21 12.40625
+L 21 0
+L 10.6875 0
+z
+" id="DejaVuSans-46"/>
+ <path d="M 40.578125 39.3125
+Q 47.65625 37.796875 51.625 33
+Q 55.609375 28.21875 55.609375 21.1875
+Q 55.609375 10.40625 48.1875 4.484375
+Q 40.765625 -1.421875 27.09375 -1.421875
+Q 22.515625 -1.421875 17.65625 -0.515625
+Q 12.796875 0.390625 7.625 2.203125
+L 7.625 11.71875
+Q 11.71875 9.328125 16.59375 8.109375
+Q 21.484375 6.890625 26.8125 6.890625
+Q 36.078125 6.890625 40.9375 10.546875
+Q 45.796875 14.203125 45.796875 21.1875
+Q 45.796875 27.640625 41.28125 31.265625
+Q 36.765625 34.90625 28.71875 34.90625
+L 20.21875 34.90625
+L 20.21875 43.015625
+L 29.109375 43.015625
+Q 36.375 43.015625 40.234375 45.921875
+Q 44.09375 48.828125 44.09375 54.296875
+Q 44.09375 59.90625 40.109375 62.90625
+Q 36.140625 65.921875 28.71875 65.921875
+Q 24.65625 65.921875 20.015625 65.03125
+Q 15.375 64.15625 9.8125 62.3125
+L 9.8125 71.09375
+Q 15.4375 72.65625 20.34375 73.4375
+Q 25.25 74.21875 29.59375 74.21875
+Q 40.828125 74.21875 47.359375 69.109375
+Q 53.90625 64.015625 53.90625 55.328125
+Q 53.90625 49.265625 50.4375 45.09375
+Q 46.96875 40.921875 40.578125 39.3125
+z
+" id="DejaVuSans-51"/>
+ </defs>
+ <use xlink:href="#DejaVuSans-49"/>
+ <use x="63.623047" xlink:href="#DejaVuSans-46"/>
+ <use x="95.410156" xlink:href="#DejaVuSans-51"/>
+ </g>
+ </g>
+ </g>
+ <g id="xtick_2">
+ <g id="line2d_2">
+ <g>
+ <use style="stroke:#000000;stroke-width:0.8;" x="169.734825" xlink:href="#m78350b8f3e" y="301.9975"/>
+ </g>
+ </g>
+ <g id="text_2">
+ <!-- 1.4 -->
+ <g transform="translate(161.783262 316.595937)scale(0.1 -0.1)">
+ <defs>
+ <path d="M 37.796875 64.3125
+L 12.890625 25.390625
+L 37.796875 25.390625
+z
+M 35.203125 72.90625
+L 47.609375 72.90625
+L 47.609375 25.390625
+L 58.015625 25.390625
+L 58.015625 17.1875
+L 47.609375 17.1875
+L 47.609375 0
+L 37.796875 0
+L 37.796875 17.1875
+L 4.890625 17.1875
+L 4.890625 26.703125
+z
+" id="DejaVuSans-52"/>
+ </defs>
+ <use xlink:href="#DejaVuSans-49"/>
+ <use x="63.623047" xlink:href="#DejaVuSans-46"/>
+ <use x="95.410156" xlink:href="#DejaVuSans-52"/>
+ </g>
+ </g>
+ </g>
+ <g id="xtick_3">
+ <g id="line2d_3">
+ <g>
+ <use style="stroke:#000000;stroke-width:0.8;" x="246.662597" xlink:href="#m78350b8f3e" y="301.9975"/>
+ </g>
+ </g>
+ <g id="text_3">
+ <!-- 1.5 -->
+ <g transform="translate(238.711035 316.595937)scale(0.1 -0.1)">
+ <defs>
+ <path d="M 10.796875 72.90625
+L 49.515625 72.90625
+L 49.515625 64.59375
+L 19.828125 64.59375
+L 19.828125 46.734375
+Q 21.96875 47.46875 24.109375 47.828125
+Q 26.265625 48.1875 28.421875 48.1875
+Q 40.625 48.1875 47.75 41.5
+Q 54.890625 34.8125 54.890625 23.390625
+Q 54.890625 11.625 47.5625 5.09375
+Q 40.234375 -1.421875 26.90625 -1.421875
+Q 22.3125 -1.421875 17.546875 -0.640625
+Q 12.796875 0.140625 7.71875 1.703125
+L 7.71875 11.625
+Q 12.109375 9.234375 16.796875 8.0625
+Q 21.484375 6.890625 26.703125 6.890625
+Q 35.15625 6.890625 40.078125 11.328125
+Q 45.015625 15.765625 45.015625 23.390625
+Q 45.015625 31 40.078125 35.4375
+Q 35.15625 39.890625 26.703125 39.890625
+Q 22.75 39.890625 18.8125 39.015625
+Q 14.890625 38.140625 10.796875 36.28125
+z
+" id="DejaVuSans-53"/>
+ </defs>
+ <use xlink:href="#DejaVuSans-49"/>
+ <use x="63.623047" xlink:href="#DejaVuSans-46"/>
+ <use x="95.410156" xlink:href="#DejaVuSans-53"/>
+ </g>
+ </g>
+ </g>
+ <g id="xtick_4">
+ <g id="line2d_4">
+ <g>
+ <use style="stroke:#000000;stroke-width:0.8;" x="323.59037" xlink:href="#m78350b8f3e" y="301.9975"/>
+ </g>
+ </g>
+ <g id="text_4">
+ <!-- 1.6 -->
+ <g transform="translate(315.638807 316.595937)scale(0.1 -0.1)">
+ <defs>
+ <path d="M 33.015625 40.375
+Q 26.375 40.375 22.484375 35.828125
+Q 18.609375 31.296875 18.609375 23.390625
+Q 18.609375 15.53125 22.484375 10.953125
+Q 26.375 6.390625 33.015625 6.390625
+Q 39.65625 6.390625 43.53125 10.953125
+Q 47.40625 15.53125 47.40625 23.390625
+Q 47.40625 31.296875 43.53125 35.828125
+Q 39.65625 40.375 33.015625 40.375
+z
+M 52.59375 71.296875
+L 52.59375 62.3125
+Q 48.875 64.0625 45.09375 64.984375
+Q 41.3125 65.921875 37.59375 65.921875
+Q 27.828125 65.921875 22.671875 59.328125
+Q 17.53125 52.734375 16.796875 39.40625
+Q 19.671875 43.65625 24.015625 45.921875
+Q 28.375 48.1875 33.59375 48.1875
+Q 44.578125 48.1875 50.953125 41.515625
+Q 57.328125 34.859375 57.328125 23.390625
+Q 57.328125 12.15625 50.6875 5.359375
+Q 44.046875 -1.421875 33.015625 -1.421875
+Q 20.359375 -1.421875 13.671875 8.265625
+Q 6.984375 17.96875 6.984375 36.375
+Q 6.984375 53.65625 15.1875 63.9375
+Q 23.390625 74.21875 37.203125 74.21875
+Q 40.921875 74.21875 44.703125 73.484375
+Q 48.484375 72.75 52.59375 71.296875
+z
+" id="DejaVuSans-54"/>
+ </defs>
+ <use xlink:href="#DejaVuSans-49"/>
+ <use x="63.623047" xlink:href="#DejaVuSans-46"/>
+ <use x="95.410156" xlink:href="#DejaVuSans-54"/>
+ </g>
+ </g>
+ </g>
+ <g id="xtick_5">
+ <g id="line2d_5">
+ <g>
+ <use style="stroke:#000000;stroke-width:0.8;" x="400.518142" xlink:href="#m78350b8f3e" y="301.9975"/>
+ </g>
+ </g>
+ <g id="text_5">
+ <!-- 1.7 -->
+ <g transform="translate(392.566579 316.595937)scale(0.1 -0.1)">
+ <defs>
+ <path d="M 8.203125 72.90625
+L 55.078125 72.90625
+L 55.078125 68.703125
+L 28.609375 0
+L 18.3125 0
+L 43.21875 64.59375
+L 8.203125 64.59375
+z
+" id="DejaVuSans-55"/>
+ </defs>
+ <use xlink:href="#DejaVuSans-49"/>
+ <use x="63.623047" xlink:href="#DejaVuSans-46"/>
+ <use x="95.410156" xlink:href="#DejaVuSans-55"/>
+ </g>
+ </g>
+ </g>
+ <g id="text_6">
+ <!-- $x$ [code units] -->
+ <g transform="translate(215.945 330.274062)scale(0.1 -0.1)">
+ <defs>
+ <path d="M 60.015625 54.6875
+L 34.90625 27.875
+L 50.296875 0
+L 39.984375 0
+L 28.421875 21.6875
+L 8.296875 0
+L -2.59375 0
+L 24.3125 28.8125
+L 10.015625 54.6875
+L 20.3125 54.6875
+L 30.8125 34.90625
+L 49.125 54.6875
+z
+" id="DejaVuSans-Oblique-120"/>
+ <path id="DejaVuSans-32"/>
+ <path d="M 8.59375 75.984375
+L 29.296875 75.984375
+L 29.296875 69
+L 17.578125 69
+L 17.578125 -6.203125
+L 29.296875 -6.203125
+L 29.296875 -13.1875
+L 8.59375 -13.1875
+z
+" id="DejaVuSans-91"/>
+ <path d="M 48.78125 52.59375
+L 48.78125 44.1875
+Q 44.96875 46.296875 41.140625 47.34375
+Q 37.3125 48.390625 33.40625 48.390625
+Q 24.65625 48.390625 19.8125 42.84375
+Q 14.984375 37.3125 14.984375 27.296875
+Q 14.984375 17.28125 19.8125 11.734375
+Q 24.65625 6.203125 33.40625 6.203125
+Q 37.3125 6.203125 41.140625 7.25
+Q 44.96875 8.296875 48.78125 10.40625
+L 48.78125 2.09375
+Q 45.015625 0.34375 40.984375 -0.53125
+Q 36.96875 -1.421875 32.421875 -1.421875
+Q 20.0625 -1.421875 12.78125 6.34375
+Q 5.515625 14.109375 5.515625 27.296875
+Q 5.515625 40.671875 12.859375 48.328125
+Q 20.21875 56 33.015625 56
+Q 37.15625 56 41.109375 55.140625
+Q 45.0625 54.296875 48.78125 52.59375
+z
+" id="DejaVuSans-99"/>
+ <path d="M 30.609375 48.390625
+Q 23.390625 48.390625 19.1875 42.75
+Q 14.984375 37.109375 14.984375 27.296875
+Q 14.984375 17.484375 19.15625 11.84375
+Q 23.34375 6.203125 30.609375 6.203125
+Q 37.796875 6.203125 41.984375 11.859375
+Q 46.1875 17.53125 46.1875 27.296875
+Q 46.1875 37.015625 41.984375 42.703125
+Q 37.796875 48.390625 30.609375 48.390625
+z
+M 30.609375 56
+Q 42.328125 56 49.015625 48.375
+Q 55.71875 40.765625 55.71875 27.296875
+Q 55.71875 13.875 49.015625 6.21875
+Q 42.328125 -1.421875 30.609375 -1.421875
+Q 18.84375 -1.421875 12.171875 6.21875
+Q 5.515625 13.875 5.515625 27.296875
+Q 5.515625 40.765625 12.171875 48.375
+Q 18.84375 56 30.609375 56
+z
+" id="DejaVuSans-111"/>
+ <path d="M 45.40625 46.390625
+L 45.40625 75.984375
+L 54.390625 75.984375
+L 54.390625 0
+L 45.40625 0
+L 45.40625 8.203125
+Q 42.578125 3.328125 38.25 0.953125
+Q 33.9375 -1.421875 27.875 -1.421875
+Q 17.96875 -1.421875 11.734375 6.484375
+Q 5.515625 14.40625 5.515625 27.296875
+Q 5.515625 40.1875 11.734375 48.09375
+Q 17.96875 56 27.875 56
+Q 33.9375 56 38.25 53.625
+Q 42.578125 51.265625 45.40625 46.390625
+z
+M 14.796875 27.296875
+Q 14.796875 17.390625 18.875 11.75
+Q 22.953125 6.109375 30.078125 6.109375
+Q 37.203125 6.109375 41.296875 11.75
+Q 45.40625 17.390625 45.40625 27.296875
+Q 45.40625 37.203125 41.296875 42.84375
+Q 37.203125 48.484375 30.078125 48.484375
+Q 22.953125 48.484375 18.875 42.84375
+Q 14.796875 37.203125 14.796875 27.296875
+z
+" id="DejaVuSans-100"/>
+ <path d="M 56.203125 29.59375
+L 56.203125 25.203125
+L 14.890625 25.203125
+Q 15.484375 15.921875 20.484375 11.0625
+Q 25.484375 6.203125 34.421875 6.203125
+Q 39.59375 6.203125 44.453125 7.46875
+Q 49.3125 8.734375 54.109375 11.28125
+L 54.109375 2.78125
+Q 49.265625 0.734375 44.1875 -0.34375
+Q 39.109375 -1.421875 33.890625 -1.421875
+Q 20.796875 -1.421875 13.15625 6.1875
+Q 5.515625 13.8125 5.515625 26.8125
+Q 5.515625 40.234375 12.765625 48.109375
+Q 20.015625 56 32.328125 56
+Q 43.359375 56 49.78125 48.890625
+Q 56.203125 41.796875 56.203125 29.59375
+z
+M 47.21875 32.234375
+Q 47.125 39.59375 43.09375 43.984375
+Q 39.0625 48.390625 32.421875 48.390625
+Q 24.90625 48.390625 20.390625 44.140625
+Q 15.875 39.890625 15.1875 32.171875
+z
+" id="DejaVuSans-101"/>
+ <path d="M 8.5 21.578125
+L 8.5 54.6875
+L 17.484375 54.6875
+L 17.484375 21.921875
+Q 17.484375 14.15625 20.5 10.265625
+Q 23.53125 6.390625 29.59375 6.390625
+Q 36.859375 6.390625 41.078125 11.03125
+Q 45.3125 15.671875 45.3125 23.6875
+L 45.3125 54.6875
+L 54.296875 54.6875
+L 54.296875 0
+L 45.3125 0
+L 45.3125 8.40625
+Q 42.046875 3.421875 37.71875 1
+Q 33.40625 -1.421875 27.6875 -1.421875
+Q 18.265625 -1.421875 13.375 4.4375
+Q 8.5 10.296875 8.5 21.578125
+z
+M 31.109375 56
+z
+" id="DejaVuSans-117"/>
+ <path d="M 54.890625 33.015625
+L 54.890625 0
+L 45.90625 0
+L 45.90625 32.71875
+Q 45.90625 40.484375 42.875 44.328125
+Q 39.84375 48.1875 33.796875 48.1875
+Q 26.515625 48.1875 22.3125 43.546875
+Q 18.109375 38.921875 18.109375 30.90625
+L 18.109375 0
+L 9.078125 0
+L 9.078125 54.6875
+L 18.109375 54.6875
+L 18.109375 46.1875
+Q 21.34375 51.125 25.703125 53.5625
+Q 30.078125 56 35.796875 56
+Q 45.21875 56 50.046875 50.171875
+Q 54.890625 44.34375 54.890625 33.015625
+z
+" id="DejaVuSans-110"/>
+ <path d="M 9.421875 54.6875
+L 18.40625 54.6875
+L 18.40625 0
+L 9.421875 0
+z
+M 9.421875 75.984375
+L 18.40625 75.984375
+L 18.40625 64.59375
+L 9.421875 64.59375
+z
+" id="DejaVuSans-105"/>
+ <path d="M 18.3125 70.21875
+L 18.3125 54.6875
+L 36.8125 54.6875
+L 36.8125 47.703125
+L 18.3125 47.703125
+L 18.3125 18.015625
+Q 18.3125 11.328125 20.140625 9.421875
+Q 21.96875 7.515625 27.59375 7.515625
+L 36.8125 7.515625
+L 36.8125 0
+L 27.59375 0
+Q 17.1875 0 13.234375 3.875
+Q 9.28125 7.765625 9.28125 18.015625
+L 9.28125 47.703125
+L 2.6875 47.703125
+L 2.6875 54.6875
+L 9.28125 54.6875
+L 9.28125 70.21875
+z
+" id="DejaVuSans-116"/>
+ <path d="M 44.28125 53.078125
+L 44.28125 44.578125
+Q 40.484375 46.53125 36.375 47.5
+Q 32.28125 48.484375 27.875 48.484375
+Q 21.1875 48.484375 17.84375 46.4375
+Q 14.5 44.390625 14.5 40.28125
+Q 14.5 37.15625 16.890625 35.375
+Q 19.28125 33.59375 26.515625 31.984375
+L 29.59375 31.296875
+Q 39.15625 29.25 43.1875 25.515625
+Q 47.21875 21.78125 47.21875 15.09375
+Q 47.21875 7.46875 41.1875 3.015625
+Q 35.15625 -1.421875 24.609375 -1.421875
+Q 20.21875 -1.421875 15.453125 -0.5625
+Q 10.6875 0.296875 5.421875 2
+L 5.421875 11.28125
+Q 10.40625 8.6875 15.234375 7.390625
+Q 20.0625 6.109375 24.8125 6.109375
+Q 31.15625 6.109375 34.5625 8.28125
+Q 37.984375 10.453125 37.984375 14.40625
+Q 37.984375 18.0625 35.515625 20.015625
+Q 33.0625 21.96875 24.703125 23.78125
+L 21.578125 24.515625
+Q 13.234375 26.265625 9.515625 29.90625
+Q 5.8125 33.546875 5.8125 39.890625
+Q 5.8125 47.609375 11.28125 51.796875
+Q 16.75 56 26.8125 56
+Q 31.78125 56 36.171875 55.265625
+Q 40.578125 54.546875 44.28125 53.078125
+z
+" id="DejaVuSans-115"/>
+ <path d="M 30.421875 75.984375
+L 30.421875 -13.1875
+L 9.71875 -13.1875
+L 9.71875 -6.203125
+L 21.390625 -6.203125
+L 21.390625 69
+L 9.71875 69
+L 9.71875 75.984375
+z
+" id="DejaVuSans-93"/>
+ </defs>
+ <use transform="translate(0 0.015625)" xlink:href="#DejaVuSans-Oblique-120"/>
+ <use transform="translate(59.179688 0.015625)" xlink:href="#DejaVuSans-32"/>
+ <use transform="translate(90.966797 0.015625)" xlink:href="#DejaVuSans-91"/>
+ <use transform="translate(129.980469 0.015625)" xlink:href="#DejaVuSans-99"/>
+ <use transform="translate(184.960938 0.015625)" xlink:href="#DejaVuSans-111"/>
+ <use transform="translate(246.142578 0.015625)" xlink:href="#DejaVuSans-100"/>
+ <use transform="translate(309.619141 0.015625)" xlink:href="#DejaVuSans-101"/>
+ <use transform="translate(371.142578 0.015625)" xlink:href="#DejaVuSans-32"/>
+ <use transform="translate(402.929688 0.015625)" xlink:href="#DejaVuSans-117"/>
+ <use transform="translate(466.308594 0.015625)" xlink:href="#DejaVuSans-110"/>
+ <use transform="translate(529.6875 0.015625)" xlink:href="#DejaVuSans-105"/>
+ <use transform="translate(557.470703 0.015625)" xlink:href="#DejaVuSans-116"/>
+ <use transform="translate(596.679688 0.015625)" xlink:href="#DejaVuSans-115"/>
+ <use transform="translate(648.779297 0.015625)" xlink:href="#DejaVuSans-93"/>
+ </g>
+ </g>
+ </g>
+ <g id="matplotlib.axis_2">
+ <g id="ytick_1">
+ <g id="line2d_6">
+ <defs>
+ <path d="M 0 0
+L -3.5 0
+" id="m64c81340d8" style="stroke:#000000;stroke-width:0.8;"/>
+ </defs>
+ <g>
+ <use style="stroke:#000000;stroke-width:0.8;" x="50.69" xlink:href="#m64c81340d8" y="265.633067"/>
+ </g>
+ </g>
+ <g id="text_7">
+ <!-- 5.2 -->
+ <g transform="translate(27.786875 269.432285)scale(0.1 -0.1)">
+ <defs>
+ <path d="M 19.1875 8.296875
+L 53.609375 8.296875
+L 53.609375 0
+L 7.328125 0
+L 7.328125 8.296875
+Q 12.9375 14.109375 22.625 23.890625
+Q 32.328125 33.6875 34.8125 36.53125
+Q 39.546875 41.84375 41.421875 45.53125
+Q 43.3125 49.21875 43.3125 52.78125
+Q 43.3125 58.59375 39.234375 62.25
+Q 35.15625 65.921875 28.609375 65.921875
+Q 23.96875 65.921875 18.8125 64.3125
+Q 13.671875 62.703125 7.8125 59.421875
+L 7.8125 69.390625
+Q 13.765625 71.78125 18.9375 73
+Q 24.125 74.21875 28.421875 74.21875
+Q 39.75 74.21875 46.484375 68.546875
+Q 53.21875 62.890625 53.21875 53.421875
+Q 53.21875 48.921875 51.53125 44.890625
+Q 49.859375 40.875 45.40625 35.40625
+Q 44.1875 33.984375 37.640625 27.21875
+Q 31.109375 20.453125 19.1875 8.296875
+z
+" id="DejaVuSans-50"/>
+ </defs>
+ <use xlink:href="#DejaVuSans-53"/>
+ <use x="63.623047" xlink:href="#DejaVuSans-46"/>
+ <use x="95.410156" xlink:href="#DejaVuSans-50"/>
+ </g>
+ </g>
+ </g>
+ <g id="ytick_2">
+ <g id="line2d_7">
+ <g>
+ <use style="stroke:#000000;stroke-width:0.8;" x="50.69" xlink:href="#m64c81340d8" y="209.394596"/>
+ </g>
+ </g>
+ <g id="text_8">
+ <!-- 5.4 -->
+ <g transform="translate(27.786875 213.193815)scale(0.1 -0.1)">
+ <use xlink:href="#DejaVuSans-53"/>
+ <use x="63.623047" xlink:href="#DejaVuSans-46"/>
+ <use x="95.410156" xlink:href="#DejaVuSans-52"/>
+ </g>
+ </g>
+ </g>
+ <g id="ytick_3">
+ <g id="line2d_8">
+ <g>
+ <use style="stroke:#000000;stroke-width:0.8;" x="50.69" xlink:href="#m64c81340d8" y="153.156126"/>
+ </g>
+ </g>
+ <g id="text_9">
+ <!-- 5.6 -->
+ <g transform="translate(27.786875 156.955344)scale(0.1 -0.1)">
+ <use xlink:href="#DejaVuSans-53"/>
+ <use x="63.623047" xlink:href="#DejaVuSans-46"/>
+ <use x="95.410156" xlink:href="#DejaVuSans-54"/>
+ </g>
+ </g>
+ </g>
+ <g id="ytick_4">
+ <g id="line2d_9">
+ <g>
+ <use style="stroke:#000000;stroke-width:0.8;" x="50.69" xlink:href="#m64c81340d8" y="96.917655"/>
+ </g>
+ </g>
+ <g id="text_10">
+ <!-- 5.8 -->
+ <g transform="translate(27.786875 100.716874)scale(0.1 -0.1)">
+ <defs>
+ <path d="M 31.78125 34.625
+Q 24.75 34.625 20.71875 30.859375
+Q 16.703125 27.09375 16.703125 20.515625
+Q 16.703125 13.921875 20.71875 10.15625
+Q 24.75 6.390625 31.78125 6.390625
+Q 38.8125 6.390625 42.859375 10.171875
+Q 46.921875 13.96875 46.921875 20.515625
+Q 46.921875 27.09375 42.890625 30.859375
+Q 38.875 34.625 31.78125 34.625
+z
+M 21.921875 38.8125
+Q 15.578125 40.375 12.03125 44.71875
+Q 8.5 49.078125 8.5 55.328125
+Q 8.5 64.0625 14.71875 69.140625
+Q 20.953125 74.21875 31.78125 74.21875
+Q 42.671875 74.21875 48.875 69.140625
+Q 55.078125 64.0625 55.078125 55.328125
+Q 55.078125 49.078125 51.53125 44.71875
+Q 48 40.375 41.703125 38.8125
+Q 48.828125 37.15625 52.796875 32.3125
+Q 56.78125 27.484375 56.78125 20.515625
+Q 56.78125 9.90625 50.3125 4.234375
+Q 43.84375 -1.421875 31.78125 -1.421875
+Q 19.734375 -1.421875 13.25 4.234375
+Q 6.78125 9.90625 6.78125 20.515625
+Q 6.78125 27.484375 10.78125 32.3125
+Q 14.796875 37.15625 21.921875 38.8125
+z
+M 18.3125 54.390625
+Q 18.3125 48.734375 21.84375 45.5625
+Q 25.390625 42.390625 31.78125 42.390625
+Q 38.140625 42.390625 41.71875 45.5625
+Q 45.3125 48.734375 45.3125 54.390625
+Q 45.3125 60.0625 41.71875 63.234375
+Q 38.140625 66.40625 31.78125 66.40625
+Q 25.390625 66.40625 21.84375 63.234375
+Q 18.3125 60.0625 18.3125 54.390625
+z
+" id="DejaVuSans-56"/>
+ </defs>
+ <use xlink:href="#DejaVuSans-53"/>
+ <use x="63.623047" xlink:href="#DejaVuSans-46"/>
+ <use x="95.410156" xlink:href="#DejaVuSans-56"/>
+ </g>
+ </g>
+ </g>
+ <g id="ytick_5">
+ <g id="line2d_10">
+ <g>
+ <use style="stroke:#000000;stroke-width:0.8;" x="50.69" xlink:href="#m64c81340d8" y="40.679185"/>
+ </g>
+ </g>
+ <g id="text_11">
+ <!-- 6.0 -->
+ <g transform="translate(27.786875 44.478403)scale(0.1 -0.1)">
+ <defs>
+ <path d="M 31.78125 66.40625
+Q 24.171875 66.40625 20.328125 58.90625
+Q 16.5 51.421875 16.5 36.375
+Q 16.5 21.390625 20.328125 13.890625
+Q 24.171875 6.390625 31.78125 6.390625
+Q 39.453125 6.390625 43.28125 13.890625
+Q 47.125 21.390625 47.125 36.375
+Q 47.125 51.421875 43.28125 58.90625
+Q 39.453125 66.40625 31.78125 66.40625
+z
+M 31.78125 74.21875
+Q 44.046875 74.21875 50.515625 64.515625
+Q 56.984375 54.828125 56.984375 36.375
+Q 56.984375 17.96875 50.515625 8.265625
+Q 44.046875 -1.421875 31.78125 -1.421875
+Q 19.53125 -1.421875 13.0625 8.265625
+Q 6.59375 17.96875 6.59375 36.375
+Q 6.59375 54.828125 13.0625 64.515625
+Q 19.53125 74.21875 31.78125 74.21875
+z
+" id="DejaVuSans-48"/>
+ </defs>
+ <use xlink:href="#DejaVuSans-54"/>
+ <use x="63.623047" xlink:href="#DejaVuSans-46"/>
+ <use x="95.410156" xlink:href="#DejaVuSans-48"/>
+ </g>
+ </g>
+ </g>
+ <g id="text_12">
+ <!-- $y$ [code units] -->
+ <g transform="translate(21.686875 190.79875)rotate(-90)scale(0.1 -0.1)">
+ <defs>
+ <path d="M 24.8125 -5.078125
+Q 18.5625 -15.578125 14.625 -18.1875
+Q 10.6875 -20.796875 4.59375 -20.796875
+L -2.484375 -20.796875
+L -0.984375 -13.28125
+L 4.203125 -13.28125
+Q 7.953125 -13.28125 10.59375 -11.234375
+Q 13.234375 -9.1875 16.5 -3.21875
+L 19.28125 2
+L 7.171875 54.6875
+L 16.703125 54.6875
+L 25.78125 12.796875
+L 50.875 54.6875
+L 60.296875 54.6875
+z
+" id="DejaVuSans-Oblique-121"/>
+ </defs>
+ <use transform="translate(0 0.015625)" xlink:href="#DejaVuSans-Oblique-121"/>
+ <use transform="translate(59.179688 0.015625)" xlink:href="#DejaVuSans-32"/>
+ <use transform="translate(90.966797 0.015625)" xlink:href="#DejaVuSans-91"/>
+ <use transform="translate(129.980469 0.015625)" xlink:href="#DejaVuSans-99"/>
+ <use transform="translate(184.960938 0.015625)" xlink:href="#DejaVuSans-111"/>
+ <use transform="translate(246.142578 0.015625)" xlink:href="#DejaVuSans-100"/>
+ <use transform="translate(309.619141 0.015625)" xlink:href="#DejaVuSans-101"/>
+ <use transform="translate(371.142578 0.015625)" xlink:href="#DejaVuSans-32"/>
+ <use transform="translate(402.929688 0.015625)" xlink:href="#DejaVuSans-117"/>
+ <use transform="translate(466.308594 0.015625)" xlink:href="#DejaVuSans-110"/>
+ <use transform="translate(529.6875 0.015625)" xlink:href="#DejaVuSans-105"/>
+ <use transform="translate(557.470703 0.015625)" xlink:href="#DejaVuSans-116"/>
+ <use transform="translate(596.679688 0.015625)" xlink:href="#DejaVuSans-115"/>
+ <use transform="translate(648.779297 0.015625)" xlink:href="#DejaVuSans-93"/>
+ </g>
+ </g>
+ </g>
+ <g id="line2d_11">
+ <path clip-path="url(#p444e95376c)" d="M 369.666423 288.740154
+L 377.756239 284.542414
+L 385.483034 280.206496
+L 392.794483 275.705775
+L 399.638716 271.01483
+L 402.869346 268.590509
+L 405.96337 266.109957
+L 408.913756 263.570753
+L 411.713127 260.970831
+L 414.353718 258.308572
+L 416.827387 255.582891
+L 419.125695 252.793329
+L 421.240068 249.940128
+L 423.162057 247.024283
+L 424.88368 244.047579
+L 426.397866 241.012579
+L 427.698957 237.922582
+L 428.783248 234.781535
+L 429.647954 231.592887
+L 430.290438 228.358295
+L 430.713195 225.081129
+L 430.918877 221.76338
+L 430.912275 218.406915
+L 430.694702 215.013392
+L 430.272394 211.584404
+L 429.650889 208.121489
+L 428.835854 204.626258
+L 427.8334 201.100501
+L 426.649488 197.546278
+L 425.289671 193.965983
+L 423.758904 190.362392
+L 422.061432 186.738676
+L 420.20076 183.098388
+L 418.179698 179.445416
+L 416.002461 175.784451
+L 413.676104 172.12108
+L 411.20209 168.459255
+L 408.584305 164.80408
+L 405.826236 161.160067
+L 402.930955 157.531662
+L 399.902011 153.923127
+L 396.743534 150.338534
+L 390.058411 143.256247
+L 382.925155 136.311828
+L 375.40626 129.526555
+L 367.572751 122.915813
+L 359.487995 116.506246
+L 351.217864 110.325976
+L 342.825917 104.401422
+L 334.371975 98.75303
+L 325.908974 93.395537
+L 317.483839 88.337535
+L 309.139761 83.581426
+L 300.918339 79.123857
+L 292.85824 74.957735
+L 284.986263 71.076825
+L 277.325388 67.473759
+L 269.889476 64.147967
+L 262.686672 61.094013
+L 255.716609 58.302297
+L 245.675261 54.574304
+L 236.067692 51.341671
+L 226.791445 48.532075
+L 217.719607 46.06895
+L 208.711086 43.87602
+L 199.618972 41.880503
+L 187.114982 39.412519
+L 173.880183 37.042213
+L 155.87764 34.063016
+L 135.796094 30.964405
+L 117.899693 28.429621
+L 98.121598 25.905421
+L 81.95633 24.03625
+L 81.95633 24.03625
+" style="fill:none;stroke:#1f77b4;stroke-dasharray:6,4.5,6;stroke-dashoffset:0;stroke-opacity:0.2;stroke-width:1.5;"/>
+ <defs>
+ <path d="M -3 3
+L 3 -3
+M -3 -3
+L 3 3
+" id="ma85ef77801" style="stroke:#1f77b4;stroke-opacity:0.2;"/>
+ </defs>
+ <g clip-path="url(#p444e95376c)">
+ <use style="fill:#1f77b4;fill-opacity:0.2;stroke:#1f77b4;stroke-opacity:0.2;" x="369.666423" xlink:href="#ma85ef77801" y="288.740154"/>
+ <use style="fill:#1f77b4;fill-opacity:0.2;stroke:#1f77b4;stroke-opacity:0.2;" x="373.75351" xlink:href="#ma85ef77801" y="286.656793"/>
+ <use style="fill:#1f77b4;fill-opacity:0.2;stroke:#1f77b4;stroke-opacity:0.2;" x="377.756239" xlink:href="#ma85ef77801" y="284.542414"/>
+ <use style="fill:#1f77b4;fill-opacity:0.2;stroke:#1f77b4;stroke-opacity:0.2;" x="381.668295" xlink:href="#ma85ef77801" y="282.393433"/>
+ <use style="fill:#1f77b4;fill-opacity:0.2;stroke:#1f77b4;stroke-opacity:0.2;" x="385.483034" xlink:href="#ma85ef77801" y="280.206496"/>
+ <use style="fill:#1f77b4;fill-opacity:0.2;stroke:#1f77b4;stroke-opacity:0.2;" x="389.193929" xlink:href="#ma85ef77801" y="277.97835"/>
+ <use style="fill:#1f77b4;fill-opacity:0.2;stroke:#1f77b4;stroke-opacity:0.2;" x="392.794483" xlink:href="#ma85ef77801" y="275.705775"/>
+ <use style="fill:#1f77b4;fill-opacity:0.2;stroke:#1f77b4;stroke-opacity:0.2;" x="396.278238" xlink:href="#ma85ef77801" y="273.385618"/>
+ <use style="fill:#1f77b4;fill-opacity:0.2;stroke:#1f77b4;stroke-opacity:0.2;" x="399.638716" xlink:href="#ma85ef77801" y="271.01483"/>
+ <use style="fill:#1f77b4;fill-opacity:0.2;stroke:#1f77b4;stroke-opacity:0.2;" x="402.869346" xlink:href="#ma85ef77801" y="268.590509"/>
+ <use style="fill:#1f77b4;fill-opacity:0.2;stroke:#1f77b4;stroke-opacity:0.2;" x="405.96337" xlink:href="#ma85ef77801" y="266.109957"/>
+ <use style="fill:#1f77b4;fill-opacity:0.2;stroke:#1f77b4;stroke-opacity:0.2;" x="408.913756" xlink:href="#ma85ef77801" y="263.570753"/>
+ <use style="fill:#1f77b4;fill-opacity:0.2;stroke:#1f77b4;stroke-opacity:0.2;" x="411.713127" xlink:href="#ma85ef77801" y="260.970831"/>
+ <use style="fill:#1f77b4;fill-opacity:0.2;stroke:#1f77b4;stroke-opacity:0.2;" x="414.353718" xlink:href="#ma85ef77801" y="258.308572"/>
+ <use style="fill:#1f77b4;fill-opacity:0.2;stroke:#1f77b4;stroke-opacity:0.2;" x="416.827387" xlink:href="#ma85ef77801" y="255.582891"/>
+ <use style="fill:#1f77b4;fill-opacity:0.2;stroke:#1f77b4;stroke-opacity:0.2;" x="419.125695" xlink:href="#ma85ef77801" y="252.793329"/>
+ <use style="fill:#1f77b4;fill-opacity:0.2;stroke:#1f77b4;stroke-opacity:0.2;" x="421.240068" xlink:href="#ma85ef77801" y="249.940128"/>
+ <use style="fill:#1f77b4;fill-opacity:0.2;stroke:#1f77b4;stroke-opacity:0.2;" x="423.162057" xlink:href="#ma85ef77801" y="247.024283"/>
+ <use style="fill:#1f77b4;fill-opacity:0.2;stroke:#1f77b4;stroke-opacity:0.2;" x="424.88368" xlink:href="#ma85ef77801" y="244.047579"/>
+ <use style="fill:#1f77b4;fill-opacity:0.2;stroke:#1f77b4;stroke-opacity:0.2;" x="426.397866" xlink:href="#ma85ef77801" y="241.012579"/>
+ <use style="fill:#1f77b4;fill-opacity:0.2;stroke:#1f77b4;stroke-opacity:0.2;" x="427.698957" xlink:href="#ma85ef77801" y="237.922582"/>
+ <use style="fill:#1f77b4;fill-opacity:0.2;stroke:#1f77b4;stroke-opacity:0.2;" x="428.783248" xlink:href="#ma85ef77801" y="234.781535"/>
+ <use style="fill:#1f77b4;fill-opacity:0.2;stroke:#1f77b4;stroke-opacity:0.2;" x="429.647954" xlink:href="#ma85ef77801" y="231.592887"/>
+ <use style="fill:#1f77b4;fill-opacity:0.2;stroke:#1f77b4;stroke-opacity:0.2;" x="430.290438" xlink:href="#ma85ef77801" y="228.358295"/>
+ <use style="fill:#1f77b4;fill-opacity:0.2;stroke:#1f77b4;stroke-opacity:0.2;" x="430.713195" xlink:href="#ma85ef77801" y="225.081129"/>
+ <use style="fill:#1f77b4;fill-opacity:0.2;stroke:#1f77b4;stroke-opacity:0.2;" x="430.918877" xlink:href="#ma85ef77801" y="221.76338"/>
+ <use style="fill:#1f77b4;fill-opacity:0.2;stroke:#1f77b4;stroke-opacity:0.2;" x="430.912275" xlink:href="#ma85ef77801" y="218.406915"/>
+ <use style="fill:#1f77b4;fill-opacity:0.2;stroke:#1f77b4;stroke-opacity:0.2;" x="430.694702" xlink:href="#ma85ef77801" y="215.013392"/>
+ <use style="fill:#1f77b4;fill-opacity:0.2;stroke:#1f77b4;stroke-opacity:0.2;" x="430.272394" xlink:href="#ma85ef77801" y="211.584404"/>
+ <use style="fill:#1f77b4;fill-opacity:0.2;stroke:#1f77b4;stroke-opacity:0.2;" x="429.650889" xlink:href="#ma85ef77801" y="208.121489"/>
+ <use style="fill:#1f77b4;fill-opacity:0.2;stroke:#1f77b4;stroke-opacity:0.2;" x="428.835854" xlink:href="#ma85ef77801" y="204.626258"/>
+ <use style="fill:#1f77b4;fill-opacity:0.2;stroke:#1f77b4;stroke-opacity:0.2;" x="427.8334" xlink:href="#ma85ef77801" y="201.100501"/>
+ <use style="fill:#1f77b4;fill-opacity:0.2;stroke:#1f77b4;stroke-opacity:0.2;" x="426.649488" xlink:href="#ma85ef77801" y="197.546278"/>
+ <use style="fill:#1f77b4;fill-opacity:0.2;stroke:#1f77b4;stroke-opacity:0.2;" x="425.289671" xlink:href="#ma85ef77801" y="193.965983"/>
+ <use style="fill:#1f77b4;fill-opacity:0.2;stroke:#1f77b4;stroke-opacity:0.2;" x="423.758904" xlink:href="#ma85ef77801" y="190.362392"/>
+ <use style="fill:#1f77b4;fill-opacity:0.2;stroke:#1f77b4;stroke-opacity:0.2;" x="422.061432" xlink:href="#ma85ef77801" y="186.738676"/>
+ <use style="fill:#1f77b4;fill-opacity:0.2;stroke:#1f77b4;stroke-opacity:0.2;" x="420.20076" xlink:href="#ma85ef77801" y="183.098388"/>
+ <use style="fill:#1f77b4;fill-opacity:0.2;stroke:#1f77b4;stroke-opacity:0.2;" x="418.179698" xlink:href="#ma85ef77801" y="179.445416"/>
+ <use style="fill:#1f77b4;fill-opacity:0.2;stroke:#1f77b4;stroke-opacity:0.2;" x="416.002461" xlink:href="#ma85ef77801" y="175.784451"/>
+ <use style="fill:#1f77b4;fill-opacity:0.2;stroke:#1f77b4;stroke-opacity:0.2;" x="413.676104" xlink:href="#ma85ef77801" y="172.12108"/>
+ <use style="fill:#1f77b4;fill-opacity:0.2;stroke:#1f77b4;stroke-opacity:0.2;" x="411.20209" xlink:href="#ma85ef77801" y="168.459255"/>
+ <use style="fill:#1f77b4;fill-opacity:0.2;stroke:#1f77b4;stroke-opacity:0.2;" x="408.584305" xlink:href="#ma85ef77801" y="164.80408"/>
+ <use style="fill:#1f77b4;fill-opacity:0.2;stroke:#1f77b4;stroke-opacity:0.2;" x="405.826236" xlink:href="#ma85ef77801" y="161.160067"/>
+ <use style="fill:#1f77b4;fill-opacity:0.2;stroke:#1f77b4;stroke-opacity:0.2;" x="402.930955" xlink:href="#ma85ef77801" y="157.531662"/>
+ <use style="fill:#1f77b4;fill-opacity:0.2;stroke:#1f77b4;stroke-opacity:0.2;" x="399.902011" xlink:href="#ma85ef77801" y="153.923127"/>
+ <use style="fill:#1f77b4;fill-opacity:0.2;stroke:#1f77b4;stroke-opacity:0.2;" x="396.743534" xlink:href="#ma85ef77801" y="150.338534"/>
+ <use style="fill:#1f77b4;fill-opacity:0.2;stroke:#1f77b4;stroke-opacity:0.2;" x="393.460416" xlink:href="#ma85ef77801" y="146.781722"/>
+ <use style="fill:#1f77b4;fill-opacity:0.2;stroke:#1f77b4;stroke-opacity:0.2;" x="390.058411" xlink:href="#ma85ef77801" y="143.256247"/>
+ <use style="fill:#1f77b4;fill-opacity:0.2;stroke:#1f77b4;stroke-opacity:0.2;" x="386.544163" xlink:href="#ma85ef77801" y="139.76533"/>
+ <use style="fill:#1f77b4;fill-opacity:0.2;stroke:#1f77b4;stroke-opacity:0.2;" x="382.925155" xlink:href="#ma85ef77801" y="136.311828"/>
+ <use style="fill:#1f77b4;fill-opacity:0.2;stroke:#1f77b4;stroke-opacity:0.2;" x="379.209599" xlink:href="#ma85ef77801" y="132.898209"/>
+ <use style="fill:#1f77b4;fill-opacity:0.2;stroke:#1f77b4;stroke-opacity:0.2;" x="375.40626" xlink:href="#ma85ef77801" y="129.526555"/>
+ <use style="fill:#1f77b4;fill-opacity:0.2;stroke:#1f77b4;stroke-opacity:0.2;" x="371.524256" xlink:href="#ma85ef77801" y="126.19859"/>
+ <use style="fill:#1f77b4;fill-opacity:0.2;stroke:#1f77b4;stroke-opacity:0.2;" x="367.572751" xlink:href="#ma85ef77801" y="122.915813"/>
+ <use style="fill:#1f77b4;fill-opacity:0.2;stroke:#1f77b4;stroke-opacity:0.2;" x="363.55772" xlink:href="#ma85ef77801" y="119.683992"/>
+ <use style="fill:#1f77b4;fill-opacity:0.2;stroke:#1f77b4;stroke-opacity:0.2;" x="359.487995" xlink:href="#ma85ef77801" y="116.506246"/>
+ <use style="fill:#1f77b4;fill-opacity:0.2;stroke:#1f77b4;stroke-opacity:0.2;" x="355.37213" xlink:href="#ma85ef77801" y="113.385628"/>
+ <use style="fill:#1f77b4;fill-opacity:0.2;stroke:#1f77b4;stroke-opacity:0.2;" x="351.217864" xlink:href="#ma85ef77801" y="110.325976"/>
+ <use style="fill:#1f77b4;fill-opacity:0.2;stroke:#1f77b4;stroke-opacity:0.2;" x="347.033222" xlink:href="#ma85ef77801" y="107.330329"/>
+ <use style="fill:#1f77b4;fill-opacity:0.2;stroke:#1f77b4;stroke-opacity:0.2;" x="342.825917" xlink:href="#ma85ef77801" y="104.401422"/>
+ <use style="fill:#1f77b4;fill-opacity:0.2;stroke:#1f77b4;stroke-opacity:0.2;" x="338.603225" xlink:href="#ma85ef77801" y="101.541644"/>
+ <use style="fill:#1f77b4;fill-opacity:0.2;stroke:#1f77b4;stroke-opacity:0.2;" x="334.371975" xlink:href="#ma85ef77801" y="98.75303"/>
+ <use style="fill:#1f77b4;fill-opacity:0.2;stroke:#1f77b4;stroke-opacity:0.2;" x="330.138557" xlink:href="#ma85ef77801" y="96.037237"/>
+ <use style="fill:#1f77b4;fill-opacity:0.2;stroke:#1f77b4;stroke-opacity:0.2;" x="325.908974" xlink:href="#ma85ef77801" y="93.395537"/>
+ <use style="fill:#1f77b4;fill-opacity:0.2;stroke:#1f77b4;stroke-opacity:0.2;" x="321.688915" xlink:href="#ma85ef77801" y="90.828807"/>
+ <use style="fill:#1f77b4;fill-opacity:0.2;stroke:#1f77b4;stroke-opacity:0.2;" x="317.483839" xlink:href="#ma85ef77801" y="88.337535"/>
+ <use style="fill:#1f77b4;fill-opacity:0.2;stroke:#1f77b4;stroke-opacity:0.2;" x="313.299048" xlink:href="#ma85ef77801" y="85.921827"/>
+ <use style="fill:#1f77b4;fill-opacity:0.2;stroke:#1f77b4;stroke-opacity:0.2;" x="309.139761" xlink:href="#ma85ef77801" y="83.581426"/>
+ <use style="fill:#1f77b4;fill-opacity:0.2;stroke:#1f77b4;stroke-opacity:0.2;" x="305.011145" xlink:href="#ma85ef77801" y="81.315738"/>
+ <use style="fill:#1f77b4;fill-opacity:0.2;stroke:#1f77b4;stroke-opacity:0.2;" x="300.918339" xlink:href="#ma85ef77801" y="79.123857"/>
+ <use style="fill:#1f77b4;fill-opacity:0.2;stroke:#1f77b4;stroke-opacity:0.2;" x="296.866432" xlink:href="#ma85ef77801" y="77.004607"/>
+ <use style="fill:#1f77b4;fill-opacity:0.2;stroke:#1f77b4;stroke-opacity:0.2;" x="292.85824" xlink:href="#ma85ef77801" y="74.957735"/>
+ <use style="fill:#1f77b4;fill-opacity:0.2;stroke:#1f77b4;stroke-opacity:0.2;" x="288.896988" xlink:href="#ma85ef77801" y="72.982281"/>
+ <use style="fill:#1f77b4;fill-opacity:0.2;stroke:#1f77b4;stroke-opacity:0.2;" x="284.986263" xlink:href="#ma85ef77801" y="71.076825"/>
+ <use style="fill:#1f77b4;fill-opacity:0.2;stroke:#1f77b4;stroke-opacity:0.2;" x="281.128557" xlink:href="#ma85ef77801" y="69.240226"/>
+ <use style="fill:#1f77b4;fill-opacity:0.2;stroke:#1f77b4;stroke-opacity:0.2;" x="277.325388" xlink:href="#ma85ef77801" y="67.473759"/>
+ <use style="fill:#1f77b4;fill-opacity:0.2;stroke:#1f77b4;stroke-opacity:0.2;" x="273.578568" xlink:href="#ma85ef77801" y="65.776541"/>
+ <use style="fill:#1f77b4;fill-opacity:0.2;stroke:#1f77b4;stroke-opacity:0.2;" x="269.889476" xlink:href="#ma85ef77801" y="64.147967"/>
+ <use style="fill:#1f77b4;fill-opacity:0.2;stroke:#1f77b4;stroke-opacity:0.2;" x="266.258764" xlink:href="#ma85ef77801" y="62.587485"/>
+ <use style="fill:#1f77b4;fill-opacity:0.2;stroke:#1f77b4;stroke-opacity:0.2;" x="262.686672" xlink:href="#ma85ef77801" y="61.094013"/>
+ <use style="fill:#1f77b4;fill-opacity:0.2;stroke:#1f77b4;stroke-opacity:0.2;" x="259.172913" xlink:href="#ma85ef77801" y="59.666172"/>
+ <use style="fill:#1f77b4;fill-opacity:0.2;stroke:#1f77b4;stroke-opacity:0.2;" x="255.716609" xlink:href="#ma85ef77801" y="58.302297"/>
+ <use style="fill:#1f77b4;fill-opacity:0.2;stroke:#1f77b4;stroke-opacity:0.2;" x="252.316318" xlink:href="#ma85ef77801" y="57.00048"/>
+ <use style="fill:#1f77b4;fill-opacity:0.2;stroke:#1f77b4;stroke-opacity:0.2;" x="248.970041" xlink:href="#ma85ef77801" y="55.758588"/>
+ <use style="fill:#1f77b4;fill-opacity:0.2;stroke:#1f77b4;stroke-opacity:0.2;" x="245.675261" xlink:href="#ma85ef77801" y="54.574304"/>
+ <use style="fill:#1f77b4;fill-opacity:0.2;stroke:#1f77b4;stroke-opacity:0.2;" x="242.428973" xlink:href="#ma85ef77801" y="53.445144"/>
+ <use style="fill:#1f77b4;fill-opacity:0.2;stroke:#1f77b4;stroke-opacity:0.2;" x="239.227731" xlink:href="#ma85ef77801" y="52.368501"/>
+ <use style="fill:#1f77b4;fill-opacity:0.2;stroke:#1f77b4;stroke-opacity:0.2;" x="236.067692" xlink:href="#ma85ef77801" y="51.341671"/>
+ <use style="fill:#1f77b4;fill-opacity:0.2;stroke:#1f77b4;stroke-opacity:0.2;" x="232.944667" xlink:href="#ma85ef77801" y="50.361878"/>
+ <use style="fill:#1f77b4;fill-opacity:0.2;stroke:#1f77b4;stroke-opacity:0.2;" x="229.854166" xlink:href="#ma85ef77801" y="49.426299"/>
+ <use style="fill:#1f77b4;fill-opacity:0.2;stroke:#1f77b4;stroke-opacity:0.2;" x="226.791445" xlink:href="#ma85ef77801" y="48.532075"/>
+ <use style="fill:#1f77b4;fill-opacity:0.2;stroke:#1f77b4;stroke-opacity:0.2;" x="223.751551" xlink:href="#ma85ef77801" y="47.676344"/>
+ <use style="fill:#1f77b4;fill-opacity:0.2;stroke:#1f77b4;stroke-opacity:0.2;" x="220.729358" xlink:href="#ma85ef77801" y="46.856249"/>
+ <use style="fill:#1f77b4;fill-opacity:0.2;stroke:#1f77b4;stroke-opacity:0.2;" x="217.719607" xlink:href="#ma85ef77801" y="46.06895"/>
+ <use style="fill:#1f77b4;fill-opacity:0.2;stroke:#1f77b4;stroke-opacity:0.2;" x="214.716937" xlink:href="#ma85ef77801" y="45.311644"/>
+ <use style="fill:#1f77b4;fill-opacity:0.2;stroke:#1f77b4;stroke-opacity:0.2;" x="211.715919" xlink:href="#ma85ef77801" y="44.581569"/>
+ <use style="fill:#1f77b4;fill-opacity:0.2;stroke:#1f77b4;stroke-opacity:0.2;" x="208.711086" xlink:href="#ma85ef77801" y="43.87602"/>
+ <use style="fill:#1f77b4;fill-opacity:0.2;stroke:#1f77b4;stroke-opacity:0.2;" x="205.696955" xlink:href="#ma85ef77801" y="43.192356"/>
+ <use style="fill:#1f77b4;fill-opacity:0.2;stroke:#1f77b4;stroke-opacity:0.2;" x="202.668059" xlink:href="#ma85ef77801" y="42.528012"/>
+ <use style="fill:#1f77b4;fill-opacity:0.2;stroke:#1f77b4;stroke-opacity:0.2;" x="199.618972" xlink:href="#ma85ef77801" y="41.880503"/>
+ <use style="fill:#1f77b4;fill-opacity:0.2;stroke:#1f77b4;stroke-opacity:0.2;" x="196.544332" xlink:href="#ma85ef77801" y="41.247438"/>
+ <use style="fill:#1f77b4;fill-opacity:0.2;stroke:#1f77b4;stroke-opacity:0.2;" x="193.43887" xlink:href="#ma85ef77801" y="40.626523"/>
+ <use style="fill:#1f77b4;fill-opacity:0.2;stroke:#1f77b4;stroke-opacity:0.2;" x="190.297428" xlink:href="#ma85ef77801" y="40.015574"/>
+ <use style="fill:#1f77b4;fill-opacity:0.2;stroke:#1f77b4;stroke-opacity:0.2;" x="187.114982" xlink:href="#ma85ef77801" y="39.412519"/>
+ <use style="fill:#1f77b4;fill-opacity:0.2;stroke:#1f77b4;stroke-opacity:0.2;" x="183.88666" xlink:href="#ma85ef77801" y="38.815409"/>
+ <use style="fill:#1f77b4;fill-opacity:0.2;stroke:#1f77b4;stroke-opacity:0.2;" x="180.60775" xlink:href="#ma85ef77801" y="38.222423"/>
+ <use style="fill:#1f77b4;fill-opacity:0.2;stroke:#1f77b4;stroke-opacity:0.2;" x="177.273714" xlink:href="#ma85ef77801" y="37.631874"/>
+ <use style="fill:#1f77b4;fill-opacity:0.2;stroke:#1f77b4;stroke-opacity:0.2;" x="173.880183" xlink:href="#ma85ef77801" y="37.042213"/>
+ <use style="fill:#1f77b4;fill-opacity:0.2;stroke:#1f77b4;stroke-opacity:0.2;" x="170.422951" xlink:href="#ma85ef77801" y="36.452032"/>
+ <use style="fill:#1f77b4;fill-opacity:0.2;stroke:#1f77b4;stroke-opacity:0.2;" x="166.897962" xlink:href="#ma85ef77801" y="35.860069"/>
+ <use style="fill:#1f77b4;fill-opacity:0.2;stroke:#1f77b4;stroke-opacity:0.2;" x="163.30129" xlink:href="#ma85ef77801" y="35.265204"/>
+ <use style="fill:#1f77b4;fill-opacity:0.2;stroke:#1f77b4;stroke-opacity:0.2;" x="159.629105" xlink:href="#ma85ef77801" y="34.666465"/>
+ <use style="fill:#1f77b4;fill-opacity:0.2;stroke:#1f77b4;stroke-opacity:0.2;" x="155.87764" xlink:href="#ma85ef77801" y="34.063016"/>
+ <use style="fill:#1f77b4;fill-opacity:0.2;stroke:#1f77b4;stroke-opacity:0.2;" x="152.043152" xlink:href="#ma85ef77801" y="33.454162"/>
+ <use style="fill:#1f77b4;fill-opacity:0.2;stroke:#1f77b4;stroke-opacity:0.2;" x="148.121869" xlink:href="#ma85ef77801" y="32.839337"/>
+ <use style="fill:#1f77b4;fill-opacity:0.2;stroke:#1f77b4;stroke-opacity:0.2;" x="144.109828" xlink:href="#ma85ef77801" y="32.218475"/>
+ <use style="fill:#1f77b4;fill-opacity:0.2;stroke:#1f77b4;stroke-opacity:0.2;" x="140.002551" xlink:href="#ma85ef77801" y="31.593492"/>
+ <use style="fill:#1f77b4;fill-opacity:0.2;stroke:#1f77b4;stroke-opacity:0.2;" x="135.796094" xlink:href="#ma85ef77801" y="30.964405"/>
+ <use style="fill:#1f77b4;fill-opacity:0.2;stroke:#1f77b4;stroke-opacity:0.2;" x="131.486507" xlink:href="#ma85ef77801" y="30.332494"/>
+ <use style="fill:#1f77b4;fill-opacity:0.2;stroke:#1f77b4;stroke-opacity:0.2;" x="127.069724" xlink:href="#ma85ef77801" y="29.698759"/>
+ <use style="fill:#1f77b4;fill-opacity:0.2;stroke:#1f77b4;stroke-opacity:0.2;" x="122.541993" xlink:href="#ma85ef77801" y="29.064176"/>
+ <use style="fill:#1f77b4;fill-opacity:0.2;stroke:#1f77b4;stroke-opacity:0.2;" x="117.899693" xlink:href="#ma85ef77801" y="28.429621"/>
+ <use style="fill:#1f77b4;fill-opacity:0.2;stroke:#1f77b4;stroke-opacity:0.2;" x="113.13943" xlink:href="#ma85ef77801" y="27.795884"/>
+ <use style="fill:#1f77b4;fill-opacity:0.2;stroke:#1f77b4;stroke-opacity:0.2;" x="108.25809" xlink:href="#ma85ef77801" y="27.163623"/>
+ <use style="fill:#1f77b4;fill-opacity:0.2;stroke:#1f77b4;stroke-opacity:0.2;" x="103.252918" xlink:href="#ma85ef77801" y="26.533354"/>
+ <use style="fill:#1f77b4;fill-opacity:0.2;stroke:#1f77b4;stroke-opacity:0.2;" x="98.121598" xlink:href="#ma85ef77801" y="25.905421"/>
+ <use style="fill:#1f77b4;fill-opacity:0.2;stroke:#1f77b4;stroke-opacity:0.2;" x="92.862356" xlink:href="#ma85ef77801" y="25.279983"/>
+ <use style="fill:#1f77b4;fill-opacity:0.2;stroke:#1f77b4;stroke-opacity:0.2;" x="87.474061" xlink:href="#ma85ef77801" y="24.657005"/>
+ <use style="fill:#1f77b4;fill-opacity:0.2;stroke:#1f77b4;stroke-opacity:0.2;" x="81.95633" xlink:href="#ma85ef77801" y="24.03625"/>
+ </g>
+ </g>
+ <g id="line2d_12">
+ <path clip-path="url(#p444e95376c)" d="M 369.666423 288.740154
+L 377.756088 284.542788
+L 383.588135 281.305296
+L 389.193864 277.97876
+L 394.551307 274.552278
+L 399.638644 271.01531
+L 404.433763 267.35795
+L 408.913623 263.571342
+L 411.712984 260.971465
+L 414.35358 258.309253
+L 416.827276 255.58362
+L 419.12564 252.794103
+L 421.240102 249.940942
+L 423.162215 247.02513
+L 424.883997 244.048447
+L 426.398373 241.013453
+L 427.699673 237.923446
+L 428.784101 234.782323
+L 429.648408 231.593186
+L 430.291482 228.358851
+L 430.71424 225.081709
+L 430.919868 221.763946
+L 430.912302 218.407476
+L 430.695132 215.013982
+L 430.272999 211.585027
+L 429.651362 208.122144
+L 428.836189 204.626945
+L 427.833602 201.101219
+L 426.649568 197.547022
+L 425.289642 193.966746
+L 422.930616 188.553612
+L 420.200534 183.099156
+L 417.110481 177.616468
+L 413.67582 172.12164
+L 409.910608 166.631117
+L 405.826066 161.160623
+L 401.432902 155.725187
+L 396.743578 150.339036
+L 391.773964 145.01532
+L 386.544227 139.765754
+L 381.078889 134.600292
+L 375.406103 129.526947
+L 369.556276 124.551894
+L 363.557108 119.68485
+L 357.43497 114.938929
+L 349.128198 108.820375
+L 340.715252 102.963143
+L 332.254273 97.386313
+L 323.796507 92.103122
+L 315.387675 87.120603
+L 307.07037 82.43966
+L 298.88597 78.055617
+L 288.89587 72.982641
+L 279.218618 68.349026
+L 269.888039 64.14831
+L 260.92095 60.372334
+L 252.314606 57.000844
+L 244.044447 54.003377
+L 236.065773 51.342074
+L 228.317601 48.974611
+L 220.727185 46.856675
+L 211.713567 44.582004
+L 202.665524 42.528457
+L 191.870237 40.320394
+L 178.944985 37.927416
+L 163.298176 35.265684
+L 144.10637 32.219325
+L 124.8165 29.38199
+L 108.254581 27.16413
+L 90.180889 24.968764
+L 81.952859 24.036851
+L 81.952859 24.036851
+" style="fill:none;stroke:#ff7f0e;stroke-dasharray:3,4.5,3;stroke-dashoffset:0;stroke-opacity:0.2;stroke-width:1.5;"/>
+ <defs>
+ <path d="M -3 3
+L 3 -3
+M -3 -3
+L 3 3
+" id="mb7e8bbea43" style="stroke:#ff7f0e;stroke-opacity:0.2;"/>
+ </defs>
+ <g clip-path="url(#p444e95376c)">
+ <use style="fill:#ff7f0e;fill-opacity:0.2;stroke:#ff7f0e;stroke-opacity:0.2;" x="369.666423" xlink:href="#mb7e8bbea43" y="288.740154"/>
+ <use style="fill:#ff7f0e;fill-opacity:0.2;stroke:#ff7f0e;stroke-opacity:0.2;" x="371.719962" xlink:href="#mb7e8bbea43" y="287.702345"/>
+ <use style="fill:#ff7f0e;fill-opacity:0.2;stroke:#ff7f0e;stroke-opacity:0.2;" x="373.753301" xlink:href="#mb7e8bbea43" y="286.657127"/>
+ <use style="fill:#ff7f0e;fill-opacity:0.2;stroke:#ff7f0e;stroke-opacity:0.2;" x="375.765619" xlink:href="#mb7e8bbea43" y="285.60408"/>
+ <use style="fill:#ff7f0e;fill-opacity:0.2;stroke:#ff7f0e;stroke-opacity:0.2;" x="377.756088" xlink:href="#mb7e8bbea43" y="284.542788"/>
+ <use style="fill:#ff7f0e;fill-opacity:0.2;stroke:#ff7f0e;stroke-opacity:0.2;" x="379.723882" xlink:href="#mb7e8bbea43" y="283.472835"/>
+ <use style="fill:#ff7f0e;fill-opacity:0.2;stroke:#ff7f0e;stroke-opacity:0.2;" x="381.668172" xlink:href="#mb7e8bbea43" y="282.393809"/>
+ <use style="fill:#ff7f0e;fill-opacity:0.2;stroke:#ff7f0e;stroke-opacity:0.2;" x="383.588135" xlink:href="#mb7e8bbea43" y="281.305296"/>
+ <use style="fill:#ff7f0e;fill-opacity:0.2;stroke:#ff7f0e;stroke-opacity:0.2;" x="385.482948" xlink:href="#mb7e8bbea43" y="280.206888"/>
+ <use style="fill:#ff7f0e;fill-opacity:0.2;stroke:#ff7f0e;stroke-opacity:0.2;" x="387.351796" xlink:href="#mb7e8bbea43" y="279.098178"/>
+ <use style="fill:#ff7f0e;fill-opacity:0.2;stroke:#ff7f0e;stroke-opacity:0.2;" x="389.193864" xlink:href="#mb7e8bbea43" y="277.97876"/>
+ <use style="fill:#ff7f0e;fill-opacity:0.2;stroke:#ff7f0e;stroke-opacity:0.2;" x="391.008343" xlink:href="#mb7e8bbea43" y="276.848234"/>
+ <use style="fill:#ff7f0e;fill-opacity:0.2;stroke:#ff7f0e;stroke-opacity:0.2;" x="392.794426" xlink:href="#mb7e8bbea43" y="275.706204"/>
+ <use style="fill:#ff7f0e;fill-opacity:0.2;stroke:#ff7f0e;stroke-opacity:0.2;" x="394.551307" xlink:href="#mb7e8bbea43" y="274.552278"/>
+ <use style="fill:#ff7f0e;fill-opacity:0.2;stroke:#ff7f0e;stroke-opacity:0.2;" x="396.278178" xlink:href="#mb7e8bbea43" y="273.38607"/>
+ <use style="fill:#ff7f0e;fill-opacity:0.2;stroke:#ff7f0e;stroke-opacity:0.2;" x="397.974229" xlink:href="#mb7e8bbea43" y="272.207204"/>
+ <use style="fill:#ff7f0e;fill-opacity:0.2;stroke:#ff7f0e;stroke-opacity:0.2;" x="399.638644" xlink:href="#mb7e8bbea43" y="271.01531"/>
+ <use style="fill:#ff7f0e;fill-opacity:0.2;stroke:#ff7f0e;stroke-opacity:0.2;" x="401.270598" xlink:href="#mb7e8bbea43" y="269.81003"/>
+ <use style="fill:#ff7f0e;fill-opacity:0.2;stroke:#ff7f0e;stroke-opacity:0.2;" x="402.869254" xlink:href="#mb7e8bbea43" y="268.591021"/>
+ <use style="fill:#ff7f0e;fill-opacity:0.2;stroke:#ff7f0e;stroke-opacity:0.2;" x="404.433763" xlink:href="#mb7e8bbea43" y="267.35795"/>
+ <use style="fill:#ff7f0e;fill-opacity:0.2;stroke:#ff7f0e;stroke-opacity:0.2;" x="405.963256" xlink:href="#mb7e8bbea43" y="266.110506"/>
+ <use style="fill:#ff7f0e;fill-opacity:0.2;stroke:#ff7f0e;stroke-opacity:0.2;" x="407.456846" xlink:href="#mb7e8bbea43" y="264.848393"/>
+ <use style="fill:#ff7f0e;fill-opacity:0.2;stroke:#ff7f0e;stroke-opacity:0.2;" x="408.913623" xlink:href="#mb7e8bbea43" y="263.571342"/>
+ <use style="fill:#ff7f0e;fill-opacity:0.2;stroke:#ff7f0e;stroke-opacity:0.2;" x="410.332655" xlink:href="#mb7e8bbea43" y="262.279106"/>
+ <use style="fill:#ff7f0e;fill-opacity:0.2;stroke:#ff7f0e;stroke-opacity:0.2;" x="411.712984" xlink:href="#mb7e8bbea43" y="260.971465"/>
+ <use style="fill:#ff7f0e;fill-opacity:0.2;stroke:#ff7f0e;stroke-opacity:0.2;" x="413.053628" xlink:href="#mb7e8bbea43" y="259.648232"/>
+ <use style="fill:#ff7f0e;fill-opacity:0.2;stroke:#ff7f0e;stroke-opacity:0.2;" x="414.35358" xlink:href="#mb7e8bbea43" y="258.309253"/>
+ <use style="fill:#ff7f0e;fill-opacity:0.2;stroke:#ff7f0e;stroke-opacity:0.2;" x="415.611812" xlink:href="#mb7e8bbea43" y="256.954408"/>
+ <use style="fill:#ff7f0e;fill-opacity:0.2;stroke:#ff7f0e;stroke-opacity:0.2;" x="416.827276" xlink:href="#mb7e8bbea43" y="255.58362"/>
+ <use style="fill:#ff7f0e;fill-opacity:0.2;stroke:#ff7f0e;stroke-opacity:0.2;" x="417.998909" xlink:href="#mb7e8bbea43" y="254.196849"/>
+ <use style="fill:#ff7f0e;fill-opacity:0.2;stroke:#ff7f0e;stroke-opacity:0.2;" x="419.12564" xlink:href="#mb7e8bbea43" y="252.794103"/>
+ <use style="fill:#ff7f0e;fill-opacity:0.2;stroke:#ff7f0e;stroke-opacity:0.2;" x="420.206393" xlink:href="#mb7e8bbea43" y="251.375435"/>
+ <use style="fill:#ff7f0e;fill-opacity:0.2;stroke:#ff7f0e;stroke-opacity:0.2;" x="421.240102" xlink:href="#mb7e8bbea43" y="249.940942"/>
+ <use style="fill:#ff7f0e;fill-opacity:0.2;stroke:#ff7f0e;stroke-opacity:0.2;" x="422.225717" xlink:href="#mb7e8bbea43" y="248.490775"/>
+ <use style="fill:#ff7f0e;fill-opacity:0.2;stroke:#ff7f0e;stroke-opacity:0.2;" x="423.162215" xlink:href="#mb7e8bbea43" y="247.02513"/>
+ <use style="fill:#ff7f0e;fill-opacity:0.2;stroke:#ff7f0e;stroke-opacity:0.2;" x="424.048617" xlink:href="#mb7e8bbea43" y="245.544255"/>
+ <use style="fill:#ff7f0e;fill-opacity:0.2;stroke:#ff7f0e;stroke-opacity:0.2;" x="424.883997" xlink:href="#mb7e8bbea43" y="244.048447"/>
+ <use style="fill:#ff7f0e;fill-opacity:0.2;stroke:#ff7f0e;stroke-opacity:0.2;" x="425.667504" xlink:href="#mb7e8bbea43" y="242.538049"/>
+ <use style="fill:#ff7f0e;fill-opacity:0.2;stroke:#ff7f0e;stroke-opacity:0.2;" x="426.398373" xlink:href="#mb7e8bbea43" y="241.013453"/>
+ <use style="fill:#ff7f0e;fill-opacity:0.2;stroke:#ff7f0e;stroke-opacity:0.2;" x="427.075942" xlink:href="#mb7e8bbea43" y="239.475094"/>
+ <use style="fill:#ff7f0e;fill-opacity:0.2;stroke:#ff7f0e;stroke-opacity:0.2;" x="427.699673" xlink:href="#mb7e8bbea43" y="237.923446"/>
+ <use style="fill:#ff7f0e;fill-opacity:0.2;stroke:#ff7f0e;stroke-opacity:0.2;" x="428.269168" xlink:href="#mb7e8bbea43" y="236.359024"/>
+ <use style="fill:#ff7f0e;fill-opacity:0.2;stroke:#ff7f0e;stroke-opacity:0.2;" x="428.784101" xlink:href="#mb7e8bbea43" y="234.782323"/>
+ <use style="fill:#ff7f0e;fill-opacity:0.2;stroke:#ff7f0e;stroke-opacity:0.2;" x="429.243864" xlink:href="#mb7e8bbea43" y="233.19356"/>
+ <use style="fill:#ff7f0e;fill-opacity:0.2;stroke:#ff7f0e;stroke-opacity:0.2;" x="429.648408" xlink:href="#mb7e8bbea43" y="231.593186"/>
+ <use style="fill:#ff7f0e;fill-opacity:0.2;stroke:#ff7f0e;stroke-opacity:0.2;" x="429.9976" xlink:href="#mb7e8bbea43" y="229.981516"/>
+ <use style="fill:#ff7f0e;fill-opacity:0.2;stroke:#ff7f0e;stroke-opacity:0.2;" x="430.291482" xlink:href="#mb7e8bbea43" y="228.358851"/>
+ <use style="fill:#ff7f0e;fill-opacity:0.2;stroke:#ff7f0e;stroke-opacity:0.2;" x="430.53025" xlink:href="#mb7e8bbea43" y="226.725488"/>
+ <use style="fill:#ff7f0e;fill-opacity:0.2;stroke:#ff7f0e;stroke-opacity:0.2;" x="430.71424" xlink:href="#mb7e8bbea43" y="225.081709"/>
+ <use style="fill:#ff7f0e;fill-opacity:0.2;stroke:#ff7f0e;stroke-opacity:0.2;" x="430.843917" xlink:href="#mb7e8bbea43" y="223.427779"/>
+ <use style="fill:#ff7f0e;fill-opacity:0.2;stroke:#ff7f0e;stroke-opacity:0.2;" x="430.919868" xlink:href="#mb7e8bbea43" y="221.763946"/>
+ <use style="fill:#ff7f0e;fill-opacity:0.2;stroke:#ff7f0e;stroke-opacity:0.2;" x="430.942629" xlink:href="#mb7e8bbea43" y="220.090441"/>
+ <use style="fill:#ff7f0e;fill-opacity:0.2;stroke:#ff7f0e;stroke-opacity:0.2;" x="430.912302" xlink:href="#mb7e8bbea43" y="218.407476"/>
+ <use style="fill:#ff7f0e;fill-opacity:0.2;stroke:#ff7f0e;stroke-opacity:0.2;" x="430.829646" xlink:href="#mb7e8bbea43" y="216.715257"/>
+ <use style="fill:#ff7f0e;fill-opacity:0.2;stroke:#ff7f0e;stroke-opacity:0.2;" x="430.695132" xlink:href="#mb7e8bbea43" y="215.013982"/>
+ <use style="fill:#ff7f0e;fill-opacity:0.2;stroke:#ff7f0e;stroke-opacity:0.2;" x="430.509361" xlink:href="#mb7e8bbea43" y="213.303842"/>
+ <use style="fill:#ff7f0e;fill-opacity:0.2;stroke:#ff7f0e;stroke-opacity:0.2;" x="430.272999" xlink:href="#mb7e8bbea43" y="211.585027"/>
+ <use style="fill:#ff7f0e;fill-opacity:0.2;stroke:#ff7f0e;stroke-opacity:0.2;" x="429.986753" xlink:href="#mb7e8bbea43" y="209.857728"/>
+ <use style="fill:#ff7f0e;fill-opacity:0.2;stroke:#ff7f0e;stroke-opacity:0.2;" x="429.651362" xlink:href="#mb7e8bbea43" y="208.122144"/>
+ <use style="fill:#ff7f0e;fill-opacity:0.2;stroke:#ff7f0e;stroke-opacity:0.2;" x="429.267584" xlink:href="#mb7e8bbea43" y="206.378477"/>
+ <use style="fill:#ff7f0e;fill-opacity:0.2;stroke:#ff7f0e;stroke-opacity:0.2;" x="428.836189" xlink:href="#mb7e8bbea43" y="204.626945"/>
+ <use style="fill:#ff7f0e;fill-opacity:0.2;stroke:#ff7f0e;stroke-opacity:0.2;" x="428.357943" xlink:href="#mb7e8bbea43" y="202.867776"/>
+ <use style="fill:#ff7f0e;fill-opacity:0.2;stroke:#ff7f0e;stroke-opacity:0.2;" x="427.833602" xlink:href="#mb7e8bbea43" y="201.101219"/>
+ <use style="fill:#ff7f0e;fill-opacity:0.2;stroke:#ff7f0e;stroke-opacity:0.2;" x="427.263907" xlink:href="#mb7e8bbea43" y="199.327538"/>
+ <use style="fill:#ff7f0e;fill-opacity:0.2;stroke:#ff7f0e;stroke-opacity:0.2;" x="426.649568" xlink:href="#mb7e8bbea43" y="197.547022"/>
+ <use style="fill:#ff7f0e;fill-opacity:0.2;stroke:#ff7f0e;stroke-opacity:0.2;" x="425.991266" xlink:href="#mb7e8bbea43" y="195.75998"/>
+ <use style="fill:#ff7f0e;fill-opacity:0.2;stroke:#ff7f0e;stroke-opacity:0.2;" x="425.289642" xlink:href="#mb7e8bbea43" y="193.966746"/>
+ <use style="fill:#ff7f0e;fill-opacity:0.2;stroke:#ff7f0e;stroke-opacity:0.2;" x="424.545298" xlink:href="#mb7e8bbea43" y="192.16768"/>
+ <use style="fill:#ff7f0e;fill-opacity:0.2;stroke:#ff7f0e;stroke-opacity:0.2;" x="423.758786" xlink:href="#mb7e8bbea43" y="190.363166"/>
+ <use style="fill:#ff7f0e;fill-opacity:0.2;stroke:#ff7f0e;stroke-opacity:0.2;" x="422.930616" xlink:href="#mb7e8bbea43" y="188.553612"/>
+ <use style="fill:#ff7f0e;fill-opacity:0.2;stroke:#ff7f0e;stroke-opacity:0.2;" x="422.061248" xlink:href="#mb7e8bbea43" y="186.739452"/>
+ <use style="fill:#ff7f0e;fill-opacity:0.2;stroke:#ff7f0e;stroke-opacity:0.2;" x="421.151097" xlink:href="#mb7e8bbea43" y="184.921141"/>
+ <use style="fill:#ff7f0e;fill-opacity:0.2;stroke:#ff7f0e;stroke-opacity:0.2;" x="420.200534" xlink:href="#mb7e8bbea43" y="183.099156"/>
+ <use style="fill:#ff7f0e;fill-opacity:0.2;stroke:#ff7f0e;stroke-opacity:0.2;" x="419.209888" xlink:href="#mb7e8bbea43" y="181.273993"/>
+ <use style="fill:#ff7f0e;fill-opacity:0.2;stroke:#ff7f0e;stroke-opacity:0.2;" x="418.179523" xlink:href="#mb7e8bbea43" y="179.446184"/>
+ <use style="fill:#ff7f0e;fill-opacity:0.2;stroke:#ff7f0e;stroke-opacity:0.2;" x="417.110481" xlink:href="#mb7e8bbea43" y="177.616468"/>
+ <use style="fill:#ff7f0e;fill-opacity:0.2;stroke:#ff7f0e;stroke-opacity:0.2;" x="416.003136" xlink:href="#mb7e8bbea43" y="175.785385"/>
+ <use style="fill:#ff7f0e;fill-opacity:0.2;stroke:#ff7f0e;stroke-opacity:0.2;" x="414.858059" xlink:href="#mb7e8bbea43" y="173.95357"/>
+ <use style="fill:#ff7f0e;fill-opacity:0.2;stroke:#ff7f0e;stroke-opacity:0.2;" x="413.67582" xlink:href="#mb7e8bbea43" y="172.12164"/>
+ <use style="fill:#ff7f0e;fill-opacity:0.2;stroke:#ff7f0e;stroke-opacity:0.2;" x="412.456883" xlink:href="#mb7e8bbea43" y="170.290196"/>
+ <use style="fill:#ff7f0e;fill-opacity:0.2;stroke:#ff7f0e;stroke-opacity:0.2;" x="411.201678" xlink:href="#mb7e8bbea43" y="168.459828"/>
+ <use style="fill:#ff7f0e;fill-opacity:0.2;stroke:#ff7f0e;stroke-opacity:0.2;" x="409.910608" xlink:href="#mb7e8bbea43" y="166.631117"/>
+ <use style="fill:#ff7f0e;fill-opacity:0.2;stroke:#ff7f0e;stroke-opacity:0.2;" x="408.58406" xlink:href="#mb7e8bbea43" y="164.804637"/>
+ <use style="fill:#ff7f0e;fill-opacity:0.2;stroke:#ff7f0e;stroke-opacity:0.2;" x="407.222417" xlink:href="#mb7e8bbea43" y="162.980953"/>
+ <use style="fill:#ff7f0e;fill-opacity:0.2;stroke:#ff7f0e;stroke-opacity:0.2;" x="405.826066" xlink:href="#mb7e8bbea43" y="161.160623"/>
+ <use style="fill:#ff7f0e;fill-opacity:0.2;stroke:#ff7f0e;stroke-opacity:0.2;" x="404.395409" xlink:href="#mb7e8bbea43" y="159.344194"/>
+ <use style="fill:#ff7f0e;fill-opacity:0.2;stroke:#ff7f0e;stroke-opacity:0.2;" x="402.930869" xlink:href="#mb7e8bbea43" y="157.532206"/>
+ <use style="fill:#ff7f0e;fill-opacity:0.2;stroke:#ff7f0e;stroke-opacity:0.2;" x="401.432902" xlink:href="#mb7e8bbea43" y="155.725187"/>
+ <use style="fill:#ff7f0e;fill-opacity:0.2;stroke:#ff7f0e;stroke-opacity:0.2;" x="399.901999" xlink:href="#mb7e8bbea43" y="153.923652"/>
+ <use style="fill:#ff7f0e;fill-opacity:0.2;stroke:#ff7f0e;stroke-opacity:0.2;" x="398.338696" xlink:href="#mb7e8bbea43" y="152.128106"/>
+ <use style="fill:#ff7f0e;fill-opacity:0.2;stroke:#ff7f0e;stroke-opacity:0.2;" x="396.743578" xlink:href="#mb7e8bbea43" y="150.339036"/>
+ <use style="fill:#ff7f0e;fill-opacity:0.2;stroke:#ff7f0e;stroke-opacity:0.2;" x="395.11728" xlink:href="#mb7e8bbea43" y="148.556915"/>
+ <use style="fill:#ff7f0e;fill-opacity:0.2;stroke:#ff7f0e;stroke-opacity:0.2;" x="393.460493" xlink:href="#mb7e8bbea43" y="146.782198"/>
+ <use style="fill:#ff7f0e;fill-opacity:0.2;stroke:#ff7f0e;stroke-opacity:0.2;" x="391.773964" xlink:href="#mb7e8bbea43" y="145.01532"/>
+ <use style="fill:#ff7f0e;fill-opacity:0.2;stroke:#ff7f0e;stroke-opacity:0.2;" x="390.058496" xlink:href="#mb7e8bbea43" y="143.256695"/>
+ <use style="fill:#ff7f0e;fill-opacity:0.2;stroke:#ff7f0e;stroke-opacity:0.2;" x="388.314946" xlink:href="#mb7e8bbea43" y="141.506717"/>
+ <use style="fill:#ff7f0e;fill-opacity:0.2;stroke:#ff7f0e;stroke-opacity:0.2;" x="386.544227" xlink:href="#mb7e8bbea43" y="139.765754"/>
+ <use style="fill:#ff7f0e;fill-opacity:0.2;stroke:#ff7f0e;stroke-opacity:0.2;" x="384.747299" xlink:href="#mb7e8bbea43" y="138.034152"/>
+ <use style="fill:#ff7f0e;fill-opacity:0.2;stroke:#ff7f0e;stroke-opacity:0.2;" x="382.92517" xlink:href="#mb7e8bbea43" y="136.312233"/>
+ <use style="fill:#ff7f0e;fill-opacity:0.2;stroke:#ff7f0e;stroke-opacity:0.2;" x="381.078889" xlink:href="#mb7e8bbea43" y="134.600292"/>
+ <use style="fill:#ff7f0e;fill-opacity:0.2;stroke:#ff7f0e;stroke-opacity:0.2;" x="379.209538" xlink:href="#mb7e8bbea43" y="132.898602"/>
+ <use style="fill:#ff7f0e;fill-opacity:0.2;stroke:#ff7f0e;stroke-opacity:0.2;" x="377.318232" xlink:href="#mb7e8bbea43" y="131.207412"/>
+ <use style="fill:#ff7f0e;fill-opacity:0.2;stroke:#ff7f0e;stroke-opacity:0.2;" x="375.406103" xlink:href="#mb7e8bbea43" y="129.526947"/>
+ <use style="fill:#ff7f0e;fill-opacity:0.2;stroke:#ff7f0e;stroke-opacity:0.2;" x="373.474302" xlink:href="#mb7e8bbea43" y="127.857411"/>
+ <use style="fill:#ff7f0e;fill-opacity:0.2;stroke:#ff7f0e;stroke-opacity:0.2;" x="371.523985" xlink:href="#mb7e8bbea43" y="126.198991"/>
+ <use style="fill:#ff7f0e;fill-opacity:0.2;stroke:#ff7f0e;stroke-opacity:0.2;" x="369.556276" xlink:href="#mb7e8bbea43" y="124.551894"/>
+ <use style="fill:#ff7f0e;fill-opacity:0.2;stroke:#ff7f0e;stroke-opacity:0.2;" x="367.571848" xlink:href="#mb7e8bbea43" y="122.916924"/>
+ <use style="fill:#ff7f0e;fill-opacity:0.2;stroke:#ff7f0e;stroke-opacity:0.2;" x="365.571802" xlink:href="#mb7e8bbea43" y="121.294409"/>
+ <use style="fill:#ff7f0e;fill-opacity:0.2;stroke:#ff7f0e;stroke-opacity:0.2;" x="363.557108" xlink:href="#mb7e8bbea43" y="119.68485"/>
+ <use style="fill:#ff7f0e;fill-opacity:0.2;stroke:#ff7f0e;stroke-opacity:0.2;" x="361.528725" xlink:href="#mb7e8bbea43" y="118.088775"/>
+ <use style="fill:#ff7f0e;fill-opacity:0.2;stroke:#ff7f0e;stroke-opacity:0.2;" x="359.48767" xlink:href="#mb7e8bbea43" y="116.506651"/>
+ <use style="fill:#ff7f0e;fill-opacity:0.2;stroke:#ff7f0e;stroke-opacity:0.2;" x="357.43497" xlink:href="#mb7e8bbea43" y="114.938929"/>
+ <use style="fill:#ff7f0e;fill-opacity:0.2;stroke:#ff7f0e;stroke-opacity:0.2;" x="355.371654" xlink:href="#mb7e8bbea43" y="113.38604"/>
+ <use style="fill:#ff7f0e;fill-opacity:0.2;stroke:#ff7f0e;stroke-opacity:0.2;" x="353.298748" xlink:href="#mb7e8bbea43" y="111.848393"/>
+ <use style="fill:#ff7f0e;fill-opacity:0.2;stroke:#ff7f0e;stroke-opacity:0.2;" x="351.217264" xlink:href="#mb7e8bbea43" y="110.326381"/>
+ <use style="fill:#ff7f0e;fill-opacity:0.2;stroke:#ff7f0e;stroke-opacity:0.2;" x="349.128198" xlink:href="#mb7e8bbea43" y="108.820375"/>
+ <use style="fill:#ff7f0e;fill-opacity:0.2;stroke:#ff7f0e;stroke-opacity:0.2;" x="347.032526" xlink:href="#mb7e8bbea43" y="107.330728"/>
+ <use style="fill:#ff7f0e;fill-opacity:0.2;stroke:#ff7f0e;stroke-opacity:0.2;" x="344.9312" xlink:href="#mb7e8bbea43" y="105.85777"/>
+ <use style="fill:#ff7f0e;fill-opacity:0.2;stroke:#ff7f0e;stroke-opacity:0.2;" x="342.825143" xlink:href="#mb7e8bbea43" y="104.401812"/>
+ <use style="fill:#ff7f0e;fill-opacity:0.2;stroke:#ff7f0e;stroke-opacity:0.2;" x="340.715252" xlink:href="#mb7e8bbea43" y="102.963143"/>
+ <use style="fill:#ff7f0e;fill-opacity:0.2;stroke:#ff7f0e;stroke-opacity:0.2;" x="338.602394" xlink:href="#mb7e8bbea43" y="101.542026"/>
+ <use style="fill:#ff7f0e;fill-opacity:0.2;stroke:#ff7f0e;stroke-opacity:0.2;" x="336.487409" xlink:href="#mb7e8bbea43" y="100.138707"/>
+ <use style="fill:#ff7f0e;fill-opacity:0.2;stroke:#ff7f0e;stroke-opacity:0.2;" x="334.371107" xlink:href="#mb7e8bbea43" y="98.753404"/>
+ <use style="fill:#ff7f0e;fill-opacity:0.2;stroke:#ff7f0e;stroke-opacity:0.2;" x="332.254273" xlink:href="#mb7e8bbea43" y="97.386313"/>
+ <use style="fill:#ff7f0e;fill-opacity:0.2;stroke:#ff7f0e;stroke-opacity:0.2;" x="330.137667" xlink:href="#mb7e8bbea43" y="96.037604"/>
+ <use style="fill:#ff7f0e;fill-opacity:0.2;stroke:#ff7f0e;stroke-opacity:0.2;" x="328.022028" xlink:href="#mb7e8bbea43" y="94.707426"/>
+ <use style="fill:#ff7f0e;fill-opacity:0.2;stroke:#ff7f0e;stroke-opacity:0.2;" x="325.908075" xlink:href="#mb7e8bbea43" y="93.395899"/>
+ <use style="fill:#ff7f0e;fill-opacity:0.2;stroke:#ff7f0e;stroke-opacity:0.2;" x="323.796507" xlink:href="#mb7e8bbea43" y="92.103122"/>
+ <use style="fill:#ff7f0e;fill-opacity:0.2;stroke:#ff7f0e;stroke-opacity:0.2;" x="321.688013" xlink:href="#mb7e8bbea43" y="90.829167"/>
+ <use style="fill:#ff7f0e;fill-opacity:0.2;stroke:#ff7f0e;stroke-opacity:0.2;" x="319.583267" xlink:href="#mb7e8bbea43" y="89.574082"/>
+ <use style="fill:#ff7f0e;fill-opacity:0.2;stroke:#ff7f0e;stroke-opacity:0.2;" x="317.482935" xlink:href="#mb7e8bbea43" y="88.337894"/>
+ <use style="fill:#ff7f0e;fill-opacity:0.2;stroke:#ff7f0e;stroke-opacity:0.2;" x="315.387675" xlink:href="#mb7e8bbea43" y="87.120603"/>
+ <use style="fill:#ff7f0e;fill-opacity:0.2;stroke:#ff7f0e;stroke-opacity:0.2;" x="313.298141" xlink:href="#mb7e8bbea43" y="85.922188"/>
+ <use style="fill:#ff7f0e;fill-opacity:0.2;stroke:#ff7f0e;stroke-opacity:0.2;" x="311.214981" xlink:href="#mb7e8bbea43" y="84.742605"/>
+ <use style="fill:#ff7f0e;fill-opacity:0.2;stroke:#ff7f0e;stroke-opacity:0.2;" x="309.138842" xlink:href="#mb7e8bbea43" y="83.581791"/>
+ <use style="fill:#ff7f0e;fill-opacity:0.2;stroke:#ff7f0e;stroke-opacity:0.2;" x="307.07037" xlink:href="#mb7e8bbea43" y="82.43966"/>
+ <use style="fill:#ff7f0e;fill-opacity:0.2;stroke:#ff7f0e;stroke-opacity:0.2;" x="305.010207" xlink:href="#mb7e8bbea43" y="81.316108"/>
+ <use style="fill:#ff7f0e;fill-opacity:0.2;stroke:#ff7f0e;stroke-opacity:0.2;" x="302.958994" xlink:href="#mb7e8bbea43" y="80.211012"/>
+ <use style="fill:#ff7f0e;fill-opacity:0.2;stroke:#ff7f0e;stroke-opacity:0.2;" x="300.917371" xlink:href="#mb7e8bbea43" y="79.124234"/>
+ <use style="fill:#ff7f0e;fill-opacity:0.2;stroke:#ff7f0e;stroke-opacity:0.2;" x="298.88597" xlink:href="#mb7e8bbea43" y="78.055617"/>
+ <use style="fill:#ff7f0e;fill-opacity:0.2;stroke:#ff7f0e;stroke-opacity:0.2;" x="296.865124" xlink:href="#mb7e8bbea43" y="77.005152"/>
+ <use style="fill:#ff7f0e;fill-opacity:0.2;stroke:#ff7f0e;stroke-opacity:0.2;" x="294.855272" xlink:href="#mb7e8bbea43" y="75.972739"/>
+ <use style="fill:#ff7f0e;fill-opacity:0.2;stroke:#ff7f0e;stroke-opacity:0.2;" x="292.85687" xlink:href="#mb7e8bbea43" y="74.958253"/>
+ <use style="fill:#ff7f0e;fill-opacity:0.2;stroke:#ff7f0e;stroke-opacity:0.2;" x="290.870276" xlink:href="#mb7e8bbea43" y="73.961599"/>
+ <use style="fill:#ff7f0e;fill-opacity:0.2;stroke:#ff7f0e;stroke-opacity:0.2;" x="288.89587" xlink:href="#mb7e8bbea43" y="72.982641"/>
+ <use style="fill:#ff7f0e;fill-opacity:0.2;stroke:#ff7f0e;stroke-opacity:0.2;" x="286.934024" xlink:href="#mb7e8bbea43" y="72.021228"/>
+ <use style="fill:#ff7f0e;fill-opacity:0.2;stroke:#ff7f0e;stroke-opacity:0.2;" x="284.985098" xlink:href="#mb7e8bbea43" y="71.077192"/>
+ <use style="fill:#ff7f0e;fill-opacity:0.2;stroke:#ff7f0e;stroke-opacity:0.2;" x="283.049417" xlink:href="#mb7e8bbea43" y="70.150398"/>
+ <use style="fill:#ff7f0e;fill-opacity:0.2;stroke:#ff7f0e;stroke-opacity:0.2;" x="281.127164" xlink:href="#mb7e8bbea43" y="69.241055"/>
+ <use style="fill:#ff7f0e;fill-opacity:0.2;stroke:#ff7f0e;stroke-opacity:0.2;" x="279.218618" xlink:href="#mb7e8bbea43" y="68.349026"/>
+ <use style="fill:#ff7f0e;fill-opacity:0.2;stroke:#ff7f0e;stroke-opacity:0.2;" x="277.323992" xlink:href="#mb7e8bbea43" y="67.474334"/>
+ <use style="fill:#ff7f0e;fill-opacity:0.2;stroke:#ff7f0e;stroke-opacity:0.2;" x="275.443466" xlink:href="#mb7e8bbea43" y="66.61697"/>
+ <use style="fill:#ff7f0e;fill-opacity:0.2;stroke:#ff7f0e;stroke-opacity:0.2;" x="273.577209" xlink:href="#mb7e8bbea43" y="65.776891"/>
+ <use style="fill:#ff7f0e;fill-opacity:0.2;stroke:#ff7f0e;stroke-opacity:0.2;" x="271.725363" xlink:href="#mb7e8bbea43" y="64.954033"/>
+ <use style="fill:#ff7f0e;fill-opacity:0.2;stroke:#ff7f0e;stroke-opacity:0.2;" x="269.888039" xlink:href="#mb7e8bbea43" y="64.14831"/>
+ <use style="fill:#ff7f0e;fill-opacity:0.2;stroke:#ff7f0e;stroke-opacity:0.2;" x="268.06532" xlink:href="#mb7e8bbea43" y="63.359616"/>
+ <use style="fill:#ff7f0e;fill-opacity:0.2;stroke:#ff7f0e;stroke-opacity:0.2;" x="266.257254" xlink:href="#mb7e8bbea43" y="62.587827"/>
+ <use style="fill:#ff7f0e;fill-opacity:0.2;stroke:#ff7f0e;stroke-opacity:0.2;" x="264.463856" xlink:href="#mb7e8bbea43" y="61.832795"/>
+ <use style="fill:#ff7f0e;fill-opacity:0.2;stroke:#ff7f0e;stroke-opacity:0.2;" x="262.685106" xlink:href="#mb7e8bbea43" y="61.094358"/>
+ <use style="fill:#ff7f0e;fill-opacity:0.2;stroke:#ff7f0e;stroke-opacity:0.2;" x="260.92095" xlink:href="#mb7e8bbea43" y="60.372334"/>
+ <use style="fill:#ff7f0e;fill-opacity:0.2;stroke:#ff7f0e;stroke-opacity:0.2;" x="259.171294" xlink:href="#mb7e8bbea43" y="59.666522"/>
+ <use style="fill:#ff7f0e;fill-opacity:0.2;stroke:#ff7f0e;stroke-opacity:0.2;" x="257.436013" xlink:href="#mb7e8bbea43" y="58.976706"/>
+ <use style="fill:#ff7f0e;fill-opacity:0.2;stroke:#ff7f0e;stroke-opacity:0.2;" x="255.714942" xlink:href="#mb7e8bbea43" y="58.302654"/>
+ <use style="fill:#ff7f0e;fill-opacity:0.2;stroke:#ff7f0e;stroke-opacity:0.2;" x="254.007885" xlink:href="#mb7e8bbea43" y="57.64412"/>
+ <use style="fill:#ff7f0e;fill-opacity:0.2;stroke:#ff7f0e;stroke-opacity:0.2;" x="252.314606" xlink:href="#mb7e8bbea43" y="57.000844"/>
+ <use style="fill:#ff7f0e;fill-opacity:0.2;stroke:#ff7f0e;stroke-opacity:0.2;" x="250.634841" xlink:href="#mb7e8bbea43" y="56.372552"/>
+ <use style="fill:#ff7f0e;fill-opacity:0.2;stroke:#ff7f0e;stroke-opacity:0.2;" x="248.968288" xlink:href="#mb7e8bbea43" y="55.758961"/>
+ <use style="fill:#ff7f0e;fill-opacity:0.2;stroke:#ff7f0e;stroke-opacity:0.2;" x="247.314618" xlink:href="#mb7e8bbea43" y="55.159773"/>
+ <use style="fill:#ff7f0e;fill-opacity:0.2;stroke:#ff7f0e;stroke-opacity:0.2;" x="245.673468" xlink:href="#mb7e8bbea43" y="54.574683"/>
+ <use style="fill:#ff7f0e;fill-opacity:0.2;stroke:#ff7f0e;stroke-opacity:0.2;" x="244.044447" xlink:href="#mb7e8bbea43" y="54.003377"/>
+ <use style="fill:#ff7f0e;fill-opacity:0.2;stroke:#ff7f0e;stroke-opacity:0.2;" x="242.427139" xlink:href="#mb7e8bbea43" y="53.445531"/>
+ <use style="fill:#ff7f0e;fill-opacity:0.2;stroke:#ff7f0e;stroke-opacity:0.2;" x="240.821098" xlink:href="#mb7e8bbea43" y="52.900817"/>
+ <use style="fill:#ff7f0e;fill-opacity:0.2;stroke:#ff7f0e;stroke-opacity:0.2;" x="239.225855" xlink:href="#mb7e8bbea43" y="52.368897"/>
+ <use style="fill:#ff7f0e;fill-opacity:0.2;stroke:#ff7f0e;stroke-opacity:0.2;" x="237.640918" xlink:href="#mb7e8bbea43" y="51.849431"/>
+ <use style="fill:#ff7f0e;fill-opacity:0.2;stroke:#ff7f0e;stroke-opacity:0.2;" x="236.065773" xlink:href="#mb7e8bbea43" y="51.342074"/>
+ <use style="fill:#ff7f0e;fill-opacity:0.2;stroke:#ff7f0e;stroke-opacity:0.2;" x="234.499886" xlink:href="#mb7e8bbea43" y="50.846477"/>
+ <use style="fill:#ff7f0e;fill-opacity:0.2;stroke:#ff7f0e;stroke-opacity:0.2;" x="232.942703" xlink:href="#mb7e8bbea43" y="50.362287"/>
+ <use style="fill:#ff7f0e;fill-opacity:0.2;stroke:#ff7f0e;stroke-opacity:0.2;" x="231.393654" xlink:href="#mb7e8bbea43" y="49.889151"/>
+ <use style="fill:#ff7f0e;fill-opacity:0.2;stroke:#ff7f0e;stroke-opacity:0.2;" x="229.852154" xlink:href="#mb7e8bbea43" y="49.426712"/>
+ <use style="fill:#ff7f0e;fill-opacity:0.2;stroke:#ff7f0e;stroke-opacity:0.2;" x="228.317601" xlink:href="#mb7e8bbea43" y="48.974611"/>
+ <use style="fill:#ff7f0e;fill-opacity:0.2;stroke:#ff7f0e;stroke-opacity:0.2;" x="226.789382" xlink:href="#mb7e8bbea43" y="48.532493"/>
+ <use style="fill:#ff7f0e;fill-opacity:0.2;stroke:#ff7f0e;stroke-opacity:0.2;" x="225.266872" xlink:href="#mb7e8bbea43" y="48.099997"/>
+ <use style="fill:#ff7f0e;fill-opacity:0.2;stroke:#ff7f0e;stroke-opacity:0.2;" x="223.749435" xlink:href="#mb7e8bbea43" y="47.676766"/>
+ <use style="fill:#ff7f0e;fill-opacity:0.2;stroke:#ff7f0e;stroke-opacity:0.2;" x="222.236424" xlink:href="#mb7e8bbea43" y="47.262444"/>
+ <use style="fill:#ff7f0e;fill-opacity:0.2;stroke:#ff7f0e;stroke-opacity:0.2;" x="220.727185" xlink:href="#mb7e8bbea43" y="46.856675"/>
+ <use style="fill:#ff7f0e;fill-opacity:0.2;stroke:#ff7f0e;stroke-opacity:0.2;" x="219.221058" xlink:href="#mb7e8bbea43" y="46.459104"/>
+ <use style="fill:#ff7f0e;fill-opacity:0.2;stroke:#ff7f0e;stroke-opacity:0.2;" x="217.717375" xlink:href="#mb7e8bbea43" y="46.06938"/>
+ <use style="fill:#ff7f0e;fill-opacity:0.2;stroke:#ff7f0e;stroke-opacity:0.2;" x="216.215463" xlink:href="#mb7e8bbea43" y="45.687153"/>
+ <use style="fill:#ff7f0e;fill-opacity:0.2;stroke:#ff7f0e;stroke-opacity:0.2;" x="214.714645" xlink:href="#mb7e8bbea43" y="45.312076"/>
+ <use style="fill:#ff7f0e;fill-opacity:0.2;stroke:#ff7f0e;stroke-opacity:0.2;" x="213.214241" xlink:href="#mb7e8bbea43" y="44.943807"/>
+ <use style="fill:#ff7f0e;fill-opacity:0.2;stroke:#ff7f0e;stroke-opacity:0.2;" x="211.713567" xlink:href="#mb7e8bbea43" y="44.582004"/>
+ <use style="fill:#ff7f0e;fill-opacity:0.2;stroke:#ff7f0e;stroke-opacity:0.2;" x="210.211939" xlink:href="#mb7e8bbea43" y="44.226332"/>
+ <use style="fill:#ff7f0e;fill-opacity:0.2;stroke:#ff7f0e;stroke-opacity:0.2;" x="208.708672" xlink:href="#mb7e8bbea43" y="43.876458"/>
+ <use style="fill:#ff7f0e;fill-opacity:0.2;stroke:#ff7f0e;stroke-opacity:0.2;" x="207.20308" xlink:href="#mb7e8bbea43" y="43.532055"/>
+ <use style="fill:#ff7f0e;fill-opacity:0.2;stroke:#ff7f0e;stroke-opacity:0.2;" x="205.69448" xlink:href="#mb7e8bbea43" y="43.192798"/>
+ <use style="fill:#ff7f0e;fill-opacity:0.2;stroke:#ff7f0e;stroke-opacity:0.2;" x="204.182188" xlink:href="#mb7e8bbea43" y="42.85837"/>
+ <use style="fill:#ff7f0e;fill-opacity:0.2;stroke:#ff7f0e;stroke-opacity:0.2;" x="202.665524" xlink:href="#mb7e8bbea43" y="42.528457"/>
+ <use style="fill:#ff7f0e;fill-opacity:0.2;stroke:#ff7f0e;stroke-opacity:0.2;" x="201.143812" xlink:href="#mb7e8bbea43" y="42.202752"/>
+ <use style="fill:#ff7f0e;fill-opacity:0.2;stroke:#ff7f0e;stroke-opacity:0.2;" x="199.616379" xlink:href="#mb7e8bbea43" y="41.880952"/>
+ <use style="fill:#ff7f0e;fill-opacity:0.2;stroke:#ff7f0e;stroke-opacity:0.2;" x="198.082557" xlink:href="#mb7e8bbea43" y="41.562762"/>
+ <use style="fill:#ff7f0e;fill-opacity:0.2;stroke:#ff7f0e;stroke-opacity:0.2;" x="196.541684" xlink:href="#mb7e8bbea43" y="41.247891"/>
+ <use style="fill:#ff7f0e;fill-opacity:0.2;stroke:#ff7f0e;stroke-opacity:0.2;" x="194.993104" xlink:href="#mb7e8bbea43" y="40.936056"/>
+ <use style="fill:#ff7f0e;fill-opacity:0.2;stroke:#ff7f0e;stroke-opacity:0.2;" x="193.436169" xlink:href="#mb7e8bbea43" y="40.62698"/>
+ <use style="fill:#ff7f0e;fill-opacity:0.2;stroke:#ff7f0e;stroke-opacity:0.2;" x="191.870237" xlink:href="#mb7e8bbea43" y="40.320394"/>
+ <use style="fill:#ff7f0e;fill-opacity:0.2;stroke:#ff7f0e;stroke-opacity:0.2;" x="190.294676" xlink:href="#mb7e8bbea43" y="40.016035"/>
+ <use style="fill:#ff7f0e;fill-opacity:0.2;stroke:#ff7f0e;stroke-opacity:0.2;" x="188.708862" xlink:href="#mb7e8bbea43" y="39.713647"/>
+ <use style="fill:#ff7f0e;fill-opacity:0.2;stroke:#ff7f0e;stroke-opacity:0.2;" x="187.112181" xlink:href="#mb7e8bbea43" y="39.412983"/>
+ <use style="fill:#ff7f0e;fill-opacity:0.2;stroke:#ff7f0e;stroke-opacity:0.2;" x="185.504029" xlink:href="#mb7e8bbea43" y="39.113803"/>
+ <use style="fill:#ff7f0e;fill-opacity:0.2;stroke:#ff7f0e;stroke-opacity:0.2;" x="183.883812" xlink:href="#mb7e8bbea43" y="38.815876"/>
+ <use style="fill:#ff7f0e;fill-opacity:0.2;stroke:#ff7f0e;stroke-opacity:0.2;" x="182.250945" xlink:href="#mb7e8bbea43" y="38.518978"/>
+ <use style="fill:#ff7f0e;fill-opacity:0.2;stroke:#ff7f0e;stroke-opacity:0.2;" x="180.604857" xlink:href="#mb7e8bbea43" y="38.222893"/>
+ <use style="fill:#ff7f0e;fill-opacity:0.2;stroke:#ff7f0e;stroke-opacity:0.2;" x="178.944985" xlink:href="#mb7e8bbea43" y="37.927416"/>
+ <use style="fill:#ff7f0e;fill-opacity:0.2;stroke:#ff7f0e;stroke-opacity:0.2;" x="177.270777" xlink:href="#mb7e8bbea43" y="37.632347"/>
+ <use style="fill:#ff7f0e;fill-opacity:0.2;stroke:#ff7f0e;stroke-opacity:0.2;" x="175.581693" xlink:href="#mb7e8bbea43" y="37.337498"/>
+ <use style="fill:#ff7f0e;fill-opacity:0.2;stroke:#ff7f0e;stroke-opacity:0.2;" x="173.877202" xlink:href="#mb7e8bbea43" y="37.042688"/>
+ <use style="fill:#ff7f0e;fill-opacity:0.2;stroke:#ff7f0e;stroke-opacity:0.2;" x="172.156784" xlink:href="#mb7e8bbea43" y="36.747746"/>
+ <use style="fill:#ff7f0e;fill-opacity:0.2;stroke:#ff7f0e;stroke-opacity:0.2;" x="170.419926" xlink:href="#mb7e8bbea43" y="36.452509"/>
+ <use style="fill:#ff7f0e;fill-opacity:0.2;stroke:#ff7f0e;stroke-opacity:0.2;" x="168.666128" xlink:href="#mb7e8bbea43" y="36.156824"/>
+ <use style="fill:#ff7f0e;fill-opacity:0.2;stroke:#ff7f0e;stroke-opacity:0.2;" x="166.894894" xlink:href="#mb7e8bbea43" y="35.860547"/>
+ <use style="fill:#ff7f0e;fill-opacity:0.2;stroke:#ff7f0e;stroke-opacity:0.2;" x="165.105737" xlink:href="#mb7e8bbea43" y="35.563543"/>
+ <use style="fill:#ff7f0e;fill-opacity:0.2;stroke:#ff7f0e;stroke-opacity:0.2;" x="163.298176" xlink:href="#mb7e8bbea43" y="35.265684"/>
+ <use style="fill:#ff7f0e;fill-opacity:0.2;stroke:#ff7f0e;stroke-opacity:0.2;" x="161.471736" xlink:href="#mb7e8bbea43" y="34.966855"/>
+ <use style="fill:#ff7f0e;fill-opacity:0.2;stroke:#ff7f0e;stroke-opacity:0.2;" x="159.625945" xlink:href="#mb7e8bbea43" y="34.666945"/>
+ <use style="fill:#ff7f0e;fill-opacity:0.2;stroke:#ff7f0e;stroke-opacity:0.2;" x="157.760333" xlink:href="#mb7e8bbea43" y="34.365857"/>
+ <use style="fill:#ff7f0e;fill-opacity:0.2;stroke:#ff7f0e;stroke-opacity:0.2;" x="155.874433" xlink:href="#mb7e8bbea43" y="34.063497"/>
+ <use style="fill:#ff7f0e;fill-opacity:0.2;stroke:#ff7f0e;stroke-opacity:0.2;" x="153.967777" xlink:href="#mb7e8bbea43" y="33.759785"/>
+ <use style="fill:#ff7f0e;fill-opacity:0.2;stroke:#ff7f0e;stroke-opacity:0.2;" x="152.039896" xlink:href="#mb7e8bbea43" y="33.454644"/>
+ <use style="fill:#ff7f0e;fill-opacity:0.2;stroke:#ff7f0e;stroke-opacity:0.2;" x="150.090316" xlink:href="#mb7e8bbea43" y="33.148009"/>
+ <use style="fill:#ff7f0e;fill-opacity:0.2;stroke:#ff7f0e;stroke-opacity:0.2;" x="148.118562" xlink:href="#mb7e8bbea43" y="32.83982"/>
+ <use style="fill:#ff7f0e;fill-opacity:0.2;stroke:#ff7f0e;stroke-opacity:0.2;" x="146.124091" xlink:href="#mb7e8bbea43" y="32.530216"/>
+ <use style="fill:#ff7f0e;fill-opacity:0.2;stroke:#ff7f0e;stroke-opacity:0.2;" x="144.10637" xlink:href="#mb7e8bbea43" y="32.219325"/>
+ <use style="fill:#ff7f0e;fill-opacity:0.2;stroke:#ff7f0e;stroke-opacity:0.2;" x="142.064896" xlink:href="#mb7e8bbea43" y="31.907201"/>
+ <use style="fill:#ff7f0e;fill-opacity:0.2;stroke:#ff7f0e;stroke-opacity:0.2;" x="139.999141" xlink:href="#mb7e8bbea43" y="31.594002"/>
+ <use style="fill:#ff7f0e;fill-opacity:0.2;stroke:#ff7f0e;stroke-opacity:0.2;" x="137.908586" xlink:href="#mb7e8bbea43" y="31.279852"/>
+ <use style="fill:#ff7f0e;fill-opacity:0.2;stroke:#ff7f0e;stroke-opacity:0.2;" x="135.792725" xlink:href="#mb7e8bbea43" y="30.964877"/>
+ <use style="fill:#ff7f0e;fill-opacity:0.2;stroke:#ff7f0e;stroke-opacity:0.2;" x="133.651055" xlink:href="#mb7e8bbea43" y="30.649203"/>
+ <use style="fill:#ff7f0e;fill-opacity:0.2;stroke:#ff7f0e;stroke-opacity:0.2;" x="131.48308" xlink:href="#mb7e8bbea43" y="30.332955"/>
+ <use style="fill:#ff7f0e;fill-opacity:0.2;stroke:#ff7f0e;stroke-opacity:0.2;" x="129.288314" xlink:href="#mb7e8bbea43" y="30.016257"/>
+ <use style="fill:#ff7f0e;fill-opacity:0.2;stroke:#ff7f0e;stroke-opacity:0.2;" x="127.066277" xlink:href="#mb7e8bbea43" y="29.699229"/>
+ <use style="fill:#ff7f0e;fill-opacity:0.2;stroke:#ff7f0e;stroke-opacity:0.2;" x="124.8165" xlink:href="#mb7e8bbea43" y="29.38199"/>
+ <use style="fill:#ff7f0e;fill-opacity:0.2;stroke:#ff7f0e;stroke-opacity:0.2;" x="122.538524" xlink:href="#mb7e8bbea43" y="29.064652"/>
+ <use style="fill:#ff7f0e;fill-opacity:0.2;stroke:#ff7f0e;stroke-opacity:0.2;" x="120.231903" xlink:href="#mb7e8bbea43" y="28.747323"/>
+ <use style="fill:#ff7f0e;fill-opacity:0.2;stroke:#ff7f0e;stroke-opacity:0.2;" x="117.896206" xlink:href="#mb7e8bbea43" y="28.430106"/>
+ <use style="fill:#ff7f0e;fill-opacity:0.2;stroke:#ff7f0e;stroke-opacity:0.2;" x="115.531014" xlink:href="#mb7e8bbea43" y="28.113095"/>
+ <use style="fill:#ff7f0e;fill-opacity:0.2;stroke:#ff7f0e;stroke-opacity:0.2;" x="113.135929" xlink:href="#mb7e8bbea43" y="27.796378"/>
+ <use style="fill:#ff7f0e;fill-opacity:0.2;stroke:#ff7f0e;stroke-opacity:0.2;" x="110.71057" xlink:href="#mb7e8bbea43" y="27.480033"/>
+ <use style="fill:#ff7f0e;fill-opacity:0.2;stroke:#ff7f0e;stroke-opacity:0.2;" x="108.254581" xlink:href="#mb7e8bbea43" y="27.16413"/>
+ <use style="fill:#ff7f0e;fill-opacity:0.2;stroke:#ff7f0e;stroke-opacity:0.2;" x="105.767628" xlink:href="#mb7e8bbea43" y="26.848728"/>
+ <use style="fill:#ff7f0e;fill-opacity:0.2;stroke:#ff7f0e;stroke-opacity:0.2;" x="103.249406" xlink:href="#mb7e8bbea43" y="26.533876"/>
+ <use style="fill:#ff7f0e;fill-opacity:0.2;stroke:#ff7f0e;stroke-opacity:0.2;" x="100.69964" xlink:href="#mb7e8bbea43" y="26.219611"/>
+ <use style="fill:#ff7f0e;fill-opacity:0.2;stroke:#ff7f0e;stroke-opacity:0.2;" x="98.11809" xlink:href="#mb7e8bbea43" y="25.905959"/>
+ <use style="fill:#ff7f0e;fill-opacity:0.2;stroke:#ff7f0e;stroke-opacity:0.2;" x="95.50455" xlink:href="#mb7e8bbea43" y="25.592935"/>
+ <use style="fill:#ff7f0e;fill-opacity:0.2;stroke:#ff7f0e;stroke-opacity:0.2;" x="92.858856" xlink:href="#mb7e8bbea43" y="25.28054"/>
+ <use style="fill:#ff7f0e;fill-opacity:0.2;stroke:#ff7f0e;stroke-opacity:0.2;" x="90.180889" xlink:href="#mb7e8bbea43" y="24.968764"/>
+ <use style="fill:#ff7f0e;fill-opacity:0.2;stroke:#ff7f0e;stroke-opacity:0.2;" x="87.470574" xlink:href="#mb7e8bbea43" y="24.657583"/>
+ <use style="fill:#ff7f0e;fill-opacity:0.2;stroke:#ff7f0e;stroke-opacity:0.2;" x="84.727888" xlink:href="#mb7e8bbea43" y="24.346961"/>
+ <use style="fill:#ff7f0e;fill-opacity:0.2;stroke:#ff7f0e;stroke-opacity:0.2;" x="81.952859" xlink:href="#mb7e8bbea43" y="24.036851"/>
+ </g>
+ </g>
+ <g id="line2d_13">
+ <path clip-path="url(#p444e95376c)" d="M 369.666423 288.740154
+L 376.76364 285.074491
+L 383.58814 281.305297
+L 389.19387 277.978761
+L 394.551311 274.55228
+L 399.638644 271.015314
+L 403.655827 267.976269
+L 407.456839 264.848402
+L 411.027709 261.627234
+L 414.35357 258.309266
+L 417.418631 254.892249
+L 419.671827 252.086772
+L 421.73899 249.217825
+L 423.61175 246.286598
+L 425.282308 243.295067
+L 426.743891 240.245982
+L 427.991267 237.142813
+L 429.020884 233.989387
+L 429.829978 230.78874
+L 430.417787 227.543482
+L 430.785878 224.255989
+L 430.937865 220.928379
+L 430.877515 217.562502
+L 430.608632 214.16
+L 430.136073 210.722418
+L 429.465469 207.251301
+L 428.357924 202.86777
+L 426.962243 198.43811
+L 425.289599 193.96674
+L 423.349826 189.458986
+L 421.151037 184.921132
+L 418.699597 180.360377
+L 416.003058 175.785363
+L 413.070839 171.205797
+L 409.91054 166.631093
+L 406.528492 162.07031
+L 402.185974 156.628016
+L 397.545017 151.232702
+L 392.620838 145.89772
+L 387.432858 140.635051
+L 382.004907 135.454961
+L 376.364611 130.365789
+L 369.556093 124.551965
+L 362.544461 118.885065
+L 355.371533 113.386007
+L 348.080993 108.073454
+L 340.715108 102.963111
+L 332.254123 97.386282
+L 323.796355 92.103092
+L 315.387522 87.120575
+L 307.070214 82.439634
+L 297.873992 77.528121
+L 288.895698 72.982617
+L 279.218438 68.349003
+L 269.887847 64.148287
+L 260.92075 60.372312
+L 252.314401 57.000823
+L 244.044238 54.003358
+L 236.065559 51.342056
+L 227.552518 48.752309
+L 219.22083 46.459086
+L 210.211702 44.226314
+L 200.380609 42.041365
+L 189.502842 39.864591
+L 176.427884 37.484887
+L 159.625709 34.666924
+L 138.956776 31.437014
+L 121.388621 28.905959
+L 104.512265 26.691215
+L 86.103126 24.502198
+L 81.952714 24.036846
+L 81.952714 24.036846
+" style="fill:none;stroke:#2ca02c;stroke-dasharray:1.5,4.5,1.5;stroke-dashoffset:0;stroke-opacity:0.2;stroke-width:1.5;"/>
+ <defs>
+ <path d="M -3 3
+L 3 -3
+M -3 -3
+L 3 3
+" id="mee8b134b61" style="stroke:#2ca02c;stroke-opacity:0.2;"/>
+ </defs>
+ <g clip-path="url(#p444e95376c)">
+ <use style="fill:#2ca02c;fill-opacity:0.2;stroke:#2ca02c;stroke-opacity:0.2;" x="369.666423" xlink:href="#mee8b134b61" y="288.740154"/>
+ <use style="fill:#2ca02c;fill-opacity:0.2;stroke:#2ca02c;stroke-opacity:0.2;" x="370.695667" xlink:href="#mee8b134b61" y="288.222149"/>
+ <use style="fill:#2ca02c;fill-opacity:0.2;stroke:#2ca02c;stroke-opacity:0.2;" x="371.719963" xlink:href="#mee8b134b61" y="287.702345"/>
+ <use style="fill:#2ca02c;fill-opacity:0.2;stroke:#2ca02c;stroke-opacity:0.2;" x="372.739209" xlink:href="#mee8b134b61" y="287.180688"/>
+ <use style="fill:#2ca02c;fill-opacity:0.2;stroke:#2ca02c;stroke-opacity:0.2;" x="373.753304" xlink:href="#mee8b134b61" y="286.657127"/>
+ <use style="fill:#2ca02c;fill-opacity:0.2;stroke:#2ca02c;stroke-opacity:0.2;" x="374.762142" xlink:href="#mee8b134b61" y="286.131608"/>
+ <use style="fill:#2ca02c;fill-opacity:0.2;stroke:#2ca02c;stroke-opacity:0.2;" x="375.765622" xlink:href="#mee8b134b61" y="285.60408"/>
+ <use style="fill:#2ca02c;fill-opacity:0.2;stroke:#2ca02c;stroke-opacity:0.2;" x="376.76364" xlink:href="#mee8b134b61" y="285.074491"/>
+ <use style="fill:#2ca02c;fill-opacity:0.2;stroke:#2ca02c;stroke-opacity:0.2;" x="377.756093" xlink:href="#mee8b134b61" y="284.542788"/>
+ <use style="fill:#2ca02c;fill-opacity:0.2;stroke:#2ca02c;stroke-opacity:0.2;" x="378.742876" xlink:href="#mee8b134b61" y="284.00892"/>
+ <use style="fill:#2ca02c;fill-opacity:0.2;stroke:#2ca02c;stroke-opacity:0.2;" x="379.723887" xlink:href="#mee8b134b61" y="283.472836"/>
+ <use style="fill:#2ca02c;fill-opacity:0.2;stroke:#2ca02c;stroke-opacity:0.2;" x="380.699022" xlink:href="#mee8b134b61" y="282.934482"/>
+ <use style="fill:#2ca02c;fill-opacity:0.2;stroke:#2ca02c;stroke-opacity:0.2;" x="381.668178" xlink:href="#mee8b134b61" y="282.393809"/>
+ <use style="fill:#2ca02c;fill-opacity:0.2;stroke:#2ca02c;stroke-opacity:0.2;" x="382.631251" xlink:href="#mee8b134b61" y="281.850764"/>
+ <use style="fill:#2ca02c;fill-opacity:0.2;stroke:#2ca02c;stroke-opacity:0.2;" x="383.58814" xlink:href="#mee8b134b61" y="281.305297"/>
+ <use style="fill:#2ca02c;fill-opacity:0.2;stroke:#2ca02c;stroke-opacity:0.2;" x="384.538742" xlink:href="#mee8b134b61" y="280.757355"/>
+ <use style="fill:#2ca02c;fill-opacity:0.2;stroke:#2ca02c;stroke-opacity:0.2;" x="385.482954" xlink:href="#mee8b134b61" y="280.206889"/>
+ <use style="fill:#2ca02c;fill-opacity:0.2;stroke:#2ca02c;stroke-opacity:0.2;" x="386.420675" xlink:href="#mee8b134b61" y="279.653847"/>
+ <use style="fill:#2ca02c;fill-opacity:0.2;stroke:#2ca02c;stroke-opacity:0.2;" x="387.351802" xlink:href="#mee8b134b61" y="279.098178"/>
+ <use style="fill:#2ca02c;fill-opacity:0.2;stroke:#2ca02c;stroke-opacity:0.2;" x="388.276234" xlink:href="#mee8b134b61" y="278.539833"/>
+ <use style="fill:#2ca02c;fill-opacity:0.2;stroke:#2ca02c;stroke-opacity:0.2;" x="389.19387" xlink:href="#mee8b134b61" y="277.978761"/>
+ <use style="fill:#2ca02c;fill-opacity:0.2;stroke:#2ca02c;stroke-opacity:0.2;" x="390.104608" xlink:href="#mee8b134b61" y="277.414912"/>
+ <use style="fill:#2ca02c;fill-opacity:0.2;stroke:#2ca02c;stroke-opacity:0.2;" x="391.008348" xlink:href="#mee8b134b61" y="276.848236"/>
+ <use style="fill:#2ca02c;fill-opacity:0.2;stroke:#2ca02c;stroke-opacity:0.2;" x="391.90499" xlink:href="#mee8b134b61" y="276.278683"/>
+ <use style="fill:#2ca02c;fill-opacity:0.2;stroke:#2ca02c;stroke-opacity:0.2;" x="392.794431" xlink:href="#mee8b134b61" y="275.706206"/>
+ <use style="fill:#2ca02c;fill-opacity:0.2;stroke:#2ca02c;stroke-opacity:0.2;" x="393.676572" xlink:href="#mee8b134b61" y="275.130754"/>
+ <use style="fill:#2ca02c;fill-opacity:0.2;stroke:#2ca02c;stroke-opacity:0.2;" x="394.551311" xlink:href="#mee8b134b61" y="274.55228"/>
+ <use style="fill:#2ca02c;fill-opacity:0.2;stroke:#2ca02c;stroke-opacity:0.2;" x="395.418548" xlink:href="#mee8b134b61" y="273.970736"/>
+ <use style="fill:#2ca02c;fill-opacity:0.2;stroke:#2ca02c;stroke-opacity:0.2;" x="396.278181" xlink:href="#mee8b134b61" y="273.386073"/>
+ <use style="fill:#2ca02c;fill-opacity:0.2;stroke:#2ca02c;stroke-opacity:0.2;" x="397.130109" xlink:href="#mee8b134b61" y="272.798246"/>
+ <use style="fill:#2ca02c;fill-opacity:0.2;stroke:#2ca02c;stroke-opacity:0.2;" x="397.974231" xlink:href="#mee8b134b61" y="272.207207"/>
+ <use style="fill:#2ca02c;fill-opacity:0.2;stroke:#2ca02c;stroke-opacity:0.2;" x="398.810443" xlink:href="#mee8b134b61" y="271.612912"/>
+ <use style="fill:#2ca02c;fill-opacity:0.2;stroke:#2ca02c;stroke-opacity:0.2;" x="399.638644" xlink:href="#mee8b134b61" y="271.015314"/>
+ <use style="fill:#2ca02c;fill-opacity:0.2;stroke:#2ca02c;stroke-opacity:0.2;" x="400.45873" xlink:href="#mee8b134b61" y="270.41437"/>
+ <use style="fill:#2ca02c;fill-opacity:0.2;stroke:#2ca02c;stroke-opacity:0.2;" x="401.270597" xlink:href="#mee8b134b61" y="269.810035"/>
+ <use style="fill:#2ca02c;fill-opacity:0.2;stroke:#2ca02c;stroke-opacity:0.2;" x="402.074139" xlink:href="#mee8b134b61" y="269.202268"/>
+ <use style="fill:#2ca02c;fill-opacity:0.2;stroke:#2ca02c;stroke-opacity:0.2;" x="402.869252" xlink:href="#mee8b134b61" y="268.591026"/>
+ <use style="fill:#2ca02c;fill-opacity:0.2;stroke:#2ca02c;stroke-opacity:0.2;" x="403.655827" xlink:href="#mee8b134b61" y="267.976269"/>
+ <use style="fill:#2ca02c;fill-opacity:0.2;stroke:#2ca02c;stroke-opacity:0.2;" x="404.433759" xlink:href="#mee8b134b61" y="267.357956"/>
+ <use style="fill:#2ca02c;fill-opacity:0.2;stroke:#2ca02c;stroke-opacity:0.2;" x="405.202936" xlink:href="#mee8b134b61" y="266.73605"/>
+ <use style="fill:#2ca02c;fill-opacity:0.2;stroke:#2ca02c;stroke-opacity:0.2;" x="405.96325" xlink:href="#mee8b134b61" y="266.110513"/>
+ <use style="fill:#2ca02c;fill-opacity:0.2;stroke:#2ca02c;stroke-opacity:0.2;" x="406.714589" xlink:href="#mee8b134b61" y="265.481308"/>
+ <use style="fill:#2ca02c;fill-opacity:0.2;stroke:#2ca02c;stroke-opacity:0.2;" x="407.456839" xlink:href="#mee8b134b61" y="264.848402"/>
+ <use style="fill:#2ca02c;fill-opacity:0.2;stroke:#2ca02c;stroke-opacity:0.2;" x="408.189886" xlink:href="#mee8b134b61" y="264.21176"/>
+ <use style="fill:#2ca02c;fill-opacity:0.2;stroke:#2ca02c;stroke-opacity:0.2;" x="408.913615" xlink:href="#mee8b134b61" y="263.571352"/>
+ <use style="fill:#2ca02c;fill-opacity:0.2;stroke:#2ca02c;stroke-opacity:0.2;" x="409.627908" xlink:href="#mee8b134b61" y="262.927146"/>
+ <use style="fill:#2ca02c;fill-opacity:0.2;stroke:#2ca02c;stroke-opacity:0.2;" x="410.332646" xlink:href="#mee8b134b61" y="262.279116"/>
+ <use style="fill:#2ca02c;fill-opacity:0.2;stroke:#2ca02c;stroke-opacity:0.2;" x="411.027709" xlink:href="#mee8b134b61" y="261.627234"/>
+ <use style="fill:#2ca02c;fill-opacity:0.2;stroke:#2ca02c;stroke-opacity:0.2;" x="411.712974" xlink:href="#mee8b134b61" y="260.971477"/>
+ <use style="fill:#2ca02c;fill-opacity:0.2;stroke:#2ca02c;stroke-opacity:0.2;" x="412.388319" xlink:href="#mee8b134b61" y="260.31182"/>
+ <use style="fill:#2ca02c;fill-opacity:0.2;stroke:#2ca02c;stroke-opacity:0.2;" x="413.053618" xlink:href="#mee8b134b61" y="259.648245"/>
+ <use style="fill:#2ca02c;fill-opacity:0.2;stroke:#2ca02c;stroke-opacity:0.2;" x="413.708744" xlink:href="#mee8b134b61" y="258.980732"/>
+ <use style="fill:#2ca02c;fill-opacity:0.2;stroke:#2ca02c;stroke-opacity:0.2;" x="414.35357" xlink:href="#mee8b134b61" y="258.309266"/>
+ <use style="fill:#2ca02c;fill-opacity:0.2;stroke:#2ca02c;stroke-opacity:0.2;" x="414.987967" xlink:href="#mee8b134b61" y="257.633834"/>
+ <use style="fill:#2ca02c;fill-opacity:0.2;stroke:#2ca02c;stroke-opacity:0.2;" x="415.611803" xlink:href="#mee8b134b61" y="256.954423"/>
+ <use style="fill:#2ca02c;fill-opacity:0.2;stroke:#2ca02c;stroke-opacity:0.2;" x="416.224947" xlink:href="#mee8b134b61" y="256.271025"/>
+ <use style="fill:#2ca02c;fill-opacity:0.2;stroke:#2ca02c;stroke-opacity:0.2;" x="416.827268" xlink:href="#mee8b134b61" y="255.583635"/>
+ <use style="fill:#2ca02c;fill-opacity:0.2;stroke:#2ca02c;stroke-opacity:0.2;" x="417.418631" xlink:href="#mee8b134b61" y="254.892249"/>
+ <use style="fill:#2ca02c;fill-opacity:0.2;stroke:#2ca02c;stroke-opacity:0.2;" x="417.998903" xlink:href="#mee8b134b61" y="254.196866"/>
+ <use style="fill:#2ca02c;fill-opacity:0.2;stroke:#2ca02c;stroke-opacity:0.2;" x="418.567949" xlink:href="#mee8b134b61" y="253.497488"/>
+ <use style="fill:#2ca02c;fill-opacity:0.2;stroke:#2ca02c;stroke-opacity:0.2;" x="419.125635" xlink:href="#mee8b134b61" y="252.794121"/>
+ <use style="fill:#2ca02c;fill-opacity:0.2;stroke:#2ca02c;stroke-opacity:0.2;" x="419.671827" xlink:href="#mee8b134b61" y="252.086772"/>
+ <use style="fill:#2ca02c;fill-opacity:0.2;stroke:#2ca02c;stroke-opacity:0.2;" x="420.206391" xlink:href="#mee8b134b61" y="251.375453"/>
+ <use style="fill:#2ca02c;fill-opacity:0.2;stroke:#2ca02c;stroke-opacity:0.2;" x="420.729194" xlink:href="#mee8b134b61" y="250.660177"/>
+ <use style="fill:#2ca02c;fill-opacity:0.2;stroke:#2ca02c;stroke-opacity:0.2;" x="421.240104" xlink:href="#mee8b134b61" y="249.940961"/>
+ <use style="fill:#2ca02c;fill-opacity:0.2;stroke:#2ca02c;stroke-opacity:0.2;" x="421.73899" xlink:href="#mee8b134b61" y="249.217825"/>
+ <use style="fill:#2ca02c;fill-opacity:0.2;stroke:#2ca02c;stroke-opacity:0.2;" x="422.225722" xlink:href="#mee8b134b61" y="248.490794"/>
+ <use style="fill:#2ca02c;fill-opacity:0.2;stroke:#2ca02c;stroke-opacity:0.2;" x="422.700176" xlink:href="#mee8b134b61" y="247.759892"/>
+ <use style="fill:#2ca02c;fill-opacity:0.2;stroke:#2ca02c;stroke-opacity:0.2;" x="423.162225" xlink:href="#mee8b134b61" y="247.025149"/>
+ <use style="fill:#2ca02c;fill-opacity:0.2;stroke:#2ca02c;stroke-opacity:0.2;" x="423.61175" xlink:href="#mee8b134b61" y="246.286598"/>
+ <use style="fill:#2ca02c;fill-opacity:0.2;stroke:#2ca02c;stroke-opacity:0.2;" x="424.048631" xlink:href="#mee8b134b61" y="245.544274"/>
+ <use style="fill:#2ca02c;fill-opacity:0.2;stroke:#2ca02c;stroke-opacity:0.2;" x="424.472757" xlink:href="#mee8b134b61" y="244.798216"/>
+ <use style="fill:#2ca02c;fill-opacity:0.2;stroke:#2ca02c;stroke-opacity:0.2;" x="424.884017" xlink:href="#mee8b134b61" y="244.048465"/>
+ <use style="fill:#2ca02c;fill-opacity:0.2;stroke:#2ca02c;stroke-opacity:0.2;" x="425.282308" xlink:href="#mee8b134b61" y="243.295067"/>
+ <use style="fill:#2ca02c;fill-opacity:0.2;stroke:#2ca02c;stroke-opacity:0.2;" x="425.66753" xlink:href="#mee8b134b61" y="242.538067"/>
+ <use style="fill:#2ca02c;fill-opacity:0.2;stroke:#2ca02c;stroke-opacity:0.2;" x="426.039591" xlink:href="#mee8b134b61" y="241.777517"/>
+ <use style="fill:#2ca02c;fill-opacity:0.2;stroke:#2ca02c;stroke-opacity:0.2;" x="426.398404" xlink:href="#mee8b134b61" y="241.01347"/>
+ <use style="fill:#2ca02c;fill-opacity:0.2;stroke:#2ca02c;stroke-opacity:0.2;" x="426.743891" xlink:href="#mee8b134b61" y="240.245982"/>
+ <use style="fill:#2ca02c;fill-opacity:0.2;stroke:#2ca02c;stroke-opacity:0.2;" x="427.07598" xlink:href="#mee8b134b61" y="239.47511"/>
+ <use style="fill:#2ca02c;fill-opacity:0.2;stroke:#2ca02c;stroke-opacity:0.2;" x="427.394607" xlink:href="#mee8b134b61" y="238.700915"/>
+ <use style="fill:#2ca02c;fill-opacity:0.2;stroke:#2ca02c;stroke-opacity:0.2;" x="427.699718" xlink:href="#mee8b134b61" y="237.923461"/>
+ <use style="fill:#2ca02c;fill-opacity:0.2;stroke:#2ca02c;stroke-opacity:0.2;" x="427.991267" xlink:href="#mee8b134b61" y="237.142813"/>
+ <use style="fill:#2ca02c;fill-opacity:0.2;stroke:#2ca02c;stroke-opacity:0.2;" x="428.269219" xlink:href="#mee8b134b61" y="236.359038"/>
+ <use style="fill:#2ca02c;fill-opacity:0.2;stroke:#2ca02c;stroke-opacity:0.2;" x="428.533507" xlink:href="#mee8b134b61" y="235.572179"/>
+ <use style="fill:#2ca02c;fill-opacity:0.2;stroke:#2ca02c;stroke-opacity:0.2;" x="428.784071" xlink:href="#mee8b134b61" y="234.782278"/>
+ <use style="fill:#2ca02c;fill-opacity:0.2;stroke:#2ca02c;stroke-opacity:0.2;" x="429.020884" xlink:href="#mee8b134b61" y="233.989387"/>
+ <use style="fill:#2ca02c;fill-opacity:0.2;stroke:#2ca02c;stroke-opacity:0.2;" x="429.243902" xlink:href="#mee8b134b61" y="233.193547"/>
+ <use style="fill:#2ca02c;fill-opacity:0.2;stroke:#2ca02c;stroke-opacity:0.2;" x="429.4531" xlink:href="#mee8b134b61" y="232.394798"/>
+ <use style="fill:#2ca02c;fill-opacity:0.2;stroke:#2ca02c;stroke-opacity:0.2;" x="429.648461" xlink:href="#mee8b134b61" y="231.593182"/>
+ <use style="fill:#2ca02c;fill-opacity:0.2;stroke:#2ca02c;stroke-opacity:0.2;" x="429.829978" xlink:href="#mee8b134b61" y="230.78874"/>
+ <use style="fill:#2ca02c;fill-opacity:0.2;stroke:#2ca02c;stroke-opacity:0.2;" x="429.997653" xlink:href="#mee8b134b61" y="229.98151"/>
+ <use style="fill:#2ca02c;fill-opacity:0.2;stroke:#2ca02c;stroke-opacity:0.2;" x="430.151498" xlink:href="#mee8b134b61" y="229.171533"/>
+ <use style="fill:#2ca02c;fill-opacity:0.2;stroke:#2ca02c;stroke-opacity:0.2;" x="430.291533" xlink:href="#mee8b134b61" y="228.358845"/>
+ <use style="fill:#2ca02c;fill-opacity:0.2;stroke:#2ca02c;stroke-opacity:0.2;" x="430.417787" xlink:href="#mee8b134b61" y="227.543482"/>
+ <use style="fill:#2ca02c;fill-opacity:0.2;stroke:#2ca02c;stroke-opacity:0.2;" x="430.530299" xlink:href="#mee8b134b61" y="226.725481"/>
+ <use style="fill:#2ca02c;fill-opacity:0.2;stroke:#2ca02c;stroke-opacity:0.2;" x="430.629114" xlink:href="#mee8b134b61" y="225.904877"/>
+ <use style="fill:#2ca02c;fill-opacity:0.2;stroke:#2ca02c;stroke-opacity:0.2;" x="430.714286" xlink:href="#mee8b134b61" y="225.081702"/>
+ <use style="fill:#2ca02c;fill-opacity:0.2;stroke:#2ca02c;stroke-opacity:0.2;" x="430.785878" xlink:href="#mee8b134b61" y="224.255989"/>
+ <use style="fill:#2ca02c;fill-opacity:0.2;stroke:#2ca02c;stroke-opacity:0.2;" x="430.843959" xlink:href="#mee8b134b61" y="223.427771"/>
+ <use style="fill:#2ca02c;fill-opacity:0.2;stroke:#2ca02c;stroke-opacity:0.2;" x="430.888607" xlink:href="#mee8b134b61" y="222.597077"/>
+ <use style="fill:#2ca02c;fill-opacity:0.2;stroke:#2ca02c;stroke-opacity:0.2;" x="430.919905" xlink:href="#mee8b134b61" y="221.763937"/>
+ <use style="fill:#2ca02c;fill-opacity:0.2;stroke:#2ca02c;stroke-opacity:0.2;" x="430.937865" xlink:href="#mee8b134b61" y="220.928379"/>
+ <use style="fill:#2ca02c;fill-opacity:0.2;stroke:#2ca02c;stroke-opacity:0.2;" x="430.94255" xlink:href="#mee8b134b61" y="220.090431"/>
+ <use style="fill:#2ca02c;fill-opacity:0.2;stroke:#2ca02c;stroke-opacity:0.2;" x="430.934019" xlink:href="#mee8b134b61" y="219.250118"/>
+ <use style="fill:#2ca02c;fill-opacity:0.2;stroke:#2ca02c;stroke-opacity:0.2;" x="430.91232" xlink:href="#mee8b134b61" y="218.407466"/>
+ <use style="fill:#2ca02c;fill-opacity:0.2;stroke:#2ca02c;stroke-opacity:0.2;" x="430.877515" xlink:href="#mee8b134b61" y="217.562502"/>
+ <use style="fill:#2ca02c;fill-opacity:0.2;stroke:#2ca02c;stroke-opacity:0.2;" x="430.829672" xlink:href="#mee8b134b61" y="216.715249"/>
+ <use style="fill:#2ca02c;fill-opacity:0.2;stroke:#2ca02c;stroke-opacity:0.2;" x="430.768859" xlink:href="#mee8b134b61" y="215.865731"/>
+ <use style="fill:#2ca02c;fill-opacity:0.2;stroke:#2ca02c;stroke-opacity:0.2;" x="430.695153" xlink:href="#mee8b134b61" y="215.013974"/>
+ <use style="fill:#2ca02c;fill-opacity:0.2;stroke:#2ca02c;stroke-opacity:0.2;" x="430.608632" xlink:href="#mee8b134b61" y="214.16"/>
+ <use style="fill:#2ca02c;fill-opacity:0.2;stroke:#2ca02c;stroke-opacity:0.2;" x="430.509377" xlink:href="#mee8b134b61" y="213.303834"/>
+ <use style="fill:#2ca02c;fill-opacity:0.2;stroke:#2ca02c;stroke-opacity:0.2;" x="430.397473" xlink:href="#mee8b134b61" y="212.445499"/>
+ <use style="fill:#2ca02c;fill-opacity:0.2;stroke:#2ca02c;stroke-opacity:0.2;" x="430.273009" xlink:href="#mee8b134b61" y="211.585019"/>
+ <use style="fill:#2ca02c;fill-opacity:0.2;stroke:#2ca02c;stroke-opacity:0.2;" x="430.136073" xlink:href="#mee8b134b61" y="210.722418"/>
+ <use style="fill:#2ca02c;fill-opacity:0.2;stroke:#2ca02c;stroke-opacity:0.2;" x="429.986757" xlink:href="#mee8b134b61" y="209.857721"/>
+ <use style="fill:#2ca02c;fill-opacity:0.2;stroke:#2ca02c;stroke-opacity:0.2;" x="429.825155" xlink:href="#mee8b134b61" y="208.990952"/>
+ <use style="fill:#2ca02c;fill-opacity:0.2;stroke:#2ca02c;stroke-opacity:0.2;" x="429.65136" xlink:href="#mee8b134b61" y="208.122137"/>
+ <use style="fill:#2ca02c;fill-opacity:0.2;stroke:#2ca02c;stroke-opacity:0.2;" x="429.465469" xlink:href="#mee8b134b61" y="207.251301"/>
+ <use style="fill:#2ca02c;fill-opacity:0.2;stroke:#2ca02c;stroke-opacity:0.2;" x="429.267577" xlink:href="#mee8b134b61" y="206.37847"/>
+ <use style="fill:#2ca02c;fill-opacity:0.2;stroke:#2ca02c;stroke-opacity:0.2;" x="429.057781" xlink:href="#mee8b134b61" y="205.503674"/>
+ <use style="fill:#2ca02c;fill-opacity:0.2;stroke:#2ca02c;stroke-opacity:0.2;" x="428.836176" xlink:href="#mee8b134b61" y="204.626938"/>
+ <use style="fill:#2ca02c;fill-opacity:0.2;stroke:#2ca02c;stroke-opacity:0.2;" x="428.602859" xlink:href="#mee8b134b61" y="203.748294"/>
+ <use style="fill:#2ca02c;fill-opacity:0.2;stroke:#2ca02c;stroke-opacity:0.2;" x="428.357924" xlink:href="#mee8b134b61" y="202.86777"/>
+ <use style="fill:#2ca02c;fill-opacity:0.2;stroke:#2ca02c;stroke-opacity:0.2;" x="428.101467" xlink:href="#mee8b134b61" y="201.985399"/>
+ <use style="fill:#2ca02c;fill-opacity:0.2;stroke:#2ca02c;stroke-opacity:0.2;" x="427.833579" xlink:href="#mee8b134b61" y="201.101213"/>
+ <use style="fill:#2ca02c;fill-opacity:0.2;stroke:#2ca02c;stroke-opacity:0.2;" x="427.554352" xlink:href="#mee8b134b61" y="200.215246"/>
+ <use style="fill:#2ca02c;fill-opacity:0.2;stroke:#2ca02c;stroke-opacity:0.2;" x="427.263878" xlink:href="#mee8b134b61" y="199.327533"/>
+ <use style="fill:#2ca02c;fill-opacity:0.2;stroke:#2ca02c;stroke-opacity:0.2;" x="426.962243" xlink:href="#mee8b134b61" y="198.43811"/>
+ <use style="fill:#2ca02c;fill-opacity:0.2;stroke:#2ca02c;stroke-opacity:0.2;" x="426.649534" xlink:href="#mee8b134b61" y="197.547016"/>
+ <use style="fill:#2ca02c;fill-opacity:0.2;stroke:#2ca02c;stroke-opacity:0.2;" x="426.325834" xlink:href="#mee8b134b61" y="196.654291"/>
+ <use style="fill:#2ca02c;fill-opacity:0.2;stroke:#2ca02c;stroke-opacity:0.2;" x="425.991227" xlink:href="#mee8b134b61" y="195.759974"/>
+ <use style="fill:#2ca02c;fill-opacity:0.2;stroke:#2ca02c;stroke-opacity:0.2;" x="425.645789" xlink:href="#mee8b134b61" y="194.864109"/>
+ <use style="fill:#2ca02c;fill-opacity:0.2;stroke:#2ca02c;stroke-opacity:0.2;" x="425.289599" xlink:href="#mee8b134b61" y="193.96674"/>
+ <use style="fill:#2ca02c;fill-opacity:0.2;stroke:#2ca02c;stroke-opacity:0.2;" x="424.922729" xlink:href="#mee8b134b61" y="193.067913"/>
+ <use style="fill:#2ca02c;fill-opacity:0.2;stroke:#2ca02c;stroke-opacity:0.2;" x="424.54525" xlink:href="#mee8b134b61" y="192.167674"/>
+ <use style="fill:#2ca02c;fill-opacity:0.2;stroke:#2ca02c;stroke-opacity:0.2;" x="424.157231" xlink:href="#mee8b134b61" y="191.266073"/>
+ <use style="fill:#2ca02c;fill-opacity:0.2;stroke:#2ca02c;stroke-opacity:0.2;" x="423.758735" xlink:href="#mee8b134b61" y="190.363159"/>
+ <use style="fill:#2ca02c;fill-opacity:0.2;stroke:#2ca02c;stroke-opacity:0.2;" x="423.349826" xlink:href="#mee8b134b61" y="189.458986"/>
+ <use style="fill:#2ca02c;fill-opacity:0.2;stroke:#2ca02c;stroke-opacity:0.2;" x="422.930562" xlink:href="#mee8b134b61" y="188.553605"/>
+ <use style="fill:#2ca02c;fill-opacity:0.2;stroke:#2ca02c;stroke-opacity:0.2;" x="422.500999" xlink:href="#mee8b134b61" y="187.647072"/>
+ <use style="fill:#2ca02c;fill-opacity:0.2;stroke:#2ca02c;stroke-opacity:0.2;" x="422.061191" xlink:href="#mee8b134b61" y="186.739444"/>
+ <use style="fill:#2ca02c;fill-opacity:0.2;stroke:#2ca02c;stroke-opacity:0.2;" x="421.611188" xlink:href="#mee8b134b61" y="185.830777"/>
+ <use style="fill:#2ca02c;fill-opacity:0.2;stroke:#2ca02c;stroke-opacity:0.2;" x="421.151037" xlink:href="#mee8b134b61" y="184.921132"/>
+ <use style="fill:#2ca02c;fill-opacity:0.2;stroke:#2ca02c;stroke-opacity:0.2;" x="420.680784" xlink:href="#mee8b134b61" y="184.010567"/>
+ <use style="fill:#2ca02c;fill-opacity:0.2;stroke:#2ca02c;stroke-opacity:0.2;" x="420.200472" xlink:href="#mee8b134b61" y="183.099146"/>
+ <use style="fill:#2ca02c;fill-opacity:0.2;stroke:#2ca02c;stroke-opacity:0.2;" x="419.710139" xlink:href="#mee8b134b61" y="182.186929"/>
+ <use style="fill:#2ca02c;fill-opacity:0.2;stroke:#2ca02c;stroke-opacity:0.2;" x="419.209825" xlink:href="#mee8b134b61" y="181.273982"/>
+ <use style="fill:#2ca02c;fill-opacity:0.2;stroke:#2ca02c;stroke-opacity:0.2;" x="418.699597" xlink:href="#mee8b134b61" y="180.360377"/>
+ <use style="fill:#2ca02c;fill-opacity:0.2;stroke:#2ca02c;stroke-opacity:0.2;" x="418.179587" xlink:href="#mee8b134b61" y="179.446207"/>
+ <use style="fill:#2ca02c;fill-opacity:0.2;stroke:#2ca02c;stroke-opacity:0.2;" x="417.64984" xlink:href="#mee8b134b61" y="178.531539"/>
+ <use style="fill:#2ca02c;fill-opacity:0.2;stroke:#2ca02c;stroke-opacity:0.2;" x="417.11045" xlink:href="#mee8b134b61" y="177.616457"/>
+ <use style="fill:#2ca02c;fill-opacity:0.2;stroke:#2ca02c;stroke-opacity:0.2;" x="416.561498" xlink:href="#mee8b134b61" y="176.701039"/>
+ <use style="fill:#2ca02c;fill-opacity:0.2;stroke:#2ca02c;stroke-opacity:0.2;" x="416.003058" xlink:href="#mee8b134b61" y="175.785363"/>
+ <use style="fill:#2ca02c;fill-opacity:0.2;stroke:#2ca02c;stroke-opacity:0.2;" x="415.435198" xlink:href="#mee8b134b61" y="174.869507"/>
+ <use style="fill:#2ca02c;fill-opacity:0.2;stroke:#2ca02c;stroke-opacity:0.2;" x="414.857985" xlink:href="#mee8b134b61" y="173.953547"/>
+ <use style="fill:#2ca02c;fill-opacity:0.2;stroke:#2ca02c;stroke-opacity:0.2;" x="414.271481" xlink:href="#mee8b134b61" y="173.037559"/>
+ <use style="fill:#2ca02c;fill-opacity:0.2;stroke:#2ca02c;stroke-opacity:0.2;" x="413.675747" xlink:href="#mee8b134b61" y="172.121617"/>
+ <use style="fill:#2ca02c;fill-opacity:0.2;stroke:#2ca02c;stroke-opacity:0.2;" x="413.070839" xlink:href="#mee8b134b61" y="171.205797"/>
+ <use style="fill:#2ca02c;fill-opacity:0.2;stroke:#2ca02c;stroke-opacity:0.2;" x="412.456812" xlink:href="#mee8b134b61" y="170.290173"/>
+ <use style="fill:#2ca02c;fill-opacity:0.2;stroke:#2ca02c;stroke-opacity:0.2;" x="411.833718" xlink:href="#mee8b134b61" y="169.374818"/>
+ <use style="fill:#2ca02c;fill-opacity:0.2;stroke:#2ca02c;stroke-opacity:0.2;" x="411.201608" xlink:href="#mee8b134b61" y="168.459805"/>
+ <use style="fill:#2ca02c;fill-opacity:0.2;stroke:#2ca02c;stroke-opacity:0.2;" x="410.560533" xlink:href="#mee8b134b61" y="167.545206"/>
+ <use style="fill:#2ca02c;fill-opacity:0.2;stroke:#2ca02c;stroke-opacity:0.2;" x="409.91054" xlink:href="#mee8b134b61" y="166.631093"/>
+ <use style="fill:#2ca02c;fill-opacity:0.2;stroke:#2ca02c;stroke-opacity:0.2;" x="409.251677" xlink:href="#mee8b134b61" y="165.717539"/>
+ <use style="fill:#2ca02c;fill-opacity:0.2;stroke:#2ca02c;stroke-opacity:0.2;" x="408.583994" xlink:href="#mee8b134b61" y="164.804613"/>
+ <use style="fill:#2ca02c;fill-opacity:0.2;stroke:#2ca02c;stroke-opacity:0.2;" x="407.907536" xlink:href="#mee8b134b61" y="163.892386"/>
+ <use style="fill:#2ca02c;fill-opacity:0.2;stroke:#2ca02c;stroke-opacity:0.2;" x="407.222352" xlink:href="#mee8b134b61" y="162.980929"/>
+ <use style="fill:#2ca02c;fill-opacity:0.2;stroke:#2ca02c;stroke-opacity:0.2;" x="406.528492" xlink:href="#mee8b134b61" y="162.07031"/>
+ <use style="fill:#2ca02c;fill-opacity:0.2;stroke:#2ca02c;stroke-opacity:0.2;" x="405.826003" xlink:href="#mee8b134b61" y="161.160598"/>
+ <use style="fill:#2ca02c;fill-opacity:0.2;stroke:#2ca02c;stroke-opacity:0.2;" x="405.114938" xlink:href="#mee8b134b61" y="160.251862"/>
+ <use style="fill:#2ca02c;fill-opacity:0.2;stroke:#2ca02c;stroke-opacity:0.2;" x="404.395348" xlink:href="#mee8b134b61" y="159.344169"/>
+ <use style="fill:#2ca02c;fill-opacity:0.2;stroke:#2ca02c;stroke-opacity:0.2;" x="403.667286" xlink:href="#mee8b134b61" y="158.437586"/>
+ <use style="fill:#2ca02c;fill-opacity:0.2;stroke:#2ca02c;stroke-opacity:0.2;" x="402.930809" xlink:href="#mee8b134b61" y="157.53218"/>
+ <use style="fill:#2ca02c;fill-opacity:0.2;stroke:#2ca02c;stroke-opacity:0.2;" x="402.185974" xlink:href="#mee8b134b61" y="156.628016"/>
+ <use style="fill:#2ca02c;fill-opacity:0.2;stroke:#2ca02c;stroke-opacity:0.2;" x="401.432843" xlink:href="#mee8b134b61" y="155.725159"/>
+ <use style="fill:#2ca02c;fill-opacity:0.2;stroke:#2ca02c;stroke-opacity:0.2;" x="400.671476" xlink:href="#mee8b134b61" y="154.823674"/>
+ <use style="fill:#2ca02c;fill-opacity:0.2;stroke:#2ca02c;stroke-opacity:0.2;" x="399.90194" xlink:href="#mee8b134b61" y="153.923624"/>
+ <use style="fill:#2ca02c;fill-opacity:0.2;stroke:#2ca02c;stroke-opacity:0.2;" x="399.124304" xlink:href="#mee8b134b61" y="153.025071"/>
+ <use style="fill:#2ca02c;fill-opacity:0.2;stroke:#2ca02c;stroke-opacity:0.2;" x="398.338638" xlink:href="#mee8b134b61" y="152.128076"/>
+ <use style="fill:#2ca02c;fill-opacity:0.2;stroke:#2ca02c;stroke-opacity:0.2;" x="397.545017" xlink:href="#mee8b134b61" y="151.232702"/>
+ <use style="fill:#2ca02c;fill-opacity:0.2;stroke:#2ca02c;stroke-opacity:0.2;" x="396.74352" xlink:href="#mee8b134b61" y="150.339006"/>
+ <use style="fill:#2ca02c;fill-opacity:0.2;stroke:#2ca02c;stroke-opacity:0.2;" x="395.934226" xlink:href="#mee8b134b61" y="149.447047"/>
+ <use style="fill:#2ca02c;fill-opacity:0.2;stroke:#2ca02c;stroke-opacity:0.2;" x="395.117221" xlink:href="#mee8b134b61" y="148.556884"/>
+ <use style="fill:#2ca02c;fill-opacity:0.2;stroke:#2ca02c;stroke-opacity:0.2;" x="394.292594" xlink:href="#mee8b134b61" y="147.668572"/>
+ <use style="fill:#2ca02c;fill-opacity:0.2;stroke:#2ca02c;stroke-opacity:0.2;" x="393.460434" xlink:href="#mee8b134b61" y="146.782166"/>
+ <use style="fill:#2ca02c;fill-opacity:0.2;stroke:#2ca02c;stroke-opacity:0.2;" x="392.620838" xlink:href="#mee8b134b61" y="145.89772"/>
+ <use style="fill:#2ca02c;fill-opacity:0.2;stroke:#2ca02c;stroke-opacity:0.2;" x="391.773904" xlink:href="#mee8b134b61" y="145.015287"/>
+ <use style="fill:#2ca02c;fill-opacity:0.2;stroke:#2ca02c;stroke-opacity:0.2;" x="390.919734" xlink:href="#mee8b134b61" y="144.134917"/>
+ <use style="fill:#2ca02c;fill-opacity:0.2;stroke:#2ca02c;stroke-opacity:0.2;" x="390.058434" xlink:href="#mee8b134b61" y="143.256661"/>
+ <use style="fill:#2ca02c;fill-opacity:0.2;stroke:#2ca02c;stroke-opacity:0.2;" x="389.190112" xlink:href="#mee8b134b61" y="142.380567"/>
+ <use style="fill:#2ca02c;fill-opacity:0.2;stroke:#2ca02c;stroke-opacity:0.2;" x="388.314882" xlink:href="#mee8b134b61" y="141.506682"/>
+ <use style="fill:#2ca02c;fill-opacity:0.2;stroke:#2ca02c;stroke-opacity:0.2;" x="387.432858" xlink:href="#mee8b134b61" y="140.635051"/>
+ <use style="fill:#2ca02c;fill-opacity:0.2;stroke:#2ca02c;stroke-opacity:0.2;" x="386.54416" xlink:href="#mee8b134b61" y="139.765719"/>
+ <use style="fill:#2ca02c;fill-opacity:0.2;stroke:#2ca02c;stroke-opacity:0.2;" x="385.648909" xlink:href="#mee8b134b61" y="138.898727"/>
+ <use style="fill:#2ca02c;fill-opacity:0.2;stroke:#2ca02c;stroke-opacity:0.2;" x="384.747229" xlink:href="#mee8b134b61" y="138.034116"/>
+ <use style="fill:#2ca02c;fill-opacity:0.2;stroke:#2ca02c;stroke-opacity:0.2;" x="383.839249" xlink:href="#mee8b134b61" y="137.171927"/>
+ <use style="fill:#2ca02c;fill-opacity:0.2;stroke:#2ca02c;stroke-opacity:0.2;" x="382.925097" xlink:href="#mee8b134b61" y="136.312197"/>
+ <use style="fill:#2ca02c;fill-opacity:0.2;stroke:#2ca02c;stroke-opacity:0.2;" x="382.004907" xlink:href="#mee8b134b61" y="135.454961"/>
+ <use style="fill:#2ca02c;fill-opacity:0.2;stroke:#2ca02c;stroke-opacity:0.2;" x="381.078812" xlink:href="#mee8b134b61" y="134.600256"/>
+ <use style="fill:#2ca02c;fill-opacity:0.2;stroke:#2ca02c;stroke-opacity:0.2;" x="380.14695" xlink:href="#mee8b134b61" y="133.748113"/>
+ <use style="fill:#2ca02c;fill-opacity:0.2;stroke:#2ca02c;stroke-opacity:0.2;" x="379.209458" xlink:href="#mee8b134b61" y="132.898566"/>
+ <use style="fill:#2ca02c;fill-opacity:0.2;stroke:#2ca02c;stroke-opacity:0.2;" x="378.266476" xlink:href="#mee8b134b61" y="132.051644"/>
+ <use style="fill:#2ca02c;fill-opacity:0.2;stroke:#2ca02c;stroke-opacity:0.2;" x="377.318147" xlink:href="#mee8b134b61" y="131.207376"/>
+ <use style="fill:#2ca02c;fill-opacity:0.2;stroke:#2ca02c;stroke-opacity:0.2;" x="376.364611" xlink:href="#mee8b134b61" y="130.365789"/>
+ <use style="fill:#2ca02c;fill-opacity:0.2;stroke:#2ca02c;stroke-opacity:0.2;" x="375.406014" xlink:href="#mee8b134b61" y="129.526911"/>
+ <use style="fill:#2ca02c;fill-opacity:0.2;stroke:#2ca02c;stroke-opacity:0.2;" x="374.442497" xlink:href="#mee8b134b61" y="128.690765"/>
+ <use style="fill:#2ca02c;fill-opacity:0.2;stroke:#2ca02c;stroke-opacity:0.2;" x="373.474208" xlink:href="#mee8b134b61" y="127.857376"/>
+ <use style="fill:#2ca02c;fill-opacity:0.2;stroke:#2ca02c;stroke-opacity:0.2;" x="372.501289" xlink:href="#mee8b134b61" y="127.026766"/>
+ <use style="fill:#2ca02c;fill-opacity:0.2;stroke:#2ca02c;stroke-opacity:0.2;" x="371.523886" xlink:href="#mee8b134b61" y="126.198956"/>
+ <use style="fill:#2ca02c;fill-opacity:0.2;stroke:#2ca02c;stroke-opacity:0.2;" x="370.542127" xlink:href="#mee8b134b61" y="125.373988"/>
+ <use style="fill:#2ca02c;fill-opacity:0.2;stroke:#2ca02c;stroke-opacity:0.2;" x="369.556093" xlink:href="#mee8b134b61" y="124.551965"/>
+ <use style="fill:#2ca02c;fill-opacity:0.2;stroke:#2ca02c;stroke-opacity:0.2;" x="368.565923" xlink:href="#mee8b134b61" y="123.732919"/>
+ <use style="fill:#2ca02c;fill-opacity:0.2;stroke:#2ca02c;stroke-opacity:0.2;" x="367.571725" xlink:href="#mee8b134b61" y="122.916929"/>
+ <use style="fill:#2ca02c;fill-opacity:0.2;stroke:#2ca02c;stroke-opacity:0.2;" x="366.573613" xlink:href="#mee8b134b61" y="122.104059"/>
+ <use style="fill:#2ca02c;fill-opacity:0.2;stroke:#2ca02c;stroke-opacity:0.2;" x="365.571708" xlink:href="#mee8b134b61" y="121.294376"/>
+ <use style="fill:#2ca02c;fill-opacity:0.2;stroke:#2ca02c;stroke-opacity:0.2;" x="364.566131" xlink:href="#mee8b134b61" y="120.487941"/>
+ <use style="fill:#2ca02c;fill-opacity:0.2;stroke:#2ca02c;stroke-opacity:0.2;" x="363.557007" xlink:href="#mee8b134b61" y="119.684817"/>
+ <use style="fill:#2ca02c;fill-opacity:0.2;stroke:#2ca02c;stroke-opacity:0.2;" x="362.544461" xlink:href="#mee8b134b61" y="118.885065"/>
+ <use style="fill:#2ca02c;fill-opacity:0.2;stroke:#2ca02c;stroke-opacity:0.2;" x="361.528619" xlink:href="#mee8b134b61" y="118.088742"/>
+ <use style="fill:#2ca02c;fill-opacity:0.2;stroke:#2ca02c;stroke-opacity:0.2;" x="360.509608" xlink:href="#mee8b134b61" y="117.295908"/>
+ <use style="fill:#2ca02c;fill-opacity:0.2;stroke:#2ca02c;stroke-opacity:0.2;" x="359.487558" xlink:href="#mee8b134b61" y="116.506619"/>
+ <use style="fill:#2ca02c;fill-opacity:0.2;stroke:#2ca02c;stroke-opacity:0.2;" x="358.462597" xlink:href="#mee8b134b61" y="115.72093"/>
+ <use style="fill:#2ca02c;fill-opacity:0.2;stroke:#2ca02c;stroke-opacity:0.2;" x="357.434853" xlink:href="#mee8b134b61" y="114.938897"/>
+ <use style="fill:#2ca02c;fill-opacity:0.2;stroke:#2ca02c;stroke-opacity:0.2;" x="356.404456" xlink:href="#mee8b134b61" y="114.160572"/>
+ <use style="fill:#2ca02c;fill-opacity:0.2;stroke:#2ca02c;stroke-opacity:0.2;" x="355.371533" xlink:href="#mee8b134b61" y="113.386007"/>
+ <use style="fill:#2ca02c;fill-opacity:0.2;stroke:#2ca02c;stroke-opacity:0.2;" x="354.336213" xlink:href="#mee8b134b61" y="112.615254"/>
+ <use style="fill:#2ca02c;fill-opacity:0.2;stroke:#2ca02c;stroke-opacity:0.2;" x="353.298622" xlink:href="#mee8b134b61" y="111.848361"/>
+ <use style="fill:#2ca02c;fill-opacity:0.2;stroke:#2ca02c;stroke-opacity:0.2;" x="352.258888" xlink:href="#mee8b134b61" y="111.085377"/>
+ <use style="fill:#2ca02c;fill-opacity:0.2;stroke:#2ca02c;stroke-opacity:0.2;" x="351.217134" xlink:href="#mee8b134b61" y="110.326349"/>
+ <use style="fill:#2ca02c;fill-opacity:0.2;stroke:#2ca02c;stroke-opacity:0.2;" x="350.173485" xlink:href="#mee8b134b61" y="109.571323"/>
+ <use style="fill:#2ca02c;fill-opacity:0.2;stroke:#2ca02c;stroke-opacity:0.2;" x="349.128065" xlink:href="#mee8b134b61" y="108.820343"/>
+ <use style="fill:#2ca02c;fill-opacity:0.2;stroke:#2ca02c;stroke-opacity:0.2;" x="348.080993" xlink:href="#mee8b134b61" y="108.073454"/>
+ <use style="fill:#2ca02c;fill-opacity:0.2;stroke:#2ca02c;stroke-opacity:0.2;" x="347.03239" xlink:href="#mee8b134b61" y="107.330696"/>
+ <use style="fill:#2ca02c;fill-opacity:0.2;stroke:#2ca02c;stroke-opacity:0.2;" x="345.982373" xlink:href="#mee8b134b61" y="106.592111"/>
+ <use style="fill:#2ca02c;fill-opacity:0.2;stroke:#2ca02c;stroke-opacity:0.2;" x="344.93106" xlink:href="#mee8b134b61" y="105.857738"/>
+ <use style="fill:#2ca02c;fill-opacity:0.2;stroke:#2ca02c;stroke-opacity:0.2;" x="343.878565" xlink:href="#mee8b134b61" y="105.127616"/>
+ <use style="fill:#2ca02c;fill-opacity:0.2;stroke:#2ca02c;stroke-opacity:0.2;" x="342.825001" xlink:href="#mee8b134b61" y="104.401781"/>
+ <use style="fill:#2ca02c;fill-opacity:0.2;stroke:#2ca02c;stroke-opacity:0.2;" x="341.770479" xlink:href="#mee8b134b61" y="103.680268"/>
+ <use style="fill:#2ca02c;fill-opacity:0.2;stroke:#2ca02c;stroke-opacity:0.2;" x="340.715108" xlink:href="#mee8b134b61" y="102.963111"/>
+ <use style="fill:#2ca02c;fill-opacity:0.2;stroke:#2ca02c;stroke-opacity:0.2;" x="339.658996" xlink:href="#mee8b134b61" y="102.250343"/>
+ <use style="fill:#2ca02c;fill-opacity:0.2;stroke:#2ca02c;stroke-opacity:0.2;" x="338.602248" xlink:href="#mee8b134b61" y="101.541995"/>
+ <use style="fill:#2ca02c;fill-opacity:0.2;stroke:#2ca02c;stroke-opacity:0.2;" x="337.54497" xlink:href="#mee8b134b61" y="100.838096"/>
+ <use style="fill:#2ca02c;fill-opacity:0.2;stroke:#2ca02c;stroke-opacity:0.2;" x="336.487262" xlink:href="#mee8b134b61" y="100.138676"/>
+ <use style="fill:#2ca02c;fill-opacity:0.2;stroke:#2ca02c;stroke-opacity:0.2;" x="335.429225" xlink:href="#mee8b134b61" y="99.443759"/>
+ <use style="fill:#2ca02c;fill-opacity:0.2;stroke:#2ca02c;stroke-opacity:0.2;" x="334.370958" xlink:href="#mee8b134b61" y="98.753373"/>
+ <use style="fill:#2ca02c;fill-opacity:0.2;stroke:#2ca02c;stroke-opacity:0.2;" x="333.312559" xlink:href="#mee8b134b61" y="98.06754"/>
+ <use style="fill:#2ca02c;fill-opacity:0.2;stroke:#2ca02c;stroke-opacity:0.2;" x="332.254123" xlink:href="#mee8b134b61" y="97.386282"/>
+ <use style="fill:#2ca02c;fill-opacity:0.2;stroke:#2ca02c;stroke-opacity:0.2;" x="331.195745" xlink:href="#mee8b134b61" y="96.70962"/>
+ <use style="fill:#2ca02c;fill-opacity:0.2;stroke:#2ca02c;stroke-opacity:0.2;" x="330.137517" xlink:href="#mee8b134b61" y="96.037574"/>
+ <use style="fill:#2ca02c;fill-opacity:0.2;stroke:#2ca02c;stroke-opacity:0.2;" x="329.079531" xlink:href="#mee8b134b61" y="95.37016"/>
+ <use style="fill:#2ca02c;fill-opacity:0.2;stroke:#2ca02c;stroke-opacity:0.2;" x="328.021878" xlink:href="#mee8b134b61" y="94.707395"/>
+ <use style="fill:#2ca02c;fill-opacity:0.2;stroke:#2ca02c;stroke-opacity:0.2;" x="326.964646" xlink:href="#mee8b134b61" y="94.049294"/>
+ <use style="fill:#2ca02c;fill-opacity:0.2;stroke:#2ca02c;stroke-opacity:0.2;" x="325.907923" xlink:href="#mee8b134b61" y="93.395869"/>
+ <use style="fill:#2ca02c;fill-opacity:0.2;stroke:#2ca02c;stroke-opacity:0.2;" x="324.851798" xlink:href="#mee8b134b61" y="92.747132"/>
+ <use style="fill:#2ca02c;fill-opacity:0.2;stroke:#2ca02c;stroke-opacity:0.2;" x="323.796355" xlink:href="#mee8b134b61" y="92.103092"/>
+ <use style="fill:#2ca02c;fill-opacity:0.2;stroke:#2ca02c;stroke-opacity:0.2;" x="322.741682" xlink:href="#mee8b134b61" y="91.463758"/>
+ <use style="fill:#2ca02c;fill-opacity:0.2;stroke:#2ca02c;stroke-opacity:0.2;" x="321.687861" xlink:href="#mee8b134b61" y="90.829137"/>
+ <use style="fill:#2ca02c;fill-opacity:0.2;stroke:#2ca02c;stroke-opacity:0.2;" x="320.634977" xlink:href="#mee8b134b61" y="90.199234"/>
+ <use style="fill:#2ca02c;fill-opacity:0.2;stroke:#2ca02c;stroke-opacity:0.2;" x="319.583114" xlink:href="#mee8b134b61" y="89.574054"/>
+ <use style="fill:#2ca02c;fill-opacity:0.2;stroke:#2ca02c;stroke-opacity:0.2;" x="318.532355" xlink:href="#mee8b134b61" y="88.953597"/>
+ <use style="fill:#2ca02c;fill-opacity:0.2;stroke:#2ca02c;stroke-opacity:0.2;" x="317.482782" xlink:href="#mee8b134b61" y="88.337866"/>
+ <use style="fill:#2ca02c;fill-opacity:0.2;stroke:#2ca02c;stroke-opacity:0.2;" x="316.434477" xlink:href="#mee8b134b61" y="87.726859"/>
+ <use style="fill:#2ca02c;fill-opacity:0.2;stroke:#2ca02c;stroke-opacity:0.2;" x="315.387522" xlink:href="#mee8b134b61" y="87.120575"/>
+ <use style="fill:#2ca02c;fill-opacity:0.2;stroke:#2ca02c;stroke-opacity:0.2;" x="314.341998" xlink:href="#mee8b134b61" y="86.519011"/>
+ <use style="fill:#2ca02c;fill-opacity:0.2;stroke:#2ca02c;stroke-opacity:0.2;" x="313.297987" xlink:href="#mee8b134b61" y="85.922161"/>
+ <use style="fill:#2ca02c;fill-opacity:0.2;stroke:#2ca02c;stroke-opacity:0.2;" x="312.255569" xlink:href="#mee8b134b61" y="85.330019"/>
+ <use style="fill:#2ca02c;fill-opacity:0.2;stroke:#2ca02c;stroke-opacity:0.2;" x="311.214826" xlink:href="#mee8b134b61" y="84.742579"/>
+ <use style="fill:#2ca02c;fill-opacity:0.2;stroke:#2ca02c;stroke-opacity:0.2;" x="310.175839" xlink:href="#mee8b134b61" y="84.15983"/>
+ <use style="fill:#2ca02c;fill-opacity:0.2;stroke:#2ca02c;stroke-opacity:0.2;" x="309.138687" xlink:href="#mee8b134b61" y="83.581765"/>
+ <use style="fill:#2ca02c;fill-opacity:0.2;stroke:#2ca02c;stroke-opacity:0.2;" x="308.103452" xlink:href="#mee8b134b61" y="83.00837"/>
+ <use style="fill:#2ca02c;fill-opacity:0.2;stroke:#2ca02c;stroke-opacity:0.2;" x="307.070214" xlink:href="#mee8b134b61" y="82.439634"/>
+ <use style="fill:#2ca02c;fill-opacity:0.2;stroke:#2ca02c;stroke-opacity:0.2;" x="306.039053" xlink:href="#mee8b134b61" y="81.875543"/>
+ <use style="fill:#2ca02c;fill-opacity:0.2;stroke:#2ca02c;stroke-opacity:0.2;" x="305.010049" xlink:href="#mee8b134b61" y="81.316082"/>
+ <use style="fill:#2ca02c;fill-opacity:0.2;stroke:#2ca02c;stroke-opacity:0.2;" x="303.983284" xlink:href="#mee8b134b61" y="80.761236"/>
+ <use style="fill:#2ca02c;fill-opacity:0.2;stroke:#2ca02c;stroke-opacity:0.2;" x="302.958836" xlink:href="#mee8b134b61" y="80.210988"/>
+ <use style="fill:#2ca02c;fill-opacity:0.2;stroke:#2ca02c;stroke-opacity:0.2;" x="301.936785" xlink:href="#mee8b134b61" y="79.665318"/>
+ <use style="fill:#2ca02c;fill-opacity:0.2;stroke:#2ca02c;stroke-opacity:0.2;" x="300.917211" xlink:href="#mee8b134b61" y="79.124209"/>
+ <use style="fill:#2ca02c;fill-opacity:0.2;stroke:#2ca02c;stroke-opacity:0.2;" x="299.900192" xlink:href="#mee8b134b61" y="78.587642"/>
+ <use style="fill:#2ca02c;fill-opacity:0.2;stroke:#2ca02c;stroke-opacity:0.2;" x="298.885766" xlink:href="#mee8b134b61" y="78.055617"/>
+ <use style="fill:#2ca02c;fill-opacity:0.2;stroke:#2ca02c;stroke-opacity:0.2;" x="297.873992" xlink:href="#mee8b134b61" y="77.528121"/>
+ <use style="fill:#2ca02c;fill-opacity:0.2;stroke:#2ca02c;stroke-opacity:0.2;" x="296.864925" xlink:href="#mee8b134b61" y="77.005145"/>
+ <use style="fill:#2ca02c;fill-opacity:0.2;stroke:#2ca02c;stroke-opacity:0.2;" x="295.858613" xlink:href="#mee8b134b61" y="76.48668"/>
+ <use style="fill:#2ca02c;fill-opacity:0.2;stroke:#2ca02c;stroke-opacity:0.2;" x="294.855106" xlink:href="#mee8b134b61" y="75.972714"/>
+ <use style="fill:#2ca02c;fill-opacity:0.2;stroke:#2ca02c;stroke-opacity:0.2;" x="293.854452" xlink:href="#mee8b134b61" y="75.463235"/>
+ <use style="fill:#2ca02c;fill-opacity:0.2;stroke:#2ca02c;stroke-opacity:0.2;" x="292.856702" xlink:href="#mee8b134b61" y="74.958229"/>
+ <use style="fill:#2ca02c;fill-opacity:0.2;stroke:#2ca02c;stroke-opacity:0.2;" x="291.861904" xlink:href="#mee8b134b61" y="74.457681"/>
+ <use style="fill:#2ca02c;fill-opacity:0.2;stroke:#2ca02c;stroke-opacity:0.2;" x="290.870106" xlink:href="#mee8b134b61" y="73.961575"/>
+ <use style="fill:#2ca02c;fill-opacity:0.2;stroke:#2ca02c;stroke-opacity:0.2;" x="289.881355" xlink:href="#mee8b134b61" y="73.469893"/>
+ <use style="fill:#2ca02c;fill-opacity:0.2;stroke:#2ca02c;stroke-opacity:0.2;" x="288.895698" xlink:href="#mee8b134b61" y="72.982617"/>
+ <use style="fill:#2ca02c;fill-opacity:0.2;stroke:#2ca02c;stroke-opacity:0.2;" x="287.913182" xlink:href="#mee8b134b61" y="72.499728"/>
+ <use style="fill:#2ca02c;fill-opacity:0.2;stroke:#2ca02c;stroke-opacity:0.2;" x="286.933851" xlink:href="#mee8b134b61" y="72.021205"/>
+ <use style="fill:#2ca02c;fill-opacity:0.2;stroke:#2ca02c;stroke-opacity:0.2;" x="285.95775" xlink:href="#mee8b134b61" y="71.547026"/>
+ <use style="fill:#2ca02c;fill-opacity:0.2;stroke:#2ca02c;stroke-opacity:0.2;" x="284.984923" xlink:href="#mee8b134b61" y="71.077169"/>
+ <use style="fill:#2ca02c;fill-opacity:0.2;stroke:#2ca02c;stroke-opacity:0.2;" x="284.015402" xlink:href="#mee8b134b61" y="70.611636"/>
+ <use style="fill:#2ca02c;fill-opacity:0.2;stroke:#2ca02c;stroke-opacity:0.2;" x="283.049215" xlink:href="#mee8b134b61" y="70.150442"/>
+ <use style="fill:#2ca02c;fill-opacity:0.2;stroke:#2ca02c;stroke-opacity:0.2;" x="282.086398" xlink:href="#mee8b134b61" y="69.693576"/>
+ <use style="fill:#2ca02c;fill-opacity:0.2;stroke:#2ca02c;stroke-opacity:0.2;" x="281.126978" xlink:href="#mee8b134b61" y="69.241046"/>
+ <use style="fill:#2ca02c;fill-opacity:0.2;stroke:#2ca02c;stroke-opacity:0.2;" x="280.170982" xlink:href="#mee8b134b61" y="68.792855"/>
+ <use style="fill:#2ca02c;fill-opacity:0.2;stroke:#2ca02c;stroke-opacity:0.2;" x="279.218438" xlink:href="#mee8b134b61" y="68.349003"/>
+ <use style="fill:#2ca02c;fill-opacity:0.2;stroke:#2ca02c;stroke-opacity:0.2;" x="278.269372" xlink:href="#mee8b134b61" y="67.909488"/>
+ <use style="fill:#2ca02c;fill-opacity:0.2;stroke:#2ca02c;stroke-opacity:0.2;" x="277.323808" xlink:href="#mee8b134b61" y="67.47431"/>
+ <use style="fill:#2ca02c;fill-opacity:0.2;stroke:#2ca02c;stroke-opacity:0.2;" x="276.381771" xlink:href="#mee8b134b61" y="67.043465"/>
+ <use style="fill:#2ca02c;fill-opacity:0.2;stroke:#2ca02c;stroke-opacity:0.2;" x="275.44328" xlink:href="#mee8b134b61" y="66.616947"/>
+ <use style="fill:#2ca02c;fill-opacity:0.2;stroke:#2ca02c;stroke-opacity:0.2;" x="274.508357" xlink:href="#mee8b134b61" y="66.19475"/>
+ <use style="fill:#2ca02c;fill-opacity:0.2;stroke:#2ca02c;stroke-opacity:0.2;" x="273.577021" xlink:href="#mee8b134b61" y="65.776868"/>
+ <use style="fill:#2ca02c;fill-opacity:0.2;stroke:#2ca02c;stroke-opacity:0.2;" x="272.649288" xlink:href="#mee8b134b61" y="65.363291"/>
+ <use style="fill:#2ca02c;fill-opacity:0.2;stroke:#2ca02c;stroke-opacity:0.2;" x="271.725172" xlink:href="#mee8b134b61" y="64.95401"/>
+ <use style="fill:#2ca02c;fill-opacity:0.2;stroke:#2ca02c;stroke-opacity:0.2;" x="270.804688" xlink:href="#mee8b134b61" y="64.549012"/>
+ <use style="fill:#2ca02c;fill-opacity:0.2;stroke:#2ca02c;stroke-opacity:0.2;" x="269.887847" xlink:href="#mee8b134b61" y="64.148287"/>
+ <use style="fill:#2ca02c;fill-opacity:0.2;stroke:#2ca02c;stroke-opacity:0.2;" x="268.974657" xlink:href="#mee8b134b61" y="63.751819"/>
+ <use style="fill:#2ca02c;fill-opacity:0.2;stroke:#2ca02c;stroke-opacity:0.2;" x="268.065126" xlink:href="#mee8b134b61" y="63.359593"/>
+ <use style="fill:#2ca02c;fill-opacity:0.2;stroke:#2ca02c;stroke-opacity:0.2;" x="267.159259" xlink:href="#mee8b134b61" y="62.971594"/>
+ <use style="fill:#2ca02c;fill-opacity:0.2;stroke:#2ca02c;stroke-opacity:0.2;" x="266.257058" xlink:href="#mee8b134b61" y="62.587804"/>
+ <use style="fill:#2ca02c;fill-opacity:0.2;stroke:#2ca02c;stroke-opacity:0.2;" x="265.358525" xlink:href="#mee8b134b61" y="62.208203"/>
+ <use style="fill:#2ca02c;fill-opacity:0.2;stroke:#2ca02c;stroke-opacity:0.2;" x="264.463659" xlink:href="#mee8b134b61" y="61.832773"/>
+ <use style="fill:#2ca02c;fill-opacity:0.2;stroke:#2ca02c;stroke-opacity:0.2;" x="263.572455" xlink:href="#mee8b134b61" y="61.461491"/>
+ <use style="fill:#2ca02c;fill-opacity:0.2;stroke:#2ca02c;stroke-opacity:0.2;" x="262.684908" xlink:href="#mee8b134b61" y="61.094336"/>
+ <use style="fill:#2ca02c;fill-opacity:0.2;stroke:#2ca02c;stroke-opacity:0.2;" x="261.80101" xlink:href="#mee8b134b61" y="60.731284"/>
+ <use style="fill:#2ca02c;fill-opacity:0.2;stroke:#2ca02c;stroke-opacity:0.2;" x="260.92075" xlink:href="#mee8b134b61" y="60.372312"/>
+ <use style="fill:#2ca02c;fill-opacity:0.2;stroke:#2ca02c;stroke-opacity:0.2;" x="260.044116" xlink:href="#mee8b134b61" y="60.017392"/>
+ <use style="fill:#2ca02c;fill-opacity:0.2;stroke:#2ca02c;stroke-opacity:0.2;" x="259.171093" xlink:href="#mee8b134b61" y="59.6665"/>
+ <use style="fill:#2ca02c;fill-opacity:0.2;stroke:#2ca02c;stroke-opacity:0.2;" x="258.301665" xlink:href="#mee8b134b61" y="59.319606"/>
+ <use style="fill:#2ca02c;fill-opacity:0.2;stroke:#2ca02c;stroke-opacity:0.2;" x="257.435811" xlink:href="#mee8b134b61" y="58.976684"/>
+ <use style="fill:#2ca02c;fill-opacity:0.2;stroke:#2ca02c;stroke-opacity:0.2;" x="256.57351" xlink:href="#mee8b134b61" y="58.637703"/>
+ <use style="fill:#2ca02c;fill-opacity:0.2;stroke:#2ca02c;stroke-opacity:0.2;" x="255.714739" xlink:href="#mee8b134b61" y="58.302633"/>
+ <use style="fill:#2ca02c;fill-opacity:0.2;stroke:#2ca02c;stroke-opacity:0.2;" x="254.859472" xlink:href="#mee8b134b61" y="57.971442"/>
+ <use style="fill:#2ca02c;fill-opacity:0.2;stroke:#2ca02c;stroke-opacity:0.2;" x="254.007681" xlink:href="#mee8b134b61" y="57.644099"/>
+ <use style="fill:#2ca02c;fill-opacity:0.2;stroke:#2ca02c;stroke-opacity:0.2;" x="253.159334" xlink:href="#mee8b134b61" y="57.320571"/>
+ <use style="fill:#2ca02c;fill-opacity:0.2;stroke:#2ca02c;stroke-opacity:0.2;" x="252.314401" xlink:href="#mee8b134b61" y="57.000823"/>
+ <use style="fill:#2ca02c;fill-opacity:0.2;stroke:#2ca02c;stroke-opacity:0.2;" x="251.472847" xlink:href="#mee8b134b61" y="56.684822"/>
+ <use style="fill:#2ca02c;fill-opacity:0.2;stroke:#2ca02c;stroke-opacity:0.2;" x="250.634635" xlink:href="#mee8b134b61" y="56.372532"/>
+ <use style="fill:#2ca02c;fill-opacity:0.2;stroke:#2ca02c;stroke-opacity:0.2;" x="249.799726" xlink:href="#mee8b134b61" y="56.063917"/>
+ <use style="fill:#2ca02c;fill-opacity:0.2;stroke:#2ca02c;stroke-opacity:0.2;" x="248.968081" xlink:href="#mee8b134b61" y="55.75894"/>
+ <use style="fill:#2ca02c;fill-opacity:0.2;stroke:#2ca02c;stroke-opacity:0.2;" x="248.139657" xlink:href="#mee8b134b61" y="55.457565"/>
+ <use style="fill:#2ca02c;fill-opacity:0.2;stroke:#2ca02c;stroke-opacity:0.2;" x="247.31441" xlink:href="#mee8b134b61" y="55.159753"/>
+ <use style="fill:#2ca02c;fill-opacity:0.2;stroke:#2ca02c;stroke-opacity:0.2;" x="246.492293" xlink:href="#mee8b134b61" y="54.865465"/>
+ <use style="fill:#2ca02c;fill-opacity:0.2;stroke:#2ca02c;stroke-opacity:0.2;" x="245.673259" xlink:href="#mee8b134b61" y="54.574664"/>
+ <use style="fill:#2ca02c;fill-opacity:0.2;stroke:#2ca02c;stroke-opacity:0.2;" x="244.857258" xlink:href="#mee8b134b61" y="54.287308"/>
+ <use style="fill:#2ca02c;fill-opacity:0.2;stroke:#2ca02c;stroke-opacity:0.2;" x="244.044238" xlink:href="#mee8b134b61" y="54.003358"/>
+ <use style="fill:#2ca02c;fill-opacity:0.2;stroke:#2ca02c;stroke-opacity:0.2;" x="243.234146" xlink:href="#mee8b134b61" y="53.722773"/>
+ <use style="fill:#2ca02c;fill-opacity:0.2;stroke:#2ca02c;stroke-opacity:0.2;" x="242.426929" xlink:href="#mee8b134b61" y="53.445512"/>
+ <use style="fill:#2ca02c;fill-opacity:0.2;stroke:#2ca02c;stroke-opacity:0.2;" x="241.622528" xlink:href="#mee8b134b61" y="53.171534"/>
+ <use style="fill:#2ca02c;fill-opacity:0.2;stroke:#2ca02c;stroke-opacity:0.2;" x="240.820887" xlink:href="#mee8b134b61" y="52.900798"/>
+ <use style="fill:#2ca02c;fill-opacity:0.2;stroke:#2ca02c;stroke-opacity:0.2;" x="240.021945" xlink:href="#mee8b134b61" y="52.63326"/>
+ <use style="fill:#2ca02c;fill-opacity:0.2;stroke:#2ca02c;stroke-opacity:0.2;" x="239.225643" xlink:href="#mee8b134b61" y="52.368878"/>
+ <use style="fill:#2ca02c;fill-opacity:0.2;stroke:#2ca02c;stroke-opacity:0.2;" x="238.431917" xlink:href="#mee8b134b61" y="52.10761"/>
+ <use style="fill:#2ca02c;fill-opacity:0.2;stroke:#2ca02c;stroke-opacity:0.2;" x="237.640705" xlink:href="#mee8b134b61" y="51.849412"/>
+ <use style="fill:#2ca02c;fill-opacity:0.2;stroke:#2ca02c;stroke-opacity:0.2;" x="236.851941" xlink:href="#mee8b134b61" y="51.594242"/>
+ <use style="fill:#2ca02c;fill-opacity:0.2;stroke:#2ca02c;stroke-opacity:0.2;" x="236.065559" xlink:href="#mee8b134b61" y="51.342056"/>
+ <use style="fill:#2ca02c;fill-opacity:0.2;stroke:#2ca02c;stroke-opacity:0.2;" x="235.281491" xlink:href="#mee8b134b61" y="51.092809"/>
+ <use style="fill:#2ca02c;fill-opacity:0.2;stroke:#2ca02c;stroke-opacity:0.2;" x="234.49967" xlink:href="#mee8b134b61" y="50.846458"/>
+ <use style="fill:#2ca02c;fill-opacity:0.2;stroke:#2ca02c;stroke-opacity:0.2;" x="233.720025" xlink:href="#mee8b134b61" y="50.60296"/>
+ <use style="fill:#2ca02c;fill-opacity:0.2;stroke:#2ca02c;stroke-opacity:0.2;" x="232.942486" xlink:href="#mee8b134b61" y="50.362269"/>
+ <use style="fill:#2ca02c;fill-opacity:0.2;stroke:#2ca02c;stroke-opacity:0.2;" x="232.166981" xlink:href="#mee8b134b61" y="50.124341"/>
+ <use style="fill:#2ca02c;fill-opacity:0.2;stroke:#2ca02c;stroke-opacity:0.2;" x="231.393436" xlink:href="#mee8b134b61" y="49.889133"/>
+ <use style="fill:#2ca02c;fill-opacity:0.2;stroke:#2ca02c;stroke-opacity:0.2;" x="230.621779" xlink:href="#mee8b134b61" y="49.656598"/>
+ <use style="fill:#2ca02c;fill-opacity:0.2;stroke:#2ca02c;stroke-opacity:0.2;" x="229.851935" xlink:href="#mee8b134b61" y="49.426693"/>
+ <use style="fill:#2ca02c;fill-opacity:0.2;stroke:#2ca02c;stroke-opacity:0.2;" x="229.083827" xlink:href="#mee8b134b61" y="49.199373"/>
+ <use style="fill:#2ca02c;fill-opacity:0.2;stroke:#2ca02c;stroke-opacity:0.2;" x="228.317381" xlink:href="#mee8b134b61" y="48.974593"/>
+ <use style="fill:#2ca02c;fill-opacity:0.2;stroke:#2ca02c;stroke-opacity:0.2;" x="227.552518" xlink:href="#mee8b134b61" y="48.752309"/>
+ <use style="fill:#2ca02c;fill-opacity:0.2;stroke:#2ca02c;stroke-opacity:0.2;" x="226.789161" xlink:href="#mee8b134b61" y="48.532475"/>
+ <use style="fill:#2ca02c;fill-opacity:0.2;stroke:#2ca02c;stroke-opacity:0.2;" x="226.027231" xlink:href="#mee8b134b61" y="48.315046"/>
+ <use style="fill:#2ca02c;fill-opacity:0.2;stroke:#2ca02c;stroke-opacity:0.2;" x="225.26665" xlink:href="#mee8b134b61" y="48.099979"/>
+ <use style="fill:#2ca02c;fill-opacity:0.2;stroke:#2ca02c;stroke-opacity:0.2;" x="224.507336" xlink:href="#mee8b134b61" y="47.887228"/>
+ <use style="fill:#2ca02c;fill-opacity:0.2;stroke:#2ca02c;stroke-opacity:0.2;" x="223.749211" xlink:href="#mee8b134b61" y="47.676748"/>
+ <use style="fill:#2ca02c;fill-opacity:0.2;stroke:#2ca02c;stroke-opacity:0.2;" x="222.992192" xlink:href="#mee8b134b61" y="47.468496"/>
+ <use style="fill:#2ca02c;fill-opacity:0.2;stroke:#2ca02c;stroke-opacity:0.2;" x="222.236198" xlink:href="#mee8b134b61" y="47.262426"/>
+ <use style="fill:#2ca02c;fill-opacity:0.2;stroke:#2ca02c;stroke-opacity:0.2;" x="221.481148" xlink:href="#mee8b134b61" y="47.058495"/>
+ <use style="fill:#2ca02c;fill-opacity:0.2;stroke:#2ca02c;stroke-opacity:0.2;" x="220.726958" xlink:href="#mee8b134b61" y="46.856657"/>
+ <use style="fill:#2ca02c;fill-opacity:0.2;stroke:#2ca02c;stroke-opacity:0.2;" x="219.973547" xlink:href="#mee8b134b61" y="46.656868"/>
+ <use style="fill:#2ca02c;fill-opacity:0.2;stroke:#2ca02c;stroke-opacity:0.2;" x="219.22083" xlink:href="#mee8b134b61" y="46.459086"/>
+ <use style="fill:#2ca02c;fill-opacity:0.2;stroke:#2ca02c;stroke-opacity:0.2;" x="218.468724" xlink:href="#mee8b134b61" y="46.263265"/>
+ <use style="fill:#2ca02c;fill-opacity:0.2;stroke:#2ca02c;stroke-opacity:0.2;" x="217.717146" xlink:href="#mee8b134b61" y="46.069362"/>
+ <use style="fill:#2ca02c;fill-opacity:0.2;stroke:#2ca02c;stroke-opacity:0.2;" x="216.96601" xlink:href="#mee8b134b61" y="45.877333"/>
+ <use style="fill:#2ca02c;fill-opacity:0.2;stroke:#2ca02c;stroke-opacity:0.2;" x="216.215232" xlink:href="#mee8b134b61" y="45.687135"/>
+ <use style="fill:#2ca02c;fill-opacity:0.2;stroke:#2ca02c;stroke-opacity:0.2;" x="215.464728" xlink:href="#mee8b134b61" y="45.498724"/>
+ <use style="fill:#2ca02c;fill-opacity:0.2;stroke:#2ca02c;stroke-opacity:0.2;" x="214.714413" xlink:href="#mee8b134b61" y="45.312058"/>
+ <use style="fill:#2ca02c;fill-opacity:0.2;stroke:#2ca02c;stroke-opacity:0.2;" x="213.964201" xlink:href="#mee8b134b61" y="45.127094"/>
+ <use style="fill:#2ca02c;fill-opacity:0.2;stroke:#2ca02c;stroke-opacity:0.2;" x="213.214007" xlink:href="#mee8b134b61" y="44.943789"/>
+ <use style="fill:#2ca02c;fill-opacity:0.2;stroke:#2ca02c;stroke-opacity:0.2;" x="212.463745" xlink:href="#mee8b134b61" y="44.7621"/>
+ <use style="fill:#2ca02c;fill-opacity:0.2;stroke:#2ca02c;stroke-opacity:0.2;" x="211.713331" xlink:href="#mee8b134b61" y="44.581986"/>
+ <use style="fill:#2ca02c;fill-opacity:0.2;stroke:#2ca02c;stroke-opacity:0.2;" x="210.962679" xlink:href="#mee8b134b61" y="44.403405"/>
+ <use style="fill:#2ca02c;fill-opacity:0.2;stroke:#2ca02c;stroke-opacity:0.2;" x="210.211702" xlink:href="#mee8b134b61" y="44.226314"/>
+ <use style="fill:#2ca02c;fill-opacity:0.2;stroke:#2ca02c;stroke-opacity:0.2;" x="209.460316" xlink:href="#mee8b134b61" y="44.050673"/>
+ <use style="fill:#2ca02c;fill-opacity:0.2;stroke:#2ca02c;stroke-opacity:0.2;" x="208.708434" xlink:href="#mee8b134b61" y="43.87644"/>
+ <use style="fill:#2ca02c;fill-opacity:0.2;stroke:#2ca02c;stroke-opacity:0.2;" x="207.955971" xlink:href="#mee8b134b61" y="43.703575"/>
+ <use style="fill:#2ca02c;fill-opacity:0.2;stroke:#2ca02c;stroke-opacity:0.2;" x="207.202841" xlink:href="#mee8b134b61" y="43.532037"/>
+ <use style="fill:#2ca02c;fill-opacity:0.2;stroke:#2ca02c;stroke-opacity:0.2;" x="206.448959" xlink:href="#mee8b134b61" y="43.361785"/>
+ <use style="fill:#2ca02c;fill-opacity:0.2;stroke:#2ca02c;stroke-opacity:0.2;" x="205.694239" xlink:href="#mee8b134b61" y="43.19278"/>
+ <use style="fill:#2ca02c;fill-opacity:0.2;stroke:#2ca02c;stroke-opacity:0.2;" x="204.938597" xlink:href="#mee8b134b61" y="43.024982"/>
+ <use style="fill:#2ca02c;fill-opacity:0.2;stroke:#2ca02c;stroke-opacity:0.2;" x="204.181946" xlink:href="#mee8b134b61" y="42.858352"/>
+ <use style="fill:#2ca02c;fill-opacity:0.2;stroke:#2ca02c;stroke-opacity:0.2;" x="203.424203" xlink:href="#mee8b134b61" y="42.692851"/>
+ <use style="fill:#2ca02c;fill-opacity:0.2;stroke:#2ca02c;stroke-opacity:0.2;" x="202.665282" xlink:href="#mee8b134b61" y="42.528439"/>
+ <use style="fill:#2ca02c;fill-opacity:0.2;stroke:#2ca02c;stroke-opacity:0.2;" x="201.905098" xlink:href="#mee8b134b61" y="42.36508"/>
+ <use style="fill:#2ca02c;fill-opacity:0.2;stroke:#2ca02c;stroke-opacity:0.2;" x="201.143569" xlink:href="#mee8b134b61" y="42.202734"/>
+ <use style="fill:#2ca02c;fill-opacity:0.2;stroke:#2ca02c;stroke-opacity:0.2;" x="200.380609" xlink:href="#mee8b134b61" y="42.041365"/>
+ <use style="fill:#2ca02c;fill-opacity:0.2;stroke:#2ca02c;stroke-opacity:0.2;" x="199.616135" xlink:href="#mee8b134b61" y="41.880934"/>
+ <use style="fill:#2ca02c;fill-opacity:0.2;stroke:#2ca02c;stroke-opacity:0.2;" x="198.850064" xlink:href="#mee8b134b61" y="41.721406"/>
+ <use style="fill:#2ca02c;fill-opacity:0.2;stroke:#2ca02c;stroke-opacity:0.2;" x="198.082312" xlink:href="#mee8b134b61" y="41.562744"/>
+ <use style="fill:#2ca02c;fill-opacity:0.2;stroke:#2ca02c;stroke-opacity:0.2;" x="197.312798" xlink:href="#mee8b134b61" y="41.404911"/>
+ <use style="fill:#2ca02c;fill-opacity:0.2;stroke:#2ca02c;stroke-opacity:0.2;" x="196.541438" xlink:href="#mee8b134b61" y="41.247873"/>
+ <use style="fill:#2ca02c;fill-opacity:0.2;stroke:#2ca02c;stroke-opacity:0.2;" x="195.768152" xlink:href="#mee8b134b61" y="41.091593"/>
+ <use style="fill:#2ca02c;fill-opacity:0.2;stroke:#2ca02c;stroke-opacity:0.2;" x="194.992858" xlink:href="#mee8b134b61" y="40.936038"/>
+ <use style="fill:#2ca02c;fill-opacity:0.2;stroke:#2ca02c;stroke-opacity:0.2;" x="194.215475" xlink:href="#mee8b134b61" y="40.781172"/>
+ <use style="fill:#2ca02c;fill-opacity:0.2;stroke:#2ca02c;stroke-opacity:0.2;" x="193.435922" xlink:href="#mee8b134b61" y="40.626962"/>
+ <use style="fill:#2ca02c;fill-opacity:0.2;stroke:#2ca02c;stroke-opacity:0.2;" x="192.65412" xlink:href="#mee8b134b61" y="40.473374"/>
+ <use style="fill:#2ca02c;fill-opacity:0.2;stroke:#2ca02c;stroke-opacity:0.2;" x="191.86999" xlink:href="#mee8b134b61" y="40.320375"/>
+ <use style="fill:#2ca02c;fill-opacity:0.2;stroke:#2ca02c;stroke-opacity:0.2;" x="191.083452" xlink:href="#mee8b134b61" y="40.167933"/>
+ <use style="fill:#2ca02c;fill-opacity:0.2;stroke:#2ca02c;stroke-opacity:0.2;" x="190.294429" xlink:href="#mee8b134b61" y="40.016016"/>
+ <use style="fill:#2ca02c;fill-opacity:0.2;stroke:#2ca02c;stroke-opacity:0.2;" x="189.502842" xlink:href="#mee8b134b61" y="39.864591"/>
+ <use style="fill:#2ca02c;fill-opacity:0.2;stroke:#2ca02c;stroke-opacity:0.2;" x="188.708615" xlink:href="#mee8b134b61" y="39.713628"/>
+ <use style="fill:#2ca02c;fill-opacity:0.2;stroke:#2ca02c;stroke-opacity:0.2;" x="187.911671" xlink:href="#mee8b134b61" y="39.563096"/>
+ <use style="fill:#2ca02c;fill-opacity:0.2;stroke:#2ca02c;stroke-opacity:0.2;" x="187.111935" xlink:href="#mee8b134b61" y="39.412964"/>
+ <use style="fill:#2ca02c;fill-opacity:0.2;stroke:#2ca02c;stroke-opacity:0.2;" x="186.30933" xlink:href="#mee8b134b61" y="39.263204"/>
+ <use style="fill:#2ca02c;fill-opacity:0.2;stroke:#2ca02c;stroke-opacity:0.2;" x="185.503783" xlink:href="#mee8b134b61" y="39.113785"/>
+ <use style="fill:#2ca02c;fill-opacity:0.2;stroke:#2ca02c;stroke-opacity:0.2;" x="184.695219" xlink:href="#mee8b134b61" y="38.964679"/>
+ <use style="fill:#2ca02c;fill-opacity:0.2;stroke:#2ca02c;stroke-opacity:0.2;" x="183.883566" xlink:href="#mee8b134b61" y="38.815857"/>
+ <use style="fill:#2ca02c;fill-opacity:0.2;stroke:#2ca02c;stroke-opacity:0.2;" x="183.06875" xlink:href="#mee8b134b61" y="38.667293"/>
+ <use style="fill:#2ca02c;fill-opacity:0.2;stroke:#2ca02c;stroke-opacity:0.2;" x="182.2507" xlink:href="#mee8b134b61" y="38.518959"/>
+ <use style="fill:#2ca02c;fill-opacity:0.2;stroke:#2ca02c;stroke-opacity:0.2;" x="181.429344" xlink:href="#mee8b134b61" y="38.370828"/>
+ <use style="fill:#2ca02c;fill-opacity:0.2;stroke:#2ca02c;stroke-opacity:0.2;" x="180.604612" xlink:href="#mee8b134b61" y="38.222874"/>
+ <use style="fill:#2ca02c;fill-opacity:0.2;stroke:#2ca02c;stroke-opacity:0.2;" x="179.776434" xlink:href="#mee8b134b61" y="38.075072"/>
+ <use style="fill:#2ca02c;fill-opacity:0.2;stroke:#2ca02c;stroke-opacity:0.2;" x="178.94474" xlink:href="#mee8b134b61" y="37.927396"/>
+ <use style="fill:#2ca02c;fill-opacity:0.2;stroke:#2ca02c;stroke-opacity:0.2;" x="178.109463" xlink:href="#mee8b134b61" y="37.779823"/>
+ <use style="fill:#2ca02c;fill-opacity:0.2;stroke:#2ca02c;stroke-opacity:0.2;" x="177.270533" xlink:href="#mee8b134b61" y="37.632328"/>
+ <use style="fill:#2ca02c;fill-opacity:0.2;stroke:#2ca02c;stroke-opacity:0.2;" x="176.427884" xlink:href="#mee8b134b61" y="37.484887"/>
+ <use style="fill:#2ca02c;fill-opacity:0.2;stroke:#2ca02c;stroke-opacity:0.2;" x="175.58145" xlink:href="#mee8b134b61" y="37.337478"/>
+ <use style="fill:#2ca02c;fill-opacity:0.2;stroke:#2ca02c;stroke-opacity:0.2;" x="174.731163" xlink:href="#mee8b134b61" y="37.190079"/>
+ <use style="fill:#2ca02c;fill-opacity:0.2;stroke:#2ca02c;stroke-opacity:0.2;" x="173.876959" xlink:href="#mee8b134b61" y="37.042668"/>
+ <use style="fill:#2ca02c;fill-opacity:0.2;stroke:#2ca02c;stroke-opacity:0.2;" x="173.018774" xlink:href="#mee8b134b61" y="36.895224"/>
+ <use style="fill:#2ca02c;fill-opacity:0.2;stroke:#2ca02c;stroke-opacity:0.2;" x="172.156542" xlink:href="#mee8b134b61" y="36.747726"/>
+ <use style="fill:#2ca02c;fill-opacity:0.2;stroke:#2ca02c;stroke-opacity:0.2;" x="171.2902" xlink:href="#mee8b134b61" y="36.600154"/>
+ <use style="fill:#2ca02c;fill-opacity:0.2;stroke:#2ca02c;stroke-opacity:0.2;" x="170.419685" xlink:href="#mee8b134b61" y="36.452489"/>
+ <use style="fill:#2ca02c;fill-opacity:0.2;stroke:#2ca02c;stroke-opacity:0.2;" x="169.544935" xlink:href="#mee8b134b61" y="36.304712"/>
+ <use style="fill:#2ca02c;fill-opacity:0.2;stroke:#2ca02c;stroke-opacity:0.2;" x="168.665888" xlink:href="#mee8b134b61" y="36.156804"/>
+ <use style="fill:#2ca02c;fill-opacity:0.2;stroke:#2ca02c;stroke-opacity:0.2;" x="167.782481" xlink:href="#mee8b134b61" y="36.008748"/>
+ <use style="fill:#2ca02c;fill-opacity:0.2;stroke:#2ca02c;stroke-opacity:0.2;" x="166.894654" xlink:href="#mee8b134b61" y="35.860527"/>
+ <use style="fill:#2ca02c;fill-opacity:0.2;stroke:#2ca02c;stroke-opacity:0.2;" x="166.002347" xlink:href="#mee8b134b61" y="35.712123"/>
+ <use style="fill:#2ca02c;fill-opacity:0.2;stroke:#2ca02c;stroke-opacity:0.2;" x="165.105498" xlink:href="#mee8b134b61" y="35.563522"/>
+ <use style="fill:#2ca02c;fill-opacity:0.2;stroke:#2ca02c;stroke-opacity:0.2;" x="164.204049" xlink:href="#mee8b134b61" y="35.414707"/>
+ <use style="fill:#2ca02c;fill-opacity:0.2;stroke:#2ca02c;stroke-opacity:0.2;" x="163.297939" xlink:href="#mee8b134b61" y="35.265663"/>
+ <use style="fill:#2ca02c;fill-opacity:0.2;stroke:#2ca02c;stroke-opacity:0.2;" x="162.387109" xlink:href="#mee8b134b61" y="35.116377"/>
+ <use style="fill:#2ca02c;fill-opacity:0.2;stroke:#2ca02c;stroke-opacity:0.2;" x="161.4715" xlink:href="#mee8b134b61" y="34.966834"/>
+ <use style="fill:#2ca02c;fill-opacity:0.2;stroke:#2ca02c;stroke-opacity:0.2;" x="160.551053" xlink:href="#mee8b134b61" y="34.81702"/>
+ <use style="fill:#2ca02c;fill-opacity:0.2;stroke:#2ca02c;stroke-opacity:0.2;" x="159.625709" xlink:href="#mee8b134b61" y="34.666924"/>
+ <use style="fill:#2ca02c;fill-opacity:0.2;stroke:#2ca02c;stroke-opacity:0.2;" x="158.695411" xlink:href="#mee8b134b61" y="34.516533"/>
+ <use style="fill:#2ca02c;fill-opacity:0.2;stroke:#2ca02c;stroke-opacity:0.2;" x="157.760099" xlink:href="#mee8b134b61" y="34.365835"/>
+ <use style="fill:#2ca02c;fill-opacity:0.2;stroke:#2ca02c;stroke-opacity:0.2;" x="156.819714" xlink:href="#mee8b134b61" y="34.21482"/>
+ <use style="fill:#2ca02c;fill-opacity:0.2;stroke:#2ca02c;stroke-opacity:0.2;" x="155.8742" xlink:href="#mee8b134b61" y="34.063476"/>
+ <use style="fill:#2ca02c;fill-opacity:0.2;stroke:#2ca02c;stroke-opacity:0.2;" x="154.923496" xlink:href="#mee8b134b61" y="33.911794"/>
+ <use style="fill:#2ca02c;fill-opacity:0.2;stroke:#2ca02c;stroke-opacity:0.2;" x="153.967545" xlink:href="#mee8b134b61" y="33.759763"/>
+ <use style="fill:#2ca02c;fill-opacity:0.2;stroke:#2ca02c;stroke-opacity:0.2;" x="153.006287" xlink:href="#mee8b134b61" y="33.607376"/>
+ <use style="fill:#2ca02c;fill-opacity:0.2;stroke:#2ca02c;stroke-opacity:0.2;" x="152.039665" xlink:href="#mee8b134b61" y="33.454623"/>
+ <use style="fill:#2ca02c;fill-opacity:0.2;stroke:#2ca02c;stroke-opacity:0.2;" x="151.067618" xlink:href="#mee8b134b61" y="33.301496"/>
+ <use style="fill:#2ca02c;fill-opacity:0.2;stroke:#2ca02c;stroke-opacity:0.2;" x="150.090087" xlink:href="#mee8b134b61" y="33.147987"/>
+ <use style="fill:#2ca02c;fill-opacity:0.2;stroke:#2ca02c;stroke-opacity:0.2;" x="149.107012" xlink:href="#mee8b134b61" y="32.99409"/>
+ <use style="fill:#2ca02c;fill-opacity:0.2;stroke:#2ca02c;stroke-opacity:0.2;" x="148.118326" xlink:href="#mee8b134b61" y="32.839822"/>
+ <use style="fill:#2ca02c;fill-opacity:0.2;stroke:#2ca02c;stroke-opacity:0.2;" x="147.123961" xlink:href="#mee8b134b61" y="32.685198"/>
+ <use style="fill:#2ca02c;fill-opacity:0.2;stroke:#2ca02c;stroke-opacity:0.2;" x="146.123854" xlink:href="#mee8b134b61" y="32.530224"/>
+ <use style="fill:#2ca02c;fill-opacity:0.2;stroke:#2ca02c;stroke-opacity:0.2;" x="145.117938" xlink:href="#mee8b134b61" y="32.37492"/>
+ <use style="fill:#2ca02c;fill-opacity:0.2;stroke:#2ca02c;stroke-opacity:0.2;" x="144.106146" xlink:href="#mee8b134b61" y="32.219301"/>
+ <use style="fill:#2ca02c;fill-opacity:0.2;stroke:#2ca02c;stroke-opacity:0.2;" x="143.088413" xlink:href="#mee8b134b61" y="32.063382"/>
+ <use style="fill:#2ca02c;fill-opacity:0.2;stroke:#2ca02c;stroke-opacity:0.2;" x="142.064675" xlink:href="#mee8b134b61" y="31.907178"/>
+ <use style="fill:#2ca02c;fill-opacity:0.2;stroke:#2ca02c;stroke-opacity:0.2;" x="141.034865" xlink:href="#mee8b134b61" y="31.750705"/>
+ <use style="fill:#2ca02c;fill-opacity:0.2;stroke:#2ca02c;stroke-opacity:0.2;" x="139.99892" xlink:href="#mee8b134b61" y="31.593979"/>
+ <use style="fill:#2ca02c;fill-opacity:0.2;stroke:#2ca02c;stroke-opacity:0.2;" x="138.956776" xlink:href="#mee8b134b61" y="31.437014"/>
+ <use style="fill:#2ca02c;fill-opacity:0.2;stroke:#2ca02c;stroke-opacity:0.2;" x="137.908368" xlink:href="#mee8b134b61" y="31.279828"/>
+ <use style="fill:#2ca02c;fill-opacity:0.2;stroke:#2ca02c;stroke-opacity:0.2;" x="136.853633" xlink:href="#mee8b134b61" y="31.122436"/>
+ <use style="fill:#2ca02c;fill-opacity:0.2;stroke:#2ca02c;stroke-opacity:0.2;" x="135.792509" xlink:href="#mee8b134b61" y="30.964853"/>
+ <use style="fill:#2ca02c;fill-opacity:0.2;stroke:#2ca02c;stroke-opacity:0.2;" x="134.724932" xlink:href="#mee8b134b61" y="30.807096"/>
+ <use style="fill:#2ca02c;fill-opacity:0.2;stroke:#2ca02c;stroke-opacity:0.2;" x="133.650841" xlink:href="#mee8b134b61" y="30.64918"/>
+ <use style="fill:#2ca02c;fill-opacity:0.2;stroke:#2ca02c;stroke-opacity:0.2;" x="132.570174" xlink:href="#mee8b134b61" y="30.49112"/>
+ <use style="fill:#2ca02c;fill-opacity:0.2;stroke:#2ca02c;stroke-opacity:0.2;" x="131.482869" xlink:href="#mee8b134b61" y="30.332932"/>
+ <use style="fill:#2ca02c;fill-opacity:0.2;stroke:#2ca02c;stroke-opacity:0.2;" x="130.388866" xlink:href="#mee8b134b61" y="30.174632"/>
+ <use style="fill:#2ca02c;fill-opacity:0.2;stroke:#2ca02c;stroke-opacity:0.2;" x="129.288105" xlink:href="#mee8b134b61" y="30.016234"/>
+ <use style="fill:#2ca02c;fill-opacity:0.2;stroke:#2ca02c;stroke-opacity:0.2;" x="128.180526" xlink:href="#mee8b134b61" y="29.857754"/>
+ <use style="fill:#2ca02c;fill-opacity:0.2;stroke:#2ca02c;stroke-opacity:0.2;" x="127.06607" xlink:href="#mee8b134b61" y="29.699207"/>
+ <use style="fill:#2ca02c;fill-opacity:0.2;stroke:#2ca02c;stroke-opacity:0.2;" x="125.944679" xlink:href="#mee8b134b61" y="29.540607"/>
+ <use style="fill:#2ca02c;fill-opacity:0.2;stroke:#2ca02c;stroke-opacity:0.2;" x="124.816295" xlink:href="#mee8b134b61" y="29.381968"/>
+ <use style="fill:#2ca02c;fill-opacity:0.2;stroke:#2ca02c;stroke-opacity:0.2;" x="123.680862" xlink:href="#mee8b134b61" y="29.223305"/>
+ <use style="fill:#2ca02c;fill-opacity:0.2;stroke:#2ca02c;stroke-opacity:0.2;" x="122.538322" xlink:href="#mee8b134b61" y="29.064631"/>
+ <use style="fill:#2ca02c;fill-opacity:0.2;stroke:#2ca02c;stroke-opacity:0.2;" x="121.388621" xlink:href="#mee8b134b61" y="28.905959"/>
+ <use style="fill:#2ca02c;fill-opacity:0.2;stroke:#2ca02c;stroke-opacity:0.2;" x="120.231704" xlink:href="#mee8b134b61" y="28.747303"/>
+ <use style="fill:#2ca02c;fill-opacity:0.2;stroke:#2ca02c;stroke-opacity:0.2;" x="119.067518" xlink:href="#mee8b134b61" y="28.588674"/>
+ <use style="fill:#2ca02c;fill-opacity:0.2;stroke:#2ca02c;stroke-opacity:0.2;" x="117.896009" xlink:href="#mee8b134b61" y="28.430086"/>
+ <use style="fill:#2ca02c;fill-opacity:0.2;stroke:#2ca02c;stroke-opacity:0.2;" x="116.717127" xlink:href="#mee8b134b61" y="28.27155"/>
+ <use style="fill:#2ca02c;fill-opacity:0.2;stroke:#2ca02c;stroke-opacity:0.2;" x="115.53082" xlink:href="#mee8b134b61" y="28.113076"/>
+ <use style="fill:#2ca02c;fill-opacity:0.2;stroke:#2ca02c;stroke-opacity:0.2;" x="114.33704" xlink:href="#mee8b134b61" y="27.954676"/>
+ <use style="fill:#2ca02c;fill-opacity:0.2;stroke:#2ca02c;stroke-opacity:0.2;" x="113.135738" xlink:href="#mee8b134b61" y="27.79636"/>
+ <use style="fill:#2ca02c;fill-opacity:0.2;stroke:#2ca02c;stroke-opacity:0.2;" x="111.926868" xlink:href="#mee8b134b61" y="27.638137"/>
+ <use style="fill:#2ca02c;fill-opacity:0.2;stroke:#2ca02c;stroke-opacity:0.2;" x="110.710383" xlink:href="#mee8b134b61" y="27.480016"/>
+ <use style="fill:#2ca02c;fill-opacity:0.2;stroke:#2ca02c;stroke-opacity:0.2;" x="109.48624" xlink:href="#mee8b134b61" y="27.322005"/>
+ <use style="fill:#2ca02c;fill-opacity:0.2;stroke:#2ca02c;stroke-opacity:0.2;" x="108.254397" xlink:href="#mee8b134b61" y="27.164113"/>
+ <use style="fill:#2ca02c;fill-opacity:0.2;stroke:#2ca02c;stroke-opacity:0.2;" x="107.014812" xlink:href="#mee8b134b61" y="27.006347"/>
+ <use style="fill:#2ca02c;fill-opacity:0.2;stroke:#2ca02c;stroke-opacity:0.2;" x="105.767447" xlink:href="#mee8b134b61" y="26.848712"/>
+ <use style="fill:#2ca02c;fill-opacity:0.2;stroke:#2ca02c;stroke-opacity:0.2;" x="104.512265" xlink:href="#mee8b134b61" y="26.691215"/>
+ <use style="fill:#2ca02c;fill-opacity:0.2;stroke:#2ca02c;stroke-opacity:0.2;" x="103.249229" xlink:href="#mee8b134b61" y="26.533861"/>
+ <use style="fill:#2ca02c;fill-opacity:0.2;stroke:#2ca02c;stroke-opacity:0.2;" x="101.978307" xlink:href="#mee8b134b61" y="26.376653"/>
+ <use style="fill:#2ca02c;fill-opacity:0.2;stroke:#2ca02c;stroke-opacity:0.2;" x="100.699467" xlink:href="#mee8b134b61" y="26.219597"/>
+ <use style="fill:#2ca02c;fill-opacity:0.2;stroke:#2ca02c;stroke-opacity:0.2;" x="99.41268" xlink:href="#mee8b134b61" y="26.062694"/>
+ <use style="fill:#2ca02c;fill-opacity:0.2;stroke:#2ca02c;stroke-opacity:0.2;" x="98.11792" xlink:href="#mee8b134b61" y="25.905946"/>
+ <use style="fill:#2ca02c;fill-opacity:0.2;stroke:#2ca02c;stroke-opacity:0.2;" x="96.815162" xlink:href="#mee8b134b61" y="25.749356"/>
+ <use style="fill:#2ca02c;fill-opacity:0.2;stroke:#2ca02c;stroke-opacity:0.2;" x="95.504384" xlink:href="#mee8b134b61" y="25.592924"/>
+ <use style="fill:#2ca02c;fill-opacity:0.2;stroke:#2ca02c;stroke-opacity:0.2;" x="94.185567" xlink:href="#mee8b134b61" y="25.436648"/>
+ <use style="fill:#2ca02c;fill-opacity:0.2;stroke:#2ca02c;stroke-opacity:0.2;" x="92.858695" xlink:href="#mee8b134b61" y="25.28053"/>
+ <use style="fill:#2ca02c;fill-opacity:0.2;stroke:#2ca02c;stroke-opacity:0.2;" x="91.523754" xlink:href="#mee8b134b61" y="25.124566"/>
+ <use style="fill:#2ca02c;fill-opacity:0.2;stroke:#2ca02c;stroke-opacity:0.2;" x="90.180732" xlink:href="#mee8b134b61" y="24.968755"/>
+ <use style="fill:#2ca02c;fill-opacity:0.2;stroke:#2ca02c;stroke-opacity:0.2;" x="88.829623" xlink:href="#mee8b134b61" y="24.813093"/>
+ <use style="fill:#2ca02c;fill-opacity:0.2;stroke:#2ca02c;stroke-opacity:0.2;" x="87.470421" xlink:href="#mee8b134b61" y="24.657576"/>
+ <use style="fill:#2ca02c;fill-opacity:0.2;stroke:#2ca02c;stroke-opacity:0.2;" x="86.103126" xlink:href="#mee8b134b61" y="24.502198"/>
+ <use style="fill:#2ca02c;fill-opacity:0.2;stroke:#2ca02c;stroke-opacity:0.2;" x="84.727739" xlink:href="#mee8b134b61" y="24.346956"/>
+ <use style="fill:#2ca02c;fill-opacity:0.2;stroke:#2ca02c;stroke-opacity:0.2;" x="83.344266" xlink:href="#mee8b134b61" y="24.191841"/>
+ <use style="fill:#2ca02c;fill-opacity:0.2;stroke:#2ca02c;stroke-opacity:0.2;" x="81.952714" xlink:href="#mee8b134b61" y="24.036846"/>
+ </g>
+ </g>
+ <g id="line2d_14">
+ <path clip-path="url(#p444e95376c)" d="M 369.97072 288.76125
+L 378.217139 284.547929
+L 386.100185 280.181013
+L 393.564916 275.638159
+L 397.1232 273.294339
+L 400.556055 270.89915
+L 403.856182 268.450503
+L 407.015959 265.946541
+L 410.02736 263.38568
+L 412.881919 260.766652
+L 415.570732 258.088549
+L 418.084536 255.350858
+L 420.413853 252.553494
+L 422.549221 249.696807
+L 424.481471 246.781578
+L 426.202062 243.808992
+L 427.703402 240.780587
+L 428.979134 237.698187
+L 430.024318 234.563833
+L 430.835369 231.37972
+L 431.409701 228.14822
+L 431.746979 224.87166
+L 431.84878 221.552317
+L 431.718701 218.192388
+L 431.362228 214.794082
+L 430.78654 211.359574
+L 430.000073 207.89106
+L 429.012031 204.390823
+L 427.831884 200.86128
+L 426.468895 197.305025
+L 424.931731 193.72486
+L 423.228193 190.123808
+L 421.365077 186.505105
+L 419.348151 182.872176
+L 417.182234 179.228608
+L 414.871452 175.578129
+L 412.42042 171.924661
+L 409.832975 168.272219
+L 404.262633 160.98679
+L 398.187032 153.755048
+L 391.634145 146.61025
+L 384.639204 139.584782
+L 377.248722 132.709132
+L 369.521564 126.010942
+L 361.527594 119.514348
+L 353.340054 113.240821
+L 345.027129 107.209974
+L 336.646238 101.439844
+L 328.242811 95.946643
+L 319.852907 90.744209
+L 311.507656 85.843343
+L 303.237026 81.251176
+L 295.071238 76.970688
+L 287.03972 73.000742
+L 279.168646 69.336549
+L 271.478063 65.969862
+L 263.980814 62.889047
+L 256.68201 60.079269
+L 249.578813 57.522906
+L 242.660497 55.200142
+L 232.588108 52.107023
+L 222.800806 49.415926
+L 213.18347 47.050739
+L 203.601814 44.937441
+L 193.90997 43.006511
+L 180.558175 40.608673
+L 166.406267 38.303197
+L 147.13944 35.428149
+L 125.673487 32.496755
+L 106.642225 30.111477
+L 85.769209 27.707352
+L 68.842656 25.898847
+L 68.842656 25.898847
+" style="fill:none;stroke:#d62728;stroke-dasharray:6,4.5,6;stroke-dashoffset:0;stroke-opacity:0.2;stroke-width:1.5;"/>
+ <defs>
+ <path d="M -3 3
+L 3 -3
+M -3 -3
+L 3 3
+" id="m6d92a482c2" style="stroke:#d62728;stroke-opacity:0.2;"/>
+ </defs>
+ <g clip-path="url(#p444e95376c)">
+ <use style="fill:#d62728;fill-opacity:0.2;stroke:#d62728;stroke-opacity:0.2;" x="369.97072" xlink:href="#m6d92a482c2" y="288.76125"/>
+ <use style="fill:#d62728;fill-opacity:0.2;stroke:#d62728;stroke-opacity:0.2;" x="374.135905" xlink:href="#m6d92a482c2" y="286.672322"/>
+ <use style="fill:#d62728;fill-opacity:0.2;stroke:#d62728;stroke-opacity:0.2;" x="378.217139" xlink:href="#m6d92a482c2" y="284.547929"/>
+ <use style="fill:#d62728;fill-opacity:0.2;stroke:#d62728;stroke-opacity:0.2;" x="382.207523" xlink:href="#m6d92a482c2" y="282.385111"/>
+ <use style="fill:#d62728;fill-opacity:0.2;stroke:#d62728;stroke-opacity:0.2;" x="386.100185" xlink:href="#m6d92a482c2" y="280.181013"/>
+ <use style="fill:#d62728;fill-opacity:0.2;stroke:#d62728;stroke-opacity:0.2;" x="389.888269" xlink:href="#m6d92a482c2" y="277.932898"/>
+ <use style="fill:#d62728;fill-opacity:0.2;stroke:#d62728;stroke-opacity:0.2;" x="393.564916" xlink:href="#m6d92a482c2" y="275.638159"/>
+ <use style="fill:#d62728;fill-opacity:0.2;stroke:#d62728;stroke-opacity:0.2;" x="397.1232" xlink:href="#m6d92a482c2" y="273.294339"/>
+ <use style="fill:#d62728;fill-opacity:0.2;stroke:#d62728;stroke-opacity:0.2;" x="400.556055" xlink:href="#m6d92a482c2" y="270.89915"/>
+ <use style="fill:#d62728;fill-opacity:0.2;stroke:#d62728;stroke-opacity:0.2;" x="403.856182" xlink:href="#m6d92a482c2" y="268.450503"/>
+ <use style="fill:#d62728;fill-opacity:0.2;stroke:#d62728;stroke-opacity:0.2;" x="407.015959" xlink:href="#m6d92a482c2" y="265.946541"/>
+ <use style="fill:#d62728;fill-opacity:0.2;stroke:#d62728;stroke-opacity:0.2;" x="410.02736" xlink:href="#m6d92a482c2" y="263.38568"/>
+ <use style="fill:#d62728;fill-opacity:0.2;stroke:#d62728;stroke-opacity:0.2;" x="412.881919" xlink:href="#m6d92a482c2" y="260.766652"/>
+ <use style="fill:#d62728;fill-opacity:0.2;stroke:#d62728;stroke-opacity:0.2;" x="415.570732" xlink:href="#m6d92a482c2" y="258.088549"/>
+ <use style="fill:#d62728;fill-opacity:0.2;stroke:#d62728;stroke-opacity:0.2;" x="418.084536" xlink:href="#m6d92a482c2" y="255.350858"/>
+ <use style="fill:#d62728;fill-opacity:0.2;stroke:#d62728;stroke-opacity:0.2;" x="420.413853" xlink:href="#m6d92a482c2" y="252.553494"/>
+ <use style="fill:#d62728;fill-opacity:0.2;stroke:#d62728;stroke-opacity:0.2;" x="422.549221" xlink:href="#m6d92a482c2" y="249.696807"/>
+ <use style="fill:#d62728;fill-opacity:0.2;stroke:#d62728;stroke-opacity:0.2;" x="424.481471" xlink:href="#m6d92a482c2" y="246.781578"/>
+ <use style="fill:#d62728;fill-opacity:0.2;stroke:#d62728;stroke-opacity:0.2;" x="426.202062" xlink:href="#m6d92a482c2" y="243.808992"/>
+ <use style="fill:#d62728;fill-opacity:0.2;stroke:#d62728;stroke-opacity:0.2;" x="427.703402" xlink:href="#m6d92a482c2" y="240.780587"/>
+ <use style="fill:#d62728;fill-opacity:0.2;stroke:#d62728;stroke-opacity:0.2;" x="428.979134" xlink:href="#m6d92a482c2" y="237.698187"/>
+ <use style="fill:#d62728;fill-opacity:0.2;stroke:#d62728;stroke-opacity:0.2;" x="430.024318" xlink:href="#m6d92a482c2" y="234.563833"/>
+ <use style="fill:#d62728;fill-opacity:0.2;stroke:#d62728;stroke-opacity:0.2;" x="430.835369" xlink:href="#m6d92a482c2" y="231.37972"/>
+ <use style="fill:#d62728;fill-opacity:0.2;stroke:#d62728;stroke-opacity:0.2;" x="431.409701" xlink:href="#m6d92a482c2" y="228.14822"/>
+ <use style="fill:#d62728;fill-opacity:0.2;stroke:#d62728;stroke-opacity:0.2;" x="431.746979" xlink:href="#m6d92a482c2" y="224.87166"/>
+ <use style="fill:#d62728;fill-opacity:0.2;stroke:#d62728;stroke-opacity:0.2;" x="431.84878" xlink:href="#m6d92a482c2" y="221.552317"/>
+ <use style="fill:#d62728;fill-opacity:0.2;stroke:#d62728;stroke-opacity:0.2;" x="431.718701" xlink:href="#m6d92a482c2" y="218.192388"/>
+ <use style="fill:#d62728;fill-opacity:0.2;stroke:#d62728;stroke-opacity:0.2;" x="431.362228" xlink:href="#m6d92a482c2" y="214.794082"/>
+ <use style="fill:#d62728;fill-opacity:0.2;stroke:#d62728;stroke-opacity:0.2;" x="430.78654" xlink:href="#m6d92a482c2" y="211.359574"/>
+ <use style="fill:#d62728;fill-opacity:0.2;stroke:#d62728;stroke-opacity:0.2;" x="430.000073" xlink:href="#m6d92a482c2" y="207.89106"/>
+ <use style="fill:#d62728;fill-opacity:0.2;stroke:#d62728;stroke-opacity:0.2;" x="429.012031" xlink:href="#m6d92a482c2" y="204.390823"/>
+ <use style="fill:#d62728;fill-opacity:0.2;stroke:#d62728;stroke-opacity:0.2;" x="427.831884" xlink:href="#m6d92a482c2" y="200.86128"/>
+ <use style="fill:#d62728;fill-opacity:0.2;stroke:#d62728;stroke-opacity:0.2;" x="426.468895" xlink:href="#m6d92a482c2" y="197.305025"/>
+ <use style="fill:#d62728;fill-opacity:0.2;stroke:#d62728;stroke-opacity:0.2;" x="424.931731" xlink:href="#m6d92a482c2" y="193.72486"/>
+ <use style="fill:#d62728;fill-opacity:0.2;stroke:#d62728;stroke-opacity:0.2;" x="423.228193" xlink:href="#m6d92a482c2" y="190.123808"/>
+ <use style="fill:#d62728;fill-opacity:0.2;stroke:#d62728;stroke-opacity:0.2;" x="421.365077" xlink:href="#m6d92a482c2" y="186.505105"/>
+ <use style="fill:#d62728;fill-opacity:0.2;stroke:#d62728;stroke-opacity:0.2;" x="419.348151" xlink:href="#m6d92a482c2" y="182.872176"/>
+ <use style="fill:#d62728;fill-opacity:0.2;stroke:#d62728;stroke-opacity:0.2;" x="417.182234" xlink:href="#m6d92a482c2" y="179.228608"/>
+ <use style="fill:#d62728;fill-opacity:0.2;stroke:#d62728;stroke-opacity:0.2;" x="414.871452" xlink:href="#m6d92a482c2" y="175.578129"/>
+ <use style="fill:#d62728;fill-opacity:0.2;stroke:#d62728;stroke-opacity:0.2;" x="412.42042" xlink:href="#m6d92a482c2" y="171.924661"/>
+ <use style="fill:#d62728;fill-opacity:0.2;stroke:#d62728;stroke-opacity:0.2;" x="409.832975" xlink:href="#m6d92a482c2" y="168.272219"/>
+ <use style="fill:#d62728;fill-opacity:0.2;stroke:#d62728;stroke-opacity:0.2;" x="407.112588" xlink:href="#m6d92a482c2" y="164.624881"/>
+ <use style="fill:#d62728;fill-opacity:0.2;stroke:#d62728;stroke-opacity:0.2;" x="404.262633" xlink:href="#m6d92a482c2" y="160.98679"/>
+ <use style="fill:#d62728;fill-opacity:0.2;stroke:#d62728;stroke-opacity:0.2;" x="401.286346" xlink:href="#m6d92a482c2" y="157.362121"/>
+ <use style="fill:#d62728;fill-opacity:0.2;stroke:#d62728;stroke-opacity:0.2;" x="398.187032" xlink:href="#m6d92a482c2" y="153.755048"/>
+ <use style="fill:#d62728;fill-opacity:0.2;stroke:#d62728;stroke-opacity:0.2;" x="394.968279" xlink:href="#m6d92a482c2" y="150.169725"/>
+ <use style="fill:#d62728;fill-opacity:0.2;stroke:#d62728;stroke-opacity:0.2;" x="391.634145" xlink:href="#m6d92a482c2" y="146.61025"/>
+ <use style="fill:#d62728;fill-opacity:0.2;stroke:#d62728;stroke-opacity:0.2;" x="388.189315" xlink:href="#m6d92a482c2" y="143.080638"/>
+ <use style="fill:#d62728;fill-opacity:0.2;stroke:#d62728;stroke-opacity:0.2;" x="384.639204" xlink:href="#m6d92a482c2" y="139.584782"/>
+ <use style="fill:#d62728;fill-opacity:0.2;stroke:#d62728;stroke-opacity:0.2;" x="380.99001" xlink:href="#m6d92a482c2" y="136.126426"/>
+ <use style="fill:#d62728;fill-opacity:0.2;stroke:#d62728;stroke-opacity:0.2;" x="377.248722" xlink:href="#m6d92a482c2" y="132.709132"/>
+ <use style="fill:#d62728;fill-opacity:0.2;stroke:#d62728;stroke-opacity:0.2;" x="373.423086" xlink:href="#m6d92a482c2" y="129.33626"/>
+ <use style="fill:#d62728;fill-opacity:0.2;stroke:#d62728;stroke-opacity:0.2;" x="369.521564" xlink:href="#m6d92a482c2" y="126.010942"/>
+ <use style="fill:#d62728;fill-opacity:0.2;stroke:#d62728;stroke-opacity:0.2;" x="365.553273" xlink:href="#m6d92a482c2" y="122.736067"/>
+ <use style="fill:#d62728;fill-opacity:0.2;stroke:#d62728;stroke-opacity:0.2;" x="361.527594" xlink:href="#m6d92a482c2" y="119.514348"/>
+ <use style="fill:#d62728;fill-opacity:0.2;stroke:#d62728;stroke-opacity:0.2;" x="357.453556" xlink:href="#m6d92a482c2" y="116.348437"/>
+ <use style="fill:#d62728;fill-opacity:0.2;stroke:#d62728;stroke-opacity:0.2;" x="353.340054" xlink:href="#m6d92a482c2" y="113.240821"/>
+ <use style="fill:#d62728;fill-opacity:0.2;stroke:#d62728;stroke-opacity:0.2;" x="349.195388" xlink:href="#m6d92a482c2" y="110.193897"/>
+ <use style="fill:#d62728;fill-opacity:0.2;stroke:#d62728;stroke-opacity:0.2;" x="345.027129" xlink:href="#m6d92a482c2" y="107.209974"/>
+ <use style="fill:#d62728;fill-opacity:0.2;stroke:#d62728;stroke-opacity:0.2;" x="340.842074" xlink:href="#m6d92a482c2" y="104.291258"/>
+ <use style="fill:#d62728;fill-opacity:0.2;stroke:#d62728;stroke-opacity:0.2;" x="336.646238" xlink:href="#m6d92a482c2" y="101.439844"/>
+ <use style="fill:#d62728;fill-opacity:0.2;stroke:#d62728;stroke-opacity:0.2;" x="332.444918" xlink:href="#m6d92a482c2" y="98.657699"/>
+ <use style="fill:#d62728;fill-opacity:0.2;stroke:#d62728;stroke-opacity:0.2;" x="328.242811" xlink:href="#m6d92a482c2" y="95.946643"/>
+ <use style="fill:#d62728;fill-opacity:0.2;stroke:#d62728;stroke-opacity:0.2;" x="324.044161" xlink:href="#m6d92a482c2" y="93.308326"/>
+ <use style="fill:#d62728;fill-opacity:0.2;stroke:#d62728;stroke-opacity:0.2;" x="319.852907" xlink:href="#m6d92a482c2" y="90.744209"/>
+ <use style="fill:#d62728;fill-opacity:0.2;stroke:#d62728;stroke-opacity:0.2;" x="315.672829" xlink:href="#m6d92a482c2" y="88.255542"/>
+ <use style="fill:#d62728;fill-opacity:0.2;stroke:#d62728;stroke-opacity:0.2;" x="311.507656" xlink:href="#m6d92a482c2" y="85.843343"/>
+ <use style="fill:#d62728;fill-opacity:0.2;stroke:#d62728;stroke-opacity:0.2;" x="307.361128" xlink:href="#m6d92a482c2" y="83.508384"/>
+ <use style="fill:#d62728;fill-opacity:0.2;stroke:#d62728;stroke-opacity:0.2;" x="303.237026" xlink:href="#m6d92a482c2" y="81.251176"/>
+ <use style="fill:#d62728;fill-opacity:0.2;stroke:#d62728;stroke-opacity:0.2;" x="299.139145" xlink:href="#m6d92a482c2" y="79.071957"/>
+ <use style="fill:#d62728;fill-opacity:0.2;stroke:#d62728;stroke-opacity:0.2;" x="295.071238" xlink:href="#m6d92a482c2" y="76.970688"/>
+ <use style="fill:#d62728;fill-opacity:0.2;stroke:#d62728;stroke-opacity:0.2;" x="291.036946" xlink:href="#m6d92a482c2" y="74.94711"/>
+ <use style="fill:#d62728;fill-opacity:0.2;stroke:#d62728;stroke-opacity:0.2;" x="287.03972" xlink:href="#m6d92a482c2" y="73.000742"/>
+ <use style="fill:#d62728;fill-opacity:0.2;stroke:#d62728;stroke-opacity:0.2;" x="283.082706" xlink:href="#m6d92a482c2" y="71.130863"/>
+ <use style="fill:#d62728;fill-opacity:0.2;stroke:#d62728;stroke-opacity:0.2;" x="279.168646" xlink:href="#m6d92a482c2" y="69.336549"/>
+ <use style="fill:#d62728;fill-opacity:0.2;stroke:#d62728;stroke-opacity:0.2;" x="275.299816" xlink:href="#m6d92a482c2" y="67.616666"/>
+ <use style="fill:#d62728;fill-opacity:0.2;stroke:#d62728;stroke-opacity:0.2;" x="271.478063" xlink:href="#m6d92a482c2" y="65.969862"/>
+ <use style="fill:#d62728;fill-opacity:0.2;stroke:#d62728;stroke-opacity:0.2;" x="267.704767" xlink:href="#m6d92a482c2" y="64.394577"/>
+ <use style="fill:#d62728;fill-opacity:0.2;stroke:#d62728;stroke-opacity:0.2;" x="263.980814" xlink:href="#m6d92a482c2" y="62.889047"/>
+ <use style="fill:#d62728;fill-opacity:0.2;stroke:#d62728;stroke-opacity:0.2;" x="260.306596" xlink:href="#m6d92a482c2" y="61.451321"/>
+ <use style="fill:#d62728;fill-opacity:0.2;stroke:#d62728;stroke-opacity:0.2;" x="256.68201" xlink:href="#m6d92a482c2" y="60.079269"/>
+ <use style="fill:#d62728;fill-opacity:0.2;stroke:#d62728;stroke-opacity:0.2;" x="253.106449" xlink:href="#m6d92a482c2" y="58.770605"/>
+ <use style="fill:#d62728;fill-opacity:0.2;stroke:#d62728;stroke-opacity:0.2;" x="249.578813" xlink:href="#m6d92a482c2" y="57.522906"/>
+ <use style="fill:#d62728;fill-opacity:0.2;stroke:#d62728;stroke-opacity:0.2;" x="246.097513" xlink:href="#m6d92a482c2" y="56.33363"/>
+ <use style="fill:#d62728;fill-opacity:0.2;stroke:#d62728;stroke-opacity:0.2;" x="242.660497" xlink:href="#m6d92a482c2" y="55.200142"/>
+ <use style="fill:#d62728;fill-opacity:0.2;stroke:#d62728;stroke-opacity:0.2;" x="239.265267" xlink:href="#m6d92a482c2" y="54.11973"/>
+ <use style="fill:#d62728;fill-opacity:0.2;stroke:#d62728;stroke-opacity:0.2;" x="235.908906" xlink:href="#m6d92a482c2" y="53.089625"/>
+ <use style="fill:#d62728;fill-opacity:0.2;stroke:#d62728;stroke-opacity:0.2;" x="232.588108" xlink:href="#m6d92a482c2" y="52.107023"/>
+ <use style="fill:#d62728;fill-opacity:0.2;stroke:#d62728;stroke-opacity:0.2;" x="229.299206" xlink:href="#m6d92a482c2" y="51.169095"/>
+ <use style="fill:#d62728;fill-opacity:0.2;stroke:#d62728;stroke-opacity:0.2;" x="226.038204" xlink:href="#m6d92a482c2" y="50.273006"/>
+ <use style="fill:#d62728;fill-opacity:0.2;stroke:#d62728;stroke-opacity:0.2;" x="222.800806" xlink:href="#m6d92a482c2" y="49.415926"/>
+ <use style="fill:#d62728;fill-opacity:0.2;stroke:#d62728;stroke-opacity:0.2;" x="219.582452" xlink:href="#m6d92a482c2" y="48.595041"/>
+ <use style="fill:#d62728;fill-opacity:0.2;stroke:#d62728;stroke-opacity:0.2;" x="216.378341" xlink:href="#m6d92a482c2" y="47.807562"/>
+ <use style="fill:#d62728;fill-opacity:0.2;stroke:#d62728;stroke-opacity:0.2;" x="213.18347" xlink:href="#m6d92a482c2" y="47.050739"/>
+ <use style="fill:#d62728;fill-opacity:0.2;stroke:#d62728;stroke-opacity:0.2;" x="209.99266" xlink:href="#m6d92a482c2" y="46.321867"/>
+ <use style="fill:#d62728;fill-opacity:0.2;stroke:#d62728;stroke-opacity:0.2;" x="206.800586" xlink:href="#m6d92a482c2" y="45.618295"/>
+ <use style="fill:#d62728;fill-opacity:0.2;stroke:#d62728;stroke-opacity:0.2;" x="203.601814" xlink:href="#m6d92a482c2" y="44.937441"/>
+ <use style="fill:#d62728;fill-opacity:0.2;stroke:#d62728;stroke-opacity:0.2;" x="200.390829" xlink:href="#m6d92a482c2" y="44.276792"/>
+ <use style="fill:#d62728;fill-opacity:0.2;stroke:#d62728;stroke-opacity:0.2;" x="197.162072" xlink:href="#m6d92a482c2" y="43.633926"/>
+ <use style="fill:#d62728;fill-opacity:0.2;stroke:#d62728;stroke-opacity:0.2;" x="193.90997" xlink:href="#m6d92a482c2" y="43.006511"/>
+ <use style="fill:#d62728;fill-opacity:0.2;stroke:#d62728;stroke-opacity:0.2;" x="190.628974" xlink:href="#m6d92a482c2" y="42.392324"/>
+ <use style="fill:#d62728;fill-opacity:0.2;stroke:#d62728;stroke-opacity:0.2;" x="187.313592" xlink:href="#m6d92a482c2" y="41.789255"/>
+ <use style="fill:#d62728;fill-opacity:0.2;stroke:#d62728;stroke-opacity:0.2;" x="183.95842" xlink:href="#m6d92a482c2" y="41.195321"/>
+ <use style="fill:#d62728;fill-opacity:0.2;stroke:#d62728;stroke-opacity:0.2;" x="180.558175" xlink:href="#m6d92a482c2" y="40.608673"/>
+ <use style="fill:#d62728;fill-opacity:0.2;stroke:#d62728;stroke-opacity:0.2;" x="177.107717" xlink:href="#m6d92a482c2" y="40.027608"/>
+ <use style="fill:#d62728;fill-opacity:0.2;stroke:#d62728;stroke-opacity:0.2;" x="173.602074" xlink:href="#m6d92a482c2" y="39.450575"/>
+ <use style="fill:#d62728;fill-opacity:0.2;stroke:#d62728;stroke-opacity:0.2;" x="170.036456" xlink:href="#m6d92a482c2" y="38.87618"/>
+ <use style="fill:#d62728;fill-opacity:0.2;stroke:#d62728;stroke-opacity:0.2;" x="166.406267" xlink:href="#m6d92a482c2" y="38.303197"/>
+ <use style="fill:#d62728;fill-opacity:0.2;stroke:#d62728;stroke-opacity:0.2;" x="162.707105" xlink:href="#m6d92a482c2" y="37.730565"/>
+ <use style="fill:#d62728;fill-opacity:0.2;stroke:#d62728;stroke-opacity:0.2;" x="158.934765" xlink:href="#m6d92a482c2" y="37.15739"/>
+ <use style="fill:#d62728;fill-opacity:0.2;stroke:#d62728;stroke-opacity:0.2;" x="155.085234" xlink:href="#m6d92a482c2" y="36.582947"/>
+ <use style="fill:#d62728;fill-opacity:0.2;stroke:#d62728;stroke-opacity:0.2;" x="151.154678" xlink:href="#m6d92a482c2" y="36.006669"/>
+ <use style="fill:#d62728;fill-opacity:0.2;stroke:#d62728;stroke-opacity:0.2;" x="147.13944" xlink:href="#m6d92a482c2" y="35.428149"/>
+ <use style="fill:#d62728;fill-opacity:0.2;stroke:#d62728;stroke-opacity:0.2;" x="143.03602" xlink:href="#m6d92a482c2" y="34.84712"/>
+ <use style="fill:#d62728;fill-opacity:0.2;stroke:#d62728;stroke-opacity:0.2;" x="138.840998" xlink:href="#m6d92a482c2" y="34.263444"/>
+ <use style="fill:#d62728;fill-opacity:0.2;stroke:#d62728;stroke-opacity:0.2;" x="134.55105" xlink:href="#m6d92a482c2" y="33.677096"/>
+ <use style="fill:#d62728;fill-opacity:0.2;stroke:#d62728;stroke-opacity:0.2;" x="130.162936" xlink:href="#m6d92a482c2" y="33.088149"/>
+ <use style="fill:#d62728;fill-opacity:0.2;stroke:#d62728;stroke-opacity:0.2;" x="125.673487" xlink:href="#m6d92a482c2" y="32.496755"/>
+ <use style="fill:#d62728;fill-opacity:0.2;stroke:#d62728;stroke-opacity:0.2;" x="121.079614" xlink:href="#m6d92a482c2" y="31.903126"/>
+ <use style="fill:#d62728;fill-opacity:0.2;stroke:#d62728;stroke-opacity:0.2;" x="116.378327" xlink:href="#m6d92a482c2" y="31.307517"/>
+ <use style="fill:#d62728;fill-opacity:0.2;stroke:#d62728;stroke-opacity:0.2;" x="111.566764" xlink:href="#m6d92a482c2" y="30.710206"/>
+ <use style="fill:#d62728;fill-opacity:0.2;stroke:#d62728;stroke-opacity:0.2;" x="106.642225" xlink:href="#m6d92a482c2" y="30.111477"/>
+ <use style="fill:#d62728;fill-opacity:0.2;stroke:#d62728;stroke-opacity:0.2;" x="101.602226" xlink:href="#m6d92a482c2" y="29.511602"/>
+ <use style="fill:#d62728;fill-opacity:0.2;stroke:#d62728;stroke-opacity:0.2;" x="96.444561" xlink:href="#m6d92a482c2" y="28.910827"/>
+ <use style="fill:#d62728;fill-opacity:0.2;stroke:#d62728;stroke-opacity:0.2;" x="91.167367" xlink:href="#m6d92a482c2" y="28.309357"/>
+ <use style="fill:#d62728;fill-opacity:0.2;stroke:#d62728;stroke-opacity:0.2;" x="85.769209" xlink:href="#m6d92a482c2" y="27.707352"/>
+ <use style="fill:#d62728;fill-opacity:0.2;stroke:#d62728;stroke-opacity:0.2;" x="80.249156" xlink:href="#m6d92a482c2" y="27.104913"/>
+ <use style="fill:#d62728;fill-opacity:0.2;stroke:#d62728;stroke-opacity:0.2;" x="74.606865" xlink:href="#m6d92a482c2" y="26.502084"/>
+ <use style="fill:#d62728;fill-opacity:0.2;stroke:#d62728;stroke-opacity:0.2;" x="68.842656" xlink:href="#m6d92a482c2" y="25.898847"/>
+ </g>
+ </g>
+ <g id="line2d_15">
+ <path clip-path="url(#p444e95376c)" d="M 369.97072 288.76125
+L 378.217168 284.547948
+L 384.16654 281.288426
+L 389.888306 277.932939
+L 395.359299 274.472586
+L 400.556016 270.899221
+L 405.454012 267.205635
+L 410.027221 263.385799
+L 412.881766 260.766789
+L 415.570586 258.088703
+L 418.08442 255.351027
+L 420.413797 252.553673
+L 422.549249 249.69699
+L 424.481605 246.781758
+L 426.202312 243.80916
+L 427.703764 240.780735
+L 428.979586 237.698309
+L 430.024817 234.563926
+L 430.835793 231.379801
+L 431.410243 228.148263
+L 431.747646 224.871661
+L 431.849508 221.552286
+L 431.719405 218.192354
+L 431.362854 214.794048
+L 430.787035 211.359538
+L 430.000393 207.891028
+L 429.01215 204.390798
+L 427.831795 200.861263
+L 426.468613 197.305012
+L 424.099799 191.926732
+L 421.364431 186.505073
+L 418.282858 181.051435
+L 414.870956 175.578058
+L 411.142964 170.097965
+L 407.111927 164.624759
+L 402.789428 159.172372
+L 398.186443 153.754877
+L 393.314834 148.386301
+L 388.18883 143.080403
+L 382.826084 137.850423
+L 377.248182 132.70884
+L 371.480635 127.667161
+L 365.552527 122.735755
+L 359.495025 117.923985
+L 351.269854 111.709368
+L 342.93486 105.742088
+L 334.544444 100.039761
+L 326.141306 94.618065
+L 317.759785 89.490144
+L 309.430387 84.665939
+L 301.183091 80.151584
+L 293.048101 75.949
+L 285.054326 72.056094
+L 277.226666 68.467191
+L 269.583452 65.173197
+L 262.135583 62.161657
+L 253.104482 58.770439
+L 244.371589 55.759929
+L 235.90682 53.089485
+L 227.663327 50.715865
+L 219.580221 48.59491
+L 209.990325 46.321737
+L 200.388395 44.276665
+L 188.973399 42.089402
+L 175.359523 39.738555
+L 158.932179 37.157261
+L 140.947593 34.555477
+L 121.077114 31.902992
+L 101.599892 29.511504
+L 80.247082 27.104891
+L 68.840733 25.898877
+L 68.840733 25.898877
+" style="fill:none;stroke:#9467bd;stroke-dasharray:3,4.5,3;stroke-dashoffset:0;stroke-opacity:0.2;stroke-width:1.5;"/>
+ <defs>
+ <path d="M -3 3
+L 3 -3
+M -3 -3
+L 3 3
+" id="me6acc25a23" style="stroke:#9467bd;stroke-opacity:0.2;"/>
+ </defs>
+ <g clip-path="url(#p444e95376c)">
+ <use style="fill:#9467bd;fill-opacity:0.2;stroke:#9467bd;stroke-opacity:0.2;" x="369.97072" xlink:href="#me6acc25a23" y="288.76125"/>
+ <use style="fill:#9467bd;fill-opacity:0.2;stroke:#9467bd;stroke-opacity:0.2;" x="372.063383" xlink:href="#me6acc25a23" y="287.721036"/>
+ <use style="fill:#9467bd;fill-opacity:0.2;stroke:#9467bd;stroke-opacity:0.2;" x="374.13592" xlink:href="#me6acc25a23" y="286.672332"/>
+ <use style="fill:#9467bd;fill-opacity:0.2;stroke:#9467bd;stroke-opacity:0.2;" x="376.18747" xlink:href="#me6acc25a23" y="285.614761"/>
+ <use style="fill:#9467bd;fill-opacity:0.2;stroke:#9467bd;stroke-opacity:0.2;" x="378.217168" xlink:href="#me6acc25a23" y="284.547948"/>
+ <use style="fill:#9467bd;fill-opacity:0.2;stroke:#9467bd;stroke-opacity:0.2;" x="380.224153" xlink:href="#me6acc25a23" y="283.471527"/>
+ <use style="fill:#9467bd;fill-opacity:0.2;stroke:#9467bd;stroke-opacity:0.2;" x="382.207563" xlink:href="#me6acc25a23" y="282.385137"/>
+ <use style="fill:#9467bd;fill-opacity:0.2;stroke:#9467bd;stroke-opacity:0.2;" x="384.16654" xlink:href="#me6acc25a23" y="281.288426"/>
+ <use style="fill:#9467bd;fill-opacity:0.2;stroke:#9467bd;stroke-opacity:0.2;" x="386.100227" xlink:href="#me6acc25a23" y="280.181047"/>
+ <use style="fill:#9467bd;fill-opacity:0.2;stroke:#9467bd;stroke-opacity:0.2;" x="388.007768" xlink:href="#me6acc25a23" y="279.062661"/>
+ <use style="fill:#9467bd;fill-opacity:0.2;stroke:#9467bd;stroke-opacity:0.2;" x="389.888306" xlink:href="#me6acc25a23" y="277.932939"/>
+ <use style="fill:#9467bd;fill-opacity:0.2;stroke:#9467bd;stroke-opacity:0.2;" x="391.740983" xlink:href="#me6acc25a23" y="276.791559"/>
+ <use style="fill:#9467bd;fill-opacity:0.2;stroke:#9467bd;stroke-opacity:0.2;" x="393.564937" xlink:href="#me6acc25a23" y="275.638209"/>
+ <use style="fill:#9467bd;fill-opacity:0.2;stroke:#9467bd;stroke-opacity:0.2;" x="395.359299" xlink:href="#me6acc25a23" y="274.472586"/>
+ <use style="fill:#9467bd;fill-opacity:0.2;stroke:#9467bd;stroke-opacity:0.2;" x="397.123195" xlink:href="#me6acc25a23" y="273.294398"/>
+ <use style="fill:#9467bd;fill-opacity:0.2;stroke:#9467bd;stroke-opacity:0.2;" x="398.855735" xlink:href="#me6acc25a23" y="272.103366"/>
+ <use style="fill:#9467bd;fill-opacity:0.2;stroke:#9467bd;stroke-opacity:0.2;" x="400.556016" xlink:href="#me6acc25a23" y="270.899221"/>
+ <use style="fill:#9467bd;fill-opacity:0.2;stroke:#9467bd;stroke-opacity:0.2;" x="402.223121" xlink:href="#me6acc25a23" y="269.681709"/>
+ <use style="fill:#9467bd;fill-opacity:0.2;stroke:#9467bd;stroke-opacity:0.2;" x="403.856107" xlink:href="#me6acc25a23" y="268.450588"/>
+ <use style="fill:#9467bd;fill-opacity:0.2;stroke:#9467bd;stroke-opacity:0.2;" x="405.454012" xlink:href="#me6acc25a23" y="267.205635"/>
+ <use style="fill:#9467bd;fill-opacity:0.2;stroke:#9467bd;stroke-opacity:0.2;" x="407.015848" xlink:href="#me6acc25a23" y="265.946642"/>
+ <use style="fill:#9467bd;fill-opacity:0.2;stroke:#9467bd;stroke-opacity:0.2;" x="408.540599" xlink:href="#me6acc25a23" y="264.67342"/>
+ <use style="fill:#9467bd;fill-opacity:0.2;stroke:#9467bd;stroke-opacity:0.2;" x="410.027221" xlink:href="#me6acc25a23" y="263.385799"/>
+ <use style="fill:#9467bd;fill-opacity:0.2;stroke:#9467bd;stroke-opacity:0.2;" x="411.474644" xlink:href="#me6acc25a23" y="262.08363"/>
+ <use style="fill:#9467bd;fill-opacity:0.2;stroke:#9467bd;stroke-opacity:0.2;" x="412.881766" xlink:href="#me6acc25a23" y="260.766789"/>
+ <use style="fill:#9467bd;fill-opacity:0.2;stroke:#9467bd;stroke-opacity:0.2;" x="414.247463" xlink:href="#me6acc25a23" y="259.435172"/>
+ <use style="fill:#9467bd;fill-opacity:0.2;stroke:#9467bd;stroke-opacity:0.2;" x="415.570586" xlink:href="#me6acc25a23" y="258.088703"/>
+ <use style="fill:#9467bd;fill-opacity:0.2;stroke:#9467bd;stroke-opacity:0.2;" x="416.849965" xlink:href="#me6acc25a23" y="256.72733"/>
+ <use style="fill:#9467bd;fill-opacity:0.2;stroke:#9467bd;stroke-opacity:0.2;" x="418.08442" xlink:href="#me6acc25a23" y="255.351027"/>
+ <use style="fill:#9467bd;fill-opacity:0.2;stroke:#9467bd;stroke-opacity:0.2;" x="419.272761" xlink:href="#me6acc25a23" y="253.959798"/>
+ <use style="fill:#9467bd;fill-opacity:0.2;stroke:#9467bd;stroke-opacity:0.2;" x="420.413797" xlink:href="#me6acc25a23" y="252.553673"/>
+ <use style="fill:#9467bd;fill-opacity:0.2;stroke:#9467bd;stroke-opacity:0.2;" x="421.506347" xlink:href="#me6acc25a23" y="251.132709"/>
+ <use style="fill:#9467bd;fill-opacity:0.2;stroke:#9467bd;stroke-opacity:0.2;" x="422.549249" xlink:href="#me6acc25a23" y="249.69699"/>
+ <use style="fill:#9467bd;fill-opacity:0.2;stroke:#9467bd;stroke-opacity:0.2;" x="423.541367" xlink:href="#me6acc25a23" y="248.246627"/>
+ <use style="fill:#9467bd;fill-opacity:0.2;stroke:#9467bd;stroke-opacity:0.2;" x="424.481605" xlink:href="#me6acc25a23" y="246.781758"/>
+ <use style="fill:#9467bd;fill-opacity:0.2;stroke:#9467bd;stroke-opacity:0.2;" x="425.368916" xlink:href="#me6acc25a23" y="245.302541"/>
+ <use style="fill:#9467bd;fill-opacity:0.2;stroke:#9467bd;stroke-opacity:0.2;" x="426.202312" xlink:href="#me6acc25a23" y="243.80916"/>
+ <use style="fill:#9467bd;fill-opacity:0.2;stroke:#9467bd;stroke-opacity:0.2;" x="426.980875" xlink:href="#me6acc25a23" y="242.301817"/>
+ <use style="fill:#9467bd;fill-opacity:0.2;stroke:#9467bd;stroke-opacity:0.2;" x="427.703764" xlink:href="#me6acc25a23" y="240.780735"/>
+ <use style="fill:#9467bd;fill-opacity:0.2;stroke:#9467bd;stroke-opacity:0.2;" x="428.370223" xlink:href="#me6acc25a23" y="239.246148"/>
+ <use style="fill:#9467bd;fill-opacity:0.2;stroke:#9467bd;stroke-opacity:0.2;" x="428.979586" xlink:href="#me6acc25a23" y="237.698309"/>
+ <use style="fill:#9467bd;fill-opacity:0.2;stroke:#9467bd;stroke-opacity:0.2;" x="429.531279" xlink:href="#me6acc25a23" y="236.137478"/>
+ <use style="fill:#9467bd;fill-opacity:0.2;stroke:#9467bd;stroke-opacity:0.2;" x="430.024817" xlink:href="#me6acc25a23" y="234.563926"/>
+ <use style="fill:#9467bd;fill-opacity:0.2;stroke:#9467bd;stroke-opacity:0.2;" x="430.459768" xlink:href="#me6acc25a23" y="232.977937"/>
+ <use style="fill:#9467bd;fill-opacity:0.2;stroke:#9467bd;stroke-opacity:0.2;" x="430.835793" xlink:href="#me6acc25a23" y="231.379801"/>
+ <use style="fill:#9467bd;fill-opacity:0.2;stroke:#9467bd;stroke-opacity:0.2;" x="431.152661" xlink:href="#me6acc25a23" y="229.769811"/>
+ <use style="fill:#9467bd;fill-opacity:0.2;stroke:#9467bd;stroke-opacity:0.2;" x="431.410243" xlink:href="#me6acc25a23" y="228.148263"/>
+ <use style="fill:#9467bd;fill-opacity:0.2;stroke:#9467bd;stroke-opacity:0.2;" x="431.608531" xlink:href="#me6acc25a23" y="226.51545"/>
+ <use style="fill:#9467bd;fill-opacity:0.2;stroke:#9467bd;stroke-opacity:0.2;" x="431.747646" xlink:href="#me6acc25a23" y="224.871661"/>
+ <use style="fill:#9467bd;fill-opacity:0.2;stroke:#9467bd;stroke-opacity:0.2;" x="431.827843" xlink:href="#me6acc25a23" y="223.217181"/>
+ <use style="fill:#9467bd;fill-opacity:0.2;stroke:#9467bd;stroke-opacity:0.2;" x="431.849508" xlink:href="#me6acc25a23" y="221.552286"/>
+ <use style="fill:#9467bd;fill-opacity:0.2;stroke:#9467bd;stroke-opacity:0.2;" x="431.81315" xlink:href="#me6acc25a23" y="219.877252"/>
+ <use style="fill:#9467bd;fill-opacity:0.2;stroke:#9467bd;stroke-opacity:0.2;" x="431.719405" xlink:href="#me6acc25a23" y="218.192354"/>
+ <use style="fill:#9467bd;fill-opacity:0.2;stroke:#9467bd;stroke-opacity:0.2;" x="431.569021" xlink:href="#me6acc25a23" y="216.497862"/>
+ <use style="fill:#9467bd;fill-opacity:0.2;stroke:#9467bd;stroke-opacity:0.2;" x="431.362854" xlink:href="#me6acc25a23" y="214.794048"/>
+ <use style="fill:#9467bd;fill-opacity:0.2;stroke:#9467bd;stroke-opacity:0.2;" x="431.101849" xlink:href="#me6acc25a23" y="213.081182"/>
+ <use style="fill:#9467bd;fill-opacity:0.2;stroke:#9467bd;stroke-opacity:0.2;" x="430.787035" xlink:href="#me6acc25a23" y="211.359538"/>
+ <use style="fill:#9467bd;fill-opacity:0.2;stroke:#9467bd;stroke-opacity:0.2;" x="430.419503" xlink:href="#me6acc25a23" y="209.629393"/>
+ <use style="fill:#9467bd;fill-opacity:0.2;stroke:#9467bd;stroke-opacity:0.2;" x="430.000393" xlink:href="#me6acc25a23" y="207.891028"/>
+ <use style="fill:#9467bd;fill-opacity:0.2;stroke:#9467bd;stroke-opacity:0.2;" x="429.530879" xlink:href="#me6acc25a23" y="206.144731"/>
+ <use style="fill:#9467bd;fill-opacity:0.2;stroke:#9467bd;stroke-opacity:0.2;" x="429.01215" xlink:href="#me6acc25a23" y="204.390798"/>
+ <use style="fill:#9467bd;fill-opacity:0.2;stroke:#9467bd;stroke-opacity:0.2;" x="428.445396" xlink:href="#me6acc25a23" y="202.629536"/>
+ <use style="fill:#9467bd;fill-opacity:0.2;stroke:#9467bd;stroke-opacity:0.2;" x="427.831795" xlink:href="#me6acc25a23" y="200.861263"/>
+ <use style="fill:#9467bd;fill-opacity:0.2;stroke:#9467bd;stroke-opacity:0.2;" x="427.172497" xlink:href="#me6acc25a23" y="199.086307"/>
+ <use style="fill:#9467bd;fill-opacity:0.2;stroke:#9467bd;stroke-opacity:0.2;" x="426.468613" xlink:href="#me6acc25a23" y="197.305012"/>
+ <use style="fill:#9467bd;fill-opacity:0.2;stroke:#9467bd;stroke-opacity:0.2;" x="425.721207" xlink:href="#me6acc25a23" y="195.517735"/>
+ <use style="fill:#9467bd;fill-opacity:0.2;stroke:#9467bd;stroke-opacity:0.2;" x="424.931287" xlink:href="#me6acc25a23" y="193.724847"/>
+ <use style="fill:#9467bd;fill-opacity:0.2;stroke:#9467bd;stroke-opacity:0.2;" x="424.099799" xlink:href="#me6acc25a23" y="191.926732"/>
+ <use style="fill:#9467bd;fill-opacity:0.2;stroke:#9467bd;stroke-opacity:0.2;" x="423.227627" xlink:href="#me6acc25a23" y="190.123789"/>
+ <use style="fill:#9467bd;fill-opacity:0.2;stroke:#9467bd;stroke-opacity:0.2;" x="422.315587" xlink:href="#me6acc25a23" y="188.316428"/>
+ <use style="fill:#9467bd;fill-opacity:0.2;stroke:#9467bd;stroke-opacity:0.2;" x="421.364431" xlink:href="#me6acc25a23" y="186.505073"/>
+ <use style="fill:#9467bd;fill-opacity:0.2;stroke:#9467bd;stroke-opacity:0.2;" x="420.374849" xlink:href="#me6acc25a23" y="184.690159"/>
+ <use style="fill:#9467bd;fill-opacity:0.2;stroke:#9467bd;stroke-opacity:0.2;" x="419.347467" xlink:href="#me6acc25a23" y="182.872128"/>
+ <use style="fill:#9467bd;fill-opacity:0.2;stroke:#9467bd;stroke-opacity:0.2;" x="418.282858" xlink:href="#me6acc25a23" y="181.051435"/>
+ <use style="fill:#9467bd;fill-opacity:0.2;stroke:#9467bd;stroke-opacity:0.2;" x="417.181542" xlink:href="#me6acc25a23" y="179.228541"/>
+ <use style="fill:#9467bd;fill-opacity:0.2;stroke:#9467bd;stroke-opacity:0.2;" x="416.044059" xlink:href="#me6acc25a23" y="177.403921"/>
+ <use style="fill:#9467bd;fill-opacity:0.2;stroke:#9467bd;stroke-opacity:0.2;" x="414.870956" xlink:href="#me6acc25a23" y="175.578058"/>
+ <use style="fill:#9467bd;fill-opacity:0.2;stroke:#9467bd;stroke-opacity:0.2;" x="413.662738" xlink:href="#me6acc25a23" y="173.751444"/>
+ <use style="fill:#9467bd;fill-opacity:0.2;stroke:#9467bd;stroke-opacity:0.2;" x="412.419912" xlink:href="#me6acc25a23" y="171.924578"/>
+ <use style="fill:#9467bd;fill-opacity:0.2;stroke:#9467bd;stroke-opacity:0.2;" x="411.142964" xlink:href="#me6acc25a23" y="170.097965"/>
+ <use style="fill:#9467bd;fill-opacity:0.2;stroke:#9467bd;stroke-opacity:0.2;" x="409.832358" xlink:href="#me6acc25a23" y="168.272115"/>
+ <use style="fill:#9467bd;fill-opacity:0.2;stroke:#9467bd;stroke-opacity:0.2;" x="408.488538" xlink:href="#me6acc25a23" y="166.44754"/>
+ <use style="fill:#9467bd;fill-opacity:0.2;stroke:#9467bd;stroke-opacity:0.2;" x="407.111927" xlink:href="#me6acc25a23" y="164.624759"/>
+ <use style="fill:#9467bd;fill-opacity:0.2;stroke:#9467bd;stroke-opacity:0.2;" x="405.702938" xlink:href="#me6acc25a23" y="162.804289"/>
+ <use style="fill:#9467bd;fill-opacity:0.2;stroke:#9467bd;stroke-opacity:0.2;" x="404.261971" xlink:href="#me6acc25a23" y="160.986653"/>
+ <use style="fill:#9467bd;fill-opacity:0.2;stroke:#9467bd;stroke-opacity:0.2;" x="402.789428" xlink:href="#me6acc25a23" y="159.172372"/>
+ <use style="fill:#9467bd;fill-opacity:0.2;stroke:#9467bd;stroke-opacity:0.2;" x="401.285713" xlink:href="#me6acc25a23" y="157.361968"/>
+ <use style="fill:#9467bd;fill-opacity:0.2;stroke:#9467bd;stroke-opacity:0.2;" x="399.75124" xlink:href="#me6acc25a23" y="155.555963"/>
+ <use style="fill:#9467bd;fill-opacity:0.2;stroke:#9467bd;stroke-opacity:0.2;" x="398.186443" xlink:href="#me6acc25a23" y="153.754877"/>
+ <use style="fill:#9467bd;fill-opacity:0.2;stroke:#9467bd;stroke-opacity:0.2;" x="396.591779" xlink:href="#me6acc25a23" y="151.959229"/>
+ <use style="fill:#9467bd;fill-opacity:0.2;stroke:#9467bd;stroke-opacity:0.2;" x="394.967735" xlink:href="#me6acc25a23" y="150.169534"/>
+ <use style="fill:#9467bd;fill-opacity:0.2;stroke:#9467bd;stroke-opacity:0.2;" x="393.314834" xlink:href="#me6acc25a23" y="148.386301"/>
+ <use style="fill:#9467bd;fill-opacity:0.2;stroke:#9467bd;stroke-opacity:0.2;" x="391.633638" xlink:href="#me6acc25a23" y="146.610037"/>
+ <use style="fill:#9467bd;fill-opacity:0.2;stroke:#9467bd;stroke-opacity:0.2;" x="389.924753" xlink:href="#me6acc25a23" y="144.841241"/>
+ <use style="fill:#9467bd;fill-opacity:0.2;stroke:#9467bd;stroke-opacity:0.2;" x="388.18883" xlink:href="#me6acc25a23" y="143.080403"/>
+ <use style="fill:#9467bd;fill-opacity:0.2;stroke:#9467bd;stroke-opacity:0.2;" x="386.426569" xlink:href="#me6acc25a23" y="141.328007"/>
+ <use style="fill:#9467bd;fill-opacity:0.2;stroke:#9467bd;stroke-opacity:0.2;" x="384.638721" xlink:href="#me6acc25a23" y="139.584525"/>
+ <use style="fill:#9467bd;fill-opacity:0.2;stroke:#9467bd;stroke-opacity:0.2;" x="382.826084" xlink:href="#me6acc25a23" y="137.850423"/>
+ <use style="fill:#9467bd;fill-opacity:0.2;stroke:#9467bd;stroke-opacity:0.2;" x="380.989508" xlink:href="#me6acc25a23" y="136.12615"/>
+ <use style="fill:#9467bd;fill-opacity:0.2;stroke:#9467bd;stroke-opacity:0.2;" x="379.129892" xlink:href="#me6acc25a23" y="134.412147"/>
+ <use style="fill:#9467bd;fill-opacity:0.2;stroke:#9467bd;stroke-opacity:0.2;" x="377.248182" xlink:href="#me6acc25a23" y="132.70884"/>
+ <use style="fill:#9467bd;fill-opacity:0.2;stroke:#9467bd;stroke-opacity:0.2;" x="375.34537" xlink:href="#me6acc25a23" y="131.016643"/>
+ <use style="fill:#9467bd;fill-opacity:0.2;stroke:#9467bd;stroke-opacity:0.2;" x="373.422495" xlink:href="#me6acc25a23" y="129.335956"/>
+ <use style="fill:#9467bd;fill-opacity:0.2;stroke:#9467bd;stroke-opacity:0.2;" x="371.480635" xlink:href="#me6acc25a23" y="127.667161"/>
+ <use style="fill:#9467bd;fill-opacity:0.2;stroke:#9467bd;stroke-opacity:0.2;" x="369.520913" xlink:href="#me6acc25a23" y="126.010629"/>
+ <use style="fill:#9467bd;fill-opacity:0.2;stroke:#9467bd;stroke-opacity:0.2;" x="367.544488" xlink:href="#me6acc25a23" y="124.366712"/>
+ <use style="fill:#9467bd;fill-opacity:0.2;stroke:#9467bd;stroke-opacity:0.2;" x="365.552527" xlink:href="#me6acc25a23" y="122.735755"/>
+ <use style="fill:#9467bd;fill-opacity:0.2;stroke:#9467bd;stroke-opacity:0.2;" x="363.54619" xlink:href="#me6acc25a23" y="121.118097"/>
+ <use style="fill:#9467bd;fill-opacity:0.2;stroke:#9467bd;stroke-opacity:0.2;" x="361.526641" xlink:href="#me6acc25a23" y="119.514066"/>
+ <use style="fill:#9467bd;fill-opacity:0.2;stroke:#9467bd;stroke-opacity:0.2;" x="359.495025" xlink:href="#me6acc25a23" y="117.923985"/>
+ <use style="fill:#9467bd;fill-opacity:0.2;stroke:#9467bd;stroke-opacity:0.2;" x="357.452464" xlink:href="#me6acc25a23" y="116.348168"/>
+ <use style="fill:#9467bd;fill-opacity:0.2;stroke:#9467bd;stroke-opacity:0.2;" x="355.400049" xlink:href="#me6acc25a23" y="114.786925"/>
+ <use style="fill:#9467bd;fill-opacity:0.2;stroke:#9467bd;stroke-opacity:0.2;" x="353.338841" xlink:href="#me6acc25a23" y="113.240559"/>
+ <use style="fill:#9467bd;fill-opacity:0.2;stroke:#9467bd;stroke-opacity:0.2;" x="351.269854" xlink:href="#me6acc25a23" y="111.709368"/>
+ <use style="fill:#9467bd;fill-opacity:0.2;stroke:#9467bd;stroke-opacity:0.2;" x="349.194063" xlink:href="#me6acc25a23" y="110.193644"/>
+ <use style="fill:#9467bd;fill-opacity:0.2;stroke:#9467bd;stroke-opacity:0.2;" x="347.112392" xlink:href="#me6acc25a23" y="108.69367"/>
+ <use style="fill:#9467bd;fill-opacity:0.2;stroke:#9467bd;stroke-opacity:0.2;" x="345.025716" xlink:href="#me6acc25a23" y="107.209728"/>
+ <use style="fill:#9467bd;fill-opacity:0.2;stroke:#9467bd;stroke-opacity:0.2;" x="342.93486" xlink:href="#me6acc25a23" y="105.742088"/>
+ <use style="fill:#9467bd;fill-opacity:0.2;stroke:#9467bd;stroke-opacity:0.2;" x="340.8406" xlink:href="#me6acc25a23" y="104.291018"/>
+ <use style="fill:#9467bd;fill-opacity:0.2;stroke:#9467bd;stroke-opacity:0.2;" x="338.743663" xlink:href="#me6acc25a23" y="102.856774"/>
+ <use style="fill:#9467bd;fill-opacity:0.2;stroke:#9467bd;stroke-opacity:0.2;" x="336.644731" xlink:href="#me6acc25a23" y="101.439608"/>
+ <use style="fill:#9467bd;fill-opacity:0.2;stroke:#9467bd;stroke-opacity:0.2;" x="334.544444" xlink:href="#me6acc25a23" y="100.039761"/>
+ <use style="fill:#9467bd;fill-opacity:0.2;stroke:#9467bd;stroke-opacity:0.2;" x="332.443403" xlink:href="#me6acc25a23" y="98.657466"/>
+ <use style="fill:#9467bd;fill-opacity:0.2;stroke:#9467bd;stroke-opacity:0.2;" x="330.342177" xlink:href="#me6acc25a23" y="97.292945"/>
+ <use style="fill:#9467bd;fill-opacity:0.2;stroke:#9467bd;stroke-opacity:0.2;" x="328.241306" xlink:href="#me6acc25a23" y="95.946411"/>
+ <use style="fill:#9467bd;fill-opacity:0.2;stroke:#9467bd;stroke-opacity:0.2;" x="326.141306" xlink:href="#me6acc25a23" y="94.618065"/>
+ <use style="fill:#9467bd;fill-opacity:0.2;stroke:#9467bd;stroke-opacity:0.2;" x="324.042675" xlink:href="#me6acc25a23" y="93.308095"/>
+ <use style="fill:#9467bd;fill-opacity:0.2;stroke:#9467bd;stroke-opacity:0.2;" x="321.945897" xlink:href="#me6acc25a23" y="92.016678"/>
+ <use style="fill:#9467bd;fill-opacity:0.2;stroke:#9467bd;stroke-opacity:0.2;" x="319.851444" xlink:href="#me6acc25a23" y="90.743978"/>
+ <use style="fill:#9467bd;fill-opacity:0.2;stroke:#9467bd;stroke-opacity:0.2;" x="317.759785" xlink:href="#me6acc25a23" y="89.490144"/>
+ <use style="fill:#9467bd;fill-opacity:0.2;stroke:#9467bd;stroke-opacity:0.2;" x="315.671384" xlink:href="#me6acc25a23" y="88.255311"/>
+ <use style="fill:#9467bd;fill-opacity:0.2;stroke:#9467bd;stroke-opacity:0.2;" x="313.586707" xlink:href="#me6acc25a23" y="87.039599"/>
+ <use style="fill:#9467bd;fill-opacity:0.2;stroke:#9467bd;stroke-opacity:0.2;" x="311.506218" xlink:href="#me6acc25a23" y="85.843112"/>
+ <use style="fill:#9467bd;fill-opacity:0.2;stroke:#9467bd;stroke-opacity:0.2;" x="309.430387" xlink:href="#me6acc25a23" y="84.665939"/>
+ <use style="fill:#9467bd;fill-opacity:0.2;stroke:#9467bd;stroke-opacity:0.2;" x="307.359685" xlink:href="#me6acc25a23" y="83.508153"/>
+ <use style="fill:#9467bd;fill-opacity:0.2;stroke:#9467bd;stroke-opacity:0.2;" x="305.294585" xlink:href="#me6acc25a23" y="82.369809"/>
+ <use style="fill:#9467bd;fill-opacity:0.2;stroke:#9467bd;stroke-opacity:0.2;" x="303.235562" xlink:href="#me6acc25a23" y="81.250946"/>
+ <use style="fill:#9467bd;fill-opacity:0.2;stroke:#9467bd;stroke-opacity:0.2;" x="301.183091" xlink:href="#me6acc25a23" y="80.151584"/>
+ <use style="fill:#9467bd;fill-opacity:0.2;stroke:#9467bd;stroke-opacity:0.2;" x="299.137646" xlink:href="#me6acc25a23" y="79.071728"/>
+ <use style="fill:#9467bd;fill-opacity:0.2;stroke:#9467bd;stroke-opacity:0.2;" x="297.099693" xlink:href="#me6acc25a23" y="78.011364"/>
+ <use style="fill:#9467bd;fill-opacity:0.2;stroke:#9467bd;stroke-opacity:0.2;" x="295.069694" xlink:href="#me6acc25a23" y="76.970469"/>
+ <use style="fill:#9467bd;fill-opacity:0.2;stroke:#9467bd;stroke-opacity:0.2;" x="293.048101" xlink:href="#me6acc25a23" y="75.949"/>
+ <use style="fill:#9467bd;fill-opacity:0.2;stroke:#9467bd;stroke-opacity:0.2;" x="291.035353" xlink:href="#me6acc25a23" y="74.946904"/>
+ <use style="fill:#9467bd;fill-opacity:0.2;stroke:#9467bd;stroke-opacity:0.2;" x="289.031875" xlink:href="#me6acc25a23" y="73.964111"/>
+ <use style="fill:#9467bd;fill-opacity:0.2;stroke:#9467bd;stroke-opacity:0.2;" x="287.038071" xlink:href="#me6acc25a23" y="73.00054"/>
+ <use style="fill:#9467bd;fill-opacity:0.2;stroke:#9467bd;stroke-opacity:0.2;" x="285.054326" xlink:href="#me6acc25a23" y="72.056094"/>
+ <use style="fill:#9467bd;fill-opacity:0.2;stroke:#9467bd;stroke-opacity:0.2;" x="283.080997" xlink:href="#me6acc25a23" y="71.130665"/>
+ <use style="fill:#9467bd;fill-opacity:0.2;stroke:#9467bd;stroke-opacity:0.2;" x="281.118415" xlink:href="#me6acc25a23" y="70.224131"/>
+ <use style="fill:#9467bd;fill-opacity:0.2;stroke:#9467bd;stroke-opacity:0.2;" x="279.166881" xlink:href="#me6acc25a23" y="69.336356"/>
+ <use style="fill:#9467bd;fill-opacity:0.2;stroke:#9467bd;stroke-opacity:0.2;" x="277.226666" xlink:href="#me6acc25a23" y="68.467191"/>
+ <use style="fill:#9467bd;fill-opacity:0.2;stroke:#9467bd;stroke-opacity:0.2;" x="275.298016" xlink:href="#me6acc25a23" y="67.616475"/>
+ <use style="fill:#9467bd;fill-opacity:0.2;stroke:#9467bd;stroke-opacity:0.2;" x="273.381144" xlink:href="#me6acc25a23" y="66.784031"/>
+ <use style="fill:#9467bd;fill-opacity:0.2;stroke:#9467bd;stroke-opacity:0.2;" x="271.476238" xlink:href="#me6acc25a23" y="65.969673"/>
+ <use style="fill:#9467bd;fill-opacity:0.2;stroke:#9467bd;stroke-opacity:0.2;" x="269.583452" xlink:href="#me6acc25a23" y="65.173197"/>
+ <use style="fill:#9467bd;fill-opacity:0.2;stroke:#9467bd;stroke-opacity:0.2;" x="267.702913" xlink:href="#me6acc25a23" y="64.39439"/>
+ <use style="fill:#9467bd;fill-opacity:0.2;stroke:#9467bd;stroke-opacity:0.2;" x="265.834717" xlink:href="#me6acc25a23" y="63.633025"/>
+ <use style="fill:#9467bd;fill-opacity:0.2;stroke:#9467bd;stroke-opacity:0.2;" x="263.978929" xlink:href="#me6acc25a23" y="62.888864"/>
+ <use style="fill:#9467bd;fill-opacity:0.2;stroke:#9467bd;stroke-opacity:0.2;" x="262.135583" xlink:href="#me6acc25a23" y="62.161657"/>
+ <use style="fill:#9467bd;fill-opacity:0.2;stroke:#9467bd;stroke-opacity:0.2;" x="260.304682" xlink:href="#me6acc25a23" y="61.451143"/>
+ <use style="fill:#9467bd;fill-opacity:0.2;stroke:#9467bd;stroke-opacity:0.2;" x="258.486197" xlink:href="#me6acc25a23" y="60.75705"/>
+ <use style="fill:#9467bd;fill-opacity:0.2;stroke:#9467bd;stroke-opacity:0.2;" x="256.680068" xlink:href="#me6acc25a23" y="60.079096"/>
+ <use style="fill:#9467bd;fill-opacity:0.2;stroke:#9467bd;stroke-opacity:0.2;" x="254.886204" xlink:href="#me6acc25a23" y="59.416992"/>
+ <use style="fill:#9467bd;fill-opacity:0.2;stroke:#9467bd;stroke-opacity:0.2;" x="253.104482" xlink:href="#me6acc25a23" y="58.770439"/>
+ <use style="fill:#9467bd;fill-opacity:0.2;stroke:#9467bd;stroke-opacity:0.2;" x="251.334749" xlink:href="#me6acc25a23" y="58.139128"/>
+ <use style="fill:#9467bd;fill-opacity:0.2;stroke:#9467bd;stroke-opacity:0.2;" x="249.576821" xlink:href="#me6acc25a23" y="57.522745"/>
+ <use style="fill:#9467bd;fill-opacity:0.2;stroke:#9467bd;stroke-opacity:0.2;" x="247.830485" xlink:href="#me6acc25a23" y="56.920971"/>
+ <use style="fill:#9467bd;fill-opacity:0.2;stroke:#9467bd;stroke-opacity:0.2;" x="246.095498" xlink:href="#me6acc25a23" y="56.333476"/>
+ <use style="fill:#9467bd;fill-opacity:0.2;stroke:#9467bd;stroke-opacity:0.2;" x="244.371589" xlink:href="#me6acc25a23" y="55.759929"/>
+ <use style="fill:#9467bd;fill-opacity:0.2;stroke:#9467bd;stroke-opacity:0.2;" x="242.658459" xlink:href="#me6acc25a23" y="55.199993"/>
+ <use style="fill:#9467bd;fill-opacity:0.2;stroke:#9467bd;stroke-opacity:0.2;" x="240.955782" xlink:href="#me6acc25a23" y="54.653327"/>
+ <use style="fill:#9467bd;fill-opacity:0.2;stroke:#9467bd;stroke-opacity:0.2;" x="239.263205" xlink:href="#me6acc25a23" y="54.119586"/>
+ <use style="fill:#9467bd;fill-opacity:0.2;stroke:#9467bd;stroke-opacity:0.2;" x="237.580352" xlink:href="#me6acc25a23" y="53.598422"/>
+ <use style="fill:#9467bd;fill-opacity:0.2;stroke:#9467bd;stroke-opacity:0.2;" x="235.90682" xlink:href="#me6acc25a23" y="53.089485"/>
+ <use style="fill:#9467bd;fill-opacity:0.2;stroke:#9467bd;stroke-opacity:0.2;" x="234.242185" xlink:href="#me6acc25a23" y="52.592425"/>
+ <use style="fill:#9467bd;fill-opacity:0.2;stroke:#9467bd;stroke-opacity:0.2;" x="232.585997" xlink:href="#me6acc25a23" y="52.106886"/>
+ <use style="fill:#9467bd;fill-opacity:0.2;stroke:#9467bd;stroke-opacity:0.2;" x="230.937788" xlink:href="#me6acc25a23" y="51.632517"/>
+ <use style="fill:#9467bd;fill-opacity:0.2;stroke:#9467bd;stroke-opacity:0.2;" x="229.297068" xlink:href="#me6acc25a23" y="51.168961"/>
+ <use style="fill:#9467bd;fill-opacity:0.2;stroke:#9467bd;stroke-opacity:0.2;" x="227.663327" xlink:href="#me6acc25a23" y="50.715865"/>
+ <use style="fill:#9467bd;fill-opacity:0.2;stroke:#9467bd;stroke-opacity:0.2;" x="226.036036" xlink:href="#me6acc25a23" y="50.272874"/>
+ <use style="fill:#9467bd;fill-opacity:0.2;stroke:#9467bd;stroke-opacity:0.2;" x="224.414651" xlink:href="#me6acc25a23" y="49.839635"/>
+ <use style="fill:#9467bd;fill-opacity:0.2;stroke:#9467bd;stroke-opacity:0.2;" x="222.798608" xlink:href="#me6acc25a23" y="49.415795"/>
+ <use style="fill:#9467bd;fill-opacity:0.2;stroke:#9467bd;stroke-opacity:0.2;" x="221.187329" xlink:href="#me6acc25a23" y="49.001003"/>
+ <use style="fill:#9467bd;fill-opacity:0.2;stroke:#9467bd;stroke-opacity:0.2;" x="219.580221" xlink:href="#me6acc25a23" y="48.59491"/>
+ <use style="fill:#9467bd;fill-opacity:0.2;stroke:#9467bd;stroke-opacity:0.2;" x="217.976677" xlink:href="#me6acc25a23" y="48.197168"/>
+ <use style="fill:#9467bd;fill-opacity:0.2;stroke:#9467bd;stroke-opacity:0.2;" x="216.376077" xlink:href="#me6acc25a23" y="47.807432"/>
+ <use style="fill:#9467bd;fill-opacity:0.2;stroke:#9467bd;stroke-opacity:0.2;" x="214.777789" xlink:href="#me6acc25a23" y="47.425359"/>
+ <use style="fill:#9467bd;fill-opacity:0.2;stroke:#9467bd;stroke-opacity:0.2;" x="213.181171" xlink:href="#me6acc25a23" y="47.050609"/>
+ <use style="fill:#9467bd;fill-opacity:0.2;stroke:#9467bd;stroke-opacity:0.2;" x="211.58557" xlink:href="#me6acc25a23" y="46.682847"/>
+ <use style="fill:#9467bd;fill-opacity:0.2;stroke:#9467bd;stroke-opacity:0.2;" x="209.990325" xlink:href="#me6acc25a23" y="46.321737"/>
+ <use style="fill:#9467bd;fill-opacity:0.2;stroke:#9467bd;stroke-opacity:0.2;" x="208.394766" xlink:href="#me6acc25a23" y="45.966952"/>
+ <use style="fill:#9467bd;fill-opacity:0.2;stroke:#9467bd;stroke-opacity:0.2;" x="206.798217" xlink:href="#me6acc25a23" y="45.618166"/>
+ <use style="fill:#9467bd;fill-opacity:0.2;stroke:#9467bd;stroke-opacity:0.2;" x="205.199994" xlink:href="#me6acc25a23" y="45.275058"/>
+ <use style="fill:#9467bd;fill-opacity:0.2;stroke:#9467bd;stroke-opacity:0.2;" x="203.599411" xlink:href="#me6acc25a23" y="44.937312"/>
+ <use style="fill:#9467bd;fill-opacity:0.2;stroke:#9467bd;stroke-opacity:0.2;" x="201.995776" xlink:href="#me6acc25a23" y="44.604616"/>
+ <use style="fill:#9467bd;fill-opacity:0.2;stroke:#9467bd;stroke-opacity:0.2;" x="200.388395" xlink:href="#me6acc25a23" y="44.276665"/>
+ <use style="fill:#9467bd;fill-opacity:0.2;stroke:#9467bd;stroke-opacity:0.2;" x="198.776571" xlink:href="#me6acc25a23" y="43.953157"/>
+ <use style="fill:#9467bd;fill-opacity:0.2;stroke:#9467bd;stroke-opacity:0.2;" x="197.159608" xlink:href="#me6acc25a23" y="43.633799"/>
+ <use style="fill:#9467bd;fill-opacity:0.2;stroke:#9467bd;stroke-opacity:0.2;" x="195.53681" xlink:href="#me6acc25a23" y="43.318302"/>
+ <use style="fill:#9467bd;fill-opacity:0.2;stroke:#9467bd;stroke-opacity:0.2;" x="193.90748" xlink:href="#me6acc25a23" y="43.006385"/>
+ <use style="fill:#9467bd;fill-opacity:0.2;stroke:#9467bd;stroke-opacity:0.2;" x="192.270927" xlink:href="#me6acc25a23" y="42.697773"/>
+ <use style="fill:#9467bd;fill-opacity:0.2;stroke:#9467bd;stroke-opacity:0.2;" x="190.626461" xlink:href="#me6acc25a23" y="42.392199"/>
+ <use style="fill:#9467bd;fill-opacity:0.2;stroke:#9467bd;stroke-opacity:0.2;" x="188.973399" xlink:href="#me6acc25a23" y="42.089402"/>
+ <use style="fill:#9467bd;fill-opacity:0.2;stroke:#9467bd;stroke-opacity:0.2;" x="187.31106" xlink:href="#me6acc25a23" y="41.789131"/>
+ <use style="fill:#9467bd;fill-opacity:0.2;stroke:#9467bd;stroke-opacity:0.2;" x="185.638773" xlink:href="#me6acc25a23" y="41.491141"/>
+ <use style="fill:#9467bd;fill-opacity:0.2;stroke:#9467bd;stroke-opacity:0.2;" x="183.955872" xlink:href="#me6acc25a23" y="41.195197"/>
+ <use style="fill:#9467bd;fill-opacity:0.2;stroke:#9467bd;stroke-opacity:0.2;" x="182.261702" xlink:href="#me6acc25a23" y="40.901073"/>
+ <use style="fill:#9467bd;fill-opacity:0.2;stroke:#9467bd;stroke-opacity:0.2;" x="180.555614" xlink:href="#me6acc25a23" y="40.60855"/>
+ <use style="fill:#9467bd;fill-opacity:0.2;stroke:#9467bd;stroke-opacity:0.2;" x="178.836972" xlink:href="#me6acc25a23" y="40.31742"/>
+ <use style="fill:#9467bd;fill-opacity:0.2;stroke:#9467bd;stroke-opacity:0.2;" x="177.105147" xlink:href="#me6acc25a23" y="40.027485"/>
+ <use style="fill:#9467bd;fill-opacity:0.2;stroke:#9467bd;stroke-opacity:0.2;" x="175.359523" xlink:href="#me6acc25a23" y="39.738555"/>
+ <use style="fill:#9467bd;fill-opacity:0.2;stroke:#9467bd;stroke-opacity:0.2;" x="173.599497" xlink:href="#me6acc25a23" y="39.450451"/>
+ <use style="fill:#9467bd;fill-opacity:0.2;stroke:#9467bd;stroke-opacity:0.2;" x="171.824474" xlink:href="#me6acc25a23" y="39.163004"/>
+ <use style="fill:#9467bd;fill-opacity:0.2;stroke:#9467bd;stroke-opacity:0.2;" x="170.033874" xlink:href="#me6acc25a23" y="38.876056"/>
+ <use style="fill:#9467bd;fill-opacity:0.2;stroke:#9467bd;stroke-opacity:0.2;" x="168.227129" xlink:href="#me6acc25a23" y="38.589458"/>
+ <use style="fill:#9467bd;fill-opacity:0.2;stroke:#9467bd;stroke-opacity:0.2;" x="166.403682" xlink:href="#me6acc25a23" y="38.303072"/>
+ <use style="fill:#9467bd;fill-opacity:0.2;stroke:#9467bd;stroke-opacity:0.2;" x="164.562989" xlink:href="#me6acc25a23" y="38.016771"/>
+ <use style="fill:#9467bd;fill-opacity:0.2;stroke:#9467bd;stroke-opacity:0.2;" x="162.704518" xlink:href="#me6acc25a23" y="37.730438"/>
+ <use style="fill:#9467bd;fill-opacity:0.2;stroke:#9467bd;stroke-opacity:0.2;" x="160.827751" xlink:href="#me6acc25a23" y="37.443966"/>
+ <use style="fill:#9467bd;fill-opacity:0.2;stroke:#9467bd;stroke-opacity:0.2;" x="158.932179" xlink:href="#me6acc25a23" y="37.157261"/>
+ <use style="fill:#9467bd;fill-opacity:0.2;stroke:#9467bd;stroke-opacity:0.2;" x="157.017307" xlink:href="#me6acc25a23" y="36.870236"/>
+ <use style="fill:#9467bd;fill-opacity:0.2;stroke:#9467bd;stroke-opacity:0.2;" x="155.08265" xlink:href="#me6acc25a23" y="36.582815"/>
+ <use style="fill:#9467bd;fill-opacity:0.2;stroke:#9467bd;stroke-opacity:0.2;" x="153.127734" xlink:href="#me6acc25a23" y="36.294934"/>
+ <use style="fill:#9467bd;fill-opacity:0.2;stroke:#9467bd;stroke-opacity:0.2;" x="151.152098" xlink:href="#me6acc25a23" y="36.006536"/>
+ <use style="fill:#9467bd;fill-opacity:0.2;stroke:#9467bd;stroke-opacity:0.2;" x="149.155289" xlink:href="#me6acc25a23" y="35.717574"/>
+ <use style="fill:#9467bd;fill-opacity:0.2;stroke:#9467bd;stroke-opacity:0.2;" x="147.136866" xlink:href="#me6acc25a23" y="35.428013"/>
+ <use style="fill:#9467bd;fill-opacity:0.2;stroke:#9467bd;stroke-opacity:0.2;" x="145.096394" xlink:href="#me6acc25a23" y="35.137822"/>
+ <use style="fill:#9467bd;fill-opacity:0.2;stroke:#9467bd;stroke-opacity:0.2;" x="143.033444" xlink:href="#me6acc25a23" y="34.846981"/>
+ <use style="fill:#9467bd;fill-opacity:0.2;stroke:#9467bd;stroke-opacity:0.2;" x="140.947593" xlink:href="#me6acc25a23" y="34.555477"/>
+ <use style="fill:#9467bd;fill-opacity:0.2;stroke:#9467bd;stroke-opacity:0.2;" x="138.838424" xlink:href="#me6acc25a23" y="34.263303"/>
+ <use style="fill:#9467bd;fill-opacity:0.2;stroke:#9467bd;stroke-opacity:0.2;" x="136.705525" xlink:href="#me6acc25a23" y="33.97046"/>
+ <use style="fill:#9467bd;fill-opacity:0.2;stroke:#9467bd;stroke-opacity:0.2;" x="134.548487" xlink:href="#me6acc25a23" y="33.676955"/>
+ <use style="fill:#9467bd;fill-opacity:0.2;stroke:#9467bd;stroke-opacity:0.2;" x="132.366908" xlink:href="#me6acc25a23" y="33.382799"/>
+ <use style="fill:#9467bd;fill-opacity:0.2;stroke:#9467bd;stroke-opacity:0.2;" x="130.160389" xlink:href="#me6acc25a23" y="33.088009"/>
+ <use style="fill:#9467bd;fill-opacity:0.2;stroke:#9467bd;stroke-opacity:0.2;" x="127.928536" xlink:href="#me6acc25a23" y="32.792606"/>
+ <use style="fill:#9467bd;fill-opacity:0.2;stroke:#9467bd;stroke-opacity:0.2;" x="125.670961" xlink:href="#me6acc25a23" y="32.496617"/>
+ <use style="fill:#9467bd;fill-opacity:0.2;stroke:#9467bd;stroke-opacity:0.2;" x="123.387279" xlink:href="#me6acc25a23" y="32.200068"/>
+ <use style="fill:#9467bd;fill-opacity:0.2;stroke:#9467bd;stroke-opacity:0.2;" x="121.077114" xlink:href="#me6acc25a23" y="31.902992"/>
+ <use style="fill:#9467bd;fill-opacity:0.2;stroke:#9467bd;stroke-opacity:0.2;" x="118.740095" xlink:href="#me6acc25a23" y="31.60542"/>
+ <use style="fill:#9467bd;fill-opacity:0.2;stroke:#9467bd;stroke-opacity:0.2;" x="116.375859" xlink:href="#me6acc25a23" y="31.307388"/>
+ <use style="fill:#9467bd;fill-opacity:0.2;stroke:#9467bd;stroke-opacity:0.2;" x="113.984053" xlink:href="#me6acc25a23" y="31.008932"/>
+ <use style="fill:#9467bd;fill-opacity:0.2;stroke:#9467bd;stroke-opacity:0.2;" x="111.564334" xlink:href="#me6acc25a23" y="30.710085"/>
+ <use style="fill:#9467bd;fill-opacity:0.2;stroke:#9467bd;stroke-opacity:0.2;" x="109.116369" xlink:href="#me6acc25a23" y="30.410886"/>
+ <use style="fill:#9467bd;fill-opacity:0.2;stroke:#9467bd;stroke-opacity:0.2;" x="106.639839" xlink:href="#me6acc25a23" y="30.111367"/>
+ <use style="fill:#9467bd;fill-opacity:0.2;stroke:#9467bd;stroke-opacity:0.2;" x="104.134442" xlink:href="#me6acc25a23" y="29.811562"/>
+ <use style="fill:#9467bd;fill-opacity:0.2;stroke:#9467bd;stroke-opacity:0.2;" x="101.599892" xlink:href="#me6acc25a23" y="29.511504"/>
+ <use style="fill:#9467bd;fill-opacity:0.2;stroke:#9467bd;stroke-opacity:0.2;" x="99.03592" xlink:href="#me6acc25a23" y="29.211223"/>
+ <use style="fill:#9467bd;fill-opacity:0.2;stroke:#9467bd;stroke-opacity:0.2;" x="96.442283" xlink:href="#me6acc25a23" y="28.910744"/>
+ <use style="fill:#9467bd;fill-opacity:0.2;stroke:#9467bd;stroke-opacity:0.2;" x="93.818759" xlink:href="#me6acc25a23" y="28.610094"/>
+ <use style="fill:#9467bd;fill-opacity:0.2;stroke:#9467bd;stroke-opacity:0.2;" x="91.165153" xlink:href="#me6acc25a23" y="28.309293"/>
+ <use style="fill:#9467bd;fill-opacity:0.2;stroke:#9467bd;stroke-opacity:0.2;" x="88.481299" xlink:href="#me6acc25a23" y="28.008359"/>
+ <use style="fill:#9467bd;fill-opacity:0.2;stroke:#9467bd;stroke-opacity:0.2;" x="85.767063" xlink:href="#me6acc25a23" y="27.707307"/>
+ <use style="fill:#9467bd;fill-opacity:0.2;stroke:#9467bd;stroke-opacity:0.2;" x="83.022345" xlink:href="#me6acc25a23" y="27.406149"/>
+ <use style="fill:#9467bd;fill-opacity:0.2;stroke:#9467bd;stroke-opacity:0.2;" x="80.247082" xlink:href="#me6acc25a23" y="27.104891"/>
+ <use style="fill:#9467bd;fill-opacity:0.2;stroke:#9467bd;stroke-opacity:0.2;" x="77.44125" xlink:href="#me6acc25a23" y="26.803537"/>
+ <use style="fill:#9467bd;fill-opacity:0.2;stroke:#9467bd;stroke-opacity:0.2;" x="74.604866" xlink:href="#me6acc25a23" y="26.502087"/>
+ <use style="fill:#9467bd;fill-opacity:0.2;stroke:#9467bd;stroke-opacity:0.2;" x="71.737992" xlink:href="#me6acc25a23" y="26.200536"/>
+ <use style="fill:#9467bd;fill-opacity:0.2;stroke:#9467bd;stroke-opacity:0.2;" x="68.840733" xlink:href="#me6acc25a23" y="25.898877"/>
+ </g>
+ </g>
+ <g id="line2d_16">
+ <path clip-path="url(#p444e95376c)" d="M 369.97072 288.76125
+L 377.205106 285.082533
+L 384.166542 281.288428
+L 389.888307 277.932942
+L 395.359297 274.47259
+L 400.55601 270.899227
+L 404.659496 267.829861
+L 408.540586 264.673428
+L 412.183298 261.42706
+L 415.570573 258.088714
+L 418.68442 254.657289
+L 420.966201 251.845053
+L 423.051727 248.973643
+L 424.931948 246.043942
+L 426.598518 243.05723
+L 428.044113 240.015121
+L 429.262698 236.919506
+L 430.249657 233.772472
+L 431.00166 230.57627
+L 431.51683 227.333244
+L 431.795128 224.045736
+L 431.838582 220.716014
+L 431.651273 217.346285
+L 431.239165 213.938723
+L 430.609797 210.495504
+L 429.771856 207.018845
+L 428.73468 203.511056
+L 427.172457 199.086299
+L 425.331445 194.62196
+L 423.22756 190.123778
+L 420.87433 185.598021
+L 418.282781 181.05142
+L 415.461864 176.491098
+L 411.785594 171.01119
+L 407.804219 165.535872
+L 403.529534 160.079038
+L 398.972516 154.654748
+L 394.14477 149.27705
+L 389.060039 143.959766
+L 383.735354 138.716241
+L 378.191634 133.559096
+L 372.453755 128.500013
+L 365.552391 122.735722
+L 358.474888 117.134241
+L 351.269682 111.709337
+L 343.980576 106.473823
+L 335.594526 100.737476
+L 327.190971 95.279924
+L 318.805041 90.114667
+L 310.467493 85.25208
+L 302.208273 80.698801
+L 294.057605 76.457286
+L 286.044694 72.52591
+L 278.195109 68.899435
+L 269.583211 65.173176
+L 261.218329 61.804309
+L 253.104225 58.770419
+L 245.231913 56.044961
+L 236.742175 53.342428
+L 228.479075 50.941109
+L 219.579926 48.594891
+L 210.787636 46.501462
+L 201.192274 44.440047
+L 190.626125 42.39218
+L 177.972401 40.172296
+L 162.704169 37.730419
+L 145.096047 35.137803
+L 125.670622 32.496599
+L 105.390445 29.961487
+L 84.398221 27.55674
+L 68.840455 25.898885
+L 68.840455 25.898885
+" style="fill:none;stroke:#8c564b;stroke-dasharray:1.5,4.5,1.5;stroke-dashoffset:0;stroke-opacity:0.2;stroke-width:1.5;"/>
+ <defs>
+ <path d="M -3 3
+L 3 -3
+M -3 -3
+L 3 3
+" id="m4f5604d9a2" style="stroke:#8c564b;stroke-opacity:0.2;"/>
+ </defs>
+ <g clip-path="url(#p444e95376c)">
+ <use style="fill:#8c564b;fill-opacity:0.2;stroke:#8c564b;stroke-opacity:0.2;" x="369.97072" xlink:href="#m4f5604d9a2" y="288.76125"/>
+ <use style="fill:#8c564b;fill-opacity:0.2;stroke:#8c564b;stroke-opacity:0.2;" x="371.019513" xlink:href="#m4f5604d9a2" y="288.24218"/>
+ <use style="fill:#8c564b;fill-opacity:0.2;stroke:#8c564b;stroke-opacity:0.2;" x="372.063383" xlink:href="#m4f5604d9a2" y="287.721036"/>
+ <use style="fill:#8c564b;fill-opacity:0.2;stroke:#8c564b;stroke-opacity:0.2;" x="373.102222" xlink:href="#m4f5604d9a2" y="287.197769"/>
+ <use style="fill:#8c564b;fill-opacity:0.2;stroke:#8c564b;stroke-opacity:0.2;" x="374.135921" xlink:href="#m4f5604d9a2" y="286.672333"/>
+ <use style="fill:#8c564b;fill-opacity:0.2;stroke:#8c564b;stroke-opacity:0.2;" x="375.164374" xlink:href="#m4f5604d9a2" y="286.144679"/>
+ <use style="fill:#8c564b;fill-opacity:0.2;stroke:#8c564b;stroke-opacity:0.2;" x="376.187471" xlink:href="#m4f5604d9a2" y="285.614761"/>
+ <use style="fill:#8c564b;fill-opacity:0.2;stroke:#8c564b;stroke-opacity:0.2;" x="377.205106" xlink:href="#m4f5604d9a2" y="285.082533"/>
+ <use style="fill:#8c564b;fill-opacity:0.2;stroke:#8c564b;stroke-opacity:0.2;" x="378.21717" xlink:href="#m4f5604d9a2" y="284.547949"/>
+ <use style="fill:#8c564b;fill-opacity:0.2;stroke:#8c564b;stroke-opacity:0.2;" x="379.223555" xlink:href="#m4f5604d9a2" y="284.010962"/>
+ <use style="fill:#8c564b;fill-opacity:0.2;stroke:#8c564b;stroke-opacity:0.2;" x="380.224154" xlink:href="#m4f5604d9a2" y="283.471528"/>
+ <use style="fill:#8c564b;fill-opacity:0.2;stroke:#8c564b;stroke-opacity:0.2;" x="381.21886" xlink:href="#m4f5604d9a2" y="282.929602"/>
+ <use style="fill:#8c564b;fill-opacity:0.2;stroke:#8c564b;stroke-opacity:0.2;" x="382.207565" xlink:href="#m4f5604d9a2" y="282.385139"/>
+ <use style="fill:#8c564b;fill-opacity:0.2;stroke:#8c564b;stroke-opacity:0.2;" x="383.190161" xlink:href="#m4f5604d9a2" y="281.838095"/>
+ <use style="fill:#8c564b;fill-opacity:0.2;stroke:#8c564b;stroke-opacity:0.2;" x="384.166542" xlink:href="#m4f5604d9a2" y="281.288428"/>
+ <use style="fill:#8c564b;fill-opacity:0.2;stroke:#8c564b;stroke-opacity:0.2;" x="385.1366" xlink:href="#m4f5604d9a2" y="280.736093"/>
+ <use style="fill:#8c564b;fill-opacity:0.2;stroke:#8c564b;stroke-opacity:0.2;" x="386.100229" xlink:href="#m4f5604d9a2" y="280.181049"/>
+ <use style="fill:#8c564b;fill-opacity:0.2;stroke:#8c564b;stroke-opacity:0.2;" x="387.057321" xlink:href="#m4f5604d9a2" y="279.623253"/>
+ <use style="fill:#8c564b;fill-opacity:0.2;stroke:#8c564b;stroke-opacity:0.2;" x="388.007769" xlink:href="#m4f5604d9a2" y="279.062664"/>
+ <use style="fill:#8c564b;fill-opacity:0.2;stroke:#8c564b;stroke-opacity:0.2;" x="388.951467" xlink:href="#m4f5604d9a2" y="278.49924"/>
+ <use style="fill:#8c564b;fill-opacity:0.2;stroke:#8c564b;stroke-opacity:0.2;" x="389.888307" xlink:href="#m4f5604d9a2" y="277.932942"/>
+ <use style="fill:#8c564b;fill-opacity:0.2;stroke:#8c564b;stroke-opacity:0.2;" x="390.818181" xlink:href="#m4f5604d9a2" y="277.363729"/>
+ <use style="fill:#8c564b;fill-opacity:0.2;stroke:#8c564b;stroke-opacity:0.2;" x="391.740983" xlink:href="#m4f5604d9a2" y="276.791562"/>
+ <use style="fill:#8c564b;fill-opacity:0.2;stroke:#8c564b;stroke-opacity:0.2;" x="392.656603" xlink:href="#m4f5604d9a2" y="276.216402"/>
+ <use style="fill:#8c564b;fill-opacity:0.2;stroke:#8c564b;stroke-opacity:0.2;" x="393.564935" xlink:href="#m4f5604d9a2" y="275.638212"/>
+ <use style="fill:#8c564b;fill-opacity:0.2;stroke:#8c564b;stroke-opacity:0.2;" x="394.46587" xlink:href="#m4f5604d9a2" y="275.056953"/>
+ <use style="fill:#8c564b;fill-opacity:0.2;stroke:#8c564b;stroke-opacity:0.2;" x="395.359297" xlink:href="#m4f5604d9a2" y="274.47259"/>
+ <use style="fill:#8c564b;fill-opacity:0.2;stroke:#8c564b;stroke-opacity:0.2;" x="396.245108" xlink:href="#m4f5604d9a2" y="273.885084"/>
+ <use style="fill:#8c564b;fill-opacity:0.2;stroke:#8c564b;stroke-opacity:0.2;" x="397.123191" xlink:href="#m4f5604d9a2" y="273.294403"/>
+ <use style="fill:#8c564b;fill-opacity:0.2;stroke:#8c564b;stroke-opacity:0.2;" x="397.993436" xlink:href="#m4f5604d9a2" y="272.70051"/>
+ <use style="fill:#8c564b;fill-opacity:0.2;stroke:#8c564b;stroke-opacity:0.2;" x="398.85573" xlink:href="#m4f5604d9a2" y="272.103371"/>
+ <use style="fill:#8c564b;fill-opacity:0.2;stroke:#8c564b;stroke-opacity:0.2;" x="399.709959" xlink:href="#m4f5604d9a2" y="271.502954"/>
+ <use style="fill:#8c564b;fill-opacity:0.2;stroke:#8c564b;stroke-opacity:0.2;" x="400.55601" xlink:href="#m4f5604d9a2" y="270.899227"/>
+ <use style="fill:#8c564b;fill-opacity:0.2;stroke:#8c564b;stroke-opacity:0.2;" x="401.393767" xlink:href="#m4f5604d9a2" y="270.292157"/>
+ <use style="fill:#8c564b;fill-opacity:0.2;stroke:#8c564b;stroke-opacity:0.2;" x="402.223113" xlink:href="#m4f5604d9a2" y="269.681714"/>
+ <use style="fill:#8c564b;fill-opacity:0.2;stroke:#8c564b;stroke-opacity:0.2;" x="403.04393" xlink:href="#m4f5604d9a2" y="269.06787"/>
+ <use style="fill:#8c564b;fill-opacity:0.2;stroke:#8c564b;stroke-opacity:0.2;" x="403.856098" xlink:href="#m4f5604d9a2" y="268.450594"/>
+ <use style="fill:#8c564b;fill-opacity:0.2;stroke:#8c564b;stroke-opacity:0.2;" x="404.659496" xlink:href="#m4f5604d9a2" y="267.829861"/>
+ <use style="fill:#8c564b;fill-opacity:0.2;stroke:#8c564b;stroke-opacity:0.2;" x="405.454002" xlink:href="#m4f5604d9a2" y="267.205642"/>
+ <use style="fill:#8c564b;fill-opacity:0.2;stroke:#8c564b;stroke-opacity:0.2;" x="406.239491" xlink:href="#m4f5604d9a2" y="266.577913"/>
+ <use style="fill:#8c564b;fill-opacity:0.2;stroke:#8c564b;stroke-opacity:0.2;" x="407.015836" xlink:href="#m4f5604d9a2" y="265.946649"/>
+ <use style="fill:#8c564b;fill-opacity:0.2;stroke:#8c564b;stroke-opacity:0.2;" x="407.782911" xlink:href="#m4f5604d9a2" y="265.311828"/>
+ <use style="fill:#8c564b;fill-opacity:0.2;stroke:#8c564b;stroke-opacity:0.2;" x="408.540586" xlink:href="#m4f5604d9a2" y="264.673428"/>
+ <use style="fill:#8c564b;fill-opacity:0.2;stroke:#8c564b;stroke-opacity:0.2;" x="409.288729" xlink:href="#m4f5604d9a2" y="264.031427"/>
+ <use style="fill:#8c564b;fill-opacity:0.2;stroke:#8c564b;stroke-opacity:0.2;" x="410.027208" xlink:href="#m4f5604d9a2" y="263.385807"/>
+ <use style="fill:#8c564b;fill-opacity:0.2;stroke:#8c564b;stroke-opacity:0.2;" x="410.755887" xlink:href="#m4f5604d9a2" y="262.73655"/>
+ <use style="fill:#8c564b;fill-opacity:0.2;stroke:#8c564b;stroke-opacity:0.2;" x="411.47463" xlink:href="#m4f5604d9a2" y="262.08364"/>
+ <use style="fill:#8c564b;fill-opacity:0.2;stroke:#8c564b;stroke-opacity:0.2;" x="412.183298" xlink:href="#m4f5604d9a2" y="261.42706"/>
+ <use style="fill:#8c564b;fill-opacity:0.2;stroke:#8c564b;stroke-opacity:0.2;" x="412.881752" xlink:href="#m4f5604d9a2" y="260.766799"/>
+ <use style="fill:#8c564b;fill-opacity:0.2;stroke:#8c564b;stroke-opacity:0.2;" x="413.56985" xlink:href="#m4f5604d9a2" y="260.102843"/>
+ <use style="fill:#8c564b;fill-opacity:0.2;stroke:#8c564b;stroke-opacity:0.2;" x="414.24745" xlink:href="#m4f5604d9a2" y="259.435183"/>
+ <use style="fill:#8c564b;fill-opacity:0.2;stroke:#8c564b;stroke-opacity:0.2;" x="414.914405" xlink:href="#m4f5604d9a2" y="258.763809"/>
+ <use style="fill:#8c564b;fill-opacity:0.2;stroke:#8c564b;stroke-opacity:0.2;" x="415.570573" xlink:href="#m4f5604d9a2" y="258.088714"/>
+ <use style="fill:#8c564b;fill-opacity:0.2;stroke:#8c564b;stroke-opacity:0.2;" x="416.215805" xlink:href="#m4f5604d9a2" y="257.409893"/>
+ <use style="fill:#8c564b;fill-opacity:0.2;stroke:#8c564b;stroke-opacity:0.2;" x="416.849954" xlink:href="#m4f5604d9a2" y="256.727341"/>
+ <use style="fill:#8c564b;fill-opacity:0.2;stroke:#8c564b;stroke-opacity:0.2;" x="417.472872" xlink:href="#m4f5604d9a2" y="256.041056"/>
+ <use style="fill:#8c564b;fill-opacity:0.2;stroke:#8c564b;stroke-opacity:0.2;" x="418.08441" xlink:href="#m4f5604d9a2" y="255.351039"/>
+ <use style="fill:#8c564b;fill-opacity:0.2;stroke:#8c564b;stroke-opacity:0.2;" x="418.68442" xlink:href="#m4f5604d9a2" y="254.657289"/>
+ <use style="fill:#8c564b;fill-opacity:0.2;stroke:#8c564b;stroke-opacity:0.2;" x="419.272753" xlink:href="#m4f5604d9a2" y="253.95981"/>
+ <use style="fill:#8c564b;fill-opacity:0.2;stroke:#8c564b;stroke-opacity:0.2;" x="419.849259" xlink:href="#m4f5604d9a2" y="253.258607"/>
+ <use style="fill:#8c564b;fill-opacity:0.2;stroke:#8c564b;stroke-opacity:0.2;" x="420.413791" xlink:href="#m4f5604d9a2" y="252.553685"/>
+ <use style="fill:#8c564b;fill-opacity:0.2;stroke:#8c564b;stroke-opacity:0.2;" x="420.966201" xlink:href="#m4f5604d9a2" y="251.845053"/>
+ <use style="fill:#8c564b;fill-opacity:0.2;stroke:#8c564b;stroke-opacity:0.2;" x="421.506344" xlink:href="#m4f5604d9a2" y="251.132721"/>
+ <use style="fill:#8c564b;fill-opacity:0.2;stroke:#8c564b;stroke-opacity:0.2;" x="422.034074" xlink:href="#m4f5604d9a2" y="250.416699"/>
+ <use style="fill:#8c564b;fill-opacity:0.2;stroke:#8c564b;stroke-opacity:0.2;" x="422.549249" xlink:href="#m4f5604d9a2" y="249.697002"/>
+ <use style="fill:#8c564b;fill-opacity:0.2;stroke:#8c564b;stroke-opacity:0.2;" x="423.051727" xlink:href="#m4f5604d9a2" y="248.973643"/>
+ <use style="fill:#8c564b;fill-opacity:0.2;stroke:#8c564b;stroke-opacity:0.2;" x="423.54137" xlink:href="#m4f5604d9a2" y="248.246639"/>
+ <use style="fill:#8c564b;fill-opacity:0.2;stroke:#8c564b;stroke-opacity:0.2;" x="424.018043" xlink:href="#m4f5604d9a2" y="247.516008"/>
+ <use style="fill:#8c564b;fill-opacity:0.2;stroke:#8c564b;stroke-opacity:0.2;" x="424.481612" xlink:href="#m4f5604d9a2" y="246.781769"/>
+ <use style="fill:#8c564b;fill-opacity:0.2;stroke:#8c564b;stroke-opacity:0.2;" x="424.931948" xlink:href="#m4f5604d9a2" y="246.043942"/>
+ <use style="fill:#8c564b;fill-opacity:0.2;stroke:#8c564b;stroke-opacity:0.2;" x="425.368926" xlink:href="#m4f5604d9a2" y="245.302551"/>
+ <use style="fill:#8c564b;fill-opacity:0.2;stroke:#8c564b;stroke-opacity:0.2;" x="425.792425" xlink:href="#m4f5604d9a2" y="244.557619"/>
+ <use style="fill:#8c564b;fill-opacity:0.2;stroke:#8c564b;stroke-opacity:0.2;" x="426.202326" xlink:href="#m4f5604d9a2" y="243.809169"/>
+ <use style="fill:#8c564b;fill-opacity:0.2;stroke:#8c564b;stroke-opacity:0.2;" x="426.598518" xlink:href="#m4f5604d9a2" y="243.05723"/>
+ <use style="fill:#8c564b;fill-opacity:0.2;stroke:#8c564b;stroke-opacity:0.2;" x="426.980892" xlink:href="#m4f5604d9a2" y="242.301826"/>
+ <use style="fill:#8c564b;fill-opacity:0.2;stroke:#8c564b;stroke-opacity:0.2;" x="427.349347" xlink:href="#m4f5604d9a2" y="241.542987"/>
+ <use style="fill:#8c564b;fill-opacity:0.2;stroke:#8c564b;stroke-opacity:0.2;" x="427.703784" xlink:href="#m4f5604d9a2" y="240.780742"/>
+ <use style="fill:#8c564b;fill-opacity:0.2;stroke:#8c564b;stroke-opacity:0.2;" x="428.044113" xlink:href="#m4f5604d9a2" y="240.015121"/>
+ <use style="fill:#8c564b;fill-opacity:0.2;stroke:#8c564b;stroke-opacity:0.2;" x="428.370246" xlink:href="#m4f5604d9a2" y="239.246155"/>
+ <use style="fill:#8c564b;fill-opacity:0.2;stroke:#8c564b;stroke-opacity:0.2;" x="428.682103" xlink:href="#m4f5604d9a2" y="238.473876"/>
+ <use style="fill:#8c564b;fill-opacity:0.2;stroke:#8c564b;stroke-opacity:0.2;" x="428.97961" xlink:href="#m4f5604d9a2" y="237.698315"/>
+ <use style="fill:#8c564b;fill-opacity:0.2;stroke:#8c564b;stroke-opacity:0.2;" x="429.262698" xlink:href="#m4f5604d9a2" y="236.919506"/>
+ <use style="fill:#8c564b;fill-opacity:0.2;stroke:#8c564b;stroke-opacity:0.2;" x="429.531304" xlink:href="#m4f5604d9a2" y="236.137483"/>
+ <use style="fill:#8c564b;fill-opacity:0.2;stroke:#8c564b;stroke-opacity:0.2;" x="429.785368" xlink:href="#m4f5604d9a2" y="235.352279"/>
+ <use style="fill:#8c564b;fill-opacity:0.2;stroke:#8c564b;stroke-opacity:0.2;" x="430.024836" xlink:href="#m4f5604d9a2" y="234.563931"/>
+ <use style="fill:#8c564b;fill-opacity:0.2;stroke:#8c564b;stroke-opacity:0.2;" x="430.249657" xlink:href="#m4f5604d9a2" y="233.772472"/>
+ <use style="fill:#8c564b;fill-opacity:0.2;stroke:#8c564b;stroke-opacity:0.2;" x="430.459786" xlink:href="#m4f5604d9a2" y="232.977941"/>
+ <use style="fill:#8c564b;fill-opacity:0.2;stroke:#8c564b;stroke-opacity:0.2;" x="430.655185" xlink:href="#m4f5604d9a2" y="232.180372"/>
+ <use style="fill:#8c564b;fill-opacity:0.2;stroke:#8c564b;stroke-opacity:0.2;" x="430.835818" xlink:href="#m4f5604d9a2" y="231.379803"/>
+ <use style="fill:#8c564b;fill-opacity:0.2;stroke:#8c564b;stroke-opacity:0.2;" x="431.00166" xlink:href="#m4f5604d9a2" y="230.57627"/>
+ <use style="fill:#8c564b;fill-opacity:0.2;stroke:#8c564b;stroke-opacity:0.2;" x="431.152691" xlink:href="#m4f5604d9a2" y="229.769811"/>
+ <use style="fill:#8c564b;fill-opacity:0.2;stroke:#8c564b;stroke-opacity:0.2;" x="431.288898" xlink:href="#m4f5604d9a2" y="228.960463"/>
+ <use style="fill:#8c564b;fill-opacity:0.2;stroke:#8c564b;stroke-opacity:0.2;" x="431.410277" xlink:href="#m4f5604d9a2" y="228.148262"/>
+ <use style="fill:#8c564b;fill-opacity:0.2;stroke:#8c564b;stroke-opacity:0.2;" x="431.51683" xlink:href="#m4f5604d9a2" y="227.333244"/>
+ <use style="fill:#8c564b;fill-opacity:0.2;stroke:#8c564b;stroke-opacity:0.2;" x="431.608568" xlink:href="#m4f5604d9a2" y="226.515447"/>
+ <use style="fill:#8c564b;fill-opacity:0.2;stroke:#8c564b;stroke-opacity:0.2;" x="431.685511" xlink:href="#m4f5604d9a2" y="225.694906"/>
+ <use style="fill:#8c564b;fill-opacity:0.2;stroke:#8c564b;stroke-opacity:0.2;" x="431.747685" xlink:href="#m4f5604d9a2" y="224.871657"/>
+ <use style="fill:#8c564b;fill-opacity:0.2;stroke:#8c564b;stroke-opacity:0.2;" x="431.795128" xlink:href="#m4f5604d9a2" y="224.045736"/>
+ <use style="fill:#8c564b;fill-opacity:0.2;stroke:#8c564b;stroke-opacity:0.2;" x="431.827883" xlink:href="#m4f5604d9a2" y="223.217176"/>
+ <use style="fill:#8c564b;fill-opacity:0.2;stroke:#8c564b;stroke-opacity:0.2;" x="431.846002" xlink:href="#m4f5604d9a2" y="222.386013"/>
+ <use style="fill:#8c564b;fill-opacity:0.2;stroke:#8c564b;stroke-opacity:0.2;" x="431.849545" xlink:href="#m4f5604d9a2" y="221.552281"/>
+ <use style="fill:#8c564b;fill-opacity:0.2;stroke:#8c564b;stroke-opacity:0.2;" x="431.838582" xlink:href="#m4f5604d9a2" y="220.716014"/>
+ <use style="fill:#8c564b;fill-opacity:0.2;stroke:#8c564b;stroke-opacity:0.2;" x="431.813186" xlink:href="#m4f5604d9a2" y="219.877247"/>
+ <use style="fill:#8c564b;fill-opacity:0.2;stroke:#8c564b;stroke-opacity:0.2;" x="431.773441" xlink:href="#m4f5604d9a2" y="219.036014"/>
+ <use style="fill:#8c564b;fill-opacity:0.2;stroke:#8c564b;stroke-opacity:0.2;" x="431.719438" xlink:href="#m4f5604d9a2" y="218.192349"/>
+ <use style="fill:#8c564b;fill-opacity:0.2;stroke:#8c564b;stroke-opacity:0.2;" x="431.651273" xlink:href="#m4f5604d9a2" y="217.346285"/>
+ <use style="fill:#8c564b;fill-opacity:0.2;stroke:#8c564b;stroke-opacity:0.2;" x="431.56905" xlink:href="#m4f5604d9a2" y="216.497856"/>
+ <use style="fill:#8c564b;fill-opacity:0.2;stroke:#8c564b;stroke-opacity:0.2;" x="431.47288" xlink:href="#m4f5604d9a2" y="215.647097"/>
+ <use style="fill:#8c564b;fill-opacity:0.2;stroke:#8c564b;stroke-opacity:0.2;" x="431.362878" xlink:href="#m4f5604d9a2" y="214.794041"/>
+ <use style="fill:#8c564b;fill-opacity:0.2;stroke:#8c564b;stroke-opacity:0.2;" x="431.239165" xlink:href="#m4f5604d9a2" y="213.938723"/>
+ <use style="fill:#8c564b;fill-opacity:0.2;stroke:#8c564b;stroke-opacity:0.2;" x="431.101867" xlink:href="#m4f5604d9a2" y="213.081175"/>
+ <use style="fill:#8c564b;fill-opacity:0.2;stroke:#8c564b;stroke-opacity:0.2;" x="430.951117" xlink:href="#m4f5604d9a2" y="212.221433"/>
+ <use style="fill:#8c564b;fill-opacity:0.2;stroke:#8c564b;stroke-opacity:0.2;" x="430.787047" xlink:href="#m4f5604d9a2" y="211.359531"/>
+ <use style="fill:#8c564b;fill-opacity:0.2;stroke:#8c564b;stroke-opacity:0.2;" x="430.609797" xlink:href="#m4f5604d9a2" y="210.495504"/>
+ <use style="fill:#8c564b;fill-opacity:0.2;stroke:#8c564b;stroke-opacity:0.2;" x="430.419507" xlink:href="#m4f5604d9a2" y="209.629386"/>
+ <use style="fill:#8c564b;fill-opacity:0.2;stroke:#8c564b;stroke-opacity:0.2;" x="430.216323" xlink:href="#m4f5604d9a2" y="208.761213"/>
+ <use style="fill:#8c564b;fill-opacity:0.2;stroke:#8c564b;stroke-opacity:0.2;" x="430.00039" xlink:href="#m4f5604d9a2" y="207.891021"/>
+ <use style="fill:#8c564b;fill-opacity:0.2;stroke:#8c564b;stroke-opacity:0.2;" x="429.771856" xlink:href="#m4f5604d9a2" y="207.018845"/>
+ <use style="fill:#8c564b;fill-opacity:0.2;stroke:#8c564b;stroke-opacity:0.2;" x="429.530868" xlink:href="#m4f5604d9a2" y="206.144723"/>
+ <use style="fill:#8c564b;fill-opacity:0.2;stroke:#8c564b;stroke-opacity:0.2;" x="429.277577" xlink:href="#m4f5604d9a2" y="205.268693"/>
+ <use style="fill:#8c564b;fill-opacity:0.2;stroke:#8c564b;stroke-opacity:0.2;" x="429.012131" xlink:href="#m4f5604d9a2" y="204.390791"/>
+ <use style="fill:#8c564b;fill-opacity:0.2;stroke:#8c564b;stroke-opacity:0.2;" x="428.73468" xlink:href="#m4f5604d9a2" y="203.511056"/>
+ <use style="fill:#8c564b;fill-opacity:0.2;stroke:#8c564b;stroke-opacity:0.2;" x="428.44537" xlink:href="#m4f5604d9a2" y="202.629529"/>
+ <use style="fill:#8c564b;fill-opacity:0.2;stroke:#8c564b;stroke-opacity:0.2;" x="428.14435" xlink:href="#m4f5604d9a2" y="201.746248"/>
+ <use style="fill:#8c564b;fill-opacity:0.2;stroke:#8c564b;stroke-opacity:0.2;" x="427.831762" xlink:href="#m4f5604d9a2" y="200.861255"/>
+ <use style="fill:#8c564b;fill-opacity:0.2;stroke:#8c564b;stroke-opacity:0.2;" x="427.507751" xlink:href="#m4f5604d9a2" y="199.974591"/>
+ <use style="fill:#8c564b;fill-opacity:0.2;stroke:#8c564b;stroke-opacity:0.2;" x="427.172457" xlink:href="#m4f5604d9a2" y="199.086299"/>
+ <use style="fill:#8c564b;fill-opacity:0.2;stroke:#8c564b;stroke-opacity:0.2;" x="426.826017" xlink:href="#m4f5604d9a2" y="198.196422"/>
+ <use style="fill:#8c564b;fill-opacity:0.2;stroke:#8c564b;stroke-opacity:0.2;" x="426.468567" xlink:href="#m4f5604d9a2" y="197.305004"/>
+ <use style="fill:#8c564b;fill-opacity:0.2;stroke:#8c564b;stroke-opacity:0.2;" x="426.100236" xlink:href="#m4f5604d9a2" y="196.41209"/>
+ <use style="fill:#8c564b;fill-opacity:0.2;stroke:#8c564b;stroke-opacity:0.2;" x="425.721154" xlink:href="#m4f5604d9a2" y="195.517727"/>
+ <use style="fill:#8c564b;fill-opacity:0.2;stroke:#8c564b;stroke-opacity:0.2;" x="425.331445" xlink:href="#m4f5604d9a2" y="194.62196"/>
+ <use style="fill:#8c564b;fill-opacity:0.2;stroke:#8c564b;stroke-opacity:0.2;" x="424.931229" xlink:href="#m4f5604d9a2" y="193.724838"/>
+ <use style="fill:#8c564b;fill-opacity:0.2;stroke:#8c564b;stroke-opacity:0.2;" x="424.520622" xlink:href="#m4f5604d9a2" y="192.826409"/>
+ <use style="fill:#8c564b;fill-opacity:0.2;stroke:#8c564b;stroke-opacity:0.2;" x="424.099736" xlink:href="#m4f5604d9a2" y="191.926722"/>
+ <use style="fill:#8c564b;fill-opacity:0.2;stroke:#8c564b;stroke-opacity:0.2;" x="423.668681" xlink:href="#m4f5604d9a2" y="191.025828"/>
+ <use style="fill:#8c564b;fill-opacity:0.2;stroke:#8c564b;stroke-opacity:0.2;" x="423.22756" xlink:href="#m4f5604d9a2" y="190.123778"/>
+ <use style="fill:#8c564b;fill-opacity:0.2;stroke:#8c564b;stroke-opacity:0.2;" x="422.776474" xlink:href="#m4f5604d9a2" y="189.220623"/>
+ <use style="fill:#8c564b;fill-opacity:0.2;stroke:#8c564b;stroke-opacity:0.2;" x="422.315517" xlink:href="#m4f5604d9a2" y="188.316417"/>
+ <use style="fill:#8c564b;fill-opacity:0.2;stroke:#8c564b;stroke-opacity:0.2;" x="421.844783" xlink:href="#m4f5604d9a2" y="187.411211"/>
+ <use style="fill:#8c564b;fill-opacity:0.2;stroke:#8c564b;stroke-opacity:0.2;" x="421.364359" xlink:href="#m4f5604d9a2" y="186.505061"/>
+ <use style="fill:#8c564b;fill-opacity:0.2;stroke:#8c564b;stroke-opacity:0.2;" x="420.87433" xlink:href="#m4f5604d9a2" y="185.598021"/>
+ <use style="fill:#8c564b;fill-opacity:0.2;stroke:#8c564b;stroke-opacity:0.2;" x="420.374775" xlink:href="#m4f5604d9a2" y="184.690145"/>
+ <use style="fill:#8c564b;fill-opacity:0.2;stroke:#8c564b;stroke-opacity:0.2;" x="419.865771" xlink:href="#m4f5604d9a2" y="183.781491"/>
+ <use style="fill:#8c564b;fill-opacity:0.2;stroke:#8c564b;stroke-opacity:0.2;" x="419.347392" xlink:href="#m4f5604d9a2" y="182.872114"/>
+ <use style="fill:#8c564b;fill-opacity:0.2;stroke:#8c564b;stroke-opacity:0.2;" x="418.819706" xlink:href="#m4f5604d9a2" y="181.962071"/>
+ <use style="fill:#8c564b;fill-opacity:0.2;stroke:#8c564b;stroke-opacity:0.2;" x="418.282781" xlink:href="#m4f5604d9a2" y="181.05142"/>
+ <use style="fill:#8c564b;fill-opacity:0.2;stroke:#8c564b;stroke-opacity:0.2;" x="417.73668" xlink:href="#m4f5604d9a2" y="180.140219"/>
+ <use style="fill:#8c564b;fill-opacity:0.2;stroke:#8c564b;stroke-opacity:0.2;" x="417.181472" xlink:href="#m4f5604d9a2" y="179.228526"/>
+ <use style="fill:#8c564b;fill-opacity:0.2;stroke:#8c564b;stroke-opacity:0.2;" x="416.617223" xlink:href="#m4f5604d9a2" y="178.316401"/>
+ <use style="fill:#8c564b;fill-opacity:0.2;stroke:#8c564b;stroke-opacity:0.2;" x="416.043998" xlink:href="#m4f5604d9a2" y="177.403905"/>
+ <use style="fill:#8c564b;fill-opacity:0.2;stroke:#8c564b;stroke-opacity:0.2;" x="415.461864" xlink:href="#m4f5604d9a2" y="176.491098"/>
+ <use style="fill:#8c564b;fill-opacity:0.2;stroke:#8c564b;stroke-opacity:0.2;" x="414.870888" xlink:href="#m4f5604d9a2" y="175.578042"/>
+ <use style="fill:#8c564b;fill-opacity:0.2;stroke:#8c564b;stroke-opacity:0.2;" x="414.271133" xlink:href="#m4f5604d9a2" y="174.664797"/>
+ <use style="fill:#8c564b;fill-opacity:0.2;stroke:#8c564b;stroke-opacity:0.2;" x="413.662664" xlink:href="#m4f5604d9a2" y="173.751427"/>
+ <use style="fill:#8c564b;fill-opacity:0.2;stroke:#8c564b;stroke-opacity:0.2;" x="413.045544" xlink:href="#m4f5604d9a2" y="172.837994"/>
+ <use style="fill:#8c564b;fill-opacity:0.2;stroke:#8c564b;stroke-opacity:0.2;" x="412.419834" xlink:href="#m4f5604d9a2" y="171.92456"/>
+ <use style="fill:#8c564b;fill-opacity:0.2;stroke:#8c564b;stroke-opacity:0.2;" x="411.785594" xlink:href="#m4f5604d9a2" y="171.01119"/>
+ <use style="fill:#8c564b;fill-opacity:0.2;stroke:#8c564b;stroke-opacity:0.2;" x="411.142882" xlink:href="#m4f5604d9a2" y="170.097946"/>
+ <use style="fill:#8c564b;fill-opacity:0.2;stroke:#8c564b;stroke-opacity:0.2;" x="410.491757" xlink:href="#m4f5604d9a2" y="169.184893"/>
+ <use style="fill:#8c564b;fill-opacity:0.2;stroke:#8c564b;stroke-opacity:0.2;" x="409.832274" xlink:href="#m4f5604d9a2" y="168.272095"/>
+ <use style="fill:#8c564b;fill-opacity:0.2;stroke:#8c564b;stroke-opacity:0.2;" x="409.164488" xlink:href="#m4f5604d9a2" y="167.359616"/>
+ <use style="fill:#8c564b;fill-opacity:0.2;stroke:#8c564b;stroke-opacity:0.2;" x="408.488452" xlink:href="#m4f5604d9a2" y="166.44752"/>
+ <use style="fill:#8c564b;fill-opacity:0.2;stroke:#8c564b;stroke-opacity:0.2;" x="407.804219" xlink:href="#m4f5604d9a2" y="165.535872"/>
+ <use style="fill:#8c564b;fill-opacity:0.2;stroke:#8c564b;stroke-opacity:0.2;" x="407.11184" xlink:href="#m4f5604d9a2" y="164.624738"/>
+ <use style="fill:#8c564b;fill-opacity:0.2;stroke:#8c564b;stroke-opacity:0.2;" x="406.411367" xlink:href="#m4f5604d9a2" y="163.714181"/>
+ <use style="fill:#8c564b;fill-opacity:0.2;stroke:#8c564b;stroke-opacity:0.2;" x="405.70285" xlink:href="#m4f5604d9a2" y="162.804268"/>
+ <use style="fill:#8c564b;fill-opacity:0.2;stroke:#8c564b;stroke-opacity:0.2;" x="404.986339" xlink:href="#m4f5604d9a2" y="161.895062"/>
+ <use style="fill:#8c564b;fill-opacity:0.2;stroke:#8c564b;stroke-opacity:0.2;" x="404.261883" xlink:href="#m4f5604d9a2" y="160.986631"/>
+ <use style="fill:#8c564b;fill-opacity:0.2;stroke:#8c564b;stroke-opacity:0.2;" x="403.529534" xlink:href="#m4f5604d9a2" y="160.079038"/>
+ <use style="fill:#8c564b;fill-opacity:0.2;stroke:#8c564b;stroke-opacity:0.2;" x="402.78934" xlink:href="#m4f5604d9a2" y="159.172349"/>
+ <use style="fill:#8c564b;fill-opacity:0.2;stroke:#8c564b;stroke-opacity:0.2;" x="402.041353" xlink:href="#m4f5604d9a2" y="158.266629"/>
+ <use style="fill:#8c564b;fill-opacity:0.2;stroke:#8c564b;stroke-opacity:0.2;" x="401.285625" xlink:href="#m4f5604d9a2" y="157.361944"/>
+ <use style="fill:#8c564b;fill-opacity:0.2;stroke:#8c564b;stroke-opacity:0.2;" x="400.522206" xlink:href="#m4f5604d9a2" y="156.458359"/>
+ <use style="fill:#8c564b;fill-opacity:0.2;stroke:#8c564b;stroke-opacity:0.2;" x="399.751152" xlink:href="#m4f5604d9a2" y="155.555938"/>
+ <use style="fill:#8c564b;fill-opacity:0.2;stroke:#8c564b;stroke-opacity:0.2;" x="398.972516" xlink:href="#m4f5604d9a2" y="154.654748"/>
+ <use style="fill:#8c564b;fill-opacity:0.2;stroke:#8c564b;stroke-opacity:0.2;" x="398.186355" xlink:href="#m4f5604d9a2" y="153.754852"/>
+ <use style="fill:#8c564b;fill-opacity:0.2;stroke:#8c564b;stroke-opacity:0.2;" x="397.392727" xlink:href="#m4f5604d9a2" y="152.856315"/>
+ <use style="fill:#8c564b;fill-opacity:0.2;stroke:#8c564b;stroke-opacity:0.2;" x="396.591691" xlink:href="#m4f5604d9a2" y="151.959203"/>
+ <use style="fill:#8c564b;fill-opacity:0.2;stroke:#8c564b;stroke-opacity:0.2;" x="395.78331" xlink:href="#m4f5604d9a2" y="151.063579"/>
+ <use style="fill:#8c564b;fill-opacity:0.2;stroke:#8c564b;stroke-opacity:0.2;" x="394.967648" xlink:href="#m4f5604d9a2" y="150.169507"/>
+ <use style="fill:#8c564b;fill-opacity:0.2;stroke:#8c564b;stroke-opacity:0.2;" x="394.14477" xlink:href="#m4f5604d9a2" y="149.27705"/>
+ <use style="fill:#8c564b;fill-opacity:0.2;stroke:#8c564b;stroke-opacity:0.2;" x="393.314747" xlink:href="#m4f5604d9a2" y="148.386273"/>
+ <use style="fill:#8c564b;fill-opacity:0.2;stroke:#8c564b;stroke-opacity:0.2;" x="392.477649" xlink:href="#m4f5604d9a2" y="147.497239"/>
+ <use style="fill:#8c564b;fill-opacity:0.2;stroke:#8c564b;stroke-opacity:0.2;" x="391.63355" xlink:href="#m4f5604d9a2" y="146.610008"/>
+ <use style="fill:#8c564b;fill-opacity:0.2;stroke:#8c564b;stroke-opacity:0.2;" x="390.782529" xlink:href="#m4f5604d9a2" y="145.724645"/>
+ <use style="fill:#8c564b;fill-opacity:0.2;stroke:#8c564b;stroke-opacity:0.2;" x="389.924664" xlink:href="#m4f5604d9a2" y="144.841211"/>
+ <use style="fill:#8c564b;fill-opacity:0.2;stroke:#8c564b;stroke-opacity:0.2;" x="389.060039" xlink:href="#m4f5604d9a2" y="143.959766"/>
+ <use style="fill:#8c564b;fill-opacity:0.2;stroke:#8c564b;stroke-opacity:0.2;" x="388.188739" xlink:href="#m4f5604d9a2" y="143.080372"/>
+ <use style="fill:#8c564b;fill-opacity:0.2;stroke:#8c564b;stroke-opacity:0.2;" x="387.310855" xlink:href="#m4f5604d9a2" y="142.203089"/>
+ <use style="fill:#8c564b;fill-opacity:0.2;stroke:#8c564b;stroke-opacity:0.2;" x="386.426477" xlink:href="#m4f5604d9a2" y="141.327975"/>
+ <use style="fill:#8c564b;fill-opacity:0.2;stroke:#8c564b;stroke-opacity:0.2;" x="385.535702" xlink:href="#m4f5604d9a2" y="140.455091"/>
+ <use style="fill:#8c564b;fill-opacity:0.2;stroke:#8c564b;stroke-opacity:0.2;" x="384.638627" xlink:href="#m4f5604d9a2" y="139.584493"/>
+ <use style="fill:#8c564b;fill-opacity:0.2;stroke:#8c564b;stroke-opacity:0.2;" x="383.735354" xlink:href="#m4f5604d9a2" y="138.716241"/>
+ <use style="fill:#8c564b;fill-opacity:0.2;stroke:#8c564b;stroke-opacity:0.2;" x="382.825987" xlink:href="#m4f5604d9a2" y="137.85039"/>
+ <use style="fill:#8c564b;fill-opacity:0.2;stroke:#8c564b;stroke-opacity:0.2;" x="381.910636" xlink:href="#m4f5604d9a2" y="136.986997"/>
+ <use style="fill:#8c564b;fill-opacity:0.2;stroke:#8c564b;stroke-opacity:0.2;" x="380.989409" xlink:href="#m4f5604d9a2" y="136.126116"/>
+ <use style="fill:#8c564b;fill-opacity:0.2;stroke:#8c564b;stroke-opacity:0.2;" x="380.062422" xlink:href="#m4f5604d9a2" y="135.267804"/>
+ <use style="fill:#8c564b;fill-opacity:0.2;stroke:#8c564b;stroke-opacity:0.2;" x="379.12979" xlink:href="#m4f5604d9a2" y="134.412113"/>
+ <use style="fill:#8c564b;fill-opacity:0.2;stroke:#8c564b;stroke-opacity:0.2;" x="378.191634" xlink:href="#m4f5604d9a2" y="133.559096"/>
+ <use style="fill:#8c564b;fill-opacity:0.2;stroke:#8c564b;stroke-opacity:0.2;" x="377.248077" xlink:href="#m4f5604d9a2" y="132.708806"/>
+ <use style="fill:#8c564b;fill-opacity:0.2;stroke:#8c564b;stroke-opacity:0.2;" x="376.299243" xlink:href="#m4f5604d9a2" y="131.861293"/>
+ <use style="fill:#8c564b;fill-opacity:0.2;stroke:#8c564b;stroke-opacity:0.2;" x="375.345262" xlink:href="#m4f5604d9a2" y="131.016608"/>
+ <use style="fill:#8c564b;fill-opacity:0.2;stroke:#8c564b;stroke-opacity:0.2;" x="374.386264" xlink:href="#m4f5604d9a2" y="130.174801"/>
+ <use style="fill:#8c564b;fill-opacity:0.2;stroke:#8c564b;stroke-opacity:0.2;" x="373.422383" xlink:href="#m4f5604d9a2" y="129.33592"/>
+ <use style="fill:#8c564b;fill-opacity:0.2;stroke:#8c564b;stroke-opacity:0.2;" x="372.453755" xlink:href="#m4f5604d9a2" y="128.500013"/>
+ <use style="fill:#8c564b;fill-opacity:0.2;stroke:#8c564b;stroke-opacity:0.2;" x="371.48052" xlink:href="#m4f5604d9a2" y="127.667125"/>
+ <use style="fill:#8c564b;fill-opacity:0.2;stroke:#8c564b;stroke-opacity:0.2;" x="370.502818" xlink:href="#m4f5604d9a2" y="126.837304"/>
+ <use style="fill:#8c564b;fill-opacity:0.2;stroke:#8c564b;stroke-opacity:0.2;" x="369.520794" xlink:href="#m4f5604d9a2" y="126.010593"/>
+ <use style="fill:#8c564b;fill-opacity:0.2;stroke:#8c564b;stroke-opacity:0.2;" x="368.534594" xlink:href="#m4f5604d9a2" y="125.187036"/>
+ <use style="fill:#8c564b;fill-opacity:0.2;stroke:#8c564b;stroke-opacity:0.2;" x="367.544362" xlink:href="#m4f5604d9a2" y="124.366677"/>
+ <use style="fill:#8c564b;fill-opacity:0.2;stroke:#8c564b;stroke-opacity:0.2;" x="366.550246" xlink:href="#m4f5604d9a2" y="123.549558"/>
+ <use style="fill:#8c564b;fill-opacity:0.2;stroke:#8c564b;stroke-opacity:0.2;" x="365.552391" xlink:href="#m4f5604d9a2" y="122.735722"/>
+ <use style="fill:#8c564b;fill-opacity:0.2;stroke:#8c564b;stroke-opacity:0.2;" x="364.550943" xlink:href="#m4f5604d9a2" y="121.92521"/>
+ <use style="fill:#8c564b;fill-opacity:0.2;stroke:#8c564b;stroke-opacity:0.2;" x="363.546048" xlink:href="#m4f5604d9a2" y="121.118064"/>
+ <use style="fill:#8c564b;fill-opacity:0.2;stroke:#8c564b;stroke-opacity:0.2;" x="362.537851" xlink:href="#m4f5604d9a2" y="120.314325"/>
+ <use style="fill:#8c564b;fill-opacity:0.2;stroke:#8c564b;stroke-opacity:0.2;" x="361.526494" xlink:href="#m4f5604d9a2" y="119.514033"/>
+ <use style="fill:#8c564b;fill-opacity:0.2;stroke:#8c564b;stroke-opacity:0.2;" x="360.512121" xlink:href="#m4f5604d9a2" y="118.717229"/>
+ <use style="fill:#8c564b;fill-opacity:0.2;stroke:#8c564b;stroke-opacity:0.2;" x="359.494873" xlink:href="#m4f5604d9a2" y="117.923952"/>
+ <use style="fill:#8c564b;fill-opacity:0.2;stroke:#8c564b;stroke-opacity:0.2;" x="358.474888" xlink:href="#m4f5604d9a2" y="117.134241"/>
+ <use style="fill:#8c564b;fill-opacity:0.2;stroke:#8c564b;stroke-opacity:0.2;" x="357.452306" xlink:href="#m4f5604d9a2" y="116.348135"/>
+ <use style="fill:#8c564b;fill-opacity:0.2;stroke:#8c564b;stroke-opacity:0.2;" x="356.42726" xlink:href="#m4f5604d9a2" y="115.565673"/>
+ <use style="fill:#8c564b;fill-opacity:0.2;stroke:#8c564b;stroke-opacity:0.2;" x="355.399886" xlink:href="#m4f5604d9a2" y="114.786892"/>
+ <use style="fill:#8c564b;fill-opacity:0.2;stroke:#8c564b;stroke-opacity:0.2;" x="354.370314" xlink:href="#m4f5604d9a2" y="114.011831"/>
+ <use style="fill:#8c564b;fill-opacity:0.2;stroke:#8c564b;stroke-opacity:0.2;" x="353.338672" xlink:href="#m4f5604d9a2" y="113.240527"/>
+ <use style="fill:#8c564b;fill-opacity:0.2;stroke:#8c564b;stroke-opacity:0.2;" x="352.305087" xlink:href="#m4f5604d9a2" y="112.473017"/>
+ <use style="fill:#8c564b;fill-opacity:0.2;stroke:#8c564b;stroke-opacity:0.2;" x="351.269682" xlink:href="#m4f5604d9a2" y="111.709337"/>
+ <use style="fill:#8c564b;fill-opacity:0.2;stroke:#8c564b;stroke-opacity:0.2;" x="350.232575" xlink:href="#m4f5604d9a2" y="110.949523"/>
+ <use style="fill:#8c564b;fill-opacity:0.2;stroke:#8c564b;stroke-opacity:0.2;" x="349.193886" xlink:href="#m4f5604d9a2" y="110.193613"/>
+ <use style="fill:#8c564b;fill-opacity:0.2;stroke:#8c564b;stroke-opacity:0.2;" x="348.153728" xlink:href="#m4f5604d9a2" y="109.44164"/>
+ <use style="fill:#8c564b;fill-opacity:0.2;stroke:#8c564b;stroke-opacity:0.2;" x="347.112211" xlink:href="#m4f5604d9a2" y="108.69364"/>
+ <use style="fill:#8c564b;fill-opacity:0.2;stroke:#8c564b;stroke-opacity:0.2;" x="346.069444" xlink:href="#m4f5604d9a2" y="107.949647"/>
+ <use style="fill:#8c564b;fill-opacity:0.2;stroke:#8c564b;stroke-opacity:0.2;" x="345.025532" xlink:href="#m4f5604d9a2" y="107.209697"/>
+ <use style="fill:#8c564b;fill-opacity:0.2;stroke:#8c564b;stroke-opacity:0.2;" x="343.980576" xlink:href="#m4f5604d9a2" y="106.473823"/>
+ <use style="fill:#8c564b;fill-opacity:0.2;stroke:#8c564b;stroke-opacity:0.2;" x="342.934674" xlink:href="#m4f5604d9a2" y="105.742058"/>
+ <use style="fill:#8c564b;fill-opacity:0.2;stroke:#8c564b;stroke-opacity:0.2;" x="341.887922" xlink:href="#m4f5604d9a2" y="105.014436"/>
+ <use style="fill:#8c564b;fill-opacity:0.2;stroke:#8c564b;stroke-opacity:0.2;" x="340.840412" xlink:href="#m4f5604d9a2" y="104.290988"/>
+ <use style="fill:#8c564b;fill-opacity:0.2;stroke:#8c564b;stroke-opacity:0.2;" x="339.792233" xlink:href="#m4f5604d9a2" y="103.571747"/>
+ <use style="fill:#8c564b;fill-opacity:0.2;stroke:#8c564b;stroke-opacity:0.2;" x="338.743473" xlink:href="#m4f5604d9a2" y="102.856745"/>
+ <use style="fill:#8c564b;fill-opacity:0.2;stroke:#8c564b;stroke-opacity:0.2;" x="337.694215" xlink:href="#m4f5604d9a2" y="102.146012"/>
+ <use style="fill:#8c564b;fill-opacity:0.2;stroke:#8c564b;stroke-opacity:0.2;" x="336.644539" xlink:href="#m4f5604d9a2" y="101.439579"/>
+ <use style="fill:#8c564b;fill-opacity:0.2;stroke:#8c564b;stroke-opacity:0.2;" x="335.594526" xlink:href="#m4f5604d9a2" y="100.737476"/>
+ <use style="fill:#8c564b;fill-opacity:0.2;stroke:#8c564b;stroke-opacity:0.2;" x="334.544251" xlink:href="#m4f5604d9a2" y="100.039732"/>
+ <use style="fill:#8c564b;fill-opacity:0.2;stroke:#8c564b;stroke-opacity:0.2;" x="333.493788" xlink:href="#m4f5604d9a2" y="99.346376"/>
+ <use style="fill:#8c564b;fill-opacity:0.2;stroke:#8c564b;stroke-opacity:0.2;" x="332.443209" xlink:href="#m4f5604d9a2" y="98.657437"/>
+ <use style="fill:#8c564b;fill-opacity:0.2;stroke:#8c564b;stroke-opacity:0.2;" x="331.392585" xlink:href="#m4f5604d9a2" y="97.972941"/>
+ <use style="fill:#8c564b;fill-opacity:0.2;stroke:#8c564b;stroke-opacity:0.2;" x="330.341983" xlink:href="#m4f5604d9a2" y="97.292916"/>
+ <use style="fill:#8c564b;fill-opacity:0.2;stroke:#8c564b;stroke-opacity:0.2;" x="329.29147" xlink:href="#m4f5604d9a2" y="96.617388"/>
+ <use style="fill:#8c564b;fill-opacity:0.2;stroke:#8c564b;stroke-opacity:0.2;" x="328.241112" xlink:href="#m4f5604d9a2" y="95.946383"/>
+ <use style="fill:#8c564b;fill-opacity:0.2;stroke:#8c564b;stroke-opacity:0.2;" x="327.190971" xlink:href="#m4f5604d9a2" y="95.279924"/>
+ <use style="fill:#8c564b;fill-opacity:0.2;stroke:#8c564b;stroke-opacity:0.2;" x="326.141112" xlink:href="#m4f5604d9a2" y="94.618036"/>
+ <use style="fill:#8c564b;fill-opacity:0.2;stroke:#8c564b;stroke-opacity:0.2;" x="325.091595" xlink:href="#m4f5604d9a2" y="93.960743"/>
+ <use style="fill:#8c564b;fill-opacity:0.2;stroke:#8c564b;stroke-opacity:0.2;" x="324.042481" xlink:href="#m4f5604d9a2" y="93.308067"/>
+ <use style="fill:#8c564b;fill-opacity:0.2;stroke:#8c564b;stroke-opacity:0.2;" x="322.99383" xlink:href="#m4f5604d9a2" y="92.660029"/>
+ <use style="fill:#8c564b;fill-opacity:0.2;stroke:#8c564b;stroke-opacity:0.2;" x="321.945702" xlink:href="#m4f5604d9a2" y="92.016651"/>
+ <use style="fill:#8c564b;fill-opacity:0.2;stroke:#8c564b;stroke-opacity:0.2;" x="320.898155" xlink:href="#m4f5604d9a2" y="91.377952"/>
+ <use style="fill:#8c564b;fill-opacity:0.2;stroke:#8c564b;stroke-opacity:0.2;" x="319.851249" xlink:href="#m4f5604d9a2" y="90.743951"/>
+ <use style="fill:#8c564b;fill-opacity:0.2;stroke:#8c564b;stroke-opacity:0.2;" x="318.805041" xlink:href="#m4f5604d9a2" y="90.114667"/>
+ <use style="fill:#8c564b;fill-opacity:0.2;stroke:#8c564b;stroke-opacity:0.2;" x="317.759589" xlink:href="#m4f5604d9a2" y="89.490117"/>
+ <use style="fill:#8c564b;fill-opacity:0.2;stroke:#8c564b;stroke-opacity:0.2;" x="316.714953" xlink:href="#m4f5604d9a2" y="88.870317"/>
+ <use style="fill:#8c564b;fill-opacity:0.2;stroke:#8c564b;stroke-opacity:0.2;" x="315.671188" xlink:href="#m4f5604d9a2" y="88.255284"/>
+ <use style="fill:#8c564b;fill-opacity:0.2;stroke:#8c564b;stroke-opacity:0.2;" x="314.628355" xlink:href="#m4f5604d9a2" y="87.645031"/>
+ <use style="fill:#8c564b;fill-opacity:0.2;stroke:#8c564b;stroke-opacity:0.2;" x="313.58651" xlink:href="#m4f5604d9a2" y="87.039572"/>
+ <use style="fill:#8c564b;fill-opacity:0.2;stroke:#8c564b;stroke-opacity:0.2;" x="312.545713" xlink:href="#m4f5604d9a2" y="86.438919"/>
+ <use style="fill:#8c564b;fill-opacity:0.2;stroke:#8c564b;stroke-opacity:0.2;" x="311.506021" xlink:href="#m4f5604d9a2" y="85.843085"/>
+ <use style="fill:#8c564b;fill-opacity:0.2;stroke:#8c564b;stroke-opacity:0.2;" x="310.467493" xlink:href="#m4f5604d9a2" y="85.25208"/>
+ <use style="fill:#8c564b;fill-opacity:0.2;stroke:#8c564b;stroke-opacity:0.2;" x="309.430188" xlink:href="#m4f5604d9a2" y="84.665913"/>
+ <use style="fill:#8c564b;fill-opacity:0.2;stroke:#8c564b;stroke-opacity:0.2;" x="308.394166" xlink:href="#m4f5604d9a2" y="84.084593"/>
+ <use style="fill:#8c564b;fill-opacity:0.2;stroke:#8c564b;stroke-opacity:0.2;" x="307.359484" xlink:href="#m4f5604d9a2" y="83.508127"/>
+ <use style="fill:#8c564b;fill-opacity:0.2;stroke:#8c564b;stroke-opacity:0.2;" x="306.326204" xlink:href="#m4f5604d9a2" y="82.936522"/>
+ <use style="fill:#8c564b;fill-opacity:0.2;stroke:#8c564b;stroke-opacity:0.2;" x="305.294383" xlink:href="#m4f5604d9a2" y="82.369783"/>
+ <use style="fill:#8c564b;fill-opacity:0.2;stroke:#8c564b;stroke-opacity:0.2;" x="304.264081" xlink:href="#m4f5604d9a2" y="81.807915"/>
+ <use style="fill:#8c564b;fill-opacity:0.2;stroke:#8c564b;stroke-opacity:0.2;" x="303.235358" xlink:href="#m4f5604d9a2" y="81.25092"/>
+ <use style="fill:#8c564b;fill-opacity:0.2;stroke:#8c564b;stroke-opacity:0.2;" x="302.208273" xlink:href="#m4f5604d9a2" y="80.698801"/>
+ <use style="fill:#8c564b;fill-opacity:0.2;stroke:#8c564b;stroke-opacity:0.2;" x="301.182885" xlink:href="#m4f5604d9a2" y="80.151558"/>
+ <use style="fill:#8c564b;fill-opacity:0.2;stroke:#8c564b;stroke-opacity:0.2;" x="300.159254" xlink:href="#m4f5604d9a2" y="79.609193"/>
+ <use style="fill:#8c564b;fill-opacity:0.2;stroke:#8c564b;stroke-opacity:0.2;" x="299.137438" xlink:href="#m4f5604d9a2" y="79.071703"/>
+ <use style="fill:#8c564b;fill-opacity:0.2;stroke:#8c564b;stroke-opacity:0.2;" x="298.117494" xlink:href="#m4f5604d9a2" y="78.539086"/>
+ <use style="fill:#8c564b;fill-opacity:0.2;stroke:#8c564b;stroke-opacity:0.2;" x="297.099482" xlink:href="#m4f5604d9a2" y="78.01134"/>
+ <use style="fill:#8c564b;fill-opacity:0.2;stroke:#8c564b;stroke-opacity:0.2;" x="296.083459" xlink:href="#m4f5604d9a2" y="77.488462"/>
+ <use style="fill:#8c564b;fill-opacity:0.2;stroke:#8c564b;stroke-opacity:0.2;" x="295.069481" xlink:href="#m4f5604d9a2" y="76.970445"/>
+ <use style="fill:#8c564b;fill-opacity:0.2;stroke:#8c564b;stroke-opacity:0.2;" x="294.057605" xlink:href="#m4f5604d9a2" y="76.457286"/>
+ <use style="fill:#8c564b;fill-opacity:0.2;stroke:#8c564b;stroke-opacity:0.2;" x="293.047885" xlink:href="#m4f5604d9a2" y="75.948977"/>
+ <use style="fill:#8c564b;fill-opacity:0.2;stroke:#8c564b;stroke-opacity:0.2;" x="292.040378" xlink:href="#m4f5604d9a2" y="75.445511"/>
+ <use style="fill:#8c564b;fill-opacity:0.2;stroke:#8c564b;stroke-opacity:0.2;" x="291.035135" xlink:href="#m4f5604d9a2" y="74.946881"/>
+ <use style="fill:#8c564b;fill-opacity:0.2;stroke:#8c564b;stroke-opacity:0.2;" x="290.03221" xlink:href="#m4f5604d9a2" y="74.453076"/>
+ <use style="fill:#8c564b;fill-opacity:0.2;stroke:#8c564b;stroke-opacity:0.2;" x="289.031654" xlink:href="#m4f5604d9a2" y="73.964088"/>
+ <use style="fill:#8c564b;fill-opacity:0.2;stroke:#8c564b;stroke-opacity:0.2;" x="288.033517" xlink:href="#m4f5604d9a2" y="73.479906"/>
+ <use style="fill:#8c564b;fill-opacity:0.2;stroke:#8c564b;stroke-opacity:0.2;" x="287.037848" xlink:href="#m4f5604d9a2" y="73.000517"/>
+ <use style="fill:#8c564b;fill-opacity:0.2;stroke:#8c564b;stroke-opacity:0.2;" x="286.044694" xlink:href="#m4f5604d9a2" y="72.52591"/>
+ <use style="fill:#8c564b;fill-opacity:0.2;stroke:#8c564b;stroke-opacity:0.2;" x="285.0541" xlink:href="#m4f5604d9a2" y="72.056072"/>
+ <use style="fill:#8c564b;fill-opacity:0.2;stroke:#8c564b;stroke-opacity:0.2;" x="284.066111" xlink:href="#m4f5604d9a2" y="71.590988"/>
+ <use style="fill:#8c564b;fill-opacity:0.2;stroke:#8c564b;stroke-opacity:0.2;" x="283.080769" xlink:href="#m4f5604d9a2" y="71.130643"/>
+ <use style="fill:#8c564b;fill-opacity:0.2;stroke:#8c564b;stroke-opacity:0.2;" x="282.098114" xlink:href="#m4f5604d9a2" y="70.675023"/>
+ <use style="fill:#8c564b;fill-opacity:0.2;stroke:#8c564b;stroke-opacity:0.2;" x="281.118184" xlink:href="#m4f5604d9a2" y="70.224109"/>
+ <use style="fill:#8c564b;fill-opacity:0.2;stroke:#8c564b;stroke-opacity:0.2;" x="280.141017" xlink:href="#m4f5604d9a2" y="69.777886"/>
+ <use style="fill:#8c564b;fill-opacity:0.2;stroke:#8c564b;stroke-opacity:0.2;" x="279.166648" xlink:href="#m4f5604d9a2" y="69.336334"/>
+ <use style="fill:#8c564b;fill-opacity:0.2;stroke:#8c564b;stroke-opacity:0.2;" x="278.195109" xlink:href="#m4f5604d9a2" y="68.899435"/>
+ <use style="fill:#8c564b;fill-opacity:0.2;stroke:#8c564b;stroke-opacity:0.2;" x="277.226432" xlink:href="#m4f5604d9a2" y="68.467169"/>
+ <use style="fill:#8c564b;fill-opacity:0.2;stroke:#8c564b;stroke-opacity:0.2;" x="276.260646" xlink:href="#m4f5604d9a2" y="68.039516"/>
+ <use style="fill:#8c564b;fill-opacity:0.2;stroke:#8c564b;stroke-opacity:0.2;" x="275.29778" xlink:href="#m4f5604d9a2" y="67.616453"/>
+ <use style="fill:#8c564b;fill-opacity:0.2;stroke:#8c564b;stroke-opacity:0.2;" x="274.337859" xlink:href="#m4f5604d9a2" y="67.197959"/>
+ <use style="fill:#8c564b;fill-opacity:0.2;stroke:#8c564b;stroke-opacity:0.2;" x="273.380907" xlink:href="#m4f5604d9a2" y="66.78401"/>
+ <use style="fill:#8c564b;fill-opacity:0.2;stroke:#8c564b;stroke-opacity:0.2;" x="272.426946" xlink:href="#m4f5604d9a2" y="66.374582"/>
+ <use style="fill:#8c564b;fill-opacity:0.2;stroke:#8c564b;stroke-opacity:0.2;" x="271.475998" xlink:href="#m4f5604d9a2" y="65.969651"/>
+ <use style="fill:#8c564b;fill-opacity:0.2;stroke:#8c564b;stroke-opacity:0.2;" x="270.528081" xlink:href="#m4f5604d9a2" y="65.569191"/>
+ <use style="fill:#8c564b;fill-opacity:0.2;stroke:#8c564b;stroke-opacity:0.2;" x="269.583211" xlink:href="#m4f5604d9a2" y="65.173176"/>
+ <use style="fill:#8c564b;fill-opacity:0.2;stroke:#8c564b;stroke-opacity:0.2;" x="268.641403" xlink:href="#m4f5604d9a2" y="64.781577"/>
+ <use style="fill:#8c564b;fill-opacity:0.2;stroke:#8c564b;stroke-opacity:0.2;" x="267.70267" xlink:href="#m4f5604d9a2" y="64.394369"/>
+ <use style="fill:#8c564b;fill-opacity:0.2;stroke:#8c564b;stroke-opacity:0.2;" x="266.767023" xlink:href="#m4f5604d9a2" y="64.011521"/>
+ <use style="fill:#8c564b;fill-opacity:0.2;stroke:#8c564b;stroke-opacity:0.2;" x="265.834472" xlink:href="#m4f5604d9a2" y="63.633004"/>
+ <use style="fill:#8c564b;fill-opacity:0.2;stroke:#8c564b;stroke-opacity:0.2;" x="264.905024" xlink:href="#m4f5604d9a2" y="63.258789"/>
+ <use style="fill:#8c564b;fill-opacity:0.2;stroke:#8c564b;stroke-opacity:0.2;" x="263.978683" xlink:href="#m4f5604d9a2" y="62.888843"/>
+ <use style="fill:#8c564b;fill-opacity:0.2;stroke:#8c564b;stroke-opacity:0.2;" x="263.055453" xlink:href="#m4f5604d9a2" y="62.523137"/>
+ <use style="fill:#8c564b;fill-opacity:0.2;stroke:#8c564b;stroke-opacity:0.2;" x="262.135335" xlink:href="#m4f5604d9a2" y="62.161636"/>
+ <use style="fill:#8c564b;fill-opacity:0.2;stroke:#8c564b;stroke-opacity:0.2;" x="261.218329" xlink:href="#m4f5604d9a2" y="61.804309"/>
+ <use style="fill:#8c564b;fill-opacity:0.2;stroke:#8c564b;stroke-opacity:0.2;" x="260.304432" xlink:href="#m4f5604d9a2" y="61.451122"/>
+ <use style="fill:#8c564b;fill-opacity:0.2;stroke:#8c564b;stroke-opacity:0.2;" x="259.39364" xlink:href="#m4f5604d9a2" y="61.10204"/>
+ <use style="fill:#8c564b;fill-opacity:0.2;stroke:#8c564b;stroke-opacity:0.2;" x="258.485946" xlink:href="#m4f5604d9a2" y="60.757029"/>
+ <use style="fill:#8c564b;fill-opacity:0.2;stroke:#8c564b;stroke-opacity:0.2;" x="257.581341" xlink:href="#m4f5604d9a2" y="60.416053"/>
+ <use style="fill:#8c564b;fill-opacity:0.2;stroke:#8c564b;stroke-opacity:0.2;" x="256.679815" xlink:href="#m4f5604d9a2" y="60.079076"/>
+ <use style="fill:#8c564b;fill-opacity:0.2;stroke:#8c564b;stroke-opacity:0.2;" x="255.781356" xlink:href="#m4f5604d9a2" y="59.746061"/>
+ <use style="fill:#8c564b;fill-opacity:0.2;stroke:#8c564b;stroke-opacity:0.2;" x="254.885949" xlink:href="#m4f5604d9a2" y="59.416972"/>
+ <use style="fill:#8c564b;fill-opacity:0.2;stroke:#8c564b;stroke-opacity:0.2;" x="253.993578" xlink:href="#m4f5604d9a2" y="59.091771"/>
+ <use style="fill:#8c564b;fill-opacity:0.2;stroke:#8c564b;stroke-opacity:0.2;" x="253.104225" xlink:href="#m4f5604d9a2" y="58.770419"/>
+ <use style="fill:#8c564b;fill-opacity:0.2;stroke:#8c564b;stroke-opacity:0.2;" x="252.21787" xlink:href="#m4f5604d9a2" y="58.452877"/>
+ <use style="fill:#8c564b;fill-opacity:0.2;stroke:#8c564b;stroke-opacity:0.2;" x="251.33449" xlink:href="#m4f5604d9a2" y="58.139108"/>
+ <use style="fill:#8c564b;fill-opacity:0.2;stroke:#8c564b;stroke-opacity:0.2;" x="250.454063" xlink:href="#m4f5604d9a2" y="57.829071"/>
+ <use style="fill:#8c564b;fill-opacity:0.2;stroke:#8c564b;stroke-opacity:0.2;" x="249.576561" xlink:href="#m4f5604d9a2" y="57.522726"/>
+ <use style="fill:#8c564b;fill-opacity:0.2;stroke:#8c564b;stroke-opacity:0.2;" x="248.701958" xlink:href="#m4f5604d9a2" y="57.220033"/>
+ <use style="fill:#8c564b;fill-opacity:0.2;stroke:#8c564b;stroke-opacity:0.2;" x="247.830223" xlink:href="#m4f5604d9a2" y="56.920951"/>
+ <use style="fill:#8c564b;fill-opacity:0.2;stroke:#8c564b;stroke-opacity:0.2;" x="246.961327" xlink:href="#m4f5604d9a2" y="56.625439"/>
+ <use style="fill:#8c564b;fill-opacity:0.2;stroke:#8c564b;stroke-opacity:0.2;" x="246.095235" xlink:href="#m4f5604d9a2" y="56.333457"/>
+ <use style="fill:#8c564b;fill-opacity:0.2;stroke:#8c564b;stroke-opacity:0.2;" x="245.231913" xlink:href="#m4f5604d9a2" y="56.044961"/>
+ <use style="fill:#8c564b;fill-opacity:0.2;stroke:#8c564b;stroke-opacity:0.2;" x="244.371324" xlink:href="#m4f5604d9a2" y="55.75991"/>
+ <use style="fill:#8c564b;fill-opacity:0.2;stroke:#8c564b;stroke-opacity:0.2;" x="243.513431" xlink:href="#m4f5604d9a2" y="55.478262"/>
+ <use style="fill:#8c564b;fill-opacity:0.2;stroke:#8c564b;stroke-opacity:0.2;" x="242.658192" xlink:href="#m4f5604d9a2" y="55.199974"/>
+ <use style="fill:#8c564b;fill-opacity:0.2;stroke:#8c564b;stroke-opacity:0.2;" x="241.805568" xlink:href="#m4f5604d9a2" y="54.925004"/>
+ <use style="fill:#8c564b;fill-opacity:0.2;stroke:#8c564b;stroke-opacity:0.2;" x="240.955514" xlink:href="#m4f5604d9a2" y="54.653308"/>
+ <use style="fill:#8c564b;fill-opacity:0.2;stroke:#8c564b;stroke-opacity:0.2;" x="240.107985" xlink:href="#m4f5604d9a2" y="54.384843"/>
+ <use style="fill:#8c564b;fill-opacity:0.2;stroke:#8c564b;stroke-opacity:0.2;" x="239.262936" xlink:href="#m4f5604d9a2" y="54.119567"/>
+ <use style="fill:#8c564b;fill-opacity:0.2;stroke:#8c564b;stroke-opacity:0.2;" x="238.420317" xlink:href="#m4f5604d9a2" y="53.857434"/>
+ <use style="fill:#8c564b;fill-opacity:0.2;stroke:#8c564b;stroke-opacity:0.2;" x="237.580081" xlink:href="#m4f5604d9a2" y="53.598403"/>
+ <use style="fill:#8c564b;fill-opacity:0.2;stroke:#8c564b;stroke-opacity:0.2;" x="236.742175" xlink:href="#m4f5604d9a2" y="53.342428"/>
+ <use style="fill:#8c564b;fill-opacity:0.2;stroke:#8c564b;stroke-opacity:0.2;" x="235.906547" xlink:href="#m4f5604d9a2" y="53.089466"/>
+ <use style="fill:#8c564b;fill-opacity:0.2;stroke:#8c564b;stroke-opacity:0.2;" x="235.073144" xlink:href="#m4f5604d9a2" y="52.839474"/>
+ <use style="fill:#8c564b;fill-opacity:0.2;stroke:#8c564b;stroke-opacity:0.2;" x="234.241909" xlink:href="#m4f5604d9a2" y="52.592406"/>
+ <use style="fill:#8c564b;fill-opacity:0.2;stroke:#8c564b;stroke-opacity:0.2;" x="233.412787" xlink:href="#m4f5604d9a2" y="52.348218"/>
+ <use style="fill:#8c564b;fill-opacity:0.2;stroke:#8c564b;stroke-opacity:0.2;" x="232.58572" xlink:href="#m4f5604d9a2" y="52.106867"/>
+ <use style="fill:#8c564b;fill-opacity:0.2;stroke:#8c564b;stroke-opacity:0.2;" x="231.760647" xlink:href="#m4f5604d9a2" y="51.868309"/>
+ <use style="fill:#8c564b;fill-opacity:0.2;stroke:#8c564b;stroke-opacity:0.2;" x="230.937509" xlink:href="#m4f5604d9a2" y="51.632498"/>
+ <use style="fill:#8c564b;fill-opacity:0.2;stroke:#8c564b;stroke-opacity:0.2;" x="230.116243" xlink:href="#m4f5604d9a2" y="51.39939"/>
+ <use style="fill:#8c564b;fill-opacity:0.2;stroke:#8c564b;stroke-opacity:0.2;" x="229.296786" xlink:href="#m4f5604d9a2" y="51.168942"/>
+ <use style="fill:#8c564b;fill-opacity:0.2;stroke:#8c564b;stroke-opacity:0.2;" x="228.479075" xlink:href="#m4f5604d9a2" y="50.941109"/>
+ <use style="fill:#8c564b;fill-opacity:0.2;stroke:#8c564b;stroke-opacity:0.2;" x="227.663043" xlink:href="#m4f5604d9a2" y="50.715846"/>
+ <use style="fill:#8c564b;fill-opacity:0.2;stroke:#8c564b;stroke-opacity:0.2;" x="226.848624" xlink:href="#m4f5604d9a2" y="50.493109"/>
+ <use style="fill:#8c564b;fill-opacity:0.2;stroke:#8c564b;stroke-opacity:0.2;" x="226.035751" xlink:href="#m4f5604d9a2" y="50.272855"/>
+ <use style="fill:#8c564b;fill-opacity:0.2;stroke:#8c564b;stroke-opacity:0.2;" x="225.224353" xlink:href="#m4f5604d9a2" y="50.055039"/>
+ <use style="fill:#8c564b;fill-opacity:0.2;stroke:#8c564b;stroke-opacity:0.2;" x="224.414363" xlink:href="#m4f5604d9a2" y="49.839616"/>
+ <use style="fill:#8c564b;fill-opacity:0.2;stroke:#8c564b;stroke-opacity:0.2;" x="223.605708" xlink:href="#m4f5604d9a2" y="49.626543"/>
+ <use style="fill:#8c564b;fill-opacity:0.2;stroke:#8c564b;stroke-opacity:0.2;" x="222.798318" xlink:href="#m4f5604d9a2" y="49.415776"/>
+ <use style="fill:#8c564b;fill-opacity:0.2;stroke:#8c564b;stroke-opacity:0.2;" x="221.992118" xlink:href="#m4f5604d9a2" y="49.207271"/>
+ <use style="fill:#8c564b;fill-opacity:0.2;stroke:#8c564b;stroke-opacity:0.2;" x="221.187036" xlink:href="#m4f5604d9a2" y="49.000984"/>
+ <use style="fill:#8c564b;fill-opacity:0.2;stroke:#8c564b;stroke-opacity:0.2;" x="220.382997" xlink:href="#m4f5604d9a2" y="48.796872"/>
+ <use style="fill:#8c564b;fill-opacity:0.2;stroke:#8c564b;stroke-opacity:0.2;" x="219.579926" xlink:href="#m4f5604d9a2" y="48.594891"/>
+ <use style="fill:#8c564b;fill-opacity:0.2;stroke:#8c564b;stroke-opacity:0.2;" x="218.777746" xlink:href="#m4f5604d9a2" y="48.394998"/>
+ <use style="fill:#8c564b;fill-opacity:0.2;stroke:#8c564b;stroke-opacity:0.2;" x="217.976379" xlink:href="#m4f5604d9a2" y="48.197149"/>
+ <use style="fill:#8c564b;fill-opacity:0.2;stroke:#8c564b;stroke-opacity:0.2;" x="217.175749" xlink:href="#m4f5604d9a2" y="48.001302"/>
+ <use style="fill:#8c564b;fill-opacity:0.2;stroke:#8c564b;stroke-opacity:0.2;" x="216.375777" xlink:href="#m4f5604d9a2" y="47.807413"/>
+ <use style="fill:#8c564b;fill-opacity:0.2;stroke:#8c564b;stroke-opacity:0.2;" x="215.576382" xlink:href="#m4f5604d9a2" y="47.61544"/>
+ <use style="fill:#8c564b;fill-opacity:0.2;stroke:#8c564b;stroke-opacity:0.2;" x="214.777486" xlink:href="#m4f5604d9a2" y="47.42534"/>
+ <use style="fill:#8c564b;fill-opacity:0.2;stroke:#8c564b;stroke-opacity:0.2;" x="213.979008" xlink:href="#m4f5604d9a2" y="47.237071"/>
+ <use style="fill:#8c564b;fill-opacity:0.2;stroke:#8c564b;stroke-opacity:0.2;" x="213.180866" xlink:href="#m4f5604d9a2" y="47.05059"/>
+ <use style="fill:#8c564b;fill-opacity:0.2;stroke:#8c564b;stroke-opacity:0.2;" x="212.382978" xlink:href="#m4f5604d9a2" y="46.865856"/>
+ <use style="fill:#8c564b;fill-opacity:0.2;stroke:#8c564b;stroke-opacity:0.2;" x="211.585263" xlink:href="#m4f5604d9a2" y="46.682827"/>
+ <use style="fill:#8c564b;fill-opacity:0.2;stroke:#8c564b;stroke-opacity:0.2;" x="210.787636" xlink:href="#m4f5604d9a2" y="46.501462"/>
+ <use style="fill:#8c564b;fill-opacity:0.2;stroke:#8c564b;stroke-opacity:0.2;" x="209.990015" xlink:href="#m4f5604d9a2" y="46.321718"/>
+ <use style="fill:#8c564b;fill-opacity:0.2;stroke:#8c564b;stroke-opacity:0.2;" x="209.192316" xlink:href="#m4f5604d9a2" y="46.143556"/>
+ <use style="fill:#8c564b;fill-opacity:0.2;stroke:#8c564b;stroke-opacity:0.2;" x="208.394454" xlink:href="#m4f5604d9a2" y="45.966933"/>
+ <use style="fill:#8c564b;fill-opacity:0.2;stroke:#8c564b;stroke-opacity:0.2;" x="207.596344" xlink:href="#m4f5604d9a2" y="45.791811"/>
+ <use style="fill:#8c564b;fill-opacity:0.2;stroke:#8c564b;stroke-opacity:0.2;" x="206.797902" xlink:href="#m4f5604d9a2" y="45.618147"/>
+ <use style="fill:#8c564b;fill-opacity:0.2;stroke:#8c564b;stroke-opacity:0.2;" x="205.999041" xlink:href="#m4f5604d9a2" y="45.445904"/>
+ <use style="fill:#8c564b;fill-opacity:0.2;stroke:#8c564b;stroke-opacity:0.2;" x="205.199677" xlink:href="#m4f5604d9a2" y="45.275039"/>
+ <use style="fill:#8c564b;fill-opacity:0.2;stroke:#8c564b;stroke-opacity:0.2;" x="204.399722" xlink:href="#m4f5604d9a2" y="45.105516"/>
+ <use style="fill:#8c564b;fill-opacity:0.2;stroke:#8c564b;stroke-opacity:0.2;" x="203.599091" xlink:href="#m4f5604d9a2" y="44.937293"/>
+ <use style="fill:#8c564b;fill-opacity:0.2;stroke:#8c564b;stroke-opacity:0.2;" x="202.797697" xlink:href="#m4f5604d9a2" y="44.770333"/>
+ <use style="fill:#8c564b;fill-opacity:0.2;stroke:#8c564b;stroke-opacity:0.2;" x="201.995454" xlink:href="#m4f5604d9a2" y="44.604597"/>
+ <use style="fill:#8c564b;fill-opacity:0.2;stroke:#8c564b;stroke-opacity:0.2;" x="201.192274" xlink:href="#m4f5604d9a2" y="44.440047"/>
+ <use style="fill:#8c564b;fill-opacity:0.2;stroke:#8c564b;stroke-opacity:0.2;" x="200.38807" xlink:href="#m4f5604d9a2" y="44.276646"/>
+ <use style="fill:#8c564b;fill-opacity:0.2;stroke:#8c564b;stroke-opacity:0.2;" x="199.582756" xlink:href="#m4f5604d9a2" y="44.114355"/>
+ <use style="fill:#8c564b;fill-opacity:0.2;stroke:#8c564b;stroke-opacity:0.2;" x="198.776245" xlink:href="#m4f5604d9a2" y="43.953138"/>
+ <use style="fill:#8c564b;fill-opacity:0.2;stroke:#8c564b;stroke-opacity:0.2;" x="197.968448" xlink:href="#m4f5604d9a2" y="43.792959"/>
+ <use style="fill:#8c564b;fill-opacity:0.2;stroke:#8c564b;stroke-opacity:0.2;" x="197.15928" xlink:href="#m4f5604d9a2" y="43.63378"/>
+ <use style="fill:#8c564b;fill-opacity:0.2;stroke:#8c564b;stroke-opacity:0.2;" x="196.348652" xlink:href="#m4f5604d9a2" y="43.475567"/>
+ <use style="fill:#8c564b;fill-opacity:0.2;stroke:#8c564b;stroke-opacity:0.2;" x="195.536479" xlink:href="#m4f5604d9a2" y="43.318283"/>
+ <use style="fill:#8c564b;fill-opacity:0.2;stroke:#8c564b;stroke-opacity:0.2;" x="194.722673" xlink:href="#m4f5604d9a2" y="43.161895"/>
+ <use style="fill:#8c564b;fill-opacity:0.2;stroke:#8c564b;stroke-opacity:0.2;" x="193.907147" xlink:href="#m4f5604d9a2" y="43.006366"/>
+ <use style="fill:#8c564b;fill-opacity:0.2;stroke:#8c564b;stroke-opacity:0.2;" x="193.089816" xlink:href="#m4f5604d9a2" y="42.851664"/>
+ <use style="fill:#8c564b;fill-opacity:0.2;stroke:#8c564b;stroke-opacity:0.2;" x="192.270593" xlink:href="#m4f5604d9a2" y="42.697755"/>
+ <use style="fill:#8c564b;fill-opacity:0.2;stroke:#8c564b;stroke-opacity:0.2;" x="191.449391" xlink:href="#m4f5604d9a2" y="42.544604"/>
+ <use style="fill:#8c564b;fill-opacity:0.2;stroke:#8c564b;stroke-opacity:0.2;" x="190.626125" xlink:href="#m4f5604d9a2" y="42.39218"/>
+ <use style="fill:#8c564b;fill-opacity:0.2;stroke:#8c564b;stroke-opacity:0.2;" x="189.80071" xlink:href="#m4f5604d9a2" y="42.240451"/>
+ <use style="fill:#8c564b;fill-opacity:0.2;stroke:#8c564b;stroke-opacity:0.2;" x="188.973061" xlink:href="#m4f5604d9a2" y="42.089383"/>
+ <use style="fill:#8c564b;fill-opacity:0.2;stroke:#8c564b;stroke-opacity:0.2;" x="188.143092" xlink:href="#m4f5604d9a2" y="41.938947"/>
+ <use style="fill:#8c564b;fill-opacity:0.2;stroke:#8c564b;stroke-opacity:0.2;" x="187.310721" xlink:href="#m4f5604d9a2" y="41.789112"/>
+ <use style="fill:#8c564b;fill-opacity:0.2;stroke:#8c564b;stroke-opacity:0.2;" x="186.475862" xlink:href="#m4f5604d9a2" y="41.639847"/>
+ <use style="fill:#8c564b;fill-opacity:0.2;stroke:#8c564b;stroke-opacity:0.2;" x="185.638432" xlink:href="#m4f5604d9a2" y="41.491122"/>
+ <use style="fill:#8c564b;fill-opacity:0.2;stroke:#8c564b;stroke-opacity:0.2;" x="184.798349" xlink:href="#m4f5604d9a2" y="41.342909"/>
+ <use style="fill:#8c564b;fill-opacity:0.2;stroke:#8c564b;stroke-opacity:0.2;" x="183.955531" xlink:href="#m4f5604d9a2" y="41.195178"/>
+ <use style="fill:#8c564b;fill-opacity:0.2;stroke:#8c564b;stroke-opacity:0.2;" x="183.109894" xlink:href="#m4f5604d9a2" y="41.047903"/>
+ <use style="fill:#8c564b;fill-opacity:0.2;stroke:#8c564b;stroke-opacity:0.2;" x="182.261359" xlink:href="#m4f5604d9a2" y="40.901054"/>
+ <use style="fill:#8c564b;fill-opacity:0.2;stroke:#8c564b;stroke-opacity:0.2;" x="181.409845" xlink:href="#m4f5604d9a2" y="40.754605"/>
+ <use style="fill:#8c564b;fill-opacity:0.2;stroke:#8c564b;stroke-opacity:0.2;" x="180.55527" xlink:href="#m4f5604d9a2" y="40.608531"/>
+ <use style="fill:#8c564b;fill-opacity:0.2;stroke:#8c564b;stroke-opacity:0.2;" x="179.697558" xlink:href="#m4f5604d9a2" y="40.462805"/>
+ <use style="fill:#8c564b;fill-opacity:0.2;stroke:#8c564b;stroke-opacity:0.2;" x="178.836627" xlink:href="#m4f5604d9a2" y="40.317401"/>
+ <use style="fill:#8c564b;fill-opacity:0.2;stroke:#8c564b;stroke-opacity:0.2;" x="177.972401" xlink:href="#m4f5604d9a2" y="40.172296"/>
+ <use style="fill:#8c564b;fill-opacity:0.2;stroke:#8c564b;stroke-opacity:0.2;" x="177.104801" xlink:href="#m4f5604d9a2" y="40.027466"/>
+ <use style="fill:#8c564b;fill-opacity:0.2;stroke:#8c564b;stroke-opacity:0.2;" x="176.233752" xlink:href="#m4f5604d9a2" y="39.882887"/>
+ <use style="fill:#8c564b;fill-opacity:0.2;stroke:#8c564b;stroke-opacity:0.2;" x="175.359177" xlink:href="#m4f5604d9a2" y="39.738536"/>
+ <use style="fill:#8c564b;fill-opacity:0.2;stroke:#8c564b;stroke-opacity:0.2;" x="174.481001" xlink:href="#m4f5604d9a2" y="39.594392"/>
+ <use style="fill:#8c564b;fill-opacity:0.2;stroke:#8c564b;stroke-opacity:0.2;" x="173.59915" xlink:href="#m4f5604d9a2" y="39.450432"/>
+ <use style="fill:#8c564b;fill-opacity:0.2;stroke:#8c564b;stroke-opacity:0.2;" x="172.713549" xlink:href="#m4f5604d9a2" y="39.306637"/>
+ <use style="fill:#8c564b;fill-opacity:0.2;stroke:#8c564b;stroke-opacity:0.2;" x="171.824127" xlink:href="#m4f5604d9a2" y="39.162985"/>
+ <use style="fill:#8c564b;fill-opacity:0.2;stroke:#8c564b;stroke-opacity:0.2;" x="170.930809" xlink:href="#m4f5604d9a2" y="39.019458"/>
+ <use style="fill:#8c564b;fill-opacity:0.2;stroke:#8c564b;stroke-opacity:0.2;" x="170.033526" xlink:href="#m4f5604d9a2" y="38.876037"/>
+ <use style="fill:#8c564b;fill-opacity:0.2;stroke:#8c564b;stroke-opacity:0.2;" x="169.132207" xlink:href="#m4f5604d9a2" y="38.732703"/>
+ <use style="fill:#8c564b;fill-opacity:0.2;stroke:#8c564b;stroke-opacity:0.2;" x="168.226781" xlink:href="#m4f5604d9a2" y="38.589439"/>
+ <use style="fill:#8c564b;fill-opacity:0.2;stroke:#8c564b;stroke-opacity:0.2;" x="167.317179" xlink:href="#m4f5604d9a2" y="38.446227"/>
+ <use style="fill:#8c564b;fill-opacity:0.2;stroke:#8c564b;stroke-opacity:0.2;" x="166.403333" xlink:href="#m4f5604d9a2" y="38.303053"/>
+ <use style="fill:#8c564b;fill-opacity:0.2;stroke:#8c564b;stroke-opacity:0.2;" x="165.485176" xlink:href="#m4f5604d9a2" y="38.159899"/>
+ <use style="fill:#8c564b;fill-opacity:0.2;stroke:#8c564b;stroke-opacity:0.2;" x="164.56264" xlink:href="#m4f5604d9a2" y="38.016752"/>
+ <use style="fill:#8c564b;fill-opacity:0.2;stroke:#8c564b;stroke-opacity:0.2;" x="163.63566" xlink:href="#m4f5604d9a2" y="37.873596"/>
+ <use style="fill:#8c564b;fill-opacity:0.2;stroke:#8c564b;stroke-opacity:0.2;" x="162.704169" xlink:href="#m4f5604d9a2" y="37.730419"/>
+ <use style="fill:#8c564b;fill-opacity:0.2;stroke:#8c564b;stroke-opacity:0.2;" x="161.768105" xlink:href="#m4f5604d9a2" y="37.587207"/>
+ <use style="fill:#8c564b;fill-opacity:0.2;stroke:#8c564b;stroke-opacity:0.2;" x="160.827402" xlink:href="#m4f5604d9a2" y="37.443947"/>
+ <use style="fill:#8c564b;fill-opacity:0.2;stroke:#8c564b;stroke-opacity:0.2;" x="159.881998" xlink:href="#m4f5604d9a2" y="37.30063"/>
+ <use style="fill:#8c564b;fill-opacity:0.2;stroke:#8c564b;stroke-opacity:0.2;" x="158.93183" xlink:href="#m4f5604d9a2" y="37.157242"/>
+ <use style="fill:#8c564b;fill-opacity:0.2;stroke:#8c564b;stroke-opacity:0.2;" x="157.976837" xlink:href="#m4f5604d9a2" y="37.013774"/>
+ <use style="fill:#8c564b;fill-opacity:0.2;stroke:#8c564b;stroke-opacity:0.2;" x="157.016958" xlink:href="#m4f5604d9a2" y="36.870217"/>
+ <use style="fill:#8c564b;fill-opacity:0.2;stroke:#8c564b;stroke-opacity:0.2;" x="156.052133" xlink:href="#m4f5604d9a2" y="36.72656"/>
+ <use style="fill:#8c564b;fill-opacity:0.2;stroke:#8c564b;stroke-opacity:0.2;" x="155.082301" xlink:href="#m4f5604d9a2" y="36.582796"/>
+ <use style="fill:#8c564b;fill-opacity:0.2;stroke:#8c564b;stroke-opacity:0.2;" x="154.107405" xlink:href="#m4f5604d9a2" y="36.438917"/>
+ <use style="fill:#8c564b;fill-opacity:0.2;stroke:#8c564b;stroke-opacity:0.2;" x="153.127386" xlink:href="#m4f5604d9a2" y="36.294915"/>
+ <use style="fill:#8c564b;fill-opacity:0.2;stroke:#8c564b;stroke-opacity:0.2;" x="152.142187" xlink:href="#m4f5604d9a2" y="36.150783"/>
+ <use style="fill:#8c564b;fill-opacity:0.2;stroke:#8c564b;stroke-opacity:0.2;" x="151.15175" xlink:href="#m4f5604d9a2" y="36.006516"/>
+ <use style="fill:#8c564b;fill-opacity:0.2;stroke:#8c564b;stroke-opacity:0.2;" x="150.15602" xlink:href="#m4f5604d9a2" y="35.862109"/>
+ <use style="fill:#8c564b;fill-opacity:0.2;stroke:#8c564b;stroke-opacity:0.2;" x="149.154942" xlink:href="#m4f5604d9a2" y="35.717555"/>
+ <use style="fill:#8c564b;fill-opacity:0.2;stroke:#8c564b;stroke-opacity:0.2;" x="148.14846" xlink:href="#m4f5604d9a2" y="35.572851"/>
+ <use style="fill:#8c564b;fill-opacity:0.2;stroke:#8c564b;stroke-opacity:0.2;" x="147.136519" xlink:href="#m4f5604d9a2" y="35.427993"/>
+ <use style="fill:#8c564b;fill-opacity:0.2;stroke:#8c564b;stroke-opacity:0.2;" x="146.119066" xlink:href="#m4f5604d9a2" y="35.282978"/>
+ <use style="fill:#8c564b;fill-opacity:0.2;stroke:#8c564b;stroke-opacity:0.2;" x="145.096047" xlink:href="#m4f5604d9a2" y="35.137803"/>
+ <use style="fill:#8c564b;fill-opacity:0.2;stroke:#8c564b;stroke-opacity:0.2;" x="144.067408" xlink:href="#m4f5604d9a2" y="34.992464"/>
+ <use style="fill:#8c564b;fill-opacity:0.2;stroke:#8c564b;stroke-opacity:0.2;" x="143.033097" xlink:href="#m4f5604d9a2" y="34.846962"/>
+ <use style="fill:#8c564b;fill-opacity:0.2;stroke:#8c564b;stroke-opacity:0.2;" x="141.993061" xlink:href="#m4f5604d9a2" y="34.701293"/>
+ <use style="fill:#8c564b;fill-opacity:0.2;stroke:#8c564b;stroke-opacity:0.2;" x="140.947247" xlink:href="#m4f5604d9a2" y="34.555457"/>
+ <use style="fill:#8c564b;fill-opacity:0.2;stroke:#8c564b;stroke-opacity:0.2;" x="139.895603" xlink:href="#m4f5604d9a2" y="34.409454"/>
+ <use style="fill:#8c564b;fill-opacity:0.2;stroke:#8c564b;stroke-opacity:0.2;" x="138.838079" xlink:href="#m4f5604d9a2" y="34.263284"/>
+ <use style="fill:#8c564b;fill-opacity:0.2;stroke:#8c564b;stroke-opacity:0.2;" x="137.774621" xlink:href="#m4f5604d9a2" y="34.116946"/>
+ <use style="fill:#8c564b;fill-opacity:0.2;stroke:#8c564b;stroke-opacity:0.2;" x="136.70518" xlink:href="#m4f5604d9a2" y="33.970441"/>
+ <use style="fill:#8c564b;fill-opacity:0.2;stroke:#8c564b;stroke-opacity:0.2;" x="135.629705" xlink:href="#m4f5604d9a2" y="33.823771"/>
+ <use style="fill:#8c564b;fill-opacity:0.2;stroke:#8c564b;stroke-opacity:0.2;" x="134.548144" xlink:href="#m4f5604d9a2" y="33.676936"/>
+ <use style="fill:#8c564b;fill-opacity:0.2;stroke:#8c564b;stroke-opacity:0.2;" x="133.460447" xlink:href="#m4f5604d9a2" y="33.529938"/>
+ <use style="fill:#8c564b;fill-opacity:0.2;stroke:#8c564b;stroke-opacity:0.2;" x="132.366566" xlink:href="#m4f5604d9a2" y="33.38278"/>
+ <use style="fill:#8c564b;fill-opacity:0.2;stroke:#8c564b;stroke-opacity:0.2;" x="131.266449" xlink:href="#m4f5604d9a2" y="33.235463"/>
+ <use style="fill:#8c564b;fill-opacity:0.2;stroke:#8c564b;stroke-opacity:0.2;" x="130.160048" xlink:href="#m4f5604d9a2" y="33.08799"/>
+ <use style="fill:#8c564b;fill-opacity:0.2;stroke:#8c564b;stroke-opacity:0.2;" x="129.047313" xlink:href="#m4f5604d9a2" y="32.940365"/>
+ <use style="fill:#8c564b;fill-opacity:0.2;stroke:#8c564b;stroke-opacity:0.2;" x="127.928196" xlink:href="#m4f5604d9a2" y="32.792589"/>
+ <use style="fill:#8c564b;fill-opacity:0.2;stroke:#8c564b;stroke-opacity:0.2;" x="126.802648" xlink:href="#m4f5604d9a2" y="32.644666"/>
+ <use style="fill:#8c564b;fill-opacity:0.2;stroke:#8c564b;stroke-opacity:0.2;" x="125.670622" xlink:href="#m4f5604d9a2" y="32.496599"/>
+ <use style="fill:#8c564b;fill-opacity:0.2;stroke:#8c564b;stroke-opacity:0.2;" x="124.532068" xlink:href="#m4f5604d9a2" y="32.348393"/>
+ <use style="fill:#8c564b;fill-opacity:0.2;stroke:#8c564b;stroke-opacity:0.2;" x="123.386941" xlink:href="#m4f5604d9a2" y="32.200051"/>
+ <use style="fill:#8c564b;fill-opacity:0.2;stroke:#8c564b;stroke-opacity:0.2;" x="122.235193" xlink:href="#m4f5604d9a2" y="32.051577"/>
+ <use style="fill:#8c564b;fill-opacity:0.2;stroke:#8c564b;stroke-opacity:0.2;" x="121.076778" xlink:href="#m4f5604d9a2" y="31.902975"/>
+ <use style="fill:#8c564b;fill-opacity:0.2;stroke:#8c564b;stroke-opacity:0.2;" x="119.911649" xlink:href="#m4f5604d9a2" y="31.754249"/>
+ <use style="fill:#8c564b;fill-opacity:0.2;stroke:#8c564b;stroke-opacity:0.2;" x="118.73976" xlink:href="#m4f5604d9a2" y="31.605404"/>
+ <use style="fill:#8c564b;fill-opacity:0.2;stroke:#8c564b;stroke-opacity:0.2;" x="117.561068" xlink:href="#m4f5604d9a2" y="31.456444"/>
+ <use style="fill:#8c564b;fill-opacity:0.2;stroke:#8c564b;stroke-opacity:0.2;" x="116.375527" xlink:href="#m4f5604d9a2" y="31.307373"/>
+ <use style="fill:#8c564b;fill-opacity:0.2;stroke:#8c564b;stroke-opacity:0.2;" x="115.183093" xlink:href="#m4f5604d9a2" y="31.158196"/>
+ <use style="fill:#8c564b;fill-opacity:0.2;stroke:#8c564b;stroke-opacity:0.2;" x="113.983723" xlink:href="#m4f5604d9a2" y="31.008917"/>
+ <use style="fill:#8c564b;fill-opacity:0.2;stroke:#8c564b;stroke-opacity:0.2;" x="112.777375" xlink:href="#m4f5604d9a2" y="30.859541"/>
+ <use style="fill:#8c564b;fill-opacity:0.2;stroke:#8c564b;stroke-opacity:0.2;" x="111.564006" xlink:href="#m4f5604d9a2" y="30.710072"/>
+ <use style="fill:#8c564b;fill-opacity:0.2;stroke:#8c564b;stroke-opacity:0.2;" x="110.343575" xlink:href="#m4f5604d9a2" y="30.560514"/>
+ <use style="fill:#8c564b;fill-opacity:0.2;stroke:#8c564b;stroke-opacity:0.2;" x="109.116043" xlink:href="#m4f5604d9a2" y="30.410873"/>
+ <use style="fill:#8c564b;fill-opacity:0.2;stroke:#8c564b;stroke-opacity:0.2;" x="107.881369" xlink:href="#m4f5604d9a2" y="30.261151"/>
+ <use style="fill:#8c564b;fill-opacity:0.2;stroke:#8c564b;stroke-opacity:0.2;" x="106.639516" xlink:href="#m4f5604d9a2" y="30.111355"/>
+ <use style="fill:#8c564b;fill-opacity:0.2;stroke:#8c564b;stroke-opacity:0.2;" x="105.390445" xlink:href="#m4f5604d9a2" y="29.961487"/>
+ <use style="fill:#8c564b;fill-opacity:0.2;stroke:#8c564b;stroke-opacity:0.2;" x="104.134121" xlink:href="#m4f5604d9a2" y="29.811551"/>
+ <use style="fill:#8c564b;fill-opacity:0.2;stroke:#8c564b;stroke-opacity:0.2;" x="102.870509" xlink:href="#m4f5604d9a2" y="29.661553"/>
+ <use style="fill:#8c564b;fill-opacity:0.2;stroke:#8c564b;stroke-opacity:0.2;" x="101.599573" xlink:href="#m4f5604d9a2" y="29.511494"/>
+ <use style="fill:#8c564b;fill-opacity:0.2;stroke:#8c564b;stroke-opacity:0.2;" x="100.321282" xlink:href="#m4f5604d9a2" y="29.36138"/>
+ <use style="fill:#8c564b;fill-opacity:0.2;stroke:#8c564b;stroke-opacity:0.2;" x="99.035605" xlink:href="#m4f5604d9a2" y="29.211214"/>
+ <use style="fill:#8c564b;fill-opacity:0.2;stroke:#8c564b;stroke-opacity:0.2;" x="97.74251" xlink:href="#m4f5604d9a2" y="29.060998"/>
+ <use style="fill:#8c564b;fill-opacity:0.2;stroke:#8c564b;stroke-opacity:0.2;" x="96.441971" xlink:href="#m4f5604d9a2" y="28.910736"/>
+ <use style="fill:#8c564b;fill-opacity:0.2;stroke:#8c564b;stroke-opacity:0.2;" x="95.133959" xlink:href="#m4f5604d9a2" y="28.760432"/>
+ <use style="fill:#8c564b;fill-opacity:0.2;stroke:#8c564b;stroke-opacity:0.2;" x="93.818449" xlink:href="#m4f5604d9a2" y="28.610087"/>
+ <use style="fill:#8c564b;fill-opacity:0.2;stroke:#8c564b;stroke-opacity:0.2;" x="92.495419" xlink:href="#m4f5604d9a2" y="28.459705"/>
+ <use style="fill:#8c564b;fill-opacity:0.2;stroke:#8c564b;stroke-opacity:0.2;" x="91.164846" xlink:href="#m4f5604d9a2" y="28.309288"/>
+ <use style="fill:#8c564b;fill-opacity:0.2;stroke:#8c564b;stroke-opacity:0.2;" x="89.826711" xlink:href="#m4f5604d9a2" y="28.158837"/>
+ <use style="fill:#8c564b;fill-opacity:0.2;stroke:#8c564b;stroke-opacity:0.2;" x="88.480996" xlink:href="#m4f5604d9a2" y="28.008355"/>
+ <use style="fill:#8c564b;fill-opacity:0.2;stroke:#8c564b;stroke-opacity:0.2;" x="87.127684" xlink:href="#m4f5604d9a2" y="27.857844"/>
+ <use style="fill:#8c564b;fill-opacity:0.2;stroke:#8c564b;stroke-opacity:0.2;" x="85.766763" xlink:href="#m4f5604d9a2" y="27.707305"/>
+ <use style="fill:#8c564b;fill-opacity:0.2;stroke:#8c564b;stroke-opacity:0.2;" x="84.398221" xlink:href="#m4f5604d9a2" y="27.55674"/>
+ <use style="fill:#8c564b;fill-opacity:0.2;stroke:#8c564b;stroke-opacity:0.2;" x="83.022049" xlink:href="#m4f5604d9a2" y="27.406148"/>
+ <use style="fill:#8c564b;fill-opacity:0.2;stroke:#8c564b;stroke-opacity:0.2;" x="81.63824" xlink:href="#m4f5604d9a2" y="27.255532"/>
+ <use style="fill:#8c564b;fill-opacity:0.2;stroke:#8c564b;stroke-opacity:0.2;" x="80.246789" xlink:href="#m4f5604d9a2" y="27.104892"/>
+ <use style="fill:#8c564b;fill-opacity:0.2;stroke:#8c564b;stroke-opacity:0.2;" x="78.847696" xlink:href="#m4f5604d9a2" y="26.954228"/>
+ <use style="fill:#8c564b;fill-opacity:0.2;stroke:#8c564b;stroke-opacity:0.2;" x="77.440961" xlink:href="#m4f5604d9a2" y="26.80354"/>
+ <use style="fill:#8c564b;fill-opacity:0.2;stroke:#8c564b;stroke-opacity:0.2;" x="76.026587" xlink:href="#m4f5604d9a2" y="26.652827"/>
+ <use style="fill:#8c564b;fill-opacity:0.2;stroke:#8c564b;stroke-opacity:0.2;" x="74.604581" xlink:href="#m4f5604d9a2" y="26.502091"/>
+ <use style="fill:#8c564b;fill-opacity:0.2;stroke:#8c564b;stroke-opacity:0.2;" x="73.174951" xlink:href="#m4f5604d9a2" y="26.351329"/>
+ <use style="fill:#8c564b;fill-opacity:0.2;stroke:#8c564b;stroke-opacity:0.2;" x="71.73771" xlink:href="#m4f5604d9a2" y="26.200542"/>
+ <use style="fill:#8c564b;fill-opacity:0.2;stroke:#8c564b;stroke-opacity:0.2;" x="70.292872" xlink:href="#m4f5604d9a2" y="26.049728"/>
+ <use style="fill:#8c564b;fill-opacity:0.2;stroke:#8c564b;stroke-opacity:0.2;" x="68.840455" xlink:href="#m4f5604d9a2" y="25.898885"/>
+ </g>
+ </g>
+ <g id="patch_3">
+ <path d="M 50.69 301.9975
+L 50.69 10.8
+" style="fill:none;stroke:#000000;stroke-linecap:square;stroke-linejoin:miter;stroke-width:0.8;"/>
+ </g>
+ <g id="patch_4">
+ <path d="M 450 301.9975
+L 450 10.8
+" style="fill:none;stroke:#000000;stroke-linecap:square;stroke-linejoin:miter;stroke-width:0.8;"/>
+ </g>
+ <g id="patch_5">
+ <path d="M 50.69 301.9975
+L 450 301.9975
+" style="fill:none;stroke:#000000;stroke-linecap:square;stroke-linejoin:miter;stroke-width:0.8;"/>
+ </g>
+ <g id="patch_6">
+ <path d="M 50.69 10.8
+L 450 10.8
+" style="fill:none;stroke:#000000;stroke-linecap:square;stroke-linejoin:miter;stroke-width:0.8;"/>
+ </g>
+ <g id="legend_1">
+ <g id="patch_7">
+ <path d="M 390.64 78.048125
+L 445.1 78.048125
+Q 446.5 78.048125 446.5 76.648125
+L 446.5 15.7
+Q 446.5 14.3 445.1 14.3
+L 390.64 14.3
+Q 389.24 14.3 389.24 15.7
+L 389.24 76.648125
+Q 389.24 78.048125 390.64 78.048125
+z
+" style="fill:#ffffff;opacity:0.8;stroke:#cccccc;stroke-linejoin:miter;"/>
+ </g>
+ <g id="line2d_17">
+ <path d="M 392.04 19.968906
+L 406.04 19.968906
+" style="fill:none;stroke:#1f77b4;stroke-dasharray:6,4.5,6;stroke-dashoffset:0;stroke-opacity:0.2;stroke-width:1.5;"/>
+ </g>
+ <g id="line2d_18">
+ <g>
+ <use style="fill:#1f77b4;fill-opacity:0.2;stroke:#1f77b4;stroke-opacity:0.2;" x="399.04" xlink:href="#ma85ef77801" y="19.968906"/>
+ </g>
+ </g>
+ <g id="text_13">
+ <!-- I4O3$f = 1$ -->
+ <g transform="translate(411.64 22.418906)scale(0.07 -0.07)">
+ <defs>
+ <path d="M 9.8125 72.90625
+L 19.671875 72.90625
+L 19.671875 0
+L 9.8125 0
+z
+" id="DejaVuSans-73"/>
+ <path d="M 39.40625 66.21875
+Q 28.65625 66.21875 22.328125 58.203125
+Q 16.015625 50.203125 16.015625 36.375
+Q 16.015625 22.609375 22.328125 14.59375
+Q 28.65625 6.59375 39.40625 6.59375
+Q 50.140625 6.59375 56.421875 14.59375
+Q 62.703125 22.609375 62.703125 36.375
+Q 62.703125 50.203125 56.421875 58.203125
+Q 50.140625 66.21875 39.40625 66.21875
+z
+M 39.40625 74.21875
+Q 54.734375 74.21875 63.90625 63.9375
+Q 73.09375 53.65625 73.09375 36.375
+Q 73.09375 19.140625 63.90625 8.859375
+Q 54.734375 -1.421875 39.40625 -1.421875
+Q 24.03125 -1.421875 14.8125 8.828125
+Q 5.609375 19.09375 5.609375 36.375
+Q 5.609375 53.65625 14.8125 63.9375
+Q 24.03125 74.21875 39.40625 74.21875
+z
+" id="DejaVuSans-79"/>
+ <path d="M 47.796875 75.984375
+L 46.390625 68.5
+L 37.796875 68.5
+Q 32.90625 68.5 30.6875 66.578125
+Q 28.46875 64.65625 27.390625 59.515625
+L 26.421875 54.6875
+L 41.21875 54.6875
+L 39.890625 47.703125
+L 25.09375 47.703125
+L 15.828125 0
+L 6.78125 0
+L 16.109375 47.703125
+L 7.515625 47.703125
+L 8.796875 54.6875
+L 17.390625 54.6875
+L 18.109375 58.5
+Q 19.96875 68.171875 24.625 72.078125
+Q 29.296875 75.984375 39.3125 75.984375
+z
+" id="DejaVuSans-Oblique-102"/>
+ <path d="M 10.59375 45.40625
+L 73.1875 45.40625
+L 73.1875 37.203125
+L 10.59375 37.203125
+z
+M 10.59375 25.484375
+L 73.1875 25.484375
+L 73.1875 17.1875
+L 10.59375 17.1875
+z
+" id="DejaVuSans-61"/>
+ </defs>
+ <use transform="translate(0 0.015625)" xlink:href="#DejaVuSans-73"/>
+ <use transform="translate(29.492188 0.015625)" xlink:href="#DejaVuSans-52"/>
+ <use transform="translate(93.115234 0.015625)" xlink:href="#DejaVuSans-79"/>
+ <use transform="translate(171.826172 0.015625)" xlink:href="#DejaVuSans-51"/>
+ <use transform="translate(235.449219 0.015625)" xlink:href="#DejaVuSans-Oblique-102"/>
+ <use transform="translate(290.136719 0.015625)" xlink:href="#DejaVuSans-61"/>
+ <use transform="translate(393.408203 0.015625)" xlink:href="#DejaVuSans-49"/>
+ </g>
+ </g>
+ <g id="line2d_19">
+ <path d="M 392.04 30.243594
+L 406.04 30.243594
+" style="fill:none;stroke:#ff7f0e;stroke-dasharray:3,4.5,3;stroke-dashoffset:0;stroke-opacity:0.2;stroke-width:1.5;"/>
+ </g>
+ <g id="line2d_20">
+ <g>
+ <use style="fill:#ff7f0e;fill-opacity:0.2;stroke:#ff7f0e;stroke-opacity:0.2;" x="399.04" xlink:href="#mb7e8bbea43" y="30.243594"/>
+ </g>
+ </g>
+ <g id="text_14">
+ <!-- I4O3$f = 2$ -->
+ <g transform="translate(411.64 32.693594)scale(0.07 -0.07)">
+ <use transform="translate(0 0.015625)" xlink:href="#DejaVuSans-73"/>
+ <use transform="translate(29.492188 0.015625)" xlink:href="#DejaVuSans-52"/>
+ <use transform="translate(93.115234 0.015625)" xlink:href="#DejaVuSans-79"/>
+ <use transform="translate(171.826172 0.015625)" xlink:href="#DejaVuSans-51"/>
+ <use transform="translate(235.449219 0.015625)" xlink:href="#DejaVuSans-Oblique-102"/>
+ <use transform="translate(290.136719 0.015625)" xlink:href="#DejaVuSans-61"/>
+ <use transform="translate(393.408203 0.015625)" xlink:href="#DejaVuSans-50"/>
+ </g>
+ </g>
+ <g id="line2d_21">
+ <path d="M 392.04 40.518281
+L 406.04 40.518281
+" style="fill:none;stroke:#2ca02c;stroke-dasharray:1.5,4.5,1.5;stroke-dashoffset:0;stroke-opacity:0.2;stroke-width:1.5;"/>
+ </g>
+ <g id="line2d_22">
+ <g>
+ <use style="fill:#2ca02c;fill-opacity:0.2;stroke:#2ca02c;stroke-opacity:0.2;" x="399.04" xlink:href="#mee8b134b61" y="40.518281"/>
+ </g>
+ </g>
+ <g id="text_15">
+ <!-- I4O3$f = 4$ -->
+ <g transform="translate(411.64 42.968281)scale(0.07 -0.07)">
+ <use transform="translate(0 0.015625)" xlink:href="#DejaVuSans-73"/>
+ <use transform="translate(29.492188 0.015625)" xlink:href="#DejaVuSans-52"/>
+ <use transform="translate(93.115234 0.015625)" xlink:href="#DejaVuSans-79"/>
+ <use transform="translate(171.826172 0.015625)" xlink:href="#DejaVuSans-51"/>
+ <use transform="translate(235.449219 0.015625)" xlink:href="#DejaVuSans-Oblique-102"/>
+ <use transform="translate(290.136719 0.015625)" xlink:href="#DejaVuSans-61"/>
+ <use transform="translate(393.408203 0.015625)" xlink:href="#DejaVuSans-52"/>
+ </g>
+ </g>
+ <g id="line2d_23">
+ <path d="M 392.04 50.792969
+L 406.04 50.792969
+" style="fill:none;stroke:#d62728;stroke-dasharray:6,4.5,6;stroke-dashoffset:0;stroke-opacity:0.2;stroke-width:1.5;"/>
+ </g>
+ <g id="line2d_24">
+ <g>
+ <use style="fill:#d62728;fill-opacity:0.2;stroke:#d62728;stroke-opacity:0.2;" x="399.04" xlink:href="#m6d92a482c2" y="50.792969"/>
+ </g>
+ </g>
+ <g id="text_16">
+ <!-- I8O5$f = 1$ -->
+ <g transform="translate(411.64 53.242969)scale(0.07 -0.07)">
+ <use transform="translate(0 0.015625)" xlink:href="#DejaVuSans-73"/>
+ <use transform="translate(29.492188 0.015625)" xlink:href="#DejaVuSans-56"/>
+ <use transform="translate(93.115234 0.015625)" xlink:href="#DejaVuSans-79"/>
+ <use transform="translate(171.826172 0.015625)" xlink:href="#DejaVuSans-53"/>
+ <use transform="translate(235.449219 0.015625)" xlink:href="#DejaVuSans-Oblique-102"/>
+ <use transform="translate(290.136719 0.015625)" xlink:href="#DejaVuSans-61"/>
+ <use transform="translate(393.408203 0.015625)" xlink:href="#DejaVuSans-49"/>
+ </g>
+ </g>
+ <g id="line2d_25">
+ <path d="M 392.04 61.067656
+L 406.04 61.067656
+" style="fill:none;stroke:#9467bd;stroke-dasharray:3,4.5,3;stroke-dashoffset:0;stroke-opacity:0.2;stroke-width:1.5;"/>
+ </g>
+ <g id="line2d_26">
+ <g>
+ <use style="fill:#9467bd;fill-opacity:0.2;stroke:#9467bd;stroke-opacity:0.2;" x="399.04" xlink:href="#me6acc25a23" y="61.067656"/>
+ </g>
+ </g>
+ <g id="text_17">
+ <!-- I8O5$f = 2$ -->
+ <g transform="translate(411.64 63.517656)scale(0.07 -0.07)">
+ <use transform="translate(0 0.015625)" xlink:href="#DejaVuSans-73"/>
+ <use transform="translate(29.492188 0.015625)" xlink:href="#DejaVuSans-56"/>
+ <use transform="translate(93.115234 0.015625)" xlink:href="#DejaVuSans-79"/>
+ <use transform="translate(171.826172 0.015625)" xlink:href="#DejaVuSans-53"/>
+ <use transform="translate(235.449219 0.015625)" xlink:href="#DejaVuSans-Oblique-102"/>
+ <use transform="translate(290.136719 0.015625)" xlink:href="#DejaVuSans-61"/>
+ <use transform="translate(393.408203 0.015625)" xlink:href="#DejaVuSans-50"/>
+ </g>
+ </g>
+ <g id="line2d_27">
+ <path d="M 392.04 71.342344
+L 406.04 71.342344
+" style="fill:none;stroke:#8c564b;stroke-dasharray:1.5,4.5,1.5;stroke-dashoffset:0;stroke-opacity:0.2;stroke-width:1.5;"/>
+ </g>
+ <g id="line2d_28">
+ <g>
+ <use style="fill:#8c564b;fill-opacity:0.2;stroke:#8c564b;stroke-opacity:0.2;" x="399.04" xlink:href="#m4f5604d9a2" y="71.342344"/>
+ </g>
+ </g>
+ <g id="text_18">
+ <!-- I8O5$f = 4$ -->
+ <g transform="translate(411.64 73.792344)scale(0.07 -0.07)">
+ <use transform="translate(0 0.015625)" xlink:href="#DejaVuSans-73"/>
+ <use transform="translate(29.492188 0.015625)" xlink:href="#DejaVuSans-56"/>
+ <use transform="translate(93.115234 0.015625)" xlink:href="#DejaVuSans-79"/>
+ <use transform="translate(171.826172 0.015625)" xlink:href="#DejaVuSans-53"/>
+ <use transform="translate(235.449219 0.015625)" xlink:href="#DejaVuSans-Oblique-102"/>
+ <use transform="translate(290.136719 0.015625)" xlink:href="#DejaVuSans-61"/>
+ <use transform="translate(393.408203 0.015625)" xlink:href="#DejaVuSans-52"/>
+ </g>
+ </g>
+ </g>
+ </g>
+ </g>
+ <defs>
+ <clipPath id="p444e95376c">
+ <rect height="291.1975" width="399.31" x="50.69" y="10.8"/>
+ </clipPath>
+ </defs>
+</svg>
diff --git a/examples/convergence/fields_base.py b/examples/convergence/fields_base.py
new file mode 100644
index 0000000000000000000000000000000000000000..d2225d3245c01f2b2ff24cad482dae7cc364d6e4
--- /dev/null
+++ b/examples/convergence/fields_base.py
@@ -0,0 +1,78 @@
+import numpy as np
+import h5py
+import matplotlib.pyplot as plt
+
+def main():
+ return None
+
+def get_velocity(
+ cc,
+ iteration = 0):
+ data_file = h5py.File(
+ cc.get_checkpoint_fname(iteration = iteration),
+ 'r')
+ if 'velocity' in data_file.keys():
+ if 'real' in data_file['velocity'].keys():
+ vel = data_file['velocity/real/{0}'.format(iteration)][()]
+ else:
+ cvel = data_file['velocity/complex/{0}'.format(iteration)][()]
+ vel = ift(cvel)
+ else:
+ assert('complex' in data_file['vorticity'].keys())
+ cvort = data_file['vorticity/complex/{0}'.format(iteration)][()]
+ cvel = compute_curl(cc, cvort, inverse_curl = True)
+ vel = ift(cvel)
+ data_file.close()
+ return vel
+
+def ift(cfield):
+ ff = np.fft.irfftn(cfield, axes = (0, 1, 2))
+ field = np.transpose(ff, (1, 0, 2, 3)).copy()
+ return field*(field.shape[0]*field.shape[1]*field.shape[2])
+
+def compute_curl(
+ cc,
+ cfield,
+ inverse_curl = False):
+ kx = cc.get_data_file()['kspace/kx'][()]
+ ky = cc.get_data_file()['kspace/ky'][()]
+ kz = cc.get_data_file()['kspace/kz'][()]
+ ff = cfield.copy()
+ ff[..., 0] = 1j*(ky[:, None, None]*cfield[..., 2] - kz[None, :, None]*cfield[..., 1])
+ ff[..., 1] = 1j*(kz[None, :, None]*cfield[..., 0] - kx[None, None, :]*cfield[..., 2])
+ ff[..., 2] = 1j*(kx[None, None, :]*cfield[..., 1] - ky[:, None, None]*cfield[..., 0])
+ if inverse_curl:
+ k2 = (kx[None, None, :]**2 +
+ ky[ :, None, None]**2 +
+ kz[None, :, None]**2)
+ k2[0, 0, 0] = 1.
+ ff /= k2[:, :, :, None]
+ return ff
+
+def compute_vector_field_distance(
+ f1, f2,
+ figname = None):
+ diff = f1 - f2
+ rms1 = np.mean(np.sum(f1**2, axis = 3))**0.5
+ rms2 = np.mean(np.sum(f2**2, axis = 3))**0.5
+ rms = np.sqrt(rms1 * rms2)
+ dd = np.sum(diff**2, axis = 3)
+ L2 = np.sqrt(np.mean(dd))
+ max_val = np.max(np.sqrt(dd))
+ if type(figname) != type(None):
+ fig = plt.figure(figsize = (8, 4))
+ a = fig.add_subplot(121)
+ a.imshow(f1[:, 5, :, 2], vmin = -1, vmax = 1, interpolation = 'none')
+ a = fig.add_subplot(122)
+ a.imshow(f2[:, 5, :, 2], vmin = -1, vmax = 1, interpolation = 'none')
+ fig.tight_layout()
+ fig.savefig('field_slice_' + figname + '.pdf')
+ plt.close(fig)
+ return {'L2' : L2,
+ 'L2_rel' : L2 / rms,
+ 'max' : max_val,
+ 'max_rel' : max_val / rms}
+
+if __name__ == '__main__':
+ main()
+
diff --git a/examples/convergence/fields_temporal.py b/examples/convergence/fields_temporal.py
new file mode 100644
index 0000000000000000000000000000000000000000..d5e271be336b70303c6e2845a438293930834a6e
--- /dev/null
+++ b/examples/convergence/fields_temporal.py
@@ -0,0 +1,208 @@
+import numpy as np
+import h5py
+import matplotlib.pyplot as plt
+
+import TurTLE
+from TurTLE import DNS, PP
+
+from fields_base import *
+
+base_niterations = 512
+test_niterations = 32
+grid_size = 128
+
+def main():
+ generate_initial_conditions()
+
+ run_simulations_field(
+ err_vs_t = False)
+ plot_error_field(
+ err_vs_t = False)
+ return None
+
+def generate_initial_conditions():
+ # change these two values as needed.
+ # number of MPI processes to use
+ nprocesses = 8
+ # number of OpenMP threads per MPI process to use
+ nthreads_per_process = 1
+
+# 1. Generate quasistationary state to use for initial conditions.
+ # create a dns object
+ c0 = DNS()
+ # launch the simulation
+ c0.launch([
+ 'NSVE',
+ '-n', '{0}'.format(grid_size),
+ '--np', '{0}'.format(nprocesses),
+ '--ntpp', '{0}'.format(nthreads_per_process),
+ '--precision', 'double',
+ '--src-simname', 'B32p1e4',
+ '--src-wd', TurTLE.data_dir,
+ '--src-iteration', '0',
+ '--simname', 'base',
+ '--niter_todo', '{0}'.format(base_niterations),
+ '--niter_out', '{0}'.format(base_niterations),
+ '--overwrite-src-parameters',
+ '--kMeta', '1.5',
+ '--fk0', '2',
+ '--fk1', '3',
+ '--niter_stat', '16',
+ '--checkpoints_per_file', '{0}'.format(64)])
+
+# 3. Generate initial conditions for NSE runs.
+ # create postprocess object
+ cc = PP()
+ # launches code to compute velocity field at last iteration
+ cc.launch([
+ 'get_velocity',
+ '--np', '{0}'.format(nprocesses),
+ '--ntpp', '{0}'.format(nthreads_per_process),
+ '--simname', 'base',
+ '--precision', 'double',
+ '--iter0', '{0}'.format(base_niterations),
+ '--iter1', '{0}'.format(base_niterations)])
+ return None
+
+def run_simulations_field(err_vs_t = False):
+ # change these two values as needed.
+ # number of MPI processes to use
+ nprocesses = 8
+ # number of OpenMP threads per MPI process to use
+ nthreads_per_process = 1
+
+ if err_vs_t:
+ niter_out_factor = 1
+ else:
+ niter_out_factor = test_niterations
+
+# 1. Run NSVE for the three resolutions.
+ for factor in [1, 2, 4]:
+ # create dns object
+ cc = DNS()
+ # launch simulation
+ cc.launch([
+ 'NSVE',
+ '-n', '{0}'.format(grid_size),
+ '--np', '{0}'.format(nprocesses),
+ '--ntpp', '{0}'.format(nthreads_per_process),
+ '--src-simname', 'base',
+ '--src-iteration', '{0}'.format(base_niterations),
+ '--simname', 'nsve_{0}x'.format(factor),
+ '--precision', 'double',
+ '--dtfactor', '{0}'.format(0.5 / factor),
+ '--niter_todo', '{0}'.format(test_niterations*factor),
+ '--niter_out', '{0}'.format(niter_out_factor*factor),
+ '--niter_stat', '{0}'.format(factor)])
+
+# 2. Run NSE for the three resolutions.
+ for factor in [1, 2, 4]:
+ # create dns object
+ cc = DNS()
+ # launch simulation
+ cc.launch([
+ 'NSE',
+ '-n', '{0}'.format(grid_size),
+ '--np', '{0}'.format(nprocesses),
+ '--ntpp', '{0}'.format(nthreads_per_process),
+ '--src-simname', 'base',
+ '--src-iteration', '{0}'.format(base_niterations),
+ '--simname', 'nse_{0}x'.format(factor),
+ '--precision', 'double',
+ '--dtfactor', '{0}'.format(0.5 / factor),
+ '--niter_todo', '{0}'.format(test_niterations*factor),
+ '--niter_out', '{0}'.format(niter_out_factor*factor),
+ '--niter_out', '{0}'.format(test_niterations*factor),
+ '--niter_stat', '{0}'.format(factor)])
+ return None
+
+def plot_error_field(err_vs_t = False):
+ factor_list = [1, 2, 4]
+ c0_list = [DNS(simname = 'nsve_{0}x'.format(factor))
+ for factor in factor_list]
+ c1_list = [DNS(simname = 'nse_{0}x'.format(factor))
+ for factor in factor_list]
+ cl = [c0_list, c1_list]
+ # sanity checks
+ for cc in c0_list + c1_list:
+ cc.compute_statistics()
+
+ # errors for individual solvers
+ error_list = [[], [], []]
+ for ii in range(len(factor_list)-1):
+ factor = factor_list[ii]
+ for jj in [0, 1]:
+ vel1 = get_velocity(cl[jj][ii ], iteration = test_niterations*factor)
+ vel2 = get_velocity(cl[jj][ii+1], iteration = 2*test_niterations*factor)
+ dd = compute_vector_field_distance(vel1, vel2, figname = cl[jj][ii].simname + '_vs_' + cl[jj][ii+1].simname)
+ error_list[jj].append(dd['L2_rel'])
+
+ # comparisons of two solutions
+ for ii in range(len(factor_list)):
+ factor = factor_list[ii]
+ vel1 = get_velocity(cl[0][ii], iteration = test_niterations*factor)
+ vel2 = get_velocity(cl[1][ii], iteration = test_niterations*factor)
+ dd = compute_vector_field_distance(vel1, vel2)
+ error_list[2].append(dd['L2_rel'])
+
+ f = plt.figure(figsize = (4, 4))
+ a = f.add_subplot(111)
+ a.plot(factor_list[:len(error_list[0])],
+ error_list[0],
+ marker = '.',
+ dashes = (2, 3),
+ label = 'NSVE')
+ a.plot(factor_list[:len(error_list[1])],
+ error_list[1],
+ marker = '.',
+ dashes = (3, 5),
+ label = 'NSE')
+ a.plot(factor_list,
+ error_list[2],
+ marker = '.',
+ label = 'NSVE vs NSE')
+ fl = np.array(factor_list).astype(np.float)
+ for ee in [2, 3]:
+ a.plot(fl[:2], error_list[0][0] * fl[:2]**(-ee),
+ dashes = (ee, ee),
+ color = 'black',
+ label = '$\propto f^{{-{0}}}$'.format(ee),
+ zorder = -ee)
+ a.set_ylabel('relative error')
+ a.set_xlabel('resolution factor $f$')
+ a.set_xscale('log')
+ a.set_yscale('log')
+ a.legend(loc = 'best', fontsize = 6)
+ f.tight_layout()
+ f.savefig('err_vs_dt_field.pdf')
+ f.savefig('err_vs_dt_field.svg')
+ plt.close(f)
+
+ if err_vs_t:
+ t = range(test_niterations+1)
+ err_vs_t = [[], [], []]
+ # comparisons of two solvers
+ for ii in range(len(factor_list)):
+ factor = factor_list[ii]
+ for tt in t:
+ vel1 = get_velocity(cl[0][ii], iteration = tt*factor)
+ vel2 = get_velocity(cl[1][ii], iteration = tt*factor)
+ dd = compute_vector_field_distance(vel1, vel2)
+ err_vs_t[ii].append(dd['L2_rel'])
+ # distance between NSVE and NSE as a function of time
+ f = plt.figure(figsize = (4, 4))
+ a = f.add_subplot(111)
+ a.plot(t, err_vs_t[0], label = 'f = 1')
+ a.plot(t, err_vs_t[1], label = 'f = 2')
+ a.plot(t, err_vs_t[2], label = 'f = 4')
+ a.set_yscale('log')
+ a.legend(loc = 'best', fontsize = 6)
+ f.tight_layout()
+ f.savefig('err_vs_t_field.pdf')
+ f.savefig('err_vs_t_field.svg')
+ plt.close(f)
+ return None
+
+if __name__ == '__main__':
+ main()
+
diff --git a/examples/convergence/particles_temporal.py b/examples/convergence/particles_temporal.py
new file mode 100644
index 0000000000000000000000000000000000000000..c7b0ac15573a97031bd9a72e706674d03f6d8a8f
--- /dev/null
+++ b/examples/convergence/particles_temporal.py
@@ -0,0 +1,304 @@
+import numpy as np
+import h5py
+import matplotlib.pyplot as plt
+
+import TurTLE
+from TurTLE import DNS
+
+base_niterations = 256
+
+nparticles = 1000
+particle_random_seed = 15
+base_particle_dt = 0.5
+test_niterations_particles = 128
+
+neighbours_smoothness_list = [
+ (1, 1),
+ (3, 2)]
+
+def main():
+ generate_initial_conditions()
+
+ run_simulations_particles()
+
+ plot_error_particles()
+ plot_traj_particles()
+ return None
+
+def generate_initial_conditions():
+ # change these two values as needed.
+ # number of MPI processes to use
+ nprocesses = 8
+ # number of OpenMP threads per MPI process to use
+ nthreads_per_process = 1
+
+# 1. Generate quasistationary state to use for initial conditions.
+ # create a dns object
+ c0 = DNS()
+ # launch the simulation
+ c0.launch([
+ 'NSVE',
+ '-n', '32',
+ '--np', '{0}'.format(nprocesses),
+ '--ntpp', '{0}'.format(nthreads_per_process),
+ '--precision', 'double',
+ '--src-simname', 'B32p1e4',
+ '--src-wd', TurTLE.data_dir,
+ '--src-iteration', '0',
+ '--simname', 'base',
+ '--niter_todo', '{0}'.format(base_niterations),
+ '--niter_out', '{0}'.format(base_niterations),
+ '--overwrite-src-parameters',
+ '--kMeta', '1.',
+ '--niter_stat', '16',
+ '--checkpoints_per_file', '{0}'.format(16)])
+ return None
+
+def run_simulations_particles():
+ # change these two values as needed.
+ # number of MPI processes to use
+ nprocesses = 8
+ # number of OpenMP threads per MPI process to use
+ nthreads_per_process = 1
+
+# 1. Run NSVEparticles for a few iterations, to build up rhs values, consistent
+# with the relevant velocity field.
+ factor = 1
+ for neighbours, smoothness in neighbours_smoothness_list:
+ interp_name = 'I{0}O{1}'.format(2*neighbours+2, 2*smoothness+1)
+ # create dns object
+ cc = DNS()
+ # launch simulation
+ cc.launch([
+ 'NSVEparticles',
+ '-n', '{0}'.format(factor*32),
+ '--np', '{0}'.format(nprocesses),
+ '--ntpp', '{0}'.format(nthreads_per_process),
+ '--src-simname', 'base',
+ '--src-iteration', '{0}'.format(base_niterations),
+ '--simname', 'nsvep_base_' + interp_name,
+ '--precision', 'double',
+ '--dtfactor', '{0}'.format(base_particle_dt/4),
+ '--kMeta', '{0}'.format(factor*1.0),
+ '--niter_todo', '{0}'.format(20),
+ '--niter_out', '{0}'.format(1),
+ '--niter_stat', '{0}'.format(1),
+ '--nparticles', '{0}'.format(nparticles),
+ '--niter_part', '{0}'.format(1),
+ '--tracers0_neighbours', '{0}'.format(neighbours),
+ '--tracers0_smoothness', '{0}'.format(smoothness),
+ '--tracers0_integration_steps', '4',
+ '--cpp_random_particles', '{0}'.format(particle_random_seed)])
+
+# 2. Prepare initial conditions
+ for neighbours, smoothness in neighbours_smoothness_list:
+ interp_name = 'I{0}O{1}'.format(2*neighbours+2, 2*smoothness+1)
+ for factor in [1, 2, 4]:
+ df = h5py.File('nsvep_{0}x_{1}_checkpoint_0.h5'.format(factor, interp_name), 'w')
+ # field
+ field_iteration = 16
+ df['vorticity/complex/{0}'.format(8*factor)] = h5py.ExternalLink(
+ 'nsvep_base_{0}_checkpoint_0.h5'.format(interp_name),
+ 'vorticity/complex/{0}'.format(field_iteration))
+ # particles
+ df['tracers0/state/{0}'.format(8*factor)] = h5py.ExternalLink(
+ 'nsvep_base_{0}_checkpoint_0.h5'.format(interp_name),
+ 'tracers0/state/16')
+ # copy rhs
+ source_file = h5py.File(
+ 'nsvep_base_{0}_checkpoint_0.h5'.format(interp_name), 'r')
+ rhs = source_file['tracers0/rhs/16'][()]
+ for tt in range(1, rhs.shape[0]):
+ ii = 16 - tt*4//factor + 1
+ #print(factor, tt, ii)
+ rhs[tt] = source_file['tracers0/rhs/{0}'.format(16 - tt*4//factor + 1)][1]
+ df['tracers0/rhs/{0}'.format(8*factor)] = rhs
+ df.close()
+
+# 3. Run NSVEparticles
+ for factor in [1, 2, 4]:
+ # Test different interpolations.
+ # The loop iterates over number of neighbours and smoothness.
+ # The simulation names are defined based on "kernel size" and "order
+ # of polynomial".
+ # We do this to emphasize the one-to-one correspondence between
+ # neighbours and kernel size, and smoothness and order of polynomial.
+ for neighbours, smoothness in neighbours_smoothness_list:
+ interp_name = 'I{0}O{1}'.format(2*neighbours+2, 2*smoothness+1)
+ source_file = h5py.File('nsvep_base_{0}.h5'.format(interp_name), 'r')
+ # create dns object
+ cc = DNS()
+ # launch simulation
+ cc.launch([
+ 'NSVEparticles',
+ '-n', '{0}'.format(source_file['parameters/nx'][()]),
+ '--np', '{0}'.format(nprocesses),
+ '--ntpp', '{0}'.format(nthreads_per_process),
+ '--simname', 'nsvep_{0}x_{1}'.format(factor, interp_name),
+ '--precision', 'double',
+ '--dtfactor', '{0}'.format(base_particle_dt/factor),
+ '--niter_todo', '{0}'.format(test_niterations_particles*factor+8*factor),
+ '--niter_out', '{0}'.format(test_niterations_particles*factor+8*factor),
+ '--niter_stat', '{0}'.format(1),
+ ## ensure fluid physics is the same
+ '--dealias_type', '{0}'.format(source_file['parameters/dealias_type'][()]),
+ '--energy', '{0}'.format(source_file['parameters/energy'][()]),
+ '--famplitude', '{0}'.format(source_file['parameters/famplitude'][()]),
+ '--fk0', '{0}'.format(source_file['parameters/fk0'][()]),
+ '--fk1', '{0}'.format(source_file['parameters/fk1'][()]),
+ '--fmode', '{0}'.format(source_file['parameters/fmode'][()]),
+ '--friction_coefficient', '{0}'.format(source_file['parameters/friction_coefficient'][()]),
+ '--injection_rate', '{0}'.format(source_file['parameters/injection_rate'][()]),
+ '--nu', '{0}'.format(source_file['parameters/nu'][()]),
+ '--forcing_type', '{0}'.format(str(source_file['parameters/forcing_type'][()], 'ascii')),
+ ## particle params
+ '--nparticles', '{0}'.format(nparticles),
+ '--niter_part', '{0}'.format(1),
+ '--tracers0_neighbours', '{0}'.format(neighbours),
+ '--tracers0_smoothness', '{0}'.format(smoothness),
+ '--tracers0_integration_steps', '4',
+ '--cpp_random_particles', '0'], # turn off C++ particle initialization
+ iter0 = 8*factor)
+ source_file.close()
+ return None
+
+def plot_error_particles():
+ f = plt.figure()
+ a = f.add_subplot(111)
+ factor_list = [1, 2]
+ factor_list = np.array(factor_list).astype(np.float)
+ for neighbours, smoothness in neighbours_smoothness_list:
+ interp_name = 'I{0}O{1}'.format(2*neighbours+2, 2*smoothness+1)
+ err_max_list = []
+ err_mean_list = []
+ for factor in factor_list:
+ factor = int(factor)
+ c1 = DNS(simname = 'nsvep_{0}x_'.format(int(factor)) + interp_name)
+ c2 = DNS(simname = 'nsvep_{0}x_'.format(int(factor)*2) + interp_name)
+ for cc in [c1, c2]:
+ cc.parameters['niter_part'] = cc.get_data_file()['parameters/niter_part'][()]
+ c1.compute_statistics(iter0 = 8*factor)
+ c2.compute_statistics(iter0 = 8*factor*2)
+ final_iter_x1 = c1.get_data_file()['iteration'][()]
+ final_iter_x2 = c2.get_data_file()['iteration'][()]
+ f1 = h5py.File(c1.simname + '_checkpoint_0.h5', 'r')
+ f2 = h5py.File(c2.simname + '_checkpoint_0.h5', 'r')
+ x1 = f1['tracers0/state/{0}'.format(final_iter_x1)][()]
+ x2 = f2['tracers0/state/{0}'.format(final_iter_x2)][()]
+ diff = np.sqrt(np.sum((x2-x1)**2, axis = -1))
+ err_max_list.append(np.max(diff))
+ err_mean_list.append(np.mean(diff))
+ f1.close()
+ f2.close()
+ err_max_list = np.array(err_max_list)
+ err_mean_list = np.array(err_mean_list)
+ a.plot(factor_list, err_max_list, marker = '.', label = 'max error ' + interp_name)
+ a.plot(factor_list, err_mean_list, marker = '.', label = 'mean error ' + interp_name)
+ for ee in [2, 3, 4]:
+ a.plot(factor_list,
+ (1e-4)*factor_list**(-ee),
+ color = 'black',
+ dashes = (ee, ee),
+ label = '$\propto f^{{-{0}}}$'.format(ee))
+ a.set_xscale('log')
+ a.set_yscale('log')
+ a.set_xlabel('resolution factor f')
+ a.set_ylabel('absolute trajectory error [DNS units]')
+ a.legend(loc = 'best', fontsize = 7)
+ f.tight_layout()
+ f.savefig('err_vs_dt_particle.pdf')
+ f.savefig('err_vs_dt_particle.svg')
+ plt.close(f)
+ return None
+
+def plot_traj_particles():
+ ###########################################################################
+ f = plt.figure()
+ a = f.add_subplot(111)
+ for neighbours, smoothness in neighbours_smoothness_list:
+ interp_name = 'I{0}O{1}'.format(2*neighbours+2, 2*smoothness+1)
+ for factor in [1, 2, 4]:
+ cc = DNS(simname = 'nsvep_{0}x_'.format(factor) + interp_name)
+ cc.compute_statistics(iter0 = 8*factor)
+ tt = 3
+ xx = read_trajectory(cc, tt, iter0 = 8*factor)
+ a.plot(xx[:, 0],
+ xx[:, 1],
+ label = interp_name + '$f = {0}$'.format(factor),
+ marker = 'x',
+ dashes = (4/factor, 3, 4/factor),
+ alpha = 0.2)
+ #a.plot(xx[:, 0], xx[:, 1], dashes = (4/factor, 3, 4/factor), alpha = 0.2)
+ a.legend(loc = 'best', fontsize = 7)
+ a.set_xlabel('$x$ [code units]')
+ a.set_ylabel('$y$ [code units]')
+ f.tight_layout()
+ f.savefig('trajectories.pdf')
+ f.savefig('trajectories.svg')
+ plt.close(f)
+ ###########################################################################
+ ###########################################################################
+ traj_index = 3
+ for neighbours, smoothness in neighbours_smoothness_list:
+ interp_name = 'I{0}O{1}'.format(2*neighbours+2, 2*smoothness+1)
+ cc = DNS(simname = 'nsvep_base_' + interp_name)
+ cc.compute_statistics()
+ xx_base = read_trajectory(cc, traj_index, iter0 = 0, dset_name = 'velocity')
+ iter0 = 16
+ tt_base = np.array(range(iter0, iter0 + xx_base.shape[0]))*cc.parameters['dt']
+ f = plt.figure()
+ ax_counter = 1
+ for factor in [1, 2, 4]:
+ a = f.add_subplot(310 + ax_counter)
+ ax_counter += 1
+ a.plot(tt_base,
+ xx_base[:, 2],
+ label = 'base_' + interp_name,
+ marker = '+',
+ linewidth = 0.5,
+ alpha = 0.2)
+ cc = DNS(simname = 'nsvep_{0}x_'.format(factor) + interp_name)
+ cc.compute_statistics(iter0 = 8*factor)
+ xx = read_trajectory(cc, traj_index, iter0 = 8*factor, dset_name = 'velocity')
+ tt = np.array(range(8*factor, xx.shape[0]+8*factor))*cc.parameters['dt']
+ a.plot(tt,
+ xx[:, 2],
+ label = interp_name + '$f = {0}$'.format(factor),
+ marker = 'x',
+ dashes = (4/factor, 3, 4/factor),
+ alpha = 0.2)
+ df = h5py.File(cc.simname + '_checkpoint_0.h5', 'r')
+ xx = df['tracers0/rhs/{0}'.format(8*factor)][()]
+ for rhs_index in [1, 2, 3]:
+ a.scatter((8*factor - rhs_index)*cc.parameters['dt'],
+ xx[rhs_index, traj_index, 2],
+ marker = '*',
+ zorder = -10,
+ alpha = 0.5,
+ color = 'black')
+ df.close()
+ a.legend(loc = 'best', fontsize = 7)
+ a.set_xlabel('$t$ [code units]')
+ a.set_ylabel('$v_y$ [code units]')
+ f.tight_layout()
+ f.savefig('velocities_{0}.pdf'.format(interp_name))
+ plt.close(f)
+ ###########################################################################
+ return None
+
+def read_trajectory(
+ cc,
+ traj_index,
+ iter0,
+ dset_name = 'position'):
+ cc.parameters['niter_part'] = cc.get_data_file()['parameters/niter_part'][()]
+ df = cc.get_particle_file()
+ xx = []
+ for ii in range(iter0, cc.get_data_file()['iteration'][()]+1, cc.parameters['niter_part']):
+ xx.append(df['tracers0/' + dset_name + '/{0}'.format(ii)][traj_index])
+ df.close()
+ return np.array(xx)
+
+if __name__ == '__main__':
+ main()
+
diff --git a/examples/resize_example.sh b/examples/resize_example.sh
new file mode 100644
index 0000000000000000000000000000000000000000..c4df4b84c1285e7243af07b7d0778971f0ed3113
--- /dev/null
+++ b/examples/resize_example.sh
@@ -0,0 +1,25 @@
+#! /bin/bash
+
+# Example TurTLE resize script
+
+# run "small" simulation
+turtle DNS NSVE \
+ -n 32 --simname test32
+
+# double the size of a specific snapshot (iteration 8 in this case)
+# it's probably a good idea to always give "new_simname" as "old_simnamex2",
+# but you are free to choose as appropriate.
+# TODO: the python script should create an appropriate parameter file for new_simname.
+turtle PP resize \
+ --simname test32 \
+ --iter0 8 --iter1 8 \
+ --new_nx 64 --new_ny 64 --new_nz 64 --new_simname test32x2
+
+# create a checkpoint symlink to the resized field
+ln -s test32x2_fields.h5 test32x2_checkpoint_0.h5
+
+# run "large" simulation using the resized field
+turtle DNS NSVE \
+ -n 64 --simname test64 \
+ --src-simname test32x2 --src-iteration 8
+
diff --git a/get_version.py b/get_version.py
new file mode 100644
index 0000000000000000000000000000000000000000..3afea79d3d60df7a1b85e884ec2d851449aa281c
--- /dev/null
+++ b/get_version.py
@@ -0,0 +1,64 @@
+################################################################################
+# #
+# Copyright 2019 Max Planck Institute for Dynamics and Self-Organization #
+# #
+# This file is part of TurTLE. #
+# #
+# TurTLE is free software: you can redistribute it and/or modify #
+# it under the terms of the GNU General Public License as published #
+# by the Free Software Foundation, either version 3 of the License, #
+# or (at your option) any later version. #
+# #
+# TurTLE is distributed in the hope that it will be useful, #
+# but WITHOUT ANY WARRANTY; without even the implied warranty of #
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the #
+# GNU General Public License for more details. #
+# #
+# You should have received a copy of the GNU General Public License #
+# along with TurTLE. If not, see <http://www.gnu.org/licenses/> #
+# #
+# Contact: Cristian.Lalescu@ds.mpg.de #
+# #
+################################################################################
+
+
+
+import datetime
+import subprocess
+
+def main():
+ # get current time
+ now = datetime.datetime.now()
+ # obtain version
+ try:
+ git_branch = subprocess.check_output(['git',
+ 'rev-parse',
+ '--abbrev-ref',
+ 'HEAD']).strip().split()[-1].decode()
+ git_revision = subprocess.check_output(['git', 'rev-parse', 'HEAD']).strip()
+ git_date = datetime.datetime.fromtimestamp(int(subprocess.check_output(['git', 'log', '-1', '--format=%ct']).strip()))
+ except:
+ git_revision = ''
+ git_branch = ''
+ git_date = now
+ if git_branch == '':
+ # there's no git available or something
+ VERSION = '{0:0>4}{1:0>2}{2:0>2}.{3:0>2}{4:0>2}{5:0>2}'.format(
+ git_date.year, git_date.month, git_date.day,
+ git_date.hour, git_date.minute, git_date.second)
+ VERSION_py = VERSION
+ else:
+ VERSION = subprocess.check_output(['git', 'describe', '--tags']).strip().decode().split('-')[0]
+ if (('develop' in git_branch) or
+ ('feature' in git_branch) or
+ ('bugfix' in git_branch)):
+ VERSION_py = subprocess.check_output(
+ ['git', 'describe', '--tags', '--dirty']).strip().decode().replace('-g', '+g').replace('-dirty', '.dirty').replace('-', '.post')
+ else:
+ VERSION_py = VERSION
+ print(VERSION)
+ return VERSION_py
+
+if __name__ == '__main__':
+ main()
+
diff --git a/get_version_long.py b/get_version_long.py
new file mode 100644
index 0000000000000000000000000000000000000000..20accf0bf5a560fe621c64b56f20e210f54ec4e8
--- /dev/null
+++ b/get_version_long.py
@@ -0,0 +1,64 @@
+################################################################################
+# #
+# Copyright 2019 Max Planck Institute for Dynamics and Self-Organization #
+# #
+# This file is part of TurTLE. #
+# #
+# TurTLE is free software: you can redistribute it and/or modify #
+# it under the terms of the GNU General Public License as published #
+# by the Free Software Foundation, either version 3 of the License, #
+# or (at your option) any later version. #
+# #
+# TurTLE is distributed in the hope that it will be useful, #
+# but WITHOUT ANY WARRANTY; without even the implied warranty of #
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the #
+# GNU General Public License for more details. #
+# #
+# You should have received a copy of the GNU General Public License #
+# along with TurTLE. If not, see <http://www.gnu.org/licenses/> #
+# #
+# Contact: Cristian.Lalescu@ds.mpg.de #
+# #
+################################################################################
+
+
+
+import datetime
+import subprocess
+
+def main():
+ # get current time
+ now = datetime.datetime.now()
+ # obtain version
+ try:
+ git_branch = subprocess.check_output(['git',
+ 'rev-parse',
+ '--abbrev-ref',
+ 'HEAD']).strip().split()[-1].decode()
+ git_revision = subprocess.check_output(['git', 'rev-parse', 'HEAD']).strip()
+ git_date = datetime.datetime.fromtimestamp(int(subprocess.check_output(['git', 'log', '-1', '--format=%ct']).strip()))
+ except:
+ git_revision = ''
+ git_branch = ''
+ git_date = now
+ if git_branch == '':
+ # there's no git available or something
+ VERSION = '{0:0>4}{1:0>2}{2:0>2}.{3:0>2}{4:0>2}{5:0>2}'.format(
+ git_date.year, git_date.month, git_date.day,
+ git_date.hour, git_date.minute, git_date.second)
+ VERSION_py = VERSION
+ else:
+ VERSION = subprocess.check_output(['git', 'describe', '--tags']).strip().decode().split('-')[0]
+ if (('develop' in git_branch) or
+ ('feature' in git_branch) or
+ ('bugfix' in git_branch)):
+ VERSION_py = subprocess.check_output(
+ ['git', 'describe', '--tags', '--dirty']).strip().decode().replace('-g', '+g').replace('-dirty', '.dirty').replace('-', '.post')
+ else:
+ VERSION_py = VERSION
+ print(VERSION_py)
+ return VERSION_py
+
+if __name__ == '__main__':
+ main()
+
diff --git a/machine_settings_py.py b/machine_settings_py.py
deleted file mode 100644
index 787f1d5a10b9b0b260b42a1da18d35e67c56dacc..0000000000000000000000000000000000000000
--- a/machine_settings_py.py
+++ /dev/null
@@ -1,63 +0,0 @@
-#######################################################################
-# #
-# Copyright 2015 Max Planck Institute #
-# for Dynamics and Self-Organization #
-# #
-# This file is part of bfps. #
-# #
-# bfps is free software: you can redistribute it and/or modify #
-# it under the terms of the GNU General Public License as published #
-# by the Free Software Foundation, either version 3 of the License, #
-# or (at your option) any later version. #
-# #
-# bfps is distributed in the hope that it will be useful, #
-# but WITHOUT ANY WARRANTY; without even the implied warranty of #
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the #
-# GNU General Public License for more details. #
-# #
-# You should have received a copy of the GNU General Public License #
-# along with bfps. If not, see <http://www.gnu.org/licenses/> #
-# #
-# Contact: Cristian.Lalescu@ds.mpg.de #
-# #
-#######################################################################
-
-
-
-import os
-
-########################################################################
-# these lists should be adapted for your different environment(s)
-# personally, I have access to setups where my home folder is shared
-# between different machines, including cluster and desktop, therefore
-# I check the host name when choosing libraries etc.
-# feel free to do your own thing to the copy of this file placed in
-# ./config/bfps
-########################################################################
-
-hostname = os.getenv('HOSTNAME')
-
-compiler = 'g++'
-extra_compile_args = ['-Wall', '-O2', '-g', '-mtune=native', '-ffast-math', '-std=c++11']
-extra_libraries = ['hdf5']
-include_dirs = []
-library_dirs = []
-
-if hostname == 'chichi-G':
- include_dirs = ['/usr/local/include',
- '/usr/include/mpich']
- library_dirs = ['/usr/local/lib',
- '/usr/lib/mpich']
- extra_libraries += ['mpich']
-
-if hostname in ['tolima', 'misti']:
- local_install_dir = '/scratch.local/chichi/installs'
-
- include_dirs = ['/usr/lib64/mpi/gcc/openmpi/include',
- os.path.join(local_install_dir, 'include')]
-
- library_dirs = ['/usr/lib64/mpi/gcc/openmpi/lib64',
- os.path.join(local_install_dir, 'lib'),
- os.path.join(local_install_dir, 'lib64')]
- extra_libraries += ['mpi_cxx', 'mpi']
-
diff --git a/meta/count_nmodes.py b/meta/count_nmodes.py
new file mode 100644
index 0000000000000000000000000000000000000000..19af4ab332067ba72758bbc5244b33c8ea569dc0
--- /dev/null
+++ b/meta/count_nmodes.py
@@ -0,0 +1,34 @@
+import numpy as np
+
+def count_expensive(fk0, fk1):
+ kcomponent = np.arange(-np.floor(fk1)-1, np.floor(fk1)+2, 1).astype(np.float)
+ ksize = (kcomponent[:, None, None]**2 +
+ kcomponent[None, :, None]**2 +
+ kcomponent[None, None, :]**2)**.5
+ #print(ksize[0])
+
+ good_indices = np.where(np.logical_and(
+ ksize >= fk0,
+ ksize <= fk1))
+ #print(ksize[good_indices])
+ #print(good_indices[0].shape)
+ return np.unique(ksize[good_indices].flatten(), return_counts = True)
+
+def main():
+ for ff in [[1, 2],
+ [1.4, 2.3],
+ [1.4, 2.2]]:
+ modes, counts = count_expensive(ff[0], ff[1])
+ nmodes = np.sum(counts)
+ print(1 / ff[1], ff, nmodes)
+ modes_str = ''
+ counts_str = ''
+ for ii in range(counts.shape[0]):
+ modes_str += '{0:>5g}\t'.format(modes[ii])
+ counts_str += '{0:>5g}\t'.format(counts[ii])
+ print(modes_str + '\n' + counts_str + '\n')
+ return None
+
+if __name__ == '__main__':
+ main()
+
diff --git a/meta/fft_check/Makefile b/meta/fft_check/Makefile
new file mode 100644
index 0000000000000000000000000000000000000000..169ab9b6cf7c4e1e878f87429a3fdb826f81f321
--- /dev/null
+++ b/meta/fft_check/Makefile
@@ -0,0 +1,18 @@
+
+
+test_fftw:
+ ${MPICXX} \
+ -DPINCHECK_FOUND \
+ ${TURTLE_COMPILATION_FLAGS} \
+ -I${FFTW_ROOT}/include \
+ -I${PINCHECK_ROOT}/include \
+ -Wall \
+ -g \
+ -Wfatal-errors \
+ -fopenmp \
+ -std=gnu++11 \
+ test.cpp \
+ -o test_fft \
+ ${FFTW_OPENMP_LIB} \
+ ${FFTW_LIB}
+
diff --git a/meta/fft_check/main_code.hpp b/meta/fft_check/main_code.hpp
new file mode 100644
index 0000000000000000000000000000000000000000..9ad3cbf0daaa3be7176e23ad764aaea06fc60877
--- /dev/null
+++ b/meta/fft_check/main_code.hpp
@@ -0,0 +1,180 @@
+/**********************************************************************
+* *
+* Copyright 2021 Max Planck Institute *
+* for Dynamics and Self-Organization *
+* *
+* This file is part of TurTLE. *
+* *
+* TurTLE is free software: you can redistribute it and/or modify *
+* it under the terms of the GNU General Public License as published *
+* by the Free Software Foundation, either version 3 of the License, *
+* or (at your option) any later version. *
+* *
+* TurTLE is distributed in the hope that it will be useful, *
+* but WITHOUT ANY WARRANTY; without even the implied warranty of *
+* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the *
+* GNU General Public License for more details. *
+* *
+* You should have received a copy of the GNU General Public License *
+* along with TurTLE. If not, see <http://www.gnu.org/licenses/> *
+* *
+* Contact: Cristian.Lalescu@ds.mpg.de *
+* *
+**********************************************************************/
+
+
+
+#ifndef MAIN_CODE_HPP
+#define MAIN_CODE_HPP
+
+
+#include <mpi.h>
+#include <omp.h>
+#include <cfenv>
+#include <string>
+#include <iostream>
+#include <fftw3-mpi.h>
+#include <string>
+#include <cassert>
+#include <stdarg.h>
+
+int myrank, nprocs;
+
+#ifdef PINCHECK_FOUND
+#include <pincheck.hpp>
+
+void print_pinning_info(void)
+{
+ // obtain string with pinning information on rank 0,
+ // ranks >0 get an empty string
+ const std::string pinning_info = pincheck::pincheck();
+ if (myrank == 0)
+ {
+ std::cerr << "### pinning info begin" << std::endl;
+ std::cerr << pinning_info;
+ std::cerr << "### pinning info end" << std::endl;
+ std::cout << "### pinning info begin" << std::endl;
+ std::cout << pinning_info;
+ std::cout << "### pinning info end" << std::endl;
+ }
+}
+
+#else
+
+#define print_pinning_info(...)
+
+#endif
+
+
+#ifndef NDEBUG
+
+const int message_buffer_length = 32768;
+
+static char debug_message_buffer[message_buffer_length];
+
+inline void DEBUG_MSG(const char * format, ...)
+{
+ va_list argptr;
+ va_start(argptr, format);
+ sprintf(
+ debug_message_buffer,
+ "MPIrank%.4d ",
+ myrank);
+ vsnprintf(
+ debug_message_buffer + 12,
+ message_buffer_length - 12,
+ format,
+ argptr);
+ va_end(argptr);
+ std::cerr << debug_message_buffer;
+}
+
+inline void DEBUG_MSG_WAIT(MPI_Comm communicator, const char * format, ...)
+{
+ va_list argptr;
+ va_start(argptr, format);
+ sprintf(
+ debug_message_buffer,
+ "MPIrank%.4d ",
+ myrank);
+ vsnprintf(
+ debug_message_buffer + 12,
+ message_buffer_length - 12,
+ format,
+ argptr);
+ va_end(argptr);
+ std::cerr << debug_message_buffer;
+ MPI_Barrier(communicator);
+}
+
+#else
+
+#define DEBUG_MSG(...)
+#define DEBUG_MSG_WAIT(...)
+
+#endif//NDEBUG
+
+typedef int main_function_call (int argc, char *argv[], bool FPE);
+
+int main_wrapper(
+ int argc,
+ char *argv[],
+ bool floating_point_exceptions,
+ main_function_call *code_to_execute)
+{
+ /* floating point exception switch */
+ if (floating_point_exceptions)
+ feenableexcept(FE_INVALID | FE_OVERFLOW);
+ else
+ std::cerr << "FPE have been turned OFF" << std::endl;
+
+ /* initialize mpi with threads */
+ int mpiprovided;
+ MPI_Init_thread(&argc, &argv, MPI_THREAD_FUNNELED, &mpiprovided);
+ MPI_Pcontrol(0);
+ assert(mpiprovided >= MPI_THREAD_FUNNELED);
+ MPI_Comm_rank(MPI_COMM_WORLD, &myrank);
+ MPI_Comm_size(MPI_COMM_WORLD, &nprocs);
+
+ print_pinning_info();
+
+ /* initialize fftw with mpi and threads */
+ const int nThreads = omp_get_max_threads();
+ DEBUG_MSG("Number of threads for the FFTW = %d\n",
+ nThreads);
+ if (nThreads > 1){
+ fftw_init_threads();
+ fftwf_init_threads();
+ }
+ fftw_mpi_init();
+ fftwf_mpi_init();
+ DEBUG_MSG("There are %d processes and %d threads\n",
+ nprocs,
+ nThreads);
+ if (nThreads > 1){
+ fftw_plan_with_nthreads(nThreads);
+ fftwf_plan_with_nthreads(nThreads);
+ }
+
+ fftwf_set_timelimit(300);
+ fftw_set_timelimit(300);
+
+ int code_result = code_to_execute(argc, argv, floating_point_exceptions);
+
+ std::cerr << "main code returned " << code_result << std::endl;
+
+ /* clean up */
+ fftwf_mpi_cleanup();
+ fftw_mpi_cleanup();
+ if (nThreads > 1){
+ fftw_cleanup_threads();
+ fftwf_cleanup_threads();
+ }
+
+ MPI_Finalize();
+ return EXIT_SUCCESS;
+}
+
+
+#endif//MAIN_CODE_HPP
+
diff --git a/meta/fft_check/test.cpp b/meta/fft_check/test.cpp
new file mode 100644
index 0000000000000000000000000000000000000000..ef476296c23cf2280e857cd03307d7862705a718
--- /dev/null
+++ b/meta/fft_check/test.cpp
@@ -0,0 +1,119 @@
+#include "main_code.hpp"
+
+#include <random>
+
+/****************************/
+// parameters
+
+const int nx = 4096;
+const int ny = 8;
+const int nz = 4096;
+
+const int nsteps = 100;
+
+/****************************/
+
+int print_plan(fftw_plan &pl)
+{
+ char *plan_information = fftw_sprint_plan(pl);
+ if (myrank == 0)
+ DEBUG_MSG("\n\n%s\n\n", plan_information);
+
+ free(plan_information);
+ return EXIT_SUCCESS;
+}
+
+int test_fft(
+ int argc,
+ char *argv[],
+ bool FPE)
+{
+ ptrdiff_t nfftw[3] = {nz, ny, nx};
+ ptrdiff_t local_n0, local_0_start;
+ ptrdiff_t local_n1, local_1_start;
+
+ double *data = NULL;
+
+ fftw_plan c2r_plan;
+ fftw_plan r2c_plan;
+ unsigned fftw_plan_rigor = FFTW_MEASURE;
+
+ ptrdiff_t local_size = fftw_mpi_local_size_many_transposed(
+ 3, nfftw, 1,
+ FFTW_MPI_DEFAULT_BLOCK, FFTW_MPI_DEFAULT_BLOCK,
+ MPI_COMM_WORLD,
+ &local_n0, &local_0_start,
+ &local_n1, &local_1_start);
+
+ /************/
+ /* ALLOCATE */
+ /************/
+ data = fftw_alloc_real(local_size*2);
+
+ c2r_plan = fftw_mpi_plan_many_dft_c2r(
+ 3, nfftw, 1,
+ FFTW_MPI_DEFAULT_BLOCK, FFTW_MPI_DEFAULT_BLOCK,
+ (fftw_complex*)(data),
+ data,
+ MPI_COMM_WORLD,
+ fftw_plan_rigor | FFTW_MPI_TRANSPOSED_IN);
+
+ assert(c2r_plan != NULL);
+
+ r2c_plan = fftw_mpi_plan_many_dft_r2c(
+ 3, nfftw, 1,
+ FFTW_MPI_DEFAULT_BLOCK, FFTW_MPI_DEFAULT_BLOCK,
+ data,
+ (fftw_complex*)(data),
+ MPI_COMM_WORLD,
+ fftw_plan_rigor | FFTW_MPI_TRANSPOSED_OUT);
+
+ assert(r2c_plan != NULL);
+
+ DEBUG_MSG("r2c plan representation\n");
+ print_plan(r2c_plan);
+ DEBUG_MSG("c2r plan representation\n");
+ print_plan(c2r_plan);
+
+ // fill up data
+ std::random_device rd{};
+ std::mt19937 gen{rd()};
+
+ std::normal_distribution<> gaussian;
+
+
+ for (ptrdiff_t ii = 0; ii < local_size; ii++)
+ data[ii] = gaussian(gen);
+
+ // start mpi profiling
+ MPI_Pcontrol(5);
+
+ for (ptrdiff_t tt = 0; tt < nsteps; tt++)
+ {
+ fftw_execute(r2c_plan);
+ fftw_execute(c2r_plan);
+ #pragma omp parallel for schedule(static)
+ for(ptrdiff_t ii = 0; ii < local_size; ii++)
+ data[ii] /= nx*ny*nz;
+ }
+
+ // stop mpi profiling
+ MPI_Pcontrol(-5);
+
+
+ /************/
+ /* FREE */
+ /************/
+ fftw_free(data);
+ fftw_destroy_plan(c2r_plan);
+ fftw_destroy_plan(r2c_plan);
+
+ return EXIT_SUCCESS;
+}
+
+int main(int argc,
+ char *argv[])
+{
+ return main_wrapper(argc, argv, true, test_fft);
+}
+
diff --git a/meta/logo.py b/meta/logo.py
new file mode 100644
index 0000000000000000000000000000000000000000..e3d9da5494e5a73b7848d378846be58b7079ca4c
--- /dev/null
+++ b/meta/logo.py
@@ -0,0 +1,579 @@
+import matplotlib.path
+Path = matplotlib.path.Path
+import matplotlib.patches as mpatches
+import matplotlib.pyplot as plt
+import numpy as np
+
+MAX_PLANCK_GREEN = (0, 125/255., 122./255)
+
+def typed():
+ plt.rcParams.update({
+ 'text.usetex' : False,
+ 'font.serif' : ['Gentium Book Basic'],
+ 'font.sans-serif' : ['DejaVu Sans'],
+ 'mathtext.fallback' : 'cm',
+ })
+
+ f = plt.figure(figsize = (4, 2.25))
+ a = f.add_axes([0, 0, 1, 1])
+
+ # T
+ a.text(-1.1, 0.1, 'T', fontsize = 160,
+ horizontalalignment = 'center',
+ fontfamily = 'sans',
+ verticalalignment = 'center',
+ color = MAX_PLANCK_GREEN,
+ )
+
+ # ur
+ a.text(0, 0, 'ur', fontsize = 80,
+ horizontalalignment = 'center',
+ #fontfamily = 'serif',
+ fontfamily = 'sans',
+ #fontweight = 'bold',
+ color = MAX_PLANCK_GREEN,
+ )
+
+ # T
+ a.text(1.2, 0.1, 'T', fontsize = 160,
+ horizontalalignment = 'center',
+ fontfamily = 'sans',
+ verticalalignment = 'center',
+ color = MAX_PLANCK_GREEN,
+ )
+
+ # L
+ a.text(2.3, 0, 'L', fontsize = 80,
+ fontfamily = 'sans',
+ horizontalalignment = 'center',
+ color = MAX_PLANCK_GREEN,
+ )
+
+ # E
+ a.text(3.1, 0, 'E', fontsize = 80,
+ fontfamily = 'sans',
+ horizontalalignment = 'center',
+ color = MAX_PLANCK_GREEN,
+ )
+ a.set_aspect('equal')
+ a.set_axis_off()
+ a.set_xlim(-2.35, 3.55)
+ a.set_ylim(-1., 2.3)
+ f.savefig('TurTLE_logo.pdf')
+ return None
+
+def drawn_full():
+ plt.rcParams.update({
+ 'text.usetex' : False,
+ 'font.family' : 'sans-serif',
+ 'font.sans-serif' : ['Linux Biolinum O'],
+ 'font.weight' :'bold',
+ 'mathtext.fallback' : 'cm',
+ })
+
+ f = plt.figure(figsize = (4, 2.25))
+ a = f.add_axes([0, 0, 1, 1])
+
+ # bottom T
+ pp1 = mpatches.PathPatch(
+ Path([(-1, 1),
+ (-1.2, 1),
+ (-2, -0.7),
+ (-1.5, -0.7),
+ (-0.8, -0.7),
+ (-0.8, 1),
+ (-1, 1),
+ (-1, 1)],
+ [Path.MOVETO,
+ Path.CURVE4,
+ Path.CURVE4,
+ Path.CURVE4,
+ Path.CURVE4,
+ Path.CURVE4,
+ Path.CURVE4,
+ Path.CLOSEPOLY]),
+ fc=MAX_PLANCK_GREEN,
+ ec=MAX_PLANCK_GREEN,
+ transform=a.transData)
+ a.add_patch(pp1)
+
+ # top T
+ pp1 = mpatches.PathPatch(
+ Path([(-2.5, 0),
+ (-2, 0.125),
+ (-2, 2.1),
+ (-0.355, 2.1)] +
+ smooth_corner_coords(
+ p0 = (-0.355, 2.1),
+ p1 = ( 0.055, 1.7),
+ angle = np.pi/4) +
+ smooth_corner_coords(
+ p0 = ( 0.055, 1.7),
+ p1 = (-0.355, 1.3),
+ angle = -np.pi/4) +
+ [(-2, 1.3),
+ (-1.5, 0),
+ (-2.5, 0),
+ (-2.5, 0),
+ ],
+ [Path.MOVETO,
+ Path.CURVE4,
+ Path.CURVE4,
+ Path.CURVE4,
+ Path.CURVE4,
+ Path.CURVE4,
+ Path.CURVE4,
+ Path.CURVE4,
+ Path.CURVE4,
+ Path.CURVE4,
+ Path.CURVE4,
+ Path.CURVE4,
+ Path.CURVE4,
+ Path.CURVE4,
+ Path.CURVE4,
+ Path.CURVE4,
+ Path.CURVE4,
+ Path.CURVE4,
+ Path.CURVE4,
+ Path.CLOSEPOLY]),
+ fc = MAX_PLANCK_GREEN,
+ ec=MAX_PLANCK_GREEN,
+ transform=a.transData)
+ a.add_patch(pp1)
+
+ # ur
+ a.text(0, 0, 'ur', fontsize = 72,
+ horizontalalignment = 'center',
+ #verticalalignment = 'center',
+ color = MAX_PLANCK_GREEN,
+ )
+
+ # bottom T
+ pp1 = mpatches.PathPatch(
+ Path([(1, 1),
+ (1.2, 1),
+ (2, -0.7),
+ (1.5, -0.7),
+ (0.8, -0.7),
+ (0.8, 1),
+ (1, 1),
+ (1, 1)],
+ [Path.MOVETO,
+ Path.CURVE4,
+ Path.CURVE4,
+ Path.CURVE4,
+ Path.CURVE4,
+ Path.CURVE4,
+ Path.CURVE4,
+ Path.CLOSEPOLY]),
+ fc=MAX_PLANCK_GREEN,
+ ec=MAX_PLANCK_GREEN,
+ transform=a.transData)
+ a.add_patch(pp1)
+
+ # top T
+ pp1 = mpatches.PathPatch(
+ Path([
+ (0.515, 2.1)] +
+ smooth_corner_coords(
+ p0 = (0.515, 2.1),
+ p1 = (0.115, 1.7),
+ angle = 3*np.pi/4) +
+ smooth_corner_coords(
+ p0 = (0.115, 1.7),
+ p1 = (0.515, 1.3),
+ angle = -3*np.pi/4) +
+ [(0.9, 1.3),
+ (1.05, 1.1),
+ (1.275, 1.1),
+ (1.3, 1.1),
+ (1.35, 1.1),
+ (1.45, 1.2)] +
+ smooth_corner_coords(
+ p0 = (1.45, 1.2),
+ p1 = (1.45, 1.6),
+ angle = 0) +
+ [(1.4, 1.65),
+ (1., 2.1),
+ (0.415, 2.1),
+ (0.415, 2.1)],
+ [Path.MOVETO,
+ Path.CURVE4,
+ Path.CURVE4,
+ Path.CURVE4,
+ Path.CURVE4,
+ Path.CURVE4,
+ Path.CURVE4,
+ Path.CURVE4,
+ Path.CURVE4,
+ Path.CURVE4,
+ Path.CURVE4,
+ Path.CURVE4,
+ Path.CURVE4,
+ Path.CURVE4,
+ Path.CURVE4,
+ Path.CURVE4,
+ Path.CURVE4,
+ Path.CURVE4,
+ Path.CURVE4,
+ Path.CURVE4,
+ Path.CURVE4,
+ Path.CURVE4,
+ Path.CURVE4,
+ Path.CURVE4,
+ Path.CURVE4,
+ Path.CURVE4,
+ Path.CURVE4,
+ Path.CURVE4,
+ Path.CLOSEPOLY]),
+ fc = MAX_PLANCK_GREEN,
+ ec=MAX_PLANCK_GREEN,
+ transform=a.transData)
+ a.add_patch(pp1)
+
+ # L
+ pp1 = mpatches.PathPatch(
+ Path([
+ (1.5, 1.15)] +
+ smooth_corner_coords(
+ p0 = (1.5, 1.15),
+ p1 = (1.8, 1.15),
+ angle = np.pi/2) +
+ [(1.8, 1.15),
+ (2.15, 0.75),
+ (2.15, 0.25),
+ (2.45, 0.25),
+ (2.47, 0.2),
+ (2.6, 0.02),
+ (2.75, 0.),
+ (1.9, 0.),
+ (1.9, 0.25),
+ (1.8, 0.75),
+ (1.5, 0.95),
+ ] +
+ smooth_corner_coords(
+ p0 = (1.5, 0.95),
+ p1 = (1.5, 1.15),
+ angle = -np.pi) +
+ [(1.5, 1.15)],
+ [Path.MOVETO,
+ Path.CURVE4,
+ Path.CURVE4,
+ Path.CURVE4,
+ Path.CURVE4,
+ Path.CURVE4,
+ Path.CURVE4,
+ Path.CURVE4,
+ Path.CURVE4,
+ Path.CURVE4,
+ Path.LINETO,
+ Path.CURVE4,
+ Path.CURVE4,
+ Path.CURVE4,
+ Path.LINETO,
+ Path.CURVE4,
+ Path.CURVE4,
+ Path.CURVE4,
+ Path.CURVE4,
+ Path.CURVE4,
+ Path.CURVE4,
+ Path.CURVE4,
+ Path.CURVE4,
+ Path.CURVE4,
+ Path.CLOSEPOLY]),
+ fc = MAX_PLANCK_GREEN,
+ ec=MAX_PLANCK_GREEN,
+ transform=a.transData)
+ a.add_patch(pp1)
+
+ # E
+ pp1 = mpatches.PathPatch(
+ Path([
+ (2.5, 0.4),
+ (2.5, 1.9),
+ (3.2, 1.3),
+ (3.2, 0.75),
+ (2.7, 1.2),
+ (2.6, 0.6),
+ (3.1, 0.6),
+ (2.7, 0.6),
+ (2.6, 0.2),
+ (3.2, 0.4),
+ (3.2, -0.2),
+ (2.5, 0.),
+ (2.5, 0.4),
+ (2.5, 0.4)],
+ [Path.MOVETO,
+ Path.CURVE4,
+ Path.CURVE4,
+ Path.CURVE4,
+ Path.CURVE4,
+ Path.CURVE4,
+ Path.CURVE4,
+ Path.CURVE4,
+ Path.CURVE4,
+ Path.CURVE4,
+ Path.CURVE4,
+ Path.CURVE4,
+ Path.CURVE4,
+ Path.CLOSEPOLY]),
+ fc=MAX_PLANCK_GREEN,
+ ec=MAX_PLANCK_GREEN,
+ transform=a.transData)
+ a.add_patch(pp1)
+
+ ## LE
+ #pp1 = mpatches.PathPatch(
+ # Path([(1.9, 0),
+ # (1.9, 0.75),
+ # (1.3, 1.5),
+ # (0, 1.5),
+ # (-2, 1.5),
+ # (-1.5, 0),
+ # (-2.5, 0),
+ # (-2, 0.125),
+ # (-2, 2.2),
+ # (0, 2.2),
+ # (1.5, 2.2),
+ # (2.1, 0.8),
+ # (2.1, 0.2),
+ # (2.5, 0.2),
+ # (2.5, 1.9),
+ # (3.2, 1.3),
+ # (3.2, 0.75),
+ # (2.7, 1.2),
+ # (2.6, 0.6),
+ # (3.1, 0.6),
+ # (2.7, 0.6),
+ # (2.6, 0.2),
+ # (3.2, 0.4),
+ # (3.2, -0.2),
+ # (2.5, 0.),
+ # (1.9, 0),
+ # (1.9, 0)],
+ # [Path.MOVETO,
+ # Path.CURVE4,
+ # Path.CURVE4,
+ # Path.CURVE4,
+ # Path.CURVE4,
+ # Path.CURVE4,
+ # Path.CURVE4,
+ # Path.CURVE4,
+ # Path.CURVE4,
+ # Path.CURVE4,
+ # Path.CURVE4,
+ # Path.CURVE4,
+ # Path.CURVE3,
+ # Path.LINETO,
+ # Path.CURVE4,
+ # Path.CURVE4,
+ # Path.CURVE4,
+ # Path.CURVE4,
+ # Path.CURVE4,
+ # Path.CURVE4,
+ # Path.CURVE4,
+ # Path.CURVE4,
+ # Path.CURVE4,
+ # Path.CURVE4,
+ # Path.CURVE4,
+ # Path.CURVE4,
+ # Path.CLOSEPOLY]),
+ # #fc=MAX_PLANCK_GREEN,
+ # fc='none',
+ # linestyle = ':',
+ # ec='black',
+ # transform=a.transData)
+ #a.add_patch(pp1)
+
+ a.set_aspect('equal')
+ a.set_axis_off()
+ a.set_xlim(-2.6, 3.3)
+ a.set_ylim(-1., 2.3)
+ f.savefig('TurTLE_logo.pdf')
+ f.savefig('TurTLE_logo.svg')
+ return None
+
+def drawn():
+ plt.rcParams.update({
+ 'text.usetex' : False,
+ 'font.family' : 'sans-serif',
+ 'font.sans-serif' : ['Gentium Book Basic'],
+ 'font.weight' :'bold',
+ 'mathtext.fallback' : 'cm',
+ })
+
+
+ f = plt.figure(figsize = (4, 2.25))
+ a = f.add_axes([0, 0, 1, 1])
+
+ # T
+ pp1 = mpatches.PathPatch(
+ Path([(-1, 1), (-2.5, -1.5), (-0.5, -1.5), (-1, 1), (-1, 1)],
+ [Path.MOVETO,
+ Path.CURVE4,
+ Path.CURVE4,
+ Path.CURVE4,
+ Path.CLOSEPOLY]),
+ fc=MAX_PLANCK_GREEN,
+ ec=MAX_PLANCK_GREEN,
+ transform=a.transData)
+ a.add_patch(pp1)
+
+ # ur
+ a.text(0, 0, 'ur', fontsize = 80,
+ horizontalalignment = 'center',
+ #verticalalignment = 'center',
+ color = MAX_PLANCK_GREEN,
+ )
+
+ # T
+ pp1 = mpatches.PathPatch(
+ Path([(1, 1), (2.5, -1.5), (0.5, -1.5), (1, 1), (1, 1)],
+ [Path.MOVETO,
+ Path.CURVE4,
+ Path.CURVE4,
+ Path.CURVE4,
+ Path.CLOSEPOLY]),
+ fc=MAX_PLANCK_GREEN,
+ ec=MAX_PLANCK_GREEN,
+ transform=a.transData)
+ a.add_patch(pp1)
+
+ # LE
+ pp1 = mpatches.PathPatch(
+ Path([(1.9, 0),
+ (1.9, 0.75),
+ (1.3, 1.5),
+ (0, 1.5),
+ (-2, 1.5),
+ (-1.5, 0),
+ (-2.5, 0),
+ (-2, 0.125),
+ (-2, 2.2),
+ (0, 2.2),
+ (1.5, 2.2),
+ (2.1, 0.8),
+ (2.1, 0.2),
+ (2.5, 0.2),
+ (2.5, 1.9),
+ (3.2, 1.3),
+ (3.2, 0.75),
+ (2.7, 1.2),
+ (2.6, 0.6),
+ (3.1, 0.6),
+ (2.7, 0.6),
+ (2.6, 0.2),
+ (3.2, 0.4),
+ (3.2, -0.2),
+ (2.5, 0.),
+ (1.9, 0),
+ (1.9, 0)],
+ [Path.MOVETO,
+ Path.CURVE4,
+ Path.CURVE4,
+ Path.CURVE4,
+ Path.CURVE4,
+ Path.CURVE4,
+ Path.CURVE4,
+ Path.CURVE4,
+ Path.CURVE4,
+ Path.CURVE4,
+ Path.CURVE4,
+ Path.CURVE4,
+ Path.CURVE3,
+ Path.LINETO,
+ Path.CURVE4,
+ Path.CURVE4,
+ Path.CURVE4,
+ Path.CURVE4,
+ Path.CURVE4,
+ Path.CURVE4,
+ Path.CURVE4,
+ Path.CURVE4,
+ Path.CURVE4,
+ Path.CURVE4,
+ Path.CURVE4,
+ Path.CURVE4,
+ Path.CLOSEPOLY]),
+ fc=MAX_PLANCK_GREEN,
+ ec=MAX_PLANCK_GREEN,
+ transform=a.transData)
+ a.add_patch(pp1)
+
+ a.set_aspect('equal')
+ a.set_axis_off()
+ a.set_xlim(-2.6, 3.3)
+ a.set_ylim(-1., 2.3)
+ f.savefig('TurTLE_logo.pdf')
+ return None
+
+def smooth_corner_coords(
+ p0 = (-1, 1),
+ p1 = (1, -1),
+ angle = 0):
+ O = ((p0[0] + p1[0])/2,
+ (p0[1] + p1[1])/2)
+ dP = (p1[0] - p0[0], p1[1] - p0[1])
+ radius = 0.5*(dP[0]**2+dP[1]**2)**0.5
+ PP = (O[0] + radius*np.cos(angle),
+ O[1] + radius*np.sin(angle))
+ print(PP)
+ p2 = ((p0[0] + PP[0])/2,
+ (p0[1] + PP[1])/2)
+ p4 = ((p1[0] + PP[0])/2,
+ (p1[1] + PP[1])/2)
+ p3 = ((p2[0] + p4[0])/2,
+ (p2[1] + p4[1])/2)
+ return [p2, p2, p3, p4, p4, p1]
+
+def test_corner():
+ f = plt.figure(figsize = (4, 2.25))
+ a = f.add_axes([0, 0, 1, 1])
+ print(
+ smooth_corner_coords())
+
+ pp1 = mpatches.PathPatch(
+ Path(#[(-1, 1),
+ # (0., 1),
+ # (0., 1),
+ # (0.5, 0.5),
+ # (1, 0),
+ # (1, 0),
+ # (1, -1),
+ # (1, -1)],
+ [(-1, 1)]+
+ smooth_corner_coords(angle = np.pi/4)+
+ smooth_corner_coords(p0 = (1, -1), p1 = (-0.5, 1), angle = -3*np.pi/4)+
+ [(-1, 1)],
+ [Path.MOVETO,
+ Path.CURVE4,
+ Path.CURVE4,
+ Path.CURVE4,
+ Path.CURVE4,
+ Path.CURVE4,
+ Path.CURVE4,
+ Path.CURVE4,
+ Path.CURVE4,
+ Path.CURVE4,
+ Path.CURVE4,
+ Path.CURVE4,
+ Path.CURVE4,
+ Path.CLOSEPOLY]),
+ fc='none',
+ ec=MAX_PLANCK_GREEN,
+ transform=a.transData)
+ a.add_patch(pp1)
+ a.set_axis_off()
+
+ a.scatter([0, 0, 1, 1], [0, 1, 0, 1])
+
+ a.set_xlim(-2, 2)
+ a.set_ylim(-2, 2)
+ a.set_aspect('equal')
+ f.savefig('tmp.pdf')
+ return None
+
+if __name__ == '__main__':
+ drawn_full()
+ #test_corner()
+
diff --git a/meta/logo.tex b/meta/logo.tex
new file mode 100644
index 0000000000000000000000000000000000000000..91128a6ea34a85e110f4e426b9a136a7c472377a
--- /dev/null
+++ b/meta/logo.tex
@@ -0,0 +1,33 @@
+\documentclass{standalone}
+\usepackage{tikz}
+%\usepackage[cm-default]{fontspec}
+%\setmainfont[Mapping = {tex-text}]{DejaVu Sans}
+
+\newcommand{\codename}{%
+ \textsf{
+ \vphantom{\begin{Large}|\end{Large}}
+ \hspace{-1.2em}
+ \begin{Large}
+ \begin{tikzpicture}
+ \coordinate (0);
+ \node [inner sep=-3pt, anchor=center, style={yslant=0.5}] {T};
+ \end{tikzpicture}
+ \end{Large}
+ \hspace{-1.1em}
+ ur
+ \hspace{-1.1em}
+ \begin{Large}
+ \begin{tikzpicture}
+ \coordinate (0);
+ \node [inner sep=-3pt, anchor=center, style={yslant=-0.5}] {T};
+ \end{tikzpicture}
+ \end{Large}
+ \hspace{-1.2em}
+ \begin{footnotesize}
+ LE%
+ \end{footnotesize}}}
+
+\begin{document}
+\textbf{Tur}bulence \textbf{T}ools: \textbf{L}agrangian and \textbf{E}ulerian, abbreviated as \codename.
+\end{document}
+
diff --git a/meta/tools/core_allocation_masks/distribute_cores_evenly.py b/meta/tools/core_allocation_masks/distribute_cores_evenly.py
new file mode 100644
index 0000000000000000000000000000000000000000..29cd5971dff8352c3fda5451a2f923618324a1dc
--- /dev/null
+++ b/meta/tools/core_allocation_masks/distribute_cores_evenly.py
@@ -0,0 +1,49 @@
+import numpy as np
+
+def distribute_cores_evenly(
+ nprocesses = 4,
+ nthreads_per_process = 3,
+ total_number_of_cores = 16):
+ assert(nprocesses*nthreads_per_process <= total_number_of_cores)
+
+ # first, determine how many total cores we can allocate per process
+ max_cores_per_process = total_number_of_cores // nprocesses
+
+ # then spread useful cores evenly throughout the total cores per process
+ # start with no cores allocated
+ single_process_mask = np.zeros(max_cores_per_process, np.bool)
+ # allocate cores evenly
+ skip = max_cores_per_process // nthreads_per_process
+ for t in range(nthreads_per_process):
+ single_process_mask[t*skip] = 1
+
+ single_process_mask = sum(int(single_process_mask[i])*(2**i) for i in range(max_cores_per_process))
+
+ # now create full node mask:
+ all_masks = []
+ for p in range(nprocesses):
+ all_masks.append(single_process_mask*(2**(max_cores_per_process*p)))
+ return all_masks
+
+def main():
+ """
+ The point is, in principle, to generate a submission script from python,
+ and change the core allocation masks from within python (thus allowing
+ to generate multiple submision scripts at the same time).
+ This is why I generate an `export SLURM_CPU_BIND` line, which will be
+ specific to each script, but can be used in a generic `srun ${SLURM...`
+ line afterwards.
+ """
+ core_masks = distribute_cores_evenly(
+ nprocesses = 8,
+ nthreads_per_process = 3,
+ total_number_of_cores = 40)
+ print('place following line in SLURM script:\n' +
+ 'export SLURM_CPU_BIND_OPTION="--cpu-bind=verbose,mask_cpu:' +
+ ','.join(['0x{0:x}'.format(mm) for mm in core_masks]) + '"\n' +
+ 'then use `srun ${SLURM_CPU_BIND_OPTION}` in the script to launch the executable')
+ return None
+
+if __name__ == '__main__':
+ main()
+
diff --git a/meta/tools/core_allocation_test/compile.sh b/meta/tools/core_allocation_test/compile.sh
new file mode 100644
index 0000000000000000000000000000000000000000..3ad47d8355d863572f5759cc9d6c6c6e6b87fa15
--- /dev/null
+++ b/meta/tools/core_allocation_test/compile.sh
@@ -0,0 +1 @@
+mpicxx -Wall -std=c++11 test.cpp -o test.exe -fopenmp -g
diff --git a/meta/tools/core_allocation_test/launch.sh b/meta/tools/core_allocation_test/launch.sh
new file mode 100644
index 0000000000000000000000000000000000000000..21a3c6e2a2cd62194d5fd6a678d4890d5483a693
--- /dev/null
+++ b/meta/tools/core_allocation_test/launch.sh
@@ -0,0 +1,31 @@
+#! /bin/bash
+
+set -e
+
+# number of nodes to use
+NN=2
+# number of processes per node
+NP=16
+# number of cores per node --- depends on machine TODO: change as required
+CPN=48
+
+CURRENT_WORK_DIR="test_nn${NN}_np${NP}"
+
+rm -rf ${CURRENT_WORK_DIR}
+mkdir ${CURRENT_WORK_DIR}
+cd ${CURRENT_WORK_DIR}
+
+cp ../test.cpp ./
+bash ../compile.sh
+cp ../run_test_template.sh ./run_test.sh
+sed -i "s/NN/${NN}/" run_test.sh
+sed -i "s/NP/${NP}/" run_test.sh
+sed -i "s/ONT/$(expr ${CPN} / ${NP})/" run_test.sh
+
+# the following need to be changed according to your setup
+module load lrz/default
+module load lrztools
+module load slurm_setup
+
+sbatch run_test.sh
+
diff --git a/meta/tools/core_allocation_test/run_test_template.sh b/meta/tools/core_allocation_test/run_test_template.sh
new file mode 100644
index 0000000000000000000000000000000000000000..560e6759ba23c5be9fe938056b8ed5c3e5099f6b
--- /dev/null
+++ b/meta/tools/core_allocation_test/run_test_template.sh
@@ -0,0 +1,55 @@
+#!/bin/bash -l
+#SBATCH -J test_pinning
+#SBATCH -e err_test_pinning_nnNN_npNP
+#SBATCH -o out_test_pinning_nnNN_npNP
+#SBATCH -D ./
+#SBATCH --mail-type=END
+#SBATCH --mail-user=INSERT_EMAIL_HERE
+#SBATCH --partition=micro
+#SBATCH --nodes=NN
+#SBATCH --ntasks-per-node=NP
+#SBATCH --cpus-per-task=ONT
+#SBATCH --time=0:05:00
+#SBATCH --no-requeue
+#SBATCH --export=NONE
+#SBATCH --get-user-env
+#SBATCH --account=INSERT_ACCOUNT_TO_BE_CHARGED_HERE
+
+############################################################################
+###############################BEGIN########################################
+# feel free to rename this file to whatever, but it should load all required modules
+source ~/.config/TurTLE/bashrc
+
+export OMP_NUM_THREADS=ONT
+
+echo "Start time is `date`"
+
+echo "==========================================="
+echo "srun ./test.exe"
+srun ./test.exe
+
+echo "==========================================="
+echo "OMP_PLACES=cores mpiexec ./test.exe"
+OMP_PLACES=cores mpiexec ./test.exe
+
+echo "==========================================="
+echo "OMP_PLACES=cores mpiexec -n $SLURM_NTASKS ./test.exe"
+OMP_PLACES=cores mpiexec -n $SLURM_NTASKS ./test.exe
+
+echo "==========================================="
+echo "I_MPI_PIN_DOMAIN=omp:compact OMP_PLACES=cores mpiexec -n $SLURM_NTASKS ./test.exe"
+I_MPI_PIN_DOMAIN=omp:compact OMP_PLACES=cores mpiexec -n $SLURM_NTASKS ./test.exe
+
+echo "==========================================="
+echo "I_MPI_PIN_CELL=core I_MPI_PIN_DOMAIN=omp:compact OMP_PLACES=cores mpiexec -n $SLURM_NTASKS ./test.exe"
+I_MPI_PIN_CELL=core I_MPI_PIN_DOMAIN=omp:compact OMP_PLACES=cores mpiexec -n $SLURM_NTASKS ./test.exe
+
+echo "==========================================="
+echo "I_MPI_PIN_CELL=core I_MPI_PIN_DOMAIN=omp:compact KMP_AFFINITY=verbose,compact mpiexec -n $SLURM_NTASKS ./test.exe"
+I_MPI_PIN_CELL=core I_MPI_PIN_DOMAIN=omp:compact KMP_AFFINITY=verbose,compact mpiexec -n $SLURM_NTASKS ./test.exe
+
+echo "End time is `date`"
+exit 0
+###############################END###########################################
+#############################################################################
+
diff --git a/meta/tools/core_allocation_test/test.cpp b/meta/tools/core_allocation_test/test.cpp
new file mode 100644
index 0000000000000000000000000000000000000000..8d2ad04a81979c2892add19a97a677e43def8978
--- /dev/null
+++ b/meta/tools/core_allocation_test/test.cpp
@@ -0,0 +1,136 @@
+/**********************************************************************
+* *
+* Copyright 2015 Max Planck Institute *
+* for Dynamics and Self-Organization *
+* *
+* This file is part of TurTLE. *
+* *
+* TurTLE is free software: you can redistribute it and/or modify *
+* it under the terms of the GNU General Public License as published *
+* by the Free Software Foundation, either version 3 of the License, *
+* or (at your option) any later version. *
+* *
+* TurTLE is distributed in the hope that it will be useful, *
+* but WITHOUT ANY WARRANTY; without even the implied warranty of *
+* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the *
+* GNU General Public License for more details. *
+* *
+* You should have received a copy of the GNU General Public License *
+* along with TurTLE. If not, see <http://www.gnu.org/licenses/> *
+* *
+* Contact: berenger.bramas@inria.fr *
+* *
+**********************************************************************/
+
+/* Simple code to check MPI/OpenMP process/thread distribution on cluster */
+
+
+#ifndef _GNU_SOURCE
+#define _GNU_SOURCE
+#endif
+#include <sys/types.h>
+#include <unistd.h>
+#include <sys/syscall.h>
+#include <sched.h>
+
+#include <iostream>
+#include <vector>
+#include <cassert>
+#include <sstream>
+#include <omp.h>
+#include <mpi.h>
+#include <algorithm>
+
+std::vector<long int> GetBindingList(){
+ std::vector<long int> list;
+
+ cpu_set_t mask;
+ CPU_ZERO(&mask);
+ pid_t tid = static_cast<pid_t>(syscall(SYS_gettid));
+ // Get the affinity
+ int retValue = sched_getaffinity(tid, sizeof(mask), &mask);
+ assert(retValue == 0);
+
+ for(size_t idx = 0 ; idx < sizeof(long int)*8-1 ; ++idx){
+ if(CPU_ISSET(idx, &mask)){
+ list.push_back(idx);
+ }
+ }
+ return list;
+}
+
+int main(
+ int argc,
+ char *argv[]){
+ MPI_Init(&argc, &argv);
+
+ // Get the number of processes
+ int world_size;
+ MPI_Comm_size(MPI_COMM_WORLD, &world_size);
+
+ // Get the rank of the process
+ int world_rank;
+ MPI_Comm_rank(MPI_COMM_WORLD, &world_rank);
+
+ std::ostringstream stream;
+
+ stream << "[" << world_rank << "] is on node " << (getenv("HOSTNAME")?getenv("HOSTNAME"):"HOSTNAME is null") << "\n";
+ {
+
+ const std::vector<long int> cores = GetBindingList();
+
+ stream << "[" << world_rank << "] Available cores for master process = ";
+
+ for(long int core : cores){
+ stream << core << " ";
+ }
+
+ stream << "\n";
+ }
+
+ #pragma omp parallel
+ {
+ for(int idxThread = 0 ; idxThread < omp_get_num_threads() ; ++idxThread){
+ if(idxThread == omp_get_thread_num()){
+ const std::vector<long int> cores = GetBindingList();
+
+ stream << "[" << world_rank << "] Available cores for thread " << omp_get_thread_num() << " = ";
+
+ for(long int core : cores){
+ stream << core << " ";
+ }
+
+ stream << "\n";
+ }
+ #pragma omp barrier
+ }
+ }
+
+
+ if( world_rank == 0){
+ std::cout << "========== Process 0 =========" << std::endl;
+ std::cout << stream.str() << std::endl;
+
+ std::vector<char> buffer;
+ for(int idxProcess = 1 ; idxProcess < world_size ; ++idxProcess){
+ int length = 0;
+ MPI_Recv(&length, 1, MPI_INT, idxProcess, idxProcess*3, MPI_COMM_WORLD, MPI_STATUS_IGNORE);
+ buffer.resize(length);
+ MPI_Recv(buffer.data(), length, MPI_CHAR, idxProcess, idxProcess*3+2, MPI_COMM_WORLD, MPI_STATUS_IGNORE);
+ buffer.push_back('\0');
+ std::cout << "========== Process " << idxProcess << " =========" << std::endl;
+ std::cout << buffer.data() << std::endl;
+ }
+ }
+ else{
+ const std::string str = stream.str();
+ int length = str.size();
+ MPI_Send(&length, 1, MPI_INT, 0, world_rank*3, MPI_COMM_WORLD);
+ MPI_Send(str.c_str(), length, MPI_CHAR, 0, world_rank*3+2, MPI_COMM_WORLD);
+ }
+
+ MPI_Finalize();
+
+ return 0;
+}
+
diff --git a/meta/tools/type_check/compile.sh b/meta/tools/type_check/compile.sh
new file mode 100644
index 0000000000000000000000000000000000000000..58c5aff650297e8bbc7d1aefaaf77b1f9cfc9251
--- /dev/null
+++ b/meta/tools/type_check/compile.sh
@@ -0,0 +1 @@
+mpicxx -Wall -std=c++11 -I${HDF5_ROOT}/include main.cpp -o test.exe -fopenmp -g
diff --git a/meta/tools/type_check/main.cpp b/meta/tools/type_check/main.cpp
new file mode 100644
index 0000000000000000000000000000000000000000..39a4da651591458206c62369e1f41710ac955b6f
--- /dev/null
+++ b/meta/tools/type_check/main.cpp
@@ -0,0 +1,12 @@
+#include <iostream>
+#include <hdf5.h>
+
+int main()
+{
+ std::cout << "Checking `sizeof` for data types used in TurTLE" << std::endl;
+ std::cout << "sizeof(int)" << sizeof(int) << std::endl;
+ std::cout << "sizeof(ptrdiff_t)" << sizeof(ptrdiff_t) << std::endl;
+ std::cout << "sizeof(hsize_t)" << sizeof(hsize_t) << std::endl;
+ return EXIT_SUCCESS;
+}
+
diff --git a/pc_host_info.py b/pc_host_info.py
new file mode 100644
index 0000000000000000000000000000000000000000..44ff1c5da7bd456206b350fdd7a906b66322c715
--- /dev/null
+++ b/pc_host_info.py
@@ -0,0 +1,80 @@
+################################################################################
+# #
+# Copyright 2019 Max Planck Institute for Dynamics and Self-Organization #
+# #
+# This file is part of bfps. #
+# #
+# bfps is free software: you can redistribute it and/or modify #
+# it under the terms of the GNU General Public License as published #
+# by the Free Software Foundation, either version 3 of the License, #
+# or (at your option) any later version. #
+# #
+# bfps is distributed in the hope that it will be useful, #
+# but WITHOUT ANY WARRANTY; without even the implied warranty of #
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the #
+# GNU General Public License for more details. #
+# #
+# You should have received a copy of the GNU General Public License #
+# along with bfps. If not, see <http://www.gnu.org/licenses/> #
+# #
+# Contact: Cristian.Lalescu@ds.mpg.de #
+# #
+################################################################################
+
+
+host_info = {'type' : 'pc'}
+
+
+# template for host_info dictionary is given below:
+# {'type' : info_template_type,
+# 'MPI' : info_template_MPI,
+# 'environment' : info_template_environment,
+# 'deltanprocs' : info_template_deltanprocs,
+# 'mail_address': info_template_mail_address,
+# 'account' : info_template_account,
+# 'executable_launcher' : info_template_executable_launcher,
+# 'extra_slurm_lines' : info_template_extra_slurm_lines,
+# 'explicit_slurm_environment' : info_template_explicit_slurm_environment,
+# 'use_TurTLE_core_distribution' : info_use_TurTLE_core_distribution}
+
+# info_template_type can be one of:
+# 'pc' --- jobs run interactively
+# 'cluster' --- cluster with SGE queueing system
+# 'SLURM' --- cluster with SLURM queueing system
+# 'IBMLoadLeveler --- cluster with IBM Load Leveler queueing system
+
+# info_template_MPI can be one of:
+# 'openmpi' --- it means mpirun takes "x" as the parameter to set an environment variable
+# not defined --- use "env" instead of "x"
+
+# info_template_environment, relevant for clusters,
+# is the default queue to which jobs are submitted
+
+# info_template_deltanprocs, relevant for clusters,
+# is the number of cores per node
+
+# info_template_mail_address, relevant for clusters,
+# is the contact e-mail address placed in the job scripts.
+
+# info_template_account, relevant for some clusters,
+# is the name of the account to be budgeted for the job.
+
+# info_template_executable_launcher, relevant for 'SLURM' clusters,
+# is the binary that should be called to execute the hybrid MPI/OpenMP code.
+# by default it is 'srun', but it may be 'mpiexec' for some machines.
+
+# info_template_extra_slurm_lines, relevant for 'SLURM' clusters,
+# is a list of strings, written as extra options within the SLURM file.
+# for example something like `#SBATCH --get-user-env`
+# Here, also the virtual environment and environment variables need to be loaded,
+# e.g. by `source <TURTLE_DIR>/lib/bash_setup_for_TurTLE.sh`
+
+# info_template_explicit_slurm_environment, relevant for `SLURM` clusters,
+# is a bool specifying whether the environment must be mentioned explicitly
+# in the submission script, or whether the cluster chooses it based on the
+# resources requested
+
+# info_use_TurTLE_core_distribution, relevant for `SLURM` clusters,
+# is a bool specifying whether TurTLE should request explicit core
+# masks for the individual MPI processes.
+
diff --git a/setup.py b/setup.py
deleted file mode 100644
index 9bba17014aabf36c685395843b806f650604face..0000000000000000000000000000000000000000
--- a/setup.py
+++ /dev/null
@@ -1,285 +0,0 @@
-#######################################################################
-# #
-# Copyright 2015 Max Planck Institute #
-# for Dynamics and Self-Organization #
-# #
-# This file is part of bfps. #
-# #
-# bfps is free software: you can redistribute it and/or modify #
-# it under the terms of the GNU General Public License as published #
-# by the Free Software Foundation, either version 3 of the License, #
-# or (at your option) any later version. #
-# #
-# bfps is distributed in the hope that it will be useful, #
-# but WITHOUT ANY WARRANTY; without even the implied warranty of #
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the #
-# GNU General Public License for more details. #
-# #
-# You should have received a copy of the GNU General Public License #
-# along with bfps. If not, see <http://www.gnu.org/licenses/> #
-# #
-# Contact: Cristian.Lalescu@ds.mpg.de #
-# #
-#######################################################################
-
-
-
-AUTHOR = 'Cristian C Lalescu'
-AUTHOR_EMAIL = 'Cristian.Lalescu@ds.mpg.de'
-
-import os
-import shutil
-import datetime
-import sys
-import subprocess
-import pickle
-
-
-### compiler configuration
-# check if .config/bfps/machine_settings.py file exists, create it if not
-homefolder = os.path.expanduser('~')
-bfpsfolder = os.path.join(homefolder, '.config', 'bfps')
-if not os.path.exists(os.path.join(bfpsfolder, 'machine_settings.py')):
- if not os.path.isdir(bfpsfolder):
- os.mkdir(bfpsfolder)
- shutil.copyfile('./machine_settings_py.py', os.path.join(bfpsfolder, 'machine_settings.py'))
-# check if .config/bfps/host_information.py file exists, create it if not
-if not os.path.exists(os.path.join(bfpsfolder, 'host_information.py')):
- if not os.path.isdir(bfpsfolder):
- os.mkdir(bfpsfolder)
- open(os.path.join(bfpsfolder, 'host_information.py'),
- 'w').write('host_info = {\'type\' : \'none\'}\n')
- shutil.copyfile('./machine_settings_py.py', os.path.join(bfpsfolder, 'machine_settings.py'))
-sys.path.insert(0, bfpsfolder)
-# import stuff required for compilation of static library
-from machine_settings import compiler, include_dirs, library_dirs, extra_compile_args, extra_libraries
-
-
-### package versioning
-# get current time
-now = datetime.datetime.now()
-# obtain version
-try:
- git_branch = subprocess.check_output(['git',
- 'rev-parse',
- '--abbrev-ref',
- 'HEAD']).strip().split()[-1].decode()
- git_revision = subprocess.check_output(['git', 'rev-parse', 'HEAD']).strip()
- git_date = datetime.datetime.fromtimestamp(int(subprocess.check_output(['git', 'log', '-1', '--format=%ct']).strip()))
-except:
- git_revision = ''
- git_branch = ''
- git_date = now
-if git_branch == '':
- # there's no git available or something
- VERSION = '{0:0>4}{1:0>2}{2:0>2}.{3:0>2}{4:0>2}{5:0>2}'.format(
- git_date.year, git_date.month, git_date.day,
- git_date.hour, git_date.minute, git_date.second)
-else:
- if (('develop' in git_branch) or
- ('feature' in git_branch) or
- ('bugfix' in git_branch)):
- VERSION = subprocess.check_output(
- ['git', 'describe', '--tags', '--dirty']).strip().decode().replace('-g', '+g').replace('-dirty', '.dirty').replace('-', '.post')
- else:
- VERSION = subprocess.check_output(['git', 'describe', '--tags']).strip().decode().split('-')[0]
-print('This is bfps version ' + VERSION)
-
-
-
-### lists of files and MANIFEST.in
-src_file_list = ['full_code/joint_acc_vel_stats',
- 'full_code/test',
- 'full_code/filter_test',
- 'hdf5_tools',
- 'full_code/get_rfields',
- 'full_code/NSVE_field_stats',
- 'full_code/native_binary_to_hdf5',
- 'full_code/postprocess',
- 'full_code/code_base',
- 'full_code/direct_numerical_simulation',
- 'full_code/NSVE',
- 'field_binary_IO',
- 'vorticity_equation',
- 'field',
- 'kspace',
- 'field_layout',
- 'field_descriptor',
- 'rFFTW_distributed_particles',
- 'distributed_particles',
- 'particles',
- 'particles_base',
- 'rFFTW_interpolator',
- 'interpolator',
- 'interpolator_base',
- 'fluid_solver',
- 'fluid_solver_base',
- 'fftw_tools',
- 'spline_n1',
- 'spline_n2',
- 'spline_n3',
- 'spline_n4',
- 'spline_n5',
- 'spline_n6',
- 'spline_n7',
- 'spline_n8',
- 'spline_n9',
- 'spline_n10',
- 'Lagrange_polys',
- 'scope_timer',
- 'full_code/NSVEparticles']
-
-particle_headers = [
- 'cpp/particles/particles_distr_mpi.hpp',
- 'cpp/particles/abstract_particles_input.hpp',
- 'cpp/particles/abstract_particles_output.hpp',
- 'cpp/particles/abstract_particles_system.hpp',
- 'cpp/particles/alltoall_exchanger.hpp',
- 'cpp/particles/particles_adams_bashforth.hpp',
- 'cpp/particles/particles_field_computer.hpp',
- 'cpp/particles/particles_input_hdf5.hpp',
- 'cpp/particles/particles_generic_interp.hpp',
- 'cpp/particles/particles_output_hdf5.hpp',
- 'cpp/particles/particles_output_mpiio.hpp',
- 'cpp/particles/particles_system_builder.hpp',
- 'cpp/particles/particles_system.hpp',
- 'cpp/particles/particles_utils.hpp',
- 'cpp/particles/particles_output_sampling_hdf5.hpp',
- 'cpp/particles/particles_sampling.hpp',
- 'cpp/particles/env_utils.hpp']
-
-full_code_headers = ['cpp/full_code/main_code.hpp',
- 'cpp/full_code/codes_with_no_output.hpp',
- 'cpp/full_code/NSVE_no_output.hpp',
- 'cpp/full_code/NSVEparticles_no_output.hpp']
-
-header_list = (['cpp/base.hpp'] +
- ['cpp/fftw_interface.hpp'] +
- ['cpp/bfps_timer.hpp'] +
- ['cpp/omputils.hpp'] +
- ['cpp/shared_array.hpp'] +
- ['cpp/spline.hpp'] +
- ['cpp/' + fname + '.hpp'
- for fname in src_file_list] +
- particle_headers +
- full_code_headers)
-
-with open('MANIFEST.in', 'w') as manifest_in_file:
- for fname in (['bfps/cpp/' + ff + '.cpp' for ff in src_file_list] +
- ['bfps/' + ff for ff in header_list]):
- manifest_in_file.write('include {0}\n'.format(fname))
-
-
-
-### libraries
-libraries = extra_libraries
-
-
-import distutils.cmd
-
-class CompileLibCommand(distutils.cmd.Command):
- description = 'Compile bfps library.'
- user_options = [
- ('timing-output=', None, 'Toggle timing output.'),
- ('fftw-estimate=', None, 'Use FFTW ESTIMATE.'),
- ('disable-fftw-omp=', None, 'Turn Off FFTW OpenMP.'),
- ]
- def initialize_options(self):
- self.timing_output = 0
- self.fftw_estimate = 0
- self.disable_fftw_omp = 0
- return None
- def finalize_options(self):
- self.timing_output = (int(self.timing_output) == 1)
- self.fftw_estimate = (int(self.fftw_estimate) == 1)
- self.disable_fftw_omp = (int(self.disable_fftw_omp) == 1)
- return None
- def run(self):
- if not os.path.isdir('obj'):
- os.makedirs('obj')
- need_to_compile = True
- if not os.path.isdir('obj/full_code'):
- os.makedirs('obj/full_code')
- need_to_compile = True
- if not os.path.isfile('bfps/libbfps.a'):
- need_to_compile = True
- else:
- ofile = 'bfps/libbfps.a'
- libtime = datetime.datetime.fromtimestamp(os.path.getctime(ofile))
- latest = libtime
- for fname in header_list:
- latest = max(latest,
- datetime.datetime.fromtimestamp(os.path.getctime('bfps/' + fname)))
- need_to_compile = (latest > libtime)
- eca = extra_compile_args
- eca += ['-fPIC']
- if self.timing_output:
- eca += ['-DUSE_TIMINGOUTPUT']
- if self.fftw_estimate:
- eca += ['-DUSE_FFTWESTIMATE']
- if self.disable_fftw_omp:
- eca += ['-DNO_FFTWOMP']
- for fname in src_file_list:
- ifile = 'bfps/cpp/' + fname + '.cpp'
- ofile = 'obj/' + fname + '.o'
- if not os.path.exists(ofile):
- need_to_compile_file = True
- else:
- need_to_compile_file = (need_to_compile or
- (datetime.datetime.fromtimestamp(os.path.getctime(ofile)) <
- datetime.datetime.fromtimestamp(os.path.getctime(ifile))))
- if need_to_compile_file:
- command_strings = [compiler, '-c']
- command_strings += ['bfps/cpp/' + fname + '.cpp']
- command_strings += ['-o', 'obj/' + fname + '.o']
- command_strings += eca
- command_strings += ['-I' + idir for idir in include_dirs]
- command_strings.append('-Ibfps/cpp/')
- print(' '.join(command_strings))
- subprocess.check_call(command_strings)
- command_strings = ['ar', 'rvs', 'bfps/libbfps.a']
- command_strings += ['obj/' + fname + '.o' for fname in src_file_list]
- print(' '.join(command_strings))
- subprocess.check_call(command_strings)
-
- ### save compiling information
- pickle.dump(
- {'include_dirs' : include_dirs,
- 'library_dirs' : library_dirs,
- 'compiler' : compiler,
- 'extra_compile_args' : eca,
- 'libraries' : libraries,
- 'install_date' : now,
- 'VERSION' : VERSION,
- 'git_revision' : git_revision},
- open('bfps/install_info.pickle', 'wb'),
- protocol = 2)
- return None
-
-from setuptools import setup
-
-setup(
- name = 'bfps',
- packages = ['bfps', 'bfps/test'],
- install_requires = ['numpy>=1.8', 'h5py>=2.2.1'],
- cmdclass={'compile_library' : CompileLibCommand},
- package_data = {'bfps': header_list +
- ['libbfps.a',
- 'install_info.pickle'] +
- ['test/B32p1e4_checkpoint_0.h5']},
- entry_points = {
- 'console_scripts': [
- 'bfps = bfps.__main__:main',
- 'bfps1 = bfps.__main__:main',
- 'bfps.test_NSVEparticles = bfps.test.test_bfps_NSVEparticles:main'],
- },
- version = VERSION,
-########################################################################
-# useless stuff folows
-########################################################################
- description = 'Big Fluid and Particle Simulator',
- long_description = open('README.rst', 'r').read(),
- author = AUTHOR,
- author_email = AUTHOR_EMAIL,
- license = 'GPL version 3.0')
-
diff --git a/setup.py.in b/setup.py.in
new file mode 100644
index 0000000000000000000000000000000000000000..87261d3275b2fe97190e7268771ddd8660f3b3b5
--- /dev/null
+++ b/setup.py.in
@@ -0,0 +1,66 @@
+################################################################################
+# #
+# Copyright 2015-2019 Max Planck Institute for Dynamics and Self-Organization #
+# #
+# This file is part of TurTLE. #
+# #
+# TurTLE is free software: you can redistribute it and/or modify #
+# it under the terms of the GNU General Public License as published #
+# by the Free Software Foundation, either version 3 of the License, #
+# or (at your option) any later version. #
+# #
+# TurTLE is distributed in the hope that it will be useful, #
+# but WITHOUT ANY WARRANTY; without even the implied warranty of #
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the #
+# GNU General Public License for more details. #
+# #
+# You should have received a copy of the GNU General Public License #
+# along with TurTLE. If not, see <http://www.gnu.org/licenses/> #
+# #
+# Contact: Cristian.Lalescu@ds.mpg.de #
+# #
+################################################################################
+
+
+
+AUTHOR = 'Cristian C Lalescu'
+AUTHOR_EMAIL = 'Cristian.Lalescu@ds.mpg.de'
+
+import pickle
+
+### package versioning
+VERSION = '@TURTLE_VERSION_LONG@'
+print('This is TurTLE version ' + VERSION)
+
+from setuptools import setup
+
+setup(
+ name = 'TurTLE',
+ packages = ['TurTLE', 'TurTLE/test'],
+ install_requires = ['numpy>=1.8', 'h5py>=2.2.1'],
+ # package data is installed by CMake
+ #package_data = {'TurTLE': []},
+ entry_points = {
+ 'console_scripts': [
+ 'turtle = TurTLE.__main__:main',
+ 'turtle.test_parameter_copy = TurTLE.test.test_parameter_copy:main',
+ 'turtle.test_NSVEparticles = TurTLE.test.test_turtle_NSVEparticles:main',
+ 'turtle.test_particles = TurTLE.test.test_particles:main',
+ 'turtle.test_Parseval = TurTLE.test.test_Parseval:main',
+ 'turtle.test_fftw = TurTLE.test.test_fftw:main',
+ 'turtle.test_Heun_p2p = TurTLE.test.test_Heun_p2p:main',
+ 'turtle.test_particle_deleter = TurTLE.test.test_particle_deleter:main',
+ 'turtle.test_collisions = TurTLE.test.test_collisions:main'],
+ },
+ version = VERSION,
+########################################################################
+# useless stuff folows
+# if anyone knows how to open the README when calling this script from
+# cmake, please let me know.
+########################################################################
+ description = 'Turbulence Tools: Lagrangian and Eulerian',
+ #long_description = open('${PROJECT_SOURCE_DIR}/README.rst', 'r').read(),
+ author = AUTHOR,
+ author_email = AUTHOR_EMAIL,
+ license = 'GPL version 3.0')
+
diff --git a/tests/DNS/test_scaling.py b/tests/DNS/test_scaling.py
index 1d4b12a5e3eb4aa322d68ba276437d1a641f7eae..9ba25e38117921c073d527c26f3ae95c0bd05bbf 100644
--- a/tests/DNS/test_scaling.py
+++ b/tests/DNS/test_scaling.py
@@ -3,7 +3,7 @@ import numpy as np
import argparse
import os
-import bfps
+import TurTLE
def get_DNS_parameters(
DNS_type = 'A',
@@ -12,29 +12,39 @@ def get_DNS_parameters(
nprocesses = 1,
output_on = False,
cores_per_node = 16,
- nparticles = int(1e5)):
+ nparticles = int(1e5),
+ environment = 'express',
+ minutes = '29',
+ no_submit = True,
+ src_dirname = '/draco/ptmp/clalescu/scaling',
+ src_prefix = 'fbL',
+ src_iteration = None,
+ kMeta = 1.5,
+ fftw_plan_rigor = 'FFTW_MEASURE',
+ extra_parameters = []):
simname = (DNS_type + '{0:0>4d}'.format(N))
if output_on:
simname = DNS_type + simname
class_name = 'NSVE'
if DNS_type != 'A':
+ exponent = int(np.log10(nparticles))
simname += 'p{0}e{1}'.format(
- int(nparticles / 10**np.log10(nparticles)),
- int(np.log10(nparticles)))
+ int(nparticles / 10**exponent),
+ exponent)
class_name += 'particles'
work_dir = 'nn{0:0>4d}np{1}'.format(nnodes, nprocesses)
if not output_on:
class_name += '_no_output'
- src_simname = 'N{0:0>4d}_kMeta2'.format(N)
- src_iteration = -1
- if N == 512:
- src_iteration = 3072
- if N == 1024:
- src_iteration = 0x4000
- if N == 2048:
- src_iteration = 0x6000
- if N == 4096:
- src_iteration = 0
+ src_simname = src_prefix + '_N{0:0>4d}_kMeta{1:.1f}'.format(N, kMeta)
+ if type(src_iteration) == type(None):
+ assert (N in [1024, 2048, 4096])
+ if N == 1024:
+ src_iteration = 32*1024
+ if N == 2048:
+ src_iteration = 20*1024
+ if N == 4096:
+ src_simname = 'fb3_N2048x2_kMeta1.5'
+ src_iteration = 0
DNS_parameters = [
class_name,
'-n', '{0}'.format(N),
@@ -45,11 +55,29 @@ def get_DNS_parameters(
'--niter_todo', '12',
'--niter_out', '12',
'--niter_stat', '3']
- if src_iteration >= 0:
+ if N == 4096:
DNS_parameters += [
- '--src-wd', 'database',
+ '--precision', 'double']
+ # check that source sim exists
+ print('looking for ', os.path.join(src_dirname, src_simname + '.h5'))
+ assert(os.path.exists(os.path.join(src_dirname, src_simname + '.h5')))
+ # check that source checkpoint exists
+ dns_src = TurTLE.DNS(simname = src_simname, work_dir = src_dirname)
+ dns_src.read_parameters()
+ print('looking for ', dns_src.get_checkpoint_fname(iteration = src_iteration))
+ assert(os.path.exists(dns_src.get_checkpoint_fname(iteration = src_iteration)))
+ DNS_parameters += [
+ '--src-wd', src_dirname,
'--src-simname', src_simname,
'--src-iteration', '{0}'.format(src_iteration)]
+ # hardcode precision:
+ if N <= 1024:
+ DNS_parameters += ['--precision', 'single']
+ else:
+ DNS_parameters += ['--precision', 'double']
+ # copy parameters from source simulation
+ for parameter in ['fk0', 'fk1', 'forcing_type', 'dt', 'dealias_type', 'dkx', 'dky', 'dkz', 'energy', 'famplitude', 'fmode', 'injection_rate', 'nu']:
+ DNS_parameters += ['--' + parameter, '{0}'.format(dns_src.parameters[parameter])]
if DNS_type != 'A':
DNS_parameters += [
'--nparticles', '{0}'.format(nparticles)]
@@ -62,8 +90,14 @@ def get_DNS_parameters(
DNS_parameters += [
'--tracers0_neighbours', '{0}'.format(nneighbours),
'--tracers0_smoothness', '{0}'.format(smoothness),
- '--particle-rand-seed', '2']
- return simname, work_dir, DNS_parameters
+ '--niter_part', '6',
+ '--cpp_random_particles', '2']
+ if no_submit:
+ DNS_parameters += ['--no-submit']
+ DNS_parameters += ['--environment', environment,
+ '--minutes', '{0}'.format(minutes),
+ '--fftw_plan_rigor', fftw_plan_rigor]
+ return simname, work_dir, DNS_parameters + extra_parameters
def main():
#DNS_type = 'A',
@@ -86,28 +120,79 @@ def main():
parser.add_argument(
'--nnodes',
type = int,
+ help = 'how many nodes to use in total',
dest = 'nnodes',
default = 1)
parser.add_argument(
'--nprocesses',
type = int,
+ help = 'how many MPI processes per node to use',
dest = 'nprocesses',
default = 1)
parser.add_argument(
'--ncores',
type = int,
+ help = 'how many cores there are per node',
dest = 'ncores',
- default = 4)
+ default = 40)
parser.add_argument(
'--output-on',
action = 'store_true',
dest = 'output_on')
+ parser.add_argument(
+ '--submit',
+ action = 'store_true',
+ dest = 'submit')
parser.add_argument(
'--nparticles',
type = int,
dest = 'nparticles',
default = int(1e5))
+ parser.add_argument(
+ '--environment',
+ type = str,
+ dest = 'environment',
+ default = 'test')
+ parser.add_argument(
+ '--minutes',
+ type = int,
+ dest = 'minutes',
+ default = 29,
+ help = 'If environment supports it, this is the requested wall-clock-limit.')
+ parser.add_argument(
+ '--src-wd',
+ type = str,
+ dest = 'src_dirname',
+ default = '/draco/ptmp/clalescu/scaling')
+ parser.add_argument(
+ '--src-prefix',
+ type = str,
+ dest = 'src_prefix',
+ default = 'fbL')
+ parser.add_argument(
+ '--src-kMeta',
+ type = float,
+ dest = 'src_kMeta',
+ default = 1.5)
+ parser.add_argument(
+ '--src-iteration',
+ type = int,
+ dest = 'src_iteration',
+ default = None)
+ parser.add_argument(
+ '--profile-with-vtune',
+ action = 'store_true',
+ dest = 'use_vtune')
+ parser.add_argument(
+ '--profile-with-aps',
+ action = 'store_true',
+ dest = 'use_aps')
opt = parser.parse_args(sys.argv[1:])
+ extra_parameters = []
+ if opt.use_vtune:
+ extra_parameters.append('--profile-with-vtune')
+ if opt.use_aps:
+ extra_parameters.append('--profile-with-aps')
simname, work_dir, params = get_DNS_parameters(
DNS_type = opt.DNS_setup,
N = opt.n,
@@ -115,7 +200,15 @@ def main():
nprocesses = opt.nprocesses,
output_on = opt.output_on,
nparticles = opt.nparticles,
- cores_per_node = opt.ncores)
+ cores_per_node = opt.ncores,
+ no_submit = not opt.submit,
+ minutes = opt.minutes,
+ environment = opt.environment,
+ src_dirname = opt.src_dirname,
+ src_prefix = opt.src_prefix,
+ src_iteration = opt.src_iteration,
+ kMeta = opt.src_kMeta,
+ extra_parameters = extra_parameters)
print(work_dir + '/' + simname)
print(' '.join(params))
# these following 2 lines actually launch something
@@ -123,7 +216,7 @@ def main():
# parameter conflicts after the simname and work_dir have been decided
if not os.path.exists(work_dir):
os.makedirs(work_dir)
- c = bfps.DNS()
+ c = TurTLE.DNS()
c.launch(params)
return None
diff --git a/tests/PP/test_bandpass.py b/tests/PP/test_bandpass.py
new file mode 100644
index 0000000000000000000000000000000000000000..b824313476cdffd300203acd1b944c2d49759df2
--- /dev/null
+++ b/tests/PP/test_bandpass.py
@@ -0,0 +1,21 @@
+import subprocess
+import h5py
+import os
+
+if not os.path.exists('test.h5'):
+ subprocess.check_call([
+ 'turtle',
+ 'DNS',
+ 'NSVE'])
+
+subprocess.check_call([
+ 'turtle',
+ 'PP',
+ 'bandpass_stats',
+ '--no-submit'])
+
+f = h5py.File('test_post.h5', 'r')
+print(f['bandpass_stats/parameters/k0list'][...])
+print(f['bandpass_stats/parameters/k1list'][...])
+print(f['bandpass_stats/parameters/filter_type'][...])
+print(f['bandpass_stats/parameters/max_velocity_estimate'][...])
diff --git a/tests/base.py b/tests/base.py
index 1c06974e836d2a348bf1e4b260f2b018ec3ab7af..5c068b94a7034d00f9826e7eb9fa2eea770231a8 100644
--- a/tests/base.py
+++ b/tests/base.py
@@ -3,20 +3,20 @@
# Copyright 2015 Max Planck Institute #
# for Dynamics and Self-Organization #
# #
-# This file is part of bfps. #
+# This file is part of TurTLE. #
# #
-# bfps is free software: you can redistribute it and/or modify #
+# TurTLE is free software: you can redistribute it and/or modify #
# it under the terms of the GNU General Public License as published #
# by the Free Software Foundation, either version 3 of the License, #
# or (at your option) any later version. #
# #
-# bfps is distributed in the hope that it will be useful, #
+# TurTLE is distributed in the hope that it will be useful, #
# but WITHOUT ANY WARRANTY; without even the implied warranty of #
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the #
# GNU General Public License for more details. #
# #
# You should have received a copy of the GNU General Public License #
-# along with bfps. If not, see <http://www.gnu.org/licenses/> #
+# along with TurTLE. If not, see <http://www.gnu.org/licenses/> #
# #
# Contact: Cristian.Lalescu@ds.mpg.de #
# #
@@ -32,13 +32,12 @@ import pickle
import numpy as np
import matplotlib.pyplot as plt
-import bfps
-from bfps import FluidResize
-from bfps.tools import particle_finite_diff_test as acceleration_test
+import TurTLE
+from TurTLE.tools import particle_finite_diff_test as acceleration_test
import argparse
-def get_parser(base_class = bfps.NavierStokes,
+def get_parser(base_class = TurTLE.NavierStokes,
n = 32,
ncpu = 2,
precision = 'single',
@@ -50,6 +49,9 @@ def get_parser(base_class = bfps.NavierStokes,
parser.add_argument('-n',
type = int, dest = 'n',
default = n)
+ parser.add_argument('--np',
+ type = int, dest = 'np',
+ default = ncpu)
parser.add_argument('--ncpu',
type = int, dest = 'ncpu',
default = ncpu)
@@ -89,33 +91,13 @@ parser.add_argument(
dest = 'kMeta',
default = 2.0)
-def double(opt):
- old_simname = 'N{0:0>3x}'.format(opt.n)
- new_simname = 'N{0:0>3x}'.format(opt.n*2)
- c = FluidResize(fluid_precision = opt.precision)
- c.launch(
- args = ['--simname', old_simname + '_double',
- '--wd', opt.work_dir,
- '--nx', '{0}'.format(opt.n),
- '--ny', '{0}'.format(opt.n),
- '--nz', '{0}'.format(opt.n),
- '--dst_nx', '{0}'.format(2*opt.n),
- '--dst_ny', '{0}'.format(2*opt.n),
- '--dst_nz', '{0}'.format(2*opt.n),
- '--dst_simname', new_simname,
- '--src_simname', old_simname,
- '--src_iteration', '0',
- '--src_wd', './',
- '--niter_todo', '0'])
- return None
-
def launch(
opt,
nu = None,
dt = None,
tracer_state_file = None,
vorticity_field = None,
- code_class = bfps.NavierStokes,
+ code_class = TurTLE.DNS,
particle_class = 'particles',
interpolator_class = 'rFFTW_interpolator'):
c = code_class(
@@ -123,7 +105,7 @@ def launch(
fluid_precision = opt.precision,
frozen_fields = opt.frozen,
use_fftw_wisdom = False)
- if code_class == bfps.NavierStokes:
+ if code_class == TurTLE.NavierStokes:
c.QR_stats_on = True
c.pars_from_namespace(opt)
c.parameters['nx'] = opt.n
diff --git a/tests/ci-scripts/test.sh b/tests/ci-scripts/test.sh
index ddde2489e431412c260752f800640812ead91167..bb6eaa859fa40d8ffa975e693dc6351ebbbd63d5 100644
--- a/tests/ci-scripts/test.sh
+++ b/tests/ci-scripts/test.sh
@@ -5,41 +5,47 @@ set -x
# stops when fails
set -e
-# Init
-export destdir=$(pwd)"/ci-installdir"
-export pythonbin=/home/ubuntu/anaconda3/bin/python3
-export bfpspythonpath=$destdir/lib/python3.6/site-packages/
-export PYTHONPATH=:$bfpspythonpath$PYTHONPATH
-export PATH=$destdir/bin/:/home/ubuntu/hdf5/install/bin/:$PATH
-export LD_LIBRARY_PATH=/home/ubuntu/hdf5/install/lib/:/home/ubuntu/fftw/install/lib/
-
-echo "destdir = $destdir"
-echo "pythonbin = $pythonbin"
-echo "bfpspythonpath = $bfpspythonpath"
-
-# Remove possible previous installation
-if [[ -d $destdir ]] ; then
- rm -rf $destdir ;
-fi
-
-# Create install path
-if [[ ! -d $bfpspythonpath ]] ; then
- mkdir -p $bfpspythonpath ;
-fi
-
-# Build
-$pythonbin setup.py compile_library --timing-output 1
-# Install
-$pythonbin setup.py install --prefix=$destdir
-
-# Test
-ls $destdir
-ls $destdir/bin/
-
-$pythonbin $destdir/bin/bfps.test_NSVEparticles
-
-# Clean
-if [[ -d $destdir ]] ; then
- rm -rf $destdir ;
-fi
+echo "please check VM before turning tests back on"
+
+## Init
+#export destdir=$(pwd)"/ci-installdir"
+#export pythonbin=/home/ubuntu/anaconda3/bin/python3
+#export bfpspythonpath=$destdir/lib/python3.6/site-packages/
+#export PYTHONPATH=:$bfpspythonpath$PYTHONPATH
+#export PATH=$destdir/bin/:/home/ubuntu/hdf5/install/bin/:$PATH
+#export LD_LIBRARY_PATH=/home/ubuntu/hdf5/install/lib/:/home/ubuntu/fftw/install/lib/
+#
+#echo "destdir = $destdir"
+#echo "pythonbin = $pythonbin"
+#echo "bfpspythonpath = $bfpspythonpath"
+#
+## Remove possible previous installation
+#if [[ -d $destdir ]] ; then
+# rm -rf $destdir ;
+#fi
+#
+## Create install path
+#if [[ ! -d $bfpspythonpath ]] ; then
+# mkdir -p $bfpspythonpath ;
+#fi
+#
+## Build
+#$pythonbin setup.py compile_library --timing-output 1
+## Install
+#$pythonbin setup.py install --prefix=$destdir
+#
+## Test
+#ls $destdir
+#ls $destdir/bin/
+#
+#$pythonbin $destdir/bin/bfps.test_fftw
+#
+#$pythonbin $destdir/bin/bfps.test_Parseval
+#
+#$pythonbin $destdir/bin/bfps.test_NSVEparticles
+#
+## Clean
+#if [[ -d $destdir ]] ; then
+# rm -rf $destdir ;
+#fi
diff --git a/tests/misc/makefile b/tests/misc/makefile
new file mode 100644
index 0000000000000000000000000000000000000000..d44b9f04a10bdcb46fcf88ec9c858832c3c6e4df
--- /dev/null
+++ b/tests/misc/makefile
@@ -0,0 +1,15 @@
+test_fftw: test_fftw.c
+ mpicc \
+ -DFFTW_PLAN_RIGOR=FFTW_ESTIMATE \
+ -I/stuff/ext_installs/include \
+ -fopenmp \
+ test_fftw.c \
+ -o test_fftw \
+ -L/stuff/ext_installs/lib \
+ -lfftw3_mpi \
+ -lfftw3 \
+ -lfftw3f_mpi \
+ -lfftw3f \
+ -lfftw3_threads \
+ -lfftw3f_threads \
+ -lm
diff --git a/tests/misc/pow_overflow.cpp b/tests/misc/pow_overflow.cpp
new file mode 100644
index 0000000000000000000000000000000000000000..48cc8aaeff5b7cacb9f9175970eeeaf2112d299f
--- /dev/null
+++ b/tests/misc/pow_overflow.cpp
@@ -0,0 +1,30 @@
+#include <cfenv>
+#include <cmath>
+#include <iostream>
+#include <limits>
+
+int main()
+{
+ feenableexcept(FE_ALL_EXCEPT);
+ double p0 = 3.54;
+ double p4 = 122;
+ double p5 = 0.836;
+ double ell = 1.0;
+ double result = 0.;
+ double argument = 0.;
+
+ for (int k = 0; k<128; k++)
+ {
+ argument = p0*k*ell;
+ // double exponent = p4*pow(ell, p5);
+ // //if (exponent*log(argument) <2*std::numeric_limits<double>::min())
+ // // result = 0.;
+ // //else
+ // //{
+ // // double result0 = pow(p0*argument, p4*pow(ell, p5));
+ // // result = exp(-0.5*result0);
+ // //}
+ }
+ std::cout << argument << std::endl;
+ return 0;
+}
diff --git a/tests/misc/run.sh b/tests/misc/run.sh
new file mode 100644
index 0000000000000000000000000000000000000000..ada649cac9355848e903c19778785aaf28a935fd
--- /dev/null
+++ b/tests/misc/run.sh
@@ -0,0 +1,2 @@
+make
+mpirun -np 2 -x OMP_NUM_THREADS=1 test_fftw
diff --git a/tests/misc/test_fftw.c b/tests/misc/test_fftw.c
new file mode 100644
index 0000000000000000000000000000000000000000..af9fef7b6564bdc4b5b3db0908cda43ec3dd9945
--- /dev/null
+++ b/tests/misc/test_fftw.c
@@ -0,0 +1,341 @@
+#include <fftw3-mpi.h>
+#include <omp.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <assert.h>
+#include <math.h>
+
+#ifndef FFTW_PLAN_RIGOR
+
+#define FFTW_PLAN_RIGOR FFTW_ESTIMATE
+
+#endif
+
+//#define NO_FFTWOMP
+
+#define NX 36
+#define NY 36
+#define NZ 12
+
+const int nx = NX;
+const int ny = NY;
+const int nz = NZ;
+const int npoints = NX*NY*NZ;
+
+const double dkx = 1.0;
+const double dky = 1.0;
+const double dkz = 1.0;
+
+int myrank, nprocs;
+
+int main(
+ int argc,
+ char *argv[])
+{
+ ////////////////////////////////////
+ /* initialize MPI environment */
+#ifdef NO_FFTWOMP
+ MPI_Init(&argc, &argv);
+ MPI_Comm_rank(MPI_COMM_WORLD, &myrank);
+ MPI_Comm_size(MPI_COMM_WORLD, &nprocs);
+ fftw_mpi_init();
+ fftwf_mpi_init();
+ printf("There are %d processes\n", nprocs);
+#else
+ int mpiprovided;
+ MPI_Init_thread(&argc, &argv, MPI_THREAD_FUNNELED, &mpiprovided);
+ assert(mpiprovided >= MPI_THREAD_FUNNELED);
+ MPI_Comm_rank(MPI_COMM_WORLD, &myrank);
+ MPI_Comm_size(MPI_COMM_WORLD, &nprocs);
+ const int nThreads = omp_get_max_threads();
+ printf("Number of threads for the FFTW = %d\n",
+ nThreads);
+ if (nThreads > 1){
+ fftw_init_threads();
+ fftwf_init_threads();
+ }
+ fftw_mpi_init();
+ fftwf_mpi_init();
+ printf("There are %d processes and %d threads\n",
+ nprocs,
+ nThreads);
+ if (nThreads > 1){
+ fftw_plan_with_nthreads(nThreads);
+ fftwf_plan_with_nthreads(nThreads);
+ }
+#endif
+
+ ////////////////////////////////////
+ /* do useful work */
+
+ // declarations
+ ptrdiff_t nfftw[3];
+ ptrdiff_t tmp_local_size;
+ ptrdiff_t local_n0, local_0_start;
+ ptrdiff_t local_n1, local_1_start;
+ ptrdiff_t local_size;
+ ptrdiff_t ix, iy, iz;
+ ptrdiff_t jx, jy, jz;
+ ptrdiff_t rindex, cindex;
+ int cc;
+ float *data0, *data;
+ fftwf_complex *cdata;
+ double L2norm0, L2norm1, L2norm2, L2normk;
+ double local_L2norm0, local_L2norm1;
+ fftwf_plan c2r_plan, r2c_plan;
+ double *kx, *ky, *kz;
+
+ // get sizes
+ nfftw[0] = nz;
+ nfftw[1] = ny;
+ nfftw[2] = nx;
+ tmp_local_size = fftwf_mpi_local_size_many_transposed(
+ 3, nfftw, 3,
+ FFTW_MPI_DEFAULT_BLOCK, FFTW_MPI_DEFAULT_BLOCK, MPI_COMM_WORLD,
+ &local_n0, &local_0_start,
+ &local_n1, &local_1_start);
+
+ local_size = local_n1 * nz * nx * 3 * 2;
+
+ // allocate field
+ data = fftwf_alloc_real(
+ local_size);
+ data0 = fftwf_alloc_real(
+ local_size);
+ cdata = (fftwf_complex*)(data);
+
+ c2r_plan = fftwf_mpi_plan_many_dft_c2r(
+ 3, nfftw, 3,
+ FFTW_MPI_DEFAULT_BLOCK, FFTW_MPI_DEFAULT_BLOCK,
+ cdata,
+ data,
+ MPI_COMM_WORLD,
+ FFTW_PLAN_RIGOR | FFTW_MPI_TRANSPOSED_IN);
+
+ r2c_plan = fftwf_mpi_plan_many_dft_r2c(
+ 3, nfftw, 3,
+ FFTW_MPI_DEFAULT_BLOCK, FFTW_MPI_DEFAULT_BLOCK,
+ data,
+ cdata,
+ MPI_COMM_WORLD,
+ FFTW_PLAN_RIGOR | FFTW_MPI_TRANSPOSED_OUT);
+
+ kx = (double*)malloc(sizeof(double)*(nx/2+1));
+ ky = (double*)malloc(sizeof(double)*local_n1);
+ kz = (double*)malloc(sizeof(double)*nz);
+
+ // generate wavenumbers
+ for (jy = 0; jy < local_n1; jy++)
+ {
+ if (jy + local_1_start <= ny/2)
+ ky[jy] = dky*(jy + local_1_start);
+ else
+ ky[jy] = dky*((jy + local_1_start) - ny); }
+ for (jz = 0; jz < nz; jz++)
+ {
+ if (jz <= nz/2)
+ kz[jz] = dkz*jz;
+ else
+ kz[jz] = dkz*(jz - nz);
+ }
+ for (jx = 0; jx < nx/2+1; jx++)
+ {
+ kx[jx] = dkx*jx;
+ }
+
+ // fill field with random numbers
+ // I'm generating cindex the stupid way, but we can also use
+ // cindex = (jy*nz + jz)*(nx/2+1) + jx
+ cindex = 0;
+ for (jy = 0; jy < local_n1; jy++)
+ for (jz = 0; jz < nz; jz++)
+ {
+ for (jx = 0; jx < nx/2+1; jx++)
+ {
+ double k2 = (kx[jx]*kx[jx] +
+ ky[jy]*ky[jy] +
+ kz[jz]*kz[jz]);
+ if (jx == 0 && (jy + local_1_start) == 0 && jz == 0)
+ k2 = dkx*dkx + dky*dky + dkz*dkz;
+ for (cc = 0; cc<3; cc++)
+ {
+ cdata[cindex*3+cc][0] = (drand48()-0.5) / sqrt(k2);
+ cdata[cindex*3+cc][1] = (drand48()-0.5) / sqrt(k2);
+ }
+ cindex++;
+ }
+ }
+
+ // go back and forth so that the
+ // Fourier space representation is properly symmetrized
+ fftwf_execute(c2r_plan);
+ fftwf_execute(r2c_plan);
+ // normalize, compute Fourier space L2 norm
+ cindex = 0;
+ local_L2norm0 = 0;
+ for (jy = 0; jy < local_n1; jy++)
+ for (jz = 0; jz < nz; jz++)
+ {
+ for (cc = 0; cc<3; cc++)
+ {
+ cdata[cindex*3+cc][0] /= npoints;
+ cdata[cindex*3+cc][1] /= npoints;
+ local_L2norm0 += (cdata[cindex*3+cc][0]*cdata[cindex*3+cc][0] +
+ cdata[cindex*3+cc][1]*cdata[cindex*3+cc][1]);
+ }
+ cindex++;
+ for (jx = 1; jx < nx/2+1; jx++)
+ {
+ for (cc = 0; cc<3; cc++)
+ {
+ cdata[cindex*3+cc][0] /= npoints;
+ cdata[cindex*3+cc][1] /= npoints;
+ local_L2norm0 += 2*(cdata[cindex*3+cc][0]*cdata[cindex*3+cc][0] +
+ cdata[cindex*3+cc][1]*cdata[cindex*3+cc][1]);
+ }
+ cindex++;
+ }
+ }
+ MPI_Allreduce(
+ &local_L2norm0,
+ &L2normk,
+ 1,
+ MPI_DOUBLE,
+ MPI_SUM,
+ MPI_COMM_WORLD);
+ L2normk = sqrt(L2normk);
+
+ // go to real space
+ fftwf_execute(c2r_plan);
+
+ // rindex = (iz*ny + iy)*(nx+2) + ix
+ rindex = 0;
+ local_L2norm0 = 0;
+ for (iz = 0; iz < local_n0; iz++)
+ for (iy = 0; iy < ny; iy++)
+ {
+ for (ix = 0; ix < nx; ix++)
+ {
+ for (cc = 0; cc<3; cc++)
+ {
+ local_L2norm0 += data[rindex*3+cc]*data[rindex*3+cc];
+ }
+ rindex++;
+ }
+ for (ix = nx; ix < nx+2; ix++)
+ {
+ rindex++;
+ }
+ }
+ MPI_Allreduce(
+ &local_L2norm0,
+ &L2norm1,
+ 1,
+ MPI_DOUBLE,
+ MPI_SUM,
+ MPI_COMM_WORLD);
+ L2norm1 = sqrt(L2norm1 / npoints);
+
+ //fftwf_execute(r2c_plan);
+
+ //cindex = 0;
+ //local_L2norm0 = 0;
+ //for (jy = 0; jy < local_n1; jy++)
+ // for (jz = 0; jz < nz; jz++)
+ // {
+ // for (cc = 0; cc<3; cc++)
+ // {
+ // local_L2norm0 += (cdata[cindex*3+cc][0]*cdata[cindex*3+cc][0] +
+ // cdata[cindex*3+cc][1]*cdata[cindex*3+cc][1]);
+ // }
+ // cindex++;
+ // // I am not adding the energy from mode nx/2 as a matter of principle.
+ // for (jx = 1; jx < nx/2+1; jx++)
+ // {
+ // for (cc = 0; cc<3; cc++)
+ // {
+ // local_L2norm0 += 2*(cdata[cindex*3+cc][0]*cdata[cindex*3+cc][0] +
+ // cdata[cindex*3+cc][1]*cdata[cindex*3+cc][1]);
+ // }
+ // cindex++;
+ // }
+ // }
+ //MPI_Allreduce(
+ // &local_L2norm0,
+ // &L2normk,
+ // 1,
+ // MPI_DOUBLE,
+ // MPI_SUM,
+ // MPI_COMM_WORLD);
+ //L2normk = sqrt(L2normk) / (nx*ny*nz);
+ //fftwf_execute(c2r_plan);
+
+ //// normalize
+ //rindex = 0;
+ //local_L2norm0 = 0;
+ //local_L2norm1 = 0;
+ //for (iz = 0; iz < local_n0; iz++)
+ // for (iy = 0; iy < ny; iy++)
+ // {
+ // for (ix = 0; ix < nx; ix++)
+ // {
+ // for (cc = 0; cc<3; cc++)
+ // {
+ // data[rindex*3+cc] /= (nx*ny*nz);
+ // local_L2norm0 += data[rindex*3+cc]*data[rindex*3+cc];
+ // local_L2norm1 += ((data0[rindex*3+cc] - data[rindex*3+cc])*
+ // (data0[rindex*3+cc] - data[rindex*3+cc]));
+ // }
+ // rindex++;
+ // }
+ // for (ix = nx; ix < nx+2; ix++)
+ // {
+ // rindex++;
+ // }
+ // }
+ //MPI_Allreduce(
+ // &local_L2norm0,
+ // &L2norm1,
+ // 1,
+ // MPI_DOUBLE,
+ // MPI_SUM,
+ // MPI_COMM_WORLD);
+ //MPI_Allreduce(
+ // &local_L2norm1,
+ // &L2norm2,
+ // 1,
+ // MPI_DOUBLE,
+ // MPI_SUM,
+ // MPI_COMM_WORLD);
+ //L2norm1 = sqrt(L2norm1 / (nx*ny*nz));
+ //L2norm2 = sqrt(L2norm2 / (nx*ny*nz));
+
+ printf("FFTW_PLAN_RIGOR=%d\n", FFTW_PLAN_RIGOR);
+ printf("L2normk = %g, L2norm1 = %g, relative error = %g\n",
+ L2normk, L2norm1, fabs(L2normk - L2norm1) / (L2normk));
+
+ // deallocate
+ fftwf_destroy_plan(r2c_plan);
+ fftwf_destroy_plan(c2r_plan);
+ fftwf_free(data);
+ fftwf_free(data0);
+ free(kx);
+ free(ky);
+ free(kz);
+
+ ////////////////////////////////////
+ /* clean up */
+ fftwf_mpi_cleanup();
+ fftw_mpi_cleanup();
+
+#ifndef NO_FFTWOMP
+ if (nThreads > 1){
+ fftw_cleanup_threads();
+ fftwf_cleanup_threads();
+ }
+#endif
+
+ MPI_Finalize();
+ return EXIT_SUCCESS;
+}
+
diff --git a/tests/run_all_tests.sh b/tests/run_all_tests.sh
new file mode 100644
index 0000000000000000000000000000000000000000..227df5ee533d37d7039b70cfa41ed892f1bd3e89
--- /dev/null
+++ b/tests/run_all_tests.sh
@@ -0,0 +1,15 @@
+#!/bin/bash
+
+set -e
+
+turtle.test_fftw
+turtle.test_Parseval
+turtle.test_NSVEparticles
+
+turtle DNS static_field_with_ghost_collisions --simname dumb_test_of_ghost_collisions
+
+# test postprocessing
+turtle PP field_single_to_double --simname dns_nsveparticles --iter0 32 --iter1 32
+turtle PP get_rfields --simname dns_nsveparticles --iter0 0 --iter1 64 --TrS2_on 1
+turtle PP joint_acc_vel_stats --simname dns_nsveparticles --iter0 0 --iter1 64
+turtle PP resize --simname dns_nsveparticles --new_nx 96 --new_ny 96 --new_nz 96 --new_simname dns_nsveparticles_resized
diff --git a/tests/test_field_class.py b/tests/test_field_class.py
index 110d9be685ef42d4ed231a3a3c723ac34e3d916d..ef0cc8513c3bb289c018afb9f4a4e726a5e8a728 100644
--- a/tests/test_field_class.py
+++ b/tests/test_field_class.py
@@ -142,9 +142,9 @@ def main():
'--ncpu', '2'])
f = h5py.File('field.h5', 'r')
- #err0 = np.max(np.abs(f['scal_tmp/real/0'].value - rdata)) / np.mean(np.abs(rdata))
- #err1 = np.max(np.abs(f['scal/real/0'].value/(n**3) - rdata)) / np.mean(np.abs(rdata))
- #err2 = np.max(np.abs(f['scal_tmp/complex/0'].value/(n**3) - cdata)) / np.mean(np.abs(cdata))
+ #err0 = np.max(np.abs(f['scal_tmp/real/0'][...] - rdata)) / np.mean(np.abs(rdata))
+ #err1 = np.max(np.abs(f['scal/real/0'][...]/(n**3) - rdata)) / np.mean(np.abs(rdata))
+ #err2 = np.max(np.abs(f['scal_tmp/complex/0'][...]/(n**3) - cdata)) / np.mean(np.abs(cdata))
#print(err0, err1, err2)
#assert(err0 < 1e-5)
#assert(err1 < 1e-5)
diff --git a/tests/test_filters.py b/tests/test_filters.py
index b6e2aedbed414b2006a33e4f0deb4742ffb8ec3f..b3c24bf77a54e1bf43c25336003c9948d2490888 100644
--- a/tests/test_filters.py
+++ b/tests/test_filters.py
@@ -80,9 +80,9 @@ def filter_comparison(
dd = None,
base_name = 'filter_test_',
dim = 0):
- b = dd.df['ball/real/{0}'.format(dim)].value
- g = dd.df['Gauss/real/{0}'.format(dim)].value
- s = dd.df['sharp_Fourier_sphere/real/{0}'.format(dim)].value
+ b = dd.df['ball/real/{0}'.format(dim)][...]
+ g = dd.df['Gauss/real/{0}'.format(dim)][...]
+ s = dd.df['sharp_Fourier_sphere/real/{0}'.format(dim)][...]
d3V = dd.grid_spacing['x']*dd.grid_spacing['y']*dd.grid_spacing['z']
print(np.sum(b)*d3V)
print(np.sum(g)*d3V)
@@ -164,7 +164,7 @@ def resolution_comparison(
f = plt.figure(figsize = (6, 5))
a = f.add_subplot(111)
for dd in dlist:
- s0 = dd.df[filter_type + '/real/{0}'.format(dim)].value
+ s0 = dd.df[filter_type + '/real/{0}'.format(dim)][...]
a.plot(dd.get_coordinate('z'),
s0[:, 0, 0],
label = '{0}'.format(dd.simname))
@@ -182,7 +182,7 @@ class sim_data:
pfile = h5py.File(simname + '.h5', 'r')
self.parameters = {}
for kk in pfile['parameters'].keys():
- self.parameters[kk] = pfile['parameters/' + kk].value
+ self.parameters[kk] = pfile['parameters/' + kk][...]
self.grid_spacing = {}
for kk in ['x', 'y', 'z']:
self.grid_spacing[kk] = 2*np.pi / (self.parameters['dk' + kk] * self.parameters['n' + kk])
diff --git a/tests/test_io_03_run.py b/tests/test_io_03_run.py
index a789ac66fd99d8e5525ce69b1e861f609d969212..5b4905ba8973299b44a3dd7ef5f9fa07294e7a8b 100644
--- a/tests/test_io_03_run.py
+++ b/tests/test_io_03_run.py
@@ -35,5 +35,5 @@ if __name__ == '__main__':
c.write_src()
c.write_par()
c.set_host_info(bfps.host_info)
- c.run()
+ c.run(opt.ncpu, 1)
diff --git a/tests/test_particles.py b/tests/test_particles.py
index 7c7c86610cb3ec1813e6ecad80943d1bb688fa8d..6be076421edd7f192080a247cccb2f432ba3c7ef 100644
--- a/tests/test_particles.py
+++ b/tests/test_particles.py
@@ -118,7 +118,7 @@ class err_finder:
for c in self.clist]
self.ctraj = [None]
for i in range(1, self.clist[0].particle_species):
- self.ctraj.append([self.clist[j].get_particle_file()['tracers{0}/state'.format(i)].value.transpose((0, 2, 1))
+ self.ctraj.append([self.clist[j].get_particle_file()['tracers{0}/state'.format(i)][...].transpose((0, 2, 1))
for j in range(len(self.clist))])
return None
def get_AB_err(self, nsubsteps = 1):
@@ -178,7 +178,7 @@ if __name__ == '__main__':
a = fig.add_subplot(111)
for s in range(1, 5):
ef.get_AB_err(s)
- errlist = [np.average(np.abs(ef.clist[i].get_particle_file()['tracers{0}/state'.format(s)].value[-1, :, :3] - ef.xAB[i][-1].T))
+ errlist = [np.average(np.abs(ef.clist[i].get_particle_file()['tracers{0}/state'.format(s)][...][-1, :, :3] - ef.xAB[i][-1].T))
for i in range(len(ef.clist))]
a.plot(ef.dtlist, errlist,
label = 'directAB{0}'.format(s),
diff --git a/tests/test_plain.py b/tests/test_plain.py
deleted file mode 100644
index ad30224f869fc724758cc95d8b9e10da7b4ca2d4..0000000000000000000000000000000000000000
--- a/tests/test_plain.py
+++ /dev/null
@@ -1,156 +0,0 @@
-#! /usr/bin/env python3
-#######################################################################
-# #
-# Copyright 2015 Max Planck Institute #
-# for Dynamics and Self-Organization #
-# #
-# This file is part of bfps. #
-# #
-# bfps is free software: you can redistribute it and/or modify #
-# it under the terms of the GNU General Public License as published #
-# by the Free Software Foundation, either version 3 of the License, #
-# or (at your option) any later version. #
-# #
-# bfps is distributed in the hope that it will be useful, #
-# but WITHOUT ANY WARRANTY; without even the implied warranty of #
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the #
-# GNU General Public License for more details. #
-# #
-# You should have received a copy of the GNU General Public License #
-# along with bfps. If not, see <http://www.gnu.org/licenses/> #
-# #
-# Contact: Cristian.Lalescu@ds.mpg.de #
-# #
-#######################################################################
-
-
-
-#from base import *
-import bfps
-from bfps.tools import particle_finite_diff_test as acceleration_test
-import sys
-
-import numpy as np
-import matplotlib.pyplot as plt
-
-#parser.add_argument('--multiplejob',
-# dest = 'multiplejob', action = 'store_true')
-#
-#parser.add_argument(
-# '--particle-class',
-# default = 'particles',
-# dest = 'particle_class',
-# type = str)
-#
-#parser.add_argument(
-# '--interpolator-class',
-# default = 'interpolator',
-# dest = 'interpolator_class',
-# type = str)
-
-class NSPlain(bfps.NavierStokes):
- def specific_parser_arguments(
- self,
- parser):
- bfps.NavierStokes.specific_parser_arguments(self, parser)
- parser.add_argument(
- '--particle-class',
- default = 'rFFTW_distributed_particles',
- dest = 'particle_class',
- type = str)
- parser.add_argument(
- '--interpolator-class',
- default = 'rFFTW_interpolator',
- dest = 'interpolator_class',
- type = str)
- parser.add_argument('--neighbours',
- type = int,
- dest = 'neighbours',
- default = 3)
- parser.add_argument('--smoothness',
- type = int,
- dest = 'smoothness',
- default = 2)
- return None
- def launch(
- self,
- args = [],
- **kwargs):
- opt = self.prepare_launch(args = args)
- self.fill_up_fluid_code()
- if type(opt.nparticles) == int:
- if opt.nparticles > 0:
- self.add_3D_rFFTW_field(
- name = 'rFFTW_acc')
- self.add_interpolator(
- name = 'spline',
- neighbours = opt.neighbours,
- smoothness = opt.smoothness,
- class_name = opt.interpolator_class)
- self.add_particles(
- kcut = ['fs->kM/2', 'fs->kM/3'],
- integration_steps = 3,
- interpolator = 'spline',
- class_name = opt.particle_class)
- self.add_particles(
- integration_steps = [2, 3, 4, 6],
- interpolator = 'spline',
- acc_name = 'rFFTW_acc',
- class_name = opt.particle_class)
- self.finalize_code()
- self.launch_jobs(opt = opt)
- return None
-
-def plain(args):
- wd = opt.work_dir
- opt.work_dir = wd + '/N{0:0>3x}_1'.format(opt.n)
- c0 = launch(opt, dt = 0.2/opt.n,
- particle_class = opt.particle_class,
- interpolator_class = opt.interpolator_class)
- c0.compute_statistics()
- print ('Re = {0:.0f}'.format(c0.statistics['Re']))
- print ('Rlambda = {0:.0f}'.format(c0.statistics['Rlambda']))
- print ('Lint = {0:.4e}, etaK = {1:.4e}'.format(c0.statistics['Lint'], c0.statistics['etaK']))
- print ('Tint = {0:.4e}, tauK = {1:.4e}'.format(c0.statistics['Tint'], c0.statistics['tauK']))
- print ('kMetaK = {0:.4e}'.format(c0.statistics['kMeta']))
- for s in range(c0.particle_species):
- acceleration_test(c0, species = s, m = 1)
- if not opt.multiplejob:
- return None
- assert(opt.niter_todo % 3 == 0)
- opt.work_dir = wd + '/N{0:0>3x}_2'.format(opt.n)
- opt.njobs *= 2
- opt.niter_todo = opt.niter_todo//2
- c1 = launch(opt, dt = c0.parameters['dt'],
- particle_class = opt.particle_class,
- interpolator_class = opt.interpolator_class)
- c1.compute_statistics()
- opt.work_dir = wd + '/N{0:0>3x}_3'.format(opt.n)
- opt.njobs = 3*opt.njobs//2
- opt.niter_todo = 2*opt.niter_todo//3
- c2 = launch(opt, dt = c0.parameters['dt'],
- particle_class = opt.particle_class,
- interpolator_class = opt.interpolator_class)
- c2.compute_statistics()
- compare_stats(opt, c0, c1)
- compare_stats(opt, c0, c2)
- return None
-
-if __name__ == '__main__':
- c0 = NSPlain()
- c0.launch(
- ['-n', '32',
- '--ncpu', '4',
- '--nparticles', '1000',
- '--niter_todo', '48',
- '--wd', 'data/single'] +
- sys.argv[1:])
- c0.compute_statistics()
- print ('Re = {0:.0f}'.format(c0.statistics['Re']))
- print ('Rlambda = {0:.0f}'.format(c0.statistics['Rlambda']))
- print ('Lint = {0:.4e}, etaK = {1:.4e}'.format(c0.statistics['Lint'], c0.statistics['etaK']))
- print ('Tint = {0:.4e}, tauK = {1:.4e}'.format(c0.statistics['Tint'], c0.statistics['tauK']))
- print ('kMetaK = {0:.4e}'.format(c0.statistics['kMeta']))
- for s in range(c0.particle_species):
- acceleration_test(c0, species = s, m = 1)
-
diff --git a/tests/test_vorticity_equation.py b/tests/test_vorticity_equation.py
index dfaccb8bf352bdd252e5edf29f6e7d711689f7dc..e492bfa5c75d0f2f2b9989cccef49964b8bc90b4 100644
--- a/tests/test_vorticity_equation.py
+++ b/tests/test_vorticity_equation.py
@@ -273,12 +273,13 @@ def main():
particle_initial_condition[..., 2] = yvals[None, :, None]
particle_initial_condition = particle_initial_condition.reshape(-1, 3)
nparticles = nparticles**2
- c = bfps.NavierStokes(simname = 'fluid_solver')
+ c = bfps.DNS(simname = 'fluid_solver')
if run_NS:
run_NSVE = True
subprocess.call('rm *fluid_solver* NavierStokes*', shell = True)
c.launch(
- ['-n', '32',
+ ['NSVE',
+ '-n', '32',
'--simname', 'fluid_solver',
'--ncpu', '4',
'--niter_todo', '{0}'.format(niterations),
@@ -298,9 +299,10 @@ def main():
f = h5py.File('vorticity_equation_checkpoint_0.h5', 'w')
f['vorticity/complex/0'] = data
f.close()
- c = bfps.NSVorticityEquation()
+ c = bfps.DNS()
c.launch(
- ['-n', '32',
+ ['NSVEparticles',
+ '-n', '32',
'--simname', 'vorticity_equation',
'--np', '4',
'--ntpp', '1',
diff --git a/tox.ini b/tox.ini
deleted file mode 100644
index 8aca14f12a8a95079220ae5a70ebdcac32fd8737..0000000000000000000000000000000000000000
--- a/tox.ini
+++ /dev/null
@@ -1,13 +0,0 @@
-[tox]
-envlist = py34
-[testenv]
-sitepackages = True
-whitelist_externals =
- echo
- cp
-passenv = LD_LIBRARY_PATH
-changedir =
- {envtmpdir}
-commands =
- cp -r {toxinidir}/tests {envtmpdir}
- python tests/{posargs:DEFAULTS}
diff --git a/tox_exec.ini b/tox_exec.ini
deleted file mode 100644
index c9efeb8f210d920837bc0e9145b884e39d234ea0..0000000000000000000000000000000000000000
--- a/tox_exec.ini
+++ /dev/null
@@ -1,12 +0,0 @@
-[tox]
-envlist = py34
-[testenv]
-sitepackages = True
-whitelist_externals =
- echo
- cp
-passenv = LD_LIBRARY_PATH
-changedir =
- {envtmpdir}
-commands =
- python -m bfps {posargs:DEFAULTS}
diff --git a/tox_full.ini b/tox_full.ini
deleted file mode 100644
index d102b91ef0cb7b305d93d08d3dd899f80178a2f4..0000000000000000000000000000000000000000
--- a/tox_full.ini
+++ /dev/null
@@ -1,25 +0,0 @@
-[tox]
-envlist = py34
-[testenv]
-sitepackages = True
-whitelist_externals =
- echo
- cp
-passenv = LD_LIBRARY_PATH
-changedir =
- {envtmpdir}
-commands =
- cp -r {toxinidir}/tests {envtmpdir}
- python tests/test_io.py
- python tests/test_interpolation.py
- python tests/test_interpolation.py --precision double --wd "interp_double"
- python tests/test_plain.py
- python tests/test_plain.py --precision double --wd "data/double"
- python tests/test_scaling.py
- python tests/test_scaling.py --precision double --wd "data/double"
- python tests/test_particles.py
- python tests/test_particles.py --precision double --wd "data/double"
- python tests/test_time_step.py
- python tests/test_time_step.py --precision double --wd "data/double"
- python tests/test_convergence.py
- python tests/test_convergence.py --precision double --wd "data/double"