Commit bdfd4366 authored by Cristian Lalescu's avatar Cristian Lalescu
Browse files

Merge branch 'develop' into bugfix/particle_distribution

parents b44c214d a11e94f0
stages:
- build
simple_test:
image: gitlab-registry.mpcdf.mpg.de/mpcdf/module-image
before_script:
build-gcc-impi:
stage: build
script:
- source tests/ci-scripts/test.sh
tags:
- fftw3
- hdf5
- mpi
- python3
- module load cmake gcc impi hdf5-mpi fftw-mpi
- mkdir build; cd build
- MPICXX=mpigxx FFTW_DIR=$FFTW_HOME cmake ..
- make VERBOSE=1
only:
- develop
tags:
- docker
build-intel-impi:
stage: build
script:
- module load cmake intel impi hdf5-mpi fftw-mpi
- mkdir build; cd build
- MPICXX=mpiicpc FFTW_DIR=$FFTW_HOME cmake ..
- make VERBOSE=1
only:
- develop
tags:
- docker
......@@ -24,7 +24,7 @@
cmake_minimum_required(VERSION 3.10)
cmake_minimum_required(VERSION 3.6)
cmake_policy(VERSION 3.12)
if (DEFINED ENV{MPICXX})
......@@ -45,17 +45,101 @@ if (DEFINED ENV{CMAKE_INSTALL_PREFIX})
set(CMAKE_INSTALL_PREFIX $ENV{CMAKE_INSTALL_PREFIX})
endif()
project(BFPS)
project(TurTLE)
execute_process(COMMAND python ${PROJECT_SOURCE_DIR}/get_version.py OUTPUT_VARIABLE BFPS_VERSION OUTPUT_STRIP_TRAILING_WHITESPACE)
execute_process(COMMAND python3 ${PROJECT_SOURCE_DIR}/get_version.py OUTPUT_VARIABLE TURTLE_VERSION OUTPUT_STRIP_TRAILING_WHITESPACE)
project(BFPS
VERSION ${BFPS_VERSION}
project(TurTLE
VERSION ${TURTLE_VERSION}
LANGUAGES CXX)
set(CMAKE_MODULE_PATH ${PROJECT_SOURCE_DIR}/cmake ${CMAKE_MODULE_PATH})
set(CMAKE_MODULE_PATH ${PROJECT_SOURCE_DIR}/cmake/morse ${CMAKE_MODULE_PATH})
set(BFPS_LIBS "")
set(TURTLE_LIBS "")
option(NDEBUG "Define NDEBUG macro" ON)
if(NDEBUG)
add_definitions(-DNDEBUG)
endif()
# taken from https://vicrucann.github.io/tutorials/quick-cmake-doxygen/
# indicate the documentation build as an option and set it to ON by default
option(BUILD_DOC "Build documentation" OFF)
# check if Doxygen is installed
find_package(Doxygen)
if (DOXYGEN_FOUND)
# set input and output files
set(DOXYGEN_IN ${CMAKE_CURRENT_SOURCE_DIR}/documentation/cpp/cpp_config)
set(DOXYGEN_OUT ${CMAKE_CURRENT_BINARY_DIR}/Doxyfile)
# request to configure the file
configure_file(${DOXYGEN_IN} ${DOXYGEN_OUT} @ONLY)
message("Doxygen build started")
# note the option ALL which allows to build the docs together with the application
add_custom_target( doc_doxygen
COMMAND ${DOXYGEN_EXECUTABLE} ${DOXYGEN_OUT}
WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}
COMMENT "Generating API documentation with Doxygen"
VERBATIM )
else (DOXYGEN_FOUND)
message("Doxygen needs to be installed to generate the doxygen documentation")
endif (DOXYGEN_FOUND)
# check if sphinx is available
find_package(Sphinx)
if (SPHINX_FOUND)
if(NOT DEFINED SPHINX_THEME)
set(SPHINX_THEME default)
endif()
if(NOT DEFINED SPHINX_THEME_DIR)
set(SPHINX_THEME_DIR)
endif()
# configured documentation tools and intermediate build results
set(BINARY_BUILD_DIR "${CMAKE_CURRENT_BINARY_DIR}/sphinx_build")
# Sphinx cache with pickled ReST documents
set(SPHINX_CACHE_DIR "${CMAKE_CURRENT_BINARY_DIR}/sphinx_doctrees")
# HTML output directory
set(SPHINX_HTML_DIR "${CMAKE_CURRENT_BINARY_DIR}/sphinx_html")
# LaTeX output directory
set(SPHINX_LATEX_DIR "${CMAKE_CURRENT_BINARY_DIR}/sphinx_latex")
configure_file(
"${PROJECT_SOURCE_DIR}/documentation/conf.py.in"
"${BINARY_BUILD_DIR}/conf.py"
@ONLY)
add_custom_target(doc_html
${SPHINX_EXECUTABLE}
-q -b html
-c "${BINARY_BUILD_DIR}"
-d "${SPHINX_CACHE_DIR}"
"${PROJECT_SOURCE_DIR}/documentation"
"${SPHINX_HTML_DIR}"
COMMENT "Building HTML documentation with Sphinx")
add_custom_target(doc_latex
${SPHINX_EXECUTABLE}
-q -b latex
-c "${BINARY_BUILD_DIR}"
-d "${SPHINX_CACHE_DIR}"
"${PROJECT_SOURCE_DIR}/documentation"
"${SPHINX_LATEX_DIR}"
COMMENT "Building LaTeX documentation with Sphinx")
else (SPHINX_FOUND)
message("Sphinx needs to be installed to generate the full documentation")
endif (SPHINX_FOUND)
option(TIMING_OUTPUT "Toggle timing output. WARNING: memory usage is proportional to `niter_todo`" OFF)
if(TIMING_OUTPUT)
add_definitions(-DUSE_TIMINGOUTPUT)
endif()
#####################################################################################
## MPI
......@@ -64,7 +148,7 @@ set(CMAKE_CXX_COMPILE_FLAGS "${CMAKE_CXX_COMPILE_FLAGS} ${MPI_CXX_COMPILE_OPTION
set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} ${MPI_CXX_LINK_FLAGS}")
include_directories(${MPI_CXX_INCLUDE_DIRS})
add_definitions(${MPI_CXX_COMPILE_DEFINITIONS})
list(APPEND BFPS_LIBS "${MPI_CXX_LIBRARIES}")
list(APPEND TURTLE_LIBS "${MPI_CXX_LIBRARIES}")
#####################################################################################
## CXX Standard
......@@ -80,12 +164,12 @@ find_package(OpenMP REQUIRED)
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${OpenMP_CXX_FLAGS}")
set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} ${OpenMP_EXE_LINKER_FLAGS}")
list(APPEND BFPS_LIBS "${OpenMP_CXX_LIB_NAMES}")
list(APPEND TURTLE_LIBS "${OpenMP_CXX_LIB_NAMES}")
#####################################################################################
## Extra flags
set(CMAKE_CXX_COMPILE_FLAGS "${CMAKE_CXX_COMPILE_FLAGS} $ENV{BFPS_OPTIMIZATION_FLAGS} -Wall -g")
set(CMAKE_CXX_COMPILE_FLAGS "${CMAKE_CXX_COMPILE_FLAGS} $ENV{TURTLE_COMPILATION_FLAGS} -Wall -g -Wfatal-errors -fopenmp")
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${CMAKE_CXX_COMPILE_FLAGS}")
#####################################################################################
......@@ -97,22 +181,23 @@ if(NOT DEFINED ENV{HDF5_ROOT})
endif()
set(HDF5_PREFER_PARALLEL TRUE)
set(HDF5_NO_FIND_PACKAGE_CONFIG_FILE TRUE)
find_package(HDF5 REQUIRED)
message(STATUS "HDF5_C_INCLUDE_DIRS ${HDF5_C_INCLUDE_DIRS}")
include_directories(${HDF5_C_INCLUDE_DIRS})
add_definitions(${HDF5_C_DEFINITIONS})
list(APPEND BFPS_LIBS "${HDF5_C_LIBRARIES}")
list(APPEND TURTLE_LIBS "${HDF5_C_LIBRARIES}")
option(BFPS_HDF5_USE_SZIP "Set to on to also link against SZIP" OFF)
option(TURTLE_HDF5_USE_SZIP "Set to on to also link against SZIP" OFF)
if(BFPS_HDF5_USE_SZIP)
option(BFPS_HDF5_SZIP_LIB_PATH "Additional lib path for SZIP" "")
if(BFPS_HDF5_SZIP_LIB_PATH)
link_directories(${BFPS_HDF5_SZIP_LIB_PATH})
if(TURTLE_HDF5_USE_SZIP)
option(TURTLE_HDF5_SZIP_LIB_PATH "Additional lib path for SZIP" "")
if(TURTLE_HDF5_SZIP_LIB_PATH)
link_directories(${TURTLE_HDF5_SZIP_LIB_PATH})
endif()
list(APPEND BFPS_LIBS "z")
list(APPEND TURTLE_LIBS "z")
endif()
#####################################################################################
......@@ -123,31 +208,41 @@ if(NOT DEFINED ENV{FFTW_DIR})
message(WARNING "The environment variable FFTW_DIR is undefined, this might cause trouble in finding the FFTW")
endif()
find_package(FFTW REQUIRED OMP)
find_package(FFTW REQUIRED)
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${FFTW_CFLAGS_OTHER}")
set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} ${FFTW_LDFLAGS_OTHER}")
list(APPEND BFPS_LIBS "${FFTW_LIBRARIES}")
list(APPEND TURTLE_LIBS "${FFTW_LIBRARIES}")
include_directories(${FFTW_INCLUDE_DIRS})
link_directories(${FFTW_LIBRARY_DIRS})
find_package(FFTW REQUIRED OMP SIMPLE)
find_package(FFTW REQUIRED SIMPLE)
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${FFTW_CFLAGS_OTHER}")
set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} ${FFTW_LDFLAGS_OTHER}")
list(APPEND BFPS_LIBS "${FFTW_LIBRARIES}")
list(APPEND TURTLE_LIBS "${FFTW_LIBRARIES}")
include_directories(${FFTW_INCLUDE_DIRS})
link_directories(${FFTW_LIBRARY_DIRS})
# hack for FFTW OMP libs
find_library(
FFTWF_OMP fftw3f_omp
HINTS ${FFTW_LIBRARY_DIRS} $ENV{FFTW_LIBDIR} $ENV{FFTW_OPENMP_LIBDIR})
set(TURTLE_LIBS ${FFTWF_OMP} ${TURTLE_LIBS})
find_library(
FFTW_OMP fftw3_omp
HINTS ${FFTW_LIBRARY_DIRS} $ENV{FFTW_LIBDIR} $ENV{FFTW_OPENMP_LIBDIR})
set(TURTLE_LIBS ${FFTW_OMP} ${TURTLE_LIBS})
# hack for FFTW MPI libs
find_library(
FFTWF_MPI fftw3f_mpi
HINTS ${FFTW_LIBRARY_DIRS})
set(BFPS_LIBS ${FFTWF_MPI} ${BFPS_LIBS})
HINTS ${FFTW_LIBRARY_DIRS} $ENV{FFTW_LIBDIR} $ENV{FFTW_MPI_LIBDIR})
set(TURTLE_LIBS ${FFTWF_MPI} ${TURTLE_LIBS})
find_library(
FFTW_MPI fftw3_mpi
HINTS ${FFTW_LIBRARY_DIRS})
set(BFPS_LIBS ${FFTW_MPI} ${BFPS_LIBS})
HINTS ${FFTW_LIBRARY_DIRS} $ENV{FFTW_LIBDIR} $ENV{FFTW_MPI_LIBDIR})
set(TURTLE_LIBS ${FFTW_MPI} ${TURTLE_LIBS})
#####################################################################################
......@@ -166,13 +261,18 @@ set(cpp_for_lib
${PROJECT_SOURCE_DIR}/cpp/full_code/code_base.cpp
${PROJECT_SOURCE_DIR}/cpp/full_code/direct_numerical_simulation.cpp
${PROJECT_SOURCE_DIR}/cpp/full_code/NSVE.cpp
${PROJECT_SOURCE_DIR}/cpp/full_code/static_field.cpp
${PROJECT_SOURCE_DIR}/cpp/full_code/kraichnan_field.cpp
${PROJECT_SOURCE_DIR}/cpp/full_code/joint_acc_vel_stats.cpp
${PROJECT_SOURCE_DIR}/cpp/full_code/test.cpp
${PROJECT_SOURCE_DIR}/cpp/full_code/filter_test.cpp
${PROJECT_SOURCE_DIR}/cpp/full_code/field_test.cpp
${PROJECT_SOURCE_DIR}/cpp/full_code/write_filtered_test.cpp
${PROJECT_SOURCE_DIR}/cpp/full_code/Gauss_field_test.cpp
${PROJECT_SOURCE_DIR}/cpp/full_code/symmetrize_test.cpp
${PROJECT_SOURCE_DIR}/cpp/full_code/field_output_test.cpp
${PROJECT_SOURCE_DIR}/cpp/full_code/get_rfields.cpp
${PROJECT_SOURCE_DIR}/cpp/full_code/bandpass_stats.cpp
${PROJECT_SOURCE_DIR}/cpp/full_code/field_single_to_double.cpp
${PROJECT_SOURCE_DIR}/cpp/full_code/resize.cpp
${PROJECT_SOURCE_DIR}/cpp/full_code/NSVE_field_stats.cpp
......@@ -201,18 +301,26 @@ set(cpp_for_lib
${PROJECT_SOURCE_DIR}/cpp/full_code/NSVEparticles.cpp
${PROJECT_SOURCE_DIR}/cpp/full_code/NSVEcomplex_particles.cpp
${PROJECT_SOURCE_DIR}/cpp/full_code/NSVEp_extra_sampling.cpp
${PROJECT_SOURCE_DIR}/cpp/particles/particles_inner_computer.cpp)
${PROJECT_SOURCE_DIR}/cpp/particles/particles_inner_computer.cpp
${PROJECT_SOURCE_DIR}/cpp/full_code/ornstein_uhlenbeck_process.cpp
${PROJECT_SOURCE_DIR}/cpp/full_code/ou_vorticity_equation.cpp)
set(hpp_for_lib
${PROJECT_SOURCE_DIR}/cpp/full_code/code_base.hpp
${PROJECT_SOURCE_DIR}/cpp/full_code/direct_numerical_simulation.hpp
${PROJECT_SOURCE_DIR}/cpp/full_code/NSVE.hpp
${PROJECT_SOURCE_DIR}/cpp/full_code/static_field.hpp
${PROJECT_SOURCE_DIR}/cpp/full_code/kraichnan_field.hpp
${PROJECT_SOURCE_DIR}/cpp/full_code/joint_acc_vel_stats.hpp
${PROJECT_SOURCE_DIR}/cpp/full_code/test.hpp
${PROJECT_SOURCE_DIR}/cpp/full_code/filter_test.hpp
${PROJECT_SOURCE_DIR}/cpp/full_code/Gauss_field_test.hpp
${PROJECT_SOURCE_DIR}/cpp/full_code/field_test.hpp
${PROJECT_SOURCE_DIR}/cpp/full_code/write_filtered_test.hpp
${PROJECT_SOURCE_DIR}/cpp/full_code/symmetrize_test.hpp
${PROJECT_SOURCE_DIR}/cpp/full_code/field_output_test.hpp
${PROJECT_SOURCE_DIR}/cpp/full_code/get_rfields.hpp
${PROJECT_SOURCE_DIR}/cpp/full_code/bandpass_stats.hpp
${PROJECT_SOURCE_DIR}/cpp/full_code/field_single_to_double.hpp
${PROJECT_SOURCE_DIR}/cpp/full_code/resize.hpp
${PROJECT_SOURCE_DIR}/cpp/full_code/NSVE_field_stats.hpp
......@@ -245,11 +353,14 @@ set(hpp_for_lib
${PROJECT_SOURCE_DIR}/cpp/particles/abstract_particles_input.hpp
${PROJECT_SOURCE_DIR}/cpp/particles/abstract_particles_output.hpp
${PROJECT_SOURCE_DIR}/cpp/particles/abstract_particles_system.hpp
${PROJECT_SOURCE_DIR}/cpp/particles/abstract_particles_system_with_p2p.hpp
${PROJECT_SOURCE_DIR}/cpp/particles/alltoall_exchanger.hpp
${PROJECT_SOURCE_DIR}/cpp/particles/env_utils.hpp
${PROJECT_SOURCE_DIR}/cpp/particles/lock_free_bool_array.hpp
${PROJECT_SOURCE_DIR}/cpp/particles/p2p_computer_empty.hpp
${PROJECT_SOURCE_DIR}/cpp/particles/p2p_computer.hpp
${PROJECT_SOURCE_DIR}/cpp/particles/p2p_ghost_collisions.hpp
${PROJECT_SOURCE_DIR}/cpp/particles/p2p_cylinder_collisions.hpp
${PROJECT_SOURCE_DIR}/cpp/particles/p2p_distr_mpi.hpp
${PROJECT_SOURCE_DIR}/cpp/particles/p2p_tree.hpp
${PROJECT_SOURCE_DIR}/cpp/particles/particles_adams_bashforth.hpp
......@@ -275,35 +386,36 @@ set(hpp_for_lib
${PROJECT_SOURCE_DIR}/cpp/omputils.hpp
${PROJECT_SOURCE_DIR}/cpp/shared_array.hpp
${PROJECT_SOURCE_DIR}/cpp/spline.hpp
${PROJECT_SOURCE_DIR}/cpp/full_code/ornstein_uhlenbeck_process.hpp
${PROJECT_SOURCE_DIR}/cpp/full_code/ou_vorticity_equation.hpp
)
#file(GLOB_RECURSE hpp_for_lib ${PROJECT_SOURCE_DIR}/*.hpp)
LIST(APPEND source_files ${hpp_for_lib} ${cpp_for_lib})
add_library(bfps ${source_files})
add_library(TurTLE ${source_files})
target_link_libraries(bfps ${BFPS_LIBS})
target_link_libraries(TurTLE ${TURTLE_LIBS})
install(TARGETS bfps EXPORT BFPS_EXPORT DESTINATION lib/ )
install(DIRECTORY ${PROJECT_SOURCE_DIR}/cpp/ DESTINATION include/bfps/ FILES_MATCHING PATTERN "*.h*")
install(TARGETS TurTLE EXPORT TURTLE_EXPORT DESTINATION lib/ )
install(DIRECTORY ${PROJECT_SOURCE_DIR}/cpp/ DESTINATION include/TurTLE/ FILES_MATCHING PATTERN "*.h*")
#####################################################################################
## Export the configuration
configure_file(${PROJECT_SOURCE_DIR}/cmake/BFPSConfig.cmake.in ${PROJECT_BINARY_DIR}/BFPSConfig.cmake @ONLY)
configure_file(${PROJECT_SOURCE_DIR}/cmake/TurTLEConfig.cmake.in ${PROJECT_BINARY_DIR}/TurTLEConfig.cmake @ONLY)
install(FILES "${PROJECT_BINARY_DIR}/BFPSConfig.cmake" DESTINATION lib/)
export(TARGETS bfps FILE "${PROJECT_BINARY_DIR}/BFPSLibraryDepends.cmake")
install(EXPORT BFPS_EXPORT DESTINATION lib/)
install(FILES "${PROJECT_BINARY_DIR}/TurTLEConfig.cmake" DESTINATION lib/)
export(TARGETS TurTLE FILE "${PROJECT_BINARY_DIR}/TurTLELibraryDepends.cmake")
install(EXPORT TURTLE_EXPORT DESTINATION lib/)
#####################################################################################
## Install the python wrapper
## Install the python3 wrapper
# copy command
install(CODE "execute_process(COMMAND ${CMAKE_COMMAND} -E copy_directory ${PROJECT_SOURCE_DIR}/bfps ${PROJECT_BINARY_DIR}/python/bfps/)")
install(CODE "execute_process(COMMAND ${CMAKE_COMMAND} -E copy_directory ${PROJECT_SOURCE_DIR}/TurTLE ${PROJECT_BINARY_DIR}/python/TurTLE/)")
if(EXISTS "${PROJECT_SOURCE_DIR}/host_info.py")
install(CODE "execute_process(COMMAND ${CMAKE_COMMAND} -E copy ${PROJECT_SOURCE_DIR}/host_info.py ${PROJECT_BINARY_DIR}/python/bfps/)")
install(CODE "execute_process(COMMAND ${CMAKE_COMMAND} -E copy ${PROJECT_SOURCE_DIR}/host_info.py ${PROJECT_BINARY_DIR}/python/TurTLE/)")
else()
install(CODE "execute_process(COMMAND ${CMAKE_COMMAND} -E copy ${PROJECT_SOURCE_DIR}/pc_host_info.py ${PROJECT_BINARY_DIR}/python/bfps/host_info.py)")
install(CODE "execute_process(COMMAND ${CMAKE_COMMAND} -E copy ${PROJECT_SOURCE_DIR}/pc_host_info.py ${PROJECT_BINARY_DIR}/python/TurTLE/host_info.py)")
endif()
install(CODE "execute_process(COMMAND python ${PROJECT_SOURCE_DIR}/setup.py install --force --prefix=${CMAKE_INSTALL_PREFIX} WORKING_DIRECTORY ${PROJECT_BINARY_DIR}/python/)")
install(CODE "execute_process(COMMAND python3 ${PROJECT_SOURCE_DIR}/setup.py install --force --prefix=${CMAKE_INSTALL_PREFIX} WORKING_DIRECTORY ${PROJECT_BINARY_DIR}/python/)")
================================
Big Fluid and Particle Simulator
================================
In brief, this code runs pseudospectral direct numerical simulations
(DNS) of the incompressible Navier-Stokes equations, using FFTW 3, and
it can integrate particle trajectories in the resulting fields.
The Navier-Stokes solver has been extensively tested (tests are included
in the repository), and it is working as expected.
Parameters and statistics are stored in HDF5 format, together with code
information, so simulation data should be "future proof" --- suggestions
of possible improvements to the current approach are always welcome.
The primary aim of bfps is to reduce the time spent on setting up and
baby sitting DNS, as well as simplify the analysis of the generated
data.
The wish is that this Python package provides an easy and general way
of constructing efficient specialized DNS C++ codes for different
turbulence problems encountered in research.
At the same time, the package should provide a unified way of
postprocessing, and accessing the postprocessing results.
The code therefore consists of two main parts: the pure C++ code, a set
of loosely related "building blocks", and the Python code, which can
generate C++ code using the pure classes, but with a significant degree
of flexibility.
The code user is expected to write a small python script that will
properly define the DNS they are interested in running.
That code will generate an executable that can then be run directly on
the user's machine, or submitted to a queue on a cluster.
.. only:: html
.. image:: https://gitlab.mpcdf.mpg.de/mjr/bfps/badges/develop/build.svg
:target: https://gitlab.mpcdf.mpg.de/mjr/bfps/commits/develop
=========================================
Turbulence Tools: Lagrangian and Eulerian
=========================================
.. _sec-introduction:
------------
Introduction
------------
TurTLE implements a number of standard functionality of Fourier-based
pseudo-spectral numerical simulations, as well as the corresponding
numerical particle-tracking functionality.
The package also contains a Navier-Stokes solver, as well as a small
number of post-processing tools.
The solver is production-ready, having already been used for a number of
publications.
TurTLE is written in C++ and it uses a hybrid MPI/OpenMP programming
paradigm, relying on FFTW3 for an efficient Fourier transform
implementation.
HDF5 is used for I/O, including for parameter input and output of
statistics.
CMake is used for compilation and installation.
A Python 3 wrapper is used to prepare parameters and initial conditions,
as well as to generate job submission scripts for clusters and to
perform basic post-processing of simulation results.
The primary aim of TurTLE is to provide optimal performance, while
reducing the time spent on setting up and supervising ensembles of DNS,
with the added benefit of a unified launch-postprocess approach through
the use of the Python wrapper.
.. _sec-installation:
......@@ -37,51 +43,41 @@ the user's machine, or submitted to a queue on a cluster.
Installation
------------
So far, the code has been run on laptops, desktops, and a couple of
clusters (biggest run so far was 1536^3 on 16 nodes of 32 cores each,
with about 11 seconds per time step, for a simple incompressible
Navier-Stokes problem).
Postprocessing data may not be very computationally intensive, depending
on the amount of data involved.
TurTLE can be used on various machines, with laptops routinely being
used for development and testing, but large production runs using tens
of thousands of CPU cores on large computing clusters.
**Postprocessing only**
Use a console; navigate to the ``bfps`` folder, and type:
The Python 3 package may be installed directly if only post-processing
of existing data is desired:
.. code:: bash
python setup.py install
(add `--user` or `sudo` as appropriate).
`setup.py` should tell you about the various packages you need.
`setup.py` uses the `setuptools` package for dependency resolution.
**Full installation**
If you want to run simulations on the machine where you're installing,
you will need to use `cmake` to compile and install the full library.
Your machine needs to have an MPI compiler installed, the HDF5 C library
and FFTW >= 3.4 --- detailed instructions are
included at the end of this document.
The C++ library requires a number of dependencies, that `CMake` will
search for before compilation and installation.
In brief, an MPI compiler is required, as well as the HDF5 C library
compiled with parallel support and FFTW >= 3.4.
Detailed instructions are provided at the end of this document.
-------------
Documentation
-------------
While the code is not fully documented yet, basic information is already
available, and it is recommended that you generate the manual and go
through it carefully.
Please do ask for specific improvements to the current text where it is
found lacking.
In order to generate the manual, navigate to the repository folder, and
execute the following commands:
.. code:: bash
cd documentation
make latexpdf
Optionally, html documentation can be generated instead if needed, just
type ``make html`` instead of ``make latexpdf``.
Partial documentation is available (link to online documentation coming
soon).
Doxygen is used for the C++ source code, and the `Sphinx` and `breathe`
Python packages are used to merge the Python wrapper documentation with
the C++ documentation.
The optional `CMake` targets `doc_doxygen`, `doc_html` and `doc_latex`
generate the required documents in the build directory.
--------
Comments
......@@ -101,32 +97,56 @@ Comments
* code is used mainly with Python 3.5 and later, and it is not tested at
all with Python 2.x
-------------------------------
Installation with prerequisites
-------------------------------
-----------------
Full installation
-----------------
These installation steps assume that you have a working MPI compiler,
properly configured on your system (i.e. the various configure scripts
are able to find it), as well as the `cmake` tool.
We recommend to specify the desired MPI C++ compiler by exporting the
environment variable `MPICXX` --- the BFPS cmake configuration looks for
this variable.
We also recommend that an environment variable `BFPS_OPTIMIZATION_FLAGS`
is defined appropriately.
In particular, for clusters of unknown architecture it helps to log into
individual nodes and run the following command:
.. code:: bash
The list is a work in progress, please contact us
(Cristian.Lalescu@ds.mpg.de) if the procedure fails at any step of the
process.
gcc -march=native -Q --help=target
*Note*: the HDF5 library may require MPI standard v1, we haven't tested
this in detail. Some default OpenMPI3 installations will not work.
Detailed full installation instructions:
1. Make directory PREFIX on a local fast partition.
1.
Export the following environment variables with apropriate values:
* `MPICXX` - Preferred MPI C++ compiler
* `FFTW_DIR` - Base directory of FFTW
* `FFTW_INCDIR` - Directory containing FFTW header files
* `FFTW_LIBDIR` - Directory containing FFTW library files
* `FFTW_OPENMP_LIBDIR` - Directory containing OpenMP FFTW library files (please define if different from FFTW_LIBDIR)
* `FFTW_MPI_LIBDIR` - Directory containing MPI FFTW library files (please define if different from FFTW_LIBDIR)
* `HDF5_ROOT` - Base directory of HDF5
* `CMAKE_PREFIX_PATH` - Directory containing local cmake configuration files
We also recommend that an environment variable `TURTLE_COMPILATION_FLAGS`
is defined appropriately.
In particular, for clusters of unknown architecture it helps to log into
individual nodes and run the following command:
.. code:: bash
gcc -march=native -Q --help=target
Note that an example file `example.bashrc` is included in the
repository.
There is at least one recorded case of a cluster where different
FFTW libraries (serial, MPI and OpenMP) were located in different
folders, hence the options of specifying the details.
See also lines 143 through 160 of `CMakeLists.txt`.
2.
Make directory PREFIX on a local fast partition.
2. Download, compile, install FFTW (latest version 3.x from http://www.fftw.org/).
Execute the following commands in order, feel free to customize
optimisation flags for your own computer (see http://www.fftw.org/fftw3_doc/Installation-on-Unix.html):
3.
Download, compile, install FFTW (latest version 3.x from http://www.fftw.org/).
Execute the following commands in order, feel free to customize
optimisation flags for your own computer (see http://www.fftw.org/fftw3_doc/Installation-on-Unix.html):
.. code:: bash
......@@ -137,13 +157,15 @@ Detailed full installation instructions:
make
make install
BFPS will try to find FFTW using the FindFFTW from the Morse project.
If the package is installed in a non standard location, it is recommanded
to setup the environment variables: `FFTW_DIR` (or `FFTW_INCDIR` and `FFTW_LIBDIR`).
TurTLE will try to find FFTW using the FindFFTW from the Morse project.
If the package is installed in a non standard location, it is recommanded
to setup the environment variables: `FFTW_DIR`, `FFTW_INCDIR`,
`FFTW_LIBDIR`, `FFTW_OPEMMP_LIBDIR` and `FFTW_MPI_LIBDIR`.
3. Download, compile, install HDF5 (version 1.8.x, currently available
at https://portal.hdfgroup.org/display/support/HDF5+1.8.20#files).
We are using parallel I/O, therefore we must use the plain C interface of HDF5:
4.
Download, compile, install HDF5 (version 1.8.x, currently available
at https://portal.hdfgroup.org/display/support/HDF5+1.8.20#files).
We are using parallel I/O, therefore we must use the plain C interface of HDF5:
.. code:: bash
......@@ -151,49 +173,99 @@ Detailed full installation instructions:
make
make install
BFPS will try to find HDF5 using the regular FindHDF5.
Therefore, if the package is installed in a non standard location, it is recommanded
to setup the environment variable: HDF5_ROOT.
TurTLE will try to find HDF5 using the regular FindHDF5.
Therefore, if the package is installed in a non standard location, it is recommanded
to setup the environment variable: HDF5_ROOT.
3. Optional.
We recommend the creation of a virtual python3 environment (also under PREFIX) that will be used for installing bfps and dependencies.
Please see https://docs.python-guide.org/dev/virtualenvs/.
5. TurTLE requires `cmake` version > 3.12.
If required, download, compile and install cmake, currently
available at https://cmake.org/cmake/resources/software.html.
4. Clone bfps repository.
.. code:: bash
./bootstrap --prefix=PREFIX
make
make install
5.
Optional.
We recommend the creation of a virtual python3 environment (also
under PREFIX) that will be used for installing TurTLE and
dependencies.
Please see https://docs.python-guide.org/dev/virtualenvs/.
6.
Clone turtle repository.