diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index bf0e4b0b29a98e95df25ee13be948ce60eeecb18..10776184bcdbded0ca7496d9d76663374633efda 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -32,7 +32,29 @@ image: gitlab-registry.mpcdf.mpg.de/mpcdf/module-image
export CC=icc
export CXX=icpc
-build-gcc:
+#.allow_docker_run_as_root: &allow_docker_run_as_root |
+# export OMPI_ALLOW_RUN_AS_ROOT=1
+# export OMPI_ALLOW_RUN_AS_ROOT_CONFIRM=1
+
+build-gcc-openmpi:
+ stage: build
+ script:
+ - *load_modules
+ - *export_GCC_compilers
+ - *build
+ variables:
+ COMPILER: "gcc"
+ MPI: "openmpi"
+ MPICXX: "mpicxx"
+ tags:
+ - docker
+ artifacts:
+ paths:
+ - build/
+ expire_in: 1 hour
+ when: always
+
+build-gcc-impi:
stage: build
script:
- *load_modules
@@ -72,7 +94,7 @@ build-intel:
expire_in: 1 hour
when: always
-test-gcc:
+test-gcc-impi:
stage: test
script:
- *load_modules
@@ -87,9 +109,26 @@ test-gcc:
MPI: "impi"
MPICXX: "mpigxx"
needs:
- - job: build-gcc
+ - job: build-gcc-impi
artifacts: true
+#test-gcc-openmpi:
+# stage: test
+# script:
+# - *load_modules
+# - *export_GCC_compilers
+# - *allow_docker_run_as_root
+# - *run_tests
+# tags:
+# - docker
+# variables:
+# COMPILER: "gcc"
+# MPI: "openmpi"
+# MPICXX: "mpicxx"
+# needs:
+# - job: build-gcc-openmpi
+# artifacts: true
+
test-intel:
stage: test
script:
diff --git a/README.rst b/README.rst
index 68c73f0c17edd9158b891c6d5ddd13b6533b016b..31de5a79f1436afb8a9bab9c0e0fe6e4d420a784 100644
--- a/README.rst
+++ b/README.rst
@@ -47,8 +47,10 @@ described at http://TurTLE.pages.mpcdf.de/turtle/html
Authors
-------
-TurTLE is developed and maintained by the Wilczek group at the Max Planck Institute for Dynamics and Self-Organization and the University of Bayreuth in collaboration with the Application Support Group of the
-Max Planck Computing and Data Facility.
+TurTLE is developed and maintained by the Wilczek group at the Max Planck
+Institute for Dynamics and Self-Organization and the University of Bayreuth
+in collaboration with the Application Support Group of the Max Planck
+Computing and Data Facility.
TurTLE contains contributions from:
@@ -56,18 +58,13 @@ TurTLE contains contributions from:
.. _sec-installation:
-------------
-Installation
-------------
-
-TurTLE can be used on various machines, with laptops routinely being
-used for development and testing, but large production runs using tens
-of thousands of CPU cores on large computing clusters.
-
-**Postprocessing only**
+---------------------------------------------------
+Installation for postprocessing only
+---------------------------------------------------
The Python 3 package may be installed directly if only post-processing
-of existing data is desired:
+of existing data is desired. Simply clone the repository and install by
+executing
.. code:: bash
@@ -77,7 +74,13 @@ of existing data is desired:
(add `--user` or `sudo` as appropriate).
`setup.py` uses the `setuptools` package for dependency resolution.
-**Full installation**
+----------------------
+Full installation
+----------------------
+
+TurTLE can be used on various machines, with laptops routinely being
+used for development and testing, but large production runs using tens
+of thousands of CPU cores on large computing clusters.
The C++ library requires a number of dependencies, that `CMake` will
search for before compilation and installation.
@@ -87,209 +90,187 @@ We provide instructions for local compilation of FFTW and HDF5, because
default versions packaged with Linux variants are typically inadequately
configured.
-These installation steps assume that you have a working MPI compiler,
+These installation steps assume that you have a working C/C++ MPI compiler,
properly configured on your system (i.e. the various configure scripts
-are able to find it), as well as the `cmake` tool.
+are able to find it), as well as an installation of Python 3.
The list is a work in progress, please contact us
(Cristian.Lalescu@mpcdf.mpg.de) if the procedure fails at any step of the
process.
We recommend to first go through the instructions in full, and only
afterwards starting to execute the individual steps.
-*Note*: the HDF5 1.8x library may require MPI standard v1, we haven't tested
-this in detail. Some default OpenMPI3 installations will not work.
-
-Detailed full installation instructions:
-
-1.
- Optional.
- Make a directory `PREFIX` on a local fast partition.
- (Under unix systems, you can probably use `~/.local`).
-
-2.
- Optional.
- We recommend the creation of a virtual python3 environment that will be
- used for installing TurTLE.
- Please see https://docs.python-guide.org/dev/virtualenvs/.
- In the following, the install location of this environment is
- denoted `TurTLE_DIR`.
- In principle `TurTLE_DIR` should be a subdirectory of the previously
- created `PREFIX` folder.
-
-3.
- Export the following environment variables with apropriate values:
- * `CC` - Preferred C compiler
- * `CXX` - Preferred C++ compiler
- * `MPI_HOME` - MPI installation directory
- * `FFTW_DIR` - Base directory of FFTW
- * `FFTW_INCDIR` - Directory containing FFTW header files
- * `FFTW_LIBDIR` - Directory containing FFTW library files
- * `FFTW_OPENMP_LIBDIR` - Directory containing OpenMP FFTW library files (please define if different from FFTW_LIBDIR)
- * `FFTW_MPI_LIBDIR` - Directory containing MPI FFTW library files (please define if different from FFTW_LIBDIR)
-
- If you do not have FFTW installed already, then download, compile, install FFTW
- (latest version 3.x from http://www.fftw.org/).
- Execute the following commands in order, feel free to customize
- optimisation flags for your own computer (see http://www.fftw.org/fftw3_doc/Installation-on-Unix.html):
+**Creating a virtual environment**
+
+We recommend creating a virtual environment for TurTLE. To do this,
+choose an installation location :code:`<INSTALL_LOC>` on a local fast partition
+(Under unix systems, this could be e.g. `~/.local`) and a name for the
+environment :code:`<VENV_NAME>` (e.g. `turtle-production`).
+To create the virtual environment, execute:
+
+.. code:: bash
+
+ python -m venv <INSTALL_LOC>/<VENV_NAME>
+
+In the following, we refer to the path :code:`<INSTALL_LOC>` /
+:code:`<VENV_NAME>` as :code:`<TURTLE_DIR>`. For more information on virtual
+environments, please see https://docs.python-guide.org/dev/virtualenvs/.
+
+**Installation of requirements**
+
+TurTLE has the following requirements:
+
+- C/C++ MPI compiler of your choice
+
+- Python 3
+
+- FFTW version >= 3.3.4
+
+- HDF5 version >= 1.8. **Important**: must be compiled with the option "--enable-parallel"
+
+- cmake version > 3.12
+
+We will assume you already have a working MPI compiler and an
+installation of Python 3. For the other requirements, we provide
+installation instructions in the following. We recommend installing
+them on a local fast partition. In the following, we refer to their
+locations as :code:`<FFTW_DIR>`, :code:`<HDF5_DIR>` and
+:code:`<CMAKE_DIR>`. You may choose to install the requirements into
+your virtual environment. In that case, all of these placeholders are
+equal to :code:`<TURTLE_DIR>`.
+
+- **FFTW**
+ Download latest version from http://www.fftw.org/.
+ To compile and install it in custom location :code:`<FFTW_DIR>`,
+ execute the following commands in order, feel free to customize
+ optimisation flags for your own computer (see
+ http://www.fftw.org/fftw3_doc/Installation-on-Unix.html):
.. code:: bash
- ./configure --prefix=FFTW_DIR --enable-float --enable-sse --enable-mpi --enable-openmp --enable-threads
+ ./configure --prefix=<FFTW_DIR> --enable-float --enable-sse --enable-mpi --enable-openmp --enable-threads
make
make install
- ./configure --prefix=FFTW_DIR --enable-sse2 --enable-avx512 --enable-mpi --enable-openmp --enable-threads
+ ./configure --prefix=<FFTW_DIR> --enable-sse2 --enable-avx512 --enable-mpi --enable-openmp --enable-threads
make
make install
- In the above instructions, replace `FFTW_DIR` with an appropriate
- location (for instance it is safe to use the `PREFIX` defined above).
+..
TurTLE will try to find FFTW using the FindFFTW from the Morse project.
- If the package is installed in a non standard location, you should
- setup the environment variables listed above: `FFTW_DIR`, `FFTW_INCDIR`,
- `FFTW_LIBDIR`, `FFTW_OPEMMP_LIBDIR` and `FFTW_MPI_LIBDIR`.
-
-4.
- Export the following environment variables with apropriate values:
- * `CC` - Preferred C compiler
- * `CXX` - Preferred C++ compiler
- * `MPI_HOME` - MPI installation directory
- * `HDF5_ROOT` - Base directory of HDF5
+ If the package is installed in a non-standard location, make sure the
+ corresponding environment variables are properly exported
+ (see step 3 of installation of TurTLE).
- If you do not have HDF5 installed already, then download, compile, install
- HDF5.
- We are using parallel I/O, therefore we must use the plain C interface of HDF5:
+- **HDF5**
+ Download HDF5 from https://www.hdfgroup.org/downloads/hdf5/.
+ To compile and install it in custom location :code:`<HDF5_DIR>`,
+ execute the following commands in order:
.. code:: bash
- ./configure --prefix=HDF5_ROOT --enable-parallel
+ ./configure --prefix=<HDF5_DIR> --enable-parallel
make
make install
+ Since TurTLE is using parallel I/O, HDF5 needs to be compiled with
+ the "--enable-parallel" flag.
+
+..
TurTLE will try to find HDF5 using the regular FindHDF5, which
searches system folders, or `HDF5_ROOT`.
-5. TurTLE requires `cmake` version > 3.12, which should be available
- from your default package manager.
- If required, download, compile and install cmake, currently
- available at https://cmake.org/cmake/resources/software.html.
+- **cmake**
+ Check if cmake (version >3.12) is available from your default package manager.
+ If not, then download cmake at https://cmake.org/cmake/resources/software.html.
+ To compile and install it in custom location :code:`<CMAKE_DIR>`,
+ execute the following commands in order:
.. code:: bash
- ./bootstrap --prefix=PREFIX
+ ./bootstrap --prefix=<CMAKE_DIR>
make
make install
- The value of `PREFIX` used above is only relevant to later executing
+ The directory `<CMAKE_DIR>` is only relevant to later executing
the `cmake` binary (which can be found under `${PREFIX}/bin` after
installation).
-6.
- Clone turtle repository.
+
+**Installation of TurTLE**
+
+1. Choose a location for the source code, enter it and clone turtle
+ repository by
.. code:: bash
git clone git@gitlab.mpcdf.mpg.de:TurTLE/turtle.git
-7.
- Go into TurTLE repository, execute
+2. Execute
.. code:: bash
+ cd turtle
mkdir build
cd build
- cp ../pc_host_info.py ./host_info.py
- Edit the `host_info.py` file according to the instructions in the file.
- This is strictly required before installing on a cluster.
-
-8.
- Optional.
- *PINCHECK* is available from https://gitlab.mpcdf.mpg.de/khr/pincheck.
- If you'd like to check whether TurTLE MPI processes and OpenMP
- threads are pinned properly to hardware threads, you can simply
- place the pincheck headers under `${PREFIX}/include`.
-
-9.
- In the build folder, edit the file `bash_setup_for_TurTLE.sh` so that it
- exports the following environment variables with apropriate values:
- * `CC` - Preferred C compiler
- * `CXX` - Preferred C++ compiler
- * `MPI_HOME` - MPI installation directory
- * `FFTW_DIR` - Base directory of FFTW
- * `FFTW_INCDIR` - Directory containing FFTW header files
- * `FFTW_LIBDIR` - Directory containing FFTW library files
- * `FFTW_OPENMP_LIBDIR` - Directory containing OpenMP FFTW library files (please define if different from FFTW_LIBDIR)
- * `FFTW_MPI_LIBDIR` - Directory containing MPI FFTW library files (please define if different from FFTW_LIBDIR)
- * `HDF5_ROOT` - Base directory of HDF5
- * `PINCHECK_ROOT` - Directory under which
- `include/pincheck.hpp` may be found (OPTIONAL)
- * `TurTLE_DIR` - Directory under which TurTLE will be installed (same value from step 2).
- I.e. library will go under `${TurTLE_DIR}/lib`, headers under
- `${TurTLE_DIR}/include`, etc.
- This script should also contain the line
- `source TurTLE_DIR/lib/bash_setup_for_TurTLE.sh`, as well as the
- line `source TurTLE_DIR/bin/activate`.
-
- We also recommend that an environment variable `TURTLE_COMPILATION_FLAGS`
- is defined appropriately.
- In particular, for clusters of unknown architecture it helps to log into
- individual nodes and run the following command:
+3. Copy the file `bash_setup_template.sh` into the build folder:
.. code:: bash
- gcc -march=native -Q --help=target
+ cp ../bash_setup_template.sh ./bash_setup_for_TurTLE.sh
- Note that an example file `example.bashrc` is included in the
- repository.
- There is at least one recorded case of a cluster where different
- FFTW libraries (serial, MPI and OpenMP) were located in different
- folders, hence the options of specifying the details.
- See also lines 143 through 160 of `CMakeLists.txt`.
+ This file will set all required environment variables.
+ Please replace all placeholders by their corresponding values.
- *If using openmpi* you may need to recompile openmpi with the
- `--enable-mpi1-compatibility` flag, in order for HDF5 to compile and
- link (step 4).
-
- *Note*: in principle it is possible to add this information to your
+ *Note*: In principle it is possible to add this information to your
`.bashrc`, but we recommend against it.
-10.
- In the previously created build folder, execute
+
+4. Copy the file `pc_host_info.py` into the build folder:
+
+ .. code:: bash
+
+ cp ../pc_host_info.py ./host_info.py
+
+ This file contains information about the machine on which TurTLE
+ will run. On desktop machines, no further steps are required. On
+ clusters, please edit the `host_info.py` file according to the
+ instructions in the file.
+
+
+5.
+ Finally, we are ready to install TurTLE. Within the build folder, execute
.. code:: bash
source bash_setup_for_TurTLE.sh
- cmake .. -DCMAKE_INSTALL_PREFIX=TurTLE_DIR
- make
- # to get a verbose compilation process, use
- # VERBOSE=1 make
- # to use more than one core for the compilation use
- # make -jN
- # (where N is the number of available cores)
- make install
+ cmake .. -DCMAKE_INSTALL_PREFIX=<TURTLE_DIR>
-11.
- Using TurTLE from an external project.
- TurTLE creates and installs 3 files alongside the C++ headers and
- library:
+ Then compile TurTLE with :code:`<N>` cores using
.. code:: bash
- TurTLEConfig.cmake
- TurTLE_EXPORT.cmake
- TurTLE_EXPORT-noconfig.cmake
-
- These files are installed under `${TurTLE_DIR}/lib`.
- In case these files provide incomplete information, it is necessary to update
- the cmake input config file: `turtle/cmake/TurTLEConfig.cmake.in`.
-
-12.
- For clusters.
- In the `extra_slurm_lines` list from `host_info.py` one should add a
- corresponding `source INSTALL_DIR/lib/bash_setup_for_TurTLE.sh`
- line (where you should write the explicit install folder instead of
- `INSTALL_DIR`), such that the appropriate setup is loaded in the job
- script.
+ make -j<N>
+ make install
+
+Congratulations, you have installed TurTLE! Feel free to checkout the
+`tutorial <http://turtle.pages.mpcdf.de/turtle/sphinx_html/chapters/tutorial.html>`_.
+
+
+**Using TurTLE from an external project**.
+
+TurTLE creates and installs 3 files alongside the C++ headers and
+library:
+
+.. code:: bash
+
+ TurTLEConfig.cmake
+ TurTLE_EXPORT.cmake
+ TurTLE_EXPORT-noconfig.cmake
+
+These files are installed under `${TURTLE_DIR}/lib`.
+In case these files provide incomplete information, it is necessary to update
+the cmake input config file: `turtle/cmake/TurTLEConfig.cmake.in`.
+
**Uninstall**
diff --git a/TurTLE/_code.py b/TurTLE/_code.py
index 46403e4d484e52e61972e873404d22edcfcffc1e..4beefa12ca33fd0f85a8e3470b6e29fa8ebc9c21 100644
--- a/TurTLE/_code.py
+++ b/TurTLE/_code.py
@@ -314,8 +314,6 @@ class _code(_base):
command_atoms = ['mpirun',
'-np',
'{0}'.format(nb_processes),
- '-' + mpirun_environment_set,
- 'OMP_NUM_THREADS={0}'.format(nb_threads_per_process),
'./' + self.name,
self.simname]
if self.host_info['type'] == 'cluster':
diff --git a/bash_setup_template.sh b/bash_setup_template.sh
new file mode 100644
index 0000000000000000000000000000000000000000..3f19fb6203c1a95fff6a1656bf70368c8ede8aa8
--- /dev/null
+++ b/bash_setup_template.sh
@@ -0,0 +1,57 @@
+#! /bin/bash
+
+# Load dependencies here if necessary, e.g. `module load cmake`
+
+# module-dependent variables
+export CC=<CC> # Preferred C compiler
+export CXX=<CXX> # Preferred C++ compiler
+export MPI_HOME=<MPI_HOME> # MPI installation directory
+export FFTW_ROOT=<FFTW_DIR> # Base directory of FFTW
+export FFTW_DIR=<FFTW_DIR> # Base directory of FFTW
+export HDF5_ROOT=<HDF5_DIR> # Base directory of HDF5
+
+# OPTIONAL
+#
+# export PINCHECK_ROOT=<PINCHECK_ROOT>
+#
+# If this variable is exported, you can check whether TurTLE
+# MPI processes and OpenMP threads are pinned properly to hardware threads.
+# It is available from https://gitlab.mpcdf.mpg.de/khr/pincheck.
+# The pincheck headers need to be placed under `<PINCHECK_ROOT>/include`.
+
+# OPTIONAL
+#
+# export TURTLE_COMPILATION_FLAGS=<COMPILATION_FLAGS>
+#
+# We also recommend setting this variable to optimize compilation
+# For clusters of unknown architecture it helps to log into
+# individual nodes and run the following command:
+# gcc -march=native -Q --help=target
+
+# OPTIONAL
+#
+# export FFTW_INCDIR=<FFTW_INCDIR> # Directory containing FFTW header files
+# export FFTW_LIBDIR=<FFTW_LIBDIR> # Directory containing FFTW library files
+# export FFTW_OPENMP_LIBDIR=<FFTW_OMP_LIBDIR> # Directory containing OpenMP FFTW library files (please define if different from FFTW_LIBDIR)
+# export FFTW_MPI_LIBDIR=<FFTW_MPI_LIBDIR> # Directory containing MPI FFTW library files (please define if different from FFTW_LIBDIR)
+#
+# There is at least one recorded case of a cluster where different
+# FFTW libraries (serial, MPI and OpenMP) were located in different
+# folders, hence the options of specifying the details.
+# See also lines 264 through 282 of `CMakeLists.txt`.
+#
+# If using openmpi you may need to recompile openmpi with the
+# `--enable-mpi1-compatibility` flag, in order for HDF5 to compile and
+# link.
+
+# Turtle installation directory
+export TURTLE_ROOT=<TURTLE_DIR>
+# (library will go under ${TURTLE_ROOT}/lib, headers under ${TURTLE_ROOT}/include, etc.)
+
+# location-dependent variables
+export CUSTOM_INSTALL_PATH=${TURTLE_ROOT}
+export PATH=${CUSTOM_INSTALL_PATH}/bin:${PATH}
+export CMAKE_PREFIX_PATH=${CUSTOM_INSTALL_PATH}/lib
+
+# activate python virtual environment
+source ${TURTLE_ROOT}/bin/activate
\ No newline at end of file
diff --git a/example.bashrc b/example.bashrc
deleted file mode 100644
index 2eebaef706534756d71a2f8989ec9d8d9febb451..0000000000000000000000000000000000000000
--- a/example.bashrc
+++ /dev/null
@@ -1,17 +0,0 @@
-#! /bin/bash
-export CC=<please insert appropriate C compiler executable here>
-export CXX=<please insert appropriate C++ compiler executable here>
-export MPI_ROOT=<please insert appropriate path here>
-export CUSTOM_INSTALL_PATH=<please insert your prefered path here. We recommend the location of your python virtual environment>
-export PATH=${CUSTOM_INSTALL_PATH}/bin:${PATH}
-export FFTW_DIR=${CUSTOM_INSTALL_PATH}
-export FFTW_BASE=${FFTW_DIR}
-export FFTW_OPENMP_LIB="-L${FFTW_BASE}/lib -lfftw3_omp -lfftw3f_omp"
-export FFTW_LIB="-L${FFTW_BASE}/lib -lfftw3_mpi -lfftw3 -lfftw3f_mpi -lfftw3f"
-export HDF5_DIR=${CUSTOM_INSTALL_PATH}
-export HDF5_BASE=${HDF5_DIR}
-export HDF5_ROOT=${HDF5_DIR}
-export CMAKE_PREFIX_PATH=${CUSTOM_INSTALL_PATH}/lib
-
-#export TURTLE_COMPILATION_FLAGS="-O3 -mtune=native -ffast-math"
-export TURTLE_COMPILATION_FLAGS=""
diff --git a/pc_host_info.py b/pc_host_info.py
index 8e66eb3535c0d13f12d1434f5407fb068f5683fc..e955be0c6adb0807083b895333e300c719a16786 100644
--- a/pc_host_info.py
+++ b/pc_host_info.py
@@ -65,6 +65,8 @@ host_info = {'type' : 'pc'}
# info_template_extra_slurm_lines, relevant for 'SLURM' clusters,
# is a list of strings, written as extra options within the SLURM file.
# for example something like `#SBATCH --get-user-env`
+# Here, also the virtual environment and environment variables need to be loaded,
+# e.g. by `source <TURTLE_DIR>/lib/bash_setup_for_TurTLE.sh`
# info_template_explicit_slurm_environment, relevant for `SLURM` clusters,
# is a bool specifying whether the environment must be mentioned explicitly