From aa4ee5e3d925d129da594d4940bfb4e74882da3e Mon Sep 17 00:00:00 2001
From: Lukas Bentkamp <lukas.bentkamp@mailbox.org>
Date: Wed, 22 Dec 2021 16:30:48 +0100
Subject: [PATCH 01/19] venv command
---
README.rst | 7 +++++++
1 file changed, 7 insertions(+)
diff --git a/README.rst b/README.rst
index 017a5bf0..ae05695f 100644
--- a/README.rst
+++ b/README.rst
@@ -101,6 +101,9 @@ this in detail. Some default OpenMPI3 installations will not work.
Detailed full installation instructions:
+***Preparation***
+
+
1.
Optional.
Make a directory `PREFIX` on a local fast partition.
@@ -116,6 +119,10 @@ Detailed full installation instructions:
In principle `TurTLE_DIR` should be a subdirectory of the previously
created `PREFIX` folder.
+ .. code:: bash
+
+ python -m venv TurTLE_DIR
+
3.
Export the following environment variables with apropriate values:
* `MPICXX` - Preferred MPI C++ compiler
--
GitLab
From 48819916cbe38e246f85164d5bec4415b7e36c69 Mon Sep 17 00:00:00 2001
From: Lukas Bentkamp <lukas.bentkamp@mailbox.org>
Date: Wed, 22 Dec 2021 17:07:57 +0100
Subject: [PATCH 02/19] WIP: rearranging README
---
README.rst | 88 ++++++++++++++++++++++++++++--------------------------
1 file changed, 46 insertions(+), 42 deletions(-)
diff --git a/README.rst b/README.rst
index ae05695f..34635548 100644
--- a/README.rst
+++ b/README.rst
@@ -101,40 +101,16 @@ this in detail. Some default OpenMPI3 installations will not work.
Detailed full installation instructions:
-***Preparation***
+**Preparation**
+TurTLE has the following requirements:
+1. FFTW version >= 3.3.4
+2. HDF5 version >= 1.8, *Important*: must be compiled with the option "--enable-parallel"
+3. cmake version > 3.12
-1.
- Optional.
- Make a directory `PREFIX` on a local fast partition.
- (Under unix systems, you can probably use `~/.local`).
-
-2.
- Optional.
- We recommend the creation of a virtual python3 environment that will be
- used for installing TurTLE.
- Please see https://docs.python-guide.org/dev/virtualenvs/.
- In the following, the install location of this environment is
- denoted `TurTLE_DIR`.
- In principle `TurTLE_DIR` should be a subdirectory of the previously
- created `PREFIX` folder.
-
- .. code:: bash
-
- python -m venv TurTLE_DIR
-
-3.
- Export the following environment variables with apropriate values:
- * `MPICXX` - Preferred MPI C++ compiler
- * `FFTW_DIR` - Base directory of FFTW
- * `FFTW_INCDIR` - Directory containing FFTW header files
- * `FFTW_LIBDIR` - Directory containing FFTW library files
- * `FFTW_OPENMP_LIBDIR` - Directory containing OpenMP FFTW library files (please define if different from FFTW_LIBDIR)
- * `FFTW_MPI_LIBDIR` - Directory containing MPI FFTW library files (please define if different from FFTW_LIBDIR)
-
- If you do not have FFTW installed already, then download, compile, install FFTW
- (latest version 3.x from http://www.fftw.org/).
- Execute the following commands in order, feel free to customize
+1. *Installation of FFTW*
+ Download latest version from http://www.fftw.org/.
+ To compile and install in custom location FFTW_DIR, execute the following commands in order, feel free to customize
optimisation flags for your own computer (see http://www.fftw.org/fftw3_doc/Installation-on-Unix.html):
.. code:: bash
@@ -145,22 +121,15 @@ Detailed full installation instructions:
./configure --prefix=FFTW_DIR --enable-sse2 --enable-avx512 --enable-mpi --enable-openmp --enable-threads
make
make install
-
- In the above instructions, replace `FFTW_DIR` with an appropriate
- location (for instance it is safe to use the `PREFIX` defined above).
+..
TurTLE will try to find FFTW using the FindFFTW from the Morse project.
If the package is installed in a non standard location, you should
setup the environment variables listed above: `FFTW_DIR`, `FFTW_INCDIR`,
`FFTW_LIBDIR`, `FFTW_OPEMMP_LIBDIR` and `FFTW_MPI_LIBDIR`.
-4.
- Export the following environment variables with apropriate values:
- * `MPICXX` - Preferred MPI C++ compiler
- * `HDF5_ROOT` - Base directory of HDF5
+2. *Installation of HDF5*
- If you do not have HDF5 installed already, then download, compile, install
- HDF5.
- We are using parallel I/O, therefore we must use the plain C interface of HDF5:
+ Download HDF5. To compile and install, execute the following commands in order:
.. code:: bash
@@ -168,6 +137,9 @@ Detailed full installation instructions:
make
make install
+
+ Since TurTLE is using parallel I/O, HDF5 needs to be compiled with the "--enable-parallel" flag.
+..
TurTLE will try to find HDF5 using the regular FindHDF5, which
searches system folders, or `HDF5_ROOT`.
@@ -186,6 +158,38 @@ Detailed full installation instructions:
the `cmake` binary (which can be found under `${PREFIX}/bin` after
installation).
+
+ Export the following environment variables with apropriate values:
+ * `MPICXX` - Preferred MPI C++ compiler
+ * `FFTW_DIR` - Base directory of FFTW
+ * `FFTW_INCDIR` - Directory containing FFTW header files
+ * `FFTW_LIBDIR` - Directory containing FFTW library files
+ * `FFTW_OPENMP_LIBDIR` - Directory containing OpenMP FFTW library files (please define if different from FFTW_LIBDIR)
+ * `FFTW_MPI_LIBDIR` - Directory containing MPI FFTW library files (please define if different from FFTW_LIBDIR)
+
+ Export the following environment variables with apropriate values:
+ * `MPICXX` - Preferred MPI C++ compiler
+ * `HDF5_ROOT` - Base directory of HDF5
+
+**Installation**
+
+Make a directory `PREFIX` on a local fast partition.
+ (Under unix systems, you can probably use `~/.local`).
+
+2.
+ Optional.
+ We recommend the creation of a virtual python3 environment that will be
+ used for installing TurTLE.
+ Please see https://docs.python-guide.org/dev/virtualenvs/.
+ In the following, the install location of this environment is
+ denoted `TurTLE_DIR`.
+ In principle `TurTLE_DIR` should be a subdirectory of the previously
+ created `PREFIX` folder.
+
+ .. code:: bash
+
+ python -m venv TurTLE_DIR
+
6.
Clone turtle repository.
--
GitLab
From 3ba082ba47c86996138e3ed1d3b4767514976b59 Mon Sep 17 00:00:00 2001
From: Lukas Bentkamp <lukas.bentkamp@mailbox.org>
Date: Wed, 22 Dec 2021 17:11:39 +0100
Subject: [PATCH 03/19] WIP: rearranging README
---
README.rst | 23 ++++++++++++-----------
1 file changed, 12 insertions(+), 11 deletions(-)
diff --git a/README.rst b/README.rst
index 34635548..a303b2ac 100644
--- a/README.rst
+++ b/README.rst
@@ -104,11 +104,14 @@ Detailed full installation instructions:
**Preparation**
TurTLE has the following requirements:
+
1. FFTW version >= 3.3.4
-2. HDF5 version >= 1.8, *Important*: must be compiled with the option "--enable-parallel"
+
+2. HDF5 version >= 1.8, **Important**: must be compiled with the option "--enable-parallel"
+
3. cmake version > 3.12
-1. *Installation of FFTW*
+1. **Installation of FFTW**
Download latest version from http://www.fftw.org/.
To compile and install in custom location FFTW_DIR, execute the following commands in order, feel free to customize
optimisation flags for your own computer (see http://www.fftw.org/fftw3_doc/Installation-on-Unix.html):
@@ -127,9 +130,8 @@ TurTLE has the following requirements:
setup the environment variables listed above: `FFTW_DIR`, `FFTW_INCDIR`,
`FFTW_LIBDIR`, `FFTW_OPEMMP_LIBDIR` and `FFTW_MPI_LIBDIR`.
-2. *Installation of HDF5*
-
- Download HDF5. To compile and install, execute the following commands in order:
+2. **Installation of HDF5**
+ Download HDF5. To compile and install in custom location HDF5_ROOT, execute the following commands in order:
.. code:: bash
@@ -137,23 +139,22 @@ TurTLE has the following requirements:
make
make install
-
Since TurTLE is using parallel I/O, HDF5 needs to be compiled with the "--enable-parallel" flag.
..
TurTLE will try to find HDF5 using the regular FindHDF5, which
searches system folders, or `HDF5_ROOT`.
-5. TurTLE requires `cmake` version > 3.12, which should be available
+5. **Installation of cmake**
+ TurTLE requires `cmake` version > 3.12, which should be available
from your default package manager.
- If required, download, compile and install cmake, currently
- available at https://cmake.org/cmake/resources/software.html.
+ If not possible, then download cmake at https://cmake.org/cmake/resources/software.html. To compile and install in custom location CMAKE_DIR, execute the following commands in order:
.. code:: bash
- ./bootstrap --prefix=PREFIX
+ ./bootstrap --prefix=CMAKE_DIR
make
make install
-
+..
The value of `PREFIX` used above is only relevant to later executing
the `cmake` binary (which can be found under `${PREFIX}/bin` after
installation).
--
GitLab
From 92646f71132a1b0e817fa32f9ad6c2eb829cd40e Mon Sep 17 00:00:00 2001
From: Lukas Bentkamp <lukas.bentkamp@mailbox.org>
Date: Wed, 22 Dec 2021 17:47:35 +0100
Subject: [PATCH 04/19] WIP: making README more readable
---
README.rst | 42 +++++++++++++++++-------------------------
1 file changed, 17 insertions(+), 25 deletions(-)
diff --git a/README.rst b/README.rst
index a303b2ac..e23b228f 100644
--- a/README.rst
+++ b/README.rst
@@ -107,7 +107,7 @@ TurTLE has the following requirements:
1. FFTW version >= 3.3.4
-2. HDF5 version >= 1.8, **Important**: must be compiled with the option "--enable-parallel"
+2. HDF5 version >= 1.8. **Important**: must be compiled with the option "--enable-parallel"
3. cmake version > 3.12
@@ -131,7 +131,7 @@ TurTLE has the following requirements:
`FFTW_LIBDIR`, `FFTW_OPEMMP_LIBDIR` and `FFTW_MPI_LIBDIR`.
2. **Installation of HDF5**
- Download HDF5. To compile and install in custom location HDF5_ROOT, execute the following commands in order:
+ Download HDF5 from https://www.hdfgroup.org/downloads/hdf5/. To compile and install in custom location HDF5_ROOT, execute the following commands in order:
.. code:: bash
@@ -144,7 +144,7 @@ TurTLE has the following requirements:
TurTLE will try to find HDF5 using the regular FindHDF5, which
searches system folders, or `HDF5_ROOT`.
-5. **Installation of cmake**
+3. **Installation of cmake**
TurTLE requires `cmake` version > 3.12, which should be available
from your default package manager.
If not possible, then download cmake at https://cmake.org/cmake/resources/software.html. To compile and install in custom location CMAKE_DIR, execute the following commands in order:
@@ -174,32 +174,24 @@ TurTLE has the following requirements:
**Installation**
-Make a directory `PREFIX` on a local fast partition.
- (Under unix systems, you can probably use `~/.local`).
-
-2.
- Optional.
- We recommend the creation of a virtual python3 environment that will be
- used for installing TurTLE.
- Please see https://docs.python-guide.org/dev/virtualenvs/.
- In the following, the install location of this environment is
- denoted `TurTLE_DIR`.
- In principle `TurTLE_DIR` should be a subdirectory of the previously
- created `PREFIX` folder.
+1. We recommend creating a virtual environment for TurTLE. To do this,
+ choose an installation location INSTALL_LOC on a local fast partition
+ (Under unix systems, this could be e.g. `~/.local`) and a name for the
+ environment VENV_NAME (e.g. `turtle-production`). To create the virtual environment, execute:
.. code:: bash
- python -m venv TurTLE_DIR
+ python -m venv INSTALL_LOC/VENV_NAME
-6.
- Clone turtle repository.
+ In the following, we refer to the path `INSTALL_LOC/VENV_NAME` with `TurTLE_DIR`.
+
+2. Clone turtle repository.
.. code:: bash
git clone git@gitlab.mpcdf.mpg.de:TurTLE/turtle.git
-7.
- Go into TurTLE repository, execute
+3. Go into TurTLE repository, execute
.. code:: bash
@@ -210,14 +202,14 @@ Make a directory `PREFIX` on a local fast partition.
Edit the `host_info.py` file according to the instructions in the file.
This is strictly required before installing on a cluster.
-8.
+4.
Optional.
*PINCHECK* is available from https://gitlab.mpcdf.mpg.de/khr/pincheck.
If you'd like to check whether TurTLE MPI processes and OpenMP
threads are pinned properly to hardware threads, you can simply
place the pincheck headers under `${PREFIX}/include`.
-9.
+5.
In the build folder, edit the file `bash_setup_for_TurTLE.sh` so that it
exports the following environment variables with apropriate values:
* `MPICXX` - Preferred MPI C++ compiler
@@ -259,7 +251,7 @@ Make a directory `PREFIX` on a local fast partition.
*Note*: in principle it is possible to add this information to your
`.bashrc`, but we recommend against it.
-10.
+6.
In the previously created build folder, execute
.. code:: bash
@@ -274,7 +266,7 @@ Make a directory `PREFIX` on a local fast partition.
# (where N is the number of available cores)
make install
-11.
+7.
Using TurTLE from an external project.
TurTLE creates and installs 3 files alongside the C++ headers and
library:
@@ -289,7 +281,7 @@ Make a directory `PREFIX` on a local fast partition.
In case these files provide incomplete information, it is necessary to update
the cmake input config file: `turtle/cmake/TurTLEConfig.cmake.in`.
-12.
+8.
For clusters.
In the `extra_slurm_lines` list from `host_info.py` one should add a
corresponding `source INSTALL_DIR/lib/bash_setup_for_TurTLE.sh`
--
GitLab
From 80f9952ea93f5d8398a701c3f91e83cc5431db6b Mon Sep 17 00:00:00 2001
From: Lukas Bentkamp <lukas.bentkamp@mailbox.org>
Date: Wed, 22 Dec 2021 17:50:10 +0100
Subject: [PATCH 05/19] WIP: making README more readable
---
README.rst | 18 +++++++++---------
1 file changed, 9 insertions(+), 9 deletions(-)
diff --git a/README.rst b/README.rst
index e23b228f..24081106 100644
--- a/README.rst
+++ b/README.rst
@@ -35,6 +35,10 @@ reducing the time spent on setting up and supervising ensembles of DNS,
with the added benefit of a unified launch-postprocess approach through
the use of the Python wrapper.
+TurTLE can be used on various machines, with laptops routinely being
+used for development and testing, but large production runs using tens
+of thousands of CPU cores on large computing clusters.
+
----------------------------
Background and documentation
----------------------------
@@ -57,15 +61,9 @@ TurTLE contains contributions from:
.. _sec-installation:
------------
-Installation
+Installation for postprocessing only
------------
-TurTLE can be used on various machines, with laptops routinely being
-used for development and testing, but large production runs using tens
-of thousands of CPU cores on large computing clusters.
-
-**Postprocessing only**
-
The Python 3 package may be installed directly if only post-processing
of existing data is desired:
@@ -77,7 +75,9 @@ of existing data is desired:
(add `--user` or `sudo` as appropriate).
`setup.py` uses the `setuptools` package for dependency resolution.
-**Full installation**
+------------
+Full installation
+------------
The C++ library requires a number of dependencies, that `CMake` will
search for before compilation and installation.
@@ -183,7 +183,7 @@ TurTLE has the following requirements:
python -m venv INSTALL_LOC/VENV_NAME
- In the following, we refer to the path `INSTALL_LOC/VENV_NAME` with `TurTLE_DIR`.
+ In the following, we refer to the path `INSTALL_LOC/VENV_NAME` as `TurTLE_DIR`.
2. Clone turtle repository.
--
GitLab
From 2ae046463cc85ef721733b87c37d8bfb836cda4a Mon Sep 17 00:00:00 2001
From: Lukas Bentkamp <lukas.bentkamp@uni-bayreuth.de>
Date: Mon, 24 Jan 2022 14:52:58 +0100
Subject: [PATCH 06/19] WIP: installation instructions
---
README.rst | 168 ++++++++++++++++++++++++++++--------------------
pc_host_info.py | 2 +
2 files changed, 100 insertions(+), 70 deletions(-)
diff --git a/README.rst b/README.rst
index 24081106..7bb74b35 100644
--- a/README.rst
+++ b/README.rst
@@ -60,9 +60,9 @@ TurTLE contains contributions from:
.. _sec-installation:
-------------
+---------------------------------------------------
Installation for postprocessing only
-------------
+---------------------------------------------------
The Python 3 package may be installed directly if only post-processing
of existing data is desired:
@@ -75,9 +75,9 @@ of existing data is desired:
(add `--user` or `sudo` as appropriate).
`setup.py` uses the `setuptools` package for dependency resolution.
-------------
+----------------------
Full installation
-------------
+----------------------
The C++ library requires a number of dependencies, that `CMake` will
search for before compilation and installation.
@@ -87,9 +87,9 @@ We provide instructions for local compilation of FFTW and HDF5, because
default versions packaged with Linux variants are typically inadequately
configured.
-These installation steps assume that you have a working MPI compiler,
+These installation steps assume that you have a working C/C++ MPI compiler,
properly configured on your system (i.e. the various configure scripts
-are able to find it), as well as the `cmake` tool.
+are able to find it), as well as an installation of Python 3.
The list is a work in progress, please contact us
(Cristian.Lalescu@mpcdf.mpg.de) if the procedure fails at any step of the
process.
@@ -101,7 +101,7 @@ this in detail. Some default OpenMPI3 installations will not work.
Detailed full installation instructions:
-**Preparation**
+**Installation of requirements**
TurTLE has the following requirements:
@@ -111,9 +111,11 @@ TurTLE has the following requirements:
3. cmake version > 3.12
+We recommend installing these requirements on a local fast partition. In the following, we refer to their locations as `FFTW_DIR`, `HDF5_DIR` and `CMAKE_DIR`.
+
1. **Installation of FFTW**
Download latest version from http://www.fftw.org/.
- To compile and install in custom location FFTW_DIR, execute the following commands in order, feel free to customize
+ To compile and install it in custom location FFTW_DIR, execute the following commands in order, feel free to customize
optimisation flags for your own computer (see http://www.fftw.org/fftw3_doc/Installation-on-Unix.html):
.. code:: bash
@@ -124,6 +126,7 @@ TurTLE has the following requirements:
./configure --prefix=FFTW_DIR --enable-sse2 --enable-avx512 --enable-mpi --enable-openmp --enable-threads
make
make install
+
..
TurTLE will try to find FFTW using the FindFFTW from the Morse project.
If the package is installed in a non standard location, you should
@@ -131,29 +134,30 @@ TurTLE has the following requirements:
`FFTW_LIBDIR`, `FFTW_OPEMMP_LIBDIR` and `FFTW_MPI_LIBDIR`.
2. **Installation of HDF5**
- Download HDF5 from https://www.hdfgroup.org/downloads/hdf5/. To compile and install in custom location HDF5_ROOT, execute the following commands in order:
+ Download HDF5 from https://www.hdfgroup.org/downloads/hdf5/. To compile and install it in custom location HDF5_DIR, execute the following commands in order:
.. code:: bash
- ./configure --prefix=HDF5_ROOT --enable-parallel
+ ./configure --prefix=HDF5_DIR --enable-parallel
make
make install
Since TurTLE is using parallel I/O, HDF5 needs to be compiled with the "--enable-parallel" flag.
+
..
TurTLE will try to find HDF5 using the regular FindHDF5, which
- searches system folders, or `HDF5_ROOT`.
+ searches system folders, or `HDF5_DIR`.
-3. **Installation of cmake**
- TurTLE requires `cmake` version > 3.12, which should be available
- from your default package manager.
- If not possible, then download cmake at https://cmake.org/cmake/resources/software.html. To compile and install in custom location CMAKE_DIR, execute the following commands in order:
+3. **Installation of cmake**
+ Check if cmake (version >3.12) is available from your default package manager.
+ If not, then download cmake at https://cmake.org/cmake/resources/software.html. To compile and install it in custom location CMAKE_DIR, execute the following commands in order:
.. code:: bash
./bootstrap --prefix=CMAKE_DIR
make
make install
+
..
The value of `PREFIX` used above is only relevant to later executing
the `cmake` binary (which can be found under `${PREFIX}/bin` after
@@ -172,62 +176,108 @@ TurTLE has the following requirements:
* `MPICXX` - Preferred MPI C++ compiler
* `HDF5_ROOT` - Base directory of HDF5
-**Installation**
+**Installation of TurTLE**
1. We recommend creating a virtual environment for TurTLE. To do this,
choose an installation location INSTALL_LOC on a local fast partition
(Under unix systems, this could be e.g. `~/.local`) and a name for the
- environment VENV_NAME (e.g. `turtle-production`). To create the virtual environment, execute:
+ environment VENV_NAME (e.g. `turtle-production`). To cCC=${GCC_HOME}/bin/gcc
+ In the following, we refer to the path `INSTALL_LOC/VENV_NAME` as `TURTLE_DIR`.
+
+2. Choose a location for the source code, enter it and clone turtle repository by
.. code:: bash
- python -m venv INSTALL_LOC/VENV_NAME
+ git clone git@gitlab.mpcdf.mpg.de:TurTLE/turtle.git
+
+3. Execute
- In the following, we refer to the path `INSTALL_LOC/VENV_NAME` as `TurTLE_DIR`.
+ .. code:: bash
+
+ cd turtle
+ mkdir build
+ cd build
-2. Clone turtle repository.
+4. In the build folder, create a file `bash_setup_for_TurTLE.sh`. It will set all required environment variables.
.. code:: bash
- git clone git@gitlab.mpcdf.mpg.de:TurTLE/turtle.git
+ #! /bin/bash
+
+ # module-dependent variables
+ export CC=
+ export CXX=
+ export MPI_HOME=
+ export FFTW_ROOT=FFTW_DIR
+ export FFTW_DIR=FFTW_DIR
+ export HDF5_ROOT=HDF5_DIR
+
+ # custom location for gcc compilation
+ export TURTLE_ROOT=TURTLE_DIR
+
+ # location-dependent variables
+ export CUSTOM_INSTALL_PATH=${TURTLE_ROOT}
+ export PATH=${CUSTOM_INSTALL_PATH}/bin:${PATH}
+ export CMAKE_PREFIX_PATH=${CUSTOM_INSTALL_PATH}/lib
-3. Go into TurTLE repository, execute
+ # activate python virtual environment
+ source ${TURTLE_ROOT}/bin/activate
+
+..
+ * `MPICXX` - Preferred MPI C++ compiler
+ * `FFTW_DIR` - Base directory of FFTW
+ * `FFTW_INCDIR` - Directory containing FFTW header files
+ * `FFTW_LIBDIR` - Directory containing FFTW library files
+ * `FFTW_OPENMP_LIBDIR` - Directory containing OpenMP FFTW library files (please define if different from FFTW_LIBDIR)
+ * `FFTW_MPI_LIBDIR` - Directory containing MPI FFTW library files (please define if different from FFTW_LIBDIR)
+ * `HDF5_ROOT` - Base directory of HDF5
+ * `TURTLE_DIR` - Directory under which TurTLE will be installed (same value from step 2).
+ I.e. library will go under `${TURTLE_DIR}/lib`, headers under
+ `${TURTLE_DIR}/include`, etc.
+
+ This script should also contain the
+ line `source TURTLE_DIR/bin/activate`.
+
+5. Rename the file `pc_host_info.py` in the following way
+
.. code:: bash
- mkdir build
- cd build
cp ../pc_host_info.py ./host_info.py
- Edit the `host_info.py` file according to the instructions in the file.
- This is strictly required before installing on a cluster.
+ This file contains information about the machine on which TurTLE will run.
+ On desktop machines, no further steps are required.
+ On clusters, please edit the `host_info.py` file according to the instructions in the file.
+
+
+6.
+ Finally, we are ready to install TurTLE. Within the build folder, execute
-4.
+ .. code:: bash
+
+ source bash_setup_for_TurTLE.sh
+ cmake .. -DCMAKE_INSTALL_PREFIX=TURTLE_DIR
+
+ Then compile TurTLE with `N` cores using
+
+ .. code:: bash
+
+ make -jN
+ make install
+
+Congratulations, you have installed TurTLE!
+
+**Optional**
+
+5.
Optional.
*PINCHECK* is available from https://gitlab.mpcdf.mpg.de/khr/pincheck.
If you'd like to check whether TurTLE MPI processes and OpenMP
threads are pinned properly to hardware threads, you can simply
place the pincheck headers under `${PREFIX}/include`.
+ * `PINCHECK_ROOT` - Directory under which `include/pincheck.hpp` may be found (OPTIONAL)
-5.
- In the build folder, edit the file `bash_setup_for_TurTLE.sh` so that it
- exports the following environment variables with apropriate values:
- * `MPICXX` - Preferred MPI C++ compiler
- * `FFTW_DIR` - Base directory of FFTW
- * `FFTW_INCDIR` - Directory containing FFTW header files
- * `FFTW_LIBDIR` - Directory containing FFTW library files
- * `FFTW_OPENMP_LIBDIR` - Directory containing OpenMP FFTW library files (please define if different from FFTW_LIBDIR)
- * `FFTW_MPI_LIBDIR` - Directory containing MPI FFTW library files (please define if different from FFTW_LIBDIR)
- * `HDF5_ROOT` - Base directory of HDF5
- * `PINCHECK_ROOT` - Directory under which
- `include/pincheck.hpp` may be found (OPTIONAL)
- * `TurTLE_DIR` - Directory under which TurTLE will be installed (same value from step 2).
- I.e. library will go under `${TurTLE_DIR}/lib`, headers under
- `${TurTLE_DIR}/include`, etc.
- This script should also contain the line
- `source TurTLE_DIR/lib/bash_setup_for_TurTLE.sh`, as well as the
- line `source TurTLE_DIR/bin/activate`.
-
+1.
We also recommend that an environment variable `TURTLE_COMPILATION_FLAGS`
is defined appropriately.
In particular, for clusters of unknown architecture it helps to log into
@@ -251,22 +301,7 @@ TurTLE has the following requirements:
*Note*: in principle it is possible to add this information to your
`.bashrc`, but we recommend against it.
-6.
- In the previously created build folder, execute
-
- .. code:: bash
-
- source bash_setup_for_TurTLE.sh
- cmake .. -DCMAKE_INSTALL_PREFIX=TurTLE_DIR
- make
- # to get a verbose compilation process, use
- # VERBOSE=1 make
- # to use more than one core for the compilation use
- # make -jN
- # (where N is the number of available cores)
- make install
-
-7.
+8.
Using TurTLE from an external project.
TurTLE creates and installs 3 files alongside the C++ headers and
library:
@@ -277,17 +312,10 @@ TurTLE has the following requirements:
TurTLE_EXPORT.cmake
TurTLE_EXPORT-noconfig.cmake
- These files are installed under `${TurTLE_DIR}/lib`.
+ These files are installed under `${TURTLE_DIR}/lib`.
In case these files provide incomplete information, it is necessary to update
the cmake input config file: `turtle/cmake/TurTLEConfig.cmake.in`.
-8.
- For clusters.
- In the `extra_slurm_lines` list from `host_info.py` one should add a
- corresponding `source INSTALL_DIR/lib/bash_setup_for_TurTLE.sh`
- line (where you should write the explicit install folder instead of
- `INSTALL_DIR`), such that the appropriate setup is loaded in the job
- script.
**Uninstall**
diff --git a/pc_host_info.py b/pc_host_info.py
index 8e66eb35..feea61af 100644
--- a/pc_host_info.py
+++ b/pc_host_info.py
@@ -65,6 +65,8 @@ host_info = {'type' : 'pc'}
# info_template_extra_slurm_lines, relevant for 'SLURM' clusters,
# is a list of strings, written as extra options within the SLURM file.
# for example something like `#SBATCH --get-user-env`
+# Here, also the virtual environment and environment variables need to be loaded,
+# e.g. by `source TURTLE_DIR/lib/bash_setup_for_TurTLE.sh`
# info_template_explicit_slurm_environment, relevant for `SLURM` clusters,
# is a bool specifying whether the environment must be mentioned explicitly
--
GitLab
From b697ef608be2b8665e879a3a12b7806b782ab477 Mon Sep 17 00:00:00 2001
From: Lukas Bentkamp <lukas.bentkamp@uni-bayreuth.de>
Date: Thu, 27 Jan 2022 13:47:17 +0100
Subject: [PATCH 07/19] bash setup and optionals
---
README.rst | 166 ++++++++++++-----------------------------
bash_setup_template.sh | 55 ++++++++++++++
2 files changed, 103 insertions(+), 118 deletions(-)
create mode 100644 bash_setup_template.sh
diff --git a/README.rst b/README.rst
index 7bb74b35..06388147 100644
--- a/README.rst
+++ b/README.rst
@@ -98,8 +98,20 @@ afterwards starting to execute the individual steps.
*Note*: the HDF5 1.8x library may require MPI standard v1, we haven't tested
this in detail. Some default OpenMPI3 installations will not work.
+**Is this note deprecated?**
-Detailed full installation instructions:
+**Creating a virtual environment**
+
+We recommend creating a virtual environment for TurTLE. To do this,
+choose an installation location :code:`<INSTALL_LOC>` on a local fast partition
+(Under unix systems, this could be e.g. `~/.local`) and a name for the
+environment :code:`<VENV_NAME>` (e.g. `turtle-production`). To create the virtual environment, execute:
+
+.. code:: bash
+
+ python -m venv <INSTALL_LOC>/<VENV_NAME>
+
+In the following, we refer to the path :code:`<INSTALL_LOC>` / :code:`<VENV_NAME>` as :code:`<TURTLE_DIR>`.
**Installation of requirements**
@@ -111,19 +123,20 @@ TurTLE has the following requirements:
3. cmake version > 3.12
-We recommend installing these requirements on a local fast partition. In the following, we refer to their locations as `FFTW_DIR`, `HDF5_DIR` and `CMAKE_DIR`.
+We recommend installing these requirements on a local fast partition.
+In the following, we refer to their locations as :code:`<FFTW_DIR>`, :code:`<HDF5_DIR>` and :code:`<CMAKE_DIR>`.
-1. **Installation of FFTW**
+1. **FFTW**
Download latest version from http://www.fftw.org/.
- To compile and install it in custom location FFTW_DIR, execute the following commands in order, feel free to customize
+ To compile and install it in custom location :code:`<FFTW_DIR>`, execute the following commands in order, feel free to customize
optimisation flags for your own computer (see http://www.fftw.org/fftw3_doc/Installation-on-Unix.html):
.. code:: bash
- ./configure --prefix=FFTW_DIR --enable-float --enable-sse --enable-mpi --enable-openmp --enable-threads
+ ./configure --prefix=<FFTW_DIR> --enable-float --enable-sse --enable-mpi --enable-openmp --enable-threads
make
make install
- ./configure --prefix=FFTW_DIR --enable-sse2 --enable-avx512 --enable-mpi --enable-openmp --enable-threads
+ ./configure --prefix=<FFTW_DIR> --enable-sse2 --enable-avx512 --enable-mpi --enable-openmp --enable-threads
make
make install
@@ -133,12 +146,13 @@ We recommend installing these requirements on a local fast partition. In the fol
setup the environment variables listed above: `FFTW_DIR`, `FFTW_INCDIR`,
`FFTW_LIBDIR`, `FFTW_OPEMMP_LIBDIR` and `FFTW_MPI_LIBDIR`.
-2. **Installation of HDF5**
- Download HDF5 from https://www.hdfgroup.org/downloads/hdf5/. To compile and install it in custom location HDF5_DIR, execute the following commands in order:
+2. **HDF5**
+ Download HDF5 from https://www.hdfgroup.org/downloads/hdf5/.
+ To compile and install it in custom location :code:`<HDF5_DIR>`, execute the following commands in order:
.. code:: bash
- ./configure --prefix=HDF5_DIR --enable-parallel
+ ./configure --prefix=<HDF5_DIR> --enable-parallel
make
make install
@@ -148,49 +162,27 @@ We recommend installing these requirements on a local fast partition. In the fol
TurTLE will try to find HDF5 using the regular FindHDF5, which
searches system folders, or `HDF5_DIR`.
-3. **Installation of cmake**
+3. **cmake**
Check if cmake (version >3.12) is available from your default package manager.
- If not, then download cmake at https://cmake.org/cmake/resources/software.html. To compile and install it in custom location CMAKE_DIR, execute the following commands in order:
+ If not, then download cmake at https://cmake.org/cmake/resources/software.html.
+ To compile and install it in custom location :code:`<CMAKE_DIR>`, execute the following commands in order:
.. code:: bash
- ./bootstrap --prefix=CMAKE_DIR
+ ./bootstrap --prefix=<CMAKE_DIR>
make
make install
-..
- The value of `PREFIX` used above is only relevant to later executing
- the `cmake` binary (which can be found under `${PREFIX}/bin` after
- installation).
-
-
- Export the following environment variables with apropriate values:
- * `MPICXX` - Preferred MPI C++ compiler
- * `FFTW_DIR` - Base directory of FFTW
- * `FFTW_INCDIR` - Directory containing FFTW header files
- * `FFTW_LIBDIR` - Directory containing FFTW library files
- * `FFTW_OPENMP_LIBDIR` - Directory containing OpenMP FFTW library files (please define if different from FFTW_LIBDIR)
- * `FFTW_MPI_LIBDIR` - Directory containing MPI FFTW library files (please define if different from FFTW_LIBDIR)
-
- Export the following environment variables with apropriate values:
- * `MPICXX` - Preferred MPI C++ compiler
- * `HDF5_ROOT` - Base directory of HDF5
**Installation of TurTLE**
-1. We recommend creating a virtual environment for TurTLE. To do this,
- choose an installation location INSTALL_LOC on a local fast partition
- (Under unix systems, this could be e.g. `~/.local`) and a name for the
- environment VENV_NAME (e.g. `turtle-production`). To cCC=${GCC_HOME}/bin/gcc
- In the following, we refer to the path `INSTALL_LOC/VENV_NAME` as `TURTLE_DIR`.
-
-2. Choose a location for the source code, enter it and clone turtle repository by
+1. Choose a location for the source code, enter it and clone turtle repository by
.. code:: bash
git clone git@gitlab.mpcdf.mpg.de:TurTLE/turtle.git
-3. Execute
+2. Execute
.. code:: bash
@@ -198,48 +190,19 @@ We recommend installing these requirements on a local fast partition. In the fol
mkdir build
cd build
-4. In the build folder, create a file `bash_setup_for_TurTLE.sh`. It will set all required environment variables.
+3. Copy the file `bash_setup_template.sh` into the build folder:
.. code:: bash
- #! /bin/bash
+ cp ../bash_setup_template.sh ./bash_setup_for_TurTLE.sh
- # module-dependent variables
- export CC=
- export CXX=
- export MPI_HOME=
- export FFTW_ROOT=FFTW_DIR
- export FFTW_DIR=FFTW_DIR
- export HDF5_ROOT=HDF5_DIR
+ This file is supposed to set all required environment variables. Please replace all placeholders by their corresponding values.
- # custom location for gcc compilation
- export TURTLE_ROOT=TURTLE_DIR
-
- # location-dependent variables
- export CUSTOM_INSTALL_PATH=${TURTLE_ROOT}
- export PATH=${CUSTOM_INSTALL_PATH}/bin:${PATH}
- export CMAKE_PREFIX_PATH=${CUSTOM_INSTALL_PATH}/lib
-
- # activate python virtual environment
- source ${TURTLE_ROOT}/bin/activate
+ *Note*: In principle it is possible to add this information to your
+ `.bashrc`, but we recommend against it.
-..
- * `MPICXX` - Preferred MPI C++ compiler
- * `FFTW_DIR` - Base directory of FFTW
- * `FFTW_INCDIR` - Directory containing FFTW header files
- * `FFTW_LIBDIR` - Directory containing FFTW library files
- * `FFTW_OPENMP_LIBDIR` - Directory containing OpenMP FFTW library files (please define if different from FFTW_LIBDIR)
- * `FFTW_MPI_LIBDIR` - Directory containing MPI FFTW library files (please define if different from FFTW_LIBDIR)
- * `HDF5_ROOT` - Base directory of HDF5
- * `TURTLE_DIR` - Directory under which TurTLE will be installed (same value from step 2).
- I.e. library will go under `${TURTLE_DIR}/lib`, headers under
- `${TURTLE_DIR}/include`, etc.
-
- This script should also contain the
- line `source TURTLE_DIR/bin/activate`.
-
-5. Rename the file `pc_host_info.py` in the following way
+4. Copy the file `pc_host_info.py` in the following way
.. code:: bash
@@ -250,7 +213,7 @@ We recommend installing these requirements on a local fast partition. In the fol
On clusters, please edit the `host_info.py` file according to the instructions in the file.
-6.
+5.
Finally, we are ready to install TurTLE. Within the build folder, execute
.. code:: bash
@@ -258,63 +221,30 @@ We recommend installing these requirements on a local fast partition. In the fol
source bash_setup_for_TurTLE.sh
cmake .. -DCMAKE_INSTALL_PREFIX=TURTLE_DIR
- Then compile TurTLE with `N` cores using
+ Then compile TurTLE with :code:`<N>` cores using
.. code:: bash
- make -jN
+ make -j<N>
make install
Congratulations, you have installed TurTLE!
-**Optional**
-
-5.
- Optional.
- *PINCHECK* is available from https://gitlab.mpcdf.mpg.de/khr/pincheck.
- If you'd like to check whether TurTLE MPI processes and OpenMP
- threads are pinned properly to hardware threads, you can simply
- place the pincheck headers under `${PREFIX}/include`.
- * `PINCHECK_ROOT` - Directory under which `include/pincheck.hpp` may be found (OPTIONAL)
-
-1.
- We also recommend that an environment variable `TURTLE_COMPILATION_FLAGS`
- is defined appropriately.
- In particular, for clusters of unknown architecture it helps to log into
- individual nodes and run the following command:
-
- .. code:: bash
-
- gcc -march=native -Q --help=target
-
- Note that an example file `example.bashrc` is included in the
- repository.
- There is at least one recorded case of a cluster where different
- FFTW libraries (serial, MPI and OpenMP) were located in different
- folders, hence the options of specifying the details.
- See also lines 143 through 160 of `CMakeLists.txt`.
-
- *If using openmpi* you may need to recompile openmpi with the
- `--enable-mpi1-compatibility` flag, in order for HDF5 to compile and
- link (step 4).
- *Note*: in principle it is possible to add this information to your
- `.bashrc`, but we recommend against it.
+**Using TurTLE from an external project**.
-8.
- Using TurTLE from an external project.
- TurTLE creates and installs 3 files alongside the C++ headers and
- library:
+TurTLE creates and installs 3 files alongside the C++ headers and
+library:
- .. code:: bash
+.. code:: bash
- TurTLEConfig.cmake
- TurTLE_EXPORT.cmake
- TurTLE_EXPORT-noconfig.cmake
+ TurTLEConfig.cmake
+ TurTLE_EXPORT.cmake
+ TurTLE_EXPORT-noconfig.cmake
- These files are installed under `${TURTLE_DIR}/lib`.
- In case these files provide incomplete information, it is necessary to update
- the cmake input config file: `turtle/cmake/TurTLEConfig.cmake.in`.
+These files are installed under `${TURTLE_DIR}/lib`.
+In case these files provide incomplete information, it is necessary to update
+the cmake input config file: `turtle/cmake/TurTLEConfig.cmake.in`.
**Uninstall**
diff --git a/bash_setup_template.sh b/bash_setup_template.sh
new file mode 100644
index 00000000..9ebd754d
--- /dev/null
+++ b/bash_setup_template.sh
@@ -0,0 +1,55 @@
+#! /bin/bash
+
+# module-dependent variables
+export CC=<CC> # executable of the C compiler
+export CXX=<CXX> # executable of the C++ compiler
+export MPI_HOME=<MPI_HOME> # location of your MPI installation
+export FFTW_ROOT=<FFTW_DIR>
+export FFTW_DIR=<FFTW_DIR>
+export HDF5_ROOT=<HDF5_DIR>
+
+# OPTIONAL
+#
+# export PINCHECK_ROOT=<PINCHECK_ROOT>
+#
+# If this variable is exported, TurTLE can use the pincheck tool to display
+# the pinning of MPI processes and OpenMP to hardware threads.
+# It is available from https://gitlab.mpcdf.mpg.de/khr/pincheck.
+# The pincheck headers need to be placed under `<PINCHECK_ROOT>/include`.
+
+# OPTIONAL
+#
+# export TURTLE_COMPILATION_FLAGS=<COMPILATION_FLAGS>
+#
+# We also recommend setting this variable to optimize compilation
+# For clusters of unknown architecture it helps to log into
+# individual nodes and run the following command:
+# gcc -march=native -Q --help=target
+
+# OPTIONAL
+#
+# export FFTW_INCDIR=<FFTW_DIR>/include
+# export FFTW_LIBDIR=<FFTW_DIR>/lib
+# export FFTW_OPENMP_LIBDIR=<FFTW_DIR>/lib
+# export FFTW_MPI_LIBDIR=<FFTW_DIR>/lib
+#
+# There is at least one recorded case of a cluster where different
+# FFTW libraries (serial, MPI and OpenMP) were located in different
+# folders, hence the options of specifying the details.
+# See also lines 264 through 282 of `CMakeLists.txt`.
+
+# *If using openmpi* you may need to recompile openmpi with the
+# `--enable-mpi1-compatibility` flag, in order for HDF5 to compile and
+# link.
+
+
+# custom location for gcc compilation
+export TURTLE_ROOT=<TURTLE_DIR>
+
+# location-dependent variables
+export CUSTOM_INSTALL_PATH=${TURTLE_ROOT}
+export PATH=${CUSTOM_INSTALL_PATH}/bin:${PATH}
+export CMAKE_PREFIX_PATH=${CUSTOM_INSTALL_PATH}/lib
+
+# activate python virtual environment
+source ${TURTLE_ROOT}/bin/activate
\ No newline at end of file
--
GitLab
From aff39234a5b228d200c6f481bccb3505fc407858 Mon Sep 17 00:00:00 2001
From: Lukas Bentkamp <lukas.bentkamp@uni-bayreuth.de>
Date: Thu, 27 Jan 2022 13:48:29 +0100
Subject: [PATCH 08/19] brackets
---
README.rst | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/README.rst b/README.rst
index 06388147..e0f8e61f 100644
--- a/README.rst
+++ b/README.rst
@@ -219,7 +219,7 @@ In the following, we refer to their locations as :code:`<FFTW_DIR>`, :code:`<HDF
.. code:: bash
source bash_setup_for_TurTLE.sh
- cmake .. -DCMAKE_INSTALL_PREFIX=TURTLE_DIR
+ cmake .. -DCMAKE_INSTALL_PREFIX=<TURTLE_DIR>
Then compile TurTLE with :code:`<N>` cores using
--
GitLab
From d246990db702050c0ea8e3e10ab610aa000dfbc3 Mon Sep 17 00:00:00 2001
From: Lukas Bentkamp <lukas.bentkamp@uni-bayreuth.de>
Date: Thu, 27 Jan 2022 13:51:35 +0100
Subject: [PATCH 09/19] copy host_info
---
README.rst | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/README.rst b/README.rst
index e0f8e61f..d6712ba5 100644
--- a/README.rst
+++ b/README.rst
@@ -202,7 +202,7 @@ In the following, we refer to their locations as :code:`<FFTW_DIR>`, :code:`<HDF
`.bashrc`, but we recommend against it.
-4. Copy the file `pc_host_info.py` in the following way
+4. Copy the file `pc_host_info.py` into the build folder:
.. code:: bash
--
GitLab
From eb1b1391be74480237b30ff8bca20e082c105ef9 Mon Sep 17 00:00:00 2001
From: Lukas Bentkamp <lukas.bentkamp@uni-bayreuth.de>
Date: Fri, 28 Jan 2022 15:45:39 +0100
Subject: [PATCH 10/19] minor adjustments
---
README.rst | 15 ++++++++-------
bash_setup_template.sh | 34 ++++++++++++++++++----------------
2 files changed, 26 insertions(+), 23 deletions(-)
diff --git a/README.rst b/README.rst
index d6712ba5..2be76aa8 100644
--- a/README.rst
+++ b/README.rst
@@ -35,10 +35,6 @@ reducing the time spent on setting up and supervising ensembles of DNS,
with the added benefit of a unified launch-postprocess approach through
the use of the Python wrapper.
-TurTLE can be used on various machines, with laptops routinely being
-used for development and testing, but large production runs using tens
-of thousands of CPU cores on large computing clusters.
-
----------------------------
Background and documentation
----------------------------
@@ -65,7 +61,7 @@ Installation for postprocessing only
---------------------------------------------------
The Python 3 package may be installed directly if only post-processing
-of existing data is desired:
+of existing data is desired. Simply clone the repository and install by executing
.. code:: bash
@@ -79,6 +75,10 @@ of existing data is desired:
Full installation
----------------------
+TurTLE can be used on various machines, with laptops routinely being
+used for development and testing, but large production runs using tens
+of thousands of CPU cores on large computing clusters.
+
The C++ library requires a number of dependencies, that `CMake` will
search for before compilation and installation.
In brief, an MPI compiler is required, as well as the HDF5 C library
@@ -124,7 +124,7 @@ TurTLE has the following requirements:
3. cmake version > 3.12
We recommend installing these requirements on a local fast partition.
-In the following, we refer to their locations as :code:`<FFTW_DIR>`, :code:`<HDF5_DIR>` and :code:`<CMAKE_DIR>`.
+In the following, we refer to their locations as :code:`<FFTW_DIR>`, :code:`<HDF5_DIR>` and :code:`<CMAKE_DIR>`. You may choose to install the requirements into your virtual environment. In that case, all of these placeholders are equal to :code"`<TURTLE_DIR>`.
1. **FFTW**
Download latest version from http://www.fftw.org/.
@@ -228,7 +228,8 @@ In the following, we refer to their locations as :code:`<FFTW_DIR>`, :code:`<HDF
make -j<N>
make install
-Congratulations, you have installed TurTLE!
+Congratulations, you have installed TurTLE! Feel free to checkout the
+`tutorial <http://turtle.pages.mpcdf.de/turtle/sphinx_html/chapters/tutorial.html>`_.
**Using TurTLE from an external project**.
diff --git a/bash_setup_template.sh b/bash_setup_template.sh
index 9ebd754d..3f19fb62 100644
--- a/bash_setup_template.sh
+++ b/bash_setup_template.sh
@@ -1,19 +1,21 @@
#! /bin/bash
+# Load dependencies here if necessary, e.g. `module load cmake`
+
# module-dependent variables
-export CC=<CC> # executable of the C compiler
-export CXX=<CXX> # executable of the C++ compiler
-export MPI_HOME=<MPI_HOME> # location of your MPI installation
-export FFTW_ROOT=<FFTW_DIR>
-export FFTW_DIR=<FFTW_DIR>
-export HDF5_ROOT=<HDF5_DIR>
+export CC=<CC> # Preferred C compiler
+export CXX=<CXX> # Preferred C++ compiler
+export MPI_HOME=<MPI_HOME> # MPI installation directory
+export FFTW_ROOT=<FFTW_DIR> # Base directory of FFTW
+export FFTW_DIR=<FFTW_DIR> # Base directory of FFTW
+export HDF5_ROOT=<HDF5_DIR> # Base directory of HDF5
# OPTIONAL
#
# export PINCHECK_ROOT=<PINCHECK_ROOT>
#
-# If this variable is exported, TurTLE can use the pincheck tool to display
-# the pinning of MPI processes and OpenMP to hardware threads.
+# If this variable is exported, you can check whether TurTLE
+# MPI processes and OpenMP threads are pinned properly to hardware threads.
# It is available from https://gitlab.mpcdf.mpg.de/khr/pincheck.
# The pincheck headers need to be placed under `<PINCHECK_ROOT>/include`.
@@ -28,23 +30,23 @@ export HDF5_ROOT=<HDF5_DIR>
# OPTIONAL
#
-# export FFTW_INCDIR=<FFTW_DIR>/include
-# export FFTW_LIBDIR=<FFTW_DIR>/lib
-# export FFTW_OPENMP_LIBDIR=<FFTW_DIR>/lib
-# export FFTW_MPI_LIBDIR=<FFTW_DIR>/lib
+# export FFTW_INCDIR=<FFTW_INCDIR> # Directory containing FFTW header files
+# export FFTW_LIBDIR=<FFTW_LIBDIR> # Directory containing FFTW library files
+# export FFTW_OPENMP_LIBDIR=<FFTW_OMP_LIBDIR> # Directory containing OpenMP FFTW library files (please define if different from FFTW_LIBDIR)
+# export FFTW_MPI_LIBDIR=<FFTW_MPI_LIBDIR> # Directory containing MPI FFTW library files (please define if different from FFTW_LIBDIR)
#
# There is at least one recorded case of a cluster where different
# FFTW libraries (serial, MPI and OpenMP) were located in different
# folders, hence the options of specifying the details.
# See also lines 264 through 282 of `CMakeLists.txt`.
-
-# *If using openmpi* you may need to recompile openmpi with the
+#
+# If using openmpi you may need to recompile openmpi with the
# `--enable-mpi1-compatibility` flag, in order for HDF5 to compile and
# link.
-
-# custom location for gcc compilation
+# Turtle installation directory
export TURTLE_ROOT=<TURTLE_DIR>
+# (library will go under ${TURTLE_ROOT}/lib, headers under ${TURTLE_ROOT}/include, etc.)
# location-dependent variables
export CUSTOM_INSTALL_PATH=${TURTLE_ROOT}
--
GitLab
From b6603feb782e2c854bc79faf5475d426ddd9119b Mon Sep 17 00:00:00 2001
From: Lukas Bentkamp <lukas.bentkamp@uni-bayreuth.de>
Date: Fri, 28 Jan 2022 16:04:17 +0100
Subject: [PATCH 11/19] incorporating develop changes
---
README.rst | 8 ++++++--
pc_host_info.py | 2 +-
2 files changed, 7 insertions(+), 3 deletions(-)
diff --git a/README.rst b/README.rst
index 2be76aa8..06dc6f15 100644
--- a/README.rst
+++ b/README.rst
@@ -160,7 +160,7 @@ In the following, we refer to their locations as :code:`<FFTW_DIR>`, :code:`<HDF
..
TurTLE will try to find HDF5 using the regular FindHDF5, which
- searches system folders, or `HDF5_DIR`.
+ searches system folders, or `HDF5_ROOT`.
3. **cmake**
Check if cmake (version >3.12) is available from your default package manager.
@@ -173,6 +173,10 @@ In the following, we refer to their locations as :code:`<FFTW_DIR>`, :code:`<HDF
make
make install
+ The directory `<CMAKE_DIR>` is only relevant to later executing
+ the `cmake` binary (which can be found under `${PREFIX}/bin` after
+ installation).
+
**Installation of TurTLE**
@@ -196,7 +200,7 @@ In the following, we refer to their locations as :code:`<FFTW_DIR>`, :code:`<HDF
cp ../bash_setup_template.sh ./bash_setup_for_TurTLE.sh
- This file is supposed to set all required environment variables. Please replace all placeholders by their corresponding values.
+ This file will set all required environment variables. Please replace all placeholders by their corresponding values.
*Note*: In principle it is possible to add this information to your
`.bashrc`, but we recommend against it.
diff --git a/pc_host_info.py b/pc_host_info.py
index feea61af..e955be0c 100644
--- a/pc_host_info.py
+++ b/pc_host_info.py
@@ -66,7 +66,7 @@ host_info = {'type' : 'pc'}
# is a list of strings, written as extra options within the SLURM file.
# for example something like `#SBATCH --get-user-env`
# Here, also the virtual environment and environment variables need to be loaded,
-# e.g. by `source TURTLE_DIR/lib/bash_setup_for_TurTLE.sh`
+# e.g. by `source <TURTLE_DIR>/lib/bash_setup_for_TurTLE.sh`
# info_template_explicit_slurm_environment, relevant for `SLURM` clusters,
# is a bool specifying whether the environment must be mentioned explicitly
--
GitLab
From 5ea965de876c46dda6c9393b87e8153ead3953a6 Mon Sep 17 00:00:00 2001
From: Lukas Bentkamp <lukas.bentkamp@uni-bayreuth.de>
Date: Fri, 28 Jan 2022 16:47:32 +0100
Subject: [PATCH 12/19] requirements
---
README.rst | 29 +++++++++++++++++------------
1 file changed, 17 insertions(+), 12 deletions(-)
diff --git a/README.rst b/README.rst
index 06dc6f15..4450c65d 100644
--- a/README.rst
+++ b/README.rst
@@ -117,16 +117,21 @@ In the following, we refer to the path :code:`<INSTALL_LOC>` / :code:`<VENV_NAME
TurTLE has the following requirements:
-1. FFTW version >= 3.3.4
+- C/C++ MPI compiler of your choice
-2. HDF5 version >= 1.8. **Important**: must be compiled with the option "--enable-parallel"
+- Python 3
-3. cmake version > 3.12
+- FFTW version >= 3.3.4
-We recommend installing these requirements on a local fast partition.
-In the following, we refer to their locations as :code:`<FFTW_DIR>`, :code:`<HDF5_DIR>` and :code:`<CMAKE_DIR>`. You may choose to install the requirements into your virtual environment. In that case, all of these placeholders are equal to :code"`<TURTLE_DIR>`.
+- HDF5 version >= 1.8. **Important**: must be compiled with the option "--enable-parallel"
-1. **FFTW**
+- cmake version > 3.12
+
+We will assume you already have a working MPI compiler and an installation of Python 3. For the other requirements, we provide installation instructions in the following.
+We recommend installing them on a local fast partition.
+In the following, we refer to their locations as :code:`<FFTW_DIR>`, :code:`<HDF5_DIR>` and :code:`<CMAKE_DIR>`. You may choose to install the requirements into your virtual environment. In that case, all of these placeholders are equal to :code:`<TURTLE_DIR>`.
+
+- **FFTW**
Download latest version from http://www.fftw.org/.
To compile and install it in custom location :code:`<FFTW_DIR>`, execute the following commands in order, feel free to customize
optimisation flags for your own computer (see http://www.fftw.org/fftw3_doc/Installation-on-Unix.html):
@@ -142,11 +147,11 @@ In the following, we refer to their locations as :code:`<FFTW_DIR>`, :code:`<HDF
..
TurTLE will try to find FFTW using the FindFFTW from the Morse project.
- If the package is installed in a non standard location, you should
- setup the environment variables listed above: `FFTW_DIR`, `FFTW_INCDIR`,
- `FFTW_LIBDIR`, `FFTW_OPEMMP_LIBDIR` and `FFTW_MPI_LIBDIR`.
+ If the package is installed in a non-standard location, make sure the
+ corresponding environment variables are properly exported
+ (see step 3 of installation of TurTLE).
-2. **HDF5**
+- **HDF5**
Download HDF5 from https://www.hdfgroup.org/downloads/hdf5/.
To compile and install it in custom location :code:`<HDF5_DIR>`, execute the following commands in order:
@@ -157,12 +162,12 @@ In the following, we refer to their locations as :code:`<FFTW_DIR>`, :code:`<HDF
make install
Since TurTLE is using parallel I/O, HDF5 needs to be compiled with the "--enable-parallel" flag.
-
+
..
TurTLE will try to find HDF5 using the regular FindHDF5, which
searches system folders, or `HDF5_ROOT`.
-3. **cmake**
+- **cmake**
Check if cmake (version >3.12) is available from your default package manager.
If not, then download cmake at https://cmake.org/cmake/resources/software.html.
To compile and install it in custom location :code:`<CMAKE_DIR>`, execute the following commands in order:
--
GitLab
From 91ce0bd85f2229cd4ac4ec033064cdfaac87a88d Mon Sep 17 00:00:00 2001
From: Cristian C Lalescu <chichilalescu@gmail.com>
Date: Mon, 31 Jan 2022 10:02:36 +0100
Subject: [PATCH 13/19] adds test of gcc-openmpi
---
.gitlab-ci.yml | 40 +++++++++++++++++++++++++++++++++++++---
1 file changed, 37 insertions(+), 3 deletions(-)
diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 60a3fe60..057f28c8 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -32,7 +32,25 @@ image: gitlab-registry.mpcdf.mpg.de/mpcdf/module-image
export CC=icc
export CXX=icpc
-build-gcc:
+build-gcc-openmpi:
+ stage: build
+ script:
+ - *load_modules
+ - *export_GCC_compilers
+ - *build
+ variables:
+ COMPILER: "gcc"
+ MPI: "openmpi"
+ MPICXX: "mpicxx"
+ tags:
+ - docker
+ artifacts:
+ paths:
+ - build/
+ expire_in: 1 hour
+ when: always
+
+build-gcc-impi:
stage: build
script:
- *load_modules
@@ -68,7 +86,7 @@ build-intel:
expire_in: 1 hour
when: always
-test-gcc:
+test-gcc-impi:
stage: test
script:
- *load_modules
@@ -81,7 +99,23 @@ test-gcc:
MPI: "impi"
MPICXX: "mpigxx"
needs:
- - job: build-gcc
+ - job: build-gcc-impi
+ artifacts: true
+
+test-gcc-openmpi:
+ stage: test
+ script:
+ - *load_modules
+ - *export_GCC_compilers
+ - *run_tests
+ tags:
+ - docker
+ variables:
+ COMPILER: "gcc"
+ MPI: "openmpi"
+ MPICXX: "mpicxx"
+ needs:
+ - job: build-gcc-openmpi
artifacts: true
test-intel:
--
GitLab
From bca420a438170f89d057a475856bff5b521a52a6 Mon Sep 17 00:00:00 2001
From: Cristian C Lalescu <chichilalescu@gmail.com>
Date: Mon, 31 Jan 2022 11:09:29 +0100
Subject: [PATCH 14/19] fixes mpirun call
---
README.rst | 4 ----
TurTLE/_code.py | 2 --
2 files changed, 6 deletions(-)
diff --git a/README.rst b/README.rst
index 4450c65d..2415c76c 100644
--- a/README.rst
+++ b/README.rst
@@ -96,10 +96,6 @@ process.
We recommend to first go through the instructions in full, and only
afterwards starting to execute the individual steps.
-*Note*: the HDF5 1.8x library may require MPI standard v1, we haven't tested
-this in detail. Some default OpenMPI3 installations will not work.
-**Is this note deprecated?**
-
**Creating a virtual environment**
We recommend creating a virtual environment for TurTLE. To do this,
diff --git a/TurTLE/_code.py b/TurTLE/_code.py
index 46403e4d..4beefa12 100644
--- a/TurTLE/_code.py
+++ b/TurTLE/_code.py
@@ -314,8 +314,6 @@ class _code(_base):
command_atoms = ['mpirun',
'-np',
'{0}'.format(nb_processes),
- '-' + mpirun_environment_set,
- 'OMP_NUM_THREADS={0}'.format(nb_threads_per_process),
'./' + self.name,
self.simname]
if self.host_info['type'] == 'cluster':
--
GitLab
From 365469909cf66e8476cf8709e44b70053af031a7 Mon Sep 17 00:00:00 2001
From: Cristian C Lalescu <chichilalescu@gmail.com>
Date: Mon, 31 Jan 2022 11:18:18 +0100
Subject: [PATCH 15/19] reformats README
---
README.rst | 60 +++++++++++++++++++++++++++++++++++-------------------
1 file changed, 39 insertions(+), 21 deletions(-)
diff --git a/README.rst b/README.rst
index 2415c76c..be46b537 100644
--- a/README.rst
+++ b/README.rst
@@ -47,8 +47,10 @@ described at http://TurTLE.pages.mpcdf.de/turtle/html
Authors
-------
-TurTLE is developed and maintained by the Wilczek group at the Max Planck Institute for Dynamics and Self-Organization and the University of Bayreuth in collaboration with the Application Support Group of the
-Max Planck Computing and Data Facility.
+TurTLE is developed and maintained by the Wilczek group at the Max Planck
+Institute for Dynamics and Self-Organization and the University of Bayreuth
+in collaboration with the Application Support Group of the Max Planck
+Computing and Data Facility.
TurTLE contains contributions from:
@@ -61,7 +63,8 @@ Installation for postprocessing only
---------------------------------------------------
The Python 3 package may be installed directly if only post-processing
-of existing data is desired. Simply clone the repository and install by executing
+of existing data is desired. Simply clone the repository and install by
+executing
.. code:: bash
@@ -101,13 +104,15 @@ afterwards starting to execute the individual steps.
We recommend creating a virtual environment for TurTLE. To do this,
choose an installation location :code:`<INSTALL_LOC>` on a local fast partition
(Under unix systems, this could be e.g. `~/.local`) and a name for the
-environment :code:`<VENV_NAME>` (e.g. `turtle-production`). To create the virtual environment, execute:
+environment :code:`<VENV_NAME>` (e.g. `turtle-production`).
+To create the virtual environment, execute:
.. code:: bash
python -m venv <INSTALL_LOC>/<VENV_NAME>
-In the following, we refer to the path :code:`<INSTALL_LOC>` / :code:`<VENV_NAME>` as :code:`<TURTLE_DIR>`.
+In the following, we refer to the path :code:`<INSTALL_LOC>` /
+:code:`<VENV_NAME>` as :code:`<TURTLE_DIR>`.
**Installation of requirements**
@@ -123,14 +128,21 @@ TurTLE has the following requirements:
- cmake version > 3.12
-We will assume you already have a working MPI compiler and an installation of Python 3. For the other requirements, we provide installation instructions in the following.
-We recommend installing them on a local fast partition.
-In the following, we refer to their locations as :code:`<FFTW_DIR>`, :code:`<HDF5_DIR>` and :code:`<CMAKE_DIR>`. You may choose to install the requirements into your virtual environment. In that case, all of these placeholders are equal to :code:`<TURTLE_DIR>`.
+We will assume you already have a working MPI compiler and an
+installation of Python 3. For the other requirements, we provide
+installation instructions in the following. We recommend installing
+them on a local fast partition. In the following, we refer to their
+locations as :code:`<FFTW_DIR>`, :code:`<HDF5_DIR>` and
+:code:`<CMAKE_DIR>`. You may choose to install the requirements into
+your virtual environment. In that case, all of these placeholders are
+equal to :code:`<TURTLE_DIR>`.
- **FFTW**
Download latest version from http://www.fftw.org/.
- To compile and install it in custom location :code:`<FFTW_DIR>`, execute the following commands in order, feel free to customize
- optimisation flags for your own computer (see http://www.fftw.org/fftw3_doc/Installation-on-Unix.html):
+ To compile and install it in custom location :code:`<FFTW_DIR>`,
+ execute the following commands in order, feel free to customize
+ optimisation flags for your own computer (see
+ http://www.fftw.org/fftw3_doc/Installation-on-Unix.html):
.. code:: bash
@@ -143,13 +155,14 @@ In the following, we refer to their locations as :code:`<FFTW_DIR>`, :code:`<HDF
..
TurTLE will try to find FFTW using the FindFFTW from the Morse project.
- If the package is installed in a non-standard location, make sure the
+ If the package is installed in a non-standard location, make sure the
corresponding environment variables are properly exported
(see step 3 of installation of TurTLE).
- **HDF5**
Download HDF5 from https://www.hdfgroup.org/downloads/hdf5/.
- To compile and install it in custom location :code:`<HDF5_DIR>`, execute the following commands in order:
+ To compile and install it in custom location :code:`<HDF5_DIR>`,
+ execute the following commands in order:
.. code:: bash
@@ -157,8 +170,9 @@ In the following, we refer to their locations as :code:`<FFTW_DIR>`, :code:`<HDF
make
make install
- Since TurTLE is using parallel I/O, HDF5 needs to be compiled with the "--enable-parallel" flag.
-
+ Since TurTLE is using parallel I/O, HDF5 needs to be compiled with
+ the "--enable-parallel" flag.
+
..
TurTLE will try to find HDF5 using the regular FindHDF5, which
searches system folders, or `HDF5_ROOT`.
@@ -166,7 +180,8 @@ In the following, we refer to their locations as :code:`<FFTW_DIR>`, :code:`<HDF
- **cmake**
Check if cmake (version >3.12) is available from your default package manager.
If not, then download cmake at https://cmake.org/cmake/resources/software.html.
- To compile and install it in custom location :code:`<CMAKE_DIR>`, execute the following commands in order:
+ To compile and install it in custom location :code:`<CMAKE_DIR>`,
+ execute the following commands in order:
.. code:: bash
@@ -181,7 +196,8 @@ In the following, we refer to their locations as :code:`<FFTW_DIR>`, :code:`<HDF
**Installation of TurTLE**
-1. Choose a location for the source code, enter it and clone turtle repository by
+1. Choose a location for the source code, enter it and clone turtle
+ repository by
.. code:: bash
@@ -201,21 +217,23 @@ In the following, we refer to their locations as :code:`<FFTW_DIR>`, :code:`<HDF
cp ../bash_setup_template.sh ./bash_setup_for_TurTLE.sh
- This file will set all required environment variables. Please replace all placeholders by their corresponding values.
+ This file will set all required environment variables.
+ Please replace all placeholders by their corresponding values.
*Note*: In principle it is possible to add this information to your
`.bashrc`, but we recommend against it.
4. Copy the file `pc_host_info.py` into the build folder:
-
+
.. code:: bash
cp ../pc_host_info.py ./host_info.py
- This file contains information about the machine on which TurTLE will run.
- On desktop machines, no further steps are required.
- On clusters, please edit the `host_info.py` file according to the instructions in the file.
+ This file contains information about the machine on which TurTLE
+ will run. On desktop machines, no further steps are required. On
+ clusters, please edit the `host_info.py` file according to the
+ instructions in the file.
5.
--
GitLab
From 9b99e03ed212795dfb47d90f43265448e8b5848f Mon Sep 17 00:00:00 2001
From: Cristian C Lalescu <chichilalescu@gmail.com>
Date: Mon, 31 Jan 2022 12:12:55 +0100
Subject: [PATCH 16/19] fixes gitlab config for docker openmpi
---
.gitlab-ci.yml | 5 +++++
1 file changed, 5 insertions(+)
diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 057f28c8..979af727 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -32,6 +32,10 @@ image: gitlab-registry.mpcdf.mpg.de/mpcdf/module-image
export CC=icc
export CXX=icpc
+.allow_docker_run_as_root: &allow_docker_run_as_root |
+ export OMPI_ALLOW_RUN_AS_ROOT=1
+ export OMPI_ALLOW_RUN_AS_ROOT_CONFIRM=1
+
build-gcc-openmpi:
stage: build
script:
@@ -107,6 +111,7 @@ test-gcc-openmpi:
script:
- *load_modules
- *export_GCC_compilers
+ - *allow_docker_run_as_root
- *run_tests
tags:
- docker
--
GitLab
From 344ba42566f08aa76e68988d639b73dc0e683327 Mon Sep 17 00:00:00 2001
From: Lukas Bentkamp <lukas.bentkamp@uni-bayreuth.de>
Date: Mon, 31 Jan 2022 11:20:13 +0000
Subject: [PATCH 17/19] Update README.rst
---
README.rst | 3 ++-
1 file changed, 2 insertions(+), 1 deletion(-)
diff --git a/README.rst b/README.rst
index be46b537..31de5a79 100644
--- a/README.rst
+++ b/README.rst
@@ -112,7 +112,8 @@ To create the virtual environment, execute:
python -m venv <INSTALL_LOC>/<VENV_NAME>
In the following, we refer to the path :code:`<INSTALL_LOC>` /
-:code:`<VENV_NAME>` as :code:`<TURTLE_DIR>`.
+:code:`<VENV_NAME>` as :code:`<TURTLE_DIR>`. For more information on virtual
+environments, please see https://docs.python-guide.org/dev/virtualenvs/.
**Installation of requirements**
--
GitLab
From c079816da2bc1f72d957124b5eda33f16d73554f Mon Sep 17 00:00:00 2001
From: Cristian C Lalescu <chichilalescu@gmail.com>
Date: Mon, 31 Jan 2022 13:16:02 +0100
Subject: [PATCH 18/19] turns off openmpi test
broken for now due to docker config.
---
.gitlab-ci.yml | 38 +++++++++++++++++++-------------------
1 file changed, 19 insertions(+), 19 deletions(-)
diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 979af727..cd673f92 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -32,9 +32,9 @@ image: gitlab-registry.mpcdf.mpg.de/mpcdf/module-image
export CC=icc
export CXX=icpc
-.allow_docker_run_as_root: &allow_docker_run_as_root |
- export OMPI_ALLOW_RUN_AS_ROOT=1
- export OMPI_ALLOW_RUN_AS_ROOT_CONFIRM=1
+#.allow_docker_run_as_root: &allow_docker_run_as_root |
+# export OMPI_ALLOW_RUN_AS_ROOT=1
+# export OMPI_ALLOW_RUN_AS_ROOT_CONFIRM=1
build-gcc-openmpi:
stage: build
@@ -106,22 +106,22 @@ test-gcc-impi:
- job: build-gcc-impi
artifacts: true
-test-gcc-openmpi:
- stage: test
- script:
- - *load_modules
- - *export_GCC_compilers
- - *allow_docker_run_as_root
- - *run_tests
- tags:
- - docker
- variables:
- COMPILER: "gcc"
- MPI: "openmpi"
- MPICXX: "mpicxx"
- needs:
- - job: build-gcc-openmpi
- artifacts: true
+#test-gcc-openmpi:
+# stage: test
+# script:
+# - *load_modules
+# - *export_GCC_compilers
+# - *allow_docker_run_as_root
+# - *run_tests
+# tags:
+# - docker
+# variables:
+# COMPILER: "gcc"
+# MPI: "openmpi"
+# MPICXX: "mpicxx"
+# needs:
+# - job: build-gcc-openmpi
+# artifacts: true
test-intel:
stage: test
--
GitLab
From 0f72352ce9ecf3812bd031febbb7a448d6261a73 Mon Sep 17 00:00:00 2001
From: Cristian C Lalescu <chichilalescu@gmail.com>
Date: Mon, 31 Jan 2022 13:21:34 +0100
Subject: [PATCH 19/19] removes redundant file
---
example.bashrc | 17 -----------------
1 file changed, 17 deletions(-)
delete mode 100644 example.bashrc
diff --git a/example.bashrc b/example.bashrc
deleted file mode 100644
index 2eebaef7..00000000
--- a/example.bashrc
+++ /dev/null
@@ -1,17 +0,0 @@
-#! /bin/bash
-export CC=<please insert appropriate C compiler executable here>
-export CXX=<please insert appropriate C++ compiler executable here>
-export MPI_ROOT=<please insert appropriate path here>
-export CUSTOM_INSTALL_PATH=<please insert your prefered path here. We recommend the location of your python virtual environment>
-export PATH=${CUSTOM_INSTALL_PATH}/bin:${PATH}
-export FFTW_DIR=${CUSTOM_INSTALL_PATH}
-export FFTW_BASE=${FFTW_DIR}
-export FFTW_OPENMP_LIB="-L${FFTW_BASE}/lib -lfftw3_omp -lfftw3f_omp"
-export FFTW_LIB="-L${FFTW_BASE}/lib -lfftw3_mpi -lfftw3 -lfftw3f_mpi -lfftw3f"
-export HDF5_DIR=${CUSTOM_INSTALL_PATH}
-export HDF5_BASE=${HDF5_DIR}
-export HDF5_ROOT=${HDF5_DIR}
-export CMAKE_PREFIX_PATH=${CUSTOM_INSTALL_PATH}/lib
-
-#export TURTLE_COMPILATION_FLAGS="-O3 -mtune=native -ffast-math"
-export TURTLE_COMPILATION_FLAGS=""
--
GitLab