Merge branch 'feature/readme' into 'develop'

Feature/readme

See merge request !38

.gitlab-ci.yml

@@ -32,7 +32,29 @@ image: gitlab-registry.mpcdf.mpg.de/mpcdf/module-image
     export CC=icc
     export CXX=icpc
 
-build-gcc:
+#.allow_docker_run_as_root: &allow_docker_run_as_root |
+#    export OMPI_ALLOW_RUN_AS_ROOT=1
+#    export OMPI_ALLOW_RUN_AS_ROOT_CONFIRM=1
+
+build-gcc-openmpi:
+    stage: build
+    script:
+      - *load_modules
+      - *export_GCC_compilers
+      - *build
+    variables:
+      COMPILER: "gcc"
+      MPI: "openmpi"
+      MPICXX: "mpicxx"
+    tags:
+      - docker
+    artifacts:
+        paths:
+            - build/
+        expire_in: 1 hour
+        when: always
+
+build-gcc-impi:
     stage: build
     script:
       - *load_modules
@@ -72,7 +94,7 @@ build-intel:
         expire_in: 1 hour
         when: always
 
-test-gcc:
+test-gcc-impi:
     stage: test
     script:
       - *load_modules
@@ -87,9 +109,26 @@ test-gcc:
       MPI: "impi"
       MPICXX: "mpigxx"
     needs:
-        - job: build-gcc
+        - job: build-gcc-impi
           artifacts: true
 
+#test-gcc-openmpi:
+#    stage: test
+#    script:
+#      - *load_modules
+#      - *export_GCC_compilers
+#      - *allow_docker_run_as_root
+#      - *run_tests
+#    tags:
+#      - docker
+#    variables:
+#      COMPILER: "gcc"
+#      MPI: "openmpi"
+#      MPICXX: "mpicxx"
+#    needs:
+#        - job: build-gcc-openmpi
+#          artifacts: true
+
 test-intel:
     stage: test
     script:

README.rst

@@ -47,8 +47,10 @@ described at http://TurTLE.pages.mpcdf.de/turtle/html
 Authors
 -------
 
-TurTLE is developed and maintained by the Wilczek group at the Max Planck Institute for Dynamics and Self-Organization and the University of Bayreuth in collaboration with the Application Support Group of the
-Max Planck Computing and Data Facility.
+TurTLE is developed and maintained by the Wilczek group at the Max Planck
+Institute for Dynamics and Self-Organization and the University of Bayreuth
+in collaboration with the Application Support Group of the Max Planck
+Computing and Data Facility.
 
 TurTLE contains contributions from:
 
@@ -56,18 +58,13 @@ TurTLE contains contributions from:
 
 .. _sec-installation:
 
-------------
-Installation
-------------
-
-TurTLE can be used on various machines, with laptops routinely being
-used for development and testing, but large production runs using tens
-of thousands of CPU cores on large computing clusters.
-
-**Postprocessing only**
+---------------------------------------------------
+Installation for postprocessing only
+---------------------------------------------------
 
 The Python 3 package may be installed directly if only post-processing
-of existing data is desired:
+of existing data is desired. Simply clone the repository and install by
+executing
 
 .. code:: bash
 
@@ -77,7 +74,13 @@ of existing data is desired:
 (add `--user` or `sudo` as appropriate).
 `setup.py` uses the `setuptools` package for dependency resolution.
 
-**Full installation**
+----------------------
+Full installation
+----------------------
+
+TurTLE can be used on various machines, with laptops routinely being
+used for development and testing, but large production runs using tens
+of thousands of CPU cores on large computing clusters.
 
 The C++ library requires a number of dependencies, that `CMake` will
 search for before compilation and installation.
@@ -87,209 +90,187 @@ We provide instructions for local compilation of FFTW and HDF5, because
 default versions packaged with Linux variants are typically inadequately
 configured.
 
-These installation steps assume that you have a working MPI compiler,
+These installation steps assume that you have a working C/C++ MPI compiler,
 properly configured on your system (i.e. the various configure scripts
-are able to find it), as well as the `cmake` tool.
+are able to find it), as well as an installation of Python 3.
 The list is a work in progress, please contact us
 (Cristian.Lalescu@mpcdf.mpg.de) if the procedure fails at any step of the
 process.
 We recommend to first go through the instructions in full, and only
 afterwards starting to execute the individual steps.
 
-*Note*: the HDF5 1.8x library may require MPI standard v1, we haven't tested
-this in detail. Some default OpenMPI3 installations will not work.
-
-Detailed full installation instructions:
-
-1.
-    Optional.
-    Make a directory `PREFIX` on a local fast partition.
-    (Under unix systems, you can probably use `~/.local`).
-
-2.
-    Optional.
-    We recommend the creation of a virtual python3 environment that will be
-    used for installing TurTLE.
-    Please see https://docs.python-guide.org/dev/virtualenvs/.
-    In the following, the install location of this environment is
-    denoted `TurTLE_DIR`.
-    In principle `TurTLE_DIR` should be a subdirectory of the previously
-    created `PREFIX` folder.
-
-3.
-    Export the following environment variables with apropriate values:
-     * `CC`                   - Preferred C compiler
-     * `CXX`                  - Preferred C++ compiler
-     * `MPI_HOME`             - MPI installation directory
-     * `FFTW_DIR`             - Base directory of FFTW
-     * `FFTW_INCDIR`          - Directory containing FFTW header files
-     * `FFTW_LIBDIR`          - Directory containing FFTW library files
-     * `FFTW_OPENMP_LIBDIR`   - Directory containing OpenMP FFTW library files (please define if different from FFTW_LIBDIR)
-     * `FFTW_MPI_LIBDIR`      - Directory containing MPI FFTW library files (please define if different from FFTW_LIBDIR)
-
-    If you do not have FFTW installed already, then download, compile, install FFTW
-    (latest version 3.x from http://www.fftw.org/).
-    Execute the following commands in order, feel free to customize
-    optimisation flags for your own computer (see http://www.fftw.org/fftw3_doc/Installation-on-Unix.html):
+**Creating a virtual environment**
+
+We recommend creating a virtual environment for TurTLE. To do this,
+choose an installation location :code:`<INSTALL_LOC>` on a local fast partition
+(Under unix systems, this could be e.g. `~/.local`) and a name for the
+environment :code:`<VENV_NAME>` (e.g. `turtle-production`).
+To create the virtual environment, execute:
+
+.. code:: bash
+
+    python -m venv <INSTALL_LOC>/<VENV_NAME>
+
+In the following, we refer to the path :code:`<INSTALL_LOC>` /
+:code:`<VENV_NAME>` as :code:`<TURTLE_DIR>`. For more information on virtual
+environments, please see https://docs.python-guide.org/dev/virtualenvs/.
+
+**Installation of requirements**
+
+TurTLE has the following requirements:
+
+- C/C++ MPI compiler of your choice
+
+- Python 3
+
+- FFTW version >= 3.3.4
+
+- HDF5 version >= 1.8. **Important**: must be compiled with the option "--enable-parallel"
+
+- cmake version > 3.12
+
+We will assume you already have a working MPI compiler and an
+installation of Python 3. For the other requirements, we provide
+installation instructions in the following.  We recommend installing
+them on a local fast partition.  In the following, we refer to their
+locations as :code:`<FFTW_DIR>`, :code:`<HDF5_DIR>` and
+:code:`<CMAKE_DIR>`. You may choose to install the requirements into
+your virtual environment. In that case, all of these placeholders are
+equal to :code:`<TURTLE_DIR>`.
+
+- **FFTW**
+    Download latest version from http://www.fftw.org/.
+    To compile and install it in custom location :code:`<FFTW_DIR>`,
+    execute the following commands in order, feel free to customize
+    optimisation flags for your own computer (see
+    http://www.fftw.org/fftw3_doc/Installation-on-Unix.html):
 
     .. code:: bash
 
-        ./configure --prefix=FFTW_DIR --enable-float --enable-sse --enable-mpi --enable-openmp --enable-threads
+        ./configure --prefix=<FFTW_DIR> --enable-float --enable-sse --enable-mpi --enable-openmp --enable-threads
         make
         make install
-        ./configure --prefix=FFTW_DIR  --enable-sse2 --enable-avx512 --enable-mpi --enable-openmp --enable-threads
+        ./configure --prefix=<FFTW_DIR>  --enable-sse2 --enable-avx512 --enable-mpi --enable-openmp --enable-threads
         make
         make install
 
-    In the above instructions, replace `FFTW_DIR` with an appropriate
-    location (for instance it is safe to use the `PREFIX` defined above).
+..
     TurTLE will try to find FFTW using the FindFFTW from the Morse project.
-    If the package is installed in a non standard location, you should
-    setup the environment variables listed above: `FFTW_DIR`, `FFTW_INCDIR`,
-    `FFTW_LIBDIR`, `FFTW_OPEMMP_LIBDIR` and `FFTW_MPI_LIBDIR`.
-
-4.
-    Export the following environment variables with apropriate values:
-     * `CC`                   - Preferred C compiler
-     * `CXX`                  - Preferred C++ compiler
-     * `MPI_HOME`             - MPI installation directory
-     * `HDF5_ROOT`            - Base directory of HDF5
+    If the package is installed in a non-standard location, make sure the
+    corresponding environment variables are properly exported
+    (see step 3 of installation of TurTLE).
 
-    If you do not have HDF5 installed already, then download, compile, install
-    HDF5.
-    We are using parallel I/O, therefore we must use the plain C interface of HDF5:
+- **HDF5**
+    Download HDF5 from https://www.hdfgroup.org/downloads/hdf5/.
+    To compile and install it in custom location :code:`<HDF5_DIR>`,
+    execute the following commands in order:
 
     .. code:: bash
 
-        ./configure --prefix=HDF5_ROOT --enable-parallel
+        ./configure --prefix=<HDF5_DIR> --enable-parallel
         make
         make install
 
+    Since TurTLE is using parallel I/O, HDF5 needs to be compiled with
+    the "--enable-parallel" flag.
+
+..
     TurTLE will try to find HDF5 using the regular FindHDF5, which
     searches system folders, or `HDF5_ROOT`.
 
-5.  TurTLE requires `cmake` version > 3.12, which should be available
-    from your default package manager.
-    If required, download, compile and install cmake, currently
-    available at https://cmake.org/cmake/resources/software.html.
+- **cmake**
+    Check if cmake (version >3.12) is available from your default package manager.
+    If not, then download cmake at https://cmake.org/cmake/resources/software.html.
+    To compile and install it in custom location :code:`<CMAKE_DIR>`,
+    execute the following commands in order:
 
     .. code:: bash
 
-        ./bootstrap --prefix=PREFIX
+        ./bootstrap --prefix=<CMAKE_DIR>
         make
         make install
 
-    The value of `PREFIX` used above is only relevant to later executing
+    The directory `<CMAKE_DIR>` is only relevant to later executing
     the `cmake` binary (which can be found under `${PREFIX}/bin` after
     installation).
 
-6.
-    Clone turtle repository.
+
+**Installation of TurTLE**
+
+1.  Choose a location for the source code, enter it and clone turtle
+    repository by
 
     .. code:: bash
 
         git clone git@gitlab.mpcdf.mpg.de:TurTLE/turtle.git
 
-7.
-    Go into TurTLE repository, execute
+2.  Execute
 
     .. code:: bash
 
+        cd turtle
         mkdir build
         cd build
-        cp ../pc_host_info.py ./host_info.py
 
-    Edit the `host_info.py` file according to the instructions in the file.
-    This is strictly required before installing on a cluster.
-
-8.
-    Optional.
-    *PINCHECK* is available from https://gitlab.mpcdf.mpg.de/khr/pincheck.
-    If you'd like to check whether TurTLE MPI processes and OpenMP
-    threads are pinned properly to hardware threads, you can simply
-    place the pincheck headers under `${PREFIX}/include`.
-
-9.
-    In the build folder, edit the file `bash_setup_for_TurTLE.sh` so that it
-    exports the following environment variables with apropriate values:
-     * `CC`                   - Preferred C compiler
-     * `CXX`                  - Preferred C++ compiler
-     * `MPI_HOME`             - MPI installation directory
-     * `FFTW_DIR`             - Base directory of FFTW
-     * `FFTW_INCDIR`          - Directory containing FFTW header files
-     * `FFTW_LIBDIR`          - Directory containing FFTW library files
-     * `FFTW_OPENMP_LIBDIR`   - Directory containing OpenMP FFTW library files (please define if different from FFTW_LIBDIR)
-     * `FFTW_MPI_LIBDIR`      - Directory containing MPI FFTW library files (please define if different from FFTW_LIBDIR)
-     * `HDF5_ROOT`            - Base directory of HDF5
-     * `PINCHECK_ROOT`        - Directory under which
-       `include/pincheck.hpp` may be found (OPTIONAL)
-     * `TurTLE_DIR`           - Directory under which TurTLE will be installed (same value from step 2).
-       I.e. library will go under `${TurTLE_DIR}/lib`, headers under
-       `${TurTLE_DIR}/include`, etc.
-    This script should also contain the line
-    `source TurTLE_DIR/lib/bash_setup_for_TurTLE.sh`, as well as the
-    line `source TurTLE_DIR/bin/activate`.
-
-    We also recommend that an environment variable `TURTLE_COMPILATION_FLAGS`
-    is defined appropriately.
-    In particular, for clusters of unknown architecture it helps to log into
-    individual nodes and run the following command:
+3.  Copy the file `bash_setup_template.sh` into the build folder:
 
     .. code:: bash
 
-        gcc -march=native -Q --help=target
+        cp ../bash_setup_template.sh ./bash_setup_for_TurTLE.sh
 
-    Note that an example file `example.bashrc` is included in the
-    repository.
-    There is at least one recorded case of a cluster where different
-    FFTW libraries (serial, MPI and OpenMP) were located in different
-    folders, hence the options of specifying the details.
-    See also lines 143 through 160 of `CMakeLists.txt`.
+    This file will set all required environment variables.
+    Please replace all placeholders by their corresponding values.
 
-    *If using openmpi* you may need to recompile openmpi with the
-    `--enable-mpi1-compatibility` flag, in order for HDF5 to compile and
-    link (step 4).
-
-    *Note*: in principle it is possible to add this information to your
+    *Note*: In principle it is possible to add this information to your
     `.bashrc`, but we recommend against it.
 
-10.
-    In the previously created build folder, execute
+
+4.  Copy the file `pc_host_info.py` into the build folder:
+
+    .. code:: bash
+
+        cp ../pc_host_info.py ./host_info.py
+
+    This file contains information about the machine on which TurTLE
+    will run.  On desktop machines, no further steps are required.  On
+    clusters, please edit the `host_info.py` file according to the
+    instructions in the file.
+
+
+5.
+    Finally, we are ready to install TurTLE. Within the build folder, execute
 
     .. code:: bash
 
         source bash_setup_for_TurTLE.sh
-        cmake .. -DCMAKE_INSTALL_PREFIX=TurTLE_DIR
-        make
-        # to get a verbose compilation process, use
-        # VERBOSE=1 make
-        # to use more than one core for the compilation use
-        # make -jN
-        # (where N is the number of available cores)
-        make install
+        cmake .. -DCMAKE_INSTALL_PREFIX=<TURTLE_DIR>
 
-11.
-    Using TurTLE from an external project.
-    TurTLE creates and installs 3 files alongside the C++ headers and
-    library:
+    Then compile TurTLE with :code:`<N>` cores using
 
     .. code:: bash
 
-        TurTLEConfig.cmake
-        TurTLE_EXPORT.cmake
-        TurTLE_EXPORT-noconfig.cmake
-
-    These files are installed under `${TurTLE_DIR}/lib`.
-    In case these files provide incomplete information, it is necessary to update
-    the cmake input config file: `turtle/cmake/TurTLEConfig.cmake.in`.
-
-12.
-    For clusters.
-    In the `extra_slurm_lines` list from `host_info.py` one should add a
-    corresponding `source INSTALL_DIR/lib/bash_setup_for_TurTLE.sh`
-    line (where you should write the explicit install folder instead of
-    `INSTALL_DIR`), such that the appropriate setup is loaded in the job
-    script.
+        make -j<N>
+        make install
+
+Congratulations, you have installed TurTLE! Feel free to checkout the
+`tutorial <http://turtle.pages.mpcdf.de/turtle/sphinx_html/chapters/tutorial.html>`_.
+
+
+**Using TurTLE from an external project**.
+
+TurTLE creates and installs 3 files alongside the C++ headers and
+library:
+
+.. code:: bash
+
+    TurTLEConfig.cmake
+    TurTLE_EXPORT.cmake
+    TurTLE_EXPORT-noconfig.cmake
+
+These files are installed under `${TURTLE_DIR}/lib`.
+In case these files provide incomplete information, it is necessary to update
+the cmake input config file: `turtle/cmake/TurTLEConfig.cmake.in`.
+
 
 **Uninstall**
 

TurTLE/_code.py

@@ -314,8 +314,6 @@ class _code(_base):
         command_atoms = ['mpirun',
                          '-np',
                          '{0}'.format(nb_processes),
-                         '-' + mpirun_environment_set,
-                         'OMP_NUM_THREADS={0}'.format(nb_threads_per_process),
                          './' + self.name,
                          self.simname]
         if self.host_info['type'] == 'cluster':

bash_setup_template.sh

+#! /bin/bash
+
+# Load dependencies here if necessary, e.g. `module load cmake`
+
+# module-dependent variables
+export CC=<CC> # Preferred C compiler
+export CXX=<CXX> # Preferred C++ compiler
+export MPI_HOME=<MPI_HOME> # MPI installation directory
+export FFTW_ROOT=<FFTW_DIR> # Base directory of FFTW
+export FFTW_DIR=<FFTW_DIR> # Base directory of FFTW
+export HDF5_ROOT=<HDF5_DIR> # Base directory of HDF5
+
+# OPTIONAL
+#
+# export PINCHECK_ROOT=<PINCHECK_ROOT>
+#
+# If this variable is exported, you can check whether TurTLE
+# MPI processes and OpenMP threads are pinned properly to hardware threads.
+# It is available from https://gitlab.mpcdf.mpg.de/khr/pincheck.
+# The pincheck headers need to be placed under `<PINCHECK_ROOT>/include`.
+
+# OPTIONAL
+#
+# export TURTLE_COMPILATION_FLAGS=<COMPILATION_FLAGS>
+#
+# We also recommend setting this variable to optimize compilation
+# For clusters of unknown architecture it helps to log into
+# individual nodes and run the following command:
+# gcc -march=native -Q --help=target
+
+# OPTIONAL
+#
+# export FFTW_INCDIR=<FFTW_INCDIR> # Directory containing FFTW header files
+# export FFTW_LIBDIR=<FFTW_LIBDIR> # Directory containing FFTW library files
+# export FFTW_OPENMP_LIBDIR=<FFTW_OMP_LIBDIR> # Directory containing OpenMP FFTW library files (please define if different from FFTW_LIBDIR)
+# export FFTW_MPI_LIBDIR=<FFTW_MPI_LIBDIR> # Directory containing MPI FFTW library files (please define if different from FFTW_LIBDIR)
+#
+# There is at least one recorded case of a cluster where different
+# FFTW libraries (serial, MPI and OpenMP) were located in different
+# folders, hence the options of specifying the details.
+# See also lines 264 through 282 of `CMakeLists.txt`.
+#
+# If using openmpi you may need to recompile openmpi with the
+# `--enable-mpi1-compatibility` flag, in order for HDF5 to compile and
+# link.
+
+# Turtle installation directory
+export TURTLE_ROOT=<TURTLE_DIR>
+# (library will go under ${TURTLE_ROOT}/lib, headers under ${TURTLE_ROOT}/include, etc.)
+
+# location-dependent variables
+export CUSTOM_INSTALL_PATH=${TURTLE_ROOT}
+export PATH=${CUSTOM_INSTALL_PATH}/bin:${PATH}
+export CMAKE_PREFIX_PATH=${CUSTOM_INSTALL_PATH}/lib
+
+# activate python virtual environment
+source ${TURTLE_ROOT}/bin/activate
\ No newline at end of file

example.bashrc

-#! /bin/bash
-export CC=<please insert appropriate C compiler executable here>
-export CXX=<please insert appropriate C++ compiler executable here>
-export MPI_ROOT=<please insert appropriate path here>
-export CUSTOM_INSTALL_PATH=<please insert your prefered path here. We recommend the location of your python virtual environment>
-export PATH=${CUSTOM_INSTALL_PATH}/bin:${PATH}
-export FFTW_DIR=${CUSTOM_INSTALL_PATH}
-export FFTW_BASE=${FFTW_DIR}
-export FFTW_OPENMP_LIB="-L${FFTW_BASE}/lib -lfftw3_omp -lfftw3f_omp"
-export FFTW_LIB="-L${FFTW_BASE}/lib -lfftw3_mpi -lfftw3 -lfftw3f_mpi -lfftw3f"
-export HDF5_DIR=${CUSTOM_INSTALL_PATH}
-export HDF5_BASE=${HDF5_DIR}
-export HDF5_ROOT=${HDF5_DIR}
-export CMAKE_PREFIX_PATH=${CUSTOM_INSTALL_PATH}/lib
-
-#export TURTLE_COMPILATION_FLAGS="-O3 -mtune=native -ffast-math"
-export TURTLE_COMPILATION_FLAGS=""

pc_host_info.py

@@ -65,6 +65,8 @@ host_info = {'type' : 'pc'}
 # info_template_extra_slurm_lines, relevant for 'SLURM' clusters,
 # is a list of strings, written as extra options within the SLURM file.
 # for example something like `#SBATCH --get-user-env`
+# Here, also the virtual environment and environment variables need to be loaded,
+# e.g. by `source <TURTLE_DIR>/lib/bash_setup_for_TurTLE.sh`
 
 # info_template_explicit_slurm_environment, relevant for `SLURM` clusters,
 # is a bool specifying whether the environment must be mentioned explicitly