diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index bf0e4b0b29a98e95df25ee13be948ce60eeecb18..10776184bcdbded0ca7496d9d76663374633efda 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -32,7 +32,29 @@ image: gitlab-registry.mpcdf.mpg.de/mpcdf/module-image export CC=icc export CXX=icpc -build-gcc: +#.allow_docker_run_as_root: &allow_docker_run_as_root | +# export OMPI_ALLOW_RUN_AS_ROOT=1 +# export OMPI_ALLOW_RUN_AS_ROOT_CONFIRM=1 + +build-gcc-openmpi: + stage: build + script: + - *load_modules + - *export_GCC_compilers + - *build + variables: + COMPILER: "gcc" + MPI: "openmpi" + MPICXX: "mpicxx" + tags: + - docker + artifacts: + paths: + - build/ + expire_in: 1 hour + when: always + +build-gcc-impi: stage: build script: - *load_modules @@ -72,7 +94,7 @@ build-intel: expire_in: 1 hour when: always -test-gcc: +test-gcc-impi: stage: test script: - *load_modules @@ -87,9 +109,26 @@ test-gcc: MPI: "impi" MPICXX: "mpigxx" needs: - - job: build-gcc + - job: build-gcc-impi artifacts: true +#test-gcc-openmpi: +# stage: test +# script: +# - *load_modules +# - *export_GCC_compilers +# - *allow_docker_run_as_root +# - *run_tests +# tags: +# - docker +# variables: +# COMPILER: "gcc" +# MPI: "openmpi" +# MPICXX: "mpicxx" +# needs: +# - job: build-gcc-openmpi +# artifacts: true + test-intel: stage: test script: diff --git a/README.rst b/README.rst index 68c73f0c17edd9158b891c6d5ddd13b6533b016b..31de5a79f1436afb8a9bab9c0e0fe6e4d420a784 100644 --- a/README.rst +++ b/README.rst @@ -47,8 +47,10 @@ described at http://TurTLE.pages.mpcdf.de/turtle/html Authors ------- -TurTLE is developed and maintained by the Wilczek group at the Max Planck Institute for Dynamics and Self-Organization and the University of Bayreuth in collaboration with the Application Support Group of the -Max Planck Computing and Data Facility. +TurTLE is developed and maintained by the Wilczek group at the Max Planck +Institute for Dynamics and Self-Organization and the University of Bayreuth +in collaboration with the Application Support Group of the Max Planck +Computing and Data Facility. TurTLE contains contributions from: @@ -56,18 +58,13 @@ TurTLE contains contributions from: .. _sec-installation: ------------- -Installation ------------- - -TurTLE can be used on various machines, with laptops routinely being -used for development and testing, but large production runs using tens -of thousands of CPU cores on large computing clusters. - -**Postprocessing only** +--------------------------------------------------- +Installation for postprocessing only +--------------------------------------------------- The Python 3 package may be installed directly if only post-processing -of existing data is desired: +of existing data is desired. Simply clone the repository and install by +executing .. code:: bash @@ -77,7 +74,13 @@ of existing data is desired: (add `--user` or `sudo` as appropriate). `setup.py` uses the `setuptools` package for dependency resolution. -**Full installation** +---------------------- +Full installation +---------------------- + +TurTLE can be used on various machines, with laptops routinely being +used for development and testing, but large production runs using tens +of thousands of CPU cores on large computing clusters. The C++ library requires a number of dependencies, that `CMake` will search for before compilation and installation. @@ -87,209 +90,187 @@ We provide instructions for local compilation of FFTW and HDF5, because default versions packaged with Linux variants are typically inadequately configured. -These installation steps assume that you have a working MPI compiler, +These installation steps assume that you have a working C/C++ MPI compiler, properly configured on your system (i.e. the various configure scripts -are able to find it), as well as the `cmake` tool. +are able to find it), as well as an installation of Python 3. The list is a work in progress, please contact us (Cristian.Lalescu@mpcdf.mpg.de) if the procedure fails at any step of the process. We recommend to first go through the instructions in full, and only afterwards starting to execute the individual steps. -*Note*: the HDF5 1.8x library may require MPI standard v1, we haven't tested -this in detail. Some default OpenMPI3 installations will not work. - -Detailed full installation instructions: - -1. - Optional. - Make a directory `PREFIX` on a local fast partition. - (Under unix systems, you can probably use `~/.local`). - -2. - Optional. - We recommend the creation of a virtual python3 environment that will be - used for installing TurTLE. - Please see https://docs.python-guide.org/dev/virtualenvs/. - In the following, the install location of this environment is - denoted `TurTLE_DIR`. - In principle `TurTLE_DIR` should be a subdirectory of the previously - created `PREFIX` folder. - -3. - Export the following environment variables with apropriate values: - * `CC` - Preferred C compiler - * `CXX` - Preferred C++ compiler - * `MPI_HOME` - MPI installation directory - * `FFTW_DIR` - Base directory of FFTW - * `FFTW_INCDIR` - Directory containing FFTW header files - * `FFTW_LIBDIR` - Directory containing FFTW library files - * `FFTW_OPENMP_LIBDIR` - Directory containing OpenMP FFTW library files (please define if different from FFTW_LIBDIR) - * `FFTW_MPI_LIBDIR` - Directory containing MPI FFTW library files (please define if different from FFTW_LIBDIR) - - If you do not have FFTW installed already, then download, compile, install FFTW - (latest version 3.x from http://www.fftw.org/). - Execute the following commands in order, feel free to customize - optimisation flags for your own computer (see http://www.fftw.org/fftw3_doc/Installation-on-Unix.html): +**Creating a virtual environment** + +We recommend creating a virtual environment for TurTLE. To do this, +choose an installation location :code:`<INSTALL_LOC>` on a local fast partition +(Under unix systems, this could be e.g. `~/.local`) and a name for the +environment :code:`<VENV_NAME>` (e.g. `turtle-production`). +To create the virtual environment, execute: + +.. code:: bash + + python -m venv <INSTALL_LOC>/<VENV_NAME> + +In the following, we refer to the path :code:`<INSTALL_LOC>` / +:code:`<VENV_NAME>` as :code:`<TURTLE_DIR>`. For more information on virtual +environments, please see https://docs.python-guide.org/dev/virtualenvs/. + +**Installation of requirements** + +TurTLE has the following requirements: + +- C/C++ MPI compiler of your choice + +- Python 3 + +- FFTW version >= 3.3.4 + +- HDF5 version >= 1.8. **Important**: must be compiled with the option "--enable-parallel" + +- cmake version > 3.12 + +We will assume you already have a working MPI compiler and an +installation of Python 3. For the other requirements, we provide +installation instructions in the following. We recommend installing +them on a local fast partition. In the following, we refer to their +locations as :code:`<FFTW_DIR>`, :code:`<HDF5_DIR>` and +:code:`<CMAKE_DIR>`. You may choose to install the requirements into +your virtual environment. In that case, all of these placeholders are +equal to :code:`<TURTLE_DIR>`. + +- **FFTW** + Download latest version from http://www.fftw.org/. + To compile and install it in custom location :code:`<FFTW_DIR>`, + execute the following commands in order, feel free to customize + optimisation flags for your own computer (see + http://www.fftw.org/fftw3_doc/Installation-on-Unix.html): .. code:: bash - ./configure --prefix=FFTW_DIR --enable-float --enable-sse --enable-mpi --enable-openmp --enable-threads + ./configure --prefix=<FFTW_DIR> --enable-float --enable-sse --enable-mpi --enable-openmp --enable-threads make make install - ./configure --prefix=FFTW_DIR --enable-sse2 --enable-avx512 --enable-mpi --enable-openmp --enable-threads + ./configure --prefix=<FFTW_DIR> --enable-sse2 --enable-avx512 --enable-mpi --enable-openmp --enable-threads make make install - In the above instructions, replace `FFTW_DIR` with an appropriate - location (for instance it is safe to use the `PREFIX` defined above). +.. TurTLE will try to find FFTW using the FindFFTW from the Morse project. - If the package is installed in a non standard location, you should - setup the environment variables listed above: `FFTW_DIR`, `FFTW_INCDIR`, - `FFTW_LIBDIR`, `FFTW_OPEMMP_LIBDIR` and `FFTW_MPI_LIBDIR`. - -4. - Export the following environment variables with apropriate values: - * `CC` - Preferred C compiler - * `CXX` - Preferred C++ compiler - * `MPI_HOME` - MPI installation directory - * `HDF5_ROOT` - Base directory of HDF5 + If the package is installed in a non-standard location, make sure the + corresponding environment variables are properly exported + (see step 3 of installation of TurTLE). - If you do not have HDF5 installed already, then download, compile, install - HDF5. - We are using parallel I/O, therefore we must use the plain C interface of HDF5: +- **HDF5** + Download HDF5 from https://www.hdfgroup.org/downloads/hdf5/. + To compile and install it in custom location :code:`<HDF5_DIR>`, + execute the following commands in order: .. code:: bash - ./configure --prefix=HDF5_ROOT --enable-parallel + ./configure --prefix=<HDF5_DIR> --enable-parallel make make install + Since TurTLE is using parallel I/O, HDF5 needs to be compiled with + the "--enable-parallel" flag. + +.. TurTLE will try to find HDF5 using the regular FindHDF5, which searches system folders, or `HDF5_ROOT`. -5. TurTLE requires `cmake` version > 3.12, which should be available - from your default package manager. - If required, download, compile and install cmake, currently - available at https://cmake.org/cmake/resources/software.html. +- **cmake** + Check if cmake (version >3.12) is available from your default package manager. + If not, then download cmake at https://cmake.org/cmake/resources/software.html. + To compile and install it in custom location :code:`<CMAKE_DIR>`, + execute the following commands in order: .. code:: bash - ./bootstrap --prefix=PREFIX + ./bootstrap --prefix=<CMAKE_DIR> make make install - The value of `PREFIX` used above is only relevant to later executing + The directory `<CMAKE_DIR>` is only relevant to later executing the `cmake` binary (which can be found under `${PREFIX}/bin` after installation). -6. - Clone turtle repository. + +**Installation of TurTLE** + +1. Choose a location for the source code, enter it and clone turtle + repository by .. code:: bash git clone git@gitlab.mpcdf.mpg.de:TurTLE/turtle.git -7. - Go into TurTLE repository, execute +2. Execute .. code:: bash + cd turtle mkdir build cd build - cp ../pc_host_info.py ./host_info.py - Edit the `host_info.py` file according to the instructions in the file. - This is strictly required before installing on a cluster. - -8. - Optional. - *PINCHECK* is available from https://gitlab.mpcdf.mpg.de/khr/pincheck. - If you'd like to check whether TurTLE MPI processes and OpenMP - threads are pinned properly to hardware threads, you can simply - place the pincheck headers under `${PREFIX}/include`. - -9. - In the build folder, edit the file `bash_setup_for_TurTLE.sh` so that it - exports the following environment variables with apropriate values: - * `CC` - Preferred C compiler - * `CXX` - Preferred C++ compiler - * `MPI_HOME` - MPI installation directory - * `FFTW_DIR` - Base directory of FFTW - * `FFTW_INCDIR` - Directory containing FFTW header files - * `FFTW_LIBDIR` - Directory containing FFTW library files - * `FFTW_OPENMP_LIBDIR` - Directory containing OpenMP FFTW library files (please define if different from FFTW_LIBDIR) - * `FFTW_MPI_LIBDIR` - Directory containing MPI FFTW library files (please define if different from FFTW_LIBDIR) - * `HDF5_ROOT` - Base directory of HDF5 - * `PINCHECK_ROOT` - Directory under which - `include/pincheck.hpp` may be found (OPTIONAL) - * `TurTLE_DIR` - Directory under which TurTLE will be installed (same value from step 2). - I.e. library will go under `${TurTLE_DIR}/lib`, headers under - `${TurTLE_DIR}/include`, etc. - This script should also contain the line - `source TurTLE_DIR/lib/bash_setup_for_TurTLE.sh`, as well as the - line `source TurTLE_DIR/bin/activate`. - - We also recommend that an environment variable `TURTLE_COMPILATION_FLAGS` - is defined appropriately. - In particular, for clusters of unknown architecture it helps to log into - individual nodes and run the following command: +3. Copy the file `bash_setup_template.sh` into the build folder: .. code:: bash - gcc -march=native -Q --help=target + cp ../bash_setup_template.sh ./bash_setup_for_TurTLE.sh - Note that an example file `example.bashrc` is included in the - repository. - There is at least one recorded case of a cluster where different - FFTW libraries (serial, MPI and OpenMP) were located in different - folders, hence the options of specifying the details. - See also lines 143 through 160 of `CMakeLists.txt`. + This file will set all required environment variables. + Please replace all placeholders by their corresponding values. - *If using openmpi* you may need to recompile openmpi with the - `--enable-mpi1-compatibility` flag, in order for HDF5 to compile and - link (step 4). - - *Note*: in principle it is possible to add this information to your + *Note*: In principle it is possible to add this information to your `.bashrc`, but we recommend against it. -10. - In the previously created build folder, execute + +4. Copy the file `pc_host_info.py` into the build folder: + + .. code:: bash + + cp ../pc_host_info.py ./host_info.py + + This file contains information about the machine on which TurTLE + will run. On desktop machines, no further steps are required. On + clusters, please edit the `host_info.py` file according to the + instructions in the file. + + +5. + Finally, we are ready to install TurTLE. Within the build folder, execute .. code:: bash source bash_setup_for_TurTLE.sh - cmake .. -DCMAKE_INSTALL_PREFIX=TurTLE_DIR - make - # to get a verbose compilation process, use - # VERBOSE=1 make - # to use more than one core for the compilation use - # make -jN - # (where N is the number of available cores) - make install + cmake .. -DCMAKE_INSTALL_PREFIX=<TURTLE_DIR> -11. - Using TurTLE from an external project. - TurTLE creates and installs 3 files alongside the C++ headers and - library: + Then compile TurTLE with :code:`<N>` cores using .. code:: bash - TurTLEConfig.cmake - TurTLE_EXPORT.cmake - TurTLE_EXPORT-noconfig.cmake - - These files are installed under `${TurTLE_DIR}/lib`. - In case these files provide incomplete information, it is necessary to update - the cmake input config file: `turtle/cmake/TurTLEConfig.cmake.in`. - -12. - For clusters. - In the `extra_slurm_lines` list from `host_info.py` one should add a - corresponding `source INSTALL_DIR/lib/bash_setup_for_TurTLE.sh` - line (where you should write the explicit install folder instead of - `INSTALL_DIR`), such that the appropriate setup is loaded in the job - script. + make -j<N> + make install + +Congratulations, you have installed TurTLE! Feel free to checkout the +`tutorial <http://turtle.pages.mpcdf.de/turtle/sphinx_html/chapters/tutorial.html>`_. + + +**Using TurTLE from an external project**. + +TurTLE creates and installs 3 files alongside the C++ headers and +library: + +.. code:: bash + + TurTLEConfig.cmake + TurTLE_EXPORT.cmake + TurTLE_EXPORT-noconfig.cmake + +These files are installed under `${TURTLE_DIR}/lib`. +In case these files provide incomplete information, it is necessary to update +the cmake input config file: `turtle/cmake/TurTLEConfig.cmake.in`. + **Uninstall** diff --git a/TurTLE/_code.py b/TurTLE/_code.py index 46403e4d484e52e61972e873404d22edcfcffc1e..4beefa12ca33fd0f85a8e3470b6e29fa8ebc9c21 100644 --- a/TurTLE/_code.py +++ b/TurTLE/_code.py @@ -314,8 +314,6 @@ class _code(_base): command_atoms = ['mpirun', '-np', '{0}'.format(nb_processes), - '-' + mpirun_environment_set, - 'OMP_NUM_THREADS={0}'.format(nb_threads_per_process), './' + self.name, self.simname] if self.host_info['type'] == 'cluster': diff --git a/bash_setup_template.sh b/bash_setup_template.sh new file mode 100644 index 0000000000000000000000000000000000000000..3f19fb6203c1a95fff6a1656bf70368c8ede8aa8 --- /dev/null +++ b/bash_setup_template.sh @@ -0,0 +1,57 @@ +#! /bin/bash + +# Load dependencies here if necessary, e.g. `module load cmake` + +# module-dependent variables +export CC=<CC> # Preferred C compiler +export CXX=<CXX> # Preferred C++ compiler +export MPI_HOME=<MPI_HOME> # MPI installation directory +export FFTW_ROOT=<FFTW_DIR> # Base directory of FFTW +export FFTW_DIR=<FFTW_DIR> # Base directory of FFTW +export HDF5_ROOT=<HDF5_DIR> # Base directory of HDF5 + +# OPTIONAL +# +# export PINCHECK_ROOT=<PINCHECK_ROOT> +# +# If this variable is exported, you can check whether TurTLE +# MPI processes and OpenMP threads are pinned properly to hardware threads. +# It is available from https://gitlab.mpcdf.mpg.de/khr/pincheck. +# The pincheck headers need to be placed under `<PINCHECK_ROOT>/include`. + +# OPTIONAL +# +# export TURTLE_COMPILATION_FLAGS=<COMPILATION_FLAGS> +# +# We also recommend setting this variable to optimize compilation +# For clusters of unknown architecture it helps to log into +# individual nodes and run the following command: +# gcc -march=native -Q --help=target + +# OPTIONAL +# +# export FFTW_INCDIR=<FFTW_INCDIR> # Directory containing FFTW header files +# export FFTW_LIBDIR=<FFTW_LIBDIR> # Directory containing FFTW library files +# export FFTW_OPENMP_LIBDIR=<FFTW_OMP_LIBDIR> # Directory containing OpenMP FFTW library files (please define if different from FFTW_LIBDIR) +# export FFTW_MPI_LIBDIR=<FFTW_MPI_LIBDIR> # Directory containing MPI FFTW library files (please define if different from FFTW_LIBDIR) +# +# There is at least one recorded case of a cluster where different +# FFTW libraries (serial, MPI and OpenMP) were located in different +# folders, hence the options of specifying the details. +# See also lines 264 through 282 of `CMakeLists.txt`. +# +# If using openmpi you may need to recompile openmpi with the +# `--enable-mpi1-compatibility` flag, in order for HDF5 to compile and +# link. + +# Turtle installation directory +export TURTLE_ROOT=<TURTLE_DIR> +# (library will go under ${TURTLE_ROOT}/lib, headers under ${TURTLE_ROOT}/include, etc.) + +# location-dependent variables +export CUSTOM_INSTALL_PATH=${TURTLE_ROOT} +export PATH=${CUSTOM_INSTALL_PATH}/bin:${PATH} +export CMAKE_PREFIX_PATH=${CUSTOM_INSTALL_PATH}/lib + +# activate python virtual environment +source ${TURTLE_ROOT}/bin/activate \ No newline at end of file diff --git a/example.bashrc b/example.bashrc deleted file mode 100644 index 2eebaef706534756d71a2f8989ec9d8d9febb451..0000000000000000000000000000000000000000 --- a/example.bashrc +++ /dev/null @@ -1,17 +0,0 @@ -#! /bin/bash -export CC=<please insert appropriate C compiler executable here> -export CXX=<please insert appropriate C++ compiler executable here> -export MPI_ROOT=<please insert appropriate path here> -export CUSTOM_INSTALL_PATH=<please insert your prefered path here. We recommend the location of your python virtual environment> -export PATH=${CUSTOM_INSTALL_PATH}/bin:${PATH} -export FFTW_DIR=${CUSTOM_INSTALL_PATH} -export FFTW_BASE=${FFTW_DIR} -export FFTW_OPENMP_LIB="-L${FFTW_BASE}/lib -lfftw3_omp -lfftw3f_omp" -export FFTW_LIB="-L${FFTW_BASE}/lib -lfftw3_mpi -lfftw3 -lfftw3f_mpi -lfftw3f" -export HDF5_DIR=${CUSTOM_INSTALL_PATH} -export HDF5_BASE=${HDF5_DIR} -export HDF5_ROOT=${HDF5_DIR} -export CMAKE_PREFIX_PATH=${CUSTOM_INSTALL_PATH}/lib - -#export TURTLE_COMPILATION_FLAGS="-O3 -mtune=native -ffast-math" -export TURTLE_COMPILATION_FLAGS="" diff --git a/pc_host_info.py b/pc_host_info.py index 8e66eb3535c0d13f12d1434f5407fb068f5683fc..e955be0c6adb0807083b895333e300c719a16786 100644 --- a/pc_host_info.py +++ b/pc_host_info.py @@ -65,6 +65,8 @@ host_info = {'type' : 'pc'} # info_template_extra_slurm_lines, relevant for 'SLURM' clusters, # is a list of strings, written as extra options within the SLURM file. # for example something like `#SBATCH --get-user-env` +# Here, also the virtual environment and environment variables need to be loaded, +# e.g. by `source <TURTLE_DIR>/lib/bash_setup_for_TurTLE.sh` # info_template_explicit_slurm_environment, relevant for `SLURM` clusters, # is a bool specifying whether the environment must be mentioned explicitly