Commit dd70d174 authored by Luka Stanisic's avatar Luka Stanisic
Browse files

retesting compilations/runs and adding recipies for different machine

parent 32fee9dc
......@@ -19,6 +19,374 @@
** ?Xeon Phi
* Ideas
* Important codes
** Recipes for compilations and runs
*** [error] dvl machine - problems at CUDA runtime
**** Machine description
- Local machines at MPCDF for development
- dvl01:
+ 20 CPU cores Intel machine (Intel(R) Xeon(R) CPU E5-2680 v2 @ 2.80GHz)
+ 2 K40 GPUs
- dvl02:
+ 20 CPU cores Intel machine (Intel(R) Xeon(R) CPU E5-2680 v2 @ 2.80GHz)
+ 2 Nvidia K20Xm GPUs
**** Installation with gnu compilers
#+BEGIN_SRC
# Loading necessary modules
module purge
module load git/2.7.4
module load gcc/4.9
module load impi/5.1.3
module load cmake/3.6
module load boost/gcc/1.57
module load fftw/gcc/3.3.4
module load cuda/7.5
# Paths
SRC_DIR="$afsdir/BioEM_fork"
BUILD_DIR="$HOME/BioEM_project/build"
mkdir -p $BUILD_DIR
cd $BUILD_DIR
# Deleting files from previous installations previous
rm -rf $BUILD_DIR/*
rm -rf $SRC_DIR/CMakeFiles $SRC_DIR/CMakeCache.txt $SRC_DIR/Makefile $SRC_DIR/cmake_install.cmake
# Configuration and compilation
cmake -DUSE_MPI=ON -DUSE_OPENMP=ON -DUSE_CUDA=ON -DPRINT_CMAKE_VARIABLES=ON -DCUDA_FORCE_GCC=ON $SRC_DIR/
make -j5 VERBOSE=1
#+END_SRC
**** Installation with Intel compilers
#+BEGIN_SRC
# Loading necessary modules
module purge
module load git/2.7.4
module load intel/15.0
module load impi/5.1.3
module load cmake/3.6
module load boost/intel/1.57
module load fftw/3.3.4
module load cuda/7.5
# Paths
SRC_DIR="$afsdir/BioEM_fork"
BUILD_DIR="$HOME/BioEM_project/build"
mkdir -p $BUILD_DIR
cd $BUILD_DIR
# Deleting files from previous installations previous
rm -rf $BUILD_DIR/*
rm -rf $SRC_DIR/CMakeFiles $SRC_DIR/CMakeCache.txt $SRC_DIR/Makefile $SRC_DIR/cmake_install.cmake
# Configuration and compilation
cmake -DUSE_MPI=ON -DUSE_OPENMP=ON -DUSE_CUDA=ON -DPRINT_CMAKE_VARIABLES=ON -DCUDA_FORCE_GCC=ON $SRC_DIR/
make -j5 VERBOSE=1
#+END_SRC
**** Running Tutorial example
#+BEGIN_SRC
# Loading necessary modules if needed (check installation gcc/Intel)
# Environment variables
export OMP_NUM_THREADS=5
export OMP_PLACES=cores
export FFTALGO=1
export GPU=1
export GPUDEVICE=-1
export GPUWORKLOAD=100
# Paths
TUTORIAL_DIR="$afsdir/BioEM_fork/Tutorial_BioEM"
BUILD_DIR="$HOME/BioEM_project/build"
# Running tutorial test
mpiexec -n 2 $BUILD_DIR/bioEM --Inputfile $TUTORIAL_DIR/Param_Input --Modelfile $TUTORIAL_DIR/Model_Text --Particlesfile $TUTORIAL_DIR/Text_Image_Form
#+END_SRC
**** Running larger example from the paper
#+BEGIN_SRC
# Loading necessary modules if needed (check installation gcc/Intel)
# Environment variables
export OMP_NUM_THREADS=5
export OMP_PLACES=cores
export FFTALGO=1
export GPU=1
export GPUDEVICE=-1
export GPUWORKLOAD=100
# Paths
INPUT_DIR="$HOME/BioEM_project/inputs"
BUILD_DIR="$HOME/BioEM_project/build"
# Running only few iterations, reading already generated maps (only if they are avaialble in the same folder
BIOEM_DEBUG_BREAK=4 BIOEM_DEBUG_OUTPUT=2 mpiexec -n 2 $BUILD_DIR/bioEM --Inputfile $INPUT_DIR/INPUT_FRH_Sep2016 --Modelfile $INPUT_DIR/Mod_X-ray_PDB --Particlesfile $INPUT_DIR/2000FRH_Part --LoadMapDump
# Running full example
BIOEM_DEBUG_OUTPUT=0 mpiexec -n 2 $BUILD_DIR/bioEM --Inputfile $INPUT_DIR/INPUT_FRH_Sep2016 --Modelfile $INPUT_DIR/Mod_X-ray_PDB --Particlesfile $INPUT_DIR/2000FRH_Part
#+END_SRC
*** [error] miy (Minsky) machine - problems at CUDA runtime
**** Machine description
- 3 identical machine with IBM Power 8+ architecture
- Local machines at MPCDF for development and testing Power8 capabilities
- miy01-miy03:
+ 2 Power8+ CPUs (160 cores?)
+ 4 Nvidia P100 GPUs
**** Installation with gnu compilers
- On Minsky machine problems with gcc options that are not supported by Power8 (IBM) machine:
+ c++: error: unrecognized command line option ‘-march=native’
+ c++: error: unrecognized command line option ‘-mfpmath=sse’
+ c++: error: unrecognized command line option ‘-minline-all-stringops’
- Without it, it is possible to compile the project
- Neither Git nor AFS are available
#+BEGIN_SRC
# Loading necessary modules
module purge
module load gcc/5.4
module load smpi/10.1
module load fftw/gcc-5.4
module load cuda/8.0
# Paths
SRC_DIR="$HOME/BioEM_project/BioEM"
BUILD_DIR="$HOME/BioEM_project/build"
mkdir -p $BUILD_DIR
cd $BUILD_DIR
# Deleting files from previous installations previous
rm -rf $BUILD_DIR/*
rm -rf $SRC_DIR/CMakeFiles $SRC_DIR/CMakeCache.txt $SRC_DIR/Makefile $SRC_DIR/cmake_install.cmake
# Configuration and compilation
cmake -DUSE_MPI=ON -DUSE_OPENMP=ON -DUSE_CUDA=ON -DPRINT_CMAKE_VARIABLES=ON -DCUDA_FORCE_GCC=ON $SRC_DIR/
make -j5 VERBOSE=1
#+END_SRC
**** Running Tutorial example
#+BEGIN_SRC
# Loading necessary modules if needed (check installation gcc/Intel)
# Environment variables
export OMP_NUM_THREADS=5
export OMP_PLACES=cores
export FFTALGO=1
export GPU=1
export GPUDEVICE=-1
export GPUWORKLOAD=100
# Paths
TUTORIAL_DIR="$HOME/BioEM_project/BioEM/Tutorial_BioEM"
BUILD_DIR="$HOME/BioEM_project/build"
# Running tutorial test
mpiexec -n 2 $BUILD_DIR/bioEM --Inputfile $TUTORIAL_DIR/Param_Input --Modelfile $TUTORIAL_DIR/Model_Text --Particlesfile $TUTORIAL_DIR/Text_Image_Form
#+END_SRC
**** Running larger example from the paper
#+BEGIN_SRC
# Loading necessary modules if needed (check installation gcc/Intel)
# Environment variables
export OMP_NUM_THREADS=5
export OMP_PLACES=cores
export FFTALGO=1
export GPU=1
export GPUDEVICE=-1
export GPUWORKLOAD=100
# Paths
INPUT_DIR="$HOME/BioEM_project/inputs"
BUILD_DIR="$HOME/BioEM_project/build"
# Running only few iterations, reading already generated maps (only if they are avaialble in the same folder
BIOEM_DEBUG_BREAK=4 BIOEM_DEBUG_OUTPUT=2 mpiexec -n 2 $BUILD_DIR/bioEM --Inputfile $INPUT_DIR/INPUT_FRH_Sep2016 --Modelfile $INPUT_DIR/Mod_X-ray_PDB --Particlesfile $INPUT_DIR/2000FRH_Part --LoadMapDump
# Running full example
BIOEM_DEBUG_OUTPUT=0 mpiexec -n 2 $BUILD_DIR/bioEM --Inputfile $INPUT_DIR/INPUT_FRH_Sep2016 --Modelfile $INPUT_DIR/Mod_X-ray_PDB --Particlesfile $INPUT_DIR/2000FRH_Part
#+END_SRC
*** [partially working] hydra machine
**** Machine description (from http://www.mpcdf.mpg.de/services/computing/hydra/about-the-system)
In October 2013, the main part of the iDataPlex HPC system HYDRA
was installed at the MPCDF with Intel Ivy Bridge processors (~
3500 nodes with 20 cores @ 2.8 GHz each), in addition to the
Intel Sandy Bridge-EP processors (610 nodes with 16 cores @ 2.6
GHz each) that are available since Sept 2012. 350 of the Ivy
Bridge nodes are equipped with accelerator cards (338 nodes with
2 NVIDIA K20X GPGPUs each, 12 nodes with 2 Intel Xeon Phi cards
each).
Most of the compute nodes have a main memory of 64 GB, 2 x 100 of
the Ivy Bridge nodes and 20 of the Sandy Bridge nodes have a main
memory of 128 GB.
In total there are ~ 83.000 cores with a main memory of 280 TB
and a peak performance of about 1.7 PetaFlop/s. The accelerator
part of the HPC cluster has a peak performance of about 1
PetaFlop/s.
In addition to the compute nodes there are 8 login nodes and 26
I/O nodes that serve the 5 PetaByte of disk storage.
The common interconnect is a fast InfiniBand FDR14 network.
The compute nodes are bundled into 5 domains, three domains
(including the old Sandy Bridge processor domain) with 628 nodes
each, one big domain with more than 1800 nodes, and one domain
consisting of the 350 nodes with accelerator cards. Within one
domain, the InfiniBand network topology is a 'fat tree' topology
for high efficient communication. The InfiniBand connection
between the domains is much weaker, so batch jobs are restricted
to a single domain.
**** [problems with compilation] Installation with gnu compilers
#+BEGIN_SRC
# Loading necessary modules
module purge
module load cmake/3.5
module load gcc/4.9
module load mpi.ibm/1.4.0
module load fftw/gcc/3.3.4
module load boost/gcc/1.61
module load cuda/7.5
# Paths
SRC_DIR="$afsdir/BioEM_fork"
BUILD_DIR="$HOME/BioEM_project/build"
mkdir -p $BUILD_DIR
cd $BUILD_DIR
# Deleting files from previous installations previous
rm -rf $BUILD_DIR/*
rm -rf $SRC_DIR/CMakeFiles $SRC_DIR/CMakeCache.txt $SRC_DIR/Makefile $SRC_DIR/cmake_install.cmake
# There are some strange problems with CUDA, however this small hack with links seems to solve the issue
ln -s /u/system/SLES11/soft/cuda/libcuda/libcuda.so.352.79 libcuda.so.1
ln -s libcuda.so.1 libcuda.so
# Configuration and compilation (need to manually add CUDA_rt_LIBRARY)
cmake -DUSE_MPI=ON -DUSE_OPENMP=ON -DUSE_CUDA=ON -DPRINT_CMAKE_VARIABLES=ON -DCUDA_FORCE_GCC=ON -DCUDA_rt_LIBRARY=/u/system/SLES11/soft/cuda/7.5/lib64/libcudart.so -DCUDA_SDK_ROOT_DIR=/u/system/SLES11/soft/cuda/7.5/samples/common -DCUDA_CUDA_LIBRARY=$PWD/libcuda.so $SRC_DIR/
make -j5 VERBOSE=1
#+END_SRC
**** [working] Installation with Intel compilers
#+BEGIN_SRC
# Loading necessary modules
module purge
module load cmake/3.5
module load gcc/4.9
module load mpi.ibm/1.4.0
module load fftw/gcc/3.3.4
module load boost/gcc/1.61
module load cuda/7.5
# Paths
SRC_DIR="$afsdir/BioEM_fork"
BUILD_DIR="$HOME/BioEM_project/build"
mkdir -p $BUILD_DIR
cd $BUILD_DIR
# Deleting files from previous installations previous
rm -rf $BUILD_DIR/*
rm -rf $SRC_DIR/CMakeFiles $SRC_DIR/CMakeCache.txt $SRC_DIR/Makefile $SRC_DIR/cmake_install.cmake
# There are some strange problems with CUDA, however this small hack with links seems to solve the issue. Probably CMake needs to be improved
ln -s /u/system/SLES11/soft/cuda/libcuda/libcuda.so.352.79 libcuda.so.1
ln -s libcuda.so.1 libcuda.so
# Configuration and compilation (need to manually add CUDA_rt_LIBRARY and CUDA_SDK_ROOT_DIR)
cmake -DUSE_MPI=ON -DUSE_OPENMP=ON -DUSE_CUDA=ON -DPRINT_CMAKE_VARIABLES=ON -DCUDA_FORCE_GCC=ON -DCUDA_rt_LIBRARY=/u/system/SLES11/soft/cuda/7.5/lib64/libcudart.so -DCUDA_SDK_ROOT_DIR=/u/system/SLES11/soft/cuda/7.5/samples/common -DCUDA_CUDA_LIBRARY=$PWD/libcuda.so $SRC_DIR/
make -j5 VERBOSE=1
#+END_SRC
**** [untested] Running interactive Tutorial example (no GPUs)
#+BEGIN_SRC
# Loading necessary modules if needed (check installation gcc/Intel)
# Environment variables
export OMP_NUM_THREADS=5
export OMP_PLACES=cores
export FFTALGO=1
export GPU=0
# Specific for interactive tests on hydra
export MP_TASK_AFFINITY=core:$OMP_NUM_THREADS
echo localhost > host.list
echo localhost >> host.list
echo localhost >> host.list
echo localhost >> host.list
# Paths
TUTORIAL_DIR="$afsdir/BioEM_fork/Tutorial_BioEM"
BUILD_DIR="$HOME/BioEM_project/build"
# Running tutorial test
mpiexec -n 4 $BUILD_DIR/bioEM --Inputfile $TUTORIAL_DIR/Param_Input --Modelfile $TUTORIAL_DIR/Model_Text --Particlesfile $TUTORIAL_DIR/Text_Image_Form
#+END_SRC
**** [untested] Running larger example from the paper
#+BEGIN_SRC
# @ shell=/bin/bash
#
# Sample script for LoadLeveler
#
# @ error = job.err.$(jobid)
# @ output = job.out.$(jobid)
# @ job_type = parallel
# @ requirements = (Feature=="gpu")
# @ node_usage= not_shared
# @ node = 1
# @ tasks_per_node = 1
# @ resources = ConsumableCpus(20)
# @ network.MPI = sn_all,not_shared,us
# @ wall_clock_limit = 01:00:00
# @ notification = complete
# @ notify_user = $(user)@rzg.mpg.de
# @ queue
# Loading necessary modules for Intel compilers
module purge
module load cmake/3.5
module load gcc/4.9
module load mpi.ibm/1.4.0
module load fftw/gcc/3.3.4
module load boost/gcc/1.61
module load cuda/7.5
# Environment variables
export OMP_NUM_THREADS=5
export OMP_PLACES=cores
export FFTALGO=1
export GPU=1
export GPUDEVICE=-1
export GPUWORKLOAD=80
# Paths
INPUT_DIR="$HOME/BioEM_project/inputs"
BUILD_DIR="$HOME/BioEM_project/build"
mpiexec -n 2 $BUILD_DIR/bioEM --Inputfile $INPUT_DIR/INPUT_FRH_Sep2016 --Modelfile $INPUT_DIR/Mod_X-ray_PDB --Particlesfile $INPUT_DIR/2000FRH_Part --LoadMapDump
# Running full example
cd /ptmp/$USER/
poe $BUILD_DIR/bioEM --Inputfile $INPUT_DIR/INPUT_FRH_Sep2016 --Modelfile $INPUT_DIR/Mod_X-ray_PDB --Particlesfile $INPUT_DIR/2000FRH_Part
#+END_SRC
* Summaries, papers, reports
=put here links to potential external repositories=
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment