diff --git a/BioEM_labbook.org b/BioEM_labbook.org index 6c85d4dc2d1cdaf16ab729e6b03ca4c664215bce..d9eae88c1a4abfa526e0b1c2c8c22276cbb3fa0e 100644 --- a/BioEM_labbook.org +++ b/BioEM_labbook.org @@ -19,6 +19,374 @@ ** ?Xeon Phi * Ideas * Important codes +** Recipes for compilations and runs +*** [error] dvl machine - problems at CUDA runtime +**** Machine description + - Local machines at MPCDF for development + - dvl01: + + 20 CPU cores Intel machine (Intel(R) Xeon(R) CPU E5-2680 v2 @ 2.80GHz) + + 2 K40 GPUs + - dvl02: + + 20 CPU cores Intel machine (Intel(R) Xeon(R) CPU E5-2680 v2 @ 2.80GHz) + + 2 Nvidia K20Xm GPUs +**** Installation with gnu compilers + +#+BEGIN_SRC +# Loading necessary modules +module purge +module load git/2.7.4 +module load gcc/4.9 +module load impi/5.1.3 +module load cmake/3.6 +module load boost/gcc/1.57 +module load fftw/gcc/3.3.4 +module load cuda/7.5 + +# Paths +SRC_DIR="$afsdir/BioEM_fork" +BUILD_DIR="$HOME/BioEM_project/build" +mkdir -p $BUILD_DIR +cd $BUILD_DIR + +# Deleting files from previous installations previous +rm -rf $BUILD_DIR/* +rm -rf $SRC_DIR/CMakeFiles $SRC_DIR/CMakeCache.txt $SRC_DIR/Makefile $SRC_DIR/cmake_install.cmake + +# Configuration and compilation +cmake -DUSE_MPI=ON -DUSE_OPENMP=ON -DUSE_CUDA=ON -DPRINT_CMAKE_VARIABLES=ON -DCUDA_FORCE_GCC=ON $SRC_DIR/ +make -j5 VERBOSE=1 +#+END_SRC + +**** Installation with Intel compilers + +#+BEGIN_SRC +# Loading necessary modules +module purge +module load git/2.7.4 +module load intel/15.0 +module load impi/5.1.3 +module load cmake/3.6 +module load boost/intel/1.57 +module load fftw/3.3.4 +module load cuda/7.5 + +# Paths +SRC_DIR="$afsdir/BioEM_fork" +BUILD_DIR="$HOME/BioEM_project/build" +mkdir -p $BUILD_DIR +cd $BUILD_DIR + +# Deleting files from previous installations previous +rm -rf $BUILD_DIR/* +rm -rf $SRC_DIR/CMakeFiles $SRC_DIR/CMakeCache.txt $SRC_DIR/Makefile $SRC_DIR/cmake_install.cmake + +# Configuration and compilation +cmake -DUSE_MPI=ON -DUSE_OPENMP=ON -DUSE_CUDA=ON -DPRINT_CMAKE_VARIABLES=ON -DCUDA_FORCE_GCC=ON $SRC_DIR/ +make -j5 VERBOSE=1 +#+END_SRC + +**** Running Tutorial example + +#+BEGIN_SRC +# Loading necessary modules if needed (check installation gcc/Intel) + +# Environment variables +export OMP_NUM_THREADS=5 +export OMP_PLACES=cores +export FFTALGO=1 +export GPU=1 +export GPUDEVICE=-1 +export GPUWORKLOAD=100 + +# Paths +TUTORIAL_DIR="$afsdir/BioEM_fork/Tutorial_BioEM" +BUILD_DIR="$HOME/BioEM_project/build" + +# Running tutorial test +mpiexec -n 2 $BUILD_DIR/bioEM --Inputfile $TUTORIAL_DIR/Param_Input --Modelfile $TUTORIAL_DIR/Model_Text --Particlesfile $TUTORIAL_DIR/Text_Image_Form +#+END_SRC + +**** Running larger example from the paper + +#+BEGIN_SRC +# Loading necessary modules if needed (check installation gcc/Intel) + +# Environment variables +export OMP_NUM_THREADS=5 +export OMP_PLACES=cores +export FFTALGO=1 +export GPU=1 +export GPUDEVICE=-1 +export GPUWORKLOAD=100 + +# Paths +INPUT_DIR="$HOME/BioEM_project/inputs" +BUILD_DIR="$HOME/BioEM_project/build" + +# Running only few iterations, reading already generated maps (only if they are avaialble in the same folder +BIOEM_DEBUG_BREAK=4 BIOEM_DEBUG_OUTPUT=2 mpiexec -n 2 $BUILD_DIR/bioEM --Inputfile $INPUT_DIR/INPUT_FRH_Sep2016 --Modelfile $INPUT_DIR/Mod_X-ray_PDB --Particlesfile $INPUT_DIR/2000FRH_Part --LoadMapDump + +# Running full example +BIOEM_DEBUG_OUTPUT=0 mpiexec -n 2 $BUILD_DIR/bioEM --Inputfile $INPUT_DIR/INPUT_FRH_Sep2016 --Modelfile $INPUT_DIR/Mod_X-ray_PDB --Particlesfile $INPUT_DIR/2000FRH_Part +#+END_SRC + +*** [error] miy (Minsky) machine - problems at CUDA runtime +**** Machine description + - 3 identical machine with IBM Power 8+ architecture + - Local machines at MPCDF for development and testing Power8 capabilities + - miy01-miy03: + + 2 Power8+ CPUs (160 cores?) + + 4 Nvidia P100 GPUs +**** Installation with gnu compilers + + - On Minsky machine problems with gcc options that are not supported by Power8 (IBM) machine: + + c++: error: unrecognized command line option ‘-march=native’ + + c++: error: unrecognized command line option ‘-mfpmath=sse’ + + c++: error: unrecognized command line option ‘-minline-all-stringops’ + - Without it, it is possible to compile the project + - Neither Git nor AFS are available + +#+BEGIN_SRC +# Loading necessary modules +module purge +module load gcc/5.4 +module load smpi/10.1 +module load fftw/gcc-5.4 +module load cuda/8.0 + +# Paths +SRC_DIR="$HOME/BioEM_project/BioEM" +BUILD_DIR="$HOME/BioEM_project/build" +mkdir -p $BUILD_DIR +cd $BUILD_DIR + +# Deleting files from previous installations previous +rm -rf $BUILD_DIR/* +rm -rf $SRC_DIR/CMakeFiles $SRC_DIR/CMakeCache.txt $SRC_DIR/Makefile $SRC_DIR/cmake_install.cmake + +# Configuration and compilation +cmake -DUSE_MPI=ON -DUSE_OPENMP=ON -DUSE_CUDA=ON -DPRINT_CMAKE_VARIABLES=ON -DCUDA_FORCE_GCC=ON $SRC_DIR/ +make -j5 VERBOSE=1 +#+END_SRC + +**** Running Tutorial example + +#+BEGIN_SRC +# Loading necessary modules if needed (check installation gcc/Intel) + +# Environment variables +export OMP_NUM_THREADS=5 +export OMP_PLACES=cores +export FFTALGO=1 +export GPU=1 +export GPUDEVICE=-1 +export GPUWORKLOAD=100 + +# Paths +TUTORIAL_DIR="$HOME/BioEM_project/BioEM/Tutorial_BioEM" +BUILD_DIR="$HOME/BioEM_project/build" + +# Running tutorial test +mpiexec -n 2 $BUILD_DIR/bioEM --Inputfile $TUTORIAL_DIR/Param_Input --Modelfile $TUTORIAL_DIR/Model_Text --Particlesfile $TUTORIAL_DIR/Text_Image_Form +#+END_SRC + +**** Running larger example from the paper + +#+BEGIN_SRC +# Loading necessary modules if needed (check installation gcc/Intel) + +# Environment variables +export OMP_NUM_THREADS=5 +export OMP_PLACES=cores +export FFTALGO=1 +export GPU=1 +export GPUDEVICE=-1 +export GPUWORKLOAD=100 + +# Paths +INPUT_DIR="$HOME/BioEM_project/inputs" +BUILD_DIR="$HOME/BioEM_project/build" + +# Running only few iterations, reading already generated maps (only if they are avaialble in the same folder +BIOEM_DEBUG_BREAK=4 BIOEM_DEBUG_OUTPUT=2 mpiexec -n 2 $BUILD_DIR/bioEM --Inputfile $INPUT_DIR/INPUT_FRH_Sep2016 --Modelfile $INPUT_DIR/Mod_X-ray_PDB --Particlesfile $INPUT_DIR/2000FRH_Part --LoadMapDump + +# Running full example +BIOEM_DEBUG_OUTPUT=0 mpiexec -n 2 $BUILD_DIR/bioEM --Inputfile $INPUT_DIR/INPUT_FRH_Sep2016 --Modelfile $INPUT_DIR/Mod_X-ray_PDB --Particlesfile $INPUT_DIR/2000FRH_Part +#+END_SRC + +*** [partially working] hydra machine +**** Machine description (from http://www.mpcdf.mpg.de/services/computing/hydra/about-the-system) + + In October 2013, the main part of the iDataPlex HPC system HYDRA + was installed at the MPCDF with Intel Ivy Bridge processors (~ + 3500 nodes with 20 cores @ 2.8 GHz each), in addition to the + Intel Sandy Bridge-EP processors (610 nodes with 16 cores @ 2.6 + GHz each) that are available since Sept 2012. 350 of the Ivy + Bridge nodes are equipped with accelerator cards (338 nodes with + 2 NVIDIA K20X GPGPUs each, 12 nodes with 2 Intel Xeon Phi cards + each). + + Most of the compute nodes have a main memory of 64 GB, 2 x 100 of + the Ivy Bridge nodes and 20 of the Sandy Bridge nodes have a main + memory of 128 GB. + + In total there are ~ 83.000 cores with a main memory of 280 TB + and a peak performance of about 1.7 PetaFlop/s. The accelerator + part of the HPC cluster has a peak performance of about 1 + PetaFlop/s. + + In addition to the compute nodes there are 8 login nodes and 26 + I/O nodes that serve the 5 PetaByte of disk storage. + + The common interconnect is a fast InfiniBand FDR14 network. + + The compute nodes are bundled into 5 domains, three domains + (including the old Sandy Bridge processor domain) with 628 nodes + each, one big domain with more than 1800 nodes, and one domain + consisting of the 350 nodes with accelerator cards. Within one + domain, the InfiniBand network topology is a 'fat tree' topology + for high efficient communication. The InfiniBand connection + between the domains is much weaker, so batch jobs are restricted + to a single domain. + +**** [problems with compilation] Installation with gnu compilers + +#+BEGIN_SRC +# Loading necessary modules +module purge +module load cmake/3.5 +module load gcc/4.9 +module load mpi.ibm/1.4.0 +module load fftw/gcc/3.3.4 +module load boost/gcc/1.61 +module load cuda/7.5 + +# Paths +SRC_DIR="$afsdir/BioEM_fork" +BUILD_DIR="$HOME/BioEM_project/build" +mkdir -p $BUILD_DIR +cd $BUILD_DIR + +# Deleting files from previous installations previous +rm -rf $BUILD_DIR/* +rm -rf $SRC_DIR/CMakeFiles $SRC_DIR/CMakeCache.txt $SRC_DIR/Makefile $SRC_DIR/cmake_install.cmake + +# There are some strange problems with CUDA, however this small hack with links seems to solve the issue +ln -s /u/system/SLES11/soft/cuda/libcuda/libcuda.so.352.79 libcuda.so.1 +ln -s libcuda.so.1 libcuda.so + +# Configuration and compilation (need to manually add CUDA_rt_LIBRARY) +cmake -DUSE_MPI=ON -DUSE_OPENMP=ON -DUSE_CUDA=ON -DPRINT_CMAKE_VARIABLES=ON -DCUDA_FORCE_GCC=ON -DCUDA_rt_LIBRARY=/u/system/SLES11/soft/cuda/7.5/lib64/libcudart.so -DCUDA_SDK_ROOT_DIR=/u/system/SLES11/soft/cuda/7.5/samples/common -DCUDA_CUDA_LIBRARY=$PWD/libcuda.so $SRC_DIR/ +make -j5 VERBOSE=1 +#+END_SRC + +**** [working] Installation with Intel compilers + +#+BEGIN_SRC +# Loading necessary modules +module purge +module load cmake/3.5 +module load gcc/4.9 +module load mpi.ibm/1.4.0 +module load fftw/gcc/3.3.4 +module load boost/gcc/1.61 +module load cuda/7.5 + +# Paths +SRC_DIR="$afsdir/BioEM_fork" +BUILD_DIR="$HOME/BioEM_project/build" +mkdir -p $BUILD_DIR +cd $BUILD_DIR + +# Deleting files from previous installations previous +rm -rf $BUILD_DIR/* +rm -rf $SRC_DIR/CMakeFiles $SRC_DIR/CMakeCache.txt $SRC_DIR/Makefile $SRC_DIR/cmake_install.cmake + +# There are some strange problems with CUDA, however this small hack with links seems to solve the issue. Probably CMake needs to be improved +ln -s /u/system/SLES11/soft/cuda/libcuda/libcuda.so.352.79 libcuda.so.1 +ln -s libcuda.so.1 libcuda.so + +# Configuration and compilation (need to manually add CUDA_rt_LIBRARY and CUDA_SDK_ROOT_DIR) +cmake -DUSE_MPI=ON -DUSE_OPENMP=ON -DUSE_CUDA=ON -DPRINT_CMAKE_VARIABLES=ON -DCUDA_FORCE_GCC=ON -DCUDA_rt_LIBRARY=/u/system/SLES11/soft/cuda/7.5/lib64/libcudart.so -DCUDA_SDK_ROOT_DIR=/u/system/SLES11/soft/cuda/7.5/samples/common -DCUDA_CUDA_LIBRARY=$PWD/libcuda.so $SRC_DIR/ +make -j5 VERBOSE=1 +#+END_SRC + +**** [untested] Running interactive Tutorial example (no GPUs) + +#+BEGIN_SRC +# Loading necessary modules if needed (check installation gcc/Intel) + +# Environment variables +export OMP_NUM_THREADS=5 +export OMP_PLACES=cores +export FFTALGO=1 +export GPU=0 + +# Specific for interactive tests on hydra +export MP_TASK_AFFINITY=core:$OMP_NUM_THREADS +echo localhost > host.list +echo localhost >> host.list +echo localhost >> host.list +echo localhost >> host.list + +# Paths +TUTORIAL_DIR="$afsdir/BioEM_fork/Tutorial_BioEM" +BUILD_DIR="$HOME/BioEM_project/build" + +# Running tutorial test +mpiexec -n 4 $BUILD_DIR/bioEM --Inputfile $TUTORIAL_DIR/Param_Input --Modelfile $TUTORIAL_DIR/Model_Text --Particlesfile $TUTORIAL_DIR/Text_Image_Form +#+END_SRC + +**** [untested] Running larger example from the paper + +#+BEGIN_SRC +# @ shell=/bin/bash +# +# Sample script for LoadLeveler +# +# @ error = job.err.$(jobid) +# @ output = job.out.$(jobid) +# @ job_type = parallel +# @ requirements = (Feature=="gpu") +# @ node_usage= not_shared +# @ node = 1 +# @ tasks_per_node = 1 +# @ resources = ConsumableCpus(20) +# @ network.MPI = sn_all,not_shared,us +# @ wall_clock_limit = 01:00:00 +# @ notification = complete +# @ notify_user = $(user)@rzg.mpg.de +# @ queue + +# Loading necessary modules for Intel compilers +module purge +module load cmake/3.5 +module load gcc/4.9 +module load mpi.ibm/1.4.0 +module load fftw/gcc/3.3.4 +module load boost/gcc/1.61 +module load cuda/7.5 + +# Environment variables +export OMP_NUM_THREADS=5 +export OMP_PLACES=cores +export FFTALGO=1 +export GPU=1 +export GPUDEVICE=-1 +export GPUWORKLOAD=80 + +# Paths +INPUT_DIR="$HOME/BioEM_project/inputs" +BUILD_DIR="$HOME/BioEM_project/build" + + +mpiexec -n 2 $BUILD_DIR/bioEM --Inputfile $INPUT_DIR/INPUT_FRH_Sep2016 --Modelfile $INPUT_DIR/Mod_X-ray_PDB --Particlesfile $INPUT_DIR/2000FRH_Part --LoadMapDump + +# Running full example +cd /ptmp/$USER/ +poe $BUILD_DIR/bioEM --Inputfile $INPUT_DIR/INPUT_FRH_Sep2016 --Modelfile $INPUT_DIR/Mod_X-ray_PDB --Particlesfile $INPUT_DIR/2000FRH_Part +#+END_SRC + * Summaries, papers, reports =put here links to potential external repositories=