Commit 588897c6 authored by Luka Stanisic's avatar Luka Stanisic

rel2: improved CI testing

parent 7e5ac2fb
before_script:
- export LANG=C
# define two stages, that way coverage only runs after all other
# tests have been done.
# For some reason doing shallow fetch doesnt work
#variables:
# GIT_DEPTH: "3"
stages:
- build
- tests
- coverage
- deploy
###############################################
# default build (without CUDA)
# default GCC build, with coverage
build-gcc-mpi:
stage: build
script:
- .gitlab-ci/build/run_build.sh gcc-mpi
artifacts:
paths:
- build/
expire_in: 30 days
# default GCC build, minimal test
build-gcc-mpi-test:
stage: build
script:
- .gitlab-ci/build/run_build.sh gcc-mpi
- .gitlab-ci/test/run_test.sh gcc-mpi 4 2
- .gitlab-ci/test/run_coverage.sh build/CMakeFiles/bioEM.dir coverage_data
- .gitlab-ci/test/run_coverage.sh build/CMakeFiles/bioEM.dir coverage_test
artifacts:
paths:
- coverage_data
- coverage_test
expire_in: 30 days
# GPU build (with CUDA)
build-gcc-mpi-cuda:
......@@ -23,3 +41,83 @@ build-gcc-mpi-cuda:
script:
- .gitlab-ci/build/run_build.sh gcc-mpi-cuda
# Intel build
build-intel-mpi:
stage: build
script:
- .gitlab-ci/build/run_build.sh intel-mpi
###############################################
# Extended tests on different inputs
verify_options:
stage: tests
dependencies:
- build-gcc-mpi
script:
- .gitlab-ci/test/verify_options.sh gcc-mpi 4 2
- .gitlab-ci/coverage/ci_coverage_collect.sh
artifacts:
paths:
- ang_prob
- environment
- coverage_data
expire_in: 30 days
# Testing tutorial options
verify_tutorial:
stage: tests
dependencies:
- build-gcc-mpi
script:
- .gitlab-ci/test/verify_tutorial.sh build/bioEM Tutorial_BioEM
- .gitlab-ci/coverage/ci_coverage_collect.sh
artifacts:
paths:
- coverage_data
expire_in: 30 days
# Testing probabilities with different environment options
verify_probabilities:
stage: tests
dependencies:
- build-gcc-mpi
script:
- .gitlab-ci/test/verify_probabilities.sh gcc-mpi 4 2
- .gitlab-ci/coverage/ci_coverage_collect.sh
artifacts:
paths:
- all_probabilities
- environment
- coverage_data
expire_in: 30 days
###############################################
# Print coverage results
total_coverage:
stage: coverage
dependencies:
- verify_options
- verify_tutorial
- verify_probabilities
script:
- echo "Generating coverage report"
- .gitlab-ci/coverage/ci_coverage_summary.sh
artifacts:
paths:
- coverage_summary
expire_in: 30 days
# Publish coverage results
pages:
stage: deploy
dependencies:
- total_coverage
script:
- echo "Publishing pages"
- mv coverage_summary/ public/
artifacts:
paths:
- public
expire_in: 30 days
\ No newline at end of file
......@@ -15,21 +15,21 @@ echo "building BioEM with ARCH=$ARCHITECTURE"
case "$ARCHITECTURE" in
intel-mpi)
module load intel/15.0 impi/5.0.3 fftw/3.3.4 boost/intel/1.56
module load intel impi fftw
rm -rf build/; mkdir build
cd build
cmake .. -DCMAKE_CXX_COMPILER=icpc -DCMAKE_C_COMPILER=icc
cmake .. -DCMAKE_CXX_COMPILER=icpc -DCMAKE_C_COMPILER=icc -DUSE_MPI=on -DUSE_OPENMP=on -DUSE_CUDA=off
make VERBOSE=1 -j 2
;;
gcc-mpi)
module load gcc/4.9 impi/5.0.3 fftw/gcc/3.3.4 boost/gcc/1.56
module load gcc/4.9 impi/5.0.3 fftw/gcc/3.3.4
rm -rf build/; mkdir build
cd build
cmake .. -DCMAKE_CXX_COMPILER=g++ -DCMAKE_C_COMPILER=gcc -DCMAKE_BUILD_TYPE=Debug -DCMAKE_CXX_FLAGS_DEBUG='-g -Wall --coverage' -DCMAKE_C_FLAGS_DEBUG='-g -Wall --coverage'
make VERBOSE=1 -j 2
;;
gcc-mpi-cuda)
module load gcc/4.9 cuda/7.5 impi/5.0.3 fftw/gcc/3.3.4 boost/gcc/1.56
module load gcc/4.9 cuda/7.5 impi/5.0.3 fftw/gcc/3.3.4
rm -rf build/; mkdir build
cd build
cmake .. -DCMAKE_CXX_COMPILER=g++ -DCMAKE_C_COMPILER=gcc -DUSE_CUDA=on -DCUDA_CUDA_LIBRARY=$CUDA_HOME/lib64/stubs/libcuda.so
......
#!/bin/bash
PATH=${HOME}/bin:${PATH}
set -e
set pipefail
. /etc/profile.d/modules.sh
module load gcc/4.9
cov_dir=coverage_data
prefix=$cov_dir/coverage_${CI_BUILD_REF}_${CI_BUILD_NAME}
mkdir -p $cov_dir/
for dir in $(find -name "*.gcno" -print0 | xargs -0 -n 1 dirname | sort -u); do
dir=${dir#./};
lcov -c -b $PWD -d $dir --to-package $prefix.${dir//\//_}.package;
done
#!/bin/bash
PATH=${HOME}/bin:${PATH}
. /etc/profile.d/modules.sh
module load gcc/4.9
for p in coverage_data/coverage_${CI_BUILD_REF}*.package ; do
cov_file=$PWD/${p%.package}.info
lcov -c -b $PWD --from-package $p --output-file $cov_file
lcov -r $cov_file "/usr/*" -o $cov_file
lcov -r $cov_file "/usr/include/*" -o $cov_file
lcov -r $cov_file "*4.9.3*" -o $cov_file
lcov -r $cov_file "/afs/*" -o $cov_file
lcov -r $cov_file "BioEM/include/MersenneTwister.h" -o $cov_file
lcov -r $cov_file "BioEM/include/autotuner.h" -o $cov_file
lcov -r $cov_file "BioEM/autotuner.cpp" -o $cov_file
done
lcov $(for i in coverage_data/coverage_${CI_BUILD_REF}_*.info; do echo -a $i; done) -o coverage_data/coverage_${CI_BUILD_REF}.info
lcov --summary coverage_data/coverage_${CI_BUILD_REF}.info 2>&1 | \
awk '/lines|functions/ {
gsub(/\.*:$/, "", $1);
gsub(/^./, "", $3);
printf "%s: %s (%s of %s), ", $1, $2, $3, $5;
}
/branches/ {
print "";
}' | \
sed 's/^/__COVERAGE__:/; s/, $//;'
mkdir -p coverage_summary
genhtml coverage_data/coverage_${CI_BUILD_REF}.info -o coverage_summary
#!/bin/bash
# Tests assume 4 nodes, with two GPUs per node.
# If this is not the case, adapt the configuration!
# Input paths
BUILD_DIR=${1}
INPUT_DIR=${2}
datafile=${3}
# Environment variables
export BIOEM="$BUILD_DIR/bioEM"
export FFTALGO=1 # Although this is needed only for BioEM<2.0
export GPUDEVICE=-1
export KMP_AFFINITY=compact,granularity=core,1
export BIOEM_DEBUG_OUTPUT=1
# Environment capture
> $datafile
$HOME/BioEM_project/get_info.sh $datafile
rm -rf Output_Probabilities
###################################
# Exp 1: Large benchmark, with best config
echo "* EXP 1: LARGE BENCH OUTPUT (FACTOR 10)" >> $datafile
#export BIOEM_ALGO=1
export GPU=1
#export GPUWORKLOAD=-1
#export BIOEM_CUDA_THREAD_COUNT=256
export BIOEM_DEBUG_BREAK=1350
#export BIOEM_DEBUG_NMAPS=2000
export OMP_NUM_THREADS=12
#export BIOEM_PROJECTIONS_AT_ONCE=1
export NP=2
# Running
mpiexec -perhost $NP $BIOEM --Inputfile $INPUT_DIR/INPUT_FRH_Sep2016 --Modelfile $INPUT_DIR/Mod_X-ray_PDB --Particlesfile $INPUT_DIR/2000FRH_Part >> $datafile
# Saving probabilities
echo "* EXP 1: PROBABILITIES" >> $datafile
cat Output_Probabilities >> $datafile
# Cleanup
rm -rf Output_Probabilities
unset BIOEM_ALGO GPU GPUWORKLOAD BIOEM_CUDA_THREAD_COUNT BIOEM_DEBUG_BREAK BIOEM_DEBUG_NMAPS OMP_NUM_THREADS BIOEM_PROJECTIONS_AT_ONCE NP
###################################
# Exp 2: Middle benchmark, with best config
echo "* EXP 2: MIDDLE BENCH OUTPUT (FACTOR 10)" >> $datafile
export BIOEM_ALGO=1
export GPU=1
export GPUWORKLOAD=-1
#export BIOEM_CUDA_THREAD_COUNT=256
export BIOEM_DEBUG_BREAK=600
export BIOEM_DEBUG_NMAPS=100
export OMP_NUM_THREADS=12
export BIOEM_PROJECTIONS_AT_ONCE=2
export NP=2
# Running
mpiexec -perhost $NP $BIOEM --Inputfile $INPUT_DIR/INPUT_FRH_Sep2016 --Modelfile $INPUT_DIR/Mod_X-ray_PDB --Particlesfile $INPUT_DIR/2000FRH_Part >> $datafile
# Saving probabilities
echo "* EXP 2: MIDDLE BENCH PROBABILITIES" >> $datafile
cat Output_Probabilities >> $datafile
# Cleanup
rm -rf Output_Probabilities
unset BIOEM_ALGO GPU GPUWORKLOAD BIOEM_CUDA_THREAD_COUNT BIOEM_DEBUG_BREAK BIOEM_DEBUG_NMAPS OMP_NUM_THREADS BIOEM_PROJECTIONS_AT_ONCE NP
###################################
# Exp 3: Small benchmark, with best ALGO=1 config
echo "* EXP 3: SMALL BENCH OUTPUT (FACTOR 10)" >> $datafile
x=1
mod=`awk -v x=$x 'NR==x' $INPUT_DIR/ListParticles`
export GPU=0
export BIOEM_ALGO=1
#export GPUWORKLOAD=100
#export BIOEM_CUDA_THREAD_COUNT=256
export BIOEM_DEBUG_BREAK=600
export BIOEM_DEBUG_NMAPS=1
export OMP_NUM_THREADS=1
export BIOEM_PROJECTIONS_AT_ONCE=1
export NP=96
# Running
mpiexec -perhost $NP $BIOEM --Modelfile $INPUT_DIR/CA-Ref_Nosym.pdb --ReadPDB --Inputfile $INPUT_DIR/Param_Input_AP-1 --Particlesfile $INPUT_DIR/IMAGES/$mod --ReadMRC --ReadOrientation $INPUT_DIR/LISTS_ORIENTATIONS/List_ori_$x >> $datafile
# Saving probabilities
echo "* EXP 3: SMALL BENCH PROBABILITIES" >> $datafile
cat Output_Probabilities >> $datafile
# Cleanup
rm -rf Output_Probabilities
unset BIOEM_ALGO GPU GPUWORKLOAD BIOEM_CUDA_THREAD_COUNT BIOEM_DEBUG_BREAK BIOEM_DEBUG_NMAPS OMP_NUM_THREADS BIOEM_PROJECTIONS_AT_ONCE NP
###################################
# Exp 4: Small benchmark, with CPU-only ALGO=2 config
echo "* EXP 4: SMALL BENCH OUTPUT (FACTOR 10)" >> $datafile
x=1
mod=`awk -v x=$x 'NR==x' $INPUT_DIR/ListParticles`
export GPU=0
export BIOEM_ALGO=2
#export GPUWORKLOAD=100
#export BIOEM_CUDA_THREAD_COUNT=256
export BIOEM_DEBUG_BREAK=600
export BIOEM_DEBUG_NMAPS=1
export OMP_NUM_THREADS=12
export BIOEM_PROJECTIONS_AT_ONCE=12
export NP=2
# Running
mpiexec -perhost $NP $BIOEM --Modelfile $INPUT_DIR/CA-Ref_Nosym.pdb --ReadPDB --Inputfile $INPUT_DIR/Param_Input_AP-1 --Particlesfile $INPUT_DIR/IMAGES/$mod --ReadMRC --ReadOrientation $INPUT_DIR/LISTS_ORIENTATIONS/List_ori_$x >> $datafile
# Saving probabilities
echo "* EXP 4: SMALL BENCH PROBABILITIES" >> $datafile
cat Output_Probabilities >> $datafile
# Cleanup
rm -rf Output_Probabilities
unset BIOEM_ALGO GPU GPUWORKLOAD BIOEM_CUDA_THREAD_COUNT BIOEM_DEBUG_BREAK BIOEM_DEBUG_NMAPS OMP_NUM_THREADS BIOEM_PROJECTIONS_AT_ONCE NP
###################################
# Exp 5: Small benchmark, with best ALGO=2 config
echo "* EXP 5: SMALL BENCH OUTPUT (FACTOR 10)" >> $datafile
x=1
mod=`awk -v x=$x 'NR==x' $INPUT_DIR/ListParticles`
export GPU=1
export BIOEM_ALGO=2
export GPUWORKLOAD=100
export BIOEM_CUDA_THREAD_COUNT=1024
export BIOEM_DEBUG_BREAK=600
export BIOEM_DEBUG_NMAPS=1
export OMP_NUM_THREADS=12
#export BIOEM_PROJECTIONS_AT_ONCE=1
export NP=2
# Running
mpiexec -perhost $NP $BIOEM --Modelfile $INPUT_DIR/CA-Ref_Nosym.pdb --ReadPDB --Inputfile $INPUT_DIR/Param_Input_AP-1 --Particlesfile $INPUT_DIR/IMAGES/$mod --ReadMRC --ReadOrientation $INPUT_DIR/LISTS_ORIENTATIONS/List_ori_$x >> $datafile
# Saving probabilities
echo "* EXP 5: SMALL BENCH PROBABILITIES" >> $datafile
cat Output_Probabilities >> $datafile
# Cleanup
rm -rf Output_Probabilities
unset BIOEM_ALGO GPU GPUWORKLOAD BIOEM_CUDA_THREAD_COUNT BIOEM_DEBUG_BREAK BIOEM_DEBUG_NMAPS OMP_NUM_THREADS BIOEM_PROJECTIONS_AT_ONCE NP
......@@ -8,7 +8,6 @@ module load gcc/4.9
mkdir $OUTPUT_DIR
lcov --no-checksum --capture --directory $INPUT_DIR --output-file coverage.info
lcov -r coverage.info */boost/* --output-file coverage.info
lcov -r coverage.info /usr/* --output-file coverage.info
lcov -r coverage.info /usr/include/* --output-file coverage.info
lcov -r coverage.info *4.9.3* --output-file coverage.info
......
#!/bin/bash
# FAIL in case of any error
set -e
function exittrap {
echo "$0 failed"
exit 1
}
trap exittrap ERR INT TERM
set -x
# Environment
ARCH=$1
NUMPROCS=${2:-4}
NUMTHRDS=${3:-2}
BIOEM=../build/bioEM
. /etc/profile.d/modules.sh
module load impi
export OMP_NUM_THREADS=$NUMTHRDS
export OMP_PLACES=cores
export OMP_STACKSIZE=128M
cat /proc/cpuinfo > environment
env >> environment
> ang_prob
cp -r Tutorial_BioEM test2
cd test2
# Compare results with a certain numerical precision
NUMDIFF=/home/rzg/soft/numdiff/bin/numdiff
# Running tests
echo "@testing help output (error allowed)"
$BIOEM --help || true
echo "@testing debugging options"
BIOEM_DEBUG_OUTPUT=2 BIOEM_DEBUG_BREAK=2 BIOEM_DEBUG_NMAPS=1 $BIOEM --Inputfile Param_Input --Modelfile Model_Text --Particlesfile Text_Image_Form
echo "@testing prior probabilities"
$BIOEM --Modelfile Model_Text --Particlesfile Text_Image_Form --Inputfile Param_Input_Priors --ReadOrientation Euler_Angle_List_Prior
$NUMDIFF --relative=1e-2 Output_Probabilities test_outputs/Output_Probabilities_prior
echo "@testing print map model"
$BIOEM --Modelfile Model.pdb --ReadPDB --PrintBestCalMap Param_Print_MAP
# Random generator is involved, so there is no sence to compare the error
#$NUMDIFF --relative=1e-4 BESTMAP test_outputs/BESTMAP
echo "@testing print posterior as a function of orientations"
mpiexec -n $NUMPROCS $BIOEM --Inputfile Param_Input_WritePAng --Modelfile Model_Text --Particlesfile Text_Image_Form
cat ANG_PROB > ang_prob
$NUMDIFF --overview ANG_PROB test_outputs/ANG_PROB || true
$NUMDIFF --relative=1e-2 ANG_PROB test_outputs/ANG_PROB
echo "@successfully finished testing several specific options"
#!/bin/bash
# FAIL in case of any error
set -e
function exittrap {
echo "$0 failed"
exit 1
}
trap exittrap ERR INT TERM
set -x
# Environment
ARCH=$1
NUMPROCS=${2:-4}
NUMTHRDS=${3:-2}
BIOEM=../build/bioEM
. /etc/profile.d/modules.sh
module load impi
export OMP_NUM_THREADS=$NUMTHRDS
export OMP_PLACES=cores
export OMP_STACKSIZE=128M
cat /proc/cpuinfo > environment
env >> environment
> all_probabilities
cp -r Tutorial_BioEM test3
cd test3
# Compare results with a certain numerical precision
NUMDIFF=/home/rzg/soft/numdiff/bin/numdiff
# Running experiments
for algo in 1 2
do
for gpu in 0
do
for proj in 1 $OMP_NUM_THREADS
do
echo "@running probabilities for BIOEM_ALGO=$algo GPU=$gpu BIOEM_PROJ_CONV_AT_ONCE=$proj"
BIOEM_ALGO=$algo GPU=$gpu BIOEM_PROJ_CONV_AT_ONCE=$proj BIOEM_DEBUG_OUTPUT=2 mpiexec -n $NUMPROCS $BIOEM --Inputfile Param_Input --Modelfile Model_Text --Particlesfile Text_Image_Form
echo "@probabilities for BIOEM_ALGO=$algo GPU=$gpu BIOEM_PROJ_CONV_AT_ONCE=$proj"
cat Output_Probabilities
cat Output_Probabilities >> all_probabilities
echo "@comparing Output_Probabilities test_outputs/Output_Probabilities_ref"
$NUMDIFF --overview Output_Probabilities test_outputs/Output_Probabilities_ref || true
echo "@verifying Output_Probabilities test_outputs/Output_Probabilities_ref"
$NUMDIFF --relative=1e-1 Output_Probabilities test_outputs/Output_Probabilities_ref
done
done
done
echo "@successfully finished testing probabilities for both algorithms"
#!/bin/bash
# FAIL in case of any error
set -e
function exittrap {
echo "$0 failed"
exit 1
}
trap exittrap ERR INT TERM
set -x
# Environment
BIOEM=../${1}
TUTORIAL_DIR=$2
echo "BioEM path: ${BIOEM}"
echo "Tutorial directory path: ${TUTORIAL_DIR}"
rm -rf testTutorial
cp -r $TUTORIAL_DIR testTutorial
cd testTutorial
# Running tests
echo "@testing all tutorial options"
echo "@@ Text model - Text Image"
$BIOEM --Inputfile Param_Input --Modelfile Model_Text --Particlesfile Text_Image_Form
echo "@@ PDB Model - Text Image "
$BIOEM --Inputfile Param_Input --Modelfile Model.pdb --ReadPDB --Particlesfile Text_Image_Form
echo "@@ PDB Model - One MRC Image"
$BIOEM --Inputfile Param_Input --Modelfile Model.pdb --ReadPDB --Particlesfile OneImage.mrc --ReadMRC
echo "@@ PDB Model - Multiple MRCs"
$BIOEM --Inputfile Param_Input --Modelfile Model.pdb --ReadPDB --Particlesfile ListMRC --ReadMRC --ReadMultipleMRC
echo "@@ Read Euler angles from file"
$BIOEM --Inputfile Param_Input --Modelfile Model.pdb --ReadPDB --Particlesfile Text_Image_Form --ReadOrientation Euler_Angle_List
echo "@@ Read quaternions from file"
$BIOEM --Inputfile Param_Input_Quat --Modelfile Model.pdb --ReadPDB --Particlesfile Text_Image_Form --ReadOrientation Quat_list_Small
echo "@@ Input-parameter suggestions"
echo "Disabled as it is too long for the current CI"
#$BIOEM --Inputfile Param_Input_Quat --Modelfile Model.pdb --ReadPDB --Particlesfile Text_Image_Form --ReadOrientation List_Quat_ProRun
echo "@@ Dump particle-images"
$BIOEM --Inputfile Param_Input --Modelfile Model.pdb --ReadPDB --Particlesfile ListMRC --ReadMRC --ReadMultipleMRC --DumpMaps
echo "@@ Load particle-images"
$BIOEM --Inputfile Param_Input --Modelfile Model.pdb --ReadPDB --LoadMapDump
echo "@@ Including prior probabilities"
$BIOEM --Modelfile Model_Text --Particlesfile Text_Image_Form --Inputfile Param_Input_Priors --ReadOrientation Euler_Angle_List_Prior
## Dont test for the PrintBestCalMap as it is in the other test (and not in Tutorial anymore)
echo "@successfully finished testing all tutorial options"
************************* HEADER:: NOTATION *******************************************
RefMap: MapNumber ; alpha[rad] - beta[rad] - gamma[rad] - logP - cal log Probability + Constant: Numerical Const.+ log (volume) + prior ang
************************* HEADER:: NOTATION *******************************************
0 -1.5708 1.15928 2.19912 -68003.1 Separated: 0.0114472 683.164 -68686.3
0 -1.5708 1.15928 -0.942478 -68035.7 Separated: 0.0106541 650.494 -68686.3
\ No newline at end of file
************************* HEADER:: NOTATION *******************************************
Notation= RefMap: MapNumber ; LogProb natural logarithm of posterior Probability ; Constant: Numerical Const. for adding Probabilities
Notation= RefMap: MapNumber ; Maximizing Param: MaxLogProb - alpha[rad] - beta[rad] - gamma[rad] - CTF amp - CTF defocus - CTF B-Env - center x - center y - normalization - offsett
**** Remark: Using Prior Proability in Angles ****
************************* HEADER:: NOTATION *******************************************
RefMap: 0 LogProb: -68442.4 Constant: 240.249
RefMap: 0 Maximizing Param: -68442.4 0.836 [] 1.369 [] 2.64 [] 0.27 [] 2.1 [micro-m] 10 [A²] -2 [pix] -8 [pix] -0.00313061 [] 0.0288172 []
************************* HEADER:: NOTATION *******************************************
Notation= RefMap: MapNumber ; LogProb natural logarithm of posterior Probability ; Constant: Numerical Const. for adding Probabilities
Notation= RefMap: MapNumber ; Maximizing Param: MaxLogProb - alpha[rad] - beta[rad] - gamma[rad] - CTF amp - CTF defocus - CTF B-Env - center x - center y - normalization - offsett
************************* HEADER:: NOTATION *******************************************
RefMap: 0 LogProb: -68317.6 Constant: 363.117
RefMap: 0 Maximizing Param: -68317.6 -1.5708 [] 1.0472 [] 1.5708 [] 0.1 [] 5 [micro-m] 200.25 [A²] -2 [pix] -6 [pix] -0.00154624 [] 0.0141542 []
\ No newline at end of file
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment