temporary hack to get back good performance, this needs to be studied (and...

temporary hack to get back good performance, this needs to be studied (and optimized) in more details

.gitlab-ci.yml

+image: gitlab-registry.mpcdf.mpg.de/mpcdf/module-image
+
 before_script:
   - export LANG=C
+  - export GCC_VER=6
 
 # For some reason doing shallow fetch doesnt work
 #variables:
@@ -17,16 +20,21 @@ stages:
 build-gcc-mpi:
   stage: build
   script:
+    - module load git
+    - git clone https://gitlab-ci-token:${CI_JOB_TOKEN}@gitlab.mpcdf.mpg.de/MPIBP-Hummer/BioEM-tutorials.git
     - .gitlab-ci/build/run_build.sh gcc-mpi
   artifacts:
     paths:
     - build/
+    - BioEM-tutorials/Tutorial_BioEM
     expire_in: 30 days
 
 # default GCC build, minimal test
 build-gcc-mpi-test:
   stage: build
   script:
+    - module load git
+    - git clone https://gitlab-ci-token:${CI_JOB_TOKEN}@gitlab.mpcdf.mpg.de/MPIBP-Hummer/BioEM-tutorials.git
     - .gitlab-ci/build/run_build.sh gcc-mpi
     - .gitlab-ci/test/run_test.sh gcc-mpi 4 2
     - .gitlab-ci/test/run_coverage.sh build/CMakeFiles/bioEM.dir coverage_test
@@ -70,7 +78,7 @@ verify_tutorial:
   dependencies:
     - build-gcc-mpi
   script:
-    - .gitlab-ci/test/verify_tutorial.sh build/bioEM Tutorial_BioEM
+    - .gitlab-ci/test/verify_tutorial.sh build/bioEM BioEM-tutorials/Tutorial_BioEM
     - .gitlab-ci/coverage/ci_coverage_collect.sh
   artifacts:
     paths:
@@ -120,4 +128,4 @@ pages:
   artifacts:
     paths:
       - public
-    expire_in: 30 days
\ No newline at end of file
+    expire_in: 30 days

.gitlab-ci/build/run_build.sh

 #!/bin/bash
 
-
 ARCHITECTURE=$1
 DEBUG=${2:-no}
 
-. /etc/profile.d/modules.sh
-module use /afs/ipp/common/usr/modules.2014/@sys/modulefiles/GPU
-
-
+module purge
 module load cmake
 
 echo "building BioEM with ARCH=$ARCHITECTURE"
 
- 
 case "$ARCHITECTURE" in
 intel-mpi)
-  module load intel impi fftw
+  module load intel impi fftw-mpi
   rm -rf build/; mkdir build
   cd build
-  cmake .. -DCMAKE_CXX_COMPILER=icpc -DCMAKE_C_COMPILER=icc -DUSE_MPI=on -DUSE_OPENMP=on -DUSE_CUDA=off
+  cmake .. -DFFTWF_LIBRARIES=$FFTW_HOME/lib/libfftw3f.a -DFFTWF_INCLUDE_DIR=$FFTW_HOME/include -DCMAKE_CXX_COMPILER=icpc -DCMAKE_C_COMPILER=icc -DUSE_MPI=on -DUSE_OPENMP=on -DUSE_CUDA=off
   make VERBOSE=1 -j 2
   ;;
 gcc-mpi)
-  module load gcc/4.9 impi/5.0.3 fftw/gcc/3.3.4
+  module load gcc/$GCC_VER impi fftw-mpi
   rm -rf build/; mkdir build
   cd build
-  cmake .. -DCMAKE_CXX_COMPILER=g++ -DCMAKE_C_COMPILER=gcc -DCMAKE_BUILD_TYPE=Debug -DCMAKE_CXX_FLAGS_DEBUG='-g -Wall --coverage' -DCMAKE_C_FLAGS_DEBUG='-g -Wall --coverage'
+  cmake .. -DFFTWF_LIBRARIES=$FFTW_HOME/lib/libfftw3f.a -DFFTWF_INCLUDE_DIR=$FFTW_HOME/include -DCMAKE_CXX_COMPILER=g++ -DCMAKE_C_COMPILER=gcc -DCMAKE_BUILD_TYPE=Debug -DCMAKE_CXX_FLAGS_DEBUG='-g -Wall --coverage' -DCMAKE_C_FLAGS_DEBUG='-g -Wall --coverage'
   make VERBOSE=1 -j 2
   ;;
 gcc-mpi-cuda)
-  module load gcc/4.9 cuda/7.5 impi/5.0.3 fftw/gcc/3.3.4
+  module load gcc/$GCC_VER cuda impi fftw-mpi
   rm -rf build/; mkdir build
   cd build
-  cmake .. -DCMAKE_CXX_COMPILER=g++ -DCMAKE_C_COMPILER=gcc -DUSE_CUDA=on -DCUDA_CUDA_LIBRARY=$CUDA_HOME/lib64/stubs/libcuda.so
+  cmake .. -DFFTWF_LIBRARIES=$FFTW_HOME/lib/libfftw3f.a -DFFTWF_INCLUDE_DIR=$FFTW_HOME/include -DCMAKE_CXX_COMPILER=g++ -DCMAKE_C_COMPILER=gcc -DUSE_CUDA=on -DCUDA_CUDA_LIBRARY=$CUDA_HOME/lib64/stubs/libcuda.so
   make VERBOSE=1 -j 2
   ;;
 *)

.gitlab-ci/coverage/ci_coverage_collect.sh

@@ -5,8 +5,9 @@ PATH=${HOME}/bin:${PATH}
 set -e
 set pipefail
 
-. /etc/profile.d/modules.sh
-module load gcc/4.9
+module purge
+module load gcc/$GCC_VER
+module load lcov
 
 cov_dir=coverage_data
 prefix=$cov_dir/coverage_${CI_BUILD_REF}_${CI_BUILD_NAME}

.gitlab-ci/coverage/ci_coverage_summary.sh

 #!/bin/bash
 PATH=${HOME}/bin:${PATH}
 
-. /etc/profile.d/modules.sh
-module load gcc/4.9
+module purge
+module load gcc/$GCC_VER
+module load lcov
 
 for p in coverage_data/coverage_${CI_BUILD_REF}*.package ; do
         cov_file=$PWD/${p%.package}.info
         lcov -c -b $PWD --from-package $p --output-file $cov_file
         lcov -r $cov_file "/usr/*" -o $cov_file
         lcov -r $cov_file "/usr/include/*" -o $cov_file
-        lcov -r $cov_file "*4.9.3*" -o $cov_file
+        lcov -r $cov_file "*$GCC_VER*" -o $cov_file
         lcov -r $cov_file "/afs/*" -o $cov_file
-        lcov -r $cov_file "BioEM/include/MersenneTwister.h" -o $cov_file
-        lcov -r $cov_file "BioEM/include/autotuner.h" -o $cov_file
-        lcov -r $cov_file "BioEM/autotuner.cpp" -o $cov_file
+        lcov -r $cov_file "include/MersenneTwister.h" -o $cov_file
+        lcov -r $cov_file "include/autotuner.h" -o $cov_file
+        lcov -r $cov_file "autotuner.cpp" -o $cov_file
 done
 
 lcov $(for i in coverage_data/coverage_${CI_BUILD_REF}_*.info; do echo -a $i; done) -o coverage_data/coverage_${CI_BUILD_REF}.info

.gitlab-ci/test/run_coverage.sh

@@ -3,15 +3,13 @@
 INPUT_DIR=$1
 OUTPUT_DIR=$2
 
-. /etc/profile.d/modules.sh
-module load gcc/4.9
+module purge
+module load gcc/$GCC_VER
+module load lcov
 
 mkdir $OUTPUT_DIR
-lcov --no-checksum --capture --directory $INPUT_DIR --output-file coverage.info 
+lcov --no-checksum --capture --directory $INPUT_DIR --output-file coverage.info
 lcov -r coverage.info /usr/*    --output-file coverage.info
 lcov -r coverage.info /usr/include/*    --output-file coverage.info
-lcov -r coverage.info *4.9.3*    --output-file coverage.info
+lcov -r coverage.info *$GCC_VER*    --output-file coverage.info
 genhtml coverage.info --output-directory $OUTPUT_DIR
-
-
-  

.gitlab-ci/test/run_test.sh

 #!/bin/bash
 
-#env
+env
 
 ARCH=$1
 NUMPROCS=${2:-4}
 NUMTHRDS=${3:-2}
-. /etc/profile.d/modules.sh
 
-module load impi
+module purge
+module load gcc/$GCC_VER impi fftw-mpi
 
 export OMP_NUM_THREADS=$NUMTHRDS
 export OMP_PLACES=cores
 export OMP_STACKSIZE=128M
 
-cp -r Tutorial_BioEM test1
+cp -r BioEM-tutorials/Tutorial_BioEM test1
 cd test1
 mpiexec -n $NUMPROCS ../build/bioEM --Inputfile Param_Input --Modelfile Model_Text --Particlesfile Text_Image_Form
-
-
-  

.gitlab-ci/test/verify_options.sh

@@ -14,9 +14,8 @@ ARCH=$1
 NUMPROCS=${2:-4}
 NUMTHRDS=${3:-2}
 BIOEM=../build/bioEM
-. /etc/profile.d/modules.sh
 
-module load impi
+module load gcc/$GCC_VER impi fftw-mpi
 
 export OMP_NUM_THREADS=$NUMTHRDS
 export OMP_PLACES=cores
@@ -26,11 +25,12 @@ cat /proc/cpuinfo > environment
 env >> environment
 > ang_prob
 
-cp -r Tutorial_BioEM test2
+cp -r BioEM-tutorials/Tutorial_BioEM test2
 cd test2
 
 # Compare results with a certain numerical precision
-NUMDIFF=/home/rzg/soft/numdiff/bin/numdiff
+module load numdiff
+NUMDIFF=numdiff
 
 # Running tests
 echo "@testing help output (error allowed)"

.gitlab-ci/test/verify_probabilities.sh

@@ -14,9 +14,8 @@ ARCH=$1
 NUMPROCS=${2:-4}
 NUMTHRDS=${3:-2}
 BIOEM=../build/bioEM
-. /etc/profile.d/modules.sh
 
-module load impi
+module load gcc/$GCC_VER impi fftw-mpi
 
 export OMP_NUM_THREADS=$NUMTHRDS
 export OMP_PLACES=cores
@@ -26,11 +25,12 @@ cat /proc/cpuinfo > environment
 env >> environment
 > all_probabilities
 
-cp -r Tutorial_BioEM test3
+cp -r BioEM-tutorials/Tutorial_BioEM test3
 cd test3
 
 # Compare results with a certain numerical precision
-NUMDIFF=/home/rzg/soft/numdiff/bin/numdiff
+module load numdiff
+NUMDIFF=numdiff
 
 # Running experiments
 for algo in 1 2

.gitlab-ci/test/verify_tutorial.sh

@@ -9,6 +9,8 @@ function exittrap {
 trap exittrap ERR INT TERM
 set -x
 
+module load gcc/$GCC_VER impi fftw-mpi
+
 # Environment
 BIOEM=../${1}
 TUTORIAL_DIR=$2
@@ -40,9 +42,13 @@ $BIOEM --Inputfile Param_Input --Modelfile Model.pdb --ReadPDB --Particlesfile T
 echo "@@ Read quaternions from file"
 $BIOEM --Inputfile Param_Input_Quat --Modelfile Model.pdb --ReadPDB --Particlesfile Text_Image_Form --ReadOrientation Quat_list_Small
 
+echo "@@ MRC Model - One MRC Image"
+echo "Disabled as it is returning many warnings"
+#$BIOEM --Inputfile Param_ModelMRC --Modelfile Model_MRC.mrc --ReadModelMRC --Particlesfile particle3.mrcs --ReadMRC --ReadOrientation Quat_list_Small
+
 echo "@@ Input-parameter suggestions"
 echo "Disabled as it is too long for the current CI"
-#$BIOEM --Inputfile Param_Input_Quat --Modelfile Model.pdb --ReadPDB --Particlesfile Text_Image_Form --ReadOrientation List_Quat_ProRun
+#$BIOEM --Inputfile Param_ProRun --Modelfile Model.pdb --ReadPDB --Particlesfile Text_Image_Form --ReadOrientation List_Quat_ProRun
 
 echo "@@ Dump particle-images"
 $BIOEM --Inputfile Param_Input --Modelfile Model.pdb --ReadPDB --Particlesfile ListMRC --ReadMRC --ReadMultipleMRC --DumpMaps
@@ -50,6 +56,15 @@ $BIOEM --Inputfile Param_Input --Modelfile Model.pdb --ReadPDB --Particlesfile L
 echo "@@ Load particle-images"
 $BIOEM --Inputfile Param_Input --Modelfile Model.pdb --ReadPDB --LoadMapDump
 
+echo "@@ Dump model"
+$BIOEM --Inputfile Param_Input --Modelfile Model.pdb --ReadPDB --Particlesfile ListMRC --ReadMRC --ReadMultipleMRC --DumpModel
+
+echo "@@ Load model"
+$BIOEM --Inputfile Param_Input --Particlesfile ListMRC --ReadMRC --ReadMultipleMRC --LoadModelDump
+
+echo "@@ Check model"
+$BIOEM --Inputfile Param_Input --Modelfile Model_Text --Particlesfile Text_Image_Form --PrintCOORDREAD
+
 echo "@@ Including prior probabilities"
 $BIOEM  --Modelfile Model_Text --Particlesfile Text_Image_Form --Inputfile Param_Input_Priors  --ReadOrientation Euler_Angle_List_Prior
 

README.md

 # BioEM:  Bayesian inference of Electron Microscopy
-# 2.0 VERSION: January, 2018
+# 2.1 VERSION: July, 2019
 
 ## Build status and test coverage
 
@@ -40,10 +40,14 @@ For a detailed description of the BioEM software see the [BioEM documentation](h
 	  --Inputfile       arg (Mandatory) Name of input parameter file
 	  --ReadOrientation arg (Optional) Read file name containing orientations
 	  --ReadPDB             (Optional) If reading model file in PDB format
+	  --ReadModelMRC        (Optional) If reading model file in MRC format
 	  --ReadMRC             (Optional) If reading particle file in MRC format
-	  --ReadMultipleMRC     (Optional) If reading Multiple MRCs
+	  --ReadMultipleMRC     (Optional) If reading multiple MRCs
 	  --DumpMaps            (Optional) Dump maps after they were read from particle-image file
 	  --LoadMapDump         (Optional) Read Maps from dump option
+	  --DumpModel           (Optional) Dump model after it was read from model file
+	  --LoadModelDump       (Optional) Read model from dump option
+	  --PrintCOORDREAD      (Optional) Print model coordinates
 	  --OutputFile      arg (Optional) For changing the outputfile name
 	  --help                (Optional) Produce help message
 
@@ -52,9 +56,9 @@ Details for the inputfiles and formats are provided in sections [1](http://bioem
 ### Output
 
 * Main output file with, default name "Output_Probabilities", provides the posterior probability for each image, as well as the parameters that give a maximum value of the posterior:
-     
+
      RefMap #(number Particle Map) Probability  #(log(P))
-     
+
      RefMap #(number Particle Map) Maximizing Param: #(Euler Angles) #(PSF parameters) #(center displacement)
 
      **Important: It is recommended to compare log(P) with respect to other Models or to Noise as in [1].
@@ -62,8 +66,8 @@ Details for the inputfiles and formats are provided in sections [1](http://bioem
 * (Optional) Write the posterior probabilities as a function of the orientations (key word: WRITE_PROB_ANGLES in InputFile, see [documentation](http://bioem.readthedocs.io/en/latest/index.html#std:inpar-WRITE_PROB_ANGLES)).
 
 ### Tutorial
- 
-A directory with example EM particles, models, and input files are provided in the Tutorial_BioEM directory. 
+
+A directory with example EM particles, models, and input files are provided in the Tutorial_BioEM directory of the [BioEM-tutorials subproject](https://github.com/bio-phys/BioEM-tutorials).
 The tutorial is described in [section 4 of the BioEM documentation](http://bioem.readthedocs.io/en/latest/index.html#tutorial). 
 
 
@@ -85,24 +89,24 @@ make VERBOSE=1
 Dependencies and software requirements:
 
 * Compiler: a modern C++ compiler which is OpenMP compliant
-              and (optionally) complies with CUDAs nvcc
-              (tested with Intel icpc 12-16, GCC 4.7-5.1)
-    -> adapt the name of the compiler using ccmake 
+       and (optionally) complies with CUDAs nvcc
+       (tested with Intel icpc 12-16, GCC 4.7-5.1)
+    -> adapt the name of the compiler using ccmake
 
     for free software see: https://gcc.gnu.org/
 
 * MPI: the Message Passing Standard library
-         (tested with Intel MPI 4.1-5.1, IBM PE 1.3-1.4)
-    -> adapt the names of the MPI compiler wrappers using ccmake 
+       (tested with Intel MPI 4.1-5.1, IBM PE 1.3-1.4)
+    -> adapt the names of the MPI compiler wrappers using ccmake
 
     for free software see: http://www.open-mpi.de/
-          
+
 * FFTW: a serial but fully thread-safe fftw3 installation or equivalent (tested with fftw 3.3)
      -> point environment variable $FFTW_ROOT to a FFTW3 installation or use ccmake to specify
 
-    for free software see: http://www.fftw.org 
+    for free software see: http://www.fftw.org
 
-* CUDA (required to build and run the GPU version) 
+* CUDA (required to build and run the GPU version)
 
     for free software see: https://developer.nvidia.com/cuda-downloads
 
@@ -115,7 +119,7 @@ The BioEM performance variables enhance or modify the code's computational perfo
 They should be tuned for the specific computing node characteristics where bioEM is executed, e.g., select the number of GPUs to use, OpenMP 
 threads etc. These are passed via environment variables. See [section 4 of the BioEM documentation](http://bioem.readthedocs.io/en/latest/index.html#performance) for a detailed description.
 
-### License 
+### License
 
 The BioEM program is a free software, under the terms of the GNU General Public License as published by the Free Software Foundation, version 3 of the License. 
 This program is distributed in the hope that it will be useful, but without any warranty.  See License.txt for more details.

Tutorial_BioEM/Euler_Angle_List

-2
-    0.836000    1.369000    2.640000
-   -1.375000    0.400000    3.018000

Tutorial_BioEM/Euler_Angle_List_Prior

-8
-    0.836000    1.369000    2.640000    0.640000
-   -1.375000    0.400000    3.018000    0.360000
-   -1.375000    0.400000    3.018000    0.360000
-    0.836000    1.369000    2.640000    0.640000
-    0.836000    1.369000    2.640000    0.640000
-   -1.375000    0.400000    3.018000    0.360000
-   -1.375000    0.400000    3.018000    0.360000

Tutorial_BioEM/ListMRC

-OneImage.mrc
-TwoImages.mrc

Tutorial_BioEM/MODEL_COMPARISION/Param_Input_ModelComparision

-##### Micrograph Parameters #######
-NUMBER_PIXELS 224
-PIXEL_SIZE 1.32
-
-##### Quaterion grid points: #######
-USE_QUATERNIONS
-
-##### Constrast transfer integration: #######
-CTF_B_ENV 2.0 300. 4
-CTF_DEFOCUS 0.5 4.5 8
-CTF_AMPLITUDE 0.01 0.601 5
-
-SIGMA_PRIOR_B_CTF 50.
-
-SIGMA_PRIOR_DEFOCUS 0.4
-PRIOR_DEFOCUS_CENTER 2.8
-
-
-##### Center displacement: #######
-DISPLACE_CENTER 40 1

Tutorial_BioEM/MODEL_COMPARISION/subtract_LogP.sh

-paste $1 $2 | grep LogProb: | awk 'BEGIN{cum=0;dif=0}{if($1=="RefMap:"){dif=$4-$10;cum+=dif;printf"MapNum: %6d Mod1: %10.5f Mod2: %10.5f Dif: %10.5f Cum: %10.5f\n",$2,$4,$10,dif,cum}}'

Tutorial_BioEM/MODEL_COMPARISON/Param_Input_ModelComparision

-##### Micrograph Parameters #######
-NUMBER_PIXELS 224
-PIXEL_SIZE 1.32
-
-##### Quaterion grid points: #######
-USE_QUATERNIONS
-
-##### Constrast transfer integration: #######
-CTF_B_ENV 2.0 300. 4
-CTF_DEFOCUS 0.5 4.5 8
-CTF_AMPLITUDE 0.01 0.601 5
-
-SIGMA_PRIOR_B_CTF 50.
-
-SIGMA_PRIOR_DEFOCUS 0.4
-PRIOR_DEFOCUS_CENTER 2.8
-
-
-##### Center displacement: #######
-DISPLACE_CENTER 40 1

Tutorial_BioEM/MODEL_COMPARISON/bioem_array_sge.sh

-### run in /bin/bash
-#$ -S /bin/bash
-#$ -cwd
-#$ -N bioem-test-array
-#$ -l h_rt=00:30:00
-### Allocating nodes with 2 GPUs each, on "phys" cluster
-#$ -pe impi_hydra 24
-#$ -P gpu
-#$ -l use_gpus=1
-#$ -l type_gpus=gtx1080
-### Allocation job-array for 20 jobs
-#$ -t 1-20
-
-set -e
-
-# Local ID of the job
-x=$SGE_TASK_ID
-SGE_TASK_COUNT=$((($SGE_TASK_LAST - $SGE_TASK_FIRST + 1) / $SGE_TASK_STEPSIZE))
-
-# Loading necessary libraries
-module purge
-module load intel impi cuda python33/python/3.3 python33/scipy/2015.10
-
-# Variables
-#TMPDIR="/tmp/"
-M=M2
-MODEL=MODEL_2
-GRID=smallGrid_125
-
-PYTHON=python3.3
-INPUTPROB=Output_${MODEL}
-ORIENTATIONS=${TMPDIR}/Quaternion_List_${M}_P${x}
-TMP1=${TMPDIR}/base${x}
-TMP2=${TMPDIR}/ll${x}
-OUTPUTPROB=${TMPDIR}/Output_${MODEL}_P${x}
-FINALOUTTMP=Output_Tmp
-FINALOUTPROB=Output_${MODEL}_Round2
-
-##########################################################
-##########################################################
-##########################################################
-# Creating list of quaternions
-grep Maxi ${INPUTPROB} | grep -v Notat | awk -v y=${x} '{if(NR==y)print $6,$8,$10,$12}' > ${TMP1}
-
-# Using the python numpy library to multiply the quaternions
-$PYTHON ./multiply_quat.py ${TMP1} ${GRID} > ${TMP2}
-
-# Generating the list of quaternions around the best orientation from base
-echo 125 > ${ORIENTATIONS}
-
-sed 's/,/ /g' ${TMP2} |  sed 's/)/ /g' |  sed 's/(/ /g' | awk '{printf"%12.8f%12.8f%12.8f%12.8f\n",$1,$2,$3,$4}' >> ${ORIENTATIONS}
-
-rm -f ${TMP1} ${TMP2}
-
-##########################################################
-##########################################################
-##########################################################
-# BioEM part Round 2
-
-# Path to your BioEM installation
-BIOEM=${HOME}/BioEM_project/build/bioEM
-
-# Running BioEM
-OMP_NUM_THREADS=12 GPU=1 GPUDEVICE=-1 BIOEM_DEBUG_OUTPUT=0 BIOEM_ALGO=2 mpiexec -perhost 2 ${BIOEM}  --Modelfile ${MODEL} --Particlesfile Particles/Particle_$x --Inputfile Param_Input_ModelComparision --ReadOrientation ${ORIENTATIONS} --OutputFile ${OUTPUTPROB}
-
-##########################################################
-##########################################################
-##########################################################
-# Writing results with a proper text in a proper order
-# Note that the order in which jobs are going to finish is undetermined, but the results are written in the good order at the end
-
-# Writing to the temporary shared file
-echo Job: ${x} $(tail -2 ${OUTPUTPROB} | head -1 | sed 's/RefMap: 0/RefMap: '${x}'/') >> ${FINALOUTTMP}
-echo Job: ${x} $(tail -1 ${OUTPUTPROB} | sed 's/RefMap: 0/RefMap: '${x}'/') >> ${FINALOUTTMP}
-
-# Cleanup
-rm -f ${ORIENTATIONS} ${OUTPUTTPROB}
-
-# If this was the last job, sort everything and remove JobID
-y=2
-if [[ $(wc -l < ${FINALOUTTMP}) == $(( $y * $SGE_TASK_COUNT )) ]]
-then
-    sort -n -k 2 ${FINALOUTTMP} | sed 's/Job: [0-9]\+ //' > ${FINALOUTPROB}
-    rm -rf ${FINALOUTTMP}
-fi

Tutorial_BioEM/MODEL_COMPARISON/bioem_array_slurm.sh

-#!/bin/bash -l
-# Standard output and error:
-#SBATCH -o ./tjob_hybrid.out.%j
-#SBATCH -e ./tjob_hybrid.err.%j
-# Initial working directory:
-#SBATCH -D ./
-# Job Name:
-#SBATCH -J bioem-test-array
-# Queue (Partition):
-#SBATCH --partition=gpu
-# Node feature:
-#SBATCH --constraint="gpu"
-# Number of nodes and MPI tasks per node:
-#SBATCH --nodes=1
-#SBATCH --ntasks-per-node=2
-# for OpenMP:
-#SBATCH --cpus-per-task=16
-#SBATCH --time=00:30:00
-# Submitting an array
-#SBATCH --array=1-20
-
-set -e
-
-echo "Hello: I'm task $SLURM_ARRAY_TASK_ID. SLURM_ARRAY_TASK_MIN=$SLURM_ARRAY_TASK_MIN SLURM_ARRAY_TASK_MAX=$SLURM_ARRAY_TASK_MAX SLURM_ARRAY_TASK_STEP=$SLURM_ARRAY_TASK_STEP"
-x=$SLURM_ARRAY_TASK_ID
-
-# The following line is only needed for the older Slurm versions
-SLURM_ARRAY_TASK_COUNT=$((($SLURM_ARRAY_TASK_MAX - $SLURM_ARRAY_TASK_MIN + 1) / $SLURM_ARRAY_TASK_STEP))
-
-##########################################################
-##########################################################
-##########################################################
-# Creating a list of quaternions for a specific particle image
-# Code similar to the create_gridOr.sh script, only for a single particle image
-
-# Loading Python
-module purge
-module load anaconda/3/4.3.1
-
-# Variables
-#TMPDIR="/tmp/"
-M=M1
-MODEL=MODEL_1
-GRID=smallGrid_125
-
-PYTHON=python3.6
-INPUTPROB=Output_${MODEL}
-ORIENTATIONS=${TMPDIR}/Quaternion_List_${M}_P${x}
-TMP1=${TMPDIR}/base${x}
-TMP2=${TMPDIR}/ll${x}
-OUTPUTPROB=${TMPDIR}/Output_${MODEL}_P${x}
-FINALOUTTMP=Output_Tmp
-FINALOUTPROB=Output_${MODEL}_Round2
-
-# Creating list of quaternions
-grep Maxi ${INPUTPROB} | grep -v Notat | awk -v y=${x} '{if(NR==y)print $6,$8,$10,$12}' > ${TMP1}
-
-# Using the python numpy library to multiply the quaternions
-$PYTHON ./multiply_quat.py ${TMP1} ${GRID} > ${TMP2}
-
-# Generating the list of quaternions around the best orientation from base
-echo 125 > ${ORIENTATIONS}
-
-sed 's/,/ /g' ${TMP2} |  sed 's/)/ /g' |  sed 's/(/ /g' | awk '{printf"%12.8f%12.8f%12.8f%12.8f\n",$1,$2,$3,$4}' >> ${ORIENTATIONS}
-
-rm -f ${TMP1} ${TMP2}
-
-##########################################################
-##########################################################
-##########################################################
-# BioEM part Round 2
-
-# Loading necessary modules for Intel compilers
-module load intel/17.0
-module load impi/2017.3
-module load fftw/3.3.6
-module load cuda/8.0
-
-# Environment variables
-export OMP_NUM_THREADS=16
-export OMP_PLACES=cores
-export GPUDEVICE=-1
-export GPUWORKLOAD=100
-
-# Environment variable to tune
-export BIOEM_CUDA_THREAD_COUNT=1024
-export BIOEM_DEBUG_OUTPUT=1
-export BIOEM_ALGO=2
-export GPU=1
-export BIOEM_PROJ_CONV_AT_ONCE=16
-
-# Path to your BioEM installation
-BIOEM=${HOME}/BioEM_project/build/bioEM
-
-# Running BioEM
-mpiexec -perhost 2 ${BIOEM} --Modelfile ${MODEL} --Particlesfile Particles/Particle_$x --Inputfile Param_Input_ModelComparision --ReadOrientation ${ORIENTATIONS} --OutputFile ${OUTPUTPROB} 
-
-##########################################################
-##########################################################
-##########################################################
-# Writing results with a proper text in a proper order
-# Note that the order in which jobs are going to finish is undetermined, but the results are written in the good order at the end
-
-# Writing to the temporary shared file
-echo Job: ${x} $(tail -2 ${OUTPUTPROB} | head -1 | sed 's/RefMap: 0/RefMap: '${x}'/') >> ${FINALOUTTMP}
-echo Job: ${x} $(tail -1 ${OUTPUTPROB} | sed 's/RefMap: 0/RefMap: '${x}'/') >> ${FINALOUTTMP}
-
-# Cleanup
-rm -f ${ORIENTATIONS} ${OUTPUTTPROB}
-
-y=2
-# If this was the last job, sort everything and remove JobID
-if [[ $(wc -l < ${FINALOUTTMP}