Commit 0158b128 authored by Luka Stanisic's avatar Luka Stanisic

temporary hack to get back good performance, this needs to be studied (and...

temporary hack to get back good performance, this needs to be studied (and optimized) in more details
parent c3843932
image: gitlab-registry.mpcdf.mpg.de/mpcdf/module-image
before_script:
- export LANG=C
- export GCC_VER=6
# For some reason doing shallow fetch doesnt work
#variables:
......@@ -17,16 +20,21 @@ stages:
build-gcc-mpi:
stage: build
script:
- module load git
- git clone https://gitlab-ci-token:${CI_JOB_TOKEN}@gitlab.mpcdf.mpg.de/MPIBP-Hummer/BioEM-tutorials.git
- .gitlab-ci/build/run_build.sh gcc-mpi
artifacts:
paths:
- build/
- BioEM-tutorials/Tutorial_BioEM
expire_in: 30 days
# default GCC build, minimal test
build-gcc-mpi-test:
stage: build
script:
- module load git
- git clone https://gitlab-ci-token:${CI_JOB_TOKEN}@gitlab.mpcdf.mpg.de/MPIBP-Hummer/BioEM-tutorials.git
- .gitlab-ci/build/run_build.sh gcc-mpi
- .gitlab-ci/test/run_test.sh gcc-mpi 4 2
- .gitlab-ci/test/run_coverage.sh build/CMakeFiles/bioEM.dir coverage_test
......@@ -70,7 +78,7 @@ verify_tutorial:
dependencies:
- build-gcc-mpi
script:
- .gitlab-ci/test/verify_tutorial.sh build/bioEM Tutorial_BioEM
- .gitlab-ci/test/verify_tutorial.sh build/bioEM BioEM-tutorials/Tutorial_BioEM
- .gitlab-ci/coverage/ci_coverage_collect.sh
artifacts:
paths:
......@@ -120,4 +128,4 @@ pages:
artifacts:
paths:
- public
expire_in: 30 days
\ No newline at end of file
expire_in: 30 days
#!/bin/bash
ARCHITECTURE=$1
DEBUG=${2:-no}
. /etc/profile.d/modules.sh
module use /afs/ipp/common/usr/modules.2014/@sys/modulefiles/GPU
module purge
module load cmake
echo "building BioEM with ARCH=$ARCHITECTURE"
case "$ARCHITECTURE" in
intel-mpi)
module load intel impi fftw
module load intel impi fftw-mpi
rm -rf build/; mkdir build
cd build
cmake .. -DCMAKE_CXX_COMPILER=icpc -DCMAKE_C_COMPILER=icc -DUSE_MPI=on -DUSE_OPENMP=on -DUSE_CUDA=off
cmake .. -DFFTWF_LIBRARIES=$FFTW_HOME/lib/libfftw3f.a -DFFTWF_INCLUDE_DIR=$FFTW_HOME/include -DCMAKE_CXX_COMPILER=icpc -DCMAKE_C_COMPILER=icc -DUSE_MPI=on -DUSE_OPENMP=on -DUSE_CUDA=off
make VERBOSE=1 -j 2
;;
gcc-mpi)
module load gcc/4.9 impi/5.0.3 fftw/gcc/3.3.4
module load gcc/$GCC_VER impi fftw-mpi
rm -rf build/; mkdir build
cd build
cmake .. -DCMAKE_CXX_COMPILER=g++ -DCMAKE_C_COMPILER=gcc -DCMAKE_BUILD_TYPE=Debug -DCMAKE_CXX_FLAGS_DEBUG='-g -Wall --coverage' -DCMAKE_C_FLAGS_DEBUG='-g -Wall --coverage'
cmake .. -DFFTWF_LIBRARIES=$FFTW_HOME/lib/libfftw3f.a -DFFTWF_INCLUDE_DIR=$FFTW_HOME/include -DCMAKE_CXX_COMPILER=g++ -DCMAKE_C_COMPILER=gcc -DCMAKE_BUILD_TYPE=Debug -DCMAKE_CXX_FLAGS_DEBUG='-g -Wall --coverage' -DCMAKE_C_FLAGS_DEBUG='-g -Wall --coverage'
make VERBOSE=1 -j 2
;;
gcc-mpi-cuda)
module load gcc/4.9 cuda/7.5 impi/5.0.3 fftw/gcc/3.3.4
module load gcc/$GCC_VER cuda impi fftw-mpi
rm -rf build/; mkdir build
cd build
cmake .. -DCMAKE_CXX_COMPILER=g++ -DCMAKE_C_COMPILER=gcc -DUSE_CUDA=on -DCUDA_CUDA_LIBRARY=$CUDA_HOME/lib64/stubs/libcuda.so
cmake .. -DFFTWF_LIBRARIES=$FFTW_HOME/lib/libfftw3f.a -DFFTWF_INCLUDE_DIR=$FFTW_HOME/include -DCMAKE_CXX_COMPILER=g++ -DCMAKE_C_COMPILER=gcc -DUSE_CUDA=on -DCUDA_CUDA_LIBRARY=$CUDA_HOME/lib64/stubs/libcuda.so
make VERBOSE=1 -j 2
;;
*)
......
......@@ -5,8 +5,9 @@ PATH=${HOME}/bin:${PATH}
set -e
set pipefail
. /etc/profile.d/modules.sh
module load gcc/4.9
module purge
module load gcc/$GCC_VER
module load lcov
cov_dir=coverage_data
prefix=$cov_dir/coverage_${CI_BUILD_REF}_${CI_BUILD_NAME}
......
#!/bin/bash
PATH=${HOME}/bin:${PATH}
. /etc/profile.d/modules.sh
module load gcc/4.9
module purge
module load gcc/$GCC_VER
module load lcov
for p in coverage_data/coverage_${CI_BUILD_REF}*.package ; do
cov_file=$PWD/${p%.package}.info
lcov -c -b $PWD --from-package $p --output-file $cov_file
lcov -r $cov_file "/usr/*" -o $cov_file
lcov -r $cov_file "/usr/include/*" -o $cov_file
lcov -r $cov_file "*4.9.3*" -o $cov_file
lcov -r $cov_file "*$GCC_VER*" -o $cov_file
lcov -r $cov_file "/afs/*" -o $cov_file
lcov -r $cov_file "BioEM/include/MersenneTwister.h" -o $cov_file
lcov -r $cov_file "BioEM/include/autotuner.h" -o $cov_file
lcov -r $cov_file "BioEM/autotuner.cpp" -o $cov_file
lcov -r $cov_file "include/MersenneTwister.h" -o $cov_file
lcov -r $cov_file "include/autotuner.h" -o $cov_file
lcov -r $cov_file "autotuner.cpp" -o $cov_file
done
lcov $(for i in coverage_data/coverage_${CI_BUILD_REF}_*.info; do echo -a $i; done) -o coverage_data/coverage_${CI_BUILD_REF}.info
......
......@@ -3,15 +3,13 @@
INPUT_DIR=$1
OUTPUT_DIR=$2
. /etc/profile.d/modules.sh
module load gcc/4.9
module purge
module load gcc/$GCC_VER
module load lcov
mkdir $OUTPUT_DIR
lcov --no-checksum --capture --directory $INPUT_DIR --output-file coverage.info
lcov --no-checksum --capture --directory $INPUT_DIR --output-file coverage.info
lcov -r coverage.info /usr/* --output-file coverage.info
lcov -r coverage.info /usr/include/* --output-file coverage.info
lcov -r coverage.info *4.9.3* --output-file coverage.info
lcov -r coverage.info *$GCC_VER* --output-file coverage.info
genhtml coverage.info --output-directory $OUTPUT_DIR
#!/bin/bash
#env
env
ARCH=$1
NUMPROCS=${2:-4}
NUMTHRDS=${3:-2}
. /etc/profile.d/modules.sh
module load impi
module purge
module load gcc/$GCC_VER impi fftw-mpi
export OMP_NUM_THREADS=$NUMTHRDS
export OMP_PLACES=cores
export OMP_STACKSIZE=128M
cp -r Tutorial_BioEM test1
cp -r BioEM-tutorials/Tutorial_BioEM test1
cd test1
mpiexec -n $NUMPROCS ../build/bioEM --Inputfile Param_Input --Modelfile Model_Text --Particlesfile Text_Image_Form
......@@ -14,9 +14,8 @@ ARCH=$1
NUMPROCS=${2:-4}
NUMTHRDS=${3:-2}
BIOEM=../build/bioEM
. /etc/profile.d/modules.sh
module load impi
module load gcc/$GCC_VER impi fftw-mpi
export OMP_NUM_THREADS=$NUMTHRDS
export OMP_PLACES=cores
......@@ -26,11 +25,12 @@ cat /proc/cpuinfo > environment
env >> environment
> ang_prob
cp -r Tutorial_BioEM test2
cp -r BioEM-tutorials/Tutorial_BioEM test2
cd test2
# Compare results with a certain numerical precision
NUMDIFF=/home/rzg/soft/numdiff/bin/numdiff
module load numdiff
NUMDIFF=numdiff
# Running tests
echo "@testing help output (error allowed)"
......
......@@ -14,9 +14,8 @@ ARCH=$1
NUMPROCS=${2:-4}
NUMTHRDS=${3:-2}
BIOEM=../build/bioEM
. /etc/profile.d/modules.sh
module load impi
module load gcc/$GCC_VER impi fftw-mpi
export OMP_NUM_THREADS=$NUMTHRDS
export OMP_PLACES=cores
......@@ -26,11 +25,12 @@ cat /proc/cpuinfo > environment
env >> environment
> all_probabilities
cp -r Tutorial_BioEM test3
cp -r BioEM-tutorials/Tutorial_BioEM test3
cd test3
# Compare results with a certain numerical precision
NUMDIFF=/home/rzg/soft/numdiff/bin/numdiff
module load numdiff
NUMDIFF=numdiff
# Running experiments
for algo in 1 2
......
......@@ -9,6 +9,8 @@ function exittrap {
trap exittrap ERR INT TERM
set -x
module load gcc/$GCC_VER impi fftw-mpi
# Environment
BIOEM=../${1}
TUTORIAL_DIR=$2
......@@ -40,9 +42,13 @@ $BIOEM --Inputfile Param_Input --Modelfile Model.pdb --ReadPDB --Particlesfile T
echo "@@ Read quaternions from file"
$BIOEM --Inputfile Param_Input_Quat --Modelfile Model.pdb --ReadPDB --Particlesfile Text_Image_Form --ReadOrientation Quat_list_Small
echo "@@ MRC Model - One MRC Image"
echo "Disabled as it is returning many warnings"
#$BIOEM --Inputfile Param_ModelMRC --Modelfile Model_MRC.mrc --ReadModelMRC --Particlesfile particle3.mrcs --ReadMRC --ReadOrientation Quat_list_Small
echo "@@ Input-parameter suggestions"
echo "Disabled as it is too long for the current CI"
#$BIOEM --Inputfile Param_Input_Quat --Modelfile Model.pdb --ReadPDB --Particlesfile Text_Image_Form --ReadOrientation List_Quat_ProRun
#$BIOEM --Inputfile Param_ProRun --Modelfile Model.pdb --ReadPDB --Particlesfile Text_Image_Form --ReadOrientation List_Quat_ProRun
echo "@@ Dump particle-images"
$BIOEM --Inputfile Param_Input --Modelfile Model.pdb --ReadPDB --Particlesfile ListMRC --ReadMRC --ReadMultipleMRC --DumpMaps
......@@ -50,6 +56,15 @@ $BIOEM --Inputfile Param_Input --Modelfile Model.pdb --ReadPDB --Particlesfile L
echo "@@ Load particle-images"
$BIOEM --Inputfile Param_Input --Modelfile Model.pdb --ReadPDB --LoadMapDump
echo "@@ Dump model"
$BIOEM --Inputfile Param_Input --Modelfile Model.pdb --ReadPDB --Particlesfile ListMRC --ReadMRC --ReadMultipleMRC --DumpModel
echo "@@ Load model"
$BIOEM --Inputfile Param_Input --Particlesfile ListMRC --ReadMRC --ReadMultipleMRC --LoadModelDump
echo "@@ Check model"
$BIOEM --Inputfile Param_Input --Modelfile Model_Text --Particlesfile Text_Image_Form --PrintCOORDREAD
echo "@@ Including prior probabilities"
$BIOEM --Modelfile Model_Text --Particlesfile Text_Image_Form --Inputfile Param_Input_Priors --ReadOrientation Euler_Angle_List_Prior
......
# BioEM: Bayesian inference of Electron Microscopy
# 2.0 VERSION: January, 2018
# 2.1 VERSION: July, 2019
## Build status and test coverage
......@@ -40,10 +40,14 @@ For a detailed description of the BioEM software see the [BioEM documentation](h
--Inputfile arg (Mandatory) Name of input parameter file
--ReadOrientation arg (Optional) Read file name containing orientations
--ReadPDB (Optional) If reading model file in PDB format
--ReadModelMRC (Optional) If reading model file in MRC format
--ReadMRC (Optional) If reading particle file in MRC format
--ReadMultipleMRC (Optional) If reading Multiple MRCs
--ReadMultipleMRC (Optional) If reading multiple MRCs
--DumpMaps (Optional) Dump maps after they were read from particle-image file
--LoadMapDump (Optional) Read Maps from dump option
--DumpModel (Optional) Dump model after it was read from model file
--LoadModelDump (Optional) Read model from dump option
--PrintCOORDREAD (Optional) Print model coordinates
--OutputFile arg (Optional) For changing the outputfile name
--help (Optional) Produce help message
......@@ -52,9 +56,9 @@ Details for the inputfiles and formats are provided in sections [1](http://bioem
### Output
* Main output file with, default name "Output_Probabilities", provides the posterior probability for each image, as well as the parameters that give a maximum value of the posterior:
RefMap #(number Particle Map) Probability #(log(P))
RefMap #(number Particle Map) Maximizing Param: #(Euler Angles) #(PSF parameters) #(center displacement)
**Important: It is recommended to compare log(P) with respect to other Models or to Noise as in [1].
......@@ -62,8 +66,8 @@ Details for the inputfiles and formats are provided in sections [1](http://bioem
* (Optional) Write the posterior probabilities as a function of the orientations (key word: WRITE_PROB_ANGLES in InputFile, see [documentation](http://bioem.readthedocs.io/en/latest/index.html#std:inpar-WRITE_PROB_ANGLES)).
### Tutorial
A directory with example EM particles, models, and input files are provided in the Tutorial_BioEM directory.
A directory with example EM particles, models, and input files are provided in the Tutorial_BioEM directory of the [BioEM-tutorials subproject](https://github.com/bio-phys/BioEM-tutorials).
The tutorial is described in [section 4 of the BioEM documentation](http://bioem.readthedocs.io/en/latest/index.html#tutorial).
......@@ -85,24 +89,24 @@ make VERBOSE=1
Dependencies and software requirements:
* Compiler: a modern C++ compiler which is OpenMP compliant
and (optionally) complies with CUDAs nvcc
(tested with Intel icpc 12-16, GCC 4.7-5.1)
-> adapt the name of the compiler using ccmake
and (optionally) complies with CUDAs nvcc
(tested with Intel icpc 12-16, GCC 4.7-5.1)
-> adapt the name of the compiler using ccmake
for free software see: https://gcc.gnu.org/
* MPI: the Message Passing Standard library
(tested with Intel MPI 4.1-5.1, IBM PE 1.3-1.4)
-> adapt the names of the MPI compiler wrappers using ccmake
(tested with Intel MPI 4.1-5.1, IBM PE 1.3-1.4)
-> adapt the names of the MPI compiler wrappers using ccmake
for free software see: http://www.open-mpi.de/
* FFTW: a serial but fully thread-safe fftw3 installation or equivalent (tested with fftw 3.3)
-> point environment variable $FFTW_ROOT to a FFTW3 installation or use ccmake to specify
for free software see: http://www.fftw.org
for free software see: http://www.fftw.org
* CUDA (required to build and run the GPU version)
* CUDA (required to build and run the GPU version)
for free software see: https://developer.nvidia.com/cuda-downloads
......@@ -115,7 +119,7 @@ The BioEM performance variables enhance or modify the code's computational perfo
They should be tuned for the specific computing node characteristics where bioEM is executed, e.g., select the number of GPUs to use, OpenMP
threads etc. These are passed via environment variables. See [section 4 of the BioEM documentation](http://bioem.readthedocs.io/en/latest/index.html#performance) for a detailed description.
### License
### License
The BioEM program is a free software, under the terms of the GNU General Public License as published by the Free Software Foundation, version 3 of the License.
This program is distributed in the hope that it will be useful, but without any warranty. See License.txt for more details.
......
2
0.836000 1.369000 2.640000
-1.375000 0.400000 3.018000
8
0.836000 1.369000 2.640000 0.640000
-1.375000 0.400000 3.018000 0.360000
-1.375000 0.400000 3.018000 0.360000
0.836000 1.369000 2.640000 0.640000
0.836000 1.369000 2.640000 0.640000
-1.375000 0.400000 3.018000 0.360000
-1.375000 0.400000 3.018000 0.360000
OneImage.mrc
TwoImages.mrc
This source diff could not be displayed because it is too large. You can view the blob instead.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
##### Micrograph Parameters #######
NUMBER_PIXELS 224
PIXEL_SIZE 1.32
##### Quaterion grid points: #######
USE_QUATERNIONS
##### Constrast transfer integration: #######
CTF_B_ENV 2.0 300. 4
CTF_DEFOCUS 0.5 4.5 8
CTF_AMPLITUDE 0.01 0.601 5
SIGMA_PRIOR_B_CTF 50.
SIGMA_PRIOR_DEFOCUS 0.4
PRIOR_DEFOCUS_CENTER 2.8
##### Center displacement: #######
DISPLACE_CENTER 40 1
This diff is collapsed.
paste $1 $2 | grep LogProb: | awk 'BEGIN{cum=0;dif=0}{if($1=="RefMap:"){dif=$4-$10;cum+=dif;printf"MapNum: %6d Mod1: %10.5f Mod2: %10.5f Dif: %10.5f Cum: %10.5f\n",$2,$4,$10,dif,cum}}'
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
##### Micrograph Parameters #######
NUMBER_PIXELS 224
PIXEL_SIZE 1.32
##### Quaterion grid points: #######
USE_QUATERNIONS
##### Constrast transfer integration: #######
CTF_B_ENV 2.0 300. 4
CTF_DEFOCUS 0.5 4.5 8
CTF_AMPLITUDE 0.01 0.601 5
SIGMA_PRIOR_B_CTF 50.
SIGMA_PRIOR_DEFOCUS 0.4
PRIOR_DEFOCUS_CENTER 2.8
##### Center displacement: #######
DISPLACE_CENTER 40 1
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
### run in /bin/bash
#$ -S /bin/bash
#$ -cwd
#$ -N bioem-test-array
#$ -l h_rt=00:30:00
### Allocating nodes with 2 GPUs each, on "phys" cluster
#$ -pe impi_hydra 24
#$ -P gpu
#$ -l use_gpus=1
#$ -l type_gpus=gtx1080
### Allocation job-array for 20 jobs
#$ -t 1-20
set -e
# Local ID of the job
x=$SGE_TASK_ID
SGE_TASK_COUNT=$((($SGE_TASK_LAST - $SGE_TASK_FIRST + 1) / $SGE_TASK_STEPSIZE))
# Loading necessary libraries
module purge
module load intel impi cuda python33/python/3.3 python33/scipy/2015.10
# Variables
#TMPDIR="/tmp/"
M=M2
MODEL=MODEL_2
GRID=smallGrid_125
PYTHON=python3.3
INPUTPROB=Output_${MODEL}
ORIENTATIONS=${TMPDIR}/Quaternion_List_${M}_P${x}
TMP1=${TMPDIR}/base${x}
TMP2=${TMPDIR}/ll${x}
OUTPUTPROB=${TMPDIR}/Output_${MODEL}_P${x}
FINALOUTTMP=Output_Tmp
FINALOUTPROB=Output_${MODEL}_Round2
##########################################################
##########################################################
##########################################################
# Creating list of quaternions
grep Maxi ${INPUTPROB} | grep -v Notat | awk -v y=${x} '{if(NR==y)print $6,$8,$10,$12}' > ${TMP1}
# Using the python numpy library to multiply the quaternions
$PYTHON ./multiply_quat.py ${TMP1} ${GRID} > ${TMP2}
# Generating the list of quaternions around the best orientation from base
echo 125 > ${ORIENTATIONS}
sed 's/,/ /g' ${TMP2} | sed 's/)/ /g' | sed 's/(/ /g' | awk '{printf"%12.8f%12.8f%12.8f%12.8f\n",$1,$2,$3,$4}' >> ${ORIENTATIONS}
rm -f ${TMP1} ${TMP2}
##########################################################
##########################################################
##########################################################
# BioEM part Round 2
# Path to your BioEM installation
BIOEM=${HOME}/BioEM_project/build/bioEM
# Running BioEM
OMP_NUM_THREADS=12 GPU=1 GPUDEVICE=-1 BIOEM_DEBUG_OUTPUT=0 BIOEM_ALGO=2 mpiexec -perhost 2 ${BIOEM} --Modelfile ${MODEL} --Particlesfile Particles/Particle_$x --Inputfile Param_Input_ModelComparision --ReadOrientation ${ORIENTATIONS} --OutputFile ${OUTPUTPROB}
##########################################################
##########################################################
##########################################################
# Writing results with a proper text in a proper order
# Note that the order in which jobs are going to finish is undetermined, but the results are written in the good order at the end
# Writing to the temporary shared file
echo Job: ${x} $(tail -2 ${OUTPUTPROB} | head -1 | sed 's/RefMap: 0/RefMap: '${x}'/') >> ${FINALOUTTMP}
echo Job: ${x} $(tail -1 ${OUTPUTPROB} | sed 's/RefMap: 0/RefMap: '${x}'/') >> ${FINALOUTTMP}
# Cleanup
rm -f ${ORIENTATIONS} ${OUTPUTTPROB}
# If this was the last job, sort everything and remove JobID
y=2
if [[ $(wc -l < ${FINALOUTTMP}) == $(( $y * $SGE_TASK_COUNT )) ]]
then
sort -n -k 2 ${FINALOUTTMP} | sed 's/Job: [0-9]\+ //' > ${FINALOUTPROB}
rm -rf ${FINALOUTTMP}
fi
#!/bin/bash -l
# Standard output and error:
#SBATCH -o ./tjob_hybrid.out.%j
#SBATCH -e ./tjob_hybrid.err.%j
# Initial working directory:
#SBATCH -D ./
# Job Name:
#SBATCH -J bioem-test-array
# Queue (Partition):
#SBATCH --partition=gpu
# Node feature:
#SBATCH --constraint="gpu"
# Number of nodes and MPI tasks per node:
#SBATCH --nodes=1
#SBATCH --ntasks-per-node=2
# for OpenMP:
#SBATCH --cpus-per-task=16
#SBATCH --time=00:30:00
# Submitting an array
#SBATCH --array=1-20
set -e
echo "Hello: I'm task $SLURM_ARRAY_TASK_ID. SLURM_ARRAY_TASK_MIN=$SLURM_ARRAY_TASK_MIN SLURM_ARRAY_TASK_MAX=$SLURM_ARRAY_TASK_MAX SLURM_ARRAY_TASK_STEP=$SLURM_ARRAY_TASK_STEP"
x=$SLURM_ARRAY_TASK_ID
# The following line is only needed for the older Slurm versions
SLURM_ARRAY_TASK_COUNT=$((($SLURM_ARRAY_TASK_MAX - $SLURM_ARRAY_TASK_MIN + 1) / $SLURM_ARRAY_TASK_STEP))
##########################################################
##########################################################
##########################################################
# Creating a list of quaternions for a specific particle image
# Code similar to the create_gridOr.sh script, only for a single particle image
# Loading Python
module purge
module load anaconda/3/4.3.1
# Variables
#TMPDIR="/tmp/"
M=M1
MODEL=MODEL_1
GRID=smallGrid_125
PYTHON=python3.6
INPUTPROB=Output_${MODEL}
ORIENTATIONS=${TMPDIR}/Quaternion_List_${M}_P${x}
TMP1=${TMPDIR}/base${x}
TMP2=${TMPDIR}/ll${x}
OUTPUTPROB=${TMPDIR}/Output_${MODEL}_P${x}
FINALOUTTMP=Output_Tmp
FINALOUTPROB=Output_${MODEL}_Round2
# Creating list of quaternions
grep Maxi ${INPUTPROB} | grep -v Notat | awk -v y=${x} '{if(NR==y)print $6,$8,$10,$12}' > ${TMP1}
# Using the python numpy library to multiply the quaternions
$PYTHON ./multiply_quat.py ${TMP1} ${GRID} > ${TMP2}
# Generating the list of quaternions around the best orientation from base
echo 125 > ${ORIENTATIONS}
sed 's/,/ /g' ${TMP2} | sed 's/)/ /g' | sed 's/(/ /g' | awk '{printf"%12.8f%12.8f%12.8f%12.8f\n",$1,$2,$3,$4}' >> ${ORIENTATIONS}
rm -f ${TMP1} ${TMP2}
##########################################################
##########################################################
##########################################################
# BioEM part Round 2
# Loading necessary modules for Intel compilers
module load intel/17.0
module load impi/2017.3
module load fftw/3.3.6
module load cuda/8.0
# Environment variables
export OMP_NUM_THREADS=16
export OMP_PLACES=cores
export GPUDEVICE=-1
export GPUWORKLOAD=100
# Environment variable to tune
export BIOEM_CUDA_THREAD_COUNT=1024
export BIOEM_DEBUG_OUTPUT=1
export BIOEM_ALGO=2
export GPU=1
export BIOEM_PROJ_CONV_AT_ONCE=16
# Path to your BioEM installation
BIOEM=${HOME}/BioEM_project/build/bioEM
# Running BioEM
mpiexec -perhost 2 ${BIOEM} --Modelfile ${MODEL} --Particlesfile Particles/Particle_$x --Inputfile Param_Input_ModelComparision --ReadOrientation ${ORIENTATIONS} --OutputFile ${OUTPUTPROB}
##########################################################
##########################################################
##########################################################
# Writing results with a proper text in a proper order
# Note that the order in which jobs are going to finish is undetermined, but the results are written in the good order at the end
# Writing to the temporary shared file
echo Job: ${x} $(tail -2 ${OUTPUTPROB} | head -1 | sed 's/RefMap: 0/RefMap: '${x}'/') >> ${FINALOUTTMP}
echo Job: ${x} $(tail -1 ${OUTPUTPROB} | sed 's/RefMap: 0/RefMap: '${x}'/') >> ${FINALOUTTMP}
# Cleanup
rm -f ${ORIENTATIONS} ${OUTPUTTPROB}
y=2
# If this was the last job, sort everything and remove JobID
if [[ $(wc -l < ${FINALOUTTMP}) == $(( $y * $SGE_TASK_COUNT )) ]]
then
sort -n -k 2 ${FINALOUTTMP} | sed 's/Job: [0-9]\+ //' > ${FINALOUTPROB}
rm -rf ${FINALOUTTMP}
fi
### to run:
# ./create_gridOr.sh $1 $2
## $1 == Output file from previous BioEM round
## $2 == Name to put on lists
# necessary python libraries:
module load python33/python/3.3 python33/scipy/2015.10
PYTHON=python3.3
TMPDIR=/tmp/
INDIR=${TMPDIR}/Quaternion_Lists_${2}
mkdir -p ${INDIR}
OUTPUTFILE=${INDIR}/Quaternion_List_${2}_P
TMP1=${TMPDIR}/base
TMP2=${TMPDIR}/ll
#Change the variable for the total number of images
numim=20
for((y=1;y<${numim}+1;y++))
do
#Extracting the best orientation from the output file from column 1
grep Maxi $1 | grep -v Notat | awk -v x=$y '{if(NR==x)print $6,$8,$10,$12}' > ${TMP1}
#Using the python numpy library to multiply the quaternions
$PYTHON ./multiply_quat.py ${TMP1} smallGrid_125 > ${TMP2}
#generating the list of quaternions around the best orientation from base
echo 125 > ${OUTPUTFILE}${y}
sed 's/,/ /g' ${TMP2} | sed 's/)/ /g' | sed 's/(/ /g' | awk '{printf"%12.8f%12.8f%12.8f%12.8f\n",$1,$2,$3,$4}' >> ${OUTPUTFILE}${y}
echo "Finished ${y}/${numim}"
rm ${TMP1} ${TMP2}