stages:
  - test



before_script:
  - export LANG=C
  - ulimit -s unlimited
  - ulimit -v unlimited
  - echo "HOST " $(hostname)
  - echo $CI_RUNNER_DESCRIPTION
  - export SLURM=yes
  - if [ "$(hostname)" = "buildtest-rzg" ]; then module purge && module load git && module list && export INTERACTIVE_RUN=yes && export SLURM=no && source ./ci_test_scripts/.ci-env-vars; fi
  - if [ "$(hostname)" = "amarek-elpa-gitlab-runner-1" ]; then module purge && module load git && module list && export INTERACTIVE_RUN=yes && export SLURM=no && source ./ci_test_scripts/.ci-env-vars; fi
  - if [ "$(hostname)" = "amarek-elpa-gitlab-runner-2" ]; then module purge && module load git && module list && export INTERACTIVE_RUN=yes && export SLURM=no && source ./ci_test_scripts/.ci-env-vars; fi
  - if [ "$(hostname)" = "amarek-elpa-gitlab-runner-3" ]; then module purge && module load git && module list && export INTERACTIVE_RUN=yes && export SLURM=no && source ./ci_test_scripts/.ci-env-vars; fi
  - if [ "$(hostname)" = "amarek-elpa-gitlab-runner-4" ]; then module purge && module load git && module list && export INTERACTIVE_RUN=yes && export SLURM=no && source ./ci_test_scripts/.ci-env-vars; fi
  - if [ "$CI_RUNNER_DESCRIPTION" = "freya01-interactive" ]; then export INTERACTIVE_RUN=yes && export SLURM=no; fi
  - if [ "$CI_RUNNER_DESCRIPTION" = "freya01-interactive-2" ]; then export INTERACTIVE_RUN=yes && export SLURM=no; fi
  - if [ "$CI_RUNNER_DESCRIPTION" = "appdev-gp02-1" ]; then export INTERACTIVE_RUN=no && export SLURM=no && export SLURMHOST=gp02 && export SLURMPARTITION=gp && export CONFIGURETIME=15 && export BUILDTIME=80 && export RUNTIME=140 && export CONTSTRAINTS="skylake" ; fi
  - if [ "$CI_RUNNER_DESCRIPTION" = "appdev-gp02-2" ]; then export INTERACTIVE_RUN=no && export SLURM=no && export SLURMHOST=gp02 && export SLURMPARTITION=gp && export CONFIGURETIME=15 && export BUILDTIME=80 && export RUNTIME=140 && export CONTSTRAINTS="skylake" ; fi
  - if [ "$CI_RUNNER_DESCRIPTION" = "appdev-gp02-3" ]; then export INTERACTIVE_RUN=no && export SLURM=no && export SLURMHOST=gp02 && export SLURMPARTITION=gp && export CONFIGURETIME=15 && export BUILDTIME=80 && export RUNTIME=140 && export CONTSTRAINTS="skylake" ; fi
  - if [ "$CI_RUNNER_DESCRIPTION" = "appdev-gp02-4" ]; then export INTERACTIVE_RUN=no && export SLURM=no && export SLURMHOST=gp02 && export SLURMPARTITION=gp && export CONFIGURETIME=15 && export BUILDTIME=80 && export RUNTIME=140 && export CONTSTRAINTS="skylake" ; fi
  - if [ "$CI_RUNNER_DESCRIPTION" = "appdev-gp02-5" ]; then export INTERACTIVE_RUN=no && export SLURM=no && export SLURMHOST=gp02 && export SLURMPARTITION=gp && export CONFIGURETIME=15 && export BUILDTIME=80 && export RUNTIME=140 && export CONTSTRAINTS="skylake" ; fi
  - if [ "$CI_RUNNER_DESCRIPTION" = "appdev-gp02-6" ]; then export INTERACTIVE_RUN=no && export SLURM=no && export SLURMHOST=gp02 && export SLURMPARTITION=gp && export CONFIGURETIME=15 && export BUILDTIME=80 && export RUNTIME=140 && export CONTSTRAINTS="skylake" ; fi
  - if [ "$CI_RUNNER_DESCRIPTION" = "appdev-gp02-7" ]; then export INTERACTIVE_RUN=no && export SLURM=no && export SLURMHOST=gp02 && export SLURMPARTITION=gp && export CONFIGURETIME=15 && export BUILDTIME=80 && export RUNTIME=140 && export CONTSTRAINTS="skylake" ; fi
  - if [ "$CI_RUNNER_DESCRIPTION" = "appdev-gp02-8" ]; then export INTERACTIVE_RUN=no && export SLURM=no && export SLURMHOST=gp02 && export SLURMPARTITION=gp && export CONFIGURETIME=15 && export BUILDTIME=80 && export RUNTIME=140 && export CONTSTRAINTS="skylake" ; fi
  - if [ "$CI_RUNNER_DESCRIPTION" = "appdev-gp02-9" ]; then export INTERACTIVE_RUN=no && export SLURM=no && export SLURMHOST=gp02 && export SLURMPARTITION=gp && export CONFIGURETIME=15 && export BUILDTIME=80 && export RUNTIME=140 && export CONTSTRAINTS="skylake" ; fi
  - if [ "$CI_RUNNER_DESCRIPTION" = "appdev-gp02-10" ]; then export INTERACTIVE_RUN=no && export SLURM=no && export SLURMHOST=gp02 && export SLURMPARTITION=gp && export CONFIGURETIME=15 && export BUILDTIME=80 && export RUNTIME=140 && export CONTSTRAINTS="skylake" ; fi


  - if [ "$CI_RUNNER_DESCRIPTION" = "appdev-knl1-1" ]; then export INTERACTIVE_RUN=no && export SLURM=no && export SLURMHOST=knl1 && export SLURMPARTITION=knl && export CONFIGURETIME=15 && export BUILDTIME=80 && export RUNTIME=140 && export CONTSTRAINTS="knl" ;  fi
  - if [ "$CI_RUNNER_DESCRIPTION" = "appdev-knl1-2" ]; then export INTERACTIVE_RUN=no && export SLURM=no && export SLURMHOST=knl1 && export SLURMPARTITION=knl && export CONFIGURETIME=15 && export BUILDTIME=80 && export RUNTIME=140 && export CONTSTRAINTS="knl" ;  fi
  - if [ "$CI_RUNNER_DESCRIPTION" = "appdev-knl1-3" ]; then export INTERACTIVE_RUN=no && export SLURM=no && export SLURMHOST=knl1 && export SLURMPARTITION=knl && export CONFIGURETIME=15 && export BUILDTIME=80 && export RUNTIME=140 && export CONTSTRAINTS="knl" ;  fi
  - if [ "$CI_RUNNER_DESCRIPTION" = "appdev-knl1-4" ]; then export INTERACTIVE_RUN=no && export SLURM=no && export SLURMHOST=knl1 && export SLURMPARTITION=knl && export CONFIGURETIME=15 && export BUILDTIME=80 && export RUNTIME=140 && export CONTSTRAINTS="knl" ;  fi
  - if [ "$CI_RUNNER_DESCRIPTION" = "appdev-knl1-5" ]; then export INTERACTIVE_RUN=no && export SLURM=no && export SLURMHOST=knl1 && export SLURMPARTITION=knl && export CONFIGURETIME=15 && export BUILDTIME=80 && export RUNTIME=140 && export CONTSTRAINTS="knl" ;  fi
  - if [ "$CI_RUNNER_DESCRIPTION" = "appdev-knl1-6" ]; then export INTERACTIVE_RUN=no && export SLURM=no && export SLURMHOST=knl1 && export SLURMPARTITION=knl && export CONFIGURETIME=15 && export BUILDTIME=80 && export RUNTIME=140 && export CONTSTRAINTS="knl" ;  fi
  - if [ "$CI_RUNNER_DESCRIPTION" = "appdev-knl1-7" ]; then export INTERACTIVE_RUN=no && export SLURM=no && export SLURMHOST=knl1 && export SLURMPARTITION=knl && export CONFIGURETIME=15 && export BUILDTIME=80 && export RUNTIME=140 && export CONTSTRAINTS="knl" ;  fi
  - if [ "$CI_RUNNER_DESCRIPTION" = "appdev-knl1-8" ]; then export INTERACTIVE_RUN=no && export SLURM=no && export SLURMHOST=knl1 && export SLURMPARTITION=knl && export CONFIGURETIME=15 && export BUILDTIME=80 && export RUNTIME=140 && export CONTSTRAINTS="knl" ;  fi
  - if [ "$CI_RUNNER_DESCRIPTION" = "appdev-knl1-9" ]; then export INTERACTIVE_RUN=no && export SLURM=no && export SLURMHOST=knl1 && export SLURMPARTITION=knl && export CONFIGURETIME=15 && export BUILDTIME=80 && export RUNTIME=140 && export CONTSTRAINTS="knl" ;  fi
  - if [ "$CI_RUNNER_DESCRIPTION" = "appdev-knl1-10" ]; then export INTERACTIVE_RUN=no && export SLURM=no && export SLURMHOST=knl1 && export SLURMPARTITION=knl && export CONFIGURETIME=15 && export BUILDTIME=80 && export RUNTIME=140 && export CONTSTRAINTS="knl" ;  fi
  - if [ "$CI_RUNNER_DESCRIPTION" = "appdev-knl1-11" ]; then export INTERACTIVE_RUN=no && export SLURM=no && export SLURMHOST=knl1 && export SLURMPARTITION=knl && export CONFIGURETIME=15 && export BUILDTIME=80 && export RUNTIME=140 && export CONTSTRAINTS="knl" ;  fi
  - if [ "$CI_RUNNER_DESCRIPTION" = "appdev-knl1-12" ]; then export INTERACTIVE_RUN=no && export SLURM=no && export SLURMHOST=knl1 && export SLURMPARTITION=knl && export CONFIGURETIME=15 && export BUILDTIME=80 && export RUNTIME=140 && export CONTSTRAINTS="knl" ;  fi
  - if [ "$CI_RUNNER_DESCRIPTION" = "appdev-knl1-13" ]; then export INTERACTIVE_RUN=no && export SLURM=no && export SLURMHOST=knl1 && export SLURMPARTITION=knl && export CONFIGURETIME=15 && export BUILDTIME=80 && export RUNTIME=140 && export CONTSTRAINTS="knl" ;  fi
  - if [ "$CI_RUNNER_DESCRIPTION" = "appdev-knl1-14" ]; then export INTERACTIVE_RUN=no && export SLURM=no && export SLURMHOST=knl1 && export SLURMPARTITION=knl && export CONFIGURETIME=15 && export BUILDTIME=80 && export RUNTIME=140 && export CONTSTRAINTS="knl" ;  fi
  - if [ "$CI_RUNNER_DESCRIPTION" = "appdev-knl1-15" ]; then export INTERACTIVE_RUN=no && export SLURM=no && export SLURMHOST=knl1 && export SLURMPARTITION=knl && export CONFIGURETIME=15 && export BUILDTIME=80 && export RUNTIME=140 && export CONTSTRAINTS="knl" ;  fi
  - if [ "$CI_RUNNER_DESCRIPTION" = "appdev-knl1-16" ]; then export INTERACTIVE_RUN=no && export SLURM=no && export SLURMHOST=knl1 && export SLURMPARTITION=knl && export CONFIGURETIME=15 && export BUILDTIME=80 && export RUNTIME=140 && export CONTSTRAINTS="knl" ;  fi
  - if [ "$CI_RUNNER_DESCRIPTION" = "appdev-knl1-17" ]; then export INTERACTIVE_RUN=no && export SLURM=no && export SLURMHOST=knl1 && export SLURMPARTITION=knl && export CONFIGURETIME=15 && export BUILDTIME=80 && export RUNTIME=140 && export CONTSTRAINTS="knl" ;  fi
  - if [ "$CI_RUNNER_DESCRIPTION" = "appdev-knl1-18" ]; then export INTERACTIVE_RUN=no && export SLURM=no && export SLURMHOST=knl1 && export SLURMPARTITION=knl && export CONFIGURETIME=15 && export BUILDTIME=80 && export RUNTIME=140 && export CONTSTRAINTS="knl" ;  fi
  - if [ "$CI_RUNNER_DESCRIPTION" = "appdev-knl1-19" ]; then export INTERACTIVE_RUN=no && export SLURM=no && export SLURMHOST=knl1 && export SLURMPARTITION=knl && export CONFIGURETIME=15 && export BUILDTIME=80 && export RUNTIME=140 && export CONTSTRAINTS="knl" ;  fi
  - if [ "$CI_RUNNER_DESCRIPTION" = "appdev-knl1-20" ]; then export INTERACTIVE_RUN=no && export SLURM=no && export SLURMHOST=knl1 && export SLURMPARTITION=knl && export CONFIGURETIME=15 && export BUILDTIME=80 && export RUNTIME=140 && export CONTSTRAINTS="knl" ;  fi


  - if [ "$CI_RUNNER_DESCRIPTION" = "appdev-knl2" ]; then export INTERACTIVE_RUN=no && export SLURM=no && export SLURMHOST=knl2 && export SLURMPARTITION=knl && export CONFIGURETIME=15 && export BUILDTIME=80 && export RUNTIME=140 && export CONTSTRAINTS="knl" ; fi


  - if [ "$CI_RUNNER_DESCRIPTION" = "appdev-knl3" ]; then export INTERACTIVE_RUN=no && export SLURM=no && export SLURMHOST=knl3 && export SLURMPARTITION=knl && export CONFIGURETIME=15 && export BUILDTIME=80 && export RUNTIME=140 && export CONTSTRAINTS="knl" ; fi


  - if [ "$CI_RUNNER_DESCRIPTION" = "appdev-knl4" ]; then export INTERACTIVE_RUN=no && export SLURM=no && export SLURMHOST=knl4 && export SLURMPARTITION=knl && export CONFIGURETIME=15 && export BUILDTIME=80 && export RUNTIME=140 && export CONTSTRAINTS="knl" ; fi


  - if [ "$CI_RUNNER_DESCRIPTION" = "appdev-maik" ]; then export INTERACTIVE_RUN=no && export SLURM=no && export SLURMHOST=maik && export SLURMPARTITION=maik && export CONFIGURETIME=15 && export BUILDTIME=80 && export RUNTIME=140 && export CONTSTRAINTS="knl" ; fi
  - if [ "$CI_RUNNER_DESCRIPTION" = "appdev-dvl01" ]; then export INTERACTIVE_RUN=no && export SLURM=no && export SLURMHOST=dvl01 && export SLURMPARTITION=dvl && export CONFIGURETIME=15 && export BUILDTIME=80 && export RUNTIME=140  && export CONTSTRAINTS="x86_64&gpu0&gpu1" && export GEOMETRYRESERVATION="gpu:2" ; fi


  - if [ "$CI_RUNNER_DESCRIPTION" = "appdev-dvl02" ]; then export INTERACTIVE_RUN=no && export SLURM=no && export SLURMHOST=dvl02 && export SLURMPARTITION=dvl && export CONFIGURETIME=15 && export BUILDTIME=80 && export RUNTIME=140 && export CONTSTRAINTS="x86_64&gpu0&gpu1" && export GEOMETRYRESERVATION="gpu:2" ; fi


  - if [ "$CI_RUNNER_DESCRIPTION" = "appdev-miy01" ]; then export INTERACTIVE_RUN=no && export SLURM=no && export SLURMHOST=miy01 && export SLURMPARTITION=minsky && export CONFIGURETIME=15 && export BUILDTIME=80 && export RUNTIME=140 && export CONTSTRAINTS="power8&gpu0&gpu1&gpu2&gpu3" && export GEOMETRYRESERVATION="gpu:4" ; fi
  - if [ "$CI_RUNNER_DESCRIPTION" = "appdev-miy02" ]; then export INTERACTIVE_RUN=no && export SLURM=no && export SLURMHOST=miy02 && export SLURMPARTITION=minsky && export CONFIGURETIME=15 && export BUILDTIME=80 && export RUNTIME=140 && export CONTSTRAINTS="power8&gpu0&gpu1&gpu2&gpu3" && export GEOMETRYRESERVATION="gpu:4" ; fi
  - if [ "$CI_RUNNER_DESCRIPTION" = "appdev-miy03" ]; then export INTERACTIVE_RUN=no && export SLURM=no && export SLURMHOST=miy03 && export SLURMPARTITION=minsky && export CONFIGURETIME=15 && export BUILDTIME=80 && export RUNTIME=140 && export CONTSTRAINTS="power8&gpu0&gpu1&gpu2&gpu3" && export GEOMETRYRESERVATION="gpu:4" ; fi


  - export MATRIX_SIZE=150
  - export NUMBER_OF_EIGENVECTORS=150
  - export BLOCK_SIZE=16
  - if [ "$MEDIUM_MATRIX" = "yes" ]; then export MATRIX_SIZE=1500 && export NUMBER_OF_EIGENVECTORS=750; fi
  - if [ "$LARGE_MATRIX" = "yes" ]; then export MATRIX_SIZE=5000 && export NUMBER_OF_EIGENVECTORS=500; fi
  - if [ "$GPU_BLOCKSIZE" = "yes" ]; then export BLOCK_SIZE=128 ; fi
  - echo "This test will run with matrix size na = $MATRIX_SIZE, nev= $NUMBER_OF_EIGENVECTORS, on a blacs grid with blocksize nblk= $BLOCK_SIZE " 
  - export SKIP_STEP=0
  - ./autogen.sh
  - export SKIP_STEP=0
  - if [ -f /etc/profile.d/modules.sh ]; then source /etc/profile.d/modules.sh ; else source /etc/profile.d/mpcdf_modules.sh; fi  && . ./ci_test_scripts/.ci-env-vars



# For some reason sometimes not-writable files remain, which cause trouble the
# next time a runner tries to clean-up
after_script:
  - chmod u+w -R .



# gnu-gnu-mpi-noopenmp-double-precision-assumed-size-band-to-full-blocking-with-gpu-no-coverage-avx-no-address-sanitize
gnu-gnu-mpi-noopenmp-double-precision-assumed-size-band-to-full-blocking-with-gpu-no-coverage-avx-no-address-sanitize-jobs:
  only:
    - /.*master.*/
  tags:
    - gpu
  artifacts:
    when: on_success
    expire_in: 2 month
  script:
    - export REQUESTED_MEMORY=2Gb


    - echo "The tasks will be submitted to SLURM PARTITION " $SLURMPARTITION " on host " $SLURMHOST " with constraints " $CONTSTRAINTS " with the geometry " $GEOMETRYRESERVATION
    - export SRUN_COMMANDLINE_CONFIGURE="--partition=$SLURMPARTITION --nodelist=$SLURMHOST --time=$CONFIGURETIME --constraint=$CONTSTRAINTS --gres=$GEOMETRYRESERVATION " 
    - export SRUN_COMMANDLINE_BUILD="--partition=$SLURMPARTITION --nodelist=$SLURMHOST --time=$BUILDTIME --constraint=$CONTSTRAINTS --gres=$GEOMETRYRESERVATION " 
    - export SRUN_COMMANDLINE_RUN="--partition=$SLURMPARTITION --nodelist=$SLURMHOST --time=$RUNTIME --constraint=$CONTSTRAINTS --gres=$GEOMETRYRESERVATION " 
    - ./ci_test_scripts/run_ci_tests.sh -c " CC=\"mpicc\" CFLAGS=\"-O3 -mavx\" FC=\"mpif90\" FCFLAGS=\"-O3 -mavx\"    SCALAPACK_LDFLAGS=\"\$MKL_GFORTRAN_SCALAPACK_LDFLAGS_MPI_NO_OMP  -L\$CUDA_HOME/lib64 -lcublas -I\$CUDA_HOME/include\"  SCALAPACK_FCFLAGS=\"\$MKL_GFORTRAN_SCALAPACK_FCFLAGS_MPI_NO_OMP  -I\$CUDA_HOME/include\" --enable-option-checking=fatal --with-mpi=yes --disable-mpi-module --disable-openmp --disable-single-precision --enable-assumed-size --enable-band-to-full-blocking --enable-gpu --with-cuda-path=\$CUDA_HOME/ --enable-avx --disable-avx2" -j 8 -t 2 -m $MATRIX_SIZE -n $NUMBER_OF_EIGENVECTORS -b $BLOCK_SIZE -s $SKIP_STEP -q "srun" -i $INTERACTIVE_RUN -S $SLURM
    -  echo "na= $MATRIX_SIZE, nev= $NUMBER_OF_EIGENVECTORS nblock= $BLOCK_SIZE"