.gitlab-ci.yml

stages:
  - test
  - coverage
  - deploy



before_script:
  - export LANG=C
  - ulimit -s unlimited
  - ulimit -v unlimited
  - echo "HOST " $(hostname)
  - if [ "$(hostname)" = "buildtest-rzg" ]; then module purge && module load git && module list && source ./ci_test_scripts/.ci-env-vars; fi
  - if [ "$(hostname)" = "amarek-elpa-gitlab-runner-1" ]; then module purge && module load git && module list && source ./ci_test_scripts/.ci-env-vars; fi
  - if [ "$(hostname)" = "amarek-elpa-gitlab-runner-2" ]; then module purge && module load git && module list && source ./ci_test_scripts/.ci-env-vars; fi
  - if [ "$(hostname)" = "amarek-elpa-gitlab-runner-3" ]; then module purge && module load git && module list && source ./ci_test_scripts/.ci-env-vars; fi
  - if [ "$(hostname)" = "amarek-elpa-gitlab-runner-4" ]; then module purge && module load git && module list && source ./ci_test_scripts/.ci-env-vars; fi
  - if [ "$CI_RUNNER_DESCRIPTION" = "appdev-gp02-1" ]; then export SLURMHOST=gp02 && export SLURMPARTITION=gp && export CONFIGURETIME=15 && export BUILDTIME=80 && export RUNTIME=140 && export CONTSTRAINTS="skylake" ; fi
  - if [ "$CI_RUNNER_DESCRIPTION" = "appdev-gp02-2" ]; then export SLURMHOST=gp02 && export SLURMPARTITION=gp && export CONFIGURETIME=15 && export BUILDTIME=80 && export RUNTIME=140 && export CONTSTRAINTS="skylake" ; fi
  - if [ "$CI_RUNNER_DESCRIPTION" = "appdev-gp02-3" ]; then export SLURMHOST=gp02 && export SLURMPARTITION=gp && export CONFIGURETIME=15 && export BUILDTIME=80 && export RUNTIME=140 && export CONTSTRAINTS="skylake" ; fi
  - if [ "$CI_RUNNER_DESCRIPTION" = "appdev-gp02-4" ]; then export SLURMHOST=gp02 && export SLURMPARTITION=gp && export CONFIGURETIME=15 && export BUILDTIME=80 && export RUNTIME=140 && export CONTSTRAINTS="skylake" ; fi
  - if [ "$CI_RUNNER_DESCRIPTION" = "appdev-gp02-5" ]; then export SLURMHOST=gp02 && export SLURMPARTITION=gp && export CONFIGURETIME=15 && export BUILDTIME=80 && export RUNTIME=140 && export CONTSTRAINTS="skylake" ; fi
  - if [ "$CI_RUNNER_DESCRIPTION" = "appdev-gp02-6" ]; then export SLURMHOST=gp02 && export SLURMPARTITION=gp && export CONFIGURETIME=15 && export BUILDTIME=80 && export RUNTIME=140 && export CONTSTRAINTS="skylake" ; fi
  - if [ "$CI_RUNNER_DESCRIPTION" = "appdev-gp02-7" ]; then export SLURMHOST=gp02 && export SLURMPARTITION=gp && export CONFIGURETIME=15 && export BUILDTIME=80 && export RUNTIME=140 && export CONTSTRAINTS="skylake" ; fi
  - if [ "$CI_RUNNER_DESCRIPTION" = "appdev-gp02-8" ]; then export SLURMHOST=gp02 && export SLURMPARTITION=gp && export CONFIGURETIME=15 && export BUILDTIME=80 && export RUNTIME=140 && export CONTSTRAINTS="skylake" ; fi
  - if [ "$CI_RUNNER_DESCRIPTION" = "appdev-gp02-9" ]; then export SLURMHOST=gp02 && export SLURMPARTITION=gp && export CONFIGURETIME=15 && export BUILDTIME=80 && export RUNTIME=140 && export CONTSTRAINTS="skylake" ; fi
  - if [ "$CI_RUNNER_DESCRIPTION" = "appdev-gp02-10" ]; then export SLURMHOST=gp02 && export SLURMPARTITION=gp && export CONFIGURETIME=15 && export BUILDTIME=80 && export RUNTIME=140 && export CONTSTRAINTS="skylake" ; fi


  - if [ "$CI_RUNNER_DESCRIPTION" = "appdev-knl1-1" ]; then export SLURMHOST=knl1 && export SLURMPARTITION=knl && export CONFIGURETIME=15 && export BUILDTIME=80 && export RUNTIME=140 && export CONTSTRAINTS="knl" ;  fi
  - if [ "$CI_RUNNER_DESCRIPTION" = "appdev-knl1-2" ]; then export SLURMHOST=knl1 && export SLURMPARTITION=knl && export CONFIGURETIME=15 && export BUILDTIME=80 && export RUNTIME=140 && export CONTSTRAINTS="knl" ;  fi
  - if [ "$CI_RUNNER_DESCRIPTION" = "appdev-knl1-3" ]; then export SLURMHOST=knl1 && export SLURMPARTITION=knl && export CONFIGURETIME=15 && export BUILDTIME=80 && export RUNTIME=140 && export CONTSTRAINTS="knl" ;  fi
  - if [ "$CI_RUNNER_DESCRIPTION" = "appdev-knl1-4" ]; then export SLURMHOST=knl1 && export SLURMPARTITION=knl && export CONFIGURETIME=15 && export BUILDTIME=80 && export RUNTIME=140 && export CONTSTRAINTS="knl" ;  fi
  - if [ "$CI_RUNNER_DESCRIPTION" = "appdev-knl1-5" ]; then export SLURMHOST=knl1 && export SLURMPARTITION=knl && export CONFIGURETIME=15 && export BUILDTIME=80 && export RUNTIME=140 && export CONTSTRAINTS="knl" ;  fi
  - if [ "$CI_RUNNER_DESCRIPTION" = "appdev-knl1-6" ]; then export SLURMHOST=knl1 && export SLURMPARTITION=knl && export CONFIGURETIME=15 && export BUILDTIME=80 && export RUNTIME=140 && export CONTSTRAINTS="knl" ;  fi
  - if [ "$CI_RUNNER_DESCRIPTION" = "appdev-knl1-7" ]; then export SLURMHOST=knl1 && export SLURMPARTITION=knl && export CONFIGURETIME=15 && export BUILDTIME=80 && export RUNTIME=140 && export CONTSTRAINTS="knl" ;  fi
  - if [ "$CI_RUNNER_DESCRIPTION" = "appdev-knl1-8" ]; then export SLURMHOST=knl1 && export SLURMPARTITION=knl && export CONFIGURETIME=15 && export BUILDTIME=80 && export RUNTIME=140 && export CONTSTRAINTS="knl" ;  fi
  - if [ "$CI_RUNNER_DESCRIPTION" = "appdev-knl1-9" ]; then export SLURMHOST=knl1 && export SLURMPARTITION=knl && export CONFIGURETIME=15 && export BUILDTIME=80 && export RUNTIME=140 && export CONTSTRAINTS="knl" ;  fi
  - if [ "$CI_RUNNER_DESCRIPTION" = "appdev-knl1-10" ]; then export SLURMHOST=knl1 && export SLURMPARTITION=knl && export CONFIGURETIME=15 && export BUILDTIME=80 && export RUNTIME=140 && export CONTSTRAINTS="knl" ;  fi
  - if [ "$CI_RUNNER_DESCRIPTION" = "appdev-knl1-11" ]; then export SLURMHOST=knl1 && export SLURMPARTITION=knl && export CONFIGURETIME=15 && export BUILDTIME=80 && export RUNTIME=140 && export CONTSTRAINTS="knl" ;  fi
  - if [ "$CI_RUNNER_DESCRIPTION" = "appdev-knl1-12" ]; then export SLURMHOST=knl1 && export SLURMPARTITION=knl && export CONFIGURETIME=15 && export BUILDTIME=80 && export RUNTIME=140 && export CONTSTRAINTS="knl" ;  fi
  - if [ "$CI_RUNNER_DESCRIPTION" = "appdev-knl1-13" ]; then export SLURMHOST=knl1 && export SLURMPARTITION=knl && export CONFIGURETIME=15 && export BUILDTIME=80 && export RUNTIME=140 && export CONTSTRAINTS="knl" ;  fi
  - if [ "$CI_RUNNER_DESCRIPTION" = "appdev-knl1-14" ]; then export SLURMHOST=knl1 && export SLURMPARTITION=knl && export CONFIGURETIME=15 && export BUILDTIME=80 && export RUNTIME=140 && export CONTSTRAINTS="knl" ;  fi
  - if [ "$CI_RUNNER_DESCRIPTION" = "appdev-knl1-15" ]; then export SLURMHOST=knl1 && export SLURMPARTITION=knl && export CONFIGURETIME=15 && export BUILDTIME=80 && export RUNTIME=140 && export CONTSTRAINTS="knl" ;  fi
  - if [ "$CI_RUNNER_DESCRIPTION" = "appdev-knl1-16" ]; then export SLURMHOST=knl1 && export SLURMPARTITION=knl && export CONFIGURETIME=15 && export BUILDTIME=80 && export RUNTIME=140 && export CONTSTRAINTS="knl" ;  fi
  - if [ "$CI_RUNNER_DESCRIPTION" = "appdev-knl1-17" ]; then export SLURMHOST=knl1 && export SLURMPARTITION=knl && export CONFIGURETIME=15 && export BUILDTIME=80 && export RUNTIME=140 && export CONTSTRAINTS="knl" ;  fi
  - if [ "$CI_RUNNER_DESCRIPTION" = "appdev-knl1-18" ]; then export SLURMHOST=knl1 && export SLURMPARTITION=knl && export CONFIGURETIME=15 && export BUILDTIME=80 && export RUNTIME=140 && export CONTSTRAINTS="knl" ;  fi
  - if [ "$CI_RUNNER_DESCRIPTION" = "appdev-knl1-19" ]; then export SLURMHOST=knl1 && export SLURMPARTITION=knl && export CONFIGURETIME=15 && export BUILDTIME=80 && export RUNTIME=140 && export CONTSTRAINTS="knl" ;  fi
  - if [ "$CI_RUNNER_DESCRIPTION" = "appdev-knl1-20" ]; then export SLURMHOST=knl1 && export SLURMPARTITION=knl && export CONFIGURETIME=15 && export BUILDTIME=80 && export RUNTIME=140 && export CONTSTRAINTS="knl" ;  fi


  - if [ "$CI_RUNNER_DESCRIPTION" = "appdev-knl2" ]; then export SLURMHOST=knl2 && export SLURMPARTITION=knl && export CONFIGURETIME=15 && export BUILDTIME=80 && export RUNTIME=140 && export CONTSTRAINTS="knl" ; fi


  - if [ "$CI_RUNNER_DESCRIPTION" = "appdev-knl3" ]; then export SLURMHOST=knl3 && export SLURMPARTITION=knl && export CONFIGURETIME=15 && export BUILDTIME=80 && export RUNTIME=140 && export CONTSTRAINTS="knl" ; fi


  - if [ "$CI_RUNNER_DESCRIPTION" = "appdev-knl4" ]; then export SLURMHOST=knl4 && export SLURMPARTITION=knl && export CONFIGURETIME=15 && export BUILDTIME=80 && export RUNTIME=140 && export CONTSTRAINTS="knl" ; fi


  - if [ "$CI_RUNNER_DESCRIPTION" = "appdev-maik" ]; then export SLURMHOST=maik && export SLURMPARTITION=maik && export CONFIGURETIME=15 && export BUILDTIME=80 && export RUNTIME=140 && export CONTSTRAINTS="knl" ; fi
  - if [ "$CI_RUNNER_DESCRIPTION" = "appdev-dvl01" ]; then export SLURMHOST=dvl01 && export SLURMPARTITION=dvl && export CONFIGURETIME=15 && export BUILDTIME=80 && export RUNTIME=140  && export CONTSTRAINTS="x86_64&gpu0&gpu1" && export GEOMETRYRESERVATION="gpu:2" ; fi


  - if [ "$CI_RUNNER_DESCRIPTION" = "appdev-dvl02" ]; then export SLURMHOST=dvl02 && export SLURMPARTITION=dvl && export CONFIGURETIME=15 && export BUILDTIME=80 && export RUNTIME=140 && export CONTSTRAINTS="x86_64&gpu0&gpu1" && export GEOMETRYRESERVATION="gpu:2" ; fi


  - if [ "$CI_RUNNER_DESCRIPTION" = "appdev-miy01" ]; then export SLURMHOST=miy01 && export SLURMPARTITION=minsky && export CONFIGURETIME=15 && export BUILDTIME=80 && export RUNTIME=140 && export CONTSTRAINTS="power8&gpu0&gpu1&gpu2&gpu3" && export GEOMETRYRESERVATION="gpu:4" ; fi
  - if [ "$CI_RUNNER_DESCRIPTION" = "appdev-miy02" ]; then export SLURMHOST=miy02 && export SLURMPARTITION=minsky && export CONFIGURETIME=15 && export BUILDTIME=80 && export RUNTIME=140 && export CONTSTRAINTS="power8&gpu0&gpu1&gpu2&gpu3" && export GEOMETRYRESERVATION="gpu:4" ; fi
  - if [ "$CI_RUNNER_DESCRIPTION" = "appdev-miy03" ]; then export SLURMHOST=miy03 && export SLURMPARTITION=minsky && export CONFIGURETIME=15 && export BUILDTIME=80 && export RUNTIME=140 && export CONTSTRAINTS="power8&gpu0&gpu1&gpu2&gpu3" && export GEOMETRYRESERVATION="gpu:4" ; fi


  - export MATRIX_SIZE=150
  - export NUMBER_OF_EIGENVECTORS=150
  - export BLOCK_SIZE=16
  - if [ "$MEDIUM_MATRIX" = "yes" ]; then export MATRIX_SIZE=1500 && export NUMBER_OF_EIGENVECTORS=750; fi
  - if [ "$LARGE_MATRIX" = "yes" ]; then export MATRIX_SIZE=5000 && export NUMBER_OF_EIGENVECTORS=500; fi
  - if [ "$GPU_BLOCKSIZE" = "yes" ]; then export BLOCK_SIZE=128 ; fi
  - echo "This test will run with matrix size na = $MATRIX_SIZE, nev= $NUMBER_OF_EIGENVECTORS, on a blacs grid with blocksize nblk= $BLOCK_SIZE " 
  - export SKIP_STEP=0
  - ./autogen.sh
  - export SKIP_STEP=0
  - source /etc/profile.d/modules.sh && . ./ci_test_scripts/.ci-env-vars



# For some reason sometimes not-writable files remain, which cause trouble the
# next time a runner tries to clean-up
after_script:
  - chmod u+w -R .



# print coverage results
total_coverage:
  only:
    - /.*master.*/
  stage: coverage
  tags:
    - coverage
  script:
    - echo "Generating coverage report"
    - ./ci_test_scripts/ci_coverage_summary
  artifacts:
    paths:
      - public



pages:
  stage: deploy
  tags:
    - coverage
  script:
    - echo "Publishing pages"
  artifacts:
    paths:
      - public
  only:
    - master



static-build:
  tags:
    - avx
  script:
    - ./configure --enable-option-checking=fatal CFLAGS="-O3 -mavx" FCFLAGS="-O3 -axAVX" SCALAPACK_LDFLAGS="$MKL_INTEL_SCALAPACK_LDFLAGS_NO_MPI_NO_OMP" SCALAPACK_FCFLAGS="$MKL_INTEL_SCALAPACK_FCFLAGS_NO_MPI_NO_OMP" --with-mpi=no FC=ifort --enable-shared=no --enable-static=yes --disable-avx2 || { cat config.log; exit 1; }
    - make -j 8
    - export LD_LIBRARY_PATH=$MKL_HOME/lib/intel64:$LD_LIBRARY_PATH
    - make check TASKS=2 TEST_FLAGS='150 50 16' || { cat test-suite.log; exit 1; }
    - grep -i "Expected %stop" test-suite.log && exit 1 || true ;



# test distcheck
distcheck:
  tags:
    - buildtest
  script:
    - ./configure --enable-option-checking=fatal --with-mpi=no --disable-sse-assembly --disable-sse --disable-avx --disable-avx2 || { cat config.log; exit 1; }
    # stupid 'make distcheck' leaves behind write-protected files that the stupid gitlab runner cannot remove
    - make distcheck DISTCHECK_CONFIGURE_FLAGS="--with-mpi=no --disable-sse-assembly --disable-sse --disable-avx --disable-avx2" TASKS=2 TEST_FLAGS="150 50 16" || { chmod u+rwX -R . ; exit 1 ; }



# test_project_1stage_legacy_api_gnu
test_project_1stage_legacy_api_gnu:
  tags:
    - buildtest
  script:
    - mkdir build
    - pushd build
    - ../autogen.sh
    - ../configure --enable-option-checking=fatal FCFLAGS="-march=native" CFLAGS="-march=native" --disable-avx2 SCALAPACK_LDFLAGS="$MKL_GFORTRAN_SCALAPACK_LDFLAGS_MPI_NO_OMP" SCALAPACK_FCFLAGS="$MKL_GFORTRAN_SCALAPACK_FCFLAGS_MPI_NO_OMP" FC=mpif90 --prefix=$PWD/installdest --disable-avx2 || { cat config.log; exit 1; }
    - make -j 8
    - make install
    - popd
    - mkdir test_project_1stage_legacy_api/build
    - pushd test_project_1stage_legacy_api/build
    - ../autogen.sh
    - ../configure --enable-option-checking=fatal PKG_CONFIG_PATH=../../build/installdest/lib/pkgconfig FC=mpif90 || { cat config.log; exit 1; }
    - make -j 8
    - export LD_LIBRARY_PATH=$MKL_HOME/lib/intel64:$LD_LIBRARY_PATH
    - ./test_real
    - make distclean
    - popd
    - pushd build
    - make distclean
    - rm -rf installdest
    - popd



# test_project_1stage_gnu
test_project_1stage_gnu:
  tags:
    - buildtest
  script:
    - mkdir build
    - pushd build
    - ../autogen.sh
    - ../configure --enable-option-checking=fatal FCFLAGS="-march=native" CFLAGS="-march=native" --disable-avx2 SCALAPACK_LDFLAGS="$MKL_GFORTRAN_SCALAPACK_LDFLAGS_MPI_NO_OMP" SCALAPACK_FCFLAGS="$MKL_GFORTRAN_SCALAPACK_FCFLAGS_MPI_NO_OMP" FC=mpif90 --prefix=$PWD/installdest --disable-avx2 || { cat config.log; exit 1; }
    - make -j 8
    - make install
    - popd
    - mkdir test_project_1stage/build
    - pushd test_project_1stage/build
    - ../autogen.sh
    - ../configure --enable-option-checking=fatal PKG_CONFIG_PATH=../../build/installdest/lib/pkgconfig FC=mpif90 || { cat config.log; exit 1; }
    - make -j 8
    - export LD_LIBRARY_PATH=$MKL_HOME/lib/intel64:$LD_LIBRARY_PATH
    - ./test_real
    - make distclean
    - popd
    - pushd build
    - make distclean
    - rm -rf installdest
    - popd



# test_project_2stage_legacy_api_gnu
test_project_2stage_legacy_api_gnu:
  tags:
    - buildtest
  script:
    - mkdir build
    - pushd build
    - ../autogen.sh
    - ../configure --enable-option-checking=fatal FCFLAGS="-march=native" CFLAGS="-march=native" --disable-avx2 SCALAPACK_LDFLAGS="$MKL_GFORTRAN_SCALAPACK_LDFLAGS_MPI_NO_OMP" SCALAPACK_FCFLAGS="$MKL_GFORTRAN_SCALAPACK_FCFLAGS_MPI_NO_OMP" FC=mpif90 --prefix=$PWD/installdest --disable-avx2 || { cat config.log; exit 1; }
    - make -j 8
    - make install
    - popd
    - mkdir test_project_2stage_legacy_api/build
    - pushd test_project_2stage_legacy_api/build
    - ../autogen.sh
    - ../configure --enable-option-checking=fatal PKG_CONFIG_PATH=../../build/installdest/lib/pkgconfig FC=mpif90 || { cat config.log; exit 1; }
    - make -j 8
    - export LD_LIBRARY_PATH=$MKL_HOME/lib/intel64:$LD_LIBRARY_PATH
    - ./test_real2
    - make distclean
    - popd
    - pushd build
    - make distclean
    - rm -rf installdest
    - popd



# test_project_2stage_gnu
test_project_2stage_gnu:
  tags:
    - buildtest
  script:
    - mkdir build
    - pushd build
    - ../autogen.sh
    - ../configure --enable-option-checking=fatal FCFLAGS="-march=native" CFLAGS="-march=native" --disable-avx2 SCALAPACK_LDFLAGS="$MKL_GFORTRAN_SCALAPACK_LDFLAGS_MPI_NO_OMP" SCALAPACK_FCFLAGS="$MKL_GFORTRAN_SCALAPACK_FCFLAGS_MPI_NO_OMP" FC=mpif90 --prefix=$PWD/installdest --disable-avx2 || { cat config.log; exit 1; }
    - make -j 8
    - make install
    - popd
    - mkdir test_project_2stage/build
    - pushd test_project_2stage/build
    - ../autogen.sh
    - ../configure --enable-option-checking=fatal PKG_CONFIG_PATH=../../build/installdest/lib/pkgconfig FC=mpif90 || { cat config.log; exit 1; }
    - make -j 8
    - export LD_LIBRARY_PATH=$MKL_HOME/lib/intel64:$LD_LIBRARY_PATH
    - ./test_real2
    - make distclean
    - popd
    - pushd build
    - make distclean
    - rm -rf installdest
    - popd



# test_project_1stage_legacy_api_intel
test_project_1stage_legacy_api_intel:
  tags:
    - buildtest
  script:
    - mkdir build
    - pushd build
    - ../autogen.sh
    - ../configure --enable-option-checking=fatal FCFLAGS="-march=native" CFLAGS="-march=native" --disable-avx2 SCALAPACK_LDFLAGS="$MKL_INTEL_SCALAPACK_LDFLAGS_MPI_NO_OMP" SCALAPACK_FCFLAGS="$MKL_INTEL_SCALAPACK_FCFLAGS_MPI_NO_OMP" FC=mpiifort --prefix=$PWD/installdest --disable-avx2 || { cat config.log; exit 1; }
    - make -j 8
    - make install
    - popd
    - mkdir test_project_1stage_legacy_api/build
    - pushd test_project_1stage_legacy_api/build
    - ../autogen.sh
    - ../configure --enable-option-checking=fatal PKG_CONFIG_PATH=../../build/installdest/lib/pkgconfig FC=mpiifort || { cat config.log; exit 1; }
    - make -j 8
    - export LD_LIBRARY_PATH=$MKL_HOME/lib/intel64:$LD_LIBRARY_PATH
    - ./test_real
    - make distclean
    - popd
    - pushd build
    - make distclean
    - rm -rf installdest
    - popd



# test_project_1stage_intel
test_project_1stage_intel:
  tags:
    - buildtest
  script:
    - mkdir build
    - pushd build
    - ../autogen.sh
    - ../configure --enable-option-checking=fatal FCFLAGS="-march=native" CFLAGS="-march=native" --disable-avx2 SCALAPACK_LDFLAGS="$MKL_INTEL_SCALAPACK_LDFLAGS_MPI_NO_OMP" SCALAPACK_FCFLAGS="$MKL_INTEL_SCALAPACK_FCFLAGS_MPI_NO_OMP" FC=mpiifort --prefix=$PWD/installdest --disable-avx2 || { cat config.log; exit 1; }
    - make -j 8
    - make install
    - popd
    - mkdir test_project_1stage/build
    - pushd test_project_1stage/build
    - ../autogen.sh
    - ../configure --enable-option-checking=fatal PKG_CONFIG_PATH=../../build/installdest/lib/pkgconfig FC=mpiifort || { cat config.log; exit 1; }
    - make -j 8
    - export LD_LIBRARY_PATH=$MKL_HOME/lib/intel64:$LD_LIBRARY_PATH
    - ./test_real
    - make distclean
    - popd
    - pushd build
    - make distclean
    - rm -rf installdest
    - popd



# test_project_2stage_legacy_api_intel
test_project_2stage_legacy_api_intel:
  tags:
    - buildtest
  script:
    - mkdir build
    - pushd build
    - ../autogen.sh
    - ../configure --enable-option-checking=fatal FCFLAGS="-march=native" CFLAGS="-march=native" --disable-avx2 SCALAPACK_LDFLAGS="$MKL_INTEL_SCALAPACK_LDFLAGS_MPI_NO_OMP" SCALAPACK_FCFLAGS="$MKL_INTEL_SCALAPACK_FCFLAGS_MPI_NO_OMP" FC=mpiifort --prefix=$PWD/installdest --disable-avx2 || { cat config.log; exit 1; }
    - make -j 8
    - make install
    - popd
    - mkdir test_project_2stage_legacy_api/build
    - pushd test_project_2stage_legacy_api/build
    - ../autogen.sh
    - ../configure --enable-option-checking=fatal PKG_CONFIG_PATH=../../build/installdest/lib/pkgconfig FC=mpiifort || { cat config.log; exit 1; }
    - make -j 8
    - export LD_LIBRARY_PATH=$MKL_HOME/lib/intel64:$LD_LIBRARY_PATH
    - ./test_real2
    - make distclean
    - popd
    - pushd build
    - make distclean
    - rm -rf installdest
    - popd



# test_project_2stage_intel
test_project_2stage_intel:
  tags:
    - buildtest
  script:
    - mkdir build
    - pushd build
    - ../autogen.sh
    - ../configure --enable-option-checking=fatal FCFLAGS="-march=native" CFLAGS="-march=native" --disable-avx2 SCALAPACK_LDFLAGS="$MKL_INTEL_SCALAPACK_LDFLAGS_MPI_NO_OMP" SCALAPACK_FCFLAGS="$MKL_INTEL_SCALAPACK_FCFLAGS_MPI_NO_OMP" FC=mpiifort --prefix=$PWD/installdest --disable-avx2 || { cat config.log; exit 1; }
    - make -j 8
    - make install
    - popd
    - mkdir test_project_2stage/build
    - pushd test_project_2stage/build
    - ../autogen.sh
    - ../configure --enable-option-checking=fatal PKG_CONFIG_PATH=../../build/installdest/lib/pkgconfig FC=mpiifort || { cat config.log; exit 1; }
    - make -j 8
    - export LD_LIBRARY_PATH=$MKL_HOME/lib/intel64:$LD_LIBRARY_PATH
    - ./test_real2
    - make distclean
    - popd
    - pushd build
    - make distclean
    - rm -rf installdest
    - popd



#The tests follow here
# gnu-gnu-mpi-noopenmp-double-precision-assumed-size-band-to-full-blocking-no-gpu-coverage-avx-no-address-sanitize
gnu-gnu-mpi-noopenmp-double-precision-assumed-size-band-to-full-blocking-no-gpu-coverage-avx-no-address-sanitize-jobs:
  only:
    - /.*master.*/
  tags:
    - coverage
  script:
   - ./ci_test_scripts/run_ci_tests.sh -c " CC=\"mpicc\" CFLAGS=\"-O3 -mavx --coverage -O0\" FC=\"mpif90\" FCFLAGS=\"-O3 -mavx --coverage -O0\"    SCALAPACK_LDFLAGS=\"$MKL_GFORTRAN_SCALAPACK_LDFLAGS_MPI_NO_OMP \"  SCALAPACK_FCFLAGS=\"$MKL_GFORTRAN_SCALAPACK_FCFLAGS_MPI_NO_OMP \" --enable-option-checking=fatal --with-mpi=yes --disable-mpi-module --disable-openmp --disable-single-precision --enable-assumed-size --enable-band-to-full-blocking --disable-gpu --enable-avx --disable-avx2" -j 8 -t 2 -m $MATRIX_SIZE -n $NUMBER_OF_EIGENVECTORS -b $BLOCK_SIZE -s $SKIP_STEP 



# gnu-gnu-mpi-noopenmp-double-precision-assumed-size-band-to-full-blocking-no-gpu-coverage-avx2-no-address-sanitize
gnu-gnu-mpi-noopenmp-double-precision-assumed-size-band-to-full-blocking-no-gpu-coverage-avx2-no-address-sanitize-jobs:
  only:
    - /.*master.*/
  tags:
    - avx2-coverage
  script:
    - export REQUESTED_MEMORY=2Gb


    - echo "The tasks will be submitted to SLURM PARTITION " $SLURMPARTITION " on host " $SLURMHOST "with constraints " $CONTSTRAINTS 
    - export SRUN_COMMANDLINE_CONFIGURE="--partition=$SLURMPARTITION --nodelist=$SLURMHOST --time=$CONFIGURETIME --constraint=$CONTSTRAINTS --mem=$REQUESTED_MEMORY" 
    - export SRUN_COMMANDLINE_BUILD="--partition=$SLURMPARTITION --nodelist=$SLURMHOST --time=$BUILDTIME --constraint=$CONTSTRAINTS --mem=$REQUESTED_MEMORY " 
    - export SRUN_COMMANDLINE_RUN="--partition=$SLURMPARTITION --nodelist=$SLURMHOST --time=$RUNTIME --constraint=$CONTSTRAINTS --mem=$REQUESTED_MEMORY " 
    - ./ci_test_scripts/run_ci_tests.sh -c " CC=\"mpicc\" CFLAGS=\"-O3 -mavx2 -mfma --coverage -O0\" FC=\"mpif90\" FCFLAGS=\"-O3 -mavx2 -mfma --coverage -O0\"    SCALAPACK_LDFLAGS=\"\$MKL_GFORTRAN_SCALAPACK_LDFLAGS_MPI_NO_OMP \"  SCALAPACK_FCFLAGS=\"\$MKL_GFORTRAN_SCALAPACK_FCFLAGS_MPI_NO_OMP \" --enable-option-checking=fatal --enable-scalapack-tests --with-mpi=yes --disable-mpi-module --disable-openmp --disable-single-precision --enable-assumed-size --enable-band-to-full-blocking --disable-gpu --enable-avx2" -j 8 -t 2 -m $MATRIX_SIZE -n $NUMBER_OF_EIGENVECTORS -b $BLOCK_SIZE -s $SKIP_STEP -q "srun" 
    - sleep 1
    - sleep 1
    -  echo "na= $MATRIX_SIZE, nev= $NUMBER_OF_EIGENVECTORS nblock= $BLOCK_SIZE" 
    - ./ci_test_scripts/ci_coverage_collect
  artifacts:
    paths:
      - coverage_data



# gnu-gnu-mpi-noopenmp-double-precision-assumed-size-band-to-full-blocking-no-gpu-coverage-avx512-no-address-sanitize
gnu-gnu-mpi-noopenmp-double-precision-assumed-size-band-to-full-blocking-no-gpu-coverage-avx512-no-address-sanitize-jobs:
  only:
    - /.*master.*/
  tags:
    - avx512-coverage
  script:
    - export REQUESTED_MEMORY=2Gb


    - echo "The tasks will be submitted to SLURM PARTITION " $SLURMPARTITION " on host " $SLURMHOST "with constraints " $CONTSTRAINTS 
    - export SRUN_COMMANDLINE_CONFIGURE="--partition=$SLURMPARTITION --nodelist=$SLURMHOST --time=$CONFIGURETIME --constraint=$CONTSTRAINTS --mem=$REQUESTED_MEMORY" 
    - export SRUN_COMMANDLINE_BUILD="--partition=$SLURMPARTITION --nodelist=$SLURMHOST --time=$BUILDTIME --constraint=$CONTSTRAINTS --mem=$REQUESTED_MEMORY " 
    - export SRUN_COMMANDLINE_RUN="--partition=$SLURMPARTITION --nodelist=$SLURMHOST --time=$RUNTIME --constraint=$CONTSTRAINTS --mem=$REQUESTED_MEMORY " 
    - ./ci_test_scripts/run_ci_tests.sh -c " CC=\"mpicc\" CFLAGS=\"-O3 -march=skylake-avx512 --coverage -O0\" FC=\"mpif90\" FCFLAGS=\"-O3 -march=skylake-avx512 --coverage -O0\"    SCALAPACK_LDFLAGS=\"\$MKL_GFORTRAN_SCALAPACK_LDFLAGS_MPI_NO_OMP \"  SCALAPACK_FCFLAGS=\"\$MKL_GFORTRAN_SCALAPACK_FCFLAGS_MPI_NO_OMP \" --enable-option-checking=fatal --with-mpi=yes --disable-mpi-module --disable-openmp --disable-single-precision --enable-assumed-size --enable-band-to-full-blocking --disable-gpu --enable-avx512" -j 8 -t 2 -m $MATRIX_SIZE -n $NUMBER_OF_EIGENVECTORS -b $BLOCK_SIZE -s $SKIP_STEP -q "srun" 
    - sleep 1
    - sleep 1
    -  echo "na= $MATRIX_SIZE, nev= $NUMBER_OF_EIGENVECTORS nblock= $BLOCK_SIZE" 
    - ./ci_test_scripts/ci_coverage_collect
  artifacts:
    paths:
      - coverage_data



# gnu-gnu-mpi-noopenmp-double-precision-assumed-size-band-to-full-blocking-no-gpu-no-coverage-avx-address-sanitize
gnu-gnu-mpi-noopenmp-double-precision-assumed-size-band-to-full-blocking-no-gpu-no-coverage-avx-address-sanitize-jobs:
  only:
    - /.*master.*/
  tags:
    - avx
  script:
   - ./ci_test_scripts/run_ci_tests.sh -c " CC=\"mpicc\" CFLAGS=\"-O3 -mavx -fsanitize=address\" FC=\"mpif90\" FCFLAGS=\"-O3 -mavx -fsanitize=address\"    SCALAPACK_LDFLAGS=\"$MKL_GFORTRAN_SCALAPACK_LDFLAGS_MPI_NO_OMP \"  SCALAPACK_FCFLAGS=\"$MKL_GFORTRAN_SCALAPACK_FCFLAGS_MPI_NO_OMP \" --enable-option-checking=fatal --with-mpi=yes --disable-mpi-module --disable-openmp --disable-single-precision --enable-assumed-size --enable-band-to-full-blocking --disable-gpu --enable-avx --disable-avx2" -j 8 -t 2 -m $MATRIX_SIZE -n $NUMBER_OF_EIGENVECTORS -b $BLOCK_SIZE -s $SKIP_STEP 



# gnu-gnu-mpi-noopenmp-double-precision-assumed-size-band-to-full-blocking-no-gpu-no-coverage-avx-no-address-sanitize
gnu-gnu-mpi-noopenmp-double-precision-assumed-size-band-to-full-blocking-no-gpu-no-coverage-avx-no-address-sanitize-jobs:
  only:
    - /.*master.*/
  tags:
    - avx
  script:
   - ./ci_test_scripts/run_ci_tests.sh -c " CC=\"mpicc\" CFLAGS=\"-O3 -mavx\" FC=\"mpif90\" FCFLAGS=\"-O3 -mavx\"    SCALAPACK_LDFLAGS=\"$MKL_GFORTRAN_SCALAPACK_LDFLAGS_MPI_NO_OMP \"  SCALAPACK_FCFLAGS=\"$MKL_GFORTRAN_SCALAPACK_FCFLAGS_MPI_NO_OMP \" --enable-option-checking=fatal --with-mpi=yes --disable-mpi-module --disable-openmp --disable-single-precision --enable-assumed-size --enable-band-to-full-blocking --disable-gpu --enable-avx --disable-avx2" -j 8 -t 2 -m $MATRIX_SIZE -n $NUMBER_OF_EIGENVECTORS -b $BLOCK_SIZE -s $SKIP_STEP 



# gnu-gnu-mpi-noopenmp-double-precision-assumed-size-band-to-full-blocking-no-gpu-no-coverage-avx2-address-sanitize
gnu-gnu-mpi-noopenmp-double-precision-assumed-size-band-to-full-blocking-no-gpu-no-coverage-avx2-address-sanitize-jobs:
  only:
    - /.*master.*/
  tags:
    - avx2
  script:
    - export REQUESTED_MEMORY=2Gb


    - echo "The tasks will be submitted to SLURM PARTITION " $SLURMPARTITION " on host " $SLURMHOST "with constraints " $CONTSTRAINTS 
    - export SRUN_COMMANDLINE_CONFIGURE="--partition=$SLURMPARTITION --nodelist=$SLURMHOST --time=$CONFIGURETIME --constraint=$CONTSTRAINTS --mem=$REQUESTED_MEMORY" 
    - export SRUN_COMMANDLINE_BUILD="--partition=$SLURMPARTITION --nodelist=$SLURMHOST --time=$BUILDTIME --constraint=$CONTSTRAINTS --mem=$REQUESTED_MEMORY " 
    - export SRUN_COMMANDLINE_RUN="--partition=$SLURMPARTITION --nodelist=$SLURMHOST --time=$RUNTIME --constraint=$CONTSTRAINTS --mem=$REQUESTED_MEMORY " 
    - ./ci_test_scripts/run_ci_tests.sh -c " CC=\"mpicc\" CFLAGS=\"-O3 -mavx2 -mfma -fsanitize=address\" FC=\"mpif90\" FCFLAGS=\"-O3 -mavx2 -mfma -fsanitize=address\"    SCALAPACK_LDFLAGS=\"\$MKL_GFORTRAN_SCALAPACK_LDFLAGS_MPI_NO_OMP \"  SCALAPACK_FCFLAGS=\"\$MKL_GFORTRAN_SCALAPACK_FCFLAGS_MPI_NO_OMP \" --enable-option-checking=fatal --with-mpi=yes --disable-mpi-module --disable-openmp --disable-single-precision --enable-assumed-size --enable-band-to-full-blocking --disable-gpu --enable-avx2" -j 8 -t 2 -m $MATRIX_SIZE -n $NUMBER_OF_EIGENVECTORS -b $BLOCK_SIZE -s $SKIP_STEP -q "srun" 
    - sleep 1
    - sleep 1
    -  echo "na= $MATRIX_SIZE, nev= $NUMBER_OF_EIGENVECTORS nblock= $BLOCK_SIZE" 



# gnu-gnu-mpi-noopenmp-double-precision-assumed-size-band-to-full-blocking-no-gpu-no-coverage-avx2-no-address-sanitize
gnu-gnu-mpi-noopenmp-double-precision-assumed-size-band-to-full-blocking-no-gpu-no-coverage-avx2-no-address-sanitize-jobs:
  only:
    - /.*master.*/
  tags:
    - avx2
  script:
    - export REQUESTED_MEMORY=2Gb


    - echo "The tasks will be submitted to SLURM PARTITION " $SLURMPARTITION " on host " $SLURMHOST "with constraints " $CONTSTRAINTS 
    - export SRUN_COMMANDLINE_CONFIGURE="--partition=$SLURMPARTITION --nodelist=$SLURMHOST --time=$CONFIGURETIME --constraint=$CONTSTRAINTS --mem=$REQUESTED_MEMORY" 
    - export SRUN_COMMANDLINE_BUILD="--partition=$SLURMPARTITION --nodelist=$SLURMHOST --time=$BUILDTIME --constraint=$CONTSTRAINTS --mem=$REQUESTED_MEMORY " 
    - export SRUN_COMMANDLINE_RUN="--partition=$SLURMPARTITION --nodelist=$SLURMHOST --time=$RUNTIME --constraint=$CONTSTRAINTS --mem=$REQUESTED_MEMORY " 
    - ./ci_test_scripts/run_ci_tests.sh -c " CC=\"mpicc\" CFLAGS=\"-O3 -mavx2 -mfma\" FC=\"mpif90\" FCFLAGS=\"-O3 -mavx2 -mfma\"    SCALAPACK_LDFLAGS=\"\$MKL_GFORTRAN_SCALAPACK_LDFLAGS_MPI_NO_OMP \"  SCALAPACK_FCFLAGS=\"\$MKL_GFORTRAN_SCALAPACK_FCFLAGS_MPI_NO_OMP \" --enable-option-checking=fatal --with-mpi=yes --disable-mpi-module --disable-openmp --disable-single-precision --enable-assumed-size --enable-band-to-full-blocking --disable-gpu --enable-avx2" -j 8 -t 2 -m $MATRIX_SIZE -n $NUMBER_OF_EIGENVECTORS -b $BLOCK_SIZE -s $SKIP_STEP -q "srun" 
    - sleep 1
    - sleep 1
    -  echo "na= $MATRIX_SIZE, nev= $NUMBER_OF_EIGENVECTORS nblock= $BLOCK_SIZE" 



# gnu-gnu-mpi-noopenmp-double-precision-assumed-size-band-to-full-blocking-no-gpu-no-coverage-avx512-address-sanitize
gnu-gnu-mpi-noopenmp-double-precision-assumed-size-band-to-full-blocking-no-gpu-no-coverage-avx512-address-sanitize-jobs:
  only:
    - /.*master.*/
  tags:
    - avx512
  script:
    - export REQUESTED_MEMORY=2Gb


    - echo "The tasks will be submitted to SLURM PARTITION " $SLURMPARTITION " on host " $SLURMHOST "with constraints " $CONTSTRAINTS 
    - export SRUN_COMMANDLINE_CONFIGURE="--partition=$SLURMPARTITION --nodelist=$SLURMHOST --time=$CONFIGURETIME --constraint=$CONTSTRAINTS --mem=$REQUESTED_MEMORY" 
    - export SRUN_COMMANDLINE_BUILD="--partition=$SLURMPARTITION --nodelist=$SLURMHOST --time=$BUILDTIME --constraint=$CONTSTRAINTS --mem=$REQUESTED_MEMORY " 
    - export SRUN_COMMANDLINE_RUN="--partition=$SLURMPARTITION --nodelist=$SLURMHOST --time=$RUNTIME --constraint=$CONTSTRAINTS --mem=$REQUESTED_MEMORY " 
    - ./ci_test_scripts/run_ci_tests.sh -c " CC=\"mpicc\" CFLAGS=\"-O3 -march=skylake-avx512 -fsanitize=address\" FC=\"mpif90\" FCFLAGS=\"-O3 -march=skylake-avx512 -fsanitize=address\"    SCALAPACK_LDFLAGS=\"\$MKL_GFORTRAN_SCALAPACK_LDFLAGS_MPI_NO_OMP \"  SCALAPACK_FCFLAGS=\"\$MKL_GFORTRAN_SCALAPACK_FCFLAGS_MPI_NO_OMP \" --enable-option-checking=fatal --with-mpi=yes --disable-mpi-module --disable-openmp --disable-single-precision --enable-assumed-size --enable-band-to-full-blocking --disable-gpu --enable-avx512" -j 8 -t 2 -m $MATRIX_SIZE -n $NUMBER_OF_EIGENVECTORS -b $BLOCK_SIZE -s $SKIP_STEP -q "srun" 
    - sleep 1
    - sleep 1
    -  echo "na= $MATRIX_SIZE, nev= $NUMBER_OF_EIGENVECTORS nblock= $BLOCK_SIZE" 



# gnu-gnu-mpi-noopenmp-double-precision-assumed-size-band-to-full-blocking-no-gpu-no-coverage-avx512-no-address-sanitize
gnu-gnu-mpi-noopenmp-double-precision-assumed-size-band-to-full-blocking-no-gpu-no-coverage-avx512-no-address-sanitize-jobs:
  only:
    - /.*master.*/
  tags:
    - avx512
  script:
    - export REQUESTED_MEMORY=2Gb


    - echo "The tasks will be submitted to SLURM PARTITION " $SLURMPARTITION " on host " $SLURMHOST "with constraints " $CONTSTRAINTS 
    - export SRUN_COMMANDLINE_CONFIGURE="--partition=$SLURMPARTITION --nodelist=$SLURMHOST --time=$CONFIGURETIME --constraint=$CONTSTRAINTS --mem=$REQUESTED_MEMORY" 
    - export SRUN_COMMANDLINE_BUILD="--partition=$SLURMPARTITION --nodelist=$SLURMHOST --time=$BUILDTIME --constraint=$CONTSTRAINTS --mem=$REQUESTED_MEMORY " 
    - export SRUN_COMMANDLINE_RUN="--partition=$SLURMPARTITION --nodelist=$SLURMHOST --time=$RUNTIME --constraint=$CONTSTRAINTS --mem=$REQUESTED_MEMORY " 
    - ./ci_test_scripts/run_ci_tests.sh -c " CC=\"mpicc\" CFLAGS=\"-O3 -march=skylake-avx512\" FC=\"mpif90\" FCFLAGS=\"-O3 -march=skylake-avx512\"    SCALAPACK_LDFLAGS=\"\$MKL_GFORTRAN_SCALAPACK_LDFLAGS_MPI_NO_OMP \"  SCALAPACK_FCFLAGS=\"\$MKL_GFORTRAN_SCALAPACK_FCFLAGS_MPI_NO_OMP \" --enable-option-checking=fatal --with-mpi=yes --disable-mpi-module --disable-openmp --disable-single-precision --enable-assumed-size --enable-band-to-full-blocking --disable-gpu --enable-avx512" -j 8 -t 2 -m $MATRIX_SIZE -n $NUMBER_OF_EIGENVECTORS -b $BLOCK_SIZE -s $SKIP_STEP -q "srun" 
    - sleep 1
    - sleep 1
    -  echo "na= $MATRIX_SIZE, nev= $NUMBER_OF_EIGENVECTORS nblock= $BLOCK_SIZE" 



# gnu-gnu-mpi-noopenmp-double-precision-assumed-size-band-to-full-blocking-no-gpu-no-coverage-sse-address-sanitize
gnu-gnu-mpi-noopenmp-double-precision-assumed-size-band-to-full-blocking-no-gpu-no-coverage-sse-address-sanitize-jobs:
  only:
    - /.*master.*/
  tags:
    - sse
  script:
   - if [ $MATRIX_SIZE -gt 150 ]; then export SKIP_STEP=1 ; fi # our SSE test machines do not have a lot of memory
   - ./ci_test_scripts/run_ci_tests.sh -c " CC=\"mpicc\" CFLAGS=\"-O3 -msse4.2 -fsanitize=address\" FC=\"mpif90\" FCFLAGS=\"-O3 -msse4.2 -fsanitize=address\"    SCALAPACK_LDFLAGS=\"$MKL_GFORTRAN_SCALAPACK_LDFLAGS_MPI_NO_OMP \"  SCALAPACK_FCFLAGS=\"$MKL_GFORTRAN_SCALAPACK_FCFLAGS_MPI_NO_OMP \" --enable-option-checking=fatal --with-mpi=yes --disable-mpi-module --disable-openmp --disable-single-precision --enable-assumed-size --enable-band-to-full-blocking --disable-gpu --enable-sse --enable-sse-assembly --disable-avx --disable-avx2" -j 8 -t 2 -m $MATRIX_SIZE -n $NUMBER_OF_EIGENVECTORS -b $BLOCK_SIZE -s $SKIP_STEP 



# gnu-gnu-mpi-noopenmp-double-precision-assumed-size-band-to-full-blocking-no-gpu-no-coverage-sse-no-address-sanitize
gnu-gnu-mpi-noopenmp-double-precision-assumed-size-band-to-full-blocking-no-gpu-no-coverage-sse-no-address-sanitize-jobs:
  only:
    - /.*master.*/
  tags:
    - sse
  script:
   - if [ $MATRIX_SIZE -gt 150 ]; then export SKIP_STEP=1 ; fi # our SSE test machines do not have a lot of memory
   - ./ci_test_scripts/run_ci_tests.sh -c " CC=\"mpicc\" CFLAGS=\"-O3 -msse4.2\" FC=\"mpif90\" FCFLAGS=\"-O3 -msse4.2\"    SCALAPACK_LDFLAGS=\"$MKL_GFORTRAN_SCALAPACK_LDFLAGS_MPI_NO_OMP \"  SCALAPACK_FCFLAGS=\"$MKL_GFORTRAN_SCALAPACK_FCFLAGS_MPI_NO_OMP \" --enable-option-checking=fatal --with-mpi=yes --disable-mpi-module --disable-openmp --disable-single-precision --enable-assumed-size --enable-band-to-full-blocking --disable-gpu --enable-sse --enable-sse-assembly --disable-avx --disable-avx2" -j 8 -t 2 -m $MATRIX_SIZE -n $NUMBER_OF_EIGENVECTORS -b $BLOCK_SIZE -s $SKIP_STEP 



# gnu-gnu-mpi-noopenmp-double-precision-assumed-size-band-to-full-blocking-with-gpu-no-coverage-avx-no-address-sanitize
gnu-gnu-mpi-noopenmp-double-precision-assumed-size-band-to-full-blocking-with-gpu-no-coverage-avx-no-address-sanitize-jobs:
  only:
    - /.*master.*/
  tags:
    - gpu
  script:
    - export REQUESTED_MEMORY=2Gb


    - echo "The tasks will be submitted to SLURM PARTITION " $SLURMPARTITION " on host " $SLURMHOST " with constraints " $CONTSTRAINTS " with the geometry " $GEOMETRYRESERVATION
    - export SRUN_COMMANDLINE_CONFIGURE="--partition=$SLURMPARTITION --nodelist=$SLURMHOST --time=$CONFIGURETIME --constraint=$CONTSTRAINTS --gres=$GEOMETRYRESERVATION " 
    - export SRUN_COMMANDLINE_BUILD="--partition=$SLURMPARTITION --nodelist=$SLURMHOST --time=$BUILDTIME --constraint=$CONTSTRAINTS --gres=$GEOMETRYRESERVATION " 
    - export SRUN_COMMANDLINE_RUN="--partition=$SLURMPARTITION --nodelist=$SLURMHOST --time=$RUNTIME --constraint=$CONTSTRAINTS --gres=$GEOMETRYRESERVATION " 
    - ./ci_test_scripts/run_ci_tests.sh -c " CC=\"mpicc\" CFLAGS=\"-O3 -mavx\" FC=\"mpif90\" FCFLAGS=\"-O3 -mavx\"    SCALAPACK_LDFLAGS=\"\$MKL_GFORTRAN_SCALAPACK_LDFLAGS_MPI_NO_OMP  -L\$CUDA_HOME/lib64 -lcublas -I\$CUDA_HOME/include\"  SCALAPACK_FCFLAGS=\"\$MKL_GFORTRAN_SCALAPACK_FCFLAGS_MPI_NO_OMP  -I\$CUDA_HOME/include\" --enable-option-checking=fatal --with-mpi=yes --disable-mpi-module --disable-openmp --disable-single-precision --enable-assumed-size --enable-band-to-full-blocking --enable-gpu --with-cuda-path=\$CUDA_HOME/ --enable-avx --disable-avx2" -j 8 -t 2 -m $MATRIX_SIZE -n $NUMBER_OF_EIGENVECTORS -b $BLOCK_SIZE -s $SKIP_STEP -q "srun" 
    - sleep 1
    - sleep 1
    -  echo "na= $MATRIX_SIZE, nev= $NUMBER_OF_EIGENVECTORS nblock= $BLOCK_SIZE" 



# gnu-gnu-mpi-noopenmp-double-precision-assumed-size-band-to-full-blocking-with-gpu-no-coverage-power8-no-address-sanitize
gnu-gnu-mpi-noopenmp-double-precision-assumed-size-band-to-full-blocking-with-gpu-no-coverage-power8-no-address-sanitize-jobs:
  only:
    - /.*master.*/
  allow_failure: true
  tags:
    - minsky
  script:
    - export REQUESTED_MEMORY=2Gb


    - echo "The tasks will be submitted to SLURM PARTITION " $SLURMPARTITION " on host " $SLURMHOST " with constraints " $CONTSTRAINTS " with the geometry " $GEOMETRYRESERVATION
    - export SRUN_COMMANDLINE_CONFIGURE="--partition=$SLURMPARTITION --nodelist=$SLURMHOST --time=$CONFIGURETIME --constraint=$CONTSTRAINTS --gres=$GEOMETRYRESERVATION " 
    - export SRUN_COMMANDLINE_BUILD="--partition=$SLURMPARTITION --nodelist=$SLURMHOST --time=$BUILDTIME --constraint=$CONTSTRAINTS --gres=$GEOMETRYRESERVATION " 
    - export SRUN_COMMANDLINE_RUN="--partition=$SLURMPARTITION --nodelist=$SLURMHOST --time=$RUNTIME --constraint=$CONTSTRAINTS --gres=$GEOMETRYRESERVATION " 
    - ./ci_test_scripts/run_ci_tests.sh -c " CC=\"mpicc\" CFLAGS=\"-O2 -I\$CUDA_HOME/include\" FC=\"mpifort\" FCFLAGS=\"-O2\" LIBS=\" -lessl -lreflapack -lessl -lcublas -lgfortran \"  LDFLAGS=\" -L/home/elpa/libs/scalapack/lib -L\$CUDA_HOME/lib64 \"    --enable-option-checking=fatal --with-mpi=yes --disable-mpi-module --disable-openmp --disable-single-precision --enable-assumed-size --enable-band-to-full-blocking --enable-gpu --with-cuda-path=\$CUDA_HOME/ --enable-vsx --disable-sse --disable-sse-assembly --disable-avx --disable-avx2 --disable-mpi-module --with-GPU-compute-capability=sm_60 " -j 8 -t 2 -m $MATRIX_SIZE -n $NUMBER_OF_EIGENVECTORS -b $BLOCK_SIZE -s $SKIP_STEP -q "srun" 
    - sleep 1
    - sleep 1
    -  echo "na= $MATRIX_SIZE, nev= $NUMBER_OF_EIGENVECTORS nblock= $BLOCK_SIZE" 



# gnu-gnu-mpi-noopenmp-double-precision-assumed-size-no-band-to-full-blocking-no-gpu-coverage-avx-no-address-sanitize
gnu-gnu-mpi-noopenmp-double-precision-assumed-size-no-band-to-full-blocking-no-gpu-coverage-avx-no-address-sanitize-jobs:
  only:
    - /.*master.*/
  tags:
    - coverage
  script:
   - ./ci_test_scripts/run_ci_tests.sh -c " CC=\"mpicc\" CFLAGS=\"-O3 -mavx --coverage -O0\" FC=\"mpif90\" FCFLAGS=\"-O3 -mavx --coverage -O0\"    SCALAPACK_LDFLAGS=\"$MKL_GFORTRAN_SCALAPACK_LDFLAGS_MPI_NO_OMP \"  SCALAPACK_FCFLAGS=\"$MKL_GFORTRAN_SCALAPACK_FCFLAGS_MPI_NO_OMP \" --enable-option-checking=fatal --with-mpi=yes --disable-mpi-module --disable-openmp --disable-single-precision --enable-assumed-size --disable-band-to-full-blocking --disable-gpu --enable-avx --disable-avx2" -j 8 -t 2 -m $MATRIX_SIZE -n $NUMBER_OF_EIGENVECTORS -b $BLOCK_SIZE -s $SKIP_STEP 



# gnu-gnu-mpi-noopenmp-double-precision-assumed-size-no-band-to-full-blocking-no-gpu-coverage-avx2-no-address-sanitize
gnu-gnu-mpi-noopenmp-double-precision-assumed-size-no-band-to-full-blocking-no-gpu-coverage-avx2-no-address-sanitize-jobs:
  only:
    - /.*master.*/
  tags:
    - avx2-coverage
  script:
    - export REQUESTED_MEMORY=2Gb


    - echo "The tasks will be submitted to SLURM PARTITION " $SLURMPARTITION " on host " $SLURMHOST "with constraints " $CONTSTRAINTS 
    - export SRUN_COMMANDLINE_CONFIGURE="--partition=$SLURMPARTITION --nodelist=$SLURMHOST --time=$CONFIGURETIME --constraint=$CONTSTRAINTS --mem=$REQUESTED_MEMORY" 
    - export SRUN_COMMANDLINE_BUILD="--partition=$SLURMPARTITION --nodelist=$SLURMHOST --time=$BUILDTIME --constraint=$CONTSTRAINTS --mem=$REQUESTED_MEMORY " 
    - export SRUN_COMMANDLINE_RUN="--partition=$SLURMPARTITION --nodelist=$SLURMHOST --time=$RUNTIME --constraint=$CONTSTRAINTS --mem=$REQUESTED_MEMORY " 
    - ./ci_test_scripts/run_ci_tests.sh -c " CC=\"mpicc\" CFLAGS=\"-O3 -mavx2 -mfma --coverage -O0\" FC=\"mpif90\" FCFLAGS=\"-O3 -mavx2 -mfma --coverage -O0\"    SCALAPACK_LDFLAGS=\"\$MKL_GFORTRAN_SCALAPACK_LDFLAGS_MPI_NO_OMP \"  SCALAPACK_FCFLAGS=\"\$MKL_GFORTRAN_SCALAPACK_FCFLAGS_MPI_NO_OMP \" --enable-option-checking=fatal --enable-scalapack-tests --with-mpi=yes --disable-mpi-module --disable-openmp --disable-single-precision --enable-assumed-size --disable-band-to-full-blocking --disable-gpu --enable-avx2" -j 8 -t 2 -m $MATRIX_SIZE -n $NUMBER_OF_EIGENVECTORS -b $BLOCK_SIZE -s $SKIP_STEP -q "srun" 
    - sleep 1
    - sleep 1
    -  echo "na= $MATRIX_SIZE, nev= $NUMBER_OF_EIGENVECTORS nblock= $BLOCK_SIZE" 
    - ./ci_test_scripts/ci_coverage_collect
  artifacts:
    paths:
      - coverage_data



# gnu-gnu-mpi-noopenmp-double-precision-assumed-size-no-band-to-full-blocking-no-gpu-coverage-avx512-no-address-sanitize
gnu-gnu-mpi-noopenmp-double-precision-assumed-size-no-band-to-full-blocking-no-gpu-coverage-avx512-no-address-sanitize-jobs:
  only:
    - /.*master.*/
  tags:
    - avx512-coverage
  script:
    - export REQUESTED_MEMORY=2Gb


    - echo "The tasks will be submitted to SLURM PARTITION " $SLURMPARTITION " on host " $SLURMHOST "with constraints " $CONTSTRAINTS 
    - export SRUN_COMMANDLINE_CONFIGURE="--partition=$SLURMPARTITION --nodelist=$SLURMHOST --time=$CONFIGURETIME --constraint=$CONTSTRAINTS --mem=$REQUESTED_MEMORY" 
    - export SRUN_COMMANDLINE_BUILD="--partition=$SLURMPARTITION --nodelist=$SLURMHOST --time=$BUILDTIME --constraint=$CONTSTRAINTS --mem=$REQUESTED_MEMORY " 
    - export SRUN_COMMANDLINE_RUN="--partition=$SLURMPARTITION --nodelist=$SLURMHOST --time=$RUNTIME --constraint=$CONTSTRAINTS --mem=$REQUESTED_MEMORY " 
    - ./ci_test_scripts/run_ci_tests.sh -c " CC=\"mpicc\" CFLAGS=\"-O3 -march=skylake-avx512 --coverage -O0\" FC=\"mpif90\" FCFLAGS=\"-O3 -march=skylake-avx512 --coverage -O0\"    SCALAPACK_LDFLAGS=\"\$MKL_GFORTRAN_SCALAPACK_LDFLAGS_MPI_NO_OMP \"  SCALAPACK_FCFLAGS=\"\$MKL_GFORTRAN_SCALAPACK_FCFLAGS_MPI_NO_OMP \" --enable-option-checking=fatal --with-mpi=yes --disable-mpi-module --disable-openmp --disable-single-precision --enable-assumed-size --disable-band-to-full-blocking --disable-gpu --enable-avx512" -j 8 -t 2 -m $MATRIX_SIZE -n $NUMBER_OF_EIGENVECTORS -b $BLOCK_SIZE -s $SKIP_STEP -q "srun" 
    - sleep 1
    - sleep 1
    -  echo "na= $MATRIX_SIZE, nev= $NUMBER_OF_EIGENVECTORS nblock= $BLOCK_SIZE" 
    - ./ci_test_scripts/ci_coverage_collect
  artifacts:
    paths:
      - coverage_data



# gnu-gnu-mpi-noopenmp-double-precision-assumed-size-no-band-to-full-blocking-no-gpu-no-coverage-avx-address-sanitize
gnu-gnu-mpi-noopenmp-double-precision-assumed-size-no-band-to-full-blocking-no-gpu-no-coverage-avx-address-sanitize-jobs:
  only:
    - /.*master.*/
  tags:
    - avx
  script:
   - ./ci_test_scripts/run_ci_tests.sh -c " CC=\"mpicc\" CFLAGS=\"-O3 -mavx -fsanitize=address\" FC=\"mpif90\" FCFLAGS=\"-O3 -mavx -fsanitize=address\"    SCALAPACK_LDFLAGS=\"$MKL_GFORTRAN_SCALAPACK_LDFLAGS_MPI_NO_OMP \"  SCALAPACK_FCFLAGS=\"$MKL_GFORTRAN_SCALAPACK_FCFLAGS_MPI_NO_OMP \" --enable-option-checking=fatal --with-mpi=yes --disable-mpi-module --disable-openmp --disable-single-precision --enable-assumed-size --disable-band-to-full-blocking --disable-gpu --enable-avx --disable-avx2" -j 8 -t 2 -m $MATRIX_SIZE -n $NUMBER_OF_EIGENVECTORS -b $BLOCK_SIZE -s $SKIP_STEP 



# gnu-gnu-mpi-noopenmp-double-precision-assumed-size-no-band-to-full-blocking-no-gpu-no-coverage-avx-no-address-sanitize
gnu-gnu-mpi-noopenmp-double-precision-assumed-size-no-band-to-full-blocking-no-gpu-no-coverage-avx-no-address-sanitize-jobs:
  only:
    - /.*master.*/
  tags:
    - avx
  script:
   - ./ci_test_scripts/run_ci_tests.sh -c " CC=\"mpicc\" CFLAGS=\"-O3 -mavx\" FC=\"mpif90\" FCFLAGS=\"-O3 -mavx\"    SCALAPACK_LDFLAGS=\"$MKL_GFORTRAN_SCALAPACK_LDFLAGS_MPI_NO_OMP \"  SCALAPACK_FCFLAGS=\"$MKL_GFORTRAN_SCALAPACK_FCFLAGS_MPI_NO_OMP \" --enable-option-checking=fatal --with-mpi=yes --disable-mpi-module --disable-openmp --disable-single-precision --enable-assumed-size --disable-band-to-full-blocking --disable-gpu --enable-avx --disable-avx2" -j 8 -t 2 -m $MATRIX_SIZE -n $NUMBER_OF_EIGENVECTORS -b $BLOCK_SIZE -s $SKIP_STEP 



# gnu-gnu-mpi-noopenmp-double-precision-assumed-size-no-band-to-full-blocking-no-gpu-no-coverage-avx2-address-sanitize
gnu-gnu-mpi-noopenmp-double-precision-assumed-size-no-band-to-full-blocking-no-gpu-no-coverage-avx2-address-sanitize-jobs:
  only:
    - /.*master.*/
  tags:
    - avx2
  script:
    - export REQUESTED_MEMORY=2Gb


    - echo "The tasks will be submitted to SLURM PARTITION " $SLURMPARTITION " on host " $SLURMHOST "with constraints " $CONTSTRAINTS 
    - export SRUN_COMMANDLINE_CONFIGURE="--partition=$SLURMPARTITION --nodelist=$SLURMHOST --time=$CONFIGURETIME --constraint=$CONTSTRAINTS --mem=$REQUESTED_MEMORY" 
    - export SRUN_COMMANDLINE_BUILD="--partition=$SLURMPARTITION --nodelist=$SLURMHOST --time=$BUILDTIME --constraint=$CONTSTRAINTS --mem=$REQUESTED_MEMORY " 
    - export SRUN_COMMANDLINE_RUN="--partition=$SLURMPARTITION --nodelist=$SLURMHOST --time=$RUNTIME --constraint=$CONTSTRAINTS --mem=$REQUESTED_MEMORY " 
    - ./ci_test_scripts/run_ci_tests.sh -c " CC=\"mpicc\" CFLAGS=\"-O3 -mavx2 -mfma -fsanitize=address\" FC=\"mpif90\" FCFLAGS=\"-O3 -mavx2 -mfma -fsanitize=address\"    SCALAPACK_LDFLAGS=\"\$MKL_GFORTRAN_SCALAPACK_LDFLAGS_MPI_NO_OMP \"  SCALAPACK_FCFLAGS=\"\$MKL_GFORTRAN_SCALAPACK_FCFLAGS_MPI_NO_OMP \" --enable-option-checking=fatal --with-mpi=yes --disable-mpi-module --disable-openmp --disable-single-precision --enable-assumed-size --disable-band-to-full-blocking --disable-gpu --enable-avx2" -j 8 -t 2 -m $MATRIX_SIZE -n $NUMBER_OF_EIGENVECTORS -b $BLOCK_SIZE -s $SKIP_STEP -q "srun" 
    - sleep 1
    - sleep 1
    -  echo "na= $MATRIX_SIZE, nev= $NUMBER_OF_EIGENVECTORS nblock= $BLOCK_SIZE" 



# gnu-gnu-mpi-noopenmp-double-precision-assumed-size-no-band-to-full-blocking-no-gpu-no-coverage-avx2-no-address-sanitize
gnu-gnu-mpi-noopenmp-double-precision-assumed-size-no-band-to-full-blocking-no-gpu-no-coverage-avx2-no-address-sanitize-jobs:
  only:
    - /.*master.*/
  tags:
    - avx2
  script:
    - export REQUESTED_MEMORY=2Gb


    - echo "The tasks will be submitted to SLURM PARTITION " $SLURMPARTITION " on host " $SLURMHOST "with constraints " $CONTSTRAINTS 
    - export SRUN_COMMANDLINE_CONFIGURE="--partition=$SLURMPARTITION --nodelist=$SLURMHOST --time=$CONFIGURETIME --constraint=$CONTSTRAINTS --mem=$REQUESTED_MEMORY" 
    - export SRUN_COMMANDLINE_BUILD="--partition=$SLURMPARTITION --nodelist=$SLURMHOST --time=$BUILDTIME --constraint=$CONTSTRAINTS --mem=$REQUESTED_MEMORY " 
    - export SRUN_COMMANDLINE_RUN="--partition=$SLURMPARTITION --nodelist=$SLURMHOST --time=$RUNTIME --constraint=$CONTSTRAINTS --mem=$REQUESTED_MEMORY " 
    - ./ci_test_scripts/run_ci_tests.sh -c " CC=\"mpicc\" CFLAGS=\"-O3 -mavx2 -mfma\" FC=\"mpif90\" FCFLAGS=\"-O3 -mavx2 -mfma\"    SCALAPACK_LDFLAGS=\"\$MKL_GFORTRAN_SCALAPACK_LDFLAGS_MPI_NO_OMP \"  SCALAPACK_FCFLAGS=\"\$MKL_GFORTRAN_SCALAPACK_FCFLAGS_MPI_NO_OMP \" --enable-option-checking=fatal --with-mpi=yes --disable-mpi-module --disable-openmp --disable-single-precision --enable-assumed-size --disable-band-to-full-blocking --disable-gpu --enable-avx2" -j 8 -t 2 -m $MATRIX_SIZE -n $NUMBER_OF_EIGENVECTORS -b $BLOCK_SIZE -s $SKIP_STEP -q "srun" 
    - sleep 1
    - sleep 1
    -  echo "na= $MATRIX_SIZE, nev= $NUMBER_OF_EIGENVECTORS nblock= $BLOCK_SIZE" 



# gnu-gnu-mpi-noopenmp-double-precision-assumed-size-no-band-to-full-blocking-no-gpu-no-coverage-avx512-address-sanitize
gnu-gnu-mpi-noopenmp-double-precision-assumed-size-no-band-to-full-blocking-no-gpu-no-coverage-avx512-address-sanitize-jobs:
  only:
    - /.*master.*/
  tags:
    - avx512
  script:
    - export REQUESTED_MEMORY=2Gb


    - echo "The tasks will be submitted to SLURM PARTITION " $SLURMPARTITION " on host " $SLURMHOST "with constraints " $CONTSTRAINTS 
    - export SRUN_COMMANDLINE_CONFIGURE="--partition=$SLURMPARTITION --nodelist=$SLURMHOST --time=$CONFIGURETIME --constraint=$CONTSTRAINTS --mem=$REQUESTED_MEMORY" 
    - export SRUN_COMMANDLINE_BUILD="--partition=$SLURMPARTITION --nodelist=$SLURMHOST --time=$BUILDTIME --constraint=$CONTSTRAINTS --mem=$REQUESTED_MEMORY " 
    - export SRUN_COMMANDLINE_RUN="--partition=$SLURMPARTITION --nodelist=$SLURMHOST --time=$RUNTIME --constraint=$CONTSTRAINTS --mem=$REQUESTED_MEMORY " 
    - ./ci_test_scripts/run_ci_tests.sh -c " CC=\"mpicc\" CFLAGS=\"-O3 -march=skylake-avx512 -fsanitize=address\" FC=\"mpif90\" FCFLAGS=\"-O3 -march=skylake-avx512 -fsanitize=address\"    SCALAPACK_LDFLAGS=\"\$MKL_GFORTRAN_SCALAPACK_LDFLAGS_MPI_NO_OMP \"  SCALAPACK_FCFLAGS=\"\$MKL_GFORTRAN_SCALAPACK_FCFLAGS_MPI_NO_OMP \" --enable-option-checking=fatal --with-mpi=yes --disable-mpi-module --disable-openmp --disable-single-precision --enable-assumed-size --disable-band-to-full-blocking --disable-gpu --enable-avx512" -j 8 -t 2 -m $MATRIX_SIZE -n $NUMBER_OF_EIGENVECTORS -b $BLOCK_SIZE -s $SKIP_STEP -q "srun" 
    - sleep 1
    - sleep 1
    -  echo "na= $MATRIX_SIZE, nev= $NUMBER_OF_EIGENVECTORS nblock= $BLOCK_SIZE" 



# gnu-gnu-mpi-noopenmp-double-precision-assumed-size-no-band-to-full-blocking-no-gpu-no-coverage-avx512-no-address-sanitize
gnu-gnu-mpi-noopenmp-double-precision-assumed-size-no-band-to-full-blocking-no-gpu-no-coverage-avx512-no-address-sanitize-jobs:
  only:
    - /.*master.*/
  tags:
    - avx512
  script:
    - export REQUESTED_MEMORY=2Gb


    - echo "The tasks will be submitted to SLURM PARTITION " $SLURMPARTITION " on host " $SLURMHOST "with constraints " $CONTSTRAINTS 
    - export SRUN_COMMANDLINE_CONFIGURE="--partition=$SLURMPARTITION --nodelist=$SLURMHOST --time=$CONFIGURETIME --constraint=$CONTSTRAINTS --mem=$REQUESTED_MEMORY" 
    - export SRUN_COMMANDLINE_BUILD="--partition=$SLURMPARTITION --nodelist=$SLURMHOST --time=$BUILDTIME --constraint=$CONTSTRAINTS --mem=$REQUESTED_MEMORY " 
    - export SRUN_COMMANDLINE_RUN="--partition=$SLURMPARTITION --nodelist=$SLURMHOST --time=$RUNTIME --constraint=$CONTSTRAINTS --mem=$REQUESTED_MEMORY " 
    - ./ci_test_scripts/run_ci_tests.sh -c " CC=\"mpicc\" CFLAGS=\"-O3 -march=skylake-avx512\" FC=\"mpif90\" FCFLAGS=\"-O3 -march=skylake-avx512\"    SCALAPACK_LDFLAGS=\"\$MKL_GFORTRAN_SCALAPACK_LDFLAGS_MPI_NO_OMP \"  SCALAPACK_FCFLAGS=\"\$MKL_GFORTRAN_SCALAPACK_FCFLAGS_MPI_NO_OMP \" --enable-option-checking=fatal --with-mpi=yes --disable-mpi-module --disable-openmp --disable-single-precision --enable-assumed-size --disable-band-to-full-blocking --disable-gpu --enable-avx512" -j 8 -t 2 -m $MATRIX_SIZE -n $NUMBER_OF_EIGENVECTORS -b $BLOCK_SIZE -s $SKIP_STEP -q "srun" 
    - sleep 1
    - sleep 1
    -  echo "na= $MATRIX_SIZE, nev= $NUMBER_OF_EIGENVECTORS nblock= $BLOCK_SIZE" 



# gnu-gnu-mpi-noopenmp-double-precision-assumed-size-no-band-to-full-blocking-no-gpu-no-coverage-sse-address-sanitize
gnu-gnu-mpi-noopenmp-double-precision-assumed-size-no-band-to-full-blocking-no-gpu-no-coverage-sse-address-sanitize-jobs:
  only:
    - /.*master.*/
  tags:
    - sse
  script:
   - if [ $MATRIX_SIZE -gt 150 ]; then export SKIP_STEP=1 ; fi # our SSE test machines do not have a lot of memory
   - ./ci_test_scripts/run_ci_tests.sh -c " CC=\"mpicc\" CFLAGS=\"-O3 -msse4.2 -fsanitize=address\" FC=\"mpif90\" FCFLAGS=\"-O3 -msse4.2 -fsanitize=address\"    SCALAPACK_LDFLAGS=\"$MKL_GFORTRAN_SCALAPACK_LDFLAGS_MPI_NO_OMP \"  SCALAPACK_FCFLAGS=\"$MKL_GFORTRAN_SCALAPACK_FCFLAGS_MPI_NO_OMP \" --enable-option-checking=fatal --with-mpi=yes --disable-mpi-module --disable-openmp --disable-single-precision --enable-assumed-size --disable-band-to-full-blocking --disable-gpu --enable-sse --enable-sse-assembly --disable-avx --disable-avx2" -j 8 -t 2 -m $MATRIX_SIZE -n $NUMBER_OF_EIGENVECTORS -b $BLOCK_SIZE -s $SKIP_STEP 



# gnu-gnu-mpi-noopenmp-double-precision-assumed-size-no-band-to-full-blocking-no-gpu-no-coverage-sse-no-address-sanitize
gnu-gnu-mpi-noopenmp-double-precision-assumed-size-no-band-to-full-blocking-no-gpu-no-coverage-sse-no-address-sanitize-jobs:
  only:
    - /.*master.*/
  tags:
    - sse
  script:
   - if [ $MATRIX_SIZE -gt 150 ]; then export SKIP_STEP=1 ; fi # our SSE test machines do not have a lot of memory
   - ./ci_test_scripts/run_ci_tests.sh -c " CC=\"mpicc\" CFLAGS=\"-O3 -msse4.2\" FC=\"mpif90\" FCFLAGS=\"-O3 -msse4.2\"    SCALAPACK_LDFLAGS=\"$MKL_GFORTRAN_SCALAPACK_LDFLAGS_MPI_NO_OMP \"  SCALAPACK_FCFLAGS=\"$MKL_GFORTRAN_SCALAPACK_FCFLAGS_MPI_NO_OMP \" --enable-option-checking=fatal --with-mpi=yes --disable-mpi-module --disable-openmp --disable-single-precision --enable-assumed-size --disable-band-to-full-blocking --disable-gpu --enable-sse --enable-sse-assembly --disable-avx --disable-avx2" -j 8 -t 2 -m $MATRIX_SIZE -n $NUMBER_OF_EIGENVECTORS -b $BLOCK_SIZE -s $SKIP_STEP 



# gnu-gnu-mpi-noopenmp-double-precision-assumed-size-no-band-to-full-blocking-with-gpu-no-coverage-avx-no-address-sanitize
gnu-gnu-mpi-noopenmp-double-precision-assumed-size-no-band-to-full-blocking-with-gpu-no-coverage-avx-no-address-sanitize-jobs:
  only:
    - /.*master.*/
  tags:
    - gpu
  script:
    - export REQUESTED_MEMORY=2Gb


    - echo "The tasks will be submitted to SLURM PARTITION " $SLURMPARTITION " on host " $SLURMHOST " with constraints " $CONTSTRAINTS " with the geometry " $GEOMETRYRESERVATION
    - export SRUN_COMMANDLINE_CONFIGURE="--partition=$SLURMPARTITION --nodelist=$SLURMHOST --time=$CONFIGURETIME --constraint=$CONTSTRAINTS --gres=$GEOMETRYRESERVATION " 
    - export SRUN_COMMANDLINE_BUILD="--partition=$SLURMPARTITION --nodelist=$SLURMHOST --time=$BUILDTIME --constraint=$CONTSTRAINTS --gres=$GEOMETRYRESERVATION " 
    - export SRUN_COMMANDLINE_RUN="--partition=$SLURMPARTITION --nodelist=$SLURMHOST --time=$RUNTIME --constraint=$CONTSTRAINTS --gres=$GEOMETRYRESERVATION " 
    - ./ci_test_scripts/run_ci_tests.sh -c " CC=\"mpicc\" CFLAGS=\"-O3 -mavx\" FC=\"mpif90\" FCFLAGS=\"-O3 -mavx\"    SCALAPACK_LDFLAGS=\"\$MKL_GFORTRAN_SCALAPACK_LDFLAGS_MPI_NO_OMP  -L\$CUDA_HOME/lib64 -lcublas -I\$CUDA_HOME/include\"  SCALAPACK_FCFLAGS=\"\$MKL_GFORTRAN_SCALAPACK_FCFLAGS_MPI_NO_OMP  -I\$CUDA_HOME/include\" --enable-option-checking=fatal --with-mpi=yes --disable-mpi-module --disable-openmp --disable-single-precision --enable-assumed-size --disable-band-to-full-blocking --enable-gpu --with-cuda-path=\$CUDA_HOME/ --enable-avx --disable-avx2" -j 8 -t 2 -m $MATRIX_SIZE -n $NUMBER_OF_EIGENVECTORS -b $BLOCK_SIZE -s $SKIP_STEP -q "srun" 
    - sleep 1
    - sleep 1
    -  echo "na= $MATRIX_SIZE, nev= $NUMBER_OF_EIGENVECTORS nblock= $BLOCK_SIZE" 



# gnu-gnu-mpi-noopenmp-double-precision-assumed-size-no-band-to-full-blocking-with-gpu-no-coverage-power8-no-address-sanitize
gnu-gnu-mpi-noopenmp-double-precision-assumed-size-no-band-to-full-blocking-with-gpu-no-coverage-power8-no-address-sanitize-jobs:
  only:
    - /.*master.*/
  allow_failure: true
  tags:
    - minsky
  script:
    - export REQUESTED_MEMORY=2Gb


    - echo "The tasks will be submitted to SLURM PARTITION " $SLURMPARTITION " on host " $SLURMHOST " with constraints " $CONTSTRAINTS " with the geometry " $GEOMETRYRESERVATION
    - export SRUN_COMMANDLINE_CONFIGURE="--partition=$SLURMPARTITION --nodelist=$SLURMHOST --time=$CONFIGURETIME --constraint=$CONTSTRAINTS --gres=$GEOMETRYRESERVATION " 
    - export SRUN_COMMANDLINE_BUILD="--partition=$SLURMPARTITION --nodelist=$SLURMHOST --time=$BUILDTIME --constraint=$CONTSTRAINTS --gres=$GEOMETRYRESERVATION " 
    - export SRUN_COMMANDLINE_RUN="--partition=$SLURMPARTITION --nodelist=$SLURMHOST --time=$RUNTIME --constraint=$CONTSTRAINTS --gres=$GEOMETRYRESERVATION " 
    - ./ci_test_scripts/run_ci_tests.sh -c " CC=\"mpicc\" CFLAGS=\"-O2 -I\$CUDA_HOME/include\" FC=\"mpifort\" FCFLAGS=\"-O2\" LIBS=\" -lessl -lreflapack -lessl -lcublas -lgfortran \"  LDFLAGS=\" -L/home/elpa/libs/scalapack/lib -L\$CUDA_HOME/lib64 \"    --enable-option-checking=fatal --with-mpi=yes --disable-mpi-module --disable-openmp --disable-single-precision --enable-assumed-size --disable-band-to-full-blocking --enable-gpu --with-cuda-path=\$CUDA_HOME/ --enable-vsx --disable-sse --disable-sse-assembly --disable-avx --disable-avx2 --disable-mpi-module --with-GPU-compute-capability=sm_60 " -j 8 -t 2 -m $MATRIX_SIZE -n $NUMBER_OF_EIGENVECTORS -b $BLOCK_SIZE -s $SKIP_STEP -q "srun" 
    - sleep 1
    - sleep 1
    -  echo "na= $MATRIX_SIZE, nev= $NUMBER_OF_EIGENVECTORS nblock= $BLOCK_SIZE" 



# gnu-gnu-mpi-noopenmp-double-precision-no-assumed-size-band-to-full-blocking-no-gpu-coverage-avx-no-address-sanitize
gnu-gnu-mpi-noopenmp-double-precision-no-assumed-size-band-to-full-blocking-no-gpu-coverage-avx-no-address-sanitize-jobs:
  only:
    - /.*master.*/
  tags:
    - coverage
  script:
   - ./ci_test_scripts/run_ci_tests.sh -c " CC=\"mpicc\" CFLAGS=\"-O3 -mavx --coverage -O0\" FC=\"mpif90\" FCFLAGS=\"-O3 -mavx --coverage -O0\"    SCALAPACK_LDFLAGS=\"$MKL_GFORTRAN_SCALAPACK_LDFLAGS_MPI_NO_OMP \"  SCALAPACK_FCFLAGS=\"$MKL_GFORTRAN_SCALAPACK_FCFLAGS_MPI_NO_OMP \" --enable-option-checking=fatal --with-mpi=yes --disable-mpi-module --disable-openmp --disable-single-precision --disable-assumed-size --enable-band-to-full-blocking --disable-gpu --enable-avx --disable-avx2" -j 8 -t 2 -m $MATRIX_SIZE -n $NUMBER_OF_EIGENVECTORS -b $BLOCK_SIZE -s $SKIP_STEP 



# gnu-gnu-mpi-noopenmp-double-precision-no-assumed-size-band-to-full-blocking-no-gpu-coverage-avx2-no-address-sanitize
gnu-gnu-mpi-noopenmp-double-precision-no-assumed-size-band-to-full-blocking-no-gpu-coverage-avx2-no-address-sanitize-jobs:
  only:
    - /.*master.*/
  tags:
    - avx2-coverage
  script:
    - export REQUESTED_MEMORY=2Gb


    - echo "The tasks will be submitted to SLURM PARTITION " $SLURMPARTITION " on host " $SLURMHOST "with constraints " $CONTSTRAINTS 
    - export SRUN_COMMANDLINE_CONFIGURE="--partition=$SLURMPARTITION --nodelist=$SLURMHOST --time=$CONFIGURETIME --constraint=$CONTSTRAINTS --mem=$REQUESTED_MEMORY" 
    - export SRUN_COMMANDLINE_BUILD="--partition=$SLURMPARTITION --nodelist=$SLURMHOST --time=$BUILDTIME --constraint=$CONTSTRAINTS --mem=$REQUESTED_MEMORY " 
    - export SRUN_COMMANDLINE_RUN="--partition=$SLURMPARTITION --nodelist=$SLURMHOST --time=$RUNTIME --constraint=$CONTSTRAINTS --mem=$REQUESTED_MEMORY " 
    - ./ci_test_scripts/run_ci_tests.sh -c " CC=\"mpicc\" CFLAGS=\"-O3 -mavx2 -mfma --coverage -O0\" FC=\"mpif90\" FCFLAGS=\"-O3 -mavx2 -mfma --coverage -O0\"    SCALAPACK_LDFLAGS=\"\$MKL_GFORTRAN_SCALAPACK_LDFLAGS_MPI_NO_OMP \"  SCALAPACK_FCFLAGS=\"\$MKL_GFORTRAN_SCALAPACK_FCFLAGS_MPI_NO_OMP \" --enable-option-checking=fatal --enable-scalapack-tests --with-mpi=yes --disable-mpi-module --disable-openmp --disable-single-precision --disable-assumed-size --enable-band-to-full-blocking --disable-gpu --enable-avx2" -j 8 -t 2 -m $MATRIX_SIZE -n $NUMBER_OF_EIGENVECTORS -b $BLOCK_SIZE -s $SKIP_STEP -q "srun" 
    - sleep 1
    - sleep 1
    -  echo "na= $MATRIX_SIZE, nev= $NUMBER_OF_EIGENVECTORS nblock= $BLOCK_SIZE" 
    - ./ci_test_scripts/ci_coverage_collect
  artifacts:
    paths:
      - coverage_data



# gnu-gnu-mpi-noopenmp-double-precision-no-assumed-size-band-to-full-blocking-no-gpu-coverage-avx512-no-address-sanitize
gnu-gnu-mpi-noopenmp-double-precision-no-assumed-size-band-to-full-blocking-no-gpu-coverage-avx512-no-address-sanitize-jobs:
  only:
    - /.*master.*/
  tags:
    - avx512-coverage
  script:
    - export REQUESTED_MEMORY=2Gb


    - echo "The tasks will be submitted to SLURM PARTITION " $SLURMPARTITION " on host " $SLURMHOST "with constraints " $CONTSTRAINTS 
    - export SRUN_COMMANDLINE_CONFIGURE="--partition=$SLURMPARTITION --nodelist=$SLURMHOST --time=$CONFIGURETIME --constraint=$CONTSTRAINTS --mem=$REQUESTED_MEMORY" 
    - export SRUN_COMMANDLINE_BUILD="--partition=$SLURMPARTITION --nodelist=$SLURMHOST --time=$BUILDTIME --constraint=$CONTSTRAINTS --mem=$REQUESTED_MEMORY " 
    - export SRUN_COMMANDLINE_RUN="--partition=$SLURMPARTITION --nodelist=$SLURMHOST --time=$RUNTIME --constraint=$CONTSTRAINTS --mem=$REQUESTED_MEMORY " 
    - ./ci_test_scripts/run_ci_tests.sh -c " CC=\"mpicc\" CFLAGS=\"-O3 -march=skylake-avx512 --coverage -O0\" FC=\"mpif90\" FCFLAGS=\"-O3 -march=skylake-avx512 --coverage -O0\"    SCALAPACK_LDFLAGS=\"\$MKL_GFORTRAN_SCALAPACK_LDFLAGS_MPI_NO_OMP \"  SCALAPACK_FCFLAGS=\"\$MKL_GFORTRAN_SCALAPACK_FCFLAGS_MPI_NO_OMP \" --enable-option-checking=fatal --with-mpi=yes --disable-mpi-module --disable-openmp --disable-single-precision --disable-assumed-size --enable-band-to-full-blocking --disable-gpu --enable-avx512" -j 8 -t 2 -m $MATRIX_SIZE -n $NUMBER_OF_EIGENVECTORS -b $BLOCK_SIZE -s $SKIP_STEP -q "srun" 
    - sleep 1
    - sleep 1
    -  echo "na= $MATRIX_SIZE, nev= $NUMBER_OF_EIGENVECTORS nblock= $BLOCK_SIZE" 
    - ./ci_test_scripts/ci_coverage_collect
  artifacts:
    paths:
      - coverage_data



# gnu-gnu-mpi-noopenmp-double-precision-no-assumed-size-band-to-full-blocking-no-gpu-no-coverage-avx-address-sanitize
gnu-gnu-mpi-noopenmp-double-precision-no-assumed-size-band-to-full-blocking-no-gpu-no-coverage-avx-address-sanitize-jobs:
  only:
    - /.*master.*/
  tags:
    - avx
  script:
   - ./ci_test_scripts/run_ci_tests.sh -c " CC=\"mpicc\" CFLAGS=\"-O3 -mavx -fsanitize=address\" FC=\"mpif90\" FCFLAGS=\"-O3 -mavx -fsanitize=address\"    SCALAPACK_LDFLAGS=\"$MKL_GFORTRAN_SCALAPACK_LDFLAGS_MPI_NO_OMP \"  SCALAPACK_FCFLAGS=\"$MKL_GFORTRAN_SCALAPACK_FCFLAGS_MPI_NO_OMP \" --enable-option-checking=fatal --with-mpi=yes --disable-mpi-module --disable-openmp --disable-single-precision --disable-assumed-size --enable-band-to-full-blocking --disable-gpu --enable-avx --disable-avx2" -j 8 -t 2 -m $MATRIX_SIZE -n $NUMBER_OF_EIGENVECTORS -b $BLOCK_SIZE -s $SKIP_STEP 



# gnu-gnu-mpi-noopenmp-double-precision-no-assumed-size-band-to-full-blocking-no-gpu-no-coverage-avx-no-address-sanitize
gnu-gnu-mpi-noopenmp-double-precision-no-assumed-size-band-to-full-blocking-no-gpu-no-coverage-avx-no-address-sanitize-jobs:
  only:
    - /.*master.*/
  tags:
    - avx
  script:
   - ./ci_test_scripts/run_ci_tests.sh -c " CC=\"mpicc\" CFLAGS=\"-O3 -mavx\" FC=\"mpif90\" FCFLAGS=\"-O3 -mavx\"    SCALAPACK_LDFLAGS=\"$MKL_GFORTRAN_SCALAPACK_LDFLAGS_MPI_NO_OMP \"  SCALAPACK_FCFLAGS=\"$MKL_GFORTRAN_SCALAPACK_FCFLAGS_MPI_NO_OMP \" --enable-option-checking=fatal --with-mpi=yes --disable-mpi-module --disable-openmp --disable-single-precision --disable-assumed-size --enable-band-to-full-blocking --disable-gpu --enable-avx --disable-avx2" -j 8 -t 2 -m $MATRIX_SIZE -n $NUMBER_OF_EIGENVECTORS -b $BLOCK_SIZE -s $SKIP_STEP 



# gnu-gnu-mpi-noopenmp-double-precision-no-assumed-size-band-to-full-blocking-no-gpu-no-coverage-avx2-address-sanitize
gnu-gnu-mpi-noopenmp-double-precision-no-assumed-size-band-to-full-blocking-no-gpu-no-coverage-avx2-address-sanitize-jobs:
  only:
    - /.*master.*/
  tags:
    - avx2
  script:
    - export REQUESTED_MEMORY=2Gb


    - echo "The tasks will be submitted to SLURM PARTITION " $SLURMPARTITION " on host " $SLURMHOST "with constraints " $CONTSTRAINTS 
    - export SRUN_COMMANDLINE_CONFIGURE="--partition=$SLURMPARTITION --nodelist=$SLURMHOST --time=$CONFIGURETIME --constraint=$CONTSTRAINTS --mem=$REQUESTED_MEMORY" 
    - export SRUN_COMMANDLINE_BUILD="--partition=$SLURMPARTITION --nodelist=$SLURMHOST --time=$BUILDTIME --constraint=$CONTSTRAINTS --mem=$REQUESTED_MEMORY " 
    - export SRUN_COMMANDLINE_RUN="--partition=$SLURMPARTITION --nodelist=$SLURMHOST --time=$RUNTIME --constraint=$CONTSTRAINTS --mem=$REQUESTED_MEMORY " 
    - ./ci_test_scripts/run_ci_tests.sh -c " CC=\"mpicc\" CFLAGS=\"-O3 -mavx2 -mfma -fsanitize=address\" FC=\"mpif90\" FCFLAGS=\"-O3 -mavx2 -mfma -fsanitize=address\"    SCALAPACK_LDFLAGS=\"\$MKL_GFORTRAN_SCALAPACK_LDFLAGS_MPI_NO_OMP \"  SCALAPACK_FCFLAGS=\"\$MKL_GFORTRAN_SCALAPACK_FCFLAGS_MPI_NO_OMP \" --enable-option-checking=fatal --with-mpi=yes --disable-mpi-module --disable-openmp --disable-single-precision --disable-assumed-size --enable-band-to-full-blocking --disable-gpu --enable-avx2" -j 8 -t 2 -m $MATRIX_SIZE -n $NUMBER_OF_EIGENVECTORS -b $BLOCK_SIZE -s $SKIP_STEP -q "srun" 
    - sleep 1
    - sleep 1
    -  echo "na= $MATRIX_SIZE, nev= $NUMBER_OF_EIGENVECTORS nblock= $BLOCK_SIZE" 



# gnu-gnu-mpi-noopenmp-double-precision-no-assumed-size-band-to-full-blocking-no-gpu-no-coverage-avx2-no-address-sanitize
gnu-gnu-mpi-noopenmp-double-precision-no-assumed-size-band-to-full-blocking-no-gpu-no-coverage-avx2-no-address-sanitize-jobs:
  only:
    - /.*master.*/
  tags:
    - avx2
  script:
    - export REQUESTED_MEMORY=2Gb


    - echo "The tasks will be submitted to SLURM PARTITION " $SLURMPARTITION " on host " $SLURMHOST "with constraints " $CONTSTRAINTS 
    - export SRUN_COMMANDLINE_CONFIGURE="--partition=$SLURMPARTITION --nodelist=$SLURMHOST --time=$CONFIGURETIME --constraint=$CONTSTRAINTS --mem=$REQUESTED_MEMORY" 
    - export SRUN_COMMANDLINE_BUILD="--partition=$SLURMPARTITION --nodelist=$SLURMHOST --time=$BUILDTIME --constraint=$CONTSTRAINTS --mem=$REQUESTED_MEMORY " 
    - export SRUN_COMMANDLINE_RUN="--partition=$SLURMPARTITION --nodelist=$SLURMHOST --time=$RUNTIME --constraint=$CONTSTRAINTS --mem=$REQUESTED_MEMORY " 
    - ./ci_test_scripts/run_ci_tests.sh -c " CC=\"mpicc\" CFLAGS=\"-O3 -mavx2 -mfma\" FC=\"mpif90\" FCFLAGS=\"-O3 -mavx2 -mfma\"    SCALAPACK_LDFLAGS=\"\$MKL_GFORTRAN_SCALAPACK_LDFLAGS_MPI_NO_OMP \"  SCALAPACK_FCFLAGS=\"\$MKL_GFORTRAN_SCALAPACK_FCFLAGS_MPI_NO_OMP \" --enable-option-checking=fatal --with-mpi=yes --disable-mpi-module --disable-openmp --disable-single-precision --disable-assumed-size --enable-band-to-full-blocking --disable-gpu --enable-avx2" -j 8 -t 2 -m $MATRIX_SIZE -n $NUMBER_OF_EIGENVECTORS -b $BLOCK_SIZE -s $SKIP_STEP -q "srun" 
    - sleep 1
    - sleep 1
    -  echo "na= $MATRIX_SIZE, nev= $NUMBER_OF_EIGENVECTORS nblock= $BLOCK_SIZE" 



# gnu-gnu-mpi-noopenmp-double-precision-no-assumed-size-band-to-full-blocking-no-gpu-no-coverage-avx512-address-sanitize
gnu-gnu-mpi-noopenmp-double-precision-no-assumed-size-band-to-full-blocking-no-gpu-no-coverage-avx512-address-sanitize-jobs:
  only:
    - /.*master.*/
  tags:
    - avx512
  script:
    - export REQUESTED_MEMORY=2Gb


    - echo "The tasks will be submitted to SLURM PARTITION " $SLURMPARTITION " on host " $SLURMHOST "with constraints " $CONTSTRAINTS 
    - export SRUN_COMMANDLINE_CONFIGURE="--partition=$SLURMPARTITION --nodelist=$SLURMHOST --time=$CONFIGURETIME --constraint=$CONTSTRAINTS --mem=$REQUESTED_MEMORY" 
    - export SRUN_COMMANDLINE_BUILD="--partition=$SLURMPARTITION --nodelist=$SLURMHOST --time=$BUILDTIME --constraint=$CONTSTRAINTS --mem=$REQUESTED_MEMORY " 
    - export SRUN_COMMANDLINE_RUN="--partition=$SLURMPARTITION --nodelist=$SLURMHOST --time=$RUNTIME --constraint=$CONTSTRAINTS --mem=$REQUESTED_MEMORY " 
    - ./ci_test_scripts/run_ci_tests.sh -c " CC=\"mpicc\" CFLAGS=\"-O3 -march=skylake-avx512 -fsanitize=address\" FC=\"mpif90\" FCFLAGS=\"-O3 -march=skylake-avx512 -fsanitize=address\"    SCALAPACK_LDFLAGS=\"\$MKL_GFORTRAN_SCALAPACK_LDFLAGS_MPI_NO_OMP \"  SCALAPACK_FCFLAGS=\"\$MKL_GFORTRAN_SCALAPACK_FCFLAGS_MPI_NO_OMP \" --enable-option-checking=fatal --with-mpi=yes --disable-mpi-module --disable-openmp --disable-single-precision --disable-assumed-size --enable-band-to-full-blocking --disable-gpu --enable-avx512" -j 8 -t 2 -m $MATRIX_SIZE -n $NUMBER_OF_EIGENVECTORS -b $BLOCK_SIZE -s $SKIP_STEP -q "srun" 
    - sleep 1
    - sleep 1
    -  echo "na= $MATRIX_SIZE, nev= $NUMBER_OF_EIGENVECTORS nblock= $BLOCK_SIZE" 



# gnu-gnu-mpi-noopenmp-double-precision-no-assumed-size-band-to-full-blocking-no-gpu-no-coverage-avx512-no-address-sanitize
gnu-gnu-mpi-noopenmp-double-precision-no-assumed-size-band-to-full-blocking-no-gpu-no-coverage-avx512-no-address-sanitize-jobs:
  only:
    - /.*master.*/
  tags:
    - avx512
  script:
    - export REQUESTED_MEMORY=2Gb


    - echo "The tasks will be submitted to SLURM PARTITION " $SLURMPARTITION " on host " $SLURMHOST "with constraints " $CONTSTRAINTS 
    - export SRUN_COMMANDLINE_CONFIGURE="--partition=$SLURMPARTITION --nodelist=$SLURMHOST --time=$CONFIGURETIME --constraint=$CONTSTRAINTS --mem=$REQUESTED_MEMORY" 
    - export SRUN_COMMANDLINE_BUILD="--partition=$SLURMPARTITION --nodelist=$SLURMHOST --time=$BUILDTIME --constraint=$CONTSTRAINTS --mem=$REQUESTED_MEMORY " 
    - export SRUN_COMMANDLINE_RUN="--partition=$SLURMPARTITION --nodelist=$SLURMHOST --time=$RUNTIME --constraint=$CONTSTRAINTS --mem=$REQUESTED_MEMORY " 
    - ./ci_test_scripts/run_ci_tests.sh -c " CC=\"mpicc\" CFLAGS=\"-O3 -march=skylake-avx512\" FC=\"mpif90\" FCFLAGS=\"-O3 -march=skylake-avx512\"    SCALAPACK_LDFLAGS=\"\$MKL_GFORTRAN_SCALAPACK_LDFLAGS_MPI_NO_OMP \"  SCALAPACK_FCFLAGS=\"\$MKL_GFORTRAN_SCALAPACK_FCFLAGS_MPI_NO_OMP \" --enable-option-checking=fatal --with-mpi=yes --disable-mpi-module --disable-openmp --disable-single-precision --disable-assumed-size --enable-band-to-full-blocking --disable-gpu --enable-avx512" -j 8 -t 2 -m $MATRIX_SIZE -n $NUMBER_OF_EIGENVECTORS -b $BLOCK_SIZE -s $SKIP_STEP -q "srun"