Move project tests to new runners

2c5d85c2 · Andreas Marek · c5d4fa12 · 2c5d85c2 · 2c5d85c2 · 2c5d85c2
Commit 2c5d85c2 authored Apr 08, 2019 by Andreas Marek
3 changed files
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
--- a/ci_test_scripts/generate_gitlab_ci_tests.py
+++ b/ci_test_scripts/generate_gitlab_ci_tests.py
@@ -440,37 +440,29 @@ for comp, s, a in product(
    print("# test_project_"+stage[s]+api[a]+"_"+compiler[comp])
    print("test_project_"+stage[s]+api[a]+"_"+compiler[comp]+":")
    print("  tags:")
-    print("    - buildtest")
+    print("    - project_test")
    print("  script:")
-    print("    - mkdir build")
-    print("    - pushd build")
-    print("    - ../autogen.sh")
-    if (comp == "intel"):
-        print("    - ../configure --enable-option-checking=fatal FCFLAGS=\"-march=native\" CFLAGS=\"-march=native\" --disable-avx2 SCALAPACK_LDFLAGS=\"$MKL_INTEL_SCALAPACK_LDFLAGS_MPI_NO_OMP\" SCALAPACK_FCFLAGS=\"$MKL_INTEL_SCALAPACK_FCFLAGS_MPI_NO_OMP\" FC=mpiifort --prefix=$PWD/installdest --disable-avx2 || { cat config.log; exit 1; }")
-    if (comp == "gnu"):
-        print("    - ../configure --enable-option-checking=fatal FCFLAGS=\"-march=native\" CFLAGS=\"-march=native\" --disable-avx2 SCALAPACK_LDFLAGS=\"$MKL_GFORTRAN_SCALAPACK_LDFLAGS_MPI_NO_OMP\" SCALAPACK_FCFLAGS=\"$MKL_GFORTRAN_SCALAPACK_FCFLAGS_MPI_NO_OMP\" FC=mpif90 --prefix=$PWD/installdest --disable-avx2 || { cat config.log; exit 1; }")
-    print("    - make -j 8")
-    print("    - make install")
-    print("    - popd")
-    print("    - mkdir test_project_"+stage[s]+api[a]+"/build")
-    print("    - pushd test_project_"+stage[s]+api[a]+"/build")
-    print("    - ../autogen.sh")
-    if (comp == "intel"):
-        print("    - ../configure --enable-option-checking=fatal PKG_CONFIG_PATH=../../build/installdest/lib/pkgconfig FC=mpiifort || { cat config.log; exit 1; }")
-    if (comp == "gnu"):
-        print("    - ../configure --enable-option-checking=fatal PKG_CONFIG_PATH=../../build/installdest/lib/pkgconfig FC=mpif90 || { cat config.log; exit 1; }")
-    print("    - make -j 8")
-    print("    - export LD_LIBRARY_PATH=$MKL_HOME/lib/intel64:$LD_LIBRARY_PATH")
    if ( s == "1stage"):
-        print("    - ./test_real")
+        projectBinary="test_real"
    else:
-        print("    - ./test_real2")
-    print("    - make distclean")
-    print("    - popd")
-    print("    - pushd build")
-    print("    - make distclean")
-    print("    - rm -rf installdest")
-    print("    - popd")
+        projectBinary="test_real2"
+
+    if (comp == "intel"):
+        print("    - ./ci_test_scripts/run_project_tests.sh -c \" FC=mpiifort FCFLAGS=\\\"-march=native \\\" CFLAGS=\\\"-march=native\\\" \
+                SCALAPACK_LDFLAGS=\\\"$MKL_INTEL_SCALAPACK_LDFLAGS_MPI_NO_OMP\\\" \
+                SCALAPACK_FCFLAGS=\\\"$MKL_INTEL_SCALAPACK_FCFLAGS_MPI_NO_OMP\\\" \
+                --enable-option-checking=fatal  --disable-avx2 --prefix=$PWD/installdest --disable-avx2 || { cat config.log; exit 1; } \" \
+                -t 2 -m 150 -n 50 -b 16 -S $SLURM -p test_project_"+stage[s]+api[a]+" -e "+projectBinary+" \
+                -C \" FC=mpiifort PKG_CONFIG_PATH=../../installdest/lib/pkgconfig  \
+                 --enable-option-checking=fatal || { cat config.log; exit 1; } \" ")
+    if (comp == "gnu"):
+        print("    - ./ci_test_scripts/run_project_tests.sh -c \" FC=mpif90 FCFLAGS=\\\"-march=native \\\" CFLAGS=\\\"-march=native\\\" \
+                SCALAPACK_LDFLAGS=\\\"$MKL_GFORTRAN_SCALAPACK_LDFLAGS_MPI_NO_OMP\\\" \
+                SCALAPACK_FCFLAGS=\\\"$MKL_GFORTRAN_SCALAPACK_FCFLAGS_MPI_NO_OMP\\\" \
+                --enable-option-checking=fatal  --disable-avx2 --prefix=$PWD/installdest --disable-avx2 || { cat config.log; exit 1; } \" \
+                -t 2 -m 150 -n 50 -b 16 -S $SLURM -p test_project_"+stage[s]+api[a]+" -e "+projectBinary+" \
+                -C \" FC=mpif90 PKG_CONFIG_PATH=../../installdest/lib/pkgconfig \
+                 --enable-option-checking=fatal || { cat config.log; exit 1; } \" ")
    print("\n\n")

 print("#The tests follow here")

--- a/ci_test_scripts/run_project_tests.sh
+++ b/ci_test_scripts/run_project_tests.sh
+#!/bin/bash
+set -e
+set -x
+
+#some defaults
+mpiTasks=2
+matrixSize=150
+nrEV=$matrixSize
+blockSize=16
+ompThreads=1
+configueArg=""
+batchCommand=""
+slurmBatch="no"
+projectName="unknown"
+projectExecutable=""
+projectConfigureArg=""
+
+function usage() {
+	cat >&2 <<-EOF
+
+		Call all the necessary steps to perform an ELPA CI test
+
+		Usage:
+		  run_project_tests [-c configure arguments] [-h] [-t MPI Tasks] [-m matrix size] [-n number of eigenvectors] [-b block size] [-o OpenMP threads] [-q submit command] [-S submit to Slurm] [-p projectName] [-e projectExecutable] [-C project configure arguments]"
+
+		Options:
+		 -c configure arguments
+		    Line of arguments passed to configure call
+		 -t MPI Tasks
+		    Number of MPI processes used during test runs of ELPA tests
+
+		 -m Matrix size
+		    Size of the mxm matrix used during runs of ELPA tests
+
+		 -n Number of eigenvectors
+		    Number of eigenvectors to be computed during runs of ELPA tests
+
+		 -b block size
+		    Block size of block-cyclic distribution during runs of ELPA tests
+
+		 -o OpenMP threads
+		    Number of OpenMP threads used during runs of ELPA tests
+
+		 -q submit command
+		    Job steps will be submitted via command to a batch system (default no submission)
+
+		 -S submit to slurm
+		    if "yes" a SLURM batch job will be submitted
+
+		 -p project name
+		    specifies which project to build and test
+
+		 -e project executable
+		    specifies which executable to run
+
+		 -C project configure arguments
+		    arguments for the configure of the project
+
+		 -h
+		    Print this help text
+	EOF
+}
+
+
+while getopts "c:t:j:m:n:b:o:s:q:i:S:p:e:C:h" opt; do
+	case $opt in
+		t)
+			mpiTasks=$OPTARG;;
+		m)
+			matrixSize=$OPTARG;;
+		n)
+			nrEV=$OPTARG;;
+		b)
+			blockSize=$OPTARG;;
+		o)
+			ompThreads=$OPTARG;;
+		c)
+			configureArgs=$OPTARG;;
+		q)
+			batchCommand=$OPTARG;;
+		S)
+			slurmBatch=$OPTARG;;
+		p)
+			projectName=$OPTARG;;
+		e)
+			projectExecutable=$OPTARG;;
+		C)
+			projectConfigureArgs=$OPTARG;;
+		:)
+			echo "Option -$OPTARG requires an argument" >&2;;
+		h)
+			usage
+			exit 1;;
+		*)
+			exit 1;;
+	esac
+done
+
+
+if [ "$slurmBatch" == "yes" ]
+then
+
+  # default exit code
+  exitCode=1
+  CLUSTER=""
+  if [[ "$HOST" =~ "cobra" ]]
+  then
+    CLUSTER="cobra"
+  fi
+  if [[ "$HOST" =~ "talos" ]]
+  then
+    CLUSTER="talos"
+  fi
+  if [[ "$HOST" =~ "freya" ]]
+  then
+    CLUSTER="freya"
+  fi
+  if [[ "$HOST" =~ "draco" ]]
+  then
+    CLUSTER="draco"
+  fi
+
+  echo "Running on $CLUSTER with runner $CI_RUNNER_DESCRIPTION with tag $CI_RUNNER_TAGS"
+
+  #project_test
+  if [[ "$CI_RUNNER_TAGS" =~ "project_test" ]]
+  then
+    cp $HOME/runners/job_script_templates/run_${CLUSTER}_1node.sh .
+    echo "mkdir -p build" >> ./run_${CLUSTER}_1node.sh
+    echo "pushd build"  >> ./run_${CLUSTER}_1node.sh
+    echo " "  >> ./run_${CLUSTER}_1node.sh
+    echo "Running autogen " >> ./run_${CLUSTER}_1node.sh
+    echo "../autogen.sh"  >> ./run_${CLUSTER}_1node.sh
+    echo " "  >> ./run_${CLUSTER}_1node.sh
+    echo "Running configure " >> ./run_${CLUSTER}_1node.sh
+    echo "../configure " "$configureArgs" >> ./run_${CLUSTER}_1node.sh
+    echo " " >> ./run_${CLUSTER}_1node.sh
+    echo "export TASKS=$mpiTasks" >> ./run_${CLUSTER}_1node.sh
+    echo " "  >> ./run_${CLUSTER}_1node.sh
+    echo "Running make " >> ./run_${CLUSTER}_1node.sh
+    echo "make -j 8"  >> ./run_${CLUSTER}_1node.sh
+    echo " "  >> ./run_${CLUSTER}_1node.sh
+    echo "Running make install" >> ./run_${CLUSTER}_1node.sh
+    echo "make install" >> ./run_${CLUSTER}_1node.sh
+    echo "popd" >> ./run_${CLUSTER}_1node.sh
+    echo "mkdir -p $projectName/build" >> ./run_${CLUSTER}_1node.sh
+    echo "pushd $projectName/build" >> ./run_${CLUSTER}_1node.sh
+    echo " "  >> ./run_${CLUSTER}_1node.sh
+    echo " "  >> ./run_${CLUSTER}_1node.sh
+    echo " "  >> ./run_${CLUSTER}_1node.sh
+    echo " "  >> ./run_${CLUSTER}_1node.sh
+    echo " Testting project "  >> ./run_${CLUSTER}_1node.sh
+
+    echo " "  >> ./run_${CLUSTER}_1node.sh
+    echo " "  >> ./run_${CLUSTER}_1node.sh
+
+    echo " "  >> ./run_${CLUSTER}_1node.sh
+    echo "Running autogen " >> ./run_${CLUSTER}_1node.sh
+    echo "../autogen.sh" >> ./run_${CLUSTER}_1node.sh
+    echo " "  >> ./run_${CLUSTER}_1node.sh
+    echo "Running configure " >> ./run_${CLUSTER}_1node.sh
+    echo "../configure " "$projectConfigureArgs " >> ./run_${CLUSTER}_1node.sh
+    echo " "  >> ./run_${CLUSTER}_1node.sh
+    echo "Running make " >> ./run_${CLUSTER}_1node.sh
+    echo "make -j 8" >> ./run_${CLUSTER}_1node.sh
+    echo "export LD_LIBRARY_PATH=$MKL_HOME/lib/intel64:\$LD_LIBRARY_PATH" >> ./run_${CLUSTER}_1node.sh
+    echo "./$projectExecutable" >> ./run_${CLUSTER}_1node.sh
+    echo "make distclean" >> ./run_${CLUSTER}_1node.sh
+    echo "popd" >> ./run_${CLUSTER}_1node.sh
+    echo "pushd build" >> ./run_${CLUSTER}_1node.sh
+    echo "make distclean" >> ./run_${CLUSTER}_1node.sh
+    echo "rm -rf installdest" >> ./run_${CLUSTER}_1node.sh
+    echo "popd" >> ./run_${CLUSTER}_1node.sh
+    echo " " >> ./run_${CLUSTER}_1node.sh
+    echo " "
+    echo "Job script for the run"
+    cat ./run_${CLUSTER}_1node.sh
+    echo " "
+    echo "Submitting to SLURM"
+    sbatch -W ./run_${CLUSTER}_1node.sh
+    exitCode=$?
+
+    echo " "
+    echo "Exit Code of sbatch: $exitCode"
+    echo " "
+    cat ./ELPA_CI.out.*
+    if [ $exitCode -ne 0 ]
+    then
+      cat ./ELPA_CI.err.*
+    fi
+
+  fi
+
+  if [ $exitCode -ne 0 ]
+  then
+    cat ./test-suite.log
+  fi
+
+  exit $exitCode
+
+fi