Gitlab CI: unique test dir

1872f44b · Andreas Marek · 7b1a20fb · 1872f44b · 1872f44b · 1872f44b
Commit 1872f44b authored Oct 07, 2019 by Andreas Marek
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -59,7 +59,6 @@ distcheck:
  tags:
    - distcheck
  script:
-    - ./ci_test_scripts/run_ci_tests.sh -c " CC=gcc FC=gfortran SCALAPACK_LDFLAGS=\"$MKL_GFORTRAN_SCALAPACK_LDFLAGS_NO_MPI_NO_OMP\"          SCALAPACK_FCFLAGS=\"$MKL_GFORTRAN_SCALAPACK_FCFLAGS_NO_MPI_NO_OMP\" --enable-option-checking=fatal --with-mpi=no --disable-sse-assembly         --disable-sse --disable-avx --disable-avx2 || { cat config.log; exit 1; } " -t $MPI_TASKS -m 150 -n 50 -b 16 -s $SKIP_STEP -i $INTERACTIVE_RUN -S $SLURM 
    - ./ci_test_scripts/run_distcheck_tests.sh  -c "  CC=gcc FC=gfortran SCALAPACK_LDFLAGS=\"$MKL_GFORTRAN_SCALAPACK_LDFLAGS_NO_MPI_NO_OMP\"          SCALAPACK_FCFLAGS=\"$MKL_GFORTRAN_SCALAPACK_FCFLAGS_NO_MPI_NO_OMP\" --enable-option-checking=fatal --with-mpi=no --disable-sse-assembly         --disable-sse --disable-avx --disable-avx2 " -d "  CC=gcc FC=gfortran SCALAPACK_LDFLAGS=\\\"$MKL_GFORTRAN_SCALAPACK_LDFLAGS_NO_MPI_NO_OMP\\\"          SCALAPACK_FCFLAGS=\\\"$MKL_GFORTRAN_SCALAPACK_FCFLAGS_NO_MPI_NO_OMP\\\" --enable-option-checking=fatal --with-mpi=no --disable-sse-assembly         --disable-sse --disable-avx --disable-avx2 " -t $MPI_TASKS -m 150 -n 50 -b 16 -S$SLURM 
  
  
@@ -68,8 +67,7 @@ distcheck-mpi:
  tags:
    - distcheck
  script:
-    - ./ci_test_scripts/run_ci_tests.sh -c " FC=mpiifort FCFLAGS=\"-xHost\" CFLAGS=\"-march=native\"   SCALAPACK_LDFLAGS=\"$MKL_INTEL_SCALAPACK_LDFLAGS_MPI_NO_OMP\"   SCALAPACK_FCFLAGS=\"$MKL_INTEL_SCALAPACK_FCFLAGS_MPI_NO_OMP\"   --enable-option-checking=fatal --with-mpi=yes  --disable-sse-assembly --disable-sse --disable-avx --disable-avx2 || { cat config.log; exit 1; } " -t $MPI_TASKS -m 150  -n 50 -b 16 -s $SKIP_STEP -i $INTERACTIVE_RUN -S $SLURM 
-    - ./ci_test_scripts/run_distcheck_tests.sh  -c " FC=mpiifort FCFLAGS=\"-xHost\"  CFLAGS=\"-march=native\" SCALAPACK_LDFLAGS=\"$MKL_INTEL_SCALAPACK_LDFLAGS_MPI_NO_OMP\"   SCALAPACK_FCFLAGS=\"$MKL_INTEL_SCALAPACK_FCFLAGS_MPI_NO_OMP\" --enable-option-checking=fatal  --with-mpi=yes --disable-sse-assembly --disable-sse --disable-avx --disable-avx2 " -d " FC=mpiifort FCFLAGS=\\\"-xHost\\\"  CFLAGS=\\\"-march=native\\\" SCALAPACK_LDFLAGS=\\\"$MKL_INTEL_SCALAPACK_LDFLAGS_MPI_NO_OMP\\\"   SCALAPACK_FCFLAGS=\\\"$MKL_INTEL_SCALAPACK_FCFLAGS_MPI_NO_OMP\\\" --enable-option-checking=fatal  --with-mpi=yes --disable-sse-assembly --disable-sse --disable-avx --disable-avx2 " - -t $MPI_TASKS -m 150 -n 50 -b 16 -S$SLURM 
+    - ./ci_test_scripts/run_distcheck_tests.sh  -c " FC=mpiifort FCFLAGS=\"-xHost\"  CFLAGS=\"-march=native\" SCALAPACK_LDFLAGS=\"$MKL_INTEL_SCALAPACK_LDFLAGS_MPI_NO_OMP\"   SCALAPACK_FCFLAGS=\"$MKL_INTEL_SCALAPACK_FCFLAGS_MPI_NO_OMP\" --enable-option-checking=fatal  --with-mpi=yes --disable-sse-assembly --disable-sse --disable-avx --disable-avx2 " -d " FC=mpiifort FCFLAGS=\\\"-xHost\\\"  CFLAGS=\\\"-march=native\\\" SCALAPACK_LDFLAGS=\\\"$MKL_INTEL_SCALAPACK_LDFLAGS_MPI_NO_OMP\\\"   SCALAPACK_FCFLAGS=\\\"$MKL_INTEL_SCALAPACK_FCFLAGS_MPI_NO_OMP\\\" --enable-option-checking=fatal  --with-mpi=yes --disable-sse-assembly --disable-sse --disable-avx --disable-avx2 " -t $MPI_TASKS -m 150 -n 50 -b 16 -S$SLURM 
  
  
  
@@ -77,7 +75,6 @@ distcheck-no-autotune:
  tags:
    - distcheck
  script:
-    - ./ci_test_scripts/run_ci_tests.sh -c " FC=mpiifort FCFLAGS=\"-xHost\" CFLAGS=\"-march=native\"   SCALAPACK_LDFLAGS=\"$MKL_INTEL_SCALAPACK_LDFLAGS_MPI_NO_OMP\"   SCALAPACK_FCFLAGS=\"$MKL_INTEL_SCALAPACK_FCFLAGS_MPI_NO_OMP\"   --enable-option-checking=fatal --with-mpi=yes   --disable-sse-assembly --disable-sse --disable-avx --disable-avx2 --disable-autotuning || { cat config.log; exit 1; } " -t $MPI_TASKS -m 150   -n 50 -b 16 -s $SKIP_STEP -i $INTERACTIVE_RUN -S $SLURM 
    - ./ci_test_scripts/run_distcheck_tests.sh  -c " FC=mpiifort FCFLAGS=\"-xHost\"  CFLAGS=\"-march=native\" SCALAPACK_LDFLAGS=\"$MKL_INTEL_SCALAPACK_LDFLAGS_MPI_NO_OMP\"   SCALAPACK_FCFLAGS=\"$MKL_INTEL_SCALAPACK_FCFLAGS_MPI_NO_OMP\" --enable-option-checking=fatal  --with-mpi=yes --disable-sse-assembly --disable-sse --disable-avx --disable-avx2 --disable-autotuning " -d " FC=mpiifort FCFLAGS=\\\"-xHost\\\"  CFLAGS=\\\"-march=native\\\" SCALAPACK_LDFLAGS=\\\"$MKL_INTEL_SCALAPACK_LDFLAGS_MPI_NO_OMP\\\"   SCALAPACK_FCFLAGS=\\\"$MKL_INTEL_SCALAPACK_FCFLAGS_MPI_NO_OMP\\\" --enable-option-checking=fatal  --with-mpi=yes --disable-sse-assembly --disable-sse --disable-avx --disable-avx2 --disable-autotuning " -t $MPI_TASKS -m 150 -n 50 -b 16 -S$SLURM 
  
  
@@ -97,7 +94,7 @@ python-distcheck:
  tags:
    - python-distcheck
  script:
-    - ./ci_test_scripts/run_ci_tests.sh -c "CC=\"mpiicc\" CFLAGS=\"-O3 -xAVX\" FC="mpiifort" FCFLAGS=\"-O3 -xAVX\" SCALAPACK_LDFLAGS=\"$MKL_ANACONDA_INTEL_SCALAPACK_LDFLAGS_MPI_OMP\" SCALAPACK_FCFLAGS=\"$MKL_ANACONDA_INTEL_SCALAPACK_FCFLAGS_MPI_OMP\" --enable-option-checking=fatal --with-mpi=yes --enable-openmp --disable-gpu --enable-avx --enable-python --enable-python-tests || { cat config.log; exit 1; }" -j 8 -t $MPI_TASKS -m 150 -n 50 -b 16 -s $SKIP_STEP -i $INTERACTIVE_RUN -S $SLURM
+#    - ./ci_test_scripts/run_ci_tests.sh -c "CC=\"mpiicc\" CFLAGS=\"-O3 -xAVX\" FC="mpiifort" FCFLAGS=\"-O3 -xAVX\" SCALAPACK_LDFLAGS=\"$MKL_ANACONDA_INTEL_SCALAPACK_LDFLAGS_MPI_OMP\" SCALAPACK_FCFLAGS=\"$MKL_ANACONDA_INTEL_SCALAPACK_FCFLAGS_MPI_OMP\" --enable-option-checking=fatal --with-mpi=yes --enable-openmp --disable-gpu --enable-avx --enable-python --enable-python-tests || { cat config.log; exit 1; }" -j 8 -t $MPI_TASKS -m 150 -n 50 -b 16 -s $SKIP_STEP -i $INTERACTIVE_RUN -S $SLURM
  
  
    - ./ci_test_scripts/run_distcheck_tests.sh -c "CC=\"mpiicc\" CFLAGS=\"-O3 -xAVX\" FC=\"mpiifort\" FCFLAGS=\"-O3 -xAVX\" SCALAPACK_LDFLAGS=\"$MKL_ANACONDA_INTEL_SCALAPACK_LDFLAGS_MPI_OMP \" SCALAPACK_FCFLAGS=\"$MKL_ANACONDA_INTEL_SCALAPACK_FCFLAGS_MPI_OMP \" --enable-option-checking=fatal --with-mpi=yes --enable-openmp --disable-gpu --enable-avx --enable-python --enable-python-tests "-d " CC=\\\"mpiicc\\\" CFLAGS=\\\"-O3 -xAVX\\\" FC=\\\"mpiifort\\\" FCFLAGS=\\\"-O3 -xAVX\\\" SCALAPACK_LDFLAGS=\"$MKL_ANACONDA_INTEL_SCALAPACK_LDFLAGS_MPI_OMP \" SCALAPACK_FCFLAGS=\"$MKL_ANACONDA_INTEL_SCALAPACK_FCFLAGS_MPI_OMP \" --enable-option-checking=fatal --with-mpi=yes --enable-openmp --disable-gpu --enable-avx --enable-python --enable-python-tests" -t $MPI_TASKS -m 150 -n 50 -b 16 -S $SLURM  || { chmod u+rwX -R . ; exit 1 ; }

--- a/ci_test_scripts/generate_gitlab_ci_tests.py
+++ b/ci_test_scripts/generate_gitlab_ci_tests.py
@@ -235,6 +235,8 @@ print("# next time a runner tries to clean-up")
 print("after_script:")
 print("  - chmod u+w -R .")
 print("  - if [ ! -f ./debug.keep ]; then find . -exec chmod a+rxw {} \; && rm -rf * ; fi")
+#print("  - chmod u+w -R /tmp/elpa")
+#print("  - find /tmp/elpa -exec chmod a+rxw {} \; && rm -rf /tmp/elpa ;")
 print("\n\n")


@@ -281,9 +283,9 @@ print("distcheck:")
 print("  tags:")
 print("    - distcheck")
 print("  script:")
-print("    - ./ci_test_scripts/run_ci_tests.sh -c \" CC=gcc FC=gfortran SCALAPACK_LDFLAGS=\\\"$MKL_GFORTRAN_SCALAPACK_LDFLAGS_NO_MPI_NO_OMP\\\"  \
-        SCALAPACK_FCFLAGS=\\\"$MKL_GFORTRAN_SCALAPACK_FCFLAGS_NO_MPI_NO_OMP\\\" --enable-option-checking=fatal --with-mpi=no --disable-sse-assembly \
-        --disable-sse --disable-avx --disable-avx2 || { cat config.log; exit 1; } \" -t $MPI_TASKS -m 150 -n 50 -b 16 -s $SKIP_STEP -i $INTERACTIVE_RUN -S $SLURM ")
+#print("    - ./ci_test_scripts/run_ci_tests.sh -c \" CC=gcc FC=gfortran SCALAPACK_LDFLAGS=\\\"$MKL_GFORTRAN_SCALAPACK_LDFLAGS_NO_MPI_NO_OMP\\\"  \
+#        SCALAPACK_FCFLAGS=\\\"$MKL_GFORTRAN_SCALAPACK_FCFLAGS_NO_MPI_NO_OMP\\\" --enable-option-checking=fatal --with-mpi=no --disable-sse-assembly \
+#        --disable-sse --disable-avx --disable-avx2 || { cat config.log; exit 1; } \" -t $MPI_TASKS -m 150 -n 50 -b 16 -s $SKIP_STEP -i $INTERACTIVE_RUN -S $SLURM ")
 print("    - ./ci_test_scripts/run_distcheck_tests.sh  -c \"  CC=gcc FC=gfortran SCALAPACK_LDFLAGS=\\\"$MKL_GFORTRAN_SCALAPACK_LDFLAGS_NO_MPI_NO_OMP\\\"  \
        SCALAPACK_FCFLAGS=\\\"$MKL_GFORTRAN_SCALAPACK_FCFLAGS_NO_MPI_NO_OMP\\\" --enable-option-checking=fatal --with-mpi=no --disable-sse-assembly \
        --disable-sse --disable-avx --disable-avx2 \" -d \"  CC=gcc FC=gfortran SCALAPACK_LDFLAGS=\\\\\\\"$MKL_GFORTRAN_SCALAPACK_LDFLAGS_NO_MPI_NO_OMP\\\\\\\"  \
@@ -295,12 +297,12 @@ print("distcheck-mpi:")
 print("  tags:")
 print("    - distcheck")
 print("  script:")
-print("    - ./ci_test_scripts/run_ci_tests.sh -c \" FC=mpiifort FCFLAGS=\\\"-xHost\\\" CFLAGS=\\\"-march=native\\\" \
-  SCALAPACK_LDFLAGS=\\\"$MKL_INTEL_SCALAPACK_LDFLAGS_MPI_NO_OMP\\\" \
-  SCALAPACK_FCFLAGS=\\\"$MKL_INTEL_SCALAPACK_FCFLAGS_MPI_NO_OMP\\\" \
-  --enable-option-checking=fatal --with-mpi=yes \
- --disable-sse-assembly --disable-sse --disable-avx --disable-avx2 || { cat config.log; exit 1; } \" -t $MPI_TASKS -m 150 \
- -n 50 -b 16 -s $SKIP_STEP -i $INTERACTIVE_RUN -S $SLURM ")
+#print("    - ./ci_test_scripts/run_ci_tests.sh -c \" FC=mpiifort FCFLAGS=\\\"-xHost\\\" CFLAGS=\\\"-march=native\\\" \
+#  SCALAPACK_LDFLAGS=\\\"$MKL_INTEL_SCALAPACK_LDFLAGS_MPI_NO_OMP\\\" \
+#  SCALAPACK_FCFLAGS=\\\"$MKL_INTEL_SCALAPACK_FCFLAGS_MPI_NO_OMP\\\" \
+#  --enable-option-checking=fatal --with-mpi=yes \
+# --disable-sse-assembly --disable-sse --disable-avx --disable-avx2 || { cat config.log; exit 1; } \" -t $MPI_TASKS -m 150 \
+# -n 50 -b 16 -s $SKIP_STEP -i $INTERACTIVE_RUN -S $SLURM ")
 print("    - ./ci_test_scripts/run_distcheck_tests.sh  -c \" FC=mpiifort FCFLAGS=\\\"-xHost\\\" \
 CFLAGS=\\\"-march=native\\\" SCALAPACK_LDFLAGS=\\\"$MKL_INTEL_SCALAPACK_LDFLAGS_MPI_NO_OMP\\\"  \
 SCALAPACK_FCFLAGS=\\\"$MKL_INTEL_SCALAPACK_FCFLAGS_MPI_NO_OMP\\\" --enable-option-checking=fatal \
@@ -314,12 +316,12 @@ print("distcheck-no-autotune:")
 print("  tags:")
 print("    - distcheck")
 print("  script:")
-print("    - ./ci_test_scripts/run_ci_tests.sh -c \" FC=mpiifort FCFLAGS=\\\"-xHost\\\" CFLAGS=\\\"-march=native\\\" \
-  SCALAPACK_LDFLAGS=\\\"$MKL_INTEL_SCALAPACK_LDFLAGS_MPI_NO_OMP\\\" \
-  SCALAPACK_FCFLAGS=\\\"$MKL_INTEL_SCALAPACK_FCFLAGS_MPI_NO_OMP\\\" \
-  --enable-option-checking=fatal --with-mpi=yes \
-  --disable-sse-assembly --disable-sse --disable-avx --disable-avx2 --disable-autotuning || { cat config.log; exit 1; } \" -t $MPI_TASKS -m 150 \
-  -n 50 -b 16 -s $SKIP_STEP -i $INTERACTIVE_RUN -S $SLURM ")
+#print("    - ./ci_test_scripts/run_ci_tests.sh -c \" FC=mpiifort FCFLAGS=\\\"-xHost\\\" CFLAGS=\\\"-march=native\\\" \
+#  SCALAPACK_LDFLAGS=\\\"$MKL_INTEL_SCALAPACK_LDFLAGS_MPI_NO_OMP\\\" \
+#  SCALAPACK_FCFLAGS=\\\"$MKL_INTEL_SCALAPACK_FCFLAGS_MPI_NO_OMP\\\" \
+#  --enable-option-checking=fatal --with-mpi=yes \
+#  --disable-sse-assembly --disable-sse --disable-avx --disable-avx2 --disable-autotuning || { cat config.log; exit 1; } \" -t $MPI_TASKS -m 150 \
+#  -n 50 -b 16 -s $SKIP_STEP -i $INTERACTIVE_RUN -S $SLURM ")
 print("    - ./ci_test_scripts/run_distcheck_tests.sh  -c \" FC=mpiifort FCFLAGS=\\\"-xHost\\\" \
 CFLAGS=\\\"-march=native\\\" SCALAPACK_LDFLAGS=\\\"$MKL_INTEL_SCALAPACK_LDFLAGS_MPI_NO_OMP\\\"  \
 SCALAPACK_FCFLAGS=\\\"$MKL_INTEL_SCALAPACK_FCFLAGS_MPI_NO_OMP\\\" --enable-option-checking=fatal \
@@ -357,7 +359,7 @@ python_ci_tests = [
    "  tags:",
    "    - python-distcheck",
    "  script:",
-    '    - ./ci_test_scripts/run_ci_tests.sh -c "'
+    '#    - ./ci_test_scripts/run_ci_tests.sh -c "'
    'CC=\\\"mpiicc\\\" CFLAGS=\\\"-O3 -xAVX\\\" '
    'FC="mpiifort" FCFLAGS=\\\"-O3 -xAVX\\\" '
    'SCALAPACK_LDFLAGS=\\\"$MKL_ANACONDA_INTEL_SCALAPACK_LDFLAGS_MPI_OMP\\\" '

--- a/ci_test_scripts/run_ci_tests.sh
+++ b/ci_test_scripts/run_ci_tests.sh
@@ -146,6 +146,7 @@ then
    echo " " >> ./run_${CLUSTER}_1node_2GPU.sh
    echo "#copy everything back from /tmp/elpa to runner directory"  >> ./run_${CLUSTER}_1node_2GPU.sh
    echo "cp -r * \$runner_path"  >> ./run_${CLUSTER}_1node_2GPU.sh
+    echo "cd .. && rm -rf /tmp/elpa_\$SLURM_JOB_ID" >> ./run_${CLUSTER}_1node_2GPU.sh

    echo " "
    echo "Job script for the run"
@@ -180,7 +181,7 @@ then
      echo " " >> ./run_${CLUSTER}_1node.sh
      echo "#copy everything back from /tmp/elpa to runner directory"  >> ./run_${CLUSTER}_1node.sh
      echo "cp -r * \$runner_path"  >> ./run_${CLUSTER}_1node.sh
-
+      echo "cd .. && rm -rf /tmp/elpa_\$SLURM_JOB_ID" >> ./run_${CLUSTER}_1node.sh

      echo " "
      echo "Job script for the run"

--- a/ci_test_scripts/run_distcheck_tests.sh
+++ b/ci_test_scripts/run_distcheck_tests.sh
@@ -129,6 +129,7 @@ then
    echo " " >> ./run_${CLUSTER}_1node.sh
    echo "#copy everything back from /tmp/elpa to runner directory" >> ./run_${CLUSTER}_1node.sh
    echo "cp -r * \$runner_path" >> ./run_${CLUSTER}_1node.sh
+    echo "cd .. && rm -rf /tmp/elpa_\$SLURM_JOB_ID" >> ./run_${CLUSTER}_1node.sh

    echo " "
    echo "Job script for the run"

--- a/ci_test_scripts/run_project_tests.sh
+++ b/ci_test_scripts/run_project_tests.sh
@@ -180,6 +180,7 @@ then
    echo " " >> ./run_${CLUSTER}_1node.sh
    echo "#copy everything back from /tmp/elpa to runner directory" >> ./run_${CLUSTER}_1node.sh
    echo "cp -r * \$runner_path"  >> ./run_${CLUSTER}_1node.sh
+    echo "cd .. && rm -rf /tmp/elpa_\$SLURM_JOB_ID" >> ./run_${CLUSTER}_1node.sh
    echo " "
    echo "Job script for the run"
    cat ./run_${CLUSTER}_1node.sh