Gitlab CI: Test for 64bit blas and 32bit MPI

.gitlab-ci.yml

@@ -101,6 +101,28 @@ gnu-gnu-nompi-openmp-ilp64:
    - ./ci_test_scripts/run_ci_tests.sh -c "CC=\"gcc\" CFLAGS=\"-O3 -mavx\" FC=\"gfortran\" FCFLAGS=\"-O3 -mavx\" SCALAPACK_LDFLAGS=\"$MKL_GFORTRAN_SCALAPACK_LDFLAGS_NOMPI_OMP_ILP64 \" SCALAPACK_FCFLAGS=\"$MKL_GFORTRAN_SCALAPACK_FCFLAGS_NOMPI_OMP_ILP64 \" --enable-option-checking=fatal --with-mpi=no --enable-openmp --disable-gpu --enable-avx --disable-avx2 --disable-avx512 --enable-64bit-integer-math-support || { cat config.log; exit 1; }" -j 8 -t $MPI_TASKS -m $MATRIX_SIZE -n $NUMBER_OF_EIGENVECTORS -b $BLOCK_SIZE -s $SKIP_STEP -i $INTERACTIVE_RUN -S $SLURM
+# gnu-gnu-ilp64-mpi-noomp
+gnu-gnu-mpi-noopenmp-ilp64:
+  tags:
+    - avx
+  artifacts:
+    when: on_success
+    expire_in: 2 month
+  script:
+   - ./ci_test_scripts/run_ci_tests.sh -c "CC=\"mpicc\" CFLAGS=\"-O3 -mavx\" FC=\"mpif90\" FCFLAGS=\"-O3 -mavx\" SCALAPACK_LDFLAGS=\"$MKL_GFORTRAN_SCALAPACK_LDFLAGS_MPI_NOOMP_ILP64 \" SCALAPACK_FCFLAGS=\"$MKL_GFORTRAN_SCALAPACK_FCFLAGS_MPI_NOOMP_ILP64 \" --enable-option-checking=fatal --with-mpi=yes --disable-openmp --disable-gpu --enable-avx --disable-avx2 --disable-avx512 --enable-64bit-integer-math-support || { cat config.log; exit 1; }" -j 8 -t $MPI_TASKS -m $MATRIX_SIZE -n $NUMBER_OF_EIGENVECTORS -b $BLOCK_SIZE -s $SKIP_STEP -i $INTERACTIVE_RUN -S $SLURM
+
+
+# gnu-gnu-ilp64-mpi-openmp
+gnu-gnu-mpi-openmp-ilp64:
+  tags:
+    - avx
+  artifacts:
+    when: on_success
+    expire_in: 2 month
+  script:
+   - ./ci_test_scripts/run_ci_tests.sh -c "CC=\"mpicc\" CFLAGS=\"-O3 -mavx\" FC=\"mpif90\" FCFLAGS=\"-O3 -mavx\" SCALAPACK_LDFLAGS=\"$MKL_GFORTRAN_SCALAPACK_LDFLAGS_MPI_OMP_ILP64 \" SCALAPACK_FCFLAGS=\"$MKL_GFORTRAN_SCALAPACK_FCFLAGS_MPI_OMP_ILP64 \" --enable-option-checking=fatal --with-mpi=yes --enable-openmp --disable-gpu --enable-avx --disable-avx2 --disable-avx512 --enable-64bit-integer-math-support || { cat config.log; exit 1; }" -j 8 -t $MPI_TASKS -m $MATRIX_SIZE -n $NUMBER_OF_EIGENVECTORS -b $BLOCK_SIZE -s $SKIP_STEP -i $INTERACTIVE_RUN -S $SLURM
+
+
 # python tests
 python-intel-intel-mpi-openmp:
   tags:

ci_test_scripts/.ci-env-vars

@@ -67,6 +67,17 @@ export MKL_GFORTRAN_SCALAPACK_NOMPI_OMP_ILP64_BASELINE="-L$MKL_HOME/lib/intel64
 export MKL_GFORTRAN_SCALAPACK_FCFLAGS_NOMPI_OMP_ILP64="-I$MKL_HOME/include/intel64/ilp64"
 export MKL_GFORTRAN_SCALAPACK_LDFLAGS_NOMPI_OMP_ILP64="$MKL_GFORTRAN_SCALAPACK_NOMPI_OMP_ILP64_BASELINE -Wl,-rpath,$MKL_HOME/lib/intel64"
 
+export MKL_GFORTRAN_SCALAPACK_MPI_NOOMP_ILP64_BASELINE="-L$MKL_HOME/lib/intel64 -lmkl_scalapack_ilp64 -lmkl_gf_ilp64 -lmkl_sequential -lmkl_core -lmkl_blacs_intelmpi_ilp64 -lpthread"
+export MKL_GFORTRAN_SCALAPACK_FCFLAGS_MPI_NOOMP_ILP64="-I$MKL_HOME/include/intel64/ilp64"
+export MKL_GFORTRAN_SCALAPACK_LDFLAGS_MPI_NOOMP_ILP64="$MKL_GFORTRAN_SCALAPACK_MPI_NOOMP_ILP64_BASELINE -Wl,-rpath,$MKL_HOME/lib/intel64"
+
+export MKL_GFORTRAN_SCALAPACK_MPI_OMP_ILP64_BASELINE="-L$MKL_HOME/lib/intel64 -lmkl_scalapack_ilp64 -lmkl_gf_ilp64 -lmkl_sequential -lmkl_core -lmkl_blacs_intelmpi_ilp64 -lpthread"
+export MKL_GFORTRAN_SCALAPACK_FCFLAGS_MPI_OMP_ILP64="-I$MKL_HOME/include/intel64/ilp64"
+export MKL_GFORTRAN_SCALAPACK_LDFLAGS_MPI_OMP_ILP64="$MKL_GFORTRAN_SCALAPACK_MPI_OMP_ILP64_BASELINE -Wl,-rpath,$MKL_HOME/lib/intel64"
+
+
+
+
 
 export ASAN_OPTIONS=suppressions=./ci_test_scripts/no_asan_for_mpi.supp,fast_unwind_on_malloc=0
 export LSAN_OPTIONS=suppressions=./ci_test_scripts/no_lsan_for_mpi.supp

ci_test_scripts/generate_gitlab_ci_tests.py

@@ -374,6 +374,48 @@ ilp64_no_omp_tests = [
 
 print("\n".join(ilp64_no_omp_tests))
 
+#two test for ilp64 mkl interface with 32bit intel MPI
+ilp64_no_omp_mpi_tests = [
+    "# gnu-gnu-ilp64-mpi-noomp",
+    "gnu-gnu-mpi-noopenmp-ilp64:",
+    "  tags:",
+    "    - avx",
+    "  artifacts:",
+    "    when: on_success",
+    "    expire_in: 2 month",
+    "  script:",
+    '   - ./ci_test_scripts/run_ci_tests.sh -c "'
+    'CC=\\"mpicc\\" CFLAGS=\\"-O3 -mavx\\" '
+    'FC=\\"mpif90\\" FCFLAGS=\\"-O3 -mavx\\" '
+    'SCALAPACK_LDFLAGS=\\"$MKL_GFORTRAN_SCALAPACK_LDFLAGS_MPI_NOOMP_ILP64 \\" '
+    'SCALAPACK_FCFLAGS=\\"$MKL_GFORTRAN_SCALAPACK_FCFLAGS_MPI_NOOMP_ILP64 \\" '
+    '--enable-option-checking=fatal --with-mpi=yes --disable-openmp '
+    '--disable-gpu --enable-avx --disable-avx2 --disable-avx512 --enable-64bit-integer-math-support || { cat config.log; exit 1; }'
+    '" -j 8 -t $MPI_TASKS -m $MATRIX_SIZE -n $NUMBER_OF_EIGENVECTORS -b $BLOCK_SIZE '
+    '-s $SKIP_STEP -i $INTERACTIVE_RUN -S $SLURM',
+    "\n",
+    "# gnu-gnu-ilp64-mpi-openmp",
+    "gnu-gnu-mpi-openmp-ilp64:",
+    "  tags:",
+    "    - avx",
+    "  artifacts:",
+    "    when: on_success",
+    "    expire_in: 2 month",
+    "  script:",
+    '   - ./ci_test_scripts/run_ci_tests.sh -c "'
+    'CC=\\"mpicc\\" CFLAGS=\\"-O3 -mavx\\" '
+    'FC=\\"mpif90\\" FCFLAGS=\\"-O3 -mavx\\" '
+    'SCALAPACK_LDFLAGS=\\"$MKL_GFORTRAN_SCALAPACK_LDFLAGS_MPI_OMP_ILP64 \\" '
+    'SCALAPACK_FCFLAGS=\\"$MKL_GFORTRAN_SCALAPACK_FCFLAGS_MPI_OMP_ILP64 \\" '
+    '--enable-option-checking=fatal --with-mpi=yes --enable-openmp '
+    '--disable-gpu --enable-avx --disable-avx2 --disable-avx512 --enable-64bit-integer-math-support || { cat config.log; exit 1; }'
+    '" -j 8 -t $MPI_TASKS -m $MATRIX_SIZE -n $NUMBER_OF_EIGENVECTORS -b $BLOCK_SIZE '
+    '-s $SKIP_STEP -i $INTERACTIVE_RUN -S $SLURM',
+    "\n",
+]
+
+print("\n".join(ilp64_no_omp_mpi_tests))
+
 # add python tests
 python_ci_tests = [
     "# python tests",

test/shared/test_analytic_template.F90

@@ -72,7 +72,7 @@
     logical, optional                               :: print_times
     logical                                         :: print_timer
     TEST_INT_TYPE                                   :: globI, globJ, locI, locJ, pi, pj, levels(num_primes)
-    integer(kind=c_int)                             :: loc_I, loc_J
+    integer(kind=c_int)                             :: loc_I, loc_J, p_i, p_j
 #ifdef HAVE_DETAILED_TIMINGS
     type(timer_t)                                   :: timer
 #else
@@ -105,12 +105,14 @@
     call timer%start("loop")
     do globI = 1, na
 
-      pi = int(prow(int(globI,kind=c_int), int(nblk,kind=c_int), int(np_rows,kind=c_int)),kind=INT_TYPE)
+      p_i = prow(int(globI,kind=c_int), int(nblk,kind=c_int), int(np_rows,kind=c_int))
+      pi = int(p_i,kind=INT_TYPE)
       if (my_prow .ne. pi) cycle
 
       do globJ = 1, na
 
-        pj = int(pcol(int(globJ,kind=c_int), int(nblk,kind=c_int), int(np_cols,kind=c_int)),kind=INT_TYPE)
+        p_j = pcol(int(globJ,kind=c_int), int(nblk,kind=c_int), int(np_cols,kind=c_int))
+        pj = int(p_j,kind=INT_TYPE)
         if (my_pcol .ne. pj) cycle
 
         if(map_global_array_index_to_local_index(int(globI,kind=c_int), int(globJ,kind=c_int), loc_I, loc_J, &
@@ -180,10 +182,10 @@
                                               normalization_quotient
     MATH_DATATYPE(kind=rck)                :: max_values_array(np_rows * np_cols), &
                                               corresponding_exact_value
-    TEST_INT_TYPE                          :: max_value_idx, rank_with_max, &
+    integer(kind=c_int)                    :: max_value_idx, rank_with_max, &
                                               rank_with_max_reduced,        &
                                               num_checked_evals
-    TEST_INT_TYPE                          :: max_idx_array(np_rows * np_cols), &
+    integer(kind=c_int)                    :: max_idx_array(np_rows * np_cols), &
                                               rank
     logical, optional                      :: print_times
     logical                                :: print_timer
@@ -251,7 +253,7 @@
           computed_z = z(locI, locJ)
           if(abs(computed_z) > abs(max_value_for_normalization)) then
             max_value_for_normalization = computed_z
-            max_value_idx = globI
+            max_value_idx = int(globI,kind=c_int)
           end if
         end if
       end do
@@ -291,7 +293,7 @@
                                        &MATH_DATATYPE&
                                        &_&
                                        &PRECISION&
-                                       &(na, max_value_idx, globJ)
+                                       &(na, int(max_value_idx,kind=INT_TYPE), globJ)
       call timer%stop("evaluation_helper")
       normalization_quotient = corresponding_exact_value / max_value_for_normalization
       ! write(*,*) "normalization q", normalization_quotient