Loop over all possible domain decompositions

We got reports from a user that there were troubles with certain domain
decompositions. So far the tests only looked at (approximately) square
decompositions in column-major process order.

Now, a new class of tests loops over all possible decompositions
(row * col) for a given number of total tasks.

So far, we can not confirm that there are any problems, all
possibilities work as expected.

generate_automake_test_programs.py

@@ -29,12 +29,19 @@ test_type_flag = {
         "solve_tridiagonal"  : "-D__SOLVE_TRIDIAGONAL",
 }
 
-for m, g, t, p, d, s in product(sorted(matrix_flag.keys()),
+layout_flag = {
+        "all_layouts" : "-DTEST_ALL_LAYOUTS",
+        "square" : ""
+}
+
+for m, g, t, p, d, s, l in product(
+                             sorted(matrix_flag.keys()),
                              sorted(gpu_flag.keys()),
                              sorted(test_type_flag.keys()),
                              sorted(prec_flag.keys()),
                              sorted(domain_flag.keys()),
-                             sorted(solver_flag.keys())):
+                             sorted(solver_flag.keys()),
+                             sorted(layout_flag.keys())):
 
     #todo: decide what tests we actually want
     if(m == "analytic" and (g == 1 or t != "eigenvectors" or p == "single" or d == "complex")):
@@ -59,6 +66,9 @@ for m, g, t, p, d, s in product(sorted(matrix_flag.keys()),
         elif kernel == "all_kernels":
             extra_flags.append("-DTEST_ALL_KERNELS")
 
+        if layout_flag[l]:
+            extra_flags.append(layout_flag[l])
+
         if (p == "single"):
             if (d == "real"):
                 print("if WANT_SINGLE_PRECISION_REAL")
@@ -68,7 +78,12 @@ for m, g, t, p, d, s in product(sorted(matrix_flag.keys()),
                 raise Exception("Oh no!")
             endifs += 1
 
-        name = "test_{0}_{1}_{2}_{3}{4}{5}{6}".format(d, p, t, s, "" if kernel == "nokernel" else "_" + kernel, "_gpu" if g else "", "_analytic" if m == "analytic" else "")
+        name = "test_{0}_{1}_{2}_{3}{4}{5}{6}{7}".format(
+                    d, p, t, s,
+                    "" if kernel == "nokernel" else "_" + kernel,
+                    "_gpu" if g else "",
+                    "_analytic" if m == "analytic" else "",
+                    "_all_layouts" if l == "all_layouts" else "")
         print("noinst_PROGRAMS += " + name)
         print("check_SCRIPTS += " + name + ".sh")
         print(name + "_SOURCES = test/Fortran/test.F90")

src/elpa_impl.F90

@@ -1176,7 +1176,7 @@ module elpa_impl
     !> \param nrows_c               number of rows of local (sub) matrix c
     !> \param ncols_c               number of columns of local (sub) matrix c
     !> \param error                 optional argument, error code which can be queried with elpa_strerr
-    subroutine elpa_hermitian_multiply_d (self,uplo_a, uplo_c, ncb, a, b, nrows_b, ncols_b, &
+    subroutine elpa_hermitian_multiply_d (self, uplo_a, uplo_c, ncb, a, b, nrows_b, ncols_b, &
                                           c, nrows_c, ncols_c, error)
       use iso_c_binding
       use elpa1_auxiliary_impl

test/C/driver/legacy_interface/legacy_complex_driver_c_version.c

@@ -66,7 +66,7 @@ int main(int argc, char** argv) {
 
    int np_cols, np_rows, np_colsStart;
 
-   int my_blacs_ctxt, nprow, npcol, my_prow, my_pcol;
+   int my_blacs_ctxt, my_prow, my_pcol;
 
    int mpierr;
 
@@ -138,7 +138,7 @@ int main(int argc, char** argv) {
 #else
    my_mpi_comm_world = 1;
 #endif
-   set_up_blacsgrid_f(my_mpi_comm_world, &my_blacs_ctxt, &np_rows, &np_cols, &nprow, &npcol, &my_prow, &my_pcol);
+   set_up_blacsgrid_f(my_mpi_comm_world, np_rows, np_cols, 'C', &my_blacs_ctxt, &my_prow, &my_pcol);
 
    if (myid == 0) {
      printf("\n");

test/C/driver/legacy_interface/legacy_real_driver_c_version.c

@@ -65,7 +65,7 @@ int main(int argc, char** argv) {
 
    int np_cols, np_rows, np_colsStart;
 
-   int my_blacs_ctxt, nprow, npcol, my_prow, my_pcol;
+   int my_blacs_ctxt, my_prow, my_pcol;
 
    int mpierr;
 
@@ -134,7 +134,7 @@ int main(int argc, char** argv) {
 #else
   my_mpi_comm_world = 1;
 #endif
-   set_up_blacsgrid_f(my_mpi_comm_world, &my_blacs_ctxt, &np_rows, &np_cols, &nprow, &npcol, &my_prow, &my_pcol);
+   set_up_blacsgrid_f(my_mpi_comm_world, np_rows, np_cols, 'C', &my_blacs_ctxt, &my_prow, &my_pcol);
 
    if (myid == 0) {
      printf("\n");

test/C/driver/legacy_interface/legacy_single_complex_driver_c_version.c

@@ -66,7 +66,7 @@ int main(int argc, char** argv) {
 
    int np_cols, np_rows, np_colsStart;
 
-   int my_blacs_ctxt, nprow, npcol, my_prow, my_pcol;
+   int my_blacs_ctxt, my_prow, my_pcol;
 
    int mpierr;
 
@@ -138,7 +138,7 @@ int main(int argc, char** argv) {
 #else
    my_mpi_comm_world = 1;
 #endif
-   set_up_blacsgrid_f(my_mpi_comm_world, &my_blacs_ctxt, &np_rows, &np_cols, &nprow, &npcol, &my_prow, &my_pcol);
+   set_up_blacsgrid_f(my_mpi_comm_world, np_rows, np_cols, 'C', &my_blacs_ctxt, &my_prow, &my_pcol);
 
    if (myid == 0) {
      printf("\n");

test/C/driver/legacy_interface/legacy_single_real_driver_c_version.c

@@ -65,7 +65,7 @@ int main(int argc, char** argv) {
 
    int np_cols, np_rows, np_colsStart;
 
-   int my_blacs_ctxt, nprow, npcol, my_prow, my_pcol;
+   int my_blacs_ctxt, my_prow, my_pcol;
 
    int mpierr;
 
@@ -135,7 +135,7 @@ int main(int argc, char** argv) {
 #else
   my_mpi_comm_world = 1;
 #endif
-   set_up_blacsgrid_f(my_mpi_comm_world, &my_blacs_ctxt, &np_rows, &np_cols, &nprow, &npcol, &my_prow, &my_pcol);
+   set_up_blacsgrid_f(my_mpi_comm_world, np_rows, np_cols, 'C', &my_blacs_ctxt, &my_prow, &my_pcol);
 
    if (myid == 0) {
      printf("\n");

test/C/elpa1/legacy_interface/legacy_complex_1stage_c_version.c

@@ -68,7 +68,7 @@ int main(int argc, char** argv) {
 
    int np_cols, np_rows, np_colsStart;
 
-   int my_blacs_ctxt, nprow, npcol, my_prow, my_pcol;
+   int my_blacs_ctxt, my_prow, my_pcol;
 
    int mpierr;
 
@@ -151,7 +151,7 @@ int main(int argc, char** argv) {
 #else
    my_mpi_comm_world = 1;
 #endif
-   set_up_blacsgrid_f(my_mpi_comm_world, &my_blacs_ctxt, &np_rows, &np_cols, &nprow, &npcol, &my_prow, &my_pcol);
+   set_up_blacsgrid_f(my_mpi_comm_world, np_rows, np_cols, 'C', &my_blacs_ctxt, &my_prow, &my_pcol);
 
    if (myid == 0) {
      printf("\n");

test/C/elpa1/legacy_interface/legacy_real_1stage_c_version.c

@@ -68,7 +68,7 @@ int main(int argc, char** argv) {
 
    int np_cols, np_rows, np_colsStart;
 
-   int my_blacs_ctxt, nprow, npcol, my_prow, my_pcol;
+   int my_blacs_ctxt, my_prow, my_pcol;
 
    int mpierr;
 
@@ -143,7 +143,7 @@ int main(int argc, char** argv) {
 #ifdef WITH_MPI
    my_mpi_comm_world = MPI_Comm_c2f(MPI_COMM_WORLD);
 #endif
-   set_up_blacsgrid_f(my_mpi_comm_world, &my_blacs_ctxt, &np_rows, &np_cols, &nprow, &npcol, &my_prow, &my_pcol);
+   set_up_blacsgrid_f(my_mpi_comm_world, np_rows, np_cols, 'C', &my_blacs_ctxt, &my_prow, &my_pcol);
 
    if (myid == 0) {
      printf("\n");

test/C/elpa2/legacy_interface/legacy_complex_2stage_c_version.c

@@ -68,7 +68,7 @@ int main(int argc, char** argv) {
 
    int np_cols, np_rows, np_colsStart;
 
-   int my_blacs_ctxt, nprow, npcol, my_prow, my_pcol;
+   int my_blacs_ctxt, my_prow, my_pcol;
 
    int mpierr;
 
@@ -148,7 +148,7 @@ int main(int argc, char** argv) {
 #else
    my_mpi_comm_world = 1;
 #endif
-   set_up_blacsgrid_f(my_mpi_comm_world, &my_blacs_ctxt, &np_rows, &np_cols, &nprow, &npcol, &my_prow, &my_pcol);
+   set_up_blacsgrid_f(my_mpi_comm_world, np_rows, np_cols, 'C', &my_blacs_ctxt, &my_prow, &my_pcol);
 
    if (myid == 0) {
      printf("\n");

test/C/elpa2/legacy_interface/legacy_real_2stage_c_version.c

@@ -67,7 +67,7 @@ int main(int argc, char** argv) {
 
    int np_cols, np_rows, np_colsStart;
 
-   int my_blacs_ctxt, nprow, npcol, my_prow, my_pcol;
+   int my_blacs_ctxt, my_prow, my_pcol;
 
    int mpierr;
 
@@ -143,7 +143,7 @@ int main(int argc, char** argv) {
 #else
   my_mpi_comm_world = 1;
 #endif
-   set_up_blacsgrid_f(my_mpi_comm_world, &my_blacs_ctxt, &np_rows, &np_cols, &nprow, &npcol, &my_prow, &my_pcol);
+   set_up_blacsgrid_f(my_mpi_comm_world, np_rows, np_cols, 'C', &my_blacs_ctxt, &my_prow, &my_pcol);
 
    if (myid == 0) {
      printf("\n");

test/C/test.c

@@ -97,7 +97,7 @@ int main(int argc, char** argv) {
    int mpi_comm;
 
    /* blacs */
-   int my_blacs_ctxt, sc_desc[9], info, nprow, npcol;
+   int my_blacs_ctxt, sc_desc[9], info;
 
    /* The Matrix */
    MATRIX_TYPE *a, *as, *z;
@@ -136,7 +136,7 @@ int main(int argc, char** argv) {
 #else
    mpi_comm = 0;
 #endif
-   set_up_blacsgrid_f(mpi_comm, &my_blacs_ctxt, &np_rows, &np_cols, &nprow, &npcol, &my_prow, &my_pcol);
+   set_up_blacsgrid_f(mpi_comm, np_rows, np_cols, &my_blacs_ctxt, &my_prow, &my_pcol);
    set_up_blacs_descriptor_f(na, nblk, my_prow, my_pcol, np_rows, np_cols, &na_rows, &na_cols, sc_desc, my_blacs_ctxt, &info);
 
    /* allocate the matrices needed for elpa */

test/Fortran/driver/legacy_interface/legacy_complex_driver.F90

@@ -166,8 +166,8 @@ program test_complex2
    ! consistent (i.e. 0<=my_prow<np_rows, 0<=my_pcol<np_cols and every
    ! process has a unique (my_prow,my_pcol) pair).
 
-   call set_up_blacsgrid(mpi_comm_world, my_blacs_ctxt, np_rows, np_cols, &
-                         nprow, npcol, my_prow, my_pcol)
+   call set_up_blacsgrid(mpi_comm_world, np_rows, np_cols, 'C', &
+                         my_blacs_ctxt, my_prow, my_pcol)
 
    if (myid==0) then
      print '(a)','| Past BLACS_Gridinfo.'

test/Fortran/driver/legacy_interface/legacy_real_driver.F90

@@ -165,8 +165,8 @@ program test_real2
    ! consistent (i.e. 0<=my_prow<np_rows, 0<=my_pcol<np_cols and every
    ! process has a unique (my_prow,my_pcol) pair).
 
-   call set_up_blacsgrid(mpi_comm_world, my_blacs_ctxt, np_rows, np_cols, &
-                         nprow, npcol, my_prow, my_pcol)
+   call set_up_blacsgrid(mpi_comm_world, np_rows, np_cols, 'C', &
+                         my_blacs_ctxt, my_prow, my_pcol)
 
    if (myid==0) then
      print '(a)','| Past BLACS_Gridinfo.'

test/Fortran/driver/legacy_interface/legacy_single_complex_driver.F90

@@ -166,8 +166,8 @@ program test_complex2
    ! consistent (i.e. 0<=my_prow<np_rows, 0<=my_pcol<np_cols and every
    ! process has a unique (my_prow,my_pcol) pair).
 
-   call set_up_blacsgrid(mpi_comm_world, my_blacs_ctxt, np_rows, np_cols, &
-                         nprow, npcol, my_prow, my_pcol)
+   call set_up_blacsgrid(mpi_comm_world, np_rows, np_cols, 'C', &
+                         my_blacs_ctxt, my_prow, my_pcol)
 
    if (myid==0) then
      print '(a)','| Past BLACS_Gridinfo.'

test/Fortran/driver/legacy_interface/legacy_single_real_driver.F90

@@ -165,8 +165,8 @@ program test_real2
    ! consistent (i.e. 0<=my_prow<np_rows, 0<=my_pcol<np_cols and every
    ! process has a unique (my_prow,my_pcol) pair).
 
-   call set_up_blacsgrid(mpi_comm_world, my_blacs_ctxt, np_rows, np_cols, &
-                         nprow, npcol, my_prow, my_pcol)
+   call set_up_blacsgrid(mpi_comm_world, np_rows, np_cols, 'C', &
+                         my_blacs_ctxt, my_prow, my_pcol)
 
    if (myid==0) then
      print '(a)','| Past BLACS_Gridinfo.'

test/Fortran/elpa1/legacy_interface/legacy_complex.F90

@@ -174,8 +174,8 @@ program test_complex_double_precision
    ! consistent (i.e. 0<=my_prow<np_rows, 0<=my_pcol<np_cols and every
    ! process has a unique (my_prow,my_pcol) pair).
 
-   call set_up_blacsgrid(mpi_comm_world, my_blacs_ctxt, np_rows, np_cols, &
-                         nprow, npcol, my_prow, my_pcol)
+   call set_up_blacsgrid(mpi_comm_world, np_rows, np_cols, 'C', &
+                         my_blacs_ctxt, my_prow, my_pcol)
 
    if (myid==0) then
      print '(a)','| Past BLACS_Gridinfo.'

test/Fortran/elpa1/legacy_interface/legacy_complex_cholesky.F90

@@ -138,8 +138,8 @@ program test_cholesky
    ! consistent (i.e. 0<=my_prow<np_rows, 0<=my_pcol<np_cols and every
    ! process has a unique (my_prow,my_pcol) pair).
 
-   call set_up_blacsgrid(mpi_comm_world, my_blacs_ctxt, np_rows, np_cols, &
-                         nprow, npcol, my_prow, my_pcol)
+   call set_up_blacsgrid(mpi_comm_world, np_rows, np_cols, 'C', &
+                         my_blacs_ctxt, my_prow, my_pcol)
 
    if (myid==0) then
      print '(a)','| Past BLACS_Gridinfo.'

test/Fortran/elpa1/legacy_interface/legacy_complex_gpu.F90

@@ -173,8 +173,8 @@ program test_complex_gpu_version_double_precision
    ! consistent (i.e. 0<=my_prow<np_rows, 0<=my_pcol<np_cols and every
    ! process has a unique (my_prow,my_pcol) pair).
 
-   call set_up_blacsgrid(mpi_comm_world, my_blacs_ctxt, np_rows, np_cols, &
-                         nprow, npcol, my_prow, my_pcol)
+   call set_up_blacsgrid(mpi_comm_world, np_rows, np_cols, 'C', &
+                         my_blacs_ctxt, my_prow, my_pcol)
 
    if (myid==0) then
      print '(a)','| Past BLACS_Gridinfo.'

test/Fortran/elpa1/legacy_interface/legacy_complex_invert_trm.F90

@@ -138,8 +138,8 @@ program test_invert_trm
    ! consistent (i.e. 0<=my_prow<np_rows, 0<=my_pcol<np_cols and every
    ! process has a unique (my_prow,my_pcol) pair).
 
-   call set_up_blacsgrid(mpi_comm_world, my_blacs_ctxt, np_rows, np_cols, &
-                         nprow, npcol, my_prow, my_pcol)
+   call set_up_blacsgrid(mpi_comm_world, np_rows, np_cols, 'C', &
+                         my_blacs_ctxt, my_prow, my_pcol)
 
    if (myid==0) then
      print '(a)','| Past BLACS_Gridinfo.'

test/Fortran/elpa1/legacy_interface/legacy_complex_transp_multiply.F90

@@ -132,8 +132,8 @@ program test_transpose_multiply
    ! consistent (i.e. 0<=my_prow<np_rows, 0<=my_pcol<np_cols and every
    ! process has a unique (my_prow,my_pcol) pair).
 
-   call set_up_blacsgrid(mpi_comm_world, my_blacs_ctxt, np_rows, np_cols, &
-                         nprow, npcol, my_prow, my_pcol)
+   call set_up_blacsgrid(mpi_comm_world, np_rows, np_cols, 'C', &
+                         my_blacs_ctxt, my_prow, my_pcol)
 
    if (myid==0) then
      print '(a)','| Past BLACS_Gridinfo.'