Merge branch 'master' into ELPA_GPU

Makefile.am

@@ -182,6 +182,7 @@ shared_sources = test/shared_sources/util.F90 \
 		 test/shared_sources/check_correctnes.F90 \
 		 test/shared_sources/setup_mpi.F90 \
 		 test/shared_sources/blacs_infrastructure.F90 \
+		 test/shared_sources/mod_output_types.F90 \
 		 test/shared_sources/prepare_matrix.F90
 
 elpa1_test_real_c_version@SUFFIX@_SOURCES = test/c_test_programs/elpa1_test_real_c_version.c $(shared_sources) $(redirect_sources)

test/fortran_test_programs/elpa_test_programs_print_headers.X90

@@ -180,8 +180,12 @@
    endif
 #endif /* ELPA1 */
 
-   if (write_to_file) then
-     if (myid .eq. 0) print *,"Writing output files"
+   if (write_to_file%eigenvectors) then
+     if (myid .eq. 0) print *,"Writing Eigenvectors to files"
+   endif
+
+   if (write_to_file%eigenvalues) then
+     if (myid .eq. 0) print *,"Writing Eigenvalues to files"
    endif
 
 

test/fortran_test_programs/test_complex.F90

@@ -92,6 +92,7 @@ program test_complex
 #ifdef HAVE_DETAILED_TIMINGS
  use timings
 #endif
+  use output_types
    implicit none
 
    !-------------------------------------------------------------------------------
@@ -120,8 +121,10 @@ program test_complex
 #ifdef WITH_OPENMP
    integer(kind=ik)              :: omp_get_max_threads,  required_mpi_thread_level, provided_mpi_thread_level
 #endif
-   logical                       :: write_to_file
+   type(output_t)                :: write_to_file
    logical                       :: success
+   character(len=8)              :: task_suffix
+   integer(kind=ik)              :: j
 
    success = .true.
    ! read input parameters if they are provided
@@ -273,15 +276,30 @@ program test_complex
    if(myid == 0) print *,'Time trans_ev_complex :',time_evp_back
    if(myid == 0) print *,'Total time (sum above):',time_evp_back+time_evp_solve+time_evp_fwd
 
-   if(write_to_file) then
+   if(write_to_file%eigenvectors) then
+     write(unit = task_suffix, fmt = '(i8.8)') myid
+     open(17,file="EVs_complex_out_task_"//task_suffix(1:8)//".txt",form='formatted',status='new')
+     write(17,*) "Part of eigenvectors: na_rows=",na_rows,"of na=",na," na_cols=",na_cols," of na=",na
+
+     do i=1,na_rows
+       do j=1,na_cols
+         write(17,*) "row=",i," col=",j," element of eigenvector=",z(i,j)
+       enddo
+     enddo
+     close(17)
+   endif
+
+   if(write_to_file%eigenvalues) then
       if (myid == 0) then
-         open(17,file="EVs_complex_out.txt",form='formatted',status='new')
+         open(17,file="Eigenvalues_complex_out.txt",form='formatted',status='new')
          do i=1,na
             write(17,*) i,ev(i)
          enddo
          close(17)
       endif
    endif
+
+
    !-------------------------------------------------------------------------------
    ! Test correctness of result (using plain scalapack routines)
    allocate(tmp1(na_rows,na_cols))

test/fortran_test_programs/test_complex2.F90

@@ -100,6 +100,7 @@ program test_complex2
 #ifdef HAVE_DETAILED_TIMINGS
  use timings
 #endif
+ use output_types
    implicit none
 
    !-------------------------------------------------------------------------------
@@ -130,7 +131,10 @@ program test_complex2
 #ifdef WITH_OPENMP
    integer(kind=ik)              :: omp_get_max_threads,  required_mpi_thread_level, provided_mpi_thread_level
 #endif
-   logical                       :: write_to_file
+   type(output_t)                :: write_to_file
+   logical                       :: success
+   character(len=8)              :: task_suffix
+   integer(kind=ik)              :: j
 
    logical                       :: successELPA, success
 
@@ -282,15 +286,28 @@ program test_complex2
    if(myid == 0) print *,'Time transform back EVs :',time_evp_back
    if(myid == 0) print *,'Total time (sum above)  :',time_evp_back+time_evp_solve+time_evp_fwd
 
-   if(write_to_file) then
+   if(write_to_file%eigenvectors) then
+     write(unit = task_suffix, fmt = '(i8.8)') myid
+     open(17,file="EVs_complex2_out_task_"//task_suffix(1:8)//".txt",form='formatted',status='new')
+     write(17,*) "Part of eigenvectors: na_rows=",na_rows,"of na=",na," na_cols=",na_cols," of na=",na
+
+     do i=1,na_rows
+       do j=1,na_cols
+         write(17,*) "row=",i," col=",j," element of eigenvector=",z(i,j)
+       enddo
+     enddo
+     close(17)
+   endif
+   if(write_to_file%eigenvalues) then
       if (myid == 0) then
-         open(17,file="EVs_complex2_out.txt",form='formatted',status='new')
+         open(17,file="Eigenvalues_complex2_out.txt",form='formatted',status='new')
          do i=1,na
             write(17,*) i,ev(i)
          enddo
          close(17)
       endif
    endif
+
    !-------------------------------------------------------------------------------
    ! Test correctness of result (using plain scalapack routines)
    allocate(tmp1(na_rows,na_cols))

test/fortran_test_programs/test_complex2_choose_kernel_with_api.F90

@@ -102,6 +102,7 @@ program test_complex2
 #ifdef HAVE_DETAILED_TIMINGS
  use timings
 #endif
+ use output_types
 
    implicit none
 
@@ -133,10 +134,12 @@ program test_complex2
 #ifdef WITH_OPENMP
    integer(kind=ik)              :: omp_get_max_threads,  required_mpi_thread_level, provided_mpi_thread_level
 #endif
-   logical                       :: write_to_file
    logical                       :: successELPA, success
    integer(kind=ik)              :: numberOfDevices
    logical                       :: gpuAvailable
+   type(output_t)                :: write_to_file
+   character(len=8)              :: task_suffix
+   integer(kind=ik)              :: j
 
    successELPA   = .true.
    gpuAvailable  = .false.
@@ -180,10 +183,6 @@ program test_complex2
   call timer%start("program")
 #endif
 
-   if (write_to_file) then
-     if (myid .eq. 0) print *,"Writing output files"
-   endif
-
    !-------------------------------------------------------------------------------
    ! Selection of number of processor rows/columns
    ! We try to set up the grid square-like, i.e. start the search for possible
@@ -375,15 +374,30 @@ program test_complex2
    if(myid == 0) print *,'Time transform back EVs :',time_evp_back
    if(myid == 0) print *,'Total time (sum above)  :',time_evp_back+time_evp_solve+time_evp_fwd
 
-   if(write_to_file) then
+   if(write_to_file%eigenvectors) then
+     write(unit = task_suffix, fmt = '(i8.8)') myid
+     open(17,file="EVs_complex2_out_task_"//task_suffix(1:8)//".txt",form='formatted',status='new')
+     write(17,*) "Part of eigenvectors: na_rows=",na_rows,"of na=",na," na_cols=",na_cols," of na=",na
+
+     do i=1,na_rows
+       do j=1,na_cols
+         write(17,*) "row=",i," col=",j," element of eigenvector=",z(i,j)
+       enddo
+     enddo
+     close(17)
+   endif
+
+   if(write_to_file%eigenvalues) then
       if (myid == 0) then
-         open(17,file="EVs_complex2_out.txt",form='formatted',status='new')
+         open(17,file="Eigenvalues_complex2_out.txt",form='formatted',status='new')
          do i=1,na
             write(17,*) i,ev(i)
          enddo
          close(17)
       endif
    endif
+
+
    !-------------------------------------------------------------------------------
    ! Test correctness of result (using plain scalapack routines)
    allocate(tmp1(na_rows,na_cols))

test/fortran_test_programs/test_complex2_default_kernel.F90

@@ -102,6 +102,7 @@ program test_complex2
 #ifdef HAVE_DETAILED_TIMINGS
  use timings
 #endif
+ use output_types
    implicit none
 
    !-------------------------------------------------------------------------------
@@ -133,10 +134,14 @@ program test_complex2
 #ifdef WITH_OPENMP
    integer(kind=ik)              :: omp_get_max_threads,  required_mpi_thread_level, provided_mpi_thread_level
 #endif
-   logical                       :: write_to_file
    logical                       :: successELPA, success
    integer(kind=ik)              :: numberOfDevices
    logical                       :: gpuAvailable
+   type(output_t)                :: write_to_file
+   character(len=8)              :: task_suffix
+   integer(kind=ik)              :: j
+
+   success = .true.
 
    successELPA   = .true.
    gpuAvailable  = .false.
@@ -313,15 +318,30 @@ program test_complex2
    if(myid == 0) print *,'Time transform back EVs :',time_evp_back
    if(myid == 0) print *,'Total time (sum above)  :',time_evp_back+time_evp_solve+time_evp_fwd
 
-   if(write_to_file) then
+   if(write_to_file%eigenvectors) then
+     write(unit = task_suffix, fmt = '(i8.8)') myid
+     open(17,file="EVs_complex2_out_task_"//task_suffix(1:8)//".txt",form='formatted',status='new')
+     write(17,*) "Part of eigenvectors: na_rows=",na_rows,"of na=",na," na_cols=",na_cols," of na=",na
+
+     do i=1,na_rows
+       do j=1,na_cols
+         write(17,*) "row=",i," col=",j," element of eigenvector=",z(i,j)
+       enddo
+     enddo
+     close(17)
+   endif
+
+   if(write_to_file%eigenvalues) then
       if (myid == 0) then
-         open(17,file="EVs_complex2_out.txt",form='formatted',status='new')
+         open(17,file="Eigenvalues_complex2_out.txt",form='formatted',status='new')
          do i=1,na
             write(17,*) i,ev(i)
          enddo
          close(17)
       endif
    endif
+
+
    !-------------------------------------------------------------------------------
    ! Test correctness of result (using plain scalapack routines)
    allocate(tmp1(na_rows,na_cols))

test/fortran_test_programs/test_real.F90

@@ -93,6 +93,7 @@ program test_real
 #ifdef HAVE_DETAILED_TIMINGS
   use timings
 #endif
+  use output_types
 
    implicit none
 
@@ -121,11 +122,10 @@ program test_real
    integer(kind=ik)           :: omp_get_max_threads,  required_mpi_thread_level, &
                                  provided_mpi_thread_level
 #endif
-   logical                    :: write_to_file
+   type(output_t)             :: write_to_file
    logical                    :: success
-
-   integer :: j
-
+   character(len=8)           :: task_suffix
+   integer(kind=ik)           :: j
    !-------------------------------------------------------------------------------
 
    success = .true.
@@ -267,9 +267,23 @@ program test_real
    if(myid == 0) print *,'Time solve_tridi      :',time_evp_solve
    if(myid == 0) print *,'Time trans_ev_real    :',time_evp_back
    if(myid == 0) print *,'Total time (sum above):',time_evp_back+time_evp_solve+time_evp_fwd
-   if(write_to_file) then
+
+   if(write_to_file%eigenvectors) then
+     write(unit = task_suffix, fmt = '(i8.8)') myid
+     open(17,file="EVs_real_out_task_"//task_suffix(1:8)//".txt",form='formatted',status='new')
+     write(17,*) "Part of eigenvectors: na_rows=",na_rows,"of na=",na," na_cols=",na_cols," of na=",na
+
+     do i=1,na_rows
+       do j=1,na_cols
+         write(17,*) "row=",i," col=",j," element of eigenvector=",z(i,j)
+       enddo
+     enddo
+     close(17)
+   endif
+
+   if(write_to_file%eigenvalues) then
       if (myid == 0) then
-         open(17,file="EVs_real_out.txt",form='formatted',status='new')
+         open(17,file="Eigenvalues_real_out.txt",form='formatted',status='new')
          do i=1,na
             write(17,*) i,ev(i)
          enddo
@@ -277,6 +291,7 @@ program test_real
       endif
    endif
 
+
    !-------------------------------------------------------------------------------
    ! Test correctness of result (using plain scalapack routines)
    allocate(tmp1(na_rows,na_cols))

test/fortran_test_programs/test_real2.F90

@@ -100,6 +100,7 @@ program test_real2
 #ifdef HAVE_DETAILED_TIMINGS
  use timings
 #endif
+ use output_types
    implicit none
 
    !-------------------------------------------------------------------------------
@@ -126,10 +127,12 @@ program test_real2
 #ifdef WITH_OPENMP
    integer(kind=ik)           :: omp_get_max_threads,  required_mpi_thread_level, provided_mpi_thread_level
 #endif
-   logical                    :: write_to_file
    logical                    :: successELPA, success
    integer(kind=ik)           :: numberOfDevices
    logical                    :: gpuAvailable
+   type(output_t)             :: write_to_file
+   character(len=8)           :: task_suffix
+   integer(kind=ik)           :: j
 
    successELPA   = .true.
    gpuAvailable  = .false.
@@ -280,15 +283,30 @@ program test_real2
    if(myid == 0) print *,'Total time (sum above)  :',time_evp_back+time_evp_solve+time_evp_fwd
 
 
-   if(write_to_file) then
+   if(write_to_file%eigenvectors) then
+     write(unit = task_suffix, fmt = '(i8.8)') myid
+     open(17,file="EVs_real2_out_task_"//task_suffix(1:8)//".txt",form='formatted',status='new')
+     write(17,*) "Part of eigenvectors: na_rows=",na_rows,"of na=",na," na_cols=",na_cols," of na=",na
+
+     do i=1,na_rows
+       do j=1,na_cols
+         write(17,*) "row=",i," col=",j," element of eigenvector=",z(i,j)
+       enddo
+     enddo
+     close(17)
+   endif
+
+   if(write_to_file%eigenvalues) then
       if (myid == 0) then
-         open(17,file="EVs_real2_out.txt",form='formatted',status='new')
+         open(17,file="Eigenvalues_real2_out.txt",form='formatted',status='new')
          do i=1,na
             write(17,*) i,ev(i)
          enddo
          close(17)
       endif
    endif
+
+
    !-------------------------------------------------------------------------------
    ! Test correctness of result (using plain scalapack routines)
    allocate(tmp1(na_rows,na_cols))

test/fortran_test_programs/test_real2_choose_kernel_with_api.F90

@@ -103,7 +103,7 @@ program test_real2
 #ifdef HAVE_DETAILED_TIMINGS
  use timings
 #endif
-
+ use output_types
    implicit none
 
    !-------------------------------------------------------------------------------
@@ -130,10 +130,12 @@ program test_real2
 #ifdef WITH_OPENMP
    integer(kind=ik)           :: omp_get_max_threads,  required_mpi_thread_level, provided_mpi_thread_level
 #endif
-   logical                    :: write_to_file
    logical                    :: successELPA, success
    integer(kind=ik)           :: numberOfDevices
    logical                    :: gpuAvailable
+   type(output_t)             :: write_to_file
+   character(len=8)           :: task_suffix
+   integer(kind=ik)           :: j
 
    successELPA   = .true.
    gpuAvailable  = .false.
@@ -383,15 +385,29 @@ program test_real2
    if(myid == 0) print *,'Time transform back EVs :',time_evp_back
    if(myid == 0) print *,'Total time (sum above)  :',time_evp_back+time_evp_solve+time_evp_fwd
 
-   if(write_to_file) then
+   if(write_to_file%eigenvectors) then
+     write(unit = task_suffix, fmt = '(i8.8)') myid
+     open(17,file="EVs_real2_out_task_"//task_suffix(1:8)//".txt",form='formatted',status='new')
+     write(17,*) "Part of eigenvectors: na_rows=",na_rows,"of na=",na," na_cols=",na_cols," of na=",na
+
+     do i=1,na_rows
+       do j=1,na_cols
+         write(17,*) "row=",i," col=",j," element of eigenvector=",z(i,j)
+       enddo
+     enddo
+     close(17)
+   endif
+
+   if(write_to_file%eigenvalues) then
       if (myid == 0) then
-         open(17,file="EVs_real2_out.txt",form='formatted',status='new')
+         open(17,file="Eigenvalues_real2_out.txt",form='formatted',status='new')
          do i=1,na
             write(17,*) i,ev(i)
          enddo
          close(17)
       endif
    endif
+
    !-------------------------------------------------------------------------------
    ! Test correctness of result (using plain scalapack routines)
 

test/fortran_test_programs/test_real2_default_kernel.F90

@@ -102,6 +102,7 @@ program test_real2
 #ifdef HAVE_DETAILED_TIMINGS
  use timings
 #endif
+ use output_types
    implicit none
 
    !-------------------------------------------------------------------------------
@@ -128,10 +129,12 @@ program test_real2
 #ifdef WITH_OPENMP
    integer(kind=ik)           :: omp_get_max_threads,  required_mpi_thread_level, provided_mpi_thread_level
 #endif
-   logical                    :: write_to_file
    logical                    :: successELPA, success
    integer(kind=ik)           :: numberOfDevices
    logical                    :: gpuAvailable
+   type(output_t)             :: write_to_file
+   character(len=8)           :: task_suffix
+   integer(kind=ik)           :: j
 
    successELPA = .true.
    gpuAvailable  = .false.
@@ -307,15 +310,29 @@ program test_real2
    if(myid == 0) print *,'Time transform back EVs :',time_evp_back
    if(myid == 0) print *,'Total time (sum above)  :',time_evp_back+time_evp_solve+time_evp_fwd
 
-   if(write_to_file) then
+   if(write_to_file%eigenvectors) then
+      write(unit = task_suffix, fmt = '(i8.8)') myid
+     open(17,file="EVs_real2_out_task_"//task_suffix(1:8)//".txt",form='formatted',status='new')
+     write(17,*) "Part of eigenvectors: na_rows=",na_rows,"of na=",na," na_cols=",na_cols," of na=",na
+
+     do i=1,na_rows
+       do j=1,na_cols
+         write(17,*) "row=",i," col=",j," element of eigenvector=",z(i,j)
+       enddo
+     enddo
+     close(17)
+   endif
+   if(write_to_file%eigenvalues) then
       if (myid == 0) then
-         open(17,file="EVs_real2_out.txt",form='formatted',status='new')
+         open(17,file="Eigenvalues_real2_out.txt",form='formatted',status='new')
          do i=1,na
             write(17,*) i,ev(i)
          enddo
          close(17)
       endif
    endif
+
+
    !-------------------------------------------------------------------------------
    ! Test correctness of result (using plain scalapack routines)
    allocate(tmp1(na_rows,na_cols))

test/fortran_test_programs/test_real2_default_kernel_qr_decomposition.F90

@@ -103,6 +103,7 @@ program test_real2
 #ifdef HAVE_DETAILED_TIMINGS
  use timings
 #endif
+ use output_types
    implicit none
 
    !-------------------------------------------------------------------------------
@@ -129,16 +130,17 @@ program test_real2
 #ifdef WITH_OPENMP
    integer(kind=ik)           :: omp_get_max_threads,  required_mpi_thread_level, provided_mpi_thread_level
 #endif
-   logical                    :: write_to_file
    logical                    :: successELPA, success
    integer(kind=ik)           :: numberOfDevices
    logical                    :: gpuAvailable
+   type(output_t)             :: write_to_file
+   character(len=8)           :: task_suffix
+   integer(kind=ik)           :: j
 
 
    successELPA   = .true.
    gpuAvailable  = .false.
 
-   write_to_file = .false.
 
    if (COMMAND_ARGUMENT_COUNT() /= 0) then
      write(error_unit,*) "This program does not support any command-line arguments"
@@ -321,7 +323,20 @@ program test_real2
    if(myid == 0) print *,'Time transform back EVs :',time_evp_back
    if(myid == 0) print *,'Total time (sum above)  :',time_evp_back+time_evp_solve+time_evp_fwd
 
-   if(write_to_file) then
+   if(write_to_file%eigenvectors) then
+     write(unit = task_suffix, fmt = '(i8.8)') myid
+     open(17,file="EVs_real2_out_task_"//task_suffix(1:8)//".txt",form='formatted',status='new')
+     write(17,*) "Part of eigenvectors: na_rows=",na_rows,"of na=",na," na_cols=",na_cols," of na=",na
+
+     do i=1,na_rows
+       do j=1,na_cols
+         write(17,*) "row=",i," col=",j," element of eigenvector=",z(i,j)
+       enddo
+     enddo
+     close(17)
+   endif
+
+   if(write_to_file%eigenvalues) then
       if (myid == 0) then
          open(17,file="EVs_real2_out.txt",form='formatted',status='new')
          do i=1,na
@@ -330,6 +345,8 @@ program test_real2
          close(17)
       endif
    endif
+
+
    !-------------------------------------------------------------------------------
    ! Test correctness of result (using plain scalapack routines)
    allocate(tmp1(na_rows,na_cols))

test/fortran_test_programs/test_real_with_c.F90

@@ -93,7 +93,7 @@ program test_real
 #ifdef HAVE_DETAILED_TIMINGS
  use timings
 #endif
-
+ use output_types
    implicit none
 
    !-------------------------------------------------------------------------------
@@ -125,7 +125,7 @@ program test_real
    integer(kind=ik)           :: omp_get_max_threads,  required_mpi_thread_level, &
                                  provided_mpi_thread_level
 #endif
-   logical                    :: write_to_file
+   type(output_t)             :: write_to_file
 
    integer(kind=ik)           :: checksWrong, checksWrongRecv
    logical                    :: success
@@ -138,10 +138,6 @@ program test_real
    !  MPI Initialization
    call setup_mpi(myid, nprocs)
 
-   if (write_to_file) then
-     if (myid .eq. 0) print *,"Writing output files"
-   endif
-
 #ifdef HAVE_DETAILED_TIMINGS
 
    ! initialise the timing functionality
@@ -328,15 +324,6 @@ program test_real
    if(myid == 0) print *,'Time solve_tridi      :',time_evp_solve
    if(myid == 0) print *,'Time trans_ev_real    :',time_evp_back
    if(myid == 0) print *,'Total time (sum above):',time_evp_back+time_evp_solve+time_evp_fwd
-   if(write_to_file) then
-      if (myid == 0) then
-         open(17,file="EVs_real_out.txt",form='formatted',status='new')
-         do i=1,na
-            write(17,*) i,ev(i)
-         enddo
-         close(17)
-      endif
-   endif
 
    ! call the c function
 #ifdef WITH_MPI

test/shared_sources/mod_output_types.F90

+#include "config-f90.h"
+
+module output_types
+
+ type :: output_t
+   logical :: eigenvectors
+   logical :: eigenvalues
+ end type
+
+
+end module

test/shared_sources/read_input_parameters.F90

@@ -49,24 +49,28 @@ module mod_read_input_parameters
       use ELPA_utilities, only : error_unit
       use precision
       use elpa_mpi
+      use output_types
       implicit none
 
       integer(kind=ik), intent(out) :: na, nev, nblk
-      logical, intent(out)          :: write_to_file
+
+      type(output_t), intent(out)   :: write_to_file
 
       ! Command line arguments
-      character(len=128)            :: arg1, arg2, arg3, arg4
+      character(len=128)            :: arg1, arg2, arg3, arg4, arg5
       integer(kind=ik)              :: mpierr
 
       ! default parameters