Merge branch 'likwid' into 'master_pre_stage'

Likwid

See merge request elpa/elpa!16

configure.ac

@@ -308,6 +308,51 @@ if test x"${enable_timings}" = x"yes"; then
 fi
 AC_LANG_POP([C])
 
+dnl Likwid
+AC_LANG_PUSH([Fortran])
+AC_ARG_WITH([likwid],
+            [AS_HELP_STRING([--with-likwid=[[yes|no(default)|PATH]]],
+                            [Use Likwid to measure performance in some parts of the library])],
+            [with_likwid="$withval"],
+            [with_likwid="no"])
+
+if test x"$with_likwid" != x"no" ; then
+  if test -d $with_likwid/lib ; then
+    LDFLAGS="-L$with_likwid/lib $LDFLAGS"
+  fi
+  if test -d $with_likwid/lib64 ; then
+    LDFLAGS="-L$with_likwid/lib64 $LDFLAGS"
+  fi
+  if test -d $with_likwid/include ; then
+    FCFLAGS="-I$with_likwid/include $FCFLAGS"
+  fi
+  AC_SEARCH_LIBS([likwid_markerInit], [likwid], [liblikwid_found="yes"], [liblikwid_found="no"])
+  if test x"$liblikwid_found" = x"no" ; then
+    AC_MSG_ERROR([Could not find a usable likwid library, please adjust LDFLAGS])
+  fi
+  AC_MSG_CHECKING([whether we can use the likwid module in a Fortran program])
+  AC_COMPILE_IFELSE([
+       program foo
+       use likwid
+
+       implicit none
+
+       call likwid_markerInit()
+       call likwid_markerThreadInit()
+
+       call likwid_markerStartRegion("foobar")
+       call likwid_markerStopRegion("foobar")
+
+       call likwid_markerClose()
+       end
+    ],
+    [AC_MSG_RESULT([yes])],
+    [AC_MSG_RESULT([no])
+     AC_MSG_ERROR([Could not compile a Fortran program using the likwid module, adjust FCFLAGS])])
+  AC_DEFINE([HAVE_LIKWID], [1], [Use likwid])
+fi
+AC_LANG_POP([Fortran])
+
 save_FCFLAGS=$FCFLAGS
 save_LDFLAGS=$LDFLAGS
 
@@ -530,7 +575,7 @@ dnl Test possibility of 'use mpi', if requested
 if test x"${with_mpi}" = x"yes" ; then
   AC_ARG_ENABLE([mpi-module],
                 AS_HELP_STRING([--disable-mpi-module],
-                               [do not use the Fortran MPI module, get interfaces by 'include "mpif.h')]),
+                               [do not use the Fortran MPI module, get interfaces by 'include "mpif.h']),
                 [
                  if test x"$enableval" = x"yes"; then
                    enable_mpi_module=yes
@@ -995,7 +1040,7 @@ if test x"${need_avx512}" = x"yes"; then
 
   if test x"$can_compile_avx512" = x"yes"; then
     AC_MSG_CHECKING([whether we compile for Xeon])
-    AC_RUN_IFELSE([AC_LANG_SOURCE([
+    AC_COMPILE_IFELSE([AC_LANG_SOURCE([
      #include <x86intrin.h>
      int main(int argc, char **argv){
      __m512d sign;
@@ -1011,7 +1056,7 @@ if test x"${need_avx512}" = x"yes"; then
     AC_MSG_RESULT([${can_compile_avx512_xeon}])
 
     AC_MSG_CHECKING([whether we compile for Xeon PHI])
-    AC_RUN_IFELSE([AC_LANG_SOURCE([
+    AC_COMPILE_IFELSE([AC_LANG_SOURCE([
      #include <x86intrin.h>
      int main(int argc, char **argv){
      __m512d sign;

src/elpa1/elpa1.F90

@@ -85,6 +85,9 @@ module elpa1_impl
   use, intrinsic :: iso_c_binding
   use elpa_utilities
   use elpa1_auxiliary_impl
+#ifdef HAVE_LIKWID
+  use likwid
+#endif
 
   implicit none
 

src/elpa1/elpa1_template.F90

@@ -324,16 +324,28 @@ function elpa_solve_evp_&
 
    if (do_tridiag) then
      call obj%timer%start("forward")
+#ifdef HAVE_LIKWID
+     call likwid_markerStartRegion("tridi")
+#endif
+
      call tridiag_&
      &MATH_DATATYPE&
      &_&
      &PRECISION&
      & (obj, na, a, lda, nblk, matrixCols, mpi_comm_rows, mpi_comm_cols, ev, e, tau, do_useGPU_tridiag, wantDebug, nrThreads)
+
+#ifdef HAVE_LIKWID
+     call likwid_markerStopRegion("tridi")
+#endif
      call obj%timer%stop("forward")
     endif  !do_tridiag
 
     if (do_solve) then
      call obj%timer%start("solve")
+#ifdef HAVE_LIKWID
+     call likwid_markerStartRegion("solve")
+#endif
+
      call solve_tridi_&
      &PRECISION&
      & (obj, na, nev, ev, e,  &
@@ -344,6 +356,10 @@ function elpa_solve_evp_&
         q_real, l_rows,  &
 #endif
         nblk, matrixCols, mpi_comm_rows, mpi_comm_cols, do_useGPU_solve_tridi, wantDebug, success, nrThreads)
+
+#ifdef HAVE_LIKWID
+     call likwid_markerStopRegion("solve")
+#endif
      call obj%timer%stop("solve")
      if (.not.(success)) return
    endif !do_solve
@@ -379,11 +395,19 @@ function elpa_solve_evp_&
 #endif
 
      call obj%timer%start("back")
+#ifdef HAVE_LIKWID
+     call likwid_markerStartRegion("trans_ev")
+#endif
+
      call trans_ev_&
      &MATH_DATATYPE&
      &_&
      &PRECISION&
      & (obj, na, nev, a, lda, tau, q, ldq, nblk, matrixCols, mpi_comm_rows, mpi_comm_cols, do_useGPU_trans_ev)
+
+#ifdef HAVE_LIKWID
+     call likwid_markerStopRegion("trans_ev")
+#endif
      call obj%timer%stop("back")
    endif ! do_trans_ev
 

src/elpa2/elpa2.F90

@@ -56,6 +56,9 @@
 !> \brief Fortran module which provides the routines to use the 2-stage ELPA solver. Implementation only. Should not be used directly
 module elpa2_impl
   use elpa_utilities, only : error_unit
+#ifdef HAVE_LIKWID
+  use likwid
+#endif
 
   implicit none
 

src/elpa2/elpa2_template.F90

@@ -549,6 +549,9 @@
 
     if (do_bandred) then
       call obj%timer%start("bandred")
+#ifdef HAVE_LIKWID
+      call likwid_markerStartRegion("bandred")
+#endif
       ! Reduction full -> band
       call bandred_&
       &MATH_DATATYPE&
@@ -561,6 +564,9 @@
       useQRActual, &
 #endif
        nrThreads)
+#ifdef HAVE_LIKWID
+      call likwid_markerStopRegion("bandred")
+#endif
       call obj%timer%stop("bandred")
       if (.not.(success)) return
     endif
@@ -578,6 +584,9 @@
        endif
 
        call obj%timer%start("tridiag")
+#ifdef HAVE_LIKWID
+       call likwid_markerStartRegion("tridiag")
+#endif
        call tridiag_band_&
        &MATH_DATATYPE&
        &_&
@@ -591,6 +600,9 @@
        call mpi_bcast(e, na, MPI_REAL_PRECISION, 0, mpi_comm_all, mpierr)
        call obj%timer%stop("mpi_communication")
 #endif /* WITH_MPI */
+#ifdef HAVE_LIKWID
+       call likwid_markerStopRegion("tridiag")
+#endif
        call obj%timer%stop("tridiag")
      endif ! do_tridiag
 
@@ -611,6 +623,9 @@
      ! Solve tridiagonal system
      if (do_solve_tridi) then
        call obj%timer%start("solve")
+#ifdef HAVE_LIKWID
+       call likwid_markerStartRegion("solve")
+#endif
        call solve_tridi_&
        &PRECISION &
        (obj, na, nev, ev, e, &
@@ -621,6 +636,9 @@
        q_real, ubound(q_real,dim=1), &
 #endif
        nblk, matrixCols, mpi_comm_rows, mpi_comm_cols, do_useGPU_solve_tridi, wantDebug, success, nrThreads)
+#ifdef HAVE_LIKWID
+       call likwid_markerStopRegion("solve")
+#endif
        call obj%timer%stop("solve")
        if (.not.(success)) return
      endif ! do_solve_tridi
@@ -678,7 +696,9 @@
 
        ! Backtransform stage 1
        call obj%timer%start("trans_ev_to_band")
-
+#ifdef HAVE_LIKWID
+       call likwid_markerStartRegion("trans_ev_to_band")
+#endif
        call trans_ev_tridi_to_band_&
        &MATH_DATATYPE&
        &_&
@@ -687,6 +707,9 @@
        q_dev, &
        ldq, matrixCols, hh_trans, mpi_comm_rows, mpi_comm_cols, wantDebug, do_useGPU_trans_ev_tridi_to_band, &
        nrThreads, success=success, kernel=kernel)
+#ifdef HAVE_LIKWID
+       call likwid_markerStopRegion("trans_ev_to_band")
+#endif
        call obj%timer%stop("trans_ev_to_band")
 
        if (.not.(success)) return
@@ -730,6 +753,9 @@
 
      if (do_trans_to_full) then
        call obj%timer%start("trans_ev_to_full")
+#ifdef HAVE_LIKWID
+       call likwid_markerStartRegion("trans_ev_to_full")
+#endif
        if ( (do_useGPU_trans_ev_band_to_full) .and. .not.(do_useGPU_trans_ev_tridi_to_band) ) then
          ! copy to device if we want to continue on GPU
          successCUDA = cuda_malloc(q_dev, ldq*matrixCols*size_of_datatype)
@@ -765,6 +791,9 @@
          &PRECISION " // ": error when deallocating tmat"//errorMessage
          stop 1
        endif
+#ifdef HAVE_LIKWID
+       call likwid_markerStopRegion("trans_ev_to_full")
+#endif
        call obj%timer%stop("trans_ev_to_full")
      endif ! do_trans_to_full
 

src/elpa_impl.F90

@@ -56,6 +56,9 @@ module elpa_impl
   use elpa_mpi
   use elpa_generated_fortran_interfaces
   use elpa_utilities, only : error_unit
+#ifdef HAVE_LIKWID
+  use likwid
+#endif
 
   use elpa_abstract_impl
 #ifdef ENABLE_AUTOTUNING
@@ -530,6 +533,13 @@ module elpa_impl
 
 #endif
 
+#ifdef HAVE_LIKWID
+      !initialize likwid
+      call likwid_markerInit()
+      call likwid_markerThreadInit()
+      call likwid_markerStartRegion("TOTAL")
+#endif
+
 #ifdef HAVE_DETAILED_TIMINGS
       call self%get("timings",timings, error)
       call self%get("measure_performance",performance, error)
@@ -1018,6 +1028,11 @@ module elpa_impl
         error = ELPA_OK
       endif
 
+#ifdef HAVE_LIKWID
+      call likwid_markerStopRegion("TOTAL")
+      call likwid_markerClose()
+#endif
+
 #ifdef WITH_MPI
       if (self%communicators_owned == 1) then
         call self%get("mpi_comm_rows", mpi_comm_rows, error2)