check_for_gpu.F90 4.42 KB
Newer Older
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40
!    This file is part of ELPA.
!
!    The ELPA library was originally created by the ELPA consortium,
!    consisting of the following organizations:
!
!    - Rechenzentrum Garching der Max-Planck-Gesellschaft (RZG),
!    - Bergische Universität Wuppertal, Lehrstuhl für angewandte
!      Informatik,
!    - Technische Universität München, Lehrstuhl für Informatik mit
!      Schwerpunkt Wissenschaftliches Rechnen ,
!    - Fritz-Haber-Institut, Berlin, Abt. Theorie,
!    - Max-Plack-Institut für Mathematik in den Naturwissenschaften,
!      Leipzig, Abt. Komplexe Strukutren in Biologie und Kognition,
!      and
!    - IBM Deutschland GmbH
!
!
!    More information can be found here:
!    http://elpa.rzg.mpg.de/
!
!    ELPA is free software: you can redistribute it and/or modify
!    it under the terms of the version 3 of the license of the
!    GNU Lesser General Public License as published by the Free
!    Software Foundation.
!
!    ELPA is distributed in the hope that it will be useful,
!    but WITHOUT ANY WARRANTY; without even the implied warranty of
!    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
!    GNU Lesser General Public License for more details.
!
!    You should have received a copy of the GNU Lesser General Public License
!    along with ELPA.  If not, see <http://www.gnu.org/licenses/>
!
!    ELPA reflects a substantial effort on the part of the original
!    ELPA consortium, and we ask you to respect the spirit of the
!    license that we chose: i.e., please contribute any changes you
!    may have back to the original ELPA library distribution, and keep
!    any derivatives of ELPA under the same license that we chose for
!    the original distribution, the GNU Lesser General Public License.
!
41 42
! This file was written by A. Marek, MPCDF

43 44 45 46 47 48
#include "config-f90.h"

module mod_check_for_gpu

  contains

49
    function check_for_gpu(myid, numberOfDevices, wantDebug) result(gpuAvailable)
50
      use cuda_functions
51
      use precision
52
      use elpa_mpi
53
      implicit none
54

55
      integer(kind=ik), intent(in)  :: myid
56 57
      logical, optional, intent(in) :: wantDebug
      logical                       :: success, wantDebugMessage
58 59 60
      integer(kind=ik), intent(out) :: numberOfDevices
      integer(kind=ik)              :: deviceNumber, mpierr, maxNumberOfDevices
      logical                       :: gpuAvailable
Andreas Marek's avatar
Andreas Marek committed
61
      !character(len=1024)           :: envname
62 63 64

      gpuAvailable = .false.

65 66 67 68 69 70 71 72 73 74 75 76 77
      if(cublasHandle .ne. -1) then
	gpuAvailable = .true.
	numberOfDevices = -1
	if(myid == 0) then
          print *, "Skipping GPU init, should have already been initialized "
	endif
	return
      else
	if(myid == 0) then
	  print *, "Initializing the GPU devices"
	endif
      endif

78 79 80 81 82 83 84 85 86 87
      if (.not.(present(wantDebug))) then
        wantDebugMessage = .false.
      else
        if (wantDebug) then
          wantDebugMessage=.true.
        else
          wantDebugMessage=.false.
        endif
      endif

88 89 90 91 92
      ! call getenv("CUDA_PROXY_PIPE_DIRECTORY", envname)
      success = cuda_getdevicecount(numberOfDevices)

      if (.not.(success)) then
        print *,"error in cuda_getdevicecount"
Andreas Marek's avatar
Andreas Marek committed
93
        stop 1
94 95 96 97
      endif

      ! make sure that all nodes have the same number of GPU's, otherwise
      ! we run into loadbalancing trouble
98
#ifdef WITH_MPI
99 100 101 102 103 104 105 106
      call mpi_allreduce(numberOfDevices, maxNumberOfDevices, 1, MPI_INTEGER, MPI_MAX, MPI_COMM_WORLD, mpierr)

      if (maxNumberOfDevices .ne. numberOfDevices) then
        print *,"Different number of GPU devices on MPI tasks!"
        print *,"GPUs will NOT be used!"
        gpuAvailable = .false.
        return
      endif
107
#endif
108 109 110 111 112
      if (numberOfDevices .ne. 0) then
        gpuAvailable = .true.
        ! Usage of GPU is possible since devices have been detected

        if (myid==0) then
113 114 115 116
          if (wantDebugMessage) then
            print *
            print '(3(a,i0))','Found ', numberOfDevices, ' GPUs'
          endif
117 118 119 120 121 122 123
        endif

        deviceNumber = mod(myid, numberOfDevices)
        success = cuda_setdevice(deviceNumber)

        if (.not.(success)) then
          print *,"Cannot set CudaDevice"
Andreas Marek's avatar
Andreas Marek committed
124
          stop 1
125
        endif
126 127 128
        if (wantDebugMessage) then
          print '(3(a,i0))', 'MPI rank ', myid, ' uses GPU #', deviceNumber
        endif
Pavel Kus's avatar
Pavel Kus committed
129 130 131 132 133 134 135
        
        success = cublas_create(cublasHandle)
        if (.not.(success)) then
          print *,"Cannot create cublas handle"
          stop 1
        endif
        
136 137 138 139
      endif

    end function
end module