check_for_gpu.F90 6.33 KB
Newer Older
Andreas Marek's avatar
Andreas Marek committed
1
2
!    Copyright 2014, A. Marek
!
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
!    This file is part of ELPA.
!
!    The ELPA library was originally created by the ELPA consortium,
!    consisting of the following organizations:
!
!    - Rechenzentrum Garching der Max-Planck-Gesellschaft (RZG),
!    - Bergische Universität Wuppertal, Lehrstuhl für angewandte
!      Informatik,
!    - Technische Universität München, Lehrstuhl für Informatik mit
!      Schwerpunkt Wissenschaftliches Rechnen ,
!    - Fritz-Haber-Institut, Berlin, Abt. Theorie,
!    - Max-Plack-Institut für Mathematik in den Naturwissenschaften,
!      Leipzig, Abt. Komplexe Strukutren in Biologie und Kognition,
!      and
!    - IBM Deutschland GmbH
!
!
!    More information can be found here:
Andreas Marek's avatar
Andreas Marek committed
21
!    http://elpa.mpcdf.mpg.de/
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
!
!    ELPA is free software: you can redistribute it and/or modify
!    it under the terms of the version 3 of the license of the
!    GNU Lesser General Public License as published by the Free
!    Software Foundation.
!
!    ELPA is distributed in the hope that it will be useful,
!    but WITHOUT ANY WARRANTY; without even the implied warranty of
!    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
!    GNU Lesser General Public License for more details.
!
!    You should have received a copy of the GNU Lesser General Public License
!    along with ELPA.  If not, see <http://www.gnu.org/licenses/>
!
!    ELPA reflects a substantial effort on the part of the original
!    ELPA consortium, and we ask you to respect the spirit of the
!    license that we chose: i.e., please contribute any changes you
!    may have back to the original ELPA library distribution, and keep
!    any derivatives of ELPA under the same license that we chose for
!    the original distribution, the GNU Lesser General Public License.
!
43
44
! This file was written by A. Marek, MPCDF

45
46
47
48
49
50
#include "config-f90.h"

module mod_check_for_gpu

  contains

51
    function check_for_gpu(obj, myid, numberOfDevices, wantDebug) result(gpuAvailable)
52
      use cuda_functions
53
      use precision
54
      use elpa_mpi
55
      use elpa_abstract_impl
56
      implicit none
57

58
      class(elpa_abstract_impl_t), intent(inout)                         :: obj
59
      integer(kind=ik), intent(in)  :: myid
60
61
      logical, optional, intent(in) :: wantDebug
      logical                       :: success, wantDebugMessage
62
63
64
      integer(kind=ik), intent(out) :: numberOfDevices
      integer(kind=ik)              :: deviceNumber, mpierr, maxNumberOfDevices
      logical                       :: gpuAvailable
65
      integer(kind=ik)              :: error, mpi_comm_all, use_gpu_id, min_use_gpu_id
Andreas Marek's avatar
Andreas Marek committed
66
      !character(len=1024)           :: envname
67

Wenzhe Yu's avatar
Wenzhe Yu committed
68
69
70
71
72
73
74
75
76
77
      if (.not.(present(wantDebug))) then
        wantDebugMessage = .false.
      else
        if (wantDebug) then
          wantDebugMessage=.true.
        else
          wantDebugMessage=.false.
        endif
      endif

78
79
      gpuAvailable = .false.

80
81
82
83
84
85
      call obj%get("mpi_comm_parent",mpi_comm_all,error)
      if (error .ne. ELPA_OK) then
        print *,"Problem getting option for mpi_comm_parent. Aborting..."
        stop
      endif

86
87
88
89
      if (obj%is_set("use_gpu_id") == 1) then
        call obj%get("use_gpu_id", use_gpu_id, error)
        if (use_gpu_id == -99) then
          print *,"Problem you did not set which gpu id this task should use"
Andreas Marek's avatar
Andreas Marek committed
90
        endif
91
92
 
        ! check whether gpu ud has been set for each proces
93
#ifdef WITH_MPI
94
        call mpi_allreduce(use_gpu_id, min_use_gpu_id, 1, MPI_INTEGER, MPI_MAX, mpi_comm_all, mpierr)
95

96
97
98
99
100
101
        if (min_use_gpu_id .lt. 0) then
          print *,"Not all tasks have set which GPU id should be used"
          print *,"GPUs will NOT be used!"
          gpuAvailable = .false.
          return
        endif
102
#endif
103
104
105
        gpuAvailable = .true.

        if (myid==0) then
106
107
108
109
          if (wantDebugMessage) then
            print *
            print '(3(a,i0))','Found ', numberOfDevices, ' GPUs'
          endif
110
111
        endif

112
        success = cuda_setdevice(use_gpu_id)
113
114
115

        if (.not.(success)) then
          print *,"Cannot set CudaDevice"
Andreas Marek's avatar
Andreas Marek committed
116
          stop 1
117
        endif
118
119
120
        if (wantDebugMessage) then
          print '(3(a,i0))', 'MPI rank ', myid, ' uses GPU #', deviceNumber
        endif
121
          
Pavel Kus's avatar
Pavel Kus committed
122
123
124
125
126
        success = cublas_create(cublasHandle)
        if (.not.(success)) then
          print *,"Cannot create cublas handle"
          stop 1
        endif
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
      else

        if (cublasHandle .ne. -1) then
          gpuAvailable = .true.
          numberOfDevices = -1
          if (myid == 0 .and. wantDebugMessage) then
            print *, "Skipping GPU init, should have already been initialized "
          endif
          return
        else
          if (myid == 0 .and. wantDebugMessage) then
            print *, "Initializing the GPU devices"
          endif
        endif

        ! call getenv("CUDA_PROXY_PIPE_DIRECTORY", envname)
        success = cuda_getdevicecount(numberOfDevices)

        if (.not.(success)) then
          print *,"error in cuda_getdevicecount"
          stop 1
        endif

        ! make sure that all nodes have the same number of GPU's, otherwise
        ! we run into loadbalancing trouble
#ifdef WITH_MPI
        call mpi_allreduce(numberOfDevices, maxNumberOfDevices, 1, MPI_INTEGER, MPI_MAX, mpi_comm_all, mpierr)

        if (maxNumberOfDevices .ne. numberOfDevices) then
          print *,"Different number of GPU devices on MPI tasks!"
          print *,"GPUs will NOT be used!"
          gpuAvailable = .false.
          return
        endif
#endif
        if (numberOfDevices .ne. 0) then
          gpuAvailable = .true.
          ! Usage of GPU is possible since devices have been detected

          if (myid==0) then
            if (wantDebugMessage) then
              print *
              print '(3(a,i0))','Found ', numberOfDevices, ' GPUs'
            endif
          endif

          deviceNumber = mod(myid, numberOfDevices)
          success = cuda_setdevice(deviceNumber)

          if (.not.(success)) then
            print *,"Cannot set CudaDevice"
            stop 1
          endif
          if (wantDebugMessage) then
            print '(3(a,i0))', 'MPI rank ', myid, ' uses GPU #', deviceNumber
          endif
          
          success = cublas_create(cublasHandle)
          if (.not.(success)) then
            print *,"Cannot create cublas handle"
            stop 1
          endif
          
        endif
191

192
      endif  
193
194
    end function
end module