check_for_gpu.F90 7.91 KB
Newer Older
Andreas Marek's avatar
Andreas Marek committed
1
2
!    Copyright 2014, A. Marek
!
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
!    This file is part of ELPA.
!
!    The ELPA library was originally created by the ELPA consortium,
!    consisting of the following organizations:
!
!    - Rechenzentrum Garching der Max-Planck-Gesellschaft (RZG),
!    - Bergische Universität Wuppertal, Lehrstuhl für angewandte
!      Informatik,
!    - Technische Universität München, Lehrstuhl für Informatik mit
!      Schwerpunkt Wissenschaftliches Rechnen ,
!    - Fritz-Haber-Institut, Berlin, Abt. Theorie,
!    - Max-Plack-Institut für Mathematik in den Naturwissenschaften,
!      Leipzig, Abt. Komplexe Strukutren in Biologie und Kognition,
!      and
!    - IBM Deutschland GmbH
!
!
!    More information can be found here:
Andreas Marek's avatar
Andreas Marek committed
21
!    http://elpa.mpcdf.mpg.de/
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
!
!    ELPA is free software: you can redistribute it and/or modify
!    it under the terms of the version 3 of the license of the
!    GNU Lesser General Public License as published by the Free
!    Software Foundation.
!
!    ELPA is distributed in the hope that it will be useful,
!    but WITHOUT ANY WARRANTY; without even the implied warranty of
!    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
!    GNU Lesser General Public License for more details.
!
!    You should have received a copy of the GNU Lesser General Public License
!    along with ELPA.  If not, see <http://www.gnu.org/licenses/>
!
!    ELPA reflects a substantial effort on the part of the original
!    ELPA consortium, and we ask you to respect the spirit of the
!    license that we chose: i.e., please contribute any changes you
!    may have back to the original ELPA library distribution, and keep
!    any derivatives of ELPA under the same license that we chose for
!    the original distribution, the GNU Lesser General Public License.
!
43
44
! This file was written by A. Marek, MPCDF

45
46
47
48
49
50
#include "config-f90.h"

module mod_check_for_gpu

  contains

51
    function check_for_gpu(obj, myid, numberOfDevices, wantDebug) result(gpuAvailable)
52
      use cuda_functions
53
      use hip_functions
54
      use precision
55
      use elpa_mpi
56
      use elpa_abstract_impl
57
      implicit none
58

59
      class(elpa_abstract_impl_t), intent(inout)                         :: obj
60
      integer(kind=ik), intent(in)  :: myid
61
62
      logical, optional, intent(in) :: wantDebug
      logical                       :: success, wantDebugMessage
63
64
65
      integer(kind=ik), intent(out) :: numberOfDevices
      integer(kind=ik)              :: deviceNumber, mpierr, maxNumberOfDevices
      logical                       :: gpuAvailable
66
      integer(kind=ik)              :: error, mpi_comm_all, use_gpu_id, min_use_gpu_id
Andreas Marek's avatar
Andreas Marek committed
67
      !character(len=1024)           :: envname
68

Wenzhe Yu's avatar
Wenzhe Yu committed
69
70
71
72
73
74
75
76
77
78
      if (.not.(present(wantDebug))) then
        wantDebugMessage = .false.
      else
        if (wantDebug) then
          wantDebugMessage=.true.
        else
          wantDebugMessage=.false.
        endif
      endif

79
80
      gpuAvailable = .false.

81
82
83
84
85
86
      call obj%get("mpi_comm_parent",mpi_comm_all,error)
      if (error .ne. ELPA_OK) then
        print *,"Problem getting option for mpi_comm_parent. Aborting..."
        stop
      endif

87
88
89
90
      if (obj%is_set("use_gpu_id") == 1) then
        call obj%get("use_gpu_id", use_gpu_id, error)
        if (use_gpu_id == -99) then
          print *,"Problem you did not set which gpu id this task should use"
Andreas Marek's avatar
Andreas Marek committed
91
        endif
92
93
 
        ! check whether gpu ud has been set for each proces
94
#ifdef WITH_MPI
95
        call mpi_allreduce(use_gpu_id, min_use_gpu_id, 1, MPI_INTEGER, MPI_MAX, mpi_comm_all, mpierr)
96

97
98
99
100
101
102
        if (min_use_gpu_id .lt. 0) then
          print *,"Not all tasks have set which GPU id should be used"
          print *,"GPUs will NOT be used!"
          gpuAvailable = .false.
          return
        endif
103
#endif
104
105
106
        gpuAvailable = .true.

        if (myid==0) then
107
108
109
110
          if (wantDebugMessage) then
            print *
            print '(3(a,i0))','Found ', numberOfDevices, ' GPUs'
          endif
111
112
        endif

113
        success = .true.
114
#ifdef WITH_NVIDIA_GPU_VERSION
115
        success = cuda_setdevice(use_gpu_id)
116
117
118
119
#endif
#ifdef WITH_AMD_GPU_VERSION
        success = hip_setdevice(use_gpu_id)
#endif
120
        if (.not.(success)) then
121
#ifdef WITH_NVIDIA_GPU_VERSION
122
          print *,"Cannot set CudaDevice"
123
124
125
126
#endif
#ifdef WITH_AMD_GPU_VERSION
          print *,"Cannot set HIPDevice"
#endif
Andreas Marek's avatar
Andreas Marek committed
127
          stop 1
128
        endif
129
130
131
        if (wantDebugMessage) then
          print '(3(a,i0))', 'MPI rank ', myid, ' uses GPU #', deviceNumber
        endif
132
133
 
        success = .true.        
134
#ifdef WITH_NVIDIA_GPU_VERSION
Pavel Kus's avatar
Pavel Kus committed
135
        success = cublas_create(cublasHandle)
136
137
138
139
#endif
#ifdef WITH_AMD_GPU_VERSION
        success = rocblas_create(cublasHandle)
#endif
Pavel Kus's avatar
Pavel Kus committed
140
        if (.not.(success)) then
141
#ifdef WITH_NVIDIA_GPU_VERSION
Pavel Kus's avatar
Pavel Kus committed
142
          print *,"Cannot create cublas handle"
143
144
145
146
#endif
#ifdef WITH_AMD_GPU_VERSION
          print *,"Cannot create rocblas handle"
#endif
Pavel Kus's avatar
Pavel Kus committed
147
148
          stop 1
        endif
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
      else

        if (cublasHandle .ne. -1) then
          gpuAvailable = .true.
          numberOfDevices = -1
          if (myid == 0 .and. wantDebugMessage) then
            print *, "Skipping GPU init, should have already been initialized "
          endif
          return
        else
          if (myid == 0 .and. wantDebugMessage) then
            print *, "Initializing the GPU devices"
          endif
        endif

164
        success = .true.
165
#ifdef WITH_NVIDIA_GPU_VERSION
166
167
        ! call getenv("CUDA_PROXY_PIPE_DIRECTORY", envname)
        success = cuda_getdevicecount(numberOfDevices)
168
169
170
171
172
#endif
#ifdef WITH_AMD_GPU_VERSION
        ! call getenv("CUDA_PROXY_PIPE_DIRECTORY", envname)
        success = hip_getdevicecount(numberOfDevices)
#endif
173
        if (.not.(success)) then
174
#ifdef WITH_NVIDIA_GPU_VERSION
175
          print *,"error in cuda_getdevicecount"
176
177
178
179
#endif
#ifdef WITH_AMPD_GPU_VERSION
          print *,"error in hip_getdevicecount"
#endif
180
181
          stop 1
        endif
182
183
184
#ifdef  WITH_INTEL_GPU_VERSION
      gpuAvailable = .false.
      numberOfDevices = -1
185

186
187
188
189
190
191
      numberOfDevices = 1
      print *,"Manually setting",numberOfDevices," of GPUs"
      if (numberOfDevices .ge. 1) then
        gpuAvailable = .true.
      endif
#endif
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
        ! make sure that all nodes have the same number of GPU's, otherwise
        ! we run into loadbalancing trouble
#ifdef WITH_MPI
        call mpi_allreduce(numberOfDevices, maxNumberOfDevices, 1, MPI_INTEGER, MPI_MAX, mpi_comm_all, mpierr)

        if (maxNumberOfDevices .ne. numberOfDevices) then
          print *,"Different number of GPU devices on MPI tasks!"
          print *,"GPUs will NOT be used!"
          gpuAvailable = .false.
          return
        endif
#endif
        if (numberOfDevices .ne. 0) then
          gpuAvailable = .true.
          ! Usage of GPU is possible since devices have been detected

          if (myid==0) then
            if (wantDebugMessage) then
              print *
              print '(3(a,i0))','Found ', numberOfDevices, ' GPUs'
            endif
          endif

          deviceNumber = mod(myid, numberOfDevices)
216
#ifdef WITH_NIVDIA_GPU_VERSION
217
          success = cuda_setdevice(deviceNumber)
218
219
220
221
#endif
#ifdef WITH_AMD_GPU_VERSION
          success = hip_setdevice(deviceNumber)
#endif
222
223

          if (.not.(success)) then
224
#ifdef WITH_NIVDIA_GPU_VERSION
225
            print *,"Cannot set CudaDevice"
226
227
228
229
#endif
#ifdef WITH_AMD_GPU_VERSION
            print *,"Cannot set hipDevice"
#endif
230
231
232
233
234
235
            stop 1
          endif
          if (wantDebugMessage) then
            print '(3(a,i0))', 'MPI rank ', myid, ' uses GPU #', deviceNumber
          endif
          
236
#ifdef WITH_NIVDIA_GPU_VERSION
237
          success = cublas_create(cublasHandle)
238
239
240
241
#endif
#ifdef WITH_AMD_GPU_VERSION
          success = rocblas_create(cublasHandle)
#endif
242
          if (.not.(success)) then
243
#ifdef WITH_NIVDIA_GPU_VERSION
244
            print *,"Cannot create cublas handle"
245
246
247
248
#endif
#ifdef WITH_AMD_GPU_VERSION
            print *,"Cannot create rocblas handle"
#endif
249
250
251
252
            stop 1
          endif
          
        endif
253

254
      endif  
255
256
    end function
end module