test_multiple_objs.F90 12.6 KB
Newer Older
Pavel Kus's avatar
Pavel Kus committed
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
!    This file is part of ELPA.
!
!    The ELPA library was originally created by the ELPA consortium,
!    consisting of the following organizations:
!
!    - Max Planck Computing and Data Facility (MPCDF), formerly known as
!      Rechenzentrum Garching der Max-Planck-Gesellschaft (RZG),
!    - Bergische Universität Wuppertal, Lehrstuhl für angewandte
!      Informatik,
!    - Technische Universität München, Lehrstuhl für Informatik mit
!      Schwerpunkt Wissenschaftliches Rechnen ,
!    - Fritz-Haber-Institut, Berlin, Abt. Theorie,
!    - Max-Plack-Institut für Mathematik in den Naturwissenschaften,
!      Leipzig, Abt. Komplexe Strukutren in Biologie und Kognition,
!      and
!    - IBM Deutschland GmbH
!
!
!    More information can be found here:
!    http://elpa.mpcdf.mpg.de/
!
!    ELPA is free software: you can redistribute it and/or modify
!    it under the terms of the version 3 of the license of the
!    GNU Lesser General Public License as published by the Free
!    Software Foundation.
!
!    ELPA is distributed in the hope that it will be useful,
!    but WITHOUT ANY WARRANTY; without even the implied warranty of
!    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
!    GNU Lesser General Public License for more details.
!
!    You should have received a copy of the GNU Lesser General Public License
!    along with ELPA.  If not, see <http://www.gnu.org/licenses/>
!
!    ELPA reflects a substantial effort on the part of the original
!    ELPA consortium, and we ask you to respect the spirit of the
!    license that we chose: i.e., please contribute any changes you
!    may have back to the original ELPA library distribution, and keep
!    any derivatives of ELPA under the same license that we chose for
!    the original distribution, the GNU Lesser General Public License.
!
!
#include "config-f90.h"

! Define one of TEST_REAL or TEST_COMPLEX
! Define one of TEST_SINGLE or TEST_DOUBLE
! Define one of TEST_SOLVER_1STAGE or TEST_SOLVER_2STAGE
48
49
! Define TEST_NVIDIA_GPU \in [0, 1]
! Define TEST_AMD_GPU \in [0, 1]
Pavel Kus's avatar
Pavel Kus committed
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
! Define either TEST_ALL_KERNELS or a TEST_KERNEL \in [any valid kernel]

#if !(defined(TEST_REAL) ^ defined(TEST_COMPLEX))
error: define exactly one of TEST_REAL or TEST_COMPLEX
#endif

#if !(defined(TEST_SINGLE) ^ defined(TEST_DOUBLE))
error: define exactly one of TEST_SINGLE or TEST_DOUBLE
#endif

#ifdef TEST_SINGLE
#  define EV_TYPE real(kind=C_FLOAT)
#  ifdef TEST_REAL
#    define MATRIX_TYPE real(kind=C_FLOAT)
#  else
#    define MATRIX_TYPE complex(kind=C_FLOAT_COMPLEX)
#  endif
#else
#  define EV_TYPE real(kind=C_DOUBLE)
#  ifdef TEST_REAL
#    define MATRIX_TYPE real(kind=C_DOUBLE)
#  else
#    define MATRIX_TYPE complex(kind=C_DOUBLE_COMPLEX)
#  endif
#endif


#ifdef TEST_REAL
#  define AUTOTUNE_DOMAIN ELPA_AUTOTUNE_DOMAIN_REAL
#else
#  define AUTOTUNE_DOMAIN ELPA_AUTOTUNE_DOMAIN_COMPLEX
#endif

83
#ifdef HAVE_64BIT_INTEGER_MATH_SUPPORT
Andreas Marek's avatar
Andreas Marek committed
84
85
86
87
88
89
90
#define TEST_INT_TYPE integer(kind=c_int64_t)
#define INT_TYPE c_int64_t
#else
#define TEST_INT_TYPE integer(kind=c_int32_t)
#define INT_TYPE c_int32_t
#endif

91
92
93
94
95
96
97
98
#ifdef HAVE_64BIT_INTEGER_MPI_SUPPORT
#define TEST_INT_MPI_TYPE integer(kind=c_int64_t)
#define INT_MPI_TYPE c_int64_t
#else
#define TEST_INT_MPI_TYPE integer(kind=c_int32_t)
#define INT_MPI_TYPE c_int32_t
#endif

99
100
101
102
103
#define TEST_GPU 0
#if (TEST_NVIDIA_GPU == 1) || (TEST_AMD_GPU == 1)
#undef TEST_GPU
#define TEST_GPU 1
#endif
104

Pavel Kus's avatar
Pavel Kus committed
105
106
107
108
109
#include "assert.h"

program test
   use elpa

Andreas Marek's avatar
Andreas Marek committed
110
   !use test_util
Pavel Kus's avatar
Pavel Kus committed
111
112
113
114
115
116
117
118
119
120
121
122
123
124
   use test_setup_mpi
   use test_prepare_matrix
   use test_read_input_parameters
   use test_blacs_infrastructure
   use test_check_correctness
   use test_analytic
   use iso_fortran_env

#ifdef HAVE_REDIRECT
   use test_redirect
#endif
   implicit none

   ! matrix dimensions
Andreas Marek's avatar
Andreas Marek committed
125
   TEST_INT_TYPE                     :: na, nev, nblk
Pavel Kus's avatar
Pavel Kus committed
126
127

   ! mpi
Andreas Marek's avatar
Andreas Marek committed
128
129
130
131
   TEST_INT_TYPE                     :: myid, nprocs
   TEST_INT_TYPE                     :: na_cols, na_rows  ! local matrix size
   TEST_INT_TYPE                     :: np_cols, np_rows  ! number of MPI processes per column/row
   TEST_INT_TYPE                     :: my_prow, my_pcol  ! local MPI task position (my_prow, my_pcol) in the grid (0..np_cols -1, 0..np_rows -1)
132
133
   TEST_INT_TYPE                     :: ierr
   TEST_INT_MPI_TYPE                 :: mpierr
Pavel Kus's avatar
Pavel Kus committed
134
   ! blacs
135
   character(len=1)                  :: layout
Andreas Marek's avatar
Andreas Marek committed
136
   TEST_INT_TYPE                     :: my_blacs_ctxt, sc_desc(9), info, nprow, npcol
Pavel Kus's avatar
Pavel Kus committed
137
138
139
140
141
142
143
144

   ! The Matrix
   MATRIX_TYPE, allocatable    :: a(:,:), as(:,:)
   ! eigenvectors
   MATRIX_TYPE, allocatable    :: z(:,:)
   ! eigenvalues
   EV_TYPE, allocatable        :: ev(:)

Andreas Marek's avatar
Andreas Marek committed
145
146
   TEST_INT_TYPE               :: status
   integer(kind=c_int)         :: error_elpa
Pavel Kus's avatar
Pavel Kus committed
147
148
149
150
151

   type(output_t)              :: write_to_file
   class(elpa_t), pointer      :: e1, e2, e_ptr
   class(elpa_autotune_t), pointer :: tune_state

Andreas Marek's avatar
Andreas Marek committed
152
   TEST_INT_TYPE                     :: iter
Pavel Kus's avatar
Pavel Kus committed
153
   character(len=5)            :: iter_string
Andreas Marek's avatar
Andreas Marek committed
154
   TEST_INT_TYPE                     :: timings, debug, gpu
Pavel Kus's avatar
Pavel Kus committed
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188

   call read_input_parameters(na, nev, nblk, write_to_file)
   call setup_mpi(myid, nprocs)
#ifdef HAVE_REDIRECT
#ifdef WITH_MPI
   call MPI_BARRIER(MPI_COMM_WORLD, mpierr)
   call redirect_stdout(myid)
#endif
#endif

   if (elpa_init(CURRENT_API_VERSION) /= ELPA_OK) then
     print *, "ELPA API version not supported"
     stop 1
   endif

   layout = 'C'
   do np_cols = NINT(SQRT(REAL(nprocs))),2,-1
      if(mod(nprocs,np_cols) == 0 ) exit
   enddo
   np_rows = nprocs/np_cols
   assert(nprocs == np_rows * np_cols)

   if (myid == 0) then
     print '((a,i0))', 'Matrix size: ', na
     print '((a,i0))', 'Num eigenvectors: ', nev
     print '((a,i0))', 'Blocksize: ', nblk
#ifdef WITH_MPI
     print '((a,i0))', 'Num MPI proc: ', nprocs
     print '(3(a,i0))','Number of processor rows=',np_rows,', cols=',np_cols,', total=',nprocs
     print '(a)',      'Process layout: ' // layout
#endif
     print *,''
   endif

189
   call set_up_blacsgrid(int(mpi_comm_world,kind=BLAS_KIND), np_rows, np_cols, layout, &
Pavel Kus's avatar
Pavel Kus committed
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
                         my_blacs_ctxt, my_prow, my_pcol)

   call set_up_blacs_descriptor(na, nblk, my_prow, my_pcol, np_rows, np_cols, &
                                na_rows, na_cols, sc_desc, my_blacs_ctxt, info)

   allocate(a (na_rows,na_cols))
   allocate(as(na_rows,na_cols))
   allocate(z (na_rows,na_cols))
   allocate(ev(na))

   a(:,:) = 0.0
   z(:,:) = 0.0
   ev(:) = 0.0

   call prepare_matrix_analytic(na, a, nblk, myid, np_rows, np_cols, my_prow, my_pcol, print_times=.false.)
   as(:,:) = a(:,:)

Andreas Marek's avatar
Andreas Marek committed
207
208
   e1 => elpa_allocate(error_elpa)
   !assert_elpa_ok(error_elpa)
209

Pavel Kus's avatar
Pavel Kus committed
210
211
   call set_basic_params(e1, na, nev, na_rows, na_cols, my_prow, my_pcol)

Andreas Marek's avatar
Andreas Marek committed
212
213
   call e1%set("timings",1, error_elpa)
   assert_elpa_ok(error_elpa)
214

Andreas Marek's avatar
Andreas Marek committed
215
216
   call e1%set("debug",1, error_elpa)
   assert_elpa_ok(error_elpa)
Pavel Kus's avatar
Pavel Kus committed
217

218
219
220
221
222
223
224
#if TEST_NVIDIA_GPU == 1 || (TEST_NVIDIA_GPU == 0) && (TEST_AMD_GPU == 0)
   call e1%set("nvidia-gpu", TEST_GPU, error_elpa)
   assert_elpa_ok(error_elpa)
#endif

#if TEST_AMD_GPU == 1
   call e1%set("amd-gpu", TEST_GPU, error_elpa)
Andreas Marek's avatar
Andreas Marek committed
225
   assert_elpa_ok(error_elpa)
226
#endif
Andreas Marek's avatar
Andreas Marek committed
227
   !call e1%set("max_stored_rows", 15, error_elpa)
Pavel Kus's avatar
Pavel Kus committed
228
229
230

   assert_elpa_ok(e1%setup())

Andreas Marek's avatar
Andreas Marek committed
231
232
   call e1%store_settings("initial_parameters.txt", error_elpa)
   assert_elpa_ok(error_elpa)
Pavel Kus's avatar
Pavel Kus committed
233

234
235
#ifdef WITH_MPI
     ! barrier after store settings, file created from one MPI rank only, but loaded everywhere
236
     call MPI_BARRIER(MPI_COMM_WORLD, mpierr)
237
238
#endif

Pavel Kus's avatar
Pavel Kus committed
239
   ! try to load parameters into another object
Andreas Marek's avatar
Andreas Marek committed
240
241
   e2 => elpa_allocate(error_elpa)
   assert_elpa_ok(error_elpa)
242

Pavel Kus's avatar
Pavel Kus committed
243
   call set_basic_params(e2, na, nev, na_rows, na_cols, my_prow, my_pcol)
Andreas Marek's avatar
Andreas Marek committed
244
245
   call e2%load_settings("initial_parameters.txt", error_elpa)
   assert_elpa_ok(error_elpa)
246

Pavel Kus's avatar
Pavel Kus committed
247
248
   assert_elpa_ok(e2%setup())

249
   ! test whether the user setting of e1 are correctly loade to e2
Andreas Marek's avatar
Andreas Marek committed
250
251
252
253
   call e2%get("timings", int(timings,kind=c_int), error_elpa)
   assert_elpa_ok(error_elpa)
   call e2%get("debug", int(debug,kind=c_int), error_elpa)
   assert_elpa_ok(error_elpa)
254
#if TEST_NVIDIA_GPU == 1 || (TEST_NVIDIA_GPU == 0) && (TEST_AMD_GPU == 0)
255
   call e2%get("nvidia-gpu", int(gpu,kind=c_int), error_elpa)
Andreas Marek's avatar
Andreas Marek committed
256
   assert_elpa_ok(error_elpa)
257
258
259
260
261
#endif
#if TEST_NVIDIA_AMD == 1
   call e2%get("amd-gpu", int(gpu,kind=c_int), error_elpa)
   assert_elpa_ok(error_elpa)
#endif
262
263
264
265
266
267

   if ((timings .ne. 1) .or. (debug .ne. 1) .or. (gpu .ne. 0)) then
     print *, "Parameters not stored or loaded correctly. Aborting...", timings, debug, gpu
     stop 1
   endif

Pavel Kus's avatar
Pavel Kus committed
268
   if(myid == 0) print *, "parameters of e1"
Andreas Marek's avatar
Andreas Marek committed
269
270
   call e1%print_settings(error_elpa)
   assert_elpa_ok(error_elpa)
271

Pavel Kus's avatar
Pavel Kus committed
272
273
   if(myid == 0) print *, ""
   if(myid == 0) print *, "parameters of e2"
Andreas Marek's avatar
Andreas Marek committed
274
275
   call e2%print_settings(error_elpa)
   assert_elpa_ok(error_elpa)
276

Pavel Kus's avatar
Pavel Kus committed
277
278
279
   e_ptr => e2


Andreas Marek's avatar
Andreas Marek committed
280
281
   tune_state => e_ptr%autotune_setup(ELPA_AUTOTUNE_FAST, AUTOTUNE_DOMAIN, error_elpa)
   assert_elpa_ok(error_elpa)
Pavel Kus's avatar
Pavel Kus committed
282

283

Pavel Kus's avatar
Pavel Kus committed
284
   iter=0
Andreas Marek's avatar
Andreas Marek committed
285
286
   do while (e_ptr%autotune_step(tune_state, error_elpa))
     assert_elpa_ok(error_elpa)
287
 
Pavel Kus's avatar
Pavel Kus committed
288
289
     iter=iter+1
     write(iter_string,'(I5.5)') iter
Andreas Marek's avatar
Andreas Marek committed
290
291
     call e_ptr%print_settings(error_elpa)
     assert_elpa_ok(error_elpa)
292

Andreas Marek's avatar
Andreas Marek committed
293
294
     call e_ptr%store_settings("saved_parameters_"//trim(iter_string)//".txt", error_elpa)
     assert_elpa_ok(error_elpa)
295

Pavel Kus's avatar
Pavel Kus committed
296
     call e_ptr%timer_start("eigenvectors: iteration "//trim(iter_string))
Andreas Marek's avatar
Andreas Marek committed
297
298
     call e_ptr%eigenvectors(a, ev, z, error_elpa)
     assert_elpa_ok(error_elpa)
Pavel Kus's avatar
Pavel Kus committed
299
300
     call e_ptr%timer_stop("eigenvectors: iteration "//trim(iter_string))

Andreas Marek's avatar
Andreas Marek committed
301
     assert_elpa_ok(error_elpa)
Pavel Kus's avatar
Pavel Kus committed
302
303
304
305
306
307
308
     if (myid .eq. 0) then
       print *, ""
       call e_ptr%print_times("eigenvectors: iteration "//trim(iter_string))
     endif
     status = check_correctness_analytic(na, nev, ev, z, nblk, myid, np_rows, np_cols, my_prow, my_pcol, &
                                         .true., .true., print_times=.false.)
     a(:,:) = as(:,:)
Andreas Marek's avatar
Andreas Marek committed
309
310
     call e_ptr%autotune_print_state(tune_state, error_elpa)
     assert_elpa_ok(error_elpa)
311

Andreas Marek's avatar
Andreas Marek committed
312
313
     call e_ptr%autotune_save_state(tune_state, "saved_state_"//trim(iter_string)//".txt", error_elpa)
     assert_elpa_ok(error_elpa)
314
#ifdef WITH_MPI
315
     ! barrier after save state, file created from one MPI rank only, but loaded everywhere
316
     call MPI_BARRIER(MPI_COMM_WORLD, mpierr)
317
#endif
Andreas Marek's avatar
Andreas Marek committed
318
319
     call e_ptr%autotune_load_state(tune_state, "saved_state_"//trim(iter_string)//".txt", error_elpa)
     assert_elpa_ok(error_elpa)
320

Pavel Kus's avatar
Pavel Kus committed
321
322
323
   end do

   ! set and print the autotuned-settings
Andreas Marek's avatar
Andreas Marek committed
324
325
   call e_ptr%autotune_set_best(tune_state, error_elpa)
   assert_elpa_ok(error_elpa)
326

Pavel Kus's avatar
Pavel Kus committed
327
328
329
   if (myid .eq. 0) then
     print *, "The best combination found by the autotuning:"
     flush(output_unit)
Andreas Marek's avatar
Andreas Marek committed
330
331
     call e_ptr%autotune_print_best(tune_state, error_elpa)
     assert_elpa_ok(error_elpa)
Pavel Kus's avatar
Pavel Kus committed
332
333
   endif
   ! de-allocate autotune object
Andreas Marek's avatar
Andreas Marek committed
334
335
   call elpa_autotune_deallocate(tune_state, error_elpa)
   assert_elpa_ok(error_elpa)
Pavel Kus's avatar
Pavel Kus committed
336
337
338
339
340

   if (myid .eq. 0) then
     print *, "Running once more time with the best found setting..."
   endif
   call e_ptr%timer_start("eigenvectors: best setting")
Andreas Marek's avatar
Andreas Marek committed
341
342
   call e_ptr%eigenvectors(a, ev, z, error_elpa)
   assert_elpa_ok(error_elpa)
343

Pavel Kus's avatar
Pavel Kus committed
344
   call e_ptr%timer_stop("eigenvectors: best setting")
Andreas Marek's avatar
Andreas Marek committed
345
   assert_elpa_ok(error_elpa)
Pavel Kus's avatar
Pavel Kus committed
346
347
348
349
350
351
352
   if (myid .eq. 0) then
     print *, ""
     call e_ptr%print_times("eigenvectors: best setting")
   endif
   status = check_correctness_analytic(na, nev, ev, z, nblk, myid, np_rows, np_cols, my_prow, my_pcol, &
                                       .true., .true., print_times=.false.)

Andreas Marek's avatar
Andreas Marek committed
353
354
   call elpa_deallocate(e_ptr, error_elpa)
   !assert_elpa_ok(error_elpa)
Pavel Kus's avatar
Pavel Kus committed
355
356
357
358
359
360

   deallocate(a)
   deallocate(as)
   deallocate(z)
   deallocate(ev)

Andreas Marek's avatar
Andreas Marek committed
361
362
   call elpa_uninit(error_elpa)
   !assert_elpa_ok(error_elpa)
Pavel Kus's avatar
Pavel Kus committed
363
364
365
366
367
368
369
370
371
372
373
374

#ifdef WITH_MPI
   call blacs_gridexit(my_blacs_ctxt)
   call mpi_finalize(mpierr)
#endif

   call exit(status)

contains
   subroutine set_basic_params(elpa, na, nev, na_rows, na_cols, my_prow, my_pcol)
     implicit none
     class(elpa_t), pointer      :: elpa
Andreas Marek's avatar
Andreas Marek committed
375
376
377
378
379
380
381
382
383
384
385
386
     TEST_INT_TYPE, intent(in)   :: na, nev, na_rows, na_cols, my_prow, my_pcol

     call elpa%set("na", int(na,kind=c_int), error_elpa)
     assert_elpa_ok(error_elpa)
     call elpa%set("nev", int(nev,kind=c_int), error_elpa)
     assert_elpa_ok(error_elpa)
     call elpa%set("local_nrows", int(na_rows,kind=c_int), error_elpa)
     assert_elpa_ok(error_elpa)
     call elpa%set("local_ncols", int(na_cols,kind=c_int), error_elpa)
     assert_elpa_ok(error_elpa)
     call elpa%set("nblk", int(nblk,kind=c_int), error_elpa)
     assert_elpa_ok(error_elpa)
Pavel Kus's avatar
Pavel Kus committed
387
388

#ifdef WITH_MPI
Andreas Marek's avatar
Andreas Marek committed
389
390
391
392
393
394
     call elpa%set("mpi_comm_parent", int(MPI_COMM_WORLD,kind=c_int), error_elpa)
     assert_elpa_ok(error_elpa)
     call elpa%set("process_row", int(my_prow,kind=c_int), error_elpa)
     assert_elpa_ok(error_elpa)
     call elpa%set("process_col", int(my_pcol,kind=c_int), error_elpa)
     assert_elpa_ok(error_elpa)
Pavel Kus's avatar
Pavel Kus committed
395
396
397
398
#endif
   end subroutine

end program