_code.py 12.4 KB
Newer Older
Chichi Lalescu's avatar
Chichi Lalescu committed
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
#######################################################################
#                                                                     #
#  Copyright 2015 Max Planck Institute                                #
#                 for Dynamics and Self-Organization                  #
#                                                                     #
#  This file is part of bfps.                                         #
#                                                                     #
#  bfps is free software: you can redistribute it and/or modify       #
#  it under the terms of the GNU General Public License as published  #
#  by the Free Software Foundation, either version 3 of the License,  #
#  or (at your option) any later version.                             #
#                                                                     #
#  bfps is distributed in the hope that it will be useful,            #
#  but WITHOUT ANY WARRANTY; without even the implied warranty of     #
#  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the      #
#  GNU General Public License for more details.                       #
#                                                                     #
#  You should have received a copy of the GNU General Public License  #
#  along with bfps.  If not, see <http://www.gnu.org/licenses/>       #
#                                                                     #
# Contact: Cristian.Lalescu@ds.mpg.de                                 #
#                                                                     #
#######################################################################
Cristian Lalescu's avatar
Cristian Lalescu committed
24

25
26


27
import os
Cristian Lalescu's avatar
Cristian Lalescu committed
28
import sys
29
import shutil
Cristian Lalescu's avatar
Cristian Lalescu committed
30
import subprocess
31
import argparse
Cristian Lalescu's avatar
Cristian Lalescu committed
32
import h5py
33
from datetime import datetime
Chichi Lalescu's avatar
Chichi Lalescu committed
34
import math
35

Cristian Lalescu's avatar
Cristian Lalescu committed
36
import bfps
37
from ._base import _base
38

39
class _code(_base):
40
41
    """This class is meant to stitch together the C++ code into a final source file,
    compile it, and handle all job launching.
42
    """
43
44
45
46
    def __init__(
            self,
            work_dir = './',
            simname = 'test'):
47
        _base.__init__(self, work_dir = work_dir, simname = simname)
48
49
50
51
        self.version_message = ('/***********************************************************************\n' +
                                '* this code automatically generated by bfps\n' +
                                '* version {0}\n'.format(bfps.__version__) +
                                '***********************************************************************/\n\n\n')
Cristian Lalescu's avatar
broken    
Cristian Lalescu committed
52
        self.includes = """
Chichi Lalescu's avatar
Chichi Lalescu committed
53
                //begincpp
54
55
56
                #include "base.hpp"
                #include "fluid_solver.hpp"
                #include <iostream>
57
                #include <hdf5.h>
Chichi Lalescu's avatar
Chichi Lalescu committed
58
                #include <string>
59
                #include <cstring>
60
                #include <fftw3-mpi.h>
Chichi Lalescu's avatar
Chichi Lalescu committed
61
62
                //endcpp
                """
Cristian Lalescu's avatar
broken    
Cristian Lalescu committed
63
        self.variables = 'int myrank, nprocs;\n'
Chichi Lalescu's avatar
Chichi Lalescu committed
64
        self.variables += 'int iteration;\n'
65
        self.variables += 'char simname[256], fname[256];\n'
Cristian Lalescu's avatar
Cristian Lalescu committed
66
        self.variables += ('hid_t parameter_file, stat_file, Cdset;\n')
Cristian Lalescu's avatar
broken    
Cristian Lalescu committed
67
        self.definitions = ''
68
        self.main_start = """
Chichi Lalescu's avatar
Chichi Lalescu committed
69
                //begincpp
70
71
72
73
74
                int main(int argc, char *argv[])
                {
                    MPI_Init(&argc, &argv);
                    MPI_Comm_rank(MPI_COMM_WORLD, &myrank);
                    MPI_Comm_size(MPI_COMM_WORLD, &nprocs);
Chichi Lalescu's avatar
Chichi Lalescu committed
75
76
                    fftw_mpi_init();
                    fftwf_mpi_init();
Chichi Lalescu's avatar
Chichi Lalescu committed
77
                    if (argc != 2)
Cristian Lalescu's avatar
broken    
Cristian Lalescu committed
78
79
80
81
82
83
84
85
                    {
                        std::cerr << "Wrong number of command line arguments. Stopping." << std::endl;
                        MPI_Finalize();
                        return EXIT_SUCCESS;
                    }
                    else
                    {
                        strcpy(simname, argv[1]);
86
87
88
89
                        sprintf(fname, "%s.h5", simname);
                        parameter_file = H5Fopen(fname, H5F_ACC_RDONLY, H5P_DEFAULT);
                        Cdset = H5Dopen(parameter_file, "iteration", H5P_DEFAULT);
                        H5Dread(Cdset, H5T_NATIVE_INT, H5S_ALL, H5S_ALL, H5P_DEFAULT, &iteration);
Chichi Lalescu's avatar
Chichi Lalescu committed
90
                        DEBUG_MSG("simname is %s and iteration is %d\\n", simname, iteration);
91
                        H5Dclose(Cdset);
Cristian Lalescu's avatar
broken    
Cristian Lalescu committed
92
                    }
93
94
                    read_parameters(parameter_file);
                    H5Fclose(parameter_file);
95
                    if (myrank == 0)
Cristian Lalescu's avatar
Cristian Lalescu committed
96
                        stat_file = H5Fopen(fname, H5F_ACC_RDWR, H5P_DEFAULT);
Chichi Lalescu's avatar
Chichi Lalescu committed
97
98
                //endcpp
                """
Chichi Lalescu's avatar
Chichi Lalescu committed
99
100
        for ostream in ['cout', 'cerr']:
            self.main_start += 'if (myrank == 0) std::{1} << "{0}" << std::endl;'.format(self.version_message, ostream).replace('\n', '\\n') + '\n'
101
        self.main_end = """
Chichi Lalescu's avatar
Chichi Lalescu committed
102
                //begincpp
103
                    // clean up
Chichi Lalescu's avatar
Chichi Lalescu committed
104
105
                    if (myrank == 0)
                    {
Cristian Lalescu's avatar
Cristian Lalescu committed
106
                        Cdset = H5Dopen(stat_file, "iteration", H5P_DEFAULT);
Cristian Lalescu's avatar
Cristian Lalescu committed
107
108
                        H5Dwrite(Cdset, H5T_NATIVE_INT, H5S_ALL, H5S_ALL, H5P_DEFAULT, &iteration);
                        H5Dclose(Cdset);
Cristian Lalescu's avatar
Cristian Lalescu committed
109
                        H5Fclose(stat_file);
Chichi Lalescu's avatar
Chichi Lalescu committed
110
                    }
111
112
113
114
115
                    fftwf_mpi_cleanup();
                    fftw_mpi_cleanup();
                    MPI_Finalize();
                    return EXIT_SUCCESS;
                }
Chichi Lalescu's avatar
Chichi Lalescu committed
116
117
                //endcpp
                """
118
119
120
121
122
123
        self.host_info = {'type'        : 'cluster',
                          'environment' : None,
                          'deltanprocs' : 1,
                          'queue'       : '',
                          'mail_address': '',
                          'mail_events' : None}
124
        self.main = ''
125
        return None
Cristian Lalescu's avatar
broken    
Cristian Lalescu committed
126
    def write_src(self):
127
        with open(self.name + '.cpp', 'w') as outfile:
128
            outfile.write(self.version_message)
Cristian Lalescu's avatar
broken    
Cristian Lalescu committed
129
            outfile.write(self.includes)
130
            outfile.write(self.cdef_pars())
Cristian Lalescu's avatar
broken    
Cristian Lalescu committed
131
            outfile.write(self.variables)
132
            outfile.write(self.cread_pars())
Cristian Lalescu's avatar
broken    
Cristian Lalescu committed
133
134
135
136
137
            outfile.write(self.definitions)
            outfile.write(self.main_start)
            outfile.write(self.main)
            outfile.write(self.main_end)
        return None
138
139
    def compile_code(self):
        # compile code
140
141
142
143
        if not os.path.isfile(os.path.join(bfps.header_dir, 'base.hpp')):
            raise IOError('header not there:\n' +
                          '{0}\n'.format(os.path.join(bfps.header_dir, 'base.hpp')) +
                          '{0}\n'.format(bfps.dist_loc))
144
        libraries = ['bfps']
Cristian Lalescu's avatar
Cristian Lalescu committed
145
        libraries += bfps.install_info['libraries']
146

Cristian Lalescu's avatar
Cristian Lalescu committed
147
        command_strings = ['g++']
Chichi Lalescu's avatar
Chichi Lalescu committed
148
        command_strings += [self.name + '.cpp', '-o', self.name]
149
        command_strings += bfps.install_info['extra_compile_args']
150
        command_strings += ['-I' + idir for idir in bfps.install_info['include_dirs']]
151
        command_strings.append('-I' + bfps.header_dir)
152
        command_strings += ['-L' + ldir for ldir in bfps.install_info['library_dirs']]
153
154
155
        command_strings.append('-L' + bfps.lib_dir)
        for libname in libraries:
            command_strings += ['-l' + libname]
Cristian Lalescu's avatar
Cristian Lalescu committed
156
        self.write_src()
157
        print('compiling code with command\n' + ' '.join(command_strings))
158
        return subprocess.call(command_strings)
159
160
161
162
163
    def set_host_info(
            self,
            host_info = {}):
        self.host_info.update(host_info)
        return None
Cristian Lalescu's avatar
broken    
Cristian Lalescu committed
164
165
    def run(self,
            ncpu = 2,
166
            out_file = 'out_file',
167
            err_file = 'err_file',
168
            hours = 1,
169
170
            minutes = 0,
            njobs = 1):
Chichi Lalescu's avatar
Chichi Lalescu committed
171
        self.read_parameters()
Cristian Lalescu's avatar
Cristian Lalescu committed
172
173
        with h5py.File(os.path.join(self.work_dir, self.simname + '.h5'), 'r') as data_file:
            iter0 = data_file['iteration'].value
174
175
        if not os.path.isdir(self.work_dir):
            os.makedirs(self.work_dir)
176
        if not os.path.exists(os.path.join(self.work_dir, self.name)):
177
178
179
180
181
            need_to_compile = True
        else:
            need_to_compile = (datetime.fromtimestamp(os.path.getctime(os.path.join(self.work_dir, self.name))) <
                               bfps.install_info['install_date'])
        if need_to_compile:
182
183
184
185
            assert(self.compile_code() == 0)
            if self.work_dir != './':
                shutil.copy(self.name, self.work_dir)
        current_dir = os.getcwd()
186
187
        os.chdir(self.work_dir)
        os.chdir(current_dir)
188
189
190
191
        command_atoms = ['mpirun',
                         '-np',
                         '{0}'.format(ncpu),
                         './' + self.name,
Chichi Lalescu's avatar
Chichi Lalescu committed
192
                         self.simname]
193
        if self.host_info['type'] == 'cluster':
194
195
            job_name_list = []
            for j in range(njobs):
196
197
                suffix = self.simname + '_{0}'.format(iter0 + j*self.parameters['niter_todo'])
                qsub_script_name = 'run_' + suffix + '.sh'
198
199
200
                self.write_sge_file(
                    file_name     = os.path.join(self.work_dir, qsub_script_name),
                    nprocesses    = ncpu,
201
                    name_of_run   = suffix,
202
203
204
                    command_atoms = command_atoms[3:],
                    hours         = hours,
                    minutes       = minutes,
205
206
                    out_file      = out_file + '_' + suffix,
                    err_file      = err_file + '_' + suffix)
207
208
209
                os.chdir(self.work_dir)
                qsub_atoms = ['qsub']
                if len(job_name_list) >= 1:
Cristian Lalescu's avatar
Cristian Lalescu committed
210
                    qsub_atoms += ['-hold_jid', job_name_list[-1]]
211
212
                subprocess.call(qsub_atoms + [qsub_script_name])
                os.chdir(current_dir)
213
                job_name_list.append(suffix)
214
        elif self.host_info['type'] == 'pc':
215
            os.chdir(self.work_dir)
216
            os.environ['LD_LIBRARY_PATH'] += ':{0}'.format(bfps.lib_dir)
217
            print('added to LD_LIBRARY_PATH the location {0}'.format(bfps.lib_dir))
218
            for j in range(njobs):
219
                suffix = self.simname + '_{0}'.format(iter0 + j*self.parameters['niter_todo'])
220
                print('running code with command\n' + ' '.join(command_atoms))
221
                subprocess.call(command_atoms,
222
223
                                stdout = open(out_file + '_' + suffix, 'w'),
                                stderr = open(err_file + '_' + suffix, 'w'))
224
            os.chdir(current_dir)
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
        return None
    def write_sge_file(
            self,
            file_name = None,
            nprocesses = None,
            name_of_run = None,
            command_atoms = [],
            hours = None,
            minutes = None,
            out_file = None,
            err_file = None):
        script_file = open(file_name, 'w')
        script_file.write('#!/bin/bash\n')
        # export all environment variables
        script_file.write('#$ -V\n')
        # job name
        script_file.write('#$ -N {0}\n'.format(name_of_run))
        # use current working directory
        script_file.write('#$ -cwd\n')
        # error file
        if not type(err_file) == type(None):
            script_file.write('#$ -e ' + err_file + '\n')
        # output file
        if not type(out_file) == type(None):
            script_file.write('#$ -o ' + out_file + '\n')
        if not type(self.host_info['environment']) == type(None):
Chichi Lalescu's avatar
Chichi Lalescu committed
251
            envprocs = self.host_info['deltanprocs'] * int(math.ceil((nprocesses *1.0/ self.host_info['deltanprocs'])))
252
253
254
255
            script_file.write('#$ -pe {0} {1}\n'.format(
                    self.host_info['environment'],
                    envprocs))
        script_file.write('echo "got $NSLOTS slots."\n')
256
        script_file.write('echo "Start time is `date`"\n')
Cristian Lalescu's avatar
Cristian Lalescu committed
257
258
259
260
261
262
        script_file.write('mpiexec -machinefile $TMPDIR/machines ' +
                          '-genv LD_LIBRARY_PATH ' +
                          '"' +
                          ':'.join([bfps.lib_dir] + bfps.install_info['library_dirs']) +
                          '" ' +
                          '-n {0} {1}\n'.format(nprocesses, ' '.join(command_atoms)))
263
        script_file.write('echo "End time is `date`"\n')
264
265
266
        script_file.write('exit 0\n')
        script_file.close()
        return None
267
268
269
270
271
272
273
274
275
276
277
    def prepare_launch(
            self,
            args = [],
            **kwargs):
        parser = argparse.ArgumentParser('bfps ' + type(self).__name__)
        self.add_parser_arguments(parser)
        opt = parser.parse_args(args)
        self.set_host_info(bfps.host_info)
        if type(opt.environment) != type(None):
            self.host_info['environment'] = opt.environment
        return opt
278