mpcdf_common.py 16.3 KB
Newer Older
1
2
3
4
5
from __future__ import print_function

import osc
import osc.conf
import osc.core
Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
6
import osc.oscerr
7

Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
8
from functools import partial
9
10
11
12
from xml.etree import ElementTree


def valid_mpi(compiler, mpi):
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
    """
    It might be possible to use Intel MPI libararies and compilers from
    different Parallel Studio packages, but I currently do not want to support
    it.

    Take care to keep this in sync with the file 'macros.obs_cluster' of
    the package software:dist / mpcdf_cluster_macros
    """

    valid_intel_impi = {
        ("intel_17_0_4", "impi_2017_3"),
        ("intel_17_0_7", "impi_2017_4"),
        ("intel_18_0_1", "impi_2018_1"),
        ("intel_18_0_2", "impi_2018_2"),
        ("intel_18_0_3", "impi_2018_3"),
    }

    if compiler.startswith("intel") and mpi.startswith("impi"):
        return (compiler, mpi) in valid_intel_impi
    else:
        return True
34
35
36
37
38
39


def valid_cuda(cuda, compiler):
    """
    The CUDA distribution only works with certain gcc versions,
    this little function takes care that only supported combinations
Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
40
41
42
43
    are allowed.

    Take care to keep this in sync with the file 'macros.obs_cluster' of
    the package software:dist / mpcdf_cluster_macros
44
    """
Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
45
    if cuda == "cuda_8_0":
46
        return compiler == "gcc_5"
Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
47
    if cuda == "cuda_9_1":
48
        return compiler == "gcc_6_3_0"
Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
49
50
    if cuda == "cuda_9_2":
        return compiler == "gcc_7"
51
52
53
    return False


Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
54
55
56
def project_meta(api_url, project):
    return ElementTree.fromstringlist(osc.core.show_project_meta(api_url, project))

57

58
59
60
61
class UnsetAttributeException(Exception):
    pass


Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
62
def get_attribute(api_url, project, package, attribute, with_project=False):
63
64
    attribute_meta = osc.core.show_attribute_meta(api_url, project, package, None, attribute, False, with_project)
    if attribute_meta is None:
Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
65
        raise osc.oscerr.APIError("Cannot fetch value for attribute '{0}' from {1}".format(attribute, (project, package)))
66
67
68
69

    root = ElementTree.fromstringlist(attribute_meta)
    attribute = root.find("./attribute")
    if attribute is not None:
Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
70
        return root
71
    else:
72
        raise UnsetAttributeException("Attribute not set")
73

Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
74

Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
75
76
77
78
79
80
def get_attribute_values(api_url, project, package, attribute, with_project=False):
    root = get_attribute(api_url, project, package, attribute, with_project)
    attribute = root.find("./attribute")
    return list(value.text for value in attribute.findall("./value"))


81
82
83
84
85
86
def set_attribute(api_url, project, package, attribute):
    path = ["source", project]
    if package:
        path.append(package)
    path.append("_attribute")

Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
87
    attr_meta = ElementTree.tostring(attribute, encoding=osc.core.ET_ENCODING)
Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
88

89
    url = osc.core.makeurl(api_url, path)
Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
90
    resp = ElementTree.fromstringlist(osc.core.streamfile(url, osc.core.http_POST, data=attr_meta))
Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
91
    if resp.find("./summary").text != "Ok":
Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
92
        raise osc.oscerr.APIError("Could not store attribute")
Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
93

94

95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
def set_attribute_values(api_url, project, package, attribute, values):
    root = ElementTree.Element("attributes")
    attr = ElementTree.SubElement(root, "attribute")

    namespace, name = attribute.split(":")
    attr.set("namespace", namespace)
    attr.set("name", name)

    for value in values:
        v = ElementTree.SubElement(attr, "value")
        v.text = value

    set_attribute(api_url, project, package, root)


def has_attribute(api_url, project, package, attribute):
    path = ["source", project]
    if package:
        path.append(package)
    path.append("_attribute")
    path.append(attribute)

    url = osc.core.makeurl(api_url, path)
    resp = ElementTree.fromstringlist(osc.core.streamfile(url, osc.core.http_GET))

    namespace, name = attribute.split(":")
121

122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
    for a in resp.findall("./attribute"):
        if a.get("namespace") == namespace and a.get("name") == name:
            return True
    return False


def remove_attribute(api_url, project, package, attribute_name):
    path = ["source"] + [project]
    if package:
        path.append(package)
    path.append("_attribute")
    path.append(attribute_name)

    url = osc.core.makeurl(api_url, path)
    resp = ElementTree.fromstringlist(osc.core.streamfile(url, osc.core.http_DELETE))
    if resp.find("./summary").text != "Ok":
        raise osc.oscerr.APIError("Could not remove attribute")


def mpcdf_enable_repositories(api_url, project, package, verbose=False):
142
    from itertools import product
Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
143
    import sys
144

145
146
    root = ElementTree.fromstringlist(osc.core.show_package_meta(api_url, project, package))
    build = root.find("./build")
Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
147
148
149
150
151
    if build is None:
        build = ElementTree.SubElement(root, "build")
    else:
        for enable in build.findall("./enable"):
            build.remove(enable)
152

Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
153
    compilers = get_attribute_values(api_url, project, package, "MPCDF:compiler_modules", with_project=True)
154
    all_compilers = get_attribute_values(api_url, project, None, "MPCDF:compiler_modules")
Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
155
156
    mpis = get_attribute_values(api_url, project, package, "MPCDF:mpi_modules", with_project=True)
    cudas = get_attribute_values(api_url, project, package, "MPCDF:cuda_modules", with_project=True)
157

Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
    try:
        default_compilers = get_attribute_values(api_url, project, None, "MPCDF:default_compiler")
    except UnsetAttributeException:
        print("ERROR: Attribute MPCDF:default_compiler not set for project", file=sys.stderr)
        raise SystemExit(1)

    try:
        default_mpis = get_attribute_values(api_url, project, None, "MPCDF:default_mpi")
    except UnsetAttributeException:
        print("ERROR: Attribute MPCDF:default_mpi not set for project", file=sys.stderr)
        raise SystemExit(1)

    try:
        default_cudas = get_attribute_values(api_url, project, None, "MPCDF:default_cuda")
    except UnsetAttributeException:
        print("ERROR: Attribute MPCDF:default_cuda not set for project", file=sys.stderr)
        raise SystemExit(1)
175

176
177
178
179
180
181
182
183
184
    def enable(name):
        node = ElementTree.Element("enable")
        node.set("repository", name)
        node.tail = "\n    "
        build.insert(0, node)
        if verbose:
            print("Enabling", name)

    try:
Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
185
        enable_repos = get_attribute_values(api_url, project, package, "MPCDF:enable_repositories")
Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
186
    except Exception:
187
188
189
190
        if verbose:
            print("Warning: Could not get attribute MPCDF:enable_repositories for package {0}".format(package))
        return False

191
192
193
    def actual_compilers():
        for compiler in compilers:
            if compiler == "intel":
Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
194
                for intel_compiler in filter(lambda cc: cc.startswith("intel"), all_compilers):
195
196
                    yield intel_compiler
            elif compiler == "gcc":
Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
197
                for gcc_compiler in filter(lambda cc: cc.startswith("gcc"), all_compilers):
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
                    yield gcc_compiler
            elif compiler == "default_compiler":
                for default_compiler in default_compilers:
                    yield default_compiler
            else:
                yield compiler

    def actual_mpis():
        for mpi in mpis:
            if mpi == "impi":
                for impi in filter(lambda cc: cc.startswith("impi"), mpis):
                    yield impi
            elif mpi == "default_mpi":
                for default_mpi in default_mpis:
                    yield default_mpi
            else:
                yield mpi

    def actual_cudas():
        for cuda in cudas:
            if cuda == "default_cuda":
                for default_cuda in default_cudas:
                    yield default_cuda
            else:
                yield cuda

224
225
226
227
228
229
    for flag in enable_repos:

        if flag == "system":
            enable("System")

        if flag == "compilers":
230
            for compiler in actual_compilers():
Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
231
                enable(compiler)
232
233

        if flag == "mpi":
234
235
236
            for mpi, compiler in product(actual_mpis(), actual_compilers()):
                if valid_mpi(compiler, mpi):
                    enable(mpi + "_" + compiler)
237
238

        if flag == "cuda":
239
240
241
            for cuda, compiler in product(actual_cudas(), all_compilers):
                if valid_cuda(cuda, compiler):
                    enable(cuda + "_" + compiler)
242
243

        if flag == "cuda_mpi":
244
245
246
            for cuda, mpi, compiler in product(actual_cudas(), actual_mpis(), all_compilers):
                if valid_cuda(cuda, compiler) and valid_mpi(compiler, mpi):
                        enable(cuda + "_" + mpi + "_" + compiler)
247

Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
248
249
    if len(build.getchildren()) > 0:
        build.getchildren()[-1].tail = "\n  "
250

Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
251
    pkg_meta = ElementTree.tostring(root, encoding=osc.core.ET_ENCODING)
252
253
254

    osc.core.edit_meta("pkg", (project, package), data=pkg_meta)
    return True
Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
255
256


257
def mpcdf_setup_repositories(api_url, project, distribution=None, parent=None, packages=None, dry_run=False):
Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
258
    if parent:
259
260
261
        for attribute in ["compiler_modules", "default_compiler", "mpi_modules", "default_mpi", "cuda_modules", "default_cuda"]:
            print("Copying attribute 'MPCDF:{0}' from parent project".format(attribute))
            set_attribute(api_url, project, None, get_attribute(api_url, parent, None, "MPCDF:" + attribute))
Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
262

Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
263
264
265
266
    compilers = list(get_attribute_values(api_url, project, None, "MPCDF:compiler_modules"))
    mpis = list(get_attribute_values(api_url, project, None, "MPCDF:mpi_modules"))
    cudas = list(get_attribute_values(api_url, project, None, "MPCDF:cuda_modules"))

267
268
    if distribution is None:
        # Get existing value from project meta
269
270
271
272
273
274
        dist_repo = project_meta(api_url, project if parent is None else parent).find(
            "./repository[@name='System']/path[@project='distributions']")
        if dist_repo is not None:
            distribution = dist_repo.get("repository")
        else:
            raise osc.oscerr.WrongArgs("ERROR: Specify distribution or parent project")
275
276
        print("Using '{0}' as base distribution".format(distribution))

Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
277
278
279
    distributions = project_meta(api_url, "distributions")
    dist_repo = distributions.find('./repository[@name="{0}"]'.format(distribution))
    if dist_repo is None:
Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
280
        raise osc.oscerr.WrongArgs("No repository '{0}' is defined in project 'distributions' on the server".format(distribution))
Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
    architectures = list(arch.text for arch in dist_repo.findall("./arch"))

    root = project_meta(api_url, project)

    prjconf = osc.core.show_project_conf(api_url, project)
    start_marker = "# Autogenerated by osc mpcdf_setup_repos, do not edit till end of section\n"
    end_marker = "# End of autogenerated section\n"

    try:
        start = prjconf.index(start_marker)
        end = prjconf.index(end_marker)
    except ValueError:
        start = None
        end = len(prjconf)

    prjconf_head = "".join(prjconf[:start])
Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
297
    prjconf_tail = "".join(prjconf[end + 1:])
Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
298
299
300
    prjconf = [start_marker]

    # Remove existing repositories
301
302
    for oldrepo in root.findall("./repository"):
        root.remove(oldrepo)
Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
303
304

    def repo(name, *dependencies, **kwargs):
305
        is_compiler = kwargs.pop("compiler", False)
Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
306
307
        is_mpi = kwargs.pop("mpi", False)
        is_cuda = kwargs.pop("cuda", False)
308
309
310
311
312
313
        is_cuda_mpi = kwargs.pop("cuda_mpi", False)

        have_compiler = is_compiler or is_mpi or is_cuda or is_cuda_mpi
        have_mpi = is_mpi or is_cuda_mpi
        have_cuda = is_cuda or is_cuda_mpi

Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
314
315
        r = ElementTree.SubElement(root, "repository")
        r.set("name", name)
Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
316
317
        r.text = "\n    "

Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
318
319
320
321
        def path(project, repo):
            p = ElementTree.SubElement(r, "path")
            p.set("project", project)
            p.set("repository", repo)
Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
322
            p.tail = "\n    "
Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
323
324
325

        for dep_project, dep_repo in dependencies:
            path(dep_project, dep_repo)
326
327
        if parent:
            path(parent, name)
Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
328
329
330
331
332
333
334
335
336
337

        for arch in architectures:
            a = ElementTree.SubElement(r, "arch")
            a.text = arch
            a.tail = "\n    "
        a.tail = "\n  "
        r.tail = "\n  "

        prjconf.append("%if %_repository == {0}".format(name))
        prjconf.append("Macros:")
338
339

        prjconf.append("%is_compiler_repository {0}".format(1 if is_compiler else 0))
Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
340
341
        prjconf.append("%is_mpi_repository {0}".format(1 if is_mpi else 0))
        prjconf.append("%is_cuda_repository {0}".format(1 if is_cuda else 0))
342
343
344
345
346
347
        prjconf.append("%is_cuda_mpi_repository {0}".format(1 if is_cuda_mpi else 0))

        prjconf.append("%have_mpcdf_compiler {0}".format(1 if have_compiler else 0))
        prjconf.append("%have_mpcdf_mpi {0}".format(1 if have_mpi else 0))
        prjconf.append("%have_mpcdf_cuda {0}".format(1 if have_cuda else 0))

348
349
350
        for macro, value in kwargs.items():
            prjconf.append("%{0} {1}".format(macro, value))

Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
351
352
353
354
        prjconf.append(":Macros")
        prjconf.append("%endif")
        prjconf.append("")

Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
355
356
357
358
    if parent:
        repo("System")
    else:
        repo("System", ("distributions", distribution))
Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
359
360

    for compiler in compilers:
361
        repo(compiler, (project, "System"), compiler=True)
Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
362
363

        for mpi in filter(partial(valid_mpi, compiler), mpis):
364
            repo(mpi + "_" + compiler, (project, compiler), mpi=True, mpi_repository=mpi)
Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
365
366
367
368
369
370
371
372

    for cuda in cudas:
        for compiler in filter(partial(valid_cuda, cuda), compilers):
            repo(cuda + "_" + compiler, (project, compiler), cuda=True)
            for mpi in filter(partial(valid_mpi, compiler), mpis):
                repo(cuda + "_" + mpi + "_" + compiler,
                     (project, cuda + "_" + compiler),
                     (project, mpi + "_" + compiler),
373
                     cuda_mpi=True)
Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
374
375
376
377
378
379
380
381
382
383
384
385

    # Remove build configuration
    build = root.find("./build")
    if build is None:
        build = ElementTree.Element("build")
        build.text = "\n    "
        disable = ElementTree.SubElement(build, "disable")
        disable.tail = "\n  "
        build.tail = "\n  "
        root.insert(list(root).index(root.find("./repository")), build)

    root.getchildren()[-1].tail = "\n"
Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
386
    prj = ElementTree.tostring(root, encoding=osc.core.ET_ENCODING)
Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
387
388
389
390
391
392
393
394
395
396

    prjconf.append(end_marker)
    prjconf = prjconf_head + "\n".join(prjconf) + prjconf_tail

    if dry_run:
        print("osc meta prj {0} -F - <<EOF\n{1}\nEOF\n".format(project, prj))
        print("osc meta prjconf {0} -F - <<EOF\n{1}\nEOF\n".format(project, prjconf))
    else:
        # First set-up the <enable/> flags, that way no
        # spurious builds are launched
Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
397
398
399
        if packages is None:
            packages = osc.core.meta_get_packagelist(api_url, project)
        for package in packages:
Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
400
401
402
403
404
405
            print("Updating repositories for", package)
            if not mpcdf_enable_repositories(api_url, project, package):
                print("ATTENTION: Not changing unmanaged package {0}".format(package))

        # Update repositories
        print("Updating prj meta")
406
        osc.core.edit_meta("prj", project, data=prj, force=True)
Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
407
408
        print("Updating prjconf meta")
        osc.core.edit_meta("prjconf", project, data=prjconf)
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448


def set_as_branch(api_url, my_project, my_package, main_project, main_package):
    import os

    def dirmeta(project, package):
        url = osc.core.makeurl(api_url, ["source", project, package])
        return ElementTree.fromstringlist(osc.core.streamfile(url, osc.core.http_GET))

    srcmeta = dirmeta(my_project, my_package)
    dstmeta = dirmeta(main_project, main_package)

    linkinfo = srcmeta.find('./linkinfo')
    if linkinfo is not None:
        print("ERROR: package {0} is already linked to package {1} in project {2}".format(
              my_package, linkinfo.get("package"), linkinfo.get("project")))
        return False

    linkfile = ElementTree.Element("link")
    linkfile.set("project", main_project)
    linkfile.set("package", main_package)
    linkfile.set("baserev", dstmeta.get("srcmd5"))
    patches = ElementTree.SubElement(linkfile, "patches")
    ElementTree.SubElement(patches, "branch")

    url = osc.core.makeurl(api_url, ["source", my_project, my_package, "_link"])
    resp = ElementTree.fromstringlist(osc.core.streamfile(url, osc.core.http_PUT, data=ElementTree.tostring(linkfile)))
    rev = resp.get("rev")
    if rev is None:
        print("ERROR: Could not commit _link file")
        return False
    else:
        print("Commited as revision", rev)
        this_package = osc.core.store_read_package(os.curdir)
        this_project = osc.core.store_read_project(os.curdir)
        if this_package == my_package and this_project == my_project:
            pac = osc.core.filedir_to_pac(os.curdir)
            rev = pac.latest_rev(expand=True)
            pac.update(rev)
    return True