mpcdf_common.py 28.6 KB
Newer Older
1
from __future__ import print_function
Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
2
from __future__ import division
3

4
import sys
5
6
7
import osc
import osc.conf
import osc.core
Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
8
import osc.oscerr
Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
9
from functools import partial
10
11
from xml.etree import ElementTree

12
13
14
15
16
17
18
if sys.version_info[0] < 3:
    def decode_it(arg):
        return arg
else:
    from osc.util.helper import decode_it


19
20
known_microarchs = {"sandybridge", "haswell", "skylake"}

21
22
package_attributes = ["MPCDF:enable_repositories"]
config_attributes = ["MPCDF:compiler_modules", "MPCDF:cuda_modules", "MPCDF:mpi_modules", "MPCDF:pgi_modules"]
23
default_attributes = ["MPCDF:default_compiler", "MPCDF:default_cuda", "MPCDF:default_mpi", "MPCDF:default_python2", "MPCDF:default_python3"]
24

25
26
27
28
29
30
31
32
intel_parallel_studio = {
    "mpcdf_intel_parallel_studio_2017_7": {"compiler": "intel_17_0_7", "impi": "impi_2017_4", "mkl": "mkl_2017_4-module", },
    "mpcdf_intel_parallel_studio_2018_1": {"compiler": "intel_18_0_1", "impi": "impi_2018_1", "mkl": "mkl_2018_1-module", },
    "mpcdf_intel_parallel_studio_2018_2": {"compiler": "intel_18_0_2", "impi": "impi_2018_2", "mkl": "mkl_2018_2-module", },
    "mpcdf_intel_parallel_studio_2018_3": {"compiler": "intel_18_0_3", "impi": "impi_2018_3", "mkl": "mkl_2018_3-module", },
    "mpcdf_intel_parallel_studio_2018_4": {"compiler": "intel_18_0_5", "impi": "impi_2018_4", "mkl": "mkl_2018_4-module", },
    "mpcdf_intel_parallel_studio_2019_0": {"compiler": "intel_19_0_0", "impi": "impi_2019_0", "mkl": "mkl_2019_0-module", },
    "mpcdf_intel_parallel_studio_2019_1": {"compiler": "intel_19_0_1", "impi": "impi_2019_1", "mkl": "mkl_2019_1-module", },
Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
33
    "mpcdf_intel_parallel_studio_2019_3": {"compiler": "intel_19_0_3", "impi": "impi_2019_3", "mkl": "mkl_2019_3-module", },
34
    "mpcdf_intel_parallel_studio_2019_4": {"compiler": "intel_19_0_4", "impi": "impi_2019_4", "mkl": "mkl_2019_4-module", },
35
    "mpcdf_intel_parallel_studio_2019_5": {"compiler": "intel_19_0_5", "impi": "impi_2019_5", "mkl": "mkl_2019_5-module", },
36
37
38
39
40
41
42
43
}

all_mkls = {ic["mkl"] for ic in intel_parallel_studio.values()}

# Some rotations
mpi_parallel_studio = {value["impi"]: dict({"ps": key}, **value) for key, value in intel_parallel_studio.items()}
compiler_parallel_studio = {value["compiler"]: dict({"ps": key}, **value) for key, value in intel_parallel_studio.items()}

44

45
46
47
48
49
# For the autogenerated software/software:*:* project 'prjconf' sections
prjconf_start_marker = "# Autogenerated by osc mpcdf_setup_repos, do not edit till end of section\n"
prjconf_end_marker = "# End of autogenerated section\n"


50
51
52
53
54
55
56
57
def compiler_module(compiler_repo):
    return compiler_repo.replace("_", "/", 1).replace("_", ".")


def mpi_module(mpi_repo):
    return mpi_repo.replace("_", "/", 1).replace("_", ".")


58
59
60
61
62
63
64
65
def valid_pgi_mpi(pgi, mpi):
    if "impi" not in mpi:
        return False
    if "2017" in mpi:
        return False
    return True


66
def valid_mpi(compiler, mpi):
67
68
69
70
71
72
73
74
75
    """
    It might be possible to use Intel MPI libararies and compilers from
    different Parallel Studio packages, but I currently do not want to support
    it.

    Take care to keep this in sync with the file 'macros.obs_cluster' of
    the package software:dist / mpcdf_cluster_macros
    """
    if compiler.startswith("intel") and mpi.startswith("impi"):
76
        return mpi == compiler_parallel_studio[compiler]["impi"]
77
78
    if compiler.startswith("pgi"):
        return valid_pgi_mpi(compiler, mpi)
79
80
    else:
        return True
81
82


83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
def prefers(reponame):
    prefer_ps = None
    if reponame in compiler_parallel_studio:
        prefer_ps = compiler_parallel_studio[reponame]["ps"]
    else:
        for mpi in mpi_parallel_studio:
            if reponame.startswith(mpi):
                prefer_ps = mpi_parallel_studio[mpi]["ps"]

    if prefer_ps:
        preferred_mkl = intel_parallel_studio[prefer_ps]["mkl"]
        unprefer_other_mkls = ("!" + mkl for mkl in all_mkls if mkl != preferred_mkl)
        return (prefer_ps,) + tuple(unprefer_other_mkls) + (preferred_mkl,)
    else:
        return ()


100
101
102
103
def valid_cuda(cuda, compiler):
    """
    The CUDA distribution only works with certain gcc versions,
    this little function takes care that only supported combinations
Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
104
105
106
107
    are allowed.

    Take care to keep this in sync with the file 'macros.obs_cluster' of
    the package software:dist / mpcdf_cluster_macros
108
    """
Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
109
    if cuda == "cuda_8_0":
110
        return compiler == "gcc_5"
Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
111
    if cuda == "cuda_9_1":
112
        return compiler == "gcc_6_3_0"
Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
113
    if cuda == "cuda_9_2":
Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
114
        return compiler == "gcc_6"
Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
115
116
    if cuda == "cuda_10_0":
        return compiler == "gcc_6"
Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
117
118
    if cuda == "cuda_10_1":
        return compiler == "gcc_8"
119
120
121
    return False


Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
122
123
124
def project_meta(api_url, project):
    return ElementTree.fromstringlist(osc.core.show_project_meta(api_url, project))

125

126
127
128
129
130
131
132
133
134
def package_meta(api_url, project, package):
    return ElementTree.fromstringlist(osc.core.show_package_meta(api_url, project, package))


def maintainers(api_url, project, package):
    root = package_meta(api_url, project, package)
    return {e.get("userid") for e in root.findall("./person[@role='maintainer']")}


135
136
137
138
class UnsetAttributeException(Exception):
    pass


139
140
141
142
143
144
145
146
def chunked(l, chunksize):
    n = len(l)
    i = 0
    while i * chunksize < n:
        yield l[i * chunksize: (i + 1) * chunksize]
        i += 1


Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
147
def get_attribute(api_url, project, package, attribute, with_project=False):
148
149
    attribute_meta = osc.core.show_attribute_meta(api_url, project, package, None, attribute, False, with_project)
    if attribute_meta is None:
Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
150
        raise osc.oscerr.APIError("Cannot fetch value for attribute '{0}' from {1}".format(attribute, (project, package)))
151
152
153
154

    root = ElementTree.fromstringlist(attribute_meta)
    attribute = root.find("./attribute")
    if attribute is not None:
Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
155
        return root
156
    else:
157
        raise UnsetAttributeException("Attribute not set")
158

Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
159

Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
160
161
162
163
164
165
def get_attribute_values(api_url, project, package, attribute, with_project=False):
    root = get_attribute(api_url, project, package, attribute, with_project)
    attribute = root.find("./attribute")
    return list(value.text for value in attribute.findall("./value"))


166
167
168
169
170
def get_attribute_value(api_url, project, package, attribute, with_project=False):
    value, = get_attribute_values(api_url, project, package, attribute, with_project=False)
    return value


171
172
173
174
175
176
def set_attribute(api_url, project, package, attribute):
    path = ["source", project]
    if package:
        path.append(package)
    path.append("_attribute")

Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
177
    attr_meta = ElementTree.tostring(attribute, encoding=osc.core.ET_ENCODING)
Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
178

179
    url = osc.core.makeurl(api_url, path)
Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
180
    resp = ElementTree.fromstringlist(osc.core.streamfile(url, osc.core.http_POST, data=attr_meta))
Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
181
    if resp.find("./summary").text != "Ok":
Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
182
        raise osc.oscerr.APIError("Could not store attribute")
Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
183

184

185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
def set_attribute_values(api_url, project, package, attribute, values):
    root = ElementTree.Element("attributes")
    attr = ElementTree.SubElement(root, "attribute")

    namespace, name = attribute.split(":")
    attr.set("namespace", namespace)
    attr.set("name", name)

    for value in values:
        v = ElementTree.SubElement(attr, "value")
        v.text = value

    set_attribute(api_url, project, package, root)


def has_attribute(api_url, project, package, attribute):
    path = ["source", project]
    if package:
        path.append(package)
    path.append("_attribute")
    path.append(attribute)

    url = osc.core.makeurl(api_url, path)
    resp = ElementTree.fromstringlist(osc.core.streamfile(url, osc.core.http_GET))

    namespace, name = attribute.split(":")
211

212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
    for a in resp.findall("./attribute"):
        if a.get("namespace") == namespace and a.get("name") == name:
            return True
    return False


def remove_attribute(api_url, project, package, attribute_name):
    path = ["source"] + [project]
    if package:
        path.append(package)
    path.append("_attribute")
    path.append(attribute_name)

    url = osc.core.makeurl(api_url, path)
    resp = ElementTree.fromstringlist(osc.core.streamfile(url, osc.core.http_DELETE))
    if resp.find("./summary").text != "Ok":
        raise osc.oscerr.APIError("Could not remove attribute")


231
232
233
234
def get_microarchitecture(project):
    if project == "software":
        # Stupid special case
        microarch = "sandybridge"
235
    elif project.startswith("software:"):
236
        microarch = project.split(":")[2]
237
238
239
240
    elif project.startswith("home:"):
        microarch = "sandybridge"
    else:
        raise Exception("Cannot determine micro-architecture for project '{0}'".format(project))
241
242
243
244
245
246
247

    if microarch in known_microarchs:
        return microarch
    else:
        raise Exception("Unknown micro-architecture '{0}'".format(microarch))


248
def mpcdf_enable_repositories(api_url, project, package, verbose=False, filter_repos=None):
249
    from itertools import product
Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
250
    import sys
251

252
253
254
    if filter_repos is None:
        filter_repos = ()

255
256
257
    pkg_meta = osc.core.show_package_meta(api_url, project, package)
    root = ElementTree.fromstringlist(pkg_meta)

258
    build = root.find("./build")
Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
259
260
    if build is None:
        build = ElementTree.SubElement(root, "build")
261
262
263
264
265

    pkg_meta = ElementTree.tostring(root, encoding=osc.core.ET_ENCODING)

    for enable in build.findall("./enable"):
        build.remove(enable)
266

Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
267
    try:
268
269
270
271
272
        enable_repos = get_attribute_values(api_url, project, package, "MPCDF:enable_repositories")
    except Exception:
        if verbose:
            print("Warning: Could not get attribute MPCDF:enable_repositories for package {0}, skipping".format(package))
        return False
Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
273

Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
274
    def try_get_attribute(package, attribute, with_project=False):
275
276
277
        try:
            return get_attribute_values(api_url, project, package, "MPCDF:" + attribute, with_project=with_project)
        except UnsetAttributeException:
278
279
280
            print("ERROR: Attribute MPCDF:" + attribute + " is not set for "
                  + ("package '{0}'".format(package) if package else "project '{0}'".format(project))
                  + ", aborting here", file=sys.stderr)
281
            raise SystemExit(1)
Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
282

283
284
285
    compilers = try_get_attribute(package, "compiler_modules", with_project=True)
    mpis = try_get_attribute(package, "mpi_modules", with_project=True)
    cudas = try_get_attribute(package, "cuda_modules", with_project=True)
286
287
288
289
290
291
    pgis = try_get_attribute(package, "pgi_modules", with_project=True)

    all_compilers = try_get_attribute(None, "compiler_modules")
    all_mpis = try_get_attribute(None, "mpi_modules")
    all_cudas = try_get_attribute(None, "cuda_modules")
    all_pgis = try_get_attribute(None, "pgi_modules")
292
293
294
295

    default_compilers = try_get_attribute(None, "default_compiler")
    default_mpis = try_get_attribute(None, "default_mpi")
    default_cudas = try_get_attribute(None, "default_cuda")
296

297
298
299
300
301
302
303
    def sort_key(string):
        name, *version = string.split("_")
        return (name,) + tuple(map(int, version))

    latest_intel = sorted((c for c in all_compilers if c.startswith("intel")), key=sort_key)[-1]
    latest_gcc = sorted((c for c in all_compilers if c.startswith("gcc")), key=sort_key)[-1]

304
    def enable(name):
305
306
        if any(filtered_repo in name for filtered_repo in filter_repos):
            return
307
308
309
310
311
312
313
        node = ElementTree.Element("enable")
        node.set("repository", name)
        node.tail = "\n    "
        build.insert(0, node)
        if verbose:
            print("Enabling", name)

314
    def actual_compilers():
315
        for compiler in (c for c in compilers if c in all_compilers + ["default_compiler", "intel", "gcc", "latest_intel", "latest_gcc"]):
316
            if compiler == "intel":
Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
317
                for intel_compiler in [cc for cc in all_compilers if cc.startswith("intel")]:
318
319
                    yield intel_compiler
            elif compiler == "gcc":
Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
320
                for gcc_compiler in [cc for cc in all_compilers if cc.startswith("gcc")]:
321
322
323
324
                    yield gcc_compiler
            elif compiler == "default_compiler":
                for default_compiler in default_compilers:
                    yield default_compiler
325
326
327
328
            elif compiler == "latest_intel":
                yield latest_intel
            elif compiler == "latest_gcc":
                yield latest_gcc
329
330
331
332
            else:
                yield compiler

    def actual_mpis():
333
        for mpi in (m for m in mpis if m in all_mpis + ["default_mpi", "impi"]):
334
            if mpi == "impi":
Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
335
                for impi in [mpi for mpi in mpis if mpi.startswith("impi")]:
336
337
338
339
340
341
342
343
                    yield impi
            elif mpi == "default_mpi":
                for default_mpi in default_mpis:
                    yield default_mpi
            else:
                yield mpi

    def actual_cudas():
344
        for cuda in (c for c in cudas if c in all_cudas + ["default_cuda"]):
345
346
347
348
349
350
            if cuda == "default_cuda":
                for default_cuda in default_cudas:
                    yield default_cuda
            else:
                yield cuda

351
352
353
354
    def actual_pgis():
        for pgi in (p for p in pgis if p in all_pgis):
            yield pgi

355
356
357
358
359
360
    for flag in enable_repos:

        if flag == "system":
            enable("System")

        if flag == "compilers":
361
            for compiler in actual_compilers():
Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
362
                enable(compiler)
363
364

        if flag == "mpi":
365
366
367
            for mpi, compiler in product(actual_mpis(), actual_compilers()):
                if valid_mpi(compiler, mpi):
                    enable(mpi + "_" + compiler)
368
369

        if flag == "cuda":
370
371
372
            for cuda, compiler in product(actual_cudas(), all_compilers):
                if valid_cuda(cuda, compiler):
                    enable(cuda + "_" + compiler)
373
374

        if flag == "cuda_mpi":
375
376
            for cuda, mpi, compiler in product(actual_cudas(), actual_mpis(), all_compilers):
                if valid_cuda(cuda, compiler) and valid_mpi(compiler, mpi):
377
                    enable(cuda + "_" + mpi + "_" + compiler)
378

379
380
381
382
383
384
        if flag == "pgi":
            for pgi in actual_pgis():
                enable(pgi)

        if flag == "pgi_mpi":
            for mpi, pgi in product(actual_mpis(), actual_pgis()):
385
386
                if valid_pgi_mpi(pgi, mpi):
                    enable(mpi + "_" + pgi)
387

Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
388
389
    if len(build.getchildren()) > 0:
        build.getchildren()[-1].tail = "\n  "
390

391
392
    new_pkg_meta = ElementTree.tostring(root, encoding=osc.core.ET_ENCODING)
    if pkg_meta != new_pkg_meta:
393
        print("Updating repositories for", package)
394
        osc.core.edit_meta("pkg", (project, package), data=new_pkg_meta)
395
396

    return True
Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
397
398


399
400
def mpcdf_setup_repositories(api_url, project, microarchitecture=None, distribution=None, parent=None,
                             packages=None, dry_run=False, filter_repos=None, only_project=False, remove_old=False):
401
402
    import threading

Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
403
    if parent:
404
405
406
        for attribute in config_attributes + default_attributes:
            print("Copying attribute '{0}' from parent project".format(attribute))
            set_attribute(api_url, project, None, get_attribute(api_url, parent, None, attribute))
Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
407

408
409
410
411
412
413
    compilers = get_attribute_values(api_url, project, None, "MPCDF:compiler_modules")
    mpis = get_attribute_values(api_url, project, None, "MPCDF:mpi_modules")
    cudas = get_attribute_values(api_url, project, None, "MPCDF:cuda_modules")
    pgis = get_attribute_values(api_url, project, None, "MPCDF:pgi_modules")
    default_python2 = get_attribute_value(api_url, project, None, "MPCDF:default_python2")
    default_python3 = get_attribute_value(api_url, project, None, "MPCDF:default_python3")
Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
414

415
416
    if distribution is None:
        # Get existing value from project meta
417
418
419
420
421
422
        dist_repo = project_meta(api_url, project if parent is None else parent).find(
            "./repository[@name='System']/path[@project='distributions']")
        if dist_repo is not None:
            distribution = dist_repo.get("repository")
        else:
            raise osc.oscerr.WrongArgs("ERROR: Specify distribution or parent project")
423
424
        print("Using '{0}' as base distribution".format(distribution))

Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
425
426
427
    distributions = project_meta(api_url, "distributions")
    dist_repo = distributions.find('./repository[@name="{0}"]'.format(distribution))
    if dist_repo is None:
Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
428
        raise osc.oscerr.WrongArgs("No repository '{0}' is defined in project 'distributions' on the server".format(distribution))
Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
429
430
    architectures = list(arch.text for arch in dist_repo.findall("./arch"))

431
432
433
    if microarchitecture is None:
        microarchitecture = get_microarchitecture(project)

Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
434
435
    root = project_meta(api_url, project)

Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
436
    prjconf = list(map(decode_it, osc.core.show_project_conf(api_url, project)))
Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
437
438

    try:
439
440
        start = prjconf.index(prjconf_start_marker)
        end = prjconf.index(prjconf_end_marker)
Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
441
442
443
444
445
    except ValueError:
        start = None
        end = len(prjconf)

    prjconf_head = "".join(prjconf[:start])
Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
446
    prjconf_tail = "".join(prjconf[end + 1:])
447
    prjconf = [prjconf_start_marker]
Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
448

449
450
451
452
453
454
455
456
457
    prjconf.append("Constraint: hostlabel {0}".format(microarchitecture))
    prjconf.append(
        """
%if %_repository != System
Macros:
%microarchitecture {0}
:Macros
%endif""".format(microarchitecture))
    prjconf.append("")
458
459
460
461
    prjconf.append("Prefer: mpcdf_python2_" + default_python2)
    prjconf.append("Prefer: mpcdf_python3_" + default_python3)
    prjconf.append("")

Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
462
    # Remove existing repositories
463
464
465
    if remove_old:
        for oldrepo in root.findall("./repository"):
            root.remove(oldrepo)
Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
466
467

    def repo(name, *dependencies, **kwargs):
468
        is_compiler = kwargs.pop("compiler", False)
Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
469
470
        is_mpi = kwargs.pop("mpi", False)
        is_cuda = kwargs.pop("cuda", False)
471
472
        is_cuda_mpi = kwargs.pop("cuda_mpi", False)

473
474
        cuda_repo = kwargs.pop("cuda_repo", "")

475
476
477
478
        have_compiler = is_compiler or is_mpi or is_cuda or is_cuda_mpi
        have_mpi = is_mpi or is_cuda_mpi
        have_cuda = is_cuda or is_cuda_mpi

479
480
481
482
483
484
        existing_repo = root.find("./repository[@name='{0}']".format(name))
        if existing_repo is not None:
            root.remove(existing_repo)
        else:
            print("New repository", name)

Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
485
486
        r = ElementTree.SubElement(root, "repository")
        r.set("name", name)
Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
487
488
        r.text = "\n    "

Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
489
490
491
492
        def path(project, repo):
            p = ElementTree.SubElement(r, "path")
            p.set("project", project)
            p.set("repository", repo)
Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
493
            p.tail = "\n    "
Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
494

495
496
        if parent:
            path(parent, name)
497
498
        for dep_project, dep_repo in dependencies:
            path(dep_project, dep_repo)
Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
499
500
501
502
503
504
505
506

        for arch in architectures:
            a = ElementTree.SubElement(r, "arch")
            a.text = arch
            a.tail = "\n    "
        a.tail = "\n  "
        r.tail = "\n  "

Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
507
508
509
510
        # In order to be able to figure out the matching MKL version for a given
        # compiler/MPI repository we emit a new macro '%matching_mkl_version' in
        # the cases this makes sense
        matching_mkl = []
Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
511
        prjconf.append("%if %_repository == {0}".format(name))
512
        for prefer in prefers(name):
Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
513
514
            if prefer.startswith("mkl_"):
                matching_mkl.append(prefer)
515
            prjconf.append("Prefer: " + prefer)
Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
516

Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
517
        prjconf.append("Macros:")
518
519

        prjconf.append("%is_compiler_repository {0}".format(1 if is_compiler else 0))
Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
520
521
        prjconf.append("%is_mpi_repository {0}".format(1 if is_mpi else 0))
        prjconf.append("%is_cuda_repository {0}".format(1 if is_cuda else 0))
522
523
524
525
526
527
        prjconf.append("%is_cuda_mpi_repository {0}".format(1 if is_cuda_mpi else 0))

        prjconf.append("%have_mpcdf_compiler {0}".format(1 if have_compiler else 0))
        prjconf.append("%have_mpcdf_mpi {0}".format(1 if have_mpi else 0))
        prjconf.append("%have_mpcdf_cuda {0}".format(1 if have_cuda else 0))

528
529
530
        if is_cuda:
            prjconf.append("%cuda_repository {0}".format(cuda_repo))

Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
531
532
533
534
535
536
        if matching_mkl:
            matching_mkl, = matching_mkl
            matching_mkl, _ = matching_mkl[len("mkl_"):].split("-module")
            matching_mkl = matching_mkl.replace("_", ".")
            prjconf.append("%matching_mkl_version {0}".format(matching_mkl))

537
538
539
        for macro, value in kwargs.items():
            prjconf.append("%{0} {1}".format(macro, value))

Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
540
541
542
543
        prjconf.append(":Macros")
        prjconf.append("%endif")
        prjconf.append("")

Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
544
545
546
547
    if parent:
        repo("System")
    else:
        repo("System", ("distributions", distribution))
Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
548

549
    for compiler in compilers + pgis:
550
        repo(compiler, (project, "System"), compiler=True,
551
             compiler_repository=compiler, compiler_module=compiler_module(compiler))
Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
552
553

        for mpi in filter(partial(valid_mpi, compiler), mpis):
554
            repo(mpi + "_" + compiler, (project, compiler), mpi=True,
555
                 mpi_repository=mpi, mpi_module=mpi_module(mpi))
Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
556
557
558

    for cuda in cudas:
        for compiler in filter(partial(valid_cuda, cuda), compilers):
559
            repo(cuda + "_" + compiler, (project, compiler), cuda=True, cuda_repo=cuda)
Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
560
561
562
563
            for mpi in filter(partial(valid_mpi, compiler), mpis):
                repo(cuda + "_" + mpi + "_" + compiler,
                     (project, cuda + "_" + compiler),
                     (project, mpi + "_" + compiler),
564
                     cuda_mpi=True)
Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
565
566
567
568
569
570
571
572
573
574
575
576

    # Remove build configuration
    build = root.find("./build")
    if build is None:
        build = ElementTree.Element("build")
        build.text = "\n    "
        disable = ElementTree.SubElement(build, "disable")
        disable.tail = "\n  "
        build.tail = "\n  "
        root.insert(list(root).index(root.find("./repository")), build)

    root.getchildren()[-1].tail = "\n"
Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
577
    prj = ElementTree.tostring(root, encoding=osc.core.ET_ENCODING)
Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
578

579
    prjconf.append(prjconf_end_marker)
Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
580
581
582
583
584
585
    prjconf = prjconf_head + "\n".join(prjconf) + prjconf_tail

    if dry_run:
        print("osc meta prj {0} -F - <<EOF\n{1}\nEOF\n".format(project, prj))
        print("osc meta prjconf {0} -F - <<EOF\n{1}\nEOF\n".format(project, prjconf))
    else:
586
587
588
589
590
        if not only_project:
            # First set-up the <enable/> flags, that way no
            # spurious builds are launched
            if packages is None:
                packages = osc.core.meta_get_packagelist(api_url, project)
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609

            if len(packages) > 40:
                chunksize = len(packages) // 20
            else:
                chunksize = len(packages)

            def work(packagelist):
                for package in packagelist:
                    if not mpcdf_enable_repositories(api_url, project, package, filter_repos=filter_repos):
                        print("ATTENTION: Not changing unmanaged package {0}".format(package))

            threads = []
            for packagelist in chunked(packages, chunksize):
                t = threading.Thread(target=work, args=(packagelist,))
                threads.append(t)
                t.start()

            for t in threads:
                t.join()
Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
610
611
612

        # Update repositories
        print("Updating prj meta")
613
        osc.core.edit_meta("prj", project, data=prj, force=True)
Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
614
615
        print("Updating prjconf meta")
        osc.core.edit_meta("prjconf", project, data=prjconf)
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655


def set_as_branch(api_url, my_project, my_package, main_project, main_package):
    import os

    def dirmeta(project, package):
        url = osc.core.makeurl(api_url, ["source", project, package])
        return ElementTree.fromstringlist(osc.core.streamfile(url, osc.core.http_GET))

    srcmeta = dirmeta(my_project, my_package)
    dstmeta = dirmeta(main_project, main_package)

    linkinfo = srcmeta.find('./linkinfo')
    if linkinfo is not None:
        print("ERROR: package {0} is already linked to package {1} in project {2}".format(
              my_package, linkinfo.get("package"), linkinfo.get("project")))
        return False

    linkfile = ElementTree.Element("link")
    linkfile.set("project", main_project)
    linkfile.set("package", main_package)
    linkfile.set("baserev", dstmeta.get("srcmd5"))
    patches = ElementTree.SubElement(linkfile, "patches")
    ElementTree.SubElement(patches, "branch")

    url = osc.core.makeurl(api_url, ["source", my_project, my_package, "_link"])
    resp = ElementTree.fromstringlist(osc.core.streamfile(url, osc.core.http_PUT, data=ElementTree.tostring(linkfile)))
    rev = resp.get("rev")
    if rev is None:
        print("ERROR: Could not commit _link file")
        return False
    else:
        print("Commited as revision", rev)
        this_package = osc.core.store_read_package(os.curdir)
        this_project = osc.core.store_read_project(os.curdir)
        if this_package == my_package and this_project == my_project:
            pac = osc.core.filedir_to_pac(os.curdir)
            rev = pac.latest_rev(expand=True)
            pac.update(rev)
    return True
656
657


658
def sync_projects(api_url, package=None, from_project="software", to_projects=None, redo_all=False, add_to_maintainers=True):
659
660
661
662
    if package is not None and redo_all:
        raise osc.oscerr.WrongArgs('Cannot specify `redo_all` and package')

    if to_projects is None:
Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
663
664
        to_projects = [p for p in osc.core.meta_get_project_list(api_url)
                       if p.startswith("software:") and not (p == "software:dist" or p == "software:images")]
665
666
667
668
669
670
671
672
673
674
675
676
677

    for to_project in to_projects:
        print("Syncing {0} with {1}".format(to_project, from_project))

        to_packages = osc.core.meta_get_packagelist(api_url, to_project)

        if package is None:
            from_packages = osc.core.meta_get_packagelist(api_url, from_project)
            if not redo_all:
                from_packages = list(sorted(set(from_packages) - set(to_packages)))
        else:
            from_packages = [package]

678
        for attribute in config_attributes:
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
            try:
                get_attribute(api_url, to_project, None, attribute)
            except Exception:
                attr = get_attribute(api_url, from_project, None, attribute)
                print("Setting attribute", attribute)
                set_attribute(api_url, to_project, None, attr)

        for orig_package in from_packages:
            if orig_package not in to_packages:
                filelist = osc.core.meta_get_filelist(api_url, from_project, orig_package)
                if "_link" in filelist:
                    print("Not branching package {0}, is a link".format(orig_package))
                else:
                    print("Branching package {0}".format(orig_package))
                    exists, targetprj, targetpkg, srcprj, srcpkg = \
                        osc.core.branch_pkg(api_url, from_project, orig_package, target_project=to_project, nodevelproject=True)
            else:
                print("Not branching package {0}, already present in target".format(orig_package))

698
            for attribute in package_attributes + config_attributes:
699
700
701
702
703
704
705
706
                try:
                    attr = get_attribute(api_url, from_project, orig_package, attribute)
                except UnsetAttributeException:
                    if has_attribute(api_url, to_project, orig_package, attribute):
                        remove_attribute(api_url, to_project, orig_package, attribute)
                    continue
                set_attribute(api_url, to_project, orig_package, attr)

707
708
709
710
711
712
713
714
715
716
717
718
719
            if add_to_maintainers:
                from_maintainers = maintainers(api_url, from_project, orig_package)
                if from_maintainers:
                    to_maintainers = maintainers(api_url, to_project, orig_package)
                    if from_maintainers - to_maintainers:
                        new_maintainers = from_maintainers - to_maintainers
                        to_meta = package_meta(api_url, to_project, orig_package)
                        for userid in sorted(new_maintainers):
                            person = ElementTree.Element("person")
                            person.set("userid", userid)
                            person.set("role", "maintainer")
                            to_meta.insert(2, person)
                        osc.core.edit_meta("pkg", (to_project, orig_package), data=ElementTree.tostring(to_meta))
720

721
722
723
724
725
726
727
728
729
730
        if package is None and redo_all:
            # Check if distribution is already set in to_project
            to_prj_meta = project_meta(api_url, to_project)
            sys_repo = to_prj_meta.find('./repository[@name="System"]')
            if sys_repo is None:
                distribution = project_meta(api_url, from_project).find('./repository[@name="System"]/path[@project="distributions"]').get("repository")
            else:
                distribution = sys_repo.find('./path[@project="distributions"]').get("repository")

            print("Creating repository configuration")
731
            mpcdf_setup_repositories(api_url, to_project, distribution=distribution)
732
733
734
735
        else:
            for orig_package in from_packages:
                if not mpcdf_enable_repositories(api_url, to_project, orig_package):
                    print("ATTENTION: Not changing unmanaged package {0}".format(orig_package))