mpcdf_common.py 30.5 KB
Newer Older
1
from __future__ import print_function
Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
2
from __future__ import division
3

4
import sys
5
6
7
import osc
import osc.conf
import osc.core
Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
8
import osc.oscerr
Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
9
from functools import partial
10
11
from xml.etree import ElementTree

12
13
14
15
16
17
18
if sys.version_info[0] < 3:
    def decode_it(arg):
        return arg
else:
    from osc.util.helper import decode_it


19
20
known_microarchs = {"sandybridge", "haswell", "skylake"}

21
22
package_attributes = ["MPCDF:enable_repositories"]
config_attributes = ["MPCDF:compiler_modules", "MPCDF:cuda_modules", "MPCDF:mpi_modules", "MPCDF:pgi_modules"]
23
default_attributes = ["MPCDF:default_compiler", "MPCDF:default_cuda", "MPCDF:default_mpi", "MPCDF:default_python2", "MPCDF:default_python3"]
24

25
26
27
28
29
30
31
32
intel_parallel_studio = {
    "mpcdf_intel_parallel_studio_2017_7": {"compiler": "intel_17_0_7", "impi": "impi_2017_4", "mkl": "mkl_2017_4-module", },
    "mpcdf_intel_parallel_studio_2018_1": {"compiler": "intel_18_0_1", "impi": "impi_2018_1", "mkl": "mkl_2018_1-module", },
    "mpcdf_intel_parallel_studio_2018_2": {"compiler": "intel_18_0_2", "impi": "impi_2018_2", "mkl": "mkl_2018_2-module", },
    "mpcdf_intel_parallel_studio_2018_3": {"compiler": "intel_18_0_3", "impi": "impi_2018_3", "mkl": "mkl_2018_3-module", },
    "mpcdf_intel_parallel_studio_2018_4": {"compiler": "intel_18_0_5", "impi": "impi_2018_4", "mkl": "mkl_2018_4-module", },
    "mpcdf_intel_parallel_studio_2019_0": {"compiler": "intel_19_0_0", "impi": "impi_2019_0", "mkl": "mkl_2019_0-module", },
    "mpcdf_intel_parallel_studio_2019_1": {"compiler": "intel_19_0_1", "impi": "impi_2019_1", "mkl": "mkl_2019_1-module", },
Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
33
    "mpcdf_intel_parallel_studio_2019_3": {"compiler": "intel_19_0_3", "impi": "impi_2019_3", "mkl": "mkl_2019_3-module", },
34
    "mpcdf_intel_parallel_studio_2019_4": {"compiler": "intel_19_0_4", "impi": "impi_2019_4", "mkl": "mkl_2019_4-module", },
35
    "mpcdf_intel_parallel_studio_2019_5": {"compiler": "intel_19_0_5", "impi": "impi_2019_5", "mkl": "mkl_2019_5-module", },
36
37
38
39
40
41
42
43
}

all_mkls = {ic["mkl"] for ic in intel_parallel_studio.values()}

# Some rotations
mpi_parallel_studio = {value["impi"]: dict({"ps": key}, **value) for key, value in intel_parallel_studio.items()}
compiler_parallel_studio = {value["compiler"]: dict({"ps": key}, **value) for key, value in intel_parallel_studio.items()}

44

45
46
47
48
49
# For the autogenerated software/software:*:* project 'prjconf' sections
prjconf_start_marker = "# Autogenerated by osc mpcdf_setup_repos, do not edit till end of section\n"
prjconf_end_marker = "# End of autogenerated section\n"


50
51
52
53
def check_for_update():
    import os
    import sys
    import time
Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
54
    from subprocess import check_output, call
55
56
57
58
59
60
61
62

    if hasattr(sys.modules["mpcdf_common"], "checked_for_updates"):
        return
    else:
        setattr(sys.modules["mpcdf_common"], "checked_for_updates", 1)

    plugin_dir = os.path.dirname(os.path.realpath(__file__))
    git_dir = os.path.join(plugin_dir, ".git")
Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
63
64
65
66
67
68
69
70
    local_rev = check_output(["git", "--git-dir", git_dir, "rev-parse", "HEAD"], encoding="utf-8").strip()

    url = "https://gitlab.mpcdf.mpg.de/mpcdf/osc-plugins.git"

    def update_server_rev():
        server_rev, _ = check_output(["git", "ls-remote", url, "master"], encoding="utf-8").split(maxsplit=1)
        with open(rev_file, "w") as fd:
            fd.write(server_rev)
71
72
73
74
75

    # Check for update on server just once a day
    rev_file = os.path.join(plugin_dir, ".remote_head_rev")
    try:
        mtime = os.stat(rev_file).st_mtime
Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
76
    except EnvironmentError:
77
78
79
        mtime = 0

    if time.time() - mtime > 86400:
Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
80
        update_server_rev()
81

Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
82
    with open(rev_file, "r") as fd:
83
        server_rev = fd.read().strip()
Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
84
85
86
87
88
89
90
91
92
93
94
95
96
97

    if call(["git", "--git-dir", git_dir, "merge-base", "--is-ancestor", server_rev, "HEAD"]) == 0:
        # Server rev is older than ours. Check again
        update_server_rev()

    with open(rev_file, "r") as fd:
        server_rev = fd.read().strip()

    if server_rev != local_rev:
        if call(["git", "--git-dir", git_dir, "merge-base", "--is-ancestor", server_rev, "HEAD"]) == 0:
            print(file=sys.stderr)
            print("You have unpushed commits in", plugin_dir, "- consider pushing them", file=sys.stderr)
            print(file=sys.stderr)
        else:
98
99
100
101
102
103
104
105
            print(file=sys.stderr)
            print("Your plugin directory is out-of-date, new commits available on", url, file=sys.stderr)
            print(file=sys.stderr)


check_for_update()


106
107
108
109
110
111
112
113
def compiler_module(compiler_repo):
    return compiler_repo.replace("_", "/", 1).replace("_", ".")


def mpi_module(mpi_repo):
    return mpi_repo.replace("_", "/", 1).replace("_", ".")


114
115
116
117
118
119
120
121
def valid_pgi_mpi(pgi, mpi):
    if "impi" not in mpi:
        return False
    if "2017" in mpi:
        return False
    return True


122
def valid_mpi(compiler, mpi):
123
124
125
126
127
128
129
130
131
    """
    It might be possible to use Intel MPI libararies and compilers from
    different Parallel Studio packages, but I currently do not want to support
    it.

    Take care to keep this in sync with the file 'macros.obs_cluster' of
    the package software:dist / mpcdf_cluster_macros
    """
    if compiler.startswith("intel") and mpi.startswith("impi"):
132
        return mpi == compiler_parallel_studio[compiler]["impi"]
133
134
    if compiler.startswith("pgi"):
        return valid_pgi_mpi(compiler, mpi)
135
136
    else:
        return True
137
138


139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
def prefers(reponame):
    prefer_ps = None
    if reponame in compiler_parallel_studio:
        prefer_ps = compiler_parallel_studio[reponame]["ps"]
    else:
        for mpi in mpi_parallel_studio:
            if reponame.startswith(mpi):
                prefer_ps = mpi_parallel_studio[mpi]["ps"]

    if prefer_ps:
        preferred_mkl = intel_parallel_studio[prefer_ps]["mkl"]
        unprefer_other_mkls = ("!" + mkl for mkl in all_mkls if mkl != preferred_mkl)
        return (prefer_ps,) + tuple(unprefer_other_mkls) + (preferred_mkl,)
    else:
        return ()


156
157
158
159
def valid_cuda(cuda, compiler):
    """
    The CUDA distribution only works with certain gcc versions,
    this little function takes care that only supported combinations
Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
160
161
162
163
    are allowed.

    Take care to keep this in sync with the file 'macros.obs_cluster' of
    the package software:dist / mpcdf_cluster_macros
164
    """
Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
165
    if cuda == "cuda_8_0":
166
        return compiler == "gcc_5"
Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
167
    if cuda == "cuda_9_1":
168
        return compiler == "gcc_6_3_0"
Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
169
    if cuda == "cuda_9_2":
Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
170
        return compiler == "gcc_6"
Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
171
172
    if cuda == "cuda_10_0":
        return compiler == "gcc_6"
Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
173
174
    if cuda == "cuda_10_1":
        return compiler == "gcc_8"
175
176
177
    return False


Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
178
179
180
def project_meta(api_url, project):
    return ElementTree.fromstringlist(osc.core.show_project_meta(api_url, project))

181

182
183
184
185
186
187
188
189
190
def package_meta(api_url, project, package):
    return ElementTree.fromstringlist(osc.core.show_package_meta(api_url, project, package))


def maintainers(api_url, project, package):
    root = package_meta(api_url, project, package)
    return {e.get("userid") for e in root.findall("./person[@role='maintainer']")}


191
192
193
194
class UnsetAttributeException(Exception):
    pass


195
196
197
198
199
200
201
202
def chunked(l, chunksize):
    n = len(l)
    i = 0
    while i * chunksize < n:
        yield l[i * chunksize: (i + 1) * chunksize]
        i += 1


Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
203
def get_attribute(api_url, project, package, attribute, with_project=False):
204
205
    attribute_meta = osc.core.show_attribute_meta(api_url, project, package, None, attribute, False, with_project)
    if attribute_meta is None:
Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
206
        raise osc.oscerr.APIError("Cannot fetch value for attribute '{0}' from {1}".format(attribute, (project, package)))
207
208
209
210

    root = ElementTree.fromstringlist(attribute_meta)
    attribute = root.find("./attribute")
    if attribute is not None:
Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
211
        return root
212
    else:
213
        raise UnsetAttributeException("Attribute not set")
214

Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
215

Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
216
217
218
219
220
221
def get_attribute_values(api_url, project, package, attribute, with_project=False):
    root = get_attribute(api_url, project, package, attribute, with_project)
    attribute = root.find("./attribute")
    return list(value.text for value in attribute.findall("./value"))


222
223
224
225
226
def get_attribute_value(api_url, project, package, attribute, with_project=False):
    value, = get_attribute_values(api_url, project, package, attribute, with_project=False)
    return value


227
228
229
230
231
232
def set_attribute(api_url, project, package, attribute):
    path = ["source", project]
    if package:
        path.append(package)
    path.append("_attribute")

Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
233
    attr_meta = ElementTree.tostring(attribute, encoding=osc.core.ET_ENCODING)
Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
234

235
    url = osc.core.makeurl(api_url, path)
Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
236
    resp = ElementTree.fromstringlist(osc.core.streamfile(url, osc.core.http_POST, data=attr_meta))
Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
237
    if resp.find("./summary").text != "Ok":
Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
238
        raise osc.oscerr.APIError("Could not store attribute")
Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
239

240

241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
def set_attribute_values(api_url, project, package, attribute, values):
    root = ElementTree.Element("attributes")
    attr = ElementTree.SubElement(root, "attribute")

    namespace, name = attribute.split(":")
    attr.set("namespace", namespace)
    attr.set("name", name)

    for value in values:
        v = ElementTree.SubElement(attr, "value")
        v.text = value

    set_attribute(api_url, project, package, root)


def has_attribute(api_url, project, package, attribute):
    path = ["source", project]
    if package:
        path.append(package)
    path.append("_attribute")
    path.append(attribute)

    url = osc.core.makeurl(api_url, path)
    resp = ElementTree.fromstringlist(osc.core.streamfile(url, osc.core.http_GET))

    namespace, name = attribute.split(":")
267

268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
    for a in resp.findall("./attribute"):
        if a.get("namespace") == namespace and a.get("name") == name:
            return True
    return False


def remove_attribute(api_url, project, package, attribute_name):
    path = ["source"] + [project]
    if package:
        path.append(package)
    path.append("_attribute")
    path.append(attribute_name)

    url = osc.core.makeurl(api_url, path)
    resp = ElementTree.fromstringlist(osc.core.streamfile(url, osc.core.http_DELETE))
    if resp.find("./summary").text != "Ok":
        raise osc.oscerr.APIError("Could not remove attribute")


287
288
289
290
def get_microarchitecture(project):
    if project == "software":
        # Stupid special case
        microarch = "sandybridge"
291
    elif project.startswith("software:"):
292
        microarch = project.split(":")[2]
293
294
295
296
    elif project.startswith("home:"):
        microarch = "sandybridge"
    else:
        raise Exception("Cannot determine micro-architecture for project '{0}'".format(project))
297
298
299
300
301
302
303

    if microarch in known_microarchs:
        return microarch
    else:
        raise Exception("Unknown micro-architecture '{0}'".format(microarch))


304
def mpcdf_enable_repositories(api_url, project, package, verbose=False, filter_repos=None):
305
    from itertools import product
Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
306
    import sys
307

308
309
310
    if filter_repos is None:
        filter_repos = ()

311
312
313
    pkg_meta = osc.core.show_package_meta(api_url, project, package)
    root = ElementTree.fromstringlist(pkg_meta)

314
    build = root.find("./build")
Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
315
316
    if build is None:
        build = ElementTree.SubElement(root, "build")
317
318
319
320
321

    pkg_meta = ElementTree.tostring(root, encoding=osc.core.ET_ENCODING)

    for enable in build.findall("./enable"):
        build.remove(enable)
322

Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
323
    try:
324
325
326
327
328
        enable_repos = get_attribute_values(api_url, project, package, "MPCDF:enable_repositories")
    except Exception:
        if verbose:
            print("Warning: Could not get attribute MPCDF:enable_repositories for package {0}, skipping".format(package))
        return False
Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
329

Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
330
    def try_get_attribute(package, attribute, with_project=False):
331
332
333
        try:
            return get_attribute_values(api_url, project, package, "MPCDF:" + attribute, with_project=with_project)
        except UnsetAttributeException:
334
335
336
            print("ERROR: Attribute MPCDF:" + attribute + " is not set for "
                  + ("package '{0}'".format(package) if package else "project '{0}'".format(project))
                  + ", aborting here", file=sys.stderr)
337
            raise SystemExit(1)
Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
338

339
340
341
    compilers = try_get_attribute(package, "compiler_modules", with_project=True)
    mpis = try_get_attribute(package, "mpi_modules", with_project=True)
    cudas = try_get_attribute(package, "cuda_modules", with_project=True)
342
343
344
345
346
347
    pgis = try_get_attribute(package, "pgi_modules", with_project=True)

    all_compilers = try_get_attribute(None, "compiler_modules")
    all_mpis = try_get_attribute(None, "mpi_modules")
    all_cudas = try_get_attribute(None, "cuda_modules")
    all_pgis = try_get_attribute(None, "pgi_modules")
348
349
350
351

    default_compilers = try_get_attribute(None, "default_compiler")
    default_mpis = try_get_attribute(None, "default_mpi")
    default_cudas = try_get_attribute(None, "default_cuda")
352

353
    def sort_key(string):
354
355
        name, version = string.split("_", 1)
        version = version.split("_")
356
357
358
359
360
        return (name,) + tuple(map(int, version))

    latest_intel = sorted((c for c in all_compilers if c.startswith("intel")), key=sort_key)[-1]
    latest_gcc = sorted((c for c in all_compilers if c.startswith("gcc")), key=sort_key)[-1]

361
    def enable(name):
362
363
        if any(filtered_repo in name for filtered_repo in filter_repos):
            return
364
365
366
367
368
369
370
        node = ElementTree.Element("enable")
        node.set("repository", name)
        node.tail = "\n    "
        build.insert(0, node)
        if verbose:
            print("Enabling", name)

371
    def actual_compilers():
372
        for compiler in (c for c in compilers if c in all_compilers + ["default_compiler", "intel", "gcc", "latest_intel", "latest_gcc"]):
373
            if compiler == "intel":
Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
374
                for intel_compiler in [cc for cc in all_compilers if cc.startswith("intel")]:
375
376
                    yield intel_compiler
            elif compiler == "gcc":
Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
377
                for gcc_compiler in [cc for cc in all_compilers if cc.startswith("gcc")]:
378
379
380
381
                    yield gcc_compiler
            elif compiler == "default_compiler":
                for default_compiler in default_compilers:
                    yield default_compiler
382
383
384
385
            elif compiler == "latest_intel":
                yield latest_intel
            elif compiler == "latest_gcc":
                yield latest_gcc
386
387
388
389
            else:
                yield compiler

    def actual_mpis():
390
        for mpi in (m for m in mpis if m in all_mpis + ["default_mpi", "impi"]):
391
            if mpi == "impi":
392
                for impi in [mpi for mpi in all_mpis if mpi.startswith("impi")]:
393
394
395
396
397
398
399
400
                    yield impi
            elif mpi == "default_mpi":
                for default_mpi in default_mpis:
                    yield default_mpi
            else:
                yield mpi

    def actual_cudas():
401
        for cuda in (c for c in cudas if c in all_cudas + ["default_cuda"]):
402
403
404
405
406
407
            if cuda == "default_cuda":
                for default_cuda in default_cudas:
                    yield default_cuda
            else:
                yield cuda

408
409
410
411
    def actual_pgis():
        for pgi in (p for p in pgis if p in all_pgis):
            yield pgi

412
413
414
415
416
417
    for flag in enable_repos:

        if flag == "system":
            enable("System")

        if flag == "compilers":
418
            for compiler in actual_compilers():
Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
419
                enable(compiler)
420
421

        if flag == "mpi":
422
423
424
            for mpi, compiler in product(actual_mpis(), actual_compilers()):
                if valid_mpi(compiler, mpi):
                    enable(mpi + "_" + compiler)
425
426

        if flag == "cuda":
427
428
429
            for cuda, compiler in product(actual_cudas(), all_compilers):
                if valid_cuda(cuda, compiler):
                    enable(cuda + "_" + compiler)
430
431

        if flag == "cuda_mpi":
432
433
            for cuda, mpi, compiler in product(actual_cudas(), actual_mpis(), all_compilers):
                if valid_cuda(cuda, compiler) and valid_mpi(compiler, mpi):
434
                    enable(cuda + "_" + mpi + "_" + compiler)
435

436
437
438
439
440
441
        if flag == "pgi":
            for pgi in actual_pgis():
                enable(pgi)

        if flag == "pgi_mpi":
            for mpi, pgi in product(actual_mpis(), actual_pgis()):
442
443
                if valid_pgi_mpi(pgi, mpi):
                    enable(mpi + "_" + pgi)
444

Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
445
446
    if len(build.getchildren()) > 0:
        build.getchildren()[-1].tail = "\n  "
447

448
449
    new_pkg_meta = ElementTree.tostring(root, encoding=osc.core.ET_ENCODING)
    if pkg_meta != new_pkg_meta:
450
        print("Updating repositories for", package)
451
        osc.core.edit_meta("pkg", (project, package), data=new_pkg_meta)
452
453

    return True
Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
454
455


456
457
def mpcdf_setup_repositories(api_url, project, microarchitecture=None, distribution=None, parent=None,
                             packages=None, dry_run=False, filter_repos=None, only_project=False, remove_old=False):
458
459
    import threading

Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
460
    if parent:
461
462
463
        for attribute in config_attributes + default_attributes:
            print("Copying attribute '{0}' from parent project".format(attribute))
            set_attribute(api_url, project, None, get_attribute(api_url, parent, None, attribute))
Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
464

465
466
467
468
469
470
    compilers = get_attribute_values(api_url, project, None, "MPCDF:compiler_modules")
    mpis = get_attribute_values(api_url, project, None, "MPCDF:mpi_modules")
    cudas = get_attribute_values(api_url, project, None, "MPCDF:cuda_modules")
    pgis = get_attribute_values(api_url, project, None, "MPCDF:pgi_modules")
    default_python2 = get_attribute_value(api_url, project, None, "MPCDF:default_python2")
    default_python3 = get_attribute_value(api_url, project, None, "MPCDF:default_python3")
Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
471

472
473
    if distribution is None:
        # Get existing value from project meta
474
475
476
477
478
479
        dist_repo = project_meta(api_url, project if parent is None else parent).find(
            "./repository[@name='System']/path[@project='distributions']")
        if dist_repo is not None:
            distribution = dist_repo.get("repository")
        else:
            raise osc.oscerr.WrongArgs("ERROR: Specify distribution or parent project")
480
481
        print("Using '{0}' as base distribution".format(distribution))

Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
482
483
484
    distributions = project_meta(api_url, "distributions")
    dist_repo = distributions.find('./repository[@name="{0}"]'.format(distribution))
    if dist_repo is None:
Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
485
        raise osc.oscerr.WrongArgs("No repository '{0}' is defined in project 'distributions' on the server".format(distribution))
Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
486
487
    architectures = list(arch.text for arch in dist_repo.findall("./arch"))

488
489
490
    if microarchitecture is None:
        microarchitecture = get_microarchitecture(project)

Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
491
492
    root = project_meta(api_url, project)

Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
493
    prjconf = list(map(decode_it, osc.core.show_project_conf(api_url, project)))
Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
494
495

    try:
496
497
        start = prjconf.index(prjconf_start_marker)
        end = prjconf.index(prjconf_end_marker)
Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
498
499
500
501
502
    except ValueError:
        start = None
        end = len(prjconf)

    prjconf_head = "".join(prjconf[:start])
Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
503
    prjconf_tail = "".join(prjconf[end + 1:])
504
    prjconf = [prjconf_start_marker]
Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
505

506
507
508
509
510
511
512
513
514
    prjconf.append("Constraint: hostlabel {0}".format(microarchitecture))
    prjconf.append(
        """
%if %_repository != System
Macros:
%microarchitecture {0}
:Macros
%endif""".format(microarchitecture))
    prjconf.append("")
515
516
517
518
    prjconf.append("Prefer: mpcdf_python2_" + default_python2)
    prjconf.append("Prefer: mpcdf_python3_" + default_python3)
    prjconf.append("")

Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
519
    # Remove existing repositories
520
521
522
    if remove_old:
        for oldrepo in root.findall("./repository"):
            root.remove(oldrepo)
Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
523
524

    def repo(name, *dependencies, **kwargs):
525
        is_compiler = kwargs.pop("compiler", False)
Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
526
527
        is_mpi = kwargs.pop("mpi", False)
        is_cuda = kwargs.pop("cuda", False)
528
529
        is_cuda_mpi = kwargs.pop("cuda_mpi", False)

530
531
        cuda_repo = kwargs.pop("cuda_repo", "")

532
533
534
535
        have_compiler = is_compiler or is_mpi or is_cuda or is_cuda_mpi
        have_mpi = is_mpi or is_cuda_mpi
        have_cuda = is_cuda or is_cuda_mpi

536
537
538
539
540
541
        existing_repo = root.find("./repository[@name='{0}']".format(name))
        if existing_repo is not None:
            root.remove(existing_repo)
        else:
            print("New repository", name)

Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
542
543
        r = ElementTree.SubElement(root, "repository")
        r.set("name", name)
Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
544
545
        r.text = "\n    "

Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
546
547
548
549
        def path(project, repo):
            p = ElementTree.SubElement(r, "path")
            p.set("project", project)
            p.set("repository", repo)
Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
550
            p.tail = "\n    "
Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
551

552
553
        if parent:
            path(parent, name)
554
555
        for dep_project, dep_repo in dependencies:
            path(dep_project, dep_repo)
Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
556
557
558
559
560
561
562
563

        for arch in architectures:
            a = ElementTree.SubElement(r, "arch")
            a.text = arch
            a.tail = "\n    "
        a.tail = "\n  "
        r.tail = "\n  "

Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
564
565
566
567
        # In order to be able to figure out the matching MKL version for a given
        # compiler/MPI repository we emit a new macro '%matching_mkl_version' in
        # the cases this makes sense
        matching_mkl = []
Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
568
        prjconf.append("%if %_repository == {0}".format(name))
569
        for prefer in prefers(name):
Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
570
571
            if prefer.startswith("mkl_"):
                matching_mkl.append(prefer)
572
            prjconf.append("Prefer: " + prefer)
Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
573

Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
574
        prjconf.append("Macros:")
575
576

        prjconf.append("%is_compiler_repository {0}".format(1 if is_compiler else 0))
Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
577
578
        prjconf.append("%is_mpi_repository {0}".format(1 if is_mpi else 0))
        prjconf.append("%is_cuda_repository {0}".format(1 if is_cuda else 0))
579
580
581
582
583
584
        prjconf.append("%is_cuda_mpi_repository {0}".format(1 if is_cuda_mpi else 0))

        prjconf.append("%have_mpcdf_compiler {0}".format(1 if have_compiler else 0))
        prjconf.append("%have_mpcdf_mpi {0}".format(1 if have_mpi else 0))
        prjconf.append("%have_mpcdf_cuda {0}".format(1 if have_cuda else 0))

585
586
587
        if is_cuda:
            prjconf.append("%cuda_repository {0}".format(cuda_repo))

Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
588
589
590
591
592
593
        if matching_mkl:
            matching_mkl, = matching_mkl
            matching_mkl, _ = matching_mkl[len("mkl_"):].split("-module")
            matching_mkl = matching_mkl.replace("_", ".")
            prjconf.append("%matching_mkl_version {0}".format(matching_mkl))

594
595
596
        for macro, value in kwargs.items():
            prjconf.append("%{0} {1}".format(macro, value))

Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
597
598
599
600
        prjconf.append(":Macros")
        prjconf.append("%endif")
        prjconf.append("")

Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
601
602
603
604
    if parent:
        repo("System")
    else:
        repo("System", ("distributions", distribution))
Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
605

606
    for compiler in compilers + pgis:
607
        repo(compiler, (project, "System"), compiler=True,
608
             compiler_repository=compiler, compiler_module=compiler_module(compiler))
Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
609
610

        for mpi in filter(partial(valid_mpi, compiler), mpis):
611
            repo(mpi + "_" + compiler, (project, compiler), mpi=True,
612
                 mpi_repository=mpi, mpi_module=mpi_module(mpi))
Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
613
614
615

    for cuda in cudas:
        for compiler in filter(partial(valid_cuda, cuda), compilers):
616
            repo(cuda + "_" + compiler, (project, compiler), cuda=True, cuda_repo=cuda)
Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
617
618
619
620
            for mpi in filter(partial(valid_mpi, compiler), mpis):
                repo(cuda + "_" + mpi + "_" + compiler,
                     (project, cuda + "_" + compiler),
                     (project, mpi + "_" + compiler),
621
                     cuda_mpi=True)
Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
622
623
624
625
626
627
628
629
630
631
632
633

    # Remove build configuration
    build = root.find("./build")
    if build is None:
        build = ElementTree.Element("build")
        build.text = "\n    "
        disable = ElementTree.SubElement(build, "disable")
        disable.tail = "\n  "
        build.tail = "\n  "
        root.insert(list(root).index(root.find("./repository")), build)

    root.getchildren()[-1].tail = "\n"
Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
634
    prj = ElementTree.tostring(root, encoding=osc.core.ET_ENCODING)
Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
635

636
    prjconf.append(prjconf_end_marker)
Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
637
638
639
640
641
642
    prjconf = prjconf_head + "\n".join(prjconf) + prjconf_tail

    if dry_run:
        print("osc meta prj {0} -F - <<EOF\n{1}\nEOF\n".format(project, prj))
        print("osc meta prjconf {0} -F - <<EOF\n{1}\nEOF\n".format(project, prjconf))
    else:
643
644
645
646
647
        if not only_project:
            # First set-up the <enable/> flags, that way no
            # spurious builds are launched
            if packages is None:
                packages = osc.core.meta_get_packagelist(api_url, project)
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666

            if len(packages) > 40:
                chunksize = len(packages) // 20
            else:
                chunksize = len(packages)

            def work(packagelist):
                for package in packagelist:
                    if not mpcdf_enable_repositories(api_url, project, package, filter_repos=filter_repos):
                        print("ATTENTION: Not changing unmanaged package {0}".format(package))

            threads = []
            for packagelist in chunked(packages, chunksize):
                t = threading.Thread(target=work, args=(packagelist,))
                threads.append(t)
                t.start()

            for t in threads:
                t.join()
Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
667
668
669

        # Update repositories
        print("Updating prj meta")
670
        osc.core.edit_meta("prj", project, data=prj, force=True)
Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
671
672
        print("Updating prjconf meta")
        osc.core.edit_meta("prjconf", project, data=prjconf)
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712


def set_as_branch(api_url, my_project, my_package, main_project, main_package):
    import os

    def dirmeta(project, package):
        url = osc.core.makeurl(api_url, ["source", project, package])
        return ElementTree.fromstringlist(osc.core.streamfile(url, osc.core.http_GET))

    srcmeta = dirmeta(my_project, my_package)
    dstmeta = dirmeta(main_project, main_package)

    linkinfo = srcmeta.find('./linkinfo')
    if linkinfo is not None:
        print("ERROR: package {0} is already linked to package {1} in project {2}".format(
              my_package, linkinfo.get("package"), linkinfo.get("project")))
        return False

    linkfile = ElementTree.Element("link")
    linkfile.set("project", main_project)
    linkfile.set("package", main_package)
    linkfile.set("baserev", dstmeta.get("srcmd5"))
    patches = ElementTree.SubElement(linkfile, "patches")
    ElementTree.SubElement(patches, "branch")

    url = osc.core.makeurl(api_url, ["source", my_project, my_package, "_link"])
    resp = ElementTree.fromstringlist(osc.core.streamfile(url, osc.core.http_PUT, data=ElementTree.tostring(linkfile)))
    rev = resp.get("rev")
    if rev is None:
        print("ERROR: Could not commit _link file")
        return False
    else:
        print("Commited as revision", rev)
        this_package = osc.core.store_read_package(os.curdir)
        this_project = osc.core.store_read_project(os.curdir)
        if this_package == my_package and this_project == my_project:
            pac = osc.core.filedir_to_pac(os.curdir)
            rev = pac.latest_rev(expand=True)
            pac.update(rev)
    return True
713
714


715
def sync_projects(api_url, package=None, from_project="software", to_projects=None, redo_all=False, add_to_maintainers=True):
716
717
718
719
    if package is not None and redo_all:
        raise osc.oscerr.WrongArgs('Cannot specify `redo_all` and package')

    if to_projects is None:
Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
720
721
        to_projects = [p for p in osc.core.meta_get_project_list(api_url)
                       if p.startswith("software:") and not (p == "software:dist" or p == "software:images")]
722
723
724
725
726
727
728
729
730
731
732
733
734

    for to_project in to_projects:
        print("Syncing {0} with {1}".format(to_project, from_project))

        to_packages = osc.core.meta_get_packagelist(api_url, to_project)

        if package is None:
            from_packages = osc.core.meta_get_packagelist(api_url, from_project)
            if not redo_all:
                from_packages = list(sorted(set(from_packages) - set(to_packages)))
        else:
            from_packages = [package]

735
        for attribute in config_attributes:
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
            try:
                get_attribute(api_url, to_project, None, attribute)
            except Exception:
                attr = get_attribute(api_url, from_project, None, attribute)
                print("Setting attribute", attribute)
                set_attribute(api_url, to_project, None, attr)

        for orig_package in from_packages:
            if orig_package not in to_packages:
                filelist = osc.core.meta_get_filelist(api_url, from_project, orig_package)
                if "_link" in filelist:
                    print("Not branching package {0}, is a link".format(orig_package))
                else:
                    print("Branching package {0}".format(orig_package))
                    exists, targetprj, targetpkg, srcprj, srcpkg = \
                        osc.core.branch_pkg(api_url, from_project, orig_package, target_project=to_project, nodevelproject=True)
            else:
                print("Not branching package {0}, already present in target".format(orig_package))

755
            for attribute in package_attributes + config_attributes:
756
757
758
759
760
761
762
763
                try:
                    attr = get_attribute(api_url, from_project, orig_package, attribute)
                except UnsetAttributeException:
                    if has_attribute(api_url, to_project, orig_package, attribute):
                        remove_attribute(api_url, to_project, orig_package, attribute)
                    continue
                set_attribute(api_url, to_project, orig_package, attr)

764
765
766
767
768
769
770
771
772
773
774
775
776
            if add_to_maintainers:
                from_maintainers = maintainers(api_url, from_project, orig_package)
                if from_maintainers:
                    to_maintainers = maintainers(api_url, to_project, orig_package)
                    if from_maintainers - to_maintainers:
                        new_maintainers = from_maintainers - to_maintainers
                        to_meta = package_meta(api_url, to_project, orig_package)
                        for userid in sorted(new_maintainers):
                            person = ElementTree.Element("person")
                            person.set("userid", userid)
                            person.set("role", "maintainer")
                            to_meta.insert(2, person)
                        osc.core.edit_meta("pkg", (to_project, orig_package), data=ElementTree.tostring(to_meta))
777

778
779
780
781
782
783
784
785
786
787
        if package is None and redo_all:
            # Check if distribution is already set in to_project
            to_prj_meta = project_meta(api_url, to_project)
            sys_repo = to_prj_meta.find('./repository[@name="System"]')
            if sys_repo is None:
                distribution = project_meta(api_url, from_project).find('./repository[@name="System"]/path[@project="distributions"]').get("repository")
            else:
                distribution = sys_repo.find('./path[@project="distributions"]').get("repository")

            print("Creating repository configuration")
788
            mpcdf_setup_repositories(api_url, to_project, distribution=distribution)
789
790
791
792
        else:
            for orig_package in from_packages:
                if not mpcdf_enable_repositories(api_url, to_project, orig_package):
                    print("ATTENTION: Not changing unmanaged package {0}".format(orig_package))