mpcdf_common.py 35.4 KB
Newer Older
1
from __future__ import print_function
Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
2
from __future__ import division
3

4
import sys
5
6
7
import osc
import osc.conf
import osc.core
Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
8
import osc.oscerr
9
import textwrap
Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
10
from functools import partial
11
12
from xml.etree import ElementTree

13
14
15
16
17
18
19
if sys.version_info[0] < 3:
    def decode_it(arg):
        return arg
else:
    from osc.util.helper import decode_it


20
21
known_microarchs = {"sandybridge", "haswell", "skylake"}

22
23
package_attributes = ["MPCDF:enable_repositories"]
config_attributes = ["MPCDF:compiler_modules", "MPCDF:cuda_modules", "MPCDF:mpi_modules", "MPCDF:pgi_modules"]
24
default_attributes = ["MPCDF:default_compiler", "MPCDF:default_cuda", "MPCDF:default_mpi"]
25

26
27
28
29
30
31
32
33
intel_parallel_studio = {
    "mpcdf_intel_parallel_studio_2017_7": {"compiler": "intel_17_0_7", "impi": "impi_2017_4", "mkl": "mkl_2017_4-module", },
    "mpcdf_intel_parallel_studio_2018_1": {"compiler": "intel_18_0_1", "impi": "impi_2018_1", "mkl": "mkl_2018_1-module", },
    "mpcdf_intel_parallel_studio_2018_2": {"compiler": "intel_18_0_2", "impi": "impi_2018_2", "mkl": "mkl_2018_2-module", },
    "mpcdf_intel_parallel_studio_2018_3": {"compiler": "intel_18_0_3", "impi": "impi_2018_3", "mkl": "mkl_2018_3-module", },
    "mpcdf_intel_parallel_studio_2018_4": {"compiler": "intel_18_0_5", "impi": "impi_2018_4", "mkl": "mkl_2018_4-module", },
    "mpcdf_intel_parallel_studio_2019_0": {"compiler": "intel_19_0_0", "impi": "impi_2019_0", "mkl": "mkl_2019_0-module", },
    "mpcdf_intel_parallel_studio_2019_1": {"compiler": "intel_19_0_1", "impi": "impi_2019_1", "mkl": "mkl_2019_1-module", },
Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
34
    "mpcdf_intel_parallel_studio_2019_3": {"compiler": "intel_19_0_3", "impi": "impi_2019_3", "mkl": "mkl_2019_3-module", },
35
    "mpcdf_intel_parallel_studio_2019_4": {"compiler": "intel_19_0_4", "impi": "impi_2019_4", "mkl": "mkl_2019_4-module", },
36
    "mpcdf_intel_parallel_studio_2019_5": {"compiler": "intel_19_0_5", "impi": "impi_2019_5", "mkl": "mkl_2019_5-module", },
Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
37
    "mpcdf_intel_parallel_studio_2020": {"compiler": "intel_19_1_0", "impi": "impi_2019_6", "mkl": "mkl_2020-module", },
38
39
40
41
42
43
44
45
}

all_mkls = {ic["mkl"] for ic in intel_parallel_studio.values()}

# Some rotations
mpi_parallel_studio = {value["impi"]: dict({"ps": key}, **value) for key, value in intel_parallel_studio.items()}
compiler_parallel_studio = {value["compiler"]: dict({"ps": key}, **value) for key, value in intel_parallel_studio.items()}

46

47
48
49
50
# For the autogenerated software/software:*:* project 'prjconf' sections
prjconf_start_marker = "# Autogenerated by osc mpcdf_setup_repos, do not edit till end of section\n"
prjconf_end_marker = "# End of autogenerated section\n"

51

Lorenz Hüdepohl's avatar
Lorenz Hüdepohl committed
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
def dist_prjconf_tags(distribution):
    centos_prjconf_tags = textwrap.dedent(
        """
        Prefer: perl-Error
        Substitute: c_compiler gcc
        Substitute: c++_compiler gcc-c++
        Substitute: ca-certificates-mozilla ca-certificates
        """).strip()

    centos8_prjconf_tags = textwrap.dedent(
        """
        ExpandFlags: module:python36-3.6
        """).strip()

    res = ""
    if "CentOS" in distribution:
        res += centos_prjconf_tags
    if "CentOS_8" in distribution:
        res += "\n" + centos8_prjconf_tags

    return res


def dist_prjconf_macros(distribution):
    centos_macros = textwrap.dedent(
        """
        # Disable all problematic automatic RPM stuff
        # like byte-compiling (with the wrong Python version)
        # or debug packages that fail for many binary-only packages
        %__no_python_bytecompile 1
        %debug_package %{nil}
        """).strip()

    res = ""
    if "CentOS" in distribution:
        res += centos_macros

    return res
90

91

92
93
94
95
def check_for_update():
    import os
    import sys
    import time
96
97
98
    from subprocess import check_output, call

    DEVNULL = open(os.devnull, 'w')
99
100
101
102
103
104

    if hasattr(sys.modules["mpcdf_common"], "checked_for_updates"):
        return
    else:
        setattr(sys.modules["mpcdf_common"], "checked_for_updates", 1)

105
    if (sys.version_info > (3, 0)):
Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
106
        set_encoding = {"encoding": "utf-8"}
107
108
109
    else:
        set_encoding = {}

110
111
    plugin_dir = os.path.dirname(os.path.realpath(__file__))
    git_dir = os.path.join(plugin_dir, ".git")
112
    local_rev = check_output(["git", "--git-dir", git_dir, "rev-parse", "HEAD"], **set_encoding).strip()
Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
113

Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
114
    url = "https://gitlab.mpcdf.mpg.de/mpcdf/obs/osc-plugins.git"
Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
115
116

    def update_server_rev():
117
        server_rev, _ = check_output(["git", "ls-remote", url, "master"], **set_encoding).split(None, 1)
Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
118
119
        with open(rev_file, "w") as fd:
            fd.write(server_rev)
120
121
122
123
124

    # Check for update on server just once a day
    rev_file = os.path.join(plugin_dir, ".remote_head_rev")
    try:
        mtime = os.stat(rev_file).st_mtime
Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
125
    except EnvironmentError:
126
127
128
        mtime = 0

    if time.time() - mtime > 86400:
Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
129
        update_server_rev()
130

Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
131
    with open(rev_file, "r") as fd:
132
        server_rev = fd.read().strip()
Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
133

134
135
136
137
    if server_rev != local_rev:
        if call(["git", "--git-dir", git_dir, "merge-base", "--is-ancestor", server_rev, "HEAD"], stderr=DEVNULL) == 0:
            # Server rev is older than ours. Check again
            update_server_rev()
Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
138
139
140
141
142

    with open(rev_file, "r") as fd:
        server_rev = fd.read().strip()

    if server_rev != local_rev:
Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
143
        print("Note from MPCDF osc plugins:", file=sys.stderr)
144
        if call(["git", "--git-dir", git_dir, "merge-base", "--is-ancestor", server_rev, "HEAD"], stderr=DEVNULL) == 0:
Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
145
            print(" You have unpushed commits in", plugin_dir, "- consider pushing them", file=sys.stderr)
Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
146
147
            print(file=sys.stderr)
        else:
Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
148
149
            print(" Your plugin directory is out-of-date, new commits available on", url, file=sys.stderr)
            print(" (careful: the repo has been moved, update your path to point to the new URL)", file=sys.stderr)
150
151
152
153
154
155
            print(file=sys.stderr)


check_for_update()


156
157
158
159
160
161
162
163
def compiler_module(compiler_repo):
    return compiler_repo.replace("_", "/", 1).replace("_", ".")


def mpi_module(mpi_repo):
    return mpi_repo.replace("_", "/", 1).replace("_", ".")


164
def valid_pgi_mpi(pgi, mpi):
165
166
167
168
169
170
171
172
173
174
    if "impi" in mpi:
        if "2017" in mpi:
            return False
        else:
            return True

    if "openmpi_4" == mpi:
        return True

    return False
175
176


177
def valid_mpi(compiler, mpi):
178
179
180
181
182
183
184
185
186
    """
    It might be possible to use Intel MPI libararies and compilers from
    different Parallel Studio packages, but I currently do not want to support
    it.

    Take care to keep this in sync with the file 'macros.obs_cluster' of
    the package software:dist / mpcdf_cluster_macros
    """
    if compiler.startswith("intel") and mpi.startswith("impi"):
187
        return mpi == compiler_parallel_studio[compiler]["impi"]
188
189
    if compiler.startswith("pgi"):
        return valid_pgi_mpi(compiler, mpi)
190
191
    else:
        return True
192
193


194
195
196
197
198
199
200
201
202
203
204
def prefers(reponame):
    prefer_ps = None
    if reponame in compiler_parallel_studio:
        prefer_ps = compiler_parallel_studio[reponame]["ps"]
    else:
        for mpi in mpi_parallel_studio:
            if reponame.startswith(mpi):
                prefer_ps = mpi_parallel_studio[mpi]["ps"]

    if prefer_ps:
        preferred_mkl = intel_parallel_studio[prefer_ps]["mkl"]
205
        unprefer_other_mkls = sorted("!" + mkl for mkl in all_mkls if mkl != preferred_mkl)
206
207
208
209
210
        return (prefer_ps,) + tuple(unprefer_other_mkls) + (preferred_mkl,)
    else:
        return ()


211
212
213
214
def valid_cuda(cuda, compiler):
    """
    The CUDA distribution only works with certain gcc versions,
    this little function takes care that only supported combinations
Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
215
216
217
218
    are allowed.

    Take care to keep this in sync with the file 'macros.obs_cluster' of
    the package software:dist / mpcdf_cluster_macros
219
    """
Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
220
    if cuda == "cuda_8_0":
221
        return compiler == "gcc_5"
Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
222
    if cuda == "cuda_9_1":
223
        return compiler == "gcc_6_3_0"
Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
224
    if cuda == "cuda_9_2":
Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
225
        return compiler == "gcc_6"
Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
226
227
    if cuda == "cuda_10_0":
        return compiler == "gcc_6"
Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
228
229
    if cuda == "cuda_10_1":
        return compiler == "gcc_8"
Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
230
231
    if cuda == "cuda_10_2":
        return compiler == "gcc_8"
232
233
234
    return False


Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
235
236
237
def project_meta(api_url, project):
    return ElementTree.fromstringlist(osc.core.show_project_meta(api_url, project))

238

239
240
241
242
243
244
245
246
247
def package_meta(api_url, project, package):
    return ElementTree.fromstringlist(osc.core.show_package_meta(api_url, project, package))


def maintainers(api_url, project, package):
    root = package_meta(api_url, project, package)
    return {e.get("userid") for e in root.findall("./person[@role='maintainer']")}


248
249
250
251
class UnsetAttributeException(Exception):
    pass


252
253
254
255
class UnmanagedPackageException(Exception):
    pass


256
257
258
259
260
261
262
263
def chunked(l, chunksize):
    n = len(l)
    i = 0
    while i * chunksize < n:
        yield l[i * chunksize: (i + 1) * chunksize]
        i += 1


Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
264
def get_attribute(api_url, project, package, attribute, with_project=False):
265
266
    attribute_meta = osc.core.show_attribute_meta(api_url, project, package, None, attribute, False, with_project)
    if attribute_meta is None:
Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
267
        raise osc.oscerr.APIError("Cannot fetch value for attribute '{0}' from {1}".format(attribute, (project, package)))
268
269
270
271

    root = ElementTree.fromstringlist(attribute_meta)
    attribute = root.find("./attribute")
    if attribute is not None:
Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
272
        return root
273
    else:
274
        raise UnsetAttributeException("Attribute not set")
275

Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
276

277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
def overloaded_project_attribute(api_url, project, attribute):
    try:
        return get_attribute_values(api_url, project, None, attribute)
    except UnsetAttributeException:
        return get_attribute_values(api_url, "software", None, attribute)


def overloaded_package_attribute(api_url, project, package, attribute):
    try:
        return get_attribute_values(api_url, project, package, attribute)
    except UnsetAttributeException:
        pass

    try:
        return get_attribute_values(api_url, project, None, attribute)
    except UnsetAttributeException:
        pass

    return get_attribute_values(api_url, "software", None, attribute)


def get_allowed_attribute_values(api_url, attribute):
    path = ["attribute"] + attribute.split(":") + ["_meta"]
    url = osc.core.makeurl(api_url, path, [])
    try:
        f = osc.core.http_GET(url)
    except osc.core.HTTPError as e:
        e.osc_msg = 'Error getting meta for attribute "{0}"'.format(attribute)
        raise

    root = ElementTree.fromstringlist(f.readlines())
    return list(value.text for value in root.findall("./allowed/value"))


Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
311
312
313
314
315
316
def get_attribute_values(api_url, project, package, attribute, with_project=False):
    root = get_attribute(api_url, project, package, attribute, with_project)
    attribute = root.find("./attribute")
    return list(value.text for value in attribute.findall("./value"))


317
318
319
320
321
def get_attribute_value(api_url, project, package, attribute, with_project=False):
    value, = get_attribute_values(api_url, project, package, attribute, with_project=False)
    return value


322
323
324
325
326
327
def set_attribute(api_url, project, package, attribute):
    path = ["source", project]
    if package:
        path.append(package)
    path.append("_attribute")

Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
328
    attr_meta = ElementTree.tostring(attribute, encoding=osc.core.ET_ENCODING)
Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
329

330
    url = osc.core.makeurl(api_url, path)
Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
331
    resp = ElementTree.fromstringlist(osc.core.streamfile(url, osc.core.http_POST, data=attr_meta))
Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
332
    if resp.find("./summary").text != "Ok":
Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
333
        raise osc.oscerr.APIError("Could not store attribute")
Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
334

335

336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
def set_attribute_values(api_url, project, package, attribute, values):
    root = ElementTree.Element("attributes")
    attr = ElementTree.SubElement(root, "attribute")

    namespace, name = attribute.split(":")
    attr.set("namespace", namespace)
    attr.set("name", name)

    for value in values:
        v = ElementTree.SubElement(attr, "value")
        v.text = value

    set_attribute(api_url, project, package, root)


def has_attribute(api_url, project, package, attribute):
    path = ["source", project]
    if package:
        path.append(package)
    path.append("_attribute")
    path.append(attribute)

    url = osc.core.makeurl(api_url, path)
    resp = ElementTree.fromstringlist(osc.core.streamfile(url, osc.core.http_GET))

    namespace, name = attribute.split(":")
362

363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
    for a in resp.findall("./attribute"):
        if a.get("namespace") == namespace and a.get("name") == name:
            return True
    return False


def remove_attribute(api_url, project, package, attribute_name):
    path = ["source"] + [project]
    if package:
        path.append(package)
    path.append("_attribute")
    path.append(attribute_name)

    url = osc.core.makeurl(api_url, path)
    resp = ElementTree.fromstringlist(osc.core.streamfile(url, osc.core.http_DELETE))
    if resp.find("./summary").text != "Ok":
        raise osc.oscerr.APIError("Could not remove attribute")


382
def get_microarchitecture(project):
383
    if project.startswith("software:"):
384
        microarch = project.split(":")[2]
385
386
387
388
    elif project.startswith("home:"):
        microarch = "sandybridge"
    else:
        raise Exception("Cannot determine micro-architecture for project '{0}'".format(project))
389
390
391
392
393
394
395

    if microarch in known_microarchs:
        return microarch
    else:
        raise Exception("Unknown micro-architecture '{0}'".format(microarch))


396
397
398
399
400
401
def package_sort_key(string):
    name, version = string.split("_", 1)
    version = version.split("_")
    return (name,) + tuple(map(int, version))


402
def mpcdf_enable_repositories(api_url, project, package, verbose=False, dry_run=False, ignore_repos=()):
403
404
    from itertools import product

405
406
407
    pkg_meta = osc.core.show_package_meta(api_url, project, package)
    root = ElementTree.fromstringlist(pkg_meta)

408
    build = root.find("./build")
Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
409
410
    if build is None:
        build = ElementTree.SubElement(root, "build")
411
412
413
414
415

    pkg_meta = ElementTree.tostring(root, encoding=osc.core.ET_ENCODING)

    for enable in build.findall("./enable"):
        build.remove(enable)
416

Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
417
    try:
418
419
420
421
        enable_repos = get_attribute_values(api_url, project, package, "MPCDF:enable_repositories")
    except Exception:
        if verbose:
            print("Warning: Could not get attribute MPCDF:enable_repositories for package {0}, skipping".format(package))
422
        raise UnmanagedPackageException()
Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
423

424
425
    if project == "software":
        distributions = list(sorted(repo.name for repo in osc.core.get_repos_of_project(api_url, "software")))
Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
426

427
428
429
430
431
    else:
        compilers = overloaded_package_attribute(api_url, project, package, "MPCDF:compiler_modules")
        mpis = overloaded_package_attribute(api_url, project, package, "MPCDF:mpi_modules")
        cudas = overloaded_package_attribute(api_url, project, package, "MPCDF:cuda_modules")
        pgis = overloaded_package_attribute(api_url, project, package, "MPCDF:pgi_modules")
432

433
434
435
436
        all_compilers = overloaded_project_attribute(api_url, project, "MPCDF:compiler_modules")
        all_mpis = overloaded_project_attribute(api_url, project, "MPCDF:mpi_modules")
        all_cudas = overloaded_project_attribute(api_url, project, "MPCDF:cuda_modules")
        all_pgis = overloaded_project_attribute(api_url, project, "MPCDF:pgi_modules")
437

438
439
440
        default_compilers = overloaded_project_attribute(api_url, project, "MPCDF:default_compiler")
        default_mpis = overloaded_project_attribute(api_url, project, "MPCDF:default_mpi")
        default_cudas = overloaded_project_attribute(api_url, project, "MPCDF:default_cuda")
441

442
443
        latest_intel = sorted((c for c in all_compilers if c.startswith("intel")), key=package_sort_key)[-1]
        latest_gcc = sorted((c for c in all_compilers if c.startswith("gcc")), key=package_sort_key)[-1]
444

445
    def enable(name):
446
        if name in ignore_repos:
447
            return
448
449
450
451
452
453
454
        node = ElementTree.Element("enable")
        node.set("repository", name)
        node.tail = "\n    "
        build.insert(0, node)
        if verbose:
            print("Enabling", name)

455
    def actual_compilers():
456
        for compiler in (c for c in compilers if c in all_compilers + ["default_compiler", "intel", "gcc", "latest_intel", "latest_gcc"]):
457
            if compiler == "intel":
Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
458
                for intel_compiler in [cc for cc in all_compilers if cc.startswith("intel")]:
459
460
                    yield intel_compiler
            elif compiler == "gcc":
Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
461
                for gcc_compiler in [cc for cc in all_compilers if cc.startswith("gcc")]:
462
463
464
465
                    yield gcc_compiler
            elif compiler == "default_compiler":
                for default_compiler in default_compilers:
                    yield default_compiler
466
467
468
469
            elif compiler == "latest_intel":
                yield latest_intel
            elif compiler == "latest_gcc":
                yield latest_gcc
470
471
472
473
            else:
                yield compiler

    def actual_mpis():
474
        for mpi in (m for m in mpis if m in all_mpis + ["default_mpi", "impi"]):
475
            if mpi == "impi":
476
                for impi in [mpi for mpi in all_mpis if mpi.startswith("impi")]:
477
478
479
480
481
482
483
484
                    yield impi
            elif mpi == "default_mpi":
                for default_mpi in default_mpis:
                    yield default_mpi
            else:
                yield mpi

    def actual_cudas():
485
        for cuda in (c for c in cudas if c in all_cudas + ["default_cuda"]):
486
487
488
489
490
491
            if cuda == "default_cuda":
                for default_cuda in default_cudas:
                    yield default_cuda
            else:
                yield cuda

492
493
494
495
    def actual_pgis():
        for pgi in (p for p in pgis if p in all_pgis):
            yield pgi

496
    for flag in enable_repos:
497
498
499
500
        if project == "software":
            if flag == "system":
                for distribution in distributions:
                    enable(distribution)
501

502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
        else:
            if flag == "compilers":
                for compiler in actual_compilers():
                    enable(compiler)

            if flag == "mpi":
                for mpi, compiler in product(actual_mpis(), actual_compilers()):
                    if valid_mpi(compiler, mpi):
                        enable(mpi + "_" + compiler)

            if flag == "cuda":
                for cuda, compiler in product(actual_cudas(), all_compilers):
                    if valid_cuda(cuda, compiler):
                        enable(cuda + "_" + compiler)

            if flag == "cuda_mpi":
                for cuda, mpi, compiler in product(actual_cudas(), actual_mpis(), all_compilers):
                    if valid_cuda(cuda, compiler) and valid_mpi(compiler, mpi):
                        enable(cuda + "_" + mpi + "_" + compiler)

            if flag == "pgi":
                for pgi in actual_pgis():
                    enable(pgi)

            if flag == "pgi_mpi":
                for mpi, pgi in product(actual_mpis(), actual_pgis()):
                    if valid_pgi_mpi(pgi, mpi):
                        enable(mpi + "_" + pgi)
530

Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
531
532
    if len(build.getchildren()) > 0:
        build.getchildren()[-1].tail = "\n  "
533

534
535
    new_pkg_meta = ElementTree.tostring(root, encoding=osc.core.ET_ENCODING)
    if pkg_meta != new_pkg_meta:
536
        print("Updating repositories for", package)
537
538
539
        if dry_run:
            print("osc meta pkg {0} {1} -F - <<EOF\n{2}\nEOF\n".format(project, package, new_pkg_meta))
        else:
540
            osc.core.edit_meta("pkg", (project, package), data=new_pkg_meta, apiurl=api_url)
541
542
    elif dry_run:
        print("Would not do anything, package meta would be unchanged")
543
544

    return True
Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
545
546


547
548
549
550
551
552
553
554
def parse_prjconf(api_url, project):
    orig_prjconf = list(map(decode_it, osc.core.show_project_conf(api_url, project)))
    try:
        start = orig_prjconf.index(prjconf_start_marker)
        end = orig_prjconf.index(prjconf_end_marker)
    except ValueError:
        start = None
        end = len(orig_prjconf)
555

556
557
558
559
560
561
562
563
564
565
    prjconf_head = orig_prjconf[:start]
    prjconf_ours = orig_prjconf[start:end]
    prjconf_tail = orig_prjconf[end + 1:]
    return orig_prjconf, prjconf_head, prjconf_ours, prjconf_tail


def mpcdf_setup_subproject(api_url, project, distribution, microarchitecture,
                           parent=None, dry_run=False, remove_old=False, all_possible=False):

    if parent and not dry_run:
566
567
568
        for attribute in config_attributes + default_attributes:
            print("Copying attribute '{0}' from parent project".format(attribute))
            set_attribute(api_url, project, None, get_attribute(api_url, parent, None, attribute))
Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
569

570
571
572
    # Check distribution
    software_meta = project_meta(api_url, "software")
    dist_repo = software_meta.find('./repository[@name="{0}"]'.format(distribution))
Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
573
    if dist_repo is None:
574
        raise osc.oscerr.WrongArgs("Invalid distribution '{0}': No matching repository is defined in project 'software' on the server".format(distribution))
Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
575
576
577
578
    architectures = list(arch.text for arch in dist_repo.findall("./arch"))

    root = project_meta(api_url, project)

579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
    orig_prjconf, prjconf_head, prjconf_ours, prjconf_tail = parse_prjconf(api_url, project)

    prjconf_repos = {}
    cur_repo = None
    old_repos = set()
    for line in prjconf_ours:
        line = line.rstrip("\n")
        if cur_repo is None and line.startswith("%if %_repository =="):
            cur_repo = line.split()[-1]
            old_repos.add(cur_repo)
            prjconf_repos[cur_repo] = []
        if cur_repo is not None:
            prjconf_repos[cur_repo].append(line)
            if line == "%endif":
                cur_repo = None

    prjconf_ours = [prjconf_start_marker]

    prjconf_ours.append("Constraint: hostlabel {0}".format(microarchitecture))
Lorenz Hüdepohl's avatar
Lorenz Hüdepohl committed
598
    prjconf_ours.append("Preinstall: mpcdf_{0}_directory".format(microarchitecture))
599
    prjconf_ours.append("PublishFilter: ^mpcdf_.*$")
600

Lorenz Hüdepohl's avatar
Lorenz Hüdepohl committed
601
602
603
604
    extra_tags = dist_prjconf_tags(distribution)
    if extra_tags:
        prjconf_ours.append(extra_tags)
    extra_macros = dist_prjconf_macros(distribution)
605

606
    prjconf_ours.append("""
607
608
Macros:
%microarchitecture {0}
609
610
{1}
:Macros""".format(microarchitecture, extra_macros))
611
    prjconf_ours.append("")
612

Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
613
    # Remove existing repositories
614
615
616
    if remove_old:
        for oldrepo in root.findall("./repository"):
            root.remove(oldrepo)
Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
617

Lorenz Hüdepohl's avatar
Lorenz Hüdepohl committed
618
    def repo(name, dependencies, compiler=False, mpi=False, cuda=False, cuda_mpi=False, additional_prefers=(), **macros):
619
620
621
622
        old_repos.discard(name)
        have_compiler = compiler or mpi or cuda or cuda_mpi
        have_mpi = mpi or cuda_mpi
        have_cuda = cuda or cuda_mpi
623

624
625
626
        existing_repo = root.find("./repository[@name='{0}']".format(name))
        if existing_repo is not None:
            root.remove(existing_repo)
627
        elif dry_run is False:
628
629
            print("New repository", name)

Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
630
631
        r = ElementTree.SubElement(root, "repository")
        r.set("name", name)
Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
632
633
        r.text = "\n    "

Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
634
635
636
637
        def path(project, repo):
            p = ElementTree.SubElement(r, "path")
            p.set("project", project)
            p.set("repository", repo)
Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
638
            p.tail = "\n    "
Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
639

640
        if parent and name != "System":
641
            path(parent, name)
642
643
        for dep_project, dep_repo in dependencies:
            path(dep_project, dep_repo)
Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
644
645
646
647
648
649
650
651

        for arch in architectures:
            a = ElementTree.SubElement(r, "arch")
            a.text = arch
            a.tail = "\n    "
        a.tail = "\n  "
        r.tail = "\n  "

Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
652
653
654
655
        # In order to be able to figure out the matching MKL version for a given
        # compiler/MPI repository we emit a new macro '%matching_mkl_version' in
        # the cases this makes sense
        matching_mkl = []
656
657
658
        repoconf = []
        repoconf.append("%if %_repository == {0}".format(name))
        for prefer in prefers(name) + additional_prefers:
Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
659
660
            if prefer.startswith("mkl_"):
                matching_mkl.append(prefer)
661
            repoconf.append("Prefer: " + prefer)
Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
662

663
        repoconf.append("Macros:")
664

665
666
667
668
        repoconf.append("%is_compiler_repository {0}".format(1 if compiler else 0))
        repoconf.append("%is_mpi_repository {0}".format(1 if mpi else 0))
        repoconf.append("%is_cuda_repository {0}".format(1 if cuda else 0))
        repoconf.append("%is_cuda_mpi_repository {0}".format(1 if cuda_mpi else 0))
669

670
671
672
        repoconf.append("%have_mpcdf_compiler {0}".format(1 if have_compiler else 0))
        repoconf.append("%have_mpcdf_mpi {0}".format(1 if have_mpi else 0))
        repoconf.append("%have_mpcdf_cuda {0}".format(1 if have_cuda else 0))
673

Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
674
675
676
677
        if matching_mkl:
            matching_mkl, = matching_mkl
            matching_mkl, _ = matching_mkl[len("mkl_"):].split("-module")
            matching_mkl = matching_mkl.replace("_", ".")
678
            repoconf.append("%matching_mkl_version {0}".format(matching_mkl))
Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
679

680
681
        for macro, value in macros.items():
            repoconf.append("%{0} {1}".format(macro, value))
682

683
684
685
        repoconf.append(":Macros")
        repoconf.append("%endif")
        prjconf_repos[name] = repoconf
Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
686

687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
    def actual_compiler(c):
        if c.startswith("latest"):
            return False
        if c == "default_compiler":
            return False
        if c == "intel":
            return False
        if c == "gcc":
            return False
        return True

    def actual_mpi(m):
        if m == "default_mpi":
            return False
        if m == "impi":
            return False
        return True

    def actual_cuda(c):
        if c == "default_cuda":
            return False
        return True

    if all_possible:
        compilers = list(filter(actual_compiler, get_allowed_attribute_values(api_url, "MPCDF:compiler_modules")))
        mpis = list(filter(actual_mpi, get_allowed_attribute_values(api_url, "MPCDF:mpi_modules")))
        cudas = list(filter(actual_cuda, get_allowed_attribute_values(api_url, "MPCDF:cuda_modules")))
        pgis = get_allowed_attribute_values(api_url, "MPCDF:pgi_modules")
Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
715
    else:
716
717
718
719
720
721
        compilers = overloaded_project_attribute(api_url, project, "MPCDF:compiler_modules")
        mpis = overloaded_project_attribute(api_url, project, "MPCDF:mpi_modules")
        cudas = overloaded_project_attribute(api_url, project, "MPCDF:cuda_modules")
        pgis = overloaded_project_attribute(api_url, project, "MPCDF:pgi_modules")

    if parent:
Lorenz Hüdepohl's avatar
Lorenz Hüdepohl committed
722
        repo("System", (("software", distribution),))
Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
723

724
    for compiler in compilers + pgis:
725
        if project != "software":
Lorenz Hüdepohl's avatar
Lorenz Hüdepohl committed
726
            dependencies = (("software", distribution),)
727
728
729
730
731
        else:
            dependencies = ()
        repo(compiler, dependencies, compiler=True,
             compiler_repository=compiler, compiler_module=compiler_module(compiler),
             additional_prefers=("mpcdf_compiler_" + compiler,))
Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
732
733

        for mpi in filter(partial(valid_mpi, compiler), mpis):
Lorenz Hüdepohl's avatar
Lorenz Hüdepohl committed
734
            repo(mpi + "_" + compiler, ((project, compiler),), mpi=True,
735
                 mpi_repository=mpi, mpi_module=mpi_module(mpi))
Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
736
737
738

    for cuda in cudas:
        for compiler in filter(partial(valid_cuda, cuda), compilers):
Lorenz Hüdepohl's avatar
Lorenz Hüdepohl committed
739
            repo(cuda + "_" + compiler, ((project, compiler),), cuda=True, cuda_repository=cuda,
740
                 additional_prefers=("mpcdf_" + cuda,))
Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
741
742
            for mpi in filter(partial(valid_mpi, compiler), mpis):
                repo(cuda + "_" + mpi + "_" + compiler,
Lorenz Hüdepohl's avatar
Lorenz Hüdepohl committed
743
744
                     ((project, cuda + "_" + compiler),
                      (project, mpi + "_" + compiler)),
745
                     cuda_mpi=True)
Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
746

747
748
749
750
751
752
753
754
755
    if old_repos and not remove_old:
        print("Warning: Keeping the prjconf sections for the following obsolete repositories:")
        for name in sorted(old_repos):
            print(" -", name)
        print()
    else:
        for old_repo in old_repos:
            del prjconf_repos[old_repo]

Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
756
757
758
759
760
761
762
763
764
765
766
    # Remove build configuration
    build = root.find("./build")
    if build is None:
        build = ElementTree.Element("build")
        build.text = "\n    "
        disable = ElementTree.SubElement(build, "disable")
        disable.tail = "\n  "
        build.tail = "\n  "
        root.insert(list(root).index(root.find("./repository")), build)

    root.getchildren()[-1].tail = "\n"
767
    new_prj = ElementTree.tostring(root, encoding=osc.core.ET_ENCODING)
Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
768

769
770
771
    for name in sorted(prjconf_repos.keys()):
        prjconf_ours.extend(prjconf_repos[name])
        prjconf_ours.append("")
Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
772

773
    prjconf_ours.append(prjconf_end_marker)
774

775
    new_prjconf = "".join(prjconf_head) + "\n".join(prjconf_ours) + "".join(prjconf_tail)
776

777
778
779
780
781
782
783
784
    if dry_run:
        print("osc meta prjconf {0} -F - <<EOF\n{1}\nEOF\n".format(project, new_prjconf))
        print("osc meta prj {0} -F - <<EOF\n{1}\nEOF\n".format(project, new_prj))
    else:
        if new_prjconf == "".join(orig_prjconf):
            print("prjconf unchanged")
        else:
            print("Updating prjconf meta")
785
            osc.core.edit_meta("prjconf", project, data=new_prjconf, apiurl=api_url)
786

787
788
789
790
        # Create and remove the <enable/> flags before the new repsitories,
        # that way no spurious builds are launched
        print("Updating enabled repositories for all packages")
        mpcdf_enable_repositories_for_all_packages(api_url, project)
Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
791
792
793

        # Update repositories
        print("Updating prj meta")
794
        osc.core.edit_meta("prj", project, data=new_prj, force=True, apiurl=api_url)
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820


def mpcdf_enable_repositories_for_all_packages(api_url, project, ignore_repos=()):
    import threading
    packages = osc.core.meta_get_packagelist(api_url, project)

    if len(packages) > 40:
        chunksize = len(packages) // 20
    else:
        chunksize = len(packages)

    def work(packagelist):
        for package in packagelist:
            try:
                mpcdf_enable_repositories(api_url, project, package, ignore_repos=ignore_repos)
            except UnmanagedPackageException:
                print("ATTENTION: Not changing unmanaged package {0}".format(package))

    threads = []
    for packagelist in chunked(packages, chunksize):
        t = threading.Thread(target=work, args=(packagelist,))
        threads.append(t)
        t.start()

    for t in threads:
        t.join()
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860


def set_as_branch(api_url, my_project, my_package, main_project, main_package):
    import os

    def dirmeta(project, package):
        url = osc.core.makeurl(api_url, ["source", project, package])
        return ElementTree.fromstringlist(osc.core.streamfile(url, osc.core.http_GET))

    srcmeta = dirmeta(my_project, my_package)
    dstmeta = dirmeta(main_project, main_package)

    linkinfo = srcmeta.find('./linkinfo')
    if linkinfo is not None:
        print("ERROR: package {0} is already linked to package {1} in project {2}".format(
              my_package, linkinfo.get("package"), linkinfo.get("project")))
        return False

    linkfile = ElementTree.Element("link")
    linkfile.set("project", main_project)
    linkfile.set("package", main_package)
    linkfile.set("baserev", dstmeta.get("srcmd5"))
    patches = ElementTree.SubElement(linkfile, "patches")
    ElementTree.SubElement(patches, "branch")

    url = osc.core.makeurl(api_url, ["source", my_project, my_package, "_link"])
    resp = ElementTree.fromstringlist(osc.core.streamfile(url, osc.core.http_PUT, data=ElementTree.tostring(linkfile)))
    rev = resp.get("rev")
    if rev is None:
        print("ERROR: Could not commit _link file")
        return False
    else:
        print("Commited as revision", rev)
        this_package = osc.core.store_read_package(os.curdir)
        this_project = osc.core.store_read_project(os.curdir)
        if this_package == my_package and this_project == my_project:
            pac = osc.core.filedir_to_pac(os.curdir)
            rev = pac.latest_rev(expand=True)
            pac.update(rev)
    return True
861
862


863
def sync_projects(api_url, package=None, from_project="software", to_projects=None, redo_all=False, add_to_maintainers=True, verbose=False):
864
865
866
867
    if package is not None and redo_all:
        raise osc.oscerr.WrongArgs('Cannot specify `redo_all` and package')

    if to_projects is None:
Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
868
869
        to_projects = [p for p in osc.core.meta_get_project_list(api_url)
                       if p.startswith("software:") and not (p == "software:dist" or p == "software:images")]
870
871
872
873
874
875
876
877
878
879
880
881
882

    for to_project in to_projects:
        print("Syncing {0} with {1}".format(to_project, from_project))

        to_packages = osc.core.meta_get_packagelist(api_url, to_project)

        if package is None:
            from_packages = osc.core.meta_get_packagelist(api_url, from_project)
            if not redo_all:
                from_packages = list(sorted(set(from_packages) - set(to_packages)))
        else:
            from_packages = [package]

883
884
885
886
887
888
889
890
891
        def non_system_package(package):
            enable_repos = get_attribute_values(api_url, from_project, package, "MPCDF:enable_repositories")
            if enable_repos == ["system"]:
                print("Not branching package {0}, is only enabled for 'System'".format(package))
                return False
            else:
                return True

        from_packages = list(filter(non_system_package, from_packages))
892
893
894
895
896
897
898
899
900
901
902
903
904

        for orig_package in from_packages:
            if orig_package not in to_packages:
                filelist = osc.core.meta_get_filelist(api_url, from_project, orig_package)
                if "_link" in filelist:
                    print("Not branching package {0}, is a link".format(orig_package))
                else:
                    print("Branching package {0}".format(orig_package))
                    exists, targetprj, targetpkg, srcprj, srcpkg = \
                        osc.core.branch_pkg(api_url, from_project, orig_package, target_project=to_project, nodevelproject=True)
            else:
                print("Not branching package {0}, already present in target".format(orig_package))

905
            for attribute in package_attributes + config_attributes:
906
907
908
909
910
911
912
913
                try:
                    attr = get_attribute(api_url, from_project, orig_package, attribute)
                except UnsetAttributeException:
                    if has_attribute(api_url, to_project, orig_package, attribute):
                        remove_attribute(api_url, to_project, orig_package, attribute)
                    continue
                set_attribute(api_url, to_project, orig_package, attr)

914
915
916
917
918
919
920
921
922
923
924
925
            if add_to_maintainers:
                from_maintainers = maintainers(api_url, from_project, orig_package)
                if from_maintainers:
                    to_maintainers = maintainers(api_url, to_project, orig_package)
                    if from_maintainers - to_maintainers:
                        new_maintainers = from_maintainers - to_maintainers
                        to_meta = package_meta(api_url, to_project, orig_package)
                        for userid in sorted(new_maintainers):
                            person = ElementTree.Element("person")
                            person.set("userid", userid)
                            person.set("role", "maintainer")
                            to_meta.insert(2, person)
926
                        osc.core.edit_meta("pkg", (to_project, orig_package), data=ElementTree.tostring(to_meta), apiurl=api_url)
927

928
929
930
        if package is None and redo_all:
            # Check if distribution is already set in to_project
            to_prj_meta = project_meta(api_url, to_project)
931
932
933
            some_repo = to_prj_meta.find('./repository')
            if some_repo is None:
                distribution = project_meta(api_url, from_project).find('./repository/path[@project="software"]').get("repository")
934
            else:
935
                distribution = some_repo.find('./path[@project="distributions"]').get("repository")
936
937

            print("Creating repository configuration")
938
939
940
941
942
943
944
            mpcdf_setup_subproject(api_url, to_project, distribution=distribution)

        for orig_package in from_packages:
            try:
                mpcdf_enable_repositories(api_url, to_project, orig_package, verbose=verbose)
            except UnmanagedPackageException:
                print("ATTENTION: Not changing unmanaged package {0}".format(orig_package))