mpcdf_common.py 36.2 KB
Newer Older
1
from __future__ import print_function
Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
2
from __future__ import division
3

4
import sys
5
6
7
import osc
import osc.conf
import osc.core
Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
8
import osc.oscerr
9
import textwrap
Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
10
from functools import partial
11
12
from xml.etree import ElementTree

13
14
15
16
17
18
19
if sys.version_info[0] < 3:
    def decode_it(arg):
        return arg
else:
    from osc.util.helper import decode_it


20
21
known_microarchs = {"sandybridge", "haswell", "skylake"}

22
23
package_attributes = ["MPCDF:enable_repositories"]
config_attributes = ["MPCDF:compiler_modules", "MPCDF:cuda_modules", "MPCDF:mpi_modules", "MPCDF:pgi_modules"]
24
default_attributes = ["MPCDF:default_compiler", "MPCDF:default_cuda", "MPCDF:default_mpi"]
25

26
27
28
29
30
31
32
33
intel_parallel_studio = {
    "mpcdf_intel_parallel_studio_2017_7": {"compiler": "intel_17_0_7", "impi": "impi_2017_4", "mkl": "mkl_2017_4-module", },
    "mpcdf_intel_parallel_studio_2018_1": {"compiler": "intel_18_0_1", "impi": "impi_2018_1", "mkl": "mkl_2018_1-module", },
    "mpcdf_intel_parallel_studio_2018_2": {"compiler": "intel_18_0_2", "impi": "impi_2018_2", "mkl": "mkl_2018_2-module", },
    "mpcdf_intel_parallel_studio_2018_3": {"compiler": "intel_18_0_3", "impi": "impi_2018_3", "mkl": "mkl_2018_3-module", },
    "mpcdf_intel_parallel_studio_2018_4": {"compiler": "intel_18_0_5", "impi": "impi_2018_4", "mkl": "mkl_2018_4-module", },
    "mpcdf_intel_parallel_studio_2019_0": {"compiler": "intel_19_0_0", "impi": "impi_2019_0", "mkl": "mkl_2019_0-module", },
    "mpcdf_intel_parallel_studio_2019_1": {"compiler": "intel_19_0_1", "impi": "impi_2019_1", "mkl": "mkl_2019_1-module", },
Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
34
    "mpcdf_intel_parallel_studio_2019_3": {"compiler": "intel_19_0_3", "impi": "impi_2019_3", "mkl": "mkl_2019_3-module", },
35
    "mpcdf_intel_parallel_studio_2019_4": {"compiler": "intel_19_0_4", "impi": "impi_2019_4", "mkl": "mkl_2019_4-module", },
36
    "mpcdf_intel_parallel_studio_2019_5": {"compiler": "intel_19_0_5", "impi": "impi_2019_5", "mkl": "mkl_2019_5-module", },
Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
37
    "mpcdf_intel_parallel_studio_2020": {"compiler": "intel_19_1_0", "impi": "impi_2019_6", "mkl": "mkl_2020-module", },
38
    "mpcdf_intel_parallel_studio_2020_1": {"compiler": "intel_19_1_1", "impi": "impi_2019_7", "mkl": "mkl_2020_1-module", },
39
40
41
42
43
44
45
46
}

all_mkls = {ic["mkl"] for ic in intel_parallel_studio.values()}

# Some rotations
mpi_parallel_studio = {value["impi"]: dict({"ps": key}, **value) for key, value in intel_parallel_studio.items()}
compiler_parallel_studio = {value["compiler"]: dict({"ps": key}, **value) for key, value in intel_parallel_studio.items()}

47

48
49
50
51
# For the autogenerated software/software:*:* project 'prjconf' sections
prjconf_start_marker = "# Autogenerated by osc mpcdf_setup_repos, do not edit till end of section\n"
prjconf_end_marker = "# End of autogenerated section\n"

52

Lorenz Hüdepohl's avatar
Lorenz Hüdepohl committed
53
54
55
56
57
58
59
60
61
62
63
64
def dist_prjconf_tags(distribution):
    centos_prjconf_tags = textwrap.dedent(
        """
        Prefer: perl-Error
        Substitute: c_compiler gcc
        Substitute: c++_compiler gcc-c++
        Substitute: ca-certificates-mozilla ca-certificates
        """).strip()

    centos8_prjconf_tags = textwrap.dedent(
        """
        ExpandFlags: module:python36-3.6
65
66
        # Remove redhat-hardened-cc1 and redhat-annobin-cc1 spec
        Optflags: x86_64 -O2 -g -pipe -Wall -Werror=format-security -Wp,-D_FORTIFY_SOURCE=2 -Wp,-D_GLIBCXX_ASSERTIONS -fexceptions -fstack-protector-strong -grecord-gcc-switches -m64 -fasynchronous-unwind-tables -fstack-clash-protection -fcf-protection
Lorenz Hüdepohl's avatar
Lorenz Hüdepohl committed
67
68
        """).strip()

Lorenz Hüdepohl's avatar
Lorenz Hüdepohl committed
69
70
71
72
    sles15_prjconf_tags = textwrap.dedent(
        """
        %if "%{_repository}" == "gcc_6" || "%{_repository}" == "gcc_7"
        # gcc6/7 cannot deal with "-fstack-clash-protection" that has been added in SLE15 by default
73
        Optflags: x86_64 -fmessage-length=0 -grecord-gcc-switches -O2 -Wall -D_FORTIFY_SOURCE=2 -fstack-protector-strong -funwind-tables -fasynchronous-unwind-tables
Lorenz Hüdepohl's avatar
Lorenz Hüdepohl committed
74
75
76
77
        %endif
        """
    ).strip()

Lorenz Hüdepohl's avatar
Lorenz Hüdepohl committed
78
79
80
81
82
    res = ""
    if "CentOS" in distribution:
        res += centos_prjconf_tags
    if "CentOS_8" in distribution:
        res += "\n" + centos8_prjconf_tags
Lorenz Hüdepohl's avatar
Lorenz Hüdepohl committed
83
84
    if "SLE_15" in distribution:
        res += sles15_prjconf_tags
Lorenz Hüdepohl's avatar
Lorenz Hüdepohl committed
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103

    return res


def dist_prjconf_macros(distribution):
    centos_macros = textwrap.dedent(
        """
        # Disable all problematic automatic RPM stuff
        # like byte-compiling (with the wrong Python version)
        # or debug packages that fail for many binary-only packages
        %__no_python_bytecompile 1
        %debug_package %{nil}
        """).strip()

    res = ""
    if "CentOS" in distribution:
        res += centos_macros

    return res
104

105

106
107
108
109
def check_for_update():
    import os
    import sys
    import time
110
111
112
    from subprocess import check_output, call

    DEVNULL = open(os.devnull, 'w')
113
114
115
116
117
118

    if hasattr(sys.modules["mpcdf_common"], "checked_for_updates"):
        return
    else:
        setattr(sys.modules["mpcdf_common"], "checked_for_updates", 1)

119
    if (sys.version_info > (3, 0)):
Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
120
        set_encoding = {"encoding": "utf-8"}
121
122
123
    else:
        set_encoding = {}

124
125
    plugin_dir = os.path.dirname(os.path.realpath(__file__))
    git_dir = os.path.join(plugin_dir, ".git")
126
    local_rev = check_output(["git", "--git-dir", git_dir, "rev-parse", "HEAD"], **set_encoding).strip()
Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
127

Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
128
    url = "https://gitlab.mpcdf.mpg.de/mpcdf/obs/osc-plugins.git"
Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
129
130

    def update_server_rev():
131
        server_rev, _ = check_output(["git", "ls-remote", url, "master"], **set_encoding).split(None, 1)
Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
132
133
        with open(rev_file, "w") as fd:
            fd.write(server_rev)
134
135
136
137
138

    # Check for update on server just once a day
    rev_file = os.path.join(plugin_dir, ".remote_head_rev")
    try:
        mtime = os.stat(rev_file).st_mtime
Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
139
    except EnvironmentError:
140
141
142
        mtime = 0

    if time.time() - mtime > 86400:
Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
143
        update_server_rev()
144

Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
145
    with open(rev_file, "r") as fd:
146
        server_rev = fd.read().strip()
Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
147

148
149
150
151
    if server_rev != local_rev:
        if call(["git", "--git-dir", git_dir, "merge-base", "--is-ancestor", server_rev, "HEAD"], stderr=DEVNULL) == 0:
            # Server rev is older than ours. Check again
            update_server_rev()
Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
152
153
154
155
156

    with open(rev_file, "r") as fd:
        server_rev = fd.read().strip()

    if server_rev != local_rev:
Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
157
        print("Note from MPCDF osc plugins:", file=sys.stderr)
158
        if call(["git", "--git-dir", git_dir, "merge-base", "--is-ancestor", server_rev, "HEAD"], stderr=DEVNULL) == 0:
Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
159
            print(" You have unpushed commits in", plugin_dir, "- consider pushing them", file=sys.stderr)
Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
160
161
            print(file=sys.stderr)
        else:
Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
162
163
            print(" Your plugin directory is out-of-date, new commits available on", url, file=sys.stderr)
            print(" (careful: the repo has been moved, update your path to point to the new URL)", file=sys.stderr)
164
165
166
167
168
169
            print(file=sys.stderr)


check_for_update()


170
171
172
173
174
175
176
177
def compiler_module(compiler_repo):
    return compiler_repo.replace("_", "/", 1).replace("_", ".")


def mpi_module(mpi_repo):
    return mpi_repo.replace("_", "/", 1).replace("_", ".")


178
def valid_pgi_mpi(pgi, mpi):
179
180
181
182
183
184
185
186
187
188
    if "impi" in mpi:
        if "2017" in mpi:
            return False
        else:
            return True

    if "openmpi_4" == mpi:
        return True

    return False
189
190


191
def valid_mpi(compiler, mpi):
192
193
194
195
196
197
198
199
200
    """
    It might be possible to use Intel MPI libararies and compilers from
    different Parallel Studio packages, but I currently do not want to support
    it.

    Take care to keep this in sync with the file 'macros.obs_cluster' of
    the package software:dist / mpcdf_cluster_macros
    """
    if compiler.startswith("intel") and mpi.startswith("impi"):
201
        return mpi == compiler_parallel_studio[compiler]["impi"]
202
203
    if compiler.startswith("pgi"):
        return valid_pgi_mpi(compiler, mpi)
204
205
206
207
208
209
210
211
212
    if compiler.startswith("gcc") and mpi.startswith("impi"):
        gcc_version = int(compiler[len("gcc_"):])
        impi_major_version, impi_minor_version = map(int, mpi[len("impi_"):].split("_"))
        if gcc_version >= 10:
            # gcc_10 only with modern Intel MPI
            return impi_major_version > 2019 or \
                (impi_major_version == 2019 and impi_minor_version >= 7)
        else:
            return True
213
214
    else:
        return True
215
216


217
218
219
220
221
222
223
224
225
226
227
def prefers(reponame):
    prefer_ps = None
    if reponame in compiler_parallel_studio:
        prefer_ps = compiler_parallel_studio[reponame]["ps"]
    else:
        for mpi in mpi_parallel_studio:
            if reponame.startswith(mpi):
                prefer_ps = mpi_parallel_studio[mpi]["ps"]

    if prefer_ps:
        preferred_mkl = intel_parallel_studio[prefer_ps]["mkl"]
228
        unprefer_other_mkls = sorted("!" + mkl for mkl in all_mkls if mkl != preferred_mkl)
229
230
231
232
233
        return (prefer_ps,) + tuple(unprefer_other_mkls) + (preferred_mkl,)
    else:
        return ()


234
235
236
237
def valid_cuda(cuda, compiler):
    """
    The CUDA distribution only works with certain gcc versions,
    this little function takes care that only supported combinations
Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
238
239
240
241
    are allowed.

    Take care to keep this in sync with the file 'macros.obs_cluster' of
    the package software:dist / mpcdf_cluster_macros
242
    """
Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
243
    if cuda == "cuda_8_0":
244
        return compiler == "gcc_5"
Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
245
    if cuda == "cuda_9_1":
246
        return compiler == "gcc_6_3_0"
Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
247
    if cuda == "cuda_9_2":
Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
248
        return compiler == "gcc_6"
Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
249
250
    if cuda == "cuda_10_0":
        return compiler == "gcc_6"
Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
251
252
    if cuda == "cuda_10_1":
        return compiler == "gcc_8"
Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
253
254
    if cuda == "cuda_10_2":
        return compiler == "gcc_8"
255
256
257
    return False


Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
258
259
260
def project_meta(api_url, project):
    return ElementTree.fromstringlist(osc.core.show_project_meta(api_url, project))

261

262
263
264
265
266
267
268
269
270
def package_meta(api_url, project, package):
    return ElementTree.fromstringlist(osc.core.show_package_meta(api_url, project, package))


def maintainers(api_url, project, package):
    root = package_meta(api_url, project, package)
    return {e.get("userid") for e in root.findall("./person[@role='maintainer']")}


271
272
273
274
class UnsetAttributeException(Exception):
    pass


275
276
277
278
class UnmanagedPackageException(Exception):
    pass


279
280
281
282
283
284
285
286
def chunked(l, chunksize):
    n = len(l)
    i = 0
    while i * chunksize < n:
        yield l[i * chunksize: (i + 1) * chunksize]
        i += 1


Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
287
def get_attribute(api_url, project, package, attribute, with_project=False):
288
289
    attribute_meta = osc.core.show_attribute_meta(api_url, project, package, None, attribute, False, with_project)
    if attribute_meta is None:
Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
290
        raise osc.oscerr.APIError("Cannot fetch value for attribute '{0}' from {1}".format(attribute, (project, package)))
291
292
293
294

    root = ElementTree.fromstringlist(attribute_meta)
    attribute = root.find("./attribute")
    if attribute is not None:
Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
295
        return root
296
    else:
297
        raise UnsetAttributeException("Attribute not set")
298

Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
299

300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
def overloaded_project_attribute(api_url, project, attribute):
    try:
        return get_attribute_values(api_url, project, None, attribute)
    except UnsetAttributeException:
        return get_attribute_values(api_url, "software", None, attribute)


def overloaded_package_attribute(api_url, project, package, attribute):
    try:
        return get_attribute_values(api_url, project, package, attribute)
    except UnsetAttributeException:
        pass

    try:
        return get_attribute_values(api_url, project, None, attribute)
    except UnsetAttributeException:
        pass

    return get_attribute_values(api_url, "software", None, attribute)


def get_allowed_attribute_values(api_url, attribute):
    path = ["attribute"] + attribute.split(":") + ["_meta"]
    url = osc.core.makeurl(api_url, path, [])
    try:
        f = osc.core.http_GET(url)
    except osc.core.HTTPError as e:
        e.osc_msg = 'Error getting meta for attribute "{0}"'.format(attribute)
        raise

    root = ElementTree.fromstringlist(f.readlines())
    return list(value.text for value in root.findall("./allowed/value"))


Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
334
335
336
337
338
339
def get_attribute_values(api_url, project, package, attribute, with_project=False):
    root = get_attribute(api_url, project, package, attribute, with_project)
    attribute = root.find("./attribute")
    return list(value.text for value in attribute.findall("./value"))


340
341
342
343
344
def get_attribute_value(api_url, project, package, attribute, with_project=False):
    value, = get_attribute_values(api_url, project, package, attribute, with_project=False)
    return value


345
346
347
348
349
350
def set_attribute(api_url, project, package, attribute):
    path = ["source", project]
    if package:
        path.append(package)
    path.append("_attribute")

Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
351
    attr_meta = ElementTree.tostring(attribute, encoding=osc.core.ET_ENCODING)
Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
352

353
    url = osc.core.makeurl(api_url, path)
Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
354
    resp = ElementTree.fromstringlist(osc.core.streamfile(url, osc.core.http_POST, data=attr_meta))
Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
355
    if resp.find("./summary").text != "Ok":
Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
356
        raise osc.oscerr.APIError("Could not store attribute")
Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
357

358

359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
def set_attribute_values(api_url, project, package, attribute, values):
    root = ElementTree.Element("attributes")
    attr = ElementTree.SubElement(root, "attribute")

    namespace, name = attribute.split(":")
    attr.set("namespace", namespace)
    attr.set("name", name)

    for value in values:
        v = ElementTree.SubElement(attr, "value")
        v.text = value

    set_attribute(api_url, project, package, root)


def has_attribute(api_url, project, package, attribute):
    path = ["source", project]
    if package:
        path.append(package)
    path.append("_attribute")
    path.append(attribute)

    url = osc.core.makeurl(api_url, path)
    resp = ElementTree.fromstringlist(osc.core.streamfile(url, osc.core.http_GET))

    namespace, name = attribute.split(":")
385

386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
    for a in resp.findall("./attribute"):
        if a.get("namespace") == namespace and a.get("name") == name:
            return True
    return False


def remove_attribute(api_url, project, package, attribute_name):
    path = ["source"] + [project]
    if package:
        path.append(package)
    path.append("_attribute")
    path.append(attribute_name)

    url = osc.core.makeurl(api_url, path)
    resp = ElementTree.fromstringlist(osc.core.streamfile(url, osc.core.http_DELETE))
    if resp.find("./summary").text != "Ok":
        raise osc.oscerr.APIError("Could not remove attribute")


405
def get_microarchitecture(project):
406
    if project.startswith("software:"):
407
        microarch = project.split(":")[2]
408
409
410
411
    elif project.startswith("home:"):
        microarch = "sandybridge"
    else:
        raise Exception("Cannot determine micro-architecture for project '{0}'".format(project))
412
413
414
415
416
417
418

    if microarch in known_microarchs:
        return microarch
    else:
        raise Exception("Unknown micro-architecture '{0}'".format(microarch))


419
420
421
422
423
424
def package_sort_key(string):
    name, version = string.split("_", 1)
    version = version.split("_")
    return (name,) + tuple(map(int, version))


425
def mpcdf_enable_repositories(api_url, project, package, verbose=False, dry_run=False, ignore_repos=()):
426
427
    from itertools import product

428
429
430
    pkg_meta = osc.core.show_package_meta(api_url, project, package)
    root = ElementTree.fromstringlist(pkg_meta)

431
    build = root.find("./build")
Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
432
433
    if build is None:
        build = ElementTree.SubElement(root, "build")
434
435
436
437
438

    pkg_meta = ElementTree.tostring(root, encoding=osc.core.ET_ENCODING)

    for enable in build.findall("./enable"):
        build.remove(enable)
439

440
441
    disabled_repos = {disable.get("repository") for disable in build.findall("./disable")}

Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
442
    try:
443
444
445
446
        enable_repos = get_attribute_values(api_url, project, package, "MPCDF:enable_repositories")
    except Exception:
        if verbose:
            print("Warning: Could not get attribute MPCDF:enable_repositories for package {0}, skipping".format(package))
447
        raise UnmanagedPackageException()
Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
448

449
450
    if project == "software":
        distributions = list(sorted(repo.name for repo in osc.core.get_repos_of_project(api_url, "software")))
Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
451

452
453
454
455
456
    else:
        compilers = overloaded_package_attribute(api_url, project, package, "MPCDF:compiler_modules")
        mpis = overloaded_package_attribute(api_url, project, package, "MPCDF:mpi_modules")
        cudas = overloaded_package_attribute(api_url, project, package, "MPCDF:cuda_modules")
        pgis = overloaded_package_attribute(api_url, project, package, "MPCDF:pgi_modules")
457

458
459
460
461
        all_compilers = overloaded_project_attribute(api_url, project, "MPCDF:compiler_modules")
        all_mpis = overloaded_project_attribute(api_url, project, "MPCDF:mpi_modules")
        all_cudas = overloaded_project_attribute(api_url, project, "MPCDF:cuda_modules")
        all_pgis = overloaded_project_attribute(api_url, project, "MPCDF:pgi_modules")
462

463
464
465
        default_compilers = overloaded_project_attribute(api_url, project, "MPCDF:default_compiler")
        default_mpis = overloaded_project_attribute(api_url, project, "MPCDF:default_mpi")
        default_cudas = overloaded_project_attribute(api_url, project, "MPCDF:default_cuda")
466

467
468
        latest_intel = sorted((c for c in all_compilers if c.startswith("intel")), key=package_sort_key)[-1]
        latest_gcc = sorted((c for c in all_compilers if c.startswith("gcc")), key=package_sort_key)[-1]
469

470
471
    i = len(build)

472
    def enable(name):
473
        if name in ignore_repos or name in disabled_repos:
474
            return
475
476
477
        node = ElementTree.Element("enable")
        node.set("repository", name)
        node.tail = "\n    "
478
        build.insert(i, node)
479
480
481
        if verbose:
            print("Enabling", name)

482
    def actual_compilers():
483
        for compiler in (c for c in compilers if c in all_compilers + ["default_compiler", "intel", "gcc", "latest_intel", "latest_gcc"]):
484
            if compiler == "intel":
Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
485
                for intel_compiler in [cc for cc in all_compilers if cc.startswith("intel")]:
486
487
                    yield intel_compiler
            elif compiler == "gcc":
Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
488
                for gcc_compiler in [cc for cc in all_compilers if cc.startswith("gcc")]:
489
490
491
492
                    yield gcc_compiler
            elif compiler == "default_compiler":
                for default_compiler in default_compilers:
                    yield default_compiler
493
494
495
496
            elif compiler == "latest_intel":
                yield latest_intel
            elif compiler == "latest_gcc":
                yield latest_gcc
497
498
499
500
            else:
                yield compiler

    def actual_mpis():
501
        for mpi in (m for m in mpis if m in all_mpis + ["default_mpi", "impi"]):
502
            if mpi == "impi":
503
                for impi in [mpi for mpi in all_mpis if mpi.startswith("impi")]:
504
505
506
507
508
509
510
511
                    yield impi
            elif mpi == "default_mpi":
                for default_mpi in default_mpis:
                    yield default_mpi
            else:
                yield mpi

    def actual_cudas():
512
        for cuda in (c for c in cudas if c in all_cudas + ["default_cuda"]):
513
514
515
516
517
518
            if cuda == "default_cuda":
                for default_cuda in default_cudas:
                    yield default_cuda
            else:
                yield cuda

519
520
521
522
    def actual_pgis():
        for pgi in (p for p in pgis if p in all_pgis):
            yield pgi

523
    for flag in enable_repos:
524
525
        if flag == "system":
            if project == "software":
526
527
                for distribution in distributions:
                    enable(distribution)
528
529
            elif project.startswith("home:"):
                enable("System")
530

531
        elif project != "software":
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
            if flag == "compilers":
                for compiler in actual_compilers():
                    enable(compiler)

            if flag == "mpi":
                for mpi, compiler in product(actual_mpis(), actual_compilers()):
                    if valid_mpi(compiler, mpi):
                        enable(mpi + "_" + compiler)

            if flag == "cuda":
                for cuda, compiler in product(actual_cudas(), all_compilers):
                    if valid_cuda(cuda, compiler):
                        enable(cuda + "_" + compiler)

            if flag == "cuda_mpi":
                for cuda, mpi, compiler in product(actual_cudas(), actual_mpis(), all_compilers):
                    if valid_cuda(cuda, compiler) and valid_mpi(compiler, mpi):
                        enable(cuda + "_" + mpi + "_" + compiler)

            if flag == "pgi":
                for pgi in actual_pgis():
                    enable(pgi)

            if flag == "pgi_mpi":
                for mpi, pgi in product(actual_mpis(), actual_pgis()):
                    if valid_pgi_mpi(pgi, mpi):
                        enable(mpi + "_" + pgi)
559

Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
560
561
    if len(build.getchildren()) > 0:
        build.getchildren()[-1].tail = "\n  "
562

563
564
    new_pkg_meta = ElementTree.tostring(root, encoding=osc.core.ET_ENCODING)
    if pkg_meta != new_pkg_meta:
565
        print("Updating repositories for", package)
566
567
568
        if dry_run:
            print("osc meta pkg {0} {1} -F - <<EOF\n{2}\nEOF\n".format(project, package, new_pkg_meta))
        else:
569
            osc.core.edit_meta("pkg", (project, package), data=new_pkg_meta, apiurl=api_url)
570
571
    elif dry_run:
        print("Would not do anything, package meta would be unchanged")
572
573

    return True
Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
574
575


576
577
578
579
580
581
582
583
def parse_prjconf(api_url, project):
    orig_prjconf = list(map(decode_it, osc.core.show_project_conf(api_url, project)))
    try:
        start = orig_prjconf.index(prjconf_start_marker)
        end = orig_prjconf.index(prjconf_end_marker)
    except ValueError:
        start = None
        end = len(orig_prjconf)
584

585
586
587
588
589
590
591
    prjconf_head = orig_prjconf[:start]
    prjconf_ours = orig_prjconf[start:end]
    prjconf_tail = orig_prjconf[end + 1:]
    return orig_prjconf, prjconf_head, prjconf_ours, prjconf_tail


def mpcdf_setup_subproject(api_url, project, distribution, microarchitecture,
592
                           parent=None, dry_run=False, remove_old=False, all_possible=False, only_project=False):
593
594

    if parent and not dry_run:
595
596
        for attribute in config_attributes + default_attributes:
            print("Copying attribute '{0}' from parent project".format(attribute))
597
            set_attribute_values(api_url, project, None, attribute, overloaded_project_attribute(api_url, parent, attribute))
Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
598

599
600
601
    # Check distribution
    software_meta = project_meta(api_url, "software")
    dist_repo = software_meta.find('./repository[@name="{0}"]'.format(distribution))
Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
602
    if dist_repo is None:
603
        raise osc.oscerr.WrongArgs("Invalid distribution '{0}': No matching repository is defined in project 'software' on the server".format(distribution))
Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
604
605
606
607
    architectures = list(arch.text for arch in dist_repo.findall("./arch"))

    root = project_meta(api_url, project)

608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
    orig_prjconf, prjconf_head, prjconf_ours, prjconf_tail = parse_prjconf(api_url, project)

    prjconf_repos = {}
    cur_repo = None
    old_repos = set()
    for line in prjconf_ours:
        line = line.rstrip("\n")
        if cur_repo is None and line.startswith("%if %_repository =="):
            cur_repo = line.split()[-1]
            old_repos.add(cur_repo)
            prjconf_repos[cur_repo] = []
        if cur_repo is not None:
            prjconf_repos[cur_repo].append(line)
            if line == "%endif":
                cur_repo = None

    prjconf_ours = [prjconf_start_marker]

    prjconf_ours.append("Constraint: hostlabel {0}".format(microarchitecture))
Lorenz Hüdepohl's avatar
Lorenz Hüdepohl committed
627
    prjconf_ours.append("Preinstall: mpcdf_{0}_directory".format(microarchitecture))
628
    prjconf_ours.append("PublishFilter: ^mpcdf_.*$")
Lorenz Hüdepohl's avatar
Lorenz Hüdepohl committed
629
    prjconf_ours.append("")
630

Lorenz Hüdepohl's avatar
Lorenz Hüdepohl committed
631
632
633
634
    extra_tags = dist_prjconf_tags(distribution)
    if extra_tags:
        prjconf_ours.append(extra_tags)
    extra_macros = dist_prjconf_macros(distribution)
635

636
    prjconf_ours.append("""
637
638
Macros:
%microarchitecture {0}
639
640
{1}
:Macros""".format(microarchitecture, extra_macros))
641
    prjconf_ours.append("")
642

Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
643
    # Remove existing repositories
644
645
646
    if remove_old:
        for oldrepo in root.findall("./repository"):
            root.remove(oldrepo)
Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
647

Lorenz Hüdepohl's avatar
Lorenz Hüdepohl committed
648
    def repo(name, dependencies, compiler=False, mpi=False, cuda=False, cuda_mpi=False, additional_prefers=(), **macros):
649
650
651
652
        old_repos.discard(name)
        have_compiler = compiler or mpi or cuda or cuda_mpi
        have_mpi = mpi or cuda_mpi
        have_cuda = cuda or cuda_mpi
653

654
655
656
        existing_repo = root.find("./repository[@name='{0}']".format(name))
        if existing_repo is not None:
            root.remove(existing_repo)
657
        elif dry_run is False:
658
659
            print("New repository", name)

Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
660
661
        r = ElementTree.SubElement(root, "repository")
        r.set("name", name)
Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
662
663
        r.text = "\n    "

Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
664
665
666
667
        def path(project, repo):
            p = ElementTree.SubElement(r, "path")
            p.set("project", project)
            p.set("repository", repo)
Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
668
            p.tail = "\n    "
Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
669

670
        if parent and name != "System":
671
            path(parent, name)
672
673
        for dep_project, dep_repo in dependencies:
            path(dep_project, dep_repo)
Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
674
675
676
677
678
679
680
681

        for arch in architectures:
            a = ElementTree.SubElement(r, "arch")
            a.text = arch
            a.tail = "\n    "
        a.tail = "\n  "
        r.tail = "\n  "

Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
682
683
684
685
        # In order to be able to figure out the matching MKL version for a given
        # compiler/MPI repository we emit a new macro '%matching_mkl_version' in
        # the cases this makes sense
        matching_mkl = []
686
687
688
        repoconf = []
        repoconf.append("%if %_repository == {0}".format(name))
        for prefer in prefers(name) + additional_prefers:
Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
689
690
            if prefer.startswith("mkl_"):
                matching_mkl.append(prefer)
691
            repoconf.append("Prefer: " + prefer)
Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
692

693
        repoconf.append("Macros:")
694

695
696
697
698
        repoconf.append("%is_compiler_repository {0}".format(1 if compiler else 0))
        repoconf.append("%is_mpi_repository {0}".format(1 if mpi else 0))
        repoconf.append("%is_cuda_repository {0}".format(1 if cuda else 0))
        repoconf.append("%is_cuda_mpi_repository {0}".format(1 if cuda_mpi else 0))
699

700
701
702
        repoconf.append("%have_mpcdf_compiler {0}".format(1 if have_compiler else 0))
        repoconf.append("%have_mpcdf_mpi {0}".format(1 if have_mpi else 0))
        repoconf.append("%have_mpcdf_cuda {0}".format(1 if have_cuda else 0))
703

Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
704
705
706
707
        if matching_mkl:
            matching_mkl, = matching_mkl
            matching_mkl, _ = matching_mkl[len("mkl_"):].split("-module")
            matching_mkl = matching_mkl.replace("_", ".")
708
            repoconf.append("%matching_mkl_version {0}".format(matching_mkl))
Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
709

710
711
        for macro, value in macros.items():
            repoconf.append("%{0} {1}".format(macro, value))
712

713
714
715
        repoconf.append(":Macros")
        repoconf.append("%endif")
        prjconf_repos[name] = repoconf
Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
716

717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
    def actual_compiler(c):
        if c.startswith("latest"):
            return False
        if c == "default_compiler":
            return False
        if c == "intel":
            return False
        if c == "gcc":
            return False
        return True

    def actual_mpi(m):
        if m == "default_mpi":
            return False
        if m == "impi":
            return False
        return True

    def actual_cuda(c):
        if c == "default_cuda":
            return False
        return True

    if all_possible:
        compilers = list(filter(actual_compiler, get_allowed_attribute_values(api_url, "MPCDF:compiler_modules")))
        mpis = list(filter(actual_mpi, get_allowed_attribute_values(api_url, "MPCDF:mpi_modules")))
        cudas = list(filter(actual_cuda, get_allowed_attribute_values(api_url, "MPCDF:cuda_modules")))
        pgis = get_allowed_attribute_values(api_url, "MPCDF:pgi_modules")
Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
745
    else:
746
747
748
749
750
751
        compilers = overloaded_project_attribute(api_url, project, "MPCDF:compiler_modules")
        mpis = overloaded_project_attribute(api_url, project, "MPCDF:mpi_modules")
        cudas = overloaded_project_attribute(api_url, project, "MPCDF:cuda_modules")
        pgis = overloaded_project_attribute(api_url, project, "MPCDF:pgi_modules")

    if parent:
Lorenz Hüdepohl's avatar
Lorenz Hüdepohl committed
752
        repo("System", (("software", distribution),))
Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
753

754
    for compiler in compilers + pgis:
755
        if project != "software":
Lorenz Hüdepohl's avatar
Lorenz Hüdepohl committed
756
            dependencies = (("software", distribution),)
757
758
759
760
761
        else:
            dependencies = ()
        repo(compiler, dependencies, compiler=True,
             compiler_repository=compiler, compiler_module=compiler_module(compiler),
             additional_prefers=("mpcdf_compiler_" + compiler,))
Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
762
763

        for mpi in filter(partial(valid_mpi, compiler), mpis):
Lorenz Hüdepohl's avatar
Lorenz Hüdepohl committed
764
            repo(mpi + "_" + compiler, ((project, compiler),), mpi=True,
765
                 mpi_repository=mpi, mpi_module=mpi_module(mpi))
Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
766
767
768

    for cuda in cudas:
        for compiler in filter(partial(valid_cuda, cuda), compilers):
Lorenz Hüdepohl's avatar
Lorenz Hüdepohl committed
769
            repo(cuda + "_" + compiler, ((project, compiler),), cuda=True, cuda_repository=cuda,
770
                 additional_prefers=("mpcdf_" + cuda,))
Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
771
772
            for mpi in filter(partial(valid_mpi, compiler), mpis):
                repo(cuda + "_" + mpi + "_" + compiler,
Lorenz Hüdepohl's avatar
Lorenz Hüdepohl committed
773
774
                     ((project, cuda + "_" + compiler),
                      (project, mpi + "_" + compiler)),
775
                     cuda_mpi=True)
Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
776

777
778
779
780
781
782
783
784
785
    if old_repos and not remove_old:
        print("Warning: Keeping the prjconf sections for the following obsolete repositories:")
        for name in sorted(old_repos):
            print(" -", name)
        print()
    else:
        for old_repo in old_repos:
            del prjconf_repos[old_repo]

Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
786
787
788
789
790
791
792
793
794
795
796
    # Remove build configuration
    build = root.find("./build")
    if build is None:
        build = ElementTree.Element("build")
        build.text = "\n    "
        disable = ElementTree.SubElement(build, "disable")
        disable.tail = "\n  "
        build.tail = "\n  "
        root.insert(list(root).index(root.find("./repository")), build)

    root.getchildren()[-1].tail = "\n"
797
    new_prj = ElementTree.tostring(root, encoding=osc.core.ET_ENCODING)
Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
798

799
800
801
    for name in sorted(prjconf_repos.keys()):
        prjconf_ours.extend(prjconf_repos[name])
        prjconf_ours.append("")
Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
802

803
    prjconf_ours.append(prjconf_end_marker)
804

805
    new_prjconf = "".join(prjconf_head) + "\n".join(prjconf_ours) + "".join(prjconf_tail)
806

807
808
809
810
811
812
813
814
    if dry_run:
        print("osc meta prjconf {0} -F - <<EOF\n{1}\nEOF\n".format(project, new_prjconf))
        print("osc meta prj {0} -F - <<EOF\n{1}\nEOF\n".format(project, new_prj))
    else:
        if new_prjconf == "".join(orig_prjconf):
            print("prjconf unchanged")
        else:
            print("Updating prjconf meta")
815
            osc.core.edit_meta("prjconf", project, data=new_prjconf, apiurl=api_url)
816

817
818
        # Create and remove the <enable/> flags before the new repsitories,
        # that way no spurious builds are launched
819
820
821
        if not only_project:
            print("Updating enabled repositories for all packages")
            mpcdf_enable_repositories_for_all_packages(api_url, project)
Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
822
823
824

        # Update repositories
        print("Updating prj meta")
825
        osc.core.edit_meta("prj", project, data=new_prj, force=True, apiurl=api_url)
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851


def mpcdf_enable_repositories_for_all_packages(api_url, project, ignore_repos=()):
    import threading
    packages = osc.core.meta_get_packagelist(api_url, project)

    if len(packages) > 40:
        chunksize = len(packages) // 20
    else:
        chunksize = len(packages)

    def work(packagelist):
        for package in packagelist:
            try:
                mpcdf_enable_repositories(api_url, project, package, ignore_repos=ignore_repos)
            except UnmanagedPackageException:
                print("ATTENTION: Not changing unmanaged package {0}".format(package))

    threads = []
    for packagelist in chunked(packages, chunksize):
        t = threading.Thread(target=work, args=(packagelist,))
        threads.append(t)
        t.start()

    for t in threads:
        t.join()
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891


def set_as_branch(api_url, my_project, my_package, main_project, main_package):
    import os

    def dirmeta(project, package):
        url = osc.core.makeurl(api_url, ["source", project, package])
        return ElementTree.fromstringlist(osc.core.streamfile(url, osc.core.http_GET))

    srcmeta = dirmeta(my_project, my_package)
    dstmeta = dirmeta(main_project, main_package)

    linkinfo = srcmeta.find('./linkinfo')
    if linkinfo is not None:
        print("ERROR: package {0} is already linked to package {1} in project {2}".format(
              my_package, linkinfo.get("package"), linkinfo.get("project")))
        return False

    linkfile = ElementTree.Element("link")
    linkfile.set("project", main_project)
    linkfile.set("package", main_package)
    linkfile.set("baserev", dstmeta.get("srcmd5"))
    patches = ElementTree.SubElement(linkfile, "patches")
    ElementTree.SubElement(patches, "branch")

    url = osc.core.makeurl(api_url, ["source", my_project, my_package, "_link"])
    resp = ElementTree.fromstringlist(osc.core.streamfile(url, osc.core.http_PUT, data=ElementTree.tostring(linkfile)))
    rev = resp.get("rev")
    if rev is None:
        print("ERROR: Could not commit _link file")
        return False
    else:
        print("Commited as revision", rev)
        this_package = osc.core.store_read_package(os.curdir)
        this_project = osc.core.store_read_project(os.curdir)
        if this_package == my_package and this_project == my_project:
            pac = osc.core.filedir_to_pac(os.curdir)
            rev = pac.latest_rev(expand=True)
            pac.update(rev)
    return True
892
893


Lorenz Hüdepohl's avatar
Lorenz Hüdepohl committed
894
def sync_projects(api_url, package=None, from_project="software", to_projects=None, add_to_maintainers=True, verbose=False):
895
    if to_projects is None:
Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
896
897
        to_projects = [p for p in osc.core.meta_get_project_list(api_url)
                       if p.startswith("software:") and not (p == "software:dist" or p == "software:images")]
898
899
900
901
902
903
904
905
906
907
908
909

    for to_project in to_projects:
        print("Syncing {0} with {1}".format(to_project, from_project))

        to_packages = osc.core.meta_get_packagelist(api_url, to_project)

        if package is None:
            from_packages = osc.core.meta_get_packagelist(api_url, from_project)
        else:
            from_packages = [package]

        for orig_package in from_packages:
910
            enable_repos = get_attribute_values(api_url, from_project, orig_package, "MPCDF:enable_repositories")
911
            if orig_package not in to_packages:
912
                if len(enable_repos) == 0:
913
                    print("Not branching package {0}, is disabled".format(orig_package))
914
915
                    continue
                elif enable_repos == ["system"]:
916
                    print("Not branching package {0}, is only enabled for 'system'".format(orig_package))
917
                    continue
918
919
920
921
922
923
924
925
926
927
                filelist = osc.core.meta_get_filelist(api_url, from_project, orig_package)
                if "_link" in filelist:
                    print("Not branching package {0}, is a link".format(orig_package))
                else:
                    print("Branching package {0}".format(orig_package))
                    exists, targetprj, targetpkg, srcprj, srcpkg = \
                        osc.core.branch_pkg(api_url, from_project, orig_package, target_project=to_project, nodevelproject=True)
            else:
                print("Not branching package {0}, already present in target".format(orig_package))

928
            for attribute in package_attributes + config_attributes:
929
930
931
932
933
934
935
936
                try:
                    attr = get_attribute(api_url, from_project, orig_package, attribute)
                except UnsetAttributeException:
                    if has_attribute(api_url, to_project, orig_package, attribute):
                        remove_attribute(api_url, to_project, orig_package, attribute)
                    continue
                set_attribute(api_url, to_project, orig_package, attr)

937
938
939
940
941
942
943
944
945
946
947
948
            if add_to_maintainers:
                from_maintainers = maintainers(api_url, from_project, orig_package)
                if from_maintainers:
                    to_maintainers = maintainers(api_url, to_project, orig_package)
                    if from_maintainers - to_maintainers:
                        new_maintainers = from_maintainers - to_maintainers
                        to_meta = package_meta(api_url, to_project, orig_package)
                        for userid in sorted(new_maintainers):
                            person = ElementTree.Element("person")
                            person.set("userid", userid)
                            person.set("role", "maintainer")
                            to_meta.insert(2, person)
949
                        osc.core.edit_meta("pkg", (to_project, orig_package), data=ElementTree.tostring(to_meta), apiurl=api_url)
950

951
952
953
954
            try:
                mpcdf_enable_repositories(api_url, to_project, orig_package, verbose=verbose)
            except UnmanagedPackageException:
                print("ATTENTION: Not changing unmanaged package {0}".format(orig_package))