mpcdf_common.py 42.2 KB
Newer Older
1
from __future__ import print_function
Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
2
from __future__ import division
3

4
import sys
5
6
7
import osc
import osc.conf
import osc.core
Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
8
import osc.oscerr
9
import textwrap
Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
10
from functools import partial
11
12
from xml.etree import ElementTree

13
14
15
16
17
18
19
if sys.version_info[0] < 3:
    def decode_it(arg):
        return arg
else:
    from osc.util.helper import decode_it


20
21
known_microarchs = {"sandybridge", "haswell", "skylake"}

22
package_attributes = ["MPCDF:enable_repositories"]
23
config_attributes = ["MPCDF:compiler_modules", "MPCDF:cuda_modules", "MPCDF:mpi_modules", "MPCDF:pgi_modules", "MPCDF:openmpi_flavors"]
24

25
26
27
28
29
30
31
32
intel_parallel_studio = {
    "mpcdf_intel_parallel_studio_2017_7": {"compiler": "intel_17_0_7", "impi": "impi_2017_4", "mkl": "mkl_2017_4-module", },
    "mpcdf_intel_parallel_studio_2018_1": {"compiler": "intel_18_0_1", "impi": "impi_2018_1", "mkl": "mkl_2018_1-module", },
    "mpcdf_intel_parallel_studio_2018_2": {"compiler": "intel_18_0_2", "impi": "impi_2018_2", "mkl": "mkl_2018_2-module", },
    "mpcdf_intel_parallel_studio_2018_3": {"compiler": "intel_18_0_3", "impi": "impi_2018_3", "mkl": "mkl_2018_3-module", },
    "mpcdf_intel_parallel_studio_2018_4": {"compiler": "intel_18_0_5", "impi": "impi_2018_4", "mkl": "mkl_2018_4-module", },
    "mpcdf_intel_parallel_studio_2019_0": {"compiler": "intel_19_0_0", "impi": "impi_2019_0", "mkl": "mkl_2019_0-module", },
    "mpcdf_intel_parallel_studio_2019_1": {"compiler": "intel_19_0_1", "impi": "impi_2019_1", "mkl": "mkl_2019_1-module", },
Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
33
    "mpcdf_intel_parallel_studio_2019_3": {"compiler": "intel_19_0_3", "impi": "impi_2019_3", "mkl": "mkl_2019_3-module", },
34
    "mpcdf_intel_parallel_studio_2019_4": {"compiler": "intel_19_0_4", "impi": "impi_2019_4", "mkl": "mkl_2019_4-module", },
35
    "mpcdf_intel_parallel_studio_2019_5": {"compiler": "intel_19_0_5", "impi": "impi_2019_5", "mkl": "mkl_2019_5-module", },
Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
36
    "mpcdf_intel_parallel_studio_2020": {"compiler": "intel_19_1_0", "impi": "impi_2019_6", "mkl": "mkl_2020-module", },
37
    "mpcdf_intel_parallel_studio_2020_1": {"compiler": "intel_19_1_1", "impi": "impi_2019_7", "mkl": "mkl_2020_1-module", },
38
    "mpcdf_intel_parallel_studio_2020_2": {"compiler": "intel_19_1_2", "impi": "impi_2019_8", "mkl": "mkl_2020_2-module", },
39
    "mpcdf_intel_parallel_studio_2020_4": {"compiler": "intel_19_1_3", "impi": "impi_2019_9", "mkl": "mkl_2020_4-module", },
Lorenz Hüdepohl's avatar
Lorenz Hüdepohl committed
40
    "mpcdf_intel_oneapi_2021_2": {"compiler": "intel_2021_2_0", "impi": "impi_2021_2", "mkl": "mkl_2021_2-module", },
41
    "mpcdf_intel_oneapi_2021_3": {"compiler": "intel_2021_3_0", "impi": "impi_2021_3", "mkl": "mkl_2021_3-module", },
42
43
44
45
46
47
48
49
}

all_mkls = {ic["mkl"] for ic in intel_parallel_studio.values()}

# Some rotations
mpi_parallel_studio = {value["impi"]: dict({"ps": key}, **value) for key, value in intel_parallel_studio.items()}
compiler_parallel_studio = {value["compiler"]: dict({"ps": key}, **value) for key, value in intel_parallel_studio.items()}

50

51
52
53
54
# For the autogenerated software/software:*:* project 'prjconf' sections
prjconf_start_marker = "# Autogenerated by osc mpcdf_setup_repos, do not edit till end of section\n"
prjconf_end_marker = "# End of autogenerated section\n"

55

Lorenz Hüdepohl's avatar
Lorenz Hüdepohl committed
56
57
58
59
def dist_prjconf_tags(distribution):
    centos_prjconf_tags = textwrap.dedent(
        """
        Prefer: perl-Error
60
        Support: post-build-checks
Lorenz Hüdepohl's avatar
Lorenz Hüdepohl committed
61
62
63
64
65
66
67
        Substitute: c_compiler gcc
        Substitute: c++_compiler gcc-c++
        Substitute: ca-certificates-mozilla ca-certificates
        """).strip()

    centos8_prjconf_tags = textwrap.dedent(
        """
Lorenz Hüdepohl's avatar
Lorenz Hüdepohl committed
68
        Preinstall: libzstd
Lorenz Hüdepohl's avatar
Lorenz Hüdepohl committed
69
        ExpandFlags: module:python36-3.6
70
71
        # Remove redhat-hardened-cc1 and redhat-annobin-cc1 spec
        Optflags: x86_64 -O2 -g -pipe -Wall -Werror=format-security -Wp,-D_FORTIFY_SOURCE=2 -Wp,-D_GLIBCXX_ASSERTIONS -fexceptions -fstack-protector-strong -grecord-gcc-switches -m64 -fasynchronous-unwind-tables -fstack-clash-protection -fcf-protection
Lorenz Hüdepohl's avatar
Lorenz Hüdepohl committed
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
        """).strip()

    res = ""
    if "CentOS" in distribution:
        res += centos_prjconf_tags
    if "CentOS_8" in distribution:
        res += "\n" + centos8_prjconf_tags

    return res


def dist_prjconf_macros(distribution):
    centos_macros = textwrap.dedent(
        """
        # Disable all problematic automatic RPM stuff
        # like byte-compiling (with the wrong Python version)
        # or debug packages that fail for many binary-only packages
        %__no_python_bytecompile 1
        %debug_package %{nil}
        """).strip()

    res = ""
    if "CentOS" in distribution:
        res += centos_macros

    return res
98

99

100
101
102
103
def check_for_update():
    import os
    import sys
    import time
104
105
106
    from subprocess import check_output, call

    DEVNULL = open(os.devnull, 'w')
107
108
109
110
111
112

    if hasattr(sys.modules["mpcdf_common"], "checked_for_updates"):
        return
    else:
        setattr(sys.modules["mpcdf_common"], "checked_for_updates", 1)

113
    if (sys.version_info > (3, 0)):
Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
114
        set_encoding = {"encoding": "utf-8"}
115
116
117
    else:
        set_encoding = {}

118
119
    plugin_dir = os.path.dirname(os.path.realpath(__file__))
    git_dir = os.path.join(plugin_dir, ".git")
120
    local_rev = check_output(["git", "--git-dir", git_dir, "rev-parse", "HEAD"], **set_encoding).strip()
Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
121

Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
122
    url = "https://gitlab.mpcdf.mpg.de/mpcdf/obs/osc-plugins.git"
Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
123
124

    def update_server_rev():
125
        server_rev, _ = check_output(["git", "ls-remote", url, "master"], **set_encoding).split(None, 1)
Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
126
127
        with open(rev_file, "w") as fd:
            fd.write(server_rev)
128
129
130
131
132

    # Check for update on server just once a day
    rev_file = os.path.join(plugin_dir, ".remote_head_rev")
    try:
        mtime = os.stat(rev_file).st_mtime
Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
133
    except EnvironmentError:
134
135
136
        mtime = 0

    if time.time() - mtime > 86400:
Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
137
        update_server_rev()
138

Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
139
    with open(rev_file, "r") as fd:
140
        server_rev = fd.read().strip()
Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
141

142
143
144
145
    if server_rev != local_rev:
        if call(["git", "--git-dir", git_dir, "merge-base", "--is-ancestor", server_rev, "HEAD"], stderr=DEVNULL) == 0:
            # Server rev is older than ours. Check again
            update_server_rev()
Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
146
147
148
149
150

    with open(rev_file, "r") as fd:
        server_rev = fd.read().strip()

    if server_rev != local_rev:
Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
151
        print("Note from MPCDF osc plugins:", file=sys.stderr)
152
        if call(["git", "--git-dir", git_dir, "merge-base", "--is-ancestor", server_rev, "HEAD"], stderr=DEVNULL) == 0:
Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
153
            print(" You have unpushed commits in", plugin_dir, "- consider pushing them", file=sys.stderr)
Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
154
155
            print(file=sys.stderr)
        else:
Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
156
157
            print(" Your plugin directory is out-of-date, new commits available on", url, file=sys.stderr)
            print(" (careful: the repo has been moved, update your path to point to the new URL)", file=sys.stderr)
158
159
160
161
162
163
            print(file=sys.stderr)


check_for_update()


164
def compiler_module(compiler_repo):
165
166
167
168
169
170
171
    # special case: long and short version for intel 21.2.0
    if compiler_repo == "intel_2021_2_0":
        return "intel/21.2.0"
    elif compiler_repo == "intel_2021_3_0":
        return "intel/21.3.0"
    else:
        return compiler_repo.replace("_", "/", 1).replace("_", ".")
172
173
174
175
176
177


def mpi_module(mpi_repo):
    return mpi_repo.replace("_", "/", 1).replace("_", ".")


178
def valid_pgi_mpi(pgi, mpi):
179
180
181
182
183
184
185
186
187
188
    if "impi" in mpi:
        if "2017" in mpi:
            return False
        else:
            return True

    if "openmpi_4" == mpi:
        return True

    return False
189
190


191
def valid_mpi(compiler, mpi):
192
193
194
195
196
197
198
199
200
    """
    It might be possible to use Intel MPI libararies and compilers from
    different Parallel Studio packages, but I currently do not want to support
    it.

    Take care to keep this in sync with the file 'macros.obs_cluster' of
    the package software:dist / mpcdf_cluster_macros
    """
    if compiler.startswith("intel") and mpi.startswith("impi"):
201
        return mpi == compiler_parallel_studio[compiler]["impi"]
202
203
    if compiler.startswith("pgi"):
        return valid_pgi_mpi(compiler, mpi)
204
205
206
207
208
209
210
211
212
    if compiler.startswith("gcc") and mpi.startswith("impi"):
        gcc_version = int(compiler[len("gcc_"):])
        impi_major_version, impi_minor_version = map(int, mpi[len("impi_"):].split("_"))
        if gcc_version >= 10:
            # gcc_10 only with modern Intel MPI
            return impi_major_version > 2019 or \
                (impi_major_version == 2019 and impi_minor_version >= 7)
        else:
            return True
213
214
    else:
        return True
215
216


Lorenz Hüdepohl's avatar
Lorenz Hüdepohl committed
217
218
219
220
221
222
223
224
def repo_tags(reponame, distribution=None):
    if "gcc_6" in reponame or "gcc_7" in reponame and distribution == "SLE_15":
        # gcc6/7 cannot deal with "-fstack-clash-protection" that has been added in SLE15 by default
        optflags = ("Optflags: * -fmessage-length=0 -grecord-gcc-switches -O2 -Wall -D_FORTIFY_SOURCE=2 -fstack-protector-strong -funwind-tables -fasynchronous-unwind-tables",)
    else:
        optflags = ()

    ps = None
225
    if reponame in compiler_parallel_studio:
Lorenz Hüdepohl's avatar
Lorenz Hüdepohl committed
226
        ps = compiler_parallel_studio[reponame]["ps"]
227
228
229
    else:
        for mpi in mpi_parallel_studio:
            if reponame.startswith(mpi):
Lorenz Hüdepohl's avatar
Lorenz Hüdepohl committed
230
                ps = mpi_parallel_studio[mpi]["ps"]
231

Lorenz Hüdepohl's avatar
Lorenz Hüdepohl committed
232
233
234
235
    if ps:
        preferred_mkl = "Prefer: " + intel_parallel_studio[ps]["mkl"]
        unprefer_other_mkls = sorted("Prefer: !" + mkl for mkl in all_mkls if mkl != preferred_mkl)
        prefers = ("Prefer: " + ps,) + tuple(unprefer_other_mkls) + (preferred_mkl,)
236
    else:
Lorenz Hüdepohl's avatar
Lorenz Hüdepohl committed
237
238
239
        prefers = ()

    return optflags + prefers
240
241


242
243
244
245
def valid_cuda(cuda, compiler):
    """
    The CUDA distribution only works with certain gcc versions,
    this little function takes care that only supported combinations
Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
246
247
248
249
    are allowed.

    Take care to keep this in sync with the file 'macros.obs_cluster' of
    the package software:dist / mpcdf_cluster_macros
250
    """
Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
251
    if cuda == "cuda_8_0":
252
        return compiler == "gcc_5"
Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
253
    if cuda == "cuda_9_1":
254
        return compiler == "gcc_6_3_0"
Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
255
    if cuda == "cuda_9_2":
Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
256
        return compiler == "gcc_6"
Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
257
258
    if cuda == "cuda_10_0":
        return compiler == "gcc_6"
Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
259
260
    if cuda == "cuda_10_1":
        return compiler == "gcc_8"
Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
261
262
    if cuda == "cuda_10_2":
        return compiler == "gcc_8"
Lorenz Hüdepohl's avatar
Lorenz Hüdepohl committed
263
264
    if cuda == "cuda_11_0":
        return compiler == "gcc_9"
Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
265
266
    if cuda == "cuda_11_2":
        return compiler == "gcc_10"
Lorenz Hüdepohl's avatar
Lorenz Hüdepohl committed
267
268
    if cuda == "cuda_11_4":
        return compiler == "gcc_11"
269
270
271
    return False


Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
272
273
274
def project_meta(api_url, project):
    return ElementTree.fromstringlist(osc.core.show_project_meta(api_url, project))

275

276
277
278
279
280
281
282
283
284
def package_meta(api_url, project, package):
    return ElementTree.fromstringlist(osc.core.show_package_meta(api_url, project, package))


def maintainers(api_url, project, package):
    root = package_meta(api_url, project, package)
    return {e.get("userid") for e in root.findall("./person[@role='maintainer']")}


285
286
287
288
class UnsetAttributeException(Exception):
    pass


289
290
291
292
class UnmanagedPackageException(Exception):
    pass


293
294
def chunked(items, chunksize):
    n = len(items)
295
296
    i = 0
    while i * chunksize < n:
297
        yield items[i * chunksize: (i + 1) * chunksize]
298
299
300
        i += 1


Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
301
def get_attribute(api_url, project, package, attribute, with_project=False):
302
303
    attribute_meta = osc.core.show_attribute_meta(api_url, project, package, None, attribute, False, with_project)
    if attribute_meta is None:
Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
304
        raise osc.oscerr.APIError("Cannot fetch value for attribute '{0}' from {1}".format(attribute, (project, package)))
305
306
307
308

    root = ElementTree.fromstringlist(attribute_meta)
    attribute = root.find("./attribute")
    if attribute is not None:
Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
309
        return root
310
    else:
311
        raise UnsetAttributeException("Attribute not set")
312

Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
313

314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
def overloaded_project_attribute(api_url, project, attribute):
    try:
        return get_attribute_values(api_url, project, None, attribute)
    except UnsetAttributeException:
        return get_attribute_values(api_url, "software", None, attribute)


def overloaded_package_attribute(api_url, project, package, attribute):
    try:
        return get_attribute_values(api_url, project, package, attribute)
    except UnsetAttributeException:
        pass

    try:
        return get_attribute_values(api_url, project, None, attribute)
    except UnsetAttributeException:
        pass

    return get_attribute_values(api_url, "software", None, attribute)


def get_allowed_attribute_values(api_url, attribute):
    path = ["attribute"] + attribute.split(":") + ["_meta"]
    url = osc.core.makeurl(api_url, path, [])
    try:
        f = osc.core.http_GET(url)
    except osc.core.HTTPError as e:
        e.osc_msg = 'Error getting meta for attribute "{0}"'.format(attribute)
        raise

    root = ElementTree.fromstringlist(f.readlines())
    return list(value.text for value in root.findall("./allowed/value"))


Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
348
349
350
351
352
353
def get_attribute_values(api_url, project, package, attribute, with_project=False):
    root = get_attribute(api_url, project, package, attribute, with_project)
    attribute = root.find("./attribute")
    return list(value.text for value in attribute.findall("./value"))


354
355
356
357
358
def get_attribute_value(api_url, project, package, attribute, with_project=False):
    value, = get_attribute_values(api_url, project, package, attribute, with_project=False)
    return value


359
360
361
362
363
364
def set_attribute(api_url, project, package, attribute):
    path = ["source", project]
    if package:
        path.append(package)
    path.append("_attribute")

Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
365
    attr_meta = ElementTree.tostring(attribute, encoding=osc.core.ET_ENCODING)
Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
366

367
    url = osc.core.makeurl(api_url, path)
Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
368
    resp = ElementTree.fromstringlist(osc.core.streamfile(url, osc.core.http_POST, data=attr_meta))
Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
369
    if resp.find("./summary").text != "Ok":
Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
370
        raise osc.oscerr.APIError("Could not store attribute")
Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
371

372

373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
def set_attribute_values(api_url, project, package, attribute, values):
    root = ElementTree.Element("attributes")
    attr = ElementTree.SubElement(root, "attribute")

    namespace, name = attribute.split(":")
    attr.set("namespace", namespace)
    attr.set("name", name)

    for value in values:
        v = ElementTree.SubElement(attr, "value")
        v.text = value

    set_attribute(api_url, project, package, root)


def has_attribute(api_url, project, package, attribute):
    path = ["source", project]
    if package:
        path.append(package)
    path.append("_attribute")
    path.append(attribute)

    url = osc.core.makeurl(api_url, path)
    resp = ElementTree.fromstringlist(osc.core.streamfile(url, osc.core.http_GET))

    namespace, name = attribute.split(":")
399

400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
    for a in resp.findall("./attribute"):
        if a.get("namespace") == namespace and a.get("name") == name:
            return True
    return False


def remove_attribute(api_url, project, package, attribute_name):
    path = ["source"] + [project]
    if package:
        path.append(package)
    path.append("_attribute")
    path.append(attribute_name)

    url = osc.core.makeurl(api_url, path)
    resp = ElementTree.fromstringlist(osc.core.streamfile(url, osc.core.http_DELETE))
    if resp.find("./summary").text != "Ok":
        raise osc.oscerr.APIError("Could not remove attribute")


419
def get_microarchitecture(project):
420
    if project.startswith("software:"):
421
        microarch = project.split(":")[2]
422
423
424
425
    elif project.startswith("home:"):
        microarch = "sandybridge"
    else:
        raise Exception("Cannot determine micro-architecture for project '{0}'".format(project))
426
427
428
429
430
431
432

    if microarch in known_microarchs:
        return microarch
    else:
        raise Exception("Unknown micro-architecture '{0}'".format(microarch))


433
434
435
436
437
438
def package_sort_key(string):
    name, version = string.split("_", 1)
    version = version.split("_")
    return (name,) + tuple(map(int, version))


439
def mpcdf_enable_repositories(api_url, project, package, verbose=False, dry_run=False, ignore_repos=()):
440
441
    from itertools import product

442
443
444
    pkg_meta = osc.core.show_package_meta(api_url, project, package)
    root = ElementTree.fromstringlist(pkg_meta)

445
    build = root.find("./build")
Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
446
447
    if build is None:
        build = ElementTree.SubElement(root, "build")
448
449
450
451
452

    pkg_meta = ElementTree.tostring(root, encoding=osc.core.ET_ENCODING)

    for enable in build.findall("./enable"):
        build.remove(enable)
453

454
455
    disabled_repos = {disable.get("repository") for disable in build.findall("./disable")}

Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
456
    try:
457
458
459
460
        enable_repos = get_attribute_values(api_url, project, package, "MPCDF:enable_repositories")
    except Exception:
        if verbose:
            print("Warning: Could not get attribute MPCDF:enable_repositories for package {0}, skipping".format(package))
461
        raise UnmanagedPackageException()
Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
462

463
464
    if project == "software":
        distributions = list(sorted(repo.name for repo in osc.core.get_repos_of_project(api_url, "software")))
Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
465

466
467
468
469
470
    else:
        compilers = overloaded_package_attribute(api_url, project, package, "MPCDF:compiler_modules")
        mpis = overloaded_package_attribute(api_url, project, package, "MPCDF:mpi_modules")
        cudas = overloaded_package_attribute(api_url, project, package, "MPCDF:cuda_modules")
        pgis = overloaded_package_attribute(api_url, project, package, "MPCDF:pgi_modules")
471
        openmpi_flavors = overloaded_package_attribute(api_url, project, package, "MPCDF:openmpi_flavors")
472

473
474
475
476
        all_compilers = overloaded_project_attribute(api_url, project, "MPCDF:compiler_modules")
        all_mpis = overloaded_project_attribute(api_url, project, "MPCDF:mpi_modules")
        all_cudas = overloaded_project_attribute(api_url, project, "MPCDF:cuda_modules")
        all_pgis = overloaded_project_attribute(api_url, project, "MPCDF:pgi_modules")
477
        all_openmpi_flavors = overloaded_project_attribute(api_url, project, "MPCDF:openmpi_flavors")
478

479
480
        latest_intel = sorted((c for c in all_compilers if c.startswith("intel")), key=package_sort_key)[-1]
        latest_gcc = sorted((c for c in all_compilers if c.startswith("gcc")), key=package_sort_key)[-1]
481

482
483
    i = len(build)

484
    def enable(name):
485
        if name in ignore_repos or name in disabled_repos:
486
            return
487
488
489
        node = ElementTree.Element("enable")
        node.set("repository", name)
        node.tail = "\n    "
490
        build.insert(i, node)
491
492
493
        if verbose:
            print("Enabling", name)

494
    def actual_compilers():
495
        for compiler in (c for c in compilers if c in all_compilers + ["intel", "gcc", "latest_intel", "latest_gcc"]):
496
            if compiler == "intel":
Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
497
                for intel_compiler in [cc for cc in all_compilers if cc.startswith("intel")]:
498
499
                    yield intel_compiler
            elif compiler == "gcc":
Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
500
                for gcc_compiler in [cc for cc in all_compilers if cc.startswith("gcc")]:
501
                    yield gcc_compiler
502
503
504
505
            elif compiler == "latest_intel":
                yield latest_intel
            elif compiler == "latest_gcc":
                yield latest_gcc
506
507
508
509
            else:
                yield compiler

    def actual_mpis():
510
        for mpi in (m for m in mpis if m in all_mpis + ["impi"]):
511
            if mpi == "impi":
512
                for impi in [mpi for mpi in all_mpis if mpi.startswith("impi")]:
513
514
515
516
517
                    yield impi
            else:
                yield mpi

    def actual_cudas():
518
519
        for cuda in (c for c in cudas if c in all_cudas):
            yield cuda
520

521
522
523
524
    def actual_pgis():
        for pgi in (p for p in pgis if p in all_pgis):
            yield pgi

525
526
527
528
    def actual_openmpi_flavors():
        for of in (f for f in openmpi_flavors if f in all_openmpi_flavors):
            yield of

529
    for flag in enable_repos:
530
531
        if flag == "system":
            if project == "software":
532
533
                for distribution in distributions:
                    enable(distribution)
534
535
            elif project.startswith("home:"):
                enable("System")
536

537
        elif project != "software":
538
539
540
541
542
543
544
545
546
            if flag == "compilers":
                for compiler in actual_compilers():
                    enable(compiler)

            if flag == "mpi":
                for mpi, compiler in product(actual_mpis(), actual_compilers()):
                    if valid_mpi(compiler, mpi):
                        enable(mpi + "_" + compiler)

547
548
549
550
551
552
            if flag == "openmpi_flavors":
                for mpi, compiler in product(actual_mpis(), actual_compilers()):
                    if "openmpi" in mpi and valid_mpi(compiler, mpi):
                        for of in actual_openmpi_flavors():
                            enable(mpi + "_" + compiler + "_" + of)

553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
            if flag == "cuda":
                for cuda, compiler in product(actual_cudas(), all_compilers):
                    if valid_cuda(cuda, compiler):
                        enable(cuda + "_" + compiler)

            if flag == "cuda_mpi":
                for cuda, mpi, compiler in product(actual_cudas(), actual_mpis(), all_compilers):
                    if valid_cuda(cuda, compiler) and valid_mpi(compiler, mpi):
                        enable(cuda + "_" + mpi + "_" + compiler)

            if flag == "pgi":
                for pgi in actual_pgis():
                    enable(pgi)

            if flag == "pgi_mpi":
                for mpi, pgi in product(actual_mpis(), actual_pgis()):
                    if valid_pgi_mpi(pgi, mpi):
                        enable(mpi + "_" + pgi)
571

572
573
            if flag == "cuda_aware_mpi":
                for cuda, mpi, compiler in product(actual_cudas(), actual_mpis(), all_compilers):
574
                    if "openmpi" in mpi and valid_cuda(cuda, compiler) and valid_mpi(compiler, mpi):
575
576
                        enable(cuda + "_aware_" + mpi + "_" + compiler)

577
578
579
580
581
582
            if flag == "cuda_aware_openmpi_flavors":
                for cuda, mpi, compiler in product(actual_cudas(), actual_mpis(), all_compilers):
                    if "openmpi" in mpi and valid_cuda(cuda, compiler) and valid_mpi(compiler, mpi):
                        for of in actual_openmpi_flavors():
                            enable(cuda + "_aware_" + mpi + "_" + compiler + "_" + of)

Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
583
584
    if len(build.getchildren()) > 0:
        build.getchildren()[-1].tail = "\n  "
585

586
587
    new_pkg_meta = ElementTree.tostring(root, encoding=osc.core.ET_ENCODING)
    if pkg_meta != new_pkg_meta:
588
        print("Updating repositories for", package)
589
590
591
        if dry_run:
            print("osc meta pkg {0} {1} -F - <<EOF\n{2}\nEOF\n".format(project, package, new_pkg_meta))
        else:
592
            osc.core.edit_meta("pkg", (project, package), data=new_pkg_meta, apiurl=api_url)
593
594
    elif dry_run:
        print("Would not do anything, package meta would be unchanged")
595
596

    return True
Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
597
598


599
600
601
602
603
604
605
606
def parse_prjconf(api_url, project):
    orig_prjconf = list(map(decode_it, osc.core.show_project_conf(api_url, project)))
    try:
        start = orig_prjconf.index(prjconf_start_marker)
        end = orig_prjconf.index(prjconf_end_marker)
    except ValueError:
        start = None
        end = len(orig_prjconf)
607

608
609
610
611
612
613
    prjconf_head = orig_prjconf[:start]
    prjconf_ours = orig_prjconf[start:end]
    prjconf_tail = orig_prjconf[end + 1:]
    return orig_prjconf, prjconf_head, prjconf_ours, prjconf_tail


614
615
616
617
618
619
620
621
622
def openmpi_flavor_dependencies(flavor, distribution):
    if flavor.startswith("mofed_"):
        mofed_version = flavor[len("mofed_"):].replace("_", ".")
        mofed_repo = "extern:mofed:" + mofed_version
        return ((mofed_repo, distribution),)
    else:
        return ()


623
def mpcdf_setup_subproject(api_url, project, distribution, microarchitecture,
624
625
                           parent=None, dry_run=False, diff=False, remove_old=False, remove_old_matching=None, all_possible=False, only_project=False):
    import re
626
627

    if parent and not dry_run:
628
        for attribute in config_attributes:
629
            print("Copying attribute '{0}' from parent project".format(attribute))
630
            set_attribute_values(api_url, project, None, attribute, overloaded_project_attribute(api_url, parent, attribute))
Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
631

632
633
634
    # Check distribution
    software_meta = project_meta(api_url, "software")
    dist_repo = software_meta.find('./repository[@name="{0}"]'.format(distribution))
Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
635
    if dist_repo is None:
636
        raise osc.oscerr.WrongArgs("Invalid distribution '{0}': No matching repository is defined in project 'software' on the server".format(distribution))
Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
637
638
639
640
    architectures = list(arch.text for arch in dist_repo.findall("./arch"))

    root = project_meta(api_url, project)

641
642
643
644
645
646
647
    orig_prjconf, prjconf_head, prjconf_ours, prjconf_tail = parse_prjconf(api_url, project)

    prjconf_repos = {}
    cur_repo = None
    old_repos = set()
    for line in prjconf_ours:
        line = line.rstrip("\n")
Lorenz Hüdepohl's avatar
Lorenz Hüdepohl committed
648
649
        if cur_repo is None and (line.startswith("%if %_repository ==") or line.startswith("%if \"%{_repository}\" == ")) and len(line.split()) == 4:
            cur_repo = line.split()[-1].strip('"')
650
651
652
653
654
655
656
657
658
659
            old_repos.add(cur_repo)
            prjconf_repos[cur_repo] = []
        if cur_repo is not None:
            prjconf_repos[cur_repo].append(line)
            if line == "%endif":
                cur_repo = None

    prjconf_ours = [prjconf_start_marker]

    prjconf_ours.append("Constraint: hostlabel {0}".format(microarchitecture))
Lorenz Hüdepohl's avatar
Lorenz Hüdepohl committed
660
    prjconf_ours.append("Preinstall: mpcdf_{0}_directory".format(microarchitecture))
661
    prjconf_ours.append("PublishFilter: ^mpcdf_.*$")
Lorenz Hüdepohl's avatar
Lorenz Hüdepohl committed
662
    prjconf_ours.append("")
663

Lorenz Hüdepohl's avatar
Lorenz Hüdepohl committed
664
665
666
667
    extra_tags = dist_prjconf_tags(distribution)
    if extra_tags:
        prjconf_ours.append(extra_tags)
    extra_macros = dist_prjconf_macros(distribution)
668

669
670
671
    if not project.startswith("home:"):
        extra_macros = "%microarchitecture {0}\n{1}".format(microarchitecture, extra_macros)

672
    prjconf_ours.append("""
673
Macros:
674
675
{0}
:Macros""".format(extra_macros))
676
    prjconf_ours.append("")
677

Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
678
    # Remove existing repositories
679
680
681
    if remove_old:
        for oldrepo in root.findall("./repository"):
            root.remove(oldrepo)
682
683
684
685
    if remove_old_matching:
        for oldrepo in root.findall("./repository"):
            if re.search(remove_old_matching, oldrepo.attrib["name"]):
                root.remove(oldrepo)
Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
686

687
    def repo(name, dependencies, compiler=False, mpi=False, cuda=False, cuda_mpi=False, cuda_aware_mpi=False, additional_tags=(), **macros):
688
        old_repos.discard(name)
689
690
691
        have_compiler = compiler or mpi or cuda or cuda_mpi or cuda_aware_mpi
        have_mpi = mpi or cuda_mpi or cuda_aware_mpi
        have_cuda = cuda or cuda_mpi or cuda_aware_mpi
692

693
694
695
        existing_repo = root.find("./repository[@name='{0}']".format(name))
        if existing_repo is not None:
            root.remove(existing_repo)
696
        elif dry_run is False:
697
698
            print("New repository", name)

Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
699
700
        r = ElementTree.SubElement(root, "repository")
        r.set("name", name)
Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
701
702
        r.text = "\n    "

Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
703
704
705
706
        def path(project, repo):
            p = ElementTree.SubElement(r, "path")
            p.set("project", project)
            p.set("repository", repo)
Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
707
            p.tail = "\n    "
Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
708

709
        # TODO: Order inverted here?
710
711
        for dep_project, dep_repo in dependencies:
            path(dep_project, dep_repo)
Lorenz Hüdepohl's avatar
Lorenz Hüdepohl committed
712
713
        if parent and name != "System":
            path(parent, name)
Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
714
715
716
717
718
719
720
721

        for arch in architectures:
            a = ElementTree.SubElement(r, "arch")
            a.text = arch
            a.tail = "\n    "
        a.tail = "\n  "
        r.tail = "\n  "

Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
722
723
724
725
        # In order to be able to figure out the matching MKL version for a given
        # compiler/MPI repository we emit a new macro '%matching_mkl_version' in
        # the cases this makes sense
        matching_mkl = []
726
        repoconf = []
Lorenz Hüdepohl's avatar
Lorenz Hüdepohl committed
727
728
729
730
731
        repoconf.append("%if \"%{{_repository}}\" == \"{0}\"".format(name))
        for tag in repo_tags(name, distribution=distribution) + additional_tags:
            if tag.startswith("Prefer: mkl"):
                matching_mkl.append(tag.split()[1])
            repoconf.append(tag)
Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
732

733
        repoconf.append("Macros:")
734
735
        if project.startswith("home:") and name != "System":
            repoconf.append("%microarchitecture {0}".format(microarchitecture))
736

737
738
739
740
        repoconf.append("%is_compiler_repository {0}".format(1 if compiler else 0))
        repoconf.append("%is_mpi_repository {0}".format(1 if mpi else 0))
        repoconf.append("%is_cuda_repository {0}".format(1 if cuda else 0))
        repoconf.append("%is_cuda_mpi_repository {0}".format(1 if cuda_mpi else 0))
741
        repoconf.append("%is_cuda_aware_mpi_repository {0}".format(1 if cuda_aware_mpi else 0))
742

743
744
745
        repoconf.append("%have_mpcdf_compiler {0}".format(1 if have_compiler else 0))
        repoconf.append("%have_mpcdf_mpi {0}".format(1 if have_mpi else 0))
        repoconf.append("%have_mpcdf_cuda {0}".format(1 if have_cuda else 0))
746

Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
747
748
749
750
        if matching_mkl:
            matching_mkl, = matching_mkl
            matching_mkl, _ = matching_mkl[len("mkl_"):].split("-module")
            matching_mkl = matching_mkl.replace("_", ".")
751
            repoconf.append("%matching_mkl_version {0}".format(matching_mkl))
Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
752

753
754
        for macro, value in macros.items():
            repoconf.append("%{0} {1}".format(macro, value))
755

756
757
758
        repoconf.append(":Macros")
        repoconf.append("%endif")
        prjconf_repos[name] = repoconf
Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
759

760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
    def actual_compiler(c):
        if c.startswith("latest"):
            return False
        if c == "intel":
            return False
        if c == "gcc":
            return False
        return True

    def actual_mpi(m):
        if m == "impi":
            return False
        return True

    def actual_cuda(c):
        return True

    if all_possible:
        compilers = list(filter(actual_compiler, get_allowed_attribute_values(api_url, "MPCDF:compiler_modules")))
        mpis = list(filter(actual_mpi, get_allowed_attribute_values(api_url, "MPCDF:mpi_modules")))
        cudas = list(filter(actual_cuda, get_allowed_attribute_values(api_url, "MPCDF:cuda_modules")))
        pgis = get_allowed_attribute_values(api_url, "MPCDF:pgi_modules")
782
        openmpi_flavors = get_allowed_attribute_values(api_url, "MPCDF:openmpi_flavors")
Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
783
    else:
784
785
786
787
        compilers = overloaded_project_attribute(api_url, project, "MPCDF:compiler_modules")
        mpis = overloaded_project_attribute(api_url, project, "MPCDF:mpi_modules")
        cudas = overloaded_project_attribute(api_url, project, "MPCDF:cuda_modules")
        pgis = overloaded_project_attribute(api_url, project, "MPCDF:pgi_modules")
788
        openmpi_flavors = overloaded_project_attribute(api_url, project, "MPCDF:openmpi_flavors")
789
790

    if parent:
Lorenz Hüdepohl's avatar
Lorenz Hüdepohl committed
791
        repo("System", (("software", distribution),))
Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
792

793
    for compiler in compilers + pgis:
Lorenz Hüdepohl's avatar
Lorenz Hüdepohl committed
794
795
796
        if project.startswith("home:"):
            dependencies = ((project, "System"),)
        elif project != "software":
Lorenz Hüdepohl's avatar
Lorenz Hüdepohl committed
797
            dependencies = (("software", distribution),)
798
799
800
801
        else:
            dependencies = ()
        repo(compiler, dependencies, compiler=True,
             compiler_repository=compiler, compiler_module=compiler_module(compiler),
Lorenz Hüdepohl's avatar
Lorenz Hüdepohl committed
802
             additional_tags=("Prefer: mpcdf_compiler_" + compiler,))
Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
803
804

        for mpi in filter(partial(valid_mpi, compiler), mpis):
Lorenz Hüdepohl's avatar
Lorenz Hüdepohl committed
805
            repo(mpi + "_" + compiler, ((project, compiler),), mpi=True,
806
                 mpi_repository=mpi, mpi_module=mpi_module(mpi))
Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
807

808
809
810
            if "openmpi" in mpi:
                for of in openmpi_flavors:
                    dependencies = ((project, compiler),)
811
812
                    if not parent:
                        dependencies = openmpi_flavor_dependencies(of, distribution) + dependencies
813
                    flavor_kind = re.sub("(_[0-9]+)*$", "", of)
814
815
                    repo(mpi + "_" + compiler + "_" + of, dependencies, mpi=True,
                         mpi_repository=mpi, mpi_module=mpi_module(mpi),
816
                         openmpi_flavor=flavor_kind)
817

Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
818
819
    for cuda in cudas:
        for compiler in filter(partial(valid_cuda, cuda), compilers):
Lorenz Hüdepohl's avatar
Lorenz Hüdepohl committed
820
            repo(cuda + "_" + compiler, ((project, compiler),), cuda=True, cuda_repository=cuda,
Lorenz Hüdepohl's avatar
Lorenz Hüdepohl committed
821
                 additional_tags=("Prefer: mpcdf_" + cuda,))
Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
822
823
            for mpi in filter(partial(valid_mpi, compiler), mpis):
                repo(cuda + "_" + mpi + "_" + compiler,
Lorenz Hüdepohl's avatar
Lorenz Hüdepohl committed
824
825
                     ((project, cuda + "_" + compiler),
                      (project, mpi + "_" + compiler)),
826
                     cuda_mpi=True)
Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
827

828
829
830
                if "openmpi" in mpi:
                    repo(cuda + "_aware_" + mpi + "_" + compiler,
                         ((project, cuda + "_" + mpi + "_" + compiler),),
831
                         cuda_aware_mpi=True,
832
833
834
835
                         additional_tags=("Prefer: mpcdf_mpi_" + mpi + "_" + cuda,
                                          "Prefer: mpcdf_" + cuda,))
                    for of in openmpi_flavors:
                        dependencies = ((project, cuda + "_" + mpi + "_" + compiler),)
836
837
                        if not parent:
                            dependencies = openmpi_flavor_dependencies(of, distribution) + dependencies
838
839
840
841
                        repo(cuda + "_aware_" + mpi + "_" + compiler + "_" + of,
                             dependencies,
                             cuda_aware_mpi=True, openmpi_flavor=of,
                             additional_tags=("Prefer: mpcdf_" + cuda,))
842

843
844
845
846
847
848
849
850
851
    if old_repos and not remove_old:
        print("Warning: Keeping the prjconf sections for the following obsolete repositories:")
        for name in sorted(old_repos):
            print(" -", name)
        print()
    else:
        for old_repo in old_repos:
            del prjconf_repos[old_repo]

Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
852
853
854
855
856
857
858
859
860
861
    # Remove build configuration
    build = root.find("./build")
    if build is None:
        build = ElementTree.Element("build")
        build.text = "\n    "
        disable = ElementTree.SubElement(build, "disable")
        disable.tail = "\n  "
        build.tail = "\n  "
        root.insert(list(root).index(root.find("./repository")), build)

862
863
864
865
866
867
868
869
870
871
872
    def sort_repos(root):
        # Sort existing repositories to be able to compare
        repos = root.findall("./repository")
        for repo in repos:
            root.remove(repo)
        repos = sorted(repos, key=lambda child: child.get("name"))
        for repo in repos:
            root.append(repo)
        root.getchildren()[-1].tail = "\n"

    sort_repos(root)
873
    new_prj = ElementTree.tostring(root, encoding=osc.core.ET_ENCODING)
Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
874

875
    for name in sorted(prjconf_repos.keys()):
876
877
878
879
880
881
        if name not in old_repos:
            prjconf_ours.extend(prjconf_repos[name])
            prjconf_ours.append("")

    if not remove_old:
        prjconf_ours.append("# Obsolete repos: ")
882
        prjconf_ours.append("")
883
884
885
        for name in sorted(old_repos):
            prjconf_ours.extend(prjconf_repos[name])
            prjconf_ours.append("")
Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
886

887
    prjconf_ours.append(prjconf_end_marker)
888

889
    new_prjconf = "".join(prjconf_head) + "\n".join(prjconf_ours) + "".join(prjconf_tail)
890

891
892
893
894
    old_prj_root = project_meta(api_url, project)
    sort_repos(old_prj_root)
    old_prj = ElementTree.tostring(old_prj_root, encoding=osc.core.ET_ENCODING)

895
896
897
898
899
900
901
902
903
904
905
906
907
908
    if diff:
        def stringdiff(s1, label1, s2, label2):
            import os
            from subprocess import call
            from tempfile import NamedTemporaryFile

            old = NamedTemporaryFile("w+", delete=False)
            new = NamedTemporaryFile("w+", delete=False)

            old.write(s1)
            old.close()
            new.write(s2)
            new.close()

909
            call(["diff", "--color=auto", "-s", "-u", old.name, "--label", label1, new.name, "--label", label2])
910
911
912
913
914
915
            os.unlink(old.name)
            os.unlink(new.name)

        stringdiff("".join(orig_prjconf), "old-prjconf", new_prjconf, "new-prjconf")
        stringdiff(old_prj + "\n", "old-prj-meta", new_prj + "\n", "new-prj-meta")

916
    if dry_run:
917
918
919
        if not diff:
            print("osc meta prjconf {0} -F - <<EOF\n{1}\nEOF\n".format(project, new_prjconf))
            print("osc meta prj {0} -F - <<EOF\n{1}\nEOF\n".format(project, new_prj))
920
921
922
923
924
    else:
        if new_prjconf == "".join(orig_prjconf):
            print("prjconf unchanged")
        else:
            print("Updating prjconf meta")
925
            osc.core.edit_meta("prjconf", project, data=new_prjconf, apiurl=api_url)
926

927
928
        # Create and remove the <enable/> flags before the new repsitories,
        # that way no spurious builds are launched
929
930
931
        if not only_project:
            print("Updating enabled repositories for all packages")
            mpcdf_enable_repositories_for_all_packages(api_url, project)
Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
932
933

        # Update repositories
934
935
936
937
938
        if new_prj == old_prj:
            print("prj meta unchanged")
        else:
            print("Updating prj meta")
            osc.core.edit_meta("prj", project, data=new_prj, force=True, apiurl=api_url)
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964


def mpcdf_enable_repositories_for_all_packages(api_url, project, ignore_repos=()):
    import threading
    packages = osc.core.meta_get_packagelist(api_url, project)

    if len(packages) > 40:
        chunksize = len(packages) // 20
    else:
        chunksize = len(packages)

    def work(packagelist):
        for package in packagelist:
            try:
                mpcdf_enable_repositories(api_url, project, package, ignore_repos=ignore_repos)
            except UnmanagedPackageException:
                print("ATTENTION: Not changing unmanaged package {0}".format(package))

    threads = []
    for packagelist in chunked(packages, chunksize):
        t = threading.Thread(target=work, args=(packagelist,))
        threads.append(t)
        t.start()

    for t in threads:
        t.join()
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
1001
1002
1003
1004


def set_as_branch(api_url, my_project, my_package, main_project, main_package):
    import os

    def dirmeta(project, package):
        url = osc.core.makeurl(api_url, ["source", project, package])
        return ElementTree.fromstringlist(osc.core.streamfile(url, osc.core.http_GET))

    srcmeta = dirmeta(my_project, my_package)
    dstmeta = dirmeta(main_project, main_package)

    linkinfo = srcmeta.find('./linkinfo')
    if linkinfo is not None:
        print("ERROR: package {0} is already linked to package {1} in project {2}".format(
              my_package, linkinfo.get("package"), linkinfo.get("project")))
        return False

    linkfile = ElementTree.Element("link")
    linkfile.set("project", main_project)
    linkfile.set("package", main_package)
    linkfile.set("baserev", dstmeta.get("srcmd5"))
    patches = ElementTree.SubElement(linkfile, "patches")
    ElementTree.SubElement(patches, "branch")

    url = osc.core.makeurl(api_url, ["source", my_project, my_package, "_link"])
    resp = ElementTree.fromstringlist(osc.core.streamfile(url, osc.core.http_PUT, data=ElementTree.tostring(linkfile)))
    rev = resp.get("rev")
    if rev is None:
        print("ERROR: Could not commit _link file")
        return False
    else:
        print("Commited as revision", rev)
        this_package = osc.core.store_read_package(os.curdir)
        this_project = osc.core.store_read_project(os.curdir)
        if this_package == my_package and this_project == my_project:
            pac = osc.core.filedir_to_pac(os.curdir)
            rev = pac.latest_rev(expand=True)
            pac.update(rev)
    return True
1005
1006


Lorenz Hüdepohl's avatar
Lorenz Hüdepohl committed
1007
def sync_projects(api_url, package=None, from_project="software", to_projects=None, add_to_maintainers=True, verbose=False):
1008
    if to_projects is None:
Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
1009
1010
        to_projects = [p for p in osc.core.meta_get_project_list(api_url)
                       if p.startswith("software:") and not (p == "software:dist" or p == "software:images")]
1011

1012
1013
1014
1015
    allowed_attribute_values = {}
    for attribute in package_attributes + config_attributes:
        allowed_attribute_values[attribute] = set(get_allowed_attribute_values(api_url, attribute))

1016
1017
1018
1019
1020
1021
1022
1023
1024
1025
1026
    for to_project in to_projects:
        print("Syncing {0} with {1}".format(to_project, from_project))

        to_packages = osc.core.meta_get_packagelist(api_url, to_project)

        if package is None:
            from_packages = osc.core.meta_get_packagelist(api_url, from_project)
        else:
            from_packages = [package]

        for orig_package in from_packages:
1027
            enable_repos = get_attribute_values(api_url, from_project, orig_package, "MPCDF:enable_repositories")
1028
            if orig_package not in to_packages:
1029
                if len(enable_repos) == 0:
1030
                    print("Not branching package {0}, is disabled".format(orig_package))
1031
                    continue
1032
                elif enable_repos == ["system"] and "System" not in osc.core.get_repositories_of_project(api_url, to_project):
1033
                    print("Not branching package {0}, is only enabled for 'system'".format(orig_package))
1034
                    continue
1035
1036
1037
1038
1039
1040
1041
1042
1043
1044
                filelist = osc.core.meta_get_filelist(api_url, from_project, orig_package)
                if "_link" in filelist:
                    print("Not branching package {0}, is a link".format(orig_package))
                else:
                    print("Branching package {0}".format(orig_package))
                    exists, targetprj, targetpkg, srcprj, srcpkg = \
                        osc.core.branch_pkg(api_url, from_project, orig_package, target_project=to_project, nodevelproject=True)
            else:
                print("Not branching package {0}, already present in target".format(orig_package))

1045
            for attribute in package_attributes + config_attributes:
1046
                try:
1047
1048
                    values = list(filter(lambda q: q in allowed_attribute_values[attribute],
                                         get_attribute_values(api_url, from_project, orig_package, attribute)))
1049
1050
1051
1052
                except UnsetAttributeException:
                    if has_attribute(api_url, to_project, orig_package, attribute):
                        remove_attribute(api_url, to_project, orig_package, attribute)
                    continue
1053
                set_attribute_values(api_url, to_project, orig_package, attribute, values)
1054

1055
1056
1057
1058
1059
1060
1061
1062
1063
1064
1065
1066
            if add_to_maintainers:
                from_maintainers = maintainers(api_url, from_project, orig_package)
                if from_maintainers:
                    to_maintainers = maintainers(api_url, to_project, orig_package)
                    if from_maintainers - to_maintainers:
                        new_maintainers = from_maintainers - to_maintainers
                        to_meta = package_meta(api_url, to_project, orig_package)
                        for userid in sorted(new_maintainers):
                            person = ElementTree.Element("person")
                            person.set("userid", userid)
                            person.set("role", "maintainer")
                            to_meta.insert(2, person)
1067
                        osc.core.edit_meta("pkg", (to_project, orig_package), data=ElementTree.tostring(to_meta), apiurl=api_url)
1068

1069
1070
1071
1072
            try:
                mpcdf_enable_repositories(api_url, to_project, orig_package, verbose=verbose)
            except UnmanagedPackageException:
                print("ATTENTION: Not changing unmanaged package {0}".format(orig_package))