mpcdf_common.py 26.8 KB
Newer Older
1
from __future__ import print_function
Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
2
from __future__ import division
3
4
5
6

import osc
import osc.conf
import osc.core
Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
7
import osc.oscerr
Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
8
from osc.util.helper import decode_it
Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
9
from functools import partial
10
11
from xml.etree import ElementTree

12
13
known_microarchs = {"sandybridge", "haswell", "skylake"}

14
15
package_attributes = ["MPCDF:enable_repositories"]
config_attributes = ["MPCDF:compiler_modules", "MPCDF:cuda_modules", "MPCDF:mpi_modules", "MPCDF:pgi_modules"]
16
default_attributes = ["MPCDF:default_compiler", "MPCDF:default_cuda", "MPCDF:default_mpi", "MPCDF:default_python2", "MPCDF:default_python3"]
17

18
19
20
21
22
23
24
25
intel_parallel_studio = {
    "mpcdf_intel_parallel_studio_2017_7": {"compiler": "intel_17_0_7", "impi": "impi_2017_4", "mkl": "mkl_2017_4-module", },
    "mpcdf_intel_parallel_studio_2018_1": {"compiler": "intel_18_0_1", "impi": "impi_2018_1", "mkl": "mkl_2018_1-module", },
    "mpcdf_intel_parallel_studio_2018_2": {"compiler": "intel_18_0_2", "impi": "impi_2018_2", "mkl": "mkl_2018_2-module", },
    "mpcdf_intel_parallel_studio_2018_3": {"compiler": "intel_18_0_3", "impi": "impi_2018_3", "mkl": "mkl_2018_3-module", },
    "mpcdf_intel_parallel_studio_2018_4": {"compiler": "intel_18_0_5", "impi": "impi_2018_4", "mkl": "mkl_2018_4-module", },
    "mpcdf_intel_parallel_studio_2019_0": {"compiler": "intel_19_0_0", "impi": "impi_2019_0", "mkl": "mkl_2019_0-module", },
    "mpcdf_intel_parallel_studio_2019_1": {"compiler": "intel_19_0_1", "impi": "impi_2019_1", "mkl": "mkl_2019_1-module", },
Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
26
    "mpcdf_intel_parallel_studio_2019_3": {"compiler": "intel_19_0_3", "impi": "impi_2019_3", "mkl": "mkl_2019_3-module", },
27
    "mpcdf_intel_parallel_studio_2019_4": {"compiler": "intel_19_0_4", "impi": "impi_2019_4", "mkl": "mkl_2019_4-module", },
28
29
30
31
32
33
34
35
}

all_mkls = {ic["mkl"] for ic in intel_parallel_studio.values()}

# Some rotations
mpi_parallel_studio = {value["impi"]: dict({"ps": key}, **value) for key, value in intel_parallel_studio.items()}
compiler_parallel_studio = {value["compiler"]: dict({"ps": key}, **value) for key, value in intel_parallel_studio.items()}

36

37
38
39
40
41
42
43
44
def valid_pgi_mpi(pgi, mpi):
    if "impi" not in mpi:
        return False
    if "2017" in mpi:
        return False
    return True


45
def valid_mpi(compiler, mpi):
46
47
48
49
50
51
52
53
54
    """
    It might be possible to use Intel MPI libararies and compilers from
    different Parallel Studio packages, but I currently do not want to support
    it.

    Take care to keep this in sync with the file 'macros.obs_cluster' of
    the package software:dist / mpcdf_cluster_macros
    """
    if compiler.startswith("intel") and mpi.startswith("impi"):
55
        return mpi == compiler_parallel_studio[compiler]["impi"]
56
57
    if compiler.startswith("pgi"):
        return valid_pgi_mpi(compiler, mpi)
58
59
    else:
        return True
60
61


62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
def prefers(reponame):
    prefer_ps = None
    if reponame in compiler_parallel_studio:
        prefer_ps = compiler_parallel_studio[reponame]["ps"]
    else:
        for mpi in mpi_parallel_studio:
            if reponame.startswith(mpi):
                prefer_ps = mpi_parallel_studio[mpi]["ps"]

    if prefer_ps:
        preferred_mkl = intel_parallel_studio[prefer_ps]["mkl"]
        unprefer_other_mkls = ("!" + mkl for mkl in all_mkls if mkl != preferred_mkl)
        return (prefer_ps,) + tuple(unprefer_other_mkls) + (preferred_mkl,)
    else:
        return ()


79
80
81
82
def valid_cuda(cuda, compiler):
    """
    The CUDA distribution only works with certain gcc versions,
    this little function takes care that only supported combinations
Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
83
84
85
86
    are allowed.

    Take care to keep this in sync with the file 'macros.obs_cluster' of
    the package software:dist / mpcdf_cluster_macros
87
    """
Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
88
    if cuda == "cuda_8_0":
89
        return compiler == "gcc_5"
Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
90
    if cuda == "cuda_9_1":
91
        return compiler == "gcc_6_3_0"
Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
92
    if cuda == "cuda_9_2":
Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
93
        return compiler == "gcc_6"
Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
94
95
    if cuda == "cuda_10_0":
        return compiler == "gcc_6"
Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
96
97
    if cuda == "cuda_10_1":
        return compiler == "gcc_8"
98
99
100
    return False


Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
101
102
103
def project_meta(api_url, project):
    return ElementTree.fromstringlist(osc.core.show_project_meta(api_url, project))

104

105
106
107
108
109
110
111
112
113
def package_meta(api_url, project, package):
    return ElementTree.fromstringlist(osc.core.show_package_meta(api_url, project, package))


def maintainers(api_url, project, package):
    root = package_meta(api_url, project, package)
    return {e.get("userid") for e in root.findall("./person[@role='maintainer']")}


114
115
116
117
class UnsetAttributeException(Exception):
    pass


118
119
120
121
122
123
124
125
def chunked(l, chunksize):
    n = len(l)
    i = 0
    while i * chunksize < n:
        yield l[i * chunksize: (i + 1) * chunksize]
        i += 1


Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
126
def get_attribute(api_url, project, package, attribute, with_project=False):
127
128
    attribute_meta = osc.core.show_attribute_meta(api_url, project, package, None, attribute, False, with_project)
    if attribute_meta is None:
Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
129
        raise osc.oscerr.APIError("Cannot fetch value for attribute '{0}' from {1}".format(attribute, (project, package)))
130
131
132
133

    root = ElementTree.fromstringlist(attribute_meta)
    attribute = root.find("./attribute")
    if attribute is not None:
Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
134
        return root
135
    else:
136
        raise UnsetAttributeException("Attribute not set")
137

Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
138

Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
139
140
141
142
143
144
def get_attribute_values(api_url, project, package, attribute, with_project=False):
    root = get_attribute(api_url, project, package, attribute, with_project)
    attribute = root.find("./attribute")
    return list(value.text for value in attribute.findall("./value"))


145
146
147
148
149
def get_attribute_value(api_url, project, package, attribute, with_project=False):
    value, = get_attribute_values(api_url, project, package, attribute, with_project=False)
    return value


150
151
152
153
154
155
def set_attribute(api_url, project, package, attribute):
    path = ["source", project]
    if package:
        path.append(package)
    path.append("_attribute")

Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
156
    attr_meta = ElementTree.tostring(attribute, encoding=osc.core.ET_ENCODING)
Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
157

158
    url = osc.core.makeurl(api_url, path)
Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
159
    resp = ElementTree.fromstringlist(osc.core.streamfile(url, osc.core.http_POST, data=attr_meta))
Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
160
    if resp.find("./summary").text != "Ok":
Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
161
        raise osc.oscerr.APIError("Could not store attribute")
Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
162

163

164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
def set_attribute_values(api_url, project, package, attribute, values):
    root = ElementTree.Element("attributes")
    attr = ElementTree.SubElement(root, "attribute")

    namespace, name = attribute.split(":")
    attr.set("namespace", namespace)
    attr.set("name", name)

    for value in values:
        v = ElementTree.SubElement(attr, "value")
        v.text = value

    set_attribute(api_url, project, package, root)


def has_attribute(api_url, project, package, attribute):
    path = ["source", project]
    if package:
        path.append(package)
    path.append("_attribute")
    path.append(attribute)

    url = osc.core.makeurl(api_url, path)
    resp = ElementTree.fromstringlist(osc.core.streamfile(url, osc.core.http_GET))

    namespace, name = attribute.split(":")
190

191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
    for a in resp.findall("./attribute"):
        if a.get("namespace") == namespace and a.get("name") == name:
            return True
    return False


def remove_attribute(api_url, project, package, attribute_name):
    path = ["source"] + [project]
    if package:
        path.append(package)
    path.append("_attribute")
    path.append(attribute_name)

    url = osc.core.makeurl(api_url, path)
    resp = ElementTree.fromstringlist(osc.core.streamfile(url, osc.core.http_DELETE))
    if resp.find("./summary").text != "Ok":
        raise osc.oscerr.APIError("Could not remove attribute")


210
211
212
213
214
215
216
217
218
219
220
221
222
def get_microarchitecture(project):
    if project == "software":
        # Stupid special case
        microarch = "sandybridge"
    else:
        microarch = project.split(":")[2]

    if microarch in known_microarchs:
        return microarch
    else:
        raise Exception("Unknown micro-architecture '{0}'".format(microarch))


223
def mpcdf_enable_repositories(api_url, project, package, verbose=False, filter_repos=None):
224
    from itertools import product
Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
225
    import sys
226

227
228
229
    if filter_repos is None:
        filter_repos = ()

230
231
232
    pkg_meta = osc.core.show_package_meta(api_url, project, package)
    root = ElementTree.fromstringlist(pkg_meta)

233
    build = root.find("./build")
Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
234
235
    if build is None:
        build = ElementTree.SubElement(root, "build")
236
237
238
239
240

    pkg_meta = ElementTree.tostring(root, encoding=osc.core.ET_ENCODING)

    for enable in build.findall("./enable"):
        build.remove(enable)
241

Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
242
    try:
243
244
245
246
247
        enable_repos = get_attribute_values(api_url, project, package, "MPCDF:enable_repositories")
    except Exception:
        if verbose:
            print("Warning: Could not get attribute MPCDF:enable_repositories for package {0}, skipping".format(package))
        return False
Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
248

Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
249
    def try_get_attribute(package, attribute, with_project=False):
250
251
252
        try:
            return get_attribute_values(api_url, project, package, "MPCDF:" + attribute, with_project=with_project)
        except UnsetAttributeException:
253
254
255
            print("ERROR: Attribute MPCDF:" + attribute + " is not set for "
                  + ("package '{0}'".format(package) if package else "project '{0}'".format(project))
                  + ", aborting here", file=sys.stderr)
256
            raise SystemExit(1)
Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
257

258
259
260
    compilers = try_get_attribute(package, "compiler_modules", with_project=True)
    mpis = try_get_attribute(package, "mpi_modules", with_project=True)
    cudas = try_get_attribute(package, "cuda_modules", with_project=True)
261
262
263
264
265
266
    pgis = try_get_attribute(package, "pgi_modules", with_project=True)

    all_compilers = try_get_attribute(None, "compiler_modules")
    all_mpis = try_get_attribute(None, "mpi_modules")
    all_cudas = try_get_attribute(None, "cuda_modules")
    all_pgis = try_get_attribute(None, "pgi_modules")
267
268
269
270

    default_compilers = try_get_attribute(None, "default_compiler")
    default_mpis = try_get_attribute(None, "default_mpi")
    default_cudas = try_get_attribute(None, "default_cuda")
271

272
    def enable(name):
273
274
        if any(filtered_repo in name for filtered_repo in filter_repos):
            return
275
276
277
278
279
280
281
        node = ElementTree.Element("enable")
        node.set("repository", name)
        node.tail = "\n    "
        build.insert(0, node)
        if verbose:
            print("Enabling", name)

282
    def actual_compilers():
283
        for compiler in (c for c in compilers if c in all_compilers + ["default_compiler", "intel", "gcc"]):
284
            if compiler == "intel":
Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
285
                for intel_compiler in [cc for cc in all_compilers if cc.startswith("intel")]:
286
287
                    yield intel_compiler
            elif compiler == "gcc":
Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
288
                for gcc_compiler in [cc for cc in all_compilers if cc.startswith("gcc")]:
289
290
291
292
293
294
295
296
                    yield gcc_compiler
            elif compiler == "default_compiler":
                for default_compiler in default_compilers:
                    yield default_compiler
            else:
                yield compiler

    def actual_mpis():
297
        for mpi in (m for m in mpis if m in all_mpis + ["default_mpi", "impi"]):
298
            if mpi == "impi":
Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
299
                for impi in [mpi for mpi in mpis if mpi.startswith("impi")]:
300
301
302
303
304
305
306
307
                    yield impi
            elif mpi == "default_mpi":
                for default_mpi in default_mpis:
                    yield default_mpi
            else:
                yield mpi

    def actual_cudas():
308
        for cuda in (c for c in cudas if c in all_cudas + ["default_cuda"]):
309
310
311
312
313
314
            if cuda == "default_cuda":
                for default_cuda in default_cudas:
                    yield default_cuda
            else:
                yield cuda

315
316
317
318
    def actual_pgis():
        for pgi in (p for p in pgis if p in all_pgis):
            yield pgi

319
320
321
322
323
324
    for flag in enable_repos:

        if flag == "system":
            enable("System")

        if flag == "compilers":
325
            for compiler in actual_compilers():
Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
326
                enable(compiler)
327
328

        if flag == "mpi":
329
330
331
            for mpi, compiler in product(actual_mpis(), actual_compilers()):
                if valid_mpi(compiler, mpi):
                    enable(mpi + "_" + compiler)
332
333

        if flag == "cuda":
334
335
336
            for cuda, compiler in product(actual_cudas(), all_compilers):
                if valid_cuda(cuda, compiler):
                    enable(cuda + "_" + compiler)
337
338

        if flag == "cuda_mpi":
339
340
            for cuda, mpi, compiler in product(actual_cudas(), actual_mpis(), all_compilers):
                if valid_cuda(cuda, compiler) and valid_mpi(compiler, mpi):
341
                    enable(cuda + "_" + mpi + "_" + compiler)
342

343
344
345
346
347
348
        if flag == "pgi":
            for pgi in actual_pgis():
                enable(pgi)

        if flag == "pgi_mpi":
            for mpi, pgi in product(actual_mpis(), actual_pgis()):
349
350
                if valid_pgi_mpi(pgi, mpi):
                    enable(mpi + "_" + pgi)
351

Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
352
353
    if len(build.getchildren()) > 0:
        build.getchildren()[-1].tail = "\n  "
354

355
356
357
    new_pkg_meta = ElementTree.tostring(root, encoding=osc.core.ET_ENCODING)
    if pkg_meta != new_pkg_meta:
        osc.core.edit_meta("pkg", (project, package), data=new_pkg_meta)
358
359

    return True
Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
360
361


362
363
def mpcdf_setup_repositories(api_url, project, microarchitecture=None, distribution=None, parent=None,
                             packages=None, dry_run=False, filter_repos=None, only_project=False, remove_old=False):
364
365
    import threading

Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
366
    if parent:
367
368
369
        for attribute in config_attributes + default_attributes:
            print("Copying attribute '{0}' from parent project".format(attribute))
            set_attribute(api_url, project, None, get_attribute(api_url, parent, None, attribute))
Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
370

371
372
373
374
375
376
    compilers = get_attribute_values(api_url, project, None, "MPCDF:compiler_modules")
    mpis = get_attribute_values(api_url, project, None, "MPCDF:mpi_modules")
    cudas = get_attribute_values(api_url, project, None, "MPCDF:cuda_modules")
    pgis = get_attribute_values(api_url, project, None, "MPCDF:pgi_modules")
    default_python2 = get_attribute_value(api_url, project, None, "MPCDF:default_python2")
    default_python3 = get_attribute_value(api_url, project, None, "MPCDF:default_python3")
Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
377

378
379
    if distribution is None:
        # Get existing value from project meta
380
381
382
383
384
385
        dist_repo = project_meta(api_url, project if parent is None else parent).find(
            "./repository[@name='System']/path[@project='distributions']")
        if dist_repo is not None:
            distribution = dist_repo.get("repository")
        else:
            raise osc.oscerr.WrongArgs("ERROR: Specify distribution or parent project")
386
387
        print("Using '{0}' as base distribution".format(distribution))

Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
388
389
390
    distributions = project_meta(api_url, "distributions")
    dist_repo = distributions.find('./repository[@name="{0}"]'.format(distribution))
    if dist_repo is None:
Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
391
        raise osc.oscerr.WrongArgs("No repository '{0}' is defined in project 'distributions' on the server".format(distribution))
Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
392
393
    architectures = list(arch.text for arch in dist_repo.findall("./arch"))

394
395
396
    if microarchitecture is None:
        microarchitecture = get_microarchitecture(project)

Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
397
398
    root = project_meta(api_url, project)

Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
399
    prjconf = list(map(decode_it, osc.core.show_project_conf(api_url, project)))
Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
400
401
402
403
404
405
406
407
408
409
410
    start_marker = "# Autogenerated by osc mpcdf_setup_repos, do not edit till end of section\n"
    end_marker = "# End of autogenerated section\n"

    try:
        start = prjconf.index(start_marker)
        end = prjconf.index(end_marker)
    except ValueError:
        start = None
        end = len(prjconf)

    prjconf_head = "".join(prjconf[:start])
Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
411
    prjconf_tail = "".join(prjconf[end + 1:])
Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
412
413
    prjconf = [start_marker]

414
415
416
417
418
419
420
421
422
    prjconf.append("Constraint: hostlabel {0}".format(microarchitecture))
    prjconf.append(
        """
%if %_repository != System
Macros:
%microarchitecture {0}
:Macros
%endif""".format(microarchitecture))
    prjconf.append("")
423
424
425
426
    prjconf.append("Prefer: mpcdf_python2_" + default_python2)
    prjconf.append("Prefer: mpcdf_python3_" + default_python3)
    prjconf.append("")

Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
427
    # Remove existing repositories
428
429
430
    if remove_old:
        for oldrepo in root.findall("./repository"):
            root.remove(oldrepo)
Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
431
432

    def repo(name, *dependencies, **kwargs):
433
        is_compiler = kwargs.pop("compiler", False)
Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
434
435
        is_mpi = kwargs.pop("mpi", False)
        is_cuda = kwargs.pop("cuda", False)
436
437
        is_cuda_mpi = kwargs.pop("cuda_mpi", False)

438
439
        cuda_repo = kwargs.pop("cuda_repo", "")

440
441
442
443
        have_compiler = is_compiler or is_mpi or is_cuda or is_cuda_mpi
        have_mpi = is_mpi or is_cuda_mpi
        have_cuda = is_cuda or is_cuda_mpi

444
445
446
447
448
449
        existing_repo = root.find("./repository[@name='{0}']".format(name))
        if existing_repo is not None:
            root.remove(existing_repo)
        else:
            print("New repository", name)

Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
450
451
        r = ElementTree.SubElement(root, "repository")
        r.set("name", name)
Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
452
453
        r.text = "\n    "

Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
454
455
456
457
        def path(project, repo):
            p = ElementTree.SubElement(r, "path")
            p.set("project", project)
            p.set("repository", repo)
Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
458
            p.tail = "\n    "
Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
459

460
461
        if parent:
            path(parent, name)
462
463
        for dep_project, dep_repo in dependencies:
            path(dep_project, dep_repo)
Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
464
465
466
467
468
469
470
471
472

        for arch in architectures:
            a = ElementTree.SubElement(r, "arch")
            a.text = arch
            a.tail = "\n    "
        a.tail = "\n  "
        r.tail = "\n  "

        prjconf.append("%if %_repository == {0}".format(name))
473
474
        for prefer in prefers(name):
            prjconf.append("Prefer: " + prefer)
Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
475
        prjconf.append("Macros:")
476
477

        prjconf.append("%is_compiler_repository {0}".format(1 if is_compiler else 0))
Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
478
479
        prjconf.append("%is_mpi_repository {0}".format(1 if is_mpi else 0))
        prjconf.append("%is_cuda_repository {0}".format(1 if is_cuda else 0))
480
481
482
483
484
485
        prjconf.append("%is_cuda_mpi_repository {0}".format(1 if is_cuda_mpi else 0))

        prjconf.append("%have_mpcdf_compiler {0}".format(1 if have_compiler else 0))
        prjconf.append("%have_mpcdf_mpi {0}".format(1 if have_mpi else 0))
        prjconf.append("%have_mpcdf_cuda {0}".format(1 if have_cuda else 0))

486
487
488
        if is_cuda:
            prjconf.append("%cuda_repository {0}".format(cuda_repo))

489
490
491
        for macro, value in kwargs.items():
            prjconf.append("%{0} {1}".format(macro, value))

Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
492
493
494
495
        prjconf.append(":Macros")
        prjconf.append("%endif")
        prjconf.append("")

Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
496
497
498
499
    if parent:
        repo("System")
    else:
        repo("System", ("distributions", distribution))
Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
500

501
    for compiler in compilers + pgis:
502
503
        repo(compiler, (project, "System"), compiler=True,
             compiler_repository=compiler)
Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
504
505

        for mpi in filter(partial(valid_mpi, compiler), mpis):
506
507
            repo(mpi + "_" + compiler, (project, compiler), mpi=True,
                 mpi_repository=mpi, compiler_repository=compiler)
Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
508
509
510

    for cuda in cudas:
        for compiler in filter(partial(valid_cuda, cuda), compilers):
511
            repo(cuda + "_" + compiler, (project, compiler), cuda=True, cuda_repo=cuda)
Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
512
513
514
515
            for mpi in filter(partial(valid_mpi, compiler), mpis):
                repo(cuda + "_" + mpi + "_" + compiler,
                     (project, cuda + "_" + compiler),
                     (project, mpi + "_" + compiler),
516
                     cuda_mpi=True)
Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
517
518
519
520
521
522
523
524
525
526
527
528

    # Remove build configuration
    build = root.find("./build")
    if build is None:
        build = ElementTree.Element("build")
        build.text = "\n    "
        disable = ElementTree.SubElement(build, "disable")
        disable.tail = "\n  "
        build.tail = "\n  "
        root.insert(list(root).index(root.find("./repository")), build)

    root.getchildren()[-1].tail = "\n"
Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
529
    prj = ElementTree.tostring(root, encoding=osc.core.ET_ENCODING)
Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
530
531
532
533
534
535
536
537

    prjconf.append(end_marker)
    prjconf = prjconf_head + "\n".join(prjconf) + prjconf_tail

    if dry_run:
        print("osc meta prj {0} -F - <<EOF\n{1}\nEOF\n".format(project, prj))
        print("osc meta prjconf {0} -F - <<EOF\n{1}\nEOF\n".format(project, prjconf))
    else:
538
539
540
541
542
        if not only_project:
            # First set-up the <enable/> flags, that way no
            # spurious builds are launched
            if packages is None:
                packages = osc.core.meta_get_packagelist(api_url, project)
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562

            if len(packages) > 40:
                chunksize = len(packages) // 20
            else:
                chunksize = len(packages)

            def work(packagelist):
                for package in packagelist:
                    print("Updating repositories for", package)
                    if not mpcdf_enable_repositories(api_url, project, package, filter_repos=filter_repos):
                        print("ATTENTION: Not changing unmanaged package {0}".format(package))

            threads = []
            for packagelist in chunked(packages, chunksize):
                t = threading.Thread(target=work, args=(packagelist,))
                threads.append(t)
                t.start()

            for t in threads:
                t.join()
Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
563
564
565

        # Update repositories
        print("Updating prj meta")
566
        osc.core.edit_meta("prj", project, data=prj, force=True)
Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
567
568
        print("Updating prjconf meta")
        osc.core.edit_meta("prjconf", project, data=prjconf)
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608


def set_as_branch(api_url, my_project, my_package, main_project, main_package):
    import os

    def dirmeta(project, package):
        url = osc.core.makeurl(api_url, ["source", project, package])
        return ElementTree.fromstringlist(osc.core.streamfile(url, osc.core.http_GET))

    srcmeta = dirmeta(my_project, my_package)
    dstmeta = dirmeta(main_project, main_package)

    linkinfo = srcmeta.find('./linkinfo')
    if linkinfo is not None:
        print("ERROR: package {0} is already linked to package {1} in project {2}".format(
              my_package, linkinfo.get("package"), linkinfo.get("project")))
        return False

    linkfile = ElementTree.Element("link")
    linkfile.set("project", main_project)
    linkfile.set("package", main_package)
    linkfile.set("baserev", dstmeta.get("srcmd5"))
    patches = ElementTree.SubElement(linkfile, "patches")
    ElementTree.SubElement(patches, "branch")

    url = osc.core.makeurl(api_url, ["source", my_project, my_package, "_link"])
    resp = ElementTree.fromstringlist(osc.core.streamfile(url, osc.core.http_PUT, data=ElementTree.tostring(linkfile)))
    rev = resp.get("rev")
    if rev is None:
        print("ERROR: Could not commit _link file")
        return False
    else:
        print("Commited as revision", rev)
        this_package = osc.core.store_read_package(os.curdir)
        this_project = osc.core.store_read_project(os.curdir)
        if this_package == my_package and this_project == my_project:
            pac = osc.core.filedir_to_pac(os.curdir)
            rev = pac.latest_rev(expand=True)
            pac.update(rev)
    return True
609
610


611
def sync_projects(api_url, package=None, from_project="software", to_projects=None, redo_all=False, add_to_maintainers=True):
612
613
614
615
    if package is not None and redo_all:
        raise osc.oscerr.WrongArgs('Cannot specify `redo_all` and package')

    if to_projects is None:
Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
616
617
        to_projects = [p for p in osc.core.meta_get_project_list(api_url)
                       if p.startswith("software:") and not (p == "software:dist" or p == "software:images")]
618
619
620
621
622
623
624
625
626
627
628
629
630

    for to_project in to_projects:
        print("Syncing {0} with {1}".format(to_project, from_project))

        to_packages = osc.core.meta_get_packagelist(api_url, to_project)

        if package is None:
            from_packages = osc.core.meta_get_packagelist(api_url, from_project)
            if not redo_all:
                from_packages = list(sorted(set(from_packages) - set(to_packages)))
        else:
            from_packages = [package]

631
        for attribute in config_attributes:
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
            try:
                get_attribute(api_url, to_project, None, attribute)
            except Exception:
                attr = get_attribute(api_url, from_project, None, attribute)
                print("Setting attribute", attribute)
                set_attribute(api_url, to_project, None, attr)

        for orig_package in from_packages:
            if orig_package not in to_packages:
                filelist = osc.core.meta_get_filelist(api_url, from_project, orig_package)
                if "_link" in filelist:
                    print("Not branching package {0}, is a link".format(orig_package))
                else:
                    print("Branching package {0}".format(orig_package))
                    exists, targetprj, targetpkg, srcprj, srcpkg = \
                        osc.core.branch_pkg(api_url, from_project, orig_package, target_project=to_project, nodevelproject=True)
            else:
                print("Not branching package {0}, already present in target".format(orig_package))

651
            for attribute in package_attributes + config_attributes:
652
653
654
655
656
657
658
659
                try:
                    attr = get_attribute(api_url, from_project, orig_package, attribute)
                except UnsetAttributeException:
                    if has_attribute(api_url, to_project, orig_package, attribute):
                        remove_attribute(api_url, to_project, orig_package, attribute)
                    continue
                set_attribute(api_url, to_project, orig_package, attr)

660
661
662
663
664
665
666
667
668
669
670
671
672
            if add_to_maintainers:
                from_maintainers = maintainers(api_url, from_project, orig_package)
                if from_maintainers:
                    to_maintainers = maintainers(api_url, to_project, orig_package)
                    if from_maintainers - to_maintainers:
                        new_maintainers = from_maintainers - to_maintainers
                        to_meta = package_meta(api_url, to_project, orig_package)
                        for userid in sorted(new_maintainers):
                            person = ElementTree.Element("person")
                            person.set("userid", userid)
                            person.set("role", "maintainer")
                            to_meta.insert(2, person)
                        osc.core.edit_meta("pkg", (to_project, orig_package), data=ElementTree.tostring(to_meta))
673

674
675
676
677
678
679
680
681
682
683
        if package is None and redo_all:
            # Check if distribution is already set in to_project
            to_prj_meta = project_meta(api_url, to_project)
            sys_repo = to_prj_meta.find('./repository[@name="System"]')
            if sys_repo is None:
                distribution = project_meta(api_url, from_project).find('./repository[@name="System"]/path[@project="distributions"]').get("repository")
            else:
                distribution = sys_repo.find('./path[@project="distributions"]').get("repository")

            print("Creating repository configuration")
684
            mpcdf_setup_repositories(api_url, to_project, distribution=distribution)
685
686
687
688
        else:
            for orig_package in from_packages:
                if not mpcdf_enable_repositories(api_url, to_project, orig_package):
                    print("ATTENTION: Not changing unmanaged package {0}".format(orig_package))