mpcdf_common.py 27.7 KB
Newer Older
1
from __future__ import print_function
Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
2
from __future__ import division
3

4
import sys
5
6
7
import osc
import osc.conf
import osc.core
Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
8
import osc.oscerr
Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
9
from functools import partial
10
11
from xml.etree import ElementTree

12
13
14
15
16
17
18
if sys.version_info[0] < 3:
    def decode_it(arg):
        return arg
else:
    from osc.util.helper import decode_it


19
20
known_microarchs = {"sandybridge", "haswell", "skylake"}

21
22
package_attributes = ["MPCDF:enable_repositories"]
config_attributes = ["MPCDF:compiler_modules", "MPCDF:cuda_modules", "MPCDF:mpi_modules", "MPCDF:pgi_modules"]
23
default_attributes = ["MPCDF:default_compiler", "MPCDF:default_cuda", "MPCDF:default_mpi", "MPCDF:default_python2", "MPCDF:default_python3"]
24

25
26
27
28
29
30
31
32
intel_parallel_studio = {
    "mpcdf_intel_parallel_studio_2017_7": {"compiler": "intel_17_0_7", "impi": "impi_2017_4", "mkl": "mkl_2017_4-module", },
    "mpcdf_intel_parallel_studio_2018_1": {"compiler": "intel_18_0_1", "impi": "impi_2018_1", "mkl": "mkl_2018_1-module", },
    "mpcdf_intel_parallel_studio_2018_2": {"compiler": "intel_18_0_2", "impi": "impi_2018_2", "mkl": "mkl_2018_2-module", },
    "mpcdf_intel_parallel_studio_2018_3": {"compiler": "intel_18_0_3", "impi": "impi_2018_3", "mkl": "mkl_2018_3-module", },
    "mpcdf_intel_parallel_studio_2018_4": {"compiler": "intel_18_0_5", "impi": "impi_2018_4", "mkl": "mkl_2018_4-module", },
    "mpcdf_intel_parallel_studio_2019_0": {"compiler": "intel_19_0_0", "impi": "impi_2019_0", "mkl": "mkl_2019_0-module", },
    "mpcdf_intel_parallel_studio_2019_1": {"compiler": "intel_19_0_1", "impi": "impi_2019_1", "mkl": "mkl_2019_1-module", },
Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
33
    "mpcdf_intel_parallel_studio_2019_3": {"compiler": "intel_19_0_3", "impi": "impi_2019_3", "mkl": "mkl_2019_3-module", },
34
    "mpcdf_intel_parallel_studio_2019_4": {"compiler": "intel_19_0_4", "impi": "impi_2019_4", "mkl": "mkl_2019_4-module", },
35
36
37
38
39
40
41
42
}

all_mkls = {ic["mkl"] for ic in intel_parallel_studio.values()}

# Some rotations
mpi_parallel_studio = {value["impi"]: dict({"ps": key}, **value) for key, value in intel_parallel_studio.items()}
compiler_parallel_studio = {value["compiler"]: dict({"ps": key}, **value) for key, value in intel_parallel_studio.items()}

43

44
45
46
47
48
# For the autogenerated software/software:*:* project 'prjconf' sections
prjconf_start_marker = "# Autogenerated by osc mpcdf_setup_repos, do not edit till end of section\n"
prjconf_end_marker = "# End of autogenerated section\n"


49
50
51
52
53
54
55
56
def valid_pgi_mpi(pgi, mpi):
    if "impi" not in mpi:
        return False
    if "2017" in mpi:
        return False
    return True


57
def valid_mpi(compiler, mpi):
58
59
60
61
62
63
64
65
66
    """
    It might be possible to use Intel MPI libararies and compilers from
    different Parallel Studio packages, but I currently do not want to support
    it.

    Take care to keep this in sync with the file 'macros.obs_cluster' of
    the package software:dist / mpcdf_cluster_macros
    """
    if compiler.startswith("intel") and mpi.startswith("impi"):
67
        return mpi == compiler_parallel_studio[compiler]["impi"]
68
69
    if compiler.startswith("pgi"):
        return valid_pgi_mpi(compiler, mpi)
70
71
    else:
        return True
72
73


74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
def prefers(reponame):
    prefer_ps = None
    if reponame in compiler_parallel_studio:
        prefer_ps = compiler_parallel_studio[reponame]["ps"]
    else:
        for mpi in mpi_parallel_studio:
            if reponame.startswith(mpi):
                prefer_ps = mpi_parallel_studio[mpi]["ps"]

    if prefer_ps:
        preferred_mkl = intel_parallel_studio[prefer_ps]["mkl"]
        unprefer_other_mkls = ("!" + mkl for mkl in all_mkls if mkl != preferred_mkl)
        return (prefer_ps,) + tuple(unprefer_other_mkls) + (preferred_mkl,)
    else:
        return ()


91
92
93
94
def valid_cuda(cuda, compiler):
    """
    The CUDA distribution only works with certain gcc versions,
    this little function takes care that only supported combinations
Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
95
96
97
98
    are allowed.

    Take care to keep this in sync with the file 'macros.obs_cluster' of
    the package software:dist / mpcdf_cluster_macros
99
    """
Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
100
    if cuda == "cuda_8_0":
101
        return compiler == "gcc_5"
Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
102
    if cuda == "cuda_9_1":
103
        return compiler == "gcc_6_3_0"
Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
104
    if cuda == "cuda_9_2":
Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
105
        return compiler == "gcc_6"
Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
106
107
    if cuda == "cuda_10_0":
        return compiler == "gcc_6"
Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
108
109
    if cuda == "cuda_10_1":
        return compiler == "gcc_8"
110
111
112
    return False


Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
113
114
115
def project_meta(api_url, project):
    return ElementTree.fromstringlist(osc.core.show_project_meta(api_url, project))

116

117
118
119
120
121
122
123
124
125
def package_meta(api_url, project, package):
    return ElementTree.fromstringlist(osc.core.show_package_meta(api_url, project, package))


def maintainers(api_url, project, package):
    root = package_meta(api_url, project, package)
    return {e.get("userid") for e in root.findall("./person[@role='maintainer']")}


126
127
128
129
class UnsetAttributeException(Exception):
    pass


130
131
132
133
134
135
136
137
def chunked(l, chunksize):
    n = len(l)
    i = 0
    while i * chunksize < n:
        yield l[i * chunksize: (i + 1) * chunksize]
        i += 1


Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
138
def get_attribute(api_url, project, package, attribute, with_project=False):
139
140
    attribute_meta = osc.core.show_attribute_meta(api_url, project, package, None, attribute, False, with_project)
    if attribute_meta is None:
Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
141
        raise osc.oscerr.APIError("Cannot fetch value for attribute '{0}' from {1}".format(attribute, (project, package)))
142
143
144
145

    root = ElementTree.fromstringlist(attribute_meta)
    attribute = root.find("./attribute")
    if attribute is not None:
Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
146
        return root
147
    else:
148
        raise UnsetAttributeException("Attribute not set")
149

Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
150

Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
151
152
153
154
155
156
def get_attribute_values(api_url, project, package, attribute, with_project=False):
    root = get_attribute(api_url, project, package, attribute, with_project)
    attribute = root.find("./attribute")
    return list(value.text for value in attribute.findall("./value"))


157
158
159
160
161
def get_attribute_value(api_url, project, package, attribute, with_project=False):
    value, = get_attribute_values(api_url, project, package, attribute, with_project=False)
    return value


162
163
164
165
166
167
def set_attribute(api_url, project, package, attribute):
    path = ["source", project]
    if package:
        path.append(package)
    path.append("_attribute")

Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
168
    attr_meta = ElementTree.tostring(attribute, encoding=osc.core.ET_ENCODING)
Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
169

170
    url = osc.core.makeurl(api_url, path)
Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
171
    resp = ElementTree.fromstringlist(osc.core.streamfile(url, osc.core.http_POST, data=attr_meta))
Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
172
    if resp.find("./summary").text != "Ok":
Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
173
        raise osc.oscerr.APIError("Could not store attribute")
Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
174

175

176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
def set_attribute_values(api_url, project, package, attribute, values):
    root = ElementTree.Element("attributes")
    attr = ElementTree.SubElement(root, "attribute")

    namespace, name = attribute.split(":")
    attr.set("namespace", namespace)
    attr.set("name", name)

    for value in values:
        v = ElementTree.SubElement(attr, "value")
        v.text = value

    set_attribute(api_url, project, package, root)


def has_attribute(api_url, project, package, attribute):
    path = ["source", project]
    if package:
        path.append(package)
    path.append("_attribute")
    path.append(attribute)

    url = osc.core.makeurl(api_url, path)
    resp = ElementTree.fromstringlist(osc.core.streamfile(url, osc.core.http_GET))

    namespace, name = attribute.split(":")
202

203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
    for a in resp.findall("./attribute"):
        if a.get("namespace") == namespace and a.get("name") == name:
            return True
    return False


def remove_attribute(api_url, project, package, attribute_name):
    path = ["source"] + [project]
    if package:
        path.append(package)
    path.append("_attribute")
    path.append(attribute_name)

    url = osc.core.makeurl(api_url, path)
    resp = ElementTree.fromstringlist(osc.core.streamfile(url, osc.core.http_DELETE))
    if resp.find("./summary").text != "Ok":
        raise osc.oscerr.APIError("Could not remove attribute")


222
223
224
225
def get_microarchitecture(project):
    if project == "software":
        # Stupid special case
        microarch = "sandybridge"
226
    elif project.startswith("software:"):
227
        microarch = project.split(":")[2]
228
229
230
231
    elif project.startswith("home:"):
        microarch = "sandybridge"
    else:
        raise Exception("Cannot determine micro-architecture for project '{0}'".format(project))
232
233
234
235
236
237
238

    if microarch in known_microarchs:
        return microarch
    else:
        raise Exception("Unknown micro-architecture '{0}'".format(microarch))


239
def mpcdf_enable_repositories(api_url, project, package, verbose=False, filter_repos=None):
240
    from itertools import product
Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
241
    import sys
242

243
244
245
    if filter_repos is None:
        filter_repos = ()

246
247
248
    pkg_meta = osc.core.show_package_meta(api_url, project, package)
    root = ElementTree.fromstringlist(pkg_meta)

249
    build = root.find("./build")
Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
250
251
    if build is None:
        build = ElementTree.SubElement(root, "build")
252
253
254
255
256

    pkg_meta = ElementTree.tostring(root, encoding=osc.core.ET_ENCODING)

    for enable in build.findall("./enable"):
        build.remove(enable)
257

Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
258
    try:
259
260
261
262
263
        enable_repos = get_attribute_values(api_url, project, package, "MPCDF:enable_repositories")
    except Exception:
        if verbose:
            print("Warning: Could not get attribute MPCDF:enable_repositories for package {0}, skipping".format(package))
        return False
Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
264

Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
265
    def try_get_attribute(package, attribute, with_project=False):
266
267
268
        try:
            return get_attribute_values(api_url, project, package, "MPCDF:" + attribute, with_project=with_project)
        except UnsetAttributeException:
269
270
271
            print("ERROR: Attribute MPCDF:" + attribute + " is not set for "
                  + ("package '{0}'".format(package) if package else "project '{0}'".format(project))
                  + ", aborting here", file=sys.stderr)
272
            raise SystemExit(1)
Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
273

274
275
276
    compilers = try_get_attribute(package, "compiler_modules", with_project=True)
    mpis = try_get_attribute(package, "mpi_modules", with_project=True)
    cudas = try_get_attribute(package, "cuda_modules", with_project=True)
277
278
279
280
281
282
    pgis = try_get_attribute(package, "pgi_modules", with_project=True)

    all_compilers = try_get_attribute(None, "compiler_modules")
    all_mpis = try_get_attribute(None, "mpi_modules")
    all_cudas = try_get_attribute(None, "cuda_modules")
    all_pgis = try_get_attribute(None, "pgi_modules")
283
284
285
286

    default_compilers = try_get_attribute(None, "default_compiler")
    default_mpis = try_get_attribute(None, "default_mpi")
    default_cudas = try_get_attribute(None, "default_cuda")
287

288
    def enable(name):
289
290
        if any(filtered_repo in name for filtered_repo in filter_repos):
            return
291
292
293
294
295
296
297
        node = ElementTree.Element("enable")
        node.set("repository", name)
        node.tail = "\n    "
        build.insert(0, node)
        if verbose:
            print("Enabling", name)

298
    def actual_compilers():
299
        for compiler in (c for c in compilers if c in all_compilers + ["default_compiler", "intel", "gcc"]):
300
            if compiler == "intel":
Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
301
                for intel_compiler in [cc for cc in all_compilers if cc.startswith("intel")]:
302
303
                    yield intel_compiler
            elif compiler == "gcc":
Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
304
                for gcc_compiler in [cc for cc in all_compilers if cc.startswith("gcc")]:
305
306
307
308
309
310
311
312
                    yield gcc_compiler
            elif compiler == "default_compiler":
                for default_compiler in default_compilers:
                    yield default_compiler
            else:
                yield compiler

    def actual_mpis():
313
        for mpi in (m for m in mpis if m in all_mpis + ["default_mpi", "impi"]):
314
            if mpi == "impi":
Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
315
                for impi in [mpi for mpi in mpis if mpi.startswith("impi")]:
316
317
318
319
320
321
322
323
                    yield impi
            elif mpi == "default_mpi":
                for default_mpi in default_mpis:
                    yield default_mpi
            else:
                yield mpi

    def actual_cudas():
324
        for cuda in (c for c in cudas if c in all_cudas + ["default_cuda"]):
325
326
327
328
329
330
            if cuda == "default_cuda":
                for default_cuda in default_cudas:
                    yield default_cuda
            else:
                yield cuda

331
332
333
334
    def actual_pgis():
        for pgi in (p for p in pgis if p in all_pgis):
            yield pgi

335
336
337
338
339
340
    for flag in enable_repos:

        if flag == "system":
            enable("System")

        if flag == "compilers":
341
            for compiler in actual_compilers():
Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
342
                enable(compiler)
343
344

        if flag == "mpi":
345
346
347
            for mpi, compiler in product(actual_mpis(), actual_compilers()):
                if valid_mpi(compiler, mpi):
                    enable(mpi + "_" + compiler)
348
349

        if flag == "cuda":
350
351
352
            for cuda, compiler in product(actual_cudas(), all_compilers):
                if valid_cuda(cuda, compiler):
                    enable(cuda + "_" + compiler)
353
354

        if flag == "cuda_mpi":
355
356
            for cuda, mpi, compiler in product(actual_cudas(), actual_mpis(), all_compilers):
                if valid_cuda(cuda, compiler) and valid_mpi(compiler, mpi):
357
                    enable(cuda + "_" + mpi + "_" + compiler)
358

359
360
361
362
363
364
        if flag == "pgi":
            for pgi in actual_pgis():
                enable(pgi)

        if flag == "pgi_mpi":
            for mpi, pgi in product(actual_mpis(), actual_pgis()):
365
366
                if valid_pgi_mpi(pgi, mpi):
                    enable(mpi + "_" + pgi)
367

Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
368
369
    if len(build.getchildren()) > 0:
        build.getchildren()[-1].tail = "\n  "
370

371
372
    new_pkg_meta = ElementTree.tostring(root, encoding=osc.core.ET_ENCODING)
    if pkg_meta != new_pkg_meta:
373
        print("Updating repositories for", package)
374
        osc.core.edit_meta("pkg", (project, package), data=new_pkg_meta)
375
376

    return True
Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
377
378


379
380
def mpcdf_setup_repositories(api_url, project, microarchitecture=None, distribution=None, parent=None,
                             packages=None, dry_run=False, filter_repos=None, only_project=False, remove_old=False):
381
382
    import threading

Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
383
    if parent:
384
385
386
        for attribute in config_attributes + default_attributes:
            print("Copying attribute '{0}' from parent project".format(attribute))
            set_attribute(api_url, project, None, get_attribute(api_url, parent, None, attribute))
Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
387

388
389
390
391
392
393
    compilers = get_attribute_values(api_url, project, None, "MPCDF:compiler_modules")
    mpis = get_attribute_values(api_url, project, None, "MPCDF:mpi_modules")
    cudas = get_attribute_values(api_url, project, None, "MPCDF:cuda_modules")
    pgis = get_attribute_values(api_url, project, None, "MPCDF:pgi_modules")
    default_python2 = get_attribute_value(api_url, project, None, "MPCDF:default_python2")
    default_python3 = get_attribute_value(api_url, project, None, "MPCDF:default_python3")
Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
394

395
396
    if distribution is None:
        # Get existing value from project meta
397
398
399
400
401
402
        dist_repo = project_meta(api_url, project if parent is None else parent).find(
            "./repository[@name='System']/path[@project='distributions']")
        if dist_repo is not None:
            distribution = dist_repo.get("repository")
        else:
            raise osc.oscerr.WrongArgs("ERROR: Specify distribution or parent project")
403
404
        print("Using '{0}' as base distribution".format(distribution))

Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
405
406
407
    distributions = project_meta(api_url, "distributions")
    dist_repo = distributions.find('./repository[@name="{0}"]'.format(distribution))
    if dist_repo is None:
Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
408
        raise osc.oscerr.WrongArgs("No repository '{0}' is defined in project 'distributions' on the server".format(distribution))
Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
409
410
    architectures = list(arch.text for arch in dist_repo.findall("./arch"))

411
412
413
    if microarchitecture is None:
        microarchitecture = get_microarchitecture(project)

Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
414
415
    root = project_meta(api_url, project)

Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
416
    prjconf = list(map(decode_it, osc.core.show_project_conf(api_url, project)))
Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
417
418

    try:
419
420
        start = prjconf.index(prjconf_start_marker)
        end = prjconf.index(prjconf_end_marker)
Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
421
422
423
424
425
    except ValueError:
        start = None
        end = len(prjconf)

    prjconf_head = "".join(prjconf[:start])
Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
426
    prjconf_tail = "".join(prjconf[end + 1:])
427
    prjconf = [prjconf_start_marker]
Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
428

429
430
431
432
433
434
435
436
437
    prjconf.append("Constraint: hostlabel {0}".format(microarchitecture))
    prjconf.append(
        """
%if %_repository != System
Macros:
%microarchitecture {0}
:Macros
%endif""".format(microarchitecture))
    prjconf.append("")
438
439
440
441
    prjconf.append("Prefer: mpcdf_python2_" + default_python2)
    prjconf.append("Prefer: mpcdf_python3_" + default_python3)
    prjconf.append("")

Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
442
    # Remove existing repositories
443
444
445
    if remove_old:
        for oldrepo in root.findall("./repository"):
            root.remove(oldrepo)
Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
446
447

    def repo(name, *dependencies, **kwargs):
448
        is_compiler = kwargs.pop("compiler", False)
Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
449
450
        is_mpi = kwargs.pop("mpi", False)
        is_cuda = kwargs.pop("cuda", False)
451
452
        is_cuda_mpi = kwargs.pop("cuda_mpi", False)

453
454
        cuda_repo = kwargs.pop("cuda_repo", "")

455
456
457
458
        have_compiler = is_compiler or is_mpi or is_cuda or is_cuda_mpi
        have_mpi = is_mpi or is_cuda_mpi
        have_cuda = is_cuda or is_cuda_mpi

459
460
461
462
463
464
        existing_repo = root.find("./repository[@name='{0}']".format(name))
        if existing_repo is not None:
            root.remove(existing_repo)
        else:
            print("New repository", name)

Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
465
466
        r = ElementTree.SubElement(root, "repository")
        r.set("name", name)
Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
467
468
        r.text = "\n    "

Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
469
470
471
472
        def path(project, repo):
            p = ElementTree.SubElement(r, "path")
            p.set("project", project)
            p.set("repository", repo)
Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
473
            p.tail = "\n    "
Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
474

475
476
        if parent:
            path(parent, name)
477
478
        for dep_project, dep_repo in dependencies:
            path(dep_project, dep_repo)
Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
479
480
481
482
483
484
485
486

        for arch in architectures:
            a = ElementTree.SubElement(r, "arch")
            a.text = arch
            a.tail = "\n    "
        a.tail = "\n  "
        r.tail = "\n  "

Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
487
488
489
490
        # In order to be able to figure out the matching MKL version for a given
        # compiler/MPI repository we emit a new macro '%matching_mkl_version' in
        # the cases this makes sense
        matching_mkl = []
Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
491
        prjconf.append("%if %_repository == {0}".format(name))
492
        for prefer in prefers(name):
Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
493
494
            if prefer.startswith("mkl_"):
                matching_mkl.append(prefer)
495
            prjconf.append("Prefer: " + prefer)
Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
496

Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
497
        prjconf.append("Macros:")
498
499

        prjconf.append("%is_compiler_repository {0}".format(1 if is_compiler else 0))
Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
500
501
        prjconf.append("%is_mpi_repository {0}".format(1 if is_mpi else 0))
        prjconf.append("%is_cuda_repository {0}".format(1 if is_cuda else 0))
502
503
504
505
506
507
        prjconf.append("%is_cuda_mpi_repository {0}".format(1 if is_cuda_mpi else 0))

        prjconf.append("%have_mpcdf_compiler {0}".format(1 if have_compiler else 0))
        prjconf.append("%have_mpcdf_mpi {0}".format(1 if have_mpi else 0))
        prjconf.append("%have_mpcdf_cuda {0}".format(1 if have_cuda else 0))

508
509
510
        if is_cuda:
            prjconf.append("%cuda_repository {0}".format(cuda_repo))

Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
511
512
513
514
515
516
        if matching_mkl:
            matching_mkl, = matching_mkl
            matching_mkl, _ = matching_mkl[len("mkl_"):].split("-module")
            matching_mkl = matching_mkl.replace("_", ".")
            prjconf.append("%matching_mkl_version {0}".format(matching_mkl))

517
518
519
        for macro, value in kwargs.items():
            prjconf.append("%{0} {1}".format(macro, value))

Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
520
521
522
523
        prjconf.append(":Macros")
        prjconf.append("%endif")
        prjconf.append("")

Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
524
525
526
527
    if parent:
        repo("System")
    else:
        repo("System", ("distributions", distribution))
Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
528

529
    for compiler in compilers + pgis:
530
531
        repo(compiler, (project, "System"), compiler=True,
             compiler_repository=compiler)
Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
532
533

        for mpi in filter(partial(valid_mpi, compiler), mpis):
534
535
            repo(mpi + "_" + compiler, (project, compiler), mpi=True,
                 mpi_repository=mpi, compiler_repository=compiler)
Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
536
537
538

    for cuda in cudas:
        for compiler in filter(partial(valid_cuda, cuda), compilers):
539
            repo(cuda + "_" + compiler, (project, compiler), cuda=True, cuda_repo=cuda)
Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
540
541
542
543
            for mpi in filter(partial(valid_mpi, compiler), mpis):
                repo(cuda + "_" + mpi + "_" + compiler,
                     (project, cuda + "_" + compiler),
                     (project, mpi + "_" + compiler),
544
                     cuda_mpi=True)
Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
545
546
547
548
549
550
551
552
553
554
555
556

    # Remove build configuration
    build = root.find("./build")
    if build is None:
        build = ElementTree.Element("build")
        build.text = "\n    "
        disable = ElementTree.SubElement(build, "disable")
        disable.tail = "\n  "
        build.tail = "\n  "
        root.insert(list(root).index(root.find("./repository")), build)

    root.getchildren()[-1].tail = "\n"
Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
557
    prj = ElementTree.tostring(root, encoding=osc.core.ET_ENCODING)
Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
558

559
    prjconf.append(prjconf_end_marker)
Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
560
561
562
563
564
565
    prjconf = prjconf_head + "\n".join(prjconf) + prjconf_tail

    if dry_run:
        print("osc meta prj {0} -F - <<EOF\n{1}\nEOF\n".format(project, prj))
        print("osc meta prjconf {0} -F - <<EOF\n{1}\nEOF\n".format(project, prjconf))
    else:
566
567
568
569
570
        if not only_project:
            # First set-up the <enable/> flags, that way no
            # spurious builds are launched
            if packages is None:
                packages = osc.core.meta_get_packagelist(api_url, project)
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589

            if len(packages) > 40:
                chunksize = len(packages) // 20
            else:
                chunksize = len(packages)

            def work(packagelist):
                for package in packagelist:
                    if not mpcdf_enable_repositories(api_url, project, package, filter_repos=filter_repos):
                        print("ATTENTION: Not changing unmanaged package {0}".format(package))

            threads = []
            for packagelist in chunked(packages, chunksize):
                t = threading.Thread(target=work, args=(packagelist,))
                threads.append(t)
                t.start()

            for t in threads:
                t.join()
Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
590
591
592

        # Update repositories
        print("Updating prj meta")
593
        osc.core.edit_meta("prj", project, data=prj, force=True)
Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
594
595
        print("Updating prjconf meta")
        osc.core.edit_meta("prjconf", project, data=prjconf)
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635


def set_as_branch(api_url, my_project, my_package, main_project, main_package):
    import os

    def dirmeta(project, package):
        url = osc.core.makeurl(api_url, ["source", project, package])
        return ElementTree.fromstringlist(osc.core.streamfile(url, osc.core.http_GET))

    srcmeta = dirmeta(my_project, my_package)
    dstmeta = dirmeta(main_project, main_package)

    linkinfo = srcmeta.find('./linkinfo')
    if linkinfo is not None:
        print("ERROR: package {0} is already linked to package {1} in project {2}".format(
              my_package, linkinfo.get("package"), linkinfo.get("project")))
        return False

    linkfile = ElementTree.Element("link")
    linkfile.set("project", main_project)
    linkfile.set("package", main_package)
    linkfile.set("baserev", dstmeta.get("srcmd5"))
    patches = ElementTree.SubElement(linkfile, "patches")
    ElementTree.SubElement(patches, "branch")

    url = osc.core.makeurl(api_url, ["source", my_project, my_package, "_link"])
    resp = ElementTree.fromstringlist(osc.core.streamfile(url, osc.core.http_PUT, data=ElementTree.tostring(linkfile)))
    rev = resp.get("rev")
    if rev is None:
        print("ERROR: Could not commit _link file")
        return False
    else:
        print("Commited as revision", rev)
        this_package = osc.core.store_read_package(os.curdir)
        this_project = osc.core.store_read_project(os.curdir)
        if this_package == my_package and this_project == my_project:
            pac = osc.core.filedir_to_pac(os.curdir)
            rev = pac.latest_rev(expand=True)
            pac.update(rev)
    return True
636
637


638
def sync_projects(api_url, package=None, from_project="software", to_projects=None, redo_all=False, add_to_maintainers=True):
639
640
641
642
    if package is not None and redo_all:
        raise osc.oscerr.WrongArgs('Cannot specify `redo_all` and package')

    if to_projects is None:
Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
643
644
        to_projects = [p for p in osc.core.meta_get_project_list(api_url)
                       if p.startswith("software:") and not (p == "software:dist" or p == "software:images")]
645
646
647
648
649
650
651
652
653
654
655
656
657

    for to_project in to_projects:
        print("Syncing {0} with {1}".format(to_project, from_project))

        to_packages = osc.core.meta_get_packagelist(api_url, to_project)

        if package is None:
            from_packages = osc.core.meta_get_packagelist(api_url, from_project)
            if not redo_all:
                from_packages = list(sorted(set(from_packages) - set(to_packages)))
        else:
            from_packages = [package]

658
        for attribute in config_attributes:
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
            try:
                get_attribute(api_url, to_project, None, attribute)
            except Exception:
                attr = get_attribute(api_url, from_project, None, attribute)
                print("Setting attribute", attribute)
                set_attribute(api_url, to_project, None, attr)

        for orig_package in from_packages:
            if orig_package not in to_packages:
                filelist = osc.core.meta_get_filelist(api_url, from_project, orig_package)
                if "_link" in filelist:
                    print("Not branching package {0}, is a link".format(orig_package))
                else:
                    print("Branching package {0}".format(orig_package))
                    exists, targetprj, targetpkg, srcprj, srcpkg = \
                        osc.core.branch_pkg(api_url, from_project, orig_package, target_project=to_project, nodevelproject=True)
            else:
                print("Not branching package {0}, already present in target".format(orig_package))

678
            for attribute in package_attributes + config_attributes:
679
680
681
682
683
684
685
686
                try:
                    attr = get_attribute(api_url, from_project, orig_package, attribute)
                except UnsetAttributeException:
                    if has_attribute(api_url, to_project, orig_package, attribute):
                        remove_attribute(api_url, to_project, orig_package, attribute)
                    continue
                set_attribute(api_url, to_project, orig_package, attr)

687
688
689
690
691
692
693
694
695
696
697
698
699
            if add_to_maintainers:
                from_maintainers = maintainers(api_url, from_project, orig_package)
                if from_maintainers:
                    to_maintainers = maintainers(api_url, to_project, orig_package)
                    if from_maintainers - to_maintainers:
                        new_maintainers = from_maintainers - to_maintainers
                        to_meta = package_meta(api_url, to_project, orig_package)
                        for userid in sorted(new_maintainers):
                            person = ElementTree.Element("person")
                            person.set("userid", userid)
                            person.set("role", "maintainer")
                            to_meta.insert(2, person)
                        osc.core.edit_meta("pkg", (to_project, orig_package), data=ElementTree.tostring(to_meta))
700

701
702
703
704
705
706
707
708
709
710
        if package is None and redo_all:
            # Check if distribution is already set in to_project
            to_prj_meta = project_meta(api_url, to_project)
            sys_repo = to_prj_meta.find('./repository[@name="System"]')
            if sys_repo is None:
                distribution = project_meta(api_url, from_project).find('./repository[@name="System"]/path[@project="distributions"]').get("repository")
            else:
                distribution = sys_repo.find('./path[@project="distributions"]').get("repository")

            print("Creating repository configuration")
711
            mpcdf_setup_repositories(api_url, to_project, distribution=distribution)
712
713
714
715
        else:
            for orig_package in from_packages:
                if not mpcdf_enable_repositories(api_url, to_project, orig_package):
                    print("ATTENTION: Not changing unmanaged package {0}".format(orig_package))