mpcdf_common.py 22.5 KB
Newer Older
1
2
3
4
5
from __future__ import print_function

import osc
import osc.conf
import osc.core
Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
6
import osc.oscerr
7

Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
8
from functools import partial
9
10
from xml.etree import ElementTree

11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
intel_parallel_studio = {
    "mpcdf_intel_parallel_studio_2017_7": {"compiler": "intel_17_0_7", "impi": "impi_2017_4", "mkl": "mkl_2017_4-module", },
    "mpcdf_intel_parallel_studio_2018_1": {"compiler": "intel_18_0_1", "impi": "impi_2018_1", "mkl": "mkl_2018_1-module", },
    "mpcdf_intel_parallel_studio_2018_2": {"compiler": "intel_18_0_2", "impi": "impi_2018_2", "mkl": "mkl_2018_2-module", },
    "mpcdf_intel_parallel_studio_2018_3": {"compiler": "intel_18_0_3", "impi": "impi_2018_3", "mkl": "mkl_2018_3-module", },
    "mpcdf_intel_parallel_studio_2018_4": {"compiler": "intel_18_0_5", "impi": "impi_2018_4", "mkl": "mkl_2018_4-module", },
    "mpcdf_intel_parallel_studio_2019_0": {"compiler": "intel_19_0_0", "impi": "impi_2019_0", "mkl": "mkl_2019_0-module", },
    "mpcdf_intel_parallel_studio_2019_1": {"compiler": "intel_19_0_1", "impi": "impi_2019_1", "mkl": "mkl_2019_1-module", },
}

all_mkls = {ic["mkl"] for ic in intel_parallel_studio.values()}

# Some rotations
mpi_parallel_studio = {value["impi"]: dict({"ps": key}, **value) for key, value in intel_parallel_studio.items()}
compiler_parallel_studio = {value["compiler"]: dict({"ps": key}, **value) for key, value in intel_parallel_studio.items()}

27
28

def valid_mpi(compiler, mpi):
29
30
31
32
33
34
35
36
37
    """
    It might be possible to use Intel MPI libararies and compilers from
    different Parallel Studio packages, but I currently do not want to support
    it.

    Take care to keep this in sync with the file 'macros.obs_cluster' of
    the package software:dist / mpcdf_cluster_macros
    """
    if compiler.startswith("intel") and mpi.startswith("impi"):
38
        return mpi == compiler_parallel_studio[compiler]["impi"]
39
40
    else:
        return True
41
42


43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
def prefers(reponame):
    prefer_ps = None
    if reponame in compiler_parallel_studio:
        prefer_ps = compiler_parallel_studio[reponame]["ps"]
    else:
        for mpi in mpi_parallel_studio:
            if reponame.startswith(mpi):
                prefer_ps = mpi_parallel_studio[mpi]["ps"]

    if prefer_ps:
        preferred_mkl = intel_parallel_studio[prefer_ps]["mkl"]
        unprefer_other_mkls = ("!" + mkl for mkl in all_mkls if mkl != preferred_mkl)
        return (prefer_ps,) + tuple(unprefer_other_mkls) + (preferred_mkl,)
    else:
        return ()


60
61
62
63
def valid_cuda(cuda, compiler):
    """
    The CUDA distribution only works with certain gcc versions,
    this little function takes care that only supported combinations
Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
64
65
66
67
    are allowed.

    Take care to keep this in sync with the file 'macros.obs_cluster' of
    the package software:dist / mpcdf_cluster_macros
68
    """
Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
69
    if cuda == "cuda_8_0":
70
        return compiler == "gcc_5"
Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
71
    if cuda == "cuda_9_1":
72
        return compiler == "gcc_6_3_0"
Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
73
    if cuda == "cuda_9_2":
Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
74
        return compiler == "gcc_6"
Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
75
76
    if cuda == "cuda_10_0":
        return compiler == "gcc_6"
77
78
79
    return False


Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
80
81
82
def project_meta(api_url, project):
    return ElementTree.fromstringlist(osc.core.show_project_meta(api_url, project))

83

84
85
86
87
class UnsetAttributeException(Exception):
    pass


Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
88
def get_attribute(api_url, project, package, attribute, with_project=False):
89
90
    attribute_meta = osc.core.show_attribute_meta(api_url, project, package, None, attribute, False, with_project)
    if attribute_meta is None:
Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
91
        raise osc.oscerr.APIError("Cannot fetch value for attribute '{0}' from {1}".format(attribute, (project, package)))
92
93
94
95

    root = ElementTree.fromstringlist(attribute_meta)
    attribute = root.find("./attribute")
    if attribute is not None:
Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
96
        return root
97
    else:
98
        raise UnsetAttributeException("Attribute not set")
99

Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
100

Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
101
102
103
104
105
106
def get_attribute_values(api_url, project, package, attribute, with_project=False):
    root = get_attribute(api_url, project, package, attribute, with_project)
    attribute = root.find("./attribute")
    return list(value.text for value in attribute.findall("./value"))


107
108
109
110
111
112
def set_attribute(api_url, project, package, attribute):
    path = ["source", project]
    if package:
        path.append(package)
    path.append("_attribute")

Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
113
    attr_meta = ElementTree.tostring(attribute, encoding=osc.core.ET_ENCODING)
Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
114

115
    url = osc.core.makeurl(api_url, path)
Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
116
    resp = ElementTree.fromstringlist(osc.core.streamfile(url, osc.core.http_POST, data=attr_meta))
Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
117
    if resp.find("./summary").text != "Ok":
Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
118
        raise osc.oscerr.APIError("Could not store attribute")
Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
119

120

121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
def set_attribute_values(api_url, project, package, attribute, values):
    root = ElementTree.Element("attributes")
    attr = ElementTree.SubElement(root, "attribute")

    namespace, name = attribute.split(":")
    attr.set("namespace", namespace)
    attr.set("name", name)

    for value in values:
        v = ElementTree.SubElement(attr, "value")
        v.text = value

    set_attribute(api_url, project, package, root)


def has_attribute(api_url, project, package, attribute):
    path = ["source", project]
    if package:
        path.append(package)
    path.append("_attribute")
    path.append(attribute)

    url = osc.core.makeurl(api_url, path)
    resp = ElementTree.fromstringlist(osc.core.streamfile(url, osc.core.http_GET))

    namespace, name = attribute.split(":")
147

148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
    for a in resp.findall("./attribute"):
        if a.get("namespace") == namespace and a.get("name") == name:
            return True
    return False


def remove_attribute(api_url, project, package, attribute_name):
    path = ["source"] + [project]
    if package:
        path.append(package)
    path.append("_attribute")
    path.append(attribute_name)

    url = osc.core.makeurl(api_url, path)
    resp = ElementTree.fromstringlist(osc.core.streamfile(url, osc.core.http_DELETE))
    if resp.find("./summary").text != "Ok":
        raise osc.oscerr.APIError("Could not remove attribute")


167
def mpcdf_enable_repositories(api_url, project, package, verbose=False, filter_repos=None):
168
    from itertools import product
Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
169
    import sys
170

171
172
173
    if filter_repos is None:
        filter_repos = ()

174
175
    root = ElementTree.fromstringlist(osc.core.show_package_meta(api_url, project, package))
    build = root.find("./build")
Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
176
177
178
179
180
    if build is None:
        build = ElementTree.SubElement(root, "build")
    else:
        for enable in build.findall("./enable"):
            build.remove(enable)
181

Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
182
    try:
183
184
185
186
187
        enable_repos = get_attribute_values(api_url, project, package, "MPCDF:enable_repositories")
    except Exception:
        if verbose:
            print("Warning: Could not get attribute MPCDF:enable_repositories for package {0}, skipping".format(package))
        return False
Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
188

Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
189
    def try_get_attribute(package, attribute, with_project=False):
190
191
192
        try:
            return get_attribute_values(api_url, project, package, "MPCDF:" + attribute, with_project=with_project)
        except UnsetAttributeException:
193
194
195
            print("ERROR: Attribute MPCDF:" + attribute + " is not set for "
                  + ("package '{0}'".format(package) if package else "project '{0}'".format(project))
                  + ", aborting here", file=sys.stderr)
196
            raise SystemExit(1)
Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
197

198
199
200
    compilers = try_get_attribute(package, "compiler_modules", with_project=True)
    mpis = try_get_attribute(package, "mpi_modules", with_project=True)
    cudas = try_get_attribute(package, "cuda_modules", with_project=True)
201
202
203
204
205
206
    pgis = try_get_attribute(package, "pgi_modules", with_project=True)

    all_compilers = try_get_attribute(None, "compiler_modules")
    all_mpis = try_get_attribute(None, "mpi_modules")
    all_cudas = try_get_attribute(None, "cuda_modules")
    all_pgis = try_get_attribute(None, "pgi_modules")
207
208
209
210

    default_compilers = try_get_attribute(None, "default_compiler")
    default_mpis = try_get_attribute(None, "default_mpi")
    default_cudas = try_get_attribute(None, "default_cuda")
211

212
    def enable(name):
213
214
        if any(filtered_repo in name for filtered_repo in filter_repos):
            return
215
216
217
218
219
220
221
        node = ElementTree.Element("enable")
        node.set("repository", name)
        node.tail = "\n    "
        build.insert(0, node)
        if verbose:
            print("Enabling", name)

222
    def actual_compilers():
223
        for compiler in (c for c in compilers if c in all_compilers + ["default_compiler", "intel", "gcc"]):
224
            if compiler == "intel":
Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
225
                for intel_compiler in filter(lambda cc: cc.startswith("intel"), all_compilers):
226
227
                    yield intel_compiler
            elif compiler == "gcc":
Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
228
                for gcc_compiler in filter(lambda cc: cc.startswith("gcc"), all_compilers):
229
230
231
232
233
234
235
236
                    yield gcc_compiler
            elif compiler == "default_compiler":
                for default_compiler in default_compilers:
                    yield default_compiler
            else:
                yield compiler

    def actual_mpis():
237
        for mpi in (m for m in mpis if m in all_mpis + ["default_mpi", "impi"]):
238
239
240
241
242
243
244
245
246
247
            if mpi == "impi":
                for impi in filter(lambda cc: cc.startswith("impi"), mpis):
                    yield impi
            elif mpi == "default_mpi":
                for default_mpi in default_mpis:
                    yield default_mpi
            else:
                yield mpi

    def actual_cudas():
248
        for cuda in (c for c in cudas if c in all_cudas + ["default_cuda"]):
249
250
251
252
253
254
            if cuda == "default_cuda":
                for default_cuda in default_cudas:
                    yield default_cuda
            else:
                yield cuda

255
256
257
258
    def actual_pgis():
        for pgi in (p for p in pgis if p in all_pgis):
            yield pgi

259
260
261
262
263
264
    for flag in enable_repos:

        if flag == "system":
            enable("System")

        if flag == "compilers":
265
            for compiler in actual_compilers():
Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
266
                enable(compiler)
267
268

        if flag == "mpi":
269
270
271
            for mpi, compiler in product(actual_mpis(), actual_compilers()):
                if valid_mpi(compiler, mpi):
                    enable(mpi + "_" + compiler)
272
273

        if flag == "cuda":
274
275
276
            for cuda, compiler in product(actual_cudas(), all_compilers):
                if valid_cuda(cuda, compiler):
                    enable(cuda + "_" + compiler)
277
278

        if flag == "cuda_mpi":
279
280
281
            for cuda, mpi, compiler in product(actual_cudas(), actual_mpis(), all_compilers):
                if valid_cuda(cuda, compiler) and valid_mpi(compiler, mpi):
                        enable(cuda + "_" + mpi + "_" + compiler)
282

283
284
285
286
287
288
289
290
        if flag == "pgi":
            for pgi in actual_pgis():
                enable(pgi)

        if flag == "pgi_mpi":
            for mpi, pgi in product(actual_mpis(), actual_pgis()):
                enable(mpi + "_" + pgi)

Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
291
292
    if len(build.getchildren()) > 0:
        build.getchildren()[-1].tail = "\n  "
293

Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
294
    pkg_meta = ElementTree.tostring(root, encoding=osc.core.ET_ENCODING)
295
296
297

    osc.core.edit_meta("pkg", (project, package), data=pkg_meta)
    return True
Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
298
299


300
def mpcdf_setup_repositories(api_url, project, distribution=None, parent=None, packages=None, dry_run=False, filter_repos=None, only_project=False):
Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
301
    if parent:
302
        for attribute in ["compiler_modules", "default_compiler", "mpi_modules", "default_mpi", "cuda_modules", "default_cuda", "pgi_modules"]:
303
304
            print("Copying attribute 'MPCDF:{0}' from parent project".format(attribute))
            set_attribute(api_url, project, None, get_attribute(api_url, parent, None, "MPCDF:" + attribute))
Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
305

Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
306
307
308
    compilers = list(get_attribute_values(api_url, project, None, "MPCDF:compiler_modules"))
    mpis = list(get_attribute_values(api_url, project, None, "MPCDF:mpi_modules"))
    cudas = list(get_attribute_values(api_url, project, None, "MPCDF:cuda_modules"))
309
    pgis = list(get_attribute_values(api_url, project, None, "MPCDF:pgi_modules"))
Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
310

311
312
    if distribution is None:
        # Get existing value from project meta
313
314
315
316
317
318
        dist_repo = project_meta(api_url, project if parent is None else parent).find(
            "./repository[@name='System']/path[@project='distributions']")
        if dist_repo is not None:
            distribution = dist_repo.get("repository")
        else:
            raise osc.oscerr.WrongArgs("ERROR: Specify distribution or parent project")
319
320
        print("Using '{0}' as base distribution".format(distribution))

Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
321
322
323
    distributions = project_meta(api_url, "distributions")
    dist_repo = distributions.find('./repository[@name="{0}"]'.format(distribution))
    if dist_repo is None:
Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
324
        raise osc.oscerr.WrongArgs("No repository '{0}' is defined in project 'distributions' on the server".format(distribution))
Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
    architectures = list(arch.text for arch in dist_repo.findall("./arch"))

    root = project_meta(api_url, project)

    prjconf = osc.core.show_project_conf(api_url, project)
    start_marker = "# Autogenerated by osc mpcdf_setup_repos, do not edit till end of section\n"
    end_marker = "# End of autogenerated section\n"

    try:
        start = prjconf.index(start_marker)
        end = prjconf.index(end_marker)
    except ValueError:
        start = None
        end = len(prjconf)

    prjconf_head = "".join(prjconf[:start])
Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
341
    prjconf_tail = "".join(prjconf[end + 1:])
Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
342
343
344
    prjconf = [start_marker]

    # Remove existing repositories
345
346
    for oldrepo in root.findall("./repository"):
        root.remove(oldrepo)
Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
347
348

    def repo(name, *dependencies, **kwargs):
349
        is_compiler = kwargs.pop("compiler", False)
Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
350
351
        is_mpi = kwargs.pop("mpi", False)
        is_cuda = kwargs.pop("cuda", False)
352
353
        is_cuda_mpi = kwargs.pop("cuda_mpi", False)

354
355
        cuda_repo = kwargs.pop("cuda_repo", "")

356
357
358
359
        have_compiler = is_compiler or is_mpi or is_cuda or is_cuda_mpi
        have_mpi = is_mpi or is_cuda_mpi
        have_cuda = is_cuda or is_cuda_mpi

Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
360
361
        r = ElementTree.SubElement(root, "repository")
        r.set("name", name)
Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
362
363
        r.text = "\n    "

Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
364
365
366
367
        def path(project, repo):
            p = ElementTree.SubElement(r, "path")
            p.set("project", project)
            p.set("repository", repo)
Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
368
            p.tail = "\n    "
Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
369

370
371
        if parent:
            path(parent, name)
372
373
        for dep_project, dep_repo in dependencies:
            path(dep_project, dep_repo)
Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
374
375
376
377
378
379
380
381
382

        for arch in architectures:
            a = ElementTree.SubElement(r, "arch")
            a.text = arch
            a.tail = "\n    "
        a.tail = "\n  "
        r.tail = "\n  "

        prjconf.append("%if %_repository == {0}".format(name))
383
384
        for prefer in prefers(name):
            prjconf.append("Prefer: " + prefer)
Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
385
        prjconf.append("Macros:")
386
387

        prjconf.append("%is_compiler_repository {0}".format(1 if is_compiler else 0))
Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
388
389
        prjconf.append("%is_mpi_repository {0}".format(1 if is_mpi else 0))
        prjconf.append("%is_cuda_repository {0}".format(1 if is_cuda else 0))
390
391
392
393
394
395
        prjconf.append("%is_cuda_mpi_repository {0}".format(1 if is_cuda_mpi else 0))

        prjconf.append("%have_mpcdf_compiler {0}".format(1 if have_compiler else 0))
        prjconf.append("%have_mpcdf_mpi {0}".format(1 if have_mpi else 0))
        prjconf.append("%have_mpcdf_cuda {0}".format(1 if have_cuda else 0))

396
397
398
        if is_cuda:
            prjconf.append("%cuda_repository {0}".format(cuda_repo))

399
400
401
        for macro, value in kwargs.items():
            prjconf.append("%{0} {1}".format(macro, value))

Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
402
403
404
405
        prjconf.append(":Macros")
        prjconf.append("%endif")
        prjconf.append("")

Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
406
407
408
409
    if parent:
        repo("System")
    else:
        repo("System", ("distributions", distribution))
Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
410

411
    for compiler in compilers + pgis:
412
        repo(compiler, (project, "System"), compiler=True)
Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
413
414

        for mpi in filter(partial(valid_mpi, compiler), mpis):
415
            repo(mpi + "_" + compiler, (project, compiler), mpi=True, mpi_repository=mpi)
Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
416
417
418

    for cuda in cudas:
        for compiler in filter(partial(valid_cuda, cuda), compilers):
419
            repo(cuda + "_" + compiler, (project, compiler), cuda=True, cuda_repo=cuda)
Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
420
421
422
423
            for mpi in filter(partial(valid_mpi, compiler), mpis):
                repo(cuda + "_" + mpi + "_" + compiler,
                     (project, cuda + "_" + compiler),
                     (project, mpi + "_" + compiler),
424
                     cuda_mpi=True)
Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
425
426
427
428
429
430
431
432
433
434
435
436

    # Remove build configuration
    build = root.find("./build")
    if build is None:
        build = ElementTree.Element("build")
        build.text = "\n    "
        disable = ElementTree.SubElement(build, "disable")
        disable.tail = "\n  "
        build.tail = "\n  "
        root.insert(list(root).index(root.find("./repository")), build)

    root.getchildren()[-1].tail = "\n"
Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
437
    prj = ElementTree.tostring(root, encoding=osc.core.ET_ENCODING)
Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
438
439
440
441
442
443
444
445

    prjconf.append(end_marker)
    prjconf = prjconf_head + "\n".join(prjconf) + prjconf_tail

    if dry_run:
        print("osc meta prj {0} -F - <<EOF\n{1}\nEOF\n".format(project, prj))
        print("osc meta prjconf {0} -F - <<EOF\n{1}\nEOF\n".format(project, prjconf))
    else:
446
447
448
449
450
451
452
453
454
        if not only_project:
            # First set-up the <enable/> flags, that way no
            # spurious builds are launched
            if packages is None:
                packages = osc.core.meta_get_packagelist(api_url, project)
            for package in packages:
                print("Updating repositories for", package)
                if not mpcdf_enable_repositories(api_url, project, package, filter_repos=filter_repos):
                    print("ATTENTION: Not changing unmanaged package {0}".format(package))
Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
455
456
457

        # Update repositories
        print("Updating prj meta")
458
        osc.core.edit_meta("prj", project, data=prj, force=True)
Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
459
460
        print("Updating prjconf meta")
        osc.core.edit_meta("prjconf", project, data=prjconf)
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500


def set_as_branch(api_url, my_project, my_package, main_project, main_package):
    import os

    def dirmeta(project, package):
        url = osc.core.makeurl(api_url, ["source", project, package])
        return ElementTree.fromstringlist(osc.core.streamfile(url, osc.core.http_GET))

    srcmeta = dirmeta(my_project, my_package)
    dstmeta = dirmeta(main_project, main_package)

    linkinfo = srcmeta.find('./linkinfo')
    if linkinfo is not None:
        print("ERROR: package {0} is already linked to package {1} in project {2}".format(
              my_package, linkinfo.get("package"), linkinfo.get("project")))
        return False

    linkfile = ElementTree.Element("link")
    linkfile.set("project", main_project)
    linkfile.set("package", main_package)
    linkfile.set("baserev", dstmeta.get("srcmd5"))
    patches = ElementTree.SubElement(linkfile, "patches")
    ElementTree.SubElement(patches, "branch")

    url = osc.core.makeurl(api_url, ["source", my_project, my_package, "_link"])
    resp = ElementTree.fromstringlist(osc.core.streamfile(url, osc.core.http_PUT, data=ElementTree.tostring(linkfile)))
    rev = resp.get("rev")
    if rev is None:
        print("ERROR: Could not commit _link file")
        return False
    else:
        print("Commited as revision", rev)
        this_package = osc.core.store_read_package(os.curdir)
        this_project = osc.core.store_read_project(os.curdir)
        if this_package == my_package and this_project == my_project:
            pac = osc.core.filedir_to_pac(os.curdir)
            rev = pac.latest_rev(expand=True)
            pac.update(rev)
    return True
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569


def sync_projects(api_url, package=None, from_project="software", to_projects=None, redo_all=False):
    if package is not None and redo_all:
        raise osc.oscerr.WrongArgs('Cannot specify `redo_all` and package')

    if to_projects is None:
        to_projects = filter(lambda s: s.startswith("software:") and not (s == "software:dist" or s == "software:images"),
                             osc.core.meta_get_project_list(api_url))

    for to_project in to_projects:
        print("Syncing {0} with {1}".format(to_project, from_project))

        to_packages = osc.core.meta_get_packagelist(api_url, to_project)

        if package is None:
            from_packages = osc.core.meta_get_packagelist(api_url, from_project)
            if not redo_all:
                from_packages = list(sorted(set(from_packages) - set(to_packages)))
        else:
            from_packages = [package]

        project_attributes = ["MPCDF:compiler_modules", "MPCDF:cuda_modules", "MPCDF:mpi_modules"]
        package_attributes = ["MPCDF:enable_repositories"] + project_attributes

        for attribute in project_attributes:
            try:
                get_attribute(api_url, to_project, None, attribute)
            except Exception:
                attr = get_attribute(api_url, from_project, None, attribute)
                print("Setting attribute", attribute)
                set_attribute(api_url, to_project, None, attr)

        for orig_package in from_packages:
            if orig_package not in to_packages:
                filelist = osc.core.meta_get_filelist(api_url, from_project, orig_package)
                if "_link" in filelist:
                    print("Not branching package {0}, is a link".format(orig_package))
                else:
                    print("Branching package {0}".format(orig_package))
                    exists, targetprj, targetpkg, srcprj, srcpkg = \
                        osc.core.branch_pkg(api_url, from_project, orig_package, target_project=to_project, nodevelproject=True)
            else:
                print("Not branching package {0}, already present in target".format(orig_package))

            for attribute in package_attributes:
                try:
                    attr = get_attribute(api_url, from_project, orig_package, attribute)
                except UnsetAttributeException:
                    if has_attribute(api_url, to_project, orig_package, attribute):
                        remove_attribute(api_url, to_project, orig_package, attribute)
                    continue
                set_attribute(api_url, to_project, orig_package, attr)

        if package is None and redo_all:
            # Check if distribution is already set in to_project
            to_prj_meta = project_meta(api_url, to_project)
            sys_repo = to_prj_meta.find('./repository[@name="System"]')
            if sys_repo is None:
                distribution = project_meta(api_url, from_project).find('./repository[@name="System"]/path[@project="distributions"]').get("repository")
            else:
                distribution = sys_repo.find('./path[@project="distributions"]').get("repository")

            print("Creating repository configuration")
            mpcdf_setup_repositories(api_url, to_project, distribution)
        else:
            for orig_package in from_packages:
                if not mpcdf_enable_repositories(api_url, to_project, orig_package):
                    print("ATTENTION: Not changing unmanaged package {0}".format(orig_package))