mpcdf_common.py 39.2 KB
Newer Older
1
from __future__ import print_function
Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
2
from __future__ import division
3

4
import sys
5
6
7
import osc
import osc.conf
import osc.core
Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
8
import osc.oscerr
9
import textwrap
Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
10
from functools import partial
11
12
from xml.etree import ElementTree

13
14
15
16
17
18
19
if sys.version_info[0] < 3:
    def decode_it(arg):
        return arg
else:
    from osc.util.helper import decode_it


20
21
known_microarchs = {"sandybridge", "haswell", "skylake"}

22
package_attributes = ["MPCDF:enable_repositories"]
23
config_attributes = ["MPCDF:compiler_modules", "MPCDF:cuda_modules", "MPCDF:mpi_modules", "MPCDF:pgi_modules", "MPCDF:openmpi_flavors"]
24
default_attributes = ["MPCDF:default_compiler", "MPCDF:default_cuda", "MPCDF:default_mpi"]
25

26
27
28
29
30
31
32
33
intel_parallel_studio = {
    "mpcdf_intel_parallel_studio_2017_7": {"compiler": "intel_17_0_7", "impi": "impi_2017_4", "mkl": "mkl_2017_4-module", },
    "mpcdf_intel_parallel_studio_2018_1": {"compiler": "intel_18_0_1", "impi": "impi_2018_1", "mkl": "mkl_2018_1-module", },
    "mpcdf_intel_parallel_studio_2018_2": {"compiler": "intel_18_0_2", "impi": "impi_2018_2", "mkl": "mkl_2018_2-module", },
    "mpcdf_intel_parallel_studio_2018_3": {"compiler": "intel_18_0_3", "impi": "impi_2018_3", "mkl": "mkl_2018_3-module", },
    "mpcdf_intel_parallel_studio_2018_4": {"compiler": "intel_18_0_5", "impi": "impi_2018_4", "mkl": "mkl_2018_4-module", },
    "mpcdf_intel_parallel_studio_2019_0": {"compiler": "intel_19_0_0", "impi": "impi_2019_0", "mkl": "mkl_2019_0-module", },
    "mpcdf_intel_parallel_studio_2019_1": {"compiler": "intel_19_0_1", "impi": "impi_2019_1", "mkl": "mkl_2019_1-module", },
Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
34
    "mpcdf_intel_parallel_studio_2019_3": {"compiler": "intel_19_0_3", "impi": "impi_2019_3", "mkl": "mkl_2019_3-module", },
35
    "mpcdf_intel_parallel_studio_2019_4": {"compiler": "intel_19_0_4", "impi": "impi_2019_4", "mkl": "mkl_2019_4-module", },
36
    "mpcdf_intel_parallel_studio_2019_5": {"compiler": "intel_19_0_5", "impi": "impi_2019_5", "mkl": "mkl_2019_5-module", },
Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
37
    "mpcdf_intel_parallel_studio_2020": {"compiler": "intel_19_1_0", "impi": "impi_2019_6", "mkl": "mkl_2020-module", },
38
    "mpcdf_intel_parallel_studio_2020_1": {"compiler": "intel_19_1_1", "impi": "impi_2019_7", "mkl": "mkl_2020_1-module", },
39
    "mpcdf_intel_parallel_studio_2020_2": {"compiler": "intel_19_1_2", "impi": "impi_2019_8", "mkl": "mkl_2020_2-module", },
40
    "mpcdf_intel_parallel_studio_2020_4": {"compiler": "intel_19_1_3", "impi": "impi_2019_9", "mkl": "mkl_2020_4-module", },
Lorenz Hüdepohl's avatar
Lorenz Hüdepohl committed
41
    "mpcdf_intel_oneapi_2021_2": {"compiler": "intel_2021_2_0", "impi": "impi_2021_2", "mkl": "mkl_2021_2-module", },
42
43
44
45
46
47
48
49
}

all_mkls = {ic["mkl"] for ic in intel_parallel_studio.values()}

# Some rotations
mpi_parallel_studio = {value["impi"]: dict({"ps": key}, **value) for key, value in intel_parallel_studio.items()}
compiler_parallel_studio = {value["compiler"]: dict({"ps": key}, **value) for key, value in intel_parallel_studio.items()}

50

51
52
53
54
# For the autogenerated software/software:*:* project 'prjconf' sections
prjconf_start_marker = "# Autogenerated by osc mpcdf_setup_repos, do not edit till end of section\n"
prjconf_end_marker = "# End of autogenerated section\n"

55

Lorenz Hüdepohl's avatar
Lorenz Hüdepohl committed
56
57
58
59
60
61
62
63
64
65
66
def dist_prjconf_tags(distribution):
    centos_prjconf_tags = textwrap.dedent(
        """
        Prefer: perl-Error
        Substitute: c_compiler gcc
        Substitute: c++_compiler gcc-c++
        Substitute: ca-certificates-mozilla ca-certificates
        """).strip()

    centos8_prjconf_tags = textwrap.dedent(
        """
Lorenz Hüdepohl's avatar
Lorenz Hüdepohl committed
67
        Preinstall: libzstd
Lorenz Hüdepohl's avatar
Lorenz Hüdepohl committed
68
        ExpandFlags: module:python36-3.6
69
70
        # Remove redhat-hardened-cc1 and redhat-annobin-cc1 spec
        Optflags: x86_64 -O2 -g -pipe -Wall -Werror=format-security -Wp,-D_FORTIFY_SOURCE=2 -Wp,-D_GLIBCXX_ASSERTIONS -fexceptions -fstack-protector-strong -grecord-gcc-switches -m64 -fasynchronous-unwind-tables -fstack-clash-protection -fcf-protection
Lorenz Hüdepohl's avatar
Lorenz Hüdepohl committed
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
        """).strip()

    res = ""
    if "CentOS" in distribution:
        res += centos_prjconf_tags
    if "CentOS_8" in distribution:
        res += "\n" + centos8_prjconf_tags

    return res


def dist_prjconf_macros(distribution):
    centos_macros = textwrap.dedent(
        """
        # Disable all problematic automatic RPM stuff
        # like byte-compiling (with the wrong Python version)
        # or debug packages that fail for many binary-only packages
        %__no_python_bytecompile 1
        %debug_package %{nil}
        """).strip()

    res = ""
    if "CentOS" in distribution:
        res += centos_macros

    return res
97

98

99
100
101
102
def check_for_update():
    import os
    import sys
    import time
103
104
105
    from subprocess import check_output, call

    DEVNULL = open(os.devnull, 'w')
106
107
108
109
110
111

    if hasattr(sys.modules["mpcdf_common"], "checked_for_updates"):
        return
    else:
        setattr(sys.modules["mpcdf_common"], "checked_for_updates", 1)

112
    if (sys.version_info > (3, 0)):
Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
113
        set_encoding = {"encoding": "utf-8"}
114
115
116
    else:
        set_encoding = {}

117
118
    plugin_dir = os.path.dirname(os.path.realpath(__file__))
    git_dir = os.path.join(plugin_dir, ".git")
119
    local_rev = check_output(["git", "--git-dir", git_dir, "rev-parse", "HEAD"], **set_encoding).strip()
Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
120

Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
121
    url = "https://gitlab.mpcdf.mpg.de/mpcdf/obs/osc-plugins.git"
Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
122
123

    def update_server_rev():
124
        server_rev, _ = check_output(["git", "ls-remote", url, "master"], **set_encoding).split(None, 1)
Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
125
126
        with open(rev_file, "w") as fd:
            fd.write(server_rev)
127
128
129
130
131

    # Check for update on server just once a day
    rev_file = os.path.join(plugin_dir, ".remote_head_rev")
    try:
        mtime = os.stat(rev_file).st_mtime
Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
132
    except EnvironmentError:
133
134
135
        mtime = 0

    if time.time() - mtime > 86400:
Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
136
        update_server_rev()
137

Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
138
    with open(rev_file, "r") as fd:
139
        server_rev = fd.read().strip()
Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
140

141
142
143
144
    if server_rev != local_rev:
        if call(["git", "--git-dir", git_dir, "merge-base", "--is-ancestor", server_rev, "HEAD"], stderr=DEVNULL) == 0:
            # Server rev is older than ours. Check again
            update_server_rev()
Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
145
146
147
148
149

    with open(rev_file, "r") as fd:
        server_rev = fd.read().strip()

    if server_rev != local_rev:
Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
150
        print("Note from MPCDF osc plugins:", file=sys.stderr)
151
        if call(["git", "--git-dir", git_dir, "merge-base", "--is-ancestor", server_rev, "HEAD"], stderr=DEVNULL) == 0:
Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
152
            print(" You have unpushed commits in", plugin_dir, "- consider pushing them", file=sys.stderr)
Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
153
154
            print(file=sys.stderr)
        else:
Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
155
156
            print(" Your plugin directory is out-of-date, new commits available on", url, file=sys.stderr)
            print(" (careful: the repo has been moved, update your path to point to the new URL)", file=sys.stderr)
157
158
159
160
161
162
            print(file=sys.stderr)


check_for_update()


163
164
165
166
167
168
169
170
def compiler_module(compiler_repo):
    return compiler_repo.replace("_", "/", 1).replace("_", ".")


def mpi_module(mpi_repo):
    return mpi_repo.replace("_", "/", 1).replace("_", ".")


171
def valid_pgi_mpi(pgi, mpi):
172
173
174
175
176
177
178
179
180
181
    if "impi" in mpi:
        if "2017" in mpi:
            return False
        else:
            return True

    if "openmpi_4" == mpi:
        return True

    return False
182
183


184
def valid_mpi(compiler, mpi):
185
186
187
188
189
190
191
192
193
    """
    It might be possible to use Intel MPI libararies and compilers from
    different Parallel Studio packages, but I currently do not want to support
    it.

    Take care to keep this in sync with the file 'macros.obs_cluster' of
    the package software:dist / mpcdf_cluster_macros
    """
    if compiler.startswith("intel") and mpi.startswith("impi"):
194
        return mpi == compiler_parallel_studio[compiler]["impi"]
195
196
    if compiler.startswith("pgi"):
        return valid_pgi_mpi(compiler, mpi)
197
198
199
200
201
202
203
204
205
    if compiler.startswith("gcc") and mpi.startswith("impi"):
        gcc_version = int(compiler[len("gcc_"):])
        impi_major_version, impi_minor_version = map(int, mpi[len("impi_"):].split("_"))
        if gcc_version >= 10:
            # gcc_10 only with modern Intel MPI
            return impi_major_version > 2019 or \
                (impi_major_version == 2019 and impi_minor_version >= 7)
        else:
            return True
206
207
    else:
        return True
208
209


Lorenz Hüdepohl's avatar
Lorenz Hüdepohl committed
210
211
212
213
214
215
216
217
def repo_tags(reponame, distribution=None):
    if "gcc_6" in reponame or "gcc_7" in reponame and distribution == "SLE_15":
        # gcc6/7 cannot deal with "-fstack-clash-protection" that has been added in SLE15 by default
        optflags = ("Optflags: * -fmessage-length=0 -grecord-gcc-switches -O2 -Wall -D_FORTIFY_SOURCE=2 -fstack-protector-strong -funwind-tables -fasynchronous-unwind-tables",)
    else:
        optflags = ()

    ps = None
218
    if reponame in compiler_parallel_studio:
Lorenz Hüdepohl's avatar
Lorenz Hüdepohl committed
219
        ps = compiler_parallel_studio[reponame]["ps"]
220
221
222
    else:
        for mpi in mpi_parallel_studio:
            if reponame.startswith(mpi):
Lorenz Hüdepohl's avatar
Lorenz Hüdepohl committed
223
                ps = mpi_parallel_studio[mpi]["ps"]
224

Lorenz Hüdepohl's avatar
Lorenz Hüdepohl committed
225
226
227
228
    if ps:
        preferred_mkl = "Prefer: " + intel_parallel_studio[ps]["mkl"]
        unprefer_other_mkls = sorted("Prefer: !" + mkl for mkl in all_mkls if mkl != preferred_mkl)
        prefers = ("Prefer: " + ps,) + tuple(unprefer_other_mkls) + (preferred_mkl,)
229
    else:
Lorenz Hüdepohl's avatar
Lorenz Hüdepohl committed
230
231
232
        prefers = ()

    return optflags + prefers
233
234


235
236
237
238
def valid_cuda(cuda, compiler):
    """
    The CUDA distribution only works with certain gcc versions,
    this little function takes care that only supported combinations
Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
239
240
241
242
    are allowed.

    Take care to keep this in sync with the file 'macros.obs_cluster' of
    the package software:dist / mpcdf_cluster_macros
243
    """
Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
244
    if cuda == "cuda_8_0":
245
        return compiler == "gcc_5"
Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
246
    if cuda == "cuda_9_1":
247
        return compiler == "gcc_6_3_0"
Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
248
    if cuda == "cuda_9_2":
Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
249
        return compiler == "gcc_6"
Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
250
251
    if cuda == "cuda_10_0":
        return compiler == "gcc_6"
Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
252
253
    if cuda == "cuda_10_1":
        return compiler == "gcc_8"
Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
254
255
    if cuda == "cuda_10_2":
        return compiler == "gcc_8"
Lorenz Hüdepohl's avatar
Lorenz Hüdepohl committed
256
257
    if cuda == "cuda_11_0":
        return compiler == "gcc_9"
Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
258
259
    if cuda == "cuda_11_2":
        return compiler == "gcc_10"
260
261
262
    return False


Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
263
264
265
def project_meta(api_url, project):
    return ElementTree.fromstringlist(osc.core.show_project_meta(api_url, project))

266

267
268
269
270
271
272
273
274
275
def package_meta(api_url, project, package):
    return ElementTree.fromstringlist(osc.core.show_package_meta(api_url, project, package))


def maintainers(api_url, project, package):
    root = package_meta(api_url, project, package)
    return {e.get("userid") for e in root.findall("./person[@role='maintainer']")}


276
277
278
279
class UnsetAttributeException(Exception):
    pass


280
281
282
283
class UnmanagedPackageException(Exception):
    pass


284
285
def chunked(items, chunksize):
    n = len(items)
286
287
    i = 0
    while i * chunksize < n:
288
        yield items[i * chunksize: (i + 1) * chunksize]
289
290
291
        i += 1


Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
292
def get_attribute(api_url, project, package, attribute, with_project=False):
293
294
    attribute_meta = osc.core.show_attribute_meta(api_url, project, package, None, attribute, False, with_project)
    if attribute_meta is None:
Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
295
        raise osc.oscerr.APIError("Cannot fetch value for attribute '{0}' from {1}".format(attribute, (project, package)))
296
297
298
299

    root = ElementTree.fromstringlist(attribute_meta)
    attribute = root.find("./attribute")
    if attribute is not None:
Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
300
        return root
301
    else:
302
        raise UnsetAttributeException("Attribute not set")
303

Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
304

305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
def overloaded_project_attribute(api_url, project, attribute):
    try:
        return get_attribute_values(api_url, project, None, attribute)
    except UnsetAttributeException:
        return get_attribute_values(api_url, "software", None, attribute)


def overloaded_package_attribute(api_url, project, package, attribute):
    try:
        return get_attribute_values(api_url, project, package, attribute)
    except UnsetAttributeException:
        pass

    try:
        return get_attribute_values(api_url, project, None, attribute)
    except UnsetAttributeException:
        pass

    return get_attribute_values(api_url, "software", None, attribute)


def get_allowed_attribute_values(api_url, attribute):
    path = ["attribute"] + attribute.split(":") + ["_meta"]
    url = osc.core.makeurl(api_url, path, [])
    try:
        f = osc.core.http_GET(url)
    except osc.core.HTTPError as e:
        e.osc_msg = 'Error getting meta for attribute "{0}"'.format(attribute)
        raise

    root = ElementTree.fromstringlist(f.readlines())
    return list(value.text for value in root.findall("./allowed/value"))


Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
339
340
341
342
343
344
def get_attribute_values(api_url, project, package, attribute, with_project=False):
    root = get_attribute(api_url, project, package, attribute, with_project)
    attribute = root.find("./attribute")
    return list(value.text for value in attribute.findall("./value"))


345
346
347
348
349
def get_attribute_value(api_url, project, package, attribute, with_project=False):
    value, = get_attribute_values(api_url, project, package, attribute, with_project=False)
    return value


350
351
352
353
354
355
def set_attribute(api_url, project, package, attribute):
    path = ["source", project]
    if package:
        path.append(package)
    path.append("_attribute")

Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
356
    attr_meta = ElementTree.tostring(attribute, encoding=osc.core.ET_ENCODING)
Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
357

358
    url = osc.core.makeurl(api_url, path)
Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
359
    resp = ElementTree.fromstringlist(osc.core.streamfile(url, osc.core.http_POST, data=attr_meta))
Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
360
    if resp.find("./summary").text != "Ok":
Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
361
        raise osc.oscerr.APIError("Could not store attribute")
Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
362

363

364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
def set_attribute_values(api_url, project, package, attribute, values):
    root = ElementTree.Element("attributes")
    attr = ElementTree.SubElement(root, "attribute")

    namespace, name = attribute.split(":")
    attr.set("namespace", namespace)
    attr.set("name", name)

    for value in values:
        v = ElementTree.SubElement(attr, "value")
        v.text = value

    set_attribute(api_url, project, package, root)


def has_attribute(api_url, project, package, attribute):
    path = ["source", project]
    if package:
        path.append(package)
    path.append("_attribute")
    path.append(attribute)

    url = osc.core.makeurl(api_url, path)
    resp = ElementTree.fromstringlist(osc.core.streamfile(url, osc.core.http_GET))

    namespace, name = attribute.split(":")
390

391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
    for a in resp.findall("./attribute"):
        if a.get("namespace") == namespace and a.get("name") == name:
            return True
    return False


def remove_attribute(api_url, project, package, attribute_name):
    path = ["source"] + [project]
    if package:
        path.append(package)
    path.append("_attribute")
    path.append(attribute_name)

    url = osc.core.makeurl(api_url, path)
    resp = ElementTree.fromstringlist(osc.core.streamfile(url, osc.core.http_DELETE))
    if resp.find("./summary").text != "Ok":
        raise osc.oscerr.APIError("Could not remove attribute")


410
def get_microarchitecture(project):
411
    if project.startswith("software:"):
412
        microarch = project.split(":")[2]
413
414
415
416
    elif project.startswith("home:"):
        microarch = "sandybridge"
    else:
        raise Exception("Cannot determine micro-architecture for project '{0}'".format(project))
417
418
419
420
421
422
423

    if microarch in known_microarchs:
        return microarch
    else:
        raise Exception("Unknown micro-architecture '{0}'".format(microarch))


424
425
426
427
428
429
def package_sort_key(string):
    name, version = string.split("_", 1)
    version = version.split("_")
    return (name,) + tuple(map(int, version))


430
def mpcdf_enable_repositories(api_url, project, package, verbose=False, dry_run=False, ignore_repos=()):
431
432
    from itertools import product

433
434
435
    pkg_meta = osc.core.show_package_meta(api_url, project, package)
    root = ElementTree.fromstringlist(pkg_meta)

436
    build = root.find("./build")
Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
437
438
    if build is None:
        build = ElementTree.SubElement(root, "build")
439
440
441
442
443

    pkg_meta = ElementTree.tostring(root, encoding=osc.core.ET_ENCODING)

    for enable in build.findall("./enable"):
        build.remove(enable)
444

445
446
    disabled_repos = {disable.get("repository") for disable in build.findall("./disable")}

Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
447
    try:
448
449
450
451
        enable_repos = get_attribute_values(api_url, project, package, "MPCDF:enable_repositories")
    except Exception:
        if verbose:
            print("Warning: Could not get attribute MPCDF:enable_repositories for package {0}, skipping".format(package))
452
        raise UnmanagedPackageException()
Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
453

454
455
    if project == "software":
        distributions = list(sorted(repo.name for repo in osc.core.get_repos_of_project(api_url, "software")))
Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
456

457
458
459
460
461
    else:
        compilers = overloaded_package_attribute(api_url, project, package, "MPCDF:compiler_modules")
        mpis = overloaded_package_attribute(api_url, project, package, "MPCDF:mpi_modules")
        cudas = overloaded_package_attribute(api_url, project, package, "MPCDF:cuda_modules")
        pgis = overloaded_package_attribute(api_url, project, package, "MPCDF:pgi_modules")
462
        openmpi_flavors = overloaded_package_attribute(api_url, project, package, "MPCDF:openmpi_flavors")
463

464
465
466
467
        all_compilers = overloaded_project_attribute(api_url, project, "MPCDF:compiler_modules")
        all_mpis = overloaded_project_attribute(api_url, project, "MPCDF:mpi_modules")
        all_cudas = overloaded_project_attribute(api_url, project, "MPCDF:cuda_modules")
        all_pgis = overloaded_project_attribute(api_url, project, "MPCDF:pgi_modules")
468
        all_openmpi_flavors = overloaded_project_attribute(api_url, project, "MPCDF:openmpi_flavors")
469

470
471
472
        default_compilers = overloaded_project_attribute(api_url, project, "MPCDF:default_compiler")
        default_mpis = overloaded_project_attribute(api_url, project, "MPCDF:default_mpi")
        default_cudas = overloaded_project_attribute(api_url, project, "MPCDF:default_cuda")
473

474
475
        latest_intel = sorted((c for c in all_compilers if c.startswith("intel")), key=package_sort_key)[-1]
        latest_gcc = sorted((c for c in all_compilers if c.startswith("gcc")), key=package_sort_key)[-1]
476

477
478
    i = len(build)

479
    def enable(name):
480
        if name in ignore_repos or name in disabled_repos:
481
            return
482
483
484
        node = ElementTree.Element("enable")
        node.set("repository", name)
        node.tail = "\n    "
485
        build.insert(i, node)
486
487
488
        if verbose:
            print("Enabling", name)

489
    def actual_compilers():
490
        for compiler in (c for c in compilers if c in all_compilers + ["default_compiler", "intel", "gcc", "latest_intel", "latest_gcc"]):
491
            if compiler == "intel":
Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
492
                for intel_compiler in [cc for cc in all_compilers if cc.startswith("intel")]:
493
494
                    yield intel_compiler
            elif compiler == "gcc":
Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
495
                for gcc_compiler in [cc for cc in all_compilers if cc.startswith("gcc")]:
496
497
498
499
                    yield gcc_compiler
            elif compiler == "default_compiler":
                for default_compiler in default_compilers:
                    yield default_compiler
500
501
502
503
            elif compiler == "latest_intel":
                yield latest_intel
            elif compiler == "latest_gcc":
                yield latest_gcc
504
505
506
507
            else:
                yield compiler

    def actual_mpis():
508
        for mpi in (m for m in mpis if m in all_mpis + ["default_mpi", "impi"]):
509
            if mpi == "impi":
510
                for impi in [mpi for mpi in all_mpis if mpi.startswith("impi")]:
511
512
513
514
515
516
517
518
                    yield impi
            elif mpi == "default_mpi":
                for default_mpi in default_mpis:
                    yield default_mpi
            else:
                yield mpi

    def actual_cudas():
519
        for cuda in (c for c in cudas if c in all_cudas + ["default_cuda"]):
520
521
522
523
524
525
            if cuda == "default_cuda":
                for default_cuda in default_cudas:
                    yield default_cuda
            else:
                yield cuda

526
527
528
529
    def actual_pgis():
        for pgi in (p for p in pgis if p in all_pgis):
            yield pgi

530
531
532
533
    def actual_openmpi_flavors():
        for of in (f for f in openmpi_flavors if f in all_openmpi_flavors):
            yield of

534
    for flag in enable_repos:
535
536
        if flag == "system":
            if project == "software":
537
538
                for distribution in distributions:
                    enable(distribution)
539
540
            elif project.startswith("home:"):
                enable("System")
541

542
        elif project != "software":
543
544
545
546
547
548
549
550
551
            if flag == "compilers":
                for compiler in actual_compilers():
                    enable(compiler)

            if flag == "mpi":
                for mpi, compiler in product(actual_mpis(), actual_compilers()):
                    if valid_mpi(compiler, mpi):
                        enable(mpi + "_" + compiler)

552
553
554
555
556
557
            if flag == "openmpi_flavors":
                for mpi, compiler in product(actual_mpis(), actual_compilers()):
                    if "openmpi" in mpi and valid_mpi(compiler, mpi):
                        for of in actual_openmpi_flavors():
                            enable(mpi + "_" + compiler + "_" + of)

558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
            if flag == "cuda":
                for cuda, compiler in product(actual_cudas(), all_compilers):
                    if valid_cuda(cuda, compiler):
                        enable(cuda + "_" + compiler)

            if flag == "cuda_mpi":
                for cuda, mpi, compiler in product(actual_cudas(), actual_mpis(), all_compilers):
                    if valid_cuda(cuda, compiler) and valid_mpi(compiler, mpi):
                        enable(cuda + "_" + mpi + "_" + compiler)

            if flag == "pgi":
                for pgi in actual_pgis():
                    enable(pgi)

            if flag == "pgi_mpi":
                for mpi, pgi in product(actual_mpis(), actual_pgis()):
                    if valid_pgi_mpi(pgi, mpi):
                        enable(mpi + "_" + pgi)
576

577
578
579
580
581
            if flag == "cuda_aware_mpi":
                for cuda, mpi, compiler in product(actual_cudas(), actual_mpis(), all_compilers):
                    if valid_cuda(cuda, compiler) and valid_mpi(compiler, mpi):
                        enable(cuda + "_aware_" + mpi + "_" + compiler)

Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
582
583
    if len(build.getchildren()) > 0:
        build.getchildren()[-1].tail = "\n  "
584

585
586
    new_pkg_meta = ElementTree.tostring(root, encoding=osc.core.ET_ENCODING)
    if pkg_meta != new_pkg_meta:
587
        print("Updating repositories for", package)
588
589
590
        if dry_run:
            print("osc meta pkg {0} {1} -F - <<EOF\n{2}\nEOF\n".format(project, package, new_pkg_meta))
        else:
591
            osc.core.edit_meta("pkg", (project, package), data=new_pkg_meta, apiurl=api_url)
592
593
    elif dry_run:
        print("Would not do anything, package meta would be unchanged")
594
595

    return True
Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
596
597


598
599
600
601
602
603
604
605
def parse_prjconf(api_url, project):
    orig_prjconf = list(map(decode_it, osc.core.show_project_conf(api_url, project)))
    try:
        start = orig_prjconf.index(prjconf_start_marker)
        end = orig_prjconf.index(prjconf_end_marker)
    except ValueError:
        start = None
        end = len(orig_prjconf)
606

607
608
609
610
611
612
613
    prjconf_head = orig_prjconf[:start]
    prjconf_ours = orig_prjconf[start:end]
    prjconf_tail = orig_prjconf[end + 1:]
    return orig_prjconf, prjconf_head, prjconf_ours, prjconf_tail


def mpcdf_setup_subproject(api_url, project, distribution, microarchitecture,
614
                           parent=None, dry_run=False, remove_old=False, all_possible=False, only_project=False):
615
616

    if parent and not dry_run:
617
618
        for attribute in config_attributes + default_attributes:
            print("Copying attribute '{0}' from parent project".format(attribute))
619
            set_attribute_values(api_url, project, None, attribute, overloaded_project_attribute(api_url, parent, attribute))
Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
620

621
622
623
    # Check distribution
    software_meta = project_meta(api_url, "software")
    dist_repo = software_meta.find('./repository[@name="{0}"]'.format(distribution))
Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
624
    if dist_repo is None:
625
        raise osc.oscerr.WrongArgs("Invalid distribution '{0}': No matching repository is defined in project 'software' on the server".format(distribution))
Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
626
627
628
629
    architectures = list(arch.text for arch in dist_repo.findall("./arch"))

    root = project_meta(api_url, project)

630
631
632
633
634
635
636
    orig_prjconf, prjconf_head, prjconf_ours, prjconf_tail = parse_prjconf(api_url, project)

    prjconf_repos = {}
    cur_repo = None
    old_repos = set()
    for line in prjconf_ours:
        line = line.rstrip("\n")
Lorenz Hüdepohl's avatar
Lorenz Hüdepohl committed
637
638
        if cur_repo is None and (line.startswith("%if %_repository ==") or line.startswith("%if \"%{_repository}\" == ")) and len(line.split()) == 4:
            cur_repo = line.split()[-1].strip('"')
639
640
641
642
643
644
645
646
647
648
            old_repos.add(cur_repo)
            prjconf_repos[cur_repo] = []
        if cur_repo is not None:
            prjconf_repos[cur_repo].append(line)
            if line == "%endif":
                cur_repo = None

    prjconf_ours = [prjconf_start_marker]

    prjconf_ours.append("Constraint: hostlabel {0}".format(microarchitecture))
Lorenz Hüdepohl's avatar
Lorenz Hüdepohl committed
649
    prjconf_ours.append("Preinstall: mpcdf_{0}_directory".format(microarchitecture))
650
    prjconf_ours.append("PublishFilter: ^mpcdf_.*$")
Lorenz Hüdepohl's avatar
Lorenz Hüdepohl committed
651
    prjconf_ours.append("")
652

Lorenz Hüdepohl's avatar
Lorenz Hüdepohl committed
653
654
655
656
    extra_tags = dist_prjconf_tags(distribution)
    if extra_tags:
        prjconf_ours.append(extra_tags)
    extra_macros = dist_prjconf_macros(distribution)
657

658
    prjconf_ours.append("""
659
660
Macros:
%microarchitecture {0}
661
662
{1}
:Macros""".format(microarchitecture, extra_macros))
663
    prjconf_ours.append("")
664

Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
665
    # Remove existing repositories
666
667
668
    if remove_old:
        for oldrepo in root.findall("./repository"):
            root.remove(oldrepo)
Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
669

670
    def repo(name, dependencies, compiler=False, mpi=False, cuda=False, cuda_mpi=False, cuda_aware_mpi=False, additional_tags=(), **macros):
671
        old_repos.discard(name)
672
673
674
        have_compiler = compiler or mpi or cuda or cuda_mpi or cuda_aware_mpi
        have_mpi = mpi or cuda_mpi or cuda_aware_mpi
        have_cuda = cuda or cuda_mpi or cuda_aware_mpi
675

676
677
678
        existing_repo = root.find("./repository[@name='{0}']".format(name))
        if existing_repo is not None:
            root.remove(existing_repo)
679
        elif dry_run is False:
680
681
            print("New repository", name)

Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
682
683
        r = ElementTree.SubElement(root, "repository")
        r.set("name", name)
Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
684
685
        r.text = "\n    "

Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
686
687
688
689
        def path(project, repo):
            p = ElementTree.SubElement(r, "path")
            p.set("project", project)
            p.set("repository", repo)
Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
690
            p.tail = "\n    "
Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
691

692
693
        for dep_project, dep_repo in dependencies:
            path(dep_project, dep_repo)
Lorenz Hüdepohl's avatar
Lorenz Hüdepohl committed
694
695
        if parent and name != "System":
            path(parent, name)
Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
696
697
698
699
700
701
702
703

        for arch in architectures:
            a = ElementTree.SubElement(r, "arch")
            a.text = arch
            a.tail = "\n    "
        a.tail = "\n  "
        r.tail = "\n  "

Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
704
705
706
707
        # In order to be able to figure out the matching MKL version for a given
        # compiler/MPI repository we emit a new macro '%matching_mkl_version' in
        # the cases this makes sense
        matching_mkl = []
708
        repoconf = []
Lorenz Hüdepohl's avatar
Lorenz Hüdepohl committed
709
710
711
712
713
        repoconf.append("%if \"%{{_repository}}\" == \"{0}\"".format(name))
        for tag in repo_tags(name, distribution=distribution) + additional_tags:
            if tag.startswith("Prefer: mkl"):
                matching_mkl.append(tag.split()[1])
            repoconf.append(tag)
Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
714

715
        repoconf.append("Macros:")
716

717
718
719
720
        repoconf.append("%is_compiler_repository {0}".format(1 if compiler else 0))
        repoconf.append("%is_mpi_repository {0}".format(1 if mpi else 0))
        repoconf.append("%is_cuda_repository {0}".format(1 if cuda else 0))
        repoconf.append("%is_cuda_mpi_repository {0}".format(1 if cuda_mpi else 0))
721
        repoconf.append("%is_cuda_aware_mpi_repository {0}".format(1 if cuda_aware_mpi else 0))
722

723
724
725
        repoconf.append("%have_mpcdf_compiler {0}".format(1 if have_compiler else 0))
        repoconf.append("%have_mpcdf_mpi {0}".format(1 if have_mpi else 0))
        repoconf.append("%have_mpcdf_cuda {0}".format(1 if have_cuda else 0))
726

Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
727
728
729
730
        if matching_mkl:
            matching_mkl, = matching_mkl
            matching_mkl, _ = matching_mkl[len("mkl_"):].split("-module")
            matching_mkl = matching_mkl.replace("_", ".")
731
            repoconf.append("%matching_mkl_version {0}".format(matching_mkl))
Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
732

733
734
        for macro, value in macros.items():
            repoconf.append("%{0} {1}".format(macro, value))
735

736
737
738
        repoconf.append(":Macros")
        repoconf.append("%endif")
        prjconf_repos[name] = repoconf
Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
739

740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
    def actual_compiler(c):
        if c.startswith("latest"):
            return False
        if c == "default_compiler":
            return False
        if c == "intel":
            return False
        if c == "gcc":
            return False
        return True

    def actual_mpi(m):
        if m == "default_mpi":
            return False
        if m == "impi":
            return False
        return True

    def actual_cuda(c):
        if c == "default_cuda":
            return False
        return True

    if all_possible:
        compilers = list(filter(actual_compiler, get_allowed_attribute_values(api_url, "MPCDF:compiler_modules")))
        mpis = list(filter(actual_mpi, get_allowed_attribute_values(api_url, "MPCDF:mpi_modules")))
        cudas = list(filter(actual_cuda, get_allowed_attribute_values(api_url, "MPCDF:cuda_modules")))
        pgis = get_allowed_attribute_values(api_url, "MPCDF:pgi_modules")
768
        openmpi_flavors = get_allowed_attribute_values(api_url, "MPCDF:openmpi_flavors")
Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
769
    else:
770
771
772
773
        compilers = overloaded_project_attribute(api_url, project, "MPCDF:compiler_modules")
        mpis = overloaded_project_attribute(api_url, project, "MPCDF:mpi_modules")
        cudas = overloaded_project_attribute(api_url, project, "MPCDF:cuda_modules")
        pgis = overloaded_project_attribute(api_url, project, "MPCDF:pgi_modules")
774
        openmpi_flavors = overloaded_project_attribute(api_url, project, "MPCDF:openmpi_flavors")
775
776

    if parent:
Lorenz Hüdepohl's avatar
Lorenz Hüdepohl committed
777
        repo("System", (("software", distribution),))
Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
778

779
    for compiler in compilers + pgis:
Lorenz Hüdepohl's avatar
Lorenz Hüdepohl committed
780
781
782
        if project.startswith("home:"):
            dependencies = ((project, "System"),)
        elif project != "software":
Lorenz Hüdepohl's avatar
Lorenz Hüdepohl committed
783
            dependencies = (("software", distribution),)
784
785
786
787
        else:
            dependencies = ()
        repo(compiler, dependencies, compiler=True,
             compiler_repository=compiler, compiler_module=compiler_module(compiler),
Lorenz Hüdepohl's avatar
Lorenz Hüdepohl committed
788
             additional_tags=("Prefer: mpcdf_compiler_" + compiler,))
Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
789
790

        for mpi in filter(partial(valid_mpi, compiler), mpis):
Lorenz Hüdepohl's avatar
Lorenz Hüdepohl committed
791
            repo(mpi + "_" + compiler, ((project, compiler),), mpi=True,
792
                 mpi_repository=mpi, mpi_module=mpi_module(mpi))
Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
793

794
795
796
797
798
799
800
801
802
803
            if "openmpi" in mpi:
                for of in openmpi_flavors:
                    dependencies = ((project, compiler),)
                    if "mofed" in of:
                        dependencies = (("extern:" + of, distribution),) + dependencies

                    repo(mpi + "_" + compiler + "_" + of, dependencies, mpi=True,
                         mpi_repository=mpi, mpi_module=mpi_module(mpi),
                         openmpi_flavor=of)

Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
804
805
    for cuda in cudas:
        for compiler in filter(partial(valid_cuda, cuda), compilers):
Lorenz Hüdepohl's avatar
Lorenz Hüdepohl committed
806
            repo(cuda + "_" + compiler, ((project, compiler),), cuda=True, cuda_repository=cuda,
Lorenz Hüdepohl's avatar
Lorenz Hüdepohl committed
807
                 additional_tags=("Prefer: mpcdf_" + cuda,))
Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
808
809
            for mpi in filter(partial(valid_mpi, compiler), mpis):
                repo(cuda + "_" + mpi + "_" + compiler,
Lorenz Hüdepohl's avatar
Lorenz Hüdepohl committed
810
811
                     ((project, cuda + "_" + compiler),
                      (project, mpi + "_" + compiler)),
812
                     cuda_mpi=True)
Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
813

814
815
816
817
                repo(cuda + "_aware_" + mpi + "_" + compiler,
                     ((project, cuda + "_" + mpi + "_" + compiler),),
                     cuda_aware_mpi=True)

818
819
820
821
822
823
824
825
826
    if old_repos and not remove_old:
        print("Warning: Keeping the prjconf sections for the following obsolete repositories:")
        for name in sorted(old_repos):
            print(" -", name)
        print()
    else:
        for old_repo in old_repos:
            del prjconf_repos[old_repo]

Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
827
828
829
830
831
832
833
834
835
836
837
    # Remove build configuration
    build = root.find("./build")
    if build is None:
        build = ElementTree.Element("build")
        build.text = "\n    "
        disable = ElementTree.SubElement(build, "disable")
        disable.tail = "\n  "
        build.tail = "\n  "
        root.insert(list(root).index(root.find("./repository")), build)

    root.getchildren()[-1].tail = "\n"
838
    new_prj = ElementTree.tostring(root, encoding=osc.core.ET_ENCODING)
Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
839

840
841
842
    for name in sorted(prjconf_repos.keys()):
        prjconf_ours.extend(prjconf_repos[name])
        prjconf_ours.append("")
Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
843

844
    prjconf_ours.append(prjconf_end_marker)
845

846
    new_prjconf = "".join(prjconf_head) + "\n".join(prjconf_ours) + "".join(prjconf_tail)
847

848
849
850
851
852
853
854
855
    if dry_run:
        print("osc meta prjconf {0} -F - <<EOF\n{1}\nEOF\n".format(project, new_prjconf))
        print("osc meta prj {0} -F - <<EOF\n{1}\nEOF\n".format(project, new_prj))
    else:
        if new_prjconf == "".join(orig_prjconf):
            print("prjconf unchanged")
        else:
            print("Updating prjconf meta")
856
            osc.core.edit_meta("prjconf", project, data=new_prjconf, apiurl=api_url)
857

858
859
        # Create and remove the <enable/> flags before the new repsitories,
        # that way no spurious builds are launched
860
861
862
        if not only_project:
            print("Updating enabled repositories for all packages")
            mpcdf_enable_repositories_for_all_packages(api_url, project)
Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
863
864
865

        # Update repositories
        print("Updating prj meta")
866
        osc.core.edit_meta("prj", project, data=new_prj, force=True, apiurl=api_url)
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892


def mpcdf_enable_repositories_for_all_packages(api_url, project, ignore_repos=()):
    import threading
    packages = osc.core.meta_get_packagelist(api_url, project)

    if len(packages) > 40:
        chunksize = len(packages) // 20
    else:
        chunksize = len(packages)

    def work(packagelist):
        for package in packagelist:
            try:
                mpcdf_enable_repositories(api_url, project, package, ignore_repos=ignore_repos)
            except UnmanagedPackageException:
                print("ATTENTION: Not changing unmanaged package {0}".format(package))

    threads = []
    for packagelist in chunked(packages, chunksize):
        t = threading.Thread(target=work, args=(packagelist,))
        threads.append(t)
        t.start()

    for t in threads:
        t.join()
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932


def set_as_branch(api_url, my_project, my_package, main_project, main_package):
    import os

    def dirmeta(project, package):
        url = osc.core.makeurl(api_url, ["source", project, package])
        return ElementTree.fromstringlist(osc.core.streamfile(url, osc.core.http_GET))

    srcmeta = dirmeta(my_project, my_package)
    dstmeta = dirmeta(main_project, main_package)

    linkinfo = srcmeta.find('./linkinfo')
    if linkinfo is not None:
        print("ERROR: package {0} is already linked to package {1} in project {2}".format(
              my_package, linkinfo.get("package"), linkinfo.get("project")))
        return False

    linkfile = ElementTree.Element("link")
    linkfile.set("project", main_project)
    linkfile.set("package", main_package)
    linkfile.set("baserev", dstmeta.get("srcmd5"))
    patches = ElementTree.SubElement(linkfile, "patches")
    ElementTree.SubElement(patches, "branch")

    url = osc.core.makeurl(api_url, ["source", my_project, my_package, "_link"])
    resp = ElementTree.fromstringlist(osc.core.streamfile(url, osc.core.http_PUT, data=ElementTree.tostring(linkfile)))
    rev = resp.get("rev")
    if rev is None:
        print("ERROR: Could not commit _link file")
        return False
    else:
        print("Commited as revision", rev)
        this_package = osc.core.store_read_package(os.curdir)
        this_project = osc.core.store_read_project(os.curdir)
        if this_package == my_package and this_project == my_project:
            pac = osc.core.filedir_to_pac(os.curdir)
            rev = pac.latest_rev(expand=True)
            pac.update(rev)
    return True
933
934


Lorenz Hüdepohl's avatar
Lorenz Hüdepohl committed
935
def sync_projects(api_url, package=None, from_project="software", to_projects=None, add_to_maintainers=True, verbose=False):
936
    if to_projects is None:
Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
937
938
        to_projects = [p for p in osc.core.meta_get_project_list(api_url)
                       if p.startswith("software:") and not (p == "software:dist" or p == "software:images")]
939

940
941
942
943
    allowed_attribute_values = {}
    for attribute in package_attributes + config_attributes:
        allowed_attribute_values[attribute] = set(get_allowed_attribute_values(api_url, attribute))

944
945
946
947
948
949
950
951
952
953
954
    for to_project in to_projects:
        print("Syncing {0} with {1}".format(to_project, from_project))

        to_packages = osc.core.meta_get_packagelist(api_url, to_project)

        if package is None:
            from_packages = osc.core.meta_get_packagelist(api_url, from_project)
        else:
            from_packages = [package]

        for orig_package in from_packages:
955
            enable_repos = get_attribute_values(api_url, from_project, orig_package, "MPCDF:enable_repositories")
956
            if orig_package not in to_packages:
957
                if len(enable_repos) == 0:
958
                    print("Not branching package {0}, is disabled".format(orig_package))
959
                    continue
960
                elif enable_repos == ["system"] and "System" not in osc.core.get_repositories_of_project(api_url, to_project):
961
                    print("Not branching package {0}, is only enabled for 'system'".format(orig_package))
962
                    continue
963
964
965
966
967
968
969
970
971
972
                filelist = osc.core.meta_get_filelist(api_url, from_project, orig_package)
                if "_link" in filelist:
                    print("Not branching package {0}, is a link".format(orig_package))
                else:
                    print("Branching package {0}".format(orig_package))
                    exists, targetprj, targetpkg, srcprj, srcpkg = \
                        osc.core.branch_pkg(api_url, from_project, orig_package, target_project=to_project, nodevelproject=True)
            else:
                print("Not branching package {0}, already present in target".format(orig_package))

973
            for attribute in package_attributes + config_attributes:
974
                try:
975
976
                    values = list(filter(lambda q: q in allowed_attribute_values[attribute],
                                         get_attribute_values(api_url, from_project, orig_package, attribute)))
977
978
979
980
                except UnsetAttributeException:
                    if has_attribute(api_url, to_project, orig_package, attribute):
                        remove_attribute(api_url, to_project, orig_package, attribute)
                    continue
981
                set_attribute_values(api_url, to_project, orig_package, attribute, values)
982

983
984
985
986
987
988
989
990
991
992
993
994
            if add_to_maintainers:
                from_maintainers = maintainers(api_url, from_project, orig_package)
                if from_maintainers:
                    to_maintainers = maintainers(api_url, to_project, orig_package)
                    if from_maintainers - to_maintainers:
                        new_maintainers = from_maintainers - to_maintainers
                        to_meta = package_meta(api_url, to_project, orig_package)
                        for userid in sorted(new_maintainers):
                            person = ElementTree.Element("person")
                            person.set("userid", userid)
                            person.set("role", "maintainer")
                            to_meta.insert(2, person)
995
                        osc.core.edit_meta("pkg", (to_project, orig_package), data=ElementTree.tostring(to_meta), apiurl=api_url)
996

997
998
999
1000
            try:
                mpcdf_enable_repositories(api_url, to_project, orig_package, verbose=verbose)
            except UnmanagedPackageException:
                print("ATTENTION: Not changing unmanaged package {0}".format(orig_package))