mpcdf_common.py

from __future__ import print_function
from __future__ import division

import sys
import osc
import osc.conf
import osc.core
import osc.oscerr
import textwrap
from functools import partial
from xml.etree import ElementTree

if sys.version_info[0] < 3:
    def decode_it(arg):
        return arg
else:
    from osc.util.helper import decode_it


known_microarchs = {"sandybridge", "haswell", "skylake"}

package_attributes = ["MPCDF:enable_repositories"]
config_attributes = ["MPCDF:compiler_modules", "MPCDF:cuda_modules", "MPCDF:mpi_modules", "MPCDF:pgi_modules", "MPCDF:openmpi_flavors"]

intel_parallel_studio = {
    "mpcdf_intel_parallel_studio_2017_7": {"compiler": "intel_17_0_7", "impi": "impi_2017_4", "mkl": "mkl_2017_4-module", },
    "mpcdf_intel_parallel_studio_2018_1": {"compiler": "intel_18_0_1", "impi": "impi_2018_1", "mkl": "mkl_2018_1-module", },
    "mpcdf_intel_parallel_studio_2018_2": {"compiler": "intel_18_0_2", "impi": "impi_2018_2", "mkl": "mkl_2018_2-module", },
    "mpcdf_intel_parallel_studio_2018_3": {"compiler": "intel_18_0_3", "impi": "impi_2018_3", "mkl": "mkl_2018_3-module", },
    "mpcdf_intel_parallel_studio_2018_4": {"compiler": "intel_18_0_5", "impi": "impi_2018_4", "mkl": "mkl_2018_4-module", },
    "mpcdf_intel_parallel_studio_2019_0": {"compiler": "intel_19_0_0", "impi": "impi_2019_0", "mkl": "mkl_2019_0-module", },
    "mpcdf_intel_parallel_studio_2019_1": {"compiler": "intel_19_0_1", "impi": "impi_2019_1", "mkl": "mkl_2019_1-module", },
    "mpcdf_intel_parallel_studio_2019_3": {"compiler": "intel_19_0_3", "impi": "impi_2019_3", "mkl": "mkl_2019_3-module", },
    "mpcdf_intel_parallel_studio_2019_4": {"compiler": "intel_19_0_4", "impi": "impi_2019_4", "mkl": "mkl_2019_4-module", },
    "mpcdf_intel_parallel_studio_2019_5": {"compiler": "intel_19_0_5", "impi": "impi_2019_5", "mkl": "mkl_2019_5-module", },
    "mpcdf_intel_parallel_studio_2020": {"compiler": "intel_19_1_0", "impi": "impi_2019_6", "mkl": "mkl_2020-module", },
    "mpcdf_intel_parallel_studio_2020_1": {"compiler": "intel_19_1_1", "impi": "impi_2019_7", "mkl": "mkl_2020_1-module", },
    "mpcdf_intel_parallel_studio_2020_2": {"compiler": "intel_19_1_2", "impi": "impi_2019_8", "mkl": "mkl_2020_2-module", },
    "mpcdf_intel_parallel_studio_2020_4": {"compiler": "intel_19_1_3", "impi": "impi_2019_9", "mkl": "mkl_2020_4-module", },
    "mpcdf_intel_oneapi_2021_2": {"compiler": "intel_2021_2_0", "impi": "impi_2021_2", "mkl": "mkl_2021_2-module", },
    "mpcdf_intel_oneapi_2021_3": {"compiler": "intel_2021_3_0", "impi": "impi_2021_3", "mkl": "mkl_2021_3-module", },
}

all_mkls = {ic["mkl"] for ic in intel_parallel_studio.values()}

# Some rotations
mpi_parallel_studio = {value["impi"]: dict({"ps": key}, **value) for key, value in intel_parallel_studio.items()}
compiler_parallel_studio = {value["compiler"]: dict({"ps": key}, **value) for key, value in intel_parallel_studio.items()}


# For the autogenerated software/software:*:* project 'prjconf' sections
prjconf_start_marker = "# Autogenerated by osc mpcdf_setup_repos, do not edit till end of section\n"
prjconf_end_marker = "# End of autogenerated section\n"


def dist_prjconf_tags(distribution):
    centos_prjconf_tags = textwrap.dedent(
        """
        Prefer: perl-Error
        Support: post-build-checks
        Substitute: c_compiler gcc
        Substitute: c++_compiler gcc-c++
        Substitute: ca-certificates-mozilla ca-certificates
        """).strip()

    centos8_prjconf_tags = textwrap.dedent(
        """
        Preinstall: libzstd
        ExpandFlags: module:python36-3.6
        # Remove redhat-hardened-cc1 and redhat-annobin-cc1 spec
        Optflags: x86_64 -O2 -g -pipe -Wall -Werror=format-security -Wp,-D_FORTIFY_SOURCE=2 -Wp,-D_GLIBCXX_ASSERTIONS -fexceptions -fstack-protector-strong -grecord-gcc-switches -m64 -fasynchronous-unwind-tables -fstack-clash-protection -fcf-protection
        """).strip()

    res = ""
    if "CentOS" in distribution:
        res += centos_prjconf_tags
    if "CentOS_8" in distribution:
        res += "\n" + centos8_prjconf_tags

    return res


def dist_prjconf_macros(distribution):
    centos_macros = textwrap.dedent(
        """
        # Disable all problematic automatic RPM stuff
        # like byte-compiling (with the wrong Python version)
        # or debug packages that fail for many binary-only packages
        %__no_python_bytecompile 1
        %debug_package %{nil}
        """).strip()

    res = ""
    if "CentOS" in distribution:
        res += centos_macros

    return res


def check_for_update():
    import os
    import sys
    import time
    from subprocess import check_output, call

    DEVNULL = open(os.devnull, 'w')

    if hasattr(sys.modules["mpcdf_common"], "checked_for_updates"):
        return
    else:
        setattr(sys.modules["mpcdf_common"], "checked_for_updates", 1)

    if (sys.version_info > (3, 0)):
        set_encoding = {"encoding": "utf-8"}
    else:
        set_encoding = {}

    plugin_dir = os.path.dirname(os.path.realpath(__file__))
    git_dir = os.path.join(plugin_dir, ".git")
    local_rev = check_output(["git", "--git-dir", git_dir, "rev-parse", "HEAD"], **set_encoding).strip()

    url = "https://gitlab.mpcdf.mpg.de/mpcdf/obs/osc-plugins.git"

    def update_server_rev():
        server_rev, _ = check_output(["git", "ls-remote", url, "master"], **set_encoding).split(None, 1)
        with open(rev_file, "w") as fd:
            fd.write(server_rev)

    # Check for update on server just once a day
    rev_file = os.path.join(plugin_dir, ".remote_head_rev")
    try:
        mtime = os.stat(rev_file).st_mtime
    except EnvironmentError:
        mtime = 0

    if time.time() - mtime > 86400:
        update_server_rev()

    with open(rev_file, "r") as fd:
        server_rev = fd.read().strip()

    if server_rev != local_rev:
        if call(["git", "--git-dir", git_dir, "merge-base", "--is-ancestor", server_rev, "HEAD"], stderr=DEVNULL) == 0:
            # Server rev is older than ours. Check again
            update_server_rev()

    with open(rev_file, "r") as fd:
        server_rev = fd.read().strip()

    if server_rev != local_rev:
        print("Note from MPCDF osc plugins:", file=sys.stderr)
        if call(["git", "--git-dir", git_dir, "merge-base", "--is-ancestor", server_rev, "HEAD"], stderr=DEVNULL) == 0:
            print(" You have unpushed commits in", plugin_dir, "- consider pushing them", file=sys.stderr)
            print(file=sys.stderr)
        else:
            print(" Your plugin directory is out-of-date, new commits available on", url, file=sys.stderr)
            print(" (careful: the repo has been moved, update your path to point to the new URL)", file=sys.stderr)
            print(file=sys.stderr)


check_for_update()


def compiler_module(compiler_repo):
    # special case: long and short version for intel 21.2.0
    if compiler_repo == "intel_2021_2_0":
        return "intel/21.2.0"
    elif compiler_repo == "intel_2021_3_0":
        return "intel/21.3.0"
    else:
        return compiler_repo.replace("_", "/", 1).replace("_", ".")


def mpi_module(mpi_repo):
    return mpi_repo.replace("_", "/", 1).replace("_", ".")


def valid_pgi_mpi(pgi, mpi):
    if "impi" in mpi:
        if "2017" in mpi:
            return False
        else:
            return True

    if "openmpi_4" == mpi:
        return True

    return False


def valid_mpi(compiler, mpi):
    """
    It might be possible to use Intel MPI libararies and compilers from
    different Parallel Studio packages, but I currently do not want to support
    it.

    Take care to keep this in sync with the file 'macros.obs_cluster' of
    the package software:dist / mpcdf_cluster_macros
    """
    if compiler.startswith("intel") and mpi.startswith("impi"):
        return mpi == compiler_parallel_studio[compiler]["impi"]
    if compiler.startswith("pgi"):
        return valid_pgi_mpi(compiler, mpi)
    if compiler.startswith("gcc") and mpi.startswith("impi"):
        gcc_version = int(compiler[len("gcc_"):])
        impi_major_version, impi_minor_version = map(int, mpi[len("impi_"):].split("_"))
        if gcc_version >= 10:
            # gcc_10 only with modern Intel MPI
            return impi_major_version > 2019 or \
                (impi_major_version == 2019 and impi_minor_version >= 7)
        else:
            return True
    else:
        return True


def repo_tags(reponame, distribution=None):
    if "gcc_6" in reponame or "gcc_7" in reponame and distribution == "SLE_15":
        # gcc6/7 cannot deal with "-fstack-clash-protection" that has been added in SLE15 by default
        optflags = ("Optflags: * -fmessage-length=0 -grecord-gcc-switches -O2 -Wall -D_FORTIFY_SOURCE=2 -fstack-protector-strong -funwind-tables -fasynchronous-unwind-tables",)
    else:
        optflags = ()

    ps = None
    if reponame in compiler_parallel_studio:
        ps = compiler_parallel_studio[reponame]["ps"]
    else:
        for mpi in mpi_parallel_studio:
            if reponame.startswith(mpi):
                ps = mpi_parallel_studio[mpi]["ps"]

    if ps:
        preferred_mkl = "Prefer: " + intel_parallel_studio[ps]["mkl"]
        unprefer_other_mkls = sorted("Prefer: !" + mkl for mkl in all_mkls if mkl != preferred_mkl)
        prefers = ("Prefer: " + ps,) + tuple(unprefer_other_mkls) + (preferred_mkl,)
    else:
        prefers = ()

    return optflags + prefers


def valid_cuda(cuda, compiler):
    """
    The CUDA distribution only works with certain gcc versions,
    this little function takes care that only supported combinations
    are allowed.

    Take care to keep this in sync with the file 'macros.obs_cluster' of
    the package software:dist / mpcdf_cluster_macros
    """
    if cuda == "cuda_8_0":
        return compiler == "gcc_5"
    if cuda == "cuda_9_1":
        return compiler == "gcc_6_3_0"
    if cuda == "cuda_9_2":
        return compiler == "gcc_6"
    if cuda == "cuda_10_0":
        return compiler == "gcc_6"
    if cuda == "cuda_10_1":
        return compiler == "gcc_8"
    if cuda == "cuda_10_2":
        return compiler == "gcc_8"
    if cuda == "cuda_11_0":
        return compiler == "gcc_9"
    if cuda == "cuda_11_2":
        return compiler == "gcc_10"
    if cuda == "cuda_11_4":
        return compiler == "gcc_11"
    return False


def project_meta(api_url, project):
    return ElementTree.fromstringlist(osc.core.show_project_meta(api_url, project))


def package_meta(api_url, project, package):
    return ElementTree.fromstringlist(osc.core.show_package_meta(api_url, project, package))


def maintainers(api_url, project, package):
    root = package_meta(api_url, project, package)
    return {e.get("userid") for e in root.findall("./person[@role='maintainer']")}


class UnsetAttributeException(Exception):
    pass


class UnmanagedPackageException(Exception):
    pass


def chunked(items, chunksize):
    n = len(items)
    i = 0
    while i * chunksize < n:
        yield items[i * chunksize: (i + 1) * chunksize]
        i += 1


def get_attribute(api_url, project, package, attribute, with_project=False):
    attribute_meta = osc.core.show_attribute_meta(api_url, project, package, None, attribute, False, with_project)
    if attribute_meta is None:
        raise osc.oscerr.APIError("Cannot fetch value for attribute '{0}' from {1}".format(attribute, (project, package)))

    root = ElementTree.fromstringlist(attribute_meta)
    attribute = root.find("./attribute")
    if attribute is not None:
        return root
    else:
        raise UnsetAttributeException("Attribute not set")


def overloaded_project_attribute(api_url, project, attribute):
    try:
        return get_attribute_values(api_url, project, None, attribute)
    except UnsetAttributeException:
        return get_attribute_values(api_url, "software", None, attribute)


def overloaded_package_attribute(api_url, project, package, attribute):
    try:
        return get_attribute_values(api_url, project, package, attribute)
    except UnsetAttributeException:
        pass

    try:
        return get_attribute_values(api_url, project, None, attribute)
    except UnsetAttributeException:
        pass

    return get_attribute_values(api_url, "software", None, attribute)


def get_allowed_attribute_values(api_url, attribute):
    path = ["attribute"] + attribute.split(":") + ["_meta"]
    url = osc.core.makeurl(api_url, path, [])
    try:
        f = osc.core.http_GET(url)
    except osc.core.HTTPError as e:
        e.osc_msg = 'Error getting meta for attribute "{0}"'.format(attribute)
        raise

    root = ElementTree.fromstringlist(f.readlines())
    return list(value.text for value in root.findall("./allowed/value"))


def get_attribute_values(api_url, project, package, attribute, with_project=False):
    root = get_attribute(api_url, project, package, attribute, with_project)
    attribute = root.find("./attribute")
    return list(value.text for value in attribute.findall("./value"))


def get_attribute_value(api_url, project, package, attribute, with_project=False):
    value, = get_attribute_values(api_url, project, package, attribute, with_project=False)
    return value


def set_attribute(api_url, project, package, attribute):
    path = ["source", project]
    if package:
        path.append(package)
    path.append("_attribute")

    attr_meta = ElementTree.tostring(attribute, encoding=osc.core.ET_ENCODING)

    url = osc.core.makeurl(api_url, path)
    resp = ElementTree.fromstringlist(osc.core.streamfile(url, osc.core.http_POST, data=attr_meta))
    if resp.find("./summary").text != "Ok":
        raise osc.oscerr.APIError("Could not store attribute")


def set_attribute_values(api_url, project, package, attribute, values):
    root = ElementTree.Element("attributes")
    attr = ElementTree.SubElement(root, "attribute")

    namespace, name = attribute.split(":")
    attr.set("namespace", namespace)
    attr.set("name", name)

    for value in values:
        v = ElementTree.SubElement(attr, "value")
        v.text = value

    set_attribute(api_url, project, package, root)


def has_attribute(api_url, project, package, attribute):
    path = ["source", project]
    if package:
        path.append(package)
    path.append("_attribute")
    path.append(attribute)

    url = osc.core.makeurl(api_url, path)
    resp = ElementTree.fromstringlist(osc.core.streamfile(url, osc.core.http_GET))

    namespace, name = attribute.split(":")

    for a in resp.findall("./attribute"):
        if a.get("namespace") == namespace and a.get("name") == name:
            return True
    return False


def remove_attribute(api_url, project, package, attribute_name):
    path = ["source"] + [project]
    if package:
        path.append(package)
    path.append("_attribute")
    path.append(attribute_name)

    url = osc.core.makeurl(api_url, path)
    resp = ElementTree.fromstringlist(osc.core.streamfile(url, osc.core.http_DELETE))
    if resp.find("./summary").text != "Ok":
        raise osc.oscerr.APIError("Could not remove attribute")


def get_microarchitecture(project):
    if project.startswith("software:"):
        microarch = project.split(":")[2]
    elif project.startswith("home:"):
        microarch = "sandybridge"
    else:
        raise Exception("Cannot determine micro-architecture for project '{0}'".format(project))

    if microarch in known_microarchs:
        return microarch
    else:
        raise Exception("Unknown micro-architecture '{0}'".format(microarch))


def package_sort_key(string):
    name, version = string.split("_", 1)
    version = version.split("_")
    return (name,) + tuple(map(int, version))


def mpcdf_enable_repositories(api_url, project, package, verbose=False, dry_run=False, ignore_repos=()):
    from itertools import product

    pkg_meta = osc.core.show_package_meta(api_url, project, package)
    root = ElementTree.fromstringlist(pkg_meta)

    build = root.find("./build")
    if build is None:
        build = ElementTree.SubElement(root, "build")

    pkg_meta = ElementTree.tostring(root, encoding=osc.core.ET_ENCODING)

    for enable in build.findall("./enable"):
        build.remove(enable)

    disabled_repos = {disable.get("repository") for disable in build.findall("./disable")}

    try:
        enable_repos = get_attribute_values(api_url, project, package, "MPCDF:enable_repositories")
    except Exception:
        if verbose:
            print("Warning: Could not get attribute MPCDF:enable_repositories for package {0}, skipping".format(package))
        raise UnmanagedPackageException()

    if project == "software":
        distributions = list(sorted(repo.name for repo in osc.core.get_repos_of_project(api_url, "software")))

    else:
        compilers = overloaded_package_attribute(api_url, project, package, "MPCDF:compiler_modules")
        mpis = overloaded_package_attribute(api_url, project, package, "MPCDF:mpi_modules")
        cudas = overloaded_package_attribute(api_url, project, package, "MPCDF:cuda_modules")
        pgis = overloaded_package_attribute(api_url, project, package, "MPCDF:pgi_modules")
        openmpi_flavors = overloaded_package_attribute(api_url, project, package, "MPCDF:openmpi_flavors")

        all_compilers = overloaded_project_attribute(api_url, project, "MPCDF:compiler_modules")
        all_mpis = overloaded_project_attribute(api_url, project, "MPCDF:mpi_modules")
        all_cudas = overloaded_project_attribute(api_url, project, "MPCDF:cuda_modules")
        all_pgis = overloaded_project_attribute(api_url, project, "MPCDF:pgi_modules")
        all_openmpi_flavors = overloaded_project_attribute(api_url, project, "MPCDF:openmpi_flavors")

        latest_intel = sorted((c for c in all_compilers if c.startswith("intel")), key=package_sort_key)[-1]
        latest_gcc = sorted((c for c in all_compilers if c.startswith("gcc")), key=package_sort_key)[-1]

    i = len(build)

    def enable(name):
        if name in ignore_repos or name in disabled_repos:
            return
        node = ElementTree.Element("enable")
        node.set("repository", name)
        node.tail = "\n    "
        build.insert(i, node)
        if verbose:
            print("Enabling", name)

    def actual_compilers():
        for compiler in (c for c in compilers if c in all_compilers + ["intel", "gcc", "latest_intel", "latest_gcc"]):
            if compiler == "intel":
                for intel_compiler in [cc for cc in all_compilers if cc.startswith("intel")]:
                    yield intel_compiler
            elif compiler == "gcc":
                for gcc_compiler in [cc for cc in all_compilers if cc.startswith("gcc")]:
                    yield gcc_compiler
            elif compiler == "latest_intel":
                yield latest_intel
            elif compiler == "latest_gcc":
                yield latest_gcc
            else:
                yield compiler

    def actual_mpis():
        for mpi in (m for m in mpis if m in all_mpis + ["impi"]):
            if mpi == "impi":
                for impi in [mpi for mpi in all_mpis if mpi.startswith("impi")]:
                    yield impi
            else:
                yield mpi

    def actual_cudas():
        for cuda in (c for c in cudas if c in all_cudas):
            yield cuda

    def actual_pgis():
        for pgi in (p for p in pgis if p in all_pgis):
            yield pgi

    def actual_openmpi_flavors():
        for of in (f for f in openmpi_flavors if f in all_openmpi_flavors):
            yield of

    for flag in enable_repos:
        if flag == "system":
            if project == "software":
                for distribution in distributions:
                    enable(distribution)
            elif project.startswith("home:"):
                enable("System")

        elif project != "software":
            if flag == "compilers":
                for compiler in actual_compilers():
                    enable(compiler)

            if flag == "mpi":
                for mpi, compiler in product(actual_mpis(), actual_compilers()):
                    if valid_mpi(compiler, mpi):
                        enable(mpi + "_" + compiler)

            if flag == "openmpi_flavors":
                for mpi, compiler in product(actual_mpis(), actual_compilers()):
                    if "openmpi" in mpi and valid_mpi(compiler, mpi):
                        for of in actual_openmpi_flavors():
                            enable(mpi + "_" + compiler + "_" + of)

            if flag == "cuda":
                for cuda, compiler in product(actual_cudas(), all_compilers):
                    if valid_cuda(cuda, compiler):
                        enable(cuda + "_" + compiler)

            if flag == "cuda_mpi":
                for cuda, mpi, compiler in product(actual_cudas(), actual_mpis(), all_compilers):
                    if valid_cuda(cuda, compiler) and valid_mpi(compiler, mpi):
                        enable(cuda + "_" + mpi + "_" + compiler)

            if flag == "pgi":
                for pgi in actual_pgis():
                    enable(pgi)

            if flag == "pgi_mpi":
                for mpi, pgi in product(actual_mpis(), actual_pgis()):
                    if valid_pgi_mpi(pgi, mpi):
                        enable(mpi + "_" + pgi)

            if flag == "cuda_aware_mpi":
                for cuda, mpi, compiler in product(actual_cudas(), actual_mpis(), all_compilers):
                    if "openmpi" in mpi and valid_cuda(cuda, compiler) and valid_mpi(compiler, mpi):
                        enable(cuda + "_aware_" + mpi + "_" + compiler)

            if flag == "cuda_aware_openmpi_flavors":
                for cuda, mpi, compiler in product(actual_cudas(), actual_mpis(), all_compilers):
                    if "openmpi" in mpi and valid_cuda(cuda, compiler) and valid_mpi(compiler, mpi):
                        for of in actual_openmpi_flavors():
                            enable(cuda + "_aware_" + mpi + "_" + compiler + "_" + of)

    if len(build.getchildren()) > 0:
        build.getchildren()[-1].tail = "\n  "

    new_pkg_meta = ElementTree.tostring(root, encoding=osc.core.ET_ENCODING)
    if pkg_meta != new_pkg_meta:
        print("Updating repositories for", package)
        if dry_run:
            print("osc meta pkg {0} {1} -F - <<EOF\n{2}\nEOF\n".format(project, package, new_pkg_meta))
        else:
            osc.core.edit_meta("pkg", (project, package), data=new_pkg_meta, apiurl=api_url)
    elif dry_run:
        print("Would not do anything, package meta would be unchanged")

    return True


def parse_prjconf(api_url, project):
    orig_prjconf = list(map(decode_it, osc.core.show_project_conf(api_url, project)))
    try:
        start = orig_prjconf.index(prjconf_start_marker)
        end = orig_prjconf.index(prjconf_end_marker)
    except ValueError:
        start = None
        end = len(orig_prjconf)

    prjconf_head = orig_prjconf[:start]
    prjconf_ours = orig_prjconf[start:end]
    prjconf_tail = orig_prjconf[end + 1:]
    return orig_prjconf, prjconf_head, prjconf_ours, prjconf_tail


def openmpi_flavor_dependencies(flavor, distribution):
    if flavor.startswith("mofed_"):
        mofed_version = flavor[len("mofed_"):].replace("_", ".")
        mofed_repo = "extern:mofed:" + mofed_version
        return ((mofed_repo, distribution),)
    else:
        return ()


def mpcdf_setup_subproject(api_url, project, distribution, microarchitecture,
                           parent=None, dry_run=False, diff=False, remove_old=False, remove_old_matching=None, all_possible=False, only_project=False):
    import re

    if parent and not dry_run:
        for attribute in config_attributes:
            print("Copying attribute '{0}' from parent project".format(attribute))
            set_attribute_values(api_url, project, None, attribute, overloaded_project_attribute(api_url, parent, attribute))

    # Check distribution
    software_meta = project_meta(api_url, "software")
    dist_repo = software_meta.find('./repository[@name="{0}"]'.format(distribution))
    if dist_repo is None:
        raise osc.oscerr.WrongArgs("Invalid distribution '{0}': No matching repository is defined in project 'software' on the server".format(distribution))
    architectures = list(arch.text for arch in dist_repo.findall("./arch"))

    root = project_meta(api_url, project)

    orig_prjconf, prjconf_head, prjconf_ours, prjconf_tail = parse_prjconf(api_url, project)

    prjconf_repos = {}
    cur_repo = None
    old_repos = set()
    for line in prjconf_ours:
        line = line.rstrip("\n")
        if cur_repo is None and (line.startswith("%if %_repository ==") or line.startswith("%if \"%{_repository}\" == ")) and len(line.split()) == 4:
            cur_repo = line.split()[-1].strip('"')
            old_repos.add(cur_repo)
            prjconf_repos[cur_repo] = []
        if cur_repo is not None:
            prjconf_repos[cur_repo].append(line)
            if line == "%endif":
                cur_repo = None

    prjconf_ours = [prjconf_start_marker]

    prjconf_ours.append("Constraint: hostlabel {0}".format(microarchitecture))
    prjconf_ours.append("Preinstall: mpcdf_{0}_directory".format(microarchitecture))
    prjconf_ours.append("PublishFilter: ^mpcdf_.*$")
    prjconf_ours.append("")

    extra_tags = dist_prjconf_tags(distribution)
    if extra_tags:
        prjconf_ours.append(extra_tags)
    extra_macros = dist_prjconf_macros(distribution)

    if not project.startswith("home:"):
        extra_macros = "%microarchitecture {0}\n{1}".format(microarchitecture, extra_macros)

    prjconf_ours.append("""
Macros:
{0}
:Macros""".format(extra_macros))
    prjconf_ours.append("")

    # Remove existing repositories
    if remove_old:
        for oldrepo in root.findall("./repository"):
            root.remove(oldrepo)
    if remove_old_matching:
        for oldrepo in root.findall("./repository"):
            if re.search(remove_old_matching, oldrepo.attrib["name"]):
                root.remove(oldrepo)

    def repo(name, dependencies, compiler=False, mpi=False, cuda=False, cuda_mpi=False, cuda_aware_mpi=False, additional_tags=(), **macros):
        old_repos.discard(name)
        have_compiler = compiler or mpi or cuda or cuda_mpi or cuda_aware_mpi
        have_mpi = mpi or cuda_mpi or cuda_aware_mpi
        have_cuda = cuda or cuda_mpi or cuda_aware_mpi

        existing_repo = root.find("./repository[@name='{0}']".format(name))
        if existing_repo is not None:
            root.remove(existing_repo)
        elif dry_run is False:
            print("New repository", name)

        r = ElementTree.SubElement(root, "repository")
        r.set("name", name)
        r.text = "\n    "

        def path(project, repo):
            p = ElementTree.SubElement(r, "path")
            p.set("project", project)
            p.set("repository", repo)
            p.tail = "\n    "

        # TODO: Order inverted here?
        for dep_project, dep_repo in dependencies:
            path(dep_project, dep_repo)
        if parent and name != "System":
            path(parent, name)

        for arch in architectures:
            a = ElementTree.SubElement(r, "arch")
            a.text = arch
            a.tail = "\n    "
        a.tail = "\n  "
        r.tail = "\n  "

        # In order to be able to figure out the matching MKL version for a given
        # compiler/MPI repository we emit a new macro '%matching_mkl_version' in
        # the cases this makes sense
        matching_mkl = []
        repoconf = []
        repoconf.append("%if \"%{{_repository}}\" == \"{0}\"".format(name))
        for tag in repo_tags(name, distribution=distribution) + additional_tags:
            if tag.startswith("Prefer: mkl"):
                matching_mkl.append(tag.split()[1])
            repoconf.append(tag)

        repoconf.append("Macros:")
        if project.startswith("home:") and name != "System":
            repoconf.append("%microarchitecture {0}".format(microarchitecture))

        repoconf.append("%is_compiler_repository {0}".format(1 if compiler else 0))
        repoconf.append("%is_mpi_repository {0}".format(1 if mpi else 0))
        repoconf.append("%is_cuda_repository {0}".format(1 if cuda else 0))
        repoconf.append("%is_cuda_mpi_repository {0}".format(1 if cuda_mpi else 0))
        repoconf.append("%is_cuda_aware_mpi_repository {0}".format(1 if cuda_aware_mpi else 0))

        repoconf.append("%have_mpcdf_compiler {0}".format(1 if have_compiler else 0))
        repoconf.append("%have_mpcdf_mpi {0}".format(1 if have_mpi else 0))
        repoconf.append("%have_mpcdf_cuda {0}".format(1 if have_cuda else 0))

        if matching_mkl:
            matching_mkl, = matching_mkl
            matching_mkl, _ = matching_mkl[len("mkl_"):].split("-module")
            matching_mkl = matching_mkl.replace("_", ".")
            repoconf.append("%matching_mkl_version {0}".format(matching_mkl))

        for macro, value in macros.items():
            repoconf.append("%{0} {1}".format(macro, value))

        repoconf.append(":Macros")
        repoconf.append("%endif")
        prjconf_repos[name] = repoconf

    def actual_compiler(c):
        if c.startswith("latest"):
            return False
        if c == "intel":
            return False
        if c == "gcc":
            return False
        return True

    def actual_mpi(m):
        if m == "impi":
            return False
        return True

    def actual_cuda(c):
        return True

    if all_possible:
        compilers = list(filter(actual_compiler, get_allowed_attribute_values(api_url, "MPCDF:compiler_modules")))
        mpis = list(filter(actual_mpi, get_allowed_attribute_values(api_url, "MPCDF:mpi_modules")))
        cudas = list(filter(actual_cuda, get_allowed_attribute_values(api_url, "MPCDF:cuda_modules")))
        pgis = get_allowed_attribute_values(api_url, "MPCDF:pgi_modules")
        openmpi_flavors = get_allowed_attribute_values(api_url, "MPCDF:openmpi_flavors")
    else:
        compilers = overloaded_project_attribute(api_url, project, "MPCDF:compiler_modules")
        mpis = overloaded_project_attribute(api_url, project, "MPCDF:mpi_modules")
        cudas = overloaded_project_attribute(api_url, project, "MPCDF:cuda_modules")
        pgis = overloaded_project_attribute(api_url, project, "MPCDF:pgi_modules")
        openmpi_flavors = overloaded_project_attribute(api_url, project, "MPCDF:openmpi_flavors")

    if parent:
        repo("System", (("software", distribution),))

    for compiler in compilers + pgis:
        if project.startswith("home:"):
            dependencies = ((project, "System"),)
        elif project != "software":
            dependencies = (("software", distribution),)
        else:
            dependencies = ()
        repo(compiler, dependencies, compiler=True,
             compiler_repository=compiler, compiler_module=compiler_module(compiler),
             additional_tags=("Prefer: mpcdf_compiler_" + compiler,))

        for mpi in filter(partial(valid_mpi, compiler), mpis):
            repo(mpi + "_" + compiler, ((project, compiler),), mpi=True,
                 mpi_repository=mpi, mpi_module=mpi_module(mpi))

            if "openmpi" in mpi:
                for of in openmpi_flavors:
                    dependencies = ((project, compiler),)
                    if not parent:
                        dependencies = openmpi_flavor_dependencies(of, distribution) + dependencies
                    flavor_kind = re.sub("(_[0-9]+)*$", "", of)
                    repo(mpi + "_" + compiler + "_" + of, dependencies, mpi=True,
                         mpi_repository=mpi, mpi_module=mpi_module(mpi),
                         openmpi_flavor=flavor_kind)

    for cuda in cudas:
        for compiler in filter(partial(valid_cuda, cuda), compilers):
            repo(cuda + "_" + compiler, ((project, compiler),), cuda=True, cuda_repository=cuda,
                 additional_tags=("Prefer: mpcdf_" + cuda,))
            for mpi in filter(partial(valid_mpi, compiler), mpis):
                repo(cuda + "_" + mpi + "_" + compiler,
                     ((project, cuda + "_" + compiler),
                      (project, mpi + "_" + compiler)),
                     cuda_mpi=True)

                if "openmpi" in mpi:
                    repo(cuda + "_aware_" + mpi + "_" + compiler,
                         ((project, cuda + "_" + mpi + "_" + compiler),),
                         cuda_aware_mpi=True,
                         additional_tags=("Prefer: mpcdf_mpi_" + mpi + "_" + cuda,
                                          "Prefer: mpcdf_" + cuda,))
                    for of in openmpi_flavors:
                        dependencies = ((project, cuda + "_" + mpi + "_" + compiler),)
                        if not parent:
                            dependencies = openmpi_flavor_dependencies(of, distribution) + dependencies
                        repo(cuda + "_aware_" + mpi + "_" + compiler + "_" + of,
                             dependencies,
                             cuda_aware_mpi=True, openmpi_flavor=of,
                             additional_tags=("Prefer: mpcdf_" + cuda,))

    if old_repos and not remove_old:
        print("Warning: Keeping the prjconf sections for the following obsolete repositories:")
        for name in sorted(old_repos):
            print(" -", name)
        print()
    else:
        for old_repo in old_repos:
            del prjconf_repos[old_repo]

    # Remove build configuration
    build = root.find("./build")
    if build is None:
        build = ElementTree.Element("build")
        build.text = "\n    "
        disable = ElementTree.SubElement(build, "disable")
        disable.tail = "\n  "
        build.tail = "\n  "
        root.insert(list(root).index(root.find("./repository")), build)

    def sort_repos(root):
        # Sort existing repositories to be able to compare
        repos = root.findall("./repository")
        for repo in repos:
            root.remove(repo)
        repos = sorted(repos, key=lambda child: child.get("name"))
        for repo in repos:
            root.append(repo)
        root.getchildren()[-1].tail = "\n"

    sort_repos(root)
    new_prj = ElementTree.tostring(root, encoding=osc.core.ET_ENCODING)

    for name in sorted(prjconf_repos.keys()):
        if name not in old_repos:
            prjconf_ours.extend(prjconf_repos[name])
            prjconf_ours.append("")

    if not remove_old:
        prjconf_ours.append("# Obsolete repos: ")
        prjconf_ours.append("")
        for name in sorted(old_repos):
            prjconf_ours.extend(prjconf_repos[name])
            prjconf_ours.append("")

    prjconf_ours.append(prjconf_end_marker)

    new_prjconf = "".join(prjconf_head) + "\n".join(prjconf_ours) + "".join(prjconf_tail)

    old_prj_root = project_meta(api_url, project)
    sort_repos(old_prj_root)
    old_prj = ElementTree.tostring(old_prj_root, encoding=osc.core.ET_ENCODING)

    if diff:
        def stringdiff(s1, label1, s2, label2):
            import os
            from subprocess import call
            from tempfile import NamedTemporaryFile

            old = NamedTemporaryFile("w+", delete=False)
            new = NamedTemporaryFile("w+", delete=False)

            old.write(s1)
            old.close()
            new.write(s2)
            new.close()

            call(["diff", "--color=auto", "-s", "-u", old.name, "--label", label1, new.name, "--label", label2])
            os.unlink(old.name)
            os.unlink(new.name)

        stringdiff("".join(orig_prjconf), "old-prjconf", new_prjconf, "new-prjconf")
        stringdiff(old_prj + "\n", "old-prj-meta", new_prj + "\n", "new-prj-meta")

    if dry_run:
        if not diff:
            print("osc meta prjconf {0} -F - <<EOF\n{1}\nEOF\n".format(project, new_prjconf))
            print("osc meta prj {0} -F - <<EOF\n{1}\nEOF\n".format(project, new_prj))
    else:
        if new_prjconf == "".join(orig_prjconf):
            print("prjconf unchanged")
        else:
            print("Updating prjconf meta")
            osc.core.edit_meta("prjconf", project, data=new_prjconf, apiurl=api_url)

        # Create and remove the <enable/> flags before the new repsitories,
        # that way no spurious builds are launched
        if not only_project:
            print("Updating enabled repositories for all packages")
            mpcdf_enable_repositories_for_all_packages(api_url, project)

        # Update repositories
        if new_prj == old_prj:
            print("prj meta unchanged")
        else:
            print("Updating prj meta")
            osc.core.edit_meta("prj", project, data=new_prj, force=True, apiurl=api_url)


def mpcdf_enable_repositories_for_all_packages(api_url, project, ignore_repos=()):
    import threading
    packages = osc.core.meta_get_packagelist(api_url, project)

    if len(packages) > 40:
        chunksize = len(packages) // 20
    else:
        chunksize = len(packages)

    def work(packagelist):
        for package in packagelist:
            try:
                mpcdf_enable_repositories(api_url, project, package, ignore_repos=ignore_repos)
            except UnmanagedPackageException:
                print("ATTENTION: Not changing unmanaged package {0}".format(package))

    threads = []
    for packagelist in chunked(packages, chunksize):
        t = threading.Thread(target=work, args=(packagelist,))
        threads.append(t)
        t.start()

    for t in threads:
        t.join()


def set_as_branch(api_url, my_project, my_package, main_project, main_package):
    import os

    def dirmeta(project, package):
        url = osc.core.makeurl(api_url, ["source", project, package])
        return ElementTree.fromstringlist(osc.core.streamfile(url, osc.core.http_GET))

    srcmeta = dirmeta(my_project, my_package)
    dstmeta = dirmeta(main_project, main_package)

    linkinfo = srcmeta.find('./linkinfo')
    if linkinfo is not None:
        print("ERROR: package {0} is already linked to package {1} in project {2}".format(
              my_package, linkinfo.get("package"), linkinfo.get("project")))
        return False

    linkfile = ElementTree.Element("link")
    linkfile.set("project", main_project)
    linkfile.set("package", main_package)
    linkfile.set("baserev", dstmeta.get("srcmd5"))
    patches = ElementTree.SubElement(linkfile, "patches")
    ElementTree.SubElement(patches, "branch")

    url = osc.core.makeurl(api_url, ["source", my_project, my_package, "_link"])
    resp = ElementTree.fromstringlist(osc.core.streamfile(url, osc.core.http_PUT, data=ElementTree.tostring(linkfile)))
    rev = resp.get("rev")
    if rev is None:
        print("ERROR: Could not commit _link file")
        return False
    else:
        print("Commited as revision", rev)
        this_package = osc.core.store_read_package(os.curdir)
        this_project = osc.core.store_read_project(os.curdir)
        if this_package == my_package and this_project == my_project:
            pac = osc.core.filedir_to_pac(os.curdir)
            rev = pac.latest_rev(expand=True)
            pac.update(rev)
    return True


def sync_projects(api_url, package=None, from_project="software", to_projects=None, add_to_maintainers=True, verbose=False):
    if to_projects is None:
        to_projects = [p for p in osc.core.meta_get_project_list(api_url)
                       if p.startswith("software:") and not (p == "software:dist" or p == "software:images")]

    allowed_attribute_values = {}
    for attribute in package_attributes + config_attributes:
        allowed_attribute_values[attribute] = set(get_allowed_attribute_values(api_url, attribute))

    for to_project in to_projects:
        print("Syncing {0} with {1}".format(to_project, from_project))

        to_packages = osc.core.meta_get_packagelist(api_url, to_project)

        if package is None:
            from_packages = osc.core.meta_get_packagelist(api_url, from_project)
        else:
            from_packages = [package]

        for orig_package in from_packages:
            enable_repos = get_attribute_values(api_url, from_project, orig_package, "MPCDF:enable_repositories")
            if orig_package not in to_packages:
                if len(enable_repos) == 0:
                    print("Not branching package {0}, is disabled".format(orig_package))
                    continue
                elif enable_repos == ["system"] and "System" not in osc.core.get_repositories_of_project(api_url, to_project):
                    print("Not branching package {0}, is only enabled for 'system'".format(orig_package))
                    continue
                filelist = osc.core.meta_get_filelist(api_url, from_project, orig_package)
                if "_link" in filelist:
                    print("Not branching package {0}, is a link".format(orig_package))
                else:
                    print("Branching package {0}".format(orig_package))
                    exists, targetprj, targetpkg, srcprj, srcpkg = \
                        osc.core.branch_pkg(api_url, from_project, orig_package, target_project=to_project, nodevelproject=True)
            else:
                print("Not branching package {0}, already present in target".format(orig_package))

            for attribute in package_attributes + config_attributes:
                try:
                    values = list(filter(lambda q: q in allowed_attribute_values[attribute],
                                         get_attribute_values(api_url, from_project, orig_package, attribute)))
                except UnsetAttributeException:
                    if has_attribute(api_url, to_project, orig_package, attribute):
                        remove_attribute(api_url, to_project, orig_package, attribute)
                    continue
                set_attribute_values(api_url, to_project, orig_package, attribute, values)

            if add_to_maintainers:
                from_maintainers = maintainers(api_url, from_project, orig_package)
                if from_maintainers:
                    to_maintainers = maintainers(api_url, to_project, orig_package)
                    if from_maintainers - to_maintainers:
                        new_maintainers = from_maintainers - to_maintainers
                        to_meta = package_meta(api_url, to_project, orig_package)
                        for userid in sorted(new_maintainers):
                            person = ElementTree.Element("person")
                            person.set("userid", userid)
                            person.set("role", "maintainer")
                            to_meta.insert(2, person)
                        osc.core.edit_meta("pkg", (to_project, orig_package), data=ElementTree.tostring(to_meta), apiurl=api_url)

            try:
                mpcdf_enable_repositories(api_url, to_project, orig_package, verbose=verbose)
            except UnmanagedPackageException:
                print("ATTENTION: Not changing unmanaged package {0}".format(orig_package))