Changes for new structure of obs-api.mpcdf.mpg.de

mpcdf_common.py

@@ -20,7 +20,7 @@ known_microarchs = {"sandybridge", "haswell", "skylake"}
 
 package_attributes = ["MPCDF:enable_repositories"]
 config_attributes = ["MPCDF:compiler_modules", "MPCDF:cuda_modules", "MPCDF:mpi_modules", "MPCDF:pgi_modules"]
-default_attributes = ["MPCDF:default_compiler", "MPCDF:default_cuda", "MPCDF:default_mpi", "MPCDF:default_python2", "MPCDF:default_python3"]
+default_attributes = ["MPCDF:default_compiler", "MPCDF:default_cuda", "MPCDF:default_mpi"]
 
 intel_parallel_studio = {
     "mpcdf_intel_parallel_studio_2017_7": {"compiler": "intel_17_0_7", "impi": "impi_2017_4", "mkl": "mkl_2017_4-module", },
@@ -161,7 +161,7 @@ def prefers(reponame):
 
     if prefer_ps:
         preferred_mkl = intel_parallel_studio[prefer_ps]["mkl"]
-        unprefer_other_mkls = ("!" + mkl for mkl in all_mkls if mkl != preferred_mkl)
+        unprefer_other_mkls = sorted("!" + mkl for mkl in all_mkls if mkl != preferred_mkl)
         return (prefer_ps,) + tuple(unprefer_other_mkls) + (preferred_mkl,)
     else:
         return ()
@@ -208,6 +208,10 @@ class UnsetAttributeException(Exception):
     pass
 
 
+class UnmanagedPackageException(Exception):
+    pass
+
+
 def chunked(l, chunksize):
     n = len(l)
     i = 0
@@ -229,6 +233,40 @@ def get_attribute(api_url, project, package, attribute, with_project=False):
         raise UnsetAttributeException("Attribute not set")
 
 
+def overloaded_project_attribute(api_url, project, attribute):
+    try:
+        return get_attribute_values(api_url, project, None, attribute)
+    except UnsetAttributeException:
+        return get_attribute_values(api_url, "software", None, attribute)
+
+
+def overloaded_package_attribute(api_url, project, package, attribute):
+    try:
+        return get_attribute_values(api_url, project, package, attribute)
+    except UnsetAttributeException:
+        pass
+
+    try:
+        return get_attribute_values(api_url, project, None, attribute)
+    except UnsetAttributeException:
+        pass
+
+    return get_attribute_values(api_url, "software", None, attribute)
+
+
+def get_allowed_attribute_values(api_url, attribute):
+    path = ["attribute"] + attribute.split(":") + ["_meta"]
+    url = osc.core.makeurl(api_url, path, [])
+    try:
+        f = osc.core.http_GET(url)
+    except osc.core.HTTPError as e:
+        e.osc_msg = 'Error getting meta for attribute "{0}"'.format(attribute)
+        raise
+
+    root = ElementTree.fromstringlist(f.readlines())
+    return list(value.text for value in root.findall("./allowed/value"))
+
+
 def get_attribute_values(api_url, project, package, attribute, with_project=False):
     root = get_attribute(api_url, project, package, attribute, with_project)
     attribute = root.find("./attribute")
@@ -301,10 +339,7 @@ def remove_attribute(api_url, project, package, attribute_name):
 
 
 def get_microarchitecture(project):
-    if project == "software":
-        # Stupid special case
-        microarch = "sandybridge"
-    elif project.startswith("software:"):
+    if project.startswith("software:"):
         microarch = project.split(":")[2]
     elif project.startswith("home:"):
         microarch = "sandybridge"
@@ -323,13 +358,10 @@ def package_sort_key(string):
     return (name,) + tuple(map(int, version))
 
 
-def mpcdf_enable_repositories(api_url, project, package, verbose=False, filter_repos=None):
+def mpcdf_enable_repositories(api_url, project, package, verbose=False, ignore_repos=()):
     from itertools import product
     import sys
 
-    if filter_repos is None:
-        filter_repos = ()
-
     pkg_meta = osc.core.show_package_meta(api_url, project, package)
     root = ElementTree.fromstringlist(pkg_meta)
 
@@ -347,36 +379,31 @@ def mpcdf_enable_repositories(api_url, project, package, verbose=False, filter_r
     except Exception:
         if verbose:
             print("Warning: Could not get attribute MPCDF:enable_repositories for package {0}, skipping".format(package))
-        return False
+        raise UnmanagedPackageException()
 
-    def try_get_attribute(package, attribute, with_project=False):
-        try:
-            return get_attribute_values(api_url, project, package, "MPCDF:" + attribute, with_project=with_project)
-        except UnsetAttributeException:
-            print("ERROR: Attribute MPCDF:" + attribute, "is not set for",
-                  "package '{0}'".format(package) if package else "project '{0}'".format(project),
-                  "- aborting here", file=sys.stderr)
-            raise SystemExit(1)
+    if project == "software":
+        distributions = list(sorted(repo.name for repo in osc.core.get_repos_of_project(api_url, "software")))
 
-    compilers = try_get_attribute(package, "compiler_modules", with_project=True)
-    mpis = try_get_attribute(package, "mpi_modules", with_project=True)
-    cudas = try_get_attribute(package, "cuda_modules", with_project=True)
-    pgis = try_get_attribute(package, "pgi_modules", with_project=True)
+    else:
+        compilers = overloaded_package_attribute(api_url, project, package, "MPCDF:compiler_modules")
+        mpis = overloaded_package_attribute(api_url, project, package, "MPCDF:mpi_modules")
+        cudas = overloaded_package_attribute(api_url, project, package, "MPCDF:cuda_modules")
+        pgis = overloaded_package_attribute(api_url, project, package, "MPCDF:pgi_modules")
 
-    all_compilers = try_get_attribute(None, "compiler_modules")
-    all_mpis = try_get_attribute(None, "mpi_modules")
-    all_cudas = try_get_attribute(None, "cuda_modules")
-    all_pgis = try_get_attribute(None, "pgi_modules")
+        all_compilers = overloaded_project_attribute(api_url, project, "MPCDF:compiler_modules")
+        all_mpis = overloaded_project_attribute(api_url, project, "MPCDF:mpi_modules")
+        all_cudas = overloaded_project_attribute(api_url, project, "MPCDF:cuda_modules")
+        all_pgis = overloaded_project_attribute(api_url, project, "MPCDF:pgi_modules")
 
-    default_compilers = try_get_attribute(None, "default_compiler")
-    default_mpis = try_get_attribute(None, "default_mpi")
-    default_cudas = try_get_attribute(None, "default_cuda")
+        default_compilers = overloaded_project_attribute(api_url, project, "MPCDF:default_compiler")
+        default_mpis = overloaded_project_attribute(api_url, project, "MPCDF:default_mpi")
+        default_cudas = overloaded_project_attribute(api_url, project, "MPCDF:default_cuda")
 
         latest_intel = sorted((c for c in all_compilers if c.startswith("intel")), key=package_sort_key)[-1]
         latest_gcc = sorted((c for c in all_compilers if c.startswith("gcc")), key=package_sort_key)[-1]
 
     def enable(name):
-        if any(filtered_repo in name for filtered_repo in filter_repos):
+        if name in ignore_repos:
             return
         node = ElementTree.Element("enable")
         node.set("repository", name)
@@ -427,10 +454,12 @@ def mpcdf_enable_repositories(api_url, project, package, verbose=False, filter_r
             yield pgi
 
     for flag in enable_repos:
-
+        if project == "software":
             if flag == "system":
-            enable("System")
+                for distribution in distributions:
+                    enable(distribution)
 
+        else:
             if flag == "compilers":
                 for compiler in actual_compilers():
                     enable(compiler)
@@ -470,90 +499,79 @@ def mpcdf_enable_repositories(api_url, project, package, verbose=False, filter_r
     return True
 
 
-def mpcdf_setup_repositories(api_url, project, microarchitecture=None, distribution=None, parent=None,
-                             packages=None, dry_run=False, filter_repos=None, only_project=False, remove_old=False):
-    import threading
+def parse_prjconf(api_url, project):
+    orig_prjconf = list(map(decode_it, osc.core.show_project_conf(api_url, project)))
+    try:
+        start = orig_prjconf.index(prjconf_start_marker)
+        end = orig_prjconf.index(prjconf_end_marker)
+    except ValueError:
+        start = None
+        end = len(orig_prjconf)
 
-    if parent:
+    prjconf_head = orig_prjconf[:start]
+    prjconf_ours = orig_prjconf[start:end]
+    prjconf_tail = orig_prjconf[end + 1:]
+    return orig_prjconf, prjconf_head, prjconf_ours, prjconf_tail
+
+
+def mpcdf_setup_subproject(api_url, project, distribution, microarchitecture,
+                           parent=None, dry_run=False, remove_old=False, all_possible=False):
+
+    if parent and not dry_run:
         for attribute in config_attributes + default_attributes:
             print("Copying attribute '{0}' from parent project".format(attribute))
             set_attribute(api_url, project, None, get_attribute(api_url, parent, None, attribute))
 
-    compilers = get_attribute_values(api_url, project, None, "MPCDF:compiler_modules")
-    mpis = get_attribute_values(api_url, project, None, "MPCDF:mpi_modules")
-    cudas = get_attribute_values(api_url, project, None, "MPCDF:cuda_modules")
-    pgis = get_attribute_values(api_url, project, None, "MPCDF:pgi_modules")
-    default_python2 = get_attribute_value(api_url, project, None, "MPCDF:default_python2")
-    default_python3 = get_attribute_value(api_url, project, None, "MPCDF:default_python3")
-
-    if distribution is None:
-        # Get existing value from project meta
-        dist_repo = project_meta(api_url, project if parent is None else parent).find(
-            "./repository[@name='System']/path[@project='distributions']")
-        if dist_repo is not None:
-            distribution = dist_repo.get("repository")
-        else:
-            raise osc.oscerr.WrongArgs("ERROR: Specify distribution or parent project")
-        print("Using '{0}' as base distribution".format(distribution))
-
-    distributions = project_meta(api_url, "distributions")
-    dist_repo = distributions.find('./repository[@name="{0}"]'.format(distribution))
+    # Check distribution
+    software_meta = project_meta(api_url, "software")
+    dist_repo = software_meta.find('./repository[@name="{0}"]'.format(distribution))
     if dist_repo is None:
-        raise osc.oscerr.WrongArgs("No repository '{0}' is defined in project 'distributions' on the server".format(distribution))
+        raise osc.oscerr.WrongArgs("Invalid distribution '{0}': No matching repository is defined in project 'software' on the server".format(distribution))
     architectures = list(arch.text for arch in dist_repo.findall("./arch"))
 
-    if microarchitecture is None:
-        microarchitecture = get_microarchitecture(project)
-
     root = project_meta(api_url, project)
 
-    prjconf = list(map(decode_it, osc.core.show_project_conf(api_url, project)))
-
-    try:
-        start = prjconf.index(prjconf_start_marker)
-        end = prjconf.index(prjconf_end_marker)
-    except ValueError:
-        start = None
-        end = len(prjconf)
-
-    prjconf_head = "".join(prjconf[:start])
-    prjconf_tail = "".join(prjconf[end + 1:])
-    prjconf = [prjconf_start_marker]
-
-    prjconf.append("Constraint: hostlabel {0}".format(microarchitecture))
-    prjconf.append(
-        """
-%if %_repository != System
+    orig_prjconf, prjconf_head, prjconf_ours, prjconf_tail = parse_prjconf(api_url, project)
+
+    prjconf_repos = {}
+    cur_repo = None
+    old_repos = set()
+    for line in prjconf_ours:
+        line = line.rstrip("\n")
+        if cur_repo is None and line.startswith("%if %_repository =="):
+            cur_repo = line.split()[-1]
+            old_repos.add(cur_repo)
+            prjconf_repos[cur_repo] = []
+        if cur_repo is not None:
+            prjconf_repos[cur_repo].append(line)
+            if line == "%endif":
+                cur_repo = None
+
+    prjconf_ours = [prjconf_start_marker]
+
+    prjconf_ours.append("Constraint: hostlabel {0}".format(microarchitecture))
+    prjconf_ours.append("PublishFilter: ^mpcdf_.*$")
+    prjconf_ours.append("""
 Macros:
 %microarchitecture {0}
-:Macros
-%endif""".format(microarchitecture))
-    prjconf.append("")
-    prjconf.append("Prefer: mpcdf_python2_" + default_python2)
-    prjconf.append("Prefer: mpcdf_python3_" + default_python3)
-    prjconf.append("")
+:Macros""".format(microarchitecture))
+    prjconf_ours.append("")
 
     # Remove existing repositories
     if remove_old:
         for oldrepo in root.findall("./repository"):
             root.remove(oldrepo)
 
-    def repo(name, *dependencies, **kwargs):
-        is_compiler = kwargs.pop("compiler", False)
-        is_mpi = kwargs.pop("mpi", False)
-        is_cuda = kwargs.pop("cuda", False)
-        is_cuda_mpi = kwargs.pop("cuda_mpi", False)
-
-        cuda_repo = kwargs.pop("cuda_repo", "")
-
-        have_compiler = is_compiler or is_mpi or is_cuda or is_cuda_mpi
-        have_mpi = is_mpi or is_cuda_mpi
-        have_cuda = is_cuda or is_cuda_mpi
+    def repo(name, *dependencies, compiler=False, mpi=False, cuda=False, cuda_mpi=False, additional_prefers=(), **macros):
+        old_repos.discard(name)
+        have_compiler = compiler or mpi or cuda or cuda_mpi
+        have_mpi = mpi or cuda_mpi
+        have_cuda = cuda or cuda_mpi
 
         existing_repo = root.find("./repository[@name='{0}']".format(name))
         if existing_repo is not None:
             root.remove(existing_repo)
-        else:
+        elif dry_run is False:
             print("New repository", name)
 
         r = ElementTree.SubElement(root, "repository")
@@ -566,7 +584,7 @@ Macros:
             p.set("repository", repo)
             p.tail = "\n    "
 
-        if parent:
+        if parent and name != "System":
             path(parent, name)
         for dep_project, dep_repo in dependencies:
             path(dep_project, dep_repo)
@@ -582,47 +600,82 @@ Macros:
         # compiler/MPI repository we emit a new macro '%matching_mkl_version' in
         # the cases this makes sense
         matching_mkl = []
-        prjconf.append("%if %_repository == {0}".format(name))
-        for prefer in prefers(name):
+        repoconf = []
+        repoconf.append("%if %_repository == {0}".format(name))
+        for prefer in prefers(name) + additional_prefers:
             if prefer.startswith("mkl_"):
                 matching_mkl.append(prefer)
-            prjconf.append("Prefer: " + prefer)
+            repoconf.append("Prefer: " + prefer)
 
-        prjconf.append("Macros:")
+        repoconf.append("Macros:")
 
-        prjconf.append("%is_compiler_repository {0}".format(1 if is_compiler else 0))
-        prjconf.append("%is_mpi_repository {0}".format(1 if is_mpi else 0))
-        prjconf.append("%is_cuda_repository {0}".format(1 if is_cuda else 0))
-        prjconf.append("%is_cuda_mpi_repository {0}".format(1 if is_cuda_mpi else 0))
+        repoconf.append("%is_compiler_repository {0}".format(1 if compiler else 0))
+        repoconf.append("%is_mpi_repository {0}".format(1 if mpi else 0))
+        repoconf.append("%is_cuda_repository {0}".format(1 if cuda else 0))
+        repoconf.append("%is_cuda_mpi_repository {0}".format(1 if cuda_mpi else 0))
 
-        prjconf.append("%have_mpcdf_compiler {0}".format(1 if have_compiler else 0))
-        prjconf.append("%have_mpcdf_mpi {0}".format(1 if have_mpi else 0))
-        prjconf.append("%have_mpcdf_cuda {0}".format(1 if have_cuda else 0))
-
-        if is_cuda:
-            prjconf.append("%cuda_repository {0}".format(cuda_repo))
+        repoconf.append("%have_mpcdf_compiler {0}".format(1 if have_compiler else 0))
+        repoconf.append("%have_mpcdf_mpi {0}".format(1 if have_mpi else 0))
+        repoconf.append("%have_mpcdf_cuda {0}".format(1 if have_cuda else 0))
 
         if matching_mkl:
             matching_mkl, = matching_mkl
             matching_mkl, _ = matching_mkl[len("mkl_"):].split("-module")
             matching_mkl = matching_mkl.replace("_", ".")
-            prjconf.append("%matching_mkl_version {0}".format(matching_mkl))
+            repoconf.append("%matching_mkl_version {0}".format(matching_mkl))
 
-        for macro, value in kwargs.items():
-            prjconf.append("%{0} {1}".format(macro, value))
+        for macro, value in macros.items():
+            repoconf.append("%{0} {1}".format(macro, value))
 
-        prjconf.append(":Macros")
-        prjconf.append("%endif")
-        prjconf.append("")
+        repoconf.append(":Macros")
+        repoconf.append("%endif")
+        prjconf_repos[name] = repoconf
 
-    if parent:
-        repo("System")
+    def actual_compiler(c):
+        if c.startswith("latest"):
+            return False
+        if c == "default_compiler":
+            return False
+        if c == "intel":
+            return False
+        if c == "gcc":
+            return False
+        return True
+
+    def actual_mpi(m):
+        if m == "default_mpi":
+            return False
+        if m == "impi":
+            return False
+        return True
+
+    def actual_cuda(c):
+        if c == "default_cuda":
+            return False
+        return True
+
+    if all_possible:
+        compilers = list(filter(actual_compiler, get_allowed_attribute_values(api_url, "MPCDF:compiler_modules")))
+        mpis = list(filter(actual_mpi, get_allowed_attribute_values(api_url, "MPCDF:mpi_modules")))
+        cudas = list(filter(actual_cuda, get_allowed_attribute_values(api_url, "MPCDF:cuda_modules")))
+        pgis = get_allowed_attribute_values(api_url, "MPCDF:pgi_modules")
     else:
-        repo("System", ("distributions", distribution))
+        compilers = overloaded_project_attribute(api_url, project, "MPCDF:compiler_modules")
+        mpis = overloaded_project_attribute(api_url, project, "MPCDF:mpi_modules")
+        cudas = overloaded_project_attribute(api_url, project, "MPCDF:cuda_modules")
+        pgis = overloaded_project_attribute(api_url, project, "MPCDF:pgi_modules")
+
+    if parent:
+        repo("System", ("software", distribution))
 
     for compiler in compilers + pgis:
-        repo(compiler, (project, "System"), compiler=True,
-             compiler_repository=compiler, compiler_module=compiler_module(compiler))
+        if project != "software":
+            dependencies = ("software", distribution)
+        else:
+            dependencies = ()
+        repo(compiler, dependencies, compiler=True,
+             compiler_repository=compiler, compiler_module=compiler_module(compiler),
+             additional_prefers=("mpcdf_compiler_" + compiler,))
 
         for mpi in filter(partial(valid_mpi, compiler), mpis):
             repo(mpi + "_" + compiler, (project, compiler), mpi=True,
@@ -630,13 +683,22 @@ Macros:
 
     for cuda in cudas:
         for compiler in filter(partial(valid_cuda, cuda), compilers):
-            repo(cuda + "_" + compiler, (project, compiler), cuda=True, cuda_repo=cuda)
+            repo(cuda + "_" + compiler, (project, compiler), cuda=True, cuda_repository=cuda)
             for mpi in filter(partial(valid_mpi, compiler), mpis):
                 repo(cuda + "_" + mpi + "_" + compiler,
                      (project, cuda + "_" + compiler),
                      (project, mpi + "_" + compiler),
                      cuda_mpi=True)
 
+    if old_repos and not remove_old:
+        print("Warning: Keeping the prjconf sections for the following obsolete repositories:")
+        for name in sorted(old_repos):
+            print(" -", name)
+        print()
+    else:
+        for old_repo in old_repos:
+            del prjconf_repos[old_repo]
+
     # Remove build configuration
     build = root.find("./build")
     if build is None:
@@ -648,19 +710,38 @@ Macros:
         root.insert(list(root).index(root.find("./repository")), build)
 
     root.getchildren()[-1].tail = "\n"
-    prj = ElementTree.tostring(root, encoding=osc.core.ET_ENCODING)
+    new_prj = ElementTree.tostring(root, encoding=osc.core.ET_ENCODING)
+
+    for name in sorted(prjconf_repos.keys()):
+        prjconf_ours.extend(prjconf_repos[name])
+        prjconf_ours.append("")
 
-    prjconf.append(prjconf_end_marker)
-    prjconf = prjconf_head + "\n".join(prjconf) + prjconf_tail
+    prjconf_ours.append(prjconf_end_marker)
+
+    new_prjconf = "".join(prjconf_head) + "\n".join(prjconf_ours) + "".join(prjconf_tail)
 
     if dry_run:
-        print("osc meta prj {0} -F - <<EOF\n{1}\nEOF\n".format(project, prj))
-        print("osc meta prjconf {0} -F - <<EOF\n{1}\nEOF\n".format(project, prjconf))
+        print("osc meta prjconf {0} -F - <<EOF\n{1}\nEOF\n".format(project, new_prjconf))
+        print("osc meta prj {0} -F - <<EOF\n{1}\nEOF\n".format(project, new_prj))
+    else:
+        if new_prjconf == "".join(orig_prjconf):
+            print("prjconf unchanged")
         else:
-        if not only_project:
-            # First set-up the <enable/> flags, that way no
-            # spurious builds are launched
-            if packages is None:
+            print("Updating prjconf meta")
+            osc.core.edit_meta("prjconf", project, data=new_prjconf)
+
+        # Create and remove the <enable/> flags before the new repsitories,
+        # that way no spurious builds are launched
+        print("Updating enabled repositories for all packages")
+        mpcdf_enable_repositories_for_all_packages(api_url, project)
+
+        # Update repositories
+        print("Updating prj meta")
+        osc.core.edit_meta("prj", project, data=new_prj, force=True)
+
+
+def mpcdf_enable_repositories_for_all_packages(api_url, project, ignore_repos=()):
+    import threading
     packages = osc.core.meta_get_packagelist(api_url, project)
 
     if len(packages) > 40:
@@ -670,7 +751,9 @@ Macros:
 
     def work(packagelist):
         for package in packagelist:
-                    if not mpcdf_enable_repositories(api_url, project, package, filter_repos=filter_repos):
+            try:
+                mpcdf_enable_repositories(api_url, project, package, ignore_repos=ignore_repos)
+            except UnmanagedPackageException:
                 print("ATTENTION: Not changing unmanaged package {0}".format(package))
 
     threads = []
@@ -682,12 +765,6 @@ Macros:
     for t in threads:
         t.join()
 
-        # Update repositories
-        print("Updating prj meta")
-        osc.core.edit_meta("prj", project, data=prj, force=True)
-        print("Updating prjconf meta")
-        osc.core.edit_meta("prjconf", project, data=prjconf)
-
 
 def set_as_branch(api_url, my_project, my_package, main_project, main_package):
     import os
@@ -729,7 +806,7 @@ def set_as_branch(api_url, my_project, my_package, main_project, main_package):
     return True
 
 
-def sync_projects(api_url, package=None, from_project="software", to_projects=None, redo_all=False, add_to_maintainers=True):
+def sync_projects(api_url, package=None, from_project="software", to_projects=None, redo_all=False, add_to_maintainers=True, verbose=False):
     if package is not None and redo_all:
         raise osc.oscerr.WrongArgs('Cannot specify `redo_all` and package')
 
@@ -749,13 +826,15 @@ def sync_projects(api_url, package=None, from_project="software", to_projects=No
         else:
             from_packages = [package]
 
-        for attribute in config_attributes:
-            try:
-                get_attribute(api_url, to_project, None, attribute)
-            except Exception:
-                attr = get_attribute(api_url, from_project, None, attribute)
-                print("Setting attribute", attribute)
-                set_attribute(api_url, to_project, None, attr)
+        def non_system_package(package):
+            enable_repos = get_attribute_values(api_url, from_project, package, "MPCDF:enable_repositories")
+            if enable_repos == ["system"]:
+                print("Not branching package {0}, is only enabled for 'System'".format(package))
+                return False
+            else:
+                return True
+
+        from_packages = list(filter(non_system_package, from_packages))
 
         for orig_package in from_packages:
             if orig_package not in to_packages:
@@ -795,15 +874,17 @@ def sync_projects(api_url, package=None, from_project="software", to_projects=No
         if package is None and redo_all:
             # Check if distribution is already set in to_project
             to_prj_meta = project_meta(api_url, to_project)
-            sys_repo = to_prj_meta.find('./repository[@name="System"]')
-            if sys_repo is None:
-                distribution = project_meta(api_url, from_project).find('./repository[@name="System"]/path[@project="distributions"]').get("repository")
+            some_repo = to_prj_meta.find('./repository')
+            if some_repo is None:
+                distribution = project_meta(api_url, from_project).find('./repository/path[@project="software"]').get("repository")
             else:
-                distribution = sys_repo.find('./path[@project="distributions"]').get("repository")
+                distribution = some_repo.find('./path[@project="distributions"]').get("repository")
 
             print("Creating repository configuration")
-            mpcdf_setup_repositories(api_url, to_project, distribution=distribution)
-        else:
+            mpcdf_setup_subproject(api_url, to_project, distribution=distribution)
+
         for orig_package in from_packages:
-                if not mpcdf_enable_repositories(api_url, to_project, orig_package):
+            try:
+                mpcdf_enable_repositories(api_url, to_project, orig_package, verbose=verbose)
+            except UnmanagedPackageException:
                 print("ATTENTION: Not changing unmanaged package {0}".format(orig_package))

mpcdf_enable_repositories.py

@@ -90,8 +90,8 @@ def do_mpcdf_enable_repositories(self, subcmd, opts, *args):
 
     if opts.recreate or opts.set or opts.disable:
         mpcdf_common.mpcdf_enable_repositories(api_url, project, package, verbose=True)
-        if project == "software":
-            mpcdf_common.sync_projects(api_url, package)
+        if project == "software" and opts.set != "system":
+            mpcdf_common.sync_projects(api_url, package, verbose=True)
 
     elif (opts.compiler_modules or opts.mpi_modules or opts.cuda_modules):
         print("ERROR: Invalid arguments, try --help")

mpcdf_refresh_aggregates.py

-#!/usr/bin/python2
-from __future__ import print_function
-
-import os
-import osc
-import osc.conf
-import osc.core
-import osc.cmdln
-
-import mpcdf_common
-
-from xml.etree import ElementTree
-
-
-def do_mpcdf_refresh_aggregates(self, subcmd, opts, *args):
-    """${cmd_name}: Recreates the aggregate packages in software:dist
-
-    Use this only if you know what this means
-
-    Usage:
-        osc ${cmd_name}
-
-    ${cmd_option_list}
-
-    """
-
-    if len(args) != 0:
-        raise osc.oscerr.WrongArgs("Too many arguments")
-
-    if osc.core.store_read_project(os.curdir) != "software:dist" or \
-            osc.core.is_package_dir(os.curdir):
-        raise osc.oscerr.WrongArgs("You must be in the project checkout of software:dist")
-
-    dist = osc.core.Project(".", False)
-
-    apiurl = self.get_api_url()
-
-    def refresh_aggregate(aggregatename, project, source, target):
-        if not os.path.exists(aggregatename):
-            package = osc.core.Package.init_package(apiurl, "software:dist", aggregatename, aggregatename)
-            dist.addPackage(aggregatename)
-        else:
-            package = dist.get_pacobj(aggregatename)
-
-        with open(aggregatename + "/_aggregate", "w") as fd:
-            print('<aggregatelist>', file=fd)
-            print('  <aggregate project="{project}">'.format(project=project), file=fd)
-            print('    <nosources/>', file=fd)
-            print('    <repository target="{target}" source="{source}" />'.format(target=target, source=source), file=fd)
-            print('  </aggregate>', file=fd)
-            print('</aggregatelist>', file=fd)
-
-        if package.status("_aggregate") == "?":
-            package.addfile("_aggregate")
-
-        if package.status("_aggregate") in ("A", "M"):
-            dist.commit((aggregatename,), msg="Refresh aggregate", verbose=True)
-
-        pkg_meta = osc.core.show_package_meta(apiurl, "software:dist", aggregatename)
-        root = ElementTree.fromstringlist(pkg_meta)
-        pkg_meta = ElementTree.tostring(root, encoding=osc.core.ET_ENCODING)
-
-        build = root.find("./build")
-        if build is None:
-            build = ElementTree.SubElement(root, "build")
-
-        for enable in build.findall("./enable"):
-            build.remove(enable)
-
-        node = ElementTree.Element("enable")
-        node.set("repository", target)
-        node.tail = "\n  "
-        build.insert(0, node)
-
-        new_pkg_meta = ElementTree.tostring(root, encoding=osc.core.ET_ENCODING)
-        if pkg_meta != new_pkg_meta:
-            osc.core.edit_meta("pkg", ("software:dist", aggregatename),
-                               data=new_pkg_meta)
-
-    projects = [p for p in osc.core.meta_get_project_list(apiurl)
-                if p.startswith("software") and not (p == "software:dist" or p == "software:images")]
-
-    macros = {}
-    for project in projects:
-        if project == "software":
-            # Stupid special case
-            target = "SLE_12_SP3-sandybridge"
-        else:
-            target = project[len("software:"):].replace(":", "-")
-
-        macros[target] = []
-        for attribute in mpcdf_common.config_attributes + mpcdf_common.default_attributes:
-            _, name = attribute.split(":")
-            values = mpcdf_common.get_attribute_values(apiurl, project, None, attribute)
-            macros[target].append("%available_{0} {1}".format(name, ",".join(sorted(values))))
-
-        all_compilers = mpcdf_common.get_attribute_values(apiurl, project, None, "MPCDF:compiler_modules")
-        latest_intel = sorted((c for c in all_compilers if c.startswith("intel")), key=mpcdf_common.package_sort_key)[-1]
-        latest_gcc = sorted((c for c in all_compilers if c.startswith("gcc")), key=mpcdf_common.package_sort_key)[-1]
-        macros[target].append("%latest_intel " + latest_intel)
-        macros[target].append("%latest_gcc " + latest_gcc)
-
-        for repo in osc.core.get_repositories_of_project(apiurl, project):
-            aggregatename = "zz_aggregate_" + project.replace(":", "-") + "_" + repo
-            refresh_aggregate(aggregatename, project, repo, target)
-
-    prjconf = list(map(mpcdf_common.decode_it, osc.core.show_project_conf(apiurl, "software:dist")))
-    start_marker = "# Autogenerated by osc mpcdf_refresh_aggregates, do not edit till end of section\n"
-    end_marker = "# End of autogenerated section\n"
-
-    try:
-        start = prjconf.index(start_marker)
-        end = prjconf.index(end_marker)
-    except ValueError:
-        start = None
-        end = len(prjconf)
-
-    prjconf_head = "".join(prjconf[:start])
-    prjconf_tail = "".join(prjconf[end + 1:])
-    prjconf = [start_marker]
-
-    distributions = osc.core.get_repositories_of_project(apiurl, "distributions")
-    for swdistrepo in osc.core.get_repositories_of_project(apiurl, "software:dist"):
-        for distrepo in distributions:
-            if swdistrepo.startswith(distrepo):
-                refresh_aggregate("zz_aggregate_" + swdistrepo + "_distributions", "distributions", distrepo, swdistrepo)
-                break
-            distrepo = None
-        prjconf.append('%if "%_repository" == "{0}"'.format(swdistrepo))
-        prjconf.append("Macros:")
-        prjconf.extend(macros[swdistrepo])
-        prjconf.append("%distribution {0}".format(distrepo))
-        prjconf.append(":Macros")
-        prjconf.append("%endif")
-        prjconf.append("")
-
-    prjconf.append(end_marker)
-    prjconf = prjconf_head + "\n".join(prjconf) + prjconf_tail
-    osc.core.edit_meta("prjconf", "software:dist", data=prjconf)

mpcdf_setup_clusters_project.py

+from __future__ import print_function
+
+import osc
+import osc.conf
+import osc.core
+import osc.cmdln
+
+import mpcdf_common
+
+
+@osc.cmdln.alias("mpcdf_setup_clusters")
+def do_mpcdf_setup_cluster_project(self, subcmd, opts, *args):
+    """${cmd_name}: Set-up the 'clusters' project
+
+    This creates the 'prjconf' for the 'clusters' project
+
+    Usage:
+        osc ${cmd_name}
+
+    ${cmd_option_list}
+
+    """
+
+    if len(args) != 0:
+        raise osc.oscerr.WrongArgs("Too many arguments")
+
+    api_url = self.get_api_url()
+
+    projects = [p for p in osc.core.meta_get_project_list(api_url)
+                if p.startswith("software:")]
+
+    macros = {}
+    for project in projects:
+        target = project[len("software:"):].replace(":", "-")
+
+        if target in macros:
+            raise Exception("Internal error")
+
+        macros[target] = []
+        for attribute in mpcdf_common.config_attributes + mpcdf_common.default_attributes:
+            _, name = attribute.split(":")
+            values = mpcdf_common.overloaded_project_attribute(api_url, project, attribute)
+            macros[target].append("%available_{0} {1}".format(name, ",".join(sorted(values))))
+
+        all_compilers = mpcdf_common.overloaded_project_attribute(api_url, project, "MPCDF:compiler_modules")
+        latest_intel = sorted((c for c in all_compilers if c.startswith("intel")), key=mpcdf_common.package_sort_key)[-1]
+        latest_gcc = sorted((c for c in all_compilers if c.startswith("gcc")), key=mpcdf_common.package_sort_key)[-1]
+        macros[target].append("%latest_intel " + latest_intel)
+        macros[target].append("%latest_gcc " + latest_gcc)
+
+    prjconf = list(map(mpcdf_common.decode_it, osc.core.show_project_conf(api_url, "clusters")))
+    start_marker = "# Autogenerated by osc mpcdf_setup_clusters_project, do not edit till end of section\n"
+    end_marker = "# End of autogenerated section\n"
+
+    try:
+        start = prjconf.index(start_marker)
+        end = prjconf.index(end_marker)
+    except ValueError:
+        start = None
+        end = len(prjconf)
+
+    prjconf_head = "".join(prjconf[:start])
+    prjconf_tail = "".join(prjconf[end + 1:])
+    prjconf = [start_marker]
+
+    distributions = osc.core.get_repositories_of_project(api_url, "software")
+    for repo in osc.core.get_repositories_of_project(api_url, "clusters"):
+        for dist in distributions:
+            if repo.startswith(dist):
+                break
+            dist = None
+        if dist is None:
+            raise Exception("Internal error")
+        prjconf.append('%if "%_repository" == "{0}"'.format(repo))
+        prjconf.append("Macros:")
+        prjconf.extend(macros[repo])
+        prjconf.append("%distribution {0}".format(dist))
+        prjconf.append(":Macros")
+        prjconf.append("%endif")
+        prjconf.append("")
+
+    prjconf.append(end_marker)
+    prjconf = prjconf_head + "\n".join(prjconf) + prjconf_tail
+    osc.core.edit_meta("prjconf", "clusters", data=prjconf)

mpcdf_setup_home_project.py

+#!/usr/bin/python2
+from __future__ import print_function
+
+from mpcdf_common import mpcdf_setup_subproject, project_meta, decode_it
+
+import os
+import osc
+import osc.conf
+import osc.core
+import osc.cmdln
+
+@osc.cmdln.option('-n', '--dry-run', action="store_true",
+                  help="Do not actually run anything but output the resulting XML configuration")
+@osc.cmdln.option('--remove-old', action="store_true", default=False,
+                  help="Remove all obsolete repositories instead of only disabling builds for packages there")
+@osc.cmdln.option('--distribution',
+                  help="Base distribution, necessary argument unless set previously for this project")
+@osc.cmdln.option('--microarchitecture', metavar="ARCH", nargs=1,
+                  help="Configure project to use ARCH as microarchitecture")
+@osc.cmdln.alias("mpcdf_setup_home")
+def do_mpcdf_setup_home_project(self, subcmd, opts, *args):
+    """${cmd_name}: Setup a home project based on a software: sub-project
+
+    Set-up a home:$USER project for building packages for the 'software'
+    project hierarchy (i.e. application group packages)
+
+    Usage:
+        osc ${cmd_name} [home:USER]
+
+    ${cmd_option_list}
+
+    """
+
+    if len(args) == 0:
+        if osc.core.is_project_dir(os.curdir) or osc.core.is_package_dir(os.curdir):
+            project = osc.core.store_read_project(os.curdir)
+        else:
+            raise osc.oscerr.WrongArgs('Specify PROJECT or run command in an osc checkout directory')
+
+    elif len(args) == 1:
+        project, = args
+    else:
+        raise osc.oscerr.WrongArgs("Too many arguments")
+
+    if project.split(":")[0] != "home":
+        raise osc.oscerr.WrongArgs('Given project is not a home: project')
+
+    api_url = self.get_api_url()
+
+    if opts.distribution is None:
+        # Get existing value from project meta
+        dist_repo = project_meta(api_url, project).find(
+            "./repository[@name='System']/path")
+        if dist_repo is not None:
+            distribution = dist_repo.get("repository")
+        else:
+            raise osc.oscerr.WrongArgs('Could not determine desired distribution, please specify --distribution explicitly')
+    else:
+        distribution = opts.distribution
+
+    microarchitecture = None
+    if opts.microarchitecture is None:
+        for prjconfline in map(decode_it, osc.core.show_project_conf(api_url, project)):
+            if prjconfline.startwith("Constraint: hostlabel"):
+                microarchitecture = prjconfline.split()[2]
+                break
+    else:
+        microarchitecture = opts.microarchitecture
+
+    if microarchitecture is None:
+        raise osc.oscerr.WrongArgs('Could not determine desired microarchitecture, please specify --microarchitecture explicitly')
+
+    parent = "software:{0}:{1}".format(distribution, microarchitecture)
+
+    mpcdf_setup_subproject(api_url,
+                           project, distribution, microarchitecture,
+                           parent=parent, dry_run=opts.dry_run, remove_old=opts.remove_old)

mpcdf_setup_software_project.py

+#!/usr/bin/python2
+from __future__ import print_function
+
+import osc
+import osc.conf
+import osc.core
+import osc.cmdln
+
+import mpcdf_common
+
+
+@osc.cmdln.option('-i', '--ignore-repo', action="append", default=[], metavar="REPO",
+                  help="Do not enable for repository REPO")
+@osc.cmdln.alias("mpcdf_setup_software")
+def do_mpcdf_setup_software_project(self, subcmd, opts, *args):
+    """${cmd_name}: Set-up the 'software' project
+
+    This creates the 'prjconf' for the 'software' project and
+    enables the repositories for packages set to build for
+    'system'
+
+    Usage:
+        osc ${cmd_name}
+
+    ${cmd_option_list}
+
+    """
+
+    if len(args) != 0:
+        raise osc.oscerr.WrongArgs("Too many arguments")
+
+    api_url = self.get_api_url()
+
+    prjconf = list(map(mpcdf_common.decode_it, osc.core.show_project_conf(api_url, "software")))
+    start_marker = "# Autogenerated by osc mpcdf_setup_software_project, do not edit till end of section\n"
+    end_marker = "# End of autogenerated section\n"
+
+    try:
+        start = prjconf.index(start_marker)
+        end = prjconf.index(end_marker)
+    except ValueError:
+        start = None
+        end = len(prjconf)
+
+    prjconf_head = "".join(prjconf[:start])
+    prjconf_tail = "".join(prjconf[end + 1:])
+    prjconf = [start_marker]
+
+    prjconf.append("Constraint: sandbox lxc")
+    prjconf.append("Constraint: hostlabel sandybridge")
+    prjconf.append("")
+    prjconf.append(r"PublishFilter: ^.*\.src\.rpm$")
+    prjconf.append("")
+    prjconf.append("Preinstall: mpcdf")
+    prjconf.append("Preinstall: mpcdf_modules")
+    prjconf.append("Preinstall: brp_mpcdf_modules")
+
+    prjconf.append(end_marker)
+    prjconf = prjconf_head + "\n".join(prjconf) + prjconf_tail
+    osc.core.edit_meta("prjconf", "software", data=prjconf)
+
+    if opts.ignore_repo:
+        ignore_repos = opts.ignore_repo
+    else:
+        ignore_repos = ()
+
+    mpcdf_common.mpcdf_enable_repositories_for_all_packages(api_url, "software", ignore_repos=ignore_repos)

mpcdf_setup_repositories.py → mpcdf_setup_subproject.py

 #!/usr/bin/python2
 from __future__ import print_function
 
-from mpcdf_common import mpcdf_setup_repositories
+from mpcdf_common import mpcdf_setup_subproject
 
 import os
 import osc
@@ -9,30 +9,23 @@ import osc.conf
 import osc.core
 import osc.cmdln
 
-
 @osc.cmdln.option('-n', '--dry-run', action="store_true",
                   help="Do not actually run anything but output the resulting XML configuration")
-@osc.cmdln.option('--parent', metavar="PARENT",
-                  help="Setup the repositories to be based on the upstream project PARENT (e.g. for home: projects)")
-@osc.cmdln.option('--distribution',
-                  help="Base distribution, necessary argument unless set previously for this project")
-@osc.cmdln.option('--disable-repo', metavar="REPO", action="append",
-                  help="Temporarily disable all repositories containing REPO")
-@osc.cmdln.option('--only-project', action="store_true", default=False,
-                  help="Only change project metadata 'prj' and 'prjconf', leave individual packages unchanged")
 @osc.cmdln.option('--remove-old', action="store_true", default=False,
                   help="Remove all obsolete repositories instead of only disabling builds for packages there")
-@osc.cmdln.option('--microarchitecture', metavar="ARCH", nargs=1,
-                  help="Configure project to use ARCH as microarchitecture")
-@osc.cmdln.alias("mpcdf_setup_repos")
-def do_mpcdf_setup_repositories(self, subcmd, opts, *args):
-    """${cmd_name}: Create all repository combinations for an MPCDF project
+@osc.cmdln.alias("mpcdf_setup_sub")
+def do_mpcdf_setup_subproject(self, subcmd, opts, *args):
+    """${cmd_name}: Setup a software: sub-project
+
+    Given a list of compilers, MPI libraries, and possibly CUDA versions via
+    the projects attributes, this command creates repositories for all the
+    resulting combinations
 
-    Given a list of compilers, MPI libraries, and possibly CUDA versions, this command
-    creates repositories for all the resulting combinations
+    Additionally, the prjconf is set-up to enforce the microarchitecture via
+    a "Constraint:" statement
 
     Usage:
-        osc ${cmd_name} [PROJECT]
+        osc ${cmd_name} [software:OS:microarchitecture]
 
     ${cmd_option_list}
 
@@ -49,7 +42,11 @@ def do_mpcdf_setup_repositories(self, subcmd, opts, *args):
     else:
         raise osc.oscerr.WrongArgs("Too many arguments")
 
-    mpcdf_setup_repositories(self.get_api_url(),
-                             project, microarchitecture=opts.microarchitecture, distribution=opts.distribution,
-                             parent=opts.parent, dry_run=opts.dry_run, filter_repos=opts.disable_repo,
-                             only_project=opts.only_project, remove_old=opts.remove_old)
+    sw, distribution, microarchitecture = project.split(":")
+
+    if sw != "software":
+        raise osc.oscerr.WrongArgs('Given project is not below the "software" project')
+
+    mpcdf_setup_subproject(self.get_api_url(),
+                           project, distribution, microarchitecture,
+                           dry_run=opts.dry_run, remove_old=opts.remove_old)