New commands `mpcdf_info`, `mpcdf_enable_repositories`

.gitignore

+*.pyc

mpcdf_common.py

+from __future__ import print_function
+
+__all__ = ["valid_mpi", "valid_cuda", "get_attribute", "mpcdf_enable_repositories"]
+
+import osc
+import osc.conf
+import osc.core
+
+from xml.etree import ElementTree
+
+
+def valid_mpi(compiler, mpi):
+    """
+    It might be possible to use Intel MPI libararies and compilers from
+    different Parallel Studio packages, but I currently do not want to support
+    it
+    """
+    if compiler == "intel_17_0":
+        if mpi.startswith("impi"):
+            return mpi == "impi_2017_3"
+    if compiler == "intel_16_0":
+        if mpi.startswith("impi"):
+            return mpi == "impi_5_1_3"
+    return True
+
+
+def valid_cuda(cuda, compiler):
+    """
+    The CUDA distribution only works with certain gcc versions,
+    this little function takes care that only supported combinations
+    are allowed
+    """
+    if cuda == "cuda_8":
+        return compiler == "gcc_5"
+    return False
+
+
+def get_attribute(api_url, project, package, attribute, with_project=False):
+
+    attribute_meta = osc.core.show_attribute_meta(api_url, project, package, None, attribute, False, with_project)
+    if attribute_meta is None:
+        raise oscerr.APIError("Cannot fetch value for attribute '{0}' from {1}".format(attribute, (project, package)))
+
+    root = ElementTree.fromstringlist(attribute_meta)
+    attribute = root.find("./attribute")
+    if attribute is not None:
+        return list(value.text for value in attribute.findall("./value"))
+    else:
+        raise Exception("Attribute not set")
+
+
+def mpcdf_enable_repositories(api_url, project, package, verbose=False):
+
+    root = ElementTree.fromstringlist(osc.core.show_package_meta(api_url, project, package))
+    build = root.find("./build")
+    for enable in build.findall("./enable"):
+        build.remove(enable)
+
+    compilers = get_attribute(api_url, project, package, "MPCDF:compiler_modules", with_project=True)
+    mpis = get_attribute(api_url, project, package, "MPCDF:mpi_modules", with_project=True)
+    cudas = get_attribute(api_url, project, package, "MPCDF:cuda_modules", with_project=True)
+
+    def enable(name):
+        node = ElementTree.Element("enable")
+        node.set("repository", name)
+        node.tail = "\n    "
+        build.insert(0, node)
+        if verbose:
+            print("Enabling", name)
+
+    try:
+        enable_repos = get_attribute(api_url, project, package, "MPCDF:enable_repositories")
+    except:
+        if verbose:
+            print("Warning: Could not get attribute MPCDF:enable_repositories for package {0}".format(package))
+        return False
+
+    for flag in enable_repos:
+
+        if flag == "system":
+            enable("System")
+
+        if flag == "compilers":
+            for compiler in compilers:
+                enable(compiler)
+
+        if flag == "mpi":
+            for compiler in compilers:
+                for mpi in mpis:
+                    if valid_mpi(compiler, mpi):
+                       enable(mpi + "_" + compiler)
+
+        if flag == "cuda":
+            for cuda in cudas:
+                for compiler in compilers:
+                    if valid_cuda(cuda, compiler):
+                        enable(cuda + "_" + compiler)
+
+        if flag == "cuda_mpi":
+            for cuda in cudas:
+                for compiler in compilers:
+                    if valid_cuda(cuda, compiler):
+                        for mpi in mpis:
+                            if valid_mpi(compiler, mpi):
+                                enable(cuda + "_" + compiler)
+
+    build.getchildren()[-1].tail = "\n  "
+
+    pkg_meta = ET.tostring(root, encoding=osc.core.ET_ENCODING)
+
+    osc.core.edit_meta("pkg", (project, package), data=pkg_meta)
+    return True

mpcdf_enable_repositories.py

+#!/usr/bin/python2
+from __future__ import print_function
+import sys
+import argparse
+from functools import partial
+from xml.etree import ElementTree
+
+from mpcdf_common import *
+
+import osc
+import osc.conf
+import osc.core
+import osc.cmdln
+
+@osc.cmdln.option('--cuda-mpi', action="store_true",
+                  help="Enable for all CUDA+MPI repositories")
+@osc.cmdln.option('--cuda', action="store_true",
+                  help="Enable for all CUDA repositories")
+@osc.cmdln.option('--mpi', action="store_true",
+                  help="Enable for all MPI module repositories")
+@osc.cmdln.option('--compilers', action="store_true",
+                  help="Enable for all compiler module repositories")
+@osc.cmdln.option('--system', action="store_true",
+                  help="Enable for System repository")
+@osc.cmdln.option('--reset', action="store_true",
+                  help="Re-create the set of enabled repositories from the MPCDF:enable_repositores attribute")
+@osc.cmdln.option('--set', action="store_true",
+                  help="Modify the set of enabled repositories, without this the current setup is displayed")
+def do_mpcdf_enable_repositories(self, subcmd, opts, *args):
+    """${cmd_name}: Select all appropriate repositories for an MPCDF package
+
+    Due to the large number of repository combinations at MPCDF it is
+    cumbersome to enable all the appropriate repositories for a given package
+    by hand.
+
+    This command allows you to set the <enable/> flags for certain kinds of
+    repositories at once.
+
+    Without --set this command only displays the current configuration.
+
+    It is possible to combine this with user-defined <disabled/> flags that
+    override the settings of this command, for example to disable the build of
+    the package with a certain repository from a given set (e.g. one
+    troublesome compiler)
+
+    Usage:
+        osc ${cmd_name}                 [[PROJECT] PACKAGE]
+        osc ${cmd_name} --set [ flags ] [[PROJECT] PACKAGE]
+        osc ${cmd_name} --reset         [[PROJECT] PACKAGE]
+
+    ${cmd_option_list}
+    """
+
+    if len(args) == 0:
+        if is_package_dir(os.curdir):
+            package = osc.core.store_read_package(os.curdir)
+            project = osc.core.store_read_project(os.curdir)
+        else:
+            raise oscerr.WrongArgs('Specify PACKAGE or run command in an osc package checkout directory')
+
+    elif len(args) == 1:
+        package, = args
+        project = osc.core.store_read_project(os.curdir)
+
+    elif len(args) == 2:
+        project, package = args
+    else:
+        raise oscerr.WrongArgs("Too many arguments")
+
+    api_url = self.get_api_url()
+
+    if opts.set:
+        root = ElementTree.fromstringlist(osc.core.show_attribute_meta(api_url, project, package, None, "MPCDF:enable_repositories", False, False))
+
+        attribute = root.find("./attribute")
+        if attribute is None:
+            attribute = ElementTree.SubElement(root, "attribute")
+            attribute.set("namespace", "MPCDF")
+            attribute.set("name", "enable_repositories")
+
+        for value in attribute.findall("./value"):
+            attribute.remove(value)
+
+        def value(content):
+            v = ElementTree.SubElement(attribute, "value")
+            v.text = content
+
+        if opts.system:
+            value("system")
+        if opts.compilers:
+            value("compilers")
+        if opts.mpi:
+            value("mpi")
+        if opts.cuda:
+            value("cuda")
+        if opts.cuda_mpi:
+            value("cuda_mpi")
+
+        attr_meta = ET.tostring(root, encoding=osc.core.ET_ENCODING)
+
+        url = osc.core.makeurl(api_url, ["source", project, package, "_attribute"])
+        resp = ElementTree.fromstringlist(osc.core.streamfile(url, http_POST, data=attr_meta))
+        if resp.find("./summary").text != "Ok":
+            raise osc.core.oscerr.APIError("Could not store attribute MPCDF:enabled_repositories")
+
+    if opts.reset or opts.set:
+        mpcdf_enable_repositories(api_url, project, package, verbose=True)
+    else:
+        print("Enabled for:", *get_attribute(api_url, project, package, "MPCDF:enable_repositories"))

mpcdf_info.py

+#!/usr/bin/python2
+from __future__ import print_function
+import sys
+import argparse
+
+from mpcdf_common import *
+
+import osc
+import osc.conf
+import osc.core
+import osc.cmdln
+
+def do_mpcdf_info(self, subcmd, opts, *args):
+    """${cmd_name}: Basic information about an MPCDF OBS project
+
+    Usage:
+        osc ${cmd_name} [PROJECT]
+
+    ${cmd_option_list}
+
+    """
+
+    if len(args) == 0:
+        if is_project_dir(os.curdir) or is_package_dir(os.curdir):
+            project = osc.core.store_read_project(os.curdir)
+        else:
+            raise oscerr.WrongArgs('Specify PROJECT or run command in an osc checkout directory')
+
+    elif len(args) == 1:
+        project, = args
+    else:
+        raise oscerr.WrongArgs("Too many arguments")
+
+    print("Project `{0}`\n".format(project))
+
+    api_url = self.get_api_url()
+
+    def print_attribute(description, attribute):
+        values = list(sorted(get_attribute(api_url, project, None, attribute)))
+        if values:
+            print(" {0}:\n   ".format(description) + "\n   ".join(values))
+            print()
+
+    print_attribute("Compilers", "MPCDF:compiler_modules")
+    print_attribute("MPI libraries", "MPCDF:mpi_modules")
+    print_attribute("CUDA versions", "MPCDF:cuda_modules")
+
+    table = []
+    unmanaged = []
+
+    for package in osc.core.meta_get_packagelist(api_url, project):
+        try:
+            table.append((package, set(get_attribute(api_url, project, package, "MPCDF:enable_repositories"))))
+        except:
+            unmanaged.append(package)
+
+    pkg_name_width = reduce(max, (len(t[0]) for t in table))
+    pkg_name_width = max(pkg_name_width, len("Package"))
+
+    r = osc.core.http_request("GET", api_url + "/attribute/MPCDF/enable_repositories/_meta")
+    root = ET.parse(r).getroot()
+    kinds = list(value.text for value in root.findall("./allowed/value"))
+
+    pkg_name_fmt = "{{:{0}}}".format(pkg_name_width)
+
+    print(" " + pkg_name_fmt.format("Package"), "Enabled repositories")
+    print(" " + "-" * (pkg_name_width), "-" * len("Enabled repositories"))
+
+    for package, enabled_repo_kind in table:
+        print(" " + pkg_name_fmt.format(package), ", ".join(filter(lambda q : q in enabled_repo_kind, kinds)))
+    print()
+
+    if unmanaged:
+        print(" Unmanaged packages")
+        print(" " + "-" * (1 + pkg_name_width))
+        for package in unmanaged:
+            print(" " + package)
+        print()

mpcdf-repositories.py → mpcdf_setup_repos.py

@@ -2,54 +2,28 @@
 from __future__ import print_function
 import sys
 import argparse
+from functools import partial
 from xml.etree import ElementTree
 
+from mpcdf_common import *
+
 import osc
 import osc.conf
 import osc.core
 import osc.cmdln
 
-all_compilers = "gcc_4_9 gcc_5 gcc_6 gcc_7 intel_16_0 intel_17_0"
-all_mpis = "openmpi_1_10 impi_5_1_3 impi_2017_3"
-all_cudas = "cuda_8"
 default_distribution = "SLE_12_SP1"
-default_architectures = "x86_64"
-
-def valid_mpi(compiler, mpi):
-    if compiler == "intel_17_0":
-        if mpi.startswith("impi"):
-            return mpi == "impi_2017_3"
-    if compiler == "intel_16_0":
-        if mpi.startswith("impi"):
-            return mpi == "impi_5_1_3"
-    return True
-
-@osc.cmdln.option('--distribution',
-                  default=default_distribution,
-                  help="Base distribution [default: %default]")
-
-@osc.cmdln.option('--architectures',
-                  default=default_architectures,
-                  help="Architectures [default: %default]")
-
-@osc.cmdln.option('--compilers',
-                  default=all_compilers,
-                  help="List of compilers, [default: %default]")
 
-@osc.cmdln.option('--mpis',
-                  default=all_mpis,
-                  help="List of MPI libraries, [default: %default]")
-
-@osc.cmdln.option('--cudas',
-                  default=all_cudas,
-                  help="List of CUDA versions, [default: %default]")
+@osc.cmdln.option('-n', '--dry-run', action="store_true",
+                  help="Do not actually run anything but output the resulting XML configuration")
 
 @osc.cmdln.option('--parent', metavar="PARENT",
                   help="Setup the repositories to be based on the upstream project PARENT (e.g. for home: projects)")
 
-@osc.cmdln.option('-n', '--dry-run', action="store_true",
-                  help="Do not actually run anything but output the resulting XML configuration")
-def do_mpcdf_repositories(self, subcmd, opts, *args):
+@osc.cmdln.option('--distribution',
+                  default=default_distribution,
+                  help="Base distribution [default: %default]")
+def do_mpcdf_setup_repos(self, subcmd, opts, *args):
     """${cmd_name}: Create all repository combinations for an MPCDF project
 
     Given a list of compilers, MPI libraries, and possibly CUDA versions, this command
@@ -73,19 +47,47 @@ def do_mpcdf_repositories(self, subcmd, opts, *args):
     else:
         raise oscerr.WrongArgs("Too many arguments")
 
-    compilers = opts.compilers.split()
-    architectures = opts.architectures.split()
-    mpis = opts.mpis.split()
-    cudas = opts.cudas.split()
+    api_url = self.get_api_url()
+
+    compilers = list(get_attribute(api_url, project, None, "MPCDF:compiler_modules"))
+    mpis = list(get_attribute(api_url, project, None, "MPCDF:mpi_modules"))
+    cudas = list(get_attribute(api_url, project, None, "MPCDF:cuda_modules"))
+
+    def project_meta(project):
+        return ElementTree.fromstringlist(osc.core.show_project_meta(api_url, project))
+
+    distributions = project_meta("distributions")
+    dist_repo = distributions.find('./repository[@name="{0}"]'.format(opts.distribution))
+    if dist_repo is None:
+        raise oscerr.WrongArgs("No repository '{0}' is defined in project 'distributions' on the server".format(opts.distribution))
+    architectures = list(arch.text for arch in dist_repo.findall("./arch"))
+
+    root = project_meta(project)
+
+    prjconf = osc.core.show_project_conf(api_url, project)
+    start_marker = "# Autogenerated by osc mpcdf_setup_repos, do not edit till end of section\n"
+    end_marker = "# End of autogenerated section\n"
+
+    try:
+        start = prjconf.index(start_marker)
+        end = prjconf.index(end_marker)
+    except ValueError:
+        start = None
+        end = len(prjconf)
 
-    project_meta = osc.core.show_project_meta(osc.conf.config["apiurl"], project)
-    root = ElementTree.fromstringlist(project_meta)
+    prjconf_head = "".join(prjconf[:start])
+    prjconf_tail = "".join(prjconf[end+1:])
+    prjconf = [start_marker]
 
     # Remove existing repositories
     for repo in root.findall("./repository"):
         root.remove(repo)
 
-    def repo(name, *dependencies):
+    def repo(name, *dependencies, **kwargs):
+        is_mpi = kwargs.pop("mpi", False)
+        is_cuda = kwargs.pop("cuda", False)
+        if kwargs:
+            raise Exception("Invalid argument")
         r = ElementTree.SubElement(root, "repository")
         r.set("name", name)
         r.text="\n    "
@@ -107,32 +109,56 @@ def do_mpcdf_repositories(self, subcmd, opts, *args):
         a.tail = "\n  "
         r.tail = "\n  "
 
+        prjconf.append("%if %_repository == {0}".format(name))
+        prjconf.append("Macros:")
+        prjconf.append("%is_mpi_repository {0}".format(1 if is_mpi else 0))
+        prjconf.append("%is_cuda_repository {0}".format(1 if is_cuda else 0))
+        prjconf.append(":Macros")
+        prjconf.append("%endif")
+        prjconf.append("")
+
+
     repo("System", ("distributions", opts.distribution))
 
-    for compiler in sorted(compilers):
+    for compiler in compilers:
         repo(compiler, (project, "System"))
 
-    for mpi in sorted(mpis):
-        for compiler in sorted(compilers):
-            if not valid_mpi(compiler, mpi):
-                continue
-            repo(mpi + "_" + compiler, (project, compiler))
-
-    for cuda in sorted(cudas):
-        repo(cuda, (project, "System"))
-        for compiler in sorted(compilers):
-            repo(cuda + "_" + compiler, (project, cuda), (project, compiler))
-        for mpi in sorted(mpis):
-            for compiler in sorted(compilers):
-                if not valid_mpi(compiler, mpi):
-                    continue
+        for mpi in filter(partial(valid_mpi, compiler), mpis):
+            repo(mpi + "_" + compiler, (project, compiler), mpi=True)
+
+    for cuda in cudas:
+        for compiler in filter(partial(valid_cuda, cuda), compilers):
+            repo(cuda + "_" + compiler, (project, compiler), cuda=True)
+            for mpi in filter(partial(valid_mpi, compiler), mpis):
                 repo(cuda + "_" + mpi + "_" + compiler,
                      (project, cuda + "_" + compiler),
-                     (project, mpi + "_" + compiler))
+                     (project, mpi + "_" + compiler),
+                     mpi=True, cuda=True)
 
     root.getchildren()[-1].tail = "\n"
-    meta = ET.tostring(root, encoding=osc.core.ET_ENCODING)
+    prj = ET.tostring(root, encoding=osc.core.ET_ENCODING)
+
+    prjconf.append(end_marker)
+    prjconf = prjconf_head + "\n".join(prjconf) + prjconf_tail
+
     if opts.dry_run:
-        print(meta)
+        if len(args) == 0:
+            arg = ""
+        else:
+            arg = " " + project
+        print("osc meta prj{0} -F - <<EOF\n{1}\nEOF\n".format(arg, prj))
+        print("osc meta prjconf{0} -F - <<EOF\n{1}\nEOF\n".format(arg, prjconf))
     else:
-        osc.core.edit_meta("prj", project, data=meta)
+        # First set-up the <enable/> flags, that way no
+        # spurious builds are launched
+        for package in osc.core.meta_get_packagelist(api_url, project):
+            print("Updating repositories for", package)
+            if not mpcdf_enable_repositories(api_url, project, package):
+                print("ATTENTION: Not changing unmanaged package {0}".format(package))
+
+        # Update repositories
+        print("Updating prj meta")
+        osc.core.edit_meta("prj", project, data=prj)
+        print("Updating prjconf meta")
+        osc.core.edit_meta("prjconf", project, data=prjconf)
+