Merge branch 'obs-api' into 'master'

Merge changes for new OBS from obs-api branch

See merge request !3

README.md

@@ -8,7 +8,7 @@ Put the `.py` files into `~/.osc-plugins` to make them available for the osc com
 for example by
 
 ```
-$> git clone https://gitlab.mpcdf.mpg.de/mpcdf/osc-plugins ~/.osc-plugins
+$> git clone https://gitlab.mpcdf.mpg.de/mpcdf/obs/osc-plugins ~/.osc-plugins
 ```
 
 Background

mpcdf_branch.py

-#!/usr/bin/python2
 from __future__ import print_function
 
 import mpcdf_common

mpcdf_enable_repositories.py

-#!/usr/bin/python2
 from __future__ import print_function
 
 import mpcdf_common
@@ -10,6 +9,8 @@ import osc.core
 import osc.cmdln
 
 
+@osc.cmdln.option('-n', '--dry-run', action="store_true",
+                  help="Do not actually perform any changes but print what would be stored")
 @osc.cmdln.option('--recreate', action="store_true",
                   help="Re-create the set of enabled repositories from the stored attributes on the server")
 @osc.cmdln.option('--compiler-modules', nargs=1,
@@ -89,28 +90,29 @@ def do_mpcdf_enable_repositories(self, subcmd, opts, *args):
         set_or_remove(opts.pgi_modules, "MPCDF:pgi_modules")
 
     if opts.recreate or opts.set or opts.disable:
-        mpcdf_common.mpcdf_enable_repositories(api_url, project, package, verbose=True)
-        if project == "software":
-            mpcdf_common.sync_projects(api_url, package)
+        mpcdf_common.mpcdf_enable_repositories(api_url, project, package, verbose=True, dry_run=opts.dry_run)
+        enabled = mpcdf_common.get_attribute_values(api_url, project, package, "MPCDF:enable_repositories")
+        if project == "software" and enabled != ["system"] and len(enabled) > 1 and not opts.dry_run:
+            mpcdf_common.sync_projects(api_url, package, verbose=True)
 
     elif (opts.compiler_modules or opts.mpi_modules or opts.cuda_modules):
         print("ERROR: Invalid arguments, try --help")
 
     else:
         try:
-            repos = mpcdf_common.get_attribute_values(api_url, project, package, "MPCDF:enable_repositories")
+            enabled = mpcdf_common.get_attribute_values(api_url, project, package, "MPCDF:enable_repositories")
         except Exception:
-            repos = ()
+            enabled = ()
             print("ERRROR: No attribute MPCDF:enable_repositories present, package unmanaged")
-        if repos:
+        if enabled:
             def print_attr(description, attribute_name):
                 try:
                     values = mpcdf_common.get_attribute_values(api_url, project, package, attribute_name)
                 except Exception:
                     return
-                print(description, ", ".join(values))
+                print(description, ",".join(values))
 
-            print("Enabled for:", *repos)
+            print("Enabled for:", ",".join(enabled))
             print_attr("- Subset of compiler modules set to:", "MPCDF:compiler_modules")
             print_attr("- Subset of MPI modules set to:", "MPCDF:mpi_modules")
             print_attr("- Subset of CUDA modules set to:", "MPCDF:cuda_modules")

mpcdf_push.py

-#!/usr/bin/python2
 from __future__ import print_function
 
 import time
@@ -112,7 +111,7 @@ def do_mpcdf_push(self, subcmd, opts, *args):
             person.set("userid", user)
             person.set("role", "maintainer")
             to_meta.insert(2, person)
-            osc.core.edit_meta("pkg", (to_project, package), data=ElementTree.tostring(to_meta))
+            osc.core.edit_meta("pkg", (to_project, package), data=ElementTree.tostring(to_meta), apiurl=api_url)
         print()
 
     # Give the system some time, sadly there is no transactional guarantee

mpcdf_refresh_aggregates.py

-#!/usr/bin/python2
-from __future__ import print_function
-
-import os
-import osc
-import osc.conf
-import osc.core
-import osc.cmdln
-
-import mpcdf_common
-
-from xml.etree import ElementTree
-
-
-def do_mpcdf_refresh_aggregates(self, subcmd, opts, *args):
-    """${cmd_name}: Recreates the aggregate packages in software:dist
-
-    Use this only if you know what this means
-
-    Usage:
-        osc ${cmd_name}
-
-    ${cmd_option_list}
-
-    """
-
-    if len(args) != 0:
-        raise osc.oscerr.WrongArgs("Too many arguments")
-
-    if osc.core.store_read_project(os.curdir) != "software:dist" or \
-            osc.core.is_package_dir(os.curdir):
-        raise osc.oscerr.WrongArgs("You must be in the project checkout of software:dist")
-
-    dist = osc.core.Project(".", False)
-
-    apiurl = self.get_api_url()
-
-    def refresh_aggregate(aggregatename, project, source, target):
-        if not os.path.exists(aggregatename):
-            package = osc.core.Package.init_package(apiurl, "software:dist", aggregatename, aggregatename)
-            dist.addPackage(aggregatename)
-        else:
-            package = dist.get_pacobj(aggregatename)
-
-        with open(aggregatename + "/_aggregate", "w") as fd:
-            print('<aggregatelist>', file=fd)
-            print('  <aggregate project="{project}">'.format(project=project), file=fd)
-            print('    <nosources/>', file=fd)
-            print('    <repository target="{target}" source="{source}" />'.format(target=target, source=source), file=fd)
-            print('  </aggregate>', file=fd)
-            print('</aggregatelist>', file=fd)
-
-        if package.status("_aggregate") == "?":
-            package.addfile("_aggregate")
-
-        if package.status("_aggregate") in ("A", "M"):
-            dist.commit((aggregatename,), msg="Refresh aggregate", verbose=True)
-
-        pkg_meta = osc.core.show_package_meta(apiurl, "software:dist", aggregatename)
-        root = ElementTree.fromstringlist(pkg_meta)
-        pkg_meta = ElementTree.tostring(root, encoding=osc.core.ET_ENCODING)
-
-        build = root.find("./build")
-        if build is None:
-            build = ElementTree.SubElement(root, "build")
-
-        for enable in build.findall("./enable"):
-            build.remove(enable)
-
-        node = ElementTree.Element("enable")
-        node.set("repository", target)
-        node.tail = "\n  "
-        build.insert(0, node)
-
-        new_pkg_meta = ElementTree.tostring(root, encoding=osc.core.ET_ENCODING)
-        if pkg_meta != new_pkg_meta:
-            osc.core.edit_meta("pkg", ("software:dist", aggregatename),
-                               data=new_pkg_meta)
-
-    projects = [p for p in osc.core.meta_get_project_list(apiurl)
-                if p.startswith("software") and not (p == "software:dist" or p == "software:images")]
-
-    macros = {}
-    for project in projects:
-        if project == "software":
-            # Stupid special case
-            target = "SLE_12_SP3-sandybridge"
-        else:
-            target = project[len("software:"):].replace(":", "-")
-
-        macros[target] = []
-        for attribute in mpcdf_common.config_attributes + mpcdf_common.default_attributes:
-            _, name = attribute.split(":")
-            values = mpcdf_common.get_attribute_values(apiurl, project, None, attribute)
-            macros[target].append("%available_{0} {1}".format(name, ",".join(sorted(values))))
-
-        all_compilers = mpcdf_common.get_attribute_values(apiurl, project, None, "MPCDF:compiler_modules")
-        latest_intel = sorted((c for c in all_compilers if c.startswith("intel")), key=mpcdf_common.package_sort_key)[-1]
-        latest_gcc = sorted((c for c in all_compilers if c.startswith("gcc")), key=mpcdf_common.package_sort_key)[-1]
-        macros[target].append("%latest_intel " + latest_intel)
-        macros[target].append("%latest_gcc " + latest_gcc)
-
-        for repo in osc.core.get_repositories_of_project(apiurl, project):
-            aggregatename = "zz_aggregate_" + project.replace(":", "-") + "_" + repo
-            refresh_aggregate(aggregatename, project, repo, target)
-
-    prjconf = list(map(mpcdf_common.decode_it, osc.core.show_project_conf(apiurl, "software:dist")))
-    start_marker = "# Autogenerated by osc mpcdf_refresh_aggregates, do not edit till end of section\n"
-    end_marker = "# End of autogenerated section\n"
-
-    try:
-        start = prjconf.index(start_marker)
-        end = prjconf.index(end_marker)
-    except ValueError:
-        start = None
-        end = len(prjconf)
-
-    prjconf_head = "".join(prjconf[:start])
-    prjconf_tail = "".join(prjconf[end + 1:])
-    prjconf = [start_marker]
-
-    distributions = osc.core.get_repositories_of_project(apiurl, "distributions")
-    for swdistrepo in osc.core.get_repositories_of_project(apiurl, "software:dist"):
-        for distrepo in distributions:
-            if swdistrepo.startswith(distrepo):
-                refresh_aggregate("zz_aggregate_" + swdistrepo + "_distributions", "distributions", distrepo, swdistrepo)
-                break
-            distrepo = None
-        prjconf.append('%if "%_repository" == "{0}"'.format(swdistrepo))
-        prjconf.append("Macros:")
-        prjconf.extend(macros[swdistrepo])
-        prjconf.append("%distribution {0}".format(distrepo))
-        prjconf.append(":Macros")
-        prjconf.append("%endif")
-        prjconf.append("")
-
-    prjconf.append(end_marker)
-    prjconf = prjconf_head + "\n".join(prjconf) + prjconf_tail
-    osc.core.edit_meta("prjconf", "software:dist", data=prjconf)

mpcdf_remove.py

-#!/usr/bin/python2
 from __future__ import print_function
 
 import os

mpcdf_set_as_branch.py

-#!/usr/bin/python2
 from __future__ import print_function
 
 import mpcdf_common

mpcdf_setup_clusters_project.py

+from __future__ import print_function
+
+import osc
+import osc.conf
+import osc.core
+import osc.cmdln
+
+import mpcdf_common
+
+
+@osc.cmdln.alias("mpcdf_setup_clusters")
+def do_mpcdf_setup_cluster_project(self, subcmd, opts, *args):
+    """${cmd_name}: Set-up the 'clusters' project
+
+    This creates the 'prjconf' for the 'clusters' project
+
+    Usage:
+        osc ${cmd_name}
+
+    ${cmd_option_list}
+
+    """
+
+    if len(args) != 0:
+        raise osc.oscerr.WrongArgs("Too many arguments")
+
+    api_url = self.get_api_url()
+
+    projects = [p for p in osc.core.meta_get_project_list(api_url)
+                if p.startswith("software:")]
+
+    macros = {}
+    for project in projects:
+        target = project[len("software:"):].replace(":", "-")
+
+        if target in macros:
+            raise Exception("Internal error")
+
+        macros[target] = []
+        for attribute in mpcdf_common.config_attributes + mpcdf_common.default_attributes:
+            _, name = attribute.split(":")
+            values = mpcdf_common.overloaded_project_attribute(api_url, project, attribute)
+            macros[target].append("%available_{0} {1}".format(name, ",".join(sorted(values))))
+
+        all_compilers = mpcdf_common.overloaded_project_attribute(api_url, project, "MPCDF:compiler_modules")
+        latest_intel = sorted((c for c in all_compilers if c.startswith("intel")), key=mpcdf_common.package_sort_key)[-1]
+        latest_gcc = sorted((c for c in all_compilers if c.startswith("gcc")), key=mpcdf_common.package_sort_key)[-1]
+        macros[target].append("%latest_intel " + latest_intel)
+        macros[target].append("%latest_gcc " + latest_gcc)
+
+    prjconf = list(map(mpcdf_common.decode_it, osc.core.show_project_conf(api_url, "clusters")))
+    start_marker = "# Autogenerated by osc mpcdf_setup_clusters_project, do not edit till end of section\n"
+    end_marker = "# End of autogenerated section\n"
+
+    try:
+        start = prjconf.index(start_marker)
+        end = prjconf.index(end_marker)
+    except ValueError:
+        start = None
+        end = len(prjconf)
+
+    prjconf_head = "".join(prjconf[:start])
+    prjconf_tail = "".join(prjconf[end + 1:])
+    prjconf = [start_marker]
+
+    distributions = osc.core.get_repositories_of_project(api_url, "software")
+    for repo in osc.core.get_repositories_of_project(api_url, "clusters"):
+        for dist in distributions:
+            if repo.startswith(dist):
+                break
+            dist = None
+        if dist is None:
+            raise Exception("Internal error")
+        prjconf.append('%if "%_repository" == "{0}"'.format(repo))
+        prjconf.append("Macros:")
+        prjconf.extend(macros[repo])
+        prjconf.append("%distribution {0}".format(dist))
+        prjconf.append(":Macros")
+        prjconf.append("%endif")
+        prjconf.append("")
+
+    prjconf.append(end_marker)
+    prjconf = prjconf_head + "\n".join(prjconf) + prjconf_tail
+    osc.core.edit_meta("prjconf", "clusters", data=prjconf, apiurl=api_url)

mpcdf_setup_repositories.py → mpcdf_setup_home_project.py

-#!/usr/bin/python2
 from __future__ import print_function
 
-from mpcdf_common import mpcdf_setup_repositories
+from mpcdf_common import mpcdf_setup_subproject, project_meta, decode_it
 
 import os
 import osc
@@ -12,27 +11,21 @@ import osc.cmdln
 
 @osc.cmdln.option('-n', '--dry-run', action="store_true",
                   help="Do not actually run anything but output the resulting XML configuration")
-@osc.cmdln.option('--parent', metavar="PARENT",
-                  help="Setup the repositories to be based on the upstream project PARENT (e.g. for home: projects)")
-@osc.cmdln.option('--distribution',
-                  help="Base distribution, necessary argument unless set previously for this project")
-@osc.cmdln.option('--disable-repo', metavar="REPO", action="append",
-                  help="Temporarily disable all repositories containing REPO")
-@osc.cmdln.option('--only-project', action="store_true", default=False,
-                  help="Only change project metadata 'prj' and 'prjconf', leave individual packages unchanged")
 @osc.cmdln.option('--remove-old', action="store_true", default=False,
                   help="Remove all obsolete repositories instead of only disabling builds for packages there")
+@osc.cmdln.option('--distribution',
+                  help="Base distribution, necessary argument unless set previously for this project")
 @osc.cmdln.option('--microarchitecture', metavar="ARCH", nargs=1,
                   help="Configure project to use ARCH as microarchitecture")
-@osc.cmdln.alias("mpcdf_setup_repos")
-def do_mpcdf_setup_repositories(self, subcmd, opts, *args):
-    """${cmd_name}: Create all repository combinations for an MPCDF project
+@osc.cmdln.alias("mpcdf_setup_home")
+def do_mpcdf_setup_home_project(self, subcmd, opts, *args):
+    """${cmd_name}: Setup a home project based on a software: sub-project
 
-    Given a list of compilers, MPI libraries, and possibly CUDA versions, this command
-    creates repositories for all the resulting combinations
+    Set-up a home:$USER project for building packages for the 'software'
+    project hierarchy (i.e. application group packages)
 
     Usage:
-        osc ${cmd_name} [PROJECT]
+        osc ${cmd_name} [home:USER]
 
     ${cmd_option_list}
 
@@ -49,7 +42,36 @@ def do_mpcdf_setup_repositories(self, subcmd, opts, *args):
     else:
         raise osc.oscerr.WrongArgs("Too many arguments")
 
-    mpcdf_setup_repositories(self.get_api_url(),
-                             project, microarchitecture=opts.microarchitecture, distribution=opts.distribution,
-                             parent=opts.parent, dry_run=opts.dry_run, filter_repos=opts.disable_repo,
-                             only_project=opts.only_project, remove_old=opts.remove_old)
+    if project.split(":")[0] != "home":
+        raise osc.oscerr.WrongArgs('Given project is not a home: project')
+
+    api_url = self.get_api_url()
+
+    if opts.distribution is None:
+        # Get existing value from project meta
+        dist_repo = project_meta(api_url, project).find(
+            "./repository[@name='System']/path")
+        if dist_repo is not None:
+            distribution = dist_repo.get("repository")
+        else:
+            raise osc.oscerr.WrongArgs('Could not determine desired distribution, please specify --distribution explicitly')
+    else:
+        distribution = opts.distribution
+
+    microarchitecture = None
+    if opts.microarchitecture is None:
+        for prjconfline in map(decode_it, osc.core.show_project_conf(api_url, project)):
+            if prjconfline.startwith("Constraint: hostlabel"):
+                microarchitecture = prjconfline.split()[2]
+                break
+    else:
+        microarchitecture = opts.microarchitecture
+
+    if microarchitecture is None:
+        raise osc.oscerr.WrongArgs('Could not determine desired microarchitecture, please specify --microarchitecture explicitly')
+
+    parent = "software:{0}:{1}".format(distribution, microarchitecture)
+
+    mpcdf_setup_subproject(api_url,
+                           project, distribution, microarchitecture,
+                           parent=parent, dry_run=opts.dry_run, remove_old=opts.remove_old)

mpcdf_setup_software_project.py

+from __future__ import print_function
+
+import osc
+import osc.conf
+import osc.core
+import osc.cmdln
+
+import mpcdf_common
+
+
+@osc.cmdln.option('-n', '--dry-run', action="store_true",
+                  help="Do not actually run anything but output the resulting XML configuration")
+@osc.cmdln.option('-i', '--ignore-repo', action="append", default=[], metavar="REPO",
+                  help="Do not enable for repository REPO")
+@osc.cmdln.alias("mpcdf_setup_software")
+def do_mpcdf_setup_software_project(self, subcmd, opts, *args):
+    """${cmd_name}: Set-up the 'software' project
+
+    This creates the 'prjconf' for the 'software' project and
+    enables the repositories for packages set to build for
+    'system'
+
+    Usage:
+        osc ${cmd_name}
+
+    ${cmd_option_list}
+
+    """
+
+    if len(args) != 0:
+        raise osc.oscerr.WrongArgs("Too many arguments")
+
+    api_url = self.get_api_url()
+
+    prjconf = list(map(mpcdf_common.decode_it, osc.core.show_project_conf(api_url, "software")))
+    start_marker = "# Autogenerated by osc mpcdf_setup_software_project, do not edit till end of section\n"
+    end_marker = "# End of autogenerated section\n"
+
+    try:
+        start = prjconf.index(start_marker)
+        end = prjconf.index(end_marker)
+    except ValueError:
+        start = None
+        end = len(prjconf)
+
+    prjconf_head = "".join(prjconf[:start])
+    prjconf_tail = "".join(prjconf[end + 1:])
+    prjconf = [start_marker]
+
+    prjconf.append("Constraint: sandbox lxc")
+    prjconf.append("Constraint: hostlabel sandybridge")
+    prjconf.append("")
+    prjconf.append(r"PublishFilter: ^.*\.src\.rpm$")
+    prjconf.append("")
+    prjconf.append("Preinstall: mpcdf")
+    prjconf.append("Runscripts: mpcdf")
+    prjconf.append("Preinstall: mpcdf_modules")
+    prjconf.append("Preinstall: brp_mpcdf_modules")
+    prjconf.append("")
+    prjconf.append("Macros:")
+    prjconf.append("%have_mpcdf_compiler 0")
+    prjconf.append("%have_mpcdf_mpi 0")
+    prjconf.append("%have_mpcdf_cuda 0")
+    prjconf.append(":Macros")
+
+    software_meta = mpcdf_common.project_meta(api_url, "software")
+
+    distributions = sorted(repo.get("name") for repo in software_meta.findall('./repository'))
+    for distribution in distributions:
+        extra_tags = mpcdf_common.dist_prjconf_tags(distribution)
+        extra_macros = mpcdf_common.dist_prjconf_macros(distribution)
+
+        if len(extra_tags) > 0 or len(extra_macros) > 0:
+            prjconf.append("")
+            prjconf.append("%if \"%_repository\" == \"{0}\"".format(distribution))
+            prjconf.append(extra_tags)
+            prjconf.append("")
+            prjconf.append("Macros:")
+            prjconf.append(extra_macros)
+            prjconf.append(":Macros")
+            prjconf.append("%endif")
+
+    prjconf.append(end_marker)
+    prjconf = prjconf_head + "\n".join(prjconf) + prjconf_tail
+
+    if opts.dry_run:
+        print("osc meta prjconf software -F - <<EOF\n{0}\nEOF\n".format(prjconf))
+    else:
+        osc.core.edit_meta("prjconf", "software", data=prjconf, apiurl=api_url)
+
+        if opts.ignore_repo:
+            ignore_repos = opts.ignore_repo
+        else:
+            ignore_repos = ()
+
+        mpcdf_common.mpcdf_enable_repositories_for_all_packages(api_url, "software", ignore_repos=ignore_repos)

mpcdf_setup_subproject.py

+from __future__ import print_function
+
+from mpcdf_common import mpcdf_setup_subproject
+
+import os
+import osc
+import osc.conf
+import osc.core
+import osc.cmdln
+
+
+@osc.cmdln.option('-n', '--dry-run', action="store_true",
+                  help="Do not actually run anything but output the resulting XML configuration")
+@osc.cmdln.option('--remove-old', action="store_true", default=False,
+                  help="Remove all obsolete repositories instead of only disabling builds for packages there")
+@osc.cmdln.alias("mpcdf_setup_sub")
+def do_mpcdf_setup_subproject(self, subcmd, opts, *args):
+    """${cmd_name}: Setup a software: sub-project
+
+    Given a list of compilers, MPI libraries, and possibly CUDA versions via
+    the projects attributes, this command creates repositories for all the
+    resulting combinations
+
+    Additionally, the prjconf is set-up to enforce the microarchitecture via
+    a "Constraint:" statement
+
+    Usage:
+        osc ${cmd_name} [software:OS:microarchitecture]
+
+    ${cmd_option_list}
+
+    """
+
+    if len(args) == 0:
+        if osc.core.is_project_dir(os.curdir) or osc.core.is_package_dir(os.curdir):
+            project = osc.core.store_read_project(os.curdir)
+        else:
+            raise osc.oscerr.WrongArgs('Specify PROJECT or run command in an osc checkout directory')
+
+    elif len(args) == 1:
+        project, = args
+    else:
+        raise osc.oscerr.WrongArgs("Too many arguments")
+
+    sw, distribution, microarchitecture = project.split(":")
+
+    if sw != "software":
+        raise osc.oscerr.WrongArgs('Given project is not below the "software" project')
+
+    mpcdf_setup_subproject(self.get_api_url(),
+                           project, distribution, microarchitecture,
+                           dry_run=opts.dry_run, remove_old=opts.remove_old)

mpcdf_sync_projects.py

-#!/usr/bin/python2
 from __future__ import print_function
 
 import mpcdf_common