Changes for new structure of obs-api.mpcdf.mpg.de

mpcdf_enable_repositories.py

@@ -90,8 +90,8 @@ def do_mpcdf_enable_repositories(self, subcmd, opts, *args):
 
     if opts.recreate or opts.set or opts.disable:
         mpcdf_common.mpcdf_enable_repositories(api_url, project, package, verbose=True)
-        if project == "software":
-            mpcdf_common.sync_projects(api_url, package)
+        if project == "software" and opts.set != "system":
+            mpcdf_common.sync_projects(api_url, package, verbose=True)
 
     elif (opts.compiler_modules or opts.mpi_modules or opts.cuda_modules):
         print("ERROR: Invalid arguments, try --help")

mpcdf_refresh_aggregates.py

-#!/usr/bin/python2
-from __future__ import print_function
-
-import os
-import osc
-import osc.conf
-import osc.core
-import osc.cmdln
-
-import mpcdf_common
-
-from xml.etree import ElementTree
-
-
-def do_mpcdf_refresh_aggregates(self, subcmd, opts, *args):
-    """${cmd_name}: Recreates the aggregate packages in software:dist
-
-    Use this only if you know what this means
-
-    Usage:
-        osc ${cmd_name}
-
-    ${cmd_option_list}
-
-    """
-
-    if len(args) != 0:
-        raise osc.oscerr.WrongArgs("Too many arguments")
-
-    if osc.core.store_read_project(os.curdir) != "software:dist" or \
-            osc.core.is_package_dir(os.curdir):
-        raise osc.oscerr.WrongArgs("You must be in the project checkout of software:dist")
-
-    dist = osc.core.Project(".", False)
-
-    apiurl = self.get_api_url()
-
-    def refresh_aggregate(aggregatename, project, source, target):
-        if not os.path.exists(aggregatename):
-            package = osc.core.Package.init_package(apiurl, "software:dist", aggregatename, aggregatename)
-            dist.addPackage(aggregatename)
-        else:
-            package = dist.get_pacobj(aggregatename)
-
-        with open(aggregatename + "/_aggregate", "w") as fd:
-            print('<aggregatelist>', file=fd)
-            print('  <aggregate project="{project}">'.format(project=project), file=fd)
-            print('    <nosources/>', file=fd)
-            print('    <repository target="{target}" source="{source}" />'.format(target=target, source=source), file=fd)
-            print('  </aggregate>', file=fd)
-            print('</aggregatelist>', file=fd)
-
-        if package.status("_aggregate") == "?":
-            package.addfile("_aggregate")
-
-        if package.status("_aggregate") in ("A", "M"):
-            dist.commit((aggregatename,), msg="Refresh aggregate", verbose=True)
-
-        pkg_meta = osc.core.show_package_meta(apiurl, "software:dist", aggregatename)
-        root = ElementTree.fromstringlist(pkg_meta)
-        pkg_meta = ElementTree.tostring(root, encoding=osc.core.ET_ENCODING)
-
-        build = root.find("./build")
-        if build is None:
-            build = ElementTree.SubElement(root, "build")
-
-        for enable in build.findall("./enable"):
-            build.remove(enable)
-
-        node = ElementTree.Element("enable")
-        node.set("repository", target)
-        node.tail = "\n  "
-        build.insert(0, node)
-
-        new_pkg_meta = ElementTree.tostring(root, encoding=osc.core.ET_ENCODING)
-        if pkg_meta != new_pkg_meta:
-            osc.core.edit_meta("pkg", ("software:dist", aggregatename),
-                               data=new_pkg_meta)
-
-    projects = [p for p in osc.core.meta_get_project_list(apiurl)
-                if p.startswith("software") and not (p == "software:dist" or p == "software:images")]
-
-    macros = {}
-    for project in projects:
-        if project == "software":
-            # Stupid special case
-            target = "SLE_12_SP3-sandybridge"
-        else:
-            target = project[len("software:"):].replace(":", "-")
-
-        macros[target] = []
-        for attribute in mpcdf_common.config_attributes + mpcdf_common.default_attributes:
-            _, name = attribute.split(":")
-            values = mpcdf_common.get_attribute_values(apiurl, project, None, attribute)
-            macros[target].append("%available_{0} {1}".format(name, ",".join(sorted(values))))
-
-        all_compilers = mpcdf_common.get_attribute_values(apiurl, project, None, "MPCDF:compiler_modules")
-        latest_intel = sorted((c for c in all_compilers if c.startswith("intel")), key=mpcdf_common.package_sort_key)[-1]
-        latest_gcc = sorted((c for c in all_compilers if c.startswith("gcc")), key=mpcdf_common.package_sort_key)[-1]
-        macros[target].append("%latest_intel " + latest_intel)
-        macros[target].append("%latest_gcc " + latest_gcc)
-
-        for repo in osc.core.get_repositories_of_project(apiurl, project):
-            aggregatename = "zz_aggregate_" + project.replace(":", "-") + "_" + repo
-            refresh_aggregate(aggregatename, project, repo, target)
-
-    prjconf = list(map(mpcdf_common.decode_it, osc.core.show_project_conf(apiurl, "software:dist")))
-    start_marker = "# Autogenerated by osc mpcdf_refresh_aggregates, do not edit till end of section\n"
-    end_marker = "# End of autogenerated section\n"
-
-    try:
-        start = prjconf.index(start_marker)
-        end = prjconf.index(end_marker)
-    except ValueError:
-        start = None
-        end = len(prjconf)
-
-    prjconf_head = "".join(prjconf[:start])
-    prjconf_tail = "".join(prjconf[end + 1:])
-    prjconf = [start_marker]
-
-    distributions = osc.core.get_repositories_of_project(apiurl, "distributions")
-    for swdistrepo in osc.core.get_repositories_of_project(apiurl, "software:dist"):
-        for distrepo in distributions:
-            if swdistrepo.startswith(distrepo):
-                refresh_aggregate("zz_aggregate_" + swdistrepo + "_distributions", "distributions", distrepo, swdistrepo)
-                break
-            distrepo = None
-        prjconf.append('%if "%_repository" == "{0}"'.format(swdistrepo))
-        prjconf.append("Macros:")
-        prjconf.extend(macros[swdistrepo])
-        prjconf.append("%distribution {0}".format(distrepo))
-        prjconf.append(":Macros")
-        prjconf.append("%endif")
-        prjconf.append("")
-
-    prjconf.append(end_marker)
-    prjconf = prjconf_head + "\n".join(prjconf) + prjconf_tail
-    osc.core.edit_meta("prjconf", "software:dist", data=prjconf)

mpcdf_setup_clusters_project.py

+from __future__ import print_function
+
+import osc
+import osc.conf
+import osc.core
+import osc.cmdln
+
+import mpcdf_common
+
+
+@osc.cmdln.alias("mpcdf_setup_clusters")
+def do_mpcdf_setup_cluster_project(self, subcmd, opts, *args):
+    """${cmd_name}: Set-up the 'clusters' project
+
+    This creates the 'prjconf' for the 'clusters' project
+
+    Usage:
+        osc ${cmd_name}
+
+    ${cmd_option_list}
+
+    """
+
+    if len(args) != 0:
+        raise osc.oscerr.WrongArgs("Too many arguments")
+
+    api_url = self.get_api_url()
+
+    projects = [p for p in osc.core.meta_get_project_list(api_url)
+                if p.startswith("software:")]
+
+    macros = {}
+    for project in projects:
+        target = project[len("software:"):].replace(":", "-")
+
+        if target in macros:
+            raise Exception("Internal error")
+
+        macros[target] = []
+        for attribute in mpcdf_common.config_attributes + mpcdf_common.default_attributes:
+            _, name = attribute.split(":")
+            values = mpcdf_common.overloaded_project_attribute(api_url, project, attribute)
+            macros[target].append("%available_{0} {1}".format(name, ",".join(sorted(values))))
+
+        all_compilers = mpcdf_common.overloaded_project_attribute(api_url, project, "MPCDF:compiler_modules")
+        latest_intel = sorted((c for c in all_compilers if c.startswith("intel")), key=mpcdf_common.package_sort_key)[-1]
+        latest_gcc = sorted((c for c in all_compilers if c.startswith("gcc")), key=mpcdf_common.package_sort_key)[-1]
+        macros[target].append("%latest_intel " + latest_intel)
+        macros[target].append("%latest_gcc " + latest_gcc)
+
+    prjconf = list(map(mpcdf_common.decode_it, osc.core.show_project_conf(api_url, "clusters")))
+    start_marker = "# Autogenerated by osc mpcdf_setup_clusters_project, do not edit till end of section\n"
+    end_marker = "# End of autogenerated section\n"
+
+    try:
+        start = prjconf.index(start_marker)
+        end = prjconf.index(end_marker)
+    except ValueError:
+        start = None
+        end = len(prjconf)
+
+    prjconf_head = "".join(prjconf[:start])
+    prjconf_tail = "".join(prjconf[end + 1:])
+    prjconf = [start_marker]
+
+    distributions = osc.core.get_repositories_of_project(api_url, "software")
+    for repo in osc.core.get_repositories_of_project(api_url, "clusters"):
+        for dist in distributions:
+            if repo.startswith(dist):
+                break
+            dist = None
+        if dist is None:
+            raise Exception("Internal error")
+        prjconf.append('%if "%_repository" == "{0}"'.format(repo))
+        prjconf.append("Macros:")
+        prjconf.extend(macros[repo])
+        prjconf.append("%distribution {0}".format(dist))
+        prjconf.append(":Macros")
+        prjconf.append("%endif")
+        prjconf.append("")
+
+    prjconf.append(end_marker)
+    prjconf = prjconf_head + "\n".join(prjconf) + prjconf_tail
+    osc.core.edit_meta("prjconf", "clusters", data=prjconf)

mpcdf_setup_home_project.py

+#!/usr/bin/python2
+from __future__ import print_function
+
+from mpcdf_common import mpcdf_setup_subproject, project_meta, decode_it
+
+import os
+import osc
+import osc.conf
+import osc.core
+import osc.cmdln
+
+@osc.cmdln.option('-n', '--dry-run', action="store_true",
+                  help="Do not actually run anything but output the resulting XML configuration")
+@osc.cmdln.option('--remove-old', action="store_true", default=False,
+                  help="Remove all obsolete repositories instead of only disabling builds for packages there")
+@osc.cmdln.option('--distribution',
+                  help="Base distribution, necessary argument unless set previously for this project")
+@osc.cmdln.option('--microarchitecture', metavar="ARCH", nargs=1,
+                  help="Configure project to use ARCH as microarchitecture")
+@osc.cmdln.alias("mpcdf_setup_home")
+def do_mpcdf_setup_home_project(self, subcmd, opts, *args):
+    """${cmd_name}: Setup a home project based on a software: sub-project
+
+    Set-up a home:$USER project for building packages for the 'software'
+    project hierarchy (i.e. application group packages)
+
+    Usage:
+        osc ${cmd_name} [home:USER]
+
+    ${cmd_option_list}
+
+    """
+
+    if len(args) == 0:
+        if osc.core.is_project_dir(os.curdir) or osc.core.is_package_dir(os.curdir):
+            project = osc.core.store_read_project(os.curdir)
+        else:
+            raise osc.oscerr.WrongArgs('Specify PROJECT or run command in an osc checkout directory')
+
+    elif len(args) == 1:
+        project, = args
+    else:
+        raise osc.oscerr.WrongArgs("Too many arguments")
+
+    if project.split(":")[0] != "home":
+        raise osc.oscerr.WrongArgs('Given project is not a home: project')
+
+    api_url = self.get_api_url()
+
+    if opts.distribution is None:
+        # Get existing value from project meta
+        dist_repo = project_meta(api_url, project).find(
+            "./repository[@name='System']/path")
+        if dist_repo is not None:
+            distribution = dist_repo.get("repository")
+        else:
+            raise osc.oscerr.WrongArgs('Could not determine desired distribution, please specify --distribution explicitly')
+    else:
+        distribution = opts.distribution
+
+    microarchitecture = None
+    if opts.microarchitecture is None:
+        for prjconfline in map(decode_it, osc.core.show_project_conf(api_url, project)):
+            if prjconfline.startwith("Constraint: hostlabel"):
+                microarchitecture = prjconfline.split()[2]
+                break
+    else:
+        microarchitecture = opts.microarchitecture
+
+    if microarchitecture is None:
+        raise osc.oscerr.WrongArgs('Could not determine desired microarchitecture, please specify --microarchitecture explicitly')
+
+    parent = "software:{0}:{1}".format(distribution, microarchitecture)
+
+    mpcdf_setup_subproject(api_url,
+                           project, distribution, microarchitecture,
+                           parent=parent, dry_run=opts.dry_run, remove_old=opts.remove_old)

mpcdf_setup_software_project.py

+#!/usr/bin/python2
+from __future__ import print_function
+
+import osc
+import osc.conf
+import osc.core
+import osc.cmdln
+
+import mpcdf_common
+
+
+@osc.cmdln.option('-i', '--ignore-repo', action="append", default=[], metavar="REPO",
+                  help="Do not enable for repository REPO")
+@osc.cmdln.alias("mpcdf_setup_software")
+def do_mpcdf_setup_software_project(self, subcmd, opts, *args):
+    """${cmd_name}: Set-up the 'software' project
+
+    This creates the 'prjconf' for the 'software' project and
+    enables the repositories for packages set to build for
+    'system'
+
+    Usage:
+        osc ${cmd_name}
+
+    ${cmd_option_list}
+
+    """
+
+    if len(args) != 0:
+        raise osc.oscerr.WrongArgs("Too many arguments")
+
+    api_url = self.get_api_url()
+
+    prjconf = list(map(mpcdf_common.decode_it, osc.core.show_project_conf(api_url, "software")))
+    start_marker = "# Autogenerated by osc mpcdf_setup_software_project, do not edit till end of section\n"
+    end_marker = "# End of autogenerated section\n"
+
+    try:
+        start = prjconf.index(start_marker)
+        end = prjconf.index(end_marker)
+    except ValueError:
+        start = None
+        end = len(prjconf)
+
+    prjconf_head = "".join(prjconf[:start])
+    prjconf_tail = "".join(prjconf[end + 1:])
+    prjconf = [start_marker]
+
+    prjconf.append("Constraint: sandbox lxc")
+    prjconf.append("Constraint: hostlabel sandybridge")
+    prjconf.append("")
+    prjconf.append(r"PublishFilter: ^.*\.src\.rpm$")
+    prjconf.append("")
+    prjconf.append("Preinstall: mpcdf")
+    prjconf.append("Preinstall: mpcdf_modules")
+    prjconf.append("Preinstall: brp_mpcdf_modules")
+
+    prjconf.append(end_marker)
+    prjconf = prjconf_head + "\n".join(prjconf) + prjconf_tail
+    osc.core.edit_meta("prjconf", "software", data=prjconf)
+
+    if opts.ignore_repo:
+        ignore_repos = opts.ignore_repo
+    else:
+        ignore_repos = ()
+
+    mpcdf_common.mpcdf_enable_repositories_for_all_packages(api_url, "software", ignore_repos=ignore_repos)

mpcdf_setup_repositories.py → mpcdf_setup_subproject.py

 #!/usr/bin/python2
 from __future__ import print_function
 
-from mpcdf_common import mpcdf_setup_repositories
+from mpcdf_common import mpcdf_setup_subproject
 
 import os
 import osc
@@ -9,30 +9,23 @@ import osc.conf
 import osc.core
 import osc.cmdln
 
-
 @osc.cmdln.option('-n', '--dry-run', action="store_true",
                   help="Do not actually run anything but output the resulting XML configuration")
-@osc.cmdln.option('--parent', metavar="PARENT",
-                  help="Setup the repositories to be based on the upstream project PARENT (e.g. for home: projects)")
-@osc.cmdln.option('--distribution',
-                  help="Base distribution, necessary argument unless set previously for this project")
-@osc.cmdln.option('--disable-repo', metavar="REPO", action="append",
-                  help="Temporarily disable all repositories containing REPO")
-@osc.cmdln.option('--only-project', action="store_true", default=False,
-                  help="Only change project metadata 'prj' and 'prjconf', leave individual packages unchanged")
 @osc.cmdln.option('--remove-old', action="store_true", default=False,
                   help="Remove all obsolete repositories instead of only disabling builds for packages there")
-@osc.cmdln.option('--microarchitecture', metavar="ARCH", nargs=1,
-                  help="Configure project to use ARCH as microarchitecture")
-@osc.cmdln.alias("mpcdf_setup_repos")
-def do_mpcdf_setup_repositories(self, subcmd, opts, *args):
-    """${cmd_name}: Create all repository combinations for an MPCDF project
+@osc.cmdln.alias("mpcdf_setup_sub")
+def do_mpcdf_setup_subproject(self, subcmd, opts, *args):
+    """${cmd_name}: Setup a software: sub-project
+
+    Given a list of compilers, MPI libraries, and possibly CUDA versions via
+    the projects attributes, this command creates repositories for all the
+    resulting combinations
 
-    Given a list of compilers, MPI libraries, and possibly CUDA versions, this command
-    creates repositories for all the resulting combinations
+    Additionally, the prjconf is set-up to enforce the microarchitecture via
+    a "Constraint:" statement
 
     Usage:
-        osc ${cmd_name} [PROJECT]
+        osc ${cmd_name} [software:OS:microarchitecture]
 
     ${cmd_option_list}
 
@@ -49,7 +42,11 @@ def do_mpcdf_setup_repositories(self, subcmd, opts, *args):
     else:
         raise osc.oscerr.WrongArgs("Too many arguments")
 
-    mpcdf_setup_repositories(self.get_api_url(),
-                             project, microarchitecture=opts.microarchitecture, distribution=opts.distribution,
-                             parent=opts.parent, dry_run=opts.dry_run, filter_repos=opts.disable_repo,
-                             only_project=opts.only_project, remove_old=opts.remove_old)
+    sw, distribution, microarchitecture = project.split(":")
+
+    if sw != "software":
+        raise osc.oscerr.WrongArgs('Given project is not below the "software" project')
+
+    mpcdf_setup_subproject(self.get_api_url(),
+                           project, distribution, microarchitecture,
+                           dry_run=opts.dry_run, remove_old=opts.remove_old)