From c6a734320a448040f2ae51088170de2b9f70b3d1 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Lorenz=20H=C3=BCdepohl?= <lorenz.huedepohl@rzg.mpg.de>
Date: Fri, 10 Nov 2017 14:13:20 +0100
Subject: [PATCH] New commands `mpcdf_info`, `mpcdf_enable_repositories`
---
.gitignore | 1 +
mpcdf-repositories.py | 138 -----------------------------
mpcdf_common.py | 112 ++++++++++++++++++++++++
mpcdf_enable_repositories.py | 109 +++++++++++++++++++++++
mpcdf_info.py | 78 +++++++++++++++++
mpcdf_setup_repos.py | 164 +++++++++++++++++++++++++++++++++++
6 files changed, 464 insertions(+), 138 deletions(-)
create mode 100644 .gitignore
delete mode 100755 mpcdf-repositories.py
create mode 100644 mpcdf_common.py
create mode 100644 mpcdf_enable_repositories.py
create mode 100644 mpcdf_info.py
create mode 100644 mpcdf_setup_repos.py
diff --git a/.gitignore b/.gitignore
new file mode 100644
index 0000000..0d20b64
--- /dev/null
+++ b/.gitignore
@@ -0,0 +1 @@
+*.pyc
diff --git a/mpcdf-repositories.py b/mpcdf-repositories.py
deleted file mode 100755
index 0558452..0000000
--- a/mpcdf-repositories.py
+++ /dev/null
@@ -1,138 +0,0 @@
-#!/usr/bin/python2
-from __future__ import print_function
-import sys
-import argparse
-from xml.etree import ElementTree
-
-import osc
-import osc.conf
-import osc.core
-import osc.cmdln
-
-all_compilers = "gcc_4_9 gcc_5 gcc_6 gcc_7 intel_16_0 intel_17_0"
-all_mpis = "openmpi_1_10 impi_5_1_3 impi_2017_3"
-all_cudas = "cuda_8"
-default_distribution = "SLE_12_SP1"
-default_architectures = "x86_64"
-
-def valid_mpi(compiler, mpi):
- if compiler == "intel_17_0":
- if mpi.startswith("impi"):
- return mpi == "impi_2017_3"
- if compiler == "intel_16_0":
- if mpi.startswith("impi"):
- return mpi == "impi_5_1_3"
- return True
-
-@osc.cmdln.option('--distribution',
- default=default_distribution,
- help="Base distribution [default: %default]")
-
-@osc.cmdln.option('--architectures',
- default=default_architectures,
- help="Architectures [default: %default]")
-
-@osc.cmdln.option('--compilers',
- default=all_compilers,
- help="List of compilers, [default: %default]")
-
-@osc.cmdln.option('--mpis',
- default=all_mpis,
- help="List of MPI libraries, [default: %default]")
-
-@osc.cmdln.option('--cudas',
- default=all_cudas,
- help="List of CUDA versions, [default: %default]")
-
-@osc.cmdln.option('--parent', metavar="PARENT",
- help="Setup the repositories to be based on the upstream project PARENT (e.g. for home: projects)")
-
-@osc.cmdln.option('-n', '--dry-run', action="store_true",
- help="Do not actually run anything but output the resulting XML configuration")
-def do_mpcdf_repositories(self, subcmd, opts, *args):
- """${cmd_name}: Create all repository combinations for an MPCDF project
-
- Given a list of compilers, MPI libraries, and possibly CUDA versions, this command
- creates repositories for all the resulting combinations
-
- Usage:
- osc ${cmd_name} [PROJECT]
-
- ${cmd_option_list}
-
- """
-
- if len(args) == 0:
- if is_project_dir(os.curdir) or is_package_dir(os.curdir):
- project = osc.core.store_read_project(os.curdir)
- else:
- raise oscerr.WrongArgs('Specify PROJECT or run command in an osc checkout directory')
-
- elif len(args) == 1:
- project, = args
- else:
- raise oscerr.WrongArgs("Too many arguments")
-
- compilers = opts.compilers.split()
- architectures = opts.architectures.split()
- mpis = opts.mpis.split()
- cudas = opts.cudas.split()
-
- project_meta = osc.core.show_project_meta(osc.conf.config["apiurl"], project)
- root = ElementTree.fromstringlist(project_meta)
-
- # Remove existing repositories
- for repo in root.findall("./repository"):
- root.remove(repo)
-
- def repo(name, *dependencies):
- r = ElementTree.SubElement(root, "repository")
- r.set("name", name)
- r.text="\n "
- def path(project, repo):
- p = ElementTree.SubElement(r, "path")
- p.set("project", project)
- p.set("repository", repo)
- p.tail="\n "
-
- if opts.parent:
- path(opts.parent, name)
- for dep_project, dep_repo in dependencies:
- path(dep_project, dep_repo)
-
- for arch in architectures:
- a = ElementTree.SubElement(r, "arch")
- a.text = arch
- a.tail = "\n "
- a.tail = "\n "
- r.tail = "\n "
-
- repo("System", ("distributions", opts.distribution))
-
- for compiler in sorted(compilers):
- repo(compiler, (project, "System"))
-
- for mpi in sorted(mpis):
- for compiler in sorted(compilers):
- if not valid_mpi(compiler, mpi):
- continue
- repo(mpi + "_" + compiler, (project, compiler))
-
- for cuda in sorted(cudas):
- repo(cuda, (project, "System"))
- for compiler in sorted(compilers):
- repo(cuda + "_" + compiler, (project, cuda), (project, compiler))
- for mpi in sorted(mpis):
- for compiler in sorted(compilers):
- if not valid_mpi(compiler, mpi):
- continue
- repo(cuda + "_" + mpi + "_" + compiler,
- (project, cuda + "_" + compiler),
- (project, mpi + "_" + compiler))
-
- root.getchildren()[-1].tail = "\n"
- meta = ET.tostring(root, encoding=osc.core.ET_ENCODING)
- if opts.dry_run:
- print(meta)
- else:
- osc.core.edit_meta("prj", project, data=meta)
diff --git a/mpcdf_common.py b/mpcdf_common.py
new file mode 100644
index 0000000..f07044f
--- /dev/null
+++ b/mpcdf_common.py
@@ -0,0 +1,112 @@
+from __future__ import print_function
+
+__all__ = ["valid_mpi", "valid_cuda", "get_attribute", "mpcdf_enable_repositories"]
+
+import osc
+import osc.conf
+import osc.core
+
+from xml.etree import ElementTree
+
+
+def valid_mpi(compiler, mpi):
+ """
+ It might be possible to use Intel MPI libararies and compilers from
+ different Parallel Studio packages, but I currently do not want to support
+ it
+ """
+ if compiler == "intel_17_0":
+ if mpi.startswith("impi"):
+ return mpi == "impi_2017_3"
+ if compiler == "intel_16_0":
+ if mpi.startswith("impi"):
+ return mpi == "impi_5_1_3"
+ return True
+
+
+def valid_cuda(cuda, compiler):
+ """
+ The CUDA distribution only works with certain gcc versions,
+ this little function takes care that only supported combinations
+ are allowed
+ """
+ if cuda == "cuda_8":
+ return compiler == "gcc_5"
+ return False
+
+
+def get_attribute(api_url, project, package, attribute, with_project=False):
+
+ attribute_meta = osc.core.show_attribute_meta(api_url, project, package, None, attribute, False, with_project)
+ if attribute_meta is None:
+ raise oscerr.APIError("Cannot fetch value for attribute '{0}' from {1}".format(attribute, (project, package)))
+
+ root = ElementTree.fromstringlist(attribute_meta)
+ attribute = root.find("./attribute")
+ if attribute is not None:
+ return list(value.text for value in attribute.findall("./value"))
+ else:
+ raise Exception("Attribute not set")
+
+
+def mpcdf_enable_repositories(api_url, project, package, verbose=False):
+
+ root = ElementTree.fromstringlist(osc.core.show_package_meta(api_url, project, package))
+ build = root.find("./build")
+ for enable in build.findall("./enable"):
+ build.remove(enable)
+
+ compilers = get_attribute(api_url, project, package, "MPCDF:compiler_modules", with_project=True)
+ mpis = get_attribute(api_url, project, package, "MPCDF:mpi_modules", with_project=True)
+ cudas = get_attribute(api_url, project, package, "MPCDF:cuda_modules", with_project=True)
+
+ def enable(name):
+ node = ElementTree.Element("enable")
+ node.set("repository", name)
+ node.tail = "\n "
+ build.insert(0, node)
+ if verbose:
+ print("Enabling", name)
+
+ try:
+ enable_repos = get_attribute(api_url, project, package, "MPCDF:enable_repositories")
+ except:
+ if verbose:
+ print("Warning: Could not get attribute MPCDF:enable_repositories for package {0}".format(package))
+ return False
+
+ for flag in enable_repos:
+
+ if flag == "system":
+ enable("System")
+
+ if flag == "compilers":
+ for compiler in compilers:
+ enable(compiler)
+
+ if flag == "mpi":
+ for compiler in compilers:
+ for mpi in mpis:
+ if valid_mpi(compiler, mpi):
+ enable(mpi + "_" + compiler)
+
+ if flag == "cuda":
+ for cuda in cudas:
+ for compiler in compilers:
+ if valid_cuda(cuda, compiler):
+ enable(cuda + "_" + compiler)
+
+ if flag == "cuda_mpi":
+ for cuda in cudas:
+ for compiler in compilers:
+ if valid_cuda(cuda, compiler):
+ for mpi in mpis:
+ if valid_mpi(compiler, mpi):
+ enable(cuda + "_" + compiler)
+
+ build.getchildren()[-1].tail = "\n "
+
+ pkg_meta = ET.tostring(root, encoding=osc.core.ET_ENCODING)
+
+ osc.core.edit_meta("pkg", (project, package), data=pkg_meta)
+ return True
diff --git a/mpcdf_enable_repositories.py b/mpcdf_enable_repositories.py
new file mode 100644
index 0000000..e9cc2bf
--- /dev/null
+++ b/mpcdf_enable_repositories.py
@@ -0,0 +1,109 @@
+#!/usr/bin/python2
+from __future__ import print_function
+import sys
+import argparse
+from functools import partial
+from xml.etree import ElementTree
+
+from mpcdf_common import *
+
+import osc
+import osc.conf
+import osc.core
+import osc.cmdln
+
+@osc.cmdln.option('--cuda-mpi', action="store_true",
+ help="Enable for all CUDA+MPI repositories")
+@osc.cmdln.option('--cuda', action="store_true",
+ help="Enable for all CUDA repositories")
+@osc.cmdln.option('--mpi', action="store_true",
+ help="Enable for all MPI module repositories")
+@osc.cmdln.option('--compilers', action="store_true",
+ help="Enable for all compiler module repositories")
+@osc.cmdln.option('--system', action="store_true",
+ help="Enable for System repository")
+@osc.cmdln.option('--reset', action="store_true",
+ help="Re-create the set of enabled repositories from the MPCDF:enable_repositores attribute")
+@osc.cmdln.option('--set', action="store_true",
+ help="Modify the set of enabled repositories, without this the current setup is displayed")
+def do_mpcdf_enable_repositories(self, subcmd, opts, *args):
+ """${cmd_name}: Select all appropriate repositories for an MPCDF package
+
+ Due to the large number of repository combinations at MPCDF it is
+ cumbersome to enable all the appropriate repositories for a given package
+ by hand.
+
+ This command allows you to set the <enable/> flags for certain kinds of
+ repositories at once.
+
+ Without --set this command only displays the current configuration.
+
+ It is possible to combine this with user-defined <disabled/> flags that
+ override the settings of this command, for example to disable the build of
+ the package with a certain repository from a given set (e.g. one
+ troublesome compiler)
+
+ Usage:
+ osc ${cmd_name} [[PROJECT] PACKAGE]
+ osc ${cmd_name} --set [ flags ] [[PROJECT] PACKAGE]
+ osc ${cmd_name} --reset [[PROJECT] PACKAGE]
+
+ ${cmd_option_list}
+ """
+
+ if len(args) == 0:
+ if is_package_dir(os.curdir):
+ package = osc.core.store_read_package(os.curdir)
+ project = osc.core.store_read_project(os.curdir)
+ else:
+ raise oscerr.WrongArgs('Specify PACKAGE or run command in an osc package checkout directory')
+
+ elif len(args) == 1:
+ package, = args
+ project = osc.core.store_read_project(os.curdir)
+
+ elif len(args) == 2:
+ project, package = args
+ else:
+ raise oscerr.WrongArgs("Too many arguments")
+
+ api_url = self.get_api_url()
+
+ if opts.set:
+ root = ElementTree.fromstringlist(osc.core.show_attribute_meta(api_url, project, package, None, "MPCDF:enable_repositories", False, False))
+
+ attribute = root.find("./attribute")
+ if attribute is None:
+ attribute = ElementTree.SubElement(root, "attribute")
+ attribute.set("namespace", "MPCDF")
+ attribute.set("name", "enable_repositories")
+
+ for value in attribute.findall("./value"):
+ attribute.remove(value)
+
+ def value(content):
+ v = ElementTree.SubElement(attribute, "value")
+ v.text = content
+
+ if opts.system:
+ value("system")
+ if opts.compilers:
+ value("compilers")
+ if opts.mpi:
+ value("mpi")
+ if opts.cuda:
+ value("cuda")
+ if opts.cuda_mpi:
+ value("cuda_mpi")
+
+ attr_meta = ET.tostring(root, encoding=osc.core.ET_ENCODING)
+
+ url = osc.core.makeurl(api_url, ["source", project, package, "_attribute"])
+ resp = ElementTree.fromstringlist(osc.core.streamfile(url, http_POST, data=attr_meta))
+ if resp.find("./summary").text != "Ok":
+ raise osc.core.oscerr.APIError("Could not store attribute MPCDF:enabled_repositories")
+
+ if opts.reset or opts.set:
+ mpcdf_enable_repositories(api_url, project, package, verbose=True)
+ else:
+ print("Enabled for:", *get_attribute(api_url, project, package, "MPCDF:enable_repositories"))
diff --git a/mpcdf_info.py b/mpcdf_info.py
new file mode 100644
index 0000000..36b12b1
--- /dev/null
+++ b/mpcdf_info.py
@@ -0,0 +1,78 @@
+#!/usr/bin/python2
+from __future__ import print_function
+import sys
+import argparse
+
+from mpcdf_common import *
+
+import osc
+import osc.conf
+import osc.core
+import osc.cmdln
+
+def do_mpcdf_info(self, subcmd, opts, *args):
+ """${cmd_name}: Basic information about an MPCDF OBS project
+
+ Usage:
+ osc ${cmd_name} [PROJECT]
+
+ ${cmd_option_list}
+
+ """
+
+ if len(args) == 0:
+ if is_project_dir(os.curdir) or is_package_dir(os.curdir):
+ project = osc.core.store_read_project(os.curdir)
+ else:
+ raise oscerr.WrongArgs('Specify PROJECT or run command in an osc checkout directory')
+
+ elif len(args) == 1:
+ project, = args
+ else:
+ raise oscerr.WrongArgs("Too many arguments")
+
+ print("Project `{0}`\n".format(project))
+
+ api_url = self.get_api_url()
+
+ def print_attribute(description, attribute):
+ values = list(sorted(get_attribute(api_url, project, None, attribute)))
+ if values:
+ print(" {0}:\n ".format(description) + "\n ".join(values))
+ print()
+
+ print_attribute("Compilers", "MPCDF:compiler_modules")
+ print_attribute("MPI libraries", "MPCDF:mpi_modules")
+ print_attribute("CUDA versions", "MPCDF:cuda_modules")
+
+ table = []
+ unmanaged = []
+
+ for package in osc.core.meta_get_packagelist(api_url, project):
+ try:
+ table.append((package, set(get_attribute(api_url, project, package, "MPCDF:enable_repositories"))))
+ except:
+ unmanaged.append(package)
+
+ pkg_name_width = reduce(max, (len(t[0]) for t in table))
+ pkg_name_width = max(pkg_name_width, len("Package"))
+
+ r = osc.core.http_request("GET", api_url + "/attribute/MPCDF/enable_repositories/_meta")
+ root = ET.parse(r).getroot()
+ kinds = list(value.text for value in root.findall("./allowed/value"))
+
+ pkg_name_fmt = "{{:{0}}}".format(pkg_name_width)
+
+ print(" " + pkg_name_fmt.format("Package"), "Enabled repositories")
+ print(" " + "-" * (pkg_name_width), "-" * len("Enabled repositories"))
+
+ for package, enabled_repo_kind in table:
+ print(" " + pkg_name_fmt.format(package), ", ".join(filter(lambda q : q in enabled_repo_kind, kinds)))
+ print()
+
+ if unmanaged:
+ print(" Unmanaged packages")
+ print(" " + "-" * (1 + pkg_name_width))
+ for package in unmanaged:
+ print(" " + package)
+ print()
diff --git a/mpcdf_setup_repos.py b/mpcdf_setup_repos.py
new file mode 100644
index 0000000..4cddb60
--- /dev/null
+++ b/mpcdf_setup_repos.py
@@ -0,0 +1,164 @@
+#!/usr/bin/python2
+from __future__ import print_function
+import sys
+import argparse
+from functools import partial
+from xml.etree import ElementTree
+
+from mpcdf_common import *
+
+import osc
+import osc.conf
+import osc.core
+import osc.cmdln
+
+default_distribution = "SLE_12_SP1"
+
+@osc.cmdln.option('-n', '--dry-run', action="store_true",
+ help="Do not actually run anything but output the resulting XML configuration")
+
+@osc.cmdln.option('--parent', metavar="PARENT",
+ help="Setup the repositories to be based on the upstream project PARENT (e.g. for home: projects)")
+
+@osc.cmdln.option('--distribution',
+ default=default_distribution,
+ help="Base distribution [default: %default]")
+def do_mpcdf_setup_repos(self, subcmd, opts, *args):
+ """${cmd_name}: Create all repository combinations for an MPCDF project
+
+ Given a list of compilers, MPI libraries, and possibly CUDA versions, this command
+ creates repositories for all the resulting combinations
+
+ Usage:
+ osc ${cmd_name} [PROJECT]
+
+ ${cmd_option_list}
+
+ """
+
+ if len(args) == 0:
+ if is_project_dir(os.curdir) or is_package_dir(os.curdir):
+ project = osc.core.store_read_project(os.curdir)
+ else:
+ raise oscerr.WrongArgs('Specify PROJECT or run command in an osc checkout directory')
+
+ elif len(args) == 1:
+ project, = args
+ else:
+ raise oscerr.WrongArgs("Too many arguments")
+
+ api_url = self.get_api_url()
+
+ compilers = list(get_attribute(api_url, project, None, "MPCDF:compiler_modules"))
+ mpis = list(get_attribute(api_url, project, None, "MPCDF:mpi_modules"))
+ cudas = list(get_attribute(api_url, project, None, "MPCDF:cuda_modules"))
+
+ def project_meta(project):
+ return ElementTree.fromstringlist(osc.core.show_project_meta(api_url, project))
+
+ distributions = project_meta("distributions")
+ dist_repo = distributions.find('./repository[@name="{0}"]'.format(opts.distribution))
+ if dist_repo is None:
+ raise oscerr.WrongArgs("No repository '{0}' is defined in project 'distributions' on the server".format(opts.distribution))
+ architectures = list(arch.text for arch in dist_repo.findall("./arch"))
+
+ root = project_meta(project)
+
+ prjconf = osc.core.show_project_conf(api_url, project)
+ start_marker = "# Autogenerated by osc mpcdf_setup_repos, do not edit till end of section\n"
+ end_marker = "# End of autogenerated section\n"
+
+ try:
+ start = prjconf.index(start_marker)
+ end = prjconf.index(end_marker)
+ except ValueError:
+ start = None
+ end = len(prjconf)
+
+ prjconf_head = "".join(prjconf[:start])
+ prjconf_tail = "".join(prjconf[end+1:])
+ prjconf = [start_marker]
+
+ # Remove existing repositories
+ for repo in root.findall("./repository"):
+ root.remove(repo)
+
+ def repo(name, *dependencies, **kwargs):
+ is_mpi = kwargs.pop("mpi", False)
+ is_cuda = kwargs.pop("cuda", False)
+ if kwargs:
+ raise Exception("Invalid argument")
+ r = ElementTree.SubElement(root, "repository")
+ r.set("name", name)
+ r.text="\n "
+ def path(project, repo):
+ p = ElementTree.SubElement(r, "path")
+ p.set("project", project)
+ p.set("repository", repo)
+ p.tail="\n "
+
+ if opts.parent:
+ path(opts.parent, name)
+ for dep_project, dep_repo in dependencies:
+ path(dep_project, dep_repo)
+
+ for arch in architectures:
+ a = ElementTree.SubElement(r, "arch")
+ a.text = arch
+ a.tail = "\n "
+ a.tail = "\n "
+ r.tail = "\n "
+
+ prjconf.append("%if %_repository == {0}".format(name))
+ prjconf.append("Macros:")
+ prjconf.append("%is_mpi_repository {0}".format(1 if is_mpi else 0))
+ prjconf.append("%is_cuda_repository {0}".format(1 if is_cuda else 0))
+ prjconf.append(":Macros")
+ prjconf.append("%endif")
+ prjconf.append("")
+
+
+ repo("System", ("distributions", opts.distribution))
+
+ for compiler in compilers:
+ repo(compiler, (project, "System"))
+
+ for mpi in filter(partial(valid_mpi, compiler), mpis):
+ repo(mpi + "_" + compiler, (project, compiler), mpi=True)
+
+ for cuda in cudas:
+ for compiler in filter(partial(valid_cuda, cuda), compilers):
+ repo(cuda + "_" + compiler, (project, compiler), cuda=True)
+ for mpi in filter(partial(valid_mpi, compiler), mpis):
+ repo(cuda + "_" + mpi + "_" + compiler,
+ (project, cuda + "_" + compiler),
+ (project, mpi + "_" + compiler),
+ mpi=True, cuda=True)
+
+ root.getchildren()[-1].tail = "\n"
+ prj = ET.tostring(root, encoding=osc.core.ET_ENCODING)
+
+ prjconf.append(end_marker)
+ prjconf = prjconf_head + "\n".join(prjconf) + prjconf_tail
+
+ if opts.dry_run:
+ if len(args) == 0:
+ arg = ""
+ else:
+ arg = " " + project
+ print("osc meta prj{0} -F - <<EOF\n{1}\nEOF\n".format(arg, prj))
+ print("osc meta prjconf{0} -F - <<EOF\n{1}\nEOF\n".format(arg, prjconf))
+ else:
+ # First set-up the <enable/> flags, that way no
+ # spurious builds are launched
+ for package in osc.core.meta_get_packagelist(api_url, project):
+ print("Updating repositories for", package)
+ if not mpcdf_enable_repositories(api_url, project, package):
+ print("ATTENTION: Not changing unmanaged package {0}".format(package))
+
+ # Update repositories
+ print("Updating prj meta")
+ osc.core.edit_meta("prj", project, data=prj)
+ print("Updating prjconf meta")
+ osc.core.edit_meta("prjconf", project, data=prjconf)
+
--
GitLab