Commit c6a73432 authored by Lorenz Huedepohl's avatar Lorenz Huedepohl
Browse files

New commands `mpcdf_info`, `mpcdf_enable_repositories`

parent 78c6dbd5
from __future__ import print_function
__all__ = ["valid_mpi", "valid_cuda", "get_attribute", "mpcdf_enable_repositories"]
import osc
import osc.conf
import osc.core
from xml.etree import ElementTree
def valid_mpi(compiler, mpi):
"""
It might be possible to use Intel MPI libararies and compilers from
different Parallel Studio packages, but I currently do not want to support
it
"""
if compiler == "intel_17_0":
if mpi.startswith("impi"):
return mpi == "impi_2017_3"
if compiler == "intel_16_0":
if mpi.startswith("impi"):
return mpi == "impi_5_1_3"
return True
def valid_cuda(cuda, compiler):
"""
The CUDA distribution only works with certain gcc versions,
this little function takes care that only supported combinations
are allowed
"""
if cuda == "cuda_8":
return compiler == "gcc_5"
return False
def get_attribute(api_url, project, package, attribute, with_project=False):
attribute_meta = osc.core.show_attribute_meta(api_url, project, package, None, attribute, False, with_project)
if attribute_meta is None:
raise oscerr.APIError("Cannot fetch value for attribute '{0}' from {1}".format(attribute, (project, package)))
root = ElementTree.fromstringlist(attribute_meta)
attribute = root.find("./attribute")
if attribute is not None:
return list(value.text for value in attribute.findall("./value"))
else:
raise Exception("Attribute not set")
def mpcdf_enable_repositories(api_url, project, package, verbose=False):
root = ElementTree.fromstringlist(osc.core.show_package_meta(api_url, project, package))
build = root.find("./build")
for enable in build.findall("./enable"):
build.remove(enable)
compilers = get_attribute(api_url, project, package, "MPCDF:compiler_modules", with_project=True)
mpis = get_attribute(api_url, project, package, "MPCDF:mpi_modules", with_project=True)
cudas = get_attribute(api_url, project, package, "MPCDF:cuda_modules", with_project=True)
def enable(name):
node = ElementTree.Element("enable")
node.set("repository", name)
node.tail = "\n "
build.insert(0, node)
if verbose:
print("Enabling", name)
try:
enable_repos = get_attribute(api_url, project, package, "MPCDF:enable_repositories")
except:
if verbose:
print("Warning: Could not get attribute MPCDF:enable_repositories for package {0}".format(package))
return False
for flag in enable_repos:
if flag == "system":
enable("System")
if flag == "compilers":
for compiler in compilers:
enable(compiler)
if flag == "mpi":
for compiler in compilers:
for mpi in mpis:
if valid_mpi(compiler, mpi):
enable(mpi + "_" + compiler)
if flag == "cuda":
for cuda in cudas:
for compiler in compilers:
if valid_cuda(cuda, compiler):
enable(cuda + "_" + compiler)
if flag == "cuda_mpi":
for cuda in cudas:
for compiler in compilers:
if valid_cuda(cuda, compiler):
for mpi in mpis:
if valid_mpi(compiler, mpi):
enable(cuda + "_" + compiler)
build.getchildren()[-1].tail = "\n "
pkg_meta = ET.tostring(root, encoding=osc.core.ET_ENCODING)
osc.core.edit_meta("pkg", (project, package), data=pkg_meta)
return True
#!/usr/bin/python2
from __future__ import print_function
import sys
import argparse
from functools import partial
from xml.etree import ElementTree
from mpcdf_common import *
import osc
import osc.conf
import osc.core
import osc.cmdln
@osc.cmdln.option('--cuda-mpi', action="store_true",
help="Enable for all CUDA+MPI repositories")
@osc.cmdln.option('--cuda', action="store_true",
help="Enable for all CUDA repositories")
@osc.cmdln.option('--mpi', action="store_true",
help="Enable for all MPI module repositories")
@osc.cmdln.option('--compilers', action="store_true",
help="Enable for all compiler module repositories")
@osc.cmdln.option('--system', action="store_true",
help="Enable for System repository")
@osc.cmdln.option('--reset', action="store_true",
help="Re-create the set of enabled repositories from the MPCDF:enable_repositores attribute")
@osc.cmdln.option('--set', action="store_true",
help="Modify the set of enabled repositories, without this the current setup is displayed")
def do_mpcdf_enable_repositories(self, subcmd, opts, *args):
"""${cmd_name}: Select all appropriate repositories for an MPCDF package
Due to the large number of repository combinations at MPCDF it is
cumbersome to enable all the appropriate repositories for a given package
by hand.
This command allows you to set the <enable/> flags for certain kinds of
repositories at once.
Without --set this command only displays the current configuration.
It is possible to combine this with user-defined <disabled/> flags that
override the settings of this command, for example to disable the build of
the package with a certain repository from a given set (e.g. one
troublesome compiler)
Usage:
osc ${cmd_name} [[PROJECT] PACKAGE]
osc ${cmd_name} --set [ flags ] [[PROJECT] PACKAGE]
osc ${cmd_name} --reset [[PROJECT] PACKAGE]
${cmd_option_list}
"""
if len(args) == 0:
if is_package_dir(os.curdir):
package = osc.core.store_read_package(os.curdir)
project = osc.core.store_read_project(os.curdir)
else:
raise oscerr.WrongArgs('Specify PACKAGE or run command in an osc package checkout directory')
elif len(args) == 1:
package, = args
project = osc.core.store_read_project(os.curdir)
elif len(args) == 2:
project, package = args
else:
raise oscerr.WrongArgs("Too many arguments")
api_url = self.get_api_url()
if opts.set:
root = ElementTree.fromstringlist(osc.core.show_attribute_meta(api_url, project, package, None, "MPCDF:enable_repositories", False, False))
attribute = root.find("./attribute")
if attribute is None:
attribute = ElementTree.SubElement(root, "attribute")
attribute.set("namespace", "MPCDF")
attribute.set("name", "enable_repositories")
for value in attribute.findall("./value"):
attribute.remove(value)
def value(content):
v = ElementTree.SubElement(attribute, "value")
v.text = content
if opts.system:
value("system")
if opts.compilers:
value("compilers")
if opts.mpi:
value("mpi")
if opts.cuda:
value("cuda")
if opts.cuda_mpi:
value("cuda_mpi")
attr_meta = ET.tostring(root, encoding=osc.core.ET_ENCODING)
url = osc.core.makeurl(api_url, ["source", project, package, "_attribute"])
resp = ElementTree.fromstringlist(osc.core.streamfile(url, http_POST, data=attr_meta))
if resp.find("./summary").text != "Ok":
raise osc.core.oscerr.APIError("Could not store attribute MPCDF:enabled_repositories")
if opts.reset or opts.set:
mpcdf_enable_repositories(api_url, project, package, verbose=True)
else:
print("Enabled for:", *get_attribute(api_url, project, package, "MPCDF:enable_repositories"))
#!/usr/bin/python2
from __future__ import print_function
import sys
import argparse
from mpcdf_common import *
import osc
import osc.conf
import osc.core
import osc.cmdln
def do_mpcdf_info(self, subcmd, opts, *args):
"""${cmd_name}: Basic information about an MPCDF OBS project
Usage:
osc ${cmd_name} [PROJECT]
${cmd_option_list}
"""
if len(args) == 0:
if is_project_dir(os.curdir) or is_package_dir(os.curdir):
project = osc.core.store_read_project(os.curdir)
else:
raise oscerr.WrongArgs('Specify PROJECT or run command in an osc checkout directory')
elif len(args) == 1:
project, = args
else:
raise oscerr.WrongArgs("Too many arguments")
print("Project `{0}`\n".format(project))
api_url = self.get_api_url()
def print_attribute(description, attribute):
values = list(sorted(get_attribute(api_url, project, None, attribute)))
if values:
print(" {0}:\n ".format(description) + "\n ".join(values))
print()
print_attribute("Compilers", "MPCDF:compiler_modules")
print_attribute("MPI libraries", "MPCDF:mpi_modules")
print_attribute("CUDA versions", "MPCDF:cuda_modules")
table = []
unmanaged = []
for package in osc.core.meta_get_packagelist(api_url, project):
try:
table.append((package, set(get_attribute(api_url, project, package, "MPCDF:enable_repositories"))))
except:
unmanaged.append(package)
pkg_name_width = reduce(max, (len(t[0]) for t in table))
pkg_name_width = max(pkg_name_width, len("Package"))
r = osc.core.http_request("GET", api_url + "/attribute/MPCDF/enable_repositories/_meta")
root = ET.parse(r).getroot()
kinds = list(value.text for value in root.findall("./allowed/value"))
pkg_name_fmt = "{{:{0}}}".format(pkg_name_width)
print(" " + pkg_name_fmt.format("Package"), "Enabled repositories")
print(" " + "-" * (pkg_name_width), "-" * len("Enabled repositories"))
for package, enabled_repo_kind in table:
print(" " + pkg_name_fmt.format(package), ", ".join(filter(lambda q : q in enabled_repo_kind, kinds)))
print()
if unmanaged:
print(" Unmanaged packages")
print(" " + "-" * (1 + pkg_name_width))
for package in unmanaged:
print(" " + package)
print()
......@@ -2,54 +2,28 @@
from __future__ import print_function
import sys
import argparse
from functools import partial
from xml.etree import ElementTree
from mpcdf_common import *
import osc
import osc.conf
import osc.core
import osc.cmdln
all_compilers = "gcc_4_9 gcc_5 gcc_6 gcc_7 intel_16_0 intel_17_0"
all_mpis = "openmpi_1_10 impi_5_1_3 impi_2017_3"
all_cudas = "cuda_8"
default_distribution = "SLE_12_SP1"
default_architectures = "x86_64"
def valid_mpi(compiler, mpi):
if compiler == "intel_17_0":
if mpi.startswith("impi"):
return mpi == "impi_2017_3"
if compiler == "intel_16_0":
if mpi.startswith("impi"):
return mpi == "impi_5_1_3"
return True
@osc.cmdln.option('--distribution',
default=default_distribution,
help="Base distribution [default: %default]")
@osc.cmdln.option('--architectures',
default=default_architectures,
help="Architectures [default: %default]")
@osc.cmdln.option('--compilers',
default=all_compilers,
help="List of compilers, [default: %default]")
@osc.cmdln.option('--mpis',
default=all_mpis,
help="List of MPI libraries, [default: %default]")
@osc.cmdln.option('--cudas',
default=all_cudas,
help="List of CUDA versions, [default: %default]")
@osc.cmdln.option('-n', '--dry-run', action="store_true",
help="Do not actually run anything but output the resulting XML configuration")
@osc.cmdln.option('--parent', metavar="PARENT",
help="Setup the repositories to be based on the upstream project PARENT (e.g. for home: projects)")
@osc.cmdln.option('-n', '--dry-run', action="store_true",
help="Do not actually run anything but output the resulting XML configuration")
def do_mpcdf_repositories(self, subcmd, opts, *args):
@osc.cmdln.option('--distribution',
default=default_distribution,
help="Base distribution [default: %default]")
def do_mpcdf_setup_repos(self, subcmd, opts, *args):
"""${cmd_name}: Create all repository combinations for an MPCDF project
Given a list of compilers, MPI libraries, and possibly CUDA versions, this command
......@@ -73,19 +47,47 @@ def do_mpcdf_repositories(self, subcmd, opts, *args):
else:
raise oscerr.WrongArgs("Too many arguments")
compilers = opts.compilers.split()
architectures = opts.architectures.split()
mpis = opts.mpis.split()
cudas = opts.cudas.split()
api_url = self.get_api_url()
compilers = list(get_attribute(api_url, project, None, "MPCDF:compiler_modules"))
mpis = list(get_attribute(api_url, project, None, "MPCDF:mpi_modules"))
cudas = list(get_attribute(api_url, project, None, "MPCDF:cuda_modules"))
def project_meta(project):
return ElementTree.fromstringlist(osc.core.show_project_meta(api_url, project))
distributions = project_meta("distributions")
dist_repo = distributions.find('./repository[@name="{0}"]'.format(opts.distribution))
if dist_repo is None:
raise oscerr.WrongArgs("No repository '{0}' is defined in project 'distributions' on the server".format(opts.distribution))
architectures = list(arch.text for arch in dist_repo.findall("./arch"))
root = project_meta(project)
prjconf = osc.core.show_project_conf(api_url, project)
start_marker = "# Autogenerated by osc mpcdf_setup_repos, do not edit till end of section\n"
end_marker = "# End of autogenerated section\n"
try:
start = prjconf.index(start_marker)
end = prjconf.index(end_marker)
except ValueError:
start = None
end = len(prjconf)
project_meta = osc.core.show_project_meta(osc.conf.config["apiurl"], project)
root = ElementTree.fromstringlist(project_meta)
prjconf_head = "".join(prjconf[:start])
prjconf_tail = "".join(prjconf[end+1:])
prjconf = [start_marker]
# Remove existing repositories
for repo in root.findall("./repository"):
root.remove(repo)
def repo(name, *dependencies):
def repo(name, *dependencies, **kwargs):
is_mpi = kwargs.pop("mpi", False)
is_cuda = kwargs.pop("cuda", False)
if kwargs:
raise Exception("Invalid argument")
r = ElementTree.SubElement(root, "repository")
r.set("name", name)
r.text="\n "
......@@ -107,32 +109,56 @@ def do_mpcdf_repositories(self, subcmd, opts, *args):
a.tail = "\n "
r.tail = "\n "
prjconf.append("%if %_repository == {0}".format(name))
prjconf.append("Macros:")
prjconf.append("%is_mpi_repository {0}".format(1 if is_mpi else 0))
prjconf.append("%is_cuda_repository {0}".format(1 if is_cuda else 0))
prjconf.append(":Macros")
prjconf.append("%endif")
prjconf.append("")
repo("System", ("distributions", opts.distribution))
for compiler in sorted(compilers):
for compiler in compilers:
repo(compiler, (project, "System"))
for mpi in sorted(mpis):
for compiler in sorted(compilers):
if not valid_mpi(compiler, mpi):
continue
repo(mpi + "_" + compiler, (project, compiler))
for cuda in sorted(cudas):
repo(cuda, (project, "System"))
for compiler in sorted(compilers):
repo(cuda + "_" + compiler, (project, cuda), (project, compiler))
for mpi in sorted(mpis):
for compiler in sorted(compilers):
if not valid_mpi(compiler, mpi):
continue
for mpi in filter(partial(valid_mpi, compiler), mpis):
repo(mpi + "_" + compiler, (project, compiler), mpi=True)
for cuda in cudas:
for compiler in filter(partial(valid_cuda, cuda), compilers):
repo(cuda + "_" + compiler, (project, compiler), cuda=True)
for mpi in filter(partial(valid_mpi, compiler), mpis):
repo(cuda + "_" + mpi + "_" + compiler,
(project, cuda + "_" + compiler),
(project, mpi + "_" + compiler))
(project, mpi + "_" + compiler),
mpi=True, cuda=True)
root.getchildren()[-1].tail = "\n"
meta = ET.tostring(root, encoding=osc.core.ET_ENCODING)
prj = ET.tostring(root, encoding=osc.core.ET_ENCODING)
prjconf.append(end_marker)
prjconf = prjconf_head + "\n".join(prjconf) + prjconf_tail
if opts.dry_run:
print(meta)
if len(args) == 0:
arg = ""
else:
arg = " " + project
print("osc meta prj{0} -F - <<EOF\n{1}\nEOF\n".format(arg, prj))
print("osc meta prjconf{0} -F - <<EOF\n{1}\nEOF\n".format(arg, prjconf))
else:
osc.core.edit_meta("prj", project, data=meta)
# First set-up the <enable/> flags, that way no
# spurious builds are launched
for package in osc.core.meta_get_packagelist(api_url, project):
print("Updating repositories for", package)
if not mpcdf_enable_repositories(api_url, project, package):
print("ATTENTION: Not changing unmanaged package {0}".format(package))
# Update repositories
print("Updating prj meta")
osc.core.edit_meta("prj", project, data=prj)
print("Updating prjconf meta")
osc.core.edit_meta("prjconf", project, data=prjconf)
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment