Commit 90656e1d authored by Lorenz Huedepohl's avatar Lorenz Huedepohl
Browse files

New command 'mpcdf_push'

and some internal re-orginisation
parent 374f5616
Pipeline #67076 failed with stage
in 11 seconds
from __future__ import print_function
__all__ = ["valid_mpi", "valid_cuda", "get_attribute", "mpcdf_enable_repositories"]
__all__ = ["valid_mpi", "valid_cuda", "project_meta", "get_attribute", "get_attribute_values", "set_attribute", "mpcdf_enable_repositories", "mpcdf_setup_repos"]
import osc
import osc.conf
import osc.core
from functools import partial
from xml.etree import ElementTree
......@@ -35,8 +36,11 @@ def valid_cuda(cuda, compiler):
return False
def get_attribute(api_url, project, package, attribute, with_project=False):
def project_meta(api_url, project):
return ElementTree.fromstringlist(osc.core.show_project_meta(api_url, project))
def get_attribute(api_url, project, package, attribute, with_project=False):
attribute_meta = osc.core.show_attribute_meta(api_url, project, package, None, attribute, False, with_project)
if attribute_meta is None:
raise oscerr.APIError("Cannot fetch value for attribute '{0}' from {1}".format(attribute, (project, package)))
......@@ -44,21 +48,38 @@ def get_attribute(api_url, project, package, attribute, with_project=False):
root = ElementTree.fromstringlist(attribute_meta)
attribute = root.find("./attribute")
if attribute is not None:
return list(value.text for value in attribute.findall("./value"))
return root
else:
raise Exception("Attribute not set")
def get_attribute_values(api_url, project, package, attribute, with_project=False):
root = get_attribute(api_url, project, package, attribute, with_project)
attribute = root.find("./attribute")
return list(value.text for value in attribute.findall("./value"))
def set_attribute(api_url, path, attribute):
attr_meta = ET.tostring(attribute, encoding=osc.core.ET_ENCODING)
url = osc.core.makeurl(api_url, ["source"] + list(path) + ["_attribute"])
resp = ElementTree.fromstringlist(osc.core.streamfile(url, http_POST, data=attr_meta))
if resp.find("./summary").text != "Ok":
raise osc.core.oscerr.APIError("Could not store attribute")
def mpcdf_enable_repositories(api_url, project, package, verbose=False):
root = ElementTree.fromstringlist(osc.core.show_package_meta(api_url, project, package))
build = root.find("./build")
for enable in build.findall("./enable"):
build.remove(enable)
if build is None:
build = ElementTree.SubElement(root, "build")
else:
for enable in build.findall("./enable"):
build.remove(enable)
compilers = get_attribute(api_url, project, package, "MPCDF:compiler_modules", with_project=True)
mpis = get_attribute(api_url, project, package, "MPCDF:mpi_modules", with_project=True)
cudas = get_attribute(api_url, project, package, "MPCDF:cuda_modules", with_project=True)
compilers = get_attribute_values(api_url, project, package, "MPCDF:compiler_modules", with_project=True)
mpis = get_attribute_values(api_url, project, package, "MPCDF:mpi_modules", with_project=True)
cudas = get_attribute_values(api_url, project, package, "MPCDF:cuda_modules", with_project=True)
def enable(name):
node = ElementTree.Element("enable")
......@@ -69,7 +90,7 @@ def mpcdf_enable_repositories(api_url, project, package, verbose=False):
print("Enabling", name)
try:
enable_repos = get_attribute(api_url, project, package, "MPCDF:enable_repositories")
enable_repos = get_attribute_values(api_url, project, package, "MPCDF:enable_repositories")
except:
if verbose:
print("Warning: Could not get attribute MPCDF:enable_repositories for package {0}".format(package))
......@@ -104,9 +125,130 @@ def mpcdf_enable_repositories(api_url, project, package, verbose=False):
if valid_mpi(compiler, mpi):
enable(cuda + "_" + compiler)
build.getchildren()[-1].tail = "\n "
if len(build.getchildren()) > 0:
build.getchildren()[-1].tail = "\n "
pkg_meta = ET.tostring(root, encoding=osc.core.ET_ENCODING)
osc.core.edit_meta("pkg", (project, package), data=pkg_meta)
return True
def mpcdf_setup_repos(api_url, project, distribution, parent=None, dry_run=False):
compilers = list(get_attribute_values(api_url, project, None, "MPCDF:compiler_modules"))
mpis = list(get_attribute_values(api_url, project, None, "MPCDF:mpi_modules"))
cudas = list(get_attribute_values(api_url, project, None, "MPCDF:cuda_modules"))
distributions = project_meta(api_url, "distributions")
dist_repo = distributions.find('./repository[@name="{0}"]'.format(distribution))
if dist_repo is None:
raise oscerr.WrongArgs("No repository '{0}' is defined in project 'distributions' on the server".format(distribution))
architectures = list(arch.text for arch in dist_repo.findall("./arch"))
root = project_meta(api_url, project)
prjconf = osc.core.show_project_conf(api_url, project)
start_marker = "# Autogenerated by osc mpcdf_setup_repos, do not edit till end of section\n"
end_marker = "# End of autogenerated section\n"
try:
start = prjconf.index(start_marker)
end = prjconf.index(end_marker)
except ValueError:
start = None
end = len(prjconf)
prjconf_head = "".join(prjconf[:start])
prjconf_tail = "".join(prjconf[end+1:])
prjconf = [start_marker]
# Remove existing repositories
for repo in root.findall("./repository"):
root.remove(repo)
def repo(name, *dependencies, **kwargs):
is_mpi = kwargs.pop("mpi", False)
is_cuda = kwargs.pop("cuda", False)
if kwargs:
raise Exception("Invalid argument")
r = ElementTree.SubElement(root, "repository")
r.set("name", name)
r.text="\n "
def path(project, repo):
p = ElementTree.SubElement(r, "path")
p.set("project", project)
p.set("repository", repo)
p.tail="\n "
if parent:
path(parent, name)
for dep_project, dep_repo in dependencies:
path(dep_project, dep_repo)
for arch in architectures:
a = ElementTree.SubElement(r, "arch")
a.text = arch
a.tail = "\n "
a.tail = "\n "
r.tail = "\n "
prjconf.append("%if %_repository == {0}".format(name))
prjconf.append("Macros:")
prjconf.append("%is_mpi_repository {0}".format(1 if is_mpi else 0))
prjconf.append("%is_cuda_repository {0}".format(1 if is_cuda else 0))
prjconf.append(":Macros")
prjconf.append("%endif")
prjconf.append("")
repo("System", ("distributions", distribution))
for compiler in compilers:
repo(compiler, (project, "System"))
for mpi in filter(partial(valid_mpi, compiler), mpis):
repo(mpi + "_" + compiler, (project, compiler), mpi=True)
for cuda in cudas:
for compiler in filter(partial(valid_cuda, cuda), compilers):
repo(cuda + "_" + compiler, (project, compiler), cuda=True)
for mpi in filter(partial(valid_mpi, compiler), mpis):
repo(cuda + "_" + mpi + "_" + compiler,
(project, cuda + "_" + compiler),
(project, mpi + "_" + compiler),
mpi=True, cuda=True)
# Remove build configuration
build = root.find("./build")
if build is None:
build = ElementTree.Element("build")
build.text = "\n "
disable = ElementTree.SubElement(build, "disable")
disable.tail = "\n "
build.tail = "\n "
root.insert(list(root).index(root.find("./repository")), build)
root.getchildren()[-1].tail = "\n"
prj = ET.tostring(root, encoding=osc.core.ET_ENCODING)
prjconf.append(end_marker)
prjconf = prjconf_head + "\n".join(prjconf) + prjconf_tail
if dry_run:
print("osc meta prj {0} -F - <<EOF\n{1}\nEOF\n".format(project, prj))
print("osc meta prjconf {0} -F - <<EOF\n{1}\nEOF\n".format(project, prjconf))
else:
# First set-up the <enable/> flags, that way no
# spurious builds are launched
for package in osc.core.meta_get_packagelist(api_url, project):
print("Updating repositories for", package)
if not mpcdf_enable_repositories(api_url, project, package):
print("ATTENTION: Not changing unmanaged package {0}".format(package))
# Update repositories
print("Updating prj meta")
osc.core.edit_meta("prj", project, data=prj)
print("Updating prjconf meta")
osc.core.edit_meta("prjconf", project, data=prjconf)
......@@ -96,14 +96,9 @@ def do_mpcdf_enable_repositories(self, subcmd, opts, *args):
if opts.cuda_mpi:
value("cuda_mpi")
attr_meta = ET.tostring(root, encoding=osc.core.ET_ENCODING)
url = osc.core.makeurl(api_url, ["source", project, package, "_attribute"])
resp = ElementTree.fromstringlist(osc.core.streamfile(url, http_POST, data=attr_meta))
if resp.find("./summary").text != "Ok":
raise osc.core.oscerr.APIError("Could not store attribute MPCDF:enabled_repositories")
set_attribute(api_url, (project, package), root)
if opts.reset or opts.set:
mpcdf_enable_repositories(api_url, project, package, verbose=True)
else:
print("Enabled for:", *get_attribute(api_url, project, package, "MPCDF:enable_repositories"))
print("Enabled for:", *get_attribute_values(api_url, project, package, "MPCDF:enable_repositories"))
......@@ -36,7 +36,7 @@ def do_mpcdf_info(self, subcmd, opts, *args):
api_url = self.get_api_url()
def print_attribute(description, attribute):
values = list(sorted(get_attribute(api_url, project, None, attribute)))
values = list(sorted(get_attribute_values(api_url, project, None, attribute)))
if values:
print(" {0}:\n ".format(description) + "\n ".join(values))
print()
......@@ -50,7 +50,7 @@ def do_mpcdf_info(self, subcmd, opts, *args):
for package in osc.core.meta_get_packagelist(api_url, project):
try:
table.append((package, set(get_attribute(api_url, project, package, "MPCDF:enable_repositories"))))
table.append((package, set(get_attribute_values(api_url, project, package, "MPCDF:enable_repositories"))))
except:
unmanaged.append(package)
......
#!/usr/bin/python2
from __future__ import print_function
import sys
import argparse
from functools import partial
from xml.etree import ElementTree
from mpcdf_common import *
from mpcdf_setup_repos import do_mpcdf_setup_repos
import osc
import osc.conf
import osc.core
import osc.cmdln
def do_mpcdf_push(self, subcmd, opts, from_project, to_project):
"""${cmd_name}: Branch all packages from one project to another"
All packages from FROM_PROJECT (that do not yet exist in the target
project) are branched into TO_PROJECT.
Packages that already exist are not branched/modified but the repository
configuration is updated to match the attribute settings from FROM_PROJECT.
${cmd_usage}
${cmd_option_list}
"""
api_url = self.get_api_url()
from_packages = osc.core.meta_get_packagelist(api_url, from_project)
to_packages = osc.core.meta_get_packagelist(api_url, to_project)
project_attributes = ["MPCDF:compiler_modules", "MPCDF:cuda_modules", "MPCDF:mpi_modules"]
package_attributes = ["MPCDF:enable_repositories"] + project_attributes
for attribute in project_attributes:
try:
get_attribute(api_url, to_project, None, attribute)
except:
attr = get_attribute(api_url, from_project, None, attribute)
print("Setting attribute", attribute)
set_attribute(api_url, (to_project,), attr)
for package in from_packages:
if not package in to_packages:
l = osc.core.meta_get_filelist(api_url, from_project, package)
if "_link" in l:
print("Skipping package {0}, is a link".format(package))
else:
print("Branching package {0}".format(package))
exists, targetprj, targetpkg, srcprj, srcpkg = \
branch_pkg(api_url, from_project, package, target_project=to_project)
else:
print("Skipping package {0}, already present in target".format(package))
for attribute in package_attributes:
try:
attr = get_attribute(api_url, from_project, package, attribute)
except:
continue
set_attribute(api_url, (to_project, package), attr)
# Check if distribution is already set in to_project
to_prj_meta = project_meta(api_url, to_project)
sys_repo = to_prj_meta.find('./repository[@name="System"]')
if sys_repo is None:
distribution = project_meta(api_url, from_project).find('./repository[@name="System"]/path[@project="distributions"]').get("repository")
else:
distribution = sys_repo.find('./path[@project="distributions"]').get("repository")
print("Creating repository configuration")
mpcdf_setup_repos(api_url, to_project, distribution)
......@@ -2,7 +2,6 @@
from __future__ import print_function
import sys
import argparse
from functools import partial
from xml.etree import ElementTree
from mpcdf_common import *
......@@ -47,118 +46,4 @@ def do_mpcdf_setup_repos(self, subcmd, opts, *args):
else:
raise oscerr.WrongArgs("Too many arguments")
api_url = self.get_api_url()
compilers = list(get_attribute(api_url, project, None, "MPCDF:compiler_modules"))
mpis = list(get_attribute(api_url, project, None, "MPCDF:mpi_modules"))
cudas = list(get_attribute(api_url, project, None, "MPCDF:cuda_modules"))
def project_meta(project):
return ElementTree.fromstringlist(osc.core.show_project_meta(api_url, project))
distributions = project_meta("distributions")
dist_repo = distributions.find('./repository[@name="{0}"]'.format(opts.distribution))
if dist_repo is None:
raise oscerr.WrongArgs("No repository '{0}' is defined in project 'distributions' on the server".format(opts.distribution))
architectures = list(arch.text for arch in dist_repo.findall("./arch"))
root = project_meta(project)
prjconf = osc.core.show_project_conf(api_url, project)
start_marker = "# Autogenerated by osc mpcdf_setup_repos, do not edit till end of section\n"
end_marker = "# End of autogenerated section\n"
try:
start = prjconf.index(start_marker)
end = prjconf.index(end_marker)
except ValueError:
start = None
end = len(prjconf)
prjconf_head = "".join(prjconf[:start])
prjconf_tail = "".join(prjconf[end+1:])
prjconf = [start_marker]
# Remove existing repositories
for repo in root.findall("./repository"):
root.remove(repo)
def repo(name, *dependencies, **kwargs):
is_mpi = kwargs.pop("mpi", False)
is_cuda = kwargs.pop("cuda", False)
if kwargs:
raise Exception("Invalid argument")
r = ElementTree.SubElement(root, "repository")
r.set("name", name)
r.text="\n "
def path(project, repo):
p = ElementTree.SubElement(r, "path")
p.set("project", project)
p.set("repository", repo)
p.tail="\n "
if opts.parent:
path(opts.parent, name)
for dep_project, dep_repo in dependencies:
path(dep_project, dep_repo)
for arch in architectures:
a = ElementTree.SubElement(r, "arch")
a.text = arch
a.tail = "\n "
a.tail = "\n "
r.tail = "\n "
prjconf.append("%if %_repository == {0}".format(name))
prjconf.append("Macros:")
prjconf.append("%is_mpi_repository {0}".format(1 if is_mpi else 0))
prjconf.append("%is_cuda_repository {0}".format(1 if is_cuda else 0))
prjconf.append(":Macros")
prjconf.append("%endif")
prjconf.append("")
repo("System", ("distributions", opts.distribution))
for compiler in compilers:
repo(compiler, (project, "System"))
for mpi in filter(partial(valid_mpi, compiler), mpis):
repo(mpi + "_" + compiler, (project, compiler), mpi=True)
for cuda in cudas:
for compiler in filter(partial(valid_cuda, cuda), compilers):
repo(cuda + "_" + compiler, (project, compiler), cuda=True)
for mpi in filter(partial(valid_mpi, compiler), mpis):
repo(cuda + "_" + mpi + "_" + compiler,
(project, cuda + "_" + compiler),
(project, mpi + "_" + compiler),
mpi=True, cuda=True)
root.getchildren()[-1].tail = "\n"
prj = ET.tostring(root, encoding=osc.core.ET_ENCODING)
prjconf.append(end_marker)
prjconf = prjconf_head + "\n".join(prjconf) + prjconf_tail
if opts.dry_run:
if len(args) == 0:
arg = ""
else:
arg = " " + project
print("osc meta prj{0} -F - <<EOF\n{1}\nEOF\n".format(arg, prj))
print("osc meta prjconf{0} -F - <<EOF\n{1}\nEOF\n".format(arg, prjconf))
else:
# First set-up the <enable/> flags, that way no
# spurious builds are launched
for package in osc.core.meta_get_packagelist(api_url, project):
print("Updating repositories for", package)
if not mpcdf_enable_repositories(api_url, project, package):
print("ATTENTION: Not changing unmanaged package {0}".format(package))
# Update repositories
print("Updating prj meta")
osc.core.edit_meta("prj", project, data=prj)
print("Updating prjconf meta")
osc.core.edit_meta("prjconf", project, data=prjconf)
mpcdf_setup_repos(self.get_api_url(), project, opts.distribution, parent=opts.parent, dry_run=opts.dry_run)
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment