Commit 68724d08 authored by Lorenz Huedepohl's avatar Lorenz Huedepohl
Browse files

Merge branch 'obs-api' into 'master'

Merge changes for new OBS from obs-api branch

See merge request !3
parents e34a3079 53d7c88e
Pipeline #71891 passed with stage
in 13 seconds
......@@ -8,7 +8,7 @@ Put the `.py` files into `~/.osc-plugins` to make them available for the osc com
for example by
```
$> git clone https://gitlab.mpcdf.mpg.de/mpcdf/osc-plugins ~/.osc-plugins
$> git clone https://gitlab.mpcdf.mpg.de/mpcdf/obs/osc-plugins ~/.osc-plugins
```
Background
......
#!/usr/bin/python2
from __future__ import print_function
import mpcdf_common
......
This diff is collapsed.
#!/usr/bin/python2
from __future__ import print_function
import mpcdf_common
......@@ -10,6 +9,8 @@ import osc.core
import osc.cmdln
@osc.cmdln.option('-n', '--dry-run', action="store_true",
help="Do not actually perform any changes but print what would be stored")
@osc.cmdln.option('--recreate', action="store_true",
help="Re-create the set of enabled repositories from the stored attributes on the server")
@osc.cmdln.option('--compiler-modules', nargs=1,
......@@ -89,28 +90,29 @@ def do_mpcdf_enable_repositories(self, subcmd, opts, *args):
set_or_remove(opts.pgi_modules, "MPCDF:pgi_modules")
if opts.recreate or opts.set or opts.disable:
mpcdf_common.mpcdf_enable_repositories(api_url, project, package, verbose=True)
if project == "software":
mpcdf_common.sync_projects(api_url, package)
mpcdf_common.mpcdf_enable_repositories(api_url, project, package, verbose=True, dry_run=opts.dry_run)
enabled = mpcdf_common.get_attribute_values(api_url, project, package, "MPCDF:enable_repositories")
if project == "software" and enabled != ["system"] and len(enabled) > 1 and not opts.dry_run:
mpcdf_common.sync_projects(api_url, package, verbose=True)
elif (opts.compiler_modules or opts.mpi_modules or opts.cuda_modules):
print("ERROR: Invalid arguments, try --help")
else:
try:
repos = mpcdf_common.get_attribute_values(api_url, project, package, "MPCDF:enable_repositories")
enabled = mpcdf_common.get_attribute_values(api_url, project, package, "MPCDF:enable_repositories")
except Exception:
repos = ()
enabled = ()
print("ERRROR: No attribute MPCDF:enable_repositories present, package unmanaged")
if repos:
if enabled:
def print_attr(description, attribute_name):
try:
values = mpcdf_common.get_attribute_values(api_url, project, package, attribute_name)
except Exception:
return
print(description, ", ".join(values))
print(description, ",".join(values))
print("Enabled for:", *repos)
print("Enabled for:", ",".join(enabled))
print_attr("- Subset of compiler modules set to:", "MPCDF:compiler_modules")
print_attr("- Subset of MPI modules set to:", "MPCDF:mpi_modules")
print_attr("- Subset of CUDA modules set to:", "MPCDF:cuda_modules")
......
#!/usr/bin/python2
from __future__ import print_function
import time
......@@ -112,7 +111,7 @@ def do_mpcdf_push(self, subcmd, opts, *args):
person.set("userid", user)
person.set("role", "maintainer")
to_meta.insert(2, person)
osc.core.edit_meta("pkg", (to_project, package), data=ElementTree.tostring(to_meta))
osc.core.edit_meta("pkg", (to_project, package), data=ElementTree.tostring(to_meta), apiurl=api_url)
print()
# Give the system some time, sadly there is no transactional guarantee
......
#!/usr/bin/python2
from __future__ import print_function
import os
import osc
import osc.conf
import osc.core
import osc.cmdln
import mpcdf_common
from xml.etree import ElementTree
def do_mpcdf_refresh_aggregates(self, subcmd, opts, *args):
"""${cmd_name}: Recreates the aggregate packages in software:dist
Use this only if you know what this means
Usage:
osc ${cmd_name}
${cmd_option_list}
"""
if len(args) != 0:
raise osc.oscerr.WrongArgs("Too many arguments")
if osc.core.store_read_project(os.curdir) != "software:dist" or \
osc.core.is_package_dir(os.curdir):
raise osc.oscerr.WrongArgs("You must be in the project checkout of software:dist")
dist = osc.core.Project(".", False)
apiurl = self.get_api_url()
def refresh_aggregate(aggregatename, project, source, target):
if not os.path.exists(aggregatename):
package = osc.core.Package.init_package(apiurl, "software:dist", aggregatename, aggregatename)
dist.addPackage(aggregatename)
else:
package = dist.get_pacobj(aggregatename)
with open(aggregatename + "/_aggregate", "w") as fd:
print('<aggregatelist>', file=fd)
print(' <aggregate project="{project}">'.format(project=project), file=fd)
print(' <nosources/>', file=fd)
print(' <repository target="{target}" source="{source}" />'.format(target=target, source=source), file=fd)
print(' </aggregate>', file=fd)
print('</aggregatelist>', file=fd)
if package.status("_aggregate") == "?":
package.addfile("_aggregate")
if package.status("_aggregate") in ("A", "M"):
dist.commit((aggregatename,), msg="Refresh aggregate", verbose=True)
pkg_meta = osc.core.show_package_meta(apiurl, "software:dist", aggregatename)
root = ElementTree.fromstringlist(pkg_meta)
pkg_meta = ElementTree.tostring(root, encoding=osc.core.ET_ENCODING)
build = root.find("./build")
if build is None:
build = ElementTree.SubElement(root, "build")
for enable in build.findall("./enable"):
build.remove(enable)
node = ElementTree.Element("enable")
node.set("repository", target)
node.tail = "\n "
build.insert(0, node)
new_pkg_meta = ElementTree.tostring(root, encoding=osc.core.ET_ENCODING)
if pkg_meta != new_pkg_meta:
osc.core.edit_meta("pkg", ("software:dist", aggregatename),
data=new_pkg_meta)
projects = [p for p in osc.core.meta_get_project_list(apiurl)
if p.startswith("software") and not (p == "software:dist" or p == "software:images")]
macros = {}
for project in projects:
if project == "software":
# Stupid special case
target = "SLE_12_SP3-sandybridge"
else:
target = project[len("software:"):].replace(":", "-")
macros[target] = []
for attribute in mpcdf_common.config_attributes + mpcdf_common.default_attributes:
_, name = attribute.split(":")
values = mpcdf_common.get_attribute_values(apiurl, project, None, attribute)
macros[target].append("%available_{0} {1}".format(name, ",".join(sorted(values))))
all_compilers = mpcdf_common.get_attribute_values(apiurl, project, None, "MPCDF:compiler_modules")
latest_intel = sorted((c for c in all_compilers if c.startswith("intel")), key=mpcdf_common.package_sort_key)[-1]
latest_gcc = sorted((c for c in all_compilers if c.startswith("gcc")), key=mpcdf_common.package_sort_key)[-1]
macros[target].append("%latest_intel " + latest_intel)
macros[target].append("%latest_gcc " + latest_gcc)
for repo in osc.core.get_repositories_of_project(apiurl, project):
aggregatename = "zz_aggregate_" + project.replace(":", "-") + "_" + repo
refresh_aggregate(aggregatename, project, repo, target)
prjconf = list(map(mpcdf_common.decode_it, osc.core.show_project_conf(apiurl, "software:dist")))
start_marker = "# Autogenerated by osc mpcdf_refresh_aggregates, do not edit till end of section\n"
end_marker = "# End of autogenerated section\n"
try:
start = prjconf.index(start_marker)
end = prjconf.index(end_marker)
except ValueError:
start = None
end = len(prjconf)
prjconf_head = "".join(prjconf[:start])
prjconf_tail = "".join(prjconf[end + 1:])
prjconf = [start_marker]
distributions = osc.core.get_repositories_of_project(apiurl, "distributions")
for swdistrepo in osc.core.get_repositories_of_project(apiurl, "software:dist"):
for distrepo in distributions:
if swdistrepo.startswith(distrepo):
refresh_aggregate("zz_aggregate_" + swdistrepo + "_distributions", "distributions", distrepo, swdistrepo)
break
distrepo = None
prjconf.append('%if "%_repository" == "{0}"'.format(swdistrepo))
prjconf.append("Macros:")
prjconf.extend(macros[swdistrepo])
prjconf.append("%distribution {0}".format(distrepo))
prjconf.append(":Macros")
prjconf.append("%endif")
prjconf.append("")
prjconf.append(end_marker)
prjconf = prjconf_head + "\n".join(prjconf) + prjconf_tail
osc.core.edit_meta("prjconf", "software:dist", data=prjconf)
#!/usr/bin/python2
from __future__ import print_function
import os
......
#!/usr/bin/python2
from __future__ import print_function
import mpcdf_common
......
from __future__ import print_function
import osc
import osc.conf
import osc.core
import osc.cmdln
import mpcdf_common
@osc.cmdln.alias("mpcdf_setup_clusters")
def do_mpcdf_setup_cluster_project(self, subcmd, opts, *args):
"""${cmd_name}: Set-up the 'clusters' project
This creates the 'prjconf' for the 'clusters' project
Usage:
osc ${cmd_name}
${cmd_option_list}
"""
if len(args) != 0:
raise osc.oscerr.WrongArgs("Too many arguments")
api_url = self.get_api_url()
projects = [p for p in osc.core.meta_get_project_list(api_url)
if p.startswith("software:")]
macros = {}
for project in projects:
target = project[len("software:"):].replace(":", "-")
if target in macros:
raise Exception("Internal error")
macros[target] = []
for attribute in mpcdf_common.config_attributes + mpcdf_common.default_attributes:
_, name = attribute.split(":")
values = mpcdf_common.overloaded_project_attribute(api_url, project, attribute)
macros[target].append("%available_{0} {1}".format(name, ",".join(sorted(values))))
all_compilers = mpcdf_common.overloaded_project_attribute(api_url, project, "MPCDF:compiler_modules")
latest_intel = sorted((c for c in all_compilers if c.startswith("intel")), key=mpcdf_common.package_sort_key)[-1]
latest_gcc = sorted((c for c in all_compilers if c.startswith("gcc")), key=mpcdf_common.package_sort_key)[-1]
macros[target].append("%latest_intel " + latest_intel)
macros[target].append("%latest_gcc " + latest_gcc)
prjconf = list(map(mpcdf_common.decode_it, osc.core.show_project_conf(api_url, "clusters")))
start_marker = "# Autogenerated by osc mpcdf_setup_clusters_project, do not edit till end of section\n"
end_marker = "# End of autogenerated section\n"
try:
start = prjconf.index(start_marker)
end = prjconf.index(end_marker)
except ValueError:
start = None
end = len(prjconf)
prjconf_head = "".join(prjconf[:start])
prjconf_tail = "".join(prjconf[end + 1:])
prjconf = [start_marker]
distributions = osc.core.get_repositories_of_project(api_url, "software")
for repo in osc.core.get_repositories_of_project(api_url, "clusters"):
for dist in distributions:
if repo.startswith(dist):
break
dist = None
if dist is None:
raise Exception("Internal error")
prjconf.append('%if "%_repository" == "{0}"'.format(repo))
prjconf.append("Macros:")
prjconf.extend(macros[repo])
prjconf.append("%distribution {0}".format(dist))
prjconf.append(":Macros")
prjconf.append("%endif")
prjconf.append("")
prjconf.append(end_marker)
prjconf = prjconf_head + "\n".join(prjconf) + prjconf_tail
osc.core.edit_meta("prjconf", "clusters", data=prjconf, apiurl=api_url)
#!/usr/bin/python2
from __future__ import print_function
from mpcdf_common import mpcdf_setup_repositories
from mpcdf_common import mpcdf_setup_subproject, project_meta, decode_it
import os
import osc
......@@ -12,27 +11,21 @@ import osc.cmdln
@osc.cmdln.option('-n', '--dry-run', action="store_true",
help="Do not actually run anything but output the resulting XML configuration")
@osc.cmdln.option('--parent', metavar="PARENT",
help="Setup the repositories to be based on the upstream project PARENT (e.g. for home: projects)")
@osc.cmdln.option('--distribution',
help="Base distribution, necessary argument unless set previously for this project")
@osc.cmdln.option('--disable-repo', metavar="REPO", action="append",
help="Temporarily disable all repositories containing REPO")
@osc.cmdln.option('--only-project', action="store_true", default=False,
help="Only change project metadata 'prj' and 'prjconf', leave individual packages unchanged")
@osc.cmdln.option('--remove-old', action="store_true", default=False,
help="Remove all obsolete repositories instead of only disabling builds for packages there")
@osc.cmdln.option('--distribution',
help="Base distribution, necessary argument unless set previously for this project")
@osc.cmdln.option('--microarchitecture', metavar="ARCH", nargs=1,
help="Configure project to use ARCH as microarchitecture")
@osc.cmdln.alias("mpcdf_setup_repos")
def do_mpcdf_setup_repositories(self, subcmd, opts, *args):
"""${cmd_name}: Create all repository combinations for an MPCDF project
@osc.cmdln.alias("mpcdf_setup_home")
def do_mpcdf_setup_home_project(self, subcmd, opts, *args):
"""${cmd_name}: Setup a home project based on a software: sub-project
Given a list of compilers, MPI libraries, and possibly CUDA versions, this command
creates repositories for all the resulting combinations
Set-up a home:$USER project for building packages for the 'software'
project hierarchy (i.e. application group packages)
Usage:
osc ${cmd_name} [PROJECT]
osc ${cmd_name} [home:USER]
${cmd_option_list}
......@@ -49,7 +42,36 @@ def do_mpcdf_setup_repositories(self, subcmd, opts, *args):
else:
raise osc.oscerr.WrongArgs("Too many arguments")
mpcdf_setup_repositories(self.get_api_url(),
project, microarchitecture=opts.microarchitecture, distribution=opts.distribution,
parent=opts.parent, dry_run=opts.dry_run, filter_repos=opts.disable_repo,
only_project=opts.only_project, remove_old=opts.remove_old)
if project.split(":")[0] != "home":
raise osc.oscerr.WrongArgs('Given project is not a home: project')
api_url = self.get_api_url()
if opts.distribution is None:
# Get existing value from project meta
dist_repo = project_meta(api_url, project).find(
"./repository[@name='System']/path")
if dist_repo is not None:
distribution = dist_repo.get("repository")
else:
raise osc.oscerr.WrongArgs('Could not determine desired distribution, please specify --distribution explicitly')
else:
distribution = opts.distribution
microarchitecture = None
if opts.microarchitecture is None:
for prjconfline in map(decode_it, osc.core.show_project_conf(api_url, project)):
if prjconfline.startwith("Constraint: hostlabel"):
microarchitecture = prjconfline.split()[2]
break
else:
microarchitecture = opts.microarchitecture
if microarchitecture is None:
raise osc.oscerr.WrongArgs('Could not determine desired microarchitecture, please specify --microarchitecture explicitly')
parent = "software:{0}:{1}".format(distribution, microarchitecture)
mpcdf_setup_subproject(api_url,
project, distribution, microarchitecture,
parent=parent, dry_run=opts.dry_run, remove_old=opts.remove_old)
from __future__ import print_function
import osc
import osc.conf
import osc.core
import osc.cmdln
import mpcdf_common
@osc.cmdln.option('-n', '--dry-run', action="store_true",
help="Do not actually run anything but output the resulting XML configuration")
@osc.cmdln.option('-i', '--ignore-repo', action="append", default=[], metavar="REPO",
help="Do not enable for repository REPO")
@osc.cmdln.alias("mpcdf_setup_software")
def do_mpcdf_setup_software_project(self, subcmd, opts, *args):
"""${cmd_name}: Set-up the 'software' project
This creates the 'prjconf' for the 'software' project and
enables the repositories for packages set to build for
'system'
Usage:
osc ${cmd_name}
${cmd_option_list}
"""
if len(args) != 0:
raise osc.oscerr.WrongArgs("Too many arguments")
api_url = self.get_api_url()
prjconf = list(map(mpcdf_common.decode_it, osc.core.show_project_conf(api_url, "software")))
start_marker = "# Autogenerated by osc mpcdf_setup_software_project, do not edit till end of section\n"
end_marker = "# End of autogenerated section\n"
try:
start = prjconf.index(start_marker)
end = prjconf.index(end_marker)
except ValueError:
start = None
end = len(prjconf)
prjconf_head = "".join(prjconf[:start])
prjconf_tail = "".join(prjconf[end + 1:])
prjconf = [start_marker]
prjconf.append("Constraint: sandbox lxc")
prjconf.append("Constraint: hostlabel sandybridge")
prjconf.append("")
prjconf.append(r"PublishFilter: ^.*\.src\.rpm$")
prjconf.append("")
prjconf.append("Preinstall: mpcdf")
prjconf.append("Runscripts: mpcdf")
prjconf.append("Preinstall: mpcdf_modules")
prjconf.append("Preinstall: brp_mpcdf_modules")
prjconf.append("")
prjconf.append("Macros:")
prjconf.append("%have_mpcdf_compiler 0")
prjconf.append("%have_mpcdf_mpi 0")
prjconf.append("%have_mpcdf_cuda 0")
prjconf.append(":Macros")
software_meta = mpcdf_common.project_meta(api_url, "software")
distributions = sorted(repo.get("name") for repo in software_meta.findall('./repository'))
for distribution in distributions:
extra_tags = mpcdf_common.dist_prjconf_tags(distribution)
extra_macros = mpcdf_common.dist_prjconf_macros(distribution)
if len(extra_tags) > 0 or len(extra_macros) > 0:
prjconf.append("")
prjconf.append("%if \"%_repository\" == \"{0}\"".format(distribution))
prjconf.append(extra_tags)
prjconf.append("")
prjconf.append("Macros:")
prjconf.append(extra_macros)
prjconf.append(":Macros")
prjconf.append("%endif")
prjconf.append(end_marker)
prjconf = prjconf_head + "\n".join(prjconf) + prjconf_tail
if opts.dry_run:
print("osc meta prjconf software -F - <<EOF\n{0}\nEOF\n".format(prjconf))
else:
osc.core.edit_meta("prjconf", "software", data=prjconf, apiurl=api_url)
if opts.ignore_repo:
ignore_repos = opts.ignore_repo
else:
ignore_repos = ()
mpcdf_common.mpcdf_enable_repositories_for_all_packages(api_url, "software", ignore_repos=ignore_repos)
from __future__ import print_function
from mpcdf_common import mpcdf_setup_subproject
import os
import osc
import osc.conf
import osc.core
import osc.cmdln
@osc.cmdln.option('-n', '--dry-run', action="store_true",
help="Do not actually run anything but output the resulting XML configuration")
@osc.cmdln.option('--remove-old', action="store_true", default=False,
help="Remove all obsolete repositories instead of only disabling builds for packages there")
@osc.cmdln.alias("mpcdf_setup_sub")
def do_mpcdf_setup_subproject(self, subcmd, opts, *args):
"""${cmd_name}: Setup a software: sub-project
Given a list of compilers, MPI libraries, and possibly CUDA versions via
the projects attributes, this command creates repositories for all the
resulting combinations
Additionally, the prjconf is set-up to enforce the microarchitecture via
a "Constraint:" statement
Usage:
osc ${cmd_name} [software:OS:microarchitecture]
${cmd_option_list}
"""
if len(args) == 0:
if osc.core.is_project_dir(os.curdir) or osc.core.is_package_dir(os.curdir):
project = osc.core.store_read_project(os.curdir)
else:
raise osc.oscerr.WrongArgs('Specify PROJECT or run command in an osc checkout directory')
elif len(args) == 1:
project, = args
else:
raise osc.oscerr.WrongArgs("Too many arguments")
sw, distribution, microarchitecture = project.split(":")
if sw != "software":
raise osc.oscerr.WrongArgs('Given project is not below the "software" project')
mpcdf_setup_subproject(self.get_api_url(),
project, distribution, microarchitecture,
dry_run=opts.dry_run, remove_old=opts.remove_old)
#!/usr/bin/python2
from __future__ import print_function
import mpcdf_common
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment