Commit 1ae99546 authored by Lorenz Huedepohl's avatar Lorenz Huedepohl Committed by Lorenz Hüdepohl
Browse files

Changes for new structure of obs-api.mpcdf.mpg.de

parent e34a3079
This diff is collapsed.
......@@ -90,8 +90,8 @@ def do_mpcdf_enable_repositories(self, subcmd, opts, *args):
if opts.recreate or opts.set or opts.disable:
mpcdf_common.mpcdf_enable_repositories(api_url, project, package, verbose=True)
if project == "software":
mpcdf_common.sync_projects(api_url, package)
if project == "software" and opts.set != "system":
mpcdf_common.sync_projects(api_url, package, verbose=True)
elif (opts.compiler_modules or opts.mpi_modules or opts.cuda_modules):
print("ERROR: Invalid arguments, try --help")
......
#!/usr/bin/python2
from __future__ import print_function
import os
import osc
import osc.conf
import osc.core
import osc.cmdln
import mpcdf_common
from xml.etree import ElementTree
def do_mpcdf_refresh_aggregates(self, subcmd, opts, *args):
"""${cmd_name}: Recreates the aggregate packages in software:dist
Use this only if you know what this means
Usage:
osc ${cmd_name}
${cmd_option_list}
"""
if len(args) != 0:
raise osc.oscerr.WrongArgs("Too many arguments")
if osc.core.store_read_project(os.curdir) != "software:dist" or \
osc.core.is_package_dir(os.curdir):
raise osc.oscerr.WrongArgs("You must be in the project checkout of software:dist")
dist = osc.core.Project(".", False)
apiurl = self.get_api_url()
def refresh_aggregate(aggregatename, project, source, target):
if not os.path.exists(aggregatename):
package = osc.core.Package.init_package(apiurl, "software:dist", aggregatename, aggregatename)
dist.addPackage(aggregatename)
else:
package = dist.get_pacobj(aggregatename)
with open(aggregatename + "/_aggregate", "w") as fd:
print('<aggregatelist>', file=fd)
print(' <aggregate project="{project}">'.format(project=project), file=fd)
print(' <nosources/>', file=fd)
print(' <repository target="{target}" source="{source}" />'.format(target=target, source=source), file=fd)
print(' </aggregate>', file=fd)
print('</aggregatelist>', file=fd)
if package.status("_aggregate") == "?":
package.addfile("_aggregate")
if package.status("_aggregate") in ("A", "M"):
dist.commit((aggregatename,), msg="Refresh aggregate", verbose=True)
pkg_meta = osc.core.show_package_meta(apiurl, "software:dist", aggregatename)
root = ElementTree.fromstringlist(pkg_meta)
pkg_meta = ElementTree.tostring(root, encoding=osc.core.ET_ENCODING)
build = root.find("./build")
if build is None:
build = ElementTree.SubElement(root, "build")
for enable in build.findall("./enable"):
build.remove(enable)
node = ElementTree.Element("enable")
node.set("repository", target)
node.tail = "\n "
build.insert(0, node)
new_pkg_meta = ElementTree.tostring(root, encoding=osc.core.ET_ENCODING)
if pkg_meta != new_pkg_meta:
osc.core.edit_meta("pkg", ("software:dist", aggregatename),
data=new_pkg_meta)
projects = [p for p in osc.core.meta_get_project_list(apiurl)
if p.startswith("software") and not (p == "software:dist" or p == "software:images")]
macros = {}
for project in projects:
if project == "software":
# Stupid special case
target = "SLE_12_SP3-sandybridge"
else:
target = project[len("software:"):].replace(":", "-")
macros[target] = []
for attribute in mpcdf_common.config_attributes + mpcdf_common.default_attributes:
_, name = attribute.split(":")
values = mpcdf_common.get_attribute_values(apiurl, project, None, attribute)
macros[target].append("%available_{0} {1}".format(name, ",".join(sorted(values))))
all_compilers = mpcdf_common.get_attribute_values(apiurl, project, None, "MPCDF:compiler_modules")
latest_intel = sorted((c for c in all_compilers if c.startswith("intel")), key=mpcdf_common.package_sort_key)[-1]
latest_gcc = sorted((c for c in all_compilers if c.startswith("gcc")), key=mpcdf_common.package_sort_key)[-1]
macros[target].append("%latest_intel " + latest_intel)
macros[target].append("%latest_gcc " + latest_gcc)
for repo in osc.core.get_repositories_of_project(apiurl, project):
aggregatename = "zz_aggregate_" + project.replace(":", "-") + "_" + repo
refresh_aggregate(aggregatename, project, repo, target)
prjconf = list(map(mpcdf_common.decode_it, osc.core.show_project_conf(apiurl, "software:dist")))
start_marker = "# Autogenerated by osc mpcdf_refresh_aggregates, do not edit till end of section\n"
end_marker = "# End of autogenerated section\n"
try:
start = prjconf.index(start_marker)
end = prjconf.index(end_marker)
except ValueError:
start = None
end = len(prjconf)
prjconf_head = "".join(prjconf[:start])
prjconf_tail = "".join(prjconf[end + 1:])
prjconf = [start_marker]
distributions = osc.core.get_repositories_of_project(apiurl, "distributions")
for swdistrepo in osc.core.get_repositories_of_project(apiurl, "software:dist"):
for distrepo in distributions:
if swdistrepo.startswith(distrepo):
refresh_aggregate("zz_aggregate_" + swdistrepo + "_distributions", "distributions", distrepo, swdistrepo)
break
distrepo = None
prjconf.append('%if "%_repository" == "{0}"'.format(swdistrepo))
prjconf.append("Macros:")
prjconf.extend(macros[swdistrepo])
prjconf.append("%distribution {0}".format(distrepo))
prjconf.append(":Macros")
prjconf.append("%endif")
prjconf.append("")
prjconf.append(end_marker)
prjconf = prjconf_head + "\n".join(prjconf) + prjconf_tail
osc.core.edit_meta("prjconf", "software:dist", data=prjconf)
from __future__ import print_function
import osc
import osc.conf
import osc.core
import osc.cmdln
import mpcdf_common
@osc.cmdln.alias("mpcdf_setup_clusters")
def do_mpcdf_setup_cluster_project(self, subcmd, opts, *args):
"""${cmd_name}: Set-up the 'clusters' project
This creates the 'prjconf' for the 'clusters' project
Usage:
osc ${cmd_name}
${cmd_option_list}
"""
if len(args) != 0:
raise osc.oscerr.WrongArgs("Too many arguments")
api_url = self.get_api_url()
projects = [p for p in osc.core.meta_get_project_list(api_url)
if p.startswith("software:")]
macros = {}
for project in projects:
target = project[len("software:"):].replace(":", "-")
if target in macros:
raise Exception("Internal error")
macros[target] = []
for attribute in mpcdf_common.config_attributes + mpcdf_common.default_attributes:
_, name = attribute.split(":")
values = mpcdf_common.overloaded_project_attribute(api_url, project, attribute)
macros[target].append("%available_{0} {1}".format(name, ",".join(sorted(values))))
all_compilers = mpcdf_common.overloaded_project_attribute(api_url, project, "MPCDF:compiler_modules")
latest_intel = sorted((c for c in all_compilers if c.startswith("intel")), key=mpcdf_common.package_sort_key)[-1]
latest_gcc = sorted((c for c in all_compilers if c.startswith("gcc")), key=mpcdf_common.package_sort_key)[-1]
macros[target].append("%latest_intel " + latest_intel)
macros[target].append("%latest_gcc " + latest_gcc)
prjconf = list(map(mpcdf_common.decode_it, osc.core.show_project_conf(api_url, "clusters")))
start_marker = "# Autogenerated by osc mpcdf_setup_clusters_project, do not edit till end of section\n"
end_marker = "# End of autogenerated section\n"
try:
start = prjconf.index(start_marker)
end = prjconf.index(end_marker)
except ValueError:
start = None
end = len(prjconf)
prjconf_head = "".join(prjconf[:start])
prjconf_tail = "".join(prjconf[end + 1:])
prjconf = [start_marker]
distributions = osc.core.get_repositories_of_project(api_url, "software")
for repo in osc.core.get_repositories_of_project(api_url, "clusters"):
for dist in distributions:
if repo.startswith(dist):
break
dist = None
if dist is None:
raise Exception("Internal error")
prjconf.append('%if "%_repository" == "{0}"'.format(repo))
prjconf.append("Macros:")
prjconf.extend(macros[repo])
prjconf.append("%distribution {0}".format(dist))
prjconf.append(":Macros")
prjconf.append("%endif")
prjconf.append("")
prjconf.append(end_marker)
prjconf = prjconf_head + "\n".join(prjconf) + prjconf_tail
osc.core.edit_meta("prjconf", "clusters", data=prjconf)
#!/usr/bin/python2
from __future__ import print_function
from mpcdf_common import mpcdf_setup_subproject, project_meta, decode_it
import os
import osc
import osc.conf
import osc.core
import osc.cmdln
@osc.cmdln.option('-n', '--dry-run', action="store_true",
help="Do not actually run anything but output the resulting XML configuration")
@osc.cmdln.option('--remove-old', action="store_true", default=False,
help="Remove all obsolete repositories instead of only disabling builds for packages there")
@osc.cmdln.option('--distribution',
help="Base distribution, necessary argument unless set previously for this project")
@osc.cmdln.option('--microarchitecture', metavar="ARCH", nargs=1,
help="Configure project to use ARCH as microarchitecture")
@osc.cmdln.alias("mpcdf_setup_home")
def do_mpcdf_setup_home_project(self, subcmd, opts, *args):
"""${cmd_name}: Setup a home project based on a software: sub-project
Set-up a home:$USER project for building packages for the 'software'
project hierarchy (i.e. application group packages)
Usage:
osc ${cmd_name} [home:USER]
${cmd_option_list}
"""
if len(args) == 0:
if osc.core.is_project_dir(os.curdir) or osc.core.is_package_dir(os.curdir):
project = osc.core.store_read_project(os.curdir)
else:
raise osc.oscerr.WrongArgs('Specify PROJECT or run command in an osc checkout directory')
elif len(args) == 1:
project, = args
else:
raise osc.oscerr.WrongArgs("Too many arguments")
if project.split(":")[0] != "home":
raise osc.oscerr.WrongArgs('Given project is not a home: project')
api_url = self.get_api_url()
if opts.distribution is None:
# Get existing value from project meta
dist_repo = project_meta(api_url, project).find(
"./repository[@name='System']/path")
if dist_repo is not None:
distribution = dist_repo.get("repository")
else:
raise osc.oscerr.WrongArgs('Could not determine desired distribution, please specify --distribution explicitly')
else:
distribution = opts.distribution
microarchitecture = None
if opts.microarchitecture is None:
for prjconfline in map(decode_it, osc.core.show_project_conf(api_url, project)):
if prjconfline.startwith("Constraint: hostlabel"):
microarchitecture = prjconfline.split()[2]
break
else:
microarchitecture = opts.microarchitecture
if microarchitecture is None:
raise osc.oscerr.WrongArgs('Could not determine desired microarchitecture, please specify --microarchitecture explicitly')
parent = "software:{0}:{1}".format(distribution, microarchitecture)
mpcdf_setup_subproject(api_url,
project, distribution, microarchitecture,
parent=parent, dry_run=opts.dry_run, remove_old=opts.remove_old)
#!/usr/bin/python2
from __future__ import print_function
import osc
import osc.conf
import osc.core
import osc.cmdln
import mpcdf_common
@osc.cmdln.option('-i', '--ignore-repo', action="append", default=[], metavar="REPO",
help="Do not enable for repository REPO")
@osc.cmdln.alias("mpcdf_setup_software")
def do_mpcdf_setup_software_project(self, subcmd, opts, *args):
"""${cmd_name}: Set-up the 'software' project
This creates the 'prjconf' for the 'software' project and
enables the repositories for packages set to build for
'system'
Usage:
osc ${cmd_name}
${cmd_option_list}
"""
if len(args) != 0:
raise osc.oscerr.WrongArgs("Too many arguments")
api_url = self.get_api_url()
prjconf = list(map(mpcdf_common.decode_it, osc.core.show_project_conf(api_url, "software")))
start_marker = "# Autogenerated by osc mpcdf_setup_software_project, do not edit till end of section\n"
end_marker = "# End of autogenerated section\n"
try:
start = prjconf.index(start_marker)
end = prjconf.index(end_marker)
except ValueError:
start = None
end = len(prjconf)
prjconf_head = "".join(prjconf[:start])
prjconf_tail = "".join(prjconf[end + 1:])
prjconf = [start_marker]
prjconf.append("Constraint: sandbox lxc")
prjconf.append("Constraint: hostlabel sandybridge")
prjconf.append("")
prjconf.append(r"PublishFilter: ^.*\.src\.rpm$")
prjconf.append("")
prjconf.append("Preinstall: mpcdf")
prjconf.append("Preinstall: mpcdf_modules")
prjconf.append("Preinstall: brp_mpcdf_modules")
prjconf.append(end_marker)
prjconf = prjconf_head + "\n".join(prjconf) + prjconf_tail
osc.core.edit_meta("prjconf", "software", data=prjconf)
if opts.ignore_repo:
ignore_repos = opts.ignore_repo
else:
ignore_repos = ()
mpcdf_common.mpcdf_enable_repositories_for_all_packages(api_url, "software", ignore_repos=ignore_repos)
#!/usr/bin/python2
from __future__ import print_function
from mpcdf_common import mpcdf_setup_repositories
from mpcdf_common import mpcdf_setup_subproject
import os
import osc
......@@ -9,30 +9,23 @@ import osc.conf
import osc.core
import osc.cmdln
@osc.cmdln.option('-n', '--dry-run', action="store_true",
help="Do not actually run anything but output the resulting XML configuration")
@osc.cmdln.option('--parent', metavar="PARENT",
help="Setup the repositories to be based on the upstream project PARENT (e.g. for home: projects)")
@osc.cmdln.option('--distribution',
help="Base distribution, necessary argument unless set previously for this project")
@osc.cmdln.option('--disable-repo', metavar="REPO", action="append",
help="Temporarily disable all repositories containing REPO")
@osc.cmdln.option('--only-project', action="store_true", default=False,
help="Only change project metadata 'prj' and 'prjconf', leave individual packages unchanged")
@osc.cmdln.option('--remove-old', action="store_true", default=False,
help="Remove all obsolete repositories instead of only disabling builds for packages there")
@osc.cmdln.option('--microarchitecture', metavar="ARCH", nargs=1,
help="Configure project to use ARCH as microarchitecture")
@osc.cmdln.alias("mpcdf_setup_repos")
def do_mpcdf_setup_repositories(self, subcmd, opts, *args):
"""${cmd_name}: Create all repository combinations for an MPCDF project
@osc.cmdln.alias("mpcdf_setup_sub")
def do_mpcdf_setup_subproject(self, subcmd, opts, *args):
"""${cmd_name}: Setup a software: sub-project
Given a list of compilers, MPI libraries, and possibly CUDA versions via
the projects attributes, this command creates repositories for all the
resulting combinations
Given a list of compilers, MPI libraries, and possibly CUDA versions, this command
creates repositories for all the resulting combinations
Additionally, the prjconf is set-up to enforce the microarchitecture via
a "Constraint:" statement
Usage:
osc ${cmd_name} [PROJECT]
osc ${cmd_name} [software:OS:microarchitecture]
${cmd_option_list}
......@@ -49,7 +42,11 @@ def do_mpcdf_setup_repositories(self, subcmd, opts, *args):
else:
raise osc.oscerr.WrongArgs("Too many arguments")
mpcdf_setup_repositories(self.get_api_url(),
project, microarchitecture=opts.microarchitecture, distribution=opts.distribution,
parent=opts.parent, dry_run=opts.dry_run, filter_repos=opts.disable_repo,
only_project=opts.only_project, remove_old=opts.remove_old)
sw, distribution, microarchitecture = project.split(":")
if sw != "software":
raise osc.oscerr.WrongArgs('Given project is not below the "software" project')
mpcdf_setup_subproject(self.get_api_url(),
project, distribution, microarchitecture,
dry_run=opts.dry_run, remove_old=opts.remove_old)
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment