mpcdf_common.py 13.6 KB
Newer Older
1
2
3
4
5
from __future__ import print_function

import osc
import osc.conf
import osc.core
Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
6
import osc.oscerr
7

Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
8
from functools import partial
9
10
11
12
13
14
15
16
17
18
19
from xml.etree import ElementTree


def valid_mpi(compiler, mpi):
    return True


def valid_cuda(cuda, compiler):
    """
    The CUDA distribution only works with certain gcc versions,
    this little function takes care that only supported combinations
Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
20
21
22
23
    are allowed.

    Take care to keep this in sync with the file 'macros.obs_cluster' of
    the package software:dist / mpcdf_cluster_macros
24
    """
Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
25
    if cuda == "cuda_8_0":
26
        return compiler == "gcc_5"
Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
27
    if cuda == "cuda_9_1":
Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
28
        return compiler == "gcc_6"
29
30
31
    return False


Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
32
33
34
def project_meta(api_url, project):
    return ElementTree.fromstringlist(osc.core.show_project_meta(api_url, project))

35

36
37
38
39
class UnsetAttributeException(Exception):
    pass


Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
40
def get_attribute(api_url, project, package, attribute, with_project=False):
41
42
    attribute_meta = osc.core.show_attribute_meta(api_url, project, package, None, attribute, False, with_project)
    if attribute_meta is None:
Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
43
        raise osc.oscerr.APIError("Cannot fetch value for attribute '{0}' from {1}".format(attribute, (project, package)))
44
45
46
47

    root = ElementTree.fromstringlist(attribute_meta)
    attribute = root.find("./attribute")
    if attribute is not None:
Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
48
        return root
49
    else:
50
        raise UnsetAttributeException("Attribute not set")
51

Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
52

Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
53
54
55
56
57
58
def get_attribute_values(api_url, project, package, attribute, with_project=False):
    root = get_attribute(api_url, project, package, attribute, with_project)
    attribute = root.find("./attribute")
    return list(value.text for value in attribute.findall("./value"))


59
60
61
62
63
64
def set_attribute(api_url, project, package, attribute):
    path = ["source", project]
    if package:
        path.append(package)
    path.append("_attribute")

Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
65
    attr_meta = ElementTree.tostring(attribute, encoding=osc.core.ET_ENCODING)
Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
66

67
    url = osc.core.makeurl(api_url, path)
Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
68
    resp = ElementTree.fromstringlist(osc.core.streamfile(url, osc.core.http_POST, data=attr_meta))
Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
69
    if resp.find("./summary").text != "Ok":
Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
70
        raise osc.oscerr.APIError("Could not store attribute")
Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
71

72

73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
def set_attribute_values(api_url, project, package, attribute, values):
    root = ElementTree.Element("attributes")
    attr = ElementTree.SubElement(root, "attribute")

    namespace, name = attribute.split(":")
    attr.set("namespace", namespace)
    attr.set("name", name)

    for value in values:
        v = ElementTree.SubElement(attr, "value")
        v.text = value

    set_attribute(api_url, project, package, root)


def has_attribute(api_url, project, package, attribute):
    path = ["source", project]
    if package:
        path.append(package)
    path.append("_attribute")
    path.append(attribute)

    url = osc.core.makeurl(api_url, path)
    resp = ElementTree.fromstringlist(osc.core.streamfile(url, osc.core.http_GET))

    namespace, name = attribute.split(":")
99

100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
    for a in resp.findall("./attribute"):
        if a.get("namespace") == namespace and a.get("name") == name:
            return True
    return False


def remove_attribute(api_url, project, package, attribute_name):
    path = ["source"] + [project]
    if package:
        path.append(package)
    path.append("_attribute")
    path.append(attribute_name)

    url = osc.core.makeurl(api_url, path)
    resp = ElementTree.fromstringlist(osc.core.streamfile(url, osc.core.http_DELETE))
    if resp.find("./summary").text != "Ok":
        raise osc.oscerr.APIError("Could not remove attribute")


def mpcdf_enable_repositories(api_url, project, package, verbose=False):
120
121
    root = ElementTree.fromstringlist(osc.core.show_package_meta(api_url, project, package))
    build = root.find("./build")
Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
122
123
124
125
126
    if build is None:
        build = ElementTree.SubElement(root, "build")
    else:
        for enable in build.findall("./enable"):
            build.remove(enable)
127

Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
128
    compilers = get_attribute_values(api_url, project, package, "MPCDF:compiler_modules", with_project=True)
129
    all_compilers = get_attribute_values(api_url, project, None, "MPCDF:compiler_modules")
Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
130
131
    mpis = get_attribute_values(api_url, project, package, "MPCDF:mpi_modules", with_project=True)
    cudas = get_attribute_values(api_url, project, package, "MPCDF:cuda_modules", with_project=True)
132
133
134
135
136
137
138
139
140
141

    def enable(name):
        node = ElementTree.Element("enable")
        node.set("repository", name)
        node.tail = "\n    "
        build.insert(0, node)
        if verbose:
            print("Enabling", name)

    try:
Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
142
        enable_repos = get_attribute_values(api_url, project, package, "MPCDF:enable_repositories")
Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
143
    except Exception:
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
        if verbose:
            print("Warning: Could not get attribute MPCDF:enable_repositories for package {0}".format(package))
        return False

    for flag in enable_repos:

        if flag == "system":
            enable("System")

        if flag == "compilers":
            for compiler in compilers:
                enable(compiler)

        if flag == "mpi":
            for compiler in compilers:
                for mpi in mpis:
                    if valid_mpi(compiler, mpi):
Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
161
                        enable(mpi + "_" + compiler)
162
163
164

        if flag == "cuda":
            for cuda in cudas:
165
                for compiler in all_compilers:
166
167
168
169
170
                    if valid_cuda(cuda, compiler):
                        enable(cuda + "_" + compiler)

        if flag == "cuda_mpi":
            for cuda in cudas:
171
                for compiler in all_compilers:
172
173
174
                    if valid_cuda(cuda, compiler):
                        for mpi in mpis:
                            if valid_mpi(compiler, mpi):
175
                                enable(cuda + "_" + mpi + "_" + compiler)
176

Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
177
178
    if len(build.getchildren()) > 0:
        build.getchildren()[-1].tail = "\n  "
179

Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
180
    pkg_meta = ElementTree.tostring(root, encoding=osc.core.ET_ENCODING)
181
182
183

    osc.core.edit_meta("pkg", (project, package), data=pkg_meta)
    return True
Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
184
185


186
def mpcdf_setup_repositories(api_url, project, distribution=None, parent=None, packages=None, dry_run=False):
Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
187
188
    if parent:
        for attribute in ["MPCDF:compiler_modules", "MPCDF:mpi_modules", "MPCDF:cuda_modules"]:
189
            print("Copying attribute '{0}' from parent project".format(attribute))
190
            set_attribute(api_url, project, None, get_attribute(api_url, parent, None, attribute))
Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
191

Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
192
193
194
195
    compilers = list(get_attribute_values(api_url, project, None, "MPCDF:compiler_modules"))
    mpis = list(get_attribute_values(api_url, project, None, "MPCDF:mpi_modules"))
    cudas = list(get_attribute_values(api_url, project, None, "MPCDF:cuda_modules"))

196
197
    if distribution is None:
        # Get existing value from project meta
198
199
200
201
202
203
        dist_repo = project_meta(api_url, project if parent is None else parent).find(
            "./repository[@name='System']/path[@project='distributions']")
        if dist_repo is not None:
            distribution = dist_repo.get("repository")
        else:
            raise osc.oscerr.WrongArgs("ERROR: Specify distribution or parent project")
204
205
        print("Using '{0}' as base distribution".format(distribution))

Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
206
207
208
    distributions = project_meta(api_url, "distributions")
    dist_repo = distributions.find('./repository[@name="{0}"]'.format(distribution))
    if dist_repo is None:
Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
209
        raise osc.oscerr.WrongArgs("No repository '{0}' is defined in project 'distributions' on the server".format(distribution))
Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
    architectures = list(arch.text for arch in dist_repo.findall("./arch"))

    root = project_meta(api_url, project)

    prjconf = osc.core.show_project_conf(api_url, project)
    start_marker = "# Autogenerated by osc mpcdf_setup_repos, do not edit till end of section\n"
    end_marker = "# End of autogenerated section\n"

    try:
        start = prjconf.index(start_marker)
        end = prjconf.index(end_marker)
    except ValueError:
        start = None
        end = len(prjconf)

    prjconf_head = "".join(prjconf[:start])
Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
226
    prjconf_tail = "".join(prjconf[end + 1:])
Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
227
228
229
    prjconf = [start_marker]

    # Remove existing repositories
230
231
    for oldrepo in root.findall("./repository"):
        root.remove(oldrepo)
Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
232
233

    def repo(name, *dependencies, **kwargs):
234
        is_compiler = kwargs.pop("compiler", False)
Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
235
236
        is_mpi = kwargs.pop("mpi", False)
        is_cuda = kwargs.pop("cuda", False)
237
238
239
240
241
242
        is_cuda_mpi = kwargs.pop("cuda_mpi", False)

        have_compiler = is_compiler or is_mpi or is_cuda or is_cuda_mpi
        have_mpi = is_mpi or is_cuda_mpi
        have_cuda = is_cuda or is_cuda_mpi

Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
243
244
        r = ElementTree.SubElement(root, "repository")
        r.set("name", name)
Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
245
246
        r.text = "\n    "

Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
247
248
249
250
        def path(project, repo):
            p = ElementTree.SubElement(r, "path")
            p.set("project", project)
            p.set("repository", repo)
Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
251
            p.tail = "\n    "
Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
252
253
254

        for dep_project, dep_repo in dependencies:
            path(dep_project, dep_repo)
255
256
        if parent:
            path(parent, name)
Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
257
258
259
260
261
262
263
264
265
266

        for arch in architectures:
            a = ElementTree.SubElement(r, "arch")
            a.text = arch
            a.tail = "\n    "
        a.tail = "\n  "
        r.tail = "\n  "

        prjconf.append("%if %_repository == {0}".format(name))
        prjconf.append("Macros:")
267
268

        prjconf.append("%is_compiler_repository {0}".format(1 if is_compiler else 0))
Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
269
270
        prjconf.append("%is_mpi_repository {0}".format(1 if is_mpi else 0))
        prjconf.append("%is_cuda_repository {0}".format(1 if is_cuda else 0))
271
272
273
274
275
276
        prjconf.append("%is_cuda_mpi_repository {0}".format(1 if is_cuda_mpi else 0))

        prjconf.append("%have_mpcdf_compiler {0}".format(1 if have_compiler else 0))
        prjconf.append("%have_mpcdf_mpi {0}".format(1 if have_mpi else 0))
        prjconf.append("%have_mpcdf_cuda {0}".format(1 if have_cuda else 0))

277
278
279
        for macro, value in kwargs.items():
            prjconf.append("%{0} {1}".format(macro, value))

Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
280
281
282
283
        prjconf.append(":Macros")
        prjconf.append("%endif")
        prjconf.append("")

Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
284
285
286
287
    if parent:
        repo("System")
    else:
        repo("System", ("distributions", distribution))
Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
288
289

    for compiler in compilers:
290
        repo(compiler, (project, "System"), compiler=True)
Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
291
292

        for mpi in filter(partial(valid_mpi, compiler), mpis):
293
            repo(mpi + "_" + compiler, (project, compiler), mpi=True, mpi_repository=mpi)
Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
294
295
296
297
298
299
300
301

    for cuda in cudas:
        for compiler in filter(partial(valid_cuda, cuda), compilers):
            repo(cuda + "_" + compiler, (project, compiler), cuda=True)
            for mpi in filter(partial(valid_mpi, compiler), mpis):
                repo(cuda + "_" + mpi + "_" + compiler,
                     (project, cuda + "_" + compiler),
                     (project, mpi + "_" + compiler),
302
                     cuda_mpi=True)
Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
303
304
305
306
307
308
309
310
311
312
313
314

    # Remove build configuration
    build = root.find("./build")
    if build is None:
        build = ElementTree.Element("build")
        build.text = "\n    "
        disable = ElementTree.SubElement(build, "disable")
        disable.tail = "\n  "
        build.tail = "\n  "
        root.insert(list(root).index(root.find("./repository")), build)

    root.getchildren()[-1].tail = "\n"
Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
315
    prj = ElementTree.tostring(root, encoding=osc.core.ET_ENCODING)
Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
316
317
318
319
320
321
322
323
324
325

    prjconf.append(end_marker)
    prjconf = prjconf_head + "\n".join(prjconf) + prjconf_tail

    if dry_run:
        print("osc meta prj {0} -F - <<EOF\n{1}\nEOF\n".format(project, prj))
        print("osc meta prjconf {0} -F - <<EOF\n{1}\nEOF\n".format(project, prjconf))
    else:
        # First set-up the <enable/> flags, that way no
        # spurious builds are launched
Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
326
327
328
        if packages is None:
            packages = osc.core.meta_get_packagelist(api_url, project)
        for package in packages:
Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
329
330
331
332
333
334
335
336
337
            print("Updating repositories for", package)
            if not mpcdf_enable_repositories(api_url, project, package):
                print("ATTENTION: Not changing unmanaged package {0}".format(package))

        # Update repositories
        print("Updating prj meta")
        osc.core.edit_meta("prj", project, data=prj)
        print("Updating prjconf meta")
        osc.core.edit_meta("prjconf", project, data=prjconf)
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377


def set_as_branch(api_url, my_project, my_package, main_project, main_package):
    import os

    def dirmeta(project, package):
        url = osc.core.makeurl(api_url, ["source", project, package])
        return ElementTree.fromstringlist(osc.core.streamfile(url, osc.core.http_GET))

    srcmeta = dirmeta(my_project, my_package)
    dstmeta = dirmeta(main_project, main_package)

    linkinfo = srcmeta.find('./linkinfo')
    if linkinfo is not None:
        print("ERROR: package {0} is already linked to package {1} in project {2}".format(
              my_package, linkinfo.get("package"), linkinfo.get("project")))
        return False

    linkfile = ElementTree.Element("link")
    linkfile.set("project", main_project)
    linkfile.set("package", main_package)
    linkfile.set("baserev", dstmeta.get("srcmd5"))
    patches = ElementTree.SubElement(linkfile, "patches")
    ElementTree.SubElement(patches, "branch")

    url = osc.core.makeurl(api_url, ["source", my_project, my_package, "_link"])
    resp = ElementTree.fromstringlist(osc.core.streamfile(url, osc.core.http_PUT, data=ElementTree.tostring(linkfile)))
    rev = resp.get("rev")
    if rev is None:
        print("ERROR: Could not commit _link file")
        return False
    else:
        print("Commited as revision", rev)
        this_package = osc.core.store_read_package(os.curdir)
        this_project = osc.core.store_read_project(os.curdir)
        if this_package == my_package and this_project == my_project:
            pac = osc.core.filedir_to_pac(os.curdir)
            rev = pac.latest_rev(expand=True)
            pac.update(rev)
    return True