mpcdf_common.py 11.1 KB
Newer Older
1
2
from __future__ import print_function

3
__all__ = ["valid_mpi", "valid_cuda", "project_meta", "get_attribute", "get_attribute_values", "set_attribute", "mpcdf_enable_repositories", "mpcdf_setup_repositories"]
4
5
6
7

import osc
import osc.conf
import osc.core
Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
8
import osc.oscerr
9

Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
10
from functools import partial
11
12
13
14
15
16
17
18
19
20
21
from xml.etree import ElementTree


def valid_mpi(compiler, mpi):
    return True


def valid_cuda(cuda, compiler):
    """
    The CUDA distribution only works with certain gcc versions,
    this little function takes care that only supported combinations
Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
22
23
24
25
    are allowed.

    Take care to keep this in sync with the file 'macros.obs_cluster' of
    the package software:dist / mpcdf_cluster_macros
26
    """
Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
27
    if cuda == "cuda_8_0":
28
        return compiler == "gcc_5"
Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
29
    if cuda == "cuda_9_1":
Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
30
        return compiler == "gcc_6"
31
32
33
    return False


Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
34
35
36
def project_meta(api_url, project):
    return ElementTree.fromstringlist(osc.core.show_project_meta(api_url, project))

37

Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
38
def get_attribute(api_url, project, package, attribute, with_project=False):
39
40
    attribute_meta = osc.core.show_attribute_meta(api_url, project, package, None, attribute, False, with_project)
    if attribute_meta is None:
Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
41
        raise osc.oscerr.APIError("Cannot fetch value for attribute '{0}' from {1}".format(attribute, (project, package)))
42
43
44
45

    root = ElementTree.fromstringlist(attribute_meta)
    attribute = root.find("./attribute")
    if attribute is not None:
Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
46
        return root
47
48
49
    else:
        raise Exception("Attribute not set")

Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
50

Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
51
52
53
54
55
56
def get_attribute_values(api_url, project, package, attribute, with_project=False):
    root = get_attribute(api_url, project, package, attribute, with_project)
    attribute = root.find("./attribute")
    return list(value.text for value in attribute.findall("./value"))


57
58
59
60
61
62
def set_attribute(api_url, project, package, attribute):
    path = ["source", project]
    if package:
        path.append(package)
    path.append("_attribute")

Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
63
    attr_meta = ElementTree.tostring(attribute, encoding=osc.core.ET_ENCODING)
Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
64

65
    url = osc.core.makeurl(api_url, path)
Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
66
    resp = ElementTree.fromstringlist(osc.core.streamfile(url, osc.core.http_POST, data=attr_meta))
Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
67
    if resp.find("./summary").text != "Ok":
Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
68
        raise osc.oscerr.APIError("Could not store attribute")
Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
69

70

71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
def set_attribute_values(api_url, project, package, attribute, values):
    root = ElementTree.Element("attributes")
    attr = ElementTree.SubElement(root, "attribute")

    namespace, name = attribute.split(":")
    attr.set("namespace", namespace)
    attr.set("name", name)

    for value in values:
        v = ElementTree.SubElement(attr, "value")
        v.text = value

    set_attribute(api_url, project, package, root)


def has_attribute(api_url, project, package, attribute):
    path = ["source", project]
    if package:
        path.append(package)
    path.append("_attribute")
    path.append(attribute)

    url = osc.core.makeurl(api_url, path)
    resp = ElementTree.fromstringlist(osc.core.streamfile(url, osc.core.http_GET))

    namespace, name = attribute.split(":")
97

98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
    for a in resp.findall("./attribute"):
        if a.get("namespace") == namespace and a.get("name") == name:
            return True
    return False


def remove_attribute(api_url, project, package, attribute_name):
    path = ["source"] + [project]
    if package:
        path.append(package)
    path.append("_attribute")
    path.append(attribute_name)

    url = osc.core.makeurl(api_url, path)
    resp = ElementTree.fromstringlist(osc.core.streamfile(url, osc.core.http_DELETE))
    if resp.find("./summary").text != "Ok":
        raise osc.oscerr.APIError("Could not remove attribute")


def mpcdf_enable_repositories(api_url, project, package, verbose=False):
118
119
    root = ElementTree.fromstringlist(osc.core.show_package_meta(api_url, project, package))
    build = root.find("./build")
Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
120
121
122
123
124
    if build is None:
        build = ElementTree.SubElement(root, "build")
    else:
        for enable in build.findall("./enable"):
            build.remove(enable)
125

Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
126
127
128
    compilers = get_attribute_values(api_url, project, package, "MPCDF:compiler_modules", with_project=True)
    mpis = get_attribute_values(api_url, project, package, "MPCDF:mpi_modules", with_project=True)
    cudas = get_attribute_values(api_url, project, package, "MPCDF:cuda_modules", with_project=True)
129
130
131
132
133
134
135
136
137
138

    def enable(name):
        node = ElementTree.Element("enable")
        node.set("repository", name)
        node.tail = "\n    "
        build.insert(0, node)
        if verbose:
            print("Enabling", name)

    try:
Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
139
        enable_repos = get_attribute_values(api_url, project, package, "MPCDF:enable_repositories")
Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
140
    except Exception:
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
        if verbose:
            print("Warning: Could not get attribute MPCDF:enable_repositories for package {0}".format(package))
        return False

    for flag in enable_repos:

        if flag == "system":
            enable("System")

        if flag == "compilers":
            for compiler in compilers:
                enable(compiler)

        if flag == "mpi":
            for compiler in compilers:
                for mpi in mpis:
                    if valid_mpi(compiler, mpi):
Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
158
                        enable(mpi + "_" + compiler)
159
160
161
162
163
164
165
166
167
168
169
170
171

        if flag == "cuda":
            for cuda in cudas:
                for compiler in compilers:
                    if valid_cuda(cuda, compiler):
                        enable(cuda + "_" + compiler)

        if flag == "cuda_mpi":
            for cuda in cudas:
                for compiler in compilers:
                    if valid_cuda(cuda, compiler):
                        for mpi in mpis:
                            if valid_mpi(compiler, mpi):
172
                                enable(cuda + "_" + mpi + "_" + compiler)
173

Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
174
175
    if len(build.getchildren()) > 0:
        build.getchildren()[-1].tail = "\n  "
176

Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
177
    pkg_meta = ElementTree.tostring(root, encoding=osc.core.ET_ENCODING)
178
179
180

    osc.core.edit_meta("pkg", (project, package), data=pkg_meta)
    return True
Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
181
182


183
def mpcdf_setup_repositories(api_url, project, distribution=None, parent=None, packages=None, dry_run=False):
Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
184
185
    if parent:
        for attribute in ["MPCDF:compiler_modules", "MPCDF:mpi_modules", "MPCDF:cuda_modules"]:
186
            print("Copying attribute '{0}' from parent project".format(attribute))
187
            set_attribute(api_url, project, None, get_attribute(api_url, parent, None, attribute))
Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
188

Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
189
190
191
192
    compilers = list(get_attribute_values(api_url, project, None, "MPCDF:compiler_modules"))
    mpis = list(get_attribute_values(api_url, project, None, "MPCDF:mpi_modules"))
    cudas = list(get_attribute_values(api_url, project, None, "MPCDF:cuda_modules"))

193
194
195
196
197
198
    if distribution is None:
        # Get existing value from project meta
        distribution = project_meta(api_url, project if parent is None else parent).find(
            "./repository[@name='System']/path[@project='distributions']").get("repository")
        print("Using '{0}' as base distribution".format(distribution))

Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
199
200
201
    distributions = project_meta(api_url, "distributions")
    dist_repo = distributions.find('./repository[@name="{0}"]'.format(distribution))
    if dist_repo is None:
Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
202
        raise osc.oscerr.WrongArgs("No repository '{0}' is defined in project 'distributions' on the server".format(distribution))
Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
    architectures = list(arch.text for arch in dist_repo.findall("./arch"))

    root = project_meta(api_url, project)

    prjconf = osc.core.show_project_conf(api_url, project)
    start_marker = "# Autogenerated by osc mpcdf_setup_repos, do not edit till end of section\n"
    end_marker = "# End of autogenerated section\n"

    try:
        start = prjconf.index(start_marker)
        end = prjconf.index(end_marker)
    except ValueError:
        start = None
        end = len(prjconf)

    prjconf_head = "".join(prjconf[:start])
Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
219
    prjconf_tail = "".join(prjconf[end + 1:])
Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
220
221
222
223
224
225
226
227
228
229
230
231
232
    prjconf = [start_marker]

    # Remove existing repositories
    for repo in root.findall("./repository"):
        root.remove(repo)

    def repo(name, *dependencies, **kwargs):
        is_mpi = kwargs.pop("mpi", False)
        is_cuda = kwargs.pop("cuda", False)
        if kwargs:
            raise Exception("Invalid argument")
        r = ElementTree.SubElement(root, "repository")
        r.set("name", name)
Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
233
234
        r.text = "\n    "

Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
235
236
237
238
        def path(project, repo):
            p = ElementTree.SubElement(r, "path")
            p.set("project", project)
            p.set("repository", repo)
Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
239
            p.tail = "\n    "
Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
240
241
242

        for dep_project, dep_repo in dependencies:
            path(dep_project, dep_repo)
243
244
        if parent:
            path(parent, name)
Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260

        for arch in architectures:
            a = ElementTree.SubElement(r, "arch")
            a.text = arch
            a.tail = "\n    "
        a.tail = "\n  "
        r.tail = "\n  "

        prjconf.append("%if %_repository == {0}".format(name))
        prjconf.append("Macros:")
        prjconf.append("%is_mpi_repository {0}".format(1 if is_mpi else 0))
        prjconf.append("%is_cuda_repository {0}".format(1 if is_cuda else 0))
        prjconf.append(":Macros")
        prjconf.append("%endif")
        prjconf.append("")

Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
261
262
263
264
    if parent:
        repo("System")
    else:
        repo("System", ("distributions", distribution))
Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291

    for compiler in compilers:
        repo(compiler, (project, "System"))

        for mpi in filter(partial(valid_mpi, compiler), mpis):
            repo(mpi + "_" + compiler, (project, compiler), mpi=True)

    for cuda in cudas:
        for compiler in filter(partial(valid_cuda, cuda), compilers):
            repo(cuda + "_" + compiler, (project, compiler), cuda=True)
            for mpi in filter(partial(valid_mpi, compiler), mpis):
                repo(cuda + "_" + mpi + "_" + compiler,
                     (project, cuda + "_" + compiler),
                     (project, mpi + "_" + compiler),
                     mpi=True, cuda=True)

    # Remove build configuration
    build = root.find("./build")
    if build is None:
        build = ElementTree.Element("build")
        build.text = "\n    "
        disable = ElementTree.SubElement(build, "disable")
        disable.tail = "\n  "
        build.tail = "\n  "
        root.insert(list(root).index(root.find("./repository")), build)

    root.getchildren()[-1].tail = "\n"
Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
292
    prj = ElementTree.tostring(root, encoding=osc.core.ET_ENCODING)
Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
293
294
295
296
297
298
299
300
301
302

    prjconf.append(end_marker)
    prjconf = prjconf_head + "\n".join(prjconf) + prjconf_tail

    if dry_run:
        print("osc meta prj {0} -F - <<EOF\n{1}\nEOF\n".format(project, prj))
        print("osc meta prjconf {0} -F - <<EOF\n{1}\nEOF\n".format(project, prjconf))
    else:
        # First set-up the <enable/> flags, that way no
        # spurious builds are launched
Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
303
304
305
        if packages is None:
            packages = osc.core.meta_get_packagelist(api_url, project)
        for package in packages:
Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
306
307
308
309
310
311
312
313
314
            print("Updating repositories for", package)
            if not mpcdf_enable_repositories(api_url, project, package):
                print("ATTENTION: Not changing unmanaged package {0}".format(package))

        # Update repositories
        print("Updating prj meta")
        osc.core.edit_meta("prj", project, data=prj)
        print("Updating prjconf meta")
        osc.core.edit_meta("prjconf", project, data=prjconf)