mpcdf_common.py 25 KB
Newer Older
1
2
3
4
5
from __future__ import print_function

import osc
import osc.conf
import osc.core
Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
6
import osc.oscerr
7

Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
8
from functools import partial
9
10
from xml.etree import ElementTree

11
12
13
14
package_attributes = ["MPCDF:enable_repositories"]
config_attributes = ["MPCDF:compiler_modules", "MPCDF:cuda_modules", "MPCDF:mpi_modules", "MPCDF:pgi_modules"]
default_attributes = ["MPCDF:default_compiler", "MPCDF:default_cuda", "MPCDF:default_mpi"]

15
16
17
18
19
20
21
22
intel_parallel_studio = {
    "mpcdf_intel_parallel_studio_2017_7": {"compiler": "intel_17_0_7", "impi": "impi_2017_4", "mkl": "mkl_2017_4-module", },
    "mpcdf_intel_parallel_studio_2018_1": {"compiler": "intel_18_0_1", "impi": "impi_2018_1", "mkl": "mkl_2018_1-module", },
    "mpcdf_intel_parallel_studio_2018_2": {"compiler": "intel_18_0_2", "impi": "impi_2018_2", "mkl": "mkl_2018_2-module", },
    "mpcdf_intel_parallel_studio_2018_3": {"compiler": "intel_18_0_3", "impi": "impi_2018_3", "mkl": "mkl_2018_3-module", },
    "mpcdf_intel_parallel_studio_2018_4": {"compiler": "intel_18_0_5", "impi": "impi_2018_4", "mkl": "mkl_2018_4-module", },
    "mpcdf_intel_parallel_studio_2019_0": {"compiler": "intel_19_0_0", "impi": "impi_2019_0", "mkl": "mkl_2019_0-module", },
    "mpcdf_intel_parallel_studio_2019_1": {"compiler": "intel_19_0_1", "impi": "impi_2019_1", "mkl": "mkl_2019_1-module", },
Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
23
    "mpcdf_intel_parallel_studio_2019_3": {"compiler": "intel_19_0_3", "impi": "impi_2019_3", "mkl": "mkl_2019_3-module", },
24
25
26
27
28
29
30
31
}

all_mkls = {ic["mkl"] for ic in intel_parallel_studio.values()}

# Some rotations
mpi_parallel_studio = {value["impi"]: dict({"ps": key}, **value) for key, value in intel_parallel_studio.items()}
compiler_parallel_studio = {value["compiler"]: dict({"ps": key}, **value) for key, value in intel_parallel_studio.items()}

32

33
34
35
36
37
38
39
40
def valid_pgi_mpi(pgi, mpi):
    if "impi" not in mpi:
        return False
    if "2017" in mpi:
        return False
    return True


41
def valid_mpi(compiler, mpi):
42
43
44
45
46
47
48
49
50
    """
    It might be possible to use Intel MPI libararies and compilers from
    different Parallel Studio packages, but I currently do not want to support
    it.

    Take care to keep this in sync with the file 'macros.obs_cluster' of
    the package software:dist / mpcdf_cluster_macros
    """
    if compiler.startswith("intel") and mpi.startswith("impi"):
51
        return mpi == compiler_parallel_studio[compiler]["impi"]
52
53
    if compiler.startswith("pgi"):
        return valid_pgi_mpi(compiler, mpi)
54
55
    else:
        return True
56
57


58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
def prefers(reponame):
    prefer_ps = None
    if reponame in compiler_parallel_studio:
        prefer_ps = compiler_parallel_studio[reponame]["ps"]
    else:
        for mpi in mpi_parallel_studio:
            if reponame.startswith(mpi):
                prefer_ps = mpi_parallel_studio[mpi]["ps"]

    if prefer_ps:
        preferred_mkl = intel_parallel_studio[prefer_ps]["mkl"]
        unprefer_other_mkls = ("!" + mkl for mkl in all_mkls if mkl != preferred_mkl)
        return (prefer_ps,) + tuple(unprefer_other_mkls) + (preferred_mkl,)
    else:
        return ()


75
76
77
78
def valid_cuda(cuda, compiler):
    """
    The CUDA distribution only works with certain gcc versions,
    this little function takes care that only supported combinations
Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
79
80
81
82
    are allowed.

    Take care to keep this in sync with the file 'macros.obs_cluster' of
    the package software:dist / mpcdf_cluster_macros
83
    """
Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
84
    if cuda == "cuda_8_0":
85
        return compiler == "gcc_5"
Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
86
    if cuda == "cuda_9_1":
87
        return compiler == "gcc_6_3_0"
Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
88
    if cuda == "cuda_9_2":
Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
89
        return compiler == "gcc_6"
Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
90
91
    if cuda == "cuda_10_0":
        return compiler == "gcc_6"
Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
92
93
    if cuda == "cuda_10_1":
        return compiler == "gcc_8"
94
95
96
    return False


Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
97
98
99
def project_meta(api_url, project):
    return ElementTree.fromstringlist(osc.core.show_project_meta(api_url, project))

100

101
102
103
104
105
106
107
108
109
def package_meta(api_url, project, package):
    return ElementTree.fromstringlist(osc.core.show_package_meta(api_url, project, package))


def maintainers(api_url, project, package):
    root = package_meta(api_url, project, package)
    return {e.get("userid") for e in root.findall("./person[@role='maintainer']")}


110
111
112
113
class UnsetAttributeException(Exception):
    pass


114
115
116
117
118
119
120
121
def chunked(l, chunksize):
    n = len(l)
    i = 0
    while i * chunksize < n:
        yield l[i * chunksize: (i + 1) * chunksize]
        i += 1


Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
122
def get_attribute(api_url, project, package, attribute, with_project=False):
123
124
    attribute_meta = osc.core.show_attribute_meta(api_url, project, package, None, attribute, False, with_project)
    if attribute_meta is None:
Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
125
        raise osc.oscerr.APIError("Cannot fetch value for attribute '{0}' from {1}".format(attribute, (project, package)))
126
127
128
129

    root = ElementTree.fromstringlist(attribute_meta)
    attribute = root.find("./attribute")
    if attribute is not None:
Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
130
        return root
131
    else:
132
        raise UnsetAttributeException("Attribute not set")
133

Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
134

Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
135
136
137
138
139
140
def get_attribute_values(api_url, project, package, attribute, with_project=False):
    root = get_attribute(api_url, project, package, attribute, with_project)
    attribute = root.find("./attribute")
    return list(value.text for value in attribute.findall("./value"))


141
142
143
144
145
146
def set_attribute(api_url, project, package, attribute):
    path = ["source", project]
    if package:
        path.append(package)
    path.append("_attribute")

Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
147
    attr_meta = ElementTree.tostring(attribute, encoding=osc.core.ET_ENCODING)
Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
148

149
    url = osc.core.makeurl(api_url, path)
Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
150
    resp = ElementTree.fromstringlist(osc.core.streamfile(url, osc.core.http_POST, data=attr_meta))
Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
151
    if resp.find("./summary").text != "Ok":
Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
152
        raise osc.oscerr.APIError("Could not store attribute")
Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
153

154

155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
def set_attribute_values(api_url, project, package, attribute, values):
    root = ElementTree.Element("attributes")
    attr = ElementTree.SubElement(root, "attribute")

    namespace, name = attribute.split(":")
    attr.set("namespace", namespace)
    attr.set("name", name)

    for value in values:
        v = ElementTree.SubElement(attr, "value")
        v.text = value

    set_attribute(api_url, project, package, root)


def has_attribute(api_url, project, package, attribute):
    path = ["source", project]
    if package:
        path.append(package)
    path.append("_attribute")
    path.append(attribute)

    url = osc.core.makeurl(api_url, path)
    resp = ElementTree.fromstringlist(osc.core.streamfile(url, osc.core.http_GET))

    namespace, name = attribute.split(":")
181

182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
    for a in resp.findall("./attribute"):
        if a.get("namespace") == namespace and a.get("name") == name:
            return True
    return False


def remove_attribute(api_url, project, package, attribute_name):
    path = ["source"] + [project]
    if package:
        path.append(package)
    path.append("_attribute")
    path.append(attribute_name)

    url = osc.core.makeurl(api_url, path)
    resp = ElementTree.fromstringlist(osc.core.streamfile(url, osc.core.http_DELETE))
    if resp.find("./summary").text != "Ok":
        raise osc.oscerr.APIError("Could not remove attribute")


201
def mpcdf_enable_repositories(api_url, project, package, verbose=False, filter_repos=None):
202
    from itertools import product
Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
203
    import sys
204

205
206
207
    if filter_repos is None:
        filter_repos = ()

208
    root = package_meta(api_url, project, package)
209
    build = root.find("./build")
Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
210
211
212
213
214
    if build is None:
        build = ElementTree.SubElement(root, "build")
    else:
        for enable in build.findall("./enable"):
            build.remove(enable)
215

Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
216
    try:
217
218
219
220
221
        enable_repos = get_attribute_values(api_url, project, package, "MPCDF:enable_repositories")
    except Exception:
        if verbose:
            print("Warning: Could not get attribute MPCDF:enable_repositories for package {0}, skipping".format(package))
        return False
Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
222

Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
223
    def try_get_attribute(package, attribute, with_project=False):
224
225
226
        try:
            return get_attribute_values(api_url, project, package, "MPCDF:" + attribute, with_project=with_project)
        except UnsetAttributeException:
227
228
229
            print("ERROR: Attribute MPCDF:" + attribute + " is not set for "
                  + ("package '{0}'".format(package) if package else "project '{0}'".format(project))
                  + ", aborting here", file=sys.stderr)
230
            raise SystemExit(1)
Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
231

232
233
234
    compilers = try_get_attribute(package, "compiler_modules", with_project=True)
    mpis = try_get_attribute(package, "mpi_modules", with_project=True)
    cudas = try_get_attribute(package, "cuda_modules", with_project=True)
235
236
237
238
239
240
    pgis = try_get_attribute(package, "pgi_modules", with_project=True)

    all_compilers = try_get_attribute(None, "compiler_modules")
    all_mpis = try_get_attribute(None, "mpi_modules")
    all_cudas = try_get_attribute(None, "cuda_modules")
    all_pgis = try_get_attribute(None, "pgi_modules")
241
242
243
244

    default_compilers = try_get_attribute(None, "default_compiler")
    default_mpis = try_get_attribute(None, "default_mpi")
    default_cudas = try_get_attribute(None, "default_cuda")
245

246
    def enable(name):
247
248
        if any(filtered_repo in name for filtered_repo in filter_repos):
            return
249
250
251
252
253
254
255
        node = ElementTree.Element("enable")
        node.set("repository", name)
        node.tail = "\n    "
        build.insert(0, node)
        if verbose:
            print("Enabling", name)

256
    def actual_compilers():
257
        for compiler in (c for c in compilers if c in all_compilers + ["default_compiler", "intel", "gcc"]):
258
            if compiler == "intel":
Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
259
                for intel_compiler in filter(lambda cc: cc.startswith("intel"), all_compilers):
260
261
                    yield intel_compiler
            elif compiler == "gcc":
Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
262
                for gcc_compiler in filter(lambda cc: cc.startswith("gcc"), all_compilers):
263
264
265
266
267
268
269
270
                    yield gcc_compiler
            elif compiler == "default_compiler":
                for default_compiler in default_compilers:
                    yield default_compiler
            else:
                yield compiler

    def actual_mpis():
271
        for mpi in (m for m in mpis if m in all_mpis + ["default_mpi", "impi"]):
272
273
274
275
276
277
278
279
280
281
            if mpi == "impi":
                for impi in filter(lambda cc: cc.startswith("impi"), mpis):
                    yield impi
            elif mpi == "default_mpi":
                for default_mpi in default_mpis:
                    yield default_mpi
            else:
                yield mpi

    def actual_cudas():
282
        for cuda in (c for c in cudas if c in all_cudas + ["default_cuda"]):
283
284
285
286
287
288
            if cuda == "default_cuda":
                for default_cuda in default_cudas:
                    yield default_cuda
            else:
                yield cuda

289
290
291
292
    def actual_pgis():
        for pgi in (p for p in pgis if p in all_pgis):
            yield pgi

293
294
295
296
297
298
    for flag in enable_repos:

        if flag == "system":
            enable("System")

        if flag == "compilers":
299
            for compiler in actual_compilers():
Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
300
                enable(compiler)
301
302

        if flag == "mpi":
303
304
305
            for mpi, compiler in product(actual_mpis(), actual_compilers()):
                if valid_mpi(compiler, mpi):
                    enable(mpi + "_" + compiler)
306
307

        if flag == "cuda":
308
309
310
            for cuda, compiler in product(actual_cudas(), all_compilers):
                if valid_cuda(cuda, compiler):
                    enable(cuda + "_" + compiler)
311
312

        if flag == "cuda_mpi":
313
314
            for cuda, mpi, compiler in product(actual_cudas(), actual_mpis(), all_compilers):
                if valid_cuda(cuda, compiler) and valid_mpi(compiler, mpi):
315
                    enable(cuda + "_" + mpi + "_" + compiler)
316

317
318
319
320
321
322
        if flag == "pgi":
            for pgi in actual_pgis():
                enable(pgi)

        if flag == "pgi_mpi":
            for mpi, pgi in product(actual_mpis(), actual_pgis()):
323
324
                if valid_pgi_mpi(pgi, mpi):
                    enable(mpi + "_" + pgi)
325

Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
326
327
    if len(build.getchildren()) > 0:
        build.getchildren()[-1].tail = "\n  "
328

Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
329
    pkg_meta = ElementTree.tostring(root, encoding=osc.core.ET_ENCODING)
330
331
332

    osc.core.edit_meta("pkg", (project, package), data=pkg_meta)
    return True
Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
333
334


335
def mpcdf_setup_repositories(api_url, project, distribution=None, parent=None, packages=None, dry_run=False, filter_repos=None, only_project=False, remove_old=False):
336
337
    import threading

Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
338
    if parent:
339
340
341
        for attribute in config_attributes + default_attributes:
            print("Copying attribute '{0}' from parent project".format(attribute))
            set_attribute(api_url, project, None, get_attribute(api_url, parent, None, attribute))
Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
342

Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
343
344
345
    compilers = list(get_attribute_values(api_url, project, None, "MPCDF:compiler_modules"))
    mpis = list(get_attribute_values(api_url, project, None, "MPCDF:mpi_modules"))
    cudas = list(get_attribute_values(api_url, project, None, "MPCDF:cuda_modules"))
346
    pgis = list(get_attribute_values(api_url, project, None, "MPCDF:pgi_modules"))
Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
347

348
349
    if distribution is None:
        # Get existing value from project meta
350
351
352
353
354
355
        dist_repo = project_meta(api_url, project if parent is None else parent).find(
            "./repository[@name='System']/path[@project='distributions']")
        if dist_repo is not None:
            distribution = dist_repo.get("repository")
        else:
            raise osc.oscerr.WrongArgs("ERROR: Specify distribution or parent project")
356
357
        print("Using '{0}' as base distribution".format(distribution))

Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
358
359
360
    distributions = project_meta(api_url, "distributions")
    dist_repo = distributions.find('./repository[@name="{0}"]'.format(distribution))
    if dist_repo is None:
Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
361
        raise osc.oscerr.WrongArgs("No repository '{0}' is defined in project 'distributions' on the server".format(distribution))
Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
    architectures = list(arch.text for arch in dist_repo.findall("./arch"))

    root = project_meta(api_url, project)

    prjconf = osc.core.show_project_conf(api_url, project)
    start_marker = "# Autogenerated by osc mpcdf_setup_repos, do not edit till end of section\n"
    end_marker = "# End of autogenerated section\n"

    try:
        start = prjconf.index(start_marker)
        end = prjconf.index(end_marker)
    except ValueError:
        start = None
        end = len(prjconf)

    prjconf_head = "".join(prjconf[:start])
Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
378
    prjconf_tail = "".join(prjconf[end + 1:])
Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
379
380
381
    prjconf = [start_marker]

    # Remove existing repositories
382
383
384
    if remove_old:
        for oldrepo in root.findall("./repository"):
            root.remove(oldrepo)
Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
385
386

    def repo(name, *dependencies, **kwargs):
387
        is_compiler = kwargs.pop("compiler", False)
Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
388
389
        is_mpi = kwargs.pop("mpi", False)
        is_cuda = kwargs.pop("cuda", False)
390
391
        is_cuda_mpi = kwargs.pop("cuda_mpi", False)

392
393
        cuda_repo = kwargs.pop("cuda_repo", "")

394
395
396
397
        have_compiler = is_compiler or is_mpi or is_cuda or is_cuda_mpi
        have_mpi = is_mpi or is_cuda_mpi
        have_cuda = is_cuda or is_cuda_mpi

398
399
400
401
402
403
404
        existing_repo = root.find("./repository[@name='{0}']".format(name))
        if existing_repo is not None:
            print("Modifying existing repository", name)
            root.remove(existing_repo)
        else:
            print("New repository", name)

Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
405
406
        r = ElementTree.SubElement(root, "repository")
        r.set("name", name)
Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
407
408
        r.text = "\n    "

Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
409
410
411
412
        def path(project, repo):
            p = ElementTree.SubElement(r, "path")
            p.set("project", project)
            p.set("repository", repo)
Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
413
            p.tail = "\n    "
Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
414

415
416
        if parent:
            path(parent, name)
417
418
        for dep_project, dep_repo in dependencies:
            path(dep_project, dep_repo)
Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
419
420
421
422
423
424
425
426
427

        for arch in architectures:
            a = ElementTree.SubElement(r, "arch")
            a.text = arch
            a.tail = "\n    "
        a.tail = "\n  "
        r.tail = "\n  "

        prjconf.append("%if %_repository == {0}".format(name))
428
429
        for prefer in prefers(name):
            prjconf.append("Prefer: " + prefer)
Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
430
        prjconf.append("Macros:")
431
432

        prjconf.append("%is_compiler_repository {0}".format(1 if is_compiler else 0))
Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
433
434
        prjconf.append("%is_mpi_repository {0}".format(1 if is_mpi else 0))
        prjconf.append("%is_cuda_repository {0}".format(1 if is_cuda else 0))
435
436
437
438
439
440
        prjconf.append("%is_cuda_mpi_repository {0}".format(1 if is_cuda_mpi else 0))

        prjconf.append("%have_mpcdf_compiler {0}".format(1 if have_compiler else 0))
        prjconf.append("%have_mpcdf_mpi {0}".format(1 if have_mpi else 0))
        prjconf.append("%have_mpcdf_cuda {0}".format(1 if have_cuda else 0))

441
442
443
        if is_cuda:
            prjconf.append("%cuda_repository {0}".format(cuda_repo))

444
445
446
        for macro, value in kwargs.items():
            prjconf.append("%{0} {1}".format(macro, value))

Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
447
448
449
450
        prjconf.append(":Macros")
        prjconf.append("%endif")
        prjconf.append("")

Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
451
452
453
454
    if parent:
        repo("System")
    else:
        repo("System", ("distributions", distribution))
Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
455

456
    for compiler in compilers + pgis:
457
458
        repo(compiler, (project, "System"), compiler=True,
             compiler_repository=compiler)
Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
459
460

        for mpi in filter(partial(valid_mpi, compiler), mpis):
461
462
            repo(mpi + "_" + compiler, (project, compiler), mpi=True,
                 mpi_repository=mpi, compiler_repository=compiler)
Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
463
464
465

    for cuda in cudas:
        for compiler in filter(partial(valid_cuda, cuda), compilers):
466
            repo(cuda + "_" + compiler, (project, compiler), cuda=True, cuda_repo=cuda)
Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
467
468
469
470
            for mpi in filter(partial(valid_mpi, compiler), mpis):
                repo(cuda + "_" + mpi + "_" + compiler,
                     (project, cuda + "_" + compiler),
                     (project, mpi + "_" + compiler),
471
                     cuda_mpi=True)
Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
472
473
474
475
476
477
478
479
480
481
482
483

    # Remove build configuration
    build = root.find("./build")
    if build is None:
        build = ElementTree.Element("build")
        build.text = "\n    "
        disable = ElementTree.SubElement(build, "disable")
        disable.tail = "\n  "
        build.tail = "\n  "
        root.insert(list(root).index(root.find("./repository")), build)

    root.getchildren()[-1].tail = "\n"
Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
484
    prj = ElementTree.tostring(root, encoding=osc.core.ET_ENCODING)
Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
485
486
487
488
489
490
491
492

    prjconf.append(end_marker)
    prjconf = prjconf_head + "\n".join(prjconf) + prjconf_tail

    if dry_run:
        print("osc meta prj {0} -F - <<EOF\n{1}\nEOF\n".format(project, prj))
        print("osc meta prjconf {0} -F - <<EOF\n{1}\nEOF\n".format(project, prjconf))
    else:
493
494
495
496
497
        if not only_project:
            # First set-up the <enable/> flags, that way no
            # spurious builds are launched
            if packages is None:
                packages = osc.core.meta_get_packagelist(api_url, project)
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517

            if len(packages) > 40:
                chunksize = len(packages) // 20
            else:
                chunksize = len(packages)

            def work(packagelist):
                for package in packagelist:
                    print("Updating repositories for", package)
                    if not mpcdf_enable_repositories(api_url, project, package, filter_repos=filter_repos):
                        print("ATTENTION: Not changing unmanaged package {0}".format(package))

            threads = []
            for packagelist in chunked(packages, chunksize):
                t = threading.Thread(target=work, args=(packagelist,))
                threads.append(t)
                t.start()

            for t in threads:
                t.join()
Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
518
519
520

        # Update repositories
        print("Updating prj meta")
521
        osc.core.edit_meta("prj", project, data=prj, force=True)
Lorenz Huedepohl's avatar
Lorenz Huedepohl committed
522
523
        print("Updating prjconf meta")
        osc.core.edit_meta("prjconf", project, data=prjconf)
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563


def set_as_branch(api_url, my_project, my_package, main_project, main_package):
    import os

    def dirmeta(project, package):
        url = osc.core.makeurl(api_url, ["source", project, package])
        return ElementTree.fromstringlist(osc.core.streamfile(url, osc.core.http_GET))

    srcmeta = dirmeta(my_project, my_package)
    dstmeta = dirmeta(main_project, main_package)

    linkinfo = srcmeta.find('./linkinfo')
    if linkinfo is not None:
        print("ERROR: package {0} is already linked to package {1} in project {2}".format(
              my_package, linkinfo.get("package"), linkinfo.get("project")))
        return False

    linkfile = ElementTree.Element("link")
    linkfile.set("project", main_project)
    linkfile.set("package", main_package)
    linkfile.set("baserev", dstmeta.get("srcmd5"))
    patches = ElementTree.SubElement(linkfile, "patches")
    ElementTree.SubElement(patches, "branch")

    url = osc.core.makeurl(api_url, ["source", my_project, my_package, "_link"])
    resp = ElementTree.fromstringlist(osc.core.streamfile(url, osc.core.http_PUT, data=ElementTree.tostring(linkfile)))
    rev = resp.get("rev")
    if rev is None:
        print("ERROR: Could not commit _link file")
        return False
    else:
        print("Commited as revision", rev)
        this_package = osc.core.store_read_package(os.curdir)
        this_project = osc.core.store_read_project(os.curdir)
        if this_package == my_package and this_project == my_project:
            pac = osc.core.filedir_to_pac(os.curdir)
            rev = pac.latest_rev(expand=True)
            pac.update(rev)
    return True
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585


def sync_projects(api_url, package=None, from_project="software", to_projects=None, redo_all=False):
    if package is not None and redo_all:
        raise osc.oscerr.WrongArgs('Cannot specify `redo_all` and package')

    if to_projects is None:
        to_projects = filter(lambda s: s.startswith("software:") and not (s == "software:dist" or s == "software:images"),
                             osc.core.meta_get_project_list(api_url))

    for to_project in to_projects:
        print("Syncing {0} with {1}".format(to_project, from_project))

        to_packages = osc.core.meta_get_packagelist(api_url, to_project)

        if package is None:
            from_packages = osc.core.meta_get_packagelist(api_url, from_project)
            if not redo_all:
                from_packages = list(sorted(set(from_packages) - set(to_packages)))
        else:
            from_packages = [package]

586
        for attribute in config_attributes:
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
            try:
                get_attribute(api_url, to_project, None, attribute)
            except Exception:
                attr = get_attribute(api_url, from_project, None, attribute)
                print("Setting attribute", attribute)
                set_attribute(api_url, to_project, None, attr)

        for orig_package in from_packages:
            if orig_package not in to_packages:
                filelist = osc.core.meta_get_filelist(api_url, from_project, orig_package)
                if "_link" in filelist:
                    print("Not branching package {0}, is a link".format(orig_package))
                else:
                    print("Branching package {0}".format(orig_package))
                    exists, targetprj, targetpkg, srcprj, srcpkg = \
                        osc.core.branch_pkg(api_url, from_project, orig_package, target_project=to_project, nodevelproject=True)
            else:
                print("Not branching package {0}, already present in target".format(orig_package))

606
            for attribute in package_attributes + config_attributes:
607
608
609
610
611
612
613
614
                try:
                    attr = get_attribute(api_url, from_project, orig_package, attribute)
                except UnsetAttributeException:
                    if has_attribute(api_url, to_project, orig_package, attribute):
                        remove_attribute(api_url, to_project, orig_package, attribute)
                    continue
                set_attribute(api_url, to_project, orig_package, attr)

615
616
617
618
619
620
621
622
623
624
625
626
627
            from_maintainers = maintainers(api_url, from_project, orig_package)
            if from_maintainers:
                to_maintainers = maintainers(api_url, to_project, orig_package)
                if from_maintainers - to_maintainers:
                    new_maintainers = from_maintainers - to_maintainers
                    to_meta = package_meta(api_url, to_project, orig_package)
                    for userid in new_maintainers:
                        person = ElementTree.Element("person")
                        person.set("userid", userid)
                        person.set("role", "maintainer")
                        to_meta.insert(2, person)
                    osc.core.edit_meta("pkg", (to_project, orig_package), data=ElementTree.tostring(to_meta))

628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
        if package is None and redo_all:
            # Check if distribution is already set in to_project
            to_prj_meta = project_meta(api_url, to_project)
            sys_repo = to_prj_meta.find('./repository[@name="System"]')
            if sys_repo is None:
                distribution = project_meta(api_url, from_project).find('./repository[@name="System"]/path[@project="distributions"]').get("repository")
            else:
                distribution = sys_repo.find('./path[@project="distributions"]').get("repository")

            print("Creating repository configuration")
            mpcdf_setup_repositories(api_url, to_project, distribution)
        else:
            for orig_package in from_packages:
                if not mpcdf_enable_repositories(api_url, to_project, orig_package):
                    print("ATTENTION: Not changing unmanaged package {0}".format(orig_package))