diff --git a/mpcdf_common.py b/mpcdf_common.py index 63582c46b22f64b45b7f80e1ed74ed23812cae2f..9f943280891807bc3d1e82df0590b600687e541e 100644 --- a/mpcdf_common.py +++ b/mpcdf_common.py @@ -1,6 +1,3 @@ -from __future__ import print_function -from __future__ import division - import sys import osc import osc.conf @@ -8,6 +5,7 @@ import osc.core import osc.oscerr import textwrap import urllib +from typing import Optional from functools import partial from xml.etree import ElementTree @@ -229,6 +227,26 @@ def valid_pgi_mpi(pgi, mpi): return False +def is_intel_compiler(compiler: str) -> bool: + return compiler.startswith("intel") + + +def is_gcc_compiler(compiler: str) -> bool: + return compiler.startswith("gcc") + + +def is_pgi_compiler(compiler: str) -> bool: + return compiler.startswith("pgi") + + +def is_intel_mpi(mpi: str) -> bool: + return mpi.startswith("impi") + + +def is_openmpi(mpi: str) -> bool: + return mpi.startswith("openmpi") + + def valid_mpi(compiler, mpi): """ It might be possible to use Intel MPI libararies and compilers from @@ -238,13 +256,13 @@ def valid_mpi(compiler, mpi): Take care to keep this in sync with the file 'macros.obs_cluster' of the package software:dist / mpcdf_cluster_macros """ - if compiler.startswith("intel") and mpi.startswith("impi"): + if is_intel_compiler(compiler) and is_intel_mpi(mpi): if compiler in compiler_x_parallel_studio: return mpi == compiler_x_parallel_studio[compiler]["impi"] return mpi == compiler_parallel_studio[compiler]["impi"] - if compiler.startswith("pgi"): + if is_pgi_compiler(compiler): return valid_pgi_mpi(compiler, mpi) - if compiler.startswith("gcc") and mpi.startswith("impi"): + if is_gcc_compiler(compiler) and is_intel_mpi(mpi): gcc_version = int(compiler[len("gcc_"):]) impi_major_version, impi_minor_version = map(int, mpi[len("impi_"):].split("_")) if gcc_version >= 10: @@ -541,8 +559,8 @@ def mpcdf_enable_repositories(api_url, project, package, verbose=False, dry_run= all_pgis = overloaded_project_attribute(api_url, project, "MPCDF:pgi_modules") all_openmpi_flavors = overloaded_project_attribute(api_url, project, "MPCDF:openmpi_flavors") - latest_intel = latest_package(c for c in all_compilers if c.startswith("intel")) - latest_gcc = latest_package(c for c in all_compilers if c.startswith("gcc")) + latest_intel = latest_package(filter(is_intel_compiler, all_compilers)) + latest_gcc = latest_package(filter(is_gcc_compiler, all_compilers)) i = len(build) @@ -559,10 +577,10 @@ def mpcdf_enable_repositories(api_url, project, package, verbose=False, dry_run= def actual_compilers(): for compiler in (c for c in compilers if c in all_compilers + ["intel", "gcc", "latest_intel", "latest_gcc"]): if compiler == "intel": - for intel_compiler in [cc for cc in all_compilers if cc.startswith("intel")]: + for intel_compiler in filter(is_intel_compiler, all_compilers): yield intel_compiler elif compiler == "gcc": - for gcc_compiler in [cc for cc in all_compilers if cc.startswith("gcc")]: + for gcc_compiler in filter(is_gcc_compiler, all_compilers): yield gcc_compiler elif compiler == "latest_intel" and latest_intel is not None: yield latest_intel @@ -574,7 +592,7 @@ def mpcdf_enable_repositories(api_url, project, package, verbose=False, dry_run= def actual_mpis(): for mpi in (m for m in mpis if m in all_mpis + ["impi"]): if mpi == "impi": - for impi in [mpi for mpi in all_mpis if mpi.startswith("impi")]: + for impi in filter(is_intel_mpi, all_mpis): yield impi else: yield mpi @@ -826,6 +844,9 @@ Macros: have_mpi = mpi or cuda_mpi or cuda_aware_mpi have_cuda = cuda or cuda_mpi or cuda_aware_mpi + if additional_tags is None: + additional_tags = () + existing_repo = root.find("./repository[@name='{0}']".format(name)) if existing_repo is not None: root.remove(existing_repo) @@ -858,7 +879,7 @@ Macros: # In order to be able to figure out the matching MKL version for a given # compiler/MPI repository we emit a new macro '%matching_mkl_version' in # the cases this makes sense - matching_mkl = [] + matching_mkl: list[str] = [] repoconf = [] repoconf.append("%if \"%{{_repository}}\" == \"{0}\"".format(name)) for tag in repo_tags(name, distribution=distribution) + additional_tags: @@ -881,10 +902,10 @@ Macros: repoconf.append("%have_mpcdf_cuda {0}".format(1 if have_cuda else 0)) if matching_mkl: - matching_mkl, = matching_mkl - matching_mkl, _ = matching_mkl[len("mkl_"):].split("-module") - matching_mkl = matching_mkl.replace("_", ".") - repoconf.append("%matching_mkl_version {0}".format(matching_mkl)) + single_matching_mkl, = matching_mkl + single_matching_mkl, _ = single_matching_mkl[len("mkl_"):].split("-module") + single_matching_mkl = single_matching_mkl.replace("_", ".") + repoconf.append("%matching_mkl_version {0}".format(single_matching_mkl)) for macro, value in macros.items(): repoconf.append("%{0} {1}".format(macro, value))