diff --git a/environment.yml b/environment.yml index df21ad60f170fd475c639dc8c0fb9851d0b70e21..6a5003dffa24483ec621b05a1f86654f9aa9136b 100644 --- a/environment.yml +++ b/environment.yml @@ -4,6 +4,10 @@ channels: - conda-forge - nodefaults dependencies: + # Note: nbclassic (notebook<7) and RISE require python<=3.11 for now. + # In case the RISE presentation mode is not necessary, newer Python + # versions can be used and the last three lines of the present file + # can be commented or removed. - python=3.11 - pip - libopenblas=*=*openmp* @@ -27,6 +31,7 @@ dependencies: - hdf5=*=mpi* - h5py=*=mpi* - jupyterlab + # Required for RISE presentation mode from nbclassic: - notebook<7 - pip: - RISE diff --git a/examples/cython/c_interface/pyproject.toml b/examples/cython/c_interface/pyproject.toml index 7af57b83c23e1a3f27ed9c9c506a01296cb225ee..1dc7e2adc24de0b4a195d22d92bc3866cc33bab1 100644 --- a/examples/cython/c_interface/pyproject.toml +++ b/examples/cython/c_interface/pyproject.toml @@ -6,5 +6,7 @@ build-backend = "setuptools.build_meta" name = "foobar" version = "0.0.1" -[tool.setuptools.package-data] -foobar = ["*.h"] +[tool.setuptools] +ext-modules = [ + {name = "foobar.hello", sources = ["src/foobar/hello.pyx", "src/foobar/c_hello.c"]} +] diff --git a/examples/cython/c_interface/setup.py b/examples/cython/c_interface/setup.py deleted file mode 100644 index 048970016010be13a3e1201306dd5b6634a43425..0000000000000000000000000000000000000000 --- a/examples/cython/c_interface/setup.py +++ /dev/null @@ -1,10 +0,0 @@ -import os -from setuptools import setup, Extension - -ext = Extension("foobar.hello", - sources=["foobar/hello.pyx", "foobar/c_hello.c"] -) - -setup( - ext_modules=[ext], -) diff --git a/examples/cython/c_interface/foobar/__init__.py b/examples/cython/c_interface/src/foobar/__init__.py similarity index 100% rename from examples/cython/c_interface/foobar/__init__.py rename to examples/cython/c_interface/src/foobar/__init__.py diff --git a/examples/cython/c_interface/foobar/c_hello.c b/examples/cython/c_interface/src/foobar/c_hello.c similarity index 100% rename from examples/cython/c_interface/foobar/c_hello.c rename to examples/cython/c_interface/src/foobar/c_hello.c diff --git a/examples/cython/c_interface/foobar/c_hello.h b/examples/cython/c_interface/src/foobar/c_hello.h similarity index 100% rename from examples/cython/c_interface/foobar/c_hello.h rename to examples/cython/c_interface/src/foobar/c_hello.h diff --git a/examples/cython/c_interface/foobar/hello.pyx b/examples/cython/c_interface/src/foobar/hello.pyx similarity index 100% rename from examples/cython/c_interface/foobar/hello.pyx rename to examples/cython/c_interface/src/foobar/hello.pyx diff --git a/examples/cython/c_interface/test_foobar.py b/examples/cython/c_interface/test_foobar.py index eea2b682ccb88e4f18f8bf930646f04acf40d6ee..cf9307a0ac1567945284882ce0e25785093c58c4 100755 --- a/examples/cython/c_interface/test_foobar.py +++ b/examples/cython/c_interface/test_foobar.py @@ -3,7 +3,6 @@ try: from foobar import hello except: - print("Please compile first using `pip install -e .`") + print("Please compile first using `pip install .`") else: hello.say_hello() - diff --git a/examples/cython/c_numpy/setup.py b/examples/cython/c_numpy/setup.py index a038f2e16a28c1f6906e62bcd2479f858fc06ae3..622ae2783317a7f3fa9ba7aaa55f404f42e6286b 100644 --- a/examples/cython/c_numpy/setup.py +++ b/examples/cython/c_numpy/setup.py @@ -20,7 +20,7 @@ include_dirs.append(np.get_include()) ld_flags = c_flags ext = Extension("ctonumpy.cube", - sources=["ctonumpy/cube.pyx", "ctonumpy/c_cube.c"], + sources=["src/ctonumpy/cube.pyx", "src/ctonumpy/c_cube.c"], extra_compile_args=c_flags, extra_link_args=ld_flags, include_dirs=[np.get_include()] diff --git a/examples/cython/c_numpy/ctonumpy/__init__.py b/examples/cython/c_numpy/src/ctonumpy/__init__.py similarity index 100% rename from examples/cython/c_numpy/ctonumpy/__init__.py rename to examples/cython/c_numpy/src/ctonumpy/__init__.py diff --git a/examples/cython/c_numpy/ctonumpy/c_cube.c b/examples/cython/c_numpy/src/ctonumpy/c_cube.c similarity index 100% rename from examples/cython/c_numpy/ctonumpy/c_cube.c rename to examples/cython/c_numpy/src/ctonumpy/c_cube.c diff --git a/examples/cython/c_numpy/ctonumpy/c_cube.h b/examples/cython/c_numpy/src/ctonumpy/c_cube.h similarity index 100% rename from examples/cython/c_numpy/ctonumpy/c_cube.h rename to examples/cython/c_numpy/src/ctonumpy/c_cube.h diff --git a/examples/cython/c_numpy/ctonumpy/cube.pyx b/examples/cython/c_numpy/src/ctonumpy/cube.pyx similarity index 100% rename from examples/cython/c_numpy/ctonumpy/cube.pyx rename to examples/cython/c_numpy/src/ctonumpy/cube.pyx diff --git a/examples/f2py/README.md b/examples/f2py/README.md index 1738f5f7293c3fc0b2ea10e344a60cebc722291c..af280bcef2cce01569329bb94765d2ae7daa8b9f 100644 --- a/examples/f2py/README.md +++ b/examples/f2py/README.md @@ -1,9 +1,6 @@ # Minimal example on how-to interface with Fortran code using 'f2py' -Options: - -* compile on the command line using `f2py`, cf. 'compile_with_f2py.sh' -* compile using 'setup.py', using the command `python setup.py build_ext --inplace` - -Run the module using 'test_fibonacci.py'. +* build python package using the meson build-system, `pip install .` +* test the module using 'test_fibonacci.py'. +* to get a standalone python module run `compile_with_f2py` diff --git a/examples/f2py/meson.build b/examples/f2py/meson.build new file mode 100644 index 0000000000000000000000000000000000000000..31ea43a8b83f5d8cee770a46a587d3180415978d --- /dev/null +++ b/examples/f2py/meson.build @@ -0,0 +1,37 @@ +project('f2py_example', 'c', + meson_version: '>=0.64.0', + default_options : ['warning_level=2', 'optimization=3', 'c_args=-march=native'], +) + +add_languages('fortran') + +py_mod = import('python') +py = py_mod.find_installation(pure: false) +py_dep = py.dependency() + +incdir_numpy = run_command(py, + ['-c', 'import numpy; print(numpy.get_include())'], + check : true +).stdout().strip() + +incdir_f2py = run_command(py, + ['-c', 'import numpy.f2py; print(numpy.f2py.get_include())'], + check : true +).stdout().strip() + +fibby_source = custom_target('fibmodule.c', + input : ['fib.f90'], + output : ['fibmodule.c', 'fib-f2pywrappers.f'], + command : [py, '-m', 'numpy.f2py', '@INPUT@', '-m', 'fib', '--lower'] +) + +inc_np = include_directories(incdir_numpy, incdir_f2py) + +py.extension_module('fib', + ['fib.f90', fibby_source], + incdir_f2py / 'fortranobject.c', + include_directories: inc_np, + dependencies : py_dep, + install : true, + subdir: 'f2py_example' +) diff --git a/examples/f2py/pyproject.toml b/examples/f2py/pyproject.toml new file mode 100644 index 0000000000000000000000000000000000000000..d819b5fb25e0cb75754158809f4999352034a24c --- /dev/null +++ b/examples/f2py/pyproject.toml @@ -0,0 +1,7 @@ +[build-system] +requires = ['meson-python', 'numpy'] +build-backend = 'mesonpy' + +[project] +name = 'f2py_example' +version = '1.0.0' diff --git a/examples/f2py/setup.py b/examples/f2py/setup.py deleted file mode 100644 index 8141faa643db6190786cd39e92c087baa39aa9c2..0000000000000000000000000000000000000000 --- a/examples/f2py/setup.py +++ /dev/null @@ -1,9 +0,0 @@ -#!/usr/bin/env python - -from numpy.distutils.core import setup, Extension - -ext = Extension(name = 'fib', - sources = ['fib.f90']) - -setup(name = 'f2py_example', - ext_modules = [ext]) diff --git a/examples/f2py/test_fibonacci.py b/examples/f2py/test_fibonacci.py index d8675707b7f23285776f39f1784d6ececd036fe9..5d4a808f0bc3b375bea7504c59497ddcf6f23154 100755 --- a/examples/f2py/test_fibonacci.py +++ b/examples/f2py/test_fibonacci.py @@ -1,6 +1,6 @@ #!/usr/bin/env python -import fib +from f2py_example import fib print("# automatically generated docstring") print(fib.fib.__doc__) diff --git a/examples/nanobind/CMakeLists.txt b/examples/nanobind/CMakeLists.txt new file mode 100644 index 0000000000000000000000000000000000000000..6c71ede7a65bc2207275fbcf82e0189ef549aaf9 --- /dev/null +++ b/examples/nanobind/CMakeLists.txt @@ -0,0 +1,26 @@ +cmake_minimum_required(VERSION 3.18) +project(${SKBUILD_PROJECT_NAME}) + +if (NOT SKBUILD) + message(WARNING "\ + This CMake file is meant to be executed using 'scikit-build-core'. + Install this package with: pip install .") +endif() + +find_package(OpenMP) + +# find python +set(DEV_MODULE Development.Module) +find_package(Python 3.8 COMPONENTS Interpreter ${DEV_MODULE} REQUIRED) + +# find nanobind +execute_process( + COMMAND "${Python_EXECUTABLE}" -m nanobind --cmake_dir + OUTPUT_STRIP_TRAILING_WHITESPACE OUTPUT_VARIABLE nanobind_ROOT) +find_package(nanobind CONFIG REQUIRED) + +# configure extension module +nanobind_add_module(_pybex src/pybex.cpp) +target_link_libraries(_pybex PRIVATE OpenMP::OpenMP_CXX) + +install(TARGETS _pybex LIBRARY DESTINATION ${SKBUILD_PROJECT_NAME}) diff --git a/examples/nanobind/README.md b/examples/nanobind/README.md new file mode 100644 index 0000000000000000000000000000000000000000..35c9df8c795c69b90f46aff4d94343ac8f354c5c --- /dev/null +++ b/examples/nanobind/README.md @@ -0,0 +1,5 @@ +# Minimal example showing how to wrap C/C++ code into Python/Numpy using nanobind + +* install with `pip install .` +* run `python test_pybex.py` to see the 'pybex' module in action + diff --git a/examples/nanobind/pyproject.toml b/examples/nanobind/pyproject.toml new file mode 100644 index 0000000000000000000000000000000000000000..922d14f7909d82de838f0d5655fc8739a142f9be --- /dev/null +++ b/examples/nanobind/pyproject.toml @@ -0,0 +1,12 @@ +[build-system] +requires = ["nanobind", "scikit-build-core"] +build-backend = "scikit_build_core.build" + +[project] +name = "pybex" +version = "0.0.1" +requires-python = ">= 3.8" +dependencies = ["numpy"] + +[tool.scikit-build] +cmake.version = ">=3.18" diff --git a/examples/nanobind/src/pybex.cpp b/examples/nanobind/src/pybex.cpp new file mode 100644 index 0000000000000000000000000000000000000000..68481a3e1c9f2067736d267a04e1318be5afe31c --- /dev/null +++ b/examples/nanobind/src/pybex.cpp @@ -0,0 +1,46 @@ +// nanobind example module 'pybex' + +#include <vector> +#include <numeric> +#include <functional> + +#include <nanobind/nanobind.h> +#include <nanobind/ndarray.h> + +namespace nb = nanobind; + +// C/C++ implementation of the function to be wrapped +void c_cube(const double * v_in, double * v_out, size_t n_elem) { + #pragma omp parallel for simd default(none) shared(v_in, v_out, n_elem) + for (size_t i=0; i<n_elem; ++i) { + v_out[i] = v_in[i] * v_in[i] * v_in[i]; + } +} + +using array = nb::ndarray<double, nb::numpy, nb::c_contig>; + +// wrapper function, accepting a NumPy array as input and returning a NumPy array +array py_cube(array np_in) +{ + // create output buffer + double *out_buffer = new double[np_in.size()]; + + // call C/C++ function with proper arguments + c_cube(np_in.data(), out_buffer, np_in.size()); + + // Delete 'data' when the 'owner' capsule expires + nb::capsule owner(out_buffer, [](void *p) noexcept { + delete[] (double *) p; + }); + + return array(out_buffer, np_in.ndim(), (const size_t*)np_in.shape_ptr(), owner); +} + + +// define Python module, expose py_cube function as "cube" to python +NB_MODULE(_pybex, m) { + m.doc() = "nanobind example module"; // module docstring + + m.def("cube", &py_cube, "a function that cubes a double-precision numpy array"); +} + diff --git a/examples/nanobind/src/pybex/__init__.py b/examples/nanobind/src/pybex/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..62b93b9f456f2abeb3456b9a2e374fc99511b0d6 --- /dev/null +++ b/examples/nanobind/src/pybex/__init__.py @@ -0,0 +1 @@ +from ._pybex import cube diff --git a/examples/nanobind/test_pybex.py b/examples/nanobind/test_pybex.py new file mode 100755 index 0000000000000000000000000000000000000000..932fa156e6a9ac5589b9f7ef607782f7c2f14efa --- /dev/null +++ b/examples/nanobind/test_pybex.py @@ -0,0 +1,33 @@ +#!/usr/bin/env python + +import numpy as np +from time import time +from pybex import cube + + +def test_pybex_cube(N=384): + v = np.full((N, N, N), fill_value=2.0) + + # (1) compute the result using our C extension + t0 = time() + w_ext = cube(v) + dt0 = time() - t0 + + # (2) compute the result using NumPy, note the relative inefficiency + t0 = time() + w_ref = v * v * v + dt1 = time() - t0 + + # (3) compare both the results, be careful with NumPy temporary arrays + try: + assert(np.allclose(w_ext, w_ref)) + #assert(np.allclose(w_ext[-1,-1,-1], w_ref[-1,-1,-1])) + except: + raise + else: + print("OK! t(C)=%f, t(NumPy)=%f" % (dt0, dt1)) + + +if __name__ == "__main__": + test_pybex_cube() + diff --git a/examples/pybind11/README.md b/examples/pybind11/README.md index 4a8cf22df67a7d61760b0b0310b389a05a11fe6f..9b4e57ea2f16b9c455863497803a181583200b0f 100644 --- a/examples/pybind11/README.md +++ b/examples/pybind11/README.md @@ -1,5 +1,5 @@ # Minimal example showing how to wrap C/C++ code into Python/Numpy using pybind11 -* you may need to run `pip install --user pybind11` if not yet installed +* install with `pip install .` * run `python test_pybex.py` to see the 'pybex' module in action diff --git a/examples/pybind11/pyproject.toml b/examples/pybind11/pyproject.toml index 01f1c1e406892f373a099ada6c688bf70c15a76d..a55f2f74dcb45081d989d7f710feeeb6e7f94c28 100644 --- a/examples/pybind11/pyproject.toml +++ b/examples/pybind11/pyproject.toml @@ -1,5 +1,5 @@ [build-system] -requires = ["setuptools", "pybind11[global]"] +requires = ["setuptools", "pybind11"] build-backend = "setuptools.build_meta" [project] diff --git a/notebooks/01--Introduction.ipynb b/notebooks/01--Introduction.ipynb index 5f8cee4c4e60ab29485c4fd35bd2d71b5004337d..fcaac67ec7026c3cef26cc54cfdad8c0805fb26d 100644 --- a/notebooks/01--Introduction.ipynb +++ b/notebooks/01--Introduction.ipynb @@ -46,7 +46,7 @@ } }, "source": [ - "## Examples for \"real\" HPC Codes using Python\n", + "## Examples for \"real\" HPC Simulation Codes using Python\n", "\n", "### [GPAW](https://wiki.fysik.dtu.dk/gpaw/)\n", "\n", @@ -125,7 +125,7 @@ " * Parallel programming with MPI\n", " * Parallel programming with OpenMP\n", " * Advanced parallel programming with OpenMP and MPI\n", - "* Watch out for these courses which are regularly offered by HLRS, LRZ, NHR/RRZE" + "* Watch out for these courses which are regularly offered by HLRS, LRZ, RRZE" ] }, { @@ -154,7 +154,7 @@ "source": [ "## Software prerequisites for the Jupyter notebooks and exercises\n", "* Python >=3.6\n", - "* Jupyter\n", + "* JupyterLab\n", "* NumPy, SciPy\n", "* matplotlib\n", "* Cython\n", @@ -179,7 +179,7 @@ "* Each instance provides access to several virtual CPU cores and a few GB of RAM\n", "* Please keep the following points in mind\n", " * Use the JupyterLab menu **File $\\to$ Shut down** to free resources when finished\n", - " * A session is terminated after 12h\n", + " * A session is terminated after 8 hours, or after 2 hours of inactivity\n", " * The storage is only temporary, i.e. download your modified notebooks and data in time via the GUI" ] }, @@ -209,16 +209,9 @@ }, "source": [ "### Software Option 3: Python environment for your local computer\n", - "* On a recent Linux system, install all the necessary packages via the package manager\n", + "* On a recent Linux system, install all the necessary packages via the package manager and `pip`\n", "* Alternatively, install [Miniforge](https://conda-forge.org/miniforge/) and use the `conda` package manager and the file `environment.yml` that is provided together with the course material to add all necessary packages" ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [] } ], "metadata": { @@ -238,7 +231,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.12.7" + "version": "3.11.10" } }, "nbformat": 4, diff --git a/notebooks/03--Python_Refresher.ipynb b/notebooks/03--Python_Refresher.ipynb index 7a354976aba33004ffa844514300dde0fe8f6dd1..bf2a97107174f8cc7e999c2953291654f7f55cb9 100644 --- a/notebooks/03--Python_Refresher.ipynb +++ b/notebooks/03--Python_Refresher.ipynb @@ -29,10 +29,12 @@ }, "source": [ "### Python: History and Status\n", + "\n", "* First version released in 1991 by G. van Rossum\n", "* [Implementations](https://wiki.python.org/moin/PythonImplementations): **cPython**, PyPy, Pyston, ...\n", - "* Language versions: 2.7 (legacy, ✞2020), now 3 (relevant versions are 3.8 - 3.13) \n", - " (to migrate legacy code, the packages `2to3`, `six`, `future` are helpful)" + "* Language versions \n", + " * Python 3, relevant versions are 3.8 - 3.13\n", + " * Python 2.7 (✞2020), to migrate legacy code, the packages `2to3`, `six`, `future` are helpful" ] }, { @@ -49,7 +51,7 @@ " $\\rightarrow$ code is interpreted in most implementations (or just-in-time compiled) \n", " $\\rightarrow$ certain [compiler optimizations](https://en.wikipedia.org/wiki/Optimizing_compiler) are not possible $\\rightarrow$ performance overhead\n", "* imperative, procedural, functional, or object-oriented programming styles possible\n", - "* syntax claimed to make programs comparably easy to read \n", + "* syntax makes programs comparably easy to read \n", " e.g. indentation (typ. 4 spaces per level) defines logical blocks" ] }, @@ -135,10 +137,10 @@ "```python\n", "a = 5\n", "```\n", - "* in Python everything is an object, i.e. contains data along with attributes and/or methods\n", - "* immutable objects may not change once created: integer, float, boolean, string, tuple\n", - "* mutable objects may change in place: list, set, dictionary, most user-defined classes\n", - "* if unsure, check via the object ID (memory location)" + "* in Python everything is an object, i.e. contains data along with attributes and methods \n", + " * immutable objects may not change once created, e.g. integer, float, boolean, string, tuple\n", + " * mutable objects may change in place, e.g. list, set, dictionary, most user-defined classes\n", + " * if unsure, check via the object ID (memory location in cPython)" ] }, { @@ -1438,7 +1440,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.12.7" + "version": "3.11.10" } }, "nbformat": 4, diff --git a/notebooks/07--Numba_Jax_JIT.ipynb b/notebooks/07--Numba_Jax_JIT.ipynb index 6b14322193be305acf651215b8c66d23a427867f..7768b457d19fb85c5ae9691a1e4f95607152230c 100644 --- a/notebooks/07--Numba_Jax_JIT.ipynb +++ b/notebooks/07--Numba_Jax_JIT.ipynb @@ -451,6 +451,7 @@ "* for the JIT interface, both provide a simple `jit` interface\n", "* both use LLVM as the compiler backend; jax with a layer of indirection via the XLA compiler\n", "* jax supports CPU/GPU/TPU backends with exactly the same codebase (e.g., no cuda knowledge necessary)\n", + "* jax excells when used on accelerator devices\n", "* jax will fail on code it doesn't know\n", "* jax provides automatic differentiation in addition\n", "* jax is general purpose but many concepts have a background in machine learning (e.g. batch parallelization)" @@ -555,9 +556,7 @@ "\n", "* jax also provides multivariate differentiation via `jacfwd`, `jacrev`, `jvp`, `vjp`\n", "* jax has support for vectorization and parallelization: `vmap`, `pmap`\n", - "* jax is still under active development but becoming increasingly popular:\n", - " * the [alphafold project](https://alphafold.com/) from DeepMind\n", - " * the [flax](https://flax.readthedocs.io/) neural net library" + "* jax is still under active development but becoming increasingly [popular](https://github.com/n2cholas/awesome-jax)" ] }, { @@ -1431,7 +1430,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.10.12" + "version": "3.11.10" } }, "nbformat": 4, diff --git a/notebooks/08--Diffusion.ipynb b/notebooks/08--Diffusion.ipynb index 6b4dd19a4f024a9b22a78987e6328b037c7e3e45..588d876204579fe268cf6a923a38c2d8cca1b537 100644 --- a/notebooks/08--Diffusion.ipynb +++ b/notebooks/08--Diffusion.ipynb @@ -5634,9 +5634,8 @@ "\n", "def main_loop_jax(evolve_func, grid):\n", " \"\"\"Main loop function, calling evolve_func on grid.\"\"\"\n", - " grid_tmp = np.empty_like(grid)\n", " for i in range(1, n_iterations+1):\n", - " grid = evolve_func(grid, grid_tmp, n_points, dt, D)\n", + " grid = evolve_func(grid, n_points, dt, D)\n", " return grid" ] }, @@ -5660,10 +5659,9 @@ " - 4 * grid\n", "\n", "@jit\n", - "def evolve_np_roll_jax(grid, grid_tmp, n_points, dt, D):\n", + "def evolve_np_roll_jax(grid, n_points, dt, D):\n", " \"\"\"Time step based on the NumPy-roll-Laplacian.\"\"\"\n", - " grid_tmp = grid_tmp.at[:].set(grid + dt * D * laplacian_np_roll(grid))\n", - " return grid_tmp" + " return grid_tmp.at[:].set(grid + dt * D * laplacian_np_roll(grid))" ] }, { @@ -5760,7 +5758,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.10.12" + "version": "3.11.10" } }, "nbformat": 4, diff --git a/notebooks/09--Interfacing_with_C_and_F.ipynb b/notebooks/09--Interfacing_with_C_and_F.ipynb index dc0a03b840f297737e87f0165a2d56964a60edf1..f13b9cfb1d82addab8d3cc65b0c859dc80a74386 100644 --- a/notebooks/09--Interfacing_with_C_and_F.ipynb +++ b/notebooks/09--Interfacing_with_C_and_F.ipynb @@ -34,7 +34,8 @@ "## Steps\n", "\n", "* create software interface between Python/NumPy and C/C++/Fortran\n", - "* compile and link to become a Python-importable module" + "* compile and link to become a Python-importable module $\\leftrightarrow$ usually integrated with packaging, \\\n", + " see `/notebooks/09--Interfacing_with_C_and_F.ipynb`" ] }, { @@ -51,7 +52,7 @@ "\n", "* Fortran $\\longrightarrow$ **f2py**\n", "* C/C++/CUDA $\\longrightarrow$ **Cython**\n", - "* C++ $\\longrightarrow$ **pybind11**\n", + "* C++ $\\longrightarrow$ **pybind11**, **nanobind**\n", "\n", "Some other options (not covered here):\n", "\n", @@ -76,10 +77,8 @@ " * automatically takes care of type casting and non-contiguous arrays\n", "* two ways of usage\n", " * direct calling of the `f2py` executable\n", - " * via a build-system\n", - " * define an extension in `setup.py` using `numpy.distutils` (deprecated with python 3.12)\n", - " * use external build-systems: `cmake`, `meson`, `scikit-build`, see (cf. the [f2py user guide](https://numpy.org/doc/stable/f2py/index.html))\n", - "* see examples in './f2py' " + " * (define an extension in `setup.py` using `numpy.distutils`; **deprecated with python 3.12**)\n", + " * via an external build-system: `meson`, `cmake`, `scikit-build`, see (cf. the [f2py user guide](https://numpy.org/doc/stable/f2py/index.html))" ] }, { @@ -121,11 +120,7 @@ } }, "source": [ - "### Compile and link Fortran code to Python module using 'setup.py'\n", - "\n", - "* Easy, especially when a `setup.py` already exists\n", - "* Necessary change: use `numpy.distutils` instead of `distutils` \n", - " $\\rightarrow$ extension sources may contain a Fortran file" + "### Compiling and packaging with fortran sources using meson" ] }, { @@ -136,16 +131,29 @@ } }, "source": [ - "```python\n", - "# setup.py\n", - "from numpy.distutils.core import setup, Extension\n", - "\n", - "ext = Extension(name = 'fib', sources = ['fib.f90'])\n", + "```pyproject.toml\n", + "# pyproject.toml\n", + "[build-system]\n", + "requires = ['meson-python', 'numpy']\n", + "build-backend = 'mesonpy'\n", "\n", - "setup(name = 'fibonacci', ext_modules = [ext])\n", + "[project]\n", + "name = 'f2py_example'\n", + "version = '1.0.0'\n", "```" ] }, + { + "cell_type": "markdown", + "metadata": { + "slideshow": { + "slide_type": "fragment" + } + }, + "source": [ + "* meson reads the build-config from `meson.build`, see `examples/f2py`" + ] + }, { "cell_type": "markdown", "metadata": { @@ -155,9 +163,9 @@ }, "source": [ "```bash\n", - "$ python setup.py build_ext --inplace\n", + "$ pip install .\n", "(...)\n", - "$ python -c \"import fib; a=fib.fib(16); print(a[-1])\"\n", + "$ python -c \"from f2p_example import fib; a=fib.fib(16); print(a[-1])\"\n", "610\n", "```" ] @@ -245,15 +253,20 @@ } }, "source": [ - "```python\n", - "# setup.py \n", - "from setuptools import setup, Extension\n", + "```pyproject.toml\n", + "# pyproject.toml\n", + "[build-system]\n", + "requires = [\"setuptools\", \"cython\"]\n", + "build-backend = \"setuptools.build_meta\"\n", "\n", - "ext = Extension(\"foobar.hello\",\n", - " sources=[\"foobar/hello.pyx\", \"foobar/c_hello.c\"])\n", + "[project]\n", + "name = \"foobar\"\n", + "version = \"0.0.1\"\n", "\n", - "setup(name=\"foobar\",\n", - " ext_modules=[ext])\n", + "[tool.setuptools]\n", + "ext-modules = [\n", + " {name = \"foobar.hello\", sources = [\"src/foobar/hello.pyx\", \"src/foobar/c_hello.c\"]}\n", + "]\n", "```" ] }, @@ -296,7 +309,7 @@ " * OpenMP threading (not affected by the Python GIL)\n", " * vectorization\n", " * cache optimization, etc.\n", - " * use optimizing compiler flags in `setup.py`" + " * specify optimizing compiler flags according to the build-backend in use; e.g., with `setuptools` this is `pyroject.toml/setup.py`" ] }, { @@ -428,14 +441,15 @@ "cell_type": "markdown", "metadata": { "slideshow": { - "slide_type": "subslide" + "slide_type": "slide" } }, "source": [ "## pybind11\n", "\n", "* lightweight header-only library to create interfaces to modern C++ code\n", - "* highlights: STL, iterators, classes and inheritance, smart pointers, move semantics, NumPy $\\leftrightarrow$ Eigen, etc.\n", + "* highlights: STL, iterators, classes and inheritance, smart pointers, move semantics, NumPy $\\leftrightarrow$ C++, Eigen $\\leftrightarrow$ Python, etc.\n", + "* support for various build-systems/build-backends: **`cmake`**, `meson`, `cppimport`, `setuptools`\n", "* documentation: https://pybind11.readthedocs.io/en/latest/" ] }, @@ -447,19 +461,13 @@ } }, "source": [ - "### Easy integration with pyproject.toml (PEP518)\n", + "### Example integration with setuptools\n", "\n", "```toml\n", "[build-system]\n", - "requires = [\"setuptools\", \"pybind11[global]\"]\n", + "requires = [\"setuptools\", \"pybind11\"]\n", "build-backend = \"setuptools.build_meta\"\n", - "\n", - "[project]\n", - "...\n", - "```\n", - "\n", - "* the `global` specifier ensures the availability of C++ headers at build-time\n", - " (see `Software_Engineering.ipynb` notebook for more information on the setup of python projects)" + "```" ] }, { @@ -500,14 +508,19 @@ } }, "source": [ - "### Build-system support\n", - "\n", - "* direct inegration with `setuptools`\n", - "* `CMake` for large projects with dependencies on the C++ side\n", - "* and some more, see: https://pybind11.readthedocs.io/en/stable/compiling.html\n", + "## nanobind\n", "\n", - "* [scikit-build](https://scikit-build.readthedocs.io/en/latest/) for glueing setuptools with CMake\n" + "* stripped down version of pybind11 but with signifact performance improvements both at run- and compile-time\n", + "* can be built and packaged conveniently with `cmake` and `scikit-build-core`, see `examples/nanobind`\n", + "* documentation: https://nanobind.readthedocs.io/en/latest/" ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] } ], "metadata": { @@ -527,7 +540,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.9.7" + "version": "3.11.10" } }, "nbformat": 4, diff --git a/notebooks/10--Profiling.ipynb b/notebooks/10--Profiling.ipynb index 9396eb83b29b1653d5d4c27fe6f8f21842da8ec4..79565b3d264e6a1afbf00169f957cca4df52fad4 100644 --- a/notebooks/10--Profiling.ipynb +++ b/notebooks/10--Profiling.ipynb @@ -472,6 +472,7 @@ " * `perf record`: run application and collect profile\n", " * `perf report -g`: explore recorded profile, get callgraph, identify hotspots, see assembly\n", " * use Brendan Gregg's flamegraph visualization \n", + "* Python >= 3.12: [native perf support](https://docs.python.org/3/howto/perf_profiling.html)\n", "* References, tutorials\n", " * See the example in the folder `profiling`\n", " * http://www.brendangregg.com/perf.html\n", @@ -623,7 +624,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.9.7" + "version": "3.11.10" } }, "nbformat": 4, diff --git a/notebooks/11--Software_Engineering.ipynb b/notebooks/11--Software_Engineering.ipynb index 00f436f35f03b177e6eb461a5672cb55474f0270..7241fa4d9a87d7d0edddf3fad19b2004e9a2fed7 100644 --- a/notebooks/11--Software_Engineering.ipynb +++ b/notebooks/11--Software_Engineering.ipynb @@ -57,7 +57,7 @@ "## Software Packaging\n", "* The Python ecosytem is inherently geared toward software packaging\n", " * great for \"professional\" software distribution (e.g., via PyPI)\n", - " * create zipfiles, tarballs, wheel, or egg packages easily\n", + " * create source (source code archives) or binary distributions (wheels)\n", " * metadata specification (version, authorship, license, **dependencies**, ...)\n", " * Compilation of Cython, C/C++, Fortran extensions (Makefile replacement)\n", " * Installation to system, per-user, or arbitrary locations\n", @@ -75,18 +75,14 @@ "source": [ "### Package configuration:\n", "\n", - "* **legacy**: `setup.py` using `distutils`/`setuptools`\n", - "* `distutils` is deprecated with removal in python 3.12\n", - "\n", - "\n", - "* modern packages contain the package specification in `pyproject.toml` (PEP518, PEP517)\n", + "* the python community converged to define package specification in `pyproject.toml` (PEP518, PEP517)\n", "* packaging is split into\n", - " * *frontend*, e.g.: `pip wheel`, `build`, ...\n", - " * *backend*, e.g., `setuptools`, `flit`, `hatch`, `pdm`, `poetry`, ...\n", + " * *frontend*, e.g.: `pip`, `wheel`, `build`, ...\n", + " * *backend*, e.g., `setuptools` (the historical reference), `scikit-build-core`, `flit`, `hatch`, `pdm`, `poetry`, ...\n", "\n", "\n", "* build system isolation (PEP518)\n", - "* core metadata specification in `pyproject.toml` (PEP621)" + "* core metadata specification in `pyproject.toml` (PEP621), different backends then may use more/different files for the build configuration on top" ] }, { @@ -145,7 +141,7 @@ "### Installing and distributing packages\n", "* Install package with `pip install .`(implying build of the package)<br>\n", " add `--user` $\\to$ installation path under `~/.local/lib/...`;<br>\n", - " this will make the package available for your local user\n", + " this will make the package available for your local user (**not recommended in general**)\n", "* Explicitly build package including extensions with `python -m build`\n", "* Create distribution packages for sharing\n", " * Source package with `python -m build --sdist`\n", @@ -185,7 +181,7 @@ "### Components of a full-fledged software package\n", "\n", "* source code\n", - "* package metadata and specification of extensions (`pyproject.toml` and `setup.py` in case)\n", + "* package metadata and specification of extensions (`pyproject.toml` + backend dependend configuration)\n", "* user documentation - how to use the package\n", "* developer documentation - how to develop code for the package\n", "* software tests\n", @@ -478,8 +474,8 @@ } }, "source": [ - "### pytest example with setup.py\n", - "$\\to$ See the directory `setuptools/py-test` for an example on how to integrate tests into `setup.py`" + "### pytest example with setuptools\n", + "$\\to$ See the directory `examples/setuptools/py-test` for an example on how to integrate tests into a package" ] }, { @@ -541,7 +537,7 @@ "* https://pypi.python.org/pypi\n", "* Contains nearly 330.000 packages (Oct 2021)\n", "* Zip files, tar balls, wheel archives are accepted \n", - " (typically generated locally by `setup.py`/`python -m build` first)\n", + " (typically generated locally by front-end, e.g., `python -m build`)\n", "* Anybody may upload packages after registration" ] }, @@ -593,7 +589,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.9.7" + "version": "3.11.10" } }, "nbformat": 4, diff --git a/notebooks/12--Parallel_Programming_Shared_Memory.ipynb b/notebooks/12--Parallel_Programming_Shared_Memory.ipynb index 31d7167667bf6622c88a6b8d3664732bbea417a5..5776638a226801c0e092a996f14ae42fb13fbb20 100644 --- a/notebooks/12--Parallel_Programming_Shared_Memory.ipynb +++ b/notebooks/12--Parallel_Programming_Shared_Memory.ipynb @@ -299,6 +299,7 @@ "* cPython's global interpreter lock (GIL)\n", " * only one thread may execute Python bytecode at a time\n", " * no parallelization of computations possible with threads (although I/O or GUI might benefit)\n", + "* new in python 3.13: [GIL can be made optional](https://peps.python.org/pep-0703/), but not yet mature\n", "\n", "$\\to$ **Python threads are not useful for HPC** \n", "$\\to$ **use several Python processes instead**" diff --git a/notebooks/15--Parallel_Programming_Distributed_Memory_MPI.ipynb b/notebooks/15--Parallel_Programming_Distributed_Memory_MPI.ipynb index 6a5b01937d4b540daa4a4d1248fc8574aa65a5f9..8a162fc102023ce2df673b56af88bc1b81807064 100644 --- a/notebooks/15--Parallel_Programming_Distributed_Memory_MPI.ipynb +++ b/notebooks/15--Parallel_Programming_Distributed_Memory_MPI.ipynb @@ -157,7 +157,27 @@ " * Objects need to be compatible with pickle\n", " * Numpy arrays\n", " * Upper-case functions (Send, Recv, ...)\n", - " * More efficient: underlying memory used directly" + " * More efficient: underlying memory used directly\n", + "* Since `mpi4py` version 4.0.0: `Pool` class" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "slideshow": { + "slide_type": "subslide" + } + }, + "source": [ + "## `mpi4py` `Pool` class\n", + "* Available since `mpi4py` version 4.0.0\n", + "* Drop-in replacement for `multiprocessing.Pool`:\n", + "\n", + "```python\n", + "from mpi4py.util.pool import Pool\n", + "with Pool(10) as p:\n", + " result = p.map(f, array)\n", + "```" ] }, {