Skip to content
Snippets Groups Projects
Commit a95c8dbf authored by Repo Updater's avatar Repo Updater
Browse files

5af64cc7 remove CleanCommand to keep setup.py simpler

parent 8eca8dc0
No related branches found
No related tags found
No related merge requests found
Hide whitespace changes
Inline Side-by-side
Showing
with 78 additions and 93 deletions
examples/cython/c_interface/setup.py
+ 2
− 22
View file @ a95c8dbf
import os
from setuptools import setup, Extension, Command
from setuptools import setup, Extension
ext = Extension("foobar.hello",
sources=["foobar/hello.pyx", "foobar/c_hello.c"])
class CleanCommand(Command):
"""Enable `python setup.py clean` to tidy up properly."""
user_options = []
def initialize_options(self):
pass
def finalize_options(self):
pass
def run(self):
os.system('rm -vrf build')
os.system('rm -vrf dist')
os.system('rm -vrf foobar/hello.c')
os.system('rm -vrf foobar/__pycache__')
os.system('rm -vrf foobar.egg-info')
os.system("find foobar -name '*.so' -delete -print")
os.system("find foobar -name '*.pyc' -delete -print")
setup(name="foobar",
ext_modules=[ext],
cmdclass={'clean': CleanCommand})
)
examples/cython/c_interface_shared_object/setup.py
+ 2
− 25
View file @ a95c8dbf
import os
from setuptools import setup, Extension, Command
from setuptools import setup, Extension
libhello_dir = os.path.abspath("./libhello")
......@@ -16,34 +16,11 @@ ld_flags.append("-L"+libhello_dir)
ld_flags.append("-Wl,-rpath,"+libhello_dir)
ld_flags.append("-lhello") # link libhello
ext = Extension("wrap_libhello.hello",
sources=["wrap_libhello/hello.pyx"],
extra_link_args=ld_flags,
include_dirs=include_dirs)
class CleanCommand(Command):
"""Enable `python setup.py clean` to tidy up properly."""
user_options = []
def initialize_options(self):
pass
def finalize_options(self):
pass
def run(self):
os.system('rm -vrf build')
os.system('rm -vrf dist')
os.system('rm -vrf wrap_libhello/__pycache__')
os.system('rm -vrf wrap_libhello/hello.c')
os.system('rm -vrf wrap_libhello.egg-info')
os.system("find wrap_libhello libhello -name '*.o' -delete -print")
os.system("find wrap_libhello libhello -name '*.so' -delete -print")
os.system("find wrap_libhello libhello -name '*.pyc' -delete -print")
setup(name="wrap_libhello",
ext_modules=[ext],
cmdclass={'clean': CleanCommand})
)
examples/cython/c_numpy/setup.py
+ 2
− 22
View file @ a95c8dbf
import os
import numpy as np
from setuptools import setup, Extension, Command
from setuptools import setup, Extension
# assuming gcc, set (aggressive) optimization flags, to be appended to the default compile line
......@@ -25,27 +25,7 @@ ext = Extension("ctonumpy.cube",
extra_link_args=ld_flags,
include_dirs=[np.get_include()])
class CleanCommand(Command):
"""Enable `python setup.py clean` to tidy up properly."""
user_options = []
def initialize_options(self):
pass
def finalize_options(self):
pass
def run(self):
# better: use plain Python instead of UNIX commands
os.system('rm -vrf build')
os.system('rm -vrf dist')
os.system('rm -vrf ctonumpy/cube.c')
os.system('rm -vrf ctonumpy/__pycache__')
os.system('rm -vrf ctonumpy.egg-info')
os.system("find ctonumpy -name '*.so' -delete -print")
os.system("find ctonumpy -name '*.pyc' -delete -print")
setup(name="ctonumpy",
ext_modules=[ext],
cmdclass={'clean': CleanCommand})
)
examples/pybind11/setup.py
+ 16
− 0
View file @ a95c8dbf
from setuptools import setup
# run `pip install --user pybind11` if not yet installed
from pybind11.setup_helpers import Pybind11Extension, build_ext
ext_modules = [
Pybind11Extension(
"pybex",
["src/pybex.cpp",],
),
]
setup(
cmdclass={"build_ext": build_ext},
ext_modules=ext_modules
)
examples/pybind11/src/cube.cpp deleted 100644 → 0
+ 0
− 0
View file @ 8eca8dc0
examples/pybind11/src/pybex.cpp 0 → 100644
+ 32
− 0
View file @ a95c8dbf
// pybind11 example module 'pybex'
#include <pybind11/pybind11.h>
#include <pybind11/numpy.h>
namespace py = pybind11;
// C/C++ implementation of the function to be wrapped
void c_cube(const double * v_in, double * v_out, size_t n_elem) {
for (size_t i=0; i<n_elem; ++i) {
v_out[i] = v_in[i] * v_in[i] * v_in[i];
}
}
// wrapper function, accepting a NumPy array as input and returning a NumPy array
py::array_t<double> py_cube(py::array_t<double, py::array::c_style> v_in)
{
// allocate buffer to be returned as a NumPy array
auto v_out = py::array_t<double>(v_in.size());
double * v_out_ptr = (double*) v_out.request().ptr;
c_cube(v_in.data(), v_out_ptr, v_in.size());
return v_out;
}
PYBIND11_MODULE(pybex, m) {
m.doc() = "pybind11 example module"; // module docstring
m.def("cube", &py_cube, "a function that cubes a double-precision numpy array");
}
notebooks/2e--Interfacing_with_C_and_F.ipynb
+ 24
− 24
View file @ a95c8dbf
%% Cell type:markdown id: tags:
# Interfacing Python/NumPy with C/C++ and Fortran Code
**Python for HPC course**
**Python for HPC**
2018-2021 Sebastian Ohlmann, Klaus Reuter
Max Planck Computing and Data Facility, Garching
%% Cell type:markdown id: tags:
## Motivation
* Call compiled C/C++ or Fortran code from the Python layer
* Existing (legacy) code
* Newly developed and optimized code (hotspots, numerical kernels)
* Newly developed and optimized code (numerical kernels)
$\rightarrow$ combine the convenience of using high-level Python with high-performance compiled code
## Steps
* create software interface between Python/NumPy and C/C++/Fortran
* compile and link to a Python module
* compile and link to become a Python-importable module
%% Cell type:markdown id: tags:
## Interfacing Options
We're looking into the following options:
* Fortran $\longrightarrow$ **f2py**
* C/C++/CUDA $\longrightarrow$ **Cython**
* C++ $\longrightarrow$ **pybind11**
Some other options (not covered here):
* Swig, an interface generator
* Boost.Python for C++ codes
* Python's low-level C API (different for Python 2 and 3!)
* Python's low-level C API
%% Cell type:markdown id: tags:
## `f2py`: Interfacing with Fortran code
* `f2py`
* is part of NumPy
* scans the Fortran code and generates signature files (.pyf)
* automatically takes care of type casting and non-contiguous arrays
* 2 ways of usage
* direct calling of `f2py`
* extension in `setup.py` using `numpy.distutils`
* two ways of usage
* direct calling of the `f2py` executable
* define an extension in `setup.py` using `numpy.distutils`
* see examples in './f2py' (cf. the [f2py user guide](https://sysbio.ioc.ee/projects/f2py2e/usersguide/index.html))
%% Cell type:markdown id: tags:
### Compile and link Fortran code to Python module using `f2py`
### Compile and link Fortran code to a Python module using `f2py`
* Given the Fortran file `fib.f90`
```Fortran
subroutine fib(a,n)
integer, intent(in) :: n
integer(kind=8), intent(out) :: a(n)
do i=1,n
if (i.eq.1) then
a(i) = 0
elseif (i.eq.2) then
a(i) = 1
else
a(i) = a(i-1) + a(i-2)
endif
enddo
end
```
* Generate '.so' module file by calling `f2py -c fib.f90 -m fib` (second parameter is module name)
* Generate 'fib' Python module file by calling `f2py -c fib.f90 -m fib` (second parameter is module name)
%% Cell type:markdown id: tags:
### Compile and link Fortran code to Python module using 'setup.py'
* Easy, especially when a `setup.py` already exists
* Necessary change: use `numpy.distutils` instead of `distutils`
$\rightarrow$ extension sources may contain a Fortran file
%% Cell type:markdown id: tags:
```python
# setup.py
from numpy.distutils.core import setup, Extension
ext = Extension(name = 'fib', sources = ['fib.f90'])
setup(name = 'fibonacci', ext_modules = [ext])
```
%% Cell type:markdown id: tags:
```bash
$ python setup.py build_ext --inplace
(...)
$ python -c "import fib; a=fib.fib(16); print(a[-1])"
610
```
%% Cell type:markdown id: tags:
## Cython: Interfacing with C/C++ code
%% Cell type:markdown id: tags:
### Strategy
* write a Cython extension that defines Python functions which call the external C code
* see the example in the directory `cython/c_interface`
%% Cell type:markdown id: tags:
```C
/* foobar/c_hello.h */
void hello(void);
```
%% Cell type:markdown id: tags:
```C
/* foobar/c_hello.c */
#include <stdio.h>
#include "c_hello.h"
void hello(void) {
printf("Hello World!\n");
}
```
%% Cell type:markdown id: tags:
```cython
# foobar/hello.pyx
cdef extern from "c_hello.h":
void hello()
def say_hello():
hello()
```
%% Cell type:markdown id: tags:
```python
# setup.py
from setuptools import setup, Extension
ext = Extension("foobar.hello",
sources=["foobar/hello.pyx", "foobar/c_hello.c"])
setup(name="foobar",
ext_modules=[ext])
```
%% Cell type:markdown id: tags:
## Interfacing with C from NumPy-Code using Cython
## Interfacing with C code from NumPy code using Cython
* Goals
* Pass NumPy arrays down to C code to perform computation,
and get the result back as a NumPy array
* Leave the memory management to the Python/NumPy layer
* See the comprehensive example in the directory `cython/c_numpy`
%% Cell type:markdown id: tags:
### Tipps and tricks for interfacing NumPy with C
* Cython layer
* check the datatypes to be passed carefully
* pass the pointer to the NumPy array (`a`) via `a.data` down to the C function
* handle (temporary) memory allocation conveniently via NumPy arrays here, not in the C layer
* make sure to only pass contiguous NumPy arrays to C, in case of non-contiguous views create a copy first
* C layer
* write the extension stateless, beware of memory leaks
* you may use all kinds of code optimization techniques
* OpenMP threading (not affected by the Python GIL)
* vectorization
* cache optimization, etc.
* tweak the compiler flags in `setup.py`
* use optimizing compiler flags in `setup.py`
%% Cell type:markdown id: tags:
## Interfacing with a C library (shared object) using Cython
## Interfacing with a C library ('.so', shared object) using Cython
* Goal: Use a C library from Python
* third-party library for which no Python bindings exist, yet
* third-party library for which no Python bindings exist
* legacy code you want to wrap into Python
* CUDA code (compile into `.so` file independently using `nvcc`, first)
* see the comprehensive example in `cython/c_interface_shared_object`
%% Cell type:markdown id: tags:
### Tipps and tricks for interfacing shared objects
* `setup.py`
* usability: locate the library and its headers, options
* installation location passed via environment variable,
often the case on HPC systems via environment modules (`MKL_ROOT`, `GSL_HOME`, `FFTW_HOME`)
* installation location passed as arguments to `setup.py`
* installed at sytem location
* deployment: add the RPATH to the link line pointing to library location, to avoid `LD_LIBRARY_PATH`
* for CUDA code, it is much easier to go via a library than using `setup.py` with `nvcc`
* hint: for CUDA code, it is often easier to go via a library than injecting `nvcc` into `setup.py`
%% Cell type:markdown id: tags:
## Cython and C++
* Cython supports most of the C++ features: classes, templates, overloading
* Cython supports many object-oriented/advanced C++ features: classes, templates, overloading
* typical use case: write a Python wrapper class for a C++ class
* not covered here, please see for further details
http://cython.readthedocs.io/en/latest/src/userguide/wrapping_CPlusPlus.html
%% Cell type:markdown id: tags:
### Cython provides interfaces to libc and c++ STL
### Cython provides interfaces to libc and C++ STL (!)
%% Cell type:code id: tags:
``` python
%load_ext Cython
```
%% Cell type:code id: tags:
``` python
%%cython
from libc.math cimport sin
print(sin(1.0))
```
%% Output
0.8414709848078965
%% Cell type:code id: tags:
``` python
%%cython
# distutils: language = c++
from libcpp.vector cimport vector
cdef vector[int] v = range(10)
print(v)
```
%% Output
[0, 1, 2, 3, 4, 5, 6, 7, 8, 9]
%% Cell type:markdown id: tags:
## pybind11
%% Cell type:code id: tags:
``` python
```
* lightweight header-only library to create interfaces to modern C++ code
* highlights: STL, iterators, classes and inheritance, smart pointers, move semantics, NumPy $\leftrightarrow$ Eigen, etc.
* cf. the simple NumPy example in `pybind11`
* documentation: https://pybind11.readthedocs.io/en/latest/
......
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Please register or to comment