Commit bb4b79c0 authored by Martin Reinecke's avatar Martin Reinecke

switch to pyHealpix and adjust dependencies accordingly

parent a0fe1025
...@@ -12,8 +12,8 @@ variables: ...@@ -12,8 +12,8 @@ variables:
before_script: before_script:
- apt-get update - apt-get update
- > - >
apt-get install -y build-essential python python-pip python-dev git apt-get install -y build-essential python python-pip python-dev git
gfortran autoconf gsl-bin libgsl-dev wget autoconf gsl-bin libgsl-dev wget python-numpy cython
- pip install -r ci/requirements_base.txt - pip install -r ci/requirements_base.txt
- chmod +x ci/*.sh - chmod +x ci/*.sh
...@@ -26,10 +26,8 @@ test_min: ...@@ -26,10 +26,8 @@ test_min:
test_mpi: test_mpi:
stage: test stage: test
script: script:
- apt-get install -y openmpi-bin libopenmpi-dev - apt-get install -y openmpi-bin libopenmpi-dev python-mpi4py
- pip install astropy healpy mpi4py - ci/install_pyHealpix.sh
- pip install git+https://github.com/mrbell/gfft
- ci/install_libsharp.sh
- python setup.py build_ext --inplace - python setup.py build_ext --inplace
- nosetests -vv - nosetests -vv
...@@ -40,11 +38,8 @@ test_mpi_fftw: ...@@ -40,11 +38,8 @@ test_mpi_fftw:
- > - >
apt-get install -y libatlas-base-dev libfftw3-bin libfftw3-dev apt-get install -y libatlas-base-dev libfftw3-bin libfftw3-dev
libfftw3-double3 libfftw3-long3 libfftw3-mpi-dev libfftw3-mpi3 libfftw3-double3 libfftw3-long3 libfftw3-mpi-dev libfftw3-mpi3
libfftw3-quad3 libfftw3-single3 libfftw3-quad3 libfftw3-single3 python-mpi4py python-pyfftw
- pip install astropy healpy mpi4py - ci/install_pyHealpix.sh
- pip install git+https://github.com/mrbell/gfft
- ci/install_libsharp.sh
- ci/install_pyfftw.sh
- python setup.py build_ext --inplace - python setup.py build_ext --inplace
- nosetests -vv - nosetests -vv
...@@ -58,12 +53,8 @@ test_mpi_fftw_hdf5: ...@@ -58,12 +53,8 @@ test_mpi_fftw_hdf5:
libfftw3-quad3 libfftw3-single3 libfftw3-quad3 libfftw3-single3
- > - >
apt-get install -y libhdf5-10 libhdf5-dev libhdf5-openmpi-10 apt-get install -y libhdf5-10 libhdf5-dev libhdf5-openmpi-10
libhdf5-openmpi-dev hdf5-tools libhdf5-openmpi-dev hdf5-tools python-mpi4py python-pyfftw python-h5py
- pip install astropy healpy mpi4py - ci/install_pyHealpix.sh
- pip install git+https://github.com/mrbell/gfft
- ci/install_h5py.sh
- ci/install_libsharp.sh
- ci/install_pyfftw.sh
- python setup.py build_ext --inplace - python setup.py build_ext --inplace
- nosetests -vv --with-coverage --cover-package=nifty --cover-branches - nosetests -vv --with-coverage --cover-package=nifty --cover-branches
- > - >
......
...@@ -8,17 +8,15 @@ RUN \ ...@@ -8,17 +8,15 @@ RUN \
libopenmpi-dev libatlas-base-dev libfftw3-bin libfftw3-dev \ libopenmpi-dev libatlas-base-dev libfftw3-bin libfftw3-dev \
libfftw3-double3 libfftw3-long3 libfftw3-mpi-dev libfftw3-mpi3 \ libfftw3-double3 libfftw3-long3 libfftw3-mpi-dev libfftw3-mpi3 \
libfftw3-quad3 libfftw3-single3 libhdf5-10 libhdf5-dev \ libfftw3-quad3 libfftw3-single3 libhdf5-10 libhdf5-dev \
libhdf5-openmpi-10 libhdf5-openmpi-dev hdf5-tools python-h5py libhdf5-openmpi-10 libhdf5-openmpi-dev hdf5-tools python-h5py python-pyfftw
# python dependencies # python dependencies
ADD ci/requirements.txt /tmp/requirements.txt ADD ci/requirements.txt /tmp/requirements.txt
RUN pip install -r /tmp/requirements.txt RUN pip install -r /tmp/requirements.txt
# install libsharp and pyfftw # install pyHealpix
ADD ci/install_libsharp.sh /tmp/install_libsharp.sh ADD ci/install_pyHealpix.sh /tmp/install_pyHealpix.sh
ADD ci/install_pyfftw.sh /tmp/install_pyfftw.sh RUN cd /tmp && chmod +x install_pyHealpix.sh && ./install_pyHealpix.sh
RUN cd /tmp && chmod +x install_libsharp.sh && ./install_libsharp.sh
RUN cd /tmp && chmod +x install_pyfftw.sh && ./install_pyfftw.sh
# copy sources and install nifty # copy sources and install nifty
COPY . /tmp/NIFTy COPY . /tmp/NIFTy
......
...@@ -38,36 +38,26 @@ certain grids, **fields** that are defined on spaces, and **operators** ...@@ -38,36 +38,26 @@ certain grids, **fields** that are defined on spaces, and **operators**
that apply to fields. that apply to fields.
- [Spaces](http://www.mpa-garching.mpg.de/ift/nifty/space.html) - [Spaces](http://www.mpa-garching.mpg.de/ift/nifty/space.html)
- `point_space` - unstructured list of points
- `rg_space` - *n*-dimensional regular Euclidean grid - `rg_space` - *n*-dimensional regular Euclidean grid
- `lm_space` - spherical harmonics - `lm_space` - spherical harmonics
- `gl_space` - Gauss-Legendre grid on the 2-sphere - `gl_space` - Gauss-Legendre grid on the 2-sphere
- `hp_space` - [HEALPix](http://sourceforge.net/projects/healpix/) - `hp_space` - [HEALPix](http://sourceforge.net/projects/healpix/)
grid on the 2-sphere grid on the 2-sphere
- `nested_space` - arbitrary product of grids
- [Fields](http://www.mpa-garching.mpg.de/ift/nifty/field.html) - [Fields](http://www.mpa-garching.mpg.de/ift/nifty/field.html)
- `field` - generic class for (discretized) fields - `field` - generic class for (discretized) fields
<!-- --> <!-- -->
field.cast_domain field.hat field.power field.smooth field.conjugate field.dim field.norm
field.conjugate field.inverse_hat field.pseudo_dot field.tensor_dot field.dot field.set_val field.weight
field.dim field.norm field.set_target field.transform
field.dot field.plot field.set_val field.weight
- [Operators](http://www.mpa-garching.mpg.de/ift/nifty/operator.html) - [Operators](http://www.mpa-garching.mpg.de/ift/nifty/operator.html)
- `diagonal_operator` - purely diagonal matrices in a specified - `diagonal_operator` - purely diagonal matrices in a specified
basis basis
- `projection_operator` - projections onto subsets of a specified - `projection_operator` - projections onto subsets of a specified
basis basis
- `vecvec_operator` - matrices derived from the outer product of a
vector
- `response_operator` - exemplary responses that include a
convolution, masking and projection
- `propagator_operator` - information propagator in Wiener filter - `propagator_operator` - information propagator in Wiener filter
theory theory
- `explicit_operator` - linear operators with an explicit matrix
representation
- (and more) - (and more)
- (and more) - (and more)
...@@ -81,19 +71,7 @@ Installation ...@@ -81,19 +71,7 @@ Installation
- [Python](http://www.python.org/) (v2.7.x) - [Python](http://www.python.org/) (v2.7.x)
- [NumPy](http://www.numpy.org/) - [NumPy](http://www.numpy.org/)
- [SciPy](http://www.scipy.org/)
- [Cython](http://cython.org/) - [Cython](http://cython.org/)
- [matplotlib](http://matplotlib.org/)
- [GFFT](https://github.com/mrbell/gfft) (v0.1.0) - Generalized Fast
Fourier Transformations for Python - **optional**
- [HEALPy](https://github.com/healpy/healpy) (v1.8.1 without openmp) -
A Python wrapper for
[HEALPix](http://sourceforge.net/projects/healpix/) -**optional,
only needed for spherical spaces**
- [libsharp-wrapper](https://github.com/mselig/libsharp-wrapper)
(v0.1.2 without openmp) - A Python wrapper for the
[libsharp](http://sourceforge.net/projects/libsharp/) library
-**optional, only needed for spherical spaces**
### Download ### Download
...@@ -112,57 +90,35 @@ Starting with a fresh Ubuntu installation move to a folder like ...@@ -112,57 +90,35 @@ Starting with a fresh Ubuntu installation move to a folder like
- Install basic packages like python, python-dev, gsl and others: - Install basic packages like python, python-dev, gsl and others:
sudo apt-get install curl git autoconf sudo apt-get install curl git autoconf
sudo apt-get install python-dev python-pip gsl-bin libgsl0-dev libfreetype6-dev libpng-dev libatlas-base-dev gfortran sudo apt-get install python-dev python-pip gsl-bin libgsl0-dev libfreetype6-dev libpng-dev libatlas-base-dev
- Install matplotlib: - Using pip install numpy etc...:
sudo apt-get install python-matplotlib sudo pip install numpy cython
- Using pip install numpy, scipy, etc...: - Install pyHealpix:
sudo pip install numpy scipy cython pyfits healpy git clone https://gitlab.mpcdf.mpg.de/mtr/pyHealpix.git
cd pyHealpix
- Now install the 'non-standard' dependencies. First of all gfft: autoreconf -i && ./configure && sudo make install
curl -LOk https://github.com/mrbell/gfft/tarball/master
tar -xzf master
cd mrbell-gfft*
sudo python setup.py install
cd ..
- Libsharp:
git clone http://git.code.sf.net/p/libsharp/code libsharp-code
cd libsharp-code
sudo autoconf
./configure --enable-pic --disable-openmp
sudo make
cd ..
- Libsharpwrapper:
git clone http://github.com/mselig/libsharp-wrapper.git libsharp-wrapper
cd libsharp-wrapper
sudo python setup.py build_ext
sudo python setup.py install
cd .. cd ..
- Finally, NIFTy: - Finally, NIFTy:
git clone https://gitlab.mpcdf.mpg.de/ift/NIFTy.git git clone https://gitlab.mpcdf.mpg.de/ift/NIFTy.git
cd nifty cd nifty
sudo python setup.py install sudo python setup.py install
cd .. cd ..
### Installation on a linux cluster ### Installation on a Linux cluster
This is for you if you want to install NIFTy on a HPC machine or cluster This is for you if you want to install NIFTy on a HPC machine or cluster
that is hosted by your university or institute. Most of the dependencies that is hosted by your university or institute. Most of the dependencies
will most likely already be there, but you won't have superuser will most likely already be there, but you won't have superuser
priviledges. In this case, instead: privileges. In this case, instead of:
sudo python setup.py install sudo python setup.py install
use: use:
...@@ -175,82 +131,40 @@ or: ...@@ -175,82 +131,40 @@ or:
in the instruction above. This will install the python packages into in the instruction above. This will install the python packages into
your local user directory. your local user directory.
For pyHealpix, use:
git clone https://gitlab.mpcdf.mpg.de/mtr/pyHealpix.git
cd pyHealpix
autoreconf -i && ./configure --prefix=$HOME/.local && make install
cd ..
### Installation on OS X 10.11 ### Installation on OS X 10.11
We advice to install the following packages in the order as they appear We advise to install the following packages in the order as they appear
below. We strongly recommend to install all needed packages via below. We strongly recommend to install all needed packages via
MacPorts. Please be aware that not all packages are available on MacPorts. Please be aware that not all packages are available on
MacPorts, missing ones need to be installed manually. It may also be MacPorts, missing ones need to be installed manually. It may also be
mentioned that one should only use one package manager, as multiple ones mentioned that one should only use one package manager, as multiple ones
may cause trouble. may cause trouble.
- Install basic packages python, scipy, matplotlib and cython: - Install basic packages numpy and cython:
sudo port install py27-numpy sudo port install py27-numpy
sudo port install py27-scipy
sudo port install py27-matplotlib
sudo port install py27-cython sudo port install py27-cython
- Install gfft. **Depending where you installed GSL you may need to - Install pyHealpix:
change the path in setup.py!**:
sudo port install gsl
git clone https://github.com/mrbell/gfft.git}{https://github.com/mrbell/gfft.git
sudo python setup.py install
- Install healpy:
sudo port install py27-pyfits git clone https://gitlab.mpcdf.mpg.de/mtr/pyHealpix.git
git clone https://github.com/healpy/healpy.git cd pyHealpix
cd healpy autoreconf -i && ./configure && sudo make install
sudo python setup.py install
cd ..
- Install libsharp and therefore autoconf, automake and libtool.
Installations instructions for libsharp may be found here:
<https://sourceforge.net/p/libsharp/code/ci/master/tree/>:
curl -OL http://ftpmirror.gnu.org/autoconf/autoconf-2.69.tar.gz
tar -xzf autoconf-2.69.tar.gz
cd autoconf-2.69
./configure && make && sudo make install
cd ..
curl -OL http://ftpmirror.gnu.org/automake/automake-1.14.tar.gz
tar -xzf automake-1.14.tar.gz
cd automake-1.14
./configure && make && sudo make install
cd ..
curl -OL http://ftpmirror.gnu.org/libtool/libtool-2.4.2.tar.gz
tar -xzf libtool-2.4.2.tar.gz
cd libtool-2.4.2
./configure && make && sudo make install
cd ..
git clone http://git.code.sf.net/p/libsharp/code libsharp-code
cd libsharp-code
sudo autoconf
./configure --enable-pic --disable-openmp
sudo make
cd ..
- Install libsharp-wrapper. **Adopt the path of the libsharp
installation in setup.py** :
sudo port install gcc
sudo port select gcc mp-gcc5
git clone https://github.com/mselig/libsharp-wrapper.git
cd libsharp-wrapper
sudo python setup.py install
cd .. cd ..
- Install NIFTy: - Install NIFTy:
git clone https://gitlab.mpcdf.mpg.de/ift/NIFTy.git git clone https://gitlab.mpcdf.mpg.de/ift/NIFTy.git
cd nifty cd nifty
sudo python setup.py install sudo python setup.py install
cd .. cd ..
### Installation using pypi ### Installation using pypi
...@@ -269,10 +183,10 @@ In oder to run the tests one needs two additional packages: ...@@ -269,10 +183,10 @@ In oder to run the tests one needs two additional packages:
pip install nose pip install nose
pip install parameterized pip install parameterized
Afterwards the tests (including a coverage report) are run using the following Afterwards the tests (including a coverage report) are run using the following
command in the repository root: command in the repository root:
nosetests --exe --cover-html nosetests --exe --cover-html
...@@ -282,7 +196,7 @@ For a quickstart, you can browse through the [informal ...@@ -282,7 +196,7 @@ For a quickstart, you can browse through the [informal
introduction](http://www.mpa-garching.mpg.de/ift/nifty/start.html) or introduction](http://www.mpa-garching.mpg.de/ift/nifty/start.html) or
dive into NIFTY by running one of the demonstrations, e.g.: dive into NIFTY by running one of the demonstrations, e.g.:
>>> run -m nifty.demos.demo_wf1 python demos/wiener_filter.py
Acknowledgement Acknowledgement
--------------- ---------------
......
#!/bin/bash
wget https://api.github.com/repos/h5py/h5py/tags -O - | grep tarball_url | grep -v rc | head -n 1 | cut -d '"' -f 4 | wget -i - -O h5py.tar.gz
tar xzf h5py.tar.gz
cd h5py-h5py*
export CC=mpicc
export HDF5_DIR=/usr/lib/x86_64-linux-gnu/hdf5/openmpi
python setup.py configure --mpi
python setup.py build
python setup.py install
cd ..
rm -r h5py-h5py*
rm h5py.tar.gz
#!/bin/bash
git clone http://git.code.sf.net/p/libsharp/code libsharp-code
cd libsharp-code
autoconf && ./configure --enable-pic --disable-openmp && make
cd ..
git clone https://github.com/mselig/libsharp-wrapper libsharp-wrapper
cd libsharp-wrapper
python setup.py build_ext install
cd ..
rm -r libsharp-code
rm -r libsharp-wrapper
#!/bin/bash
git clone -b mpi https://github.com/fredRos/pyFFTW.git
cd pyFFTW/
CC=mpicc python setup.py build_ext install
cd ..
rm -r pyFFTW
numpy
scipy
cython
astropy
healpy
mpi4py
ipython==5.3.0
nose nose
nose-parameterized parameterized
coverage
git+https://gitlab.mpcdf.mpg.de/ift/mpi_dummy.git git+https://gitlab.mpcdf.mpg.de/ift/mpi_dummy.git
git+https://gitlab.mpcdf.mpg.de/ift/keepers.git git+https://gitlab.mpcdf.mpg.de/ift/keepers.git
git+https://gitlab.mpcdf.mpg.de/ift/D2O.git git+https://gitlab.mpcdf.mpg.de/ift/D2O.git
numpy
scipy
cython
nose nose
nose-parameterized parameterized
coverage coverage
git+https://gitlab.mpcdf.mpg.de/ift/mpi_dummy.git git+https://gitlab.mpcdf.mpg.de/ift/mpi_dummy.git
git+https://gitlab.mpcdf.mpg.de/ift/keepers.git git+https://gitlab.mpcdf.mpg.de/ift/keepers.git
......
...@@ -25,12 +25,10 @@ import keepers ...@@ -25,12 +25,10 @@ import keepers
# Setup the dependency injector # Setup the dependency injector
dependency_injector = keepers.DependencyInjector( dependency_injector = keepers.DependencyInjector(
[('mpi4py.MPI', 'MPI'), [('mpi4py.MPI', 'MPI'),
'healpy', 'pyHealpix'])
'libsharp_wrapper_gl'])
dependency_injector.register('pyfftw', lambda z: hasattr(z, 'FFTW_MPI')) dependency_injector.register('pyfftw', lambda z: hasattr(z, 'FFTW_MPI'))
# Initialize the variables # Initialize the variables
variable_fft_module = keepers.Variable( variable_fft_module = keepers.Variable(
'fft_module', 'fft_module',
...@@ -38,35 +36,23 @@ variable_fft_module = keepers.Variable( ...@@ -38,35 +36,23 @@ variable_fft_module = keepers.Variable(
lambda z: (('pyfftw' in dependency_injector) lambda z: (('pyfftw' in dependency_injector)
if z == 'fftw' else True)) if z == 'fftw' else True))
def _pyHealpix_validator(use_pyHealpix):
def _healpy_validator(use_healpy): if not isinstance(use_pyHealpix, bool):
if not isinstance(use_healpy, bool):
return False return False
if not use_healpy: if not use_pyHealpix:
return True return True
if 'healpy' not in dependency_injector: if 'pyHealpix' not in dependency_injector:
return False
healpy = dependency_injector['healpy']
if lv(healpy.__version__) < lv('1.8.1'):
return False return False
pyHealpix = dependency_injector['pyHealpix']
return True return True
variable_use_healpy = keepers.Variable( variable_use_pyHealpix = keepers.Variable(
'use_healpy', 'use_pyHealpix',
[True, False], [True, False],
_healpy_validator, _pyHealpix_validator,
genus='boolean') genus='boolean')
variable_use_libsharp = keepers.Variable(
'use_libsharp',
[True, False],
lambda z: (('libsharp_wrapper_gl' in
dependency_injector)
if z else True) and isinstance(z, bool),
genus='boolean')
def _dtype_validator(dtype): def _dtype_validator(dtype):
try: try:
np.dtype(dtype) np.dtype(dtype)
...@@ -91,8 +77,7 @@ variable_default_distribution_strategy = keepers.Variable( ...@@ -91,8 +77,7 @@ variable_default_distribution_strategy = keepers.Variable(
nifty_configuration = keepers.get_Configuration( nifty_configuration = keepers.get_Configuration(
name='NIFTy', name='NIFTy',
variables=[variable_fft_module, variables=[variable_fft_module,
variable_use_healpy, variable_use_pyHealpix,
variable_use_libsharp,
variable_default_field_dtype, variable_default_field_dtype,
variable_default_distribution_strategy], variable_default_distribution_strategy],
file_name='NIFTy.conf', file_name='NIFTy.conf',
......
...@@ -23,7 +23,7 @@ from nifty import GLSpace, LMSpace ...@@ -23,7 +23,7 @@ from nifty import GLSpace, LMSpace
from slicing_transformation import SlicingTransformation from slicing_transformation import SlicingTransformation
import lm_transformation_factory as ltf import lm_transformation_factory as ltf
libsharp = gdi.get('libsharp_wrapper_gl') pyHealpix = gdi.get('pyHealpix')
class GLLMTransformation(SlicingTransformation): class GLLMTransformation(SlicingTransformation):
...@@ -31,9 +31,9 @@ class GLLMTransformation(SlicingTransformation): ...@@ -31,9 +31,9 @@ class GLLMTransformation(SlicingTransformation):
# ---Overwritten properties and methods--- # ---Overwritten properties and methods---
def __init__(self, domain, codomain=None, module=None): def __init__(self, domain, codomain=None, module=None):
if 'libsharp_wrapper_gl' not in gdi: if 'pyHealpix' not in gdi:
raise ImportError( raise ImportError(
"The module libsharp is needed but not available.") "The module pyHealpix is needed but not available.")
super(GLLMTransformation, self).__init__(domain, codomain, super(GLLMTransformation, self).__init__(domain, codomain,
module=module) module=module)
...@@ -108,14 +108,12 @@ class GLLMTransformation(SlicingTransformation): ...@@ -108,14 +108,12 @@ class GLLMTransformation(SlicingTransformation):
lmax = self.codomain.lmax lmax = self.codomain.lmax
mmax = self.codomain.mmax mmax = self.codomain.mmax
sjob=pyHealpix.sharpjob_d()
sjob.set_Gauss_geometry(nlat,nlot)
sjob.set_triangular_alm_info(lmax,mmax)
if issubclass(inp.dtype.type, np.complexfloating): if issubclass(inp.dtype.type, np.complexfloating):
[resultReal, resultImag] = [self.libsharpMap2Alm(x, [resultReal, resultImag] = [sjob.map2alm(x)
nlat=nlat,
nlon=nlon,
lmax=lmax,
mmax=mmax,
**kwargs)
for x in (inp.real, inp.imag)] for x in (inp.real, inp.imag)]
[resultReal, resultImag] = [ltf.buildIdx(x, lmax=lmax) [resultReal, resultImag] = [ltf.buildIdx(x, lmax=lmax)
...@@ -123,22 +121,7 @@ class GLLMTransformation(SlicingTransformation): ...@@ -123,22 +121,7 @@ class GLLMTransformation(SlicingTransformation):
result = self._combine_complex_result(resultReal, resultImag) result = self._combine_complex_result(resultReal, resultImag)
else: else:
result = self.libsharpMap2Alm(inp, nlat=nlat, nlon=nlon, lmax=lmax, result = sjob.map2alm(inp)