Commit 34e9b78d authored by Theo Steininger's avatar Theo Steininger
Browse files

Merge branch 'master' into 'plotting'

Master

See merge request !69
parents 2794762d fe6bf4e5
Pipeline #11674 failed with stage
in 9 minutes and 56 seconds
......@@ -12,9 +12,9 @@ variables:
before_script:
- apt-get update
- >
apt-get install -y build-essential python python-pip python-dev git
gfortran autoconf gsl-bin libgsl-dev wget
- pip install -r ci/requirements_base.txt
apt-get install -y build-essential python python-pip python-dev git
autoconf gsl-bin libgsl-dev wget python-numpy cython
- pip install --upgrade -r ci/requirements_base.txt
- chmod +x ci/*.sh
test_min:
......@@ -27,9 +27,8 @@ test_mpi:
stage: test
script:
- apt-get install -y openmpi-bin libopenmpi-dev
- pip install astropy healpy mpi4py
- pip install git+https://github.com/mrbell/gfft
- ci/install_libsharp.sh
- pip install mpi4py
- ci/install_pyHealpix.sh
- python setup.py build_ext --inplace
- nosetests -vv
......@@ -41,9 +40,8 @@ test_mpi_fftw:
apt-get install -y libatlas-base-dev libfftw3-bin libfftw3-dev
libfftw3-double3 libfftw3-long3 libfftw3-mpi-dev libfftw3-mpi3
libfftw3-quad3 libfftw3-single3
- pip install astropy healpy mpi4py
- pip install git+https://github.com/mrbell/gfft
- ci/install_libsharp.sh
- pip install mpi4py
- ci/install_pyHealpix.sh
- ci/install_pyfftw.sh
- python setup.py build_ext --inplace
- nosetests -vv
......@@ -59,10 +57,9 @@ test_mpi_fftw_hdf5:
- >
apt-get install -y libhdf5-10 libhdf5-dev libhdf5-openmpi-10
libhdf5-openmpi-dev hdf5-tools
- pip install astropy healpy mpi4py
- pip install git+https://github.com/mrbell/gfft
- pip install mpi4py
- ci/install_pyHealpix.sh
- ci/install_h5py.sh
- ci/install_libsharp.sh
- ci/install_pyfftw.sh
- python setup.py build_ext --inplace
- nosetests -vv --with-coverage --cover-package=nifty --cover-branches
......
......@@ -4,21 +4,23 @@ FROM ubuntu:latest
RUN \
apt-get update && \
apt-get install -y build-essential python python-pip python-dev git \
gfortran autoconf gsl-bin libgsl-dev python-matplotlib openmpi-bin \
autoconf gsl-bin libgsl-dev openmpi-bin wget \
libopenmpi-dev libatlas-base-dev libfftw3-bin libfftw3-dev \
libfftw3-double3 libfftw3-long3 libfftw3-mpi-dev libfftw3-mpi3 \
libfftw3-quad3 libfftw3-single3 libhdf5-10 libhdf5-dev \
libhdf5-openmpi-10 libhdf5-openmpi-dev hdf5-tools python-h5py
libhdf5-openmpi-10 libhdf5-openmpi-dev hdf5-tools
# python dependencies
ADD ci/requirements.txt /tmp/requirements.txt
RUN pip install -r /tmp/requirements.txt
RUN pip install --upgrade -r /tmp/requirements.txt
# install libsharp and pyfftw
ADD ci/install_libsharp.sh /tmp/install_libsharp.sh
# install pyHealpix, pyfftw and h5py
ADD ci/install_pyHealpix.sh /tmp/install_pyHealpix.sh
RUN cd /tmp && chmod +x install_pyHealpix.sh && ./install_pyHealpix.sh
ADD ci/install_pyfftw.sh /tmp/install_pyfftw.sh
RUN cd /tmp && chmod +x install_libsharp.sh && ./install_libsharp.sh
RUN cd /tmp && chmod +x install_pyfftw.sh && ./install_pyfftw.sh
ADD ci/install_h5py.sh /tmp/install_h5py.sh
RUN cd /tmp && chmod +x install_h5py.sh && ./install_h5py.sh
# copy sources and install nifty
COPY . /tmp/NIFTy
......
......@@ -38,36 +38,26 @@ certain grids, **fields** that are defined on spaces, and **operators**
that apply to fields.
- [Spaces](http://www.mpa-garching.mpg.de/ift/nifty/space.html)
- `point_space` - unstructured list of points
- `rg_space` - *n*-dimensional regular Euclidean grid
- `lm_space` - spherical harmonics
- `gl_space` - Gauss-Legendre grid on the 2-sphere
- `hp_space` - [HEALPix](http://sourceforge.net/projects/healpix/)
grid on the 2-sphere
- `nested_space` - arbitrary product of grids
- [Fields](http://www.mpa-garching.mpg.de/ift/nifty/field.html)
- `field` - generic class for (discretized) fields
<!-- -->
field.cast_domain field.hat field.power field.smooth
field.conjugate field.inverse_hat field.pseudo_dot field.tensor_dot
field.dim field.norm field.set_target field.transform
field.dot field.plot field.set_val field.weight
field.conjugate field.dim field.norm
field.dot field.set_val field.weight
- [Operators](http://www.mpa-garching.mpg.de/ift/nifty/operator.html)
- `diagonal_operator` - purely diagonal matrices in a specified
basis
- `projection_operator` - projections onto subsets of a specified
basis
- `vecvec_operator` - matrices derived from the outer product of a
vector
- `response_operator` - exemplary responses that include a
convolution, masking and projection
- `propagator_operator` - information propagator in Wiener filter
theory
- `explicit_operator` - linear operators with an explicit matrix
representation
- (and more)
- (and more)
......@@ -81,19 +71,7 @@ Installation
- [Python](http://www.python.org/) (v2.7.x)
- [NumPy](http://www.numpy.org/)
- [SciPy](http://www.scipy.org/)
- [Cython](http://cython.org/)
- [matplotlib](http://matplotlib.org/)
- [GFFT](https://github.com/mrbell/gfft) (v0.1.0) - Generalized Fast
Fourier Transformations for Python - **optional**
- [HEALPy](https://github.com/healpy/healpy) (v1.8.1 without openmp) -
A Python wrapper for
[HEALPix](http://sourceforge.net/projects/healpix/) -**optional,
only needed for spherical spaces**
- [libsharp-wrapper](https://github.com/mselig/libsharp-wrapper)
(v0.1.2 without openmp) - A Python wrapper for the
[libsharp](http://sourceforge.net/projects/libsharp/) library
-**optional, only needed for spherical spaces**
### Download
......@@ -112,57 +90,35 @@ Starting with a fresh Ubuntu installation move to a folder like
- Install basic packages like python, python-dev, gsl and others:
sudo apt-get install curl git autoconf
sudo apt-get install python-dev python-pip gsl-bin libgsl0-dev libfreetype6-dev libpng-dev libatlas-base-dev gfortran
sudo apt-get install curl git autoconf
sudo apt-get install python-dev python-pip gsl-bin libgsl0-dev libfreetype6-dev libpng-dev libatlas-base-dev
- Install matplotlib:
- Using pip install numpy etc...:
sudo apt-get install python-matplotlib
sudo pip install numpy cython
- Using pip install numpy, scipy, etc...:
- Install pyHealpix:
sudo pip install numpy scipy cython pyfits healpy
- Now install the 'non-standard' dependencies. First of all gfft:
curl -LOk https://github.com/mrbell/gfft/tarball/master
tar -xzf master
cd mrbell-gfft*
sudo python setup.py install
cd ..
- Libsharp:
git clone http://git.code.sf.net/p/libsharp/code libsharp-code
cd libsharp-code
sudo autoconf
./configure --enable-pic --disable-openmp
sudo make
cd ..
- Libsharpwrapper:
git clone http://github.com/mselig/libsharp-wrapper.git libsharp-wrapper
cd libsharp-wrapper
sudo python setup.py build_ext
sudo python setup.py install
git clone https://gitlab.mpcdf.mpg.de/mtr/pyHealpix.git
cd pyHealpix
autoreconf -i && ./configure && sudo make install
cd ..
- Finally, NIFTy:
git clone https://gitlab.mpcdf.mpg.de/ift/NIFTy.git
cd nifty
sudo python setup.py install
cd ..
sudo python setup.py install
cd ..
### Installation on a linux cluster
### Installation on a Linux cluster
This is for you if you want to install NIFTy on a HPC machine or cluster
that is hosted by your university or institute. Most of the dependencies
will most likely already be there, but you won't have superuser
priviledges. In this case, instead:
privileges. In this case, instead of:
sudo python setup.py install
sudo python setup.py install
use:
......@@ -175,82 +131,40 @@ or:
in the instruction above. This will install the python packages into
your local user directory.
For pyHealpix, use:
git clone https://gitlab.mpcdf.mpg.de/mtr/pyHealpix.git
cd pyHealpix
autoreconf -i && ./configure --prefix=$HOME/.local && make install
cd ..
### Installation on OS X 10.11
We advice to install the following packages in the order as they appear
We advise to install the following packages in the order as they appear
below. We strongly recommend to install all needed packages via
MacPorts. Please be aware that not all packages are available on
MacPorts, missing ones need to be installed manually. It may also be
mentioned that one should only use one package manager, as multiple ones
may cause trouble.
- Install basic packages python, scipy, matplotlib and cython:
- Install basic packages numpy and cython:
sudo port install py27-numpy
sudo port install py27-scipy
sudo port install py27-matplotlib
sudo port install py27-cython
- Install gfft. **Depending where you installed GSL you may need to
change the path in setup.py!**:
sudo port install gsl
git clone https://github.com/mrbell/gfft.git}{https://github.com/mrbell/gfft.git
sudo python setup.py install
- Install healpy:
sudo port install py27-pyfits
git clone https://github.com/healpy/healpy.git
cd healpy
sudo python setup.py install
cd ..
- Install libsharp and therefore autoconf, automake and libtool.
Installations instructions for libsharp may be found here:
<https://sourceforge.net/p/libsharp/code/ci/master/tree/>:
- Install pyHealpix:
curl -OL http://ftpmirror.gnu.org/autoconf/autoconf-2.69.tar.gz
tar -xzf autoconf-2.69.tar.gz
cd autoconf-2.69
./configure && make && sudo make install
cd ..
curl -OL http://ftpmirror.gnu.org/automake/automake-1.14.tar.gz
tar -xzf automake-1.14.tar.gz
cd automake-1.14
./configure && make && sudo make install
cd ..
curl -OL http://ftpmirror.gnu.org/libtool/libtool-2.4.2.tar.gz
tar -xzf libtool-2.4.2.tar.gz
cd libtool-2.4.2
./configure && make && sudo make install
cd ..
git clone http://git.code.sf.net/p/libsharp/code libsharp-code
cd libsharp-code
sudo autoconf
./configure --enable-pic --disable-openmp
sudo make
cd ..
- Install libsharp-wrapper. **Adopt the path of the libsharp
installation in setup.py** :
sudo port install gcc
sudo port select gcc mp-gcc5
git clone https://github.com/mselig/libsharp-wrapper.git
cd libsharp-wrapper
sudo python setup.py install
git clone https://gitlab.mpcdf.mpg.de/mtr/pyHealpix.git
cd pyHealpix
autoreconf -i && ./configure && sudo make install
cd ..
- Install NIFTy:
git clone https://gitlab.mpcdf.mpg.de/ift/NIFTy.git
cd nifty
sudo python setup.py install
cd ..
sudo python setup.py install
cd ..
### Installation using pypi
......@@ -263,13 +177,26 @@ Alternatively, a private or user specific installation can be done by:
pip install --user ift_nifty
### Running the tests
In oder to run the tests one needs two additional packages:
pip install nose
pip install parameterized
Afterwards the tests (including a coverage report) are run using the following
command in the repository root:
nosetests --exe --cover-html
### First Steps
For a quickstart, you can browse through the [informal
introduction](http://www.mpa-garching.mpg.de/ift/nifty/start.html) or
dive into NIFTY by running one of the demonstrations, e.g.:
>>> run -m nifty.demos.demo_wf1
python demos/wiener_filter.py
Acknowledgement
---------------
......
#!/bin/bash
git clone http://git.code.sf.net/p/libsharp/code libsharp-code
cd libsharp-code
autoconf && ./configure --enable-pic --disable-openmp && make
cd ..
git clone https://github.com/mselig/libsharp-wrapper libsharp-wrapper
cd libsharp-wrapper
python setup.py build_ext install
cd ..
rm -r libsharp-code
rm -r libsharp-wrapper
#!/bin/bash
git clone https://gitlab.mpcdf.mpg.de/ift/pyHealpix.git
cd pyHealpix
autoreconf -i && ./configure && make -j4 install
cd ..
rm -rf pyHealpix
numpy
scipy
cython
astropy
healpy
mpi4py
ipython
matplotlib
ipython==5.3.0
nose
nose-parameterized
parameterized
coverage
git+https://gitlab.mpcdf.mpg.de/ift/mpi_dummy.git
git+https://gitlab.mpcdf.mpg.de/ift/keepers.git
git+https://gitlab.mpcdf.mpg.de/ift/D2O.git
numpy
scipy
cython
nose
nose-parameterized
parameterized
coverage
git+https://gitlab.mpcdf.mpg.de/ift/mpi_dummy.git
git+https://gitlab.mpcdf.mpg.de/ift/keepers.git
......
## NIFTY (Numerical Information Field Theory) has been developed at the
## Max-Planck-Institute for Astrophysics.
##
## Copyright (C) 2013 Max-Planck-Society
##
## Author: Marco Selig
## Project homepage: <http://www.mpa-garching.mpg.de/ift/nifty/>
##
## This program is free software: you can redistribute it and/or modify
## it under the terms of the GNU General Public License as published by
## the Free Software Foundation, either version 3 of the License, or
## (at your option) any later version.
##
## This program is distributed in the hope that it will be useful,
## but WITHOUT ANY WARRANTY; without even the implied warranty of
## MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
## See the GNU General Public License for more details.
##
## You should have received a copy of the GNU General Public License
## along with this program. If not, see <http://www.gnu.org/licenses/>.
# NIFTy
# Copyright (C) 2017 Theo Steininger
#
# Author: Theo Steininger
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
from __future__ import division
......@@ -27,8 +24,6 @@ from .version import __version__
from keepers import MPILogger
logger = MPILogger()
import dummys
# it is important to import config before d2o such that NIFTy is able to
# pre-create d2o's configuration object with the corrected path
from config import dependency_injector,\
......@@ -58,5 +53,3 @@ from operators import *
from probing import *
from sugar import *
#import pyximport; pyximport.install(pyimport = True)
# NIFTy
# Copyright (C) 2017 Theo Steininger
#
# Author: Theo Steininger
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
import numpy as np
from d2o import distributed_data_object
......@@ -9,24 +26,15 @@ __all__ = ['cos', 'sin', 'cosh', 'sinh', 'tan', 'tanh', 'arccos', 'arcsin',
'conjugate']
def _math_helper(x, function, inplace=False):
def _math_helper(x, function):
if isinstance(x, Field):
if inplace:
x.val.apply_scalar_function(function, inplace=True)
result = x
else:
result_val = x.val.apply_scalar_function(function)
result = x.copy_empty(dtype=result_val.dtype)
result.val = result_val
result_val = x.val.apply_scalar_function(function)
result = x.copy_empty(dtype=result_val.dtype)
result.val = result_val
elif isinstance(x, distributed_data_object):
result = x.apply_scalar_function(function, inplace=inplace)
result = x.apply_scalar_function(function, inplace=False)
else:
result = function(np.asarray(x))
if inplace:
x[:] = result
result = x
return result
......
# -*- coding: utf-8 -*-
# NIFTy
# Copyright (C) 2017 Theo Steininger
#
# Author: Theo Steininger
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
from nifty_config import dependency_injector,\
nifty_configuration
......
# -*- coding: utf-8 -*-
# NIFTy
# Copyright (C) 2017 Theo Steininger
#
# Author: Theo Steininger
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
import os
......@@ -8,6 +24,6 @@ import keepers
d2o_configuration = keepers.get_Configuration(
name='D2O',
file_name='D2O.conf',
search_pathes=[os.path.expanduser('~') + "/.config/nifty/",
os.path.expanduser('~') + "/.config/",
'./'])
search_paths=[os.path.expanduser('~') + "/.config/nifty/",
os.path.expanduser('~') + "/.config/",
'./'])
# -*- coding: utf-8 -*-
# NIFTy
# Copyright (C) 2017 Theo Steininger
#
# Author: Theo Steininger
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
import os
from distutils.version import LooseVersion as lv
import numpy as np
import keepers
......@@ -9,48 +24,16 @@ import keepers
# Setup the dependency injector
dependency_injector = keepers.DependencyInjector(
[('mpi4py.MPI', 'MPI'),
'gfft',
('nifty.dummys.gfft_dummy', 'gfft_dummy'),
'healpy',
'libsharp_wrapper_gl',
'plotly'])
'pyHealpix'])
dependency_injector.register('pyfftw', lambda z: hasattr(z, 'FFTW_MPI'))
# Initialize the variables
variable_fft_module = keepers.Variable(
'fft_module',
['pyfftw', 'gfft', 'gfft_dummy'],
lambda z: z in dependency_injector)
def _healpy_validator(use_healpy):
if not isinstance(use_healpy, bool):
return False
if not use_healpy:
return True
if 'healpy' not in dependency_injector:
return False
healpy = dependency_injector['healpy']
if lv(healpy.__version__) < lv('1.8.1'):
return False
return True
variable_use_healpy = keepers.Variable(
'use_healpy',
[True, False],
_healpy_validator,
genus='boolean')
variable_use_libsharp = keepers.Variable(
'use_libsharp',
[True, False],
lambda z: (('libsharp_wrapper_gl' in
dependency_injector)
if z else True) and isinstance(z, bool),
genus='boolean')
['fftw', 'numpy'],
lambda z: (('pyfftw' in dependency_injector)
if z == 'fftw' else True))
def _dtype_validator(dtype):
......@@ -69,7 +52,7 @@ variable_default_field_dtype = keepers.Variable(
variable_default_distribution_strategy = keepers.Variable(
'default_distribution_strategy',
['fftw', 'equal'],
['fftw', 'equal', 'not'],
lambda z: (('pyfftw' in dependency_injector)
if z == 'fftw' else True),
genus='str')
......@@ -77,19 +60,16 @@ variable_default_distribution_strategy = keepers.Variable(
nifty_configuration = keepers.get_Configuration(
name='NIFTy',
variables=[variable_fft_module,
variable_use_healpy,
variable_use_libsharp,
variable_default_field_dtype,
variable_default_distribution_strategy],
file_name='NIFTy.conf',
search_pathes=[os.path.expanduser('~') + "/.config/nifty/",
os.path.expanduser('~') + "/.config/",
'./'])
search_paths=[os.path.expanduser('~') + "/.config/nifty/",
os.path.expanduser('~') + "/.config/",