Commit 25e88936 authored by Philipp Arras's avatar Philipp Arras
Browse files

Merge remote-tracking branch 'dev/NIFTy_5' into NIFTy_5

parents 9e80b9a0 87a53824
Pipeline #43112 failed with stages
in 13 seconds
...@@ -10,6 +10,8 @@ setup.cfg ...@@ -10,6 +10,8 @@ setup.cfg
.document .document
.svn/ .svn/
*.csv *.csv
.pytest_cache/
*.png
# from https://github.com/github/gitignore/blob/master/Python.gitignore # from https://github.com/github/gitignore/blob/master/Python.gitignore
......
image: $CONTAINER_TEST_IMAGE image: $CONTAINER_TEST_IMAGE
variables: variables:
CONTAINER_TEST_IMAGE: gitlab-registry.mpcdf.mpg.de/ift/nifty:$CI_BUILD_REF_NAME CONTAINER_TEST_IMAGE: gitlab-registry.mpcdf.mpg.de/ift/nifty-dev:$CI_BUILD_REF_NAME
OMP_NUM_THREADS: 1 OMP_NUM_THREADS: 1
stages: stages:
...@@ -15,6 +15,8 @@ build_docker_from_scratch: ...@@ -15,6 +15,8 @@ build_docker_from_scratch:
- schedules - schedules
image: docker:stable image: docker:stable
stage: build_docker stage: build_docker
before_script:
- ls
script: script:
- docker login -u gitlab-ci-token -p $CI_BUILD_TOKEN gitlab-registry.mpcdf.mpg.de - docker login -u gitlab-ci-token -p $CI_BUILD_TOKEN gitlab-registry.mpcdf.mpg.de
- docker build -t $CONTAINER_TEST_IMAGE --no-cache . - docker build -t $CONTAINER_TEST_IMAGE --no-cache .
...@@ -25,177 +27,87 @@ build_docker_from_cache: ...@@ -25,177 +27,87 @@ build_docker_from_cache:
- schedules - schedules
image: docker:stable image: docker:stable
stage: build_docker stage: build_docker
before_script:
- ls
script: script:
- docker login -u gitlab-ci-token -p $CI_BUILD_TOKEN gitlab-registry.mpcdf.mpg.de - docker login -u gitlab-ci-token -p $CI_BUILD_TOKEN gitlab-registry.mpcdf.mpg.de
- docker build -t $CONTAINER_TEST_IMAGE . - docker build -t $CONTAINER_TEST_IMAGE .
- docker push $CONTAINER_TEST_IMAGE - docker push $CONTAINER_TEST_IMAGE
test_python2_with_coverage: test_serial:
stage: test stage: test
script: script:
- python setup.py install --user -f - pytest-3 -q --cov=nifty5 test
- mpiexec -n 2 --bind-to none nosetests -q 2> /dev/null
- nosetests -q --with-coverage --cover-package=nifty5 --cover-erase
- > - >
coverage report --omit "*plotting*,*distributed_do*" python3 -m coverage report --omit "*plot*,*distributed_do*" | tee coverage.txt
- > - >
coverage report --omit "*plotting*,*distributed_do*" | grep TOTAL | awk '{ print "TOTAL: "$4; }' grep TOTAL coverage.txt | awk '{ print "TOTAL: "$4; }'
test_python3: test_mpi:
stage: test stage: test
variables:
OMPI_MCA_btl_vader_single_copy_mechanism: none
script: script:
- python3 setup.py install --user -f - mpiexec -n 2 --bind-to none pytest-3 -q test
- mpiexec -n 2 --bind-to none nosetests3 -q 2> /dev/null
- nosetests3 -q
pages: pages:
stage: release stage: release
script: script:
- python setup.py install --user -f
- sh docs/generate.sh - sh docs/generate.sh
- mv docs/build/ public/ - mv docs/build/ public/
artifacts: artifacts:
paths: paths:
- public - public
only: only:
- NIFTy_4 - NIFTy_5
before_script:
- export MPLBACKEND="agg"
run_critical_filtering:
stage: demo_runs
script:
- ls
- python setup.py install --user -f
- python3 setup.py install --user -f
- python demos/critical_filtering.py
- python3 demos/critical_filtering.py
artifacts:
paths:
- '*.png'
run_nonlinear_critical_filter:
stage: demo_runs
script:
- python setup.py install --user -f
- python3 setup.py install --user -f
- python demos/nonlinear_critical_filter.py
- python3 demos/nonlinear_critical_filter.py
artifacts:
paths:
- '*.png'
run_nonlinear_wiener_filter:
stage: demo_runs
script:
- python setup.py install --user -f
- python3 setup.py install --user -f
- python demos/nonlinear_wiener_filter.py
- python3 demos/nonlinear_wiener_filter.py
only:
- run_demos
artifacts:
paths:
- '*.png'
# FIXME: disable for now. Fixing it is part of issue #244.
#run_poisson_demo:
# stage: demo_runs
# script:
# - python setup.py install --user -f
# - python3 setup.py install --user -f
# - python demos/poisson_demo.py
# - python3 demos/poisson_demo.py
# artifacts:
# paths:
# - '*.png'
run_probing:
stage: demo_runs
script:
- python setup.py install --user -f
- python3 setup.py install --user -f
- python demos/probing.py
- python3 demos/probing.py
artifacts:
paths:
- '*.png'
run_sampling: before_script:
stage: demo_runs
script:
- python setup.py install --user -f
- python3 setup.py install --user -f - python3 setup.py install --user -f
- python demos/sampling.py
- python3 demos/sampling.py
artifacts:
paths:
- '*.png'
run_tomography: run_ipynb:
stage: demo_runs stage: demo_runs
script: script:
- python setup.py install --user -f - jupyter nbconvert --execute --ExecutePreprocessor.timeout=None demos/Wiener_Filter.ipynb
- python3 setup.py install --user -f
- python demos/tomography.py
- python3 demos/tomography.py
artifacts:
paths:
- '*.png'
run_wiener_filter_data_space_noiseless: run_getting_started_1:
stage: demo_runs stage: demo_runs
script: script:
- python setup.py install --user -f - python3 demos/getting_started_1.py
- python3 setup.py install --user -f - mpiexec -n 2 --bind-to none python3 demos/getting_started_1.py 2> /dev/null
- python demos/wiener_filter_data_space_noiseless.py
- python3 demos/wiener_filter_data_space_noiseless.py
artifacts: artifacts:
paths: paths:
- '*.png' - '*.png'
run_wiener_filter_easy.py: run_getting_started_2:
stage: demo_runs stage: demo_runs
script: script:
- python setup.py install --user -f - python3 demos/getting_started_2.py
- python3 setup.py install --user -f - mpiexec -n 2 --bind-to none python3 demos/getting_started_2.py 2> /dev/null
- python demos/wiener_filter_easy.py
- python3 demos/wiener_filter_easy.py
artifacts: artifacts:
paths: paths:
- '*.png' - '*.png'
run_wiener_filter_via_curvature.py: run_getting_started_3:
stage: demo_runs stage: demo_runs
script: script:
- pip install --user numericalunits - python3 demos/getting_started_3.py
- pip3 install --user numericalunits
- python setup.py install --user -f
- python3 setup.py install --user -f
- python demos/wiener_filter_via_curvature.py
- python3 demos/wiener_filter_via_curvature.py
artifacts: artifacts:
paths: paths:
- '*.png' - '*.png'
run_wiener_filter_via_hamiltonian.py: run_bernoulli:
stage: demo_runs stage: demo_runs
script: script:
- python setup.py install --user -f - python3 demos/bernoulli_demo.py
- python3 setup.py install --user -f
- python demos/wiener_filter_via_hamiltonian.py
- python3 demos/wiener_filter_via_hamiltonian.py
artifacts: artifacts:
paths: paths:
- '*.png' - '*.png'
run_ipynb: run_curve_fitting:
stage: demo_runs stage: demo_runs
script: script:
- python setup.py install --user -f - python3 demos/polynomial_fit.py
- python3 setup.py install --user -f
- jupyter nbconvert --execute --ExecutePreprocessor.timeout=None demos/Wiener_Filter.ipynb
artifacts: artifacts:
paths: paths:
- '*.png' - '*.png'
FROM debian:testing-slim FROM debian:testing-slim
RUN apt-get update && apt-get install -y \ RUN apt-get update && apt-get install -y \
# Needed for gitlab tests # Needed for setup
git \ git python3-pip \
# Packages needed for NIFTy # Packages needed for NIFTy
libfftw3-dev \ python3-scipy \
python python-pip python-dev python-future python-scipy cython \
python3 python3-pip python3-dev python3-future python3-scipy cython3 \
# Documentation build dependencies # Documentation build dependencies
python-sphinx python-sphinx-rtd-theme python-numpydoc \ python3-sphinx-rtd-theme dvipng texlive-latex-base texlive-latex-extra \
# Testing dependencies # Testing dependencies
python-nose python-parameterized \ python3-pytest-cov jupyter \
python3-nose python3-parameterized \
# Optional NIFTy dependencies # Optional NIFTy dependencies
openmpi-bin libopenmpi-dev python-mpi4py python3-mpi4py \ libfftw3-dev python3-mpi4py python3-matplotlib \
# Packages needed for NIFTy # more optional NIFTy dependencies
&& pip install pyfftw \
&& pip3 install pyfftw \ && pip3 install pyfftw \
# Optional NIFTy dependencies
&& pip install git+https://gitlab.mpcdf.mpg.de/ift/pyHealpix.git \
&& pip3 install git+https://gitlab.mpcdf.mpg.de/ift/pyHealpix.git \ && pip3 install git+https://gitlab.mpcdf.mpg.de/ift/pyHealpix.git \
# Testing dependencies && pip3 install jupyter \
&& pip install coverage \
&& rm -rf /var/lib/apt/lists/* && rm -rf /var/lib/apt/lists/*
# Needed for demos to be running # Set matplotlib backend
RUN apt-get update && apt-get install -y python-matplotlib python3-matplotlib \ ENV MPLBACKEND agg
&& python3 -m pip install --upgrade pip && python3 -m pip install jupyter && python -m pip install --upgrade pip && python -m pip install jupyter \
&& rm -rf /var/lib/apt/lists/*
# Create user (openmpi does not like to be run as root) # Create user (openmpi does not like to be run as root)
RUN useradd -ms /bin/bash testinguser RUN useradd -ms /bin/bash testinguser
......
...@@ -37,11 +37,11 @@ Installation ...@@ -37,11 +37,11 @@ Installation
### Requirements ### Requirements
- [Python](https://www.python.org/) (v2.7.x or 3.5.x) - [Python 3](https://www.python.org/) (3.5.x or later)
- [SciPy](https://www.scipy.org/) - [SciPy](https://www.scipy.org/)
- [pyFFTW](https://pypi.python.org/pypi/pyFFTW)
Optional dependencies: Optional dependencies:
- [pyFFTW](https://pypi.python.org/pypi/pyFFTW) for faster Fourier transforms
- [pyHealpix](https://gitlab.mpcdf.mpg.de/ift/pyHealpix) (for harmonic - [pyHealpix](https://gitlab.mpcdf.mpg.de/ift/pyHealpix) (for harmonic
transforms involving domains on the sphere) transforms involving domains on the sphere)
- [mpi4py](https://mpi4py.scipy.org) (for MPI-parallel execution) - [mpi4py](https://mpi4py.scipy.org) (for MPI-parallel execution)
...@@ -61,45 +61,48 @@ distributions, the "apt" lines will need slight changes. ...@@ -61,45 +61,48 @@ distributions, the "apt" lines will need slight changes.
NIFTy5 and its mandatory dependencies can be installed via: NIFTy5 and its mandatory dependencies can be installed via:
sudo apt-get install git libfftw3-dev python python-pip python-dev sudo apt-get install git python3 python3-pip python3-dev
pip install --user git+https://gitlab.mpcdf.mpg.de/ift/NIFTy.git@NIFTy_5 pip3 install --user git+https://gitlab.mpcdf.mpg.de/ift/NIFTy.git@NIFTy_5
Plotting support is added via:
sudo apt-get install python3-matplotlib
NIFTy uses Numpy's FFT implementation by default. For large problems FFTW may be
used because of its higher performance. It can be installed via:
sudo apt-get install libfftw3-dev
pip3 install --user pyfftw
To enable FFTW usage in NIFTy, call
nifty5.fft.enable_fftw()
at the beginning of your code.
(Note: If you encounter problems related to `pyFFTW`, make sure that you are (Note: If you encounter problems related to `pyFFTW`, make sure that you are
using a pip-installed `pyFFTW` package. Unfortunately, some distributions are using a pip-installed `pyFFTW` package. Unfortunately, some distributions are
shipping an incorrectly configured `pyFFTW` package, which does not cooperate shipping an incorrectly configured `pyFFTW` package, which does not cooperate
with the installed `FFTW3` libraries.) with the installed `FFTW3` libraries.)
Plotting support is added via:
pip install --user matplotlib
Support for spherical harmonic transforms is added via: Support for spherical harmonic transforms is added via:
pip install --user git+https://gitlab.mpcdf.mpg.de/ift/pyHealpix.git pip3 install --user git+https://gitlab.mpcdf.mpg.de/ift/pyHealpix.git
MPI support is added via: MPI support is added via:
sudo apt-get install openmpi-bin libopenmpi-dev sudo apt-get install python3-mpi4py
pip install --user mpi4py
### Installation for Python 3
If you want to run NIFTy with Python 3, you need to make the following changes
to the instructions above:
- in all `apt-get` commands, replace `python-*` by `python3-*`
- in all `pip` commands, replace `pip` by `pip3`
### Running the tests ### Running the tests
In oder to run the tests one needs two additional packages: To run the tests, additional packages are required:
pip install --user nose parameterized coverage sudo apt-get install python3-pytest-cov
Afterwards the tests (including a coverage report) can be run using the Afterwards the tests (including a coverage report) can be run using the
following command in the repository root: following command in the repository root:
nosetests -x --with-coverage --cover-html --cover-package=nifty5 pytest-3 --cov=nifty5 test
### First Steps ### First Steps
...@@ -108,7 +111,7 @@ For a quick start, you can browse through the [informal ...@@ -108,7 +111,7 @@ For a quick start, you can browse through the [informal
introduction](http://ift.pages.mpcdf.de/NIFTy/code.html) or introduction](http://ift.pages.mpcdf.de/NIFTy/code.html) or
dive into NIFTy by running one of the demonstrations, e.g.: dive into NIFTy by running one of the demonstrations, e.g.:
python demos/wiener_filter_via_curvature.py python3 demos/getting_started_1.py
### Acknowledgement ### Acknowledgement
......
%% Cell type:markdown id: tags: %% Cell type:markdown id: tags:
# A NIFTy demonstration # A NIFTy demonstration
%% Cell type:markdown id: tags: %% Cell type:markdown id: tags:
## IFT: Big Picture ## IFT: Big Picture
IFT starting point: IFT starting point:
$$d = Rs+n$$ $$d = Rs+n$$
Typically, $s$ is a continuous field, $d$ a discrete data vector. Particularly, $R$ is not invertible. Typically, $s$ is a continuous field, $d$ a discrete data vector. Particularly, $R$ is not invertible.
IFT aims at **inverting** the above uninvertible problem in the **best possible way** using Bayesian statistics. IFT aims at **inverting** the above uninvertible problem in the **best possible way** using Bayesian statistics.
## NIFTy ## NIFTy
NIFTy (Numerical Information Field Theory) is a Python framework in which IFT problems can be tackled easily. NIFTy (Numerical Information Field Theory) is a Python framework in which IFT problems can be tackled easily.
Main Interfaces: Main Interfaces:
- **Spaces**: Cartesian, 2-Spheres (Healpix, Gauss-Legendre) and their respective harmonic spaces. - **Spaces**: Cartesian, 2-Spheres (Healpix, Gauss-Legendre) and their respective harmonic spaces.
- **Fields**: Defined on spaces. - **Fields**: Defined on spaces.
- **Operators**: Acting on fields. - **Operators**: Acting on fields.
%% Cell type:markdown id: tags: %% Cell type:markdown id: tags:
## Wiener Filter: Formulae ## Wiener Filter: Formulae
### Assumptions ### Assumptions
- $d=Rs+n$, $R$ linear operator. - $d=Rs+n$, $R$ linear operator.
- $\mathcal P (s) = \mathcal G (s,S)$, $\mathcal P (n) = \mathcal G (n,N)$ where $S, N$ are positive definite matrices. - $\mathcal P (s) = \mathcal G (s,S)$, $\mathcal P (n) = \mathcal G (n,N)$ where $S, N$ are positive definite matrices.
### Posterior ### Posterior
The Posterior is given by: The Posterior is given by:
$$\mathcal P (s|d) \propto P(s,d) = \mathcal G(d-Rs,N) \,\mathcal G(s,S) \propto \mathcal G (m,D) $$ $$\mathcal P (s|d) \propto P(s,d) = \mathcal G(d-Rs,N) \,\mathcal G(s,S) \propto \mathcal G (s-m,D) $$
where where
$$\begin{align} $$\begin{align}
m &= Dj \\ m &= Dj \\
D^{-1}&= (S^{-1} +R^\dagger N^{-1} R )\\ D^{-1}&= (S^{-1} +R^\dagger N^{-1} R )\\
j &= R^\dagger N^{-1} d j &= R^\dagger N^{-1} d
\end{align}$$ \end{align}$$
Let us implement this in NIFTy! Let us implement this in NIFTy!
%% Cell type:markdown id: tags: %% Cell type:markdown id: tags:
## Wiener Filter: Example ## Wiener Filter: Example
- We assume statistical homogeneity and isotropy. Therefore the signal covariance $S$ is diagonal in harmonic space, and is described by a one-dimensional power spectrum, assumed here as $$P(k) = P_0\,\left(1+\left(\frac{k}{k_0}\right)^2\right)^{-\gamma /2},$$ - We assume statistical homogeneity and isotropy. Therefore the signal covariance $S$ is diagonal in harmonic space, and is described by a one-dimensional power spectrum, assumed here as $$P(k) = P_0\,\left(1+\left(\frac{k}{k_0}\right)^2\right)^{-\gamma /2},$$
with $P_0 = 0.2, k_0 = 5, \gamma = 4$. with $P_0 = 0.2, k_0 = 5, \gamma = 4$.
- $N = 0.2 \cdot \mathbb{1}$. - $N = 0.2 \cdot \mathbb{1}$.
- Number of data points $N_{pix} = 512$. - Number of data points $N_{pix} = 512$.
- reconstruction in harmonic space. - reconstruction in harmonic space.
- Response operator: - Response operator:
$$R = FFT_{\text{harmonic} \rightarrow \text{position}}$$ $$R = FFT_{\text{harmonic} \rightarrow \text{position}}$$
%% Cell type:code id: tags: %% Cell type:code id: tags:
``` python ``` python
N_pixels = 512 # Number of pixels N_pixels = 512 # Number of pixels
def pow_spec(k): def pow_spec(k):
P0, k0, gamma = [.2, 5, 4] P0, k0, gamma = [.2, 5, 4]
return P0 / ((1. + (k/k0)**2)**(gamma / 2)) return P0 / ((1. + (k/k0)**2)**(gamma / 2))
``` ```
%% Cell type:markdown id: tags: %% Cell type:markdown id: tags:
## Wiener Filter: Implementation ## Wiener Filter: Implementation
%% Cell type:markdown id: tags: %% Cell type:markdown id: tags:
### Import Modules ### Import Modules
%% Cell type:code id: tags: %% Cell type:code id: tags:
``` python ``` python
import numpy as np import numpy as np
np.random.seed(40) np.random.seed(40)
import nifty5 as ift import nifty5 as ift
import matplotlib.pyplot as plt import matplotlib.pyplot as plt
%matplotlib inline %matplotlib inline
``` ```
%% Cell type:markdown id: tags: %% Cell type:markdown id: tags:
### Implement Propagator ### Implement Propagator
%% Cell type:code id: tags: %% Cell type:code id: tags:
``` python ``` python
def Curvature(R, N, Sh): def Curvature(R, N, Sh):
IC = ift.GradientNormController(iteration_limit=50000, IC = ift.GradientNormController(iteration_limit=50000,
tol_abs_gradnorm=0.1) tol_abs_gradnorm=0.1)
# WienerFilterCurvature is (R.adjoint*N.inverse*R + Sh.inverse) plus some handy # WienerFilterCurvature is (R.adjoint*N.inverse*R + Sh.inverse) plus some handy
# helper methods. # helper methods.
return ift.library.WienerFilterCurvature(R,N,Sh,iteration_controller=IC,iteration_controller_sampling=IC) return ift.WienerFilterCurvature(R,N,Sh,iteration_controller=IC,iteration_controller_sampling=IC)