Commit 25e88936 authored by Philipp Arras's avatar Philipp Arras
Browse files

Merge remote-tracking branch 'dev/NIFTy_5' into NIFTy_5

parents 9e80b9a0 87a53824
Pipeline #43112 failed with stages
in 13 seconds
......@@ -10,6 +10,8 @@ setup.cfg
.document
.svn/
*.csv
.pytest_cache/
*.png
# from https://github.com/github/gitignore/blob/master/Python.gitignore
......
image: $CONTAINER_TEST_IMAGE
variables:
CONTAINER_TEST_IMAGE: gitlab-registry.mpcdf.mpg.de/ift/nifty:$CI_BUILD_REF_NAME
CONTAINER_TEST_IMAGE: gitlab-registry.mpcdf.mpg.de/ift/nifty-dev:$CI_BUILD_REF_NAME
OMP_NUM_THREADS: 1
stages:
......@@ -15,6 +15,8 @@ build_docker_from_scratch:
- schedules
image: docker:stable
stage: build_docker
before_script:
- ls
script:
- docker login -u gitlab-ci-token -p $CI_BUILD_TOKEN gitlab-registry.mpcdf.mpg.de
- docker build -t $CONTAINER_TEST_IMAGE --no-cache .
......@@ -25,177 +27,87 @@ build_docker_from_cache:
- schedules
image: docker:stable
stage: build_docker
before_script:
- ls
script:
- docker login -u gitlab-ci-token -p $CI_BUILD_TOKEN gitlab-registry.mpcdf.mpg.de
- docker build -t $CONTAINER_TEST_IMAGE .
- docker push $CONTAINER_TEST_IMAGE
test_python2_with_coverage:
test_serial:
stage: test
script:
- python setup.py install --user -f
- mpiexec -n 2 --bind-to none nosetests -q 2> /dev/null
- nosetests -q --with-coverage --cover-package=nifty5 --cover-erase
- pytest-3 -q --cov=nifty5 test
- >
coverage report --omit "*plotting*,*distributed_do*"
python3 -m coverage report --omit "*plot*,*distributed_do*" | tee coverage.txt
- >
coverage report --omit "*plotting*,*distributed_do*" | grep TOTAL | awk '{ print "TOTAL: "$4; }'
grep TOTAL coverage.txt | awk '{ print "TOTAL: "$4; }'
test_python3:
test_mpi:
stage: test
variables:
OMPI_MCA_btl_vader_single_copy_mechanism: none
script:
- python3 setup.py install --user -f
- mpiexec -n 2 --bind-to none nosetests3 -q 2> /dev/null
- nosetests3 -q
- mpiexec -n 2 --bind-to none pytest-3 -q test
pages:
stage: release
script:
- python setup.py install --user -f
- sh docs/generate.sh
- mv docs/build/ public/
artifacts:
paths:
- public
only:
- NIFTy_4
before_script:
- export MPLBACKEND="agg"
run_critical_filtering:
stage: demo_runs
script:
- ls
- python setup.py install --user -f
- python3 setup.py install --user -f
- python demos/critical_filtering.py
- python3 demos/critical_filtering.py
artifacts:
paths:
- '*.png'
- NIFTy_5
run_nonlinear_critical_filter:
stage: demo_runs
script:
- python setup.py install --user -f
- python3 setup.py install --user -f
- python demos/nonlinear_critical_filter.py
- python3 demos/nonlinear_critical_filter.py
artifacts:
paths:
- '*.png'
run_nonlinear_wiener_filter:
stage: demo_runs
script:
- python setup.py install --user -f
- python3 setup.py install --user -f
- python demos/nonlinear_wiener_filter.py
- python3 demos/nonlinear_wiener_filter.py
only:
- run_demos
artifacts:
paths:
- '*.png'
# FIXME: disable for now. Fixing it is part of issue #244.
#run_poisson_demo:
# stage: demo_runs
# script:
# - python setup.py install --user -f
# - python3 setup.py install --user -f
# - python demos/poisson_demo.py
# - python3 demos/poisson_demo.py
# artifacts:
# paths:
# - '*.png'
run_probing:
stage: demo_runs
script:
- python setup.py install --user -f
- python3 setup.py install --user -f
- python demos/probing.py
- python3 demos/probing.py
artifacts:
paths:
- '*.png'
run_sampling:
stage: demo_runs
script:
- python setup.py install --user -f
- python3 setup.py install --user -f
- python demos/sampling.py
- python3 demos/sampling.py
artifacts:
paths:
- '*.png'
before_script:
- python3 setup.py install --user -f
run_tomography:
run_ipynb:
stage: demo_runs
script:
- python setup.py install --user -f
- python3 setup.py install --user -f
- python demos/tomography.py
- python3 demos/tomography.py
artifacts:
paths:
- '*.png'
- jupyter nbconvert --execute --ExecutePreprocessor.timeout=None demos/Wiener_Filter.ipynb
run_wiener_filter_data_space_noiseless:
run_getting_started_1:
stage: demo_runs
script:
- python setup.py install --user -f
- python3 setup.py install --user -f
- python demos/wiener_filter_data_space_noiseless.py
- python3 demos/wiener_filter_data_space_noiseless.py
- python3 demos/getting_started_1.py
- mpiexec -n 2 --bind-to none python3 demos/getting_started_1.py 2> /dev/null
artifacts:
paths:
- '*.png'
run_wiener_filter_easy.py:
run_getting_started_2:
stage: demo_runs
script:
- python setup.py install --user -f
- python3 setup.py install --user -f
- python demos/wiener_filter_easy.py
- python3 demos/wiener_filter_easy.py
- python3 demos/getting_started_2.py
- mpiexec -n 2 --bind-to none python3 demos/getting_started_2.py 2> /dev/null
artifacts:
paths:
- '*.png'
run_wiener_filter_via_curvature.py:
run_getting_started_3:
stage: demo_runs
script:
- pip install --user numericalunits
- pip3 install --user numericalunits
- python setup.py install --user -f
- python3 setup.py install --user -f
- python demos/wiener_filter_via_curvature.py
- python3 demos/wiener_filter_via_curvature.py
- python3 demos/getting_started_3.py
artifacts:
paths:
- '*.png'
run_wiener_filter_via_hamiltonian.py:
run_bernoulli:
stage: demo_runs
script:
- python setup.py install --user -f
- python3 setup.py install --user -f
- python demos/wiener_filter_via_hamiltonian.py
- python3 demos/wiener_filter_via_hamiltonian.py
- python3 demos/bernoulli_demo.py
artifacts:
paths:
- '*.png'
run_ipynb:
run_curve_fitting:
stage: demo_runs
script:
- python setup.py install --user -f
- python3 setup.py install --user -f
- jupyter nbconvert --execute --ExecutePreprocessor.timeout=None demos/Wiener_Filter.ipynb
- python3 demos/polynomial_fit.py
artifacts:
paths:
- '*.png'
FROM debian:testing-slim
RUN apt-get update && apt-get install -y \
# Needed for gitlab tests
git \
# Needed for setup
git python3-pip \
# Packages needed for NIFTy
libfftw3-dev \
python python-pip python-dev python-future python-scipy cython \
python3 python3-pip python3-dev python3-future python3-scipy cython3 \
python3-scipy \
# Documentation build dependencies
python-sphinx python-sphinx-rtd-theme python-numpydoc \
python3-sphinx-rtd-theme dvipng texlive-latex-base texlive-latex-extra \
# Testing dependencies
python-nose python-parameterized \
python3-nose python3-parameterized \
python3-pytest-cov jupyter \
# Optional NIFTy dependencies
openmpi-bin libopenmpi-dev python-mpi4py python3-mpi4py \
# Packages needed for NIFTy
&& pip install pyfftw \
libfftw3-dev python3-mpi4py python3-matplotlib \
# more optional NIFTy dependencies
&& pip3 install pyfftw \
# Optional NIFTy dependencies
&& pip install git+https://gitlab.mpcdf.mpg.de/ift/pyHealpix.git \
&& pip3 install git+https://gitlab.mpcdf.mpg.de/ift/pyHealpix.git \
# Testing dependencies
&& pip install coverage \
&& pip3 install jupyter \
&& rm -rf /var/lib/apt/lists/*
# Needed for demos to be running
RUN apt-get update && apt-get install -y python-matplotlib python3-matplotlib \
&& python3 -m pip install --upgrade pip && python3 -m pip install jupyter && python -m pip install --upgrade pip && python -m pip install jupyter \
&& rm -rf /var/lib/apt/lists/*
# Set matplotlib backend
ENV MPLBACKEND agg
# Create user (openmpi does not like to be run as root)
RUN useradd -ms /bin/bash testinguser
......
......@@ -37,11 +37,11 @@ Installation
### Requirements
- [Python](https://www.python.org/) (v2.7.x or 3.5.x)
- [Python 3](https://www.python.org/) (3.5.x or later)
- [SciPy](https://www.scipy.org/)
- [pyFFTW](https://pypi.python.org/pypi/pyFFTW)
Optional dependencies:
- [pyFFTW](https://pypi.python.org/pypi/pyFFTW) for faster Fourier transforms
- [pyHealpix](https://gitlab.mpcdf.mpg.de/ift/pyHealpix) (for harmonic
transforms involving domains on the sphere)
- [mpi4py](https://mpi4py.scipy.org) (for MPI-parallel execution)
......@@ -61,45 +61,48 @@ distributions, the "apt" lines will need slight changes.
NIFTy5 and its mandatory dependencies can be installed via:
sudo apt-get install git libfftw3-dev python python-pip python-dev
pip install --user git+https://gitlab.mpcdf.mpg.de/ift/NIFTy.git@NIFTy_5
sudo apt-get install git python3 python3-pip python3-dev
pip3 install --user git+https://gitlab.mpcdf.mpg.de/ift/NIFTy.git@NIFTy_5
Plotting support is added via:
sudo apt-get install python3-matplotlib
NIFTy uses Numpy's FFT implementation by default. For large problems FFTW may be
used because of its higher performance. It can be installed via:
sudo apt-get install libfftw3-dev
pip3 install --user pyfftw
To enable FFTW usage in NIFTy, call
nifty5.fft.enable_fftw()
at the beginning of your code.
(Note: If you encounter problems related to `pyFFTW`, make sure that you are
using a pip-installed `pyFFTW` package. Unfortunately, some distributions are
shipping an incorrectly configured `pyFFTW` package, which does not cooperate
with the installed `FFTW3` libraries.)
Plotting support is added via:
pip install --user matplotlib
Support for spherical harmonic transforms is added via:
pip install --user git+https://gitlab.mpcdf.mpg.de/ift/pyHealpix.git
pip3 install --user git+https://gitlab.mpcdf.mpg.de/ift/pyHealpix.git
MPI support is added via:
sudo apt-get install openmpi-bin libopenmpi-dev
pip install --user mpi4py
### Installation for Python 3
If you want to run NIFTy with Python 3, you need to make the following changes
to the instructions above:
- in all `apt-get` commands, replace `python-*` by `python3-*`
- in all `pip` commands, replace `pip` by `pip3`
sudo apt-get install python3-mpi4py
### Running the tests
In oder to run the tests one needs two additional packages:
To run the tests, additional packages are required:
pip install --user nose parameterized coverage
sudo apt-get install python3-pytest-cov
Afterwards the tests (including a coverage report) can be run using the
following command in the repository root:
nosetests -x --with-coverage --cover-html --cover-package=nifty5
pytest-3 --cov=nifty5 test
### First Steps
......@@ -108,7 +111,7 @@ For a quick start, you can browse through the [informal
introduction](http://ift.pages.mpcdf.de/NIFTy/code.html) or
dive into NIFTy by running one of the demonstrations, e.g.:
python demos/wiener_filter_via_curvature.py
python3 demos/getting_started_1.py
### Acknowledgement
......
......@@ -58,7 +58,7 @@
"### Posterior\n",
"The Posterior is given by:\n",
"\n",
"$$\\mathcal P (s|d) \\propto P(s,d) = \\mathcal G(d-Rs,N) \\,\\mathcal G(s,S) \\propto \\mathcal G (m,D) $$\n",
"$$\\mathcal P (s|d) \\propto P(s,d) = \\mathcal G(d-Rs,N) \\,\\mathcal G(s,S) \\propto \\mathcal G (s-m,D) $$\n",
"\n",
"where\n",
"$$\\begin{align}\n",
......@@ -172,7 +172,7 @@
" tol_abs_gradnorm=0.1)\n",
" # WienerFilterCurvature is (R.adjoint*N.inverse*R + Sh.inverse) plus some handy\n",
" # helper methods.\n",
" return ift.library.WienerFilterCurvature(R,N,Sh,iteration_controller=IC,iteration_controller_sampling=IC)"
" return ift.WienerFilterCurvature(R,N,Sh,iteration_controller=IC,iteration_controller_sampling=IC)"
]
},
{
......@@ -429,8 +429,8 @@
"mask[l:h] = 0\n",
"mask = ift.Field.from_global_data(s_space, mask)\n",
"\n",
"R = ift.DiagonalOperator(mask)*HT\n",
"n = n.to_global_data()\n",
"R = ift.DiagonalOperator(mask)(HT)\n",
"n = n.to_global_data_rw()\n",
"n[l:h] = 0\n",
"n = ift.Field.from_global_data(s_space, n)\n",
"\n",
......@@ -501,7 +501,7 @@
"m_data = HT(m).to_global_data()\n",
"m_var_data = m_var.to_global_data()\n",
"uncertainty = np.sqrt(m_var_data)\n",
"d_data = d.to_global_data()\n",
"d_data = d.to_global_data_rw()\n",
"\n",
"# Set lost data to NaN for proper plotting\n",
"d_data[d_data == 0] = np.nan"
......@@ -585,8 +585,8 @@
"mask[l:h,l:h] = 0.\n",
"mask = ift.Field.from_global_data(s_space, mask)\n",
"\n",
"R = ift.DiagonalOperator(mask)*HT\n",
"n = n.to_global_data()\n",
"R = ift.DiagonalOperator(mask)(HT)\n",
"n = n.to_global_data_rw()\n",
"n[l:h, l:h] = 0\n",
"n = ift.Field.from_global_data(s_space, n)\n",
"curv = Curvature(R=R, N=N, Sh=Sh)\n",
......@@ -717,21 +717,21 @@
"metadata": {
"celltoolbar": "Slideshow",
"kernelspec": {
"display_name": "Python 2",
"display_name": "Python 3",
"language": "python",
"name": "python2"
"name": "python3"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 2
"version": 3
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython2",
"version": "2.7.15"
"pygments_lexer": "ipython3",
"version": "3.6.6"
}
},
"nbformat": 4,
......
%% Cell type:markdown id: tags:
# A NIFTy demonstration
%% Cell type:markdown id: tags:
## IFT: Big Picture
IFT starting point:
$$d = Rs+n$$
Typically, $s$ is a continuous field, $d$ a discrete data vector. Particularly, $R$ is not invertible.
IFT aims at **inverting** the above uninvertible problem in the **best possible way** using Bayesian statistics.
## NIFTy
NIFTy (Numerical Information Field Theory) is a Python framework in which IFT problems can be tackled easily.
Main Interfaces:
- **Spaces**: Cartesian, 2-Spheres (Healpix, Gauss-Legendre) and their respective harmonic spaces.
- **Fields**: Defined on spaces.
- **Operators**: Acting on fields.
%% Cell type:markdown id: tags:
## Wiener Filter: Formulae
### Assumptions
- $d=Rs+n$, $R$ linear operator.
- $\mathcal P (s) = \mathcal G (s,S)$, $\mathcal P (n) = \mathcal G (n,N)$ where $S, N$ are positive definite matrices.
### Posterior
The Posterior is given by:
$$\mathcal P (s|d) \propto P(s,d) = \mathcal G(d-Rs,N) \,\mathcal G(s,S) \propto \mathcal G (m,D) $$
$$\mathcal P (s|d) \propto P(s,d) = \mathcal G(d-Rs,N) \,\mathcal G(s,S) \propto \mathcal G (s-m,D) $$
where
$$\begin{align}
m &= Dj \\
D^{-1}&= (S^{-1} +R^\dagger N^{-1} R )\\
j &= R^\dagger N^{-1} d
\end{align}$$
Let us implement this in NIFTy!
%% Cell type:markdown id: tags:
## Wiener Filter: Example
- We assume statistical homogeneity and isotropy. Therefore the signal covariance $S$ is diagonal in harmonic space, and is described by a one-dimensional power spectrum, assumed here as $$P(k) = P_0\,\left(1+\left(\frac{k}{k_0}\right)^2\right)^{-\gamma /2},$$
with $P_0 = 0.2, k_0 = 5, \gamma = 4$.
- $N = 0.2 \cdot \mathbb{1}$.
- Number of data points $N_{pix} = 512$.
- reconstruction in harmonic space.
- Response operator:
$$R = FFT_{\text{harmonic} \rightarrow \text{position}}$$
%% Cell type:code id: tags:
``` python
N_pixels = 512 # Number of pixels
def pow_spec(k):
P0, k0, gamma = [.2, 5, 4]
return P0 / ((1. + (k/k0)**2)**(gamma / 2))
```
%% Cell type:markdown id: tags:
## Wiener Filter: Implementation
%% Cell type:markdown id: tags:
### Import Modules
%% Cell type:code id: tags:
``` python
import numpy as np
np.random.seed(40)
import nifty5 as ift
import matplotlib.pyplot as plt
%matplotlib inline
```
%% Cell type:markdown id: tags:
### Implement Propagator
%% Cell type:code id: tags:
``` python
def Curvature(R, N, Sh):
IC = ift.GradientNormController(iteration_limit=50000,
tol_abs_gradnorm=0.1)
# WienerFilterCurvature is (R.adjoint*N.inverse*R + Sh.inverse) plus some handy
# helper methods.
return ift.library.WienerFilterCurvature(R,N,Sh,iteration_controller=IC,iteration_controller_sampling=IC)
return ift.WienerFilterCurvature(R,N,Sh,iteration_controller=IC,iteration_controller_sampling=IC)
```
%% Cell type:markdown id: tags:
### Conjugate Gradient Preconditioning
- $D$ is defined via:
$$D^{-1} = \mathcal S_h^{-1} + R^\dagger N^{-1} R.$$
In the end, we want to apply $D$ to $j$, i.e. we need the inverse action of $D^{-1}$. This is done numerically (algorithm: *Conjugate Gradient*).
<!--
- One can define the *condition number* of a non-singular and normal matrix $A$:
$$\kappa (A) := \frac{|\lambda_{\text{max}}|}{|\lambda_{\text{min}}|},$$
where $\lambda_{\text{max}}$ and $\lambda_{\text{min}}$ are the largest and smallest eigenvalue of $A$, respectively.
- The larger $\kappa$ the slower Conjugate Gradient.
- By default, conjugate gradient solves: $D^{-1} m = j$ for $m$, where $D^{-1}$ can be badly conditioned. If one knows a non-singular matrix $T$ for which $TD^{-1}$ is better conditioned, one can solve the equivalent problem:
$$\tilde A m = \tilde j,$$
where $\tilde A = T D^{-1}$ and $\tilde j = Tj$.
- In our case $S^{-1}$ is responsible for the bad conditioning of $D$ depending on the chosen power spectrum. Thus, we choose
$$T = \mathcal F^\dagger S_h^{-1} \mathcal F.$$
-->
%% Cell type:markdown id: tags:
### Generate Mock data
- Generate a field $s$ and $n$ with given covariances.
- Calculate $d$.
%% Cell type:code id: tags:
``` python
s_space = ift.RGSpace(N_pixels)
h_space = s_space.get_default_codomain()
HT = ift.HarmonicTransformOperator(h_space, target=s_space)
# Operators
Sh = ift.create_power_operator(h_space, power_spectrum=pow_spec)
R = HT #*ift.create_harmonic_smoothing_operator((h_space,), 0, 0.02)
# Fields and data
sh = Sh.draw_sample()
noiseless_data=R(sh)
noise_amplitude = np.sqrt(0.2)
N = ift.ScalingOperator(noise_amplitude**2, s_space)
n = ift.Field.from_random(domain=s_space, random_type='normal',
std=noise_amplitude, mean=0)
d = noiseless_data + n
j = R.adjoint_times(N.inverse_times(d))
curv = Curvature(R=R, N=N, Sh=Sh)
D = curv.inverse
```
%% Cell type:markdown id: tags:
### Run Wiener Filter
%% Cell type:code id: tags:
``` python
m = D(j)
```
%% Cell type:markdown id: tags:
### Signal Reconstruction
%% Cell type:code id: tags:
``` python
# Get signal data and reconstruction data
s_data = HT(sh).to_global_data()
m_data = HT(m).to_global_data()
d_data = d.to_global_data()
plt.figure(figsize=(15,10))
plt.plot(s_data, 'r', label="Signal", linewidth=3)
plt.plot(d_data, 'k.', label="Data")
plt.plot(m_data, 'k', label="Reconstruction",linewidth=3)
plt.title("Reconstruction")
plt.legend()
plt.show()
```
%% Cell type:code id: tags:
``` python
plt.figure(figsize=(15,10))
plt.plot(s_data - s_data, 'r', label="Signal", linewidth=3)
plt.plot(d_data - s_data, 'k.', label="Data")
plt.plot(m_data - s_data, 'k', label="Reconstruction",linewidth=3)
plt.axhspan(-noise_amplitude,noise_amplitude, facecolor='0.9', alpha=.5)
plt.title("Residuals")
plt.legend()
plt.show()
```
%% Cell type:markdown id: tags:
### Power Spectrum
%% Cell type:code id: tags:
``` python
s_power_data = ift.power_analyze(sh).to_global_data()
m_power_data = ift.power_analyze(m).to_global_data()
plt.figure(figsize=(15,10))
plt.loglog()
plt.xlim(1, int(N_pixels/2))
ymin = min(m_power_data)
plt.ylim(ymin, 1)
xs = np.arange(1,int(N_pixels/2),.1)
plt.plot(xs, pow_spec(xs), label="True Power Spectrum", color='k',alpha=0.5)
plt.plot(s_power_data, 'r', label="Signal")
plt.plot(m_power_data, 'k', label="Reconstruction")
plt.axhline(noise_amplitude**2 / N_pixels, color="k", linestyle='--', label="Noise level", alpha=.5)
plt.axhspan(noise_amplitude**2 / N_pixels, ymin, facecolor='0.9', alpha=.5)
plt.title("Power Spectrum")
plt.legend()
plt.show()
```
%% Cell type:markdown id: tags:
## Wiener Filter on Incomplete Data
%% Cell type:code id: tags:
``` python
# Operators
Sh = ift.create_power_operator(h_space, power_spectrum=pow_spec)
N = ift.ScalingOperator(noise_amplitude**2,s_space)
# R is defined below
# Fields
sh = Sh.draw_sample()
s = HT(sh)
n = ift.Field.from_random(domain=s_space, random_type='normal',
std=noise_amplitude, mean=0)
```
%% Cell type:markdown id: tags:
### Partially Lose Data
%% Cell type:code id: tags:
``` python
l = int(N_pixels * 0.2)
h = int(N_pixels * 0.2 * 2)
mask = np.full(s_space.shape, 1.)
mask[l:h] = 0
mask = ift.Field.from_global_data(s_space, mask)
R = ift.DiagonalOperator(mask)*HT
n = n.to_global_data()
R = ift.DiagonalOperator(mask)(HT)
n = n.to_global_data_rw()
n[l:h] = 0
n = ift.Field.from_global_data(s_space, n)
d = R(sh) + n
```
%% Cell type:code id: tags:
``` python
curv = Curvature(R=R, N=N, Sh=Sh)
D = curv.inverse
j = R.adjoint_times(N.inverse_times(d))
m = D(j)
```
%% Cell type:markdown id: tags:
### Compute Uncertainty
%% Cell type:code id: tags:
``` python
m_mean, m_var = ift.probe_with_posterior_samples(curv, HT, 200)
```
%% Cell type:markdown id: tags:
### Get data
%% Cell type:code id: tags:
``` python
# Get signal data and reconstruction data
s_data = s.to_global_data()
m_data = HT(m).to_global_data()
m_var_data = m_var.to_global_data()
uncertainty = np.sqrt(m_var_data)
d_data = d.to_global_data()
d_data = d.to_global_data_rw()
# Set lost data to NaN for proper plotting
d_data[d_data == 0] = np.nan
```
%% Cell type:code id: tags:
``` python
fig = plt.figure(figsize=(15,10))
plt.axvspan(l, h, facecolor='0.8',alpha=0.5)
plt.fill_between(range(N_pixels), m_data - uncertainty, m_data + uncertainty, facecolor='0.5', alpha=0.5)
plt.plot(s_data, 'r', label="Signal", alpha=1, linewidth=3)
plt.plot(d_data, 'k.', label="Data")
plt.plot(m_data, 'k', label="Reconstruction", linewidth=3)
plt.title("Reconstruction of incomplete data")
plt.legend()
```
%% Cell type:markdown id: tags:
# 2d Example
%% Cell type:code id: tags:
``` python
N_pixels = 256 # Number of pixels
sigma2 = 2. # Noise variance
def pow_spec(k):
P0, k0, gamma = [.2, 2, 4]
return P0 * (1. + (k/k0)**2)**(-gamma/2)
s_space = ift.RGSpace([N_pixels, N_pixels])
```
%% Cell type:code id: tags:
``` python
h_space = s_space.get_default_codomain()
HT = ift.HarmonicTransformOperator(h_space,s_space)
# Operators
Sh = ift.create_power_operator(h_space, power_spectrum=pow_spec)
N = ift.ScalingOperator(sigma2,s_space)
# Fields and data
sh = Sh.draw_sample()
n = ift.Field.from_random(domain=s_space, random_type='normal',
std=np.sqrt(sigma2), mean=0)
# Lose some data
l = int(N_pixels * 0.33)
h = int(N_pixels * 0.33 * 2)
mask = np.full(s_space.shape, 1.)
mask[l:h,l:h] = 0.
mask = ift.Field.from_global_data(s_space, mask)
R = ift.DiagonalOperator(mask)*HT
n = n.to_global_data()
R = ift.DiagonalOperator(mask)(HT)
n = n.to_global_data_rw()
n[l:h, l:h] = 0
n = ift.Field.from_global_data(s_space, n)
curv = Curvature(R=R, N=N, Sh=Sh)
D = curv.inverse
d = R(sh) + n
j = R.adjoint_times(N.inverse_times(d))
# Run Wiener filter
m = D(j)
# Uncertainty
m_mean, m_var = ift.probe_with_posterior_samples(curv, HT, 20)
# Get data
s_data = HT(sh).to_global_data()
m_data = HT(m).to_global_data()
m_var_data = m_var.to_global_data()
d_data = d.to_global_data()
uncertainty = np.sqrt(np.abs(m_var_data))
```
%% Cell type:code id: tags:
``` python
cm = ['magma', 'inferno', 'plasma', 'viridis'][1]
mi = np.min(s_data)
ma = np.max(s_data)
fig, axes = plt.subplots(1, 2, figsize=(15, 7))
data = [s_data, d_data]
caption = ["Signal", "Data"]
for ax in axes.flat:
im = ax.imshow(data.pop(0), interpolation='nearest', cmap=cm, vmin=mi,
vmax=ma)
ax.set_title(caption.pop(0))
fig.subplots_adjust(right=0.8)
cbar_ax = fig.add_axes([0.85, 0.15, 0.05, 0.7])
fig.colorbar(im, cax=cbar_ax)
```
%% Cell type:code id: tags:
``` python
mi = np.min(s_data)
ma = np.max(s_data)
fig, axes = plt.subplots(3, 2, figsize=(15, 22.5))
sample = HT(curv.draw_sample(from_inverse=True)+m).to_global_data()
post_mean = (m_mean + HT(m)).to_global_data()
data = [s_data, m_data, post_mean, sample, s_data - m_data, uncertainty]
caption = ["Signal", "Reconstruction", "Posterior mean", "Sample", "Residuals", "Uncertainty Map"]
for ax in axes.flat:
im = ax.imshow(data.pop(0), interpolation='nearest', cmap=cm, vmin=mi, vmax=ma)
ax.set_title(caption.pop(0))
fig.subplots_adjust(right=0.8)
cbar_ax = fig.add_axes([.85, 0.15, 0.05, 0.7])
fig.colorbar(im, cax=cbar_ax)
```
%% Cell type:markdown id: tags:
### Is the uncertainty map reliable?
%% Cell type:code id: tags:
``` python
precise = (np.abs(s_data-m_data) < uncertainty)
print("Error within uncertainty map bounds: " + str(np.sum(precise) * 100 / N_pixels**2) + "%")
plt.figure(figsize=(15,10))
plt.imshow(precise.astype(float), cmap="brg")
plt.colorbar()
```
%% Cell type:markdown id: tags:
# Start Coding
## NIFTy Repository + Installation guide
https://gitlab.mpcdf.mpg.de/ift/NIFTy
NIFTy v5 **more or less stable!**
......
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
#
# Copyright(C) 2013-2019 Max-Planck-Society
#
# NIFTy is being developed at the Max-Planck-Institut fuer Astrophysik.
#####################################################################
# Bernoulli reconstruction
# Reconstruct an event probability field with values between 0 and 1
# from the observed events
# 1D (set mode=0), 2D (mode=1), or on the sphere (mode=2)
#####################################################################
import numpy as np
import nifty5 as ift
if __name__ == '__main__':
np.random.seed(41)
# Set up the position space of the signal
mode = 2
if mode == 0:
# One-dimensional regular grid
position_space = ift.RGSpace(1024)
elif mode == 1:
# Two-dimensional regular grid
position_space = ift.RGSpace([512, 512])
else:
# Sphere
position_space = ift.HPSpace(128)
# Define harmonic space and transform
harmonic_space = position_space.get_default_codomain()
HT = ift.HarmonicTransformOperator(harmonic_space, position_space)