...
 
Commits (1769)
......@@ -2,6 +2,7 @@
git_version.py
# custom
*.txt
setup.cfg
.idea
.DS_Store
......@@ -10,6 +11,8 @@ setup.cfg
.document
.svn/
*.csv
.pytest_cache/
*.png
# from https://github.com/github/gitignore/blob/master/Python.gitignore
......
image: $CONTAINER_TEST_IMAGE
variables:
CONTAINER_TEST_IMAGE: gitlab-registry.mpcdf.mpg.de/ift/nifty:$CI_BUILD_REF_NAME
CONTAINER_TEST_IMAGE: gitlab-registry.mpcdf.mpg.de/$CI_PROJECT_PATH:$CI_BUILD_REF_NAME
OMP_NUM_THREADS: 1
stages:
- build_docker
- test
- release
- demo_runs
build_docker_from_scratch:
only:
- schedules
image: docker:stable
stage: build_docker
before_script:
- ls
script:
- docker login -u gitlab-ci-token -p $CI_BUILD_TOKEN gitlab-registry.mpcdf.mpg.de
- docker build -t $CONTAINER_TEST_IMAGE --no-cache .
......@@ -24,37 +27,87 @@ build_docker_from_cache:
- schedules
image: docker:stable
stage: build_docker
before_script:
- ls
script:
- docker login -u gitlab-ci-token -p $CI_BUILD_TOKEN gitlab-registry.mpcdf.mpg.de
- docker build -t $CONTAINER_TEST_IMAGE .
- docker push $CONTAINER_TEST_IMAGE
test_python2_with_coverage:
test_serial:
stage: test
script:
- python setup.py install --user -f
- mpiexec -n 2 --bind-to none nosetests -q 2> /dev/null
- nosetests -q --with-coverage --cover-package=nifty4 --cover-erase
- pytest-3 -q --cov=nifty5 test
- >
coverage report --omit "*plotting*,*distributed_do*"
python3 -m coverage report --omit "*plot*,*distributed_do*" | tee coverage.txt
- >
coverage report --omit "*plotting*,*distributed_do*" | grep TOTAL | awk '{ print "TOTAL: "$4; }'
grep TOTAL coverage.txt | awk '{ print "TOTAL: "$4; }'
test_python3:
test_mpi:
stage: test
variables:
OMPI_MCA_btl_vader_single_copy_mechanism: none
script:
- python3 setup.py install --user -f
- mpiexec -n 2 --bind-to none nosetests3 -q 2> /dev/null
- nosetests3 -q
- mpiexec -n 2 --bind-to none pytest-3 -q test
pages:
stage: release
script:
- python setup.py install --user -f
- sh docs/generate.sh
- mv docs/build/ public/
artifacts:
paths:
- public
only:
- NIFTy_4
- NIFTy_5
before_script:
- python3 setup.py install --user -f
run_ipynb:
stage: demo_runs
script:
- jupyter nbconvert --execute --ExecutePreprocessor.timeout=None demos/Wiener_Filter.ipynb
run_getting_started_1:
stage: demo_runs
script:
- python3 demos/getting_started_1.py
- mpiexec -n 2 --bind-to none python3 demos/getting_started_1.py 2> /dev/null
artifacts:
paths:
- '*.png'
run_getting_started_2:
stage: demo_runs
script:
- python3 demos/getting_started_2.py
- mpiexec -n 2 --bind-to none python3 demos/getting_started_2.py 2> /dev/null
artifacts:
paths:
- '*.png'
run_getting_started_3:
stage: demo_runs
script:
- python3 demos/getting_started_3.py
artifacts:
paths:
- '*.png'
run_bernoulli:
stage: demo_runs
script:
- python3 demos/bernoulli_demo.py
artifacts:
paths:
- '*.png'
run_curve_fitting:
stage: demo_runs
script:
- python3 demos/polynomial_fit.py
artifacts:
paths:
- '*.png'
FROM debian:testing-slim
RUN apt-get update && apt-get install -y \
# Needed for gitlab tests
git \
# Needed for setup
git python3-pip \
# Packages needed for NIFTy
libfftw3-dev \
python python-pip python-dev python-future python-scipy \
python3 python3-pip python3-dev python3-future python3-scipy \
python3-scipy \
# Documentation build dependencies
python-sphinx python-sphinx-rtd-theme python-numpydoc \
python3-sphinx-rtd-theme dvipng texlive-latex-base texlive-latex-extra \
# Testing dependencies
python-nose python-parameterized \
python3-nose python3-parameterized \
python3-pytest-cov jupyter \
# Optional NIFTy dependencies
openmpi-bin libopenmpi-dev python-mpi4py python3-mpi4py \
# Packages needed for NIFTy
&& pip install pyfftw \
&& pip3 install pyfftw \
# Optional NIFTy dependencies
&& pip install git+https://gitlab.mpcdf.mpg.de/ift/pyHealpix.git \
python3-mpi4py python3-matplotlib \
# more optional NIFTy dependencies
&& pip3 install git+https://gitlab.mpcdf.mpg.de/ift/pyHealpix.git \
# Testing dependencies
&& pip install coverage \
&& pip3 install git+https://gitlab.mpcdf.mpg.de/ift/nifty_gridder.git \
&& pip3 install git+https://gitlab.mpcdf.mpg.de/mtr/pypocketfft.git \
&& pip3 install jupyter \
&& rm -rf /var/lib/apt/lists/*
# Set matplotlib backend
ENV MPLBACKEND agg
# Create user (openmpi does not like to be run as root)
RUN useradd -ms /bin/bash testinguser
USER testinguser
......
NIFTy - Numerical Information Field Theory
==========================================
[![build status](https://gitlab.mpcdf.mpg.de/ift/NIFTy/badges/NIFTy_4/build.svg)](https://gitlab.mpcdf.mpg.de/ift/NIFTy/commits/NIFTy_4)
[![coverage report](https://gitlab.mpcdf.mpg.de/ift/NIFTy/badges/NIFTy_4/coverage.svg)](https://gitlab.mpcdf.mpg.de/ift/NIFTy/commits/NIFTy_4)
[![build status](https://gitlab.mpcdf.mpg.de/ift/NIFTy/badges/NIFTy_5/build.svg)](https://gitlab.mpcdf.mpg.de/ift/NIFTy/commits/NIFTy_5)
[![coverage report](https://gitlab.mpcdf.mpg.de/ift/NIFTy/badges/NIFTy_5/coverage.svg)](https://gitlab.mpcdf.mpg.de/ift/NIFTy/commits/NIFTy_5)
**NIFTy** project homepage:
[http://ift.pages.mpcdf.de/NIFTy](http://ift.pages.mpcdf.de/NIFTy)
[http://ift.pages.mpcdf.de/nifty](http://ift.pages.mpcdf.de/nifty)
Summary
-------
......@@ -13,23 +13,31 @@ Summary
**NIFTy**, "**N**umerical **I**nformation **F**ield **T**heor<strong>y</strong>", is
a versatile library designed to enable the development of signal
inference algorithms that operate regardless of the underlying spatial
grid and its resolution. Its object-oriented framework is written in
Python, although it accesses libraries written in C++ and C for
efficiency.
inference algorithms that operate regardless of the underlying grids
(spatial, spectral, temporal, …) and their resolutions.
Its object-oriented framework is written in Python, although it accesses
libraries written in C++ and C for efficiency.
NIFTy offers a toolkit that abstracts discretized representations of
continuous spaces, fields in these spaces, and operators acting on
fields into classes. The correct normalization of operations on
fields is taken care of automatically without concerning the user. This
allows for an abstract formulation and programming of inference
these fields into classes.
This allows for an abstract formulation and programming of inference
algorithms, including those derived within information field theory.
Thus, NIFTy permits its user to rapidly prototype algorithms in 1D, and
then apply the developed code in higher-dimensional settings of real
world problems. The set of spaces on which NIFTy operates comprises
point sets, *n*-dimensional regular grids, spherical spaces, their
harmonic counterparts, and product spaces constructed as combinations of
those.
NIFTy's interface is designed to resemble IFT formulae in the sense
that the user implements algorithms in NIFTy independent of the topology
of the underlying spaces and the discretization scheme.
Thus, the user can develop algorithms on subsets of problems and on
spaces where the detailed performance of the algorithm can be properly
evaluated and then easily generalize them to other, more complex spaces
and the full problem, respectively.
The set of spaces on which NIFTy operates comprises point sets,
*n*-dimensional regular grids, spherical spaces, their harmonic
counterparts, and product spaces constructed as combinations of those.
NIFTy takes care of numerical subtleties like the normalization of
operations on fields and the numerical representation of model
components, allowing the user to focus on formulating the abstract
inference procedures and process-specific model properties.
Installation
......@@ -37,81 +45,74 @@ Installation
### Requirements
- [Python](https://www.python.org/) (v2.7.x or 3.5.x)
- [Python 3](https://www.python.org/) (3.5.x or later)
- [SciPy](https://www.scipy.org/)
- [pyFFTW](https://pypi.python.org/pypi/pyFFTW)
- [pypocketfft](https://gitlab.mpcdf.mpg.de/mtr/pypocketfft)
Optional dependencies:
- [pyHealpix](https://gitlab.mpcdf.mpg.de/ift/pyHealpix) (for harmonic
transforms involving domains on the sphere)
- [nifty_gridder](https://gitlab.mpcdf.mpg.de/ift/nifty_gridder) (for radio
interferometry responses)
- [mpi4py](https://mpi4py.scipy.org) (for MPI-parallel execution)
- [matplotlib](https://matplotlib.org/) (for field plotting)
### Sources
The current version of Nifty4 can be obtained by cloning the repository and
switching to the NIFTy_4 branch:
The current version of Nifty5 can be obtained by cloning the repository and
switching to the NIFTy_5 branch:
git clone https://gitlab.mpcdf.mpg.de/ift/NIFTy.git
git clone https://gitlab.mpcdf.mpg.de/ift/nifty.git
### Installation
In the following, we assume a Debian-based distribution. For other
distributions, the "apt" lines will need slight changes.
NIFTy4 and its mandatory dependencies can be installed via:
NIFTy5 and its mandatory dependencies can be installed via:
sudo apt-get install git libfftw3-dev python python-pip python-dev
pip install --user git+https://gitlab.mpcdf.mpg.de/ift/NIFTy.git@NIFTy_4
(Note: If you encounter problems related to `pyFFTW`, make sure that you are
using a pip-installed `pyFFTW` package. Unfortunately, some distributions are
shipping an incorrectly configured `pyFFTW` package, which does not cooperate
with the installed `FFTW3` libraries.)
sudo apt-get install git python3 python3-pip python3-dev
pip3 install --user git+https://gitlab.mpcdf.mpg.de/ift/nifty.git@NIFTy_5
pip3 install --user git+https://gitlab.mpcdf.mpg.de/mtr/pypocketfft
Plotting support is added via:
pip install --user matplotlib
sudo apt-get install python3-matplotlib
Support for spherical harmonic transforms is added via:
pip install --user git+https://gitlab.mpcdf.mpg.de/ift/pyHealpix.git
MPI support is added via:
pip3 install --user git+https://gitlab.mpcdf.mpg.de/ift/pyHealpix.git
sudo apt-get install openmpi-bin libopenmpi-dev
pip install --user mpi4py
Support for the radio interferometry gridder is added via:
### Installation for Python 3
pip3 install --user git+https://gitlab.mpcdf.mpg.de/ift/nifty_gridder.git
If you want to run NIFTy with Python 3, you need to make the following changes
to the instructions above:
MPI support is added via:
- in all `apt-get` commands, replace `python-*` by `python3-*`
- in all `pip` commands, replace `pip` by `pip3`
sudo apt-get install python3-mpi4py
### Running the tests
In oder to run the tests one needs two additional packages:
To run the tests, additional packages are required:
pip install --user nose parameterized coverage
sudo apt-get install python3-pytest-cov
Afterwards the tests (including a coverage report) can be run using the
following command in the repository root:
nosetests -x --with-coverage --cover-html --cover-package=nifty4
pytest-3 --cov=nifty5 test
### First Steps
For a quick start, you can browse through the [informal
introduction](http://ift.pages.mpcdf.de/NIFTy/code.html) or
introduction](http://ift.pages.mpcdf.de/nifty/code.html) or
dive into NIFTy by running one of the demonstrations, e.g.:
python demos/wiener_filter_via_curvature.py
python3 demos/getting_started_1.py
### Acknowledgement
### Acknowledgements
Please acknowledge the use of NIFTy in your publication(s) by using a
phrase such as the following:
......@@ -119,10 +120,10 @@ phrase such as the following:
> "Some of the results in this publication have been derived using the
> NIFTy package [(https://gitlab.mpcdf.mpg.de/ift/NIFTy)](https://gitlab.mpcdf.mpg.de/ift/NIFTy)"
and a citation to one of the [publications](http://ift.pages.mpcdf.de/NIFTy/citations.html).
and a citation to one of the [publications](http://ift.pages.mpcdf.de/nifty/citations.html).
### Release Notes
### Licensing terms
The NIFTy package is licensed under the terms of the
[GPLv3](https://www.gnu.org/licenses/gpl.html) and is distributed
......
......@@ -58,7 +58,7 @@
"### Posterior\n",
"The Posterior is given by:\n",
"\n",
"$$\\mathcal P (s|d) \\propto P(s,d) = \\mathcal G(d-Rs,N) \\,\\mathcal G(s,S) \\propto \\mathcal G (m,D) $$\n",
"$$\\mathcal P (s|d) \\propto P(s,d) = \\mathcal G(d-Rs,N) \\,\\mathcal G(s,S) \\propto \\mathcal G (s-m,D) $$\n",
"\n",
"where\n",
"$$\\begin{align}\n",
......@@ -132,6 +132,7 @@
"cell_type": "code",
"execution_count": null,
"metadata": {
"scrolled": true,
"slideshow": {
"slide_type": "-"
}
......@@ -140,7 +141,7 @@
"source": [
"import numpy as np\n",
"np.random.seed(40)\n",
"import nifty4 as ift\n",
"import nifty5 as ift\n",
"import matplotlib.pyplot as plt\n",
"%matplotlib inline"
]
......@@ -169,10 +170,9 @@
"def Curvature(R, N, Sh):\n",
" IC = ift.GradientNormController(iteration_limit=50000,\n",
" tol_abs_gradnorm=0.1)\n",
" inverter = ift.ConjugateGradient(controller=IC)\n",
" # WienerFilterCurvature is (R.adjoint*N.inverse*R + Sh.inverse) plus some handy\n",
" # helper methods.\n",
" return ift.library.WienerFilterCurvature(R,N,Sh,inverter)"
" return ift.WienerFilterCurvature(R,N,Sh,iteration_controller=IC,iteration_controller_sampling=IC)"
]
},
{
......@@ -223,7 +223,9 @@
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"metadata": {
"scrolled": true
},
"outputs": [],
"source": [
"s_space = ift.RGSpace(N_pixels)\n",
......@@ -427,8 +429,8 @@
"mask[l:h] = 0\n",
"mask = ift.Field.from_global_data(s_space, mask)\n",
"\n",
"R = ift.DiagonalOperator(mask)*HT\n",
"n = n.to_global_data()\n",
"R = ift.DiagonalOperator(mask)(HT)\n",
"n = n.to_global_data_rw()\n",
"n[l:h] = 0\n",
"n = ift.Field.from_global_data(s_space, n)\n",
"\n",
......@@ -499,7 +501,7 @@
"m_data = HT(m).to_global_data()\n",
"m_var_data = m_var.to_global_data()\n",
"uncertainty = np.sqrt(m_var_data)\n",
"d_data = d.to_global_data()\n",
"d_data = d.to_global_data_rw()\n",
"\n",
"# Set lost data to NaN for proper plotting\n",
"d_data[d_data == 0] = np.nan"
......@@ -583,8 +585,8 @@
"mask[l:h,l:h] = 0.\n",
"mask = ift.Field.from_global_data(s_space, mask)\n",
"\n",
"R = ift.DiagonalOperator(mask)*HT\n",
"n = n.to_global_data()\n",
"R = ift.DiagonalOperator(mask)(HT)\n",
"n = n.to_global_data_rw()\n",
"n[l:h, l:h] = 0\n",
"n = ift.Field.from_global_data(s_space, n)\n",
"curv = Curvature(R=R, N=N, Sh=Sh)\n",
......@@ -708,28 +710,28 @@
"\n",
"https://gitlab.mpcdf.mpg.de/ift/NIFTy\n",
"\n",
"NIFTy v4 **more or less stable!**"
"NIFTy v5 **more or less stable!**"
]
}
],
"metadata": {
"celltoolbar": "Slideshow",
"kernelspec": {
"display_name": "Python 2",
"display_name": "Python 3",
"language": "python",
"name": "python2"
"name": "python3"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 2
"version": 3
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython2",
"version": "2.7.12"
"pygments_lexer": "ipython3",
"version": "3.6.6"
}
},
"nbformat": 4,
......
from time import time
import matplotlib.pyplot as plt
import numpy as np
import nifty5 as ift
np.random.seed(40)
N0s, a0s, b0s, c0s = [], [], [], []
for ii in range(10, 26):
nu = 1024
nv = 1024
N = int(2**ii)
print('N = {}'.format(N))
uv = np.random.rand(N, 2) - 0.5
vis = np.random.randn(N) + 1j*np.random.randn(N)
uvspace = ift.RGSpace((nu, nv))
visspace = ift.UnstructuredDomain(N)
img = np.random.randn(nu*nv)
img = img.reshape((nu, nv))
img = ift.from_global_data(uvspace, img)
t0 = time()
GM = ift.GridderMaker(uvspace, eps=1e-7, uv=uv)
vis = ift.from_global_data(visspace, vis)
op = GM.getFull().adjoint
t1 = time()
op(img).to_global_data()
t2 = time()
op.adjoint(vis).to_global_data()
t3 = time()
print(t2-t1, t3-t2)
N0s.append(N)
a0s.append(t1 - t0)
b0s.append(t2 - t1)
c0s.append(t3 - t2)
print('Measure rest operator')
sc = ift.StatCalculator()
op = GM.getRest().adjoint
for _ in range(10):
t0 = time()
res = op(img)
sc.add(time() - t0)
t_fft = sc.mean
print('FFT shape', res.shape)
plt.scatter(N0s, a0s, label='Gridder mr')
plt.legend()
# no idea why this is necessary, but if it is omitted, the range is wrong
plt.ylim(min(a0s), max(a0s))
plt.ylabel('time [s]')
plt.title('Initialization')
plt.loglog()
plt.savefig('bench0.png')
plt.close()
plt.scatter(N0s, b0s, color='k', marker='^', label='Gridder mr times')
plt.scatter(N0s, c0s, color='k', label='Gridder mr adjoint times')
plt.axhline(sc.mean, label='FFT')
plt.axhline(sc.mean + np.sqrt(sc.var))
plt.axhline(sc.mean - np.sqrt(sc.var))
plt.legend()
plt.ylabel('time [s]')
plt.title('Apply')
plt.loglog()
plt.savefig('bench1.png')
plt.close()
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
#
# Copyright(C) 2013-2019 Max-Planck-Society
#
# NIFTy is being developed at the Max-Planck-Institut fuer Astrophysik.
#####################################################################
# Bernoulli reconstruction
# Reconstruct an event probability field with values between 0 and 1
# from the observed events
# 1D (set mode=0), 2D (mode=1), or on the sphere (mode=2)
#####################################################################
import numpy as np
import nifty5 as ift
if __name__ == '__main__':
np.random.seed(41)
# Set up the position space of the signal
mode = 2
if mode == 0:
# One-dimensional regular grid
position_space = ift.RGSpace(1024)
elif mode == 1:
# Two-dimensional regular grid
position_space = ift.RGSpace([512, 512])
else:
# Sphere
position_space = ift.HPSpace(128)
# Define harmonic space and transform
harmonic_space = position_space.get_default_codomain()
HT = ift.HarmonicTransformOperator(harmonic_space, position_space)
position = ift.from_random('normal', harmonic_space)
# Define power spectrum and amplitudes
def sqrtpspec(k):
return 1./(20. + k**2)
A = ift.create_power_operator(harmonic_space, sqrtpspec)
# Set up a sky operator and instrumental response
sky = ift.sigmoid(HT(A))
GR = ift.GeometryRemover(position_space)
R = GR
# Generate mock data
p = R(sky)
mock_position = ift.from_random('normal', harmonic_space)
tmp = p(mock_position).to_global_data().astype(np.float64)
data = np.random.binomial(1, tmp)
data = ift.Field.from_global_data(R.target, data)
# Compute likelihood and Hamiltonian
position = ift.from_random('normal', harmonic_space)
likelihood = ift.BernoulliEnergy(data)(p)
ic_newton = ift.DeltaEnergyController(
name='Newton', iteration_limit=100, tol_rel_deltaE=1e-8)
minimizer = ift.NewtonCG(ic_newton)
ic_sampling = ift.GradientNormController(iteration_limit=100)
# Minimize the Hamiltonian
H = ift.StandardHamiltonian(likelihood, ic_sampling)
H = ift.EnergyAdapter(position, H, want_metric=True)
# minimizer = ift.L_BFGS(ic_newton)
H, convergence = minimizer(H)
reconstruction = sky(H.position)
plot = ift.Plot()
plot.add(reconstruction, title='reconstruction')
plot.add(GR.adjoint_times(data), title='data')
plot.add(sky(mock_position), title='truth')
plot.output(nx=3, xsize=16, ysize=9, title="results", name="bernoulli.png")
import nifty4 as ift
from nifty4.library.nonlinearities import Linear, Tanh, Exponential
import numpy as np
np.random.seed(42)
def adjust_zero_mode(m0, t0):
mtmp = m0.to_global_data().copy()
zero_position = len(m0.shape)*(0,)
zero_mode = mtmp[zero_position]
mtmp[zero_position] = zero_mode / abs(zero_mode)
ttmp = t0.to_global_data().copy()
ttmp[0] += 2 * np.log(abs(zero_mode))
return (ift.Field.from_global_data(m0.domain, mtmp),
ift.Field.from_global_data(t0.domain, ttmp))
if __name__ == "__main__":
noise_level = 1.
p_spec = (lambda k: (.5 / (k + 1) ** 3))
nonlinearity = Linear()
# Set up position space
s_space = ift.RGSpace((128, 128))
h_space = s_space.get_default_codomain()
# Define harmonic transformation and associated harmonic space
HT = ift.HarmonicTransformOperator(h_space, target=s_space)
# Setting up power space
p_space = ift.PowerSpace(h_space,
binbounds=ift.PowerSpace.useful_binbounds(
h_space, logarithmic=True))
# Choosing the prior correlation structure and defining
# correlation operator
p = ift.PS_field(p_space, p_spec)
log_p = ift.log(p)
S = ift.create_power_operator(h_space, power_spectrum=lambda k: 1e-5)
# Drawing a sample sh from the prior distribution in harmonic space
sh = S.draw_sample()
# Choosing the measurement instrument
# Instrument = SmoothingOperator(s_space, sigma=0.01)
mask = np.ones(s_space.shape)
# mask[6000:8000] = 0.
mask[30:70, 30:70] = 0.
mask = ift.Field.from_global_data(s_space, mask)
MaskOperator = ift.DiagonalOperator(mask)
R = ift.GeometryRemover(s_space)
R = R*MaskOperator
# R = R*HT
# R = R * ift.create_harmonic_smoothing_operator((harmonic_space,), 0,
# response_sigma)
MeasurementOperator = R
d_space = MeasurementOperator.target
Distributor = ift.PowerDistributor(target=h_space, power_space=p_space)
power = Distributor(ift.exp(0.5*log_p))
# Creating the mock data
true_sky = nonlinearity(HT(power*sh))
noiseless_data = MeasurementOperator(true_sky)
noise_amplitude = noiseless_data.val.std()*noise_level
N = ift.ScalingOperator(noise_amplitude**2, d_space)
n = N.draw_sample()
# Creating the mock data
d = noiseless_data + n
m0 = ift.full(h_space, 1e-7)
t0 = ift.full(p_space, -4.)
power0 = Distributor.times(ift.exp(0.5 * t0))
plotdict = {"colormap": "Planck-like"}
zmin = true_sky.min()
zmax = true_sky.max()
ift.plot(true_sky, title="True sky", name="true_sky.png", **plotdict)
ift.plot(MeasurementOperator.adjoint_times(d), title="Data",
name="data.png", **plotdict)
IC1 = ift.GradientNormController(name="IC1", iteration_limit=100,
tol_abs_gradnorm=1e-3)
LS = ift.LineSearchStrongWolfe(c2=0.02)
minimizer = ift.RelaxedNewton(IC1, line_searcher=LS)
ICI = ift.GradientNormController(iteration_limit=500,
tol_abs_gradnorm=1e-3)
inverter = ift.ConjugateGradient(controller=ICI)
for i in range(20):
power0 = Distributor(ift.exp(0.5*t0))
map0_energy = ift.library.NonlinearWienerFilterEnergy(
m0, d, MeasurementOperator, nonlinearity, HT, power0, N, S,
inverter=inverter)
# Minimization with chosen minimizer
map0_energy, convergence = minimizer(map0_energy)
m0 = map0_energy.position
# Updating parameters for correlation structure reconstruction
D0 = map0_energy.curvature
# Initializing the power energy with updated parameters
power0_energy = ift.library.NonlinearPowerEnergy(
position=t0, d=d, N=N, xi=m0, D=D0, ht=HT,
Instrument=MeasurementOperator, nonlinearity=nonlinearity,
Distributor=Distributor, sigma=1., samples=2, inverter=inverter)
power0_energy = minimizer(power0_energy)[0]
# Setting new power spectrum
t0 = power0_energy.position
# break degeneracy between amplitude and excitation by setting
# excitation monopole to 1
m0, t0 = adjust_zero_mode(m0, t0)
ift.plot(nonlinearity(HT(power0*m0)), title="Reconstructed sky",
name="reconstructed_sky.png", zmin=zmin, zmax=zmax, **plotdict)
ymin = np.min(p.to_global_data())
ift.plot([ift.exp(t0), p], title="Power spectra",
name="ps.png", ymin=ymin, **plotdict)
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
#
# Copyright(C) 2013-2019 Max-Planck-Society
#
# NIFTy is being developed at the Max-Planck-Institut fuer Astrophysik.
###############################################################################
# Compute a Wiener filter solution with NIFTy
# Shows how measurement gaps are filled in
# 1D (set mode=0), 2D (mode=1), or on the sphere (mode=2)
###############################################################################
import sys
import numpy as np
import nifty5 as ift
def make_checkerboard_mask(position_space):
# Checkerboard mask for 2D mode
mask = np.ones(position_space.shape)
for i in range(4):
for j in range(4):
if (i + j) % 2 == 0:
mask[i*128//4:(i + 1)*128//4, j*128//4:(j + 1)*128//4] = 0
return mask
def make_random_mask():
# Random mask for spherical mode
mask = ift.from_random('pm1', position_space)
mask = (mask + 1)/2
return mask.to_global_data()
if __name__ == '__main__':
np.random.seed(42)
# Choose space on which the signal field is defined
if len(sys.argv) == 2:
mode = int(sys.argv[1])
else:
mode = 1
if mode == 0:
# One-dimensional regular grid
position_space = ift.RGSpace([1024])
mask = np.zeros(position_space.shape)
elif mode == 1:
# Two-dimensional regular grid with checkerboard mask
position_space = ift.RGSpace([128, 128])
mask = make_checkerboard_mask(position_space)
else:
# Sphere with half of its pixels randomly masked
position_space = ift.HPSpace(128)
mask = make_random_mask()
# Specify harmonic space corresponding to signal
harmonic_space = position_space.get_default_codomain()
# Harmonic transform from harmonic space to position space
HT = ift.HarmonicTransformOperator(harmonic_space, target=position_space)
# Set prior correlation covariance with a power spectrum leading to
# homogeneous and isotropic statistics
def power_spectrum(k):
return 100./(20. + k**3)
# 1D spectral space on which the power spectrum is defined
power_space = ift.PowerSpace(harmonic_space)
# Mapping to (higher dimensional) harmonic space
PD = ift.PowerDistributor(harmonic_space, power_space)
# Apply the mapping
prior_correlation_structure = PD(ift.PS_field(power_space, power_spectrum))
# Insert the result into the diagonal of an harmonic space operator
S = ift.DiagonalOperator(prior_correlation_structure)
# S is the prior field covariance
# Build instrument response consisting of a discretization, mask
# and harmonic transformaion
# Masking operator to model that parts of the field have not been observed
mask = ift.Field.from_global_data(position_space, mask)
Mask = ift.MaskOperator(mask)
# The response operator consists of
# - a harmonic transform (to get to image space)
# - the application of the mask
# - the removal of geometric information
# The removal of geometric information is included in the MaskOperator
# it can also be implemented with a GeometryRemover
# Operators can be composed either with parenthesis
R = Mask(HT)
# or with @
R = Mask @ HT
data_space = R.target
# Set the noise covariance N
noise = 5.
N = ift.ScalingOperator(noise, data_space)
# Create mock data
MOCK_SIGNAL = S.draw_sample()
MOCK_NOISE = N.draw_sample()
data = R(MOCK_SIGNAL) + MOCK_NOISE
# Build inverse propagator D and information source j
D_inv = R.adjoint @ N.inverse @ R + S.inverse
j = R.adjoint_times(N.inverse_times(data))
# Make D_inv invertible (via Conjugate Gradient)
IC = ift.GradientNormController(iteration_limit=500, tol_abs_gradnorm=1e-3)
D = ift.InversionEnabler(D_inv, IC, approximation=S.inverse).inverse
# Calculate WIENER FILTER solution
m = D(j)
# Plotting
rg = isinstance(position_space, ift.RGSpace)
plot = ift.Plot()
filename = "getting_started_1_mode_{}.png".format(mode)
if rg and len(position_space.shape) == 1:
plot.add(
[HT(MOCK_SIGNAL), Mask.adjoint(data),
HT(m)],
label=['Mock signal', 'Data', 'Reconstruction'],
alpha=[1, .3, 1])
plot.add(Mask.adjoint(Mask(HT(m - MOCK_SIGNAL))), title='Residuals')
plot.output(nx=2, ny=1, xsize=10, ysize=4, name=filename)
else:
plot.add(HT(MOCK_SIGNAL), title='Mock Signal')
plot.add(Mask.adjoint(data), title='Data')
plot.add(HT(m), title='Reconstruction')
plot.add(Mask.adjoint(Mask(HT(m) - HT(MOCK_SIGNAL))),
title='Residuals')
plot.output(nx=2, ny=2, xsize=10, ysize=10, name=filename)
print("Saved results as '{}'.".format(filename))
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
#
# Copyright(C) 2013-2019 Max-Planck-Society
#
# NIFTy is being developed at the Max-Planck-Institut fuer Astrophysik.
###############################################################################
# Log-normal field reconstruction from Poissonian data with inhomogenous
# exposure (in case for 2D mode)
# 1D (set mode=0), 2D (mode=1), or on the sphere (mode=2)
###############################################################################
import sys
import numpy as np
import nifty5 as ift
def exposure_2d():
# Structured exposure for 2D mode
x_shape, y_shape = position_space.shape
exposure = np.ones(position_space.shape)
exposure[x_shape//3:x_shape//2, :] *= 2.
exposure[x_shape*4//5:x_shape, :] *= .1
exposure[x_shape//2:x_shape*3//2, :] *= 3.
exposure[:, x_shape//3:x_shape//2] *= 2.
exposure[:, x_shape*4//5:x_shape] *= .1
exposure[:, x_shape//2:x_shape*3//2] *= 3.
return ift.Field.from_global_data(position_space, exposure)
if __name__ == '__main__':
np.random.seed(42)
# Choose space on which the signal field is defined
if len(sys.argv) == 2:
mode = int(sys.argv[1])
else:
mode = 1
if mode == 0:
# One-dimensional regular grid with uniform exposure of 10
position_space = ift.RGSpace(1024)
exposure = ift.Field.full(position_space, 10.)
elif mode == 1:
# Two-dimensional regular grid with inhomogeneous exposure
position_space = ift.RGSpace([512, 512])
exposure = exposure_2d()
else:
# Sphere with uniform exposure of 100
position_space = ift.HPSpace(128)
exposure = ift.Field.full(position_space, 100.)
# Define harmonic space and harmonic transform
harmonic_space = position_space.get_default_codomain()
HT = ift.HarmonicTransformOperator(harmonic_space, position_space)
# Domain on which the field's degrees of freedom are defined
domain = ift.DomainTuple.make(harmonic_space)
# Define amplitude (square root of power spectrum)
def sqrtpspec(k):
return 1./(20. + k**2)
p_space = ift.PowerSpace(harmonic_space)
pd = ift.PowerDistributor(harmonic_space, p_space)
a = ift.PS_field(p_space, sqrtpspec)
A = pd(a)
# Define sky operator
sky = ift.exp(HT(ift.makeOp(A)))
M = ift.DiagonalOperator(exposure)
GR = ift.GeometryRemover(position_space)
# Define instrumental response
R = GR(M)
# Generate mock data and define likelihood operator
d_space = R.target[0]
lamb = R(sky)
mock_position = ift.from_random('normal', domain)
data = lamb(mock_position)
data = np.random.poisson(data.to_global_data().astype(np.float64))
data = ift.Field.from_global_data(d_space, data)
likelihood = ift.PoissonianEnergy(data)(lamb)
# Settings for minimization
ic_newton = ift.DeltaEnergyController(
name='Newton', iteration_limit=100, tol_rel_deltaE=1e-8)
minimizer = ift.NewtonCG(ic_newton)
# Compute MAP solution by minimizing the information Hamiltonian
H = ift.StandardHamiltonian(likelihood)
initial_position = ift.from_random('normal', domain)
H = ift.EnergyAdapter(initial_position, H, want_metric=True)
H, convergence = minimizer(H)
# Plotting
signal = sky(mock_position)
reconst = sky(H.position)
filename = "getting_started_2_mode_{}.png".format(mode)
plot = ift.Plot()
plot.add(signal, title='Signal')
plot.add(GR.adjoint(data), title='Data')
plot.add(reconst, title='Reconstruction')
plot.add(reconst - signal, title='Residuals')
plot.output(xsize=12, ysize=10, name=filename)
print("Saved results as '{}'.".format(filename))
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
#
# Copyright(C) 2013-2019 Max-Planck-Society
#
# NIFTy is being developed at the Max-Planck-Institut fuer Astrophysik.
############################################################
# Non-linear tomography
#
# The signal is a sigmoid-normal distributed field.
# The data is the field integrated along lines of sight that are
# randomly (set mode=0) or radially (mode=1) distributed
#
# Demo takes a while to compute
#############################################################
import sys
import numpy as np
import nifty5 as ift
def random_los(n_los):
starts = list(np.random.uniform(0, 1, (n_los, 2)).T)
ends = list(np.random.uniform(0, 1, (n_los, 2)).T)
return starts, ends
def radial_los(n_los):
starts = list(np.random.uniform(0, 1, (n_los, 2)).T)
ends = list(0.5 + 0*np.random.uniform(0, 1, (n_los, 2)).T)
return starts, ends
if __name__ == '__main__':
np.random.seed(420)
# Choose between random line-of-sight response (mode=0) and radial lines
# of sight (mode=1)
if len(sys.argv) == 2:
mode = int(sys.argv[1])
else:
mode = 0
filename = "getting_started_3_mode_{}_".format(mode) + "{}.png"
position_space = ift.RGSpace([128, 128])
harmonic_space = position_space.get_default_codomain()
ht = ift.HarmonicTransformOperator(harmonic_space, position_space)
power_space = ift.PowerSpace(harmonic_space)
# Set up an amplitude operator for the field
dct = {
'target': power_space,
'n_pix': 64, # 64 spectral bins
# Spectral smoothness (affects Gaussian process part)
'a': 3, # relatively high variance of spectral curbvature
'k0': .4, # quefrency mode below which cepstrum flattens
# Power-law part of spectrum:
'sm': -5, # preferred power-law slope
'sv': .5, # low variance of power-law slope
'im': 0, # y-intercept mean, in-/decrease for more/less contrast
'iv': .3 # y-intercept variance
}
A = ift.SLAmplitude(**dct)
# Build the operator for a correlated signal
power_distributor = ift.PowerDistributor(harmonic_space, power_space)
vol = harmonic_space.scalar_dvol**-0.5
xi = ift.ducktape(harmonic_space, None, 'xi')
correlated_field = ht(vol*power_distributor(A)*xi)
# Alternatively, one can use:
# correlated_field = ift.CorrelatedField(position_space, A)
# Apply a nonlinearity
signal = ift.sigmoid(correlated_field)
# Build the line-of-sight response and define signal response
LOS_starts, LOS_ends = random_los(100) if mode == 0 else radial_los(100)
R = ift.LOSResponse(position_space, starts=LOS_starts, ends=LOS_ends)
signal_response = R(signal)
# Specify noise
data_space = R.target
noise = .001
N = ift.ScalingOperator(noise, data_space)
# Generate mock signal and data
mock_position = ift.from_random('normal', signal_response.domain)
data = signal_response(mock_position) + N.draw_sample()
# Minimization parameters
ic_sampling = ift.AbsDeltaEnergyController(
name='Sampling', deltaE=0.05, iteration_limit=100)
ic_newton = ift.AbsDeltaEnergyController(
name='Newton', deltaE=0.5, iteration_limit=35)
minimizer = ift.NewtonCG(ic_newton)
# Set up likelihood and information Hamiltonian
likelihood = ift.GaussianEnergy(mean=data,
inverse_covariance=N.inverse)(signal_response)
H = ift.StandardHamiltonian(likelihood, ic_sampling)
initial_mean = ift.MultiField.full(H.domain, 0.)
mean = initial_mean
plot = ift.Plot()
plot.add(signal(mock_position), title='Ground Truth')
plot.add(R.adjoint_times(data), title='Data')
plot.add([A.force(mock_position)], title='Power Spectrum')
plot.output(ny=1, nx=3, xsize=24, ysize=6, name=filename.format("setup"))
# number of samples used to estimate the KL
N_samples = 20
# Draw new samples to approximate the KL five times
for i in range(5):
# Draw new samples and minimize KL
KL = ift.MetricGaussianKL(mean, H, N_samples)
KL, convergence = minimizer(KL)
mean = KL.position
# Plot current reconstruction
plot = ift.Plot()
plot.add(signal(KL.position), title="reconstruction")
plot.add([A.force(KL.position), A.force(mock_position)], title="power")
plot.output(ny=1, ysize=6, xsize=16,
name=filename.format("loop_{:02d}".format(i)))
# Draw posterior samples
KL = ift.MetricGaussianKL(mean, H, N_samples)
sc = ift.StatCalculator()
for sample in KL.samples:
sc.add(signal(sample + KL.position))
# Plotting
filename_res = filename.format("results")
plot = ift.Plot()
plot.add(sc.mean, title="Posterior Mean")
plot.add(ift.sqrt(sc.var), title="Posterior Standard Deviation")
powers = [A.force(s + KL.position) for s in KL.samples]
plot.add(
powers + [A.force(KL.position),
A.force(mock_position)],
title="Sampled Posterior Power Spectrum",
linewidth=[1.]*len(powers) + [3., 3.])
plot.output(ny=1, nx=3, xsize=24, ysize=6, name=filename_res)
print("Saved results as '{}'.".format(filename_res))
import nifty4 as ift
import numpy as np
import matplotlib.pyplot as plt
from nifty4.sugar import create_power_operator
np.random.seed(42)
x_space = ift.RGSpace(1024)
h_space = x_space.get_default_codomain()
d_space = x_space
N_hat = np.full(d_space.shape, 10.)
N_hat[400:450] = 0.0001
N_hat = ift.Field.from_global_data(d_space, N_hat)
N = ift.DiagonalOperator(N_hat)
FFT = ift.HarmonicTransformOperator(h_space, x_space)
R = ift.ScalingOperator(1., x_space)
def ampspec(k): return 1. / (1. + k**2.)
S = ift.ScalingOperator(1., h_space)
A = create_power_operator(h_space, ampspec)
s_h = S.draw_sample()
sky = FFT * A
s_x = sky(s_h)
n = N.draw_sample()
d = R(s_x) + n
R_p = R * FFT * A
j = R_p.adjoint(N.inverse(d))
D_inv = ift.SandwichOperator.make(R_p, N.inverse) + S.inverse
N_samps = 200
N_iter = 100
IC = ift.GradientNormController(tol_abs_gradnorm=1e-3, iteration_limit=N_iter)
m, samps = ift.library.generate_krylov_samples(D_inv, S, j, N_samps, IC)
m_x = sky(m)
inverter = ift.ConjugateGradient(IC)
curv = ift.library.WienerFilterCurvature(S=S, N=N, R=R_p, inverter=inverter)
samps_old = [curv.draw_sample(from_inverse=True) for i in range(N_samps)]
plt.plot(d.to_global_data(), '+', label="data", alpha=.5)
plt.plot(s_x.to_global_data(), label="original")
plt.plot(m_x.to_global_data(), label="reconstruction")
plt.legend()
plt.savefig('Krylov_reconstruction.png')
plt.close()
pltdict = {'alpha': .3, 'linewidth': .2}
for i in range(N_samps):
if i == 0:
plt.plot(sky(samps_old[i]).to_global_data(), color='b',
label='Traditional samples (residuals)',
**pltdict)
plt.plot(sky(samps[i]).to_global_data(), color='r',
label='Krylov samples (residuals)',
**pltdict)
else:
plt.plot(sky(samps_old[i]).to_global_data(), color='b', **pltdict)
plt.plot(sky(samps[i]).to_global_data(), color='r', **pltdict)
plt.plot((s_x - m_x).to_global_data(), color='k', label='signal - mean')
plt.legend()
plt.savefig('Krylov_samples_residuals.png')
plt.close()
D_hat_old = ift.full(x_space, 0.).to_global_data()
D_hat_new = ift.full(x_space, 0.).to_global_data()
for i in range(N_samps):
D_hat_old += sky(samps_old[i]).to_global_data()**2
D_hat_new += sky(samps[i]).to_global_data()**2
plt.plot(np.sqrt(D_hat_old / N_samps), 'r--', label='Traditional uncertainty')
plt.plot(-np.sqrt(D_hat_old / N_samps), 'r--')
plt.fill_between(range(len(D_hat_new)), -np.sqrt(D_hat_new / N_samps), np.sqrt(
D_hat_new / N_samps), facecolor='0.5', alpha=0.5,
label='Krylov uncertainty')
plt.plot((s_x - m_x).to_global_data(), color='k', label='signal - mean')
plt.legend()
plt.savefig('Krylov_uncertainty.png')
plt.close()
import numpy as np
import nifty5 as ift
def convtest(test_signal, delta, func):
domain = test_signal.domain
# Create Convolution Operator
conv_op = ift.FuncConvolutionOperator(domain, func)
# Convolve, Adjoint-Convolve
conv_signal = conv_op(test_signal)
cac_signal = conv_op.adjoint_times(conv_signal)
print(test_signal.integrate(), conv_signal.integrate(),
cac_signal.integrate())
# generate kernel image
conv_delta = conv_op(delta)
# Plot results
plot = ift.Plot()
plot.add(signal, title='Signal')
plot.add(conv_signal, title='Signal Convolved')
plot.add(cac_signal, title='Signal, Conv, Adj-Conv')
plot.add(conv_delta, title='Kernel')
plot.output()
# Healpix test
nside = 64
npix = 12 * nside * nside
domain = ift.HPSpace(nside)
# Define test signal (some point sources)
signal_vals = np.zeros(npix, dtype=np.float64)
for i in range(0, npix, npix//12 + 27):
signal_vals[i] = 500.
signal = ift.from_global_data(domain, signal_vals)
delta_vals = np.zeros(npix, dtype=np.float64)
delta_vals[0] = 1.0
delta = ift.from_global_data(domain, delta_vals)
# Define kernel function
def func(theta):
ct = np.cos(theta)
return 1. * np.logical_and(ct > 0.7, ct <= 0.8)