Commit d90436e4 authored by Martin Reinecke's avatar Martin Reinecke
Browse files

Merge branch 'NIFTy_5' into docstrings_torsten

parents 0df19473 928858f4
......@@ -10,6 +10,7 @@ setup.cfg
.document
.svn/
*.csv
.pytest_cache/
# from https://github.com/github/gitignore/blob/master/Python.gitignore
......
......@@ -34,18 +34,22 @@ build_docker_from_cache:
- docker build -t $CONTAINER_TEST_IMAGE .
- docker push $CONTAINER_TEST_IMAGE
test:
test_serial:
stage: test
variables:
OMPI_MCA_btl_vader_single_copy_mechanism: none
script:
- mpiexec -n 2 --bind-to none pytest-3 -q test
- pytest-3 -q --cov=nifty5 test
- >
python3 -m coverage report --omit "*plotting*,*distributed_do*"
- >
python3 -m coverage report --omit "*plotting*,*distributed_do*" | grep TOTAL | awk '{ print "TOTAL: "$4; }'
test_mpi:
stage: test
variables:
OMPI_MCA_btl_vader_single_copy_mechanism: none
script:
- mpiexec -n 2 --bind-to none pytest-3 -q test
pages:
stage: release
before_script:
......
......@@ -7,9 +7,9 @@ RUN apt-get update && apt-get install -y \
libfftw3-dev \
python3 python3-pip python3-dev python3-future python3-scipy cython3 \
# Documentation build dependencies
python3-sphinx python3-sphinx-rtd-theme python3-numpydoc \
python3-sphinx python3-sphinx-rtd-theme \
# Testing dependencies
python3-coverage python3-parameterized python3-pytest python3-pytest-cov \
python3-coverage python3-pytest python3-pytest-cov \
# Optional NIFTy dependencies
openmpi-bin libopenmpi-dev python3-mpi4py \
# Packages needed for NIFTy
......
rm -rf docs/build docs/source/mod
sphinx-apidoc -l -e -d 2 -o docs/source/mod nifty5
sphinx-build -b html docs/source/ docs/build/
.. currentmodule:: nifty5
=============
Code Overview
......@@ -37,9 +36,12 @@ Domains
Abstract base class
-------------------
.. currentmodule:: nifty5.domains.domain
One of the fundamental building blocks of the NIFTy5 framework is the *domain*.
Its required capabilities are expressed by the abstract :class:`Domain` class.
Its required capabilities are expressed by the abstract :py:class:`Domain` class.
A domain must be able to answer the following queries:
m
- its total number of data entries (pixels), which is accessible via the
:attr:`~Domain.size` property
......@@ -51,6 +53,8 @@ A domain must be able to answer the following queries:
Unstructured domains
--------------------
.. currentmodule:: nifty5.domains.unstructured_domain
Domains can be either *structured* (i.e. there is geometrical information
associated with them, like position in space and volume factors),
or *unstructured* (meaning that the data points have no associated manifold).
......@@ -62,6 +66,8 @@ Unstructured domains can be described by instances of NIFTy's
Structured domains
------------------
.. currentmodule:: nifty5.domains.structured_domain
In contrast to unstructured domains, these domains have an assigned geometry.
NIFTy requires them to provide the volume elements of their grid cells.
The additional methods are specified in the abstract class
......@@ -81,15 +87,17 @@ The additional methods are specified in the abstract class
NIFTy comes with several concrete subclasses of :class:`StructuredDomain`:
- :class:`RGSpace` represents a regular Cartesian grid with an arbitrary
.. currentmodule:: nifty5.domains
- :class:`rg_space.RGSpace` represents a regular Cartesian grid with an arbitrary
number of dimensions, which is supposed to be periodic in each dimension.
- :class:`HPSpace` and :class:`GLSpace` describe pixelisations of the
2-sphere; their counterpart in harmonic space is :class:`LMSpace`, which
- :class:`hp_space.HPSpace` and :class:`gl_space.GLSpace` describe pixelisations of the
2-sphere; their counterpart in harmonic space is :class:`lm_space.LMSpace`, which
contains spherical harmonic coefficients.
- :class:`PowerSpace` is used to describe one-dimensional power spectra.
- :class:`power_space.PowerSpace` is used to describe one-dimensional power spectra.
Among these, :class:`RGSpace` can be harmonic or not (depending on constructor arguments), :class:`GLSpace`, :class:`HPSpace`, and :class:`PowerSpace` are
pure position domains (i.e. nonharmonic), and :class:`LMSpace` is always
Among these, :class:`rg_space.RGSpace` can be harmonic or not (depending on constructor arguments), :class:`gl_space.GLSpace`, :class:`hp_space.HPSpace`, and :class:`power_space.PowerSpace` are
pure position domains (i.e. nonharmonic), and :class:`lm_space.LMSpace` is always
harmonic.
......@@ -158,7 +166,7 @@ be extracted first, then changed, and a new field has to be created from the
result.
Fields defined on a MultiDomain
------------------------------
-------------------------------
The :class:`MultiField` class can be seen as a dictionary of individual
:class:`Field` s, each identified by a name, which is defined on a
......@@ -300,7 +308,7 @@ As an example one may consider the following combination of ``x``, which is an o
Basic operators
------------
---------------
# FIXME All this is outdated!
Basic operator classes provided by NIFTy are
......
import nifty5
import sphinx_rtd_theme
napoleon_google_docstring = False
napoleon_numpy_docstring = True
napoleon_use_ivar = True
napoleon_use_param = False
napoleon_use_keyword = False
autodoc_member_order = 'groupwise'
numpydoc_show_inherited_class_members = False
numpydoc_class_members_toctree = False
extensions = [
'sphinx.ext.autodoc', 'numpydoc', 'sphinx.ext.autosummary',
'sphinx.ext.napoleon', 'sphinx.ext.imgmath', 'sphinx.ext.viewcode'
'sphinx.ext.napoleon', # Support for NumPy and Google style docstrings
'sphinx.ext.imgmath', # Render math as images
'sphinx.ext.viewcode' # Add links to highlighted source code
]
templates_path = ['_templates']
source_suffix = '.rst'
master_doc = 'index'
napoleon_google_docstring = False
napoleon_numpy_docstring = True
napoleon_use_ivar = True
project = u'NIFTy5'
copyright = u'2013-2019, Max-Planck-Society'
author = u'Martin Reinecke'
......@@ -29,19 +21,6 @@ version = release[:-2]
language = None
exclude_patterns = []
add_module_names = False
pygments_style = 'sphinx'
todo_include_todos = True
html_theme = "sphinx_rtd_theme"
html_theme_options = {
'collapse_navigation': False,
'display_version': False,
}
html_theme_path = [sphinx_rtd_theme.get_html_theme_path()]
html_logo = 'nifty_logo_black.png'
html_static_path = []
html_last_updated_fmt = '%b %d, %Y'
html_domain_indices = False
html_use_index = False
html_show_sourcelink = False
htmlhelp_basename = 'NIFTydoc'
......@@ -104,6 +104,7 @@ with :math:`{R}` the measurement response, which maps the continous signal field
This is called a free theory, as the information Hamiltonian
associate professor
.. math::
\mathcal{H}(d,s)= -\log \mathcal{P}(d,s)= \frac{1}{2} s^\dagger S^{-1} s + \frac{1}{2} (d-R\,s)^\dagger N^{-1} (d-R\,s) + \mathrm{const}
......@@ -179,23 +180,22 @@ NIFTy takes advantage of this formulation in several ways:
The reconstruction of a non-Gaussian signal with unknown covarinance from a non-trivial (tomographic) response is demonstrated in demos/getting_started_3.py. Here, the uncertainty of the field and the power spectrum of its generating process are probed via posterior samples provided by the MGVI algorithm.
+-------------------------------------------------+
| .. image:: images/getting_started_3_setup.png |
| :width: 30 % |
+-------------------------------------------------+
| .. image:: images/getting_started_3_results.png |
| :width: 30 % |
+-------------------------------------------------+
| Output of tomography demo getting_started_3.py. |
| **Top row:** Non-Gaussian signal field, |
| data backprojected into the image domain, power |
| spectrum of underlying Gausssian process. |
| **Bottom row:** Posterior mean field signal |
| reconstruction, its uncertainty, and the power |
| spectrum of the process for different posterior |
| samples in comparison to the correct one (thick |
| orange line). |
+-------------------------------------------------+
+----------------------------------------------------+
| **Output of tomography demo getting_started_3.py** |
+----------------------------------------------------+
| .. image:: images/getting_started_3_setup.png |
| |
+----------------------------------------------------+
| Non-Gaussian signal field, |
| data backprojected into the image domain, power |
| spectrum of underlying Gausssian process. |
+----------------------------------------------------+
| .. image:: images/getting_started_3_results.png |
| |
+----------------------------------------------------+
| Posterior mean field signal |
| reconstruction, its uncertainty, and the power |
| spectrum of the process for different posterior |
| samples in comparison to the correct one (thick |
| orange line). |
+----------------------------------------------------+
......@@ -17,6 +17,7 @@
import sys
from functools import reduce
import numpy as np
from mpi4py import MPI
......
......@@ -18,11 +18,10 @@
# Data object module for NIFTy that uses simple numpy ndarrays.
import numpy as np
from numpy import empty, empty_like, exp, full, log
from numpy import absolute, clip, cos, cosh, empty, empty_like, exp, full, log
from numpy import ndarray as data_object
from numpy import ones, sqrt, tanh, vdot, zeros
from numpy import sin, cos, tan, sinh, cosh, sinc
from numpy import absolute, sign, clip
from numpy import ones, sign, sin, sinc, sinh, sqrt, tan, tanh, vdot, zeros
from .random import Random
__all__ = ["ntask", "rank", "master", "local_shape", "data_object", "full",
......
......@@ -111,23 +111,27 @@ def _SlopePowerSpectrum(logk_space, sm, sv, im, iv):
def AmplitudeOperator(s_space, Npixdof, ceps_a, ceps_k, sm, sv, im, iv,
keys=['tau', 'phi'], zero_mode=True):
'''
''' Operator for parametrizing smooth power spectra.
Computes a smooth power spectrum.
Output is defined on a PowerSpace.
Parameters
----------
Npixdof : #pix in dof_space
ceps_a, ceps_k0 : Smoothness parameters in ceps_kernel
eg. ceps_kernel(k) = (a/(1+(k/k0)**2))**2
a = ceps_a, k0 = ceps_k0
sm, sv : slope_mean = expected exponent of power law (e.g. -4),
slope_variance (default=1)
im, iv : y-intercept_mean, y-intercept_variance of power_slope
Npixdof : int
#pix in dof_space
ceps_a : float
Smoothness parameters in ceps_kernel eg. ceps_kernel(k) = (a/(1+(k/k0)**2))**2 a = ceps_a, k0 = ceps_k0
ceps_k0 : float
Smoothness parameters in ceps_kernel eg. ceps_kernel(k) = (a/(1+(k/k0)**2))**2 a = ceps_a, k0 = ceps_k0
sm : float
slope_mean = expected exponent of power law (e.g. -4)
sv : float
slope_variance (default=1)
im : float
y-intercept_mean
iv : float
y-intercept_variance of power_slope
'''
from ..operators.exp_transform import ExpTransform
......
......@@ -28,14 +28,14 @@ from ..field import Field
from ..operators.linear_operator import LinearOperator
def _gaussian_error_function(x):
return 0.5/erfc(x*np.sqrt(2.))
def _gaussian_sf(x):
return 0.5*erfc(x/np.sqrt(2.))
def _comp_traverse(start, end, shp, dist, lo, mid, hi, erf):
def _comp_traverse(start, end, shp, dist, lo, mid, hi, sig, erf):
ndim = start.shape[0]
nlos = start.shape[1]
inc = np.full(len(shp), 1)
inc = np.full(len(shp), 1, dtype=np.int64)
for i in range(-2, -len(shp)-1, -1):
inc[i] = inc[i+1]*shp[i+1]
......@@ -59,7 +59,7 @@ def _comp_traverse(start, end, shp, dist, lo, mid, hi, erf):
dmin += 1e-7
dmax -= 1e-7
if dmin >= dmax: # no intersection
out[i] = (np.full(0, 0), np.full(0, 0.))
out[i] = (np.full(0, 0, dtype=np.int64), np.full(0, 0.))
continue
# determine coordinates of first cell crossing
c_first = np.ceil(start[:, i]+dir*dmin)
......@@ -75,7 +75,7 @@ def _comp_traverse(start, end, shp, dist, lo, mid, hi, erf):
tmp = np.arange(start=c_first[j], stop=dmax,
step=abs(1./dir[j]))
cdist = np.append(cdist, tmp)
add = np.append(add, np.full(len(tmp), step))
add = np.append(add, np.full(len(tmp), step, dtype=np.int64))
idx = np.argsort(cdist)
cdist = cdist[idx]
add = add[idx]
......@@ -85,21 +85,19 @@ def _comp_traverse(start, end, shp, dist, lo, mid, hi, erf):
cdist *= corfac
wgt = np.diff(cdist)
mdist = 0.5*(cdist[:-1]+cdist[1:])
wgt = apply_erf(wgt, mdist, lo[i], mid[i], hi[i], erf)
wgt = apply_erf(wgt, mdist, lo[i], mid[i], hi[i], sig[i], erf)
add = np.append(pos1, add)
add = np.cumsum(add)
out[i] = (add, wgt)
return out
def apply_erf(wgt, dist, lo, mid, hi, erf):
def apply_erf(wgt, dist, lo, mid, hi, sig, erf):
wgt = wgt.copy()
mask = dist > hi
wgt[mask] = 0.
mask = (dist > mid) & (dist <= hi)
wgt[mask] *= erf((dist[mask]-mid)/(hi-mid))
mask = (dist <= mid) & (dist > lo)
wgt[mask] *= erf((dist[mask]-mid)/(mid-lo))
mask = (dist > lo) & (dist <= hi)
wgt[mask] *= erf((-1/dist[mask]+1/mid)/sig)
return wgt
......@@ -119,17 +117,32 @@ class LOSResponse(LinearOperator):
of sight. The first dimension must have as many entries as `domain`
has dimensions. The second dimensions must be identical for both arrays
and indicated the total number of lines of sight.
sigmas_low, sigmas_up : numpy.ndarray(float) (optional)
For expert use. If unsure, leave blank.
sigmas: numpy.ndarray(float) (optional)
If this is not None, the inverse of the lengths of the LOSs are assumed
to be Gaussian distributed with these sigmas. The start point will
remain the same, but the endpoint is assumed to be unknown.
This is a typical statistical model for astrophysical parallaxes.
The LOS response then returns the expected integral
over the input given that the length of the LOS is unknown and
therefore the result is averaged over different endpoints.
default: None
truncation: float (optional)
Use only if the sigmas keyword argument is used!
This truncates the probability of the endpoint lying more sigmas away
than the truncation. Used to speed up computation and to avoid negative
distances. It should hold that `1./(1./length-sigma*truncation)>0`
for all lengths of the LOSs and all corresponding sigma of sigmas.
If unsure, leave blank.
default: 3.
Notes
-----
`starts, `ends`, `sigmas_low`, and `sigmas_up` have to be identical on
`starts, `ends`, `sigmas`, and `truncation` have to be identical on
every calling MPI task (i.e. the full LOS information has to be provided on
every task).
"""
def __init__(self, domain, starts, ends, sigmas_low=None, sigmas_up=None):
def __init__(self, domain, starts, ends, sigmas=None, truncation=3.):
self._domain = DomainTuple.make(domain)
self._capability = self.TIMES | self.ADJOINT_TIMES
......@@ -141,20 +154,15 @@ class LOSResponse(LinearOperator):
starts = np.array(starts)
nlos = starts.shape[1]
ends = np.array(ends)
if sigmas_low is None:
sigmas_low = np.zeros(nlos, dtype=np.float32)
if sigmas_up is None:
sigmas_up = np.zeros(nlos, dtype=np.float32)
sigmas_low = np.array(sigmas_low)
sigmas_up = np.array(sigmas_up)
if sigmas is None:
sigmas = np.zeros(nlos, dtype=np.float32)
sigmas = np.array(sigmas)
if starts.shape[0] != ndim:
raise TypeError("dimension mismatch")
if nlos != sigmas_low.shape[0]:
if nlos != sigmas.shape[0]:
raise TypeError("dimension mismatch")
if starts.shape != ends.shape:
raise TypeError("dimension mismatch")
if sigmas_low.shape != sigmas_up.shape:
raise TypeError("dimension mismatch")
self._local_shape = dobj.local_shape(self.domain[0].shape)
local_zero_point = (np.array(
......@@ -164,7 +172,11 @@ class LOSResponse(LinearOperator):
diffs = ends-starts
difflen = np.linalg.norm(diffs, axis=0)
diffs /= difflen
real_ends = ends + sigmas_up*diffs
real_distances = 1./(1./difflen - truncation*sigmas)
if np.any(real_distances < 0):
raise ValueError("parallax error truncation to high: "
"getting negative distances")
real_ends = starts + diffs*real_distances
lzp = local_zero_point.reshape((-1, 1))
dist = np.array(self.domain[0].distances).reshape((-1, 1))
localized_pixel_starts = (starts-lzp)/dist + 0.5
......@@ -175,8 +187,11 @@ class LOSResponse(LinearOperator):
localized_pixel_ends,
self._local_shape,
np.array(self.domain[0].distances),
difflen-sigmas_low, difflen, difflen+sigmas_up,
_gaussian_error_function)
1./(1./difflen+truncation*sigmas),
difflen,
1./(1./difflen-truncation*sigmas),
sigmas,
_gaussian_sf)
boxsz = 16
nlos = len(w_i)
......@@ -202,7 +217,7 @@ class LOSResponse(LinearOperator):
tmp += (fullidx[j]//boxsz)*fct
fct *= self._local_shape[j]
tmp += cnt/float(nlos)
tmp += iarr[ofs:ofs+nval]/float(nlos*npix)
tmp += iarr[ofs:ofs+nval]/(float(nlos)*float(npix))
pri[ofs:ofs+nval] = tmp
ofs += nval
cnt += 1
......
......@@ -110,11 +110,11 @@ class Linearization(object):
def __truediv__(self, other):
if isinstance(other, Linearization):
return self.__mul__(other.inverse())
return self.__mul__(other.one_over())
return self.__mul__(1./other)
def __rtruediv__(self, other):
return self.inverse().__mul__(other)
return self.one_over().__mul__(other)
def __pow__(self, power):
if not np.isscalar(power):
......@@ -122,9 +122,6 @@ class Linearization(object):
return self.new(self._val**power,
makeOp(self._val**(power-1)).scale(power)(self._jac))
def inverse(self):
return self.new(1./self._val, makeOp(-1./(self._val**2))(self._jac))
def __mul__(self, other):
from .sugar import makeOp
if isinstance(other, Linearization):
......
......@@ -23,18 +23,18 @@ from .linear_operator import LinearOperator
class DomainTupleFieldInserter(LinearOperator):
def __init__(self, domain, new_space, index, position):
'''Writes the content of a field into one slice of a DomainTuple.
'''Writes the content of a field into one slice of a DomainTuple.
Parameters
----------
domain : Domain, tuple of Domain or DomainTuple
new_space : Domain, tuple of Domain or DomainTuple
index : Integer
Index at which new_space shall be added to domain.
position : tuple
Slice in new_space in which the input field shall be written into.
'''
Parameters
----------
domain : Domain, tuple of Domain or DomainTuple
new_space : Domain, tuple of Domain or DomainTuple
index : Integer
Index at which new_space shall be added to domain.
position : tuple
Slice in new_space in which the input field shall be written into.
'''
def __init__(self, domain, new_space, index, position):
self._domain = DomainTuple.make(domain)
tgt = list(self.domain)
tgt.insert(index, new_space)
......
......@@ -15,20 +15,12 @@
#
# NIFTy is being developed at the Max-Planck-Institut fuer Astrophysik.
from builtins import str
import pytest
import numpy as np
from parameterized import parameterized
np.seterr(all='raise', under='ignore')
def list2fixture(lst):
@pytest.fixture(params=lst)
def myfixture(request):
return request.param
def _custom_name_func(testcase_func, param_num, param):
return "{}_{}".format(
testcase_func.__name__,
parameterized.to_safe_name("_".join(str(x) for x in param.args)),
)
def expand(*args, **kwargs):
return parameterized.expand(*args, func=_custom_name_func, **kwargs)
return myfixture
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
#
# Copyright(C) 2013-2019 Max-Planck-Society
#
# NIFTy is being developed at the Max-Planck-Institut fuer Astrophysik.
import unittest
from itertools import product
from test.common import expand
import nifty5 as ift
import numpy as np
class Energy_Tests(unittest.TestCase):
def make_operator(self, **kwargs):
np.random.seed(kwargs['seed'])
S = ift.ScalingOperator(1., kwargs['space'])
s = S.draw_sample()
return ift.MultiField.from_dict({kwargs['space_key']: s})
@expand(product(
[ift.GLSpace(15),
ift.RGSpace(64, distances=.789),
ift.RGSpace([32, 32], distances=.789)],
[4, 78, 23]
))
def testGaussian(self, space, seed):
op = self.make_operator(
space_key='s1', space=space, seed=seed)['s1']
energy = ift.GaussianEnergy(domain=space)
ift.extra.check_value_gradient_consistency(energy, op)
# @expand(product(
# [ift.GLSpace(15),
# ift.RGSpace(64, distances=.789),
# ift.RGSpace([32, 32], distances=.789)],
# [4, 78, 23]
# ))
# def testQuadratic(self, type1, space, seed):
# np.random.seed(seed)
# S = ift.ScalingOperator(1., space)
# s = [S.draw_sample() for _ in range(3)]
# energy = ift.QuadraticEnergy(s[0], ift.makeOp(s[1]), s[2])
# ift.extra.check_value_gradient_consistency(energy)
@expand(
product([
ift.GLSpace(15),
ift.RGSpace(64, distances=.789),
ift.RGSpace([32, 32], distances=.789)
], [4, 78, 23]))
def testInverseGammaLikelihood(self, space, seed):
op = self.make_operator(space_key='s1', space=space, seed=seed)['s1']
op = op.exp()
d = np.random.normal(10, size=space.shape)**2
d = ift.Field.from_global_data(space, d)
energy = ift.InverseGammaLikelihood(d)
ift.extra.check_value_gradient_consistency(energy, op, tol=1e-7)
@expand(product(
[ift.GLSpace(15),
ift.RGSpace(64, distances=.789),
ift.RGSpace([32, 32], distances=.789)],
[4, 78, 23]
))