Commit 4ed58632 by Martin Reinecke

### Merge branch 'immutable_fields' into 'NIFTy_5'

Immutable fields

See merge request ift/nifty-dev!39
parents 8cf302cb 83400bbe
 %% Cell type:markdown id: tags: # A NIFTy demonstration %% Cell type:markdown id: tags: ## IFT: Big Picture IFT starting point: $$d = Rs+n$$ Typically, $s$ is a continuous field, $d$ a discrete data vector. Particularly, $R$ is not invertible. IFT aims at **inverting** the above uninvertible problem in the **best possible way** using Bayesian statistics. ## NIFTy NIFTy (Numerical Information Field Theory) is a Python framework in which IFT problems can be tackled easily. Main Interfaces: - **Spaces**: Cartesian, 2-Spheres (Healpix, Gauss-Legendre) and their respective harmonic spaces. - **Fields**: Defined on spaces. - **Operators**: Acting on fields. %% Cell type:markdown id: tags: ## Wiener Filter: Formulae ### Assumptions - $d=Rs+n$, $R$ linear operator. - $\mathcal P (s) = \mathcal G (s,S)$, $\mathcal P (n) = \mathcal G (n,N)$ where $S, N$ are positive definite matrices. ### Posterior The Posterior is given by: $$\mathcal P (s|d) \propto P(s,d) = \mathcal G(d-Rs,N) \,\mathcal G(s,S) \propto \mathcal G (m,D)$$ where \begin{align} m &= Dj \\ D^{-1}&= (S^{-1} +R^\dagger N^{-1} R )\\ j &= R^\dagger N^{-1} d \end{align} Let us implement this in NIFTy! %% Cell type:markdown id: tags: ## Wiener Filter: Example - We assume statistical homogeneity and isotropy. Therefore the signal covariance $S$ is diagonal in harmonic space, and is described by a one-dimensional power spectrum, assumed here as $$P(k) = P_0\,\left(1+\left(\frac{k}{k_0}\right)^2\right)^{-\gamma /2},$$ with $P_0 = 0.2, k_0 = 5, \gamma = 4$. - $N = 0.2 \cdot \mathbb{1}$. - Number of data points $N_{pix} = 512$. - reconstruction in harmonic space. - Response operator: $$R = FFT_{\text{harmonic} \rightarrow \text{position}}$$ %% Cell type:code id: tags:  python N_pixels = 512 # Number of pixels def pow_spec(k): P0, k0, gamma = [.2, 5, 4] return P0 / ((1. + (k/k0)**2)**(gamma / 2))  %% Cell type:markdown id: tags: ## Wiener Filter: Implementation %% Cell type:markdown id: tags: ### Import Modules %% Cell type:code id: tags:  python import numpy as np np.random.seed(40) import nifty5 as ift import matplotlib.pyplot as plt %matplotlib inline  %% Cell type:markdown id: tags: ### Implement Propagator %% Cell type:code id: tags:  python def Curvature(R, N, Sh): IC = ift.GradientNormController(iteration_limit=50000, tol_abs_gradnorm=0.1) # WienerFilterCurvature is (R.adjoint*N.inverse*R + Sh.inverse) plus some handy # helper methods. return ift.WienerFilterCurvature(R,N,Sh,iteration_controller=IC,iteration_controller_sampling=IC)  %% Cell type:markdown id: tags: ### Conjugate Gradient Preconditioning - $D$ is defined via: $$D^{-1} = \mathcal S_h^{-1} + R^\dagger N^{-1} R.$$ In the end, we want to apply $D$ to $j$, i.e. we need the inverse action of $D^{-1}$. This is done numerically (algorithm: *Conjugate Gradient*). %% Cell type:markdown id: tags: ### Generate Mock data - Generate a field $s$ and $n$ with given covariances. - Calculate $d$. %% Cell type:code id: tags:  python s_space = ift.RGSpace(N_pixels) h_space = s_space.get_default_codomain() HT = ift.HarmonicTransformOperator(h_space, target=s_space) # Operators Sh = ift.create_power_operator(h_space, power_spectrum=pow_spec) R = HT #*ift.create_harmonic_smoothing_operator((h_space,), 0, 0.02) # Fields and data sh = Sh.draw_sample() noiseless_data=R(sh) noise_amplitude = np.sqrt(0.2) N = ift.ScalingOperator(noise_amplitude**2, s_space) n = ift.Field.from_random(domain=s_space, random_type='normal', std=noise_amplitude, mean=0) d = noiseless_data + n j = R.adjoint_times(N.inverse_times(d)) curv = Curvature(R=R, N=N, Sh=Sh) D = curv.inverse  %% Cell type:markdown id: tags: ### Run Wiener Filter %% Cell type:code id: tags:  python m = D(j)  %% Cell type:markdown id: tags: ### Signal Reconstruction %% Cell type:code id: tags:  python # Get signal data and reconstruction data s_data = HT(sh).to_global_data() m_data = HT(m).to_global_data() d_data = d.to_global_data() plt.figure(figsize=(15,10)) plt.plot(s_data, 'r', label="Signal", linewidth=3) plt.plot(d_data, 'k.', label="Data") plt.plot(m_data, 'k', label="Reconstruction",linewidth=3) plt.title("Reconstruction") plt.legend() plt.show()  %% Cell type:code id: tags:  python plt.figure(figsize=(15,10)) plt.plot(s_data - s_data, 'r', label="Signal", linewidth=3) plt.plot(d_data - s_data, 'k.', label="Data") plt.plot(m_data - s_data, 'k', label="Reconstruction",linewidth=3) plt.axhspan(-noise_amplitude,noise_amplitude, facecolor='0.9', alpha=.5) plt.title("Residuals") plt.legend() plt.show()  %% Cell type:markdown id: tags: ### Power Spectrum %% Cell type:code id: tags:  python s_power_data = ift.power_analyze(sh).to_global_data() m_power_data = ift.power_analyze(m).to_global_data() plt.figure(figsize=(15,10)) plt.loglog() plt.xlim(1, int(N_pixels/2)) ymin = min(m_power_data) plt.ylim(ymin, 1) xs = np.arange(1,int(N_pixels/2),.1) plt.plot(xs, pow_spec(xs), label="True Power Spectrum", color='k',alpha=0.5) plt.plot(s_power_data, 'r', label="Signal") plt.plot(m_power_data, 'k', label="Reconstruction") plt.axhline(noise_amplitude**2 / N_pixels, color="k", linestyle='--', label="Noise level", alpha=.5) plt.axhspan(noise_amplitude**2 / N_pixels, ymin, facecolor='0.9', alpha=.5) plt.title("Power Spectrum") plt.legend() plt.show()  %% Cell type:markdown id: tags: ## Wiener Filter on Incomplete Data %% Cell type:code id: tags:  python # Operators Sh = ift.create_power_operator(h_space, power_spectrum=pow_spec) N = ift.ScalingOperator(noise_amplitude**2,s_space) # R is defined below # Fields sh = Sh.draw_sample() s = HT(sh) n = ift.Field.from_random(domain=s_space, random_type='normal', std=noise_amplitude, mean=0)  %% Cell type:markdown id: tags: ### Partially Lose Data %% Cell type:code id: tags:  python l = int(N_pixels * 0.2) h = int(N_pixels * 0.2 * 2) mask = np.full(s_space.shape, 1.) mask[l:h] = 0 mask = ift.Field.from_global_data(s_space, mask) R = ift.DiagonalOperator(mask)*HT n = n.to_global_data() n = n.to_global_data().copy() n[l:h] = 0 n = ift.Field.from_global_data(s_space, n) d = R(sh) + n  %% Cell type:code id: tags:  python curv = Curvature(R=R, N=N, Sh=Sh) D = curv.inverse j = R.adjoint_times(N.inverse_times(d)) m = D(j)  %% Cell type:markdown id: tags: ### Compute Uncertainty %% Cell type:code id: tags:  python m_mean, m_var = ift.probe_with_posterior_samples(curv, HT, 200)  %% Cell type:markdown id: tags: ### Get data %% Cell type:code id: tags:  python # Get signal data and reconstruction data s_data = s.to_global_data() m_data = HT(m).to_global_data() m_var_data = m_var.to_global_data() uncertainty = np.sqrt(m_var_data) d_data = d.to_global_data() d_data = d.to_global_data().copy() # Set lost data to NaN for proper plotting d_data[d_data == 0] = np.nan  %% Cell type:code id: tags:  python fig = plt.figure(figsize=(15,10)) plt.axvspan(l, h, facecolor='0.8',alpha=0.5) plt.fill_between(range(N_pixels), m_data - uncertainty, m_data + uncertainty, facecolor='0.5', alpha=0.5) plt.plot(s_data, 'r', label="Signal", alpha=1, linewidth=3) plt.plot(d_data, 'k.', label="Data") plt.plot(m_data, 'k', label="Reconstruction", linewidth=3) plt.title("Reconstruction of incomplete data") plt.legend()  %% Cell type:markdown id: tags: # 2d Example %% Cell type:code id: tags:  python N_pixels = 256 # Number of pixels sigma2 = 2. # Noise variance def pow_spec(k): P0, k0, gamma = [.2, 2, 4] return P0 * (1. + (k/k0)**2)**(-gamma/2) s_space = ift.RGSpace([N_pixels, N_pixels])  %% Cell type:code id: tags:  python h_space = s_space.get_default_codomain() HT = ift.HarmonicTransformOperator(h_space,s_space) # Operators Sh = ift.create_power_operator(h_space, power_spectrum=pow_spec) N = ift.ScalingOperator(sigma2,s_space) # Fields and data sh = Sh.draw_sample() n = ift.Field.from_random(domain=s_space, random_type='normal', std=np.sqrt(sigma2), mean=0) # Lose some data l = int(N_pixels * 0.33) h = int(N_pixels * 0.33 * 2) mask = np.full(s_space.shape, 1.) mask[l:h,l:h] = 0. mask = ift.Field.from_global_data(s_space, mask) R = ift.DiagonalOperator(mask)*HT n = n.to_global_data() n = n.to_global_data().copy() n[l:h, l:h] = 0 n = ift.Field.from_global_data(s_space, n) curv = Curvature(R=R, N=N, Sh=Sh) D = curv.inverse d = R(sh) + n j = R.adjoint_times(N.inverse_times(d)) # Run Wiener filter m = D(j) # Uncertainty m_mean, m_var = ift.probe_with_posterior_samples(curv, HT, 20) # Get data s_data = HT(sh).to_global_data() m_data = HT(m).to_global_data() m_var_data = m_var.to_global_data() d_data = d.to_global_data() uncertainty = np.sqrt(np.abs(m_var_data))  %% Cell type:code id: tags:  python cm = ['magma', 'inferno', 'plasma', 'viridis'][1] mi = np.min(s_data) ma = np.max(s_data) fig, axes = plt.subplots(1, 2, figsize=(15, 7)) data = [s_data, d_data] caption = ["Signal", "Data"] for ax in axes.flat: im = ax.imshow(data.pop(0), interpolation='nearest', cmap=cm, vmin=mi, vmax=ma) ax.set_title(caption.pop(0)) fig.subplots_adjust(right=0.8) cbar_ax = fig.add_axes([0.85, 0.15, 0.05, 0.7]) fig.colorbar(im, cax=cbar_ax)  %% Cell type:code id: tags:  python mi = np.min(s_data) ma = np.max(s_data) fig, axes = plt.subplots(3, 2, figsize=(15, 22.5)) sample = HT(curv.draw_sample(from_inverse=True)+m).to_global_data() post_mean = (m_mean + HT(m)).to_global_data() data = [s_data, m_data, post_mean, sample, s_data - m_data, uncertainty] caption = ["Signal", "Reconstruction", "Posterior mean", "Sample", "Residuals", "Uncertainty Map"] for ax in axes.flat: im = ax.imshow(data.pop(0), interpolation='nearest', cmap=cm, vmin=mi, vmax=ma) ax.set_title(caption.pop(0)) fig.subplots_adjust(right=0.8) cbar_ax = fig.add_axes([.85, 0.15, 0.05, 0.7]) fig.colorbar(im, cax=cbar_ax)  %% Cell type:markdown id: tags: ### Is the uncertainty map reliable? %% Cell type:code id: tags:  python precise = (np.abs(s_data-m_data) < uncertainty) print("Error within uncertainty map bounds: " + str(np.sum(precise) * 100 / N_pixels**2) + "%") plt.figure(figsize=(15,10)) plt.imshow(precise.astype(float), cmap="brg") plt.colorbar()  %% Cell type:markdown id: tags: # Start Coding ## NIFTy Repository + Installation guide https://gitlab.mpcdf.mpg.de/ift/NIFTy NIFTy v5 **more or less stable!** ... ...
 ... ... @@ -5,7 +5,6 @@ import numpy as np def get_random_LOS(n_los): starts = list(np.random.uniform(0, 1, (n_los, 2)).T) ends = list(np.random.uniform(0, 1, (n_los, 2)).T) return starts, ends ... ... @@ -87,20 +86,12 @@ if __name__ == '__main__': ift.plot([A.at(position).value, A.at(MOCK_POSITION).value], name='power.pdf') avrg = 0. va = 0. powers = [] sc = ift.StatCalculator() for sample in samples: sam = signal.at(sample + position).value powers.append(A.at(sample+position).value) avrg += sam va += sam**2 avrg /= len(samples) va /= len(samples) va -= avrg**2 std = ift.sqrt(va) ift.plot(avrg, name='avrg.pdf') ift.plot(std, name='std.pdf') sc.add(signal.at(sample+position).value) ift.plot(sc.mean, name='avrg.pdf') ift.plot(ift.sqrt(sc.var), name='std.pdf') powers = [A.at(s+position).value for s in samples] ift.plot([A.at(position).value, A.at(MOCK_POSITION).value]+powers, name='power.pdf')
 ... ... @@ -142,12 +142,11 @@ There is also a set of convenience functions to generate fields with constant values or fields filled with random numbers according to a user-specified distribution. Fields are the only fundamental NIFTy objects which can change state after they have been constructed: while their data type, domain, and array shape cannot be modified, the actual data content of the array may be manipulated during the lifetime of the object. This is a slight deviation from the philosophy that all NIFTy objects should be immutable, but this choice offers considerable performance benefits. Like almost all NIFTy objects, fields are immutable: their value or any other attribute cannot be modified after construction. To manipulate a field in ways that are not covered by the provided standard operations, its data content must be extracted first, then changed, and a new field has to be created from the result. Linear Operators ... ...
 ... ... @@ -54,7 +54,8 @@ extensions = [ 'sphinx.ext.napoleon', # 'sphinx.ext.coverage', # 'sphinx.ext.todo', 'sphinx.ext.mathjax', # 'sphinx.ext.mathjax', 'sphinx.ext.imgmath', 'sphinx.ext.viewcode' ] ... ... @@ -82,9 +83,9 @@ author = u'Theo Steininger / Martin Reinecke' # built documents. # # The short X.Y version. version = u'4.0' version = u'5.0' # The full version, including alpha/beta/rc tags. release = u'4.0.0' release = u'5.0.0' # The language for content autogenerated by Sphinx. Refer to documentation # for a list of supported languages. ... ...
 ... ... @@ -89,7 +89,7 @@ from .library.bernoulli_energy import BernoulliEnergy from . import extra from .utilities import memo from .utilities import memo, frozendict from .logger import logger ... ...
 ... ... @@ -16,11 +16,13 @@ # NIFTy is being developed at the Max-Planck-Institut fuer Astrophysik # and financially supported by the Studienstiftung des deutschen Volkes. from __future__ import (absolute_import, division, print_function) from builtins import * from functools import reduce import numpy as np from .random import Random from mpi4py import MPI import sys from functools import reduce _comm = MPI.COMM_WORLD ntask = _comm.Get_size() ... ... @@ -62,6 +64,9 @@ class data_object(object): if local_shape(self._shape, self._distaxis) != self._data.shape: raise ValueError("shape mismatch") def copy(self): return data_object(self._shape, self._data.copy(), self._distaxis) # def _sanity_checks(self): # # check whether the distaxis is consistent # if self._distaxis < -1 or self._distaxis >= len(self._shape): ... ...
 ... ... @@ -16,6 +16,8 @@ # NIFTy is being developed at the Max-Planck-Institut fuer Astrophysik # and financially supported by the Studienstiftung des deutschen Volkes. from __future__ import (absolute_import, division, print_function) from builtins import * from functools import reduce from .domains.domain import Domain ... ... @@ -104,6 +106,16 @@ class DomainTuple(object): """ return self._shape @property def local_shape(self): """tuple of int: number of pixels along each axis on the local task The shape of the array-like object required to store information living on part of the domain which is stored on the local MPI task. """ from .dobj import local_shape return local_shape(self._shape) @property def size(self): """int : total number of pixels. ... ...
 ... ... @@ -88,6 +88,16 @@ class Domain(NiftyMetaBase()): """ raise NotImplementedError @property def local_shape(self): """tuple of int: number of pixels along each axis on the local task The shape of the array-like object required to store information living on part of the domain which is stored on the local MPI task. """ from ..dobj import local_shape return local_shape(self.shape) @abc.abstractproperty def size(self): """int: total number of pixels. ... ...
 ... ... @@ -101,12 +101,7 @@ class LMSpace(StructuredDomain): # by Challinor et al. # http://arxiv.org/abs/astro-ph/0008228 from ..sugar import exp res = x+1. res *= x res *= -0.5*sigma*sigma exp(res, out=res) return res return exp((x+1.) * x * (-0.5*sigma*sigma)) def get_fft_smoothing_kernel_function(self, sigma):