Commit 3d9bc2a7 authored by Theo Steininger's avatar Theo Steininger
Browse files

Merge branch 'power_spectra_smoothing_v2' into 'master'

Power spectra smoothing v2



See merge request !36
parents 90d6e2f7 1c53be26
......@@ -59,14 +59,14 @@ class FFTOperator(LinearOperator):
forward_class = self.transformation_dictionary[
(self.domain[0].__class__, self.target[0].__class__)]
except KeyError:
raise TypeError(
raise ValueError(
"No forward transformation for domain-target pair "
"found.")
try:
backward_class = self.transformation_dictionary[
(self.target[0].__class__, self.domain[0].__class__)]
except KeyError:
raise TypeError(
raise ValueError(
"No backward transformation for domain-target pair "
"found.")
......@@ -156,13 +156,13 @@ class FFTOperator(LinearOperator):
try:
codomain_class = cls.default_codomain_dictionary[domain_class]
except KeyError:
raise TypeError("unknown domain")
raise ValueError("Unknown domain")
try:
transform_class = cls.transformation_dictionary[(domain_class,
codomain_class)]
except KeyError:
raise TypeError(
raise ValueError(
"No transformation for domain-codomain pair found.")
return transform_class.get_codomain(domain)
#cython: nonecheck=False
#cython: boundscheck=True
#cython: wraparound=False
#cython: cdivision=True
import numpy as np
cimport numpy as np
#defining NPY_NO_DEPRECATED_API NPY_1_7_API_VERSION
cdef int SIGMA_RANGE = 3
cpdef np.float32_t gaussianKernel_f(np.ndarray[np.float32_t, ndim=1] mpower,
np.ndarray[np.float32_t, ndim=1] mk,
np.float32_t mu,
np.float32_t smooth_length):
cdef np.ndarray[np.float32_t, ndim=1] C = \
np.exp(-(mk - mu) ** 2 / (2. * smooth_length ** 2))
return np.sum(C * mpower) / np.sum(C)
cpdef np.float64_t gaussianKernel(np.ndarray[np.float64_t, ndim=1] mpower,
np.ndarray[np.float64_t, ndim=1] mk,
np.float64_t mu,
np.float64_t smooth_length):
cdef np.ndarray[np.float64_t, ndim=1] C = \
np.exp(-(mk - mu) ** 2 / (2. * smooth_length ** 2))
return np.sum(C * mpower) / np.sum(C)
cpdef np.ndarray[np.float64_t, ndim=1] apply_kernel_along_array(
np.ndarray[np.float64_t, ndim=1] power,
np.int_t startindex,
np.int_t endindex,
np.ndarray[np.float64_t, ndim=1] distances,
np.float64_t smooth_length,
np.float64_t smoothing_width):
if smooth_length == 0.0:
return power[startindex:endindex]
if (smooth_length is None) or (smooth_length < 0):
smooth_length = distances[1]-distances[0]
cdef np.ndarray[np.float64_t, ndim=1] p_smooth = \
np.empty(endindex-startindex, dtype=np.float64)
cdef np.int64_t i, l, u
for i in xrange(startindex, endindex):
current_location = distances[i]
l = np.searchsorted(distances,
current_location - smoothing_width*smooth_length)
u = np.searchsorted(distances,
current_location + smoothing_width*smooth_length)
p_smooth[i-startindex] = gaussianKernel(power[l:u],
distances[l:u],
distances[i],
smooth_length)
return p_smooth
cpdef np.ndarray[np.float32_t, ndim=1] apply_kernel_along_array_f(
np.ndarray[np.float32_t, ndim=1] power,
np.int_t startindex,
np.int_t endindex,
np.ndarray[np.float32_t, ndim=1] distances,
np.float64_t smooth_length,
np.float64_t smoothing_width):
if smooth_length == 0.0:
return power[startindex:endindex]
if (smooth_length is None) or (smooth_length < 0):
smooth_length = distances[1]-distances[0]
cdef np.ndarray[np.float32_t, ndim=1] p_smooth = \
np.empty(endindex-startindex, dtype=np.float64)
cdef np.int64_t i, l, u
for i in xrange(startindex, endindex):
current_location = distances[i]
l = np.searchsorted(distances,
current_location - smoothing_width*smooth_length)
u = np.searchsorted(distances,
current_location + smoothing_width*smooth_length)
p_smooth[i-startindex] = gaussianKernel(power[l:u],
distances[l:u],
distances[i],
smooth_length)
return p_smooth
def getShape(a):
return tuple(a.shape)
cpdef np.ndarray[np.float64_t, ndim=1] apply_along_axis(
np.int_t axis,
np.ndarray arr,
np.int_t startindex,
np.int_t endindex,
np.ndarray[np.float64_t, ndim=1] distances,
np.float64_t smooth_length,
np.float64_t smoothing_width):
cdef np.int_t nd = arr.ndim
if axis < 0:
axis += nd
if (axis >= nd):
raise ValueError("axis must be less than arr.ndim; axis=%d, rank=%d."
% (axis, nd))
cdef np.ndarray[np.int_t, ndim=1] ind = np.zeros(nd-1, dtype=np.int)
cdef np.ndarray i = np.zeros(nd, dtype=object)
cdef tuple shape = getShape(arr)
cdef np.ndarray[np.int_t, ndim=1] indlist = np.asarray(range(nd))
indlist = np.delete(indlist,axis)
i[axis] = slice(None, None)
cdef np.ndarray[np.int_t, ndim=1] outshape = \
np.asarray(shape).take(indlist)
i.put(indlist, ind)
cdef np.ndarray[np.float64_t, ndim=1] slicedArr
cdef np.ndarray[np.float64_t, ndim=1] res
cdef np.int_t Ntot = np.product(outshape)
cdef np.ndarray[np.int_t, ndim=1] holdshape = outshape
slicedArr = arr[tuple(i.tolist())]
res = apply_kernel_along_array(slicedArr,
startindex,
endindex,
distances,
smooth_length,
smoothing_width)
outshape = np.asarray(getShape(arr))
outshape[axis] = endindex - startindex
outarr = np.zeros(outshape, dtype=np.float64)
outarr[tuple(i.tolist())] = res
cdef np.int_t n, k = 1
while k < Ntot:
# increment the index
ind[nd-1] += 1
n = -1
while (ind[n] >= holdshape[n]) and (n > (1-nd)):
ind[n-1] += 1
ind[n] = 0
n -= 1
i.put(indlist, ind)
slicedArr = arr[tuple(i.tolist())]
res = apply_kernel_along_array(slicedArr,
startindex,
endindex,
distances,
smooth_length,
smoothing_width)
outarr[tuple(i.tolist())] = res
k += 1
return outarr
cpdef np.ndarray[np.float32_t, ndim=1] apply_along_axis_f(
np.int_t axis,
np.ndarray arr,
np.int_t startindex,
np.int_t endindex,
np.ndarray[np.float32_t, ndim=1] distances,
np.float64_t smooth_length,
np.float64_t smoothing_width):
cdef np.int_t nd = arr.ndim
if axis < 0:
axis += nd
if (axis >= nd):
raise ValueError("axis must be less than arr.ndim; axis=%d, rank=%d."
% (axis, nd))
cdef np.ndarray[np.int_t, ndim=1] ind = np.zeros(nd-1, dtype=np.int)
cdef np.ndarray i = np.zeros(nd, dtype=object)
cdef tuple shape = getShape(arr)
cdef np.ndarray[np.int_t, ndim=1] indlist = np.asarray(range(nd))
indlist = np.delete(indlist,axis)
i[axis] = slice(None, None)
cdef np.ndarray[np.int_t, ndim=1] outshape = \
np.asarray(shape).take(indlist)
i.put(indlist, ind)
cdef np.ndarray[np.float32_t, ndim=1] slicedArr
cdef np.ndarray[np.float32_t, ndim=1] res
cdef np.int_t Ntot = np.product(outshape)
cdef np.ndarray[np.int_t, ndim=1] holdshape = outshape
slicedArr = arr[tuple(i.tolist())]
res = apply_kernel_along_array_f(slicedArr,
startindex,
endindex,
distances,
smooth_length,
smoothing_width)
outshape = np.asarray(getShape(arr))
outshape[axis] = endindex - startindex
outarr = np.zeros(outshape, dtype=np.float32)
outarr[tuple(i.tolist())] = res
cdef np.int_t n, k = 1
while k < Ntot:
# increment the index
ind[nd-1] += 1
n = -1
while (ind[n] >= holdshape[n]) and (n > (1-nd)):
ind[n-1] += 1
ind[n] = 0
n -= 1
i.put(indlist, ind)
slicedArr = arr[tuple(i.tolist())]
res = apply_kernel_along_array(slicedArr,
startindex,
endindex,
distances,
smooth_length,
smoothing_width)
outarr[tuple(i.tolist())] = res
k += 1
return outarr
import numpy as np
import nifty.nifty_utilities as utilities
from nifty import RGSpace, LMSpace
from nifty.operators.endomorphic_operator import EndomorphicOperator
from nifty.operators.fft_operator import FFTOperator
import smooth_util as su
from d2o import STRATEGIES
class SmoothingOperator(EndomorphicOperator):
# ---Overwritten properties and methods---
def __init__(self, domain=(), field_type=(), sigma=0):
def __init__(self, domain=(), field_type=(), sigma=0,
log_distances=False):
self._domain = self._parse_domain(domain)
self._field_type = self._parse_field_type(field_type)
......@@ -17,8 +18,8 @@ class SmoothingOperator(EndomorphicOperator):
if len(self.domain) != 1:
raise ValueError(
'ERROR: SmoothOperator accepts only exactly one '
'space as input domain.'
'ERROR: SmoothOperator accepts only exactly one '
'space as input domain.'
)
if self.field_type != ():
......@@ -27,13 +28,16 @@ class SmoothingOperator(EndomorphicOperator):
'empty tuple.'
)
self._sigma = sigma
self.sigma = sigma
self.log_distances = log_distances
self._direct_smoothing_width = 3.
def _inverse_times(self, x, spaces, types):
return self._smooth_helper(x, spaces, types, inverse=True)
return self._smoothing_helper(x, spaces, inverse=True)
def _times(self, x, spaces, types):
return self._smooth_helper(x, spaces, types)
return self._smoothing_helper(x, spaces, inverse=False)
# ---Mandatory properties and methods---
@property
......@@ -50,18 +54,31 @@ class SmoothingOperator(EndomorphicOperator):
@property
def symmetric(self):
return True
return False
@property
def unitary(self):
return False
# ---Added properties and methods---
@property
def sigma(self):
return self._sigma
def _smooth_helper(self, x, spaces, types, inverse=False):
@sigma.setter
def sigma(self, sigma):
self._sigma = np.float(sigma)
@property
def log_distances(self):
return self._log_distances
@log_distances.setter
def log_distances(self, log_distances):
self._log_distances = bool(log_distances)
def _smoothing_helper(self, x, spaces, inverse):
if self.sigma == 0:
return x.copy()
......@@ -74,6 +91,13 @@ class SmoothingOperator(EndomorphicOperator):
else:
spaces = utilities.cast_axis_to_tuple(spaces, len(x.domain))
try:
result = self._fft_smoothing(x, spaces, inverse)
except ValueError:
result = self._direct_smoothing(x, spaces, inverse)
return result
def _fft_smoothing(self, x, spaces, inverse):
Transformator = FFTOperator(x.domain[spaces[0]])
# transform to the (global-)default codomain and perform all remaining
......@@ -87,9 +111,13 @@ class SmoothingOperator(EndomorphicOperator):
transformed_x.val.get_axes_local_distribution_strategy(axes=coaxes)
kernel = codomain.get_distance_array(
distribution_strategy=axes_local_distribution_strategy)
distribution_strategy=axes_local_distribution_strategy)
if self.log_distances:
kernel.apply_scalar_function(np.log, inplace=True)
kernel.apply_scalar_function(
codomain.get_smoothing_kernel_function(self.sigma),
codomain.get_fft_smoothing_kernel_function(self.sigma),
inplace=True)
# now, apply the kernel to transformed_x
......@@ -98,7 +126,7 @@ class SmoothingOperator(EndomorphicOperator):
local_transformed_x = transformed_x.val.get_local_data(copy=False)
local_kernel = kernel.get_local_data(copy=False)
reshaper = [local_transformed_x.shape[i] if i in coaxes else 1
reshaper = [transformed_x.shape[i] if i in coaxes else 1
for i in xrange(len(transformed_x.shape))]
local_kernel = np.reshape(local_kernel, reshaper)
......@@ -113,3 +141,127 @@ class SmoothingOperator(EndomorphicOperator):
result = Transformator.inverse_times(transformed_x, spaces=spaces)
return result
def _direct_smoothing(self, x, spaces, inverse):
# infer affected axes
# we rely on the knowledge, that `spaces` is a tuple with length 1.
affected_axes = x.domain_axes[spaces[0]]
if len(affected_axes) > 1:
raise ValueError("By this implementation only one-dimensional "
"spaces can be smoothed directly.")
affected_axis = affected_axes[0]
distance_array = x.domain[spaces[0]].get_distance_array(
distribution_strategy='not')
distance_array = distance_array.get_local_data(copy=False)
if self.log_distances:
np.log(distance_array, out=distance_array)
# collect the local data + ghost cells
local_data_Q = False
if x.distribution_strategy == 'not':
local_data_Q = True
elif x.distribution_strategy in STRATEGIES['slicing']:
# infer the local start/end based on the slicing information of
# x's d2o. Only gets non-trivial for axis==0.
if 0 != affected_axis:
local_data_Q = True
else:
start_index = x.val.distributor.local_start
start_distance = distance_array[start_index]
augmented_start_distance = \
(start_distance - self._direct_smoothing_width*self.sigma)
augmented_start_index = \
np.searchsorted(distance_array, augmented_start_distance)
true_start = start_index - augmented_start_index
end_index = x.val.distributor.local_end
end_distance = distance_array[end_index-1]
augmented_end_distance = \
(end_distance + self._direct_smoothing_width*self.sigma)
augmented_end_index = \
np.searchsorted(distance_array, augmented_end_distance)
true_end = true_start + x.val.distributor.local_length
augmented_slice = slice(augmented_start_index,
augmented_end_index)
augmented_data = x.val.get_data(augmented_slice,
local_keys=True,
copy=False)
augmented_data = augmented_data.get_local_data(copy=False)
augmented_distance_array = distance_array[augmented_slice]
else:
raise ValueError("Direct smoothing not implemented for given"
"distribution strategy.")
if local_data_Q:
# if the needed data resides on the nodes already, the necessary
# are the same; no matter what the distribution strategy was.
augmented_data = x.val.get_local_data(copy=False)
augmented_distance_array = distance_array
true_start = 0
true_end = x.shape[affected_axis]
# perform the convolution along the affected axes
# currently only one axis is supported
data_axis = affected_axes[0]
local_result = self._direct_smoothing_single_axis(
augmented_data,
data_axis,
augmented_distance_array,
true_start,
true_end,
inverse)
result = x.copy_empty()
result.val.set_local_data(local_result, copy=False)
return result
def _direct_smoothing_single_axis(self, data, data_axis, distances,
true_start, true_end, inverse):
if inverse:
true_sigma = 1. / self.sigma
else:
true_sigma = self.sigma
if data.dtype is np.dtype('float32'):
distances = distances.astype(np.float32, copy=False)
smoothed_data = su.apply_along_axis_f(
data_axis, data,
startindex=true_start,
endindex=true_end,
distances=distances,
smooth_length=true_sigma,
smoothing_width=self._direct_smoothing_width)
elif data.dtype is np.dtype('float64'):
distances = distances.astype(np.float64, copy=False)
smoothed_data = su.apply_along_axis(
data_axis, data,
startindex=true_start,
endindex=true_end,
distances=distances,
smooth_length=true_sigma,
smoothing_width=self._direct_smoothing_width)
elif np.issubdtype(data.dtype, np.complexfloating):
real = self._direct_smoothing_single_axis(data.real,
data_axis,
distances,
true_start,
true_end, inverse)
imag = self._direct_smoothing_single_axis(data.imag,
data_axis,
distances,
true_start,
true_end, inverse)
return real + 1j*imag
else:
raise TypeError("Dtype %s not supported" % str(data.dtype))
return smoothed_data
......@@ -3,7 +3,8 @@ from __future__ import division
import itertools
import numpy as np
from d2o import arange, STRATEGIES as DISTRIBUTION_STRATEGIES
import d2o
from d2o import STRATEGIES as DISTRIBUTION_STRATEGIES
from nifty.spaces.space import Space
from nifty.config import nifty_configuration as gc,\
......@@ -154,8 +155,8 @@ class GLSpace(Space):
return result_x
def get_distance_array(self, distribution_strategy):
dists = arange(start=0, stop=self.shape[0],
distribution_strategy=distribution_strategy)
dists = d2o.arange(start=0, stop=self.shape[0],
distribution_strategy=distribution_strategy)
dists = dists.apply_scalar_function(
lambda x: self._distance_array_helper(divmod(x, self.nlon)),
......@@ -172,7 +173,7 @@ class GLSpace(Space):
return np.arctan(numerator / denominator)
def get_smoothing_kernel_function(self, sigma):
def get_fft_smoothing_kernel_function(self, sigma):
if sigma is None:
sigma = np.sqrt(2) * np.pi
......
......@@ -35,6 +35,8 @@ from __future__ import division
import numpy as np
import d2o
from nifty.spaces.space import Space
from nifty.config import nifty_configuration as gc, \
dependency_injector as gdi
......@@ -169,7 +171,7 @@ class HPSpace(Space):
-------
dists: distributed_data_object
"""
dists = arange(
dists = d2o.arange(
start=0, stop=self.shape[0],
distribution_strategy=distribution_strategy
)
......@@ -183,7 +185,7 @@ class HPSpace(Space):
return dists
def get_smoothing_kernel_function(self, sigma):
def get_fft_smoothing_kernel_function(self, sigma):
if sigma is None:
sigma = np.sqrt(2) * np.pi
......
......@@ -156,7 +156,7 @@ class LMSpace(Space):
return dists
def get_smoothing_kernel_function(self, sigma):
def get_fft_smoothing_kernel_function(self, sigma):
if sigma is None:
sigma = np.sqrt(2) * np.pi / (self.lmax + 1)
......
......@@ -2,6 +2,8 @@
import numpy as np
import d2o
from power_index_factory import PowerIndexFactory
from nifty.spaces.space import Space
......@@ -106,13 +108,14 @@ class PowerSpace(Space):
return result_x
def get_distance_array(self, distribution_strategy):
raise NotImplementedError(
"There is no get_distance_array implementation for "
"PowerSpace.")
result = d2o.distributed_data_object(
self.kindex,
distribution_strategy=distribution_strategy)