Merge branch 'power_spectra_smoothing_v2' into 'master'

Power spectra smoothing v2



See merge request !36

nifty/operators/fft_operator/fft_operator.py

@@ -59,14 +59,14 @@ class FFTOperator(LinearOperator):
             forward_class = self.transformation_dictionary[
                 (self.domain[0].__class__, self.target[0].__class__)]
         except KeyError:
-            raise TypeError(
+            raise ValueError(
                 "No forward transformation for domain-target pair "
                 "found.")
         try:
             backward_class = self.transformation_dictionary[
                 (self.target[0].__class__, self.domain[0].__class__)]
         except KeyError:
-            raise TypeError(
+            raise ValueError(
                 "No backward transformation for domain-target pair "
                 "found.")
 
@@ -156,13 +156,13 @@ class FFTOperator(LinearOperator):
         try:
             codomain_class = cls.default_codomain_dictionary[domain_class]
         except KeyError:
-            raise TypeError("unknown domain")
+            raise ValueError("Unknown domain")
 
         try:
             transform_class = cls.transformation_dictionary[(domain_class,
                                                              codomain_class)]
         except KeyError:
-            raise TypeError(
+            raise ValueError(
                 "No transformation for domain-codomain pair found.")
 
         return transform_class.get_codomain(domain)

nifty/operators/smoothing_operator/smooth_util.pyx

+#cython: nonecheck=False
+#cython: boundscheck=True
+#cython: wraparound=False
+#cython: cdivision=True
+
+import numpy as np
+cimport numpy as np
+#defining NPY_NO_DEPRECATED_API NPY_1_7_API_VERSION
+
+cdef int SIGMA_RANGE = 3
+
+cpdef np.float32_t gaussianKernel_f(np.ndarray[np.float32_t, ndim=1] mpower,
+                                    np.ndarray[np.float32_t, ndim=1] mk,
+                                    np.float32_t mu,
+                                    np.float32_t smooth_length):
+    cdef np.ndarray[np.float32_t, ndim=1] C = \
+        np.exp(-(mk - mu) ** 2 / (2. * smooth_length ** 2))
+    return np.sum(C * mpower) / np.sum(C)
+
+cpdef np.float64_t gaussianKernel(np.ndarray[np.float64_t, ndim=1] mpower,
+                                  np.ndarray[np.float64_t, ndim=1] mk,
+                                  np.float64_t mu,
+                                  np.float64_t smooth_length):
+    cdef np.ndarray[np.float64_t, ndim=1] C = \
+        np.exp(-(mk - mu) ** 2 / (2. * smooth_length ** 2))
+    return np.sum(C * mpower) / np.sum(C)
+
+
+cpdef np.ndarray[np.float64_t, ndim=1] apply_kernel_along_array(
+                                    np.ndarray[np.float64_t, ndim=1] power,
+                                    np.int_t startindex,
+                                    np.int_t endindex,
+                                    np.ndarray[np.float64_t, ndim=1] distances,
+                                    np.float64_t smooth_length,
+                                    np.float64_t smoothing_width):
+
+    if smooth_length == 0.0:
+        return power[startindex:endindex]
+
+    if (smooth_length is None) or (smooth_length < 0):
+        smooth_length = distances[1]-distances[0]
+
+    cdef np.ndarray[np.float64_t, ndim=1] p_smooth = \
+        np.empty(endindex-startindex, dtype=np.float64)
+    cdef np.int64_t i, l, u
+    for i in xrange(startindex, endindex):
+        current_location = distances[i]
+        l = np.searchsorted(distances,
+                            current_location - smoothing_width*smooth_length)
+        u = np.searchsorted(distances,
+                            current_location + smoothing_width*smooth_length)
+        p_smooth[i-startindex] = gaussianKernel(power[l:u],
+                                                distances[l:u],
+                                                distances[i],
+                                                smooth_length)
+
+    return p_smooth
+
+cpdef np.ndarray[np.float32_t, ndim=1] apply_kernel_along_array_f(
+                                    np.ndarray[np.float32_t, ndim=1] power,
+                                    np.int_t startindex,
+                                    np.int_t endindex,
+                                    np.ndarray[np.float32_t, ndim=1] distances,
+                                    np.float64_t smooth_length,
+                                    np.float64_t smoothing_width):
+
+    if smooth_length == 0.0:
+        return power[startindex:endindex]
+
+    if (smooth_length is None) or (smooth_length < 0):
+        smooth_length = distances[1]-distances[0]
+
+    cdef np.ndarray[np.float32_t, ndim=1] p_smooth = \
+        np.empty(endindex-startindex, dtype=np.float64)
+    cdef np.int64_t i, l, u
+    for i in xrange(startindex, endindex):
+        current_location = distances[i]
+        l = np.searchsorted(distances,
+                            current_location - smoothing_width*smooth_length)
+        u = np.searchsorted(distances,
+                            current_location + smoothing_width*smooth_length)
+        p_smooth[i-startindex] = gaussianKernel(power[l:u],
+                                                distances[l:u],
+                                                distances[i],
+                                                smooth_length)
+
+    return p_smooth
+
+def getShape(a):
+    return tuple(a.shape)
+
+cpdef np.ndarray[np.float64_t, ndim=1] apply_along_axis(
+                                    np.int_t axis,
+                                    np.ndarray arr,
+                                    np.int_t startindex,
+                                    np.int_t endindex,
+                                    np.ndarray[np.float64_t, ndim=1] distances,
+                                    np.float64_t smooth_length,
+                                    np.float64_t smoothing_width):
+    cdef np.int_t nd = arr.ndim
+    if axis < 0:
+        axis += nd
+    if (axis >= nd):
+        raise ValueError("axis must be less than arr.ndim; axis=%d, rank=%d."
+            % (axis, nd))
+    cdef np.ndarray[np.int_t, ndim=1] ind = np.zeros(nd-1, dtype=np.int)
+    cdef np.ndarray i = np.zeros(nd, dtype=object)
+    cdef tuple shape = getShape(arr)
+    cdef np.ndarray[np.int_t, ndim=1] indlist = np.asarray(range(nd))
+    indlist = np.delete(indlist,axis)
+    i[axis] = slice(None, None)
+    cdef np.ndarray[np.int_t, ndim=1] outshape = \
+        np.asarray(shape).take(indlist)
+
+    i.put(indlist, ind)
+
+    cdef np.ndarray[np.float64_t, ndim=1] slicedArr
+    cdef np.ndarray[np.float64_t, ndim=1] res
+
+    cdef np.int_t Ntot = np.product(outshape)
+    cdef np.ndarray[np.int_t, ndim=1] holdshape = outshape
+    slicedArr = arr[tuple(i.tolist())]
+
+    res = apply_kernel_along_array(slicedArr,
+                                   startindex,
+                                   endindex,
+                                   distances,
+                                   smooth_length,
+                                   smoothing_width)
+
+    outshape = np.asarray(getShape(arr))
+    outshape[axis] = endindex - startindex
+    outarr = np.zeros(outshape, dtype=np.float64)
+    outarr[tuple(i.tolist())] = res
+    cdef np.int_t n, k = 1
+    while k < Ntot:
+        # increment the index
+        ind[nd-1] += 1
+        n = -1
+        while (ind[n] >= holdshape[n]) and (n > (1-nd)):
+            ind[n-1] += 1
+            ind[n] = 0
+            n -= 1
+        i.put(indlist, ind)
+        slicedArr = arr[tuple(i.tolist())]
+        res = apply_kernel_along_array(slicedArr,
+                                       startindex,
+                                       endindex,
+                                       distances,
+                                       smooth_length,
+                                       smoothing_width)
+        outarr[tuple(i.tolist())] = res
+        k += 1
+
+    return outarr
+
+
+cpdef np.ndarray[np.float32_t, ndim=1] apply_along_axis_f(
+                                    np.int_t axis,
+                                    np.ndarray arr,
+                                    np.int_t startindex,
+                                    np.int_t endindex,
+                                    np.ndarray[np.float32_t, ndim=1] distances,
+                                    np.float64_t smooth_length,
+                                    np.float64_t smoothing_width):
+    cdef np.int_t nd = arr.ndim
+    if axis < 0:
+        axis += nd
+    if (axis >= nd):
+        raise ValueError("axis must be less than arr.ndim; axis=%d, rank=%d."
+            % (axis, nd))
+    cdef np.ndarray[np.int_t, ndim=1] ind = np.zeros(nd-1, dtype=np.int)
+    cdef np.ndarray i = np.zeros(nd, dtype=object)
+    cdef tuple shape = getShape(arr)
+    cdef np.ndarray[np.int_t, ndim=1] indlist = np.asarray(range(nd))
+    indlist = np.delete(indlist,axis)
+    i[axis] = slice(None, None)
+    cdef np.ndarray[np.int_t, ndim=1] outshape = \
+        np.asarray(shape).take(indlist)
+
+    i.put(indlist, ind)
+
+    cdef np.ndarray[np.float32_t, ndim=1] slicedArr
+    cdef np.ndarray[np.float32_t, ndim=1] res
+
+    cdef np.int_t Ntot = np.product(outshape)
+    cdef np.ndarray[np.int_t, ndim=1] holdshape = outshape
+    slicedArr = arr[tuple(i.tolist())]
+
+    res = apply_kernel_along_array_f(slicedArr,
+                                     startindex,
+                                     endindex,
+                                     distances,
+                                     smooth_length,
+                                     smoothing_width)
+
+    outshape = np.asarray(getShape(arr))
+    outshape[axis] = endindex - startindex
+    outarr = np.zeros(outshape, dtype=np.float32)
+    outarr[tuple(i.tolist())] = res
+    cdef np.int_t n, k = 1
+    while k < Ntot:
+        # increment the index
+        ind[nd-1] += 1
+        n = -1
+        while (ind[n] >= holdshape[n]) and (n > (1-nd)):
+            ind[n-1] += 1
+            ind[n] = 0
+            n -= 1
+        i.put(indlist, ind)
+        slicedArr = arr[tuple(i.tolist())]
+        res = apply_kernel_along_array(slicedArr,
+                                       startindex,
+                                       endindex,
+                                       distances,
+                                       smooth_length,
+                                       smoothing_width)
+        outarr[tuple(i.tolist())] = res
+        k += 1
+
+    return outarr
+

nifty/operators/smoothing_operator/smoothing_operator.py

 import numpy as np
 
 import nifty.nifty_utilities as utilities
-from nifty import RGSpace, LMSpace
 from nifty.operators.endomorphic_operator import EndomorphicOperator
 from nifty.operators.fft_operator import FFTOperator
+import smooth_util as su
+from d2o import STRATEGIES
 
 
 class SmoothingOperator(EndomorphicOperator):
-
     # ---Overwritten properties and methods---
-    def __init__(self, domain=(), field_type=(), sigma=0):
+    def __init__(self, domain=(), field_type=(), sigma=0,
+                 log_distances=False):
 
         self._domain = self._parse_domain(domain)
         self._field_type = self._parse_field_type(field_type)
@@ -17,8 +18,8 @@ class SmoothingOperator(EndomorphicOperator):
         if len(self.domain) != 1:
             raise ValueError(
 
-                    'ERROR: SmoothOperator accepts only exactly one '
-                    'space as input domain.'
+                'ERROR: SmoothOperator accepts only exactly one '
+                'space as input domain.'
             )
 
         if self.field_type != ():
@@ -27,13 +28,16 @@ class SmoothingOperator(EndomorphicOperator):
                 'empty tuple.'
             )
 
-        self._sigma = sigma
+        self.sigma = sigma
+        self.log_distances = log_distances
+
+        self._direct_smoothing_width = 3.
 
     def _inverse_times(self, x, spaces, types):
-        return self._smooth_helper(x, spaces, types, inverse=True)
+        return self._smoothing_helper(x, spaces, inverse=True)
 
     def _times(self, x, spaces, types):
-        return self._smooth_helper(x, spaces, types)
+        return self._smoothing_helper(x, spaces, inverse=False)
 
     # ---Mandatory properties and methods---
     @property
@@ -50,18 +54,31 @@ class SmoothingOperator(EndomorphicOperator):
 
     @property
     def symmetric(self):
-        return True
+        return False
 
     @property
     def unitary(self):
         return False
 
     # ---Added properties and methods---
+
     @property
     def sigma(self):
         return self._sigma
 
-    def _smooth_helper(self, x, spaces, types, inverse=False):
+    @sigma.setter
+    def sigma(self, sigma):
+        self._sigma = np.float(sigma)
+
+    @property
+    def log_distances(self):
+        return self._log_distances
+
+    @log_distances.setter
+    def log_distances(self, log_distances):
+        self._log_distances = bool(log_distances)
+
+    def _smoothing_helper(self, x, spaces, inverse):
         if self.sigma == 0:
             return x.copy()
 
@@ -74,6 +91,13 @@ class SmoothingOperator(EndomorphicOperator):
         else:
             spaces = utilities.cast_axis_to_tuple(spaces, len(x.domain))
 
+        try:
+            result = self._fft_smoothing(x, spaces, inverse)
+        except ValueError:
+            result = self._direct_smoothing(x, spaces, inverse)
+        return result
+
+    def _fft_smoothing(self, x, spaces, inverse):
         Transformator = FFTOperator(x.domain[spaces[0]])
 
         # transform to the (global-)default codomain and perform all remaining
@@ -87,9 +111,13 @@ class SmoothingOperator(EndomorphicOperator):
             transformed_x.val.get_axes_local_distribution_strategy(axes=coaxes)
 
         kernel = codomain.get_distance_array(
-                        distribution_strategy=axes_local_distribution_strategy)
+            distribution_strategy=axes_local_distribution_strategy)
+
+        if self.log_distances:
+            kernel.apply_scalar_function(np.log, inplace=True)
+
         kernel.apply_scalar_function(
-            codomain.get_smoothing_kernel_function(self.sigma),
+            codomain.get_fft_smoothing_kernel_function(self.sigma),
             inplace=True)
 
         # now, apply the kernel to transformed_x
@@ -98,7 +126,7 @@ class SmoothingOperator(EndomorphicOperator):
         local_transformed_x = transformed_x.val.get_local_data(copy=False)
         local_kernel = kernel.get_local_data(copy=False)
 
-        reshaper = [local_transformed_x.shape[i] if i in coaxes else 1
+        reshaper = [transformed_x.shape[i] if i in coaxes else 1
                     for i in xrange(len(transformed_x.shape))]
         local_kernel = np.reshape(local_kernel, reshaper)
 
@@ -113,3 +141,127 @@ class SmoothingOperator(EndomorphicOperator):
         result = Transformator.inverse_times(transformed_x, spaces=spaces)
 
         return result
+
+    def _direct_smoothing(self, x, spaces, inverse):
+        # infer affected axes
+        # we rely on the knowledge, that `spaces` is a tuple with length 1.
+        affected_axes = x.domain_axes[spaces[0]]
+
+        if len(affected_axes) > 1:
+            raise ValueError("By this implementation only one-dimensional "
+                             "spaces can be smoothed directly.")
+
+        affected_axis = affected_axes[0]
+
+        distance_array = x.domain[spaces[0]].get_distance_array(
+            distribution_strategy='not')
+        distance_array = distance_array.get_local_data(copy=False)
+
+        if self.log_distances:
+            np.log(distance_array, out=distance_array)
+
+        # collect the local data + ghost cells
+        local_data_Q = False
+
+        if x.distribution_strategy == 'not':
+            local_data_Q = True
+        elif x.distribution_strategy in STRATEGIES['slicing']:
+            # infer the local start/end based on the slicing information of
+            # x's d2o. Only gets non-trivial for axis==0.
+            if 0 != affected_axis:
+                local_data_Q = True
+            else:
+                start_index = x.val.distributor.local_start
+                start_distance = distance_array[start_index]
+                augmented_start_distance = \
+                    (start_distance - self._direct_smoothing_width*self.sigma)
+                augmented_start_index = \
+                    np.searchsorted(distance_array, augmented_start_distance)
+                true_start = start_index - augmented_start_index
+                end_index = x.val.distributor.local_end
+                end_distance = distance_array[end_index-1]
+                augmented_end_distance = \
+                    (end_distance + self._direct_smoothing_width*self.sigma)
+                augmented_end_index = \
+                    np.searchsorted(distance_array, augmented_end_distance)
+                true_end = true_start + x.val.distributor.local_length
+                augmented_slice = slice(augmented_start_index,
+                                        augmented_end_index)
+
+                augmented_data = x.val.get_data(augmented_slice,
+                                                local_keys=True,
+                                                copy=False)
+                augmented_data = augmented_data.get_local_data(copy=False)
+
+                augmented_distance_array = distance_array[augmented_slice]
+
+        else:
+            raise ValueError("Direct smoothing not implemented for given"
+                             "distribution strategy.")
+
+        if local_data_Q:
+            # if the needed data resides on the nodes already, the necessary
+            # are the same; no matter what the distribution strategy was.
+            augmented_data = x.val.get_local_data(copy=False)
+            augmented_distance_array = distance_array
+            true_start = 0
+            true_end = x.shape[affected_axis]
+
+        # perform the convolution along the affected axes
+        # currently only one axis is supported
+        data_axis = affected_axes[0]
+        local_result = self._direct_smoothing_single_axis(
+                                                    augmented_data,
+                                                    data_axis,
+                                                    augmented_distance_array,
+                                                    true_start,
+                                                    true_end,
+                                                    inverse)
+        result = x.copy_empty()
+        result.val.set_local_data(local_result, copy=False)
+        return result
+
+    def _direct_smoothing_single_axis(self, data, data_axis, distances,
+                                      true_start, true_end, inverse):
+        if inverse:
+            true_sigma = 1. / self.sigma
+        else:
+            true_sigma = self.sigma
+
+        if data.dtype is np.dtype('float32'):
+            distances = distances.astype(np.float32, copy=False)
+            smoothed_data = su.apply_along_axis_f(
+                                  data_axis, data,
+                                  startindex=true_start,
+                                  endindex=true_end,
+                                  distances=distances,
+                                  smooth_length=true_sigma,
+                                  smoothing_width=self._direct_smoothing_width)
+        elif data.dtype is np.dtype('float64'):
+            distances = distances.astype(np.float64, copy=False)
+            smoothed_data = su.apply_along_axis(
+                                  data_axis, data,
+                                  startindex=true_start,
+                                  endindex=true_end,
+                                  distances=distances,
+                                  smooth_length=true_sigma,
+                                  smoothing_width=self._direct_smoothing_width)
+
+        elif np.issubdtype(data.dtype, np.complexfloating):
+            real = self._direct_smoothing_single_axis(data.real,
+                                                      data_axis,
+                                                      distances,
+                                                      true_start,
+                                                      true_end, inverse)
+            imag = self._direct_smoothing_single_axis(data.imag,
+                                                      data_axis,
+                                                      distances,
+                                                      true_start,
+                                                      true_end, inverse)
+
+            return real + 1j*imag
+
+        else:
+            raise TypeError("Dtype %s not supported" % str(data.dtype))
+
+        return smoothed_data

nifty/spaces/gl_space/gl_space.py

@@ -3,7 +3,8 @@ from __future__ import division
 import itertools
 import numpy as np
 
-from d2o import arange, STRATEGIES as DISTRIBUTION_STRATEGIES
+import d2o
+from d2o import STRATEGIES as DISTRIBUTION_STRATEGIES
 
 from nifty.spaces.space import Space
 from nifty.config import nifty_configuration as gc,\
@@ -154,8 +155,8 @@ class GLSpace(Space):
         return result_x
 
     def get_distance_array(self, distribution_strategy):
-        dists = arange(start=0, stop=self.shape[0],
-                       distribution_strategy=distribution_strategy)
+        dists = d2o.arange(start=0, stop=self.shape[0],
+                           distribution_strategy=distribution_strategy)
 
         dists = dists.apply_scalar_function(
             lambda x: self._distance_array_helper(divmod(x, self.nlon)),
@@ -172,7 +173,7 @@ class GLSpace(Space):
 
         return np.arctan(numerator / denominator)
 
-    def get_smoothing_kernel_function(self, sigma):
+    def get_fft_smoothing_kernel_function(self, sigma):
         if sigma is None:
             sigma = np.sqrt(2) * np.pi
 

nifty/spaces/hp_space/hp_space.py

@@ -35,6 +35,8 @@ from __future__ import division
 
 import numpy as np
 
+import d2o
+
 from nifty.spaces.space import Space
 from nifty.config import nifty_configuration as gc, \
                          dependency_injector as gdi
@@ -169,7 +171,7 @@ class HPSpace(Space):
         -------
         dists: distributed_data_object
         """
-        dists = arange(
+        dists = d2o.arange(
             start=0, stop=self.shape[0],
             distribution_strategy=distribution_strategy
         )
@@ -183,7 +185,7 @@ class HPSpace(Space):
 
         return dists
 
-    def get_smoothing_kernel_function(self, sigma):
+    def get_fft_smoothing_kernel_function(self, sigma):
         if sigma is None:
             sigma = np.sqrt(2) * np.pi
 

nifty/spaces/lm_space/lm_space.py

@@ -156,7 +156,7 @@ class LMSpace(Space):
 
         return dists
 
-    def get_smoothing_kernel_function(self, sigma):
+    def get_fft_smoothing_kernel_function(self, sigma):
         if sigma is None:
             sigma = np.sqrt(2) * np.pi / (self.lmax + 1)
 

nifty/spaces/power_space/power_space.py

@@ -2,6 +2,8 @@
 
 import numpy as np
 
+import d2o
+
 from power_index_factory import PowerIndexFactory
 
 from nifty.spaces.space import Space
@@ -106,13 +108,14 @@ class PowerSpace(Space):
         return result_x
 
     def get_distance_array(self, distribution_strategy):
-        raise NotImplementedError(
-            "There is no get_distance_array implementation for "
-            "PowerSpace.")
+        result = d2o.distributed_data_object(
+                                self.kindex,
+                                distribution_strategy=distribution_strategy)