Commit 10e11a0a authored by Ultima's avatar Ultima
Browse files

Test Suite coverage now at 88%.

-> Many bugfixes and refactorings.
parent d31b3ce6
...@@ -33,7 +33,7 @@ ...@@ -33,7 +33,7 @@
""" """
from __future__ import division from __future__ import division
from nifty import * # version 0.8.0 from nifty import * # version 0.8.0
about.warnings.off() about.warnings.on()
# some signal space; e.g., a two-dimensional regular grid # some signal space; e.g., a two-dimensional regular grid
x_space = rg_space([1280, 1280], datamodel = 'd2o') # define signal space x_space = rg_space([1280, 1280], datamodel = 'd2o') # define signal space
......
...@@ -24,7 +24,7 @@ class _COMM_WORLD(): ...@@ -24,7 +24,7 @@ class _COMM_WORLD():
def Get_size(self): def Get_size(self):
return self.size return self.size
def _scattergather_helper(self, sendbuf, recvbuf=None): def _scattergather_helper(self, sendbuf, recvbuf=None, **kwargs):
sendbuf = self._unwrapper(sendbuf) sendbuf = self._unwrapper(sendbuf)
recvbuf = self._unwrapper(recvbuf) recvbuf = self._unwrapper(recvbuf)
if recvbuf != None: if recvbuf != None:
...@@ -68,13 +68,16 @@ class _COMM_WORLD(): ...@@ -68,13 +68,16 @@ class _COMM_WORLD():
return self._scattergather_helper(*args, **kwargs) return self._scattergather_helper(*args, **kwargs)
def Allreduce(self, sendbuf, recvbuf, op, **kwargs): def Allreduce(self, sendbuf, recvbuf, op, **kwargs):
recvbuf[None] = op(sendbuf) recvbuf[:] = op(sendbuf)
return recvbuf return recvbuf
def allreduce(self, sendbuf, recvbuf, op, **kwargs): def allreduce(self, sendbuf, recvbuf, op, **kwargs):
recvbuf[None] = op(sendbuf) recvbuf[:] = op(sendbuf)
return recvbuf return recvbuf
def sendrecv(self, sendobj, **kwargs):
return sendobj
def _unwrapper(self, x): def _unwrapper(self, x):
if isinstance(x, list): if isinstance(x, list):
return x[0] return x[0]
......
...@@ -1221,10 +1221,10 @@ class point_space(space): ...@@ -1221,10 +1221,10 @@ class point_space(space):
"mean" : lambda y: getattr(y, 'mean')(), "mean" : lambda y: getattr(y, 'mean')(),
"std" : lambda y: getattr(y, 'std')(), "std" : lambda y: getattr(y, 'std')(),
"var" : lambda y: getattr(y, 'var')(), "var" : lambda y: getattr(y, 'var')(),
"argmin" : lambda y: getattr(y, 'argmin')(), "argmin" : lambda y: getattr(y, 'argmin_nonflat')(),
"argmin_flat" : lambda y: getattr(y, 'argmin_flat')(), "argmin_flat" : lambda y: getattr(y, 'argmin')(),
"argmax" : lambda y: getattr(y, 'argmax')(), "argmax" : lambda y: getattr(y, 'argmax_nonflat')(),
"argmax_flat" : lambda y: getattr(y, 'argmax_flat')(), "argmax_flat" : lambda y: getattr(y, 'argmax')(),
"conjugate" : lambda y: getattr(y, 'conjugate')(), "conjugate" : lambda y: getattr(y, 'conjugate')(),
"sum" : lambda y: getattr(y, 'sum')(), "sum" : lambda y: getattr(y, 'sum')(),
"prod" : lambda y: getattr(y, 'prod')(), "prod" : lambda y: getattr(y, 'prod')(),
......
...@@ -52,6 +52,10 @@ except(ImportError): ...@@ -52,6 +52,10 @@ except(ImportError):
FOUND['h5py_parallel'] = False FOUND['h5py_parallel'] = False
ALL_DISTRIBUTION_STRATEGIES = ['not', 'equal', 'fftw', 'freeform']
GLOBAL_DISTRIBUTION_STRATEGIES = ['not', 'equal', 'fftw']
LOCAL_DISTRIBUTION_STRATEGIES = ['freeform']
HDF5_DISTRIBUTION_STRATEGIES = ['equal', 'fftw']
COMM = MPI.COMM_WORLD COMM = MPI.COMM_WORLD
...@@ -192,6 +196,8 @@ class distributed_data_object(object): ...@@ -192,6 +196,8 @@ class distributed_data_object(object):
self.distribution_strategy = distribution_strategy self.distribution_strategy = distribution_strategy
self.dtype = self.distributor.dtype self.dtype = self.distributor.dtype
self.shape = self.distributor.global_shape self.shape = self.distributor.global_shape
self.local_shape = self.distributor.local_shape
self.comm = self.distributor.comm
self.init_args = args self.init_args = args
self.init_kwargs = kwargs self.init_kwargs = kwargs
...@@ -200,7 +206,7 @@ class distributed_data_object(object): ...@@ -200,7 +206,7 @@ class distributed_data_object(object):
global_data = global_data, global_data = global_data,
local_data = local_data, local_data = local_data,
alias = alias, alias = alias,
path = alias, path = path,
hermitian = hermitian, hermitian = hermitian,
copy = copy) copy = copy)
self.index = d2o_librarian.register(self) self.index = d2o_librarian.register(self)
...@@ -224,7 +230,7 @@ class distributed_data_object(object): ...@@ -224,7 +230,7 @@ class distributed_data_object(object):
temp_d2o = self.copy_empty(dtype=dtype, temp_d2o = self.copy_empty(dtype=dtype,
distribution_strategy=distribution_strategy, distribution_strategy=distribution_strategy,
**kwargs) **kwargs)
if distribution_strategy == None or \ if distribution_strategy is None or \
distribution_strategy == self.distribution_strategy: distribution_strategy == self.distribution_strategy:
temp_d2o.set_local_data(self.get_local_data(), copy=True) temp_d2o.set_local_data(self.get_local_data(), copy=True)
else: else:
...@@ -235,11 +241,23 @@ class distributed_data_object(object): ...@@ -235,11 +241,23 @@ class distributed_data_object(object):
def copy_empty(self, global_shape=None, local_shape=None, dtype=None, def copy_empty(self, global_shape=None, local_shape=None, dtype=None,
distribution_strategy=None, **kwargs): distribution_strategy=None, **kwargs):
if global_shape == None: if self.distribution_strategy == 'not' and \
distribution_strategy in LOCAL_DISTRIBUTION_STRATEGIES and \
local_shape == None:
result = self.copy_empty(global_shape = global_shape,
local_shape = local_shape,
dtype = dtype,
distribution_strategy = 'equal',
**kwargs)
return result.copy_empty(distribution_strategy = 'freeform')
if global_shape is None:
global_shape = self.shape global_shape = self.shape
if dtype == None: if local_shape is None:
local_shape = self.local_shape
if dtype is None:
dtype = self.dtype dtype = self.dtype
if distribution_strategy == None: if distribution_strategy is None:
distribution_strategy = self.distribution_strategy distribution_strategy = self.distribution_strategy
kwargs.update(self.init_kwargs) kwargs.update(self.init_kwargs)
...@@ -248,6 +266,7 @@ class distributed_data_object(object): ...@@ -248,6 +266,7 @@ class distributed_data_object(object):
local_shape = local_shape, local_shape = local_shape,
dtype = dtype, dtype = dtype,
distribution_strategy = distribution_strategy, distribution_strategy = distribution_strategy,
comm = self.comm,
*self.init_args, *self.init_args,
**kwargs) **kwargs)
return temp_d2o return temp_d2o
...@@ -257,17 +276,22 @@ class distributed_data_object(object): ...@@ -257,17 +276,22 @@ class distributed_data_object(object):
if inplace == True: if inplace == True:
temp = self temp = self
if dtype != None and self.dtype != np.dtype(dtype): if dtype is not None and self.dtype != np.dtype(dtype):
about.warnings.cprint(\ about.warnings.cprint(\
"WARNING: Inplace dtype conversion is not possible!") "WARNING: Inplace dtype conversion is not possible!")
else: else:
temp = self.copy_empty(dtype=dtype) temp = self.copy_empty(dtype=dtype)
try: if np.prod(self.local_shape) != 0:
temp.data[:] = function(self.data) try:
except: temp.data[:] = function(self.data)
temp.data[:] = np.vectorize(function)(self.data) except:
temp.data[:] = np.vectorize(function)(self.data)
else:
## Noting to do here. The value-empty array
## is also geometrically empty
pass
if function in (np.exp, np.log): if function in (np.exp, np.log):
temp.hermitian = remember_hermitianQ temp.hermitian = remember_hermitianQ
...@@ -305,7 +329,7 @@ class distributed_data_object(object): ...@@ -305,7 +329,7 @@ class distributed_data_object(object):
return result return result
## Case 3: 'other' is None ## Case 3: 'other' is None
elif other == None: elif other is None:
return False return False
## Case 4: 'other' is something different ## Case 4: 'other' is something different
...@@ -377,7 +401,12 @@ class distributed_data_object(object): ...@@ -377,7 +401,12 @@ class distributed_data_object(object):
copy = True) copy = True)
return temp_d2o return temp_d2o
def __builtin_helper__(self, operator, other, inplace=False): def _builtin_helper(self, operator, other, inplace=False):
if isinstance(other, distributed_data_object):
other_is_real = other.isreal()
else:
other_is_real = np.isreal(other)
## Case 1: other is not a scalar ## Case 1: other is not a scalar
if not (np.isscalar(other) or np.shape(other) == (1,)): if not (np.isscalar(other) or np.shape(other) == (1,)):
## if self.shape != other.shape: ## if self.shape != other.shape:
...@@ -392,9 +421,8 @@ class distributed_data_object(object): ...@@ -392,9 +421,8 @@ class distributed_data_object(object):
temp_data = operator(temp_data) temp_data = operator(temp_data)
## Case 2: other is a real scalar -> preserve hermitianity ## Case 2: other is a real scalar -> preserve hermitianity
elif np.isreal(other) or (self.dtype not in ( elif other_is_real or (self.dtype not in (np.dtype('complex128'),
np.dtype('complex128'), np.dtype('complex256'))):
np.dtype('complex256'))):
hermitian_Q = self.hermitian hermitian_Q = self.hermitian
temp_data = operator(other) temp_data = operator(other)
## Case 3: other is complex ## Case 3: other is complex
...@@ -432,79 +460,86 @@ class distributed_data_object(object): ...@@ -432,79 +460,86 @@ class distributed_data_object(object):
""" """
def __add__(self, other): def __add__(self, other):
return self.__builtin_helper__(self.get_local_data().__add__, other) return self._builtin_helper(self.get_local_data().__add__, other)
def __radd__(self, other): def __radd__(self, other):
return self.__builtin_helper__(self.get_local_data().__radd__, other) return self._builtin_helper(self.get_local_data().__radd__, other)
def __iadd__(self, other): def __iadd__(self, other):
return self.__builtin_helper__(self.get_local_data().__iadd__, return self._builtin_helper(self.get_local_data().__iadd__,
other, other,
inplace = True) inplace = True)
def __sub__(self, other): def __sub__(self, other):
return self.__builtin_helper__(self.get_local_data().__sub__, other) return self._builtin_helper(self.get_local_data().__sub__, other)
def __rsub__(self, other): def __rsub__(self, other):
return self.__builtin_helper__(self.get_local_data().__rsub__, other) return self._builtin_helper(self.get_local_data().__rsub__, other)
def __isub__(self, other): def __isub__(self, other):
return self.__builtin_helper__(self.get_local_data().__isub__, return self._builtin_helper(self.get_local_data().__isub__,
other, other,
inplace = True) inplace = True)
def __div__(self, other): def __div__(self, other):
return self.__builtin_helper__(self.get_local_data().__div__, other) return self._builtin_helper(self.get_local_data().__div__, other)
def __truediv__(self, other): def __truediv__(self, other):
return self.__div__(other) return self.__div__(other)
def __rdiv__(self, other): def __rdiv__(self, other):
return self.__builtin_helper__(self.get_local_data().__rdiv__, other) return self._builtin_helper(self.get_local_data().__rdiv__, other)
def __rtruediv__(self, other): def __rtruediv__(self, other):
return self.__rdiv__(other) return self.__rdiv__(other)
def __idiv__(self, other): def __idiv__(self, other):
return self.__builtin_helper__(self.get_local_data().__idiv__, return self._builtin_helper(self.get_local_data().__idiv__,
other, other,
inplace = True) inplace = True)
def __itruediv__(self, other): def __itruediv__(self, other):
return self.__idiv__(other) return self.__idiv__(other)
def __floordiv__(self, other): def __floordiv__(self, other):
return self.__builtin_helper__(self.get_local_data().__floordiv__, return self._builtin_helper(self.get_local_data().__floordiv__,
other) other)
def __rfloordiv__(self, other): def __rfloordiv__(self, other):
return self.__builtin_helper__(self.get_local_data().__rfloordiv__, return self._builtin_helper(self.get_local_data().__rfloordiv__,
other) other)
def __ifloordiv__(self, other): def __ifloordiv__(self, other):
return self.__builtin_helper__( return self._builtin_helper(
self.get_local_data().__ifloordiv__, other, self.get_local_data().__ifloordiv__, other,
inplace = True) inplace = True)
def __mul__(self, other): def __mul__(self, other):
return self.__builtin_helper__(self.get_local_data().__mul__, other) return self._builtin_helper(self.get_local_data().__mul__, other)
def __rmul__(self, other): def __rmul__(self, other):
return self.__builtin_helper__(self.get_local_data().__rmul__, other) return self._builtin_helper(self.get_local_data().__rmul__, other)
def __imul__(self, other): def __imul__(self, other):
return self.__builtin_helper__(self.get_local_data().__imul__, return self._builtin_helper(self.get_local_data().__imul__,
other, other,
inplace = True) inplace = True)
def __pow__(self, other): def __pow__(self, other):
return self.__builtin_helper__(self.get_local_data().__pow__, other) return self._builtin_helper(self.get_local_data().__pow__, other)
def __rpow__(self, other): def __rpow__(self, other):
return self.__builtin_helper__(self.get_local_data().__rpow__, other) return self._builtin_helper(self.get_local_data().__rpow__, other)
def __ipow__(self, other): def __ipow__(self, other):
return self.__builtin_helper__(self.get_local_data().__ipow__, return self._builtin_helper(self.get_local_data().__ipow__,
other, other,
inplace = True) inplace = True)
def __mod__(self, other):
return self._builtin_helper(self.get_local_data().__mod__, other)
def __rmod__(self, other):
return self._builtin_helper(self.get_local_data().__rmod__, other)
def __imod__(self, other):
return self._builtin_helper(self.get_local_data().__imod__,
other,
inplace = True)
def __len__(self): def __len__(self):
return self.shape[0] return self.shape[0]
...@@ -550,10 +585,23 @@ class distributed_data_object(object): ...@@ -550,10 +585,23 @@ class distributed_data_object(object):
self.set_data(data, key) self.set_data(data, key)
def _contraction_helper(self, function, **kwargs): def _contraction_helper(self, function, **kwargs):
local = function(self.data, **kwargs) if np.prod(self.data.shape) == 0:
local = 0
include = False
else:
local = function(self.data, **kwargs)
include = True
local_list = self.distributor._allgather(local) local_list = self.distributor._allgather(local)
global_ = function(local_list, axis=0) local_list = np.array(local_list, dtype = np.dtype(local_list[0]))
return global_ include_list = np.array(self.distributor._allgather(include))
work_list = local_list[include_list]
if work_list.shape[0] == 0:
raise ValueError("ERROR: Zero-size array to reduction operation "+
"which has no identity")
else:
result = function(work_list, axis=0)
return result
def amin(self, **kwargs): def amin(self, **kwargs):
return self._contraction_helper(np.amin, **kwargs) return self._contraction_helper(np.amin, **kwargs)
...@@ -575,19 +623,34 @@ class distributed_data_object(object): ...@@ -575,19 +623,34 @@ class distributed_data_object(object):
def mean(self, power=1): def mean(self, power=1):
## compute the local means and the weights for the mean-mean. ## compute the local means and the weights for the mean-mean.
local_mean = np.mean(self.data**power) if np.prod(self.data.shape) == 0:
local_mean = 0
include = False
else:
local_mean = np.mean(self.data**power)
include = True
local_weight = np.prod(self.data.shape) local_weight = np.prod(self.data.shape)
## collect the local means and cast the result to a ndarray ## collect the local means and cast the result to a ndarray
local_mean_weight_list = self.distributor._allgather((local_mean, local_mean_list = self.distributor._allgather(local_mean)
local_weight)) local_weight_list = self.distributor._allgather(local_weight)
local_mean_weight_list =np.array(local_mean_weight_list)
## compute the denominator for the weighted mean-mean local_mean_list =np.array(local_mean_list,
global_weight = np.sum(local_mean_weight_list[:,1]) dtype = np.dtype(local_mean_list[0]))
## compute the numerator local_weight_list = np.array(local_weight_list)
numerator = np.sum(local_mean_weight_list[:,0]*\ ## extract the parts from the non-empty nodes
local_mean_weight_list[:,1]) include_list = np.array(self.distributor._allgather(include))
global_mean = numerator/global_weight work_mean_list = local_mean_list[include_list]
return global_mean work_weight_list = local_weight_list[include_list]
if work_mean_list.shape[0] == 0:
raise ValueError("ERROR: Mean of empty slice.")
else:
## compute the denominator for the weighted mean-mean
global_weight = np.sum(work_weight_list)
## compute the numerator
numerator = np.sum(work_mean_list * work_weight_list)
global_mean = numerator/global_weight
return global_mean
def var(self): def var(self):
mean_of_the_square = self.mean(power=2) mean_of_the_square = self.mean(power=2)
...@@ -608,42 +671,53 @@ class distributed_data_object(object): ...@@ -608,42 +671,53 @@ class distributed_data_object(object):
# ('index', int)]) # ('index', int)])
# return local_argmin_list # return local_argmin_list
# #
def argmin_flat(self): def argmin(self):
local_argmin = np.argmin(self.data) if np.prod(self.data.shape) == 0:
local_argmin_value = self.data[np.unravel_index(local_argmin, local_argmin = np.nan
self.data.shape)] local_argmin_value = np.nan
globalized_local_argmin = self.distributor.globalize_flat_index( globalized_local_argmin = np.nan
else:
local_argmin = np.argmin(self.data)
local_argmin_value = self.data[np.unravel_index(local_argmin,
self.data.shape)]
globalized_local_argmin = self.distributor.globalize_flat_index(
local_argmin) local_argmin)
local_argmin_list = self.distributor._allgather((local_argmin_value, local_argmin_list = self.distributor._allgather((local_argmin_value,
globalized_local_argmin)) globalized_local_argmin))
local_argmin_list = np.array(local_argmin_list, dtype=[ local_argmin_list = np.array(local_argmin_list, dtype=[
('value', local_argmin_value.dtype), ('value', np.dtype('complex128')),
('index', int)]) ('index', np.dtype('float'))])
local_argmin_list = np.sort(local_argmin_list, local_argmin_list = np.sort(local_argmin_list,
order=['value', 'index']) order=['value', 'index'])
return local_argmin_list[0][1] return np.int(local_argmin_list[0][1])
def argmax_flat(self): def argmax(self):
local_argmax = np.argmax(self.data) if np.prod(self.data.shape) == 0:
local_argmax_value = -self.data[np.unravel_index(local_argmax, local_argmax = np.nan
self.data.shape)] local_argmax_value = np.nan
globalized_local_argmax = self.distributor.globalize_flat_index( globalized_local_argmax = np.nan
else:
local_argmax = np.argmax(self.data)
local_argmax_value = -self.data[np.unravel_index(local_argmax,
self.data.shape)]
globalized_local_argmax = self.distributor.globalize_flat_index(
local_argmax) local_argmax)
local_argmax_list = self.distributor._allgather((local_argmax_value, local_argmax_list = self.distributor._allgather((local_argmax_value,
globalized_local_argmax)) globalized_local_argmax))
local_argmax_list = np.array(local_argmax_list, dtype=[ local_argmax_list = np.array(local_argmax_list, dtype=[
('value', local_argmax_value.dtype), ('value', np.dtype('complex128')),
('index', int)]) ('index', np.dtype('float'))])
local_argmax_list = np.sort(local_argmax_list, local_argmax_list = np.sort(local_argmax_list,
order=['value', 'index']) order=['value', 'index'])
return local_argmax_list[0][1] return np.int(local_argmax_list[0][1])
def argmin(self): def argmin_nonflat(self):
return np.unravel_index(self.argmin_flat(), self.shape) return np.unravel_index(self.argmin(), self.shape)
def argmax(self): def argmax_nonflat(self):
return np.unravel_index(self.argmax_flat(), self.shape) return np.unravel_index(self.argmax(), self.shape)
def conjugate(self): def conjugate(self):
temp_d2o = self.copy_empty() temp_d2o = self.copy_empty()
...@@ -662,12 +736,12 @@ class distributed_data_object(object): ...@@ -662,12 +736,12 @@ class distributed_data_object(object):
return median return median
def iscomplex(self): def iscomplex(self):
temp_d2o = self.copy_empty(dtype=np.bool_) temp_d2o = self.copy_empty(dtype=np.dtype('bool'))
temp_d2o.set_local_data(np.iscomplex(self.data)) temp_d2o.set_local_data(np.iscomplex(self.data))
return temp_d2o return temp_d2o
def isreal(self): def isreal(self):
temp_d2o = self.copy_empty(dtype=np.bool_) temp_d2o = self.copy_empty(dtype=np.dtype('bool'))
temp_d2o.set_local_data(np.isreal(self.data)) temp_d2o.set_local_data(np.isreal(self.data))
return temp_d2o return temp_d2o
...@@ -706,11 +780,17 @@ class distributed_data_object(object): ...@@ -706,11 +780,17 @@ class distributed_data_object(object):
local_counts = np.bincount(self.get_local_data().flatten(), local_counts = np.bincount(self.get_local_data().flatten(),
weights = local_weights, weights = local_weights,
minlength = minlength)