Commit 10e11a0a authored by Ultima's avatar Ultima
Browse files

Test Suite coverage now at 88%.

-> Many bugfixes and refactorings.
parent d31b3ce6
......@@ -33,7 +33,7 @@
"""
from __future__ import division
from nifty import * # version 0.8.0
about.warnings.off()
about.warnings.on()
# some signal space; e.g., a two-dimensional regular grid
x_space = rg_space([1280, 1280], datamodel = 'd2o') # define signal space
......
......@@ -24,7 +24,7 @@ class _COMM_WORLD():
def Get_size(self):
return self.size
def _scattergather_helper(self, sendbuf, recvbuf=None):
def _scattergather_helper(self, sendbuf, recvbuf=None, **kwargs):
sendbuf = self._unwrapper(sendbuf)
recvbuf = self._unwrapper(recvbuf)
if recvbuf != None:
......@@ -68,13 +68,16 @@ class _COMM_WORLD():
return self._scattergather_helper(*args, **kwargs)
def Allreduce(self, sendbuf, recvbuf, op, **kwargs):
recvbuf[None] = op(sendbuf)
recvbuf[:] = op(sendbuf)
return recvbuf
def allreduce(self, sendbuf, recvbuf, op, **kwargs):
recvbuf[None] = op(sendbuf)
recvbuf[:] = op(sendbuf)
return recvbuf
def sendrecv(self, sendobj, **kwargs):
return sendobj
def _unwrapper(self, x):
if isinstance(x, list):
return x[0]
......
......@@ -1221,10 +1221,10 @@ class point_space(space):
"mean" : lambda y: getattr(y, 'mean')(),
"std" : lambda y: getattr(y, 'std')(),
"var" : lambda y: getattr(y, 'var')(),
"argmin" : lambda y: getattr(y, 'argmin')(),
"argmin_flat" : lambda y: getattr(y, 'argmin_flat')(),
"argmax" : lambda y: getattr(y, 'argmax')(),
"argmax_flat" : lambda y: getattr(y, 'argmax_flat')(),
"argmin" : lambda y: getattr(y, 'argmin_nonflat')(),
"argmin_flat" : lambda y: getattr(y, 'argmin')(),
"argmax" : lambda y: getattr(y, 'argmax_nonflat')(),
"argmax_flat" : lambda y: getattr(y, 'argmax')(),
"conjugate" : lambda y: getattr(y, 'conjugate')(),
"sum" : lambda y: getattr(y, 'sum')(),
"prod" : lambda y: getattr(y, 'prod')(),
......
......@@ -52,6 +52,10 @@ except(ImportError):
FOUND['h5py_parallel'] = False
ALL_DISTRIBUTION_STRATEGIES = ['not', 'equal', 'fftw', 'freeform']
GLOBAL_DISTRIBUTION_STRATEGIES = ['not', 'equal', 'fftw']
LOCAL_DISTRIBUTION_STRATEGIES = ['freeform']
HDF5_DISTRIBUTION_STRATEGIES = ['equal', 'fftw']
COMM = MPI.COMM_WORLD
......@@ -192,6 +196,8 @@ class distributed_data_object(object):
self.distribution_strategy = distribution_strategy
self.dtype = self.distributor.dtype
self.shape = self.distributor.global_shape
self.local_shape = self.distributor.local_shape
self.comm = self.distributor.comm
self.init_args = args
self.init_kwargs = kwargs
......@@ -200,7 +206,7 @@ class distributed_data_object(object):
global_data = global_data,
local_data = local_data,
alias = alias,
path = alias,
path = path,
hermitian = hermitian,
copy = copy)
self.index = d2o_librarian.register(self)
......@@ -224,7 +230,7 @@ class distributed_data_object(object):
temp_d2o = self.copy_empty(dtype=dtype,
distribution_strategy=distribution_strategy,
**kwargs)
if distribution_strategy == None or \
if distribution_strategy is None or \
distribution_strategy == self.distribution_strategy:
temp_d2o.set_local_data(self.get_local_data(), copy=True)
else:
......@@ -235,11 +241,23 @@ class distributed_data_object(object):
def copy_empty(self, global_shape=None, local_shape=None, dtype=None,
distribution_strategy=None, **kwargs):
if global_shape == None:
if self.distribution_strategy == 'not' and \
distribution_strategy in LOCAL_DISTRIBUTION_STRATEGIES and \
local_shape == None:
result = self.copy_empty(global_shape = global_shape,
local_shape = local_shape,
dtype = dtype,
distribution_strategy = 'equal',
**kwargs)
return result.copy_empty(distribution_strategy = 'freeform')
if global_shape is None:
global_shape = self.shape
if dtype == None:
if local_shape is None:
local_shape = self.local_shape
if dtype is None:
dtype = self.dtype
if distribution_strategy == None:
if distribution_strategy is None:
distribution_strategy = self.distribution_strategy
kwargs.update(self.init_kwargs)
......@@ -248,6 +266,7 @@ class distributed_data_object(object):
local_shape = local_shape,
dtype = dtype,
distribution_strategy = distribution_strategy,
comm = self.comm,
*self.init_args,
**kwargs)
return temp_d2o
......@@ -257,17 +276,22 @@ class distributed_data_object(object):
if inplace == True:
temp = self
if dtype != None and self.dtype != np.dtype(dtype):
if dtype is not None and self.dtype != np.dtype(dtype):
about.warnings.cprint(\
"WARNING: Inplace dtype conversion is not possible!")
else:
temp = self.copy_empty(dtype=dtype)
try:
temp.data[:] = function(self.data)
except:
temp.data[:] = np.vectorize(function)(self.data)
if np.prod(self.local_shape) != 0:
try:
temp.data[:] = function(self.data)
except:
temp.data[:] = np.vectorize(function)(self.data)
else:
## Noting to do here. The value-empty array
## is also geometrically empty
pass
if function in (np.exp, np.log):
temp.hermitian = remember_hermitianQ
......@@ -305,7 +329,7 @@ class distributed_data_object(object):
return result
## Case 3: 'other' is None
elif other == None:
elif other is None:
return False
## Case 4: 'other' is something different
......@@ -377,7 +401,12 @@ class distributed_data_object(object):
copy = True)
return temp_d2o
def __builtin_helper__(self, operator, other, inplace=False):
def _builtin_helper(self, operator, other, inplace=False):
if isinstance(other, distributed_data_object):
other_is_real = other.isreal()
else:
other_is_real = np.isreal(other)
## Case 1: other is not a scalar
if not (np.isscalar(other) or np.shape(other) == (1,)):
## if self.shape != other.shape:
......@@ -392,9 +421,8 @@ class distributed_data_object(object):
temp_data = operator(temp_data)
## Case 2: other is a real scalar -> preserve hermitianity
elif np.isreal(other) or (self.dtype not in (
np.dtype('complex128'),
np.dtype('complex256'))):
elif other_is_real or (self.dtype not in (np.dtype('complex128'),
np.dtype('complex256'))):
hermitian_Q = self.hermitian
temp_data = operator(other)
## Case 3: other is complex
......@@ -432,79 +460,86 @@ class distributed_data_object(object):
"""
def __add__(self, other):
return self.__builtin_helper__(self.get_local_data().__add__, other)
return self._builtin_helper(self.get_local_data().__add__, other)
def __radd__(self, other):
return self.__builtin_helper__(self.get_local_data().__radd__, other)
return self._builtin_helper(self.get_local_data().__radd__, other)
def __iadd__(self, other):
return self.__builtin_helper__(self.get_local_data().__iadd__,
return self._builtin_helper(self.get_local_data().__iadd__,
other,
inplace = True)
def __sub__(self, other):
return self.__builtin_helper__(self.get_local_data().__sub__, other)
return self._builtin_helper(self.get_local_data().__sub__, other)
def __rsub__(self, other):
return self.__builtin_helper__(self.get_local_data().__rsub__, other)
return self._builtin_helper(self.get_local_data().__rsub__, other)
def __isub__(self, other):
return self.__builtin_helper__(self.get_local_data().__isub__,
return self._builtin_helper(self.get_local_data().__isub__,
other,
inplace = True)
def __div__(self, other):
return self.__builtin_helper__(self.get_local_data().__div__, other)
return self._builtin_helper(self.get_local_data().__div__, other)
def __truediv__(self, other):
return self.__div__(other)
def __rdiv__(self, other):
return self.__builtin_helper__(self.get_local_data().__rdiv__, other)
return self._builtin_helper(self.get_local_data().__rdiv__, other)
def __rtruediv__(self, other):
return self.__rdiv__(other)
def __idiv__(self, other):
return self.__builtin_helper__(self.get_local_data().__idiv__,
return self._builtin_helper(self.get_local_data().__idiv__,
other,
inplace = True)
def __itruediv__(self, other):
return self.__idiv__(other)
def __floordiv__(self, other):
return self.__builtin_helper__(self.get_local_data().__floordiv__,
return self._builtin_helper(self.get_local_data().__floordiv__,
other)
def __rfloordiv__(self, other):
return self.__builtin_helper__(self.get_local_data().__rfloordiv__,
return self._builtin_helper(self.get_local_data().__rfloordiv__,
other)
def __ifloordiv__(self, other):
return self.__builtin_helper__(
return self._builtin_helper(
self.get_local_data().__ifloordiv__, other,
inplace = True)
def __mul__(self, other):
return self.__builtin_helper__(self.get_local_data().__mul__, other)
return self._builtin_helper(self.get_local_data().__mul__, other)
def __rmul__(self, other):
return self.__builtin_helper__(self.get_local_data().__rmul__, other)
return self._builtin_helper(self.get_local_data().__rmul__, other)
def __imul__(self, other):
return self.__builtin_helper__(self.get_local_data().__imul__,
return self._builtin_helper(self.get_local_data().__imul__,
other,
inplace = True)
def __pow__(self, other):
return self.__builtin_helper__(self.get_local_data().__pow__, other)
return self._builtin_helper(self.get_local_data().__pow__, other)
def __rpow__(self, other):
return self.__builtin_helper__(self.get_local_data().__rpow__, other)
return self._builtin_helper(self.get_local_data().__rpow__, other)
def __ipow__(self, other):
return self.__builtin_helper__(self.get_local_data().__ipow__,
return self._builtin_helper(self.get_local_data().__ipow__,
other,
inplace = True)
def __mod__(self, other):
return self._builtin_helper(self.get_local_data().__mod__, other)
def __rmod__(self, other):
return self._builtin_helper(self.get_local_data().__rmod__, other)
def __imod__(self, other):
return self._builtin_helper(self.get_local_data().__imod__,
other,
inplace = True)
def __len__(self):
return self.shape[0]
......@@ -550,10 +585,23 @@ class distributed_data_object(object):
self.set_data(data, key)
def _contraction_helper(self, function, **kwargs):
local = function(self.data, **kwargs)
if np.prod(self.data.shape) == 0:
local = 0
include = False
else:
local = function(self.data, **kwargs)
include = True
local_list = self.distributor._allgather(local)
global_ = function(local_list, axis=0)
return global_
local_list = np.array(local_list, dtype = np.dtype(local_list[0]))
include_list = np.array(self.distributor._allgather(include))
work_list = local_list[include_list]
if work_list.shape[0] == 0:
raise ValueError("ERROR: Zero-size array to reduction operation "+
"which has no identity")
else:
result = function(work_list, axis=0)
return result
def amin(self, **kwargs):
return self._contraction_helper(np.amin, **kwargs)
......@@ -575,19 +623,34 @@ class distributed_data_object(object):
def mean(self, power=1):
## compute the local means and the weights for the mean-mean.
local_mean = np.mean(self.data**power)
if np.prod(self.data.shape) == 0:
local_mean = 0
include = False
else:
local_mean = np.mean(self.data**power)
include = True
local_weight = np.prod(self.data.shape)
## collect the local means and cast the result to a ndarray
local_mean_weight_list = self.distributor._allgather((local_mean,
local_weight))
local_mean_weight_list =np.array(local_mean_weight_list)
## compute the denominator for the weighted mean-mean
global_weight = np.sum(local_mean_weight_list[:,1])
## compute the numerator
numerator = np.sum(local_mean_weight_list[:,0]*\
local_mean_weight_list[:,1])
global_mean = numerator/global_weight
return global_mean
local_mean_list = self.distributor._allgather(local_mean)
local_weight_list = self.distributor._allgather(local_weight)
local_mean_list =np.array(local_mean_list,
dtype = np.dtype(local_mean_list[0]))
local_weight_list = np.array(local_weight_list)
## extract the parts from the non-empty nodes
include_list = np.array(self.distributor._allgather(include))
work_mean_list = local_mean_list[include_list]
work_weight_list = local_weight_list[include_list]
if work_mean_list.shape[0] == 0:
raise ValueError("ERROR: Mean of empty slice.")
else:
## compute the denominator for the weighted mean-mean
global_weight = np.sum(work_weight_list)
## compute the numerator
numerator = np.sum(work_mean_list * work_weight_list)
global_mean = numerator/global_weight
return global_mean
def var(self):
mean_of_the_square = self.mean(power=2)
......@@ -608,42 +671,53 @@ class distributed_data_object(object):
# ('index', int)])
# return local_argmin_list
#
def argmin_flat(self):
local_argmin = np.argmin(self.data)
local_argmin_value = self.data[np.unravel_index(local_argmin,
self.data.shape)]
globalized_local_argmin = self.distributor.globalize_flat_index(
def argmin(self):
if np.prod(self.data.shape) == 0:
local_argmin = np.nan
local_argmin_value = np.nan
globalized_local_argmin = np.nan
else:
local_argmin = np.argmin(self.data)
local_argmin_value = self.data[np.unravel_index(local_argmin,
self.data.shape)]
globalized_local_argmin = self.distributor.globalize_flat_index(
local_argmin)
local_argmin_list = self.distributor._allgather((local_argmin_value,
globalized_local_argmin))
local_argmin_list = np.array(local_argmin_list, dtype=[
('value', local_argmin_value.dtype),
('index', int)])
('value', np.dtype('complex128')),
('index', np.dtype('float'))])
local_argmin_list = np.sort(local_argmin_list,
order=['value', 'index'])
return local_argmin_list[0][1]
return np.int(local_argmin_list[0][1])
def argmax_flat(self):
local_argmax = np.argmax(self.data)
local_argmax_value = -self.data[np.unravel_index(local_argmax,
self.data.shape)]
globalized_local_argmax = self.distributor.globalize_flat_index(
def argmax(self):
if np.prod(self.data.shape) == 0:
local_argmax = np.nan
local_argmax_value = np.nan
globalized_local_argmax = np.nan
else:
local_argmax = np.argmax(self.data)
local_argmax_value = -self.data[np.unravel_index(local_argmax,
self.data.shape)]
globalized_local_argmax = self.distributor.globalize_flat_index(
local_argmax)
local_argmax_list = self.distributor._allgather((local_argmax_value,
globalized_local_argmax))
local_argmax_list = np.array(local_argmax_list, dtype=[
('value', local_argmax_value.dtype),
('index', int)])
('value', np.dtype('complex128')),
('index', np.dtype('float'))])
local_argmax_list = np.sort(local_argmax_list,
order=['value', 'index'])
return local_argmax_list[0][1]
return np.int(local_argmax_list[0][1])
def argmin(self):
return np.unravel_index(self.argmin_flat(), self.shape)
def argmin_nonflat(self):
return np.unravel_index(self.argmin(), self.shape)
def argmax(self):
return np.unravel_index(self.argmax_flat(), self.shape)
def argmax_nonflat(self):
return np.unravel_index(self.argmax(), self.shape)
def conjugate(self):
temp_d2o = self.copy_empty()
......@@ -662,12 +736,12 @@ class distributed_data_object(object):
return median
def iscomplex(self):
temp_d2o = self.copy_empty(dtype=np.bool_)
temp_d2o = self.copy_empty(dtype=np.dtype('bool'))
temp_d2o.set_local_data(np.iscomplex(self.data))
return temp_d2o
def isreal(self):
temp_d2o = self.copy_empty(dtype=np.bool_)
temp_d2o = self.copy_empty(dtype=np.dtype('bool'))
temp_d2o.set_local_data(np.isreal(self.data))
return temp_d2o
......@@ -706,11 +780,17 @@ class distributed_data_object(object):
local_counts = np.bincount(self.get_local_data().flatten(),
weights = local_weights,
minlength = minlength)
list_of_counts = self.distributor._allgather(local_counts)
counts = np.sum(list_of_counts, axis = 0)
return counts
if self.distribution_strategy == 'not':
return local_counts
else:
list_of_counts = self.distributor._allgather(local_counts)
counts = np.sum(list_of_counts, axis = 0)
return counts
def where(self):
return self.distributor.where(self.data)
def set_local_data(self, data, hermitian=False, copy=True):
"""
Stores data directly in the local data attribute. No distribution
......@@ -728,9 +808,13 @@ class distributed_data_object(object):
"""
self.hermitian = hermitian
self.data = np.array(data, dtype=self.dtype, copy=copy, order='C')
if copy == True:
self.data[:] = data
else:
self.data = np.array(data, dtype=self.dtype,
copy=False, order='C').reshape(self.local_shape)
def set_data(self, data, to_key, from_key=None, local_to_keys=False,
def set_data(self, data, to_key, from_key=None, local_keys=False,
hermitian=False, copy=True, **kwargs):
"""
Stores the supplied data in the region which is specified by key.
......@@ -756,7 +840,7 @@ class distributed_data_object(object):
to_key = to_key,
data_update = data,
from_key = from_key,
local_to_keys = local_to_keys,
local_keys = local_keys,
copy = copy,
**kwargs)
#
......@@ -830,6 +914,18 @@ class distributed_data_object(object):
global_data[key] : numpy.ndarray
"""
if key is None:
return self.copy()
elif isinstance(key, slice):
if key == slice(None):
return self.copy()
elif isinstance(key, tuple):
try:
if all(x == slice(None) for x in key):
return self.copy()
except(ValueError):
pass
return self.distributor.collect_data(self.data,
key,
local_keys = local_keys,
......@@ -863,23 +959,47 @@ class distributed_data_object(object):
return self.distributor.consolidate_data(self.data,
target_rank = target_rank)
def inject(self, to_slices=(slice(None),), data=None,
from_slices=(slice(None),)):
if data == None:
def inject(self, to_key=(slice(None),), data=None,
from_key=(slice(None),)):
if data is None:
return self
self.distributor.inject(self.data, to_slices, data, from_slices)
self.distributor.inject(self.data, to_key, data, from_key)
def flatten(self, inplace = False):
flat_shape = (np.prod(self.shape),)
temp_d2o = self.copy_empty(global_shape = flat_shape)
flat_data = self.distributor.flatten(self.data, inplace = inplace)
temp_d2o.set_local_data(data = flat_data)
flat_data = self.distributor.flatten(self.data, inplace = inplace)
flat_global_shape = (np.prod(self.shape),)
flat_local_shape = np.shape(flat_data)
## Try to keep the distribution strategy. Therefore
## create an empty copy of self which has the new shape
temp_d2o = self.copy_empty(global_shape = flat_global_shape,
local_shape = flat_local_shape)
## Check if the local shapes match.
if temp_d2o.local_shape == flat_local_shape:
work_d2o = temp_d2o
## if the shapes do not match, create a freeform d2o
else:
work_d2o = self.copy_empty(local_shape = flat_local_shape,
distribution_strategy = 'freeform')
## Feed the work_d2o with the flat data
work_d2o.set_local_data(data = flat_data,
copy = False)
if inplace == True:
self = temp_d2o
self = work_d2o
return self
else:
return temp_d2o
return work_d2o
#
# flat_data = self.distributor.flatten(self.data, inplace = inplace)
# temp_d2o.set_local_data(data = flat_data)
# if inplace == True:
# self = temp_d2o
# return self
# else:
# return temp_d2o
......@@ -946,7 +1066,10 @@ class _distributor_factory(object):
## Check that all nodes got the same distribution_strategy
strat_list = comm.allgather(distribution_strategy)
assert(all(x == strat_list[0] for x in strat_list))
if all(x == strat_list[0] for x in strat_list) == False:
raise ValueError(about._errors.cstring(
"ERROR: The distribution-strategy must be the same on "+
"all nodes!"))
## Check for an hdf5 file and open it if given
if FOUND['h5py'] == True and alias is not None:
......@@ -978,8 +1101,8 @@ class _distributor_factory(object):
elif distribution_strategy in ['not', 'equal', 'fftw']:
if dtype is None:
if global_data is None:
raise ValueError(about._errors.cstring(
"ERROR: Neither global_data nor dtype supplied!"))
dtype = np.dtype('float64')
about.infos.cprint('INFO: dtype set was set to default.')
else:
try:
dtype = global_data.dtype
......@@ -990,19 +1113,23 @@ class _distributor_factory(object):
elif distribution_strategy in ['freeform']:
if dtype is None:
if global_data is None and local_data is None:
raise ValueError(about._errors.cstring(
"ERROR: Neither nor local_data nor dtype supplied!"))
else:
if isinstance(global_data, distributed_data_object):
dtype = global_data.dtype
elif local_data is not None: