Skip to content
GitLab
Commit a43a3b04 authored by Ultima's avatar Ultima
Browse files

Improved .diag method in diagonal_operator. 

Implemented interface for slicing methods in slicing distributors. fftw_distributor is now a special case.
Implemented flatten routines for d2o.
parent afca20a4
Show whitespace changes
Inline Side-by-side
demos/demo_wf1.py
View file @ a43a3b04
......@@ -33,7 +33,7 @@
"""
from __future__ import division
from nifty import * # version 0.8.0
about.warnings.off()
# some signal space; e.g., a two-dimensional regular grid
x_space = rg_space([128, 128]) # define signal space
......@@ -61,10 +61,10 @@ d = R(s) + n # compute data
j = R.adjoint_times(N.inverse_times(d)) # define information source
D = propagator_operator(S=S, N=N, R=R) # define information propagator
m = D(j, W=S, tol=1E-3, note=True) # reconstruct map
m = D(j, W=S, tol=1E-1, note=True) # reconstruct map
s.plot(title="signal", save = 'plot_s.png') # plot signal
d_ = field(x_space, val=d.val, target=k_space)
d_.plot(title="data", vmin=s.min(), vmax=s.max(), save = 'plot_d.png') # plot data
m.plot(title="reconstructed map", vmin=s.min(), vmax=s.max(), save = 'plot_m.png') # plot map
#s.plot(title="signal", save = 'plot_s.png') # plot signal
#d_ = field(x_space, val=d.val, target=k_space)
#d_.plot(title="data", vmin=s.min(), vmax=s.max(), save = 'plot_d.png') # plot data
#m.plot(title="reconstructed map", vmin=s.min(), vmax=s.max(), save = 'plot_m.png') # plot map
nifty_mpi_data.py
View file @ a43a3b04
......@@ -51,6 +51,7 @@ except(ImportError):
found['h5py_parallel'] = False
class distributed_data_object(object):
"""
......@@ -342,13 +343,21 @@ class distributed_data_object(object):
def __div__(self, other):
return self.__builtin_helper__(self.get_local_data().__div__, other)
def __truediv__(self, other):
return self.__div__(other)
def __rdiv__(self, other):
return self.__builtin_helper__(self.get_local_data().__rdiv__, other)
def __rtruediv__(self, other):
return self.__rdiv__(other)
def __idiv__(self, other):
return self.__builtin_helper__(self.get_local_data().__idiv__,
other,
inplace = True)
def __itruediv(self, other):
return self.__idiv__(other)
def __floordiv__(self, other):
return self.__builtin_helper__(self.get_local_data().__floordiv__,
......@@ -702,6 +711,17 @@ class distributed_data_object(object):
self.distributor.inject(self.data, to_slices, data, from_slices)
def flatten(self, inplace = False):
flat_shape = (np.prod(self.shape),)
temp_d2o = self.copy_empty(global_shape = flat_shape)
flat_data = self.distributor.flatten(self.data, inplace = inplace)
temp_d2o.set_local_data(data = flat_data)
if inplace == True:
self = temp_d2o
return self
else:
return temp_d2o
def _get_distributor(self, distribution_strategy):
'''
......@@ -714,6 +734,7 @@ class distributed_data_object(object):
distributor_dict={
'fftw': _fftw_distributor,
'equal': _equal_distributor,
'not': _not_distributor
}
if not distributor_dict.has_key(distribution_strategy):
......@@ -814,8 +835,8 @@ class distributed_data_object(object):
class _fftw_distributor(object):
def __init__(self, global_data=None, global_shape=None, dtype=None,
class _slicing_distributor(object):
def __init__(self, slicer, global_data=None, global_shape=None, dtype=None,
comm=MPI.COMM_WORLD, alias=None, path=None):
if alias != None:
......@@ -876,9 +897,14 @@ class _fftw_distributor(object):
self.mpi_dtype = self._my_dtype_converter.to_mpi(self.dtype)
self._local_size = pyfftw.local_size(self.global_shape)
self.local_start = self._local_size[2]
self.local_end = self.local_start + self._local_size[1]
#self._local_size = pyfftw.local_size(self.global_shape)
#self.local_start = self._local_size[2]
#self.local_end = self.local_start + self._local_size[1]
self.slicer = lambda global_shape: slicer(global_shape, comm = comm)
self._local_size = self.slicer(self.global_shape)
self.local_start = self._local_size[0]
self.local_end = self._local_size[1]
self.local_length = self.local_end-self.local_start
self.local_shape = (self.local_length,) + tuple(self.global_shape[1:])
self.local_dim = np.product(self.local_shape)
......@@ -1022,36 +1048,112 @@ class _fftw_distributor(object):
if from_slices == None:
update_slice = (slice(o[r], o[r]+l),)
else:
## Determine the part of the source array, which is relevant
## for the target rank
if (from_slices[0].step > 0) or (from_slices[0].step is None):
## f_relative_start: index of start of source data in
## source array
f_lower_end = from_slices[0].start
if f_lower_end is None:
f_lower_end = 0
## f_start: index of start of specific source data in
## source array
f_start = f_lower_end + o[r]
## f_stop: index of stop of specific source data
f_stop = f_start + l
elif from_slices[0].step < 0:
## f_relative_start: index of start of source data in
## source array
f_upper_end = from_slices[0].start
if f_upper_end is None:
f_upper_end = data_update.shape[0] - 1
## f_start: index of start of specific source data in
## source array
f_start = f_upper_end - o[r]
## f_stop: index of stop of specific source data
f_stop = f_start - l
f_step = from_slices[0].step
if f_step == None:
f_step = 1
f_direction = np.sign(f_step)
f_relative_start = from_slices[0].start
if f_relative_start != None:
f_start = f_relative_start + f_direction*o[r]
else:
f_start = None
f_relative_start = 0
f_stop = f_relative_start + f_direction*(o[r]+l*np.abs(f_step))
if f_stop < 0:
f_stop = None
raise ValueError(about._errors.cstring(\
"ERROR: step size == 0!"))
## combine the slicing for the first dimension
update_slice = (slice(f_start,
f_stop,
f_step),
)
## add the rest of the from_slicing
update_slice += from_slices[1:]
from_slices[0].step),)
update_slice += from_slices[1:]
data[local_slice] = np.array(data_update[update_slice],\
copy=False).astype(self.dtype)
# ## TODO: Fallunterscheidung, ob direction positiv oder negativ!!
# if from_slices[0].step > 0:
# f_relative_start = from_slices[0].start
# else:
# f_relative_start = from_slices[0].stop + 1
#
# if f_relative_start is None:
# f_relative_start = 0
#
# local_start = f_relative_start + o[r]
# print ('rank', rank,
# 'f_relative_start', f_relative_start,
# 'local_start', local_start,
# 'o[r]', o[r])
#
#
# update_slice = self._backshift_and_decycle(
# slice_object = from_slices[0],
# shifted_start = local_start,
# shifted_stop = local_start+l,
# global_length = data_update.shape[0])
#
# print ('rank', rank, update_slice)
# f_step = from_slices[0].step
# if f_step == None:
# f_step = 1
#
# f_direction = np.sign(f_step)
#
# f_relative_start = from_slices[0].start
#
# ## Case 1: f_direction is positive
# if f_direction > 0:
# if f_relative_start != None:
# f_start = f_relative_start + o[r]
#
#
#
#
# if f_relative_start != None:
# f_start = f_relative_start + f_direction*o[r]
# else:
# f_start = None
# f_relative_start = self.local_start + l - 1
#
#
# f_stop = f_relative_start + f_direction*(o[r]+l*np.abs(f_step))
# print (rank,
# 'f_start', f_start,
# 'offset', self.local_start,
# 'f_relative_start', f_relative_start,
# 'f_stop', f_stop)
# if f_stop < 0:
# f_stop = None
#
#
# ## combine the slicing for the first dimension
# update_slice = (slice(f_start,
# f_stop,
# f_step),
# )
# ## add the rest of the from_slicing
# update_slice += from_slices[1:]
#
# data[local_slice] = np.array(data_update[update_slice],\
# copy=False).astype(self.dtype)
#
else:
## Scatterv the relevant part from the source_rank to the others
## and plug it into data[local_slice]
......@@ -1231,8 +1333,11 @@ class _fftw_distributor(object):
(global_length-shifted_stop)%step #stepsize compensation
else:
local_start = slice_object.start - shifted_start
## if the local_start is negative, pull it up to zero
local_start = 0 if local_start < 0 else local_start
## if the local_start is negative, immediately return the
## values for an empty slice
if local_start < 0:
return 0, 0
## if the local_start is greater than the local length, pull
## it down
if local_start > local_length-1:
......@@ -1244,8 +1349,8 @@ class _fftw_distributor(object):
local_stop = None
else:
local_stop = slice_object.stop - shifted_start
## if local_stop is negative, pull it up to zero
local_stop = 0 if local_stop < 0 else local_stop
## if local_stop is negative, pull it up to None
local_stop = None if local_stop < 0 else local_stop
## Note: if start or stop are greater than the array length,
## numpy will automatically cut the index value down into the
## array's range
......@@ -1285,7 +1390,7 @@ class _fftw_distributor(object):
try:
(data_object, matching_dimensions) = \
self._reshape_foreign_data(data_object)
## if the shape-casting fails, try to fix things via locall data
## if the shape-casting fails, try to fix things via local data
## matching
except(ValueError):
## Check if all the local shapes match the supplied data
......@@ -1386,6 +1491,38 @@ class _fftw_distributor(object):
root=target_rank)
return _gathered_data
def flatten(self, data, inplace = False):
## it can be the case, that the portion af data changes due to
## the flattening. E.g. this is the case if some nodes had no data at
## all, but the higher dimensions are large.
## Check if the amount of data changes because of the flattening
size_now = np.prod(self.local_shape)
(start_then, end_then) = self.slicer((np.prod(self.global_shape),))
size_then = end_then - start_then
if size_now == size_then:
if inplace == True:
return data.ravel()
else:
return data.flatten()
else:
about.warnings.cprint(
"WARNING: Local datasets on the nodes must be changed "+
"and will be exchanged! "+
"The data will be completely gathered!")
if inplace == True:
about.warnings.cprint(
"WARNING: Inplace flattening is not possible when "+
"data-exchange is necessary!")
## TODO: Improve this by making something smarter than a gather_all
full_data = self.consolidate_data(data).ravel()
## extract the local portion
new_data = full_data[slice(start_then, end_then)]
return new_data
print (size_now, size_then)
if found['h5py']:
def save_data(self, data, alias, path=None, overwriteQ=True, comm=None):
if comm == None:
......@@ -1443,7 +1580,54 @@ class _fftw_distributor(object):
def _fftw_slicer(global_shape, comm):
local_size = pyfftw.local_size(global_shape, comm = comm)
start = local_size[2]
end = start + local_size[1]
return (start, end)
class _fftw_distributor(_slicing_distributor):
def __init__(self, global_data=None, global_shape=None, dtype=None,
comm=MPI.COMM_WORLD, alias=None, path=None):
super(_fftw_distributor, self).__init__(slicer = _fftw_slicer,
global_data = global_data,
global_shape = global_shape,
dtype = dtype,
comm = comm,
alias = alias,
path = path)
def _equal_slicer(global_shape, comm=MPI.COMM_WORLD):
rank = comm.rank
size = comm.size
global_length = global_shape[0]
## compute the smallest number of rows the node will get
local_length = global_length // size
## calculate how many nodes will get an extra row
number_of_extras = global_length - local_length * size
## calculate the individual offset
offset = rank*local_length + min(rank, number_of_extras)*1
## check if local node will get an extra row or not
if number_of_extras > rank:
## if yes, increase the local_length by one
local_length += 1
return (offset, offset+local_length)
class _equal_distributor(_slicing_distributor):
def __init__(self, global_data=None, global_shape=None, dtype=None,
comm=MPI.COMM_WORLD, alias=None, path=None):
super(_equal_distributor, self).__init__(slicer = _equal_slicer,
global_data = global_data,
global_shape = global_shape,
dtype = dtype,
comm = comm,
alias = alias,
path = path)
class _not_distributor(object):
def __init__(self, global_data=None, global_shape=None, dtype=None, *args, **kwargs):
......@@ -1493,6 +1677,12 @@ class _not_distributor(object):
def extract_local_data(self, data_object):
return data_object.get_full_data()
def flatten(self, data, inplace = False):
if inplace == False:
return data.flatten()
else:
return data.reshape(data.size)
def save_data(self, *args, **kwargs):
raise AttributeError(about._errors.cstring(
"ERROR: save_data not implemented"))
......
operators/nifty_operators.py
View file @ a43a3b04
......@@ -1611,13 +1611,18 @@ class diagonal_operator(operator):
"""
if (domain is None) or (domain == self.domain):
if(not self.domain.discrete)and(bare):
diag = self.domain.calc_weight(self.val, power=-1)
else:
diag = self.val
else:
diag = super(diagonal_operator, self).diag(bare=bare,
domain=domain,
nrun=1,
random='pm1',
varQ=False,
**kwargs)
if varQ == True:
return (diag, diag.domain.cast(1))
else:
......@@ -1712,13 +1717,20 @@ class diagonal_operator(operator):
entries; e.g., as variance in case of an covariance operator.
"""
if (domain is None) or (domain == self.domain):
inverse_val = 1./self.val
if(not self.domain.discrete)and(bare):
inverse_diag = self.domain.calc_weight(inverse_val, power=-1)
else:
inverse_diag = inverse_val
else:
inverse_diag = super(diagonal_operator, self).inverse_diag(bare=bare,
domain=domain,
nrun=1,
random='pm1',
varQ=False,
**kwargs)
if varQ == True:
return (inverse_diag, inverse_diag.domain.cast(1))
else:
......@@ -3034,7 +3046,8 @@ class response_operator(operator):
Whether to consider the arguments as densities or not.
Mandatory for the correct incorporation of volume weights.
"""
def __init__(self,domain,sigma=0,mask=1,assign=None,den=False,target=None):
def __init__(self, domain, sigma=0, mask=1, assign=None, den=False,
target=None):
"""
Sets the standard properties and `density`, `sigma`, `mask` and `assignment(s)`.
......
rg/nifty_rg.py
View file @ a43a3b04
......@@ -774,11 +774,20 @@ class rg_space(point_space):
## Case 2: normal distribution with zero-mean and a given standard
## deviation or variance
elif arg[0] == 'gau':
var = arg[3]
if np.isscalar(var) == True or var is None:
processed_var = var
else:
try:
processed_var = sample.distributor.extract_local_data(var)
except(AttributeError):
processed_var = var
gen = lambda s: random.gau(datatype=self.datatype,
shape = s,
mean = arg[1],
dev = arg[2],
var = arg[3])
var = processed_var)
sample.apply_generator(gen)
if hermitianizeQ == True:
......@@ -2255,9 +2264,14 @@ class utilities(object):
## flip in every direction
for i in xrange(dimensions):
slice_picker = slice_primitive[:]
slice_picker[i] = slice(1, None)
slice_picker[i] = slice(1, None,None)
slice_inverter = slice_primitive[:]
slice_inverter[i] = slice(None, 0, -1)
try:
y.inject(to_slices=slice_picker, data=y,
from_slices=slice_inverter)
except(AttributeError):
y[slice_picker] = y[slice_inverter]
return y
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment