distributed_do.py 15.8 KB
Newer Older
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program.  If not, see <http://www.gnu.org/licenses/>.
#
# Copyright(C) 2013-2018 Max-Planck-Society
#
# NIFTy is being developed at the Max-Planck-Institut fuer Astrophysik
# and financially supported by the Studienstiftung des deutschen Volkes.

19
20
21
import numpy as np
from .random import Random
from mpi4py import MPI
22
import sys
23

Martin Reinecke's avatar
tweaks    
Martin Reinecke committed
24
25
26
_comm = MPI.COMM_WORLD
ntask = _comm.Get_size()
rank = _comm.Get_rank()
Martin Reinecke's avatar
Martin Reinecke committed
27
master = (rank == 0)
28
29


Martin Reinecke's avatar
Martin Reinecke committed
30
31
32
33
def is_numpy():
    return False


Martin Reinecke's avatar
tweaks    
Martin Reinecke committed
34
def _shareSize(nwork, nshares, myshare):
Martin Reinecke's avatar
Martin Reinecke committed
35
    return (nwork//nshares) + int(myshare < nwork % nshares)
Martin Reinecke's avatar
Martin Reinecke committed
36

Martin Reinecke's avatar
tweaks    
Martin Reinecke committed
37
38

def _shareRange(nwork, nshares, myshare):
Martin Reinecke's avatar
Martin Reinecke committed
39
40
    nbase = nwork//nshares
    additional = nwork % nshares
Martin Reinecke's avatar
Martin Reinecke committed
41
    lo = myshare*nbase + min(myshare, additional)
Martin Reinecke's avatar
Martin Reinecke committed
42
    hi = lo + nbase + int(myshare < additional)
Martin Reinecke's avatar
Martin Reinecke committed
43
44
    return lo, hi

45

46
def local_shape(shape, distaxis=0):
Martin Reinecke's avatar
Martin Reinecke committed
47
    if len(shape) == 0 or distaxis == -1:
48
        return shape
Martin Reinecke's avatar
Martin Reinecke committed
49
50
    shape2 = list(shape)
    shape2[distaxis] = _shareSize(shape[distaxis], ntask, rank)
51
52
    return tuple(shape2)

Martin Reinecke's avatar
Martin Reinecke committed
53

54
55
class data_object(object):
    def __init__(self, shape, data, distaxis):
Martin Reinecke's avatar
Martin Reinecke committed
56
        self._shape = tuple(shape)
Martin Reinecke's avatar
Martin Reinecke committed
57
        if len(self._shape) == 0:
Martin Reinecke's avatar
fixes    
Martin Reinecke committed
58
            distaxis = -1
59
60
61
        self._distaxis = distaxis
        self._data = data

Martin Reinecke's avatar
Martin Reinecke committed
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
#     def _sanity_checks(self):
#         # check whether the distaxis is consistent
#         if self._distaxis < -1 or self._distaxis >= len(self._shape):
#             raise ValueError
#         itmp = np.array(self._distaxis)
#         otmp = np.empty(ntask, dtype=np.int)
#         _comm.Allgather(itmp, otmp)
#         if np.any(otmp != self._distaxis):
#             raise ValueError
#         # check whether the global shape is consistent
#         itmp = np.array(self._shape)
#         otmp = np.empty((ntask, len(self._shape)), dtype=np.int)
#         _comm.Allgather(itmp, otmp)
#         for i in range(ntask):
#             if np.any(otmp[i, :] != self._shape):
#                 raise ValueError
#         # check shape of local data
#         if self._distaxis < 0:
#             if self._data.shape != self._shape:
#                 raise ValueError
#         else:
#             itmp = np.array(self._shape)
#             itmp[self._distaxis] = _shareSize(self._shape[self._distaxis],
#                                               ntask, rank)
#             if np.any(self._data.shape != itmp):
#                 raise ValueError
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102

    @property
    def dtype(self):
        return self._data.dtype

    @property
    def shape(self):
        return self._shape

    @property
    def size(self):
        return np.prod(self._shape)

    @property
    def real(self):
Martin Reinecke's avatar
Martin Reinecke committed
103
        return data_object(self._shape, self._data.real, self._distaxis)
104
105
106

    @property
    def imag(self):
Martin Reinecke's avatar
Martin Reinecke committed
107
        return data_object(self._shape, self._data.imag, self._distaxis)
108

Martin Reinecke's avatar
Martin Reinecke committed
109
110
111
112
113
114
    def conj(self):
        return data_object(self._shape, self._data.conj(), self._distaxis)

    def conjugate(self):
        return data_object(self._shape, self._data.conjugate(), self._distaxis)

Martin Reinecke's avatar
Martin Reinecke committed
115
    def _contraction_helper(self, op, mpiop, axis):
116
        if axis is not None:
Martin Reinecke's avatar
Martin Reinecke committed
117
            if len(axis) == len(self._data.shape):
118
119
                axis = None
        if axis is None:
Martin Reinecke's avatar
Martin Reinecke committed
120
            res = np.array(getattr(self._data, op)())
Martin Reinecke's avatar
Martin Reinecke committed
121
            if (self._distaxis == -1):
Martin Reinecke's avatar
fixes    
Martin Reinecke committed
122
                return res[()]
Martin Reinecke's avatar
Martin Reinecke committed
123
124
            res2 = np.empty((), dtype=res.dtype)
            _comm.Allreduce(res, res2, mpiop)
Martin Reinecke's avatar
fixes    
Martin Reinecke committed
125
            return res2[()]
126
127

        if self._distaxis in axis:
Martin Reinecke's avatar
Martin Reinecke committed
128
129
            res = getattr(self._data, op)(axis=axis)
            res2 = np.empty_like(res)
Martin Reinecke's avatar
Martin Reinecke committed
130
            _comm.Allreduce(res, res2, mpiop)
Martin Reinecke's avatar
Martin Reinecke committed
131
            return from_global_data(res2, distaxis=0)
132
        else:
Martin Reinecke's avatar
Martin Reinecke committed
133
            # perform the contraction on the local data
Martin Reinecke's avatar
Martin Reinecke committed
134
135
            res = getattr(self._data, op)(axis=axis)
            if self._distaxis == -1:
Martin Reinecke's avatar
Martin Reinecke committed
136
                return from_global_data(res, distaxis=0)
Martin Reinecke's avatar
Martin Reinecke committed
137
            shp = list(res.shape)
Martin Reinecke's avatar
Martin Reinecke committed
138
            shift = 0
Martin Reinecke's avatar
Martin Reinecke committed
139
            for ax in axis:
Martin Reinecke's avatar
Martin Reinecke committed
140
141
                if ax < self._distaxis:
                    shift += 1
Martin Reinecke's avatar
Martin Reinecke committed
142
143
            shp[self._distaxis-shift] = self.shape[self._distaxis]
            return from_local_data(shp, res, self._distaxis-shift)
144
145
146

    def sum(self, axis=None):
        return self._contraction_helper("sum", MPI.SUM, axis)
Martin Reinecke's avatar
Martin Reinecke committed
147

148
149
150
    def prod(self, axis=None):
        return self._contraction_helper("prod", MPI.PROD, axis)

Martin Reinecke's avatar
fixes    
Martin Reinecke committed
151
152
    def min(self, axis=None):
        return self._contraction_helper("min", MPI.MIN, axis)
Martin Reinecke's avatar
Martin Reinecke committed
153

Martin Reinecke's avatar
fixes    
Martin Reinecke committed
154
155
    def max(self, axis=None):
        return self._contraction_helper("max", MPI.MAX, axis)
156

157
158
159
160
161
162
    def mean(self, axis=None):
        if axis is None:
            sz = self.size
        else:
            sz = reduce(lambda x, y: x*y, [self.shape[i] for i in axis])
        return self.sum(axis)/sz
Martin Reinecke's avatar
Martin Reinecke committed
163

164
165
    def std(self, axis=None):
        return np.sqrt(self.var(axis))
Martin Reinecke's avatar
Martin Reinecke committed
166

Martin Reinecke's avatar
Martin Reinecke committed
167
    # FIXME: to be improved!
168
169
170
    def var(self, axis=None):
        if axis is not None and len(axis) != len(self.shape):
            raise ValueError("functionality not yet supported")
Martin Reinecke's avatar
tweaks    
Martin Reinecke committed
171
172
        return (abs(self-self.mean())**2).mean()

173
    def _binary_helper(self, other, op):
Martin Reinecke's avatar
Martin Reinecke committed
174
        a = self
175
        if isinstance(other, data_object):
Martin Reinecke's avatar
Martin Reinecke committed
176
            b = other
177
178
179
180
            if a._shape != b._shape:
                raise ValueError("shapes are incompatible.")
            if a._distaxis != b._distaxis:
                raise ValueError("distributions are incompatible.")
Martin Reinecke's avatar
Martin Reinecke committed
181
182
            a = a._data
            b = b._data
Martin Reinecke's avatar
Martin Reinecke committed
183
184
185
186
        elif np.isscalar(other):
            a = a._data
            b = other
        elif isinstance(other, np.ndarray):
Martin Reinecke's avatar
Martin Reinecke committed
187
            a = a._data
188
            b = other
Martin Reinecke's avatar
Martin Reinecke committed
189
190
        else:
            return NotImplemented
191
192

        tval = getattr(a, op)(b)
Martin Reinecke's avatar
Martin Reinecke committed
193
194
195
196
        if tval is a:
            return self
        else:
            return data_object(self._shape, tval, self._distaxis)
197
198

    def __neg__(self):
Martin Reinecke's avatar
Martin Reinecke committed
199
        return data_object(self._shape, -self._data, self._distaxis)
200
201

    def __abs__(self):
202
        return data_object(self._shape, abs(self._data), self._distaxis)
203
204

    def all(self):
Martin Reinecke's avatar
fixes    
Martin Reinecke committed
205
        return self.sum() == self.size
206
207

    def any(self):
Martin Reinecke's avatar
fixes    
Martin Reinecke committed
208
        return self.sum() != 0
209

Martin Reinecke's avatar
fixes    
Martin Reinecke committed
210
211
    def fill(self, value):
        self._data.fill(value)
212

213
214
215
216
217
218
219
220
221
222
223
224
225
226
for op in ["__add__", "__radd__", "__iadd__",
           "__sub__", "__rsub__", "__isub__",
           "__mul__", "__rmul__", "__imul__",
           "__div__", "__rdiv__", "__idiv__",
           "__truediv__", "__rtruediv__", "__itruediv__",
           "__floordiv__", "__rfloordiv__", "__ifloordiv__",
           "__pow__", "__rpow__", "__ipow__",
           "__lt__", "__le__", "__gt__", "__ge__", "__eq__", "__ne__"]:
    def func(op):
        def func2(self, other):
            return self._binary_helper(other, op=op)
        return func2
    setattr(data_object, op, func(op))

Martin Reinecke's avatar
Martin Reinecke committed
227

Martin Reinecke's avatar
Martin Reinecke committed
228
def full(shape, fill_value, dtype=None, distaxis=0):
Martin Reinecke's avatar
Martin Reinecke committed
229
230
    return data_object(shape, np.full(local_shape(shape, distaxis),
                                      fill_value, dtype), distaxis)
231
232


Martin Reinecke's avatar
fixes    
Martin Reinecke committed
233
def empty(shape, dtype=None, distaxis=0):
Martin Reinecke's avatar
Martin Reinecke committed
234
235
    return data_object(shape, np.empty(local_shape(shape, distaxis),
                                       dtype), distaxis)
236
237


Martin Reinecke's avatar
fixes    
Martin Reinecke committed
238
def zeros(shape, dtype=None, distaxis=0):
Martin Reinecke's avatar
Martin Reinecke committed
239
240
    return data_object(shape, np.zeros(local_shape(shape, distaxis), dtype),
                       distaxis)
241
242


Martin Reinecke's avatar
fixes    
Martin Reinecke committed
243
def ones(shape, dtype=None, distaxis=0):
Martin Reinecke's avatar
Martin Reinecke committed
244
245
    return data_object(shape, np.ones(local_shape(shape, distaxis), dtype),
                       distaxis)
246
247
248
249
250
251
252


def empty_like(a, dtype=None):
    return data_object(np.empty_like(a._data, dtype))


def vdot(a, b):
Martin Reinecke's avatar
fixes    
Martin Reinecke committed
253
    tmp = np.array(np.vdot(a._data, b._data))
Martin Reinecke's avatar
Martin Reinecke committed
254
255
    res = np.empty((), dtype=tmp.dtype)
    _comm.Allreduce(tmp, res, MPI.SUM)
Martin Reinecke's avatar
fixes    
Martin Reinecke committed
256
    return res[()]
257
258
259


def _math_helper(x, function, out):
260
    function = getattr(np, function)
261
262
263
264
    if out is not None:
        function(x._data, out=out._data)
        return out
    else:
Martin Reinecke's avatar
Martin Reinecke committed
265
        return data_object(x.shape, function(x._data), x._distaxis)
266
267


268
_current_module = sys.modules[__name__]
Martin Reinecke's avatar
Martin Reinecke committed
269

270
for f in ["sqrt", "exp", "log", "tanh", "conjugate"]:
271
272
273
274
275
    def func(f):
        def func2(x, out=None):
            return _math_helper(x, f, out)
        return func2
    setattr(_current_module, f, func(f))
276
277


Martin Reinecke's avatar
Martin Reinecke committed
278
279
280
281
282
283
284
285
286
287
288
289
def from_object(object, dtype, copy, set_locked):
    if dtype is None:
        dtype = object.dtype
    dtypes_equal = dtype == object.dtype
    if set_locked and dtypes_equal and locked(object):
        return object
    if not dtypes_equal and not copy:
        raise ValueError("cannot change data type without copying")
    if set_locked and not copy:
        raise ValueError("cannot lock object without copying")
    data = np.array(object._data, dtype=dtype, copy=copy)
    if set_locked:
Martin Reinecke's avatar
fix    
Martin Reinecke committed
290
        data.flags.writeable = False
Martin Reinecke's avatar
Martin Reinecke committed
291
    return data_object(object._shape, data, distaxis=object._distaxis)
292
293


Martin Reinecke's avatar
Martin Reinecke committed
294
295
# This function draws all random numbers on all tasks, to produce the same
# array independent on the number of tasks
Martin Reinecke's avatar
Martin Reinecke committed
296
297
298
# MR FIXME: depending on what is really wanted/needed (i.e. same result
# independent of number of tasks, performance etc.) we need to adjust the
# algorithm.
Martin Reinecke's avatar
Martin Reinecke committed
299
def from_random(random_type, shape, dtype=np.float64, **kwargs):
300
    generator_function = getattr(Random, random_type)
Martin Reinecke's avatar
Martin Reinecke committed
301
302
303
304
305
306
307
    for i in range(ntask):
        lshape = list(shape)
        lshape[0] = _shareSize(shape[0], ntask, i)
        ldat = generator_function(dtype=dtype, shape=lshape, **kwargs)
        if i == rank:
            outdat = ldat
    return from_local_data(shape, outdat, distaxis=0)
308

Martin Reinecke's avatar
Martin Reinecke committed
309

Martin Reinecke's avatar
Martin Reinecke committed
310
311
312
313
def local_data(arr):
    return arr._data


314
315
def ibegin_from_shape(glob_shape, distaxis=0):
    res = [0] * len(glob_shape)
Martin Reinecke's avatar
Martin Reinecke committed
316
    if distaxis < 0:
317
318
319
320
321
        return res
    res[distaxis] = _shareRange(glob_shape[distaxis], ntask, rank)[0]
    return tuple(res)


Martin Reinecke's avatar
fixes    
Martin Reinecke committed
322
323
def ibegin(arr):
    res = [0] * arr._data.ndim
Martin Reinecke's avatar
Martin Reinecke committed
324
    res[arr._distaxis] = _shareRange(arr._shape[arr._distaxis], ntask, rank)[0]
Martin Reinecke's avatar
fixes    
Martin Reinecke committed
325
    return tuple(res)
Martin Reinecke's avatar
Martin Reinecke committed
326
327


Martin Reinecke's avatar
fixes    
Martin Reinecke committed
328
329
def np_allreduce_sum(arr):
    res = np.empty_like(arr)
Martin Reinecke's avatar
Martin Reinecke committed
330
    _comm.Allreduce(arr, res, MPI.SUM)
Martin Reinecke's avatar
fixes    
Martin Reinecke committed
331
    return res
Martin Reinecke's avatar
Martin Reinecke committed
332
333


334
335
336
337
338
339
def np_allreduce_min(arr):
    res = np.empty_like(arr)
    _comm.Allreduce(arr, res, MPI.MIN)
    return res


Martin Reinecke's avatar
Martin Reinecke committed
340
341
342
343
def distaxis(arr):
    return arr._distaxis


Martin Reinecke's avatar
Martin Reinecke committed
344
def from_local_data(shape, arr, distaxis=0):
Martin Reinecke's avatar
Martin Reinecke committed
345
346
347
    return data_object(shape, arr, distaxis)


348
349
350
def from_global_data(arr, sum_up=False, distaxis=0):
    if sum_up:
        arr = np_allreduce_sum(arr)
Martin Reinecke's avatar
Martin Reinecke committed
351
    if distaxis == -1:
Martin Reinecke's avatar
Martin Reinecke committed
352
        return data_object(arr.shape, arr, distaxis)
Martin Reinecke's avatar
Martin Reinecke committed
353
    lo, hi = _shareRange(arr.shape[distaxis], ntask, rank)
Martin Reinecke's avatar
Martin Reinecke committed
354
    sl = [slice(None)]*len(arr.shape)
Martin Reinecke's avatar
Martin Reinecke committed
355
    sl[distaxis] = slice(lo, hi)
Martin Reinecke's avatar
Martin Reinecke committed
356
357
358
    return data_object(arr.shape, arr[sl], distaxis)


Martin Reinecke's avatar
Martin Reinecke committed
359
360
def to_global_data(arr):
    if arr._distaxis == -1:
Martin Reinecke's avatar
fixes    
Martin Reinecke committed
361
362
363
364
365
        return arr._data
    tmp = redistribute(arr, dist=-1)
    return tmp._data


Martin Reinecke's avatar
Martin Reinecke committed
366
def redistribute(arr, dist=None, nodist=None):
Martin Reinecke's avatar
Martin Reinecke committed
367
368
369
    if dist is not None:
        if nodist is not None:
            raise ValueError
Martin Reinecke's avatar
Martin Reinecke committed
370
        if dist == arr._distaxis:
Martin Reinecke's avatar
Martin Reinecke committed
371
372
373
374
375
376
            return arr
    else:
        if nodist is None:
            raise ValueError
        if arr._distaxis not in nodist:
            return arr
Martin Reinecke's avatar
Martin Reinecke committed
377
        dist = -1
Martin Reinecke's avatar
Martin Reinecke committed
378
379
        for i in range(len(arr.shape)):
            if i not in nodist:
Martin Reinecke's avatar
Martin Reinecke committed
380
                dist = i
Martin Reinecke's avatar
Martin Reinecke committed
381
                break
Martin Reinecke's avatar
Martin Reinecke committed
382

Martin Reinecke's avatar
Martin Reinecke committed
383
    if arr._distaxis == -1:  # all data available, just pick the proper subset
384
        return from_global_data(arr._data, distaxis=dist)
Martin Reinecke's avatar
Martin Reinecke committed
385
    if dist == -1:  # gather all data on all tasks
Martin Reinecke's avatar
Martin Reinecke committed
386
        tmp = np.moveaxis(arr._data, arr._distaxis, 0)
Martin Reinecke's avatar
Martin Reinecke committed
387
388
        slabsize = np.prod(tmp.shape[1:])*tmp.itemsize
        sz = np.empty(ntask, dtype=np.int)
Martin Reinecke's avatar
Martin Reinecke committed
389
        for i in range(ntask):
Martin Reinecke's avatar
Martin Reinecke committed
390
391
392
393
            sz[i] = slabsize*_shareSize(arr.shape[arr._distaxis], ntask, i)
        disp = np.empty(ntask, dtype=np.int)
        disp[0] = 0
        disp[1:] = np.cumsum(sz[:-1])
Martin Reinecke's avatar
Martin Reinecke committed
394
        tmp = np.require(tmp, requirements="C")
Martin Reinecke's avatar
Martin Reinecke committed
395
396
        out = np.empty(arr.size, dtype=arr.dtype)
        _comm.Allgatherv(tmp, [out, sz, disp, MPI.BYTE])
Martin Reinecke's avatar
Martin Reinecke committed
397
398
399
400
        shp = np.array(arr._shape)
        shp[1:arr._distaxis+1] = shp[0:arr._distaxis]
        shp[0] = arr.shape[arr._distaxis]
        out = out.reshape(shp)
Martin Reinecke's avatar
Martin Reinecke committed
401
        out = np.moveaxis(out, 0, arr._distaxis)
Martin Reinecke's avatar
Martin Reinecke committed
402
        return from_global_data(out, distaxis=-1)
Martin Reinecke's avatar
Martin Reinecke committed
403

Martin Reinecke's avatar
Martin Reinecke committed
404
    # real redistribution via Alltoallv
Martin Reinecke's avatar
Martin Reinecke committed
405
    ssz0 = arr._data.size//arr.shape[dist]
Martin Reinecke's avatar
Martin Reinecke committed
406
    ssz = np.empty(ntask, dtype=np.int)
Martin Reinecke's avatar
Martin Reinecke committed
407
408
409
    rszall = arr.size//arr.shape[dist]*_shareSize(arr.shape[dist], ntask, rank)
    rbuf = np.empty(rszall, dtype=arr.dtype)
    rsz0 = rszall//arr.shape[arr._distaxis]
Martin Reinecke's avatar
Martin Reinecke committed
410
    rsz = np.empty(ntask, dtype=np.int)
Martin Reinecke's avatar
Martin Reinecke committed
411
412
413
414
415
416
417
418
419
420
421
422
    if dist == 0:  # shortcut possible
        sbuf = np.ascontiguousarray(arr._data)
        for i in range(ntask):
            lo, hi = _shareRange(arr.shape[dist], ntask, i)
            ssz[i] = ssz0*(hi-lo)
            rsz[i] = rsz0*_shareSize(arr.shape[arr._distaxis], ntask, i)
    else:
        sbuf = np.empty(arr._data.size, dtype=arr.dtype)
        sslice = [slice(None)]*arr._data.ndim
        ofs = 0
        for i in range(ntask):
            lo, hi = _shareRange(arr.shape[dist], ntask, i)
Martin Reinecke's avatar
Martin Reinecke committed
423
            sslice[dist] = slice(lo, hi)
Martin Reinecke's avatar
Martin Reinecke committed
424
425
426
427
428
429
            ssz[i] = ssz0*(hi-lo)
            sbuf[ofs:ofs+ssz[i]] = arr._data[sslice].flat
            ofs += ssz[i]
            rsz[i] = rsz0*_shareSize(arr.shape[arr._distaxis], ntask, i)
    ssz *= arr._data.itemsize
    rsz *= arr._data.itemsize
Martin Reinecke's avatar
Martin Reinecke committed
430
431
    sdisp = np.append(0, np.cumsum(ssz[:-1]))
    rdisp = np.append(0, np.cumsum(rsz[:-1]))
Martin Reinecke's avatar
Martin Reinecke committed
432
433
    s_msg = [sbuf, (ssz, sdisp), MPI.BYTE]
    r_msg = [rbuf, (rsz, rdisp), MPI.BYTE]
Martin Reinecke's avatar
tweaks    
Martin Reinecke committed
434
    _comm.Alltoallv(s_msg, r_msg)
Martin Reinecke's avatar
Martin Reinecke committed
435
    del sbuf  # free memory
Martin Reinecke's avatar
Martin Reinecke committed
436
437
438
439
440
441
442
443
444
    if arr._distaxis == 0:
        rbuf = rbuf.reshape(local_shape(arr.shape, dist))
        arrnew = from_local_data(arr.shape, rbuf, distaxis=dist)
    else:
        arrnew = empty(arr.shape, dtype=arr.dtype, distaxis=dist)
        rslice = [slice(None)]*arr._data.ndim
        ofs = 0
        for i in range(ntask):
            lo, hi = _shareRange(arr.shape[arr._distaxis], ntask, i)
Martin Reinecke's avatar
Martin Reinecke committed
445
            rslice[arr._distaxis] = slice(lo, hi)
Martin Reinecke's avatar
Martin Reinecke committed
446
447
448
449
            sz = rsz[i]//arr._data.itemsize
            arrnew._data[rslice].flat = rbuf[ofs:ofs+sz]
            ofs += sz
    return arrnew
Martin Reinecke's avatar
Martin Reinecke committed
450
451


Martin Reinecke's avatar
Martin Reinecke committed
452
453
def transpose(arr):
    if len(arr.shape) != 2 or arr._distaxis != 0:
Martin Reinecke's avatar
Martin Reinecke committed
454
        raise ValueError("bad input")
Martin Reinecke's avatar
Martin Reinecke committed
455
456
457
458
459
460
461
462
463
464
465
    ssz0 = arr._data.size//arr.shape[1]
    ssz = np.empty(ntask, dtype=np.int)
    rszall = arr.size//arr.shape[1]*_shareSize(arr.shape[1], ntask, rank)
    rbuf = np.empty(rszall, dtype=arr.dtype)
    rsz0 = rszall//arr.shape[0]
    rsz = np.empty(ntask, dtype=np.int)
    sbuf = np.empty(arr._data.size, dtype=arr.dtype)
    ofs = 0
    for i in range(ntask):
        lo, hi = _shareRange(arr.shape[1], ntask, i)
        ssz[i] = ssz0*(hi-lo)
Martin Reinecke's avatar
Martin Reinecke committed
466
        sbuf[ofs:ofs+ssz[i]] = arr._data[:, lo:hi].flat
Martin Reinecke's avatar
Martin Reinecke committed
467
468
469
470
471
472
473
474
475
476
477
478
        ofs += ssz[i]
        rsz[i] = rsz0*_shareSize(arr.shape[0], ntask, i)
    ssz *= arr._data.itemsize
    rsz *= arr._data.itemsize
    sdisp = np.append(0, np.cumsum(ssz[:-1]))
    rdisp = np.append(0, np.cumsum(rsz[:-1]))
    s_msg = [sbuf, (ssz, sdisp), MPI.BYTE]
    r_msg = [rbuf, (rsz, rdisp), MPI.BYTE]
    _comm.Alltoallv(s_msg, r_msg)
    del sbuf  # free memory
    arrnew = empty((arr.shape[1], arr.shape[0]), dtype=arr.dtype, distaxis=0)
    ofs = 0
Martin Reinecke's avatar
Martin Reinecke committed
479
    sz2 = _shareSize(arr.shape[1], ntask, rank)
Martin Reinecke's avatar
Martin Reinecke committed
480
481
482
    for i in range(ntask):
        lo, hi = _shareRange(arr.shape[0], ntask, i)
        sz = rsz[i]//arr._data.itemsize
Martin Reinecke's avatar
Martin Reinecke committed
483
        arrnew._data[:, lo:hi] = rbuf[ofs:ofs+sz].reshape(hi-lo, sz2).T
Martin Reinecke's avatar
Martin Reinecke committed
484
485
486
487
        ofs += sz
    return arrnew


Martin Reinecke's avatar
Martin Reinecke committed
488
489
def default_distaxis():
    return 0
490
491
492
493
494
495
496
497


def lock(arr):
    arr._data.flags.writeable = False


def locked(arr):
    return not arr._data.flags.writeable