distributed_do.py 14.8 KB
Newer Older
Martin Reinecke's avatar
Martin Reinecke committed
1
from __future__ import print_function
2
3
4
5
import numpy as np
from .random import Random
from mpi4py import MPI

Martin Reinecke's avatar
tweaks    
Martin Reinecke committed
6
7
8
_comm = MPI.COMM_WORLD
ntask = _comm.Get_size()
rank = _comm.Get_rank()
Martin Reinecke's avatar
Martin Reinecke committed
9
master = (rank == 0)
10
11


Martin Reinecke's avatar
Martin Reinecke committed
12
13
14
15
16
def mprint(*args):
    if master:
        print(*args)


Martin Reinecke's avatar
tweaks    
Martin Reinecke committed
17
def _shareSize(nwork, nshares, myshare):
Martin Reinecke's avatar
Martin Reinecke committed
18
    return (nwork//nshares) + int(myshare < nwork % nshares)
Martin Reinecke's avatar
Martin Reinecke committed
19

Martin Reinecke's avatar
tweaks    
Martin Reinecke committed
20
21

def _shareRange(nwork, nshares, myshare):
Martin Reinecke's avatar
Martin Reinecke committed
22
23
    nbase = nwork//nshares
    additional = nwork % nshares
Martin Reinecke's avatar
Martin Reinecke committed
24
    lo = myshare*nbase + min(myshare, additional)
Martin Reinecke's avatar
Martin Reinecke committed
25
    hi = lo + nbase + int(myshare < additional)
Martin Reinecke's avatar
Martin Reinecke committed
26
27
    return lo, hi

28

Martin Reinecke's avatar
tweaks    
Martin Reinecke committed
29
def local_shape(shape, distaxis):
Martin Reinecke's avatar
Martin Reinecke committed
30
    if len(shape) == 0 or distaxis == -1:
31
        return shape
Martin Reinecke's avatar
Martin Reinecke committed
32
33
    shape2 = list(shape)
    shape2[distaxis] = _shareSize(shape[distaxis], ntask, rank)
34
35
    return tuple(shape2)

Martin Reinecke's avatar
Martin Reinecke committed
36

37
38
class data_object(object):
    def __init__(self, shape, data, distaxis):
Martin Reinecke's avatar
Martin Reinecke committed
39
        self._shape = tuple(shape)
Martin Reinecke's avatar
Martin Reinecke committed
40
        if len(self._shape) == 0:
Martin Reinecke's avatar
fixes    
Martin Reinecke committed
41
            distaxis = -1
42
43
44
        self._distaxis = distaxis
        self._data = data

Martin Reinecke's avatar
tweaks    
Martin Reinecke committed
45
    def _sanity_checks(self):
46
        # check whether the distaxis is consistent
Martin Reinecke's avatar
Martin Reinecke committed
47
        if self._distaxis < -1 or self._distaxis >= len(self._shape):
48
            raise ValueError
Martin Reinecke's avatar
Martin Reinecke committed
49
50
51
52
        itmp = np.array(self._distaxis)
        otmp = np.empty(ntask, dtype=np.int)
        _comm.Allgather(itmp, otmp)
        if np.any(otmp != self._distaxis):
53
54
            raise ValueError
        # check whether the global shape is consistent
Martin Reinecke's avatar
Martin Reinecke committed
55
56
57
        itmp = np.array(self._shape)
        otmp = np.empty((ntask, len(self._shape)), dtype=np.int)
        _comm.Allgather(itmp, otmp)
58
        for i in range(ntask):
Martin Reinecke's avatar
Martin Reinecke committed
59
            if np.any(otmp[i, :] != self._shape):
60
61
                raise ValueError
        # check shape of local data
Martin Reinecke's avatar
Martin Reinecke committed
62
63
        if self._distaxis < 0:
            if self._data.shape != self._shape:
64
65
                raise ValueError
        else:
Martin Reinecke's avatar
Martin Reinecke committed
66
67
68
69
            itmp = np.array(self._shape)
            itmp[self._distaxis] = _shareSize(self._shape[self._distaxis],
                                              ntask, rank)
            if np.any(self._data.shape != itmp):
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
                raise ValueError

    @property
    def dtype(self):
        return self._data.dtype

    @property
    def shape(self):
        return self._shape

    @property
    def size(self):
        return np.prod(self._shape)

    @property
    def real(self):
Martin Reinecke's avatar
Martin Reinecke committed
86
        return data_object(self._shape, self._data.real, self._distaxis)
87
88
89

    @property
    def imag(self):
Martin Reinecke's avatar
Martin Reinecke committed
90
        return data_object(self._shape, self._data.imag, self._distaxis)
91

Martin Reinecke's avatar
Martin Reinecke committed
92
93
94
95
96
97
    def conj(self):
        return data_object(self._shape, self._data.conj(), self._distaxis)

    def conjugate(self):
        return data_object(self._shape, self._data.conjugate(), self._distaxis)

Martin Reinecke's avatar
Martin Reinecke committed
98
    def _contraction_helper(self, op, mpiop, axis):
99
        if axis is not None:
Martin Reinecke's avatar
Martin Reinecke committed
100
            if len(axis) == len(self._data.shape):
101
102
                axis = None
        if axis is None:
Martin Reinecke's avatar
Martin Reinecke committed
103
            res = np.array(getattr(self._data, op)())
Martin Reinecke's avatar
Martin Reinecke committed
104
            if (self._distaxis == -1):
Martin Reinecke's avatar
fixes    
Martin Reinecke committed
105
                return res[()]
Martin Reinecke's avatar
Martin Reinecke committed
106
107
            res2 = np.empty((), dtype=res.dtype)
            _comm.Allreduce(res, res2, mpiop)
Martin Reinecke's avatar
fixes    
Martin Reinecke committed
108
            return res2[()]
109
110

        if self._distaxis in axis:
Martin Reinecke's avatar
Martin Reinecke committed
111
112
            res = getattr(self._data, op)(axis=axis)
            res2 = np.empty_like(res)
Martin Reinecke's avatar
Martin Reinecke committed
113
            _comm.Allreduce(res, res2, mpiop)
Martin Reinecke's avatar
Martin Reinecke committed
114
            return from_global_data(res2, distaxis=0)
115
        else:
Martin Reinecke's avatar
Martin Reinecke committed
116
            # perform the contraction on the local data
Martin Reinecke's avatar
Martin Reinecke committed
117
118
            res = getattr(self._data, op)(axis=axis)
            if self._distaxis == -1:
Martin Reinecke's avatar
Martin Reinecke committed
119
                return from_global_data(res, distaxis=0)
Martin Reinecke's avatar
Martin Reinecke committed
120
            shp = list(res.shape)
Martin Reinecke's avatar
Martin Reinecke committed
121
            shift = 0
Martin Reinecke's avatar
Martin Reinecke committed
122
            for ax in axis:
Martin Reinecke's avatar
Martin Reinecke committed
123
124
                if ax < self._distaxis:
                    shift += 1
Martin Reinecke's avatar
Martin Reinecke committed
125
126
            shp[self._distaxis-shift] = self.shape[self._distaxis]
            return from_local_data(shp, res, self._distaxis-shift)
127
128
129

    def sum(self, axis=None):
        return self._contraction_helper("sum", MPI.SUM, axis)
Martin Reinecke's avatar
Martin Reinecke committed
130

Martin Reinecke's avatar
fixes    
Martin Reinecke committed
131
132
    def min(self, axis=None):
        return self._contraction_helper("min", MPI.MIN, axis)
Martin Reinecke's avatar
Martin Reinecke committed
133

Martin Reinecke's avatar
fixes    
Martin Reinecke committed
134
135
    def max(self, axis=None):
        return self._contraction_helper("max", MPI.MAX, axis)
136

Martin Reinecke's avatar
tweaks    
Martin Reinecke committed
137
138
    def mean(self):
        return self.sum()/self.size
Martin Reinecke's avatar
Martin Reinecke committed
139

Martin Reinecke's avatar
tweaks    
Martin Reinecke committed
140
141
    def std(self):
        return np.sqrt(self.var())
Martin Reinecke's avatar
Martin Reinecke committed
142

Martin Reinecke's avatar
Martin Reinecke committed
143
    # FIXME: to be improved!
Martin Reinecke's avatar
tweaks    
Martin Reinecke committed
144
145
146
    def var(self):
        return (abs(self-self.mean())**2).mean()

147
    def _binary_helper(self, other, op):
Martin Reinecke's avatar
Martin Reinecke committed
148
        a = self
149
        if isinstance(other, data_object):
Martin Reinecke's avatar
Martin Reinecke committed
150
            b = other
151
152
153
154
            if a._shape != b._shape:
                raise ValueError("shapes are incompatible.")
            if a._distaxis != b._distaxis:
                raise ValueError("distributions are incompatible.")
Martin Reinecke's avatar
Martin Reinecke committed
155
156
            a = a._data
            b = b._data
Martin Reinecke's avatar
Martin Reinecke committed
157
158
159
160
        elif np.isscalar(other):
            a = a._data
            b = other
        elif isinstance(other, np.ndarray):
Martin Reinecke's avatar
Martin Reinecke committed
161
            a = a._data
162
            b = other
Martin Reinecke's avatar
Martin Reinecke committed
163
164
        else:
            return NotImplemented
165
166

        tval = getattr(a, op)(b)
Martin Reinecke's avatar
Martin Reinecke committed
167
168
169
170
        if tval is a:
            return self
        else:
            return data_object(self._shape, tval, self._distaxis)
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204

    def __add__(self, other):
        return self._binary_helper(other, op='__add__')

    def __radd__(self, other):
        return self._binary_helper(other, op='__radd__')

    def __iadd__(self, other):
        return self._binary_helper(other, op='__iadd__')

    def __sub__(self, other):
        return self._binary_helper(other, op='__sub__')

    def __rsub__(self, other):
        return self._binary_helper(other, op='__rsub__')

    def __isub__(self, other):
        return self._binary_helper(other, op='__isub__')

    def __mul__(self, other):
        return self._binary_helper(other, op='__mul__')

    def __rmul__(self, other):
        return self._binary_helper(other, op='__rmul__')

    def __imul__(self, other):
        return self._binary_helper(other, op='__imul__')

    def __div__(self, other):
        return self._binary_helper(other, op='__div__')

    def __rdiv__(self, other):
        return self._binary_helper(other, op='__rdiv__')

Martin Reinecke's avatar
tweaks    
Martin Reinecke committed
205
206
207
    def __idiv__(self, other):
        return self._binary_helper(other, op='__idiv__')

208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
    def __truediv__(self, other):
        return self._binary_helper(other, op='__truediv__')

    def __rtruediv__(self, other):
        return self._binary_helper(other, op='__rtruediv__')

    def __pow__(self, other):
        return self._binary_helper(other, op='__pow__')

    def __rpow__(self, other):
        return self._binary_helper(other, op='__rpow__')

    def __ipow__(self, other):
        return self._binary_helper(other, op='__ipow__')

    def __eq__(self, other):
        return self._binary_helper(other, op='__eq__')

    def __ne__(self, other):
        return self._binary_helper(other, op='__ne__')

    def __neg__(self):
Martin Reinecke's avatar
Martin Reinecke committed
230
        return data_object(self._shape, -self._data, self._distaxis)
231
232

    def __abs__(self):
Martin Reinecke's avatar
Martin Reinecke committed
233
        return data_object(self._shape, np.abs(self._data), self._distaxis)
234
235

    def all(self):
Martin Reinecke's avatar
fixes    
Martin Reinecke committed
236
        return self.sum() == self.size
237
238

    def any(self):
Martin Reinecke's avatar
fixes    
Martin Reinecke committed
239
        return self.sum() != 0
240

Martin Reinecke's avatar
fixes    
Martin Reinecke committed
241
242
    def fill(self, value):
        self._data.fill(value)
243

Martin Reinecke's avatar
Martin Reinecke committed
244
def full(shape, fill_value, dtype=None, distaxis=0):
Martin Reinecke's avatar
Martin Reinecke committed
245
246
    return data_object(shape, np.full(local_shape(shape, distaxis),
                                      fill_value, dtype), distaxis)
247
248


Martin Reinecke's avatar
fixes    
Martin Reinecke committed
249
def empty(shape, dtype=None, distaxis=0):
Martin Reinecke's avatar
Martin Reinecke committed
250
251
    return data_object(shape, np.empty(local_shape(shape, distaxis),
                                       dtype), distaxis)
252
253


Martin Reinecke's avatar
fixes    
Martin Reinecke committed
254
def zeros(shape, dtype=None, distaxis=0):
Martin Reinecke's avatar
Martin Reinecke committed
255
256
    return data_object(shape, np.zeros(local_shape(shape, distaxis), dtype),
                       distaxis)
257
258


Martin Reinecke's avatar
fixes    
Martin Reinecke committed
259
def ones(shape, dtype=None, distaxis=0):
Martin Reinecke's avatar
Martin Reinecke committed
260
261
    return data_object(shape, np.ones(local_shape(shape, distaxis), dtype),
                       distaxis)
262
263
264
265
266
267
268


def empty_like(a, dtype=None):
    return data_object(np.empty_like(a._data, dtype))


def vdot(a, b):
Martin Reinecke's avatar
fixes    
Martin Reinecke committed
269
    tmp = np.array(np.vdot(a._data, b._data))
Martin Reinecke's avatar
Martin Reinecke committed
270
271
    res = np.empty((), dtype=tmp.dtype)
    _comm.Allreduce(tmp, res, MPI.SUM)
Martin Reinecke's avatar
fixes    
Martin Reinecke committed
272
    return res[()]
273
274
275
276
277
278
279


def _math_helper(x, function, out):
    if out is not None:
        function(x._data, out=out._data)
        return out
    else:
Martin Reinecke's avatar
Martin Reinecke committed
280
        return data_object(x.shape, function(x._data), x._distaxis)
281
282
283
284
285
286
287
288
289
290
291
292
293
294


def abs(a, out=None):
    return _math_helper(a, np.abs, out)


def exp(a, out=None):
    return _math_helper(a, np.exp, out)


def log(a, out=None):
    return _math_helper(a, np.log, out)


Martin Reinecke's avatar
Martin Reinecke committed
295
296
297
298
def tanh(a, out=None):
    return _math_helper(a, np.tanh, out)


299
300
301
302
303
def sqrt(a, out=None):
    return _math_helper(a, np.sqrt, out)


def from_object(object, dtype=None, copy=True):
Martin Reinecke's avatar
Martin Reinecke committed
304
305
306
    return data_object(object._shape, np.array(object._data, dtype=dtype,
                                               copy=copy),
                       distaxis=object._distaxis)
307
308


Martin Reinecke's avatar
Martin Reinecke committed
309
310
# This function draws all random numbers on all tasks, to produce the same
# array independent on the number of tasks
Martin Reinecke's avatar
Martin Reinecke committed
311
312
313
# MR FIXME: depending on what is really wanted/needed (i.e. same result
# independent of number of tasks, performance etc.) we need to adjust the
# algorithm.
Martin Reinecke's avatar
Martin Reinecke committed
314
def from_random(random_type, shape, dtype=np.float64, **kwargs):
315
    generator_function = getattr(Random, random_type)
Martin Reinecke's avatar
Martin Reinecke committed
316
317
318
319
320
321
322
    for i in range(ntask):
        lshape = list(shape)
        lshape[0] = _shareSize(shape[0], ntask, i)
        ldat = generator_function(dtype=dtype, shape=lshape, **kwargs)
        if i == rank:
            outdat = ldat
    return from_local_data(shape, outdat, distaxis=0)
323

Martin Reinecke's avatar
Martin Reinecke committed
324

Martin Reinecke's avatar
Martin Reinecke committed
325
326
327
328
def local_data(arr):
    return arr._data


Martin Reinecke's avatar
fixes    
Martin Reinecke committed
329
330
def ibegin(arr):
    res = [0] * arr._data.ndim
Martin Reinecke's avatar
Martin Reinecke committed
331
    res[arr._distaxis] = _shareRange(arr._shape[arr._distaxis], ntask, rank)[0]
Martin Reinecke's avatar
fixes    
Martin Reinecke committed
332
    return tuple(res)
Martin Reinecke's avatar
Martin Reinecke committed
333
334


Martin Reinecke's avatar
fixes    
Martin Reinecke committed
335
336
def np_allreduce_sum(arr):
    res = np.empty_like(arr)
Martin Reinecke's avatar
Martin Reinecke committed
337
    _comm.Allreduce(arr, res, MPI.SUM)
Martin Reinecke's avatar
fixes    
Martin Reinecke committed
338
    return res
Martin Reinecke's avatar
Martin Reinecke committed
339
340
341
342
343
344


def distaxis(arr):
    return arr._distaxis


Martin Reinecke's avatar
Martin Reinecke committed
345
def from_local_data(shape, arr, distaxis):
Martin Reinecke's avatar
Martin Reinecke committed
346
347
348
    return data_object(shape, arr, distaxis)


Martin Reinecke's avatar
Martin Reinecke committed
349
350
def from_global_data(arr, distaxis=0):
    if distaxis == -1:
Martin Reinecke's avatar
Martin Reinecke committed
351
        return data_object(arr.shape, arr, distaxis)
Martin Reinecke's avatar
Martin Reinecke committed
352
    lo, hi = _shareRange(arr.shape[distaxis], ntask, rank)
Martin Reinecke's avatar
Martin Reinecke committed
353
    sl = [slice(None)]*len(arr.shape)
Martin Reinecke's avatar
Martin Reinecke committed
354
    sl[distaxis] = slice(lo, hi)
Martin Reinecke's avatar
Martin Reinecke committed
355
356
357
    return data_object(arr.shape, arr[sl], distaxis)


Martin Reinecke's avatar
Martin Reinecke committed
358
359
def to_global_data(arr):
    if arr._distaxis == -1:
Martin Reinecke's avatar
fixes    
Martin Reinecke committed
360
361
362
363
364
        return arr._data
    tmp = redistribute(arr, dist=-1)
    return tmp._data


Martin Reinecke's avatar
Martin Reinecke committed
365
def redistribute(arr, dist=None, nodist=None):
Martin Reinecke's avatar
Martin Reinecke committed
366
367
368
    if dist is not None:
        if nodist is not None:
            raise ValueError
Martin Reinecke's avatar
Martin Reinecke committed
369
        if dist == arr._distaxis:
Martin Reinecke's avatar
Martin Reinecke committed
370
371
372
373
374
375
            return arr
    else:
        if nodist is None:
            raise ValueError
        if arr._distaxis not in nodist:
            return arr
Martin Reinecke's avatar
Martin Reinecke committed
376
        dist = -1
Martin Reinecke's avatar
Martin Reinecke committed
377
378
        for i in range(len(arr.shape)):
            if i not in nodist:
Martin Reinecke's avatar
Martin Reinecke committed
379
                dist = i
Martin Reinecke's avatar
Martin Reinecke committed
380
                break
Martin Reinecke's avatar
Martin Reinecke committed
381

Martin Reinecke's avatar
Martin Reinecke committed
382
    if arr._distaxis == -1:  # all data available, just pick the proper subset
Martin Reinecke's avatar
Martin Reinecke committed
383
        return from_global_data(arr._data, dist)
Martin Reinecke's avatar
Martin Reinecke committed
384
    if dist == -1:  # gather all data on all tasks
Martin Reinecke's avatar
Martin Reinecke committed
385
        tmp = np.moveaxis(arr._data, arr._distaxis, 0)
Martin Reinecke's avatar
Martin Reinecke committed
386
387
        slabsize = np.prod(tmp.shape[1:])*tmp.itemsize
        sz = np.empty(ntask, dtype=np.int)
Martin Reinecke's avatar
Martin Reinecke committed
388
        for i in range(ntask):
Martin Reinecke's avatar
Martin Reinecke committed
389
390
391
392
            sz[i] = slabsize*_shareSize(arr.shape[arr._distaxis], ntask, i)
        disp = np.empty(ntask, dtype=np.int)
        disp[0] = 0
        disp[1:] = np.cumsum(sz[:-1])
Martin Reinecke's avatar
Martin Reinecke committed
393
        tmp = np.require(tmp, requirements="C")
Martin Reinecke's avatar
Martin Reinecke committed
394
395
        out = np.empty(arr.size, dtype=arr.dtype)
        _comm.Allgatherv(tmp, [out, sz, disp, MPI.BYTE])
Martin Reinecke's avatar
Martin Reinecke committed
396
397
398
399
        shp = np.array(arr._shape)
        shp[1:arr._distaxis+1] = shp[0:arr._distaxis]
        shp[0] = arr.shape[arr._distaxis]
        out = out.reshape(shp)
Martin Reinecke's avatar
Martin Reinecke committed
400
        out = np.moveaxis(out, 0, arr._distaxis)
Martin Reinecke's avatar
Martin Reinecke committed
401
        return from_global_data(out, distaxis=-1)
Martin Reinecke's avatar
Martin Reinecke committed
402

Martin Reinecke's avatar
Martin Reinecke committed
403
    # real redistribution via Alltoallv
Martin Reinecke's avatar
Martin Reinecke committed
404
    ssz0 = arr._data.size//arr.shape[dist]
Martin Reinecke's avatar
Martin Reinecke committed
405
    ssz = np.empty(ntask, dtype=np.int)
Martin Reinecke's avatar
Martin Reinecke committed
406
407
408
    rszall = arr.size//arr.shape[dist]*_shareSize(arr.shape[dist], ntask, rank)
    rbuf = np.empty(rszall, dtype=arr.dtype)
    rsz0 = rszall//arr.shape[arr._distaxis]
Martin Reinecke's avatar
Martin Reinecke committed
409
    rsz = np.empty(ntask, dtype=np.int)
Martin Reinecke's avatar
Martin Reinecke committed
410
411
412
413
414
415
416
417
418
419
420
421
    if dist == 0:  # shortcut possible
        sbuf = np.ascontiguousarray(arr._data)
        for i in range(ntask):
            lo, hi = _shareRange(arr.shape[dist], ntask, i)
            ssz[i] = ssz0*(hi-lo)
            rsz[i] = rsz0*_shareSize(arr.shape[arr._distaxis], ntask, i)
    else:
        sbuf = np.empty(arr._data.size, dtype=arr.dtype)
        sslice = [slice(None)]*arr._data.ndim
        ofs = 0
        for i in range(ntask):
            lo, hi = _shareRange(arr.shape[dist], ntask, i)
Martin Reinecke's avatar
Martin Reinecke committed
422
            sslice[dist] = slice(lo, hi)
Martin Reinecke's avatar
Martin Reinecke committed
423
424
425
426
427
428
            ssz[i] = ssz0*(hi-lo)
            sbuf[ofs:ofs+ssz[i]] = arr._data[sslice].flat
            ofs += ssz[i]
            rsz[i] = rsz0*_shareSize(arr.shape[arr._distaxis], ntask, i)
    ssz *= arr._data.itemsize
    rsz *= arr._data.itemsize
Martin Reinecke's avatar
Martin Reinecke committed
429
430
    sdisp = np.append(0, np.cumsum(ssz[:-1]))
    rdisp = np.append(0, np.cumsum(rsz[:-1]))
Martin Reinecke's avatar
Martin Reinecke committed
431
432
    s_msg = [sbuf, (ssz, sdisp), MPI.BYTE]
    r_msg = [rbuf, (rsz, rdisp), MPI.BYTE]
Martin Reinecke's avatar
tweaks    
Martin Reinecke committed
433
    _comm.Alltoallv(s_msg, r_msg)
Martin Reinecke's avatar
Martin Reinecke committed
434
    del sbuf  # free memory
Martin Reinecke's avatar
Martin Reinecke committed
435
436
437
438
439
440
441
442
443
    if arr._distaxis == 0:
        rbuf = rbuf.reshape(local_shape(arr.shape, dist))
        arrnew = from_local_data(arr.shape, rbuf, distaxis=dist)
    else:
        arrnew = empty(arr.shape, dtype=arr.dtype, distaxis=dist)
        rslice = [slice(None)]*arr._data.ndim
        ofs = 0
        for i in range(ntask):
            lo, hi = _shareRange(arr.shape[arr._distaxis], ntask, i)
Martin Reinecke's avatar
Martin Reinecke committed
444
            rslice[arr._distaxis] = slice(lo, hi)
Martin Reinecke's avatar
Martin Reinecke committed
445
446
447
448
            sz = rsz[i]//arr._data.itemsize
            arrnew._data[rslice].flat = rbuf[ofs:ofs+sz]
            ofs += sz
    return arrnew
Martin Reinecke's avatar
Martin Reinecke committed
449
450


Martin Reinecke's avatar
Martin Reinecke committed
451
452
def transpose(arr):
    if len(arr.shape) != 2 or arr._distaxis != 0:
Martin Reinecke's avatar
Martin Reinecke committed
453
        raise ValueError("bad input")
Martin Reinecke's avatar
Martin Reinecke committed
454
455
456
457
458
459
460
461
462
463
464
    ssz0 = arr._data.size//arr.shape[1]
    ssz = np.empty(ntask, dtype=np.int)
    rszall = arr.size//arr.shape[1]*_shareSize(arr.shape[1], ntask, rank)
    rbuf = np.empty(rszall, dtype=arr.dtype)
    rsz0 = rszall//arr.shape[0]
    rsz = np.empty(ntask, dtype=np.int)
    sbuf = np.empty(arr._data.size, dtype=arr.dtype)
    ofs = 0
    for i in range(ntask):
        lo, hi = _shareRange(arr.shape[1], ntask, i)
        ssz[i] = ssz0*(hi-lo)
Martin Reinecke's avatar
Martin Reinecke committed
465
        sbuf[ofs:ofs+ssz[i]] = arr._data[:, lo:hi].flat
Martin Reinecke's avatar
Martin Reinecke committed
466
467
468
469
470
471
472
473
474
475
476
477
        ofs += ssz[i]
        rsz[i] = rsz0*_shareSize(arr.shape[0], ntask, i)
    ssz *= arr._data.itemsize
    rsz *= arr._data.itemsize
    sdisp = np.append(0, np.cumsum(ssz[:-1]))
    rdisp = np.append(0, np.cumsum(rsz[:-1]))
    s_msg = [sbuf, (ssz, sdisp), MPI.BYTE]
    r_msg = [rbuf, (rsz, rdisp), MPI.BYTE]
    _comm.Alltoallv(s_msg, r_msg)
    del sbuf  # free memory
    arrnew = empty((arr.shape[1], arr.shape[0]), dtype=arr.dtype, distaxis=0)
    ofs = 0
Martin Reinecke's avatar
Martin Reinecke committed
478
    sz2 = _shareSize(arr.shape[1], ntask, rank)
Martin Reinecke's avatar
Martin Reinecke committed
479
480
481
    for i in range(ntask):
        lo, hi = _shareRange(arr.shape[0], ntask, i)
        sz = rsz[i]//arr._data.itemsize
Martin Reinecke's avatar
Martin Reinecke committed
482
        arrnew._data[:, lo:hi] = rbuf[ofs:ofs+sz].reshape(hi-lo, sz2).T
Martin Reinecke's avatar
Martin Reinecke committed
483
484
485
486
        ofs += sz
    return arrnew


Martin Reinecke's avatar
Martin Reinecke committed
487
488
def default_distaxis():
    return 0