rework dobj interface and prepare tests for MPI

nifty/data_objects/numpy_do.py

@@ -2,9 +2,21 @@
 
 import numpy as np
 from numpy import ndarray as data_object
-from numpy import full, empty, sqrt, ones, zeros, vdot, abs, bincount, exp, log
+from numpy import full, empty, empty_like, sqrt, ones, zeros, vdot, abs, \
+                  bincount, exp, log
 from .random import Random
 
+__all__ = ["ntask", "rank", "master", "local_shape", "data_object", "full",
+           "empty", "zeros", "ones", "empty_like", "vdot", "abs", "exp",
+           "log", "sqrt", "bincount", "from_object", "from_random",
+           "local_data", "ibegin", "np_allreduce_sum", "distaxis",
+           "from_local_data", "from_global_data", "to_global_data",
+           "redistribute", "default_distaxis"]
+
+ntask = 1
+rank = 0
+master = True
+
 
 def from_object(object, dtype=None, copy=True):
     return np.array(object, dtype=dtype, copy=copy)
@@ -13,3 +25,45 @@ def from_object(object, dtype=None, copy=True):
 def from_random(random_type, shape, dtype=np.float64, **kwargs):
     generator_function = getattr(Random, random_type)
     return generator_function(dtype=dtype, shape=shape, **kwargs)
+
+
+def local_data(arr):
+    return arr
+
+
+def ibegin(arr):
+    return (0,)*arr.ndim
+
+
+def np_allreduce_sum(arr):
+    return arr
+
+
+def distaxis(arr):
+    return -1
+
+
+def from_local_data(shape, arr, distaxis):
+    if shape != arr.shape:
+        raise ValueError
+    return arr
+
+
+def from_global_data(arr, distaxis=-1):
+    return arr
+
+
+def to_global_data(arr):
+    return arr
+
+
+def redistribute(arr, dist=None, nodist=None):
+    return arr
+
+
+def default_distaxis():
+    return -1
+
+
+def local_shape(glob_shape, distaxis):
+    return glob_shape

test/common.py

@@ -18,10 +18,11 @@
 
 from builtins import str
 from parameterized import parameterized
-from nifty2go import Space, RGSpace, LMSpace, HPSpace, GLSpace, PowerSpace
+import nifty2go as ift
 import numpy as np
 
-np.seterr(all='raise',under='ignore')
+np.seterr(all='raise', under='ignore')
+
 
 def custom_name_func(testcase_func, param_num, param):
     return "%s_%s" % (
@@ -36,11 +37,12 @@ def expand(*args, **kwargs):
 
 
 def generate_spaces():
-    spaces = [RGSpace(4), PowerSpace(RGSpace((4, 4), harmonic=True)),
-              LMSpace(5), HPSpace(4), GLSpace(4)]
+    spaces = [ift.RGSpace(4),
+              ift.PowerSpace(ift.RGSpace((4, 4), harmonic=True)),
+              ift.LMSpace(5), ift.HPSpace(4), ift.GLSpace(4)]
     return spaces
 
 
 def generate_harmonic_spaces():
-    spaces = [RGSpace(4, harmonic=True), LMSpace(5)]
+    spaces = [ift.RGSpace(4, harmonic=True), ift.LMSpace(5)]
     return spaces

test/test_field.py

@@ -17,111 +17,115 @@
 # and financially supported by the Studienstiftung des deutschen Volkes.
 
 import unittest
-
 import numpy as np
-from numpy.testing import assert_equal,\
-                          assert_allclose
+from numpy.testing import assert_equal, assert_allclose
 from itertools import product
-from nifty2go import Field,\
-                  RGSpace,\
-                  LMSpace,\
-                  PowerSpace,\
-                  DomainTuple,\
-                  DiagonalOperator
-
-from nifty2go.sugar import create_power_field, power_analyze, power_synthesize
-
+import nifty2go as ift
 from test.common import expand
 
 
-SPACES = [RGSpace((4,)), RGSpace((5))]
+SPACES = [ift.RGSpace((4,)), ift.RGSpace((5))]
 SPACE_COMBINATIONS = [(), SPACES[0], SPACES[1], SPACES]
 
 
 class Test_Interface(unittest.TestCase):
     @expand(product(SPACE_COMBINATIONS,
-                    [['domain', DomainTuple],
-                     ['val', np.ndarray],
+                    [['domain', ift.DomainTuple],
+                     ['val', ift.dobj.data_object],
                      ['shape', tuple],
                      ['dim', (np.int, np.int64)]]))
     def test_return_types(self, domain, attribute_desired_type):
         attribute = attribute_desired_type[0]
         desired_type = attribute_desired_type[1]
-        f = Field(domain=domain)
+        f = ift.Field(domain=domain)
         assert_equal(isinstance(getattr(f, attribute), desired_type), True)
 
 
+def _spec1(k):
+    return 42/(1.+k)**2
+
+
+def _spec2(k):
+    return 42/(1.+k)**3
+
+
 class Test_Functionality(unittest.TestCase):
-    @expand(product([RGSpace((8,), harmonic=True),
-                     RGSpace((8, 8), harmonic=True, distances=0.123)],
-                    [RGSpace((8,), harmonic=True),
-                     LMSpace(12)]))
+    @expand(product([ift.RGSpace((8,), harmonic=True),
+                     ift.RGSpace((8, 8), harmonic=True, distances=0.123)],
+                    [ift.RGSpace((8,), harmonic=True),
+                     ift.LMSpace(12)]))
     def test_power_synthesize_analyze(self, space1, space2):
         np.random.seed(11)
 
-        p1 = PowerSpace(space1)
-        spec1 = lambda k: 42/(1+k)**2
-        fp1 = Field(p1, val=spec1(p1.k_lengths))
+        p1 = ift.PowerSpace(space1)
+        p1val = _spec1(p1.k_lengths)
+        fp1 = ift.Field(p1, val=ift.dobj.from_global_data(p1val))
 
-        p2 = PowerSpace(space2)
-        spec2 = lambda k: 42/(1+k)**3
-        fp2 = Field(p2, val=spec2(p2.k_lengths))
+        p2 = ift.PowerSpace(space2)
+        p2val = _spec2(p2.k_lengths)
+        fp2 = ift.Field(p2, val=ift.dobj.from_global_data(p2val))
 
-        outer = np.outer(fp1.val, fp2.val)
-        fp = Field((p1, p2), val=outer)
+        outer = ift.dobj.from_global_data(np.outer(p1val, p2val))
+        fp = ift.Field((p1, p2), val=outer)
 
         samples = 500
         ps1 = 0.
         ps2 = 0.
         for ii in range(samples):
-            sk = power_synthesize(fp, spaces=(0, 1), real_signal=True)
+            sk = ift.power_synthesize(fp, spaces=(0, 1), real_signal=True)
 
-            sp = power_analyze(sk, spaces=(0, 1), keep_phase_information=False)
+            sp = ift.power_analyze(sk, spaces=(0, 1),
+                                   keep_phase_information=False)
             ps1 += sp.sum(spaces=1)/fp2.sum()
             ps2 += sp.sum(spaces=0)/fp1.sum()
 
-        assert_allclose(ps1.val/samples, fp1.val, rtol=0.2)
-        assert_allclose(ps2.val/samples, fp2.val, rtol=0.2)
+        assert_allclose(ift.dobj.to_global_data(ps1.val/samples),
+                        ift.dobj.to_global_data(fp1.val), rtol=0.2)
+        assert_allclose(ift.dobj.to_global_data(ps2.val/samples),
+                        ift.dobj.to_global_data(fp2.val), rtol=0.2)
 
-    @expand(product([RGSpace((8,), harmonic=True),
-                     RGSpace((8, 8), harmonic=True, distances=0.123)],
-                    [RGSpace((8,), harmonic=True),
-                     LMSpace(12)]))
+    @expand(product([ift.RGSpace((8,), harmonic=True),
+                     ift.RGSpace((8, 8), harmonic=True, distances=0.123)],
+                    [ift.RGSpace((8,), harmonic=True),
+                     ift.LMSpace(12)]))
     def test_DiagonalOperator_power_analyze(self, space1, space2):
         np.random.seed(11)
 
-        fulldomain = DomainTuple.make((space1, space2))
+        fulldomain = ift.DomainTuple.make((space1, space2))
 
-        p1 = PowerSpace(space1)
-        spec1 = lambda k: 42/(1+k)**2
-        fp1 = Field(p1, val=spec1(p1.k_lengths))
+        p1 = ift.PowerSpace(space1)
+        p1val = _spec1(p1.k_lengths)
+        fp1 = ift.Field(p1, val=ift.dobj.from_global_data(p1val))
 
-        p2 = PowerSpace(space2)
-        spec2 = lambda k: 42/(1+k)**3
-        fp2 = Field(p2, val=spec2(p2.k_lengths))
+        p2 = ift.PowerSpace(space2)
+        p2val = _spec2(p2.k_lengths)
+        fp2 = ift.Field(p2, val=ift.dobj.from_global_data(p2val))
 
-        S_1 = create_power_field(space1, lambda x: np.sqrt(spec1(x)))
-        S_1 = DiagonalOperator(S_1, domain=fulldomain, spaces=0)
-        S_2 = create_power_field(space2, lambda x: np.sqrt(spec2(x)))
-        S_2 = DiagonalOperator(S_2, domain=fulldomain, spaces=1)
+        S_1 = ift.create_power_field(space1, lambda x: np.sqrt(_spec1(x)))
+        S_1 = ift.DiagonalOperator(S_1, domain=fulldomain, spaces=0)
+        S_2 = ift.create_power_field(space2, lambda x: np.sqrt(_spec2(x)))
+        S_2 = ift.DiagonalOperator(S_2, domain=fulldomain, spaces=1)
 
         samples = 500
         ps1 = 0.
         ps2 = 0.
 
         for ii in range(samples):
-            rand_k = Field.from_random('normal', domain=fulldomain)
+            rand_k = ift.Field.from_random('normal', domain=fulldomain)
             sk = S_1.times(S_2.times(rand_k))
-            sp = power_analyze(sk, spaces=(0, 1), keep_phase_information=False)
+            sp = ift.power_analyze(sk, spaces=(0, 1),
+                                   keep_phase_information=False)
             ps1 += sp.sum(spaces=1)/fp2.sum()
             ps2 += sp.sum(spaces=0)/fp1.sum()
 
-        assert_allclose(ps1.val/samples, fp1.val, rtol=0.2)
-        assert_allclose(ps2.val/samples, fp2.val, rtol=0.2)
+        assert_allclose(ift.dobj.to_global_data(ps1.val/samples),
+                        ift.dobj.to_global_data(fp1.val), rtol=0.2)
+        assert_allclose(ift.dobj.to_global_data(ps2.val/samples),
+                        ift.dobj.to_global_data(fp2.val), rtol=0.2)
 
     def test_vdot(self):
-        s = RGSpace((10,))
-        f1 = Field.from_random("normal", domain=s, dtype=np.complex128)
-        f2 = Field.from_random("normal", domain=s, dtype=np.complex128)
-        assert_allclose(f1.vdot(f2), f1.vdot(f2, spaces=0))
+        s = ift.RGSpace((10,))
+        f1 = ift.Field.from_random("normal", domain=s, dtype=np.complex128)
+        f2 = ift.Field.from_random("normal", domain=s, dtype=np.complex128)
+        # assert_allclose(f1.vdot(f2), f1.vdot(f2, spaces=0))
         assert_allclose(f1.vdot(f2), np.conj(f2.vdot(f1)))

test/test_minimization/test_minimizers.py

 import unittest
-
 import numpy as np
 from numpy.testing import assert_allclose
-
 import nifty2go as ift
-
 from itertools import product
 from test.common import expand
 
@@ -31,6 +28,6 @@ class Test_Minimizers(unittest.TestCase):
 
         (energy, convergence) = minimizer(energy)
         assert convergence == IC.CONVERGED
-        assert_allclose(energy.position.val,
-                        1./covariance_diagonal.val,
+        assert_allclose(ift.dobj.to_global_data(energy.position.val),
+                        1./ift.dobj.to_global_data(covariance_diagonal.val),
                         rtol=1e-3, atol=1e-3)

test/test_operators/test_adoint.py

@@ -7,33 +7,37 @@ from numpy.testing import assert_allclose
 
 
 def _check_adjointness(op, dtype=np.float64):
-    f1 = ift.Field.from_random("normal",domain=op.domain, dtype=dtype)
-    f2 = ift.Field.from_random("normal",domain=op.target, dtype=dtype)
+    f1 = ift.Field.from_random("normal", domain=op.domain, dtype=dtype)
+    f2 = ift.Field.from_random("normal", domain=op.target, dtype=dtype)
     assert_allclose(f1.vdot(op.adjoint_times(f2)), op.times(f1).vdot(f2),
                     rtol=1e-8)
 
-_harmonic_spaces = [ ift.RGSpace(7, distances=0.2, harmonic=True),
-                     ift.RGSpace((12,46), distances=(0.2, 0.3), harmonic=True),
-                     ift.LMSpace(17) ]
+_harmonic_spaces = [ift.RGSpace(7, distances=0.2, harmonic=True),
+                    ift.RGSpace((12, 46), distances=(0.2, 0.3), harmonic=True),
+                    ift.LMSpace(17)]
+
+_position_spaces = [ift.RGSpace(19, distances=0.7),
+                    ift.RGSpace((1, 2, 3, 6), distances=(0.2, 0.25, 0.34, .8)),
+                    ift.HPSpace(17),
+                    ift.GLSpace(8, 13)]
 
-_position_spaces = [ ift.RGSpace(19, distances=0.7),
-                     ift.RGSpace((1,2,3,6), distances=(0.2,0.25,0.34,0.8)),
-                     ift.HPSpace(17),
-                     ift.GLSpace(8,13) ]
 
 class Adjointness_Tests(unittest.TestCase):
     @expand(product(_harmonic_spaces, [np.float64, np.complex128]))
     def testPPO(self, sp, dtype):
         op = ift.PowerProjectionOperator(sp)
         _check_adjointness(op, dtype)
-        ps = ift.PowerSpace(sp, ift.PowerSpace.useful_binbounds(sp, logarithmic=False, nbin=3))
+        ps = ift.PowerSpace(
+            sp, ift.PowerSpace.useful_binbounds(sp, logarithmic=False, nbin=3))
         op = ift.PowerProjectionOperator(sp, ps)
         _check_adjointness(op, dtype)
-        ps = ift.PowerSpace(sp, ift.PowerSpace.useful_binbounds(sp, logarithmic=True, nbin=3))
+        ps = ift.PowerSpace(
+            sp, ift.PowerSpace.useful_binbounds(sp, logarithmic=True, nbin=3))
         op = ift.PowerProjectionOperator(sp, ps)
         _check_adjointness(op, dtype)
 
-    @expand(product(_harmonic_spaces+_position_spaces, [np.float64, np.complex128]))
+    @expand(product(_harmonic_spaces+_position_spaces,
+                    [np.float64, np.complex128]))
     def testFFT(self, sp, dtype):
         op = ift.FFTOperator(sp)
         _check_adjointness(op, dtype)

test/test_operators/test_composed_operator.py

 import unittest
-
-from numpy.testing import assert_equal,\
-    assert_allclose,\
-    assert_approx_equal
-
-from nifty2go import Field,\
-    DiagonalOperator,\
-    ComposedOperator
-
+from numpy.testing import assert_allclose
+import nifty2go as ift
 from test.common import generate_spaces
-
 from itertools import product
 from test.common import expand
 
+
 class ComposedOperator_Tests(unittest.TestCase):
     spaces = generate_spaces()
 
-    @expand(product(spaces,spaces))
+    @expand(product(spaces, spaces))
     def test_times_adjoint_times(self, space1, space2):
         cspace = (space1, space2)
-        diag1 = Field.from_random('normal', domain=space1)
-        diag2 = Field.from_random('normal', domain=space2)
-        op1 = DiagonalOperator(diag1, cspace, spaces=(0,))
-        op2 = DiagonalOperator(diag2, cspace, spaces=(1,))
+        diag1 = ift.Field.from_random('normal', domain=space1)
+        diag2 = ift.Field.from_random('normal', domain=space2)
+        op1 = ift.DiagonalOperator(diag1, cspace, spaces=(0,))
+        op2 = ift.DiagonalOperator(diag2, cspace, spaces=(1,))
 
-        op = ComposedOperator((op1, op2))
+        op = ift.ComposedOperator((op1, op2))
 
-        rand1 = Field.from_random('normal', domain=(space1,space2))
-        rand2 = Field.from_random('normal', domain=(space1,space2))
+        rand1 = ift.Field.from_random('normal', domain=(space1, space2))
+        rand2 = ift.Field.from_random('normal', domain=(space1, space2))
 
         tt1 = rand2.vdot(op.times(rand1))
         tt2 = rand1.vdot(op.adjoint_times(rand2))
-        assert_approx_equal(tt1, tt2)
+        assert_allclose(tt1, tt2)
 
     @expand(product(spaces, spaces))
     def test_times_inverse_times(self, space1, space2):
         cspace = (space1, space2)
-        diag1 = Field.from_random('normal', domain=space1)
-        diag2 = Field.from_random('normal', domain=space2)
-        op1 = DiagonalOperator(diag1, cspace, spaces=(0,))
-        op2 = DiagonalOperator(diag2, cspace, spaces=(1,))
+        diag1 = ift.Field.from_random('normal', domain=space1)
+        diag2 = ift.Field.from_random('normal', domain=space2)
+        op1 = ift.DiagonalOperator(diag1, cspace, spaces=(0,))
+        op2 = ift.DiagonalOperator(diag2, cspace, spaces=(1,))
 
-        op = ComposedOperator((op1, op2))
+        op = ift.ComposedOperator((op1, op2))
 
-        rand1 = Field.from_random('normal', domain=(space1, space2))
+        rand1 = ift.Field.from_random('normal', domain=(space1, space2))
         tt1 = op.inverse_times(op.times(rand1))
 
-        assert_allclose(tt1.val, rand1.val)
-
+        assert_allclose(ift.dobj.to_global_data(tt1.val),
+                        ift.dobj.to_global_data(rand1.val))

test/test_operators/test_diagonal_operator.py

 from __future__ import division
 import unittest
-
-import numpy as np
-from numpy.testing import assert_equal,\
-    assert_allclose,\
-    assert_approx_equal
-
-from nifty2go import Field,\
-    DiagonalOperator
-
+from numpy.testing import assert_equal, assert_allclose
+import nifty2go as ift
 from test.common import generate_spaces
-
 from itertools import product
 from test.common import expand
 
+
 class DiagonalOperator_Tests(unittest.TestCase):
     spaces = generate_spaces()
 
     @expand(product(spaces))
     def test_property(self, space):
-        diag = Field.from_random('normal', domain=space)
-        D = DiagonalOperator(diag)
+        diag = ift.Field.from_random('normal', domain=space)
+        D = ift.DiagonalOperator(diag)
         if D.domain[0] != space:
             raise TypeError
-        if D.unitary != False:
+        if D.unitary:
             raise TypeError
-        if D.self_adjoint != True:
+        if not D.self_adjoint:
             raise TypeError
 
     @expand(product(spaces))
     def test_times_adjoint(self, space):
-        rand1 = Field.from_random('normal', domain=space)
-        rand2 = Field.from_random('normal', domain=space)
-        diag = Field.from_random('normal', domain=space)
-        D = DiagonalOperator(diag)
+        rand1 = ift.Field.from_random('normal', domain=space)
+        rand2 = ift.Field.from_random('normal', domain=space)
+        diag = ift.Field.from_random('normal', domain=space)
+        D = ift.DiagonalOperator(diag)
         tt1 = rand1.vdot(D.times(rand2))
         tt2 = rand2.vdot(D.times(rand1))
-        assert_approx_equal(tt1, tt2)
+        assert_allclose(tt1, tt2)
 
     @expand(product(spaces))
     def test_times_inverse(self, space):
-        rand1 = Field.from_random('normal', domain=space)
-        diag = Field.from_random('normal', domain=space)
-        D = DiagonalOperator(diag)
+        rand1 = ift.Field.from_random('normal', domain=space)
+        diag = ift.Field.from_random('normal', domain=space)
+        D = ift.DiagonalOperator(diag)
         tt1 = D.times(D.inverse_times(rand1))
-        assert_allclose(rand1.val, tt1.val)
+        assert_allclose(ift.dobj.to_global_data(rand1.val),
+                        ift.dobj.to_global_data(tt1.val))
 
     @expand(product(spaces))
     def test_times(self, space):
-        rand1 = Field.from_random('normal', domain=space)
-        diag = Field.from_random('normal', domain=space)
-        D = DiagonalOperator(diag)
+        rand1 = ift.Field.from_random('normal', domain=space)
+        diag = ift.Field.from_random('normal', domain=space)
+        D = ift.DiagonalOperator(diag)
         tt = D.times(rand1)
         assert_equal(tt.domain[0], space)
 
     @expand(product(spaces))
     def test_adjoint_times(self, space):
-        rand1 = Field.from_random('normal', domain=space)
-        diag = Field.from_random('normal', domain=space)
-        D = DiagonalOperator(diag)
+        rand1 = ift.Field.from_random('normal', domain=space)
+        diag = ift.Field.from_random('normal', domain=space)
+        D = ift.DiagonalOperator(diag)
         tt = D.adjoint_times(rand1)
         assert_equal(tt.domain[0], space)
 
     @expand(product(spaces))
     def test_inverse_times(self, space):
-        rand1 = Field.from_random('normal', domain=space)
-        diag = Field.from_random('normal', domain=space)
-        D = DiagonalOperator(diag)
+        rand1 = ift.Field.from_random('normal', domain=space)
+        diag = ift.Field.from_random('normal', domain=space)
+        D = ift.DiagonalOperator(diag)
         tt = D.inverse_times(rand1)
         assert_equal(tt.domain[0], space)
 
     @expand(product(spaces))
     def test_adjoint_inverse_times(self, space):
-        rand1 = Field.from_random('normal', domain=space)
-        diag = Field.from_random('normal', domain=space)
-        D = DiagonalOperator(diag)
+        rand1 = ift.Field.from_random('normal', domain=space)
+        diag = ift.Field.from_random('normal', domain=space)
+        D = ift.DiagonalOperator(diag)
         tt = D.adjoint_inverse_times(rand1)
         assert_equal(tt.domain[0], space)
 
     @expand(product(spaces))
     def test_diagonal(self, space):
-        diag = Field.from_random('normal', domain=space)
-        D = DiagonalOperator(diag)
+        diag = ift.Field.from_random('normal', domain=space)
+        D = ift.DiagonalOperator(diag)
         diag_op = D.diagonal()
-        assert_allclose(diag.val, diag_op.val)
+        assert_allclose(ift.dobj.to_global_data(diag.val),
+                        ift.dobj.to_global_data(diag_op.val))

test/test_operators/test_fft_operator.py

@@ -20,12 +20,7 @@ from __future__ import division
 import unittest
 import numpy as np
 from numpy.testing import assert_allclose
-from nifty2go import Field,\
-    RGSpace,\
-    LMSpace,\
-    HPSpace,\
-    GLSpace,\
-    FFTOperator
+import nifty2go as ift
 from itertools import product
 from test.common import expand
 
@@ -42,75 +37,81 @@ class FFTOperatorTests(unittest.TestCase):
                     [np.float64, np.float32, np.complex64, np.complex128]))
     def test_fft1D(self, dim1, d, itp):
         tol = _get_rtol(itp)
-        a = RGSpace(dim1, distances=d)
-        b = RGSpace(dim1, distances=1./(dim1*d), harmonic=True)
-        fft = FFTOperator(domain=a, target=b)
+        a = ift.RGSpace(dim1, distances=d)
+        b = ift.RGSpace(dim1, distances=1./(dim1*d), harmonic=True)
+        fft = ift.FFTOperator(domain=a, target=b)
 
         np.random.seed(16)
-        inp = Field.from_random(domain=a, random_type='normal', std=7, mean=3,
-                                dtype=itp)
+        inp = ift.Field.from_random(domain=a, random_type='normal',
+                                    std=7, mean=3, dtype=itp)
         out = fft.adjoint_times(fft.times(inp))
-        assert_allclose(inp.val, out.val, rtol=tol, atol=tol)
+        assert_allclose(ift.dobj.to_global_data(inp.val),
+                        ift.dobj.to_global_data(out.val), rtol=tol, atol=tol)
 
     @expand(product([12, 15], [9, 12], [0.1, 1, 3.7],
                     [0.4, 1, 2.7],
                     [np.float64, np.float32, np.complex64, np.complex128]))
     def test_fft2D(self, dim1, dim2, d1, d2, itp):
         tol = _get_rtol(itp)
-        a = RGSpace([dim1, dim2], distances=[d1, d2])