WIP: initial ci config

.gitlab-ci.yml

+image: ubuntu:latest
+
+stages:
+  - test
+  - release
+
+variables:
+  DOCKER_DRIVER: overlay
+  RELEASE_IMAGE: iftmpa/nifty:ci_test
+
+before_script:
+  - apt-get update
+  - >
+    apt-get install -y build-essential python python-pip python-dev git 
+    gfortran autoconf gsl-bin libgsl-dev
+  - pip install -r ci/requirements_base.txt
+  - chmod +x ci/*.sh
+
+test_min:
+  stage: test
+  script:
+    - python setup.py build_ext --inplace
+    - nosetests -vv
+
+test_mpi:
+  stage: test
+  script:
+    - apt-get install -y openmpi-bin libopenmpi-dev
+    - pip install astropy healpy mpi4py
+    - pip install git+https://github.com/mrbell/gfft
+    - ci/install_libsharp.sh
+    - python setup.py build_ext --inplace
+    - nosetests -vv
+
+test_mpi_fftw:
+  stage: test
+  script:
+    - apt-get install -y openmpi-bin libopenmpi-dev
+    - >
+      apt-get install -y libatlas-base-dev libfftw3-bin libfftw3-dev
+      libfftw3-double3 libfftw3-long3 libfftw3-mpi-dev libfftw3-mpi3
+      libfftw3-quad3 libfftw3-single3
+    - pip install astropy healpy mpi4py
+    - pip install git+https://github.com/mrbell/gfft
+    - ci/install_libsharp.sh
+    - ci/install_pyfftw.sh
+    - python setup.py build_ext --inplace
+    - nosetests -vv
+
+test_mpi_fftw_hdf5:
+  stage: test
+  script:
+    - apt-get install -y openmpi-bin libopenmpi-dev
+    - >
+      apt-get install -y libatlas-base-dev libfftw3-bin libfftw3-dev
+      libfftw3-double3 libfftw3-long3 libfftw3-mpi-dev libfftw3-mpi3
+      libfftw3-quad3 libfftw3-single3
+    - >
+      apt-get install -y libhdf5-10 libhdf5-dev libhdf5-openmpi-10
+      libhdf5-openmpi-dev hdf5-tools python-h5py
+    - pip install astropy healpy mpi4py
+    - pip install git+https://github.com/mrbell/gfft
+    - ci/install_libsharp.sh
+    - ci/install_pyfftw.sh
+    - python setup.py build_ext --inplace
+    - nosetests -vv
+
+release_image:
+  image: docker:latest
+  stage: release
+  before_script:
+    - pwd
+  services:
+    - docker:dind
+  script:
+    - docker login -u foxymoron -p SupernovaProgenitorsII1@3
+    - docker build -t $RELEASE_IMAGE .
+    - docker push $RELEASE_IMAGE

Dockerfile

+FROM ubuntu:latest
+
+# dependencies via apt
+RUN \
+    apt-get update && \
+    apt-get install -y build-essential python python-pip python-dev git \
+    gfortran autoconf gsl-bin libgsl-dev python-matplotlib openmpi-bin \
+    libopenmpi-dev libatlas-base-dev libfftw3-bin libfftw3-dev \
+    libfftw3-double3 libfftw3-long3 libfftw3-mpi-dev libfftw3-mpi3 \
+    libfftw3-quad3 libfftw3-single3 libhdf5-10 libhdf5-dev \
+    libhdf5-openmpi-10 libhdf5-openmpi-dev hdf5-tools python-h5py
+
+# python dependencies
+ADD ci/requirements.txt /tmp/requirements.txt
+RUN pip install -r /tmp/requirements.txt
+
+# install libsharp and pyfftw
+ADD ci/install_libsharp.sh /tmp/install_libsharp.sh
+ADD ci/install_pyfftw.sh /tmp/install_pyfftw.sh
+RUN cd /tmp && chmod +x install_libsharp.sh && ./install_libsharp.sh
+RUN cd /tmp && chmod +x install_pyfftw.sh && ./install_pyfftw.sh
+
+# copy sources and install nifty
+COPY . /tmp/NIFTy
+RUN pip install /tmp/NIFTy
+
+# Cleanup
+RUN rm -r /tmp/*

ci/install_libsharp.sh

+#!/bin/bash
+
+git clone http://git.code.sf.net/p/libsharp/code libsharp-code
+cd libsharp-code
+autoconf && ./configure --enable-pic --disable-openmp && make
+cd ..
+git clone https://github.com/mselig/libsharp-wrapper libsharp-wrapper
+cd libsharp-wrapper
+python setup.py build_ext install
+cd ..
+rm -r libsharp-code
+rm -r libsharp-wrapper

ci/install_pyfftw.sh

+#!/bin/bash
+
+git clone -b mpi https://github.com/fredRos/pyFFTW.git
+cd pyFFTW/
+CC=mpicc python setup.py build_ext install
+cd ..
+rm -r pyFFTW

ci/requirements.txt

+numpy
+scipy
+cython
+astropy
+healpy
+mpi4py
+ipython
+nose
+nose-parameterized
+git+https://gitlab.mpcdf.mpg.de/ift/mpi_dummy.git
+git+https://gitlab.mpcdf.mpg.de/ift/keepers.git
+git+https://gitlab.mpcdf.mpg.de/ift/D2O.git

ci/requirements_base.txt

+numpy
+scipy
+cython
+nose
+nose-parameterized
+git+https://gitlab.mpcdf.mpg.de/ift/mpi_dummy.git
+git+https://gitlab.mpcdf.mpg.de/ift/keepers.git
+git+https://gitlab.mpcdf.mpg.de/ift/D2O.git

nifty/__init__.py

@@ -21,9 +21,6 @@
 
 from __future__ import division
 
-import matplotlib as mpl
-mpl.use('Agg')
-
 from .version import __version__
 
 # initialize the logger instance

test/test_nifty.py

+import nifty as nt
+import numpy as np
+
+import unittest
+
+def weighted_np_transform(val, domain, codomain, axes=None):
+    if codomain.harmonic:
+        # correct for forward fft
+        val = domain.weight(val, power=1, axes=axes)
+        # Perform the transformation
+
+    Tval = np.fft.fftn(val, axes=axes)
+
+    if not codomain.harmonic:
+        # correct for inverse fft
+        Tval = codomain.weight(Tval, power=-1, axes=axes)
+
+    return Tval
+
+def test_simple_fft():
+    x = nt.RGSpace((16,))
+    x_p = nt.FFTOperator.get_default_codomain(x)
+    f = nt.Field((x, x), val=1)
+    val_p = np.ones((16,16))
+    fft = nt.FFTOperator(x)
+
+    assert np.allclose(
+        fft(f, spaces=(1,)).val,
+        weighted_np_transform(val_p, x, x_p, axes=(1,))
+    )

test/test_nifty_field.py

-# -*- coding: utf-8 -*-
-
-from numpy.testing import assert_equal, \
-    assert_almost_equal, \
-    assert_raises
-
-from nose_parameterized import parameterized
-import unittest
-import itertools
-import numpy as np
-
-from d2o import distributed_data_object
-
-from nifty import Space, \
-    point_space
-
-from nifty.rg import RGSpace
-
-from nifty.lm import *
-
-from nifty.field import Field
-
-from nifty.nifty_core import POINT_DISTRIBUTION_STRATEGIES
-from nifty.rg.rg_space import RG_DISTRIBUTION_STRATEGIES, \
-    gc as RG_GC
-from nifty.lm.nifty_lm import LM_DISTRIBUTION_STRATEGIES, \
-    GL_DISTRIBUTION_STRATEGIES, \
-    HP_DISTRIBUTION_STRATEGIES
-
-
-###############################################################################
-
-def custom_name_func(testcase_func, param_num, param):
-    return "%s_%s" % (
-        testcase_func.__name__,
-        parameterized.to_safe_name("_".join(str(x) for x in param.args)),
-    )
-
-
-###############################################################################
-###############################################################################
-
-all_point_datatypes = [np.dtype('bool'),
-                       np.dtype('int16'),
-                       np.dtype('int32'),
-                       np.dtype('int64'),
-                       np.dtype('float32'),
-                       np.dtype('float64'),
-                       np.dtype('complex64'),
-                       np.dtype('complex128')]
-
-all_lm_datatypes = [np.dtype('complex64'),
-                    np.dtype('complex128')]
-
-all_gl_datatypes = [np.dtype('float64'),
-                    np.dtype('float128')]
-
-all_hp_datatypes = [np.dtype('float64')]
-
-###############################################################################
-
-DATAMODELS = {}
-DATAMODELS['point_space'] = POINT_DISTRIBUTION_STRATEGIES
-DATAMODELS['rg_space'] = RG_DISTRIBUTION_STRATEGIES
-DATAMODELS['lm_space'] = LM_DISTRIBUTION_STRATEGIES
-DATAMODELS['gl_space'] = GL_DISTRIBUTION_STRATEGIES
-DATAMODELS['hp_space'] = HP_DISTRIBUTION_STRATEGIES
-
-###############################################################################
-
-fft_modules = []
-for name in ['gfft', 'gfft_dummy', 'pyfftw']:
-    if RG_GC.validQ('fft_module', name):
-        fft_modules += [name]
-
-###############################################################################
-
-point_like_spaces = ['point_space', 'rg_space', 'lm_space', 'hp_space',
-                     'gl_space']
-
-###############################################################################
-
-space_list = []
-
-# Add point_spaces
-for param in itertools.product([1, 10],
-                               all_point_datatypes):
-    space_list += [[point_space(num=param[0], dtype=param[1])]]
-
-# Add rg_spaces
-for param in itertools.product([(1,), (4, 6), (5, 8)],
-                               [False, True],
-                               [0, 1, 2],
-                               [None, 0.3],
-                               [False],
-                               DATAMODELS['rg_space'],
-                               fft_modules):
-    space_list += [[RGSpace(shape=param[0],
-                             zerocenter=param[1],
-                             complexity=param[2],
-                             distances=param[3],
-                             harmonic=param[4],
-                             fft_module=param[6]), param[5]]]
-
-
-def generate_space_with_size(name, num):
-    space_dict = {'space': Space(),
-                  'point_space': point_space(num),
-                  'rg_space': RGSpace((num, num)),
-                  'lm_space': LMSpace(mmax=num+1, lmax=num+1),
-                  'hp_space': HPSpace(num),
-                  'gl_space': GLSpace(nlat=num, nlon=2*num-1),
-                  }
-    return space_dict[name]
-
-
-def generate_data(space):
-    a = np.arange(space.dim).reshape(space.shape)
-    return distributed_data_object(a)
-
-
-###############################################################################
-###############################################################################
-
-class Test_field_init(unittest.TestCase):
-    @parameterized.expand(
-        itertools.product([(1,), (4, 6), (5, 8)],
-                          [False, True],
-                          [0, 1, 2],
-                          [None, 0.3],
-                          [False],
-                          fft_modules,
-                          DATAMODELS['rg_space']),
-        testcase_func_name=custom_name_func)
-    def test_successfull_init_and_attributes(self, shape, zerocenter,
-                                             complexity, distances, harmonic,
-                                             fft_module, datamodel):
-        s = RGSpace(shape=shape, zerocenter=zerocenter,
-                     complexity=complexity, distances=distances,
-                     harmonic=harmonic, fft_module=fft_module)
-        f = Field(domain=(s,), dtype=s.dtype, datamodel=datamodel)
-        assert (f.domain[0] is s)
-        assert (s.check_codomain(f.codomain[0]))
-        assert (s.shape == f.shape)
-
-
-class Test_field_init2(unittest.TestCase):
-    @parameterized.expand(
-        itertools.product(point_like_spaces, [4],
-                          DATAMODELS['rg_space']),
-        testcase_func_name=custom_name_func)
-    def test_successfull_init_and_attributes(self, name, num, datamodel):
-        s = generate_space_with_size(name, num)
-        d = generate_data(s)
-        f = Field(val=d, domain=(s,), dtype=s.dtype, datamodel=datamodel)
-        assert (f.domain[0] is s)
-        assert (s.check_codomain(f.codomain[0]))
-        assert (s.shape == f.shape)
-
-
-class Test_field_multiple_rg_init(unittest.TestCase):
-    @parameterized.expand(
-        itertools.product([(1,)],
-                          [True],
-                          [0],
-                          [None],
-                          [False],
-                          fft_modules,
-                          DATAMODELS['rg_space']),
-        testcase_func_name=custom_name_func)
-    def test_multiple_space_init(self, shape, zerocenter,
-                                 complexity, distances, harmonic,
-                                 fft_module, datamodel):
-        s1 = RGSpace(shape=shape, zerocenter=zerocenter,
-                      complexity=complexity, distances=distances,
-                      harmonic=harmonic, fft_module=fft_module)
-        s2 = RGSpace(shape=shape, zerocenter=zerocenter,
-                      complexity=complexity, distances=distances,
-                      harmonic=harmonic, fft_module=fft_module)
-        f = Field(domain=(s1, s2), dtype=s1.dtype, datamodel=datamodel)
-        assert (f.domain[0] is s1)
-        assert (f.domain[1] is s2)
-        assert (s1.check_codomain(f.codomain[0]))
-        assert (s2.check_codomain(f.codomain[1]))
-        assert (s1.shape + s2.shape == f.shape)
-
-
-class Test_field_multiple_init(unittest.TestCase):
-    @parameterized.expand(
-        itertools.product(point_like_spaces, point_like_spaces, [4]),
-        testcase_func_name=custom_name_func)
-    def test_multiple_space_init(self, space1, space2, shape):
-        s1 = generate_space_with_size(space1, shape)
-        s2 = generate_space_with_size(space2, shape)
-        f = Field(domain=(s1, s2))
-        assert (f.domain[0] is s1)
-        assert (f.domain[1] is s2)
-        assert (s1.check_codomain(f.codomain[0]))
-        assert (s2.check_codomain(f.codomain[1]))
-        assert (s1.shape + s2.shape == f.shape)
-        s3 = generate_space_with_size('hp_space', shape)
-        f = Field(domain=(s1, s2, s3))
-        assert (f.domain[0] is s1)
-        assert (f.domain[1] is s2)
-        assert (f.domain[2] is s3)
-        assert (s1.check_codomain(f.codomain[0]))
-        assert (s2.check_codomain(f.codomain[1]))
-        assert (s3.check_codomain(f.codomain[2]))
-        assert (s1.shape + s2.shape + s3.shape ==
-                f.shape)
-
-
-class Test_axis(unittest.TestCase):
-    @parameterized.expand(
-        itertools.product(point_like_spaces, [4],
-                          ['sum', 'prod', 'mean', 'var', 'std', 'median', 'all',
-                           'any', 'min', 'nanmin', 'argmin', 'max', 'nanmax',
-                           'argmax'],
-                          [None, (0,)],
-                          DATAMODELS['rg_space']),
-        testcase_func_name=custom_name_func)
-    def test_unary_operations(self, name, num, op, axis, datamodel):
-        s = generate_space_with_size(name, num)
-        d = generate_data(s)
-        a = d.get_full_data()
-        f = Field(val=d, domain=(s,), dtype=s.dtype, datamodel=datamodel)
-        if op in ['argmin', 'argmax']:
-            assert_almost_equal(getattr(f, op)(),
-                                getattr(np, op)(a), decimal=4)
-        else:
-            assert_almost_equal(getattr(f, op)(axis=axis),
-                                getattr(np, op)(a, axis=axis), decimal=4)
-
-
-binary_operations = [('add','__add__'),('radd','__radd__'),('iadd','__iadd__'),
-                     ('sub','__sub__'),('rsub','__rsub__'),('isub','__isub__'),
-                     ('mul','__mul__'),('rmul','__rmul__'),('imul','__imul__'),
-                     ('div','__div__'),('rdiv','__rdiv__'),('idiv','__idiv__'),
-                     ('pow','__pow__'),('rpow','__rpow__'),('ipow','__ipow__'),
-                     ('ne','__ne__'),('lt','__lt__'),('eq','__eq__'),
-                     ('ge','__ge__'),('gt','__gt__')]
-
-class Test_binary_operation(unittest.TestCase):
-    @parameterized.expand(
-    itertools.product([point_like_spaces[0]], [4], binary_operations,
-                      DATAMODELS['rg_space']),
-    testcase_func_name=custom_name_func)
-    def test_binary_operations(self, name, num, op, datamodel):
-        s = generate_space_with_size(name, num)
-        d = generate_data(s)
-        a = d.get_full_data()
-        f = Field(val=d, domain=(s,), dtype=s.dtype, datamodel=datamodel)
-        d2 = d[::-1]
-        a2 = np.copy(a[::-1])
-        if op[0] in ['iadd','isub','imul','idiv']:
-            getattr(a, op[1])(a2)
-            f.binary_operation(d, d2, op[0])
-            assert_almost_equal(a,d,4)
-        else:
-            assert_almost_equal(getattr(a, op[1])(a2),f.binary_operation(d, d2,
-                                                                      op[0]), 4)
\ No newline at end of file

test/test_nifty_operators.py

-# -*- coding: utf-8 -*-
-
-from numpy.testing import assert_equal,\
-    assert_almost_equal,\
-    assert_raises
-
-from nose_parameterized import parameterized
-import unittest
-import itertools
-import numpy as np
-
-from nifty import Space,\
-                  RGSpace,\
-                  Field,\
-                  distributed_data_object
-
-from nifty.operators import operator,\
-                            diagonal_operator,\
-                            power_operator,\
-                            projection_operator,\
-                            vecvec_operator,\
-                            response_operator,\
-                            invertible_operator,\
-                            propagator_operator,\
-                            identity_operator
-
-from nifty.nifty_core import POINT_DISTRIBUTION_STRATEGIES
-
-from nifty.rg.rg_space import RG_DISTRIBUTION_STRATEGIES,\
-                              gc as RG_GC
-from nifty.lm.nifty_lm import LM_DISTRIBUTION_STRATEGIES,\
-                              GL_DISTRIBUTION_STRATEGIES,\
-                              HP_DISTRIBUTION_STRATEGIES
-
-available = []
-try:
-    from nifty import LMSpace
-except ImportError:
-    pass
-else:
-    available += ['lm_space']
-try:
-    from nifty import  GLSpace
-except ImportError:
-    pass
-else:
-    available += ['gl_space']
-try:
-    from nifty import  HPSpace
-except ImportError:
-    pass
-else:
-    available += ['hp_space']
-
-
-
-###############################################################################
-
-def custom_name_func(testcase_func, param_num, param):
-    return "%s_%s" % (
-        testcase_func.__name__,
-        parameterized.to_safe_name("_".join(str(x) for x in param.args)),
-    )
-
-###############################################################################
-###############################################################################
-
-all_point_datatypes = [np.dtype('bool'),
-                       np.dtype('int16'),
-                       np.dtype('int32'),
-                       np.dtype('int64'),
-                       np.dtype('float32'),
-                       np.dtype('float64'),
-                       np.dtype('complex64'),
-                       np.dtype('complex128')]
-
-all_lm_datatypes = [np.dtype('complex64'),
-                    np.dtype('complex128')]
-
-all_gl_datatypes = [np.dtype('float64'),
-                    np.dtype('float128')]
-
-all_hp_datatypes = [np.dtype('float64')]
-
-###############################################################################
-
-DATAMODELS = {}
-DATAMODELS['point_space'] = ['np'] + POINT_DISTRIBUTION_STRATEGIES
-DATAMODELS['rg_space'] = ['np'] + RG_DISTRIBUTION_STRATEGIES
-DATAMODELS['lm_space'] = ['np'] + LM_DISTRIBUTION_STRATEGIES
-DATAMODELS['gl_space'] = ['np'] + GL_DISTRIBUTION_STRATEGIES
-DATAMODELS['hp_space'] = ['np'] + HP_DISTRIBUTION_STRATEGIES
-
-###############################################################################
-
-all_spaces = ['space', 'point_space', 'rg_space']
-if 'lm_space' in available:
-    all_spaces += ['lm_space']
-if 'gl_space' in available:
-    all_spaces += ['gl_space']
-if 'hp_space' in available:
-    all_spaces += ['hp_space']
-
-
-point_like_spaces = ['point_space', 'rg_space']
-if 'lm_space' in available:
-    point_like_spaces += ['lm_space']
-if 'gl_space' in available:
-    point_like_spaces += ['gl_space']
-if 'hp_space' in available:
-    point_like_spaces += ['hp_space']
-
-###############################################################################
-
-np_spaces = point_like_spaces
-d2o_spaces = []
-if POINT_DISTRIBUTION_STRATEGIES != []:
-    d2o_spaces += ['point_space']
-if RG_DISTRIBUTION_STRATEGIES != []:
-    d2o_spaces += ['rg_space']
-if LM_DISTRIBUTION_STRATEGIES != []:
-    d2o_spaces += ['lm_space']
-if GL_DISTRIBUTION_STRATEGIES != []:
-    d2o_spaces += ['gl_space']
-if HP_DISTRIBUTION_STRATEGIES != []:
-    d2o_spaces += ['hp_space']
-###############################################################################
-
-all_operators = ['operator', 'diagonal_operator', 'power_operator',
-                 'projection_operator', 'vecvec_operator', 'response_operator',
-                 'invertible_operator', 'propagator_operator',
-                 'identity_operator']
-
-
-###############################################################################
-
-def generate_operator(name):
-    x = RGSpace((8, 8))
-    k = x.get_codomain()
-    operator_dict = {'operator': operator(domain=x),
-                     'diagonal_operator': diagonal_operator(domain=x),
-                     'identity_operator': identity_operator(domain=x),
-                     'power_operator': power_operator(domain=k),
-                     'projection_operator': projection_operator(domain=x),
-                     'vecvec_operator': vecvec_operator(domain=x),
-                     'response_operator': response_operator(domain=x),
-                     'invertible_operator': invertible_operator(domain=x)}
-    return operator_dict[name]
-
-
-def generate_data(space):