WIP: initial ci config

.gitlab-ci.yml

+image: ubuntu:latest
+
+stages:
+  - test
+  - release
+
+variables:
+  DOCKER_DRIVER: overlay
+  RELEASE_IMAGE: iftmpa/nifty:ci_test
+
+before_script:
+  - apt-get update
+  - >
+    apt-get install -y build-essential python python-pip python-dev git 
+    gfortran autoconf gsl-bin libgsl-dev
+  - pip install -r ci/requirements_base.txt
+  - chmod +x ci/*.sh
+
+test_min:
+  stage: test
+  script:
+    - python setup.py build_ext --inplace
+    - nosetests -vv
+
+test_mpi:
+  stage: test
+  script:
+    - apt-get install -y openmpi-bin libopenmpi-dev
+    - pip install astropy healpy mpi4py
+    - pip install git+https://github.com/mrbell/gfft
+    - ci/install_libsharp.sh
+    - python setup.py build_ext --inplace
+    - nosetests -vv
+
+test_mpi_fftw:
+  stage: test
+  script:
+    - apt-get install -y openmpi-bin libopenmpi-dev
+    - >
+      apt-get install -y libatlas-base-dev libfftw3-bin libfftw3-dev
+      libfftw3-double3 libfftw3-long3 libfftw3-mpi-dev libfftw3-mpi3
+      libfftw3-quad3 libfftw3-single3
+    - pip install astropy healpy mpi4py
+    - pip install git+https://github.com/mrbell/gfft
+    - ci/install_libsharp.sh
+    - ci/install_pyfftw.sh
+    - python setup.py build_ext --inplace
+    - nosetests -vv
+
+test_mpi_fftw_hdf5:
+  stage: test
+  script:
+    - apt-get install -y openmpi-bin libopenmpi-dev
+    - >
+      apt-get install -y libatlas-base-dev libfftw3-bin libfftw3-dev
+      libfftw3-double3 libfftw3-long3 libfftw3-mpi-dev libfftw3-mpi3
+      libfftw3-quad3 libfftw3-single3
+    - >
+      apt-get install -y libhdf5-10 libhdf5-dev libhdf5-openmpi-10
+      libhdf5-openmpi-dev hdf5-tools python-h5py
+    - pip install astropy healpy mpi4py
+    - pip install git+https://github.com/mrbell/gfft
+    - ci/install_libsharp.sh
+    - ci/install_pyfftw.sh
+    - python setup.py build_ext --inplace
+    - nosetests -vv
+
+release_image:
+  image: docker:latest
+  stage: release
+  before_script:
+    - pwd
+  services:
+    - docker:dind
+  script:
+    - docker login -u foxymoron -p SupernovaProgenitorsII1@3
+    - docker build -t $RELEASE_IMAGE .
+    - docker push $RELEASE_IMAGE

Dockerfile

+FROM ubuntu:latest
+
+# dependencies via apt
+RUN \
+    apt-get update && \
+    apt-get install -y build-essential python python-pip python-dev git \
+    gfortran autoconf gsl-bin libgsl-dev python-matplotlib openmpi-bin \
+    libopenmpi-dev libatlas-base-dev libfftw3-bin libfftw3-dev \
+    libfftw3-double3 libfftw3-long3 libfftw3-mpi-dev libfftw3-mpi3 \
+    libfftw3-quad3 libfftw3-single3 libhdf5-10 libhdf5-dev \
+    libhdf5-openmpi-10 libhdf5-openmpi-dev hdf5-tools python-h5py
+
+# python dependencies
+ADD ci/requirements.txt /tmp/requirements.txt
+RUN pip install -r /tmp/requirements.txt
+
+# install libsharp and pyfftw
+ADD ci/install_libsharp.sh /tmp/install_libsharp.sh
+ADD ci/install_pyfftw.sh /tmp/install_pyfftw.sh
+RUN cd /tmp && chmod +x install_libsharp.sh && ./install_libsharp.sh
+RUN cd /tmp && chmod +x install_pyfftw.sh && ./install_pyfftw.sh
+
+# copy sources and install nifty
+COPY . /tmp/NIFTy
+RUN pip install /tmp/NIFTy
+
+# Cleanup
+RUN rm -r /tmp/*

ci/install_libsharp.sh

+#!/bin/bash
+
+git clone http://git.code.sf.net/p/libsharp/code libsharp-code
+cd libsharp-code
+autoconf && ./configure --enable-pic --disable-openmp && make
+cd ..
+git clone https://github.com/mselig/libsharp-wrapper libsharp-wrapper
+cd libsharp-wrapper
+python setup.py build_ext install
+cd ..
+rm -r libsharp-code
+rm -r libsharp-wrapper

ci/install_pyfftw.sh

+#!/bin/bash
+
+git clone -b mpi https://github.com/fredRos/pyFFTW.git
+cd pyFFTW/
+CC=mpicc python setup.py build_ext install
+cd ..
+rm -r pyFFTW

ci/requirements.txt

+numpy
+scipy
+cython
+astropy
+healpy
+mpi4py
+ipython
+nose
+nose-parameterized
+git+https://gitlab.mpcdf.mpg.de/ift/mpi_dummy.git
+git+https://gitlab.mpcdf.mpg.de/ift/keepers.git
+git+https://gitlab.mpcdf.mpg.de/ift/D2O.git

ci/requirements_base.txt

+numpy
+scipy
+cython
+nose
+nose-parameterized
+git+https://gitlab.mpcdf.mpg.de/ift/mpi_dummy.git
+git+https://gitlab.mpcdf.mpg.de/ift/keepers.git
+git+https://gitlab.mpcdf.mpg.de/ift/D2O.git

nifty/__init__.py

@@ -21,9 +21,6 @@
 
 from __future__ import division
 
-import matplotlib as mpl
-mpl.use('Agg')
-
 from .version import __version__
 
 # initialize the logger instance

test/test_nifty.py

+import nifty as nt
+import numpy as np
+
+import unittest
+
+def weighted_np_transform(val, domain, codomain, axes=None):
+    if codomain.harmonic:
+        # correct for forward fft
+        val = domain.weight(val, power=1, axes=axes)
+        # Perform the transformation
+
+    Tval = np.fft.fftn(val, axes=axes)
+
+    if not codomain.harmonic:
+        # correct for inverse fft
+        Tval = codomain.weight(Tval, power=-1, axes=axes)
+
+    return Tval
+
+def test_simple_fft():
+    x = nt.RGSpace((16,))
+    x_p = nt.FFTOperator.get_default_codomain(x)
+    f = nt.Field((x, x), val=1)
+    val_p = np.ones((16,16))
+    fft = nt.FFTOperator(x)
+
+    assert np.allclose(
+        fft(f, spaces=(1,)).val,
+        weighted_np_transform(val_p, x, x_p, axes=(1,))
+    )

test/test_nifty_operators.py

-# -*- coding: utf-8 -*-
-
-from numpy.testing import assert_equal,\
-    assert_almost_equal,\
-    assert_raises
-
-from nose_parameterized import parameterized
-import unittest
-import itertools
-import numpy as np
-
-from nifty import Space,\
-                  RGSpace,\
-                  Field,\
-                  distributed_data_object
-
-from nifty.operators import operator,\
-                            diagonal_operator,\
-                            power_operator,\
-                            projection_operator,\
-                            vecvec_operator,\
-                            response_operator,\
-                            invertible_operator,\
-                            propagator_operator,\
-                            identity_operator
-
-from nifty.nifty_core import POINT_DISTRIBUTION_STRATEGIES
-
-from nifty.rg.rg_space import RG_DISTRIBUTION_STRATEGIES,\
-                              gc as RG_GC
-from nifty.lm.nifty_lm import LM_DISTRIBUTION_STRATEGIES,\
-                              GL_DISTRIBUTION_STRATEGIES,\
-                              HP_DISTRIBUTION_STRATEGIES
-
-available = []
-try:
-    from nifty import LMSpace
-except ImportError:
-    pass
-else:
-    available += ['lm_space']
-try:
-    from nifty import  GLSpace
-except ImportError:
-    pass
-else:
-    available += ['gl_space']
-try:
-    from nifty import  HPSpace
-except ImportError:
-    pass
-else:
-    available += ['hp_space']
-
-
-
-###############################################################################
-
-def custom_name_func(testcase_func, param_num, param):
-    return "%s_%s" % (
-        testcase_func.__name__,
-        parameterized.to_safe_name("_".join(str(x) for x in param.args)),
-    )
-
-###############################################################################
-###############################################################################
-
-all_point_datatypes = [np.dtype('bool'),
-                       np.dtype('int16'),
-                       np.dtype('int32'),
-                       np.dtype('int64'),
-                       np.dtype('float32'),
-                       np.dtype('float64'),
-                       np.dtype('complex64'),
-                       np.dtype('complex128')]
-
-all_lm_datatypes = [np.dtype('complex64'),
-                    np.dtype('complex128')]
-
-all_gl_datatypes = [np.dtype('float64'),
-                    np.dtype('float128')]
-
-all_hp_datatypes = [np.dtype('float64')]
-
-###############################################################################
-
-DATAMODELS = {}
-DATAMODELS['point_space'] = ['np'] + POINT_DISTRIBUTION_STRATEGIES
-DATAMODELS['rg_space'] = ['np'] + RG_DISTRIBUTION_STRATEGIES
-DATAMODELS['lm_space'] = ['np'] + LM_DISTRIBUTION_STRATEGIES
-DATAMODELS['gl_space'] = ['np'] + GL_DISTRIBUTION_STRATEGIES
-DATAMODELS['hp_space'] = ['np'] + HP_DISTRIBUTION_STRATEGIES
-
-###############################################################################
-
-all_spaces = ['space', 'point_space', 'rg_space']
-if 'lm_space' in available:
-    all_spaces += ['lm_space']
-if 'gl_space' in available:
-    all_spaces += ['gl_space']
-if 'hp_space' in available:
-    all_spaces += ['hp_space']
-
-
-point_like_spaces = ['point_space', 'rg_space']
-if 'lm_space' in available:
-    point_like_spaces += ['lm_space']
-if 'gl_space' in available:
-    point_like_spaces += ['gl_space']
-if 'hp_space' in available:
-    point_like_spaces += ['hp_space']
-
-###############################################################################
-
-np_spaces = point_like_spaces
-d2o_spaces = []
-if POINT_DISTRIBUTION_STRATEGIES != []:
-    d2o_spaces += ['point_space']
-if RG_DISTRIBUTION_STRATEGIES != []:
-    d2o_spaces += ['rg_space']
-if LM_DISTRIBUTION_STRATEGIES != []:
-    d2o_spaces += ['lm_space']
-if GL_DISTRIBUTION_STRATEGIES != []:
-    d2o_spaces += ['gl_space']
-if HP_DISTRIBUTION_STRATEGIES != []:
-    d2o_spaces += ['hp_space']
-###############################################################################
-
-all_operators = ['operator', 'diagonal_operator', 'power_operator',
-                 'projection_operator', 'vecvec_operator', 'response_operator',
-                 'invertible_operator', 'propagator_operator',
-                 'identity_operator']
-
-
-###############################################################################
-
-def generate_operator(name):
-    x = RGSpace((8, 8))
-    k = x.get_codomain()
-    operator_dict = {'operator': operator(domain=x),
-                     'diagonal_operator': diagonal_operator(domain=x),
-                     'identity_operator': identity_operator(domain=x),
-                     'power_operator': power_operator(domain=k),
-                     'projection_operator': projection_operator(domain=x),
-                     'vecvec_operator': vecvec_operator(domain=x),
-                     'response_operator': response_operator(domain=x),
-                     'invertible_operator': invertible_operator(domain=x)}
-    return operator_dict[name]
-
-
-def generate_data(space):
-    a = np.arange(space.dim).reshape(space.shape)
-    data = space.cast(a)
-    return data
-
-
-def check_equality(space, data1, data2):
-    return space.unary_operation(space.binary_operation(data1, data2, 'eq'),
-                                 'all')
-
-
-def check_almost_equality(space, data1, data2, integers=7):
-    return space.unary_operation(
-        space.binary_operation(
-            space.unary_operation(
-                space.binary_operation(data1, data2, 'sub'),
-                'abs'),
-            10.**(-1. * integers), 'le'),
-        'all')
-
-
-###############################################################################
-###############################################################################
-
-class Test_Operator_Base_Class(unittest.TestCase):
-
-    @parameterized.expand(all_operators,
-                          testcase_func_name=custom_name_func)
-    def test_successfull_init_and_methods(self, name):
-        op = generate_operator(name)
-        assert(callable(op.set_val))
-        assert(callable(op.get_val))
-        assert(callable(op._multiply))
-        assert(callable(op._adjoint_multiply))
-        assert(callable(op._inverse_multiply))
-        assert(callable(op._adjoint_inverse_multiply))
-        assert(callable(op._inverse_adjoint_multiply))
-        assert(callable(op._briefing))
-        assert(callable(op._debriefing))
-        assert(callable(op.times))
-        assert(callable(op.__call__))
-        assert(callable(op.adjoint_times))
-        assert(callable(op.inverse_times))
-        assert(callable(op.adjoint_inverse_times))
-        assert(callable(op.inverse_adjoint_times))
-        assert(callable(op.tr))
-        assert(callable(op.inverse_tr))
-        assert(callable(op.diag))
-        assert(callable(op.det))
-        assert(callable(op.inverse_det))
-        assert(callable(op.log_det))
-        assert(callable(op.tr_log))
-        assert(callable(op.hat))
-        assert(callable(op.inverse_hat))
-        assert(callable(op.hathat))
-        assert(callable(op.inverse_hathat))
-        assert(callable(op.__repr__))
-
-    @parameterized.expand(all_operators,
-                          testcase_func_name=custom_name_func)
-    def test_successfull_init_and_attributes(self, name):
-        op = generate_operator(name)
-        assert(isinstance(op.sym, bool))
-        assert(isinstance(op.uni, bool))
-        assert(isinstance(op.bare, bool))
-        assert(isinstance(op.imp, bool))
-
-        assert(isinstance(op.domain, space))
-        assert(isinstance(op.codomain, space))
-        assert(isinstance(op.target, space))
-        assert(isinstance(op.cotarget, space))
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-

test/test_nifty_transforms.py

-import itertools
-import unittest
-
-import d2o
-import numpy as np
-from nifty.rg.rg_space import gc as RG_GC
-from nose_parameterized import parameterized
-from numpy.testing import assert_raises
-
-from nifty import RGSpace, LMSpace, HPSpace, GLSpace
-from nifty import transformator
-from nifty.operators.fft_operator.transformations.rgrgtransformation import RGRGTransformation
-
-
-###############################################################################
-
-def custom_name_func(testcase_func, param_num, param):
-    return "%s_%s" % (
-        testcase_func.__name__,
-        parameterized.to_safe_name("_".join(str(x) for x in param.args)),
-    )
-
-def weighted_np_transform(val, domain, codomain, axes=None):
-    if codomain.harmonic:
-        # correct for forward fft
-        val = domain.calc_weight(val, power=1)
-
-    # Perform the transformation
-    Tval = np.fft.fftn(val, axes=axes)
-
-    if not codomain.harmonic:
-        # correct for inverse fft
-        Tval = codomain.calc_weight(Tval, power=-1)
-
-    return Tval
-
-###############################################################################
-
-rg_rg_fft_modules = []
-for name in ['gfft', 'gfft_dummy', 'pyfftw']:
-    if RG_GC.validQ('fft_module', name):
-        rg_rg_fft_modules += [name]
-
-rg_rg_test_shapes = [(128, 128), (179, 179), (512, 512)]
-
-rg_rg_test_spaces = [(GLSpace(8),), (HPSpace(8),), (LMSpace(8),)]
-gl_hp_lm_test_spaces = [(GLSpace(8),), (HPSpace(8),), (RGSpace(8),)]
-lm_gl_hp_test_spaces = [(LMSpace(8),), (RGSpace(8),)]
-
-###############################################################################
-
-class TestRGRGTransformation(unittest.TestCase):
-    @parameterized.expand(
-        rg_rg_test_spaces,
-        testcase_func_name=custom_name_func
-    )
-    def test_check_codomain_mismatch(self, space):
-        x = RGSpace((8, 8))
-        with assert_raises(TypeError):
-            transformator.create(x, space)
-
-    @parameterized.expand(
-        itertools.product([0, 1, 2], [None, (1, 1), (10, 10)], [False, True]),
-        testcase_func_name=custom_name_func
-    )
-    def test_check_codomain_rgspecific(self, complexity, distances, harmonic):
-        x = RGSpace((8, 8), complexity=complexity,
-                    distances=distances, harmonic=harmonic)
-        assert (RGRGTransformation.check_codomain(x, x.get_codomain()))
-        assert (RGRGTransformation.check_codomain(x, x.get_codomain()))
-
-    @parameterized.expand(rg_rg_fft_modules, testcase_func_name=custom_name_func)
-    def test_shapemismatch(self, module):
-        x = RGSpace((8, 8))
-        b = d2o.distributed_data_object(np.ones((8, 8)))
-        with assert_raises(ValueError):
-            transformator.create(
-                x, module=module
-            ).transform(b, axes=(0, 1, 2))
-
-    @parameterized.expand(
-        itertools.product(rg_rg_fft_modules, rg_rg_test_shapes),
-        testcase_func_name=custom_name_func
-    )
-    def test_local_ndarray(self, module, shape):
-        x = RGSpace(shape)
-        a = np.ones(shape)
-        assert np.allclose(
-            transformator.create(
-                x, module=module
-            ).transform(a),
-            weighted_np_transform(a, x, x.get_codomain())
-        )
-
-    @parameterized.expand(
-        itertools.product(rg_rg_fft_modules, rg_rg_test_shapes),
-        testcase_func_name=custom_name_func
-    )
-    def test_local_notzero(self, module, shape):
-        x = RGSpace(shape[0])  # all tests along axis 1
-        a = np.ones(shape)
-        b = d2o.distributed_data_object(a)
-        assert np.allclose(
-            transformator.create(
-                x, module=module
-            ).transform(b, axes=(1,)),
-            weighted_np_transform(a, x, x.get_codomain(), axes=(1,))
-        )
-
-    @parameterized.expand(
-        itertools.product(rg_rg_fft_modules, rg_rg_test_shapes),
-        testcase_func_name=custom_name_func
-    )
-    def test_not(self, module, shape):
-        x = RGSpace(shape)
-        a = np.ones(shape)
-        b = d2o.distributed_data_object(a, distribution_strategy='not')
-        assert np.allclose(
-            transformator.create(
-                x, module=module
-            ).transform(b),
-            weighted_np_transform(a, x, x.get_codomain())
-        )
-
-    @parameterized.expand(
-        itertools.product(rg_rg_test_shapes),
-        testcase_func_name=custom_name_func
-    )
-    def test_mpi_axesnone(self, shape):
-        x = RGSpace(shape)
-        a = np.ones(shape)
-        b = d2o.distributed_data_object(a)
-        assert np.allclose(
-            transformator.create(
-                x, module='pyfftw'
-            ).transform(b),
-            weighted_np_transform(a, x, x.get_codomain())
-        )
-
-    @parameterized.expand(
-        itertools.product(rg_rg_test_shapes),
-        testcase_func_name=custom_name_func
-    )
-    def test_mpi_axesnone_equal(self, shape):
-        x = RGSpace(shape)
-        a = np.ones(shape)
-        b = d2o.distributed_data_object(a, distribution_strategy='equal')
-        assert np.allclose(
-            transformator.create(
-                x, module='pyfftw'
-            ).transform(b),
-            weighted_np_transform(a, x, x.get_codomain())
-        )
-
-class TestGLLMTransformation(unittest.TestCase):
-    @parameterized.expand(
-        gl_hp_lm_test_spaces,
-        testcase_func_name=custom_name_func
-    )
-    def test_check_codomain_mismatch(self, space):
-        x = GLSpace(8)
-        with assert_raises(TypeError):
-            transformator.create(x, space)
-
-class TestHPLMTransformation(unittest.TestCase):
-
-    @parameterized.expand(
-        gl_hp_lm_test_spaces,