Commit b9a1dd5e authored by Jait Dixit's avatar Jait Dixit

WIP: initial ci config

parent c286d744
Pipeline #9315 failed with stages
in 31 minutes and 11 seconds
image: ubuntu:latest
stages:
- test
- release
variables:
DOCKER_DRIVER: overlay
RELEASE_IMAGE: iftmpa/nifty:ci_test
before_script:
- apt-get update
- >
apt-get install -y build-essential python python-pip python-dev git
gfortran autoconf gsl-bin libgsl-dev
- pip install -r ci/requirements_base.txt
- chmod +x ci/*.sh
test_min:
stage: test
script:
- python setup.py build_ext --inplace
- nosetests -vv
test_mpi:
stage: test
script:
- apt-get install -y openmpi-bin libopenmpi-dev
- pip install astropy healpy mpi4py
- pip install git+https://github.com/mrbell/gfft
- ci/install_libsharp.sh
- python setup.py build_ext --inplace
- nosetests -vv
test_mpi_fftw:
stage: test
script:
- apt-get install -y openmpi-bin libopenmpi-dev
- >
apt-get install -y libatlas-base-dev libfftw3-bin libfftw3-dev
libfftw3-double3 libfftw3-long3 libfftw3-mpi-dev libfftw3-mpi3
libfftw3-quad3 libfftw3-single3
- pip install astropy healpy mpi4py
- pip install git+https://github.com/mrbell/gfft
- ci/install_libsharp.sh
- ci/install_pyfftw.sh
- python setup.py build_ext --inplace
- nosetests -vv
test_mpi_fftw_hdf5:
stage: test
script:
- apt-get install -y openmpi-bin libopenmpi-dev
- >
apt-get install -y libatlas-base-dev libfftw3-bin libfftw3-dev
libfftw3-double3 libfftw3-long3 libfftw3-mpi-dev libfftw3-mpi3
libfftw3-quad3 libfftw3-single3
- >
apt-get install -y libhdf5-10 libhdf5-dev libhdf5-openmpi-10
libhdf5-openmpi-dev hdf5-tools python-h5py
- pip install astropy healpy mpi4py
- pip install git+https://github.com/mrbell/gfft
- ci/install_libsharp.sh
- ci/install_pyfftw.sh
- python setup.py build_ext --inplace
- nosetests -vv
release_image:
image: docker:latest
stage: release
before_script:
- pwd
services:
- docker:dind
script:
- docker login -u foxymoron -p SupernovaProgenitorsII1@3
- docker build -t $RELEASE_IMAGE .
- docker push $RELEASE_IMAGE
FROM ubuntu:latest
# dependencies via apt
RUN \
apt-get update && \
apt-get install -y build-essential python python-pip python-dev git \
gfortran autoconf gsl-bin libgsl-dev python-matplotlib openmpi-bin \
libopenmpi-dev libatlas-base-dev libfftw3-bin libfftw3-dev \
libfftw3-double3 libfftw3-long3 libfftw3-mpi-dev libfftw3-mpi3 \
libfftw3-quad3 libfftw3-single3 libhdf5-10 libhdf5-dev \
libhdf5-openmpi-10 libhdf5-openmpi-dev hdf5-tools python-h5py
# python dependencies
ADD ci/requirements.txt /tmp/requirements.txt
RUN pip install -r /tmp/requirements.txt
# install libsharp and pyfftw
ADD ci/install_libsharp.sh /tmp/install_libsharp.sh
ADD ci/install_pyfftw.sh /tmp/install_pyfftw.sh
RUN cd /tmp && chmod +x install_libsharp.sh && ./install_libsharp.sh
RUN cd /tmp && chmod +x install_pyfftw.sh && ./install_pyfftw.sh
# copy sources and install nifty
COPY . /tmp/NIFTy
RUN pip install /tmp/NIFTy
# Cleanup
RUN rm -r /tmp/*
#!/bin/bash
git clone http://git.code.sf.net/p/libsharp/code libsharp-code
cd libsharp-code
autoconf && ./configure --enable-pic --disable-openmp && make
cd ..
git clone https://github.com/mselig/libsharp-wrapper libsharp-wrapper
cd libsharp-wrapper
python setup.py build_ext install
cd ..
rm -r libsharp-code
rm -r libsharp-wrapper
#!/bin/bash
git clone -b mpi https://github.com/fredRos/pyFFTW.git
cd pyFFTW/
CC=mpicc python setup.py build_ext install
cd ..
rm -r pyFFTW
numpy
scipy
cython
astropy
healpy
mpi4py
ipython
nose
nose-parameterized
git+https://gitlab.mpcdf.mpg.de/ift/mpi_dummy.git
git+https://gitlab.mpcdf.mpg.de/ift/keepers.git
git+https://gitlab.mpcdf.mpg.de/ift/D2O.git
numpy
scipy
cython
nose
nose-parameterized
git+https://gitlab.mpcdf.mpg.de/ift/mpi_dummy.git
git+https://gitlab.mpcdf.mpg.de/ift/keepers.git
git+https://gitlab.mpcdf.mpg.de/ift/D2O.git
......@@ -21,9 +21,6 @@
from __future__ import division
import matplotlib as mpl
mpl.use('Agg')
from .version import __version__
# initialize the logger instance
......
import nifty as nt
import numpy as np
import unittest
def weighted_np_transform(val, domain, codomain, axes=None):
if codomain.harmonic:
# correct for forward fft
val = domain.weight(val, power=1, axes=axes)
# Perform the transformation
Tval = np.fft.fftn(val, axes=axes)
if not codomain.harmonic:
# correct for inverse fft
Tval = codomain.weight(Tval, power=-1, axes=axes)
return Tval
def test_simple_fft():
x = nt.RGSpace((16,))
x_p = nt.FFTOperator.get_default_codomain(x)
f = nt.Field((x, x), val=1)
val_p = np.ones((16,16))
fft = nt.FFTOperator(x)
assert np.allclose(
fft(f, spaces=(1,)).val,
weighted_np_transform(val_p, x, x_p, axes=(1,))
)
This diff is collapsed.
# -*- coding: utf-8 -*-
from numpy.testing import assert_equal,\
assert_almost_equal,\
assert_raises
from nose_parameterized import parameterized
import unittest
import itertools
import numpy as np
from nifty import Space,\
RGSpace,\
Field,\
distributed_data_object
from nifty.operators import operator,\
diagonal_operator,\
power_operator,\
projection_operator,\
vecvec_operator,\
response_operator,\
invertible_operator,\
propagator_operator,\
identity_operator
from nifty.nifty_core import POINT_DISTRIBUTION_STRATEGIES
from nifty.rg.rg_space import RG_DISTRIBUTION_STRATEGIES,\
gc as RG_GC
from nifty.lm.nifty_lm import LM_DISTRIBUTION_STRATEGIES,\
GL_DISTRIBUTION_STRATEGIES,\
HP_DISTRIBUTION_STRATEGIES
available = []
try:
from nifty import LMSpace
except ImportError:
pass
else:
available += ['lm_space']
try:
from nifty import GLSpace
except ImportError:
pass
else:
available += ['gl_space']
try:
from nifty import HPSpace
except ImportError:
pass
else:
available += ['hp_space']
###############################################################################
def custom_name_func(testcase_func, param_num, param):
return "%s_%s" % (
testcase_func.__name__,
parameterized.to_safe_name("_".join(str(x) for x in param.args)),
)
###############################################################################
###############################################################################
all_point_datatypes = [np.dtype('bool'),
np.dtype('int16'),
np.dtype('int32'),
np.dtype('int64'),
np.dtype('float32'),
np.dtype('float64'),
np.dtype('complex64'),
np.dtype('complex128')]
all_lm_datatypes = [np.dtype('complex64'),
np.dtype('complex128')]
all_gl_datatypes = [np.dtype('float64'),
np.dtype('float128')]
all_hp_datatypes = [np.dtype('float64')]
###############################################################################
DATAMODELS = {}
DATAMODELS['point_space'] = ['np'] + POINT_DISTRIBUTION_STRATEGIES
DATAMODELS['rg_space'] = ['np'] + RG_DISTRIBUTION_STRATEGIES
DATAMODELS['lm_space'] = ['np'] + LM_DISTRIBUTION_STRATEGIES
DATAMODELS['gl_space'] = ['np'] + GL_DISTRIBUTION_STRATEGIES
DATAMODELS['hp_space'] = ['np'] + HP_DISTRIBUTION_STRATEGIES
###############################################################################
all_spaces = ['space', 'point_space', 'rg_space']
if 'lm_space' in available:
all_spaces += ['lm_space']
if 'gl_space' in available:
all_spaces += ['gl_space']
if 'hp_space' in available:
all_spaces += ['hp_space']
point_like_spaces = ['point_space', 'rg_space']
if 'lm_space' in available:
point_like_spaces += ['lm_space']
if 'gl_space' in available:
point_like_spaces += ['gl_space']
if 'hp_space' in available:
point_like_spaces += ['hp_space']
###############################################################################
np_spaces = point_like_spaces
d2o_spaces = []
if POINT_DISTRIBUTION_STRATEGIES != []:
d2o_spaces += ['point_space']
if RG_DISTRIBUTION_STRATEGIES != []:
d2o_spaces += ['rg_space']
if LM_DISTRIBUTION_STRATEGIES != []:
d2o_spaces += ['lm_space']
if GL_DISTRIBUTION_STRATEGIES != []:
d2o_spaces += ['gl_space']
if HP_DISTRIBUTION_STRATEGIES != []:
d2o_spaces += ['hp_space']
###############################################################################
all_operators = ['operator', 'diagonal_operator', 'power_operator',
'projection_operator', 'vecvec_operator', 'response_operator',
'invertible_operator', 'propagator_operator',
'identity_operator']
###############################################################################
def generate_operator(name):
x = RGSpace((8, 8))
k = x.get_codomain()
operator_dict = {'operator': operator(domain=x),
'diagonal_operator': diagonal_operator(domain=x),
'identity_operator': identity_operator(domain=x),
'power_operator': power_operator(domain=k),
'projection_operator': projection_operator(domain=x),
'vecvec_operator': vecvec_operator(domain=x),
'response_operator': response_operator(domain=x),
'invertible_operator': invertible_operator(domain=x)}
return operator_dict[name]
def generate_data(space):
a = np.arange(space.dim).reshape(space.shape)
data = space.cast(a)
return data
def check_equality(space, data1, data2):
return space.unary_operation(space.binary_operation(data1, data2, 'eq'),
'all')
def check_almost_equality(space, data1, data2, integers=7):
return space.unary_operation(
space.binary_operation(
space.unary_operation(
space.binary_operation(data1, data2, 'sub'),
'abs'),
10.**(-1. * integers), 'le'),
'all')
###############################################################################
###############################################################################
class Test_Operator_Base_Class(unittest.TestCase):
@parameterized.expand(all_operators,
testcase_func_name=custom_name_func)
def test_successfull_init_and_methods(self, name):
op = generate_operator(name)
assert(callable(op.set_val))
assert(callable(op.get_val))
assert(callable(op._multiply))
assert(callable(op._adjoint_multiply))
assert(callable(op._inverse_multiply))
assert(callable(op._adjoint_inverse_multiply))
assert(callable(op._inverse_adjoint_multiply))
assert(callable(op._briefing))
assert(callable(op._debriefing))
assert(callable(op.times))
assert(callable(op.__call__))
assert(callable(op.adjoint_times))
assert(callable(op.inverse_times))
assert(callable(op.adjoint_inverse_times))
assert(callable(op.inverse_adjoint_times))
assert(callable(op.tr))
assert(callable(op.inverse_tr))
assert(callable(op.diag))
assert(callable(op.det))
assert(callable(op.inverse_det))
assert(callable(op.log_det))
assert(callable(op.tr_log))
assert(callable(op.hat))
assert(callable(op.inverse_hat))
assert(callable(op.hathat))
assert(callable(op.inverse_hathat))
assert(callable(op.__repr__))
@parameterized.expand(all_operators,
testcase_func_name=custom_name_func)
def test_successfull_init_and_attributes(self, name):
op = generate_operator(name)
assert(isinstance(op.sym, bool))
assert(isinstance(op.uni, bool))
assert(isinstance(op.bare, bool))
assert(isinstance(op.imp, bool))
assert(isinstance(op.domain, space))
assert(isinstance(op.codomain, space))
assert(isinstance(op.target, space))
assert(isinstance(op.cotarget, space))
This diff is collapsed.
import itertools
import unittest
import d2o
import numpy as np
from nifty.rg.rg_space import gc as RG_GC
from nose_parameterized import parameterized
from numpy.testing import assert_raises
from nifty import RGSpace, LMSpace, HPSpace, GLSpace
from nifty import transformator
from nifty.operators.fft_operator.transformations.rgrgtransformation import RGRGTransformation
###############################################################################
def custom_name_func(testcase_func, param_num, param):
return "%s_%s" % (
testcase_func.__name__,
parameterized.to_safe_name("_".join(str(x) for x in param.args)),
)
def weighted_np_transform(val, domain, codomain, axes=None):
if codomain.harmonic:
# correct for forward fft
val = domain.calc_weight(val, power=1)
# Perform the transformation
Tval = np.fft.fftn(val, axes=axes)
if not codomain.harmonic:
# correct for inverse fft
Tval = codomain.calc_weight(Tval, power=-1)
return Tval
###############################################################################
rg_rg_fft_modules = []
for name in ['gfft', 'gfft_dummy', 'pyfftw']:
if RG_GC.validQ('fft_module', name):
rg_rg_fft_modules += [name]
rg_rg_test_shapes = [(128, 128), (179, 179), (512, 512)]
rg_rg_test_spaces = [(GLSpace(8),), (HPSpace(8),), (LMSpace(8),)]
gl_hp_lm_test_spaces = [(GLSpace(8),), (HPSpace(8),), (RGSpace(8),)]
lm_gl_hp_test_spaces = [(LMSpace(8),), (RGSpace(8),)]
###############################################################################
class TestRGRGTransformation(unittest.TestCase):
@parameterized.expand(
rg_rg_test_spaces,
testcase_func_name=custom_name_func
)
def test_check_codomain_mismatch(self, space):
x = RGSpace((8, 8))
with assert_raises(TypeError):
transformator.create(x, space)
@parameterized.expand(
itertools.product([0, 1, 2], [None, (1, 1), (10, 10)], [False, True]),
testcase_func_name=custom_name_func
)
def test_check_codomain_rgspecific(self, complexity, distances, harmonic):
x = RGSpace((8, 8), complexity=complexity,
distances=distances, harmonic=harmonic)
assert (RGRGTransformation.check_codomain(x, x.get_codomain()))
assert (RGRGTransformation.check_codomain(x, x.get_codomain()))
@parameterized.expand(rg_rg_fft_modules, testcase_func_name=custom_name_func)
def test_shapemismatch(self, module):
x = RGSpace((8, 8))
b = d2o.distributed_data_object(np.ones((8, 8)))
with assert_raises(ValueError):
transformator.create(
x, module=module
).transform(b, axes=(0, 1, 2))
@parameterized.expand(
itertools.product(rg_rg_fft_modules, rg_rg_test_shapes),
testcase_func_name=custom_name_func
)
def test_local_ndarray(self, module, shape):
x = RGSpace(shape)
a = np.ones(shape)
assert np.allclose(
transformator.create(
x, module=module
).transform(a),
weighted_np_transform(a, x, x.get_codomain())
)
@parameterized.expand(
itertools.product(rg_rg_fft_modules, rg_rg_test_shapes),
testcase_func_name=custom_name_func
)
def test_local_notzero(self, module, shape):
x = RGSpace(shape[0]) # all tests along axis 1
a = np.ones(shape)
b = d2o.distributed_data_object(a)
assert np.allclose(
transformator.create(
x, module=module
).transform(b, axes=(1,)),
weighted_np_transform(a, x, x.get_codomain(), axes=(1,))
)
@parameterized.expand(
itertools.product(rg_rg_fft_modules, rg_rg_test_shapes),
testcase_func_name=custom_name_func
)
def test_not(self, module, shape):
x = RGSpace(shape)
a = np.ones(shape)
b = d2o.distributed_data_object(a, distribution_strategy='not')
assert np.allclose(
transformator.create(
x, module=module
).transform(b),
weighted_np_transform(a, x, x.get_codomain())
)
@parameterized.expand(
itertools.product(rg_rg_test_shapes),
testcase_func_name=custom_name_func
)
def test_mpi_axesnone(self, shape):
x = RGSpace(shape)
a = np.ones(shape)
b = d2o.distributed_data_object(a)
assert np.allclose(
transformator.create(
x, module='pyfftw'
).transform(b),
weighted_np_transform(a, x, x.get_codomain())
)
@parameterized.expand(
itertools.product(rg_rg_test_shapes),
testcase_func_name=custom_name_func
)
def test_mpi_axesnone_equal(self, shape):
x = RGSpace(shape)
a = np.ones(shape)
b = d2o.distributed_data_object(a, distribution_strategy='equal')
assert np.allclose(
transformator.create(
x, module='pyfftw'
).transform(b),
weighted_np_transform(a, x, x.get_codomain())
)
class TestGLLMTransformation(unittest.TestCase):
@parameterized.expand(
gl_hp_lm_test_spaces,
testcase_func_name=custom_name_func
)
def test_check_codomain_mismatch(self, space):
x = GLSpace(8)
with assert_raises(TypeError):
transformator.create(x, space)
class TestHPLMTransformation(unittest.TestCase):
@parameterized.expand(
gl_hp_lm_test_spaces,