Skip to content

GitLab

Commit b9a1dd5e authored by Jait Dixit's avatar Jait Dixit
Browse files

WIP: initial ci config

parent c286d744
Pipeline #9315 failed with stages
in 31 minutes and 11 seconds
Expand all Hide whitespace changes
Inline Side-by-side
Showing with 198 additions and 2100 deletions
.gitlab-ci.yml 0 → 100644
View file @ b9a1dd5e
image: ubuntu:latest
stages:
- test
- release
variables:
DOCKER_DRIVER: overlay
RELEASE_IMAGE: iftmpa/nifty:ci_test
before_script:
- apt-get update
- >
apt-get install -y build-essential python python-pip python-dev git
gfortran autoconf gsl-bin libgsl-dev
- pip install -r ci/requirements_base.txt
- chmod +x ci/*.sh
test_min:
stage: test
script:
- python setup.py build_ext --inplace
- nosetests -vv
test_mpi:
stage: test
script:
- apt-get install -y openmpi-bin libopenmpi-dev
- pip install astropy healpy mpi4py
- pip install git+https://github.com/mrbell/gfft
- ci/install_libsharp.sh
- python setup.py build_ext --inplace
- nosetests -vv
test_mpi_fftw:
stage: test
script:
- apt-get install -y openmpi-bin libopenmpi-dev
- >
apt-get install -y libatlas-base-dev libfftw3-bin libfftw3-dev
libfftw3-double3 libfftw3-long3 libfftw3-mpi-dev libfftw3-mpi3
libfftw3-quad3 libfftw3-single3
- pip install astropy healpy mpi4py
- pip install git+https://github.com/mrbell/gfft
- ci/install_libsharp.sh
- ci/install_pyfftw.sh
- python setup.py build_ext --inplace
- nosetests -vv
test_mpi_fftw_hdf5:
stage: test
script:
- apt-get install -y openmpi-bin libopenmpi-dev
- >
apt-get install -y libatlas-base-dev libfftw3-bin libfftw3-dev
libfftw3-double3 libfftw3-long3 libfftw3-mpi-dev libfftw3-mpi3
libfftw3-quad3 libfftw3-single3
- >
apt-get install -y libhdf5-10 libhdf5-dev libhdf5-openmpi-10
libhdf5-openmpi-dev hdf5-tools python-h5py
- pip install astropy healpy mpi4py
- pip install git+https://github.com/mrbell/gfft
- ci/install_libsharp.sh
- ci/install_pyfftw.sh
- python setup.py build_ext --inplace
- nosetests -vv
release_image:
image: docker:latest
stage: release
before_script:
- pwd
services:
- docker:dind
script:
- docker login -u foxymoron -p SupernovaProgenitorsII1@3
- docker build -t $RELEASE_IMAGE .
- docker push $RELEASE_IMAGE
Dockerfile 0 → 100644
View file @ b9a1dd5e
FROM ubuntu:latest
# dependencies via apt
RUN \
apt-get update && \
apt-get install -y build-essential python python-pip python-dev git \
gfortran autoconf gsl-bin libgsl-dev python-matplotlib openmpi-bin \
libopenmpi-dev libatlas-base-dev libfftw3-bin libfftw3-dev \
libfftw3-double3 libfftw3-long3 libfftw3-mpi-dev libfftw3-mpi3 \
libfftw3-quad3 libfftw3-single3 libhdf5-10 libhdf5-dev \
libhdf5-openmpi-10 libhdf5-openmpi-dev hdf5-tools python-h5py
# python dependencies
ADD ci/requirements.txt /tmp/requirements.txt
RUN pip install -r /tmp/requirements.txt
# install libsharp and pyfftw
ADD ci/install_libsharp.sh /tmp/install_libsharp.sh
ADD ci/install_pyfftw.sh /tmp/install_pyfftw.sh
RUN cd /tmp && chmod +x install_libsharp.sh && ./install_libsharp.sh
RUN cd /tmp && chmod +x install_pyfftw.sh && ./install_pyfftw.sh
# copy sources and install nifty
COPY . /tmp/NIFTy
RUN pip install /tmp/NIFTy
# Cleanup
RUN rm -r /tmp/*
ci/install_libsharp.sh 0 → 100644
View file @ b9a1dd5e
#!/bin/bash
git clone http://git.code.sf.net/p/libsharp/code libsharp-code
cd libsharp-code
autoconf && ./configure --enable-pic --disable-openmp && make
cd ..
git clone https://github.com/mselig/libsharp-wrapper libsharp-wrapper
cd libsharp-wrapper
python setup.py build_ext install
cd ..
rm -r libsharp-code
rm -r libsharp-wrapper
ci/install_pyfftw.sh 0 → 100644
View file @ b9a1dd5e
#!/bin/bash
git clone -b mpi https://github.com/fredRos/pyFFTW.git
cd pyFFTW/
CC=mpicc python setup.py build_ext install
cd ..
rm -r pyFFTW
ci/requirements.txt 0 → 100644
View file @ b9a1dd5e
numpy
scipy
cython
astropy
healpy
mpi4py
ipython
nose
nose-parameterized
git+https://gitlab.mpcdf.mpg.de/ift/mpi_dummy.git
git+https://gitlab.mpcdf.mpg.de/ift/keepers.git
git+https://gitlab.mpcdf.mpg.de/ift/D2O.git
ci/requirements_base.txt 0 → 100644
View file @ b9a1dd5e
numpy
scipy
cython
nose
nose-parameterized
git+https://gitlab.mpcdf.mpg.de/ift/mpi_dummy.git
git+https://gitlab.mpcdf.mpg.de/ift/keepers.git
git+https://gitlab.mpcdf.mpg.de/ift/D2O.git
nifty/__init__.py
View file @ b9a1dd5e
......@@ -21,9 +21,6 @@
from __future__ import division
import matplotlib as mpl
mpl.use('Agg')
from .version import __version__
# initialize the logger instance
......
test/test_nifty.py 0 → 100644
View file @ b9a1dd5e
import nifty as nt
import numpy as np
import unittest
def weighted_np_transform(val, domain, codomain, axes=None):
if codomain.harmonic:
# correct for forward fft
val = domain.weight(val, power=1, axes=axes)
# Perform the transformation
Tval = np.fft.fftn(val, axes=axes)
if not codomain.harmonic:
# correct for inverse fft
Tval = codomain.weight(Tval, power=-1, axes=axes)
return Tval
def test_simple_fft():
x = nt.RGSpace((16,))
x_p = nt.FFTOperator.get_default_codomain(x)
f = nt.Field((x, x), val=1)
val_p = np.ones((16,16))
fft = nt.FFTOperator(x)
assert np.allclose(
fft(f, spaces=(1,)).val,
weighted_np_transform(val_p, x, x_p, axes=(1,))
)
test/test_nifty_field.py deleted 100644 → 0
View file @ c286d744
# -*- coding: utf-8 -*-
from numpy.testing import assert_equal, \
assert_almost_equal, \
assert_raises
from nose_parameterized import parameterized
import unittest
import itertools
import numpy as np
from d2o import distributed_data_object
from nifty import Space, \
point_space
from nifty.rg import RGSpace
from nifty.lm import *
from nifty.field import Field
from nifty.nifty_core import POINT_DISTRIBUTION_STRATEGIES
from nifty.rg.rg_space import RG_DISTRIBUTION_STRATEGIES, \
gc as RG_GC
from nifty.lm.nifty_lm import LM_DISTRIBUTION_STRATEGIES, \
GL_DISTRIBUTION_STRATEGIES, \
HP_DISTRIBUTION_STRATEGIES
###############################################################################
def custom_name_func(testcase_func, param_num, param):
return "%s_%s" % (
testcase_func.__name__,
parameterized.to_safe_name("_".join(str(x) for x in param.args)),
)
###############################################################################
###############################################################################
all_point_datatypes = [np.dtype('bool'),
np.dtype('int16'),
np.dtype('int32'),
np.dtype('int64'),
np.dtype('float32'),
np.dtype('float64'),
np.dtype('complex64'),
np.dtype('complex128')]
all_lm_datatypes = [np.dtype('complex64'),
np.dtype('complex128')]
all_gl_datatypes = [np.dtype('float64'),
np.dtype('float128')]
all_hp_datatypes = [np.dtype('float64')]
###############################################################################
DATAMODELS = {}
DATAMODELS['point_space'] = POINT_DISTRIBUTION_STRATEGIES
DATAMODELS['rg_space'] = RG_DISTRIBUTION_STRATEGIES
DATAMODELS['lm_space'] = LM_DISTRIBUTION_STRATEGIES
DATAMODELS['gl_space'] = GL_DISTRIBUTION_STRATEGIES
DATAMODELS['hp_space'] = HP_DISTRIBUTION_STRATEGIES
###############################################################################
fft_modules = []
for name in ['gfft', 'gfft_dummy', 'pyfftw']:
if RG_GC.validQ('fft_module', name):
fft_modules += [name]
###############################################################################
point_like_spaces = ['point_space', 'rg_space', 'lm_space', 'hp_space',
'gl_space']
###############################################################################
space_list = []
# Add point_spaces
for param in itertools.product([1, 10],
all_point_datatypes):
space_list += [[point_space(num=param[0], dtype=param[1])]]
# Add rg_spaces
for param in itertools.product([(1,), (4, 6), (5, 8)],
[False, True],
[0, 1, 2],
[None, 0.3],
[False],
DATAMODELS['rg_space'],
fft_modules):
space_list += [[RGSpace(shape=param[0],
zerocenter=param[1],
complexity=param[2],
distances=param[3],
harmonic=param[4],
fft_module=param[6]), param[5]]]
def generate_space_with_size(name, num):
space_dict = {'space': Space(),
'point_space': point_space(num),
'rg_space': RGSpace((num, num)),
'lm_space': LMSpace(mmax=num+1, lmax=num+1),
'hp_space': HPSpace(num),
'gl_space': GLSpace(nlat=num, nlon=2*num-1),
}
return space_dict[name]
def generate_data(space):
a = np.arange(space.dim).reshape(space.shape)
return distributed_data_object(a)
###############################################################################
###############################################################################
class Test_field_init(unittest.TestCase):
@parameterized.expand(
itertools.product([(1,), (4, 6), (5, 8)],
[False, True],
[0, 1, 2],
[None, 0.3],
[False],
fft_modules,
DATAMODELS['rg_space']),
testcase_func_name=custom_name_func)
def test_successfull_init_and_attributes(self, shape, zerocenter,
complexity, distances, harmonic,
fft_module, datamodel):
s = RGSpace(shape=shape, zerocenter=zerocenter,
complexity=complexity, distances=distances,
harmonic=harmonic, fft_module=fft_module)
f = Field(domain=(s,), dtype=s.dtype, datamodel=datamodel)
assert (f.domain[0] is s)
assert (s.check_codomain(f.codomain[0]))
assert (s.shape == f.shape)
class Test_field_init2(unittest.TestCase):
@parameterized.expand(
itertools.product(point_like_spaces, [4],
DATAMODELS['rg_space']),
testcase_func_name=custom_name_func)
def test_successfull_init_and_attributes(self, name, num, datamodel):
s = generate_space_with_size(name, num)
d = generate_data(s)
f = Field(val=d, domain=(s,), dtype=s.dtype, datamodel=datamodel)
assert (f.domain[0] is s)
assert (s.check_codomain(f.codomain[0]))
assert (s.shape == f.shape)
class Test_field_multiple_rg_init(unittest.TestCase):
@parameterized.expand(
itertools.product([(1,)],
[True],
[0],
[None],
[False],
fft_modules,
DATAMODELS['rg_space']),
testcase_func_name=custom_name_func)
def test_multiple_space_init(self, shape, zerocenter,
complexity, distances, harmonic,
fft_module, datamodel):
s1 = RGSpace(shape=shape, zerocenter=zerocenter,
complexity=complexity, distances=distances,
harmonic=harmonic, fft_module=fft_module)
s2 = RGSpace(shape=shape, zerocenter=zerocenter,
complexity=complexity, distances=distances,
harmonic=harmonic, fft_module=fft_module)
f = Field(domain=(s1, s2), dtype=s1.dtype, datamodel=datamodel)
assert (f.domain[0] is s1)
assert (f.domain[1] is s2)
assert (s1.check_codomain(f.codomain[0]))
assert (s2.check_codomain(f.codomain[1]))
assert (s1.shape + s2.shape == f.shape)
class Test_field_multiple_init(unittest.TestCase):
@parameterized.expand(
itertools.product(point_like_spaces, point_like_spaces, [4]),
testcase_func_name=custom_name_func)
def test_multiple_space_init(self, space1, space2, shape):
s1 = generate_space_with_size(space1, shape)
s2 = generate_space_with_size(space2, shape)
f = Field(domain=(s1, s2))
assert (f.domain[0] is s1)
assert (f.domain[1] is s2)
assert (s1.check_codomain(f.codomain[0]))
assert (s2.check_codomain(f.codomain[1]))
assert (s1.shape + s2.shape == f.shape)
s3 = generate_space_with_size('hp_space', shape)
f = Field(domain=(s1, s2, s3))
assert (f.domain[0] is s1)
assert (f.domain[1] is s2)
assert (f.domain[2] is s3)
assert (s1.check_codomain(f.codomain[0]))
assert (s2.check_codomain(f.codomain[1]))
assert (s3.check_codomain(f.codomain[2]))
assert (s1.shape + s2.shape + s3.shape ==
f.shape)
class Test_axis(unittest.TestCase):
@parameterized.expand(
itertools.product(point_like_spaces, [4],
['sum', 'prod', 'mean', 'var', 'std', 'median', 'all',
'any', 'min', 'nanmin', 'argmin', 'max', 'nanmax',
'argmax'],
[None, (0,)],
DATAMODELS['rg_space']),
testcase_func_name=custom_name_func)
def test_unary_operations(self, name, num, op, axis, datamodel):
s = generate_space_with_size(name, num)
d = generate_data(s)
a = d.get_full_data()
f = Field(val=d, domain=(s,), dtype=s.dtype, datamodel=datamodel)
if op in ['argmin', 'argmax']:
assert_almost_equal(getattr(f, op)(),
getattr(np, op)(a), decimal=4)
else:
assert_almost_equal(getattr(f, op)(axis=axis),
getattr(np, op)(a, axis=axis), decimal=4)
binary_operations = [('add','__add__'),('radd','__radd__'),('iadd','__iadd__'),
('sub','__sub__'),('rsub','__rsub__'),('isub','__isub__'),
('mul','__mul__'),('rmul','__rmul__'),('imul','__imul__'),
('div','__div__'),('rdiv','__rdiv__'),('idiv','__idiv__'),
('pow','__pow__'),('rpow','__rpow__'),('ipow','__ipow__'),
('ne','__ne__'),('lt','__lt__'),('eq','__eq__'),
('ge','__ge__'),('gt','__gt__')]
class Test_binary_operation(unittest.TestCase):
@parameterized.expand(
itertools.product([point_like_spaces[0]], [4], binary_operations,
DATAMODELS['rg_space']),
testcase_func_name=custom_name_func)
def test_binary_operations(self, name, num, op, datamodel):
s = generate_space_with_size(name, num)
d = generate_data(s)
a = d.get_full_data()
f = Field(val=d, domain=(s,), dtype=s.dtype, datamodel=datamodel)
d2 = d[::-1]
a2 = np.copy(a[::-1])
if op[0] in ['iadd','isub','imul','idiv']:
getattr(a, op[1])(a2)
f.binary_operation(d, d2, op[0])
assert_almost_equal(a,d,4)
else:
assert_almost_equal(getattr(a, op[1])(a2),f.binary_operation(d, d2,
op[0]), 4)
\ No newline at end of file
test/test_nifty_operators.py deleted 100644 → 0
View file @ c286d744
# -*- coding: utf-8 -*-
from numpy.testing import assert_equal,\
assert_almost_equal,\
assert_raises
from nose_parameterized import parameterized
import unittest
import itertools
import numpy as np
from nifty import Space,\
RGSpace,\
Field,\
distributed_data_object
from nifty.operators import operator,\
diagonal_operator,\
power_operator,\
projection_operator,\
vecvec_operator,\
response_operator,\
invertible_operator,\
propagator_operator,\
identity_operator
from nifty.nifty_core import POINT_DISTRIBUTION_STRATEGIES
from nifty.rg.rg_space import RG_DISTRIBUTION_STRATEGIES,\
gc as RG_GC
from nifty.lm.nifty_lm import LM_DISTRIBUTION_STRATEGIES,\
GL_DISTRIBUTION_STRATEGIES,\
HP_DISTRIBUTION_STRATEGIES
available = []
try:
from nifty import LMSpace
except ImportError:
pass
else:
available += ['lm_space']
try:
from nifty import GLSpace
except ImportError:
pass
else:
available += ['gl_space']
try:
from nifty import HPSpace
except ImportError:
pass
else:
available += ['hp_space']
###############################################################################
def custom_name_func(testcase_func, param_num, param):
return "%s_%s" % (
testcase_func.__name__,
parameterized.to_safe_name("_".join(str(x) for x in param.args)),
)
###############################################################################
###############################################################################
all_point_datatypes = [np.dtype('bool'),
np.dtype('int16'),
np.dtype('int32'),
np.dtype('int64'),
np.dtype('float32'),
np.dtype('float64'),
np.dtype('complex64'),
np.dtype('complex128')]
all_lm_datatypes = [np.dtype('complex64'),
np.dtype('complex128')]
all_gl_datatypes = [np.dtype('float64'),
np.dtype('float128')]
all_hp_datatypes = [np.dtype('float64')]
###############################################################################
DATAMODELS = {}
DATAMODELS['point_space'] = ['np'] + POINT_DISTRIBUTION_STRATEGIES
DATAMODELS['rg_space'] = ['np'] + RG_DISTRIBUTION_STRATEGIES
DATAMODELS['lm_space'] = ['np'] + LM_DISTRIBUTION_STRATEGIES
DATAMODELS['gl_space'] = ['np'] + GL_DISTRIBUTION_STRATEGIES
DATAMODELS['hp_space'] = ['np'] + HP_DISTRIBUTION_STRATEGIES
###############################################################################
all_spaces = ['space', 'point_space', 'rg_space']
if 'lm_space' in available:
all_spaces += ['lm_space']
if 'gl_space' in available:
all_spaces += ['gl_space']
if 'hp_space' in available:
all_spaces += ['hp_space']
point_like_spaces = ['point_space', 'rg_space']
if 'lm_space' in available:
point_like_spaces += ['lm_space']
if 'gl_space' in available:
point_like_spaces += ['gl_space']
if 'hp_space' in available:
point_like_spaces += ['hp_space']
###############################################################################
np_spaces = point_like_spaces
d2o_spaces = []
if POINT_DISTRIBUTION_STRATEGIES != []:
d2o_spaces += ['point_space']
if RG_DISTRIBUTION_STRATEGIES != []:
d2o_spaces += ['rg_space']
if LM_DISTRIBUTION_STRATEGIES != []:
d2o_spaces += ['lm_space']
if GL_DISTRIBUTION_STRATEGIES != []:
d2o_spaces += ['gl_space']
if HP_DISTRIBUTION_STRATEGIES != []:
d2o_spaces += ['hp_space']
###############################################################################
all_operators = ['operator', 'diagonal_operator', 'power_operator',
'projection_operator', 'vecvec_operator', 'response_operator',
'invertible_operator', 'propagator_operator',
'identity_operator']
###############################################################################
def generate_operator(name):
x = RGSpace((8, 8))
k = x.get_codomain()
operator_dict = {'operator': operator(domain=x),
'diagonal_operator': diagonal_operator(domain=x),
'identity_operator': identity_operator(domain=x),
'power_operator': power_operator(domain=k),
'projection_operator': projection_operator(domain=x),
'vecvec_operator': vecvec_operator(domain=x),
'response_operator': response_operator(domain=x),
'invertible_operator': invertible_operator(domain=x)}
return operator_dict[name]