Commit 32707f07 authored by Theo Steininger's avatar Theo Steininger

Merge branch 'ci' into 'master'

Ci



See merge request !1
parents b8f6f2d9 a5263ffc
Pipeline #8816 passed with stage
in 9 minutes and 51 seconds
image: python:2.7-wheezy
before_script:
- pip install nose
- pip install nose_parameterized
- pip install numpy
- chmod +x continuous_integration/*
test_minimal:
script:
- python setup.py install
- nosetests
test_mpi:
script:
- continuous_integration/install_mpi.sh
- python setup.py install
- nosetests
- mpirun -n 2 nosetests
- mpirun -n 5 nosetests
test_mpi_fftw:
script:
- continuous_integration/install_mpi.sh
- continuous_integration/install_fftw.sh
- python setup.py install
- mpirun -n 2 nosetests
test_mpi_fftw_hdf5:
script:
- continuous_integration/install_mpi.sh
- continuous_integration/install_fftw.sh
- continuous_integration/install_h5py.sh
- python setup.py install
- mpirun -n 2 nosetests
#!/bin/bash
pip install cython
apt-get install -y libfftw3-3 libfftw3-bin libfftw3-dev libfftw3-mpi-dev libfftw3-mpi3
git clone -b mpi https://github.com/fredRos/pyFFTW.git
cd pyFFTW/
#export LDFLAGS="-L/usr/include"
#export CFLAGS="-I/usr/include"
CC=mpicc python setup.py build_ext install
cd ..
#!/bin/bash
pip install cython
apt-get update -qy
apt-get install -y libhdf5-openmpi-dev
curl -s https://api.github.com/repos/h5py/h5py/tags | grep tarball_url | head -n 1 | cut -d '"' -f 4 | wget -i - -O h5py.tar.gz
tar xzf h5py.tar.gz
cd h5py-h5py*
export CC=mpicc
python setup.py configure --mpi
python setup.py build
python setup.py install
cd ..
#!/bin/bash
apt-get update -qy
apt-get install -y openmpi-bin libopenmpi-dev
pip install mpi4py
......@@ -22,7 +22,7 @@ import keepers
dependency_injector = keepers.DependencyInjector(
['h5py',
('mpi4py.MPI', 'MPI'),
('d2o.mpi_dummy.mpi_dummy', 'MPI_dummy')]
('mpi_dummy', 'MPI_dummy')]
)
dependency_injector.register('pyfftw', lambda z: hasattr(z, 'FFTW_MPI'))
......
......@@ -1282,6 +1282,7 @@ class distributed_data_object(Versionable, object):
('index', np.dtype('float'))])
local_argmax_list = np.sort(local_argmax_list,
order=['value', 'index'])
# take the last entry here and correct the minus sign of the index
return -np.int(local_argmax_list[-1][1])
......
This diff is collapsed.
# D2O
# Copyright (C) 2016 Theo Steininger
#
# Author: Theo Steininger
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
import mpi_dummy
\ No newline at end of file
# D2O
# Copyright (C) 2016 Theo Steininger
#
# Author: Theo Steininger
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
import copy
import numpy as np
class Op(object):
@classmethod
def Create(cls, function, commute=False):
pass
MIN = Op()
MAX = Op()
SUM = Op()
PROD = Op()
LAND = Op()
LOR = Op()
BAND = Op()
BOR = Op()
class Comm(object):
pass
class Intracomm(Comm):
def __init__(self, name):
if not running_single_threadedQ():
raise RuntimeError("ERROR: MPI_dummy module is running in a " +
"mpirun with n>1.")
self.name = name
self.rank = 0
self.size = 1
def Get_rank(self):
return self.rank
def Get_size(self):
return self.size
def _scattergather_helper(self, sendbuf, recvbuf=None, **kwargs):
sendbuf = self._unwrapper(sendbuf)
recvbuf = self._unwrapper(recvbuf)
if recvbuf is not None:
recvbuf[:] = sendbuf
return recvbuf
else:
recvbuf = np.copy(sendbuf)
return recvbuf
def bcast(self, sendbuf, *args, **kwargs):
return sendbuf
def Bcast(self, sendbuf, *args, **kwargs):
return sendbuf
def scatter(self, sendbuf, *args, **kwargs):
return sendbuf[0]
def Scatter(self, *args, **kwargs):
return self._scattergather_helper(*args, **kwargs)
def Scatterv(self, *args, **kwargs):
return self._scattergather_helper(*args, **kwargs)
def gather(self, sendbuf, *args, **kwargs):
return [sendbuf]
def Gather(self, *args, **kwargs):
return self._scattergather_helper(*args, **kwargs)
def Gatherv(self, *args, **kwargs):
return self._scattergather_helper(*args, **kwargs)
def allgather(self, sendbuf, *args, **kwargs):
return [sendbuf]
def Allgather(self, *args, **kwargs):
return self._scattergather_helper(*args, **kwargs)
def Allgatherv(self, *args, **kwargs):
return self._scattergather_helper(*args, **kwargs)
def Allreduce(self, sendbuf, recvbuf, op, **kwargs):
sendbuf = self._unwrapper(sendbuf)
recvbuf = self._unwrapper(recvbuf)
recvbuf[:] = sendbuf
return recvbuf
def allreduce(self, sendobj, op=SUM, **kwargs):
if np.isscalar(sendobj):
return sendobj
return copy.copy(sendobj)
def sendrecv(self, sendobj, **kwargs):
return sendobj
def _unwrapper(self, x):
if isinstance(x, list):
return x[0]
else:
return x
def Barrier(self):
pass
class _datatype():
def __init__(self, name):
self.name = str(name)
def running_single_threadedQ():
try:
from mpi4py import MPI
except ImportError:
return True
else:
if MPI.COMM_WORLD.size != 1:
return False
else:
return True
BYTE = _datatype('MPI_BYTE')
SHORT = _datatype('MPI_SHORT')
UNSIGNED_SHORT = _datatype("MPI_UNSIGNED_SHORT")
UNSIGNED_INT = _datatype("MPI_UNSIGNED_INT")
INT = _datatype("MPI_INT")
LONG = _datatype("MPI_LONG")
UNSIGNED_LONG = _datatype("MPI_UNSIGNED_LONG")
LONG_LONG = _datatype("MPI_LONG_LONG")
UNSIGNED_LONG_LONG = _datatype("MPI_UNSIGNED_LONG_LONG")
FLOAT = _datatype("MPI_FLOAT")
DOUBLE = _datatype("MPI_DOUBLE")
LONG_DOUBLE = _datatype("MPI_LONG_DOUBLE")
COMPLEX = _datatype("MPI_COMPLEX")
DOUBLE_COMPLEX = _datatype("MPI_DOUBLE_COMPLEX")
class _comm_wrapper(Intracomm):
def __init__(self, name):
self.cache = None
self.name = name
self.size = 1
self.rank = 0
@property
def comm(self):
if self.cache is None:
self.cache = Intracomm(self.name)
return self.cache
def __getattr__(self, x):
return self.comm.__getattribute__(x)
COMM_WORLD = _comm_wrapper('MPI_dummy_COMM_WORLD')
#COMM_WORLD.__class__ = COMM_WORLD.comm.__class__
......@@ -37,11 +37,12 @@ setup(
"computing in Python"),
keywords = "parallelization, numerics, MPI",
url = "https://gitlab.mpcdf.mpg.de/ift/D2O",
packages=['d2o', 'd2o.config', 'd2o.mpi_dummy', 'test'],
packages=['d2o', 'd2o.config', 'test'],
zip_safe=False,
dependency_links = [
"git+https://gitlab.mpcdf.mpg.de/ift/keepers.git#egg=keepers-0.3.4"],
install_requires=['keepers>=0.3.4'],
"git+https://gitlab.mpcdf.mpg.de/ift/keepers.git#egg=keepers-0.3.4",
"git+https://gitlab.mpcdf.mpg.de/ift/mpi_dummy.git#egg=mpi_dummy-1.0.0"],
install_requires=['keepers>=0.3.4', 'mpi_dummy>=1.0.0'],
long_description=read('README.rst'),
license = "GPLv3",
classifiers=[
......
......@@ -18,7 +18,8 @@
from numpy.testing import assert_equal,\
assert_almost_equal,\
assert_raises
assert_raises,\
assert_allclose
from nose_parameterized import parameterized
import unittest
......@@ -136,9 +137,9 @@ def generate_data(global_shape, dtype, distribution_strategy,
local_shape[0] = 0
else:
local_shape[0] = global_shape[0] // np.ceil(size / 2.)
number_of_extras = global_shape[
0] - local_shape[0] * np.ceil(size / 2.)
if number_of_extras > rank:
number_of_extras = (global_shape[0] -
local_shape[0] * np.ceil(size / 2.))
if number_of_extras > rank//2:
local_shape[0] += 1
local_shape = tuple(local_shape)
......@@ -1530,8 +1531,8 @@ class Test_contractions(unittest.TestCase):
(a, obj) = generate_data(global_shape, dtype,
distribution_strategy,
strictly_positive=True)
assert_almost_equal(getattr(obj, function)(), getattr(np, function)(a),
decimal=4)
assert_allclose(getattr(obj, function)(), getattr(np, function)(a),
rtol=1e-4)
###############################################################################
......@@ -1547,8 +1548,8 @@ class Test_contractions(unittest.TestCase):
(a, obj) = generate_data(global_shape, dtype,
distribution_strategy,
strictly_positive=True)
assert_almost_equal(getattr(obj, function)(), getattr(np, function)(a),
decimal=4)
assert_allclose(getattr(obj, function)(), getattr(np, function)(a),
rtol=1e-4)
###############################################################################
......@@ -1557,9 +1558,13 @@ class Test_contractions(unittest.TestCase):
all_distribution_strategies
))
def test_argmin_argmax(self, dtype, distribution_strategy):
print (dtype, distribution_strategy)
global_shape = (8, 8)
(a, obj) = generate_data(global_shape, dtype,
distribution_strategy)
distribution_strategy,
strictly_positive=True)
o_full = obj.get_full_data()
print (a, o_full)
assert_equal(obj.argmax(), np.argmax(a))
assert_equal(obj.argmin(), np.argmin(a))
assert_equal(obj.argmin_nonflat(),
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment