Commit 1297585d authored by Steininger, Theo (theos)'s avatar Steininger, Theo (theos)

Merge branch 'python3' into 'master'

Add Python3 compatibility

See merge request !156
parents da29a8f4 a476b875
Pipeline #17025 passed with stages
in 24 minutes and 49 seconds
......@@ -14,21 +14,22 @@ before_script:
- chmod +x ci/*.sh
- ci/install_basics.sh
- pip install --upgrade -r ci/requirements.txt
- pip3 install --upgrade -r ci/requirements.txt
test_min:
stage: test
script:
- python setup.py build_ext --inplace
- nosetests -vv
- nosetests3 -vv
test_mpi:
stage: test
script:
- ci/install_pyHealpix.sh
- ci/install_mpi4py.sh
- python setup.py build_ext --inplace
- nosetests -vv
- nosetests3 -vv
test_mpi_fftw:
stage: test
......@@ -36,8 +37,8 @@ test_mpi_fftw:
- ci/install_pyHealpix.sh
- ci/install_mpi4py.sh
- ci/install_pyfftw.sh
- python setup.py build_ext --inplace
- nosetests -vv
- nosetests3 -vv
test_mpi_fftw_hdf5:
stage: test
......@@ -46,9 +47,10 @@ test_mpi_fftw_hdf5:
- ci/install_mpi4py.sh
- ci/install_pyfftw.sh
- ci/install_h5py.sh
- python setup.py build_ext --inplace
- mpiexec --allow-run-as-root -n 2 nosetests -x
- mpiexec --allow-run-as-root -n 2 nosetests3 -x
- mpiexec --allow-run-as-root -n 4 nosetests -x
- mpiexec --allow-run-as-root -n 4 nosetests3 -x
- nosetests -x --with-coverage --cover-package=nifty --cover-branches
- >
coverage report | grep TOTAL | awk '{ print "TOTAL: "$6; }'
......
#!/bin/bash
apt-get install -y build-essential python python-pip python-dev git autoconf libtool gsl-bin libgsl-dev wget
apt-get install -y build-essential python python-pip python-dev git autoconf libtool gsl-bin libgsl-dev wget python3 python3-pip python3-dev python3-nose
......@@ -2,3 +2,4 @@
apt-get install -y openmpi-bin libopenmpi-dev
pip install mpi4py
pip3 install mpi4py
......@@ -2,4 +2,5 @@
git clone https://gitlab.mpcdf.mpg.de/ift/pyHealpix.git
(cd pyHealpix && autoreconf -i && ./configure --enable-openmp && make -j4 install)
(cd pyHealpix && autoreconf -i && PYTHON=python3 ./configure --enable-openmp && make -j4 install)
rm -rf pyHealpix
......@@ -4,4 +4,5 @@ apt-get install -y libatlas-base-dev libfftw3-bin libfftw3-dev libfftw3-double3
git clone -b mpi https://github.com/fredros/pyFFTW.git
(cd pyFFTW && CC=mpicc python setup.py build_ext install)
(cd pyFFTW && CC=mpicc python3 setup.py build_ext install)
rm -rf pyFFTW
......@@ -7,8 +7,9 @@ from nifty import plotting
from keepers import Repository
if __name__ == "__main__":
ift.nifty_configuration['default_distribution_strategy'] = 'fftw'
signal_to_noise = 1.5 # The signal to noise ratioa
signal_to_noise = 1.5 # The signal to noise ratio
# Setting up parameters |\label{code:wf_parameters}|
......@@ -29,7 +30,7 @@ if __name__ == "__main__":
harmonic_space_1 = ift.FFTOperator.get_default_codomain(signal_space_1)
fft_1 = ift.FFTOperator(harmonic_space_1, target=signal_space_1,
domain_dtype=np.complex, target_dtype=np.complex)
power_space_1 = ift.PowerSpace(harmonic_space_1, distribution_strategy='fftw')
power_space_1 = ift.PowerSpace(harmonic_space_1)
mock_power_1 = ift.Field(power_space_1, val=power_spectrum_1,
distribution_strategy='not')
......@@ -67,20 +68,18 @@ if __name__ == "__main__":
distribution_strategy='not')
diagonal = mock_power.power_synthesize(spaces=(0, 1), mean=1, std=0,
real_signal=False,
distribution_strategy='fftw')**2
real_signal=False)**2
S = ift.DiagonalOperator(domain=(harmonic_space_1, harmonic_space_2),
diagonal=diagonal)
np.random.seed(10)
mock_signal = fft(mock_power.power_synthesize(real_signal=True,
distribution_strategy='fftw'))
mock_signal = fft(mock_power.power_synthesize(real_signal=True))
# Setting up a exemplary response
N1_10 = int(N_pixels_1/10)
mask_1 = ift.Field(signal_space_1, val=1., distribution_strategy='fftw')
mask_1 = ift.Field(signal_space_1, val=1.)
mask_1.val[N1_10*7:N1_10*9] = 0.
N2_10 = int(N_pixels_2/10)
......@@ -95,12 +94,10 @@ if __name__ == "__main__":
# Setting up the noise covariance and drawing a random noise realization
N = ift.DiagonalOperator(data_domain, diagonal=mock_signal.var()/signal_to_noise,
bare=True,
distribution_strategy='fftw')
bare=True)
noise = ift.Field.from_random(domain=data_domain, random_type='normal',
std=mock_signal.std()/np.sqrt(signal_to_noise),
mean=0,
distribution_strategy='fftw')
mean=0)
data = R(mock_signal) + noise #|\label{code:wf_mock_data}|
# Wiener filter
......@@ -133,7 +130,7 @@ if __name__ == "__main__":
plotter.plot.zmin = 0.
plotter.plot.zmax = 3.
sm = ift.SmoothingOperator(plot_space, sigma=0.03)
sm = ift.SmoothingOperator.make(plot_space, sigma=0.03)
plotter(ift.log(ift.sqrt(sm(ift.Field(plot_space, val=variance.val.real)))), path='uncertainty.html')
plotter.plot.zmin = np.real(mock_signal.min());
......
......@@ -57,7 +57,7 @@ if __name__ == "__main__":
proby = Proby(signal_space, probe_count=800)
proby(lambda z: fft(wiener_curvature.inverse_times(fft.inverse_times(z)))) #|\label{code:wf_variance_fft_wrap}|
sm = ift.SmoothingOperator(signal_space, sigma=0.03)
sm = ift.SmoothingOperator.make(signal_space, sigma=0.03)
variance = ift.sqrt(sm(proby.diagonal.weight(-1))) #|\label{code:wf_variance_weighting}|
repo = Repository('repo_800.h5')
......
# -*- coding: utf-8 -*-
from __future__ import print_function
from nifty import Field, RGSpace, DiagonalProberMixin, TraceProberMixin,\
Prober, DiagonalOperator
......@@ -19,12 +20,9 @@ diagOp = DiagonalOperator(domain=x, diagonal=f)
diagProber = DiagonalProber(domain=x)
diagProber(diagOp)
print (f - diagProber.diagonal).norm()
print((f - diagProber.diagonal).norm())
multiProber = MultiProber(domain=x)
multiProber(diagOp)
print (f - multiProber.diagonal).norm()
print f.sum() - multiProber.trace
print((f - multiProber.diagonal).norm())
print(f.sum() - multiProber.trace)
......@@ -26,36 +26,34 @@ logger = MPILogger()
# it is important to import config before d2o such that NIFTy is able to
# pre-create d2o's configuration object with the corrected path
from config import dependency_injector,\
from .config import dependency_injector,\
nifty_configuration,\
d2o_configuration
from d2o import distributed_data_object, d2o_librarian
from .field import Field
from .random import Random
from field import Field
from .basic_arithmetics import *
from random import Random
from .nifty_utilities import *
from basic_arithmetics import *
from .field_types import *
from nifty_utilities import *
from .energies import *
from field_types import *
from .minimization import *
from energies import *
from .spaces import *
from minimization import *
from .operators import *
from spaces import *
from .probing import *
from operators import *
from .sugar import *
from probing import *
from . import plotting
from sugar import *
import library
import plotting
from . import library
......@@ -16,9 +16,10 @@
# NIFTy is being developed at the Max-Planck-Institut fuer Astrophysik
# and financially supported by the Studienstiftung des deutschen Volkes.
from __future__ import division
import numpy as np
from d2o import distributed_data_object
from nifty.field import Field
from .field import Field
__all__ = ['cos', 'sin', 'cosh', 'sinh', 'tan', 'tanh', 'arccos', 'arcsin',
......
......@@ -17,7 +17,7 @@
# and financially supported by the Studienstiftung des deutschen Volkes.
from nifty_config import dependency_injector,\
from .nifty_config import dependency_injector,\
nifty_configuration
from d2o_config import d2o_configuration
from .d2o_config import d2o_configuration
......@@ -16,7 +16,7 @@
# NIFTy is being developed at the Max-Planck-Institut fuer Astrophysik
# and financially supported by the Studienstiftung des deutschen Volkes.
import matplotlib_init
from . import matplotlib_init
import os
......
......@@ -16,14 +16,17 @@
# NIFTy is being developed at the Max-Planck-Institut fuer Astrophysik
# and financially supported by the Studienstiftung des deutschen Volkes.
from __future__ import division
import abc
from nifty.nifty_meta import NiftyMeta
from .nifty_meta import NiftyMeta
from keepers import Loggable,\
Versionable
from future.utils import with_metaclass
class DomainObject(Versionable, Loggable, object):
class DomainObject(with_metaclass(NiftyMeta, type('NewBase',
(Versionable, Loggable, object), {}))):
"""The abstract class that can be used as a domain for a field.
This holds all the information and functionality a field needs to know
......@@ -39,8 +42,6 @@ class DomainObject(Versionable, Loggable, object):
"""
__metaclass__ = NiftyMeta
def __init__(self):
# _global_id is used in the Versioning module from keepers
self._ignore_for_hash = ['_global_id']
......@@ -56,7 +57,7 @@ class DomainObject(Versionable, Loggable, object):
item = vars(self)[key]
if key in self._ignore_for_hash or key == '_ignore_for_hash':
continue
result_hash ^= item.__hash__() ^ int(hash(key)/117)
result_hash ^= item.__hash__() ^ int(hash(key)//117)
return result_hash
def __eq__(self, x):
......@@ -75,7 +76,7 @@ class DomainObject(Versionable, Loggable, object):
"""
if isinstance(x, type(self)):
for key in vars(self).keys():
for key in list(vars(self).keys()):
item1 = vars(self)[key]
if key in self._ignore_for_hash or key == '_ignore_for_hash':
continue
......
......@@ -16,6 +16,6 @@
# NIFTy is being developed at the Max-Planck-Institut fuer Astrophysik
# and financially supported by the Studienstiftung des deutschen Volkes.
from energy import Energy
from line_energy import LineEnergy
from memoization import memo
from .energy import Energy
from .line_energy import LineEnergy
from .memoization import memo
......@@ -19,9 +19,10 @@
from nifty.nifty_meta import NiftyMeta
from keepers import Loggable
from future.utils import with_metaclass
class Energy(Loggable, object):
class Energy(with_metaclass(NiftyMeta, type('NewBase', (Loggable, object), {}))):
""" Provides the functional used by minimization schemes.
The Energy object is an implementation of a scalar function including its
......@@ -63,8 +64,6 @@ class Energy(Loggable, object):
"""
__metaclass__ = NiftyMeta
def __init__(self, position):
super(Energy, self).__init__()
self._cache = {}
......
......@@ -16,6 +16,8 @@
# NIFTy is being developed at the Max-Planck-Institut fuer Astrophysik
# and financially supported by the Studienstiftung des deutschen Volkes.
from __future__ import print_function
class LineEnergy(object):
""" Evaluates an underlying Energy along a certain line direction.
......@@ -114,6 +116,6 @@ class LineEnergy(object):
def directional_derivative(self):
res = self.energy.gradient.vdot(self.line_direction)
if abs(res.imag) / max(abs(res.real), 1.) > 1e-12:
print "directional derivative has non-negligible " \
"imaginary part:", res
print ("directional derivative has non-negligible "
"imaginary part:", res)
return res.real
......@@ -17,6 +17,9 @@
# and financially supported by the Studienstiftung des deutschen Volkes.
from __future__ import division
from builtins import zip
#from builtins import str
from builtins import range
import ast
import numpy as np
......@@ -27,14 +30,15 @@ from keepers import Versionable,\
from d2o import distributed_data_object,\
STRATEGIES as DISTRIBUTION_STRATEGIES
from nifty.config import nifty_configuration as gc
from .config import nifty_configuration as gc
from nifty.domain_object import DomainObject
from .domain_object import DomainObject
from nifty.spaces.power_space import PowerSpace
from .spaces.power_space import PowerSpace
import nifty.nifty_utilities as utilities
from nifty.random import Random
from . import nifty_utilities as utilities
from .random import Random
from functools import reduce
class Field(Loggable, Versionable, object):
......@@ -346,7 +350,7 @@ class Field(Loggable, Versionable, object):
# check if the `spaces` input is valid
spaces = utilities.cast_axis_to_tuple(spaces, len(self.domain))
if spaces is None:
spaces = range(len(self.domain))
spaces = list(range(len(self.domain)))
if len(spaces) == 0:
raise ValueError(
......@@ -526,7 +530,7 @@ class Field(Loggable, Versionable, object):
spaces = utilities.cast_axis_to_tuple(spaces, len(self.domain))
if spaces is None:
spaces = range(len(self.domain))
spaces = list(range(len(self.domain)))
for power_space_index in spaces:
power_space = self.domain[power_space_index]
......@@ -821,7 +825,7 @@ class Field(Loggable, Versionable, object):
dim_tuple = tuple(sp.dim for sp in self.domain)
try:
return reduce(lambda x, y: x * y, dim_tuple)
return int(reduce(lambda x, y: x * y, dim_tuple))
except TypeError:
return 0
......@@ -1010,7 +1014,7 @@ class Field(Loggable, Versionable, object):
fast_copyable = True
try:
for i in xrange(len(self.domain)):
for i in range(len(self.domain)):
if self.domain[i] is not domain[i]:
fast_copyable = False
break
......@@ -1032,7 +1036,7 @@ class Field(Loggable, Versionable, object):
# repair its class
new_field.__class__ = self.__class__
# copy domain, codomain and val
for key, value in self.__dict__.items():
for key, value in list(self.__dict__.items()):
if key != '_val':
new_field.__dict__[key] = value
else:
......@@ -1069,7 +1073,7 @@ class Field(Loggable, Versionable, object):
spaces = utilities.cast_axis_to_tuple(spaces, len(self.domain))
if spaces is None:
spaces = range(len(self.domain))
spaces = list(range(len(self.domain)))
for ind, sp in enumerate(self.domain):
if ind in spaces:
......@@ -1202,7 +1206,7 @@ class Field(Loggable, Versionable, object):
def _contraction_helper(self, op, spaces):
# build a list of all axes
if spaces is None:
spaces = xrange(len(self.domain))
spaces = range(len(self.domain))
else:
spaces = utilities.cast_axis_to_tuple(spaces, len(self.domain))
......@@ -1222,7 +1226,7 @@ class Field(Loggable, Versionable, object):
return data
else:
return_domain = tuple(self.domain[i]
for i in xrange(len(self.domain))
for i in range(len(self.domain))
if i not in spaces)
return_field = Field(domain=return_domain,
......@@ -1270,7 +1274,7 @@ class Field(Loggable, Versionable, object):
if isinstance(other, Field):
try:
assert len(other.domain) == len(self.domain)
for index in xrange(len(self.domain)):
for index in range(len(self.domain)):
assert other.domain[index] == self.domain[index]
except AssertionError:
raise ValueError(
......@@ -1398,6 +1402,17 @@ class Field(Loggable, Versionable, object):
return self._binary_helper(other, op='__div__')
def __truediv__(self, other):
""" x.__truediv__(y) <==> x/y
See Also
--------
_builtin_helper
"""
return self._binary_helper(other, op='__truediv__')
def __rdiv__(self, other):
""" x.__rdiv__(y) <==> y/x
......@@ -1409,6 +1424,17 @@ class Field(Loggable, Versionable, object):
return self._binary_helper(other, op='__rdiv__')
def __rtruediv__(self, other):
""" x.__rtruediv__(y) <==> y/x
See Also
--------
_builtin_helper
"""
return self._binary_helper(other, op='__rtruediv__')
def __idiv__(self, other):
""" x.__idiv__(y) <==> x/=y
......
......@@ -16,5 +16,5 @@
# NIFTy is being developed at the Max-Planck-Institut fuer Astrophysik
# and financially supported by the Studienstiftung des deutschen Volkes.
from field_type import FieldType
from field_array import FieldArray
from .field_type import FieldType
from .field_array import FieldArray
......@@ -16,7 +16,8 @@
# NIFTy is being developed at the Max-Planck-Institut fuer Astrophysik
# and financially supported by the Studienstiftung des deutschen Volkes.
from field_type import FieldType
from .field_type import FieldType
from functools import reduce
class FieldArray(FieldType):
......
from critical_filter import *
from log_normal_wiener_filter import *
from wiener_filter import *
from .critical_filter import *
from .log_normal_wiener_filter import *
from .wiener_filter import *
# -*- coding: utf-8 -*-
from critical_power_curvature import CriticalPowerCurvature
from critical_power_energy import CriticalPowerEnergy
from .critical_power_curvature import CriticalPowerCurvature
from .critical_power_energy import CriticalPowerEnergy
# -*- coding: utf-8 -*-
from log_normal_wiener_filter_curvature import LogNormalWienerFilterCurvature
from log_normal_wiener_filter_energy import LogNormalWienerFilterEnergy
from .log_normal_wiener_filter_curvature import LogNormalWienerFilterCurvature
from .log_normal_wiener_filter_energy import LogNormalWienerFilterEnergy
# -*- coding: utf-8 -*-
from wiener_filter_curvature import WienerFilterCurvature
from wiener_filter_energy import WienerFilterEnergy
from .wiener_filter_curvature import WienerFilterCurvature
from .wiener_filter_energy import WienerFilterEnergy
......@@ -16,9 +16,9 @@
# NIFTy is being developed at the Max-Planck-Institut fuer Astrophysik
# and financially supported by the Studienstiftung des deutschen Volkes.
from line_searching import *
from conjugate_gradient import ConjugateGradient
from descent_minimizer import DescentMinimizer
from steepest_descent import SteepestDescent
from vl_bfgs import VL_BFGS
from relaxed_newton import RelaxedNewton
from .line_searching import *
from .conjugate_gradient import ConjugateGradient
from .descent_minimizer import DescentMinimizer
from .steepest_descent import SteepestDescent
from .vl_bfgs import VL_BFGS
from .relaxed_newton import RelaxedNewton
......@@ -16,6 +16,7 @@
# NIFTy is being developed at the Max-Planck-Institut fuer Astrophysik
# and financially supported by the Studienstiftung des deutschen Volkes.
from __future__ import division
import abc
from nifty.nifty_meta import NiftyMeta
......@@ -24,9 +25,10 @@ import numpy as np
from keepers import Loggable
from .line_searching import LineSearchStrongWolfe
from future.utils import with_metaclass
class DescentMinimizer(Loggable, object):
class DescentMinimizer(with_metaclass(NiftyMeta, type('NewBase', (Loggable, object), {}))):
""" A base class used by gradient methods to find a local minimum.
Descent minimization methods are used to find a local minimum of a scalar
......@@ -77,8 +79,6 @@ class DescentMinimizer(Loggable, object):
"""
__metaclass__ = NiftyMeta
def __init__(self, line_searcher=LineSearchStrongWolfe(), callback=None,
convergence_tolerance=1E-4, convergence_level=3,
iteration_limit=None):
......
......@@ -16,5 +16,5 @@
# NIFTy is being developed at the Max-Planck-Institut fuer Astrophysik
# and financially supported by the Studienstiftung des deutschen Volkes.
from line_search import LineSearch
from line_search_strong_wolfe import LineSearchStrongWolfe
from .line_search import LineSearch
from .line_search_strong_wolfe import LineSearchStrongWolfe
......@@ -21,9 +21,10 @@ import abc
from keepers import Loggable
from nifty import LineEnergy
from future.utils import with_metaclass
class LineSearch(Loggable, object):
class LineSearch(with_metaclass(abc.ABCMeta, type('NewBase', (Loggable, object), {}))):
"""Class for determining the optimal step size along some descent direction.
Initialize the line search procedure which can be used by a specific line
......
......@@ -16,6 +16,9 @@
# NIFTy is being developed at the Max-Planck-Institut fuer Astrophysik
# and financially supported by the Studienstiftung des deutschen Volkes.
from __future__ import print_function
from __future__ import division
from builtins import range
import numpy as np
from .line_search import LineSearch
......@@ -124,7 +127,7 @@ class LineSearchStrongWolfe(LineSearch):
alpha1 = 1.0/pk.norm()
# start the minimization loop
for i in xrange(self.max_iterations):
for i in range(self.max_iterations):
if alpha1 == 0:
self.logger.warn("Increment size became 0.")
return le_0.energy
......@@ -152,7 +155,7 @@ class LineSearchStrongWolfe(LineSearch):
# update alphas
alpha0, alpha1 = alpha1, min(2*alpha1, self.max_step_size)
if alpha1 == self.max_step_size:
print "reached max step size, bailing out"
print ("reached max step size, bailing out")
return le_alpha1.energy
phi_alpha0 = phi_alpha1
......@@ -207,7 +210,7 @@ class LineSearchStrongWolfe(LineSearch):
assert phi_lo <= phi_0 + self.c1*alpha_lo*phiprime_0
assert phiprime_lo*(alpha_hi-alpha_lo) < 0.
for i in xrange(self.max_zoom_iterations):
for i in range(self.max_zoom_iterations):
# assert phi_lo <= phi_0 + self.c1*alpha_lo*phiprime_0
# assert phiprime_lo*(alpha_hi-alpha_lo)<0.
delta_alpha = alpha_hi - alpha_lo
......
from __future__ import division
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
......@@ -16,6 +17,7 @@
# NIFTy is being developed at the Max-Planck-Institut fuer Astrophysik
# and financially supported by the Studienstiftung des deutschen Volkes.
from __future__ import division
from .descent_minimizer import DescentMinimizer
......
......@@ -16,6 +16,9 @@
# NIFTy is being developed at the Max-Planck-Institut fuer Astrophysik
# and financially supported by the Studienstiftung des deutschen Volkes.
from __future__ import division
from builtins import range
from builtins import object
import numpy as np
from .descent_minimizer import DescentMinimizer
......@@ -81,7 +84,7 @@ class VL_BFGS(DescentMinimizer):
delta = self._information_store.delta
descent_direction = delta[0] * b[0]
for i in xrange(1, len(delta)):
for i in range(1, len(delta)):
descent_direction += delta[i] * b[i]
return descent_direction
......@@ -154,14 +157,15 @@ class InformationStore(object):
result = []
m = self.history_length
mmax = self.max_history_length
k = self.k
k = self.k
s = self.s
for i in xrange(m):