Planned maintenance on Wednesday, 2021-01-20, 17:00-18:00. Expect some interruptions during that time

Commit e5620e98 authored by Martin Reinecke's avatar Martin Reinecke

merge attempt

parents c016b764 f3dea2e4
Pipeline #71317 failed with stages
in 47 seconds
Changes since NIFTy 5: Changes since NIFTy 5:
MPI parallelisation over samples in MetricGaussianKL
====================================================
The classes `MetricGaussianKL` and `MetricGaussianKL_MPI` have been unified
into one `MetricGaussianKL` class which has MPI support built in.
New approach for random number generation
=========================================
The code now uses `numpy`'s new `SeedSequence` and `Generator` classes for the
production of random numbers (introduced in numpy 1.17. This greatly simplifies
the generation of reproducible random numbers in the presence of MPI parallelism
and leads to cleaner code overall. Please see the documentation of
`nifty6.random` for details.
Interface Change for non-linear Operators Interface Change for non-linear Operators
========================================= =========================================
...@@ -17,7 +32,6 @@ behaviour since both `Operator._check_input()` and ...@@ -17,7 +32,6 @@ behaviour since both `Operator._check_input()` and
`extra.check_jacobian_consistency()` tests for the new conditions to be `extra.check_jacobian_consistency()` tests for the new conditions to be
fulfilled. fulfilled.
Special functions for complete Field reduction operations Special functions for complete Field reduction operations
========================================================= =========================================================
...@@ -66,12 +80,3 @@ User-visible changes: ...@@ -66,12 +80,3 @@ User-visible changes:
replaced by a single function called `makeField` replaced by a single function called `makeField`
- the property `local_shape` has been removed from `Domain` (and subclasses) - the property `local_shape` has been removed from `Domain` (and subclasses)
and `DomainTuple`. and `DomainTuple`.
Transfer of MPI parallelization into operators:
===============================================
As was already the case with the `MetricGaussianKL_MPI` in NIFTy5, MPI
parallelization in NIFTy6 is handled by specialized MPI-enabled operators.
They are accessible via the `nifty6.mpi` namespace, from which they can be
imported directly: `from nifty6.mpi import MPIenabledOperator`.
...@@ -140,7 +140,6 @@ ...@@ -140,7 +140,6 @@
"outputs": [], "outputs": [],
"source": [ "source": [
"import numpy as np\n", "import numpy as np\n",
"np.random.seed(40)\n",
"import nifty6 as ift\n", "import nifty6 as ift\n",
"import matplotlib.pyplot as plt\n", "import matplotlib.pyplot as plt\n",
"%matplotlib inline" "%matplotlib inline"
......
...@@ -5,8 +5,6 @@ import numpy as np ...@@ -5,8 +5,6 @@ import numpy as np
import nifty6 as ift import nifty6 as ift
np.random.seed(40)
N0s, a0s, b0s, c0s = [], [], [], [] N0s, a0s, b0s, c0s = [], [], [], []
for ii in range(10, 26): for ii in range(10, 26):
...@@ -15,15 +13,16 @@ for ii in range(10, 26): ...@@ -15,15 +13,16 @@ for ii in range(10, 26):
N = int(2**ii) N = int(2**ii)
print('N = {}'.format(N)) print('N = {}'.format(N))
uv = np.random.rand(N, 2) - 0.5 rng = ift.random.current_rng()
vis = np.random.randn(N) + 1j*np.random.randn(N) uv = rng.uniform(-.5, .5, (N,2))
vis = rng.normal(0., 1., N) + 1j*rng.normal(0., 1., N)
uvspace = ift.RGSpace((nu, nv)) uvspace = ift.RGSpace((nu, nv))
visspace = ift.TMP_UnstructuredSpace(N) visspace = ift.TMP_UnstructuredSpace(N)
img = np.random.randn(nu*nv) img = rng.standard_normal((nu, nv))
img = img.reshape((nu, nv)) img = ift.makeField(uvspace, img)
img = ift.makeTMP_fld(uvspace, img) img = ift.makeTMP_fld(uvspace, img)
t0 = time() t0 = time()
......
...@@ -27,8 +27,6 @@ import numpy as np ...@@ -27,8 +27,6 @@ import numpy as np
import nifty6 as ift import nifty6 as ift
if __name__ == '__main__': if __name__ == '__main__':
np.random.seed(41)
# Set up the position space of the signal # Set up the position space of the signal
mode = 2 mode = 2
if mode == 0: if mode == 0:
...@@ -62,7 +60,7 @@ if __name__ == '__main__': ...@@ -62,7 +60,7 @@ if __name__ == '__main__':
p = R(sky) p = R(sky)
mock_position = ift.from_random('normal', harmonic_space) mock_position = ift.from_random('normal', harmonic_space)
tmp = p(mock_position).val.astype(np.float64) tmp = p(mock_position).val.astype(np.float64)
data = np.random.binomial(1, tmp) data = ift.random.current_rng().binomial(1, tmp)
data = ift.TMP_fld.from_raw(R.target, data) data = ift.TMP_fld.from_raw(R.target, data)
# Compute likelihood and Hamiltonian # Compute likelihood and Hamiltonian
......
...@@ -46,8 +46,6 @@ def make_random_mask(): ...@@ -46,8 +46,6 @@ def make_random_mask():
if __name__ == '__main__': if __name__ == '__main__':
np.random.seed(42)
# Choose space on which the signal field is defined # Choose space on which the signal field is defined
if len(sys.argv) == 2: if len(sys.argv) == 2:
mode = int(sys.argv[1]) mode = int(sys.argv[1])
......
...@@ -44,8 +44,6 @@ def exposure_2d(): ...@@ -44,8 +44,6 @@ def exposure_2d():
if __name__ == '__main__': if __name__ == '__main__':
np.random.seed(42)
# Choose space on which the signal field is defined # Choose space on which the signal field is defined
if len(sys.argv) == 2: if len(sys.argv) == 2:
mode = int(sys.argv[1]) mode = int(sys.argv[1])
...@@ -94,7 +92,7 @@ if __name__ == '__main__': ...@@ -94,7 +92,7 @@ if __name__ == '__main__':
lamb = R(sky) lamb = R(sky)
mock_position = ift.from_random('normal', domain) mock_position = ift.from_random('normal', domain)
data = lamb(mock_position) data = lamb(mock_position)
data = np.random.poisson(data.val.astype(np.float64)) data = ift.random.current_rng().poisson(data.val.astype(np.float64))
data = ift.TMP_fld.from_raw(d_space, data) data = ift.TMP_fld.from_raw(d_space, data)
likelihood = ift.PoissonianEnergy(data) @ lamb likelihood = ift.PoissonianEnergy(data) @ lamb
......
...@@ -33,20 +33,18 @@ import nifty6 as ift ...@@ -33,20 +33,18 @@ import nifty6 as ift
def random_los(n_los): def random_los(n_los):
starts = list(np.random.uniform(0, 1, (n_los, 2)).T) starts = list(ift.random.current_rng().random((n_los, 2)).T)
ends = list(np.random.uniform(0, 1, (n_los, 2)).T) ends = list(ift.random.current_rng().random((n_los, 2)).T)
return starts, ends return starts, ends
def radial_los(n_los): def radial_los(n_los):
starts = list(np.random.uniform(0, 1, (n_los, 2)).T) starts = list(ift.random.current_rng().random((n_los, 2)).T)
ends = list(0.5 + 0*np.random.uniform(0, 1, (n_los, 2)).T) ends = list(0.5 + 0*ift.random.current_rng().random((n_los, 2)).T)
return starts, ends return starts, ends
if __name__ == '__main__': if __name__ == '__main__':
np.random.seed(420)
# Choose between random line-of-sight response (mode=0) and radial lines # Choose between random line-of-sight response (mode=0) and radial lines
# of sight (mode=1) # of sight (mode=1)
if len(sys.argv) == 2: if len(sys.argv) == 2:
......
...@@ -44,20 +44,18 @@ class SingleTMP_Space(ift.LinearOperator): ...@@ -44,20 +44,18 @@ class SingleTMP_Space(ift.LinearOperator):
def random_los(n_los): def random_los(n_los):
starts = list(np.random.uniform(0, 1, (n_los, 2)).T) starts = list(ift.random.current_rng().random((n_los, 2)).T)
ends = list(np.random.uniform(0, 1, (n_los, 2)).T) ends = list(ift.random.current_rng().random((n_los, 2)).T)
return starts, ends return starts, ends
def radial_los(n_los): def radial_los(n_los):
starts = list(np.random.uniform(0, 1, (n_los, 2)).T) starts = list(ift.random.current_rng().random((n_los, 2)).T)
ends = list(0.5 + 0*np.random.uniform(0, 1, (n_los, 2)).T) ends = list(0.5 + 0*ift.random.current_rng().random((n_los, 2)).T)
return starts, ends return starts, ends
if __name__ == '__main__': if __name__ == '__main__':
np.random.seed(43)
# Choose between random line-of-sight response (mode=0) and radial lines # Choose between random line-of-sight response (mode=0) and radial lines
# of sight (mode=1) # of sight (mode=1)
if len(sys.argv) == 2: if len(sys.argv) == 2:
......
...@@ -19,7 +19,6 @@ import matplotlib.pyplot as plt ...@@ -19,7 +19,6 @@ import matplotlib.pyplot as plt
import numpy as np import numpy as np
import nifty6 as ift import nifty6 as ift
np.random.seed(12)
def polynomial(coefficients, sampling_points): def polynomial(coefficients, sampling_points):
...@@ -86,7 +85,7 @@ if __name__ == '__main__': ...@@ -86,7 +85,7 @@ if __name__ == '__main__':
N_params = 10 N_params = 10
N_samples = 100 N_samples = 100
size = (12,) size = (12,)
x = np.random.random(size) * 10 x = ift.random.current_rng().random(size) * 10
y = np.sin(x**2) * x**3 y = np.sin(x**2) * x**3
var = np.full_like(y, y.var() / 10) var = np.full_like(y, y.var() / 10)
var[-2] *= 4 var[-2] *= 4
......
from .version import __version__ from .version import __version__
from . import random
from .domains.domain import TMP_Space from .domains.domain import TMP_Space
from .domains.structured_domain import TMP_StructuredSpace from .domains.structured_domain import TMP_StructuredSpace
from .domains.unstructured_domain import TMP_UnstructuredSpace from .domains.unstructured_domain import TMP_UnstructuredSpace
......
...@@ -278,17 +278,21 @@ def check_jacobian_consistency(op, loc, tol=1e-8, ntries=100, perf_check=True): ...@@ -278,17 +278,21 @@ def check_jacobian_consistency(op, loc, tol=1e-8, ntries=100, perf_check=True):
dir = loc2-loc dir = loc2-loc
locnext = loc2 locnext = loc2
dirnorm = dir.norm() dirnorm = dir.norm()
hist = []
for i in range(50): for i in range(50):
locmid = loc + 0.5*dir locmid = loc + 0.5*dir
linmid = op(Linearization.make_var(locmid)) linmid = op(Linearization.make_var(locmid))
dirder = linmid.jac(dir) dirder = linmid.jac(dir)
numgrad = (lin2.val-lin.val) numgrad = (lin2.val-lin.val)
xtol = tol * dirder.norm() / np.sqrt(dirder.size) xtol = tol * dirder.norm() / np.sqrt(dirder.size)
hist.append((numgrad-dirder).norm())
# print(len(hist),hist[-1])
if (abs(numgrad-dirder) <= xtol).s_all(): if (abs(numgrad-dirder) <= xtol).s_all():
break break
dir = dir*0.5 dir = dir*0.5
dirnorm *= 0.5 dirnorm *= 0.5
loc2, lin2 = locmid, linmid loc2, lin2 = locmid, linmid
else: else:
print(hist)
raise ValueError("gradient and value seem inconsistent") raise ValueError("gradient and value seem inconsistent")
loc = locnext loc = locnext
...@@ -25,6 +25,7 @@ from ..field import TMP_fld ...@@ -25,6 +25,7 @@ from ..field import TMP_fld
from ..linearization import Linearization from ..linearization import Linearization
from ..operators.operator import Operator from ..operators.operator import Operator
from ..sugar import makeOp from ..sugar import makeOp
from .. import random
def _f_on_np(f, arr): def _f_on_np(f, arr):
...@@ -67,7 +68,8 @@ class _InterpolationOperator(Operator): ...@@ -67,7 +68,8 @@ class _InterpolationOperator(Operator):
if table_func is not None: if table_func is not None:
if inv_table_func is None: if inv_table_func is None:
raise ValueError raise ValueError
a = func(np.random.randn(10)) # MR FIXME: not sure whether we should have this in production code
a = func(random.current_rng().random(10))
a1 = _f_on_np(lambda x: inv_table_func(table_func(x)), a) a1 = _f_on_np(lambda x: inv_table_func(table_func(x)), a)
np.testing.assert_allclose(a, a1) np.testing.assert_allclose(a, a1)
self._table = _f_on_np(table_func, self._table) self._table = _f_on_np(table_func, self._table)
......
This diff is collapsed.
This diff is collapsed.
from .minimization.metric_gaussian_kl_mpi import MetricGaussianKL_MPI
...@@ -27,10 +27,6 @@ class Operator(metaclass=NiftyMeta): ...@@ -27,10 +27,6 @@ class Operator(metaclass=NiftyMeta):
domain, and can also provide the Jacobian. domain, and can also provide the Jacobian.
""" """
VALUE_ONLY = 0
WITH_JAC = 1
WITH_METRIC = 2
@property @property
def domain(self): def domain(self):
"""The domain on which the Operator's input TMP_fld is defined. """The domain on which the Operator's input TMP_fld is defined.
...@@ -202,7 +198,7 @@ class Operator(metaclass=NiftyMeta): ...@@ -202,7 +198,7 @@ class Operator(metaclass=NiftyMeta):
return self.apply(x.trivial_jac()).prepend_jac(x.jac) return self.apply(x.trivial_jac()).prepend_jac(x.jac)
elif isinstance(x, (TMP_fld, TMP_Field)): elif isinstance(x, (TMP_fld, TMP_Field)):
return self.apply(x) return self.apply(x)
raise TypeError('Operator can only consume TMP_fld, TMP_Fields and Linearizations') return self @ x
def ducktape(self, name): def ducktape(self, name):
from .simple_linear_operators import ducktape from .simple_linear_operators import ducktape
......
...@@ -11,21 +11,150 @@ ...@@ -11,21 +11,150 @@
# You should have received a copy of the GNU General Public License # You should have received a copy of the GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>. # along with this program. If not, see <http://www.gnu.org/licenses/>.
# #
# Copyright(C) 2013-2019 Max-Planck-Society # Copyright(C) 2013-2020 Max-Planck-Society
# #
# NIFTy is being developed at the Max-Planck-Institut fuer Astrophysik. # NIFTy is being developed at the Max-Planck-Institut fuer Astrophysik.
"""
Some remarks on NIFTy's treatment of random numbers
NIFTy makes use of the `Generator` and `SeedSequence` classes introduced to
`numpy.random` in numpy 1.17.
On first load of the `nifty6.random` module, it creates a stack of
`SeedSequence` objects which contains a single `SeedSequence` with a fixed seed,
and also a stack of `Generator` objects, which contains a single generator
derived from the above seed sequence. Without user intervention, this generator
will be used for all random number generation tasks within NIFTy. This means
- that random numbers drawn by NIFTy will be reproducible across multiple runs
(assuming there are no complications like MPI-enabled runs with a varying
number of tasks), and
- that trying to change random seeds via `numpy.random.seed` will have no
effect on the random numbers drawn by NIFTy.
Users who want to change the random seed for a given run can achieve this
by calling `push_sseq_from_seed()` with a seed of their choice. This will push
a new seed sequence generated from that seed onto the seed sequence stack, and a
generator derived from this seed sequence onto the generator stack. Since all
NIFTy RNG-related calls will use the generator on the top of the stack, all
calls from this point on will use the new generator.
If the user already has a `SeedSequence` object at hand, they can pass this to
NIFTy via `push_sseq`. A new generator derived from this sequence will then also
be pushed onto the generator stack.
These operations can be reverted (and should be, as soon as the new generator is
no longer needed) by a call to `pop_sseq()`.
When users need direct access to the RNG currently in use, they can access it
via the `current_rng` function.
Example for using multiple seed sequences:
Assume that N samples are needed to compute a KL, which are distributed over
a variable number of MPI tasks. In this situation, whenever random numbers
need to be drawn for these samples:
- each MPI task should spawn as many seed sequences as there are samples
_in total_, using `sseq = spawn_sseq(N)`
- each task loops over the local samples
- first pushing the seed sequence for the global(!) index of the
current sample via `push_sseq(sseq[iglob])`
- drawing the required random numbers
- then popping the seed sequence again via `pop_sseq()`
That way, random numbers should be reproducible and independent of the number
of MPI tasks.
WARNING: do not push/pop the same `SeedSequence` object more than once - this
will lead to repeated random sequences! Whenever you have to push `SeedSequence`
objects, generate new ones via `spawn_sseq()`.
"""
import numpy as np import numpy as np
# Stack of SeedSequence objects. Will always start out with a well-defined
# default. Users can change the "random seed" used by a calculation by pushing
# a different SeedSequence before invoking any other nifty6.random calls
_sseq = [np.random.SeedSequence(42)]
# Stack of random number generators associated with _sseq.
_rng = [np.random.default_rng(_sseq[-1])]
def spawn_sseq(n, parent=None):
"""Returns a list of `n` SeedSequence objects which are children of `parent`
Parameters
----------
n : int
number of requested SeedSequence objects
parent : SeedSequence
the object from which the returned objects will be derived
If `None`, the top of the current SeedSequence stack will be used
Returns
-------
list(SeedSequence)
the requested SeedSequence objects
"""
if parent is None:
global _sseq
parent = _sseq[-1]
return parent.spawn(n)
def current_rng():
"""Returns the RNG object currently in use by NIFTy
Returns
-------
Generator
the current Generator object (top of the generatir stack)
"""
return _rng[-1]
def push_sseq(sseq):
"""Pushes a new SeedSequence object onto the SeedSequence stack.
This also pushes a new Generator object built from the new SeedSequence
to the generator stack.
Parameters
----------
sseq: SeedSequence
the SeedSequence object to be used from this point
"""
_sseq.append(sseq)
_rng.append(np.random.default_rng(_sseq[-1]))
def push_sseq_from_seed(seed):
"""Pushes a new SeedSequence object derived from an integer seed onto the
SeedSequence stack.
This also pushes a new Generator object built from the new SeedSequence
to the generator stack.
Parameters
----------
seed: int
the seed from which the new SeedSequence will be built
"""
_sseq.append(np.random.SeedSequence(seed))
_rng.append(np.random.default_rng(_sseq[-1]))
def pop_sseq():
"""Pops the top of the SeedSequence and generator stacks."""
_sseq.pop()
_rng.pop()
class Random(object): class Random(object):
@staticmethod @staticmethod
def pm1(dtype, shape): def pm1(dtype, shape):
if np.issubdtype(dtype, np.complexfloating): if np.issubdtype(dtype, np.complexfloating):
x = np.array([1+0j, 0+1j, -1+0j, 0-1j], dtype=dtype) x = np.array([1+0j, 0+1j, -1+0j, 0-1j], dtype=dtype)
x = x[np.random.randint(4, size=shape)] x = x[_rng[-1].integers(0, 4, size=shape)]
else: else:
x = 2*np.random.randint(2, size=shape) - 1 x = 2*_rng[-1].integers(0, 2, size=shape)-1
return x.astype(dtype, copy=False) return x.astype(dtype, copy=False)
@staticmethod @staticmethod
...@@ -42,10 +171,10 @@ class Random(object): ...@@ -42,10 +171,10 @@ class Random(object):
raise TypeError("mean must not be complex for a real result field") raise TypeError("mean must not be complex for a real result field")
if np.issubdtype(dtype, np.complexfloating): if np.issubdtype(dtype, np.complexfloating):
x = np.empty(shape, dtype=dtype) x = np.empty(shape, dtype=dtype)
x.real = np.random.normal(mean.real, std*np.sqrt(0.5), shape) x.real = _rng[-1].normal(mean.real, std*np.sqrt(0.5), shape)
x.imag = np.random.normal(mean.imag, std*np.sqrt(0.5), shape) x.imag = _rng[-1].normal(mean.imag, std*np.sqrt(0.5), shape)
else: else:
x = np.random.normal(mean, std, shape).astype(dtype, copy=False) x = _rng[-1].normal(mean, std, shape).astype(dtype, copy=False)
return x return x
@staticmethod @staticmethod
...@@ -57,13 +186,13 @@ class Random(object): ...@@ -57,13 +186,13 @@ class Random(object):
raise TypeError("low and high must not be complex") raise TypeError("low and high must not be complex")
if np.issubdtype(dtype, np.complexfloating): if np.issubdtype(dtype, np.complexfloating):
x = np.empty(shape, dtype=dtype) x = np.empty(shape, dtype=dtype)
x.real = np.random.uniform(low, high, shape) x.real = _rng[-1].uniform(low, high, shape)
x.imag = np.random.uniform(low, high, shape) x.imag = _rng[-1].uniform(low, high, shape)
elif np.issubdtype(dtype, np.integer): elif np.issubdtype(dtype, np.integer):
if not (np.issubdtype(type(low), np.integer) and if not (np.issubdtype(type(low), np.integer) and
np.issubdtype(type(high), np.integer)): np.issubdtype(type(high), np.integer)):
raise TypeError("low and high must be integer") raise TypeError("low and high must be integer")
x = np.random.randint(low, high+1, shape) x = _rng[-1].integers(low, high+1, shape)
else: else:
x = np.random.uniform(low, high, shape) x = _rng[-1].uniform(low, high, shape)
return x.astype(dtype, copy=False) return x.astype(dtype, copy=False)
...@@ -39,8 +39,8 @@ setup(name="nifty6", ...@@ -39,8 +39,8 @@ setup(name="nifty6",
packages=find_packages(include=["nifty6", "nifty6.*"]), packages=find_packages(include=["nifty6", "nifty6.*"]),
zip_safe=True, zip_safe=True,
license="GPLv3", license="GPLv3",
setup_requires=['scipy>=1.4.1'], setup_requires=['scipy>=1.4.1', 'numpy>=1.17'],
install_requires=['scipy>=1.4.1'], install_requires=['scipy>=1.4.1', 'numpy>=1.17'],
python_requires='>=3.5', python_requires='>=3.5',
classifiers=[ classifiers=[
"Development Status :: 4 - Beta", "Development Status :: 4 - Beta",
......