From 80ff8fdcb5d655aed1b8f8dd59a4e21ed9ffb0a9 Mon Sep 17 00:00:00 2001
From: Theo Steininger <theo.steininger@ultimanet.de>
Date: Thu, 18 May 2017 09:13:30 +0200
Subject: [PATCH] Updated energy and minimization docstrings.
---
.gitignore | 102 +++++++++++--
nifty/energies/energy.py | 79 ++++++----
nifty/energies/line_energy.py | 57 ++++----
nifty/minimization/.DS_Store | Bin 6148 -> 0 bytes
nifty/minimization/conjugate_gradient.py | 76 +++++-----
nifty/minimization/descent_minimizer.py | 106 +++++++-------
nifty/minimization/relaxed_newton.py | 38 ++---
nifty/minimization/steepest_descent.py | 36 ++---
nifty/minimization/vl_bfgs.py | 175 +++++++++--------------
9 files changed, 363 insertions(+), 306 deletions(-)
delete mode 100644 nifty/minimization/.DS_Store
diff --git a/.gitignore b/.gitignore
index f1ff57973..f2515d0a7 100644
--- a/.gitignore
+++ b/.gitignore
@@ -1,18 +1,98 @@
-*.html
+# custom
+setup.cfg
+.idea
+.DS_Store
+
+# from https://github.com/github/gitignore/blob/master/Python.gitignore
+
+# Byte-compiled / optimized / DLL files
+__pycache__/
+*.py[cod]
+*$py.class
+
+# C extensions
*.c
*.o
*.so
-*.pyc
-*.log
+
+# Distribution / packaging
+.Python
+env/
+build/
+develop-eggs/
+dist/
+downloads/
+eggs/
+.eggs/
+lib/
+lib64/
+parts/
+sdist/
+var/
+*.egg-info/
+.installed.cfg
*.egg
-*.egg-info
-*~
-.git/
-.svn/
-.spyderproject
-.document
-build
+# PyInstaller
+# Usually these files are written by a python script from a template
+# before PyInstaller builds the exe, so as to inject date/other infos into it.
+*.manifest
+*.spec
-.idea
+# Installer logs
+pip-log.txt
+pip-delete-this-directory.txt
+
+# Unit test / coverage reports
+htmlcov/
+.tox/
.coverage
+.coverage.*
+.cache
+nosetests.xml
+coverage.xml
+*,cover
+.hypothesis/
+
+# Translations
+*.mo
+*.pot
+
+# Django stuff:
+*.log
+local_settings.py
+
+# Flask stuff:
+instance/
+.webassets-cache
+
+# Scrapy stuff:
+.scrapy
+
+# Sphinx documentation
+docs/_build/
+
+# PyBuilder
+target/
+
+# IPython Notebook
+.ipynb_checkpoints
+
+# pyenv
+.python-version
+
+# celery beat schedule file
+celerybeat-schedule
+
+# dotenv
+.env
+
+# virtualenv
+venv/
+ENV/
+
+# Spyder project settings
+.spyderproject
+
+# Rope project settings
+.ropeproject
\ No newline at end of file
diff --git a/nifty/energies/energy.py b/nifty/energies/energy.py
index 9e6024844..851f201e7 100644
--- a/nifty/energies/energy.py
+++ b/nifty/energies/energy.py
@@ -22,44 +22,44 @@ from keepers import Loggable
class Energy(Loggable, object):
- """ The Energy object provides the structure required for minimization schemes.
+ """ Provides the functional used by minimization schemes.
- The implementation of a scalar function with its gradient and curvature at some position.
+ The Energy object is an implementation of a scalar function including its
+ gradient and curvature at some position.
Parameters
----------
- position : Field, float
- The parameter of the scalar function and its first and second derivative.
+ position : Field
+ The input parameter of the scalar function.
Attributes
----------
- position : Field, float
- The Field location in parameter space where value, gradient and curvature is evaluated.
- value : float
- The evaluation of the energy functional at given position.
- gradient : Field, float
- The gradient at given position in parameter direction.
- curvature : callable
- A positive semi-definite operator or function describing the curvature of the potential
- at given position.
-
- Raises
- ------
- NotImplementedError
- Raised if
- * value, gradient or curvature is called
- AttributeError
- Raised if
- * copying of the position fails
+ position : Field
+ The Field location in parameter space where value, gradient and
+ curvature are evaluated.
+ value : np.float
+ The value of the energy functional at given `position`.
+ gradient : Field
+ The gradient at given `position` in parameter direction.
+ curvature : LinearOperator, callable
+ A positive semi-definite operator or function describing the curvature
+ of the potential at the given `position`.
Notes
-----
- The Energy object gives the blueprint how to formulate the model in order to apply
- various inference schemes. The functions value, gradient and curvature have to be
- implemented according to the concrete inference problem.
+ An instance of the Energy class is defined at a certain location. If one
+ is interested in the value, gradient or curvature of the abstract energy
+ functional one has to 'jump' to the new position using the `at` method.
+ This method returns a new energy instance residing at the new position. By
+ this approach, intermediate results from computing e.g. the gradient can
+ safely be reused for e.g. the value or the curvature.
- Memorizing the evaluations of some quantities minimizes the computational effort
- for multiple calls.
+ Memorizing the evaluations of some quantities (using the memo decorator)
+ minimizes the computational effort for multiple calls.
+
+ See also
+ --------
+ memo
"""
@@ -74,7 +74,7 @@ class Energy(Loggable, object):
self._position = position
def at(self, position):
- """ Initializes and returns new Energy object at new position.
+ """ Initializes and returns a new Energy object at the new position.
Parameters
----------
@@ -87,20 +87,43 @@ class Energy(Loggable, object):
Energy object at new position.
"""
+
return self.__class__(position)
@property
def position(self):
+ """
+ The Field location in parameter space where value, gradient and
+ curvature are evaluated.
+
+ """
+
return self._position
@property
def value(self):
+ """
+ The value of the energy functional at given `position`.
+
+ """
+
raise NotImplementedError
@property
def gradient(self):
+ """
+ The gradient at given `position` in parameter direction.
+
+ """
+
raise NotImplementedError
@property
def curvature(self):
+ """
+ A positive semi-definite operator or function describing the curvature
+ of the potential at the given `position`.
+
+ """
+
raise NotImplementedError
diff --git a/nifty/energies/line_energy.py b/nifty/energies/line_energy.py
index 408a69cd9..27f09399f 100644
--- a/nifty/energies/line_energy.py
+++ b/nifty/energies/line_energy.py
@@ -20,52 +20,58 @@ from .energy import Energy
class LineEnergy(Energy):
- """A Energy object restricting an underlying Energy along only some line direction.
- Given some Energy and line direction, its position is parametrized by a scalar
- step size along the descent direction.
+ """ Evaluates an underlying Energy along a certain line direction.
+
+ Given an Energy class and a line direction, its position is parametrized by
+ a scalar step size along the descent direction relative to a zero point.
Parameters
----------
position : float
The step length parameter along the given line direction.
energy : Energy
- The Energy object which will be restricted along the given line direction
- line_direction : Field, float
- Line direction restricting the Energy.
- zero_point : Field, float
- Fixing the zero point of the line restriction. Used to memorize this position in new
- initializations (default : None)
+ The Energy object which will be evaluated along the given direction.
+ line_direction : Field
+ Direction used for line evaluation.
+ zero_point : Field *optional*
+ Fixing the zero point of the line restriction. Used to memorize this
+ position in new initializations. By the default the current position
+ of the supplied `energy` instance is used (default : None).
Attributes
----------
- position : float
- The step length along the given line direction.
+ position : float
+ The position along the given line direction relative to the zero point.
value : float
- The evaluation of the energy functional at given position.
+ The value of the energy functional at given `position`.
gradient : float
- The gradient along the line direction projected on the current line position.
+ The gradient of the underlying energy instance along the line direction
+ projected on the line direction.
curvature : callable
- A positive semi-definite operator or function describing the curvature of the potential
- at given position.
+ A positive semi-definite operator or function describing the curvature
+ of the potential at given `position`.
line_direction : Field
- Direction along which the movement is restricted. Does not have to be normalized.
+ Direction along which the movement is restricted. Does not have to be
+ normalized.
energy : Energy
- The underlying Energy at the resulting position along the line according to the step length.
+ The underlying Energy at the `position` along the line direction.
Raises
------
NotImplementedError
Raised if
- * value, gradient or curvature of the attribute energy is not implemented.
+ * value, gradient or curvature of the attribute energy are not
+ implemented.
Notes
-----
- The LineEnergy is used in minimization schemes in order to determine the step size along
- some descent direction using a line search. It describes an underlying Energy which is restricted
- along one direction, only requiring the step size parameter to determine a new position.
-
+ The LineEnergy is used in minimization schemes in order perform line
+ searches. It describes an underlying Energy which is restricted along one
+ direction, only requiring the step size parameter to determine a new
+ position.
"""
+
def __init__(self, position, energy, line_direction, zero_point=None):
super(LineEnergy, self).__init__(position=position)
self.line_direction = line_direction
@@ -78,19 +84,20 @@ class LineEnergy(Energy):
self.energy = energy.at(position=position_on_line)
def at(self, position):
- """ Initializes and returns new LineEnergy object at new position, memorizing the zero point.
+ """ Returns LineEnergy at new position, memorizing the zero point.
Parameters
----------
position : float
- Parameter for the new position.
+ Parameter for the new position on the line direction.
Returns
-------
out : LineEnergy
- LineEnergy object at new position with same zero point.
+ LineEnergy object at new position with same zero point as `self`.
"""
+
return self.__class__(position,
self.energy,
self.line_direction,
diff --git a/nifty/minimization/.DS_Store b/nifty/minimization/.DS_Store
deleted file mode 100644
index f66870e1152c97241fa7e3f01e25c5e9ebb4ef7d..0000000000000000000000000000000000000000
GIT binary patch
literal 0
HcmV?d00001
literal 6148
zcmZQzU|@7AO)+F(5MW?n;9!8z3~dZp0Z1N%F(jFwB8(vOz-GiV6f>kUBr+5+Btzv0
zQ6Rm13^@#$40#Nx49NO3z+&lT!9{sF`FZIOI|vy!DnA+mqaiRF0;3@?8UmvsKp+G_
zbuXy?=O&<TRC+W7Mnhoag#f60P=L0(9iVgrga%15FfuTJy8w&~3@otFVFdRB7(j9$
ztsokt6-0xyGBAQzU^Bp485p5j8NuBUkUmhC1Vn?iGcYoMwKFg>fVDF)f^BACV1#IA
zV1(Mt2<@RTLbNk5LbNk5LbSu2H%gC&z-S0SLx34V3xMi>R|W=L{eOt6QF1f{Mnhm&
zh5#drOR$R*xKhUMKTur@s!tQ3Y7Rh^F@ox8h!{u-sA>mS$4rnxMG2@fkT^&?hz3{3
Vj0}+4e6%qD3!zbZGz91y0s!f~7ajlr
diff --git a/nifty/minimization/conjugate_gradient.py b/nifty/minimization/conjugate_gradient.py
index 833d01eaf..181e6d3c1 100644
--- a/nifty/minimization/conjugate_gradient.py
+++ b/nifty/minimization/conjugate_gradient.py
@@ -23,56 +23,58 @@ from keepers import Loggable
class ConjugateGradient(Loggable, object):
- """Implementation of the Conjugate Gradient scheme.
-
+ """ Implementation of the Conjugate Gradient scheme.
+
It is an iterative method for solving a linear system of equations:
Ax = b
-
- SUGGESTED LITERATURE:
- Thomas V. Mikosch et al., "Numerical Optimization", Second Edition,
- 2006, Springer-Verlag New York
-
+
Parameters
----------
- convergence_tolerance : scalar
- Tolerance specifying convergence. (default: 1E-4)
- convergence_level : integer
- Number of times the tolerance should be undershot before exiting.
- (default: 3)
+ convergence_tolerance : float *optional*
+ Tolerance specifying the case of convergence. (default: 1E-4)
+ convergence_level : integer *optional*
+ Number of times the tolerance must be undershot before convergence
+ is reached. (default: 3)
iteration_limit : integer *optional*
- Maximum number of iterations performed. (default: None)
- reset_count : integer, *optional*
+ Maximum number of iterations performed (default: None).
+ reset_count : integer *optional*
Number of iterations after which to restart; i.e., forget previous
- conjugated directions. (default: None)
- preconditioner : function *optional*
- The user can provide a function which transforms the variables of the
- system to make the converge more favorable.(default: None)
- callback : function, *optional*
- Function f(energy, iteration_number) specified by the user to print
- iteration number and energy value at every iteration step. It accepts
- an Energy object(energy) and integer(iteration_number). (default: None)
+ conjugated directions (default: None).
+ preconditioner : Operator *optional*
+ This operator can be provided which transforms the variables of the
+ system to improve the conditioning (default: None).
+ callback : callable *optional*
+ Function f(energy, iteration_number) supplied by the user to perform
+ in-situ analysis at every iteration step. When being called the
+ current energy and iteration_number are passed. (default: None)
Attributes
----------
convergence_tolerance : float
- Tolerance specifying convergence.
- convergence_level : float
- Number of times the tolerance should be undershot before exiting.
+ Tolerance specifying the case of convergence.
+ convergence_level : integer
+ Number of times the tolerance must be undershot before convergence
+ is reached. (default: 3)
iteration_limit : integer
Maximum number of iterations performed.
reset_count : integer
Number of iterations after which to restart; i.e., forget previous
conjugated directions.
preconditioner : function
- The user can provide a function which transforms the variables of the
- system to make the converge more favorable.
- callback : function
- Function f(energy, iteration_number) specified by the user to print
- iteration number and energy value at every iteration step. It accepts
- an Energy object(energy) and integer(iteration_number).
-
- """
-
+ This operator can be provided which transforms the variables of the
+ system to improve the conditioning (default: None).
+ callback : callable
+ Function f(energy, iteration_number) supplied by the user to perform
+ in-situ analysis at every iteration step. When being called the
+ current energy and iteration_number are passed. (default: None)
+
+ References
+ ----------
+ Thomas V. Mikosch et al., "Numerical Optimization", Second Edition,
+ 2006, Springer-Verlag New York
+
+ """
+
def __init__(self, convergence_tolerance=1E-4, convergence_level=3,
iteration_limit=None, reset_count=None,
preconditioner=None, callback=None):
@@ -95,14 +97,15 @@ class ConjugateGradient(Loggable, object):
self.callback = callback
def __call__(self, A, b, x0):
- """Runs the conjugate gradient minimization.
+ """ Runs the conjugate gradient minimization.
+ For `Ax = b` the variable `x` is infered.
Parameters
----------
A : Operator
Operator `A` applicable to a Field.
b : Field
- Resulting Field of the operation `A(x)`.
+ Result of the operation `A(x)`.
x0 : Field
Starting guess for the minimization.
@@ -115,6 +118,7 @@ class ConjugateGradient(Loggable, object):
has converged or not.
"""
+
r = b - A(x0)
d = self.preconditioner(r)
previous_gamma = r.dot(d)
diff --git a/nifty/minimization/descent_minimizer.py b/nifty/minimization/descent_minimizer.py
index d8996afbb..9e32e524e 100644
--- a/nifty/minimization/descent_minimizer.py
+++ b/nifty/minimization/descent_minimizer.py
@@ -27,53 +27,56 @@ from .line_searching import LineSearchStrongWolfe
class DescentMinimizer(Loggable, object):
- """A class used by other minimization methods to find a local minimum.
-
- Descent minimization methods are used to find a local minimum of a scalar function
- by following a descent direction. This class implements the minimization procedure,
- the descent direction has to be implemented separately.
-
+ """ A base class used by gradient methods to find a local minimum.
+
+ Descent minimization methods are used to find a local minimum of a scalar
+ function by following a descent direction. This class implements the
+ minimization procedure once a descent direction is known. The descent
+ direction has to be implemented separately.
+
Parameters
----------
- line_searcher : callable
- Function which finds the step size in descent direction. (default:
- LineSearchStrongWolfe())
- callback : function, *optional*
- Function f(energy, iteration_number) specified by the user to print
- iteration number and energy value at every iteration step. It accepts
- an Energy object(energy) and integer(iteration_number). (default: None)
- convergence_tolerance : scalar
- Tolerance specifying convergence. (default: 1E-4)
- convergence_level : integer
- Number of times the tolerance should be undershot before
- exiting. (default: 3)
+ line_searcher : callable *optional*
+ Function which infers the step size in the descent direction
+ (default : LineSearchStrongWolfe()).
+ callback : callable *optional*
+ Function f(energy, iteration_number) supplied by the user to perform
+ in-situ analysis at every iteration step. When being called the
+ current energy and iteration_number are passed. (default: None)
+ convergence_tolerance : float *optional*
+ Tolerance specifying the case of convergence. (default: 1E-4)
+ convergence_level : integer *optional*
+ Number of times the tolerance must be undershot before convergence
+ is reached. (default: 3)
iteration_limit : integer *optional*
- Maximum number of iterations performed. (default: None)
-
+ Maximum number of iterations performed (default: None).
+
Attributes
----------
convergence_tolerance : float
- Tolerance specifying convergence.
- convergence_level : float
- Number of times the tolerance should be undershot before
- exiting.
+ Tolerance specifying the case of convergence.
+ convergence_level : integer
+ Number of times the tolerance must be undershot before convergence
+ is reached. (default: 3)
iteration_limit : integer
Maximum number of iterations performed.
- line_searcher : callable
- Function which finds the step size into the descent direction
+ line_searcher : LineSearch
+ Function which infers the optimal step size for functional minization
+ given a descent direction.
callback : function
- Function f(energy, iteration_number) specified by the user to print
- iteration number and energy value at every iteration step. It accepts
- an Energy object(energy) and integer(iteration_number).
-
- Raises
+ Function f(energy, iteration_number) supplied by the user to perform
+ in-situ analysis at every iteration step. When being called the
+ current energy and iteration_number are passed.
+
+ Notes
------
- StopIteration
- Raised if
- *callback function does not match the specified form.
+ The callback function can be used to externally stop the minimization by
+ raising a `StopIteration` exception.
+ Check `get_descent_direction` of a derived class for information on the
+ concrete minization scheme.
+
+ """
- """
-
__metaclass__ = NiftyMeta
def __init__(self, line_searcher=LineSearchStrongWolfe(), callback=None,
@@ -81,7 +84,7 @@ class DescentMinimizer(Loggable, object):
iteration_limit=None):
self.convergence_tolerance = np.float(convergence_tolerance)
- self.convergence_level = np.float(convergence_level)
+ self.convergence_level = np.int(convergence_level)
if iteration_limit is not None:
iteration_limit = int(iteration_limit)
@@ -91,16 +94,13 @@ class DescentMinimizer(Loggable, object):
self.callback = callback
def __call__(self, energy):
- """Runs the minimization on the provided Energy class.
+ """ Performs the minimization of the provided Energy functional.
- Accepts the NIFTY Energy class which describes our system and it runs
- the minimization to find the minimum of the system.
-
Parameters
----------
energy : Energy object
- Energy object provided by the user from which we can calculate the
- energy, gradient and curvature at a specific point.
+ Energy object which provides value, gradient and curvature at a
+ specific position in parameter space.
Returns
-------
@@ -109,16 +109,16 @@ class DescentMinimizer(Loggable, object):
convergence : integer
Latest convergence level indicating whether the minimization
has converged or not.
-
+
Note
----
- It stops the minimization if:
- *callback function does not match the specified form.
- *a perfectly flat point is reached.
- *according to line-search the minimum is found.
- *target convergence level is reached.
- *iteration limit is reached.
-
+ The minimization is stopped if
+ * the callback function raises a `StopIteration` exception,
+ * a perfectly flat point is reached,
+ * according to the line-search the minimum is found,
+ * the target convergence level is reached,
+ * the iteration limit is reached.
+
"""
convergence = 0
@@ -146,7 +146,7 @@ class DescentMinimizer(Loggable, object):
break
# current position is encoded in energy object
- descend_direction = self._get_descend_direction(energy)
+ descend_direction = self.get_descend_direction(energy)
# compute the step length, which minimizes energy.value along the
# search direction
@@ -188,5 +188,5 @@ class DescentMinimizer(Loggable, object):
return energy, convergence
@abc.abstractmethod
- def _get_descend_direction(self, energy):
+ def get_descend_direction(self, energy):
raise NotImplementedError
diff --git a/nifty/minimization/relaxed_newton.py b/nifty/minimization/relaxed_newton.py
index cd573f34c..eeb1650c8 100644
--- a/nifty/minimization/relaxed_newton.py
+++ b/nifty/minimization/relaxed_newton.py
@@ -21,26 +21,7 @@ from .line_searching import LineSearchStrongWolfe
class RelaxedNewton(DescentMinimizer):
- """ A implementation of the relaxed Newton minimization scheme.
- The relaxed Newton minimization exploits gradient and curvature information to
- propose a step. A linesearch optimizes along this direction.
- Parameter
- ---------
- line_searcher : LineSearch,
- An implementation of a line-search algorithm.
- callback : function, *optional*
- Function f(energy, iteration_number) specified by the user to print
- iteration number and energy value at every iteration step. It accepts
- an Energy object(energy) and integer(iteration_number). (default: None)
- convergence_tolerance : float,
- Specifies the required accuracy for convergence. (default : 10e-4)
- convergence_level : integer
- Specifies the demanded level of convergence. (default : 3)
- iteration_limit : integer
- Limiting the maximum number of steps. (default : None)
-
- """
def __init__(self, line_searcher=LineSearchStrongWolfe(), callback=None,
convergence_tolerance=1E-4, convergence_level=3,
iteration_limit=None):
@@ -53,27 +34,28 @@ class RelaxedNewton(DescentMinimizer):
self.line_searcher.prefered_initial_step_size = 1.
- def _get_descend_direction(self, energy):
+ def get_descend_direction(self, energy):
""" Calculates the descent direction according to a Newton scheme.
+
The descent direction is determined by weighting the gradient at the
- current parameter position with the inverse local curvature, provided by the
- Energy object.
+ current parameter position with the inverse local curvature, provided
+ by the Energy object.
+
Parameters
----------
energy : Energy
- The energy object providing implementations of the to be minimized function,
- its gradient and curvature.
+ An instance of the Energy class which shall be minized. The
+ position of `energy` is used as the starting point of minization.
Returns
-------
- out : Field
- Returns the descent direction with proposed step length. In a quadratic
- potential this corresponds to the optimal step.
+ descend_direction : Field
+ Returns the descent direction with proposed step length. In a
+ quadratic potential this corresponds to the optimal step.
"""
gradient = energy.gradient
curvature = energy.curvature
descend_direction = curvature.inverse_times(gradient)
return descend_direction * -1
-# return descend_direction * -1
diff --git a/nifty/minimization/steepest_descent.py b/nifty/minimization/steepest_descent.py
index b853c4bf0..8d2311935 100644
--- a/nifty/minimization/steepest_descent.py
+++ b/nifty/minimization/steepest_descent.py
@@ -20,23 +20,25 @@ from .descent_minimizer import DescentMinimizer
class SteepestDescent(DescentMinimizer):
- """Implementation of the steepest descent minimization scheme.
-
- It uses the gradient of the minimized function to get to the minimum.
-
- Parameters
- ----------
- energy : Energy object
- The energy object providing implementations of the to be minimized
- function and gradient.
-
- Returns
- -------
- descend_direction : Field
- Returns the descent direction.
-
- """
- def _get_descend_direction(self, energy):
+ def get_descend_direction(self, energy):
+ """ Implementation of the steepest descent minimization scheme.
+
+ Also known as 'gradient descent'. This algorithm simply follows the
+ functionals gradient for minization.
+
+ Parameters
+ ----------
+ energy : Energy
+ An instance of the Energy class which shall be minized. The
+ position of `energy` is used as the starting point of minization.
+
+ Returns
+ -------
+ descend_direction : Field
+ Returns the descent direction.
+
+ """
+
descend_direction = energy.gradient
norm = descend_direction.norm()
if norm != 1:
diff --git a/nifty/minimization/vl_bfgs.py b/nifty/minimization/vl_bfgs.py
index ab74971b7..5eae3af90 100644
--- a/nifty/minimization/vl_bfgs.py
+++ b/nifty/minimization/vl_bfgs.py
@@ -23,41 +23,6 @@ from .line_searching import LineSearchStrongWolfe
class VL_BFGS(DescentMinimizer):
- """Implementation of the Vector-free L-BFGS minimization scheme.
-
- Find the descent direction by using the inverse Hessian.
- Instead of storing the whole matrix, it stores only the last few updates,
- which are used to do operations requiring the inverse Hessian product. The
- updates are represented in a new basis to optimize the algorithm.
- LITERATURE:
- W. Chen, Z. Wang, J. Zhou, Large-scale L-BFGS using MapReduce, 2014,
- Microsoft
-
- Parameters
- ----------
- line_searcher : callable
- Function which finds the step size in descent direction. (default:
- LineSearchStrongWolfe())
- callback : function *optional*
- Function f(energy, iteration_number) specified by the user to print
- iteration number and energy value at every iteration step. It accepts
- an Energy object(energy) and integer(iteration_number). (default: None)
- convergence_tolerance : scalar
- Tolerance specifying convergence. (default: 1E-4)
- convergence_level : integer
- Number of times the tolerance should be undershot before exiting.
- (default: 3)
- iteration_limit : integer *optional*
- Maximum number of iterations performed. (default: None)
- max_history_length : integer
- Maximum number of stored past updates. (default: 10)
-
- Attributes
- ----------
- max_history_length : integer
- Maximum number of stored past updates.
-
- """
def __init__(self, line_searcher=LineSearchStrongWolfe(), callback=None,
convergence_tolerance=1E-4, convergence_level=3,
iteration_limit=None, max_history_length=10):
@@ -72,42 +37,36 @@ class VL_BFGS(DescentMinimizer):
self.max_history_length = max_history_length
def __call__(self, energy):
- """Runs the inherited __call__ method from QuasiNewtonMinimizer.
-
- Parameters
- ----------
- energy : Energy object
- Energy object which describes our system.
-
- Returns
- -------
- energy : Energy object
- Latest `energy` of the minimization.
- convergence : integer
- Latest convergence level indicating whether the minimization
- has converged or not.
-
- """
self._information_store = None
return super(VL_BFGS, self).__call__(energy)
- def _get_descend_direction(self, energy):
- """Initializes the storage of updates and gets descent direction.
-
- Gets the new basis vectors b and sums them to get the descend
- direction.
-
+ def get_descend_direction(self, energy):
+ """Implementation of the Vector-free L-BFGS minimization scheme.
+
+ Find the descent direction by using the inverse Hessian.
+ Instead of storing the whole matrix, it stores only the last few
+ updates, which are used to do operations requiring the inverse
+ Hessian product. The updates are represented in a new basis to optimize
+ the algorithm.
+
Parameters
----------
- energy : Energy object
- Energy object which describes our system.
-
+ energy : Energy
+ An instance of the Energy class which shall be minized. The
+ position of `energy` is used as the starting point of minization.
+
Returns
-------
descend_direction : Field
- Descend direction.
-
+ Returns the descent direction.
+
+ References
+ ----------
+ W. Chen, Z. Wang, J. Zhou, "Large-scale L-BFGS using MapReduce", 2014,
+ Microsoft
+
"""
+
x = energy.position
gradient = energy.gradient
# initialize the information store if it doesn't already exist
@@ -133,7 +92,7 @@ class VL_BFGS(DescentMinimizer):
class InformationStore(object):
"""Class for storing a list of past updates.
-
+
Parameters
----------
max_history_length : integer
@@ -142,7 +101,7 @@ class InformationStore(object):
Initial position in variable space.
gradient : Field
Gradient at position x0.
-
+
Attributes
----------
max_history_length : integer
@@ -163,7 +122,7 @@ class InformationStore(object):
Dictionary of scalar products between elements of s and y.
_yy_store : dictionary
Dictionary of scalar products between different elements of y.
-
+
"""
def __init__(self, max_history_length, x0, gradient):
self.max_history_length = max_history_length
@@ -180,19 +139,19 @@ class InformationStore(object):
@property
def history_length(self):
"""Returns the number of currently stored updates.
-
+
"""
return min(self.k, self.max_history_length)
@property
def b(self):
- """Combines s, y and gradient to form the new base vectors b.
-
+ """Combines s, y and gradient to form the new base vectors b.
+
Returns
-------
result : List
List of new basis vectors.
-
+
"""
result = []
m = self.history_length
@@ -213,15 +172,15 @@ class InformationStore(object):
@property
def b_dot_b(self):
"""Generates the (2m+1) * (2m+1) scalar matrix.
-
+
The i,j-th element of the matrix is a scalar product between the i-th
- and j-th base vector.
-
+ and j-th base vector.
+
Returns
-------
- result : numpy.ndarray
+ result : numpy.ndarray
Scalar matrix.
-
+
"""
m = self.history_length
k = self.k
@@ -252,12 +211,12 @@ class InformationStore(object):
@property
def delta(self):
"""Calculates the new scalar coefficients (deltas).
-
+
Returns
-------
delta : List
List of the new scalar coefficients (deltas).
-
+
"""
m = self.history_length
b_dot_b = self.b_dot_b
@@ -284,21 +243,21 @@ class InformationStore(object):
def ss_store(self, i, j):
"""Updates the dictionary _ss_store with a new scalar product.
-
- Returns the scalar product of s_i and s_j.
-
+
+ Returns the scalar product of s_i and s_j.
+
Parameters
----------
i : integer
s index.
j : integer
s index.
-
+
Returns
-------
_ss_store[key] : float
Scalar product of s_i and s_j.
-
+
"""
key = tuple(sorted((i, j)))
if key not in self._ss_store:
@@ -307,21 +266,21 @@ class InformationStore(object):
def sy_store(self, i, j):
"""Updates the dictionary _sy_store with a new scalar product.
-
- Returns the scalar product of s_i and y_j.
-
+
+ Returns the scalar product of s_i and y_j.
+
Parameters
----------
i : integer
s index.
j : integer
y index.
-
+
Returns
-------
_sy_store[key] : float
Scalar product of s_i and y_j.
-
+
"""
key = (i, j)
if key not in self._sy_store:
@@ -330,21 +289,21 @@ class InformationStore(object):
def yy_store(self, i, j):
"""Updates the dictionary _yy_store with a new scalar product.
-
- Returns the scalar product of y_i and y_j.
-
+
+ Returns the scalar product of y_i and y_j.
+
Parameters
----------
i : integer
y index.
j : integer
y index.
-
+
Returns
------
_yy_store[key] : float
Scalar product of y_i and y_j.
-
+
"""
key = tuple(sorted((i, j)))
if key not in self._yy_store:
@@ -353,43 +312,43 @@ class InformationStore(object):
def sgrad_store(self, i):
"""Returns scalar product between s_i and gradient on initial position.
-
+
Returns
-------
scalar product : float
Scalar product.
-
+
"""
return self.s[i].dot(self.last_gradient)
def ygrad_store(self, i):
"""Returns scalar product between y_i and gradient on initial position.
-
+
Returns
-------
scalar product : float
Scalar product.
-
+
"""
return self.y[i].dot(self.last_gradient)
def gradgrad_store(self):
"""Returns scalar product of gradient on initial position with itself.
-
+
Returns
-------
scalar product : float
Scalar product.
-
+
"""
return self.last_gradient.dot(self.last_gradient)
def add_new_point(self, x, gradient):
"""Updates the s list and y list.
- Calculates the new position and gradient differences and adds them to
- the respective list.
-
+ Calculates the new position and gradient differences and adds them to
+ the respective list.
+
"""
self.k += 1
@@ -405,12 +364,12 @@ class InformationStore(object):
class LimitedList(object):
"""Class for creating a list of limited length.
-
+
Parameters
----------
history_length : integer
Maximum number of stored past updates.
-
+
Attributes
----------
history_length : integer
@@ -420,7 +379,7 @@ class LimitedList(object):
length.
_storage : list
List where input values are stored.
-
+
"""
def __init__(self, history_length):
self.history_length = int(history_length)
@@ -429,12 +388,12 @@ class LimitedList(object):
def __getitem__(self, index):
"""Returns the element with index [index-offset].
-
+
Parameters
----------
index : integer
Index of the selected element.
-
+
Returns
-------
selected element
@@ -443,15 +402,15 @@ class LimitedList(object):
def add(self, value):
"""Adds a new element to the list.
-
+
If the list is of length maximum history then it removes the first
element first.
-
+
Parameters
----------
value : anything
New element in the list.
-
+
"""
if len(self._storage) == self.history_length:
self._storage.pop(0)
--
GitLab