Skip to content
GitLab
Menu
Projects
Groups
Snippets
Help
Help
Support
Community forum
Keyboard shortcuts
?
Submit feedback
Contribute to GitLab
Sign in
Toggle navigation
Menu
Open sidebar
ift
NIFTy
Commits
eb3a20e3
Commit
eb3a20e3
authored
Feb 16, 2018
by
Martin Reinecke
Browse files
more docs
parent
59a5251e
Pipeline
#25039
passed with stages
in 6 minutes and 42 seconds
Changes
9
Pipelines
1
Hide whitespace changes
Inline
Side-by-side
docs/source/concepts/minimization.rst
View file @
eb3a20e3
...
...
@@ -7,18 +7,25 @@ Description of Minimization
:maxdepth: 1
:caption: Minimization Concepts
Conjugate Gradient <../mod/nifty4.minimization.conjugate_gradient>
Descent Minimizer <../mod/nifty4.minimization.descent_minimizer>
Energy <../mod/nifty4.minimization.energy>
Iteration Controller <../mod/nifty4.minimization.iteration_controller>
Line Energy <../mod/nifty4.minimization.line_energy>
Line Search <../mod/nifty4.minimization.line_search>
Line Search Strong Wolfe <../mod/nifty4.minimization.line_search_strong_wolfe>
QuadraticEnergy <../mod/nifty4.minimization.quadratic_energy>
IterationController <../mod/nifty4.minimization.iteration_controller>
GradientNormController <../mod/nifty4.minimization.gradient_norm_controller>
Minimizer <../mod/nifty4.minimization.minimizer>
Nonlinear Cg <../mod/nifty4.minimization.nonlinear_cg>
Quadratic Energy <../mod/nifty4.minimization.quadratic_energy>
Relaxed Newton <../mod/nifty4.minimization.relaxed_newton>
Scipy Minimizer <../mod/nifty4.minimization.scipy_minimizer>
Steepest Descent <../mod/nifty4.minimization.steepest_descent>
VL BFGS <../mod/nifty4.minimization.vl_bfgs>
Gradient Norm Controller <../mod/nifty4.minimization.gradient_norm_controller>
ConjugateGradient <../mod/nifty4.minimization.conjugate_gradient>
NonlinearCG <../mod/nifty4.minimization.nonlinear_cg>
ScipyMinimizer <../mod/nifty4.minimization.scipy_minimizer>
LineEnergy <../mod/nifty4.minimization.line_energy>
LineSearch <../mod/nifty4.minimization.line_search>
LineSearchStrongWolfe <../mod/nifty4.minimization.line_search_strong_wolfe>
DescentMinimizer <../mod/nifty4.minimization.descent_minimizer>
SteepestDescent <../mod/nifty4.minimization.steepest_descent>
RelaxedNewton <../mod/nifty4.minimization.relaxed_newton>
VL_BFGS <../mod/nifty4.minimization.vl_bfgs>
nifty4/minimization/descent_minimizer.py
View file @
eb3a20e3
...
...
@@ -11,7 +11,7 @@
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
#
# Copyright(C) 2013-201
7
Max-Planck-Society
# Copyright(C) 2013-201
8
Max-Planck-Society
#
# NIFTy is being developed at the Max-Planck-Institut fuer Astrophysik
# and financially supported by the Studienstiftung des deutschen Volkes.
...
...
@@ -50,7 +50,7 @@ class DescentMinimizer(Minimizer):
Parameters
----------
nergy : Energy
e
nergy : Energy
Energy object which provides value, gradient and curvature at a
specific position in parameter space.
...
...
@@ -105,4 +105,17 @@ class DescentMinimizer(Minimizer):
@
abc
.
abstractmethod
def
get_descent_direction
(
self
,
energy
):
""" Calculates the next descent direction.
Parameters
----------
energy : Energy
An instance of the Energy class which shall be minimized. The
position of `energy` is used as the starting point of minimization.
Returns
-------
Field
The descent direction.
"""
raise
NotImplementedError
nifty4/minimization/energy.py
View file @
eb3a20e3
...
...
@@ -11,7 +11,7 @@
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
#
# Copyright(C) 2013-201
7
Max-Planck-Society
# Copyright(C) 2013-201
8
Max-Planck-Society
#
# NIFTy is being developed at the Max-Planck-Institut fuer Astrophysik
# and financially supported by the Studienstiftung des deutschen Volkes.
...
...
@@ -31,19 +31,6 @@ class Energy(with_metaclass(NiftyMeta, type('NewBase', (object,), {}))):
position : Field
The input parameter of the scalar function.
Attributes
----------
position : Field
The Field location in parameter space where value, gradient and
curvature are evaluated.
value : np.float
The value of the energy functional at given `position`.
gradient : Field
The gradient at given `position`.
curvature : LinearOperator, callable
A positive semi-definite operator or function describing the curvature
of the potential at the given `position`.
Notes
-----
An instance of the Energy class is defined at a certain location. If one
...
...
@@ -76,7 +63,7 @@ class Energy(with_metaclass(NiftyMeta, type('NewBase', (object,), {}))):
Returns
-------
out :
Energy
Energy
Energy object at new position.
"""
return
self
.
__class__
(
position
)
...
...
@@ -92,14 +79,16 @@ class Energy(with_metaclass(NiftyMeta, type('NewBase', (object,), {}))):
@
property
def
value
(
self
):
"""
The value of the energy functional at given `position`.
float
The value of the energy functional at given `position`.
"""
raise
NotImplementedError
@
property
def
gradient
(
self
):
"""
The gradient at given `position`.
Field
The gradient at given `position`.
"""
raise
NotImplementedError
...
...
@@ -107,14 +96,16 @@ class Energy(with_metaclass(NiftyMeta, type('NewBase', (object,), {}))):
@
memo
def
gradient_norm
(
self
):
"""
The length of the gradient at given `position`.
float
The length of the gradient at given `position`.
"""
return
self
.
gradient
.
norm
()
@
property
def
curvature
(
self
):
"""
A positive semi-definite operator or function describing the curvature
of the potential at the given `position`.
LinearOperator
A positive semi-definite operator or function describing the
curvature of the potential at the given `position`.
"""
raise
NotImplementedError
nifty4/minimization/line_energy.py
View file @
eb3a20e3
...
...
@@ -11,7 +11,7 @@
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
#
# Copyright(C) 2013-201
7
Max-Planck-Society
# Copyright(C) 2013-201
8
Max-Planck-Society
#
# NIFTy is being developed at the Max-Planck-Institut fuer Astrophysik
# and financially supported by the Studienstiftung des deutschen Volkes.
...
...
@@ -37,34 +37,12 @@ class LineEnergy(object):
energy.position = zero_point + offset*line_direction
(default : 0.).
Attributes
----------
line_position : float
The position along the given line direction relative to the zero point.
value : float
The value of the energy functional at the given position
directional_derivative : float
The directional derivative at the given position
line_direction : Field
Direction along which the movement is restricted. Does not have to be
normalized.
energy : Energy
The underlying Energy at the given position
Raises
------
NotImplementedError
Raised if
* value, gradient or curvature of the attribute energy are not
implemented.
Notes
-----
The LineEnergy is used in minimization schemes in order perform line
searches. It describes an underlying Energy which is restricted along one
direction, only requiring the step size parameter to determine a new
position.
"""
def
__init__
(
self
,
line_position
,
energy
,
line_direction
,
offset
=
0.
):
...
...
@@ -73,11 +51,11 @@ class LineEnergy(object):
self
.
_line_direction
=
line_direction
if
self
.
_line_position
==
float
(
offset
):
self
.
energy
=
energy
self
.
_
energy
=
energy
else
:
pos
=
energy
.
position
\
+
(
self
.
_line_position
-
float
(
offset
))
*
self
.
_line_direction
self
.
energy
=
energy
.
at
(
position
=
pos
)
self
.
_
energy
=
energy
.
at
(
position
=
pos
)
def
at
(
self
,
line_position
):
""" Returns LineEnergy at new position, memorizing the zero point.
...
...
@@ -93,24 +71,32 @@ class LineEnergy(object):
"""
return
LineEnergy
(
line_position
,
self
.
energy
,
self
.
line_direction
,
offset
=
self
.
line_position
)
return
LineEnergy
(
line_position
,
self
.
_
energy
,
self
.
_
line_direction
,
offset
=
self
.
_
line_position
)
@
property
def
value
(
self
):
return
self
.
energy
.
value
@
property
def
line_position
(
self
):
return
self
.
_
line_position
def
energy
(
self
):
"""
Energy
The underlying Energy object
"""
return
self
.
_
energy
@
property
def
line_direction
(
self
):
return
self
.
_line_direction
def
value
(
self
):
"""
float
The value of the energy functional at given `position`.
"""
return
self
.
_energy
.
value
@
property
def
directional_derivative
(
self
):
res
=
self
.
energy
.
gradient
.
vdot
(
self
.
line_direction
)
"""
float
The directional derivative at the given `position`.
"""
res
=
self
.
_energy
.
gradient
.
vdot
(
self
.
_line_direction
)
if
abs
(
res
.
imag
)
/
max
(
abs
(
res
.
real
),
1.
)
>
1e-12
:
from
..dobj
import
mprint
mprint
(
"directional derivative has non-negligible "
...
...
nifty4/minimization/minimizer.py
View file @
eb3a20e3
...
...
@@ -11,7 +11,7 @@
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
#
# Copyright(C) 2013-201
7
Max-Planck-Society
# Copyright(C) 2013-201
8
Max-Planck-Society
#
# NIFTy is being developed at the Max-Planck-Institut fuer Astrophysik
# and financially supported by the Studienstiftung des deutschen Volkes.
...
...
@@ -30,7 +30,7 @@ class Minimizer(with_metaclass(NiftyMeta, type('NewBase', (object,), {}))):
Parameters
----------
energy : Energy
object
energy : Energy
Energy object which provides value, gradient and curvature at a
specific position in parameter space.
...
...
nifty4/minimization/relaxed_newton.py
View file @
eb3a20e3
...
...
@@ -11,7 +11,7 @@
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
#
# Copyright(C) 2013-201
7
Max-Planck-Society
# Copyright(C) 2013-201
8
Max-Planck-Society
#
# NIFTy is being developed at the Max-Planck-Institut fuer Astrophysik
# and financially supported by the Studienstiftung des deutschen Volkes.
...
...
@@ -21,6 +21,11 @@ from .line_search_strong_wolfe import LineSearchStrongWolfe
class
RelaxedNewton
(
DescentMinimizer
):
""" Calculates the descent direction according to a Newton scheme.
The descent direction is determined by weighting the gradient at the
current parameter position with the inverse local curvature.
"""
def
__init__
(
self
,
controller
,
line_searcher
=
LineSearchStrongWolfe
()):
super
(
RelaxedNewton
,
self
).
__init__
(
controller
=
controller
,
line_searcher
=
line_searcher
)
...
...
@@ -28,23 +33,4 @@ class RelaxedNewton(DescentMinimizer):
self
.
line_searcher
.
preferred_initial_step_size
=
1.
def
get_descent_direction
(
self
,
energy
):
""" Calculates the descent direction according to a Newton scheme.
The descent direction is determined by weighting the gradient at the
current parameter position with the inverse local curvature, provided
by the Energy object.
Parameters
----------
energy : Energy
An instance of the Energy class which shall be minized. The
position of `energy` is used as the starting point of minization.
Returns
-------
descent_direction : Field
Returns the descent direction with proposed step length. In a
quadratic potential this corresponds to the optimal step.
"""
return
-
energy
.
curvature
.
inverse_times
(
energy
.
gradient
)
nifty4/minimization/scipy_minimizer.py
View file @
eb3a20e3
...
...
@@ -11,7 +11,7 @@
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
#
# Copyright(C) 2013-201
7
Max-Planck-Society
# Copyright(C) 2013-201
8
Max-Planck-Society
#
# NIFTy is being developed at the Max-Planck-Institut fuer Astrophysik
# and financially supported by the Studienstiftung des deutschen Volkes.
...
...
nifty4/minimization/steepest_descent.py
View file @
eb3a20e3
...
...
@@ -12,7 +12,7 @@ from __future__ import division
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
#
# Copyright(C) 2013-201
7
Max-Planck-Society
# Copyright(C) 2013-201
8
Max-Planck-Society
#
# NIFTy is being developed at the Max-Planck-Institut fuer Astrophysik
# and financially supported by the Studienstiftung des deutschen Volkes.
...
...
@@ -22,21 +22,10 @@ from .descent_minimizer import DescentMinimizer
class
SteepestDescent
(
DescentMinimizer
):
def
get_descent_direction
(
self
,
energy
):
""" Implementation of the steepest descent minimization scheme.
Also known as 'gradient descent'. This algorithm simply follows the
functional's gradient for minization.
""" Implementation of the steepest descent minimization scheme.
Parameters
----------
energy : Energy
An instance of the Energy class which shall be minized. The
position of `energy` is used as the starting point of minization.
Returns
-------
descent_direction : Field
the descent direction.
"""
Also known as 'gradient descent'. This algorithm simply follows the
functional's gradient for minimization.
"""
def
get_descent_direction
(
self
,
energy
):
return
-
energy
.
gradient
nifty4/minimization/vl_bfgs.py
View file @
eb3a20e3
...
...
@@ -11,7 +11,7 @@
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
#
# Copyright(C) 2013-201
7
Max-Planck-Society
# Copyright(C) 2013-201
8
Max-Planck-Society
#
# NIFTy is being developed at the Max-Planck-Institut fuer Astrophysik
# and financially supported by the Studienstiftung des deutschen Volkes.
...
...
@@ -25,6 +25,20 @@ from .line_search_strong_wolfe import LineSearchStrongWolfe
class
VL_BFGS
(
DescentMinimizer
):
"""Implementation of the Vector-free L-BFGS minimization scheme.
Find the descent direction by using the inverse Hessian.
Instead of storing the whole matrix, it stores only the last few
updates, which are used to do operations requiring the inverse
Hessian product. The updates are represented in a new basis to optimize
the algorithm.
References
----------
W. Chen, Z. Wang, J. Zhou, "Large-scale L-BFGS using MapReduce", 2014,
Microsoft
"""
def
__init__
(
self
,
controller
,
line_searcher
=
LineSearchStrongWolfe
(),
max_history_length
=
5
):
super
(
VL_BFGS
,
self
).
__init__
(
controller
=
controller
,
...
...
@@ -36,41 +50,14 @@ class VL_BFGS(DescentMinimizer):
return
super
(
VL_BFGS
,
self
).
__call__
(
energy
)
def
get_descent_direction
(
self
,
energy
):
"""Implementation of the Vector-free L-BFGS minimization scheme.
Find the descent direction by using the inverse Hessian.
Instead of storing the whole matrix, it stores only the last few
updates, which are used to do operations requiring the inverse
Hessian product. The updates are represented in a new basis to optimize
the algorithm.
Parameters
----------
energy : Energy
An instance of the Energy class which shall be minized. The
position of `energy` is used as the starting point of minization.
Returns
-------
descent_direction : Field
Returns the descent direction.
References
----------
W. Chen, Z. Wang, J. Zhou, "Large-scale L-BFGS using MapReduce", 2014,
Microsoft
"""
x
=
energy
.
position
gradient
=
energy
.
gradient
# initialize the information store if it doesn't already exist
try
:
self
.
_information_store
.
add_new_point
(
x
,
gradient
)
except
AttributeError
:
self
.
_information_store
=
InformationStore
(
self
.
max_history_length
,
x0
=
x
,
gradient
=
gradient
)
self
.
_information_store
=
_InformationStore
(
self
.
max_history_length
,
x0
=
x
,
gradient
=
gradient
)
b
=
self
.
_information_store
.
b
delta
=
self
.
_information_store
.
delta
...
...
@@ -82,7 +69,7 @@ class VL_BFGS(DescentMinimizer):
return
descent_direction
class
InformationStore
(
object
):
class
_
InformationStore
(
object
):
"""Class for storing a list of past updates.
Parameters
...
...
Write
Preview
Markdown
is supported
0%
Try again
or
attach a new file
.
Attach a file
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment