Commit 5cc52382 authored by Martin Reinecke's avatar Martin Reinecke

merge master; currently broken

parents c7c8e176 aa91ea8c
Pipeline #43304 failed with stages
in 31 seconds
...@@ -10,6 +10,8 @@ setup.cfg ...@@ -10,6 +10,8 @@ setup.cfg
.document .document
.svn/ .svn/
*.csv *.csv
.pytest_cache/
*.png
# from https://github.com/github/gitignore/blob/master/Python.gitignore # from https://github.com/github/gitignore/blob/master/Python.gitignore
......
image: $CONTAINER_TEST_IMAGE image: $CONTAINER_TEST_IMAGE
variables: variables:
CONTAINER_TEST_IMAGE: gitlab-registry.mpcdf.mpg.de/ift/nifty:$CI_BUILD_REF_NAME CONTAINER_TEST_IMAGE: gitlab-registry.mpcdf.mpg.de/$CI_PROJECT_PATH:$CI_BUILD_REF_NAME
OMP_NUM_THREADS: 1 OMP_NUM_THREADS: 1
stages: stages:
...@@ -15,6 +15,8 @@ build_docker_from_scratch: ...@@ -15,6 +15,8 @@ build_docker_from_scratch:
- schedules - schedules
image: docker:stable image: docker:stable
stage: build_docker stage: build_docker
before_script:
- ls
script: script:
- docker login -u gitlab-ci-token -p $CI_BUILD_TOKEN gitlab-registry.mpcdf.mpg.de - docker login -u gitlab-ci-token -p $CI_BUILD_TOKEN gitlab-registry.mpcdf.mpg.de
- docker build -t $CONTAINER_TEST_IMAGE --no-cache . - docker build -t $CONTAINER_TEST_IMAGE --no-cache .
...@@ -25,33 +27,32 @@ build_docker_from_cache: ...@@ -25,33 +27,32 @@ build_docker_from_cache:
- schedules - schedules
image: docker:stable image: docker:stable
stage: build_docker stage: build_docker
before_script:
- ls
script: script:
- docker login -u gitlab-ci-token -p $CI_BUILD_TOKEN gitlab-registry.mpcdf.mpg.de - docker login -u gitlab-ci-token -p $CI_BUILD_TOKEN gitlab-registry.mpcdf.mpg.de
- docker build -t $CONTAINER_TEST_IMAGE . - docker build -t $CONTAINER_TEST_IMAGE .
- docker push $CONTAINER_TEST_IMAGE - docker push $CONTAINER_TEST_IMAGE
test_python2_with_coverage: test_serial:
stage: test stage: test
script: script:
- python setup.py install --user -f - pytest-3 -q --cov=nifty5 test
- mpiexec -n 2 --bind-to none nosetests -q 2> /dev/null
- nosetests -q --with-coverage --cover-package=nifty5 --cover-erase
- > - >
coverage report --omit "*plotting*,*distributed_do*" python3 -m coverage report --omit "*plot*,*distributed_do*" | tee coverage.txt
- > - >
coverage report --omit "*plotting*,*distributed_do*" | grep TOTAL | awk '{ print "TOTAL: "$4; }' grep TOTAL coverage.txt | awk '{ print "TOTAL: "$4; }'
test_python3: test_mpi:
stage: test stage: test
variables:
OMPI_MCA_btl_vader_single_copy_mechanism: none
script: script:
- python3 setup.py install --user -f - mpiexec -n 2 --bind-to none pytest-3 -q test
- mpiexec -n 2 --bind-to none nosetests3 -q 2> /dev/null
- nosetests3 -q
pages: pages:
stage: release stage: release
script: script:
- python setup.py install --user -f
- sh docs/generate.sh - sh docs/generate.sh
- mv docs/build/ public/ - mv docs/build/ public/
artifacts: artifacts:
...@@ -60,142 +61,53 @@ pages: ...@@ -60,142 +61,53 @@ pages:
only: only:
- NIFTy_5 - NIFTy_5
before_script:
- export MPLBACKEND="agg"
run_critical_filtering:
stage: demo_runs
script:
- ls
- python setup.py install --user -f
- python3 setup.py install --user -f
- python demos/critical_filtering.py
- python3 demos/critical_filtering.py
artifacts:
paths:
- '*.png'
run_nonlinear_critical_filter:
stage: demo_runs
script:
- python setup.py install --user -f
- python3 setup.py install --user -f
- python demos/nonlinear_critical_filter.py
- python3 demos/nonlinear_critical_filter.py
artifacts:
paths:
- '*.png'
run_nonlinear_wiener_filter:
stage: demo_runs
script:
- python setup.py install --user -f
- python3 setup.py install --user -f
- python demos/nonlinear_wiener_filter.py
- python3 demos/nonlinear_wiener_filter.py
only:
- run_demos
artifacts:
paths:
- '*.png'
# FIXME: disable for now. Fixing it is part of issue #244.
#run_poisson_demo:
# stage: demo_runs
# script:
# - python setup.py install --user -f
# - python3 setup.py install --user -f
# - python demos/poisson_demo.py
# - python3 demos/poisson_demo.py
# artifacts:
# paths:
# - '*.png'
run_probing:
stage: demo_runs
script:
- python setup.py install --user -f
- python3 setup.py install --user -f
- python demos/probing.py
- python3 demos/probing.py
artifacts:
paths:
- '*.png'
run_sampling: before_script:
stage: demo_runs
script:
- python setup.py install --user -f
- python3 setup.py install --user -f - python3 setup.py install --user -f
- python demos/sampling.py
- python3 demos/sampling.py
artifacts:
paths:
- '*.png'
run_tomography: run_ipynb:
stage: demo_runs stage: demo_runs
script: script:
- python setup.py install --user -f - jupyter nbconvert --execute --ExecutePreprocessor.timeout=None demos/Wiener_Filter.ipynb
- python3 setup.py install --user -f
- python demos/tomography.py
- python3 demos/tomography.py
artifacts:
paths:
- '*.png'
run_wiener_filter_data_space_noiseless: run_getting_started_1:
stage: demo_runs stage: demo_runs
script: script:
- python setup.py install --user -f - python3 demos/getting_started_1.py
- python3 setup.py install --user -f - mpiexec -n 2 --bind-to none python3 demos/getting_started_1.py 2> /dev/null
- python demos/wiener_filter_data_space_noiseless.py
- python3 demos/wiener_filter_data_space_noiseless.py
artifacts: artifacts:
paths: paths:
- '*.png' - '*.png'
run_wiener_filter_easy.py: run_getting_started_2:
stage: demo_runs stage: demo_runs
script: script:
- python setup.py install --user -f - python3 demos/getting_started_2.py
- python3 setup.py install --user -f - mpiexec -n 2 --bind-to none python3 demos/getting_started_2.py 2> /dev/null
- python demos/wiener_filter_easy.py
- python3 demos/wiener_filter_easy.py
artifacts: artifacts:
paths: paths:
- '*.png' - '*.png'
run_wiener_filter_via_curvature.py: run_getting_started_3:
stage: demo_runs stage: demo_runs
script: script:
- pip install --user numericalunits - python3 demos/getting_started_3.py
- pip3 install --user numericalunits
- python setup.py install --user -f
- python3 setup.py install --user -f
- python demos/wiener_filter_via_curvature.py
- python3 demos/wiener_filter_via_curvature.py
artifacts: artifacts:
paths: paths:
- '*.png' - '*.png'
run_wiener_filter_via_hamiltonian.py: run_bernoulli:
stage: demo_runs stage: demo_runs
script: script:
- python setup.py install --user -f - python3 demos/bernoulli_demo.py
- python3 setup.py install --user -f
- python demos/wiener_filter_via_hamiltonian.py
- python3 demos/wiener_filter_via_hamiltonian.py
artifacts: artifacts:
paths: paths:
- '*.png' - '*.png'
run_ipynb: run_curve_fitting:
stage: demo_runs stage: demo_runs
script: script:
- python setup.py install --user -f - python3 demos/polynomial_fit.py
- python3 setup.py install --user -f
- jupyter nbconvert --execute --ExecutePreprocessor.timeout=None demos/Wiener_Filter.ipynb
artifacts: artifacts:
paths: paths:
- '*.png' - '*.png'
FROM debian:testing-slim FROM debian:testing-slim
RUN apt-get update && apt-get install -y \ RUN apt-get update && apt-get install -y \
# Needed for gitlab tests # Needed for setup
git \ git python3-pip \
# Packages needed for NIFTy # Packages needed for NIFTy
libfftw3-dev \ python3-scipy \
python python-pip python-dev python-future python-scipy \
python3 python3-pip python3-dev python3-future python3-scipy \
# Documentation build dependencies # Documentation build dependencies
python-sphinx python-sphinx-rtd-theme python-numpydoc \ python3-sphinx-rtd-theme dvipng texlive-latex-base texlive-latex-extra \
# Testing dependencies # Testing dependencies
python-nose python-parameterized \ python3-pytest-cov jupyter \
python3-nose python3-parameterized \
# Optional NIFTy dependencies # Optional NIFTy dependencies
openmpi-bin libopenmpi-dev python-mpi4py python3-mpi4py \ libfftw3-dev python3-mpi4py python3-matplotlib \
# Packages needed for NIFTy # more optional NIFTy dependencies
&& pip install pyfftw \
&& pip3 install pyfftw \ && pip3 install pyfftw \
# Optional NIFTy dependencies
&& pip install git+https://gitlab.mpcdf.mpg.de/ift/pyHealpix.git \
&& pip3 install git+https://gitlab.mpcdf.mpg.de/ift/pyHealpix.git \ && pip3 install git+https://gitlab.mpcdf.mpg.de/ift/pyHealpix.git \
# Testing dependencies && pip3 install jupyter \
&& pip install coverage \
&& rm -rf /var/lib/apt/lists/* && rm -rf /var/lib/apt/lists/*
# Needed for demos to be running # Set matplotlib backend
RUN apt-get update && apt-get install -y python-matplotlib python3-matplotlib \ ENV MPLBACKEND agg
&& python3 -m pip install --upgrade pip && python3 -m pip install jupyter && python -m pip install --upgrade pip && python -m pip install jupyter \
&& rm -rf /var/lib/apt/lists/*
# Create user (openmpi does not like to be run as root) # Create user (openmpi does not like to be run as root)
RUN useradd -ms /bin/bash testinguser RUN useradd -ms /bin/bash testinguser
......
...@@ -4,7 +4,7 @@ NIFTy - Numerical Information Field Theory ...@@ -4,7 +4,7 @@ NIFTy - Numerical Information Field Theory
[![coverage report](https://gitlab.mpcdf.mpg.de/ift/NIFTy/badges/NIFTy_5/coverage.svg)](https://gitlab.mpcdf.mpg.de/ift/NIFTy/commits/NIFTy_5) [![coverage report](https://gitlab.mpcdf.mpg.de/ift/NIFTy/badges/NIFTy_5/coverage.svg)](https://gitlab.mpcdf.mpg.de/ift/NIFTy/commits/NIFTy_5)
**NIFTy** project homepage: **NIFTy** project homepage:
[http://ift.pages.mpcdf.de/NIFTy](http://ift.pages.mpcdf.de/NIFTy) [http://ift.pages.mpcdf.de/nifty](http://ift.pages.mpcdf.de/nifty)
Summary Summary
------- -------
...@@ -13,23 +13,31 @@ Summary ...@@ -13,23 +13,31 @@ Summary
**NIFTy**, "**N**umerical **I**nformation **F**ield **T**heor<strong>y</strong>", is **NIFTy**, "**N**umerical **I**nformation **F**ield **T**heor<strong>y</strong>", is
a versatile library designed to enable the development of signal a versatile library designed to enable the development of signal
inference algorithms that operate regardless of the underlying spatial inference algorithms that operate regardless of the underlying grids
grid and its resolution. Its object-oriented framework is written in (spatial, spectral, temporal, …) and their resolutions.
Python, although it accesses libraries written in C++ and C for Its object-oriented framework is written in Python, although it accesses
efficiency. libraries written in C++ and C for efficiency.
NIFTy offers a toolkit that abstracts discretized representations of NIFTy offers a toolkit that abstracts discretized representations of
continuous spaces, fields in these spaces, and operators acting on continuous spaces, fields in these spaces, and operators acting on
fields into classes. The correct normalization of operations on these fields into classes.
fields is taken care of automatically without concerning the user. This This allows for an abstract formulation and programming of inference
allows for an abstract formulation and programming of inference
algorithms, including those derived within information field theory. algorithms, including those derived within information field theory.
Thus, NIFTy permits its user to rapidly prototype algorithms in 1D, and NIFTy's interface is designed to resemble IFT formulae in the sense
then apply the developed code in higher-dimensional settings of real that the user implements algorithms in NIFTy independent of the topology
world problems. The set of spaces on which NIFTy operates comprises of the underlying spaces and the discretization scheme.
point sets, *n*-dimensional regular grids, spherical spaces, their Thus, the user can develop algorithms on subsets of problems and on
harmonic counterparts, and product spaces constructed as combinations of spaces where the detailed performance of the algorithm can be properly
those. evaluated and then easily generalize them to other, more complex spaces
and the full problem, respectively.
The set of spaces on which NIFTy operates comprises point sets,
*n*-dimensional regular grids, spherical spaces, their harmonic
counterparts, and product spaces constructed as combinations of those.
NIFTy takes care of numerical subtleties like the normalization of
operations on fields and the numerical representation of model
components, allowing the user to focus on formulating the abstract
inference procedures and process-specific model properties.
Installation Installation
...@@ -37,11 +45,11 @@ Installation ...@@ -37,11 +45,11 @@ Installation
### Requirements ### Requirements
- [Python](https://www.python.org/) (v2.7.x or 3.5.x) - [Python 3](https://www.python.org/) (3.5.x or later)
- [SciPy](https://www.scipy.org/) - [SciPy](https://www.scipy.org/)
- [pyFFTW](https://pypi.python.org/pypi/pyFFTW)
Optional dependencies: Optional dependencies:
- [pyFFTW](https://pypi.python.org/pypi/pyFFTW) for faster Fourier transforms
- [pyHealpix](https://gitlab.mpcdf.mpg.de/ift/pyHealpix) (for harmonic - [pyHealpix](https://gitlab.mpcdf.mpg.de/ift/pyHealpix) (for harmonic
transforms involving domains on the sphere) transforms involving domains on the sphere)
- [mpi4py](https://mpi4py.scipy.org) (for MPI-parallel execution) - [mpi4py](https://mpi4py.scipy.org) (for MPI-parallel execution)
...@@ -61,57 +69,60 @@ distributions, the "apt" lines will need slight changes. ...@@ -61,57 +69,60 @@ distributions, the "apt" lines will need slight changes.
NIFTy5 and its mandatory dependencies can be installed via: NIFTy5 and its mandatory dependencies can be installed via:
sudo apt-get install git libfftw3-dev python python-pip python-dev sudo apt-get install git python3 python3-pip python3-dev
pip install --user git+https://gitlab.mpcdf.mpg.de/ift/NIFTy.git@NIFTy_5 pip3 install --user git+https://gitlab.mpcdf.mpg.de/ift/NIFTy.git@NIFTy_5
Plotting support is added via:
sudo apt-get install python3-matplotlib
NIFTy uses Numpy's FFT implementation by default. For large problems FFTW may be
used because of its higher performance. It can be installed via:
sudo apt-get install libfftw3-dev
pip3 install --user pyfftw
To enable FFTW usage in NIFTy, call
nifty5.fft.enable_fftw()
at the beginning of your code.
(Note: If you encounter problems related to `pyFFTW`, make sure that you are (Note: If you encounter problems related to `pyFFTW`, make sure that you are
using a pip-installed `pyFFTW` package. Unfortunately, some distributions are using a pip-installed `pyFFTW` package. Unfortunately, some distributions are
shipping an incorrectly configured `pyFFTW` package, which does not cooperate shipping an incorrectly configured `pyFFTW` package, which does not cooperate
with the installed `FFTW3` libraries.) with the installed `FFTW3` libraries.)
Plotting support is added via:
pip install --user matplotlib
Support for spherical harmonic transforms is added via: Support for spherical harmonic transforms is added via:
pip install --user git+https://gitlab.mpcdf.mpg.de/ift/pyHealpix.git pip3 install --user git+https://gitlab.mpcdf.mpg.de/ift/pyHealpix.git
MPI support is added via: MPI support is added via:
sudo apt-get install openmpi-bin libopenmpi-dev sudo apt-get install python3-mpi4py
pip install --user mpi4py
### Installation for Python 3
If you want to run NIFTy with Python 3, you need to make the following changes
to the instructions above:
- in all `apt-get` commands, replace `python-*` by `python3-*`
- in all `pip` commands, replace `pip` by `pip3`
### Running the tests ### Running the tests
In oder to run the tests one needs two additional packages: To run the tests, additional packages are required:
pip install --user nose parameterized coverage sudo apt-get install python3-pytest-cov
Afterwards the tests (including a coverage report) can be run using the Afterwards the tests (including a coverage report) can be run using the
following command in the repository root: following command in the repository root:
nosetests -x --with-coverage --cover-html --cover-package=nifty5 pytest-3 --cov=nifty5 test
### First Steps ### First Steps
For a quick start, you can browse through the [informal For a quick start, you can browse through the [informal
introduction](http://ift.pages.mpcdf.de/NIFTy/code.html) or introduction](http://ift.pages.mpcdf.de/nifty/code.html) or
dive into NIFTy by running one of the demonstrations, e.g.: dive into NIFTy by running one of the demonstrations, e.g.:
python demos/wiener_filter_via_curvature.py python3 demos/getting_started_1.py
### Acknowledgement ### Acknowledgements
Please acknowledge the use of NIFTy in your publication(s) by using a Please acknowledge the use of NIFTy in your publication(s) by using a
phrase such as the following: phrase such as the following:
...@@ -119,10 +130,10 @@ phrase such as the following: ...@@ -119,10 +130,10 @@ phrase such as the following:
> "Some of the results in this publication have been derived using the > "Some of the results in this publication have been derived using the
> NIFTy package [(https://gitlab.mpcdf.mpg.de/ift/NIFTy)](https://gitlab.mpcdf.mpg.de/ift/NIFTy)" > NIFTy package [(https://gitlab.mpcdf.mpg.de/ift/NIFTy)](https://gitlab.mpcdf.mpg.de/ift/NIFTy)"
and a citation to one of the [publications](http://ift.pages.mpcdf.de/NIFTy/citations.html). and a citation to one of the [publications](http://ift.pages.mpcdf.de/nifty/citations.html).
### Release Notes ### Licensing terms
The NIFTy package is licensed under the terms of the The NIFTy package is licensed under the terms of the
[GPLv3](https://www.gnu.org/licenses/gpl.html) and is distributed [GPLv3](https://www.gnu.org/licenses/gpl.html) and is distributed
......
...@@ -58,7 +58,7 @@ ...@@ -58,7 +58,7 @@
"### Posterior\n", "### Posterior\n",
"The Posterior is given by:\n", "The Posterior is given by:\n",
"\n", "\n",
"$$\\mathcal P (s|d) \\propto P(s,d) = \\mathcal G(d-Rs,N) \\,\\mathcal G(s,S) \\propto \\mathcal G (m,D) $$\n", "$$\\mathcal P (s|d) \\propto P(s,d) = \\mathcal G(d-Rs,N) \\,\\mathcal G(s,S) \\propto \\mathcal G (s-m,D) $$\n",
"\n", "\n",
"where\n", "where\n",
"$$\\begin{align}\n", "$$\\begin{align}\n",
...@@ -172,7 +172,7 @@ ...@@ -172,7 +172,7 @@
" tol_abs_gradnorm=0.1)\n", " tol_abs_gradnorm=0.1)\n",
" # WienerFilterCurvature is (R.adjoint*N.inverse*R + Sh.inverse) plus some handy\n", " # WienerFilterCurvature is (R.adjoint*N.inverse*R + Sh.inverse) plus some handy\n",
" # helper methods.\n", " # helper methods.\n",
" return ift.library.WienerFilterCurvature(R,N,Sh,iteration_controller=IC,iteration_controller_sampling=IC)" " return ift.WienerFilterCurvature(R,N,Sh,iteration_controller=IC,iteration_controller_sampling=IC)"
] ]
}, },
{ {
...@@ -429,8 +429,8 @@ ...@@ -429,8 +429,8 @@
"mask[l:h] = 0\n", "mask[l:h] = 0\n",
"mask = ift.Field.from_global_data(s_space, mask)\n", "mask = ift.Field.from_global_data(s_space, mask)\n",
"\n", "\n",
"R = ift.DiagonalOperator(mask)*HT\n", "R = ift.DiagonalOperator(mask)(HT)\n",
"n = n.to_global_data()\n", "n = n.to_global_data_rw()\n",
"n[l:h] = 0\n", "n[l:h] = 0\n",
"n = ift.Field.from_global_data(s_space, n)\n", "n = ift.Field.from_global_data(s_space, n)\n",
"\n", "\n",
...@@ -501,7 +501,7 @@ ...@@ -501,7 +501,7 @@
"m_data = HT(m).to_global_data()\n", "m_data = HT(m).to_global_data()\n",
"m_var_data = m_var.to_global_data()\n", "m_var_data = m_var.to_global_data()\n",
"uncertainty = np.sqrt(m_var_data)\n", "uncertainty = np.sqrt(m_var_data)\n",
"d_data = d.to_global_data()\n", "d_data = d.to_global_data_rw()\n",
"\n", "\n",
"# Set lost data to NaN for proper plotting\n", "# Set lost data to NaN for proper plotting\n",
"d_data[d_data == 0] = np.nan" "d_data[d_data == 0] = np.nan"
...@@ -585,8 +585,8 @@ ...@@ -585,8 +585,8 @@
"mask[l:h,l:h] = 0.\n", "mask[l:h,l:h] = 0.\n",
"mask = ift.Field.from_global_data(s_space, mask)\n", "mask = ift.Field.from_global_data(s_space, mask)\n",
"\n", "\n",
"R = ift.DiagonalOperator(mask)*HT\n", "R = ift.DiagonalOperator(mask)(HT)\n",
"n = n.to_global_data()\n", "n = n.to_global_data_rw()\n",
"n[l:h, l:h] = 0\n", "n[l:h, l:h] = 0\n",
"n = ift.Field.from_global_data(s_space, n)\n", "n = ift.Field.from_global_data(s_space, n)\n",
"curv = Curvature(R=R, N=N, Sh=Sh)\n", "curv = Curvature(R=R, N=N, Sh=Sh)\n",
...@@ -717,21 +717,21 @@ ...@@ -717,21 +717,21 @@
"metadata": { "metadata": {
"celltoolbar": "Slideshow", "celltoolbar": "Slideshow",
"kernelspec": { "kernelspec": {
"display_name": "Python 2", "display_name": "Python 3",
"language": "python", "language": "python",
"name": "python2" "name": "python3"
}, },
"language_info": { "language_info": {
"codemirror_mode": { "codemirror_mode": {
"name": "ipython", "name": "ipython",
"version": 2 "version": 3
}, },
"file_extension": ".py", "file_extension": ".py",
"mimetype": "text/x-python", "mimetype": "text/x-python",
"name": "python", "name": "python",
"nbconvert_exporter": "python", "nbconvert_exporter": "python",
"pygments_lexer": "ipython2", "pygments_lexer": "ipython3",
"version": "2.7.15" "version": "3.6.6"
} }