Commit 25e88936 authored by Philipp Arras's avatar Philipp Arras

Merge remote-tracking branch 'dev/NIFTy_5' into NIFTy_5

parents 9e80b9a0 87a53824
Pipeline #43112 failed with stages
in 13 seconds
...@@ -10,6 +10,8 @@ setup.cfg ...@@ -10,6 +10,8 @@ setup.cfg
.document .document
.svn/ .svn/
*.csv *.csv
.pytest_cache/
*.png
# from https://github.com/github/gitignore/blob/master/Python.gitignore # from https://github.com/github/gitignore/blob/master/Python.gitignore
......
image: $CONTAINER_TEST_IMAGE image: $CONTAINER_TEST_IMAGE
variables: variables:
CONTAINER_TEST_IMAGE: gitlab-registry.mpcdf.mpg.de/ift/nifty:$CI_BUILD_REF_NAME CONTAINER_TEST_IMAGE: gitlab-registry.mpcdf.mpg.de/ift/nifty-dev:$CI_BUILD_REF_NAME
OMP_NUM_THREADS: 1 OMP_NUM_THREADS: 1
stages: stages:
...@@ -15,6 +15,8 @@ build_docker_from_scratch: ...@@ -15,6 +15,8 @@ build_docker_from_scratch:
- schedules - schedules
image: docker:stable image: docker:stable
stage: build_docker stage: build_docker
before_script:
- ls
script: script:
- docker login -u gitlab-ci-token -p $CI_BUILD_TOKEN gitlab-registry.mpcdf.mpg.de - docker login -u gitlab-ci-token -p $CI_BUILD_TOKEN gitlab-registry.mpcdf.mpg.de
- docker build -t $CONTAINER_TEST_IMAGE --no-cache . - docker build -t $CONTAINER_TEST_IMAGE --no-cache .
...@@ -25,177 +27,87 @@ build_docker_from_cache: ...@@ -25,177 +27,87 @@ build_docker_from_cache:
- schedules - schedules
image: docker:stable image: docker:stable
stage: build_docker stage: build_docker
before_script:
- ls
script: script:
- docker login -u gitlab-ci-token -p $CI_BUILD_TOKEN gitlab-registry.mpcdf.mpg.de - docker login -u gitlab-ci-token -p $CI_BUILD_TOKEN gitlab-registry.mpcdf.mpg.de
- docker build -t $CONTAINER_TEST_IMAGE . - docker build -t $CONTAINER_TEST_IMAGE .
- docker push $CONTAINER_TEST_IMAGE - docker push $CONTAINER_TEST_IMAGE
test_python2_with_coverage: test_serial:
stage: test stage: test
script: script:
- python setup.py install --user -f - pytest-3 -q --cov=nifty5 test
- mpiexec -n 2 --bind-to none nosetests -q 2> /dev/null
- nosetests -q --with-coverage --cover-package=nifty5 --cover-erase
- > - >
coverage report --omit "*plotting*,*distributed_do*" python3 -m coverage report --omit "*plot*,*distributed_do*" | tee coverage.txt
- > - >
coverage report --omit "*plotting*,*distributed_do*" | grep TOTAL | awk '{ print "TOTAL: "$4; }' grep TOTAL coverage.txt | awk '{ print "TOTAL: "$4; }'
test_python3: test_mpi:
stage: test stage: test
variables:
OMPI_MCA_btl_vader_single_copy_mechanism: none
script: script:
- python3 setup.py install --user -f - mpiexec -n 2 --bind-to none pytest-3 -q test
- mpiexec -n 2 --bind-to none nosetests3 -q 2> /dev/null
- nosetests3 -q
pages: pages:
stage: release stage: release
script: script:
- python setup.py install --user -f
- sh docs/generate.sh - sh docs/generate.sh
- mv docs/build/ public/ - mv docs/build/ public/
artifacts: artifacts:
paths: paths:
- public - public
only: only:
- NIFTy_4 - NIFTy_5
before_script:
- export MPLBACKEND="agg"
run_critical_filtering:
stage: demo_runs
script:
- ls
- python setup.py install --user -f
- python3 setup.py install --user -f
- python demos/critical_filtering.py
- python3 demos/critical_filtering.py
artifacts:
paths:
- '*.png'
run_nonlinear_critical_filter:
stage: demo_runs
script:
- python setup.py install --user -f
- python3 setup.py install --user -f
- python demos/nonlinear_critical_filter.py
- python3 demos/nonlinear_critical_filter.py
artifacts:
paths:
- '*.png'
run_nonlinear_wiener_filter: before_script:
stage: demo_runs - python3 setup.py install --user -f
script:
- python setup.py install --user -f
- python3 setup.py install --user -f
- python demos/nonlinear_wiener_filter.py
- python3 demos/nonlinear_wiener_filter.py
only:
- run_demos
artifacts:
paths:
- '*.png'
# FIXME: disable for now. Fixing it is part of issue #244.
#run_poisson_demo:
# stage: demo_runs
# script:
# - python setup.py install --user -f
# - python3 setup.py install --user -f
# - python demos/poisson_demo.py
# - python3 demos/poisson_demo.py
# artifacts:
# paths:
# - '*.png'
run_probing:
stage: demo_runs
script:
- python setup.py install --user -f
- python3 setup.py install --user -f
- python demos/probing.py
- python3 demos/probing.py
artifacts:
paths:
- '*.png'
run_sampling:
stage: demo_runs
script:
- python setup.py install --user -f
- python3 setup.py install --user -f
- python demos/sampling.py
- python3 demos/sampling.py
artifacts:
paths:
- '*.png'
run_tomography: run_ipynb:
stage: demo_runs stage: demo_runs
script: script:
- python setup.py install --user -f - jupyter nbconvert --execute --ExecutePreprocessor.timeout=None demos/Wiener_Filter.ipynb
- python3 setup.py install --user -f
- python demos/tomography.py
- python3 demos/tomography.py
artifacts:
paths:
- '*.png'
run_wiener_filter_data_space_noiseless: run_getting_started_1:
stage: demo_runs stage: demo_runs
script: script:
- python setup.py install --user -f - python3 demos/getting_started_1.py
- python3 setup.py install --user -f - mpiexec -n 2 --bind-to none python3 demos/getting_started_1.py 2> /dev/null
- python demos/wiener_filter_data_space_noiseless.py
- python3 demos/wiener_filter_data_space_noiseless.py
artifacts: artifacts:
paths: paths:
- '*.png' - '*.png'
run_wiener_filter_easy.py: run_getting_started_2:
stage: demo_runs stage: demo_runs
script: script:
- python setup.py install --user -f - python3 demos/getting_started_2.py
- python3 setup.py install --user -f - mpiexec -n 2 --bind-to none python3 demos/getting_started_2.py 2> /dev/null
- python demos/wiener_filter_easy.py
- python3 demos/wiener_filter_easy.py
artifacts: artifacts:
paths: paths:
- '*.png' - '*.png'
run_wiener_filter_via_curvature.py: run_getting_started_3:
stage: demo_runs stage: demo_runs
script: script:
- pip install --user numericalunits - python3 demos/getting_started_3.py
- pip3 install --user numericalunits
- python setup.py install --user -f
- python3 setup.py install --user -f
- python demos/wiener_filter_via_curvature.py
- python3 demos/wiener_filter_via_curvature.py
artifacts: artifacts:
paths: paths:
- '*.png' - '*.png'
run_wiener_filter_via_hamiltonian.py: run_bernoulli:
stage: demo_runs stage: demo_runs
script: script:
- python setup.py install --user -f - python3 demos/bernoulli_demo.py
- python3 setup.py install --user -f
- python demos/wiener_filter_via_hamiltonian.py
- python3 demos/wiener_filter_via_hamiltonian.py
artifacts: artifacts:
paths: paths:
- '*.png' - '*.png'
run_ipynb: run_curve_fitting:
stage: demo_runs stage: demo_runs
script: script:
- python setup.py install --user -f - python3 demos/polynomial_fit.py
- python3 setup.py install --user -f
- jupyter nbconvert --execute --ExecutePreprocessor.timeout=None demos/Wiener_Filter.ipynb
artifacts: artifacts:
paths: paths:
- '*.png' - '*.png'
FROM debian:testing-slim FROM debian:testing-slim
RUN apt-get update && apt-get install -y \ RUN apt-get update && apt-get install -y \
# Needed for gitlab tests # Needed for setup
git \ git python3-pip \
# Packages needed for NIFTy # Packages needed for NIFTy
libfftw3-dev \ python3-scipy \
python python-pip python-dev python-future python-scipy cython \
python3 python3-pip python3-dev python3-future python3-scipy cython3 \
# Documentation build dependencies # Documentation build dependencies
python-sphinx python-sphinx-rtd-theme python-numpydoc \ python3-sphinx-rtd-theme dvipng texlive-latex-base texlive-latex-extra \
# Testing dependencies # Testing dependencies
python-nose python-parameterized \ python3-pytest-cov jupyter \
python3-nose python3-parameterized \
# Optional NIFTy dependencies # Optional NIFTy dependencies
openmpi-bin libopenmpi-dev python-mpi4py python3-mpi4py \ libfftw3-dev python3-mpi4py python3-matplotlib \
# Packages needed for NIFTy # more optional NIFTy dependencies
&& pip install pyfftw \
&& pip3 install pyfftw \ && pip3 install pyfftw \
# Optional NIFTy dependencies
&& pip install git+https://gitlab.mpcdf.mpg.de/ift/pyHealpix.git \
&& pip3 install git+https://gitlab.mpcdf.mpg.de/ift/pyHealpix.git \ && pip3 install git+https://gitlab.mpcdf.mpg.de/ift/pyHealpix.git \
# Testing dependencies && pip3 install jupyter \
&& pip install coverage \
&& rm -rf /var/lib/apt/lists/* && rm -rf /var/lib/apt/lists/*
# Needed for demos to be running # Set matplotlib backend
RUN apt-get update && apt-get install -y python-matplotlib python3-matplotlib \ ENV MPLBACKEND agg
&& python3 -m pip install --upgrade pip && python3 -m pip install jupyter && python -m pip install --upgrade pip && python -m pip install jupyter \
&& rm -rf /var/lib/apt/lists/*
# Create user (openmpi does not like to be run as root) # Create user (openmpi does not like to be run as root)
RUN useradd -ms /bin/bash testinguser RUN useradd -ms /bin/bash testinguser
......
...@@ -37,11 +37,11 @@ Installation ...@@ -37,11 +37,11 @@ Installation
### Requirements ### Requirements
- [Python](https://www.python.org/) (v2.7.x or 3.5.x) - [Python 3](https://www.python.org/) (3.5.x or later)
- [SciPy](https://www.scipy.org/) - [SciPy](https://www.scipy.org/)
- [pyFFTW](https://pypi.python.org/pypi/pyFFTW)
Optional dependencies: Optional dependencies:
- [pyFFTW](https://pypi.python.org/pypi/pyFFTW) for faster Fourier transforms
- [pyHealpix](https://gitlab.mpcdf.mpg.de/ift/pyHealpix) (for harmonic - [pyHealpix](https://gitlab.mpcdf.mpg.de/ift/pyHealpix) (for harmonic
transforms involving domains on the sphere) transforms involving domains on the sphere)
- [mpi4py](https://mpi4py.scipy.org) (for MPI-parallel execution) - [mpi4py](https://mpi4py.scipy.org) (for MPI-parallel execution)
...@@ -61,45 +61,48 @@ distributions, the "apt" lines will need slight changes. ...@@ -61,45 +61,48 @@ distributions, the "apt" lines will need slight changes.
NIFTy5 and its mandatory dependencies can be installed via: NIFTy5 and its mandatory dependencies can be installed via:
sudo apt-get install git libfftw3-dev python python-pip python-dev sudo apt-get install git python3 python3-pip python3-dev
pip install --user git+https://gitlab.mpcdf.mpg.de/ift/NIFTy.git@NIFTy_5 pip3 install --user git+https://gitlab.mpcdf.mpg.de/ift/NIFTy.git@NIFTy_5
Plotting support is added via:
sudo apt-get install python3-matplotlib
NIFTy uses Numpy's FFT implementation by default. For large problems FFTW may be
used because of its higher performance. It can be installed via:
sudo apt-get install libfftw3-dev
pip3 install --user pyfftw
To enable FFTW usage in NIFTy, call
nifty5.fft.enable_fftw()
at the beginning of your code.
(Note: If you encounter problems related to `pyFFTW`, make sure that you are (Note: If you encounter problems related to `pyFFTW`, make sure that you are
using a pip-installed `pyFFTW` package. Unfortunately, some distributions are using a pip-installed `pyFFTW` package. Unfortunately, some distributions are
shipping an incorrectly configured `pyFFTW` package, which does not cooperate shipping an incorrectly configured `pyFFTW` package, which does not cooperate
with the installed `FFTW3` libraries.) with the installed `FFTW3` libraries.)
Plotting support is added via:
pip install --user matplotlib
Support for spherical harmonic transforms is added via: Support for spherical harmonic transforms is added via:
pip install --user git+https://gitlab.mpcdf.mpg.de/ift/pyHealpix.git pip3 install --user git+https://gitlab.mpcdf.mpg.de/ift/pyHealpix.git
MPI support is added via: MPI support is added via:
sudo apt-get install openmpi-bin libopenmpi-dev sudo apt-get install python3-mpi4py
pip install --user mpi4py
### Installation for Python 3
If you want to run NIFTy with Python 3, you need to make the following changes
to the instructions above:
- in all `apt-get` commands, replace `python-*` by `python3-*`
- in all `pip` commands, replace `pip` by `pip3`
### Running the tests ### Running the tests
In oder to run the tests one needs two additional packages: To run the tests, additional packages are required:
pip install --user nose parameterized coverage sudo apt-get install python3-pytest-cov
Afterwards the tests (including a coverage report) can be run using the Afterwards the tests (including a coverage report) can be run using the
following command in the repository root: following command in the repository root:
nosetests -x --with-coverage --cover-html --cover-package=nifty5 pytest-3 --cov=nifty5 test
### First Steps ### First Steps
...@@ -108,7 +111,7 @@ For a quick start, you can browse through the [informal ...@@ -108,7 +111,7 @@ For a quick start, you can browse through the [informal
introduction](http://ift.pages.mpcdf.de/NIFTy/code.html) or introduction](http://ift.pages.mpcdf.de/NIFTy/code.html) or
dive into NIFTy by running one of the demonstrations, e.g.: dive into NIFTy by running one of the demonstrations, e.g.:
python demos/wiener_filter_via_curvature.py python3 demos/getting_started_1.py
### Acknowledgement ### Acknowledgement
......
...@@ -58,7 +58,7 @@ ...@@ -58,7 +58,7 @@
"### Posterior\n", "### Posterior\n",
"The Posterior is given by:\n", "The Posterior is given by:\n",
"\n", "\n",
"$$\\mathcal P (s|d) \\propto P(s,d) = \\mathcal G(d-Rs,N) \\,\\mathcal G(s,S) \\propto \\mathcal G (m,D) $$\n", "$$\\mathcal P (s|d) \\propto P(s,d) = \\mathcal G(d-Rs,N) \\,\\mathcal G(s,S) \\propto \\mathcal G (s-m,D) $$\n",
"\n", "\n",
"where\n", "where\n",
"$$\\begin{align}\n", "$$\\begin{align}\n",
...@@ -172,7 +172,7 @@ ...@@ -172,7 +172,7 @@
" tol_abs_gradnorm=0.1)\n", " tol_abs_gradnorm=0.1)\n",
" # WienerFilterCurvature is (R.adjoint*N.inverse*R + Sh.inverse) plus some handy\n", " # WienerFilterCurvature is (R.adjoint*N.inverse*R + Sh.inverse) plus some handy\n",
" # helper methods.\n", " # helper methods.\n",
" return ift.library.WienerFilterCurvature(R,N,Sh,iteration_controller=IC,iteration_controller_sampling=IC)" " return ift.WienerFilterCurvature(R,N,Sh,iteration_controller=IC,iteration_controller_sampling=IC)"
] ]
}, },
{ {
...@@ -429,8 +429,8 @@ ...@@ -429,8 +429,8 @@
"mask[l:h] = 0\n", "mask[l:h] = 0\n",
"mask = ift.Field.from_global_data(s_space, mask)\n", "mask = ift.Field.from_global_data(s_space, mask)\n",
"\n", "\n",
"R = ift.DiagonalOperator(mask)*HT\n", "R = ift.DiagonalOperator(mask)(HT)\n",
"n = n.to_global_data()\n", "n = n.to_global_data_rw()\n",
"n[l:h] = 0\n", "n[l:h] = 0\n",
"n = ift.Field.from_global_data(s_space, n)\n", "n = ift.Field.from_global_data(s_space, n)\n",
"\n", "\n",
...@@ -501,7 +501,7 @@ ...@@ -501,7 +501,7 @@
"m_data = HT(m).to_global_data()\n", "m_data = HT(m).to_global_data()\n",
"m_var_data = m_var.to_global_data()\n", "m_var_data = m_var.to_global_data()\n",
"uncertainty = np.sqrt(m_var_data)\n", "uncertainty = np.sqrt(m_var_data)\n",
"d_data = d.to_global_data()\n", "d_data = d.to_global_data_rw()\n",
"\n", "\n",
"# Set lost data to NaN for proper plotting\n", "# Set lost data to NaN for proper plotting\n",
"d_data[d_data == 0] = np.nan" "d_data[d_data == 0] = np.nan"
...@@ -585,8 +585,8 @@ ...@@ -585,8 +585,8 @@
"mask[l:h,l:h] = 0.\n", "mask[l:h,l:h] = 0.\n",
"mask = ift.Field.from_global_data(s_space, mask)\n", "mask = ift.Field.from_global_data(s_space, mask)\n",
"\n", "\n",
"R = ift.DiagonalOperator(mask)*HT\n", "R = ift.DiagonalOperator(mask)(HT)\n",
"n = n.to_global_data()\n", "n = n.to_global_data_rw()\n",
"n[l:h, l:h] = 0\n", "n[l:h, l:h] = 0\n",
"n = ift.Field.from_global_data(s_space, n)\n", "n = ift.Field.from_global_data(s_space, n)\n",
"curv = Curvature(R=R, N=N, Sh=Sh)\n", "curv = Curvature(R=R, N=N, Sh=Sh)\n",
...@@ -717,21 +717,21 @@ ...@@ -717,21 +717,21 @@
"metadata": { "metadata": {
"celltoolbar": "Slideshow", "celltoolbar": "Slideshow",
"kernelspec": { "kernelspec": {
"display_name": "Python 2", "display_name": "Python 3",
"language": "python", "language": "python",
"name": "python2" "name": "python3"
}, },
"language_info": { "language_info": {
"codemirror_mode": { "codemirror_mode": {
"name": "ipython", "name": "ipython",
"version": 2 "version": 3
}, },
"file_extension": ".py", "file_extension": ".py",
"mimetype": "text/x-python", "mimetype": "text/x-python",
"name": "python", "name": "python",
"nbconvert_exporter": "python", "nbconvert_exporter": "python",
"pygments_lexer": "ipython2", "pygments_lexer": "ipython3",
"version": "2.7.15" "version": "3.6.6"
} }
}, },
"nbformat": 4, "nbformat": 4,
......
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
#
# Copyright(C) 2013-2019 Max-Planck-Society
#
# NIFTy is being developed at the Max-Planck-Institut fuer Astrophysik.
#####################################################################
# Bernoulli reconstruction
# Reconstruct an event probability field with values between 0 and 1
# from the observed events
# 1D (set mode=0), 2D (mode=1), or on the sphere (mode=2)
#####################################################################
import numpy as np
import nifty5 as ift
if __name__ == '__main__':
np.random.seed(41)
# Set up the position space of the signal
mode = 2
if mode == 0:
# One-dimensional regular grid
position_space = ift.RGSpace(1024)
elif mode == 1:
# Two-dimensional regular grid
position_space = ift.RGSpace([512, 512])
else:
# Sphere
position_space = ift.HPSpace(128)
# Define harmonic space and transform
harmonic_space = position_space.get_default_codomain()
HT = ift.HarmonicTransformOperator(harmonic_space, position_space)
position = ift.from_random('normal', harmonic_space)
# Define power spectrum and amplitudes
def sqrtpspec(k):
return 1./(20. + k**2)
A = ift.create_power_operator(harmonic_space, sqrtpspec)
# Set up a sky operator and instrumental response
sky = ift.sigmoid(HT(A))
GR = ift.GeometryRemover(position_space)
R = GR
# Generate mock data
p = R(sky)
mock_position = ift.from_random('normal', harmonic_space)
tmp = p(mock_position).to_global_data().astype(np.float64)
data = np.random.binomial(1, tmp)
data = ift.Field.from_global_data(R.target, data)
# Compute likelihood and Hamiltonian
position = ift.from_random('normal', harmonic_space)
likelihood = ift.BernoulliEnergy(data)(p)
ic_newton = ift.DeltaEnergyController(
name='Newton', iteration_limit=100, tol_rel_deltaE=1e-8)
minimizer = ift.NewtonCG(ic_newton)
ic_sampling = ift.GradientNormController(iteration_limit=100)
# Minimize the Hamiltonian
H = ift.StandardHamiltonian(likelihood, ic_sampling)