big bunch of reorganization

.gitlab-ci.yml

@@ -2,7 +2,6 @@ image: $CONTAINER_TEST_IMAGE
 
 variables:
   CONTAINER_TEST_IMAGE: gitlab-registry.mpcdf.mpg.de/$CI_PROJECT_PATH:$CI_BUILD_REF_NAME
-  OMP_NUM_THREADS: 2
 
 stages:
   - build_docker
@@ -21,13 +20,13 @@ test_gcc:
   stage: testing
   script:
     - python3 setup.py install --user -f
-    - pytest-3 -q pypocketfft/test pysharp/test pyHealpix/test nifty_gridder/test pyinterpol_ng/test
+    - pytest-3 -q fft/test sht/test healpix/test wgridder/test totalconvolve/test
 
 test_clang:
   stage: testing
   script:
     - CC="clang -fsized-deallocation" python3 setup.py install --user -f
-    - pytest-3 -q pypocketfft/test pysharp/test pyHealpix/test nifty_gridder/test pyinterpol_ng/test
+    - pytest-3 -q fft/test sht/test healpix/test wgridder/test totalconvolve/test
 
 release:
   stage: build_tarballs

MANIFEST.in

 prune .
 
 include src/mr_util/infra/aligned_array.h
-#include src/mr_util/infra/communication.cc
-#include src/mr_util/infra/communication.h
 include src/mr_util/infra/error_handling.h
 include src/mr_util/infra/mav.h
 include src/mr_util/infra/simd.h
@@ -13,8 +11,6 @@ include src/mr_util/infra/system.h
 include src/mr_util/infra/threading.cc
 include src/mr_util/infra/threading.h
 include src/mr_util/infra/timers.h
-#include src/mr_util/infra/types.cc
-#include src/mr_util/infra/types.h
 include src/mr_util/infra/useful_macros.h
 
 include src/mr_util/bindings/pybind_utils.h
@@ -52,15 +48,17 @@ include src/mr_util/healpix/healpix_base.h
 include src/mr_util/healpix/healpix_tables.cc
 include src/mr_util/healpix/healpix_tables.h
 
-include pypocketfft/pypocketfft.cc
+include python/fft.cc
+include python/sht.cc
+include python/healpix.cc
+include python/gridder_cxx.h
+include python/wgridder.cc
+include python/alm.h
+include python/totalconvolve.h
+include python/totalconvolve.cc
 
-include pysharp/pysharp.cc
-
-include pyHealpix/pyHealpix.cc
-
-include nifty_gridder/gridder_cxx.h
-include nifty_gridder/nifty_gridder.cc
-
-include pyinterpol_ng/alm.h
-include pyinterpol_ng/interpol_ng.h
-include pyinterpol_ng/pyinterpol_ng.cc
+include python/test/test_fft.py
+include python/test/test_healpix.py
+include python/test/test_sht.py
+include python/test/test_totalconvolve.py
+include python/test/test_wgridder.py

README.md

+DUCC 0.1
+========
+
+This is a collection of basic programming tools which can be handy in many
+situations.
+
+
+Installation
+------------
+
+### Requirements
+
+- [Python 3](https://www.python.org/)
+- [pybind11](https://github.com/pybind/pybind11)
+- a C++17-capable C++ compiler (e.g. g++ from version 7 on or a recent clang++)
+
+### Sources
+
+The current version of DUCC can be obtained by cloning the repository via
+
+    git clone https://gitlab.mpcdf.mpg.de/mtr/ducc.git
+
+### Installation
+
+In the following, we assume a Debian-based distribution. For other
+distributions, the "apt" lines will need slight changes.
+
+DUCC and its mandatory dependencies can be installed via:
+
+    sudo apt-get install git python3 python3-pip python3-dev python3-pybind11 pybind11-dev
+    pip3 install --user git+https://gitlab.mpcdf.mpg.de/mtr/ducc.git
+
+
+DUCC components
+===============
+
+ducc.fft
+--------
+
+This package provides Fast Fourier, trigonometric and Hartley transforms with a
+simple Python interface.
+
+The central algorithms are derived from Paul Swarztrauber's FFTPACK code
+(http://www.netlib.org/fftpack).
+
+Features
+- supports fully complex and half-complex (i.e. complex-to-real and
+  real-to-complex) FFTs, discrete sine/cosine transforms and Hartley transforms
+- achieves very high accuracy for all transforms
+- supports multidimensional arrays and selection of the axes to be transformed
+- supports single, double, and long double precision
+- makes use of CPU vector instructions when performing 2D and higher-dimensional
+  transforms
+- supports prime-length transforms without degrading to O(N**2) performance
+- has optional multi-threading support for multidimensional transforms
+
+
+ducc.sht
+--------
+
+This package provides efficient spherical harmonic trasforms (SHTs). Its code
+is derived from `libsharp`.
+
+
+ducc.healpix
+------------
+
+This library provides Python bindings for the most important
+functionality in Healpix C++. The design goals are
+- similarity to the C++ interface (while respecting some Python peculiarities)
+- simplicity (no optional function parameters)
+- low function calling overhead
+
+
+ducc.totalconvolve
+------------------
+
+Library for high-accuracy 4pi convolution on the sphere, which generates a
+total convolution data cube from a set of sky and beam `a_lm` and computes
+interpolated values for a given list of detector pointings.
+
+Algorithmic details:
+- the code uses `ducc.sht` SHTs to compute the data cube
+- shared-memory parallelization is provided via standard C++ threads.
+- for interpolation, the algorithm and kernel described in
+  https://arxiv.org/abs/1808.06736 are used. This allows very efficient
+  interpolation with user-adjustable accuracy.
+
+
+ducc.wgridder
+-------------
+
+Library for high-accuracy gridding/degridding of radio interferometry datasets
+
+Programming aspects
+- written in C++11, fully portable
+- shared-memory parallelization via and C++ threads.
+- Python interface available
+- kernel computation is performed on the fly, avoiding inaccuracies
+  due to table lookup and reducing overall memory bandwidth
+
+Numerical aspects
+- uses the analytical gridding kernel presented in
+  https://arxiv.org/abs/1808.06736
+- uses the "improved W-stacking method" described in
+  https://www.repository.cam.ac.uk/handle/1810/292298 (p. 139ff)
+- in combination these two aspects allow extremely accurate gridding/degridding
+  operations (L2 error compared to explicit DFTs can go below 1e-12) with
+  reasonable resource consumption

pysharp/demo.py → python/demos/sht_demo.py

-# Elementary demo for pysharp interface using a Gauss-Legendre grid
+# Elementary demo for the ducc_0_1.sht interface using a Gauss-Legendre grid
 # I'm not sure I have a perfect equivalent for the DH grid(s) at the moment,
 # since they apparently do not include the South Pole. The Clenshaw-Curtis
 # and Fejer quadrature rules are very similar (see the documentation in

ducc.cc → python/ducc.cc

@@ -13,19 +13,19 @@
 #include "mr_util/healpix/healpix_base.cc"
 
 #include <pybind11/pybind11.h>
-#include "pysharp/pysharp.cc"
-#include "pypocketfft/pypocketfft.cc"
-#include "pyinterpol_ng/pyinterpol_ng.cc"
-#include "nifty_gridder/nifty_gridder.cc"
-#include "pyHealpix/pyHealpix.cc"
+#include "python/sht.cc"
+#include "python/fft.cc"
+#include "python/totalconvolve.cc"
+#include "python/wgridder.cc"
+#include "python/healpix.cc"
 
 using namespace mr;
 
 PYBIND11_MODULE(PKGNAME, m)
   {
-  add_pypocketfft(m);
-  add_pysharp(m);
-  add_pyinterpol_ng(m);
-  add_nifty_gridder(m);
-  add_pyHealpix(m);
+  add_fft(m);
+  add_sht(m);
+  add_totalconvolve(m);
+  add_wgridder(m);
+  add_healpix(m);
   }

pypocketfft/pypocketfft.cc → python/fft.cc

@@ -21,7 +21,7 @@
 
 namespace mr {
 
-namespace detail_pypocketfft {
+namespace detail_pymodule_fft {
 
 namespace {
 
@@ -346,7 +346,7 @@ PyObject * good_size(PyObject * /*self*/, PyObject * args)
     real ? util1d::good_size_real(n) : util1d::good_size_cmplx(n));
   }
 
-const char *pypocketfft_DS = R"""(Fast Fourier and Hartley transforms.
+const char *fft_DS = R"""(Fast Fourier and Hartley transforms.
 
 This module supports
 - single, double, and long double precision
@@ -649,11 +649,11 @@ out : int
 
 } // unnamed namespace
 
-void add_pypocketfft(py::module &msup)
+void add_fft(py::module &msup)
   {
   using namespace pybind11::literals;
   auto m = msup.def_submodule("fft");
-  m.doc() = pypocketfft_DS;
+  m.doc() = fft_DS;
   m.def("c2c", c2c, c2c_DS, "a"_a, "axes"_a=None, "forward"_a=true,
     "inorm"_a=0, "out"_a=None, "nthreads"_a=1);
   m.def("r2c", r2c, r2c_DS, "a"_a, "axes"_a=None, "forward"_a=true,
@@ -678,6 +678,6 @@ void add_pypocketfft(py::module &msup)
 
 }
 
-using detail_pypocketfft::add_pypocketfft;
+using detail_pymodule_fft::add_fft;
 
 }

pyHealpix/pyHealpix.cc → python/healpix.cc

@@ -41,7 +41,7 @@
 
 namespace mr {
 
-namespace detail_pyHealpix {
+namespace detail_pymodule_healpix {
 
 using namespace std;
 using namespace mr;
@@ -245,7 +245,7 @@ py::array local_v_angle (const py::array &v1, const py::array &v2)
   return move(angle);
   }
 
-const char *pyHealpix_DS = R"""(
+const char *healpix_DS = R"""(
 Python interface for some of the HEALPix C++ functionality
 
 All angles are interpreted as radians.
@@ -345,11 +345,11 @@ that their last dimension is removed.
 The employed algorithm is highly accurate, even for angles close to 0 or pi.
 )""";
 
-void add_pyHealpix(py::module &msup)
+void add_healpix(py::module &msup)
   {
   using namespace pybind11::literals;
   auto m = msup.def_submodule("healpix");
-  m.doc() = pyHealpix_DS;
+  m.doc() = healpix_DS;
 
   py::class_<Pyhpbase> (m, "Healpix_Base", py::module_local())
     .def(py::init<int,const string &>(),"nside"_a,"scheme"_a)
@@ -385,6 +385,6 @@ void add_pyHealpix(py::module &msup)
 
 }
 
-using detail_pyHealpix::add_pyHealpix;
+using detail_pymodule_healpix::add_healpix;
 
 }

pysharp/pysharp.cc → python/sht.cc

@@ -44,7 +44,7 @@
 
 namespace mr {
 
-namespace detail_pysharp {
+namespace detail_pymodule_sht {
 
 using namespace std;
 using namespace mr;
@@ -188,7 +188,7 @@ template<typename T> class py_sharpjob
       }
   };
 
-const char *pysharp_DS = R"""(
+const char *sht_DS = R"""(
 Python interface for some of the libsharp functionality
 
 Error conditions are reported by raising exceptions.
@@ -268,11 +268,11 @@ py::array py_upsample_to_cc(const py::array &in, size_t nrings_out, bool has_np,
   return move(out);
   }
 
-void add_pysharp(py::module &msup)
+void add_sht(py::module &msup)
   {
   using namespace pybind11::literals;
   auto m = msup.def_submodule("sht");
-  m.doc() = pysharp_DS;
+  m.doc() = sht_DS;
 
   py::class_<py_sharpjob<double>> (m, "sharpjob_d", py::module_local())
     .def(py::init<>())
@@ -306,7 +306,7 @@ void add_pysharp(py::module &msup)
 
 }
 
-using detail_pysharp::add_pysharp;
+using detail_pymodule_sht::add_sht;
 
 }
 

pyinterpol_ng/pyinterpol_ng.cc → python/totalconvolve.cc

@@ -5,11 +5,11 @@
 
 #include <pybind11/pybind11.h>
 #include <pybind11/numpy.h>
-#include "interpol_ng.h"
+#include "totalconvolve.h"
 
 namespace mr {
 
-namespace detail_pyinterpol_ng {
+namespace detail_pymodule_totalconvolve {
 
 using namespace std;
 using namespace mr;
@@ -94,7 +94,7 @@ template<typename T> py::array pyrotate_alm(const py::array &alm_, int64_t lmax,
   }
 #endif
 
-constexpr const char *pyinterpol_ng_DS = R"""(
+constexpr const char *totalconvolve_DS = R"""(
 Python interface for total convolution/interpolation library
 
 All arrays containing spherical harmonic coefficients are assumed to have the
@@ -234,12 +234,12 @@ Notes
     - must be the last call to the object
 )""";
 
-void add_pyinterpol_ng(py::module &msup)
+void add_totalconvolve(py::module &msup)
   {
   using namespace pybind11::literals;
   auto m = msup.def_submodule("totalconvolve");
 
-  m.doc() = pyinterpol_ng_DS;
+  m.doc() = totalconvolve_DS;
 
   using inter_d = PyInterpolator<double>;
   py::class_<inter_d> (m, "PyInterpolator", py::module_local(), pyinterpolator_DS)
@@ -272,6 +272,6 @@ void add_pyinterpol_ng(py::module &msup)
 
 }
 
-using detail_pyinterpol_ng::add_pyinterpol_ng;
+using detail_pymodule_totalconvolve::add_totalconvolve;
 
 }

nifty_gridder/nifty_gridder.cc → python/wgridder.cc

@@ -26,7 +26,7 @@
 
 namespace mr {
 
-namespace detail_nifty_gridder {
+namespace detail_pymodule_wgridder {
 
 using namespace std;
 using namespace gridder;
@@ -190,7 +190,7 @@ np.array((nrows, nchan,), dtype=complex of same precision as `dirty`)
     the measurement set data.
 )""";
 
-void add_nifty_gridder(py::module &msup)
+void add_wgridder(py::module &msup)
   {
   using namespace pybind11::literals;
   auto m = msup.def_submodule("wgridder");
@@ -205,6 +205,6 @@ void add_nifty_gridder(py::module &msup)
 
 }
 
-using detail_nifty_gridder::add_nifty_gridder;
+using detail_pymodule_wgridder::add_wgridder;
 
 }

setup.py

@@ -22,7 +22,7 @@ def _get_files_by_suffix(directory, suffix):
     return list(itertools.chain.from_iterable(iterable_sources))
 
 
-include_dirs = ['./src/',
+include_dirs = ['.', './src/',
                 _deferred_pybind11_include(True),
                 _deferred_pybind11_include()]
 extra_compile_args = ['--std=c++17', '-march=native', '-ffast-math', '-O3']
@@ -47,7 +47,7 @@ def get_extension_modules():
     depfiles = _get_files_by_suffix('.', 'h') + _get_files_by_suffix('.', 'cc') + ['setup.py']
     return [Extension(pkgname,
                       language='c++',
-                      sources=['ducc.cc'],
+                      sources=['python/ducc.cc'],
                       depends=depfiles,
                       include_dirs=include_dirs,
                       define_macros=define_macros,