big bunch of reorganization

.gitlab-ci.yml

@@ -2,7 +2,6 @@ image: $CONTAINER_TEST_IMAGE
 
 variables:
   CONTAINER_TEST_IMAGE: gitlab-registry.mpcdf.mpg.de/$CI_PROJECT_PATH:$CI_BUILD_REF_NAME
-  OMP_NUM_THREADS: 2
 
 stages:
   - build_docker
@@ -21,13 +20,13 @@ test_gcc:
   stage: testing
   script:
     - python3 setup.py install --user -f
-    - pytest-3 -q pypocketfft/test pysharp/test pyHealpix/test nifty_gridder/test pyinterpol_ng/test
+    - pytest-3 -q fft/test sht/test healpix/test wgridder/test totalconvolve/test
 
 test_clang:
   stage: testing
   script:
     - CC="clang -fsized-deallocation" python3 setup.py install --user -f
-    - pytest-3 -q pypocketfft/test pysharp/test pyHealpix/test nifty_gridder/test pyinterpol_ng/test
+    - pytest-3 -q fft/test sht/test healpix/test wgridder/test totalconvolve/test
 
 release:
   stage: build_tarballs

MANIFEST.in

 prune .
 
 include src/mr_util/infra/aligned_array.h
-#include src/mr_util/infra/communication.cc
-#include src/mr_util/infra/communication.h
 include src/mr_util/infra/error_handling.h
 include src/mr_util/infra/mav.h
 include src/mr_util/infra/simd.h
@@ -13,8 +11,6 @@ include src/mr_util/infra/system.h
 include src/mr_util/infra/threading.cc
 include src/mr_util/infra/threading.h
 include src/mr_util/infra/timers.h
-#include src/mr_util/infra/types.cc
-#include src/mr_util/infra/types.h
 include src/mr_util/infra/useful_macros.h
 
 include src/mr_util/bindings/pybind_utils.h
@@ -52,15 +48,17 @@ include src/mr_util/healpix/healpix_base.h
 include src/mr_util/healpix/healpix_tables.cc
 include src/mr_util/healpix/healpix_tables.h
 
-include pypocketfft/pypocketfft.cc
+include python/fft.cc
+include python/sht.cc
+include python/healpix.cc
+include python/gridder_cxx.h
+include python/wgridder.cc
+include python/alm.h
+include python/totalconvolve.h
+include python/totalconvolve.cc
 
-include pysharp/pysharp.cc
-
-include pyHealpix/pyHealpix.cc
-
-include nifty_gridder/gridder_cxx.h
-include nifty_gridder/nifty_gridder.cc
-
-include pyinterpol_ng/alm.h
-include pyinterpol_ng/interpol_ng.h
-include pyinterpol_ng/pyinterpol_ng.cc
+include python/test/test_fft.py
+include python/test/test_healpix.py
+include python/test/test_sht.py
+include python/test/test_totalconvolve.py
+include python/test/test_wgridder.py

README.md

+DUCC 0.1
+========
+
+This is a collection of basic programming tools which can be handy in many
+situations.
+
+
+Installation
+------------
+
+### Requirements
+
+- [Python 3](https://www.python.org/)
+- [pybind11](https://github.com/pybind/pybind11)
+- a C++17-capable C++ compiler (e.g. g++ from version 7 on or a recent clang++)
+
+### Sources
+
+The current version of DUCC can be obtained by cloning the repository via
+
+    git clone https://gitlab.mpcdf.mpg.de/mtr/ducc.git
+
+### Installation
+
+In the following, we assume a Debian-based distribution. For other
+distributions, the "apt" lines will need slight changes.
+
+DUCC and its mandatory dependencies can be installed via:
+
+    sudo apt-get install git python3 python3-pip python3-dev python3-pybind11 pybind11-dev
+    pip3 install --user git+https://gitlab.mpcdf.mpg.de/mtr/ducc.git
+
+
+DUCC components
+===============
+
+ducc.fft
+--------
+
+This package provides Fast Fourier, trigonometric and Hartley transforms with a
+simple Python interface.
+
+The central algorithms are derived from Paul Swarztrauber's FFTPACK code
+(http://www.netlib.org/fftpack).
+
+Features
+- supports fully complex and half-complex (i.e. complex-to-real and
+  real-to-complex) FFTs, discrete sine/cosine transforms and Hartley transforms
+- achieves very high accuracy for all transforms
+- supports multidimensional arrays and selection of the axes to be transformed
+- supports single, double, and long double precision
+- makes use of CPU vector instructions when performing 2D and higher-dimensional
+  transforms
+- supports prime-length transforms without degrading to O(N**2) performance
+- has optional multi-threading support for multidimensional transforms
+
+
+ducc.sht
+--------
+
+This package provides efficient spherical harmonic trasforms (SHTs). Its code
+is derived from `libsharp`.
+
+
+ducc.healpix
+------------
+
+This library provides Python bindings for the most important
+functionality in Healpix C++. The design goals are
+- similarity to the C++ interface (while respecting some Python peculiarities)
+- simplicity (no optional function parameters)
+- low function calling overhead
+
+
+ducc.totalconvolve
+------------------
+
+Library for high-accuracy 4pi convolution on the sphere, which generates a
+total convolution data cube from a set of sky and beam `a_lm` and computes
+interpolated values for a given list of detector pointings.
+
+Algorithmic details:
+- the code uses `ducc.sht` SHTs to compute the data cube
+- shared-memory parallelization is provided via standard C++ threads.
+- for interpolation, the algorithm and kernel described in
+  https://arxiv.org/abs/1808.06736 are used. This allows very efficient
+  interpolation with user-adjustable accuracy.
+
+
+ducc.wgridder
+-------------
+
+Library for high-accuracy gridding/degridding of radio interferometry datasets
+
+Programming aspects
+- written in C++11, fully portable
+- shared-memory parallelization via and C++ threads.
+- Python interface available
+- kernel computation is performed on the fly, avoiding inaccuracies
+  due to table lookup and reducing overall memory bandwidth
+
+Numerical aspects
+- uses the analytical gridding kernel presented in
+  https://arxiv.org/abs/1808.06736
+- uses the "improved W-stacking method" described in
+  https://www.repository.cam.ac.uk/handle/1810/292298 (p. 139ff)
+- in combination these two aspects allow extremely accurate gridding/degridding
+  operations (L2 error compared to explicit DFTs can go below 1e-12) with
+  reasonable resource consumption

pysharp/demo.py → python/demos/sht_demo.py

-# Elementary demo for pysharp interface using a Gauss-Legendre grid
+# Elementary demo for the ducc_0_1.sht interface using a Gauss-Legendre grid
 # I'm not sure I have a perfect equivalent for the DH grid(s) at the moment,
 # since they apparently do not include the South Pole. The Clenshaw-Curtis
 # and Fejer quadrature rules are very similar (see the documentation in

ducc.cc → python/ducc.cc

@@ -13,19 +13,19 @@
 #include "mr_util/healpix/healpix_base.cc"
 
 #include <pybind11/pybind11.h>
-#include "pysharp/pysharp.cc"
-#include "pypocketfft/pypocketfft.cc"
-#include "pyinterpol_ng/pyinterpol_ng.cc"
-#include "nifty_gridder/nifty_gridder.cc"
-#include "pyHealpix/pyHealpix.cc"
+#include "python/sht.cc"
+#include "python/fft.cc"
+#include "python/totalconvolve.cc"
+#include "python/wgridder.cc"
+#include "python/healpix.cc"
 
 using namespace mr;
 
 PYBIND11_MODULE(PKGNAME, m)
   {
-  add_pypocketfft(m);
-  add_pysharp(m);
-  add_pyinterpol_ng(m);
-  add_nifty_gridder(m);
-  add_pyHealpix(m);
+  add_fft(m);
+  add_sht(m);
+  add_totalconvolve(m);
+  add_wgridder(m);
+  add_healpix(m);
   }

pypocketfft/pypocketfft.cc → python/fft.cc

@@ -21,7 +21,7 @@
 
 namespace mr {
 
-namespace detail_pypocketfft {
+namespace detail_pymodule_fft {
 
 namespace {
 
@@ -346,7 +346,7 @@ PyObject * good_size(PyObject * /*self*/, PyObject * args)
     real ? util1d::good_size_real(n) : util1d::good_size_cmplx(n));
   }
 
-const char *pypocketfft_DS = R"""(Fast Fourier and Hartley transforms.
+const char *fft_DS = R"""(Fast Fourier and Hartley transforms.
 
 This module supports
 - single, double, and long double precision
@@ -649,11 +649,11 @@ out : int
 
 } // unnamed namespace
 
-void add_pypocketfft(py::module &msup)
+void add_fft(py::module &msup)
   {
   using namespace pybind11::literals;
   auto m = msup.def_submodule("fft");
-  m.doc() = pypocketfft_DS;
+  m.doc() = fft_DS;
   m.def("c2c", c2c, c2c_DS, "a"_a, "axes"_a=None, "forward"_a=true,
     "inorm"_a=0, "out"_a=None, "nthreads"_a=1);
   m.def("r2c", r2c, r2c_DS, "a"_a, "axes"_a=None, "forward"_a=true,
@@ -678,6 +678,6 @@ void add_pypocketfft(py::module &msup)
 
 }
 
-using detail_pypocketfft::add_pypocketfft;
+using detail_pymodule_fft::add_fft;
 
 }

pyHealpix/pyHealpix.cc → python/healpix.cc

@@ -41,7 +41,7 @@
 
 namespace mr {
 
-namespace detail_pyHealpix {
+namespace detail_pymodule_healpix {
 
 using namespace std;
 using namespace mr;
@@ -245,7 +245,7 @@ py::array local_v_angle (const py::array &v1, const py::array &v2)
   return move(angle);
   }
 
-const char *pyHealpix_DS = R"""(
+const char *healpix_DS = R"""(
 Python interface for some of the HEALPix C++ functionality
 
 All angles are interpreted as radians.
@@ -345,11 +345,11 @@ that their last dimension is removed.
 The employed algorithm is highly accurate, even for angles close to 0 or pi.
 )""";
 
-void add_pyHealpix(py::module &msup)
+void add_healpix(py::module &msup)
   {
   using namespace pybind11::literals;
   auto m = msup.def_submodule("healpix");
-  m.doc() = pyHealpix_DS;
+  m.doc() = healpix_DS;
 
   py::class_<Pyhpbase> (m, "Healpix_Base", py::module_local())
     .def(py::init<int,const string &>(),"nside"_a,"scheme"_a)
@@ -385,6 +385,6 @@ void add_pyHealpix(py::module &msup)
 
 }
 
-using detail_pyHealpix::add_pyHealpix;
+using detail_pymodule_healpix::add_healpix;
 
 }

pysharp/pysharp.cc → python/sht.cc

@@ -44,7 +44,7 @@
 
 namespace mr {
 
-namespace detail_pysharp {
+namespace detail_pymodule_sht {
 
 using namespace std;
 using namespace mr;
@@ -188,7 +188,7 @@ template<typename T> class py_sharpjob
       }
   };
 
-const char *pysharp_DS = R"""(
+const char *sht_DS = R"""(
 Python interface for some of the libsharp functionality
 
 Error conditions are reported by raising exceptions.
@@ -268,11 +268,11 @@ py::array py_upsample_to_cc(const py::array &in, size_t nrings_out, bool has_np,
   return move(out);
   }
 
-void add_pysharp(py::module &msup)
+void add_sht(py::module &msup)
   {
   using namespace pybind11::literals;
   auto m = msup.def_submodule("sht");
-  m.doc() = pysharp_DS;
+  m.doc() = sht_DS;
 
   py::class_<py_sharpjob<double>> (m, "sharpjob_d", py::module_local())
     .def(py::init<>())
@@ -306,7 +306,7 @@ void add_pysharp(py::module &msup)
 
 }
 
-using detail_pysharp::add_pysharp;
+using detail_pymodule_sht::add_sht;
 
 }