finalize split file output functionality

src/RMHD_converter.cpp

 #include "RMHD_converter.hpp"
+#include <string>
 
 extern int myrank, nprocs;
 
@@ -39,8 +40,16 @@ inline void zindex_to_grid3D(
 
 RMHD_converter::RMHD_converter(
         int n0, int n1, int n2,
-        int N0, int N1, int N2)
+        int N0, int N1, int N2,
+        int nfiles)
 {
+    if (nprocs % nfiles != 0)
+    {
+        std::cerr <<
+            "Number of output files incompatible with number of processes.\n"
+            "Aborting.\n" << std::endl;
+        exit(EXIT_FAILURE);
+    }
     int n[7];
 
     // first 3 arguments are dimensions for input array
@@ -106,6 +115,15 @@ RMHD_converter::RMHD_converter(
     n[1] = 8*8*8*2;
     this->dzcubbie = new field_descriptor(2, n, MPI_REAL4, MPI_COMM_WORLD);
 
+    //set up output file descriptor
+    int out_rank, out_nprocs;
+    out_nprocs = nprocs/nfiles;
+    this->out_group = myrank / out_nprocs;
+    out_rank = myrank % out_nprocs;
+    n[0] = ((N0/8) * (N1/8) * (N2/8)) / nfiles;
+    n[1] = 8*8*8*2;
+    MPI_Comm_split(MPI_COMM_WORLD, this->out_group, out_rank, &this->out_communicator);
+    this->dout = new field_descriptor(2, n, MPI_REAL4, this->out_communicator);
 }
 
 RMHD_converter::~RMHD_converter()
@@ -118,6 +136,9 @@ RMHD_converter::~RMHD_converter()
     if (this->f4r != NULL) delete this->f4r;
     if (this->drcubbie != NULL) delete this->drcubbie;
     if (this->dzcubbie != NULL) delete this->dzcubbie;
+    if (this->dout != NULL) delete this->dout;
+
+    MPI_Comm_free(&this->out_communicator);
 
     if (this->c0  != NULL) fftwf_free(this->c0);
     if (this->c12 != NULL) fftwf_free(this->c12);
@@ -232,7 +253,11 @@ int RMHD_converter::convert(
     rtmp = tpointer;
 
     fftwf_free(rtmp);
-    this->dzcubbie->write(ofile, (void*)this->r3);
+    char temp_char[200];
+    sprintf(temp_char, "%s_Z%.7lx\0", ofile, this->out_group*this->dout->sizes[0]);
+    this->dout->write(
+            temp_char,
+            this->r3 + this->out_group*this->dout->sizes[0]);
     return EXIT_SUCCESS;
 }
 

src/RMHD_converter.hpp

@@ -26,6 +26,13 @@ class RMHD_converter
         // descriptor for NZ x 8 x 8 x 8 x 2 array
         field_descriptor *dzcubbie = NULL;
 
+        // descriptor for (NZ/nfiles) x 8 x 8 x 8 x 2 array
+        field_descriptor *dout = NULL;
+
+        // communicator to use for output
+        MPI_Comm out_communicator;
+        int out_group;
+
         fftwf_complex *c0  = NULL; // array to store 2D input
         fftwf_complex *c12 = NULL; // array to store transposed input
         fftwf_complex *c3  = NULL; // array to store resized Fourier data
@@ -37,7 +44,8 @@ class RMHD_converter
         /* methods */
         RMHD_converter(
                 int n0, int n1, int n2,
-                int N0, int N1, int N2);
+                int N0, int N1, int N2,
+                int nfiles);
         ~RMHD_converter();
 
         int convert(

src/field_descriptor.cpp

@@ -60,7 +60,7 @@ int field_descriptor::read(
     MPI_File f;
 
     MPI_File_open(
-            MPI_COMM_WORLD,
+            this->comm,
             fname,
             MPI_MODE_RDONLY,
             info,
@@ -92,7 +92,7 @@ int field_descriptor::write(
     MPI_File f;
 
     MPI_File_open(
-            MPI_COMM_WORLD,
+            this->comm,
             fname,
             MPI_MODE_CREATE | MPI_MODE_WRONLY,
             info,
@@ -155,7 +155,7 @@ int field_descriptor::transpose(
     tplan = fftwf_mpi_plan_transpose(
             this->sizes[0], dim1,
             input, output,
-            MPI_COMM_WORLD,
+            this->comm,
             FFTW_ESTIMATE);
     fftwf_execute(tplan);
     fftwf_destroy_plan(tplan);
@@ -188,7 +188,7 @@ int field_descriptor::transpose(
                     FFTW_MPI_DEFAULT_BLOCK,
                     FFTW_MPI_DEFAULT_BLOCK,
                     (float*)input, (float*)output,
-                    MPI_COMM_WORLD,
+                    this->comm,
                     FFTW_ESTIMATE);
             fftwf_execute(tplan);
             fftwf_destroy_plan(tplan);

src/full.cpp

@@ -10,7 +10,8 @@ int main(int argc, char *argv[])
 
     RMHD_converter *bla = new RMHD_converter(
             atoi(argv[1]), atoi(argv[2]), atoi(argv[3]),
-            atoi(argv[4]), atoi(argv[5]), atoi(argv[6]));
+            atoi(argv[4]), atoi(argv[5]), atoi(argv[6]),
+            2);
 
     bla->convert("Kdata0", "Kdata1", "Rdata");
     delete bla;

test.ipynb

 {
  "metadata": {
   "name": "",
-  "signature": "sha256:ae900b72d559af23210256bab2cd50894706b98d9b53b4846df0f6db9ae56322"
+  "signature": "sha256:89382dc0480b222aff0422843abff870b50845f17d2bc41415431ef40ac300ba"
  },
  "nbformat": 3,
  "nbformat_minor": 0,
@@ -176,7 +176,7 @@
       "if subprocess.call(['make', 'full']) == 0:\n",
       "    subprocess.call(['mpirun.mpich',\n",
       "                     '-np',\n",
-      "                     '1',\n",
+      "                     '8',\n",
       "                     './full',\n",
       "                     '{0}'.format(n/2+1),\n",
       "                     '{0}'.format(n),\n",
@@ -184,16 +184,20 @@
       "                     '{0}'.format(N),\n",
       "                     '{0}'.format(N),\n",
       "                     '{0}'.format(N)])\n",
-      "    Rdata = np.fromfile(\n",
-      "        \"Rdata\",\n",
+      "    Rdata0 = np.fromfile(\n",
+      "        \"Rdata_Z0000000\",\n",
       "        dtype = np.float32).reshape(-1, 8, 8, 8, 2)\n",
+      "    Rdata1 = np.fromfile(\n",
+      "        \"Rdata_Z0000800\",\n",
+      "        dtype = np.float32).reshape(-1, 8, 8, 8, 2)\n",
+      "    Rdata = np.concatenate([Rdata0, Rdata1])\n",
       "else:\n",
       "    print ('compilation error')"
      ],
      "language": "python",
      "metadata": {},
      "outputs": [],
-     "prompt_number": 4
+     "prompt_number": 13
     },
     {
      "cell_type": "code",
@@ -218,13 +222,13 @@
        "output_type": "stream",
        "stream": "stdout",
        "text": [
-        "2.86102e-06\n",
-        "[12  9  2]\n",
-        "1634 1634 0.0\n"
+        "13.4568\n",
+        "[ 7 13  7]\n",
+        "1519 1519 0.0\n"
        ]
       }
      ],
-     "prompt_number": 6
+     "prompt_number": 14
     },
     {
      "cell_type": "code",
@@ -246,7 +250,7 @@
      "language": "python",
      "metadata": {},
      "outputs": [],
-     "prompt_number": 8
+     "prompt_number": 15
     },
     {
      "cell_type": "code",