From 34b2ecc4b21b171f8ac77d8002c65777e8091893 Mon Sep 17 00:00:00 2001
From: Chichi Lalescu <clalesc1@jhu.edu>
Date: Wed, 25 Feb 2015 09:34:40 -0500
Subject: [PATCH] fix mpi_array_type creation
I don't know if my current fix is the best option, but it seems to be a
working solution to the problem.
I use FFTW to figure out how to distribute the fields between CPUS.
This yields nice results for numbers of CPUs that are powers of 2, and
fields that have dimensions which are powers of 2 (or multiples of large
enough powers of 2).
However, it can keep one or more CPUs empty in case it decides that's
faster (and it probably is).
MPI is stupid, and when I try to call MPI_Type_create_subarray with
subsizes that are 0 it fails telling me that I'm an idiot.
Therefore what I do right now is to keep track of which processes
actually have data, and then only create the subarray for those.
I still need to check whether the results are now sane though.
---
src/base.hpp | 24 +++++++++++++++---------
src/field_descriptor.cpp | 34 ++++++++++++++++++----------------
2 files changed, 33 insertions(+), 25 deletions(-)
diff --git a/src/base.hpp b/src/base.hpp
index 310ce6cc..31c99876 100644
--- a/src/base.hpp
+++ b/src/base.hpp
@@ -29,22 +29,28 @@ inline void DEBUG_MSG(const char * format, ...)
std::cerr << debug_message_buffer;
}
-inline void DEBUG_WAIT(MPI_Comm communicator)
+inline void DEBUG_MSG_WAIT(MPI_Comm communicator, const char * format, ...)
{
+ va_list argptr;
+ va_start(argptr, format);
+ sprintf(
+ debug_message_buffer,
+ "cpu%.4d ",
+ myrank);
+ vsnprintf(
+ debug_message_buffer + 8,
+ message_buffer_length - 8,
+ format,
+ argptr);
+ va_end(argptr);
+ std::cerr << debug_message_buffer;
MPI_Barrier(communicator);
}
-#define DEBUG_WAIT_ALL() MPI_Barrier(MPI_COMM_WORLD)
-
-#define CHECK_POINT() DEBUG_MSG("%s %s", __FILE__, __LINE__)
-
#else
#define DEBUG_MSG(...)
-#define DEBUG_WAIT_ALL()
-#define DEBUG_WAIT(...)
-
-#define CHECK_POINT()
+#define DEBUG_MSG_WAIT(...)
#endif//NDEBUG
diff --git a/src/field_descriptor.cpp b/src/field_descriptor.cpp
index ac9c28bf..35341f96 100644
--- a/src/field_descriptor.cpp
+++ b/src/field_descriptor.cpp
@@ -11,7 +11,6 @@ field_descriptor::field_descriptor(
MPI_Comm COMM_TO_USE)
{
DEBUG_MSG("entered field_descriptor::field_descriptor\n");
- //CHECK_POINT();
this->comm = COMM_TO_USE;
MPI_Comm_rank(this->comm, &this->myrank);
MPI_Comm_size(this->comm, &this->nprocs);
@@ -45,14 +44,14 @@ field_descriptor::field_descriptor(
this->slice_size *= this->subsizes[i];
this->full_size *= this->sizes[i];
}
- DEBUG_MSG(
+ DEBUG_MSG_WAIT(
+ this->comm,
"inside field_descriptor constructor, about to call "
"MPI_Type_create_subarray\n"
"%d %d %d\n",
this->sizes[0],
this->subsizes[0],
this->starts[0]);
- DEBUG_WAIT_ALL();
int local_zero_array[this->nprocs], zero_array[this->nprocs];
for (int i=0; i<this->nprocs; i++)
local_zero_array[i] = 0;
@@ -130,27 +129,30 @@ field_descriptor::field_descriptor(
field_descriptor::~field_descriptor()
{
- delete[] this->sizes;
- delete[] this->subsizes;
- delete[] this->starts;
- delete[] this->rank;
- DEBUG_MSG(this->io_comm == MPI_COMM_NULL ? "null\n" : "not null\n");
- DEBUG_WAIT_ALL();
- DEBUG_MSG("subsizes[0] = %d \n", this->subsizes[0]);
- DEBUG_WAIT_ALL();
+ DEBUG_MSG_WAIT(
+ MPI_COMM_WORLD,
+ this->io_comm == MPI_COMM_NULL ? "null\n" : "not null\n");
+ DEBUG_MSG_WAIT(
+ MPI_COMM_WORLD,
+ "subsizes[0] = %d \n", this->subsizes[0]);
if (this->subsizes[0] > 0)
{
- DEBUG_MSG("deallocating mpi_array_dtype\n");
- DEBUG_WAIT(this->io_comm);
+ DEBUG_MSG_WAIT(
+ this->io_comm,
+ "deallocating mpi_array_dtype\n");
MPI_Type_free(&this->mpi_array_dtype);
}
- DEBUG_WAIT_ALL();
if (this->nprocs != this->io_nprocs && this->io_myrank != MPI_PROC_NULL)
{
- DEBUG_MSG("freeing io_comm\n");
- DEBUG_WAIT(this->io_comm);
+ DEBUG_MSG_WAIT(
+ this->io_comm,
+ "freeing io_comm\n");
MPI_Comm_free(&this->io_comm);
}
+ delete[] this->sizes;
+ delete[] this->subsizes;
+ delete[] this->starts;
+ delete[] this->rank;
}
int field_descriptor::read(
--
GitLab