From 2d2fcea8181c606e131341ca1cf93ff707689378 Mon Sep 17 00:00:00 2001
From: Oliver Zier <ozier@web.de>
Date: Thu, 8 Oct 2020 21:29:02 +0200
Subject: [PATCH] Fix init_entropy by adding synchronization
---
src/pm/pm_periodic.cc | 1 -
src/sph/init_entropy.cc | 34 +++++++++++++++++++++++++++-------
2 files changed, 27 insertions(+), 8 deletions(-)
diff --git a/src/pm/pm_periodic.cc b/src/pm/pm_periodic.cc
index 1eee5b5..7667d94 100644
--- a/src/pm/pm_periodic.cc
+++ b/src/pm/pm_periodic.cc
@@ -253,7 +253,6 @@ void pm_periodic::pmforce_zoom_optimized_prepare_density(int mode, int *typelist
large_numpart_type num_on_grid = ((large_numpart_type)NSource) << 3;
/* bring the part-field into the order of the accessed cells. This allows the removal of duplicates */
- // mycxxsort(part_sortindex, part_sortindex + num_on_grid, pm_periodic_compare_sortindex, part);
mycxxsort(part_sortindex, part_sortindex + num_on_grid, pm_periodic_sortindex_comparator(part));
diff --git a/src/sph/init_entropy.cc b/src/sph/init_entropy.cc
index 231eeb4..7ef8c71 100644
--- a/src/sph/init_entropy.cc
+++ b/src/sph/init_entropy.cc
@@ -78,21 +78,41 @@ void sph::init_entropy(void)
Foreign_Points = (foreign_sphpoint_data *)Mem.mymalloc_movable(&Foreign_Points, "Foreign_Points",
MaxForeignPoints * sizeof(foreign_sphpoint_data));
- tree_initialize_leaf_node_access_info();
- max_ncycles = 0;
-
- prepare_shared_memory_access();
double tstart = Logs.second();
+ int global_left_particles = 0;
+
+ MPI_Allreduce(&ndensities, &global_left_particles, 1, MPI_INT, MPI_SUM, D->Communicator);
+
do
{
double t0 = Logs.second();
+ /* Since EntropyToInvGammaPred of remote particles can change, we have to import the particles in every iteration */
+
+ MaxForeignNodes = nspace;
+ MaxForeignPoints = 8 * nspace;
+ NumForeignNodes = 0;
+ NumForeignPoints = 0;
+
+ sum_NumForeignNodes = 0;
+ sum_NumForeignPoints = 0;
+
+ tree_initialize_leaf_node_access_info();
+
+ max_ncycles = 0;
+
+ prepare_shared_memory_access();
+
/* now do the primary work with this call */
densities_determine(ndensities, targetList);
+ MPI_Allreduce(MPI_IN_PLACE, &max_ncycles, 1, MPI_INT, MPI_MAX, D->Communicator);
+
+ cleanup_shared_memory_access();
+
/* do final operations on results */
double entropy_old;
@@ -132,6 +152,8 @@ void sph::init_entropy(void)
ndensities = npleft;
+ MPI_Allreduce(&ndensities, &global_left_particles, 1, MPI_INT, MPI_SUM, D->Communicator);
+
double t1 = Logs.second();
if(npleft > 0)
@@ -147,13 +169,11 @@ void sph::init_entropy(void)
else
D->mpi_printf("SPH-INIT-ENTROPY: ngb iteration %4d: took %8.3f\n", ++iter, Logs.timediff(t0, t1));
}
- while(ndensities > 0);
+ while(global_left_particles > 0);
- MPI_Allreduce(MPI_IN_PLACE, &max_ncycles, 1, MPI_INT, MPI_MAX, D->Communicator);
TIMER_STOP(CPU_DENSITY);
- cleanup_shared_memory_access();
/* free temporary buffers */
--
GitLab