diff --git a/src/io/restart.cc b/src/io/restart.cc index 1f2912ad49b1a3c4aa3e60bd0070e6d905d144cb..af956a28f49ab69289c4d1185be56c44a432e6a0 100644 --- a/src/io/restart.cc +++ b/src/io/restart.cc @@ -298,6 +298,9 @@ void restart::work_files(int modus) MPI_Gather(&seq_loc, sizeof(seq_data), MPI_BYTE, seq, sizeof(seq_data), MPI_BYTE, 0, Communicator); + if(modus == MODUS_READ) + MPI_Comm_split(Communicator, Shmem.Island_Smallest_WorldTask, 0, &Sim->NgbTree.TreeSharedMemComm); + if(ThisTask == 0) { std::sort(seq, seq + NTask); @@ -357,6 +360,9 @@ void restart::work_files(int modus) /* send back completion notice */ MPI_Ssend(&ThisTask, 1, MPI_INT, 0, TAG_KEY, Communicator); } + + if(modus == MODUS_READ) + Sim->NgbTree.treeallocate_share_topnode_addresses(); } void restart::contents_restart_file(int modus) @@ -523,12 +529,16 @@ void restart::contents_restart_file(int modus) { byten(Sim->NgbTree.Nodes + Sim->NgbTree.MaxPart + Sim->Domain.NTopnodes, (Sim->NgbTree.NumNodes - Sim->Domain.NTopnodes) * sizeof(ngbnode), modus); - byten(Sim->NgbTree.TopNodes + Sim->NgbTree.MaxPart, Sim->Domain.NTopnodes * sizeof(ngbnode), modus); - byten(Sim->NgbTree.NodeIndex, Sim->Domain.NTopleaves * sizeof(int), modus); - byten(Sim->NgbTree.NodeSibling, Sim->Domain.NTopleaves * sizeof(int), modus); - byten(Sim->NgbTree.NodeLevel, Sim->Domain.NTopleaves * sizeof(unsigned char), modus); byten(Sim->NgbTree.Nextnode, (Sim->NgbTree.MaxPart + Sim->Domain.NTopleaves) * sizeof(int), modus); byten(Sim->NgbTree.Father, Sim->NgbTree.MaxPart * sizeof(int), modus); + + if(Sim->NgbTree.TreeSharedMem_ThisTask == 0) + { + byten(Sim->NgbTree.TopNodes + Sim->NgbTree.MaxPart, Sim->Domain.NTopnodes * sizeof(ngbnode), modus); + byten(Sim->NgbTree.NodeIndex, Sim->Domain.NTopleaves * sizeof(int), modus); + byten(Sim->NgbTree.NodeSibling, Sim->Domain.NTopleaves * sizeof(int), modus); + byten(Sim->NgbTree.NodeLevel, Sim->Domain.NTopleaves * sizeof(unsigned char), modus); + } } byten(Sim->Domain.TopNodes, Sim->Domain.NTopnodes * Sim->Domain.domain_sizeof_topnode_data(), modus); diff --git a/src/tree/tree.cc b/src/tree/tree.cc index c7be9840ef45e0aa231e4fe109aa17c5b29378aa..a487a55824f9a367f59b4875d28516ca8ce38086 100644 --- a/src/tree/tree.cc +++ b/src/tree/tree.cc @@ -778,10 +778,12 @@ void tree<node, partset, point_data, foreign_point_data>::treeallocate(int max_p D = Dptr; Tp = Tp_ptr; + /* split up the communicator into pieces overlap with different shared memory regions */ if(max_partindex != -1) - { - MPI_Allreduce(&max_partindex, &MaxPart, 1, MPI_INT, MPI_MAX, D->Communicator); - } + MPI_Comm_split(D->Communicator, Shmem.Island_Smallest_WorldTask, 0, &TreeSharedMemComm); + + if(max_partindex != -1) + MPI_Allreduce(&max_partindex, &MaxPart, 1, MPI_INT, MPI_MAX, D->Communicator); if(MaxPart == 0) return; // nothing to be done @@ -816,9 +818,6 @@ void tree<node, partset, point_data, foreign_point_data>::treeallocate(int max_p max_partindex = MaxPart; } - /* now split up the communicator into pieces overlap with different shared memory regions */ - MPI_Comm_split(D->Communicator, Shmem.Island_Smallest_WorldTask, 0, &TreeSharedMemComm); - MPI_Comm_rank(TreeSharedMemComm, &TreeSharedMem_ThisTask); MPI_Comm_size(TreeSharedMemComm, &TreeSharedMem_NTask); @@ -880,6 +879,16 @@ void tree<node, partset, point_data, foreign_point_data>::treeallocate(int max_p } } + Nodes = (node *)Mem.mymalloc_movable(&Nodes, "Nodes", (MaxNodes - D->NTopnodes + 1) * sizeof(node)); + Nodes -= (MaxPart + D->NTopnodes); + + if(max_partindex != -1) + treeallocate_share_topnode_addresses(); +} + +template <typename node, typename partset, typename point_data, typename foreign_point_data> +void tree<node, partset, point_data, foreign_point_data>::treeallocate_share_topnode_addresses(void) +{ MPI_Bcast(&TreeInfoHandle, 1, MPI_INT, 0, TreeSharedMemComm); ptrdiff_t off[4] = {((char *)NodeLevel - Mem.Base), ((char *)NodeSibling - Mem.Base), ((char *)NodeIndex - Mem.Base), @@ -894,9 +903,6 @@ void tree<node, partset, point_data, foreign_point_data>::treeallocate(int max_p NodeSibling = (int *)((char *)Shmem.SharedMemBaseAddr[shmrank] + off[1]); NodeIndex = (int *)((char *)Shmem.SharedMemBaseAddr[shmrank] + off[2]); TopNodes = (node *)((char *)Shmem.SharedMemBaseAddr[shmrank] + off[3]); - - Nodes = (node *)Mem.mymalloc_movable(&Nodes, "Nodes", (MaxNodes - D->NTopnodes + 1) * sizeof(node)); - Nodes -= (MaxPart + D->NTopnodes); } template <typename node, typename partset, typename point_data, typename foreign_point_data> @@ -1221,13 +1227,13 @@ void tree<node, partset, point_data, foreign_point_data>::tree_fetch_foreign_nod template <typename node, typename partset, typename point_data, typename foreign_point_data> void tree<node, partset, point_data, foreign_point_data>::treefree(void) { + MPI_Comm_free(&TreeSharedMemComm); + if(MaxPart == 0) return; // nothing to be done if(Nodes) { - MPI_Comm_free(&TreeSharedMemComm); - if(Father) { Mem.myfree_movable(Father); diff --git a/src/tree/tree.h b/src/tree/tree.h index eddc65c160b6d61b6c25e31f8cd3295b8b1e92d9..5385a880d2f82817226262434623fcce535306d6 100644 --- a/src/tree/tree.h +++ b/src/tree/tree.h @@ -321,6 +321,7 @@ class tree int treebuild(int ninsert, int *indexlist); void treefree(void); void treeallocate(int max_partindex, partset *Pptr, domain<partset> *Dptr); + void treeallocate_share_topnode_addresses(void); void tree_export_node_threads(int no, int i, thread_data *thread, offset_tuple off = 0); void tree_export_node_threads_by_task_and_node(int task, int nodeindex, int i, thread_data *thread, offset_tuple off = 0);