From 209877f58051a1af32a14bbec9dd5735998dff4c Mon Sep 17 00:00:00 2001
From: Hartmut Kaiser <hartmut.kaiser@gmail.com>
Date: Thu, 19 Nov 2020 21:00:14 -0600
Subject: [PATCH] Make sure spmd_init finishes before returning

---
 flecsi/execution/hpx/runtime_driver.cc | 20 ++++++++++++++++++++
 1 file changed, 20 insertions(+)

diff --git a/flecsi/execution/hpx/runtime_driver.cc b/flecsi/execution/hpx/runtime_driver.cc
index 8c3c4bc1b..1655e8030 100644
--- a/flecsi/execution/hpx/runtime_driver.cc
+++ b/flecsi/execution/hpx/runtime_driver.cc
@@ -13,6 +13,9 @@
                                                                               */
 /*! @file */
 
+#include <hpx/modules/execution_base.hpp> // yield_while
+#include <hpx/modules/runtime_local.hpp> // get_thread_manager
+
 #include <cstddef>
 #include <cstdint>
 
@@ -27,6 +30,17 @@ clog_register_tag(runtime_driver);
 namespace flecsi {
 namespace execution {
 
+void
+termination_detection() {
+  auto & tm = hpx::threads::get_thread_manager();
+  hpx::util::yield_while(
+    [&tm]() -> bool {
+      return tm.get_thread_count() >
+             std::int64_t(1) + tm.get_background_thread_count();
+    },
+    "termination_detection");
+}
+
 //----------------------------------------------------------------------------//
 // Implementation of FleCSI runtime driver task.
 //----------------------------------------------------------------------------//
@@ -97,6 +111,9 @@ hpx_runtime_driver(int argc, char ** argv) {
   // Execute the specialization driver.
   specialization_tlt_init(argc, argv);
   annotation::end<annotation::spl_tlt_init>();
+
+  // make sure all activity has ceased
+  termination_detection();
 #endif // FLECSI_ENABLE_SPECIALIZATION_TLT_INIT
 
   remap_shared_entities();
@@ -149,6 +166,9 @@ hpx_runtime_driver(int argc, char ** argv) {
   annotation::begin<annotation::spl_spmd_init>();
   specialization_spmd_init(argc, argv);
   annotation::end<annotation::spl_spmd_init>();
+
+  // make sure all activity has ceased
+  termination_detection();
 #endif // FLECSI_ENABLE_SPECIALIZATION_SPMD_INIT
 
   context_.advance_state();