Enable the ORTE progress thread for apps (not needed in the tools as they already continuously loop in the event lib). This appears to be working, at least for MPI apps that only use shared memory (a simple "hello"). More testing is required to identify where problems will occur - this is only intended to allow further development.

In order to use the progress thread, you must configure with:

--enable-orte-progress-threads --enable-event-thread-support

This commit was SVN r26457.

configure.ac

@@ -155,10 +155,6 @@ AC_MSG_RESULT([$ltversion])
 #
 # Hardwire all progress threads to be off
 #
-enable_progress_threads="no"
-ORTE_ENABLE_PROGRESS_THREADS=0
-AC_DEFINE_UNQUOTED([ORTE_ENABLE_PROGRESS_THREADS], [$ORTE_ENABLE_PROGRESS_THREADS],
-                   [Hardcode the ORTE progress thread to be off])
 OMPI_ENABLE_PROGRESS_THREADS=0
 AC_DEFINE_UNQUOTED([OMPI_ENABLE_PROGRESS_THREADS], [$OMPI_ENABLE_PROGRESS_THREADS],
                    [Hardcode the OMPI progress thread to be off])

opal/mca/event/libevent2019/Makefile.am

@@ -85,3 +85,7 @@ noinst_LTLIBRARIES = $(component_noinst)
 libmca_event_libevent2019_la_SOURCES =$(sources)
 libmca_event_libevent2019_la_LDFLAGS = -module -avoid-version
 libmca_event_libevent2019_la_LIBADD = $(builddir)/libevent/libevent.la
+if OPAL_EVENT_HAVE_THREAD_SUPPORT
+libmca_event_libevent2019_la_LIBADD += $(builddir)/libevent/libevent_pthreads.la
+endif
+

opal/mca/event/libevent2019/configure.m4

@@ -94,7 +94,7 @@ AC_DEFUN([MCA_opal_event_libevent2019_CONFIG],[
 
     AC_ARG_ENABLE(event-thread-support,
                   AC_HELP_STRING([--enable-event-thread-support], [enable event library internal thread support]))
-    if test "$enable_event_thread_support" = "yes"; then
+    if test "$enable_event_thread_support" = "yes" -o "$require_event_thread_support" = 1; then
         AC_DEFINE_UNQUOTED(OPAL_EVENT_HAVE_THREAD_SUPPORT, 1,
                            [Thread support was configured into the event library])
     else
@@ -102,6 +102,7 @@ AC_DEFUN([MCA_opal_event_libevent2019_CONFIG],[
         AC_DEFINE_UNQUOTED(OPAL_EVENT_HAVE_THREAD_SUPPORT, 0,
                            [Thread support was not configured into the event library])
     fi
+    AM_CONDITIONAL(OPAL_EVENT_HAVE_THREAD_SUPPORT, test "$enable_event_thread_support" = "yes")
 
     AC_MSG_RESULT([$event_args])
 

orte/config/orte_configure_options.m4

@@ -115,21 +115,23 @@ AC_DEFINE_UNQUOTED([ORTE_ENABLE_HEARTBEAT],
 
 #
 # Do we want a separate orte progress thread?
-AC_MSG_CHECKING([if want orte progress thread])
+AC_MSG_CHECKING([if want orte progress threads])
-AC_ARG_ENABLE([orte-progress-thread],
+AC_ARG_ENABLE([orte-progress-threads],
-              [AC_HELP_STRING([--enable-orte-progress-thread],
+              [AC_HELP_STRING([--enable-orte-progress-threads],
               [Enable orte progress thread - for experiment by developers only! (default: disabled)])])
-if test "$enable_orte_progress_thread" = "yes"; then
+if test "$enable_orte_progress_threads" = "yes"; then
     AC_MSG_RESULT([yes])
-    orte_enable_progress_thread=1
+    orte_enable_progress_threads=1
+    require_event_thread_support=1
     AC_DEFINE_UNQUOTED(OPAL_EVENT_HAVE_THREAD_SUPPORT, 1,
                       [Thread support must be configured into the event library])
 else
     AC_MSG_RESULT([no])
-    orte_enable_progress_thread=0
+    orte_enable_progress_threads=0
+    require_event_thread_support=0
 fi
-AC_DEFINE_UNQUOTED([ORTE_ENABLE_PROGRESS_THREAD],
+AC_DEFINE_UNQUOTED([ORTE_ENABLE_PROGRESS_THREADS],
-                   [$orte_enable_progress_thread],
+                   [$orte_enable_progress_threads],
-                   [Whether we want an orte progress thread enabled])
+                   [Whether we want orte progress threads enabled])
 
 ])dnl

orte/mca/oob/tcp/oob_tcp_recv.c

@@ -179,7 +179,7 @@ int mca_oob_tcp_recv_cancel(
 
     /* wait for any previously matched messages to be processed */
     OPAL_THREAD_LOCK(&mca_oob_tcp_component.tcp_match_lock);
-#if ORTE_ENABLE_PROGRESS_THREADS
+#if 0 && ORTE_ENABLE_PROGRESS_THREADS
     if(opal_event_progress_thread() == false) {
         while(mca_oob_tcp_component.tcp_match_count) {
             opal_condition_wait(

orte/mca/routed/base/routed_base_fns.c

@@ -23,6 +23,13 @@
 #include "orte/constants.h"
 #include "orte/types.h"
 
+#if HAVE_TIME_H
+#include <time.h>
+#endif
+#if HAVE_SYS_TIME_H
+#include <sys/time.h>
+#endif
+
 #include "opal/dss/dss.h"
 #include "opal/runtime/opal_progress.h"
 
@@ -307,7 +314,17 @@ int orte_routed_base_register_sync(bool setup)
     
     /* it is okay to block here as we are -not- in an event */
     while (!sync_recvd) {
+#if !ORTE_ENABLE_PROGRESS_THREADS
         opal_progress();
+#else
+        {
+            /* provide a very short quiet period so we
+             * don't hammer the cpu while we wait
+             */
+            struct timespec tp = {0, 100};
+            nanosleep(&tp, NULL);
+        }
+#endif
     }
     
     OPAL_OUTPUT_VERBOSE((5, orte_routed_base_output,

orte/runtime/orte_finalize.c

@@ -59,15 +59,6 @@ int orte_finalize(void)
     /* close the orte_show_help system */
     orte_show_help_finalize();
 
-#if !ORTE_DISABLE_FULL_SUPPORT && ORTE_ENABLE_PROGRESS_THREAD
-    if (ORTE_PROC_IS_APP) {
-        /* stop the progress thread */
-        orte_event_base_active = false;
-        opal_thread_join(&orte_progress_thread, NULL);
-        OBJ_DESTRUCT(&orte_progress_thread);
-    }
-#endif
-
     /* call the finalize function for this environment */
     orte_ess.finalize();
     
@@ -85,6 +76,21 @@ int orte_finalize(void)
     if( NULL != orte_default_hostfile ) {
         free(orte_default_hostfile);
     }
+#if ORTE_ENABLE_PROGRESS_THREADS
+    if (ORTE_PROC_IS_APP) {
+        /* stop the progress thread */
+        orte_event_base_active = false;
+        /* must trigger the "finalize" event to break us
+         * out of the event loop
+         */
+        opal_event_active(&orte_finalize_event, OPAL_EV_WRITE, 1);
+        /* wait for thread to exit */
+        opal_thread_join(&orte_progress_thread, NULL);
+        OBJ_DESTRUCT(&orte_progress_thread);
+        /* release the event base */
+        opal_event_base_free(orte_event_base);
+    }
+#endif
 #endif
 
     /* Close the general debug stream */

orte/runtime/orte_globals.c

@@ -188,8 +188,9 @@ bool orte_map_reduce = false;
 bool orte_map_stddiag_to_stderr = false;
 
 /* progress thread */
-#if ORTE_ENABLE_PROGRESS_THREAD
+#if ORTE_ENABLE_PROGRESS_THREADS
 opal_thread_t orte_progress_thread;
+opal_event_t orte_finalize_event;
 #endif
 
 #endif /* !ORTE_DISABLE_FULL_RTE */

orte/runtime/orte_globals.h

@@ -137,8 +137,9 @@ typedef struct orte_app_context_t orte_app_context_t;
 
 #else
 
-#if ORTE_ENABLE_PROGRESS_THREAD
+#if ORTE_ENABLE_PROGRESS_THREADS
 ORTE_DECLSPEC extern opal_thread_t orte_progress_thread;
+ORTE_DECLSPEC extern opal_event_t orte_finalize_event;
 #endif
 
 #define ORTE_GLOBAL_ARRAY_BLOCK_SIZE    64

orte/runtime/orte_init.c

@@ -67,7 +67,7 @@ orte_process_name_t orte_name_wildcard = {ORTE_JOBID_WILDCARD, ORTE_VPID_WILDCAR
 
 orte_process_name_t orte_name_invalid = {ORTE_JOBID_INVALID, ORTE_VPID_INVALID}; 
 
-#if !ORTE_DISABLE_FULL_SUPPORT && ORTE_ENABLE_PROGRESS_THREAD
+#if !ORTE_DISABLE_FULL_SUPPORT && ORTE_ENABLE_PROGRESS_THREADS
 static void* orte_progress_thread_engine(opal_object_t *obj);
 #endif
 
@@ -78,6 +78,13 @@ static void* orte_progress_thread_engine(opal_object_t *obj);
 #endif
 const char orte_version_string[] = ORTE_IDENT_STRING;
 
+#if !ORTE_DISABLE_FULL_SUPPORT && ORTE_ENABLE_PROGRESS_THREADS
+static void ignore_callback(int fd, short args, void *cbdata)
+{
+    /* nothing to do here */
+}
+#endif
+
 int orte_init(int* pargc, char*** pargv, orte_proc_type_t flags)
 {
     int ret;
@@ -134,9 +141,16 @@ int orte_init(int* pargc, char*** pargv, orte_proc_type_t flags)
     }
 
     if (ORTE_PROC_IS_APP) {
-#if !ORTE_DISABLE_FULL_SUPPORT && ORTE_ENABLE_PROGRESS_THREAD
+#if !ORTE_DISABLE_FULL_SUPPORT && ORTE_ENABLE_PROGRESS_THREADS
+#if OPAL_EVENT_HAVE_THREAD_SUPPORT
         /* get a separate orte event base */
         orte_event_base = opal_event_base_create();
+        /* setup the finalize event - we'll need it
+         * to break the thread out of the event lib
+         * when we want to stop it
+         */
+        opal_event_set(orte_event_base, &orte_finalize_event, -1, OPAL_EV_WRITE, ignore_callback, NULL);
+        opal_event_set_priority(&orte_finalize_event, ORTE_ERROR_PRI);
         /* construct the thread object */
         OBJ_CONSTRUCT(&orte_progress_thread, opal_thread_t);
         /* fork off a thread to progress it */
@@ -145,6 +159,11 @@ int orte_init(int* pargc, char*** pargv, orte_proc_type_t flags)
             error = "orte progress thread start";
             goto error;
         }
+#else
+        error = "event thread support is not configured";
+        ret = ORTE_ERROR;
+        goto error;
+#endif
 #else
         /* set the event base to the opal one */
         orte_event_base = opal_event_base;
@@ -175,7 +194,7 @@ int orte_init(int* pargc, char*** pargv, orte_proc_type_t flags)
 }
 
 
-#if !ORTE_DISABLE_FULL_SUPPORT && ORTE_ENABLE_PROGRESS_THREAD
+#if !ORTE_DISABLE_FULL_SUPPORT && ORTE_ENABLE_PROGRESS_THREADS
 static void* orte_progress_thread_engine(opal_object_t *obj)
 {
     while (orte_event_base_active) {