diff --git a/configure.ac b/configure.ac index 9b9114ee86..9166fc7d97 100644 --- a/configure.ac +++ b/configure.ac @@ -155,10 +155,6 @@ AC_MSG_RESULT([$ltversion]) # # Hardwire all progress threads to be off # -enable_progress_threads="no" -ORTE_ENABLE_PROGRESS_THREADS=0 -AC_DEFINE_UNQUOTED([ORTE_ENABLE_PROGRESS_THREADS], [$ORTE_ENABLE_PROGRESS_THREADS], - [Hardcode the ORTE progress thread to be off]) OMPI_ENABLE_PROGRESS_THREADS=0 AC_DEFINE_UNQUOTED([OMPI_ENABLE_PROGRESS_THREADS], [$OMPI_ENABLE_PROGRESS_THREADS], [Hardcode the OMPI progress thread to be off]) diff --git a/opal/mca/event/libevent2019/Makefile.am b/opal/mca/event/libevent2019/Makefile.am index f0116d27f9..6ea7ac1ff2 100644 --- a/opal/mca/event/libevent2019/Makefile.am +++ b/opal/mca/event/libevent2019/Makefile.am @@ -85,3 +85,7 @@ noinst_LTLIBRARIES = $(component_noinst) libmca_event_libevent2019_la_SOURCES =$(sources) libmca_event_libevent2019_la_LDFLAGS = -module -avoid-version libmca_event_libevent2019_la_LIBADD = $(builddir)/libevent/libevent.la +if OPAL_EVENT_HAVE_THREAD_SUPPORT +libmca_event_libevent2019_la_LIBADD += $(builddir)/libevent/libevent_pthreads.la +endif + diff --git a/opal/mca/event/libevent2019/configure.m4 b/opal/mca/event/libevent2019/configure.m4 index 9a264edec8..2e5c0ceed4 100644 --- a/opal/mca/event/libevent2019/configure.m4 +++ b/opal/mca/event/libevent2019/configure.m4 @@ -94,7 +94,7 @@ AC_DEFUN([MCA_opal_event_libevent2019_CONFIG],[ AC_ARG_ENABLE(event-thread-support, AC_HELP_STRING([--enable-event-thread-support], [enable event library internal thread support])) - if test "$enable_event_thread_support" = "yes"; then + if test "$enable_event_thread_support" = "yes" -o "$require_event_thread_support" = 1; then AC_DEFINE_UNQUOTED(OPAL_EVENT_HAVE_THREAD_SUPPORT, 1, [Thread support was configured into the event library]) else @@ -102,6 +102,7 @@ AC_DEFUN([MCA_opal_event_libevent2019_CONFIG],[ AC_DEFINE_UNQUOTED(OPAL_EVENT_HAVE_THREAD_SUPPORT, 0, [Thread support was not configured into the event library]) fi + AM_CONDITIONAL(OPAL_EVENT_HAVE_THREAD_SUPPORT, test "$enable_event_thread_support" = "yes") AC_MSG_RESULT([$event_args]) diff --git a/orte/config/orte_configure_options.m4 b/orte/config/orte_configure_options.m4 index 72ebe798a9..c7ad235714 100644 --- a/orte/config/orte_configure_options.m4 +++ b/orte/config/orte_configure_options.m4 @@ -115,21 +115,23 @@ AC_DEFINE_UNQUOTED([ORTE_ENABLE_HEARTBEAT], # # Do we want a separate orte progress thread? -AC_MSG_CHECKING([if want orte progress thread]) -AC_ARG_ENABLE([orte-progress-thread], - [AC_HELP_STRING([--enable-orte-progress-thread], +AC_MSG_CHECKING([if want orte progress threads]) +AC_ARG_ENABLE([orte-progress-threads], + [AC_HELP_STRING([--enable-orte-progress-threads], [Enable orte progress thread - for experiment by developers only! (default: disabled)])]) -if test "$enable_orte_progress_thread" = "yes"; then +if test "$enable_orte_progress_threads" = "yes"; then AC_MSG_RESULT([yes]) - orte_enable_progress_thread=1 + orte_enable_progress_threads=1 + require_event_thread_support=1 AC_DEFINE_UNQUOTED(OPAL_EVENT_HAVE_THREAD_SUPPORT, 1, [Thread support must be configured into the event library]) else AC_MSG_RESULT([no]) - orte_enable_progress_thread=0 + orte_enable_progress_threads=0 + require_event_thread_support=0 fi -AC_DEFINE_UNQUOTED([ORTE_ENABLE_PROGRESS_THREAD], - [$orte_enable_progress_thread], - [Whether we want an orte progress thread enabled]) +AC_DEFINE_UNQUOTED([ORTE_ENABLE_PROGRESS_THREADS], + [$orte_enable_progress_threads], + [Whether we want orte progress threads enabled]) ])dnl diff --git a/orte/mca/oob/tcp/oob_tcp_recv.c b/orte/mca/oob/tcp/oob_tcp_recv.c index 237960ad13..432a9245bf 100644 --- a/orte/mca/oob/tcp/oob_tcp_recv.c +++ b/orte/mca/oob/tcp/oob_tcp_recv.c @@ -179,7 +179,7 @@ int mca_oob_tcp_recv_cancel( /* wait for any previously matched messages to be processed */ OPAL_THREAD_LOCK(&mca_oob_tcp_component.tcp_match_lock); -#if ORTE_ENABLE_PROGRESS_THREADS +#if 0 && ORTE_ENABLE_PROGRESS_THREADS if(opal_event_progress_thread() == false) { while(mca_oob_tcp_component.tcp_match_count) { opal_condition_wait( diff --git a/orte/mca/routed/base/routed_base_fns.c b/orte/mca/routed/base/routed_base_fns.c index bbc0c09fc5..f89e965203 100644 --- a/orte/mca/routed/base/routed_base_fns.c +++ b/orte/mca/routed/base/routed_base_fns.c @@ -23,6 +23,13 @@ #include "orte/constants.h" #include "orte/types.h" +#if HAVE_TIME_H +#include +#endif +#if HAVE_SYS_TIME_H +#include +#endif + #include "opal/dss/dss.h" #include "opal/runtime/opal_progress.h" @@ -307,7 +314,17 @@ int orte_routed_base_register_sync(bool setup) /* it is okay to block here as we are -not- in an event */ while (!sync_recvd) { +#if !ORTE_ENABLE_PROGRESS_THREADS opal_progress(); +#else + { + /* provide a very short quiet period so we + * don't hammer the cpu while we wait + */ + struct timespec tp = {0, 100}; + nanosleep(&tp, NULL); + } +#endif } OPAL_OUTPUT_VERBOSE((5, orte_routed_base_output, diff --git a/orte/runtime/orte_finalize.c b/orte/runtime/orte_finalize.c index 415727faef..85bd613c3d 100644 --- a/orte/runtime/orte_finalize.c +++ b/orte/runtime/orte_finalize.c @@ -59,15 +59,6 @@ int orte_finalize(void) /* close the orte_show_help system */ orte_show_help_finalize(); -#if !ORTE_DISABLE_FULL_SUPPORT && ORTE_ENABLE_PROGRESS_THREAD - if (ORTE_PROC_IS_APP) { - /* stop the progress thread */ - orte_event_base_active = false; - opal_thread_join(&orte_progress_thread, NULL); - OBJ_DESTRUCT(&orte_progress_thread); - } -#endif - /* call the finalize function for this environment */ orte_ess.finalize(); @@ -85,6 +76,21 @@ int orte_finalize(void) if( NULL != orte_default_hostfile ) { free(orte_default_hostfile); } +#if ORTE_ENABLE_PROGRESS_THREADS + if (ORTE_PROC_IS_APP) { + /* stop the progress thread */ + orte_event_base_active = false; + /* must trigger the "finalize" event to break us + * out of the event loop + */ + opal_event_active(&orte_finalize_event, OPAL_EV_WRITE, 1); + /* wait for thread to exit */ + opal_thread_join(&orte_progress_thread, NULL); + OBJ_DESTRUCT(&orte_progress_thread); + /* release the event base */ + opal_event_base_free(orte_event_base); + } +#endif #endif /* Close the general debug stream */ diff --git a/orte/runtime/orte_globals.c b/orte/runtime/orte_globals.c index ca0e4c964a..e5d8d4e68d 100644 --- a/orte/runtime/orte_globals.c +++ b/orte/runtime/orte_globals.c @@ -188,8 +188,9 @@ bool orte_map_reduce = false; bool orte_map_stddiag_to_stderr = false; /* progress thread */ -#if ORTE_ENABLE_PROGRESS_THREAD +#if ORTE_ENABLE_PROGRESS_THREADS opal_thread_t orte_progress_thread; +opal_event_t orte_finalize_event; #endif #endif /* !ORTE_DISABLE_FULL_RTE */ diff --git a/orte/runtime/orte_globals.h b/orte/runtime/orte_globals.h index 493659aa6b..aa6d43fef4 100644 --- a/orte/runtime/orte_globals.h +++ b/orte/runtime/orte_globals.h @@ -137,8 +137,9 @@ typedef struct orte_app_context_t orte_app_context_t; #else -#if ORTE_ENABLE_PROGRESS_THREAD +#if ORTE_ENABLE_PROGRESS_THREADS ORTE_DECLSPEC extern opal_thread_t orte_progress_thread; +ORTE_DECLSPEC extern opal_event_t orte_finalize_event; #endif #define ORTE_GLOBAL_ARRAY_BLOCK_SIZE 64 diff --git a/orte/runtime/orte_init.c b/orte/runtime/orte_init.c index a3c6863666..cbde482c08 100644 --- a/orte/runtime/orte_init.c +++ b/orte/runtime/orte_init.c @@ -67,7 +67,7 @@ orte_process_name_t orte_name_wildcard = {ORTE_JOBID_WILDCARD, ORTE_VPID_WILDCAR orte_process_name_t orte_name_invalid = {ORTE_JOBID_INVALID, ORTE_VPID_INVALID}; -#if !ORTE_DISABLE_FULL_SUPPORT && ORTE_ENABLE_PROGRESS_THREAD +#if !ORTE_DISABLE_FULL_SUPPORT && ORTE_ENABLE_PROGRESS_THREADS static void* orte_progress_thread_engine(opal_object_t *obj); #endif @@ -78,6 +78,13 @@ static void* orte_progress_thread_engine(opal_object_t *obj); #endif const char orte_version_string[] = ORTE_IDENT_STRING; +#if !ORTE_DISABLE_FULL_SUPPORT && ORTE_ENABLE_PROGRESS_THREADS +static void ignore_callback(int fd, short args, void *cbdata) +{ + /* nothing to do here */ +} +#endif + int orte_init(int* pargc, char*** pargv, orte_proc_type_t flags) { int ret; @@ -134,9 +141,16 @@ int orte_init(int* pargc, char*** pargv, orte_proc_type_t flags) } if (ORTE_PROC_IS_APP) { -#if !ORTE_DISABLE_FULL_SUPPORT && ORTE_ENABLE_PROGRESS_THREAD +#if !ORTE_DISABLE_FULL_SUPPORT && ORTE_ENABLE_PROGRESS_THREADS +#if OPAL_EVENT_HAVE_THREAD_SUPPORT /* get a separate orte event base */ orte_event_base = opal_event_base_create(); + /* setup the finalize event - we'll need it + * to break the thread out of the event lib + * when we want to stop it + */ + opal_event_set(orte_event_base, &orte_finalize_event, -1, OPAL_EV_WRITE, ignore_callback, NULL); + opal_event_set_priority(&orte_finalize_event, ORTE_ERROR_PRI); /* construct the thread object */ OBJ_CONSTRUCT(&orte_progress_thread, opal_thread_t); /* fork off a thread to progress it */ @@ -145,6 +159,11 @@ int orte_init(int* pargc, char*** pargv, orte_proc_type_t flags) error = "orte progress thread start"; goto error; } +#else + error = "event thread support is not configured"; + ret = ORTE_ERROR; + goto error; +#endif #else /* set the event base to the opal one */ orte_event_base = opal_event_base; @@ -175,7 +194,7 @@ int orte_init(int* pargc, char*** pargv, orte_proc_type_t flags) } -#if !ORTE_DISABLE_FULL_SUPPORT && ORTE_ENABLE_PROGRESS_THREAD +#if !ORTE_DISABLE_FULL_SUPPORT && ORTE_ENABLE_PROGRESS_THREADS static void* orte_progress_thread_engine(opal_object_t *obj) { while (orte_event_base_active) {