From 83d69b6c959bdd2c4d883a4dbfa145a08e078a90 Mon Sep 17 00:00:00 2001 From: Ralph Castain Date: Sun, 20 May 2012 15:14:43 +0000 Subject: [PATCH] Enable the ORTE progress thread for apps (not needed in the tools as they already continuously loop in the event lib). This appears to be working, at least for MPI apps that only use shared memory (a simple "hello"). More testing is required to identify where problems will occur - this is only intended to allow further development. In order to use the progress thread, you must configure with: --enable-orte-progress-threads --enable-event-thread-support This commit was SVN r26457. --- configure.ac | 4 ---- opal/mca/event/libevent2019/Makefile.am | 4 ++++ opal/mca/event/libevent2019/configure.m4 | 3 ++- orte/config/orte_configure_options.m4 | 20 ++++++++++--------- orte/mca/oob/tcp/oob_tcp_recv.c | 2 +- orte/mca/routed/base/routed_base_fns.c | 17 ++++++++++++++++ orte/runtime/orte_finalize.c | 24 ++++++++++++++--------- orte/runtime/orte_globals.c | 3 ++- orte/runtime/orte_globals.h | 3 ++- orte/runtime/orte_init.c | 25 +++++++++++++++++++++--- 10 files changed, 76 insertions(+), 29 deletions(-) diff --git a/configure.ac b/configure.ac index 9b9114ee86..9166fc7d97 100644 --- a/configure.ac +++ b/configure.ac @@ -155,10 +155,6 @@ AC_MSG_RESULT([$ltversion]) # # Hardwire all progress threads to be off # -enable_progress_threads="no" -ORTE_ENABLE_PROGRESS_THREADS=0 -AC_DEFINE_UNQUOTED([ORTE_ENABLE_PROGRESS_THREADS], [$ORTE_ENABLE_PROGRESS_THREADS], - [Hardcode the ORTE progress thread to be off]) OMPI_ENABLE_PROGRESS_THREADS=0 AC_DEFINE_UNQUOTED([OMPI_ENABLE_PROGRESS_THREADS], [$OMPI_ENABLE_PROGRESS_THREADS], [Hardcode the OMPI progress thread to be off]) diff --git a/opal/mca/event/libevent2019/Makefile.am b/opal/mca/event/libevent2019/Makefile.am index f0116d27f9..6ea7ac1ff2 100644 --- a/opal/mca/event/libevent2019/Makefile.am +++ b/opal/mca/event/libevent2019/Makefile.am @@ -85,3 +85,7 @@ noinst_LTLIBRARIES = $(component_noinst) libmca_event_libevent2019_la_SOURCES =$(sources) libmca_event_libevent2019_la_LDFLAGS = -module -avoid-version libmca_event_libevent2019_la_LIBADD = $(builddir)/libevent/libevent.la +if OPAL_EVENT_HAVE_THREAD_SUPPORT +libmca_event_libevent2019_la_LIBADD += $(builddir)/libevent/libevent_pthreads.la +endif + diff --git a/opal/mca/event/libevent2019/configure.m4 b/opal/mca/event/libevent2019/configure.m4 index 9a264edec8..2e5c0ceed4 100644 --- a/opal/mca/event/libevent2019/configure.m4 +++ b/opal/mca/event/libevent2019/configure.m4 @@ -94,7 +94,7 @@ AC_DEFUN([MCA_opal_event_libevent2019_CONFIG],[ AC_ARG_ENABLE(event-thread-support, AC_HELP_STRING([--enable-event-thread-support], [enable event library internal thread support])) - if test "$enable_event_thread_support" = "yes"; then + if test "$enable_event_thread_support" = "yes" -o "$require_event_thread_support" = 1; then AC_DEFINE_UNQUOTED(OPAL_EVENT_HAVE_THREAD_SUPPORT, 1, [Thread support was configured into the event library]) else @@ -102,6 +102,7 @@ AC_DEFUN([MCA_opal_event_libevent2019_CONFIG],[ AC_DEFINE_UNQUOTED(OPAL_EVENT_HAVE_THREAD_SUPPORT, 0, [Thread support was not configured into the event library]) fi + AM_CONDITIONAL(OPAL_EVENT_HAVE_THREAD_SUPPORT, test "$enable_event_thread_support" = "yes") AC_MSG_RESULT([$event_args]) diff --git a/orte/config/orte_configure_options.m4 b/orte/config/orte_configure_options.m4 index 72ebe798a9..c7ad235714 100644 --- a/orte/config/orte_configure_options.m4 +++ b/orte/config/orte_configure_options.m4 @@ -115,21 +115,23 @@ AC_DEFINE_UNQUOTED([ORTE_ENABLE_HEARTBEAT], # # Do we want a separate orte progress thread? -AC_MSG_CHECKING([if want orte progress thread]) -AC_ARG_ENABLE([orte-progress-thread], - [AC_HELP_STRING([--enable-orte-progress-thread], +AC_MSG_CHECKING([if want orte progress threads]) +AC_ARG_ENABLE([orte-progress-threads], + [AC_HELP_STRING([--enable-orte-progress-threads], [Enable orte progress thread - for experiment by developers only! (default: disabled)])]) -if test "$enable_orte_progress_thread" = "yes"; then +if test "$enable_orte_progress_threads" = "yes"; then AC_MSG_RESULT([yes]) - orte_enable_progress_thread=1 + orte_enable_progress_threads=1 + require_event_thread_support=1 AC_DEFINE_UNQUOTED(OPAL_EVENT_HAVE_THREAD_SUPPORT, 1, [Thread support must be configured into the event library]) else AC_MSG_RESULT([no]) - orte_enable_progress_thread=0 + orte_enable_progress_threads=0 + require_event_thread_support=0 fi -AC_DEFINE_UNQUOTED([ORTE_ENABLE_PROGRESS_THREAD], - [$orte_enable_progress_thread], - [Whether we want an orte progress thread enabled]) +AC_DEFINE_UNQUOTED([ORTE_ENABLE_PROGRESS_THREADS], + [$orte_enable_progress_threads], + [Whether we want orte progress threads enabled]) ])dnl diff --git a/orte/mca/oob/tcp/oob_tcp_recv.c b/orte/mca/oob/tcp/oob_tcp_recv.c index 237960ad13..432a9245bf 100644 --- a/orte/mca/oob/tcp/oob_tcp_recv.c +++ b/orte/mca/oob/tcp/oob_tcp_recv.c @@ -179,7 +179,7 @@ int mca_oob_tcp_recv_cancel( /* wait for any previously matched messages to be processed */ OPAL_THREAD_LOCK(&mca_oob_tcp_component.tcp_match_lock); -#if ORTE_ENABLE_PROGRESS_THREADS +#if 0 && ORTE_ENABLE_PROGRESS_THREADS if(opal_event_progress_thread() == false) { while(mca_oob_tcp_component.tcp_match_count) { opal_condition_wait( diff --git a/orte/mca/routed/base/routed_base_fns.c b/orte/mca/routed/base/routed_base_fns.c index bbc0c09fc5..f89e965203 100644 --- a/orte/mca/routed/base/routed_base_fns.c +++ b/orte/mca/routed/base/routed_base_fns.c @@ -23,6 +23,13 @@ #include "orte/constants.h" #include "orte/types.h" +#if HAVE_TIME_H +#include +#endif +#if HAVE_SYS_TIME_H +#include +#endif + #include "opal/dss/dss.h" #include "opal/runtime/opal_progress.h" @@ -307,7 +314,17 @@ int orte_routed_base_register_sync(bool setup) /* it is okay to block here as we are -not- in an event */ while (!sync_recvd) { +#if !ORTE_ENABLE_PROGRESS_THREADS opal_progress(); +#else + { + /* provide a very short quiet period so we + * don't hammer the cpu while we wait + */ + struct timespec tp = {0, 100}; + nanosleep(&tp, NULL); + } +#endif } OPAL_OUTPUT_VERBOSE((5, orte_routed_base_output, diff --git a/orte/runtime/orte_finalize.c b/orte/runtime/orte_finalize.c index 415727faef..85bd613c3d 100644 --- a/orte/runtime/orte_finalize.c +++ b/orte/runtime/orte_finalize.c @@ -59,15 +59,6 @@ int orte_finalize(void) /* close the orte_show_help system */ orte_show_help_finalize(); -#if !ORTE_DISABLE_FULL_SUPPORT && ORTE_ENABLE_PROGRESS_THREAD - if (ORTE_PROC_IS_APP) { - /* stop the progress thread */ - orte_event_base_active = false; - opal_thread_join(&orte_progress_thread, NULL); - OBJ_DESTRUCT(&orte_progress_thread); - } -#endif - /* call the finalize function for this environment */ orte_ess.finalize(); @@ -85,6 +76,21 @@ int orte_finalize(void) if( NULL != orte_default_hostfile ) { free(orte_default_hostfile); } +#if ORTE_ENABLE_PROGRESS_THREADS + if (ORTE_PROC_IS_APP) { + /* stop the progress thread */ + orte_event_base_active = false; + /* must trigger the "finalize" event to break us + * out of the event loop + */ + opal_event_active(&orte_finalize_event, OPAL_EV_WRITE, 1); + /* wait for thread to exit */ + opal_thread_join(&orte_progress_thread, NULL); + OBJ_DESTRUCT(&orte_progress_thread); + /* release the event base */ + opal_event_base_free(orte_event_base); + } +#endif #endif /* Close the general debug stream */ diff --git a/orte/runtime/orte_globals.c b/orte/runtime/orte_globals.c index ca0e4c964a..e5d8d4e68d 100644 --- a/orte/runtime/orte_globals.c +++ b/orte/runtime/orte_globals.c @@ -188,8 +188,9 @@ bool orte_map_reduce = false; bool orte_map_stddiag_to_stderr = false; /* progress thread */ -#if ORTE_ENABLE_PROGRESS_THREAD +#if ORTE_ENABLE_PROGRESS_THREADS opal_thread_t orte_progress_thread; +opal_event_t orte_finalize_event; #endif #endif /* !ORTE_DISABLE_FULL_RTE */ diff --git a/orte/runtime/orte_globals.h b/orte/runtime/orte_globals.h index 493659aa6b..aa6d43fef4 100644 --- a/orte/runtime/orte_globals.h +++ b/orte/runtime/orte_globals.h @@ -137,8 +137,9 @@ typedef struct orte_app_context_t orte_app_context_t; #else -#if ORTE_ENABLE_PROGRESS_THREAD +#if ORTE_ENABLE_PROGRESS_THREADS ORTE_DECLSPEC extern opal_thread_t orte_progress_thread; +ORTE_DECLSPEC extern opal_event_t orte_finalize_event; #endif #define ORTE_GLOBAL_ARRAY_BLOCK_SIZE 64 diff --git a/orte/runtime/orte_init.c b/orte/runtime/orte_init.c index a3c6863666..cbde482c08 100644 --- a/orte/runtime/orte_init.c +++ b/orte/runtime/orte_init.c @@ -67,7 +67,7 @@ orte_process_name_t orte_name_wildcard = {ORTE_JOBID_WILDCARD, ORTE_VPID_WILDCAR orte_process_name_t orte_name_invalid = {ORTE_JOBID_INVALID, ORTE_VPID_INVALID}; -#if !ORTE_DISABLE_FULL_SUPPORT && ORTE_ENABLE_PROGRESS_THREAD +#if !ORTE_DISABLE_FULL_SUPPORT && ORTE_ENABLE_PROGRESS_THREADS static void* orte_progress_thread_engine(opal_object_t *obj); #endif @@ -78,6 +78,13 @@ static void* orte_progress_thread_engine(opal_object_t *obj); #endif const char orte_version_string[] = ORTE_IDENT_STRING; +#if !ORTE_DISABLE_FULL_SUPPORT && ORTE_ENABLE_PROGRESS_THREADS +static void ignore_callback(int fd, short args, void *cbdata) +{ + /* nothing to do here */ +} +#endif + int orte_init(int* pargc, char*** pargv, orte_proc_type_t flags) { int ret; @@ -134,9 +141,16 @@ int orte_init(int* pargc, char*** pargv, orte_proc_type_t flags) } if (ORTE_PROC_IS_APP) { -#if !ORTE_DISABLE_FULL_SUPPORT && ORTE_ENABLE_PROGRESS_THREAD +#if !ORTE_DISABLE_FULL_SUPPORT && ORTE_ENABLE_PROGRESS_THREADS +#if OPAL_EVENT_HAVE_THREAD_SUPPORT /* get a separate orte event base */ orte_event_base = opal_event_base_create(); + /* setup the finalize event - we'll need it + * to break the thread out of the event lib + * when we want to stop it + */ + opal_event_set(orte_event_base, &orte_finalize_event, -1, OPAL_EV_WRITE, ignore_callback, NULL); + opal_event_set_priority(&orte_finalize_event, ORTE_ERROR_PRI); /* construct the thread object */ OBJ_CONSTRUCT(&orte_progress_thread, opal_thread_t); /* fork off a thread to progress it */ @@ -145,6 +159,11 @@ int orte_init(int* pargc, char*** pargv, orte_proc_type_t flags) error = "orte progress thread start"; goto error; } +#else + error = "event thread support is not configured"; + ret = ORTE_ERROR; + goto error; +#endif #else /* set the event base to the opal one */ orte_event_base = opal_event_base; @@ -175,7 +194,7 @@ int orte_init(int* pargc, char*** pargv, orte_proc_type_t flags) } -#if !ORTE_DISABLE_FULL_SUPPORT && ORTE_ENABLE_PROGRESS_THREAD +#if !ORTE_DISABLE_FULL_SUPPORT && ORTE_ENABLE_PROGRESS_THREADS static void* orte_progress_thread_engine(opal_object_t *obj) { while (orte_event_base_active) {