Fix #1253: default libevent to use select/poll and only use the other

mechanisms (such as epoll) if someone (ompi_mpi_init()) requests
otherwise.  See big comment in opal/event/event.c for a full
explanation.

This commit was SVN r17956.

ompi/runtime/ompi_mpi_init.c

@@ -234,15 +234,37 @@ int ompi_mpi_init(int argc, char **argv, int requested, int *provided)
     /* see comment below about sched_yield */
     int num_processors;
 #endif
-    
-    /* Join the run-time environment - do the things that don't hit
-       the registry */
 
-    if (ORTE_SUCCESS != (ret = opal_init())) {
-        error = "ompi_mpi_init: opal_init failed";
+    /* Setup enough to check get/set MCA params */
+
+    if (ORTE_SUCCESS != (ret = opal_init_util())) {
+        error = "ompi_mpi_init: opal_init_util failed";
         goto error;
     }
 
+    /* _After_ opal_init_util() but _before_ orte_init(), we need to
+       set an MCA param that tells libevent that it's ok to use any
+       mechanism in libevent that si available on this platform (e.g.,
+       epoll and friends).  Per opal/event/event.s, we default to
+       select/poll -- but we know that MPI processes won't be using
+       pty's with the event engine, so it's ok to relax this
+       constraint and let any fd-monitoring mechanism be used. */
+    ret = mca_base_param_reg_string_name("opal", "event_include",
+                                         "Internal orted MCA param: tell opal_init() to use a specific mechanism in libevent",
+                                         false, false, "all", NULL);
+    if (ret >= 0) {
+        /* We have to explicitly "set" the MCA param value here
+           because libevent initialization will re-register the MCA
+           param and therefore override the default. Setting the value
+           here puts the desired value ("all") in different storage
+           that is not overwritten if/when the MCA param is
+           re-registered.  Note that we do *NOT* set this value as an
+           environment variable, just so that it won't be inherited by
+           any spawned processes and potentially cause unintented
+           side-effects with launching ORTE tools... */
+        mca_base_param_set_string(ret, "all");
+    }
+    
     /* check to see if we want timing information */
     param = mca_base_param_reg_int_name("ompi", "timing",
                                         "Request that critical timing loops be measured",

opal/event/event.c

@@ -256,15 +256,42 @@ opal_event_init(void)
 
 #if OPAL_HAVE_WORKING_EVENTOPS
 
-    /**
-     * Retrieve the upper level specified event system, if any.
+    /* Retrieve the upper level specified event system, if any.
+     * Default to select() on OS X and poll() everywhere else because
+     * various parts of OMPI / ORTE use libevent with pty's.  pty's
+     * *only* work with select on OS X (tested on Tiger and Leopard);
+     * we *know* that both select and poll works with pty's everywhere
+     * else we care about (other mechansisms such as epoll *may* work
+     * with pty's -- we have not tested comprehensively with newer
+     * versions of Linux, etc.).  So the safe thing to do is:
+     *
+     * - On OS X, default to using "select" only
+     * - Everywhere else, default to using "poll" only (because poll
+     *   is more scalable than select)
+     *
+     * An upper layer may override this setting if it knows that pty's
+     * won't be used with libevent.  For example, we currently have
+     * ompi_mpi_init() set to use "all" (to include epoll and friends)
+     * so that the TCP BTL can be a bit more scalable -- because we
+     * *know* that MPI apps don't use pty's with libevent.  
+     * Note that other tools explicitly *do* use pty's with libevent:
+     *
+     * - orted
+     * - orterun (probably only if it launches locally)
+     * - ...?
      */
     mca_base_param_reg_string_name("opal", "event_include",
-                                   "Comma-delimited list of libevent subsystems to use (kqueue, devpoll, epoll, poll, select, and rtsig)",
-                                   false, false, "all", &event_module_include);
+                                   "Comma-delimited list of libevent subsystems to use (kqueue, devpoll, epoll, poll, select, and rtsig -- depending on your platform)",
+                                   false, false, 
+#ifdef __APPLE__
+                                   "select",
+#else
+                                   "poll",
+#endif
+                                   &event_module_include);
     if (NULL == event_module_include) {
         /* Shouldn't happen, but... */
-        event_module_include = strdup("all");
+        event_module_include = strdup("select");
     }
     opal_event_module_include = opal_argv_split(event_module_include,',');
     free(event_module_include);

orte/orted/orted_main.c

@@ -178,7 +178,6 @@ int orte_daemon(int argc, char *argv[])
     char log_file[PATH_MAX];
     char *jobidstring;
     char *rml_uri;
-    char *tmp1, *tmp2;
     int i;
     opal_buffer_t *buffer;
     char hostname[100];
@@ -264,36 +263,6 @@ int orte_daemon(int argc, char *argv[])
         if (1000 < i) i=0;        
     }
 
-    /* _After_ opal_init_util() (and various other bookkeeping) but
-       _before_ orte_init(), we need to set an MCA param that tells
-       the orted not to use any other libevent mechanism except
-       "select" or "poll" (per potential pty issues with scalable
-       fd-monitoring mechanisms such as epoll() and friends -- these
-       issues *may* have been fixed in later OS releases and/or newer
-       versions of libevent, but we weren't willing to do all the
-       testing to figure it out.  So force the orted to use
-       select()/poll() *only* -- there's so few fd's in the orted that
-       it really doesn't matter. 
-       
-       Note that pty's work fine with poll() on most systems, so we
-       prefer that (because it's more scalable than select()).
-       However, poll() does *not* work with ptys on OS X, so we use
-       select() there. */
-    mca_base_param_reg_string_name("opal", "event_include",
-                                   "Internal orted MCA param: tell opal_init() to use a specific mechanism in libevent",
-                                   true, true, 
-#ifdef __APPLE__
-                                   "select",
-#else
-                                   "poll",
-#endif
-                                   NULL);
-    tmp1 = mca_base_param_environ_variable("opal", NULL, "event_include");
-    asprintf(&tmp2, "%s=select", tmp1);
-    putenv(tmp2);
-    free(tmp1);
-    free(tmp2);
-    
     /* Okay, now on to serious business! */
     
     if (orted_globals.hnp) {