First cut of the MPI thread leave determination stuff (i.e., actually

listen to what the modules say, but don't make any decisions based on that
yet)

This commit was SVN r630.

src/mca/mpi/base/base.h

@@ -31,7 +31,9 @@ extern "C" {
   int mca_mpi_alloc_mem(MPI_Aint size, MPI_Info info, void *baseptr);
   int mca_mpi_free_mem(void *baseptr);
 
-  int mca_mpi_init_select_modules(int requested, int *provided);
+  int mca_mpi_init_select_modules(int requested, 
+                                  bool allow_multi_user_threads,
+                                  bool have_hidden_threads, int *provided);
 
 #if 0
   /* JMS Not implemented yet */

src/mca/mpi/base/mca_mpi_init_select_modules.c

@@ -26,9 +26,12 @@
  *
  * The contents of this function will likely be replaced 
  */
-int mca_mpi_init_select_modules(int requested, int *provided)
+int mca_mpi_init_select_modules(int requested, 
+                                bool allow_multi_user_threads,
+                                bool have_hidden_threads, int *provided)
 {
   lam_list_t colls;
+  bool user_threads, hidden_threads;
 
   /* Make final lists of available modules (i.e., call the query/init
      functions and see if they return happiness).  For pml, there will
@@ -37,18 +40,26 @@ int mca_mpi_init_select_modules(int requested, int *provided)
   /* JMS: At some point, we'll need to feed it the thread level to
      ensure to pick one high enough (e.g., if we need CR) */
 
-  if (LAM_SUCCESS != mca_pml_base_select(&mca_pml)) {
+  if (LAM_SUCCESS != mca_pml_base_select(&mca_pml, 
+                                         &user_threads, &hidden_threads)) {
     return LAM_ERROR;
   }
+  allow_multi_user_threads |= user_threads;
+  have_hidden_threads |= hidden_threads;
 
-  if (LAM_SUCCESS != mca_ptl_base_select()) {
+  if (LAM_SUCCESS != mca_ptl_base_select(&user_threads, &hidden_threads)) {
     return LAM_ERROR;
   }
+  allow_multi_user_threads |= user_threads;
+  have_hidden_threads |= hidden_threads;
 
   lam_list_init(&colls);
-  if (LAM_SUCCESS != mca_coll_base_select(&colls)) {
+  if (LAM_SUCCESS != mca_coll_base_select(&colls, &user_threads, 
+                                          &hidden_threads)) {
     return LAM_ERROR;
   }
+  allow_multi_user_threads |= user_threads;
+  have_hidden_threads |= hidden_threads;
 
   /* Now that we have a final list of all available modules, do the
      selection.  pml is already selected. */

src/mca/mpi/coll/base/base.h

@@ -20,7 +20,9 @@
 extern "C" {
 #endif
   int mca_coll_base_open(void);
-  int mca_coll_base_select(lam_list_t *available);
+  int mca_coll_base_select(lam_list_t *selected, 
+                           bool *allow_multi_user_threads, 
+                           bool *have_hidden_threads);
   const mca_coll_1_0_0_t *
     mca_coll_basic_query(MPI_Comm comm, int *priority);
   int mca_coll_base_close(void);

src/mca/mpi/coll/base/coll_base_select.c

@@ -9,8 +9,14 @@
 #include "mca/mpi/coll/base/base.h"
 
 
-int mca_coll_base_select(lam_list_t *selected)
+int mca_coll_base_select(lam_list_t *selected, bool *allow_multi_user_threads, 
+                         bool *have_hidden_threads)
 {
+  /* JMS Need to implement */
+
+  *allow_multi_user_threads = true;
+  *have_hidden_threads = false;
+
   /* All done */
 
   return LAM_SUCCESS;

src/mca/mpi/pml/base/base.h

@@ -19,7 +19,9 @@
 extern "C" {
 #endif
   int mca_pml_base_open(void);
-  int mca_pml_base_select(mca_pml_t *selected);
+  int mca_pml_base_select(mca_pml_t *selected, 
+                          bool *allow_multi_user_threads, 
+                          bool *have_hidden_threads);
   int mca_pml_base_close(void);
 #if defined(c_plusplus) || defined(__cplusplus)
 }

src/mca/mpi/pml/base/pml_base_close.c

@@ -19,8 +19,8 @@ int mca_pml_base_close(void)
      anyway?  This module is going away, so errors don't matter
      anymore) */
 
-  if (NULL != mca_pml.pml_finalize) {
-    mca_pml.pml_finalize();
+  if (NULL != mca_pml_base_selected_module.pmlm_finalize) {
+    mca_pml_base_selected_module.pmlm_finalize();
   }
 
   /* Close all remaining available modules (may be one if this is a

src/mca/mpi/pml/base/pml_base_open.c

@@ -47,7 +47,7 @@ int mca_pml_base_open(void)
   /* Set a sentinel in case we don't select any modules (e.g.,
      laminfo) */
 
-  mca_pml.pml_finalize = NULL;
+  mca_pml_base_selected_module.pmlm_finalize = NULL;
 
   /* All done */
 

src/mca/mpi/pml/base/pml_base_select.c

@@ -16,7 +16,6 @@ typedef struct opened_module_t {
   lam_list_item_t super;
 
   mca_pml_base_module_t *om_module;
-  mca_pml_t *om_actions;
 } opened_module_t;
 
 
@@ -30,10 +29,12 @@ typedef struct opened_module_t {
  * will have all of its function pointers saved and returned to the
  * caller.
  */
-int mca_pml_base_select(mca_pml_t *selected)
+int mca_pml_base_select(mca_pml_t *selected, bool *allow_multi_user_threads,
+                        bool *have_hidden_threads)
 {
   int priority, best_priority;
-  bool allow_multi_user_threads, have_hidden_threads;
+  bool user_threads, hidden_threads;
+  bool best_user_threads, best_hidden_threads;
   lam_list_item_t *item;
   mca_base_module_list_item_t *mli;
   mca_pml_base_module_t *module, *best_module;
@@ -61,8 +62,8 @@ int mca_pml_base_select(mca_pml_t *selected)
       lam_output_verbose(10, mca_pml_base_output,
                          "select: no init function; ignoring module");
     } else {
-      actions = module->pmlm_init(&priority, &allow_multi_user_threads,
-                                  &have_hidden_threads);
+      actions = module->pmlm_init(&priority, &user_threads,
+                                  &hidden_threads);
       if (NULL == actions) {
         lam_output_verbose(10, mca_pml_base_output,
                            "select: init returned failure");
@@ -71,6 +72,8 @@ int mca_pml_base_select(mca_pml_t *selected)
                            "select: init returned priority %d", priority);
         if (priority > best_priority) {
           best_priority = priority;
+          best_user_threads = user_threads;
+          best_hidden_threads = hidden_threads;
           best_module = module;
         }
 
@@ -80,7 +83,6 @@ int mca_pml_base_select(mca_pml_t *selected)
         }
         lam_list_item_init((lam_list_item_t *) om);
         om->om_module = module;
-        om->om_actions = actions;
         lam_list_append(&opened, (lam_list_item_t*) om);
       }
     }
@@ -103,13 +105,13 @@ int mca_pml_base_select(mca_pml_t *selected)
 
       /* Finalize */
 
-      if (NULL != om->om_actions->pml_finalize) {
+      if (NULL != om->om_module->pmlm_finalize) {
 
         /* Blatently ignore the return code (what would we do to
            recover, anyway?  This module is going away, so errors
            don't matter anymore) */
 
-        om->om_actions->pml_finalize();
+        om->om_module->pmlm_finalize();
         lam_output_verbose(10, mca_pml_base_output, 
                            "select: module %s not selected / finalized",
                            module->pmlm_version.mca_module_name);
@@ -129,6 +131,8 @@ int mca_pml_base_select(mca_pml_t *selected)
 
   mca_pml_base_selected_module = *best_module;
   *selected = *actions;
+  *allow_multi_user_threads = best_user_threads;
+  *have_hidden_threads = best_hidden_threads;
   lam_output_verbose(10, mca_pml_base_output, 
                      "select: module %s selected",
                      module->pmlm_version.mca_module_name);

src/mca/mpi/pml/pml.h

@@ -49,6 +49,8 @@ typedef struct mca_pml_1_0_0_t * (*mca_pml_base_module_init_fn_t)(
     bool *allow_multi_user_threads,
     bool *have_hidden_threads);
 
+typedef int (*mca_pml_base_module_finalize_fn_t)(void);
+
 /**
  * PML module version and interface functions.
  */
@@ -57,6 +59,7 @@ struct mca_pml_base_module_1_0_0_t {
    mca_base_module_t pmlm_version;
    mca_base_module_data_1_0_0_t pmlm_data;
    mca_pml_base_module_init_fn_t pmlm_init;
+   mca_pml_base_module_finalize_fn_t pmlm_finalize;
 };
 typedef struct mca_pml_base_module_1_0_0_t mca_pml_base_module_1_0_0_t;
 typedef mca_pml_base_module_1_0_0_t mca_pml_base_module_t;
@@ -71,7 +74,6 @@ typedef int (*mca_pml_base_del_comm_fn_t)(struct lam_communicator_t*);
 typedef int (*mca_pml_base_add_procs_fn_t)(struct lam_proc_t **procs, size_t nprocs);
 typedef int (*mca_pml_base_del_procs_fn_t)(struct lam_proc_t **procs, size_t nprocs);
 typedef int (*mca_pml_base_add_ptls_fn_t)(lam_list_t *ptls);
-typedef int (*mca_pml_base_finalize_fn_t)(void);
 typedef int (*mca_pml_base_progress_fn_t)(void);
 
 typedef int (*mca_pml_base_irecv_init_fn_t)(
@@ -146,7 +148,6 @@ struct mca_pml_1_0_0_t {
     mca_pml_base_add_procs_fn_t    pml_add_procs;
     mca_pml_base_del_procs_fn_t    pml_del_procs;
     mca_pml_base_add_ptls_fn_t     pml_add_ptls;
-    mca_pml_base_finalize_fn_t     pml_finalize;
     mca_pml_base_progress_fn_t     pml_progress;
 
     /* downcalls from MPI to PML */

src/mca/mpi/ptl/base/base.h

@@ -29,7 +29,8 @@ typedef struct mca_ptl_base_selected_module_t mca_ptl_base_selected_module_t;
 extern "C" {
 #endif
   int mca_ptl_base_open(void);
-  int mca_ptl_base_select(void);
+  int mca_ptl_base_select(bool *allow_multi_user_threads, 
+                          bool *have_hidden_threads);
   int mca_ptl_base_close(void);
 #if defined(c_plusplus) || defined(__cplusplus)
 }

src/mca/mpi/ptl/base/ptl_base_select.c

@@ -19,10 +19,11 @@
  * will be closed and unloaded.  The selected modules will be returned
  * to the caller in a lam_list_t.
  */
-int mca_ptl_base_select(void)
+int mca_ptl_base_select(bool *allow_multi_user_threads, 
+                        bool *have_hidden_threads)
 {
   int i, num_ptls;
-  bool allow_multi_user_threads, have_hidden_threads;
+  bool user_threads, hidden_threads;
   lam_list_item_t *item;
   mca_base_module_list_item_t *mli;
   mca_ptl_base_module_t *module;
@@ -46,8 +47,8 @@ int mca_ptl_base_select(void)
       lam_output_verbose(10, mca_ptl_base_output,
                          "select: no init function; ignoring module");
     } else {
-      actions = module->ptlm_init(&num_ptls, &allow_multi_user_threads,
-                                  &have_hidden_threads);
+      actions = module->ptlm_init(&num_ptls, &user_threads,
+                                  &hidden_threads);
 
       /* If the module didn't initialize, unload it */
 
@@ -64,6 +65,9 @@ int mca_ptl_base_select(void)
       /* Otherwise, it initialized properly.  Save it. */
 
       else {
+        *allow_multi_user_threads |= user_threads;
+        *have_hidden_threads |= hidden_threads;
+
         lam_output_verbose(10, mca_ptl_base_output,
                            "select: init returned success");