Don't allow the BTL to be closed before the end of the process. Count the

number of times the BTLs are opened, and then don't remove them until
close was called the same number of times.

This commit was SVN r15376.

ompi/mca/bml/base/bml_base_close.c

@@ -2,7 +2,7 @@
  * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
  *                         University Research and Technology
  *                         Corporation.  All rights reserved.
- * Copyright (c) 2004-2006 The University of Tennessee and The University
+ * Copyright (c) 2004-2007 The University of Tennessee and The University
  *                         of Tennessee Research Foundation.  All rights
  *                         reserved.
  * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, 
@@ -20,10 +20,13 @@
 #include "ompi/mca/btl/base/base.h" 
 #include "ompi/mca/bml/base/base.h" 
 
-int mca_bml_base_close( void ) {
+int mca_bml_base_close( void )
+{
     int rc; 
+
     if(OMPI_SUCCESS != (rc = mca_btl_base_close()))
         return rc;
+
     return OMPI_SUCCESS; 
 }
 

ompi/mca/btl/base/btl_base_close.c

@@ -2,7 +2,7 @@
  * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
  *                         University Research and Technology
  *                         Corporation.  All rights reserved.
- * Copyright (c) 2004-2005 The University of Tennessee and The University
+ * Copyright (c) 2004-2007 The University of Tennessee and The University
  *                         of Tennessee Research Foundation.  All rights
  *                         reserved.
  * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, 
@@ -28,50 +28,55 @@
 #include "ompi/mca/btl/btl.h"
 #include "ompi/mca/btl/base/base.h"
 
-extern bool already_opened;
+extern int already_opened;
 
 int mca_btl_base_close(void)
 {
-  opal_list_item_t *item;
-  mca_btl_base_selected_module_t *sm;
+    opal_list_item_t *item;
+    mca_btl_base_selected_module_t *sm;
 
-  /* disable event processing while cleaning up btls */
-  opal_event_disable();
+    if( already_opened <= 0 ) {
+        return OMPI_ERROR;
+    } else {
+        if( --already_opened > 0 ) {
+            return OMPI_SUCCESS;
+        }
+    }
+    /* disable event processing while cleaning up btls */
+    opal_event_disable();
 
-  /* Finalize all the btl components and free their list items */
+    /* Finalize all the btl components and free their list items */
 
-  for (item = opal_list_remove_first(&mca_btl_base_modules_initialized);
-       NULL != item; 
-       item = opal_list_remove_first(&mca_btl_base_modules_initialized)) {
-    sm = (mca_btl_base_selected_module_t *) item;
+    for (item = opal_list_remove_first(&mca_btl_base_modules_initialized);
+         NULL != item; 
+         item = opal_list_remove_first(&mca_btl_base_modules_initialized)) {
+        sm = (mca_btl_base_selected_module_t *) item;
 
-    /* Blatebtly ignore the return code (what would we do to recover,
-       anyway?  This component is going away, so errors don't matter
-       anymore) */
+        /* Blatebtly ignore the return code (what would we do to recover,
+           anyway?  This component is going away, so errors don't matter
+           anymore) */
 
-    sm->btl_module->btl_finalize(sm->btl_module);
-    free(sm);
-  }
+        sm->btl_module->btl_finalize(sm->btl_module);
+        free(sm);
+    }
 
-  /* Close all remaining opened components (may be one if this is a
-     OMPI RTE program, or [possibly] multiple if this is ompi_info) */
+    /* Close all remaining opened components (may be one if this is a
+       OMPI RTE program, or [possibly] multiple if this is ompi_info) */
   
-  if (0 != opal_list_get_size(&mca_btl_base_components_opened)) {
-      mca_base_components_close(mca_btl_base_output, 
-                                &mca_btl_base_components_opened, NULL);
-  }
+    if (0 != opal_list_get_size(&mca_btl_base_components_opened)) {
+        mca_base_components_close(mca_btl_base_output, 
+                                  &mca_btl_base_components_opened, NULL);
+    }
 
-  /* cleanup */
-  if(NULL != mca_btl_base_include)
-     free(mca_btl_base_include);
-  if(NULL != mca_btl_base_exclude)
-     free(mca_btl_base_exclude);
+    /* cleanup */
+    if(NULL != mca_btl_base_include)
+        free(mca_btl_base_include);
+    if(NULL != mca_btl_base_exclude)
+        free(mca_btl_base_exclude);
 
-  /* restore event processing */
-  opal_event_enable();
+    /* restore event processing */
+    opal_event_enable();
 
-  already_opened = false;
-
-  /* All done */
-  return OMPI_SUCCESS;
+    /* All done */
+    return OMPI_SUCCESS;
 }

ompi/mca/btl/base/btl_base_open.c

@@ -2,7 +2,7 @@
  * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
  *                         University Research and Technology
  *                         Corporation.  All rights reserved.
- * Copyright (c) 2004-2005 The University of Tennessee and The University
+ * Copyright (c) 2004-2007 The University of Tennessee and The University
  *                         of Tennessee Research Foundation.  All rights
  *                         reserved.
  * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, 
@@ -74,7 +74,7 @@ int mca_btl_base_warn_component_unused = 1;
 opal_list_t mca_btl_base_components_opened;
 opal_list_t mca_btl_base_modules_initialized;
 
-bool already_opened = false;
+int already_opened = 0;
 
 /**
  * Function for finding and opening either all MCA components, or the one
@@ -82,16 +82,14 @@ bool already_opened = false;
  */
 int mca_btl_base_open(void)
 {
-    if (already_opened) return OMPI_SUCCESS;
-    already_opened = true;
+    if( ++already_opened > 1 ) return OMPI_SUCCESS;
 
-    mca_base_param_reg_int_name(
-                                "btl", 
-                                "base_debug", 
-                                "If btl_base_debug is 1 standard debug is output, if > 1 verbose debug is output", 
-                                false, false, 
-                                0, 
-                                &mca_btl_base_debug);
+    mca_base_param_reg_int_name( "btl", 
+                                 "base_debug", 
+                                 "If btl_base_debug is 1 standard debug is output, if > 1 verbose debug is output", 
+                                 false, false, 
+                                 0, 
+                                 &mca_btl_base_debug );
 
     if( mca_btl_base_debug > 0 ) {
         mca_btl_base_output = opal_output_open(NULL);

ompi/mca/btl/base/btl_base_select.c

@@ -2,7 +2,7 @@
  * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
  *                         University Research and Technology
  *                         Corporation.  All rights reserved.
- * Copyright (c) 2004-2005 The University of Tennessee and The University
+ * Copyright (c) 2004-2007 The University of Tennessee and The University
  *                         of Tennessee Research Foundation.  All rights
  *                         reserved.
  * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, 
@@ -29,11 +29,10 @@
 #include "ompi/mca/btl/base/base.h"
 #include "orte/mca/errmgr/errmgr.h"
 
-OBJ_CLASS_INSTANCE(
-   mca_btl_base_selected_module_t,
-   opal_list_item_t,
-   NULL,
-   NULL);
+OBJ_CLASS_INSTANCE( mca_btl_base_selected_module_t,
+                    opal_list_item_t,
+                    NULL,
+                    NULL );
 
 /**
  * Function for weeding out btl components that don't want to run.
@@ -46,112 +45,112 @@ OBJ_CLASS_INSTANCE(
 int mca_btl_base_select(bool enable_progress_threads,
                         bool enable_mpi_threads)
 {
-  int i, num_btls;
-  opal_list_item_t *item;
-  mca_base_component_list_item_t *cli;
-  mca_btl_base_component_t *component;
-  mca_btl_base_module_t **modules;
-  mca_btl_base_selected_module_t *sm;
+    int i, num_btls;
+    opal_list_item_t *item;
+    mca_base_component_list_item_t *cli;
+    mca_btl_base_component_t *component;
+    mca_btl_base_module_t **modules;
+    mca_btl_base_selected_module_t *sm;
 
-  char** include = opal_argv_split(mca_btl_base_include, ',');
-  char** exclude = opal_argv_split(mca_btl_base_exclude, ',');
+    char** include = opal_argv_split(mca_btl_base_include, ',');
+    char** exclude = opal_argv_split(mca_btl_base_exclude, ',');
 
-  /* Traverse the list of opened modules; call their init
-     functions. */
+    /* Traverse the list of opened modules; call their init
+       functions. */
 
-  item  = opal_list_get_first(&mca_btl_base_components_opened);
-  while(item != opal_list_get_end(&mca_btl_base_components_opened)) {
-    opal_list_item_t *next = opal_list_get_next(item);
-    cli = (mca_base_component_list_item_t *) item;
+    item  = opal_list_get_first(&mca_btl_base_components_opened);
+    while(item != opal_list_get_end(&mca_btl_base_components_opened)) {
+        opal_list_item_t *next = opal_list_get_next(item);
+        cli = (mca_base_component_list_item_t *) item;
 
-    component = (mca_btl_base_component_t *) cli->cli_component;
+        component = (mca_btl_base_component_t *) cli->cli_component;
 
-    /* if there is an include list - item must be in the list to be included */
-    if ( NULL != include ) {
-        char** argv = include; 
-        bool found = false;
-        while(argv && *argv) {
-            if(strcmp(component->btl_version.mca_component_name,*argv) == 0) {
-                found = true;
-                break;
+        /* if there is an include list - item must be in the list to be included */
+        if ( NULL != include ) {
+            char** argv = include; 
+            bool found = false;
+            while(argv && *argv) {
+                if(strcmp(component->btl_version.mca_component_name,*argv) == 0) {
+                    found = true;
+                    break;
+                }
+                argv++;
             }
-            argv++;
-        }
-        if(found == false) {
-            item = next;
-            continue;
-        }
-
-    /* otherwise - check the exclude list to see if this item has been specifically excluded */
-    } else if ( NULL != exclude ) {
-        char** argv = exclude; 
-        bool found = false;
-        while(argv && *argv) {
-            if(strcmp(component->btl_version.mca_component_name,*argv) == 0) {
-                found = true;
-                break;
+            if(found == false) {
+                item = next;
+                continue;
+            }
+
+            /* otherwise - check the exclude list to see if this item has been specifically excluded */
+        } else if ( NULL != exclude ) {
+            char** argv = exclude; 
+            bool found = false;
+            while(argv && *argv) {
+                if(strcmp(component->btl_version.mca_component_name,*argv) == 0) {
+                    found = true;
+                    break;
+                }
+                argv++;
+            }
+            if(found == true) {
+                item = next;
+                continue;
             }
-            argv++;
         }
-        if(found == true) {
-            item = next;
-            continue;
-        }
-    }
 
-    opal_output_verbose(10, mca_btl_base_output, 
-                       "select: initializing %s component %s",
-                       component->btl_version.mca_type_name,
-                       component->btl_version.mca_component_name);
-    if (NULL == component->btl_init) {
-      opal_output_verbose(10, mca_btl_base_output,
-                         "select: no init function; ignoring component");
-    } else {
-      modules = component->btl_init(&num_btls, enable_progress_threads,
-                                     enable_mpi_threads);
-
-      /* If the component didn't initialize, remove it from the opened
-         list and remove it from the component repository */
-
-      if (NULL == modules) {
-        opal_output_verbose(10, mca_btl_base_output,
-                           "select: init returned failure");
-        opal_output_verbose(10, mca_btl_base_output,
-                            "select: module %s unloaded",
+        opal_output_verbose(10, mca_btl_base_output, 
+                            "select: initializing %s component %s",
+                            component->btl_version.mca_type_name,
                             component->btl_version.mca_component_name);
+        if (NULL == component->btl_init) {
+            opal_output_verbose(10, mca_btl_base_output,
+                                "select: no init function; ignoring component");
+        } else {
+            modules = component->btl_init(&num_btls, enable_progress_threads,
+                                          enable_mpi_threads);
 
-        mca_base_component_repository_release((mca_base_component_t *) component);
-        opal_list_remove_item(&mca_btl_base_components_opened, item);
-      } 
+            /* If the component didn't initialize, remove it from the opened
+               list and remove it from the component repository */
 
-      /* Otherwise, it initialized properly.  Save it. */
+            if (NULL == modules) {
+                opal_output_verbose(10, mca_btl_base_output,
+                                    "select: init returned failure");
+                opal_output_verbose(10, mca_btl_base_output,
+                                    "select: module %s unloaded",
+                                    component->btl_version.mca_component_name);
 
-      else {
-        opal_output_verbose(10, mca_btl_base_output,
-                           "select: init returned success");
+                mca_base_component_repository_release((mca_base_component_t *) component);
+                opal_list_remove_item(&mca_btl_base_components_opened, item);
+            } 
 
-        for (i = 0; i < num_btls; ++i) {
-          sm = OBJ_NEW(mca_btl_base_selected_module_t);
-          if (NULL == sm) {
-            return OMPI_ERR_OUT_OF_RESOURCE;
-          }
-          sm->btl_component = component;
-          sm->btl_module = modules[i];
-          opal_list_append(&mca_btl_base_modules_initialized,
-                          (opal_list_item_t*) sm);
+            /* Otherwise, it initialized properly.  Save it. */
+
+            else {
+                opal_output_verbose(10, mca_btl_base_output,
+                                    "select: init returned success");
+
+                for (i = 0; i < num_btls; ++i) {
+                    sm = OBJ_NEW(mca_btl_base_selected_module_t);
+                    if (NULL == sm) {
+                        return OMPI_ERR_OUT_OF_RESOURCE;
+                    }
+                    sm->btl_component = component;
+                    sm->btl_module = modules[i];
+                    opal_list_append(&mca_btl_base_modules_initialized,
+                                     (opal_list_item_t*) sm);
+                }
+                free(modules);
+            }
         }
-        free(modules);
-      }
+        item = next;
     }
-    item = next;
-  }
 
-  /* Finished querying all components.  Check for the bozo case. */
+    /* Finished querying all components.  Check for the bozo case. */
 
-  if (0 == opal_list_get_size(&mca_btl_base_modules_initialized)) {
-     opal_show_help("help-mca-base.txt", "find-available:none-found", true,
-                    "btl");
-     orte_errmgr.error_detected(1, NULL);
-  }
-  return OMPI_SUCCESS;
+    if (0 == opal_list_get_size(&mca_btl_base_modules_initialized)) {
+        opal_show_help("help-mca-base.txt", "find-available:none-found", true,
+                       "btl");
+        orte_errmgr.error_detected(1, NULL);
+    }
+    return OMPI_SUCCESS;
 }

ompi/mca/pml/dr/pml_dr.c

@@ -3,7 +3,7 @@
  * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
  *                         University Research and Technology
  *                         Corporation.  All rights reserved.
- * Copyright (c) 2004-2006 The University of Tennessee and The University
+ * Copyright (c) 2004-2007 The University of Tennessee and The University
  *                         of Tennessee Research Foundation.  All rights
  *                         reserved.
  * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
@@ -275,21 +275,6 @@ int mca_pml_dr_del_procs(ompi_proc_t** procs, size_t nprocs)
     return mca_bml.bml_del_procs(nprocs, procs);
 }
 
-int mca_pml_dr_component_fini(void)
-{
-    if(!mca_pml_dr.enabled)
-        return OMPI_SUCCESS; /* never selected.. return success.. */  
-    mca_pml_dr.enabled = false;  /* not anymore */
-
-    OBJ_DESTRUCT(&mca_pml_dr.send_pending);
-    OBJ_DESTRUCT(&mca_pml_dr.send_active);
-    OBJ_DESTRUCT(&mca_pml_dr.acks_pending);
-    OBJ_DESTRUCT(&mca_pml_dr.recv_frags);
-    OBJ_DESTRUCT(&mca_pml_dr.buffers);
-
-    return OMPI_SUCCESS;
-}
-
 int mca_pml_dr_dump(
     struct ompi_communicator_t* comm,
     int verbose)

ompi/mca/pml/dr/pml_dr.h

@@ -92,25 +92,6 @@ typedef struct mca_pml_dr_t mca_pml_dr_t;
 
 extern mca_pml_dr_t mca_pml_dr;
 
-
-/*
- * PML module functions.
- */
-
-
-extern int mca_pml_dr_component_open(void);
-extern int mca_pml_dr_component_close(void);
-
-extern mca_pml_base_module_t* mca_pml_dr_component_init(
-    int *priority, 
-    bool enable_progress_threads,
-    bool enable_mpi_threads
-);
-
-extern int mca_pml_dr_component_fini(void);
-
-
-
 /*
  * PML interface functions.
  */

ompi/mca/pml/dr/pml_dr_component.c

@@ -34,6 +34,14 @@
 #include "ompi/mca/bml/base/base.h" 
 #include "pml_dr_component.h"
 
+static int mca_pml_dr_component_open(void);
+static int mca_pml_dr_component_close(void);
+static mca_pml_base_module_t*
+mca_pml_dr_component_init( int* priority, 
+                           bool enable_progress_threads,
+                           bool enable_mpi_threads );
+static int mca_pml_dr_component_fini(void);
+
 mca_pml_base_component_1_0_0_t mca_pml_dr_component = {
 
     /* First, the mca_base_component_t struct containing meta
@@ -65,17 +73,15 @@ mca_pml_base_component_1_0_0_t mca_pml_dr_component = {
 };
 
 
-
-static inline int mca_pml_dr_param_register_int(
-    const char* param_name,
-    int default_value)
+static inline int
+mca_pml_dr_param_register_int( const char* param_name,
+                               int default_value )
 {
     int id = mca_base_param_register_int("pml","dr",param_name,NULL,default_value);
     int param_value = default_value;
     mca_base_param_lookup_int(id,&param_value);
     return param_value;
 }
-                                                                                                                        
 
 int mca_pml_dr_component_open(void)
 {
@@ -117,7 +123,6 @@ int mca_pml_dr_component_open(void)
     return mca_bml_base_open();
 }
 
-
 int mca_pml_dr_component_close(void)
 {
     int rc;
@@ -128,7 +133,6 @@ int mca_pml_dr_component_close(void)
     return OMPI_SUCCESS;
 }
 
-
 mca_pml_base_module_t* mca_pml_dr_component_init(int* priority, 
                                                   bool enable_progress_threads,
                                                   bool enable_mpi_threads)
@@ -156,3 +160,24 @@ mca_pml_base_module_t* mca_pml_dr_component_init(int* priority,
     return &mca_pml_dr.super;
 }
 
+int mca_pml_dr_component_fini(void)
+{
+    int rc;
+
+    /* Shutdown BML */
+    if(OMPI_SUCCESS != (rc = mca_bml.bml_finalize()))
+        return rc;
+
+    if(!mca_pml_dr.enabled)
+        return OMPI_SUCCESS; /* never selected.. return success.. */  
+    mca_pml_dr.enabled = false;  /* not anymore */
+
+    OBJ_DESTRUCT(&mca_pml_dr.send_pending);
+    OBJ_DESTRUCT(&mca_pml_dr.send_active);
+    OBJ_DESTRUCT(&mca_pml_dr.acks_pending);
+    OBJ_DESTRUCT(&mca_pml_dr.recv_frags);
+    OBJ_DESTRUCT(&mca_pml_dr.buffers);
+
+    return OMPI_SUCCESS;
+}
+

ompi/mca/pml/ob1/pml_ob1_component.c

@@ -207,7 +207,6 @@ int mca_pml_ob1_component_fini(void)
     if(OMPI_SUCCESS != (rc = mca_bml.bml_finalize()))
         return rc;
 
-
     /* Shutdown buffered send */
     if(OMPI_SUCCESS != (rc = mca_pml_base_bsend_fini())) {
         return rc;