After a nice, calm, rational discussion between Brian, Jeff, and myself, we decided to revert r24864 and r24862 to restore the reference counters in opal_init/finalize. The rationale was that we should instead change orte_init/finalize to also use reference counters to support multi-embedded libraries. Jeff and Brian will discuss proposing a similar change to mpi_init/finalize to the MPI Forum so that all three libraries will behave in similar manners.

It was agreed that opal_init_util had wound up being used in unintended ways, which raised the problem of getting reference counts to work right. However, fixing it would involve more pain than it was worth - and so long as the other layers are made to behave similarly, I have no preference either way.

Complete implementation will follow - for now, this just reverts the prior changes.

This commit was SVN r24886.

The following SVN revision numbers were found above:
  r24862 --> open-mpi/ompi@aa92e0c4eb
  r24864 --> open-mpi/ompi@a5062385c2

opal/runtime/opal_finalize.c

@@ -54,16 +54,18 @@
 #include "opal/runtime/opal_cr.h"
 #include "opal/mca/crs/base/base.h"
 
-extern bool opal_initialized;
-extern bool opal_util_initialized;
+extern int opal_initialized;
+extern int opal_util_initialized;
 
 int
 opal_finalize_util(void)
 {
-    if (!opal_util_initialized) {
+    if( --opal_util_initialized != 0 ) {
+        if( opal_util_initialized < 0 ) {
+            return OPAL_ERROR;
+        }
         return OPAL_SUCCESS;
     }
-    opal_util_initialized = false;
 
     /* Clear out all the registered MCA params */
     mca_base_param_finalize();
@@ -109,10 +111,12 @@ opal_finalize_util(void)
 int
 opal_finalize(void)
 {
-    if (!opal_initialized) {
+    if( --opal_initialized != 0 ) {
+        if( opal_initialized < 0 ) {
+            return OPAL_ERROR;
+        }
         return OPAL_SUCCESS;
     }
-    opal_initialized = false;
 
     /* close the checkpoint and restart service */
     opal_cr_finalize();

opal/runtime/opal_init.c

@@ -68,8 +68,8 @@
 #endif
 const char opal_version_string[] = OPAL_IDENT_STRING;
 
-bool opal_initialized = false;
-bool opal_util_initialized = false;
+int opal_initialized = 0;
+int opal_util_initialized = 0;
 int opal_cache_line_size;
 
 static int
@@ -219,10 +219,12 @@ opal_init_util(int* pargc, char*** pargv)
     int ret;
     char *error = NULL;
 
-    if (opal_util_initialized) {
+    if( ++opal_util_initialized != 1 ) {
+        if( opal_util_initialized < 1 ) {
+            return OPAL_ERROR;
+        }
         return OPAL_SUCCESS;
     }
-    opal_util_initialized = true;
 
     /* JMS See note in runtime/opal.h -- this is temporary; to be
        replaced with real hwloc information soon (in trunk/v1.5 and
@@ -322,10 +324,12 @@ opal_init(int* pargc, char*** pargv)
     int ret;
     char *error = NULL;
 
-    if (opal_initialized) {
+    if( ++opal_initialized != 1 ) {
+        if( opal_initialized < 1 ) {
+            return OPAL_ERROR;
+        }
         return OPAL_SUCCESS;
     }
-    opal_initialized = true;
 
     /* initialize util code */
     if (OPAL_SUCCESS != (ret = opal_init_util(pargc, pargv))) {

orte/orted/orted_main.c

@@ -328,7 +328,11 @@ int orte_daemon(int argc, char *argv[])
             return ret;
         }
     }
- 
+    /* finalize the OPAL utils. As they are opened again from orte_init->opal_init
+     * we continue to have a reference count on them. So we have to finalize them twice...
+     */
+    opal_finalize_util();
+
     if ((int)ORTE_VPID_INVALID != orted_globals.fail) {
         orted_globals.abort=false;
         /* some vpid was ordered to fail. The value can be positive

orte/tools/orterun/orterun.c

@@ -618,6 +618,10 @@ int orterun(int argc, char *argv[])
         ORTE_ERROR_LOG(rc);
         return rc;
     }
+    /* finalize the OPAL utils. As they are opened again from orte_init->opal_init
+     * we continue to have a reference count on them. So we have to finalize them twice...
+     */
+    opal_finalize_util();
 
     /* check for request to report uri */
     if (NULL != orterun_globals.report_uri) {