Convert the C++ Comm keyval creation and intercept callbacks to *not*

use the STL.  This is the first step in removing the STL from the C++
bindings (Solaris has 2 versions of the STL; if OMPI uses one and an
MPI application wants to use another, Bad Things happen).

The main idea is to wrap up the C++ callback function pointers and the
user's extra_state into our own struct that is passed as the
extra_state to the C keyval registration along with the intercept
routines in intercepts.cc.  When the C++ intercepts are activated,
they unwrap the user's callback and extra state and call them.

It got a little more complicated than that, however:

 * I realized that we were returning errors back from
   Comm::create_keyval() incorrectly, so I fixed that.
 * Instead of using STL maps to store associations, we now use an
   opal_list_t which has to be guaranteed to be initialized correctly
   and only once in a multi-threaded environment.
 * Because of whackyness in the C++ bindings, it is possible to call
   Comm::Create_keyval with C callbacks (!).  If both registered
   callbacks are C functions, then ensure to avoid all the C++
   machinery.

This commit was SVN r17125.

ompi/mpi/cxx/comm.cc

@@ -10,7 +10,7 @@
 //                         University of Stuttgart.  All rights reserved.
 // Copyright (c) 2004-2005 The Regents of the University of California.
 //                         All rights reserved.
-// Copyright (c) 2007      Cisco Systems, Inc.  All rights reserved.
+// Copyright (c) 2007-2008 Cisco Systems, Inc.  All rights reserved.
 // $COPYRIGHT$
 // 
 // Additional copyrights may follow
@@ -21,13 +21,43 @@
 // do not include ompi_config.h because it kills the free/malloc defines
 #include "mpi.h"
 #include "ompi/mpi/cxx/mpicxx.h"
+
+#ifdef HAVE_SCHED_H
+#include <sched.h>
+#endif
+
+#include "opal/threads/mutex.h"
+#include "opal/class/opal_object.h"
 #include "opal/threads/mutex.h"
 
 #include "ompi/communicator/communicator.h"
 #include "ompi/attribute/attribute.h"
 #include "ompi/errhandler/errhandler.h"
 
-static void cxx_comm_keyval_destructor(int keyval);
+// Struct to make a linked list of keyval intercept data
+struct keyval_intercept_data_item_t {
+    opal_list_item_t super;
+    int kid_keyval;
+    MPI::Comm::keyval_intercept_data_t *kid_data;
+};
+
+// We are explicitly *not* using the STL here (just for the sake of
+// avoiding using the STL; e.g., Solaris has 2 STL's -- which one
+// should OMPI use?  What if OMPI uses one and the MPI app wants to
+// use the other?), so use the C++-like opal_list_t stuff.
+OBJ_CLASS_DECLARATION(keyval_intercept_data_item_t);
+OBJ_CLASS_INSTANCE(keyval_intercept_data_item_t, opal_list_item_t, NULL, NULL);
+
+// List to hold the cxx_extra_state structs that are new'ed when C++
+// keyvals are created
+static opal_list_t cxx_extra_states;
+// Whether or not cxx_extra_states has been initialized yet
+static volatile bool cxx_extra_states_init = false;
+// Will be set to 1 by the thread who is actually doing the initialization
+static volatile int32_t cxx_extra_states_init_thread = 0;
+// Lock to protect cxx_extra_states from being accessed by multiple
+// threads at the same time
+opal_mutex_t MPI::Comm::cxx_extra_states_lock;
 
 //
 // These functions are all not inlined because they need to use locks to
@@ -61,6 +91,30 @@ MPI::Comm::Set_errhandler(const MPI::Errhandler& errhandler)
 }
 
 
+// This function is called back out of the keyval destructor in the C
+// layer when the keyval is not be used by any attributes anymore,
+// anywhere.  So we can definitely safely remove the entry for this
+// keyval's C++ intercept extra_state from the list
+static void cxx_comm_keyval_destructor(int keyval) 
+{
+    opal_list_item_t *item;
+    keyval_intercept_data_item_t *kid;
+
+    // Search the list until we find the item with the same keyval
+    for (item = opal_list_get_first(&cxx_extra_states);
+         opal_list_get_end(&cxx_extra_states) != item; 
+         item = opal_list_get_next(item)) {
+        kid = (keyval_intercept_data_item_t *) item;
+        if (kid->kid_keyval == keyval) {
+            delete kid->kid_data;
+            opal_list_remove_item(&cxx_extra_states, item);
+            OBJ_RELEASE(item);
+            break;
+        }
+    }
+}
+
+
 //JGS I took the const out because it causes problems when trying to
 //call this function with the predefined NULL_COPY_FN etc.
 int
@@ -68,13 +122,61 @@ MPI::Comm::do_create_keyval(MPI_Comm_copy_attr_function* c_copy_fn,
                             MPI_Comm_delete_attr_function* c_delete_fn,
                             Copy_attr_function* cxx_copy_fn,
                             Delete_attr_function* cxx_delete_fn,
-                            void* extra_state)
+                            void* extra_state, int &keyval)
 {
-    int keyval, ret, count = 0;
+    int ret, count = 0;
     ompi_attribute_fn_ptr_union_t copy_fn;
     ompi_attribute_fn_ptr_union_t delete_fn;
-    Copy_attr_function *cxx_pair_copy = NULL;
-    Delete_attr_function *cxx_pair_delete = NULL;
+    keyval_intercept_data_t *cxx_extra_state;
+
+    // If both the callbacks are C, then do the simple thing -- no
+    // need for all the C++ machinery.
+    if (NULL != c_copy_fn && NULL != c_delete_fn) {
+        copy_fn.attr_communicator_copy_fn = 
+            (MPI_Comm_internal_copy_attr_function*) c_copy_fn;
+        delete_fn.attr_communicator_delete_fn = c_delete_fn;
+        ret = ompi_attr_create_keyval(COMM_ATTR, copy_fn, delete_fn,
+                                      &keyval, extra_state, 0, NULL);
+        if (MPI_SUCCESS != ret) {
+            return OMPI_ERRHANDLER_INVOKE(MPI_COMM_WORLD, ret,
+                                          "MPI::Comm::Create_keyval");
+        }
+    }
+
+    // If either callback is C++, then we have to use the C++
+    // callbacks for both, because we have to generate a new
+    // extra_state.  And since we only get one extra_state (i.e., we
+    // don't get one extra_state for the copy callback and another
+    // extra_state for the delete callback), we have to use the C++
+    // callbacks for both (and therefore translate the C++-special
+    // extra_state into the user's original extra_state).
+    cxx_extra_state = new keyval_intercept_data_t;
+    if (NULL == cxx_extra_state) {
+        return OMPI_ERRHANDLER_INVOKE(MPI_COMM_WORLD, MPI_ERR_NO_MEM, 
+                                      "MPI::Comm::Create_keyval");
+    }
+    cxx_extra_state->c_copy_fn = c_copy_fn;
+    cxx_extra_state->cxx_copy_fn = cxx_copy_fn;
+    cxx_extra_state->c_delete_fn = c_delete_fn;
+    cxx_extra_state->cxx_delete_fn = cxx_delete_fn;
+
+    // Error check.  Must have exactly 2 non-NULL function pointers.
+    if (NULL != c_copy_fn) {
+        ++count;
+    }
+    if (NULL != c_delete_fn) {
+        ++count;
+    }
+    if (NULL != cxx_copy_fn) {
+        ++count;
+    }
+    if (NULL != cxx_delete_fn) {
+        ++count;
+    }
+    if (2 != count) {
+        return OMPI_ERRHANDLER_INVOKE(MPI_COMM_WORLD, MPI_ERR_ARG, 
+                                      "MPI::Comm::Create_keyval");
+    }
 
     // We do not call MPI_Comm_create_keyval() here because we need to
     // pass in a special destructor to the backend keyval creation
@@ -86,61 +188,45 @@ MPI::Comm::do_create_keyval(MPI_Comm_copy_attr_function* c_copy_fn,
     // ensure to set the destructor atomicly when the keyval is
     // created).
 
-    // Error check.  Must have exactly 2 non-NULL function pointers.
-    if (NULL != c_copy_fn) {
-        copy_fn.attr_communicator_copy_fn = 
-            (MPI_Comm_internal_copy_attr_function*) c_copy_fn;
-        ++count;
-    }
-    if (NULL != c_delete_fn) {
-        delete_fn.attr_communicator_delete_fn = c_delete_fn;
-        ++count;
-    }
-    if (NULL != cxx_copy_fn) {
-        copy_fn.attr_communicator_copy_fn =
-            (MPI_Comm_internal_copy_attr_function*) ompi_mpi_cxx_comm_copy_attr_intercept;
-        cxx_pair_copy = cxx_copy_fn;
-        ++count;
-    }
-    if (NULL != cxx_delete_fn) {
-        delete_fn.attr_communicator_delete_fn =
-            ompi_mpi_cxx_comm_delete_attr_intercept;
-        cxx_pair_delete = cxx_delete_fn;
-        ++count;
-    }
-    if (2 != count) {
-        return OMPI_ERRHANDLER_INVOKE(MPI_COMM_WORLD, MPI_ERR_ARG, 
+    copy_fn.attr_communicator_copy_fn =
+        (MPI_Comm_internal_copy_attr_function*) 
+        ompi_mpi_cxx_comm_copy_attr_intercept;
+    delete_fn.attr_communicator_delete_fn =
+        ompi_mpi_cxx_comm_delete_attr_intercept;
+    ret = ompi_attr_create_keyval(COMM_ATTR, copy_fn, delete_fn,
+                                  &keyval, cxx_extra_state, 0,
+                                  cxx_comm_keyval_destructor);
+    if (OMPI_SUCCESS != ret) {
+        return OMPI_ERRHANDLER_INVOKE(MPI_COMM_WORLD, ret,
                                       "MPI::Comm::Create_keyval");
     }
 
-    ret = ompi_attr_create_keyval(COMM_ATTR, copy_fn, delete_fn,
-                                  &keyval, extra_state, 0,
-                                  cxx_comm_keyval_destructor);
-    if (OMPI_SUCCESS != ret) {
-        return ret;
+    // Ensure to initialize the list safely
+    if (opal_atomic_cmpset_32(&cxx_extra_states_init_thread, 0, 1)) {
+        OBJ_CONSTRUCT(&cxx_extra_states, opal_list_t);
+        OBJ_CONSTRUCT(&cxx_extra_states_lock, opal_mutex_t);
+        cxx_extra_states_init = true;
+    } else {
+        while (!cxx_extra_states_init) {
+#if defined(__WINDOWS__)
+            SwitchToThread();
+#else
+            sched_yield();
+#endif  /* defined(__WINDOWS__) */
+        }
     }
 
-    keyval_pair_t* copy_and_delete = 
-        new keyval_pair_t(cxx_pair_copy, cxx_pair_delete);
-    OPAL_THREAD_LOCK(MPI::mpi_map_mutex);
-    MPI::Comm::mpi_comm_keyval_fn_map[keyval] = copy_and_delete;
-    OPAL_THREAD_UNLOCK(MPI::mpi_map_mutex);
-    return keyval;
-}
-
-// This function is called back out of the keyval destructor in the C
-// layer when the keyval is not be used by any attributes anymore,
-// anywhere.  So we can definitely safely remove the entry for this
-// keyval from the C++ map.
-static void cxx_comm_keyval_destructor(int keyval) 
-{
-    OPAL_THREAD_LOCK(MPI::mpi_map_mutex);
-    if (MPI::Comm::mpi_comm_keyval_fn_map.end() !=
-        MPI::Comm::mpi_comm_keyval_fn_map.find(keyval) &&
-        NULL != MPI::Comm::mpi_comm_keyval_fn_map[keyval]) {
-        delete MPI::Comm::mpi_comm_keyval_fn_map[keyval];
-        MPI::Comm::mpi_comm_keyval_fn_map.erase(keyval);
-    }
-    OPAL_THREAD_UNLOCK(MPI::mpi_map_mutex);
+    // Put this cxx_extra_state in a place where the
+    // cxx_comm_keyval_destructor can find it based on the keyval
+    // (because that's all the cxx_comm_keyval_destructor gets as an
+    // argument)
+    keyval_intercept_data_item_t *kid = OBJ_NEW(keyval_intercept_data_item_t);
+    kid->kid_keyval = keyval;
+    kid->kid_data = cxx_extra_state;
+    OPAL_THREAD_LOCK(&cxx_extra_states_lock);
+    opal_list_append(&cxx_extra_states, &kid->super);
+    OPAL_THREAD_UNLOCK(&cxx_extra_states_lock);
+
+    return MPI_SUCCESS;
 }
 

ompi/mpi/cxx/comm.h

@@ -10,7 +10,7 @@
 //                         University of Stuttgart.  All rights reserved.
 // Copyright (c) 2004-2005 The Regents of the University of California.
 //                         All rights reserved.
-// Copyright (c) 2006-2007 Cisco Systems, Inc.  All rights reserved.
+// Copyright (c) 2006-2008 Cisco Systems, Inc.  All rights reserved.
 // $COPYRIGHT$
 // 
 // Additional copyrights may follow
@@ -412,7 +412,7 @@ protected:
                               MPI_Comm_delete_attr_function* c_delete_fn,
                               Copy_attr_function* cxx_copy_fn,
                               Delete_attr_function* cxx_delete_fn,
-                              void* extra_state);
+                              void* extra_state, int &keyval);
 
 public:
 
@@ -450,14 +450,22 @@ public: // JGS hmmm, these used by errhandler_intercept
   typedef ::std::map<MPI_Comm, Comm*> mpi_comm_err_map_t;
   static mpi_comm_err_map_t mpi_comm_err_map;
   
-  typedef ::std::pair<Comm::_MPI2CPP_COPYATTRFN_*, Comm::_MPI2CPP_DELETEATTRFN_*> keyval_pair_t;
-  typedef ::std::map<int, keyval_pair_t*> mpi_comm_keyval_fn_map_t;
-  static mpi_comm_keyval_fn_map_t mpi_comm_keyval_fn_map;
-  
   void init() {
     my_errhandler = (Errhandler*)0;
   }
 
+    // Data that is passed through keyval create when C++ callback
+    // functions are used
+    struct keyval_intercept_data_t {
+        MPI_Comm_copy_attr_function *c_copy_fn;
+        MPI_Comm_delete_attr_function *c_delete_fn;
+        Copy_attr_function* cxx_copy_fn;
+        Delete_attr_function* cxx_delete_fn;
+        void *extra_state;
+    };
+
+    // Protect the global list from multiple thread access
+    static opal_mutex_t cxx_extra_states_lock;
 #endif
 
 };

ompi/mpi/cxx/comm_inln.h

@@ -10,7 +10,7 @@
 //                         University of Stuttgart.  All rights reserved.
 // Copyright (c) 2004-2005 The Regents of the University of California.
 //                         All rights reserved.
-// Copyright (c) 2007      Cisco Systems, Inc.  All rights reserved.
+// Copyright (c) 2007-2008 Cisco Systems, Inc.  All rights reserved.
 // $COPYRIGHT$
 // 
 // Additional copyrights may follow
@@ -560,9 +560,11 @@ MPI::Comm::Create_keyval(MPI::Comm::Copy_attr_function* comm_copy_attr_fn,
                          void* extra_state)
 {
     // Back-end function does the heavy lifting
-    return do_create_keyval(NULL, NULL, 
-                            comm_copy_attr_fn, comm_delete_attr_fn,
-                            extra_state);
+    int ret, keyval;
+    ret = do_create_keyval(NULL, NULL, 
+                           comm_copy_attr_fn, comm_delete_attr_fn,
+                           extra_state, keyval);
+    return (MPI_SUCCESS == ret) ? keyval : ret;
 }
 
 // 2) overload Create_keyval to take the first 2 arguments as C
@@ -573,9 +575,11 @@ MPI::Comm::Create_keyval(MPI_Comm_copy_attr_function* comm_copy_attr_fn,
                          void* extra_state)
 {
     // Back-end function does the heavy lifting
+    int ret, keyval;
     return do_create_keyval(comm_copy_attr_fn, comm_delete_attr_fn,
                             NULL, NULL,
-                            extra_state);
+                            extra_state, keyval);
+    return (MPI_SUCCESS == ret) ? keyval : ret;
 }
 
 // 3) overload Create_keyval to take the first 2 arguments as C++ & C
@@ -586,9 +590,11 @@ MPI::Comm::Create_keyval(MPI::Comm::Copy_attr_function* comm_copy_attr_fn,
                          void* extra_state)
 {
     // Back-end function does the heavy lifting
+    int ret, keyval;
     return do_create_keyval(NULL, comm_delete_attr_fn,
                             comm_copy_attr_fn, NULL,
-                            extra_state);
+                            extra_state, keyval);
+    return (MPI_SUCCESS == ret) ? keyval : ret;
 }
 
 // 4) overload Create_keyval to take the first 2 arguments as C & C++
@@ -599,9 +605,11 @@ MPI::Comm::Create_keyval(MPI_Comm_copy_attr_function* comm_copy_attr_fn,
                          void* extra_state)
 {
     // Back-end function does the heavy lifting
+    int ret, keyval;
     return do_create_keyval(comm_copy_attr_fn, NULL,
                             NULL, comm_delete_attr_fn,
-                            extra_state);
+                            extra_state, keyval);
+    return (MPI_SUCCESS == ret) ? keyval : ret;
 }
 
 inline void

ompi/mpi/cxx/intercepts.cc

@@ -10,7 +10,7 @@
 //                         University of Stuttgart.  All rights reserved.
 // Copyright (c) 2004-2005 The Regents of the University of California.
 //                         All rights reserved.
-// Copyright (c) 2006-2007 Cisco Systems, Inc.  All rights reserved.
+// Copyright (c) 2006-2008 Cisco Systems, Inc.  All rights reserved.
 // $COPYRIGHT$
 // 
 // Additional copyrights may follow
@@ -28,7 +28,6 @@
 #include "opal/threads/mutex.h"
 
 MPI::Comm::mpi_comm_err_map_t MPI::Comm::mpi_comm_err_map;
-MPI::Comm::mpi_comm_keyval_fn_map_t MPI::Comm::mpi_comm_keyval_fn_map;
 
 MPI::Win::mpi_win_map_t MPI::Win::mpi_win_map;
 MPI::Win::mpi_win_keyval_fn_map_t MPI::Win::mpi_win_keyval_fn_map;
@@ -288,19 +287,25 @@ ompi_mpi_cxx_op_intercept(void *invec, void *outvec, int *len,
 //
 extern "C" int
 ompi_mpi_cxx_comm_copy_attr_intercept(MPI_Comm comm, int keyval, 
-                                      void *extra_state, void *attribute_val_in, 
+                                      void *extra_state, 
+                                      void *attribute_val_in, 
                                       void *attribute_val_out, int *flag,
                                       MPI_Comm newcomm)
 {
   int ret = 0;
-  MPI::Comm::keyval_pair_t* copy_and_delete; 
-  MPI::Comm::Copy_attr_function* copy_fn;
+  MPI::Comm::keyval_intercept_data_t *kid = 
+      (MPI::Comm::keyval_intercept_data_t*) extra_state;
+
+  // The callback may be in C or C++.  If it's in C, it's easy - just
+  // call it with no extra C++ machinery.
+
+  if (NULL != kid->c_copy_fn) {
+      return kid->c_copy_fn(comm, keyval, kid->extra_state, attribute_val_in,
+                            attribute_val_out, flag);
+  }
+
+  // If the callback was C++, we have to do a little more work
 
-  OPAL_THREAD_LOCK(MPI::mpi_map_mutex);
-  copy_and_delete = MPI::Comm::mpi_comm_keyval_fn_map[keyval];
-  copy_fn = copy_and_delete->first;
-  OPAL_THREAD_UNLOCK(MPI::mpi_map_mutex);
-  
   MPI::Intracomm intracomm;
   MPI::Intercomm intercomm;
   MPI::Graphcomm graphcomm;
@@ -308,23 +313,27 @@ ompi_mpi_cxx_comm_copy_attr_intercept(MPI_Comm comm, int keyval,
   
   bool bflag = OPAL_INT_TO_BOOL(*flag); 
 
-  if (NULL != copy_fn) {
+  if (NULL != kid->cxx_copy_fn) {
       if (OMPI_COMM_IS_GRAPH(comm)) {
           graphcomm = MPI::Graphcomm(comm);
-          ret = copy_fn(graphcomm, keyval, extra_state,
-                        attribute_val_in, attribute_val_out, bflag);
+          ret = kid->cxx_copy_fn(graphcomm, keyval, kid->extra_state,
+                                 attribute_val_in, attribute_val_out, 
+                                 bflag);
       } else if (OMPI_COMM_IS_CART(comm)) {
           cartcomm = MPI::Cartcomm(comm);
-          ret = copy_fn(cartcomm, keyval, extra_state,
-                        attribute_val_in, attribute_val_out, bflag);
+          ret = kid->cxx_copy_fn(cartcomm, keyval, kid->extra_state,
+                                 attribute_val_in, attribute_val_out, 
+                                 bflag);
       } else if (OMPI_COMM_IS_INTRA(comm)) {
           intracomm = MPI::Intracomm(comm);
-          ret = copy_fn(intracomm, keyval, extra_state,
-                        attribute_val_in, attribute_val_out, bflag);
+          ret = kid->cxx_copy_fn(intracomm, keyval, kid->extra_state,
+                                 attribute_val_in, attribute_val_out, 
+                                 bflag);
       } else if (OMPI_COMM_IS_INTER(comm)) {
           intercomm = MPI::Intercomm(comm);
-          ret = copy_fn(intercomm, keyval, extra_state,
-                        attribute_val_in, attribute_val_out, bflag);
+          ret = kid->cxx_copy_fn(intercomm, keyval, kid->extra_state,
+                                 attribute_val_in, attribute_val_out, 
+                                 bflag);
       } else {
           ret = MPI::ERR_COMM;
       }
@@ -341,32 +350,40 @@ ompi_mpi_cxx_comm_delete_attr_intercept(MPI_Comm comm, int keyval,
                                         void *attribute_val, void *extra_state)
 {
   int ret = 0;
-  MPI::Comm::keyval_pair_t * copy_and_delete;
-  MPI::Comm::Delete_attr_function* delete_fn;  
+  MPI::Comm::keyval_intercept_data_t *kid = 
+      (MPI::Comm::keyval_intercept_data_t*) extra_state;
 
-  OPAL_THREAD_LOCK(MPI::mpi_map_mutex);
-  copy_and_delete = MPI::Comm::mpi_comm_keyval_fn_map[keyval];
-  delete_fn = copy_and_delete->second;
-  OPAL_THREAD_UNLOCK(MPI::mpi_map_mutex);
+  // The callback may be in C or C++.  If it's in C, it's easy - just
+  // call it with no extra C++ machinery.
+
+  if (NULL != kid->c_delete_fn) {
+      return kid->c_delete_fn(comm, keyval, attribute_val, kid->extra_state);
+  }
+
+  // If the callback was C++, we have to do a little more work
 
   MPI::Intracomm intracomm;
   MPI::Intercomm intercomm;
   MPI::Graphcomm graphcomm;
   MPI::Cartcomm cartcomm;
   
-  if (NULL != delete_fn) {
+  if (NULL != kid->cxx_delete_fn) {
       if (OMPI_COMM_IS_GRAPH(comm)) {
           graphcomm = MPI::Graphcomm(comm);
-          ret = delete_fn(graphcomm, keyval, attribute_val, extra_state);
+          ret = kid->cxx_delete_fn(graphcomm, keyval, attribute_val, 
+                                   kid->extra_state);
       } else if (OMPI_COMM_IS_CART(comm)) {
           cartcomm = MPI::Cartcomm(comm);
-          ret = delete_fn(cartcomm, keyval, attribute_val, extra_state);
+          ret = kid->cxx_delete_fn(cartcomm, keyval, attribute_val, 
+                                   kid->extra_state);
       } else if (OMPI_COMM_IS_INTRA(comm)) {
           intracomm = MPI::Intracomm(comm);
-          ret = delete_fn(intracomm, keyval, attribute_val, extra_state);
+          ret = kid->cxx_delete_fn(intracomm, keyval, attribute_val, 
+                                   kid->extra_state);
       } else if (OMPI_COMM_IS_INTER(comm)) {
           intercomm = MPI::Intercomm(comm);
-          ret = delete_fn(intercomm, keyval, attribute_val, extra_state);
+          ret = kid->cxx_delete_fn(intercomm, keyval, attribute_val, 
+                                   kid->extra_state);
       } else {
           ret = MPI::ERR_COMM;
       }

ompi/mpi/cxx/mpicxx.h

@@ -10,7 +10,7 @@
 //                         University of Stuttgart.  All rights reserved.
 // Copyright (c) 2004-2005 The Regents of the University of California.
 //                         All rights reserved.
-// Copyright (c) 2006-2007 Cisco Systems, Inc.  All rights reserved.
+// Copyright (c) 2006-2008 Cisco Systems, Inc.  All rights reserved.
 // $COPYRIGHT$
 // 
 // Additional copyrights may follow
@@ -212,6 +212,7 @@ namespace MPI {
 #include "openmpi/ompi/mpi/cxx/info.h"
 #endif
 
+    // JMS To be deleted when the all the STL is removed from these bindings
     extern opal_mutex_t *mpi_map_mutex;
 }