Fix a problem in the selection logic for MX. Basically we need to be able to

open MTL MX and BTL MX and initialize them at the same time. The problem is
that both call mx_init and mx_finalize, solution is to add an external entity
that does the init and finalize (based on ref counting).

This commit was SVN r13576.

ompi/mca/btl/mx/Makefile.am

@@ -42,6 +42,7 @@ mcacomponent_LTLIBRARIES = $(component_install)
 mca_btl_mx_la_SOURCES = $(btl_mx_sources)
 mca_btl_mx_la_LIBADD = \
     $(btl_mx_LIBS) \
+    $(top_ompi_builddir)/ompi/mca/common/mx/libmca_common_mx.la \
     $(top_ompi_builddir)/ompi/libmpi.la \
     $(top_ompi_builddir)/orte/libopen-rte.la \
     $(top_ompi_builddir)/opal/libopen-pal.la

ompi/mca/btl/mx/btl_mx_component.c

@@ -28,6 +28,8 @@
 #include "orte/mca/errmgr/errmgr.h"
 #include "ompi/mca/pml/base/pml_base_module_exchange.h"
 #include "ompi/mca/btl/base/btl_base_error.h"
+#include "ompi/mca/common/mx/common_mx.h"
+
 #include "btl_mx.h"
 #include "btl_mx_frag.h"
 #include "btl_mx_endpoint.h" 
@@ -176,8 +178,10 @@ int mca_btl_mx_component_close(void)
 {
     if( NULL == mca_btl_mx_component.mx_btls )
         return OMPI_SUCCESS;
-
-    mx_finalize();
+    
+    if(OMPI_SUCCESS != ompi_common_mx_finalize()) { 
+        return OMPI_ERROR;
+    }
 
     /* release resources */
     OBJ_DESTRUCT(&mca_btl_mx_component.mx_send_eager_frags);
@@ -376,16 +380,12 @@ mca_btl_base_module_t** mca_btl_mx_component_init(int *num_btl_modules,
     opal_setenv( "MX_PIPELINE_LOG", "0", true, &environ );
 
     /* First check if MX is available ... */
-    if( MX_SUCCESS != (status = mx_init()) ) {
-        if(MX_ALREADY_INITIALIZED != status) {
-            opal_output( 0, "mca_btl_mx_component_init: mx_init() failed with status = %d (%s)\n",
-                         status, mx_strerror(status) );
-        }
+    if(OMPI_SUCCESS!=ompi_common_mx_initialize()) { 
         mca_pml_base_modex_send(&mca_btl_mx_component.super.btl_version, 
                                 NULL, 0);
         return NULL;
     }
-
+        
     /* initialize objects */
     OBJ_CONSTRUCT(&mca_btl_mx_component.mx_send_eager_frags, ompi_free_list_t);
     OBJ_CONSTRUCT(&mca_btl_mx_component.mx_send_user_frags, ompi_free_list_t);

ompi/mca/common/mx/Makefile.am

@@ -0,0 +1,95 @@
+#
+# Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
+#                         University Research and Technology
+#                         Corporation.  All rights reserved.
+# Copyright (c) 2004-2005 The University of Tennessee and The University
+#                         of Tennessee Research Foundation.  All rights
+#                         reserved.
+# Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, 
+#                         University of Stuttgart.  All rights reserved.
+# Copyright (c) 2004-2005 The Regents of the University of California.
+#                         All rights reserved.
+# $COPYRIGHT$
+# 
+# Additional copyrights may follow
+# 
+# $HEADER$
+#
+
+# A word of explanation...
+#
+# This library is linked against various MCA components because all
+# shared-memory based components (e.g., mpool, ptl, etc.)  need to
+# share some common code and data.  There's two cases:
+#
+# 1. libmca_common_portals.la is a shared library.  By linking that shared
+# library to all components that need it, the OS linker will
+# automatically load it into the process as necessary, and there will
+# only be one copy (i.e., all the components will share *one* copy of
+# the code and data).
+# 2. libmca_common_portals.la is a static library.  In this case, it
+# will be rolled up into the top-level libmpi.la.  It will also be
+# rolled into each component, but then the component will also be
+# rolled up into the upper-level libmpi.la.  Libtool sorts this all
+# out and it all works out in the end.
+#
+# Note that building this common component statically and linking
+# against other dynamic components is *not* supported!
+
+
+AM_CPPFLAGS = $(common_mx_CPPFLAGS)
+
+# Header files
+
+headers = \
+        common_mx.h
+
+# Source files
+
+sources = \
+        common_mx.c
+
+
+lib_LTLIBRARIES =
+noinst_LTLIBRARIES =
+comp_inst = libmca_common_mx.la
+comp_noinst = libmca_common_mx_noinst.la
+
+if OMPI_BUILD_common_mx_DSO
+lib_LTLIBRARIES += $(comp_inst)
+else
+noinst_LTLIBRARIES += $(comp_noinst)
+endif
+
+libmca_common_mx_la_SOURCES = $(headers) $(sources)
+libmca_common_mx_la_LDFLAGS = $(common_mx_LDFLAGS)
+libmca_common_mx_la_LIBADD = $(common_mx_LIBS)
+
+libmca_common_mx_noinst_la_SOURCES = $(libmca_common_mx_la_SOURCES)
+libmca_common_mx_noinst_la_LDFLAGS = $(common_mx_LDFLAGS)
+libmca_common_mx_noinst_la_LIBADD = $(common_mx_LIBS)
+
+
+# Conditionally install the header files
+
+if WANT_INSTALL_HEADERS
+ompidir = $(includedir)/openmpi/ompi/mca/common/mx
+ompi_HEADERS = $(headers)
+else
+ompidir = $(includedir)
+endif
+
+# These two rules will sym link the "noinst" libtool library filename
+# to the installable libtool library filename in the case where we are
+# compiling this component statically (case 2), described above).
+
+all-local:
+	if test -z "$(lib_LTLIBRARIES)"; then \
+	  rm -f "$(comp_inst)"; \
+	  $(LN_S) "$(comp_noinst)" "$(comp_inst)"; \
+	fi
+
+clean-local:
+	if test -z "$(lib_LTLIBRARIES)"; then \
+	  rm -f "$(comp_inst)"; \
+	fi

ompi/mca/common/mx/common_mx.c

@@ -0,0 +1,68 @@
+/*
+ * Copyright (c) 2004-2006 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2005 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2006 The Regents of the University of California.
+ *                         All rights reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+
+#include "ompi_config.h"
+#include "opal/util/output.h"
+#include "ompi/constants.h"
+#include "common_mx.h"
+
+int ompi_common_mx_initialize_ref_cnt = 0;
+int
+ompi_common_mx_initialize(void)
+{
+    mx_return_t mx_return;
+    ompi_common_mx_initialize_ref_cnt++;
+    
+    if(ompi_common_mx_initialize_ref_cnt == 1) { 
+        /* set the MX error handle to always return. This function is the
+         * only MX function allowed to be called before mx_init in order
+         * to make sure that if the MX is not up and running the MX
+         * library does not exit the application.
+         */
+        mx_set_error_handler(MX_ERRORS_RETURN);
+        
+        /* initialize the mx library */
+        mx_return = mx_init(); 
+        
+        if(MX_SUCCESS != mx_return){
+            opal_output(0,
+                        "Error in mx_init (error %s)\n",
+                        mx_strerror(mx_return));
+            return OMPI_ERR_NOT_AVAILABLE;
+        }
+        
+    } 
+    return OMPI_SUCCESS;
+}
+
+
+int
+ompi_common_mx_finalize(void)
+{
+    mx_return_t mx_return;
+    ompi_common_mx_initialize_ref_cnt--;
+    if(ompi_common_mx_initialize == 0) { 
+        mx_return = mx_finalize(); 
+        if(mx_return != MX_SUCCESS){ 
+            opal_output(0, "Error in mx_finalize (error %s)\n", mx_strerror(mx_return));
+            return OMPI_ERROR;
+        } 
+    } else {
+        return OMPI_SUCCESS;
+    }
+}

ompi/mca/common/mx/common_mx.h

@@ -0,0 +1,42 @@
+/*
+ * Copyright (c) 2004-2006 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2005 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2006 The Regents of the University of California.
+ *                         All rights reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+
+#ifndef OMPI_MCA_COMMON_MX_H
+#define OMPI_MCA_COMMON_MX_H
+
+#include "myriexpress.h"
+
+
+/**
+ * Initialize mx library
+ * *
+ * @retval OMPI_SUCCESS MX successfully initialized
+ * @retval OMPI_ERR_NOT_AVAILABLE MX could not be initialized
+ */
+int ompi_common_mx_initialize(void);
+
+
+/**
+ * Shut down mx library 
+ *
+ */
+int ompi_common_mx_finalize(void);
+
+
+
+#endif /* OMPI_MCA_COMMON_PORTALS_H */

ompi/mca/common/mx/configure.m4

@@ -0,0 +1,41 @@
+# -*- shell-script -*-
+#
+# Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
+#                         University Research and Technology
+#                         Corporation.  All rights reserved.
+# Copyright (c) 2004-2005 The University of Tennessee and The University
+#                         of Tennessee Research Foundation.  All rights
+#                         reserved.
+# Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
+#                         University of Stuttgart.  All rights reserved.
+# Copyright (c) 2004-2005 The Regents of the University of California.
+#                         All rights reserved.
+# $COPYRIGHT$
+#
+# Additional copyrights may follow
+#
+# $HEADER$
+#
+
+
+# MCA_mtl_mx_CONFIG([action-if-can-compile],
+#                      [action-if-cant-compile])
+# ------------------------------------------------
+AC_DEFUN([MCA_common_mx_CONFIG],[
+    OMPI_CHECK_MX([common_mx],
+                     [common_mx_happy="yes"],
+                     [common_mx_happy="no"])
+
+    AS_IF([test "$common_mx_happy" = "yes"],
+          [common_mx_WRAPPER_EXTRA_LDFLAGS="$common_mx_LDFLAGS"
+           common_mx_WRAPPER_EXTRA_LIBS="$common_mx_LIBS"
+           $1],
+          [$2])
+
+    # substitute in the things needed to build mx
+    AC_SUBST([common_mx_CFLAGS])
+    AC_SUBST([common_mx_CPPFLAGS])
+    AC_SUBST([common_mx_LDFLAGS])
+    AC_SUBST([common_mx_LIBS])
+])dnl
+

ompi/mca/common/mx/configure.params

@@ -0,0 +1,24 @@
+# -*- shell-script -*-
+#
+# Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
+#                         University Research and Technology
+#                         Corporation.  All rights reserved.
+# Copyright (c) 2004-2005 The University of Tennessee and The University
+#                         of Tennessee Research Foundation.  All rights
+#                         reserved.
+# Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, 
+#                         University of Stuttgart.  All rights reserved.
+# Copyright (c) 2004-2005 The Regents of the University of California.
+#                         All rights reserved.
+# Copyright (c) 2007      Los Alamos National Security, LLC.  All rights
+#                         reserved. 
+# $COPYRIGHT$
+# 
+# Additional copyrights may follow
+# 
+# $HEADER$
+#
+
+# Specific to this module
+
+PARAM_CONFIG_FILES="Makefile"

ompi/mca/mtl/mx/Makefile.am

@@ -50,6 +50,7 @@ mcacomponent_LTLIBRARIES = $(component_install)
 mca_mtl_mx_la_SOURCES = $(mtl_mx_sources)
 mca_mtl_mx_la_LIBADD = \
     $(mtl_mx_LIBS) \
+    $(top_ompi_builddir)/ompi/mca/common/mx/libmca_common_mx.la \
     $(top_ompi_builddir)/ompi/libmpi.la \
     $(top_ompi_builddir)/orte/libopen-rte.la \
     $(top_ompi_builddir)/opal/libopen-pal.la

ompi/mca/mtl/mx/mtl_mx.c

@@ -25,7 +25,7 @@
 #include "opal/class/opal_list.h"
 #include "ompi/mca/pml/base/pml_base_module_exchange.h"
 #include "ompi/mca/mtl/base/mtl_base_datatype.h"
-
+#include "ompi/mca/common/mx/common_mx.h"
 #include "mtl_mx.h"
 #include "mtl_mx_types.h"
 #include "mtl_mx_endpoint.h"
@@ -114,7 +114,7 @@ int ompi_mtl_mx_module_init(){
 int
 ompi_mtl_mx_finalize(struct mca_mtl_base_module_t* mtl) { 
     mx_return_t mx_return;
-
+    
     opal_progress_unregister(ompi_mtl_mx_progress);
     
     /* free resources */
@@ -123,14 +123,9 @@ ompi_mtl_mx_finalize(struct mca_mtl_base_module_t* mtl) {
         opal_output(ompi_mtl_base_output, "Error in mx_close_endpoint (error %s)\n", mx_strerror(mx_return));
         return OMPI_ERROR;
     }
-
-    mx_return = mx_finalize(); 
-    if(mx_return != MX_SUCCESS){ 
-        opal_output(ompi_mtl_base_output, "Error in mx_finalize (error %s)\n", mx_strerror(mx_return));
-        return OMPI_ERROR;
-    }
     
-    return OMPI_SUCCESS;
+    return ompi_common_mx_finalize();
+    
 }
 
 int

ompi/mca/mtl/mx/mtl_mx_component.c

@@ -23,6 +23,7 @@
 #include "opal/util/output.h"
 #include "ompi/datatype/convertor.h"
 #include "ompi/mca/mtl/base/base.h"
+#include "ompi/mca/common/mx/common_mx.h"
 
 #include "mtl_mx.h"
 #include "mtl_mx_types.h"
@@ -113,27 +114,11 @@ static mca_mtl_base_module_t*
 ompi_mtl_mx_component_init(bool enable_progress_threads,
                            bool enable_mpi_threads)
 {
-    mx_return_t mx_return;
-    int ret;
-
-    /* set the MX error handle to always return. This function is the
-     * only MX function allowed to be called before mx_init in order
-     * to make sure that if the MX is not up and running the MX
-     * library does not exit the application.
-     */
-    mx_set_error_handler(MX_ERRORS_RETURN);
+   int ret;
     
-    /* initialize the mx library */
-    mx_return = mx_init(); 
-    
-    if(MX_SUCCESS != mx_return){
-        if(MX_ALREADY_INITIALIZED  != mx_return) {
-            opal_output(ompi_mtl_base_output,
-                        "Error in mx_init (error %s)\n",
-                        mx_strerror(mx_return));
-        } else { 
-            return NULL;
-        }
+    ret = ompi_common_mx_initialize();
+    if(OMPI_SUCCESS != ret) { 
+        return NULL;
     }
         
     ret = ompi_mtl_mx_module_init();

ompi/mca/pml/cm/pml_cm_component.c

@@ -27,6 +27,7 @@ static int mca_pml_cm_component_close(void);
 static mca_pml_base_module_t* mca_pml_cm_component_init( int* priority,
                             bool enable_progress_threads, bool enable_mpi_threads);
 static int mca_pml_cm_component_fini(void);
+extern mca_mtl_base_component_t* ompi_mtl_base_selected_component;
 
 mca_pml_base_component_1_0_0_t mca_pml_cm_component = {
 
@@ -132,7 +133,14 @@ mca_pml_cm_component_init(int* priority,
     if (OMPI_SUCCESS != ret) { 
         *priority = -1;
         return NULL;
+    } else if(strcmp(ompi_mtl_base_selected_component->mtl_version.mca_component_name, "psm") != 0) {
+        /* if mtl is not PSM then back down priority, and require the user to */
+        /*  specify pml cm directly if that is what they want priority */
+        /*  of 1 is sufficient in that case as it is the only pml that */ 
+        /*  will be considered */
+        *priority = 1;
     }
+
     
     /* update our tag / context id max values based on MTL
        information */