Update common_portals to allow using portals interconnect with a modex rather

than relying on cnos to get the nid/pid map. 

This commit was SVN r17588.

LICENSE

@@ -23,6 +23,8 @@ Copyright (c) 2006-2007 Sun Microsystems, Inc.  All rights reserved.
                         Use is subject to license terms.
 Copyright (c) 2006-2007 The University of Houston. All rights reserved.
 Copyright (c) 2006      Myricom, Inc.  All rights reserved.
+Copyright (c) 2007-2008 UT-Battelle, LLC. All rights reserved.
+
 $COPYRIGHT$
 
 Additional copyrights may follow

config/ompi_check_portals.m4

@@ -43,7 +43,7 @@ AC_DEFUN([OMPI_CHECK_PORTALS],[
     AC_ARG_WITH([portals-config],
             AC_HELP_STRING([--with-portals-config],
                            [configuration to use for Portals support.
-                            One of "utcp", "xt3".  (default: utcp)]))
+                            One of "utcp", "xt3", "xt3-modex".  (default: utcp)]))
     AC_ARG_WITH([portals-libs], 
         [AC_HELP_STRING([--with-portals-libs=LIBS],
                        [Libraries to link with for portals])])
@@ -53,9 +53,12 @@ AC_DEFUN([OMPI_CHECK_PORTALS],[
         with_portals_config="utcp"
     elif test "$with_portals_config" = "redstorm" ; then
         with_portals_config="xt3"
+    elif test "$with_portals_config" = "cnl_modex" ; then
+	with_portals_config="xt3-modex"
     fi
     OMPI_PORTALS_UTCP=0
     OMPI_PORTALS_CRAYXT3=0
+    OMPI_PORTALS_CRAYXT3_MODEX=0
     case "$with_portals_config" in
         "utcp")
             OMPI_PORTALS_UTCP=1
@@ -71,6 +74,13 @@ AC_DEFUN([OMPI_CHECK_PORTALS],[
             check_portals_header_prefix="portals/"
             AC_MSG_RESULT([Cray XT3])
             ;;
+        "xt3-modex")
+            OMPI_PORTALS_CRAYXT3_MODEX=1
+            OMPI_PORTALS_HAVE_EVENT_UNLINK=0
+            check_portals_LIBS=
+            check_portals_header_prefix="portals/"
+            AC_MSG_RESULT([Cray XT3 - Modex])
+            ;;
         *)
             # ok to call ERROR here - the user specified something invalid.
             # that should be brought to his attention
@@ -84,9 +94,13 @@ AC_DEFUN([OMPI_CHECK_PORTALS],[
 
     AC_DEFINE_UNQUOTED([OMPI_PORTALS_UTCP], [$OMPI_PORTALS_UTCP],
                        [Use the UTCP reference implementation of Portals])
+
     AC_DEFINE_UNQUOTED([OMPI_PORTALS_CRAYXT3], [$OMPI_PORTALS_CRAYXT3],
                        [Use the Cray XT-3 implementation of Portals])
     
+    AC_DEFINE_UNQUOTED([OMPI_PORTALS_CRAYXT3_MODEX], [$OMPI_PORTALS_CRAYXT3_MODEX],
+                       [Use the Cray XT-3 implementation of Portals using Modex])
+
     # Add correct -I and -L flags
     AS_IF([test -n "$with_portals"],
           [AS_IF([test -d "$with_portals/include"],

contrib/platform/cray_xt_cnl_romio

@@ -4,7 +4,7 @@ enable_shared=no
 with_threads=no
 enable_pretty_print_stacktrace=no
 enable_dlopen=no
-with_portals_config=redstorm
+with_portals_config=cnl_modex
 with_memory_manager=none
 enable_mca_no_build=maffinity-first_use,maffinity-libnuma,paffinity-linux,pls-rsh,pml-dr,filem-rsh,grpcomm-cnos,pls-cnos,rmgr-cnos,rml-cnos,routed-cnos,sds-portals_utcp,sds-cnos
 enable_heterogeneous=no

ompi/mca/btl/portals/btl_portals.c

@@ -526,14 +526,14 @@ mca_btl_portals_finalize(struct mca_btl_base_module_t *btl_base)
         for (i = 0 ; i < OMPI_BTL_PORTALS_EQ_SIZE ; ++i) {
             int ptl_ret = PtlEQFree(mca_btl_portals_module.portals_eq_handles[i]);
             if (PTL_OK != ptl_ret) {
-#if OMPI_PORTALS_CRAYXT3
+#if (OMPI_PORTALS_CRAYXT3 || OMPI_PORTALS_CRAYXT3_MODEX)
                 if (i != OMPI_BTL_PORTALS_EQ_SEND && PTL_EQ_IN_USE != ptl_ret) {
                     /* The PML isn't great about cleaning up after itself.
                        Ignore related errors. */
 #endif
                     opal_output(mca_btl_portals_component.portals_output,
                             "Error freeing EQ %d: %d", i, ptl_ret);
-#if OMPI_PORTALS_CRAYXT3
+#if (OMPI_PORTALS_CRAYXT3 || OMPI_PORTALS_CRAYXT3_MODEX)
                 }
 #endif
             }

ompi/mca/btl/portals/btl_portals.h

@@ -61,6 +61,9 @@ struct mca_btl_portals_component_t {
 
     /* number of eager fragments */
     int portals_free_list_eager_max_num;
+    
+    /* shall I use portals to send to thyself? */
+    int portals_support_self;
 };
 typedef struct mca_btl_portals_component_t mca_btl_portals_component_t;
 

ompi/mca/btl/portals/btl_portals_component.c

@@ -125,6 +125,14 @@ mca_btl_portals_component_open(void)
                            32,
                            &(mca_btl_portals_component.portals_free_list_eager_max_num));
 
+    mca_base_param_reg_int(&mca_btl_portals_component.super.btl_version,
+                           "support_self",
+                           "Use portals for send to self",
+                           false,
+                           false,
+                           1, /* default to true.. */ 
+                           &(mca_btl_portals_component.portals_support_self));
+
     /* 
      * fill default module state 
      */
@@ -222,6 +230,7 @@ mca_btl_portals_component_init(int *num_btls,
                                bool enable_mpi_threads)
 {
     mca_btl_base_module_t ** btls = malloc(sizeof(mca_btl_base_module_t*));
+    bool accel;
     btls[0] = (mca_btl_base_module_t*) &mca_btl_portals_module;
 
     if (enable_progress_threads || enable_mpi_threads) {
@@ -232,7 +241,7 @@ mca_btl_portals_component_init(int *num_btls,
 
     /* initialize portals btl.  note that this is in the compat code because
        it's fairly non-portable between implementations */
-    if (OMPI_SUCCESS != ompi_common_portals_initialize()) {
+    if (OMPI_SUCCESS != ompi_common_portals_initialize(&mca_btl_portals_module.portals_ni_h, &accel)) {
         opal_output_verbose(20, mca_btl_portals_component.portals_output,
                             "disabled because compatibility init failed");
         return NULL;

ompi/mca/common/portals/common_portals.c

@@ -29,6 +29,9 @@
 
 #include "common_portals_crayxt3.c"
 
+#elif OMPI_PORTALS_CRAYXT3_MODEX
+#include "common_portals_cray_xt_modex.c"
+
 #else
 
 #error "Unknown Portals library configuration"
@@ -107,7 +110,7 @@ ompi_common_portals_error_ptl_to_ompi(int ptl_error)
     case PTL_SR_INDEX_INVALID:
         ret = OMPI_ERR_BAD_PARAM;
         break;
-#if !OMPI_PORTALS_CRAYXT3
+#if !(OMPI_PORTALS_CRAYXT3 || OMPI_PORTALS_CRAYXT3_MODEX)
     case PTL_UNKNOWN_ERROR:
         ret = OMPI_ERROR;
         break;

ompi/mca/common/portals/common_portals.h

@@ -34,8 +34,7 @@
 #define OMPI_MTL_PORTALS_READ_TABLE_ID 3
 #define OMPI_MTL_PORTALS_ACK_TABLE_ID  4
 
-#elif OMPI_PORTALS_CRAYXT3
+#elif (OMPI_PORTALS_CRAYXT3 || OMPI_PORTALS_CRAYXT3_MODEX)
-
 #include <portals/portals3.h>
 #define PTL_EQ_HANDLER_NONE NULL
 /* Cray's definition, differs from the spec */
@@ -91,11 +90,13 @@ int ompi_common_portals_register_mca(void);
  * may not be possible if use of the modex is required to setup the
  * network (as is the case with the utcp reference implementation).
  *
+ * @param ni_handle (OUT) network interface handle
+ * @param bool (OUT) true if using accelerated Portals, false otherwise
+ *
  * @retval OMPI_SUCCESS Portals successfully initialized
  * @retval OMPI_ERR_NOT_AVAILABLE Portals could not be initialized
  */
-int ompi_common_portals_initialize(void);
+int ompi_common_portals_initialize(ptl_handle_ni_t *ni_handle, bool *accel);
-
 
 /**
  * Initialize network interface
@@ -112,8 +113,7 @@ int ompi_common_portals_initialize(void);
  * @retval OMPI_SUCCESS Portals network interface successfully initialized
  * @retval OMPI_ERROR Something bad happened
  */
-int ompi_common_portals_ni_initialize(ptl_handle_ni_t *ni_handle,
+int ompi_common_portals_ni_initialize(ptl_handle_ni_t *ni_handle, bool *accel);
-                                      bool *accel);
 
 
 /**

ompi/mca/common/portals/common_portals_crayxt3.c

@@ -42,7 +42,7 @@ ompi_common_portals_register_mca(void)
 
 
 int
-ompi_common_portals_initialize(void)
+ompi_common_portals_initialize(ptl_handle_ni_t *ni_handle, bool *accel)
 {
     int ret, max_interfaces;
 

ompi/mca/common/portals/common_portals_utcp.c

@@ -80,7 +80,7 @@ ompi_common_portals_register_mca(void)
 
 
 int
-ompi_common_portals_initialize(void)
+ompi_common_portals_initialize(ptl_handle_ni_t *ni_handle, bool *accel)
 {
     int ret;
     ptl_process_id_t info;

ompi/mca/mtl/portals/mtl_portals.c

@@ -86,6 +86,9 @@ ompi_mtl_portals_add_procs(struct mca_mtl_base_module_t *mtl,
     ptl_process_id_t *portals_procs = NULL;
     ptl_md_t md;
     size_t i;
+    ptl_match_bits_t match_bits;
+    ptl_match_bits_t ignore_bits;
+    ptl_process_id_t anyid = { PTL_NID_ANY, PTL_PID_ANY };
     bool accel;
     
     assert(mtl == &ompi_mtl_portals.base);
@@ -95,13 +98,12 @@ ompi_mtl_portals_add_procs(struct mca_mtl_base_module_t *mtl,
        environment setup the common code does for the utcp
        implementation, we can't do it until modex information can be
        received. */
+    
     if (PTL_INVALID_HANDLE == ompi_mtl_portals.ptl_ni_h) {
-        ptl_match_bits_t match_bits;
-        ptl_match_bits_t ignore_bits;
-	ptl_process_id_t anyid = { PTL_NID_ANY, PTL_PID_ANY };
         
         ret = ompi_common_portals_ni_initialize(&(ompi_mtl_portals.ptl_ni_h), &accel);
         if (OMPI_SUCCESS != ret) goto cleanup;
+    }   
     
     /* event queue for expected events */
     ret = PtlEQAlloc(ompi_mtl_portals.ptl_ni_h,
@@ -229,7 +231,7 @@ ompi_mtl_portals_add_procs(struct mca_mtl_base_module_t *mtl,
     /* set up the short copy blocks */
     ompi_mtl_portals_short_setup();
 
-    }
+    
 
     /* get the list of ptl_process_id_t structures for the given proc
        structures.  If the Portals runtime environment supports

ompi/mca/mtl/portals/mtl_portals_component.c

@@ -166,11 +166,12 @@ static mca_mtl_base_module_t*
 ompi_mtl_portals_component_init(bool enable_progress_threads,
                                 bool enable_mpi_threads)
 {
+    bool accel;
     /* we don't run with no stinkin' threads */
     if (enable_progress_threads || enable_mpi_threads) return NULL;
 
     /* initialize our interface */
-    if (OMPI_SUCCESS != ompi_common_portals_initialize()) {
+    if (OMPI_SUCCESS != ompi_common_portals_initialize(&(ompi_mtl_portals.ptl_ni_h), &accel)) {
         return NULL;
     }