Update common_portals to allow using portals interconnect with a modex rather

than relying on cnos to get the nid/pid map. 

This commit was SVN r17588.

LICENSE

@@ -23,6 +23,8 @@ Copyright (c) 2006-2007 Sun Microsystems, Inc.  All rights reserved.
                         Use is subject to license terms.
 Copyright (c) 2006-2007 The University of Houston. All rights reserved.
 Copyright (c) 2006      Myricom, Inc.  All rights reserved.
+Copyright (c) 2007-2008 UT-Battelle, LLC. All rights reserved.
+
 $COPYRIGHT$
 
 Additional copyrights may follow

config/ompi_check_portals.m4

@@ -43,7 +43,7 @@ AC_DEFUN([OMPI_CHECK_PORTALS],[
     AC_ARG_WITH([portals-config],
             AC_HELP_STRING([--with-portals-config],
                            [configuration to use for Portals support.
-                            One of "utcp", "xt3".  (default: utcp)]))
+                            One of "utcp", "xt3", "xt3-modex".  (default: utcp)]))
     AC_ARG_WITH([portals-libs], 
         [AC_HELP_STRING([--with-portals-libs=LIBS],
                        [Libraries to link with for portals])])
@@ -53,9 +53,12 @@ AC_DEFUN([OMPI_CHECK_PORTALS],[
         with_portals_config="utcp"
     elif test "$with_portals_config" = "redstorm" ; then
         with_portals_config="xt3"
+    elif test "$with_portals_config" = "cnl_modex" ; then
+	with_portals_config="xt3-modex"
     fi
     OMPI_PORTALS_UTCP=0
     OMPI_PORTALS_CRAYXT3=0
+    OMPI_PORTALS_CRAYXT3_MODEX=0
     case "$with_portals_config" in
         "utcp")
             OMPI_PORTALS_UTCP=1
@@ -71,6 +74,13 @@ AC_DEFUN([OMPI_CHECK_PORTALS],[
             check_portals_header_prefix="portals/"
             AC_MSG_RESULT([Cray XT3])
             ;;
+        "xt3-modex")
+            OMPI_PORTALS_CRAYXT3_MODEX=1
+            OMPI_PORTALS_HAVE_EVENT_UNLINK=0
+            check_portals_LIBS=
+            check_portals_header_prefix="portals/"
+            AC_MSG_RESULT([Cray XT3 - Modex])
+            ;;
         *)
             # ok to call ERROR here - the user specified something invalid.
             # that should be brought to his attention
@@ -84,8 +94,12 @@ AC_DEFUN([OMPI_CHECK_PORTALS],[
 
     AC_DEFINE_UNQUOTED([OMPI_PORTALS_UTCP], [$OMPI_PORTALS_UTCP],
                        [Use the UTCP reference implementation of Portals])
+
     AC_DEFINE_UNQUOTED([OMPI_PORTALS_CRAYXT3], [$OMPI_PORTALS_CRAYXT3],
                        [Use the Cray XT-3 implementation of Portals])
+    
+    AC_DEFINE_UNQUOTED([OMPI_PORTALS_CRAYXT3_MODEX], [$OMPI_PORTALS_CRAYXT3_MODEX],
+                       [Use the Cray XT-3 implementation of Portals using Modex])
 
     # Add correct -I and -L flags
     AS_IF([test -n "$with_portals"],

contrib/platform/cray_xt_cnl_romio

@@ -4,7 +4,7 @@ enable_shared=no
 with_threads=no
 enable_pretty_print_stacktrace=no
 enable_dlopen=no
-with_portals_config=redstorm
+with_portals_config=cnl_modex
 with_memory_manager=none
 enable_mca_no_build=maffinity-first_use,maffinity-libnuma,paffinity-linux,pls-rsh,pml-dr,filem-rsh,grpcomm-cnos,pls-cnos,rmgr-cnos,rml-cnos,routed-cnos,sds-portals_utcp,sds-cnos
 enable_heterogeneous=no

ompi/mca/btl/portals/btl_portals.c

@@ -526,14 +526,14 @@ mca_btl_portals_finalize(struct mca_btl_base_module_t *btl_base)
         for (i = 0 ; i < OMPI_BTL_PORTALS_EQ_SIZE ; ++i) {
             int ptl_ret = PtlEQFree(mca_btl_portals_module.portals_eq_handles[i]);
             if (PTL_OK != ptl_ret) {
-#if OMPI_PORTALS_CRAYXT3
+#if (OMPI_PORTALS_CRAYXT3 || OMPI_PORTALS_CRAYXT3_MODEX)
                 if (i != OMPI_BTL_PORTALS_EQ_SEND && PTL_EQ_IN_USE != ptl_ret) {
                     /* The PML isn't great about cleaning up after itself.
                        Ignore related errors. */
 #endif
                     opal_output(mca_btl_portals_component.portals_output,
                             "Error freeing EQ %d: %d", i, ptl_ret);
-#if OMPI_PORTALS_CRAYXT3
+#if (OMPI_PORTALS_CRAYXT3 || OMPI_PORTALS_CRAYXT3_MODEX)
                 }
 #endif
             }

ompi/mca/btl/portals/btl_portals.h

@@ -61,6 +61,9 @@ struct mca_btl_portals_component_t {
 
     /* number of eager fragments */
     int portals_free_list_eager_max_num;
+    
+    /* shall I use portals to send to thyself? */
+    int portals_support_self;
 };
 typedef struct mca_btl_portals_component_t mca_btl_portals_component_t;
 

ompi/mca/btl/portals/btl_portals_component.c

@@ -125,6 +125,14 @@ mca_btl_portals_component_open(void)
                            32,
                            &(mca_btl_portals_component.portals_free_list_eager_max_num));
 
+    mca_base_param_reg_int(&mca_btl_portals_component.super.btl_version,
+                           "support_self",
+                           "Use portals for send to self",
+                           false,
+                           false,
+                           1, /* default to true.. */ 
+                           &(mca_btl_portals_component.portals_support_self));
+
     /* 
      * fill default module state 
      */
@@ -222,6 +230,7 @@ mca_btl_portals_component_init(int *num_btls,
                                bool enable_mpi_threads)
 {
     mca_btl_base_module_t ** btls = malloc(sizeof(mca_btl_base_module_t*));
+    bool accel;
     btls[0] = (mca_btl_base_module_t*) &mca_btl_portals_module;
 
     if (enable_progress_threads || enable_mpi_threads) {
@@ -232,7 +241,7 @@ mca_btl_portals_component_init(int *num_btls,
 
     /* initialize portals btl.  note that this is in the compat code because
        it's fairly non-portable between implementations */
-    if (OMPI_SUCCESS != ompi_common_portals_initialize()) {
+    if (OMPI_SUCCESS != ompi_common_portals_initialize(&mca_btl_portals_module.portals_ni_h, &accel)) {
         opal_output_verbose(20, mca_btl_portals_component.portals_output,
                             "disabled because compatibility init failed");
         return NULL;

ompi/mca/common/portals/common_portals.c

@@ -29,6 +29,9 @@
 
 #include "common_portals_crayxt3.c"
 
+#elif OMPI_PORTALS_CRAYXT3_MODEX
+#include "common_portals_cray_xt_modex.c"
+
 #else
 
 #error "Unknown Portals library configuration"
@@ -107,7 +110,7 @@ ompi_common_portals_error_ptl_to_ompi(int ptl_error)
     case PTL_SR_INDEX_INVALID:
         ret = OMPI_ERR_BAD_PARAM;
         break;
-#if !OMPI_PORTALS_CRAYXT3
+#if !(OMPI_PORTALS_CRAYXT3 || OMPI_PORTALS_CRAYXT3_MODEX)
     case PTL_UNKNOWN_ERROR:
         ret = OMPI_ERROR;
         break;

ompi/mca/common/portals/common_portals.h

@@ -34,8 +34,7 @@
 #define OMPI_MTL_PORTALS_READ_TABLE_ID 3
 #define OMPI_MTL_PORTALS_ACK_TABLE_ID  4
 
-#elif OMPI_PORTALS_CRAYXT3
+#elif (OMPI_PORTALS_CRAYXT3 || OMPI_PORTALS_CRAYXT3_MODEX)
-
 #include <portals/portals3.h>
 #define PTL_EQ_HANDLER_NONE NULL
 /* Cray's definition, differs from the spec */
@@ -91,11 +90,13 @@ int ompi_common_portals_register_mca(void);
  * may not be possible if use of the modex is required to setup the
  * network (as is the case with the utcp reference implementation).
  *
+ * @param ni_handle (OUT) network interface handle
+ * @param bool (OUT) true if using accelerated Portals, false otherwise
+ *
  * @retval OMPI_SUCCESS Portals successfully initialized
  * @retval OMPI_ERR_NOT_AVAILABLE Portals could not be initialized
  */
-int ompi_common_portals_initialize(void);
+int ompi_common_portals_initialize(ptl_handle_ni_t *ni_handle, bool *accel);
-
 
 /**
  * Initialize network interface
@@ -112,8 +113,7 @@ int ompi_common_portals_initialize(void);
  * @retval OMPI_SUCCESS Portals network interface successfully initialized
  * @retval OMPI_ERROR Something bad happened
  */
-int ompi_common_portals_ni_initialize(ptl_handle_ni_t *ni_handle,
+int ompi_common_portals_ni_initialize(ptl_handle_ni_t *ni_handle, bool *accel);
-                                      bool *accel);
 
 
 /**

ompi/mca/common/portals/common_portals_crayxt3.c

@@ -42,7 +42,7 @@ ompi_common_portals_register_mca(void)
 
 
 int
-ompi_common_portals_initialize(void)
+ompi_common_portals_initialize(ptl_handle_ni_t *ni_handle, bool *accel)
 {
     int ret, max_interfaces;
 

ompi/mca/common/portals/common_portals_utcp.c

@@ -80,7 +80,7 @@ ompi_common_portals_register_mca(void)
 
 
 int
-ompi_common_portals_initialize(void)
+ompi_common_portals_initialize(ptl_handle_ni_t *ni_handle, bool *accel)
 {
     int ret;
     ptl_process_id_t info;

ompi/mca/mtl/portals/mtl_portals.c

@@ -86,8 +86,11 @@ ompi_mtl_portals_add_procs(struct mca_mtl_base_module_t *mtl,
     ptl_process_id_t *portals_procs = NULL;
     ptl_md_t md;
     size_t i;
+    ptl_match_bits_t match_bits;
+    ptl_match_bits_t ignore_bits;
+    ptl_process_id_t anyid = { PTL_NID_ANY, PTL_PID_ANY };
     bool accel;
-
+    
     assert(mtl == &ompi_mtl_portals.base);
 
     /* if we havne't already initialized the network, do so now.  We
@@ -95,141 +98,140 @@ ompi_mtl_portals_add_procs(struct mca_mtl_base_module_t *mtl,
        environment setup the common code does for the utcp
        implementation, we can't do it until modex information can be
        received. */
+    
     if (PTL_INVALID_HANDLE == ompi_mtl_portals.ptl_ni_h) {
-        ptl_match_bits_t match_bits;
+        
-        ptl_match_bits_t ignore_bits;
-	ptl_process_id_t anyid = { PTL_NID_ANY, PTL_PID_ANY };
-
         ret = ompi_common_portals_ni_initialize(&(ompi_mtl_portals.ptl_ni_h), &accel);
         if (OMPI_SUCCESS != ret) goto cleanup;
+    }   
+    
+    /* event queue for expected events */
+    ret = PtlEQAlloc(ompi_mtl_portals.ptl_ni_h,
+                     ompi_mtl_portals.ptl_expected_queue_size,
+                     PTL_EQ_HANDLER_NONE,
+                     &(ompi_mtl_portals.ptl_eq_h));
+    assert(ret == PTL_OK);
+    
+    /* event queue for unexpected receives */
+    ret = PtlEQAlloc(ompi_mtl_portals.ptl_ni_h,
+                     ompi_mtl_portals.ptl_unexpected_queue_size,
+                     PTL_EQ_HANDLER_NONE,
+                     &(ompi_mtl_portals.ptl_unex_eq_h));
+    assert(ret == PTL_OK);
 
-        /* event queue for expected events */
+    /* empty event queue for PtlMEMDPost() */
-        ret = PtlEQAlloc(ompi_mtl_portals.ptl_ni_h,
+    ret = PtlEQAlloc(ompi_mtl_portals.ptl_ni_h,
-                         ompi_mtl_portals.ptl_expected_queue_size,
+                     1,
-                         PTL_EQ_HANDLER_NONE,
+                     PTL_EQ_HANDLER_NONE,
-                         &(ompi_mtl_portals.ptl_eq_h));
+                     &(ompi_mtl_portals.ptl_empty_eq_h));
-        assert(ret == PTL_OK);
+    assert(ret == PTL_OK);
 
-	/* event queue for unexpected receives */
+    /* attach the long unex msg buffer */
-        ret = PtlEQAlloc(ompi_mtl_portals.ptl_ni_h,
+    match_bits  = PTL_LONG_MSG;
-                         ompi_mtl_portals.ptl_unexpected_queue_size,
+    ignore_bits = ~(PTL_LONG_MSG);
-                         PTL_EQ_HANDLER_NONE,
-                         &(ompi_mtl_portals.ptl_unex_eq_h));
-        assert(ret == PTL_OK);
 
-        /* empty event queue for PtlMEMDPost() */
+    ret = PtlMEAttach(ompi_mtl_portals.ptl_ni_h,
-        ret = PtlEQAlloc(ompi_mtl_portals.ptl_ni_h,
+                      OMPI_MTL_PORTALS_SEND_TABLE_ID,
-                         1,
+                      anyid,
-                         PTL_EQ_HANDLER_NONE,
+                      match_bits,
-                         &(ompi_mtl_portals.ptl_empty_eq_h));
+                      ignore_bits,
-        assert(ret == PTL_OK);
+                      PTL_RETAIN,
+                      PTL_INS_AFTER,
+                      &(ompi_mtl_portals.ptl_unex_long_me_h));
+    assert(ret == PTL_OK);
 
-	/* attach the long unex msg buffer */
+    md.start     = NULL;
-	match_bits  = PTL_LONG_MSG;
+    md.length    = 0;
-	ignore_bits = ~(PTL_LONG_MSG);
+    md.threshold = PTL_MD_THRESH_INF;
+    md.max_size  = 0;
+    md.options   = PTL_MD_OP_PUT | PTL_MD_TRUNCATE | PTL_MD_ACK_DISABLE;
+    md.eq_handle = ompi_mtl_portals.ptl_unex_eq_h;
+    md.user_ptr  = NULL;
 
-        ret = PtlMEAttach(ompi_mtl_portals.ptl_ni_h,
+    ret = PtlMDAttach(ompi_mtl_portals.ptl_unex_long_me_h,
-                          OMPI_MTL_PORTALS_SEND_TABLE_ID,
+                      md,
-                          anyid,
+                      PTL_RETAIN,
-                          match_bits,
+                      &unex_long_md_h);
-                          ignore_bits,
+    assert(ret == PTL_OK);
-                          PTL_RETAIN,
-                          PTL_INS_AFTER,
-                          &(ompi_mtl_portals.ptl_unex_long_me_h));
-        assert(ret == PTL_OK);
 
-        md.start     = NULL;
+    /* attach catchalls to the send, ack, and read portals */
-        md.length    = 0;
+    catchall_request.event_callback = ompi_mtl_portals_catchall_callback;
-        md.threshold = PTL_MD_THRESH_INF;
+    md.eq_handle = ompi_mtl_portals.ptl_eq_h;
-        md.max_size  = 0;
+    md.user_ptr = &catchall_request;
-        md.options   = PTL_MD_OP_PUT | PTL_MD_TRUNCATE | PTL_MD_ACK_DISABLE;
-        md.eq_handle = ompi_mtl_portals.ptl_unex_eq_h;
-        md.user_ptr  = NULL;
 
-        ret = PtlMDAttach(ompi_mtl_portals.ptl_unex_long_me_h,
+    /* catchall for the send portal */
-                          md,
+    ret = PtlMEMDPost(ompi_mtl_portals.ptl_ni_h,
-                          PTL_RETAIN,
+                      ompi_mtl_portals.ptl_unex_long_me_h,
-                          &unex_long_md_h);
+                      anyid,
-        assert(ret == PTL_OK);
+                      0,
+                      ~0,
+                      PTL_RETAIN,
+                      PTL_INS_AFTER,
+                      md,
+                      PTL_UNLINK,
+                      &(ompi_mtl_portals.ptl_send_catchall_me_h),
+                      &send_catchall_md_h,
+                      ompi_mtl_portals.ptl_empty_eq_h);
+    assert(ret == PTL_OK);
 
-	/* attach catchalls to the send, ack, and read portals */
+    /* catchall for ack portal */
-        catchall_request.event_callback = ompi_mtl_portals_catchall_callback;
+    ret = PtlMEAttach(ompi_mtl_portals.ptl_ni_h,
-        md.eq_handle = ompi_mtl_portals.ptl_eq_h;
+                      OMPI_MTL_PORTALS_ACK_TABLE_ID,
-        md.user_ptr = &catchall_request;
+                      anyid,
+                      0,
+                      ~0,
+                      PTL_RETAIN,
+                      PTL_INS_AFTER,
+                      &(ompi_mtl_portals.ptl_ack_catchall_me_h));
+    assert(ret == PTL_OK);
 
-	/* catchall for the send portal */
+    ret = PtlMDAttach(ompi_mtl_portals.ptl_ack_catchall_me_h,
-        ret = PtlMEMDPost(ompi_mtl_portals.ptl_ni_h,
+                      md,
-                          ompi_mtl_portals.ptl_unex_long_me_h,
+                      PTL_UNLINK,
-                          anyid,
+                      &ack_catchall_md_h);
-                          0,
+    assert(ret == PTL_OK);
-                          ~0,
-                          PTL_RETAIN,
-                          PTL_INS_AFTER,
-                          md,
-                          PTL_UNLINK,
-                          &(ompi_mtl_portals.ptl_send_catchall_me_h),
-                          &send_catchall_md_h,
-                          ompi_mtl_portals.ptl_empty_eq_h);
-        assert(ret == PTL_OK);
 
-        /* catchall for ack portal */
+    /* catchall for read portal */
-        ret = PtlMEAttach(ompi_mtl_portals.ptl_ni_h,
+    ret = PtlMEAttach(ompi_mtl_portals.ptl_ni_h,
-                          OMPI_MTL_PORTALS_ACK_TABLE_ID,
+                      OMPI_MTL_PORTALS_READ_TABLE_ID,
-                          anyid,
+                      anyid,
-                          0,
+                      0,
-                          ~0,
+                      ~0,
-                          PTL_RETAIN,
+                      PTL_RETAIN,
-                          PTL_INS_AFTER,
+                      PTL_INS_AFTER,
-                          &(ompi_mtl_portals.ptl_ack_catchall_me_h));
+                      &(ompi_mtl_portals.ptl_read_catchall_me_h));
-        assert(ret == PTL_OK);
+    assert(ret == PTL_OK);
 
-        ret = PtlMDAttach(ompi_mtl_portals.ptl_ack_catchall_me_h,
+    ret = PtlMDAttach(ompi_mtl_portals.ptl_read_catchall_me_h,
-                          md,
+                      md,
-                          PTL_UNLINK,
+                      PTL_RETAIN,
-                          &ack_catchall_md_h);
+                      &read_catchall_md_h);
-        assert(ret == PTL_OK);
+    assert(ret == PTL_OK);
 
-        /* catchall for read portal */
+    /* attach short unex recv blocks */
-        ret = PtlMEAttach(ompi_mtl_portals.ptl_ni_h,
+    ret = ompi_mtl_portals_recv_short_enable((mca_mtl_portals_module_t*) mtl);
-                          OMPI_MTL_PORTALS_READ_TABLE_ID,
-                          anyid,
-                          0,
-                          ~0,
-                          PTL_RETAIN,
-                          PTL_INS_AFTER,
-                          &(ompi_mtl_portals.ptl_read_catchall_me_h));
-        assert(ret == PTL_OK);
 
-        ret = PtlMDAttach(ompi_mtl_portals.ptl_read_catchall_me_h,
+    opal_progress_register(ompi_mtl_portals_progress);
-                          md,
-                          PTL_RETAIN,
-                          &read_catchall_md_h);
-        assert(ret == PTL_OK);
 
-	/* attach short unex recv blocks */
+    /* bind zero-length md for sending zero-length msgs and acks */
-        ret = ompi_mtl_portals_recv_short_enable((mca_mtl_portals_module_t*) mtl);
+    md.start     = NULL;
+    md.length    = 0;
+    md.threshold = PTL_MD_THRESH_INF;
+    md.max_size  = 0;
+    md.options   = PTL_MD_EVENT_START_DISABLE | PTL_MD_EVENT_END_DISABLE;
+    md.user_ptr  = NULL;
+    md.eq_handle = PTL_EQ_NONE;
 
-        opal_progress_register(ompi_mtl_portals_progress);
+    ret = PtlMDBind(ompi_mtl_portals.ptl_ni_h,
+                    md,
+                    PTL_RETAIN,
+                    &ompi_mtl_portals.ptl_zero_md_h ); 
+    assert(ret == PTL_OK);
 
-        /* bind zero-length md for sending zero-length msgs and acks */
+    /* set up the short copy blocks */
-        md.start     = NULL;
+    ompi_mtl_portals_short_setup();
-        md.length    = 0;
-        md.threshold = PTL_MD_THRESH_INF;
-        md.max_size  = 0;
-        md.options   = PTL_MD_EVENT_START_DISABLE | PTL_MD_EVENT_END_DISABLE;
-        md.user_ptr  = NULL;
-        md.eq_handle = PTL_EQ_NONE;
 
-        ret = PtlMDBind(ompi_mtl_portals.ptl_ni_h,
+    
-                        md,
-                        PTL_RETAIN,
-                        &ompi_mtl_portals.ptl_zero_md_h ); 
-        assert(ret == PTL_OK);
-
-	/* set up the short copy blocks */
-        ompi_mtl_portals_short_setup();
-
-    }
 
     /* get the list of ptl_process_id_t structures for the given proc
        structures.  If the Portals runtime environment supports

ompi/mca/mtl/portals/mtl_portals_component.c

@@ -166,11 +166,12 @@ static mca_mtl_base_module_t*
 ompi_mtl_portals_component_init(bool enable_progress_threads,
                                 bool enable_mpi_threads)
 {
+    bool accel;
     /* we don't run with no stinkin' threads */
     if (enable_progress_threads || enable_mpi_threads) return NULL;
 
     /* initialize our interface */
-    if (OMPI_SUCCESS != ompi_common_portals_initialize()) {
+    if (OMPI_SUCCESS != ompi_common_portals_initialize(&(ompi_mtl_portals.ptl_ni_h), &accel)) {
         return NULL;
     }