Added support for Accelerated Portals for the btl.

This commit was SVN r16771.

ompi/mca/btl/portals/btl_portals.c

@@ -86,6 +86,7 @@ mca_btl_portals_add_procs(struct mca_btl_base_module_t* btl_base,
     size_t i;
     unsigned long distance;
     bool need_activate = false;
+    bool accel;
 
     assert(&mca_btl_portals_module == (mca_btl_portals_module_t*) btl_base);
     opal_output_verbose(50, mca_btl_portals_component.portals_output,
@@ -94,7 +95,7 @@ mca_btl_portals_add_procs(struct mca_btl_base_module_t* btl_base,
 
     /* if we havne't already, get our network handle */
     if (mca_btl_portals_module.portals_ni_h == PTL_INVALID_HANDLE) {
-        ret = ompi_common_portals_ni_initialize(&mca_btl_portals_module.portals_ni_h);
+        ret = ompi_common_portals_ni_initialize(&mca_btl_portals_module.portals_ni_h, &accel);
         if (OMPI_SUCCESS != ret) return ret;
     }
 
@@ -119,15 +120,18 @@ mca_btl_portals_add_procs(struct mca_btl_base_module_t* btl_base,
         if (NULL == peers[i]) return OMPI_ERROR;
         *((mca_btl_base_endpoint_t*) peers[i]) = portals_procs[i];
 
-        /* make sure we can reach the process - this is supposed to be
-           a cheap-ish operation */
-        ret = PtlNIDist(mca_btl_portals_module.portals_ni_h,
-                        portals_procs[i],
-                        &distance);
-        if (ret != PTL_OK) {
-            opal_output_verbose(10, mca_btl_portals_component.portals_output,
-                                "Could not find distance to process %d", i);
-            continue;
+        /* accelerated doesn't support PtlNIDist() */
+        if (accel == false) {
+            /* make sure we can reach the process - this is supposed to be
+               a cheap-ish operation */
+            ret = PtlNIDist(mca_btl_portals_module.portals_ni_h,
+                            portals_procs[i],
+                            &distance);
+            if (ret != PTL_OK) {
+                opal_output_verbose(10, mca_btl_portals_component.portals_output,
+                                    "Could not find distance to process %d", i);
+                continue;
+            }
         }
 
         OPAL_THREAD_ADD32(&mca_btl_portals_module.portals_num_procs, 1);
@@ -516,7 +520,7 @@ mca_btl_portals_finalize(struct mca_btl_base_module_t *btl_base)
 
     /* finalize all communication */
     while (mca_btl_portals_module.portals_outstanding_ops > 0) {
-      mca_btl_portals_component_progress();
+        mca_btl_portals_component_progress();
     }
 
     if (mca_btl_portals_module.portals_num_procs != 0) {

ompi/mca/common/portals/common_portals.h

@@ -107,11 +107,13 @@ int ompi_common_portals_initialize(void);
  * called after the modex data is available.
  *
  * @param ni_handle (OUT) network interface handle
+ * @param bool (OUT) true if using accelerated Portals, false otherwise
  *
  * @retval OMPI_SUCCESS Portals network interface successfully initialized
  * @retval OMPI_ERROR Something bad happened
  */
-int ompi_common_portals_ni_initialize(ptl_handle_ni_t *ni_handle);
+int ompi_common_portals_ni_initialize(ptl_handle_ni_t *ni_handle,
+                                      bool *accel);
 
 
 /**

ompi/mca/common/portals/common_portals_crayxt3.c

@@ -22,6 +22,7 @@
 #include "ompi/constants.h"
 #include "ompi/proc/proc.h"
 
+static bool use_accelerated;
 
 char *
 ompi_common_portals_nodeid(void)
@@ -59,12 +60,25 @@ ompi_common_portals_initialize(void)
 
 
 int
-ompi_common_portals_ni_initialize(ptl_handle_ni_t *ni_handle)
+ompi_common_portals_ni_initialize(ptl_handle_ni_t *ni_handle, bool *accel)
 {
     ptl_interface_t ni_iface = PTL_IFACE_DEFAULT;
     int max_interfaces;
     int launcher;
     int ret;
+    int tmp = 0;
+
+#if defined(CRAY_ACCEL)
+    mca_base_param_reg_int_name("mca",
+                                "use_accelerated_portals",
+                                "Use Accelerated Portals",
+                                false,
+                                false,
+                                0,
+                                &tmp);
+#endif
+
+    *accel = use_accelerated = (tmp == 0) ? false : true;
 
     launcher = cnos_launcher();
 
@@ -75,6 +89,11 @@ ompi_common_portals_ni_initialize(ptl_handle_ni_t *ni_handle)
     if( launcher != CNOS_LAUNCHER_YOD ) {
         ni_iface = IFACE_FROM_BRIDGE_AND_NALID(PTL_BRIDGE_UK,PTL_IFACE_SS);
     }
+#if defined(CRAY_ACCEL)
+    else if (use_accelerated == true) {
+        ni_iface = CRAY_ACCEL;
+    }
+#endif
 
     /*
      * Initialize Portals interface
@@ -112,6 +131,7 @@ ompi_common_portals_get_procs(size_t nprocs,
 {
     int nptl_procs = 0;
     cnos_nidpid_map_t *map;
+    int pid_space_offset = 0;
     int i;
 
     /*
@@ -126,12 +146,16 @@ ompi_common_portals_get_procs(size_t nprocs,
         return OMPI_ERR_FATAL;
     }
 
+#if defined(CRAY_ACCEL)
+    pid_space_offset = (use_accelerated == true) ? ACCEL_PTLS_PID_SPACE_OFFSET : 0;
+#endif
+
     for (i = 0 ; i < nprocs ; ++i) {
         size_t idx = (size_t) procs[i]->proc_name.vpid;
         if (idx >= nptl_procs) return OMPI_ERR_NOT_FOUND;
 
 	portals_procs[i].nid = map[idx].nid;
-	portals_procs[i].pid = map[idx].pid;
+	portals_procs[i].pid = map[idx].pid + pid_space_offset;
     }
 
     return OMPI_SUCCESS;

ompi/mca/common/portals/common_portals_utcp.c

@@ -164,10 +164,12 @@ ompi_common_portals_initialize(void)
 
 
 int
-ompi_common_portals_ni_initialize(ptl_handle_ni_t *ni_handle)
+ompi_common_portals_ni_initialize(ptl_handle_ni_t *ni_handle, bool *accel)
 {
     int ret;
 
+    *accel = false;
+
     OPAL_THREAD_ADD32(&ni_usage_count, 1);
     if (PTL_INVALID_HANDLE != active_ni_h) {
         *ni_handle = active_ni_h;