Fix bug in distance computation code when deciding which devices to use on a NUMA node.

Also add a verbose flag so one can see what devices are selected as well as another flag to override
locality information and use all devices on the node.  

This commit was SVN r30287.

ompi/mca/btl/openib/btl_openib.h

@@ -16,7 +16,7 @@
  *                         reserved.
  * Copyright (c) 2006-2007 Voltaire All rights reserved.
  * Copyright (c) 2009-2010 Oracle and/or its affiliates.  All rights reserved.
- * Copyright (c) 2013      NVIDIA Corporation.  All rights reserved.
+ * Copyright (c) 2013-2014 NVIDIA Corporation.  All rights reserved.
  * $COPYRIGHT$
  *
  * Additional copyrights may follow
@@ -304,6 +304,8 @@ struct mca_btl_openib_component_t {
     bool enable_srq_resize;
     int memory_registration_verbose_level;
     int memory_registration_verbose;
+    int device_selection_verbose;
+    int ignore_locality;
 #if BTL_OPENIB_FAILOVER_ENABLED
     int verbose_failover;
 #endif

ompi/mca/btl/openib/btl_openib_component.c

@@ -16,7 +16,7 @@
  *                         reserved.
  * Copyright (c) 2006-2007 Voltaire All rights reserved.
  * Copyright (c) 2009-2012 Oracle and/or its affiliates.  All rights reserved.
- * Copyright (c) 2011-2013 NVIDIA Corporation.  All rights reserved.
+ * Copyright (c) 2011-2014 NVIDIA Corporation.  All rights reserved.
  * Copyright (c) 2012      Oak Ridge National Laboratory.  All rights reserved
  * Copyright (c) 2013      Intel, Inc. All rights reserved
  * $COPYRIGHT$
@@ -2116,6 +2116,11 @@ static int get_ib_dev_distance(struct ibv_device *dev)
        because we have no way of measuring. */
     int distance = 0;
 
+    /* Override any distance logic so all devices are used */
+    if (0 != mca_btl_openib_component.ignore_locality) {
+        return distance;
+    }
+
 #if OPAL_HAVE_HWLOC
     int a, b, i;
     hwloc_cpuset_t my_cpuset = NULL, ibv_cpuset = NULL;
@@ -2202,10 +2207,10 @@ static int get_ib_dev_distance(struct ibv_device *dev)
         if (NULL != my_obj) {
             /* Distance may be asymetrical, so calculate both of them
                and take the max */
-            a = hwloc_distances->latency[my_obj->logical_index *
+            a = hwloc_distances->latency[my_obj->logical_index +
                                          (ibv_obj->logical_index * 
                                           hwloc_distances->nbobjs)];
-            b = hwloc_distances->latency[ibv_obj->logical_index *
+            b = hwloc_distances->latency[ibv_obj->logical_index +
                                          (my_obj->logical_index * 
                                           hwloc_distances->nbobjs)];
             distance = (a > b) ? a : b;
@@ -2224,10 +2229,10 @@ static int get_ib_dev_distance(struct ibv_device *dev)
                                                             ibv_obj->cpuset, 
                                                             HWLOC_OBJ_NODE, ++i)) {
 
-            a = hwloc_distances->latency[node_obj->logical_index *
+            a = hwloc_distances->latency[node_obj->logical_index +
                                          (ibv_obj->logical_index * 
                                           hwloc_distances->nbobjs)];
-            b = hwloc_distances->latency[ibv_obj->logical_index *
+            b = hwloc_distances->latency[ibv_obj->logical_index +
                                          (node_obj->logical_index * 
                                           hwloc_distances->nbobjs)];
             a = (a > b) ? a : b;
@@ -2552,8 +2557,11 @@ btl_openib_component_init(int *num_btl_modules,
                 mca_btl_openib_component.ib_max_btls); i++) {
         if (0 != mca_btl_openib_component.ib_num_btls &&
             distance != dev_sorted[i].distance) {
-            BTL_VERBOSE(("openib: skipping device %s; it's too far away", 
-                         ibv_get_device_name(dev_sorted[i].ib_dev)));
+            if (mca_btl_openib_component.device_selection_verbose) {
+                opal_output(0, "[rank=%d] openib: skipping device %s; it is too far away", 
+                            ORTE_PROC_MY_NAME->vpid,
+                            ibv_get_device_name(dev_sorted[i].ib_dev));
+            }
             break;
         }
 
@@ -2592,6 +2600,12 @@ btl_openib_component_init(int *num_btl_modules,
         if (OMPI_SUCCESS != ret && OMPI_ERR_NOT_SUPPORTED != ret) {
             free(dev_sorted);
             goto no_btls;
+        } else {
+            if (mca_btl_openib_component.device_selection_verbose) {
+                opal_output(0, "[rank=%d] openib: using device %s", 
+                            ORTE_PROC_MY_NAME->vpid,
+                            ibv_get_device_name(dev_sorted[i].ib_dev));
+            }
         }
     }
     free(dev_sorted);

ompi/mca/btl/openib/btl_openib_mca.c

@@ -15,7 +15,7 @@
  *                         reserved.
  * Copyright (c) 2006-2007 Voltaire All rights reserved.
  * Copyright (c) 2009-2010 Oracle and/or its affiliates.  All rights reserved.
- * Copyright (c) 2013      NVIDIA Corporation.  All rights reserved.
+ * Copyright (c) 2013-2014 NVIDIA Corporation.  All rights reserved.
  * $COPYRIGHT$
  *
  * Additional copyrights may follow
@@ -537,6 +537,17 @@ int btl_openib_register_mca_params(void)
                   "(0 = no output, nonzero = output)", 0,
 		  &mca_btl_openib_component.memory_registration_verbose_level, 0));
 
+    /* Help see which devices are being used */
+    CHECK(reg_int("device_selection_verbose", NULL,
+                  "Output some verbose device selection information "
+                  "(0 = no output, nonzero = output)", 0,
+                  &mca_btl_openib_component.device_selection_verbose, REGINT_GE_ZERO));
+
+    CHECK(reg_int("ignore_locality", NULL,
+                  "Ignore any locality information and use all devices "
+                  "(0 = use locality informaiton and use only close devices, nonzero = ignore locality information)", 0,
+                  &mca_btl_openib_component.ignore_locality, REGINT_GE_ZERO));
+
     /* Info only */
     tmp = mca_base_component_var_register(&mca_btl_openib_component.super.btl_version,
                                           "have_fork_support",