Merge pull request #7356 from hppritcha/topic/pr7201_to_v40x

Topic/pr7201 to v40x

opal/mca/hwloc/base/hwloc_base_util.c

@@ -19,7 +19,9 @@
  * Copyright (C) 2018      Mellanox Technologies, Ltd. 
  *                         All rights reserved.
  * Copyright (c) 2018      Amazon.com, Inc. or its affiliates.  All Rights reserved.
- * Copyright (c) 2019 IBM Corporation. All rights reserved.
+ * Copyright (c) 2019      IBM Corporation. All rights reserved.
+ * Copyright (c) 2019-2020 Inria.  All rights reserved.
+ * Copyright (c) 2020      Triad National Security, LLC. All rights reserved.
  * $COPYRIGHT$
  *
  * Additional copyrights may follow
@@ -58,6 +60,7 @@
 
 #include "opal/mca/hwloc/hwloc-internal.h"
 #include "opal/mca/hwloc/base/base.h"
+#include "opal/util/printf.h"
 
 static bool topo_in_shmem = false;
 
@@ -1215,16 +1218,84 @@ int opal_hwloc_base_cpu_list_parse(const char *slot_str,
     return OPAL_SUCCESS;
 }
 
+static void opal_hwloc_base_get_relative_locality_by_depth(hwloc_topology_t topo, unsigned d,
+                                                           hwloc_cpuset_t loc1, hwloc_cpuset_t loc2,
+                                                           opal_hwloc_locality_t *locality, bool *shared)
+{
+    unsigned width, w;
+    hwloc_obj_t obj;
+    int sect1, sect2;
+
+    /* get the width of the topology at this depth */
+    width = hwloc_get_nbobjs_by_depth(topo, d);
+
+    /* scan all objects at this depth to see if
+     * our locations overlap with them
+     */
+    for (w=0; w < width; w++) {
+        /* get the object at this depth/index */
+        obj = hwloc_get_obj_by_depth(topo, d, w);
+        /* see if our locations intersect with the cpuset for this obj */
+        sect1 = hwloc_bitmap_intersects(obj->cpuset, loc1);
+        sect2 = hwloc_bitmap_intersects(obj->cpuset, loc2);
+        /* if both intersect, then we share this level */
+        if (sect1 && sect2) {
+            *shared = true;
+            switch(obj->type) {
+            case HWLOC_OBJ_NODE:
+                *locality |= OPAL_PROC_ON_NUMA;
+                break;
+            case HWLOC_OBJ_SOCKET:
+                *locality |= OPAL_PROC_ON_SOCKET;
+                break;
+#if HWLOC_API_VERSION < 0x20000
+            case HWLOC_OBJ_CACHE:
+                if (3 == obj->attr->cache.depth) {
+                    *locality |= OPAL_PROC_ON_L3CACHE;
+                } else if (2 == obj->attr->cache.depth) {
+                    *locality |= OPAL_PROC_ON_L2CACHE;
+                } else {
+                    *locality |= OPAL_PROC_ON_L1CACHE;
+                }
+                break;
+#else
+            case HWLOC_OBJ_L3CACHE:
+                *locality |= OPAL_PROC_ON_L3CACHE;
+                break;
+            case HWLOC_OBJ_L2CACHE:
+                *locality |= OPAL_PROC_ON_L2CACHE;
+                break;
+            case HWLOC_OBJ_L1CACHE:
+                *locality |= OPAL_PROC_ON_L1CACHE;
+                break;
+#endif
+            case HWLOC_OBJ_CORE:
+                *locality |= OPAL_PROC_ON_CORE;
+                break;
+            case HWLOC_OBJ_PU:
+                *locality |= OPAL_PROC_ON_HWTHREAD;
+                break;
+            default:
+                /* just ignore it */
+                break;
+            }
+            break;
+        }
+        /* otherwise, we don't share this
+         * object - but we still might share another object
+         * on this level, so we have to keep searching
+         */
+    }
+}
+
 opal_hwloc_locality_t opal_hwloc_base_get_relative_locality(hwloc_topology_t topo,
                                                             char *cpuset1, char *cpuset2)
 {
     opal_hwloc_locality_t locality;
-    hwloc_obj_t obj;
+    hwloc_cpuset_t loc1, loc2;
-    unsigned depth, d, width, w;
+    unsigned depth, d;
     bool shared;
     hwloc_obj_type_t type;
-    int sect1, sect2;
-    hwloc_cpuset_t loc1, loc2;
 
     /* start with what we know - they share a node on a cluster
      * NOTE: we may alter that latter part as hwloc's ability to
@@ -1265,66 +1336,8 @@ opal_hwloc_locality_t opal_hwloc_base_get_relative_locality(hwloc_topology_t top
             HWLOC_OBJ_PU != type) {
             continue;
         }
-        /* get the width of the topology at this depth */
+        opal_hwloc_base_get_relative_locality_by_depth(topo, d, loc1, loc2, &locality, &shared);
-        width = hwloc_get_nbobjs_by_depth(topo, d);
 
-        /* scan all objects at this depth to see if
-         * our locations overlap with them
-         */
-        for (w=0; w < width; w++) {
-            /* get the object at this depth/index */
-            obj = hwloc_get_obj_by_depth(topo, d, w);
-            /* see if our locations intersect with the cpuset for this obj */
-            sect1 = hwloc_bitmap_intersects(obj->cpuset, loc1);
-            sect2 = hwloc_bitmap_intersects(obj->cpuset, loc2);
-            /* if both intersect, then we share this level */
-            if (sect1 && sect2) {
-                shared = true;
-                switch(obj->type) {
-                case HWLOC_OBJ_NODE:
-                    locality |= OPAL_PROC_ON_NUMA;
-                    break;
-                case HWLOC_OBJ_SOCKET:
-                    locality |= OPAL_PROC_ON_SOCKET;
-                    break;
-#if HWLOC_API_VERSION < 0x20000
-                case HWLOC_OBJ_CACHE:
-                    if (3 == obj->attr->cache.depth) {
-                        locality |= OPAL_PROC_ON_L3CACHE;
-                    } else if (2 == obj->attr->cache.depth) {
-                        locality |= OPAL_PROC_ON_L2CACHE;
-                    } else {
-                        locality |= OPAL_PROC_ON_L1CACHE;
-                    }
-                    break;
-#else
-                case HWLOC_OBJ_L3CACHE:
-                    locality |= OPAL_PROC_ON_L3CACHE;
-                    break;
-                case HWLOC_OBJ_L2CACHE:
-                    locality |= OPAL_PROC_ON_L2CACHE;
-                    break;
-                case HWLOC_OBJ_L1CACHE:
-                    locality |= OPAL_PROC_ON_L1CACHE;
-                    break;
-#endif
-                case HWLOC_OBJ_CORE:
-                    locality |= OPAL_PROC_ON_CORE;
-                    break;
-                case HWLOC_OBJ_PU:
-                    locality |= OPAL_PROC_ON_HWTHREAD;
-                    break;
-                default:
-                    /* just ignore it */
-                    break;
-                }
-                break;
-            }
-            /* otherwise, we don't share this
-             * object - but we still might share another object
-             * on this level, so we have to keep searching
-             */
-        }
         /* if we spanned the entire width without finding
          * a point of intersection, then no need to go
          * deeper
@@ -1333,6 +1346,9 @@ opal_hwloc_locality_t opal_hwloc_base_get_relative_locality(hwloc_topology_t top
             break;
         }
     }
+#if HWLOC_API_VERSION >= 0x20000
+    opal_hwloc_base_get_relative_locality_by_depth(topo, HWLOC_TYPE_DEPTH_NUMANODE, loc1, loc2, &locality, &shared);
+#endif
 
     opal_output_verbose(5, opal_hwloc_base_framework.framework_output,
                         "locality: %s",
@@ -1349,9 +1365,10 @@ opal_hwloc_locality_t opal_hwloc_base_get_relative_locality(hwloc_topology_t top
  */
 char* opal_hwloc_base_find_coprocessors(hwloc_topology_t topo)
 {
+#if HAVE_DECL_HWLOC_OBJ_OSDEV_COPROC
     hwloc_obj_t osdev;
-    unsigned i;
     char **cps = NULL;
+#endif
     char *cpstring = NULL;
     int depth;
 
@@ -1369,6 +1386,7 @@ char* opal_hwloc_base_find_coprocessors(hwloc_topology_t topo)
     while (NULL != osdev) {
         if (HWLOC_OBJ_OSDEV_COPROC == osdev->attr->osdev.type) {
             /* got one! find and save its serial number */
+            unsigned i;
             for (i=0; i < osdev->infos_count; i++) {
                 if (0 == strncmp(osdev->infos[i].name, "MICSerialNumber", strlen("MICSerialNumber"))) {
                     OPAL_OUTPUT_VERBOSE((5, opal_hwloc_base_framework.framework_output,
@@ -2063,12 +2081,40 @@ char* opal_hwloc_base_get_topo_signature(hwloc_topology_t topo)
     return sig;
 }
 
+static int opal_hwloc_base_get_locality_string_by_depth(hwloc_topology_t topo,
+                                                        int d,
+                                                        hwloc_cpuset_t cpuset,
+                                                        hwloc_cpuset_t result)
+{
+    hwloc_obj_t obj;
+    unsigned width, w;
+
+    /* get the width of the topology at this depth */
+    width = hwloc_get_nbobjs_by_depth(topo, d);
+    if (0 == width) {
+        return -1;
+    }
+
+    /* scan all objects at this depth to see if
+     * the location overlaps with them
+     */
+    for (w=0; w < width; w++) {
+        /* get the object at this depth/index */
+        obj = hwloc_get_obj_by_depth(topo, d, w);
+        /* see if the location intersects with it */
+        if (hwloc_bitmap_intersects(obj->cpuset, cpuset)) {
+            hwloc_bitmap_set(result, w);
+        }
+    }
+
+    return 0;
+}
+
 char* opal_hwloc_base_get_locality_string(hwloc_topology_t topo,
                                           char *bitmap)
 {
-    hwloc_obj_t obj;
     char *locality=NULL, *tmp, *t2;
-    unsigned depth, d, width, w;
+    unsigned depth, d;
     hwloc_cpuset_t cpuset, result;
     hwloc_obj_type_t type;
 
@@ -2111,28 +2157,15 @@ char* opal_hwloc_base_get_locality_string(hwloc_topology_t topo,
             continue;
         }
 
-        /* get the width of the topology at this depth */
+        if (opal_hwloc_base_get_locality_string_by_depth(topo, d, cpuset, result) < 0) {
-        width = hwloc_get_nbobjs_by_depth(topo, d);
-        if (0 == width) {
             continue;
         }
 
-        /* scan all objects at this depth to see if
-         * the location overlaps with them
-         */
-        for (w=0; w < width; w++) {
-            /* get the object at this depth/index */
-            obj = hwloc_get_obj_by_depth(topo, d, w);
-            /* see if the location intersects with it */
-            if (hwloc_bitmap_intersects(obj->cpuset, cpuset)) {
-                hwloc_bitmap_set(result, w);
-            }
-        }
         /* it should be impossible, but allow for the possibility
          * that we came up empty at this depth */
         if (!hwloc_bitmap_iszero(result)) {
             hwloc_bitmap_list_asprintf(&tmp, result);
-            switch(obj->type) {
+            switch(type) {
                 case HWLOC_OBJ_NODE:
                     asprintf(&t2, "%sNM%s:", (NULL == locality) ? "" : locality, tmp);
                     if (NULL != locality) {
@@ -2148,15 +2181,16 @@ char* opal_hwloc_base_get_locality_string(hwloc_topology_t topo,
                     locality = t2;
                     break;
 #if HWLOC_API_VERSION < 0x20000
-                case HWLOC_OBJ_CACHE:
+                case HWLOC_OBJ_CACHE: {
-                    if (3 == obj->attr->cache.depth) {
+                    unsigned cachedepth = hwloc_get_obj_by_depth(topo, d, 0)->attr->cache.depth;
+                    if (3 == cachedepth) {
                         asprintf(&t2, "%sL3%s:", (NULL == locality) ? "" : locality, tmp);
                         if (NULL != locality) {
                             free(locality);
                         }
                         locality = t2;
                         break;
-                    } else if (2 == obj->attr->cache.depth) {
+                    } else if (2 == cachedepth) {
                         asprintf(&t2, "%sL2%s:", (NULL == locality) ? "" : locality, tmp);
                         if (NULL != locality) {
                             free(locality);
@@ -2172,6 +2206,7 @@ char* opal_hwloc_base_get_locality_string(hwloc_topology_t topo,
                         break;
                     }
                     break;
+                }
 #else
                 case HWLOC_OBJ_L3CACHE:
                     asprintf(&t2, "%sL3%s:", (NULL == locality) ? "" : locality, tmp);
@@ -2217,6 +2252,24 @@ char* opal_hwloc_base_get_locality_string(hwloc_topology_t topo,
         }
         hwloc_bitmap_zero(result);
     }
+
+#if HWLOC_API_VERSION >= 0x20000
+    if (opal_hwloc_base_get_locality_string_by_depth(topo, HWLOC_TYPE_DEPTH_NUMANODE, cpuset, result) == 0) {
+        /* it should be impossible, but allow for the possibility
+         * that we came up empty at this depth */
+        if (!hwloc_bitmap_iszero(result)) {
+            hwloc_bitmap_list_asprintf(&tmp, result);
+            asprintf(&t2, "%sNM%s:", (NULL == locality) ? "" : locality, tmp);
+            if (NULL != locality) {
+                free(locality);
+            }
+            locality = t2;
+            free(tmp);
+        }
+        hwloc_bitmap_zero(result);
+    }
+#endif
+
     hwloc_bitmap_free(result);
     hwloc_bitmap_free(cpuset);