diff --git a/opal/mca/hwloc/base/hwloc_base_util.c b/opal/mca/hwloc/base/hwloc_base_util.c index ba26ba0ac6..80d6b95ab0 100644 --- a/opal/mca/hwloc/base/hwloc_base_util.c +++ b/opal/mca/hwloc/base/hwloc_base_util.c @@ -19,7 +19,9 @@ * Copyright (C) 2018 Mellanox Technologies, Ltd. * All rights reserved. * Copyright (c) 2018 Amazon.com, Inc. or its affiliates. All Rights reserved. - * Copyright (c) 2019 IBM Corporation. All rights reserved. + * Copyright (c) 2019 IBM Corporation. All rights reserved. + * Copyright (c) 2019-2020 Inria. All rights reserved. + * Copyright (c) 2020 Triad National Security, LLC. All rights reserved. * $COPYRIGHT$ * * Additional copyrights may follow @@ -58,6 +60,7 @@ #include "opal/mca/hwloc/hwloc-internal.h" #include "opal/mca/hwloc/base/base.h" +#include "opal/util/printf.h" static bool topo_in_shmem = false; @@ -1215,16 +1218,84 @@ int opal_hwloc_base_cpu_list_parse(const char *slot_str, return OPAL_SUCCESS; } +static void opal_hwloc_base_get_relative_locality_by_depth(hwloc_topology_t topo, unsigned d, + hwloc_cpuset_t loc1, hwloc_cpuset_t loc2, + opal_hwloc_locality_t *locality, bool *shared) +{ + unsigned width, w; + hwloc_obj_t obj; + int sect1, sect2; + + /* get the width of the topology at this depth */ + width = hwloc_get_nbobjs_by_depth(topo, d); + + /* scan all objects at this depth to see if + * our locations overlap with them + */ + for (w=0; w < width; w++) { + /* get the object at this depth/index */ + obj = hwloc_get_obj_by_depth(topo, d, w); + /* see if our locations intersect with the cpuset for this obj */ + sect1 = hwloc_bitmap_intersects(obj->cpuset, loc1); + sect2 = hwloc_bitmap_intersects(obj->cpuset, loc2); + /* if both intersect, then we share this level */ + if (sect1 && sect2) { + *shared = true; + switch(obj->type) { + case HWLOC_OBJ_NODE: + *locality |= OPAL_PROC_ON_NUMA; + break; + case HWLOC_OBJ_SOCKET: + *locality |= OPAL_PROC_ON_SOCKET; + break; +#if HWLOC_API_VERSION < 0x20000 + case HWLOC_OBJ_CACHE: + if (3 == obj->attr->cache.depth) { + *locality |= OPAL_PROC_ON_L3CACHE; + } else if (2 == obj->attr->cache.depth) { + *locality |= OPAL_PROC_ON_L2CACHE; + } else { + *locality |= OPAL_PROC_ON_L1CACHE; + } + break; +#else + case HWLOC_OBJ_L3CACHE: + *locality |= OPAL_PROC_ON_L3CACHE; + break; + case HWLOC_OBJ_L2CACHE: + *locality |= OPAL_PROC_ON_L2CACHE; + break; + case HWLOC_OBJ_L1CACHE: + *locality |= OPAL_PROC_ON_L1CACHE; + break; +#endif + case HWLOC_OBJ_CORE: + *locality |= OPAL_PROC_ON_CORE; + break; + case HWLOC_OBJ_PU: + *locality |= OPAL_PROC_ON_HWTHREAD; + break; + default: + /* just ignore it */ + break; + } + break; + } + /* otherwise, we don't share this + * object - but we still might share another object + * on this level, so we have to keep searching + */ + } +} + opal_hwloc_locality_t opal_hwloc_base_get_relative_locality(hwloc_topology_t topo, char *cpuset1, char *cpuset2) { opal_hwloc_locality_t locality; - hwloc_obj_t obj; - unsigned depth, d, width, w; + hwloc_cpuset_t loc1, loc2; + unsigned depth, d; bool shared; hwloc_obj_type_t type; - int sect1, sect2; - hwloc_cpuset_t loc1, loc2; /* start with what we know - they share a node on a cluster * NOTE: we may alter that latter part as hwloc's ability to @@ -1265,66 +1336,8 @@ opal_hwloc_locality_t opal_hwloc_base_get_relative_locality(hwloc_topology_t top HWLOC_OBJ_PU != type) { continue; } - /* get the width of the topology at this depth */ - width = hwloc_get_nbobjs_by_depth(topo, d); + opal_hwloc_base_get_relative_locality_by_depth(topo, d, loc1, loc2, &locality, &shared); - /* scan all objects at this depth to see if - * our locations overlap with them - */ - for (w=0; w < width; w++) { - /* get the object at this depth/index */ - obj = hwloc_get_obj_by_depth(topo, d, w); - /* see if our locations intersect with the cpuset for this obj */ - sect1 = hwloc_bitmap_intersects(obj->cpuset, loc1); - sect2 = hwloc_bitmap_intersects(obj->cpuset, loc2); - /* if both intersect, then we share this level */ - if (sect1 && sect2) { - shared = true; - switch(obj->type) { - case HWLOC_OBJ_NODE: - locality |= OPAL_PROC_ON_NUMA; - break; - case HWLOC_OBJ_SOCKET: - locality |= OPAL_PROC_ON_SOCKET; - break; -#if HWLOC_API_VERSION < 0x20000 - case HWLOC_OBJ_CACHE: - if (3 == obj->attr->cache.depth) { - locality |= OPAL_PROC_ON_L3CACHE; - } else if (2 == obj->attr->cache.depth) { - locality |= OPAL_PROC_ON_L2CACHE; - } else { - locality |= OPAL_PROC_ON_L1CACHE; - } - break; -#else - case HWLOC_OBJ_L3CACHE: - locality |= OPAL_PROC_ON_L3CACHE; - break; - case HWLOC_OBJ_L2CACHE: - locality |= OPAL_PROC_ON_L2CACHE; - break; - case HWLOC_OBJ_L1CACHE: - locality |= OPAL_PROC_ON_L1CACHE; - break; -#endif - case HWLOC_OBJ_CORE: - locality |= OPAL_PROC_ON_CORE; - break; - case HWLOC_OBJ_PU: - locality |= OPAL_PROC_ON_HWTHREAD; - break; - default: - /* just ignore it */ - break; - } - break; - } - /* otherwise, we don't share this - * object - but we still might share another object - * on this level, so we have to keep searching - */ - } /* if we spanned the entire width without finding * a point of intersection, then no need to go * deeper @@ -1333,6 +1346,9 @@ opal_hwloc_locality_t opal_hwloc_base_get_relative_locality(hwloc_topology_t top break; } } +#if HWLOC_API_VERSION >= 0x20000 + opal_hwloc_base_get_relative_locality_by_depth(topo, HWLOC_TYPE_DEPTH_NUMANODE, loc1, loc2, &locality, &shared); +#endif opal_output_verbose(5, opal_hwloc_base_framework.framework_output, "locality: %s", @@ -1349,9 +1365,10 @@ opal_hwloc_locality_t opal_hwloc_base_get_relative_locality(hwloc_topology_t top */ char* opal_hwloc_base_find_coprocessors(hwloc_topology_t topo) { +#if HAVE_DECL_HWLOC_OBJ_OSDEV_COPROC hwloc_obj_t osdev; - unsigned i; char **cps = NULL; +#endif char *cpstring = NULL; int depth; @@ -1369,6 +1386,7 @@ char* opal_hwloc_base_find_coprocessors(hwloc_topology_t topo) while (NULL != osdev) { if (HWLOC_OBJ_OSDEV_COPROC == osdev->attr->osdev.type) { /* got one! find and save its serial number */ + unsigned i; for (i=0; i < osdev->infos_count; i++) { if (0 == strncmp(osdev->infos[i].name, "MICSerialNumber", strlen("MICSerialNumber"))) { OPAL_OUTPUT_VERBOSE((5, opal_hwloc_base_framework.framework_output, @@ -2063,12 +2081,40 @@ char* opal_hwloc_base_get_topo_signature(hwloc_topology_t topo) return sig; } +static int opal_hwloc_base_get_locality_string_by_depth(hwloc_topology_t topo, + int d, + hwloc_cpuset_t cpuset, + hwloc_cpuset_t result) +{ + hwloc_obj_t obj; + unsigned width, w; + + /* get the width of the topology at this depth */ + width = hwloc_get_nbobjs_by_depth(topo, d); + if (0 == width) { + return -1; + } + + /* scan all objects at this depth to see if + * the location overlaps with them + */ + for (w=0; w < width; w++) { + /* get the object at this depth/index */ + obj = hwloc_get_obj_by_depth(topo, d, w); + /* see if the location intersects with it */ + if (hwloc_bitmap_intersects(obj->cpuset, cpuset)) { + hwloc_bitmap_set(result, w); + } + } + + return 0; +} + char* opal_hwloc_base_get_locality_string(hwloc_topology_t topo, char *bitmap) { - hwloc_obj_t obj; char *locality=NULL, *tmp, *t2; - unsigned depth, d, width, w; + unsigned depth, d; hwloc_cpuset_t cpuset, result; hwloc_obj_type_t type; @@ -2111,28 +2157,15 @@ char* opal_hwloc_base_get_locality_string(hwloc_topology_t topo, continue; } - /* get the width of the topology at this depth */ - width = hwloc_get_nbobjs_by_depth(topo, d); - if (0 == width) { + if (opal_hwloc_base_get_locality_string_by_depth(topo, d, cpuset, result) < 0) { continue; } - /* scan all objects at this depth to see if - * the location overlaps with them - */ - for (w=0; w < width; w++) { - /* get the object at this depth/index */ - obj = hwloc_get_obj_by_depth(topo, d, w); - /* see if the location intersects with it */ - if (hwloc_bitmap_intersects(obj->cpuset, cpuset)) { - hwloc_bitmap_set(result, w); - } - } /* it should be impossible, but allow for the possibility * that we came up empty at this depth */ if (!hwloc_bitmap_iszero(result)) { hwloc_bitmap_list_asprintf(&tmp, result); - switch(obj->type) { + switch(type) { case HWLOC_OBJ_NODE: asprintf(&t2, "%sNM%s:", (NULL == locality) ? "" : locality, tmp); if (NULL != locality) { @@ -2148,15 +2181,16 @@ char* opal_hwloc_base_get_locality_string(hwloc_topology_t topo, locality = t2; break; #if HWLOC_API_VERSION < 0x20000 - case HWLOC_OBJ_CACHE: - if (3 == obj->attr->cache.depth) { + case HWLOC_OBJ_CACHE: { + unsigned cachedepth = hwloc_get_obj_by_depth(topo, d, 0)->attr->cache.depth; + if (3 == cachedepth) { asprintf(&t2, "%sL3%s:", (NULL == locality) ? "" : locality, tmp); if (NULL != locality) { free(locality); } locality = t2; break; - } else if (2 == obj->attr->cache.depth) { + } else if (2 == cachedepth) { asprintf(&t2, "%sL2%s:", (NULL == locality) ? "" : locality, tmp); if (NULL != locality) { free(locality); @@ -2172,6 +2206,7 @@ char* opal_hwloc_base_get_locality_string(hwloc_topology_t topo, break; } break; + } #else case HWLOC_OBJ_L3CACHE: asprintf(&t2, "%sL3%s:", (NULL == locality) ? "" : locality, tmp); @@ -2217,6 +2252,24 @@ char* opal_hwloc_base_get_locality_string(hwloc_topology_t topo, } hwloc_bitmap_zero(result); } + +#if HWLOC_API_VERSION >= 0x20000 + if (opal_hwloc_base_get_locality_string_by_depth(topo, HWLOC_TYPE_DEPTH_NUMANODE, cpuset, result) == 0) { + /* it should be impossible, but allow for the possibility + * that we came up empty at this depth */ + if (!hwloc_bitmap_iszero(result)) { + hwloc_bitmap_list_asprintf(&tmp, result); + asprintf(&t2, "%sNM%s:", (NULL == locality) ? "" : locality, tmp); + if (NULL != locality) { + free(locality); + } + locality = t2; + free(tmp); + } + hwloc_bitmap_zero(result); + } +#endif + hwloc_bitmap_free(result); hwloc_bitmap_free(cpuset);