diff --git a/ompi/communicator/comm.c b/ompi/communicator/comm.c index a43e1c9493..a3f6b1e560 100644 --- a/ompi/communicator/comm.c +++ b/ompi/communicator/comm.c @@ -10,7 +10,7 @@ * Copyright (c) 2004-2005 The Regents of the University of California. * All rights reserved. * Copyright (c) 2007-2011 University of Houston. All rights reserved. - * Copyright (c) 2007-2008 Cisco Systems, Inc. All rights reserved. + * Copyright (c) 2007-2012 Cisco Systems, Inc. All rights reserved. * Copyright (c) 2009 Sun Microsystems, Inc. All rights reserved. * Copyright (c) 2012 Oak Ridge National Labs. All rights reserved. * $COPYRIGHT$ @@ -25,6 +25,7 @@ #include #include "ompi/constants.h" +#include "opal/mca/hwloc/base/base.h" #include "opal/dss/dss.h" #include "orte/util/name_fns.h" diff --git a/ompi/mca/btl/openib/btl_openib_component.c b/ompi/mca/btl/openib/btl_openib_component.c index 7e249fb669..7d6e24bba5 100644 --- a/ompi/mca/btl/openib/btl_openib_component.c +++ b/ompi/mca/btl/openib/btl_openib_component.c @@ -10,7 +10,7 @@ * University of Stuttgart. All rights reserved. * Copyright (c) 2004-2005 The Regents of the University of California. * All rights reserved. - * Copyright (c) 2006-2011 Cisco Systems, Inc. All rights reserved. + * Copyright (c) 2006-2012 Cisco Systems, Inc. All rights reserved. * Copyright (c) 2006-2009 Mellanox Technologies. All rights reserved. * Copyright (c) 2006-2012 Los Alamos National Security, LLC. All rights * reserved. @@ -57,9 +57,8 @@ const char *ibv_get_sysfs_path(void); #include "opal/util/argv.h" #include "opal/memoryhooks/memory.h" #include "opal/mca/base/mca_base_param.h" -#include "opal/mca/carto/carto.h" -#include "opal/mca/carto/base/base.h" -#include "opal/mca/paffinity/base/base.h" +#include "opal/mca/hwloc/hwloc.h" +#include "opal/mca/hwloc/base/base.h" #include "opal/mca/installdirs/installdirs.h" #include "opal_stdint.h" @@ -2337,58 +2336,6 @@ static void ibv_free_device_list_compat(struct ibv_device **ib_devs) #endif } -static opal_carto_graph_t *host_topo; - -static int get_ib_dev_distance(struct ibv_device *dev) -{ - opal_paffinity_base_cpu_set_t cpus; - opal_carto_base_node_t *device_node; - int min_distance = -1, i, num_processors; - const char *device = ibv_get_device_name(dev); - - if(opal_paffinity_base_get_processor_info(&num_processors) != OMPI_SUCCESS) { - num_processors = 100; /* Choose something big enough */ - } - - device_node = opal_carto_base_find_node(host_topo, device); - - /* no topology info for device found. Assume that it is close */ - if(NULL == device_node) - return 0; - - OPAL_PAFFINITY_CPU_ZERO(cpus); - opal_paffinity_base_get(&cpus); - - for (i = 0; i < num_processors; i++) { - opal_carto_base_node_t *slot_node; - int distance, socket, core; - char *slot; - - if(!OPAL_PAFFINITY_CPU_ISSET(i, cpus)) - continue; - - opal_paffinity_base_get_map_to_socket_core(i, &socket, &core); - asprintf(&slot, "slot%d", socket); - - slot_node = opal_carto_base_find_node(host_topo, slot); - - free(slot); - - if(NULL == slot_node) - return 0; - - distance = opal_carto_base_spf(host_topo, slot_node, device_node); - - if(distance < 0) - return 0; - - if(min_distance < 0 || min_distance > distance) - min_distance = distance; - } - - return min_distance; -} - struct dev_distance { struct ibv_device *ib_dev; int distance; @@ -2402,23 +2349,165 @@ static int compare_distance(const void *p1, const void *p2) return d1->distance - d2->distance; } +static int get_ib_dev_distance(struct ibv_device *dev) +{ + /* If we don't have hwloc, we'll default to a distance of 0, + because we have no way of measuring. */ + int distance = 0; + +#if OPAL_HAVE_HWLOC + int a, b, i; + hwloc_cpuset_t my_cpuset = NULL, ibv_cpuset = NULL; + hwloc_obj_t my_obj, ibv_obj, node_obj; + + /* Note that this struct is owned by hwloc; there's no need to + free it at the end of time */ + static const struct hwloc_distances_s *hwloc_distances = NULL; + + if (NULL == hwloc_distances) { + hwloc_distances = + hwloc_get_whole_distance_matrix_by_type(opal_hwloc_topology, + HWLOC_OBJ_NODE); + } + + /* If we got no info, just return 0 */ + if (NULL == hwloc_distances || NULL == hwloc_distances->latency) { + goto out; + } + + /* Next, find the NUMA node where this IBV device is located */ + ibv_cpuset = hwloc_bitmap_alloc(); + if (NULL == ibv_cpuset) { + goto out; + } + if (0 != hwloc_ibv_get_device_cpuset(opal_hwloc_topology, dev, ibv_cpuset)) { + goto out; + } + ibv_obj = hwloc_get_obj_covering_cpuset(opal_hwloc_topology, ibv_cpuset); + if (NULL == ibv_obj) { + goto out; + } + + /* If ibv_obj is a NUMA node or below, we're good. */ + switch (ibv_obj->type) { + case HWLOC_OBJ_NODE: + case HWLOC_OBJ_SOCKET: + case HWLOC_OBJ_CACHE: + case HWLOC_OBJ_CORE: + case HWLOC_OBJ_PU: + while (NULL != ibv_obj && ibv_obj->type != HWLOC_OBJ_NODE) { + ibv_obj = ibv_obj->parent; + } + break; + + default: + /* If it's above a NUMA node, then I don't know how to compute + the distance... */ + ibv_obj = NULL; + break; + } + + /* If we don't have an object for this ibv device, give up */ + if (NULL == ibv_obj) { + goto out; + } + + /* This function is only called if the process is bound, so let's + find out where we are bound to. For the moment, we only care + about the NUMA node to which we are bound. */ + my_cpuset = hwloc_bitmap_alloc(); + if (NULL == my_cpuset) { + goto out; + } + if (0 != hwloc_get_cpubind(opal_hwloc_topology, my_cpuset, 0)) { + hwloc_bitmap_free(my_cpuset); + goto out; + } + my_obj = hwloc_get_obj_covering_cpuset(opal_hwloc_topology, my_cpuset); + if (NULL == my_obj) { + goto out; + } + + /* If my_obj is a NUMA node or below, we're good. */ + switch (my_obj->type) { + case HWLOC_OBJ_NODE: + case HWLOC_OBJ_SOCKET: + case HWLOC_OBJ_CACHE: + case HWLOC_OBJ_CORE: + case HWLOC_OBJ_PU: + while (NULL != my_obj && my_obj->type != HWLOC_OBJ_NODE) { + my_obj = my_obj->parent; + } + if (NULL != my_obj) { + /* Distance may be asymetrical, so calculate both of them + and take the max */ + a = hwloc_distances->latency[my_obj->logical_index * + (ibv_obj->logical_index * + hwloc_distances->nbobjs)]; + b = hwloc_distances->latency[ibv_obj->logical_index * + (my_obj->logical_index * + hwloc_distances->nbobjs)]; + distance = (a > b) ? a : b; + } + break; + + default: + /* If the obj is above a NUMA node, then we're bound to more than + one NUMA node. Find the max distance. */ + i = 0; + for (node_obj = hwloc_get_obj_inside_cpuset_by_type(opal_hwloc_topology, + ibv_obj->cpuset, + HWLOC_OBJ_NODE, i); + NULL != node_obj; + node_obj = hwloc_get_obj_inside_cpuset_by_type(opal_hwloc_topology, + ibv_obj->cpuset, + HWLOC_OBJ_NODE, ++i)) { + + a = hwloc_distances->latency[node_obj->logical_index * + (ibv_obj->logical_index * + hwloc_distances->nbobjs)]; + b = hwloc_distances->latency[ibv_obj->logical_index * + (node_obj->logical_index * + hwloc_distances->nbobjs)]; + a = (a > b) ? a : b; + distance = (a > distance) ? a : distance; + } + break; + } + + out: + if (NULL != ibv_cpuset) { + hwloc_bitmap_free(ibv_cpuset); + } + if (NULL != my_cpuset) { + hwloc_bitmap_free(my_cpuset); + } +#endif + + return distance; +} + static struct dev_distance * sort_devs_by_distance(struct ibv_device **ib_devs, int count) { int i; struct dev_distance *devs = (struct dev_distance *) malloc(count * sizeof(struct dev_distance)); - opal_carto_base_get_host_graph(&host_topo, "Infiniband"); - for (i = 0; i < count; i++) { devs[i].ib_dev = ib_devs[i]; - devs[i].distance = get_ib_dev_distance(ib_devs[i]); + if (OPAL_HAVE_HWLOC && orte_proc_is_bound) { + /* If this process is bound to one or more PUs, we can get + an accurate distance. */ + devs[i].distance = get_ib_dev_distance(ib_devs[i]); + } else { + /* Since we're not bound, just assume that the device is + close. */ + devs[i].distance = 0; + } } qsort(devs, count, sizeof(struct dev_distance), compare_distance); - opal_carto_base_free_graph(host_topo); - return devs; } diff --git a/ompi/mca/btl/sm/btl_sm.c b/ompi/mca/btl/sm/btl_sm.c index 5b2f4c4073..4800f5a50b 100644 --- a/ompi/mca/btl/sm/btl_sm.c +++ b/ompi/mca/btl/sm/btl_sm.c @@ -10,8 +10,8 @@ * Copyright (c) 2004-2005 The Regents of the University of California. * All rights reserved. * Copyright (c) 2006-2007 Voltaire. All rights reserved. - * Copyright (c) 2009 Cisco Systems, Inc. All rights reserved. - * Copyright (c) 2010 Los Alamos National Security, LLC. + * Copyright (c) 2009-2012 Cisco Systems, Inc. All rights reserved. + * Copyright (c) 2010-2012 Los Alamos National Security, LLC. * All rights reserved. * Copyright (c) 2010-2012 IBM Corporation. All rights reserved. * $COPYRIGHT$ @@ -41,10 +41,7 @@ #include "opal/class/opal_bitmap.h" #include "opal/util/output.h" #include "opal/util/printf.h" -#include "opal/mca/carto/carto.h" -#include "opal/mca/carto/base/base.h" -#include "opal/mca/paffinity/base/base.h" -#include "opal/mca/maffinity/base/base.h" +#include "opal/mca/hwloc/base/base.h" #include "orte/util/proc_info.h" #include "opal/datatype/opal_convertor.h" #include "ompi/class/ompi_free_list.h" @@ -128,133 +125,119 @@ static void *mpool_calloc(size_t nmemb, size_t size) return buf; } -static void init_maffinity(int *my_mem_node, int *max_mem_node) -{ - opal_carto_graph_t *topo; - opal_value_array_t dists; - int i, num_core, socket; - opal_paffinity_base_cpu_set_t cpus; - char *myslot = NULL; - opal_carto_node_distance_t *dist; - opal_carto_base_node_t *slot_node; - - *my_mem_node = 0; - *max_mem_node = 1; - - if (OMPI_SUCCESS != opal_carto_base_get_host_graph(&topo, "Memory")) { - return; - } - - OBJ_CONSTRUCT(&dists, opal_value_array_t); - opal_value_array_init(&dists, sizeof(opal_carto_node_distance_t)); - - if (OMPI_SUCCESS != opal_paffinity_base_get_processor_info(&num_core)) { - num_core = 100; /* set something large */ - } - - OPAL_PAFFINITY_CPU_ZERO(cpus); - opal_paffinity_base_get(&cpus); - - /* find core we are running on */ - for (i = 0; i < num_core; i++) { - if (OPAL_PAFFINITY_CPU_ISSET(i, cpus)) { - break; - } - } - - if (OMPI_SUCCESS != opal_paffinity_base_get_map_to_socket_core(i, &socket, &i)) { - /* no topology info available */ - goto out; - } - - asprintf(&myslot, "slot%d", socket); - - slot_node = opal_carto_base_find_node(topo, myslot); - - if(NULL == slot_node) { - goto out; - } - - opal_carto_base_get_nodes_distance(topo, slot_node, "Memory", &dists); - if((*max_mem_node = opal_value_array_get_size(&dists)) < 2) { - goto out; - } - - dist = (opal_carto_node_distance_t *) opal_value_array_get_item(&dists, 0); - opal_maffinity_base_node_name_to_id(dist->node->node_name, my_mem_node); -out: - if (myslot) { - free(myslot); - } - OBJ_DESTRUCT(&dists); - opal_carto_base_free_graph(topo); -} static int sm_btl_first_time_init(mca_btl_sm_t *sm_btl, int n) { size_t size, length, length_payload; char *sm_ctl_file; sm_fifo_t *my_fifos; - int my_mem_node=-1, num_mem_nodes=-1, i; + int my_mem_node, num_mem_nodes, i; ompi_proc_t **procs; size_t num_procs; + mca_mpool_base_resources_t res; + mca_btl_sm_component_t* m = &mca_btl_sm_component; - init_maffinity(&my_mem_node, &num_mem_nodes); - mca_btl_sm_component.mem_node = my_mem_node; - mca_btl_sm_component.num_mem_nodes = num_mem_nodes; + /* Assume we don't have hwloc support and fill in dummy info */ + mca_btl_sm_component.mem_node = my_mem_node = 0; + mca_btl_sm_component.num_mem_nodes = num_mem_nodes = 1; + +#if OPAL_HAVE_HWLOC + /* If we have hwloc support, then get accurate information */ + if (NULL != opal_hwloc_topology) { + i = opal_hwloc_base_get_nbobjs_by_type(opal_hwloc_topology, + HWLOC_OBJ_NODE, 0, + OPAL_HWLOC_AVAILABLE); + + /* If we find >0 NUMA nodes, then investigate further */ + if (i > 0) { + opal_hwloc_level_t bind_level; + unsigned int bind_index; + + /* JMS This tells me how many numa nodes are *available*, + but it's not how many are being used *by this job*. + Note that this is the value we've previously used (from + the previous carto-based implementation), but it really + should be improved to be how many NUMA nodes are being + used *in this job*. */ + mca_btl_sm_component.num_mem_nodes = num_mem_nodes = i; + + /* Fill opal_hwloc_my_cpuset and find out to what level + this process is bound (if at all) */ + opal_hwloc_base_get_local_cpuset(); + opal_hwloc_base_get_level_and_index(opal_hwloc_my_cpuset, + &bind_level, &bind_index); + if (OPAL_HWLOC_NODE_LEVEL != bind_level) { + /* We are bound to *something* (i.e., our binding + level is less than "node", meaning the entire + machine), so discover which NUMA node this process + is bound */ + if (OPAL_HWLOC_NUMA_LEVEL == bind_level) { + mca_btl_sm_component.mem_node = my_mem_node = (int) bind_index; + } else { + if (OPAL_SUCCESS == + opal_hwloc_base_get_local_index(HWLOC_OBJ_NODE, 0, &bind_index)) { + mca_btl_sm_component.mem_node = my_mem_node = (int) bind_index; + } else { + /* Weird. We can't figure out what NUMA node + we're on. :-( */ + mca_btl_sm_component.mem_node = my_mem_node = -1; + } + } + } + } + } +#endif /* lookup shared memory pool */ mca_btl_sm_component.sm_mpools = (mca_mpool_base_module_t **) calloc(num_mem_nodes, sizeof(mca_mpool_base_module_t*)); - /* create mpool for each memory node */ - for(i = 0; i < num_mem_nodes; i++) { - mca_mpool_base_resources_t res; - mca_btl_sm_component_t* m = &mca_btl_sm_component; + /* Create one mpool. Per discussion with George and a UTK Euro + MPI 2010 paper, it may be beneficial to create multiple mpools. + Leaving that for a future optimization, however. */ + /* Disable memory binding, because each MPI process will claim + pages in the mpool for their local NUMA node */ + res.mem_node = -1; - /* disable memory binding if there is only one memory node */ - res.mem_node = (num_mem_nodes == 1) ? -1 : i; + /* determine how much memory to create */ + /* + * This heuristic formula mostly says that we request memory for: + * - nfifos FIFOs, each comprising: + * . a sm_fifo_t structure + * . many pointers (fifo_size of them per FIFO) + * - eager fragments (2*n of them, allocated in sm_free_list_inc chunks) + * - max fragments (sm_free_list_num of them) + * + * On top of all that, we sprinkle in some number of + * "opal_cache_line_size" additions to account for some + * padding and edge effects that may lie in the allocator. + */ + res.size = + FIFO_MAP_NUM(n) * ( sizeof(sm_fifo_t) + sizeof(void *) * m->fifo_size + 4 * opal_cache_line_size ) + + ( 2 * n + m->sm_free_list_inc ) * ( m->eager_limit + 2 * opal_cache_line_size ) + + m->sm_free_list_num * ( m->max_frag_size + 2 * opal_cache_line_size ); - /* determine how much memory to create */ - /* - * This heuristic formula mostly says that we request memory for: - * - nfifos FIFOs, each comprising: - * . a sm_fifo_t structure - * . many pointers (fifo_size of them per FIFO) - * - eager fragments (2*n of them, allocated in sm_free_list_inc chunks) - * - max fragments (sm_free_list_num of them) - * - * On top of all that, we sprinkle in some number of "opal_cache_line_size" - * additions to account for some padding and edge effects that may lie - * in the allocator. - */ - res.size = - FIFO_MAP_NUM(n) * ( sizeof(sm_fifo_t) + sizeof(void *) * m->fifo_size + 4 * opal_cache_line_size ) - + ( 2 * n + m->sm_free_list_inc ) * ( m->eager_limit + 2 * opal_cache_line_size ) - + m->sm_free_list_num * ( m->max_frag_size + 2 * opal_cache_line_size ); - - /* before we multiply by n, make sure the result won't overflow */ - /* Stick that little pad in, particularly since we'll eventually - * need a little extra space. E.g., in mca_mpool_sm_init() in - * mpool_sm_component.c when sizeof(mca_common_sm_module_t) is - * added. - */ - if ( ((double) res.size) * n > LONG_MAX - 4096 ) + /* before we multiply by n, make sure the result won't overflow */ + /* Stick that little pad in, particularly since we'll eventually + * need a little extra space. E.g., in mca_mpool_sm_init() in + * mpool_sm_component.c when sizeof(mca_common_sm_module_t) is + * added. + */ + if ( ((double) res.size) * n > LONG_MAX - 4096 ) { + return OMPI_ERR_OUT_OF_RESOURCE; + } + res.size *= n; + + /* now, create it */ + mca_btl_sm_component.sm_mpools[0] = + mca_mpool_base_module_create(mca_btl_sm_component.sm_mpool_name, + sm_btl, &res); + /* Sanity check to ensure that we found it */ + if (NULL == mca_btl_sm_component.sm_mpools[0]) { return OMPI_ERR_OUT_OF_RESOURCE; - res.size *= n; - - /* now, create it */ - mca_btl_sm_component.sm_mpools[i] = - mca_mpool_base_module_create(mca_btl_sm_component.sm_mpool_name, - sm_btl, &res); - /* Sanity check to ensure that we found it */ - if(NULL == mca_btl_sm_component.sm_mpools[i]) - return OMPI_ERR_OUT_OF_RESOURCE; - - if(i == my_mem_node) - mca_btl_sm_component.sm_mpool = mca_btl_sm_component.sm_mpools[i]; } + mca_btl_sm_component.sm_mpool = mca_btl_sm_component.sm_mpools[0]; mca_btl_sm_component.sm_mpool_base = mca_btl_sm_component.sm_mpools[0]->mpool_base(mca_btl_sm_component.sm_mpools[0]); @@ -262,17 +245,19 @@ static int sm_btl_first_time_init(mca_btl_sm_t *sm_btl, int n) /* create a list of peers */ mca_btl_sm_component.sm_peers = (struct mca_btl_base_endpoint_t**) calloc(n, sizeof(struct mca_btl_base_endpoint_t*)); - if(NULL == mca_btl_sm_component.sm_peers) + if (NULL == mca_btl_sm_component.sm_peers) { return OMPI_ERR_OUT_OF_RESOURCE; + } /* Allocate Shared Memory BTL process coordination * data structure. This will reside in shared memory */ /* set file name */ - if(asprintf(&sm_ctl_file, "%s"OPAL_PATH_SEP"shared_mem_btl_module.%s", - orte_process_info.job_session_dir, - orte_process_info.nodename) < 0) + if (asprintf(&sm_ctl_file, "%s"OPAL_PATH_SEP"shared_mem_btl_module.%s", + orte_process_info.job_session_dir, + orte_process_info.nodename) < 0) { return OMPI_ERR_OUT_OF_RESOURCE; + } /* Pass in a data segment alignment of 0 to get no data segment (only the shared control structure) */ diff --git a/ompi/mca/btl/smcuda/btl_smcuda.c b/ompi/mca/btl/smcuda/btl_smcuda.c index 2eb130279a..2d64e757b7 100644 --- a/ompi/mca/btl/smcuda/btl_smcuda.c +++ b/ompi/mca/btl/smcuda/btl_smcuda.c @@ -10,8 +10,8 @@ * Copyright (c) 2004-2005 The Regents of the University of California. * All rights reserved. * Copyright (c) 2006-2007 Voltaire. All rights reserved. - * Copyright (c) 2009 Cisco Systems, Inc. All rights reserved. - * Copyright (c) 2010 Los Alamos National Security, LLC. + * Copyright (c) 2009-2012 Cisco Systems, Inc. All rights reserved. + * Copyright (c) 2010-2012 Los Alamos National Security, LLC. * All rights reserved. * Copyright (c) 2012 NVIDIA Corporation. All rights reserved. * $COPYRIGHT$ @@ -37,10 +37,7 @@ #include "opal/class/opal_bitmap.h" #include "opal/util/output.h" #include "opal/util/printf.h" -#include "opal/mca/carto/carto.h" -#include "opal/mca/carto/base/base.h" -#include "opal/mca/paffinity/base/base.h" -#include "opal/mca/maffinity/base/base.h" +#include "opal/mca/hwloc/base/base.h" #include "orte/util/proc_info.h" #include "opal/datatype/opal_convertor.h" #include "ompi/class/ompi_free_list.h" @@ -127,68 +124,6 @@ static void *mpool_calloc(size_t nmemb, size_t size) return buf; } -static void init_maffinity(int *my_mem_node, int *max_mem_node) -{ - opal_carto_graph_t *topo; - opal_value_array_t dists; - int i, num_core, socket; - opal_paffinity_base_cpu_set_t cpus; - char *myslot = NULL; - opal_carto_node_distance_t *dist; - opal_carto_base_node_t *slot_node; - - *my_mem_node = 0; - *max_mem_node = 1; - - if (OMPI_SUCCESS != opal_carto_base_get_host_graph(&topo, "Memory")) { - return; - } - - OBJ_CONSTRUCT(&dists, opal_value_array_t); - opal_value_array_init(&dists, sizeof(opal_carto_node_distance_t)); - - if (OMPI_SUCCESS != opal_paffinity_base_get_processor_info(&num_core)) { - num_core = 100; /* set something large */ - } - - OPAL_PAFFINITY_CPU_ZERO(cpus); - opal_paffinity_base_get(&cpus); - - /* find core we are running on */ - for (i = 0; i < num_core; i++) { - if (OPAL_PAFFINITY_CPU_ISSET(i, cpus)) { - break; - } - } - - if (OMPI_SUCCESS != opal_paffinity_base_get_map_to_socket_core(i, &socket, &i)) { - /* no topology info available */ - goto out; - } - - asprintf(&myslot, "slot%d", socket); - - slot_node = opal_carto_base_find_node(topo, myslot); - - if(NULL == slot_node) { - goto out; - } - - opal_carto_base_get_nodes_distance(topo, slot_node, "Memory", &dists); - if((*max_mem_node = opal_value_array_get_size(&dists)) < 2) { - goto out; - } - - dist = (opal_carto_node_distance_t *) opal_value_array_get_item(&dists, 0); - opal_maffinity_base_node_name_to_id(dist->node->node_name, my_mem_node); -out: - if (myslot) { - free(myslot); - } - OBJ_DESTRUCT(&dists); - opal_carto_base_free_graph(topo); -} - static int smcuda_btl_first_time_init(mca_btl_smcuda_t *smcuda_btl, int n) { size_t size, length, length_payload; @@ -197,61 +132,111 @@ static int smcuda_btl_first_time_init(mca_btl_smcuda_t *smcuda_btl, int n) int my_mem_node=-1, num_mem_nodes=-1, i; ompi_proc_t **procs; size_t num_procs; + mca_mpool_base_resources_t res; + mca_btl_smcuda_component_t* m = &mca_btl_smcuda_component; - init_maffinity(&my_mem_node, &num_mem_nodes); - mca_btl_smcuda_component.mem_node = my_mem_node; - mca_btl_smcuda_component.num_mem_nodes = num_mem_nodes; + /* Assume we don't have hwloc support and fill in dummy info */ + mca_btl_sm_component.mem_node = my_mem_node = 0; + mca_btl_sm_component.num_mem_nodes = num_mem_nodes = 1; + +#if OPAL_HAVE_HWLOC + /* If we have hwloc support, then get accurate information */ + if (NULL != opal_hwloc_topology) { + i = opal_hwloc_base_get_nbobjs_by_type(opal_hwloc_topology, + HWLOC_OBJ_NODE, 0, + OPAL_HWLOC_AVAILABLE); + + /* If we find >0 NUMA nodes, then investigate further */ + if (i > 0) { + opal_hwloc_level_t bind_level; + unsigned int bind_index; + + /* JMS This tells me how many numa nodes are *available*, + but it's not how many are being used *by this job*. + Note that this is the value we've previously used (from + the previous carto-based implementation), but it really + should be improved to be how many NUMA nodes are being + used *in this job*. */ + mca_btl_sm_component.num_mem_nodes = num_mem_nodes = i; + + /* Fill opal_hwloc_my_cpuset and find out to what level + this process is bound (if at all) */ + opal_hwloc_base_get_local_cpuset(); + opal_hwloc_base_get_level_and_index(opal_hwloc_my_cpuset, + &bind_level, &bind_index); + if (OPAL_HWLOC_NODE_LEVEL != bind_level) { + /* We are bound to *something* (i.e., our binding + level is less than "node", meaning the entire + machine), so discover which NUMA node this process + is bound */ + if (OPAL_HWLOC_NUMA_LEVEL == bind_level) { + mca_btl_sm_component.mem_node = my_mem_node = (int) bind_index; + } else { + if (OPAL_SUCCESS == + opal_hwloc_base_get_local_index(HWLOC_OBJ_NODE, 0, &bind_index)) { + mca_btl_sm_component.mem_node = my_mem_node = (int) bind_index; + } else { + /* Weird. We can't figure out what NUMA node + we're on. :-( */ + mca_btl_sm_component.mem_node = my_mem_node = -1; + } + } + } + } + } +#endif /* lookup shared memory pool */ mca_btl_smcuda_component.sm_mpools = (mca_mpool_base_module_t **) calloc(num_mem_nodes, sizeof(mca_mpool_base_module_t*)); - /* create mpool for each memory node */ - for(i = 0; i < num_mem_nodes; i++) { - mca_mpool_base_resources_t res; - mca_btl_smcuda_component_t* m = &mca_btl_smcuda_component; + /* Create one mpool. Per discussion with George and a UTK Euro + MPI 2010 paper, it may be beneficial to create multiple mpools. + Leaving that for a future optimization, however. */ + /* Disable memory binding, because each MPI process will claim + pages in the mpool for their local NUMA node */ + res.mem_node = -1; - /* disable memory binding if there is only one memory node */ - res.mem_node = (num_mem_nodes == 1) ? -1 : i; + /* determine how much memory to create */ + /* + * This heuristic formula mostly says that we request memory for: + * - nfifos FIFOs, each comprising: + * . a sm_fifo_t structure + * . many pointers (fifo_size of them per FIFO) + * - eager fragments (2*n of them, allocated in sm_free_list_inc chunks) + * - max fragments (sm_free_list_num of them) + * + * On top of all that, we sprinkle in some number of "opal_cache_line_size" + * additions to account for some padding and edge effects that may lie + * in the allocator. + */ + res.size = + FIFO_MAP_NUM(n) * ( sizeof(sm_fifo_t) + sizeof(void *) * m->fifo_size + 4 * opal_cache_line_size ) + + ( 2 * n + m->sm_free_list_inc ) * ( m->eager_limit + 2 * opal_cache_line_size ) + + m->sm_free_list_num * ( m->max_frag_size + 2 * opal_cache_line_size ); - /* determine how much memory to create */ - /* - * This heuristic formula mostly says that we request memory for: - * - nfifos FIFOs, each comprising: - * . a sm_fifo_t structure - * . many pointers (fifo_size of them per FIFO) - * - eager fragments (2*n of them, allocated in sm_free_list_inc chunks) - * - max fragments (sm_free_list_num of them) - * - * On top of all that, we sprinkle in some number of "opal_cache_line_size" - * additions to account for some padding and edge effects that may lie - * in the allocator. - */ - res.size = - FIFO_MAP_NUM(n) * ( sizeof(sm_fifo_t) + sizeof(void *) * m->fifo_size + 4 * opal_cache_line_size ) - + ( 2 * n + m->sm_free_list_inc ) * ( m->eager_limit + 2 * opal_cache_line_size ) - + m->sm_free_list_num * ( m->max_frag_size + 2 * opal_cache_line_size ); + /* before we multiply by n, make sure the result won't overflow */ + /* Stick that little pad in, particularly since we'll eventually + * need a little extra space. E.g., in mca_mpool_sm_init() in + * mpool_sm_component.c when sizeof(mca_common_sm_module_t) is + * added. + */ + if ( ((double) res.size) * n > LONG_MAX - 4096 ) { + return OMPI_ERR_OUT_OF_RESOURCE; + } + res.size *= n; - /* before we multiply by n, make sure the result won't overflow */ - /* Stick that little pad in, particularly since we'll eventually - * need a little extra space. E.g., in mca_mpool_sm_init() in - * mpool_sm_component.c when sizeof(mca_common_sm_module_t) is - * added. - */ - if ( ((double) res.size) * n > LONG_MAX - 4096 ) + /* now, create it */ + mca_btl_smcuda_component.sm_mpools[0] = + mca_mpool_base_module_create(mca_btl_smcuda_component.sm_mpool_name, + smcuda_btl, &res); + /* Sanity check to ensure that we found it */ + if (NULL == mca_btl_smcuda_component.sm_mpools[0]) { return OMPI_ERR_OUT_OF_RESOURCE; - res.size *= n; + } - /* now, create it */ - mca_btl_smcuda_component.sm_mpools[i] = - mca_mpool_base_module_create(mca_btl_smcuda_component.sm_mpool_name, - smcuda_btl, &res); - /* Sanity check to ensure that we found it */ - if(NULL == mca_btl_smcuda_component.sm_mpools[i]) - return OMPI_ERR_OUT_OF_RESOURCE; + mca_btl_smcuda_component.sm_mpool = mca_btl_smcuda_component.sm_mpools[0]; - if(i == my_mem_node) - mca_btl_smcuda_component.sm_mpool = mca_btl_smcuda_component.sm_mpools[i]; #if OMPI_CUDA_SUPPORT /* Create a local memory pool that sends handles to the remote * side. Note that the res argument is not really used, but diff --git a/ompi/mca/btl/wv/btl_wv_component.c b/ompi/mca/btl/wv/btl_wv_component.c index 12dc8c8e25..42a51b8e64 100644 --- a/ompi/mca/btl/wv/btl_wv_component.c +++ b/ompi/mca/btl/wv/btl_wv_component.c @@ -10,7 +10,7 @@ * University of Stuttgart. All rights reserved. * Copyright (c) 2004-2005 The Regents of the University of California. * All rights reserved. - * Copyright (c) 2006-2011 Cisco Systems, Inc. All rights reserved. + * Copyright (c) 2006-2012 Cisco Systems, Inc. All rights reserved. * Copyright (c) 2006-2009 Mellanox Technologies. All rights reserved. * Copyright (c) 2006-2012 Los Alamos National Security, LLC. All rights * reserved. @@ -41,9 +41,7 @@ #include "opal/util/argv.h" #include "opal/memoryhooks/memory.h" #include "opal/mca/base/mca_base_param.h" -#include "opal/mca/carto/carto.h" -#include "opal/mca/carto/base/base.h" -#include "opal/mca/paffinity/base/base.h" +#include "opal/mca/hwloc/base/base.h" #include "opal/mca/installdirs/installdirs.h" #include "opal_stdint.h" @@ -1926,63 +1924,149 @@ static void wv_free_device_list_compat(struct wv_device **ib_devs) free(ib_devs); } -static opal_carto_graph_t *host_topo; - -static int get_ib_dev_distance(struct wv_device *dev) -{ - opal_paffinity_base_cpu_set_t cpus; - opal_carto_base_node_t *device_node; - int min_distance = -1, i, num_processors; - const char *device = dev->name; - - if(opal_paffinity_base_get_processor_info(&num_processors) != OMPI_SUCCESS) { - num_processors = 100; /* Choose something big enough */ - } - - device_node = opal_carto_base_find_node(host_topo, device); - - /* no topology info for device found. Assume that it is close */ - if(NULL == device_node) - return 0; - - OPAL_PAFFINITY_CPU_ZERO(cpus); - opal_paffinity_base_get(&cpus); - - for (i = 0; i < num_processors; i++) { - opal_carto_base_node_t *slot_node; - int distance, socket, core; - char *slot; - - if(!OPAL_PAFFINITY_CPU_ISSET(i, cpus)) - continue; - - opal_paffinity_base_get_map_to_socket_core(i, &socket, &core); - asprintf(&slot, "slot%d", socket); - - slot_node = opal_carto_base_find_node(host_topo, slot); - - free(slot); - - if(NULL == slot_node) - return 0; - - distance = opal_carto_base_spf(host_topo, slot_node, device_node); - - if(distance < 0) - return 0; - - if(min_distance < 0 || min_distance > distance) - min_distance = distance; - } - - return min_distance; -} - struct dev_distance { struct wv_device *ib_dev; int distance; }; +static int get_ib_dev_distance(struct ibv_device *dev) +{ + /* If we don't have hwloc, we'll default to a distance of 0, + because we have no way of measuring. */ + int distance = 0; + +#if OPAL_HAVE_HWLOC + int a, b, i; + hwloc_cpuset_t my_cpuset = NULL, ibv_cpuset = NULL; + hwloc_obj_t my_obj, ibv_obj, node_obj; + + /* Note that this struct is owned by hwloc; there's no need to + free it at the end of time */ + static const struct hwloc_distances_s *hwloc_distances = NULL; + + if (NULL == hwloc_distances) { + hwloc_distances = + hwloc_get_whole_distance_matrix_by_type(opal_hwloc_topology, + HWLOC_OBJ_NODE); + } + + /* If we got no info, just return 0 */ + if (NULL == hwloc_distances || NULL == hwloc_distances->latency) { + goto out; + } + + /* Next, find the NUMA node where this IBV device is located */ + ibv_cpuset = hwloc_bitmap_alloc(); + if (NULL == ibv_cpuset) { + goto out; + } + if (0 != hwloc_ibv_get_device_cpuset(opal_hwloc_topology, dev, ibv_cpuset)) { + goto out; + } + ibv_obj = hwloc_get_obj_covering_cpuset(opal_hwloc_topology, ibv_cpuset); + if (NULL == ibv_obj) { + goto out; + } + + /* If ibv_obj is a NUMA node or below, we're good. */ + switch (ibv_obj->type) { + case HWLOC_OBJ_NODE: + case HWLOC_OBJ_SOCKET: + case HWLOC_OBJ_CACHE: + case HWLOC_OBJ_CORE: + case HWLOC_OBJ_PU: + while (NULL != ibv_obj && ibv_obj->type != HWLOC_OBJ_NODE) { + ibv_obj = ibv_obj->parent; + } + break; + + default: + /* If it's above a NUMA node, then I don't know how to compute + the distance... */ + ibv_obj = NULL; + break; + } + + /* If we don't have an object for this ibv device, give up */ + if (NULL == ibv_obj) { + goto out; + } + + /* This function is only called if the process is bound, so let's + find out where we are bound to. For the moment, we only care + about the NUMA node to which we are bound. */ + my_cpuset = hwloc_bitmap_alloc(); + if (NULL == my_cpuset) { + goto out; + } + if (0 != hwloc_get_cpubind(opal_hwloc_topology, my_cpuset, 0)) { + hwloc_bitmap_free(my_cpuset); + goto out; + } + my_obj = hwloc_get_obj_covering_cpuset(opal_hwloc_topology, my_cpuset); + if (NULL == my_obj) { + goto out; + } + + /* If my_obj is a NUMA node or below, we're good. */ + switch (my_obj->type) { + case HWLOC_OBJ_NODE: + case HWLOC_OBJ_SOCKET: + case HWLOC_OBJ_CACHE: + case HWLOC_OBJ_CORE: + case HWLOC_OBJ_PU: + while (NULL != my_obj && my_obj->type != HWLOC_OBJ_NODE) { + my_obj = my_obj->parent; + } + if (NULL != my_obj) { + /* Distance may be asymetrical, so calculate both of them + and take the max */ + a = hwloc_distances->latency[my_obj->logical_index * + (ibv_obj->logical_index * + hwloc_distances->nbobjs)]; + b = hwloc_distances->latency[ibv_obj->logical_index * + (my_obj->logical_index * + hwloc_distances->nbobjs)]; + distance = (a > b) ? a : b; + } + break; + + default: + /* If the obj is above a NUMA node, then we're bound to more than + one NUMA node. Find the max distance. */ + i = 0; + for (node_obj = hwloc_get_obj_inside_cpuset_by_type(opal_hwloc_topology, + ibv_obj->cpuset, + HWLOC_OBJ_NODE, i); + NULL != node_obj; + node_obj = hwloc_get_obj_inside_cpuset_by_type(opal_hwloc_topology, + ibv_obj->cpuset, + HWLOC_OBJ_NODE, ++i)) { + + a = hwloc_distances->latency[node_obj->logical_index * + (ibv_obj->logical_index * + hwloc_distances->nbobjs)]; + b = hwloc_distances->latency[ibv_obj->logical_index * + (node_obj->logical_index * + hwloc_distances->nbobjs)]; + a = (a > b) ? a : b; + distance = (a > distance) ? a : distance; + } + break; + } + + out: + if (NULL != ibv_cpuset) { + hwloc_bitmap_free(ibv_cpuset); + } + if (NULL != my_cpuset) { + hwloc_bitmap_free(my_cpuset); + } +#endif + + return distance; +} + static int compare_distance(const void *p1, const void *p2) { const struct dev_distance *d1 = (const struct dev_distance *) p1; @@ -2001,17 +2085,24 @@ sort_devs_by_distance(struct wv_device **ib_devs, int count) * if we have found more than one device, search for the shortest path. * otherwise, just use the one we got */ - if( count == 1) { + if (1 == count) { devs[0].ib_dev = ib_devs[0]; devs[0].distance = 0; return devs; } - opal_carto_base_get_host_graph(&host_topo, "Infiniband"); for (i = 0; i < count; i++) { devs[i].ib_dev = ib_devs[i]; - devs[i].distance = get_ib_dev_distance(ib_devs[i]); + if (orte_proc_is_bound) { + /* If this process is bound to one or more PUs, we can get + an accurate distance. */ + devs[i].distance = get_ib_dev_distance(ib_devs[i]); + } else { + /* Since we're not bound, just assume that the device is + close. */ + devs[i].distance = 0; + } } qsort(devs, count, sizeof(struct dev_distance), compare_distance); diff --git a/ompi/mca/coll/fca/coll_fca_module.c b/ompi/mca/coll/fca/coll_fca_module.c index 246f3c9628..aa05609b5e 100644 --- a/ompi/mca/coll/fca/coll_fca_module.c +++ b/ompi/mca/coll/fca/coll_fca_module.c @@ -1,5 +1,6 @@ /** Copyright (c) 2011 Mellanox Technologies. All rights reserved. + Copyright (c) 2012 Cisco Systems, Inc. All rights reserved. $COPYRIGHT$ Additional copyrights may follow @@ -7,7 +8,6 @@ $HEADER$ */ #include "coll_fca.h" -#include "opal/mca/paffinity/paffinity.h" /* * Initial query function that is invoked during MPI_INIT, allowing diff --git a/ompi/mca/coll/hierarch/coll_hierarch.c b/ompi/mca/coll/hierarch/coll_hierarch.c index e752642c12..3b4ac73731 100644 --- a/ompi/mca/coll/hierarch/coll_hierarch.c +++ b/ompi/mca/coll/hierarch/coll_hierarch.c @@ -12,6 +12,7 @@ * All rights reserved. * Copyright (c) 2007-2008 University of Houston. All rights reserved. * Copyright (c) 2008 Sun Microsystems, Inc. All rights reserved. + * Copyright (c) 2012 Cisco Systems, Inc. All rights reserved. * $COPYRIGHT$ * * Additional copyrights may follow @@ -24,6 +25,8 @@ #include +#include "opal/mca/hwloc/base/base.h" + #include "mpi.h" #include "ompi/communicator/communicator.h" #include "ompi/group/group.h" diff --git a/ompi/mca/coll/sm/coll_sm_module.c b/ompi/mca/coll/sm/coll_sm_module.c index 8c1bba49de..11e831039e 100644 --- a/ompi/mca/coll/sm/coll_sm_module.c +++ b/ompi/mca/coll/sm/coll_sm_module.c @@ -10,8 +10,8 @@ * Copyright (c) 2004-2005 The Regents of the University of California. * All rights reserved. * Copyright (c) 2008 Sun Microsystems, Inc. All rights reserved. - * Copyright (c) 2009-2010 Cisco Systems, Inc. All rights reserved. - * Copyright (c) 2010-2011 Los Alamos National Security, LLC. + * Copyright (c) 2009-2012 Cisco Systems, Inc. All rights reserved. + * Copyright (c) 2010-2012 Los Alamos National Security, LLC. * All rights reserved. * $COPYRIGHT$ * @@ -48,8 +48,7 @@ #include "mpi.h" #include "opal_stdint.h" -#include "opal/mca/maffinity/maffinity.h" -#include "opal/mca/maffinity/base/base.h" +#include "opal/mca/hwloc/base/base.h" #include "opal/util/os_path.h" #include "orte/util/proc_info.h" @@ -257,7 +256,7 @@ int ompi_coll_sm_lazy_enable(mca_coll_base_module_t *module, mca_coll_sm_comm_t *data = NULL; size_t control_size, frag_size; mca_coll_sm_component_t *c = &mca_coll_sm_component; - opal_maffinity_base_segment_t *maffinity; + opal_hwloc_base_memory_segment_t *maffinity; int parent, min_child, max_child, num_children; unsigned char *base = NULL; const int num_barrier_buffers = 2; @@ -270,8 +269,8 @@ int ompi_coll_sm_lazy_enable(mca_coll_base_module_t *module, /* Get some space to setup memory affinity (just easier to try to alloc here to handle the error case) */ - maffinity = (opal_maffinity_base_segment_t*) - malloc(sizeof(opal_maffinity_base_segment_t) * + maffinity = (opal_hwloc_base_memory_segment_t*) + malloc(sizeof(opal_hwloc_base_memory_segment_t) * c->sm_comm_num_segments * 3); if (NULL == maffinity) { opal_output_verbose(10, mca_coll_base_output, @@ -458,7 +457,7 @@ int ompi_coll_sm_lazy_enable(mca_coll_base_module_t *module, /* Setup memory affinity so that the pages that belong to this process are local to this process */ - opal_maffinity_base_set(maffinity, j); + opal_hwloc_base_memory_set(maffinity, j); free(maffinity); /* Zero out the control structures that belong to this process */ diff --git a/ompi/mca/mpool/sm/mpool_sm_module.c b/ompi/mca/mpool/sm/mpool_sm_module.c index 3319d2621b..87d543c99a 100644 --- a/ompi/mca/mpool/sm/mpool_sm_module.c +++ b/ompi/mca/mpool/sm/mpool_sm_module.c @@ -9,8 +9,8 @@ * University of Stuttgart. All rights reserved. * Copyright (c) 2004-2005 The Regents of the University of California. * All rights reserved. - * Copyright (c) 2009 Cisco Systems, Inc. All rights reserved. - * Copyright (c) 2011 Los Alamos National Security, LLC. + * Copyright (c) 2009-2012 Cisco Systems, Inc. All rights reserved. + * Copyright (c) 2011-2012 Los Alamos National Security, LLC. * All rights reserved. * Copyright (c) 2011 NVIDIA Corporation. All rights reserved. * $COPYRIGHT$ @@ -28,9 +28,7 @@ #ifdef HAVE_UNISTD_H #include #endif -#include "opal/mca/maffinity/maffinity.h" -#include "opal/mca/maffinity/maffinity_types.h" -#include "opal/mca/maffinity/base/base.h" +#include "opal/mca/hwloc/base/base.h" #include "orte/util/proc_info.h" #if OPAL_ENABLE_FT_CR == 1 @@ -87,14 +85,14 @@ void* mca_mpool_sm_alloc( mca_mpool_base_registration_t** registration) { mca_mpool_sm_module_t* mpool_sm = (mca_mpool_sm_module_t*)mpool; - opal_maffinity_base_segment_t mseg; + opal_hwloc_base_memory_segment_t mseg; mseg.mbs_start_addr = mpool_sm->sm_allocator->alc_alloc(mpool_sm->sm_allocator, size, align, registration); if(mpool_sm->mem_node >= 0) { mseg.mbs_len = size; - opal_maffinity_base_bind(&mseg, 1, mpool_sm->mem_node); + opal_hwloc_base_membind(&mseg, 1, mpool_sm->mem_node); } #if OPAL_CUDA_SUPPORT @@ -117,14 +115,14 @@ void* mca_mpool_sm_realloc( mca_mpool_base_registration_t** registration) { mca_mpool_sm_module_t* mpool_sm = (mca_mpool_sm_module_t*)mpool; - opal_maffinity_base_segment_t mseg; + opal_hwloc_base_memory_segment_t mseg; mseg.mbs_start_addr = mpool_sm->sm_allocator->alc_realloc(mpool_sm->sm_allocator, addr, size, registration); if(mpool_sm->mem_node >= 0) { mseg.mbs_len = size; - opal_maffinity_base_bind(&mseg, 1, mpool_sm->mem_node); + opal_hwloc_base_membind(&mseg, 1, mpool_sm->mem_node); } return mseg.mbs_start_addr; diff --git a/ompi/mpiext/affinity/c/mpiext_affinity_str.c b/ompi/mpiext/affinity/c/mpiext_affinity_str.c index da99803b43..cc1ba09856 100644 --- a/ompi/mpiext/affinity/c/mpiext_affinity_str.c +++ b/ompi/mpiext/affinity/c/mpiext_affinity_str.c @@ -4,6 +4,8 @@ * Corporation. All rights reserved. * Copyright (c) 2010-2012 Cisco Systems, Inc. All rights reserved. * Copyright (c) 2010 Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2012 Los Alamos National Security, LLC. All rights + * reserved. * $COPYRIGHT$ * * Additional copyrights may follow @@ -24,7 +26,10 @@ #include #include -#include "opal/mca/paffinity/base/base.h" +#include "opal/mca/hwloc/base/base.h" +#include "opal/runtime/opal.h" + +#include "orte/runtime/orte_globals.h" #include "ompi/communicator/communicator.h" #include "ompi/errhandler/errhandler.h" @@ -40,6 +45,8 @@ static int get_layout_ompi_bound(char str[OMPI_AFFINITY_STRING_MAX]); static int get_layout_current_binding(char str[OMPI_AFFINITY_STRING_MAX]); static int get_layout_exists(char str[OMPI_AFFINITY_STRING_MAX]); +/*------------------------------------------------------------------------------*/ + int OMPI_Affinity_str(ompi_affinity_fmt_t fmt_type, char ompi_bound[OMPI_AFFINITY_STRING_MAX], char current_binding[OMPI_AFFINITY_STRING_MAX], @@ -50,18 +57,27 @@ int OMPI_Affinity_str(ompi_affinity_fmt_t fmt_type, memset(ompi_bound, 0, sizeof(ompi_bound)); memset(current_binding, 0, sizeof(current_binding)); - switch(fmt_type) { + /* If we have no hwloc support, return nothing */ + if (NULL == opal_hwloc_topology) { + strncpy(ompi_bound, "Not supported", sizeof(ompi_bound)); + strncpy(current_binding, "Not supported", sizeof(current_binding)); + strncpy(exists, "Not supported", sizeof(exists)); + return MPI_SUCCESS; + } + + /* Otherwise, return useful information */ + switch (fmt_type) { case OMPI_AFFINITY_RSRC_STRING_FMT: - if (OPAL_SUCCESS != (ret = get_rsrc_ompi_bound(ompi_bound)) || - OPAL_SUCCESS != (ret = get_rsrc_current_binding(current_binding)) || - OPAL_SUCCESS != (ret = get_rsrc_exists(exists))) { + if (OMPI_SUCCESS != (ret = get_rsrc_ompi_bound(ompi_bound)) || + OMPI_SUCCESS != (ret = get_rsrc_current_binding(current_binding)) || + OMPI_SUCCESS != (ret = get_rsrc_exists(exists))) { return OMPI_ERRHANDLER_INVOKE(MPI_COMM_WORLD, ret, FUNC_NAME); } break; case OMPI_AFFINITY_LAYOUT_FMT: - if (OPAL_SUCCESS != (ret = get_layout_ompi_bound(ompi_bound)) || - OPAL_SUCCESS != (ret = get_layout_current_binding(current_binding)) || - OPAL_SUCCESS != (ret = get_layout_exists(exists))) { + if (OMPI_SUCCESS != (ret = get_layout_ompi_bound(ompi_bound)) || + OMPI_SUCCESS != (ret = get_layout_current_binding(current_binding)) || + OMPI_SUCCESS != (ret = get_layout_exists(exists))) { return OMPI_ERRHANDLER_INVOKE(MPI_COMM_WORLD, ret, FUNC_NAME); } break; @@ -72,165 +88,554 @@ int OMPI_Affinity_str(ompi_affinity_fmt_t fmt_type, return MPI_SUCCESS; } +/*------------------------------------------------------------------------------*/ + +/* + * Turn an int bitmap to a "a-b,c" range kind of string + */ +static char *bitmap2rangestr(int bitmap) +{ + size_t i; + int range_start, range_end; + bool first, isset; + char tmp[BUFSIZ]; + const int stmp = sizeof(tmp) - 1; + static char ret[BUFSIZ]; + + memset(ret, 0, sizeof(ret)); + + first = true; + range_start = -999; + for (i = 0; i < sizeof(int) * 8; ++i) { + isset = (bitmap & (1 << i)); + + /* Do we have a running range? */ + if (range_start >= 0) { + if (isset) { + continue; + } else { + /* A range just ended; output it */ + if (!first) { + strncat(ret, ",", sizeof(ret) - strlen(ret)); + first = false; + } + + range_end = i - 1; + if (range_start == range_end) { + snprintf(tmp, stmp, "%d", range_start); + } else { + snprintf(tmp, stmp, "%d-%d", range_start, range_end); + } + strncat(ret, tmp, sizeof(ret) - strlen(ret)); + + range_start = -999; + } + } + + /* No running range */ + else { + if (isset) { + range_start = i; + } + } + } + + /* If we ended the bitmap with a range open, output it */ + if (range_start >= 0) { + if (!first) { + strncat(ret, ",", sizeof(ret) - strlen(ret)); + first = false; + } + + range_end = i - 1; + if (range_start == range_end) { + snprintf(tmp, stmp, "%d", range_start); + } else { + snprintf(tmp, stmp, "%d-%d", range_start, range_end); + } + strncat(ret, tmp, sizeof(ret) - strlen(ret)); + } + + return ret; +} + +/* + * Make a map of socket/core/hwthread tuples + */ +static int build_map(int *num_sockets_arg, int *num_cores_arg, + hwloc_cpuset_t cpuset, int ***map) +{ + static int num_sockets = -1, num_cores = -1; + int socket_index, core_index, pu_index; + hwloc_obj_t socket, core, pu; + int **data; + + /* Find out how many sockets we have (cached so that we don't have + to look this up every time) */ + if (num_sockets < 0) { + num_sockets = hwloc_get_nbobjs_by_type(opal_hwloc_topology, HWLOC_OBJ_CORE); + + /* Lazy: take the total number of cores that we have in the + topology; that'll be less than the max number of cores + under any given socket */ + num_cores = hwloc_get_nbobjs_by_type(opal_hwloc_topology, HWLOC_OBJ_CORE); + } + *num_sockets_arg = num_sockets; + *num_cores_arg = num_cores; + + /* Alloc a 2D array: sockets x cores. */ + data = malloc(num_sockets * sizeof(int *)); + if (NULL == data) { + return OMPI_ERR_OUT_OF_RESOURCE; + } + data[0] = calloc(num_sockets * num_cores, sizeof(int)); + if (NULL == data[0]) { + free(data); + return OMPI_ERR_OUT_OF_RESOURCE; + } + for (socket_index = 1; socket_index < num_sockets; ++socket_index) { + data[socket_index] = data[socket_index - 1] + num_cores; + } + + /* Iterate the PUs in this cpuset; fill in the data[][] array with + the socket/core/pu triples */ + for (pu_index = 0, + pu = hwloc_get_obj_inside_cpuset_by_type(opal_hwloc_topology, + cpuset, HWLOC_OBJ_PU, + pu_index); + NULL != pu; + pu = hwloc_get_obj_inside_cpuset_by_type(opal_hwloc_topology, + cpuset, HWLOC_OBJ_PU, + ++pu_index)) { + /* Go upward and find the core this PU belongs to */ + core = pu; + while (NULL != core && core->type != HWLOC_OBJ_CORE) { + core = core->parent; + } + core_index = 0; + if (NULL != core) { + core_index = core->os_index; + } + + /* Go upward and find the socket this PU belongs to */ + socket = pu; + while (NULL != socket && socket->type != HWLOC_OBJ_SOCKET) { + socket = socket->parent; + } + socket_index = 0; + if (NULL != socket) { + socket_index = socket->os_index; + } + + /* Save this socket/core/pu combo. LAZY: Assuming that we + won't have more PU's per core than (sizeof(int)*8). */ + data[socket_index][core_index] |= (1 << pu->sibling_rank); + } + + *map = data; + return OMPI_SUCCESS; +} + +/* + * Make a prettyprint string for a hwloc_cpuset_t + */ +static int cset2str(char *str, int len, hwloc_cpuset_t cpuset) +{ + bool first; + int num_sockets, num_cores; + int ret, socket_index, core_index; + char tmp[BUFSIZ]; + const int stmp = sizeof(tmp) - 1; + int **map; + + str[0] = tmp[stmp] = '\0'; + + if (OMPI_SUCCESS != (ret = build_map(&num_sockets, &num_cores, cpuset, &map))) { + return ret; + } + + /* Iterate over the data maxtrix and build up the string */ + first = true; + for (socket_index = 0; socket_index < num_sockets; ++socket_index) { + for (core_index = 0; core_index < num_cores; ++core_index) { + if (map[socket_index][core_index] > 0) { + if (!first) { + strncat(str, ", ", len - strlen(str)); + } + first = false; + + snprintf(tmp, stmp, "socket %d[core %d[hwt %s]]", + socket_index, core_index, + bitmap2rangestr(map[socket_index][core_index])); + strncat(str, tmp, len - strlen(str)); + } + } + } + free(map[0]); + free(map); + + return OMPI_SUCCESS; +} + +/** + * Make a prettyprint string for a cset in a map format. + * Example: [B./..] + * Key: [] - signifies socket + * / - signifies core + * . - signifies PU a process not bound to + * B - signifies PU a process is bound to + */ +static int cset2mapstr(char *str, int len, hwloc_cpuset_t cpuset) +{ + char tmp[BUFSIZ]; + int core_index, pu_index; + const int stmp = sizeof(tmp) - 1; + hwloc_obj_t socket, core, pu; + + str[0] = tmp[stmp] = '\0'; + + /* Iterate over all existing sockets */ + for (socket = hwloc_get_obj_by_type(opal_hwloc_topology, + HWLOC_OBJ_SOCKET, 0); + NULL != socket; + socket = socket->next_cousin) { + strncat(str, "[", len - strlen(str)); + + /* Iterate over all existing cores in this socket */ + core_index = 0; + for (core = hwloc_get_obj_inside_cpuset_by_type(opal_hwloc_topology, + socket->cpuset, + HWLOC_OBJ_CORE, core_index); + NULL != core; + core = hwloc_get_obj_inside_cpuset_by_type(opal_hwloc_topology, + socket->cpuset, + HWLOC_OBJ_CORE, ++core_index)) { + if (core_index > 0) { + strncat(str, "/", len - strlen(str)); + } + + /* Iterate over all existing PUs in this core */ + pu_index = 0; + for (pu = hwloc_get_obj_inside_cpuset_by_type(opal_hwloc_topology, + core->cpuset, + HWLOC_OBJ_PU, pu_index); + NULL != pu; + pu = hwloc_get_obj_inside_cpuset_by_type(opal_hwloc_topology, + core->cpuset, + HWLOC_OBJ_PU, ++pu_index)) { + + /* Is this PU in the cpuset? */ + if (hwloc_bitmap_isset(cpuset, pu->os_index)) { + strncat(str, "B", len - strlen(str)); + } else { + strncat(str, ".", len - strlen(str)); + } + } + } + strncat(str, "]", len - strlen(str)); + } + + return OMPI_SUCCESS; +} + +/*------------------------------------------------------------------------------*/ + +/* + * Where did OMPI bind this process? (prettyprint) + */ static int get_rsrc_ompi_bound(char str[OMPI_AFFINITY_STRING_MAX]) { int ret; - opal_paffinity_base_cpu_set_t cset; /* If OMPI did not bind, indicate that */ - if (!opal_paffinity_base_bound) { - const char tmp[] = "Open MPI did not bind this process"; + if (!orte_proc_is_bound) { + const char tmp[] = "This process is not bound"; strncpy(str, tmp, OMPI_AFFINITY_STRING_MAX - 1); - return OPAL_SUCCESS; + return OMPI_SUCCESS; } - /* Find out what OMPI bound us to and prettyprint it */ - ret = - opal_paffinity_base_parse_binding(opal_paffinity_base_applied_binding, - &cset); - if (OPAL_SUCCESS != ret) { - return ret; - } - - return opal_paffinity_base_cset2str(str, OMPI_AFFINITY_STRING_MAX, &cset); + ret = cset2str(str, OMPI_AFFINITY_STRING_MAX, orte_proc_applied_binding); + return ret; } + +/* + * Where is this process currently bound? (prettyprint) + */ static int get_rsrc_current_binding(char str[OMPI_AFFINITY_STRING_MAX]) { - int ret, flag; - opal_paffinity_base_cpu_set_t cset; + int ret; + hwloc_obj_t root; + hwloc_cpuset_t boundset, rootset; + bool bound = false; - /* Get our binding */ - ret = opal_paffinity_base_get(&cset); - if (OPAL_SUCCESS != ret) { - return ret; + /* get our root object */ + root = hwloc_get_root_obj(opal_hwloc_topology); + rootset = opal_hwloc_base_get_available_cpus(opal_hwloc_topology, root); + + /* get our bindings */ + boundset = hwloc_bitmap_alloc(); + if (hwloc_get_cpubind(opal_hwloc_topology, boundset, + HWLOC_CPUBIND_PROCESS) < 0) { + /* we are NOT bound if get_cpubind fails, nor can we be bound + - the environment does not support it */ + bound = false; + } else { + /* we are bound if the two cpusets are not equal, or if there + is only ONE PU available to us */ + if (0 != hwloc_bitmap_compare(boundset, rootset) || + opal_hwloc_base_single_cpu(rootset) || + opal_hwloc_base_single_cpu(boundset)) { + bound = true; + } } - /* Are we bound anywhere? */ - OPAL_PAFFINITY_PROCESS_IS_BOUND(cset, &flag); - if (!flag) { + /* If we are not bound, indicate that */ + if (!bound) { const char tmp[] = "Not bound (or bound to all available processors)"; strncat(str, tmp, OMPI_AFFINITY_STRING_MAX - 1); - return OPAL_SUCCESS; + ret = OMPI_SUCCESS; } - return opal_paffinity_base_cset2str(str, OMPI_AFFINITY_STRING_MAX, &cset); + /* If we are bound, print it out */ + else { + ret = cset2str(str, OMPI_AFFINITY_STRING_MAX, boundset); + } + hwloc_bitmap_free(boundset); + + return OMPI_SUCCESS; } -/* Prettyprint a list of all available processors */ +/* + * Prettyprint a list of all available sockets and cores. Note that + * this is *everything* -- not just the ones that are available to + * this process. + */ static int get_rsrc_exists(char str[OMPI_AFFINITY_STRING_MAX]) { - int ret, i, num_sockets, num_cores; + bool first = true; + int i, num_cores, num_pus; char tmp[BUFSIZ]; const int stmp = sizeof(tmp) - 1; + hwloc_obj_t socket, core, c2; - tmp[stmp] = '\0'; - - /* Loop over the number of sockets in this machine */ - ret = opal_paffinity_base_get_socket_info(&num_sockets); - if (OPAL_SUCCESS != ret) { - return ret; - } - for (i = 0; i < num_sockets; ++i) { - if (i > 0) { - strncat(str, ", ", OMPI_AFFINITY_STRING_MAX - strlen(str)); + str[0] = '\0'; + for (socket = hwloc_get_obj_by_type(opal_hwloc_topology, + HWLOC_OBJ_SOCKET, 0); + NULL != socket; socket = socket->next_cousin) { + /* If this isn't the first socket, add a delimiter */ + if (!first) { + strncat(str, "; ", OMPI_AFFINITY_STRING_MAX - strlen(str)); } - snprintf(tmp, stmp, "socket %d has ", i); + first = false; + + snprintf(tmp, stmp, "socket %d has ", socket->os_index); strncat(str, tmp, OMPI_AFFINITY_STRING_MAX - strlen(str)); - /* Loop over the number of cores in this socket */ - ret = opal_paffinity_base_get_core_info(i, &num_cores); - if (OPAL_SUCCESS != ret) { - return ret; - } - if (1 == num_cores) { - strncat(str, "1 core", OMPI_AFFINITY_STRING_MAX - strlen(str)); - } else { - snprintf(tmp, stmp, "%d cores", num_cores); - strncat(str, tmp, OMPI_AFFINITY_STRING_MAX - strlen(str)); + /* Find out how many cores are inside this socket, and get an + object pointing to the first core. Also count how many PUs + are in the first core. */ + num_cores = hwloc_get_nbobjs_inside_cpuset_by_type(opal_hwloc_topology, + socket->cpuset, + HWLOC_OBJ_CORE); + core = hwloc_get_obj_inside_cpuset_by_type(opal_hwloc_topology, + socket->cpuset, + HWLOC_OBJ_CORE, 0); + if (NULL != core) { + num_pus = + hwloc_get_nbobjs_inside_cpuset_by_type(opal_hwloc_topology, + core->cpuset, + HWLOC_OBJ_PU); + + /* Only 1 core */ + if (1 == num_cores) { + strncat(str, "1 core with ", + OMPI_AFFINITY_STRING_MAX - strlen(str)); + if (1 == num_pus) { + strncat(str, "1 hwt", + OMPI_AFFINITY_STRING_MAX - strlen(str)); + } else { + snprintf(tmp, stmp, "%d hwts", num_pus); + strncat(str, tmp, OMPI_AFFINITY_STRING_MAX - strlen(str)); + } + } + + /* Multiple cores */ + else { + bool same = true; + + snprintf(tmp, stmp, "%d cores", num_cores); + strncat(str, tmp, OMPI_AFFINITY_STRING_MAX - strlen(str)); + + /* Do all the cores have the same number of PUs? */ + for (c2 = core; NULL != c2; c2 = c2->next_cousin) { + if (hwloc_get_nbobjs_inside_cpuset_by_type(opal_hwloc_topology, + core->cpuset, + HWLOC_OBJ_PU) != + num_pus) { + same = false; + break; + } + } + + /* Yes, they all have the same number of PUs */ + if (same) { + snprintf(tmp, stmp, ", each with %d hwt", num_pus); + strncat(str, tmp, OMPI_AFFINITY_STRING_MAX - strlen(str)); + if (num_pus != 1) { + strncat(str, "s", OMPI_AFFINITY_STRING_MAX - strlen(str)); + } + } + + /* No, they have differing numbers of PUs */ + else { + bool first = true; + + strncat(str, "with (", OMPI_AFFINITY_STRING_MAX - strlen(str)); + for (c2 = core; NULL != c2; c2 = c2->next_cousin) { + if (!first) { + strncat(str, ", ", + OMPI_AFFINITY_STRING_MAX - strlen(str)); + } + first = false; + + i = hwloc_get_nbobjs_inside_cpuset_by_type(opal_hwloc_topology, + core->cpuset, + HWLOC_OBJ_PU); + snprintf(tmp, stmp, "%d", i); + strncat(str, tmp, OMPI_AFFINITY_STRING_MAX - strlen(str)); + } + strncat(str, ") hwts", + OMPI_AFFINITY_STRING_MAX - strlen(str)); + } + } } } - return OPAL_SUCCESS; + return OMPI_SUCCESS; } +/*------------------------------------------------------------------------------*/ + +/* + * Where did OMPI bind this process? (layout string) + */ static int get_layout_ompi_bound(char str[OMPI_AFFINITY_STRING_MAX]) { int ret; - opal_paffinity_base_cpu_set_t cset; /* If OMPI did not bind, indicate that */ - if (!opal_paffinity_base_bound) { - const char tmp[] = "Open MPI did not bind this process"; + if (!orte_proc_is_bound) { + const char tmp[] = "This process is not bound"; strncpy(str, tmp, OMPI_AFFINITY_STRING_MAX - 1); - return OPAL_SUCCESS; + return OMPI_SUCCESS; } /* Find out what OMPI bound us to and prettyprint it */ - ret = - opal_paffinity_base_parse_binding(opal_paffinity_base_applied_binding, - &cset); - if (OPAL_SUCCESS != ret) { - return ret; - } - - return opal_paffinity_base_cset2mapstr(str, OMPI_AFFINITY_STRING_MAX, &cset); + ret = cset2mapstr(str, OMPI_AFFINITY_STRING_MAX, orte_proc_applied_binding); + return ret; } +/* + * Where is this process currently bound? (layout string) + */ static int get_layout_current_binding(char str[OMPI_AFFINITY_STRING_MAX]) { - int ret = OPAL_SUCCESS, flag; - opal_paffinity_base_cpu_set_t cset; + int ret; + hwloc_obj_t root; + hwloc_cpuset_t boundset, rootset; + bool bound = false; - /* Get our binding */ - ret = opal_paffinity_base_get(&cset); - if (OPAL_SUCCESS != ret) { - return ret; + /* get our root object */ + root = hwloc_get_root_obj(opal_hwloc_topology); + rootset = opal_hwloc_base_get_available_cpus(opal_hwloc_topology, root); + + /* get our bindings */ + boundset = hwloc_bitmap_alloc(); + if (hwloc_get_cpubind(opal_hwloc_topology, boundset, + HWLOC_CPUBIND_PROCESS) < 0) { + /* we are NOT bound if get_cpubind fails, nor can we be bound + - the environment does not support it */ + bound = false; + } else { + /* we are bound if the two cpusets are not equal, or if there + is only ONE PU available to us */ + if (0 != hwloc_bitmap_compare(boundset, rootset) || + opal_hwloc_base_single_cpu(rootset) || + opal_hwloc_base_single_cpu(boundset)) { + bound = true; + } } - /* Are we bound anywhere? */ - OPAL_PAFFINITY_PROCESS_IS_BOUND(cset, &flag); - if (!flag) { + /* If we are not bound, indicate that */ + if (!bound) { const char tmp[] = "Not bound (or bound to all available processors)"; strncat(str, tmp, OMPI_AFFINITY_STRING_MAX - 1); - return OPAL_SUCCESS; + ret = OMPI_SUCCESS; } - return opal_paffinity_base_cset2mapstr(str, OMPI_AFFINITY_STRING_MAX, &cset); + /* If we are bound, print it out */ + else { + ret = cset2mapstr(str, OMPI_AFFINITY_STRING_MAX, boundset); + } + hwloc_bitmap_free(boundset); + + return OMPI_SUCCESS; } -/* Prettyprint a list of all available processors in layout format*/ +/* + * Make a layout string of all available sockets and cores. Note that + * this is *everything* -- not just the ones that are available to + * this process. + * + * Example: [../..] + * Key: [] - signifies socket + * / - signifies core + * . - signifies PU + */ static int get_layout_exists(char str[OMPI_AFFINITY_STRING_MAX]) { - int ret, i, j, num_sockets, num_cores; + int core_index, pu_index; int len = OMPI_AFFINITY_STRING_MAX; + hwloc_obj_t socket, core, pu; str[0] = '\0'; - /* Loop over the number of sockets in this machine */ - ret = opal_paffinity_base_get_socket_info(&num_sockets); - if (OPAL_SUCCESS != ret) { - return ret; - } - for (i = 0; i < num_sockets; ++i) { - strncat(str, "[", len - strlen(str)); - /* Loop over the number of cores in this socket */ - ret = opal_paffinity_base_get_core_info(i, &num_cores); - if (OPAL_SUCCESS != ret) { - return ret; + /* Iterate over all existing sockets */ + for (socket = hwloc_get_obj_by_type(opal_hwloc_topology, + HWLOC_OBJ_SOCKET, 0); + NULL != socket; + socket = socket->next_cousin) { + strncat(str, "[", len - strlen(str)); + + /* Iterate over all existing cores in this socket */ + core_index = 0; + for (core = hwloc_get_obj_inside_cpuset_by_type(opal_hwloc_topology, + socket->cpuset, + HWLOC_OBJ_CORE, core_index); + NULL != core; + core = hwloc_get_obj_inside_cpuset_by_type(opal_hwloc_topology, + socket->cpuset, + HWLOC_OBJ_CORE, ++core_index)) { + if (core_index > 0) { + strncat(str, "/", len - strlen(str)); + } + + /* Iterate over all existing PUs in this core */ + pu_index = 0; + for (pu = hwloc_get_obj_inside_cpuset_by_type(opal_hwloc_topology, + core->cpuset, + HWLOC_OBJ_PU, pu_index); + NULL != pu; + pu = hwloc_get_obj_inside_cpuset_by_type(opal_hwloc_topology, + core->cpuset, + HWLOC_OBJ_PU, ++pu_index)) { + strncat(str, ".", len - strlen(str)); + } } - for (j = 0; j < num_cores; j++) { - if (0 < j) { - /* add space after first core is printed */ - strncat(str, " ", len - strlen(str)); - } - - /* mark core exists */ - strncat(str, ".", len - strlen(str)); - } - strncat(str, "]", len - strlen(str)); + strncat(str, "]", len - strlen(str)); } - return OPAL_SUCCESS; + return OMPI_SUCCESS; } diff --git a/ompi/proc/proc.h b/ompi/proc/proc.h index c561d72e24..36d5424c07 100644 --- a/ompi/proc/proc.h +++ b/ompi/proc/proc.h @@ -9,8 +9,8 @@ * University of Stuttgart. All rights reserved. * Copyright (c) 2004-2005 The Regents of the University of California. * All rights reserved. - * Copyright (c) 2006 Cisco Systems, Inc. All rights reserved. - * Copyright (c) 2007 Los Alamos National Security, LLC. All rights + * Copyright (c) 2006-2012 Cisco Systems, Inc. All rights reserved. + * Copyright (c) 2007-2012 Los Alamos National Security, LLC. All rights * reserved. * $COPYRIGHT$ * @@ -35,7 +35,7 @@ #include "ompi/types.h" #include "opal/class/opal_list.h" #include "opal/dss/dss_types.h" -#include "opal/mca/paffinity/paffinity.h" +#include "opal/mca/hwloc/hwloc.h" #include "orte/types.h" @@ -63,7 +63,7 @@ struct ompi_proc_t { /** architecture of this process */ uint32_t proc_arch; /** flags for this proc */ - opal_paffinity_locality_t proc_flags; + opal_hwloc_locality_t proc_flags; /** Base convertor for the proc described by this process */ struct opal_convertor_t* proc_convertor; /** A pointer to the name of this host - data is diff --git a/ompi/runtime/help-mpi-runtime.txt b/ompi/runtime/help-mpi-runtime.txt index 3fa45ea148..1368e0f468 100644 --- a/ompi/runtime/help-mpi-runtime.txt +++ b/ompi/runtime/help-mpi-runtime.txt @@ -10,7 +10,7 @@ # University of Stuttgart. All rights reserved. # Copyright (c) 2004-2005 The Regents of the University of California. # All rights reserved. -# Copyright (c) 2007-2011 Cisco Systems, Inc. All rights reserved. +# Copyright (c) 2007-2012 Cisco Systems, Inc. All rights reserved. # $COPYRIGHT$ # # Additional copyrights may follow @@ -53,16 +53,6 @@ mpi_param_check value has therefore been ignored. WARNING: Cannot set both the MCA parameters mpi_leave_pinned and mpi_leave_pinned_pipeline to "true". Defaulting to mpi_leave_pinned ONLY. -[mpi_init:startup:paffinity-unavailable] -The MCA parameter "opal_paffinity_alone" was set to a nonzero value, -but Open MPI was unable to bind MPI_COMM_WORLD rank %s to a processor. - -Typical causes for this problem include: - - - A node was oversubscribed (more processes than processors), in - which case Open MPI will not bind any processes on that node - - A startup mechanism was used which did not tell Open MPI which - processors to bind processes to # [mpi_finalize:invoked_multiple_times] The function MPI_FINALIZE was invoked multiple times in a single diff --git a/ompi/runtime/ompi_mpi_finalize.c b/ompi/runtime/ompi_mpi_finalize.c index 976e69b66a..fb4a25feb9 100644 --- a/ompi/runtime/ompi_mpi_finalize.c +++ b/ompi/runtime/ompi_mpi_finalize.c @@ -10,8 +10,8 @@ * University of Stuttgart. All rights reserved. * Copyright (c) 2004-2005 The Regents of the University of California. * All rights reserved. - * Copyright (c) 2006-2009 Cisco Systems, Inc. All rights reserved. - * Copyright (c) 2006-2011 Los Alamos National Security, LLC. All rights + * Copyright (c) 2006-2012 Cisco Systems, Inc. All rights reserved. + * Copyright (c) 2006-2012 Los Alamos National Security, LLC. All rights * reserved. * Copyright (c) 2006 University of Houston. All rights reserved. * Copyright (c) 2009 Sun Microsystems, Inc. All rights reserved. @@ -42,7 +42,6 @@ #include "opal/mca/event/event.h" #include "opal/util/output.h" #include "opal/runtime/opal_progress.h" -#include "opal/mca/maffinity/base/base.h" #include "opal/mca/base/base.h" #include "orte/util/show_help.h" #include "opal/sys/atomic.h" @@ -157,11 +156,6 @@ int ompi_mpi_finalize(void) MPI lifetime, to get better latency when not using TCP */ opal_progress_event_users_increment(); - /* If maffinity was setup, tear it down */ - if (opal_maffinity_setup) { - opal_maffinity_base_close(); - } - /* check to see if we want timing information */ mca_base_param_reg_int_name("ompi", "timing", "Request that critical timing loops be measured", diff --git a/ompi/runtime/ompi_mpi_init.c b/ompi/runtime/ompi_mpi_init.c index a6dae3eaf3..08d0f41e98 100644 --- a/ompi/runtime/ompi_mpi_init.c +++ b/ompi/runtime/ompi_mpi_init.c @@ -39,8 +39,6 @@ #include "opal/class/opal_list.h" #include "opal/mca/base/base.h" #include "opal/mca/hwloc/base/base.h" -#include "opal/mca/paffinity/base/base.h" -#include "opal/mca/maffinity/base/base.h" #include "opal/runtime/opal_progress.h" #include "opal/threads/threads.h" #include "opal/util/output.h" @@ -129,8 +127,6 @@ int ompi_mpi_thread_provided = MPI_THREAD_SINGLE; opal_thread_t *ompi_mpi_main_thread = NULL; -bool ompi_mpi_maffinity_setup = false; - bool ompi_warn_on_fork; /* @@ -609,213 +605,6 @@ int ompi_mpi_init(int argc, char **argv, int requested, int *provided) (ompistop.tv_usec - ompistart.tv_usec))); gettimeofday(&ompistart, NULL); } - -#if OPAL_HAVE_HWLOC - { - hwloc_obj_t node, obj; - hwloc_cpuset_t cpus, nodeset; - bool paffinity_enabled=false; - orte_node_rank_t nrank; - hwloc_obj_type_t target; - unsigned int cache_level = 0; - struct hwloc_topology_support *support; - - /* see if we were bound when launched */ - if (NULL == getenv("OMPI_MCA_opal_bound_at_launch")) { - /* we were not bound at launch */ - if (NULL != opal_hwloc_topology) { - support = (struct hwloc_topology_support*)hwloc_topology_get_support(opal_hwloc_topology); - /* get our node object */ - node = hwloc_get_root_obj(opal_hwloc_topology); - nodeset = opal_hwloc_base_get_available_cpus(opal_hwloc_topology, node); - /* get our bindings */ - cpus = hwloc_bitmap_alloc(); - if (hwloc_get_cpubind(opal_hwloc_topology, cpus, HWLOC_CPUBIND_PROCESS) < 0) { - /* we are NOT bound if get_cpubind fails, nor can we be bound - the - * environment does not support it - */ - hwloc_bitmap_free(cpus); - goto MOVEON; - } - /* we are bound if the two cpusets are not equal, - * or if there is only ONE cpu available to us - */ - if (0 != hwloc_bitmap_compare(cpus, nodeset) || - opal_hwloc_base_single_cpu(nodeset) || - opal_hwloc_base_single_cpu(cpus)) { - /* someone external set it - indicate it is set - * so that we know - */ - paffinity_enabled = true; - hwloc_bitmap_free(cpus); - } else if (support->cpubind->set_thisproc_cpubind && - OPAL_BINDING_POLICY_IS_SET(opal_hwloc_binding_policy) && - OPAL_BIND_TO_NONE != OPAL_GET_BINDING_POLICY(opal_hwloc_binding_policy)) { - /* the system is capable of doing processor affinity, but it - * has not yet been set - see if a slot_list was given - */ - hwloc_bitmap_zero(cpus); - if (OPAL_BIND_TO_CPUSET == OPAL_GET_BINDING_POLICY(opal_hwloc_binding_policy)) { - if (ORTE_SUCCESS != (ret = opal_hwloc_base_slot_list_parse(opal_hwloc_base_slot_list, - opal_hwloc_topology, cpus))) { - error = "Setting processor affinity failed"; - hwloc_bitmap_free(cpus); - goto error; - } - if (0 > hwloc_set_cpubind(opal_hwloc_topology, cpus, 0)) { - error = "Setting processor affinity failed"; - hwloc_bitmap_free(cpus); - goto error; - } - /* try to find a level and index for this location */ - opal_hwloc_base_get_level_and_index(cpus, &orte_process_info.bind_level, &orte_process_info.bind_idx); - /* cleanup */ - hwloc_bitmap_free(cpus); - paffinity_enabled = true; - } else { - /* cleanup */ - hwloc_bitmap_free(cpus); - /* get the node rank */ - if (ORTE_NODE_RANK_INVALID == (nrank = orte_ess.get_node_rank(ORTE_PROC_MY_NAME))) { - /* this is not an error - could be due to being - * direct launched - so just ignore and leave - * us unbound - */ - goto MOVEON; - } - /* if the binding policy is hwthread, then we bind to the nrank-th - * hwthread on this node - */ - if (OPAL_BIND_TO_HWTHREAD == OPAL_GET_BINDING_POLICY(opal_hwloc_binding_policy)) { - if (NULL == (obj = opal_hwloc_base_get_obj_by_type(opal_hwloc_topology, HWLOC_OBJ_PU, - 0, nrank, OPAL_HWLOC_LOGICAL))) { - ret = OMPI_ERR_NOT_FOUND; - error = "Getting hwthread object"; - goto error; - } - cpus = opal_hwloc_base_get_available_cpus(opal_hwloc_topology, obj); - if (0 > hwloc_set_cpubind(opal_hwloc_topology, cpus, 0)) { - ret = OMPI_ERROR; - error = "Setting processor affinity failed"; - goto error; - } - orte_process_info.bind_level = OPAL_HWLOC_L1CACHE_LEVEL; - orte_process_info.bind_idx = nrank; - } else if (OPAL_BIND_TO_CORE == OPAL_GET_BINDING_POLICY(opal_hwloc_binding_policy)) { - /* if the binding policy is core, then we bind to the nrank-th - * core on this node - */ - if (NULL == (obj = opal_hwloc_base_get_obj_by_type(opal_hwloc_topology, HWLOC_OBJ_CORE, - 0, nrank, OPAL_HWLOC_LOGICAL))) { - ret = OMPI_ERR_NOT_FOUND; - error = "Getting core object"; - goto error; - } - cpus = opal_hwloc_base_get_available_cpus(opal_hwloc_topology, obj); - if (0 > hwloc_set_cpubind(opal_hwloc_topology, cpus, 0)) { - error = "Setting processor affinity failed"; - ret = OMPI_ERROR; - goto error; - } - orte_process_info.bind_level = OPAL_HWLOC_CORE_LEVEL; - orte_process_info.bind_idx = nrank; - } else { - /* for all higher binding policies, we bind to the specified - * object that the nrank-th core belongs to - */ - if (NULL == (obj = opal_hwloc_base_get_obj_by_type(opal_hwloc_topology, HWLOC_OBJ_CORE, - 0, nrank, OPAL_HWLOC_LOGICAL))) { - ret = OMPI_ERR_NOT_FOUND; - error = "Getting core object"; - goto error; - } - if (OPAL_BIND_TO_L1CACHE == OPAL_GET_BINDING_POLICY(opal_hwloc_binding_policy)) { - target = HWLOC_OBJ_CACHE; - cache_level = 1; - orte_process_info.bind_level = OPAL_HWLOC_L1CACHE_LEVEL; - } else if (OPAL_BIND_TO_L2CACHE == OPAL_GET_BINDING_POLICY(opal_hwloc_binding_policy)) { - target = HWLOC_OBJ_CACHE; - cache_level = 2; - orte_process_info.bind_level = OPAL_HWLOC_L2CACHE_LEVEL; - } else if (OPAL_BIND_TO_L3CACHE == OPAL_GET_BINDING_POLICY(opal_hwloc_binding_policy)) { - target = HWLOC_OBJ_CACHE; - cache_level = 3; - orte_process_info.bind_level = OPAL_HWLOC_L3CACHE_LEVEL; - } else if (OPAL_BIND_TO_SOCKET == OPAL_GET_BINDING_POLICY(opal_hwloc_binding_policy)) { - target = HWLOC_OBJ_SOCKET; - orte_process_info.bind_level = OPAL_HWLOC_SOCKET_LEVEL; - } else if (OPAL_BIND_TO_NUMA == OPAL_GET_BINDING_POLICY(opal_hwloc_binding_policy)) { - target = HWLOC_OBJ_NODE; - orte_process_info.bind_level = OPAL_HWLOC_NUMA_LEVEL; - } else { - ret = OMPI_ERR_NOT_FOUND; - error = "Binding policy not known"; - goto error; - } - for (obj = obj->parent; NULL != obj; obj = obj->parent) { - if (target == obj->type) { - if (HWLOC_OBJ_CACHE == target && cache_level != obj->attr->cache.depth) { - continue; - } - /* this is the place! */ - cpus = opal_hwloc_base_get_available_cpus(opal_hwloc_topology, obj); - if (0 > hwloc_set_cpubind(opal_hwloc_topology, cpus, 0)) { - ret = OMPI_ERROR; - error = "Setting processor affinity failed"; - goto error; - } - orte_process_info.bind_idx = opal_hwloc_base_get_obj_idx(opal_hwloc_topology, - obj, OPAL_HWLOC_LOGICAL); - paffinity_enabled = true; - break; - } - } - if (!paffinity_enabled) { - ret = OMPI_ERROR; - error = "Setting processor affinity failed"; - goto error; - } - } - paffinity_enabled = true; - } - } - /* If we were able to set processor affinity, try setting up - memory affinity */ - if (!opal_maffinity_setup && paffinity_enabled) { - if (OPAL_SUCCESS == opal_maffinity_base_open() && - OPAL_SUCCESS == opal_maffinity_base_select()) { - opal_maffinity_setup = true; - } - } - } - } - } - -MOVEON: - /* get or update our local cpuset - it will get used multiple - * times, so it's more efficient to keep a global copy - */ - opal_hwloc_base_get_local_cpuset(); - /* report bindings, if requested */ - if (opal_hwloc_report_bindings) { - char bindings[64]; - hwloc_obj_t root; - hwloc_cpuset_t cpus; - /* get the root object for this node */ - root = hwloc_get_root_obj(opal_hwloc_topology); - cpus = opal_hwloc_base_get_available_cpus(opal_hwloc_topology, root); - /* we are not bound if this equals our cpuset */ - if (0 == hwloc_bitmap_compare(cpus, opal_hwloc_my_cpuset)) { - opal_output(0, "%s is not bound", - ORTE_NAME_PRINT(ORTE_PROC_MY_NAME)); - } else { - hwloc_bitmap_list_snprintf(bindings, 64, opal_hwloc_my_cpuset); - opal_output(0, "%s is bound to cpus %s", - ORTE_NAME_PRINT(ORTE_PROC_MY_NAME), - bindings); - } - } -#endif /* select buffered send allocator component to be used */ ret=mca_pml_base_bsend_init(OMPI_ENABLE_THREAD_MULTIPLE); diff --git a/ompi/tools/ompi_info/components.c b/ompi/tools/ompi_info/components.c index ad3cfe6822..da64649222 100644 --- a/ompi/tools/ompi_info/components.c +++ b/ompi/tools/ompi_info/components.c @@ -9,8 +9,8 @@ * University of Stuttgart. All rights reserved. * Copyright (c) 2004-2005 The Regents of the University of California. * All rights reserved. - * Copyright (c) 2006-2010 Cisco Systems, Inc. All rights reserved. - * Copyright (c) 2010-2011 Los Alamos National Security, LLC. + * Copyright (c) 2006-2012 Cisco Systems, Inc. All rights reserved. + * Copyright (c) 2010-2012 Los Alamos National Security, LLC. * All rights reserved. * Copyright (c) 2011 University of Houston. All rights reserved. * $COPYRIGHT$ @@ -35,14 +35,8 @@ #include "opal/mca/base/base.h" #include "opal/mca/backtrace/backtrace.h" #include "opal/mca/backtrace/base/base.h" -#include "opal/mca/paffinity/paffinity.h" -#include "opal/mca/paffinity/base/base.h" -#include "opal/mca/carto/carto.h" -#include "opal/mca/carto/base/base.h" #include "opal/mca/shmem/shmem.h" #include "opal/mca/shmem/base/base.h" -#include "opal/mca/maffinity/maffinity.h" -#include "opal/mca/maffinity/base/base.h" #include "opal/mca/memory/memory.h" #include "opal/mca/memory/base/base.h" #include "opal/mca/memchecker/memchecker.h" @@ -308,22 +302,6 @@ void ompi_info_open_components(void) map->components = &opal_memchecker_base_components_opened; opal_pointer_array_add(&component_map, map); - if (OPAL_SUCCESS != opal_paffinity_base_open()) { - goto error; - } - map = OBJ_NEW(ompi_info_component_map_t); - map->type = strdup("paffinity"); - map->components = &opal_paffinity_base_components_opened; - opal_pointer_array_add(&component_map, map); - - if (OPAL_SUCCESS != opal_carto_base_open()) { - goto error; - } - map = OBJ_NEW(ompi_info_component_map_t); - map->type = strdup("carto"); - map->components = &opal_carto_base_components_opened; - opal_pointer_array_add(&component_map, map); - if (OPAL_SUCCESS != opal_shmem_base_open()) { goto error; } @@ -332,14 +310,6 @@ void ompi_info_open_components(void) map->components = &opal_shmem_base_components_opened; opal_pointer_array_add(&component_map, map); - if (OPAL_SUCCESS != opal_maffinity_base_open()) { - goto error; - } - map = OBJ_NEW(ompi_info_component_map_t); - map->type = strdup("maffinity"); - map->components = &opal_maffinity_base_components_opened; - opal_pointer_array_add(&component_map, map); - #if OPAL_HAVE_HWLOC if (OPAL_SUCCESS != opal_hwloc_base_open()) { goto error; @@ -804,9 +774,6 @@ void ompi_info_close_components() (void) opal_backtrace_base_close(); (void) opal_memory_base_close(); (void) opal_memchecker_base_close(); - (void) opal_paffinity_base_close(); - (void) opal_carto_base_close(); - (void) opal_maffinity_base_close(); (void) opal_timer_base_close(); #if OPAL_HAVE_HWLOC (void) opal_hwloc_base_close(); diff --git a/ompi/tools/ompi_info/ompi_info.c b/ompi/tools/ompi_info/ompi_info.c index 2111d61a2a..58f01cfb66 100644 --- a/ompi/tools/ompi_info/ompi_info.c +++ b/ompi/tools/ompi_info/ompi_info.c @@ -210,10 +210,7 @@ int main(int argc, char *argv[]) opal_pointer_array_add(&mca_types, "backtrace"); opal_pointer_array_add(&mca_types, "memchecker"); opal_pointer_array_add(&mca_types, "memory"); - opal_pointer_array_add(&mca_types, "paffinity"); - opal_pointer_array_add(&mca_types, "carto"); opal_pointer_array_add(&mca_types, "shmem"); - opal_pointer_array_add(&mca_types, "maffinity"); opal_pointer_array_add(&mca_types, "timer"); opal_pointer_array_add(&mca_types, "installdirs"); opal_pointer_array_add(&mca_types, "hwloc"); diff --git a/opal/include/opal/constants.h b/opal/include/opal/constants.h index 32f054b639..ed9b3aeeba 100644 --- a/opal/include/opal/constants.h +++ b/opal/include/opal/constants.h @@ -70,7 +70,8 @@ enum { OPAL_ERR_SLOT_LIST_RANGE = (OPAL_ERR_BASE - 41), OPAL_ERR_NETWORK_NOT_PARSEABLE = (OPAL_ERR_BASE - 42), OPAL_ERR_SILENT = (OPAL_ERR_BASE - 43), - OPAL_ERR_NOT_INITIALIZED = (OPAL_ERR_BASE - 44) + OPAL_ERR_NOT_INITIALIZED = (OPAL_ERR_BASE - 44), + OPAL_ERR_NOT_BOUND = (OPAL_ERR_BASE - 45) }; #define OPAL_ERR_MAX (OPAL_ERR_BASE - 100) diff --git a/opal/mca/carto/Makefile.am b/opal/mca/carto/Makefile.am deleted file mode 100644 index f6710a0f07..0000000000 --- a/opal/mca/carto/Makefile.am +++ /dev/null @@ -1,37 +0,0 @@ -# -# Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana -# University Research and Technology -# Corporation. All rights reserved. -# Copyright (c) 2004-2005 The University of Tennessee and The University -# of Tennessee Research Foundation. All rights -# reserved. -# Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, -# University of Stuttgart. All rights reserved. -# Copyright (c) 2004-2005 The Regents of the University of California. -# All rights reserved. -# Copyright (c) 2007-2010 Cisco Systems, Inc. All rights reserved. -# $COPYRIGHT$ -# -# Additional copyrights may follow -# -# $HEADER$ -# - -# main library setup -noinst_LTLIBRARIES = libmca_carto.la -libmca_carto_la_SOURCES = - -# local files -headers = carto.h -libmca_carto_la_SOURCES += $(headers) - -# Conditionally install the header files -if WANT_INSTALL_HEADERS -opaldir = $(includedir)/openmpi/$(subdir) -nobase_opal_HEADERS = $(headers) -endif - -include base/Makefile.am - -distclean-local: - rm -f base/static-components.h diff --git a/opal/mca/carto/auto_detect/.windows b/opal/mca/carto/auto_detect/.windows deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/opal/mca/carto/auto_detect/Makefile.am b/opal/mca/carto/auto_detect/Makefile.am deleted file mode 100644 index f536b23347..0000000000 --- a/opal/mca/carto/auto_detect/Makefile.am +++ /dev/null @@ -1,47 +0,0 @@ -# -# Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana -# University Research and Technology -# Corporation. All rights reserved. -# Copyright (c) 2004-2005 The University of Tennessee and The University -# of Tennessee Research Foundation. All rights -# reserved. -# Copyright (c) 2004-2009 High Performance Computing Center Stuttgart, -# University of Stuttgart. All rights reserved. -# Copyright (c) 2004-2005 The Regents of the University of California. -# All rights reserved. -# Copyright (c) 2007-2010 Cisco Systems, Inc. All rights reserved. -# $COPYRIGHT$ -# -# Additional copyrights may follow -# -# $HEADER$ -# - -EXTRA_DIST = .windows - -sources = \ - carto_auto_detect.h \ - carto_auto_detect_component.c \ - carto_auto_detect_module.c - -# Make the output library in this directory, and name it either -# mca__.la (for DSO builds) or libmca__.la -# (for static builds). - -if MCA_BUILD_opal_carto_auto_detect_DSO -component_noinst = -component_install = mca_carto_auto_detect.la -else -component_noinst = libmca_carto_auto_detect.la -component_install = -endif - -mcacomponentdir = $(pkglibdir) -mcacomponent_LTLIBRARIES = $(component_install) -mca_carto_auto_detect_la_SOURCES = $(sources) -mca_carto_auto_detect_la_LDFLAGS = -module -avoid-version - -noinst_LTLIBRARIES = $(component_noinst) -libmca_carto_auto_detect_la_SOURCES =$(sources) -libmca_carto_auto_detect_la_LDFLAGS = -module -avoid-version - diff --git a/opal/mca/carto/auto_detect/carto_auto_detect.h b/opal/mca/carto/auto_detect/carto_auto_detect.h deleted file mode 100644 index f71640b1ee..0000000000 --- a/opal/mca/carto/auto_detect/carto_auto_detect.h +++ /dev/null @@ -1,58 +0,0 @@ -/* - * Copyright (c) 2004-2008 The Trustees of Indiana University and Indiana - * University Research and Technology - * Corporation. All rights reserved. - * Copyright (c) 2004-2005 The University of Tennessee and The University - * of Tennessee Research Foundation. All rights - * reserved. - * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, - * University of Stuttgart. All rights reserved. - * Copyright (c) 2004-2005 The Regents of the University of California. - * All rights reserved. - * Copyright (c) 2006-2007 Cisco Systems, Inc. All rights reserved. - * - * $COPYRIGHT$ - * - * Additional copyrights may follow - * - * $HEADER$ - */ - -/** - * @file - * - * The auto detect component discover automaticly the structure - * of the host. - */ - -#ifndef MCA_CARTO_AUTO_DETECT_H -#define MCA_CARTO_AUTO_DETECT_H - -#include "opal_config.h" - -#include "opal/mca/mca.h" -#include "opal/mca/carto/carto.h" - -BEGIN_C_DECLS - -/** - * Globally exported variable - */ -OPAL_DECLSPEC extern const opal_carto_base_component_2_0_0_t -mca_carto_auto_detect_component; - - -/** - * carto query API function - * - * Query function for carto components. Simply returns a priority - * to rank it against other available carto components (assumedly, - * only one component will be available per platform, but it's - * possible that there could be more than one available). - */ -int opal_carto_auto_detect_component_query(mca_base_module_t **module, int *priority); - -END_C_DECLS - -#endif /* MCA_CARTO_FILE_EXPORT_H */ - diff --git a/opal/mca/carto/auto_detect/carto_auto_detect_component.c b/opal/mca/carto/auto_detect/carto_auto_detect_component.c deleted file mode 100644 index 10d6938776..0000000000 --- a/opal/mca/carto/auto_detect/carto_auto_detect_component.c +++ /dev/null @@ -1,83 +0,0 @@ -/* - * Copyright (c) 2004-2008 The Trustees of Indiana University and Indiana - * University Research and Technology - * Corporation. All rights reserved. - * Copyright (c) 2004-2005 The University of Tennessee and The University - * of Tennessee Research Foundation. All rights - * reserved. - * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, - * University of Stuttgart. All rights reserved. - * Copyright (c) 2004-2005 The Regents of the University of California. - * All rights reserved. - * Copyright (c) 2007 Cisco Systems, Inc. All rights reserved. - * $COPYRIGHT$ - * - * Additional copyrights may follow - * - * $HEADER$ - * - * These symbols are in a file by themselves to provide nice linker - * semantics. Since linkers generally pull in symbols by object - * files, keeping these symbols as the only symbols in this file - * prevents utility programs such as "ompi_info" from having to import - * entire components just to query their version and parameters. - */ - -#include "opal_config.h" - -#include "opal/constants.h" -#include "opal/mca/carto/carto.h" -#include "carto_auto_detect.h" - -/* - * Public string showing the carto ompi_file component version number - */ -const char *opal_carto_auto_detect_component_version_string = - "OPAL auto detect carto MCA component version " OPAL_VERSION; - -/* - * Local function - */ -static int auto_detect_open(void); - -/* - * Instantiate the public struct with all of our public information - * and pointers to our public functions in it - */ - -const opal_carto_base_component_2_0_0_t mca_carto_auto_detect_component = { - - /* First, the mca_component_t struct containing meta information - about the component itself */ - - { - OPAL_CARTO_BASE_VERSION_2_0_0, - - /* Component name and version */ - "auto_detect", - OPAL_MAJOR_VERSION, - OPAL_MINOR_VERSION, - OPAL_RELEASE_VERSION, - - /* Component open and close functions */ - auto_detect_open, - NULL, - opal_carto_auto_detect_component_query - }, - { - /* The component is checkpoint ready */ - MCA_BASE_METADATA_PARAM_CHECKPOINT - } -}; - - -static int auto_detect_open(void) -{ - mca_base_param_reg_int(&mca_carto_auto_detect_component.base_version, - "priority", - "Priority of the auto_detect carto component", - false, false, 11, NULL); - - return OPAL_SUCCESS; -} - diff --git a/opal/mca/carto/auto_detect/carto_auto_detect_module.c b/opal/mca/carto/auto_detect/carto_auto_detect_module.c deleted file mode 100644 index 2f09ad2721..0000000000 --- a/opal/mca/carto/auto_detect/carto_auto_detect_module.c +++ /dev/null @@ -1,107 +0,0 @@ -/* - * Copyright (c) 2004-2008 The Trustees of Indiana University and Indiana - * University Research and Technology - * Corporation. All rights reserved. - * Copyright (c) 2004-2005 The University of Tennessee and The University - * of Tennessee Research Foundation. All rights - * reserved. - * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, - * University of Stuttgart. All rights reserved. - * Copyright (c) 2004-2005 The Regents of the University of California. - * All rights reserved. - * Copyright (c) 2006-2007 Cisco Systems, Inc. All rights reserved. - * - * $COPYRIGHT$ - * - * Additional copyrights may follow - * - * $HEADER$ - */ - -#include "opal_config.h" - -/* This component will only be compiled on File, where we are - guaranteed to have and friends */ -#include - -#ifdef HAVE_UNISTD_H -#include -#endif /* HAVE_UNISTD_H */ -#ifdef HAVE_STDLIB_H -#include -#endif /* HAVE_STDLIB_H */ -#ifdef HAVE_STRING_H -#include -#endif /* HAVE_STRING_H */ - -#include "opal/constants.h" -#include "opal/mca/base/mca_base_param.h" -#include "opal/mca/carto/carto.h" -#include "opal/mca/carto/base/base.h" -#include "opal/mca/carto/base/carto_base_graph.h" -#include "carto_auto_detect.h" - - -static int opal_carto_auto_detect_init(void); -static int opal_carto_auto_detect_finalize(void); - - -/* - * Auto detect carto module - */ -static const opal_carto_base_module_1_0_0_t loc_module = { - opal_carto_auto_detect_init, - opal_carto_base_graph_get_host_graph_fn, - opal_carto_base_free_graph_fn, - opal_carto_base_get_nodes_distance_fn, - opal_carto_base_graph_spf_fn, - opal_carto_base_graph_find_node_fn, - opal_carto_auto_detect_finalize, -}; - -int opal_carto_auto_detect_component_query(mca_base_module_t **module, int *priority) -{ - int param; - - param = mca_base_param_find("carto", "auto_detect", "priority"); - mca_base_param_lookup_int(param, priority); - - *module = (mca_base_module_t *)&loc_module; - - return OPAL_SUCCESS; -} - - -/** - * Init the auto detect module. this function build an empty - * carto grap and does nothing more. to comleate this module, - * this function should read the system files and fill the - * carto graph - * - * @return int success or error - */ -static int opal_carto_auto_detect_init(void) -{ - /* create an empty graph */ - if (NULL == opal_carto_base_common_host_graph) { - opal_carto_base_graph_create_fn(&opal_carto_base_common_host_graph); - } - return OPAL_SUCCESS; -} - -/** - * Cleans the auto detect module. - * - * @return int success or error - */ -static int opal_carto_auto_detect_finalize(void) -{ - /* free the host cartography graph. */ - if (NULL != opal_carto_base_common_host_graph) { - opal_carto_base_free_graph_fn(opal_carto_base_common_host_graph); - } - return OPAL_SUCCESS; -} - - - diff --git a/opal/mca/carto/base/Makefile.am b/opal/mca/carto/base/Makefile.am deleted file mode 100644 index cacce2db0a..0000000000 --- a/opal/mca/carto/base/Makefile.am +++ /dev/null @@ -1,29 +0,0 @@ -# -# Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana -# University Research and Technology -# Corporation. All rights reserved. -# Copyright (c) 2004-2005 The University of Tennessee and The University -# of Tennessee Research Foundation. All rights -# reserved. -# Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, -# University of Stuttgart. All rights reserved. -# Copyright (c) 2004-2005 The Regents of the University of California. -# All rights reserved. -# Copyright (c) 2007 Cisco Systems, Inc. All rights reserved. -# $COPYRIGHT$ -# -# Additional copyrights may follow -# -# $HEADER$ -# - -headers += \ - base/base.h \ - base/carto_base_graph.h - -libmca_carto_la_SOURCES += \ - base/carto_base_close.c \ - base/carto_base_select.c \ - base/carto_base_graph.c \ - base/carto_base_wrapers.c \ - base/carto_base_open.c diff --git a/opal/mca/carto/base/base.h b/opal/mca/carto/base/base.h deleted file mode 100644 index 60f81a441a..0000000000 --- a/opal/mca/carto/base/base.h +++ /dev/null @@ -1,202 +0,0 @@ -/* - * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana - * University Research and Technology - * Corporation. All rights reserved. - * Copyright (c) 2004-2006 The University of Tennessee and The University - * of Tennessee Research Foundation. All rights - * reserved. - * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, - * University of Stuttgart. All rights reserved. - * Copyright (c) 2004-2005 The Regents of the University of California. - * All rights reserved. - * Copyright (c) 2007 Cisco Systems, Inc. All rights reserved. - * $COPYRIGHT$ - * - * Additional copyrights may follow - * - * $HEADER$ - * - */ - -#ifndef OPAL_CARTO_BASE_H -#define OPAL_CARTO_BASE_H - -#include "opal_config.h" - -#include "opal/mca/carto/carto.h" - -/* - * Global functions for MCA overall carto open and close - */ - -BEGIN_C_DECLS - -/** - * Initialize the carto MCA framework - * - * @retval OPAL_SUCCESS Upon success - * @retval OPAL_ERROR Upon failure - * - * This must be the first function invoked in the carto MCA - * framework. It initializes the carto MCA framework, finds - * and opens carto components, etc. - * - * This function is invoked during opal_init(). - * - * This function fills in the internal global variable - * opal_carto_base_components_opened, which is a list of all - * carto components that were successfully opened. This - * variable should \em only be used by other carto base - * functions -- it is not considered a public interface member -- - * and is only mentioned here for completeness. - */ -OPAL_DECLSPEC int opal_carto_base_open(void); - -/** - * Select an available component. - * - * @return OPAL_SUCCESS Upon success. - * @return OPAL_NOT_FOUND If no component can be selected. - * @return OPAL_ERROR Upon other failure. - * - * This function invokes the selection process for carto components, - * which works as follows: - * - * - If the \em carto MCA parameter is not specified, the - * selection set is all available carto components. - * - If the \em carto MCA parameter is specified, the - * selection set is just that component. - * - All components in the selection set are queried to see if - * they want to run. All components that want to run are ranked - * by their priority and the highest priority component is - * selected. All non-selected components have their "close" - * function invoked to let them know that they were not selected. - * - The selected component will have its "init" function invoked to - * let it know that it was selected. - * - * If we fall through this entire process and no component is - * selected, then return OPAL_NOT_FOUND (this is not a fatal - * error). - * - * At the end of this process, we'll either have a single - * component that is selected and initialized, or no component was - * selected. If no component was selected, subsequent invocation - * of the carto wrapper functions will return an error. - */ -OPAL_DECLSPEC int opal_carto_base_select(void); - -/** - * Shut down the carto MCA framework. - * - * @retval OPAL_SUCCESS Always - * - * This function shuts down everything in the carto MCA - * framework, and is called during opal_finalize(). - * - * It must be the last function invoked on the carto MCA - * framework. - */ -OPAL_DECLSPEC int opal_carto_base_close(void); - -/** - * Indication of whether a component was successfully selected or - * not - */ -OPAL_DECLSPEC extern bool opal_carto_base_selected; - -/** - * Global component struct for the selected component - */ -OPAL_DECLSPEC extern const opal_carto_base_component_2_0_0_t -*opal_carto_base_component; - -/** - * Global module struct for the selected module - */ -OPAL_DECLSPEC extern const opal_carto_base_module_1_0_0_t -*opal_carto_base_module; - -/** - * Indicator as to whether the list of opened carto components - * is valid or not. - */ -OPAL_DECLSPEC extern bool opal_carto_base_components_opened_valid; - -/** - * List of all opened components; created when the carto - * framework is initialized and destroyed when we reduce the list - * to all available carto components. - */ -OPAL_DECLSPEC extern opal_list_t opal_carto_base_components_opened; - - -/** - * Get the local host graph. you can reduce the graph for only - * the nodes that interst you using the node type. - * - * @param graph - * @param graph_type - * - * @return OPAL_DECLSPEC int - */ -OPAL_DECLSPEC int opal_carto_base_get_host_graph(opal_carto_graph_t **graph, const char *graph_type); - - -/** - * Frre a graph - * - * @param graph - * - * @return OPAL_DECLSPEC void - */ -OPAL_DECLSPEC void opal_carto_base_free_graph(opal_carto_graph_t *graph); - -/** - * Get the distance (weight) from a start node to all other - * nodes. you can reduce the list to the list to the node types - * that intersts you. - * - * @param graph - * @param start - * @param node_type - * @param distance_ - * - * @return OPAL_DECLSPEC int - */ -OPAL_DECLSPEC int opal_carto_base_get_nodes_distance(opal_carto_graph_t *graph, opal_carto_base_node_t *start, const char *node_type, opal_value_array_t *distance_); - -/** - * find the distance between two nodes. - * - * @param graph - * @param start - * @param end - * - * @return OPAL_DECLSPEC uint32_t - */ -OPAL_DECLSPEC uint32_t opal_carto_base_spf(opal_carto_graph_t *graph,opal_carto_base_node_t *start, opal_carto_base_node_t *end); - -/** - * Find a node in the graph - * - * @param graph - * @param node_name - * - * @return OPAL_DECLSPEC opal_carto_base_node_t - */ -OPAL_DECLSPEC opal_carto_base_node_t *opal_carto_base_find_node(opal_carto_graph_t *graph, const char *node_name); - - -/** - * Debugging output stream - */ -OPAL_DECLSPEC extern int opal_carto_base_output; - -OPAL_DECLSPEC extern opal_carto_graph_t *opal_carto_base_common_host_graph; - -OPAL_DECLSPEC extern opal_carto_base_module_1_0_0_t opal_carto_default_module; - - -END_C_DECLS - -#endif /* OPAL_BASE_CARTO_H */ diff --git a/opal/mca/carto/base/carto_base_close.c b/opal/mca/carto/base/carto_base_close.c deleted file mode 100644 index edf16a1bc3..0000000000 --- a/opal/mca/carto/base/carto_base_close.c +++ /dev/null @@ -1,50 +0,0 @@ -/* - * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana - * University Research and Technology - * Corporation. All rights reserved. - * Copyright (c) 2004-2006 The University of Tennessee and The University - * of Tennessee Research Foundation. All rights - * reserved. - * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, - * University of Stuttgart. All rights reserved. - * Copyright (c) 2004-2005 The Regents of the University of California. - * All rights reserved. - * Copyright (c) 2007 Cisco Systems, Inc. All rights reserved. - * $COPYRIGHT$ - * - * Additional copyrights may follow - * - * $HEADER$ - */ - -#include "opal_config.h" - -#include "opal/constants.h" -#include "opal/mca/mca.h" -#include "opal/mca/base/base.h" -#include "opal/mca/carto/carto.h" -#include "opal/mca/carto/base/base.h" - -int opal_carto_base_close(void) -{ - /* If there is a selected carto module, finalize it */ - - if (NULL != opal_carto_base_module && - NULL != opal_carto_base_module->carto_module_finalize) { - opal_carto_base_module->carto_module_finalize(); - } - - /* Close all components that are still open (this should only - happen during ompi_info). */ - - if (opal_carto_base_components_opened_valid) { - mca_base_components_close(opal_carto_base_output, - &opal_carto_base_components_opened, NULL); - OBJ_DESTRUCT(&opal_carto_base_components_opened); - opal_carto_base_components_opened_valid = false; - } - - /* All done */ - - return OPAL_SUCCESS; -} diff --git a/opal/mca/carto/base/carto_base_graph.c b/opal/mca/carto/base/carto_base_graph.c deleted file mode 100644 index 0a9fc8a993..0000000000 --- a/opal/mca/carto/base/carto_base_graph.c +++ /dev/null @@ -1,382 +0,0 @@ -/* - * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana - * University Research and Technology - * Corporation. All rights reserved. - * Copyright (c) 2004-2005 The University of Tennessee and The University - * of Tennessee Research Foundation. All rights - * reserved. - * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, - * University of Stuttgart. All rights reserved. - * Copyright (c) 2004-2005 The Regents of the University of California. - * All rights reserved. -# Copyright (c) 2007 Cisco Systems, Inc. All rights reserved. - * $COPYRIGHT$ - * - * Additional copyrights may follow - * - * $HEADER$ - */ -/** - * @file - * - * This file is an implementation of the carto graph base on the graph class. - * This file is common to all the carto components. - */ - -#include "opal_config.h" - - -#include "opal/class/opal_graph.h" -#include "opal/mca/carto/carto.h" -#include "opal/mca/carto/base/carto_base_graph.h" - - - -/* - * Globals - */ - -static void opal_carto_base_free_node(void *node); -static void opal_carto_base_copy_nodes(void **dst, void *src); -static void *opal_carto_base_alloc_node(void); -static int opal_carto_compare_nodes(void *node1, void *node2); -static char* opal_carto_print_node(void* node); - - - -/* - * Functions - */ - -/** - * A function to print a node. this function allocates buffer - * and prints in it the node type and the node name. this - * function will be assign to the print_vertex function pointer - * when building a new node. - * - * @param node The node we want to print - * - * @return char* the string to print. - */ -static char* opal_carto_print_node(void* node) -{ - char *print_str; - char cpu_str[] = "(CPU)"; - opal_carto_base_node_t *tmp_node = (opal_carto_base_node_t *)node; - - if (false == tmp_node->is_cpu) { - cpu_str[0] = 0; - } - asprintf(&print_str,"%s %5s -%s", tmp_node->node_type, cpu_str, tmp_node->node_name); - - return print_str; -} - -/** - * A function to free node. this function will be assigned to - * the free_vertex_data pointer - * - * @param node the node to free. - */ -static void opal_carto_base_free_node(void *node) -{ - opal_carto_base_node_t *tmp_node = (opal_carto_base_node_t *)node; - /* free the node name string */ - free(tmp_node->node_name); - free(tmp_node->node_type); - /* free the node */ - free(tmp_node); -} - -/** - * A function to copy a node. this function will be assign in - * the copy_vertex_data function pointer. - * - * @param dst the destination node. - * @param src the source node. - */ -static void opal_carto_base_copy_nodes(void **dst, void *src) -{ - opal_carto_base_node_t *src_node = (opal_carto_base_node_t *)src, - *dst_node = (opal_carto_base_node_t *)*dst; - - /* duplicate the node name */ - dst_node->node_name = strdup(src_node->node_name); - /* copy the node type */ - dst_node->node_type = strdup(src_node->node_type); - dst_node->is_cpu = src_node->is_cpu; - /* If the nodes vertex was copied, get the copied vertex */ - dst_node->vertex = src_node->vertex->sibling; -} - -/** - * Allocate memory for node. this function will be assign to the - * alloc_vertex_data function pointer. - * - * @return void* - */ -static void *opal_carto_base_alloc_node(void) -{ - opal_carto_base_node_t *node; - - /* allocate memory fore node */ - node = (opal_carto_base_node_t *)malloc(sizeof(opal_carto_base_node_t)); - /*Init the node fields */ - node->node_name = NULL; - node->node_type = NULL; - node->is_cpu = false; - node->vertex = NULL; - - return (void*)node; -} - -/** - * Compare two nodes. in our case we needs to compare only the - * node name. this function will be assign to the compare vertex - * data function pointer. - * - * @param node1 - * @param node2 - * - * @return int 0-equal, 1-the first is bigger, -1-the first is - * smaller. - */ -static int opal_carto_compare_nodes(void *node1, void *node2) -{ - opal_carto_base_node_t *tmp_node1 = (opal_carto_base_node_t *)node1, - *tmp_node2 = (opal_carto_base_node_t *)node2; - - /* use str compare to compare the node names */ - return strcmp(tmp_node1->node_name, tmp_node2->node_name); -} - -/** - * Create new carto graph. - * - * @param graph an empty graph pointer - */ -void opal_carto_base_graph_create_fn(opal_carto_graph_t **graph) -{ - *graph = (opal_carto_graph_t *)OBJ_NEW(opal_graph_t); -} - -/** - * Add a node to carto graph. - * - * @param graph the carto graph to add the node to. - * @param node the node to add. - */ -void opal_carto_base_graph_add_node_fn(opal_carto_graph_t *graph, opal_carto_base_node_t *node) -{ - /* construct new vertex */ - node->vertex = OBJ_NEW(opal_graph_vertex_t); - /* assign the node as the vertex data */ - node->vertex->vertex_data = (void *)node; - /* assign the vertex function pointers */ - node->vertex->free_vertex_data = opal_carto_base_free_node; - node->vertex->copy_vertex_data = opal_carto_base_copy_nodes; - node->vertex->alloc_vertex_data = opal_carto_base_alloc_node; - node->vertex->compare_vertex = opal_carto_compare_nodes; - node->vertex->print_vertex = opal_carto_print_node; - /* add the new node to the carto graph by adding the nodes vertex to the graph */ - opal_graph_add_vertex((opal_graph_t *)graph, node->vertex); -} - -/** - * Free a carto graph - * @param graph the graph we want to free. - */ -void opal_carto_base_free_graph_fn(opal_carto_graph_t *graph) -{ - int i, graph_order; - opal_carto_base_node_t *node; - opal_pointer_array_t *graph_vertices; - opal_graph_vertex_t *vertex; - - graph_vertices = OBJ_NEW(opal_pointer_array_t); - opal_pointer_array_init(graph_vertices, 20, INT_MAX, 20); - /* get all the graph vertices */ - graph_order = opal_graph_get_graph_vertices(graph, graph_vertices); - /* for all the vertices in the graph, free the nodes (and distract the vertices) */ - for (i = 0; i < graph_order; i++) { - vertex = (opal_graph_vertex_t *)opal_pointer_array_get_item(graph_vertices, i); - node = vertex->vertex_data; - opal_carto_base_free_node((void *)node); - } - OBJ_RELEASE(graph_vertices); - /* destruct the graph */ - OBJ_RELEASE(graph); -} - -/** - * Connect two nodes by adding an edge to the graph. - * - * @param graph the graph that the nodes belongs to. - * @param start the start node - * @param end the end node - * @param weight the weight of the connection - * - * @return int success or error (if one of the nodes does not - * belong to the graph. - */ -int opal_carto_base_connect_nodes_fn(opal_carto_graph_t *graph, opal_carto_base_node_t *start, opal_carto_base_node_t *end, uint32_t weight) -{ - opal_graph_edge_t *edge; - - /* construct anew edge */ - edge = OBJ_NEW(opal_graph_edge_t); - /* assigne the start and the end nodes vertices to the new edge */ - edge->start = start->vertex; - edge->end = end->vertex; - /* assign the weight to the edge */ - edge->weight = weight; - /* add the edge to the graph */ - return opal_graph_add_edge((opal_graph_t *)graph, edge); -} - - -/** - * Duplicate a carto graph and reduce the new graph to contain - * nodes from a ceratin type(s) - * - * @param destination The new graph. - * @param source the original graph. - * @param node_type the node type(s) that the new graph will - * include. - */ -void opal_carto_base_duplicate_graph_fn(opal_carto_graph_t **destination, const opal_carto_graph_t *source, const char *node_type) -{ - opal_pointer_array_t *vertices; - int i, graph_order; - opal_carto_base_node_t *node; - opal_graph_vertex_t *vertex; - - /* duplicate the graph */ - opal_graph_duplicate((opal_graph_t **)destination, (opal_graph_t *)source); - /* if there is no need for reduction, return */ - if (NULL == node_type) { - return; - } - vertices = OBJ_NEW(opal_pointer_array_t); - opal_pointer_array_init(vertices, 20, INT_MAX, 20); - /* get all the vertices of the new graph */ - graph_order = opal_graph_get_graph_vertices(*destination, vertices); - /* remove all the nodes that are not in the required type */ - for (i = 0; i < graph_order; i++ ) { - vertex = (opal_graph_vertex_t *)opal_pointer_array_get_item(vertices, i); - node = vertex->vertex_data; - if (!(0 == strcmp(node_type, node->node_type) || node->is_cpu)) { - opal_graph_remove_vertex(*destination, vertex); - } - } - /* free the vertices array */ - OBJ_RELEASE(vertices); -} - -/** - * opal_carto_base_get_nodes_distance - returns the distance of - * all the nodes from the reference node. - * - * @param graph - * @param reference_node - * @param node_type the type of the nodes in the returned array - * @param dist_array - * - * @return int number of nodes in the returned array. - */ -int opal_carto_base_get_nodes_distance_fn(opal_carto_graph_t *graph, opal_carto_base_node_t *reference_node, - const char *node_type, opal_value_array_t *dist_array) -{ - opal_value_array_t *distance_array; - vertex_distance_from_t *vertex_distance; - opal_carto_base_node_t *node; - uint32_t i, graph_order; - int distance_array_size; - opal_carto_node_distance_t node_distance; - - - distance_array = OBJ_NEW(opal_value_array_t); - opal_value_array_init(distance_array, sizeof(vertex_distance_from_t)); - opal_value_array_reserve(distance_array,50); - /* use dijkstra algorithm to receive the distance of all the nodes from the referenced node */ - graph_order = opal_graph_dijkstra(graph, reference_node->vertex, distance_array); - /* for all the nodes in the dijkstra array */ - for (i = 0, distance_array_size = 0; i < graph_order; i++) { - vertex_distance = opal_value_array_get_item(distance_array, i); - node = vertex_distance->vertex->vertex_data; - /* check if the node is in the correct type */ - if (NULL == node_type || 0 == strcmp(node->node_type, node_type)) { - /* assigne the result distance array */ - node_distance.node = vertex_distance->vertex->vertex_data; - node_distance.node_distance = vertex_distance->weight; - opal_value_array_append_item(dist_array, (void *)&node_distance); - } - } - /* return the result distance array */ - return distance_array_size; -} - -/** - * Find the shortest path between two nodes in the graph - * - * @param graph the graph that the nodes belongs to. - * @param node1 first node. - * @param node2 second node. - * - * @return uint32_t he distance between the nodes. - */ -uint32_t opal_carto_base_graph_spf_fn(opal_carto_graph_t *graph, opal_carto_base_node_t *node1, opal_carto_base_node_t *node2) -{ - return opal_graph_spf((opal_graph_t *)graph, node1->vertex, node2->vertex); -} - -/** - * Find a node in the graph according to its name. - * - * @param graph the graph in which we are searching. - * @param node_name the node name. - * - * @return opal_carto_base_node_t* the node with the name -if - * found or NULL. - */ -opal_carto_base_node_t *opal_carto_base_graph_find_node_fn(opal_carto_graph_t *graph, const char *node_name) -{ - opal_carto_base_node_t node; - opal_graph_vertex_t *vertex; - - /* build a temporary node */ - node.node_name = strdup(node_name); - /** - * find a vertex in the graph. the find_vertex uses the - * compare_vertex_data method. in our case, compare_vertex_data - * is assigned to opal_carto_compare_nodes that compares two - * nodes names. - */ - vertex = opal_graph_find_vertex((opal_graph_t *)graph, (void *)&node); - free(node.node_name); - if (NULL != vertex) { - /* return the fund vertex data (node) */ - return vertex->vertex_data; - } - /* if not found, return NULL */ - return NULL; -} - -/** - * Get the host cartography graph. - * - * @param graph an unallocated pointer to a graph. - * @param graph_type the type of nodes we want the returned - * graph will contain. - * - * @return int success or error - */ -int opal_carto_base_graph_get_host_graph_fn(opal_carto_graph_t **graph, const char *graph_type) -{ - /* duplicate the host graph and delete all the relevant nodes */ - opal_carto_base_duplicate_graph_fn(graph, opal_carto_base_common_host_graph, graph_type); - return OPAL_SUCCESS; -} - - - diff --git a/opal/mca/carto/base/carto_base_graph.h b/opal/mca/carto/base/carto_base_graph.h deleted file mode 100644 index d8c88ea572..0000000000 --- a/opal/mca/carto/base/carto_base_graph.h +++ /dev/null @@ -1,144 +0,0 @@ -/* - * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana - * University Research and Technology - * Corporation. All rights reserved. - * Copyright (c) 2004-2006 The University of Tennessee and The University - * of Tennessee Research Foundation. All rights - * reserved. - * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, - * University of Stuttgart. All rights reserved. - * Copyright (c) 2004-2005 The Regents of the University of California. - * All rights reserved. - * Copyright (c) 2007 Cisco Systems, Inc. All rights reserved. - * $COPYRIGHT$ - * - * Additional copyrights may follow - * - * $HEADER$ - * - */ - -#ifndef OPAL_CARTO_BASE_GRAPH_H -#define OPAL_CARTO_BASE_GRAPH_H - -/* THIS FILE CONTAINS THE ACTUAL BASE FUNCTION IMPLEMENTATIONS FOR - * THE CARTO FRAMEWORK - THESE FUNCTIONS ARE TO BE CALLED STRICTLY - * FROM THE VARIOUS COMPONENTS, AS PASSED THROUGH BY THE PUBLIC - * BASE FUNCTIONS - * - * DO -NOT- CALL THESE FUNCTIONS DIRECTLY! - */ - -#include "opal_config.h" - -#include "opal/mca/carto/carto.h" - -BEGIN_C_DECLS - -OPAL_DECLSPEC extern opal_carto_graph_t *opal_carto_base_common_host_graph; - -/** - * Create new carto graph. - * - * @param graph an empty graph pointer - */ -OPAL_DECLSPEC void opal_carto_base_graph_create_fn(opal_carto_graph_t **graph); - -/** - * Add a node to carto graph. - * - * @param graph the carto graph to add the node to. - * @param node the node to add. - */ -OPAL_DECLSPEC void -opal_carto_base_graph_add_node_fn(opal_carto_graph_t *graph, opal_carto_base_node_t *node); - -/** - * Free a carto graph - * @param graph the graph we want to free. - */ -OPAL_DECLSPEC void opal_carto_base_free_graph_fn(opal_carto_graph_t *graph); - -/** - * Connect two nodes by adding an edge to the graph. - * - * @param graph the graph that the nodes belongs to. - * @param start the start node - * @param end the end node - * @param weight the weight of the connection - * - * @return int success or error (if one of the nodes does not - * belong to the graph. - */ -OPAL_DECLSPEC int -opal_carto_base_connect_nodes_fn(opal_carto_graph_t *graph, opal_carto_base_node_t *start, - opal_carto_base_node_t *end, uint32_t weight); - -/** - * Duplicate a carto graph and reduce the new graph to contain - * nodes from a ceratin type(s) - * - * @param destination The new graph. - * @param source the original graph. - * @param node_type the node type(s) that the new graph will - * include. - */ -OPAL_DECLSPEC void -opal_carto_base_duplicate_graph_fn(opal_carto_graph_t **destination, const opal_carto_graph_t *source, - const char *node_type); - - -/** - * opal_carto_base_get_nodes_distance - returns the distance of - * all the nodes from the reference node. - * - * @param graph - * @param reference_node - * @param node_type the type of the nodes in the returned array - * @param dist_array - * - * @return int number of nodes in the returned array. - */ -OPAL_DECLSPEC int -opal_carto_base_get_nodes_distance_fn(opal_carto_graph_t *graph, opal_carto_base_node_t *reference_node, - const char *node_type, opal_value_array_t *dist_array); - -/** - * Find the shortest path between two nodes in the graph - * - * @param graph the graph that the nodes belongs to. - * @param node1 first node. - * @param node2 second node. - * - * @return uint32_t he distance between the nodes. - */ -OPAL_DECLSPEC uint32_t -opal_carto_base_graph_spf_fn(opal_carto_graph_t *graph, opal_carto_base_node_t *node1, - opal_carto_base_node_t *node2); - -/** - * Find a node in the graph according to its name. - * - * @param graph the graph in which we are searching. - * @param node_name the node name. - * - * @return opal_carto_base_node_t* the node with the name -if - * found or NULL. - */ -OPAL_DECLSPEC opal_carto_base_node_t -*opal_carto_base_graph_find_node_fn(opal_carto_graph_t *graph, const char *node_name); - -/** - * Get the host cartography graph. - * - * @param graph an unallocated pointer to a graph. - * @param graph_type the type of nodes we want the returned - * graph will contain. - * - * @return int success or error - */ -OPAL_DECLSPEC int opal_carto_base_graph_get_host_graph_fn(opal_carto_graph_t **graph, const char * graph_type); - -END_C_DECLS - -#endif diff --git a/opal/mca/carto/base/carto_base_open.c b/opal/mca/carto/base/carto_base_open.c deleted file mode 100644 index 00c418f8bd..0000000000 --- a/opal/mca/carto/base/carto_base_open.c +++ /dev/null @@ -1,115 +0,0 @@ -/* - * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana - * University Research and Technology - * Corporation. All rights reserved. - * Copyright (c) 2004-2005 The University of Tennessee and The University - * of Tennessee Research Foundation. All rights - * reserved. - * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, - * University of Stuttgart. All rights reserved. - * Copyright (c) 2004-2005 The Regents of the University of California. - * All rights reserved. - * Copyright (c) 2007 Cisco Systems, Inc. All rights reserved. - * $COPYRIGHT$ - * - * Additional copyrights may follow - * - * $HEADER$ - */ - - -#include "opal_config.h" - -#include "opal/constants.h" -#include "opal/util/output.h" -#include "opal/mca/mca.h" -#include "opal/mca/base/base.h" -#include "opal/mca/base/mca_base_param.h" -#include "opal/mca/carto/carto.h" -#include "opal/mca/carto/base/carto_base_graph.h" -#include "opal/mca/carto/base/base.h" - - -/* - * The following file was created by configure. It contains extern - * statements and the definition of an array of pointers to each - * component's public mca_base_component_t struct. - */ -#include "opal/mca/carto/base/static-components.h" - - -/* - * Globals - */ -int opal_carto_base_output = -1; -bool opal_carto_base_components_opened_valid = false; -opal_list_t opal_carto_base_components_opened; -opal_carto_graph_t *opal_carto_base_common_host_graph=NULL; - -/* - * default carto module when no components are found - */ -static int opal_carto_default_init(void) -{ - /* create an empty graph */ - if (NULL == opal_carto_base_common_host_graph) { - opal_carto_base_graph_create_fn(&opal_carto_base_common_host_graph); - } - return OPAL_SUCCESS; -} -static int opal_carto_default_finalize(void) -{ - /* free the host cartography graph. */ - if (NULL != opal_carto_base_common_host_graph) { - opal_carto_base_free_graph_fn(opal_carto_base_common_host_graph); - } - return OPAL_SUCCESS; -} - -opal_carto_base_module_1_0_0_t opal_carto_default_module = { - opal_carto_default_init, - opal_carto_base_graph_get_host_graph_fn, - opal_carto_base_free_graph_fn, - opal_carto_base_get_nodes_distance_fn, - opal_carto_base_graph_spf_fn, - opal_carto_base_graph_find_node_fn, - opal_carto_default_finalize, -}; - -/* - * Function for finding and opening either all MCA components, or the one - * that was specifically requested via a MCA parameter. - */ -int opal_carto_base_open(void) -{ - int value; - - /* Debugging / verbose output */ - - mca_base_param_reg_int_name("carto", "base_verbose", - "Verbosity level of the carto framework", - false, false, - 0, &value); - if (0 != value) { - opal_carto_base_output = opal_output_open(NULL); - } else { - opal_carto_base_output = -1; - } - - opal_carto_base_components_opened_valid = false; - - /* Open up all available components */ - - if (OPAL_SUCCESS != - mca_base_components_open("carto", opal_carto_base_output, - mca_carto_base_static_components, - &opal_carto_base_components_opened, - true)) { - return OPAL_ERROR; - } - opal_carto_base_components_opened_valid = true; - - /* All done */ - - return OPAL_SUCCESS; -} diff --git a/opal/mca/carto/base/carto_base_select.c b/opal/mca/carto/base/carto_base_select.c deleted file mode 100644 index f00a9e65b8..0000000000 --- a/opal/mca/carto/base/carto_base_select.c +++ /dev/null @@ -1,72 +0,0 @@ -/* - * Copyright (c) 2004-2008 The Trustees of Indiana University and Indiana - * University Research and Technology - * Corporation. All rights reserved. - * Copyright (c) 2004-2006 The University of Tennessee and The University - * of Tennessee Research Foundation. All rights - * reserved. - * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, - * University of Stuttgart. All rights reserved. - * Copyright (c) 2004-2005 The Regents of the University of California. - * All rights reserved. - * Copyright (c) 2007 Cisco Systems, Inc. All rights reserved. - * $COPYRIGHT$ - * - * Additional copyrights may follow - * - * $HEADER$ - */ - - -#include "opal_config.h" - -#include "opal/constants.h" -#include "opal/mca/mca.h" -#include "opal/mca/base/base.h" -#include "opal/mca/base/mca_base_param.h" -#include "opal/mca/carto/carto.h" -#include "opal/mca/carto/base/base.h" - -/* - * Globals - */ -bool opal_carto_base_selected = false; -const opal_carto_base_component_2_0_0_t *opal_carto_base_component = NULL; -const opal_carto_base_module_1_0_0_t *opal_carto_base_module = NULL; - - -int opal_carto_base_select(void) -{ - int exit_status = OPAL_SUCCESS; - opal_carto_base_component_2_0_0_t *best_component = NULL; - opal_carto_base_module_1_0_0_t *best_module = NULL; - - /* - * Select the best component - */ - if( OPAL_SUCCESS != mca_base_select("carto", opal_carto_base_output, - &opal_carto_base_components_opened, - (mca_base_module_t **) &best_module, - (mca_base_component_t **) &best_component) ) { - /* This will only happen if no component was selected, so - * use the default module instead - */ - opal_carto_base_module = &opal_carto_default_module; - opal_carto_base_selected = true; - goto cleanup; - } - - /* Save the winner */ - opal_carto_base_component = best_component; - opal_carto_base_module = best_module; - opal_carto_base_selected = true; - -cleanup: - /* Initialize the winner */ - if (NULL != opal_carto_base_module) { - exit_status = opal_carto_base_module->carto_module_init(); - } - - return exit_status; -} - diff --git a/opal/mca/carto/base/carto_base_wrapers.c b/opal/mca/carto/base/carto_base_wrapers.c deleted file mode 100644 index 941df85085..0000000000 --- a/opal/mca/carto/base/carto_base_wrapers.c +++ /dev/null @@ -1,73 +0,0 @@ -/* - * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana - * University Research and Technology - * Corporation. All rights reserved. - * Copyright (c) 2004-2006 The University of Tennessee and The University - * of Tennessee Research Foundation. All rights - * reserved. - * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, - * University of Stuttgart. All rights reserved. - * Copyright (c) 2004-2005 The Regents of the University of California. - * All rights reserved. - * Copyright (c) 2007 Cisco Systems, Inc. All rights reserved. - * - * $COPYRIGHT$ - * - * Additional copyrights may follow - * - * $HEADER$ - */ - - -#include "opal_config.h" - -#include "opal/constants.h" -#include "opal/mca/carto/carto.h" -#include "opal/mca/carto/base/base.h" - - -int opal_carto_base_get_host_graph(opal_carto_graph_t **graph, const char *graph_type) -{ - if (!opal_carto_base_selected) { - return OPAL_ERR_NOT_FOUND; - } - return opal_carto_base_module->get_host_graph(graph, graph_type); -} - -void opal_carto_base_free_graph(opal_carto_graph_t *graph) -{ - if (!opal_carto_base_selected) { - return ; - } - opal_carto_base_module->free_graph(graph); -} - - -int opal_carto_base_get_nodes_distance(opal_carto_graph_t *graph, opal_carto_base_node_t *start, const char *node_type, opal_value_array_t *distance_) -{ - if (!opal_carto_base_selected) { - return OPAL_ERR_NOT_FOUND; - } - return opal_carto_base_module->get_nodes_distance(graph, start, node_type, distance_); -} - -uint32_t opal_carto_base_spf(opal_carto_graph_t *graph,opal_carto_base_node_t *start, opal_carto_base_node_t *end) -{ - if (!opal_carto_base_selected) { - return OPAL_ERR_NOT_FOUND; - } - return opal_carto_base_module->spf(graph, start, end); -} - -opal_carto_base_node_t *opal_carto_base_find_node(opal_carto_graph_t *graph, const char *node_name) -{ - if (!opal_carto_base_selected) { - return NULL; - } - return opal_carto_base_module->find_node(graph, node_name); -} - - - - - diff --git a/opal/mca/carto/carto.h b/opal/mca/carto/carto.h deleted file mode 100644 index 191362714a..0000000000 --- a/opal/mca/carto/carto.h +++ /dev/null @@ -1,164 +0,0 @@ -/* - * Copyright (c) 2004-2008 The Trustees of Indiana University and Indiana - * University Research and Technology - * Corporation. All rights reserved. - * Copyright (c) 2004-2005 The University of Tennessee and The University - * of Tennessee Research Foundation. All rights - * reserved. - * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, - * University of Stuttgart. All rights reserved. - * Copyright (c) 2004-2005 The Regents of the University of California. - * All rights reserved. - * Copyright (c) 2007 Cisco Systems, Inc. All rights reserved. - * $COPYRIGHT$ - * - * Additional copyrights may follow - * - * $HEADER$ - */ -/** - * @file - * - * The carto framework suplies an information of the the host structure and connection between the - * host components i.e memory nodes,CPUs, Ethernet port and inifiniband ports. - */ - -#ifndef OPAL_CARTO_H -#define OPAL_CARTO_H - -#include "opal_config.h" - -#include "opal/mca/mca.h" -#include "opal/mca/base/base.h" -#include "opal/class/opal_graph.h" - -/** - * type for carto graph - */ -typedef opal_graph_t opal_carto_graph_t; - -/* A structure of carto node */ -struct opal_carto_base_node_t { - opal_graph_vertex_t *vertex; /* the parent of a node is a graph vertex */ - char *node_name; - char *node_type; /* the type of the node */ - bool is_cpu; -}; - -/** - * A definition of carto node type. - */ -typedef struct opal_carto_base_node_t opal_carto_base_node_t; - - -/** - * A structure of node distance from some other node - */ -struct opal_carto_node_distance_t { - opal_carto_base_node_t *node; /* The node */ - uint32_t node_distance; /* and the distance */ -}; - -/** - * A definition of node distance type. - */ -typedef struct opal_carto_node_distance_t opal_carto_node_distance_t; - - - -/** - * Module initialization function. Should return OPAL_SUCCESS. - */ -typedef int (*opal_carto_base_module_init_1_0_0_fn_t)(void); - -/** - * Get the local host graph. you can reduce the graph for only - * the nodes that interst you using the node type. - */ -typedef int (*opal_carto_base_get_host_graph_fn_t) - (opal_carto_graph_t **graph, const char *graph_type); - -/** - * Frre a graph - */ -typedef void (*opal_carto_base_free_graph_fn_t) - (opal_carto_graph_t *graph); - -/** - * Get the distance (weight) from a start node to all other - * nodes. you can reduce the list to the list to the node types - * that intersts you. - */ -typedef int (*opal_carto_base_get_nodes_distance_fn_t) - (opal_carto_graph_t *graph, opal_carto_base_node_t *start, const char *node_type, opal_value_array_t *distance_); - -/** - * find the distance between two nodes. - */ -typedef uint32_t (*opal_carto_base_spf_fn_t) - (opal_carto_graph_t *graph,opal_carto_base_node_t *start, opal_carto_base_node_t *end); - -/** - * Find a node in the graph - */ -typedef opal_carto_base_node_t *(*opal_carto_base_find_node_fn_t) - (opal_carto_graph_t *graph, const char *node_name); - - -/** - * Module finalize function. Invoked by the base on the selected - * module when the carto framework is being shut down. - */ -typedef int (*opal_carto_base_module_finalize_fn_t)(void); - - -/** - * Structure for carto components. - */ -struct opal_carto_base_component_2_0_0_t { - /** MCA base component */ - mca_base_component_t base_version; - /** MCA base data */ - mca_base_component_data_t base_data; -}; -/** - * Convenience typedef - */ -typedef struct opal_carto_base_component_2_0_0_t opal_carto_base_component_2_0_0_t; -typedef struct opal_carto_base_component_2_0_0_t opal_carto_base_component_t; - - -/** - * Structure for carto modules - */ -struct opal_carto_base_module_1_0_0_t { - /** Module initialization function */ - opal_carto_base_module_init_1_0_0_fn_t carto_module_init; - /** Get host graph */ - opal_carto_base_get_host_graph_fn_t get_host_graph; - /** free graph */ - opal_carto_base_free_graph_fn_t free_graph; - /** Get the distance from one node to all other nodes */ - opal_carto_base_get_nodes_distance_fn_t get_nodes_distance; - /** Find the distance between two nodes */ - opal_carto_base_spf_fn_t spf; - /** Find a node in the graph */ - opal_carto_base_find_node_fn_t find_node; - /** Shut down this module */ - opal_carto_base_module_finalize_fn_t carto_module_finalize; -}; -/** - * Convenience typedef - */ -typedef struct opal_carto_base_module_1_0_0_t opal_carto_base_module_1_0_0_t; -typedef struct opal_carto_base_module_1_0_0_t opal_carto_base_module_t; - - -/* - * Macro for use in components that are of type carto - */ -#define OPAL_CARTO_BASE_VERSION_2_0_0 \ - MCA_BASE_VERSION_2_0_0, \ - "carto", 2, 0, 0 - -#endif /* OPAL_CARTO_H */ diff --git a/opal/mca/carto/file/Makefile.am b/opal/mca/carto/file/Makefile.am deleted file mode 100644 index 31bb60a635..0000000000 --- a/opal/mca/carto/file/Makefile.am +++ /dev/null @@ -1,53 +0,0 @@ -# -# Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana -# University Research and Technology -# Corporation. All rights reserved. -# Copyright (c) 2004-2005 The University of Tennessee and The University -# of Tennessee Research Foundation. All rights -# reserved. -# Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, -# University of Stuttgart. All rights reserved. -# Copyright (c) 2004-2005 The Regents of the University of California. -# All rights reserved. -# Copyright (c) 2007-2010 Cisco Systems, Inc. All rights reserved. -# $COPYRIGHT$ -# -# Additional copyrights may follow -# -# $HEADER$ -# - -AM_LFLAGS = -Pcarto_file_ -LEX_OUTPUT_ROOT = lex.carto_file_ - -dist_pkgdata_DATA = \ - help-opal-carto-file.txt - - -sources = \ - carto_file.h \ - carto_file_lex.h \ - carto_file_lex.l \ - carto_file_component.c \ - carto_file_module.c - -# Make the output library in this directory, and name it either -# mca__.la (for DSO builds) or libmca__.la -# (for static builds). - -if MCA_BUILD_opal_carto_file_DSO -component_noinst = -component_install = mca_carto_file.la -else -component_noinst = libmca_carto_file.la -component_install = -endif - -mcacomponentdir = $(pkglibdir) -mcacomponent_LTLIBRARIES = $(component_install) -mca_carto_file_la_SOURCES = $(sources) -mca_carto_file_la_LDFLAGS = -module -avoid-version - -noinst_LTLIBRARIES = $(component_noinst) -libmca_carto_file_la_SOURCES = $(sources) -libmca_carto_file_la_LDFLAGS = -module -avoid-version diff --git a/opal/mca/carto/file/carto_file.h b/opal/mca/carto/file/carto_file.h deleted file mode 100644 index d4f009cd02..0000000000 --- a/opal/mca/carto/file/carto_file.h +++ /dev/null @@ -1,115 +0,0 @@ -/* - * Copyright (c) 2004-2008 The Trustees of Indiana University and Indiana - * University Research and Technology - * Corporation. All rights reserved. - * Copyright (c) 2004-2005 The University of Tennessee and The University - * of Tennessee Research Foundation. All rights - * reserved. - * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, - * University of Stuttgart. All rights reserved. - * Copyright (c) 2004-2005 The Regents of the University of California. - * All rights reserved. - * Copyright (c) 2006-2007 Cisco Systems, Inc. All rights reserved. - * - * $COPYRIGHT$ - * - * Additional copyrights may follow - * - * $HEADER$ - */ - -/** - * @file#this is a comment -# Node declaration Node type (Free string) Node name (Free string) -# (Reserve word) (socket is a reserve word (free string) -# for CPU socket) -#======================================================================= - EDGE Memory mem0 - EDGE Memory mem1 - EDGE Memory mem2 - EDGE Memory mem3 -# - EDGE socket socket0 - EDGE socket socket1 - EDGE socket socket2 - EDGE socket socket3 -# - EDGE Infiniband mthca0 - EDGE Infiniband mthca1 -# - EDGE Ethernet eth0 - EDGE Ethernet eth1 -# -# -# Connection decleration From node To node:weight To node:weight ...... -# (Reserve word) (declered (declered (declered -# above) above) above) -#=============================================================================================== - BRANCH mem0 socket0:0 - BRANCH mem3 socket3:0 -# - BRANCH socket0 mem0:0 socket1:1 socket2:1 mthca0:1 eth0:1 - BRANCH socket1 socket0:1 socket3:1 - BRANCH socket2 socket1:1 socket3:1 - BRANCH socket3 mem3:0 socket1:1 socket2:1 mthca1:1 eth1:1 -# -# - BRANCH mthca0 socket0:1 - BRANCH mthca1 socket3:1 -# - BRANCH eth0 socket0:1 - BRANCH eth1 socket3:1 - -#Bi-Directional connection -# - BRANCH_BI_DIR socket1 mem1:0 - BRANCH_BI_DIR socket2 mem2:0 -# -# end of carto file. - - * - * The file component uses a cartograpy file to discover the - * host cartography. - * - * An example cartography file: - * - - * - * - * - * - * - */ - -#ifndef MCA_CARTO_FILE_H -#define MCA_CARTO_FILE_H - -#include "opal_config.h" - -#include "opal/mca/mca.h" -#include "opal/mca/carto/carto.h" - -BEGIN_C_DECLS - -extern char *carto_file_path; - -/** - * Globally exported variable - */ -OPAL_DECLSPEC extern const opal_carto_base_component_2_0_0_t -mca_carto_file_component; - - -/** - * carto query API function - * - * Query function for carto components. Simply returns a priority - * to rank it against other available carto components (assumedly, - * only one component will be available per platform, but it's - * possible that there could be more than one available). - */ -int opal_carto_file_component_query(mca_base_module_t **module, int *priority); - -END_C_DECLS - -#endif /* MCA_CARTO_FILE_EXPORT_H */ diff --git a/opal/mca/carto/file/carto_file_component.c b/opal/mca/carto/file/carto_file_component.c deleted file mode 100644 index a541385f2c..0000000000 --- a/opal/mca/carto/file/carto_file_component.c +++ /dev/null @@ -1,100 +0,0 @@ -/* - * Copyright (c) 2004-2008 The Trustees of Indiana University and Indiana - * University Research and Technology - * Corporation. All rights reserved. - * Copyright (c) 2004-2005 The University of Tennessee and The University - * of Tennessee Research Foundation. All rights - * reserved. - * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, - * University of Stuttgart. All rights reserved. - * Copyright (c) 2004-2005 The Regents of the University of California. - * All rights reserved. - * Copyright (c) 2007 Cisco Systems, Inc. All rights reserved. - * $COPYRIGHT$ - * - * Additional copyrights may follow - * - * $HEADER$ - * - * These symbols are in a file by themselves to provide nice linker - * semantics. Since linkers generally pull in symbols by object - * files, keeping these symbols as the only symbols in this file - * prevents utility programs such as "ompi_info" from having to import - * entire components just to query their version and parameters. - */ - -#include "opal_config.h" - -#include "opal/constants.h" -#include "opal/mca/carto/carto.h" -#include "carto_file.h" - -/* - * Public string showing the carto ompi_file component version number - */ -const char *opal_carto_file_component_version_string = - "OPAL file carto MCA component version " OPAL_VERSION; - -/* - * Local function - */ -static int file_open(void); - -/* - * Instantiate the public struct with all of our public information - * and pointers to our public functions in it - */ - -const opal_carto_base_component_2_0_0_t mca_carto_file_component = { - - /* First, the mca_component_t struct containing meta information - about the component itself */ - - { - OPAL_CARTO_BASE_VERSION_2_0_0, - - /* Component name and version */ - "file", - OPAL_MAJOR_VERSION, - OPAL_MINOR_VERSION, - OPAL_RELEASE_VERSION, - - /* Component open and close functions */ - file_open, - NULL, - opal_carto_file_component_query - }, - { - /* The component is checkpoint ready */ - MCA_BASE_METADATA_PARAM_CHECKPOINT - } -}; - - -static int file_open(void) -{ - - - mca_base_param_reg_string(&mca_carto_file_component.base_version, - "path", - "The path to the cartography file", - false, false, NULL, &carto_file_path); - - /** - * If the user specified the carto file path (not NULL), use the - * carto file component. The auto detect component is with - * higher priority, so by default it will be chosen. - */ - if (NULL == carto_file_path) { - mca_base_param_reg_int(&mca_carto_file_component.base_version, - "priority", - "Priority of the file carto component", - false, false, 10, NULL); - }else{ - mca_base_param_reg_int(&mca_carto_file_component.base_version, - "priority", - "Priority of the file carto component", - false, false, 12, NULL); - } - return OPAL_SUCCESS; -} diff --git a/opal/mca/carto/file/carto_file_lex.h b/opal/mca/carto/file/carto_file_lex.h deleted file mode 100644 index f758d61253..0000000000 --- a/opal/mca/carto/file/carto_file_lex.h +++ /dev/null @@ -1,68 +0,0 @@ -/* -*- C -*- - * - * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana - * University Research and Technology - * Corporation. All rights reserved. - * Copyright (c) 2004-2005 The University of Tennessee and The University - * of Tennessee Research Foundation. All rights - * reserved. - * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, - * University of Stuttgart. All rights reserved. - * Copyright (c) 2004-2005 The Regents of the University of California. - * All rights reserved. - * $COPYRIGHT$ - * - * Additional copyrights may follow - * - * $HEADER$ - */ - -#ifndef CARTO_FILE_LEX_LEX_H_ -#define CARTO_FILE_LEX_LEX_H_ - -#include "opal_config.h" - -#ifdef malloc -#undef malloc -#endif -#ifdef realloc -#undef realloc -#endif -#ifdef free -#undef free -#endif - -#include - -typedef union { - int ival; - char* sval; -} orte_rds_value_t; - -extern int carto_file_lex(void); -extern FILE *carto_file_in; -extern int carto_file_line; -extern bool carto_file_done; -extern orte_rds_value_t carto_file_value; - -/* - * Make lex-generated files not issue compiler warnings - */ -#define YY_STACK_USED 0 -#define YY_ALWAYS_INTERACTIVE 0 -#define YY_NEVER_INTERACTIVE 0 -#define YY_MAIN 0 -#define YY_NO_UNPUT 1 -#define YY_SKIP_YYWRAP 1 - -#define OPAL_CARTO_FILE_NEWLINE 0 -#define OPAL_CARTO_FILE_ERROR 1 -#define OPAL_CARTO_FILE_NODE_DECELERATION 2 -#define OPAL_CARTO_FILE_CONNECTION_DECELERATION 3 -#define OPAL_CARTO_FILE_BIDIRECTION_CONNECTION 4 -#define OPAL_CARTO_FILE_INT 5 -#define OPAL_CARTO_FILE_NAME 6 -#define OPAL_CARTO_FILE_NODE_CONNECTION 7 - -#endif - diff --git a/opal/mca/carto/file/carto_file_lex.l b/opal/mca/carto/file/carto_file_lex.l deleted file mode 100644 index e0133254cf..0000000000 --- a/opal/mca/carto/file/carto_file_lex.l +++ /dev/null @@ -1,102 +0,0 @@ -%option nounput - -%{ /* -*- C -*- */ -/* - * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana - * University Research and Technology - * Corporation. All rights reserved. - * Copyright (c) 2004-2006 The University of Tennessee and The University - * of Tennessee Research Foundation. All rights - * reserved. - * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, - * University of Stuttgart. All rights reserved. - * Copyright (c) 2004-2005 The Regents of the University of California. - * All rights reserved. - * $COPYRIGHT$ - * - * Additional copyrights may follow - * - * $HEADER$ - */ -#include "opal_config.h" - -#include -#ifdef HAVE_UNISTD_H -#include -#endif - -#include "opal/mca/carto/file/carto_file_lex.h" - -/* - * local functions - */ -BEGIN_C_DECLS - -int carto_file_wrap(void); - -END_C_DECLS - -int carto_file_wrap(void) -{ - carto_file_done = true; - return 1; -} - -/* - * global variables - */ -int carto_file_line=1; -orte_rds_value_t carto_file_value; -bool carto_file_done = false; - -%} - -WHITE [\f\t\v ] - -%x comment - -%% - -{WHITE}*\n { carto_file_line++; - return OPAL_CARTO_FILE_NEWLINE; } -#.*\n { carto_file_line++; - return OPAL_CARTO_FILE_NEWLINE; } -"//".*\n { carto_file_line++; - return OPAL_CARTO_FILE_NEWLINE; } - -"/*" { BEGIN(comment); - return OPAL_CARTO_FILE_NEWLINE; } -[^*\n]* ; /* Eat up non '*'s */ -"*"+[^*/\n]* ; /* Eat '*'s not followed by a '/' */ -\n { carto_file_line++; - return OPAL_CARTO_FILE_NEWLINE; } -"*"+"/" { BEGIN(INITIAL); /* Done with Block Comment */ - return OPAL_CARTO_FILE_NEWLINE; } - -{WHITE}+ ; /* whitespace */ - - - - -EDGE { carto_file_value.sval = yytext; - return OPAL_CARTO_FILE_NODE_DECELERATION; } - -BRANCH { carto_file_value.sval = yytext; - return OPAL_CARTO_FILE_CONNECTION_DECELERATION; } - -BRANCH_BI_DIR { carto_file_value.sval = yytext; - return OPAL_CARTO_FILE_BIDIRECTION_CONNECTION; } - -[0-9] { carto_file_value.ival = atol(yytext); - return OPAL_CARTO_FILE_INT; } - -[A-Za-z0-9_-]* { carto_file_value.sval = yytext; - return OPAL_CARTO_FILE_NAME; } - - -([[A-Za-z0-9_\-]*)+":"([0-9]*) { carto_file_value.sval = yytext; - return OPAL_CARTO_FILE_NODE_CONNECTION; } - - -%% - diff --git a/opal/mca/carto/file/carto_file_module.c b/opal/mca/carto/file/carto_file_module.c deleted file mode 100644 index 75c9950986..0000000000 --- a/opal/mca/carto/file/carto_file_module.c +++ /dev/null @@ -1,330 +0,0 @@ -/* - * Copyright (c) 2004-2008 The Trustees of Indiana University and Indiana - * University Research and Technology - * Corporation. All rights reserved. - * Copyright (c) 2004-2005 The University of Tennessee and The University - * of Tennessee Research Foundation. All rights - * reserved. - * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, - * University of Stuttgart. All rights reserved. - * Copyright (c) 2004-2005 The Regents of the University of California. - * All rights reserved. - * Copyright (c) 2006-2007 Cisco Systems, Inc. All rights reserved. - * - * $COPYRIGHT$ - * - * Additional copyrights may follow - * - * $HEADER$ - */ - -#include "opal_config.h" - -/* This component will only be compiled on File, where we are - guaranteed to have and friends */ -#include - -#ifdef HAVE_UNISTD_H -#include -#endif /* HAVE_UNISTD_H */ -#ifdef HAVE_STDLIB_H -#include -#endif /* HAVE_STDLIB_H */ -#ifdef HAVE_STRING_H -#include -#endif /* HAVE_STRING_H */ - -#include "opal/constants.h" -#include "opal/mca/base/mca_base_param.h" -#include "opal/util/argv.h" -#include "opal/util/show_help.h" -#include "opal/mca/carto/carto.h" -#include "opal/mca/carto/base/base.h" -#include "opal/mca/carto/base/carto_base_graph.h" -#include "carto_file.h" -#include "carto_file_lex.h" - - - -char *carto_file_path; - -static int opal_carto_file_init(void); -static int opal_carto_file_finalize(void); -static int opal_carto_file_parse(const char *cartofile); - - -/* - * File carto module - */ -static const opal_carto_base_module_1_0_0_t loc_module = { - opal_carto_file_init, - opal_carto_base_graph_get_host_graph_fn, - opal_carto_base_free_graph_fn, - opal_carto_base_get_nodes_distance_fn, - opal_carto_base_graph_spf_fn, - opal_carto_base_graph_find_node_fn, - opal_carto_file_finalize, -}; - - -int opal_carto_file_component_query(mca_base_module_t **module, int *priority) -{ - int param; - - param = mca_base_param_find("carto", "file", "priority"); - mca_base_param_lookup_int(param, priority); - - param = mca_base_param_find("carto", "file", "path"); - mca_base_param_lookup_string(param, &carto_file_path); - - *module = (mca_base_module_t *)&loc_module; - - return OPAL_SUCCESS; -} - - -/** - * Init the carto file module. - * - * @return int success or error - */ -static int opal_carto_file_init(void) -{ - int rc; - - /* create an empty graph */ - if (NULL == opal_carto_base_common_host_graph) { - opal_carto_base_graph_create_fn(&opal_carto_base_common_host_graph); - } - if (NULL == carto_file_path) { - return OPAL_ERROR; - } - /* parse the carto file and add nodes and connections to the graph */ - rc = opal_carto_file_parse(carto_file_path); - return rc; -} - -/** - * finalize the carto file module. - * - * @return int success or error - */ -static int opal_carto_file_finalize(void) -{ - /* free the host cartography graph. */ - if (NULL != opal_carto_base_common_host_graph) { - opal_carto_base_free_graph_fn(opal_carto_base_common_host_graph); - } - return OPAL_SUCCESS; -} - - -/** - * Parse the carto file and file the host graph with nodes and - * connections. - * - * @param cartofile the path to the carto file. - * - * @return int success or error. - */ -static int opal_carto_file_parse(const char *cartofile) -{ - int token; - opal_carto_base_node_t *node, *end_node; - uint32_t weight; - char *node1_name, *node2_name, *value, **argv; - int cnt, line_number = 1; - char *token_to_string[] = { - "New-line", - "File-error", - "Node deceleration", - "Connection deceleration", - "bi directional connection", - "Integer", - "Name", - "Node connection", - "Undefined" - "Undefined" - "Undefined" - }; - - /* set the done flag to false. at the end of file the the lexical analyzer will set it to true */ - carto_file_done = false; - carto_file_in = fopen(cartofile, "r"); /* open the carto file */ - /* if the file not found, return an error */ - if (NULL == carto_file_in) { - opal_show_help("help-opal-carto-file.txt", "file not found", true, cartofile); - return OPAL_ERR_NOT_FOUND; - } - while (!carto_file_done) { - token = carto_file_lex(); - switch (token) { - case OPAL_CARTO_FILE_NEWLINE: - line_number++; - break; - case OPAL_CARTO_FILE_NODE_DECELERATION: - token = carto_file_lex(); - switch (token) { - case OPAL_CARTO_FILE_NAME: - node = (opal_carto_base_node_t *)malloc(sizeof(opal_carto_base_node_t)); - node->node_type = strdup(carto_file_value.sval); - if (0 == strcmp("socket",node->node_type)) { - node->is_cpu = true; - } - else { - node->is_cpu = false; - } - token = carto_file_lex(); - switch (token) { - case OPAL_CARTO_FILE_NAME: - node->node_name = strdup(carto_file_value.sval); - opal_carto_base_graph_add_node_fn(opal_carto_base_common_host_graph, node); - break; - default: - free(node); - opal_show_help("help-opal-carto-file.txt", "expected node name", - true, cartofile, line_number, token_to_string[token]); - goto error; - } - break; - default: - opal_show_help("help-opal-carto-file.txt", "expected node type", - true, cartofile, line_number, token_to_string[token]); - goto error; - } - break; - case OPAL_CARTO_FILE_CONNECTION_DECELERATION: - token = carto_file_lex(); - switch (token) { - case OPAL_CARTO_FILE_NAME: - node1_name = strdup(carto_file_value.sval); - while (OPAL_CARTO_FILE_NEWLINE != token) { - token = carto_file_lex(); - switch (token) { - case OPAL_CARTO_FILE_NODE_CONNECTION: - value = carto_file_value.sval; - argv = opal_argv_split (value, ':'); - cnt = opal_argv_count (argv); - if (2 == cnt) { - node2_name = strdup(argv[0]); - weight = atoi(argv[1]); - } else { - opal_show_help("help-opal-carto-file.txt", "incorrect connection", true, cartofile, line_number, value); - opal_argv_free (argv); - free(node1_name); - goto error; - } - opal_argv_free (argv); - /* find the start node of the connection */ - node = opal_carto_base_graph_find_node_fn(opal_carto_base_common_host_graph,node1_name); - if (NULL == node) { - opal_show_help("help-opal-carto-file.txt", "vertex not found", true, cartofile, line_number, node1_name); - free(node1_name); - free(node2_name); - goto error; - } - /* find the end node of the connection */ - end_node = opal_carto_base_graph_find_node_fn(opal_carto_base_common_host_graph,node2_name); - if (NULL == end_node) { - opal_show_help("help-opal-carto-file.txt", "vertex not found", true, cartofile, line_number, node2_name); - free(node1_name); - free(node2_name); - goto error; - } - opal_carto_base_connect_nodes_fn(opal_carto_base_common_host_graph, node, end_node, weight); - free(node2_name); - break; - case OPAL_CARTO_FILE_NEWLINE: - line_number++; - break; - default: - opal_show_help("help-opal-carto-file.txt", "expected Connection", - true, cartofile, line_number, token_to_string[token]); - free(node1_name); - goto error; - } - } - free(node1_name); - break; - default: - opal_show_help("help-opal-carto-file.txt", "expected node name", - true, cartofile, line_number, token_to_string[token]); - goto error; - } - break; - case OPAL_CARTO_FILE_BIDIRECTION_CONNECTION: - token = carto_file_lex(); - switch (token) { - case OPAL_CARTO_FILE_NAME: - node1_name = strdup(carto_file_value.sval); - while (OPAL_CARTO_FILE_NEWLINE != token) { - token = carto_file_lex(); - switch (token) { - case OPAL_CARTO_FILE_NODE_CONNECTION: - value = carto_file_value.sval; - argv = opal_argv_split (value, ':'); - cnt = opal_argv_count (argv); - if (2 == cnt) { - node2_name = strdup(argv[0]); - weight = atoi(argv[1]); - } else { - opal_show_help("help-opal-carto-file.txt", "incorrect connection", true, cartofile, line_number, value); - opal_argv_free (argv); - free(node1_name); - goto error; - } - opal_argv_free (argv); - /* find the start node of the connection */ - node = opal_carto_base_graph_find_node_fn(opal_carto_base_common_host_graph,node1_name); - if (NULL == node) { - opal_show_help("help-opal-carto-file.txt", "vertex not found", true, cartofile, line_number, node1_name); - free(node1_name); - free(node2_name); - goto error; - } - /* find the end node of the connection */ - end_node = opal_carto_base_graph_find_node_fn(opal_carto_base_common_host_graph,node2_name); - if (NULL == end_node) { - opal_show_help("help-opal-carto-file.txt", "vertex not found", true, cartofile, line_number, node2_name); - free(node1_name); - free(node2_name); - goto error; - } - opal_carto_base_connect_nodes_fn(opal_carto_base_common_host_graph, node, end_node, weight); - opal_carto_base_connect_nodes_fn(opal_carto_base_common_host_graph, end_node, node, weight); - free(node2_name); - break; - case OPAL_CARTO_FILE_NEWLINE: - line_number++; - break; - default: - opal_show_help("help-opal-carto-file.txt", "expected Connection", - true, cartofile, line_number, token_to_string[token]); - free(node1_name); - goto error; - } - } - free(node1_name); - break; - default: - opal_show_help("help-opal-carto-file.txt", "expected node name", - true, cartofile, line_number, token_to_string[token]); - goto error; - } - break; - default: - opal_show_help("help-opal-carto-file.txt", "expected deceleration", - true, cartofile, line_number, token_to_string[token]); - goto error; - - } - } - return OPAL_SUCCESS; -error: - fclose(carto_file_in); - return OPAL_ERR_BAD_PARAM; -} - - - - diff --git a/opal/mca/carto/file/help-opal-carto-file.txt b/opal/mca/carto/file/help-opal-carto-file.txt deleted file mode 100644 index 9969ff74d2..0000000000 --- a/opal/mca/carto/file/help-opal-carto-file.txt +++ /dev/null @@ -1,56 +0,0 @@ -# -*- text -*- -# -# Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana -# University Research and Technology -# Corporation. All rights reserved. -# Copyright (c) 2004-2005 The University of Tennessee and The University -# of Tennessee Research Foundation. All rights -# reserved. -# Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, -# University of Stuttgart. All rights reserved. -# Copyright (c) 2004-2006 The Regents of the University of California. -# All rights reserved. -# Copyright (c) 2006-2007 Cisco Systems, Inc. All rights reserved. -# Copyright (c) 2007 Mellanox Technologies. All rights reserved. -# $COPYRIGHT$ -# -# Additional copyrights may follow -# -# $HEADER$ -# -# This is the US/English help file for Open MPI's OpenFabrics support -# (the carto file). -# - -[file not found] -File %s: No such file or directory -# - -[expected node type] -File: %s line: %d expected Edge type (free string). received %s -# - -[expected node name] -File: %s line: %d expected Edge name (free string). received %s -# - -[expected Connection] -File: %s line: %d expected Edge branch (edge name:weight). received %s -# - -[expected deceleration] -File: %s line: %d expected Edge declaration (EDGE) or branch declaration (BRANCH). received %s -# - -[incorrect connection] -File: %s line: %d - %s - incorrect branch -# - -[vertex not found] -File: %s line: %d - Edge %s is not in the graph -# - -[unknown token] -File: %s line: %d - lexical analyzer returned -unknown value - %d -# diff --git a/opal/mca/hwloc/base/Makefile.am b/opal/mca/hwloc/base/Makefile.am index f3c51f64a3..7cb5a9c2bc 100644 --- a/opal/mca/hwloc/base/Makefile.am +++ b/opal/mca/hwloc/base/Makefile.am @@ -1,5 +1,5 @@ # -# Copyright (c) 2011 Cisco Systems, Inc. All rights reserved. +# Copyright (c) 2011-2012 Cisco Systems, Inc. All rights reserved. # $COPYRIGHT$ # # Additional copyrights may follow @@ -22,6 +22,6 @@ if OPAL_HAVE_HWLOC libmca_hwloc_la_SOURCES += \ base/hwloc_base_dt.c \ base/hwloc_base_util.c \ - base/hwloc_base_proc_mempolicy.c + base/hwloc_base_maffinity.c endif diff --git a/opal/mca/hwloc/base/base.h b/opal/mca/hwloc/base/base.h index cea241049a..3dc3a7cbe7 100644 --- a/opal/mca/hwloc/base/base.h +++ b/opal/mca/hwloc/base/base.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2011 Cisco Systems, Inc. All rights reserved. + * Copyright (c) 2011-2012 Cisco Systems, Inc. All rights reserved. * $COPYRIGHT$ * * Additional copyrights may follow @@ -13,7 +13,6 @@ #include "opal_config.h" #include "opal/dss/dss_types.h" -#include "opal/mca/paffinity/paffinity.h" #include "opal/mca/hwloc/hwloc.h" @@ -23,6 +22,8 @@ BEGIN_C_DECLS +/* ******************************************************************** */ + /** * Initialize the hwloc MCA framework * @@ -81,22 +82,19 @@ OPAL_DECLSPEC extern char *opal_hwloc_base_slot_list; OPAL_DECLSPEC extern char *opal_hwloc_base_cpu_set; OPAL_DECLSPEC extern hwloc_cpuset_t opal_hwloc_base_given_cpus; -/** - * Report a bind failure using the normal mechanisms if a component - * fails to bind memory -- according to the value of the - * hwloc_base_bind_failure_action MCA parameter. - */ -OPAL_DECLSPEC int opal_hwloc_base_report_bind_failure(const char *file, - int line, - const char *msg, - int rc); - -OPAL_DECLSPEC opal_paffinity_locality_t opal_hwloc_base_get_relative_locality(hwloc_topology_t topo, +OPAL_DECLSPEC opal_hwloc_locality_t opal_hwloc_base_get_relative_locality(hwloc_topology_t topo, opal_hwloc_level_t level1, unsigned int peer1, opal_hwloc_level_t level2, unsigned int peer2); +/** + * Loads opal_hwloc_my_cpuset (global variable in + * opal/mca/hwloc/hwloc.h) for this process. opal_hwloc_my_cpuset + * will be loaded with this process' binding, or, if the process is + * not bound, use the hwloc root object's (available and online) + * cpuset. + */ OPAL_DECLSPEC void opal_hwloc_base_get_local_cpuset(void); /** @@ -130,7 +128,16 @@ OPAL_DECLSPEC extern opal_hwloc_base_mbfa_t opal_hwloc_base_mbfa; /* some critical helper functions */ OPAL_DECLSPEC int opal_hwloc_base_filter_cpus(hwloc_topology_t topo); + +/** + * Discover / load the hwloc topology (i.e., call hwloc_topology_init() and + * hwloc_topology_load()). + */ OPAL_DECLSPEC int opal_hwloc_base_get_topology(void); + +/** + * Free the hwloc topology. + */ OPAL_DECLSPEC void opal_hwloc_base_free_topology(hwloc_topology_t topo); OPAL_DECLSPEC hwloc_cpuset_t opal_hwloc_base_get_available_cpus(hwloc_topology_t topo, hwloc_obj_t obj); @@ -146,14 +153,25 @@ OPAL_DECLSPEC hwloc_obj_t opal_hwloc_base_get_obj_by_type(hwloc_topology_t topo, OPAL_DECLSPEC unsigned int opal_hwloc_base_get_obj_idx(hwloc_topology_t topo, hwloc_obj_t obj, opal_hwloc_resource_type_t rtype); -OPAL_DECLSPEC void opal_hwloc_base_get_level_and_index(hwloc_cpuset_t cpus, - opal_hwloc_level_t *bind_level, - unsigned int *bind_idx); +OPAL_DECLSPEC hwloc_obj_t opal_hwloc_base_get_level_and_index(hwloc_cpuset_t cpus, + opal_hwloc_level_t *bind_level, + unsigned int *bind_idx); +OPAL_DECLSPEC int opal_hwloc_base_get_local_index(hwloc_obj_type_t type, + unsigned cache_level, + unsigned int *idx); + +/** + * Get the number of pu's under a given hwloc object. + */ OPAL_DECLSPEC unsigned int opal_hwloc_base_get_npus(hwloc_topology_t topo, hwloc_obj_t target); OPAL_DECLSPEC char* opal_hwloc_base_print_binding(opal_binding_policy_t binding); -OPAL_DECLSPEC char* opal_hwloc_base_print_locality(opal_paffinity_locality_t locality); +OPAL_DECLSPEC char* opal_hwloc_base_print_locality(opal_hwloc_locality_t locality); OPAL_DECLSPEC char* opal_hwloc_base_print_level(opal_hwloc_level_t level); + +/** + * Determine if there is a single cpu in a bitmap. + */ OPAL_DECLSPEC bool opal_hwloc_base_single_cpu(hwloc_cpuset_t cpuset); /** @@ -184,6 +202,14 @@ OPAL_DECLSPEC int opal_hwloc_base_report_bind_failure(const char *file, */ OPAL_DECLSPEC int opal_hwloc_base_set_process_membind_policy(void); +OPAL_DECLSPEC int opal_hwloc_base_membind(opal_hwloc_base_memory_segment_t *segs, + size_t count, int node_id); + +OPAL_DECLSPEC int opal_hwloc_base_node_name_to_id(char *node_name, int *id); + +OPAL_DECLSPEC int opal_hwloc_base_memory_set(opal_hwloc_base_memory_segment_t *segments, + size_t num_segments); + /* datatype support */ OPAL_DECLSPEC int opal_hwloc_pack(opal_buffer_t *buffer, const void *src, int32_t num_vals, diff --git a/opal/mca/hwloc/base/hwloc_base_dt.c b/opal/mca/hwloc/base/hwloc_base_dt.c index b43530a62d..09113c4377 100644 --- a/opal/mca/hwloc/base/hwloc_base_dt.c +++ b/opal/mca/hwloc/base/hwloc_base_dt.c @@ -1,5 +1,5 @@ /* - * Copyright (c) 2011 Cisco Systems, Inc. All rights reserved. + * Copyright (c) 2011-2012 Cisco Systems, Inc. All rights reserved. * * $COPYRIGHT$ * @@ -272,19 +272,27 @@ static void print_hwloc_obj(char **output, char *prefix, free(tmp); tmp = tmp2; } - /* print the cpusets */ - hwloc_bitmap_snprintf(string, OPAL_HWLOC_MAX_STRING, obj->cpuset); - asprintf(&tmp2, "%s%sCpuset: %s", tmp, pfx, string); - free(tmp); - tmp = tmp2; - hwloc_bitmap_snprintf(string, OPAL_HWLOC_MAX_STRING, obj->online_cpuset); - asprintf(&tmp2, "%s%sOnline: %s", tmp, pfx, string); - free(tmp); - tmp = tmp2; - hwloc_bitmap_snprintf(string, OPAL_HWLOC_MAX_STRING, obj->allowed_cpuset); - asprintf(&tmp2, "%s%sAllowed: %s", tmp, pfx, string); - free(tmp); - tmp = tmp2; + /* print the cpusets - apparently, some new HWLOC types don't + * have cpusets, so protect ourselves here + */ + if (NULL != obj->cpuset) { + hwloc_bitmap_snprintf(string, OPAL_HWLOC_MAX_STRING, obj->cpuset); + asprintf(&tmp2, "%s%sCpuset: %s", tmp, pfx, string); + free(tmp); + tmp = tmp2; + } + if (NULL != obj->online_cpuset) { + hwloc_bitmap_snprintf(string, OPAL_HWLOC_MAX_STRING, obj->online_cpuset); + asprintf(&tmp2, "%s%sOnline: %s", tmp, pfx, string); + free(tmp); + tmp = tmp2; + } + if (NULL != obj->allowed_cpuset) { + hwloc_bitmap_snprintf(string, OPAL_HWLOC_MAX_STRING, obj->allowed_cpuset); + asprintf(&tmp2, "%s%sAllowed: %s", tmp, pfx, string); + free(tmp); + tmp = tmp2; + } if (HWLOC_OBJ_MACHINE == obj->type) { /* root level object - add support values */ support = (struct hwloc_topology_support*)hwloc_topology_get_support(topo); diff --git a/opal/mca/maffinity/hwloc/maffinity_hwloc_module.c b/opal/mca/hwloc/base/hwloc_base_maffinity.c similarity index 56% rename from opal/mca/maffinity/hwloc/maffinity_hwloc_module.c rename to opal/mca/hwloc/base/hwloc_base_maffinity.c index 398cd5f9f2..98ed295597 100644 --- a/opal/mca/maffinity/hwloc/maffinity_hwloc_module.c +++ b/opal/mca/hwloc/base/hwloc_base_maffinity.c @@ -1,16 +1,5 @@ /* - * Copyright (c) 2004-2008 The Trustees of Indiana University and Indiana - * University Research and Technology - * Corporation. All rights reserved. - * Copyright (c) 2004-2005 The University of Tennessee and The University - * of Tennessee Research Foundation. All rights - * reserved. - * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, - * University of Stuttgart. All rights reserved. - * Copyright (c) 2004-2005 The Regents of the University of California. - * All rights reserved. - * Copyright (c) 2006-2011 Cisco Systems, Inc. All rights reserved. - * + * Copyright (c) 2011-2012 Cisco Systems, Inc. All rights reserved. * $COPYRIGHT$ * * Additional copyrights may follow @@ -18,71 +7,74 @@ * $HEADER$ */ + #include "opal_config.h" -/* This component will only be compiled on Hwloc, where we are - guaranteed to have and friends */ -#include - -#include -#include -#include -#include - #include "opal/constants.h" -#include "opal/mca/base/mca_base_param.h" -#include "opal/mca/maffinity/maffinity.h" -#include "opal/mca/maffinity/base/base.h" -#include "opal/mca/hwloc/base/base.h" -#include "maffinity_hwloc.h" + +#include "opal/mca/hwloc/hwloc.h" #include "opal/mca/hwloc/base/base.h" -/* - * Local functions - */ -static int hwloc_module_init(void); -static int hwloc_module_set(opal_maffinity_base_segment_t *segments, - size_t num_segments); -static int hwloc_module_node_name_to_id(char *, int *); -static int hwloc_module_bind(opal_maffinity_base_segment_t *, size_t, int); /* - * Hwloc maffinity module + * Don't use show_help() here (or print any error message at all). + * Let the upper layer output a relevant message, because doing so may + * be complicated (e.g., this might be called from the ORTE ODLS, + * which has to do some extra steps to get error messages to be + * displayed). */ -static const opal_maffinity_base_module_1_0_0_t local_module = { - /* Initialization function */ - hwloc_module_init, - - /* Module function pointers */ - hwloc_module_set, - hwloc_module_node_name_to_id, - hwloc_module_bind -}; - -int opal_maffinity_hwloc_component_query(mca_base_module_t **module, - int *priority) +int opal_hwloc_base_set_process_membind_policy(void) { + int rc = 0, flags; + hwloc_membind_policy_t policy; + hwloc_cpuset_t cpuset; + + /* Make sure opal_hwloc_topology has been set by the time we've + been called */ if (NULL == opal_hwloc_topology) { - return OPAL_ERROR; + return OPAL_ERR_BAD_PARAM; } - *priority = mca_maffinity_hwloc_component.priority; - *module = (mca_base_module_t *) &local_module; + /* Set the default memory allocation policy according to MCA + param */ + switch (opal_hwloc_base_map) { + case OPAL_HWLOC_BASE_MAP_LOCAL_ONLY: + policy = HWLOC_MEMBIND_BIND; + flags = HWLOC_MEMBIND_STRICT; + break; + + case OPAL_HWLOC_BASE_MAP_NONE: + default: + policy = HWLOC_MEMBIND_DEFAULT; + flags = 0; + break; + } + + cpuset = hwloc_bitmap_alloc(); + if (NULL == cpuset) { + rc = OPAL_ERR_OUT_OF_RESOURCE; + } else { + int e; + hwloc_get_cpubind(opal_hwloc_topology, cpuset, 0); + rc = hwloc_set_membind(opal_hwloc_topology, + cpuset, policy, flags); + e = errno; + hwloc_bitmap_free(cpuset); - return OPAL_SUCCESS; + /* See if hwloc was able to do it. If hwloc failed due to + ENOSYS, but the base_map == NONE, then it's not really an + error. */ + if (0 != rc && ENOSYS == e && + OPAL_HWLOC_BASE_MAP_NONE == opal_hwloc_base_map) { + rc = 0; + } + } + + return (0 == rc) ? OPAL_SUCCESS : OPAL_ERROR; } - -static int hwloc_module_init(void) -{ - /* Nothing to do! */ - - return OPAL_SUCCESS; -} - - -static int hwloc_module_set(opal_maffinity_base_segment_t *segments, - size_t num_segments) +int opal_hwloc_base_memory_set(opal_hwloc_base_memory_segment_t *segments, + size_t num_segments) { int rc = OPAL_SUCCESS; char *msg = NULL; @@ -128,7 +120,7 @@ static int hwloc_module_set(opal_maffinity_base_segment_t *segments, return OPAL_SUCCESS; } -static int hwloc_module_node_name_to_id(char *node_name, int *id) +int opal_hwloc_base_node_name_to_id(char *node_name, int *id) { /* GLB: fix me */ *id = atoi(node_name + 3); @@ -136,8 +128,8 @@ static int hwloc_module_node_name_to_id(char *node_name, int *id) return OPAL_SUCCESS; } -static int hwloc_module_bind(opal_maffinity_base_segment_t *segs, - size_t count, int node_id) +int opal_hwloc_base_membind(opal_hwloc_base_memory_segment_t *segs, + size_t count, int node_id) { size_t i; int rc = OPAL_SUCCESS; diff --git a/opal/mca/hwloc/base/hwloc_base_util.c b/opal/mca/hwloc/base/hwloc_base_util.c index 8ca7afbdb8..628a3a8ea6 100644 --- a/opal/mca/hwloc/base/hwloc_base_util.c +++ b/opal/mca/hwloc/base/hwloc_base_util.c @@ -10,6 +10,8 @@ * Copyright (c) 2004-2005 The Regents of the University of California. * All rights reserved. * Copyright (c) 2011-2012 Cisco Systems, Inc. All rights reserved. + * Copyright (c) 2012 Los Alamos National Security, LLC. + * All rights reserved. * $COPYRIGHT$ * * Additional copyrights may follow @@ -32,7 +34,6 @@ #include "opal/util/argv.h" #include "opal/util/output.h" #include "opal/util/show_help.h" -#include "opal/mca/paffinity/paffinity.h" #include "opal/threads/tsd.h" #include "opal/mca/hwloc/hwloc.h" @@ -277,8 +278,11 @@ void opal_hwloc_base_get_local_cpuset(void) if (NULL == opal_hwloc_my_cpuset) { opal_hwloc_my_cpuset = hwloc_bitmap_alloc(); } + /* get the cpus we are bound to */ - if (0 > hwloc_get_cpubind(opal_hwloc_topology, opal_hwloc_my_cpuset, HWLOC_CPUBIND_PROCESS)) { + if (hwloc_get_cpubind(opal_hwloc_topology, + opal_hwloc_my_cpuset, + HWLOC_CPUBIND_PROCESS) < 0) { /* we are not bound - use the root's available cpuset */ root = hwloc_get_root_obj(opal_hwloc_topology); base_cpus = opal_hwloc_base_get_available_cpus(opal_hwloc_topology, root); @@ -341,6 +345,11 @@ hwloc_cpuset_t opal_hwloc_base_get_available_cpus(hwloc_topology_t topo, return rdata->available; } + /* some hwloc object types don't have cpus */ + if (NULL == obj->online_cpuset || NULL == obj->allowed_cpuset) { + return NULL; + } + /* see if we already have this info */ if (NULL == (data = (opal_hwloc_obj_data_t*)obj->userdata)) { /* nope - create the object */ @@ -620,7 +629,7 @@ static hwloc_obj_t df_search(hwloc_topology_t topo, if (NULL == data->available) { data->available = opal_hwloc_base_get_available_cpus(topo, start); } - if (!hwloc_bitmap_iszero(data->available)) { + if (NULL != data->available && !hwloc_bitmap_iszero(data->available)) { if (NULL != num_objs) { *num_objs += 1; } else if (*idx == nobj) { @@ -1036,9 +1045,9 @@ int opal_hwloc_base_slot_list_parse(const char *slot_str, return OPAL_SUCCESS; } -static opal_paffinity_locality_t get_locality(opal_hwloc_level_t level) +static opal_hwloc_locality_t get_locality(opal_hwloc_level_t level) { - opal_paffinity_locality_t lvl; + opal_hwloc_locality_t lvl; switch(level) { case OPAL_HWLOC_NODE_LEVEL: @@ -1070,13 +1079,13 @@ static opal_paffinity_locality_t get_locality(opal_hwloc_level_t level) return lvl; } -opal_paffinity_locality_t opal_hwloc_base_get_relative_locality(hwloc_topology_t topo, - opal_hwloc_level_t level1, - unsigned int peer1, - opal_hwloc_level_t level2, - unsigned int peer2) +opal_hwloc_locality_t opal_hwloc_base_get_relative_locality(hwloc_topology_t topo, + opal_hwloc_level_t level1, + unsigned int peer1, + opal_hwloc_level_t level2, + unsigned int peer2) { - opal_paffinity_locality_t locality; + opal_hwloc_locality_t locality; hwloc_obj_t obj1, obj2; unsigned cache_level=0; opal_hwloc_level_t i, lvl; @@ -1234,9 +1243,9 @@ static hwloc_obj_t df_search_level(hwloc_obj_t start, return NULL; } -void opal_hwloc_base_get_level_and_index(hwloc_cpuset_t cpus, - opal_hwloc_level_t *bind_level, - unsigned int *bind_idx) +hwloc_obj_t opal_hwloc_base_get_level_and_index(hwloc_cpuset_t cpus, + opal_hwloc_level_t *bind_level, + unsigned int *bind_idx) { hwloc_obj_t root, obj; @@ -1244,7 +1253,7 @@ void opal_hwloc_base_get_level_and_index(hwloc_cpuset_t cpus, if (NULL == opal_hwloc_topology) { *bind_level = OPAL_HWLOC_NODE_LEVEL; *bind_idx = 0; - return; + return NULL; } /* start at the node level and do a down-first @@ -1257,11 +1266,89 @@ void opal_hwloc_base_get_level_and_index(hwloc_cpuset_t cpus, if (NULL == obj) { /* no match found */ - return; + return NULL; } /* get the index */ *bind_idx = opal_hwloc_base_get_obj_idx(opal_hwloc_topology, obj, OPAL_HWLOC_AVAILABLE); + return obj; +} + +int opal_hwloc_base_get_local_index(hwloc_obj_type_t type, + unsigned cache_level, + unsigned int *idx) +{ + opal_hwloc_level_t bind_level; + unsigned int bind_idx; + hwloc_obj_t obj; + + /* if we don't have topology info, nothing we can do */ + if (NULL == opal_hwloc_topology) { + return OPAL_ERR_NOT_AVAILABLE; + } + + /* ensure we have our local cpuset */ + opal_hwloc_base_get_local_cpuset(); + + /* if we are not bound, then this is meaningless */ + obj = opal_hwloc_base_get_level_and_index(opal_hwloc_my_cpuset, + &bind_level, &bind_idx); + if (OPAL_HWLOC_NODE_LEVEL == bind_level) { + return OPAL_ERR_NOT_BOUND; + } + + /* if the type/level match, then we are done */ + if (type == opal_hwloc_levels[bind_level]) { + if (HWLOC_OBJ_CACHE == type) { + if ((cache_level == 1 && bind_level == OPAL_HWLOC_L1CACHE_LEVEL) || + (cache_level == 2 && bind_level == OPAL_HWLOC_L2CACHE_LEVEL) || + (cache_level == 3 && bind_level == OPAL_HWLOC_L3CACHE_LEVEL)) { + *idx = bind_idx; + return OPAL_SUCCESS; + } + } else { + *idx = bind_idx; + return OPAL_SUCCESS; + } + } + + /* if the binding level is below the type, then we cannot + * answer the question as we could run on multiple objects + * of that type - e.g., if we are bound to NUMA and we are + * asked for the idx of the socket we are on, then we can + * only answer "unknown" + */ + if (type > opal_hwloc_levels[bind_level]) { + return OPAL_ERR_MULTIPLE_AFFINITIES; + } + if (type == HWLOC_OBJ_CACHE) { + if ((cache_level == 1 && OPAL_HWLOC_L1CACHE_LEVEL < bind_level) || + (cache_level == 2 && OPAL_HWLOC_L2CACHE_LEVEL < bind_level) || + (cache_level == 3 && OPAL_HWLOC_L3CACHE_LEVEL < bind_level)) { + return OPAL_ERR_MULTIPLE_AFFINITIES; + } + } + + /* move upward until we find the specified type */ + while (NULL != obj) { + obj = obj->parent; + if (obj->type == type) { + if (type == HWLOC_OBJ_CACHE) { + if (cache_level == obj->attr->cache.depth) { + break; + } + } else { + break; + } + } + } + if (NULL == obj) { + return OPAL_ERR_NOT_FOUND; + } + + /* get the index of this object */ + *idx = opal_hwloc_base_get_obj_idx(opal_hwloc_topology, obj, OPAL_HWLOC_AVAILABLE); + return OPAL_SUCCESS; } #define OPAL_HWLOC_PRINT_MAX_SIZE 50 @@ -1415,7 +1502,7 @@ char* opal_hwloc_base_print_level(opal_hwloc_level_t level) return ret; } -char* opal_hwloc_base_print_locality(opal_paffinity_locality_t locality) +char* opal_hwloc_base_print_locality(opal_hwloc_locality_t locality) { opal_hwloc_print_buffers_t *ptr; int idx; diff --git a/opal/mca/hwloc/hwloc.h b/opal/mca/hwloc/hwloc.h index 7abb6d8dc6..016d826ab9 100644 --- a/opal/mca/hwloc/hwloc.h +++ b/opal/mca/hwloc/hwloc.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2011 Cisco Systems, Inc. All rights reserved. + * Copyright (c) 2011-2012 Cisco Systems, Inc. All rights reserved. * * $COPYRIGHT$ * @@ -86,6 +86,24 @@ typedef enum { #if OPAL_HAVE_HWLOC #include MCA_hwloc_IMPLEMENTATION_HEADER +/** + * Struct used to describe a section of memory (starting address + * and length). This is really the same thing as an iovec, but + * we include a separate type for it for at least 2 reasons: + * + * 1. Some OS's iovec definitions are exceedingly lame (e.g., + * Solaris 9 has the length argument as an int, instead of a + * size_t). + * + * 2. We reserve the right to expand/change this struct in the + * future. + */ +typedef struct { + /** Starting address of segment */ + void *mbs_start_addr; + /** Length of segment */ + size_t mbs_len; +} opal_hwloc_base_memory_segment_t; /* define type of processor info requested */ typedef uint8_t opal_hwloc_resource_type_t; @@ -151,6 +169,42 @@ typedef uint16_t opal_binding_policy_t; #define OPAL_BIND_OVERLOAD_ALLOWED(n) \ (OPAL_BIND_ALLOW_OVERLOAD & (n)) +/* ******************************************************************** */ +typedef uint16_t opal_hwloc_locality_t; + +/** Process locality definitions */ +enum { + OPAL_PROC_LOCALITY_UNKNOWN = 0x0000, + OPAL_PROC_NON_LOCAL = 0x8000, + OPAL_PROC_ON_CLUSTER = 0x0400, + OPAL_PROC_ON_CU = 0x0200, + OPAL_PROC_ON_NODE = 0x0100, + OPAL_PROC_ON_BOARD = 0x0080, + OPAL_PROC_ON_NUMA = 0x0040, + OPAL_PROC_ON_SOCKET = 0x0020, + OPAL_PROC_ON_L3CACHE = 0x0010, + OPAL_PROC_ON_L2CACHE = 0x0008, + OPAL_PROC_ON_L1CACHE = 0x0004, + OPAL_PROC_ON_CORE = 0x0002, + OPAL_PROC_ON_HWTHREAD = 0x0001, + OPAL_PROC_ALL_LOCAL = 0x0fff +}; + +/** Process locality macros */ +#define OPAL_PROC_ON_LOCAL_HWTHREAD(n) ((n) & OPAL_PROC_ON_HWTHREAD) +#define OPAL_PROC_ON_LOCAL_CORE(n) ((n) & OPAL_PROC_ON_CORE) +#define OPAL_PROC_ON_LOCAL_L1CACHE(n) ((n) & OPAL_PROC_ON_L1CACHE) +#define OPAL_PROC_ON_LOCAL_L2CACHE(n) ((n) & OPAL_PROC_ON_L2CACHE) +#define OPAL_PROC_ON_LOCAL_L3CACHE(n) ((n) & OPAL_PROC_ON_L3CACHE) +#define OPAL_PROC_ON_LOCAL_SOCKET(n) ((n) & OPAL_PROC_ON_SOCKET) +#define OPAL_PROC_ON_LOCAL_NUMA(n) ((n) & OPAL_PROC_ON_NUMA) +#define OPAL_PROC_ON_LOCAL_BOARD(n) ((n) & OPAL_PROC_ON_BOARD) +#define OPAL_PROC_ON_LOCAL_NODE(n) ((n) & OPAL_PROC_ON_NODE) +#define OPAL_PROC_ON_LOCAL_CU(n) ((n) & OPAL_PROC_ON_CU) +#define OPAL_PROC_ON_LOCAL_CLUSTER(n) ((n) & OPAL_PROC_ON_CLUSTER) + +/* ******************************************************************** */ + /* some global values */ OPAL_DECLSPEC extern hwloc_topology_t opal_hwloc_topology; OPAL_DECLSPEC extern opal_binding_policy_t opal_hwloc_binding_policy; diff --git a/opal/mca/hwloc/hwloc132/configure.m4 b/opal/mca/hwloc/hwloc132/configure.m4 index 369912fdef..cbd10aaea0 100644 --- a/opal/mca/hwloc/hwloc132/configure.m4 +++ b/opal/mca/hwloc/hwloc132/configure.m4 @@ -1,6 +1,6 @@ # -*- shell-script -*- # -# Copyright (c) 2009-2011 Cisco Systems, Inc. All rights reserved. +# Copyright (c) 2009-2012 Cisco Systems, Inc. All rights reserved. # # $COPYRIGHT$ # @@ -109,6 +109,9 @@ AC_DEFUN([MCA_opal_hwloc_hwloc132_CONFIG],[ ["$HWLOC_VERSION"], [Version of hwloc]) + # Do we have verbs support? + AC_CHECK_HEADERS([infiniband/verbs.h]) + # Set these variables so that the framework m4 knows # what file to include in opal/mca/hwloc/hwloc.h opal_hwloc_hwloc132_include="$opal_hwloc_hwloc132_basedir/hwloc132.h" diff --git a/opal/mca/hwloc/hwloc132/hwloc132.h b/opal/mca/hwloc/hwloc132/hwloc132.h index feba298151..2544a506d9 100644 --- a/opal/mca/hwloc/hwloc132/hwloc132.h +++ b/opal/mca/hwloc/hwloc132/hwloc132.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2011 Cisco Systems, Inc. All rights reserved. + * Copyright (c) 2011-2012 Cisco Systems, Inc. All rights reserved. * * $COPYRIGHT$ * @@ -15,8 +15,16 @@ #ifndef MCA_OPAL_HWLOC_HWLOC132_H #define MCA_OPAL_HWLOC_HWLOC132_H +BEGIN_C_DECLS + #include "hwloc/include/hwloc.h" +/* If we have verbs.h, then include the hwloc openfabrics helpers + header file */ +#if defined(HAVE_INFINIBAND_VERBS_H) +#include "hwloc/include/hwloc/openfabrics-verbs.h" +#endif + END_C_DECLS #endif /* MCA_OPAL_HWLOC_HWLOC132_H */ diff --git a/opal/mca/maffinity/Makefile.am b/opal/mca/maffinity/Makefile.am deleted file mode 100644 index 81bedfe104..0000000000 --- a/opal/mca/maffinity/Makefile.am +++ /dev/null @@ -1,37 +0,0 @@ -# -# Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana -# University Research and Technology -# Corporation. All rights reserved. -# Copyright (c) 2004-2005 The University of Tennessee and The University -# of Tennessee Research Foundation. All rights -# reserved. -# Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, -# University of Stuttgart. All rights reserved. -# Copyright (c) 2004-2005 The Regents of the University of California. -# All rights reserved. -# Copyright (c) 2010-2011 Cisco Systems, Inc. All rights reserved. -# $COPYRIGHT$ -# -# Additional copyrights may follow -# -# $HEADER$ -# - -# main library setup -noinst_LTLIBRARIES = libmca_maffinity.la -libmca_maffinity_la_SOURCES = - -# local files -headers = maffinity.h maffinity_types.h -libmca_maffinity_la_SOURCES += $(headers) - -# Conditionally install the header files -if WANT_INSTALL_HEADERS -opaldir = $(includedir)/openmpi/$(subdir) -nobase_opal_HEADERS = $(headers) -endif - -include base/Makefile.am - -distclean-local: - rm -f base/static-components.h diff --git a/opal/mca/maffinity/base/Makefile.am b/opal/mca/maffinity/base/Makefile.am deleted file mode 100644 index a97d9fa28b..0000000000 --- a/opal/mca/maffinity/base/Makefile.am +++ /dev/null @@ -1,27 +0,0 @@ -# -# Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana -# University Research and Technology -# Corporation. All rights reserved. -# Copyright (c) 2004-2005 The University of Tennessee and The University -# of Tennessee Research Foundation. All rights -# reserved. -# Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, -# University of Stuttgart. All rights reserved. -# Copyright (c) 2004-2005 The Regents of the University of California. -# All rights reserved. -# Copyright (c) 2011 Cisco Systems, Inc. All rights reserved. -# $COPYRIGHT$ -# -# Additional copyrights may follow -# -# $HEADER$ -# - -headers += \ - base/base.h - -libmca_maffinity_la_SOURCES += \ - base/maffinity_base_close.c \ - base/maffinity_base_select.c \ - base/maffinity_base_open.c \ - base/maffinity_base_wrappers.c diff --git a/opal/mca/maffinity/base/base.h b/opal/mca/maffinity/base/base.h deleted file mode 100644 index 617d9c3daa..0000000000 --- a/opal/mca/maffinity/base/base.h +++ /dev/null @@ -1,166 +0,0 @@ -/* - * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana - * University Research and Technology - * Corporation. All rights reserved. - * Copyright (c) 2004-2006 The University of Tennessee and The University - * of Tennessee Research Foundation. All rights - * reserved. - * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, - * University of Stuttgart. All rights reserved. - * Copyright (c) 2004-2005 The Regents of the University of California. - * All rights reserved. - * Copyright (c) 2011 Cisco Systems, Inc. All rights reserved. - * $COPYRIGHT$ - * - * Additional copyrights may follow - * - * $HEADER$ - * - */ - -#ifndef OPAL_MAFFINITY_BASE_H -#define OPAL_MAFFINITY_BASE_H - -#include "opal_config.h" - -#include "opal/mca/maffinity/maffinity.h" - - -/* - * Global functions for MCA overall maffinity open and close - */ - -BEGIN_C_DECLS - -/** - * Initialize the maffinity MCA framework - * - * @retval OPAL_SUCCESS Upon success - * @retval OPAL_ERROR Upon failure - * - * This must be the first function invoked in the maffinity MCA - * framework. It initializes the maffinity MCA framework, finds - * and opens maffinity components, etc. - * - * This function fills in the internal global variable - * opal_maffinity_base_components_opened, which is a list of all - * maffinity components that were successfully opened. This - * variable should \em only be used by other maffinity base - * functions -- it is not considered a public interface member -- - * and is only mentioned here for completeness. - */ -OPAL_DECLSPEC int opal_maffinity_base_open(void); - -/** - * Select an available component. - * - * @return OPAL_SUCCESS Upon success. - * @return OPAL_NOT_FOUND If no component can be selected. - * @return OPAL_ERROR Upon other failure. - * - * This function invokes the selection process for maffinity - * components, which works as follows: - * - * - If the \em maffinity MCA parameter is not specified, the - * selection set is all available maffinity components. - * - If the \em maffinity MCA parameter is specified, the - * selection set is just that component. - * - All components in the selection set are queried to see if - * they want to run. All components that want to run are ranked - * by their priority and the highest priority component is - * selected. All non-selected components have their "close" - * function invoked to let them know that they were not selected. - * - The selected component will have its "init" function invoked to - * let it know that it was selected. - * - * If we fall through this entire process and no component is - * selected, then return OPAL_NOT_FOUND (this is not a fatal - * error). - * - * At the end of this process, we'll either have a single - * component that is selected and initialized, or no component was - * selected. If no component was selected, subsequent invocation - * of the maffinity wrapper functions will return an error. - */ -OPAL_DECLSPEC int opal_maffinity_base_select(void); - -/** - * Set memory affinity. - * - * @param segments Array describing segments and what process they - * belong to - * @param num_segments Length of the segments array - * @param am_allocator True if this process created the shared - * memory block - * - * @retval OPAL_SUCCESS upon success - * @retval OPAL_NOT_FOUND if no maffinity components are available. - * @retval OPAL_ERROR upon other error. - * - * Set the affinity of the memory segments described in the \em - * segments array. - * - * If no maffinity components were available, or if the - * opal_maffinity_base_select() was never invoked, OPAL_NOT_FOUND - * is returned. - */ -OPAL_DECLSPEC int opal_maffinity_base_set(opal_maffinity_base_segment_t *segments, size_t num_segments); - -OPAL_DECLSPEC int opal_maffinity_base_node_name_to_id(char *, int *); -OPAL_DECLSPEC int opal_maffinity_base_bind(opal_maffinity_base_segment_t *, size_t, int); - -/** - * Shut down the maffinity MCA framework. - * - * @retval OPAL_SUCCESS Always - * - * This function shuts down everything in the maffinity MCA - * framework. - * - * It must be the last function invoked on the maffinity MCA - * framework. - */ -OPAL_DECLSPEC int opal_maffinity_base_close(void); - -/** - * Indication of whether a component was successfully selected or - * not - */ -extern bool opal_maffinity_base_selected; - -/** - * Global component struct for the selected component - */ -OPAL_DECLSPEC extern const opal_maffinity_base_component_2_0_0_t -*opal_maffinity_base_component; -/** - * Global module struct for the selected module - */ -OPAL_DECLSPEC extern const opal_maffinity_base_module_1_0_0_t -*opal_maffinity_base_module; - -/** - * Indicator as to whether the list of opened maffinity components - * is valid or not. - */ -extern bool opal_maffinity_base_components_opened_valid; -/** - * List of all opened components; created when the maffinity - * framework is initialized and destroyed when we reduce the list - * to all available maffinity components. - */ -OPAL_DECLSPEC extern opal_list_t opal_maffinity_base_components_opened; - -/** - * Debugging output stream - */ -extern int opal_maffinity_base_output; - -/** - * Flag to indicate whether or not maffinity was setup - */ -OPAL_DECLSPEC extern bool opal_maffinity_setup; - -END_C_DECLS - -#endif /* OPAL_MAFFINITY_BASE_H */ diff --git a/opal/mca/maffinity/base/maffinity_base_close.c b/opal/mca/maffinity/base/maffinity_base_close.c deleted file mode 100644 index c15800868a..0000000000 --- a/opal/mca/maffinity/base/maffinity_base_close.c +++ /dev/null @@ -1,42 +0,0 @@ -/* - * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana - * University Research and Technology - * Corporation. All rights reserved. - * Copyright (c) 2004-2006 The University of Tennessee and The University - * of Tennessee Research Foundation. All rights - * reserved. - * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, - * University of Stuttgart. All rights reserved. - * Copyright (c) 2004-2005 The Regents of the University of California. - * All rights reserved. - * $COPYRIGHT$ - * - * Additional copyrights may follow - * - * $HEADER$ - */ - -#include "opal_config.h" - -#include "opal/constants.h" -#include "opal/mca/mca.h" -#include "opal/mca/base/base.h" -#include "opal/mca/maffinity/maffinity.h" -#include "opal/mca/maffinity/base/base.h" - -int opal_maffinity_base_close(void) -{ - /* Close all components that are still open (this should only - happen during ompi_info). */ - - if (opal_maffinity_base_components_opened_valid) { - mca_base_components_close(opal_maffinity_base_output, - &opal_maffinity_base_components_opened, NULL); - OBJ_DESTRUCT(&opal_maffinity_base_components_opened); - opal_maffinity_base_components_opened_valid = false; - } - - /* All done */ - - return OPAL_SUCCESS; -} diff --git a/opal/mca/maffinity/base/maffinity_base_open.c b/opal/mca/maffinity/base/maffinity_base_open.c deleted file mode 100644 index 905036c1ff..0000000000 --- a/opal/mca/maffinity/base/maffinity_base_open.c +++ /dev/null @@ -1,93 +0,0 @@ -/* - * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana - * University Research and Technology - * Corporation. All rights reserved. - * Copyright (c) 2004-2005 The University of Tennessee and The University - * of Tennessee Research Foundation. All rights - * reserved. - * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, - * University of Stuttgart. All rights reserved. - * Copyright (c) 2004-2005 The Regents of the University of California. - * All rights reserved. - * Copyright (c) 2011 Cisco Systems, Inc. All rights reserved. - * $COPYRIGHT$ - * - * Additional copyrights may follow - * - * $HEADER$ - */ - - -#include "opal_config.h" - -#include -#ifdef HAVE_SYS_TYPES_H -#include -#endif -#ifdef HAVE_UNISTD_H -#include -#endif - -#include "opal/constants.h" -#include "opal/util/output.h" -#include "opal/util/show_help.h" -#include "opal/mca/mca.h" -#include "opal/mca/base/base.h" -#include "opal/mca/base/mca_base_param.h" -#include "opal/mca/maffinity/maffinity.h" -#include "opal/mca/maffinity/base/base.h" - - -/* - * The following file was created by configure. It contains extern - * statements and the definition of an array of pointers to each - * component's public mca_base_component_t struct. - */ -#include "opal/mca/maffinity/base/static-components.h" - - -/* - * Globals - */ -int opal_maffinity_base_output = -1; -bool opal_maffinity_base_components_opened_valid = false; -opal_list_t opal_maffinity_base_components_opened; -bool opal_maffinity_setup = false; - -/* - * Function for finding and opening either all MCA components, or the one - * that was specifically requested via a MCA parameter. - */ -int opal_maffinity_base_open(void) -{ - int int_value; - - /* Debugging / verbose output */ - - mca_base_param_reg_int_name("maffinity", "base_verbose", - "Verbosity level of the maffinity framework", - false, false, - 0, &int_value); - if (0 != int_value) { - opal_maffinity_base_output = opal_output_open(NULL); - } else { - opal_maffinity_base_output = -1; - } - - opal_maffinity_base_components_opened_valid = false; - - /* Open up all available components */ - - if (OPAL_SUCCESS != - mca_base_components_open("maffinity", opal_maffinity_base_output, - mca_maffinity_base_static_components, - &opal_maffinity_base_components_opened, - true)) { - return OPAL_ERROR; - } - opal_maffinity_base_components_opened_valid = true; - - /* All done */ - - return OPAL_SUCCESS; -} diff --git a/opal/mca/maffinity/base/maffinity_base_select.c b/opal/mca/maffinity/base/maffinity_base_select.c deleted file mode 100644 index a5816743bd..0000000000 --- a/opal/mca/maffinity/base/maffinity_base_select.c +++ /dev/null @@ -1,71 +0,0 @@ -/* - * Copyright (c) 2004-2008 The Trustees of Indiana University and Indiana - * University Research and Technology - * Corporation. All rights reserved. - * Copyright (c) 2004-2006 The University of Tennessee and The University - * of Tennessee Research Foundation. All rights - * reserved. - * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, - * University of Stuttgart. All rights reserved. - * Copyright (c) 2004-2005 The Regents of the University of California. - * All rights reserved. - * $COPYRIGHT$ - * - * Additional copyrights may follow - * - * $HEADER$ - */ - - -#include "opal_config.h" - -#include "opal/constants.h" -#include "opal/mca/mca.h" -#include "opal/mca/base/base.h" -#include "opal/mca/base/mca_base_param.h" -#include "opal/mca/maffinity/maffinity.h" -#include "opal/mca/maffinity/base/base.h" - -/* - * Globals - */ -bool opal_maffinity_base_selected = false; -const opal_maffinity_base_component_2_0_0_t *opal_maffinity_base_component = NULL; -const opal_maffinity_base_module_1_0_0_t *opal_maffinity_base_module = NULL; - - -int opal_maffinity_base_select(void) -{ - int ret, exit_status = OPAL_SUCCESS; - opal_maffinity_base_component_2_0_0_t *best_component = NULL; - opal_maffinity_base_module_1_0_0_t *best_module = NULL; - - /* - * Select the best component - */ - if( OPAL_SUCCESS != mca_base_select("maffinity", opal_maffinity_base_output, - &opal_maffinity_base_components_opened, - (mca_base_module_t **) &best_module, - (mca_base_component_t **) &best_component) ) { - /* This will only happen if no component was selected */ - exit_status = OPAL_ERR_NOT_FOUND; - goto cleanup; - } - - /* Save the winner */ - opal_maffinity_base_component = best_component; - opal_maffinity_base_module = best_module; - opal_maffinity_base_selected = true; - - /* Initialize the winner */ - if (NULL != opal_maffinity_base_module) { - if (OPAL_SUCCESS != (ret = opal_maffinity_base_module->maff_module_init()) ) { - exit_status = ret; - goto cleanup; - } - } - - cleanup: - return exit_status; -} - diff --git a/opal/mca/maffinity/base/maffinity_base_wrappers.c b/opal/mca/maffinity/base/maffinity_base_wrappers.c deleted file mode 100644 index eef5c7dbbe..0000000000 --- a/opal/mca/maffinity/base/maffinity_base_wrappers.c +++ /dev/null @@ -1,63 +0,0 @@ -/* - * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana - * University Research and Technology - * Corporation. All rights reserved. - * Copyright (c) 2004-2006 The University of Tennessee and The University - * of Tennessee Research Foundation. All rights - * reserved. - * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, - * University of Stuttgart. All rights reserved. - * Copyright (c) 2004-2005 The Regents of the University of California. - * All rights reserved. - * $COPYRIGHT$ - * - * Additional copyrights may follow - * - * $HEADER$ - */ - - -#include "opal_config.h" - -#include "opal/constants.h" -#include "opal/mca/maffinity/maffinity.h" -#include "opal/mca/maffinity/base/base.h" - -int opal_maffinity_base_set(opal_maffinity_base_segment_t *segments, - size_t num_segments) -{ - if (!opal_maffinity_base_selected) { - return OPAL_ERR_NOT_FOUND; - } - return opal_maffinity_base_module->maff_module_set(segments, num_segments); -} - -int opal_maffinity_base_node_name_to_id(char *node_name, int *node_id) -{ - if (!opal_maffinity_base_selected) { - return OPAL_ERR_NOT_FOUND; - } - - if (!opal_maffinity_base_module->maff_module_name_to_id) { - *node_id = 0; - return OPAL_ERR_NOT_IMPLEMENTED; - } - - return opal_maffinity_base_module->maff_module_name_to_id(node_name, - node_id); -} - -int opal_maffinity_base_bind(opal_maffinity_base_segment_t *segments, - size_t num_segments, int node_id) -{ - if (!opal_maffinity_base_selected) { - return OPAL_ERR_NOT_FOUND; - } - - if (!opal_maffinity_base_module->maff_module_bind) { - return OPAL_ERR_NOT_IMPLEMENTED; - } - - return opal_maffinity_base_module->maff_module_bind(segments, num_segments, - node_id); -} diff --git a/opal/mca/maffinity/first_use/.windows b/opal/mca/maffinity/first_use/.windows deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/opal/mca/maffinity/first_use/Makefile.am b/opal/mca/maffinity/first_use/Makefile.am deleted file mode 100644 index c864e5a413..0000000000 --- a/opal/mca/maffinity/first_use/Makefile.am +++ /dev/null @@ -1,46 +0,0 @@ -# -# Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana -# University Research and Technology -# Corporation. All rights reserved. -# Copyright (c) 2004-2005 The University of Tennessee and The University -# of Tennessee Research Foundation. All rights -# reserved. -# Copyright (c) 2004-2009 High Performance Computing Center Stuttgart, -# University of Stuttgart. All rights reserved. -# Copyright (c) 2004-2005 The Regents of the University of California. -# All rights reserved. -# Copyright (c) 2010 Cisco Systems, Inc. All rights reserved. -# $COPYRIGHT$ -# -# Additional copyrights may follow -# -# $HEADER$ -# - -EXTRA_DIST = .windows - -sources = \ - maffinity_first_use.h \ - maffinity_first_use_component.c \ - maffinity_first_use_module.c - -# Make the output library in this directory, and name it either -# mca__.la (for DSO builds) or libmca__.la -# (for static builds). - -if MCA_BUILD_opal_maffinity_first_use_DSO -component_noinst = -component_install = mca_maffinity_first_use.la -else -component_noinst = libmca_maffinity_first_use.la -component_install = -endif - -mcacomponentdir = $(pkglibdir) -mcacomponent_LTLIBRARIES = $(component_install) -mca_maffinity_first_use_la_SOURCES = $(sources) -mca_maffinity_first_use_la_LDFLAGS = -module -avoid-version - -noinst_LTLIBRARIES = $(component_noinst) -libmca_maffinity_first_use_la_SOURCES =$(sources) -libmca_maffinity_first_use_la_LDFLAGS = -module -avoid-version diff --git a/opal/mca/maffinity/first_use/maffinity_first_use.h b/opal/mca/maffinity/first_use/maffinity_first_use.h deleted file mode 100644 index b2406a9c86..0000000000 --- a/opal/mca/maffinity/first_use/maffinity_first_use.h +++ /dev/null @@ -1,55 +0,0 @@ -/* - * Copyright (c) 2004-2008 The Trustees of Indiana University and Indiana - * University Research and Technology - * Corporation. All rights reserved. - * Copyright (c) 2004-2006 The University of Tennessee and The University - * of Tennessee Research Foundation. All rights - * reserved. - * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, - * University of Stuttgart. All rights reserved. - * Copyright (c) 2004-2005 The Regents of the University of California. - * All rights reserved. - * $COPYRIGHT$ - * - * Additional copyrights may follow - * - * $HEADER$ - */ - -/** - * @file - * - * Processor affinity for First_Use. - */ - - -#ifndef MCA_MAFFINITY_FIRST_USE_EXPORT_H -#define MCA_MAFFINITY_FIRST_USE_EXPORT_H - -#include "opal_config.h" - -#include "opal/mca/mca.h" -#include "opal/mca/maffinity/maffinity.h" - - -BEGIN_C_DECLS - - /** - * Globally exported variable - */ - OPAL_MODULE_DECLSPEC extern const opal_maffinity_base_component_2_0_0_t - mca_maffinity_first_use_component; - - - /** - * maffinity query API function - * - * Query function for maffinity components. Simply returns a priority - * to rank it against other available maffinity components (assumedly, - * only one component will be available per platform, but it's - * possible that there could be more than one available). - */ - int opal_maffinity_first_use_component_query(mca_base_module_t **module, int *priority); - -END_C_DECLS -#endif /* MCA_MAFFINITY_FIRST_USE_EXPORT_H */ diff --git a/opal/mca/maffinity/first_use/maffinity_first_use_component.c b/opal/mca/maffinity/first_use/maffinity_first_use_component.c deleted file mode 100644 index 5740ec3cf8..0000000000 --- a/opal/mca/maffinity/first_use/maffinity_first_use_component.c +++ /dev/null @@ -1,75 +0,0 @@ -/* - * Copyright (c) 2004-2008 The Trustees of Indiana University and Indiana - * University Research and Technology - * Corporation. All rights reserved. - * Copyright (c) 2004-2005 The University of Tennessee and The University - * of Tennessee Research Foundation. All rights - * reserved. - * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, - * University of Stuttgart. All rights reserved. - * Copyright (c) 2004-2005 The Regents of the University of California. - * All rights reserved. - * $COPYRIGHT$ - * - * Additional copyrights may follow - * - * $HEADER$ - */ - -#include "opal_config.h" - -#include "opal/constants.h" -#include "opal/mca/maffinity/maffinity.h" -#include "maffinity_first_use.h" - -/* - * Public string showing the maffinity ompi_first_use component version number - */ -const char *opal_maffinity_first_use_component_version_string = - "OPAL first_use maffinity MCA component version " OPAL_VERSION; - -/* - * Local function - */ -static int first_use_open(void); - -/* - * Instantiate the public struct with all of our public information - * and pointers to our public functions in it - */ - -const opal_maffinity_base_component_2_0_0_t mca_maffinity_first_use_component = { - - /* First, the mca_component_t struct containing meta information - about the component itself */ - - { - OPAL_MAFFINITY_BASE_VERSION_2_0_0, - - /* Component name and version */ - "first_use", - OPAL_MAJOR_VERSION, - OPAL_MINOR_VERSION, - OPAL_RELEASE_VERSION, - - /* Component open and close functions */ - first_use_open, - NULL, - opal_maffinity_first_use_component_query - }, - { - /* The component is checkpoint ready */ - MCA_BASE_METADATA_PARAM_CHECKPOINT - } -}; - - -static int first_use_open(void) -{ - mca_base_param_reg_int(&mca_maffinity_first_use_component.base_version, - "priority", - "Priority of the first_use maffinity component", - false, false, 10, NULL); - - return OPAL_SUCCESS; -} diff --git a/opal/mca/maffinity/first_use/maffinity_first_use_module.c b/opal/mca/maffinity/first_use/maffinity_first_use_module.c deleted file mode 100644 index 58feb0b979..0000000000 --- a/opal/mca/maffinity/first_use/maffinity_first_use_module.c +++ /dev/null @@ -1,99 +0,0 @@ -/* - * Copyright (c) 2004-2008 The Trustees of Indiana University and Indiana - * University Research and Technology - * Corporation. All rights reserved. - * Copyright (c) 2004-2005 The University of Tennessee and The University - * of Tennessee Research Foundation. All rights - * reserved. - * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, - * University of Stuttgart. All rights reserved. - * Copyright (c) 2004-2005 The Regents of the University of California. - * All rights reserved. - * Copyright (c) 2009 Cisco Systems, Inc. All rights reserved. - * $COPYRIGHT$ - * - * Additional copyrights may follow - * - * $HEADER$ - */ - -#include "opal_config.h" - -#include -#include -#ifdef HAVE_UNISTD_H -#include -#endif - -#include "opal/constants.h" -#include "opal/mca/maffinity/maffinity.h" -#include "opal/mca/maffinity/base/base.h" -#include "maffinity_first_use.h" - -/* - * Local functions - */ -static int first_use_module_init(void); -static int first_use_module_set(opal_maffinity_base_segment_t *segments, - size_t num_segments); - -/* - * First_Use maffinity module - */ -static const opal_maffinity_base_module_1_0_0_t loc_module = { - /* Initialization function */ - first_use_module_init, - - /* Module function pointers */ - first_use_module_set, - NULL, - NULL -}; - -int opal_maffinity_first_use_component_query(mca_base_module_t **module, int *priority) -{ - int param; - - param = mca_base_param_find("maffinity", "first_use", "priority"); - mca_base_param_lookup_int(param, priority); - - *module = (mca_base_module_t *)&loc_module; - - return OPAL_SUCCESS; -} - - -static int first_use_module_init(void) -{ - /* Nothing to do */ - - return OPAL_SUCCESS; -} - - -static int first_use_module_set(opal_maffinity_base_segment_t *segments, - size_t num_segments) -{ - size_t i; - uintptr_t pagesize = (uintptr_t)sysconf(_SC_PAGESIZE); - volatile char useless; - - for (i = 0; i < num_segments; ++i) { - char* ptr = (char*)segments[i].mbs_start_addr; - char* end_ptr = ptr + segments[i].mbs_len; - - /* Let's touch the first byte of the segment. If this is the - * first byte on the memory page, good. If not, at least it - * will not overwrite anything important. - */ - useless = ptr[0]; - /* Compute the address of the first byte on the next page */ - ptr = (char*)((uintptr_t)(ptr + pagesize) & ~pagesize); - while( ptr <= end_ptr) { - useless += ptr[0]; - ptr += pagesize; - }; - } - - return OPAL_SUCCESS; -} diff --git a/opal/mca/maffinity/hwloc/Makefile.am b/opal/mca/maffinity/hwloc/Makefile.am deleted file mode 100644 index 4bcbd43b3c..0000000000 --- a/opal/mca/maffinity/hwloc/Makefile.am +++ /dev/null @@ -1,53 +0,0 @@ -# -# Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana -# University Research and Technology -# Corporation. All rights reserved. -# Copyright (c) 2004-2005 The University of Tennessee and The University -# of Tennessee Research Foundation. All rights -# reserved. -# Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, -# University of Stuttgart. All rights reserved. -# Copyright (c) 2004-2005 The Regents of the University of California. -# All rights reserved. -# Copyright (c) 2010-2011 Cisco Systems, Inc. All rights reserved. -# $COPYRIGHT$ -# -# Additional copyrights may follow -# -# $HEADER$ -# - -# To find hwloc_bottom.h. EMBEDDED flags are for if we OMPI's -# internal hwloc is used; maffinity_hwloc_CPPFLAGS is if we are using -# an external install. -AM_CPPFLAGS = $(HWLOC_EMBEDDED_CPPFLAGS) $(maffinity_hwloc_CPPFLAGS) -# To get the cflags for the stuff in hwloc.h -AM_CFLAGS = $(HWLOC_EMBEDDED_CFLAGS) $(maffinity_hwloc_CFLAGS) - -dist_pkgdata_DATA = help-opal-maffinity-hwloc.txt - -sources = \ - maffinity_hwloc.h \ - maffinity_hwloc_component.c \ - maffinity_hwloc_module.c - -# Make the output library in this directory, and name it either -# mca__.la (for DSO builds) or libmca__.la -# (for static builds). - -if MCA_BUILD_opal_maffinity_hwloc_DSO -component_noinst = -component_install = mca_maffinity_hwloc.la -else -component_noinst = libmca_maffinity_hwloc.la -component_install = -endif - -mcacomponentdir = $(pkglibdir) -mcacomponent_LTLIBRARIES = $(component_install) -mca_maffinity_hwloc_la_SOURCES = $(sources) -mca_maffinity_hwloc_la_LDFLAGS = -module -avoid-version - -noinst_LTLIBRARIES = $(component_noinst) -libmca_maffinity_hwloc_la_SOURCES =$(sources) -libmca_maffinity_hwloc_la_LDFLAGS = -module -avoid-version diff --git a/opal/mca/maffinity/hwloc/configure.m4 b/opal/mca/maffinity/hwloc/configure.m4 deleted file mode 100644 index 7c8d1da622..0000000000 --- a/opal/mca/maffinity/hwloc/configure.m4 +++ /dev/null @@ -1,26 +0,0 @@ -# -*- shell-script -*- -# -# Copyright (c) 2007-2011 Cisco Systems, Inc. All rights reserved. -# -# $COPYRIGHT$ -# -# Additional copyrights may follow -# -# $HEADER$ -# - -# MCA_opal_maffinity_hwloc_CONFIG([action-if-found], [action-if-not-found]) -# ------------------------------------------------------------------------- -AC_DEFUN([MCA_opal_maffinity_hwloc_CONFIG],[ - AC_REQUIRE([MCA_opal_hwloc_CONFIG_REQUIRE]) - AC_CONFIG_FILES([opal/mca/maffinity/hwloc/Makefile]) - - # All we check for is whether $OPAL_HAVE_HWLOC is 1. - # See big comment in opal/mca/hwloc/configure.m4. - AC_MSG_CHECKING([if hwloc is enabled]) - AS_IF([test $OPAL_HAVE_HWLOC -eq 1], - [AC_MSG_RESULT([yes]) - $1], - [AC_MSG_RESULT([no]) - $2]) -])dnl diff --git a/opal/mca/maffinity/hwloc/help-opal-maffinity-hwloc.txt b/opal/mca/maffinity/hwloc/help-opal-maffinity-hwloc.txt deleted file mode 100644 index edc4c8948a..0000000000 --- a/opal/mca/maffinity/hwloc/help-opal-maffinity-hwloc.txt +++ /dev/null @@ -1,27 +0,0 @@ -# -*- text -*- -# -# Copyright (c) 2011 Cisco Systems, Inc. All rights reserved. -# $COPYRIGHT$ -# -# Additional copyrights may follow -# -# $HEADER$ -# -# This is the US/English help file for Open MPI's libnuma support -# -[invalid policy] -WARNING: An invalid value was given for the maffinity_libnuma_policy -MCA parameter. The policy determines what happens when Open MPI tries -to allocate local memory, but no local memory is available. The value -provided was: - - Value: %s - PID: %d - -Valid values are: - - strict: the memory allocation will fail - loose: the memory allocation will spill over to remote memory - -Your job will now abort. -# diff --git a/opal/mca/maffinity/hwloc/maffinity_hwloc.h b/opal/mca/maffinity/hwloc/maffinity_hwloc.h deleted file mode 100644 index 7954776720..0000000000 --- a/opal/mca/maffinity/hwloc/maffinity_hwloc.h +++ /dev/null @@ -1,60 +0,0 @@ -/* - * Copyright (c) 2004-2008 The Trustees of Indiana University and Indiana - * University Research and Technology - * Corporation. All rights reserved. - * Copyright (c) 2004-2005 The University of Tennessee and The University - * of Tennessee Research Foundation. All rights - * reserved. - * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, - * University of Stuttgart. All rights reserved. - * Copyright (c) 2004-2005 The Regents of the University of California. - * All rights reserved. - * Copyright (c) 2006-2011 Cisco Systems, Inc. All rights reserved. - * - * $COPYRIGHT$ - * - * Additional copyrights may follow - * - * $HEADER$ - */ - -#ifndef MCA_MAFFINITY_HWLOC_EXPORT_H -#define MCA_MAFFINITY_HWLOC_EXPORT_H - -#include "opal_config.h" - -#include "opal/mca/mca.h" -#include "opal/mca/maffinity/maffinity.h" -#include "hwloc.h" - - -BEGIN_C_DECLS - -typedef struct { - /* Base maffinity component */ - opal_maffinity_base_component_2_0_0_t base; - - /* This component's data */ - int priority; -} opal_maffinity_hwloc_component_2_0_0_t; - -/** - * Globally exported variable - */ -OPAL_DECLSPEC extern opal_maffinity_hwloc_component_2_0_0_t - mca_maffinity_hwloc_component; - -/** - * maffinity query API function - * - * Query function for maffinity components. Simply returns a priority - * to rank it against other available maffinity components (assumedly, - * only one component will be available per platform, but it's - * possible that there could be more than one available). - */ -int opal_maffinity_hwloc_component_query(mca_base_module_t **module, - int *priority); - -END_C_DECLS - -#endif /* MCA_MAFFINITY_HWLOC_EXPORT_H */ diff --git a/opal/mca/maffinity/hwloc/maffinity_hwloc_component.c b/opal/mca/maffinity/hwloc/maffinity_hwloc_component.c deleted file mode 100644 index 81371e9b5b..0000000000 --- a/opal/mca/maffinity/hwloc/maffinity_hwloc_component.c +++ /dev/null @@ -1,88 +0,0 @@ -/* - * Copyright (c) 2004-2008 The Trustees of Indiana University and Indiana - * University Research and Technology - * Corporation. All rights reserved. - * Copyright (c) 2004-2005 The University of Tennessee and The University - * of Tennessee Research Foundation. All rights - * reserved. - * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, - * University of Stuttgart. All rights reserved. - * Copyright (c) 2004-2005 The Regents of the University of California. - * All rights reserved. - * Copyright (c) 2007-2011 Cisco Systems, Inc. All rights reserved. - * $COPYRIGHT$ - * - * Additional copyrights may follow - * - * $HEADER$ - */ - -#include "opal_config.h" - -#include - -#include "opal/constants.h" -#include "opal/util/show_help.h" -#include "opal/mca/base/mca_base_param.h" -#include "opal/mca/hwloc/hwloc.h" -#include "opal/mca/maffinity/maffinity.h" -#include "maffinity_hwloc.h" - -/* - * Public string showing the maffinity ompi_hwloc component version number - */ -const char *opal_maffinity_hwloc_component_version_string = - "OPAL hwloc maffinity MCA component version " OPAL_VERSION; - -/* - * Local function - */ -static int hwloc_register(void); - -/* - * Instantiate the public struct with all of our public information - * and pointers to our public functions in it - */ - -opal_maffinity_hwloc_component_2_0_0_t mca_maffinity_hwloc_component = { - { - /* First, the mca_component_t struct containing meta information - about the component itself */ - { - OPAL_MAFFINITY_BASE_VERSION_2_0_0, - - /* Component name and version */ - "hwloc", - OPAL_MAJOR_VERSION, - OPAL_MINOR_VERSION, - OPAL_RELEASE_VERSION, - - /* Component open and close functions */ - NULL, - NULL, - opal_maffinity_hwloc_component_query, - hwloc_register, - }, - { - /* The component is checkpoint ready */ - MCA_BASE_METADATA_PARAM_CHECKPOINT - } - }, - - /* Priority */ - 40, - - /* NULL fill the rest of the component data */ -}; - - -static int hwloc_register(void) -{ - mca_base_param_reg_int(&mca_maffinity_hwloc_component.base.base_version, - "priority", - "Priority of the hwloc maffinity component", - false, false, 40, - &mca_maffinity_hwloc_component.priority); - - return OPAL_SUCCESS; -} diff --git a/opal/mca/maffinity/maffinity.h b/opal/mca/maffinity/maffinity.h deleted file mode 100644 index 95c656c291..0000000000 --- a/opal/mca/maffinity/maffinity.h +++ /dev/null @@ -1,137 +0,0 @@ -/* - * Copyright (c) 2004-2008 The Trustees of Indiana University and Indiana - * University Research and Technology - * Corporation. All rights reserved. - * Copyright (c) 2004-2005 The University of Tennessee and The University - * of Tennessee Research Foundation. All rights - * reserved. - * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, - * University of Stuttgart. All rights reserved. - * Copyright (c) 2004-2005 The Regents of the University of California. - * All rights reserved. - * $COPYRIGHT$ - * - * Additional copyrights may follow - * - * $HEADER$ - */ -/** - * @file - * - * maffinity (memory affinity) framework component interface - * definitions. - * - * Intent - * - * Simple component to set memory affinity for pages. Note that this - * framework assumes that processor affinity is being used (it doesn't - * make much sense to use memory affinity unless processes are bound - * to specific processors, otherwise the processes may move around and - * the memory may end up being remote). - * - * maffinity components are typically used with shared memory - * operations, but can be used elsewhere as well. The idea is to get - * memory physically located with the process that is going to use it. - * For memory allocated to a processor-bound process, functions such - * as malloc() do this naturally. However, when working with memory - * shared by multiple processes on a NUMA machine, it can be extremely - * advantageous to ensure that pages containing the data structures - * for a given process are physically local to the processor where - * that process is bound. - * - * One process will allocate a large shared memory block and all will - * need to participate to make pages local to specific processors. - * - * There is one main module function - * (opal_maffinity_base_module_set_fn_t) that takes an array of - * segment descriptions within the block. Each process will get a - * different set of segment descriptions (i.e., the segments belonging - * to that process). Components then do whatever is necessary to make - * pages local to their respective processes (i.e., the processors - * where the processes are running). - */ - -#ifndef OPAL_MAFFINITY_H -#define OPAL_MAFFINITY_H - -#include "opal_config.h" - -#include "opal/mca/mca.h" -#include "opal/mca/base/base.h" -#include "opal/mca/maffinity/maffinity_types.h" - -/** - * Module initialization function. Should return OPAL_SUCCESS. - */ -typedef int (*opal_maffinity_base_module_init_1_0_0_fn_t)(void); - - -/** - * Module function to set memory affinity. Take an array of - * maffinity_base_segment_t instances to describe which memory should - * physically reside with which process. - * - * This function is intended to be invoked by each process immediately - * after they mmap / attach the shared memory. In some cases, it will - * be a no-op for some processes (i.e., machines where a single - * function call in the creating process sets the memory affinity for - * the entire region), but in other cases all processes will need to - * participate (e.g., the first_use component, each each process will - * "touch" the pages that are supposed to be local to them). - */ -typedef int (*opal_maffinity_base_module_set_fn_t) - (opal_maffinity_base_segment_t *segments, size_t num_segments); - -/** - * translate memory node name (such as "mem0") to memory node id - */ -typedef int (*opal_maffinity_base_module_node_name_to_id_fn_t) - (char *node_name, int *node_id); - -/** - * bind memory to node - */ -typedef int (*opal_maffinity_base_module_bind_fn_t) - (opal_maffinity_base_segment_t *segments, size_t num_segments, int node_id); - -/** - * Structure for maffinity components. - */ -struct opal_maffinity_base_component_2_0_0_t { - /** MCA base component */ - mca_base_component_t base_version; - /** MCA base data */ - mca_base_component_data_t base_data; -}; -/** - * Convenience typedef - */ -typedef struct opal_maffinity_base_component_2_0_0_t opal_maffinity_base_component_2_0_0_t; - - -/** - * Structure for maffinity modules - */ -struct opal_maffinity_base_module_1_0_0_t { - /** Module initialization function */ - opal_maffinity_base_module_init_1_0_0_fn_t maff_module_init; - - /** Set memory affinity */ - opal_maffinity_base_module_set_fn_t maff_module_set; - opal_maffinity_base_module_node_name_to_id_fn_t maff_module_name_to_id; - opal_maffinity_base_module_bind_fn_t maff_module_bind; -}; -/** - * Convenience typedef - */ -typedef struct opal_maffinity_base_module_1_0_0_t opal_maffinity_base_module_1_0_0_t; - - -/* - * Macro for use in components that are of type maffinity - */ -#define OPAL_MAFFINITY_BASE_VERSION_2_0_0 \ - MCA_BASE_VERSION_2_0_0, \ - "maffinity", 2, 0, 0 - -#endif /* OPAL_MAFFINITY_H */ diff --git a/opal/mca/maffinity/maffinity_types.h b/opal/mca/maffinity/maffinity_types.h deleted file mode 100644 index 1acc7fe093..0000000000 --- a/opal/mca/maffinity/maffinity_types.h +++ /dev/null @@ -1,59 +0,0 @@ -/* - * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana - * University Research and Technology - * Corporation. All rights reserved. - * Copyright (c) 2004-2005 The University of Tennessee and The University - * of Tennessee Research Foundation. All rights - * reserved. - * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, - * University of Stuttgart. All rights reserved. - * Copyright (c) 2004-2005 The Regents of the University of California. - * All rights reserved. - * $COPYRIGHT$ - * - * Additional copyrights may follow - * - * $HEADER$ - */ -/** - * @file - * - * Common types used in the maffinity framework - */ - -#ifndef OPAL_MAFFINITY_TYPES_H -#define OPAL_MAFFINITY_TYPES_H - -#include "opal_config.h" - -#include - -BEGIN_C_DECLS - -/** - * Struct used with opal_maffinity_base_module_set_fn_t. It - * describes a section of memory (starting address and length). - * This is really the same thing as an iovec, but we include a - * separate type for it for at least 2 reasons: - * - * 1. Some OS's iovec definitions are exceedingly lame (e.g., - * Solaris 9 has the length argument as an int, instead of a - * size_t). - * - * 2. We reserve the right to expand/change this struct in the - * future. - */ -struct opal_maffinity_base_segment_t { - /** Starting address of segment */ - void *mbs_start_addr; - /** Length of segment */ - size_t mbs_len; -}; -/** - * Convenience typedef - */ -typedef struct opal_maffinity_base_segment_t opal_maffinity_base_segment_t; - -END_C_DECLS - -#endif /* OPAL_MAFFINITY_TYPES_H */ diff --git a/opal/mca/paffinity/Makefile.am b/opal/mca/paffinity/Makefile.am deleted file mode 100644 index dd148c66c5..0000000000 --- a/opal/mca/paffinity/Makefile.am +++ /dev/null @@ -1,37 +0,0 @@ -# -# Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana -# University Research and Technology -# Corporation. All rights reserved. -# Copyright (c) 2004-2005 The University of Tennessee and The University -# of Tennessee Research Foundation. All rights -# reserved. -# Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, -# University of Stuttgart. All rights reserved. -# Copyright (c) 2004-2005 The Regents of the University of California. -# All rights reserved. -# Copyright (c) 2010 Cisco Systems, Inc. All rights reserved. -# $COPYRIGHT$ -# -# Additional copyrights may follow -# -# $HEADER$ -# - -# main library setup -noinst_LTLIBRARIES = libmca_paffinity.la -libmca_paffinity_la_SOURCES = - -# local files -headers = paffinity.h -libmca_paffinity_la_SOURCES += $(headers) - -# Conditionally install the header files -if WANT_INSTALL_HEADERS -opaldir = $(includedir)/openmpi/$(subdir) -nobase_opal_HEADERS = $(headers) -endif - -include base/Makefile.am - -distclean-local: - rm -f base/static-components.h diff --git a/opal/mca/paffinity/base/Makefile.am b/opal/mca/paffinity/base/Makefile.am deleted file mode 100644 index e18fcab7bc..0000000000 --- a/opal/mca/paffinity/base/Makefile.am +++ /dev/null @@ -1,27 +0,0 @@ -# -# Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana -# University Research and Technology -# Corporation. All rights reserved. -# Copyright (c) 2004-2005 The University of Tennessee and The University -# of Tennessee Research Foundation. All rights -# reserved. -# Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, -# University of Stuttgart. All rights reserved. -# Copyright (c) 2004-2005 The Regents of the University of California. -# All rights reserved. -# $COPYRIGHT$ -# -# Additional copyrights may follow -# -# $HEADER$ -# - -headers += \ - base/base.h - -libmca_paffinity_la_SOURCES += \ - base/paffinity_base_close.c \ - base/paffinity_base_select.c \ - base/paffinity_base_open.c \ - base/paffinity_base_wrappers.c \ - base/paffinity_base_service.c diff --git a/opal/mca/paffinity/base/base.h b/opal/mca/paffinity/base/base.h deleted file mode 100644 index b71fd4d665..0000000000 --- a/opal/mca/paffinity/base/base.h +++ /dev/null @@ -1,308 +0,0 @@ -/* - * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana - * University Research and Technology - * Corporation. All rights reserved. - * Copyright (c) 2004-2006 The University of Tennessee and The University - * of Tennessee Research Foundation. All rights - * reserved. - * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, - * University of Stuttgart. All rights reserved. - * Copyright (c) 2004-2005 The Regents of the University of California. - * All rights reserved. - * Copyright (c) 2007-2010 Cisco Systems, Inc. All rights reserved. - * Copyright (c) 2010 Oracle and/or its affiliates. All rights reserved. - * $COPYRIGHT$ - * - * Additional copyrights may follow - * - * $HEADER$ - */ - -#ifndef OPAL_PAFFINITY_BASE_H -#define OPAL_PAFFINITY_BASE_H - -#include "opal_config.h" - -#include "opal/mca/paffinity/paffinity.h" - -/* - * Global functions for MCA overall paffinity open and close - */ - -BEGIN_C_DECLS - -/** - * Register MCA params for the paffinity base. - * - * @retval OPAL_SUCCESS Upon success - * - * This function is invoked by opal_register_params(). It registers - * some paffinity-wide MCA parameters. - */ -OPAL_DECLSPEC int opal_paffinity_base_register_params(void); - -/** - * Initialize the paffinity MCA framework - * - * @retval OPAL_SUCCESS Upon success - * @retval OPAL_ERROR Upon failure - * - * This must be the first function invoked in the paffinity MCA - * framework. It initializes the paffinity MCA framework, finds - * and opens paffinity components, etc. - * - * This function is invoked during opal_init(). - * - * This function fills in the internal global variable - * opal_paffinity_base_components_opened, which is a list of all - * paffinity components that were successfully opened. This - * variable should \em only be used by other paffinity base - * functions -- it is not considered a public interface member -- - * and is only mentioned here for completeness. - */ -OPAL_DECLSPEC int opal_paffinity_base_open(void); - -/** - * Select an available component. - * - * @return OPAL_SUCCESS Upon success. - * @return OPAL_NOT_FOUND If no component can be selected. - * @return OPAL_ERROR Upon other failure. - * - * This function invokes the selection process for paffinity components, - * which works as follows: - * - * - If the \em paffinity MCA parameter is not specified, the - * selection set is all available paffinity components. - * - If the \em paffinity MCA parameter is specified, the - * selection set is just that component. - * - All components in the selection set are queried to see if - * they want to run. All components that want to run are ranked - * by their priority and the highest priority component is - * selected. All non-selected components have their "close" - * function invoked to let them know that they were not selected. - * - The selected component will have its "init" function invoked to - * let it know that it was selected. - * - * If we fall through this entire process and no component is - * selected, then return OPAL_NOT_FOUND (this is not a fatal - * error). - * - * At the end of this process, we'll either have a single - * component that is selected and initialized, or no component was - * selected. If no component was selected, subsequent invocation - * of the paffinity wrapper functions will return an error. - */ -OPAL_DECLSPEC int opal_paffinity_base_select(void); - -/** - * Set this process' affinity. - * - * @param cpumask PHYSICAL processor bitmask - * - * @retval OPAL_SUCCESS upon success - * @retval OPAL_NOT_FOUND if no paffinity components are available. - * @retval OPAL_ERROR upon other error. - * - * Set this process' affinity to the PHYSICAL CPU's specified in \em cpumask - * - * If no paffinity components were available, or if the - * opal_paffinity_base_select() was never invoked, OPAL_NOT_FOUND - * is returned. - */ -OPAL_DECLSPEC int opal_paffinity_base_set(opal_paffinity_base_cpu_set_t cpumask); - -/** - * Get this process' affinity. - * - * @param cpumask Pointer to PHYSICAL processor bitmask - * - * @retval OPAL_SUCCESS upon success - * @retval OPAL_NOT_FOUND if no paffinity components are available. - * @retval OPAL_ERROR upon other error. - * - * Get this process' CPU affinity PHYSICAL bitmask and assign - * it to \em cpumask. - * - * If no paffinity components were available, or if the - * opal_paffinity_base_select() was never invoked, OPAL_NOT_FOUND - * is returned and cpumask is zeroed out. - */ -OPAL_DECLSPEC int opal_paffinity_base_get(opal_paffinity_base_cpu_set_t *cpumask); - -/** - * Shut down the paffinity MCA framework. - * - * @retval OPAL_SUCCESS Always - * - * This function shuts down everything in the paffinity MCA - * framework, and is called during opal_finalize(). - * - * It must be the last function invoked on the paffinity MCA - * framework. - */ -OPAL_DECLSPEC int opal_paffinity_base_close(void); - -/** - * Returns mapping of PHYSICAL socket:core -> PHYSICAL processor id. - * - * @param physical_socket - * @param pyshical_core - * @param physical_processor_id - * - * @return int - OPAL_SUCCESS or OPAL_ERR_NOT_SUPPORTED if not - * supported - */ -OPAL_DECLSPEC int opal_paffinity_base_get_map_to_processor_id(int physical_socket, int physical_core, int *physical_processor_id); - -/** - * Provides mapping of PHYSICAL processor id -> PHYSICAL socket:core. - * - * @param physical_processor_id - * @param physical_socket - * @param physical_core - * - * @return int - OPAL_SUCCESS or OPAL_ERR_NOT_SUPPORTED if not - * supported - */ -OPAL_DECLSPEC int opal_paffinity_base_get_map_to_socket_core(int physical_processor_id, int *physical_socket, int *physical_core); - -/** - * Return the number of LOGICAL processors - * - * @param num_logical_processors - * - * @return int - OPAL_SUCCESS or OPAL_ERR_NOT_SUPPORTED if not - * supported - */ -OPAL_DECLSPEC int opal_paffinity_base_get_processor_info(int *num_logical_processors); - -/** - * Return the number of LOGICAL sockets - * - * @param num_logical_sockets - * - * @return int - OPAL_SUCCESS or OPAL_ERR_NOT_SUPPORTED if not - * supported - */ -OPAL_DECLSPEC int opal_paffinity_base_get_socket_info(int *num_logical_sockets); - -/** - * Return the number of LOGICAL cores for a given PHYSICAL socket - * - * @param physical_socket - * @param num_logical_cores - * - * @return int - OPAL_SUCCESS or OPAL_ERR_NOT_SUPPORTED if not - * supported - */ -OPAL_DECLSPEC int opal_paffinity_base_get_core_info(int physical_socket, int *num_logical_cores); - -/** - * Return the PHYSICAL processor id that corresponds to the - * given LOGICAL processor id - * - * @param logical_processor_id - * @param physical_processor_id - * - * @return int - OPAL_SUCCESS or OPAL_ERR_* - */ -OPAL_DECLSPEC int opal_paffinity_base_get_physical_processor_id(int logical_processor_id, int *physical_processor_id); - -/** - * Return the PHYSICAL socket ID that corresponds to the given - * LOGICAL socket ID. - * - * @param logical_socket_id - * @param physical_socket_id - * - * @return int - OPAL_SUCCESS or OPAL_ERR_* - */ -OPAL_DECLSPEC int opal_paffinity_base_get_physical_socket_id(int logical_socket_id, int *physical_socket_id); - -/** - * Return the PHYSICAL core ID that corresponds to the given LOGICAL - * core id on the given PHYSICAL socket ID. - * - * @param physical_socket_id - * @param_logical_core_id - * @param physical_core_id - * - * @return int - OPAL_SUCCESS or OPAL_ERR_* - */ -OPAL_DECLSPEC int opal_paffinity_base_get_physical_core_id(int physical_socket_id, int logical_core_id, int *physical_core_id); - -/* Print a char string representation of a cpu set */ -OPAL_DECLSPEC char *opal_paffinity_base_print_binding(opal_paffinity_base_cpu_set_t cpumask); - -/* Parse the binding string created by above function back into a cpu set */ -OPAL_DECLSPEC int opal_paffinity_base_parse_binding(char *binding, opal_paffinity_base_cpu_set_t *cpumask); - -/** - * Indication of whether a component was successfully selected or - * not - */ -OPAL_DECLSPEC extern bool opal_paffinity_base_selected; - -/** - * Global component struct for the selected component - */ -OPAL_DECLSPEC extern const opal_paffinity_base_component_2_0_1_t -*opal_paffinity_base_component; -/** - * Global module struct for the selected module - */ -OPAL_DECLSPEC extern const opal_paffinity_base_module_1_1_0_t -*opal_paffinity_base_module; -/** - * Indicator as to whether the list of opened paffinity components - * is valid or not. - */ -OPAL_DECLSPEC extern bool opal_paffinity_base_components_opened_valid; -/** - * List of all opened components; created when the paffinity - * framework is initialized and destroyed when we reduce the list - * to all available paffinity components. - */ -OPAL_DECLSPEC extern opal_list_t opal_paffinity_base_components_opened; -/** - * Assigning slot_list to process - */ -OPAL_DECLSPEC int opal_paffinity_base_slot_list_set(long rank, char *slot_str, - opal_paffinity_base_cpu_set_t *cpumask); - -/** - * Make a prettyprint string for a cset. - */ -OPAL_DECLSPEC int opal_paffinity_base_cset2str(char *str, int len, - opal_paffinity_base_cpu_set_t *cset); - -/** - * Make a prettyprint string for a cset with a map format. - */ -OPAL_DECLSPEC int opal_paffinity_base_cset2mapstr(char *str, int len, - opal_paffinity_base_cpu_set_t *cset); - -/** - * Debugging output stream - */ -OPAL_DECLSPEC extern int opal_paffinity_base_output; - -/** - * Flag indicating whether or not processor affinity is to be enabled - */ -OPAL_DECLSPEC extern bool opal_paffinity_alone; - -OPAL_DECLSPEC extern char *opal_paffinity_base_slot_list; - -/* Indicator that process affinity has been set for this process - i.e., - * that OPAL set it either directly or when launched - */ -OPAL_DECLSPEC extern bool opal_paffinity_base_bound; - -/* String passed down from launcher that contains applied binding */ -OPAL_DECLSPEC extern char *opal_paffinity_base_applied_binding; - -END_C_DECLS - -#endif /* OPAL_BASE_PAFFINITY_H */ diff --git a/opal/mca/paffinity/base/paffinity_base_close.c b/opal/mca/paffinity/base/paffinity_base_close.c deleted file mode 100644 index 95b3dfa4dd..0000000000 --- a/opal/mca/paffinity/base/paffinity_base_close.c +++ /dev/null @@ -1,50 +0,0 @@ -/* - * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana - * University Research and Technology - * Corporation. All rights reserved. - * Copyright (c) 2004-2006 The University of Tennessee and The University - * of Tennessee Research Foundation. All rights - * reserved. - * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, - * University of Stuttgart. All rights reserved. - * Copyright (c) 2004-2005 The Regents of the University of California. - * All rights reserved. - * Copyright (c) 2007 Cisco Systems, Inc. All rights reserved. - * $COPYRIGHT$ - * - * Additional copyrights may follow - * - * $HEADER$ - */ - -#include "opal_config.h" - -#include "opal/constants.h" -#include "opal/mca/mca.h" -#include "opal/mca/base/base.h" -#include "opal/mca/paffinity/paffinity.h" -#include "opal/mca/paffinity/base/base.h" - -int opal_paffinity_base_close(void) -{ - /* If there is a selected paffinity module, finalize it */ - - if (NULL != opal_paffinity_base_module && - NULL != opal_paffinity_base_module->paff_module_finalize) { - opal_paffinity_base_module->paff_module_finalize(); - } - - /* Close all components that are still open (this should only - happen during ompi_info). */ - - if (opal_paffinity_base_components_opened_valid) { - mca_base_components_close(opal_paffinity_base_output, - &opal_paffinity_base_components_opened, NULL); - OBJ_DESTRUCT(&opal_paffinity_base_components_opened); - opal_paffinity_base_components_opened_valid = false; - } - - /* All done */ - - return OPAL_SUCCESS; -} diff --git a/opal/mca/paffinity/base/paffinity_base_open.c b/opal/mca/paffinity/base/paffinity_base_open.c deleted file mode 100644 index 402d5ae75f..0000000000 --- a/opal/mca/paffinity/base/paffinity_base_open.c +++ /dev/null @@ -1,140 +0,0 @@ -/* - * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana - * University Research and Technology - * Corporation. All rights reserved. - * Copyright (c) 2004-2005 The University of Tennessee and The University - * of Tennessee Research Foundation. All rights - * reserved. - * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, - * University of Stuttgart. All rights reserved. - * Copyright (c) 2004-2005 The Regents of the University of California. - * All rights reserved. - * Copyright (c) 2008-2010 Cisco Systems, Inc. All rights reserved. - * $COPYRIGHT$ - * - * Additional copyrights may follow - * - * $HEADER$ - */ - - -#include "opal_config.h" - -#include "opal/constants.h" -#include "opal/util/output.h" -#include "opal/util/show_help.h" - -#include "opal/mca/mca.h" -#include "opal/mca/base/base.h" -#include "opal/mca/base/mca_base_param.h" -#include "opal/mca/paffinity/paffinity.h" -#include "opal/mca/paffinity/base/base.h" - - -/* - * The following file was created by configure. It contains extern - * statements and the definition of an array of pointers to each - * component's public mca_base_component_t struct. - */ -#include "opal/mca/paffinity/base/static-components.h" - - -/* - * Globals - */ -OPAL_DECLSPEC int opal_paffinity_base_output = -1; -bool opal_paffinity_base_components_opened_valid = false; -opal_list_t opal_paffinity_base_components_opened; -bool opal_paffinity_alone = false; -char *opal_paffinity_base_slot_list; -bool opal_paffinity_base_bound; -char *opal_paffinity_base_applied_binding; -/* - * Register some paffinity-wide MCA params - */ -int opal_paffinity_base_register_params(void) -{ - int value, id; - static int been_here = 0; - - /* We may get called twice; be harmless in that case. */ - if (1 == been_here) { - return OPAL_SUCCESS; - } - been_here = 1; - - /* Debugging / verbose output */ - - mca_base_param_reg_int_name("paffinity", "base_verbose", - "Verbosity level of the paffinity framework", - false, false, - 0, &value); - if (0 != value) { - opal_paffinity_base_output = opal_output_open(NULL); - } else { - opal_paffinity_base_output = -1; - } - - /*** DEPRECATED - NO LONGER FUNCTIONAL, BUT MPIRUN - * WILL OUTPUT WARNING FOR NOW - ***/ - id = mca_base_param_reg_int_name("opal", "paffinity_alone", - "If nonzero, assume that this job is the only (set of) process(es) running on each node and bind processes to processors, starting with processor ID 0", - false, false, - -1, NULL); - /* register the historical mpi_paffinity_alone synonym, but don't - * declare it deprecated so we don't scare the users. - * - * Yes, this breaks the abstraction barrier, but as indicated - * on the developer list....live with it. :-) - */ - mca_base_param_reg_syn_name(id, "mpi", "paffinity_alone", false); - mca_base_param_lookup_int(id, &value); - /* we just want to know if someone set it */ - if (-1 != value) { - opal_paffinity_alone = true; - } else { - opal_paffinity_alone = false; - } - - mca_base_param_reg_int_name("paffinity", "base_bound", - "Process affinity was set by an external entity", - true, false, - false, &value); - opal_paffinity_base_bound = OPAL_INT_TO_BOOL(value); - - mca_base_param_reg_string_name("paffinity", "base_applied_binding", - "Binding from launcher", - true, false, - NULL, &opal_paffinity_base_applied_binding); - - return OPAL_SUCCESS; -} - -/* - * Function for finding and opening either all MCA components, or the one - * that was specifically requested via a MCA parameter. - */ -int opal_paffinity_base_open(void) -{ - opal_paffinity_base_components_opened_valid = false; - - mca_base_param_reg_string_name("opal", "paffinity_base_slot_list", - "Used to set list of processor IDs to bind MPI processes to (e.g., used in conjunction with rank files)", - true, false, NULL, &opal_paffinity_base_slot_list); - - /* Open up all available components */ - - if (OPAL_SUCCESS != - mca_base_components_open("paffinity", opal_paffinity_base_output, - mca_paffinity_base_static_components, - &opal_paffinity_base_components_opened, - true)) { - return OPAL_ERROR; - } - opal_paffinity_base_components_opened_valid = true; - - /* All done */ - - return OPAL_SUCCESS; -} diff --git a/opal/mca/paffinity/base/paffinity_base_select.c b/opal/mca/paffinity/base/paffinity_base_select.c deleted file mode 100644 index 98d7c6b00c..0000000000 --- a/opal/mca/paffinity/base/paffinity_base_select.c +++ /dev/null @@ -1,68 +0,0 @@ -/* - * Copyright (c) 2004-2008 The Trustees of Indiana University and Indiana - * University Research and Technology - * Corporation. All rights reserved. - * Copyright (c) 2004-2006 The University of Tennessee and The University - * of Tennessee Research Foundation. All rights - * reserved. - * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, - * University of Stuttgart. All rights reserved. - * Copyright (c) 2004-2005 The Regents of the University of California. - * All rights reserved. - * Copyright (c) 2007-2010 Cisco Systems, Inc. All rights reserved. - * $COPYRIGHT$ - * - * Additional copyrights may follow - * - * $HEADER$ - */ - - -#include "opal_config.h" - -#include "opal/constants.h" -#include "opal/mca/mca.h" -#include "opal/mca/base/base.h" -#include "opal/mca/base/mca_base_param.h" -#include "opal/mca/paffinity/paffinity.h" -#include "opal/mca/paffinity/base/base.h" - -/* - * Globals - */ -bool opal_paffinity_base_selected = false; -const opal_paffinity_base_component_2_0_1_t *opal_paffinity_base_component = NULL; -const opal_paffinity_base_module_1_1_0_t *opal_paffinity_base_module = NULL; - - -int opal_paffinity_base_select(void) -{ - int ret = OPAL_SUCCESS; - opal_paffinity_base_component_2_0_1_t *best_component = NULL; - opal_paffinity_base_module_1_1_0_t *best_module = NULL; - - /* - * Select the best component - */ - if( OPAL_SUCCESS != mca_base_select("paffinity", opal_paffinity_base_output, - &opal_paffinity_base_components_opened, - (mca_base_module_t **) &best_module, - (mca_base_component_t **) &best_component) ) { - /* It is okay if we don't find a module - we will report an - * error if/when someone tries to actually use affinity - */ - return OPAL_SUCCESS; - } - - /* Save the winner */ - opal_paffinity_base_component = best_component; - opal_paffinity_base_module = best_module; - opal_paffinity_base_selected = true; - - /* Initialize the winner */ - if (NULL != opal_paffinity_base_module) { - ret = opal_paffinity_base_module->paff_module_init(); - } - - return ret; -} diff --git a/opal/mca/paffinity/base/paffinity_base_service.c b/opal/mca/paffinity/base/paffinity_base_service.c deleted file mode 100644 index 822d19e80d..0000000000 --- a/opal/mca/paffinity/base/paffinity_base_service.c +++ /dev/null @@ -1,665 +0,0 @@ -/* - * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana - * University Research and Technology - * Corporation. All rights reserved. - * Copyright (c) 2004-2006 The University of Tennessee and The University - * of Tennessee Research Foundation. All rights - * reserved. - * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, - * University of Stuttgart. All rights reserved. - * Copyright (c) 2004-2005 The Regents of the University of California. - * All rights reserved. - * Copyright (c) 2007-2010 Cisco Systems, Inc. All rights reserved. - * Copyright (c) 2008 Voltaire. All rights reserved - * Copyright (c) 2010 Oracle and/or its affiliates. All rights reserved. - * - * $COPYRIGHT$ - * - * Additional copyrights may follow - * - * $HEADER$ - */ - -#include "opal_config.h" - -#ifdef HAVE_STRING_H -#include -#endif - -#include "opal/util/argv.h" -#include "opal/constants.h" -#include "opal/mca/paffinity/paffinity.h" -#include "opal/mca/paffinity/base/base.h" -#include "opal/runtime/opal.h" -#include "opal/util/output.h" - -static bool diag_requested; - -static int opal_paffinity_base_socket_to_cpu_set(char **socket_list, int socket_cnt, - long rank, bool logical_map, - opal_paffinity_base_cpu_set_t *cpumask) -{ - int i; - char **range; - int range_cnt; - int lower_range, upper_range; - int processor_id, num_processors; - int rc; - int phys_processor; - - /* get the number of LOGICAL processors on this node */ - if (OPAL_SUCCESS != (rc = opal_paffinity_base_get_processor_info(&num_processors))) { - return rc; - } - OPAL_PAFFINITY_CPU_ZERO(*cpumask); - for (i=0; i lower_range || num_cores < (upper_range - lower_range) || lower_range >= upper_range ) { - return OPAL_ERR_SLOT_LIST_RANGE; - } - for (core=lower_range; core<=upper_range; core++) { - if (logical_map) { - /* convert to physical core */ - if (OPAL_SUCCESS != (rc = opal_paffinity_base_get_physical_core_id(phys_socket, core, &phys_core))) { - opal_output(0, "Rank %ld: PAFFINITY cannot get physical core id for logical core %ld in physical socket %ld (%ld)", - rank, (long)core, (long)phys_socket, (long)socket); - return rc; - } - } else { - phys_core = core; - } - /* get the PHYSICAL processor id for this PHYSICAL socket/core */ - if ( OPAL_SUCCESS != (rc = opal_paffinity_base_get_map_to_processor_id (phys_socket, phys_core, &phys_processor))) { - return rc; - } - /* set the bit for this processor */ - OPAL_PAFFINITY_CPU_SET(phys_processor, *cpumask); - /* output diagnostic if requested */ - if (diag_requested) { - opal_output(0,"paffinity slot assignment: rank %ld runs on cpu #%d ( %d[%d] : %d[%d])", - rank, phys_processor, phys_socket, socket, phys_core, core); - } - } - /* tell paffinity to bind us */ - if ( OPAL_SUCCESS != (rc = opal_paffinity_base_set(*cpumask))) { - return rc; - } - break; - - default: - opal_argv_free(range); - opal_argv_free(socket_core); - return OPAL_ERROR; - } - opal_argv_free(range); - opal_argv_free(socket_core); - } - - for (i=1; i these cores are on the same socket as the last one specified, - * so we map them on that same physical socket - */ - range = opal_argv_split(socket_core[0], '-'); - range_cnt = opal_argv_count(range); - switch (range_cnt) { - case 1: /* only one core provided */ - core = atoi(range[0]); - if (logical_map) { - /* convert to physical core */ - if (OPAL_SUCCESS != (rc = opal_paffinity_base_get_physical_core_id(phys_socket, core, &phys_core))) { - return rc; - } - } else { - phys_core = core; - } - /* get the PHYSICAL processor id for this PHYSICAL socket/core */ - if ( OPAL_SUCCESS != (rc = opal_paffinity_base_get_map_to_processor_id (phys_socket, phys_core, &phys_processor))) { - return rc; - } - /* set the bit for this processor */ - OPAL_PAFFINITY_CPU_SET(phys_processor, *cpumask); - /* tell paffinity to bind us */ - if (OPAL_SUCCESS != (rc = opal_paffinity_base_set(*cpumask))) { - return rc; - } - /* output diagnostic if requested */ - if (diag_requested) { - opal_output(0, "paffinity slot assignment: rank %ld runs on physical cpu #%d ( %d[%d] : %d[%d])", - rank, phys_processor, phys_socket, socket, phys_core, core); - } - break; - - case 2: /* range of core id's was given */ - lower_range = atoi(range[0]); - upper_range = atoi(range[1]); - if ( 0 > lower_range || num_cores < (upper_range - lower_range) || lower_range >= upper_range ) { - return OPAL_ERR_SLOT_LIST_RANGE; - } - for (core=lower_range; core<=upper_range; core++) { - if (logical_map) { - /* convert to physical core */ - if (OPAL_SUCCESS != (rc = opal_paffinity_base_get_physical_core_id(phys_socket, core, &phys_core))) { - return rc; - } - } else { - phys_core = core; - } - /* get the PHYSICAL processor id for this PHYSICAL socket/core */ - if ( OPAL_SUCCESS != (rc = opal_paffinity_base_get_map_to_processor_id (phys_socket, phys_core, &phys_processor))) { - return rc; - } - /* set the bit for this processor */ - OPAL_PAFFINITY_CPU_SET(phys_processor, *cpumask); - /* output diagnostic if requested */ - if (diag_requested) { - opal_output(0, "paffinity slot assignment: rank %ld runs on physical cpu #%d ( %d[%d] : %d[%d])", - rank, phys_processor, phys_socket, socket, phys_core, core); - } - } - /* tell paffinity to bind us */ - if ( OPAL_SUCCESS != (rc = opal_paffinity_base_set(*cpumask))) { - return rc; - } - break; - - default: - opal_argv_free(range); - opal_argv_free(socket_core); - return OPAL_ERROR; - } - opal_argv_free(range); - break; - - case 2: /* colon was given => refers to a new socket! */ - socket = atoi(socket_core[0]); - if (logical_map) { - /* need to convert provided socket to a PHYSICAL socket id */ - rc = opal_paffinity_base_get_physical_socket_id(socket, &phys_socket); - if (OPAL_SUCCESS != rc) { - return rc; - } - } else { - phys_socket = socket; - } - - /* get the LOGICAL core info for this socket */ - if ( OPAL_SUCCESS != ( rc = opal_paffinity_base_get_core_info(phys_socket, &num_cores))) { - return rc; - } - - if (0 == strcmp("*",socket_core[1])) { - /* bind to all available LOGICAL cores */ - for (core = 0; core < num_cores; core++) { - /* convert to PHYSICAL core id */ - if (OPAL_SUCCESS != (rc = opal_paffinity_base_get_physical_core_id(phys_socket, core, &phys_core))) { - return rc; - } - /* get the PHYSICAL processor id for the PHYSICAL socket/core */ - if ( OPAL_SUCCESS != (rc = opal_paffinity_base_get_map_to_processor_id (phys_socket, phys_core, &phys_processor))) { - return rc; - } - /* set the bit for this processor */ - OPAL_PAFFINITY_CPU_SET(phys_processor, *cpumask); - } - /* tell paffinity to bind us */ - if (OPAL_SUCCESS != (rc = opal_paffinity_base_set(*cpumask))) { - return rc; - } - /* output diagnostic if requested */ - if (diag_requested) { - opal_output(0, "paffinity slot assignment: rank %ld runs on physical cpu #%d ( %d[%d] : %d[%d])", - rank, phys_processor, phys_socket, socket, phys_core, core); - } - } else { - range = opal_argv_split(socket_core[1], '-'); - range_cnt = opal_argv_count(range); - socket = atoi(socket_core[0]); - switch (range_cnt) { - case 1: /* only one core specified */ - core = atoi(range[0]); - if (logical_map) { - /* convert to physical core */ - if (OPAL_SUCCESS != (rc = opal_paffinity_base_get_physical_core_id(phys_socket, core, &phys_core))) { - return rc; - } - } else { - phys_core = core; - } - /* get the PHYSICAL processor id for this PHYSICAL socket/core */ - if ( OPAL_SUCCESS != (rc = opal_paffinity_base_get_map_to_processor_id (phys_socket, phys_core, &phys_processor))) { - return rc; - } - /* set the bit for this processor */ - OPAL_PAFFINITY_CPU_SET(phys_processor, *cpumask); - /* tell paffinity to bind us */ - if (OPAL_SUCCESS != (rc = opal_paffinity_base_set(*cpumask))) { - return rc; - } - /* output diagnostic if requested */ - if (diag_requested) { - opal_output(0, "paffinity slot assignment: rank %ld runs on physical cpu #%d ( %d[%d] : %d[%d])", - rank, phys_processor, phys_socket, socket, phys_core, core); - } - break; - - case 2: /* range of core id's was given */ - lower_range = atoi(range[0]); - upper_range = atoi(range[1]); - if ( 0 > lower_range || num_cores < (upper_range - lower_range) || lower_range >= upper_range ) { - return OPAL_ERR_SLOT_LIST_RANGE; - } - for (core=lower_range; core<=upper_range; core++) { - if (logical_map) { - /* convert to physical core */ - if (OPAL_SUCCESS != (rc = opal_paffinity_base_get_physical_core_id(phys_socket, core, &phys_core))) { - return rc; - } - } else { - phys_core = core; - } - /* get the PHYSICAL processor id for this PHYSICAL socket/core */ - if ( OPAL_SUCCESS != (rc = opal_paffinity_base_get_map_to_processor_id (phys_socket, phys_core, &phys_processor))) { - return rc; - } - /* set the bit for this processor */ - OPAL_PAFFINITY_CPU_SET(phys_processor, *cpumask); - /* output diagnostic if requested */ - if (diag_requested) { - opal_output(0, "paffinity slot assignment: rank %ld runs on physical cpu #%d ( %d[%d] : %d[%d])", - rank, phys_processor, phys_socket, socket, phys_core, core); - } - } - /* tell paffinity to bind us */ - if ( OPAL_SUCCESS != (rc = opal_paffinity_base_set(*cpumask))) { - return rc; - } - - default: - opal_argv_free(range); - opal_argv_free(socket_core); - return OPAL_ERROR; - } - opal_argv_free(range); - } - break; - - default: - opal_argv_free(socket_core); - return OPAL_ERROR; - } - opal_argv_free(socket_core); - } - return OPAL_SUCCESS; -} - -int opal_paffinity_base_slot_list_set(long rank, char *slot_str, opal_paffinity_base_cpu_set_t *cpumask) -{ - char **item; - char **socket_core; - int item_cnt, socket_core_cnt, rc; - bool logical_map; - - if (NULL == slot_str){ - return OPAL_ERR_BAD_PARAM; - } - - /* if the slot string is empty, that is an error */ - if (0 == strlen(slot_str)) { - return OPAL_ERR_BAD_PARAM; - } - - /* check for diag request to avoid repeatedly doing so */ - if (4 < opal_output_get_verbosity(opal_paffinity_base_output)) { - diag_requested = true; - } else { - diag_requested = false; - } - - opal_output_verbose(5, opal_paffinity_base_output, "paffinity slot assignment: slot_list == %s", slot_str); - - if ('P' == slot_str[0] || 'p' == slot_str[0]) { - /* user has specified physical mapping */ - logical_map = false; - item = opal_argv_split (&slot_str[1], ','); - } else { - logical_map = true; /* default to logical mapping */ - item = opal_argv_split (slot_str, ','); - } - - item_cnt = opal_argv_count (item); - socket_core = opal_argv_split (item[0], ':'); - socket_core_cnt = opal_argv_count(socket_core); - opal_argv_free(socket_core); - switch (socket_core_cnt) { - case 1: /* binding to cpu's */ - if (OPAL_SUCCESS != (rc = opal_paffinity_base_socket_to_cpu_set(item, item_cnt, - rank, logical_map, cpumask))) { - opal_argv_free(item); - return rc; - } - break; - case 2: /* binding to socket/core specification */ - if (OPAL_SUCCESS != (rc = opal_paffinity_base_socket_core_to_cpu_set(item, item_cnt, - rank, logical_map, cpumask))) { - opal_argv_free(item); - return rc; - } - break; - default: - opal_argv_free(item); - return OPAL_ERROR; - } - opal_argv_free(item); - return OPAL_SUCCESS; -} - - -/** - * Make a prettyprint string for a cset. - */ -int opal_paffinity_base_cset2str(char *str, int len, - opal_paffinity_base_cpu_set_t *cset) -{ - int ret, i, j, k, num_sockets, num_cores, flag, count, - range_first=0, range_last; - char tmp[BUFSIZ]; - const int stmp = sizeof(tmp) - 1; - - str[0] = tmp[stmp] = '\0'; - - /* Loop over the number of sockets in this machine */ - ret = opal_paffinity_base_get_socket_info(&num_sockets); - if (OPAL_SUCCESS != ret) { - return ret; - } - for (i = 0; i < num_sockets; ++i) { - /* Loop over the number of cores in this socket */ - ret = opal_paffinity_base_get_core_info(i, &num_cores); - if (OPAL_SUCCESS != ret) { - return ret; - } - /* Must initially set range_last to a low number -- smaller - than -1, so that the comparisons below work out - properly. */ - for (range_last = -5, count = j = 0; j < num_cores; ++j) { - ret = opal_paffinity_base_get_map_to_processor_id(i, j, &k); - if (OPAL_SUCCESS != ret) { - return ret; - } - - /* Prettyprint the cores that we're actually bound to */ - flag = OPAL_PAFFINITY_CPU_ISSET(k, *cset); - if (flag) { - if (0 == count) { - snprintf(tmp, stmp, "socket %d[core %d", i, j); - strncat(str, tmp, len - strlen(str)); - range_first = range_last = j; - } else { - if (j - 1 == range_last) { - range_last = j; - } else { - snprintf(tmp, stmp, "-%d,%d", range_last, j); - strncat(str, tmp, len - strlen(str)); - range_first = range_last = j; - } - } - ++count; - } - } - if (count > 0) { - if (range_first != range_last) { - snprintf(tmp, stmp, "-%d", range_last); - strncat(str, tmp, len - strlen(str)); - } - strncat(str, "] ", len - strlen(str)); - } - } - - return OPAL_SUCCESS; -} - -/** - * Make a prettyprint string for a cset in a map format. - * Example: [B_/__] - * Key: [] - signifies socket - * / - signifies core - * _ - signifies thread a process not bound to - * B - signifies thread a process is bound to - */ -int opal_paffinity_base_cset2mapstr(char *str, int len, - opal_paffinity_base_cpu_set_t *cset) -{ - int ret, i, j, k, num_sockets, num_cores, flag; - char tmp[BUFSIZ]; - const int stmp = sizeof(tmp) - 1; - - str[0] = tmp[stmp] = '\0'; - - /* Loop over the number of sockets in this machine */ - ret = opal_paffinity_base_get_socket_info(&num_sockets); - if (OPAL_SUCCESS != ret) { - return ret; - } - for (i = 0; i < num_sockets; ++i) { - strncat(str, "[", len - strlen(str)); - /* Loop over the number of cores in this socket */ - ret = opal_paffinity_base_get_core_info(i, &num_cores); - if (OPAL_SUCCESS != ret) { - return ret; - } - for (j = 0; j < num_cores; j++) { - if (0 < j) { - /* add space after first core is printed */ - strncat(str, " ", len - strlen(str)); - } - - ret = opal_paffinity_base_get_map_to_processor_id(i, j, &k); - if (OPAL_SUCCESS != ret) { - return ret; - } - - flag = OPAL_PAFFINITY_CPU_ISSET(k, *cset); - if (flag) { - /* mark core as bound to process */ - strncat(str, "B", len - strlen(str)); - } else { - /* mark core as no process bound to it */ - strncat(str, ".", len - strlen(str)); - } - } - strncat(str, "]", len - strlen(str)); - } - - return OPAL_SUCCESS; -} - diff --git a/opal/mca/paffinity/base/paffinity_base_wrappers.c b/opal/mca/paffinity/base/paffinity_base_wrappers.c deleted file mode 100644 index ba6805175c..0000000000 --- a/opal/mca/paffinity/base/paffinity_base_wrappers.c +++ /dev/null @@ -1,208 +0,0 @@ -/* - * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana - * University Research and Technology - * Corporation. All rights reserved. - * Copyright (c) 2004-2006 The University of Tennessee and The University - * of Tennessee Research Foundation. All rights - * reserved. - * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, - * University of Stuttgart. All rights reserved. - * Copyright (c) 2004-2005 The Regents of the University of California. - * All rights reserved. - * Copyright (c) 2007-2010 Cisco Systems, Inc. All rights reserved. - * $COPYRIGHT$ - * - * Additional copyrights may follow - * - * $HEADER$ - */ - -#include "opal_config.h" - -#ifdef HAVE_STRING_H -#include -#endif -#ifdef HAVE_UNISTD_H -#include -#endif - -#include "opal/constants.h" -#include "opal/util/output.h" -#include "opal/mca/paffinity/paffinity.h" -#include "opal/mca/paffinity/base/base.h" - -int opal_paffinity_base_set(opal_paffinity_base_cpu_set_t cpumask) -{ - int rc; - - if (!opal_paffinity_base_selected) { - return OPAL_ERR_MODULE_NOT_FOUND; - } - if (OPAL_SUCCESS == (rc = opal_paffinity_base_module->paff_module_set(cpumask))) { - opal_paffinity_base_bound = true; - } - return rc; -} - -int opal_paffinity_base_get(opal_paffinity_base_cpu_set_t *cpumask) -{ - /* zero the cpumask so we start with a clean slate - do - * it here so that any error returns no info - */ - if (NULL != cpumask) { - OPAL_PAFFINITY_CPU_ZERO(*cpumask); - } - - if (!opal_paffinity_base_selected) { - return OPAL_ERR_MODULE_NOT_FOUND; - } - if(NULL == cpumask) { - return OPAL_ERR_BAD_PARAM; - } - return opal_paffinity_base_module->paff_module_get(cpumask); -} - -int opal_paffinity_base_get_map_to_processor_id(int socket, int core, int *processor_id) -{ - if (!opal_paffinity_base_selected) { - return OPAL_ERR_MODULE_NOT_FOUND; - } - return opal_paffinity_base_module->paff_get_map_to_processor_id(socket, core, processor_id); -} - -int opal_paffinity_base_get_map_to_socket_core(int processor_id, int *socket, int *core) -{ - if (!opal_paffinity_base_selected) { - return OPAL_ERR_MODULE_NOT_FOUND; - } - return opal_paffinity_base_module->paff_get_map_to_socket_core(processor_id, socket, core); -} - - -int opal_paffinity_base_get_processor_info(int *num_processors) -{ - if (!opal_paffinity_base_selected) { - return OPAL_ERR_MODULE_NOT_FOUND; - } - return opal_paffinity_base_module->paff_get_processor_info(num_processors); -} - -int opal_paffinity_base_get_socket_info(int *num_sockets) -{ - if (!opal_paffinity_base_selected) { - return OPAL_ERR_MODULE_NOT_FOUND; - } - return opal_paffinity_base_module->paff_get_socket_info(num_sockets); -} - -int opal_paffinity_base_get_core_info(int socket, int *num_cores) -{ - if (!opal_paffinity_base_selected) { - return OPAL_ERR_MODULE_NOT_FOUND; - } - return opal_paffinity_base_module->paff_get_core_info(socket, num_cores); -} - -int opal_paffinity_base_get_physical_processor_id(int logical_processor_id, - int *physical_processor_id) -{ - if (!opal_paffinity_base_selected) { - return OPAL_ERR_MODULE_NOT_FOUND; - } - return opal_paffinity_base_module->paff_get_physical_processor_id(logical_processor_id, physical_processor_id); -} - -int opal_paffinity_base_get_physical_socket_id(int logical_socket_id, - int *physical_socket_id) -{ - if (!opal_paffinity_base_selected) { - return OPAL_ERR_MODULE_NOT_FOUND; - } - return opal_paffinity_base_module->paff_get_physical_socket_id(logical_socket_id, physical_socket_id); -} - -int opal_paffinity_base_get_physical_core_id(int physical_socket_id, int logical_core_id, int *physical_core_id) -{ - if (!opal_paffinity_base_selected) { - return OPAL_ERR_MODULE_NOT_FOUND; - } - return opal_paffinity_base_module->paff_get_physical_core_id(physical_socket_id, logical_core_id, physical_core_id); -} - -char *opal_paffinity_base_print_binding(opal_paffinity_base_cpu_set_t cpumask) -{ - char *tmp; - size_t i, j, masksize, save; - - /* get space for element separators and trailing NULL */ - masksize = (2*OPAL_PAFFINITY_CPU_SET_NUM_BYTES)+OPAL_PAFFINITY_BITMASK_NUM_ELEMENTS + 1; - tmp = (char*)malloc(masksize); - if (NULL == tmp) { - return NULL; - } - memset(tmp, 0, masksize); - masksize = sizeof(opal_paffinity_base_bitmask_t); - - /* Save the position of the last : before there are all zeros to - the right -- we don't need to print all zeros to the right; - we'll chop them off, below. */ - save = 0; - if (4 == masksize) { - for (i=0, j=0; i < OPAL_PAFFINITY_BITMASK_NUM_ELEMENTS; i++) { - sprintf(&tmp[j], "%08lx", cpumask.bitmask[i]); - j += 8; - tmp[j] = ':'; - if (cpumask.bitmask[i] > 0) { - save = j; - } - j++; - } - } else if (8 == masksize) { - for (i=0, j=0; i < OPAL_PAFFINITY_BITMASK_NUM_ELEMENTS; i++) { - sprintf(&tmp[j], "%016lx", cpumask.bitmask[i]); - j += 16; - tmp[j] = ':'; - j++; - if (cpumask.bitmask[i] > 0) { - save = j; - } - } - } - - /* If there were no non-zero values, then ensure to print one - field of zeros */ - if (0 == save) { - tmp[2 * masksize] = '\0'; - } else { - tmp[save] = '\0'; - } - - return tmp; -} - -int opal_paffinity_base_parse_binding(char *binding, opal_paffinity_base_cpu_set_t *cpumask) -{ - size_t i; - char *tmp, *save; - - if (NULL == binding || 0 == strlen(binding)) { - return OPAL_SUCCESS; - } - - OPAL_PAFFINITY_CPU_ZERO(*cpumask); - tmp = binding; - for (i=0; i < OPAL_PAFFINITY_BITMASK_NUM_ELEMENTS; i++) { - cpumask->bitmask[i] = strtoul(tmp, &save, 16); - tmp = save; - if (NULL == tmp) { - /* end of the line */ - break; - } - tmp++; - if (NULL == tmp || 0 == strlen(tmp)) { - return OPAL_SUCCESS; - } - } - - return OPAL_SUCCESS; -} diff --git a/opal/mca/paffinity/hwloc/Makefile.am b/opal/mca/paffinity/hwloc/Makefile.am deleted file mode 100644 index 653a80cfe7..0000000000 --- a/opal/mca/paffinity/hwloc/Makefile.am +++ /dev/null @@ -1,51 +0,0 @@ -# -# Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana -# University Research and Technology -# Corporation. All rights reserved. -# Copyright (c) 2004-2005 The University of Tennessee and The University -# of Tennessee Research Foundation. All rights -# reserved. -# Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, -# University of Stuttgart. All rights reserved. -# Copyright (c) 2004-2005 The Regents of the University of California. -# All rights reserved. -# Copyright (c) 2010-2011 Cisco Systems, Inc. All rights reserved. -# $COPYRIGHT$ -# -# Additional copyrights may follow -# -# $HEADER$ -# - -# To find hwloc_bottom.h. EMBEDDED flags are for if we OMPI's -# internal hwloc is used; paffinity_hwloc_CPPFLAGS is if we are using -# an external install. -AM_CPPFLAGS = $(HWLOC_EMBEDDED_CPPFLAGS) $(paffinity_hwloc_CPPFLAGS) -# To get the cflags for the stuff in hwloc.h -AM_CFLAGS = $(HWLOC_EMBEDDED_CFLAGS) $(paffinity_hwloc_CFLAGS) - -sources = \ - paffinity_hwloc.h \ - paffinity_hwloc_component.c \ - paffinity_hwloc_module.c - -# Make the output library in this directory, and name it either -# mca__.la (for DSO builds) or libmca__.la -# (for static builds). - -if MCA_BUILD_opal_paffinity_hwloc_DSO -component_noinst = -component_install = mca_paffinity_hwloc.la -else -component_noinst = libmca_paffinity_hwloc.la -component_install = -endif - -mcacomponentdir = $(pkglibdir) -mcacomponent_LTLIBRARIES = $(component_install) -mca_paffinity_hwloc_la_SOURCES = $(sources) -mca_paffinity_hwloc_la_LDFLAGS = -module -avoid-version - -noinst_LTLIBRARIES = $(component_noinst) -libmca_paffinity_hwloc_la_SOURCES =$(sources) -libmca_paffinity_hwloc_la_LDFLAGS = -module -avoid-version diff --git a/opal/mca/paffinity/hwloc/configure.m4 b/opal/mca/paffinity/hwloc/configure.m4 deleted file mode 100644 index 8943061beb..0000000000 --- a/opal/mca/paffinity/hwloc/configure.m4 +++ /dev/null @@ -1,36 +0,0 @@ -# -*- shell-script -*- -# -# Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana -# University Research and Technology -# Corporation. All rights reserved. -# Copyright (c) 2004-2005 The University of Tennessee and The University -# of Tennessee Research Foundation. All rights -# reserved. -# Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, -# University of Stuttgart. All rights reserved. -# Copyright (c) 2004-2005 The Regents of the University of California. -# All rights reserved. -# Copyright (c) 2007-2011 Cisco Systems, Inc. All rights reserved. -# -# $COPYRIGHT$ -# -# Additional copyrights may follow -# -# $HEADER$ -# - -# MCA_opal_paffinity_hwloc_CONFIG([action-if-found], [action-if-not-found]) -# ------------------------------------------------------------------------- -AC_DEFUN([MCA_opal_paffinity_hwloc_CONFIG],[ - AC_REQUIRE([MCA_opal_hwloc_CONFIG_REQUIRE]) - AC_CONFIG_FILES([opal/mca/paffinity/hwloc/Makefile]) - - # All we check for is whether $OPAL_HAVE_HWLOC is 1. - # See big comment in opal/mca/hwloc/configure.m4. - AC_MSG_CHECKING([if hwloc is enabled]) - AS_IF([test $OPAL_HAVE_HWLOC -eq 1], - [AC_MSG_RESULT([yes]) - $1], - [AC_MSG_RESULT([no]) - $2]) -])dnl diff --git a/opal/mca/paffinity/hwloc/paffinity_hwloc.h b/opal/mca/paffinity/hwloc/paffinity_hwloc.h deleted file mode 100644 index f94ace9bf3..0000000000 --- a/opal/mca/paffinity/hwloc/paffinity_hwloc.h +++ /dev/null @@ -1,52 +0,0 @@ -/* - * Copyright (c) 2004-2008 The Trustees of Indiana University and Indiana - * University Research and Technology - * Corporation. All rights reserved. - * Copyright (c) 2004-2005 The University of Tennessee and The University - * of Tennessee Research Foundation. All rights - * reserved. - * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, - * University of Stuttgart. All rights reserved. - * Copyright (c) 2004-2005 The Regents of the University of California. - * All rights reserved. - * Copyright (c) 2006-2011 Cisco Systems, Inc. All rights reserved. - * - * $COPYRIGHT$ - * - * Additional copyrights may follow - * - * $HEADER$ - */ - -#ifndef MCA_PAFFINITY_HWLOC_EXPORT_H -#define MCA_PAFFINITY_HWLOC_EXPORT_H - -#include "opal_config.h" - -#include "opal/mca/mca.h" -#include "opal/mca/paffinity/paffinity.h" -#include "hwloc.h" - - -BEGIN_C_DECLS - -/** - * Globally exported variable - */ -OPAL_DECLSPEC extern opal_paffinity_base_component_t - mca_paffinity_hwloc_component; - -/** - * paffinity query API function - * - * Query function for paffinity components. Simply returns a priority - * to rank it against other available paffinity components (assumedly, - * only one component will be available per platform, but it's - * possible that there could be more than one available). - */ -int opal_paffinity_hwloc_component_query(mca_base_module_t **module, - int *priority); - -END_C_DECLS - -#endif /* MCA_PAFFINITY_HWLOC_EXPORT_H */ diff --git a/opal/mca/paffinity/hwloc/paffinity_hwloc_component.c b/opal/mca/paffinity/hwloc/paffinity_hwloc_component.c deleted file mode 100644 index 29419dad4a..0000000000 --- a/opal/mca/paffinity/hwloc/paffinity_hwloc_component.c +++ /dev/null @@ -1,78 +0,0 @@ -/* - * Copyright (c) 2004-2008 The Trustees of Indiana University and Indiana - * University Research and Technology - * Corporation. All rights reserved. - * Copyright (c) 2004-2005 The University of Tennessee and The University - * of Tennessee Research Foundation. All rights - * reserved. - * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, - * University of Stuttgart. All rights reserved. - * Copyright (c) 2004-2005 The Regents of the University of California. - * All rights reserved. - * Copyright (c) 2007-2011 Cisco Systems, Inc. All rights reserved. - * $COPYRIGHT$ - * - * Additional copyrights may follow - * - * $HEADER$ - */ - -#include "opal_config.h" - -#include "opal/constants.h" -#include "opal/mca/base/mca_base_param.h" -#include "opal/mca/hwloc/hwloc.h" -#include "opal/mca/paffinity/paffinity.h" -#include "paffinity_hwloc.h" - -/* - * Public string showing the paffinity ompi_hwloc component version number - */ -const char *opal_paffinity_hwloc_component_version_string = - "OPAL hwloc paffinity MCA component version " OPAL_VERSION; - -/* - * Local function - */ -static int hwloc_register(void); - -/* - * Instantiate the public struct with all of our public information - * and pointers to our public functions in it - */ - -opal_paffinity_base_component_t mca_paffinity_hwloc_component = { - /* First, the mca_component_t struct containing meta information - about the component itself */ - { - OPAL_PAFFINITY_BASE_VERSION_2_0_1, - - /* Component name and version */ - "hwloc", - OPAL_MAJOR_VERSION, - OPAL_MINOR_VERSION, - OPAL_RELEASE_VERSION, - - /* Component open and close functions */ - NULL, - NULL, - opal_paffinity_hwloc_component_query, - hwloc_register, - }, - { - /* The component is checkpoint ready */ - MCA_BASE_METADATA_PARAM_CHECKPOINT - } -}; - - - -static int hwloc_register(void) -{ - mca_base_param_reg_int(&mca_paffinity_hwloc_component.base_version, - "priority", - "Priority of the hwloc paffinity component", - false, false, 40, NULL); - - return OPAL_SUCCESS; -} diff --git a/opal/mca/paffinity/hwloc/paffinity_hwloc_module.c b/opal/mca/paffinity/hwloc/paffinity_hwloc_module.c deleted file mode 100644 index e5c7930065..0000000000 --- a/opal/mca/paffinity/hwloc/paffinity_hwloc_module.c +++ /dev/null @@ -1,573 +0,0 @@ -/* - * Copyright (c) 2004-2008 The Trustees of Indiana University and Indiana - * University Research and Technology - * Corporation. All rights reserved. - * Copyright (c) 2004-2005 The University of Tennessee and The University - * of Tennessee Research Foundation. All rights - * reserved. - * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, - * University of Stuttgart. All rights reserved. - * Copyright (c) 2004-2005 The Regents of the University of California. - * All rights reserved. - * Copyright (c) 2006-2012 Cisco Systems, Inc. All rights reserved. - * - * $COPYRIGHT$ - * - * Additional copyrights may follow - * - * $HEADER$ - */ - -#include "opal_config.h" - -/* This component will only be compiled on Hwloc, where we are - guaranteed to have and friends */ -#include - -#include -#include -#include -#include - -#include "opal/constants.h" -#include "opal/mca/base/mca_base_param.h" -#include "opal/mca/paffinity/paffinity.h" -#include "opal/mca/paffinity/base/base.h" -#include "paffinity_hwloc.h" -#include "opal/mca/hwloc/hwloc.h" - -/* - * Local functions - */ -static int module_init(void); -static int module_set(opal_paffinity_base_cpu_set_t cpumask); -static int module_get(opal_paffinity_base_cpu_set_t *cpumask); -static int module_map_to_processor_id(int socket, int core, int *processor_id); -static int module_map_to_socket_core(int processor_id, int *socket, int *core); -static int module_get_processor_info(int *num_processors); -static int module_get_socket_info(int *num_sockets); -static int module_get_core_info(int socket, int *num_cores); -static int module_get_physical_processor_id(int logical_processor_id, - int *physical_processor_id); -static int module_get_physical_socket_id(int logical_socket_id, - int *physical_socket_id); -static int module_get_physical_core_id(int physical_socket_id, - int logical_core_id, - int *physical_core_id); - -/* - * Hwloc paffinity module - */ -static const opal_paffinity_base_module_1_1_0_t loc_module = { - /* Initialization function */ - module_init, - - /* Module function pointers */ - module_set, - module_get, - module_map_to_processor_id, - module_map_to_socket_core, - module_get_processor_info, - module_get_socket_info, - module_get_core_info, - module_get_physical_processor_id, - module_get_physical_socket_id, - module_get_physical_core_id, - NULL -}; - -/* - * Trivial DFS traversal recursion function - */ -static hwloc_obj_t dfs_find_os_index(hwloc_obj_t root, hwloc_obj_type_t type, - unsigned os_index) -{ - unsigned i; - hwloc_obj_t ret; - - if (root->type == type && root->os_index == os_index) { - return root; - } - for (i = 0; i < root->arity; ++i) { - ret = dfs_find_os_index(root->children[i], type, os_index); - if (NULL != ret) { - return ret; - } - } - - return NULL; -} - -/* - * Trivial DFS traversal recursion function - */ -static hwloc_obj_t dfs_find_nth_item(hwloc_obj_t root, - hwloc_obj_type_t type, - unsigned *current, - unsigned n) -{ - unsigned i; - hwloc_obj_t ret; - - if (root->type == type) { - if (*current == n) { - return root; - } - ++(*current); - } - for (i = 0; i < root->arity; ++i) { - ret = dfs_find_nth_item(root->children[i], type, current, n); - if (NULL != ret) { - return ret; - } - } - - return NULL; -} - -/* - * Trivial DFS traversal recursion function - */ -static int dfs_count_type(hwloc_obj_t root, hwloc_obj_type_t type) -{ - unsigned i; - int count = 0; - if (root->type == type) { - ++count; - } - for (i = 0; i < root->arity; ++i) { - count += dfs_count_type(root->children[i], type); - } - - return count; -} - - -int opal_paffinity_hwloc_component_query(mca_base_module_t **module, - int *priority) -{ - int param; - - param = mca_base_param_find("paffinity", "hwloc", "priority"); - mca_base_param_lookup_int(param, priority); - - *module = (mca_base_module_t *)&loc_module; - - return OPAL_SUCCESS; -} - - -static int module_init(void) -{ - /* Nothing to do */ - - return OPAL_SUCCESS; -} - - -static int module_set(opal_paffinity_base_cpu_set_t mask) -{ - int i, ret = OPAL_SUCCESS; - hwloc_bitmap_t set; - hwloc_topology_t *t; - - /* bozo check */ - if (NULL == opal_hwloc_topology) { - return OPAL_ERR_NOT_INITIALIZED; - } - t = &opal_hwloc_topology; - - set = hwloc_bitmap_alloc(); - hwloc_bitmap_zero(set); - for (i = 0; ((unsigned int) i) < OPAL_PAFFINITY_BITMASK_CPU_MAX; ++i) { - if (OPAL_PAFFINITY_CPU_ISSET(i, mask)) { - hwloc_bitmap_set(set, i); - } - } - - if (0 != hwloc_set_cpubind(*t, set, 0)) { - ret = OPAL_ERR_IN_ERRNO; - } - hwloc_bitmap_free(set); - - return ret; -} - - -static int module_get(opal_paffinity_base_cpu_set_t *mask) -{ - int i, ret = OPAL_SUCCESS; - hwloc_bitmap_t set; - hwloc_topology_t *t; - - /* bozo check */ - if (NULL == opal_hwloc_topology) { - return OPAL_ERR_NOT_INITIALIZED; - } - t = &opal_hwloc_topology; - - if (NULL == mask) { - return OPAL_ERR_BAD_PARAM; - } - - set = hwloc_bitmap_alloc(); - if (0 != hwloc_get_cpubind(*t, set, 0)) { - ret = OPAL_ERR_IN_ERRNO; - } else { - OPAL_PAFFINITY_CPU_ZERO(*mask); - for (i = 0; ((unsigned int) i) < 8 * sizeof(*mask); i++) { - if (hwloc_bitmap_isset(set, i)) { - OPAL_PAFFINITY_CPU_SET(i, *mask); - } - } - - /* Ensure that hwloc does not have any bits set above the size - of the paffinity mask. If it does, we're hosed. :-( - (remember: hwloc has a dynamic bitmask size, but paffinity - has a fixed size) */ - if (hwloc_bitmap_last(set) > OPAL_PAFFINITY_BITMASK_CPU_MAX) { - ret = OPAL_ERR_VALUE_OUT_OF_BOUNDS; - } - } - hwloc_bitmap_free(set); - - return ret; -} - -/* - * Returns mapping of PHYSICAL socket:core -> PHYSICAL processor id. - * - * Since paffinity currently does not understand hardware threads, - * return the processor ID of the first hardware thread in the target - * core. - */ -static int module_map_to_processor_id(int socket, int core, int *processor_id) -{ - hwloc_topology_t *t; - hwloc_obj_t obj; - - /* bozo check */ - if (NULL == opal_hwloc_topology) { - return OPAL_ERR_NOT_INITIALIZED; - } - t = &opal_hwloc_topology; - - /* Traverse all sockets, looking for the right physical ID number. - Once we find it, traverse all that socket's cores looking for - the right physial ID number. Once we find it, return the - physical processor ID number. */ - for (obj = hwloc_get_next_obj_by_type(*t, HWLOC_OBJ_SOCKET, NULL); - NULL != obj; - obj = hwloc_get_next_obj_by_type(*t, HWLOC_OBJ_SOCKET, obj)) { - if (obj->os_index == (unsigned int) socket) { - /* Ok, we found the right socket. Browse its descendants - looking for the core with the right os_index (don't - assume all cores are at the same level). */ - - obj = dfs_find_os_index(obj, HWLOC_OBJ_CORE, core); - if (NULL != obj) { - /* Ok, we found the right core. Get the cpuset and - return the first PU (because hwloc understands - hardware threads, of which there might be multiple - on this core). */ - - hwloc_bitmap_t good; - good = hwloc_bitmap_alloc(); - if (NULL == good) { - return OPAL_ERR_OUT_OF_RESOURCE; - } - hwloc_bitmap_and(good, obj->online_cpuset, - obj->allowed_cpuset); - *processor_id = hwloc_bitmap_first(good); - hwloc_bitmap_free(good); - return OPAL_SUCCESS; - } - - /* If we found the right socket but not the right core, we - didn't find it. */ - return OPAL_ERR_NOT_FOUND; - } - } - - /* If we didn't even find the right socket, we didn't find it. */ - return OPAL_ERR_NOT_FOUND; -} - -/* - * Provides mapping of PHYSICAL processor id -> PHYSICAL socket:core. - */ -static int module_map_to_socket_core(int processor_id, int *socket, int *core) -{ - int ret = OPAL_ERR_NOT_FOUND; - hwloc_obj_t obj; - hwloc_topology_t *t; - hwloc_bitmap_t good; - - /* bozo check */ - if (NULL == opal_hwloc_topology) { - return OPAL_ERR_NOT_INITIALIZED; - } - t = &opal_hwloc_topology; - - good = hwloc_bitmap_alloc(); - if (NULL == good) { - return OPAL_ERR_OUT_OF_RESOURCE; - } - - /* Iterate through every core and find one that contains the - processor_id. Then find the corresponding socket. */ - for (obj = hwloc_get_next_obj_by_type(*t, HWLOC_OBJ_CORE, NULL); - NULL != obj; - obj = hwloc_get_next_obj_by_type(*t, HWLOC_OBJ_CORE, obj)) { - hwloc_bitmap_and(good, obj->online_cpuset, - obj->allowed_cpuset); - - /* Does this core contain the processor_id in question? */ - if (hwloc_bitmap_isset(good, processor_id)) { - *core = obj->os_index; - - /* Go upward from the core object until we find its parent - socket. */ - while (HWLOC_OBJ_SOCKET != obj->type) { - if (NULL == obj->parent) { - /* If we get to the root without finding a socket, - er.. Hmm. Error! */ - ret = OPAL_ERR_NOT_FOUND; - goto out; - } - obj = obj->parent; - } - *socket = obj->os_index; - ret = OPAL_SUCCESS; - goto out; - } - } - - /* If we didn't even find the right core, we didn't find it. Fall - through. */ - ret = OPAL_ERR_NOT_FOUND; - - out: - hwloc_bitmap_free(good); - return ret; -} - -/* - * Provides number of LOGICAL processors in a host. Since paffinity - * does not currently understand hardware threads, we interpret - * "processors" to mean "cores". - */ -static int module_get_processor_info(int *num_processors) -{ - hwloc_topology_t *t; - - /* bozo check */ - if (NULL == opal_hwloc_topology) { - return OPAL_ERR_NOT_INITIALIZED; - } - t = &opal_hwloc_topology; - - /* Try the simple hwloc_get_nbobjs_by_type() first. If we get -1, - go aggregate ourselves (because it means that there are cores - are multiple levels in the topology). */ - *num_processors = (int) hwloc_get_nbobjs_by_type(*t, HWLOC_OBJ_CORE); - if (-1 == *num_processors) { - hwloc_obj_t obj; - - *num_processors = 0; - for (obj = hwloc_get_next_obj_by_type(*t, HWLOC_OBJ_CORE, NULL); - NULL != obj; - obj = hwloc_get_next_obj_by_type(*t, HWLOC_OBJ_CORE, obj)) { - if (HWLOC_OBJ_CORE == obj->type) { - ++*num_processors; - } - } - } - - return OPAL_SUCCESS; -} - -/* - * Provides the number of LOGICAL sockets in a host. - */ -static int module_get_socket_info(int *num_sockets) -{ - hwloc_topology_t *t; - - /* bozo check */ - if (NULL == opal_hwloc_topology) { - return OPAL_ERR_NOT_INITIALIZED; - } - t = &opal_hwloc_topology; - - /* Try the simple hwloc_get_nbobjs_by_type() first. If we get -1, - go aggregate ourselves (because it means that there are cores - are multiple levels in the topology). */ - *num_sockets = (int) hwloc_get_nbobjs_by_type(*t, HWLOC_OBJ_SOCKET); - if (-1 == *num_sockets) { - hwloc_obj_t obj; - - *num_sockets = 0; - for (obj = hwloc_get_next_obj_by_type(*t, HWLOC_OBJ_SOCKET, NULL); - NULL != obj; - obj = hwloc_get_next_obj_by_type(*t, HWLOC_OBJ_SOCKET, obj)) { - if (HWLOC_OBJ_CORE == obj->type) { - ++*num_sockets; - } - } - } - - return OPAL_SUCCESS; -} - -/* - * Provides the number of LOGICAL cores in a PHYSICAL socket. - */ -static int module_get_core_info(int socket, int *num_cores) -{ - hwloc_obj_t obj; - hwloc_topology_t *t; - - /* bozo check */ - if (NULL == opal_hwloc_topology) { - return OPAL_ERR_NOT_INITIALIZED; - } - t = &opal_hwloc_topology; - - /* Traverse all sockets, looking for the right physical ID - number. */ - for (obj = hwloc_get_next_obj_by_type(*t, HWLOC_OBJ_SOCKET, NULL); - NULL != obj; - obj = hwloc_get_next_obj_by_type(*t, HWLOC_OBJ_SOCKET, obj)) { - if (obj->os_index == (unsigned int) socket) { - /* Ok, we found the right socket. Browse its descendants - looking for all cores. */ - *num_cores = dfs_count_type(obj, HWLOC_OBJ_CORE); - return OPAL_SUCCESS; - } - } - - /* If we didn't even find the right socket, we didn't find it. */ - return OPAL_ERR_NOT_FOUND; -} - -/* - * Provide the PHYSICAL processor id that corresponds to the given - * LOGICAL processor id. Remember: paffinity does not understand - * hardware threads, so "processor" here [usually] means "core" -- - * except that on some platforms, hwloc won't find any cores; it'll - * only find PUs (!). On such platforms, then do the same calculation - * but with PUs instead of COREs. - */ -static int module_get_physical_processor_id(int logical_processor_id, - int *physical_processor_id) -{ - hwloc_obj_type_t obj_type = HWLOC_OBJ_CORE; - hwloc_obj_t obj; - hwloc_bitmap_t good; - hwloc_topology_t *t; - - /* bozo check */ - if (NULL == opal_hwloc_topology) { - return OPAL_ERR_NOT_INITIALIZED; - } - t = &opal_hwloc_topology; - - /* hwloc isn't able to find cores on all platforms. Example: - PPC64 running RHEL 5.4 (linux kernel 2.6.18) only reports NUMA - nodes and PU's. Fine. - - However, note that hwloc_get_obj_by_type() will return NULL in - 2 (effectively) different cases: - - - no objects of the requested type were found - - the Nth object of the requested type was not found - - So first we have to see if we can find *any* cores by looking - for the 0th core. If we find it, then try to find the Nth - core. Otherwise, try to find the Nth PU. */ - if (NULL == hwloc_get_obj_by_type(*t, HWLOC_OBJ_CORE, 0)) { - obj_type = HWLOC_OBJ_PU; - } - - /* Now do the actual lookup. */ - obj = hwloc_get_obj_by_type(*t, obj_type, logical_processor_id); - if (NULL == obj) { - return OPAL_ERR_NOT_FOUND; - } - - /* Found the right core (or PU). Now find the processor ID of the - first PU available in that core. */ - good = hwloc_bitmap_alloc(); - if (NULL == good) { - return OPAL_ERR_OUT_OF_RESOURCE; - } - hwloc_bitmap_and(good, obj->online_cpuset, - obj->allowed_cpuset); - *physical_processor_id = hwloc_bitmap_first(good); - hwloc_bitmap_free(good); - return OPAL_SUCCESS; -} - -/* - * Provide the PHYSICAL socket id that corresponds to the given - * LOGICAL socket id - */ -static int module_get_physical_socket_id(int logical_socket_id, - int *physical_socket_id) -{ - hwloc_obj_t obj; - hwloc_topology_t *t; - - /* bozo check */ - if (NULL == opal_hwloc_topology) { - return OPAL_ERR_NOT_INITIALIZED; - } - t = &opal_hwloc_topology; - - obj = hwloc_get_obj_by_type(*t, HWLOC_OBJ_SOCKET, logical_socket_id); - if (NULL == obj) { - return OPAL_ERR_NOT_FOUND; - } - *physical_socket_id = obj->os_index; - return OPAL_SUCCESS; -} - -/* - * Provide the PHYSICAL core id that corresponds to the given LOGICAL - * core id on the given PHYSICAL socket id - */ -static int module_get_physical_core_id(int physical_socket_id, - int logical_core_id, - int *physical_core_id) -{ - unsigned count = 0; - hwloc_obj_t obj; - hwloc_topology_t *t; - - /* bozo check */ - if (NULL == opal_hwloc_topology) { - return OPAL_ERR_NOT_INITIALIZED; - } - t = &opal_hwloc_topology; - - obj = hwloc_get_root_obj(*t); - if (NULL == obj) { - return OPAL_ERR_NOT_FOUND; - } - obj = dfs_find_os_index(obj, HWLOC_OBJ_SOCKET, physical_socket_id); - if (NULL == obj) { - return OPAL_ERR_NOT_FOUND; - } - - /* Note that we can't look at hwloc's logical_index here -- hwloc - counts logically across *all* cores. We only want to find the - Nth logical core under this particular socket. */ - obj = dfs_find_nth_item(obj, HWLOC_OBJ_CORE, &count, logical_core_id); - if (NULL == obj) { - return OPAL_ERR_NOT_FOUND; - } - *physical_core_id = obj->os_index; - return OPAL_SUCCESS; -} - diff --git a/opal/mca/paffinity/paffinity.h b/opal/mca/paffinity/paffinity.h deleted file mode 100644 index 90780575bc..0000000000 --- a/opal/mca/paffinity/paffinity.h +++ /dev/null @@ -1,418 +0,0 @@ -/* - * Copyright (c) 2004-2008 The Trustees of Indiana University and Indiana - * University Research and Technology - * Corporation. All rights reserved. - * Copyright (c) 2004-2005 The University of Tennessee and The University - * of Tennessee Research Foundation. All rights - * reserved. - * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, - * University of Stuttgart. All rights reserved. - * Copyright (c) 2004-2005 The Regents of the University of California. - * All rights reserved. - * Copyright (c) 2007-2012 Cisco Systems, Inc. All rights reserved. - * Copyright (c) 2009 Sun Microsystems, Inc. All rights reserved. - * Copyright (c) 2010 IBM Corporation. All rights reserved. - * $COPYRIGHT$ - * - * Additional copyrights may follow - * - * $HEADER$ - */ -/** - * @file - * - * paffinity (processor affinity) framework component interface - * definitions. - * - * Intent - * - * This framework is used to support the OS-specific API for placement - * of processes on processors. It does *not* decide scheduling issues - * -- it is simply for assigning the current process it to a specific - * processor set. As such, the components are likely to be extremely - * short/simple -- there will likely be one component for each OS/API - * that we support (e.g., Linux, IRIX, etc.). As a direct - * consequence, there will likely only be one component that is - * useable on a given platform (making selection easy). - * - * It is *not* an error if there is no paffinity component available; - * processor affinity services are simply not available. Hence, - * paffinity component functions are invoked through short wrapper - * functions in paffinity/base (that check to see if there is a - * selected component before invoking function pointers). If there is - * no selected component, they return an appropriate error code. - * - * In the paffinity interface, we make the distinction between LOGICAL - * and PHYSICAL processors. LOGICAL processors are defined to have - * some corresponding PHYSICAL processor that both exists and is - * currently online. LOGICAL processors numbered countiguously - * starting with 0. PHYSICAL processors are numbered according to the - * underlying operating system; they are represented by integers, but - * no guarantees are made about their values. - * - * Hence, LOGICAL processor IDs are convenient for humans and are in - * the range of [0,N-1] (assuming N processors are currently online). - * Each LOGICAL processor has a 1:1 relationship with a PHYSICAL - * processor, but the PHYSICAL processor's ID can be any unique - * integer value. - * - * ***NOTE*** Obtaining information about socket/core IDs is not well - * supported in many OS's. Programmers using this paffinity interface - * should fully expect to sometimes get OPAL_ERR_NOT_SUPPORTED back - * when calling such functions. - - * General scheme - * - * The component has one function: query(). It simply returns a - * priority (for the unlikely event where there are multiple - * components available on a given platform). - * - * Note that the only real paffinity module that matters these days is - * hwloc. Its module functions will return OPAL_ERR_NOT_INITIALIZED - * if the underlying hwloc system has not yet been initialized (which - * may not have occured yet since we tend to only initialize hwloc - * if/when necessary -- it's not nice to have N processes on a server - * all initialize hwloc simultaneously, because they might all - * simultaneously bang on /proc and /syst, etc.). - * - * The module has the following functions: - * - * - module_init: initialze the module - * - set: set this process's affinity to a specific processor set - * - get: get this process's processor affinity set - * - map physical (socket ID, core ID) -> physical processor ID - * - map physical processor ID -> physical (socket ID, core ID) - * - get the number of logical processors - * - get the number of logical sockets - * - get the number of logical cores on a specific socket - * - map logical processor ID -> physical processor ID - * - map logical socket ID -> physical socket ID - * - map physical socket ID, logical core ID -> physical core ID - * - module_finalize: finalize the module - */ - -#ifndef OPAL_PAFFINITY_H -#define OPAL_PAFFINITY_H - -#include "opal_config.h" - -#ifdef HAVE_STRING_H -#include -#endif - -#include "opal/mca/mca.h" -#include "opal/mca/base/base.h" - -/* ******************************************************************** */ -typedef uint16_t opal_paffinity_locality_t; - -/** Process locality definitions */ -#define OPAL_PROC_LOCALITY_UNKNOWN 0x0000 -#define OPAL_PROC_NON_LOCAL 0x8000 -#define OPAL_PROC_ON_CLUSTER 0x0400 -#define OPAL_PROC_ON_CU 0x0200 -#define OPAL_PROC_ON_NODE 0x0100 -#define OPAL_PROC_ON_BOARD 0x0080 -#define OPAL_PROC_ON_NUMA 0x0040 -#define OPAL_PROC_ON_SOCKET 0x0020 -#define OPAL_PROC_ON_L3CACHE 0x0010 -#define OPAL_PROC_ON_L2CACHE 0x0008 -#define OPAL_PROC_ON_L1CACHE 0x0004 -#define OPAL_PROC_ON_CORE 0x0002 -#define OPAL_PROC_ON_HWTHREAD 0x0001 -#define OPAL_PROC_ALL_LOCAL 0x0fff - -/** Process locality macros */ -#define OPAL_PROC_ON_LOCAL_HWTHREAD(n) ((n) & OPAL_PROC_ON_HWTHREAD) -#define OPAL_PROC_ON_LOCAL_CORE(n) ((n) & OPAL_PROC_ON_CORE) -#define OPAL_PROC_ON_LOCAL_L1CACHE(n) ((n) & OPAL_PROC_ON_L1CACHE) -#define OPAL_PROC_ON_LOCAL_L2CACHE(n) ((n) & OPAL_PROC_ON_L2CACHE) -#define OPAL_PROC_ON_LOCAL_L3CACHE(n) ((n) & OPAL_PROC_ON_L3CACHE) -#define OPAL_PROC_ON_LOCAL_SOCKET(n) ((n) & OPAL_PROC_ON_SOCKET) -#define OPAL_PROC_ON_LOCAL_NUMA(n) ((n) & OPAL_PROC_ON_NUMA) -#define OPAL_PROC_ON_LOCAL_BOARD(n) ((n) & OPAL_PROC_ON_BOARD) -#define OPAL_PROC_ON_LOCAL_NODE(n) ((n) & OPAL_PROC_ON_NODE) -#define OPAL_PROC_ON_LOCAL_CU(n) ((n) & OPAL_PROC_ON_CU) -#define OPAL_PROC_ON_LOCAL_CLUSTER(n) ((n) & OPAL_PROC_ON_CLUSTER) - -/* Process binding modes */ -#define OPAL_PAFFINITY_DO_NOT_BIND 0x01 -#define OPAL_PAFFINITY_BIND_TO_CORE 0x02 -#define OPAL_PAFFINITY_BIND_TO_SOCKET 0x04 -#define OPAL_PAFFINITY_BIND_TO_BOARD 0x08 -#define OPAL_PAFFINITY_BIND_IF_SUPPORTED 0x80 -/* ******************************************************************** */ - - -/** - * Buffer type for paffinity processor masks. - * Copied almost directly from PLPA. - */ - -/** - * \internal - * Internal type used for the underlying bitmask unit - */ -typedef unsigned long int opal_paffinity_base_bitmask_t; - -/** - * \internal - * Number of bits in opal_paffinity_base_bitmask_t - */ -#define OPAL_PAFFINITY_BITMASK_T_NUM_BITS (sizeof(opal_paffinity_base_bitmask_t) * 8) -/** - * \internal - * How many bits we want - */ -#define OPAL_PAFFINITY_BITMASK_CPU_MAX 1024 -/** - * \internal - * How many opal_paffinity_base_bitmask_t's we need - */ -#define OPAL_PAFFINITY_BITMASK_NUM_ELEMENTS (OPAL_PAFFINITY_BITMASK_CPU_MAX / OPAL_PAFFINITY_BITMASK_T_NUM_BITS) - -/** - * \internal - * How many bytes in a cpu set - */ -#define OPAL_PAFFINITY_CPU_SET_NUM_BYTES (OPAL_PAFFINITY_BITMASK_NUM_ELEMENTS * sizeof(opal_paffinity_base_bitmask_t)) - -/** - * Public processor bitmask type - */ -typedef struct opal_paffinity_base_cpu_set_t { - opal_paffinity_base_bitmask_t bitmask[OPAL_PAFFINITY_BITMASK_NUM_ELEMENTS]; -} opal_paffinity_base_cpu_set_t; - -/***************************************************************************/ - -/** - * \internal - * Internal macro for identifying the byte in a bitmask array - */ -#define OPAL_PAFFINITY_CPU_BYTE(num) ((num) / OPAL_PAFFINITY_BITMASK_T_NUM_BITS) - -/** - * \internal - * Internal macro for identifying the bit in a bitmask array - */ -#define OPAL_PAFFINITY_CPU_BIT(num) ((num) % OPAL_PAFFINITY_BITMASK_T_NUM_BITS) - -/***************************************************************************/ - -/** - * Public macro to zero out a OPAL_PAFFINITY cpu set - */ -#define OPAL_PAFFINITY_CPU_ZERO(cpuset) \ - memset(&(cpuset), 0, sizeof(opal_paffinity_base_cpu_set_t)) - -/** - * Public macro to set a bit in a OPAL_PAFFINITY cpu set - */ -#define OPAL_PAFFINITY_CPU_SET(num, cpuset) \ - (cpuset).bitmask[OPAL_PAFFINITY_CPU_BYTE(num)] |= ((opal_paffinity_base_bitmask_t) 1 << OPAL_PAFFINITY_CPU_BIT(num)) - -/** - * Public macro to clear a bit in a OPAL_PAFFINITY cpu set - */ -#define OPAL_PAFFINITY_CPU_CLR(num, cpuset) \ - (cpuset).bitmask[OPAL_PAFFINITY_CPU_BYTE(num)] &= ~((opal_paffinity_base_bitmask_t) 1 << OPAL_PAFFINITY_CPU_BIT(num)) - -/** - * Public macro to test if a bit is set in a OPAL_PAFFINITY cpu set - */ -#define OPAL_PAFFINITY_CPU_ISSET(num, cpuset) \ - (0 != (((cpuset).bitmask[OPAL_PAFFINITY_CPU_BYTE(num)]) & ((opal_paffinity_base_bitmask_t) 1 << OPAL_PAFFINITY_CPU_BIT(num)))) - -/** - * Public macro to test if a process is bound anywhere - */ -#define OPAL_PAFFINITY_PROCESS_IS_BOUND(cpuset, bound) \ - do { \ - int i, num_processors, num_bound; \ - *(bound) = false; \ - if (OPAL_SUCCESS == \ - opal_paffinity_base_get_processor_info(&num_processors)) { \ - num_bound = 0; \ - for (i = 0; i < OPAL_PAFFINITY_BITMASK_CPU_MAX; i++) { \ - if (OPAL_PAFFINITY_CPU_ISSET(i, (cpuset))) { \ - num_bound++; \ - } \ - } \ - if (0 < num_bound && (1 == num_processors || \ - num_bound < num_processors)) { \ - *(bound) = true; \ - } \ - } \ - } while(0); - -/***************************************************************************/ - -/** - * Module initialization function. Should return OPAL_SUCCESS. - */ -typedef int (*opal_paffinity_base_module_init_1_1_0_fn_t)(void); - -/** - * Module function to set this process' affinity to a specific set of - * PHYSICAL CPUs. - */ -typedef int (*opal_paffinity_base_module_set_fn_t)(opal_paffinity_base_cpu_set_t cpumask); - - -/** - * Module function to get this process' affinity to a specific set of - * PHYSICAL CPUs. Returns any binding in the cpumask. This function -only- - * returns something other than OPAL_SUCCESS if an actual error is encountered. - * You will need to check the mask to find out if this process is actually - * bound somewhere specific - a macro for that purpose is provided above - */ -typedef int (*opal_paffinity_base_module_get_fn_t)(opal_paffinity_base_cpu_set_t *cpumask); - -/** - * Returns mapping of PHYSICAL socket:core -> PHYSICAL processor id. - * - * Return OPAL_SUCCESS or OPAL_ERR_NOT_SUPPORTED if not - * supported - */ -typedef int (*opal_paffinity_base_module_get_map_to_processor_id_fn_t)(int physical_socket, - int physical_core, - int *physical_processor_id); - -/** - * Provides mapping of PHYSICAL processor id -> PHYSICAL socket:core. - * - * Return OPAL_SUCCESS or OPAL_ERR_NOT_SUPPORTED if not - * supported - */ -typedef int (*opal_paffinity_base_module_get_map_to_socket_core_fn_t)(int physical_processor_id, - int *physical_socket, - int *physical_core); - -/** - * Provides number of LOGICAL processors in a host. - * - * Return OPAL_SUCCESS or OPAL_ERR_NOT_SUPPORTED if not - * supported - */ -typedef int (*opal_paffinity_base_module_get_processor_info_fn_t)(int *num_processors); - -/** - * Provides the number of LOGICAL sockets in a host. - * - * Return OPAL_SUCCESS or OPAL_ERR_NOT_SUPPORTED if not - * supported - */ -typedef int (*opal_paffinity_base_module_get_socket_info_fn_t)(int *num_sockets); - -/** - * Provides the number of LOGICAL cores in a PHYSICAL socket. - * - * Returns OPAL_SUCCESS or OPAL_ERR_NOT_SUPPORTED if not - * supported. - */ -typedef int (*opal_paffinity_base_module_get_core_info_fn_t)(int physical_socket, int *num_cores); - -/** - * Provide the PHYSICAL processor ID that corresponds to the - * given LOGICAL processor ID. - * - * Return OPAL_ERR_NOT_SUPPORTED if not supported. - */ -typedef int (*opal_paffinity_base_module_get_physical_processor_id_fn_t)(int logical_processor_id, int *physical_processor_id); - -/** - * Provide the PHYSICAL socket ID that corresponds to the given - * LOGICAL socket ID. - * - * Return OPAL_ERR_NOT_SUPPORTED if not supported. - */ -typedef int (*opal_paffinity_base_module_get_physical_socket_id_fn_t)(int logical_socket_id, int *physical_socket_id); - -/** - * Provide the PHYSICAL core ID that corresponds to the given LOGICAL - * core ID on the given PHYSICAL socket ID. - * - * Return OPAL_ERR_NOT_SUPPORTED if not supported. - */ -typedef int (*opal_paffinity_base_module_get_physical_core_id_fn_t)(int physical_socket_id, int logical_core_id, int *physical_core_id); - - -/** - * Module finalize function. Invoked by the base on the selected - * module when the paffinity framework is being shut down. - */ -typedef int (*opal_paffinity_base_module_finalize_fn_t)(void); - - -/** - * Structure for paffinity components. - */ -struct opal_paffinity_base_component_2_0_1_t { - /** MCA base component */ - mca_base_component_t base_version; - /** MCA base data */ - mca_base_component_data_t base_data; -}; -/** - * Convenience typedef - */ -typedef struct opal_paffinity_base_component_2_0_1_t opal_paffinity_base_component_2_0_1_t; -typedef struct opal_paffinity_base_component_2_0_1_t opal_paffinity_base_component_t; - - -/** - * Structure for paffinity modules - */ -struct opal_paffinity_base_module_1_1_0_t { - /** Module initialization function */ - opal_paffinity_base_module_init_1_1_0_fn_t paff_module_init; - - /** Set this process' affinity */ - opal_paffinity_base_module_set_fn_t paff_module_set; - - /** Get this process' affinity */ - opal_paffinity_base_module_get_fn_t paff_module_get; - - /** Map socket:core to processor ID */ - opal_paffinity_base_module_get_map_to_processor_id_fn_t paff_get_map_to_processor_id; - - /** Map processor ID to socket:core */ - opal_paffinity_base_module_get_map_to_socket_core_fn_t paff_get_map_to_socket_core; - - /** Return the max processor ID */ - opal_paffinity_base_module_get_processor_info_fn_t paff_get_processor_info; - - /** Return the max socket number */ - opal_paffinity_base_module_get_socket_info_fn_t paff_get_socket_info; - - /** Return the max core number */ - opal_paffinity_base_module_get_core_info_fn_t paff_get_core_info; - - /* Return physical processor id */ - opal_paffinity_base_module_get_physical_processor_id_fn_t paff_get_physical_processor_id; - - /* Return physical socket id */ - opal_paffinity_base_module_get_physical_socket_id_fn_t paff_get_physical_socket_id; - - /* Return physical core id */ - opal_paffinity_base_module_get_physical_core_id_fn_t paff_get_physical_core_id; - - /** Shut down this module */ - opal_paffinity_base_module_finalize_fn_t paff_module_finalize; -}; -/** - * Convenience typedef - */ -typedef struct opal_paffinity_base_module_1_1_0_t opal_paffinity_base_module_1_1_0_t; -typedef struct opal_paffinity_base_module_1_1_0_t opal_paffinity_base_module_t; - - -/* - * Macro for use in components that are of type paffinity - */ -#define OPAL_PAFFINITY_BASE_VERSION_2_0_1 \ - MCA_BASE_VERSION_2_0_0, \ - "paffinity", 2, 0, 1 - -#endif /* OPAL_PAFFINITY_H */ diff --git a/opal/mca/paffinity/test/Makefile.am b/opal/mca/paffinity/test/Makefile.am deleted file mode 100644 index 7fd2e842c3..0000000000 --- a/opal/mca/paffinity/test/Makefile.am +++ /dev/null @@ -1,44 +0,0 @@ -# -# Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana -# University Research and Technology -# Corporation. All rights reserved. -# Copyright (c) 2004-2005 The University of Tennessee and The University -# of Tennessee Research Foundation. All rights -# reserved. -# Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, -# University of Stuttgart. All rights reserved. -# Copyright (c) 2004-2005 The Regents of the University of California. -# All rights reserved. -# Copyright (c) 2010 Cisco Systems, Inc. All rights reserved. -# $COPYRIGHT$ -# -# Additional copyrights may follow -# -# $HEADER$ -# - -sources = \ - paffinity_test.h \ - paffinity_test_component.c \ - paffinity_test_module.c - -# Make the output library in this directory, and name it either -# mca__.la (for DSO builds) or libmca__.la -# (for static builds). - -if MCA_BUILD_opal_paffinity_test_DSO -component_noinst = -component_install = mca_paffinity_test.la -else -component_noinst = libmca_paffinity_test.la -component_install = -endif - -mcacomponentdir = $(pkglibdir) -mcacomponent_LTLIBRARIES = $(component_install) -mca_paffinity_test_la_SOURCES = $(sources) -mca_paffinity_test_la_LDFLAGS = -module -avoid-version - -noinst_LTLIBRARIES = $(component_noinst) -libmca_paffinity_test_la_SOURCES =$(sources) -libmca_paffinity_test_la_LDFLAGS = -module -avoid-version diff --git a/opal/mca/paffinity/test/configure.m4 b/opal/mca/paffinity/test/configure.m4 deleted file mode 100644 index 800bab52ff..0000000000 --- a/opal/mca/paffinity/test/configure.m4 +++ /dev/null @@ -1,38 +0,0 @@ -# -*- shell-script -*- -# -# Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana -# University Research and Technology -# Corporation. All rights reserved. -# Copyright (c) 2004-2005 The University of Tennessee and The University -# of Tennessee Research Foundation. All rights -# reserved. -# Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, -# University of Stuttgart. All rights reserved. -# Copyright (c) 2004-2005 The Regents of the University of California. -# All rights reserved. -# Copyright (c) 2007-2010 Cisco Systems, Inc. All rights reserved. -# Copyright (c) 2008 Sun Microsystems, Inc. All rights reserved. -# $COPYRIGHT$ -# -# Additional copyrights may follow -# -# $HEADER$ -# - -# MCA_paffinity_test_CONFIG([action-if-found], [action-if-not-found]) -# ----------------------------------------------------------- -AC_DEFUN([MCA_opal_paffinity_test_CONFIG],[ - AC_CONFIG_FILES([opal/mca/paffinity/test/Makefile]) - - # check to see if we have - # as this is a Darwin-specific thing and - # we are a test module for that environment - - AC_MSG_CHECKING([if this is an --enable-debug build]) - AS_IF([test "$WANT_DEBUG" = "1"], - [AC_MSG_RESULT([yes]) - AC_CHECK_HEADER([mach/mach_host.h], [$1], [$2])], - [AC_MSG_RESULT([no (component disabled)]) - $2]) -])dnl - diff --git a/opal/mca/paffinity/test/paffinity_test.h b/opal/mca/paffinity/test/paffinity_test.h deleted file mode 100644 index 13f2a61170..0000000000 --- a/opal/mca/paffinity/test/paffinity_test.h +++ /dev/null @@ -1,46 +0,0 @@ -/* - * Copyright (c) 2004-2008 The Trustees of Indiana University and Indiana - * University Research and Technology - * Corporation. All rights reserved. - * Copyright (c) 2004-2005 The University of Tennessee and The University - * of Tennessee Research Foundation. All rights - * reserved. - * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, - * University of Stuttgart. All rights reserved. - * Copyright (c) 2004-2005 The Regents of the University of California. - * All rights reserved. - * Copyright (c) 2007 Cisco Systems, Inc. All rights reserved. - * $COPYRIGHT$ - * - * Additional copyrights may follow - * - * $HEADER$ - */ - -#ifndef MCA_PAFFINITY_TEST_EXPORT_H -#define MCA_PAFFINITY_TEST_EXPORT_H - -#include "opal_config.h" - -#include "opal/mca/mca.h" -#include "opal/mca/paffinity/paffinity.h" - -BEGIN_C_DECLS - -/* - * Globally exported variable - */ -typedef struct opal_paffinity_test_component_t { - opal_paffinity_base_component_t super; - bool bound; - int num_sockets; - int num_cores; -} opal_paffinity_test_component_t; - -OPAL_MODULE_DECLSPEC extern opal_paffinity_test_component_t mca_paffinity_test_component; - -extern opal_paffinity_base_module_t opal_paffinity_test_module; - -END_C_DECLS - -#endif /* MCA_PAFFINITY_TEST_EXPORT_H */ diff --git a/opal/mca/paffinity/test/paffinity_test_component.c b/opal/mca/paffinity/test/paffinity_test_component.c deleted file mode 100644 index 5bf328fe27..0000000000 --- a/opal/mca/paffinity/test/paffinity_test_component.c +++ /dev/null @@ -1,109 +0,0 @@ -/* - * Copyright (c) 2004-2008 The Trustees of Indiana University and Indiana - * University Research and Technology - * Corporation. All rights reserved. - * Copyright (c) 2004-2005 The University of Tennessee and The University - * of Tennessee Research Foundation. All rights - * reserved. - * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, - * University of Stuttgart. All rights reserved. - * Copyright (c) 2004-2005 The Regents of the University of California. - * All rights reserved. - * Copyright (c) 2007-2010 Cisco Systems, Inc. All rights reserved. - * $COPYRIGHT$ - * - * Additional copyrights may follow - * - * $HEADER$ - * - * These symbols are in a file by themselves to provide nice linker - * semantics. Since linkers generally pull in symbols by object - * files, keeping these symbols as the only symbols in this file - * prevents utility programs such as "ompi_info" from having to import - * entire components just to query their version and parameters. - */ - -#include "opal_config.h" - -#include "opal/constants.h" -#include "opal/mca/paffinity/paffinity.h" -#include "paffinity_test.h" - -/* - * Public string showing the paffinity ompi_test component version number - */ -const char *opal_paffinity_test_component_version_string = - "OPAL test paffinity MCA component version " OPAL_VERSION; - -/* - * Local function - */ -static int test_open(void); -static int test_query(mca_base_module_t **module, int *priority); - -/* - * Instantiate the public struct with all of our public information - * and pointers to our public functions in it - */ - -opal_paffinity_test_component_t mca_paffinity_test_component = { - { - /* First, the mca_component_t struct containing meta information - * about the component itself - */ - - { - /* Indicate that we are a paffinity v1.1.0 component (which also - implies a specific MCA version) */ - - OPAL_PAFFINITY_BASE_VERSION_2_0_1, - - /* Component name and version */ - - "test", - OPAL_MAJOR_VERSION, - OPAL_MINOR_VERSION, - OPAL_RELEASE_VERSION, - - /* Component open and close functions */ - - test_open, - NULL, - test_query - }, - /* Next the MCA v1.0.0 component meta data */ - { - /* The component is checkpoint ready */ - MCA_BASE_METADATA_PARAM_CHECKPOINT - } - } -}; - - -static int test_open(void) -{ - int tmp; - - mca_base_param_reg_int(&mca_paffinity_test_component.super.base_version, "bound", - "Whether or not to test as if externally bound (default=0: no)", - false, false, (int)false, &tmp); - mca_paffinity_test_component.bound = OPAL_INT_TO_BOOL(tmp); - - mca_base_param_reg_int(&mca_paffinity_test_component.super.base_version, "num_sockets", - "Number of sockets on each node (default=4)", - false, false, 4, &mca_paffinity_test_component.num_sockets); - - mca_base_param_reg_int(&mca_paffinity_test_component.super.base_version, "num_cores", - "Number of cores in each socket (default=4)", - false, false, 4, &mca_paffinity_test_component.num_cores); - return OPAL_SUCCESS; -} - -static int test_query(mca_base_module_t **module, int *priority) -{ - /* set this priority so I can only be selected if directed */ - *priority = 5; - *module = (mca_base_module_t *)&opal_paffinity_test_module; - - return OPAL_SUCCESS; -} diff --git a/opal/mca/paffinity/test/paffinity_test_module.c b/opal/mca/paffinity/test/paffinity_test_module.c deleted file mode 100644 index 487fef5ca9..0000000000 --- a/opal/mca/paffinity/test/paffinity_test_module.c +++ /dev/null @@ -1,167 +0,0 @@ -/* - * Copyright (c) 2004-2008 The Trustees of Indiana University and Indiana - * University Research and Technology - * Corporation. All rights reserved. - * Copyright (c) 2004-2005 The University of Tennessee and The University - * of Tennessee Research Foundation. All rights - * reserved. - * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, - * University of Stuttgart. All rights reserved. - * Copyright (c) 2004-2005 The Regents of the University of California. - * All rights reserved. - * Copyright (c) 2007-2010 Cisco Systems, Inc. All rights reserved. - * Copyright (c) 2008 Sun Microsystems, Inc. All rights reserved. - * - * $COPYRIGHT$ - * - * Additional copyrights may follow - * - * $HEADER$ - */ - -#include "opal_config.h" -#include "opal/constants.h" - -#include "opal/mca/base/mca_base_param.h" -#include "opal/mca/paffinity/paffinity.h" -#include "opal/mca/paffinity/base/base.h" - -#include "paffinity_test.h" - -/* - * Local functions - */ -static int init(void); -static int set(opal_paffinity_base_cpu_set_t cpumask); -static int get(opal_paffinity_base_cpu_set_t *cpumask); -static int finalize(void); -static int map_to_processor_id(int socket, int core, int *processor_id); -static int map_to_socket_core(int processor_id, int *socket, int *core); -static int get_processor_info(int *num_processors); -static int get_socket_info(int *num_sockets); -static int get_core_info(int socket, int *num_cores); -static int get_physical_processor_id(int logical_processor_id, - int *physical_processor_id); -static int get_physical_socket_id(int logical_socket_id, - int *physical_socket_id); -static int get_physical_core_id(int physical_socket_id, int logical_core_id, - int *physical_core_id); - -/* - * Test paffinity module - */ -opal_paffinity_base_module_t opal_paffinity_test_module = { - /* Initialization function */ - init, - - /* Module function pointers */ - set, - get, - map_to_processor_id, - map_to_socket_core, - get_processor_info, - get_socket_info, - get_core_info, - get_physical_processor_id, - get_physical_socket_id, - get_physical_core_id, - finalize -}; - -/* nothing to init here */ -static int init(void) -{ - return OPAL_SUCCESS; -} - -/* this gives us a cpumask which tells which CPU to bind */ -static int set(opal_paffinity_base_cpu_set_t cpumask) -{ - return OPAL_SUCCESS; -} - -/* This get function returns the CPU id that's currently binded, - * and then sets the cpumask. */ -static int get(opal_paffinity_base_cpu_set_t *cpumask) -{ - int i; - - OPAL_PAFFINITY_CPU_ZERO(*cpumask); - if (mca_paffinity_test_component.bound) { - for (i=0; i < mca_paffinity_test_component.num_sockets*mca_paffinity_test_component.num_cores; i+=2) { - OPAL_PAFFINITY_CPU_SET(i, *cpumask); - } - /* assign all cores in the 2nd socket, if it exists */ - if (mca_paffinity_test_component.num_sockets >= 2) { - for (i=mca_paffinity_test_component.num_cores; i < 2*mca_paffinity_test_component.num_cores; i++) { - OPAL_PAFFINITY_CPU_SET(i, *cpumask); - } - } - } else { - for (i=0; i < mca_paffinity_test_component.num_sockets*mca_paffinity_test_component.num_cores; i++) { - OPAL_PAFFINITY_CPU_SET(i, *cpumask); - } - } - return OPAL_SUCCESS; -} - -static int map_to_processor_id(int socket, int core, int *processor_id) -{ - *processor_id = socket*mca_paffinity_test_component.num_cores + core; - return OPAL_SUCCESS; -} - -static int map_to_socket_core(int processor_id, int *socket, int *core) -{ - *socket = processor_id / mca_paffinity_test_component.num_cores; - *core = processor_id % mca_paffinity_test_component.num_cores; - return OPAL_SUCCESS; -} - -static int get_processor_info(int *num_processors) -{ - *num_processors = mca_paffinity_test_component.num_sockets * mca_paffinity_test_component.num_cores; - return OPAL_SUCCESS; -} - -static int get_socket_info(int *num_sockets) -{ - *num_sockets = mca_paffinity_test_component.num_sockets; - return OPAL_SUCCESS; -} - -static int get_core_info(int socket, int *num_cores) -{ - *num_cores = mca_paffinity_test_component.num_cores; - return OPAL_SUCCESS; -} - -static int get_physical_processor_id(int logical_processor_id, - int *physical_processor_id) -{ - *physical_processor_id = logical_processor_id; - return OPAL_SUCCESS; -} - -static int get_physical_socket_id(int logical_socket_id, - int *physical_processor_id) -{ - *physical_processor_id = logical_socket_id; - return OPAL_SUCCESS; -} - -static int get_physical_core_id(int physical_socket_id, int logical_core_id, - int *physical_core_id) -{ - if (mca_paffinity_test_component.num_cores < logical_core_id) { - return OPAL_ERROR; - } - *physical_core_id = logical_core_id; - return OPAL_SUCCESS; -} - -static int finalize(void) -{ - return OPAL_SUCCESS; -} - diff --git a/opal/runtime/opal_cr.c b/opal/runtime/opal_cr.c index 66052a3f02..0448e5a957 100644 --- a/opal/runtime/opal_cr.c +++ b/opal/runtime/opal_cr.c @@ -12,6 +12,7 @@ * Copyright (c) 2007 Los Alamos National Security, LLC. All rights * reserved. * Copyright (c) 2011 Oak Ridge National Labs. All rights reserved. + * Copyright (c) 2012 Cisco Systems, Inc. All rights reserved. * $COPYRIGHT$ * * Additional copyrights may follow @@ -66,8 +67,6 @@ #include "opal/mca/memcpy/base/base.h" #include "opal/mca/memory/base/base.h" #include "opal/mca/timer/base/base.h" -#include "opal/mca/paffinity/base/base.h" -#include "opal/mca/paffinity/base/base.h" #include "opal/threads/mutex.h" #include "opal/threads/threads.h" diff --git a/opal/runtime/opal_finalize.c b/opal/runtime/opal_finalize.c index 5004c41202..ab47deb8cc 100644 --- a/opal/runtime/opal_finalize.c +++ b/opal/runtime/opal_finalize.c @@ -9,7 +9,7 @@ * University of Stuttgart. All rights reserved. * Copyright (c) 2004-2005 The Regents of the University of California. * All rights reserved. - * Copyright (c) 2008 Cisco Systems, Inc. All rights reserved. + * Copyright (c) 2008-2012 Cisco Systems, Inc. All rights reserved. * Copyright (c) 2010-2012 Los Alamos National Security, LLC. * All rights reserved. * $COPYRIGHT$ @@ -42,10 +42,8 @@ #include "opal/mca/backtrace/base/base.h" #include "opal/mca/timer/base/base.h" #include "opal/mca/hwloc/base/base.h" -#include "opal/mca/paffinity/base/base.h" #include "opal/mca/event/base/base.h" #include "opal/runtime/opal_progress.h" -#include "opal/mca/carto/base/base.h" #include "opal/mca/shmem/base/base.h" #if OPAL_ENABLE_FT_CR == 1 #include "opal/mca/compress/base/base.h" @@ -141,18 +139,12 @@ opal_finalize(void) /* finalize the memory manager / tracker */ opal_mem_hooks_finalize(); - /* close the carto framework */ - opal_carto_base_close(); - /* close the shmem framework */ opal_shmem_base_close(); /* close the hwloc framework */ opal_hwloc_base_close(); - /* close the processor affinity base */ - opal_paffinity_base_close(); - /* close the memcpy base */ opal_memcpy_base_close(); diff --git a/opal/runtime/opal_init.c b/opal/runtime/opal_init.c index 8d909f0701..2daeca1fda 100644 --- a/opal/runtime/opal_init.c +++ b/opal/runtime/opal_init.c @@ -38,11 +38,9 @@ #include "opal/mca/memory/base/base.h" #include "opal/mca/memcpy/base/base.h" #include "opal/mca/hwloc/base/base.h" -#include "opal/mca/paffinity/base/base.h" #include "opal/mca/timer/base/base.h" #include "opal/mca/memchecker/base/base.h" #include "opal/dss/dss.h" -#include "opal/mca/carto/base/base.h" #include "opal/mca/shmem/base/base.h" #if OPAL_ENABLE_FT_CR == 1 #include "opal/mca/compress/base/base.h" @@ -213,9 +211,12 @@ opal_err2str(int errnum, const char **errmsg) case OPAL_ERR_NOT_INITIALIZED: retval = "Not initialized"; break; + case OPAL_ERR_NOT_BOUND: + retval = "Not bound"; + break; default: retval = NULL; -} + } *errmsg = retval; return OPAL_SUCCESS; @@ -350,16 +351,6 @@ opal_init(int* pargc, char*** pargv) goto return_error; } - /* open the processor affinity base */ - if (OPAL_SUCCESS != (ret = opal_paffinity_base_open())) { - error = "opal_paffinity_base_open"; - goto return_error; - } - if (OPAL_SUCCESS != (ret = opal_paffinity_base_select())) { - error = "opal_paffinity_base_select"; - goto return_error; - } - /* the memcpy component should be one of the first who get * loaded in order to make sure we ddo have all the available * versions of memcpy correctly configured. @@ -406,17 +397,6 @@ opal_init(int* pargc, char*** pargv) goto return_error; } - /* setup the carto framework */ - if (OPAL_SUCCESS != (ret = opal_carto_base_open())) { - error = "opal_carto_base_open"; - goto return_error; - } - - if (OPAL_SUCCESS != (ret = opal_carto_base_select())) { - error = "opal_carto_base_select"; - goto return_error; - } - /* * Need to start the event and progress engines if noone else is. * opal_cr_init uses the progress engine, so it is lumped together diff --git a/opal/runtime/opal_params.c b/opal/runtime/opal_params.c index 5811c6c280..566daecfbb 100644 --- a/opal/runtime/opal_params.c +++ b/opal/runtime/opal_params.c @@ -11,7 +11,7 @@ * All rights reserved. * Copyright (c) 2006 Los Alamos National Security, LLC. All rights * reserved. - * Copyright (c) 2008-2011 Cisco Systems, Inc. All rights reserved. + * Copyright (c) 2008-2012 Cisco Systems, Inc. All rights reserved. * Copyright (c) 2009 Oak Ridge National Labs. All rights reserved. * Copyright (c) 2010 Los Alamos National Security, LLC. * All rights reserved. @@ -35,7 +35,6 @@ #include "opal/mca/base/mca_base_param.h" #include "opal/threads/mutex.h" #include "opal/threads/threads.h" -#include "opal/mca/paffinity/base/base.h" #include "opal/mca/shmem/base/base.h" int opal_register_params(void) @@ -117,6 +116,5 @@ int opal_register_params(void) return ret; } - /* Paffinity base also has some parameters */ - return opal_paffinity_base_register_params(); + return OPAL_SUCCESS; } diff --git a/orte/mca/ess/base/base.h b/orte/mca/ess/base/base.h index b4ca0c5ca3..467e2e5a6c 100644 --- a/orte/mca/ess/base/base.h +++ b/orte/mca/ess/base/base.h @@ -9,7 +9,7 @@ * University of Stuttgart. All rights reserved. * Copyright (c) 2004-2005 The Regents of the University of California. * All rights reserved. - * Copyright (c) 2011 Cisco Systems, Inc. All rights reserved. + * Copyright (c) 2011-2012 Cisco Systems, Inc. All rights reserved. * Copyright (c) 2012 Oak Ridge National Labs. All rights reserved. * $COPYRIGHT$ * @@ -83,7 +83,7 @@ ORTE_DECLSPEC int orte_ess_base_tool_finalize(void); ORTE_DECLSPEC int orte_ess_base_orted_setup(char **hosts); ORTE_DECLSPEC int orte_ess_base_orted_finalize(void); -ORTE_DECLSPEC opal_paffinity_locality_t orte_ess_base_proc_get_locality(orte_process_name_t *proc); +ORTE_DECLSPEC opal_hwloc_locality_t orte_ess_base_proc_get_locality(orte_process_name_t *proc); ORTE_DECLSPEC orte_vpid_t orte_ess_base_proc_get_daemon(orte_process_name_t *proc); ORTE_DECLSPEC char* orte_ess_base_proc_get_hostname(orte_process_name_t *proc); ORTE_DECLSPEC orte_local_rank_t orte_ess_base_proc_get_local_rank(orte_process_name_t *proc); @@ -91,6 +91,11 @@ ORTE_DECLSPEC orte_node_rank_t orte_ess_base_proc_get_node_rank(orte_process_nam ORTE_DECLSPEC int orte_ess_base_update_pidmap(opal_byte_object_t *bo); ORTE_DECLSPEC int orte_ess_base_update_nidmap(opal_byte_object_t *bo); +/* Detect whether or not this proc is bound - if not, + * see if it should bind itself + */ +ORTE_DECLSPEC int orte_ess_base_proc_binding(void); + /* * Put functions */ diff --git a/orte/mca/ess/base/ess_base_fns.c b/orte/mca/ess/base/ess_base_fns.c index 9be0a9b093..b795b1d091 100644 --- a/orte/mca/ess/base/ess_base_fns.c +++ b/orte/mca/ess/base/ess_base_fns.c @@ -9,7 +9,7 @@ * University of Stuttgart. All rights reserved. * Copyright (c) 2004-2005 The Regents of the University of California. * All rights reserved. - * Copyright (c) 2011 Cisco Systems, Inc. All rights reserved. + * Copyright (c) 2011-2012 Cisco Systems, Inc. All rights reserved. * Copyright (c) 2011-2012 Los Alamos National Security, LLC. * All rights reserved. * $COPYRIGHT$ @@ -29,12 +29,13 @@ #include #include "opal/util/output.h" -#include "opal/mca/paffinity/paffinity.h" +#include "opal/mca/hwloc/base/base.h" #include "orte/mca/errmgr/errmgr.h" #include "orte/util/name_fns.h" #include "orte/util/nidmap.h" #include "orte/util/proc_info.h" +#include "orte/util/show_help.h" #include "orte/runtime/orte_globals.h" #include "orte/mca/ess/base/base.h" @@ -51,7 +52,7 @@ static orte_proc_t* find_proc(orte_process_name_t *proc) /* used by daemons */ return (orte_proc_t*)opal_pointer_array_get_item(jdata->procs, proc->vpid); } -opal_paffinity_locality_t orte_ess_base_proc_get_locality(orte_process_name_t *proc) +opal_hwloc_locality_t orte_ess_base_proc_get_locality(orte_process_name_t *proc) { orte_pmap_t *pmap; @@ -262,3 +263,229 @@ int orte_ess_base_update_nidmap(opal_byte_object_t *bo) return rc; } +int orte_ess_base_proc_binding(void) +{ +#if OPAL_HAVE_HWLOC + hwloc_obj_t node, obj; + hwloc_cpuset_t cpus, nodeset; + orte_node_rank_t nrank; + hwloc_obj_type_t target; + unsigned int cache_level = 0; + struct hwloc_topology_support *support; + char *map; + int ret; + char *error; + + /* Determine if we were pre-bound or not */ + if (NULL != getenv("OMPI_MCA_orte_bound_at_launch")) { + orte_proc_is_bound = true; + if (NULL != (map = getenv("OMPI_MCA_orte_base_applied_binding"))) { + orte_proc_applied_binding = hwloc_bitmap_alloc(); + if (0 != (ret = hwloc_bitmap_list_sscanf(orte_proc_applied_binding, map))) { + error = "applied_binding parse"; + goto error; + } + } + } + + /* see if we were bound when launched */ + if (!orte_proc_is_bound) { + /* we were not bound at launch */ + if (NULL != opal_hwloc_topology) { + support = (struct hwloc_topology_support*)hwloc_topology_get_support(opal_hwloc_topology); + /* get our node object */ + node = hwloc_get_root_obj(opal_hwloc_topology); + nodeset = opal_hwloc_base_get_available_cpus(opal_hwloc_topology, node); + /* get our bindings */ + cpus = hwloc_bitmap_alloc(); + if (hwloc_get_cpubind(opal_hwloc_topology, cpus, HWLOC_CPUBIND_PROCESS) < 0) { + /* we are NOT bound if get_cpubind fails, nor can we be bound - the + * environment does not support it + */ + hwloc_bitmap_free(cpus); + goto MOVEON; + } + /* we are bound if the two cpusets are not equal, + * or if there is only ONE cpu available to us + */ + if (0 != hwloc_bitmap_compare(cpus, nodeset) || + opal_hwloc_base_single_cpu(nodeset) || + opal_hwloc_base_single_cpu(cpus)) { + /* someone external set it - indicate it is set + * so that we know + */ + orte_proc_is_bound = true; + hwloc_bitmap_free(cpus); + } else if (support->cpubind->set_thisproc_cpubind && + OPAL_BINDING_POLICY_IS_SET(opal_hwloc_binding_policy) && + OPAL_BIND_TO_NONE != OPAL_GET_BINDING_POLICY(opal_hwloc_binding_policy)) { + /* the system is capable of doing processor affinity, but it + * has not yet been set - see if a slot_list was given + */ + hwloc_bitmap_zero(cpus); + if (OPAL_BIND_TO_CPUSET == OPAL_GET_BINDING_POLICY(opal_hwloc_binding_policy)) { + if (OPAL_SUCCESS != (ret = opal_hwloc_base_slot_list_parse(opal_hwloc_base_slot_list, + opal_hwloc_topology, cpus))) { + error = "Setting processor affinity failed"; + hwloc_bitmap_free(cpus); + goto error; + } + if (0 > hwloc_set_cpubind(opal_hwloc_topology, cpus, 0)) { + error = "Setting processor affinity failed"; + hwloc_bitmap_free(cpus); + goto error; + } + /* try to find a level and index for this location */ + opal_hwloc_base_get_level_and_index(cpus, &orte_process_info.bind_level, &orte_process_info.bind_idx); + /* cleanup */ + hwloc_bitmap_free(cpus); + orte_proc_is_bound = true; + } else { + /* cleanup */ + hwloc_bitmap_free(cpus); + /* get the node rank */ + if (ORTE_NODE_RANK_INVALID == (nrank = orte_ess.get_node_rank(ORTE_PROC_MY_NAME))) { + /* this is not an error - could be due to being + * direct launched - so just ignore and leave + * us unbound + */ + goto MOVEON; + } + /* if the binding policy is hwthread, then we bind to the nrank-th + * hwthread on this node + */ + if (OPAL_BIND_TO_HWTHREAD == OPAL_GET_BINDING_POLICY(opal_hwloc_binding_policy)) { + if (NULL == (obj = opal_hwloc_base_get_obj_by_type(opal_hwloc_topology, HWLOC_OBJ_PU, + 0, nrank, OPAL_HWLOC_LOGICAL))) { + ret = ORTE_ERR_NOT_FOUND; + error = "Getting hwthread object"; + goto error; + } + cpus = opal_hwloc_base_get_available_cpus(opal_hwloc_topology, obj); + if (0 > hwloc_set_cpubind(opal_hwloc_topology, cpus, 0)) { + ret = ORTE_ERROR; + error = "Setting processor affinity failed"; + goto error; + } + orte_process_info.bind_level = OPAL_HWLOC_L1CACHE_LEVEL; + orte_process_info.bind_idx = nrank; + } else if (OPAL_BIND_TO_CORE == OPAL_GET_BINDING_POLICY(opal_hwloc_binding_policy)) { + /* if the binding policy is core, then we bind to the nrank-th + * core on this node + */ + if (NULL == (obj = opal_hwloc_base_get_obj_by_type(opal_hwloc_topology, HWLOC_OBJ_CORE, + 0, nrank, OPAL_HWLOC_LOGICAL))) { + ret = ORTE_ERR_NOT_FOUND; + error = "Getting core object"; + goto error; + } + cpus = opal_hwloc_base_get_available_cpus(opal_hwloc_topology, obj); + if (0 > hwloc_set_cpubind(opal_hwloc_topology, cpus, 0)) { + error = "Setting processor affinity failed"; + ret = ORTE_ERROR; + goto error; + } + orte_process_info.bind_level = OPAL_HWLOC_CORE_LEVEL; + orte_process_info.bind_idx = nrank; + } else { + /* for all higher binding policies, we bind to the specified + * object that the nrank-th core belongs to + */ + if (NULL == (obj = opal_hwloc_base_get_obj_by_type(opal_hwloc_topology, HWLOC_OBJ_CORE, + 0, nrank, OPAL_HWLOC_LOGICAL))) { + ret = ORTE_ERR_NOT_FOUND; + error = "Getting core object"; + goto error; + } + if (OPAL_BIND_TO_L1CACHE == OPAL_GET_BINDING_POLICY(opal_hwloc_binding_policy)) { + target = HWLOC_OBJ_CACHE; + cache_level = 1; + orte_process_info.bind_level = OPAL_HWLOC_L1CACHE_LEVEL; + } else if (OPAL_BIND_TO_L2CACHE == OPAL_GET_BINDING_POLICY(opal_hwloc_binding_policy)) { + target = HWLOC_OBJ_CACHE; + cache_level = 2; + orte_process_info.bind_level = OPAL_HWLOC_L2CACHE_LEVEL; + } else if (OPAL_BIND_TO_L3CACHE == OPAL_GET_BINDING_POLICY(opal_hwloc_binding_policy)) { + target = HWLOC_OBJ_CACHE; + cache_level = 3; + orte_process_info.bind_level = OPAL_HWLOC_L3CACHE_LEVEL; + } else if (OPAL_BIND_TO_SOCKET == OPAL_GET_BINDING_POLICY(opal_hwloc_binding_policy)) { + target = HWLOC_OBJ_SOCKET; + orte_process_info.bind_level = OPAL_HWLOC_SOCKET_LEVEL; + } else if (OPAL_BIND_TO_NUMA == OPAL_GET_BINDING_POLICY(opal_hwloc_binding_policy)) { + target = HWLOC_OBJ_NODE; + orte_process_info.bind_level = OPAL_HWLOC_NUMA_LEVEL; + } else { + ret = ORTE_ERR_NOT_FOUND; + error = "Binding policy not known"; + goto error; + } + for (obj = obj->parent; NULL != obj; obj = obj->parent) { + if (target == obj->type) { + if (HWLOC_OBJ_CACHE == target && cache_level != obj->attr->cache.depth) { + continue; + } + /* this is the place! */ + cpus = opal_hwloc_base_get_available_cpus(opal_hwloc_topology, obj); + if (0 > hwloc_set_cpubind(opal_hwloc_topology, cpus, 0)) { + ret = ORTE_ERROR; + error = "Setting processor affinity failed"; + goto error; + } + orte_process_info.bind_idx = opal_hwloc_base_get_obj_idx(opal_hwloc_topology, + obj, OPAL_HWLOC_LOGICAL); + orte_proc_is_bound = true; + break; + } + } + if (!orte_proc_is_bound) { + ret = ORTE_ERROR; + error = "Setting processor affinity failed"; + goto error; + } + } + } + } + } + } + + MOVEON: + /* get or update our local cpuset - it will get used multiple + * times, so it's more efficient to keep a global copy + */ + opal_hwloc_base_get_local_cpuset(); + /* report bindings, if requested */ + if (opal_hwloc_report_bindings) { + char bindings[64]; + hwloc_obj_t root; + hwloc_cpuset_t cpus; + /* get the root object for this node */ + root = hwloc_get_root_obj(opal_hwloc_topology); + cpus = opal_hwloc_base_get_available_cpus(opal_hwloc_topology, root); + /* we are not bound if this equals our cpuset */ + if (0 == hwloc_bitmap_compare(cpus, opal_hwloc_my_cpuset)) { + opal_output(0, "%s is not bound", + ORTE_NAME_PRINT(ORTE_PROC_MY_NAME)); + } else { + hwloc_bitmap_list_snprintf(bindings, 64, opal_hwloc_my_cpuset); + opal_output(0, "%s is bound to cpus %s", + ORTE_NAME_PRINT(ORTE_PROC_MY_NAME), + bindings); + } + } + + return ORTE_SUCCESS; + + error: + if (ORTE_ERR_SILENT != ret) { + orte_show_help("help-orte-runtime", + "orte_init:startup:internal-failure", + true, error, ORTE_ERROR_NAME(ret), ret); + } + + return ORTE_ERR_SILENT; + +#else + return ORTE_SUCCESS; +#endif +} diff --git a/orte/mca/ess/cnos/ess_cnos_module.c b/orte/mca/ess/cnos/ess_cnos_module.c index dc2d815248..075ab09f73 100644 --- a/orte/mca/ess/cnos/ess_cnos_module.c +++ b/orte/mca/ess/cnos/ess_cnos_module.c @@ -9,7 +9,7 @@ * University of Stuttgart. All rights reserved. * Copyright (c) 2004-2005 The Regents of the University of California. * All rights reserved. - * Copyright (c) 2011 Cisco Systems, Inc. All rights reserved. + * Copyright (c) 2011-2012 Cisco Systems, Inc. All rights reserved. * $COPYRIGHT$ * * Additional copyrights may follow @@ -44,7 +44,7 @@ static int rte_init(void); static int rte_finalize(void); static void rte_abort(int status, bool report) __opal_attribute_noreturn__; -static opal_paffinity_locality_t proc_get_locality(orte_process_name_t *proc); +static opal_hwloc_locality_t proc_get_locality(orte_process_name_t *proc); static char* proc_get_hostname(orte_process_name_t *proc); static orte_local_rank_t proc_get_local_rank(orte_process_name_t *proc); static orte_node_rank_t proc_get_node_rank(orte_process_name_t *proc); @@ -130,7 +130,7 @@ static void rte_abort(int status, bool report) exit(status); } -static opal_paffinity_locality_t proc_get_locality(orte_process_name_t *proc) +static opal_hwloc_locality_t proc_get_locality(orte_process_name_t *proc) { if (map[ORTE_PROC_MY_NAME->vpid].nid == map[proc->vpid].nid) { diff --git a/orte/mca/ess/env/ess_env_module.c b/orte/mca/ess/env/ess_env_module.c index 26c7183713..e6f2087692 100644 --- a/orte/mca/ess/env/ess_env_module.c +++ b/orte/mca/ess/env/ess_env_module.c @@ -9,7 +9,7 @@ * University of Stuttgart. All rights reserved. * Copyright (c) 2004-2005 The Regents of the University of California. * All rights reserved. - * Copyright (c) 2011 Cisco Systems, Inc. All rights reserved. + * Copyright (c) 2011-2012 Cisco Systems, Inc. All rights reserved. * $COPYRIGHT$ * * Additional copyrights may follow @@ -164,6 +164,12 @@ static int rte_init(void) goto error; } + /* setup process binding */ + if (ORTE_SUCCESS != (ret = orte_ess_base_proc_binding())) { + error = "proc_binding"; + goto error; + } + return ORTE_SUCCESS; error: diff --git a/orte/mca/ess/ess.h b/orte/mca/ess/ess.h index 915d97ee39..136472dcf0 100644 --- a/orte/mca/ess/ess.h +++ b/orte/mca/ess/ess.h @@ -11,6 +11,7 @@ * All rights reserved. * Copyright (c) 2011 Los Alamos National Security, LLC. All rights * reserved. + * Copyright (c) 2012 Cisco Systems, Inc. All rights reserved. * $COPYRIGHT$ * * Additional copyrights may follow @@ -30,7 +31,7 @@ #include "orte/types.h" #include "opal/mca/mca.h" -#include "opal/mca/paffinity/paffinity.h" +#include "opal/mca/hwloc/base/base.h" #include "orte/util/proc_info.h" @@ -70,7 +71,7 @@ typedef void (*orte_ess_base_module_abort_fn_t)(int status, bool report); * board, node, computing unit, or cluster. This function provides * a means for an MPI proc to query the locality of another proc. */ -typedef opal_paffinity_locality_t (*orte_ess_base_module_get_proc_locality_fn_t)(orte_process_name_t *proc); +typedef opal_hwloc_locality_t (*orte_ess_base_module_get_proc_locality_fn_t)(orte_process_name_t *proc); /** * Get the vpid of the daemon who hosts the specified proc diff --git a/orte/mca/ess/hnp/ess_hnp_module.c b/orte/mca/ess/hnp/ess_hnp_module.c index 5070a0981a..b7e476c516 100644 --- a/orte/mca/ess/hnp/ess_hnp_module.c +++ b/orte/mca/ess/hnp/ess_hnp_module.c @@ -10,8 +10,8 @@ * Copyright (c) 2004-2005 The Regents of the University of California. * All rights reserved. * Copyright (c) 2010-2011 Oak Ridge National Labs. All rights reserved. - * Copyright (c) 2011 Cisco Systems, Inc. All rights reserved. - * Copyright (c) 2011 Los Alamos National Security, LLC. All rights + * Copyright (c) 2011-2012 Cisco Systems, Inc. All rights reserved. + * Copyright (c) 2011-2012 Los Alamos National Security, LLC. All rights * reserved. * $COPYRIGHT$ * diff --git a/orte/mca/ess/pmi/ess_pmi_module.c b/orte/mca/ess/pmi/ess_pmi_module.c index 8e5dea8f00..943d84541e 100644 --- a/orte/mca/ess/pmi/ess_pmi_module.c +++ b/orte/mca/ess/pmi/ess_pmi_module.c @@ -9,7 +9,7 @@ * University of Stuttgart. All rights reserved. * Copyright (c) 2004-2005 The Regents of the University of California. * All rights reserved. - * Copyright (c) 2008-2011 Cisco Systems, Inc. All rights reserved. + * Copyright (c) 2008-2012 Cisco Systems, Inc. All rights reserved. * Copyright (c) 2012 Los Alamos National Security, LLC. * All rights reserved. * $COPYRIGHT$ @@ -301,6 +301,12 @@ static int rte_init(void) error = "orte_ess_base_app_setup"; goto error; } + + /* setup process binding */ + if (ORTE_SUCCESS != (ret = orte_ess_base_proc_binding())) { + error = "proc_binding"; + goto error; + } } /* set max procs */ diff --git a/orte/mca/ess/portals4_shmem/ess_portals4_shmem_module.c b/orte/mca/ess/portals4_shmem/ess_portals4_shmem_module.c index 6dd52360ce..5933751879 100644 --- a/orte/mca/ess/portals4_shmem/ess_portals4_shmem_module.c +++ b/orte/mca/ess/portals4_shmem/ess_portals4_shmem_module.c @@ -10,7 +10,7 @@ * Copyright (c) 2004-2005 The Regents of the University of California. * All rights reserved. * Copyright (c) 2010 Sandia National Laboratories. All rights reserved. - * Copyright (c) 2011 Cisco Systems, Inc. All rights reserved. + * Copyright (c) 2011-2012 Cisco Systems, Inc. All rights reserved. * $COPYRIGHT$ * * Additional copyrights may follow @@ -42,7 +42,7 @@ static int rte_init(void); static int rte_finalize(void); static void rte_abort(int status, bool report) __opal_attribute_noreturn__; -static opal_paffinity_locality_t proc_get_locality(orte_process_name_t *proc); +static opal_hwloc_locality_t proc_get_locality(orte_process_name_t *proc); static char* proc_get_hostname(orte_process_name_t *proc); static orte_local_rank_t proc_get_local_rank(orte_process_name_t *proc); static orte_node_rank_t proc_get_node_rank(orte_process_name_t *proc); @@ -128,7 +128,7 @@ static void rte_abort(int status, bool report) exit(status); } -static opal_paffinity_locality_t proc_get_locality(orte_process_name_t *proc) +static opal_hwloc_locality_t proc_get_locality(orte_process_name_t *proc) { if (map[ORTE_PROC_MY_NAME->vpid].nid == map[proc->vpid].nid) { diff --git a/orte/mca/odls/default/odls_default_module.c b/orte/mca/odls/default/odls_default_module.c index 0a3ca7a202..7fd9389029 100644 --- a/orte/mca/odls/default/odls_default_module.c +++ b/orte/mca/odls/default/odls_default_module.c @@ -11,7 +11,7 @@ * All rights reserved. * Copyright (c) 2007-2010 Oracle and/or its affiliates. All rights reserved. * Copyright (c) 2007 Evergrid, Inc. All rights reserved. - * Copyright (c) 2008-2011 Cisco Systems, Inc. All rights reserved. + * Copyright (c) 2008-2012 Cisco Systems, Inc. All rights reserved. * Copyright (c) 2010 IBM Corporation. All rights reserved. * Copyright (c) 2011-2012 Los Alamos National Security, LLC. All rights * reserved. @@ -109,7 +109,6 @@ #include "opal/mca/hwloc/hwloc.h" #include "opal/mca/hwloc/base/base.h" -#include "opal/mca/maffinity/base/base.h" #include "opal/class/opal_pointer_array.h" #include "opal/util/opal_environ.h" #include "opal/util/show_help.h" @@ -525,16 +524,15 @@ static int do_child(orte_app_context_t* context, goto PROCEED; } } - opal_maffinity_setup = true; /* Set an info MCA param that tells the launched processes that it was bound by us (e.g., so that MPI_INIT doesn't try to bind itself) */ - param = mca_base_param_environ_variable("opal","bound","at_launch"); + param = mca_base_param_environ_variable("orte","bound","at_launch"); opal_setenv(param, "1", true, &environ_copy); free(param); /* ...and provide a nice string representation of what we bound to */ - param = mca_base_param_environ_variable("opal","base","applied_binding"); + param = mca_base_param_environ_variable("orte","base","applied_binding"); opal_setenv(param, child->cpu_bitmap, true, &environ_copy); } } diff --git a/orte/mca/rmaps/base/rmaps_base_map_job.c b/orte/mca/rmaps/base/rmaps_base_map_job.c index 37eb8de0aa..1549ac0c90 100644 --- a/orte/mca/rmaps/base/rmaps_base_map_job.c +++ b/orte/mca/rmaps/base/rmaps_base_map_job.c @@ -9,7 +9,7 @@ * University of Stuttgart. All rights reserved. * Copyright (c) 2004-2005 The Regents of the University of California. * All rights reserved. - * Copyright (c) 2011 Cisco Systems, Inc. All rights reserved. + * Copyright (c) 2011-2012 Cisco Systems, Inc. All rights reserved. * Copyright (c) 2011-2012 Los Alamos National Security, LLC. * All rights reserved. * $COPYRIGHT$ @@ -261,7 +261,7 @@ void orte_rmaps_base_map_job(int fd, short args, void *cbdata) } #if OPAL_HAVE_HWLOC { - opal_paffinity_locality_t locality; + opal_hwloc_locality_t locality; orte_proc_t *p0; /* test locality - for the first node, print the locality of each proc relative to the first one */ diff --git a/orte/runtime/orte_globals.h b/orte/runtime/orte_globals.h index e5c7322f13..493659aa6b 100644 --- a/orte/runtime/orte_globals.h +++ b/orte/runtime/orte_globals.h @@ -10,7 +10,7 @@ * Copyright (c) 2004-2005 The Regents of the University of California. * All rights reserved. * Copyright (c) 2007-2010 Oracle and/or its affiliates. All rights reserved. - * Copyright (c) 2007-2011 Cisco Systems, Inc. All rights reserved. + * Copyright (c) 2007-2012 Cisco Systems, Inc. All rights reserved. * Copyright (c) 2011-2012 Los Alamos National Security, LLC. * All rights reserved. * $COPYRIGHT$ @@ -42,7 +42,7 @@ #include "opal/threads/threads.h" #include "opal/mca/event/event.h" #include "opal/mca/hwloc/hwloc.h" -#include "opal/mca/paffinity/paffinity.h" +#include "opal/mca/hwloc/base/base.h" #include "orte/mca/plm/plm_types.h" #include "orte/mca/rml/rml_types.h" @@ -65,7 +65,16 @@ ORTE_DECLSPEC extern bool orte_create_session_dirs; /* instantiated in orte/run ORTE_DECLSPEC extern bool orte_execute_quiet; /* instantiated in orte/runtime/orte_globals.c */ ORTE_DECLSPEC extern bool orte_report_silent_errors; /* instantiated in orte/runtime/orte_globals.c */ ORTE_DECLSPEC extern opal_event_base_t *orte_event_base; /* instantiated in orte/runtime/orte_init.c */ -ORTE_DECLSPEC extern bool orte_event_base_active; +ORTE_DECLSPEC extern bool orte_event_base_active; /* instantiated in orte/runtime/orte_init.c */ +ORTE_DECLSPEC extern bool orte_proc_is_bound; /* instantiated in orte/runtime/orte_init.c */ +#if OPAL_HAVE_HWLOC +/** + * Global indicating where this process was bound to at launch (will + * be NULL if !orte_proc_is_bound) + */ +OPAL_DECLSPEC extern hwloc_cpuset_t orte_proc_applied_binding; /* instantiated in orte/runtime/orte_init.c */ +#endif + /* Shortcut for some commonly used names */ #define ORTE_NAME_WILDCARD (&orte_name_wildcard) @@ -546,7 +555,7 @@ typedef struct { /* node rank */ orte_node_rank_t node_rank; /* locality */ - opal_paffinity_locality_t locality; + opal_hwloc_locality_t locality; } orte_pmap_t; ORTE_DECLSPEC OBJ_CLASS_DECLARATION(orte_pmap_t); diff --git a/orte/runtime/orte_init.c b/orte/runtime/orte_init.c index 12efbe5c8b..a3c6863666 100644 --- a/orte/runtime/orte_init.c +++ b/orte/runtime/orte_init.c @@ -11,7 +11,7 @@ * All rights reserved. * Copyright (c) 2006-2012 Los Alamos National Security, LLC. All rights * reserved. - * Copyright (c) 2007-2011 Cisco Systems, Inc. All rights reserved. + * Copyright (c) 2007-2012 Cisco Systems, Inc. All rights reserved. * Copyright (c) 2007-2008 Sun Microsystems, Inc. All rights reserved. * * $COPYRIGHT$ @@ -58,6 +58,10 @@ char *orte_prohibited_session_dirs = NULL; bool orte_create_session_dirs = true; opal_event_base_t *orte_event_base; bool orte_event_base_active = true; +bool orte_proc_is_bound = false; +#if OPAL_HAVE_HWLOC +hwloc_cpuset_t orte_proc_applied_binding = NULL; +#endif orte_process_name_t orte_name_wildcard = {ORTE_JOBID_WILDCARD, ORTE_VPID_WILDCARD}; diff --git a/orte/tools/mapreduce/mapreduce.c b/orte/tools/mapreduce/mapreduce.c index 0ce47b7778..be733debfb 100644 --- a/orte/tools/mapreduce/mapreduce.c +++ b/orte/tools/mapreduce/mapreduce.c @@ -59,7 +59,6 @@ #include "opal/mca/event/event.h" #include "opal/mca/installdirs/installdirs.h" -#include "opal/mca/paffinity/base/base.h" #include "opal/mca/base/base.h" #include "opal/util/argv.h" #include "opal/util/output.h" @@ -627,13 +626,6 @@ int main(int argc, char *argv[]) /* Setup MCA params */ orte_register_params(); - /*** NOTIFY IF DEPRECATED OPAL_PAFFINITY_ALONE WAS SET ***/ - if (opal_paffinity_alone) { - orte_show_help("help-opal-runtime.txt", - "opal_paffinity_alone:deprecated", - true); - } - /* Check for some "global" command line params */ parse_globals(argc, argv, &cmd_line); OBJ_DESTRUCT(&cmd_line); diff --git a/orte/tools/orte-info/components.c b/orte/tools/orte-info/components.c index 1bc4f79fb8..1b33c61ef5 100644 --- a/orte/tools/orte-info/components.c +++ b/orte/tools/orte-info/components.c @@ -9,7 +9,7 @@ * University of Stuttgart. All rights reserved. * Copyright (c) 2004-2005 The Regents of the University of California. * All rights reserved. - * Copyright (c) 2006-2010 Cisco Systems, Inc. All rights reserved. + * Copyright (c) 2006-2012 Cisco Systems, Inc. All rights reserved. * Copyright (c) 2010-2011 Los Alamos National Security, LLC. * All rights reserved. * $COPYRIGHT$ @@ -34,14 +34,8 @@ #include "opal/mca/backtrace/backtrace.h" #include "opal/mca/backtrace/base/base.h" #include "opal/mca/hwloc/base/base.h" -#include "opal/mca/paffinity/paffinity.h" -#include "opal/mca/paffinity/base/base.h" -#include "opal/mca/carto/carto.h" -#include "opal/mca/carto/base/base.h" #include "opal/mca/shmem/shmem.h" #include "opal/mca/shmem/base/base.h" -#include "opal/mca/maffinity/maffinity.h" -#include "opal/mca/maffinity/base/base.h" #include "opal/mca/memory/memory.h" #include "opal/mca/memory/base/base.h" #include "opal/mca/memchecker/memchecker.h" @@ -255,22 +249,6 @@ void orte_info_open_components(void) map->components = &opal_memchecker_base_components_opened; opal_pointer_array_add(&component_map, map); - if (OPAL_SUCCESS != opal_paffinity_base_open()) { - goto error; - } - map = OBJ_NEW(orte_info_component_map_t); - map->type = strdup("paffinity"); - map->components = &opal_paffinity_base_components_opened; - opal_pointer_array_add(&component_map, map); - - if (OPAL_SUCCESS != opal_carto_base_open()) { - goto error; - } - map = OBJ_NEW(orte_info_component_map_t); - map->type = strdup("carto"); - map->components = &opal_carto_base_components_opened; - opal_pointer_array_add(&component_map, map); - if (OPAL_SUCCESS != opal_shmem_base_open()) { goto error; } @@ -279,14 +257,6 @@ void orte_info_open_components(void) map->components = &opal_shmem_base_components_opened; opal_pointer_array_add(&component_map, map); - if (OPAL_SUCCESS != opal_maffinity_base_open()) { - goto error; - } - map = OBJ_NEW(orte_info_component_map_t); - map->type = strdup("maffinity"); - map->components = &opal_maffinity_base_components_opened; - opal_pointer_array_add(&component_map, map); - #if OPAL_HAVE_HWLOC if (OPAL_SUCCESS != opal_hwloc_base_open()) { goto error; @@ -540,9 +510,6 @@ void orte_info_close_components() (void) opal_backtrace_base_close(); (void) opal_memory_base_close(); (void) opal_memchecker_base_close(); - (void) opal_paffinity_base_close(); - (void) opal_carto_base_close(); - (void) opal_maffinity_base_close(); (void) opal_timer_base_close(); #if OPAL_HAVE_HWLOC (void) opal_hwloc_base_close(); diff --git a/orte/tools/orterun/orterun.c b/orte/tools/orterun/orterun.c index 798c6292d4..c258a6b1b8 100644 --- a/orte/tools/orterun/orterun.c +++ b/orte/tools/orterun/orterun.c @@ -59,7 +59,7 @@ #include "opal/mca/event/event.h" #include "opal/mca/installdirs/installdirs.h" -#include "opal/mca/paffinity/base/base.h" +#include "opal/mca/hwloc/base/base.h" #include "opal/mca/base/base.h" #include "opal/util/argv.h" #include "opal/util/output.h" @@ -674,13 +674,6 @@ int orterun(int argc, char *argv[]) /* Setup MCA params */ orte_register_params(); - /*** NOTIFY IF DEPRECATED OPAL_PAFFINITY_ALONE WAS SET ***/ - if (opal_paffinity_alone) { - orte_show_help("help-opal-runtime.txt", - "opal_paffinity_alone:deprecated", - true); - } - /* Check for some "global" command line params */ parse_globals(argc, argv, &cmd_line); OBJ_DESTRUCT(&cmd_line);