ports/openmpi
ports/openmpi
1
1
Fork 0
Code Releases Activity

Per RFC, bring in the following changes:

Browse Source 
* Remove paffinity, maffinity, and carto frameworks -- they've been
   wholly replaced by hwloc.
 * Move ompi_mpi_init() affinity-setting/checking code down to ORTE.
 * Update sm, smcuda, wv, and openib components to no longer use carto.
   Instead, use hwloc data.  There are still optimizations possible in
   the sm/smcuda BTLs (i.e., making multiple mpools).  Also, the old
   carto-based code found out how many NUMA nodes were ''available''
   -- not how many were used ''in this job''.  The new hwloc-using
   code computes the same value -- it was not updated to calculate how
   many NUMA nodes are used ''by this job.''
   * Note that I cannot compile the smcuda and wv BTLs -- I ''think''
     they're right, but they need to be verified by their owners.
 * The openib component now does a bunch of stuff to figure out where
   "near" OpenFabrics devices are.  '''THIS IS A CHANGE IN DEFAULT
   BEHAVIOR!!''' and still needs to be verified by OpenFabrics vendors
   (I do not have a NUMA machine with an OpenFabrics device that is a
   non-uniform distance from multiple different NUMA nodes).
 * Completely rewrite the OMPI_Affinity_str() routine from the
   "affinity" mpiext extension.  This extension now understands
   hyperthreads; the output format of it has changed a bit to reflect
   this new information.
 * Bunches of minor changes around the code base to update names/types
   from maffinity/paffinity-based names to hwloc-based names.
 * Add some helper functions into the hwloc base, mainly having to do
   with the fact that we have the hwloc data reporting ''all''
   topology information, but sometimes you really only want the
   (online | available) data.

This commit was SVN r26391.
This commit is contained in:
Jeff Squyres 2012-05-07 14:52:54 +00:00
parent 1b475523de
commit 2ba10c37fe
104 changed files with 1629 additions and 7706 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
ompi/communicator/comm.c
View File

@ -10,7 +10,7 @@
* Copyright (c) 2004-2005 The Regents of the University of California.
* All rights reserved.
* Copyright (c) 2007-2011 University of Houston. All rights reserved.
* Copyright (c) 2007-2008 Cisco Systems, Inc. All rights reserved.
* Copyright (c) 2007-2012 Cisco Systems, Inc. All rights reserved.
* Copyright (c) 2009 Sun Microsystems, Inc. All rights reserved.
* Copyright (c) 2012 Oak Ridge National Labs. All rights reserved.
* $COPYRIGHT$
@ -25,6 +25,7 @@
#include <stdio.h>
#include "ompi/constants.h"
#include "opal/mca/hwloc/base/base.h"
#include "opal/dss/dss.h"
#include "orte/util/name_fns.h"

211
ompi/mca/btl/openib/btl_openib_component.c
View File

@ -10,7 +10,7 @@
* University of Stuttgart. All rights reserved.
* Copyright (c) 2004-2005 The Regents of the University of California.
* All rights reserved.
* Copyright (c) 2006-2011 Cisco Systems, Inc. All rights reserved.
* Copyright (c) 2006-2012 Cisco Systems, Inc. All rights reserved.
* Copyright (c) 2006-2009 Mellanox Technologies. All rights reserved.
* Copyright (c) 2006-2012 Los Alamos National Security, LLC. All rights
* reserved.
@ -57,9 +57,8 @@ const char *ibv_get_sysfs_path(void);
#include "opal/util/argv.h"
#include "opal/memoryhooks/memory.h"
#include "opal/mca/base/mca_base_param.h"
#include "opal/mca/carto/carto.h"
#include "opal/mca/carto/base/base.h"
#include "opal/mca/paffinity/base/base.h"
#include "opal/mca/hwloc/hwloc.h"
#include "opal/mca/hwloc/base/base.h"
#include "opal/mca/installdirs/installdirs.h"
#include "opal_stdint.h"
@ -2337,58 +2336,6 @@ static void ibv_free_device_list_compat(struct ibv_device **ib_devs)
#endif
}
static opal_carto_graph_t *host_topo;
static int get_ib_dev_distance(struct ibv_device *dev)
{
opal_paffinity_base_cpu_set_t cpus;
opal_carto_base_node_t *device_node;
int min_distance = -1, i, num_processors;
const char *device = ibv_get_device_name(dev);
if(opal_paffinity_base_get_processor_info(&num_processors) != OMPI_SUCCESS) {
num_processors = 100; /* Choose something big enough */
}
device_node = opal_carto_base_find_node(host_topo, device);
/* no topology info for device found. Assume that it is close */
if(NULL == device_node)
return 0;
OPAL_PAFFINITY_CPU_ZERO(cpus);
opal_paffinity_base_get(&cpus);
for (i = 0; i < num_processors; i++) {
opal_carto_base_node_t *slot_node;
int distance, socket, core;
char *slot;
if(!OPAL_PAFFINITY_CPU_ISSET(i, cpus))
continue;
opal_paffinity_base_get_map_to_socket_core(i, &socket, &core);
asprintf(&slot, "slot%d", socket);
slot_node = opal_carto_base_find_node(host_topo, slot);
free(slot);
if(NULL == slot_node)
return 0;
distance = opal_carto_base_spf(host_topo, slot_node, device_node);
if(distance < 0)
return 0;
if(min_distance < 0 || min_distance > distance)
min_distance = distance;
}
return min_distance;
}
struct dev_distance {
struct ibv_device *ib_dev;
int distance;
@ -2402,23 +2349,165 @@ static int compare_distance(const void *p1, const void *p2)
return d1->distance - d2->distance;
}
static int get_ib_dev_distance(struct ibv_device *dev)
{
/* If we don't have hwloc, we'll default to a distance of 0,
because we have no way of measuring. */
int distance = 0;
#if OPAL_HAVE_HWLOC
int a, b, i;
hwloc_cpuset_t my_cpuset = NULL, ibv_cpuset = NULL;
hwloc_obj_t my_obj, ibv_obj, node_obj;
/* Note that this struct is owned by hwloc; there's no need to
free it at the end of time */
static const struct hwloc_distances_s *hwloc_distances = NULL;
if (NULL == hwloc_distances) {
hwloc_distances =
hwloc_get_whole_distance_matrix_by_type(opal_hwloc_topology,
HWLOC_OBJ_NODE);
}
/* If we got no info, just return 0 */
if (NULL == hwloc_distances || NULL == hwloc_distances->latency) {
goto out;
}
/* Next, find the NUMA node where this IBV device is located */
ibv_cpuset = hwloc_bitmap_alloc();
if (NULL == ibv_cpuset) {
goto out;
}
if (0 != hwloc_ibv_get_device_cpuset(opal_hwloc_topology, dev, ibv_cpuset)) {
goto out;
}
ibv_obj = hwloc_get_obj_covering_cpuset(opal_hwloc_topology, ibv_cpuset);
if (NULL == ibv_obj) {
goto out;
}
/* If ibv_obj is a NUMA node or below, we're good. */
switch (ibv_obj->type) {
case HWLOC_OBJ_NODE:
case HWLOC_OBJ_SOCKET:
case HWLOC_OBJ_CACHE:
case HWLOC_OBJ_CORE:
case HWLOC_OBJ_PU:
while (NULL != ibv_obj && ibv_obj->type != HWLOC_OBJ_NODE) {
ibv_obj = ibv_obj->parent;
}
break;
default:
/* If it's above a NUMA node, then I don't know how to compute
the distance... */
ibv_obj = NULL;
break;
}
/* If we don't have an object for this ibv device, give up */
if (NULL == ibv_obj) {
goto out;
}
/* This function is only called if the process is bound, so let's
find out where we are bound to. For the moment, we only care
about the NUMA node to which we are bound. */
my_cpuset = hwloc_bitmap_alloc();
if (NULL == my_cpuset) {
goto out;
}
if (0 != hwloc_get_cpubind(opal_hwloc_topology, my_cpuset, 0)) {
hwloc_bitmap_free(my_cpuset);
goto out;
}
my_obj = hwloc_get_obj_covering_cpuset(opal_hwloc_topology, my_cpuset);
if (NULL == my_obj) {
goto out;
}
/* If my_obj is a NUMA node or below, we're good. */
switch (my_obj->type) {
case HWLOC_OBJ_NODE:
case HWLOC_OBJ_SOCKET:
case HWLOC_OBJ_CACHE:
case HWLOC_OBJ_CORE:
case HWLOC_OBJ_PU:
while (NULL != my_obj && my_obj->type != HWLOC_OBJ_NODE) {
my_obj = my_obj->parent;
}
if (NULL != my_obj) {
/* Distance may be asymetrical, so calculate both of them
and take the max */
a = hwloc_distances->latency[my_obj->logical_index *
(ibv_obj->logical_index *
hwloc_distances->nbobjs)];
b = hwloc_distances->latency[ibv_obj->logical_index *
(my_obj->logical_index *
hwloc_distances->nbobjs)];
distance = (a > b) ? a : b;
}
break;
default:
/* If the obj is above a NUMA node, then we're bound to more than
one NUMA node. Find the max distance. */
i = 0;
for (node_obj = hwloc_get_obj_inside_cpuset_by_type(opal_hwloc_topology,
ibv_obj->cpuset,
HWLOC_OBJ_NODE, i);
NULL != node_obj;
node_obj = hwloc_get_obj_inside_cpuset_by_type(opal_hwloc_topology,
ibv_obj->cpuset,
HWLOC_OBJ_NODE, ++i)) {
a = hwloc_distances->latency[node_obj->logical_index *
(ibv_obj->logical_index *
hwloc_distances->nbobjs)];
b = hwloc_distances->latency[ibv_obj->logical_index *
(node_obj->logical_index *
hwloc_distances->nbobjs)];
a = (a > b) ? a : b;
distance = (a > distance) ? a : distance;
}
break;
}
out:
if (NULL != ibv_cpuset) {
hwloc_bitmap_free(ibv_cpuset);
}
if (NULL != my_cpuset) {
hwloc_bitmap_free(my_cpuset);
}
#endif
return distance;
}
static struct dev_distance *
sort_devs_by_distance(struct ibv_device **ib_devs, int count)
{
int i;
struct dev_distance *devs = (struct dev_distance *) malloc(count * sizeof(struct dev_distance));
opal_carto_base_get_host_graph(&host_topo, "Infiniband");
for (i = 0; i < count; i++) {
devs[i].ib_dev = ib_devs[i];
devs[i].distance = get_ib_dev_distance(ib_devs[i]);
if (OPAL_HAVE_HWLOC && orte_proc_is_bound) {
/* If this process is bound to one or more PUs, we can get
an accurate distance. */
devs[i].distance = get_ib_dev_distance(ib_devs[i]);
} else {
/* Since we're not bound, just assume that the device is
close. */
devs[i].distance = 0;
}
}
qsort(devs, count, sizeof(struct dev_distance), compare_distance);
opal_carto_base_free_graph(host_topo);
return devs;
}

221
ompi/mca/btl/sm/btl_sm.c
View File

@ -10,8 +10,8 @@
* Copyright (c) 2004-2005 The Regents of the University of California.
* All rights reserved.
* Copyright (c) 2006-2007 Voltaire. All rights reserved.
* Copyright (c) 2009 Cisco Systems, Inc. All rights reserved.
* Copyright (c) 2010 Los Alamos National Security, LLC.
* Copyright (c) 2009-2012 Cisco Systems, Inc. All rights reserved.
* Copyright (c) 2010-2012 Los Alamos National Security, LLC.
* All rights reserved.
* Copyright (c) 2010-2012 IBM Corporation. All rights reserved.
* $COPYRIGHT$
@ -41,10 +41,7 @@
#include "opal/class/opal_bitmap.h"
#include "opal/util/output.h"
#include "opal/util/printf.h"
#include "opal/mca/carto/carto.h"
#include "opal/mca/carto/base/base.h"
#include "opal/mca/paffinity/base/base.h"
#include "opal/mca/maffinity/base/base.h"
#include "opal/mca/hwloc/base/base.h"
#include "orte/util/proc_info.h"
#include "opal/datatype/opal_convertor.h"
#include "ompi/class/ompi_free_list.h"
@ -128,133 +125,119 @@ static void *mpool_calloc(size_t nmemb, size_t size)
return buf;
}
static void init_maffinity(int *my_mem_node, int *max_mem_node)
{
opal_carto_graph_t *topo;
opal_value_array_t dists;
int i, num_core, socket;
opal_paffinity_base_cpu_set_t cpus;
char *myslot = NULL;
opal_carto_node_distance_t *dist;
opal_carto_base_node_t *slot_node;
*my_mem_node = 0;
*max_mem_node = 1;
if (OMPI_SUCCESS != opal_carto_base_get_host_graph(&topo, "Memory")) {
return;
}
OBJ_CONSTRUCT(&dists, opal_value_array_t);
opal_value_array_init(&dists, sizeof(opal_carto_node_distance_t));
if (OMPI_SUCCESS != opal_paffinity_base_get_processor_info(&num_core)) {
num_core = 100; /* set something large */
}
OPAL_PAFFINITY_CPU_ZERO(cpus);
opal_paffinity_base_get(&cpus);
/* find core we are running on */
for (i = 0; i < num_core; i++) {
if (OPAL_PAFFINITY_CPU_ISSET(i, cpus)) {
break;
}
}
if (OMPI_SUCCESS != opal_paffinity_base_get_map_to_socket_core(i, &socket, &i)) {
/* no topology info available */
goto out;
}
asprintf(&myslot, "slot%d", socket);
slot_node = opal_carto_base_find_node(topo, myslot);
if(NULL == slot_node) {
goto out;
}
opal_carto_base_get_nodes_distance(topo, slot_node, "Memory", &dists);
if((*max_mem_node = opal_value_array_get_size(&dists)) < 2) {
goto out;
}
dist = (opal_carto_node_distance_t *) opal_value_array_get_item(&dists, 0);
opal_maffinity_base_node_name_to_id(dist->node->node_name, my_mem_node);
out:
if (myslot) {
free(myslot);
}
OBJ_DESTRUCT(&dists);
opal_carto_base_free_graph(topo);
}
static int sm_btl_first_time_init(mca_btl_sm_t *sm_btl, int n)
{
size_t size, length, length_payload;
char *sm_ctl_file;
sm_fifo_t *my_fifos;
int my_mem_node=-1, num_mem_nodes=-1, i;
int my_mem_node, num_mem_nodes, i;
ompi_proc_t **procs;
size_t num_procs;
mca_mpool_base_resources_t res;
mca_btl_sm_component_t* m = &mca_btl_sm_component;
init_maffinity(&my_mem_node, &num_mem_nodes);
mca_btl_sm_component.mem_node = my_mem_node;
mca_btl_sm_component.num_mem_nodes = num_mem_nodes;
/* Assume we don't have hwloc support and fill in dummy info */
mca_btl_sm_component.mem_node = my_mem_node = 0;
mca_btl_sm_component.num_mem_nodes = num_mem_nodes = 1;
#if OPAL_HAVE_HWLOC
/* If we have hwloc support, then get accurate information */
if (NULL != opal_hwloc_topology) {
i = opal_hwloc_base_get_nbobjs_by_type(opal_hwloc_topology,
HWLOC_OBJ_NODE, 0,
OPAL_HWLOC_AVAILABLE);
/* If we find >0 NUMA nodes, then investigate further */
if (i > 0) {
opal_hwloc_level_t bind_level;
unsigned int bind_index;
/* JMS This tells me how many numa nodes are *available*,
but it's not how many are being used *by this job*.
Note that this is the value we've previously used (from
the previous carto-based implementation), but it really
should be improved to be how many NUMA nodes are being
used *in this job*. */
mca_btl_sm_component.num_mem_nodes = num_mem_nodes = i;
/* Fill opal_hwloc_my_cpuset and find out to what level
this process is bound (if at all) */
opal_hwloc_base_get_local_cpuset();
opal_hwloc_base_get_level_and_index(opal_hwloc_my_cpuset,
&bind_level, &bind_index);
if (OPAL_HWLOC_NODE_LEVEL != bind_level) {
/* We are bound to *something* (i.e., our binding
level is less than "node", meaning the entire
machine), so discover which NUMA node this process
is bound */
if (OPAL_HWLOC_NUMA_LEVEL == bind_level) {
mca_btl_sm_component.mem_node = my_mem_node = (int) bind_index;
} else {
if (OPAL_SUCCESS ==
opal_hwloc_base_get_local_index(HWLOC_OBJ_NODE, 0, &bind_index)) {
mca_btl_sm_component.mem_node = my_mem_node = (int) bind_index;
} else {
/* Weird. We can't figure out what NUMA node
we're on. :-( */
mca_btl_sm_component.mem_node = my_mem_node = -1;
}
}
}
}
}
#endif
/* lookup shared memory pool */
mca_btl_sm_component.sm_mpools = (mca_mpool_base_module_t **) calloc(num_mem_nodes,
sizeof(mca_mpool_base_module_t*));
/* create mpool for each memory node */
for(i = 0; i < num_mem_nodes; i++) {
mca_mpool_base_resources_t res;
mca_btl_sm_component_t* m = &mca_btl_sm_component;
/* Create one mpool. Per discussion with George and a UTK Euro
MPI 2010 paper, it may be beneficial to create multiple mpools.
Leaving that for a future optimization, however. */
/* Disable memory binding, because each MPI process will claim
pages in the mpool for their local NUMA node */
res.mem_node = -1;
/* disable memory binding if there is only one memory node */
res.mem_node = (num_mem_nodes == 1) ? -1 : i;
/* determine how much memory to create */
/*
* This heuristic formula mostly says that we request memory for:
* - nfifos FIFOs, each comprising:
* . a sm_fifo_t structure
* . many pointers (fifo_size of them per FIFO)
* - eager fragments (2*n of them, allocated in sm_free_list_inc chunks)
* - max fragments (sm_free_list_num of them)
*
* On top of all that, we sprinkle in some number of
* "opal_cache_line_size" additions to account for some
* padding and edge effects that may lie in the allocator.
*/
res.size =
FIFO_MAP_NUM(n) * ( sizeof(sm_fifo_t) + sizeof(void *) * m->fifo_size + 4 * opal_cache_line_size )
+ ( 2 * n + m->sm_free_list_inc ) * ( m->eager_limit + 2 * opal_cache_line_size )
+ m->sm_free_list_num * ( m->max_frag_size + 2 * opal_cache_line_size );
/* determine how much memory to create */
/*
* This heuristic formula mostly says that we request memory for:
* - nfifos FIFOs, each comprising:
* . a sm_fifo_t structure
* . many pointers (fifo_size of them per FIFO)
* - eager fragments (2*n of them, allocated in sm_free_list_inc chunks)
* - max fragments (sm_free_list_num of them)
*
* On top of all that, we sprinkle in some number of "opal_cache_line_size"
* additions to account for some padding and edge effects that may lie
* in the allocator.
*/
res.size =
FIFO_MAP_NUM(n) * ( sizeof(sm_fifo_t) + sizeof(void *) * m->fifo_size + 4 * opal_cache_line_size )
+ ( 2 * n + m->sm_free_list_inc ) * ( m->eager_limit + 2 * opal_cache_line_size )
+ m->sm_free_list_num * ( m->max_frag_size + 2 * opal_cache_line_size );
/* before we multiply by n, make sure the result won't overflow */
/* Stick that little pad in, particularly since we'll eventually
* need a little extra space. E.g., in mca_mpool_sm_init() in
* mpool_sm_component.c when sizeof(mca_common_sm_module_t) is
* added.
*/
if ( ((double) res.size) * n > LONG_MAX - 4096 )
/* before we multiply by n, make sure the result won't overflow */
/* Stick that little pad in, particularly since we'll eventually
* need a little extra space. E.g., in mca_mpool_sm_init() in
* mpool_sm_component.c when sizeof(mca_common_sm_module_t) is
* added.
*/
if ( ((double) res.size) * n > LONG_MAX - 4096 ) {
return OMPI_ERR_OUT_OF_RESOURCE;
}
res.size *= n;
/* now, create it */
mca_btl_sm_component.sm_mpools[0] =
mca_mpool_base_module_create(mca_btl_sm_component.sm_mpool_name,
sm_btl, &res);
/* Sanity check to ensure that we found it */
if (NULL == mca_btl_sm_component.sm_mpools[0]) {
return OMPI_ERR_OUT_OF_RESOURCE;
res.size *= n;
/* now, create it */
mca_btl_sm_component.sm_mpools[i] =
mca_mpool_base_module_create(mca_btl_sm_component.sm_mpool_name,
sm_btl, &res);
/* Sanity check to ensure that we found it */
if(NULL == mca_btl_sm_component.sm_mpools[i])
return OMPI_ERR_OUT_OF_RESOURCE;
if(i == my_mem_node)
mca_btl_sm_component.sm_mpool = mca_btl_sm_component.sm_mpools[i];
}
mca_btl_sm_component.sm_mpool = mca_btl_sm_component.sm_mpools[0];
mca_btl_sm_component.sm_mpool_base =
mca_btl_sm_component.sm_mpools[0]->mpool_base(mca_btl_sm_component.sm_mpools[0]);
@ -262,17 +245,19 @@ static int sm_btl_first_time_init(mca_btl_sm_t *sm_btl, int n)
/* create a list of peers */
mca_btl_sm_component.sm_peers = (struct mca_btl_base_endpoint_t**)
calloc(n, sizeof(struct mca_btl_base_endpoint_t*));
if(NULL == mca_btl_sm_component.sm_peers)
if (NULL == mca_btl_sm_component.sm_peers) {
return OMPI_ERR_OUT_OF_RESOURCE;
}
/* Allocate Shared Memory BTL process coordination
* data structure. This will reside in shared memory */
/* set file name */
if(asprintf(&sm_ctl_file, "%s"OPAL_PATH_SEP"shared_mem_btl_module.%s",
orte_process_info.job_session_dir,
orte_process_info.nodename) < 0)
if (asprintf(&sm_ctl_file, "%s"OPAL_PATH_SEP"shared_mem_btl_module.%s",
orte_process_info.job_session_dir,
orte_process_info.nodename) < 0) {
return OMPI_ERR_OUT_OF_RESOURCE;
}
/* Pass in a data segment alignment of 0 to get no data
segment (only the shared control structure) */

207
ompi/mca/btl/smcuda/btl_smcuda.c
View File

@ -10,8 +10,8 @@
* Copyright (c) 2004-2005 The Regents of the University of California.
* All rights reserved.
* Copyright (c) 2006-2007 Voltaire. All rights reserved.
* Copyright (c) 2009 Cisco Systems, Inc. All rights reserved.
* Copyright (c) 2010 Los Alamos National Security, LLC.
* Copyright (c) 2009-2012 Cisco Systems, Inc. All rights reserved.
* Copyright (c) 2010-2012 Los Alamos National Security, LLC.
* All rights reserved.
* Copyright (c) 2012 NVIDIA Corporation. All rights reserved.
* $COPYRIGHT$
@ -37,10 +37,7 @@
#include "opal/class/opal_bitmap.h"
#include "opal/util/output.h"
#include "opal/util/printf.h"
#include "opal/mca/carto/carto.h"
#include "opal/mca/carto/base/base.h"
#include "opal/mca/paffinity/base/base.h"
#include "opal/mca/maffinity/base/base.h"
#include "opal/mca/hwloc/base/base.h"
#include "orte/util/proc_info.h"
#include "opal/datatype/opal_convertor.h"
#include "ompi/class/ompi_free_list.h"
@ -127,68 +124,6 @@ static void *mpool_calloc(size_t nmemb, size_t size)
return buf;
}
static void init_maffinity(int *my_mem_node, int *max_mem_node)
{
opal_carto_graph_t *topo;
opal_value_array_t dists;
int i, num_core, socket;
opal_paffinity_base_cpu_set_t cpus;
char *myslot = NULL;
opal_carto_node_distance_t *dist;
opal_carto_base_node_t *slot_node;
*my_mem_node = 0;
*max_mem_node = 1;
if (OMPI_SUCCESS != opal_carto_base_get_host_graph(&topo, "Memory")) {
return;
}
OBJ_CONSTRUCT(&dists, opal_value_array_t);
opal_value_array_init(&dists, sizeof(opal_carto_node_distance_t));
if (OMPI_SUCCESS != opal_paffinity_base_get_processor_info(&num_core)) {
num_core = 100; /* set something large */
}
OPAL_PAFFINITY_CPU_ZERO(cpus);
opal_paffinity_base_get(&cpus);
/* find core we are running on */
for (i = 0; i < num_core; i++) {
if (OPAL_PAFFINITY_CPU_ISSET(i, cpus)) {
break;
}
}
if (OMPI_SUCCESS != opal_paffinity_base_get_map_to_socket_core(i, &socket, &i)) {
/* no topology info available */
goto out;
}
asprintf(&myslot, "slot%d", socket);
slot_node = opal_carto_base_find_node(topo, myslot);
if(NULL == slot_node) {
goto out;
}
opal_carto_base_get_nodes_distance(topo, slot_node, "Memory", &dists);
if((*max_mem_node = opal_value_array_get_size(&dists)) < 2) {
goto out;
}
dist = (opal_carto_node_distance_t *) opal_value_array_get_item(&dists, 0);
opal_maffinity_base_node_name_to_id(dist->node->node_name, my_mem_node);
out:
if (myslot) {
free(myslot);
}
OBJ_DESTRUCT(&dists);
opal_carto_base_free_graph(topo);
}
static int smcuda_btl_first_time_init(mca_btl_smcuda_t *smcuda_btl, int n)
{
size_t size, length, length_payload;
@ -197,61 +132,111 @@ static int smcuda_btl_first_time_init(mca_btl_smcuda_t *smcuda_btl, int n)
int my_mem_node=-1, num_mem_nodes=-1, i;
ompi_proc_t **procs;
size_t num_procs;
mca_mpool_base_resources_t res;
mca_btl_smcuda_component_t* m = &mca_btl_smcuda_component;
init_maffinity(&my_mem_node, &num_mem_nodes);
mca_btl_smcuda_component.mem_node = my_mem_node;
mca_btl_smcuda_component.num_mem_nodes = num_mem_nodes;
/* Assume we don't have hwloc support and fill in dummy info */
mca_btl_sm_component.mem_node = my_mem_node = 0;
mca_btl_sm_component.num_mem_nodes = num_mem_nodes = 1;
#if OPAL_HAVE_HWLOC
/* If we have hwloc support, then get accurate information */
if (NULL != opal_hwloc_topology) {
i = opal_hwloc_base_get_nbobjs_by_type(opal_hwloc_topology,
HWLOC_OBJ_NODE, 0,
OPAL_HWLOC_AVAILABLE);
/* If we find >0 NUMA nodes, then investigate further */
if (i > 0) {
opal_hwloc_level_t bind_level;
unsigned int bind_index;
/* JMS This tells me how many numa nodes are *available*,
but it's not how many are being used *by this job*.
Note that this is the value we've previously used (from
the previous carto-based implementation), but it really
should be improved to be how many NUMA nodes are being
used *in this job*. */
mca_btl_sm_component.num_mem_nodes = num_mem_nodes = i;
/* Fill opal_hwloc_my_cpuset and find out to what level
this process is bound (if at all) */
opal_hwloc_base_get_local_cpuset();
opal_hwloc_base_get_level_and_index(opal_hwloc_my_cpuset,
&bind_level, &bind_index);
if (OPAL_HWLOC_NODE_LEVEL != bind_level) {
/* We are bound to *something* (i.e., our binding
level is less than "node", meaning the entire
machine), so discover which NUMA node this process
is bound */
if (OPAL_HWLOC_NUMA_LEVEL == bind_level) {
mca_btl_sm_component.mem_node = my_mem_node = (int) bind_index;
} else {
if (OPAL_SUCCESS ==
opal_hwloc_base_get_local_index(HWLOC_OBJ_NODE, 0, &bind_index)) {
mca_btl_sm_component.mem_node = my_mem_node = (int) bind_index;
} else {
/* Weird. We can't figure out what NUMA node
we're on. :-( */
mca_btl_sm_component.mem_node = my_mem_node = -1;
}
}
}
}
}
#endif
/* lookup shared memory pool */
mca_btl_smcuda_component.sm_mpools = (mca_mpool_base_module_t **) calloc(num_mem_nodes,
sizeof(mca_mpool_base_module_t*));
/* create mpool for each memory node */
for(i = 0; i < num_mem_nodes; i++) {
mca_mpool_base_resources_t res;
mca_btl_smcuda_component_t* m = &mca_btl_smcuda_component;
/* Create one mpool. Per discussion with George and a UTK Euro
MPI 2010 paper, it may be beneficial to create multiple mpools.
Leaving that for a future optimization, however. */
/* Disable memory binding, because each MPI process will claim
pages in the mpool for their local NUMA node */
res.mem_node = -1;
/* disable memory binding if there is only one memory node */
res.mem_node = (num_mem_nodes == 1) ? -1 : i;
/* determine how much memory to create */
/*
* This heuristic formula mostly says that we request memory for:
* - nfifos FIFOs, each comprising:
* . a sm_fifo_t structure
* . many pointers (fifo_size of them per FIFO)
* - eager fragments (2*n of them, allocated in sm_free_list_inc chunks)
* - max fragments (sm_free_list_num of them)
*
* On top of all that, we sprinkle in some number of "opal_cache_line_size"
* additions to account for some padding and edge effects that may lie
* in the allocator.
*/
res.size =
FIFO_MAP_NUM(n) * ( sizeof(sm_fifo_t) + sizeof(void *) * m->fifo_size + 4 * opal_cache_line_size )
+ ( 2 * n + m->sm_free_list_inc ) * ( m->eager_limit + 2 * opal_cache_line_size )
+ m->sm_free_list_num * ( m->max_frag_size + 2 * opal_cache_line_size );
/* determine how much memory to create */
/*
* This heuristic formula mostly says that we request memory for:
* - nfifos FIFOs, each comprising:
* . a sm_fifo_t structure
* . many pointers (fifo_size of them per FIFO)
* - eager fragments (2*n of them, allocated in sm_free_list_inc chunks)
* - max fragments (sm_free_list_num of them)
*
* On top of all that, we sprinkle in some number of "opal_cache_line_size"
* additions to account for some padding and edge effects that may lie
* in the allocator.
*/
res.size =
FIFO_MAP_NUM(n) * ( sizeof(sm_fifo_t) + sizeof(void *) * m->fifo_size + 4 * opal_cache_line_size )
+ ( 2 * n + m->sm_free_list_inc ) * ( m->eager_limit + 2 * opal_cache_line_size )
+ m->sm_free_list_num * ( m->max_frag_size + 2 * opal_cache_line_size );
/* before we multiply by n, make sure the result won't overflow */
/* Stick that little pad in, particularly since we'll eventually
* need a little extra space. E.g., in mca_mpool_sm_init() in
* mpool_sm_component.c when sizeof(mca_common_sm_module_t) is
* added.
*/
if ( ((double) res.size) * n > LONG_MAX - 4096 ) {
return OMPI_ERR_OUT_OF_RESOURCE;
}
res.size *= n;
/* before we multiply by n, make sure the result won't overflow */
/* Stick that little pad in, particularly since we'll eventually
* need a little extra space. E.g., in mca_mpool_sm_init() in
* mpool_sm_component.c when sizeof(mca_common_sm_module_t) is
* added.
*/
if ( ((double) res.size) * n > LONG_MAX - 4096 )
/* now, create it */
mca_btl_smcuda_component.sm_mpools[0] =
mca_mpool_base_module_create(mca_btl_smcuda_component.sm_mpool_name,
smcuda_btl, &res);
/* Sanity check to ensure that we found it */
if (NULL == mca_btl_smcuda_component.sm_mpools[0]) {
return OMPI_ERR_OUT_OF_RESOURCE;
res.size *= n;
}
/* now, create it */
mca_btl_smcuda_component.sm_mpools[i] =
mca_mpool_base_module_create(mca_btl_smcuda_component.sm_mpool_name,
smcuda_btl, &res);
/* Sanity check to ensure that we found it */
if(NULL == mca_btl_smcuda_component.sm_mpools[i])
return OMPI_ERR_OUT_OF_RESOURCE;
mca_btl_smcuda_component.sm_mpool = mca_btl_smcuda_component.sm_mpools[0];
if(i == my_mem_node)
mca_btl_smcuda_component.sm_mpool = mca_btl_smcuda_component.sm_mpools[i];
#if OMPI_CUDA_SUPPORT
/* Create a local memory pool that sends handles to the remote
* side. Note that the res argument is not really used, but

209
ompi/mca/btl/wv/btl_wv_component.c
View File

@ -10,7 +10,7 @@
* University of Stuttgart. All rights reserved.
* Copyright (c) 2004-2005 The Regents of the University of California.
* All rights reserved.
* Copyright (c) 2006-2011 Cisco Systems, Inc. All rights reserved.
* Copyright (c) 2006-2012 Cisco Systems, Inc. All rights reserved.
* Copyright (c) 2006-2009 Mellanox Technologies. All rights reserved.
* Copyright (c) 2006-2012 Los Alamos National Security, LLC. All rights
* reserved.
@ -41,9 +41,7 @@
#include "opal/util/argv.h"
#include "opal/memoryhooks/memory.h"
#include "opal/mca/base/mca_base_param.h"
#include "opal/mca/carto/carto.h"
#include "opal/mca/carto/base/base.h"
#include "opal/mca/paffinity/base/base.h"
#include "opal/mca/hwloc/base/base.h"
#include "opal/mca/installdirs/installdirs.h"
#include "opal_stdint.h"
@ -1926,63 +1924,149 @@ static void wv_free_device_list_compat(struct wv_device **ib_devs)
free(ib_devs);
}
static opal_carto_graph_t *host_topo;
static int get_ib_dev_distance(struct wv_device *dev)
{
opal_paffinity_base_cpu_set_t cpus;
opal_carto_base_node_t *device_node;
int min_distance = -1, i, num_processors;
const char *device = dev->name;
if(opal_paffinity_base_get_processor_info(&num_processors) != OMPI_SUCCESS) {
num_processors = 100; /* Choose something big enough */
}
device_node = opal_carto_base_find_node(host_topo, device);
/* no topology info for device found. Assume that it is close */
if(NULL == device_node)
return 0;
OPAL_PAFFINITY_CPU_ZERO(cpus);
opal_paffinity_base_get(&cpus);
for (i = 0; i < num_processors; i++) {
opal_carto_base_node_t *slot_node;
int distance, socket, core;
char *slot;
if(!OPAL_PAFFINITY_CPU_ISSET(i, cpus))
continue;
opal_paffinity_base_get_map_to_socket_core(i, &socket, &core);
asprintf(&slot, "slot%d", socket);
slot_node = opal_carto_base_find_node(host_topo, slot);
free(slot);
if(NULL == slot_node)
return 0;
distance = opal_carto_base_spf(host_topo, slot_node, device_node);
if(distance < 0)
return 0;
if(min_distance < 0 || min_distance > distance)
min_distance = distance;
}
return min_distance;
}
struct dev_distance {
struct wv_device *ib_dev;
int distance;
};
static int get_ib_dev_distance(struct ibv_device *dev)
{
/* If we don't have hwloc, we'll default to a distance of 0,
because we have no way of measuring. */
int distance = 0;
#if OPAL_HAVE_HWLOC
int a, b, i;
hwloc_cpuset_t my_cpuset = NULL, ibv_cpuset = NULL;
hwloc_obj_t my_obj, ibv_obj, node_obj;
/* Note that this struct is owned by hwloc; there's no need to
free it at the end of time */
static const struct hwloc_distances_s *hwloc_distances = NULL;
if (NULL == hwloc_distances) {
hwloc_distances =
hwloc_get_whole_distance_matrix_by_type(opal_hwloc_topology,
HWLOC_OBJ_NODE);
}
/* If we got no info, just return 0 */
if (NULL == hwloc_distances || NULL == hwloc_distances->latency) {
goto out;
}
/* Next, find the NUMA node where this IBV device is located */
ibv_cpuset = hwloc_bitmap_alloc();
if (NULL == ibv_cpuset) {
goto out;
}
if (0 != hwloc_ibv_get_device_cpuset(opal_hwloc_topology, dev, ibv_cpuset)) {
goto out;
}
ibv_obj = hwloc_get_obj_covering_cpuset(opal_hwloc_topology, ibv_cpuset);
if (NULL == ibv_obj) {
goto out;
}
/* If ibv_obj is a NUMA node or below, we're good. */
switch (ibv_obj->type) {
case HWLOC_OBJ_NODE:
case HWLOC_OBJ_SOCKET:
case HWLOC_OBJ_CACHE:
case HWLOC_OBJ_CORE:
case HWLOC_OBJ_PU:
while (NULL != ibv_obj && ibv_obj->type != HWLOC_OBJ_NODE) {
ibv_obj = ibv_obj->parent;
}
break;
default:
/* If it's above a NUMA node, then I don't know how to compute
the distance... */
ibv_obj = NULL;
break;
}
/* If we don't have an object for this ibv device, give up */
if (NULL == ibv_obj) {
goto out;
}
/* This function is only called if the process is bound, so let's
find out where we are bound to. For the moment, we only care
about the NUMA node to which we are bound. */
my_cpuset = hwloc_bitmap_alloc();
if (NULL == my_cpuset) {
goto out;
}
if (0 != hwloc_get_cpubind(opal_hwloc_topology, my_cpuset, 0)) {
hwloc_bitmap_free(my_cpuset);
goto out;
}
my_obj = hwloc_get_obj_covering_cpuset(opal_hwloc_topology, my_cpuset);
if (NULL == my_obj) {
goto out;
}
/* If my_obj is a NUMA node or below, we're good. */
switch (my_obj->type) {
case HWLOC_OBJ_NODE:
case HWLOC_OBJ_SOCKET:
case HWLOC_OBJ_CACHE:
case HWLOC_OBJ_CORE:
case HWLOC_OBJ_PU:
while (NULL != my_obj && my_obj->type != HWLOC_OBJ_NODE) {
my_obj = my_obj->parent;
}
if (NULL != my_obj) {
/* Distance may be asymetrical, so calculate both of them
and take the max */
a = hwloc_distances->latency[my_obj->logical_index *
(ibv_obj->logical_index *
hwloc_distances->nbobjs)];
b = hwloc_distances->latency[ibv_obj->logical_index *
(my_obj->logical_index *
hwloc_distances->nbobjs)];
distance = (a > b) ? a : b;
}
break;
default:
/* If the obj is above a NUMA node, then we're bound to more than
one NUMA node. Find the max distance. */
i = 0;
for (node_obj = hwloc_get_obj_inside_cpuset_by_type(opal_hwloc_topology,
ibv_obj->cpuset,
HWLOC_OBJ_NODE, i);
NULL != node_obj;
node_obj = hwloc_get_obj_inside_cpuset_by_type(opal_hwloc_topology,
ibv_obj->cpuset,
HWLOC_OBJ_NODE, ++i)) {
a = hwloc_distances->latency[node_obj->logical_index *
(ibv_obj->logical_index *
hwloc_distances->nbobjs)];
b = hwloc_distances->latency[ibv_obj->logical_index *
(node_obj->logical_index *
hwloc_distances->nbobjs)];
a = (a > b) ? a : b;
distance = (a > distance) ? a : distance;
}
break;
}
out:
if (NULL != ibv_cpuset) {
hwloc_bitmap_free(ibv_cpuset);
}
if (NULL != my_cpuset) {
hwloc_bitmap_free(my_cpuset);
}
#endif
return distance;
}
static int compare_distance(const void *p1, const void *p2)
{
const struct dev_distance *d1 = (const struct dev_distance *) p1;
@ -2001,17 +2085,24 @@ sort_devs_by_distance(struct wv_device **ib_devs, int count)
* if we have found more than one device, search for the shortest path.
* otherwise, just use the one we got
*/
if( count == 1) {
if (1 == count) {
devs[0].ib_dev = ib_devs[0];
devs[0].distance = 0;
return devs;
}
opal_carto_base_get_host_graph(&host_topo, "Infiniband");
for (i = 0; i < count; i++) {
devs[i].ib_dev = ib_devs[i];
devs[i].distance = get_ib_dev_distance(ib_devs[i]);
if (orte_proc_is_bound) {
/* If this process is bound to one or more PUs, we can get
an accurate distance. */
devs[i].distance = get_ib_dev_distance(ib_devs[i]);
} else {
/* Since we're not bound, just assume that the device is
close. */
devs[i].distance = 0;
}
}
qsort(devs, count, sizeof(struct dev_distance), compare_distance);

2
ompi/mca/coll/fca/coll_fca_module.c
View File

@ -1,5 +1,6 @@
/**
Copyright (c) 2011 Mellanox Technologies. All rights reserved.
Copyright (c) 2012 Cisco Systems, Inc. All rights reserved.
$COPYRIGHT$
Additional copyrights may follow
@ -7,7 +8,6 @@
$HEADER$
*/
#include "coll_fca.h"
#include "opal/mca/paffinity/paffinity.h"
/*
* Initial query function that is invoked during MPI_INIT, allowing

3
ompi/mca/coll/hierarch/coll_hierarch.c
View File

@ -12,6 +12,7 @@
* All rights reserved.
* Copyright (c) 2007-2008 University of Houston. All rights reserved.
* Copyright (c) 2008 Sun Microsystems, Inc. All rights reserved.
* Copyright (c) 2012 Cisco Systems, Inc. All rights reserved.
* $COPYRIGHT$
*
* Additional copyrights may follow
@ -24,6 +25,8 @@
#include <stdio.h>
#include "opal/mca/hwloc/base/base.h"
#include "mpi.h"
#include "ompi/communicator/communicator.h"
#include "ompi/group/group.h"

15
ompi/mca/coll/sm/coll_sm_module.c
View File

@ -10,8 +10,8 @@
* Copyright (c) 2004-2005 The Regents of the University of California.
* All rights reserved.
* Copyright (c) 2008 Sun Microsystems, Inc. All rights reserved.
* Copyright (c) 2009-2010 Cisco Systems, Inc. All rights reserved.
* Copyright (c) 2010-2011 Los Alamos National Security, LLC.
* Copyright (c) 2009-2012 Cisco Systems, Inc. All rights reserved.
* Copyright (c) 2010-2012 Los Alamos National Security, LLC.
* All rights reserved.
* $COPYRIGHT$
*
@ -48,8 +48,7 @@
#include "mpi.h"
#include "opal_stdint.h"
#include "opal/mca/maffinity/maffinity.h"
#include "opal/mca/maffinity/base/base.h"
#include "opal/mca/hwloc/base/base.h"
#include "opal/util/os_path.h"
#include "orte/util/proc_info.h"
@ -257,7 +256,7 @@ int ompi_coll_sm_lazy_enable(mca_coll_base_module_t *module,
mca_coll_sm_comm_t *data = NULL;
size_t control_size, frag_size;
mca_coll_sm_component_t *c = &mca_coll_sm_component;
opal_maffinity_base_segment_t *maffinity;
opal_hwloc_base_memory_segment_t *maffinity;
int parent, min_child, max_child, num_children;
unsigned char *base = NULL;
const int num_barrier_buffers = 2;
@ -270,8 +269,8 @@ int ompi_coll_sm_lazy_enable(mca_coll_base_module_t *module,
/* Get some space to setup memory affinity (just easier to try to
alloc here to handle the error case) */
maffinity = (opal_maffinity_base_segment_t*)
malloc(sizeof(opal_maffinity_base_segment_t) *
maffinity = (opal_hwloc_base_memory_segment_t*)
malloc(sizeof(opal_hwloc_base_memory_segment_t) *
c->sm_comm_num_segments * 3);
if (NULL == maffinity) {
opal_output_verbose(10, mca_coll_base_output,
@ -458,7 +457,7 @@ int ompi_coll_sm_lazy_enable(mca_coll_base_module_t *module,
/* Setup memory affinity so that the pages that belong to this
process are local to this process */
opal_maffinity_base_set(maffinity, j);
opal_hwloc_base_memory_set(maffinity, j);
free(maffinity);
/* Zero out the control structures that belong to this process */

16
ompi/mca/mpool/sm/mpool_sm_module.c
View File

@ -9,8 +9,8 @@
* University of Stuttgart. All rights reserved.
* Copyright (c) 2004-2005 The Regents of the University of California.
* All rights reserved.
* Copyright (c) 2009 Cisco Systems, Inc. All rights reserved.
* Copyright (c) 2011 Los Alamos National Security, LLC.
* Copyright (c) 2009-2012 Cisco Systems, Inc. All rights reserved.
* Copyright (c) 2011-2012 Los Alamos National Security, LLC.
* All rights reserved.
* Copyright (c) 2011 NVIDIA Corporation. All rights reserved.
* $COPYRIGHT$
@ -28,9 +28,7 @@
#ifdef HAVE_UNISTD_H
#include <unistd.h>
#endif
#include "opal/mca/maffinity/maffinity.h"
#include "opal/mca/maffinity/maffinity_types.h"
#include "opal/mca/maffinity/base/base.h"
#include "opal/mca/hwloc/base/base.h"
#include "orte/util/proc_info.h"
#if OPAL_ENABLE_FT_CR == 1
@ -87,14 +85,14 @@ void* mca_mpool_sm_alloc(
mca_mpool_base_registration_t** registration)
{
mca_mpool_sm_module_t* mpool_sm = (mca_mpool_sm_module_t*)mpool;
opal_maffinity_base_segment_t mseg;
opal_hwloc_base_memory_segment_t mseg;
mseg.mbs_start_addr =
mpool_sm->sm_allocator->alc_alloc(mpool_sm->sm_allocator, size, align, registration);
if(mpool_sm->mem_node >= 0) {
mseg.mbs_len = size;
opal_maffinity_base_bind(&mseg, 1, mpool_sm->mem_node);
opal_hwloc_base_membind(&mseg, 1, mpool_sm->mem_node);
}
#if OPAL_CUDA_SUPPORT
@ -117,14 +115,14 @@ void* mca_mpool_sm_realloc(
mca_mpool_base_registration_t** registration)
{
mca_mpool_sm_module_t* mpool_sm = (mca_mpool_sm_module_t*)mpool;
opal_maffinity_base_segment_t mseg;
opal_hwloc_base_memory_segment_t mseg;
mseg.mbs_start_addr =
mpool_sm->sm_allocator->alc_realloc(mpool_sm->sm_allocator, addr, size,
registration);
if(mpool_sm->mem_node >= 0) {
mseg.mbs_len = size;
opal_maffinity_base_bind(&mseg, 1, mpool_sm->mem_node);
opal_hwloc_base_membind(&mseg, 1, mpool_sm->mem_node);
}
return mseg.mbs_start_addr;

611
ompi/mpiext/affinity/c/mpiext_affinity_str.c
View File

@ -4,6 +4,8 @@
* Corporation. All rights reserved.
* Copyright (c) 2010-2012 Cisco Systems, Inc. All rights reserved.
* Copyright (c) 2010 Oracle and/or its affiliates. All rights reserved.
* Copyright (c) 2012 Los Alamos National Security, LLC. All rights
* reserved.
* $COPYRIGHT$
*
* Additional copyrights may follow
@ -24,7 +26,10 @@
#include <stdio.h>
#include <string.h>
#include "opal/mca/paffinity/base/base.h"
#include "opal/mca/hwloc/base/base.h"
#include "opal/runtime/opal.h"
#include "orte/runtime/orte_globals.h"
#include "ompi/communicator/communicator.h"
#include "ompi/errhandler/errhandler.h"
@ -40,6 +45,8 @@ static int get_layout_ompi_bound(char str[OMPI_AFFINITY_STRING_MAX]);
static int get_layout_current_binding(char str[OMPI_AFFINITY_STRING_MAX]);
static int get_layout_exists(char str[OMPI_AFFINITY_STRING_MAX]);
/*------------------------------------------------------------------------------*/
int OMPI_Affinity_str(ompi_affinity_fmt_t fmt_type,
char ompi_bound[OMPI_AFFINITY_STRING_MAX],
char current_binding[OMPI_AFFINITY_STRING_MAX],
@ -50,18 +57,27 @@ int OMPI_Affinity_str(ompi_affinity_fmt_t fmt_type,
memset(ompi_bound, 0, sizeof(ompi_bound));
memset(current_binding, 0, sizeof(current_binding));
switch(fmt_type) {
/* If we have no hwloc support, return nothing */
if (NULL == opal_hwloc_topology) {
strncpy(ompi_bound, "Not supported", sizeof(ompi_bound));
strncpy(current_binding, "Not supported", sizeof(current_binding));
strncpy(exists, "Not supported", sizeof(exists));
return MPI_SUCCESS;
}
/* Otherwise, return useful information */
switch (fmt_type) {
case OMPI_AFFINITY_RSRC_STRING_FMT:
if (OPAL_SUCCESS != (ret = get_rsrc_ompi_bound(ompi_bound)) ||
OPAL_SUCCESS != (ret = get_rsrc_current_binding(current_binding)) ||
OPAL_SUCCESS != (ret = get_rsrc_exists(exists))) {
if (OMPI_SUCCESS != (ret = get_rsrc_ompi_bound(ompi_bound)) ||
OMPI_SUCCESS != (ret = get_rsrc_current_binding(current_binding)) ||
OMPI_SUCCESS != (ret = get_rsrc_exists(exists))) {
return OMPI_ERRHANDLER_INVOKE(MPI_COMM_WORLD, ret, FUNC_NAME);
}
break;
case OMPI_AFFINITY_LAYOUT_FMT:
if (OPAL_SUCCESS != (ret = get_layout_ompi_bound(ompi_bound)) ||
OPAL_SUCCESS != (ret = get_layout_current_binding(current_binding)) ||
OPAL_SUCCESS != (ret = get_layout_exists(exists))) {
if (OMPI_SUCCESS != (ret = get_layout_ompi_bound(ompi_bound)) ||
OMPI_SUCCESS != (ret = get_layout_current_binding(current_binding)) ||
OMPI_SUCCESS != (ret = get_layout_exists(exists))) {
return OMPI_ERRHANDLER_INVOKE(MPI_COMM_WORLD, ret, FUNC_NAME);
}
break;
@ -72,165 +88,554 @@ int OMPI_Affinity_str(ompi_affinity_fmt_t fmt_type,
return MPI_SUCCESS;
}
/*------------------------------------------------------------------------------*/
/*
* Turn an int bitmap to a "a-b,c" range kind of string
*/
static char *bitmap2rangestr(int bitmap)
{
size_t i;
int range_start, range_end;
bool first, isset;
char tmp[BUFSIZ];
const int stmp = sizeof(tmp) - 1;
static char ret[BUFSIZ];
memset(ret, 0, sizeof(ret));
first = true;
range_start = -999;
for (i = 0; i < sizeof(int) * 8; ++i) {
isset = (bitmap & (1 << i));
/* Do we have a running range? */
if (range_start >= 0) {
if (isset) {
continue;
} else {
/* A range just ended; output it */
if (!first) {
strncat(ret, ",", sizeof(ret) - strlen(ret));
first = false;
}
range_end = i - 1;
if (range_start == range_end) {
snprintf(tmp, stmp, "%d", range_start);
} else {
snprintf(tmp, stmp, "%d-%d", range_start, range_end);
}
strncat(ret, tmp, sizeof(ret) - strlen(ret));
range_start = -999;
}
}
/* No running range */
else {
if (isset) {
range_start = i;
}
}
}
/* If we ended the bitmap with a range open, output it */
if (range_start >= 0) {
if (!first) {
strncat(ret, ",", sizeof(ret) - strlen(ret));
first = false;
}
range_end = i - 1;
if (range_start == range_end) {
snprintf(tmp, stmp, "%d", range_start);
} else {
snprintf(tmp, stmp, "%d-%d", range_start, range_end);
}
strncat(ret, tmp, sizeof(ret) - strlen(ret));
}
return ret;
}
/*
* Make a map of socket/core/hwthread tuples
*/
static int build_map(int *num_sockets_arg, int *num_cores_arg,
hwloc_cpuset_t cpuset, int ***map)
{
static int num_sockets = -1, num_cores = -1;
int socket_index, core_index, pu_index;
hwloc_obj_t socket, core, pu;
int **data;
/* Find out how many sockets we have (cached so that we don't have
to look this up every time) */
if (num_sockets < 0) {
num_sockets = hwloc_get_nbobjs_by_type(opal_hwloc_topology, HWLOC_OBJ_CORE);
/* Lazy: take the total number of cores that we have in the
topology; that'll be less than the max number of cores
under any given socket */
num_cores = hwloc_get_nbobjs_by_type(opal_hwloc_topology, HWLOC_OBJ_CORE);
}
*num_sockets_arg = num_sockets;
*num_cores_arg = num_cores;
/* Alloc a 2D array: sockets x cores. */
data = malloc(num_sockets * sizeof(int *));
if (NULL == data) {
return OMPI_ERR_OUT_OF_RESOURCE;
}
data[0] = calloc(num_sockets * num_cores, sizeof(int));
if (NULL == data[0]) {
free(data);
return OMPI_ERR_OUT_OF_RESOURCE;
}
for (socket_index = 1; socket_index < num_sockets; ++socket_index) {
data[socket_index] = data[socket_index - 1] + num_cores;
}
/* Iterate the PUs in this cpuset; fill in the data[][] array with
the socket/core/pu triples */
for (pu_index = 0,
pu = hwloc_get_obj_inside_cpuset_by_type(opal_hwloc_topology,
cpuset, HWLOC_OBJ_PU,
pu_index);
NULL != pu;
pu = hwloc_get_obj_inside_cpuset_by_type(opal_hwloc_topology,
cpuset, HWLOC_OBJ_PU,
++pu_index)) {
/* Go upward and find the core this PU belongs to */
core = pu;
while (NULL != core && core->type != HWLOC_OBJ_CORE) {
core = core->parent;
}
core_index = 0;
if (NULL != core) {
core_index = core->os_index;
}
/* Go upward and find the socket this PU belongs to */
socket = pu;
while (NULL != socket && socket->type != HWLOC_OBJ_SOCKET) {
socket = socket->parent;
}
socket_index = 0;
if (NULL != socket) {
socket_index = socket->os_index;
}
/* Save this socket/core/pu combo. LAZY: Assuming that we
won't have more PU's per core than (sizeof(int)*8). */
data[socket_index][core_index] |= (1 << pu->sibling_rank);
}
*map = data;
return OMPI_SUCCESS;
}
/*
* Make a prettyprint string for a hwloc_cpuset_t
*/
static int cset2str(char *str, int len, hwloc_cpuset_t cpuset)
{
bool first;
int num_sockets, num_cores;
int ret, socket_index, core_index;
char tmp[BUFSIZ];
const int stmp = sizeof(tmp) - 1;
int **map;
str[0] = tmp[stmp] = '\0';
if (OMPI_SUCCESS != (ret = build_map(&num_sockets, &num_cores, cpuset, &map))) {
return ret;
}
/* Iterate over the data maxtrix and build up the string */
first = true;
for (socket_index = 0; socket_index < num_sockets; ++socket_index) {
for (core_index = 0; core_index < num_cores; ++core_index) {
if (map[socket_index][core_index] > 0) {
if (!first) {
strncat(str, ", ", len - strlen(str));
}
first = false;
snprintf(tmp, stmp, "socket %d[core %d[hwt %s]]",
socket_index, core_index,
bitmap2rangestr(map[socket_index][core_index]));
strncat(str, tmp, len - strlen(str));
}
}
}
free(map[0]);
free(map);
return OMPI_SUCCESS;
}
/**
* Make a prettyprint string for a cset in a map format.
* Example: [B./..]
* Key: [] - signifies socket
* / - signifies core
* . - signifies PU a process not bound to
* B - signifies PU a process is bound to
*/
static int cset2mapstr(char *str, int len, hwloc_cpuset_t cpuset)
{
char tmp[BUFSIZ];
int core_index, pu_index;
const int stmp = sizeof(tmp) - 1;
hwloc_obj_t socket, core, pu;
str[0] = tmp[stmp] = '\0';
/* Iterate over all existing sockets */
for (socket = hwloc_get_obj_by_type(opal_hwloc_topology,
HWLOC_OBJ_SOCKET, 0);
NULL != socket;
socket = socket->next_cousin) {
strncat(str, "[", len - strlen(str));
/* Iterate over all existing cores in this socket */
core_index = 0;
for (core = hwloc_get_obj_inside_cpuset_by_type(opal_hwloc_topology,
socket->cpuset,
HWLOC_OBJ_CORE, core_index);
NULL != core;
core = hwloc_get_obj_inside_cpuset_by_type(opal_hwloc_topology,
socket->cpuset,
HWLOC_OBJ_CORE, ++core_index)) {
if (core_index > 0) {
strncat(str, "/", len - strlen(str));
}
/* Iterate over all existing PUs in this core */
pu_index = 0;
for (pu = hwloc_get_obj_inside_cpuset_by_type(opal_hwloc_topology,
core->cpuset,
HWLOC_OBJ_PU, pu_index);
NULL != pu;
pu = hwloc_get_obj_inside_cpuset_by_type(opal_hwloc_topology,
core->cpuset,
HWLOC_OBJ_PU, ++pu_index)) {
/* Is this PU in the cpuset? */
if (hwloc_bitmap_isset(cpuset, pu->os_index)) {
strncat(str, "B", len - strlen(str));
} else {
strncat(str, ".", len - strlen(str));
}
}
}
strncat(str, "]", len - strlen(str));
}
return OMPI_SUCCESS;
}
/*------------------------------------------------------------------------------*/
/*
* Where did OMPI bind this process? (prettyprint)
*/
static int get_rsrc_ompi_bound(char str[OMPI_AFFINITY_STRING_MAX])
{
int ret;
opal_paffinity_base_cpu_set_t cset;
/* If OMPI did not bind, indicate that */
if (!opal_paffinity_base_bound) {
const char tmp[] = "Open MPI did not bind this process";
if (!orte_proc_is_bound) {
const char tmp[] = "This process is not bound";
strncpy(str, tmp, OMPI_AFFINITY_STRING_MAX - 1);
return OPAL_SUCCESS;
return OMPI_SUCCESS;
}
/* Find out what OMPI bound us to and prettyprint it */
ret =
opal_paffinity_base_parse_binding(opal_paffinity_base_applied_binding,
&cset);
if (OPAL_SUCCESS != ret) {
return ret;
}
return opal_paffinity_base_cset2str(str, OMPI_AFFINITY_STRING_MAX, &cset);
ret = cset2str(str, OMPI_AFFINITY_STRING_MAX, orte_proc_applied_binding);
return ret;
}
/*
* Where is this process currently bound? (prettyprint)
*/
static int get_rsrc_current_binding(char str[OMPI_AFFINITY_STRING_MAX])
{
int ret, flag;
opal_paffinity_base_cpu_set_t cset;
int ret;
hwloc_obj_t root;
hwloc_cpuset_t boundset, rootset;
bool bound = false;
/* Get our binding */
ret = opal_paffinity_base_get(&cset);
if (OPAL_SUCCESS != ret) {
return ret;
/* get our root object */
root = hwloc_get_root_obj(opal_hwloc_topology);
rootset = opal_hwloc_base_get_available_cpus(opal_hwloc_topology, root);
/* get our bindings */
boundset = hwloc_bitmap_alloc();
if (hwloc_get_cpubind(opal_hwloc_topology, boundset,
HWLOC_CPUBIND_PROCESS) < 0) {
/* we are NOT bound if get_cpubind fails, nor can we be bound
- the environment does not support it */
bound = false;
} else {
/* we are bound if the two cpusets are not equal, or if there
is only ONE PU available to us */
if (0 != hwloc_bitmap_compare(boundset, rootset) ||
opal_hwloc_base_single_cpu(rootset) ||
opal_hwloc_base_single_cpu(boundset)) {
bound = true;
}
}
/* Are we bound anywhere? */
OPAL_PAFFINITY_PROCESS_IS_BOUND(cset, &flag);
if (!flag) {
/* If we are not bound, indicate that */
if (!bound) {
const char tmp[] = "Not bound (or bound to all available processors)";
strncat(str, tmp, OMPI_AFFINITY_STRING_MAX - 1);
return OPAL_SUCCESS;
ret = OMPI_SUCCESS;
}
return opal_paffinity_base_cset2str(str, OMPI_AFFINITY_STRING_MAX, &cset);
/* If we are bound, print it out */
else {
ret = cset2str(str, OMPI_AFFINITY_STRING_MAX, boundset);
}
hwloc_bitmap_free(boundset);
return OMPI_SUCCESS;
}
/* Prettyprint a list of all available processors */
/*
* Prettyprint a list of all available sockets and cores. Note that
* this is *everything* -- not just the ones that are available to
* this process.
*/
static int get_rsrc_exists(char str[OMPI_AFFINITY_STRING_MAX])
{
int ret, i, num_sockets, num_cores;
bool first = true;
int i, num_cores, num_pus;
char tmp[BUFSIZ];
const int stmp = sizeof(tmp) - 1;
hwloc_obj_t socket, core, c2;
tmp[stmp] = '\0';
/* Loop over the number of sockets in this machine */
ret = opal_paffinity_base_get_socket_info(&num_sockets);
if (OPAL_SUCCESS != ret) {
return ret;
}
for (i = 0; i < num_sockets; ++i) {
if (i > 0) {
strncat(str, ", ", OMPI_AFFINITY_STRING_MAX - strlen(str));
str[0] = '\0';
for (socket = hwloc_get_obj_by_type(opal_hwloc_topology,
HWLOC_OBJ_SOCKET, 0);
NULL != socket; socket = socket->next_cousin) {
/* If this isn't the first socket, add a delimiter */
if (!first) {
strncat(str, "; ", OMPI_AFFINITY_STRING_MAX - strlen(str));
}
snprintf(tmp, stmp, "socket %d has ", i);
first = false;
snprintf(tmp, stmp, "socket %d has ", socket->os_index);
strncat(str, tmp, OMPI_AFFINITY_STRING_MAX - strlen(str));
/* Loop over the number of cores in this socket */
ret = opal_paffinity_base_get_core_info(i, &num_cores);
if (OPAL_SUCCESS != ret) {
return ret;
}
if (1 == num_cores) {
strncat(str, "1 core", OMPI_AFFINITY_STRING_MAX - strlen(str));
} else {
snprintf(tmp, stmp, "%d cores", num_cores);
strncat(str, tmp, OMPI_AFFINITY_STRING_MAX - strlen(str));
/* Find out how many cores are inside this socket, and get an
object pointing to the first core. Also count how many PUs
are in the first core. */
num_cores = hwloc_get_nbobjs_inside_cpuset_by_type(opal_hwloc_topology,
socket->cpuset,
HWLOC_OBJ_CORE);
core = hwloc_get_obj_inside_cpuset_by_type(opal_hwloc_topology,
socket->cpuset,
HWLOC_OBJ_CORE, 0);
if (NULL != core) {
num_pus =
hwloc_get_nbobjs_inside_cpuset_by_type(opal_hwloc_topology,
core->cpuset,
HWLOC_OBJ_PU);
/* Only 1 core */
if (1 == num_cores) {
strncat(str, "1 core with ",
OMPI_AFFINITY_STRING_MAX - strlen(str));
if (1 == num_pus) {
strncat(str, "1 hwt",
OMPI_AFFINITY_STRING_MAX - strlen(str));
} else {
snprintf(tmp, stmp, "%d hwts", num_pus);
strncat(str, tmp, OMPI_AFFINITY_STRING_MAX - strlen(str));
}
}
/* Multiple cores */
else {
bool same = true;
snprintf(tmp, stmp, "%d cores", num_cores);
strncat(str, tmp, OMPI_AFFINITY_STRING_MAX - strlen(str));
/* Do all the cores have the same number of PUs? */
for (c2 = core; NULL != c2; c2 = c2->next_cousin) {
if (hwloc_get_nbobjs_inside_cpuset_by_type(opal_hwloc_topology,
core->cpuset,
HWLOC_OBJ_PU) !=
num_pus) {
same = false;
break;
}
}
/* Yes, they all have the same number of PUs */
if (same) {
snprintf(tmp, stmp, ", each with %d hwt", num_pus);
strncat(str, tmp, OMPI_AFFINITY_STRING_MAX - strlen(str));
if (num_pus != 1) {
strncat(str, "s", OMPI_AFFINITY_STRING_MAX - strlen(str));
}
}
/* No, they have differing numbers of PUs */
else {
bool first = true;
strncat(str, "with (", OMPI_AFFINITY_STRING_MAX - strlen(str));
for (c2 = core; NULL != c2; c2 = c2->next_cousin) {
if (!first) {
strncat(str, ", ",
OMPI_AFFINITY_STRING_MAX - strlen(str));
}
first = false;
i = hwloc_get_nbobjs_inside_cpuset_by_type(opal_hwloc_topology,
core->cpuset,
HWLOC_OBJ_PU);
snprintf(tmp, stmp, "%d", i);
strncat(str, tmp, OMPI_AFFINITY_STRING_MAX - strlen(str));
}
strncat(str, ") hwts",
OMPI_AFFINITY_STRING_MAX - strlen(str));
}
}
}
}
return OPAL_SUCCESS;
return OMPI_SUCCESS;
}
/*------------------------------------------------------------------------------*/
/*
* Where did OMPI bind this process? (layout string)
*/
static int get_layout_ompi_bound(char str[OMPI_AFFINITY_STRING_MAX])
{
int ret;
opal_paffinity_base_cpu_set_t cset;
/* If OMPI did not bind, indicate that */
if (!opal_paffinity_base_bound) {
const char tmp[] = "Open MPI did not bind this process";
if (!orte_proc_is_bound) {
const char tmp[] = "This process is not bound";
strncpy(str, tmp, OMPI_AFFINITY_STRING_MAX - 1);
return OPAL_SUCCESS;
return OMPI_SUCCESS;
}
/* Find out what OMPI bound us to and prettyprint it */
ret =
opal_paffinity_base_parse_binding(opal_paffinity_base_applied_binding,
&cset);
if (OPAL_SUCCESS != ret) {
return ret;
}
return opal_paffinity_base_cset2mapstr(str, OMPI_AFFINITY_STRING_MAX, &cset);
ret = cset2mapstr(str, OMPI_AFFINITY_STRING_MAX, orte_proc_applied_binding);
return ret;
}
/*
* Where is this process currently bound? (layout string)
*/
static int get_layout_current_binding(char str[OMPI_AFFINITY_STRING_MAX])
{
int ret = OPAL_SUCCESS, flag;
opal_paffinity_base_cpu_set_t cset;
int ret;
hwloc_obj_t root;
hwloc_cpuset_t boundset, rootset;
bool bound = false;
/* Get our binding */
ret = opal_paffinity_base_get(&cset);
if (OPAL_SUCCESS != ret) {
return ret;
/* get our root object */
root = hwloc_get_root_obj(opal_hwloc_topology);
rootset = opal_hwloc_base_get_available_cpus(opal_hwloc_topology, root);
/* get our bindings */
boundset = hwloc_bitmap_alloc();
if (hwloc_get_cpubind(opal_hwloc_topology, boundset,
HWLOC_CPUBIND_PROCESS) < 0) {
/* we are NOT bound if get_cpubind fails, nor can we be bound
- the environment does not support it */
bound = false;
} else {
/* we are bound if the two cpusets are not equal, or if there
is only ONE PU available to us */
if (0 != hwloc_bitmap_compare(boundset, rootset) ||
opal_hwloc_base_single_cpu(rootset) ||
opal_hwloc_base_single_cpu(boundset)) {
bound = true;
}
}
/* Are we bound anywhere? */
OPAL_PAFFINITY_PROCESS_IS_BOUND(cset, &flag);
if (!flag) {
/* If we are not bound, indicate that */
if (!bound) {
const char tmp[] = "Not bound (or bound to all available processors)";
strncat(str, tmp, OMPI_AFFINITY_STRING_MAX - 1);
return OPAL_SUCCESS;
ret = OMPI_SUCCESS;
}
return opal_paffinity_base_cset2mapstr(str, OMPI_AFFINITY_STRING_MAX, &cset);
/* If we are bound, print it out */
else {
ret = cset2mapstr(str, OMPI_AFFINITY_STRING_MAX, boundset);
}
hwloc_bitmap_free(boundset);
return OMPI_SUCCESS;
}
/* Prettyprint a list of all available processors in layout format*/
/*
* Make a layout string of all available sockets and cores. Note that
* this is *everything* -- not just the ones that are available to
* this process.
*
* Example: [../..]
* Key: [] - signifies socket
* / - signifies core
* . - signifies PU
*/
static int get_layout_exists(char str[OMPI_AFFINITY_STRING_MAX])
{
int ret, i, j, num_sockets, num_cores;
int core_index, pu_index;
int len = OMPI_AFFINITY_STRING_MAX;
hwloc_obj_t socket, core, pu;
str[0] = '\0';
/* Loop over the number of sockets in this machine */
ret = opal_paffinity_base_get_socket_info(&num_sockets);
if (OPAL_SUCCESS != ret) {
return ret;
}
for (i = 0; i < num_sockets; ++i) {
strncat(str, "[", len - strlen(str));
/* Loop over the number of cores in this socket */
ret = opal_paffinity_base_get_core_info(i, &num_cores);
if (OPAL_SUCCESS != ret) {
return ret;
/* Iterate over all existing sockets */
for (socket = hwloc_get_obj_by_type(opal_hwloc_topology,
HWLOC_OBJ_SOCKET, 0);
NULL != socket;
socket = socket->next_cousin) {
strncat(str, "[", len - strlen(str));
/* Iterate over all existing cores in this socket */
core_index = 0;
for (core = hwloc_get_obj_inside_cpuset_by_type(opal_hwloc_topology,
socket->cpuset,
HWLOC_OBJ_CORE, core_index);
NULL != core;
core = hwloc_get_obj_inside_cpuset_by_type(opal_hwloc_topology,
socket->cpuset,
HWLOC_OBJ_CORE, ++core_index)) {
if (core_index > 0) {
strncat(str, "/", len - strlen(str));
}
/* Iterate over all existing PUs in this core */
pu_index = 0;
for (pu = hwloc_get_obj_inside_cpuset_by_type(opal_hwloc_topology,
core->cpuset,
HWLOC_OBJ_PU, pu_index);
NULL != pu;
pu = hwloc_get_obj_inside_cpuset_by_type(opal_hwloc_topology,
core->cpuset,
HWLOC_OBJ_PU, ++pu_index)) {
strncat(str, ".", len - strlen(str));
}
}
for (j = 0; j < num_cores; j++) {
if (0 < j) {
/* add space after first core is printed */
strncat(str, " ", len - strlen(str));
}
/* mark core exists */
strncat(str, ".", len - strlen(str));
}
strncat(str, "]", len - strlen(str));
strncat(str, "]", len - strlen(str));
}
return OPAL_SUCCESS;
return OMPI_SUCCESS;
}

8
ompi/proc/proc.h
View File

@ -9,8 +9,8 @@
* University of Stuttgart. All rights reserved.
* Copyright (c) 2004-2005 The Regents of the University of California.
* All rights reserved.
* Copyright (c) 2006 Cisco Systems, Inc. All rights reserved.
* Copyright (c) 2007 Los Alamos National Security, LLC. All rights
* Copyright (c) 2006-2012 Cisco Systems, Inc. All rights reserved.
* Copyright (c) 2007-2012 Los Alamos National Security, LLC. All rights
* reserved.
* $COPYRIGHT$
*
@ -35,7 +35,7 @@
#include "ompi/types.h"
#include "opal/class/opal_list.h"
#include "opal/dss/dss_types.h"
#include "opal/mca/paffinity/paffinity.h"
#include "opal/mca/hwloc/hwloc.h"
#include "orte/types.h"
@ -63,7 +63,7 @@ struct ompi_proc_t {
/** architecture of this process */
uint32_t proc_arch;
/** flags for this proc */
opal_paffinity_locality_t proc_flags;
opal_hwloc_locality_t proc_flags;
/** Base convertor for the proc described by this process */
struct opal_convertor_t* proc_convertor;
/** A pointer to the name of this host - data is

12
ompi/runtime/help-mpi-runtime.txt
View File

@ -10,7 +10,7 @@
# University of Stuttgart. All rights reserved.
# Copyright (c) 2004-2005 The Regents of the University of California.
# All rights reserved.
# Copyright (c) 2007-2011 Cisco Systems, Inc. All rights reserved.
# Copyright (c) 2007-2012 Cisco Systems, Inc. All rights reserved.
# $COPYRIGHT$
#
# Additional copyrights may follow
@ -53,16 +53,6 @@ mpi_param_check value has therefore been ignored.
WARNING: Cannot set both the MCA parameters mpi_leave_pinned and
mpi_leave_pinned_pipeline to "true". Defaulting to mpi_leave_pinned
ONLY.
[mpi_init:startup:paffinity-unavailable]
The MCA parameter "opal_paffinity_alone" was set to a nonzero value,
but Open MPI was unable to bind MPI_COMM_WORLD rank %s to a processor.
Typical causes for this problem include:
- A node was oversubscribed (more processes than processors), in
which case Open MPI will not bind any processes on that node
- A startup mechanism was used which did not tell Open MPI which
processors to bind processes to
#
[mpi_finalize:invoked_multiple_times]
The function MPI_FINALIZE was invoked multiple times in a single

10
ompi/runtime/ompi_mpi_finalize.c
View File

@ -10,8 +10,8 @@
* University of Stuttgart. All rights reserved.
* Copyright (c) 2004-2005 The Regents of the University of California.
* All rights reserved.
* Copyright (c) 2006-2009 Cisco Systems, Inc. All rights reserved.
* Copyright (c) 2006-2011 Los Alamos National Security, LLC. All rights
* Copyright (c) 2006-2012 Cisco Systems, Inc. All rights reserved.
* Copyright (c) 2006-2012 Los Alamos National Security, LLC. All rights
* reserved.
* Copyright (c) 2006 University of Houston. All rights reserved.
* Copyright (c) 2009 Sun Microsystems, Inc. All rights reserved.
@ -42,7 +42,6 @@
#include "opal/mca/event/event.h"
#include "opal/util/output.h"
#include "opal/runtime/opal_progress.h"
#include "opal/mca/maffinity/base/base.h"
#include "opal/mca/base/base.h"
#include "orte/util/show_help.h"
#include "opal/sys/atomic.h"
@ -157,11 +156,6 @@ int ompi_mpi_finalize(void)
MPI lifetime, to get better latency when not using TCP */
opal_progress_event_users_increment();
/* If maffinity was setup, tear it down */
if (opal_maffinity_setup) {
opal_maffinity_base_close();
}
/* check to see if we want timing information */
mca_base_param_reg_int_name("ompi", "timing",
"Request that critical timing loops be measured",

211
ompi/runtime/ompi_mpi_init.c
View File

@ -39,8 +39,6 @@
#include "opal/class/opal_list.h"
#include "opal/mca/base/base.h"
#include "opal/mca/hwloc/base/base.h"
#include "opal/mca/paffinity/base/base.h"
#include "opal/mca/maffinity/base/base.h"
#include "opal/runtime/opal_progress.h"
#include "opal/threads/threads.h"
#include "opal/util/output.h"
@ -129,8 +127,6 @@ int ompi_mpi_thread_provided = MPI_THREAD_SINGLE;
opal_thread_t *ompi_mpi_main_thread = NULL;
bool ompi_mpi_maffinity_setup = false;
bool ompi_warn_on_fork;
/*
@ -609,213 +605,6 @@ int ompi_mpi_init(int argc, char **argv, int requested, int *provided)
(ompistop.tv_usec - ompistart.tv_usec)));
gettimeofday(&ompistart, NULL);
}
#if OPAL_HAVE_HWLOC
{
hwloc_obj_t node, obj;
hwloc_cpuset_t cpus, nodeset;
bool paffinity_enabled=false;
orte_node_rank_t nrank;
hwloc_obj_type_t target;
unsigned int cache_level = 0;
struct hwloc_topology_support *support;
/* see if we were bound when launched */
if (NULL == getenv("OMPI_MCA_opal_bound_at_launch")) {
/* we were not bound at launch */
if (NULL != opal_hwloc_topology) {
support = (struct hwloc_topology_support*)hwloc_topology_get_support(opal_hwloc_topology);
/* get our node object */
node = hwloc_get_root_obj(opal_hwloc_topology);
nodeset = opal_hwloc_base_get_available_cpus(opal_hwloc_topology, node);
/* get our bindings */
cpus = hwloc_bitmap_alloc();
if (hwloc_get_cpubind(opal_hwloc_topology, cpus, HWLOC_CPUBIND_PROCESS) < 0) {
/* we are NOT bound if get_cpubind fails, nor can we be bound - the
* environment does not support it
*/
hwloc_bitmap_free(cpus);
goto MOVEON;
}
/* we are bound if the two cpusets are not equal,
* or if there is only ONE cpu available to us
*/
if (0 != hwloc_bitmap_compare(cpus, nodeset) ||
opal_hwloc_base_single_cpu(nodeset) ||
opal_hwloc_base_single_cpu(cpus)) {
/* someone external set it - indicate it is set
* so that we know
*/
paffinity_enabled = true;
hwloc_bitmap_free(cpus);
} else if (support->cpubind->set_thisproc_cpubind &&
OPAL_BINDING_POLICY_IS_SET(opal_hwloc_binding_policy) &&
OPAL_BIND_TO_NONE != OPAL_GET_BINDING_POLICY(opal_hwloc_binding_policy)) {
/* the system is capable of doing processor affinity, but it
* has not yet been set - see if a slot_list was given
*/
hwloc_bitmap_zero(cpus);
if (OPAL_BIND_TO_CPUSET == OPAL_GET_BINDING_POLICY(opal_hwloc_binding_policy)) {
if (ORTE_SUCCESS != (ret = opal_hwloc_base_slot_list_parse(opal_hwloc_base_slot_list,
opal_hwloc_topology, cpus))) {
error = "Setting processor affinity failed";
hwloc_bitmap_free(cpus);
goto error;
}
if (0 > hwloc_set_cpubind(opal_hwloc_topology, cpus, 0)) {
error = "Setting processor affinity failed";
hwloc_bitmap_free(cpus);
goto error;
}
/* try to find a level and index for this location */
opal_hwloc_base_get_level_and_index(cpus, &orte_process_info.bind_level, &orte_process_info.bind_idx);
/* cleanup */
hwloc_bitmap_free(cpus);
paffinity_enabled = true;
} else {
/* cleanup */
hwloc_bitmap_free(cpus);
/* get the node rank */
if (ORTE_NODE_RANK_INVALID == (nrank = orte_ess.get_node_rank(ORTE_PROC_MY_NAME))) {
/* this is not an error - could be due to being
* direct launched - so just ignore and leave
* us unbound
*/
goto MOVEON;
}
/* if the binding policy is hwthread, then we bind to the nrank-th
* hwthread on this node
*/
if (OPAL_BIND_TO_HWTHREAD == OPAL_GET_BINDING_POLICY(opal_hwloc_binding_policy)) {
if (NULL == (obj = opal_hwloc_base_get_obj_by_type(opal_hwloc_topology, HWLOC_OBJ_PU,
0, nrank, OPAL_HWLOC_LOGICAL))) {
ret = OMPI_ERR_NOT_FOUND;
error = "Getting hwthread object";
goto error;
}
cpus = opal_hwloc_base_get_available_cpus(opal_hwloc_topology, obj);
if (0 > hwloc_set_cpubind(opal_hwloc_topology, cpus, 0)) {
ret = OMPI_ERROR;
error = "Setting processor affinity failed";
goto error;
}
orte_process_info.bind_level = OPAL_HWLOC_L1CACHE_LEVEL;
orte_process_info.bind_idx = nrank;
} else if (OPAL_BIND_TO_CORE == OPAL_GET_BINDING_POLICY(opal_hwloc_binding_policy)) {
/* if the binding policy is core, then we bind to the nrank-th
* core on this node
*/
if (NULL == (obj = opal_hwloc_base_get_obj_by_type(opal_hwloc_topology, HWLOC_OBJ_CORE,
0, nrank, OPAL_HWLOC_LOGICAL))) {
ret = OMPI_ERR_NOT_FOUND;
error = "Getting core object";
goto error;
}
cpus = opal_hwloc_base_get_available_cpus(opal_hwloc_topology, obj);
if (0 > hwloc_set_cpubind(opal_hwloc_topology, cpus, 0)) {
error = "Setting processor affinity failed";
ret = OMPI_ERROR;
goto error;
}
orte_process_info.bind_level = OPAL_HWLOC_CORE_LEVEL;
orte_process_info.bind_idx = nrank;
} else {
/* for all higher binding policies, we bind to the specified
* object that the nrank-th core belongs to
*/
if (NULL == (obj = opal_hwloc_base_get_obj_by_type(opal_hwloc_topology, HWLOC_OBJ_CORE,
0, nrank, OPAL_HWLOC_LOGICAL))) {
ret = OMPI_ERR_NOT_FOUND;
error = "Getting core object";
goto error;
}
if (OPAL_BIND_TO_L1CACHE == OPAL_GET_BINDING_POLICY(opal_hwloc_binding_policy)) {
target = HWLOC_OBJ_CACHE;
cache_level = 1;
orte_process_info.bind_level = OPAL_HWLOC_L1CACHE_LEVEL;
} else if (OPAL_BIND_TO_L2CACHE == OPAL_GET_BINDING_POLICY(opal_hwloc_binding_policy)) {
target = HWLOC_OBJ_CACHE;
cache_level = 2;
orte_process_info.bind_level = OPAL_HWLOC_L2CACHE_LEVEL;
} else if (OPAL_BIND_TO_L3CACHE == OPAL_GET_BINDING_POLICY(opal_hwloc_binding_policy)) {
target = HWLOC_OBJ_CACHE;
cache_level = 3;
orte_process_info.bind_level = OPAL_HWLOC_L3CACHE_LEVEL;
} else if (OPAL_BIND_TO_SOCKET == OPAL_GET_BINDING_POLICY(opal_hwloc_binding_policy)) {
target = HWLOC_OBJ_SOCKET;
orte_process_info.bind_level = OPAL_HWLOC_SOCKET_LEVEL;
} else if (OPAL_BIND_TO_NUMA == OPAL_GET_BINDING_POLICY(opal_hwloc_binding_policy)) {
target = HWLOC_OBJ_NODE;
orte_process_info.bind_level = OPAL_HWLOC_NUMA_LEVEL;
} else {
ret = OMPI_ERR_NOT_FOUND;
error = "Binding policy not known";
goto error;
}
for (obj = obj->parent; NULL != obj; obj = obj->parent) {
if (target == obj->type) {
if (HWLOC_OBJ_CACHE == target && cache_level != obj->attr->cache.depth) {
continue;
}
/* this is the place! */
cpus = opal_hwloc_base_get_available_cpus(opal_hwloc_topology, obj);
if (0 > hwloc_set_cpubind(opal_hwloc_topology, cpus, 0)) {
ret = OMPI_ERROR;
error = "Setting processor affinity failed";
goto error;
}
orte_process_info.bind_idx = opal_hwloc_base_get_obj_idx(opal_hwloc_topology,
obj, OPAL_HWLOC_LOGICAL);
paffinity_enabled = true;
break;
}
}
if (!paffinity_enabled) {
ret = OMPI_ERROR;
error = "Setting processor affinity failed";
goto error;
}
}
paffinity_enabled = true;
}
}
/* If we were able to set processor affinity, try setting up
memory affinity */
if (!opal_maffinity_setup && paffinity_enabled) {
if (OPAL_SUCCESS == opal_maffinity_base_open() &&
OPAL_SUCCESS == opal_maffinity_base_select()) {
opal_maffinity_setup = true;
}
}
}
}
}
MOVEON:
/* get or update our local cpuset - it will get used multiple
* times, so it's more efficient to keep a global copy
*/
opal_hwloc_base_get_local_cpuset();
/* report bindings, if requested */
if (opal_hwloc_report_bindings) {
char bindings[64];
hwloc_obj_t root;
hwloc_cpuset_t cpus;
/* get the root object for this node */
root = hwloc_get_root_obj(opal_hwloc_topology);
cpus = opal_hwloc_base_get_available_cpus(opal_hwloc_topology, root);
/* we are not bound if this equals our cpuset */
if (0 == hwloc_bitmap_compare(cpus, opal_hwloc_my_cpuset)) {
opal_output(0, "%s is not bound",
ORTE_NAME_PRINT(ORTE_PROC_MY_NAME));
} else {
hwloc_bitmap_list_snprintf(bindings, 64, opal_hwloc_my_cpuset);
opal_output(0, "%s is bound to cpus %s",
ORTE_NAME_PRINT(ORTE_PROC_MY_NAME),
bindings);
}
}
#endif
/* select buffered send allocator component to be used */
ret=mca_pml_base_bsend_init(OMPI_ENABLE_THREAD_MULTIPLE);

37
ompi/tools/ompi_info/components.c
View File

@ -9,8 +9,8 @@
* University of Stuttgart. All rights reserved.
* Copyright (c) 2004-2005 The Regents of the University of California.
* All rights reserved.
* Copyright (c) 2006-2010 Cisco Systems, Inc. All rights reserved.
* Copyright (c) 2010-2011 Los Alamos National Security, LLC.
* Copyright (c) 2006-2012 Cisco Systems, Inc. All rights reserved.
* Copyright (c) 2010-2012 Los Alamos National Security, LLC.
* All rights reserved.
* Copyright (c) 2011 University of Houston. All rights reserved.
* $COPYRIGHT$
@ -35,14 +35,8 @@
#include "opal/mca/base/base.h"
#include "opal/mca/backtrace/backtrace.h"
#include "opal/mca/backtrace/base/base.h"
#include "opal/mca/paffinity/paffinity.h"
#include "opal/mca/paffinity/base/base.h"
#include "opal/mca/carto/carto.h"
#include "opal/mca/carto/base/base.h"
#include "opal/mca/shmem/shmem.h"
#include "opal/mca/shmem/base/base.h"
#include "opal/mca/maffinity/maffinity.h"
#include "opal/mca/maffinity/base/base.h"
#include "opal/mca/memory/memory.h"
#include "opal/mca/memory/base/base.h"
#include "opal/mca/memchecker/memchecker.h"
@ -308,22 +302,6 @@ void ompi_info_open_components(void)
map->components = &opal_memchecker_base_components_opened;
opal_pointer_array_add(&component_map, map);
if (OPAL_SUCCESS != opal_paffinity_base_open()) {
goto error;
}
map = OBJ_NEW(ompi_info_component_map_t);
map->type = strdup("paffinity");
map->components = &opal_paffinity_base_components_opened;
opal_pointer_array_add(&component_map, map);
if (OPAL_SUCCESS != opal_carto_base_open()) {
goto error;
}
map = OBJ_NEW(ompi_info_component_map_t);
map->type = strdup("carto");
map->components = &opal_carto_base_components_opened;
opal_pointer_array_add(&component_map, map);
if (OPAL_SUCCESS != opal_shmem_base_open()) {
goto error;
}
@ -332,14 +310,6 @@ void ompi_info_open_components(void)
map->components = &opal_shmem_base_components_opened;
opal_pointer_array_add(&component_map, map);
if (OPAL_SUCCESS != opal_maffinity_base_open()) {
goto error;
}
map = OBJ_NEW(ompi_info_component_map_t);
map->type = strdup("maffinity");
map->components = &opal_maffinity_base_components_opened;
opal_pointer_array_add(&component_map, map);
#if OPAL_HAVE_HWLOC
if (OPAL_SUCCESS != opal_hwloc_base_open()) {
goto error;
@ -804,9 +774,6 @@ void ompi_info_close_components()
(void) opal_backtrace_base_close();
(void) opal_memory_base_close();
(void) opal_memchecker_base_close();
(void) opal_paffinity_base_close();
(void) opal_carto_base_close();
(void) opal_maffinity_base_close();
(void) opal_timer_base_close();
#if OPAL_HAVE_HWLOC
(void) opal_hwloc_base_close();

3
ompi/tools/ompi_info/ompi_info.c
View File

@ -210,10 +210,7 @@ int main(int argc, char *argv[])
opal_pointer_array_add(&mca_types, "backtrace");
opal_pointer_array_add(&mca_types, "memchecker");
opal_pointer_array_add(&mca_types, "memory");
opal_pointer_array_add(&mca_types, "paffinity");
opal_pointer_array_add(&mca_types, "carto");
opal_pointer_array_add(&mca_types, "shmem");
opal_pointer_array_add(&mca_types, "maffinity");
opal_pointer_array_add(&mca_types, "timer");
opal_pointer_array_add(&mca_types, "installdirs");
opal_pointer_array_add(&mca_types, "hwloc");

3
opal/include/opal/constants.h
View File

@ -70,7 +70,8 @@ enum {
OPAL_ERR_SLOT_LIST_RANGE = (OPAL_ERR_BASE - 41),
OPAL_ERR_NETWORK_NOT_PARSEABLE = (OPAL_ERR_BASE - 42),
OPAL_ERR_SILENT = (OPAL_ERR_BASE - 43),
OPAL_ERR_NOT_INITIALIZED = (OPAL_ERR_BASE - 44)
OPAL_ERR_NOT_INITIALIZED = (OPAL_ERR_BASE - 44),
OPAL_ERR_NOT_BOUND = (OPAL_ERR_BASE - 45)
};
#define OPAL_ERR_MAX (OPAL_ERR_BASE - 100)

37
opal/mca/carto/Makefile.am
View File

@ -1,37 +0,0 @@
#
# Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
# University Research and Technology
# Corporation. All rights reserved.
# Copyright (c) 2004-2005 The University of Tennessee and The University
# of Tennessee Research Foundation. All rights
# reserved.
# Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
# University of Stuttgart. All rights reserved.
# Copyright (c) 2004-2005 The Regents of the University of California.
# All rights reserved.
# Copyright (c) 2007-2010 Cisco Systems, Inc. All rights reserved.
# $COPYRIGHT$
#
# Additional copyrights may follow
#
# $HEADER$
#
# main library setup
noinst_LTLIBRARIES = libmca_carto.la
libmca_carto_la_SOURCES =
# local files
headers = carto.h
libmca_carto_la_SOURCES += $(headers)
# Conditionally install the header files
if WANT_INSTALL_HEADERS
opaldir = $(includedir)/openmpi/$(subdir)
nobase_opal_HEADERS = $(headers)
endif
include base/Makefile.am
distclean-local:
rm -f base/static-components.h

0
opal/mca/carto/auto_detect/.windows
View File

47
opal/mca/carto/auto_detect/Makefile.am
View File

@ -1,47 +0,0 @@
#
# Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
# University Research and Technology
# Corporation. All rights reserved.
# Copyright (c) 2004-2005 The University of Tennessee and The University
# of Tennessee Research Foundation. All rights
# reserved.
# Copyright (c) 2004-2009 High Performance Computing Center Stuttgart,
# University of Stuttgart. All rights reserved.
# Copyright (c) 2004-2005 The Regents of the University of California.
# All rights reserved.
# Copyright (c) 2007-2010 Cisco Systems, Inc. All rights reserved.
# $COPYRIGHT$
#
# Additional copyrights may follow
#
# $HEADER$
#
EXTRA_DIST = .windows
sources = \
carto_auto_detect.h \
carto_auto_detect_component.c \
carto_auto_detect_module.c
# Make the output library in this directory, and name it either
# mca_<type>_<name>.la (for DSO builds) or libmca_<type>_<name>.la
# (for static builds).
if MCA_BUILD_opal_carto_auto_detect_DSO
component_noinst =
component_install = mca_carto_auto_detect.la
else
component_noinst = libmca_carto_auto_detect.la
component_install =
endif
mcacomponentdir = $(pkglibdir)
mcacomponent_LTLIBRARIES = $(component_install)
mca_carto_auto_detect_la_SOURCES = $(sources)
mca_carto_auto_detect_la_LDFLAGS = -module -avoid-version
noinst_LTLIBRARIES = $(component_noinst)
libmca_carto_auto_detect_la_SOURCES =$(sources)
libmca_carto_auto_detect_la_LDFLAGS = -module -avoid-version

58
opal/mca/carto/auto_detect/carto_auto_detect.h
View File

@ -1,58 +0,0 @@
/*
* Copyright (c) 2004-2008 The Trustees of Indiana University and Indiana
* University Research and Technology
* Corporation. All rights reserved.
* Copyright (c) 2004-2005 The University of Tennessee and The University
* of Tennessee Research Foundation. All rights
* reserved.
* Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
* University of Stuttgart. All rights reserved.
* Copyright (c) 2004-2005 The Regents of the University of California.
* All rights reserved.
* Copyright (c) 2006-2007 Cisco Systems, Inc. All rights reserved.
*
* $COPYRIGHT$
*
* Additional copyrights may follow
*
* $HEADER$
*/
/**
* @file
*
* The auto detect component discover automaticly the structure
* of the host.
*/
#ifndef MCA_CARTO_AUTO_DETECT_H
#define MCA_CARTO_AUTO_DETECT_H
#include "opal_config.h"
#include "opal/mca/mca.h"
#include "opal/mca/carto/carto.h"
BEGIN_C_DECLS
/**
* Globally exported variable
*/
OPAL_DECLSPEC extern const opal_carto_base_component_2_0_0_t
mca_carto_auto_detect_component;
/**
* carto query API function
*
* Query function for carto components. Simply returns a priority
* to rank it against other available carto components (assumedly,
* only one component will be available per platform, but it's
* possible that there could be more than one available).
*/
int opal_carto_auto_detect_component_query(mca_base_module_t **module, int *priority);
END_C_DECLS
#endif /* MCA_CARTO_FILE_EXPORT_H */

83
opal/mca/carto/auto_detect/carto_auto_detect_component.c
View File

@ -1,83 +0,0 @@
/*
* Copyright (c) 2004-2008 The Trustees of Indiana University and Indiana
* University Research and Technology
* Corporation. All rights reserved.
* Copyright (c) 2004-2005 The University of Tennessee and The University
* of Tennessee Research Foundation. All rights
* reserved.
* Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
* University of Stuttgart. All rights reserved.
* Copyright (c) 2004-2005 The Regents of the University of California.
* All rights reserved.
* Copyright (c) 2007 Cisco Systems, Inc. All rights reserved.
* $COPYRIGHT$
*
* Additional copyrights may follow
*
* $HEADER$
*
* These symbols are in a file by themselves to provide nice linker
* semantics. Since linkers generally pull in symbols by object
* files, keeping these symbols as the only symbols in this file
* prevents utility programs such as "ompi_info" from having to import
* entire components just to query their version and parameters.
*/
#include "opal_config.h"
#include "opal/constants.h"
#include "opal/mca/carto/carto.h"
#include "carto_auto_detect.h"
/*
* Public string showing the carto ompi_file component version number
*/
const char *opal_carto_auto_detect_component_version_string =
"OPAL auto detect carto MCA component version " OPAL_VERSION;
/*
* Local function
*/
static int auto_detect_open(void);
/*
* Instantiate the public struct with all of our public information
* and pointers to our public functions in it
*/
const opal_carto_base_component_2_0_0_t mca_carto_auto_detect_component = {
/* First, the mca_component_t struct containing meta information
about the component itself */
{
OPAL_CARTO_BASE_VERSION_2_0_0,
/* Component name and version */
"auto_detect",
OPAL_MAJOR_VERSION,
OPAL_MINOR_VERSION,
OPAL_RELEASE_VERSION,
/* Component open and close functions */
auto_detect_open,
NULL,
opal_carto_auto_detect_component_query
},
{
/* The component is checkpoint ready */
MCA_BASE_METADATA_PARAM_CHECKPOINT
}
};
static int auto_detect_open(void)
{
mca_base_param_reg_int(&mca_carto_auto_detect_component.base_version,
"priority",
"Priority of the auto_detect carto component",
false, false, 11, NULL);
return OPAL_SUCCESS;
}

107
opal/mca/carto/auto_detect/carto_auto_detect_module.c
View File

@ -1,107 +0,0 @@
/*
* Copyright (c) 2004-2008 The Trustees of Indiana University and Indiana
* University Research and Technology
* Corporation. All rights reserved.
* Copyright (c) 2004-2005 The University of Tennessee and The University
* of Tennessee Research Foundation. All rights
* reserved.
* Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
* University of Stuttgart. All rights reserved.
* Copyright (c) 2004-2005 The Regents of the University of California.
* All rights reserved.
* Copyright (c) 2006-2007 Cisco Systems, Inc. All rights reserved.
*
* $COPYRIGHT$
*
* Additional copyrights may follow
*
* $HEADER$
*/
#include "opal_config.h"
/* This component will only be compiled on File, where we are
guaranteed to have <unistd.h> and friends */
#include <stdio.h>
#ifdef HAVE_UNISTD_H
#include <unistd.h>
#endif /* HAVE_UNISTD_H */
#ifdef HAVE_STDLIB_H
#include <stdlib.h>
#endif /* HAVE_STDLIB_H */
#ifdef HAVE_STRING_H
#include <string.h>
#endif /* HAVE_STRING_H */
#include "opal/constants.h"
#include "opal/mca/base/mca_base_param.h"
#include "opal/mca/carto/carto.h"
#include "opal/mca/carto/base/base.h"
#include "opal/mca/carto/base/carto_base_graph.h"
#include "carto_auto_detect.h"
static int opal_carto_auto_detect_init(void);
static int opal_carto_auto_detect_finalize(void);
/*
* Auto detect carto module
*/
static const opal_carto_base_module_1_0_0_t loc_module = {
opal_carto_auto_detect_init,
opal_carto_base_graph_get_host_graph_fn,
opal_carto_base_free_graph_fn,
opal_carto_base_get_nodes_distance_fn,
opal_carto_base_graph_spf_fn,
opal_carto_base_graph_find_node_fn,
opal_carto_auto_detect_finalize,
};
int opal_carto_auto_detect_component_query(mca_base_module_t **module, int *priority)
{
int param;
param = mca_base_param_find("carto", "auto_detect", "priority");
mca_base_param_lookup_int(param, priority);
*module = (mca_base_module_t *)&loc_module;
return OPAL_SUCCESS;
}
/**
* Init the auto detect module. this function build an empty
* carto grap and does nothing more. to comleate this module,
* this function should read the system files and fill the
* carto graph
*
* @return int success or error
*/
static int opal_carto_auto_detect_init(void)
{
/* create an empty graph */
if (NULL == opal_carto_base_common_host_graph) {
opal_carto_base_graph_create_fn(&opal_carto_base_common_host_graph);
}
return OPAL_SUCCESS;
}
/**
* Cleans the auto detect module.
*
* @return int success or error
*/
static int opal_carto_auto_detect_finalize(void)
{
/* free the host cartography graph. */
if (NULL != opal_carto_base_common_host_graph) {
opal_carto_base_free_graph_fn(opal_carto_base_common_host_graph);
}
return OPAL_SUCCESS;
}

29
opal/mca/carto/base/Makefile.am
View File

@ -1,29 +0,0 @@
#
# Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
# University Research and Technology
# Corporation. All rights reserved.
# Copyright (c) 2004-2005 The University of Tennessee and The University
# of Tennessee Research Foundation. All rights
# reserved.
# Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
# University of Stuttgart. All rights reserved.
# Copyright (c) 2004-2005 The Regents of the University of California.
# All rights reserved.
# Copyright (c) 2007 Cisco Systems, Inc. All rights reserved.
# $COPYRIGHT$
#
# Additional copyrights may follow
#
# $HEADER$
#
headers += \
base/base.h \
base/carto_base_graph.h
libmca_carto_la_SOURCES += \
base/carto_base_close.c \
base/carto_base_select.c \
base/carto_base_graph.c \
base/carto_base_wrapers.c \
base/carto_base_open.c

202
opal/mca/carto/base/base.h
View File

@ -1,202 +0,0 @@
/*
* Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
* University Research and Technology
* Corporation. All rights reserved.
* Copyright (c) 2004-2006 The University of Tennessee and The University
* of Tennessee Research Foundation. All rights
* reserved.
* Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
* University of Stuttgart. All rights reserved.
* Copyright (c) 2004-2005 The Regents of the University of California.
* All rights reserved.
* Copyright (c) 2007 Cisco Systems, Inc. All rights reserved.
* $COPYRIGHT$
*
* Additional copyrights may follow
*
* $HEADER$
*
*/
#ifndef OPAL_CARTO_BASE_H
#define OPAL_CARTO_BASE_H
#include "opal_config.h"
#include "opal/mca/carto/carto.h"
/*
* Global functions for MCA overall carto open and close
*/
BEGIN_C_DECLS
/**
* Initialize the carto MCA framework
*
* @retval OPAL_SUCCESS Upon success
* @retval OPAL_ERROR Upon failure
*
* This must be the first function invoked in the carto MCA
* framework. It initializes the carto MCA framework, finds
* and opens carto components, etc.
*
* This function is invoked during opal_init().
*
* This function fills in the internal global variable
* opal_carto_base_components_opened, which is a list of all
* carto components that were successfully opened. This
* variable should \em only be used by other carto base
* functions -- it is not considered a public interface member --
* and is only mentioned here for completeness.
*/
OPAL_DECLSPEC int opal_carto_base_open(void);
/**
* Select an available component.
*
* @return OPAL_SUCCESS Upon success.
* @return OPAL_NOT_FOUND If no component can be selected.
* @return OPAL_ERROR Upon other failure.
*
* This function invokes the selection process for carto components,
* which works as follows:
*
* - If the \em carto MCA parameter is not specified, the
* selection set is all available carto components.
* - If the \em carto MCA parameter is specified, the
* selection set is just that component.
* - All components in the selection set are queried to see if
* they want to run. All components that want to run are ranked
* by their priority and the highest priority component is
* selected. All non-selected components have their "close"
* function invoked to let them know that they were not selected.
* - The selected component will have its "init" function invoked to
* let it know that it was selected.
*
* If we fall through this entire process and no component is
* selected, then return OPAL_NOT_FOUND (this is not a fatal
* error).
*
* At the end of this process, we'll either have a single
* component that is selected and initialized, or no component was
* selected. If no component was selected, subsequent invocation
* of the carto wrapper functions will return an error.
*/
OPAL_DECLSPEC int opal_carto_base_select(void);
/**
* Shut down the carto MCA framework.
*
* @retval OPAL_SUCCESS Always
*
* This function shuts down everything in the carto MCA
* framework, and is called during opal_finalize().
*
* It must be the last function invoked on the carto MCA
* framework.
*/
OPAL_DECLSPEC int opal_carto_base_close(void);
/**
* Indication of whether a component was successfully selected or
* not
*/
OPAL_DECLSPEC extern bool opal_carto_base_selected;
/**
* Global component struct for the selected component
*/
OPAL_DECLSPEC extern const opal_carto_base_component_2_0_0_t
*opal_carto_base_component;
/**
* Global module struct for the selected module
*/
OPAL_DECLSPEC extern const opal_carto_base_module_1_0_0_t
*opal_carto_base_module;
/**
* Indicator as to whether the list of opened carto components
* is valid or not.
*/
OPAL_DECLSPEC extern bool opal_carto_base_components_opened_valid;
/**
* List of all opened components; created when the carto
* framework is initialized and destroyed when we reduce the list
* to all available carto components.
*/
OPAL_DECLSPEC extern opal_list_t opal_carto_base_components_opened;
/**
* Get the local host graph. you can reduce the graph for only
* the nodes that interst you using the node type.
*
* @param graph
* @param graph_type
*
* @return OPAL_DECLSPEC int
*/
OPAL_DECLSPEC int opal_carto_base_get_host_graph(opal_carto_graph_t **graph, const char *graph_type);
/**
* Frre a graph
*
* @param graph
*
* @return OPAL_DECLSPEC void
*/
OPAL_DECLSPEC void opal_carto_base_free_graph(opal_carto_graph_t *graph);
/**
* Get the distance (weight) from a start node to all other
* nodes. you can reduce the list to the list to the node types
* that intersts you.
*
* @param graph
* @param start
* @param node_type
* @param distance_
*
* @return OPAL_DECLSPEC int
*/
OPAL_DECLSPEC int opal_carto_base_get_nodes_distance(opal_carto_graph_t *graph, opal_carto_base_node_t *start, const char *node_type, opal_value_array_t *distance_);
/**
* find the distance between two nodes.
*
* @param graph
* @param start
* @param end
*
* @return OPAL_DECLSPEC uint32_t
*/
OPAL_DECLSPEC uint32_t opal_carto_base_spf(opal_carto_graph_t *graph,opal_carto_base_node_t *start, opal_carto_base_node_t *end);
/**
* Find a node in the graph
*
* @param graph
* @param node_name
*
* @return OPAL_DECLSPEC opal_carto_base_node_t
*/
OPAL_DECLSPEC opal_carto_base_node_t *opal_carto_base_find_node(opal_carto_graph_t *graph, const char *node_name);
/**
* Debugging output stream
*/
OPAL_DECLSPEC extern int opal_carto_base_output;
OPAL_DECLSPEC extern opal_carto_graph_t *opal_carto_base_common_host_graph;
OPAL_DECLSPEC extern opal_carto_base_module_1_0_0_t opal_carto_default_module;
END_C_DECLS
#endif /* OPAL_BASE_CARTO_H */

50
opal/mca/carto/base/carto_base_close.c
View File

@ -1,50 +0,0 @@
/*
* Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
* University Research and Technology
* Corporation. All rights reserved.
* Copyright (c) 2004-2006 The University of Tennessee and The University
* of Tennessee Research Foundation. All rights
* reserved.
* Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
* University of Stuttgart. All rights reserved.
* Copyright (c) 2004-2005 The Regents of the University of California.
* All rights reserved.
* Copyright (c) 2007 Cisco Systems, Inc. All rights reserved.
* $COPYRIGHT$
*
* Additional copyrights may follow
*
* $HEADER$
*/
#include "opal_config.h"
#include "opal/constants.h"
#include "opal/mca/mca.h"
#include "opal/mca/base/base.h"
#include "opal/mca/carto/carto.h"
#include "opal/mca/carto/base/base.h"
int opal_carto_base_close(void)
{
/* If there is a selected carto module, finalize it */
if (NULL != opal_carto_base_module &&
NULL != opal_carto_base_module->carto_module_finalize) {
opal_carto_base_module->carto_module_finalize();
}
/* Close all components that are still open (this should only
happen during ompi_info). */
if (opal_carto_base_components_opened_valid) {
mca_base_components_close(opal_carto_base_output,
&opal_carto_base_components_opened, NULL);
OBJ_DESTRUCT(&opal_carto_base_components_opened);
opal_carto_base_components_opened_valid = false;
}
/* All done */
return OPAL_SUCCESS;
}

382
opal/mca/carto/base/carto_base_graph.c
View File

@ -1,382 +0,0 @@
/*
* Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
* University Research and Technology
* Corporation. All rights reserved.
* Copyright (c) 2004-2005 The University of Tennessee and The University
* of Tennessee Research Foundation. All rights
* reserved.
* Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
* University of Stuttgart. All rights reserved.
* Copyright (c) 2004-2005 The Regents of the University of California.
* All rights reserved.
# Copyright (c) 2007 Cisco Systems, Inc. All rights reserved.
* $COPYRIGHT$
*
* Additional copyrights may follow
*
* $HEADER$
*/
/**
* @file
*
* This file is an implementation of the carto graph base on the graph class.
* This file is common to all the carto components.
*/
#include "opal_config.h"
#include "opal/class/opal_graph.h"
#include "opal/mca/carto/carto.h"
#include "opal/mca/carto/base/carto_base_graph.h"
/*
* Globals
*/
static void opal_carto_base_free_node(void *node);
static void opal_carto_base_copy_nodes(void **dst, void *src);
static void *opal_carto_base_alloc_node(void);
static int opal_carto_compare_nodes(void *node1, void *node2);
static char* opal_carto_print_node(void* node);
/*
* Functions
*/
/**
* A function to print a node. this function allocates buffer
* and prints in it the node type and the node name. this
* function will be assign to the print_vertex function pointer
* when building a new node.
*
* @param node The node we want to print
*
* @return char* the string to print.
*/
static char* opal_carto_print_node(void* node)
{
char *print_str;
char cpu_str[] = "(CPU)";
opal_carto_base_node_t *tmp_node = (opal_carto_base_node_t *)node;
if (false == tmp_node->is_cpu) {
cpu_str[0] = 0;
}
asprintf(&print_str,"%s %5s -%s", tmp_node->node_type, cpu_str, tmp_node->node_name);
return print_str;
}
/**
* A function to free node. this function will be assigned to
* the free_vertex_data pointer
*
* @param node the node to free.
*/
static void opal_carto_base_free_node(void *node)
{
opal_carto_base_node_t *tmp_node = (opal_carto_base_node_t *)node;
/* free the node name string */
free(tmp_node->node_name);
free(tmp_node->node_type);
/* free the node */
free(tmp_node);
}
/**
* A function to copy a node. this function will be assign in
* the copy_vertex_data function pointer.
*
* @param dst the destination node.
* @param src the source node.
*/
static void opal_carto_base_copy_nodes(void **dst, void *src)
{
opal_carto_base_node_t *src_node = (opal_carto_base_node_t *)src,
*dst_node = (opal_carto_base_node_t *)*dst;
/* duplicate the node name */
dst_node->node_name = strdup(src_node->node_name);
/* copy the node type */
dst_node->node_type = strdup(src_node->node_type);
dst_node->is_cpu = src_node->is_cpu;
/* If the nodes vertex was copied, get the copied vertex */
dst_node->vertex = src_node->vertex->sibling;
}
/**
* Allocate memory for node. this function will be assign to the
* alloc_vertex_data function pointer.
*
* @return void*
*/
static void *opal_carto_base_alloc_node(void)
{
opal_carto_base_node_t *node;
/* allocate memory fore node */
node = (opal_carto_base_node_t *)malloc(sizeof(opal_carto_base_node_t));
/*Init the node fields */
node->node_name = NULL;
node->node_type = NULL;
node->is_cpu = false;
node->vertex = NULL;
return (void*)node;
}
/**
* Compare two nodes. in our case we needs to compare only the
* node name. this function will be assign to the compare vertex
* data function pointer.
*
* @param node1
* @param node2
*
* @return int 0-equal, 1-the first is bigger, -1-the first is
* smaller.
*/
static int opal_carto_compare_nodes(void *node1, void *node2)
{
opal_carto_base_node_t *tmp_node1 = (opal_carto_base_node_t *)node1,
*tmp_node2 = (opal_carto_base_node_t *)node2;
/* use str compare to compare the node names */
return strcmp(tmp_node1->node_name, tmp_node2->node_name);
}
/**
* Create new carto graph.
*
* @param graph an empty graph pointer
*/
void opal_carto_base_graph_create_fn(opal_carto_graph_t **graph)
{
*graph = (opal_carto_graph_t *)OBJ_NEW(opal_graph_t);
}
/**
* Add a node to carto graph.
*
* @param graph the carto graph to add the node to.
* @param node the node to add.
*/
void opal_carto_base_graph_add_node_fn(opal_carto_graph_t *graph, opal_carto_base_node_t *node)
{
/* construct new vertex */
node->vertex = OBJ_NEW(opal_graph_vertex_t);
/* assign the node as the vertex data */
node->vertex->vertex_data = (void *)node;
/* assign the vertex function pointers */
node->vertex->free_vertex_data = opal_carto_base_free_node;
node->vertex->copy_vertex_data = opal_carto_base_copy_nodes;
node->vertex->alloc_vertex_data = opal_carto_base_alloc_node;
node->vertex->compare_vertex = opal_carto_compare_nodes;
node->vertex->print_vertex = opal_carto_print_node;
/* add the new node to the carto graph by adding the nodes vertex to the graph */
opal_graph_add_vertex((opal_graph_t *)graph, node->vertex);
}
/**
* Free a carto graph
* @param graph the graph we want to free.
*/
void opal_carto_base_free_graph_fn(opal_carto_graph_t *graph)
{
int i, graph_order;
opal_carto_base_node_t *node;
opal_pointer_array_t *graph_vertices;
opal_graph_vertex_t *vertex;
graph_vertices = OBJ_NEW(opal_pointer_array_t);
opal_pointer_array_init(graph_vertices, 20, INT_MAX, 20);
/* get all the graph vertices */
graph_order = opal_graph_get_graph_vertices(graph, graph_vertices);
/* for all the vertices in the graph, free the nodes (and distract the vertices) */
for (i = 0; i < graph_order; i++) {
vertex = (opal_graph_vertex_t *)opal_pointer_array_get_item(graph_vertices, i);
node = vertex->vertex_data;
opal_carto_base_free_node((void *)node);
}
OBJ_RELEASE(graph_vertices);
/* destruct the graph */
OBJ_RELEASE(graph);
}
/**
* Connect two nodes by adding an edge to the graph.
*
* @param graph the graph that the nodes belongs to.
* @param start the start node
* @param end the end node
* @param weight the weight of the connection
*
* @return int success or error (if one of the nodes does not
* belong to the graph.
*/
int opal_carto_base_connect_nodes_fn(opal_carto_graph_t *graph, opal_carto_base_node_t *start, opal_carto_base_node_t *end, uint32_t weight)
{
opal_graph_edge_t *edge;
/* construct anew edge */
edge = OBJ_NEW(opal_graph_edge_t);
/* assigne the start and the end nodes vertices to the new edge */
edge->start = start->vertex;
edge->end = end->vertex;
/* assign the weight to the edge */
edge->weight = weight;
/* add the edge to the graph */
return opal_graph_add_edge((opal_graph_t *)graph, edge);
}
/**
* Duplicate a carto graph and reduce the new graph to contain
* nodes from a ceratin type(s)
*
* @param destination The new graph.
* @param source the original graph.
* @param node_type the node type(s) that the new graph will
* include.
*/
void opal_carto_base_duplicate_graph_fn(opal_carto_graph_t **destination, const opal_carto_graph_t *source, const char *node_type)
{
opal_pointer_array_t *vertices;
int i, graph_order;
opal_carto_base_node_t *node;
opal_graph_vertex_t *vertex;
/* duplicate the graph */
opal_graph_duplicate((opal_graph_t **)destination, (opal_graph_t *)source);
/* if there is no need for reduction, return */
if (NULL == node_type) {
return;
}
vertices = OBJ_NEW(opal_pointer_array_t);
opal_pointer_array_init(vertices, 20, INT_MAX, 20);
/* get all the vertices of the new graph */
graph_order = opal_graph_get_graph_vertices(*destination, vertices);
/* remove all the nodes that are not in the required type */
for (i = 0; i < graph_order; i++ ) {
vertex = (opal_graph_vertex_t *)opal_pointer_array_get_item(vertices, i);
node = vertex->vertex_data;
if (!(0 == strcmp(node_type, node->node_type) || node->is_cpu)) {
opal_graph_remove_vertex(*destination, vertex);
}
}
/* free the vertices array */
OBJ_RELEASE(vertices);
}
/**
* opal_carto_base_get_nodes_distance - returns the distance of
* all the nodes from the reference node.
*
* @param graph
* @param reference_node
* @param node_type the type of the nodes in the returned array
* @param dist_array
*
* @return int number of nodes in the returned array.
*/
int opal_carto_base_get_nodes_distance_fn(opal_carto_graph_t *graph, opal_carto_base_node_t *reference_node,
const char *node_type, opal_value_array_t *dist_array)
{
opal_value_array_t *distance_array;
vertex_distance_from_t *vertex_distance;
opal_carto_base_node_t *node;
uint32_t i, graph_order;
int distance_array_size;
opal_carto_node_distance_t node_distance;
distance_array = OBJ_NEW(opal_value_array_t);
opal_value_array_init(distance_array, sizeof(vertex_distance_from_t));
opal_value_array_reserve(distance_array,50);
/* use dijkstra algorithm to receive the distance of all the nodes from the referenced node */
graph_order = opal_graph_dijkstra(graph, reference_node->vertex, distance_array);
/* for all the nodes in the dijkstra array */
for (i = 0, distance_array_size = 0; i < graph_order; i++) {
vertex_distance = opal_value_array_get_item(distance_array, i);
node = vertex_distance->vertex->vertex_data;
/* check if the node is in the correct type */
if (NULL == node_type || 0 == strcmp(node->node_type, node_type)) {
/* assigne the result distance array */
node_distance.node = vertex_distance->vertex->vertex_data;
node_distance.node_distance = vertex_distance->weight;
opal_value_array_append_item(dist_array, (void *)&node_distance);
}
}
/* return the result distance array */
return distance_array_size;
}
/**
* Find the shortest path between two nodes in the graph
*
* @param graph the graph that the nodes belongs to.
* @param node1 first node.
* @param node2 second node.
*
* @return uint32_t he distance between the nodes.
*/
uint32_t opal_carto_base_graph_spf_fn(opal_carto_graph_t *graph, opal_carto_base_node_t *node1, opal_carto_base_node_t *node2)
{
return opal_graph_spf((opal_graph_t *)graph, node1->vertex, node2->vertex);
}
/**
* Find a node in the graph according to its name.
*
* @param graph the graph in which we are searching.
* @param node_name the node name.
*
* @return opal_carto_base_node_t* the node with the name -if
* found or NULL.
*/
opal_carto_base_node_t *opal_carto_base_graph_find_node_fn(opal_carto_graph_t *graph, const char *node_name)
{
opal_carto_base_node_t node;
opal_graph_vertex_t *vertex;
/* build a temporary node */
node.node_name = strdup(node_name);
/**
* find a vertex in the graph. the find_vertex uses the
* compare_vertex_data method. in our case, compare_vertex_data
* is assigned to opal_carto_compare_nodes that compares two
* nodes names.
*/
vertex = opal_graph_find_vertex((opal_graph_t *)graph, (void *)&node);
free(node.node_name);
if (NULL != vertex) {
/* return the fund vertex data (node) */
return vertex->vertex_data;
}
/* if not found, return NULL */
return NULL;
}
/**
* Get the host cartography graph.
*
* @param graph an unallocated pointer to a graph.
* @param graph_type the type of nodes we want the returned
* graph will contain.
*
* @return int success or error
*/
int opal_carto_base_graph_get_host_graph_fn(opal_carto_graph_t **graph, const char *graph_type)
{
/* duplicate the host graph and delete all the relevant nodes */
opal_carto_base_duplicate_graph_fn(graph, opal_carto_base_common_host_graph, graph_type);
return OPAL_SUCCESS;
}

144
opal/mca/carto/base/carto_base_graph.h
View File

@ -1,144 +0,0 @@
/*
* Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
* University Research and Technology
* Corporation. All rights reserved.
* Copyright (c) 2004-2006 The University of Tennessee and The University
* of Tennessee Research Foundation. All rights
* reserved.
* Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
* University of Stuttgart. All rights reserved.
* Copyright (c) 2004-2005 The Regents of the University of California.
* All rights reserved.
* Copyright (c) 2007 Cisco Systems, Inc. All rights reserved.
* $COPYRIGHT$
*
* Additional copyrights may follow
*
* $HEADER$
*
*/
#ifndef OPAL_CARTO_BASE_GRAPH_H
#define OPAL_CARTO_BASE_GRAPH_H
/* THIS FILE CONTAINS THE ACTUAL BASE FUNCTION IMPLEMENTATIONS FOR
* THE CARTO FRAMEWORK - THESE FUNCTIONS ARE TO BE CALLED STRICTLY
* FROM THE VARIOUS COMPONENTS, AS PASSED THROUGH BY THE PUBLIC
* BASE FUNCTIONS
*
* DO -NOT- CALL THESE FUNCTIONS DIRECTLY!
*/
#include "opal_config.h"
#include "opal/mca/carto/carto.h"
BEGIN_C_DECLS
OPAL_DECLSPEC extern opal_carto_graph_t *opal_carto_base_common_host_graph;
/**
* Create new carto graph.
*
* @param graph an empty graph pointer
*/
OPAL_DECLSPEC void opal_carto_base_graph_create_fn(opal_carto_graph_t **graph);
/**
* Add a node to carto graph.
*
* @param graph the carto graph to add the node to.
* @param node the node to add.
*/
OPAL_DECLSPEC void
opal_carto_base_graph_add_node_fn(opal_carto_graph_t *graph, opal_carto_base_node_t *node);
/**
* Free a carto graph
* @param graph the graph we want to free.
*/
OPAL_DECLSPEC void opal_carto_base_free_graph_fn(opal_carto_graph_t *graph);
/**
* Connect two nodes by adding an edge to the graph.
*
* @param graph the graph that the nodes belongs to.
* @param start the start node
* @param end the end node
* @param weight the weight of the connection
*
* @return int success or error (if one of the nodes does not
* belong to the graph.
*/
OPAL_DECLSPEC int
opal_carto_base_connect_nodes_fn(opal_carto_graph_t *graph, opal_carto_base_node_t *start,
opal_carto_base_node_t *end, uint32_t weight);
/**
* Duplicate a carto graph and reduce the new graph to contain
* nodes from a ceratin type(s)
*
* @param destination The new graph.
* @param source the original graph.
* @param node_type the node type(s) that the new graph will
* include.
*/
OPAL_DECLSPEC void
opal_carto_base_duplicate_graph_fn(opal_carto_graph_t **destination, const opal_carto_graph_t *source,
const char *node_type);
/**
* opal_carto_base_get_nodes_distance - returns the distance of
* all the nodes from the reference node.
*
* @param graph
* @param reference_node
* @param node_type the type of the nodes in the returned array
* @param dist_array
*
* @return int number of nodes in the returned array.
*/
OPAL_DECLSPEC int
opal_carto_base_get_nodes_distance_fn(opal_carto_graph_t *graph, opal_carto_base_node_t *reference_node,
const char *node_type, opal_value_array_t *dist_array);
/**
* Find the shortest path between two nodes in the graph
*
* @param graph the graph that the nodes belongs to.
* @param node1 first node.
* @param node2 second node.
*
* @return uint32_t he distance between the nodes.
*/
OPAL_DECLSPEC uint32_t
opal_carto_base_graph_spf_fn(opal_carto_graph_t *graph, opal_carto_base_node_t *node1,
opal_carto_base_node_t *node2);
/**
* Find a node in the graph according to its name.
*
* @param graph the graph in which we are searching.
* @param node_name the node name.
*
* @return opal_carto_base_node_t* the node with the name -if
* found or NULL.
*/
OPAL_DECLSPEC opal_carto_base_node_t
*opal_carto_base_graph_find_node_fn(opal_carto_graph_t *graph, const char *node_name);
/**
* Get the host cartography graph.
*
* @param graph an unallocated pointer to a graph.
* @param graph_type the type of nodes we want the returned
* graph will contain.
*
* @return int success or error
*/
OPAL_DECLSPEC int opal_carto_base_graph_get_host_graph_fn(opal_carto_graph_t **graph, const char * graph_type);
END_C_DECLS
#endif

115
opal/mca/carto/base/carto_base_open.c
View File

@ -1,115 +0,0 @@
/*
* Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
* University Research and Technology
* Corporation. All rights reserved.
* Copyright (c) 2004-2005 The University of Tennessee and The University
* of Tennessee Research Foundation. All rights
* reserved.
* Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
* University of Stuttgart. All rights reserved.
* Copyright (c) 2004-2005 The Regents of the University of California.
* All rights reserved.
* Copyright (c) 2007 Cisco Systems, Inc. All rights reserved.
* $COPYRIGHT$
*
* Additional copyrights may follow
*
* $HEADER$
*/
#include "opal_config.h"
#include "opal/constants.h"
#include "opal/util/output.h"
#include "opal/mca/mca.h"
#include "opal/mca/base/base.h"
#include "opal/mca/base/mca_base_param.h"
#include "opal/mca/carto/carto.h"
#include "opal/mca/carto/base/carto_base_graph.h"
#include "opal/mca/carto/base/base.h"
/*
* The following file was created by configure. It contains extern
* statements and the definition of an array of pointers to each
* component's public mca_base_component_t struct.
*/
#include "opal/mca/carto/base/static-components.h"
/*
* Globals
*/
int opal_carto_base_output = -1;
bool opal_carto_base_components_opened_valid = false;
opal_list_t opal_carto_base_components_opened;
opal_carto_graph_t *opal_carto_base_common_host_graph=NULL;
/*
* default carto module when no components are found
*/
static int opal_carto_default_init(void)
{
/* create an empty graph */
if (NULL == opal_carto_base_common_host_graph) {
opal_carto_base_graph_create_fn(&opal_carto_base_common_host_graph);
}
return OPAL_SUCCESS;
}
static int opal_carto_default_finalize(void)
{
/* free the host cartography graph. */
if (NULL != opal_carto_base_common_host_graph) {
opal_carto_base_free_graph_fn(opal_carto_base_common_host_graph);
}
return OPAL_SUCCESS;
}
opal_carto_base_module_1_0_0_t opal_carto_default_module = {
opal_carto_default_init,
opal_carto_base_graph_get_host_graph_fn,
opal_carto_base_free_graph_fn,
opal_carto_base_get_nodes_distance_fn,
opal_carto_base_graph_spf_fn,
opal_carto_base_graph_find_node_fn,
opal_carto_default_finalize,
};
/*
* Function for finding and opening either all MCA components, or the one
* that was specifically requested via a MCA parameter.
*/
int opal_carto_base_open(void)
{
int value;
/* Debugging / verbose output */
mca_base_param_reg_int_name("carto", "base_verbose",
"Verbosity level of the carto framework",
false, false,
0, &value);
if (0 != value) {
opal_carto_base_output = opal_output_open(NULL);
} else {
opal_carto_base_output = -1;
}
opal_carto_base_components_opened_valid = false;
/* Open up all available components */
if (OPAL_SUCCESS !=
mca_base_components_open("carto", opal_carto_base_output,
mca_carto_base_static_components,
&opal_carto_base_components_opened,
true)) {
return OPAL_ERROR;
}
opal_carto_base_components_opened_valid = true;
/* All done */
return OPAL_SUCCESS;
}

72
opal/mca/carto/base/carto_base_select.c
View File

@ -1,72 +0,0 @@
/*
* Copyright (c) 2004-2008 The Trustees of Indiana University and Indiana
* University Research and Technology
* Corporation. All rights reserved.
* Copyright (c) 2004-2006 The University of Tennessee and The University
* of Tennessee Research Foundation. All rights
* reserved.
* Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
* University of Stuttgart. All rights reserved.
* Copyright (c) 2004-2005 The Regents of the University of California.
* All rights reserved.
* Copyright (c) 2007 Cisco Systems, Inc. All rights reserved.
* $COPYRIGHT$
*
* Additional copyrights may follow
*
* $HEADER$
*/
#include "opal_config.h"
#include "opal/constants.h"
#include "opal/mca/mca.h"
#include "opal/mca/base/base.h"
#include "opal/mca/base/mca_base_param.h"
#include "opal/mca/carto/carto.h"
#include "opal/mca/carto/base/base.h"
/*
* Globals
*/
bool opal_carto_base_selected = false;
const opal_carto_base_component_2_0_0_t *opal_carto_base_component = NULL;
const opal_carto_base_module_1_0_0_t *opal_carto_base_module = NULL;
int opal_carto_base_select(void)
{
int exit_status = OPAL_SUCCESS;
opal_carto_base_component_2_0_0_t *best_component = NULL;
opal_carto_base_module_1_0_0_t *best_module = NULL;
/*
* Select the best component
*/
if( OPAL_SUCCESS != mca_base_select("carto", opal_carto_base_output,
&opal_carto_base_components_opened,
(mca_base_module_t **) &best_module,
(mca_base_component_t **) &best_component) ) {
/* This will only happen if no component was selected, so
* use the default module instead
*/
opal_carto_base_module = &opal_carto_default_module;
opal_carto_base_selected = true;
goto cleanup;
}
/* Save the winner */
opal_carto_base_component = best_component;
opal_carto_base_module = best_module;
opal_carto_base_selected = true;
cleanup:
/* Initialize the winner */
if (NULL != opal_carto_base_module) {
exit_status = opal_carto_base_module->carto_module_init();
}
return exit_status;
}

73
opal/mca/carto/base/carto_base_wrapers.c
View File

@ -1,73 +0,0 @@
/*
* Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
* University Research and Technology
* Corporation. All rights reserved.
* Copyright (c) 2004-2006 The University of Tennessee and The University
* of Tennessee Research Foundation. All rights
* reserved.
* Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
* University of Stuttgart. All rights reserved.
* Copyright (c) 2004-2005 The Regents of the University of California.
* All rights reserved.
* Copyright (c) 2007 Cisco Systems, Inc. All rights reserved.
*
* $COPYRIGHT$
*
* Additional copyrights may follow
*
* $HEADER$
*/
#include "opal_config.h"
#include "opal/constants.h"
#include "opal/mca/carto/carto.h"
#include "opal/mca/carto/base/base.h"
int opal_carto_base_get_host_graph(opal_carto_graph_t **graph, const char *graph_type)
{
if (!opal_carto_base_selected) {
return OPAL_ERR_NOT_FOUND;
}
return opal_carto_base_module->get_host_graph(graph, graph_type);
}
void opal_carto_base_free_graph(opal_carto_graph_t *graph)
{
if (!opal_carto_base_selected) {
return ;
}
opal_carto_base_module->free_graph(graph);
}
int opal_carto_base_get_nodes_distance(opal_carto_graph_t *graph, opal_carto_base_node_t *start, const char *node_type, opal_value_array_t *distance_)
{
if (!opal_carto_base_selected) {
return OPAL_ERR_NOT_FOUND;
}
return opal_carto_base_module->get_nodes_distance(graph, start, node_type, distance_);
}
uint32_t opal_carto_base_spf(opal_carto_graph_t *graph,opal_carto_base_node_t *start, opal_carto_base_node_t *end)
{
if (!opal_carto_base_selected) {
return OPAL_ERR_NOT_FOUND;
}
return opal_carto_base_module->spf(graph, start, end);
}
opal_carto_base_node_t *opal_carto_base_find_node(opal_carto_graph_t *graph, const char *node_name)
{
if (!opal_carto_base_selected) {
return NULL;
}
return opal_carto_base_module->find_node(graph, node_name);
}

164
opal/mca/carto/carto.h
View File

@ -1,164 +0,0 @@
/*
* Copyright (c) 2004-2008 The Trustees of Indiana University and Indiana
* University Research and Technology
* Corporation. All rights reserved.
* Copyright (c) 2004-2005 The University of Tennessee and The University
* of Tennessee Research Foundation. All rights
* reserved.
* Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
* University of Stuttgart. All rights reserved.
* Copyright (c) 2004-2005 The Regents of the University of California.
* All rights reserved.
* Copyright (c) 2007 Cisco Systems, Inc. All rights reserved.
* $COPYRIGHT$
*
* Additional copyrights may follow
*
* $HEADER$
*/
/**
* @file
*
* The carto framework suplies an information of the the host structure and connection between the
* host components i.e memory nodes,CPUs, Ethernet port and inifiniband ports.
*/
#ifndef OPAL_CARTO_H
#define OPAL_CARTO_H
#include "opal_config.h"
#include "opal/mca/mca.h"
#include "opal/mca/base/base.h"
#include "opal/class/opal_graph.h"
/**
* type for carto graph
*/
typedef opal_graph_t opal_carto_graph_t;
/* A structure of carto node */
struct opal_carto_base_node_t {
opal_graph_vertex_t *vertex; /* the parent of a node is a graph vertex */
char *node_name;
char *node_type; /* the type of the node */
bool is_cpu;
};
/**
* A definition of carto node type.
*/
typedef struct opal_carto_base_node_t opal_carto_base_node_t;
/**
* A structure of node distance from some other node
*/
struct opal_carto_node_distance_t {
opal_carto_base_node_t *node; /* The node */
uint32_t node_distance; /* and the distance */
};
/**
* A definition of node distance type.
*/
typedef struct opal_carto_node_distance_t opal_carto_node_distance_t;
/**
* Module initialization function. Should return OPAL_SUCCESS.
*/
typedef int (*opal_carto_base_module_init_1_0_0_fn_t)(void);
/**
* Get the local host graph. you can reduce the graph for only
* the nodes that interst you using the node type.
*/
typedef int (*opal_carto_base_get_host_graph_fn_t)
(opal_carto_graph_t **graph, const char *graph_type);
/**
* Frre a graph
*/
typedef void (*opal_carto_base_free_graph_fn_t)
(opal_carto_graph_t *graph);
/**
* Get the distance (weight) from a start node to all other
* nodes. you can reduce the list to the list to the node types
* that intersts you.
*/
typedef int (*opal_carto_base_get_nodes_distance_fn_t)
(opal_carto_graph_t *graph, opal_carto_base_node_t *start, const char *node_type, opal_value_array_t *distance_);
/**
* find the distance between two nodes.
*/
typedef uint32_t (*opal_carto_base_spf_fn_t)
(opal_carto_graph_t *graph,opal_carto_base_node_t *start, opal_carto_base_node_t *end);
/**
* Find a node in the graph
*/
typedef opal_carto_base_node_t *(*opal_carto_base_find_node_fn_t)
(opal_carto_graph_t *graph, const char *node_name);
/**
* Module finalize function. Invoked by the base on the selected
* module when the carto framework is being shut down.
*/
typedef int (*opal_carto_base_module_finalize_fn_t)(void);
/**
* Structure for carto components.
*/
struct opal_carto_base_component_2_0_0_t {
/** MCA base component */
mca_base_component_t base_version;
/** MCA base data */
mca_base_component_data_t base_data;
};
/**
* Convenience typedef
*/
typedef struct opal_carto_base_component_2_0_0_t opal_carto_base_component_2_0_0_t;
typedef struct opal_carto_base_component_2_0_0_t opal_carto_base_component_t;
/**
* Structure for carto modules
*/
struct opal_carto_base_module_1_0_0_t {
/** Module initialization function */
opal_carto_base_module_init_1_0_0_fn_t carto_module_init;
/** Get host graph */
opal_carto_base_get_host_graph_fn_t get_host_graph;
/** free graph */
opal_carto_base_free_graph_fn_t free_graph;
/** Get the distance from one node to all other nodes */
opal_carto_base_get_nodes_distance_fn_t get_nodes_distance;
/** Find the distance between two nodes */
opal_carto_base_spf_fn_t spf;
/** Find a node in the graph */
opal_carto_base_find_node_fn_t find_node;
/** Shut down this module */
opal_carto_base_module_finalize_fn_t carto_module_finalize;
};
/**
* Convenience typedef
*/
typedef struct opal_carto_base_module_1_0_0_t opal_carto_base_module_1_0_0_t;
typedef struct opal_carto_base_module_1_0_0_t opal_carto_base_module_t;
/*
* Macro for use in components that are of type carto
*/
#define OPAL_CARTO_BASE_VERSION_2_0_0 \
MCA_BASE_VERSION_2_0_0, \
"carto", 2, 0, 0
#endif /* OPAL_CARTO_H */

53
opal/mca/carto/file/Makefile.am
View File

@ -1,53 +0,0 @@
#
# Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
# University Research and Technology
# Corporation. All rights reserved.
# Copyright (c) 2004-2005 The University of Tennessee and The University
# of Tennessee Research Foundation. All rights
# reserved.
# Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
# University of Stuttgart. All rights reserved.
# Copyright (c) 2004-2005 The Regents of the University of California.
# All rights reserved.
# Copyright (c) 2007-2010 Cisco Systems, Inc. All rights reserved.
# $COPYRIGHT$
#
# Additional copyrights may follow
#
# $HEADER$
#
AM_LFLAGS = -Pcarto_file_
LEX_OUTPUT_ROOT = lex.carto_file_
dist_pkgdata_DATA = \
help-opal-carto-file.txt
sources = \
carto_file.h \
carto_file_lex.h \
carto_file_lex.l \
carto_file_component.c \
carto_file_module.c
# Make the output library in this directory, and name it either
# mca_<type>_<name>.la (for DSO builds) or libmca_<type>_<name>.la
# (for static builds).
if MCA_BUILD_opal_carto_file_DSO
component_noinst =
component_install = mca_carto_file.la
else
component_noinst = libmca_carto_file.la
component_install =
endif
mcacomponentdir = $(pkglibdir)
mcacomponent_LTLIBRARIES = $(component_install)
mca_carto_file_la_SOURCES = $(sources)
mca_carto_file_la_LDFLAGS = -module -avoid-version
noinst_LTLIBRARIES = $(component_noinst)
libmca_carto_file_la_SOURCES = $(sources)
libmca_carto_file_la_LDFLAGS = -module -avoid-version

115
opal/mca/carto/file/carto_file.h
View File

@ -1,115 +0,0 @@
/*
* Copyright (c) 2004-2008 The Trustees of Indiana University and Indiana
* University Research and Technology
* Corporation. All rights reserved.
* Copyright (c) 2004-2005 The University of Tennessee and The University
* of Tennessee Research Foundation. All rights
* reserved.
* Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
* University of Stuttgart. All rights reserved.
* Copyright (c) 2004-2005 The Regents of the University of California.
* All rights reserved.
* Copyright (c) 2006-2007 Cisco Systems, Inc. All rights reserved.
*
* $COPYRIGHT$
*
* Additional copyrights may follow
*
* $HEADER$
*/
/**
* @file#this is a comment
# Node declaration Node type (Free string) Node name (Free string)
# (Reserve word) (socket is a reserve word (free string)
# for CPU socket)
#=======================================================================
EDGE Memory mem0
EDGE Memory mem1
EDGE Memory mem2
EDGE Memory mem3
#
EDGE socket socket0
EDGE socket socket1
EDGE socket socket2
EDGE socket socket3
#
EDGE Infiniband mthca0
EDGE Infiniband mthca1
#
EDGE Ethernet eth0
EDGE Ethernet eth1
#
#
# Connection decleration From node To node:weight To node:weight ......
# (Reserve word) (declered (declered (declered
# above) above) above)
#===============================================================================================
BRANCH mem0 socket0:0
BRANCH mem3 socket3:0
#
BRANCH socket0 mem0:0 socket1:1 socket2:1 mthca0:1 eth0:1
BRANCH socket1 socket0:1 socket3:1
BRANCH socket2 socket1:1 socket3:1
BRANCH socket3 mem3:0 socket1:1 socket2:1 mthca1:1 eth1:1
#
#
BRANCH mthca0 socket0:1
BRANCH mthca1 socket3:1
#
BRANCH eth0 socket0:1
BRANCH eth1 socket3:1
#Bi-Directional connection
#
BRANCH_BI_DIR socket1 mem1:0
BRANCH_BI_DIR socket2 mem2:0
#
# end of carto file.
*
* The file component uses a cartograpy file to discover the
* host cartography.
*
* An example cartography file:
*
*
*
*
*
*
*/
#ifndef MCA_CARTO_FILE_H
#define MCA_CARTO_FILE_H
#include "opal_config.h"
#include "opal/mca/mca.h"
#include "opal/mca/carto/carto.h"
BEGIN_C_DECLS
extern char *carto_file_path;
/**
* Globally exported variable
*/
OPAL_DECLSPEC extern const opal_carto_base_component_2_0_0_t
mca_carto_file_component;
/**
* carto query API function
*
* Query function for carto components. Simply returns a priority
* to rank it against other available carto components (assumedly,
* only one component will be available per platform, but it's
* possible that there could be more than one available).
*/
int opal_carto_file_component_query(mca_base_module_t **module, int *priority);
END_C_DECLS
#endif /* MCA_CARTO_FILE_EXPORT_H */

100
opal/mca/carto/file/carto_file_component.c
View File

@ -1,100 +0,0 @@
/*
* Copyright (c) 2004-2008 The Trustees of Indiana University and Indiana
* University Research and Technology
* Corporation. All rights reserved.
* Copyright (c) 2004-2005 The University of Tennessee and The University
* of Tennessee Research Foundation. All rights
* reserved.
* Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
* University of Stuttgart. All rights reserved.
* Copyright (c) 2004-2005 The Regents of the University of California.
* All rights reserved.
* Copyright (c) 2007 Cisco Systems, Inc. All rights reserved.
* $COPYRIGHT$
*
* Additional copyrights may follow
*
* $HEADER$
*
* These symbols are in a file by themselves to provide nice linker
* semantics. Since linkers generally pull in symbols by object
* files, keeping these symbols as the only symbols in this file
* prevents utility programs such as "ompi_info" from having to import
* entire components just to query their version and parameters.
*/
#include "opal_config.h"
#include "opal/constants.h"
#include "opal/mca/carto/carto.h"
#include "carto_file.h"
/*
* Public string showing the carto ompi_file component version number
*/
const char *opal_carto_file_component_version_string =
"OPAL file carto MCA component version " OPAL_VERSION;
/*
* Local function
*/
static int file_open(void);
/*
* Instantiate the public struct with all of our public information
* and pointers to our public functions in it
*/
const opal_carto_base_component_2_0_0_t mca_carto_file_component = {
/* First, the mca_component_t struct containing meta information
about the component itself */
{
OPAL_CARTO_BASE_VERSION_2_0_0,
/* Component name and version */
"file",
OPAL_MAJOR_VERSION,
OPAL_MINOR_VERSION,
OPAL_RELEASE_VERSION,
/* Component open and close functions */
file_open,
NULL,
opal_carto_file_component_query
},
{
/* The component is checkpoint ready */
MCA_BASE_METADATA_PARAM_CHECKPOINT
}
};
static int file_open(void)
{
mca_base_param_reg_string(&mca_carto_file_component.base_version,
"path",
"The path to the cartography file",
false, false, NULL, &carto_file_path);
/**
* If the user specified the carto file path (not NULL), use the
* carto file component. The auto detect component is with
* higher priority, so by default it will be chosen.
*/
if (NULL == carto_file_path) {
mca_base_param_reg_int(&mca_carto_file_component.base_version,
"priority",
"Priority of the file carto component",
false, false, 10, NULL);
}else{
mca_base_param_reg_int(&mca_carto_file_component.base_version,
"priority",
"Priority of the file carto component",
false, false, 12, NULL);
}
return OPAL_SUCCESS;
}

68
opal/mca/carto/file/carto_file_lex.h
View File

@ -1,68 +0,0 @@
/* -*- C -*-
*
* Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
* University Research and Technology
* Corporation. All rights reserved.
* Copyright (c) 2004-2005 The University of Tennessee and The University
* of Tennessee Research Foundation. All rights
* reserved.
* Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
* University of Stuttgart. All rights reserved.
* Copyright (c) 2004-2005 The Regents of the University of California.
* All rights reserved.
* $COPYRIGHT$
*
* Additional copyrights may follow
*
* $HEADER$
*/
#ifndef CARTO_FILE_LEX_LEX_H_
#define CARTO_FILE_LEX_LEX_H_
#include "opal_config.h"
#ifdef malloc
#undef malloc
#endif
#ifdef realloc
#undef realloc
#endif
#ifdef free
#undef free
#endif
#include <stdio.h>
typedef union {
int ival;
char* sval;
} orte_rds_value_t;
extern int carto_file_lex(void);
extern FILE *carto_file_in;
extern int carto_file_line;
extern bool carto_file_done;
extern orte_rds_value_t carto_file_value;
/*
* Make lex-generated files not issue compiler warnings
*/
#define YY_STACK_USED 0
#define YY_ALWAYS_INTERACTIVE 0
#define YY_NEVER_INTERACTIVE 0
#define YY_MAIN 0
#define YY_NO_UNPUT 1
#define YY_SKIP_YYWRAP 1
#define OPAL_CARTO_FILE_NEWLINE 0
#define OPAL_CARTO_FILE_ERROR 1
#define OPAL_CARTO_FILE_NODE_DECELERATION 2
#define OPAL_CARTO_FILE_CONNECTION_DECELERATION 3
#define OPAL_CARTO_FILE_BIDIRECTION_CONNECTION 4
#define OPAL_CARTO_FILE_INT 5
#define OPAL_CARTO_FILE_NAME 6
#define OPAL_CARTO_FILE_NODE_CONNECTION 7
#endif

102
opal/mca/carto/file/carto_file_lex.l
View File

@ -1,102 +0,0 @@
%option nounput
%{ /* -*- C -*- */
/*
* Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
* University Research and Technology
* Corporation. All rights reserved.
* Copyright (c) 2004-2006 The University of Tennessee and The University
* of Tennessee Research Foundation. All rights
* reserved.
* Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
* University of Stuttgart. All rights reserved.
* Copyright (c) 2004-2005 The Regents of the University of California.
* All rights reserved.
* $COPYRIGHT$
*
* Additional copyrights may follow
*
* $HEADER$
*/
#include "opal_config.h"
#include <stdio.h>
#ifdef HAVE_UNISTD_H
#include <unistd.h>
#endif
#include "opal/mca/carto/file/carto_file_lex.h"
/*
* local functions
*/
BEGIN_C_DECLS
int carto_file_wrap(void);
END_C_DECLS
int carto_file_wrap(void)
{
carto_file_done = true;
return 1;
}
/*
* global variables
*/
int carto_file_line=1;
orte_rds_value_t carto_file_value;
bool carto_file_done = false;
%}
WHITE [\f\t\v ]
%x comment
%%
{WHITE}*\n { carto_file_line++;
return OPAL_CARTO_FILE_NEWLINE; }
#.*\n { carto_file_line++;
return OPAL_CARTO_FILE_NEWLINE; }
"//".*\n { carto_file_line++;
return OPAL_CARTO_FILE_NEWLINE; }
"/*" { BEGIN(comment);
return OPAL_CARTO_FILE_NEWLINE; }
<comment>[^*\n]* ; /* Eat up non '*'s */
<comment>"*"+[^*/\n]* ; /* Eat '*'s not followed by a '/' */
<comment>\n { carto_file_line++;
return OPAL_CARTO_FILE_NEWLINE; }
<comment>"*"+"/" { BEGIN(INITIAL); /* Done with Block Comment */
return OPAL_CARTO_FILE_NEWLINE; }
{WHITE}+ ; /* whitespace */
EDGE { carto_file_value.sval = yytext;
return OPAL_CARTO_FILE_NODE_DECELERATION; }
BRANCH { carto_file_value.sval = yytext;
return OPAL_CARTO_FILE_CONNECTION_DECELERATION; }
BRANCH_BI_DIR { carto_file_value.sval = yytext;
return OPAL_CARTO_FILE_BIDIRECTION_CONNECTION; }
[0-9] { carto_file_value.ival = atol(yytext);
return OPAL_CARTO_FILE_INT; }
[A-Za-z0-9_-]* { carto_file_value.sval = yytext;
return OPAL_CARTO_FILE_NAME; }
([[A-Za-z0-9_\-]*)+":"([0-9]*) { carto_file_value.sval = yytext;
return OPAL_CARTO_FILE_NODE_CONNECTION; }
%%

330
opal/mca/carto/file/carto_file_module.c
View File

@ -1,330 +0,0 @@
/*
* Copyright (c) 2004-2008 The Trustees of Indiana University and Indiana
* University Research and Technology
* Corporation. All rights reserved.
* Copyright (c) 2004-2005 The University of Tennessee and The University
* of Tennessee Research Foundation. All rights
* reserved.
* Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
* University of Stuttgart. All rights reserved.
* Copyright (c) 2004-2005 The Regents of the University of California.
* All rights reserved.
* Copyright (c) 2006-2007 Cisco Systems, Inc. All rights reserved.
*
* $COPYRIGHT$
*
* Additional copyrights may follow
*
* $HEADER$
*/
#include "opal_config.h"
/* This component will only be compiled on File, where we are
guaranteed to have <unistd.h> and friends */
#include <stdio.h>
#ifdef HAVE_UNISTD_H
#include <unistd.h>
#endif /* HAVE_UNISTD_H */
#ifdef HAVE_STDLIB_H
#include <stdlib.h>
#endif /* HAVE_STDLIB_H */
#ifdef HAVE_STRING_H
#include <string.h>
#endif /* HAVE_STRING_H */
#include "opal/constants.h"
#include "opal/mca/base/mca_base_param.h"
#include "opal/util/argv.h"
#include "opal/util/show_help.h"
#include "opal/mca/carto/carto.h"
#include "opal/mca/carto/base/base.h"
#include "opal/mca/carto/base/carto_base_graph.h"
#include "carto_file.h"
#include "carto_file_lex.h"
char *carto_file_path;
static int opal_carto_file_init(void);
static int opal_carto_file_finalize(void);
static int opal_carto_file_parse(const char *cartofile);
/*
* File carto module
*/
static const opal_carto_base_module_1_0_0_t loc_module = {
opal_carto_file_init,
opal_carto_base_graph_get_host_graph_fn,
opal_carto_base_free_graph_fn,
opal_carto_base_get_nodes_distance_fn,
opal_carto_base_graph_spf_fn,
opal_carto_base_graph_find_node_fn,
opal_carto_file_finalize,
};
int opal_carto_file_component_query(mca_base_module_t **module, int *priority)
{
int param;
param = mca_base_param_find("carto", "file", "priority");
mca_base_param_lookup_int(param, priority);
param = mca_base_param_find("carto", "file", "path");
mca_base_param_lookup_string(param, &carto_file_path);
*module = (mca_base_module_t *)&loc_module;
return OPAL_SUCCESS;
}
/**
* Init the carto file module.
*
* @return int success or error
*/
static int opal_carto_file_init(void)
{
int rc;
/* create an empty graph */
if (NULL == opal_carto_base_common_host_graph) {
opal_carto_base_graph_create_fn(&opal_carto_base_common_host_graph);
}
if (NULL == carto_file_path) {
return OPAL_ERROR;
}
/* parse the carto file and add nodes and connections to the graph */
rc = opal_carto_file_parse(carto_file_path);
return rc;
}
/**
* finalize the carto file module.
*
* @return int success or error
*/
static int opal_carto_file_finalize(void)
{
/* free the host cartography graph. */
if (NULL != opal_carto_base_common_host_graph) {
opal_carto_base_free_graph_fn(opal_carto_base_common_host_graph);
}
return OPAL_SUCCESS;
}
/**
* Parse the carto file and file the host graph with nodes and
* connections.
*
* @param cartofile the path to the carto file.
*
* @return int success or error.
*/
static int opal_carto_file_parse(const char *cartofile)
{
int token;
opal_carto_base_node_t *node, *end_node;
uint32_t weight;
char *node1_name, *node2_name, *value, **argv;
int cnt, line_number = 1;
char *token_to_string[] = {
"New-line",
"File-error",
"Node deceleration",
"Connection deceleration",
"bi directional connection",
"Integer",
"Name",
"Node connection",
"Undefined"
"Undefined"
"Undefined"
};
/* set the done flag to false. at the end of file the the lexical analyzer will set it to true */
carto_file_done = false;
carto_file_in = fopen(cartofile, "r"); /* open the carto file */
/* if the file not found, return an error */
if (NULL == carto_file_in) {
opal_show_help("help-opal-carto-file.txt", "file not found", true, cartofile);
return OPAL_ERR_NOT_FOUND;
}
while (!carto_file_done) {
token = carto_file_lex();
switch (token) {
case OPAL_CARTO_FILE_NEWLINE:
line_number++;
break;
case OPAL_CARTO_FILE_NODE_DECELERATION:
token = carto_file_lex();
switch (token) {
case OPAL_CARTO_FILE_NAME:
node = (opal_carto_base_node_t *)malloc(sizeof(opal_carto_base_node_t));
node->node_type = strdup(carto_file_value.sval);
if (0 == strcmp("socket",node->node_type)) {
node->is_cpu = true;
}
else {
node->is_cpu = false;
}
token = carto_file_lex();
switch (token) {
case OPAL_CARTO_FILE_NAME:
node->node_name = strdup(carto_file_value.sval);
opal_carto_base_graph_add_node_fn(opal_carto_base_common_host_graph, node);
break;
default:
free(node);
opal_show_help("help-opal-carto-file.txt", "expected node name",
true, cartofile, line_number, token_to_string[token]);
goto error;
}
break;
default:
opal_show_help("help-opal-carto-file.txt", "expected node type",
true, cartofile, line_number, token_to_string[token]);
goto error;
}
break;
case OPAL_CARTO_FILE_CONNECTION_DECELERATION:
token = carto_file_lex();
switch (token) {
case OPAL_CARTO_FILE_NAME:
node1_name = strdup(carto_file_value.sval);
while (OPAL_CARTO_FILE_NEWLINE != token) {
token = carto_file_lex();
switch (token) {
case OPAL_CARTO_FILE_NODE_CONNECTION:
value = carto_file_value.sval;
argv = opal_argv_split (value, ':');
cnt = opal_argv_count (argv);
if (2 == cnt) {
node2_name = strdup(argv[0]);
weight = atoi(argv[1]);
} else {
opal_show_help("help-opal-carto-file.txt", "incorrect connection", true, cartofile, line_number, value);
opal_argv_free (argv);
free(node1_name);
goto error;
}
opal_argv_free (argv);
/* find the start node of the connection */
node = opal_carto_base_graph_find_node_fn(opal_carto_base_common_host_graph,node1_name);
if (NULL == node) {
opal_show_help("help-opal-carto-file.txt", "vertex not found", true, cartofile, line_number, node1_name);
free(node1_name);
free(node2_name);
goto error;
}
/* find the end node of the connection */
end_node = opal_carto_base_graph_find_node_fn(opal_carto_base_common_host_graph,node2_name);
if (NULL == end_node) {
opal_show_help("help-opal-carto-file.txt", "vertex not found", true, cartofile, line_number, node2_name);
free(node1_name);
free(node2_name);
goto error;
}
opal_carto_base_connect_nodes_fn(opal_carto_base_common_host_graph, node, end_node, weight);
free(node2_name);
break;
case OPAL_CARTO_FILE_NEWLINE:
line_number++;
break;
default:
opal_show_help("help-opal-carto-file.txt", "expected Connection",
true, cartofile, line_number, token_to_string[token]);
free(node1_name);
goto error;
}
}
free(node1_name);
break;
default:
opal_show_help("help-opal-carto-file.txt", "expected node name",
true, cartofile, line_number, token_to_string[token]);
goto error;
}
break;
case OPAL_CARTO_FILE_BIDIRECTION_CONNECTION:
token = carto_file_lex();
switch (token) {
case OPAL_CARTO_FILE_NAME:
node1_name = strdup(carto_file_value.sval);
while (OPAL_CARTO_FILE_NEWLINE != token) {
token = carto_file_lex();
switch (token) {
case OPAL_CARTO_FILE_NODE_CONNECTION:
value = carto_file_value.sval;
argv = opal_argv_split (value, ':');
cnt = opal_argv_count (argv);
if (2 == cnt) {
node2_name = strdup(argv[0]);
weight = atoi(argv[1]);
} else {
opal_show_help("help-opal-carto-file.txt", "incorrect connection", true, cartofile, line_number, value);
opal_argv_free (argv);
free(node1_name);
goto error;
}
opal_argv_free (argv);
/* find the start node of the connection */
node = opal_carto_base_graph_find_node_fn(opal_carto_base_common_host_graph,node1_name);
if (NULL == node) {
opal_show_help("help-opal-carto-file.txt", "vertex not found", true, cartofile, line_number, node1_name);
free(node1_name);
free(node2_name);
goto error;
}
/* find the end node of the connection */
end_node = opal_carto_base_graph_find_node_fn(opal_carto_base_common_host_graph,node2_name);
if (NULL == end_node) {
opal_show_help("help-opal-carto-file.txt", "vertex not found", true, cartofile, line_number, node2_name);
free(node1_name);
free(node2_name);
goto error;
}
opal_carto_base_connect_nodes_fn(opal_carto_base_common_host_graph, node, end_node, weight);
opal_carto_base_connect_nodes_fn(opal_carto_base_common_host_graph, end_node, node, weight);
free(node2_name);
break;
case OPAL_CARTO_FILE_NEWLINE:
line_number++;
break;
default:
opal_show_help("help-opal-carto-file.txt", "expected Connection",
true, cartofile, line_number, token_to_string[token]);
free(node1_name);
goto error;
}
}
free(node1_name);
break;
default:
opal_show_help("help-opal-carto-file.txt", "expected node name",
true, cartofile, line_number, token_to_string[token]);
goto error;
}
break;
default:
opal_show_help("help-opal-carto-file.txt", "expected deceleration",
true, cartofile, line_number, token_to_string[token]);
goto error;
}
}
return OPAL_SUCCESS;
error:
fclose(carto_file_in);
return OPAL_ERR_BAD_PARAM;
}

56
opal/mca/carto/file/help-opal-carto-file.txt
View File

@ -1,56 +0,0 @@
# -*- text -*-
#
# Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
# University Research and Technology
# Corporation. All rights reserved.
# Copyright (c) 2004-2005 The University of Tennessee and The University
# of Tennessee Research Foundation. All rights
# reserved.
# Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
# University of Stuttgart. All rights reserved.
# Copyright (c) 2004-2006 The Regents of the University of California.
# All rights reserved.
# Copyright (c) 2006-2007 Cisco Systems, Inc. All rights reserved.
# Copyright (c) 2007 Mellanox Technologies. All rights reserved.
# $COPYRIGHT$
#
# Additional copyrights may follow
#
# $HEADER$
#
# This is the US/English help file for Open MPI's OpenFabrics support
# (the carto file).
#
[file not found]
File %s: No such file or directory
#
[expected node type]
File: %s line: %d expected Edge type (free string). received %s
#
[expected node name]
File: %s line: %d expected Edge name (free string). received %s
#
[expected Connection]
File: %s line: %d expected Edge branch (edge name:weight). received %s
#
[expected deceleration]
File: %s line: %d expected Edge declaration (EDGE) or branch declaration (BRANCH). received %s
#
[incorrect connection]
File: %s line: %d - %s - incorrect branch
#
[vertex not found]
File: %s line: %d - Edge %s is not in the graph
#
[unknown token]
File: %s line: %d - lexical analyzer returned
unknown value - %d
#

4
opal/mca/hwloc/base/Makefile.am
View File

@ -1,5 +1,5 @@
#
# Copyright (c) 2011 Cisco Systems, Inc. All rights reserved.
# Copyright (c) 2011-2012 Cisco Systems, Inc. All rights reserved.
# $COPYRIGHT$
#
# Additional copyrights may follow
@ -22,6 +22,6 @@ if OPAL_HAVE_HWLOC
libmca_hwloc_la_SOURCES += \
base/hwloc_base_dt.c \
base/hwloc_base_util.c \
base/hwloc_base_proc_mempolicy.c
base/hwloc_base_maffinity.c
endif

60
opal/mca/hwloc/base/base.h
View File

@ -1,5 +1,5 @@
/*
* Copyright (c) 2011 Cisco Systems, Inc. All rights reserved.
* Copyright (c) 2011-2012 Cisco Systems, Inc. All rights reserved.
* $COPYRIGHT$
*
* Additional copyrights may follow
@ -13,7 +13,6 @@
#include "opal_config.h"
#include "opal/dss/dss_types.h"
#include "opal/mca/paffinity/paffinity.h"
#include "opal/mca/hwloc/hwloc.h"
@ -23,6 +22,8 @@
BEGIN_C_DECLS
/* ******************************************************************** */
/**
* Initialize the hwloc MCA framework
*
@ -81,22 +82,19 @@ OPAL_DECLSPEC extern char *opal_hwloc_base_slot_list;
OPAL_DECLSPEC extern char *opal_hwloc_base_cpu_set;
OPAL_DECLSPEC extern hwloc_cpuset_t opal_hwloc_base_given_cpus;
/**
* Report a bind failure using the normal mechanisms if a component
* fails to bind memory -- according to the value of the
* hwloc_base_bind_failure_action MCA parameter.
*/
OPAL_DECLSPEC int opal_hwloc_base_report_bind_failure(const char *file,
int line,
const char *msg,
int rc);
OPAL_DECLSPEC opal_paffinity_locality_t opal_hwloc_base_get_relative_locality(hwloc_topology_t topo,
OPAL_DECLSPEC opal_hwloc_locality_t opal_hwloc_base_get_relative_locality(hwloc_topology_t topo,
opal_hwloc_level_t level1,
unsigned int peer1,
opal_hwloc_level_t level2,
unsigned int peer2);
/**
* Loads opal_hwloc_my_cpuset (global variable in
* opal/mca/hwloc/hwloc.h) for this process. opal_hwloc_my_cpuset
* will be loaded with this process' binding, or, if the process is
* not bound, use the hwloc root object's (available and online)
* cpuset.
*/
OPAL_DECLSPEC void opal_hwloc_base_get_local_cpuset(void);
/**
@ -130,7 +128,16 @@ OPAL_DECLSPEC extern opal_hwloc_base_mbfa_t opal_hwloc_base_mbfa;
/* some critical helper functions */
OPAL_DECLSPEC int opal_hwloc_base_filter_cpus(hwloc_topology_t topo);
/**
* Discover / load the hwloc topology (i.e., call hwloc_topology_init() and
* hwloc_topology_load()).
*/
OPAL_DECLSPEC int opal_hwloc_base_get_topology(void);
/**
* Free the hwloc topology.
*/
OPAL_DECLSPEC void opal_hwloc_base_free_topology(hwloc_topology_t topo);
OPAL_DECLSPEC hwloc_cpuset_t opal_hwloc_base_get_available_cpus(hwloc_topology_t topo,
hwloc_obj_t obj);
@ -146,14 +153,25 @@ OPAL_DECLSPEC hwloc_obj_t opal_hwloc_base_get_obj_by_type(hwloc_topology_t topo,
OPAL_DECLSPEC unsigned int opal_hwloc_base_get_obj_idx(hwloc_topology_t topo,
hwloc_obj_t obj,
opal_hwloc_resource_type_t rtype);
OPAL_DECLSPEC void opal_hwloc_base_get_level_and_index(hwloc_cpuset_t cpus,
opal_hwloc_level_t *bind_level,
unsigned int *bind_idx);
OPAL_DECLSPEC hwloc_obj_t opal_hwloc_base_get_level_and_index(hwloc_cpuset_t cpus,
opal_hwloc_level_t *bind_level,
unsigned int *bind_idx);
OPAL_DECLSPEC int opal_hwloc_base_get_local_index(hwloc_obj_type_t type,
unsigned cache_level,
unsigned int *idx);
/**
* Get the number of pu's under a given hwloc object.
*/
OPAL_DECLSPEC unsigned int opal_hwloc_base_get_npus(hwloc_topology_t topo,
hwloc_obj_t target);
OPAL_DECLSPEC char* opal_hwloc_base_print_binding(opal_binding_policy_t binding);
OPAL_DECLSPEC char* opal_hwloc_base_print_locality(opal_paffinity_locality_t locality);
OPAL_DECLSPEC char* opal_hwloc_base_print_locality(opal_hwloc_locality_t locality);
OPAL_DECLSPEC char* opal_hwloc_base_print_level(opal_hwloc_level_t level);
/**
* Determine if there is a single cpu in a bitmap.
*/
OPAL_DECLSPEC bool opal_hwloc_base_single_cpu(hwloc_cpuset_t cpuset);
/**
@ -184,6 +202,14 @@ OPAL_DECLSPEC int opal_hwloc_base_report_bind_failure(const char *file,
*/
OPAL_DECLSPEC int opal_hwloc_base_set_process_membind_policy(void);
OPAL_DECLSPEC int opal_hwloc_base_membind(opal_hwloc_base_memory_segment_t *segs,
size_t count, int node_id);
OPAL_DECLSPEC int opal_hwloc_base_node_name_to_id(char *node_name, int *id);
OPAL_DECLSPEC int opal_hwloc_base_memory_set(opal_hwloc_base_memory_segment_t *segments,
size_t num_segments);
/* datatype support */
OPAL_DECLSPEC int opal_hwloc_pack(opal_buffer_t *buffer, const void *src,
int32_t num_vals,

36
opal/mca/hwloc/base/hwloc_base_dt.c
View File

@ -1,5 +1,5 @@
/*
* Copyright (c) 2011 Cisco Systems, Inc. All rights reserved.
* Copyright (c) 2011-2012 Cisco Systems, Inc. All rights reserved.
*
* $COPYRIGHT$
*
@ -272,19 +272,27 @@ static void print_hwloc_obj(char **output, char *prefix,
free(tmp);
tmp = tmp2;
}
/* print the cpusets */
hwloc_bitmap_snprintf(string, OPAL_HWLOC_MAX_STRING, obj->cpuset);
asprintf(&tmp2, "%s%sCpuset: %s", tmp, pfx, string);
free(tmp);
tmp = tmp2;
hwloc_bitmap_snprintf(string, OPAL_HWLOC_MAX_STRING, obj->online_cpuset);
asprintf(&tmp2, "%s%sOnline: %s", tmp, pfx, string);
free(tmp);
tmp = tmp2;
hwloc_bitmap_snprintf(string, OPAL_HWLOC_MAX_STRING, obj->allowed_cpuset);
asprintf(&tmp2, "%s%sAllowed: %s", tmp, pfx, string);
free(tmp);
tmp = tmp2;
/* print the cpusets - apparently, some new HWLOC types don't
* have cpusets, so protect ourselves here
*/
if (NULL != obj->cpuset) {
hwloc_bitmap_snprintf(string, OPAL_HWLOC_MAX_STRING, obj->cpuset);
asprintf(&tmp2, "%s%sCpuset: %s", tmp, pfx, string);
free(tmp);
tmp = tmp2;
}
if (NULL != obj->online_cpuset) {
hwloc_bitmap_snprintf(string, OPAL_HWLOC_MAX_STRING, obj->online_cpuset);
asprintf(&tmp2, "%s%sOnline: %s", tmp, pfx, string);
free(tmp);
tmp = tmp2;
}
if (NULL != obj->allowed_cpuset) {
hwloc_bitmap_snprintf(string, OPAL_HWLOC_MAX_STRING, obj->allowed_cpuset);
asprintf(&tmp2, "%s%sAllowed: %s", tmp, pfx, string);
free(tmp);
tmp = tmp2;
}
if (HWLOC_OBJ_MACHINE == obj->type) {
/* root level object - add support values */
support = (struct hwloc_topology_support*)hwloc_topology_get_support(topo);

122
opal/mca/maffinity/hwloc/maffinity_hwloc_module.c → opal/mca/hwloc/base/hwloc_base_maffinity.c
View File

@ -1,16 +1,5 @@
/*
* Copyright (c) 2004-2008 The Trustees of Indiana University and Indiana
* University Research and Technology
* Corporation. All rights reserved.
* Copyright (c) 2004-2005 The University of Tennessee and The University
* of Tennessee Research Foundation. All rights
* reserved.
* Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
* University of Stuttgart. All rights reserved.
* Copyright (c) 2004-2005 The Regents of the University of California.
* All rights reserved.
* Copyright (c) 2006-2011 Cisco Systems, Inc. All rights reserved.
*
* Copyright (c) 2011-2012 Cisco Systems, Inc. All rights reserved.
* $COPYRIGHT$
*
* Additional copyrights may follow
@ -18,71 +7,74 @@
* $HEADER$
*/
#include "opal_config.h"
/* This component will only be compiled on Hwloc, where we are
guaranteed to have <unistd.h> and friends */
#include <stdio.h>
#include <unistd.h>
#include <stdlib.h>
#include <string.h>
#include <errno.h>
#include "opal/constants.h"
#include "opal/mca/base/mca_base_param.h"
#include "opal/mca/maffinity/maffinity.h"
#include "opal/mca/maffinity/base/base.h"
#include "opal/mca/hwloc/base/base.h"
#include "maffinity_hwloc.h"
#include "opal/mca/hwloc/hwloc.h"
#include "opal/mca/hwloc/base/base.h"
/*
* Local functions
*/
static int hwloc_module_init(void);
static int hwloc_module_set(opal_maffinity_base_segment_t *segments,
size_t num_segments);
static int hwloc_module_node_name_to_id(char *, int *);
static int hwloc_module_bind(opal_maffinity_base_segment_t *, size_t, int);
/*
* Hwloc maffinity module
* Don't use show_help() here (or print any error message at all).
* Let the upper layer output a relevant message, because doing so may
* be complicated (e.g., this might be called from the ORTE ODLS,
* which has to do some extra steps to get error messages to be
* displayed).
*/
static const opal_maffinity_base_module_1_0_0_t local_module = {
/* Initialization function */
hwloc_module_init,
/* Module function pointers */
hwloc_module_set,
hwloc_module_node_name_to_id,
hwloc_module_bind
};
int opal_maffinity_hwloc_component_query(mca_base_module_t **module,
int *priority)
int opal_hwloc_base_set_process_membind_policy(void)
{
int rc = 0, flags;
hwloc_membind_policy_t policy;
hwloc_cpuset_t cpuset;
/* Make sure opal_hwloc_topology has been set by the time we've
been called */
if (NULL == opal_hwloc_topology) {
return OPAL_ERROR;
return OPAL_ERR_BAD_PARAM;
}
*priority = mca_maffinity_hwloc_component.priority;
*module = (mca_base_module_t *) &local_module;
/* Set the default memory allocation policy according to MCA
param */
switch (opal_hwloc_base_map) {
case OPAL_HWLOC_BASE_MAP_LOCAL_ONLY:
policy = HWLOC_MEMBIND_BIND;
flags = HWLOC_MEMBIND_STRICT;
break;
case OPAL_HWLOC_BASE_MAP_NONE:
default:
policy = HWLOC_MEMBIND_DEFAULT;
flags = 0;
break;
}
cpuset = hwloc_bitmap_alloc();
if (NULL == cpuset) {
rc = OPAL_ERR_OUT_OF_RESOURCE;
} else {
int e;
hwloc_get_cpubind(opal_hwloc_topology, cpuset, 0);
rc = hwloc_set_membind(opal_hwloc_topology,
cpuset, policy, flags);
e = errno;
hwloc_bitmap_free(cpuset);
return OPAL_SUCCESS;
/* See if hwloc was able to do it. If hwloc failed due to
ENOSYS, but the base_map == NONE, then it's not really an
error. */
if (0 != rc && ENOSYS == e &&
OPAL_HWLOC_BASE_MAP_NONE == opal_hwloc_base_map) {
rc = 0;
}
}
return (0 == rc) ? OPAL_SUCCESS : OPAL_ERROR;
}
static int hwloc_module_init(void)
{
/* Nothing to do! */
return OPAL_SUCCESS;
}
static int hwloc_module_set(opal_maffinity_base_segment_t *segments,
size_t num_segments)
int opal_hwloc_base_memory_set(opal_hwloc_base_memory_segment_t *segments,
size_t num_segments)
{
int rc = OPAL_SUCCESS;
char *msg = NULL;
@ -128,7 +120,7 @@ static int hwloc_module_set(opal_maffinity_base_segment_t *segments,
return OPAL_SUCCESS;
}
static int hwloc_module_node_name_to_id(char *node_name, int *id)
int opal_hwloc_base_node_name_to_id(char *node_name, int *id)
{
/* GLB: fix me */
*id = atoi(node_name + 3);
@ -136,8 +128,8 @@ static int hwloc_module_node_name_to_id(char *node_name, int *id)
return OPAL_SUCCESS;
}
static int hwloc_module_bind(opal_maffinity_base_segment_t *segs,
size_t count, int node_id)
int opal_hwloc_base_membind(opal_hwloc_base_memory_segment_t *segs,
size_t count, int node_id)
{
size_t i;
int rc = OPAL_SUCCESS;

121
opal/mca/hwloc/base/hwloc_base_util.c
View File

@ -10,6 +10,8 @@
* Copyright (c) 2004-2005 The Regents of the University of California.
* All rights reserved.
* Copyright (c) 2011-2012 Cisco Systems, Inc. All rights reserved.
* Copyright (c) 2012 Los Alamos National Security, LLC.
* All rights reserved.
* $COPYRIGHT$
*
* Additional copyrights may follow
@ -32,7 +34,6 @@
#include "opal/util/argv.h"
#include "opal/util/output.h"
#include "opal/util/show_help.h"
#include "opal/mca/paffinity/paffinity.h"
#include "opal/threads/tsd.h"
#include "opal/mca/hwloc/hwloc.h"
@ -277,8 +278,11 @@ void opal_hwloc_base_get_local_cpuset(void)
if (NULL == opal_hwloc_my_cpuset) {
opal_hwloc_my_cpuset = hwloc_bitmap_alloc();
}
/* get the cpus we are bound to */
if (0 > hwloc_get_cpubind(opal_hwloc_topology, opal_hwloc_my_cpuset, HWLOC_CPUBIND_PROCESS)) {
if (hwloc_get_cpubind(opal_hwloc_topology,
opal_hwloc_my_cpuset,
HWLOC_CPUBIND_PROCESS) < 0) {
/* we are not bound - use the root's available cpuset */
root = hwloc_get_root_obj(opal_hwloc_topology);
base_cpus = opal_hwloc_base_get_available_cpus(opal_hwloc_topology, root);
@ -341,6 +345,11 @@ hwloc_cpuset_t opal_hwloc_base_get_available_cpus(hwloc_topology_t topo,
return rdata->available;
}
/* some hwloc object types don't have cpus */
if (NULL == obj->online_cpuset || NULL == obj->allowed_cpuset) {
return NULL;
}
/* see if we already have this info */
if (NULL == (data = (opal_hwloc_obj_data_t*)obj->userdata)) {
/* nope - create the object */
@ -620,7 +629,7 @@ static hwloc_obj_t df_search(hwloc_topology_t topo,
if (NULL == data->available) {
data->available = opal_hwloc_base_get_available_cpus(topo, start);
}
if (!hwloc_bitmap_iszero(data->available)) {
if (NULL != data->available && !hwloc_bitmap_iszero(data->available)) {
if (NULL != num_objs) {
*num_objs += 1;
} else if (*idx == nobj) {
@ -1036,9 +1045,9 @@ int opal_hwloc_base_slot_list_parse(const char *slot_str,
return OPAL_SUCCESS;
}
static opal_paffinity_locality_t get_locality(opal_hwloc_level_t level)
static opal_hwloc_locality_t get_locality(opal_hwloc_level_t level)
{
opal_paffinity_locality_t lvl;
opal_hwloc_locality_t lvl;
switch(level) {
case OPAL_HWLOC_NODE_LEVEL:
@ -1070,13 +1079,13 @@ static opal_paffinity_locality_t get_locality(opal_hwloc_level_t level)
return lvl;
}
opal_paffinity_locality_t opal_hwloc_base_get_relative_locality(hwloc_topology_t topo,
opal_hwloc_level_t level1,
unsigned int peer1,
opal_hwloc_level_t level2,
unsigned int peer2)
opal_hwloc_locality_t opal_hwloc_base_get_relative_locality(hwloc_topology_t topo,
opal_hwloc_level_t level1,
unsigned int peer1,
opal_hwloc_level_t level2,
unsigned int peer2)
{
opal_paffinity_locality_t locality;
opal_hwloc_locality_t locality;
hwloc_obj_t obj1, obj2;
unsigned cache_level=0;
opal_hwloc_level_t i, lvl;
@ -1234,9 +1243,9 @@ static hwloc_obj_t df_search_level(hwloc_obj_t start,
return NULL;
}
void opal_hwloc_base_get_level_and_index(hwloc_cpuset_t cpus,
opal_hwloc_level_t *bind_level,
unsigned int *bind_idx)
hwloc_obj_t opal_hwloc_base_get_level_and_index(hwloc_cpuset_t cpus,
opal_hwloc_level_t *bind_level,
unsigned int *bind_idx)
{
hwloc_obj_t root, obj;
@ -1244,7 +1253,7 @@ void opal_hwloc_base_get_level_and_index(hwloc_cpuset_t cpus,
if (NULL == opal_hwloc_topology) {
*bind_level = OPAL_HWLOC_NODE_LEVEL;
*bind_idx = 0;
return;
return NULL;
}
/* start at the node level and do a down-first
@ -1257,11 +1266,89 @@ void opal_hwloc_base_get_level_and_index(hwloc_cpuset_t cpus,
if (NULL == obj) {
/* no match found */
return;
return NULL;
}
/* get the index */
*bind_idx = opal_hwloc_base_get_obj_idx(opal_hwloc_topology, obj, OPAL_HWLOC_AVAILABLE);
return obj;
}
int opal_hwloc_base_get_local_index(hwloc_obj_type_t type,
unsigned cache_level,
unsigned int *idx)
{
opal_hwloc_level_t bind_level;
unsigned int bind_idx;
hwloc_obj_t obj;
/* if we don't have topology info, nothing we can do */
if (NULL == opal_hwloc_topology) {
return OPAL_ERR_NOT_AVAILABLE;
}
/* ensure we have our local cpuset */
opal_hwloc_base_get_local_cpuset();
/* if we are not bound, then this is meaningless */
obj = opal_hwloc_base_get_level_and_index(opal_hwloc_my_cpuset,
&bind_level, &bind_idx);
if (OPAL_HWLOC_NODE_LEVEL == bind_level) {
return OPAL_ERR_NOT_BOUND;
}
/* if the type/level match, then we are done */
if (type == opal_hwloc_levels[bind_level]) {
if (HWLOC_OBJ_CACHE == type) {
if ((cache_level == 1 && bind_level == OPAL_HWLOC_L1CACHE_LEVEL) ||
(cache_level == 2 && bind_level == OPAL_HWLOC_L2CACHE_LEVEL) ||
(cache_level == 3 && bind_level == OPAL_HWLOC_L3CACHE_LEVEL)) {
*idx = bind_idx;
return OPAL_SUCCESS;
}
} else {
*idx = bind_idx;
return OPAL_SUCCESS;
}
}
/* if the binding level is below the type, then we cannot
* answer the question as we could run on multiple objects
* of that type - e.g., if we are bound to NUMA and we are
* asked for the idx of the socket we are on, then we can
* only answer "unknown"
*/
if (type > opal_hwloc_levels[bind_level]) {
return OPAL_ERR_MULTIPLE_AFFINITIES;
}
if (type == HWLOC_OBJ_CACHE) {
if ((cache_level == 1 && OPAL_HWLOC_L1CACHE_LEVEL < bind_level) ||
(cache_level == 2 && OPAL_HWLOC_L2CACHE_LEVEL < bind_level) ||
(cache_level == 3 && OPAL_HWLOC_L3CACHE_LEVEL < bind_level)) {
return OPAL_ERR_MULTIPLE_AFFINITIES;
}
}
/* move upward until we find the specified type */
while (NULL != obj) {
obj = obj->parent;
if (obj->type == type) {
if (type == HWLOC_OBJ_CACHE) {
if (cache_level == obj->attr->cache.depth) {
break;
}
} else {
break;
}
}
}
if (NULL == obj) {
return OPAL_ERR_NOT_FOUND;
}
/* get the index of this object */
*idx = opal_hwloc_base_get_obj_idx(opal_hwloc_topology, obj, OPAL_HWLOC_AVAILABLE);
return OPAL_SUCCESS;
}
#define OPAL_HWLOC_PRINT_MAX_SIZE 50
@ -1415,7 +1502,7 @@ char* opal_hwloc_base_print_level(opal_hwloc_level_t level)
return ret;
}
char* opal_hwloc_base_print_locality(opal_paffinity_locality_t locality)
char* opal_hwloc_base_print_locality(opal_hwloc_locality_t locality)
{
opal_hwloc_print_buffers_t *ptr;
int idx;

56
opal/mca/hwloc/hwloc.h
View File

@ -1,5 +1,5 @@
/*
* Copyright (c) 2011 Cisco Systems, Inc. All rights reserved.
* Copyright (c) 2011-2012 Cisco Systems, Inc. All rights reserved.
*
* $COPYRIGHT$
*
@ -86,6 +86,24 @@ typedef enum {
#if OPAL_HAVE_HWLOC
#include MCA_hwloc_IMPLEMENTATION_HEADER
/**
* Struct used to describe a section of memory (starting address
* and length). This is really the same thing as an iovec, but
* we include a separate type for it for at least 2 reasons:
*
* 1. Some OS's iovec definitions are exceedingly lame (e.g.,
* Solaris 9 has the length argument as an int, instead of a
* size_t).
*
* 2. We reserve the right to expand/change this struct in the
* future.
*/
typedef struct {
/** Starting address of segment */
void *mbs_start_addr;
/** Length of segment */
size_t mbs_len;
} opal_hwloc_base_memory_segment_t;
/* define type of processor info requested */
typedef uint8_t opal_hwloc_resource_type_t;
@ -151,6 +169,42 @@ typedef uint16_t opal_binding_policy_t;
#define OPAL_BIND_OVERLOAD_ALLOWED(n) \
(OPAL_BIND_ALLOW_OVERLOAD & (n))
/* ******************************************************************** */
typedef uint16_t opal_hwloc_locality_t;
/** Process locality definitions */
enum {
OPAL_PROC_LOCALITY_UNKNOWN = 0x0000,
OPAL_PROC_NON_LOCAL = 0x8000,
OPAL_PROC_ON_CLUSTER = 0x0400,
OPAL_PROC_ON_CU = 0x0200,
OPAL_PROC_ON_NODE = 0x0100,
OPAL_PROC_ON_BOARD = 0x0080,
OPAL_PROC_ON_NUMA = 0x0040,
OPAL_PROC_ON_SOCKET = 0x0020,
OPAL_PROC_ON_L3CACHE = 0x0010,
OPAL_PROC_ON_L2CACHE = 0x0008,
OPAL_PROC_ON_L1CACHE = 0x0004,
OPAL_PROC_ON_CORE = 0x0002,
OPAL_PROC_ON_HWTHREAD = 0x0001,
OPAL_PROC_ALL_LOCAL = 0x0fff
};
/** Process locality macros */
#define OPAL_PROC_ON_LOCAL_HWTHREAD(n) ((n) & OPAL_PROC_ON_HWTHREAD)
#define OPAL_PROC_ON_LOCAL_CORE(n) ((n) & OPAL_PROC_ON_CORE)
#define OPAL_PROC_ON_LOCAL_L1CACHE(n) ((n) & OPAL_PROC_ON_L1CACHE)
#define OPAL_PROC_ON_LOCAL_L2CACHE(n) ((n) & OPAL_PROC_ON_L2CACHE)
#define OPAL_PROC_ON_LOCAL_L3CACHE(n) ((n) & OPAL_PROC_ON_L3CACHE)
#define OPAL_PROC_ON_LOCAL_SOCKET(n) ((n) & OPAL_PROC_ON_SOCKET)
#define OPAL_PROC_ON_LOCAL_NUMA(n) ((n) & OPAL_PROC_ON_NUMA)
#define OPAL_PROC_ON_LOCAL_BOARD(n) ((n) & OPAL_PROC_ON_BOARD)
#define OPAL_PROC_ON_LOCAL_NODE(n) ((n) & OPAL_PROC_ON_NODE)
#define OPAL_PROC_ON_LOCAL_CU(n) ((n) & OPAL_PROC_ON_CU)
#define OPAL_PROC_ON_LOCAL_CLUSTER(n) ((n) & OPAL_PROC_ON_CLUSTER)
/* ******************************************************************** */
/* some global values */
OPAL_DECLSPEC extern hwloc_topology_t opal_hwloc_topology;
OPAL_DECLSPEC extern opal_binding_policy_t opal_hwloc_binding_policy;

5
opal/mca/hwloc/hwloc132/configure.m4
View File

@ -1,6 +1,6 @@
# -*- shell-script -*-
#
# Copyright (c) 2009-2011 Cisco Systems, Inc. All rights reserved.
# Copyright (c) 2009-2012 Cisco Systems, Inc. All rights reserved.
#
# $COPYRIGHT$
#
@ -109,6 +109,9 @@ AC_DEFUN([MCA_opal_hwloc_hwloc132_CONFIG],[
["$HWLOC_VERSION"],
[Version of hwloc])
# Do we have verbs support?
AC_CHECK_HEADERS([infiniband/verbs.h])
# Set these variables so that the framework m4 knows
# what file to include in opal/mca/hwloc/hwloc.h
opal_hwloc_hwloc132_include="$opal_hwloc_hwloc132_basedir/hwloc132.h"

10
opal/mca/hwloc/hwloc132/hwloc132.h
View File

@ -1,5 +1,5 @@
/*
* Copyright (c) 2011 Cisco Systems, Inc. All rights reserved.
* Copyright (c) 2011-2012 Cisco Systems, Inc. All rights reserved.
*
* $COPYRIGHT$
*
@ -15,8 +15,16 @@
#ifndef MCA_OPAL_HWLOC_HWLOC132_H
#define MCA_OPAL_HWLOC_HWLOC132_H
BEGIN_C_DECLS
#include "hwloc/include/hwloc.h"
/* If we have verbs.h, then include the hwloc openfabrics helpers
header file */
#if defined(HAVE_INFINIBAND_VERBS_H)
#include "hwloc/include/hwloc/openfabrics-verbs.h"
#endif
END_C_DECLS
#endif /* MCA_OPAL_HWLOC_HWLOC132_H */

37
opal/mca/maffinity/Makefile.am
View File

@ -1,37 +0,0 @@
#
# Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
# University Research and Technology
# Corporation. All rights reserved.
# Copyright (c) 2004-2005 The University of Tennessee and The University
# of Tennessee Research Foundation. All rights
# reserved.
# Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
# University of Stuttgart. All rights reserved.
# Copyright (c) 2004-2005 The Regents of the University of California.
# All rights reserved.
# Copyright (c) 2010-2011 Cisco Systems, Inc. All rights reserved.
# $COPYRIGHT$
#
# Additional copyrights may follow
#
# $HEADER$
#
# main library setup
noinst_LTLIBRARIES = libmca_maffinity.la
libmca_maffinity_la_SOURCES =
# local files
headers = maffinity.h maffinity_types.h
libmca_maffinity_la_SOURCES += $(headers)
# Conditionally install the header files
if WANT_INSTALL_HEADERS
opaldir = $(includedir)/openmpi/$(subdir)
nobase_opal_HEADERS = $(headers)
endif
include base/Makefile.am
distclean-local:
rm -f base/static-components.h

27
opal/mca/maffinity/base/Makefile.am
View File

@ -1,27 +0,0 @@
#
# Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
# University Research and Technology
# Corporation. All rights reserved.
# Copyright (c) 2004-2005 The University of Tennessee and The University
# of Tennessee Research Foundation. All rights
# reserved.
# Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
# University of Stuttgart. All rights reserved.
# Copyright (c) 2004-2005 The Regents of the University of California.
# All rights reserved.
# Copyright (c) 2011 Cisco Systems, Inc. All rights reserved.
# $COPYRIGHT$
#
# Additional copyrights may follow
#
# $HEADER$
#
headers += \
base/base.h
libmca_maffinity_la_SOURCES += \
base/maffinity_base_close.c \
base/maffinity_base_select.c \
base/maffinity_base_open.c \
base/maffinity_base_wrappers.c

166
opal/mca/maffinity/base/base.h
View File

@ -1,166 +0,0 @@
/*
* Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
* University Research and Technology
* Corporation. All rights reserved.
* Copyright (c) 2004-2006 The University of Tennessee and The University
* of Tennessee Research Foundation. All rights
* reserved.
* Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
* University of Stuttgart. All rights reserved.
* Copyright (c) 2004-2005 The Regents of the University of California.
* All rights reserved.
* Copyright (c) 2011 Cisco Systems, Inc. All rights reserved.
* $COPYRIGHT$
*
* Additional copyrights may follow
*
* $HEADER$
*
*/
#ifndef OPAL_MAFFINITY_BASE_H
#define OPAL_MAFFINITY_BASE_H
#include "opal_config.h"
#include "opal/mca/maffinity/maffinity.h"
/*
* Global functions for MCA overall maffinity open and close
*/
BEGIN_C_DECLS
/**
* Initialize the maffinity MCA framework
*
* @retval OPAL_SUCCESS Upon success
* @retval OPAL_ERROR Upon failure
*
* This must be the first function invoked in the maffinity MCA
* framework. It initializes the maffinity MCA framework, finds
* and opens maffinity components, etc.
*
* This function fills in the internal global variable
* opal_maffinity_base_components_opened, which is a list of all
* maffinity components that were successfully opened. This
* variable should \em only be used by other maffinity base
* functions -- it is not considered a public interface member --
* and is only mentioned here for completeness.
*/
OPAL_DECLSPEC int opal_maffinity_base_open(void);
/**
* Select an available component.
*
* @return OPAL_SUCCESS Upon success.
* @return OPAL_NOT_FOUND If no component can be selected.
* @return OPAL_ERROR Upon other failure.
*
* This function invokes the selection process for maffinity
* components, which works as follows:
*
* - If the \em maffinity MCA parameter is not specified, the
* selection set is all available maffinity components.
* - If the \em maffinity MCA parameter is specified, the
* selection set is just that component.
* - All components in the selection set are queried to see if
* they want to run. All components that want to run are ranked
* by their priority and the highest priority component is
* selected. All non-selected components have their "close"
* function invoked to let them know that they were not selected.
* - The selected component will have its "init" function invoked to
* let it know that it was selected.
*
* If we fall through this entire process and no component is
* selected, then return OPAL_NOT_FOUND (this is not a fatal
* error).
*
* At the end of this process, we'll either have a single
* component that is selected and initialized, or no component was
* selected. If no component was selected, subsequent invocation
* of the maffinity wrapper functions will return an error.
*/
OPAL_DECLSPEC int opal_maffinity_base_select(void);
/**
* Set memory affinity.
*
* @param segments Array describing segments and what process they
* belong to
* @param num_segments Length of the segments array
* @param am_allocator True if this process created the shared
* memory block
*
* @retval OPAL_SUCCESS upon success
* @retval OPAL_NOT_FOUND if no maffinity components are available.
* @retval OPAL_ERROR upon other error.
*
* Set the affinity of the memory segments described in the \em
* segments array.
*
* If no maffinity components were available, or if the
* opal_maffinity_base_select() was never invoked, OPAL_NOT_FOUND
* is returned.
*/
OPAL_DECLSPEC int opal_maffinity_base_set(opal_maffinity_base_segment_t *segments, size_t num_segments);
OPAL_DECLSPEC int opal_maffinity_base_node_name_to_id(char *, int *);
OPAL_DECLSPEC int opal_maffinity_base_bind(opal_maffinity_base_segment_t *, size_t, int);
/**
* Shut down the maffinity MCA framework.
*
* @retval OPAL_SUCCESS Always
*
* This function shuts down everything in the maffinity MCA
* framework.
*
* It must be the last function invoked on the maffinity MCA
* framework.
*/
OPAL_DECLSPEC int opal_maffinity_base_close(void);
/**
* Indication of whether a component was successfully selected or
* not
*/
extern bool opal_maffinity_base_selected;
/**
* Global component struct for the selected component
*/
OPAL_DECLSPEC extern const opal_maffinity_base_component_2_0_0_t
*opal_maffinity_base_component;
/**
* Global module struct for the selected module
*/
OPAL_DECLSPEC extern const opal_maffinity_base_module_1_0_0_t
*opal_maffinity_base_module;
/**
* Indicator as to whether the list of opened maffinity components
* is valid or not.
*/
extern bool opal_maffinity_base_components_opened_valid;
/**
* List of all opened components; created when the maffinity
* framework is initialized and destroyed when we reduce the list
* to all available maffinity components.
*/
OPAL_DECLSPEC extern opal_list_t opal_maffinity_base_components_opened;
/**
* Debugging output stream
*/
extern int opal_maffinity_base_output;
/**
* Flag to indicate whether or not maffinity was setup
*/
OPAL_DECLSPEC extern bool opal_maffinity_setup;
END_C_DECLS
#endif /* OPAL_MAFFINITY_BASE_H */

42
opal/mca/maffinity/base/maffinity_base_close.c
View File

@ -1,42 +0,0 @@
/*
* Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
* University Research and Technology
* Corporation. All rights reserved.
* Copyright (c) 2004-2006 The University of Tennessee and The University
* of Tennessee Research Foundation. All rights
* reserved.
* Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
* University of Stuttgart. All rights reserved.
* Copyright (c) 2004-2005 The Regents of the University of California.
* All rights reserved.
* $COPYRIGHT$
*
* Additional copyrights may follow
*
* $HEADER$
*/
#include "opal_config.h"
#include "opal/constants.h"
#include "opal/mca/mca.h"
#include "opal/mca/base/base.h"
#include "opal/mca/maffinity/maffinity.h"
#include "opal/mca/maffinity/base/base.h"
int opal_maffinity_base_close(void)
{
/* Close all components that are still open (this should only
happen during ompi_info). */
if (opal_maffinity_base_components_opened_valid) {
mca_base_components_close(opal_maffinity_base_output,
&opal_maffinity_base_components_opened, NULL);
OBJ_DESTRUCT(&opal_maffinity_base_components_opened);
opal_maffinity_base_components_opened_valid = false;
}
/* All done */
return OPAL_SUCCESS;
}

93
opal/mca/maffinity/base/maffinity_base_open.c
View File

@ -1,93 +0,0 @@
/*
* Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
* University Research and Technology
* Corporation. All rights reserved.
* Copyright (c) 2004-2005 The University of Tennessee and The University
* of Tennessee Research Foundation. All rights
* reserved.
* Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
* University of Stuttgart. All rights reserved.
* Copyright (c) 2004-2005 The Regents of the University of California.
* All rights reserved.
* Copyright (c) 2011 Cisco Systems, Inc. All rights reserved.
* $COPYRIGHT$
*
* Additional copyrights may follow
*
* $HEADER$
*/
#include "opal_config.h"
#include <string.h>
#ifdef HAVE_SYS_TYPES_H
#include <sys/types.h>
#endif
#ifdef HAVE_UNISTD_H
#include <unistd.h>
#endif
#include "opal/constants.h"
#include "opal/util/output.h"
#include "opal/util/show_help.h"
#include "opal/mca/mca.h"
#include "opal/mca/base/base.h"
#include "opal/mca/base/mca_base_param.h"
#include "opal/mca/maffinity/maffinity.h"
#include "opal/mca/maffinity/base/base.h"
/*
* The following file was created by configure. It contains extern
* statements and the definition of an array of pointers to each
* component's public mca_base_component_t struct.
*/
#include "opal/mca/maffinity/base/static-components.h"
/*
* Globals
*/
int opal_maffinity_base_output = -1;
bool opal_maffinity_base_components_opened_valid = false;
opal_list_t opal_maffinity_base_components_opened;
bool opal_maffinity_setup = false;
/*
* Function for finding and opening either all MCA components, or the one
* that was specifically requested via a MCA parameter.
*/
int opal_maffinity_base_open(void)
{
int int_value;
/* Debugging / verbose output */
mca_base_param_reg_int_name("maffinity", "base_verbose",
"Verbosity level of the maffinity framework",
false, false,
0, &int_value);
if (0 != int_value) {
opal_maffinity_base_output = opal_output_open(NULL);
} else {
opal_maffinity_base_output = -1;
}
opal_maffinity_base_components_opened_valid = false;
/* Open up all available components */
if (OPAL_SUCCESS !=
mca_base_components_open("maffinity", opal_maffinity_base_output,
mca_maffinity_base_static_components,
&opal_maffinity_base_components_opened,
true)) {
return OPAL_ERROR;
}
opal_maffinity_base_components_opened_valid = true;
/* All done */
return OPAL_SUCCESS;
}

71
opal/mca/maffinity/base/maffinity_base_select.c
View File

@ -1,71 +0,0 @@
/*
* Copyright (c) 2004-2008 The Trustees of Indiana University and Indiana
* University Research and Technology
* Corporation. All rights reserved.
* Copyright (c) 2004-2006 The University of Tennessee and The University
* of Tennessee Research Foundation. All rights
* reserved.
* Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
* University of Stuttgart. All rights reserved.
* Copyright (c) 2004-2005 The Regents of the University of California.
* All rights reserved.
* $COPYRIGHT$
*
* Additional copyrights may follow
*
* $HEADER$
*/
#include "opal_config.h"
#include "opal/constants.h"
#include "opal/mca/mca.h"
#include "opal/mca/base/base.h"
#include "opal/mca/base/mca_base_param.h"
#include "opal/mca/maffinity/maffinity.h"
#include "opal/mca/maffinity/base/base.h"
/*
* Globals
*/
bool opal_maffinity_base_selected = false;
const opal_maffinity_base_component_2_0_0_t *opal_maffinity_base_component = NULL;
const opal_maffinity_base_module_1_0_0_t *opal_maffinity_base_module = NULL;
int opal_maffinity_base_select(void)
{
int ret, exit_status = OPAL_SUCCESS;
opal_maffinity_base_component_2_0_0_t *best_component = NULL;
opal_maffinity_base_module_1_0_0_t *best_module = NULL;
/*
* Select the best component
*/
if( OPAL_SUCCESS != mca_base_select("maffinity", opal_maffinity_base_output,
&opal_maffinity_base_components_opened,
(mca_base_module_t **) &best_module,
(mca_base_component_t **) &best_component) ) {
/* This will only happen if no component was selected */
exit_status = OPAL_ERR_NOT_FOUND;
goto cleanup;
}
/* Save the winner */
opal_maffinity_base_component = best_component;
opal_maffinity_base_module = best_module;
opal_maffinity_base_selected = true;
/* Initialize the winner */
if (NULL != opal_maffinity_base_module) {
if (OPAL_SUCCESS != (ret = opal_maffinity_base_module->maff_module_init()) ) {
exit_status = ret;
goto cleanup;
}
}
cleanup:
return exit_status;
}

63
opal/mca/maffinity/base/maffinity_base_wrappers.c
View File

@ -1,63 +0,0 @@
/*
* Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
* University Research and Technology
* Corporation. All rights reserved.
* Copyright (c) 2004-2006 The University of Tennessee and The University
* of Tennessee Research Foundation. All rights
* reserved.
* Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
* University of Stuttgart. All rights reserved.
* Copyright (c) 2004-2005 The Regents of the University of California.
* All rights reserved.
* $COPYRIGHT$
*
* Additional copyrights may follow
*
* $HEADER$
*/
#include "opal_config.h"
#include "opal/constants.h"
#include "opal/mca/maffinity/maffinity.h"
#include "opal/mca/maffinity/base/base.h"
int opal_maffinity_base_set(opal_maffinity_base_segment_t *segments,
size_t num_segments)
{
if (!opal_maffinity_base_selected) {
return OPAL_ERR_NOT_FOUND;
}
return opal_maffinity_base_module->maff_module_set(segments, num_segments);
}
int opal_maffinity_base_node_name_to_id(char *node_name, int *node_id)
{
if (!opal_maffinity_base_selected) {
return OPAL_ERR_NOT_FOUND;
}
if (!opal_maffinity_base_module->maff_module_name_to_id) {
*node_id = 0;
return OPAL_ERR_NOT_IMPLEMENTED;
}
return opal_maffinity_base_module->maff_module_name_to_id(node_name,
node_id);
}
int opal_maffinity_base_bind(opal_maffinity_base_segment_t *segments,
size_t num_segments, int node_id)
{
if (!opal_maffinity_base_selected) {
return OPAL_ERR_NOT_FOUND;
}
if (!opal_maffinity_base_module->maff_module_bind) {
return OPAL_ERR_NOT_IMPLEMENTED;
}
return opal_maffinity_base_module->maff_module_bind(segments, num_segments,
node_id);
}

0
opal/mca/maffinity/first_use/.windows
View File

46
opal/mca/maffinity/first_use/Makefile.am
View File

@ -1,46 +0,0 @@
#
# Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
# University Research and Technology
# Corporation. All rights reserved.
# Copyright (c) 2004-2005 The University of Tennessee and The University
# of Tennessee Research Foundation. All rights
# reserved.
# Copyright (c) 2004-2009 High Performance Computing Center Stuttgart,
# University of Stuttgart. All rights reserved.
# Copyright (c) 2004-2005 The Regents of the University of California.
# All rights reserved.
# Copyright (c) 2010 Cisco Systems, Inc. All rights reserved.
# $COPYRIGHT$
#
# Additional copyrights may follow
#
# $HEADER$
#
EXTRA_DIST = .windows
sources = \
maffinity_first_use.h \
maffinity_first_use_component.c \
maffinity_first_use_module.c
# Make the output library in this directory, and name it either
# mca_<type>_<name>.la (for DSO builds) or libmca_<type>_<name>.la
# (for static builds).
if MCA_BUILD_opal_maffinity_first_use_DSO
component_noinst =
component_install = mca_maffinity_first_use.la
else
component_noinst = libmca_maffinity_first_use.la
component_install =
endif
mcacomponentdir = $(pkglibdir)
mcacomponent_LTLIBRARIES = $(component_install)
mca_maffinity_first_use_la_SOURCES = $(sources)
mca_maffinity_first_use_la_LDFLAGS = -module -avoid-version
noinst_LTLIBRARIES = $(component_noinst)
libmca_maffinity_first_use_la_SOURCES =$(sources)
libmca_maffinity_first_use_la_LDFLAGS = -module -avoid-version

55
opal/mca/maffinity/first_use/maffinity_first_use.h
View File

@ -1,55 +0,0 @@
/*
* Copyright (c) 2004-2008 The Trustees of Indiana University and Indiana
* University Research and Technology
* Corporation. All rights reserved.
* Copyright (c) 2004-2006 The University of Tennessee and The University
* of Tennessee Research Foundation. All rights
* reserved.
* Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
* University of Stuttgart. All rights reserved.
* Copyright (c) 2004-2005 The Regents of the University of California.
* All rights reserved.
* $COPYRIGHT$
*
* Additional copyrights may follow
*
* $HEADER$
*/
/**
* @file
*
* Processor affinity for First_Use.
*/
#ifndef MCA_MAFFINITY_FIRST_USE_EXPORT_H
#define MCA_MAFFINITY_FIRST_USE_EXPORT_H
#include "opal_config.h"
#include "opal/mca/mca.h"
#include "opal/mca/maffinity/maffinity.h"
BEGIN_C_DECLS
/**
* Globally exported variable
*/
OPAL_MODULE_DECLSPEC extern const opal_maffinity_base_component_2_0_0_t
mca_maffinity_first_use_component;
/**
* maffinity query API function
*
* Query function for maffinity components. Simply returns a priority
* to rank it against other available maffinity components (assumedly,
* only one component will be available per platform, but it's
* possible that there could be more than one available).
*/
int opal_maffinity_first_use_component_query(mca_base_module_t **module, int *priority);
END_C_DECLS
#endif /* MCA_MAFFINITY_FIRST_USE_EXPORT_H */

75
opal/mca/maffinity/first_use/maffinity_first_use_component.c
View File

@ -1,75 +0,0 @@
/*
* Copyright (c) 2004-2008 The Trustees of Indiana University and Indiana
* University Research and Technology
* Corporation. All rights reserved.
* Copyright (c) 2004-2005 The University of Tennessee and The University
* of Tennessee Research Foundation. All rights
* reserved.
* Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
* University of Stuttgart. All rights reserved.
* Copyright (c) 2004-2005 The Regents of the University of California.
* All rights reserved.
* $COPYRIGHT$
*
* Additional copyrights may follow
*
* $HEADER$
*/
#include "opal_config.h"
#include "opal/constants.h"
#include "opal/mca/maffinity/maffinity.h"
#include "maffinity_first_use.h"
/*
* Public string showing the maffinity ompi_first_use component version number
*/
const char *opal_maffinity_first_use_component_version_string =
"OPAL first_use maffinity MCA component version " OPAL_VERSION;
/*
* Local function
*/
static int first_use_open(void);
/*
* Instantiate the public struct with all of our public information
* and pointers to our public functions in it
*/
const opal_maffinity_base_component_2_0_0_t mca_maffinity_first_use_component = {
/* First, the mca_component_t struct containing meta information
about the component itself */
{
OPAL_MAFFINITY_BASE_VERSION_2_0_0,
/* Component name and version */
"first_use",
OPAL_MAJOR_VERSION,
OPAL_MINOR_VERSION,
OPAL_RELEASE_VERSION,
/* Component open and close functions */
first_use_open,
NULL,
opal_maffinity_first_use_component_query
},
{
/* The component is checkpoint ready */
MCA_BASE_METADATA_PARAM_CHECKPOINT
}
};
static int first_use_open(void)
{
mca_base_param_reg_int(&mca_maffinity_first_use_component.base_version,
"priority",
"Priority of the first_use maffinity component",
false, false, 10, NULL);
return OPAL_SUCCESS;
}

99
opal/mca/maffinity/first_use/maffinity_first_use_module.c
View File

@ -1,99 +0,0 @@
/*
* Copyright (c) 2004-2008 The Trustees of Indiana University and Indiana
* University Research and Technology
* Corporation. All rights reserved.
* Copyright (c) 2004-2005 The University of Tennessee and The University
* of Tennessee Research Foundation. All rights
* reserved.
* Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
* University of Stuttgart. All rights reserved.
* Copyright (c) 2004-2005 The Regents of the University of California.
* All rights reserved.
* Copyright (c) 2009 Cisco Systems, Inc. All rights reserved.
* $COPYRIGHT$
*
* Additional copyrights may follow
*
* $HEADER$
*/
#include "opal_config.h"
#include <string.h>
#include <stddef.h>
#ifdef HAVE_UNISTD_H
#include <unistd.h>
#endif
#include "opal/constants.h"
#include "opal/mca/maffinity/maffinity.h"
#include "opal/mca/maffinity/base/base.h"
#include "maffinity_first_use.h"
/*
* Local functions
*/
static int first_use_module_init(void);
static int first_use_module_set(opal_maffinity_base_segment_t *segments,
size_t num_segments);
/*
* First_Use maffinity module
*/
static const opal_maffinity_base_module_1_0_0_t loc_module = {
/* Initialization function */
first_use_module_init,
/* Module function pointers */
first_use_module_set,
NULL,
NULL
};
int opal_maffinity_first_use_component_query(mca_base_module_t **module, int *priority)
{
int param;
param = mca_base_param_find("maffinity", "first_use", "priority");
mca_base_param_lookup_int(param, priority);
*module = (mca_base_module_t *)&loc_module;
return OPAL_SUCCESS;
}
static int first_use_module_init(void)
{
/* Nothing to do */
return OPAL_SUCCESS;
}
static int first_use_module_set(opal_maffinity_base_segment_t *segments,
size_t num_segments)
{
size_t i;
uintptr_t pagesize = (uintptr_t)sysconf(_SC_PAGESIZE);
volatile char useless;
for (i = 0; i < num_segments; ++i) {
char* ptr = (char*)segments[i].mbs_start_addr;
char* end_ptr = ptr + segments[i].mbs_len;
/* Let's touch the first byte of the segment. If this is the
* first byte on the memory page, good. If not, at least it
* will not overwrite anything important.
*/
useless = ptr[0];
/* Compute the address of the first byte on the next page */
ptr = (char*)((uintptr_t)(ptr + pagesize) & ~pagesize);
while( ptr <= end_ptr) {
useless += ptr[0];
ptr += pagesize;
};
}
return OPAL_SUCCESS;
}

53
opal/mca/maffinity/hwloc/Makefile.am
View File

@ -1,53 +0,0 @@
#
# Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
# University Research and Technology
# Corporation. All rights reserved.
# Copyright (c) 2004-2005 The University of Tennessee and The University
# of Tennessee Research Foundation. All rights
# reserved.
# Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
# University of Stuttgart. All rights reserved.
# Copyright (c) 2004-2005 The Regents of the University of California.
# All rights reserved.
# Copyright (c) 2010-2011 Cisco Systems, Inc. All rights reserved.
# $COPYRIGHT$
#
# Additional copyrights may follow
#
# $HEADER$
#
# To find hwloc_bottom.h. EMBEDDED flags are for if we OMPI's
# internal hwloc is used; maffinity_hwloc_CPPFLAGS is if we are using
# an external install.
AM_CPPFLAGS = $(HWLOC_EMBEDDED_CPPFLAGS) $(maffinity_hwloc_CPPFLAGS)
# To get the cflags for the stuff in hwloc.h
AM_CFLAGS = $(HWLOC_EMBEDDED_CFLAGS) $(maffinity_hwloc_CFLAGS)
dist_pkgdata_DATA = help-opal-maffinity-hwloc.txt
sources = \
maffinity_hwloc.h \
maffinity_hwloc_component.c \
maffinity_hwloc_module.c
# Make the output library in this directory, and name it either
# mca_<type>_<name>.la (for DSO builds) or libmca_<type>_<name>.la
# (for static builds).
if MCA_BUILD_opal_maffinity_hwloc_DSO
component_noinst =
component_install = mca_maffinity_hwloc.la
else
component_noinst = libmca_maffinity_hwloc.la
component_install =
endif
mcacomponentdir = $(pkglibdir)
mcacomponent_LTLIBRARIES = $(component_install)
mca_maffinity_hwloc_la_SOURCES = $(sources)
mca_maffinity_hwloc_la_LDFLAGS = -module -avoid-version
noinst_LTLIBRARIES = $(component_noinst)
libmca_maffinity_hwloc_la_SOURCES =$(sources)
libmca_maffinity_hwloc_la_LDFLAGS = -module -avoid-version

26
opal/mca/maffinity/hwloc/configure.m4
View File

@ -1,26 +0,0 @@
# -*- shell-script -*-
#
# Copyright (c) 2007-2011 Cisco Systems, Inc. All rights reserved.
#
# $COPYRIGHT$
#
# Additional copyrights may follow
#
# $HEADER$
#
# MCA_opal_maffinity_hwloc_CONFIG([action-if-found], [action-if-not-found])
# -------------------------------------------------------------------------
AC_DEFUN([MCA_opal_maffinity_hwloc_CONFIG],[
AC_REQUIRE([MCA_opal_hwloc_CONFIG_REQUIRE])
AC_CONFIG_FILES([opal/mca/maffinity/hwloc/Makefile])
# All we check for is whether $OPAL_HAVE_HWLOC is 1.
# See big comment in opal/mca/hwloc/configure.m4.
AC_MSG_CHECKING([if hwloc is enabled])
AS_IF([test $OPAL_HAVE_HWLOC -eq 1],
[AC_MSG_RESULT([yes])
$1],
[AC_MSG_RESULT([no])
$2])
])dnl

27
opal/mca/maffinity/hwloc/help-opal-maffinity-hwloc.txt
View File

@ -1,27 +0,0 @@
# -*- text -*-
#
# Copyright (c) 2011 Cisco Systems, Inc. All rights reserved.
# $COPYRIGHT$
#
# Additional copyrights may follow
#
# $HEADER$
#
# This is the US/English help file for Open MPI's libnuma support
#
[invalid policy]
WARNING: An invalid value was given for the maffinity_libnuma_policy
MCA parameter. The policy determines what happens when Open MPI tries
to allocate local memory, but no local memory is available. The value
provided was:
Value: %s
PID: %d
Valid values are:
strict: the memory allocation will fail
loose: the memory allocation will spill over to remote memory
Your job will now abort.
#

60
opal/mca/maffinity/hwloc/maffinity_hwloc.h
View File

@ -1,60 +0,0 @@
/*
* Copyright (c) 2004-2008 The Trustees of Indiana University and Indiana
* University Research and Technology
* Corporation. All rights reserved.
* Copyright (c) 2004-2005 The University of Tennessee and The University
* of Tennessee Research Foundation. All rights
* reserved.
* Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
* University of Stuttgart. All rights reserved.
* Copyright (c) 2004-2005 The Regents of the University of California.
* All rights reserved.
* Copyright (c) 2006-2011 Cisco Systems, Inc. All rights reserved.
*
* $COPYRIGHT$
*
* Additional copyrights may follow
*
* $HEADER$
*/
#ifndef MCA_MAFFINITY_HWLOC_EXPORT_H
#define MCA_MAFFINITY_HWLOC_EXPORT_H
#include "opal_config.h"
#include "opal/mca/mca.h"
#include "opal/mca/maffinity/maffinity.h"
#include "hwloc.h"
BEGIN_C_DECLS
typedef struct {
/* Base maffinity component */
opal_maffinity_base_component_2_0_0_t base;
/* This component's data */
int priority;
} opal_maffinity_hwloc_component_2_0_0_t;
/**
* Globally exported variable
*/
OPAL_DECLSPEC extern opal_maffinity_hwloc_component_2_0_0_t
mca_maffinity_hwloc_component;
/**
* maffinity query API function
*
* Query function for maffinity components. Simply returns a priority
* to rank it against other available maffinity components (assumedly,
* only one component will be available per platform, but it's
* possible that there could be more than one available).
*/
int opal_maffinity_hwloc_component_query(mca_base_module_t **module,
int *priority);
END_C_DECLS
#endif /* MCA_MAFFINITY_HWLOC_EXPORT_H */

88
opal/mca/maffinity/hwloc/maffinity_hwloc_component.c
View File

@ -1,88 +0,0 @@
/*
* Copyright (c) 2004-2008 The Trustees of Indiana University and Indiana
* University Research and Technology
* Corporation. All rights reserved.
* Copyright (c) 2004-2005 The University of Tennessee and The University
* of Tennessee Research Foundation. All rights
* reserved.
* Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
* University of Stuttgart. All rights reserved.
* Copyright (c) 2004-2005 The Regents of the University of California.
* All rights reserved.
* Copyright (c) 2007-2011 Cisco Systems, Inc. All rights reserved.
* $COPYRIGHT$
*
* Additional copyrights may follow
*
* $HEADER$
*/
#include "opal_config.h"
#include <string.h>
#include "opal/constants.h"
#include "opal/util/show_help.h"
#include "opal/mca/base/mca_base_param.h"
#include "opal/mca/hwloc/hwloc.h"
#include "opal/mca/maffinity/maffinity.h"
#include "maffinity_hwloc.h"
/*
* Public string showing the maffinity ompi_hwloc component version number
*/
const char *opal_maffinity_hwloc_component_version_string =
"OPAL hwloc maffinity MCA component version " OPAL_VERSION;
/*
* Local function
*/
static int hwloc_register(void);
/*
* Instantiate the public struct with all of our public information
* and pointers to our public functions in it
*/
opal_maffinity_hwloc_component_2_0_0_t mca_maffinity_hwloc_component = {
{
/* First, the mca_component_t struct containing meta information
about the component itself */
{
OPAL_MAFFINITY_BASE_VERSION_2_0_0,
/* Component name and version */
"hwloc",
OPAL_MAJOR_VERSION,
OPAL_MINOR_VERSION,
OPAL_RELEASE_VERSION,
/* Component open and close functions */
NULL,
NULL,
opal_maffinity_hwloc_component_query,
hwloc_register,
},
{
/* The component is checkpoint ready */
MCA_BASE_METADATA_PARAM_CHECKPOINT
}
},
/* Priority */
40,
/* NULL fill the rest of the component data */
};
static int hwloc_register(void)
{
mca_base_param_reg_int(&mca_maffinity_hwloc_component.base.base_version,
"priority",
"Priority of the hwloc maffinity component",
false, false, 40,
&mca_maffinity_hwloc_component.priority);
return OPAL_SUCCESS;
}

137
opal/mca/maffinity/maffinity.h
View File

@ -1,137 +0,0 @@
/*
* Copyright (c) 2004-2008 The Trustees of Indiana University and Indiana
* University Research and Technology
* Corporation. All rights reserved.
* Copyright (c) 2004-2005 The University of Tennessee and The University
* of Tennessee Research Foundation. All rights
* reserved.
* Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
* University of Stuttgart. All rights reserved.
* Copyright (c) 2004-2005 The Regents of the University of California.
* All rights reserved.
* $COPYRIGHT$
*
* Additional copyrights may follow
*
* $HEADER$
*/
/**
* @file
*
* maffinity (memory affinity) framework component interface
* definitions.
*
* Intent
*
* Simple component to set memory affinity for pages. Note that this
* framework assumes that processor affinity is being used (it doesn't
* make much sense to use memory affinity unless processes are bound
* to specific processors, otherwise the processes may move around and
* the memory may end up being remote).
*
* maffinity components are typically used with shared memory
* operations, but can be used elsewhere as well. The idea is to get
* memory physically located with the process that is going to use it.
* For memory allocated to a processor-bound process, functions such
* as malloc() do this naturally. However, when working with memory
* shared by multiple processes on a NUMA machine, it can be extremely
* advantageous to ensure that pages containing the data structures
* for a given process are physically local to the processor where
* that process is bound.
*
* One process will allocate a large shared memory block and all will
* need to participate to make pages local to specific processors.
*
* There is one main module function
* (opal_maffinity_base_module_set_fn_t) that takes an array of
* segment descriptions within the block. Each process will get a
* different set of segment descriptions (i.e., the segments belonging
* to that process). Components then do whatever is necessary to make
* pages local to their respective processes (i.e., the processors
* where the processes are running).
*/
#ifndef OPAL_MAFFINITY_H
#define OPAL_MAFFINITY_H
#include "opal_config.h"
#include "opal/mca/mca.h"
#include "opal/mca/base/base.h"
#include "opal/mca/maffinity/maffinity_types.h"
/**
* Module initialization function. Should return OPAL_SUCCESS.
*/
typedef int (*opal_maffinity_base_module_init_1_0_0_fn_t)(void);
/**
* Module function to set memory affinity. Take an array of
* maffinity_base_segment_t instances to describe which memory should
* physically reside with which process.
*
* This function is intended to be invoked by each process immediately
* after they mmap / attach the shared memory. In some cases, it will
* be a no-op for some processes (i.e., machines where a single
* function call in the creating process sets the memory affinity for
* the entire region), but in other cases all processes will need to
* participate (e.g., the first_use component, each each process will
* "touch" the pages that are supposed to be local to them).
*/
typedef int (*opal_maffinity_base_module_set_fn_t)
(opal_maffinity_base_segment_t *segments, size_t num_segments);
/**
* translate memory node name (such as "mem0") to memory node id
*/
typedef int (*opal_maffinity_base_module_node_name_to_id_fn_t)
(char *node_name, int *node_id);
/**
* bind memory to node
*/
typedef int (*opal_maffinity_base_module_bind_fn_t)
(opal_maffinity_base_segment_t *segments, size_t num_segments, int node_id);
/**
* Structure for maffinity components.
*/
struct opal_maffinity_base_component_2_0_0_t {
/** MCA base component */
mca_base_component_t base_version;
/** MCA base data */
mca_base_component_data_t base_data;
};
/**
* Convenience typedef
*/
typedef struct opal_maffinity_base_component_2_0_0_t opal_maffinity_base_component_2_0_0_t;
/**
* Structure for maffinity modules
*/
struct opal_maffinity_base_module_1_0_0_t {
/** Module initialization function */
opal_maffinity_base_module_init_1_0_0_fn_t maff_module_init;
/** Set memory affinity */
opal_maffinity_base_module_set_fn_t maff_module_set;
opal_maffinity_base_module_node_name_to_id_fn_t maff_module_name_to_id;
opal_maffinity_base_module_bind_fn_t maff_module_bind;
};
/**
* Convenience typedef
*/
typedef struct opal_maffinity_base_module_1_0_0_t opal_maffinity_base_module_1_0_0_t;
/*
* Macro for use in components that are of type maffinity
*/
#define OPAL_MAFFINITY_BASE_VERSION_2_0_0 \
MCA_BASE_VERSION_2_0_0, \
"maffinity", 2, 0, 0
#endif /* OPAL_MAFFINITY_H */

59
opal/mca/maffinity/maffinity_types.h
View File

@ -1,59 +0,0 @@
/*
* Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
* University Research and Technology
* Corporation. All rights reserved.
* Copyright (c) 2004-2005 The University of Tennessee and The University
* of Tennessee Research Foundation. All rights
* reserved.
* Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
* University of Stuttgart. All rights reserved.
* Copyright (c) 2004-2005 The Regents of the University of California.
* All rights reserved.
* $COPYRIGHT$
*
* Additional copyrights may follow
*
* $HEADER$
*/
/**
* @file
*
* Common types used in the maffinity framework
*/
#ifndef OPAL_MAFFINITY_TYPES_H
#define OPAL_MAFFINITY_TYPES_H
#include "opal_config.h"
#include <sys/types.h>
BEGIN_C_DECLS
/**
* Struct used with opal_maffinity_base_module_set_fn_t. It
* describes a section of memory (starting address and length).
* This is really the same thing as an iovec, but we include a
* separate type for it for at least 2 reasons:
*
* 1. Some OS's iovec definitions are exceedingly lame (e.g.,
* Solaris 9 has the length argument as an int, instead of a
* size_t).
*
* 2. We reserve the right to expand/change this struct in the
* future.
*/
struct opal_maffinity_base_segment_t {
/** Starting address of segment */
void *mbs_start_addr;
/** Length of segment */
size_t mbs_len;
};
/**
* Convenience typedef
*/
typedef struct opal_maffinity_base_segment_t opal_maffinity_base_segment_t;
END_C_DECLS
#endif /* OPAL_MAFFINITY_TYPES_H */

37
opal/mca/paffinity/Makefile.am
View File

@ -1,37 +0,0 @@
#
# Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
# University Research and Technology
# Corporation. All rights reserved.
# Copyright (c) 2004-2005 The University of Tennessee and The University
# of Tennessee Research Foundation. All rights
# reserved.
# Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
# University of Stuttgart. All rights reserved.
# Copyright (c) 2004-2005 The Regents of the University of California.
# All rights reserved.
# Copyright (c) 2010 Cisco Systems, Inc. All rights reserved.
# $COPYRIGHT$
#
# Additional copyrights may follow
#
# $HEADER$
#
# main library setup
noinst_LTLIBRARIES = libmca_paffinity.la
libmca_paffinity_la_SOURCES =
# local files
headers = paffinity.h
libmca_paffinity_la_SOURCES += $(headers)
# Conditionally install the header files
if WANT_INSTALL_HEADERS
opaldir = $(includedir)/openmpi/$(subdir)
nobase_opal_HEADERS = $(headers)
endif
include base/Makefile.am
distclean-local:
rm -f base/static-components.h

27
opal/mca/paffinity/base/Makefile.am
View File

@ -1,27 +0,0 @@
#
# Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
# University Research and Technology
# Corporation. All rights reserved.
# Copyright (c) 2004-2005 The University of Tennessee and The University
# of Tennessee Research Foundation. All rights
# reserved.
# Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
# University of Stuttgart. All rights reserved.
# Copyright (c) 2004-2005 The Regents of the University of California.
# All rights reserved.
# $COPYRIGHT$
#
# Additional copyrights may follow
#
# $HEADER$
#
headers += \
base/base.h
libmca_paffinity_la_SOURCES += \
base/paffinity_base_close.c \
base/paffinity_base_select.c \
base/paffinity_base_open.c \
base/paffinity_base_wrappers.c \
base/paffinity_base_service.c

308
opal/mca/paffinity/base/base.h
View File

@ -1,308 +0,0 @@
/*
* Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
* University Research and Technology
* Corporation. All rights reserved.
* Copyright (c) 2004-2006 The University of Tennessee and The University
* of Tennessee Research Foundation. All rights
* reserved.
* Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
* University of Stuttgart. All rights reserved.
* Copyright (c) 2004-2005 The Regents of the University of California.
* All rights reserved.
* Copyright (c) 2007-2010 Cisco Systems, Inc. All rights reserved.
* Copyright (c) 2010 Oracle and/or its affiliates. All rights reserved.
* $COPYRIGHT$
*
* Additional copyrights may follow
*
* $HEADER$
*/
#ifndef OPAL_PAFFINITY_BASE_H
#define OPAL_PAFFINITY_BASE_H
#include "opal_config.h"
#include "opal/mca/paffinity/paffinity.h"
/*
* Global functions for MCA overall paffinity open and close
*/
BEGIN_C_DECLS
/**
* Register MCA params for the paffinity base.
*
* @retval OPAL_SUCCESS Upon success
*
* This function is invoked by opal_register_params(). It registers
* some paffinity-wide MCA parameters.
*/
OPAL_DECLSPEC int opal_paffinity_base_register_params(void);
/**
* Initialize the paffinity MCA framework
*
* @retval OPAL_SUCCESS Upon success
* @retval OPAL_ERROR Upon failure
*
* This must be the first function invoked in the paffinity MCA
* framework. It initializes the paffinity MCA framework, finds
* and opens paffinity components, etc.
*
* This function is invoked during opal_init().
*
* This function fills in the internal global variable
* opal_paffinity_base_components_opened, which is a list of all
* paffinity components that were successfully opened. This
* variable should \em only be used by other paffinity base
* functions -- it is not considered a public interface member --
* and is only mentioned here for completeness.
*/
OPAL_DECLSPEC int opal_paffinity_base_open(void);
/**
* Select an available component.
*
* @return OPAL_SUCCESS Upon success.
* @return OPAL_NOT_FOUND If no component can be selected.
* @return OPAL_ERROR Upon other failure.
*
* This function invokes the selection process for paffinity components,
* which works as follows:
*
* - If the \em paffinity MCA parameter is not specified, the
* selection set is all available paffinity components.
* - If the \em paffinity MCA parameter is specified, the
* selection set is just that component.
* - All components in the selection set are queried to see if
* they want to run. All components that want to run are ranked
* by their priority and the highest priority component is
* selected. All non-selected components have their "close"
* function invoked to let them know that they were not selected.
* - The selected component will have its "init" function invoked to
* let it know that it was selected.
*
* If we fall through this entire process and no component is
* selected, then return OPAL_NOT_FOUND (this is not a fatal
* error).
*
* At the end of this process, we'll either have a single
* component that is selected and initialized, or no component was
* selected. If no component was selected, subsequent invocation
* of the paffinity wrapper functions will return an error.
*/
OPAL_DECLSPEC int opal_paffinity_base_select(void);
/**
* Set this process' affinity.
*
* @param cpumask PHYSICAL processor bitmask
*
* @retval OPAL_SUCCESS upon success
* @retval OPAL_NOT_FOUND if no paffinity components are available.
* @retval OPAL_ERROR upon other error.
*
* Set this process' affinity to the PHYSICAL CPU's specified in \em cpumask
*
* If no paffinity components were available, or if the
* opal_paffinity_base_select() was never invoked, OPAL_NOT_FOUND
* is returned.
*/
OPAL_DECLSPEC int opal_paffinity_base_set(opal_paffinity_base_cpu_set_t cpumask);
/**
* Get this process' affinity.
*
* @param cpumask Pointer to PHYSICAL processor bitmask
*
* @retval OPAL_SUCCESS upon success
* @retval OPAL_NOT_FOUND if no paffinity components are available.
* @retval OPAL_ERROR upon other error.
*
* Get this process' CPU affinity PHYSICAL bitmask and assign
* it to \em cpumask.
*
* If no paffinity components were available, or if the
* opal_paffinity_base_select() was never invoked, OPAL_NOT_FOUND
* is returned and cpumask is zeroed out.
*/
OPAL_DECLSPEC int opal_paffinity_base_get(opal_paffinity_base_cpu_set_t *cpumask);
/**
* Shut down the paffinity MCA framework.
*
* @retval OPAL_SUCCESS Always
*
* This function shuts down everything in the paffinity MCA
* framework, and is called during opal_finalize().
*
* It must be the last function invoked on the paffinity MCA
* framework.
*/
OPAL_DECLSPEC int opal_paffinity_base_close(void);
/**
* Returns mapping of PHYSICAL socket:core -> PHYSICAL processor id.
*
* @param physical_socket
* @param pyshical_core
* @param physical_processor_id
*
* @return int - OPAL_SUCCESS or OPAL_ERR_NOT_SUPPORTED if not
* supported
*/
OPAL_DECLSPEC int opal_paffinity_base_get_map_to_processor_id(int physical_socket, int physical_core, int *physical_processor_id);
/**
* Provides mapping of PHYSICAL processor id -> PHYSICAL socket:core.
*
* @param physical_processor_id
* @param physical_socket
* @param physical_core
*
* @return int - OPAL_SUCCESS or OPAL_ERR_NOT_SUPPORTED if not
* supported
*/
OPAL_DECLSPEC int opal_paffinity_base_get_map_to_socket_core(int physical_processor_id, int *physical_socket, int *physical_core);
/**
* Return the number of LOGICAL processors
*
* @param num_logical_processors
*
* @return int - OPAL_SUCCESS or OPAL_ERR_NOT_SUPPORTED if not
* supported
*/
OPAL_DECLSPEC int opal_paffinity_base_get_processor_info(int *num_logical_processors);
/**
* Return the number of LOGICAL sockets
*
* @param num_logical_sockets
*
* @return int - OPAL_SUCCESS or OPAL_ERR_NOT_SUPPORTED if not
* supported
*/
OPAL_DECLSPEC int opal_paffinity_base_get_socket_info(int *num_logical_sockets);
/**
* Return the number of LOGICAL cores for a given PHYSICAL socket
*
* @param physical_socket
* @param num_logical_cores
*
* @return int - OPAL_SUCCESS or OPAL_ERR_NOT_SUPPORTED if not
* supported
*/
OPAL_DECLSPEC int opal_paffinity_base_get_core_info(int physical_socket, int *num_logical_cores);
/**
* Return the PHYSICAL processor id that corresponds to the
* given LOGICAL processor id
*
* @param logical_processor_id
* @param physical_processor_id
*
* @return int - OPAL_SUCCESS or OPAL_ERR_*
*/
OPAL_DECLSPEC int opal_paffinity_base_get_physical_processor_id(int logical_processor_id, int *physical_processor_id);
/**
* Return the PHYSICAL socket ID that corresponds to the given
* LOGICAL socket ID.
*
* @param logical_socket_id
* @param physical_socket_id
*
* @return int - OPAL_SUCCESS or OPAL_ERR_*
*/
OPAL_DECLSPEC int opal_paffinity_base_get_physical_socket_id(int logical_socket_id, int *physical_socket_id);
/**
* Return the PHYSICAL core ID that corresponds to the given LOGICAL
* core id on the given PHYSICAL socket ID.
*
* @param physical_socket_id
* @param_logical_core_id
* @param physical_core_id
*
* @return int - OPAL_SUCCESS or OPAL_ERR_*
*/
OPAL_DECLSPEC int opal_paffinity_base_get_physical_core_id(int physical_socket_id, int logical_core_id, int *physical_core_id);
/* Print a char string representation of a cpu set */
OPAL_DECLSPEC char *opal_paffinity_base_print_binding(opal_paffinity_base_cpu_set_t cpumask);
/* Parse the binding string created by above function back into a cpu set */
OPAL_DECLSPEC int opal_paffinity_base_parse_binding(char *binding, opal_paffinity_base_cpu_set_t *cpumask);
/**
* Indication of whether a component was successfully selected or
* not
*/
OPAL_DECLSPEC extern bool opal_paffinity_base_selected;
/**
* Global component struct for the selected component
*/
OPAL_DECLSPEC extern const opal_paffinity_base_component_2_0_1_t
*opal_paffinity_base_component;
/**
* Global module struct for the selected module
*/
OPAL_DECLSPEC extern const opal_paffinity_base_module_1_1_0_t
*opal_paffinity_base_module;
/**
* Indicator as to whether the list of opened paffinity components
* is valid or not.
*/
OPAL_DECLSPEC extern bool opal_paffinity_base_components_opened_valid;
/**
* List of all opened components; created when the paffinity
* framework is initialized and destroyed when we reduce the list
* to all available paffinity components.
*/
OPAL_DECLSPEC extern opal_list_t opal_paffinity_base_components_opened;
/**
* Assigning slot_list to process
*/
OPAL_DECLSPEC int opal_paffinity_base_slot_list_set(long rank, char *slot_str,
opal_paffinity_base_cpu_set_t *cpumask);
/**
* Make a prettyprint string for a cset.
*/
OPAL_DECLSPEC int opal_paffinity_base_cset2str(char *str, int len,
opal_paffinity_base_cpu_set_t *cset);
/**
* Make a prettyprint string for a cset with a map format.
*/
OPAL_DECLSPEC int opal_paffinity_base_cset2mapstr(char *str, int len,
opal_paffinity_base_cpu_set_t *cset);
/**
* Debugging output stream
*/
OPAL_DECLSPEC extern int opal_paffinity_base_output;
/**
* Flag indicating whether or not processor affinity is to be enabled
*/
OPAL_DECLSPEC extern bool opal_paffinity_alone;
OPAL_DECLSPEC extern char *opal_paffinity_base_slot_list;
/* Indicator that process affinity has been set for this process - i.e.,
* that OPAL set it either directly or when launched
*/
OPAL_DECLSPEC extern bool opal_paffinity_base_bound;
/* String passed down from launcher that contains applied binding */
OPAL_DECLSPEC extern char *opal_paffinity_base_applied_binding;
END_C_DECLS
#endif /* OPAL_BASE_PAFFINITY_H */

50
opal/mca/paffinity/base/paffinity_base_close.c
View File

@ -1,50 +0,0 @@
/*
* Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
* University Research and Technology
* Corporation. All rights reserved.
* Copyright (c) 2004-2006 The University of Tennessee and The University
* of Tennessee Research Foundation. All rights
* reserved.
* Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
* University of Stuttgart. All rights reserved.
* Copyright (c) 2004-2005 The Regents of the University of California.
* All rights reserved.
* Copyright (c) 2007 Cisco Systems, Inc. All rights reserved.
* $COPYRIGHT$
*
* Additional copyrights may follow
*
* $HEADER$
*/
#include "opal_config.h"
#include "opal/constants.h"
#include "opal/mca/mca.h"
#include "opal/mca/base/base.h"
#include "opal/mca/paffinity/paffinity.h"
#include "opal/mca/paffinity/base/base.h"
int opal_paffinity_base_close(void)
{
/* If there is a selected paffinity module, finalize it */
if (NULL != opal_paffinity_base_module &&
NULL != opal_paffinity_base_module->paff_module_finalize) {
opal_paffinity_base_module->paff_module_finalize();
}
/* Close all components that are still open (this should only
happen during ompi_info). */
if (opal_paffinity_base_components_opened_valid) {
mca_base_components_close(opal_paffinity_base_output,
&opal_paffinity_base_components_opened, NULL);
OBJ_DESTRUCT(&opal_paffinity_base_components_opened);
opal_paffinity_base_components_opened_valid = false;
}
/* All done */
return OPAL_SUCCESS;
}

140
opal/mca/paffinity/base/paffinity_base_open.c
View File

@ -1,140 +0,0 @@
/*
* Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
* University Research and Technology
* Corporation. All rights reserved.
* Copyright (c) 2004-2005 The University of Tennessee and The University
* of Tennessee Research Foundation. All rights
* reserved.
* Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
* University of Stuttgart. All rights reserved.
* Copyright (c) 2004-2005 The Regents of the University of California.
* All rights reserved.
* Copyright (c) 2008-2010 Cisco Systems, Inc. All rights reserved.
* $COPYRIGHT$
*
* Additional copyrights may follow
*
* $HEADER$
*/
#include "opal_config.h"
#include "opal/constants.h"
#include "opal/util/output.h"
#include "opal/util/show_help.h"
#include "opal/mca/mca.h"
#include "opal/mca/base/base.h"
#include "opal/mca/base/mca_base_param.h"
#include "opal/mca/paffinity/paffinity.h"
#include "opal/mca/paffinity/base/base.h"
/*
* The following file was created by configure. It contains extern
* statements and the definition of an array of pointers to each
* component's public mca_base_component_t struct.
*/
#include "opal/mca/paffinity/base/static-components.h"
/*
* Globals
*/
OPAL_DECLSPEC int opal_paffinity_base_output = -1;
bool opal_paffinity_base_components_opened_valid = false;
opal_list_t opal_paffinity_base_components_opened;
bool opal_paffinity_alone = false;
char *opal_paffinity_base_slot_list;
bool opal_paffinity_base_bound;
char *opal_paffinity_base_applied_binding;
/*
* Register some paffinity-wide MCA params
*/
int opal_paffinity_base_register_params(void)
{
int value, id;
static int been_here = 0;
/* We may get called twice; be harmless in that case. */
if (1 == been_here) {
return OPAL_SUCCESS;
}
been_here = 1;
/* Debugging / verbose output */
mca_base_param_reg_int_name("paffinity", "base_verbose",
"Verbosity level of the paffinity framework",
false, false,
0, &value);
if (0 != value) {
opal_paffinity_base_output = opal_output_open(NULL);
} else {
opal_paffinity_base_output = -1;
}
/*** DEPRECATED - NO LONGER FUNCTIONAL, BUT MPIRUN
* WILL OUTPUT WARNING FOR NOW
***/
id = mca_base_param_reg_int_name("opal", "paffinity_alone",
"If nonzero, assume that this job is the only (set of) process(es) running on each node and bind processes to processors, starting with processor ID 0",
false, false,
-1, NULL);
/* register the historical mpi_paffinity_alone synonym, but don't
* declare it deprecated so we don't scare the users.
*
* Yes, this breaks the abstraction barrier, but as indicated
* on the developer list....live with it. :-)
*/
mca_base_param_reg_syn_name(id, "mpi", "paffinity_alone", false);
mca_base_param_lookup_int(id, &value);
/* we just want to know if someone set it */
if (-1 != value) {
opal_paffinity_alone = true;
} else {
opal_paffinity_alone = false;
}
mca_base_param_reg_int_name("paffinity", "base_bound",
"Process affinity was set by an external entity",
true, false,
false, &value);
opal_paffinity_base_bound = OPAL_INT_TO_BOOL(value);
mca_base_param_reg_string_name("paffinity", "base_applied_binding",
"Binding from launcher",
true, false,
NULL, &opal_paffinity_base_applied_binding);
return OPAL_SUCCESS;
}
/*
* Function for finding and opening either all MCA components, or the one
* that was specifically requested via a MCA parameter.
*/
int opal_paffinity_base_open(void)
{
opal_paffinity_base_components_opened_valid = false;
mca_base_param_reg_string_name("opal", "paffinity_base_slot_list",
"Used to set list of processor IDs to bind MPI processes to (e.g., used in conjunction with rank files)",
true, false, NULL, &opal_paffinity_base_slot_list);
/* Open up all available components */
if (OPAL_SUCCESS !=
mca_base_components_open("paffinity", opal_paffinity_base_output,
mca_paffinity_base_static_components,
&opal_paffinity_base_components_opened,
true)) {
return OPAL_ERROR;
}
opal_paffinity_base_components_opened_valid = true;
/* All done */
return OPAL_SUCCESS;
}

68
opal/mca/paffinity/base/paffinity_base_select.c
View File

@ -1,68 +0,0 @@
/*
* Copyright (c) 2004-2008 The Trustees of Indiana University and Indiana
* University Research and Technology
* Corporation. All rights reserved.
* Copyright (c) 2004-2006 The University of Tennessee and The University
* of Tennessee Research Foundation. All rights
* reserved.
* Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
* University of Stuttgart. All rights reserved.
* Copyright (c) 2004-2005 The Regents of the University of California.
* All rights reserved.
* Copyright (c) 2007-2010 Cisco Systems, Inc. All rights reserved.
* $COPYRIGHT$
*
* Additional copyrights may follow
*
* $HEADER$
*/
#include "opal_config.h"
#include "opal/constants.h"
#include "opal/mca/mca.h"
#include "opal/mca/base/base.h"
#include "opal/mca/base/mca_base_param.h"
#include "opal/mca/paffinity/paffinity.h"
#include "opal/mca/paffinity/base/base.h"
/*
* Globals
*/
bool opal_paffinity_base_selected = false;
const opal_paffinity_base_component_2_0_1_t *opal_paffinity_base_component = NULL;
const opal_paffinity_base_module_1_1_0_t *opal_paffinity_base_module = NULL;
int opal_paffinity_base_select(void)
{
int ret = OPAL_SUCCESS;
opal_paffinity_base_component_2_0_1_t *best_component = NULL;
opal_paffinity_base_module_1_1_0_t *best_module = NULL;
/*
* Select the best component
*/
if( OPAL_SUCCESS != mca_base_select("paffinity", opal_paffinity_base_output,
&opal_paffinity_base_components_opened,
(mca_base_module_t **) &best_module,
(mca_base_component_t **) &best_component) ) {
/* It is okay if we don't find a module - we will report an
* error if/when someone tries to actually use affinity
*/
return OPAL_SUCCESS;
}
/* Save the winner */
opal_paffinity_base_component = best_component;
opal_paffinity_base_module = best_module;
opal_paffinity_base_selected = true;
/* Initialize the winner */
if (NULL != opal_paffinity_base_module) {
ret = opal_paffinity_base_module->paff_module_init();
}
return ret;
}

665
opal/mca/paffinity/base/paffinity_base_service.c
View File

@ -1,665 +0,