ports/openmpi
ports/openmpi
1
1
Форкнуть 0
Код Релизы Активность

Complete implementation of the ess.proc_get_locality API. Up to this point, the API was only capable of telling if the specified proc was sharing a node with you. However, the returned value was capable of telling you much more detailed info - e.g., if the proc shares a socket, a cache, or numa node. We just didn't have the data to provide that detail.

Просмотр исходного кода 
Use hwloc to obtain the cpuset for each process during mpi_init, and share that info in the modex. As it arrives, use a new opal_hwloc_base utility function to parse the value against the local proc's cpuset and determine where they overlap. Cache the value in the pmap object as it may be referenced multiple times.

Thus, the return value from orte_ess.proc_get_locality is a 16-bit bitmask that describes the resources being shared with you. This bitmask can be tested using the macros in opal/mca/paffinity/paffinity.h

Locality is available for all procs, whether launched via mpirun or directly with an external launcher such as slurm or aprun.

This commit was SVN r25331.
Этот коммит содержится в:
Ralph Castain 2011-10-19 20:18:14 +00:00
родитель 1bc5da0911
Коммит b44f8d4b28
36 изменённых файлов: 870 добавлений и 1424 удалений
Показать статистику Скачать Patch файл Скачать Diff файл Показать все файлы Свернуть все файлы

2
ompi/proc/proc.h
Просмотреть файл

@ -63,7 +63,7 @@ struct ompi_proc_t {
/** architecture of this process */
uint32_t proc_arch;
/** flags for this proc */
uint8_t proc_flags;
uint16_t proc_flags;
/** Base convertor for the proc described by this process */
struct opal_convertor_t* proc_convertor;
/** A pointer to the name of this host - data is

22
ompi/runtime/ompi_mpi_init.c
Просмотреть файл

@ -37,6 +37,7 @@
#include "mpi.h"
#include "opal/class/opal_list.h"
#include "opal/mca/base/base.h"
#include "opal/mca/hwloc/base/base.h"
#include "opal/mca/paffinity/base/base.h"
#include "opal/mca/maffinity/base/base.h"
#include "opal/runtime/opal_progress.h"
@ -411,6 +412,7 @@ int ompi_mpi_init(int argc, char **argv, int requested, int *provided)
goto error;
}
#if OPAL_HAVE_HWLOC
/* If orte_init() didn't fill in opal_hwloc_topology, then we need
to go fill it in ourselves. */
if (NULL == opal_hwloc_topology) {
@ -419,7 +421,8 @@ int ompi_mpi_init(int argc, char **argv, int requested, int *provided)
return OPAL_ERR_NOT_SUPPORTED;
}
}
#endif
/* Once we've joined the RTE, see if any MCA parameters were
passed to the MPI level */
@ -488,9 +491,10 @@ int ompi_mpi_init(int argc, char **argv, int requested, int *provided)
int phys_cpu;
orte_node_rank_t nrank;
if (ORTE_NODE_RANK_INVALID == (nrank = orte_ess.get_node_rank(ORTE_PROC_MY_NAME))) {
ret = OMPI_ERR_BAD_PARAM;
error = "Could not get node rank - cannot set processor affinity";
goto error;
/* this is okay - we probably were direct-launched, which means
* we won't get our node rank until the modex. So just ignore
*/
goto MOVEON;
}
OPAL_PAFFINITY_CPU_ZERO(mask);
ret = opal_paffinity_base_get_physical_processor_id(nrank, &phys_cpu);
@ -520,7 +524,15 @@ int ompi_mpi_init(int argc, char **argv, int requested, int *provided)
}
}
}
MOVEON:
#if OPAL_HAVE_HWLOC
/* get or update our local cpuset - it will get used multiple
* times, so it's more efficient to keep a global copy
*/
opal_hwloc_base_get_local_cpuset();
#endif
/* If we were able to set processor affinity, try setting up
memory affinity */
if (!opal_maffinity_setup && paffinity_enabled) {

7
opal/mca/hwloc/base/base.h
Просмотреть файл

@ -13,6 +13,7 @@
#include "opal_config.h"
#include "opal/dss/dss_types.h"
#include "opal/mca/paffinity/paffinity.h"
#include "opal/mca/hwloc/hwloc.h"
@ -107,6 +108,12 @@ OPAL_DECLSPEC int opal_hwloc_base_report_bind_failure(const char *file,
const char *msg,
int rc);
OPAL_DECLSPEC opal_paffinity_locality_t opal_hwloc_base_get_relative_locality(hwloc_topology_t topo,
hwloc_cpuset_t peer1,
hwloc_cpuset_t peer2);
OPAL_DECLSPEC void opal_hwloc_base_get_local_cpuset(void);
#endif
/**

6
opal/mca/hwloc/base/hwloc_base_close.c
Просмотреть файл

@ -34,6 +34,12 @@ int opal_hwloc_base_close(void)
OBJ_RELEASE(item);
}
OBJ_DESTRUCT(&opal_hwloc_base_components);
/* free memory */
if (NULL != opal_hwloc_my_cpuset) {
hwloc_bitmap_free(opal_hwloc_my_cpuset);
opal_hwloc_my_cpuset = NULL;
}
}
#endif

1
opal/mca/hwloc/base/hwloc_base_open.c
Просмотреть файл

@ -38,6 +38,7 @@ opal_list_t opal_hwloc_base_components;
bool opal_hwloc_base_inited = false;
#if OPAL_HAVE_HWLOC
hwloc_topology_t opal_hwloc_topology=NULL;
hwloc_cpuset_t opal_hwloc_my_cpuset=NULL;
#endif
opal_hwloc_base_map_t opal_hwloc_base_map = OPAL_HWLOC_BASE_MAP_NONE;
opal_hwloc_base_mbfa_t opal_hwloc_base_mbfa = OPAL_HWLOC_BASE_MBFA_ERROR;

160
opal/mca/hwloc/base/hwloc_base_util.c
Просмотреть файл

@ -28,14 +28,38 @@
#endif
#include "opal/constants.h"
#include "opal/util/output.h"
#include "opal/util/show_help.h"
#include "opal/mca/paffinity/paffinity.h"
#include "opal/mca/hwloc/hwloc.h"
#include "opal/mca/hwloc/base/base.h"
void opal_hwloc_base_get_local_cpuset(void)
{
hwloc_obj_t root;
if (NULL != opal_hwloc_topology) {
if (NULL == opal_hwloc_my_cpuset) {
opal_hwloc_my_cpuset = hwloc_bitmap_alloc();
}
/* get the cpus we are bound to */
hwloc_get_cpubind(opal_hwloc_topology, opal_hwloc_my_cpuset, HWLOC_CPUBIND_PROCESS);
/* if the cpuset is empty, then we are not bound */
if (hwloc_bitmap_iszero(opal_hwloc_my_cpuset)) {
OPAL_OUTPUT_VERBOSE((5, opal_hwloc_base_output,
"hwloc:base:get_local_cpuset MY LOCAL CPUSET WAS ZERO - NOT BOUND"));
/* just insert the cpuset for the root object as we are unbound */
root = hwloc_get_root_obj(opal_hwloc_topology);
hwloc_bitmap_copy(opal_hwloc_my_cpuset, root->cpuset);
}
}
}
int opal_hwloc_base_report_bind_failure(const char *file,
int line,
const char *msg, int rc)
int line,
const char *msg, int rc)
{
static int already_reported = 0;
@ -54,3 +78,135 @@ int opal_hwloc_base_report_bind_failure(const char *file,
return OPAL_SUCCESS;
}
static void recurse_locality(hwloc_obj_t obj,
opal_paffinity_locality_t *locality,
hwloc_cpuset_t peer1,
hwloc_cpuset_t peer2)
{
unsigned i;
hwloc_obj_t obj2;
#if OPAL_ENABLE_DEBUG
{
char debug[256], debug1[256], debug2[256];
hwloc_bitmap_list_snprintf(debug, 256, obj->cpuset);
hwloc_bitmap_list_snprintf(debug1, 256, peer1);
hwloc_bitmap_list_snprintf(debug2, 256, peer2);
OPAL_OUTPUT_VERBOSE((5, opal_hwloc_base_output,
"Type: %s Recurse: OBJ: %s PEER1: %s PEER2: %s",
hwloc_obj_type_string(obj->type), debug, debug1, debug2));
}
#endif
/* is peer1 on this object? */
if (!hwloc_bitmap_intersects(obj->cpuset, peer1)) {
/* no - move on */
OPAL_OUTPUT_VERBOSE((5, opal_hwloc_base_output,
"hwloc:base:recurs_locality Peer1 does not intersect"));
goto MOVEON;
}
/* is peer2 on this object? */
if (!hwloc_bitmap_intersects(obj->cpuset, peer2)) {
OPAL_OUTPUT_VERBOSE((5, opal_hwloc_base_output,
"hwloc:base:recurs_locality Peer2 does not intersect"));
/* no - move on */
goto MOVEON;
}
/* we share this object, so we must share locality
* at this resource type - figure out what that is
* and flag it
*/
switch (obj->type) {
case HWLOC_OBJ_PU:
OPAL_OUTPUT_VERBOSE((5, opal_hwloc_base_output, "hwloc:base:recurs_locality ON PU"));
*locality |= OPAL_PROC_ON_HWTHREAD;
break;
case HWLOC_OBJ_CORE:
OPAL_OUTPUT_VERBOSE((5, opal_hwloc_base_output, "hwloc:base:recurs_locality ON CORE"));
*locality |= OPAL_PROC_ON_CORE;
break;
case HWLOC_OBJ_CACHE:
OPAL_OUTPUT_VERBOSE((5, opal_hwloc_base_output, "hwloc:base:recurs_locality ON CACHE"));
/* hwloc treats caches as a special case, so we have
* to figure out which level of cache we have here
*/
if (1 == obj->attr->cache.depth) {
*locality |= OPAL_PROC_ON_L1CACHE;
} else if (2 == obj->attr->cache.depth) {
*locality |= OPAL_PROC_ON_L2CACHE;
} else if (3 == obj->attr->cache.depth) {
*locality |= OPAL_PROC_ON_L3CACHE;
}
break;
case HWLOC_OBJ_SOCKET:
OPAL_OUTPUT_VERBOSE((5, opal_hwloc_base_output, "hwloc:base:recurs_locality ON SOCKET"));
*locality |= OPAL_PROC_ON_SOCKET;
break;
case HWLOC_OBJ_NODE:
OPAL_OUTPUT_VERBOSE((5, opal_hwloc_base_output, "hwloc:base:recurs_locality ON NUMA"));
*locality |= OPAL_PROC_ON_NUMA;
break;
default:
/* ignore */
break;
}
MOVEON:
/* step to next object - it would be nice if we could
* drop down the depth chart and avoid possible duplication
* of checks. However, hwloc treats all caches as being at
* a common depth, which makes this impossible. Fortunately,
* the cpuset intersect test is lightweight, and we are not
* used in performance-critical paths, so the penalty is small
*/
for (i=0; i < obj->arity; i++) {
obj2 = obj->children[i];
/* process the child object */
recurse_locality(obj2, locality, peer1, peer2);
}
}
opal_paffinity_locality_t opal_hwloc_base_get_relative_locality(hwloc_topology_t topo,
hwloc_cpuset_t peer1,
hwloc_cpuset_t peer2)
{
opal_paffinity_locality_t locality;
hwloc_obj_t root;
/* start with what we know - they share a node on a cluster
* NOTE: we may alter that latter part as hwloc's ability to
* sense multi-cu, multi-cluster systems grows
*/
locality = OPAL_PROC_ON_CLUSTER | OPAL_PROC_ON_CU | OPAL_PROC_ON_NODE | OPAL_PROC_ON_BOARD;
/* start at the root object */
root = hwloc_get_root_obj(topo);
/* descend down the topology */
recurse_locality(root, &locality, peer1, peer2);
/* NOTE: hwloc isn't able to find cores on all platforms. Example:
PPC64 running RHEL 5.4 (linux kernel 2.6.18) only reports NUMA
nodes and PU's. Fine.
However, note that hwloc_get_obj_by_type() will return NULL in
2 (effectively) different cases:
- no objects of the requested type were found
- the Nth object of the requested type was not found
So see if we can find *any* cores by looking for the 0th core.
*/
if (NULL == hwloc_get_obj_by_type(topo, HWLOC_OBJ_CORE, 0)) {
/* nope - so if the two peer's share a HWTHREAD, also
* declare them as sharing a core
*/
if (OPAL_PROC_ON_LOCAL_HWTHREAD(locality)) {
locality |= OPAL_PROC_ON_CORE;
}
}
return locality;
}

2
opal/mca/hwloc/hwloc.h
Просмотреть файл

@ -67,6 +67,8 @@ END_C_DECLS
#include MCA_hwloc_IMPLEMENTATION_HEADER
OPAL_DECLSPEC extern hwloc_topology_t opal_hwloc_topology;
OPAL_DECLSPEC extern hwloc_cpuset_t opal_hwloc_my_cpuset;
#endif
#endif /* OPAL_HWLOC_H_ */

28
opal/mca/paffinity/paffinity.h
Просмотреть файл

@ -96,18 +96,32 @@
#include "opal/mca/base/base.h"
/* ******************************************************************** */
typedef uint16_t opal_paffinity_locality_t;
/** Process locality definitions */
#define OPAL_PROC_ON_CLUSTER 0x10
#define OPAL_PROC_ON_CU 0x08
#define OPAL_PROC_ON_NODE 0x04
#define OPAL_PROC_ON_BOARD 0x02
#define OPAL_PROC_ON_SOCKET 0x01
#define OPAL_PROC_NON_LOCAL 0x00
#define OPAL_PROC_ALL_LOCAL 0x1f
#define OPAL_PROC_LOCALITY_UNKNOWN 0x0000
#define OPAL_PROC_NON_LOCAL 0x8000
#define OPAL_PROC_ON_CLUSTER 0x0400
#define OPAL_PROC_ON_CU 0x0200
#define OPAL_PROC_ON_NODE 0x0100
#define OPAL_PROC_ON_BOARD 0x0080
#define OPAL_PROC_ON_NUMA 0x0040
#define OPAL_PROC_ON_SOCKET 0x0020
#define OPAL_PROC_ON_L3CACHE 0x0010
#define OPAL_PROC_ON_L2CACHE 0x0008
#define OPAL_PROC_ON_L1CACHE 0x0004
#define OPAL_PROC_ON_CORE 0x0002
#define OPAL_PROC_ON_HWTHREAD 0x0001
#define OPAL_PROC_ALL_LOCAL 0x0fff
/** Process locality macros */
#define OPAL_PROC_ON_LOCAL_HWTHREAD(n) ((n) & OPAL_PROC_ON_HWTHREAD)
#define OPAL_PROC_ON_LOCAL_CORE(n) ((n) & OPAL_PROC_ON_CORE)
#define OPAL_PROC_ON_LOCAL_L1CACHE(n) ((n) & OPAL_PROC_ON_L1CACHE)
#define OPAL_PROC_ON_LOCAL_L2CACHE(n) ((n) & OPAL_PROC_ON_L2CACHE)
#define OPAL_PROC_ON_LOCAL_L3CACHE(n) ((n) & OPAL_PROC_ON_L3CACHE)
#define OPAL_PROC_ON_LOCAL_SOCKET(n) ((n) & OPAL_PROC_ON_SOCKET)
#define OPAL_PROC_ON_LOCAL_NUMA(n) ((n) & OPAL_PROC_ON_NUMA)
#define OPAL_PROC_ON_LOCAL_BOARD(n) ((n) & OPAL_PROC_ON_BOARD)
#define OPAL_PROC_ON_LOCAL_NODE(n) ((n) & OPAL_PROC_ON_NODE)
#define OPAL_PROC_ON_LOCAL_CU(n) ((n) & OPAL_PROC_ON_CU)

157
orte/mca/ess/alps/ess_alps_module.c
Просмотреть файл

@ -45,26 +45,19 @@ static int alps_set_name(void);
static int rte_init(void);
static int rte_finalize(void);
static uint8_t proc_get_locality(orte_process_name_t *proc);
static orte_vpid_t proc_get_daemon(orte_process_name_t *proc);
static char* proc_get_hostname(orte_process_name_t *proc);
static orte_local_rank_t proc_get_local_rank(orte_process_name_t *proc);
static orte_node_rank_t proc_get_node_rank(orte_process_name_t *proc);
static int update_pidmap(opal_byte_object_t *bo);
static int update_nidmap(opal_byte_object_t *bo);
orte_ess_base_module_t orte_ess_alps_module = {
rte_init,
rte_finalize,
orte_ess_base_app_abort,
proc_get_locality,
proc_get_daemon,
proc_get_hostname,
proc_get_local_rank,
proc_get_node_rank,
orte_ess_base_proc_get_locality,
orte_ess_base_proc_get_daemon,
orte_ess_base_proc_get_hostname,
orte_ess_base_proc_get_local_rank,
orte_ess_base_proc_get_node_rank,
orte_ess_base_proc_get_epoch,
update_pidmap,
update_nidmap,
orte_ess_base_update_pidmap,
orte_ess_base_update_nidmap,
NULL /* ft_event */
};
@ -191,142 +184,6 @@ static int rte_finalize(void)
return ret;
}
static uint8_t proc_get_locality(orte_process_name_t *proc)
{
orte_nid_t *nid;
if (NULL == (nid = orte_util_lookup_nid(proc))) {
ORTE_ERROR_LOG(ORTE_ERR_NOT_FOUND);
return OPAL_PROC_NON_LOCAL;
}
if (nid->daemon == ORTE_PROC_MY_DAEMON->vpid) {
OPAL_OUTPUT_VERBOSE((2, orte_ess_base_output,
"%s ess:alps: proc %s on LOCAL NODE",
ORTE_NAME_PRINT(ORTE_PROC_MY_NAME),
ORTE_NAME_PRINT(proc)));
return (OPAL_PROC_ON_NODE | OPAL_PROC_ON_CU | OPAL_PROC_ON_CLUSTER);
}
OPAL_OUTPUT_VERBOSE((2, orte_ess_base_output,
"%s ess:alps: proc %s is REMOTE",
ORTE_NAME_PRINT(ORTE_PROC_MY_NAME),
ORTE_NAME_PRINT(proc)));
return OPAL_PROC_NON_LOCAL;
}
static orte_vpid_t proc_get_daemon(orte_process_name_t *proc)
{
orte_nid_t *nid;
if( ORTE_JOBID_IS_DAEMON(proc->jobid) ) {
return proc->vpid;
}
if (NULL == (nid = orte_util_lookup_nid(proc))) {
return ORTE_VPID_INVALID;
}
OPAL_OUTPUT_VERBOSE((2, orte_ess_base_output,
"%s ess:alps: proc %s is hosted by daemon %s",
ORTE_NAME_PRINT(ORTE_PROC_MY_NAME),
ORTE_NAME_PRINT(proc),
ORTE_VPID_PRINT(nid->daemon)));
return nid->daemon;
}
static char* proc_get_hostname(orte_process_name_t *proc)
{
orte_nid_t *nid;
if (NULL == (nid = orte_util_lookup_nid(proc))) {
ORTE_ERROR_LOG(ORTE_ERR_NOT_FOUND);
return NULL;
}
OPAL_OUTPUT_VERBOSE((2, orte_ess_base_output,
"%s ess:alps: proc %s is on host %s",
ORTE_NAME_PRINT(ORTE_PROC_MY_NAME),
ORTE_NAME_PRINT(proc),
nid->name));
return nid->name;
}
static orte_local_rank_t proc_get_local_rank(orte_process_name_t *proc)
{
orte_pmap_t *pmap;
if (NULL == (pmap = orte_util_lookup_pmap(proc))) {
ORTE_ERROR_LOG(ORTE_ERR_NOT_FOUND);
return ORTE_LOCAL_RANK_INVALID;
}
OPAL_OUTPUT_VERBOSE((2, orte_ess_base_output,
"%s ess:alps: proc %s has local rank %d",
ORTE_NAME_PRINT(ORTE_PROC_MY_NAME),
ORTE_NAME_PRINT(proc),
(int)pmap->local_rank));
return pmap->local_rank;
}
static orte_node_rank_t proc_get_node_rank(orte_process_name_t *proc)
{
orte_pmap_t *pmap;
orte_ns_cmp_bitmask_t mask;
mask = ORTE_NS_CMP_ALL;
/* is this me? */
if (OPAL_EQUAL == orte_util_compare_name_fields(mask, proc, ORTE_PROC_MY_NAME)) {
/* yes it is - reply with my rank. This is necessary
* because the pidmap will not have arrived when I
* am starting up, and if we use static ports, then
* I need to know my node rank during init
*/
return my_node_rank;
}
if (NULL == (pmap = orte_util_lookup_pmap(proc))) {
ORTE_ERROR_LOG(ORTE_ERR_NOT_FOUND);
return ORTE_NODE_RANK_INVALID;
}
OPAL_OUTPUT_VERBOSE((2, orte_ess_base_output,
"%s ess:alps: proc %s has node rank %d",
ORTE_NAME_PRINT(ORTE_PROC_MY_NAME),
ORTE_NAME_PRINT(proc),
(int)pmap->node_rank));
return pmap->node_rank;
}
static int update_pidmap(opal_byte_object_t *bo)
{
int ret;
/* build the pmap */
if (ORTE_SUCCESS != (ret = orte_util_decode_pidmap(bo))) {
ORTE_ERROR_LOG(ret);
}
return ret;
}
static int update_nidmap(opal_byte_object_t *bo)
{
int rc;
/* decode the nidmap - the util will know what to do */
if (ORTE_SUCCESS != (rc = orte_util_decode_nodemap(bo))) {
ORTE_ERROR_LOG(rc);
}
return rc;
}
static int alps_set_name(void)
{
int rc;

3
orte/mca/ess/base/Makefile.am
Просмотреть файл

@ -34,6 +34,7 @@ libmca_ess_la_SOURCES += \
base/ess_base_std_tool.c \
base/ess_base_std_app.c \
base/ess_base_std_orted.c \
base/ess_base_std_prolog.c
base/ess_base_std_prolog.c \
base/ess_base_fns.c
endif

11
orte/mca/ess/base/base.h
Просмотреть файл

@ -9,6 +9,7 @@
* University of Stuttgart. All rights reserved.
* Copyright (c) 2004-2005 The Regents of the University of California.
* All rights reserved.
* Copyright (c) 2011 Cisco Systems, Inc. All rights reserved.
* $COPYRIGHT$
*
* Additional copyrights may follow
@ -25,7 +26,7 @@
#include "orte/types.h"
#include "opal/mca/mca.h"
#include "opal/dss/dss_types.h"
#include "orte/mca/ess/ess.h"
@ -84,6 +85,14 @@ ORTE_DECLSPEC int orte_ess_base_orted_finalize(void);
ORTE_DECLSPEC int orte_ess_base_query_sys_info(char *node, char **keys, opal_list_t *values);
ORTE_DECLSPEC opal_paffinity_locality_t orte_ess_base_proc_get_locality(orte_process_name_t *proc);
ORTE_DECLSPEC orte_vpid_t orte_ess_base_proc_get_daemon(orte_process_name_t *proc);
ORTE_DECLSPEC char* orte_ess_base_proc_get_hostname(orte_process_name_t *proc);
ORTE_DECLSPEC orte_local_rank_t orte_ess_base_proc_get_local_rank(orte_process_name_t *proc);
ORTE_DECLSPEC orte_node_rank_t orte_ess_base_proc_get_node_rank(orte_process_name_t *proc);
ORTE_DECLSPEC int orte_ess_base_update_pidmap(opal_byte_object_t *bo);
ORTE_DECLSPEC int orte_ess_base_update_nidmap(opal_byte_object_t *bo);
/*
* Put functions
*/

164
orte/mca/ess/base/ess_base_fns.c Обычный файл
Просмотреть файл

@ -0,0 +1,164 @@
/*
* Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
* University Research and Technology
* Corporation. All rights reserved.
* Copyright (c) 2004-2011 The University of Tennessee and The University
* of Tennessee Research Foundation. All rights
* reserved.
* Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
* University of Stuttgart. All rights reserved.
* Copyright (c) 2004-2005 The Regents of the University of California.
* All rights reserved.
* Copyright (c) 2011 Cisco Systems, Inc. All rights reserved.
* $COPYRIGHT$
*
* Additional copyrights may follow
*
* $HEADER$
*/
#include "orte_config.h"
#include "orte/constants.h"
#ifdef HAVE_UNISTD_H
#include <unistd.h>
#endif
#include <stdlib.h>
#include <errno.h>
#include "opal/util/output.h"
#include "opal/mca/paffinity/paffinity.h"
#include "orte/mca/errmgr/errmgr.h"
#include "orte/util/name_fns.h"
#include "orte/util/nidmap.h"
#include "orte/util/proc_info.h"
#include "orte/runtime/orte_globals.h"
#include "orte/mca/ess/base/base.h"
opal_paffinity_locality_t orte_ess_base_proc_get_locality(orte_process_name_t *proc)
{
orte_pmap_t *pmap;
if (NULL == (pmap = orte_util_lookup_pmap(proc))) {
ORTE_ERROR_LOG(ORTE_ERR_NOT_FOUND);
return OPAL_PROC_NON_LOCAL;
}
return pmap->locality;
}
orte_vpid_t orte_ess_base_proc_get_daemon(orte_process_name_t *proc)
{
orte_nid_t *nid;
if( ORTE_JOBID_IS_DAEMON(proc->jobid) ) {
return proc->vpid;
}
if (NULL == (nid = orte_util_lookup_nid(proc))) {
return ORTE_VPID_INVALID;
}
OPAL_OUTPUT_VERBOSE((5, orte_ess_base_output,
"%s ess:base: proc %s is hosted by daemon %s",
ORTE_NAME_PRINT(ORTE_PROC_MY_NAME),
ORTE_NAME_PRINT(proc),
ORTE_VPID_PRINT(nid->daemon)));
return nid->daemon;
}
char* orte_ess_base_proc_get_hostname(orte_process_name_t *proc)
{
orte_nid_t *nid;
if (NULL == (nid = orte_util_lookup_nid(proc))) {
ORTE_ERROR_LOG(ORTE_ERR_NOT_FOUND);
return NULL;
}
OPAL_OUTPUT_VERBOSE((5, orte_ess_base_output,
"%s ess:base: proc %s is on host %s",
ORTE_NAME_PRINT(ORTE_PROC_MY_NAME),
ORTE_NAME_PRINT(proc),
nid->name));
return nid->name;
}
orte_local_rank_t orte_ess_base_proc_get_local_rank(orte_process_name_t *proc)
{
orte_pmap_t *pmap;
if (NULL == (pmap = orte_util_lookup_pmap(proc))) {
ORTE_ERROR_LOG(ORTE_ERR_NOT_FOUND);
return ORTE_LOCAL_RANK_INVALID;
}
OPAL_OUTPUT_VERBOSE((5, orte_ess_base_output,
"%s ess:base: proc %s has local rank %d",
ORTE_NAME_PRINT(ORTE_PROC_MY_NAME),
ORTE_NAME_PRINT(proc),
(int)pmap->local_rank));
return pmap->local_rank;
}
orte_node_rank_t orte_ess_base_proc_get_node_rank(orte_process_name_t *proc)
{
orte_pmap_t *pmap;
orte_ns_cmp_bitmask_t mask;
mask = ORTE_NS_CMP_JOBID | ORTE_NS_CMP_VPID;
/* is this me? */
if (OPAL_EQUAL == orte_util_compare_name_fields(mask, proc, ORTE_PROC_MY_NAME)) {
/* yes it is - reply with my rank. This is necessary
* because the pidmap will not have arrived when I
* am starting up, and if we use static ports, then
* I need to know my node rank during init
*/
return orte_process_info.my_node_rank;
}
if (NULL == (pmap = orte_util_lookup_pmap(proc))) {
return ORTE_NODE_RANK_INVALID;
}
OPAL_OUTPUT_VERBOSE((5, orte_ess_base_output,
"%s ess:base: proc %s has node rank %d",
ORTE_NAME_PRINT(ORTE_PROC_MY_NAME),
ORTE_NAME_PRINT(proc),
(int)pmap->node_rank));
return pmap->node_rank;
}
int orte_ess_base_update_pidmap(opal_byte_object_t *bo)
{
int ret;
OPAL_OUTPUT_VERBOSE((2, orte_ess_base_output,
"%s ess:base: updating pidmap",
ORTE_NAME_PRINT(ORTE_PROC_MY_NAME)));
/* build the pmap */
if (ORTE_SUCCESS != (ret = orte_util_decode_pidmap(bo))) {
ORTE_ERROR_LOG(ret);
}
return ret;
}
int orte_ess_base_update_nidmap(opal_byte_object_t *bo)
{
int rc;
/* decode the nidmap - the util will know what to do */
if (ORTE_SUCCESS != (rc = orte_util_decode_nodemap(bo))) {
ORTE_ERROR_LOG(rc);
}
return rc;
}

4
orte/mca/ess/cnos/ess_cnos_module.c
Просмотреть файл

@ -43,7 +43,7 @@
static int rte_init(void);
static int rte_finalize(void);
static void rte_abort(int status, bool report) __opal_attribute_noreturn__;
static uint8_t proc_get_locality(orte_process_name_t *proc);
static opal_paffinity_locality_t proc_get_locality(orte_process_name_t *proc);
static char* proc_get_hostname(orte_process_name_t *proc);
static orte_local_rank_t proc_get_local_rank(orte_process_name_t *proc);
static orte_node_rank_t proc_get_node_rank(orte_process_name_t *proc);
@ -130,7 +130,7 @@ static void rte_abort(int status, bool report)
exit(status);
}
static uint8_t proc_get_locality(orte_process_name_t *proc)
static opal_paffinity_locality_t proc_get_locality(orte_process_name_t *proc)
{
if (map[ORTE_PROC_MY_NAME->vpid].nid ==
map[proc->vpid].nid) {

174
orte/mca/ess/env/ess_env_module.c поставляемый
Просмотреть файл

@ -79,13 +79,6 @@ static int env_set_name(void);
static int rte_init(void);
static int rte_finalize(void);
static uint8_t proc_get_locality(orte_process_name_t *proc);
static orte_vpid_t proc_get_daemon(orte_process_name_t *proc);
static char* proc_get_hostname(orte_process_name_t *proc);
static orte_local_rank_t proc_get_local_rank(orte_process_name_t *proc);
static orte_node_rank_t proc_get_node_rank(orte_process_name_t *proc);
static int update_pidmap(opal_byte_object_t *bo);
static int update_nidmap(opal_byte_object_t *bo);
#if OPAL_ENABLE_FT_CR == 1
static int rte_ft_event(int state);
@ -95,14 +88,14 @@ orte_ess_base_module_t orte_ess_env_module = {
rte_init,
rte_finalize,
orte_ess_base_app_abort,
proc_get_locality,
proc_get_daemon,
proc_get_hostname,
proc_get_local_rank,
proc_get_node_rank,
orte_ess_base_proc_get_locality,
orte_ess_base_proc_get_daemon,
orte_ess_base_proc_get_hostname,
orte_ess_base_proc_get_local_rank,
orte_ess_base_proc_get_node_rank,
orte_ess_base_proc_get_epoch, /* proc_get_epoch */
update_pidmap,
update_nidmap,
orte_ess_base_update_pidmap,
orte_ess_base_update_nidmap,
#if OPAL_ENABLE_FT_CR == 1
rte_ft_event
#else
@ -110,11 +103,6 @@ orte_ess_base_module_t orte_ess_env_module = {
#endif
};
/*
* Local variables
*/
static orte_node_rank_t my_node_rank=ORTE_NODE_RANK_INVALID;
static int rte_init(void)
{
int ret;
@ -219,145 +207,6 @@ static int rte_finalize(void)
return ret;
}
static uint8_t proc_get_locality(orte_process_name_t *proc)
{
orte_nid_t *nid;
if (NULL == (nid = orte_util_lookup_nid(proc))) {
ORTE_ERROR_LOG(ORTE_ERR_NOT_FOUND);
return OPAL_PROC_NON_LOCAL;
}
if (nid->daemon == ORTE_PROC_MY_DAEMON->vpid) {
OPAL_OUTPUT_VERBOSE((2, orte_ess_base_output,
"%s ess:env: proc %s on LOCAL NODE",
ORTE_NAME_PRINT(ORTE_PROC_MY_NAME),
ORTE_NAME_PRINT(proc)));
return (OPAL_PROC_ON_NODE | OPAL_PROC_ON_CU | OPAL_PROC_ON_CLUSTER);
}
OPAL_OUTPUT_VERBOSE((2, orte_ess_base_output,
"%s ess:env: proc %s is REMOTE",
ORTE_NAME_PRINT(ORTE_PROC_MY_NAME),
ORTE_NAME_PRINT(proc)));
return OPAL_PROC_NON_LOCAL;
}
static orte_vpid_t proc_get_daemon(orte_process_name_t *proc)
{
orte_nid_t *nid;
if( ORTE_JOBID_IS_DAEMON(proc->jobid) ) {
return proc->vpid;
}
if (NULL == (nid = orte_util_lookup_nid(proc))) {
return ORTE_VPID_INVALID;
}
OPAL_OUTPUT_VERBOSE((2, orte_ess_base_output,
"%s ess:env: proc %s is hosted by daemon %s",
ORTE_NAME_PRINT(ORTE_PROC_MY_NAME),
ORTE_NAME_PRINT(proc),
ORTE_VPID_PRINT(nid->daemon)));
return nid->daemon;
}
static char* proc_get_hostname(orte_process_name_t *proc)
{
orte_nid_t *nid;
if (NULL == (nid = orte_util_lookup_nid(proc))) {
ORTE_ERROR_LOG(ORTE_ERR_NOT_FOUND);
return NULL;
}
OPAL_OUTPUT_VERBOSE((2, orte_ess_base_output,
"%s ess:env: proc %s is on host %s",
ORTE_NAME_PRINT(ORTE_PROC_MY_NAME),
ORTE_NAME_PRINT(proc),
nid->name));
return nid->name;
}
static orte_local_rank_t proc_get_local_rank(orte_process_name_t *proc)
{
orte_pmap_t *pmap;
if (NULL == (pmap = orte_util_lookup_pmap(proc))) {
ORTE_ERROR_LOG(ORTE_ERR_NOT_FOUND);
return ORTE_LOCAL_RANK_INVALID;
}
OPAL_OUTPUT_VERBOSE((2, orte_ess_base_output,
"%s ess:env: proc %s has local rank %d",
ORTE_NAME_PRINT(ORTE_PROC_MY_NAME),
ORTE_NAME_PRINT(proc),
(int)pmap->local_rank));
return pmap->local_rank;
}
static orte_node_rank_t proc_get_node_rank(orte_process_name_t *proc)
{
orte_pmap_t *pmap;
orte_ns_cmp_bitmask_t mask;
mask = ORTE_NS_CMP_JOBID | ORTE_NS_CMP_VPID;
/* is this me? */
if (OPAL_EQUAL == orte_util_compare_name_fields(mask, proc, ORTE_PROC_MY_NAME)) {
/* yes it is - reply with my rank. This is necessary
* because the pidmap will not have arrived when I
* am starting up, and if we use static ports, then
* I need to know my node rank during init
*/
return my_node_rank;
}
if (NULL == (pmap = orte_util_lookup_pmap(proc))) {
return ORTE_NODE_RANK_INVALID;
}
OPAL_OUTPUT_VERBOSE((2, orte_ess_base_output,
"%s ess:env: proc %s has node rank %d",
ORTE_NAME_PRINT(ORTE_PROC_MY_NAME),
ORTE_NAME_PRINT(proc),
(int)pmap->node_rank));
return pmap->node_rank;
}
static int update_pidmap(opal_byte_object_t *bo)
{
int ret;
OPAL_OUTPUT_VERBOSE((2, orte_ess_base_output,
"%s ess:env: updating pidmap",
ORTE_NAME_PRINT(ORTE_PROC_MY_NAME)));
/* build the pmap */
if (ORTE_SUCCESS != (ret = orte_util_decode_pidmap(bo))) {
ORTE_ERROR_LOG(ret);
}
return ret;
}
static int update_nidmap(opal_byte_object_t *bo)
{
int rc;
/* decode the nidmap - the util will know what to do */
if (ORTE_SUCCESS != (rc = orte_util_decode_nodemap(bo))) {
ORTE_ERROR_LOG(rc);
}
return rc;
}
static int env_set_name(void)
{
char *tmp;
@ -396,15 +245,6 @@ static int env_set_name(void)
OPAL_OUTPUT_VERBOSE((1, orte_ess_base_output,
"ess:env set name to %s", ORTE_NAME_PRINT(ORTE_PROC_MY_NAME)));
/* get my node rank in case we are using static ports - this won't
* be present for daemons, so don't error out if we don't have it
*/
mca_base_param_reg_string_name("orte", "ess_node_rank", "Process node rank",
true, false, NULL, &tmp);
if (NULL != tmp) {
my_node_rank = strtol(tmp, NULL, 10);
}
/* get the non-name common environmental variables */
if (ORTE_SUCCESS != (rc = orte_ess_env_get())) {
ORTE_ERROR_LOG(rc);

3
orte/mca/ess/ess.h
Просмотреть файл

@ -28,6 +28,7 @@
#include "orte/types.h"
#include "opal/mca/mca.h"
#include "opal/mca/paffinity/paffinity.h"
#include "orte/util/proc_info.h"
@ -68,7 +69,7 @@ typedef void (*orte_ess_base_module_abort_fn_t)(int status, bool report)
* board, node, computing unit, or cluster. This function provides
* a means for an MPI proc to query the locality of another proc.
*/
typedef uint8_t (*orte_ess_base_module_get_proc_locality_fn_t)(orte_process_name_t *proc);
typedef opal_paffinity_locality_t (*orte_ess_base_module_get_proc_locality_fn_t)(orte_process_name_t *proc);
/**
* Get the vpid of the daemon who hosts the specified proc

147
orte/mca/ess/generic/ess_generic_module.c
Просмотреть файл

@ -78,26 +78,19 @@
static int rte_init(void);
static int rte_finalize(void);
static uint8_t proc_get_locality(orte_process_name_t *proc);
static orte_vpid_t proc_get_daemon(orte_process_name_t *proc);
static char* proc_get_hostname(orte_process_name_t *proc);
static orte_local_rank_t proc_get_local_rank(orte_process_name_t *proc);
static orte_node_rank_t proc_get_node_rank(orte_process_name_t *proc);
static int update_pidmap(opal_byte_object_t *bo);
static int update_nidmap(opal_byte_object_t *bo);
orte_ess_base_module_t orte_ess_generic_module = {
rte_init,
rte_finalize,
orte_ess_base_app_abort,
proc_get_locality,
proc_get_daemon,
proc_get_hostname,
proc_get_local_rank,
proc_get_node_rank,
orte_ess_base_proc_get_locality,
orte_ess_base_proc_get_daemon,
orte_ess_base_proc_get_hostname,
orte_ess_base_proc_get_local_rank,
orte_ess_base_proc_get_node_rank,
orte_ess_base_proc_get_epoch,
update_pidmap,
update_nidmap,
orte_ess_base_update_pidmap,
orte_ess_base_update_nidmap,
NULL
};
@ -366,129 +359,3 @@ static int rte_finalize(void)
return ret;
}
static uint8_t proc_get_locality(orte_process_name_t *proc)
{
orte_nid_t *nid;
if (NULL == (nid = orte_util_lookup_nid(proc))) {
ORTE_ERROR_LOG(ORTE_ERR_NOT_FOUND);
return OPAL_PROC_NON_LOCAL;
}
if (nid->daemon == ORTE_PROC_MY_DAEMON->vpid) {
OPAL_OUTPUT_VERBOSE((2, orte_ess_base_output,
"%s ess:generic: proc %s on LOCAL NODE",
ORTE_NAME_PRINT(ORTE_PROC_MY_NAME),
ORTE_NAME_PRINT(proc)));
return (OPAL_PROC_ON_NODE | OPAL_PROC_ON_CU | OPAL_PROC_ON_CLUSTER);
}
OPAL_OUTPUT_VERBOSE((2, orte_ess_base_output,
"%s ess:generic: proc %s is REMOTE",
ORTE_NAME_PRINT(ORTE_PROC_MY_NAME),
ORTE_NAME_PRINT(proc)));
return OPAL_PROC_NON_LOCAL;
}
static orte_vpid_t proc_get_daemon(orte_process_name_t *proc)
{
orte_nid_t *nid;
if( ORTE_JOBID_IS_DAEMON(proc->jobid) ) {
return proc->vpid;
}
if (NULL == (nid = orte_util_lookup_nid(proc))) {
return ORTE_VPID_INVALID;
}
OPAL_OUTPUT_VERBOSE((2, orte_ess_base_output,
"%s ess:generic: proc %s is hosted by daemon %s",
ORTE_NAME_PRINT(ORTE_PROC_MY_NAME),
ORTE_NAME_PRINT(proc),
ORTE_VPID_PRINT(nid->daemon)));
return nid->daemon;
}
static char* proc_get_hostname(orte_process_name_t *proc)
{
orte_nid_t *nid;
if (NULL == (nid = orte_util_lookup_nid(proc))) {
ORTE_ERROR_LOG(ORTE_ERR_NOT_FOUND);
return NULL;
}
OPAL_OUTPUT_VERBOSE((2, orte_ess_base_output,
"%s ess:generic: proc %s is on host %s",
ORTE_NAME_PRINT(ORTE_PROC_MY_NAME),
ORTE_NAME_PRINT(proc),
nid->name));
return nid->name;
}
static orte_local_rank_t proc_get_local_rank(orte_process_name_t *proc)
{
orte_pmap_t *pmap;
if (NULL == (pmap = orte_util_lookup_pmap(proc))) {
ORTE_ERROR_LOG(ORTE_ERR_NOT_FOUND);
return ORTE_LOCAL_RANK_INVALID;
}
OPAL_OUTPUT_VERBOSE((2, orte_ess_base_output,
"%s ess:generic: proc %s has local rank %d",
ORTE_NAME_PRINT(ORTE_PROC_MY_NAME),
ORTE_NAME_PRINT(proc),
(int)pmap->local_rank));
return pmap->local_rank;
}
static orte_node_rank_t proc_get_node_rank(orte_process_name_t *proc)
{
orte_pmap_t *pmap;
if (NULL == (pmap = orte_util_lookup_pmap(proc))) {
return ORTE_NODE_RANK_INVALID;
}
OPAL_OUTPUT_VERBOSE((2, orte_ess_base_output,
"%s ess:generic: proc %s has node rank %d",
ORTE_NAME_PRINT(ORTE_PROC_MY_NAME),
ORTE_NAME_PRINT(proc),
(int)pmap->node_rank));
return pmap->node_rank;
}
static int update_pidmap(opal_byte_object_t *bo)
{
int ret;
OPAL_OUTPUT_VERBOSE((2, orte_ess_base_output,
"%s ess:generic: updating pidmap",
ORTE_NAME_PRINT(ORTE_PROC_MY_NAME)));
/* build the pmap */
if (ORTE_SUCCESS != (ret = orte_util_decode_pidmap(bo))) {
ORTE_ERROR_LOG(ret);
}
return ret;
}
static int update_nidmap(opal_byte_object_t *bo)
{
int rc;
/* decode the nidmap - the util will know what to do */
if (ORTE_SUCCESS != (rc = orte_util_decode_nodemap(bo))) {
ORTE_ERROR_LOG(rc);
}
return rc;
}

4
orte/mca/ess/hnp/ess_hnp_module.c
Просмотреть файл

@ -92,7 +92,7 @@
static int rte_init(void);
static int rte_finalize(void);
static void rte_abort(int status, bool report) __opal_attribute_noreturn__;
static uint8_t proc_get_locality(orte_process_name_t *proc);
static opal_paffinity_locality_t proc_get_locality(orte_process_name_t *proc);
static orte_vpid_t proc_get_daemon(orte_process_name_t *proc);
static char* proc_get_hostname(orte_process_name_t *proc);
static orte_local_rank_t proc_get_local_rank(orte_process_name_t *proc);
@ -850,7 +850,7 @@ static void rte_abort(int status, bool report)
exit(status);
}
static uint8_t proc_get_locality(orte_process_name_t *proc)
static opal_paffinity_locality_t proc_get_locality(orte_process_name_t *proc)
{
orte_node_t *node;
orte_proc_t *myproc;

158
orte/mca/ess/lsf/ess_lsf_module.c
Просмотреть файл

@ -51,26 +51,19 @@ static int lsf_set_name(void);
static int rte_init(void);
static int rte_finalize(void);
static uint8_t proc_get_locality(orte_process_name_t *proc);
static orte_vpid_t proc_get_daemon(orte_process_name_t *proc);
static char* proc_get_hostname(orte_process_name_t *proc);
static orte_local_rank_t proc_get_local_rank(orte_process_name_t *proc);
static orte_node_rank_t proc_get_node_rank(orte_process_name_t *proc);
static int update_pidmap(opal_byte_object_t *bo);
static int update_nidmap(opal_byte_object_t *bo);
orte_ess_base_module_t orte_ess_lsf_module = {
rte_init,
rte_finalize,
orte_ess_base_app_abort,
proc_get_locality,
proc_get_daemon,
proc_get_hostname,
proc_get_local_rank,
proc_get_node_rank,
orte_ess_base_proc_get_locality,
orte_ess_base_proc_get_daemon,
orte_ess_base_proc_get_hostname,
orte_ess_base_proc_get_local_rank,
orte_ess_base_proc_get_node_rank,
orte_ess_base_proc_get_epoch, /* proc_get_epoch */
update_pidmap,
update_nidmap,
orte_ess_base_update_pidmap,
orte_ess_base_update_nidmap,
NULL /* ft_event */
};
@ -185,143 +178,6 @@ static int rte_finalize(void)
return ret;
}
static uint8_t proc_get_locality(orte_process_name_t *proc)
{
orte_nid_t *nid;
if (NULL == (nid = orte_util_lookup_nid(proc))) {
ORTE_ERROR_LOG(ORTE_ERR_NOT_FOUND);
return OPAL_PROC_NON_LOCAL;
}
if (nid->daemon == ORTE_PROC_MY_DAEMON->vpid) {
OPAL_OUTPUT_VERBOSE((2, orte_ess_base_output,
"%s ess:lsf: proc %s is LOCAL",
ORTE_NAME_PRINT(ORTE_PROC_MY_NAME),
ORTE_NAME_PRINT(proc)));
return (OPAL_PROC_ON_NODE | OPAL_PROC_ON_CU | OPAL_PROC_ON_CLUSTER);
}
OPAL_OUTPUT_VERBOSE((2, orte_ess_base_output,
"%s ess:lsf: proc %s is REMOTE",
ORTE_NAME_PRINT(ORTE_PROC_MY_NAME),
ORTE_NAME_PRINT(proc)));
return OPAL_PROC_NON_LOCAL;
}
static orte_vpid_t proc_get_daemon(orte_process_name_t *proc)
{
orte_nid_t *nid;
if( ORTE_JOBID_IS_DAEMON(proc->jobid) ) {
return proc->vpid;
}
if (NULL == (nid = orte_util_lookup_nid(proc))) {
return ORTE_VPID_INVALID;
}
OPAL_OUTPUT_VERBOSE((2, orte_ess_base_output,
"%s ess:lsf: proc %s is hosted by daemon %s",
ORTE_NAME_PRINT(ORTE_PROC_MY_NAME),
ORTE_NAME_PRINT(proc),
ORTE_VPID_PRINT(nid->daemon)));
return nid->daemon;
}
static char* proc_get_hostname(orte_process_name_t *proc)
{
orte_nid_t *nid;
if (NULL == (nid = orte_util_lookup_nid(proc))) {
ORTE_ERROR_LOG(ORTE_ERR_NOT_FOUND);
return NULL;
}
OPAL_OUTPUT_VERBOSE((2, orte_ess_base_output,
"%s ess:lsf: proc %s is on host %s",
ORTE_NAME_PRINT(ORTE_PROC_MY_NAME),
ORTE_NAME_PRINT(proc),
nid->name));
return nid->name;
}
static orte_local_rank_t proc_get_local_rank(orte_process_name_t *proc)
{
orte_pmap_t *pmap;
if (NULL == (pmap = orte_util_lookup_pmap(proc))) {
ORTE_ERROR_LOG(ORTE_ERR_NOT_FOUND);
return ORTE_LOCAL_RANK_INVALID;
}
OPAL_OUTPUT_VERBOSE((2, orte_ess_base_output,
"%s ess:lsf: proc %s has local rank %d",
ORTE_NAME_PRINT(ORTE_PROC_MY_NAME),
ORTE_NAME_PRINT(proc),
(int)pmap->local_rank));
return pmap->local_rank;
}
static orte_node_rank_t proc_get_node_rank(orte_process_name_t *proc)
{
orte_pmap_t *pmap;
orte_ns_cmp_bitmask_t mask;
mask = ORTE_NS_CMP_JOBID | ORTE_NS_CMP_VPID;
/* is this me? */
if (OPAL_EQUAL == orte_util_compare_name_fields(mask, proc, ORTE_PROC_MY_NAME)) {
/* yes it is - reply with my rank. This is necessary
* because the pidmap will not have arrived when I
* am starting up, and if we use static ports, then
* I need to know my node rank during init
*/
return my_node_rank;
}
if (NULL == (pmap = orte_util_lookup_pmap(proc))) {
ORTE_ERROR_LOG(ORTE_ERR_NOT_FOUND);
return ORTE_NODE_RANK_INVALID;
}
OPAL_OUTPUT_VERBOSE((2, orte_ess_base_output,
"%s ess:lsf: proc %s has node rank %d",
ORTE_NAME_PRINT(ORTE_PROC_MY_NAME),
ORTE_NAME_PRINT(proc),
(int)pmap->node_rank));
return pmap->node_rank;
}
static int update_pidmap(opal_byte_object_t *bo)
{
int ret;
/* build the pmap */
if (ORTE_SUCCESS != (ret = orte_util_decode_pidmap(bo))) {
ORTE_ERROR_LOG(ret);
}
return ret;
}
static int update_nidmap(opal_byte_object_t *bo)
{
int rc;
/* decode the nidmap - the util will know what to do */
if (ORTE_SUCCESS != (rc = orte_util_decode_nodemap(bo))) {
ORTE_ERROR_LOG(rc);
}
return rc;
}
static int lsf_set_name(void)
{
int rc;

172
orte/mca/ess/pmi/ess_pmi_module.c
Просмотреть файл

@ -62,26 +62,19 @@
static int rte_init(void);
static int rte_finalize(void);
static void rte_abort(int error_code, bool report) __opal_attribute_noreturn__;
static uint8_t proc_get_locality(orte_process_name_t *proc);
static orte_vpid_t proc_get_daemon(orte_process_name_t *proc);
static char* proc_get_hostname(orte_process_name_t *proc);
static orte_local_rank_t proc_get_local_rank(orte_process_name_t *proc);
static orte_node_rank_t proc_get_node_rank(orte_process_name_t *proc);
static int update_pidmap(opal_byte_object_t *bo);
static int update_nidmap(opal_byte_object_t *bo);
orte_ess_base_module_t orte_ess_pmi_module = {
rte_init,
rte_finalize,
rte_abort,
proc_get_locality,
proc_get_daemon,
proc_get_hostname,
proc_get_local_rank,
proc_get_node_rank,
orte_ess_base_proc_get_locality,
orte_ess_base_proc_get_daemon,
orte_ess_base_proc_get_hostname,
orte_ess_base_proc_get_local_rank,
orte_ess_base_proc_get_node_rank,
orte_ess_base_proc_get_epoch, /* proc_get_epoch */
update_pidmap,
update_nidmap,
orte_ess_base_update_pidmap,
orte_ess_base_update_nidmap,
NULL /* ft_event */
};
@ -218,6 +211,11 @@ static int rte_init(void)
*/
nid = (orte_nid_t*)opal_pointer_array_get_item(&orte_nidmap, 0);
nid->daemon = 0;
/* setup my daemon's name - arbitrary, since we don't route
* messages
*/
ORTE_PROC_MY_DAEMON->jobid = 0;
ORTE_PROC_MY_DAEMON->vpid = 0;
/* get the job map for this job */
jmap = (orte_jmap_t*)opal_pointer_array_get_item(&orte_jobmap, 0);
@ -310,152 +308,6 @@ static void rte_abort(int error_code, bool report)
orte_ess_base_app_abort(error_code, report);
}
static uint8_t proc_get_locality(orte_process_name_t *proc)
{
orte_nid_t *nid;
if (NULL == (nid = orte_util_lookup_nid(proc))) {
OPAL_OUTPUT_VERBOSE((2, orte_ess_base_output,
"%s ess:pmi: proc %s NID NOT FOUND",
ORTE_NAME_PRINT(ORTE_PROC_MY_NAME),
ORTE_NAME_PRINT(proc)));
ORTE_ERROR_LOG(ORTE_ERR_NOT_FOUND);
return OPAL_PROC_NON_LOCAL;
}
if (nid->daemon == ORTE_PROC_MY_DAEMON->vpid) {
OPAL_OUTPUT_VERBOSE((2, orte_ess_base_output,
"%s ess:pmi: proc %s is LOCAL",
ORTE_NAME_PRINT(ORTE_PROC_MY_NAME),
ORTE_NAME_PRINT(proc)));
return (OPAL_PROC_ON_NODE | OPAL_PROC_ON_CU | OPAL_PROC_ON_CLUSTER);
}
OPAL_OUTPUT_VERBOSE((2, orte_ess_base_output,
"%s ess:pmi: proc %s is REMOTE",
ORTE_NAME_PRINT(ORTE_PROC_MY_NAME),
ORTE_NAME_PRINT(proc)));
return OPAL_PROC_NON_LOCAL;
}
static orte_vpid_t proc_get_daemon(orte_process_name_t *proc)
{
orte_nid_t *nid;
if( ORTE_JOBID_IS_DAEMON(proc->jobid) ) {
return proc->vpid;
}
if (NULL == (nid = orte_util_lookup_nid(proc))) {
OPAL_OUTPUT_VERBOSE((2, orte_ess_base_output,
"%s ess:pmi: proc %s NID NOT FOUND",
ORTE_NAME_PRINT(ORTE_PROC_MY_NAME),
ORTE_NAME_PRINT(proc)));
return ORTE_VPID_INVALID;
}
OPAL_OUTPUT_VERBOSE((2, orte_ess_base_output,
"%s ess:pmi: proc %s is hosted by daemon %s",
ORTE_NAME_PRINT(ORTE_PROC_MY_NAME),
ORTE_NAME_PRINT(proc),
ORTE_VPID_PRINT(nid->daemon)));
return nid->daemon;
}
static char* proc_get_hostname(orte_process_name_t *proc)
{
orte_nid_t *nid;
if (NULL == (nid = orte_util_lookup_nid(proc))) {
OPAL_OUTPUT_VERBOSE((2, orte_ess_base_output,
"%s ess:pmi: proc %s NID NOT FOUND",
ORTE_NAME_PRINT(ORTE_PROC_MY_NAME),
ORTE_NAME_PRINT(proc)));
ORTE_ERROR_LOG(ORTE_ERR_NOT_FOUND);
return NULL;
}
OPAL_OUTPUT_VERBOSE((2, orte_ess_base_output,
"%s ess:pmi: proc %s is on host %s",
ORTE_NAME_PRINT(ORTE_PROC_MY_NAME),
ORTE_NAME_PRINT(proc),
nid->name));
return nid->name;
}
static orte_local_rank_t proc_get_local_rank(orte_process_name_t *proc)
{
orte_pmap_t *pmap;
if (NULL == (pmap = orte_util_lookup_pmap(proc))) {
OPAL_OUTPUT_VERBOSE((2, orte_ess_base_output,
"%s ess:pmi: proc %s NID NOT FOUND",
ORTE_NAME_PRINT(ORTE_PROC_MY_NAME),
ORTE_NAME_PRINT(proc)));
ORTE_ERROR_LOG(ORTE_ERR_NOT_FOUND);
return ORTE_LOCAL_RANK_INVALID;
}
OPAL_OUTPUT_VERBOSE((2, orte_ess_base_output,
"%s ess:pmi: proc %s has local rank %d",
ORTE_NAME_PRINT(ORTE_PROC_MY_NAME),
ORTE_NAME_PRINT(proc),
(int)pmap->local_rank));
return pmap->local_rank;
}
static orte_node_rank_t proc_get_node_rank(orte_process_name_t *proc)
{
orte_pmap_t *pmap;
if (NULL == (pmap = orte_util_lookup_pmap(proc))) {
OPAL_OUTPUT_VERBOSE((2, orte_ess_base_output,
"%s ess:pmi: proc %s PMAP NOT FOUND",
ORTE_NAME_PRINT(ORTE_PROC_MY_NAME),
ORTE_NAME_PRINT(proc)));
return ORTE_NODE_RANK_INVALID;
}
OPAL_OUTPUT_VERBOSE((2, orte_ess_base_output,
"%s ess:pmi: proc %s has node rank %d",
ORTE_NAME_PRINT(ORTE_PROC_MY_NAME),
ORTE_NAME_PRINT(proc),
(int)pmap->node_rank));
return pmap->node_rank;
}
static int update_pidmap(opal_byte_object_t *bo)
{
int ret;
OPAL_OUTPUT_VERBOSE((2, orte_ess_base_output,
"%s ess:pmi: updating pidmap",
ORTE_NAME_PRINT(ORTE_PROC_MY_NAME)));
/* build the pmap */
if (ORTE_SUCCESS != (ret = orte_util_decode_pidmap(bo))) {
ORTE_ERROR_LOG(ret);
}
return ret;
}
static int update_nidmap(opal_byte_object_t *bo)
{
int rc;
/* decode the nidmap - the util will know what to do */
if (ORTE_SUCCESS != (rc = orte_util_decode_nodemap(bo))) {
ORTE_ERROR_LOG(rc);
}
return rc;
}
/* useful util */
static char* pmi_error(int pmi_err)
{

4
orte/mca/ess/portals4_shmem/ess_portals4_shmem_module.c
Просмотреть файл

@ -41,7 +41,7 @@
static int rte_init(void);
static int rte_finalize(void);
static void rte_abort(int status, bool report) __opal_attribute_noreturn__;
static uint8_t proc_get_locality(orte_process_name_t *proc);
static opal_paffinity_locality_t proc_get_locality(orte_process_name_t *proc);
static char* proc_get_hostname(orte_process_name_t *proc);
static orte_local_rank_t proc_get_local_rank(orte_process_name_t *proc);
static orte_node_rank_t proc_get_node_rank(orte_process_name_t *proc);
@ -128,7 +128,7 @@ static void rte_abort(int status, bool report)
exit(status);
}
static uint8_t proc_get_locality(orte_process_name_t *proc)
static opal_paffinity_locality_t proc_get_locality(orte_process_name_t *proc)
{
if (map[ORTE_PROC_MY_NAME->vpid].nid ==
map[proc->vpid].nid) {

161
orte/mca/ess/singleton/ess_singleton_module.c
Просмотреть файл

@ -70,26 +70,19 @@ static void set_handler_default(int sig)
static int rte_init(void);
static int rte_finalize(void);
static uint8_t proc_get_locality(orte_process_name_t *proc);
static orte_vpid_t proc_get_daemon(orte_process_name_t *proc);
static char* proc_get_hostname(orte_process_name_t *proc);
static orte_local_rank_t proc_get_local_rank(orte_process_name_t *proc);
static orte_node_rank_t proc_get_node_rank(orte_process_name_t *proc);
static int update_pidmap(opal_byte_object_t *bo);
static int update_nidmap(opal_byte_object_t *bo);
orte_ess_base_module_t orte_ess_singleton_module = {
rte_init,
rte_finalize,
orte_ess_base_app_abort,
proc_get_locality,
proc_get_daemon,
proc_get_hostname,
proc_get_local_rank,
proc_get_node_rank,
orte_ess_base_proc_get_locality,
orte_ess_base_proc_get_daemon,
orte_ess_base_proc_get_hostname,
orte_ess_base_proc_get_local_rank,
orte_ess_base_proc_get_node_rank,
orte_ess_base_proc_get_epoch, /* proc_get_epoch */
update_pidmap,
update_nidmap,
orte_ess_base_update_pidmap,
orte_ess_base_update_nidmap,
NULL /* ft_event */
};
@ -241,22 +234,18 @@ static int rte_init(void)
* library wrt pty's and stdin
*/
/* setup the nidmap and jobmap arrays */
if (ORTE_SUCCESS != (rc = orte_util_nidmap_init(NULL))) {
ORTE_ERROR_LOG(rc);
return rc;
}
if (ORTE_SUCCESS != (rc = orte_util_setup_local_nidmap_entries())) {
ORTE_ERROR_LOG(rc);
return rc;
}
/* use the std app init to complete the procedure */
if (ORTE_SUCCESS != (rc = orte_ess_base_app_setup())) {
ORTE_ERROR_LOG(rc);
return rc;
}
/* if one was provided, build my nidmap */
if (ORTE_SUCCESS != (rc = orte_util_nidmap_init(orte_process_info.sync_buf))) {
ORTE_ERROR_LOG(rc);
return rc;
}
return ORTE_SUCCESS;
}
@ -637,127 +626,3 @@ static int fork_hnp(void)
return ORTE_SUCCESS;
#endif
}
static uint8_t proc_get_locality(orte_process_name_t *proc)
{
orte_nid_t *nid;
if (NULL == (nid = orte_util_lookup_nid(proc))) {
ORTE_ERROR_LOG(ORTE_ERR_NOT_FOUND);
return OPAL_PROC_NON_LOCAL;
}
if (nid->daemon == ORTE_PROC_MY_DAEMON->vpid) {
OPAL_OUTPUT_VERBOSE((2, orte_ess_base_output,
"%s ess:singleton: proc %s is LOCAL",
ORTE_NAME_PRINT(ORTE_PROC_MY_NAME),
ORTE_NAME_PRINT(proc)));
return (OPAL_PROC_ON_NODE | OPAL_PROC_ON_CU | OPAL_PROC_ON_CLUSTER);
}
OPAL_OUTPUT_VERBOSE((2, orte_ess_base_output,
"%s ess:singleton: proc %s is REMOTE",
ORTE_NAME_PRINT(ORTE_PROC_MY_NAME),
ORTE_NAME_PRINT(proc)));
return OPAL_PROC_NON_LOCAL;
}
static orte_vpid_t proc_get_daemon(orte_process_name_t *proc)
{
orte_nid_t *nid;
if( ORTE_JOBID_IS_DAEMON(proc->jobid) ) {
return proc->vpid;
}
if (NULL == (nid = orte_util_lookup_nid(proc))) {
return ORTE_VPID_INVALID;
}
OPAL_OUTPUT_VERBOSE((2, orte_ess_base_output,
"%s ess:singleton: proc %s is hosted by daemon %s",
ORTE_NAME_PRINT(ORTE_PROC_MY_NAME),
ORTE_NAME_PRINT(proc),
ORTE_VPID_PRINT(nid->daemon)));
return nid->daemon;
}
static char* proc_get_hostname(orte_process_name_t *proc)
{
orte_nid_t *nid;
if (NULL == (nid = orte_util_lookup_nid(proc))) {
ORTE_ERROR_LOG(ORTE_ERR_NOT_FOUND);
return NULL;
}
OPAL_OUTPUT_VERBOSE((2, orte_ess_base_output,
"%s ess:singleton: proc %s is on host %s",
ORTE_NAME_PRINT(ORTE_PROC_MY_NAME),
ORTE_NAME_PRINT(proc),
nid->name));
return nid->name;
}
static orte_local_rank_t proc_get_local_rank(orte_process_name_t *proc)
{
orte_pmap_t *pmap;
if (NULL == (pmap = orte_util_lookup_pmap(proc))) {
ORTE_ERROR_LOG(ORTE_ERR_NOT_FOUND);
return ORTE_LOCAL_RANK_INVALID;
}
OPAL_OUTPUT_VERBOSE((2, orte_ess_base_output,
"%s ess:singleton: proc %s has local rank %d",
ORTE_NAME_PRINT(ORTE_PROC_MY_NAME),
ORTE_NAME_PRINT(proc),
(int)pmap->local_rank));
return pmap->local_rank;
}
static orte_node_rank_t proc_get_node_rank(orte_process_name_t *proc)
{
orte_pmap_t *pmap;
if (NULL == (pmap = orte_util_lookup_pmap(proc))) {
ORTE_ERROR_LOG(ORTE_ERR_NOT_FOUND);
return ORTE_NODE_RANK_INVALID;
}
OPAL_OUTPUT_VERBOSE((2, orte_ess_base_output,
"%s ess:singleton: proc %s has node rank %d",
ORTE_NAME_PRINT(ORTE_PROC_MY_NAME),
ORTE_NAME_PRINT(proc),
(int)pmap->node_rank));
return pmap->node_rank;
}
static int update_pidmap(opal_byte_object_t *bo)
{
int ret;
/* build the pmap */
if (ORTE_SUCCESS != (ret = orte_util_decode_pidmap(bo))) {
ORTE_ERROR_LOG(ret);
}
return ret;
}
static int update_nidmap(opal_byte_object_t *bo)
{
int rc;
/* decode the nidmap - the util will know what to do */
if (ORTE_SUCCESS != (rc = orte_util_decode_nodemap(bo))) {
ORTE_ERROR_LOG(rc);
}
return rc;
}

4
orte/mca/ess/slave/ess_slave_module.c
Просмотреть файл

@ -76,7 +76,7 @@ static int slave_set_name(void);
static int rte_init(void);
static int rte_finalize(void);
static uint8_t proc_get_locality(orte_process_name_t *proc);
static opal_paffinity_locality_t proc_get_locality(orte_process_name_t *proc);
static orte_vpid_t proc_get_daemon(orte_process_name_t *proc);
static char* proc_get_hostname(orte_process_name_t *proc);
static orte_local_rank_t proc_get_local_rank(orte_process_name_t *proc);
@ -168,7 +168,7 @@ static int rte_finalize(void)
return ret;
}
static uint8_t proc_get_locality(orte_process_name_t *proc)
static opal_paffinity_locality_t proc_get_locality(orte_process_name_t *proc)
{
/* no proc can be local */

175
orte/mca/ess/slurm/ess_slurm_module.c
Просмотреть файл

@ -55,35 +55,22 @@ static int slurm_set_name(void);
static int rte_init(void);
static int rte_finalize(void);
static uint8_t proc_get_locality(orte_process_name_t *proc);
static orte_vpid_t proc_get_daemon(orte_process_name_t *proc);
static char* proc_get_hostname(orte_process_name_t *proc);
static orte_local_rank_t proc_get_local_rank(orte_process_name_t *proc);
static orte_node_rank_t proc_get_node_rank(orte_process_name_t *proc);
static int update_pidmap(opal_byte_object_t *bo);
static int update_nidmap(opal_byte_object_t *bo);
orte_ess_base_module_t orte_ess_slurm_module = {
rte_init,
rte_finalize,
orte_ess_base_app_abort,
proc_get_locality,
proc_get_daemon,
proc_get_hostname,
proc_get_local_rank,
proc_get_node_rank,
orte_ess_base_proc_get_locality,
orte_ess_base_proc_get_daemon,
orte_ess_base_proc_get_hostname,
orte_ess_base_proc_get_local_rank,
orte_ess_base_proc_get_node_rank,
orte_ess_base_proc_get_epoch, /* proc_get_epoch */
update_pidmap,
update_nidmap,
orte_ess_base_update_pidmap,
orte_ess_base_update_nidmap,
NULL /* ft_event */
};
/*
* Local variables
*/
static orte_node_rank_t my_node_rank=ORTE_NODE_RANK_INVALID;
static int rte_init(void)
{
int ret;
@ -187,145 +174,6 @@ static int rte_finalize(void)
return ret;
}
static uint8_t proc_get_locality(orte_process_name_t *proc)
{
orte_nid_t *nid;
if (NULL == (nid = orte_util_lookup_nid(proc))) {
ORTE_ERROR_LOG(ORTE_ERR_NOT_FOUND);
return OPAL_PROC_NON_LOCAL;
}
if (nid->daemon == ORTE_PROC_MY_DAEMON->vpid) {
OPAL_OUTPUT_VERBOSE((2, orte_ess_base_output,
"%s ess:slurm: proc %s is LOCAL",
ORTE_NAME_PRINT(ORTE_PROC_MY_NAME),
ORTE_NAME_PRINT(proc)));
return (OPAL_PROC_ON_NODE | OPAL_PROC_ON_CU | OPAL_PROC_ON_CLUSTER);
}
OPAL_OUTPUT_VERBOSE((2, orte_ess_base_output,
"%s ess:slurm: proc %s is REMOTE",
ORTE_NAME_PRINT(ORTE_PROC_MY_NAME),
ORTE_NAME_PRINT(proc)));
return OPAL_PROC_NON_LOCAL;
}
static orte_vpid_t proc_get_daemon(orte_process_name_t *proc)
{
orte_nid_t *nid;
if( ORTE_JOBID_IS_DAEMON(proc->jobid) ) {
return proc->vpid;
}
if (NULL == (nid = orte_util_lookup_nid(proc))) {
return ORTE_VPID_INVALID;
}
OPAL_OUTPUT_VERBOSE((2, orte_ess_base_output,
"%s ess:slurm: proc %s is hosted by daemon %s",
ORTE_NAME_PRINT(ORTE_PROC_MY_NAME),
ORTE_NAME_PRINT(proc),
ORTE_VPID_PRINT(nid->daemon)));
return nid->daemon;
}
static char* proc_get_hostname(orte_process_name_t *proc)
{
orte_nid_t *nid;
if (NULL == (nid = orte_util_lookup_nid(proc))) {
ORTE_ERROR_LOG(ORTE_ERR_NOT_FOUND);
return NULL;
}
OPAL_OUTPUT_VERBOSE((2, orte_ess_base_output,
"%s ess:slurm: proc %s is on host %s",
ORTE_NAME_PRINT(ORTE_PROC_MY_NAME),
ORTE_NAME_PRINT(proc),
nid->name));
return nid->name;
}
static orte_local_rank_t proc_get_local_rank(orte_process_name_t *proc)
{
orte_pmap_t *pmap;
if (NULL == (pmap = orte_util_lookup_pmap(proc))) {
ORTE_ERROR_LOG(ORTE_ERR_NOT_FOUND);
return ORTE_LOCAL_RANK_INVALID;
}
OPAL_OUTPUT_VERBOSE((2, orte_ess_base_output,
"%s ess:slurm: proc %s has local rank %d",
ORTE_NAME_PRINT(ORTE_PROC_MY_NAME),
ORTE_NAME_PRINT(proc),
(int)pmap->local_rank));
return pmap->local_rank;
}
static orte_node_rank_t proc_get_node_rank(orte_process_name_t *proc)
{
orte_pmap_t *pmap;
orte_ns_cmp_bitmask_t mask;
mask = ORTE_NS_CMP_ALL;
/* is this me? */
if (OPAL_EQUAL == orte_util_compare_name_fields(mask, proc, ORTE_PROC_MY_NAME)) {
/* yes it is - reply with my rank. This is necessary
* because the pidmap will not have arrived when I
* am starting up, and if we use static ports, then
* I need to know my node rank during init
*/
return my_node_rank;
}
if (NULL == (pmap = orte_util_lookup_pmap(proc))) {
return ORTE_NODE_RANK_INVALID;
}
OPAL_OUTPUT_VERBOSE((2, orte_ess_base_output,
"%s ess:slurm: proc %s has node rank %d",
ORTE_NAME_PRINT(ORTE_PROC_MY_NAME),
ORTE_NAME_PRINT(proc),
(int)pmap->node_rank));
return pmap->node_rank;
}
static int update_pidmap(opal_byte_object_t *bo)
{
int ret;
OPAL_OUTPUT_VERBOSE((2, orte_ess_base_output,
"%s ess:slurm: updating pidmap",
ORTE_NAME_PRINT(ORTE_PROC_MY_NAME)));
/* build the pmap */
if (ORTE_SUCCESS != (ret = orte_util_decode_pidmap(bo))) {
ORTE_ERROR_LOG(ret);
}
return ret;
}
static int update_nidmap(opal_byte_object_t *bo)
{
int rc;
/* decode the nidmap - the util will know what to do */
if (ORTE_SUCCESS != (rc = orte_util_decode_nodemap(bo))) {
ORTE_ERROR_LOG(rc);
}
return rc;
}
static int slurm_set_name(void)
{
int slurm_nodeid;
@ -372,15 +220,6 @@ static int slurm_set_name(void)
OPAL_OUTPUT_VERBOSE((1, orte_ess_base_output,
"ess:slurm set name to %s", ORTE_NAME_PRINT(ORTE_PROC_MY_NAME)));
/* get my node rank in case we are using static ports - this won't
* be present for daemons, so don't error out if we don't have it
*/
mca_base_param_reg_string_name("orte", "ess_node_rank", "Process node rank",
true, false, NULL, &tmp);
if (NULL != tmp) {
my_node_rank = strtol(tmp, NULL, 10);
}
/* fix up the system info nodename to match exactly what slurm returned */
if (NULL != orte_process_info.nodename) {
free(orte_process_info.nodename);

148
orte/mca/ess/slurmd/ess_slurmd_module.c
Просмотреть файл

@ -60,26 +60,19 @@
static int rte_init(void);
static int rte_finalize(void);
static void rte_abort(int error_code, bool report) __opal_attribute_noreturn__;
static uint8_t proc_get_locality(orte_process_name_t *proc);
static orte_vpid_t proc_get_daemon(orte_process_name_t *proc);
static char* proc_get_hostname(orte_process_name_t *proc);
static orte_local_rank_t proc_get_local_rank(orte_process_name_t *proc);
static orte_node_rank_t proc_get_node_rank(orte_process_name_t *proc);
static int update_pidmap(opal_byte_object_t *bo);
static int update_nidmap(opal_byte_object_t *bo);
orte_ess_base_module_t orte_ess_slurmd_module = {
rte_init,
rte_finalize,
rte_abort,
proc_get_locality,
proc_get_daemon,
proc_get_hostname,
proc_get_local_rank,
proc_get_node_rank,
orte_ess_base_proc_get_locality,
orte_ess_base_proc_get_daemon,
orte_ess_base_proc_get_hostname,
orte_ess_base_proc_get_local_rank,
orte_ess_base_proc_get_node_rank,
orte_ess_base_proc_get_epoch, /* proc_get_epoch */
update_pidmap,
update_nidmap,
orte_ess_base_update_pidmap,
orte_ess_base_update_nidmap,
NULL /* ft_event */
};
@ -443,133 +436,6 @@ static void rte_abort(int error_code, bool report)
}
}
static uint8_t proc_get_locality(orte_process_name_t *proc)
{
orte_nid_t *nid;
if (NULL == (nid = orte_util_lookup_nid(proc))) {
ORTE_ERROR_LOG(ORTE_ERR_NOT_FOUND);
return OPAL_PROC_NON_LOCAL;
}
if (nid->daemon == ORTE_PROC_MY_DAEMON->vpid) {
OPAL_OUTPUT_VERBOSE((2, orte_ess_base_output,
"%s ess:slurmd: proc %s is LOCAL",
ORTE_NAME_PRINT(ORTE_PROC_MY_NAME),
ORTE_NAME_PRINT(proc)));
return (OPAL_PROC_ON_NODE | OPAL_PROC_ON_CU | OPAL_PROC_ON_CLUSTER);
}
OPAL_OUTPUT_VERBOSE((2, orte_ess_base_output,
"%s ess:slurmd: proc %s is REMOTE",
ORTE_NAME_PRINT(ORTE_PROC_MY_NAME),
ORTE_NAME_PRINT(proc)));
return OPAL_PROC_NON_LOCAL;
}
static orte_vpid_t proc_get_daemon(orte_process_name_t *proc)
{
orte_nid_t *nid;
if( ORTE_JOBID_IS_DAEMON(proc->jobid) ) {
return proc->vpid;
}
if (NULL == (nid = orte_util_lookup_nid(proc))) {
return ORTE_VPID_INVALID;
}
OPAL_OUTPUT_VERBOSE((2, orte_ess_base_output,
"%s ess:slurmd: proc %s is hosted by daemon %s",
ORTE_NAME_PRINT(ORTE_PROC_MY_NAME),
ORTE_NAME_PRINT(proc),
ORTE_VPID_PRINT(nid->daemon)));
return nid->daemon;
}
static char* proc_get_hostname(orte_process_name_t *proc)
{
orte_nid_t *nid;
if (NULL == (nid = orte_util_lookup_nid(proc))) {
ORTE_ERROR_LOG(ORTE_ERR_NOT_FOUND);
return NULL;
}
OPAL_OUTPUT_VERBOSE((2, orte_ess_base_output,
"%s ess:slurmd: proc %s is on host %s",
ORTE_NAME_PRINT(ORTE_PROC_MY_NAME),
ORTE_NAME_PRINT(proc),
nid->name));
return nid->name;
}
static orte_local_rank_t proc_get_local_rank(orte_process_name_t *proc)
{
orte_pmap_t *pmap;
if (NULL == (pmap = orte_util_lookup_pmap(proc))) {
ORTE_ERROR_LOG(ORTE_ERR_NOT_FOUND);
return ORTE_LOCAL_RANK_INVALID;
}
OPAL_OUTPUT_VERBOSE((2, orte_ess_base_output,
"%s ess:slurmd: proc %s has local rank %d",
ORTE_NAME_PRINT(ORTE_PROC_MY_NAME),
ORTE_NAME_PRINT(proc),
(int)pmap->local_rank));
return pmap->local_rank;
}
static orte_node_rank_t proc_get_node_rank(orte_process_name_t *proc)
{
orte_pmap_t *pmap;
if (NULL == (pmap = orte_util_lookup_pmap(proc))) {
return ORTE_NODE_RANK_INVALID;
}
OPAL_OUTPUT_VERBOSE((2, orte_ess_base_output,
"%s ess:slurmd: proc %s has node rank %d",
ORTE_NAME_PRINT(ORTE_PROC_MY_NAME),
ORTE_NAME_PRINT(proc),
(int)pmap->node_rank));
return pmap->node_rank;
}
static int update_pidmap(opal_byte_object_t *bo)
{
int ret;
OPAL_OUTPUT_VERBOSE((2, orte_ess_base_output,
"%s ess:slurmd: updating pidmap",
ORTE_NAME_PRINT(ORTE_PROC_MY_NAME)));
/* build the pmap */
if (ORTE_SUCCESS != (ret = orte_util_decode_pidmap(bo))) {
ORTE_ERROR_LOG(ret);
}
return ret;
}
static int update_nidmap(opal_byte_object_t *bo)
{
int rc;
/* decode the nidmap - the util will know what to do */
if (ORTE_SUCCESS != (rc = orte_util_decode_nodemap(bo))) {
ORTE_ERROR_LOG(rc);
}
return rc;
}
/**
* Discover the available resources.
*

160
orte/mca/ess/tm/ess_tm_module.c
Просмотреть файл

@ -53,26 +53,19 @@ static int tm_set_name(void);
static int rte_init(void);
static int rte_finalize(void);
static uint8_t proc_get_locality(orte_process_name_t *proc);
static orte_vpid_t proc_get_daemon(orte_process_name_t *proc);
static char* proc_get_hostname(orte_process_name_t *proc);
static orte_local_rank_t proc_get_local_rank(orte_process_name_t *proc);
static orte_node_rank_t proc_get_node_rank(orte_process_name_t *proc);
static int update_pidmap(opal_byte_object_t *bo);
static int update_nidmap(opal_byte_object_t *bo);
orte_ess_base_module_t orte_ess_tm_module = {
rte_init,
rte_finalize,
orte_ess_base_app_abort,
proc_get_locality,
proc_get_daemon,
proc_get_hostname,
proc_get_local_rank,
proc_get_node_rank,
orte_ess_base_proc_get_locality,
orte_ess_base_proc_get_daemon,
orte_ess_base_proc_get_hostname,
orte_ess_base_proc_get_local_rank,
orte_ess_base_proc_get_node_rank,
orte_ess_base_proc_get_epoch, /* proc_get_epoch */
update_pidmap,
update_nidmap,
orte_ess_base_update_pidmap,
orte_ess_base_update_nidmap,
NULL /* ft_event */
};
@ -187,145 +180,6 @@ static int rte_finalize(void)
return ret;
}
static uint8_t proc_get_locality(orte_process_name_t *proc)
{
orte_nid_t *nid;
if (NULL == (nid = orte_util_lookup_nid(proc))) {
ORTE_ERROR_LOG(ORTE_ERR_NOT_FOUND);
return OPAL_PROC_NON_LOCAL;
}
if (nid->daemon == ORTE_PROC_MY_DAEMON->vpid) {
OPAL_OUTPUT_VERBOSE((2, orte_ess_base_output,
"%s ess:tm: proc %s is LOCAL",
ORTE_NAME_PRINT(ORTE_PROC_MY_NAME),
ORTE_NAME_PRINT(proc)));
return (OPAL_PROC_ON_NODE | OPAL_PROC_ON_CU | OPAL_PROC_ON_CLUSTER);
}
OPAL_OUTPUT_VERBOSE((2, orte_ess_base_output,
"%s ess:tm: proc %s is REMOTE",
ORTE_NAME_PRINT(ORTE_PROC_MY_NAME),
ORTE_NAME_PRINT(proc)));
return OPAL_PROC_NON_LOCAL;
}
static orte_vpid_t proc_get_daemon(orte_process_name_t *proc)
{
orte_nid_t *nid;
if( ORTE_JOBID_IS_DAEMON(proc->jobid) ) {
return proc->vpid;
}
if (NULL == (nid = orte_util_lookup_nid(proc))) {
return ORTE_VPID_INVALID;
}
OPAL_OUTPUT_VERBOSE((2, orte_ess_base_output,
"%s ess:tm: proc %s is hosted by daemon %s",
ORTE_NAME_PRINT(ORTE_PROC_MY_NAME),
ORTE_NAME_PRINT(proc),
ORTE_VPID_PRINT(nid->daemon)));
return nid->daemon;
}
static char* proc_get_hostname(orte_process_name_t *proc)
{
orte_nid_t *nid;
if (NULL == (nid = orte_util_lookup_nid(proc))) {
ORTE_ERROR_LOG(ORTE_ERR_NOT_FOUND);
return NULL;
}
OPAL_OUTPUT_VERBOSE((2, orte_ess_base_output,
"%s ess:tm: proc %s is on host %s",
ORTE_NAME_PRINT(ORTE_PROC_MY_NAME),
ORTE_NAME_PRINT(proc),
nid->name));
return nid->name;
}
static orte_local_rank_t proc_get_local_rank(orte_process_name_t *proc)
{
orte_pmap_t *pmap;
if (NULL == (pmap = orte_util_lookup_pmap(proc))) {
ORTE_ERROR_LOG(ORTE_ERR_NOT_FOUND);
return ORTE_LOCAL_RANK_INVALID;
}
OPAL_OUTPUT_VERBOSE((2, orte_ess_base_output,
"%s ess:tm: proc %s has local rank %d",
ORTE_NAME_PRINT(ORTE_PROC_MY_NAME),
ORTE_NAME_PRINT(proc),
(int)pmap->local_rank));
return pmap->local_rank;
}
static orte_node_rank_t proc_get_node_rank(orte_process_name_t *proc)
{
orte_pmap_t *pmap;
orte_ns_cmp_bitmask_t mask;
mask = ORTE_NS_CMP_ALL;
/* is this me? */
if (OPAL_EQUAL == orte_util_compare_name_fields(mask, proc, ORTE_PROC_MY_NAME)) {
/* yes it is - reply with my rank. This is necessary
* because the pidmap will not have arrived when I
* am starting up, and if we use static ports, then
* I need to know my node rank during init
*/
return my_node_rank;
}
if (NULL == (pmap = orte_util_lookup_pmap(proc))) {
return ORTE_NODE_RANK_INVALID;
}
OPAL_OUTPUT_VERBOSE((2, orte_ess_base_output,
"%s ess:tm: proc %s has node rank %d",
ORTE_NAME_PRINT(ORTE_PROC_MY_NAME),
ORTE_NAME_PRINT(proc),
(int)pmap->node_rank));
return pmap->node_rank;
}
static int update_pidmap(opal_byte_object_t *bo)
{
int ret;
OPAL_OUTPUT_VERBOSE((2, orte_ess_base_output,
"%s ess:tm: updating pidmap",
ORTE_NAME_PRINT(ORTE_PROC_MY_NAME)));
/* build the pmap */
if (ORTE_SUCCESS != (ret = orte_util_decode_pidmap(bo))) {
ORTE_ERROR_LOG(ret);
}
return ret;
}
static int update_nidmap(opal_byte_object_t *bo)
{
int rc;
/* decode the nidmap - the util will know what to do */
if (ORTE_SUCCESS != (rc = orte_util_decode_nodemap(bo))) {
ORTE_ERROR_LOG(rc);
}
return rc;
}
static int tm_set_name(void)
{
int rc;

48
orte/mca/grpcomm/bad/grpcomm_bad_module.c
Просмотреть файл

@ -24,9 +24,10 @@
#include <string.h>
#include "orte/util/proc_info.h"
#include "opal/dss/dss.h"
#include "opal/util/opal_sos.h"
#include "opal/mca/hwloc/base/base.h"
#include "orte/mca/errmgr/errmgr.h"
#include "orte/mca/odls/base/base.h"
#include "orte/mca/odls/odls_types.h"
@ -36,6 +37,7 @@
#include "orte/mca/routed/routed.h"
#include "orte/runtime/orte_globals.h"
#include "orte/util/name_fns.h"
#include "orte/util/proc_info.h"
#include "orte/orted/orted.h"
#include "orte/runtime/orte_wait.h"
@ -280,6 +282,7 @@ static int modex(opal_list_t *procs)
{
int rc;
opal_buffer_t buf, rbuf;
char *locale=NULL;
OPAL_OUTPUT_VERBOSE((1, orte_grpcomm_base.output,
"%s grpcomm:bad: modex entered",
@ -305,6 +308,47 @@ static int modex(opal_list_t *procs)
goto cleanup;
}
#if OPAL_HAVE_HWLOC
{
if (NULL != opal_hwloc_topology) {
/* our cpuset should already be known, but check for safety */
if (NULL == opal_hwloc_my_cpuset) {
opal_hwloc_base_get_local_cpuset();
}
/* convert to a string */
hwloc_bitmap_list_asprintf(&locale, opal_hwloc_my_cpuset);
OPAL_OUTPUT_VERBOSE((5, orte_grpcomm_base.output,
"%s grpcomm:bad LOCALE %s",
ORTE_NAME_PRINT(ORTE_PROC_MY_NAME), locale));
/* pack it */
if (ORTE_SUCCESS != (rc = opal_dss.pack(&buf, &locale, 1, OPAL_STRING))) {
ORTE_ERROR_LOG(rc);
free(locale);
goto cleanup;
}
free(locale);
} else {
OPAL_OUTPUT_VERBOSE((5, orte_grpcomm_base.output,
"%s grpcomm:bad NO TOPO - ADDING PLACEHOLDER",
ORTE_NAME_PRINT(ORTE_PROC_MY_NAME)));
/* pack a placeholder */
if (ORTE_SUCCESS != (rc = opal_dss.pack(&buf, &locale, 1, OPAL_STRING))) {
ORTE_ERROR_LOG(rc);
goto cleanup;
}
}
}
#else
OPAL_OUTPUT_VERBOSE((5, orte_grpcomm_base.output,
"%s grpcomm:bad NO HWLOC - ADDING PLACEHOLDER",
ORTE_NAME_PRINT(ORTE_PROC_MY_NAME)));
/* pack a placeholder */
if (ORTE_SUCCESS != (rc = opal_dss.pack(&buf, &locale, 1, OPAL_STRING))) {
ORTE_ERROR_LOG(rc);
goto cleanup;
}
#endif
/* pack the entries we have received */
if (ORTE_SUCCESS != (rc = orte_grpcomm_base_pack_modex_entries(&buf))) {
ORTE_ERROR_LOG(rc);
@ -325,7 +369,7 @@ static int modex(opal_list_t *procs)
OPAL_OUTPUT_VERBOSE((1, orte_grpcomm_base.output,
"%s grpcomm:bad: modex posted",
ORTE_NAME_PRINT(ORTE_PROC_MY_NAME)));
cleanup:
cleanup:
OBJ_DESTRUCT(&buf);
OBJ_DESTRUCT(&rbuf);

4
orte/mca/grpcomm/base/base.h
Просмотреть файл

@ -30,6 +30,7 @@
#include "opal/mca/mca.h"
#include "opal/threads/mutex.h"
#include "opal/threads/condition.h"
#include "opal/mca/hwloc/hwloc.h"
#include "orte/mca/odls/odls_types.h"
@ -60,6 +61,9 @@ typedef struct {
opal_list_t components_available;
orte_grpcomm_base_component_t selected_component;
orte_grpcomm_daemon_collective_fn_t daemon_coll;
#if OPAL_HAVE_HWLOC
hwloc_cpuset_t working_cpuset;
#endif
} orte_grpcomm_base_t;
ORTE_DECLSPEC extern orte_grpcomm_base_t orte_grpcomm_base;

9
orte/mca/grpcomm/base/grpcomm_base_close.c
Просмотреть файл

@ -22,6 +22,8 @@
#include "opal/mca/mca.h"
#include "opal/mca/base/base.h"
#include "opal/mca/hwloc/hwloc.h"
#include "orte/mca/grpcomm/base/base.h"
@ -39,6 +41,13 @@ int orte_grpcomm_base_close(void)
mca_base_components_close(orte_grpcomm_base.output,
&orte_grpcomm_base.components_available, NULL);
#if OPAL_HAVE_HWLOC
if (NULL != orte_grpcomm_base.working_cpuset) {
hwloc_bitmap_free(orte_grpcomm_base.working_cpuset);
orte_grpcomm_base.working_cpuset = NULL;
}
#endif
/* All done */
return ORTE_SUCCESS;

238
orte/mca/grpcomm/base/grpcomm_base_modex.c
Просмотреть файл

@ -11,6 +11,7 @@
* Copyright (c) 2004-2005 The Regents of the University of California.
* All rights reserved.
* Copyright (c) 2007 Sun Microsystems, Inc. All rights reserved.
* Copyright (c) 2011 Cisco Systems, Inc. All rights reserved.
* $COPYRIGHT$
*
* Additional copyrights may follow
@ -31,6 +32,7 @@
#include "opal/util/output.h"
#include "opal/class/opal_hash_table.h"
#include "opal/dss/dss.h"
#include "opal/mca/hwloc/base/base.h"
#include "orte/util/proc_info.h"
#include "orte/mca/errmgr/errmgr.h"
@ -59,7 +61,8 @@ int orte_grpcomm_base_full_modex(opal_list_t *procs)
orte_pmap_t *pmap;
orte_vpid_t daemon;
char *hostname;
char *locale=NULL;
OPAL_OUTPUT_VERBOSE((1, orte_grpcomm_base.output,
"%s grpcomm:base:full:modex: performing modex",
ORTE_NAME_PRINT(ORTE_PROC_MY_NAME)));
@ -100,6 +103,48 @@ int orte_grpcomm_base_full_modex(opal_list_t *procs)
goto cleanup;
}
#if OPAL_HAVE_HWLOC
{
/* get and pack our cpuset so other procs can determine our locality */
if (NULL != opal_hwloc_topology) {
/* our cpuset should already be known, but check for safety */
if (NULL == opal_hwloc_my_cpuset) {
opal_hwloc_base_get_local_cpuset();
}
/* convert to a string */
hwloc_bitmap_list_asprintf(&locale, opal_hwloc_my_cpuset);
OPAL_OUTPUT_VERBOSE((5, orte_grpcomm_base.output,
"%s grpcomm:base:modex LOCALE %s",
ORTE_NAME_PRINT(ORTE_PROC_MY_NAME), locale));
/* pack it */
if (ORTE_SUCCESS != (rc = opal_dss.pack(&buf, &locale, 1, OPAL_STRING))) {
ORTE_ERROR_LOG(rc);
free(locale);
goto cleanup;
}
free(locale);
} else {
/* pack a placeholder */
OPAL_OUTPUT_VERBOSE((5, orte_grpcomm_base.output,
"%s grpcomm:base:modex NO TOPO - ADDING PLACEHOLDER",
ORTE_NAME_PRINT(ORTE_PROC_MY_NAME)));
if (ORTE_SUCCESS != (rc = opal_dss.pack(&buf, &locale, 1, OPAL_STRING))) {
ORTE_ERROR_LOG(rc);
goto cleanup;
}
}
}
#else
/* pack a placeholder */
OPAL_OUTPUT_VERBOSE((5, orte_grpcomm_base.output,
"%s grpcomm:base:modex NO HWLOC - ADDING PLACEHOLDER",
ORTE_NAME_PRINT(ORTE_PROC_MY_NAME)));
if (ORTE_SUCCESS != (rc = opal_dss.pack(&buf, &locale, 1, OPAL_STRING))) {
ORTE_ERROR_LOG(rc);
goto cleanup;
}
#endif
/* pack the entries we have received */
if (ORTE_SUCCESS != (rc = orte_grpcomm_base_pack_modex_entries(&buf))) {
ORTE_ERROR_LOG(rc);
@ -223,7 +268,7 @@ int orte_grpcomm_base_full_modex(opal_list_t *procs)
opal_pointer_array_set_item(&jmap->pmap, proc_name.vpid, pmap);
} else {
/* see if we have this proc in a pidmap */
if (NULL == orte_util_lookup_pmap(&proc_name)) {
if (NULL == (pmap = orte_util_lookup_pmap(&proc_name))) {
/* proc wasn't found - let's add it */
OPAL_OUTPUT_VERBOSE((5, orte_grpcomm_base.output,
"%s grpcomm:base:full:modex no pidmap entry for proc %s",
@ -241,7 +286,65 @@ int orte_grpcomm_base_full_modex(opal_list_t *procs)
jmap->num_procs++;
}
}
/* unpack the locality info */
cnt = 1;
if (ORTE_SUCCESS != (rc = opal_dss.unpack(&rbuf, &locale, &cnt, OPAL_STRING))) {
ORTE_ERROR_LOG(rc);
goto cleanup;
}
OPAL_OUTPUT_VERBOSE((2, orte_grpcomm_base.output,
"%s grpcomm:base:modex setting proc %s locale %s",
ORTE_NAME_PRINT(ORTE_PROC_MY_NAME),
ORTE_NAME_PRINT(&proc_name),
(NULL == locale) ? "NULL" : locale));
/* store on the pmap */
if (OPAL_EQUAL == orte_util_compare_name_fields(ORTE_NS_CMP_ALL, &proc_name, ORTE_PROC_MY_NAME)) {
/* if this data is from myself, then set locality to all */
OPAL_OUTPUT_VERBOSE((5, orte_grpcomm_base.output,
"%s grpcomm:base:modex setting proc %s locale ALL",
ORTE_NAME_PRINT(ORTE_PROC_MY_NAME),
ORTE_NAME_PRINT(&proc_name)));
pmap->locality = OPAL_PROC_ALL_LOCAL;
} else if (daemon != ORTE_PROC_MY_DAEMON->vpid) {
/* this is on a different node, then mark as non-local */
OPAL_OUTPUT_VERBOSE((5, orte_grpcomm_base.output,
"%s grpcomm:base:modex setting proc %s locale NONLOCAL",
ORTE_NAME_PRINT(ORTE_PROC_MY_NAME),
ORTE_NAME_PRINT(&proc_name)));
pmap->locality = OPAL_PROC_NON_LOCAL;
} else if (NULL == locale || 0 == strlen(locale)){
/* if we share a node, but we don't know anything more, then
* mark us as on the node as this is all we know
*/
OPAL_OUTPUT_VERBOSE((5, orte_grpcomm_base.output,
"%s grpcomm:base:modex setting proc %s locale NODE",
ORTE_NAME_PRINT(ORTE_PROC_MY_NAME),
ORTE_NAME_PRINT(&proc_name)));
pmap->locality = OPAL_PROC_ON_NODE;
} else {
/* convert the locale to a cpuset */
if (NULL == orte_grpcomm_base.working_cpuset) {
orte_grpcomm_base.working_cpuset = hwloc_bitmap_alloc();
}
if (0 != hwloc_bitmap_list_sscanf(orte_grpcomm_base.working_cpuset, locale)) {
/* got a bad locale */
ORTE_ERROR_LOG(ORTE_ERR_VALUE_OUT_OF_BOUNDS);
rc = ORTE_ERR_VALUE_OUT_OF_BOUNDS;
goto cleanup;
}
/* determine relative location on our node */
pmap->locality = opal_hwloc_base_get_relative_locality(opal_hwloc_topology,
opal_hwloc_my_cpuset,
orte_grpcomm_base.working_cpuset);
OPAL_OUTPUT_VERBOSE((5, orte_grpcomm_base.output,
"%s grpcomm:base:modex setting proc %s locale %04x",
ORTE_NAME_PRINT(ORTE_PROC_MY_NAME),
ORTE_NAME_PRINT(&proc_name), pmap->locality));
}
OPAL_OUTPUT_VERBOSE((5, orte_grpcomm_base.output,
"%s grpcomm:base:full:modex: adding modex entry for proc %s",
ORTE_NAME_PRINT(ORTE_PROC_MY_NAME),
@ -266,6 +369,9 @@ int orte_grpcomm_base_modex_unpack( opal_buffer_t* rbuf)
orte_std_cntr_t cnt;
orte_process_name_t proc_name;
int rc=ORTE_SUCCESS;
orte_vpid_t daemon;
orte_pmap_t *pmap;
char *locale;
/* process the results */
/* extract the number of procs that put data in the buffer */
@ -294,16 +400,87 @@ int orte_grpcomm_base_modex_unpack( opal_buffer_t* rbuf)
goto cleanup;
}
/* SINCE THIS IS AMONGST PEERS, THERE IS NO NEED TO UPDATE THE NIDMAP/PIDMAP */
/* SINCE THIS IS AMONGST PEERS, THERE IS NO NEED TO UPDATE THE NIDMAP/PIDMAP
* ITSELF, EXCEPT FOR LOCALITY INFO
*/
if (ORTE_VPID_INVALID == (daemon = orte_ess.proc_get_daemon(&proc_name))) {
/* clear problem */
ORTE_ERROR_LOG(ORTE_ERR_NOT_FOUND);
rc = ORTE_ERR_NOT_FOUND;
goto cleanup;
}
if (NULL == (pmap = orte_util_lookup_pmap(&proc_name))) {
/* clear problem */
ORTE_ERROR_LOG(ORTE_ERR_NOT_FOUND);
rc = ORTE_ERR_NOT_FOUND;
goto cleanup;
}
/* unpack the locality info */
cnt = 1;
if (ORTE_SUCCESS != (rc = opal_dss.unpack(rbuf, &locale, &cnt, OPAL_STRING))) {
ORTE_ERROR_LOG(rc);
goto cleanup;
}
OPAL_OUTPUT_VERBOSE((5, orte_grpcomm_base.output,
"%s grpcomm:base:modex:unpack received proc %s locale %s",
ORTE_NAME_PRINT(ORTE_PROC_MY_NAME),
ORTE_NAME_PRINT(&proc_name),
(NULL == locale) ? "NULL" : locale));
if (OPAL_EQUAL == orte_util_compare_name_fields(ORTE_NS_CMP_ALL, &proc_name, ORTE_PROC_MY_NAME)) {
/* if this data is from myself, then set locality to all */
OPAL_OUTPUT_VERBOSE((5, orte_grpcomm_base.output,
"%s grpcomm:base:modex:unpack setting proc %s locale ALL",
ORTE_NAME_PRINT(ORTE_PROC_MY_NAME),
ORTE_NAME_PRINT(&proc_name)));
pmap->locality = OPAL_PROC_ALL_LOCAL;
} else if (daemon != ORTE_PROC_MY_DAEMON->vpid) {
/* this is on a different node, then mark as non-local */
OPAL_OUTPUT_VERBOSE((5, orte_grpcomm_base.output,
"%s grpcomm:base:modex:unpack setting proc %s locale NONLOCAL",
ORTE_NAME_PRINT(ORTE_PROC_MY_NAME),
ORTE_NAME_PRINT(&proc_name)));
pmap->locality = OPAL_PROC_NON_LOCAL;
} else if (NULL == locale || 0 == strlen(locale)){
/* if we share a node, but we don't know anything more, then
* mark us as on the node as this is all we know
*/
OPAL_OUTPUT_VERBOSE((5, orte_grpcomm_base.output,
"%s grpcomm:base:modex:unpack setting proc %s locale NODE",
ORTE_NAME_PRINT(ORTE_PROC_MY_NAME),
ORTE_NAME_PRINT(&proc_name)));
pmap->locality = OPAL_PROC_ON_NODE;
} else {
/* convert the locale to a cpuset */
if (NULL == orte_grpcomm_base.working_cpuset) {
orte_grpcomm_base.working_cpuset = hwloc_bitmap_alloc();
}
if (0 != hwloc_bitmap_list_sscanf(orte_grpcomm_base.working_cpuset, locale)) {
/* got a bad locale */
ORTE_ERROR_LOG(ORTE_ERR_VALUE_OUT_OF_BOUNDS);
rc = ORTE_ERR_VALUE_OUT_OF_BOUNDS;
goto cleanup;
}
/* determine relative location on our node */
pmap->locality = opal_hwloc_base_get_relative_locality(opal_hwloc_topology,
opal_hwloc_my_cpuset,
orte_grpcomm_base.working_cpuset);
OPAL_OUTPUT_VERBOSE((5, orte_grpcomm_base.output,
"%s grpcomm:base:modex:unpack setting proc %s locale %04x",
ORTE_NAME_PRINT(ORTE_PROC_MY_NAME),
ORTE_NAME_PRINT(&proc_name), pmap->locality));
}
OPAL_OUTPUT_VERBOSE((5, orte_grpcomm_base.output,
"%s grpcomm:base:modex:unpack: adding modex entry for proc %s",
ORTE_NAME_PRINT(ORTE_PROC_MY_NAME),
ORTE_NAME_PRINT(&proc_name)));
/* pass the rest of the buffer
* to that system to update the modex database
*/
/* update the modex database */
if (ORTE_SUCCESS != (rc = orte_grpcomm_base_update_modex_entries(&proc_name, rbuf))) {
ORTE_ERROR_LOG(rc);
goto cleanup;
@ -318,7 +495,8 @@ int orte_grpcomm_base_peer_modex(void)
{
opal_buffer_t buf, rbuf;
int rc = ORTE_SUCCESS;
char *locale=NULL;
OPAL_OUTPUT_VERBOSE((1, orte_grpcomm_base.output,
"%s grpcomm:base:peer:modex: performing modex",
ORTE_NAME_PRINT(ORTE_PROC_MY_NAME)));
@ -333,6 +511,48 @@ int orte_grpcomm_base_peer_modex(void)
goto cleanup;
}
#if OPAL_HAVE_HWLOC
{
if (NULL != opal_hwloc_topology) {
/* our cpuset should already be known, but check for safety */
if (NULL == opal_hwloc_my_cpuset) {
opal_hwloc_base_get_local_cpuset();
}
/* convert to a string */
hwloc_bitmap_list_asprintf(&locale, opal_hwloc_my_cpuset);
OPAL_OUTPUT_VERBOSE((5, orte_grpcomm_base.output,
"%s grpcomm:base:peer:modex LOCALE %s",
ORTE_NAME_PRINT(ORTE_PROC_MY_NAME), locale));
/* pack it */
if (ORTE_SUCCESS != (rc = opal_dss.pack(&buf, &locale, 1, OPAL_STRING))) {
ORTE_ERROR_LOG(rc);
free(locale);
goto cleanup;
}
free(locale);
} else {
/* pack a placeholder */
OPAL_OUTPUT_VERBOSE((5, orte_grpcomm_base.output,
"%s grpcomm:base:peer:modex NO TOPO - ADDING PLACEHOLDER",
ORTE_NAME_PRINT(ORTE_PROC_MY_NAME)));
if (ORTE_SUCCESS != (rc = opal_dss.pack(&buf, &locale, 1, OPAL_STRING))) {
ORTE_ERROR_LOG(rc);
goto cleanup;
}
}
}
#else
/* pack a placeholder */
OPAL_OUTPUT_VERBOSE((5, orte_grpcomm_base.output,
"%s grpcomm:base:peer:modex NO HWLOC - ADDING PLACEHOLDER",
ORTE_NAME_PRINT(ORTE_PROC_MY_NAME)));
if (ORTE_SUCCESS != (rc = opal_dss.pack(&buf, &locale, 1, OPAL_STRING))) {
ORTE_ERROR_LOG(rc);
goto cleanup;
}
#endif
/* pack the entries we have received */
if (ORTE_SUCCESS != (rc = orte_grpcomm_base_pack_modex_entries(&buf))) {
ORTE_ERROR_LOG(rc);
@ -358,7 +578,7 @@ int orte_grpcomm_base_peer_modex(void)
goto cleanup;
}
cleanup:
cleanup:
OBJ_DESTRUCT(&buf);
OBJ_DESTRUCT(&rbuf);
return rc;

5
orte/mca/grpcomm/base/grpcomm_base_open.c
Просмотреть файл

@ -25,7 +25,6 @@
#include "opal/mca/base/base.h"
#include "opal/mca/base/mca_base_param.h"
#include "orte/mca/grpcomm/base/base.h"
@ -61,6 +60,10 @@ int orte_grpcomm_base_open(void)
orte_grpcomm_base.daemon_coll = orte_grpcomm_base_daemon_collective;
#endif
#if OPAL_HAVE_HWLOC
orte_grpcomm_base.working_cpuset = NULL;
#endif
/* Open up all available components */
if (ORTE_SUCCESS !=

96
orte/mca/grpcomm/pmi/grpcomm_pmi_module.c
Просмотреть файл

@ -17,6 +17,7 @@
#include <pmi.h>
#include "opal/dss/dss.h"
#include "opal/mca/hwloc/base/base.h"
#include "orte/mca/errmgr/errmgr.h"
#include "orte/mca/rml/rml.h"
@ -311,12 +312,44 @@ static int modex(opal_list_t *procs)
}
free(rml_uri);
#if OPAL_HAVE_HWLOC
{
char *locale;
/* provide the locality info */
if (NULL != opal_hwloc_topology) {
/* our cpuset should already be known, but check for safety */
if (NULL == opal_hwloc_my_cpuset) {
opal_hwloc_base_get_local_cpuset();
}
/* convert to a string */
hwloc_bitmap_list_asprintf(&locale, opal_hwloc_my_cpuset);
OPAL_OUTPUT_VERBOSE((2, orte_grpcomm_base.output,
"%s grpcomm:pmi LOCALE %s",
ORTE_NAME_PRINT(ORTE_PROC_MY_NAME), locale));
/* get the key */
if (ORTE_SUCCESS != (rc = setup_key(ORTE_PROC_MY_NAME, "HWLOC"))) {
ORTE_ERROR_LOG(rc);
return rc;
}
/* enter the key-value */
rc = PMI_KVS_Put(pmi_kvs_name, pmi_kvs_key, locale);
if (PMI_SUCCESS != rc) {
ORTE_PMI_ERROR(rc, "PMI_KVS_Put");
free(locale);
return ORTE_ERROR;
}
free(locale);
}
}
#endif
/* get the job map for this job */
jmap = (orte_jmap_t*)opal_pointer_array_get_item(&orte_jobmap, 0);
/* get my pidmap entry */
pmap = (orte_pmap_t*)opal_pointer_array_get_item(&jmap->pmap, ORTE_PROC_MY_NAME->vpid);
/* add our locality info */
/* add our local/node rank info */
if (ORTE_SUCCESS != (rc = setup_key(ORTE_PROC_MY_NAME, "LOCALRANK"))) {
ORTE_ERROR_LOG(rc);
return rc;
@ -369,7 +402,7 @@ static int modex(opal_list_t *procs)
ORTE_PMI_ERROR(rc, "PMI_KVS_Get");
return ORTE_ERROR;
}
OPAL_OUTPUT_VERBOSE((5, orte_grpcomm_base.output,
OPAL_OUTPUT_VERBOSE((2, orte_grpcomm_base.output,
"%s grpcomm:pmi: proc %s oob endpoint %s",
ORTE_NAME_PRINT(ORTE_PROC_MY_NAME),
ORTE_NAME_PRINT(&name), pmi_attr_val));
@ -386,7 +419,7 @@ static int modex(opal_list_t *procs)
ORTE_PMI_ERROR(rc, "PMI_KVS_Get");
return ORTE_ERROR;
}
OPAL_OUTPUT_VERBOSE((5, orte_grpcomm_base.output,
OPAL_OUTPUT_VERBOSE((2, orte_grpcomm_base.output,
"%s grpcomm:pmi: proc %s location %s",
ORTE_NAME_PRINT(ORTE_PROC_MY_NAME),
ORTE_NAME_PRINT(&name), pmi_attr_val));
@ -421,7 +454,7 @@ static int modex(opal_list_t *procs)
return rc;
}
}
/* get the proc's locality info */
/* get the proc's local/node rank info */
if (ORTE_SUCCESS != (rc = setup_key(&name, "LOCALRANK"))) {
ORTE_ERROR_LOG(rc);
return rc;
@ -442,12 +475,65 @@ static int modex(opal_list_t *procs)
return ORTE_ERROR;
}
pmap->node_rank = (uint16_t)strtoul(pmi_attr_val, NULL, 10);
OPAL_OUTPUT_VERBOSE((5, orte_grpcomm_base.output,
OPAL_OUTPUT_VERBOSE((2, orte_grpcomm_base.output,
"%s grpcomm:pmi: proc %s lrank %u nrank %u",
ORTE_NAME_PRINT(ORTE_PROC_MY_NAME),
ORTE_NAME_PRINT(&name),
(unsigned int)pmap->local_rank,
(unsigned int)pmap->node_rank));
#if OPAL_HAVE_HWLOC
/* get the proc's locality info, if available */
if (ORTE_SUCCESS != (rc = setup_key(&name, "HWLOC"))) {
ORTE_ERROR_LOG(rc);
return rc;
}
rc = PMI_KVS_Get(pmi_kvs_name, pmi_kvs_key, pmi_attr_val, pmi_vallen_max);
/* don't error out here - if not found, that's okay */
if (PMI_SUCCESS == rc) {
if (OPAL_EQUAL == orte_util_compare_name_fields(ORTE_NS_CMP_ALL, &name, ORTE_PROC_MY_NAME)) {
/* if this data is from myself, then set locality to all */
OPAL_OUTPUT_VERBOSE((2, orte_grpcomm_base.output,
"%s grpcomm:pmi setting proc %s locale ALL",
ORTE_NAME_PRINT(ORTE_PROC_MY_NAME),
ORTE_NAME_PRINT(&name)));
pmap->locality = OPAL_PROC_ALL_LOCAL;
} else if (loc->daemon != ORTE_PROC_MY_DAEMON->vpid) {
/* this is on a different node, then mark as non-local */
OPAL_OUTPUT_VERBOSE((2, orte_grpcomm_base.output,
"%s grpcomm:pmi setting proc %s locale NONLOCAL",
ORTE_NAME_PRINT(ORTE_PROC_MY_NAME),
ORTE_NAME_PRINT(&name)));
pmap->locality = OPAL_PROC_NON_LOCAL;
} else if (0 == strlen(pmi_attr_val)){
/* if we share a node, but we don't know anything more, then
* mark us as on the node as this is all we know
*/
OPAL_OUTPUT_VERBOSE((2, orte_grpcomm_base.output,
"%s grpcomm:pmi setting proc %s locale NODE",
ORTE_NAME_PRINT(ORTE_PROC_MY_NAME),
ORTE_NAME_PRINT(&name)));
pmap->locality = OPAL_PROC_ON_NODE;
} else {
/* convert the locale to a cpuset */
if (NULL == orte_grpcomm_base.working_cpuset) {
orte_grpcomm_base.working_cpuset = hwloc_bitmap_alloc();
}
if (0 != hwloc_bitmap_list_sscanf(orte_grpcomm_base.working_cpuset, pmi_attr_val)) {
/* got a bad locale */
ORTE_ERROR_LOG(ORTE_ERR_VALUE_OUT_OF_BOUNDS);
return ORTE_ERR_VALUE_OUT_OF_BOUNDS;
}
/* determine relative location on our node */
pmap->locality = opal_hwloc_base_get_relative_locality(opal_hwloc_topology,
opal_hwloc_my_cpuset,
orte_grpcomm_base.working_cpuset);
OPAL_OUTPUT_VERBOSE((2, orte_grpcomm_base.output,
"%s grpcommpmi setting proc %s locale %04x",
ORTE_NAME_PRINT(ORTE_PROC_MY_NAME),
ORTE_NAME_PRINT(&name), pmap->locality));
}
}
#endif
}
/* cycle thru the array of our peers and assign local and node ranks */

2
orte/runtime/orte_globals.c
Просмотреть файл

@ -1021,8 +1021,8 @@ static void orte_pmap_construct(orte_pmap_t *ptr)
ptr->node = -1;
ptr->local_rank = ORTE_LOCAL_RANK_INVALID;
ptr->node_rank = ORTE_NODE_RANK_INVALID;
ptr->locality = OPAL_PROC_LOCALITY_UNKNOWN;
}
OBJ_CLASS_INSTANCE(orte_pmap_t,
opal_object_t,
orte_pmap_construct,

3
orte/runtime/orte_globals.h
Просмотреть файл

@ -39,6 +39,7 @@
#include "opal/class/opal_ring_buffer.h"
#include "opal/threads/threads.h"
#include "opal/mca/hwloc/hwloc.h"
#include "opal/mca/paffinity/paffinity.h"
#include "orte/mca/plm/plm_types.h"
#include "orte/mca/rml/rml_types.h"
@ -542,6 +543,8 @@ typedef struct {
orte_local_rank_t local_rank;
/* node rank */
orte_node_rank_t node_rank;
/* locality */
opal_paffinity_locality_t locality;
} orte_pmap_t;
ORTE_DECLSPEC OBJ_CLASS_DECLARATION(orte_pmap_t);

3
orte/util/nidmap.c
Просмотреть файл

@ -48,7 +48,6 @@
#include "opal/class/opal_pointer_array.h"
#include "opal/mca/hwloc/hwloc.h"
#include "opal/util/output.h"
#include "opal/util/opal_sos.h"
#include "opal/util/argv.h"
#include "orte/mca/errmgr/errmgr.h"
@ -764,7 +763,7 @@ int orte_util_decode_pidmap(opal_byte_object_t *bo)
/* setup for next cycle */
n = 1;
}
if (ORTE_ERR_UNPACK_READ_PAST_END_OF_BUFFER == OPAL_SOS_GET_ERROR_CODE(rc)) {
if (ORTE_ERR_UNPACK_READ_PAST_END_OF_BUFFER == rc) {
rc = ORTE_SUCCESS;
}

8
orte/util/proc_info.c
Просмотреть файл

@ -61,6 +61,7 @@ ORTE_DECLSPEC orte_proc_info_t orte_process_info = {
/* .sync_buf = */ NULL,
/* .my_port = */ 0,
/* .num_restarts = */ 0,
/* .my_node_rank = */ ORTE_NODE_RANK_INVALID,
/* .tmpdir_base = */ NULL,
/* .top_session_dir = */ NULL,
/* .job_session_dir = */ NULL,
@ -169,6 +170,13 @@ int orte_proc_info(void)
true, false, 0, &tmp);
orte_process_info.app_rank = tmp;
/* get my node rank in case we are using static ports - this won't
* be present for daemons, so don't error out if we don't have it
*/
mca_base_param_reg_int_name("orte", "ess_node_rank", "Process node rank",
true, false, ORTE_NODE_RANK_INVALID, &tmp);
orte_process_info.my_node_rank = (orte_node_rank_t)tmp;
/* setup the sync buffer */
orte_process_info.sync_buf = OBJ_NEW(opal_buffer_t);

1
orte/util/proc_info.h
Просмотреть файл

@ -96,6 +96,7 @@ struct orte_proc_info_t {
opal_buffer_t *sync_buf; /**< buffer to store sync response */
uint16_t my_port; /**< TCP port for out-of-band comm */
int32_t num_restarts; /**< number of times this proc has restarted */
orte_node_rank_t my_node_rank; /**< node rank */
/* The session directory has the form
* <prefix>/<openmpi-sessions-user>/<jobid>/<procid>, where the prefix
* can either be provided by the user via the