1
1
openmpi/ompi/mca/common/ugni/common_ugni.c
Ralph Castain c4c9bc1573 As per the RFC:
http://www.open-mpi.org/community/lists/devel/2014/04/14496.php

Revamp the opal database framework, including renaming it to "dstore" to reflect that it isn't a "database". Move the "db" framework to ORTE for now, soon to move to ORCM

This commit was SVN r31557.
2014-04-29 21:49:23 +00:00

316 строки
8.6 KiB
C

/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
/*
* Copyright (c) 2011-2013 Los Alamos National Security, LLC. All rights
* reserved.
* Copyright (c) 2011 UT-Battelle, LLC. All rights reserved.
* Copyright (c) 2014 Intel, Inc. All rights reserved.
* $COPYRIGHT$
*
* Additional copyrights may follow
*
* $HEADER$
*/
#include "common_ugni.h"
#include "ompi/proc/proc.h"
#include "opal/mca/dstore/dstore.h"
#include "opal/class/opal_list.h"
#include "opal/dss/dss.h"
/* NTH: we need some options from the btl */
#include "ompi/mca/btl/ugni/btl_ugni.h"
static int ompi_common_ugni_module_ref_count = 0;
ompi_common_ugni_module_t ompi_common_ugni_module;
mca_base_component_t ompi_common_ugni_component = {
MCA_BASE_VERSION_2_0_0,
"common",
MCA_BASE_VERSION_2_0_0,
"ugni",
MCA_BASE_VERSION_2_0_0,
NULL,
NULL
};
static inline int
get_ptag(uint8_t *out_ptag)
{
/* TODO no need for tmp */
char *ptr;
uint8_t tmp_ptag;
if (NULL == (ptr = getenv("PMI_GNI_PTAG"))) {
/* TODO add err msg - better rc? */
return OMPI_ERR_NOT_FOUND;
}
errno = 0;
tmp_ptag = (uint8_t)strtoul (ptr, (char **)NULL, 10);
if (0 != errno) {
/* TODO add err msg - better rc? */
return OMPI_ERR_VALUE_OUT_OF_BOUNDS;
}
*out_ptag = tmp_ptag;
return OMPI_SUCCESS;
}
static inline int get_cookie (uint32_t *out_cookie)
{
/* TODO no need for tmp */
char *ptr;
uint32_t tmp_cookie;
if (NULL == (ptr = getenv("PMI_GNI_COOKIE"))) {
/* TODO add err msg - better rc? */
return OMPI_ERR_NOT_FOUND;
}
errno = 0;
tmp_cookie = (uint32_t) strtoul (ptr, NULL, 10);
if (0 != errno) {
/* TODO add err msg - better rc? */
return OMPI_ERR_VALUE_OUT_OF_BOUNDS;
}
*out_cookie = tmp_cookie;
return OMPI_SUCCESS;
}
static unsigned int
ompi_common_ugni_get_nic_address(int device_id)
{
unsigned int address, cpu_id;
gni_return_t status;
int i, alps_dev_id = -1;
char *token,*p_ptr;
p_ptr = getenv("PMI_GNI_DEV_ID");
if (!p_ptr) {
status = GNI_CdmGetNicAddress(device_id, &address, &cpu_id);
if(status != GNI_RC_SUCCESS) {
opal_output (0, "FAILED:GNI_CdmGetNicAddress returned error %d", status);
return (unsigned int)-1;
}
return address;
}
while (NULL != (token = strtok(p_ptr, ":"))) {
alps_dev_id = atoi(token);
if (alps_dev_id == device_id) {
break;
}
p_ptr = NULL;
}
if (OPAL_UNLIKELY(-1 == alps_dev_id)) {
return (unsigned int)-1;
}
p_ptr = getenv("PMI_GNI_LOC_ADDR");
if (OPAL_UNLIKELY(NULL == p_ptr)) {
return (unsigned int)-1;
}
i = 0;
while (NULL != (token = strtok(p_ptr, ":"))) {
if (i == alps_dev_id) {
return strtoul (token, NULL, 10);
}
p_ptr = NULL;
++i;
}
return (unsigned int)-1;
}
static int ompi_common_ugni_device_init (ompi_common_ugni_device_t *device,
int device_id)
{
int rc;
/* Create a NIC Adress */
device->dev_id = device_id; /* Minor number of the Gemini NIC */
device->dev_addr = ompi_common_ugni_get_nic_address (device->dev_id);
OPAL_OUTPUT((-1, "Got NIC Addr: 0x%08x, CPU ID: %d", device->dev_addr, device->dev_id));
/* Attach device to the communication domain */
rc = GNI_CdmAttach (ompi_common_ugni_module.cd_handle, device->dev_id,
&device->dev_pe_addr, &device->dev_handle);
if (GNI_RC_SUCCESS != rc) {
OPAL_OUTPUT((0, "Error: Creating communication domain %d\n", rc));
return ompi_common_rc_ugni_to_ompi (rc);
}
return OMPI_SUCCESS;
}
static int ompi_common_ugni_device_fini (ompi_common_ugni_device_t *dev)
{
return OMPI_SUCCESS;
}
/*
* Send local device information and other information
* required for setup
*/
static int ompi_common_ugni_send_modex (int my_rank)
{
uint32_t modex_size, total_msg_size, msg_offset;
struct ompi_common_ugni_modex_t modex;
char *modex_msg;
int rc, i;
modex_size = sizeof (struct ompi_common_ugni_modex_t);
total_msg_size = ompi_common_ugni_module.device_count * modex_size;
modex_msg = (char *) malloc (total_msg_size);
if (NULL == modex_msg) {
OPAL_OUTPUT((-1, "Error allocating memory for modex @ %s:%d",
__FILE__, __LINE__));
return OMPI_ERR_OUT_OF_RESOURCE;
}
/* pack modex for all available devices */
for (i = 0, msg_offset = 0; i < ompi_common_ugni_module.device_count ; ++i) {
ompi_common_ugni_device_t *dev = ompi_common_ugni_module.devices + i;
modex.addr = dev->dev_addr;
modex.id = my_rank;
memcpy ((void *)((uintptr_t) modex_msg + msg_offset),
(void *)&modex, modex_size);
msg_offset += modex_size;
}
rc = ompi_modex_send(&ompi_common_ugni_component,
modex_msg, total_msg_size);
free(modex_msg);
return rc;
}
int ompi_common_ugni_fini (void)
{
int i, rc;
if (0 == ompi_common_ugni_module_ref_count) {
return OMPI_SUCCESS;
}
if (1 == ompi_common_ugni_module_ref_count) {
/* tear down component */
if (ompi_common_ugni_module.devices) {
/* finalize devices */
for (i = 0 ; i < ompi_common_ugni_module.device_count ; ++i) {
ompi_common_ugni_device_fini (ompi_common_ugni_module.devices + i);
}
free (ompi_common_ugni_module.devices);
ompi_common_ugni_module.devices = NULL;
}
/* finally, tear down the communication domain */
rc = GNI_CdmDestroy (ompi_common_ugni_module.cd_handle);
if (GNI_RC_SUCCESS != rc) {
OPAL_OUTPUT((-1, "error destroying cdm"));
}
}
ompi_common_ugni_module_ref_count--;
return OMPI_SUCCESS;
}
int ompi_common_ugni_init (void)
{
ompi_proc_t *my_proc;
int modes, rc, i;
uint32_t my_rank, *ptr;
ompi_common_ugni_module_ref_count ++;
if (ompi_common_ugni_module_ref_count > 1) {
return OMPI_SUCCESS;
}
my_proc = ompi_proc_local ();
/* get a unique id from the runtime */
#if defined(OMPI_DB_GLOBAL_RANK)
{
opal_list_t myvals;
opal_value_t *kv;
ptr = &my_rank;
OBJ_CONSTRUCT(&myvals, opal_list_t);
rc = opal_dstore.fetch (opal_dstore_internal,
ORTE_NAME_PRINT(&my_proc->proc_name),
OMPI_DB_GLOBAL_RANK,
&myvals);
if (OPAL_SUCCESS == rc) {
kv = (opal_value_t*)opal_list_get_first(&myvals):
if (OPAL_SUCCESS != opal_value_unload(&kv, (void**)&ptr, OPAL_UINT32)) {
my_rank = my_proc->proc_name.vpid;
}
} else {
my_rank = my_proc->proc_name.vpid;
}
OPAL_LIST_DESTRUCT(&myvals);
}
#else
my_rank = my_proc->proc_name.vpid;
#endif
/* pull settings from ugni btl */
ompi_common_ugni_module.rdma_max_retries =
mca_btl_ugni_component.rdma_max_retries;
/* Create a communication domain */
modes = GNI_CDM_MODE_FORK_FULLCOPY | GNI_CDM_MODE_CACHED_AMO_ENABLED |
GNI_CDM_MODE_ERR_NO_KILL | GNI_CDM_MODE_FAST_DATAGRAM_POLL;
/* collect uGNI information */
rc = get_ptag(&ompi_common_ugni_module.ptag);
if (OPAL_UNLIKELY(OMPI_SUCCESS != rc)) {
return rc;
}
rc = get_cookie(&ompi_common_ugni_module.cookie);
if (OPAL_UNLIKELY(OMPI_SUCCESS != rc)) {
return rc;
}
/* create a communication domain */
rc = GNI_CdmCreate (my_rank, ompi_common_ugni_module.ptag,
ompi_common_ugni_module.cookie, modes,
&ompi_common_ugni_module.cd_handle);
if (OPAL_UNLIKELY(GNI_RC_SUCCESS != rc)) {
OPAL_OUTPUT((0, "Error: Creating communication domain %d\n",rc));
return ompi_common_rc_ugni_to_ompi (rc);
}
/* setup uGNI devices. we only support one device atm */
ompi_common_ugni_module.device_count = 1;
ompi_common_ugni_module.devices = calloc (ompi_common_ugni_module.device_count,
sizeof (ompi_common_ugni_device_t));
for (i = 0 ; i < ompi_common_ugni_module.device_count ; ++i) {
rc = ompi_common_ugni_device_init (ompi_common_ugni_module.devices + i, i);
if (OPAL_UNLIKELY(OMPI_SUCCESS != rc)) {
OPAL_OUTPUT((-1, "error initializing uGNI device"));
return rc;
}
}
/* send ugni modex */
ompi_common_ugni_send_modex (my_rank);
return OMPI_SUCCESS;
}