266 строки
8.8 KiB
C
266 строки
8.8 KiB
C
/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil indent-tabs-mode:nil -*- */
|
|
/*
|
|
* Copyright (c) 2011-2012 Los Alamos National Security, LLC. All rights
|
|
* reserved.
|
|
* $COPYRIGHT$
|
|
*
|
|
* Additional copyrights may follow
|
|
*
|
|
* $HEADER$
|
|
*
|
|
*/
|
|
|
|
#include "orte_config.h"
|
|
#include "orte/types.h"
|
|
#include "opal/types.h"
|
|
|
|
#include "orte/util/name_fns.h"
|
|
#include "orte/runtime/orte_globals.h"
|
|
#include "orte/util/proc_info.h"
|
|
|
|
#include "oob_ud.h"
|
|
|
|
static int mca_oob_ud_component_open (void);
|
|
static int mca_oob_ud_component_close (void);
|
|
static int mca_oob_ud_component_register (void);
|
|
static mca_oob_t *mca_oob_ud_component_init (int *priority);
|
|
|
|
mca_oob_ud_component_t mca_oob_ud_component = {
|
|
{
|
|
{
|
|
MCA_OOB_BASE_VERSION_2_0_0,
|
|
"ud", /* MCA module name */
|
|
ORTE_MAJOR_VERSION,
|
|
ORTE_MINOR_VERSION,
|
|
ORTE_RELEASE_VERSION,
|
|
mca_oob_ud_component_open,
|
|
mca_oob_ud_component_close,
|
|
NULL, /* component query */
|
|
mca_oob_ud_component_register
|
|
},
|
|
{
|
|
/* The component is checkpoint ready */
|
|
MCA_BASE_METADATA_PARAM_CHECKPOINT
|
|
},
|
|
mca_oob_ud_component_init
|
|
}
|
|
};
|
|
|
|
static int mca_oob_ud_component_open (void)
|
|
{
|
|
OBJ_CONSTRUCT(&mca_oob_ud_component.ud_devices, opal_list_t);
|
|
OBJ_CONSTRUCT(&mca_oob_ud_component.ud_active_sends, opal_list_t);
|
|
OBJ_CONSTRUCT(&mca_oob_ud_component.ud_pending_recvs, opal_list_t);
|
|
|
|
OBJ_CONSTRUCT(&mca_oob_ud_component.ud_active_recvs, opal_list_t);
|
|
OBJ_CONSTRUCT(&mca_oob_ud_component.ud_event_queued_reqs, opal_list_t);
|
|
|
|
OBJ_CONSTRUCT(&mca_oob_ud_component.ud_unexpected_recvs, opal_list_t);
|
|
OBJ_CONSTRUCT(&mca_oob_ud_component.ud_completed, opal_list_t);
|
|
OBJ_CONSTRUCT(&mca_oob_ud_component.ud_event_processing_msgs, opal_list_t);
|
|
|
|
OBJ_CONSTRUCT(&mca_oob_ud_component.ud_lock, opal_mutex_t);
|
|
OBJ_CONSTRUCT(&mca_oob_ud_component.ud_match_lock, opal_mutex_t);
|
|
|
|
OBJ_CONSTRUCT(&mca_oob_ud_component.ud_peers, opal_hash_table_t);
|
|
|
|
return ORTE_SUCCESS;
|
|
}
|
|
|
|
static int mca_oob_ud_component_close (void)
|
|
{
|
|
OPAL_OUTPUT_VERBOSE((5, mca_oob_base_output, "%s oob:ud:component_close entering",
|
|
ORTE_NAME_PRINT(ORTE_PROC_MY_NAME)));
|
|
|
|
OBJ_DESTRUCT(&mca_oob_ud_component.ud_devices);
|
|
OBJ_DESTRUCT(&mca_oob_ud_component.ud_active_sends);
|
|
OBJ_DESTRUCT(&mca_oob_ud_component.ud_pending_recvs);
|
|
|
|
OBJ_DESTRUCT(&mca_oob_ud_component.ud_active_recvs);
|
|
OBJ_DESTRUCT(&mca_oob_ud_component.ud_event_queued_reqs);
|
|
OBJ_DESTRUCT(&mca_oob_ud_component.ud_unexpected_recvs);
|
|
|
|
OBJ_DESTRUCT(&mca_oob_ud_component.ud_lock);
|
|
OBJ_DESTRUCT(&mca_oob_ud_component.ud_match_lock);
|
|
|
|
OBJ_DESTRUCT(&mca_oob_ud_component.ud_peers);
|
|
|
|
OBJ_DESTRUCT(&mca_oob_ud_component.ud_completed);
|
|
OBJ_DESTRUCT(&mca_oob_ud_component.ud_event_processing_msgs);
|
|
|
|
return ORTE_SUCCESS;
|
|
}
|
|
|
|
static int mca_oob_ud_component_register (void)
|
|
{
|
|
mca_base_param_reg_int (&mca_oob_ud_component.super.oob_base,
|
|
"min_qp", "Minimum number of UD queue pairs "
|
|
"to allocate (default: 8)", false, false,
|
|
8, &mca_oob_ud_component.ud_min_qp);
|
|
|
|
mca_base_param_reg_int (&mca_oob_ud_component.super.oob_base,
|
|
"max_qp", "Maximum number of UD queue pairs "
|
|
"to allocate (default: 32)", false, false,
|
|
32, &mca_oob_ud_component.ud_max_qp);
|
|
|
|
mca_base_param_reg_int (&mca_oob_ud_component.super.oob_base,
|
|
"recv_buffers", "Number of MTU sized recv "
|
|
"buffers to post (default: 512)", false, false,
|
|
512, &mca_oob_ud_component.ud_recv_buffer_count);
|
|
|
|
mca_base_param_reg_int (&mca_oob_ud_component.super.oob_base,
|
|
"send_buffers", "Number of MTU sized sent "
|
|
"buffers to post (default: 512)", false, false,
|
|
512, &mca_oob_ud_component.ud_send_buffer_count);
|
|
|
|
return ORTE_SUCCESS;
|
|
}
|
|
|
|
static int port_mtus[] = {0, 256, 512, 1024, 2048, 4096};
|
|
|
|
static inline int mca_oob_ud_port_setup (mca_oob_ud_port_t *port)
|
|
{
|
|
int rc;
|
|
struct ibv_port_attr port_attr;
|
|
|
|
rc = ibv_query_port (port->device->ib_context, port->port_num, &port_attr);
|
|
if (0 != rc || IBV_PORT_ACTIVE != port_attr.state || 0 == port_attr.lid) {
|
|
/* skip this port */
|
|
return ORTE_ERROR;
|
|
}
|
|
|
|
port->lid = port_attr.lid;
|
|
port->mtu = port_attr.active_mtu > IBV_MTU_4096 ? 2048 : port_mtus[port_attr.active_mtu];
|
|
|
|
OPAL_OUTPUT_VERBOSE((5, mca_oob_base_output, "%s oob:ud:port_setup found port: num = %u,"
|
|
"lid = %u, mtu = %u", ORTE_NAME_PRINT(ORTE_PROC_MY_NAME),
|
|
port->port_num, port->lid, port->mtu));
|
|
|
|
return rc;
|
|
}
|
|
|
|
static inline int mca_oob_ud_device_setup (mca_oob_ud_device_t *device,
|
|
struct ibv_device *ib_device)
|
|
{
|
|
int rc, port_num;
|
|
struct ibv_device_attr dev_attr;
|
|
|
|
OPAL_OUTPUT_VERBOSE((5, mca_oob_base_output, "%s oob:ud:device_setup attempting to setup ib device %p",
|
|
ORTE_NAME_PRINT(ORTE_PROC_MY_NAME), (void *) ib_device));
|
|
|
|
device->ib_context = ibv_open_device (ib_device);
|
|
if (NULL == device->ib_context) {
|
|
OPAL_OUTPUT_VERBOSE((5, mca_oob_base_output, "%s oob:ud:device_setup error opening device. errno = %d",
|
|
ORTE_NAME_PRINT(ORTE_PROC_MY_NAME), errno));
|
|
return ORTE_ERROR;
|
|
}
|
|
|
|
rc = ibv_query_device (device->ib_context, &dev_attr);
|
|
if (0 != rc) {
|
|
OPAL_OUTPUT_VERBOSE((5, mca_oob_base_output, "%s oob:ud:device_setup error querying device. errno = %d",
|
|
ORTE_NAME_PRINT(ORTE_PROC_MY_NAME), errno));
|
|
return ORTE_ERROR;
|
|
}
|
|
|
|
device->ib_channel = ibv_create_comp_channel (device->ib_context);
|
|
if (NULL == device->ib_channel) {
|
|
OPAL_OUTPUT_VERBOSE((5, mca_oob_base_output, "%s oob:ud:device_setup error completing completion channel."
|
|
"errno = %d", ORTE_NAME_PRINT(ORTE_PROC_MY_NAME), errno));
|
|
return ORTE_ERROR;
|
|
}
|
|
|
|
device->ib_pd = ibv_alloc_pd (device->ib_context);
|
|
if (NULL == device->ib_pd) {
|
|
OPAL_OUTPUT_VERBOSE((5, mca_oob_base_output, "%s oob:ud:device_setup error allocating protection domain."
|
|
"errno = %d", ORTE_NAME_PRINT(ORTE_PROC_MY_NAME), errno));
|
|
return ORTE_ERROR;
|
|
}
|
|
|
|
for (port_num = 1 ; port_num <= dev_attr.phys_port_cnt ; ++port_num) {
|
|
mca_oob_ud_port_t *port = OBJ_NEW(mca_oob_ud_port_t);
|
|
|
|
if (NULL == port) {
|
|
opal_output (0, "oob:ud:device_setup malloc failure. errno = %d", errno);
|
|
return ORTE_ERR_OUT_OF_RESOURCE;
|
|
}
|
|
|
|
port->device = device;
|
|
port->port_num = port_num;
|
|
|
|
rc = mca_oob_ud_port_setup (port);
|
|
if (ORTE_SUCCESS != rc) {
|
|
OBJ_RELEASE(port);
|
|
continue;
|
|
}
|
|
|
|
opal_list_append (&device->ports, (opal_list_item_t *) port);
|
|
|
|
break;
|
|
}
|
|
|
|
if (0 == opal_list_get_size(&device->ports)) {
|
|
OPAL_OUTPUT_VERBOSE((5, mca_oob_base_output, "%s oob:ud:device_setup could not init device. no usable "
|
|
"ports present", ORTE_NAME_PRINT(ORTE_PROC_MY_NAME)));
|
|
return ORTE_ERROR;
|
|
}
|
|
|
|
return ORTE_SUCCESS;
|
|
}
|
|
|
|
static mca_oob_t *mca_oob_ud_component_init(int *priority)
|
|
{
|
|
struct ibv_device **devices;
|
|
int num_devices, i, rc;
|
|
|
|
/* set the priority so that we will select this component
|
|
* only if someone directs to do so
|
|
*/
|
|
*priority = 0;
|
|
|
|
opal_hash_table_init (&mca_oob_ud_component.ud_peers, 1024);
|
|
|
|
devices = ibv_get_device_list (&num_devices);
|
|
if (NULL == devices || 0 == num_devices) {
|
|
OPAL_OUTPUT_VERBOSE((5, mca_oob_base_output, "%s oob:ud:component_init no devices found",
|
|
ORTE_NAME_PRINT(ORTE_PROC_MY_NAME)));
|
|
return NULL;
|
|
}
|
|
|
|
for (i = 0 ; i < num_devices ; ++i) {
|
|
mca_oob_ud_device_t *device = OBJ_NEW(mca_oob_ud_device_t);
|
|
|
|
if (NULL == device) {
|
|
opal_output (0, "oob:ud:component_init malloc failure. errno = %d",
|
|
errno);
|
|
return NULL;
|
|
}
|
|
|
|
rc = mca_oob_ud_device_setup (device, devices[i]);
|
|
if (ORTE_SUCCESS != rc) {
|
|
OBJ_RELEASE(device);
|
|
continue;
|
|
}
|
|
|
|
opal_list_append (&mca_oob_ud_component.ud_devices,
|
|
(opal_list_item_t *) device);
|
|
|
|
/* NTH: support only 1 device for now */
|
|
break;
|
|
}
|
|
|
|
ibv_free_device_list (devices);
|
|
|
|
if (0 == opal_list_get_size (&mca_oob_ud_component.ud_devices)) {
|
|
OPAL_OUTPUT_VERBOSE((5, mca_oob_base_output, "%s oob:ud:component_init no usable devices found.",
|
|
ORTE_NAME_PRINT(ORTE_PROC_MY_NAME)));
|
|
return NULL;
|
|
}
|
|
|
|
/* have to call the module init here so we can test for available qpair */
|
|
if (ORTE_SUCCESS != mca_oob_ud_module_init()) {
|
|
return NULL;
|
|
}
|
|
|
|
return &mca_oob_ud_module;
|
|
}
|