ports/openmpi
ports/openmpi
1
1
Форкнуть 0
Код Релизы Активность

Bring in the modular-wireup stuff for the openib BTL (from

Просмотр исходного кода 
/tmp/jms-modular-wireup branch):

 * This commit moves all the openib BTL connection code out of
   btl_openib_endpoint.c and into a connect "pseudo-component" area,
   meaning that different schemes for doing OFA connection schemes can
   be chosen via function pointer (i.e., MCA parameter) at run-time.
 * The connect/connect.h file includes comments describing the
   specific interface for the connect pseudo-component.
 * Two pseudo-components are in this commit (more can certainly be
   added).
   * oob: use the same old oob/rml scheme for creating OFA connections
     that we've had forever; this now just puts the logic into this
     self-contained pseudo-component.
   * rdma_cm: a currently-empty set of functions (that currently
     return NOT_IMPLEMENTED) that will someday use the RDMA connection
     manager to make OFA connections.

This commit was SVN r15786.
Этот коммит содержится в:
Jeff Squyres 2007-08-06 23:40:35 +00:00
родитель 96e132b11d
Коммит 50bae9c603
12 изменённых файлов: 1143 добавлений и 786 удалений
Показать статистику Скачать Patch файл Скачать Diff файл Показать все файлы Свернуть все файлы

9
ompi/mca/btl/openib/Makefile.am
Просмотреть файл

@ -48,7 +48,14 @@ sources = \
btl_openib_ini.c \
btl_openib_ini.h \
btl_openib_async.c \
btl_openib_async.h
btl_openib_async.h \
connect/base.h \
connect/btl_openib_connect_base.c \
connect/btl_openib_connect_oob.c \
connect/btl_openib_connect_oob.h \
connect/btl_openib_connect_rdma_cm.c \
connect/btl_openib_connect_rdma_cm.h \
connect/connect.h
# Make the output library in this directory, and name it either
# mca_<type>_<name>.la (for DSO builds) or libmca_<type>_<name>.la

33
ompi/mca/btl/openib/btl_openib_component.c
Просмотреть файл

@ -20,8 +20,12 @@
* $HEADER$
*/
#include "ompi_config.h"
#include <infiniband/verbs.h>
#include <errno.h>
#include <string.h> /* for strerror()*/
#include "ompi/constants.h"
#include "opal/event/event.h"
#include "opal/include/opal/align.h"
@ -29,19 +33,24 @@
#include "opal/util/argv.h"
#include "opal/util/output.h"
#include "opal/util/show_help.h"
#include "ompi/proc/proc.h"
#include "ompi/mca/pml/pml.h"
#include "ompi/mca/btl/btl.h"
#include "opal/sys/timer.h"
#include "opal/sys/atomic.h"
#include "opal/util/argv.h"
#include "opal/mca/base/mca_base_param.h"
#include "orte/mca/errmgr/errmgr.h"
#include "orte/util/sys_info.h"
#include "ompi/proc/proc.h"
#include "ompi/mca/pml/pml.h"
#include "ompi/mca/btl/btl.h"
#include "ompi/mca/mpool/base/base.h"
#include "ompi/mca/mpool/rdma/mpool_rdma.h"
#include "ompi/mca/btl/base/base.h"
#include "ompi/datatype/convertor.h"
#include "ompi/mca/mpool/mpool.h"
#include "ompi/runtime/ompi_module_exchange.h"
#include "btl_openib.h"
#include "btl_openib_frag.h"
#include "btl_openib_endpoint.h"
@ -49,17 +58,11 @@
#include "btl_openib_proc.h"
#include "btl_openib_ini.h"
#include "btl_openib_mca.h"
#include "ompi/datatype/convertor.h"
#include "ompi/mca/mpool/mpool.h"
#if OMPI_HAVE_THREADS
#include "btl_openib_async.h"
#endif
#include <infiniband/verbs.h>
#include <errno.h>
#include <string.h> /* for strerror()*/
#include "connect/base.h"
#include "ompi/runtime/ompi_module_exchange.h"
/*
* Local functions
@ -1097,8 +1100,10 @@ btl_openib_component_init(int *num_btl_modules,
}
/* Post OOB receive to support dynamic connection setup */
mca_btl_openib_post_recv();
/* Setup connect module */
if (OMPI_SUCCESS != ompi_btl_openib_connect_base_select()) {
return NULL;
}
btl_openib_modex_send();
*num_btl_modules = mca_btl_openib_component.ib_num_btls;

795
ompi/mca/btl/openib/btl_openib_endpoint.c
Просмотреть файл

@ -20,56 +20,32 @@
* $HEADER$
*/
#include "ompi_config.h"
#include <sys/time.h>
#include <time.h>
#include "ompi/types.h"
#include "ompi/mca/pml/base/pml_base_sendreq.h"
#include <errno.h>
#include <string.h>
#include "orte/mca/ns/base/base.h"
#include "orte/mca/oob/base/base.h"
#include "orte/mca/rml/rml.h"
#include "orte/mca/errmgr/errmgr.h"
#include "orte/dss/dss.h"
#include "ompi/types.h"
#include "ompi/mca/pml/base/pml_base_sendreq.h"
#include "ompi/class/ompi_free_list.h"
#include "btl_openib.h"
#include "btl_openib_endpoint.h"
#include "btl_openib_proc.h"
#include "btl_openib_frag.h"
#include "ompi/class/ompi_free_list.h"
#include <errno.h>
#include <string.h>
#define ENDPOINT_CONNECT_REQ 0
#define ENDPOINT_CONNECT_RSP 1
#define ENDPOINT_CONNECT_ACK 2
#include "connect/base.h"
static void mca_btl_openib_endpoint_construct(mca_btl_base_endpoint_t* endpoint);
static void mca_btl_openib_endpoint_destruct(mca_btl_base_endpoint_t* endpoint);
int mca_btl_openib_endpoint_create_qp(
mca_btl_openib_module_t* openib_btl,
struct ibv_pd* pd,
struct ibv_cq* cq,
struct ibv_srq* srq,
struct ibv_qp_attr* qp_attr,
struct ibv_qp** qp,
int qp_idx
);
int mca_btl_openib_endpoint_qp_init_query(
mca_btl_openib_module_t* openib_btl,
struct ibv_qp* qp,
struct ibv_qp_attr* attr,
uint32_t lcl_psn,
uint32_t rem_qp_num,
uint32_t rem_psn,
uint16_t rem_lid,
uint32_t rem_mtu,
uint32_t port_num
);
static int post_send(mca_btl_openib_module_t *openib_btl,
mca_btl_openib_endpoint_t *endpoint, mca_btl_openib_frag_t *frag,
const int qp, const int do_rdma)
@ -255,10 +231,7 @@ static inline int mca_btl_openib_endpoint_post_send(mca_btl_openib_module_t* ope
ib_rc, strerror(ib_rc)));
return OMPI_ERROR;
}
#if 0
mca_btl_openib_post_srr_all(openib_btl, 1);
mca_btl_openib_endpoint_post_rr_all(endpoint, 1);
#endif
return OMPI_SUCCESS;
}
@ -428,283 +401,31 @@ static void mca_btl_openib_endpoint_destruct(mca_btl_base_endpoint_t* endpoint)
OBJ_DESTRUCT(&endpoint->pending_put_frags);
}
/*
* Send connection information to remote endpoint using OOB
*
*/
static void mca_btl_openib_endpoint_send_cb(
int status,
orte_process_name_t* endpoint,
orte_buffer_t* buffer,
orte_rml_tag_t tag,
void* cbdata)
{
OBJ_RELEASE(buffer);
}
static int mca_btl_openib_endpoint_send_connect_data(mca_btl_base_endpoint_t* endpoint, uint8_t message_type)
{
orte_buffer_t* buffer = OBJ_NEW(orte_buffer_t);
int rc;
if(NULL == buffer) {
ORTE_ERROR_LOG(ORTE_ERR_OUT_OF_RESOURCE);
return ORTE_ERR_OUT_OF_RESOURCE;
}
/* pack the info in the send buffer */
opal_output(mca_btl_base_output, "packing %d of %d\n", 1, ORTE_UINT8);
rc = orte_dss.pack(buffer, &message_type, 1, ORTE_UINT8);
if(rc != ORTE_SUCCESS) {
ORTE_ERROR_LOG(rc);
return rc;
}
opal_output(mca_btl_base_output, "packing %d of %d\n", 1, ORTE_UINT64);
rc = orte_dss.pack(buffer, &endpoint->subnet_id, 1, ORTE_UINT64);
if(rc != ORTE_SUCCESS) {
ORTE_ERROR_LOG(rc);
return rc;
}
/*GMS left off here */
if(message_type != ENDPOINT_CONNECT_REQ) {
/* send the QP connect request info we respond to */
opal_output(mca_btl_base_output, "packing %d of %d\n", 1, ORTE_UINT32);
rc = orte_dss.pack(buffer,
&endpoint->rem_info.rem_qps[0].rem_qp_num, 1,
ORTE_UINT32);
if(rc != ORTE_SUCCESS) {
ORTE_ERROR_LOG(rc);
return rc;
}
opal_output(mca_btl_base_output, "packing %d of %d\n", 1, ORTE_UINT16);
rc = orte_dss.pack(buffer, &endpoint->rem_info.rem_lid, 1, ORTE_UINT16);
if(rc != ORTE_SUCCESS) {
ORTE_ERROR_LOG(rc);
return rc;
}
}
if(message_type != ENDPOINT_CONNECT_ACK) {
int qp;
/* stuff all the QP info into the buffer */
for(qp = 0; qp < mca_btl_openib_component.num_qps; qp++) {
opal_output(mca_btl_base_output, "packing %d of %d\n", 1, ORTE_UINT32);
rc = orte_dss.pack(buffer, &endpoint->qps[qp].lcl_qp->qp_num,
1, ORTE_UINT32);
if(rc != ORTE_SUCCESS) {
ORTE_ERROR_LOG(rc);
return rc;
}
opal_output(mca_btl_base_output, "packing %d of %d\n", 1, ORTE_UINT32);
rc = orte_dss.pack(buffer, &endpoint->qps[qp].lcl_psn, 1,
ORTE_UINT32);
if(rc != ORTE_SUCCESS) {
ORTE_ERROR_LOG(rc);
return rc;
}
}
opal_output(mca_btl_base_output, "packing %d of %d\n", 1, ORTE_UINT16);
rc = orte_dss.pack(buffer, &endpoint->endpoint_btl->lid, 1, ORTE_UINT16);
if(rc != ORTE_SUCCESS) {
ORTE_ERROR_LOG(rc);
return rc;
}
opal_output(mca_btl_base_output, "packing %d of %d\n", 1, ORTE_UINT32);
rc = orte_dss.pack(buffer, &endpoint->endpoint_btl->hca->mtu, 1,
ORTE_UINT32);
if(rc != ORTE_SUCCESS) {
ORTE_ERROR_LOG(rc);
return rc;
}
opal_output(mca_btl_base_output, "packing %d of %d\n", 1, ORTE_UINT32);
rc = orte_dss.pack(buffer, &endpoint->index, 1, ORTE_UINT32);
if(rc != ORTE_SUCCESS) {
ORTE_ERROR_LOG(rc);
return rc;
}
}
/* send to endpoint */
rc = orte_rml.send_buffer_nb(&endpoint->endpoint_proc->proc_guid,
buffer, ORTE_RML_TAG_DYNAMIC-1, 0,
mca_btl_openib_endpoint_send_cb, NULL);
BTL_VERBOSE(("Sent QP Info, LID = %d, SUBNET = %016x\n",
endpoint->endpoint_btl->lid,
endpoint->subnet_id));
if(rc < 0) {
ORTE_ERROR_LOG(rc);
return rc;
}
return OMPI_SUCCESS;
}
/*
* Set remote connection info
* (from OOB connection)
*
* call when the connect module has created all the qp's on an
* endpoint and needs to have some receive buffers posted
*/
static int mca_btl_openib_endpoint_set_remote_info(mca_btl_base_endpoint_t* endpoint,
mca_btl_openib_rem_info_t* rem_info)
int mca_btl_openib_endpoint_post_recvs(mca_btl_openib_endpoint_t *endpoint)
{
/* copy the rem_info stuff */
memcpy(&((mca_btl_openib_endpoint_t*) endpoint)->rem_info,
rem_info, sizeof(mca_btl_openib_rem_info_t));
/* copy over the rem qp info */
memcpy(endpoint->rem_info.rem_qps,
rem_info->rem_qps, sizeof(mca_btl_openib_rem_qp_info_t) *
mca_btl_openib_component.num_qps);
BTL_VERBOSE(("Setting QP info, LID = %d", endpoint->rem_info.rem_lid));
int qp;
return ORTE_SUCCESS;
}
static int mca_btl_openib_endpoint_qp_create_one(
mca_btl_base_endpoint_t* endpoint, int prio, int qp)
{
int rc;
mca_btl_openib_module_t *openib_btl =
(mca_btl_openib_module_t*)endpoint->endpoint_btl;
struct ibv_srq *srq =
(MCA_BTL_OPENIB_PP_QP == openib_btl->qps[qp].type) ? NULL :
openib_btl->qps[qp].u.srq_qp.srq;
/* Create the Queue Pair */
if(OMPI_SUCCESS != (rc = mca_btl_openib_endpoint_create_qp(openib_btl,
openib_btl->hca->ib_pd,
openib_btl->ib_cq[prio],
srq,
endpoint->qps[qp].lcl_qp_attr,
&endpoint->qps[qp].lcl_qp,
qp))) {
BTL_ERROR(("error creating queue pair, error code %d", rc));
return rc;
}
endpoint->qps[qp].lcl_psn = lrand48() & 0xffffff;
endpoint->qps[qp].credit_frag = NULL;
openib_btl->cq_users[prio]++;
if(MCA_BTL_OPENIB_SRQ_QP == endpoint->qps[qp].qp_type) {
mca_btl_openib_post_srr(openib_btl, 1, qp);
} else {
mca_btl_openib_endpoint_post_rr(endpoint, 1, qp);
}
return OMPI_SUCCESS;
}
static int mca_btl_openib_endpoint_qp_create_all(
mca_btl_base_endpoint_t* endpoint)
{
int qp, rc;
for(qp = 0; qp < mca_btl_openib_component.num_qps; qp++) {
if( mca_btl_openib_component.qp_infos[qp].size <=
mca_btl_openib_component.eager_limit) {
rc = mca_btl_openib_endpoint_qp_create_one(endpoint,
BTL_OPENIB_HP_CQ, qp);
if(rc != OMPI_SUCCESS)
return rc;
for (qp = 0; qp < mca_btl_openib_component.num_qps; ++qp) {
if (MCA_BTL_OPENIB_SRQ_QP == endpoint->qps[qp].qp_type) {
mca_btl_openib_post_srr(endpoint->endpoint_btl, 1, qp);
} else {
rc = mca_btl_openib_endpoint_qp_create_one(endpoint,
BTL_OPENIB_LP_CQ, qp);
if(rc != OMPI_SUCCESS)
return rc;
mca_btl_openib_endpoint_post_rr(endpoint, 1, qp);
}
}
return OMPI_SUCCESS;
}
/*
* Start to connect to the endpoint. We send our Queue Pair
* information over the TCP OOB communication mechanism.
* On completion of our send, a send completion handler
* is called
*
*/
static int mca_btl_openib_endpoint_start_connect(mca_btl_base_endpoint_t* endpoint)
{
int rc;
BTL_VERBOSE(("Initialized QP , LID = %d",
((mca_btl_openib_module_t*)endpoint->endpoint_btl)->lid));
rc = mca_btl_openib_endpoint_qp_create_all(endpoint);
if(rc != OMPI_SUCCESS)
return rc;
/* Send connection info over to remote endpoint */
endpoint->endpoint_state = MCA_BTL_IB_CONNECTING;
if(OMPI_SUCCESS != (rc = mca_btl_openib_endpoint_send_connect_data(endpoint,
ENDPOINT_CONNECT_REQ))) {
BTL_ERROR(("error sending connect request, error code %d", rc));
return rc;
}
return OMPI_SUCCESS;
}
/*
* Reply to a `start - connect' message
*
*/
static int mca_btl_openib_endpoint_reply_start_connect(
mca_btl_openib_endpoint_t *endpoint,
mca_btl_openib_rem_info_t *rem_info)
{
int rc;
BTL_VERBOSE(("Initialized QPs, LID = %d",
((mca_btl_openib_module_t*)endpoint->endpoint_btl)->lid));
rc = mca_btl_openib_endpoint_qp_create_all(endpoint);
if(rc != OMPI_SUCCESS)
return rc;
/* Set the remote side info */
mca_btl_openib_endpoint_set_remote_info(endpoint, rem_info);
/* Connect to endpoint */
rc = mca_btl_openib_endpoint_connect(endpoint);
if(rc != OMPI_SUCCESS) {
BTL_ERROR(("error in endpoint connect error code is %d", rc));
return rc;
}
/* Send connection info over to remote endpoint */
endpoint->endpoint_state = MCA_BTL_IB_CONNECT_ACK;
if(OMPI_SUCCESS != (rc = mca_btl_openib_endpoint_send_connect_data(endpoint,
ENDPOINT_CONNECT_RSP))) {
BTL_ERROR(("error in endpoint send connect request error code is %d",
rc));
return rc;
}
return OMPI_SUCCESS;
}
/*
* called when the openib has completed setup via the
* OOB channel
* called when the connect module has completed setup of an endpoint
*/
static void mca_btl_openib_endpoint_connected(mca_btl_openib_endpoint_t *endpoint)
void mca_btl_openib_endpoint_connected(mca_btl_openib_endpoint_t *endpoint)
{
opal_list_item_t *frag_item;
mca_btl_openib_frag_t *frag;
@ -713,15 +434,12 @@ static void mca_btl_openib_endpoint_connected(mca_btl_openib_endpoint_t *endpoin
endpoint->endpoint_state = MCA_BTL_IB_CONNECTED;
endpoint->endpoint_btl->poll_cq = true;
/**
* The connection is correctly setup. Now we can decrease the event trigger.
*/
/* The connection is correctly setup. Now we can decrease the
event trigger. */
opal_progress_event_users_decrement();
/* While there are frags in the list,
* process them */
while(!opal_list_is_empty(&(endpoint->pending_lazy_frags))) {
/* While there are frags in the list, process them */
while (!opal_list_is_empty(&(endpoint->pending_lazy_frags))) {
frag_item = opal_list_remove_first(&(endpoint->pending_lazy_frags));
frag = (mca_btl_openib_frag_t *) frag_item;
openib_btl = endpoint->endpoint_btl;
@ -732,269 +450,13 @@ static void mca_btl_openib_endpoint_connected(mca_btl_openib_endpoint_t *endpoin
}
}
/*
* Non blocking OOB recv callback.
* Read incoming QP and other info, and if this endpoint
* is trying to connect, reply with our QP info,
* otherwise try to modify QP's and establish
* reliable connection
*
*/
static void mca_btl_openib_endpoint_recv(
int status,
orte_process_name_t* process_name,
orte_buffer_t* buffer,
orte_rml_tag_t tag,
void* cbdata)
{
mca_btl_openib_proc_t *ib_proc;
mca_btl_openib_endpoint_t *ib_endpoint = NULL;
int endpoint_state;
int rc;
uint32_t i, lcl_qp = 0;
uint16_t lcl_lid = 0;
int32_t cnt = 1;
mca_btl_openib_rem_info_t rem_info;
uint8_t message_type;
bool master;
/* start by unpacking data first so we know who is knocking at
our door */
opal_output(mca_btl_base_output, "unpacking %d of %d\n", cnt, ORTE_UINT8);
rc = orte_dss.unpack(buffer, &message_type, &cnt, ORTE_UINT8);
if(ORTE_SUCCESS != rc) {
ORTE_ERROR_LOG(rc);
return;
}
opal_output(mca_btl_base_output, "unpacking %d of %d\n", cnt, ORTE_UINT64);
rc = orte_dss.unpack(buffer, &rem_info.rem_subnet_id, &cnt, ORTE_UINT64);
if(ORTE_SUCCESS != rc) {
ORTE_ERROR_LOG(rc);
return;
}
if(message_type != ENDPOINT_CONNECT_REQ) {
opal_output(mca_btl_base_output, "unpacking %d of %d\n", cnt, ORTE_UINT32);
rc = orte_dss.unpack(buffer, &lcl_qp, &cnt, ORTE_UINT32);
if(ORTE_SUCCESS != rc) {
ORTE_ERROR_LOG(rc);
return;
}
opal_output(mca_btl_base_output, "unpacking %d of %d\n", cnt, ORTE_UINT16);
rc = orte_dss.unpack(buffer, &lcl_lid, &cnt, ORTE_UINT16);
if(ORTE_SUCCESS != rc) {
ORTE_ERROR_LOG(rc);
return;
}
}
if(message_type != ENDPOINT_CONNECT_ACK) {
int qp;
/* get ready for the data */
rem_info.rem_qps =
(mca_btl_openib_rem_qp_info_t*) malloc(sizeof(mca_btl_openib_rem_qp_info_t) *
mca_btl_openib_component.num_qps);
/* unpack all the qp info */
for(qp = 0; qp < mca_btl_openib_component.num_qps; qp++) {
opal_output(mca_btl_base_output, "unpacking %d of %d\n", cnt, ORTE_UINT32);
rc = orte_dss.unpack(buffer, &rem_info.rem_qps[qp].rem_qp_num, &cnt,
ORTE_UINT32);
if(ORTE_SUCCESS != rc) {
ORTE_ERROR_LOG(rc);
return;
}
opal_output(mca_btl_base_output, "unpacking %d of %d\n", cnt, ORTE_UINT32);
rc = orte_dss.unpack(buffer, &rem_info.rem_qps[qp].rem_psn, &cnt,
ORTE_UINT32);
if(ORTE_SUCCESS != rc) {
ORTE_ERROR_LOG(rc);
return;
}
}
opal_output(mca_btl_base_output, "unpacking %d of %d\n", cnt, ORTE_UINT16);
rc = orte_dss.unpack(buffer, &rem_info.rem_lid, &cnt, ORTE_UINT16);
if(ORTE_SUCCESS != rc) {
ORTE_ERROR_LOG(rc);
return;
}
opal_output(mca_btl_base_output, "unpacking %d of %d\n", cnt, ORTE_UINT32);
rc = orte_dss.unpack(buffer, &rem_info.rem_mtu, &cnt, ORTE_UINT32);
if(ORTE_SUCCESS != rc) {
ORTE_ERROR_LOG(rc);
return;
}
opal_output(mca_btl_base_output, "unpacking %d of %d\n", cnt, ORTE_UINT32);
rc = orte_dss.unpack(buffer, &rem_info.rem_index, &cnt, ORTE_UINT32);
if(ORTE_SUCCESS != rc) {
ORTE_ERROR_LOG(rc);
return;
}
}
BTL_VERBOSE(("Received QP Info, LID = %d, SUBNET = %016x\n",
rem_info.rem_lid,
rem_info.rem_subnet_id));
master = orte_ns.compare_fields(ORTE_NS_CMP_ALL, orte_process_info.my_name,
process_name) > 0 ? true : false;
for(ib_proc = (mca_btl_openib_proc_t*)
opal_list_get_first(&mca_btl_openib_component.ib_procs);
ib_proc != (mca_btl_openib_proc_t*)
opal_list_get_end(&mca_btl_openib_component.ib_procs);
ib_proc = (mca_btl_openib_proc_t*)opal_list_get_next(ib_proc)) {
bool found = false;
if(orte_ns.compare_fields(ORTE_NS_CMP_ALL,
&ib_proc->proc_guid, process_name) != ORTE_EQUAL)
continue;
if(message_type != ENDPOINT_CONNECT_REQ) {
/* This is a reply message. Try to get the endpoint instance the
* reply belongs to */
for(i = 0; i < ib_proc->proc_endpoint_count; i++) {
ib_endpoint = ib_proc->proc_endpoints[i];
if(ib_endpoint->qps[0].lcl_qp != NULL &&
lcl_lid == ib_endpoint->endpoint_btl->lid &&
lcl_qp == ib_endpoint->qps[0].lcl_qp->qp_num &&
rem_info.rem_subnet_id == ib_endpoint->subnet_id) {
found = true;
break;
}
}
} else {
/* This is new connection request. If this is master try to find
* endpoint in a connecting state. If this is slave try to find
* endpoint in closed state and initiate connection back */
mca_btl_openib_endpoint_t *ib_endpoint_found = NULL;
for(i = 0; i < ib_proc->proc_endpoint_count; i++) {
ib_endpoint = ib_proc->proc_endpoints[i];
if(ib_endpoint->subnet_id != rem_info.rem_subnet_id ||
(ib_endpoint->endpoint_state != MCA_BTL_IB_CONNECTING
&& ib_endpoint->endpoint_state != MCA_BTL_IB_CLOSED))
continue;
found = true;
ib_endpoint_found = ib_endpoint;
if((master &&
ib_endpoint->endpoint_state == MCA_BTL_IB_CONNECTING) ||
(!master &&
ib_endpoint->endpoint_state == MCA_BTL_IB_CLOSED))
break; /* Found one. No point to continue */
}
ib_endpoint = ib_endpoint_found;
/* if this is slave and there is no endpoints in closed state
* then all connection are already in progress so just ignore
* this connection request */
if(found && !master &&
ib_endpoint->endpoint_state != MCA_BTL_IB_CLOSED) {
return;
}
}
if(!found) {
BTL_ERROR(("can't find suitable endpoint for this peer\n"));
return;
}
endpoint_state = ib_endpoint->endpoint_state;
/* Update status */
switch(endpoint_state) {
case MCA_BTL_IB_CLOSED :
/* We had this connection closed before.
* The endpoint is trying to connect. Move the
* status of this connection to CONNECTING,
* and then reply with our QP information */
if(master) {
rc = mca_btl_openib_endpoint_reply_start_connect(ib_endpoint, &rem_info);
} else {
rc = mca_btl_openib_endpoint_start_connect(ib_endpoint);
}
if(OMPI_SUCCESS != rc) {
BTL_ERROR(("error in endpoint reply start connect"));
break;
}
/** As long as we expect a message from the peer (in order to setup the connection)
* let the event engine pool the OOB events. Note: we increment it once peer active
* connection.
*/
opal_progress_event_users_increment();
break;
case MCA_BTL_IB_CONNECTING :
mca_btl_openib_endpoint_set_remote_info(ib_endpoint, &rem_info);
if(OMPI_SUCCESS != (rc = mca_btl_openib_endpoint_connect(ib_endpoint))) {
BTL_ERROR(("endpoint connect error: %d", rc));
break;
}
if(master) {
ib_endpoint->endpoint_state = MCA_BTL_IB_WAITING_ACK;
/* Send him an ack */
mca_btl_openib_endpoint_send_connect_data(ib_endpoint,
ENDPOINT_CONNECT_RSP);
} else {
mca_btl_openib_endpoint_send_connect_data(ib_endpoint,
ENDPOINT_CONNECT_ACK);
mca_btl_openib_endpoint_connected(ib_endpoint);
}
break;
case MCA_BTL_IB_WAITING_ACK:
mca_btl_openib_endpoint_connected(ib_endpoint);
break;
case MCA_BTL_IB_CONNECT_ACK:
mca_btl_openib_endpoint_send_connect_data(ib_endpoint,
ENDPOINT_CONNECT_ACK);
mca_btl_openib_endpoint_connected(ib_endpoint);
break;
case MCA_BTL_IB_CONNECTED :
break;
default :
BTL_ERROR(("Invalid endpoint state %d", endpoint_state));
}
OPAL_THREAD_UNLOCK(&ib_endpoint->endpoint_lock);
break;
}
}
/*
* Post the OOB recv (for receiving the peers information)
*/
void mca_btl_openib_post_recv()
{
orte_rml.recv_buffer_nb(
ORTE_NAME_WILDCARD,
ORTE_RML_TAG_DYNAMIC-1,
ORTE_RML_PERSISTENT,
mca_btl_openib_endpoint_recv,
NULL);
}
/*
* Attempt to send a fragment using a given endpoint. If the endpoint is not
* connected, queue the fragment and start the connection as required.
*/
int mca_btl_openib_endpoint_send(
mca_btl_base_endpoint_t* endpoint,
mca_btl_openib_frag_t* frag
)
int mca_btl_openib_endpoint_send(mca_btl_base_endpoint_t* endpoint,
mca_btl_openib_frag_t* frag)
{
int rc;
bool call_progress = false;
@ -1027,10 +489,11 @@ int mca_btl_openib_endpoint_send(
BTL_VERBOSE(("Connection to endpoint closed ... connecting ..."));
opal_list_append(&endpoint->pending_lazy_frags,
(opal_list_item_t *)frag);
rc = mca_btl_openib_endpoint_start_connect(endpoint);
/**
* As long as we expect a message from the peer (in order to setup the connection)
* let the event engine pool the OOB events. Note: we increment it once peer active
rc = ompi_btl_openib_connect.bcf_start_connect(endpoint);
/*
* As long as we expect a message from the peer (in order
* to setup the connection) let the event engine pool the
* OOB events. Note: we increment it once peer active
* connection.
*/
opal_progress_event_users_increment();
@ -1061,200 +524,6 @@ int mca_btl_openib_endpoint_send(
return rc;
}
/*
* Complete connection to endpoint.
*/
static int mca_btl_openib_endpoint_connect_qp(
mca_btl_openib_endpoint_t *endpoint, int prio, int qp)
{
int rc;
mca_btl_openib_module_t* openib_btl =
(mca_btl_openib_module_t*)endpoint->endpoint_btl;
/* Connection establishment RC */
rc = mca_btl_openib_endpoint_qp_init_query(openib_btl,
endpoint->qps[qp].lcl_qp,
endpoint->qps[qp].lcl_qp_attr,
endpoint->qps[qp].lcl_psn,
endpoint->rem_info.rem_qps[qp].rem_qp_num,
endpoint->rem_info.rem_qps[qp].rem_psn,
endpoint->rem_info.rem_lid,
endpoint->rem_info.rem_mtu,
openib_btl->port_num);
return rc;
}
int mca_btl_openib_endpoint_connect(
mca_btl_openib_endpoint_t *endpoint)
{
int rc;
int qp;
for(qp = 0; qp < mca_btl_openib_component.num_qps; qp++) {
if( mca_btl_openib_component.qp_infos[qp].size <= mca_btl_openib_component.eager_limit) {
rc = mca_btl_openib_endpoint_connect_qp(endpoint,
BTL_OPENIB_HP_CQ,
qp);
} else {
rc = mca_btl_openib_endpoint_connect_qp(endpoint,
BTL_OPENIB_LP_CQ,
qp);
}
if(rc != OMPI_SUCCESS) {
return rc;
}
}
return OMPI_SUCCESS;
}
/*
* Create the queue pair note that this is just the initial
* queue pair creation and we need to get the remote queue pair
* info from the peer before the qp is usable,
*/
int mca_btl_openib_endpoint_create_qp(
mca_btl_openib_module_t* openib_btl,
struct ibv_pd* pd,
struct ibv_cq* cq,
struct ibv_srq* srq,
struct ibv_qp_attr* qp_attr,
struct ibv_qp** qp,
int qp_idx
)
{
{
struct ibv_qp* my_qp;
struct ibv_qp_init_attr qp_init_attr;
memset(&qp_init_attr, 0, sizeof(struct ibv_qp_init_attr));
qp_init_attr.send_cq = cq;
qp_init_attr.recv_cq = cq;
if(MCA_BTL_OPENIB_PP_QP == mca_btl_openib_component.qp_infos[qp_idx].type) {
qp_init_attr.cap.max_recv_wr =
mca_btl_openib_component.qp_infos[qp_idx].rd_num +
mca_btl_openib_component.qp_infos[qp_idx].u.pp_qp.rd_rsv;
qp_init_attr.cap.max_send_wr =
mca_btl_openib_component.qp_infos[qp_idx].rd_num + 1;
} else {
qp_init_attr.cap.max_recv_wr =
mca_btl_openib_component.qp_infos[qp_idx].rd_num;
qp_init_attr.cap.max_send_wr =
mca_btl_openib_component.qp_infos[qp_idx].u.srq_qp.sd_max +
BTL_OPENIB_EAGER_RDMA_QP(qp_idx);
}
qp_init_attr.cap.max_send_sge = mca_btl_openib_component.ib_sg_list_size;
qp_init_attr.cap.max_recv_sge = mca_btl_openib_component.ib_sg_list_size;
qp_init_attr.qp_type = IBV_QPT_RC;
qp_init_attr.srq = srq;
my_qp = ibv_create_qp(pd, &qp_init_attr);
if(NULL == my_qp) {
BTL_ERROR(("error creating qp errno says %s", strerror(errno)));
return OMPI_ERROR;
}
(*qp) = my_qp;
openib_btl->ib_inline_max = qp_init_attr.cap.max_inline_data;
}
{
qp_attr->qp_state = IBV_QPS_INIT;
qp_attr->pkey_index = openib_btl->pkey_index; /*mca_btl_openib_component.ib_pkey_ix; */
qp_attr->port_num = openib_btl->port_num;
qp_attr->qp_access_flags = IBV_ACCESS_REMOTE_WRITE | IBV_ACCESS_REMOTE_READ;
if(ibv_modify_qp((*qp), qp_attr,
IBV_QP_STATE |
IBV_QP_PKEY_INDEX |
IBV_QP_PORT |
IBV_QP_ACCESS_FLAGS )) {
BTL_ERROR(("error modifying qp to INIT errno says %s", strerror(errno)));
return OMPI_ERROR;
}
}
return OMPI_SUCCESS;
}
/*
* The queue pair has been created and we have received the remote
* queue pair information from the peer so we init this queue pair
* and are ready to roll.
*/
int mca_btl_openib_endpoint_qp_init_query(
mca_btl_openib_module_t* openib_btl,
struct ibv_qp* qp,
struct ibv_qp_attr* attr,
uint32_t lcl_psn,
uint32_t rem_qp_num,
uint32_t rem_psn,
uint16_t rem_lid,
uint32_t rem_mtu,
uint32_t port_num
)
{
attr->qp_state = IBV_QPS_RTR;
attr->path_mtu = (openib_btl->hca->mtu < rem_mtu) ?
openib_btl->hca->mtu : rem_mtu;
if (mca_btl_openib_component.verbose) {
BTL_OUTPUT(("Set MTU to IBV value %d (%s bytes)", attr->path_mtu,
(attr->path_mtu == IBV_MTU_256) ? "256" :
(attr->path_mtu == IBV_MTU_512) ? "512" :
(attr->path_mtu == IBV_MTU_1024) ? "1024" :
(attr->path_mtu == IBV_MTU_2048) ? "2048" :
(attr->path_mtu == IBV_MTU_4096) ? "4096" :
"unknown (!)"));
}
attr->dest_qp_num = rem_qp_num;
attr->rq_psn = rem_psn;
attr->max_dest_rd_atomic = mca_btl_openib_component.ib_max_rdma_dst_ops;
attr->min_rnr_timer = mca_btl_openib_component.ib_min_rnr_timer;
attr->ah_attr.is_global = 0;
attr->ah_attr.dlid = rem_lid;
attr->ah_attr.sl = mca_btl_openib_component.ib_service_level;
attr->ah_attr.src_path_bits = openib_btl->src_path_bits;
attr->ah_attr.port_num = port_num;
/* JMS to be filled in later dynamically */
attr->ah_attr.static_rate = 0;
if(ibv_modify_qp(qp, attr,
IBV_QP_STATE |
IBV_QP_AV |
IBV_QP_PATH_MTU |
IBV_QP_DEST_QPN |
IBV_QP_RQ_PSN |
IBV_QP_MAX_DEST_RD_ATOMIC |
IBV_QP_MIN_RNR_TIMER)) {
BTL_ERROR(("error modifing QP to RTR errno says %s", strerror(errno)));
return OMPI_ERROR;
}
attr->qp_state = IBV_QPS_RTS;
attr->timeout = mca_btl_openib_component.ib_timeout;
attr->retry_cnt = mca_btl_openib_component.ib_retry_count;
attr->rnr_retry = mca_btl_openib_component.ib_rnr_retry;
attr->sq_psn = lcl_psn;
attr->max_rd_atomic = mca_btl_openib_component.ib_max_rdma_dst_ops;
if (ibv_modify_qp(qp, attr,
IBV_QP_STATE |
IBV_QP_TIMEOUT |
IBV_QP_RETRY_CNT |
IBV_QP_RNR_RETRY |
IBV_QP_SQ_PSN |
IBV_QP_MAX_QP_RD_ATOMIC)) {
BTL_ERROR(("error modifying QP to RTS errno says %s", strerror(errno)));
return OMPI_ERROR;
}
return OMPI_SUCCESS;
}
/**
* Return control fragment.
*/

13
ompi/mca/btl/openib/btl_openib_endpoint.h
Просмотреть файл

@ -33,9 +33,7 @@
#include <string.h>
#include "ompi/mca/btl/base/btl_base_error.h"
#if defined(c_plusplus) || defined(__cplusplus)
extern "C" {
#endif
BEGIN_C_DECLS
struct mca_btl_openib_frag_t;
@ -204,10 +202,10 @@ OBJ_CLASS_DECLARATION(mca_btl_openib_endpoint_t);
int mca_btl_openib_endpoint_send(mca_btl_base_endpoint_t* endpoint,
struct mca_btl_openib_frag_t* frag);
int mca_btl_openib_endpoint_connect(mca_btl_base_endpoint_t*);
void mca_btl_openib_post_recv(void);
void mca_btl_openib_endpoint_send_credits(mca_btl_base_endpoint_t*, const int);
void mca_btl_openib_endpoint_connect_eager_rdma(mca_btl_openib_endpoint_t*);
int mca_btl_openib_endpoint_post_recvs(mca_btl_openib_endpoint_t *endpoint);
void mca_btl_openib_endpoint_connected(mca_btl_openib_endpoint_t *endpoint);
@ -300,7 +298,6 @@ static inline int btl_openib_check_send_credits(
return 0;
}
#if defined(c_plusplus) || defined(__cplusplus)
}
#endif
END_C_DECLS
#endif

5
ompi/mca/btl/openib/btl_openib_mca.c
Просмотреть файл

@ -31,6 +31,7 @@
#include "opal/mca/base/mca_base_param.h"
#include "btl_openib.h"
#include "btl_openib_mca.h"
#include "connect/base.h"
/*
* Local flags
@ -601,6 +602,10 @@ static int mca_btl_openib_mca_setup_qps(void) {
mca_btl_openib_component.rdma_qp = mca_btl_openib_component.num_qps - 1;
mca_btl_openib_component.eager_rdma_qp = 0;
/* Register any MCA params for the connect pseudo-components */
ompi_btl_openib_connect_base_open();
ret = MPI_SUCCESS;
error:
if(params) {

35
ompi/mca/btl/openib/connect/base.h Обычный файл
Просмотреть файл

@ -0,0 +1,35 @@
/*
* Copyright (c) 2007 Cisco, Inc. All rights reserved.
*
* $COPYRIGHT$
*
* Additional copyrights may follow
*
* $HEADER$
*/
#ifndef BTL_OPENIB_CONNECT_BASE_H
#define BTL_OPENIB_CONNECT_BASE_H
#include "connect/connect.h"
BEGIN_C_DECLS
/**
* Global variable with the selected function pointers in it
*/
extern ompi_btl_openib_connect_base_funcs_t ompi_btl_openib_connect;
/*
* Open function
*/
int ompi_btl_openib_connect_base_open(void);
/*
* Select function
*/
int ompi_btl_openib_connect_base_select(void);
END_C_DECLS
#endif

105
ompi/mca/btl/openib/connect/btl_openib_connect_base.c Обычный файл
Просмотреть файл

@ -0,0 +1,105 @@
/*
* Copyright (c) 2007 Cisco, Inc. All rights reserved.
*
* $COPYRIGHT$
*
* Additional copyrights may follow
*
* $HEADER$
*/
#include "ompi_config.h"
#include "btl_openib.h"
#include "connect/base.h"
#include "connect/btl_openib_connect_oob.h"
#include "connect/btl_openib_connect_rdma_cm.h"
#include "opal/util/argv.h"
/*
* Global variable with the selected function pointers in it
*/
ompi_btl_openib_connect_base_funcs_t ompi_btl_openib_connect = {
"",
/* Compiler fills in the rest with NULL */
};
/*
* Array of all possible connection functions
*/
static ompi_btl_openib_connect_base_funcs_t *all[] = {
&ompi_btl_openib_connect_oob,
&ompi_btl_openib_connect_rdma_cm,
NULL
};
/*
* MCA parameter value
*/
static char *param = NULL;
/*
* Register MCA parameters
*/
int ompi_btl_openib_connect_base_open(void)
{
int i;
char **temp, *a, *b;
/* Make an MCA parameter to select which connect module to use */
temp = NULL;
for (i = 0; NULL != all[i]; ++i) {
opal_argv_append_nosize(&temp, all[i]->bcf_name);
}
a = opal_argv_join(temp, ',');
opal_argv_free(temp);
asprintf(&b,
"Method used to make OpenFabrics connections (valid values: %s)",
a);
mca_base_param_reg_string(&mca_btl_openib_component.super.btl_version,
"btl_openib_connect",
b, false, false,
"oob", &param);
/* Call the open function on all the connect modules */
for (i = 0; NULL != all[i]; ++i) {
if (NULL != all[i]->bcf_open) {
all[i]->bcf_open();
}
}
return OMPI_SUCCESS;
}
int ompi_btl_openib_connect_base_select(void)
{
int i;
/* Go through all the pseudo-components; if the btl_openib_connect
param is empty, then take the first one that returns
OMPI_SUCCESS from its init function. If
btl_openib_connect_param is not empty, find that one and ensure
that its init function returns OMPI_SUCCESS. */
if (NULL != param && '\0' == param[0]) {
param = NULL;
}
for (i = 0; NULL != all[i]; ++i) {
if ((NULL != param && 0 == strcmp(all[i]->bcf_name, param)) ||
(NULL == param)) {
if (NULL != all[i]->bcf_init &&
OMPI_SUCCESS == all[i]->bcf_init()) {
ompi_btl_openib_connect = *(all[i]);
break;
}
}
}
if (NULL == all[i]) {
/* JMS opal_show_help */
return OMPI_ERR_NOT_FOUND;
}
return OMPI_SUCCESS;
}

723
ompi/mca/btl/openib/connect/btl_openib_connect_oob.c Обычный файл
Просмотреть файл

@ -0,0 +1,723 @@
/*
* Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
* University Research and Technology
* Corporation. All rights reserved.
* Copyright (c) 2004-2005 The University of Tennessee and The University
* of Tennessee Research Foundation. All rights
* reserved.
* Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
* University of Stuttgart. All rights reserved.
* Copyright (c) 2004-2005 The Regents of the University of California.
* All rights reserved.
* Copyright (c) 2006-2007 Cisco, Inc. All rights reserved.
* Copyright (c) 2006 Los Alamos National Security, LLC. All rights
* reserved.
*
* $COPYRIGHT$
*
* Additional copyrights may follow
*
* $HEADER$
*/
#include "ompi_config.h"
#include "orte/mca/ns/base/base.h"
#include "orte/mca/oob/base/base.h"
#include "orte/mca/rml/rml.h"
#include "orte/mca/errmgr/errmgr.h"
#include "orte/dss/dss.h"
#include "btl_openib.h"
#include "btl_openib_endpoint.h"
#include "btl_openib_proc.h"
#include "connect/connect.h"
typedef enum {
ENDPOINT_CONNECT_REQUEST,
ENDPOINT_CONNECT_RESPONSE,
ENDPOINT_CONNECT_ACK
} connect_message_type_t;
#define OOB_TAG (ORTE_RML_TAG_DYNAMIC - 1)
static int oob_init(void);
static int oob_start_connect(mca_btl_base_endpoint_t *e);
static int oob_finalize(void);
static int reply_start_connect(mca_btl_openib_endpoint_t *endpoint,
mca_btl_openib_rem_info_t *rem_info);
static int set_remote_info(mca_btl_base_endpoint_t* endpoint,
mca_btl_openib_rem_info_t* rem_info);
static int qp_connect_all(mca_btl_base_endpoint_t* endpoint);
static int qp_create_all(mca_btl_base_endpoint_t* endpoint);
static int qp_create_one(mca_btl_base_endpoint_t* endpoint, int prio, int qp);
static int send_connect_data(mca_btl_base_endpoint_t* endpoint,
uint8_t message_type);
static void rml_send_cb(int status, orte_process_name_t* endpoint,
orte_buffer_t* buffer, orte_rml_tag_t tag,
void* cbdata);
static void rml_recv_cb(int status, orte_process_name_t* process_name,
orte_buffer_t* buffer, orte_rml_tag_t tag,
void* cbdata);
/*
* The "module" struct -- the top-level function pointers for the oob
* connection scheme.
*/
ompi_btl_openib_connect_base_funcs_t ompi_btl_openib_connect_oob = {
"oob",
/* No need for "open */
NULL,
/* Init */
oob_init,
/* Connect */
oob_start_connect,
/* Finalize */
oob_finalize,
};
/*
* Init function. Post non-blocking RML receive to accept incoming
* connection requests.
*/
static int oob_init(void)
{
int rc;
rc = orte_rml.recv_buffer_nb(ORTE_NAME_WILDCARD,
OOB_TAG,
ORTE_RML_PERSISTENT,
rml_recv_cb,
NULL);
return (ORTE_SUCCESS == rc) ? OMPI_SUCCESS : rc;
}
/*
* Connect function. Start initiation of connections to a remote
* peer. We send our Queue Pair information over the RML/OOB
* communication mechanism. On completion of our send, a send
* completion handler is called.
*/
static int oob_start_connect(mca_btl_base_endpoint_t *endpoint)
{
int rc;
if (OMPI_SUCCESS != (rc = qp_create_all(endpoint))) {
return rc;
}
/* Send connection info over to remote endpoint */
endpoint->endpoint_state = MCA_BTL_IB_CONNECTING;
if (OMPI_SUCCESS !=
(rc = send_connect_data(endpoint, ENDPOINT_CONNECT_REQUEST))) {
BTL_ERROR(("error sending connect request, error code %d", rc));
return rc;
}
return OMPI_SUCCESS;
}
/*
* Finalize function. Cleanup RML non-blocking receive.
*/
static int oob_finalize(void)
{
orte_rml.recv_cancel(ORTE_NAME_WILDCARD, OOB_TAG);
return OMPI_SUCCESS;
}
/**************************************************************************/
/*
* Reply to a `start - connect' message
*/
static int reply_start_connect(mca_btl_openib_endpoint_t *endpoint,
mca_btl_openib_rem_info_t *rem_info)
{
int rc;
BTL_VERBOSE(("Initialized QPs, LID = %d",
((mca_btl_openib_module_t*)endpoint->endpoint_btl)->lid));
/* Create local QP's and post receive resources */
if (OMPI_SUCCESS != (rc = qp_create_all(endpoint))) {
return rc;
}
/* Set the remote side info */
set_remote_info(endpoint, rem_info);
/* Connect to remote endpoint qp's */
if (OMPI_SUCCESS != (rc = qp_connect_all(endpoint))) {
return rc;
}
/* Send connection info over to remote endpoint */
endpoint->endpoint_state = MCA_BTL_IB_CONNECT_ACK;
if (OMPI_SUCCESS !=
(rc = send_connect_data(endpoint, ENDPOINT_CONNECT_RESPONSE))) {
BTL_ERROR(("error in endpoint send connect request error code is %d",
rc));
return rc;
}
return OMPI_SUCCESS;
}
static int set_remote_info(mca_btl_base_endpoint_t* endpoint,
mca_btl_openib_rem_info_t* rem_info)
{
/* copy the rem_info stuff */
memcpy(&((mca_btl_openib_endpoint_t*) endpoint)->rem_info,
rem_info, sizeof(mca_btl_openib_rem_info_t));
/* copy over the rem qp info */
memcpy(endpoint->rem_info.rem_qps,
rem_info->rem_qps, sizeof(mca_btl_openib_rem_qp_info_t) *
mca_btl_openib_component.num_qps);
BTL_VERBOSE(("Setting QP info, LID = %d", endpoint->rem_info.rem_lid));
return ORTE_SUCCESS;
}
/*
* Connect the local ends of all qp's to the remote side
*/
static int qp_connect_all(mca_btl_openib_endpoint_t *endpoint)
{
int i;
struct ibv_qp* qp;
struct ibv_qp_attr* attr;
mca_btl_openib_module_t* openib_btl =
(mca_btl_openib_module_t*)endpoint->endpoint_btl;
for (i = 0; i < mca_btl_openib_component.num_qps; i++) {
qp = endpoint->qps[i].lcl_qp;
attr = endpoint->qps[i].lcl_qp_attr;
attr->qp_state = IBV_QPS_RTR;
attr->path_mtu = (openib_btl->hca->mtu < endpoint->rem_info.rem_mtu) ?
openib_btl->hca->mtu : endpoint->rem_info.rem_mtu;
if (mca_btl_openib_component.verbose) {
BTL_OUTPUT(("Set MTU to IBV value %d (%s bytes)", attr->path_mtu,
(attr->path_mtu == IBV_MTU_256) ? "256" :
(attr->path_mtu == IBV_MTU_512) ? "512" :
(attr->path_mtu == IBV_MTU_1024) ? "1024" :
(attr->path_mtu == IBV_MTU_2048) ? "2048" :
(attr->path_mtu == IBV_MTU_4096) ? "4096" :
"unknown (!)"));
}
attr->dest_qp_num = endpoint->rem_info.rem_qps[i].rem_qp_num,
attr->rq_psn = endpoint->rem_info.rem_qps[i].rem_psn,
attr->max_dest_rd_atomic = mca_btl_openib_component.ib_max_rdma_dst_ops;
attr->min_rnr_timer = mca_btl_openib_component.ib_min_rnr_timer;
attr->ah_attr.is_global = 0;
attr->ah_attr.dlid = endpoint->rem_info.rem_lid,
attr->ah_attr.sl = mca_btl_openib_component.ib_service_level;
attr->ah_attr.src_path_bits = openib_btl->src_path_bits;
attr->ah_attr.port_num = openib_btl->port_num;
/* JMS to be filled in later dynamically */
attr->ah_attr.static_rate = 0;
if (ibv_modify_qp(qp, attr,
IBV_QP_STATE |
IBV_QP_AV |
IBV_QP_PATH_MTU |
IBV_QP_DEST_QPN |
IBV_QP_RQ_PSN |
IBV_QP_MAX_DEST_RD_ATOMIC |
IBV_QP_MIN_RNR_TIMER)) {
BTL_ERROR(("error modifing QP to RTR errno says %s",
strerror(errno)));
return OMPI_ERROR;
}
attr->qp_state = IBV_QPS_RTS;
attr->timeout = mca_btl_openib_component.ib_timeout;
attr->retry_cnt = mca_btl_openib_component.ib_retry_count;
attr->rnr_retry = mca_btl_openib_component.ib_rnr_retry;
attr->sq_psn = endpoint->qps[i].lcl_psn;
attr->max_rd_atomic = mca_btl_openib_component.ib_max_rdma_dst_ops;
if (ibv_modify_qp(qp, attr,
IBV_QP_STATE |
IBV_QP_TIMEOUT |
IBV_QP_RETRY_CNT |
IBV_QP_RNR_RETRY |
IBV_QP_SQ_PSN |
IBV_QP_MAX_QP_RD_ATOMIC)) {
BTL_ERROR(("error modifying QP to RTS errno says %s",
strerror(errno)));
return OMPI_ERROR;
}
}
return OMPI_SUCCESS;
}
/*
* Create the local side of all the qp's. The remote sides will be
* connected later.
*/
static int qp_create_all(mca_btl_base_endpoint_t* endpoint)
{
int qp, rc, prio;
for (qp = 0; qp < mca_btl_openib_component.num_qps; ++qp) {
/* If the size for this qp is <= the eager limit, make it a
high priority QP. Otherwise, make it a low priority QP. */
prio = (mca_btl_openib_component.qp_infos[qp].size <=
mca_btl_openib_component.eager_limit) ?
BTL_OPENIB_HP_CQ : BTL_OPENIB_LP_CQ;
rc = qp_create_one(endpoint, prio, qp);
if (OMPI_SUCCESS != rc) {
return rc;
}
}
/* Now that all the qp's are created locally, post some receive
buffers, setup credits, etc. */
return mca_btl_openib_endpoint_post_recvs(endpoint);
}
/*
* Create the local side of one qp. The remote side will be connected
* later.
*/
static int qp_create_one(mca_btl_base_endpoint_t* endpoint, int prio, int qp)
{
mca_btl_openib_module_t *openib_btl =
(mca_btl_openib_module_t*)endpoint->endpoint_btl;
struct ibv_srq *srq =
(MCA_BTL_OPENIB_PP_QP == openib_btl->qps[qp].type) ? NULL :
openib_btl->qps[qp].u.srq_qp.srq;
/* Create the Queue Pair */
#if 0
if (OMPI_SUCCESS != (rc = create_qp(openib_btl,
openib_btl->hca->ib_pd,
openib_btl->ib_cq[prio],
srq,
endpoint->qps[qp].lcl_qp_attr,
&endpoint->qps[qp].lcl_qp,
qp))) {
static int create_qp(
struct ibv_qp** qp,
int qp_idx);
BTL_ERROR(("error creating queue pair, error code %d", rc));
return rc;
}
#endif
{
struct ibv_qp* my_qp;
struct ibv_qp_init_attr qp_init_attr;
memset(&qp_init_attr, 0, sizeof(struct ibv_qp_init_attr));
qp_init_attr.send_cq =
qp_init_attr.recv_cq = openib_btl->ib_cq[prio];
if (MCA_BTL_OPENIB_PP_QP == mca_btl_openib_component.qp_infos[qp].type) {
qp_init_attr.cap.max_recv_wr =
mca_btl_openib_component.qp_infos[qp].rd_num +
mca_btl_openib_component.qp_infos[qp].u.pp_qp.rd_rsv;
qp_init_attr.cap.max_send_wr =
mca_btl_openib_component.qp_infos[qp].rd_num + 1;
} else {
qp_init_attr.cap.max_recv_wr =
mca_btl_openib_component.qp_infos[qp].rd_num;
qp_init_attr.cap.max_send_wr =
mca_btl_openib_component.qp_infos[qp].u.srq_qp.sd_max +
BTL_OPENIB_EAGER_RDMA_QP(qp);
}
qp_init_attr.cap.max_send_sge = mca_btl_openib_component.ib_sg_list_size;
qp_init_attr.cap.max_recv_sge = mca_btl_openib_component.ib_sg_list_size;
qp_init_attr.qp_type = IBV_QPT_RC;
qp_init_attr.srq = srq;
my_qp = ibv_create_qp(openib_btl->hca->ib_pd, &qp_init_attr);
if (NULL == my_qp) {
BTL_ERROR(("error creating qp errno says %s", strerror(errno)));
return OMPI_ERROR;
}
endpoint->qps[qp].lcl_qp = my_qp;
openib_btl->ib_inline_max = qp_init_attr.cap.max_inline_data;
}
{
endpoint->qps[qp].lcl_qp_attr->qp_state = IBV_QPS_INIT;
endpoint->qps[qp].lcl_qp_attr->pkey_index = openib_btl->pkey_index;
endpoint->qps[qp].lcl_qp_attr->port_num = openib_btl->port_num;
endpoint->qps[qp].lcl_qp_attr->qp_access_flags = IBV_ACCESS_REMOTE_WRITE | IBV_ACCESS_REMOTE_READ;
if (ibv_modify_qp(endpoint->qps[qp].lcl_qp,
endpoint->qps[qp].lcl_qp_attr,
IBV_QP_STATE |
IBV_QP_PKEY_INDEX |
IBV_QP_PORT |
IBV_QP_ACCESS_FLAGS )) {
BTL_ERROR(("error modifying qp to INIT errno says %s", strerror(errno)));
return OMPI_ERROR;
}
}
/* Setup meta data on the endpoint */
endpoint->qps[qp].lcl_psn = lrand48() & 0xffffff;
endpoint->qps[qp].credit_frag = NULL;
openib_btl->cq_users[prio]++;
return OMPI_SUCCESS;
}
/*
* RML send connect information to remote endpoint
*/
static int send_connect_data(mca_btl_base_endpoint_t* endpoint,
uint8_t message_type)
{
orte_buffer_t* buffer = OBJ_NEW(orte_buffer_t);
int rc;
if (NULL == buffer) {
ORTE_ERROR_LOG(ORTE_ERR_OUT_OF_RESOURCE);
return ORTE_ERR_OUT_OF_RESOURCE;
}
/* pack the info in the send buffer */
opal_output(mca_btl_base_output, "packing %d of %d\n", 1, ORTE_UINT8);
rc = orte_dss.pack(buffer, &message_type, 1, ORTE_UINT8);
if (ORTE_SUCCESS != rc) {
ORTE_ERROR_LOG(rc);
return rc;
}
opal_output(mca_btl_base_output, "packing %d of %d\n", 1, ORTE_UINT64);
rc = orte_dss.pack(buffer, &endpoint->subnet_id, 1, ORTE_UINT64);
if (ORTE_SUCCESS != rc) {
ORTE_ERROR_LOG(rc);
return rc;
}
if (message_type != ENDPOINT_CONNECT_REQUEST) {
/* send the QP connect request info we respond to */
opal_output(mca_btl_base_output, "packing %d of %d\n", 1, ORTE_UINT32);
rc = orte_dss.pack(buffer,
&endpoint->rem_info.rem_qps[0].rem_qp_num, 1,
ORTE_UINT32);
if (ORTE_SUCCESS != rc) {
ORTE_ERROR_LOG(rc);
return rc;
}
opal_output(mca_btl_base_output, "packing %d of %d\n", 1, ORTE_UINT16);
rc = orte_dss.pack(buffer, &endpoint->rem_info.rem_lid, 1, ORTE_UINT16);
if (ORTE_SUCCESS != rc) {
ORTE_ERROR_LOG(rc);
return rc;
}
}
if (message_type != ENDPOINT_CONNECT_ACK) {
int qp;
/* stuff all the QP info into the buffer */
for (qp = 0; qp < mca_btl_openib_component.num_qps; qp++) {
opal_output(mca_btl_base_output, "packing %d of %d\n", 1, ORTE_UINT32);
rc = orte_dss.pack(buffer, &endpoint->qps[qp].lcl_qp->qp_num,
1, ORTE_UINT32);
if (ORTE_SUCCESS != rc) {
ORTE_ERROR_LOG(rc);
return rc;
}
opal_output(mca_btl_base_output, "packing %d of %d\n", 1, ORTE_UINT32);
rc = orte_dss.pack(buffer, &endpoint->qps[qp].lcl_psn, 1,
ORTE_UINT32);
if (ORTE_SUCCESS != rc) {
ORTE_ERROR_LOG(rc);
return rc;
}
}
opal_output(mca_btl_base_output, "packing %d of %d\n", 1, ORTE_UINT16);
rc = orte_dss.pack(buffer, &endpoint->endpoint_btl->lid, 1, ORTE_UINT16);
if (ORTE_SUCCESS != rc) {
ORTE_ERROR_LOG(rc);
return rc;
}
opal_output(mca_btl_base_output, "packing %d of %d\n", 1, ORTE_UINT32);
rc = orte_dss.pack(buffer, &endpoint->endpoint_btl->hca->mtu, 1,
ORTE_UINT32);
if (ORTE_SUCCESS != rc) {
ORTE_ERROR_LOG(rc);
return rc;
}
opal_output(mca_btl_base_output, "packing %d of %d\n", 1, ORTE_UINT32);
rc = orte_dss.pack(buffer, &endpoint->index, 1, ORTE_UINT32);
if (ORTE_SUCCESS != rc) {
ORTE_ERROR_LOG(rc);
return rc;
}
}
/* send to remote endpoint */
rc = orte_rml.send_buffer_nb(&endpoint->endpoint_proc->proc_guid,
buffer, OOB_TAG, 0,
rml_send_cb, NULL);
if (ORTE_SUCCESS != rc) {
ORTE_ERROR_LOG(rc);
return rc;
}
BTL_VERBOSE(("Sent QP Info, LID = %d, SUBNET = %016x\n",
endpoint->endpoint_btl->lid,
endpoint->subnet_id));
return OMPI_SUCCESS;
}
/*
* Callback when we have finished RML sending the connect data to a
* remote peer
*/
static void rml_send_cb(int status, orte_process_name_t* endpoint,
orte_buffer_t* buffer, orte_rml_tag_t tag,
void* cbdata)
{
OBJ_RELEASE(buffer);
}
/*
* Non blocking RML recv callback. Read incoming QP and other info,
* and if this endpoint is trying to connect, reply with our QP info,
* otherwise try to modify QP's and establish reliable connection
*/
static void rml_recv_cb(int status, orte_process_name_t* process_name,
orte_buffer_t* buffer, orte_rml_tag_t tag,
void* cbdata)
{
mca_btl_openib_proc_t *ib_proc;
mca_btl_openib_endpoint_t *ib_endpoint = NULL;
int endpoint_state;
int rc;
uint32_t i, lcl_qp = 0;
uint16_t lcl_lid = 0;
int32_t cnt = 1;
mca_btl_openib_rem_info_t rem_info;
uint8_t message_type;
bool master;
/* start by unpacking data first so we know who is knocking at
our door */
opal_output(mca_btl_base_output, "unpacking %d of %d\n", cnt, ORTE_UINT8);
rc = orte_dss.unpack(buffer, &message_type, &cnt, ORTE_UINT8);
if (ORTE_SUCCESS != rc) {
ORTE_ERROR_LOG(rc);
return;
}
opal_output(mca_btl_base_output, "unpacking %d of %d\n", cnt, ORTE_UINT64);
rc = orte_dss.unpack(buffer, &rem_info.rem_subnet_id, &cnt, ORTE_UINT64);
if (ORTE_SUCCESS != rc) {
ORTE_ERROR_LOG(rc);
return;
}
if (ENDPOINT_CONNECT_REQUEST != message_type) {
opal_output(mca_btl_base_output, "unpacking %d of %d\n", cnt, ORTE_UINT32);
rc = orte_dss.unpack(buffer, &lcl_qp, &cnt, ORTE_UINT32);
if (ORTE_SUCCESS != rc) {
ORTE_ERROR_LOG(rc);
return;
}
opal_output(mca_btl_base_output, "unpacking %d of %d\n", cnt, ORTE_UINT16);
rc = orte_dss.unpack(buffer, &lcl_lid, &cnt, ORTE_UINT16);
if (ORTE_SUCCESS != rc) {
ORTE_ERROR_LOG(rc);
return;
}
}
if (ENDPOINT_CONNECT_ACK != message_type) {
int qp;
/* get ready for the data */
rem_info.rem_qps =
(mca_btl_openib_rem_qp_info_t*) malloc(sizeof(mca_btl_openib_rem_qp_info_t) *
mca_btl_openib_component.num_qps);
/* unpack all the qp info */
for (qp = 0; qp < mca_btl_openib_component.num_qps; ++qp) {
opal_output(mca_btl_base_output, "unpacking %d of %d\n", cnt, ORTE_UINT32);
rc = orte_dss.unpack(buffer, &rem_info.rem_qps[qp].rem_qp_num, &cnt,
ORTE_UINT32);
if (ORTE_SUCCESS != rc) {
ORTE_ERROR_LOG(rc);
return;
}
opal_output(mca_btl_base_output, "unpacking %d of %d\n", cnt, ORTE_UINT32);
rc = orte_dss.unpack(buffer, &rem_info.rem_qps[qp].rem_psn, &cnt,
ORTE_UINT32);
if (ORTE_SUCCESS != rc) {
ORTE_ERROR_LOG(rc);
return;
}
}
opal_output(mca_btl_base_output, "unpacking %d of %d\n", cnt, ORTE_UINT16);
rc = orte_dss.unpack(buffer, &rem_info.rem_lid, &cnt, ORTE_UINT16);
if (ORTE_SUCCESS != rc) {
ORTE_ERROR_LOG(rc);
return;
}
opal_output(mca_btl_base_output, "unpacking %d of %d\n", cnt, ORTE_UINT32);
rc = orte_dss.unpack(buffer, &rem_info.rem_mtu, &cnt, ORTE_UINT32);
if (ORTE_SUCCESS != rc) {
ORTE_ERROR_LOG(rc);
return;
}
opal_output(mca_btl_base_output, "unpacking %d of %d\n", cnt, ORTE_UINT32);
rc = orte_dss.unpack(buffer, &rem_info.rem_index, &cnt, ORTE_UINT32);
if (ORTE_SUCCESS != rc) {
ORTE_ERROR_LOG(rc);
return;
}
}
BTL_VERBOSE(("Received QP Info, LID = %d, SUBNET = %016x\n",
rem_info.rem_lid,
rem_info.rem_subnet_id));
master = orte_ns.compare_fields(ORTE_NS_CMP_ALL, orte_process_info.my_name,
process_name) > 0 ? true : false;
for (ib_proc = (mca_btl_openib_proc_t*)
opal_list_get_first(&mca_btl_openib_component.ib_procs);
ib_proc != (mca_btl_openib_proc_t*)
opal_list_get_end(&mca_btl_openib_component.ib_procs);
ib_proc = (mca_btl_openib_proc_t*)opal_list_get_next(ib_proc)) {
bool found = false;
if (orte_ns.compare_fields(ORTE_NS_CMP_ALL,
&ib_proc->proc_guid, process_name) != ORTE_EQUAL) {
continue;
}
if (ENDPOINT_CONNECT_REQUEST != message_type) {
/* This is a reply message. Try to get the endpoint
instance the reply belongs to */
for (i = 0; i < ib_proc->proc_endpoint_count; i++) {
ib_endpoint = ib_proc->proc_endpoints[i];
if (ib_endpoint->qps[0].lcl_qp != NULL &&
lcl_lid == ib_endpoint->endpoint_btl->lid &&
lcl_qp == ib_endpoint->qps[0].lcl_qp->qp_num &&
rem_info.rem_subnet_id == ib_endpoint->subnet_id) {
found = true;
break;
}
}
} else {
/* This is new connection request. If this is master try
to find endpoint in a connecting state. If this is
slave try to find endpoint in closed state and
initiate connection back */
mca_btl_openib_endpoint_t *ib_endpoint_found = NULL;
for (i = 0; i < ib_proc->proc_endpoint_count; i++) {
ib_endpoint = ib_proc->proc_endpoints[i];
if (ib_endpoint->subnet_id != rem_info.rem_subnet_id ||
(ib_endpoint->endpoint_state != MCA_BTL_IB_CONNECTING
&& ib_endpoint->endpoint_state != MCA_BTL_IB_CLOSED))
continue;
found = true;
ib_endpoint_found = ib_endpoint;
if ((master &&
MCA_BTL_IB_CONNECTING == ib_endpoint->endpoint_state) ||
(!master &&
MCA_BTL_IB_CLOSED == ib_endpoint->endpoint_state))
break; /* Found one. No point to continue */
}
ib_endpoint = ib_endpoint_found;
/* if this is slave and there is no endpoints in closed
state then all connection are already in progress so
just ignore this connection request */
if (found && !master &&
MCA_BTL_IB_CLOSED != ib_endpoint->endpoint_state) {
return;
}
}
if (!found) {
BTL_ERROR(("can't find suitable endpoint for this peer\n"));
return;
}
endpoint_state = ib_endpoint->endpoint_state;
/* Update status */
switch (endpoint_state) {
case MCA_BTL_IB_CLOSED :
/* We had this connection closed before. The endpoint is
trying to connect. Move the status of this connection
to CONNECTING, and then reply with our QP
information */
if (master) {
rc = reply_start_connect(ib_endpoint, &rem_info);
} else {
rc = oob_start_connect(ib_endpoint);
}
if (OMPI_SUCCESS != rc) {
BTL_ERROR(("error in endpoint reply start connect"));
break;
}
/* As long as we expect a message from the peer (in order
to setup the connection) let the event engine pool the
RML events. Note: we increment it once peer active
connection. */
opal_progress_event_users_increment();
break;
case MCA_BTL_IB_CONNECTING :
set_remote_info(ib_endpoint, &rem_info);
if (OMPI_SUCCESS != (rc = qp_connect_all(ib_endpoint))) {
BTL_ERROR(("endpoint connect error: %d", rc));
break;
}
if (master) {
ib_endpoint->endpoint_state = MCA_BTL_IB_WAITING_ACK;
/* Send him an ACK */
send_connect_data(ib_endpoint, ENDPOINT_CONNECT_RESPONSE);
} else {
send_connect_data(ib_endpoint, ENDPOINT_CONNECT_ACK);
/* Tell main BTL that we're done */
mca_btl_openib_endpoint_connected(ib_endpoint);
}
break;
case MCA_BTL_IB_WAITING_ACK:
/* Tell main BTL that we're done */
mca_btl_openib_endpoint_connected(ib_endpoint);
break;
case MCA_BTL_IB_CONNECT_ACK:
send_connect_data(ib_endpoint, ENDPOINT_CONNECT_ACK);
/* Tell main BTL that we're done */
mca_btl_openib_endpoint_connected(ib_endpoint);
break;
case MCA_BTL_IB_CONNECTED:
break;
default :
BTL_ERROR(("Invalid endpoint state %d", endpoint_state));
}
OPAL_THREAD_UNLOCK(&ib_endpoint->endpoint_lock);
break;
}
}

18
ompi/mca/btl/openib/connect/btl_openib_connect_oob.h Обычный файл
Просмотреть файл

@ -0,0 +1,18 @@
/*
* Copyright (c) 2007 Cisco, Inc. All rights reserved.
*
* $COPYRIGHT$
*
* Additional copyrights may follow
*
* $HEADER$
*/
#ifndef BTL_OPENIB_CONNECT_OOB_H
#define BTL_OPENIB_CONNECT_OOB_H
#include "connect/connect.h"
extern ompi_btl_openib_connect_base_funcs_t ompi_btl_openib_connect_oob;
#endif

56
ompi/mca/btl/openib/connect/btl_openib_connect_rdma_cm.c Обычный файл
Просмотреть файл

@ -0,0 +1,56 @@
/*
* Copyright (c) 2007 Cisco, Inc. All rights reserved.
*
* $COPYRIGHT$
*
* Additional copyrights may follow
*
* $HEADER$
*/
#include "ompi_config.h"
#include "btl_openib_endpoint.h"
#include "connect/connect.h"
static int rdma_cm_open(void);
static int rdma_cm_init(void);
static int rdma_cm_connect(mca_btl_base_endpoint_t *e);
static int rdma_cm_finalize(void);
ompi_btl_openib_connect_base_funcs_t ompi_btl_openib_connect_rdma_cm = {
"rdma_cm",
rdma_cm_open,
rdma_cm_init,
rdma_cm_connect,
rdma_cm_finalize,
};
static int rdma_cm_open(void)
{
mca_base_param_reg_int(&mca_btl_openib_component.super.btl_version,
"btl_openib_connect_rdma_cm_foo",
"A dummy help message", false, false,
17, NULL);
return OMPI_SUCCESS;
}
static int rdma_cm_init(void)
{
printf("rdma cm init\n");
return OMPI_ERR_NOT_IMPLEMENTED;
}
static int rdma_cm_connect(mca_btl_base_endpoint_t *e)
{
printf("rdma cm connect\n");
return OMPI_ERR_NOT_IMPLEMENTED;
}
static int rdma_cm_finalize(void)
{
printf("rdma cm finalize\n");
return OMPI_ERR_NOT_IMPLEMENTED;
}

20
ompi/mca/btl/openib/connect/btl_openib_connect_rdma_cm.h Обычный файл
Просмотреть файл

@ -0,0 +1,20 @@
/*
* Copyright (c) 2007 Cisco, Inc. All rights reserved.
*
* $COPYRIGHT$
*
* Additional copyrights may follow
*
* $HEADER$
*/
#ifndef BTL_OPENIB_CONNECT_RDMA_CM_H
#define BTL_OPENIB_CONNECT_RDMA_CM_H
#include "ompi_config.h"
#include "connect/connect.h"
extern ompi_btl_openib_connect_base_funcs_t ompi_btl_openib_connect_rdma_cm;
#endif

117
ompi/mca/btl/openib/connect/connect.h Обычный файл
Просмотреть файл

@ -0,0 +1,117 @@
/*
* Copyright (c) 2007 Cisco, Inc. All rights reserved.
*
* $COPYRIGHT$
*
* Additional copyrights may follow
*
* $HEADER$
*/
/**
* @file
*
* This interface is designed to hide the back-end details of how IB
* RC connections are made from the rest of the openib BTL. There are
* module-like instances of the implemented functionality (dlopen and
* friends are not used, but all the functionality is accessed through
* struct's of function pointers, so you can swap between multiple
* different implementations at run time, just like real components).
*
* Currently, the connect functions are referenced by their names
* (e.g., "oob", "rdma_cm"). The decision which to use is made during
* the openib BTL init() function call.
*
* Note that the openib BTL's open() function calls the
* connect_base_open() function, which registers an MCA parameter, and
* scans all the connect modules to see if they have open() functions.
* If they do, they are called. In this way, the connect modules can
* register MCA parameters that show up in ompi_info output.
*
* There are four main functions to this interface:
*
* - open: as described above, used to register MCA params for connect
* modules
*
* - init: to select a connect module. The module is responsible for
* setting itself up for asynchronous operation for incoming
* connection requests (e.g., putting fd's in the progress engine,
* posting non-blocking RML requests, spawning a background thread,
* etc.).
*
* - start_connect: initiate a connection to a remote peer. Similar
* to init, the module is responsible for setting itself up for
* asyncronous operation for progressing the outgoing connection
* request.
*
* - finalize: shut down all asynchronous handling. No need to clean
* up the connections that were made; that's the responsibility of the
* main openib BTL.
*
* There are two functions in the main openib BTL that the module will
* call:
*
* - ompi_btl_openib_post_recvs(endpoint): once a QP is locally
* connected to the remote side (but we don't know if the remote side
* is connected to us yet), this function is invoked to post buffers
* on the QP, setup credits for the endpoint, etc.
*
* - ompi_btl_openib_connected(endpoint): once we know that a QP is
* connected on *both* sides, this function is invoked to tell the
* main openib BTL "ok, you can use this connection now." (e.g., the
* main openib BTL will start sending out fragments that were queued
* while the connection was establing, etc.).
*/
#ifndef BTL_OPENIB_CONNECT_H
#define BTL_OPENIB_CONNECT_H
#include "btl_openib_endpoint.h"
BEGIN_C_DECLS
/**
* Function to register MCA params in the connect functions
*/
typedef int (*ompi_btl_openib_connect_base_func_open_t)(void);
/**
* Function to intialize the connection functions (i.e., it's been
* selected, so do whatever setup is necessary).
*/
typedef int (*ompi_btl_openib_connect_base_func_init_t)(void);
/**
* Function to initiate a connection to a remote process
*/
typedef int (*ompi_btl_openib_connect_base_func_start_connect_t)
(mca_btl_base_endpoint_t *e);
/**
* Function to finalize the connection functions
*/
typedef int (*ompi_btl_openib_connect_base_func_finalize_t)(void);
#define BCF_MAX_NAME 64
struct ompi_btl_openib_connect_base_funcs_t {
/** Name of this set of connection functions */
char bcf_name[BCF_MAX_NAME];
/** Open function */
ompi_btl_openib_connect_base_func_open_t bcf_open;
/** Init function */
ompi_btl_openib_connect_base_func_init_t bcf_init;
/** Connect function */
ompi_btl_openib_connect_base_func_start_connect_t bcf_start_connect;
/** Finalize function */
ompi_btl_openib_connect_base_func_open_t bcf_finalize;
};
typedef struct ompi_btl_openib_connect_base_funcs_t ompi_btl_openib_connect_base_funcs_t;
END_C_DECLS
#endif