1
1

More openib cleanup.. still note ready for public consumption ;-)

This commit was SVN r6565.
Этот коммит содержится в:
Galen Shipman 2005-07-20 15:17:18 +00:00
родитель db4d993228
Коммит 946402b980
10 изменённых файлов: 321 добавлений и 317 удалений

Просмотреть файл

@ -17,8 +17,64 @@
#ifndef MCA_BTL_BASE_ERROR_H
#define MCA_BTL_BASE_ERROR_H
#define BTL_ERROR(fmt, args...) { \
#if defined(ACCEPT_C99) && __STDC_VERSION__ >= 199901L
# define BTL_ERROR(fmt, ...) { \
opal_output(0, "[%s:%d:%d] my_name: [%lu,%lu,%lu] " fmt, __FILE__, __LINE__, __func__, \
ORTE_NAME_ARGS(orte_process_info.my_name), __VA_ARGS__); \
}
#else
# if defined(__GNUC__) && !defined(__STDC__)
#define BTL_ERROR(fmt, args...) { \
opal_output(0, "[%s:%d:%d] my_name: [%lu,%lu,%lu]" fmt, __FILE__, __LINE__, __func__,\
ORTE_NAME_ARGS(orte_process_info.my_name), ##args); \
}
#else
static inline void BTL_ERROR(char *fmt, ... )
{
va_list list;
va_start(list, fmt);
opal_output(0, "[%s:%d:%d] my_name: [%lu,%lu,%lu]", fmt, __FILE__, __LINE__, __func__,
ORTE_NAME_ARGS(orte_process_info.my_name), list);
va_end(list);
}
#define BTL_ERROR printf
#endif
#endif
#ifdef BTL_DEBUG_OUT
#if defined(ACCEPT_C99) && __STDC_VERSION__ >= 199901L
# define BTL_DEBUG_OUT(fmt, ...) { \
opal_output(0, "[%s:%d:%d " fmt, __FILE__, __LINE__, __func__, __VA_ARGS__); \
}
#else
# if defined(__GNUC__) && !defined(__STDC__)
#define BTL_DEBUG_OUT(fmt, args...) { \
opal_output(0, "[%s:%d:%d " fmt, __FILE__, __LINE__, __func__, ##args); \
}
}
#else
static inline void BTL_DEBUG_OUT(char *fmt, ... )
{
va_list list;
va_start(list, fmt);
opal_output(0, "[%s:%d:%d ", fmt, __FILE__, __LINE__, __func__, list);
va_end(list);
}
#define BTL_DEBUG_OUT printf
#endif
#endif
#else
#if defined(ACCEPT_C99) && __STDC_VERSION__ >= 199901L
# define BTL_DEBUG_OUT(fmt, ...)
#else
# if defined(__GNUC__) && !defined(__STDC__)
#define BTL_DEBUG_OUT(fmt, args...)
#else
static inline void BTL_DEBUG_OUT(char *fmt, ... )
{
}
#define BTL_DEBUG_OUT printf
#endif
#endif
#endif
#endif

Просмотреть файл

@ -63,6 +63,11 @@ mca_btl_openib_module_t mca_btl_openib_module = {
/*
* add a proc to this btl module
* creates an endpoint that is setup on the
* first send to the endpoint
*/
int mca_btl_openib_add_procs(
struct mca_btl_base_module_t* btl,
size_t nprocs,
@ -117,13 +122,17 @@ int mca_btl_openib_add_procs(
return OMPI_SUCCESS;
}
/*
* delete the proc as reachable from this btl module
*/
int mca_btl_openib_del_procs(struct mca_btl_base_module_t* btl,
size_t nprocs,
struct ompi_proc_t **procs,
struct mca_btl_base_endpoint_t ** peers)
{
/* Stub */
DEBUG_OUT("Stub\n");
/* TODO */
BTL_DEBUG_OUT("Stub\n");
return OMPI_SUCCESS;
}
@ -154,6 +163,9 @@ int mca_btl_openib_register(
*
* @param btl (IN) BTL module
* @param size (IN) Request segment size.
*
* When allocating a segment we pull a pre-alllocated segment
* from one of two free lists, an eager list and a max list
*/
mca_btl_base_descriptor_t* mca_btl_openib_alloc(
struct mca_btl_base_module_t* btl,
@ -185,6 +197,8 @@ mca_btl_base_descriptor_t* mca_btl_openib_alloc(
/**
* Return a segment
*
* Return the segment to the appropriate
* preallocated segment list
*/
int mca_btl_openib_free(
struct mca_btl_base_module_t* btl,
@ -210,11 +224,27 @@ int mca_btl_openib_free(
/**
* Pack data and return a descriptor that can be
* register user buffer or pack
* data into pre-registered buffer and return a
* descriptor that can be
* used for send/put.
*
* @param btl (IN) BTL module
* @param peer (IN) BTL peer addressing
*
* prepare source's behavior depends on the following:
* Has a valid memory registration been passed to prepare_src?
* if so we attempt to use the pre-registred user-buffer, if the memory registration
* is to small (only a portion of the user buffer) then we must reregister the user buffer
* Has the user requested the memory to be left pinned?
* if so we insert the memory registration into a memory tree for later lookup, we
* may also remove a previous registration if a MRU (most recently used) list of
* registions is full, this prevents resources from being exhausted.
* Is the requested size larger than the btl's max send size?
* if so and we aren't asked to leave the registration pinned than we register the memory if
* the users buffer is contiguous
* Otherwise we choose from two free lists of pre-registered memory in which to pack the data into.
*
*/
mca_btl_base_descriptor_t* mca_btl_openib_prepare_src(
struct mca_btl_base_module_t* btl,
@ -229,7 +259,7 @@ mca_btl_base_descriptor_t* mca_btl_openib_prepare_src(
mca_btl_openib_frag_t* frag;
mca_mpool_openib_registration_t * openib_reg;
struct iovec iov;
int32_t iov_count = 1;
uint32_t iov_count = 1;
size_t max_data = *size;
int32_t free_after;
int rc;
@ -238,13 +268,13 @@ mca_btl_base_descriptor_t* mca_btl_openib_prepare_src(
openib_btl = (mca_btl_openib_module_t*) btl;
openib_reg = (mca_mpool_openib_registration_t*) registration;
/** if the data fits in the eager limit and we aren't told to pinn then we
simply pack, if the data fits in the eager limit and the data is non contiguous
then we pack **/
if(NULL != openib_reg && 0 == ompi_convertor_need_buffers(convertor)){
bool is_leave_pinned = openib_reg->base_reg.is_leave_pinned;
size_t reg_len;
/* the memory is already pinned and we have contiguous user data */
MCA_BTL_IB_FRAG_ALLOC_FRAG(btl, frag, rc);
if(NULL == frag){
return NULL;
@ -254,15 +284,16 @@ mca_btl_base_descriptor_t* mca_btl_openib_prepare_src(
iov.iov_base = NULL;
ompi_convertor_pack(convertor, &iov, &iov_count, &max_data, &free_after);
/* first we will try to find this address in the memory tree (from MPI_Alloc_mem) */
frag->segment.seg_len = max_data;
frag->segment.seg_addr.pval = iov.iov_base;
size_t reg_len;
reg_len = (unsigned char*)openib_reg->base_reg.bound - (unsigned char*)iov.iov_base + 1;
if(frag->segment.seg_len > reg_len) {
/* the pinned region is too small! we have to re-pinn it */
size_t new_len = openib_reg->base_reg.bound - openib_reg->base_reg.base + 1
+ frag->segment.seg_len - reg_len;
void * base_addr = openib_reg->base_reg.base;
@ -299,12 +330,14 @@ mca_btl_base_descriptor_t* mca_btl_openib_prepare_src(
}
OBJ_RETAIN(openib_reg);
if(is_leave_pinned) {
/* we should leave the memory pinned so put the memory on the MRU list */
openib_reg->base_reg.is_leave_pinned = is_leave_pinned;
opal_list_append(&openib_btl->reg_mru_list, (opal_list_item_t*) openib_reg);
}
}
else if(is_leave_pinned) {
} else if(is_leave_pinned) {
/* the current memory region is large enough and we should leave the memory pinned */
if(NULL == opal_list_remove_item(&openib_btl->reg_mru_list, (opal_list_item_t*) openib_reg)) {
BTL_ERROR("error removing item from reg_mru_list");
return NULL;
@ -333,6 +366,10 @@ mca_btl_base_descriptor_t* mca_btl_openib_prepare_src(
ompi_convertor_need_buffers(convertor) == 0 &&
reserve == 0)
{
/* The user buffer is contigous and we need to leave the buffer pinned or we are asked to send
more than the max send size. Note that the memory was not already pinned because we have
no registration information passed to us */
MCA_BTL_IB_FRAG_ALLOC_FRAG(btl, frag, rc);
if(NULL == frag){
return NULL;
@ -350,14 +387,19 @@ mca_btl_base_descriptor_t* mca_btl_openib_prepare_src(
if(mca_btl_openib_component.leave_pinned) {
/* so we need to leave pinned which means we must check and see if we
have room on the MRU list */
if(mca_btl_openib_component.reg_mru_len <= openib_btl->reg_mru_list.opal_list_length ) {
/* we have the maximum number of entries on the MRU list, time to
pull something off and make room. */
mca_mpool_openib_registration_t* old_reg =
(mca_mpool_openib_registration_t*)
opal_list_remove_last(&openib_btl->reg_mru_list);
if( NULL == old_reg) {
opal_output(0,"%s:%d:%s error removing item from reg_mru_list", __FILE__, __LINE__, __func__);
BTL_ERROR("error removing item from reg_mru_list");
return NULL;
}
@ -365,7 +407,7 @@ mca_btl_base_descriptor_t* mca_btl_openib_prepare_src(
rc = mca_mpool_base_remove((void*) old_reg->base_reg.base);
if(OMPI_SUCCESS != rc) {
opal_output(0,"%s:%d:%s error removing memory region from memory pool tree", __FILE__, __LINE__, __func__);
BTL_ERROR("error removing memory region from memory pool tree");
return NULL;
}
@ -390,7 +432,8 @@ mca_btl_base_descriptor_t* mca_btl_openib_prepare_src(
opal_list_append(&openib_btl->reg_mru_list, (opal_list_item_t*) openib_reg);
} else {
} else {
/* we don't need to leave the memory pinned so just register it.. */
openib_btl->ib_pool->mpool_register(openib_btl->ib_pool,
iov.iov_base,
max_data,
@ -410,20 +453,23 @@ mca_btl_base_descriptor_t* mca_btl_openib_prepare_src(
frag->base.des_dst = NULL;
frag->base.des_dst_cnt = 0;
frag->openib_reg = openib_reg;
DEBUG_OUT("frag->sg_entry.lkey = %lu .addr = %llu", frag->sg_entry.lkey, frag->sg_entry.addr);
BTL_DEBUG_OUT("frag->sg_entry.lkey = %lu .addr = %llu", frag->sg_entry.lkey, frag->sg_entry.addr);
return &frag->base;
} else if (max_data+reserve <= btl->btl_eager_limit) {
/* the data is small enough to fit in the eager frag and
either we received no prepinned memory or leave pinned is
not set
*/
MCA_BTL_IB_FRAG_ALLOC_EAGER(btl, frag, rc);
if(NULL == frag) {
return NULL;
}
iov.iov_len = max_data;
iov.iov_base = frag->segment.seg_addr.lval + reserve;
iov.iov_base = (unsigned char*) frag->segment.seg_addr.lval + reserve;
rc = ompi_convertor_pack(convertor, &iov, &iov_count, &max_data, &free_after);
*size = max_data;
@ -484,6 +530,14 @@ mca_btl_base_descriptor_t* mca_btl_openib_prepare_src(
*
* @param btl (IN) BTL module
* @param peer (IN) BTL peer addressing
* prepare dest's behavior depends on the following:
* Has a valid memory registration been passed to prepare_src?
* if so we attempt to use the pre-registred user-buffer, if the memory registration
* is to small (only a portion of the user buffer) then we must reregister the user buffer
* Has the user requested the memory to be left pinned?
* if so we insert the memory registration into a memory tree for later lookup, we
* may also remove a previous registration if a MRU (most recently used) list of
* registions is full, this prevents resources from being exhausted.
*/
mca_btl_base_descriptor_t* mca_btl_openib_prepare_dst(
struct mca_btl_base_module_t* btl,
@ -525,13 +579,13 @@ mca_btl_base_descriptor_t* mca_btl_openib_prepare_dst(
rc = mca_mpool_base_remove((void*) openib_reg->base_reg.base);
if(OMPI_SUCCESS != rc) {
opal_output(0,"%s:%d:%s error removing memory region from memory pool tree", __FILE__, __LINE__, __func__);
BTL_ERROR("error removing memory region from memory pool tree");
return NULL;
}
if(is_leave_pinned) {
if(NULL == opal_list_remove_item(&openib_btl->reg_mru_list, (opal_list_item_t*) openib_reg)) {
opal_output(0,"%s:%d:%s error removing item from reg_mru_list", __FILE__, __LINE__, __func__);
BTL_ERROR("error removing item from reg_mru_list");
return NULL;
}
}
@ -550,7 +604,7 @@ mca_btl_base_descriptor_t* mca_btl_openib_prepare_dst(
(mca_mpool_base_registration_t*) openib_reg);
if(OMPI_SUCCESS != rc) {
opal_output(0,"%s:%d:%s error inserting memory region into memory pool tree", __FILE__, __LINE__, __func__);
BTL_ERROR("error inserting memory region into memory pool tree");
return NULL;
}
OBJ_RETAIN(openib_reg);
@ -563,7 +617,7 @@ mca_btl_base_descriptor_t* mca_btl_openib_prepare_dst(
}
else if(is_leave_pinned){
if(NULL == opal_list_remove_item(&openib_btl->reg_mru_list, (opal_list_item_t*) openib_reg)) {
opal_output(0,"%s:%d:%s error removing item from reg_mru_list", __FILE__, __LINE__, __func__);
BTL_ERROR("error removing item from reg_mru_list");
return NULL;
}
opal_list_append(&openib_btl->reg_mru_list, (opal_list_item_t*) openib_reg);
@ -581,13 +635,13 @@ mca_btl_base_descriptor_t* mca_btl_openib_prepare_dst(
opal_list_remove_last(&openib_btl->reg_mru_list);
if( NULL == old_reg) {
opal_output(0,"%s:%d:%s error removing item from reg_mru_list", __FILE__, __LINE__, __func__);
BTL_ERROR("error removing item from reg_mru_list");
return NULL;
}
rc = mca_mpool_base_remove((void*) old_reg->base_reg.base);
if(OMPI_SUCCESS !=rc ) {
opal_output(0,"%s:%d:%s error removing memory region from memory pool tree", __FILE__, __LINE__, __func__);
BTL_ERROR("error removing memory region from memory pool tree");
return NULL;
}
@ -607,7 +661,7 @@ mca_btl_base_descriptor_t* mca_btl_openib_prepare_dst(
(void*) (&openib_btl->super),
(mca_mpool_base_registration_t*) openib_reg);
if(OMPI_SUCCESS != rc){
opal_output(0,"%s:%d:%s error inserting memory region into memory pool", __FILE__, __LINE__, __func__);
BTL_ERROR("error inserting memory region into memory pool");
return NULL;
}
@ -638,7 +692,7 @@ mca_btl_base_descriptor_t* mca_btl_openib_prepare_dst(
frag->base.des_src = NULL;
frag->base.des_src_cnt = 0;
frag->openib_reg = openib_reg;
DEBUG_OUT("frag->sg_entry.lkey = %lu .addr = %llu frag->segment.seg_key.key32[0] = %lu" , frag->sg_entry.lkey, frag->sg_entry.addr, frag->segment.seg_key.key32[0]);
BTL_DEBUG_OUT("frag->sg_entry.lkey = %lu .addr = %llu frag->segment.seg_key.key32[0] = %lu" , frag->sg_entry.lkey, frag->sg_entry.addr, frag->segment.seg_key.key32[0]);
return &frag->base;
@ -648,7 +702,8 @@ int mca_btl_openib_finalize(struct mca_btl_base_module_t* btl)
{
mca_btl_openib_module_t* openib_btl;
openib_btl = (mca_btl_openib_module_t*) btl;
#if 0
if(openib_btl->send_free_eager.fl_num_allocated !=
openib_btl->send_free_eager.super.opal_list_length){
opal_output(0, "btl ib send_free_eager frags: %d allocated %d returned \n",
@ -681,15 +736,13 @@ int mca_btl_openib_finalize(struct mca_btl_base_module_t* btl)
openib_btl->recv_free_max.fl_num_allocated,
openib_btl->recv_free_max.super.opal_list_length);
}
#endif
return OMPI_SUCCESS;
}
/*
* Initiate a send. If this is the first fragment, use the fragment
* descriptor allocated with the send requests, otherwise obtain
* one from the free list. Initialize the fragment and foward
* on to the peer.
* Initiate a send.
*/
int mca_btl_openib_send(
@ -721,82 +774,34 @@ int mca_btl_openib_put( mca_btl_base_module_t* btl,
struct ibv_send_wr* bad_wr;
mca_btl_openib_frag_t* frag = (mca_btl_openib_frag_t*) descriptor;
frag->endpoint = endpoint;
frag->sr_desc.opcode = IBV_WR_RDMA_WRITE;
frag->sr_desc.send_flags = IBV_SEND_SIGNALED;
frag->sr_desc.wr.rdma.remote_addr = (uintptr_t) frag->base.des_dst->seg_addr.pval;
frag->sr_desc.wr.rdma.rkey = frag->base.des_dst->seg_key.key32[0];
frag->wr_desc.sr_desc.opcode = IBV_WR_RDMA_WRITE;
frag->wr_desc.sr_desc.send_flags = IBV_SEND_SIGNALED;
frag->wr_desc.sr_desc.wr.rdma.remote_addr = (uintptr_t) frag->base.des_dst->seg_addr.pval;
frag->wr_desc.sr_desc.wr.rdma.rkey = frag->base.des_dst->seg_key.key32[0];
frag->sg_entry.addr = (uintptr_t) frag->base.des_src->seg_addr.pval;
frag->sg_entry.length = frag->base.des_src->seg_len;
DEBUG_OUT("frag->sr_desc.wr.rdma.remote_addr = %llu .rkey = %lu frag->sg_entry.addr = %llu .length = %lu"
, frag->sr_desc.wr.rdma.remote_addr
, frag->sr_desc.wr.rdma.rkey
, frag->sg_entry.addr
, frag->sg_entry.length);
BTL_DEBUG_OUT("frag->wr_desc.sr_desc.wr.rdma.remote_addr = %llu .rkey = %lu frag->sg_entry.addr = %llu .length = %lu"
, frag->wr_desc.sr_desc.wr.rdma.remote_addr
, frag->wr_desc.sr_desc.wr.rdma.rkey
, frag->sg_entry.addr
, frag->sg_entry.length);
if(ibv_post_send(endpoint->lcl_qp_low,
&frag->sr_desc,
&frag->wr_desc.sr_desc,
&bad_wr)){
opal_output(0, "%s: error posting send request errno says %s\n", __func__, strerror(errno));
BTL_ERROR("error posting send request errno says %s", strerror(errno));
return OMPI_ERROR;
}
mca_btl_openib_endpoint_post_rr(endpoint, 1);
MCA_BTL_OPENIB_ENDPOINT_POST_RR_HIGH(endpoint, 1);
MCA_BTL_OPENIB_ENDPOINT_POST_RR_LOW(endpoint, 1);
return OMPI_SUCCESS;
}
/*
* Asynchronous event handler to detect unforseen
* events. Usually, such events are catastrophic.
* Should have a robust mechanism to handle these
* events and abort the OMPI application if necessary.
*
*/
/* static void async_event_handler(VAPI_hca_hndl_t hca_hndl, */
/* VAPI_event_record_t * event_p, */
/* void *priv_data) */
/* { */
/* switch (event_p->type) { */
/* case VAPI_QP_PATH_MIGRATED: */
/* case VAPI_EEC_PATH_MIGRATED: */
/* case VAPI_QP_COMM_ESTABLISHED: */
/* case VAPI_EEC_COMM_ESTABLISHED: */
/* case VAPI_SEND_QUEUE_DRAINED: */
/* case VAPI_PORT_ACTIVE: */
/* { */
/* DEBUG_OUT("Got an asynchronous event: %s\n", */
/* VAPI_event_record_sym(event_p->type)); */
/* break; */
/* } */
/* case VAPI_CQ_ERROR: */
/* case VAPI_LOCAL_WQ_INV_REQUEST_ERROR: */
/* case VAPI_LOCAL_WQ_ACCESS_VIOL_ERROR: */
/* case VAPI_LOCAL_WQ_CATASTROPHIC_ERROR: */
/* case VAPI_PATH_MIG_REQ_ERROR: */
/* case VAPI_LOCAL_EEC_CATASTROPHIC_ERROR: */
/* case VAPI_LOCAL_CATASTROPHIC_ERROR: */
/* case VAPI_PORT_ERROR: */
/* { */
/* opal_output(0, "Got an asynchronous event: %s (%s)", */
/* VAPI_event_record_sym(event_p->type), */
/* VAPI_event_syndrome_sym(event_p-> */
/* syndrome)); */
/* break; */
/* } */
/* default: */
/* opal_output(0, "Warning!! Got an undefined " */
/* "asynchronous event\n"); */
/* } */
/* } */
int mca_btl_openib_module_init(mca_btl_openib_module_t *openib_btl)
{
@ -809,36 +814,30 @@ int mca_btl_openib_module_init(mca_btl_openib_module_t *openib_btl)
if(NULL == openib_btl->ib_pd) {
opal_output(0, "%s: error allocating pd for %s errno says %s\n",
__func__,
ibv_get_device_name(openib_btl->ib_dev),
strerror(errno));
BTL_ERROR("error allocating pd for %s errno says %s\n",
ibv_get_device_name(openib_btl->ib_dev),
strerror(errno));
return OMPI_ERROR;
}
/* Create the low and high priority queue pairs */
openib_btl->ib_cq_low = ibv_create_cq(ctx, mca_btl_openib_component.ib_cq_size, NULL);
if(NULL == openib_btl->ib_cq_low) {
opal_output(0, "%s: error creating low priority cq for %s errno says %s\n",
__func__,
ibv_get_device_name(openib_btl->ib_dev),
strerror(errno));
BTL_ERROR("error creating low priority cq for %s errno says %s\n",
ibv_get_device_name(openib_btl->ib_dev),
strerror(errno));
return OMPI_ERROR;
}
openib_btl->ib_cq_high = ibv_create_cq(ctx, mca_btl_openib_component.ib_cq_size, NULL);
if(NULL == openib_btl->ib_cq_high) {
opal_output(0, "%s: error creating high priority cq for %s errno says %s\n",
__func__,
ibv_get_device_name(openib_btl->ib_dev),
strerror(errno));
BTL_ERROR("error creating high priority cq for %s errno says %s\n",
ibv_get_device_name(openib_btl->ib_dev),
strerror(errno));
return OMPI_ERROR;
}
/* TODO: EVAPI_set_qsync_event_handler? */
return OMPI_SUCCESS;
}

Просмотреть файл

@ -32,7 +32,7 @@
#include "mca/btl/btl.h"
#include "opal/util/output.h"
#include "mca/mpool/mpool.h"
#include "btl_openib_error.h"
#include "mca/btl/base/btl_base_error.h"
#include "mca/btl/btl.h"
#include "mca/btl/base/base.h"

Просмотреть файл

@ -507,10 +507,9 @@ int mca_btl_openib_component_progress()
for(i = 0; i < mca_btl_openib_component.ib_num_btls; i++) {
struct ibv_wc wc;
memset(&wc, 0, sizeof(struct ibv_wc));
mca_btl_openib_module_t* openib_btl = &mca_btl_openib_component.openib_btls[i];
memset(&wc, 0, sizeof(struct ibv_wc));
/* we have two completion queues, one for "high" priority and one for "low".
* we will check the high priority and process them until there are none left.
* note that low priority messages are only processed one per progress call.
@ -522,12 +521,12 @@ int mca_btl_openib_component_progress()
return OMPI_ERROR;
}
else if(wc.status != IBV_WC_SUCCESS) {
BTL_ERROR("error polling CQ with status %d for wr_id %d\n",
BTL_ERROR("error polling CQ with status %d for wr_id %llu\n",
wc.status, wc.wr_id);
return OMPI_ERROR;
}
else if(1 == ne) {
DEBUG_OUT("completion queue event says opcode is %d\n", wc.opcode);
BTL_DEBUG_OUT("completion queue event says opcode is %d\n", wc.opcode);
/* Handle work completions */
switch(wc.opcode) {
@ -538,7 +537,7 @@ int mca_btl_openib_component_progress()
case IBV_WC_RECV:
/* Process a RECV */
DEBUG_OUT("Got an recv on the completion queue");
BTL_DEBUG_OUT("Got an recv on the completion queue");
frag = (mca_btl_openib_frag_t*) wc.wr_id;
endpoint = (mca_btl_openib_endpoint_t*) frag->endpoint;
frag->rc=OMPI_SUCCESS;
@ -547,9 +546,6 @@ int mca_btl_openib_component_progress()
((unsigned char*) frag->segment.seg_addr.pval - (unsigned char*) frag->hdr);
OPAL_THREAD_ADD32(&endpoint->rr_posted_high, -1);
mca_btl_openib_endpoint_post_rr(((mca_btl_openib_frag_t*)wc.wr_id)->endpoint, 0);
/* advance the segment address past the header and subtract from the length..*/
openib_btl->ib_reg[frag->hdr->tag].cbfunc(&openib_btl->super,
@ -557,6 +553,8 @@ int mca_btl_openib_component_progress()
&frag->base,
openib_btl->ib_reg[frag->hdr->tag].cbdata);
OPAL_THREAD_ADD32(&endpoint->rr_posted_high, -1);
MCA_BTL_OPENIB_ENDPOINT_POST_RR_HIGH(((mca_btl_openib_frag_t*)wc.wr_id)->endpoint, 0);
OMPI_FREE_LIST_RETURN(&(openib_btl->recv_free_eager), (opal_list_item_t*) frag);
count++;
break;
@ -588,8 +586,8 @@ int mca_btl_openib_component_progress()
return OMPI_ERROR;
}
else if(wc.status != IBV_WC_SUCCESS) {
BTL_ERROR("error polling CQ with status %d for wr_id %d",
wc.status, wc.wr_id);
BTL_ERROR("error polling CQ with status %d for wr_id %llu",
wc.status, wc.wr_id);
return OMPI_ERROR;
}
else if(1 == ne) {
@ -601,7 +599,7 @@ int mca_btl_openib_component_progress()
case IBV_WC_RECV:
/* process a recv completion (this should only occur for a send not an rdma) */
DEBUG_OUT( "%s:%d ib recv under redesign\n", __FILE__, __LINE__);
BTL_DEBUG_OUT( "%s:%d ib recv under redesign\n", __FILE__, __LINE__);
frag = (mca_btl_openib_frag_t*) wc.wr_id;
endpoint = (mca_btl_openib_endpoint_t*) frag->endpoint;
frag->rc=OMPI_SUCCESS;
@ -611,17 +609,15 @@ int mca_btl_openib_component_progress()
wc.byte_len-
((unsigned char*) frag->segment.seg_addr.pval - (unsigned char*) frag->hdr);
OPAL_THREAD_ADD32(&endpoint->rr_posted_low, -1);
mca_btl_openib_endpoint_post_rr(((mca_btl_openib_frag_t*)wc.wr_id)->endpoint, 0);
openib_btl->ib_reg[frag->hdr->tag].cbfunc(&openib_btl->super,
frag->hdr->tag,
&frag->base,
openib_btl->ib_reg[frag->hdr->tag].cbdata);
OPAL_THREAD_ADD32(&endpoint->rr_posted_low, -1);
MCA_BTL_OPENIB_ENDPOINT_POST_RR_LOW(((mca_btl_openib_frag_t*)wc.wr_id)->endpoint, 0);
OMPI_FREE_LIST_RETURN(&(openib_btl->recv_free_max), (opal_list_item_t*) frag);
count++;
break;

Просмотреть файл

@ -58,6 +58,10 @@ int mca_btl_openib_endpoint_qp_init_query(
uint32_t port_num
);
/*
* post a send to the work queue
*/
static inline int mca_btl_openib_endpoint_post_send(mca_btl_openib_module_t* openib_btl,
mca_btl_openib_endpoint_t * endpoint,
mca_btl_openib_frag_t * frag)
@ -73,7 +77,7 @@ static inline int mca_btl_openib_endpoint_post_send(mca_btl_openib_module_t* ope
ib_qp = endpoint->lcl_qp_low;
}
frag->sr_desc.opcode = IBV_WR_SEND;
frag->wr_desc.sr_desc.opcode = IBV_WR_SEND;
frag->sg_entry.length =
frag->segment.seg_len +
@ -81,18 +85,19 @@ static inline int mca_btl_openib_endpoint_post_send(mca_btl_openib_module_t* ope
if(frag->sg_entry.length <= openib_btl->ib_inline_max) {
/* frag->sr_desc.send_flags |= IBV_SEND_INLINE; */
frag->wr_desc.sr_desc.send_flags |= IBV_SEND_INLINE;
}
if(ibv_post_send(ib_qp,
&frag->sr_desc,
&frag->wr_desc.sr_desc,
&bad_wr)) {
opal_output(0, "%s: error posting send request errno says %s\n", __func__, strerror(errno));
BTL_ERROR("error posting send request errno says %s", strerror(errno));
return OMPI_ERROR;
}
mca_btl_openib_endpoint_post_rr(endpoint, 1);
MCA_BTL_OPENIB_ENDPOINT_POST_RR_HIGH(endpoint, 1);
MCA_BTL_OPENIB_ENDPOINT_POST_RR_LOW(endpoint, 1);
return OMPI_SUCCESS;
}
@ -200,7 +205,7 @@ static int mca_btl_openib_endpoint_send_connect_req(mca_btl_base_endpoint_t* end
mca_btl_openib_endpoint_send_cb, NULL);
DEBUG_OUT("Sending High Priority QP num = %d, Low Priority QP num = %d, LID = %d",
BTL_DEBUG_OUT("Sending High Priority QP num = %d, Low Priority QP num = %d, LID = %d",
endpoint->lcl_qp_high->qp_num,
endpoint->lcl_qp_low->qp_num,
endpoint->endpoint_btl->ib_port_attr->lid);
@ -256,10 +261,7 @@ static int mca_btl_openib_endpoint_send_connect_ack(mca_btl_base_endpoint_t* end
/*
* Set remote connection info
*
* XXX: Currently size is unutilized, this shall change
* as soon as we add more info to be exchanged at connection
* setup.
* (from OOB connection)
*
*/
static int mca_btl_openib_endpoint_set_remote_info(mca_btl_base_endpoint_t* endpoint, orte_buffer_t* buffer)
@ -294,7 +296,7 @@ static int mca_btl_openib_endpoint_set_remote_info(mca_btl_base_endpoint_t* endp
ORTE_ERROR_LOG(rc);
return rc;
}
DEBUG_OUT("Received High Priority QP num = %d, Low Priority QP num %d, LID = %d",
BTL_DEBUG_OUT("Received High Priority QP num = %d, Low Priority QP num %d, LID = %d",
endpoint->rem_qp_num_high,
endpoint->rem_qp_num_low,
endpoint->rem_lid);
@ -344,7 +346,7 @@ static int mca_btl_openib_endpoint_start_connect(mca_btl_base_endpoint_t* endpoi
}
endpoint->lcl_psn_low = lrand48() & 0xffffff;
DEBUG_OUT("Initialized High Priority QP num = %d, Low Priority QP num = %d, LID = %d",
BTL_DEBUG_OUT("Initialized High Priority QP num = %d, Low Priority QP num = %d, LID = %d",
endpoint->lcl_qp_high->qp_num,
endpoint->lcl_qp_low->qp_num,
openib_btl->ib_port_attr->lid);
@ -375,8 +377,7 @@ static int mca_btl_openib_endpoint_reply_start_connect(mca_btl_openib_endpoint_t
openib_btl->ib_cq_high,
endpoint->lcl_qp_attr_high,
&endpoint->lcl_qp_high))) {
opal_output(0, "[%lu,%lu,%lu] %s:%d errcode %d\n",
ORTE_NAME_ARGS(orte_process_info.my_name), __FILE__,__LINE__,rc);
BTL_ERROR("error creating queue pair, error code %d", rc);
return rc;
}
srand48(getpid() * time(NULL));
@ -388,13 +389,12 @@ static int mca_btl_openib_endpoint_reply_start_connect(mca_btl_openib_endpoint_t
openib_btl->ib_cq_low,
endpoint->lcl_qp_attr_low,
&endpoint->lcl_qp_low))) {
opal_output(0, "[%lu,%lu,%lu] %s:%d errcode %d\n",
ORTE_NAME_ARGS(orte_process_info.my_name), __FILE__,__LINE__,rc);
BTL_ERROR("error createing queue pair, error code %d", rc);
return rc;
}
endpoint->lcl_psn_low = lrand48() & 0xffffff;
DEBUG_OUT("Initialized High Priority QP num = %d, Low Priority QP num = %d, LID = %d",
BTL_DEBUG_OUT("Initialized High Priority QP num = %d, Low Priority QP num = %d, LID = %d",
endpoint->lcl_qp_high->qp_num,
endpoint->lcl_qp_low->qp_num,
openib_btl->ib_port_attr->lid);
@ -407,15 +407,13 @@ static int mca_btl_openib_endpoint_reply_start_connect(mca_btl_openib_endpoint_t
rc = mca_btl_openib_endpoint_connect(endpoint);
if(rc != OMPI_SUCCESS) {
opal_output(0, "[%lu,%lu,%lu] %s:%d errcode %d\n",
ORTE_NAME_ARGS(orte_process_info.my_name), __FILE__,__LINE__,rc);
BTL_ERROR("error in endpoint connect error code is %d", rc);
return rc;
}
/* Send connection info over to remote endpoint */
if(OMPI_SUCCESS != (rc = mca_btl_openib_endpoint_send_connect_req(endpoint))) {
opal_output(0, "[%lu,%lu,%lu] %s:%d errcode %d\n",
ORTE_NAME_ARGS(orte_process_info.my_name), __FILE__,__LINE__,rc);
BTL_ERROR("error in endpoint send connect request error code is %d", rc);
return rc;
}
return OMPI_SUCCESS;
@ -492,8 +490,7 @@ static void mca_btl_openib_endpoint_recv(
mca_btl_openib_endpoint_set_remote_info(ib_endpoint, buffer);
if(OMPI_SUCCESS != (rc = mca_btl_openib_endpoint_connect(ib_endpoint))) {
opal_output(0, "[%lu,%lu,%lu] %s:%d errcode %d\n",
ORTE_NAME_ARGS(orte_process_info.my_name), __FILE__,__LINE__,rc);
BTL_ERROR("endpoint connect error: %d", rc);
break;
}
@ -505,7 +502,6 @@ static void mca_btl_openib_endpoint_recv(
break;
case MCA_BTL_IB_CONNECT_ACK:
DEBUG_OUT("Got a connect ack from %d\n", endpoint->vpid);
mca_btl_openib_endpoint_connected(ib_endpoint);
break;
@ -514,7 +510,7 @@ static void mca_btl_openib_endpoint_recv(
break;
default :
opal_output(0, "Connected -> Connecting not possible.\n");
BTL_ERROR("Invalid endpoint state %d", endpoint_state);
}
break;
@ -526,6 +522,9 @@ static void mca_btl_openib_endpoint_recv(
mca_btl_openib_post_recv();
}
/*
* Post the OOB recv (for receiving the peers information)
*/
void mca_btl_openib_post_recv()
{
@ -556,7 +555,7 @@ int mca_btl_openib_endpoint_send(
switch(endpoint->endpoint_state) {
case MCA_BTL_IB_CONNECTING:
DEBUG_OUT("Queing because state is connecting");
BTL_DEBUG_OUT("Queing because state is connecting");
opal_list_append(&endpoint->pending_send_frags,
(opal_list_item_t *)frag);
@ -566,7 +565,7 @@ int mca_btl_openib_endpoint_send(
case MCA_BTL_IB_CONNECT_ACK:
DEBUG_OUT("Queuing because waiting for ack");
BTL_DEBUG_OUT("Queuing because waiting for ack");
opal_list_append(&endpoint->pending_send_frags,
(opal_list_item_t *)frag);
@ -576,13 +575,10 @@ int mca_btl_openib_endpoint_send(
case MCA_BTL_IB_CLOSED:
DEBUG_OUT("Connection to endpoint closed ... connecting ...");
BTL_DEBUG_OUT("Connection to endpoint closed ... connecting ...");
opal_list_append(&endpoint->pending_send_frags,
(opal_list_item_t *)frag);
rc = mca_btl_openib_endpoint_start_connect(endpoint);
break;
case MCA_BTL_IB_FAILED:
@ -593,15 +589,11 @@ int mca_btl_openib_endpoint_send(
case MCA_BTL_IB_CONNECTED:
{
openib_btl = endpoint->endpoint_btl;
DEBUG_OUT("Send to : %d, len : %d, frag : %p",
endpoint->endpoint_proc->proc_guid.vpid,
frag->sg_entry.length,
frag);
BTL_DEBUG_OUT("Send to : %d, len : %lu, frag : %llu",
endpoint->endpoint_proc->proc_guid.vpid,
frag->sg_entry.length,
(unsigned long long) frag);
rc = mca_btl_openib_endpoint_post_send(openib_btl, endpoint, frag);
break;
}
@ -613,7 +605,10 @@ int mca_btl_openib_endpoint_send(
return rc;
}
/*
* Progress send frags, any frags that have been queued up before the connection
* was established need to be progressed.
*/
void mca_btl_openib_progress_send_frags(mca_btl_openib_endpoint_t* endpoint)
{
opal_list_item_t *frag_item;
@ -635,7 +630,7 @@ void mca_btl_openib_progress_send_frags(mca_btl_openib_endpoint_t* endpoint)
/* We need to post this one */
if(OMPI_SUCCESS != mca_btl_openib_endpoint_post_send(openib_btl, endpoint, frag))
opal_output(0, "error in mca_btl_openib_endpoint_send");
BTL_ERROR("Error posting send");
}
}
@ -685,12 +680,17 @@ int mca_btl_openib_endpoint_connect(
return rc;
}
mca_btl_openib_endpoint_post_rr(endpoint, 0);
MCA_BTL_OPENIB_ENDPOINT_POST_RR_HIGH(endpoint, 0);
MCA_BTL_OPENIB_ENDPOINT_POST_RR_LOW(endpoint, 0);
return OMPI_SUCCESS;
}
/*
* Create the queue pair note that this is just the initial
* queue pair creation and we need to get the remote queue pair
* info from the peer before the qp is usable,
*/
int mca_btl_openib_endpoint_create_qp(
mca_btl_openib_module_t* openib_btl,
@ -718,7 +718,7 @@ int mca_btl_openib_endpoint_create_qp(
my_qp = ibv_create_qp(pd, &qp_init_attr);
if(NULL == my_qp) {
opal_output(0, "%s: error creating qp errno says %s\n", __func__, strerror(errno));
BTL_ERROR("error creating qp errno says %s", strerror(errno));
return OMPI_ERROR;
}
(*qp) = my_qp;
@ -737,7 +737,7 @@ int mca_btl_openib_endpoint_create_qp(
IBV_QP_PKEY_INDEX |
IBV_QP_PORT |
IBV_QP_ACCESS_FLAGS )) {
opal_output(0, "%s: error modifying qp to INIT errno says %s\n", __func__, strerror(errno));
BTL_ERROR("error modifying qp to INIT errno says %s", strerror(errno));
return OMPI_ERROR;
}
}
@ -745,6 +745,11 @@ int mca_btl_openib_endpoint_create_qp(
return OMPI_SUCCESS;
}
/*
* The queue pair has been created and we have received the remote
* queue pair information from the peer so we init this queue pair
* and are ready to roll.
*/
int mca_btl_openib_endpoint_qp_init_query(
mca_btl_openib_module_t* openib_btl,
struct ibv_qp* qp,
@ -778,7 +783,7 @@ int mca_btl_openib_endpoint_qp_init_query(
IBV_QP_RQ_PSN |
IBV_QP_MAX_DEST_RD_ATOMIC |
IBV_QP_MIN_RNR_TIMER)) {
opal_output(0, "%s: error modifing QP to RTR errno says %s\n", __func__, strerror(errno));
BTL_ERROR("error modifing QP to RTR errno says %s", strerror(errno));
return OMPI_ERROR;
}
attr->qp_state = IBV_QPS_RTS;
@ -794,7 +799,7 @@ int mca_btl_openib_endpoint_qp_init_query(
IBV_QP_RNR_RETRY |
IBV_QP_SQ_PSN |
IBV_QP_MAX_QP_RD_ATOMIC)) {
opal_output(0, "%s: error modifying QP to RTS errno says %s\n", __func__, strerror(errno));
BTL_ERROR("error modifying QP to RTS errno says %s", strerror(errno));
return OMPI_ERROR;
}
return OMPI_SUCCESS;

Просмотреть файл

@ -26,7 +26,7 @@
#include "btl_openib.h"
#include <errno.h>
#include <string.h>
#include "mca/btl/base/btl_base_error.h"
extern int errno;
#if defined(c_plusplus) || defined(__cplusplus)
@ -133,87 +133,66 @@ void mca_btl_openib_post_recv(void);
void mca_btl_openib_progress_send_frags(mca_btl_openib_endpoint_t*);
static inline int mca_btl_openib_endpoint_post_rr_sub(int cnt,
mca_btl_openib_endpoint_t* endpoint,
ompi_free_list_t* frag_list,
uint32_t* rr_posted,
struct ibv_qp* qp
)
{
int rc, i;
opal_list_item_t* item;
mca_btl_openib_frag_t* frag;
mca_btl_openib_module_t *openib_btl = endpoint->endpoint_btl;
struct ibv_recv_wr* bad_wr;
struct ibv_recv_wr* rr_desc_post = openib_btl->rr_desc_post;
/* prepare frags and post receive requests, given, this is ugly,
* if openib doesn't plan on supporting a post_list method than
* this should be changed to simply loop through and post receives
* without bothering with the rr_desc_post array as it is not needed
*/
for(i = 0; i < cnt; i++) {
OMPI_FREE_LIST_WAIT(frag_list, item, rc);
frag = (mca_btl_openib_frag_t*) item;
frag->endpoint = endpoint;
frag->sg_entry.length = frag->size +
((unsigned char*) frag->segment.seg_addr.pval-
(unsigned char*) frag->hdr);
rr_desc_post[i] = frag->rr_desc;
}
for(i=0; i< cnt; i++){
if(ibv_post_recv(qp,
&rr_desc_post[i],
&bad_wr)) {
opal_output(0, "%s: error posting receive errno says %s\n", __func__, strerror(errno));
return OMPI_ERROR;
}
}
OPAL_THREAD_ADD32(rr_posted, cnt);
return OMPI_SUCCESS;
#define MCA_BTL_OPENIB_ENDPOINT_POST_RR_HIGH(post_rr_high_endpoint, \
post_rr_high_additional) \
{ \
mca_btl_openib_module_t * post_rr_high_openib_btl = post_rr_high_endpoint->endpoint_btl; \
OPAL_THREAD_LOCK(&post_rr_high_openib_btl->ib_lock); \
if(post_rr_high_endpoint->rr_posted_high <= mca_btl_openib_component.ib_rr_buf_min+post_rr_high_additional && \
post_rr_high_endpoint->rr_posted_high < mca_btl_openib_component.ib_rr_buf_max){ \
MCA_BTL_OPENIB_ENDPOINT_POST_RR_SUB(mca_btl_openib_component.ib_rr_buf_max - \
post_rr_high_endpoint->rr_posted_high, \
post_rr_high_endpoint, \
&post_rr_high_openib_btl->recv_free_eager, \
&post_rr_high_endpoint->rr_posted_high, \
post_rr_high_endpoint->lcl_qp_high); \
} \
OPAL_THREAD_UNLOCK(&post_rr_high_openib_btl->ib_lock); \
}
static inline int mca_btl_openib_endpoint_post_rr( mca_btl_openib_endpoint_t * endpoint, int additional){
mca_btl_openib_module_t * openib_btl = endpoint->endpoint_btl;
int rc;
OPAL_THREAD_LOCK(&openib_btl->ib_lock);
#define MCA_BTL_OPENIB_ENDPOINT_POST_RR_LOW(post_rr_low_endpoint, \
post_rr_low_additional) { \
mca_btl_openib_module_t * post_rr_low_openib_btl = post_rr_low_endpoint->endpoint_btl; \
OPAL_THREAD_LOCK(&post_rr_low_openib_btl->ib_lock); \
if(post_rr_low_endpoint->rr_posted_low <= mca_btl_openib_component.ib_rr_buf_min+post_rr_low_additional && \
post_rr_low_endpoint->rr_posted_low < mca_btl_openib_component.ib_rr_buf_max){ \
MCA_BTL_OPENIB_ENDPOINT_POST_RR_SUB(mca_btl_openib_component.ib_rr_buf_max - \
post_rr_low_endpoint->rr_posted_low, \
post_rr_low_endpoint, \
&post_rr_low_openib_btl->recv_free_max, \
&post_rr_low_endpoint->rr_posted_low, \
post_rr_low_endpoint->lcl_qp_low \
); } \
OPAL_THREAD_UNLOCK(&post_rr_low_openib_btl->ib_lock); \
}
if(endpoint->rr_posted_high <= mca_btl_openib_component.ib_rr_buf_min+additional && endpoint->rr_posted_high < mca_btl_openib_component.ib_rr_buf_max){
rc = mca_btl_openib_endpoint_post_rr_sub(mca_btl_openib_component.ib_rr_buf_max - endpoint->rr_posted_high,
endpoint,
&openib_btl->recv_free_eager,
&endpoint->rr_posted_high,
endpoint->lcl_qp_high
);
if(rc != OMPI_SUCCESS){
OPAL_THREAD_UNLOCK(&openib_btl->ib_lock);
return rc;
}
}
if(endpoint->rr_posted_low <= mca_btl_openib_component.ib_rr_buf_min+additional && endpoint->rr_posted_low < mca_btl_openib_component.ib_rr_buf_max){
rc = mca_btl_openib_endpoint_post_rr_sub(mca_btl_openib_component.ib_rr_buf_max - endpoint->rr_posted_low,
endpoint,
&openib_btl->recv_free_max,
&endpoint->rr_posted_low,
endpoint->lcl_qp_low
);
if(rc != OMPI_SUCCESS) {
OPAL_THREAD_UNLOCK(&openib_btl->ib_lock);
return rc;
}
}
OPAL_THREAD_UNLOCK(&openib_btl->ib_lock);
return OMPI_SUCCESS;
#define MCA_BTL_OPENIB_ENDPOINT_POST_RR_SUB(post_rr_sub_cnt, \
post_rr_sub_endpoint, \
post_rr_sub_frag_list, \
post_rr_sub_rr_posted, \
post_rr_sub_qp ) \
{\
uint32_t post_rr_sub_i; \
int post_rr_sub_rc; \
opal_list_item_t* post_rr_sub_item; \
mca_btl_openib_frag_t* post_rr_sub_frag; \
struct ibv_recv_wr* post_rr_sub_bad_wr; \
for(post_rr_sub_i = 0; post_rr_sub_i < post_rr_sub_cnt; post_rr_sub_i++) { \
OMPI_FREE_LIST_WAIT(post_rr_sub_frag_list, post_rr_sub_item, post_rr_sub_rc); \
post_rr_sub_frag = (mca_btl_openib_frag_t*) post_rr_sub_item; \
post_rr_sub_frag->endpoint = post_rr_sub_endpoint; \
post_rr_sub_frag->sg_entry.length = post_rr_sub_frag->size + \
((unsigned char*) post_rr_sub_frag->segment.seg_addr.pval- \
(unsigned char*) post_rr_sub_frag->hdr); \
if(ibv_post_recv(post_rr_sub_qp, \
&post_rr_sub_frag->wr_desc.rr_desc, \
&post_rr_sub_bad_wr)) { \
BTL_ERROR("error posting receive errno says %s\n", strerror(errno)); \
return OMPI_ERROR; \
}\
}\
OPAL_THREAD_ADD32(post_rr_sub_rr_posted, post_rr_sub_cnt); \
}
#define DUMP_ENDPOINT(endpoint_ptr) { \

Просмотреть файл

@ -1,37 +0,0 @@
/*
* Copyright (c) 2004-2005 The Trustees of Indiana University.
* All rights reserved.
* Copyright (c) 2004-2005 The Trustees of the University of Tennessee.
* All rights reserved.
* Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
* University of Stuttgart. All rights reserved.
* Copyright (c) 2004-2005 The Regents of the University of California.
* All rights reserved.
* $COPYRIGHT$
*
* Additional copyrights may follow
*
* $HEADER$
*/
#ifndef MCA_BTL_IB_ERROR_H
#define MCA_BTL_IB_ERROR_H
#include <infiniband/verbs.h>
/*
*
*
*/
/* Debug Print */
#if 0
#define DEBUG_OUT(fmt, args...) { \
opal_output(0, "[%s:%d:%s] " fmt, __FILE__, __LINE__, __func__, \
##args); \
}
#else
#define DEBUG_OUT(fmt, args...)
#endif
#endif

Просмотреть файл

@ -1,3 +1,19 @@
/*
* Copyright (c) 2004-2005 The Trustees of Indiana University.
* All rights reserved.
* Copyright (c) 2004-2005 The Trustees of the University of Tennessee.
* All rights reserved.
* Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
* University of Stuttgart. All rights reserved.
* Copyright (c) 2004-2005 The Regents of the University of California.
* All rights reserved.
* $COPYRIGHT$
*
* Additional copyrights may follow
*
* $HEADER$
*/
#include "btl_openib_frag.h"
#include "mca/mpool/openib/mpool_openib.h"
@ -45,12 +61,12 @@ static void mca_btl_openib_send_frag_common_constructor(mca_btl_openib_frag_t* f
frag->base.des_dst = NULL;
frag->base.des_dst_cnt = 0;
frag->sr_desc.wr_id = (uint64_t) frag;
frag->sr_desc.sg_list = &frag->sg_entry;
frag->sr_desc.num_sge = 1;
frag->sr_desc.opcode = IBV_WR_SEND;
frag->sr_desc.send_flags = IBV_SEND_SIGNALED;
frag->sr_desc.next = NULL;
frag->wr_desc.sr_desc.wr_id = (uint64_t) frag;
frag->wr_desc.sr_desc.sg_list = &frag->sg_entry;
frag->wr_desc.sr_desc.num_sge = 1;
frag->wr_desc.sr_desc.opcode = IBV_WR_SEND;
frag->wr_desc.sr_desc.send_flags = IBV_SEND_SIGNALED;
frag->wr_desc.sr_desc.next = NULL;
}
static void mca_btl_openib_recv_frag_common_constructor(mca_btl_openib_frag_t* frag)
@ -62,10 +78,10 @@ static void mca_btl_openib_recv_frag_common_constructor(mca_btl_openib_frag_t* f
frag->base.des_src = NULL;
frag->base.des_src_cnt = 0;
frag->rr_desc.wr_id = (uint64_t) frag;
frag->rr_desc.sg_list = &frag->sg_entry;
frag->rr_desc.num_sge = 1;
frag->rr_desc.next = NULL;
frag->wr_desc.rr_desc.wr_id = (uint64_t) frag;
frag->wr_desc.rr_desc.sg_list = &frag->sg_entry;
frag->wr_desc.rr_desc.num_sge = 1;
frag->wr_desc.rr_desc.next = NULL;
}
static void mca_btl_openib_send_frag_eager_constructor(mca_btl_openib_frag_t* frag)

Просмотреть файл

@ -51,11 +51,10 @@ struct mca_btl_openib_frag_t {
mca_btl_openib_frag_type_t type;
size_t size;
int rc;
union{
struct ibv_recv_wr rr_desc;
struct ibv_send_wr sr_desc;
};
} wr_desc;
struct ibv_sge sg_entry;
struct ibv_mr *mr;
mca_btl_openib_header_t *hdr;

Просмотреть файл

@ -75,16 +75,12 @@ static mca_btl_openib_proc_t* mca_btl_openib_proc_lookup_ompi(ompi_proc_t* ompi_
ib_proc != (mca_btl_openib_proc_t*)
opal_list_get_end(&mca_btl_openib_component.ib_procs);
ib_proc = (mca_btl_openib_proc_t*)opal_list_get_next(ib_proc)) {
if(ib_proc->proc_ompi == ompi_proc) {
OPAL_THREAD_UNLOCK(&mca_btl_openib_component.ib_lock);
return ib_proc;
}
}
OPAL_THREAD_UNLOCK(&mca_btl_openib_component.ib_lock);
return NULL;
}
@ -105,19 +101,15 @@ mca_btl_openib_proc_t* mca_btl_openib_proc_create(ompi_proc_t* ompi_proc)
module_proc = mca_btl_openib_proc_lookup_ompi(ompi_proc);
if(module_proc != NULL) {
/* Gotcha! */
return module_proc;
}
/* Oops! First time, gotta create a new IB proc
* out of the ompi_proc ... */
module_proc = OBJ_NEW(mca_btl_openib_proc_t);
/* Initialize number of peer */
module_proc->proc_endpoint_count = 0;
module_proc->proc_ompi = ompi_proc;
/* build a unique identifier (of arbitrary
@ -156,6 +148,5 @@ int mca_btl_openib_proc_insert(mca_btl_openib_proc_t* module_proc,
/* insert into endpoint array */
module_endpoint->endpoint_proc = module_proc;
module_proc->proc_endpoints[module_proc->proc_endpoint_count++] = module_endpoint;
return OMPI_SUCCESS;
}