ports/openmpi
ports/openmpi
1
1
Форкнуть 0
Код Релизы Активность

Rewrote qp creation/init

Просмотр исходного кода 
Rewrote cq creation/init 
Removed all static constants for vapi init and added mca params 
Finished rewrite of component and module init 

This commit was SVN r6014.
Этот коммит содержится в:
Galen Shipman 2005-06-09 16:36:51 +00:00
родитель c930025531
Коммит a8c48ac12c
15 изменённых файлов: 336 добавлений и 884 удалений
Показать статистику Скачать Patch файл Скачать Diff файл Показать все файлы Свернуть все файлы

5
src/mca/bmi/ib/Makefile.am
Просмотреть файл

@ -21,17 +21,14 @@ include $(top_ompi_srcdir)/config/Makefile.options
libmca_bmi_ib_la_SOURCES = \
bmi_ib.c \
bmi_ib.h \
bmi_ib_addr.h \
bmi_ib_component.c \
bmi_ib_endpoint.c \
bmi_ib_endpoint.h \
bmi_ib_frag.c \
bmi_ib_frag.h \
bmi_ib_priv.c \
bmi_ib_priv.h \
bmi_ib_proc.c \
bmi_ib_proc.h \
bmi_ib_vapi.h
bmi_ib_error.h
# Make the output library in this directory, and name it either
# mca_<type>_<name>.la (for DSO builds) or libmca_<type>_<name>.la

123
src/mca/bmi/ib/bmi_ib.c
Просмотреть файл

@ -3,8 +3,6 @@
* All rights reserved.
* Copyright (c) 2004-2005 The Trustees of the University of Tennessee.
* All rights reserved.
* Copyright (c) 2004 The Ohio State University.
* All rights reserved.
* Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
* University of Stuttgart. All rights reserved.
* Copyright (c) 2004-2005 The Regents of the University of California.
@ -25,10 +23,8 @@
#include "bmi_ib.h"
#include "bmi_ib_frag.h"
#include "bmi_ib_addr.h"
#include "bmi_ib_proc.h"
#include "bmi_ib_endpoint.h"
#include "bmi_ib_priv.h"
#include "datatype/convertor.h"
#include "mca/common/vapi/vapi_mem_reg.h"
@ -119,7 +115,7 @@ int mca_bmi_ib_del_procs(struct mca_bmi_base_module_t* bmi,
struct mca_bmi_base_endpoint_t ** peers)
{
/* Stub */
D_PRINT("Stub\n");
DEBUG_OUT("Stub\n");
return OMPI_SUCCESS;
}
@ -215,8 +211,8 @@ mca_bmi_base_descriptor_t* mca_bmi_ib_prepare_src(
mca_bmi_ib_module_t* ib_bmi;
mca_bmi_ib_frag_t* frag;
struct iovec iov;
uint32_t iov_count = 1;
uint32_t max_data = *size;
int32_t out_size = 0;
size_t max_data = *size;
int32_t free_after;
int rc;
void* user_out;
@ -234,7 +230,7 @@ mca_bmi_base_descriptor_t* mca_bmi_ib_prepare_src(
iov.iov_len = max_data;
iov.iov_base = frag->segment.seg_addr.pval + reserve;
rc = ompi_convertor_pack(convertor, &iov, &iov_count, &max_data, &free_after);
rc = ompi_convertor_pack(convertor, &iov, &out_size, &max_data, &free_after);
if( rc < 0 ) {
MCA_BMI_IB_FRAG_RETURN_EAGER(bmi, frag);
return NULL;
@ -255,7 +251,7 @@ mca_bmi_base_descriptor_t* mca_bmi_ib_prepare_src(
iov.iov_len = max_data;
iov.iov_base = frag->segment.seg_addr.pval + reserve;
rc = ompi_convertor_pack(convertor, &iov, &iov_count, &max_data, &free_after);
rc = ompi_convertor_pack(convertor, &iov, &out_size, &max_data, &free_after);
if( rc < 0 ) {
MCA_BMI_IB_FRAG_RETURN_MAX(bmi, frag);
return NULL;
@ -283,13 +279,13 @@ mca_bmi_base_descriptor_t* mca_bmi_ib_prepare_src(
mr_in.type = VAPI_MR;
frag = (mca_bmi_ib_send_frag_frag_t*) ib_bmi->ib_pool->mpool_alloc(ib_bmi->ib_pool, sizeof(frag) + sizeof(mca_bmi_ib_header_t) + size ,0, &user_out);
frag = (mca_bmi_ib_send_frag_frag_t*) ib_bmi->ib_pool->mpool_alloc(ib_bmi->ib_pool, sizeof(frag) + sizeof(mca_bmi_ib_header_t) + *size ,0, &user_out);
frag->base.super.user_data = user_out;
OBJ_CONSTRUCT(frag, mca_bmi_ib_send_frag_frag_t);
iov.iov_len = max_data;
iov.iov_base = NULL;
ompi_convertor_pack(convertor, &iov, &iov_count, &max_data, &free_after);
ompi_convertor_pack(convertor, &iov, &out_size, &max_data, &free_after);
frag->segment.seg_len = max_data;
frag->segment.seg_addr.pval = iov.iov_base;
@ -378,44 +374,6 @@ int mca_bmi_ib_put( mca_bmi_base_module_t* bmi,
static int mca_bmi_ib_alloc_pd(VAPI_hca_hndl_t nic,
VAPI_pd_hndl_t* ptag)
{
VAPI_ret_t ret;
ret = VAPI_alloc_pd(nic, ptag);
if(ret != VAPI_OK) {
MCA_BMI_IB_VAPI_RET(ret, "VAPI_alloc_pd");
return OMPI_ERROR;
}
return OMPI_SUCCESS;
}
static int mca_bmi_ib_create_cq(VAPI_hca_hndl_t nic,
VAPI_cq_hndl_t* cq_hndl)
{
uint32_t act_num_cqe = 0;
VAPI_ret_t ret;
ret = VAPI_create_cq(nic, DEFAULT_CQ_SIZE,
cq_hndl, &act_num_cqe);
if( (VAPI_OK != ret) || (0 == act_num_cqe)) {
MCA_BMI_IB_VAPI_RET(ret, "VAPI_create_cq");
return OMPI_ERROR;
}
return OMPI_SUCCESS;
}
/*
* Asynchronous event handler to detect unforseen
* events. Usually, such events are catastrophic.
@ -435,7 +393,7 @@ static void async_event_handler(VAPI_hca_hndl_t hca_hndl,
case VAPI_SEND_QUEUE_DRAINED:
case VAPI_PORT_ACTIVE:
{
D_PRINT("Got an asynchronous event: %s\n",
DEBUG_OUT("Got an asynchronous event: %s\n",
VAPI_event_record_sym(event_p->type));
break;
}
@ -462,60 +420,41 @@ static void async_event_handler(VAPI_hca_hndl_t hca_hndl,
}
static int mca_bmi_ib_set_async_handler(VAPI_hca_hndl_t nic,
EVAPI_async_handler_hndl_t *async_handler)
{
VAPI_ret_t ret;
ret = EVAPI_set_async_event_handler(nic,
async_event_handler, 0, async_handler);
if(VAPI_OK != ret) {
MCA_BMI_IB_VAPI_RET(ret, "EVAPI_set_async_event_handler");
return OMPI_ERROR;
}
return OMPI_SUCCESS;
}
int mca_bmi_ib_module_init(mca_bmi_ib_module_t *ib_bmi)
{
/* Get HCA handle */
/* if(mca_bmi_ib_get_hca_hndl(ib_bmi->hca_id, &ib_bmi->nic) */
/* != OMPI_SUCCESS) { */
/* return OMPI_ERROR; */
/* } */
/* Allocate a protection domain for this NIC */
if(mca_bmi_ib_alloc_pd(ib_bmi->nic, &ib_bmi->ptag)
!= OMPI_SUCCESS) {
/* Allocate Protection Domain */
VAPI_ret_t ret;
uint32_t cqe_cnt = 0;
ret = VAPI_alloc_pd(ib_bmi->nic, &ib_bmi->ptag);
if(ret != VAPI_OK) {
MCA_BMI_IB_VAPI_ERROR(ret, "VAPI_alloc_pd");
return OMPI_ERROR;
}
ret = VAPI_create_cq(ib_bmi->nic, ib_bmi->ib_cq_size,
&ib_bmi->cq_hndl, &cqe_cnt);
/* Get the properties of the HCA PORT,
* LID etc. are part of the properties */
/* if(mca_bmi_ib_query_hca_port_prop(ib_bmi->nic, &ib_bmi->port) */
/* != OMPI_SUCCESS) { */
/* return OMPI_ERROR; */
/* } */
/* Create Completion Q */
/* We use a single completion Q for sends & recvs
* This saves us overhead of polling 2 separate Qs */
if(mca_bmi_ib_create_cq(ib_bmi->nic, &ib_bmi->cq_hndl)
!= OMPI_SUCCESS) {
if( VAPI_OK != ret) {
MCA_BMI_IB_VAPI_ERROR(ret, "VAPI_create_cq");
return OMPI_ERROR;
}
if(cqe_cnt <= 0) {
ompi_output(0, "%s: error creating completion queue ", __func__);
return OMPI_ERROR;
}
/* Attach asynchronous handler */
if(mca_bmi_ib_set_async_handler(ib_bmi->nic,
&ib_bmi->async_handler)
!= OMPI_SUCCESS) {
ret = EVAPI_set_async_event_handler(ib_bmi->nic,
async_event_handler, 0, &ib_bmi->async_handler);
if(VAPI_OK != ret) {
MCA_BMI_IB_VAPI_ERROR(ret, "EVAPI_set_async_event_handler");
return OMPI_ERROR;
}
return OMPI_SUCCESS;
}

23
src/mca/bmi/ib/bmi_ib.h
Просмотреть файл

@ -4,8 +4,6 @@
* All rights reserved.
* Copyright (c) 2004-2005 The Trustees of the University of Tennessee.
* All rights reserved.
* Copyright (c) 2004 The Ohio State University.
* All rights reserved.
* Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
* University of Stuttgart. All rights reserved.
* Copyright (c) 2004-2005 The Regents of the University of California.
@ -34,8 +32,7 @@
#include "mca/bmi/bmi.h"
#include "util/output.h"
#include "mca/mpool/mpool.h"
#include "bmi_ib_vapi.h"
#include "bmi_ib_error.h"
/* InfiniBand VAPI includes */
#include "mca/bmi/bmi.h"
@ -134,7 +131,23 @@ struct mca_bmi_ib_module_t {
VAPI_rr_desc_t* rr_desc_post;
/**< an array to allow posting of rr in one swoop */
size_t ib_inline_max; /**< max size of inline send*/
size_t ib_pin_min; /** < min size to pin memory*/
size_t ib_pin_min; /**< min size to pin memory*/
uint32_t ib_cq_size; /**< Max outstanding CQE on the CQ */
uint32_t ib_wq_size; /**< Max outstanding WR on the WQ */
uint32_t ib_sg_list_size; /**< Max scatter/gather descriptor entries on the WQ*/
uint32_t ib_pkey_ix;
uint32_t ib_psn;
uint32_t ib_qp_ous_rd_atom;
uint32_t ib_mtu;
uint32_t ib_min_rnr_timer;
uint32_t ib_timeout;
uint32_t ib_retry_count;
uint32_t ib_rnr_retry;
uint32_t ib_max_rdma_dst_ops;
uint32_t ib_service_level;
uint32_t ib_static_rate;
uint32_t ib_src_path_bits;
}; typedef struct mca_bmi_ib_module_t mca_bmi_ib_module_t;

24
src/mca/bmi/ib/bmi_ib_addr.h
Просмотреть файл

@ -1,24 +0,0 @@
/*
* Copyright (c) 2004-2005 The Trustees of Indiana University.
* All rights reserved.
* Copyright (c) 2004-2005 The Trustees of the University of Tennessee.
* All rights reserved.
* Copyright (c) 2004 The Ohio State University.
* All rights reserved.
* Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
* University of Stuttgart. All rights reserved.
* Copyright (c) 2004-2005 The Regents of the University of California.
* All rights reserved.
* $COPYRIGHT$
*
* Additional copyrights may follow
*
* $HEADER$
*/
#ifndef MCA_BMI_IB_ADDR_H
#define MCA_BMI_IB_ADDR_H
#include "bmi_ib.h"
#endif

76
src/mca/bmi/ib/bmi_ib_component.c
Просмотреть файл

@ -137,6 +137,60 @@ int mca_bmi_ib_component_open(void)
mca_bmi_ib_module.super.bmi_max_send_size =
mca_bmi_ib_param_register_int ("max_send_size", 128*1024);
mca_bmi_ib_module.ib_cq_size =
mca_bmi_ib_param_register_int("ib_cq_size",
40000);
mca_bmi_ib_module.ib_wq_size =
mca_bmi_ib_param_register_int("ib_wq_size",
10000);
mca_bmi_ib_module.ib_sg_list_size =
mca_bmi_ib_param_register_int("ib_sg_list_size",
1);
mca_bmi_ib_module.ib_pkey_ix =
mca_bmi_ib_param_register_int("ib_pkey_ix",
0);
mca_bmi_ib_module.ib_psn =
mca_bmi_ib_param_register_int("ib_psn",
0);
mca_bmi_ib_module.ib_qp_ous_rd_atom =
mca_bmi_ib_param_register_int("ib_qp_ous_rd_atom",
1);
mca_bmi_ib_module.ib_mtu =
mca_bmi_ib_param_register_int("ib_mtu",
MTU1024);
mca_bmi_ib_module.ib_min_rnr_timer =
mca_bmi_ib_param_register_int("ib_min_rnr_timer",
5);
mca_bmi_ib_module.ib_timeout =
mca_bmi_ib_param_register_int("ib_timeout",
10);
mca_bmi_ib_module.ib_retry_count =
mca_bmi_ib_param_register_int("ib_retry_count",
7);
mca_bmi_ib_module.ib_rnr_retry =
mca_bmi_ib_param_register_int("ib_rnr_retry",
7);
mca_bmi_ib_module.ib_max_rdma_dst_ops =
mca_bmi_ib_param_register_int("ib_max_rdma_dst_ops",
16);
mca_bmi_ib_module.ib_service_level =
mca_bmi_ib_param_register_int("ib_service_level",
0);
mca_bmi_ib_module.ib_static_rate =
mca_bmi_ib_param_register_int("ib_static_rate",
0);
mca_bmi_ib_module.ib_src_path_bits =
mca_bmi_ib_param_register_int("ib_src_path_bits",
0);
mca_bmi_ib_component.max_send_size = mca_bmi_ib_module.super.bmi_max_send_size;
mca_bmi_ib_component.eager_limit = mca_bmi_ib_module.super.bmi_eager_limit;
@ -173,7 +227,7 @@ mca_bmi_base_module_t** mca_bmi_ib_component_init(int *num_bmi_modules,
VAPI_hca_port_t hca_port;
uint32_t num_hcas;
mca_bmi_base_module_t** bmis;
int i,j,k, length;
int i,j, length;
mca_common_vapi_hca_pd_t hca_pd;
ompi_list_t bmi_list;
mca_bmi_ib_module_t * ib_bmi;
@ -182,7 +236,7 @@ mca_bmi_base_module_t** mca_bmi_ib_component_init(int *num_bmi_modules,
/* initialization */
*num_bmi_modules = 0;
/* Determine the number of hca's available on the host */
/* Determine the number of hca's available on the host */
vapi_ret=EVAPI_list_hcas(0, &num_hcas, NULL);
if( VAPI_EAGAIN != vapi_ret || 0 == num_hcas ) {
ompi_output(0,"No hca's found on this host \n");
@ -213,14 +267,14 @@ mca_bmi_base_module_t** mca_bmi_ib_component_init(int *num_bmi_modules,
vapi_ret = EVAPI_get_hca_hndl(hca_ids[i], &hca_hndl);
if(VAPI_OK != vapi_ret) {
ompi_output(0, "%s:error getting hca handle\n", __func__);
return OMPI_ERROR;
return NULL;
}
vapi_ret = VAPI_query_hca_cap(hca_hndl, &hca_vendor, &hca_cap);
if(VAPI_OK != vapi_ret) {
ompi_output(0, "%s:error getting hca properties\n", __func__);
return OMPI_ERROR;
return NULL;
}
@ -229,7 +283,7 @@ mca_bmi_base_module_t** mca_bmi_ib_component_init(int *num_bmi_modules,
vapi_ret = VAPI_query_hca_port_prop(hca_hndl, (IB_port_t) j, &hca_port);
if(VAPI_OK != vapi_ret) {
ompi_output(0, "%s:error getting hca port properties\n", __func__);
return OMPI_ERROR;
return NULL;
}
if( PORT_ACTIVE == hca_port.state ){
@ -353,7 +407,7 @@ mca_bmi_base_module_t** mca_bmi_ib_component_init(int *num_bmi_modules,
length = sizeof(mca_bmi_ib_frag_t) +
sizeof(mca_bmi_ib_header_t);
ib_bmi->super.bmi_max_send_size;
ompi_free_list_init(&ib_bmi->send_free_frag,
@ -434,13 +488,9 @@ int mca_bmi_ib_component_progress()
mca_bmi_ib_frag_t* frag;
/* Poll for completions */
for(i = 0; i < mca_bmi_ib_component.ib_num_bmis; i++) {
mca_bmi_ib_module_t* ib_bmi = &mca_bmi_ib_component.ib_bmis[i];
int comp_type = IB_COMP_NOTHING;
void* comp_addr;
VAPI_ret_t ret;
VAPI_wc_desc_t comp;
mca_bmi_ib_module_t* ib_bmi = &mca_bmi_ib_component.ib_bmis[i];
ret = VAPI_poll_cq(ib_bmi->nic, ib_bmi->cq_hndl, &comp);
@ -464,14 +514,14 @@ int mca_bmi_ib_component_progress()
case VAPI_CQE_RQ_SEND_DATA:
D_PRINT(0, "%s:%d ib recv under redesign\n", __FILE__, __LINE__);
DEBUG_OUT(0, "%s:%d ib recv under redesign\n", __FILE__, __LINE__);
frag = (mca_bmi_ib_frag_t*) comp.id;
frag->segment.seg_len = comp.byte_len-sizeof(mca_bmi_ib_header_t);
/* advance the segment address past the header and subtract from the length..*/
ib_bmi->ib_reg[frag->hdr->tag].cbfunc(&ib_bmi->super, frag->hdr->tag, &frag->base, ib_bmi->ib_reg[frag->hdr->tag].cbdata);
OMPI_FREE_LIST_RETURN(&ib_bmi->recv_free, (ompi_free_list_item_t*) frag);
OMPI_FREE_LIST_RETURN(&(ib_bmi->recv_free), (ompi_list_item_t*) frag);
if(OMPI_THREAD_ADD32(&ib_bmi->rr_posted, -1) <= mca_bmi_ib_component.ib_rr_buf_min)
mca_bmi_ib_endpoint_post_rr(mca_bmi_ib_component.ib_rr_buf_max - ib_bmi->rr_posted,

220
src/mca/bmi/ib/bmi_ib_endpoint.c
Просмотреть файл

@ -3,8 +3,6 @@
* All rights reserved.
* Copyright (c) 2004-2005 The Trustees of the University of Tennessee.
* All rights reserved.
* Copyright (c) 2004 The Ohio State University.
* All rights reserved.
* Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
* University of Stuttgart. All rights reserved.
* Copyright (c) 2004-2005 The Regents of the University of California.
@ -29,14 +27,16 @@
#include "dps/dps.h"
#include "bmi_ib.h"
#include "bmi_ib_endpoint.h"
#include "bmi_ib_addr.h"
#include "bmi_ib_proc.h"
#include "bmi_ib_priv.h"
#include "bmi_ib_frag.h"
#include "class/ompi_free_list.h"
static void mca_bmi_ib_endpoint_construct(mca_bmi_base_endpoint_t* endpoint);
static void mca_bmi_ib_endpoint_destruct(mca_bmi_base_endpoint_t* endpoint);
int mca_bmi_ib_endpoint_create_qp(mca_bmi_ib_endpoint_t* endpoint,
int transport_type);
int mca_bmi_ib_endpoint_qp_init_query(mca_bmi_ib_endpoint_t* endpoint);
static inline int mca_bmi_ib_endpoint_post_send(mca_bmi_ib_module_t* ib_bmi, mca_bmi_ib_endpoint_t * endpoint, mca_bmi_ib_frag_t * frag)
@ -133,7 +133,7 @@ static int mca_bmi_ib_endpoint_send_connect_req(mca_bmi_base_endpoint_t* endpoin
mca_bmi_ib_endpoint_send_cb, NULL);
D_PRINT("Sending QP num = %d, LID = %d",
DEBUG_OUT("Sending QP num = %d, LID = %d",
endpoint->lcl_qp_prop.qp_num,
endpoint->endpoint_bmi->port.lid);
@ -200,7 +200,7 @@ static int mca_bmi_ib_endpoint_set_remote_info(mca_bmi_base_endpoint_t* endpoint
ORTE_ERROR_LOG(rc);
return rc;
}
D_PRINT("Received QP num = %d, LID = %d",
DEBUG_OUT("Received QP num = %d, LID = %d",
endpoint->rem_qp_num,
endpoint->rem_lid);
return ORTE_SUCCESS;
@ -223,24 +223,18 @@ static int mca_bmi_ib_endpoint_init(mca_bmi_ib_endpoint_t *endpoint)
static int mca_bmi_ib_endpoint_start_connect(mca_bmi_base_endpoint_t* endpoint)
{
mca_bmi_ib_module_t* ib_bmi = endpoint->endpoint_bmi;
int rc;
/* Create the Queue Pair */
if(OMPI_SUCCESS != (rc = mca_bmi_ib_create_qp(ib_bmi->nic,
ib_bmi->ptag,
ib_bmi->cq_hndl,
ib_bmi->cq_hndl,
&endpoint->lcl_qp_hndl,
&endpoint->lcl_qp_prop,
VAPI_TS_RC))) {
if(OMPI_SUCCESS != (rc = mca_bmi_ib_endpoint_create_qp(endpoint,
VAPI_TS_RC))) {
ompi_output(0, "[%lu,%lu,%lu] %s:%d errcode %d\n",
ORTE_NAME_ARGS(orte_process_info.my_name), __FILE__,__LINE__,rc);
return rc;
}
D_PRINT("Initialized QP num = %d, LID = %d",
DEBUG_OUT("Initialized QP num = %d, LID = %d",
endpoint->lcl_qp_prop.qp_num,
ib_bmi->port.lid);
@ -260,23 +254,17 @@ static int mca_bmi_ib_endpoint_start_connect(mca_bmi_base_endpoint_t* endpoint)
*/
static int mca_bmi_ib_endpoint_reply_start_connect(mca_bmi_ib_endpoint_t *endpoint, orte_buffer_t* buffer)
{
mca_bmi_ib_module_t* ib_bmi = endpoint->endpoint_bmi;
int rc;
/* Create the Queue Pair */
if(OMPI_SUCCESS != (rc = mca_bmi_ib_create_qp(ib_bmi->nic,
ib_bmi->ptag,
ib_bmi->cq_hndl,
ib_bmi->cq_hndl,
&endpoint->lcl_qp_hndl,
&endpoint->lcl_qp_prop,
VAPI_TS_RC))) {
if(OMPI_SUCCESS != (rc = mca_bmi_ib_endpoint_create_qp(endpoint,
VAPI_TS_RC))) {
ompi_output(0, "[%lu,%lu,%lu] %s:%d errcode %d\n",
ORTE_NAME_ARGS(orte_process_info.my_name), __FILE__,__LINE__,rc);
return rc;
}
D_PRINT("Initialized QP num = %d, LID = %d",
DEBUG_OUT("Initialized QP num = %d, LID = %d",
endpoint->lcl_qp_prop.qp_num,
ib_bmi->port.lid);
@ -408,7 +396,7 @@ static void mca_bmi_ib_endpoint_recv(
void mca_bmi_ib_post_recv()
{
D_PRINT("");
DEBUG_OUT("");
orte_rml.recv_buffer_nb(
ORTE_RML_NAME_ANY,
@ -437,7 +425,7 @@ int mca_bmi_ib_endpoint_send(
switch(endpoint->endpoint_state) {
case MCA_BMI_IB_CONNECTING:
D_PRINT("Queing because state is connecting");
DEBUG_OUT("Queing because state is connecting");
ompi_list_append(&endpoint->pending_send_frags,
(ompi_list_item_t *)frag);
@ -447,7 +435,7 @@ int mca_bmi_ib_endpoint_send(
case MCA_BMI_IB_CONNECT_ACK:
D_PRINT("Queuing because waiting for ack");
DEBUG_OUT("Queuing because waiting for ack");
ompi_list_append(&endpoint->pending_send_frags,
(ompi_list_item_t *)frag);
@ -457,7 +445,7 @@ int mca_bmi_ib_endpoint_send(
case MCA_BMI_IB_CLOSED:
D_PRINT("Connection to endpoint closed ... connecting ...");
DEBUG_OUT("Connection to endpoint closed ... connecting ...");
ompi_list_append(&endpoint->pending_send_frags,
(ompi_list_item_t *)frag);
@ -476,7 +464,7 @@ int mca_bmi_ib_endpoint_send(
ib_bmi = endpoint->endpoint_bmi;
A_PRINT("Send to : %d, len : %d, frag : %p",
DEBUG_OUT("Send to : %d, len : %d, frag : %p",
endpoint->endpoint_proc->proc_guid.vpid,
frag->ib_buf.desc.sg_entry.len,
frag);
@ -543,26 +531,176 @@ int mca_bmi_ib_endpoint_connect(
mca_bmi_ib_endpoint_t *endpoint)
{
int rc;
mca_bmi_ib_module_t *ib_bmi = endpoint->endpoint_bmi;
/* Connection establishment RC */
rc = mca_bmi_ib_qp_init(ib_bmi->nic,
endpoint->lcl_qp_hndl,
endpoint->rem_qp_num,
endpoint->rem_lid,
ib_bmi->port_id);
rc = mca_bmi_ib_endpoint_qp_init_query(endpoint);
if(rc != OMPI_SUCCESS) {
return rc;
}
rc = mca_bmi_ib_qp_query(ib_bmi,
endpoint->lcl_qp_hndl,
endpoint->rem_qp_num);
if(rc != OMPI_SUCCESS) {
return rc;
}
mca_bmi_ib_endpoint_post_rr(mca_bmi_ib_component.ib_rr_buf_max, endpoint);
return OMPI_SUCCESS;
}
int mca_bmi_ib_endpoint_create_qp(mca_bmi_ib_endpoint_t * endpoint,
int transport_type)
{
VAPI_ret_t ret;
VAPI_qp_init_attr_t qp_init_attr;
mca_bmi_ib_module_t* ib_bmi = endpoint->endpoint_bmi;
VAPI_hca_hndl_t nic = ib_bmi->nic;
VAPI_pd_hndl_t ptag = ib_bmi->ptag;
VAPI_cq_hndl_t recv_cq = ib_bmi->cq_hndl;
VAPI_cq_hndl_t send_cq = ib_bmi->cq_hndl;
VAPI_qp_hndl_t* qp_hndl = &endpoint->lcl_qp_hndl;
VAPI_qp_prop_t* qp_prop = &endpoint->lcl_qp_prop;
switch(transport_type) {
case VAPI_TS_RC: /* Set up RC qp parameters */
qp_init_attr.cap.max_oust_wr_rq = ib_bmi->ib_wq_size;
qp_init_attr.cap.max_oust_wr_sq = ib_bmi->ib_wq_size;
qp_init_attr.cap.max_sg_size_rq = ib_bmi->ib_sg_list_size;
qp_init_attr.cap.max_sg_size_sq = ib_bmi->ib_sg_list_size;
qp_init_attr.pd_hndl = ptag;
/* We don't have Reliable Datagram Handle right now */
qp_init_attr.rdd_hndl = 0;
/* Set Send and Recv completion queues */
qp_init_attr.rq_cq_hndl = recv_cq;
qp_init_attr.sq_cq_hndl = send_cq;
/* Signal all work requests on this queue pair */
qp_init_attr.rq_sig_type = VAPI_SIGNAL_REQ_WR;
qp_init_attr.sq_sig_type = VAPI_SIGNAL_REQ_WR;
/* Use Unreliable Datagram transport service */
qp_init_attr.ts_type = VAPI_TS_RC;
break;
case VAPI_TS_UD: /* Set up UD qp parameters */
default:
return OMPI_ERR_NOT_IMPLEMENTED;
}
ret = VAPI_create_qp(nic, &qp_init_attr,
qp_hndl, qp_prop);
if(VAPI_OK != ret) {
MCA_BMI_IB_VAPI_ERROR(ret, "VAPI_create_qp");
return OMPI_ERROR;
}
return OMPI_SUCCESS;
}
int mca_bmi_ib_endpoint_qp_init_query(mca_bmi_ib_endpoint_t * endpoint)
{
VAPI_ret_t ret;
VAPI_qp_attr_t qp_attr;
VAPI_qp_attr_mask_t qp_attr_mask;
VAPI_qp_init_attr_t qp_init_attr;
VAPI_qp_cap_t qp_cap;
mca_bmi_ib_module_t* ib_bmi = endpoint->endpoint_bmi;
VAPI_hca_hndl_t nic = ib_bmi->nic;
VAPI_qp_hndl_t qp_hndl = endpoint->lcl_qp_hndl;
VAPI_qp_num_t remote_qp = endpoint->rem_qp_num;
IB_lid_t remote_lid = endpoint->rem_lid;
IB_port_t port_id = ib_bmi->port_id;
/* Modifying QP to INIT */
QP_ATTR_MASK_CLR_ALL(qp_attr_mask);
qp_attr.qp_state = VAPI_INIT;
QP_ATTR_MASK_SET(qp_attr_mask, QP_ATTR_QP_STATE);
qp_attr.pkey_ix = ib_bmi->ib_pkey_ix;
QP_ATTR_MASK_SET(qp_attr_mask, QP_ATTR_PKEY_IX);
qp_attr.port = port_id;
QP_ATTR_MASK_SET(qp_attr_mask, QP_ATTR_PORT);
qp_attr.remote_atomic_flags = VAPI_EN_REM_WRITE | VAPI_EN_REM_READ;
QP_ATTR_MASK_SET(qp_attr_mask, QP_ATTR_REMOTE_ATOMIC_FLAGS);
ret = VAPI_modify_qp(nic, qp_hndl,
&qp_attr, &qp_attr_mask, &qp_cap);
if(VAPI_OK != ret) {
MCA_BMI_IB_VAPI_ERROR(ret, "VAPI_modify_qp");
return OMPI_ERROR;
}
DEBUG_OUT("Modified to init..Qp %d", qp_hndl);
/********************** INIT --> RTR ************************/
QP_ATTR_MASK_CLR_ALL(qp_attr_mask);
qp_attr.qp_state = VAPI_RTR;
QP_ATTR_MASK_SET(qp_attr_mask, QP_ATTR_QP_STATE);
qp_attr.qp_ous_rd_atom = ib_bmi->ib_qp_ous_rd_atom;
QP_ATTR_MASK_SET(qp_attr_mask, QP_ATTR_QP_OUS_RD_ATOM);
qp_attr.path_mtu = ib_bmi->ib_mtu;
QP_ATTR_MASK_SET(qp_attr_mask, QP_ATTR_PATH_MTU);
qp_attr.rq_psn = ib_bmi->ib_psn;
QP_ATTR_MASK_SET(qp_attr_mask, QP_ATTR_RQ_PSN);
qp_attr.pkey_ix = ib_bmi->ib_pkey_ix;
QP_ATTR_MASK_SET(qp_attr_mask, QP_ATTR_PKEY_IX);
qp_attr.min_rnr_timer = ib_bmi->ib_min_rnr_timer;
QP_ATTR_MASK_SET(qp_attr_mask, QP_ATTR_MIN_RNR_TIMER);
qp_attr.av.sl = ib_bmi->ib_service_level;
qp_attr.av.grh_flag = FALSE;
qp_attr.av.static_rate = ib_bmi->ib_static_rate;
qp_attr.av.src_path_bits = ib_bmi->ib_src_path_bits;
qp_attr.dest_qp_num = remote_qp;
QP_ATTR_MASK_SET(qp_attr_mask, QP_ATTR_DEST_QP_NUM);
qp_attr.av.dlid = remote_lid;
QP_ATTR_MASK_SET(qp_attr_mask, QP_ATTR_AV);
ret = VAPI_modify_qp(nic, qp_hndl,
&qp_attr, &qp_attr_mask, &qp_cap);
if(VAPI_OK != ret) {
MCA_BMI_IB_VAPI_ERROR(ret, "VAPI_modify_qp");
return OMPI_ERROR;
}
DEBUG_OUT("Modified to RTR..Qp %d", qp_hndl);
/************** RTS *******************/
QP_ATTR_MASK_CLR_ALL(qp_attr_mask);
qp_attr.qp_state = VAPI_RTS;
QP_ATTR_MASK_SET(qp_attr_mask, QP_ATTR_QP_STATE);
qp_attr.sq_psn = ib_bmi->ib_psn;
QP_ATTR_MASK_SET(qp_attr_mask, QP_ATTR_SQ_PSN);
qp_attr.timeout = ib_bmi->ib_timeout;
QP_ATTR_MASK_SET(qp_attr_mask, QP_ATTR_TIMEOUT);
qp_attr.retry_count = ib_bmi->ib_retry_count;
QP_ATTR_MASK_SET(qp_attr_mask, QP_ATTR_RETRY_COUNT);
qp_attr.rnr_retry = ib_bmi->ib_rnr_retry;
QP_ATTR_MASK_SET(qp_attr_mask, QP_ATTR_RNR_RETRY);
qp_attr.ous_dst_rd_atom = ib_bmi->ib_max_rdma_dst_ops;
QP_ATTR_MASK_SET(qp_attr_mask, QP_ATTR_OUS_DST_RD_ATOM);
ret = VAPI_modify_qp(nic, qp_hndl,
&qp_attr, &qp_attr_mask, &qp_cap);
if(VAPI_OK != ret) {
MCA_BMI_IB_VAPI_ERROR(ret, "VAPI_modify_qp");
return OMPI_ERROR;
}
DEBUG_OUT("Modified to RTS..Qp %d", qp_hndl);
ret = VAPI_query_qp(nic, qp_hndl, &qp_attr, &qp_attr_mask, &qp_init_attr );
if (ret != VAPI_OK) {
ompi_output(0, "error querying the queue pair");
return OMPI_ERROR;
}
ib_bmi->ib_inline_max = qp_init_attr.cap.max_inline_data_sq;
return OMPI_SUCCESS;
}

5
src/mca/bmi/ib/bmi_ib_endpoint.h
Просмотреть файл

@ -3,8 +3,6 @@
* All rights reserved.
* Copyright (c) 2004-2005 The Trustees of the University of Tennessee.
* All rights reserved.
* Copyright (c) 2004 The Ohio State University.
* All rights reserved.
* Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
* University of Stuttgart. All rights reserved.
* Copyright (c) 2004-2005 The Regents of the University of California.
@ -24,7 +22,6 @@
#include "mca/pml/pml.h"
#include "mca/bmi/bmi.h"
#include "bmi_ib_frag.h"
#include "bmi_ib_priv.h"
#include "bmi_ib.h"
#if defined(c_plusplus) || defined(__cplusplus)
extern "C" {
@ -138,7 +135,7 @@ static inline int mca_bmi_ib_endpoint_post_rr(int cnt, mca_bmi_ib_endpoint_t *en
cnt,
rr_desc_post);
if(VAPI_OK != frag->ret) {
MCA_BMI_IB_VAPI_RET(frag->ret, "EVAPI_post_rr_list");
MCA_BMI_IB_VAPI_ERROR(frag->ret, "EVAPI_post_rr_list");
return OMPI_ERROR;
}
OMPI_THREAD_ADD32(&ib_bmi->rr_posted, cnt);

43
src/mca/bmi/ib/bmi_ib_error.h Обычный файл
Просмотреть файл

@ -0,0 +1,43 @@
/*
* Copyright (c) 2004-2005 The Trustees of Indiana University.
* All rights reserved.
* Copyright (c) 2004-2005 The Trustees of the University of Tennessee.
* All rights reserved.
* Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
* University of Stuttgart. All rights reserved.
* Copyright (c) 2004-2005 The Regents of the University of California.
* All rights reserved.
* $COPYRIGHT$
*
* Additional copyrights may follow
*
* $HEADER$
*/
#ifndef MCA_BMI_IB_ERROR_H
#define MCA_BMI_IB_ERROR_H
#include <vapi.h>
#include <mtl_common.h>
#include <vapi_common.h>
/*
*
*
*/
#define MCA_BMI_IB_VAPI_ERROR(vapi_ret, func_name) { \
ompi_output(0,"[%s:%d] ", __FILE__, __LINE__); \
ompi_output(0,"%s : %s",func_name,VAPI_strerror(vapi_ret)); \
}
/* Debug Print */
#if 0
#define DEBUG_OUT(fmt, args...) { \
ompi_output(0, "[%s:%d:%s] " fmt, __FILE__, __LINE__, __func__, \
##args); \
}
#else
#define DEBUG_OUT(fmt, args...)
#endif
#endif

3
src/mca/bmi/ib/bmi_ib_frag.h
Просмотреть файл

@ -3,8 +3,6 @@
* All rights reserved.
* Copyright (c) 2004-2005 The Trustees of the University of Tennessee.
* All rights reserved.
* Copyright (c) 2004 The Ohio State University.
* All rights reserved.
* Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
* University of Stuttgart. All rights reserved.
* Copyright (c) 2004-2005 The Regents of the University of California.
@ -20,7 +18,6 @@
#define MCA_BMI_IB_FRAG_H
#include "ompi_config.h"
#include "bmi_ib_priv.h"
#include "bmi_ib.h"
#include <vapi.h>

377
src/mca/bmi/ib/bmi_ib_priv.c
Просмотреть файл

@ -1,377 +0,0 @@
/*
* Copyright (c) 2004-2005 The Trustees of Indiana University.
* All rights reserved.
* Copyright (c) 2004-2005 The Trustees of the University of Tennessee.
* All rights reserved.
* Copyright (c) 2004 The Ohio State University.
* All rights reserved.
* Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
* University of Stuttgart. All rights reserved.
* Copyright (c) 2004-2005 The Regents of the University of California.
* All rights reserved.
* $COPYRIGHT$
*
* Additional copyrights may follow
*
* $HEADER$
*/
#include "ompi_config.h"
#include "bmi_ib_vapi.h"
#include "bmi_ib_priv.h"
#include "bmi_ib.h"
#include "bmi_ib_frag.h"
#include "bmi_ib_endpoint.h"
/*
* This function returns the hca_id for each BMI
* in a round robin manner. Each BMI gets a different
* HCA id ...
*
* If num BMIs > num HCAs, then those bmis will be
* assigned HCA ids beginning from 0 again.
*
*/
/* static int mca_bmi_ib_get_hca_hndl(VAPI_hca_id_t hca_id, */
/* VAPI_hca_hndl_t* hca_hndl) */
/* { */
/* VAPI_ret_t ret; */
/* /\* Open the HCA *\/ */
/* ret = EVAPI_get_hca_hndl(hca_id, hca_hndl); */
/* if(VAPI_OK != ret) { */
/* MCA_BMI_IB_VAPI_RET(ret, "EVAPI_get_hca_hndl"); */
/* return OMPI_ERROR; */
/* } */
/* return OMPI_SUCCESS; */
/* } */
/* static int mca_bmi_ib_query_hca_prop(VAPI_hca_hndl_t nic) */
/* { */
/* VAPI_hca_vendor_t hca_vendor; */
/* VAPI_hca_cap_t hca_cap; */
/* VAPI_ret_t ret; */
/* ret = VAPI_query_hca_prop(nic, &hca_vendor, &hca_cap); */
/* hca_cap.phys_port_num; */
/* } */
/* static int mca_bmi_ib_query_hca_port_prop(VAPI_hca_hndl_t nic, */
/* VAPI_hca_port_t* port) */
/* { */
/* VAPI_ret_t ret; */
/* /\* Querying for port properties *\/ */
/* ret = VAPI_query_hca_port_prop(nic, */
/* (IB_port_t)DEFAULT_PORT, */
/* port); */
/* if(VAPI_OK != ret) { */
/* MCA_BMI_IB_VAPI_RET(ret, "VAPI_query_hca_port_prop"); */
/* return OMPI_ERROR; */
/* } */
/* return OMPI_SUCCESS; */
/* } */
int mca_bmi_ib_create_qp(VAPI_hca_hndl_t nic,
VAPI_pd_hndl_t ptag,
VAPI_cq_hndl_t recv_cq,
VAPI_cq_hndl_t send_cq,
VAPI_qp_hndl_t* qp_hndl,
VAPI_qp_prop_t* qp_prop,
int transport_type)
{
VAPI_ret_t ret;
VAPI_qp_init_attr_t qp_init_attr;
switch(transport_type) {
case VAPI_TS_RC: /* Set up RC qp parameters */
qp_init_attr.cap.max_oust_wr_rq = DEFAULT_WQ_SIZE;
qp_init_attr.cap.max_oust_wr_sq = DEFAULT_WQ_SIZE;
qp_init_attr.cap.max_sg_size_rq = DEFAULT_SG_LIST;
qp_init_attr.cap.max_sg_size_sq = DEFAULT_SG_LIST;
qp_init_attr.pd_hndl = ptag;
/* We don't have Reliable Datagram Handle right now */
qp_init_attr.rdd_hndl = 0;
/* Set Send and Recv completion queues */
qp_init_attr.rq_cq_hndl = recv_cq;
qp_init_attr.sq_cq_hndl = send_cq;
/* Signal all work requests on this queue pair */
qp_init_attr.rq_sig_type = VAPI_SIGNAL_REQ_WR;
qp_init_attr.sq_sig_type = VAPI_SIGNAL_REQ_WR;
/* Use Unreliable Datagram transport service */
qp_init_attr.ts_type = VAPI_TS_RC;
break;
case VAPI_TS_UD: /* Set up UD qp parameters */
default:
return OMPI_ERR_NOT_IMPLEMENTED;
}
ret = VAPI_create_qp(nic, &qp_init_attr,
qp_hndl, qp_prop);
if(VAPI_OK != ret) {
MCA_BMI_IB_VAPI_RET(ret, "VAPI_create_qp");
return OMPI_ERROR;
}
return OMPI_SUCCESS;
}
int mca_bmi_ib_qp_query(mca_bmi_ib_module_t* ib_bmi, VAPI_qp_hndl_t qp_hndl, VAPI_qp_num_t qp_num)
{
VAPI_hca_hndl_t nic = ib_bmi->nic;
VAPI_qp_attr_t qp_attr;
VAPI_qp_attr_mask_t qp_attr_mask;
VAPI_qp_init_attr_t qp_init_attr;
VAPI_ret_t ret;
ret = VAPI_query_qp(nic, qp_hndl, &qp_attr, &qp_attr_mask, &qp_init_attr );
if (ret != VAPI_OK) {
ompi_output(0, "error querying the queue pair");
return OMPI_ERROR;
}
ib_bmi->ib_inline_max = qp_init_attr.cap.max_inline_data_sq;
return OMPI_SUCCESS;
}
int mca_bmi_ib_qp_init(VAPI_hca_hndl_t nic,
VAPI_qp_hndl_t qp_hndl,
VAPI_qp_num_t remote_qp,
IB_lid_t remote_lid,
IB_port_t port_id)
{
VAPI_ret_t ret;
VAPI_qp_attr_t qp_attr;
VAPI_qp_attr_mask_t qp_attr_mask;
VAPI_qp_cap_t qp_cap;
/* Modifying QP to INIT */
QP_ATTR_MASK_CLR_ALL(qp_attr_mask);
qp_attr.qp_state = VAPI_INIT;
QP_ATTR_MASK_SET(qp_attr_mask, QP_ATTR_QP_STATE);
qp_attr.pkey_ix = DEFAULT_PKEY_IX;
QP_ATTR_MASK_SET(qp_attr_mask, QP_ATTR_PKEY_IX);
qp_attr.port = port_id;
QP_ATTR_MASK_SET(qp_attr_mask, QP_ATTR_PORT);
qp_attr.remote_atomic_flags = VAPI_EN_REM_WRITE | VAPI_EN_REM_READ;
QP_ATTR_MASK_SET(qp_attr_mask, QP_ATTR_REMOTE_ATOMIC_FLAGS);
ret = VAPI_modify_qp(nic, qp_hndl,
&qp_attr, &qp_attr_mask, &qp_cap);
if(VAPI_OK != ret) {
MCA_BMI_IB_VAPI_RET(ret, "VAPI_modify_qp");
return OMPI_ERROR;
}
D_PRINT("Modified to init..Qp %d", qp_hndl);
/********************** INIT --> RTR ************************/
QP_ATTR_MASK_CLR_ALL(qp_attr_mask);
qp_attr.qp_state = VAPI_RTR;
QP_ATTR_MASK_SET(qp_attr_mask, QP_ATTR_QP_STATE);
qp_attr.qp_ous_rd_atom = DEFAULT_QP_OUS_RD_ATOM;
QP_ATTR_MASK_SET(qp_attr_mask, QP_ATTR_QP_OUS_RD_ATOM);
qp_attr.path_mtu = DEFAULT_MTU;
QP_ATTR_MASK_SET(qp_attr_mask, QP_ATTR_PATH_MTU);
qp_attr.rq_psn = DEFAULT_PSN;
QP_ATTR_MASK_SET(qp_attr_mask, QP_ATTR_RQ_PSN);
qp_attr.pkey_ix = DEFAULT_PKEY_IX;
QP_ATTR_MASK_SET(qp_attr_mask, QP_ATTR_PKEY_IX);
qp_attr.min_rnr_timer = DEFAULT_MIN_RNR_TIMER;
QP_ATTR_MASK_SET(qp_attr_mask, QP_ATTR_MIN_RNR_TIMER);
qp_attr.av.sl = DEFAULT_SERVICE_LEVEL;
qp_attr.av.grh_flag = FALSE;
qp_attr.av.static_rate = DEFAULT_STATIC_RATE;
qp_attr.av.src_path_bits = DEFAULT_SRC_PATH_BITS;
qp_attr.dest_qp_num = remote_qp;
QP_ATTR_MASK_SET(qp_attr_mask, QP_ATTR_DEST_QP_NUM);
qp_attr.av.dlid = remote_lid;
QP_ATTR_MASK_SET(qp_attr_mask, QP_ATTR_AV);
ret = VAPI_modify_qp(nic, qp_hndl,
&qp_attr, &qp_attr_mask, &qp_cap);
if(VAPI_OK != ret) {
MCA_BMI_IB_VAPI_RET(ret, "VAPI_modify_qp");
return OMPI_ERROR;
}
D_PRINT("Modified to RTR..Qp %d", qp_hndl);
/************** RTS *******************/
QP_ATTR_MASK_CLR_ALL(qp_attr_mask);
qp_attr.qp_state = VAPI_RTS;
QP_ATTR_MASK_SET(qp_attr_mask, QP_ATTR_QP_STATE);
qp_attr.sq_psn = DEFAULT_PSN;
QP_ATTR_MASK_SET(qp_attr_mask, QP_ATTR_SQ_PSN);
qp_attr.timeout = DEFAULT_TIME_OUT;
QP_ATTR_MASK_SET(qp_attr_mask, QP_ATTR_TIMEOUT);
qp_attr.retry_count = DEFAULT_RETRY_COUNT;
QP_ATTR_MASK_SET(qp_attr_mask, QP_ATTR_RETRY_COUNT);
qp_attr.rnr_retry = DEFAULT_RNR_RETRY;
QP_ATTR_MASK_SET(qp_attr_mask, QP_ATTR_RNR_RETRY);
qp_attr.ous_dst_rd_atom = DEFAULT_MAX_RDMA_DST_OPS;
QP_ATTR_MASK_SET(qp_attr_mask, QP_ATTR_OUS_DST_RD_ATOM);
ret = VAPI_modify_qp(nic, qp_hndl,
&qp_attr, &qp_attr_mask, &qp_cap);
if(VAPI_OK != ret) {
MCA_BMI_IB_VAPI_RET(ret, "VAPI_modify_qp");
return OMPI_ERROR;
}
D_PRINT("Modified to RTS..Qp %d", qp_hndl);
return OMPI_SUCCESS;
}
/* int mca_bmi_ib_register_mem(VAPI_hca_hndl_t nic, VAPI_pd_hndl_t ptag, */
/* void* buf, int len, vapi_memhandle_t* memhandle) */
/* { */
/* VAPI_ret_t ret; */
/* VAPI_mrw_t mr_in, mr_out; */
/* vapi_memhandle_t mem_handle; */
/* mr_in.acl = VAPI_EN_LOCAL_WRITE | VAPI_EN_REMOTE_WRITE; */
/* mr_in.l_key = 0; */
/* mr_in.r_key = 0; */
/* mr_in.pd_hndl = ptag; */
/* mr_in.size = len; */
/* mr_in.start = (VAPI_virt_addr_t) (MT_virt_addr_t) buf; */
/* mr_in.type = VAPI_MR; */
/* ret = VAPI_register_mr(nic, &mr_in, &mem_handle.hndl, &mr_out); */
/* if(VAPI_OK != ret) { */
/* MCA_BMI_IB_VAPI_RET(ret, "VAPI_register_mr"); */
/* return OMPI_ERROR; */
/* } */
/* mem_handle.lkey = mr_out.l_key; */
/* mem_handle.rkey = mr_out.r_key; */
/* memhandle->lkey = mem_handle.lkey; */
/* memhandle->rkey = mem_handle.rkey; */
/* /\* D_PRINT("addr = %p, lkey = %d\n", buf, memhandle->lkey); *\/ */
/* memhandle->hndl = mem_handle.hndl; */
/* return OMPI_SUCCESS; */
/* } */
/* int mca_bmi_ib_post_send(mca_bmi_ib_module_t *ib_bmi, */
/* mca_bmi_ib_endpoint_t *peer, */
/* ib_buffer_t *ib_buf, void* addr) */
/* { */
/* VAPI_ret_t ret; */
/* int msg_len = ib_buf->desc.sg_entry.len; */
/* IB_PREPARE_SEND_DESC(ib_buf, (peer->rem_qp_num), */
/* msg_len, addr); */
/* /\* TODO - get this from NIC properties *\/ */
/* if(msg_len < 128) { /\* query this information from VAPI_query_qp(property max_inline_data_sq) *\/ */
/* ret = EVAPI_post_inline_sr(ib_bmi->nic, */
/* peer->lcl_qp_hndl, */
/* &ib_buf->desc.sr); */
/* } else { */
/* ret = VAPI_post_sr(ib_bmi->nic, */
/* peer->lcl_qp_hndl, */
/* &ib_buf->desc.sr); */
/* } */
/* if(VAPI_OK != ret) { */
/* MCA_BMI_IB_VAPI_RET(ret, "VAPI_post_sr"); */
/* return OMPI_ERROR; */
/* } */
/* return OMPI_SUCCESS; */
/* } */
/* void mca_bmi_ib_buffer_repost(VAPI_hca_hndl_t nic, void* addr) */
/* { */
/* mca_bmi_ib_recv_frag_t * frag = (mca_bmi_ib_recv_frag_t*)addr; */
/* frag->sg_entry.len = frag->size; */
/* frag->ret = VAPI_post_rr(nic, frag->endpoint->lcl_qp_hndl, &(frag->rr_desc)); */
/* if(VAPI_OK != frag->ret) { */
/* MCA_BMI_IB_VAPI_RET(frag->ret, "VAPI_post_rr"); */
/* ompi_output(0, "Error in buffer reposting"); */
/* } */
/* } */
/* void mca_bmi_ib_prepare_ack(mca_bmi_ib_module_t *ib_bmi, */
/* void* addr_to_reg, int len_to_reg, */
/* void* ack_buf, int* len_added) */
/* { */
/* mca_bmi_ib_mem_registry_info_t *info = */
/* mca_bmi_ib_register_mem_with_registry(ib_bmi, */
/* addr_to_reg, (size_t)len_to_reg); */
/* if(NULL == info) { */
/* ompi_output(0, "Error in registering"); */
/* } */
/* A_PRINT("Sending Remote key : %d", info->reply.r_key); */
/* memcpy(ack_buf,(void*) &(info->reply.r_key), sizeof(VAPI_rkey_t)); */
/* *len_added = sizeof(VAPI_rkey_t); */
/* } */
/* int mca_bmi_ib_rdma_write(mca_bmi_ib_module_t *ib_bmi, */
/* mca_bmi_ib_endpoint_t *peer, ib_buffer_t *ib_buf, */
/* void* send_buf, size_t send_len, void* remote_buf, */
/* VAPI_rkey_t remote_key, void* id_buf) */
/* { */
/* VAPI_ret_t ret; */
/* mca_bmi_ib_mem_registry_info_t *info = */
/* mca_bmi_ib_register_mem_with_registry(ib_bmi, */
/* send_buf, send_len); */
/* if (NULL == info) { */
/* return OMPI_ERROR; */
/* } */
/* /\* Prepare descriptor *\/ */
/* IB_PREPARE_RDMA_W_DESC(ib_buf, (peer->rem_qp_num), */
/* send_len, send_buf, (info->reply.l_key), remote_key, */
/* id_buf, remote_buf); */
/* ret = VAPI_post_sr(ib_bmi->nic, */
/* peer->lcl_qp_hndl, */
/* &ib_buf->desc.sr); */
/* if(ret != VAPI_OK) { */
/* MCA_BMI_IB_VAPI_RET(ret, "VAPI_post_sr"); */
/* return OMPI_ERROR; */
/* } */
/* return OMPI_SUCCESS; */
/* } */

222
src/mca/bmi/ib/bmi_ib_priv.h
Просмотреть файл

@ -1,222 +0,0 @@
/*
* Copyright (c) 2004-2005 The Trustees of Indiana University.
* All rights reserved.
* Copyright (c) 2004-2005 The Trustees of the University of Tennessee.
* All rights reserved.
* Copyright (c) 2004 The Ohio State University.
* All rights reserved.
* Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
* University of Stuttgart. All rights reserved.
* Copyright (c) 2004-2005 The Regents of the University of California.
* All rights reserved.
* $COPYRIGHT$
*
* Additional copyrights may follow
*
* $HEADER$
*/
#ifndef MCA_BMI_IB_PRIV_H
#define MCA_BMI_IB_PRIV_H
#include <stdint.h>
#include "class/ompi_free_list.h"
#include "bmi_ib_vapi.h"
#include "bmi_ib.h"
#define NUM_IB_SEND_BUF (1)
#define NUM_IB_RECV_BUF (4)
typedef enum {
IB_RECV,
IB_SEND
} IB_wr_t;
typedef enum {
IB_COMP_ERROR,
IB_COMP_RECV,
IB_COMP_SEND,
IB_COMP_RDMA_W,
IB_COMP_NOTHING
} IB_comp_t;
struct vapi_memhandle_t {
VAPI_mr_hndl_t hndl;
/* Memory region handle */
VAPI_lkey_t lkey;
/* Local key to registered memory, needed for
* posting send/recv requests */
VAPI_rkey_t rkey;
/* Remote key to registered memory, need to send this
* to remote processes for incoming RDMA ops */
};
typedef struct vapi_memhandle_t vapi_memhandle_t;
struct vapi_descriptor_t {
union {
VAPI_rr_desc_t rr;
/* Receive descriptor */
VAPI_sr_desc_t sr;
/* Send descriptor */
};
VAPI_sg_lst_entry_t sg_entry;
/* Scatter/Gather entry */
};
typedef struct vapi_descriptor_t vapi_descriptor_t;
/* struct ib_buffer_t { */
/* ompi_list_item_t super; */
/* vapi_descriptor_t desc; */
/* /\* Descriptor of the buffer *\/ */
/* vapi_memhandle_t hndl; */
/* /\* Buffer handle *\/ */
/* char buf[MCA_BMI_IB_FIRST_FRAG_SIZE]; */
/* /\* Buffer space *\/ */
/* VAPI_qp_hndl_t qp_hndl; */
/* /\* Queue pair used for this IB buffer *\/ */
/* }; */
/* typedef struct ib_buffer_t ib_buffer_t; */
#define DUMP_IB_STATE(ib_bmi) { \
ompi_output(0, "[%s:%d] ", __FILE__, __LINE__); \
ompi_output(0, "Dumping IB state"); \
ompi_output(0, "HCA ID : %s", ib_bmi->hca_id); \
ompi_output(0, "LID : %d", ib_bmi->port.lid); \
ompi_output(0, "HCA handle : %d", ib_bmi->nic); \
ompi_output(0, "Protection Domain: %d", ib_bmi->ptag); \
ompi_output(0, "Comp Q handle : %d", ib_bmi->cq_hndl); \
ompi_output(0, "Async hndl : %d", ib_bmi->async_handler); \
}
#define IB_PREPARE_RECV_DESC(ib_buf_ptr) { \
ib_buf_ptr->desc.rr.comp_type = VAPI_SIGNALED; \
ib_buf_ptr->desc.rr.opcode = VAPI_RECEIVE; \
ib_buf_ptr->desc.rr.id = (VAPI_virt_addr_t) \
(MT_virt_addr_t) ib_buf_ptr; \
ib_buf_ptr->desc.rr.sg_lst_len = 1; \
ib_buf_ptr->desc.rr.sg_lst_p = &ib_buf_ptr->desc.sg_entry; \
ib_buf_ptr->desc.sg_entry.len = MCA_BMI_IB_FIRST_FRAG_SIZE; \
ib_buf_ptr->desc.sg_entry.lkey = ib_buf_ptr->hndl.lkey; \
ib_buf_ptr->desc.sg_entry.addr = (VAPI_virt_addr_t) \
(MT_virt_addr_t) ib_buf_ptr->buf; \
}
#define IB_PREPARE_SEND_DESC(ib_buf_ptr, qp, msg_len, \
id_buf) { \
ib_buf_ptr->desc.sr.comp_type = VAPI_SIGNALED; \
ib_buf_ptr->desc.sr.opcode = VAPI_SEND; \
ib_buf_ptr->desc.sr.remote_qkey = 0; \
ib_buf_ptr->desc.sr.remote_qp = qp; \
ib_buf_ptr->desc.sr.id = (VAPI_virt_addr_t) \
(MT_virt_addr_t) id_buf; \
ib_buf_ptr->desc.sr.sg_lst_len = 1; \
ib_buf_ptr->desc.sr.sg_lst_p = &ib_buf_ptr->desc.sg_entry; \
ib_buf_ptr->desc.sg_entry.len = msg_len; \
ib_buf_ptr->desc.sg_entry.lkey = ib_buf_ptr->hndl.lkey; \
ib_buf_ptr->desc.sg_entry.addr = (VAPI_virt_addr_t) \
(MT_virt_addr_t) ib_buf_ptr->buf; \
}
#define IB_SET_REMOTE_QP_NUM(ib_buf_ptr, qp) { \
ib_buf_ptr->desc.sr.remote_qp = qp; \
}
#define IB_SET_SEND_DESC_ID(ib_buf_ptr, addr) { \
ib_buf_ptr->desc.sr.id = (VAPI_virt_addr_t) \
(MT_virt_addr_t) addr; \
}
#define IB_SET_SEND_DESC_LEN(ib_buf_ptr, msg_len) { \
ib_buf_ptr->desc.sg_entry.len = msg_len; \
}
#define IB_PREPARE_RDMA_W_DESC(ib_buf_ptr, qp, \
msg_len, user_buf, local_key, remote_key, \
id_buf, remote_buf) { \
ib_buf_ptr->desc.sr.comp_type = VAPI_SIGNALED; \
ib_buf_ptr->desc.sr.opcode = VAPI_RDMA_WRITE; \
ib_buf_ptr->desc.sr.remote_qkey = 0; \
ib_buf_ptr->desc.sr.remote_qp = qp; \
ib_buf_ptr->desc.sr.id = (VAPI_virt_addr_t) \
(MT_virt_addr_t) id_buf; \
ib_buf_ptr->desc.sr.sg_lst_len = 1; \
ib_buf_ptr->desc.sr.sg_lst_p = &ib_buf_ptr->desc.sg_entry; \
ib_buf_ptr->desc.sg_entry.len = msg_len; \
ib_buf_ptr->desc.sg_entry.lkey = local_key; \
ib_buf_ptr->desc.sg_entry.addr = (VAPI_virt_addr_t) \
(MT_virt_addr_t) user_buf; \
ib_buf_ptr->desc.sr.remote_addr = (VAPI_virt_addr_t) \
(MT_virt_addr_t) remote_buf; \
ib_buf_ptr->desc.sr.r_key = remote_key; \
}
struct mca_bmi_ib_module_t;
struct mca_bmi_base_endpoint_t;
int mca_bmi_ib_module_init(struct mca_bmi_ib_module_t*);
int mca_bmi_ib_register_mem(
VAPI_hca_hndl_t nic,
VAPI_pd_hndl_t ptag,
void* buf,
int len,
vapi_memhandle_t* memhandle);
/* int mca_bmi_ib_post_send( */
/* struct mca_bmi_ib_module_t *ib_module, */
/* struct mca_bmi_base_endpoint_t *peer, */
/* ib_buffer_t *ib_buf, void*); */
void mca_bmi_ib_buffer_repost(
VAPI_hca_hndl_t nic,
void* addr);
int mca_bmi_ib_qp_query(mca_bmi_ib_module_t* ib_bmi, VAPI_qp_hndl_t qp_hndl, VAPI_qp_num_t qp_num);
/* void mca_bmi_ib_prepare_ack( */
/* struct mca_bmi_ib_module_t *ib_module, */
/* void* addr_to_reg, int len_to_reg, */
/* void* ack_buf, int* len_added); */
/* int mca_bmi_ib_rdma_write( */
/* struct mca_bmi_ib_module_t *ib_module, */
/* struct mca_bmi_base_endpoint_t *peer, */
/* ib_buffer_t *ib_buf, */
/* void* send_buf, */
/* size_t send_len, */
/* void* remote_buf, */
/* VAPI_rkey_t remote_key, void*); */
int mca_bmi_ib_create_qp(VAPI_hca_hndl_t nic,
VAPI_pd_hndl_t ptag,
VAPI_cq_hndl_t recv_cq,
VAPI_cq_hndl_t send_cq,
VAPI_qp_hndl_t* qp_hndl,
VAPI_qp_prop_t* qp_prop,
int transport_type);
int mca_bmi_ib_qp_init(
VAPI_hca_hndl_t nic,
VAPI_qp_hndl_t qp_hndl,
VAPI_qp_num_t remote_qp,
IB_lid_t remote_lid,
IB_port_t port_id);
#endif /* MCA_BMI_IB_PRIV_H */

3
src/mca/bmi/ib/bmi_ib_proc.c
Просмотреть файл

@ -3,8 +3,6 @@
* All rights reserved.
* Copyright (c) 2004-2005 The Trustees of the University of Tennessee.
* All rights reserved.
* Copyright (c) 2004 The Ohio State University.
* All rights reserved.
* Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
* University of Stuttgart. All rights reserved.
* Copyright (c) 2004-2005 The Regents of the University of California.
@ -22,7 +20,6 @@
#include "mca/base/mca_base_module_exchange.h"
#include "bmi_ib.h"
#include "bmi_ib_vapi.h"
#include "bmi_ib_proc.h"
static void mca_bmi_ib_proc_construct(mca_bmi_ib_proc_t* proc);

4
src/mca/bmi/ib/bmi_ib_proc.h
Просмотреть файл

@ -3,8 +3,6 @@
* All rights reserved.
* Copyright (c) 2004-2005 The Trustees of the University of Tennessee.
* All rights reserved.
* Copyright (c) 2004 The Ohio State University.
* All rights reserved.
* Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
* University of Stuttgart. All rights reserved.
* Copyright (c) 2004-2005 The Regents of the University of California.
@ -23,8 +21,6 @@
#include "class/ompi_object.h"
#include "proc/proc.h"
#include "bmi_ib.h"
#include "bmi_ib_vapi.h"
#include "bmi_ib_addr.h"
#include "bmi_ib_endpoint.h"
#if defined(c_plusplus) || defined(__cplusplus)

90
src/mca/bmi/ib/bmi_ib_vapi.h
Просмотреть файл

@ -1,90 +0,0 @@
/*
* Copyright (c) 2004-2005 The Trustees of Indiana University.
* All rights reserved.
* Copyright (c) 2004-2005 The Trustees of the University of Tennessee.
* All rights reserved.
* Copyright (c) 2004 The Ohio State University.
* All rights reserved.
* Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
* University of Stuttgart. All rights reserved.
* Copyright (c) 2004-2005 The Regents of the University of California.
* All rights reserved.
* $COPYRIGHT$
*
* Additional copyrights may follow
*
* $HEADER$
*/
#ifndef MCA_BMI_IB_VAPI_H
#define MCA_BMI_IB_VAPI_H
#include <vapi.h>
#include <mtl_common.h>
#include <vapi_common.h>
/* HACK: Alert, these are dumb defines,
* all this stuff should be runtime. Ignoring for now.
*/
#define DEFAULT_PORT (1)
#define DEFAULT_CQ_SIZE (40000)
#define DEFAULT_WQ_SIZE (10000)
#define DEFAULT_SG_LIST (1)
#define DEFAULT_PKEY_IX (0)
#define DEFAULT_PSN (0)
#define DEFAULT_QP_OUS_RD_ATOM (1)
#define DEFAULT_MTU (MTU1024)
#define DEFAULT_MIN_RNR_TIMER (5)
#define DEFAULT_TIME_OUT (10)
#define DEFAULT_RETRY_COUNT (7)
#define DEFAULT_RNR_RETRY (7)
#define DEFAULT_MAX_RDMA_DST_OPS (16)
#define DEFAULT_TRAFFIC_CLASS (0)
#define DEFAULT_HOP_LIMIT (63)
#define DEFAULT_FLOW_LABEL (0)
#define DEFAULT_SERVICE_LEVEL (0)
#define DEFAULT_STATIC_RATE (0)
#define DEFAULT_SRC_PATH_BITS (0)
/* This is a convinence macro.
*
* ret : The value to return if call failed
* vapi_ret : The value which was returned from the last VAPI call
* func_name : The VAPI function which was called
*/
#define MCA_BMI_IB_VAPI_RET(vapi_ret, func_name) { \
ompi_output(0,"[%s:%d] ", __FILE__, __LINE__); \
ompi_output(0,"%s : %s",func_name,VAPI_strerror(vapi_ret)); \
}
/* Debug Print */
#if 0
#define D_PRINT(fmt, args...) { \
ompi_output(0, "[%s:%d:%s] " fmt, __FILE__, __LINE__, __func__, \
##args); \
}
#else
#define D_PRINT(fmt, args...)
#endif
#if 0
#define A_PRINT(fmt, args...) { \
ompi_output(0, "[%s:%d:%s] " fmt, __FILE__, __LINE__, __func__, \
##args); \
}
#else
#define A_PRINT(fmt, args...)
#endif
#if 0
#define B_PRINT(fmt, args...) { \
ompi_output(0, "[%s:%d:%s] " fmt, __FILE__, __LINE__, __func__, \
##args); \
}
#else
#define B_PRINT(fmt, args...)
#endif
#endif

2
src/mca/bmi/ib/configure.stub
Просмотреть файл

@ -4,8 +4,6 @@
# All rights reserved.
# Copyright (c) 2004-2005 The Trustees of the University of Tennessee.
# All rights reserved.
# Copyright (c) 2004 The Ohio State University.
# All rights reserved.
# Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
# University of Stuttgart. All rights reserved.
# Copyright (c) 2004-2005 The Regents of the University of California.