ports/openmpi
ports/openmpi
1
1
Форкнуть 0
Код Релизы Активность

Getting Dynamic connection management and UD transport to work.

Просмотр исходного кода 
This commit was SVN r2135.
Этот коммит содержится в:
Sayantan Sur 2004-08-14 00:27:00 +00:00
родитель a5848eb249
Коммит c6d6ea8482
12 изменённых файлов: 342 добавлений и 98 удалений
Показать статистику Скачать Patch файл Скачать Diff файл Показать все файлы Свернуть все файлы

3
src/mca/ptl/ib/src/Makefile.am
Просмотреть файл

@ -16,7 +16,10 @@ libmca_ptl_ib_la_SOURCES = \
ptl_ib_proc.h \
ptl_ib_peer.c \
ptl_ib_peer.h \
ptl_ib_recvfrag.c \
ptl_ib_recvfrag.h \
ptl_ib_sendfrag.c \
ptl_ib_sendfrag.h \
ptl_ib_sendreq.c \
ptl_ib_sendreq.h \
ptl_ib_vapi.h

39
src/mca/ptl/ib/src/ptl_ib.c
Просмотреть файл

@ -14,6 +14,7 @@
#include "mca/ptl/base/ptl_base_recvfrag.h"
#include "mca/base/mca_base_module_exchange.h"
#include "ptl_ib.h"
#include "ptl_ib_sendfrag.h"
mca_ptl_ib_module_t mca_ptl_ib_module = {
{
@ -44,14 +45,6 @@ mca_ptl_ib_module_t mca_ptl_ib_module = {
}
};
OBJ_CLASS_INSTANCE(mca_ptl_ib_recv_frag_t,
mca_ptl_base_recv_frag_t,
NULL, NULL);
OBJ_CLASS_INSTANCE(mca_ptl_ib_send_request_t,
mca_pml_base_send_request_t,
NULL, NULL);
int mca_ptl_ib_add_procs(struct mca_ptl_base_module_t* base_module,
size_t nprocs, struct ompi_proc_t **ompi_procs,
@ -161,9 +154,33 @@ int mca_ptl_ib_send( struct mca_ptl_base_module_t* ptl,
size_t size,
int flags)
{
/* Stub */
D_PRINT("Stub\n");
return OMPI_SUCCESS;
int rc;
mca_ptl_ib_send_frag_t* sendfrag;
ompi_list_item_t* item;
if (0 == offset) {
sendfrag = &((mca_ptl_ib_send_request_t*)sendreq)->req_frag;
} else {
OMPI_FREE_LIST_GET(&mca_ptl_ib_component.ib_send_frags, item, rc);
if(NULL == (sendfrag = (mca_ptl_ib_send_frag_t*)item)) {
return rc;
}
}
rc = mca_ptl_ib_send_frag_init(sendfrag, ptl_peer, sendreq, offset, &size, flags);
if(rc != OMPI_SUCCESS) {
return rc;
}
/* must update the offset after actual fragment
* size is determined -- and very important --
* before attempting to send the fragment
*/
sendreq->req_offset += size;
rc = mca_ptl_ib_peer_send(ptl_peer, sendfrag);
return rc;
}

5
src/mca/ptl/ib/src/ptl_ib.h
Просмотреть файл

@ -43,6 +43,7 @@ struct mca_ptl_ib_component_t {
int ib_free_list_max; /**< maximum size of free lists */
int ib_free_list_inc; /**< number of elements to alloc when growing free lists */
ompi_free_list_t ib_send_requests; /**< free list of ib send requests -- sendreq + IB */
ompi_free_list_t ib_send_frags; /**< free list of ib send fragments */
ompi_free_list_t ib_recv_frags; /**< free list of ib recv fragments */
ompi_list_t ib_procs; /**< list of ib proc structures */
ompi_event_t ib_send_event; /**< event structure for sends */
@ -73,8 +74,8 @@ struct mca_ptl_ib_module_t {
VAPI_cq_hndl_t ud_scq_hndl;/* UD send completion queue handle */
VAPI_cq_hndl_t ud_rcq_hndl;/* UD recv completion queue handle */
mca_ptl_ib_ud_buf_t* ud_buf; /* Link to UD buffer structures
which are posted on the UD interface */
mca_ptl_ib_ud_buf_t* ud_recv_buf;/* Link to UD recv buffer structures */
mca_ptl_ib_ud_buf_t* ud_send_buf;/* Link to UD bufs which are used for sending */
VAPI_qp_hndl_t ud_qp_hndl; /* UD queue pair handle */
VAPI_qp_prop_t ud_qp_prop; /* UD queue pair properties */

5
src/mca/ptl/ib/src/ptl_ib_addr.h
Просмотреть файл

@ -12,4 +12,9 @@ struct mca_ptl_ib_ud_addr_t {
};
typedef struct mca_ptl_ib_ud_addr_t mca_ptl_ib_ud_addr_t;
struct mca_ptl_ib_addr_t {
VAPI_qp_hndl_t rc_qp; /* RC qp hndl */
};
typedef struct mca_ptl_ib_addr_t mca_ptl_ib_addr_t;
#endif

47
src/mca/ptl/ib/src/ptl_ib_component.c
Просмотреть файл

@ -90,8 +90,6 @@ static inline int mca_ptl_ib_param_register_int(
int mca_ptl_ib_component_open(void)
{
D_PRINT("Opening InfiniBand component ...\n");
OBJ_CONSTRUCT(&mca_ptl_ib_component.ib_procs, ompi_list_t);
/* register super component parameters */
@ -188,8 +186,6 @@ mca_ptl_base_module_t** mca_ptl_ib_component_init(int *num_ptl_modules,
*allow_multi_user_threads = true;
*have_hidden_threads = OMPI_HAVE_THREADS;
D_PRINT("IB Component Init\n");
/* need to set ompi_using_threads() as ompi_event_init()
* will spawn a thread if supported */
if(OMPI_HAVE_THREADS) {
@ -204,8 +200,6 @@ mca_ptl_base_module_t** mca_ptl_ib_component_init(int *num_ptl_modules,
ret = mca_ptl_ib_get_num_hcas(&num_hcas);
D_PRINT("Number of HCAs found: %d\n", num_hcas);
if ((0 == num_hcas) || (OMPI_SUCCESS != ret)) {
return NULL;
}
@ -227,11 +221,6 @@ mca_ptl_base_module_t** mca_ptl_ib_component_init(int *num_ptl_modules,
mca_ptl_ib_component.ib_max_ptl_modules =
mca_ptl_ib_component.ib_num_hcas;
D_PRINT("num_hcas: %d, num_ptl_modules: %d, max_ptl_modules: %d\n",
mca_ptl_ib_component.ib_num_hcas,
mca_ptl_ib_component.ib_num_ptl_modules,
mca_ptl_ib_component.ib_max_ptl_modules);
ib_modules = (mca_ptl_ib_module_t*) malloc(sizeof(mca_ptl_ib_module_t) *
mca_ptl_ib_component.ib_num_ptl_modules);
if(NULL == ib_modules) {
@ -249,8 +238,6 @@ mca_ptl_base_module_t** mca_ptl_ib_component_init(int *num_ptl_modules,
sizeof(mca_ptl_ib_module));
}
D_PRINT("About to initialize IB modules ...\n");
/* For each ptl, do this */
for(i = 0; i < mca_ptl_ib_component.ib_num_ptl_modules; i++) {
@ -313,21 +300,31 @@ mca_ptl_base_module_t** mca_ptl_ib_component_init(int *num_ptl_modules,
return NULL;
}
/* Allocate the UD buffers */
ib_modules[i].ud_recv_buf = NULL;
ib_modules[i].ud_recv_buf = malloc(MAX_UD_PREPOST_DEPTH *
sizeof(mca_ptl_ib_ud_buf_t));
if(NULL == ib_modules[i].ud_recv_buf) {
return NULL;
}
/* Prepare the UD buffers for communication:
*
* 1. register
* 2. fill up descriptors
*/
ib_modules[i].ud_buf = NULL;
if(mca_ptl_ib_prep_ud_bufs(ib_modules[i].nic, &ib_modules[i].ud_buf)
if(mca_ptl_ib_prep_ud_bufs(ib_modules[i].nic, ib_modules[i].ud_recv_buf,
IB_RECV, MAX_UD_PREPOST_DEPTH)
!= OMPI_SUCCESS) {
return NULL;
}
/* Post the UD recv descriptors */
if(mca_ptl_ib_post_ud_recv(ib_modules[i].nic, ib_modules[i].ud_qp_hndl,
ib_modules[i].ud_buf)
ib_modules[i].ud_recv_buf, MAX_UD_PREPOST_DEPTH)
!= OMPI_SUCCESS) {
return NULL;
}
@ -392,6 +389,22 @@ mca_ptl_base_module_t** mca_ptl_ib_component_init(int *num_ptl_modules,
int mca_ptl_ib_component_control(int param, void* value, size_t size)
{
#if 0
VAPI_ret_t ret;
VAPI_wc_desc_t comp;
char* env = NULL;
if((env = getenv("POLL")) != NULL) {
while(1) {
ret = VAPI_poll_cq(mca_ptl_ib_module.nic, mca_ptl_ib_module.ud_rcq_hndl,
&comp);
if(VAPI_OK == ret) {
fprintf(stderr,"Something arrived!\n");
}
}
}
#endif
/* Stub */
D_PRINT("Stub\n");
return OMPI_SUCCESS;

143
src/mca/ptl/ib/src/ptl_ib_peer.c
Просмотреть файл

@ -1,3 +1,6 @@
#include <sys/time.h>
#include <time.h>
#include "include/types.h"
#include "mca/pml/base/pml_base_sendreq.h"
#include "mca/ns/base/base.h"
@ -5,6 +8,7 @@
#include "ptl_ib_addr.h"
#include "ptl_ib_peer.h"
#include "ptl_ib_proc.h"
#include "ptl_ib_priv.h"
#include "ptl_ib_sendfrag.h"
static void mca_ptl_ib_peer_construct(mca_ptl_base_peer_t* module_peer);
@ -23,11 +27,12 @@ static void mca_ptl_ib_peer_construct(mca_ptl_base_peer_t* module_peer)
module_peer->peer_module = 0;
module_peer->peer_proc = 0;
module_peer->peer_addr = 0;
module_peer->peer_ts = 0.0;
module_peer->peer_send_frag = 0;
module_peer->peer_recv_frag = 0;
module_peer->peer_send_event.ev_flags = 0;
module_peer->peer_recv_event.ev_flags = 0;
module_peer->peer_state = MCA_PTL_IB_NOT_OPENED;
module_peer->peer_state = MCA_PTL_IB_CLOSED;
module_peer->peer_retries = 0;
OBJ_CONSTRUCT(&module_peer->peer_frags, ompi_list_t);
OBJ_CONSTRUCT(&module_peer->peer_send_lock, ompi_mutex_t);
@ -37,3 +42,139 @@ static void mca_ptl_ib_peer_construct(mca_ptl_base_peer_t* module_peer)
static void mca_ptl_ib_peer_destruct(mca_ptl_base_peer_t* module_peer)
{
}
static int mca_ptl_ib_peer_check_timeout(mca_ptl_base_peer_t* peer)
{
return OMPI_SUCCESS;
}
static double mca_ptl_ib_get_us(void)
{
struct timeval t;
gettimeofday(&t, NULL);
return (double) t.tv_sec * (double) 1e6 + (double) t.tv_usec;
}
static int mca_ptl_ib_peer_start_connect(mca_ptl_base_peer_t* peer)
{
VAPI_ret_t ret;
VAPI_ud_av_t av;
VAPI_ud_av_hndl_t av_hndl;
mca_ptl_ib_module_t* peer_module;
peer_module = peer->peer_module;
peer_module->ud_send_buf = malloc(sizeof(mca_ptl_ib_ud_buf_t) *
MAX_UD_PREPOST_DEPTH);
mca_ptl_ib_prep_ud_bufs(peer_module->nic, peer_module->ud_send_buf,
IB_SEND, MAX_UD_PREPOST_DEPTH);
peer_module->ud_send_buf[0].buf_data->qp_hndl = 101;
av.dgid[0] = 0;
av.dlid = peer->peer_proc->proc_addrs[0].lid;
av.grh_flag = TRUE;
av.flow_label = DEFAULT_FLOW_LABEL;
av.hop_limit = DEFAULT_HOP_LIMIT;
av.sgid_index = 0;
av.sl = DEFAULT_SERVICE_LEVEL;
av.port = DEFAULT_PORT;
av.src_path_bits = DEFAULT_SRC_PATH_BITS;
av.static_rate = DEFAULT_STATIC_RATE;
av.traffic_class = DEFAULT_TRAFFIC_CLASS;
ret = VAPI_create_addr_hndl(peer_module->nic, peer_module->ptag,
&av, &av_hndl);
if (VAPI_OK != ret) {
return OMPI_ERROR;
}
peer_module->ud_send_buf[0].desc.sr.remote_qkey = 0;
peer_module->ud_send_buf[0].desc.sr.remote_qp = peer->peer_proc->proc_addrs[0].ud_qp;
peer_module->ud_send_buf[0].desc.sr.remote_ah = av_hndl;
peer_module->ud_send_buf[0].desc.sr.id = (VAPI_virt_addr_t) (MT_virt_addr_t)&peer_module->ud_send_buf[0];
peer_module->ud_send_buf[0].desc.sg_entry.addr = (VAPI_virt_addr_t) (MT_virt_addr_t)
peer_module->ud_send_buf[0].buf_data;
peer_module->ud_send_buf[0].desc.sg_entry.len = 4;
peer_module->ud_send_buf[0].desc.sg_entry.lkey = peer_module->ud_send_buf[0].memhandle.lkey;
peer_module->ud_send_buf[0].desc.sr.set_se = FALSE;
peer_module->ud_send_buf[0].desc.sr.fence = FALSE;
ret = VAPI_post_sr(peer_module->nic,
peer_module->ud_qp_hndl,
&(peer_module->ud_send_buf[0].desc.sr));
D_PRINT("Remote QP: %d, Remote LID : %d, Posted sr: %s\n",
peer_module->ud_send_buf[0].desc.sr.remote_qp,
av.dlid, VAPI_strerror(ret));
if(VAPI_OK != ret) {
return OMPI_ERROR;
}
return OMPI_SUCCESS;
}
/*
* Attempt to send a fragment using a given peer. If the peer is not
* connected, queue the fragment and start the connection as required.
*/
int mca_ptl_ib_peer_send(mca_ptl_base_peer_t* peer,
mca_ptl_ib_send_frag_t* frag)
{
int rc;
OMPI_THREAD_LOCK(&peer->peer_send_lock);
switch(peer->peer_state) {
case MCA_PTL_IB_CONNECTING:
/* Well, connecting means that I've already sent my UD
* QP across, but I haven't got any reply, so, I have
* to check for timeout */
ompi_list_append(&peer->peer_frags, (ompi_list_item_t*)frag);
rc = mca_ptl_ib_peer_check_timeout(peer);
break;
case MCA_PTL_IB_CLOSED:
ompi_list_append(&peer->peer_frags, (ompi_list_item_t*)frag);
rc = mca_ptl_ib_peer_start_connect(peer);
break;
case MCA_PTL_IB_FAILED:
rc = OMPI_ERR_UNREACH;
break;
case MCA_PTL_IB_CONNECTED:
/* Fill in this later for the send to work
*
if (NULL != ptl_peer->peer_send_frag) {
ompi_list_append(&ptl_peer->peer_frags, (ompi_list_item_t*)frag);
} else {
if(mca_ptl_ib_send_frag_handler(frag, ptl_peer->peer_sd)) {
OMPI_THREAD_UNLOCK(&ptl_peer->peer_send_lock);
mca_ptl_ib_send_frag_progress(frag);
return rc;
} else {
ptl_peer->peer_send_frag = frag;
ompi_event_add(&ptl_peer->peer_send_event, 0);
}
}
*/
break;
default:
rc = OMPI_ERR_UNREACH;
}
OMPI_THREAD_UNLOCK(&peer->peer_send_lock);
return rc;
}

23
src/mca/ptl/ib/src/ptl_ib_peer.h
Просмотреть файл

@ -8,16 +8,28 @@
#include "ptl_ib_recvfrag.h"
#include "ptl_ib_sendfrag.h"
OBJ_CLASS_DECLARATION(mca_ptl_ib_peer_t);
/**
* State of IB peer connection.
*/
typedef enum {
MCA_PTL_IB_NOT_OPENED,
/* Defines the state in which this PTL instance
* has started the process of connection, and
* is waiting for acknowledgement from peer */
MCA_PTL_IB_CONNECTING,
MCA_PTL_IB_CONNECT_ACK,
/* Connected ... both sender & receiver have
* buffers associated with this connection */
MCA_PTL_IB_CONNECTED,
/* Connection is closed, there are no resources
* associated with this */
MCA_PTL_IB_CLOSED,
/* Maximum number of retries have been used.
* Report failure on send to upper layer */
MCA_PTL_IB_FAILED
} mca_ptl_ib_state_t;
@ -32,11 +44,13 @@ struct mca_ptl_base_peer_t {
ompi_list_item_t super;
struct mca_ptl_ib_module_t* peer_module; /**< PTL instance that created this connection */
struct mca_ptl_ib_proc_t* peer_proc; /**< proc structure corresponding to peer */
struct mca_ptl_ib_ud_addr_t* peer_addr; /**< address of peer */
struct mca_ptl_ib_addr_t* peer_addr; /**< address of peer */
mca_ptl_ib_send_frag_t* peer_send_frag; /**< current send frag being processed */
mca_ptl_ib_recv_frag_t* peer_recv_frag; /**< current recv frag being processed */
mca_ptl_ib_state_t peer_state; /**< current state of the connection */
size_t peer_retries; /**< number of connection retries attempted */
double peer_ts; /**< timestamp of when the first
connection was attempted */
ompi_list_t peer_frags; /**< list of pending frags to send */
ompi_mutex_t peer_send_lock; /**< lock for concurrent access to peer state */
ompi_mutex_t peer_recv_lock; /**< lock for concurrent access to peer state */
@ -46,7 +60,6 @@ struct mca_ptl_base_peer_t {
typedef struct mca_ptl_base_peer_t mca_ptl_base_peer_t;
typedef struct mca_ptl_base_peer_t mca_ptl_ib_peer_t;
OBJ_CLASS_DECLARATION(mca_ptl_ib_peer_t);
int mca_ptl_ib_peer_send(mca_ptl_base_peer_t*, mca_ptl_ib_send_frag_t*);
#endif

101
src/mca/ptl/ib/src/ptl_ib_priv.c
Просмотреть файл

@ -14,10 +14,8 @@ static void async_event_handler(VAPI_hca_hndl_t hca_hndl,
case VAPI_SEND_QUEUE_DRAINED:
case VAPI_PORT_ACTIVE:
{
/*
D_PRINT("Got an asynchronous event: %s\n",
VAPI_event_record_sym(event_p->type));
*/
break;
}
case VAPI_CQ_ERROR:
@ -47,7 +45,8 @@ static void ud_completion_handler(VAPI_hca_hndl_t nic,
{
VAPI_ret_t ret;
D_PRINT("Got interrupt!!\n");
fprintf(stderr,"Got interrupt!!\n");
fflush(stderr);
ret = VAPI_req_comp_notif(nic, cq_hndl, VAPI_NEXT_COMP);
@ -120,8 +119,8 @@ int mca_ptl_ib_ud_qp_init(VAPI_hca_hndl_t nic,
qp_init_attr.sq_cq_hndl = ud_scq_hndl;
/* Signal all work requests on this queue pair */
qp_init_attr.rq_sig_type = VAPI_SIGNAL_ALL_WR;
qp_init_attr.sq_sig_type = VAPI_SIGNAL_ALL_WR;
qp_init_attr.rq_sig_type = VAPI_SIGNAL_REQ_WR;
qp_init_attr.sq_sig_type = VAPI_SIGNAL_REQ_WR;
/* Use Unreliable Datagram transport service */
qp_init_attr.ts_type = VAPI_TS_UD;
@ -342,59 +341,62 @@ int mca_ptl_ib_set_async_handler(VAPI_hca_hndl_t nic,
}
int mca_ptl_ib_prep_ud_bufs(VAPI_hca_hndl_t nic,
mca_ptl_ib_ud_buf_t** ud_buf_ptr)
mca_ptl_ib_ud_buf_t* ud_buf, IB_wr_t wr_type,
int num_bufs)
{
int size, len;
int i, num_ud_bufs;
int i;
vapi_descriptor_t* desc;
mca_ptl_ib_ud_buf_data_t* buf_data;
vapi_memhandle_t* memhandle;
mca_ptl_ib_ud_buf_t* ud_buf;
num_ud_bufs = MAX_UD_PREPOST_DEPTH;
for(i = 0; i < num_bufs; i++) {
size = num_ud_bufs * sizeof(mca_ptl_ib_ud_buf_t);
len = sizeof(mca_ptl_ib_ud_buf_data_t);
(mca_ptl_ib_ud_buf_t*)*ud_buf_ptr = (mca_ptl_ib_ud_buf_t*) malloc(size);
if(NULL == *ud_buf_ptr) {
return OMPI_ERR_OUT_OF_RESOURCE;
}
ud_buf = (mca_ptl_ib_ud_buf_t*)*ud_buf_ptr;
/* Walk through the list of UD bufs */
for(i = 0; i < num_ud_bufs; i++) {
desc = &ud_buf[i].desc;
buf_data = ud_buf[i].buf_data;
memhandle = &ud_buf[i].memhandle;
ud_buf[i].buf_data =
malloc(sizeof(mca_ptl_ib_ud_buf_data_t));
buf_data = (mca_ptl_ib_ud_buf_data_t*) malloc(len);
if(NULL == buf_data) {
if(NULL == ud_buf[i].buf_data) {
return OMPI_ERR_OUT_OF_RESOURCE;
}
if(mca_ptl_ib_register_mem(nic, buf_data, len, memhandle)
if(mca_ptl_ib_register_mem(nic,
(void*) ud_buf[i].buf_data,
sizeof(mca_ptl_ib_ud_buf_data_t),
&ud_buf[i].memhandle)
!= OMPI_SUCCESS) {
return OMPI_ERROR;
}
desc->rr.comp_type = VAPI_SIGNALED;
desc->rr.opcode = VAPI_RECEIVE;
desc->rr.id = (VAPI_virt_addr_t) (unsigned int) &ud_buf[i];
desc->rr.sg_lst_len = 1;
desc->rr.sg_lst_p = &(desc->sg_entry);
desc->sg_entry.len = len;
D_PRINT("databuf = %p, len = %d, lkey = %d\n",
(void*) ud_buf[i].buf_data,
sizeof(mca_ptl_ib_ud_buf_data_t),
ud_buf[i].memhandle.lkey);
D_PRINT("length = %d\n", len);
desc->sg_entry.lkey = memhandle->lkey;
if(IB_RECV == wr_type) {
desc->rr.comp_type = VAPI_SIGNALED;
desc->rr.opcode = VAPI_RECEIVE;
desc->rr.id = (VAPI_virt_addr_t)(MT_virt_addr_t) &(ud_buf[i]);
desc->rr.sg_lst_len = 1;
desc->rr.sg_lst_p = &(desc->sg_entry);
desc->sg_entry.len = sizeof(mca_ptl_ib_ud_buf_data_t);
desc->sg_entry.addr = (VAPI_virt_addr_t) (MT_virt_addr_t) ud_buf[i].buf_data;
desc->sg_entry.lkey = ud_buf[i].memhandle.lkey;
} else if (IB_SEND == wr_type) {
desc->sr.comp_type = VAPI_SIGNALED;
desc->sr.opcode = VAPI_SEND;
desc->sr.id = (VAPI_virt_addr_t)(MT_virt_addr_t) &(ud_buf[i]);
desc->sr.sg_lst_len = 1;
desc->sr.sg_lst_p = &(desc->sg_entry);
desc->sg_entry.len = sizeof(mca_ptl_ib_ud_buf_data_t);
desc->sg_entry.addr = (VAPI_virt_addr_t) (MT_virt_addr_t) ud_buf[i].buf_data;
desc->sg_entry.lkey = ud_buf[i].memhandle.lkey;
} else {
return OMPI_ERROR;
}
D_PRINT("local key = %d\n", (int) memhandle->lkey);
desc->sg_entry.addr = (VAPI_virt_addr_t) (MT_virt_addr_t) (buf_data);
D_PRINT("sg_entry.addr = %d\n", (int)desc->sg_entry.addr);
D_PRINT("databuf = %p, len = %d, lkey = %d\n",
(void*) ud_buf[i].buf_data,
sizeof(mca_ptl_ib_ud_buf_data_t),
ud_buf[i].memhandle.lkey);
}
return OMPI_SUCCESS;
@ -427,8 +429,7 @@ int mca_ptl_ib_register_mem(VAPI_hca_hndl_t nic,
memhandle->lkey = mem_handle.lkey;
memhandle->rkey = mem_handle.rkey;
D_PRINT("local key = %d, remote key = %d\n",
mem_handle.lkey, mem_handle.rkey);
D_PRINT("addr = %p, lkey = %d\n", buf, memhandle->lkey);
memhandle->hndl = mem_handle.hndl;
@ -437,18 +438,14 @@ int mca_ptl_ib_register_mem(VAPI_hca_hndl_t nic,
int mca_ptl_ib_post_ud_recv(VAPI_hca_hndl_t nic,
VAPI_qp_hndl_t ud_qp_hndl,
mca_ptl_ib_ud_buf_t* ud_buf)
mca_ptl_ib_ud_buf_t* ud_buf, int num_bufs)
{
int num_ud_bufs, i;
int i;
VAPI_ret_t ret;
vapi_descriptor_t* desc;
num_ud_bufs = MAX_UD_PREPOST_DEPTH;
for(i = 0; i < num_bufs; i++) {
for(i = 0; i< num_ud_bufs; i++) {
desc = &ud_buf[i].desc;
ret = VAPI_post_rr(nic, ud_qp_hndl, &desc->rr);
ret = VAPI_post_rr(nic, ud_qp_hndl, &ud_buf[i].desc.rr);
if(VAPI_OK != ret) {
MCA_PTL_IB_VAPI_RET(ret, "VAPI_post_rr");

36
src/mca/ptl/ib/src/ptl_ib_priv.h
Просмотреть файл

@ -10,6 +10,11 @@
*/
#define MAX_UD_PREPOST_DEPTH (1)
typedef enum {
IB_RECV,
IB_SEND
} IB_wr_t;
struct vapi_memhandle_t {
VAPI_mr_hndl_t hndl;
VAPI_lkey_t lkey;
@ -42,6 +47,31 @@ struct mca_ptl_ib_ud_buf_t {
typedef struct mca_ptl_ib_ud_buf_t mca_ptl_ib_ud_buf_t;
#define MCA_PTL_IB_UD_RECV_DESC(ud_buf, len) { \
desc->rr.comp_type = VAPI_SIGNALED; \
desc->rr.opcode = VAPI_RECEIVE; \
desc->rr.id = (VAPI_virt_addr_t)(MT_virt_addr_t) &(ud_buf); \
desc->rr.sg_lst_len = 1; \
desc->rr.sg_lst_p = &(desc->sg_entry); \
desc->sg_entry.len = len; \
desc->sg_entry.addr = \
(VAPI_virt_addr_t) (MT_virt_addr_t) ud_buf.buf_data; \
desc->sg_entry.lkey = ud_buf.memhandle.lkey; \
}
#define MCA_PTL_IB_UD_SEND_DESC(ud_buf, len) { \
desc->sr.comp_type = VAPI_SIGNALED; \
desc->sr.opcode = VAPI_SEND; \
desc->sr.id = (VAPI_virt_addr_t)(MT_virt_addr_t) ud_buf; \
desc->sr.sg_lst_len = 1; \
desc->sr.sg_lst_p = &(desc->sg_entry); \
desc->sg_entry.len = len; \
desc->sg_entry.addr = \
(VAPI_virt_addr_t) (MT_virt_addr_t) ud_buf->buf_data; \
desc->sg_entry.lkey = ud_buf->memhandle.lkey; \
}
int mca_ptl_ib_ud_cq_init(VAPI_hca_hndl_t, VAPI_cq_hndl_t*,
VAPI_cq_hndl_t*);
int mca_ptl_ib_ud_qp_init(VAPI_hca_hndl_t, VAPI_cq_hndl_t,
@ -56,8 +86,8 @@ int mca_ptl_ib_create_cq(VAPI_hca_hndl_t, VAPI_cq_hndl_t*);
int mca_ptl_ib_set_async_handler(VAPI_hca_hndl_t,
EVAPI_async_handler_hndl_t*);
int mca_ptl_ib_post_ud_recv(VAPI_hca_hndl_t, VAPI_qp_hndl_t,
mca_ptl_ib_ud_buf_t*);
int mca_ptl_ib_prep_ud_bufs(VAPI_hca_hndl_t, mca_ptl_ib_ud_buf_t**);
mca_ptl_ib_ud_buf_t*, int);
int mca_ptl_ib_prep_ud_bufs(VAPI_hca_hndl_t, mca_ptl_ib_ud_buf_t*, IB_wr_t, int);
int mca_ptl_ib_register_mem(VAPI_hca_hndl_t, void*, int,
vapi_memhandle_t*);
int mca_ptl_ib_set_comp_ev_hndl(VAPI_hca_hndl_t, VAPI_cq_hndl_t,
@ -65,7 +95,7 @@ int mca_ptl_ib_set_comp_ev_hndl(VAPI_hca_hndl_t, VAPI_cq_hndl_t,
EVAPI_compl_handler_hndl_t*);
int mca_ptl_ib_req_comp_notif(VAPI_hca_hndl_t,VAPI_cq_hndl_t);
int mca_ptl_ib_get_comp_ev_hndl(VAPI_completion_event_handler_t*);
int mca_ptl_ib_init_send(void*, VAPI_qp_hndl_t, int);
#endif /* MCA_PTL_IB_PRIV_H */

7
src/mca/ptl/ib/src/ptl_ib_proc.c
Просмотреть файл

@ -88,13 +88,11 @@ mca_ptl_ib_proc_t* mca_ptl_ib_proc_create(ompi_proc_t* ompi_proc)
mca_ptl_ib_proc_t* module_proc = NULL;
/*
module_proc = mca_ptl_ib_proc_lookup_ompi(ompi_proc);
if(module_proc != NULL) {
return module_proc;
}
*/
module_proc = OBJ_NEW(mca_ptl_ib_proc_t);
@ -160,12 +158,12 @@ mca_ptl_ib_proc_t* mca_ptl_ib_proc_create(ompi_proc_t* ompi_proc)
int mca_ptl_ib_proc_insert(mca_ptl_ib_proc_t* module_proc,
mca_ptl_base_peer_t* module_peer)
{
int i;
/* insert into peer array */
module_peer->peer_proc = module_proc;
module_proc->proc_peers[module_proc->proc_peer_count++] = module_peer;
#if 0 /* TODO: don't quite understand what this means for IB ptl.
will come back to it later */
/*
* Look through the proc instance for an address that is on the
* directly attached network. If we don't find one, pick the first
@ -178,6 +176,7 @@ int mca_ptl_ib_proc_insert(mca_ptl_ib_proc_t* module_proc,
module_peer->peer_addr = peer_addr;
}
#endif
return OMPI_SUCCESS;
}

24
src/mca/ptl/ib/src/ptl_ib_sendfrag.h
Просмотреть файл

@ -9,9 +9,7 @@
#include "ptl_ib_vapi.h"
extern ompi_class_t mca_ptl_ib_send_frag_t_class;
struct mca_ptl_base_peer_t;
OBJ_CLASS_DECLARATION(mca_ptl_ib_send_frag_t);
/**
* IB send fragment derived type.
@ -21,4 +19,24 @@ struct mca_ptl_ib_send_frag_t {
};
typedef struct mca_ptl_ib_send_frag_t mca_ptl_ib_send_frag_t;
/**
* Initialize a fragment descriptor.
*
* frag (IN) Fragment
* peer (IN) PTL peer addressing information
* request (IN) Send request
* offset (IN) Current offset into packed buffer
* size (IN/OUT) Requested size / actual size returned
* flags (IN)
*/
int mca_ptl_ib_send_frag_init(
mca_ptl_ib_send_frag_t*,
struct mca_ptl_base_peer_t*,
struct mca_pml_base_send_request_t*,
size_t offset,
size_t* size,
int flags);
#endif

7
src/mca/ptl/ib/src/ptl_ib_vapi.h
Просмотреть файл

@ -17,6 +17,13 @@
#define DEFAULT_PKEY_IX (0)
#define DEFAULT_PSN (0)
#define DEFAULT_TRAFFIC_CLASS (0)
#define DEFAULT_HOP_LIMIT (63)
#define DEFAULT_FLOW_LABEL (0)
#define DEFAULT_SERVICE_LEVEL (0)
#define DEFAULT_STATIC_RATE (0)
#define DEFAULT_SRC_PATH_BITS (0)
/* This is a convinence macro.
*
* ret : The value to return if call failed