ports/openmpi
ports/openmpi
1
1
Форкнуть 0
Код Релизы Активность

* implement btl_put for Portals

Просмотр исходного кода 
This commit was SVN r7320.
Этот коммит содержится в:
Brian Barrett 2005-09-12 20:24:43 +00:00
родитель c9fb1f32f2
Коммит 79f7ea6856
5 изменённых файлов: 80 добавлений и 54 удалений
Показать статистику Скачать Patch файл Скачать Diff файл Показать все файлы Свернуть все файлы

70
ompi/mca/btl/portals/btl_portals.c
Просмотреть файл

@ -249,6 +249,10 @@ mca_btl_portals_alloc(struct mca_btl_base_module_t* btl_base,
frag->base.des_src_cnt = 1;
frag->base.des_flags = 0;
/* can't setup off an alloc right now - we don't know how much the
caller will actually use */
frag->md_h = PTL_INVALID_HANDLE;
return &frag->base;
}
@ -261,6 +265,10 @@ mca_btl_portals_free(struct mca_btl_base_module_t* btl_base,
assert(&mca_btl_portals_module == (mca_btl_portals_module_t*) btl_base);
if (frag->md_h != PTL_INVALID_HANDLE) {
PtlMDUnlink(frag->md_h);
}
if (frag->size == 0) {
OMPI_BTL_PORTALS_FRAG_RETURN_USER(&mca_btl_portals_module.super, frag);
} else if (frag->size == mca_btl_portals_module.super.btl_eager_limit){
@ -344,12 +352,19 @@ mca_btl_portals_prepare_src(struct mca_btl_base_module_t* btl_base,
frag->base.des_src_cnt = 1;
}
/* clearly a send - delay setup of memory descriptor until send */
frag->md_h = PTL_INVALID_HANDLE;
} else {
/* no need to pack - we can send directly out of the user's
buffer. If we have reserve space, use an eager fragment
and give the caller the eager space as reserve. If we have
no reserve space needs, use a user frag */
if (0 == reserve) {
ptl_md_t md;
ptl_handle_me_t me_h;
/* user frags are always setup to use only one fragment */
OMPI_BTL_PORTALS_FRAG_ALLOC_USER(&mca_btl_portals_module.super, frag, ret);
if(NULL == frag){
@ -363,8 +378,55 @@ mca_btl_portals_prepare_src(struct mca_btl_base_module_t* btl_base,
frag->segments[0].seg_len = max_data;
frag->segments[0].seg_addr.pval = iov.iov_base;
frag->segments[0].seg_key.key64 = OPAL_THREAD_ADD64(&(mca_btl_portals_module.portals_rdma_key), 1);
frag->base.des_src_cnt = 1;
/* either a put or get. figure out which later */
OPAL_OUTPUT_VERBOSE((90, mca_btl_portals_component.portals_output,
"rdma src posted for frag 0x%x, callback 0x%x, bits %lld",
frag, frag->base.des_cbfunc, frag->segments[0].seg_key.key64));
/* create a match entry */
ret = PtlMEAttach(mca_btl_portals_module.portals_ni_h,
OMPI_BTL_PORTALS_RDMA_TABLE_ID,
*((mca_btl_base_endpoint_t*) peer),
frag->segments[0].seg_key.key64, /* match */
0, /* ignore */
PTL_UNLINK,
PTL_INS_AFTER,
&me_h);
if (PTL_OK != ret) {
opal_output(mca_btl_portals_component.portals_output,
"Error creating rdma src ME: %d", ret);
OMPI_BTL_PORTALS_FRAG_RETURN_USER(&mca_btl_portals_module.super, frag);
return NULL;
}
/* setup the memory descriptor. RDMA should never need to be
retransmitted, so we set the threshold for the event it will
receive (PUT/GET START and END). No need to track the unlinks
later :) */
md.start = frag->segments[0].seg_addr.pval;
md.length = frag->segments[0].seg_len;
md.threshold = PTL_MD_THRESH_INF;
md.max_size = 0;
md.options = PTL_MD_OP_PUT | PTL_MD_OP_GET | PTL_MD_EVENT_START_DISABLE;
md.user_ptr = frag; /* keep a pointer to ourselves */
md.eq_handle = mca_btl_portals_module.portals_eq_handles[OMPI_BTL_PORTALS_EQ];
ret = PtlMDAttach(me_h,
md,
PTL_UNLINK,
&(frag->md_h));
if (PTL_OK != ret) {
opal_output(mca_btl_portals_component.portals_output,
"Error creating rdma src MD: %d", ret);
PtlMEUnlink(me_h);
OMPI_BTL_PORTALS_FRAG_RETURN_USER(&mca_btl_portals_module.super, frag);
return NULL;
}
} else {
OMPI_BTL_PORTALS_FRAG_ALLOC_EAGER(&mca_btl_portals_module, frag, ret);
if (NULL == frag) {
@ -391,6 +453,9 @@ mca_btl_portals_prepare_src(struct mca_btl_base_module_t* btl_base,
frag->iov[0].iov_len = frag->segments[0].seg_len;
frag->iov[1].iov_base = frag->segments[1].seg_addr.pval;
frag->iov[1].iov_len = frag->segments[1].seg_len;
/* clearly a send - delay setup of memory descriptor until send */
frag->md_h = PTL_INVALID_HANDLE;
}
}
@ -414,7 +479,6 @@ mca_btl_portals_prepare_dst(struct mca_btl_base_module_t* btl_base,
mca_btl_portals_frag_t* frag;
ptl_md_t md;
ptl_handle_me_t me_h;
ptl_handle_md_t md_h;
int ret;
assert(&mca_btl_portals_module == (mca_btl_portals_module_t*) btl_base);
@ -461,7 +525,7 @@ mca_btl_portals_prepare_dst(struct mca_btl_base_module_t* btl_base,
later :) */
md.start = frag->segments[0].seg_addr.pval;
md.length = frag->segments[0].seg_len;
md.threshold = 1; /* unlink after put */
md.threshold = PTL_MD_THRESH_INF;
md.max_size = 0;
md.options = PTL_MD_OP_PUT | PTL_MD_OP_GET | PTL_MD_EVENT_START_DISABLE;
md.user_ptr = frag; /* keep a pointer to ourselves */
@ -470,7 +534,7 @@ mca_btl_portals_prepare_dst(struct mca_btl_base_module_t* btl_base,
ret = PtlMDAttach(me_h,
md,
PTL_UNLINK,
&md_h);
&(frag->md_h));
if (PTL_OK != ret) {
opal_output(mca_btl_portals_component.portals_output,
"Error creating rdma dest MD: %d", ret);

2
ompi/mca/btl/portals/btl_portals_component.c
Просмотреть файл

@ -194,7 +194,7 @@ mca_btl_portals_component_open(void)
&dummy);
mca_btl_portals_module.super.btl_bandwidth = dummy;
mca_btl_portals_module.super.btl_flags = MCA_BTL_FLAGS_PUT | MCA_BTL_FLAGS_SEND_INPLACE;
mca_btl_portals_module.super.btl_flags = MCA_BTL_FLAGS_PUT | MCA_BTL_FLAGS_GET | MCA_BTL_FLAGS_SEND_INPLACE;
mca_btl_portals_module.portals_num_procs = 0;
bzero(&(mca_btl_portals_module.portals_reg),

2
ompi/mca/btl/portals/btl_portals_frag.h
Просмотреть файл

@ -36,6 +36,8 @@ struct mca_btl_portals_frag_t {
enum { mca_btl_portals_frag_type_send,
mca_btl_portals_frag_type_recv,
mca_btl_portals_frag_type_rdma} type;
/* handle to use for communication */
ptl_handle_md_t md_h;
size_t size;
};

53
ompi/mca/btl/portals/btl_portals_rdma.c
Просмотреть файл

@ -28,11 +28,10 @@ mca_btl_portals_put(struct mca_btl_base_module_t* btl_base,
struct mca_btl_base_descriptor_t* descriptor)
{
mca_btl_portals_frag_t *frag = (mca_btl_portals_frag_t*) descriptor;
ptl_md_t md;
ptl_handle_md_t md_h;
int ret;
assert(&mca_btl_portals_module == (mca_btl_portals_module_t*) btl_base);
assert(frag->md_h != PTL_INVALID_HANDLE);
frag->endpoint = btl_peer;
frag->hdr.tag = MCA_BTL_TAG_MAX;
@ -41,26 +40,7 @@ mca_btl_portals_put(struct mca_btl_base_module_t* btl_base,
/* setup the send */
assert(1 == frag->base.des_src_cnt);
md.start = frag->segments[0].seg_addr.pval;
md.length = frag->segments[0].seg_len;
md.threshold = 2; /* unlink after send & ack */
md.max_size = 0;
md.options = PTL_MD_EVENT_START_DISABLE;
md.user_ptr = frag; /* keep a pointer to ourselves */
md.eq_handle = mca_btl_portals_module.portals_eq_handles[OMPI_BTL_PORTALS_EQ];
/* make a free-floater */
ret = PtlMDBind(mca_btl_portals_module.portals_ni_h,
md,
PTL_UNLINK,
&md_h);
if (ret != PTL_OK) {
opal_output(mca_btl_portals_component.portals_output,
"PtlMDBind failed with error %d", ret);
return OMPI_ERROR;
}
ret = PtlPut(md_h,
ret = PtlPut(frag->md_h,
PTL_ACK_REQ,
*((mca_btl_base_endpoint_t*) btl_peer),
OMPI_BTL_PORTALS_RDMA_TABLE_ID,
@ -71,7 +51,7 @@ mca_btl_portals_put(struct mca_btl_base_module_t* btl_base,
if (ret != PTL_OK) {
opal_output(mca_btl_portals_component.portals_output,
"PtlPut failed with error %d", ret);
PtlMDUnlink(md_h);
PtlMDUnlink(frag->md_h);
return OMPI_ERROR;
}
@ -85,37 +65,16 @@ mca_btl_portals_get(struct mca_btl_base_module_t* btl_base,
struct mca_btl_base_descriptor_t* descriptor)
{
mca_btl_portals_frag_t *frag = (mca_btl_portals_frag_t*) descriptor;
ptl_md_t md;
ptl_handle_md_t md_h;
int ret;
assert(&mca_btl_portals_module == (mca_btl_portals_module_t*) btl_base);
assert(frag->md_h != PTL_INVALID_HANDLE);
frag->endpoint = btl_peer;
frag->hdr.tag = MCA_BTL_TAG_MAX;
frag->type = mca_btl_portals_frag_type_rdma;
/* setup the get */
md.start = frag->segments[0].seg_addr.pval;
md.length = frag->segments[0].seg_len;
md.threshold = 2; /* unlink after send & ack */
md.max_size = 0;
md.options = PTL_MD_EVENT_START_DISABLE;
md.user_ptr = frag; /* keep a pointer to ourselves */
md.eq_handle = mca_btl_portals_module.portals_eq_handles[OMPI_BTL_PORTALS_EQ];
/* make a free-floater */
ret = PtlMDBind(mca_btl_portals_module.portals_ni_h,
md,
PTL_UNLINK,
&md_h);
if (ret != PTL_OK) {
opal_output(mca_btl_portals_component.portals_output,
"PtlMDBind failed with error %d", ret);
return OMPI_ERROR;
}
ret = PtlGet(md_h,
ret = PtlGet(frag->md_h,
*((mca_btl_base_endpoint_t*) btl_peer),
OMPI_BTL_PORTALS_RDMA_TABLE_ID,
0, /* ac_index - not used*/
@ -124,7 +83,7 @@ mca_btl_portals_get(struct mca_btl_base_module_t* btl_base,
if (ret != PTL_OK) {
opal_output(mca_btl_portals_component.portals_output,
"PtlGet failed with error %d", ret);
PtlMDUnlink(md_h);
PtlMDUnlink(frag->md_h);
return OMPI_ERROR;
}

7
ompi/mca/btl/portals/btl_portals_send.c
Просмотреть файл

@ -36,6 +36,7 @@ mca_btl_portals_send(struct mca_btl_base_module_t* btl_base,
int ret;
assert(&mca_btl_portals_module == (mca_btl_portals_module_t*) btl_base);
assert(frag->md_h == PTL_INVALID_HANDLE);
frag->endpoint = endpoint;
frag->hdr.tag = tag;
@ -57,8 +58,8 @@ mca_btl_portals_send(struct mca_btl_base_module_t* btl_base,
ret = OMPI_SUCCESS;
} else {
ptl_handle_md_t md_h;
int ret;
ptl_handle_md_t md_h;
/* setup the send */
if (1 == frag->base.des_src_cnt) {
@ -84,7 +85,7 @@ mca_btl_portals_send(struct mca_btl_base_module_t* btl_base,
"PtlMDBind failed with error %d", ret);
return OMPI_ERROR;
}
ret = PtlPut(md_h,
PTL_ACK_REQ,
*((mca_btl_base_endpoint_t*) endpoint),
@ -95,7 +96,7 @@ mca_btl_portals_send(struct mca_btl_base_module_t* btl_base,
frag->hdr.tag); /* hdr_data - tag */
if (ret != PTL_OK) {
opal_output(mca_btl_portals_component.portals_output,
"PtlPut failed with error %d", ret);
"send: PtlPut failed with error %d", ret);
PtlMDUnlink(md_h);
return OMPI_ERROR;
}