ports/openmpi
ports/openmpi
1
1
Форкнуть 0
Код Релизы Активность

btl-portals4: fix send failure for messages larger than eager_size

Просмотр исходного кода 
In the large message case, the sender issues a PtlMEAppend() in
order to generate events when the receiver issues a PtlGet().  This
commit moves the PtlMEAppend() from mca_btl_portals4_prepare_src()
to mca_btl_portals4_register_mem() which is the way it's done in
BTL 3.0.
Этот коммит содержится в:
Todd Kordenbrock 2015-04-10 14:39:20 -05:00
родитель 5ea1f1c12b
Коммит bca2522d8b
3 изменённых файлов: 53 добавлений и 48 удалений
Показать статистику Скачать Patch файл Скачать Diff файл Показать все файлы Свернуть все файлы

87
opal/mca/btl/portals4/btl_portals4.c
Просмотреть файл

@ -327,7 +327,6 @@ mca_btl_portals4_prepare_src(struct mca_btl_base_module_t* btl_base,
} else {
/* no need to pack - rdma operation out of user's buffer */
ptl_me_t me;
/* reserve space in the event queue for rdma operations immediately */
while (OPAL_THREAD_ADD32(&portals4_btl->portals_outstanding_ops, 1) >
@ -366,48 +365,6 @@ mca_btl_portals4_prepare_src(struct mca_btl_base_module_t* btl_base,
(unsigned long) frag,
(unsigned long) frag->base.des_cbfunc,
frag->segments[0].key, flags));
/* create a match entry */
me.start = frag->segments[0].base.seg_addr.pval;
me.length = frag->segments[0].base.seg_len;
me.ct_handle = PTL_CT_NONE;
me.min_free = 0;
me.uid = PTL_UID_ANY;
me.options = PTL_ME_OP_GET | PTL_ME_USE_ONCE |
PTL_ME_EVENT_LINK_DISABLE |
PTL_ME_EVENT_COMM_DISABLE |
PTL_ME_EVENT_UNLINK_DISABLE;
if (mca_btl_portals4_component.use_logical) {
me.match_id.rank = peer->ptl_proc.rank;
} else {
me.match_id.phys.nid = peer->ptl_proc.phys.nid;
me.match_id.phys.pid = peer->ptl_proc.phys.pid;
}
me.match_bits = frag->segments[0].key;
me.ignore_bits = BTL_PORTALS4_PROTOCOL_MASK |
BTL_PORTALS4_CONTEXT_MASK |
BTL_PORTALS4_SOURCE_MASK;
me.ignore_bits = 0;
ret = PtlMEAppend(portals4_btl->portals_ni_h,
portals4_btl->recv_idx,
&me,
PTL_PRIORITY_LIST,
frag,
&(frag->me_h));
if (PTL_OK != ret) {
opal_output_verbose(1, opal_btl_base_framework.framework_output,
"%s:%d: PtlMEAppend failed: %d\n",
__FILE__, __LINE__, ret);
OPAL_BTL_PORTALS4_FRAG_RETURN_USER(portals4_btl, frag);
OPAL_THREAD_ADD32(&portals4_btl->portals_outstanding_ops, -1);
return NULL;
}
OPAL_OUTPUT_VERBOSE((90, opal_btl_base_framework.framework_output,
"PtlMEAppend (prepare_src) frag=%p, me_h=%d start=%p length=%ld rank=%x nid=%x pid=%x match_bits=%lx\n",
(void *)frag, frag->me_h, me.start, me.length,
me.match_id.rank, me.match_id.phys.nid, me.match_id.phys.pid, me.match_bits));
}
frag->base.des_segments = &frag->segments[0].base;
@ -425,6 +382,8 @@ mca_btl_portals4_register_mem(mca_btl_base_module_t *btl_base,
{
struct mca_btl_portals4_module_t *portals4_btl = (struct mca_btl_portals4_module_t*) btl_base;
mca_btl_base_registration_handle_t *handle = NULL;
ptl_me_t me;
int ret;
handle = (mca_btl_base_registration_handle_t *)malloc(sizeof(mca_btl_base_registration_handle_t));
if (!handle) {
@ -437,6 +396,48 @@ mca_btl_portals4_register_mem(mca_btl_base_module_t *btl_base,
"mca_btl_portals4_register_mem NI=%d base=%p size=%ld handle=%p key=%ld\n",
portals4_btl->interface_num, base, size, (void *)handle, handle->key));
if (MCA_BTL_FLAGS_PUT == flags) {
/* create a match entry */
me.start = base;
me.length = size;
me.ct_handle = PTL_CT_NONE;
me.min_free = 0;
me.uid = PTL_UID_ANY;
me.options = PTL_ME_OP_GET | PTL_ME_USE_ONCE |
PTL_ME_EVENT_LINK_DISABLE |
PTL_ME_EVENT_COMM_DISABLE |
PTL_ME_EVENT_UNLINK_DISABLE;
if (mca_btl_portals4_component.use_logical) {
me.match_id.rank = endpoint->ptl_proc.rank;
} else {
me.match_id.phys.nid = endpoint->ptl_proc.phys.nid;
me.match_id.phys.pid = endpoint->ptl_proc.phys.pid;
}
me.match_bits = handle->key;
me.ignore_bits = BTL_PORTALS4_PROTOCOL_MASK |
BTL_PORTALS4_CONTEXT_MASK |
BTL_PORTALS4_SOURCE_MASK;
me.ignore_bits = 0;
ret = PtlMEAppend(portals4_btl->portals_ni_h,
portals4_btl->recv_idx,
&me,
PTL_PRIORITY_LIST,
handle,
&(handle->me_h));
if (PTL_OK != ret) {
opal_output_verbose(1, opal_btl_base_framework.framework_output,
"%s:%d: PtlMEAppend failed: %d\n",
__FILE__, __LINE__, ret);
OPAL_THREAD_ADD32(&portals4_btl->portals_outstanding_ops, -1);
return NULL;
}
OPAL_OUTPUT_VERBOSE((90, opal_btl_base_framework.framework_output,
"PtlMEAppend (mca_btl_portals4_register_mem) handle=%p, me_h=%d start=%p length=%ld rank=%x nid=%x pid=%x match_bits=%lx\n",
(void *)handle, handle->me_h, me.start, me.length,
me.match_id.rank, me.match_id.phys.nid, me.match_id.phys.pid, me.match_bits));
}
return handle;
}

2
opal/mca/btl/portals4/btl_portals4.h
Просмотреть файл

@ -284,6 +284,8 @@ int mca_btl_portals4_get_error(int ptl_error);
struct mca_btl_base_registration_handle_t {
/** Portals4 match bits */
ptl_match_bits_t key;
/** Portals4 me_h */
ptl_handle_me_t me_h;
};
/*

12
opal/mca/btl/portals4/btl_portals4_rdma.c
Просмотреть файл

@ -67,10 +67,8 @@ mca_btl_portals4_get(struct mca_btl_base_module_t* btl_base,
return OPAL_ERROR;
}
OPAL_OUTPUT_VERBOSE((90, opal_btl_base_framework.framework_output,
"mca_btl_portals4_prepare_src: Incrementing portals_outstanding_ops=%d\n", portals4_btl->portals_outstanding_ops));
OPAL_OUTPUT_VERBOSE((90, opal_btl_base_framework.framework_output,
"mca_btl_portals4_get frag=%p\n", (void *)frag));
"mca_btl_portals4_get: Incrementing portals_outstanding_ops=%d frag=%p",
portals4_btl->portals_outstanding_ops, (void *)frag));
frag->rdma_cb.func = cbfunc;
frag->rdma_cb.context = cbcontext;
@ -101,6 +99,10 @@ mca_btl_portals4_get(struct mca_btl_base_module_t* btl_base,
frag->match_bits = remote_handle->key;
frag->length = md.length;
frag->peer_proc = btl_peer->ptl_proc;
OPAL_OUTPUT_VERBOSE((90, opal_btl_base_framework.framework_output, "PtlGet start=%p length=%ld nid=%x pid=%x match_bits=%lx\n",
md.start, md.length, btl_peer->ptl_proc.phys.nid, btl_peer->ptl_proc.phys.pid, frag->match_bits));
ret = PtlGet(frag->md_h,
0,
md.length,
@ -117,7 +119,7 @@ mca_btl_portals4_get(struct mca_btl_base_module_t* btl_base,
frag->md_h = PTL_INVALID_HANDLE;
return OPAL_ERROR;
}
OPAL_OUTPUT_VERBOSE((90, opal_btl_base_framework.framework_output, "PtlGet start=%p length=%ld nid=%x pid=%x match_bits=%lx\n",
OPAL_OUTPUT_VERBOSE((90, opal_btl_base_framework.framework_output, "SUCCESS: PtlGet start=%p length=%ld nid=%x pid=%x match_bits=%lx\n",
md.start, md.length, btl_peer->ptl_proc.phys.nid, btl_peer->ptl_proc.phys.pid, frag->match_bits));
return OPAL_SUCCESS;