* implement btl_put for Portals
This commit was SVN r7320.
Этот коммит содержится в:
родитель
c9fb1f32f2
Коммит
79f7ea6856
@ -249,6 +249,10 @@ mca_btl_portals_alloc(struct mca_btl_base_module_t* btl_base,
|
|||||||
frag->base.des_src_cnt = 1;
|
frag->base.des_src_cnt = 1;
|
||||||
frag->base.des_flags = 0;
|
frag->base.des_flags = 0;
|
||||||
|
|
||||||
|
/* can't setup off an alloc right now - we don't know how much the
|
||||||
|
caller will actually use */
|
||||||
|
frag->md_h = PTL_INVALID_HANDLE;
|
||||||
|
|
||||||
return &frag->base;
|
return &frag->base;
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -261,6 +265,10 @@ mca_btl_portals_free(struct mca_btl_base_module_t* btl_base,
|
|||||||
|
|
||||||
assert(&mca_btl_portals_module == (mca_btl_portals_module_t*) btl_base);
|
assert(&mca_btl_portals_module == (mca_btl_portals_module_t*) btl_base);
|
||||||
|
|
||||||
|
if (frag->md_h != PTL_INVALID_HANDLE) {
|
||||||
|
PtlMDUnlink(frag->md_h);
|
||||||
|
}
|
||||||
|
|
||||||
if (frag->size == 0) {
|
if (frag->size == 0) {
|
||||||
OMPI_BTL_PORTALS_FRAG_RETURN_USER(&mca_btl_portals_module.super, frag);
|
OMPI_BTL_PORTALS_FRAG_RETURN_USER(&mca_btl_portals_module.super, frag);
|
||||||
} else if (frag->size == mca_btl_portals_module.super.btl_eager_limit){
|
} else if (frag->size == mca_btl_portals_module.super.btl_eager_limit){
|
||||||
@ -344,12 +352,19 @@ mca_btl_portals_prepare_src(struct mca_btl_base_module_t* btl_base,
|
|||||||
frag->base.des_src_cnt = 1;
|
frag->base.des_src_cnt = 1;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/* clearly a send - delay setup of memory descriptor until send */
|
||||||
|
frag->md_h = PTL_INVALID_HANDLE;
|
||||||
|
|
||||||
|
|
||||||
} else {
|
} else {
|
||||||
/* no need to pack - we can send directly out of the user's
|
/* no need to pack - we can send directly out of the user's
|
||||||
buffer. If we have reserve space, use an eager fragment
|
buffer. If we have reserve space, use an eager fragment
|
||||||
and give the caller the eager space as reserve. If we have
|
and give the caller the eager space as reserve. If we have
|
||||||
no reserve space needs, use a user frag */
|
no reserve space needs, use a user frag */
|
||||||
if (0 == reserve) {
|
if (0 == reserve) {
|
||||||
|
ptl_md_t md;
|
||||||
|
ptl_handle_me_t me_h;
|
||||||
|
|
||||||
/* user frags are always setup to use only one fragment */
|
/* user frags are always setup to use only one fragment */
|
||||||
OMPI_BTL_PORTALS_FRAG_ALLOC_USER(&mca_btl_portals_module.super, frag, ret);
|
OMPI_BTL_PORTALS_FRAG_ALLOC_USER(&mca_btl_portals_module.super, frag, ret);
|
||||||
if(NULL == frag){
|
if(NULL == frag){
|
||||||
@ -363,8 +378,55 @@ mca_btl_portals_prepare_src(struct mca_btl_base_module_t* btl_base,
|
|||||||
|
|
||||||
frag->segments[0].seg_len = max_data;
|
frag->segments[0].seg_len = max_data;
|
||||||
frag->segments[0].seg_addr.pval = iov.iov_base;
|
frag->segments[0].seg_addr.pval = iov.iov_base;
|
||||||
|
frag->segments[0].seg_key.key64 = OPAL_THREAD_ADD64(&(mca_btl_portals_module.portals_rdma_key), 1);
|
||||||
frag->base.des_src_cnt = 1;
|
frag->base.des_src_cnt = 1;
|
||||||
|
|
||||||
|
/* either a put or get. figure out which later */
|
||||||
|
|
||||||
|
OPAL_OUTPUT_VERBOSE((90, mca_btl_portals_component.portals_output,
|
||||||
|
"rdma src posted for frag 0x%x, callback 0x%x, bits %lld",
|
||||||
|
frag, frag->base.des_cbfunc, frag->segments[0].seg_key.key64));
|
||||||
|
|
||||||
|
/* create a match entry */
|
||||||
|
ret = PtlMEAttach(mca_btl_portals_module.portals_ni_h,
|
||||||
|
OMPI_BTL_PORTALS_RDMA_TABLE_ID,
|
||||||
|
*((mca_btl_base_endpoint_t*) peer),
|
||||||
|
frag->segments[0].seg_key.key64, /* match */
|
||||||
|
0, /* ignore */
|
||||||
|
PTL_UNLINK,
|
||||||
|
PTL_INS_AFTER,
|
||||||
|
&me_h);
|
||||||
|
if (PTL_OK != ret) {
|
||||||
|
opal_output(mca_btl_portals_component.portals_output,
|
||||||
|
"Error creating rdma src ME: %d", ret);
|
||||||
|
OMPI_BTL_PORTALS_FRAG_RETURN_USER(&mca_btl_portals_module.super, frag);
|
||||||
|
return NULL;
|
||||||
|
}
|
||||||
|
|
||||||
|
/* setup the memory descriptor. RDMA should never need to be
|
||||||
|
retransmitted, so we set the threshold for the event it will
|
||||||
|
receive (PUT/GET START and END). No need to track the unlinks
|
||||||
|
later :) */
|
||||||
|
md.start = frag->segments[0].seg_addr.pval;
|
||||||
|
md.length = frag->segments[0].seg_len;
|
||||||
|
md.threshold = PTL_MD_THRESH_INF;
|
||||||
|
md.max_size = 0;
|
||||||
|
md.options = PTL_MD_OP_PUT | PTL_MD_OP_GET | PTL_MD_EVENT_START_DISABLE;
|
||||||
|
md.user_ptr = frag; /* keep a pointer to ourselves */
|
||||||
|
md.eq_handle = mca_btl_portals_module.portals_eq_handles[OMPI_BTL_PORTALS_EQ];
|
||||||
|
|
||||||
|
ret = PtlMDAttach(me_h,
|
||||||
|
md,
|
||||||
|
PTL_UNLINK,
|
||||||
|
&(frag->md_h));
|
||||||
|
if (PTL_OK != ret) {
|
||||||
|
opal_output(mca_btl_portals_component.portals_output,
|
||||||
|
"Error creating rdma src MD: %d", ret);
|
||||||
|
PtlMEUnlink(me_h);
|
||||||
|
OMPI_BTL_PORTALS_FRAG_RETURN_USER(&mca_btl_portals_module.super, frag);
|
||||||
|
return NULL;
|
||||||
|
}
|
||||||
|
|
||||||
} else {
|
} else {
|
||||||
OMPI_BTL_PORTALS_FRAG_ALLOC_EAGER(&mca_btl_portals_module, frag, ret);
|
OMPI_BTL_PORTALS_FRAG_ALLOC_EAGER(&mca_btl_portals_module, frag, ret);
|
||||||
if (NULL == frag) {
|
if (NULL == frag) {
|
||||||
@ -391,6 +453,9 @@ mca_btl_portals_prepare_src(struct mca_btl_base_module_t* btl_base,
|
|||||||
frag->iov[0].iov_len = frag->segments[0].seg_len;
|
frag->iov[0].iov_len = frag->segments[0].seg_len;
|
||||||
frag->iov[1].iov_base = frag->segments[1].seg_addr.pval;
|
frag->iov[1].iov_base = frag->segments[1].seg_addr.pval;
|
||||||
frag->iov[1].iov_len = frag->segments[1].seg_len;
|
frag->iov[1].iov_len = frag->segments[1].seg_len;
|
||||||
|
|
||||||
|
/* clearly a send - delay setup of memory descriptor until send */
|
||||||
|
frag->md_h = PTL_INVALID_HANDLE;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -414,7 +479,6 @@ mca_btl_portals_prepare_dst(struct mca_btl_base_module_t* btl_base,
|
|||||||
mca_btl_portals_frag_t* frag;
|
mca_btl_portals_frag_t* frag;
|
||||||
ptl_md_t md;
|
ptl_md_t md;
|
||||||
ptl_handle_me_t me_h;
|
ptl_handle_me_t me_h;
|
||||||
ptl_handle_md_t md_h;
|
|
||||||
int ret;
|
int ret;
|
||||||
|
|
||||||
assert(&mca_btl_portals_module == (mca_btl_portals_module_t*) btl_base);
|
assert(&mca_btl_portals_module == (mca_btl_portals_module_t*) btl_base);
|
||||||
@ -461,7 +525,7 @@ mca_btl_portals_prepare_dst(struct mca_btl_base_module_t* btl_base,
|
|||||||
later :) */
|
later :) */
|
||||||
md.start = frag->segments[0].seg_addr.pval;
|
md.start = frag->segments[0].seg_addr.pval;
|
||||||
md.length = frag->segments[0].seg_len;
|
md.length = frag->segments[0].seg_len;
|
||||||
md.threshold = 1; /* unlink after put */
|
md.threshold = PTL_MD_THRESH_INF;
|
||||||
md.max_size = 0;
|
md.max_size = 0;
|
||||||
md.options = PTL_MD_OP_PUT | PTL_MD_OP_GET | PTL_MD_EVENT_START_DISABLE;
|
md.options = PTL_MD_OP_PUT | PTL_MD_OP_GET | PTL_MD_EVENT_START_DISABLE;
|
||||||
md.user_ptr = frag; /* keep a pointer to ourselves */
|
md.user_ptr = frag; /* keep a pointer to ourselves */
|
||||||
@ -470,7 +534,7 @@ mca_btl_portals_prepare_dst(struct mca_btl_base_module_t* btl_base,
|
|||||||
ret = PtlMDAttach(me_h,
|
ret = PtlMDAttach(me_h,
|
||||||
md,
|
md,
|
||||||
PTL_UNLINK,
|
PTL_UNLINK,
|
||||||
&md_h);
|
&(frag->md_h));
|
||||||
if (PTL_OK != ret) {
|
if (PTL_OK != ret) {
|
||||||
opal_output(mca_btl_portals_component.portals_output,
|
opal_output(mca_btl_portals_component.portals_output,
|
||||||
"Error creating rdma dest MD: %d", ret);
|
"Error creating rdma dest MD: %d", ret);
|
||||||
|
@ -194,7 +194,7 @@ mca_btl_portals_component_open(void)
|
|||||||
&dummy);
|
&dummy);
|
||||||
mca_btl_portals_module.super.btl_bandwidth = dummy;
|
mca_btl_portals_module.super.btl_bandwidth = dummy;
|
||||||
|
|
||||||
mca_btl_portals_module.super.btl_flags = MCA_BTL_FLAGS_PUT | MCA_BTL_FLAGS_SEND_INPLACE;
|
mca_btl_portals_module.super.btl_flags = MCA_BTL_FLAGS_PUT | MCA_BTL_FLAGS_GET | MCA_BTL_FLAGS_SEND_INPLACE;
|
||||||
|
|
||||||
mca_btl_portals_module.portals_num_procs = 0;
|
mca_btl_portals_module.portals_num_procs = 0;
|
||||||
bzero(&(mca_btl_portals_module.portals_reg),
|
bzero(&(mca_btl_portals_module.portals_reg),
|
||||||
|
@ -36,6 +36,8 @@ struct mca_btl_portals_frag_t {
|
|||||||
enum { mca_btl_portals_frag_type_send,
|
enum { mca_btl_portals_frag_type_send,
|
||||||
mca_btl_portals_frag_type_recv,
|
mca_btl_portals_frag_type_recv,
|
||||||
mca_btl_portals_frag_type_rdma} type;
|
mca_btl_portals_frag_type_rdma} type;
|
||||||
|
/* handle to use for communication */
|
||||||
|
ptl_handle_md_t md_h;
|
||||||
size_t size;
|
size_t size;
|
||||||
|
|
||||||
};
|
};
|
||||||
|
@ -28,11 +28,10 @@ mca_btl_portals_put(struct mca_btl_base_module_t* btl_base,
|
|||||||
struct mca_btl_base_descriptor_t* descriptor)
|
struct mca_btl_base_descriptor_t* descriptor)
|
||||||
{
|
{
|
||||||
mca_btl_portals_frag_t *frag = (mca_btl_portals_frag_t*) descriptor;
|
mca_btl_portals_frag_t *frag = (mca_btl_portals_frag_t*) descriptor;
|
||||||
ptl_md_t md;
|
|
||||||
ptl_handle_md_t md_h;
|
|
||||||
int ret;
|
int ret;
|
||||||
|
|
||||||
assert(&mca_btl_portals_module == (mca_btl_portals_module_t*) btl_base);
|
assert(&mca_btl_portals_module == (mca_btl_portals_module_t*) btl_base);
|
||||||
|
assert(frag->md_h != PTL_INVALID_HANDLE);
|
||||||
|
|
||||||
frag->endpoint = btl_peer;
|
frag->endpoint = btl_peer;
|
||||||
frag->hdr.tag = MCA_BTL_TAG_MAX;
|
frag->hdr.tag = MCA_BTL_TAG_MAX;
|
||||||
@ -41,26 +40,7 @@ mca_btl_portals_put(struct mca_btl_base_module_t* btl_base,
|
|||||||
/* setup the send */
|
/* setup the send */
|
||||||
assert(1 == frag->base.des_src_cnt);
|
assert(1 == frag->base.des_src_cnt);
|
||||||
|
|
||||||
md.start = frag->segments[0].seg_addr.pval;
|
ret = PtlPut(frag->md_h,
|
||||||
md.length = frag->segments[0].seg_len;
|
|
||||||
md.threshold = 2; /* unlink after send & ack */
|
|
||||||
md.max_size = 0;
|
|
||||||
md.options = PTL_MD_EVENT_START_DISABLE;
|
|
||||||
md.user_ptr = frag; /* keep a pointer to ourselves */
|
|
||||||
md.eq_handle = mca_btl_portals_module.portals_eq_handles[OMPI_BTL_PORTALS_EQ];
|
|
||||||
|
|
||||||
/* make a free-floater */
|
|
||||||
ret = PtlMDBind(mca_btl_portals_module.portals_ni_h,
|
|
||||||
md,
|
|
||||||
PTL_UNLINK,
|
|
||||||
&md_h);
|
|
||||||
if (ret != PTL_OK) {
|
|
||||||
opal_output(mca_btl_portals_component.portals_output,
|
|
||||||
"PtlMDBind failed with error %d", ret);
|
|
||||||
return OMPI_ERROR;
|
|
||||||
}
|
|
||||||
|
|
||||||
ret = PtlPut(md_h,
|
|
||||||
PTL_ACK_REQ,
|
PTL_ACK_REQ,
|
||||||
*((mca_btl_base_endpoint_t*) btl_peer),
|
*((mca_btl_base_endpoint_t*) btl_peer),
|
||||||
OMPI_BTL_PORTALS_RDMA_TABLE_ID,
|
OMPI_BTL_PORTALS_RDMA_TABLE_ID,
|
||||||
@ -71,7 +51,7 @@ mca_btl_portals_put(struct mca_btl_base_module_t* btl_base,
|
|||||||
if (ret != PTL_OK) {
|
if (ret != PTL_OK) {
|
||||||
opal_output(mca_btl_portals_component.portals_output,
|
opal_output(mca_btl_portals_component.portals_output,
|
||||||
"PtlPut failed with error %d", ret);
|
"PtlPut failed with error %d", ret);
|
||||||
PtlMDUnlink(md_h);
|
PtlMDUnlink(frag->md_h);
|
||||||
return OMPI_ERROR;
|
return OMPI_ERROR;
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -85,37 +65,16 @@ mca_btl_portals_get(struct mca_btl_base_module_t* btl_base,
|
|||||||
struct mca_btl_base_descriptor_t* descriptor)
|
struct mca_btl_base_descriptor_t* descriptor)
|
||||||
{
|
{
|
||||||
mca_btl_portals_frag_t *frag = (mca_btl_portals_frag_t*) descriptor;
|
mca_btl_portals_frag_t *frag = (mca_btl_portals_frag_t*) descriptor;
|
||||||
ptl_md_t md;
|
|
||||||
ptl_handle_md_t md_h;
|
|
||||||
int ret;
|
int ret;
|
||||||
|
|
||||||
assert(&mca_btl_portals_module == (mca_btl_portals_module_t*) btl_base);
|
assert(&mca_btl_portals_module == (mca_btl_portals_module_t*) btl_base);
|
||||||
|
assert(frag->md_h != PTL_INVALID_HANDLE);
|
||||||
|
|
||||||
frag->endpoint = btl_peer;
|
frag->endpoint = btl_peer;
|
||||||
frag->hdr.tag = MCA_BTL_TAG_MAX;
|
frag->hdr.tag = MCA_BTL_TAG_MAX;
|
||||||
frag->type = mca_btl_portals_frag_type_rdma;
|
frag->type = mca_btl_portals_frag_type_rdma;
|
||||||
|
|
||||||
/* setup the get */
|
ret = PtlGet(frag->md_h,
|
||||||
md.start = frag->segments[0].seg_addr.pval;
|
|
||||||
md.length = frag->segments[0].seg_len;
|
|
||||||
md.threshold = 2; /* unlink after send & ack */
|
|
||||||
md.max_size = 0;
|
|
||||||
md.options = PTL_MD_EVENT_START_DISABLE;
|
|
||||||
md.user_ptr = frag; /* keep a pointer to ourselves */
|
|
||||||
md.eq_handle = mca_btl_portals_module.portals_eq_handles[OMPI_BTL_PORTALS_EQ];
|
|
||||||
|
|
||||||
/* make a free-floater */
|
|
||||||
ret = PtlMDBind(mca_btl_portals_module.portals_ni_h,
|
|
||||||
md,
|
|
||||||
PTL_UNLINK,
|
|
||||||
&md_h);
|
|
||||||
if (ret != PTL_OK) {
|
|
||||||
opal_output(mca_btl_portals_component.portals_output,
|
|
||||||
"PtlMDBind failed with error %d", ret);
|
|
||||||
return OMPI_ERROR;
|
|
||||||
}
|
|
||||||
|
|
||||||
ret = PtlGet(md_h,
|
|
||||||
*((mca_btl_base_endpoint_t*) btl_peer),
|
*((mca_btl_base_endpoint_t*) btl_peer),
|
||||||
OMPI_BTL_PORTALS_RDMA_TABLE_ID,
|
OMPI_BTL_PORTALS_RDMA_TABLE_ID,
|
||||||
0, /* ac_index - not used*/
|
0, /* ac_index - not used*/
|
||||||
@ -124,7 +83,7 @@ mca_btl_portals_get(struct mca_btl_base_module_t* btl_base,
|
|||||||
if (ret != PTL_OK) {
|
if (ret != PTL_OK) {
|
||||||
opal_output(mca_btl_portals_component.portals_output,
|
opal_output(mca_btl_portals_component.portals_output,
|
||||||
"PtlGet failed with error %d", ret);
|
"PtlGet failed with error %d", ret);
|
||||||
PtlMDUnlink(md_h);
|
PtlMDUnlink(frag->md_h);
|
||||||
return OMPI_ERROR;
|
return OMPI_ERROR;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -36,6 +36,7 @@ mca_btl_portals_send(struct mca_btl_base_module_t* btl_base,
|
|||||||
int ret;
|
int ret;
|
||||||
|
|
||||||
assert(&mca_btl_portals_module == (mca_btl_portals_module_t*) btl_base);
|
assert(&mca_btl_portals_module == (mca_btl_portals_module_t*) btl_base);
|
||||||
|
assert(frag->md_h == PTL_INVALID_HANDLE);
|
||||||
|
|
||||||
frag->endpoint = endpoint;
|
frag->endpoint = endpoint;
|
||||||
frag->hdr.tag = tag;
|
frag->hdr.tag = tag;
|
||||||
@ -57,8 +58,8 @@ mca_btl_portals_send(struct mca_btl_base_module_t* btl_base,
|
|||||||
|
|
||||||
ret = OMPI_SUCCESS;
|
ret = OMPI_SUCCESS;
|
||||||
} else {
|
} else {
|
||||||
ptl_handle_md_t md_h;
|
|
||||||
int ret;
|
int ret;
|
||||||
|
ptl_handle_md_t md_h;
|
||||||
|
|
||||||
/* setup the send */
|
/* setup the send */
|
||||||
if (1 == frag->base.des_src_cnt) {
|
if (1 == frag->base.des_src_cnt) {
|
||||||
@ -84,7 +85,7 @@ mca_btl_portals_send(struct mca_btl_base_module_t* btl_base,
|
|||||||
"PtlMDBind failed with error %d", ret);
|
"PtlMDBind failed with error %d", ret);
|
||||||
return OMPI_ERROR;
|
return OMPI_ERROR;
|
||||||
}
|
}
|
||||||
|
|
||||||
ret = PtlPut(md_h,
|
ret = PtlPut(md_h,
|
||||||
PTL_ACK_REQ,
|
PTL_ACK_REQ,
|
||||||
*((mca_btl_base_endpoint_t*) endpoint),
|
*((mca_btl_base_endpoint_t*) endpoint),
|
||||||
@ -95,7 +96,7 @@ mca_btl_portals_send(struct mca_btl_base_module_t* btl_base,
|
|||||||
frag->hdr.tag); /* hdr_data - tag */
|
frag->hdr.tag); /* hdr_data - tag */
|
||||||
if (ret != PTL_OK) {
|
if (ret != PTL_OK) {
|
||||||
opal_output(mca_btl_portals_component.portals_output,
|
opal_output(mca_btl_portals_component.portals_output,
|
||||||
"PtlPut failed with error %d", ret);
|
"send: PtlPut failed with error %d", ret);
|
||||||
PtlMDUnlink(md_h);
|
PtlMDUnlink(md_h);
|
||||||
return OMPI_ERROR;
|
return OMPI_ERROR;
|
||||||
}
|
}
|
||||||
|
Загрузка…
Ссылка в новой задаче
Block a user