ports/openmpi
ports/openmpi
1
1
Форкнуть 0
Код Релизы Активность

Fixed bug trac #213 by moving the udapl btl header to being a footer.

Просмотр исходного кода 
Also fixed bug trac #346.

This commit was SVN r11760.
Этот коммит содержится в:
Terry Dontje 2006-09-22 19:28:09 +00:00
родитель 3c814fdd23
Коммит d636db5832
5 изменённых файлов: 66 добавлений и 49 удалений
Показать статистику Скачать Patch файл Скачать Diff файл Показать все файлы Свернуть все файлы

38
ompi/mca/btl/udapl/btl_udapl.c
Просмотреть файл

@ -63,7 +63,7 @@ mca_btl_udapl_module_t mca_btl_udapl_module = {
NULL, /* get */ NULL, /* get */
mca_btl_base_dump, mca_btl_base_dump,
NULL, /* mpool */ NULL, /* mpool */
NULL /* register error cb */ NULL /* register error cb */
} }
}; };
@ -347,9 +347,11 @@ mca_btl_base_descriptor_t* mca_btl_udapl_alloc(
/* Set up the LMR triplet from the frag segment */ /* Set up the LMR triplet from the frag segment */
/* Note that this triplet defines a sub-region of a registered LMR */ /* Note that this triplet defines a sub-region of a registered LMR */
frag->triplet.virtual_address = (DAT_VADDR)frag->hdr; frag->triplet.virtual_address = (DAT_VADDR)frag->segment.seg_addr.pval;
frag->ftr = (mca_btl_udapl_footer_t *)
((char *)frag->segment.seg_addr.pval + frag->segment.seg_len);
frag->triplet.segment_length = frag->triplet.segment_length =
frag->segment.seg_len + sizeof(mca_btl_base_header_t); frag->segment.seg_len + sizeof(mca_btl_udapl_footer_t);
assert(frag->triplet.lmr_context == assert(frag->triplet.lmr_context ==
((mca_mpool_udapl_registration_t*)frag->registration)->lmr_triplet.lmr_context); ((mca_mpool_udapl_registration_t*)frag->registration)->lmr_triplet.lmr_context);
@ -486,7 +488,7 @@ mca_btl_base_descriptor_t* mca_btl_udapl_prepare_src(
frag->registration = registration; frag->registration = registration;
frag->triplet.lmr_context = frag->triplet.lmr_context =
((mca_mpool_udapl_registration_t*)registration)->lmr_triplet.lmr_context; ((mca_mpool_udapl_registration_t*)registration)->lmr_triplet.lmr_context;
/* TODO - should our base addr be frag->hdr? */ /* TODO - should our base addr be frag->ftr? */
frag->segment.seg_len = max_data; frag->segment.seg_len = max_data;
frag->segment.seg_addr.pval = iov.iov_base; frag->segment.seg_addr.pval = iov.iov_base;
frag->triplet.segment_length = max_data; frag->triplet.segment_length = max_data;
@ -506,7 +508,7 @@ mca_btl_base_descriptor_t* mca_btl_udapl_prepare_src(
} }
iov.iov_len = max_data; iov.iov_len = max_data;
iov.iov_base = (unsigned char*) frag->segment.seg_addr.pval + reserve; iov.iov_base = (char *) frag->segment.seg_addr.pval + reserve;
rc = ompi_convertor_pack(convertor, rc = ompi_convertor_pack(convertor,
&iov, &iov_count, &max_data, &free_after); &iov, &iov_count, &max_data, &free_after);
@ -515,11 +517,6 @@ mca_btl_base_descriptor_t* mca_btl_udapl_prepare_src(
MCA_BTL_UDAPL_FRAG_RETURN_EAGER(btl, frag); MCA_BTL_UDAPL_FRAG_RETURN_EAGER(btl, frag);
return NULL; return NULL;
} }
frag->segment.seg_len = max_data + reserve;
frag->triplet.segment_length =
max_data + reserve + sizeof(mca_btl_base_header_t);
frag->triplet.virtual_address = (DAT_VADDR)frag->hdr;
} }
/* /*
@ -537,7 +534,7 @@ mca_btl_base_descriptor_t* mca_btl_udapl_prepare_src(
} }
iov.iov_len = max_data; iov.iov_len = max_data;
iov.iov_base = (unsigned char*) frag->segment.seg_addr.pval + reserve; iov.iov_base = (char *) frag->segment.seg_addr.pval + reserve;
rc = ompi_convertor_pack(convertor, rc = ompi_convertor_pack(convertor,
&iov, &iov_count, &max_data, &free_after); &iov, &iov_count, &max_data, &free_after);
@ -547,14 +544,17 @@ mca_btl_base_descriptor_t* mca_btl_udapl_prepare_src(
MCA_BTL_UDAPL_FRAG_RETURN_MAX(btl, frag); MCA_BTL_UDAPL_FRAG_RETURN_MAX(btl, frag);
return NULL; return NULL;
} }
/* TODO - pull this out of the if statements. */
frag->segment.seg_len = max_data + reserve;
frag->triplet.segment_length =
max_data + reserve + sizeof(mca_btl_base_header_t);
frag->triplet.virtual_address = (DAT_VADDR)frag->hdr;
} }
/* setup lengths and addresses to send out data */
frag->segment.seg_len = max_data + reserve;
frag->triplet.segment_length =
max_data + reserve + sizeof(mca_btl_udapl_footer_t);
frag->triplet.virtual_address = (DAT_VADDR)frag->segment.seg_addr.pval;
frag->ftr = (mca_btl_udapl_footer_t *)
((char *)frag->segment.seg_addr.pval + frag->segment.seg_len);
/* initialize base descriptor */
frag->base.des_src = &frag->segment; frag->base.des_src = &frag->segment;
frag->base.des_src_cnt = 1; frag->base.des_src_cnt = 1;
frag->base.des_dst = NULL; frag->base.des_dst = NULL;
@ -656,7 +656,9 @@ int mca_btl_udapl_send(
frag->btl = (mca_btl_udapl_module_t*)btl; frag->btl = (mca_btl_udapl_module_t*)btl;
frag->endpoint = endpoint; frag->endpoint = endpoint;
frag->hdr->tag = tag; frag->ftr = (mca_btl_udapl_footer_t *)
((char *)frag->segment.seg_addr.pval + frag->segment.seg_len);
frag->ftr->tag = tag;
frag->type = MCA_BTL_UDAPL_SEND; frag->type = MCA_BTL_UDAPL_SEND;
/* TODO - will inlining this give worthwhile performance? */ /* TODO - will inlining this give worthwhile performance? */

34
ompi/mca/btl/udapl/btl_udapl_component.c
Просмотреть файл

@ -54,7 +54,7 @@ mca_btl_udapl_component_t mca_btl_udapl_component = {
/* Indicate that we are a pml v1.0.0 component (which also implies a /* Indicate that we are a pml v1.0.0 component (which also implies a
specific MCA version) */ specific MCA version) */
MCA_BTL_BASE_VERSION_1_0_1, MCA_BTL_BASE_VERSION_1_0_0,
"udapl", /* MCA component name */ "udapl", /* MCA component name */
OMPI_MAJOR_VERSION, /* MCA component major version */ OMPI_MAJOR_VERSION, /* MCA component major version */
@ -182,12 +182,12 @@ int mca_btl_udapl_component_open(void)
mca_btl_udapl_component.udapl_eager_frag_size = mca_btl_udapl_component.udapl_eager_frag_size =
mca_btl_udapl_module.super.btl_eager_limit; mca_btl_udapl_module.super.btl_eager_limit;
mca_btl_udapl_module.super.btl_eager_limit -= mca_btl_udapl_module.super.btl_eager_limit -=
sizeof(mca_btl_base_header_t); sizeof(mca_btl_udapl_footer_t);
mca_btl_udapl_component.udapl_max_frag_size = mca_btl_udapl_component.udapl_max_frag_size =
mca_btl_udapl_module.super.btl_max_send_size; mca_btl_udapl_module.super.btl_max_send_size;
mca_btl_udapl_module.super.btl_max_send_size -= mca_btl_udapl_module.super.btl_max_send_size -=
sizeof(mca_btl_base_header_t); sizeof(mca_btl_udapl_footer_t);
/* leave pinned option */ /* leave pinned option */
@ -412,6 +412,7 @@ static inline int mca_btl_udapl_sendrecv(mca_btl_udapl_module_t* btl,
memcpy(frag->segment.seg_addr.pval, memcpy(frag->segment.seg_addr.pval,
&btl->udapl_addr, sizeof(mca_btl_udapl_addr_t)); &btl->udapl_addr, sizeof(mca_btl_udapl_addr_t));
frag->type = MCA_BTL_UDAPL_CONN_SEND; frag->type = MCA_BTL_UDAPL_CONN_SEND;
rc = dat_ep_post_send(endpoint, 1, rc = dat_ep_post_send(endpoint, 1,
@ -498,7 +499,7 @@ int mca_btl_udapl_component_progress()
assert(frag->triplet.segment_length == assert(frag->triplet.segment_length ==
frag->segment.seg_len + frag->segment.seg_len +
sizeof(mca_btl_base_header_t)); sizeof(mca_btl_udapl_footer_t));
cookie.as_ptr = frag; cookie.as_ptr = frag;
dat_ep_post_send(endpoint->endpoint_eager, dat_ep_post_send(endpoint->endpoint_eager,
@ -521,7 +522,7 @@ int mca_btl_udapl_component_progress()
assert(frag->triplet.segment_length == assert(frag->triplet.segment_length ==
frag->segment.seg_len + frag->segment.seg_len +
sizeof(mca_btl_base_header_t)); sizeof(mca_btl_udapl_footer_t));
cookie.as_ptr = frag; cookie.as_ptr = frag;
dat_ep_post_send(endpoint->endpoint_max, dat_ep_post_send(endpoint->endpoint_max,
@ -537,8 +538,7 @@ int mca_btl_udapl_component_progress()
} }
case MCA_BTL_UDAPL_RECV: case MCA_BTL_UDAPL_RECV:
{ {
mca_btl_base_recv_reg_t* reg = mca_btl_base_recv_reg_t* reg;
&btl->udapl_reg[frag->hdr->tag];
assert(frag->base.des_dst == &frag->segment); assert(frag->base.des_dst == &frag->segment);
assert(frag->base.des_dst_cnt == 1); assert(frag->base.des_dst_cnt == 1);
@ -546,26 +546,26 @@ int mca_btl_udapl_component_progress()
assert(frag->base.des_src_cnt == 0); assert(frag->base.des_src_cnt == 0);
assert(frag->type == MCA_BTL_UDAPL_RECV); assert(frag->type == MCA_BTL_UDAPL_RECV);
assert(frag->triplet.virtual_address == assert(frag->triplet.virtual_address ==
(DAT_VADDR)frag->hdr); (DAT_VADDR)frag->segment.seg_addr.pval);
assert(frag->triplet.segment_length == frag->size); assert(frag->triplet.segment_length == frag->size);
assert(frag->btl == btl); assert(frag->btl == btl);
/*OPAL_OUTPUT((0, "btl_udapl UDAPL_RECV %d", /* setup frag ftr location and do callback */
dto->transfered_length));*/
frag->segment.seg_addr.pval = frag->hdr + 1;
frag->segment.seg_len = dto->transfered_length - frag->segment.seg_len = dto->transfered_length -
sizeof(mca_btl_base_header_t); sizeof(mca_btl_udapl_footer_t);
frag->ftr = (mca_btl_udapl_footer_t *)
((char *)frag->segment.seg_addr.pval +
frag->segment.seg_len);
reg = &btl->udapl_reg[frag->ftr->tag];
OPAL_THREAD_UNLOCK(&mca_btl_udapl_component.udapl_lock); OPAL_THREAD_UNLOCK(&mca_btl_udapl_component.udapl_lock);
reg->cbfunc(&btl->super, reg->cbfunc(&btl->super,
frag->hdr->tag, &frag->base, reg->cbdata); frag->ftr->tag, &frag->base, reg->cbdata);
OPAL_THREAD_LOCK(&mca_btl_udapl_component.udapl_lock); OPAL_THREAD_LOCK(&mca_btl_udapl_component.udapl_lock);
/* Repost the frag */ /* Repost the frag */
frag->segment.seg_addr.pval = frag->hdr; frag->ftr = frag->segment.seg_addr.pval;
frag->segment.seg_len = frag->segment.seg_len =
frag->size - sizeof(mca_btl_base_header_t); frag->size - sizeof(mca_btl_udapl_footer_t);
frag->base.des_flags = 0; frag->base.des_flags = 0;
if(frag->size == if(frag->size ==

21
ompi/mca/btl/udapl/btl_udapl_endpoint.c
Просмотреть файл

@ -11,6 +11,8 @@
* All rights reserved. * All rights reserved.
* Copyright (c) 2006 Sandia National Laboratories. All rights * Copyright (c) 2006 Sandia National Laboratories. All rights
* reserved. * reserved.
* Copyright (c) 2006 Sun Microsystems, Inc. All rights reserved.
*
* $COPYRIGHT$ * $COPYRIGHT$
* *
* Additional copyrights may follow * Additional copyrights may follow
@ -55,7 +57,7 @@ int mca_btl_udapl_endpoint_send(mca_btl_base_endpoint_t* endpoint,
/* Fix up the segment length before we do anything with the frag */ /* Fix up the segment length before we do anything with the frag */
frag->triplet.segment_length = frag->triplet.segment_length =
frag->segment.seg_len + sizeof(mca_btl_base_header_t); frag->segment.seg_len + sizeof(mca_btl_udapl_footer_t);
OPAL_THREAD_LOCK(&endpoint->endpoint_lock); OPAL_THREAD_LOCK(&endpoint->endpoint_lock);
switch(endpoint->endpoint_state) { switch(endpoint->endpoint_state) {
@ -179,7 +181,7 @@ void mca_btl_udapl_endpoint_recv(int status, orte_process_name_t* endpoint,
mca_btl_udapl_addr_t addr; mca_btl_udapl_addr_t addr;
mca_btl_udapl_proc_t* proc; mca_btl_udapl_proc_t* proc;
mca_btl_base_endpoint_t* ep; mca_btl_base_endpoint_t* ep;
size_t cnt = 1; int32_t cnt = 1;
size_t i; size_t i;
int rc; int rc;
@ -286,6 +288,8 @@ failure_create:
/* /*
* Finish establishing a connection * Finish establishing a connection
* Note that this routine expects that the mca_btl_udapl_component.udapl.lock
* has been acquired by the callee.
*/ */
int mca_btl_udapl_endpoint_finish_connect(struct mca_btl_udapl_module_t* btl, int mca_btl_udapl_endpoint_finish_connect(struct mca_btl_udapl_module_t* btl,
@ -311,7 +315,6 @@ int mca_btl_udapl_endpoint_finish_connect(struct mca_btl_udapl_module_t* btl,
/* TODO - Check that the DAT_CONN_QUAL's match too */ /* TODO - Check that the DAT_CONN_QUAL's match too */
if(ep->endpoint_btl == btl && if(ep->endpoint_btl == btl &&
!memcmp(addr, &ep->endpoint_addr, sizeof(DAT_SOCK_ADDR))) { !memcmp(addr, &ep->endpoint_addr, sizeof(DAT_SOCK_ADDR))) {
OPAL_THREAD_LOCK(&ep->endpoint_lock); OPAL_THREAD_LOCK(&ep->endpoint_lock);
if(MCA_BTL_UDAPL_CONN_EAGER == ep->endpoint_state) { if(MCA_BTL_UDAPL_CONN_EAGER == ep->endpoint_state) {
ep->endpoint_eager = endpoint; ep->endpoint_eager = endpoint;
@ -324,7 +327,6 @@ int mca_btl_udapl_endpoint_finish_connect(struct mca_btl_udapl_module_t* btl,
ep->endpoint_state)); ep->endpoint_state));
return OMPI_ERROR; return OMPI_ERROR;
} }
return rc; return rc;
} }
} }
@ -401,9 +403,10 @@ static int mca_btl_udapl_endpoint_finish_max(mca_btl_udapl_endpoint_t* endpoint)
opal_list_remove_first(&endpoint->endpoint_eager_frags))) { opal_list_remove_first(&endpoint->endpoint_eager_frags))) {
cookie.as_ptr = frag; cookie.as_ptr = frag;
assert(frag->triplet.virtual_address == (DAT_VADDR)frag->hdr); assert(frag->triplet.virtual_address ==
(DAT_VADDR)frag->segment.seg_addr.pval);
assert(frag->triplet.segment_length == assert(frag->triplet.segment_length ==
frag->segment.seg_len + sizeof(mca_btl_base_header_t)); frag->segment.seg_len + sizeof(mca_btl_udapl_footer_t));
assert(frag->size == assert(frag->size ==
mca_btl_udapl_component.udapl_eager_frag_size); mca_btl_udapl_component.udapl_eager_frag_size);
rc = dat_ep_post_send(endpoint->endpoint_eager, 1, rc = dat_ep_post_send(endpoint->endpoint_eager, 1,
@ -427,9 +430,9 @@ static int mca_btl_udapl_endpoint_finish_max(mca_btl_udapl_endpoint_t* endpoint)
opal_list_remove_first(&endpoint->endpoint_max_frags))) { opal_list_remove_first(&endpoint->endpoint_max_frags))) {
cookie.as_ptr = frag; cookie.as_ptr = frag;
assert(frag->triplet.virtual_address == (DAT_VADDR)frag->hdr); assert(frag->triplet.virtual_address == (DAT_VADDR)frag->ftr);
assert(frag->triplet.segment_length == assert(frag->triplet.segment_length ==
frag->segment.seg_len + sizeof(mca_btl_base_header_t)); frag->segment.seg_len + sizeof(mca_btl_udapl_footer_t));
assert(frag->size == assert(frag->size ==
mca_btl_udapl_component.udapl_eager_frag_size); mca_btl_udapl_component.udapl_eager_frag_size);
@ -477,7 +480,7 @@ static int mca_btl_udapl_endpoint_post_recv(mca_btl_udapl_endpoint_t* endpoint,
assert(size == frag->size); assert(size == frag->size);
/* Set up the LMR triplet from the frag segment */ /* Set up the LMR triplet from the frag segment */
/* Note that this triplet defines a sub-region of a registered LMR */ /* Note that this triplet defines a sub-region of a registered LMR */
frag->triplet.virtual_address = (DAT_VADDR)frag->hdr; frag->triplet.virtual_address = (DAT_VADDR)frag->segment.seg_addr.pval;
frag->triplet.segment_length = frag->size; frag->triplet.segment_length = frag->size;
frag->btl = endpoint->endpoint_btl; frag->btl = endpoint->endpoint_btl;

9
ompi/mca/btl/udapl/btl_udapl_frag.c
Просмотреть файл

@ -11,6 +11,7 @@
* All rights reserved. * All rights reserved.
* Copyright (c) 2006 Sandia National Laboratories. All rights * Copyright (c) 2006 Sandia National Laboratories. All rights
* reserved. * reserved.
* Copyright (c) 2006 Sun Microsystems, Inc. All rights reserved.
* $COPYRIGHT$ * $COPYRIGHT$
* *
* Additional copyrights may follow * Additional copyrights may follow
@ -36,8 +37,8 @@ static void mca_btl_udapl_frag_common_constructor(mca_btl_udapl_frag_t* frag)
#endif #endif
frag->registration = (mca_mpool_base_registration_t*)reg; frag->registration = (mca_mpool_base_registration_t*)reg;
frag->hdr = (mca_btl_base_header_t*)(frag + 1); frag->segment.seg_addr.pval = (unsigned char*)(frag + 1);
frag->segment.seg_addr.pval = (unsigned char*)(frag->hdr + 1); frag->ftr = NULL;
/* Don't understand why yet, but there are cases where reg is NULL - /* Don't understand why yet, but there are cases where reg is NULL -
that is, this memory has not been registered. So be careful not that is, this memory has not been registered. So be careful not
@ -67,14 +68,14 @@ static void mca_btl_udapl_frag_user_constructor(mca_btl_udapl_frag_t* frag)
mca_btl_udapl_frag_common_constructor(frag); mca_btl_udapl_frag_common_constructor(frag);
frag->segment.seg_len = 0; frag->segment.seg_len = 0;
frag->segment.seg_addr.pval = NULL; frag->segment.seg_addr.pval = NULL;
frag->hdr = NULL; frag->ftr = NULL;
frag->size = 0; frag->size = 0;
} }
static void mca_btl_udapl_frag_common_destructor(mca_btl_udapl_frag_t* frag) static void mca_btl_udapl_frag_common_destructor(mca_btl_udapl_frag_t* frag)
{ {
#if OMPI_ENABLE_DEBUG #if OMPI_ENABLE_DEBUG
frag->hdr = NULL; frag->ftr = NULL;
frag->size = 0; frag->size = 0;
frag->registration = NULL; frag->registration = NULL;
frag->segment.seg_len = 0; frag->segment.seg_len = 0;

13
ompi/mca/btl/udapl/btl_udapl_frag.h
Просмотреть файл

@ -11,6 +11,8 @@
* All rights reserved. * All rights reserved.
* Copyright (c) 2006 Sandia National Laboratories. All rights * Copyright (c) 2006 Sandia National Laboratories. All rights
* reserved. * reserved.
* Copyright (c) 2006 Sun Microsystems, Inc. All rights reserved.
*
* $COPYRIGHT$ * $COPYRIGHT$
* *
* Additional copyrights may follow * Additional copyrights may follow
@ -38,6 +40,14 @@ typedef enum {
MCA_BTL_UDAPL_CONN_SEND MCA_BTL_UDAPL_CONN_SEND
} mca_btl_udapl_frag_type_t; } mca_btl_udapl_frag_type_t;
/**
* uDAPL btl footer.
* This is put after the payload packet so the PML header can be aligned.
*/
struct mca_btl_udapl_footer_t {
mca_btl_base_tag_t tag;
};
typedef struct mca_btl_udapl_footer_t mca_btl_udapl_footer_t;
/** /**
* uDAPL fragment derived type. * uDAPL fragment derived type.
@ -51,13 +61,14 @@ struct mca_btl_udapl_frag_t {
struct mca_mpool_base_registration_t* registration; struct mca_mpool_base_registration_t* registration;
DAT_LMR_TRIPLET triplet; DAT_LMR_TRIPLET triplet;
mca_btl_base_header_t *hdr; mca_btl_udapl_footer_t *ftr;
size_t size; size_t size;
mca_btl_udapl_frag_type_t type; mca_btl_udapl_frag_type_t type;
}; };
typedef struct mca_btl_udapl_frag_t mca_btl_udapl_frag_t; typedef struct mca_btl_udapl_frag_t mca_btl_udapl_frag_t;
OBJ_CLASS_DECLARATION(mca_btl_udapl_frag_t); OBJ_CLASS_DECLARATION(mca_btl_udapl_frag_t);
typedef struct mca_btl_udapl_frag_t mca_btl_udapl_frag_eager_t; typedef struct mca_btl_udapl_frag_t mca_btl_udapl_frag_eager_t;
OBJ_CLASS_DECLARATION(mca_btl_udapl_frag_eager_t); OBJ_CLASS_DECLARATION(mca_btl_udapl_frag_eager_t);