rework to use a single segment for large messages
This commit was SVN r3429.
Этот коммит содержится в:
Tim Woodall
родитель
5809eda0c2
Коммит
2b72629c23
@ -137,7 +137,6 @@ int mca_ptl_mx_send(
|
||||
{
|
||||
mca_ptl_mx_module_t* mx_ptl = (mca_ptl_mx_module_t*)ptl;
|
||||
mca_ptl_mx_send_frag_t* sendfrag;
|
||||
mca_ptl_base_header_t* hdr;
|
||||
mx_return_t mx_return;
|
||||
int rc;
|
||||
|
||||
@ -150,23 +149,6 @@ int mca_ptl_mx_send(
|
||||
return rc;
|
||||
}
|
||||
|
||||
/* setup message header */
|
||||
hdr = &sendfrag->frag_send.frag_base.frag_header;
|
||||
hdr->hdr_common.hdr_type = MCA_PTL_HDR_TYPE_MATCH;
|
||||
hdr->hdr_common.hdr_flags = flags;
|
||||
hdr->hdr_common.hdr_size = sizeof(mca_ptl_base_match_header_t);
|
||||
hdr->hdr_frag.hdr_frag_offset = offset;
|
||||
hdr->hdr_frag.hdr_frag_seq = 0;
|
||||
hdr->hdr_frag.hdr_src_ptr.lval = 0; /* for VALGRIND/PURIFY - REPLACE WITH MACRO */
|
||||
hdr->hdr_frag.hdr_src_ptr.pval = sendfrag;
|
||||
hdr->hdr_frag.hdr_dst_ptr.lval = 0;
|
||||
hdr->hdr_match.hdr_contextid = sendreq->req_base.req_comm->c_contextid;
|
||||
hdr->hdr_match.hdr_src = sendreq->req_base.req_comm->c_my_rank;
|
||||
hdr->hdr_match.hdr_dst = sendreq->req_base.req_peer;
|
||||
hdr->hdr_match.hdr_tag = sendreq->req_base.req_tag;
|
||||
hdr->hdr_match.hdr_msg_length = sendreq->req_bytes_packed;
|
||||
hdr->hdr_match.hdr_msg_seq = sendreq->req_base.req_sequence;
|
||||
|
||||
/* initialize convertor */
|
||||
sendfrag->frag_progress = 0;
|
||||
sendfrag->frag_free = 0;
|
||||
@ -208,19 +190,12 @@ int mca_ptl_mx_send(
|
||||
sendfrag->frag_free <<= 1;
|
||||
sendfrag->frag_segments[1].segment_ptr = iov.iov_base;
|
||||
sendfrag->frag_segments[1].segment_length = iov.iov_len;
|
||||
sendfrag->frag_segment_count = 2;
|
||||
sendfrag->frag_segment_count = 1;
|
||||
sendfrag->frag_send.frag_base.frag_addr = iov.iov_base;
|
||||
} else {
|
||||
sendfrag->frag_send.frag_base.frag_addr = NULL;
|
||||
sendfrag->frag_send.frag_base.frag_size = 0;
|
||||
sendfrag->frag_segment_count = 1;
|
||||
}
|
||||
hdr->hdr_frag.hdr_frag_length = size;
|
||||
|
||||
/* convert header to network byte order if required */
|
||||
if(ptl_peer->peer_byte_swap) {
|
||||
hdr->hdr_common.hdr_flags |= MCA_PTL_FLAGS_NBO;
|
||||
MCA_PTL_BASE_MATCH_HDR_HTON(hdr->hdr_match);
|
||||
sendfrag->frag_segment_count = 0;
|
||||
}
|
||||
|
||||
/* fragment state */
|
||||
@ -237,7 +212,7 @@ int mca_ptl_mx_send(
|
||||
/* start the fragment */
|
||||
mx_return = mx_isend(
|
||||
mx_ptl->mx_endpoint,
|
||||
sendfrag->frag_segments,
|
||||
sendfrag->frag_segments+1, /* no header - data only */
|
||||
sendfrag->frag_segment_count,
|
||||
ptl_peer->peer_addr,
|
||||
0,
|
||||
|
||||
@ -143,6 +143,7 @@ static void mca_ptl_mx_match(void* context, uint64_t match_value, int size)
|
||||
mca_ptl_mx_module_t* ptl = (mca_ptl_mx_module_t*)context;
|
||||
mca_ptl_mx_recv_frag_t *frag;
|
||||
mx_return_t mx_return;
|
||||
mx_segment_t* segments;
|
||||
ompi_ptr_t match;
|
||||
int rc;
|
||||
|
||||
@ -158,9 +159,11 @@ static void mca_ptl_mx_match(void* context, uint64_t match_value, int size)
|
||||
/* first fragment - post a buffer */
|
||||
if(match_value == 0) {
|
||||
|
||||
frag->frag_recv.frag_base.frag_size = size - sizeof(mca_ptl_base_header_t);
|
||||
frag->frag_recv.frag_base.frag_addr = frag->frag_data;
|
||||
frag->frag_recv.frag_is_buffered = true;
|
||||
frag->frag_segment_count = 2;
|
||||
frag->frag_segments[1].segment_ptr = frag->frag_data;
|
||||
frag->frag_segments[1].segment_length = size - sizeof(mca_ptl_base_header_t);
|
||||
segments = frag->frag_segments;
|
||||
|
||||
/* fragment has already been matched */
|
||||
} else {
|
||||
@ -170,7 +173,11 @@ static void mca_ptl_mx_match(void* context, uint64_t match_value, int size)
|
||||
ompi_proc_t *proc = ompi_comm_peer_lookup(request->req_base.req_comm,
|
||||
request->req_base.req_ompi.req_status.MPI_SOURCE);
|
||||
ompi_convertor_t* convertor = &frag->frag_recv.frag_base.frag_convertor;
|
||||
|
||||
frag->frag_size = size;
|
||||
frag->frag_recv.frag_base.frag_size = size - sizeof(mca_ptl_base_header_t);
|
||||
frag->frag_recv.frag_base.frag_header.hdr_common.hdr_type =
|
||||
MCA_PTL_HDR_TYPE_MATCH;
|
||||
|
||||
/* initialize convertor */
|
||||
ompi_convertor_copy(proc->proc_convertor, convertor);
|
||||
@ -193,14 +200,16 @@ static void mca_ptl_mx_match(void* context, uint64_t match_value, int size)
|
||||
frag->frag_recv.frag_base.frag_addr = ((unsigned char*)request->req_base.req_addr) + offset;
|
||||
}
|
||||
|
||||
frag->frag_segments[1].segment_ptr = frag->frag_recv.frag_base.frag_addr;
|
||||
frag->frag_segments[1].segment_length = frag->frag_recv.frag_base.frag_size;
|
||||
frag->frag_segment_count = 2;
|
||||
/* dont receive a header */
|
||||
frag->frag_segment_count = 1;
|
||||
segments = frag->frag_segments+1;
|
||||
}
|
||||
frag->frag_segments[1].segment_ptr = frag->frag_recv.frag_base.frag_addr;
|
||||
frag->frag_segments[1].segment_length = frag->frag_recv.frag_base.frag_size;
|
||||
|
||||
mx_return = mx_irecv(
|
||||
ptl->mx_endpoint,
|
||||
frag->frag_segments,
|
||||
segments,
|
||||
frag->frag_segment_count,
|
||||
match_value,
|
||||
MX_MATCH_MASK_NONE,
|
||||
|
||||
@ -57,7 +57,7 @@ do {
|
||||
} \
|
||||
case MCA_PTL_HDR_TYPE_FRAG: \
|
||||
{ \
|
||||
MCA_PTL_MX_RECV_FRAG_FRAG(recvfrag,hdr); \
|
||||
MCA_PTL_MX_RECV_FRAG_FRAG(recvfrag); \
|
||||
break; \
|
||||
} \
|
||||
case MCA_PTL_HDR_TYPE_ACK: \
|
||||
|
||||
@ -18,9 +18,10 @@
|
||||
struct mca_ptl_mx_recv_frag_t {
|
||||
mca_ptl_base_recv_frag_t frag_recv; /**< base receive fragment descriptor */
|
||||
mx_request_t frag_request;
|
||||
mx_segment_t frag_segments[3];
|
||||
mx_segment_t frag_segments[2];
|
||||
uint32_t frag_segment_count;
|
||||
unsigned char frag_data[32*1024];
|
||||
size_t frag_size;
|
||||
};
|
||||
typedef struct mca_ptl_mx_recv_frag_t mca_ptl_mx_recv_frag_t;
|
||||
|
||||
@ -36,8 +37,9 @@ OBJ_CLASS_DECLARATION(mca_ptl_mx_recv_frag_t);
|
||||
|
||||
#define MCA_PTL_MX_RECV_FRAG_RETURN(frag) \
|
||||
{ \
|
||||
if(frag->frag_recv.frag_is_buffered) { \
|
||||
free(frag->frag_segments[1].segment_ptr); \
|
||||
if(frag->frag_recv.frag_is_buffered && \
|
||||
frag->frag_data != frag->frag_recv.frag_base.frag_addr) { \
|
||||
free(frag->frag_recv.frag_base.frag_addr); \
|
||||
} \
|
||||
OMPI_FREE_LIST_RETURN(&mca_ptl_mx_component.mx_recv_frags, (ompi_list_item_t*)frag); \
|
||||
}
|
||||
@ -60,9 +62,7 @@ do { \
|
||||
* Process a fragment that completed.
|
||||
*/
|
||||
|
||||
static inline void MCA_PTL_MX_RECV_FRAG_FRAG(
|
||||
mca_ptl_mx_recv_frag_t* frag,
|
||||
mca_ptl_base_header_t* hdr)
|
||||
static inline void MCA_PTL_MX_RECV_FRAG_FRAG(mca_ptl_mx_recv_frag_t* frag)
|
||||
{
|
||||
/* copy into user space */
|
||||
if(frag->frag_recv.frag_is_buffered) {
|
||||
@ -83,7 +83,7 @@ static inline void MCA_PTL_MX_RECV_FRAG_FRAG(
|
||||
frag->frag_recv.frag_base.frag_owner->ptl_recv_progress(
|
||||
frag->frag_recv.frag_base.frag_owner,
|
||||
frag->frag_recv.frag_request,
|
||||
frag->frag_recv.frag_base.frag_header.hdr_frag.hdr_frag_length,
|
||||
frag->frag_size,
|
||||
frag->frag_recv.frag_base.frag_size);
|
||||
|
||||
MCA_PTL_MX_RECV_FRAG_RETURN(frag);
|
||||
|
||||
Загрузка…
x
Ссылка в новой задаче
Block a user