From 1479a90b3934f09264f0be8c26f2ae9b2be70e46 Mon Sep 17 00:00:00 2001 From: Brian Barrett Date: Tue, 28 Feb 2006 19:54:46 +0000 Subject: [PATCH] * assert() that endianness doesn't need to change if we are sending RDMA headers around, since OB1 currently doesn't do the right thing there, but that should not happen in the near future because the R2 BML should not make any RDMA networks available between machines with different architectures * Clean up the #ifs a little bit so that we don't do unneeded work when on big endian machines and heterogeneous support is disabled... This commit was SVN r9184. --- ompi/mca/pml/ob1/pml_ob1_recvfrag.c | 10 +++++-- ompi/mca/pml/ob1/pml_ob1_recvreq.c | 21 ++++++++++----- ompi/mca/pml/ob1/pml_ob1_sendreq.c | 41 ++++++++++++++++++++--------- 3 files changed, 51 insertions(+), 21 deletions(-) diff --git a/ompi/mca/pml/ob1/pml_ob1_recvfrag.c b/ompi/mca/pml/ob1/pml_ob1_recvfrag.c index 8192237542..66edad4503 100644 --- a/ompi/mca/pml/ob1/pml_ob1_recvfrag.c +++ b/ompi/mca/pml/ob1/pml_ob1_recvfrag.c @@ -94,7 +94,10 @@ void mca_pml_ob1_recv_frag_callback( case MCA_PML_OB1_HDR_TYPE_RGET: { #if !defined(WORDS_BIGENDIAN) && OMPI_ENABLE_HETEROGENEOUS_SUPPORT - /* BWB - FIX ME - Tim, what do I do with rget? */ + /* RDMA is currently disabled by bml if arch doesn't + match, so this shouldn't be needed. here to make sure + we remember if we ever change the bml. */ + assert(0 == (hdr->hdr_common.hdr_flags & MCA_PML_OB1_HDR_FLAGS_NBO)); #endif mca_pml_ob1_recv_frag_match(btl, &hdr->hdr_match, segments,des->des_dst_cnt); break; @@ -131,7 +134,10 @@ void mca_pml_ob1_recv_frag_callback( { mca_pml_ob1_send_request_t* sendreq; #if !defined(WORDS_BIGENDIAN) && OMPI_ENABLE_HETEROGENEOUS_SUPPORT - /* BWB - FIX ME - Tim, what do I do with rdma headers? */ + /* RDMA is currently disabled by bml if arch doesn't + match, so this shouldn't be needed. here to make sure + we remember if we ever change the bml. */ + assert(0 == (hdr->hdr_common.hdr_flags & MCA_PML_OB1_HDR_FLAGS_NBO)); #endif sendreq = (mca_pml_ob1_send_request_t*) hdr->hdr_rdma.hdr_req.pval; diff --git a/ompi/mca/pml/ob1/pml_ob1_recvreq.c b/ompi/mca/pml/ob1/pml_ob1_recvreq.c index 4cb1796c1b..ef6014bbc9 100644 --- a/ompi/mca/pml/ob1/pml_ob1_recvreq.c +++ b/ompi/mca/pml/ob1/pml_ob1_recvreq.c @@ -265,9 +265,10 @@ static void mca_pml_ob1_recv_request_ack( ack->hdr_dst_req.pval = recvreq; ack->hdr_rdma_offset = recvreq->req_rdma_offset; +#if OMPI_ENABLE_HETEROGENEOUS_SUPPORT #ifdef WORDS_BIGENDIAN ack->hdr_common.hdr_flags |= MCA_PML_OB1_HDR_FLAGS_NBO; -#elif OMPI_ENABLE_HETEROGENEOUS_SUPPORT +#else /* if we are little endian and the remote side is big endian, we're responsible for making sure the data is in network byte order */ @@ -275,6 +276,7 @@ static void mca_pml_ob1_recv_request_ack( ack->hdr_common.hdr_flags |= MCA_PML_OB1_HDR_FLAGS_NBO; MCA_PML_OB1_ACK_HDR_HTON(*ack); } +#endif #endif /* initialize descriptor */ @@ -364,9 +366,10 @@ static void mca_pml_ob1_rget_completion( hdr->hdr_common.hdr_type = MCA_PML_OB1_HDR_TYPE_FIN; hdr->hdr_des = frag->rdma_hdr.hdr_rget.hdr_des; +#if OMPI_ENABLE_HETEROGENEOUS_SUPPORT #ifdef WORDS_BIGENDIAN hdr->hdr_common.hdr_flags |= MCA_PML_OB1_HDR_FLAGS_NBO; -#elif OMPI_ENABLE_HETEROGENEOUS_SUPPORT +#else /* if we are little endian and the remote side is big endian, we're responsible for making sure the data is in network byte order */ @@ -375,7 +378,7 @@ static void mca_pml_ob1_rget_completion( MCA_PML_OB1_FIN_HDR_HTON(*hdr); } #endif - +#endif /* queue request */ rc = mca_bml_base_send( bml_btl, @@ -741,15 +744,19 @@ void mca_pml_ob1_recv_request_schedule(mca_pml_ob1_recv_request_t* recvreq) hdr->hdr_seg_cnt = dst->des_dst_cnt; memcpy(hdr->hdr_segs, dst->des_dst, dst->des_dst_cnt * sizeof(mca_btl_base_segment_t)); +#if OMPI_ENABLE_HETEROGENEOUS_SUPPORT #ifdef WORDS_BIGENDIAN hdr->hdr_common.hdr_flags |= MCA_PML_OB1_HDR_FLAGS_NBO; -#elif OMPI_ENABLE_HETEROGENEOUS_SUPPORT +#else /* if we are little endian and the remote side is big endian, we're responsible for making sure the data is in network byte order */ - if (recvreq->req_recv.req_base.req_proc->proc_arch & OMPI_ARCH_ISBIGENDIAN) { - /* BWB - FIX ME - TIM, what do we do here? */ - } + /* RDMA is currently disabled by bml if arch doesn't + match, so this shouldn't be needed. here to make sure + we remember if we ever change the bml. */ + assert(0 == (recvreq->req_recv.req_base.req_proc->proc_arch & + OMPI_ARCH_ISBIGENDIAN)); +#endif #endif /* update request state */ diff --git a/ompi/mca/pml/ob1/pml_ob1_sendreq.c b/ompi/mca/pml/ob1/pml_ob1_sendreq.c index c36e1b15a4..e4a80d37e4 100644 --- a/ompi/mca/pml/ob1/pml_ob1_sendreq.c +++ b/ompi/mca/pml/ob1/pml_ob1_sendreq.c @@ -337,9 +337,10 @@ int mca_pml_ob1_send_request_start_buffered( hdr->hdr_rndv.hdr_msg_length = sendreq->req_send.req_bytes_packed; hdr->hdr_rndv.hdr_src_req.pval = sendreq; +#if OMPI_ENABLE_HETEROGENEOUS_SUPPORT #ifdef WORDS_BIGENDIAN hdr->hdr_common.hdr_flags |= MCA_PML_OB1_HDR_FLAGS_NBO; -#elif OMPI_ENABLE_HETEROGENEOUS_SUPPORT +#else /* if we are little endian and the remote side is big endian, we're responsible for making sure the data is in network byte order */ @@ -347,6 +348,7 @@ int mca_pml_ob1_send_request_start_buffered( hdr->hdr_common.hdr_flags |= MCA_PML_OB1_HDR_FLAGS_NBO; MCA_PML_OB1_RNDV_HDR_HTON(hdr->hdr_rndv); } +#endif #endif /* update lengths */ @@ -450,9 +452,10 @@ int mca_pml_ob1_send_request_start_copy( hdr->hdr_match.hdr_tag = sendreq->req_send.req_base.req_tag; hdr->hdr_match.hdr_seq = sendreq->req_send.req_base.req_sequence; +#if OMPI_ENABLE_HETEROGENEOUS_SUPPORT #ifdef WORDS_BIGENDIAN hdr->hdr_common.hdr_flags |= MCA_PML_OB1_HDR_FLAGS_NBO; -#elif OMPI_ENABLE_HETEROGENEOUS_SUPPORT +#else /* if we are little endian and the remote side is big endian, we're responsible for making sure the data is in network byte order */ @@ -460,6 +463,7 @@ int mca_pml_ob1_send_request_start_copy( hdr->hdr_common.hdr_flags |= MCA_PML_OB1_HDR_FLAGS_NBO; MCA_PML_OB1_MATCH_HDR_HTON(hdr->hdr_match); } +#endif #endif /* update lengths */ @@ -522,9 +526,10 @@ int mca_pml_ob1_send_request_start_prepare( hdr->hdr_match.hdr_tag = sendreq->req_send.req_base.req_tag; hdr->hdr_match.hdr_seq = sendreq->req_send.req_base.req_sequence; +#if OMPI_ENABLE_HETEROGENEOUS_SUPPORT #ifdef WORDS_BIGENDIAN hdr->hdr_common.hdr_flags |= MCA_PML_OB1_HDR_FLAGS_NBO; -#elif OMPI_ENABLE_HETEROGENEOUS_SUPPORT +#else /* if we are little endian and the remote side is big endian, we're responsible for making sure the data is in network byte order */ @@ -532,6 +537,7 @@ int mca_pml_ob1_send_request_start_prepare( hdr->hdr_common.hdr_flags |= MCA_PML_OB1_HDR_FLAGS_NBO; MCA_PML_OB1_MATCH_HDR_HTON(hdr->hdr_match); } +#endif #endif /* short message */ @@ -616,16 +622,19 @@ int mca_pml_ob1_send_request_start_rdma( hdr->hdr_rget.hdr_des.pval = src; hdr->hdr_rget.hdr_seg_cnt = src->des_src_cnt; +#if OMPI_ENABLE_HETEROGENEOUS_SUPPORT #ifdef WORDS_BIGENDIAN hdr->hdr_common.hdr_flags |= MCA_PML_OB1_HDR_FLAGS_NBO; -#elif OMPI_ENABLE_HETEROGENEOUS_SUPPORT +#else /* if we are little endian and the remote side is big endian, we're responsible for making sure the data is in network byte order */ - if (sendreq->req_send.req_base.req_proc->proc_arch & OMPI_ARCH_ISBIGENDIAN) { - /* BWB - FIX ME - Ask Tim what we are supposed to do in this case */ - return OMPI_ERR_NOT_SUPPORTED; - } + /* RDMA is currently disabled by bml if arch doesn't + match, so this shouldn't be needed. here to make sure + we remember if we ever change the bml. */ + assert(0 == (sendreq->req_send.req_base.req_proc->proc_arch & + OMPI_ARCH_ISBIGENDIAN)); +#endif #endif for(i=0; ides_src_cnt; i++) @@ -655,9 +664,10 @@ int mca_pml_ob1_send_request_start_rdma( hdr->hdr_rndv.hdr_msg_length = sendreq->req_send.req_bytes_packed; hdr->hdr_rndv.hdr_src_req.pval = sendreq; +#if OMPI_ENABLE_HETEROGENEOUS_SUPPORT #ifdef WORDS_BIGENDIAN hdr->hdr_common.hdr_flags |= MCA_PML_OB1_HDR_FLAGS_NBO; -#elif OMPI_ENABLE_HETEROGENEOUS_SUPPORT +#else /* if we are little endian and the remote side is big endian, we're responsible for making sure the data is in network byte order */ @@ -665,6 +675,7 @@ int mca_pml_ob1_send_request_start_rdma( hdr->hdr_common.hdr_flags |= MCA_PML_OB1_HDR_FLAGS_NBO; MCA_PML_OB1_RNDV_HDR_HTON(hdr->hdr_rndv); } +#endif #endif /* update lengths with number of bytes actually packed */ @@ -736,9 +747,10 @@ int mca_pml_ob1_send_request_start_rndv( hdr->hdr_rndv.hdr_msg_length = sendreq->req_send.req_bytes_packed; hdr->hdr_rndv.hdr_src_req.pval = sendreq; +#if OMPI_ENABLE_HETEROGENEOUS_SUPPORT #ifdef WORDS_BIGENDIAN hdr->hdr_common.hdr_flags |= MCA_PML_OB1_HDR_FLAGS_NBO; -#elif OMPI_ENABLE_HETEROGENEOUS_SUPPORT +#else /* if we are little endian and the remote side is big endian, we're responsible for making sure the data is in network byte order */ @@ -746,6 +758,7 @@ int mca_pml_ob1_send_request_start_rndv( hdr->hdr_common.hdr_flags |= MCA_PML_OB1_HDR_FLAGS_NBO; MCA_PML_OB1_RNDV_HDR_HTON(hdr->hdr_rndv); } +#endif #endif /* first fragment of a long message */ @@ -870,9 +883,10 @@ int mca_pml_ob1_send_request_schedule(mca_pml_ob1_send_request_t* sendreq) hdr->hdr_src_req.pval = sendreq; hdr->hdr_dst_req = sendreq->req_recv; +#if OMPI_ENABLE_HETEROGENEOUS_SUPPORT #ifdef WORDS_BIGENDIAN hdr->hdr_common.hdr_flags |= MCA_PML_OB1_HDR_FLAGS_NBO; -#elif OMPI_ENABLE_HETEROGENEOUS_SUPPORT +#else /* if we are little endian and the remote side is big endian, we're responsible for making sure the data is in network byte order */ @@ -880,6 +894,7 @@ int mca_pml_ob1_send_request_schedule(mca_pml_ob1_send_request_t* sendreq) hdr->hdr_common.hdr_flags |= MCA_PML_OB1_HDR_FLAGS_NBO; MCA_PML_OB1_FRAG_HDR_HTON(*hdr); } +#endif #endif /* update state */ @@ -983,9 +998,10 @@ static void mca_pml_ob1_put_completion( hdr->hdr_common.hdr_type = MCA_PML_OB1_HDR_TYPE_FIN; hdr->hdr_des = frag->rdma_hdr.hdr_rdma.hdr_des; +#if OMPI_ENABLE_HETEROGENEOUS_SUPPORT #ifdef WORDS_BIGENDIAN hdr->hdr_common.hdr_flags |= MCA_PML_OB1_HDR_FLAGS_NBO; -#elif OMPI_ENABLE_HETEROGENEOUS_SUPPORT +#else /* if we are little endian and the remote side is big endian, we're responsible for making sure the data is in network byte order */ @@ -993,6 +1009,7 @@ static void mca_pml_ob1_put_completion( hdr->hdr_common.hdr_flags |= MCA_PML_OB1_HDR_FLAGS_NBO; MCA_PML_OB1_FIN_HDR_HTON(*hdr); } +#endif #endif /* queue request */