* assert() that endianness doesn't need to change if we are sending RDMA headers

around, since OB1 currently doesn't do the right thing there, but that should
  not happen in the near future because the R2 BML should not make any RDMA
  networks available between machines with different architectures
* Clean up the #ifs a little bit so that we don't do unneeded work when
  on big endian machines and heterogeneous support is disabled...

This commit was SVN r9184.

ompi/mca/pml/ob1/pml_ob1_recvfrag.c

@@ -94,7 +94,10 @@ void mca_pml_ob1_recv_frag_callback(
     case MCA_PML_OB1_HDR_TYPE_RGET:
         {
 #if !defined(WORDS_BIGENDIAN) && OMPI_ENABLE_HETEROGENEOUS_SUPPORT
-            /* BWB - FIX ME - Tim, what do I do with rget? */
+            /* RDMA is currently disabled by bml if arch doesn't
+               match, so this shouldn't be needed.  here to make sure
+               we remember if we ever change the bml. */
+            assert(0 == (hdr->hdr_common.hdr_flags & MCA_PML_OB1_HDR_FLAGS_NBO));
 #endif
             mca_pml_ob1_recv_frag_match(btl, &hdr->hdr_match, segments,des->des_dst_cnt);
             break;
@@ -131,7 +134,10 @@ void mca_pml_ob1_recv_frag_callback(
         {
             mca_pml_ob1_send_request_t* sendreq;
 #if !defined(WORDS_BIGENDIAN) && OMPI_ENABLE_HETEROGENEOUS_SUPPORT
-            /* BWB - FIX ME - Tim, what do I do with rdma headers? */
+            /* RDMA is currently disabled by bml if arch doesn't
+               match, so this shouldn't be needed.  here to make sure
+               we remember if we ever change the bml. */
+            assert(0 == (hdr->hdr_common.hdr_flags & MCA_PML_OB1_HDR_FLAGS_NBO));
 #endif
             sendreq = (mca_pml_ob1_send_request_t*)
                 hdr->hdr_rdma.hdr_req.pval;

ompi/mca/pml/ob1/pml_ob1_recvreq.c

@@ -265,9 +265,10 @@ static void mca_pml_ob1_recv_request_ack(
     ack->hdr_dst_req.pval = recvreq;
     ack->hdr_rdma_offset = recvreq->req_rdma_offset;
 
+#if OMPI_ENABLE_HETEROGENEOUS_SUPPORT
 #ifdef WORDS_BIGENDIAN
     ack->hdr_common.hdr_flags |= MCA_PML_OB1_HDR_FLAGS_NBO;
-#elif OMPI_ENABLE_HETEROGENEOUS_SUPPORT
+#else
     /* if we are little endian and the remote side is big endian,
        we're responsible for making sure the data is in network byte
        order */
@@ -275,6 +276,7 @@ static void mca_pml_ob1_recv_request_ack(
         ack->hdr_common.hdr_flags |= MCA_PML_OB1_HDR_FLAGS_NBO;
         MCA_PML_OB1_ACK_HDR_HTON(*ack);
     }
+#endif
 #endif
 
     /* initialize descriptor */
@@ -364,9 +366,10 @@ static void mca_pml_ob1_rget_completion(
     hdr->hdr_common.hdr_type = MCA_PML_OB1_HDR_TYPE_FIN;
     hdr->hdr_des = frag->rdma_hdr.hdr_rget.hdr_des;
 
+#if OMPI_ENABLE_HETEROGENEOUS_SUPPORT
 #ifdef WORDS_BIGENDIAN
     hdr->hdr_common.hdr_flags |= MCA_PML_OB1_HDR_FLAGS_NBO;
-#elif OMPI_ENABLE_HETEROGENEOUS_SUPPORT
+#else
     /* if we are little endian and the remote side is big endian,
        we're responsible for making sure the data is in network byte
        order */
@@ -375,7 +378,7 @@ static void mca_pml_ob1_rget_completion(
         MCA_PML_OB1_FIN_HDR_HTON(*hdr);
     }
 #endif
-                                                                                                            
+#endif                                                                                                            
     /* queue request */
     rc = mca_bml_base_send(
         bml_btl,
@@ -741,15 +744,19 @@ void mca_pml_ob1_recv_request_schedule(mca_pml_ob1_recv_request_t* recvreq)
                 hdr->hdr_seg_cnt = dst->des_dst_cnt;
                 memcpy(hdr->hdr_segs, dst->des_dst, dst->des_dst_cnt * sizeof(mca_btl_base_segment_t));
 
+#if OMPI_ENABLE_HETEROGENEOUS_SUPPORT
 #ifdef WORDS_BIGENDIAN
                 hdr->hdr_common.hdr_flags |= MCA_PML_OB1_HDR_FLAGS_NBO;
-#elif OMPI_ENABLE_HETEROGENEOUS_SUPPORT
+#else
                 /* if we are little endian and the remote side is big endian,
                    we're responsible for making sure the data is in network byte
                    order */
-                if (recvreq->req_recv.req_base.req_proc->proc_arch & OMPI_ARCH_ISBIGENDIAN) {
+                /* RDMA is currently disabled by bml if arch doesn't
-                    /* BWB - FIX ME - TIM, what do we do here? */
+                   match, so this shouldn't be needed.  here to make sure
-                }
+                   we remember if we ever change the bml. */
+                assert(0 == (recvreq->req_recv.req_base.req_proc->proc_arch & 
+                             OMPI_ARCH_ISBIGENDIAN));
+#endif
 #endif
 
                 /* update request state */

ompi/mca/pml/ob1/pml_ob1_sendreq.c

@@ -337,9 +337,10 @@ int mca_pml_ob1_send_request_start_buffered(
     hdr->hdr_rndv.hdr_msg_length = sendreq->req_send.req_bytes_packed;
     hdr->hdr_rndv.hdr_src_req.pval = sendreq;
 
+#if OMPI_ENABLE_HETEROGENEOUS_SUPPORT
 #ifdef WORDS_BIGENDIAN
     hdr->hdr_common.hdr_flags |= MCA_PML_OB1_HDR_FLAGS_NBO;
-#elif OMPI_ENABLE_HETEROGENEOUS_SUPPORT
+#else
     /* if we are little endian and the remote side is big endian,
        we're responsible for making sure the data is in network byte
        order */
@@ -347,6 +348,7 @@ int mca_pml_ob1_send_request_start_buffered(
         hdr->hdr_common.hdr_flags |= MCA_PML_OB1_HDR_FLAGS_NBO;
         MCA_PML_OB1_RNDV_HDR_HTON(hdr->hdr_rndv);
     }
+#endif
 #endif
 
     /* update lengths */
@@ -450,9 +452,10 @@ int mca_pml_ob1_send_request_start_copy(
     hdr->hdr_match.hdr_tag = sendreq->req_send.req_base.req_tag;
     hdr->hdr_match.hdr_seq = sendreq->req_send.req_base.req_sequence;
 
+#if OMPI_ENABLE_HETEROGENEOUS_SUPPORT
 #ifdef WORDS_BIGENDIAN
     hdr->hdr_common.hdr_flags |= MCA_PML_OB1_HDR_FLAGS_NBO;
-#elif OMPI_ENABLE_HETEROGENEOUS_SUPPORT
+#else
     /* if we are little endian and the remote side is big endian,
        we're responsible for making sure the data is in network byte
        order */
@@ -460,6 +463,7 @@ int mca_pml_ob1_send_request_start_copy(
         hdr->hdr_common.hdr_flags |= MCA_PML_OB1_HDR_FLAGS_NBO;
         MCA_PML_OB1_MATCH_HDR_HTON(hdr->hdr_match);
     }
+#endif
 #endif
 
     /* update lengths */
@@ -522,9 +526,10 @@ int mca_pml_ob1_send_request_start_prepare(
     hdr->hdr_match.hdr_tag = sendreq->req_send.req_base.req_tag;
     hdr->hdr_match.hdr_seq = sendreq->req_send.req_base.req_sequence;
 
+#if OMPI_ENABLE_HETEROGENEOUS_SUPPORT
 #ifdef WORDS_BIGENDIAN
     hdr->hdr_common.hdr_flags |= MCA_PML_OB1_HDR_FLAGS_NBO;
-#elif OMPI_ENABLE_HETEROGENEOUS_SUPPORT
+#else
     /* if we are little endian and the remote side is big endian,
        we're responsible for making sure the data is in network byte
        order */
@@ -532,6 +537,7 @@ int mca_pml_ob1_send_request_start_prepare(
         hdr->hdr_common.hdr_flags |= MCA_PML_OB1_HDR_FLAGS_NBO;
         MCA_PML_OB1_MATCH_HDR_HTON(hdr->hdr_match);
     }
+#endif
 #endif
 
     /* short message */
@@ -616,16 +622,19 @@ int mca_pml_ob1_send_request_start_rdma(
          hdr->hdr_rget.hdr_des.pval = src;
          hdr->hdr_rget.hdr_seg_cnt = src->des_src_cnt;
 
+#if OMPI_ENABLE_HETEROGENEOUS_SUPPORT
 #ifdef WORDS_BIGENDIAN
          hdr->hdr_common.hdr_flags |= MCA_PML_OB1_HDR_FLAGS_NBO;
-#elif OMPI_ENABLE_HETEROGENEOUS_SUPPORT
+#else
          /* if we are little endian and the remote side is big endian,
             we're responsible for making sure the data is in network byte
             order */
-         if (sendreq->req_send.req_base.req_proc->proc_arch & OMPI_ARCH_ISBIGENDIAN) {
+         /* RDMA is currently disabled by bml if arch doesn't
-             /* BWB - FIX ME - Ask Tim what we are supposed to do in this case */
+            match, so this shouldn't be needed.  here to make sure
-             return OMPI_ERR_NOT_SUPPORTED;
+            we remember if we ever change the bml. */
-         }
+         assert(0 == (sendreq->req_send.req_base.req_proc->proc_arch & 
+                             OMPI_ARCH_ISBIGENDIAN));
+#endif
 #endif
 
          for(i=0; i<src->des_src_cnt; i++)
@@ -655,9 +664,10 @@ int mca_pml_ob1_send_request_start_rdma(
          hdr->hdr_rndv.hdr_msg_length = sendreq->req_send.req_bytes_packed;
          hdr->hdr_rndv.hdr_src_req.pval = sendreq;
 
+#if OMPI_ENABLE_HETEROGENEOUS_SUPPORT
 #ifdef WORDS_BIGENDIAN
          hdr->hdr_common.hdr_flags |= MCA_PML_OB1_HDR_FLAGS_NBO;
-#elif OMPI_ENABLE_HETEROGENEOUS_SUPPORT
+#else
          /* if we are little endian and the remote side is big endian,
             we're responsible for making sure the data is in network byte
             order */
@@ -665,6 +675,7 @@ int mca_pml_ob1_send_request_start_rdma(
              hdr->hdr_common.hdr_flags |= MCA_PML_OB1_HDR_FLAGS_NBO;
              MCA_PML_OB1_RNDV_HDR_HTON(hdr->hdr_rndv);
          }
+#endif
 #endif
 
          /* update lengths with number of bytes actually packed */
@@ -736,9 +747,10 @@ int mca_pml_ob1_send_request_start_rndv(
     hdr->hdr_rndv.hdr_msg_length = sendreq->req_send.req_bytes_packed;
     hdr->hdr_rndv.hdr_src_req.pval = sendreq;
 
+#if OMPI_ENABLE_HETEROGENEOUS_SUPPORT
 #ifdef WORDS_BIGENDIAN
     hdr->hdr_common.hdr_flags |= MCA_PML_OB1_HDR_FLAGS_NBO;
-#elif OMPI_ENABLE_HETEROGENEOUS_SUPPORT
+#else
     /* if we are little endian and the remote side is big endian,
        we're responsible for making sure the data is in network byte
        order */
@@ -746,6 +758,7 @@ int mca_pml_ob1_send_request_start_rndv(
         hdr->hdr_common.hdr_flags |= MCA_PML_OB1_HDR_FLAGS_NBO;
         MCA_PML_OB1_RNDV_HDR_HTON(hdr->hdr_rndv);
     }
+#endif
 #endif
 
     /* first fragment of a long message */
@@ -870,9 +883,10 @@ int mca_pml_ob1_send_request_schedule(mca_pml_ob1_send_request_t* sendreq)
                 hdr->hdr_src_req.pval = sendreq;
                 hdr->hdr_dst_req = sendreq->req_recv;
 
+#if OMPI_ENABLE_HETEROGENEOUS_SUPPORT
 #ifdef WORDS_BIGENDIAN
                 hdr->hdr_common.hdr_flags |= MCA_PML_OB1_HDR_FLAGS_NBO;
-#elif OMPI_ENABLE_HETEROGENEOUS_SUPPORT
+#else
                 /* if we are little endian and the remote side is big endian,
                    we're responsible for making sure the data is in network byte
                    order */
@@ -880,6 +894,7 @@ int mca_pml_ob1_send_request_schedule(mca_pml_ob1_send_request_t* sendreq)
                     hdr->hdr_common.hdr_flags |= MCA_PML_OB1_HDR_FLAGS_NBO;
                     MCA_PML_OB1_FRAG_HDR_HTON(*hdr);
                 }
+#endif
 #endif
 
                 /* update state */
@@ -983,9 +998,10 @@ static void mca_pml_ob1_put_completion(
     hdr->hdr_common.hdr_type = MCA_PML_OB1_HDR_TYPE_FIN;
     hdr->hdr_des = frag->rdma_hdr.hdr_rdma.hdr_des;
 
+#if OMPI_ENABLE_HETEROGENEOUS_SUPPORT
 #ifdef WORDS_BIGENDIAN
     hdr->hdr_common.hdr_flags |= MCA_PML_OB1_HDR_FLAGS_NBO;
-#elif OMPI_ENABLE_HETEROGENEOUS_SUPPORT
+#else
     /* if we are little endian and the remote side is big endian,
        we're responsible for making sure the data is in network byte
        order */
@@ -993,6 +1009,7 @@ static void mca_pml_ob1_put_completion(
         hdr->hdr_common.hdr_flags |= MCA_PML_OB1_HDR_FLAGS_NBO;
         MCA_PML_OB1_FIN_HDR_HTON(*hdr);
     }
+#endif
 #endif
 
     /* queue request */