From e865a751bde7213dbd762e84699784c33df4f6aa Mon Sep 17 00:00:00 2001 From: Brian Barrett Date: Mon, 27 Feb 2006 18:47:00 +0000 Subject: [PATCH] * First whack at making the onesided component endian safe. Needs a endian-safe datatype engine to really give it a whirl ;). This commit was SVN r9176. --- ompi/mca/osc/pt2pt/osc_pt2pt_component.c | 49 ++++++++++++++++++++++++ ompi/mca/osc/pt2pt/osc_pt2pt_data_move.c | 44 ++++++++++++++++----- ompi/mca/osc/pt2pt/osc_pt2pt_header.h | 10 ++--- 3 files changed, 88 insertions(+), 15 deletions(-) diff --git a/ompi/mca/osc/pt2pt/osc_pt2pt_component.c b/ompi/mca/osc/pt2pt/osc_pt2pt_component.c index dc58268b4c..1cd23000eb 100644 --- a/ompi/mca/osc/pt2pt/osc_pt2pt_component.c +++ b/ompi/mca/osc/pt2pt/osc_pt2pt_component.c @@ -32,6 +32,7 @@ #include "ompi/mca/btl/btl.h" #include "ompi/mca/bml/bml.h" #include "ompi/mca/bml/base/base.h" +#include "ompi/datatype/dt_arch.h" ompi_osc_pt2pt_component_t mca_osc_pt2pt_component = { { /* ompi_osc_base_component_t */ @@ -334,6 +335,12 @@ ompi_osc_pt2pt_component_fragment_cb(struct mca_btl_base_module_t *btl, descriptor->des_dst[0].seg_addr.pval; payload = (void*) (header + 1); +#if !defined(WORDS_BIGENDIAN) && OMPI_ENABLE_HETEROGENEOUS_SUPPORT + if (header->hdr_base.hdr_flags & OMPI_OSC_PT2PT_HDR_FLAG_NBO) { + OMPI_OSC_PT2PT_SEND_HDR_NTOH(*header); + } +#endif + /* get our module pointer */ module = ompi_osc_pt2pt_windx_to_module(header->hdr_windx); if (NULL == module) return; @@ -351,6 +358,12 @@ ompi_osc_pt2pt_component_fragment_cb(struct mca_btl_base_module_t *btl, descriptor->des_dst[0].seg_addr.pval; payload = (void*) (header + 1); +#if !defined(WORDS_BIGENDIAN) && OMPI_ENABLE_HETEROGENEOUS_SUPPORT + if (header->hdr_base.hdr_flags & OMPI_OSC_PT2PT_HDR_FLAG_NBO) { + OMPI_OSC_PT2PT_SEND_HDR_NTOH(*header); + } +#endif + /* get our module pointer */ module = ompi_osc_pt2pt_windx_to_module(header->hdr_windx); if (NULL == module) return; @@ -372,6 +385,12 @@ ompi_osc_pt2pt_component_fragment_cb(struct mca_btl_base_module_t *btl, descriptor->des_dst[0].seg_addr.pval; payload = (void*) (header + 1); +#if !defined(WORDS_BIGENDIAN) && OMPI_ENABLE_HETEROGENEOUS_SUPPORT + if (header->hdr_base.hdr_flags & OMPI_OSC_PT2PT_HDR_FLAG_NBO) { + OMPI_OSC_PT2PT_SEND_HDR_NTOH(*header); + } +#endif + /* get our module pointer */ module = ompi_osc_pt2pt_windx_to_module(header->hdr_windx); if (NULL == module) return; @@ -407,6 +426,12 @@ ompi_osc_pt2pt_component_fragment_cb(struct mca_btl_base_module_t *btl, descriptor->des_dst[0].seg_addr.pval; payload = (void*) (header + 1); +#if !defined(WORDS_BIGENDIAN) && OMPI_ENABLE_HETEROGENEOUS_SUPPORT + if (header->hdr_base.hdr_flags & OMPI_OSC_PT2PT_HDR_FLAG_NBO) { + OMPI_OSC_PT2PT_REPLY_HDR_NTOH(*header); + } +#endif + /* get original sendreq pointer */ sendreq = (ompi_osc_pt2pt_sendreq_t*) header->hdr_origin_sendreq.pval; module = sendreq->req_module; @@ -421,6 +446,12 @@ ompi_osc_pt2pt_component_fragment_cb(struct mca_btl_base_module_t *btl, (ompi_osc_pt2pt_control_header_t*) descriptor->des_dst[0].seg_addr.pval; +#if !defined(WORDS_BIGENDIAN) && OMPI_ENABLE_HETEROGENEOUS_SUPPORT + if (header->hdr_base.hdr_flags & OMPI_OSC_PT2PT_HDR_FLAG_NBO) { + OMPI_OSC_PT2PT_CONTROL_HDR_NTOH(*header); + } +#endif + /* get our module pointer */ module = ompi_osc_pt2pt_windx_to_module(header->hdr_windx); if (NULL == module) return; @@ -434,6 +465,12 @@ ompi_osc_pt2pt_component_fragment_cb(struct mca_btl_base_module_t *btl, (ompi_osc_pt2pt_control_header_t*) descriptor->des_dst[0].seg_addr.pval; +#if !defined(WORDS_BIGENDIAN) && OMPI_ENABLE_HETEROGENEOUS_SUPPORT + if (header->hdr_base.hdr_flags & OMPI_OSC_PT2PT_HDR_FLAG_NBO) { + OMPI_OSC_PT2PT_CONTROL_HDR_NTOH(*header); + } +#endif + /* get our module pointer */ module = ompi_osc_pt2pt_windx_to_module(header->hdr_windx); if (NULL == module) return; @@ -451,6 +488,12 @@ ompi_osc_pt2pt_component_fragment_cb(struct mca_btl_base_module_t *btl, (ompi_osc_pt2pt_control_header_t*) descriptor->des_dst[0].seg_addr.pval; +#if !defined(WORDS_BIGENDIAN) && OMPI_ENABLE_HETEROGENEOUS_SUPPORT + if (header->hdr_base.hdr_flags & OMPI_OSC_PT2PT_HDR_FLAG_NBO) { + OMPI_OSC_PT2PT_CONTROL_HDR_NTOH(*header); + } +#endif + /* get our module pointer */ module = ompi_osc_pt2pt_windx_to_module(header->hdr_windx); if (NULL == module) return; @@ -470,6 +513,12 @@ ompi_osc_pt2pt_component_fragment_cb(struct mca_btl_base_module_t *btl, (ompi_osc_pt2pt_control_header_t*) descriptor->des_dst[0].seg_addr.pval; +#if !defined(WORDS_BIGENDIAN) && OMPI_ENABLE_HETEROGENEOUS_SUPPORT + if (header->hdr_base.hdr_flags & OMPI_OSC_PT2PT_HDR_FLAG_NBO) { + OMPI_OSC_PT2PT_CONTROL_HDR_NTOH(*header); + } +#endif + /* get our module pointer */ module = ompi_osc_pt2pt_windx_to_module(header->hdr_windx); if (NULL == module) return; diff --git a/ompi/mca/osc/pt2pt/osc_pt2pt_data_move.c b/ompi/mca/osc/pt2pt/osc_pt2pt_data_move.c index 809b1fbdb1..469a50bf3c 100644 --- a/ompi/mca/osc/pt2pt/osc_pt2pt_data_move.c +++ b/ompi/mca/osc/pt2pt/osc_pt2pt_data_move.c @@ -28,6 +28,7 @@ #include "ompi/mca/bml/base/base.h" #include "ompi/mca/btl/btl.h" #include "ompi/datatype/datatype.h" +#include "ompi/datatype/dt_arch.h" static inline int32_t create_send_tag(ompi_osc_pt2pt_module_t *module) @@ -104,6 +105,11 @@ ompi_osc_pt2pt_sendreq_send_cb(struct mca_btl_base_module_t* btl, in the case of get, as we really don't care when it completes - only when the data arrives. */ if (OMPI_OSC_PT2PT_HDR_GET != header->hdr_base.hdr_type) { +#if !defined(WORDS_BIGENDIAN) && OMPI_ENABLE_HETEROGENEOUS_SUPPORT + if (header->hdr_base.hdr_flags & OMPI_OSC_PT2PT_HDR_FLAG_NBO) { + OMPI_OSC_PT2PT_SEND_HDR_NTOH(*header); + } +#endif /* do we need to post a send? */ if (header->hdr_msg_length != 0) { /* sendreq is done. Mark it as so and get out of here */ @@ -191,6 +197,7 @@ ompi_osc_pt2pt_sendreq_send(ompi_osc_pt2pt_module_t *module, /* pack header */ header = (ompi_osc_pt2pt_send_header_t*) descriptor->des_src[0].seg_addr.pval; written_data += sizeof(ompi_osc_pt2pt_send_header_t); + header->hdr_base.hdr_flags = 0; header->hdr_windx = sendreq->req_module->p2p_comm->c_contextid; header->hdr_origin = sendreq->req_module->p2p_comm->c_my_rank; header->hdr_origin_sendreq.pval = (void*) sendreq; @@ -259,9 +266,13 @@ ompi_osc_pt2pt_sendreq_send(ompi_osc_pt2pt_module_t *module, header->hdr_msg_length = 0; } -#if 0 /* BWB - FIX ME */ - /* put in network byte order */ - OMPI_OSC_PT2PT_REQ_HDR_HTON(header); +#ifdef WORDS_BIGENDIAN + header->hdr_base.hdr_flags |= OMPI_OSC_PT2PT_HDR_FLAG_NBO; +#elif OMPI_ENABLE_HETEROGENEOUS_SUPPORT + if (sendreq->req_target_proc->proc_arch & OMPI_ARCH_ISBIGENDIAN) { + header->hdr_base.hdr_flags |= OMPI_OSC_PT2PT_HDR_FLAG_NBO; + OMPI_OSC_PT2PT_SEND_HDR_HTON(*header); + } #endif /* send fragment */ @@ -320,6 +331,12 @@ ompi_osc_pt2pt_replyreq_send_cb(struct mca_btl_base_module_t* btl, return; } +#if !defined(WORDS_BIGENDIAN) && OMPI_ENABLE_HETEROGENEOUS_SUPPORT + if (header->hdr_base.hdr_flags & OMPI_OSC_PT2PT_HDR_FLAG_NBO) { + OMPI_OSC_PT2PT_REPLY_HDR_NTOH(*header); + } +#endif + /* do we need to post a send? */ if (header->hdr_msg_length != 0) { /* sendreq is done. Mark it as so and get out of here */ @@ -390,6 +407,7 @@ ompi_osc_pt2pt_replyreq_send(ompi_osc_pt2pt_module_t *module, header = (ompi_osc_pt2pt_reply_header_t*) descriptor->des_src[0].seg_addr.pval; written_data += sizeof(ompi_osc_pt2pt_reply_header_t); header->hdr_base.hdr_type = OMPI_OSC_PT2PT_HDR_REPLY; + header->hdr_base.hdr_flags = 0; header->hdr_origin_sendreq = replyreq->rep_origin_sendreq; header->hdr_target_tag = 0; @@ -421,9 +439,13 @@ ompi_osc_pt2pt_replyreq_send(ompi_osc_pt2pt_module_t *module, header->hdr_target_tag = create_send_tag(module); } -#if 0 /* BWB - FIX ME */ - /* put in network byte order */ - OMPI_OSC_PT2PT_REPLYREQ_HDR_HTON(header); +#ifdef WORDS_BIGENDIAN + header->hdr_base.hdr_flags |= OMPI_OSC_PT2PT_HDR_FLAG_NBO; +#elif OMPI_ENABLE_HETEROGENEOUS_SUPPORT + if (replyreq->rep_origin_proc->proc_arch & OMPI_ARCH_ISBIGENDIAN) { + header->hdr_base.hdr_flags |= OMPI_OSC_PT2PT_HDR_FLAG_NBO; + OMPI_OSC_PT2PT_REPLY_HDR_HTON(*header); + } #endif /* send fragment */ @@ -786,9 +808,13 @@ ompi_osc_pt2pt_control_send(ompi_osc_pt2pt_module_t *module, header->hdr_value[1] = value1; header->hdr_windx = module->p2p_comm->c_contextid; -#if 0 /* BWB - FIX ME */ - /* put in network byte order */ - OMPI_OSC_PT2PT_CONTROL_HDR_HTON(header); +#ifdef WORDS_BIGENDIAN + header->hdr_base.hdr_flags |= OMPI_OSC_PT2PT_HDR_FLAG_NBO; +#elif OMPI_ENABLE_HETEROGENEOUS_SUPPORT + if (proc->proc_arch & OMPI_ARCH_ISBIGENDIAN) { + header->hdr_base.hdr_flags |= OMPI_OSC_PT2PT_HDR_FLAG_NBO; + OMPI_OSC_PT2PT_CONTROL_HDR_HTON(*header); + } #endif /* send fragment */ diff --git a/ompi/mca/osc/pt2pt/osc_pt2pt_header.h b/ompi/mca/osc/pt2pt/osc_pt2pt_header.h index 5363d2d584..d6a44a2451 100644 --- a/ompi/mca/osc/pt2pt/osc_pt2pt_header.h +++ b/ompi/mca/osc/pt2pt/osc_pt2pt_header.h @@ -32,6 +32,8 @@ #define OMPI_OSC_PT2PT_HDR_LOCK_REQ 0x0040 #define OMPI_OSC_PT2PT_HDR_UNLOCK_REQ 0x0080 +#define OMPI_OSC_PT2PT_HDR_FLAG_NBO 0x0001 + struct ompi_osc_pt2pt_base_header_t { uint8_t hdr_type; /* eventually, this will include endian information */ @@ -58,12 +60,11 @@ struct ompi_osc_pt2pt_send_header_t { }; typedef struct ompi_osc_pt2pt_send_header_t ompi_osc_pt2pt_send_header_t; -#define OMPI_OSC_PT2PT_REQ_HDR_HTON(hdr) \ +#define OMPI_OSC_PT2PT_SEND_HDR_HTON(hdr) \ do { \ OMPI_OSC_PT2PT_BASE_HDR_HTON((hdr).hdr_base) \ (hdr).hdr_windx = htons((hdr).hdr_windx); \ (hdr).hdr_origin = htonl((hdr).hdr_origin); \ - (hdr).hdr_origin_sendreq = hton64((hdr).hdr_origin_sendreq); \ (hdr).hdr_origin_tag = htonl((hdr).hdr_origin_tag); \ (hdr).hdr_target_disp = htonl((hdr).hdr_target_disp); \ (hdr).hdr_target_count = htonl((hdr).hdr_target_count); \ @@ -71,12 +72,11 @@ typedef struct ompi_osc_pt2pt_send_header_t ompi_osc_pt2pt_send_header_t; (hdr).hdr_msg_length = htonl((hdr).hdr_msg_length); \ } while (0) -#define OMPI_OSC_PT2PT_REQ_HDR_NTOH(hdr) \ +#define OMPI_OSC_PT2PT_SEND_HDR_NTOH(hdr) \ do { \ OMPI_OSC_PT2PT_BASE_HDR_NTOH((hdr).hdr_base) \ (hdr).hdr_windx = ntohs((hdr).hdr_windx); \ (hdr).hdr_origin = ntohl((hdr).hdr_origin); \ - (hdr).hdr_origin_sendreq = ntoh64((hdr).hdr_origin_sendreq); \ (hdr).hdr_origin_tag = ntohl((hdr).hdr_origin_tag); \ (hdr).hdr_target_disp = ntohl((hdr).hdr_target_disp); \ (hdr).hdr_target_count = ntohl((hdr).hdr_target_count); \ @@ -98,7 +98,6 @@ typedef struct ompi_osc_pt2pt_reply_header_t ompi_osc_pt2pt_reply_header_t; #define OMPI_OSC_PT2PT_REPLY_HDR_HTON(hdr) \ do { \ OMPI_OSC_PT2PT_BASE_HDR_HTON((hdr).hdr_base) \ - (hdr).hdr_origin_sendreq = hton64((hdr).hdr_origin_sendreq); \ (hdr).hdr_target_tag = htonl((hdr).hdr_target_tag); \ (hdr).hdr_msg_length = htonl((hdr).hdr_msg_length); \ } while (0) @@ -106,7 +105,6 @@ typedef struct ompi_osc_pt2pt_reply_header_t ompi_osc_pt2pt_reply_header_t; #define OMPI_OSC_PT2PT_REPLY_HDR_NTOH(hdr) \ do { \ OMPI_OSC_PT2PT_BASE_HDR_NTOH((hdr).hdr_base) \ - (hdr).hdr_origin_sendreq = ntoh64((hdr).hdr_origin_sendreq); \ (hdr).hdr_target_tag = ntohl((hdr).hdr_target_tag); \ (hdr).hdr_msg_length = ntohl((hdr).hdr_msg_length); \ } while (0)