diff --git a/ompi/mca/osc/rdma/osc_rdma_component.c b/ompi/mca/osc/rdma/osc_rdma_component.c index 0958f88ca5..45a7191293 100644 --- a/ompi/mca/osc/rdma/osc_rdma_component.c +++ b/ompi/mca/osc/rdma/osc_rdma_component.c @@ -11,6 +11,7 @@ * Copyright (c) 2007 Los Alamos National Security, LLC. All rights * reserved. * Copyright (c) 2006-2008 University of Houston. All rights reserved. + * Copyright (c) 2010 Oracle and/or its affiliates. All rights reserved. * $COPYRIGHT$ * * Additional copyrights may follow @@ -32,6 +33,7 @@ #include "opal/threads/condition.h" #include "opal/threads/mutex.h" #include "opal/util/arch.h" +#include "opal/align.h" #include "ompi/info/info.h" #include "ompi/communicator/communicator.h" @@ -919,6 +921,10 @@ component_fragment_cb(struct mca_btl_base_module_t *btl, } if ((base_header->hdr_flags & OMPI_OSC_RDMA_HDR_FLAG_MULTI) != 0) { + + /* The next header starts at the next aligned address in the + * buffer. Therefore, bump pointer forward if necessary. */ + payload += OPAL_ALIGN_PAD_AMOUNT(payload, sizeof(void*)); base_header = (ompi_osc_rdma_base_header_t*) payload; } else { done = true; diff --git a/ompi/mca/osc/rdma/osc_rdma_data_move.c b/ompi/mca/osc/rdma/osc_rdma_data_move.c index 5f00f6a056..18a34a03ac 100644 --- a/ompi/mca/osc/rdma/osc_rdma_data_move.c +++ b/ompi/mca/osc/rdma/osc_rdma_data_move.c @@ -9,7 +9,7 @@ * All rights reserved. * Copyright (c) 2007 Los Alamos National Security, LLC. All rights * reserved. - * Copyright (c) 2009 Sun Microsystems, Inc. All rights reserved. + * Copyright (c) 2009-2010 Oracle and/or its affiliates. All rights reserved. * $COPYRIGHT$ * * Additional copyrights may follow @@ -28,6 +28,7 @@ #include "opal/util/arch.h" #include "opal/util/output.h" #include "opal/sys/atomic.h" +#include "opal/align.h" #include "ompi/mca/pml/pml.h" #include "ompi/mca/bml/bml.h" #include "ompi/mca/bml/base/base.h" @@ -366,6 +367,12 @@ ompi_osc_rdma_sendreq_send_cb(struct mca_btl_base_module_t* btl, sizeof(ompi_osc_rdma_send_header_t) + ompi_datatype_pack_description_length(sendreq->req_target_datatype) + header->hdr_msg_length); + + /* The next header starts at the next aligned address in the + * buffer. Therefore, bump pointer forward if necessary. */ + header = (ompi_osc_rdma_send_header_t*)((char*)header + + OPAL_ALIGN_PAD_AMOUNT(header, sizeof(void*))); + if (header->hdr_base.hdr_type == OMPI_OSC_RDMA_HDR_MULTI_END) { done = true; } @@ -416,6 +423,7 @@ ompi_osc_rdma_sendreq_send(ompi_osc_rdma_module_t *module, mca_btl_base_descriptor_t *descriptor = NULL; ompi_osc_rdma_send_header_t *header = NULL; size_t written_data = 0; + size_t offset; size_t needed_len = sizeof(ompi_osc_rdma_send_header_t); const void *packed_ddt; size_t packed_ddt_len, remain; @@ -575,6 +583,19 @@ ompi_osc_rdma_sendreq_send(ompi_osc_rdma_module_t *module, /* not enough space left - send now */ ret = send_multi_buffer(module, sendreq->req_target_rank); } else { + + /* When putting multiple messages in a single buffer, the starting + * point for the next message needs to be aligned with pointer + * addresses. Therefore, the pointer and the amount written are + * adjusted forward so that the starting position for the next + * message is aligned properly. This strict alignment is required + * by certain platforms like SPARC. Without it, bus errors can + * occur. Keeping things aligned also may offer some performance + * improvements on other platforms. */ + offset = OPAL_ALIGN_PAD_AMOUNT(descriptor->des_src[0].seg_len, sizeof(void*)); + descriptor->des_src[0].seg_len += offset; + written_data += offset; + ret = OMPI_SUCCESS; } diff --git a/opal/include/opal/align.h b/opal/include/opal/align.h index c5db49033e..c445b06217 100644 --- a/opal/include/opal/align.h +++ b/opal/include/opal/align.h @@ -10,6 +10,7 @@ * Copyright (c) 2004-2005 The Regents of the University of California. * All rights reserved. * Copyright (c) 2006 Voltaire All rights reserved. + * Copyright (c) 2010 Oracle and/or its affiliates. All rights reserved. * * $COPYRIGHT$ * @@ -23,5 +24,6 @@ #define OPAL_ALIGN(x,a,t) (((x)+((t)(a)-1)) & ~(((t)(a)-1))) #define OPAL_ALIGN_PTR(x,a,t) ((t)OPAL_ALIGN((uintptr_t)x, a, uintptr_t)) +#define OPAL_ALIGN_PAD_AMOUNT(x,s) ((~((uintptr_t)(x))+1) & ((uintptr_t)(s)-1)) #endif /* OPAL_ALIGN_H */