From 86a7b317a5d9f228342cddf813dffad48e83a7d1 Mon Sep 17 00:00:00 2001 From: George Bosilca Date: Tue, 9 May 2017 16:57:15 -0400 Subject: [PATCH] Allow MPI_ANY_SOURCE in MPI_Sendrecv_replace. Signed-off-by: George Bosilca --- ompi/mpi/c/sendrecv_replace.c | 129 +++++++++++++++++----------------- 1 file changed, 64 insertions(+), 65 deletions(-) diff --git a/ompi/mpi/c/sendrecv_replace.c b/ompi/mpi/c/sendrecv_replace.c index 0063125119..98b3089bfc 100644 --- a/ompi/mpi/c/sendrecv_replace.c +++ b/ompi/mpi/c/sendrecv_replace.c @@ -2,7 +2,7 @@ * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana * University Research and Technology * Corporation. All rights reserved. - * Copyright (c) 2004-2010 The University of Tennessee and The University + * Copyright (c) 2004-2017 The University of Tennessee and The University * of Tennessee Research Foundation. All rights * reserved. * Copyright (c) 2004-2008 High Performance Computing Center Stuttgart, @@ -48,10 +48,10 @@ int MPI_Sendrecv_replace(void * buf, int count, MPI_Datatype datatype, int rc = MPI_SUCCESS; MEMCHECKER( - memchecker_datatype(datatype); - memchecker_call(&opal_memchecker_base_isdefined, buf, count, datatype); - memchecker_comm(comm); - ); + memchecker_datatype(datatype); + memchecker_call(&opal_memchecker_base_isdefined, buf, count, datatype); + memchecker_comm(comm); + ); if ( MPI_PARAM_CHECK ) { rc = MPI_SUCCESS; @@ -76,68 +76,67 @@ int MPI_Sendrecv_replace(void * buf, int count, MPI_Datatype datatype, /* simple case */ if ( source == MPI_PROC_NULL || dest == MPI_PROC_NULL || count == 0 ) { - rc = PMPI_Sendrecv(buf,count,datatype,dest,sendtag,buf,count,datatype,source,recvtag,comm,status); + rc = PMPI_Sendrecv(buf, count, datatype, dest, sendtag, buf, count, datatype, source, recvtag, comm, status); OPAL_CR_EXIT_LIBRARY(); return rc; - } else { - - opal_convertor_t convertor; - struct iovec iov; - unsigned char recv_data[2048]; - size_t packed_size, max_data; - uint32_t iov_count; - ompi_status_public_t recv_status; - ompi_proc_t* proc = ompi_comm_peer_lookup(comm,source); - if(proc == NULL) { - rc = MPI_ERR_RANK; - OMPI_ERRHANDLER_RETURN(rc, comm, rc, FUNC_NAME); - } - - /* initialize convertor to unpack recv buffer */ - OBJ_CONSTRUCT(&convertor, opal_convertor_t); - opal_convertor_copy_and_prepare_for_recv( proc->super.proc_convertor, &(datatype->super), - count, buf, 0, &convertor ); - - /* setup a buffer for recv */ - opal_convertor_get_packed_size( &convertor, &packed_size ); - if( packed_size > sizeof(recv_data) ) { - rc = PMPI_Alloc_mem(packed_size, MPI_INFO_NULL, &iov.iov_base); - if(OMPI_SUCCESS != rc) { - OMPI_ERRHANDLER_RETURN(OMPI_ERR_OUT_OF_RESOURCE, comm, MPI_ERR_BUFFER, FUNC_NAME); - } - } else { - iov.iov_base = (caddr_t)recv_data; - } - - /* recv into temporary buffer */ - rc = PMPI_Sendrecv( buf, count, datatype, dest, sendtag, iov.iov_base, packed_size, - MPI_BYTE, source, recvtag, comm, &recv_status ); - if (rc != MPI_SUCCESS) { - if(packed_size > sizeof(recv_data)) - PMPI_Free_mem(iov.iov_base); - OBJ_DESTRUCT(&convertor); - OMPI_ERRHANDLER_RETURN(rc, comm, rc, FUNC_NAME); - } - - /* unpack into users buffer */ - iov.iov_len = recv_status._ucount; - iov_count = 1; - max_data = recv_status._ucount; - opal_convertor_unpack(&convertor, &iov, &iov_count, &max_data ); - - /* return status to user */ - if(status != MPI_STATUS_IGNORE) { - *status = recv_status; - } - - /* release resources */ - if(packed_size > sizeof(recv_data)) { - PMPI_Free_mem(iov.iov_base); - } - OBJ_DESTRUCT(&convertor); - - OPAL_CR_EXIT_LIBRARY(); - return MPI_SUCCESS; } + + /** + * If we look for an optimal solution, then we should receive the data into a temporary buffer + * and once the send completes we would unpack back into the original buffer. However, if the + * sender is unknown, this approach can only be implementing by receiving with the recv datatype + * (potentially non-contiguous) and thus the allocated memory will be larger than the size of the + * datatype. A simpler, but potentially less efficient approach is to work on the data we have + * control of, aka the sent data, and pack it into a contiguous buffer before posting the receive. + * Once the send completes, we free it. + */ + opal_convertor_t convertor; + unsigned char packed_data[2048]; + struct iovec iov = { .iov_base = packed_data, .iov_len = sizeof(packed_data) }; + size_t packed_size, max_data; + uint32_t iov_count; + ompi_status_public_t recv_status; + ompi_proc_t* proc = ompi_comm_peer_lookup(comm, dest); + if(proc == NULL) { + rc = MPI_ERR_RANK; + OMPI_ERRHANDLER_RETURN(rc, comm, rc, FUNC_NAME); + } + + /* initialize convertor to unpack recv buffer */ + OBJ_CONSTRUCT(&convertor, opal_convertor_t); + opal_convertor_copy_and_prepare_for_send( proc->super.proc_convertor, &(datatype->super), + count, buf, 0, &convertor ); + + /* setup a buffer for recv */ + opal_convertor_get_packed_size( &convertor, &packed_size ); + if( packed_size > sizeof(packed_data) ) { + rc = PMPI_Alloc_mem(packed_size, MPI_INFO_NULL, &iov.iov_base); + if(OMPI_SUCCESS != rc) { + rc = OMPI_ERR_OUT_OF_RESOURCE; + goto cleanup_and_return; + } + } + max_data = packed_size; + iov_count = 1; + rc = opal_convertor_pack(&convertor, &iov, &iov_count, &max_data); + + /* recv into temporary buffer */ + rc = PMPI_Sendrecv( iov.iov_base, packed_size, MPI_PACKED, dest, sendtag, buf, count, + datatype, source, recvtag, comm, &recv_status ); + + cleanup_and_return: + /* return status to user */ + if(status != MPI_STATUS_IGNORE) { + *status = recv_status; + } + + /* release resources */ + if(packed_size > sizeof(packed_data)) { + PMPI_Free_mem(iov.iov_base); + } + OBJ_DESTRUCT(&convertor); + + OPAL_CR_EXIT_LIBRARY(); + OMPI_ERRHANDLER_RETURN(rc, comm, rc, FUNC_NAME); }