From 0e393195d9f2373ffa9d59a240092f643117cd39 Mon Sep 17 00:00:00 2001
From: Gilles Gouaillardet <gilles@rist.or.jp>
Date: Wed, 8 Jun 2016 16:48:00 +0900
Subject: [PATCH] coll/base: fix [all]reduce with non zero lower bound
 datatypes

Offset temporary buffer when a non zero lower bound datatype is used.

Thanks Hristo Iliev for the report
---
 ompi/mca/coll/base/coll_base_allreduce.c | 11 ++++++-----
 ompi/mca/coll/base/coll_base_reduce.c    | 22 ++++++++++++----------
 2 files changed, 18 insertions(+), 15 deletions(-)

diff --git a/ompi/mca/coll/base/coll_base_allreduce.c b/ompi/mca/coll/base/coll_base_allreduce.c
index c21676af1e..aa5a62796b 100644
--- a/ompi/mca/coll/base/coll_base_allreduce.c
+++ b/ompi/mca/coll/base/coll_base_allreduce.c
@@ -13,7 +13,7 @@
  * Copyright (c) 2009      University of Houston. All rights reserved.
  * Copyright (c) 2013      Los Alamos National Security, LLC. All Rights
  *                         reserved.
- * Copyright (c) 2015      Research Organization for Information Science
+ * Copyright (c) 2015-2016 Research Organization for Information Science
  *                         and Technology (RIST). All rights reserved.
  * $COPYRIGHT$
  *
@@ -134,7 +134,7 @@ ompi_coll_base_allreduce_intra_recursivedoubling(const void *sbuf, void *rbuf,
 {
     int ret, line, rank, size, adjsize, remote, distance;
     int newrank, newremote, extra_ranks;
-    char *tmpsend = NULL, *tmprecv = NULL, *tmpswap = NULL, *inplacebuf = NULL;
+    char *tmpsend = NULL, *tmprecv = NULL, *tmpswap = NULL, *inplacebuf_free = NULL, *inplacebuf;
     ompi_request_t *reqs[2] = {NULL, NULL};
     OPAL_PTRDIFF_TYPE span, gap;
 
@@ -155,8 +155,9 @@ ompi_coll_base_allreduce_intra_recursivedoubling(const void *sbuf, void *rbuf,
 
     /* Allocate and initialize temporary send buffer */
     span = opal_datatype_span(&dtype->super, count, &gap);
-    inplacebuf = (char*) malloc(span);
-    if (NULL == inplacebuf) { ret = -1; line = __LINE__; goto error_hndl; }
+    inplacebuf_free = (char*) malloc(span);
+    if (NULL == inplacebuf_free) { ret = -1; line = __LINE__; goto error_hndl; }
+    inplacebuf = inplacebuf_free - gap;
 
     if (MPI_IN_PLACE == sbuf) {
         ret = ompi_datatype_copy_content_same_ddt(dtype, count, inplacebuf, (char*)rbuf);
@@ -263,7 +264,7 @@ ompi_coll_base_allreduce_intra_recursivedoubling(const void *sbuf, void *rbuf,
         if (ret < 0) { line = __LINE__; goto error_hndl; }
     }
 
-    if (NULL != inplacebuf) free(inplacebuf);
+    if (NULL != inplacebuf_free) free(inplacebuf_free);
     return MPI_SUCCESS;
 
  error_hndl:
diff --git a/ompi/mca/coll/base/coll_base_reduce.c b/ompi/mca/coll/base/coll_base_reduce.c
index f5b2449727..23d6838772 100644
--- a/ompi/mca/coll/base/coll_base_reduce.c
+++ b/ompi/mca/coll/base/coll_base_reduce.c
@@ -12,7 +12,7 @@
  *                         All rights reserved.
  * Copyright (c) 2013      Los Alamos National Security, LLC. All Rights
  *                         reserved.
- * Copyright (c) 2015      Research Organization for Information Science
+ * Copyright (c) 2015-2016 Research Organization for Information Science
  *                         and Technology (RIST). All rights reserved.
  * $COPYRIGHT$
  *
@@ -485,6 +485,7 @@ int ompi_coll_base_reduce_intra_in_order_binary( const void *sendbuf, void *recv
     int ret, rank, size, io_root, segcount = count;
     void *use_this_sendbuf = NULL;
     void *use_this_recvbuf = NULL;
+    char *tmpbuf_free = NULL;
     size_t typelng;
     mca_coll_base_module_t *base_module = (mca_coll_base_module_t*) module;
     mca_coll_base_comm_t *data = base_module->base_data;
@@ -515,24 +516,26 @@ int ompi_coll_base_reduce_intra_in_order_binary( const void *sendbuf, void *recv
     use_this_recvbuf = recvbuf;
     if (io_root != root) {
         ptrdiff_t dsize, gap;
-        char *tmpbuf = NULL;
+        char *tmpbuf;
 
         dsize = opal_datatype_span(&datatype->super, count, &gap);
 
         if ((root == rank) && (MPI_IN_PLACE == sendbuf)) {
-            tmpbuf = (char *) malloc(dsize);
-            if (NULL == tmpbuf) {
+            tmpbuf_free = (char *) malloc(dsize);
+            if (NULL == tmpbuf_free) {
                 return MPI_ERR_INTERN;
             }
+            tmpbuf = tmpbuf_free - gap;
             ompi_datatype_copy_content_same_ddt(datatype, count,
                                                 (char*)tmpbuf,
                                                 (char*)recvbuf);
             use_this_sendbuf = tmpbuf;
         } else if (io_root == rank) {
-            tmpbuf = (char *) malloc(dsize);
-            if (NULL == tmpbuf) {
+            tmpbuf_free = (char *) malloc(dsize);
+            if (NULL == tmpbuf_free) {
                 return MPI_ERR_INTERN;
             }
+            tmpbuf = tmpbuf_free - gap;
             use_this_recvbuf = tmpbuf;
         }
     }
@@ -552,9 +555,6 @@ int ompi_coll_base_reduce_intra_in_order_binary( const void *sendbuf, void *recv
                                     MCA_COLL_BASE_TAG_REDUCE, comm,
                                     MPI_STATUS_IGNORE));
             if (MPI_SUCCESS != ret) { return ret; }
-            if (MPI_IN_PLACE == sendbuf) {
-                free(use_this_sendbuf);
-            }
 
         } else if (io_root == rank) {
             /* Send result from use_this_recvbuf to root */
@@ -562,9 +562,11 @@ int ompi_coll_base_reduce_intra_in_order_binary( const void *sendbuf, void *recv
                                     MCA_COLL_BASE_TAG_REDUCE,
                                     MCA_PML_BASE_SEND_STANDARD, comm));
             if (MPI_SUCCESS != ret) { return ret; }
-            free(use_this_recvbuf);
         }
     }
+    if (NULL != tmpbuf_free) {
+        free(tmpbuf_free);
+    }
 
     return MPI_SUCCESS;
 }