From aeb2c02d2fc4de9bcb7db5a754eb3b267da67c9f Mon Sep 17 00:00:00 2001
From: Mark Allen <markalle@us.ibm.com>
Date: Wed, 7 Jun 2017 16:53:03 -0400
Subject: [PATCH] Type_create_darray with mix of BLOCK/CYCLIC

Example (using MPI_ORDER_C so the below has 6 rows of 4 ints to parcel out)
    size = 4;
    rank = 0;
    ndims=2;
    gsizes[0] = 6;
    gsizes[1] = 4;
    distribs[0] = MPI_DISTRIBUTE_CYCLIC;
    distribs[1] = MPI_DISTRIBUTE_BLOCK;
    dargs[0] = 2;
    dargs[1] = 2;
    psizes[0] = 2;
    psizes[1] = 2;
    MPI_Type_create_darray(size, rank, ndims,
        gsizes, distribs, dargs, psizes,
        MPI_ORDER_C, MPI_INT, &mydt);

Expectation for the layout:
   inner dimension (1) is
       4 items (ints) distributed block over 2 ranks with 2 items each
       eg for rank 0: [ x x . . ]
   outer dimension (0) is:
       6 items (the above [ x x . .]) cyclic over 2 ranks with 2 items each
       eg for rank 0:
           [ x x . . ]    :  offset=0 bytes=8
           [ x x . . ]    :  ofset=16 bytes=8
           [ . . . . ]
           [ . . . . ]
           [ x x . . ]    :  offset=64 bytes=8
           [ x x . . ]    :  offset=80 bytes=8

Or more specifically a stream of ints 0,1,2,3,4,5,6,7 sent into that
type should be
    [ 0 1 . . ]
    [ 2 3 . . ]
    [ . . . . ]
    [ . . . . ]
    [ 4 5 . . ]
    [ 6 7 . . ]
The data was laying out though as
    [ 0 1 2 3 ]
    [ . . . . ]
    [ . . . . ]
    [ . . . . ]
    [ 4 5 6 7 ]
    [ . . . . ]
because the recursive construction inside the block() function (which
creates the smaller row datatype [ x x . . ]) wasn't setting the extent
of that type.

Signed-off-by: Mark Allen <markalle@us.ibm.com>
---
 ompi/datatype/ompi_datatype_create_darray.c | 17 ++++++++++++++++-
 1 file changed, 16 insertions(+), 1 deletion(-)

diff --git a/ompi/datatype/ompi_datatype_create_darray.c b/ompi/datatype/ompi_datatype_create_darray.c
index 98c81f0dc2..a245dcebce 100644
--- a/ompi/datatype/ompi_datatype_create_darray.c
+++ b/ompi/datatype/ompi_datatype_create_darray.c
@@ -15,6 +15,7 @@
  * Copyright (c) 2010      Cisco Systems, Inc.  All rights reserved.
  * Copyright (c) 2016      Los Alamos National Security, LLC. All rights
  *                         reserved.
+ * Copyright (c) 2017      IBM Corporation. All rights reserved.
  * $COPYRIGHT$
  *
  * Additional copyrights may follow
@@ -35,7 +36,7 @@ block(const int *gsize_array, int dim, int ndims, int nprocs,
       ptrdiff_t *st_offset)
 {
     int blksize, global_size, mysize, i, j, rc, start_loop, step;
-    ptrdiff_t stride;
+    ptrdiff_t stride, disps[2];
 
     global_size = gsize_array[dim];
 
@@ -71,6 +72,20 @@ block(const int *gsize_array, int dim, int ndims, int nprocs,
     /* in terms of no. of elements of type oldtype in this dimension */
     if (mysize == 0) *st_offset = 0;
 
+    /* need to set the UB for block-cyclic to work */
+    disps[0] = 0;         disps[1] = orig_extent;
+    if (order == MPI_ORDER_FORTRAN) {
+        for(i=0; i<=dim; i++) {
+            disps[1] *= gsize_array[i];
+        }
+    } else {
+        for(i=ndims-1; i>=dim; i--) {
+            disps[1] *= gsize_array[i];
+        }
+    }
+    rc = opal_datatype_resize( &(*type_new)->super, disps[0], disps[1] );
+    if (OMPI_SUCCESS != rc) return rc;
+
     return OMPI_SUCCESS;
 }