osc-portals4: add support for noncontiguous datatypes

This commit implements onesided operations for noncontiguous
datatypes using two different algorithms.

 * If the result and/or origin datatype is noncontiguous and the
   target datatype is contiguous, then an iovec MD is created for
   the result and origin.  The operation is performed using a
   single Portals4 call (unless it exceeds the max message size).
 * If the target datatype is noncontigous, then an algorithm
   similar to the one in osc-rdma is used to loop over the
   contiguous blocks of each datatype.  The operation is
   performed using multiple Portals4 calls.

This commit ensures that individual operations do not exceed the
max atomic size or the max message size supported by the device.

Signed-off-by: Todd Kordenbrock <thkgcode@gmail.com>

ompi/mca/osc/portals4/osc_portals4.h

@@ -1,6 +1,6 @@
 /* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
 /*
- * Copyright (c) 2011-2013 Sandia National Laboratories.  All rights reserved.
+ * Copyright (c) 2011-2017 Sandia National Laboratories.  All rights reserved.
  * Copyright (c) 2015      Los Alamos National Security, LLC.  All rights
  *                         reserved.
  * Copyright (c) 2015      Research Organization for Information Science
@@ -23,6 +23,8 @@
 
 #define REQ_OSC_TABLE_ID     4
 
+#define OSC_PORTALS4_IOVEC_MAX 64
+
 #define OSC_PORTALS4_MB_DATA    0x0000000000000000ULL
 #define OSC_PORTALS4_MB_CONTROL 0x1000000000000000ULL
 
@@ -95,6 +97,11 @@ struct ompi_osc_portals4_module_t {
     int64_t opcount;
     ptl_match_bits_t match_bits; /* match bits for module.  Same as cid for comm in most cases. */
 
+    ptl_iovec_t     *origin_iovec_list; /* list of memory segments that compose the noncontiguous region */
+    ptl_handle_md_t  origin_iovec_md_h; /* memory descriptor describing a noncontiguous region in this window */
+    ptl_iovec_t     *result_iovec_list; /* list of memory segments that compose the noncontiguous region */
+    ptl_handle_md_t  result_iovec_md_h; /* memory descriptor describing a noncontiguous region in this window */
+
     ptl_size_t atomic_max; /* max size of atomic messages.  Will guarantee ordering IF ordering requested */
     ptl_size_t fetch_atomic_max; /* max size of fetchatomic messages.  Will guarantee ordering IF ordering requested */
 

ompi/mca/osc/portals4/osc_portals4_component.c

@@ -1,6 +1,6 @@
 /* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
 /*
- * Copyright (c) 2011-2013 Sandia National Laboratories.  All rights reserved.
+ * Copyright (c) 2011-2017 Sandia National Laboratories.  All rights reserved.
  * Copyright (c) 2015      Los Alamos National Security, LLC.  All rights
  *                         reserved.
  * Copyright (c) 2015      Research Organization for Information Science
@@ -506,6 +506,11 @@ component_select(struct ompi_win_t *win, void **base, size_t size, int disp_unit
         goto error;
     }
 
+    module->origin_iovec_list = NULL;
+    module->origin_iovec_md_h = PTL_INVALID_HANDLE;
+    module->result_iovec_list = NULL;
+    module->result_iovec_md_h = PTL_INVALID_HANDLE;
+
     if (MPI_WIN_FLAVOR_DYNAMIC == flavor) {
         me.start = 0;
         me.length = PTL_SIZE_MAX;
@@ -650,6 +655,14 @@ ompi_osc_portals4_free(struct ompi_win_t *win)
     PtlMEUnlink(module->control_me_h);
     PtlMEUnlink(module->data_me_h);
     PtlMDRelease(module->md_h);
+    if (module->origin_iovec_md_h != PTL_INVALID_HANDLE) {
+        PtlMDRelease(module->origin_iovec_md_h);
+        free(module->origin_iovec_list);
+    }
+    if (module->result_iovec_md_h != PTL_INVALID_HANDLE) {
+        PtlMDRelease(module->result_iovec_md_h);
+        free(module->result_iovec_list);
+    }
     PtlMDRelease(module->req_md_h);
     PtlCTFree(module->ct_h);
     if (NULL != module->disp_units) free(module->disp_units);