Merge pull request #4520 from ggouaillardet/refresh/romio321

io/romio321: refresh ROMIO based on latest stable MPICH 3.2.1

ompi/mca/io/romio314/REFRESH_NOTES.txt

@@ -1,58 +0,0 @@
-1. prepare new romio directory
-
-cp -a <old romio> <new romio>
-# cp -a ompi/mca/io/romio ompi/mca/io/romio314
-rm -rf <new romio>/romio
-# rm -rf ompi/mca/io/romio314/romio
-git add <new romio>
-# git add ompi/mca/io/romio314
-# git commit
-
-
-2. import new romio
-
-cp -a <mpich dir>/src/mpi/romio <new romio>
-# cp -a /.../mpich-3.2b1/src/mpi/romio ompi/mca/io/romio314
-git add <new romio>
-# git add ompi/mca/io/romio314
-# git commit
-
-3. patch romio
-
-cd <new romio>
-# cd ompi/mca/io/romio314
-# git show 7ad376a7b202b45c8b4f574e873546ab8752e305 | patch -p5
-# cd ../../../..
-git add <new romio>
-# git add ompi/mca/io/romio314
-# git commit
-
-
-4. apply post romio 3.1.4 patches
-# cd ompi/mca/io/romio314
-# git show 53fd425a6a0843a5de0a8c544901fbf01246ed31 | patch -p5
-# git show 74a46863ca3d0806050e7a55377a4fbde612f3fe | patch -p5
-# git show 2b5c52fb05ab54e4fa72e948acabc42e3f979ebd | patch -p5
-# git show 24a6f1425734eb3e6ffb9e37f510507a4a65ebd1 | patch -p5
-# git show 23b27c510c2c626a09457abad7699cfc4740d1eb | patch -p5
-# cd ../../../..
-# git add ompi/mca/io/romio314
-# git commit
-
-
-4. prepare ompi to use new romio
-
-vi <new romio>/Makefile.am <new romio>/configure.m4 <new romio>/src/Makefile.extra
-# vi ompi/mca/io/romio314/Makefile.am ompi/mca/io/romio314/configure.m4 ompi/mca/io/romio314/src/Makefile.extra
-git add <new romio>
-# git add ompi/mca/io/romio314
-# git commit
-
-
-5. remove old romio
-
-git rm -r <old romio>
-# git rm -r ompi/mca/io/romio
-# git commit
-rm -rf <old romio>
-# rm -rf ompi/mca/io/romio

ompi/mca/io/romio314/romio/localdefs.in

@@ -1,4 +0,0 @@
-#! /bin/sh
-LIBS="@LIBS@"
-MPI_OFFSET_TYPE="@MPI_OFFSET_TYPE@"
-FORTRAN_MPI_OFFSET="@FORTRAN_MPI_OFFSET@"

ompi/mca/io/romio321/Makefile.am

@@ -11,6 +11,8 @@
 #                         All rights reserved.
 # Copyright (c) 2010-2015 Cisco Systems, Inc.  All rights reserved.
 # Copyright (c) 2017      IBM Corporation.  All rights reserved.
+# Copyright (c) 2017      Research Organization for Information Science
+#                         and Technology (RIST). All rights reserved.
 # $COPYRIGHT$
 #
 # Additional copyrights may follow
@@ -23,7 +25,7 @@ EXTRA_DIST = autogen.subdirs
 sources =
 include src/Makefile.extra
 
-if MCA_io_romio314_SHOULD_BUILD
+if MCA_io_romio321_SHOULD_BUILD
 # Need to set both SUBDIRS and DIST_SUBDIRS due to oddities with AM
 SUBDIRS = romio
 DIST_SUBDIRS = romio
@@ -33,13 +35,13 @@ endif
 # mca_<type>_<name>.la (for DSO builds) or libmca_<type>_<name>.la
 # (for static builds).
 
-if MCA_BUILD_ompi_io_romio314_DSO
+if MCA_BUILD_ompi_io_romio321_DSO
 lib =
 lib_sources =
-component = mca_io_romio314.la
+component = mca_io_romio321.la
 component_sources = $(sources)
 else
-lib = libmca_io_romio314.la
+lib = libmca_io_romio321.la
 lib_sources = $(sources)
 component =
 component_sources =
@@ -49,14 +51,13 @@ libs = romio/libromio_dist.la
 
 mcacomponentdir = $(ompilibdir)
 mcacomponent_LTLIBRARIES = $(component)
-mca_io_romio314_la_SOURCES = $(component_sources)
+mca_io_romio321_la_SOURCES = $(component_sources)
-mca_io_romio314_la_LIBADD = $(top_builddir)/ompi/lib@OMPI_LIBMPI_NAME@.la \
+mca_io_romio321_la_LIBADD = $(libs)
-	$(libs)
+mca_io_romio321_la_DEPENDENCIES = $(libs)
-mca_io_romio314_la_DEPENDENCIES = $(libs)
+mca_io_romio321_la_LDFLAGS = -module -avoid-version
-mca_io_romio314_la_LDFLAGS = -module -avoid-version
 
 noinst_LTLIBRARIES = $(lib)
-libmca_io_romio314_la_SOURCES = $(lib_sources)
+libmca_io_romio321_la_SOURCES = $(lib_sources)
-libmca_io_romio314_la_LIBADD = $(libs)
+libmca_io_romio321_la_LIBADD = $(libs)
-libmca_io_romio314_la_DEPENDENCIES = $(libs)
+libmca_io_romio321_la_DEPENDENCIES = $(libs)
-libmca_io_romio314_la_LDFLAGS = -module -avoid-version
+libmca_io_romio321_la_LDFLAGS = -module -avoid-version

ompi/mca/io/romio321/REFRESH_NOTES.txt

@@ -0,0 +1,39 @@
+1. prepare new romio directory
+cp -a <old romio> <new romio>
+# cp -a ompi/mca/io/romio ompi/mca/io/romio321
+rm -rf <new romio>/romio
+# rm -rf ompi/mca/io/romio321/romio
+git add <new romio>
+# git add ompi/mca/io/romio321
+# git commit
+
+2. import new romio
+cp -a <mpich dir>/src/mpi/romio <new romio>
+# cp -a /.../mpich-3.2b1/src/mpi/romio ompi/mca/io/romio321
+git add <new romio>
+# git add ompi/mca/io/romio321
+# git commit
+
+3. patch romio
+cd <new romio>
+# cd ompi/mca/io/romio321
+# git show 7a7968f02e2bad912426d1535119fb30d881b460 | patch -p5
+# cd ../../../..
+git add <new romio>
+# git add ompi/mca/io/romio321
+# git commit
+
+4. prepare ompi to use new romio
+vi <new romio>/Makefile.am <new romio>/configure.m4 <new romio>/src/Makefile.extra
+# vi ompi/mca/io/romio321/Makefile.am ompi/mca/io/romio321/configure.m4 ompi/mca/io/romio321/src/Makefile.extra
+git add <new romio>
+# git add ompi/mca/io/romio321
+# git commit
+
+
+5. remove old romio
+git rm -r <old romio>
+# git rm -r ompi/mca/io/romio
+# git commit
+rm -rf <old romio>
+# rm -rf ompi/mca/io/romio

ompi/mca/io/romio321/configure.m4

@@ -11,7 +11,7 @@
 # Copyright (c) 2004-2005 The Regents of the University of California.
 #                         All rights reserved.
 # Copyright (c) 2008-2015 Cisco Systems, Inc.  All rights reserved.
-# Copyright (c) 2015      Research Organization for Information Science
+# Copyright (c) 2015-2017 Research Organization for Information Science
 #                         and Technology (RIST). All rights reserved.
 # $COPYRIGHT$
 #
@@ -20,24 +20,24 @@
 # $HEADER$
 #
 
-AC_DEFUN([MCA_ompi_io_romio314_POST_CONFIG], [
+AC_DEFUN([MCA_ompi_io_romio321_POST_CONFIG], [
-    AM_CONDITIONAL([MCA_io_romio314_SHOULD_BUILD], [test $1 -eq 1])
+    AM_CONDITIONAL([MCA_io_romio321_SHOULD_BUILD], [test $1 -eq 1])
 ])
 
 
-# MCA_io_romio314_CONFIG([action-if-found], [action-if-not-found])
+# MCA_io_romio321_CONFIG([action-if-found], [action-if-not-found])
 # -----------------------------------------------------------
-AC_DEFUN([MCA_ompi_io_romio314_CONFIG],[
+AC_DEFUN([MCA_ompi_io_romio321_CONFIG],[
-    AC_CONFIG_FILES([ompi/mca/io/romio314/Makefile])
+    AC_CONFIG_FILES([ompi/mca/io/romio321/Makefile])
 
-    OPAL_VAR_SCOPE_PUSH([io_romio314_flags io_romio314_flags_define io_romio314_happy io_romio314_save_LIBS])
+    OPAL_VAR_SCOPE_PUSH([io_romio321_flags io_romio321_flags_define io_romio321_happy io_romio321_save_LIBS])
     AC_ARG_ENABLE([io-romio],
                   [AC_HELP_STRING([--disable-io-romio],
                                   [Disable the ROMIO MPI-IO component])])
     AC_ARG_WITH([io-romio-flags],
                 [AC_HELP_STRING([--with-io-romio-flags=FLAGS],
                                 [Pass FLAGS to the ROMIO distribution configuration script])])
-    AC_DEFINE_UNQUOTED([MCA_io_romio314_USER_CONFIGURE_FLAGS], ["$with_io_romio_flags"], [Set of user-defined configure flags given to ROMIOs configure script via --with-io-romio-flags])
+    AC_DEFINE_UNQUOTED([MCA_io_romio321_USER_CONFIGURE_FLAGS], ["$with_io_romio_flags"], [Set of user-defined configure flags given to ROMIOs configure script via --with-io-romio-flags])
     AC_MSG_CHECKING([if want ROMIO component])
     AS_IF([test "$enable_io_romio" = "no"],
            [AC_MSG_RESULT([no])
@@ -53,50 +53,50 @@ AC_DEFUN([MCA_ompi_io_romio314_CONFIG],[
                   [AC_MSG_RESULT([yes])
 
                    AS_IF([test -n "$with_io_romio_flags" && test "$with_io_romio_flags" != "no"],
-                         [io_romio314_flags="$with_io_romio_flags $io_romio314_flags"],
+                         [io_romio321_flags="$with_io_romio_flags $io_romio321_flags"],
-                         [io_romio314_flags=])
+                         [io_romio321_flags=])
                    # If ROMIO is going to end up in a DSO, all we need is
                    # shared library-ized objects, as we're only building a
                    # DSO (which is always shared).  Otherwise, build with
                    # same flags as OMPI, as we might need any combination of
                    # shared and static-ized objects...
                    AS_IF([test "$compile_mode" = "dso"],
-                         [io_romio314_shared=enable
+                         [io_romio321_shared=enable
-                          io_romio314_static=disable],
+                          io_romio321_static=disable],
                          [AS_IF([test "$enable_shared" = "yes"],
-                                [io_romio314_shared=enable],
+                                [io_romio321_shared=enable],
-                                [io_romio314_shared=disable])
+                                [io_romio321_shared=disable])
                           AS_IF([test "$enable_static" = "yes"],
-                                [io_romio314_static=enable],
+                                [io_romio321_static=enable],
-                                [io_romio314_static=disable])])
+                                [io_romio321_static=disable])])
                    AS_IF([test -n "$prefix" && test "$prefix" != "NONE"],
-                         [io_romio314_prefix_arg="--prefix=$prefix"],
+                         [io_romio321_prefix_arg="--prefix=$prefix"],
-                         [io_romio314_prefix_arg=])
+                         [io_romio321_prefix_arg=])
 
                    AS_IF([test "$cross_compiling" = "yes"],
-                       [AS_IF([test ! -z $build], [io_romio314_flags="$io_romio314_flags --build=$build"])
+                       [AS_IF([test ! -z $build], [io_romio321_flags="$io_romio321_flags --build=$build"])
-                        AS_IF([test ! -z $host], [io_romio314_flags="$io_romio314_flags --host=$host"])
+                        AS_IF([test ! -z $host], [io_romio321_flags="$io_romio321_flags --host=$host"])
-                        AS_IF([test ! -z $target], [io_romio314_flags="$io_romio314_flags --target=$target"])])
+                        AS_IF([test ! -z $target], [io_romio321_flags="$io_romio321_flags --target=$target"])])
-                   io_romio314_flags_define="$io_romio314_flags FROM_OMPI=yes CC='$CC' CFLAGS='$CFLAGS -D__EXTENSIONS__' CPPFLAGS='$CPPFLAGS' FFLAGS='$FFLAGS' LDFLAGS='$LDFLAGS' --$io_romio314_shared-shared --$io_romio314_static-static $io_romio314_flags $io_romio314_prefix_arg --disable-aio --disable-weak-symbols --enable-strict --disable-f77 --disable-f90"
+                   io_romio321_flags_define="$io_romio321_flags FROM_OMPI=yes CC='$CC' CFLAGS='$CFLAGS -D__EXTENSIONS__' CPPFLAGS='$CPPFLAGS' FFLAGS='$FFLAGS' LDFLAGS='$LDFLAGS' --$io_romio321_shared-shared --$io_romio321_static-static $io_romio321_flags $io_romio321_prefix_arg --disable-aio --disable-weak-symbols --enable-strict --disable-f77 --disable-f90"
-                   AC_DEFINE_UNQUOTED([MCA_io_romio314_COMPLETE_CONFIGURE_FLAGS], ["$io_romio314_flags_define"], [Complete set of command line arguments given to ROMIOs configure script])
+                   AC_DEFINE_UNQUOTED([MCA_io_romio321_COMPLETE_CONFIGURE_FLAGS], ["$io_romio321_flags_define"], [Complete set of command line arguments given to ROMIOs configure script])
 
-                   io_romio314_flags="$io_romio314_flags FROM_OMPI=yes CC="'"'"$CC"'"'" CFLAGS="'"'"$CFLAGS -D__EXTENSIONS__"'"'" CPPFLAGS="'"'"$CPPFLAGS"'"'" FFLAGS="'"'"$FFLAGS"'"'" LDFLAGS="'"'"$LDFLAGS"'"'" --$io_romio314_shared-shared --$io_romio314_static-static $io_romio314_flags $io_romio314_prefix_arg --disable-aio --disable-weak-symbols --enable-strict --disable-f77 --disable-f90"
+                   io_romio321_flags="$io_romio321_flags FROM_OMPI=yes CC="'"'"$CC"'"'" CFLAGS="'"'"$CFLAGS -D__EXTENSIONS__"'"'" CPPFLAGS="'"'"$CPPFLAGS"'"'" FFLAGS="'"'"$FFLAGS"'"'" LDFLAGS="'"'"$LDFLAGS"'"'" --$io_romio321_shared-shared --$io_romio321_static-static $io_romio321_flags $io_romio321_prefix_arg --disable-aio --disable-weak-symbols --enable-strict --disable-f77 --disable-f90"
 
                    opal_show_subtitle "Configuring ROMIO distribution"
-                   OPAL_CONFIG_SUBDIR([ompi/mca/io/romio314/romio],
+                   OPAL_CONFIG_SUBDIR([ompi/mca/io/romio321/romio],
-                                      [$io_romio314_flags],
+                                      [$io_romio321_flags],
-                                      [io_romio314_happy=1], [io_romio314_happy=0])
+                                      [io_romio321_happy=1], [io_romio321_happy=0])
 
-                   AS_IF([test "$io_romio314_happy" = "1"],
+                   AS_IF([test "$io_romio321_happy" = "1"],
                          [ # grab the libraries list from ROMIO.  We don't
                            # need this for building the component, as libtool
                            # will figure that part out.  But we do need it for
                            # the wrapper settings
-                          io_romio314_save_LIBS="$LIBS"
+                          io_romio321_save_LIBS="$LIBS"
                           LIBS=
-                          . ompi/mca/io/romio314/romio/localdefs
+                          . ompi/mca/io/romio321/romio/localdefs
-                          io_romio314_LIBS="$LIBS"
+                          io_romio321_LIBS="$LIBS"
-                          LIBS="$io_romio314_save_LIBS"
+                          LIBS="$io_romio321_save_LIBS"
 
                           echo "ROMIO distribution configured successfully"
                           $1],

ompi/mca/io/romio321/ompi.patch

@@ -1,35 +1,16 @@
-diff --git a/ompi/mca/io/romio32b1/romio/.config_params b/ompi/mca/io/romio32b1/romio/.config_params
+diff --git a/ompi/mca/io/romio321/romio/.config_params b/ompi/mca/io/romio321/romio/.config_params
 index 96f735f..fcc2f91 100644
---- a/ompi/mca/io/romio32b1/romio/.config_params
+--- a/ompi/mca/io/romio321/romio/.config_params
-+++ b/ompi/mca/io/romio32b1/romio/.config_params
++++ b/ompi/mca/io/romio321/romio/.config_params
 @@ -36,3 +36,4 @@ __sgi_mpi
  __hp_mpi
  __cray_mpi
  __lam_mpi
 +__open_mpi
-diff --git a/ompi/mca/io/romio32b1/romio/.gitignore b/ompi/mca/io/romio32b1/romio/.gitignore
+diff --git a/ompi/mca/io/romio321/romio/Makefile.am b/ompi/mca/io/romio321/romio/Makefile.am
-new file mode 100644
-index 0000000..28f1e98
---- /dev/null
-+++ b/ompi/mca/io/romio32b1/romio/.gitignore
-@@ -0,0 +1,13 @@
-+/Makefile
-+/.deps
-+/*.bb
-+/*.bbg
-+/*.gcda
-+/*.gcno
-+/.libs
-+/.libstamp*
-+/*.lo
-+/.*-cache
-+.state-cache
-+version.m4
-+confdb/config.rpath
-diff --git a/ompi/mca/io/romio32b1/romio/Makefile.am b/ompi/mca/io/romio32b1/romio/Makefile.am
 index b9d4e25..69d7014 100644
---- a/ompi/mca/io/romio32b1/romio/Makefile.am
+--- a/ompi/mca/io/romio321/romio/Makefile.am
-+++ b/ompi/mca/io/romio32b1/romio/Makefile.am
++++ b/ompi/mca/io/romio321/romio/Makefile.am
 @@ -1,9 +1,28 @@
  # -*- Mode: Makefile; -*-
 +# Copyright (c) 2004-2006 The Trustees of Indiana University and Indiana
@@ -89,69 +70,10 @@ index b9d4e25..69d7014 100644
  
  ## NOTE: ROMIO's old build system builds a bunch of _foo.o objects that contain
  ## PMPI_ implementations as well as calls to only other PMPI routines.  In
-diff --git a/ompi/mca/io/romio32b1/romio/Makefile.options b/ompi/mca/io/romio32b1/romio/Makefile.options
+diff --git a/ompi/mca/io/romio321/romio/adio/Makefile.mk b/ompi/mca/io/romio321/romio/adio/Makefile.mk
-new file mode 100644
-index 0000000..0b72829
---- /dev/null
-+++ b/ompi/mca/io/romio32b1/romio/Makefile.options
-@@ -0,0 +1,36 @@
-+# -*- makefile -*-
-+#
-+# Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
-+#                         University Research and Technology
-+#                         Corporation.  All rights reserved.
-+# Copyright (c) 2004-2005 The University of Tennessee and The University
-+#                         of Tennessee Research Foundation.  All rights
-+#                         reserved.
-+# Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
-+#                         University of Stuttgart.  All rights reserved.
-+# Copyright (c) 2004-2005 The Regents of the University of California.
-+#                         All rights reserved.
-+# $COPYRIGHT$
-+#
-+# Additional copyrights may follow
-+#
-+# $HEADER$
-+#
-+
-+AUTOMAKE_OPTIONS	= foreign dist-bzip2
-+
-+# $(OMPI_TOP_SRCDIR)                - mca_base_param.h
-+# $(OMPI_TOP_SRCDIR)/opal/include   - opal_config_bottom.h
-+# $(OMPI_TOP_BUILDDIR)/opal/include - opal_config.h
-+# $(OMPI_TOP_BUILDDIR)/ompi/include - mpi.h
-+# $(top_srcdir)/include             - vpath support
-+# $(top_srcdir)/adio/include        - vpath support
-+
-+AM_CPPFLAGS = \
-+        -DOMPI_BUILDING=1 \
-+        -I$(OMPI_TOP_SRCDIR) \
-+        -I$(OMPI_TOP_SRCDIR)/opal/include \
-+        -I$(OMPI_TOP_BUILDDIR)/opal/include \
-+        -I$(OMPI_TOP_BUILDDIR)/ompi/include \
-+	-I$(top_srcdir)/include \
-+	-I$(top_srcdir)/adio/include
-diff --git a/ompi/mca/io/romio32b1/romio/README_OMPI b/ompi/mca/io/romio32b1/romio/README_OMPI
-new file mode 100644
-index 0000000..6dba412
---- /dev/null
-+++ b/ompi/mca/io/romio32b1/romio/README_OMPI
-@@ -0,0 +1,11 @@
-+Please note that this is *NOT* a vanilla MPICH v3.2b1
-+distribution of the ROMIO package from Argonne National Labs.
-+Various customizations had to be applied to the configuration process.
-+More to the point -- if replace this copy of ROMIO with a newer version,
-+it will likely not work.  :-(
-+
-+- The Open MPI Team
-+
-+-----------------------------------------------------------------------------
-+
-+Local modifications are in ompi.patch
-diff --git a/ompi/mca/io/romio32b1/romio/adio/Makefile.mk b/ompi/mca/io/romio32b1/romio/adio/Makefile.mk
 index 505d518..ffc05cb 100644
---- a/ompi/mca/io/romio32b1/romio/adio/Makefile.mk
+--- a/ompi/mca/io/romio321/romio/adio/Makefile.mk
-+++ b/ompi/mca/io/romio32b1/romio/adio/Makefile.mk
++++ b/ompi/mca/io/romio321/romio/adio/Makefile.mk
 @@ -20,6 +20,7 @@ noinst_HEADERS +=                      \
      adio/include/mpipr.h               \
      adio/include/mpiu_greq.h           \
@@ -160,10 +82,10 @@ index 505d518..ffc05cb 100644
      adio/include/mpiu_external32.h     \
      adio/include/hint_fns.h
  
-diff --git a/ompi/mca/io/romio32b1/romio/adio/common/ad_end.c b/ompi/mca/io/romio32b1/romio/adio/common/ad_end.c
+diff --git a/ompi/mca/io/romio321/romio/adio/common/ad_end.c b/ompi/mca/io/romio321/romio/adio/common/ad_end.c
 index ea4dfeb..066c65c 100644
---- a/ompi/mca/io/romio32b1/romio/adio/common/ad_end.c
+--- a/ompi/mca/io/romio321/romio/adio/common/ad_end.c
-+++ b/ompi/mca/io/romio32b1/romio/adio/common/ad_end.c
++++ b/ompi/mca/io/romio321/romio/adio/common/ad_end.c
 @@ -16,7 +16,12 @@ void ADIO_End(int *error_code)
  
      /* if a default errhandler was set on MPI_FILE_NULL then we need to ensure
@@ -177,10 +99,10 @@ index ea4dfeb..066c65c 100644
  
  /* delete the flattened datatype list */
      curr = ADIOI_Flatlist;
-diff --git a/ompi/mca/io/romio32b1/romio/adio/common/ad_iread_coll.c b/ompi/mca/io/romio32b1/romio/adio/common/ad_iread_coll.c
+diff --git a/ompi/mca/io/romio321/romio/adio/common/ad_iread_coll.c b/ompi/mca/io/romio321/romio/adio/common/ad_iread_coll.c
-index b1311e6..6ae4359 100644
+index 2ec23fb..8487c25 100644
---- a/ompi/mca/io/romio32b1/romio/adio/common/ad_iread_coll.c
+--- a/ompi/mca/io/romio321/romio/adio/common/ad_iread_coll.c
-+++ b/ompi/mca/io/romio32b1/romio/adio/common/ad_iread_coll.c
++++ b/ompi/mca/io/romio321/romio/adio/common/ad_iread_coll.c
 @@ -16,6 +16,8 @@
  #include "mpe.h"
  #endif
@@ -190,15 +112,15 @@ index b1311e6..6ae4359 100644
  /* ADIOI_GEN_IreadStridedColl */
  struct ADIOI_GEN_IreadStridedColl_vars {
      /* requests */
-@@ -1315,3 +1317,4 @@ static int ADIOI_GEN_irc_wait_fn(int count, void **array_of_states,
+@@ -1340,3 +1342,4 @@ static int ADIOI_GEN_irc_wait_fn(int count, void **array_of_states,
      return errcode;
  }
  
 +#endif /* HAVE_MPI_GREQUEST_EXTENSIONS */
-diff --git a/ompi/mca/io/romio32b1/romio/adio/common/ad_iwrite_coll.c b/ompi/mca/io/romio32b1/romio/adio/common/ad_iwrite_coll.c
+diff --git a/ompi/mca/io/romio321/romio/adio/common/ad_iwrite_coll.c b/ompi/mca/io/romio321/romio/adio/common/ad_iwrite_coll.c
-index b456ec4..9178a8d 100644
+index d275f78..9b27b42 100644
---- a/ompi/mca/io/romio32b1/romio/adio/common/ad_iwrite_coll.c
+--- a/ompi/mca/io/romio321/romio/adio/common/ad_iwrite_coll.c
-+++ b/ompi/mca/io/romio32b1/romio/adio/common/ad_iwrite_coll.c
++++ b/ompi/mca/io/romio321/romio/adio/common/ad_iwrite_coll.c
 @@ -13,6 +13,8 @@
  #include "mpe.h"
  #endif
@@ -208,16 +130,16 @@ index b456ec4..9178a8d 100644
  /* ADIOI_GEN_IwriteStridedColl */
  struct ADIOI_GEN_IwriteStridedColl_vars {
      /* requests */
-@@ -1539,3 +1541,4 @@ static int ADIOI_GEN_iwc_wait_fn(int count, void **array_of_states,
+@@ -1565,3 +1567,4 @@ static int ADIOI_GEN_iwc_wait_fn(int count, void **array_of_states,
      return errcode;
  }
  
 +#endif /* HAVE_MPI_GREQUEST_EXTENSIONS */
-diff --git a/ompi/mca/io/romio32b1/romio/adio/include/adioi.h b/ompi/mca/io/romio32b1/romio/adio/include/adioi.h
+diff --git a/ompi/mca/io/romio321/romio/adio/include/adioi.h b/ompi/mca/io/romio321/romio/adio/include/adioi.h
-index b20ca82..73dad0d 100644
+index 0e91ead..13c0f25 100644
---- a/ompi/mca/io/romio32b1/romio/adio/include/adioi.h
+--- a/ompi/mca/io/romio321/romio/adio/include/adioi.h
-+++ b/ompi/mca/io/romio32b1/romio/adio/include/adioi.h
++++ b/ompi/mca/io/romio321/romio/adio/include/adioi.h
-@@ -429,18 +429,26 @@ void ADIOI_GEN_ReadStridedColl(ADIO_File fd, void *buf, int count,
+@@ -440,18 +440,26 @@ void ADIOI_GEN_ReadStridedColl(ADIO_File fd, void *buf, int count,
                         MPI_Datatype datatype, int file_ptr_type,
                         ADIO_Offset offset, ADIO_Status *status, int
                         *error_code);
@@ -244,61 +166,10 @@ index b20ca82..73dad0d 100644
  void ADIOI_Calc_my_off_len(ADIO_File fd, int bufcount, MPI_Datatype
  			    datatype, int file_ptr_type, ADIO_Offset 
  			    offset, ADIO_Offset **offset_list_ptr, ADIO_Offset
-diff --git a/ompi/mca/io/romio32b1/romio/adio/include/romioconf-undefs.h b/ompi/mca/io/romio32b1/romio/adio/include/romioconf-undefs.h
+diff --git a/ompi/mca/io/romio321/romio/configure.ac b/ompi/mca/io/romio321/romio/configure.ac
-new file mode 100644
+index 93f8b09..0b85d53 100644
-index 0000000..5c21607
+--- a/ompi/mca/io/romio321/romio/configure.ac
---- /dev/null
++++ b/ompi/mca/io/romio321/romio/configure.ac
-+++ b/ompi/mca/io/romio32b1/romio/adio/include/romioconf-undefs.h
-@@ -0,0 +1,45 @@
-+/*
-+ *  Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
-+ *                          University Research and Technology
-+ *                          Corporation.  All rights reserved.
-+ *  Copyright (c) 2004-2005 The University of Tennessee and The University
-+ *                          of Tennessee Research Foundation.  All rights
-+ *                          reserved.
-+ *  Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
-+ *                          University of Stuttgart. All rights reserved.
-+ *  Copyright (c) 2004-2005 The Regents of the University of California.
-+ *                          All rights reserved.
-+ *  $COPYRIGHT$
-+ *
-+ *  Additional copyrights may follow
-+ *
-+ *  $HEADER$
-+ */
-+
-+#ifndef ROMIO_CONF_UNDEFS_H
-+#define ROMIO_CONF_UNDEFS_H
-+
-+/* Need to add some undefs here so that we don't conflict with the
-+ * main ompi_config.h.  Arrgh.  Stupid autoconf not giving us the option
-+ * to not define these macros... grumble...
-+ */
-+#if defined(PACKAGE_BUGREPORT)
-+#undef PACKAGE_BUGREPORT
-+#endif
-+#if defined(PACKAGE_NAME)
-+#undef PACKAGE_NAME
-+#endif
-+#if defined(PACKAGE_STRING)
-+#undef PACKAGE_STRING
-+#endif
-+#if defined(PACKAGE_TARNAME)
-+#undef PACKAGE_TARNAME
-+#endif
-+#if defined(PACKAGE_VERSION)
-+#undef PACKAGE_VERSION
-+#endif
-+#if defined(PACKAGE_URL)
-+#undef PACKAGE_URL
-+#endif
-+
-+#endif /* ROMIOCONF_UNDEFS_H */
-diff --git a/ompi/mca/io/romio32b1/romio/configure.ac b/ompi/mca/io/romio32b1/romio/configure.ac
-index f975e1c..4d51a3e 100644
---- a/ompi/mca/io/romio32b1/romio/configure.ac
-+++ b/ompi/mca/io/romio32b1/romio/configure.ac
 @@ -3,12 +3,21 @@
  # autoconf --localdir=../confdb configure.ac
  # (or wherever the confdb is)
@@ -410,7 +281,7 @@ index f975e1c..4d51a3e 100644
  # foll. needed for f77 test programs
  F77GETARG="call getarg(i,str)"
  F77IARGC="iargc()"
-@@ -133,6 +178,17 @@ MPI_OFFSET_KIND2="!"
+@@ -133,6 +178,18 @@ MPI_OFFSET_KIND2="!"
  TEST_CC=""
  TEST_F77=""
  #
@@ -425,10 +296,11 @@ index f975e1c..4d51a3e 100644
 +MPIO_REQOBJECTS="iotest.o iotestall.o iotestany.o iotestsome.o iowait.o iowaitall.o iowaitany.o iowaitsome.o ioreq_c2f.o ioreq_f2c.o"
 +MPIO_REQ_TMP_POBJECTS="iotest.p iowait.p iowaitall.p iowaitany.p iotestall.p iotestany.p iowaitsome.p iotestsome.p"
 +MPIO_REQ_REAL_POBJECTS="_iotest.o _iowait.o _iowaitall.o _iowaitany.o _iotestall.o _iotestany.o _iowaitsome.o _iotestsome.o"
- have_aio=no
++#
- #
  known_mpi_impls="mpich_mpi mpich_mpi sgi_mpi hp_mpi cray_mpi lam_mpi open_mpi_mpi"
-@@ -171,7 +227,7 @@ AC_ARG_ENABLE(f77,
+ 
+ dnl An m4 macro for use with m4_foreach_w and friends.  You should modify this
+@@ -169,7 +226,7 @@ AC_ARG_ENABLE(f77,
  AC_ARG_ENABLE(f90,
  [--enable-f90 - Turn on support for Fortran 90 (default)],,enable_f90=yes)
  AC_ARG_ENABLE(weak-symbols,
@@ -437,7 +309,7 @@ index f975e1c..4d51a3e 100644
  AC_ARG_ENABLE(debug,
  [--enable-debug - Build a debugging version],,)
  AC_ARG_WITH(file-system,[
-@@ -194,9 +250,10 @@ if test "$enable_debug" = "yes" ; then
+@@ -192,9 +249,10 @@ if test "$enable_debug" = "yes" ; then
      DEBUG=yes
  fi
  MPI=$with_mpi
@@ -451,7 +323,7 @@ index f975e1c..4d51a3e 100644
  
  # start with the set of file systems that the user asked for
  # FILE_SYSTEM=$with_file_system
-@@ -259,6 +316,7 @@ top_build_dir=`pwd`
+@@ -257,6 +315,7 @@ top_build_dir=`pwd`
  # used in romioinstall
  AC_SUBST(top_build_dir)
  
@@ -459,7 +331,7 @@ index f975e1c..4d51a3e 100644
  #
  # Create the "autoconf" style directory names...
  # Most of these are done for us; add the documentation directories
-@@ -380,8 +438,9 @@ if test "$CC" = "gcc" -a -z "$C_DEBUG_FLAG" ; then
+@@ -378,8 +437,9 @@ if test "$CC" = "gcc" -a -z "$C_DEBUG_FLAG" ; then
  fi
  if test $DEBUG = "yes" ; then
      CFLAGS="$CFLAGS $C_DEBUG_FLAG"
@@ -471,7 +343,7 @@ index f975e1c..4d51a3e 100644
  fi
  # ---------------------------------------------------------------------------
  # Here go the rest of the tests
-@@ -428,7 +487,8 @@ else
+@@ -426,7 +486,8 @@ else
      F77=":"
  fi
  #
@@ -481,7 +353,7 @@ index f975e1c..4d51a3e 100644
  
  AC_TYPE_SIZE_T
  AC_TYPE_SSIZE_T
-@@ -651,7 +711,9 @@ AM_CONDITIONAL([BUILD_MPIO_ERRHAN],[false])
+@@ -649,7 +710,9 @@ AM_CONDITIONAL([BUILD_MPIO_ERRHAN],[false])
  
  # if we don't have weak symbol support, we must build a separate convenience
  # library in order to provide the "PMPI_" symbols
@@ -492,7 +364,7 @@ index f975e1c..4d51a3e 100644
  
  # weird: we have conflated "buid ROMIO's versions of the fortran bindings" and
  # "build ROMIO"s fortran I/O tests". Of course the common situaiton is that we
-@@ -1565,8 +1627,10 @@ if test $FROM_OMPI = yes ; then
+@@ -1447,8 +1510,10 @@ if test $FROM_OMPI = yes ; then
     HAVE_MPI_DARRAY_SUBARRAY="#define HAVE_MPI_DARRAY_SUBARRAY"
     # Open MPI: see comments in mpi-io/mpioprof.h
     AC_DEFINE(MPIO_BUILD_PROFILING, 1, [hack to make ROMIO build without profiling])
@@ -504,7 +376,7 @@ index f975e1c..4d51a3e 100644
  elif test $FROM_LAM = yes ; then
     # LAM does have the status set bytes functionality
     AC_DEFINE(HAVE_STATUS_SET_BYTES,1,[Define if have MPIR_Status_set_bytes])
-@@ -1615,7 +1679,7 @@ elif test $FROM_MPICH = yes ; then
+@@ -1497,7 +1562,7 @@ elif test $FROM_MPICH = yes ; then
     DEFINE_HAVE_MPI_GREQUEST_EXTENSIONS="#define HAVE_MPI_GREQUEST_EXTENSIONS 1"
     AC_DEFINE(HAVE_MPIU_FUNCS,1,[Define if MPICH memory tracing macros defined])
     AC_DEFINE(HAVE_MPIX_H, 1, [])
@@ -513,7 +385,7 @@ index f975e1c..4d51a3e 100644
     AC_DEFINE(HAVE_MPI_TYPE_SIZE_X, 1, [Define if MPI library provides MPI_TYPE_SIZE_X])
     AC_DEFINE(HAVE_MPI_STATUS_SET_ELEMENTS_X, 1, [Define if MPI library provides MPI_STATUS_SET_ELEMENTS_X])
     AC_DEFINE(HAVE_DECL_MPI_COMBINER_HINDEXED_BLOCK, 1, [Define if MPI library provides HINDEXED_BLOCK datatype])
-@@ -1751,9 +1815,22 @@ AC_CHECK_HEADERS(unistd.h,
+@@ -1620,9 +1685,22 @@ fi
  echo "setting SYSDEP_INC to $SYSDEP_INC"
  AC_SUBST(SYSDEP_INC)
  
@@ -536,7 +408,7 @@ index f975e1c..4d51a3e 100644
  # support gcov test coverage information
  PAC_ENABLE_COVERAGE
  
-@@ -1765,23 +1842,29 @@ echo "setting CFLAGS to $CFLAGS"
+@@ -1634,23 +1712,29 @@ echo "setting CFLAGS to $CFLAGS"
  echo "setting USER_CFLAGS to $USER_CFLAGS"
  echo "setting USER_FFLAGS to $USER_FFLAGS"
  
@@ -572,7 +444,15 @@ index f975e1c..4d51a3e 100644
  AC_SUBST(arch_IRIX)
  AC_SUBST(ROMIO_HOME)
  AC_SUBST(LIBNAME)
-@@ -1825,24 +1908,25 @@ AC_SUBST(ROMIO_TCPPFLAGS)
+@@ -1671,6 +1755,7 @@ AC_SUBST(HAVE_MPI_DARRAY_SUBARRAY)
+ AC_SUBST(BUILD_MPI_ARRAY)
+ AC_SUBST(DEFINE_MPI_OFFSET)
+ AC_SUBST(DEFINE_HAVE_MPI_GREQUEST)
++AC_SUBST(DEFINE_HAVE_MPI_GREQUEST_EXTENSIONS)
+ AC_SUBST(MPI_OFFSET_TYPE)
+ AC_SUBST(MPI_FINFO1)
+ AC_SUBST(MPI_FINFO2)
+@@ -1694,24 +1779,25 @@ AC_SUBST(ROMIO_TCPPFLAGS)
  AC_SUBST(ROMIO_TFFLAGS)
  AC_SUBST(MPIRUN)
  AC_SUBST(FORTRAN_TEST)
@@ -614,7 +494,7 @@ index f975e1c..4d51a3e 100644
  
  # Create makefiles for all of the adio devices.  Only the ones that 
  # are active will be called by the top level ROMIO make
-@@ -1850,8 +1934,6 @@ AC_OUTPUT_COMMANDS([chmod 755 util/romioinstall test/runtests])
+@@ -1719,8 +1805,6 @@ AC_OUTPUT_COMMANDS([chmod 755 util/romioinstall test/runtests])
  AC_CONFIG_FILES([
      Makefile
      localdefs
@@ -623,7 +503,7 @@ index f975e1c..4d51a3e 100644
      test/Makefile
      test/misc.c
      test/large_file.c
-@@ -1859,14 +1941,17 @@ AC_CONFIG_FILES([
+@@ -1728,14 +1812,17 @@ AC_CONFIG_FILES([
      test-internal/Makefile
      util/romioinstall
      include/mpio.h
@@ -644,10 +524,10 @@ index f975e1c..4d51a3e 100644
  AC_OUTPUT
  
  dnl PAC_SUBDIR_CACHE_CLEANUP
-diff --git a/ompi/mca/io/romio32b1/romio/doc/users-guide.tex b/ompi/mca/io/romio32b1/romio/doc/users-guide.tex
+diff --git a/ompi/mca/io/romio321/romio/doc/users-guide.tex b/ompi/mca/io/romio321/romio/doc/users-guide.tex
 index b33d483..3715431 100644
---- a/ompi/mca/io/romio32b1/romio/doc/users-guide.tex
+--- a/ompi/mca/io/romio321/romio/doc/users-guide.tex
-+++ b/ompi/mca/io/romio32b1/romio/doc/users-guide.tex
++++ b/ompi/mca/io/romio321/romio/doc/users-guide.tex
 @@ -807,13 +807,19 @@ to include the file {\tt mpio.h} for C or {\tt mpiof.h} for Fortran in
  your MPI-IO program.  
  
@@ -679,140 +559,10 @@ index b33d483..3715431 100644
  
  \item The file-open mode {\tt MPI\_MODE\_EXCL} does not work on Intel
  PFS file system, due to a bug in PFS.
-diff --git a/ompi/mca/io/romio32b1/romio/include/io_romio_conv.h b/ompi/mca/io/romio32b1/romio/include/io_romio_conv.h
+diff --git a/ompi/mca/io/romio321/romio/include/mpio.h.in b/ompi/mca/io/romio321/romio/include/mpio.h.in
-new file mode 100644
+index 1f3f382..9ef709d 100644
-index 0000000..c03873d
+--- a/ompi/mca/io/romio321/romio/include/mpio.h.in
---- /dev/null
++++ b/ompi/mca/io/romio321/romio/include/mpio.h.in
-+++ b/ompi/mca/io/romio32b1/romio/include/io_romio_conv.h
-@@ -0,0 +1,124 @@
-+/*
-+ * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
-+ *                         University Research and Technology
-+ *                         Corporation.  All rights reserved.
-+ * Copyright (c) 2004-2005 The University of Tennessee and The University
-+ *                         of Tennessee Research Foundation.  All rights
-+ *                         reserved.
-+ * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
-+ *                         University of Stuttgart.  All rights reserved.
-+ * Copyright (c) 2004-2005 The Regents of the University of California.
-+ *                         All rights reserved.
-+ * $COPYRIGHT$
-+ *
-+ * Additional copyrights may follow
-+ *
-+ * $HEADER$
-+ */
-+
-+#ifndef MCA_IO_ROMIO_CONV_H
-+#define MCA_IO_ROMIO_CONV_H
-+
-+/* Prefix that we add to all ROMIO symbols */
-+#ifdef ROMIO_PREFIX
-+#undef ROMIO_PREFIX
-+#endif
-+#define ROMIO_PREFIX(foo) mca_io_romio_dist_##foo
-+
-+/* Section 9.2 */
-+/* Begin Prototypes */
-+#define MPI_File_open ROMIO_PREFIX(MPI_File_open)
-+#define MPI_File_close ROMIO_PREFIX(MPI_File_close)
-+#define MPI_File_delete ROMIO_PREFIX(MPI_File_delete)
-+#define MPI_File_set_size ROMIO_PREFIX(MPI_File_set_size)
-+#define MPI_File_preallocate ROMIO_PREFIX(MPI_File_preallocate)
-+#define MPI_File_get_size ROMIO_PREFIX(MPI_File_get_size)
-+#define MPI_File_get_group ROMIO_PREFIX(MPI_File_get_group)
-+#define MPI_File_get_amode ROMIO_PREFIX(MPI_File_get_amode)
-+#define MPI_File_set_info ROMIO_PREFIX(MPI_File_set_info)
-+#define MPI_File_get_info ROMIO_PREFIX(MPI_File_get_info)
-+
-+/* Section 9.3 */
-+#define MPI_File_set_view ROMIO_PREFIX(MPI_File_set_view)
-+#define MPI_File_get_view ROMIO_PREFIX(MPI_File_get_view)
-+
-+/* Section 9.4.2 */
-+#define MPI_File_read_at ROMIO_PREFIX(MPI_File_read_at)
-+#define MPI_File_read_at_all ROMIO_PREFIX(MPI_File_read_at_all)
-+#define MPI_File_write_at ROMIO_PREFIX(MPI_File_write_at)
-+#define MPI_File_write_at_all ROMIO_PREFIX(MPI_File_write_at_all)
-+#define MPI_File_iread_at ROMIO_PREFIX(MPI_File_iread_at)
-+#define MPI_File_iwrite_at ROMIO_PREFIX(MPI_File_iwrite_at)
-+
-+/* Section 9.4.3 */
-+#define MPI_File_read ROMIO_PREFIX(MPI_File_read)
-+#define MPI_File_read_all ROMIO_PREFIX(MPI_File_read_all)
-+#define MPI_File_write ROMIO_PREFIX(MPI_File_write)
-+#define MPI_File_write_all ROMIO_PREFIX(MPI_File_write_all)
-+
-+#define MPI_File_iread ROMIO_PREFIX(MPI_File_iread)
-+#define MPI_File_iwrite ROMIO_PREFIX(MPI_File_iwrite)
-+
-+#define MPI_File_seek ROMIO_PREFIX(MPI_File_seek)
-+#define MPI_File_get_position ROMIO_PREFIX(MPI_File_get_position)
-+#define MPI_File_get_byte_offset ROMIO_PREFIX(MPI_File_get_byte_offset)
-+
-+/* Section 9.4.4 */
-+#define MPI_File_read_shared ROMIO_PREFIX(MPI_File_read_shared)
-+#define MPI_File_write_shared ROMIO_PREFIX(MPI_File_write_shared)
-+#define MPI_File_iread_shared ROMIO_PREFIX(MPI_File_iread_shared)
-+#define MPI_File_iwrite_shared ROMIO_PREFIX(MPI_File_iwrite_shared)
-+#define MPI_File_read_ordered ROMIO_PREFIX(MPI_File_read_ordered)
-+#define MPI_File_write_ordered ROMIO_PREFIX(MPI_File_write_ordered)
-+#define MPI_File_seek_shared ROMIO_PREFIX(MPI_File_seek_shared)
-+#define MPI_File_get_position_shared ROMIO_PREFIX(MPI_File_get_position_shared)
-+
-+/* Section 9.4.5 */
-+#define MPI_File_read_at_all_begin ROMIO_PREFIX(MPI_File_read_at_all_begin)
-+#define MPI_File_read_at_all_end ROMIO_PREFIX(MPI_File_read_at_all_end)
-+#define MPI_File_write_at_all_begin ROMIO_PREFIX(MPI_File_write_at_all_begin)
-+#define MPI_File_write_at_all_end ROMIO_PREFIX(MPI_File_write_at_all_end)
-+#define MPI_File_read_all_begin ROMIO_PREFIX(MPI_File_read_all_begin)
-+#define MPI_File_read_all_end ROMIO_PREFIX(MPI_File_read_all_end)
-+#define MPI_File_write_all_begin ROMIO_PREFIX(MPI_File_write_all_begin)
-+#define MPI_File_write_all_end ROMIO_PREFIX(MPI_File_write_all_end)
-+#define MPI_File_read_ordered_begin ROMIO_PREFIX(MPI_File_read_ordered_begin)
-+#define MPI_File_read_ordered_end ROMIO_PREFIX(MPI_File_read_ordered_end)
-+#define MPI_File_write_ordered_begin ROMIO_PREFIX(MPI_File_write_ordered_begin)
-+#define MPI_File_write_ordered_end ROMIO_PREFIX(MPI_File_write_ordered_end)
-+
-+/* Section 9.5.1 */
-+#define MPI_File_get_type_extent ROMIO_PREFIX(MPI_File_get_type_extent)
-+
-+/* Section 9.6.1 */
-+#define MPI_File_set_atomicity ROMIO_PREFIX(MPI_File_set_atomicity)
-+#define MPI_File_get_atomicity ROMIO_PREFIX(MPI_File_get_atomicity)
-+#define MPI_File_sync ROMIO_PREFIX(MPI_File_sync)
-+
-+/* Section 4.13.3 */
-+#define MPI_File_set_errhandler ROMIO_PREFIX(MPI_File_set_errhandler)
-+#define MPI_File_get_errhandler ROMIO_PREFIX(MPI_File_get_errhandler)
-+/* End Prototypes */
-+
-+#define MPI_Register_datarep ROMIO_PREFIX(MPI_Register_datarep)
-+
-+/* JMS these don't seem to work... */
-+#define MPI_File_f2c ROMIO_PREFIX(MPI_File_f2c)
-+#define MPI_File_c2f ROMIO_PREFIX(MPI_File_c2f)
-+
-+#define MPIO_Request_c2f ROMIO_PREFIX(MPIO_Request_c2f)
-+#define MPIO_Request_f2c ROMIO_PREFIX(MPIO_Request_f2c)
-+
-+/* Conversion of MPI_File and MPIO_Request */
-+#define MPI_File ROMIO_PREFIX(MPI_File)
-+
-+/* Open MPI's mpi.h #define's MPI_FILE_NULL, so we need to undef it
-+   here and allow it to be re-assigned to whatever ROMIO wants */
-+#undef MPI_FILE_NULL
-+
-+/* Let's not use MPIR_Status_set_bytes */
-+#ifndef MPIR_Status_set_bytes
-+#define MPIR_Status_set_bytes ROMIO_PREFIX(MPIR_Status_set_bytes)
-+#endif
-+
-+#endif /* MCA_IO_ROMIO_CONV_H */
-diff --git a/ompi/mca/io/romio32b1/romio/include/mpio.h.in b/ompi/mca/io/romio32b1/romio/include/mpio.h.in
-index 2238f4b..8d1314c 100644
---- a/ompi/mca/io/romio32b1/romio/include/mpio.h.in
-+++ b/ompi/mca/io/romio32b1/romio/include/mpio.h.in
 @@ -11,6 +11,16 @@
  #define MPIO_INCLUDE
  
@@ -830,7 +580,16 @@ index 2238f4b..8d1314c 100644
  
  #if defined(__cplusplus)
  extern "C" {
-@@ -279,9 +289,12 @@ int MPI_Type_create_darray(int size, int rank, int ndims, const int array_of_gsi
+@@ -38,6 +48,8 @@ typedef struct ADIOI_RequestD *MPIO_Request;
+ #endif
+ #define MPIO_REQUEST_DEFINED
+ 
++@DEFINE_HAVE_MPI_GREQUEST_EXTENSIONS@
++
+ #ifndef HAVE_MPI_OFFSET
+ @DEFINE_MPI_OFFSET@
+ /* If we needed to define MPI_Offset, then we also need to make
+@@ -279,9 +291,12 @@ int MPI_Type_create_darray(int size, int rank, int ndims, const int array_of_gsi
  #endif
  #endif
  /* above needed for some versions of mpi.h in MPICH!! */
@@ -844,7 +603,7 @@ index 2238f4b..8d1314c 100644
  
  #ifndef HAVE_MPI_GREQUEST
  /* The following functions are required if generalized requests are not
-@@ -329,15 +342,21 @@ int MPI_Info_free(MPI_Info *info);
+@@ -329,15 +344,21 @@ int MPI_Info_free(MPI_Info *info);
  #endif
  #endif
  /* above needed for some versions of mpi.h in MPICH!! */
@@ -866,7 +625,7 @@ index 2238f4b..8d1314c 100644
  
  /* Section 9.2 */
  int PMPI_File_open(MPI_Comm, const char *, int, MPI_Info, MPI_File *);
-@@ -528,6 +547,8 @@ MPI_Fint PMPI_Info_c2f(MPI_Info);
+@@ -528,6 +549,8 @@ MPI_Fint PMPI_Info_c2f(MPI_Info);
  MPI_Info PMPI_Info_f2c(MPI_Fint);
  #endif
  
@@ -875,23 +634,10 @@ index 2238f4b..8d1314c 100644
  #if defined(__cplusplus)
  }
  #endif
-diff --git a/ompi/mca/io/romio32b1/romio/localdefs.in b/ompi/mca/io/romio32b1/romio/localdefs.in
+diff --git a/ompi/mca/io/romio321/romio/mpi-io/Makefile.mk b/ompi/mca/io/romio321/romio/mpi-io/Makefile.mk
-index bf057d1..5b3d670 100644
---- a/ompi/mca/io/romio32b1/romio/localdefs.in
-+++ b/ompi/mca/io/romio32b1/romio/localdefs.in
-@@ -1,7 +1,4 @@
- #! /bin/sh
--
--# Append ROMIO library dependencies to the global list
--EXTERNAL_LIBS="$EXTERNAL_LIBS @LIBS@"
--
-+LIBS="@LIBS@"
- MPI_OFFSET_TYPE="@MPI_OFFSET_TYPE@"
- FORTRAN_MPI_OFFSET="@FORTRAN_MPI_OFFSET@"
-diff --git a/ompi/mca/io/romio32b1/romio/mpi-io/Makefile.mk b/ompi/mca/io/romio32b1/romio/mpi-io/Makefile.mk
 index d4d5a29..264976e 100644
---- a/ompi/mca/io/romio32b1/romio/mpi-io/Makefile.mk
+--- a/ompi/mca/io/romio321/romio/mpi-io/Makefile.mk
-+++ b/ompi/mca/io/romio32b1/romio/mpi-io/Makefile.mk
++++ b/ompi/mca/io/romio321/romio/mpi-io/Makefile.mk
 @@ -14,8 +14,6 @@ noinst_HEADERS += mpi-io/mpioimpl.h mpi-io/mpioprof.h
  romio_mpi_sources +=          \
      mpi-io/close.c            \
@@ -901,10 +647,10 @@ index d4d5a29..264976e 100644
      mpi-io/fsync.c            \
      mpi-io/get_amode.c        \
      mpi-io/get_atom.c         \
-diff --git a/ompi/mca/io/romio32b1/romio/mpi-io/close.c b/ompi/mca/io/romio32b1/romio/mpi-io/close.c
+diff --git a/ompi/mca/io/romio321/romio/mpi-io/close.c b/ompi/mca/io/romio321/romio/mpi-io/close.c
-index 520f206..160b661 100644
+index 3f2b0b1..12d5c47 100644
---- a/ompi/mca/io/romio32b1/romio/mpi-io/close.c
+--- a/ompi/mca/io/romio321/romio/mpi-io/close.c
-+++ b/ompi/mca/io/romio32b1/romio/mpi-io/close.c
++++ b/ompi/mca/io/romio321/romio/mpi-io/close.c
 @@ -76,8 +76,13 @@ int MPI_File_close(MPI_File *fh)
       * somehow inform the MPI library that we no longer hold a reference to any
       * user defined error handler.  We do this by setting the errhandler at this
@@ -919,10 +665,10 @@ index 520f206..160b661 100644
  
      ADIO_Close(adio_fh, &error_code);
      MPIO_File_free(fh);
-diff --git a/ompi/mca/io/romio32b1/romio/mpi-io/glue/Makefile.mk b/ompi/mca/io/romio32b1/romio/mpi-io/glue/Makefile.mk
+diff --git a/ompi/mca/io/romio321/romio/mpi-io/glue/Makefile.mk b/ompi/mca/io/romio321/romio/mpi-io/glue/Makefile.mk
 index 05954a1..66f7f9e 100644
---- a/ompi/mca/io/romio32b1/romio/mpi-io/glue/Makefile.mk
+--- a/ompi/mca/io/romio321/romio/mpi-io/glue/Makefile.mk
-+++ b/ompi/mca/io/romio32b1/romio/mpi-io/glue/Makefile.mk
++++ b/ompi/mca/io/romio321/romio/mpi-io/glue/Makefile.mk
 @@ -7,6 +7,7 @@
  
  include $(top_srcdir)/mpi-io/glue/default/Makefile.mk
@@ -931,11 +677,11 @@ index 05954a1..66f7f9e 100644
  
  if !BUILD_ROMIO_EMBEDDED
  romio_other_sources += \
-diff --git a/ompi/mca/io/romio32b1/romio/mpi-io/iread_all.c b/ompi/mca/io/romio32b1/romio/mpi-io/iread_all.c
+diff --git a/ompi/mca/io/romio321/romio/mpi-io/iread_all.c b/ompi/mca/io/romio321/romio/mpi-io/iread_all.c
-index ba36161..32a48b8 100644
+index 2615e56..00a0640 100644
---- a/ompi/mca/io/romio32b1/romio/mpi-io/iread_all.c
+--- a/ompi/mca/io/romio321/romio/mpi-io/iread_all.c
-+++ b/ompi/mca/io/romio32b1/romio/mpi-io/iread_all.c
++++ b/ompi/mca/io/romio321/romio/mpi-io/iread_all.c
-@@ -26,7 +26,7 @@ int MPIX_File_iread_all(MPI_File fh, void *buf, int count, MPI_Datatype datatype
+@@ -26,7 +26,7 @@ int MPI_File_iread_all(MPI_File fh, void *buf, int count, MPI_Datatype datatype,
  #include "mpioprof.h"
  #endif
  
@@ -944,10 +690,10 @@ index ba36161..32a48b8 100644
  #include "mpiu_greq.h"
  #endif
  
-diff --git a/ompi/mca/io/romio32b1/romio/mpi-io/mpioprof.h b/ompi/mca/io/romio32b1/romio/mpi-io/mpioprof.h
+diff --git a/ompi/mca/io/romio321/romio/mpi-io/mpioprof.h b/ompi/mca/io/romio321/romio/mpi-io/mpioprof.h
-index 15654ac..63dffd8 100644
+index f3ee3b4..c74413d 100644
---- a/ompi/mca/io/romio32b1/romio/mpi-io/mpioprof.h
+--- a/ompi/mca/io/romio321/romio/mpi-io/mpioprof.h
-+++ b/ompi/mca/io/romio32b1/romio/mpi-io/mpioprof.h
++++ b/ompi/mca/io/romio321/romio/mpi-io/mpioprof.h
 @@ -10,6 +10,15 @@
     building the profiling interface
   */

ompi/mca/io/romio321/romio/.gitignore

@@ -0,0 +1,13 @@
+/Makefile
+/.deps
+/*.bb
+/*.bbg
+/*.gcda
+/*.gcno
+/.libs
+/.libstamp*
+/*.lo
+/.*-cache
+.state-cache
+version.m4
+confdb/config.rpath

ompi/mca/io/romio321/romio/3.diff

@@ -0,0 +1,36 @@
+commit ae17908f35ee614071ec68eb4643286f7b707e86
+Author: Nathan Hjelm <hjelmn@lanl.gov>
+Date:   Tue May 1 15:11:34 2018 -0600
+
+    io/romio314: fix two more MPI-3 compliance issues
+    
+    Signed-off-by: Nathan Hjelm <hjelmn@lanl.gov>
+
+diff --git a/ompi/mca/io/romio314/romio/adio/common/cb_config_list.c b/ompi/mca/io/romio314/romio/adio/common/cb_config_list.c
+index 718205b..e9f3116 100644
+--- a/ompi/mca/io/romio314/romio/adio/common/cb_config_list.c
++++ b/ompi/mca/io/romio314/romio/adio/common/cb_config_list.c
+@@ -135,8 +135,8 @@ int ADIOI_cb_gather_name_array(MPI_Comm comm,
+ 
+     if (ADIOI_cb_config_list_keyval == MPI_KEYVAL_INVALID) {
+         /* cleaned up by ADIOI_End_call */
+-	MPI_Comm_create_keyval((MPI_Copy_function *) ADIOI_cb_copy_name_array,
+-			  (MPI_Delete_function *) ADIOI_cb_delete_name_array,
++	MPI_Comm_create_keyval((MPI_Comm_copy_attr_function *) ADIOI_cb_copy_name_array,
++			  (MPI_Comm_delete_attr_function *) ADIOI_cb_delete_name_array,
+ 			  &ADIOI_cb_config_list_keyval, NULL);
+     }
+     else {
+diff --git a/ompi/mca/io/romio314/romio/mpi-io/mpioimpl.h b/ompi/mca/io/romio314/romio/mpi-io/mpioimpl.h
+index a73561a..ff13723 100644
+--- a/ompi/mca/io/romio314/romio/mpi-io/mpioimpl.h
++++ b/ompi/mca/io/romio314/romio/mpi-io/mpioimpl.h
+@@ -58,7 +58,7 @@ struct MPIR_Info {
+ 
+ #define MPIR_INFO_COOKIE 5835657
+ 
+-MPI_Delete_function ADIOI_End_call;
++MPI_Comm_delete_attr_function ADIOI_End_call;
+ 
+ /* common initialization routine */
+ void MPIR_MPIOInit(int * error_code);

ompi/mca/io/romio321/romio/adio/ad_gpfs/ad_gpfs.c

@@ -56,6 +56,8 @@ struct ADIOI_Fns_struct ADIO_GPFS_operations = {
 #elif PEPLATFORM
     "GPFS+PE: IBM GPFS for PE",
 #else
-    "GPFS: IBM GPFS"
+    "GPFS: IBM GPFS",
 #endif
+    ADIOI_GEN_IreadStridedColl, /* IreadStridedColl */
+    ADIOI_GEN_IwriteStridedColl /* IwriteStridedColl */
 };

ompi/mca/io/romio321/romio/adio/ad_gpfs/ad_gpfs.h

@@ -24,7 +24,9 @@
 #ifdef HAVE_SIGNAL_H
 #include <signal.h>
 #endif
-#ifdef HAVE_AIO_H
+#ifdef HAVE_AIO_LITE_H
+#include <aio-lite.h>
+#elif defined  HAVE_AIO_H
 #include <aio.h>
 #endif
 

ompi/mca/io/romio321/romio/adio/ad_gpfs/ad_gpfs_aggrs.c

@@ -719,9 +719,9 @@ void ADIOI_GPFS_Calc_others_req(ADIO_File fd, int count_my_req_procs,
 	    others_req[i].lens =
 		ADIOI_Malloc(count_others_req_per_proc[i]*sizeof(ADIO_Offset));
 
-	    if ( (MPIR_Upint)others_req[i].offsets < (MPIR_Upint)recvBufForOffsets )
+	    if ( (MPIU_Upint)others_req[i].offsets < (MPIU_Upint)recvBufForOffsets )
 		recvBufForOffsets = others_req[i].offsets;
-	    if ( (MPIR_Upint)others_req[i].lens < (MPIR_Upint)recvBufForLens )
+	    if ( (MPIU_Upint)others_req[i].lens < (MPIU_Upint)recvBufForLens )
 		recvBufForLens = others_req[i].lens;
 
 	    others_req[i].mem_ptrs = (MPI_Aint *)
@@ -750,13 +750,13 @@ void ADIOI_GPFS_Calc_others_req(ADIO_File fd, int count_my_req_procs,
     for (i=0; i<nprocs; i++)
     {
 	if ( (my_req[i].count) &&
-	     ((MPIR_Upint)my_req[i].offsets <= (MPIR_Upint)sendBufForOffsets) )
+	     ((MPIU_Upint)my_req[i].offsets <= (MPIU_Upint)sendBufForOffsets) )
        {
 	  sendBufForOffsets = my_req[i].offsets;
     }
 
 	if ( (my_req[i].count) &&
-	     ((MPIR_Upint)my_req[i].lens <= (MPIR_Upint)sendBufForLens) )
+	     ((MPIU_Upint)my_req[i].lens <= (MPIU_Upint)sendBufForLens) )
        {
 	    sendBufForLens = my_req[i].lens;
       }
@@ -775,9 +775,9 @@ void ADIOI_GPFS_Calc_others_req(ADIO_File fd, int count_my_req_procs,
 	    sdispls[i] = 0;
 	else
 	    sdispls[i] =  (int)
-	                ( ( (MPIR_Upint)my_req[i].offsets -
+	                ( ( (MPIU_Upint)my_req[i].offsets -
-			   (MPIR_Upint)sendBufForOffsets ) /
+			   (MPIU_Upint)sendBufForOffsets ) /
-			  (MPIR_Upint)sizeof(ADIO_Offset) );
+			  (MPIU_Upint)sizeof(ADIO_Offset) );
 
 	/* Receive these offsets from process i.*/
 	rcounts[i] = count_others_req_per_proc[i];
@@ -785,9 +785,9 @@ void ADIOI_GPFS_Calc_others_req(ADIO_File fd, int count_my_req_procs,
 	    rdispls[i] = 0;
 	else
 	    rdispls[i] = (int)
-	                 ( ( (MPIR_Upint)others_req[i].offsets -
+	                 ( ( (MPIU_Upint)others_req[i].offsets -
-			     (MPIR_Upint)recvBufForOffsets ) /
+			     (MPIU_Upint)recvBufForOffsets ) /
-			   (MPIR_Upint)sizeof(ADIO_Offset) );
+			   (MPIU_Upint)sizeof(ADIO_Offset) );
     }
 
     /* Exchange the offsets */
@@ -809,9 +809,9 @@ void ADIOI_GPFS_Calc_others_req(ADIO_File fd, int count_my_req_procs,
 	    sdispls[i] = 0;
 	else
 	  sdispls[i] = (int)
-	               ( ( (MPIR_Upint)my_req[i].lens -
+	               ( ( (MPIU_Upint)my_req[i].lens -
-			   (MPIR_Upint)sendBufForLens ) /
+			   (MPIU_Upint)sendBufForLens ) /
-			 (MPIR_Upint) sizeof(ADIO_Offset) );
+			 (MPIU_Upint) sizeof(ADIO_Offset) );
 
 	/* Receive these offsets from process i. */
 	rcounts[i] = count_others_req_per_proc[i];
@@ -819,9 +819,9 @@ void ADIOI_GPFS_Calc_others_req(ADIO_File fd, int count_my_req_procs,
 	    rdispls[i] = 0;
 	else
 	    rdispls[i] = (int)
-	                 ( ( (MPIR_Upint)others_req[i].lens -
+	                 ( ( (MPIU_Upint)others_req[i].lens -
-			     (MPIR_Upint)recvBufForLens ) /
+			     (MPIU_Upint)recvBufForLens ) /
-			   (MPIR_Upint) sizeof(ADIO_Offset) );
+			   (MPIU_Upint) sizeof(ADIO_Offset) );
     }
 
     /* Exchange the lengths */

ompi/mca/io/romio321/romio/adio/ad_gpfs/ad_gpfs_rdcoll.c

@@ -114,6 +114,7 @@ void ADIOI_GPFS_ReadStridedColl(ADIO_File fd, void *buf, int count,
     ADIO_Offset *offset_list = NULL, *st_offsets = NULL, *fd_start = NULL,
 	*fd_end = NULL, *end_offsets = NULL;
     ADIO_Offset *gpfs_offsets0 = NULL, *gpfs_offsets = NULL;
+    ADIO_Offset *count_sizes;
     int  ii;
     ADIO_Offset *len_list = NULL;
     int *buf_idx = NULL;
@@ -174,7 +175,36 @@ void ADIOI_GPFS_ReadStridedColl(ADIO_File fd, void *buf, int count,
 	st_offsets   = (ADIO_Offset *) ADIOI_Malloc(nprocs*sizeof(ADIO_Offset));
 	end_offsets  = (ADIO_Offset *) ADIOI_Malloc(nprocs*sizeof(ADIO_Offset));
 
+    ADIO_Offset my_count_size=0;
+    /* One-sided aggregation needs the amount of data per rank as well because the difference in
+     * starting and ending offsets for 1 byte is 0 the same as 0 bytes so it cannot be distiguished.
+     */
+    if ((gpfsmpio_read_aggmethod == 1) || (gpfsmpio_read_aggmethod == 2)) {
+        count_sizes = (ADIO_Offset *) ADIOI_Malloc(nprocs*sizeof(ADIO_Offset));
+        MPI_Count buftype_size;
+        MPI_Type_size_x(datatype, &buftype_size);
+        my_count_size = (ADIO_Offset) count  * (ADIO_Offset)buftype_size;
+    }
     if (gpfsmpio_tunegather) {
+      if ((gpfsmpio_read_aggmethod == 1) || (gpfsmpio_read_aggmethod == 2)) {
+        gpfs_offsets0 = (ADIO_Offset *) ADIOI_Malloc(3*nprocs*sizeof(ADIO_Offset));
+        gpfs_offsets  = (ADIO_Offset *) ADIOI_Malloc(3*nprocs*sizeof(ADIO_Offset));
+        for (ii=0; ii<nprocs; ii++)  {
+          gpfs_offsets0[ii*3]   = 0;
+          gpfs_offsets0[ii*3+1] = 0;
+          gpfs_offsets0[ii*3+2] = 0;
+        }
+        gpfs_offsets0[myrank*3]   = start_offset;
+        gpfs_offsets0[myrank*3+1] =   end_offset;
+        gpfs_offsets0[myrank*3+2] =   my_count_size;
+        MPI_Allreduce( gpfs_offsets0, gpfs_offsets, nprocs*3, ADIO_OFFSET, MPI_MAX, fd->comm );
+        for (ii=0; ii<nprocs; ii++)  {
+          st_offsets [ii] = gpfs_offsets[ii*3]  ;
+          end_offsets[ii] = gpfs_offsets[ii*3+1];
+          count_sizes[ii] = gpfs_offsets[ii*3+2];
+        }
+      }
+      else {
 	    gpfs_offsets0 = (ADIO_Offset *) ADIOI_Malloc(2*nprocs*sizeof(ADIO_Offset));
 	    gpfs_offsets  = (ADIO_Offset *) ADIOI_Malloc(2*nprocs*sizeof(ADIO_Offset));
 	    for (ii=0; ii<nprocs; ii++)  {
@@ -190,6 +220,7 @@ void ADIOI_GPFS_ReadStridedColl(ADIO_File fd, void *buf, int count,
 		st_offsets [ii] = gpfs_offsets[ii*2]  ;
 		end_offsets[ii] = gpfs_offsets[ii*2+1];
 	    }
+      }
 	    ADIOI_Free( gpfs_offsets0 );
 	    ADIOI_Free( gpfs_offsets  );
     } else {
@@ -197,6 +228,10 @@ void ADIOI_GPFS_ReadStridedColl(ADIO_File fd, void *buf, int count,
                       ADIO_OFFSET, fd->comm);
         MPI_Allgather(&end_offset, 1, ADIO_OFFSET, end_offsets, 1,
                       ADIO_OFFSET, fd->comm);
+        if ((gpfsmpio_read_aggmethod == 1) || (gpfsmpio_read_aggmethod == 2)) {
+	      MPI_Allgather(&count_sizes, 1, ADIO_OFFSET, count_sizes, 1,
+                        ADIO_OFFSET, fd->comm);
+        }
     }
 
     GPFSMPIO_T_CIO_SET_GET( r, 1, 1, GPFSMPIO_CIO_T_PATANA, GPFSMPIO_CIO_T_GATHER )
@@ -259,18 +294,69 @@ void ADIOI_GPFS_ReadStridedColl(ADIO_File fd, void *buf, int count,
      * needs to be mapped to an actual rank in the communicator later.
      *
      */
-    if (gpfsmpio_tuneblocking)
+    int currentNonZeroDataIndex = 0;
+    if ((gpfsmpio_read_aggmethod == 1) || (gpfsmpio_read_aggmethod == 2)) {
+      /* Take out the 0-data offsets by shifting the indexes with data to the
+       * front and keeping track of the non-zero data index for use as the
+       * length.  By doing this we will optimally use all available aggs
+       * and spread the actual data across them instead of having offsets
+       * with empty data potentially dilute the file domains and create
+       * problems for the one-sided aggregation.
+       */
+      for (i=0; i<nprocs; i++) {
+        if (count_sizes[i] > 0) {
+          st_offsets[currentNonZeroDataIndex] = st_offsets[i];
+          end_offsets[currentNonZeroDataIndex] = end_offsets[i];
+          currentNonZeroDataIndex++;
+        }
+      }
+    }
+    if (gpfsmpio_tuneblocking) {
+    if ((gpfsmpio_read_aggmethod == 1) || (gpfsmpio_read_aggmethod == 2)) {
+    ADIOI_GPFS_Calc_file_domains(fd, st_offsets, end_offsets, currentNonZeroDataIndex,
+			    nprocs_for_coll, &min_st_offset,
+			    &fd_start, &fd_end, &fd_size, fd->fs_ptr);
+    }
+    else {
     ADIOI_GPFS_Calc_file_domains(fd, st_offsets, end_offsets, nprocs,
 			    nprocs_for_coll, &min_st_offset,
 			    &fd_start, &fd_end, &fd_size, fd->fs_ptr);
-    else
+    }
+    }
+    else {
+    if ((gpfsmpio_read_aggmethod == 1) || (gpfsmpio_read_aggmethod == 2)) {
+    ADIOI_Calc_file_domains(st_offsets, end_offsets, currentNonZeroDataIndex,
+			    nprocs_for_coll, &min_st_offset,
+			    &fd_start, &fd_end,
+			    fd->hints->min_fdomain_size, &fd_size,
+			    fd->hints->striping_unit);
+	}
+    else {
     ADIOI_Calc_file_domains(st_offsets, end_offsets, nprocs,
 			    nprocs_for_coll, &min_st_offset,
 			    &fd_start, &fd_end,
 			    fd->hints->min_fdomain_size, &fd_size, 
 			    fd->hints->striping_unit);
+	}
+    }
 
     GPFSMPIO_T_CIO_SET_GET( r, 1, 1, GPFSMPIO_CIO_T_MYREQ, GPFSMPIO_CIO_T_FD_PART );
+    if ((gpfsmpio_read_aggmethod == 1) || (gpfsmpio_read_aggmethod == 2)) {
+    /* If the user has specified to use a one-sided aggregation method then do that at
+     * this point instead of the two-phase I/O.
+     */
+      ADIOI_OneSidedReadAggregation(fd, offset_list, len_list, contig_access_count, buf,
+		    datatype,error_code, st_offsets, end_offsets, currentNonZeroDataIndex, fd_start, fd_end);
+      GPFSMPIO_T_CIO_REPORT( 0, fd, myrank, nprocs)
+      ADIOI_Free(offset_list);
+      ADIOI_Free(len_list);
+      ADIOI_Free(st_offsets);
+      ADIOI_Free(end_offsets);
+      ADIOI_Free(fd_start);
+      ADIOI_Free(fd_end);
+      ADIOI_Free(count_sizes);
+	  goto fn_exit;
+    }
     if (gpfsmpio_p2pcontig==1) {
 	/* For some simple yet common(?) workloads, full-on two-phase I/O is
 	 * overkill.  We can establish sub-groups of processes and their
@@ -519,9 +605,7 @@ static void ADIOI_Read_and_exch(ADIO_File fd, void *buf, MPI_Datatype
 
     ADIOI_Datatype_iscontig(datatype, &buftype_is_contig);
     if (!buftype_is_contig) {
-	ADIOI_Flatten_datatype(datatype);
+	flat_buf = ADIOI_Flatten_and_find(datatype);
-	flat_buf = ADIOI_Flatlist;
-        while (flat_buf->type != datatype) flat_buf = flat_buf->next;
     }
     MPI_Type_get_extent(datatype, &buftype_lb, &buftype_extent);
 
@@ -609,7 +693,7 @@ static void ADIOI_Read_and_exch(ADIO_File fd, void *buf, MPI_Datatype
 		    }
 		    if (req_off < real_off + real_size) {
 			count[i]++;
-      ADIOI_Assert((((ADIO_Offset)(MPIR_Upint)read_buf)+req_off-real_off) == (ADIO_Offset)(MPIR_Upint)(read_buf+req_off-real_off));
+      ADIOI_Assert((((ADIO_Offset)(MPIU_Upint)read_buf)+req_off-real_off) == (ADIO_Offset)(MPIU_Upint)(read_buf+req_off-real_off));
 			MPI_Get_address(read_buf+req_off-real_off, 
                                &(others_req[i].mem_ptrs[j]));
       ADIOI_Assert((real_off + real_size - req_off) == (int)(real_off + real_size - req_off));
@@ -693,7 +777,7 @@ static void ADIOI_Read_and_exch(ADIO_File fd, void *buf, MPI_Datatype
 
 	if (for_next_iter) {
 	    tmp_buf = (char *) ADIOI_Malloc(for_next_iter);
-      ADIOI_Assert((((ADIO_Offset)(MPIR_Upint)read_buf)+real_size-for_next_iter) == (ADIO_Offset)(MPIR_Upint)(read_buf+real_size-for_next_iter));
+      ADIOI_Assert((((ADIO_Offset)(MPIU_Upint)read_buf)+real_size-for_next_iter) == (ADIO_Offset)(MPIU_Upint)(read_buf+real_size-for_next_iter));
       ADIOI_Assert((for_next_iter+coll_bufsize) == (size_t)(for_next_iter+coll_bufsize));
 	    memcpy(tmp_buf, read_buf+real_size-for_next_iter, for_next_iter);
 	    ADIOI_Free(fd->io_buf);
@@ -908,7 +992,7 @@ static void ADIOI_R_Exchange_data(ADIO_File fd, void *buf, ADIOI_Flatlist_node
 { \
     while (size) { \
 	size_in_buf = ADIOI_MIN(size, flat_buf_sz); \
-  ADIOI_Assert((((ADIO_Offset)(MPIR_Upint)buf) + user_buf_idx) == (ADIO_Offset)(MPIR_Upint)(buf + user_buf_idx)); \
+  ADIOI_Assert((((ADIO_Offset)(MPIU_Upint)buf) + user_buf_idx) == (ADIO_Offset)(MPIU_Upint)(buf + user_buf_idx)); \
   ADIOI_Assert(size_in_buf == (size_t)size_in_buf); \
 	memcpy(((char *) buf) + user_buf_idx, \
 	       &(recv_buf[p][recv_buf_idx[p]]), size_in_buf); \

ompi/mca/io/romio321/romio/adio/ad_gpfs/ad_gpfs_tuning.c

@@ -37,9 +37,14 @@ long    bglocklessmpio_f_type;
 int     gpfsmpio_bg_nagg_pset;
 int     gpfsmpio_pthreadio;
 int     gpfsmpio_p2pcontig;
+int     gpfsmpio_write_aggmethod;
+int     gpfsmpio_read_aggmethod;
 int	gpfsmpio_balancecontig;
 int     gpfsmpio_devnullio;
 int     gpfsmpio_bridgeringagg;
+int     gpfsmpio_onesided_no_rmw;
+int     gpfsmpio_onesided_always_rmw;
+int     gpfsmpio_onesided_inform_rmw;
 
 double	gpfsmpio_prof_cw    [GPFSMPIO_CIO_LAST+1];
 double	gpfsmpio_prof_cr    [GPFSMPIO_CIO_LAST+1];
@@ -105,6 +110,40 @@ double	gpfsmpio_prof_cr    [GPFSMPIO_CIO_LAST+1];
  * 3.) There are no gaps between the offsets.
  * 4.) No single rank has a data size which spans multiple file domains.
  *
+ * - GPFSMPIO_WRITE_AGGMETHOD/GPFSMPIO_READ_AGGMETHOD -  Replaces the two-phase
+ *   collective IO aggregation
+ *   with a one-sided algorithm, significantly reducing communication and
+ *   memory overhead.  Fully
+ *   supports all datasets and datatypes, the only caveat is that any holes in the data
+ *   when writing to a pre-existing file are ignored -- there is no read-modify-write
+ *   support to maintain the correctness of regions of pre-existing data so every byte
+ *   must be explicitly written to maintain correctness.  Users must beware of middle-ware
+ *   libraries like PNETCDF which may count on read-modify-write functionality for certain
+ *   features (like fill values).  Possible values:
+ *   - 0 - Normal two-phase collective IO is used.
+ *   - 1 - A separate one-sided MPI_Put or MPI_Get is used for each contigous chunk of data
+ *         for a compute to write to or read from the collective buffer on the aggregator.
+ *   - 2 - An MPI derived datatype is created using all the contigous chunks and just one
+ *         call to MPI_Put or MPI_Get is done with the derived datatype.  On Blue Gene /Q
+ *         optimal performance for this is achieved when paired with PAMID_TYPED_ONESIDED=1.
+ *   - Default is 0
+ *
+ * - GPFSMPIO_ONESIDED_NO_RMW - For one-sided aggregation (GPFSMPIO_WRITE_AGGMETHOD = 1 or 2)
+ *   disable the detection of holes in the data when writing to a pre-existing
+ *   file requiring a read-modify-write, thereby avoiding the communication
+ *   overhead for this detection.
+ *   - 0 (hole detection enabled) or 1 (hole detection disabled)
+ *   - Default is 0
+ *
+ * - GPFSMPIO_ONESIDED_INFORM_RMW - For one-sided aggregation
+ *   (GPFSMPIO_AGGMETHOD = 1 or 2) generate an informational message informing
+ *   the user whether holes exist in the data when writing to a pre-existing
+ *   file requiring a read-modify-write, thereby educating the user to set
+ *   GPFSMPIO_ONESIDED_NO_RMW=1 on a future run to avoid the communication
+ *   overhead for this detection.
+ *   - 0 (disabled) or 1 (enabled)
+ *   - Default is 0
+ *
  * - GPFSMPIO_BALANCECONTIG -  Relevant only to BGQ.  File domain blocks are assigned
  *   to aggregators in a breadth-first fashion relative to the ions - additionally,
  *   file domains on the aggregators sharing the same bridgeset and ion have contiguous
@@ -165,6 +204,14 @@ void ad_gpfs_get_env_vars() {
     x = getenv( "GPFSMPIO_P2PCONTIG" );
     if (x) gpfsmpio_p2pcontig = atoi(x);
 
+    gpfsmpio_write_aggmethod = 0;
+    x = getenv( "GPFSMPIO_WRITE_AGGMETHOD" );
+    if (x) gpfsmpio_write_aggmethod = atoi(x);
+
+    gpfsmpio_read_aggmethod = 0;
+    x = getenv( "GPFSMPIO_READ_AGGMETHOD" );
+    if (x) gpfsmpio_read_aggmethod = atoi(x);
+
     gpfsmpio_balancecontig = 0;
     x = getenv( "GPFSMPIO_BALANCECONTIG" );
     if (x) gpfsmpio_balancecontig = atoi(x);
@@ -176,6 +223,20 @@ void ad_gpfs_get_env_vars() {
     gpfsmpio_bridgeringagg = 0;
     x = getenv( "GPFSMPIO_BRIDGERINGAGG" );
     if (x) gpfsmpio_bridgeringagg = atoi(x);
+
+    gpfsmpio_onesided_no_rmw = 0;
+    x = getenv( "GPFSMPIO_ONESIDED_NO_RMW" );
+    if (x) gpfsmpio_onesided_no_rmw = atoi(x);
+
+    gpfsmpio_onesided_always_rmw = 0;
+    x = getenv( "GPFSMPIO_ONESIDED_ALWAYS_RMW" );
+    if (x) gpfsmpio_onesided_always_rmw = atoi(x);
+    if (gpfsmpio_onesided_always_rmw)
+      gpfsmpio_onesided_no_rmw = 1;
+
+    gpfsmpio_onesided_inform_rmw = 0;
+    x = getenv( "GPFSMPIO_ONESIDED_INFORM_RMW" );
+    if (x) gpfsmpio_onesided_inform_rmw = atoi(x);
 }
 
 /* report timing breakdown for MPI I/O collective call */

ompi/mca/io/romio321/romio/adio/ad_gpfs/ad_gpfs_tuning.h

@@ -23,6 +23,7 @@
  *  Global variables for the control of
  *  1.  timing
  *  2.  select specific optimizations
+ *  3.  global flags for certain optimizations
  *-----------------------------------------*/
 
 /* timing fields */
@@ -56,7 +57,6 @@ enum {
 extern double 	gpfsmpio_prof_cw    [GPFSMPIO_CIO_LAST+1];
 extern double 	gpfsmpio_prof_cr    [GPFSMPIO_CIO_LAST+1];
 
-
 /* corresponds to environment variables to select optimizations and timing level */
 extern int 	gpfsmpio_timing;
 extern int      gpfsmpio_timing_cw_level;
@@ -66,9 +66,14 @@ extern int 	gpfsmpio_tuneblocking;
 extern long bglocklessmpio_f_type;
 extern int      gpfsmpio_pthreadio;
 extern int      gpfsmpio_p2pcontig;
+extern int      gpfsmpio_write_aggmethod;
+extern int      gpfsmpio_read_aggmethod;
 extern int  gpfsmpio_balancecontig;
 extern int      gpfsmpio_devnullio;
 extern int      gpfsmpio_bridgeringagg;
+extern int      gpfsmpio_onesided_no_rmw;
+extern int      gpfsmpio_onesided_always_rmw;
+extern int      gpfsmpio_onesided_inform_rmw;
 
 /* Default is, well, kind of complicated. Blue Gene /L and /P had "psets": one
  * i/o node and all compute nodes wired to it.  On Blue Gene /Q that

ompi/mca/io/romio321/romio/adio/ad_gpfs/ad_gpfs_wrcoll.c

@@ -18,7 +18,7 @@
 #include "ad_gpfs_aggrs.h"
 
 #ifdef BGQPLATFORM
-#include <mpix.h>
+#include "bg/ad_bg_pset.h"
 #endif
 
 #ifdef AGGREGATION_PROFILE
@@ -132,6 +132,7 @@ void ADIOI_GPFS_WriteStridedColl(ADIO_File fd, const void *buf, int count,
     ADIO_Offset *offset_list = NULL, *st_offsets = NULL, *fd_start = NULL,
 	*fd_end = NULL, *end_offsets = NULL;
     ADIO_Offset *gpfs_offsets0 = NULL, *gpfs_offsets = NULL;
+    ADIO_Offset *count_sizes;
     int  ii;
 
     int *buf_idx = NULL;
@@ -175,7 +176,37 @@ void ADIOI_GPFS_WriteStridedColl(ADIO_File fd, const void *buf, int count,
 	st_offsets = (ADIO_Offset *) ADIOI_Malloc(nprocs*sizeof(ADIO_Offset));
 	end_offsets = (ADIO_Offset *) ADIOI_Malloc(nprocs*sizeof(ADIO_Offset));
 
+    ADIO_Offset my_count_size=0;
+    /* One-sided aggregation needs the amount of data per rank as well because
+     * the difference in starting and ending offsets for 1 byte is 0 the same
+     * as 0 bytes so it cannot be distiguished.
+     */
+    if ((gpfsmpio_write_aggmethod == 1) || (gpfsmpio_write_aggmethod == 2)) {
+        count_sizes = (ADIO_Offset *) ADIOI_Malloc(nprocs*sizeof(ADIO_Offset));
+        MPI_Count buftype_size;
+        MPI_Type_size_x(datatype, &buftype_size);
+        my_count_size = (ADIO_Offset) count  * (ADIO_Offset)buftype_size;
+    }
     if (gpfsmpio_tunegather) {
+      if ((gpfsmpio_write_aggmethod == 1) || (gpfsmpio_write_aggmethod == 2)) {
+        gpfs_offsets0 = (ADIO_Offset *) ADIOI_Malloc(3*nprocs*sizeof(ADIO_Offset));
+        gpfs_offsets  = (ADIO_Offset *) ADIOI_Malloc(3*nprocs*sizeof(ADIO_Offset));
+        for (ii=0; ii<nprocs; ii++)  {
+          gpfs_offsets0[ii*3]   = 0;
+          gpfs_offsets0[ii*3+1] = 0;
+          gpfs_offsets0[ii*3+2] = 0;
+        }
+        gpfs_offsets0[myrank*3]   = start_offset;
+        gpfs_offsets0[myrank*3+1] =   end_offset;
+        gpfs_offsets0[myrank*3+2] =   my_count_size;
+        MPI_Allreduce( gpfs_offsets0, gpfs_offsets, nprocs*3, ADIO_OFFSET, MPI_MAX, fd->comm );
+        for (ii=0; ii<nprocs; ii++)  {
+          st_offsets [ii] = gpfs_offsets[ii*3]  ;
+          end_offsets[ii] = gpfs_offsets[ii*3+1];
+          count_sizes[ii] = gpfs_offsets[ii*3+2];
+        }
+      }
+      else {
             gpfs_offsets0 = (ADIO_Offset *) ADIOI_Malloc(2*nprocs*sizeof(ADIO_Offset));
             gpfs_offsets  = (ADIO_Offset *) ADIOI_Malloc(2*nprocs*sizeof(ADIO_Offset));
             for (ii=0; ii<nprocs; ii++)  {
@@ -191,6 +222,7 @@ void ADIOI_GPFS_WriteStridedColl(ADIO_File fd, const void *buf, int count,
                 st_offsets [ii] = gpfs_offsets[ii*2]  ;
                 end_offsets[ii] = gpfs_offsets[ii*2+1];
             }
+      }
             ADIOI_Free( gpfs_offsets0 );
             ADIOI_Free( gpfs_offsets  );
     } else {
@@ -198,6 +230,10 @@ void ADIOI_GPFS_WriteStridedColl(ADIO_File fd, const void *buf, int count,
 		      ADIO_OFFSET, fd->comm);
 	MPI_Allgather(&end_offset, 1, ADIO_OFFSET, end_offsets, 1,
 		      ADIO_OFFSET, fd->comm);
+        if ((gpfsmpio_write_aggmethod == 1) || (gpfsmpio_write_aggmethod == 2)) {
+	    MPI_Allgather(&count_sizes, 1, ADIO_OFFSET, count_sizes, 1,
+                     ADIO_OFFSET, fd->comm);
+        }
     }
 
     GPFSMPIO_T_CIO_SET_GET(w, 1, 1, GPFSMPIO_CIO_T_PATANA, GPFSMPIO_CIO_T_GATHER )
@@ -250,25 +286,109 @@ void ADIOI_GPFS_WriteStridedColl(ADIO_File fd, const void *buf, int count,
    done by (logically) dividing the file into file domains (FDs); each
    process may directly access only its own file domain. */
 
-    if (gpfsmpio_tuneblocking)
+    int currentValidDataIndex = 0;
+    if ((gpfsmpio_write_aggmethod == 1) || (gpfsmpio_write_aggmethod == 2)) {
+      /* Take out the 0-data offsets by shifting the indexes with data to the front
+       * and keeping track of the valid data index for use as the length.
+       */
+      for (i=0; i<nprocs; i++) {
+        if (count_sizes[i] > 0) {
+          st_offsets[currentValidDataIndex] = st_offsets[i];
+          end_offsets[currentValidDataIndex] = end_offsets[i];
+          currentValidDataIndex++;
+        }
+      }
+    }
+
+    if (gpfsmpio_tuneblocking) {
+	if ((gpfsmpio_write_aggmethod == 1) || (gpfsmpio_write_aggmethod == 2)) {
+	    ADIOI_GPFS_Calc_file_domains(fd, st_offsets, end_offsets,
+		    currentValidDataIndex,
+		    nprocs_for_coll, &min_st_offset,
+		    &fd_start, &fd_end, &fd_size, fd->fs_ptr);
+	}
+	else {
+
     ADIOI_GPFS_Calc_file_domains(fd, st_offsets, end_offsets, nprocs,
 			    nprocs_for_coll, &min_st_offset,
 			    &fd_start, &fd_end, &fd_size, fd->fs_ptr);   
-    else
+	}
+    }
+    else {
+	if ((gpfsmpio_write_aggmethod == 1) || (gpfsmpio_write_aggmethod == 2)) {
+	    ADIOI_Calc_file_domains(st_offsets, end_offsets, currentValidDataIndex,
+		    nprocs_for_coll, &min_st_offset,
+		    &fd_start, &fd_end,
+		    fd->hints->min_fdomain_size, &fd_size,
+		    fd->hints->striping_unit);
+	}
+	else {
     ADIOI_Calc_file_domains(st_offsets, end_offsets, nprocs,
 			    nprocs_for_coll, &min_st_offset,
 			    &fd_start, &fd_end,
 			    fd->hints->min_fdomain_size, &fd_size,
 			    fd->hints->striping_unit);   
+	}
+    }
 
     GPFSMPIO_T_CIO_SET_GET( w, 1, 1, GPFSMPIO_CIO_T_MYREQ, GPFSMPIO_CIO_T_FD_PART );
 
+    if ((gpfsmpio_write_aggmethod == 1) || (gpfsmpio_write_aggmethod == 2)) {
+    /* If the user has specified to use a one-sided aggregation method then do that at
+     * this point instead of the two-phase I/O.
+     */
+      int holeFound = 0;
+      ADIOI_OneSidedWriteAggregation(fd, offset_list, len_list, contig_access_count,
+	      buf, datatype, error_code, st_offsets, end_offsets,
+	      currentValidDataIndex, fd_start, fd_end, &holeFound);
+      int anyHolesFound = 0;
+      if (!gpfsmpio_onesided_no_rmw)
+        MPI_Allreduce(&holeFound, &anyHolesFound, 1, MPI_INT, MPI_MAX, fd->comm);
+      if (anyHolesFound == 0) {
+      GPFSMPIO_T_CIO_REPORT( 1, fd, myrank, nprocs)
+      ADIOI_Free(offset_list);
+      ADIOI_Free(len_list);
+      ADIOI_Free(st_offsets);
+      ADIOI_Free(end_offsets);
+      ADIOI_Free(fd_start);
+      ADIOI_Free(fd_end);
+      ADIOI_Free(count_sizes);
+	  goto fn_exit;
+	  }
+      else {
+	/* Holes are found in the data and the user has not set
+	 * gpfsmpio_onesided_no_rmw --- set gpfsmpio_onesided_always_rmw to 1
+	 * and re-call ADIOI_OneSidedWriteAggregation and if the user has
+	 * gpfsmpio_onesided_inform_rmw set then inform him of this condition
+	 * and behavior.
+         */
+
+        if (gpfsmpio_onesided_inform_rmw && (myrank ==0))
+          FPRINTF(stderr,"Information: Holes found during one-sided "
+		  "write aggregation algorithm --- re-running one-sided "
+		  "write aggregation with GPFSMPIO_ONESIDED_ALWAYS_RMW set to 1.\n");
+          gpfsmpio_onesided_always_rmw = 1;
+          int prev_gpfsmpio_onesided_no_rmw = gpfsmpio_onesided_no_rmw;
+          gpfsmpio_onesided_no_rmw = 1;
+          ADIOI_OneSidedWriteAggregation(fd, offset_list, len_list, contig_access_count, buf, datatype, error_code, st_offsets, end_offsets, currentValidDataIndex, fd_start, fd_end, &holeFound);
+          gpfsmpio_onesided_no_rmw = prev_gpfsmpio_onesided_no_rmw;
+          GPFSMPIO_T_CIO_REPORT( 1, fd, myrank, nprocs)
+          ADIOI_Free(offset_list);
+          ADIOI_Free(len_list);
+          ADIOI_Free(st_offsets);
+          ADIOI_Free(end_offsets);
+          ADIOI_Free(fd_start);
+          ADIOI_Free(fd_end);
+          ADIOI_Free(count_sizes);
+          goto fn_exit;
+      }
+    }
     if (gpfsmpio_p2pcontig==1) {
 	/* For some simple yet common(?) workloads, full-on two-phase I/O is overkill.  We can establish sub-groups of processes and their aggregator, and then these sub-groups will carry out a simplified two-phase over that sub-group.
 	 *
 	 * First verify that the filetype is contig and the offsets are
 	 * increasing in rank order*/
-	int i, inOrderAndNoGaps = 1;
+	int inOrderAndNoGaps = 1;
 	for (i=0;i<(nprocs-1);i++) {
 	    if (end_offsets[i] != (st_offsets[i+1]-1))
 		inOrderAndNoGaps = 0;
@@ -445,7 +565,7 @@ static int gpfs_find_access_for_ion(ADIO_File fd,
 	ADIO_Offset *fd_start, ADIO_Offset *fd_end,
 	ADIO_Offset *start, ADIO_Offset *end)
 {
-    int my_ionode = MPIX_IO_node_id();
+    int my_ionode = BGQ_IO_node_id();
     int *rank_to_ionode;
     int i, nprocs, rank;
     ADIO_Offset group_start=LLONG_MAX, group_end=0;
@@ -630,9 +750,7 @@ static void ADIOI_Exch_and_write(ADIO_File fd, const void *buf, MPI_Datatype
 
     ADIOI_Datatype_iscontig(datatype, &buftype_is_contig);
     if (!buftype_is_contig) {
-	ADIOI_Flatten_datatype(datatype);
+	flat_buf = ADIOI_Flatten_and_find(datatype);
-	flat_buf = ADIOI_Flatlist;
-        while (flat_buf->type != datatype) flat_buf = flat_buf->next;
     }
     MPI_Type_get_extent(datatype, &buftype_lb, &buftype_extent);
 
@@ -705,7 +823,7 @@ static void ADIOI_Exch_and_write(ADIO_File fd, const void *buf, MPI_Datatype
 		    }
 		    if (req_off < off + size) {
 			count[i]++;
-      ADIOI_Assert((((ADIO_Offset)(MPIR_Upint)write_buf)+req_off-off) == (ADIO_Offset)(MPIR_Upint)(write_buf+req_off-off));
+      ADIOI_Assert((((ADIO_Offset)(MPIU_Upint)write_buf)+req_off-off) == (ADIO_Offset)(MPIU_Upint)(write_buf+req_off-off));
 			MPI_Get_address(write_buf+req_off-off, 
                                &(others_req[i].mem_ptrs[j]));
       ADIOI_Assert((off + size - req_off) == (int)(off + size - req_off));
@@ -800,7 +918,7 @@ static void ADIOI_Exch_and_write(ADIO_File fd, const void *buf, MPI_Datatype
 		io_thread_args.io_kind = ADIOI_WRITE;
 		io_thread_args.size = size;
 		io_thread_args.offset = off;
-		io_thread_args.status = status;
+		io_thread_args.status = &status;
 		io_thread_args.error_code = *error_code;
 		if ( (pthread_create(&io_thread, NULL,
 			ADIOI_IO_Thread_Func, &(io_thread_args))) != 0)
@@ -1151,7 +1269,7 @@ static void ADIOI_W_Exchange_data(ADIO_File fd, const void *buf, char *write_buf
 { \
     while (size) { \
         size_in_buf = ADIOI_MIN(size, flat_buf_sz); \
-  ADIOI_Assert((((ADIO_Offset)(MPIR_Upint)buf) + user_buf_idx) == (ADIO_Offset)(MPIR_Upint)((MPIR_Upint)buf + user_buf_idx)); \
+  ADIOI_Assert((((ADIO_Offset)(MPIU_Upint)buf) + user_buf_idx) == (ADIO_Offset)(MPIU_Upint)((MPIU_Upint)buf + user_buf_idx)); \
   ADIOI_Assert(size_in_buf == (size_t)size_in_buf); \
         memcpy(&(send_buf[p][send_buf_idx[p]]), \
                ((char *) buf) + user_buf_idx, size_in_buf); \

ompi/mca/io/romio321/romio/adio/ad_gpfs/bg/ad_bg_aggrs.c

@@ -281,6 +281,14 @@ ADIOI_BG_compute_agg_ranklist_serial_do (const ADIOI_BG_ConfInfo_t *confInfo,
           }
         }
         currentNumHops++;
+        /* Handle the case where the numAggs is more than exists starting
+         * at gpfsmpio_bridgeringagg hops, wrap back and restart at 0 to
+         * assign the overrun - it is up to the user to realize this
+         * situation and adjust numAggs and gpfsmpio_bridgeringagg
+         * accordingly.
+         */
+        if (currentNumHops > 16)
+          currentNumHops = 0;
         /* If 3 rounds go by without selecting an agg abort to avoid
            infinite loop.
         */
@@ -428,7 +436,7 @@ ADIOI_BG_compute_agg_ranklist_serial ( ADIO_File fd,
 
 #   if AGG_DEBUG
     for (i=0; i<confInfo->nProcs; i++) {
-      DBG_FPRINTF(stderr, "\tcpuid %1d, rank = %6d\n", all_procInfo[i].coreID, all_procInfo[i].rank );
+      DBG_FPRINTF(stderr, "\trank = %6d\n", all_procInfo[i].rank );
     }
 #   endif
 
@@ -437,14 +445,13 @@ ADIOI_BG_compute_agg_ranklist_serial ( ADIO_File fd,
 
 #   define VERIFY 1
 #   if VERIFY
-    DBG_FPRINTF(stderr, "\tconfInfo = min: %3d, max: %3d, naggrs: %3d, bridge: %3d, nprocs: %3d, vpset: %3d, tsize: %3d, ratio: %.4f; naggs = %d\n",
+    DBG_FPRINTF(stderr, "\tconfInfo = min: %3d, max: %3d, naggrs: %3d, bridge: %3d, nprocs: %3d, vpset: %3d, ratio: %.4f; naggs = %d\n",
 	    confInfo->ioMinSize        ,
 	    confInfo->ioMaxSize        ,
 	    confInfo->nAggrs           ,
 	    confInfo->numBridgeRanks ,
 	    confInfo->nProcs          ,
 	    confInfo->ioMaxSize /*virtualPsetSize*/          ,
-      confInfo->cpuIDsize,
 	    confInfo->aggRatio        ,
 	    naggs );
 #   endif

ompi/mca/io/romio321/romio/adio/ad_gpfs/bg/ad_bg_pset.c

@@ -16,14 +16,14 @@
 // #define bridgeringaggtrace 1
 
 #include <stdlib.h>
+#include <stdbool.h>
 #include "../ad_gpfs.h"
 #include "ad_bg_pset.h"
 #include <spi/include/kernel/process.h>
 #include <firmware/include/personality.h>
 
-#ifdef HAVE_MPIX_H
+#define BGQ_TORUS_MAX_DIMS 5
-#include <mpix.h>
+#define BGQ_FULL_TORUS_SIZE 512
-#endif
 
 #ifndef TRACE_ERR
 #  define TRACE_ERR(fmt...)
@@ -81,8 +81,8 @@ static int intsort(const void *p1, const void *p2)
    return(i1->bridgeCoord - i2->bridgeCoord);
 }
 
-unsigned torusSize[MPIX_TORUS_MAX_DIMS];
+unsigned torusSize[BGQ_TORUS_MAX_DIMS];
-unsigned dimTorus[MPIX_TORUS_MAX_DIMS];
+bool dimTorus[BGQ_TORUS_MAX_DIMS];
 
 /* This function computes the number of hops between the torus coordinates of the
  * aggCoords and bridgeCoords parameters.
@@ -91,7 +91,7 @@ static unsigned procManhattanDistance(unsigned *aggCoords, unsigned *bridgeCoord
 
   unsigned totalDistance = 0;
   int i;
-  for (i=0;i<MPIX_TORUS_MAX_DIMS;i++) {
+  for (i=0;i<BGQ_TORUS_MAX_DIMS;i++) {
     unsigned dimDistance = abs((int)aggCoords[i] - (int)bridgeCoords[i]);
     if (dimDistance > 0) { // could torus make it closer?
       if (dimTorus[i]) {
@@ -111,6 +111,31 @@ static unsigned procManhattanDistance(unsigned *aggCoords, unsigned *bridgeCoord
   return totalDistance;
 }
 
+int BGQ_IO_node_id ()
+{
+  static unsigned long IO_node_id = ULONG_MAX;
+
+  if (IO_node_id != ULONG_MAX)
+    return (int)(IO_node_id>>32);
+
+  int rc;
+  int fd;
+  char* uci_str;
+  char buffer[4096];
+
+  fd = open("/dev/bgpers", O_RDONLY, 0);
+  assert(fd>=0);
+  rc = read(fd, buffer, sizeof(buffer));
+  assert(rc>0);
+  close(fd);
+
+  uci_str = strstr(buffer, "BG_UCI=");
+  assert(uci_str);
+  uci_str += sizeof("BG_UCI=")-1;
+
+  IO_node_id = strtoul(uci_str, NULL, 16);
+  return (int)(IO_node_id>>32);
+}
 
 void 
 ADIOI_BG_persInfo_init(ADIOI_BG_ConfInfo_t *conf, 
@@ -125,15 +150,13 @@ ADIOI_BG_persInfo_init(ADIOI_BG_ConfInfo_t *conf,
    TRACE_ERR("Entering BG_persInfo_init, size: %d, rank: %d, n_aggrs: %d, comm: %d\n", size, rank, n_aggrs, (int)comm);
 
    Personality_t pers;
-   MPIX_Hardware_t hw;
-   MPIX_Hardware(&hw);
-   TRACE_ERR("BG_persInfo_init, my coords{%u,%u,%u,%u,%u} rankInPset %u,sizeOfPset %u,idOfPset %u\n",hw.Coords[0],hw.Coords[1],hw.Coords[2],hw.Coords[3],hw.Coords[4],hw.rankInPset,hw.sizeOfPset,hw.idOfPset);
 
 
    Kernel_GetPersonality(&pers, sizeof(pers));
+   Personality_Networks_t *net = &pers.Network_Config;
 
+   TRACE_ERR("BG_persInfo_init, my coords{%u,%u,%u,%u,%u}\n",net->Acoord,net->Bcoord,net->Ccoord,net->Dcoord,net->Ecoord);
    proc->rank = rank;
-   proc->coreID = hw.coreID;
 
    if (gpfsmpio_bridgeringagg > 0) {
 #ifdef bridgeringaggtrace
@@ -143,26 +166,44 @@ ADIOI_BG_persInfo_init(ADIOI_BG_ConfInfo_t *conf,
 
      /* Set the numNodesInPartition and nodeRank for this proc
      */
-     proc->numNodesInPartition = 1;
+     unsigned dimMaxArray[BGQ_TORUS_MAX_DIMS];
+     dimMaxArray[0] = net->Anodes;
+     dimMaxArray[1] = net->Bnodes;
+     dimMaxArray[2] = net->Cnodes;
+     dimMaxArray[3] = net->Dnodes;
+     dimMaxArray[4] = net->Enodes;
+
+     unsigned hwCoordsArray[BGQ_TORUS_MAX_DIMS];
+     hwCoordsArray[0] = net->Acoord;
+     hwCoordsArray[1] = net->Bcoord;
+     hwCoordsArray[2] = net->Ccoord;
+     hwCoordsArray[3] = net->Dcoord;
+     hwCoordsArray[4] = net->Ecoord;
+     proc->numNodesInPartition = net->Anodes * net->Bnodes * net->Cnodes * net->Dnodes * net->Enodes;
      proc->nodeRank = 0;
-     for (i=0;i<MPIX_TORUS_MAX_DIMS;i++) {
+     /* Set the indicator for if a dimension in the partitions is a torus or not.
-       torusSize[i] = hw.Size[i];
+      */
-       dimTorus[i] = hw.isTorus[i];
+     dimTorus[0] = (bool) (ND_ENABLE_TORUS_DIM_A & net->NetFlags);
-       proc->numNodesInPartition *= hw.Size[i];
+     dimTorus[1] = (bool) (ND_ENABLE_TORUS_DIM_B & net->NetFlags);
+     dimTorus[2] = (bool) (ND_ENABLE_TORUS_DIM_C & net->NetFlags);
+     dimTorus[3] = (bool) (ND_ENABLE_TORUS_DIM_D & net->NetFlags);
+     dimTorus[4] = (bool) (ND_ENABLE_TORUS_DIM_E & net->NetFlags);
+     for (i=0;i<BGQ_TORUS_MAX_DIMS;i++) {
+       torusSize[i] = dimMaxArray[i];
          int baseNum = 1, j;
          for (j=0;j<i;j++)
-           baseNum *= hw.Size[j];
+           baseNum *= dimMaxArray[j];
-         proc->nodeRank += (hw.Coords[i] * baseNum);
+         proc->nodeRank += (hwCoordsArray[i] * baseNum);
 #ifdef bridgeringaggtrace
        if (rank == 0)
-         fprintf(stderr,"Dimension %d has %d elements wrap-around value is %d\n",i,torusSize[i],dimTorus[i]);
+         fprintf(stderr,"numNodesInPartition is %d Dimension %d has %d elements wrap-around value is %d\n",proc->numNodesInPartition,i,torusSize[i],dimTorus[i]);
 #endif
      }
    }
 
    MPI_Comm_size(comm, &commsize);
 
-   proc->ionID = MPIX_IO_node_id ();
+   proc->ionID = BGQ_IO_node_id ();
 
    if(size == 1)
    {
@@ -179,8 +220,6 @@ ADIOI_BG_persInfo_init(ADIOI_BG_ConfInfo_t *conf,
       conf->ioMaxSize = size;
       conf->numBridgeRanks = 1;
       conf->nProcs = size;
-      conf->cpuIDsize = hw.ppn;
-      /*conf->virtualPsetSize = conf->ioMaxSize * conf->cpuIDsize;*/
       conf->nAggrs = 1;
       conf->aggRatio = 1. * conf->nAggrs / conf->ioMinSize /*virtualPsetSize*/;
       if(conf->aggRatio > 1) conf->aggRatio = 1.;
@@ -200,11 +239,11 @@ ADIOI_BG_persInfo_init(ADIOI_BG_ConfInfo_t *conf,
                   pers.Network_Config.cnBridge_E << 2;
    ADIOI_Assert((bridgeCoords >= 0)); /* A dim is < 6 bits or sorting won't work */
 
-   if((hw.Coords[0] == pers.Network_Config.cnBridge_A) &&
+   if((net->Acoord == pers.Network_Config.cnBridge_A) &&
-      (hw.Coords[1] == pers.Network_Config.cnBridge_B) &&
+      (net->Bcoord == pers.Network_Config.cnBridge_B) &&
-      (hw.Coords[2] == pers.Network_Config.cnBridge_C) &&
+      (net->Ccoord == pers.Network_Config.cnBridge_C) &&
-      (hw.Coords[3] == pers.Network_Config.cnBridge_D) &&
+      (net->Dcoord == pers.Network_Config.cnBridge_D) &&
-      (hw.Coords[4] == pers.Network_Config.cnBridge_E)) {
+      (net->Ecoord == pers.Network_Config.cnBridge_E)) {
       iambridge = 1;      /* I am bridge */
       if (gpfsmpio_bridgeringagg > 0) {
         proc->manhattanDistanceToBridge = 0;
@@ -212,16 +251,16 @@ ADIOI_BG_persInfo_init(ADIOI_BG_ConfInfo_t *conf,
     }
     else {  // calculate manhattan distance to bridge if gpfsmpio_bridgeringagg is set
       if (gpfsmpio_bridgeringagg > 0) {
-        unsigned aggCoords[MPIX_TORUS_MAX_DIMS],manhattanBridgeCoords[MPIX_TORUS_MAX_DIMS];
+        unsigned aggCoords[BGQ_TORUS_MAX_DIMS],manhattanBridgeCoords[BGQ_TORUS_MAX_DIMS];
-        aggCoords[0] = hw.Coords[0];
+        aggCoords[0] = net->Acoord;
         manhattanBridgeCoords[0] = pers.Network_Config.cnBridge_A;
-        aggCoords[1] = hw.Coords[1];
+        aggCoords[1] = net->Bcoord;
         manhattanBridgeCoords[1] = pers.Network_Config.cnBridge_B;
-        aggCoords[2] = hw.Coords[2];
+        aggCoords[2] = net->Ccoord;
         manhattanBridgeCoords[2] = pers.Network_Config.cnBridge_C;
-        aggCoords[3] = hw.Coords[3];
+        aggCoords[3] = net->Dcoord;
         manhattanBridgeCoords[3] = pers.Network_Config.cnBridge_D;
-        aggCoords[4] = hw.Coords[4];
+        aggCoords[4] = net->Ecoord;
         manhattanBridgeCoords[4] = pers.Network_Config.cnBridge_E;
 
         proc->manhattanDistanceToBridge= procManhattanDistance(aggCoords, manhattanBridgeCoords);
@@ -342,8 +381,6 @@ ADIOI_BG_persInfo_init(ADIOI_BG_ConfInfo_t *conf,
       conf->ioMaxSize = maxcompute; /* equivalent to pset size */
       conf->numBridgeRanks = bridgeIndex+1;
       conf->nProcs = size;
-      conf->cpuIDsize = hw.ppn;
-      /*conf->virtualPsetSize = maxcompute * conf->cpuIDsize;*/
             
       conf->nAggrs = n_aggrs;
       /*    First pass gets nAggrs = -1 */

ompi/mca/io/romio321/romio/adio/ad_gpfs/bg/ad_bg_pset.h

@@ -17,9 +17,6 @@
 #ifndef AD_BG_PSET_H_
 #define AD_BG_PSET_H_
 
-#ifdef HAVE_MPIX_H
-#include <mpix.h>
-#endif
 
 /* Keeps specific information to each process, will be exchanged among processes */
 typedef struct {
@@ -28,7 +25,6 @@ typedef struct {
    int ionID;  /* ion id this cn is using */
 /*   int myCoords[5]; */
    int bridgeRank; /* my bridge node (or proxy) rank */
-   unsigned char coreID;
    unsigned char threadID; /* unlikely to be useful but better than just padding */
    unsigned char __cpad[2];
    int myIOSize;  /* number of ranks sharing my bridge/IO
@@ -73,6 +69,7 @@ typedef struct {
 
 
 /* public funcs for a pair of ADIOI_BG_ConfInfo_t and ADIOI_BG_ProcInfo_t objects */
+    int BGQ_IO_node_id ();
     void ADIOI_BG_persInfo_init( ADIOI_BG_ConfInfo_t *conf,
 				  ADIOI_BG_ProcInfo_t *proc,
 				  int s, int r, int n_aggrs, MPI_Comm comm);

ompi/mca/io/romio321/romio/adio/ad_gridftp/ad_gridftp.c

@@ -34,4 +34,6 @@ struct ADIOI_Fns_struct ADIO_GRIDFTP_operations = {
     ADIOI_GRIDFTP_Resize, /* Resize */
     ADIOI_GRIDFTP_Delete, /* Delete */
     ADIOI_GRIDFTP_Feature, /* Features */
+    ADIOI_GEN_IreadStridedColl, /* IreadStridedColl */
+    ADIOI_GEN_IwriteStridedColl /* IwriteStridedColl */
 };

ompi/mca/io/romio321/romio/adio/ad_gridftp/ad_gridftp_read.c

@@ -241,10 +241,7 @@ void ADIOI_GRIDFTP_ReadDiscontig(ADIO_File fd, void *buf, int count,
     /* from here we can assume btype_extent==btype_size */
 
     /* Flatten out fd->filetype so we know which blocks to skip */
-    ADIOI_Flatten_datatype(fd->filetype);
+    flat_file = ADIOI_Flatten_and_find(fd->filetype);
-    flat_file = ADIOI_Flatlist;
-    while (flat_file->type != fd->filetype && flat_file->next!=NULL)
-	flat_file = flat_file->next;
 
     /* Figure out how big the area to read is */
     start=(globus_off_t)(offset*etype_size);

ompi/mca/io/romio321/romio/adio/ad_gridftp/ad_gridftp_write.c

@@ -242,10 +242,7 @@ void ADIOI_GRIDFTP_WriteDiscontig(ADIO_File fd, void *buf, int count,
     /* from here we can assume btype_extent==btype_size */
 
     /* Flatten out fd->filetype so we know which blocks to skip */
-    ADIOI_Flatten_datatype(fd->filetype);
+    flat_file = ADIOI_Flatten_and_find(fd->filetype);
-    flat_file = ADIOI_Flatlist;
-    while (flat_file->type != fd->filetype && flat_file->next!=NULL)
-	flat_file = flat_file->next;
 
     /* Figure out how big the area to write is */
     /* ASSUMPTION: ftype_size is an integer multiple of btype_size or vice versa. */

ompi/mca/io/romio321/romio/adio/ad_hfs/ad_hfs.c

@@ -33,4 +33,6 @@ struct ADIOI_Fns_struct ADIO_HFS_operations = {
     ADIOI_GEN_Flush, /* Flush */
     ADIOI_HFS_Resize, /* Resize */
     ADIOI_GEN_Delete, /* Delete */
+    ADIOI_GEN_IreadStridedColl, /* IreadStridedColl */
+    ADIOI_GEN_IwriteStridedColl /* IwriteStridedColl */
 };

ompi/mca/io/romio321/romio/adio/ad_lustre/ad_lustre.c

@@ -41,4 +41,6 @@ struct ADIOI_Fns_struct ADIO_LUSTRE_operations = {
     ADIOI_GEN_Delete, /* Delete */
     ADIOI_GEN_Feature, /* Features */
     "LUSTRE:",
+    ADIOI_GEN_IreadStridedColl, /* IreadStridedColl */
+    ADIOI_GEN_IwriteStridedColl /* IwriteStridedColl */
 };

ompi/mca/io/romio321/romio/adio/ad_lustre/ad_lustre.h

@@ -35,12 +35,16 @@
 #include <signal.h>
 #endif
 
+#ifdef HAVE_AIO_LITE_H
+#include <aio-lite.h>
+#else
  #ifdef  HAVE_AIO_H
  #include <aio.h>
+ #endif
  #ifdef HAVE_SYS_AIO_H
  #include <sys/aio.h>
  #endif
-#endif /* End of HAVE_SYS_AIO_H */
+#endif /* End of HAVE_AIO_LITE_H */
 
 void ADIOI_LUSTRE_Open(ADIO_File fd, int *error_code);
 void ADIOI_LUSTRE_Close(ADIO_File fd, int *error_code);

ompi/mca/io/romio321/romio/adio/ad_lustre/ad_lustre_aggregate.c

@@ -310,8 +310,13 @@ int ADIOI_LUSTRE_Docollect(ADIO_File fd, int contig_access_count,
                fd->comm);
     MPI_Allreduce(&contig_access_count, &total_access_count, 1, MPI_INT, MPI_SUM,
                fd->comm);
+    /* avoid possible divide-by-zero) */
+    if (total_access_count != 0) {
 	/* estimate average req_size */
 	avg_req_size = (int)(total_req_size / total_access_count);
+    } else {
+	avg_req_size = 0;
+    }
     /* get hint of big_req_size */
     big_req_size = fd->hints->fs_hints.lustre.coll_threshold;
     /* Don't perform collective I/O if there are big requests */

ompi/mca/io/romio321/romio/adio/ad_lustre/ad_lustre_hints.c

@@ -20,7 +20,7 @@ void ADIOI_LUSTRE_SetInfo(ADIO_File fd, MPI_Info users_info, int *error_code)
     char *value;
     int flag;
     ADIO_Offset stripe_val[3], str_factor = -1, str_unit=0, start_iodev=-1;
-    int err, myrank;
+    int myrank;
     static char myname[] = "ADIOI_LUSTRE_SETINFO";
 
     value = (char *) ADIOI_Malloc((MPI_MAX_INFO_VAL+1)*sizeof(char));
@@ -125,6 +125,12 @@ void ADIOI_LUSTRE_SetInfo(ADIO_File fd, MPI_Info users_info, int *error_code)
     /* set the values for collective I/O and data sieving parameters */
     ADIOI_GEN_SetInfo(fd, users_info, error_code);
 
+    /* generic hints might step on striping_unit */
+    if (users_info != MPI_INFO_NULL) {
+	ADIOI_Info_check_and_install_int(fd, users_info, "striping_unit",
+		NULL, myname, error_code);
+    }
+
     if (ADIOI_Direct_read) fd->direct_read = 1;
     if (ADIOI_Direct_write) fd->direct_write = 1;
 

ompi/mca/io/romio321/romio/adio/ad_lustre/ad_lustre_open.c

@@ -147,9 +147,6 @@ void ADIOI_LUSTRE_Open(ADIO_File fd, int *error_code)
     if (fd->access_mode & ADIO_APPEND)
 	fd->fp_ind = fd->fp_sys_posn = lseek(fd->fd_sys, 0, SEEK_END);
 
-    if ((fd->fd_sys != -1) && (fd->access_mode & ADIO_APPEND))
-	fd->fp_ind = fd->fp_sys_posn = lseek(fd->fd_sys, 0, SEEK_END);
-
     fd->fd_direct = -1;
     if (fd->direct_write || fd->direct_read) {
 	fd->fd_direct = open(fd->filename, amode_direct, perm);
@@ -160,10 +157,10 @@ void ADIOI_LUSTRE_Open(ADIO_File fd, int *error_code)
 	    fd->direct_write = fd->direct_read = 0;
 	}
     }
+
 fn_exit:
     ADIOI_Free(lum);
     ADIOI_Free(value);
-
     /* --BEGIN ERROR HANDLING-- */
     if (fd->fd_sys == -1 || ((fd->fd_direct == -1) && 
 		(fd->direct_write || fd->direct_read))) {

ompi/mca/io/romio321/romio/adio/ad_lustre/ad_lustre_wrcoll.c

@@ -415,10 +415,7 @@ static void ADIOI_LUSTRE_Exch_and_write(ADIO_File fd, const void *buf,
 
     ADIOI_Datatype_iscontig(datatype, &buftype_is_contig);
     if (!buftype_is_contig) {
-	ADIOI_Flatten_datatype(datatype);
+	flat_buf = ADIOI_Flatten_and_find(datatype);
-	flat_buf = ADIOI_Flatlist;
-	while (flat_buf->type != datatype)
-	    flat_buf = flat_buf->next;
     }
     MPI_Type_get_extent(datatype, &buftype_lb, &buftype_extent);
     /* I need to check if there are any outstanding nonblocking writes to
@@ -500,7 +497,7 @@ static void ADIOI_LUSTRE_Exch_and_write(ADIO_File fd, const void *buf,
                     req_len = others_req[i].lens[j];
 		    if (req_off < iter_st_off + max_size) {
 			recv_count[i]++;
-                        ADIOI_Assert((((ADIO_Offset)(MPIR_Upint)write_buf)+req_off-off) == (ADIO_Offset)(MPIR_Upint)(write_buf+req_off-off));
+                        ADIOI_Assert((((ADIO_Offset)(MPIU_Upint)write_buf)+req_off-off) == (ADIO_Offset)(MPIU_Upint)(write_buf+req_off-off));
 			MPI_Get_address(write_buf + req_off - off,
 				        &(others_req[i].mem_ptrs[j]));
                         recv_size[i] += req_len;
@@ -862,7 +859,7 @@ static void ADIOI_LUSTRE_W_Exchange_data(ADIO_File fd, const void *buf,
 { \
     while (size) { \
         size_in_buf = ADIOI_MIN(size, flat_buf_sz); \
-        ADIOI_Assert((((ADIO_Offset)(MPIR_Upint)buf) + user_buf_idx) == (ADIO_Offset)(MPIR_Upint)((MPIR_Upint)buf + user_buf_idx)); \
+        ADIOI_Assert((((ADIO_Offset)(MPIU_Upint)buf) + user_buf_idx) == (ADIO_Offset)(MPIU_Upint)((MPIU_Upint)buf + user_buf_idx)); \
         ADIOI_Assert(size_in_buf == (size_t)size_in_buf);               \
         memcpy(&(send_buf[p][send_buf_idx[p]]), \
                ((char *) buf) + user_buf_idx, size_in_buf); \

ompi/mca/io/romio321/romio/adio/ad_lustre/ad_lustre_wrstr.c

@@ -209,10 +209,7 @@ void ADIOI_LUSTRE_WriteStrided(ADIO_File fd, const void *buf, int count,
     /* Different buftype to different filetype */
     if (!buftype_is_contig && filetype_is_contig) {
         /* noncontiguous in memory, contiguous in file. */
-	ADIOI_Flatten_datatype(datatype);
+	flat_buf = ADIOI_Flatten_and_find(datatype);
-	flat_buf = ADIOI_Flatlist;
-	while (flat_buf->type != datatype)
-	    flat_buf = flat_buf->next;
 
 	off = (file_ptr_type == ADIO_INDIVIDUAL) ? fd->fp_ind :
             fd->disp + (ADIO_Offset)etype_size * offset;
@@ -436,10 +433,7 @@ void ADIOI_LUSTRE_WriteStrided(ADIO_File fd, const void *buf, int count,
         }
         else {
 /* noncontiguous in memory as well as in file */
-
+	    flat_buf = ADIOI_Flatten_and_find(datatype);
-	        ADIOI_Flatten_datatype(datatype);
-	        flat_buf = ADIOI_Flatlist;
-            while (flat_buf->type != datatype) flat_buf = flat_buf->next;
 
 		k = num = buf_count = 0;
             i_offset = flat_buf->indices[0];

ompi/mca/io/romio321/romio/adio/ad_nfs/ad_nfs.c

@@ -37,5 +37,7 @@ struct ADIOI_Fns_struct ADIO_NFS_operations = {
     ADIOI_NFS_Resize, /* Resize */
     ADIOI_GEN_Delete, /* Delete */
     ADIOI_NFS_Feature, /* Features */
-    "NFS:"  /* fsname: just a string */
+    "NFS:",  /* fsname: just a string */
+    ADIOI_GEN_IreadStridedColl, /* IreadStridedColl */
+    ADIOI_GEN_IwriteStridedColl /* IwriteStridedColl */
 };

ompi/mca/io/romio321/romio/adio/ad_nfs/ad_nfs.h

@@ -7,10 +7,11 @@
 #ifndef AD_NFS_INCLUDE
 #define AD_NFS_INCLUDE
 
+#include "adio.h"
+
 #include <unistd.h>
 #include <sys/types.h>
 #include <fcntl.h>
-#include "adio.h"
 
 #ifdef HAVE_SIGNAL_H
 #include <signal.h>
@@ -18,12 +19,16 @@
 #ifdef HAVE_SYS_TYPES_H
 #include <sys/types.h>
 #endif
+#ifdef HAVE_AIO_LITE_H
+#include <aio-lite.h>
+#else
  #ifdef  HAVE_AIO_H
  #include <aio.h>
  #endif
  #ifdef HAVE_SYS_AIO_H
  #include <sys/aio.h>
  #endif
+#endif
 
 /* Workaround for incomplete set of definitions if __REDIRECT is not 
    defined and large file support is used in aio.h */

ompi/mca/io/romio321/romio/adio/ad_nfs/ad_nfs_iwrite.c

@@ -69,19 +69,19 @@ int ADIOI_NFS_aio(ADIO_File fd, void *buf, int len, ADIO_Offset offset,
     aiocbp->aio_buf    = buf;
     aiocbp->aio_nbytes = len;
 
-#ifdef ROMIO_HAVE_STRUCT_AIOCB_WITH_AIO_WHENCE
+#ifdef HAVE_STRUCT_AIOCB_AIO_WHENCE
     aiocbp->aio_whence = SEEK_SET;
 #endif
-#ifdef ROMIO_HAVE_STRUCT_AIOCB_WITH_AIO_FILDES
+#ifdef HAVE_STRUCT_AIOCB_AIO_FILDES
     aiocbp->aio_fildes = fd_sys;
 #endif
-#ifdef ROMIO_HAVE_STRUCT_AIOCB_WITH_AIO_SIGEVENT
+#ifdef HAVE_STRUCT_AIOCB_AIO_SIGEVENT
 # ifdef AIO_SIGNOTIFY_NONE
     aiocbp->aio_sigevent.sigev_notify = SIGEV_NONE;
 # endif
     aiocbp->aio_sigevent.sigev_signo = 0;
 #endif
-#ifdef ROMIO_HAVE_STRUCT_AIOCB_WITH_AIO_REQPRIO
+#ifdef HAVE_STRUCT_AIOCB_AIO_REQPRIO
 # ifdef AIO_PRIO_DFL
     aiocbp->aio_reqprio = AIO_PRIO_DFL;   /* not needed in DEC Unix 4.0 */
 # else

ompi/mca/io/romio321/romio/adio/ad_nfs/ad_nfs_read.c

@@ -7,68 +7,79 @@
 
 #include "ad_nfs.h"
 #include "adio_extern.h"
-#ifdef HAVE_UNISTD_H
-#include <unistd.h>
-#endif
 
 void ADIOI_NFS_ReadContig(ADIO_File fd, void *buf, int count, 
                      MPI_Datatype datatype, int file_ptr_type,
 		     ADIO_Offset offset, ADIO_Status *status, int *error_code)
 {
-    ssize_t err=-1;
+    int err=-1;
     MPI_Count datatype_size, len;
-    ADIO_Offset bytes_xfered=0;
-    size_t rd_count;
     static char myname[] = "ADIOI_NFS_READCONTIG";
-    char *p;
 
     MPI_Type_size_x(datatype, &datatype_size);
     len = datatype_size * count;
 
-    if (file_ptr_type == ADIO_INDIVIDUAL) {
+    if (file_ptr_type == ADIO_EXPLICIT_OFFSET) {
-	offset = fd->fp_ind;
+	if (fd->fp_sys_posn != offset) {
+#ifdef ADIOI_MPE_LOGGING
+            MPE_Log_event( ADIOI_MPE_lseek_a, 0, NULL );
+#endif
+	    lseek(fd->fd_sys, offset, SEEK_SET);
+#ifdef ADIOI_MPE_LOGGING
+            MPE_Log_event( ADIOI_MPE_lseek_b, 0, NULL );
+#endif
         }
-
-    p = buf;
-    while (bytes_xfered < len ) {
-        rd_count = len - bytes_xfered;
-        /* FreeBSD and Darwin workaround: bigger than INT_MAX is an error */
-        if (rd_count > INT_MAX)
-            rd_count = INT_MAX;
 	if (fd->atomicity)
-            ADIOI_WRITE_LOCK(fd, offset+bytes_xfered, SEEK_SET, rd_count);
+	    ADIOI_WRITE_LOCK(fd, offset, SEEK_SET, len);
-        else ADIOI_READ_LOCK(fd, offset+bytes_xfered, SEEK_SET, rd_count);
+	else ADIOI_READ_LOCK(fd, offset, SEEK_SET, len);
 #ifdef ADIOI_MPE_LOGGING
         MPE_Log_event( ADIOI_MPE_read_a, 0, NULL );
 #endif
-        err = pread(fd->fd_sys, p, rd_count, offset+bytes_xfered);
+	err = read(fd->fd_sys, buf, len);
-        /* --BEGIN ERROR HANDLING-- */
-        if (err == -1) {
-            *error_code = MPIO_Err_create_code(MPI_SUCCESS,
-        	    MPIR_ERR_RECOVERABLE, myname, __LINE__, MPI_ERR_IO,
-        	    "**io", "**io %s", strerror(errno));
-        }
-        /* --END ERROR HANDLING-- */
 #ifdef ADIOI_MPE_LOGGING
         MPE_Log_event( ADIOI_MPE_read_b, 0, NULL );
 #endif
-        ADIOI_UNLOCK(fd, offset+bytes_xfered, SEEK_SET, rd_count);
+	ADIOI_UNLOCK(fd, offset, SEEK_SET, len);
-        if (err == 0) {
+	fd->fp_sys_posn = offset + err;
-            /* end of file */
+	/* individual file pointer not updated */        
-            break;
     }
-        bytes_xfered += err;
+    else {  /* read from curr. location of ind. file pointer */
-        p += err;
+	offset = fd->fp_ind;
+	if (fd->fp_sys_posn != fd->fp_ind) {
+#ifdef ADIOI_MPE_LOGGING
+            MPE_Log_event( ADIOI_MPE_lseek_a, 0, NULL );
+#endif
+	    lseek(fd->fd_sys, fd->fp_ind, SEEK_SET);
+#ifdef ADIOI_MPE_LOGGING
+            MPE_Log_event( ADIOI_MPE_lseek_b, 0, NULL );
+#endif
+        }
+	if (fd->atomicity)
+	    ADIOI_WRITE_LOCK(fd, offset, SEEK_SET, len);
+	else ADIOI_READ_LOCK(fd, offset, SEEK_SET, len);
+#ifdef ADIOI_MPE_LOGGING
+        MPE_Log_event( ADIOI_MPE_read_a, 0, NULL );
+#endif
+	err = read(fd->fd_sys, buf, len);
+#ifdef ADIOI_MPE_LOGGING
+        MPE_Log_event( ADIOI_MPE_read_b, 0, NULL );
+#endif
+	ADIOI_UNLOCK(fd, offset, SEEK_SET, len);
+	fd->fp_ind += err;
+	fd->fp_sys_posn = fd->fp_ind;
     }
 
-    fd->fp_sys_posn = offset + bytes_xfered;
+    /* --BEGIN ERROR HANDLING-- */
-    if (file_ptr_type == ADIO_INDIVIDUAL) {
+    if (err == -1) {
-        fd->fp_ind += bytes_xfered;
+	*error_code = MPIO_Err_create_code(MPI_SUCCESS, MPIR_ERR_RECOVERABLE,
+					   myname, __LINE__, MPI_ERR_IO,
+					   "**io", "**io %s", strerror(errno));
+	return;
     }
     /* --END ERROR HANDLING-- */
 
 #ifdef HAVE_STATUS_SET_BYTES
-    if (err != -1) MPIR_Status_set_bytes(status, datatype, bytes_xfered);
+    MPIR_Status_set_bytes(status, datatype, err);
 #endif
 
     *error_code = MPI_SUCCESS;
@@ -206,9 +217,7 @@ void ADIOI_NFS_ReadStrided(ADIO_File fd, void *buf, int count,
 
 /* noncontiguous in memory, contiguous in file. */
 
-	ADIOI_Flatten_datatype(datatype);
+	flat_buf = ADIOI_Flatten_and_find(datatype);
-	flat_buf = ADIOI_Flatlist;
-	while (flat_buf->type != datatype) flat_buf = flat_buf->next;
 
         off = (file_ptr_type == ADIO_INDIVIDUAL) ? fd->fp_ind : 
                  fd->disp + etype_size * offset;
@@ -452,9 +461,7 @@ void ADIOI_NFS_ReadStrided(ADIO_File fd, void *buf, int count,
 /* noncontiguous in memory as well as in file */
 
             ADIO_Offset i;
-	    ADIOI_Flatten_datatype(datatype);
+	    flat_buf = ADIOI_Flatten_and_find(datatype);
-	    flat_buf = ADIOI_Flatlist;
-	    while (flat_buf->type != datatype) flat_buf = flat_buf->next;
 
 	    k = num = buf_count = 0;
 	    i = flat_buf->indices[0];

ompi/mca/io/romio321/romio/adio/ad_nfs/ad_nfs_write.c

@@ -7,64 +7,76 @@
 
 #include "ad_nfs.h"
 #include "adio_extern.h"
-#ifdef HAVE_UNISTD_H
-#include <unistd.h>
-#endif
 
 void ADIOI_NFS_WriteContig(ADIO_File fd, const void *buf, int count,
                      MPI_Datatype datatype, int file_ptr_type,
 		     ADIO_Offset offset, ADIO_Status *status, int *error_code)
 {
-    ssize_t err=-1;
+    int err=-1;
     MPI_Count datatype_size, len;
-    ADIO_Offset bytes_xfered=0;
-    size_t wr_count;
     static char myname[] = "ADIOI_NFS_WRITECONTIG";
-    char *p;
 
     MPI_Type_size_x(datatype, &datatype_size);
-    len = datatype_size * (ADIO_Offset)count;
+    len = datatype_size * count;
 
-    if (file_ptr_type == ADIO_INDIVIDUAL) {
+    if (file_ptr_type == ADIO_EXPLICIT_OFFSET) {
-	offset = fd->fp_ind;
+	if (fd->fp_sys_posn != offset) {
+#ifdef ADIOI_MPE_LOGGING
+            MPE_Log_event( ADIOI_MPE_lseek_a, 0, NULL );
+#endif
+	    lseek(fd->fd_sys, offset, SEEK_SET);
+#ifdef ADIOI_MPE_LOGGING
+            MPE_Log_event( ADIOI_MPE_lseek_b, 0, NULL );
+#endif
         }
-
+	ADIOI_WRITE_LOCK(fd, offset, SEEK_SET, len);
-    p = (char *)buf;
-    while (bytes_xfered < len) {
 #ifdef ADIOI_MPE_LOGGING
         MPE_Log_event( ADIOI_MPE_write_a, 0, NULL );
 #endif
-        wr_count = len - bytes_xfered;
+	err = write(fd->fd_sys, buf, len);
-        /* work around FreeBSD and OS X defects*/
-        if (wr_count > INT_MAX)
-            wr_count = INT_MAX;
-
-        ADIOI_WRITE_LOCK(fd, offset+bytes_xfered, SEEK_SET, wr_count);
-        err = pwrite(fd->fd_sys, p, wr_count, offset+bytes_xfered);
-        /* --BEGIN ERROR HANDLING-- */
-        if (err == -1) {
-            *error_code = MPIO_Err_create_code(MPI_SUCCESS,
-        	    MPIR_ERR_RECOVERABLE,
-        	    myname, __LINE__, MPI_ERR_IO, "**io",
-        	    "**io %s", strerror(errno));
-            fd->fp_sys_posn = -1;
-            return;
-        }
-        /* --END ERROR HANDLING-- */
 #ifdef ADIOI_MPE_LOGGING
         MPE_Log_event( ADIOI_MPE_write_b, 0, NULL );
 #endif
-        ADIOI_UNLOCK(fd, offset+bytes_xfered, SEEK_SET, wr_count);
+	ADIOI_UNLOCK(fd, offset, SEEK_SET, len);
-        bytes_xfered += err;
+	fd->fp_sys_posn = offset + err;
-        p += err;
+	/* individual file pointer not updated */        
+    }
+    else { /* write from curr. location of ind. file pointer */
+	offset = fd->fp_ind;
+	if (fd->fp_sys_posn != fd->fp_ind) {
+#ifdef ADIOI_MPE_LOGGING
+            MPE_Log_event( ADIOI_MPE_lseek_a, 0, NULL );
+#endif
+	    lseek(fd->fd_sys, fd->fp_ind, SEEK_SET);
+#ifdef ADIOI_MPE_LOGGING
+            MPE_Log_event( ADIOI_MPE_lseek_b, 0, NULL );
+#endif
+        }
+	ADIOI_WRITE_LOCK(fd, offset, SEEK_SET, len);
+#ifdef ADIOI_MPE_LOGGING
+        MPE_Log_event( ADIOI_MPE_write_a, 0, NULL );
+#endif
+	err = write(fd->fd_sys, buf, len);
+#ifdef ADIOI_MPE_LOGGING
+        MPE_Log_event( ADIOI_MPE_write_b, 0, NULL );
+#endif
+	ADIOI_UNLOCK(fd, offset, SEEK_SET, len);
+	fd->fp_ind += err;
+	fd->fp_sys_posn = fd->fp_ind;
     }
 
-    if (file_ptr_type == ADIO_INDIVIDUAL) {
+    /* --BEGIN ERROR HANDLING-- */
-        fd->fp_ind += bytes_xfered;
+    if (err == -1) {
+	*error_code = MPIO_Err_create_code(MPI_SUCCESS, MPIR_ERR_RECOVERABLE,
+					   myname, __LINE__, MPI_ERR_IO,
+					   "**io",
+					   "**io %s", strerror(errno));
+	return;
     }
+    /* --END ERROR HANDLING-- */
 
 #ifdef HAVE_STATUS_SET_BYTES
-    MPIR_Status_set_bytes(status, datatype, bytes_xfered);
+    MPIR_Status_set_bytes(status, datatype, err);
 #endif
 
     *error_code = MPI_SUCCESS;
@@ -308,9 +320,7 @@ void ADIOI_NFS_WriteStrided(ADIO_File fd, const void *buf, int count,
 
 /* noncontiguous in memory, contiguous in file. */
 
-	ADIOI_Flatten_datatype(datatype);
+	flat_buf = ADIOI_Flatten_and_find(datatype);
-	flat_buf = ADIOI_Flatlist;
-	while (flat_buf->type != datatype) flat_buf = flat_buf->next;
 
         off = (file_ptr_type == ADIO_INDIVIDUAL) ? fd->fp_ind : 
                  fd->disp + etype_size * offset;
@@ -556,9 +566,7 @@ void ADIOI_NFS_WriteStrided(ADIO_File fd, const void *buf, int count,
 /* noncontiguous in memory as well as in file */
 
 	    ADIO_Offset i;
-	    ADIOI_Flatten_datatype(datatype);
+	    flat_buf = ADIOI_Flatten_and_find(datatype);
-	    flat_buf = ADIOI_Flatlist;
-	    while (flat_buf->type != datatype) flat_buf = flat_buf->next;
 
 	    k = num = buf_count = 0;
 	    i = flat_buf->indices[0];

ompi/mca/io/romio321/romio/adio/ad_ntfs/ad_ntfs.c

@@ -34,5 +34,7 @@ struct ADIOI_Fns_struct ADIO_NTFS_operations = {
     ADIOI_NTFS_Flush, /* Flush */
     ADIOI_NTFS_Resize, /* Resize */
     ADIOI_GEN_Delete, /* Delete */
-    ADIOI_NTFS_Feature /* Features */
+    ADIOI_NTFS_Feature, /* Features */
+    ADIOI_GEN_IreadStridedColl, /* IreadStridedColl */
+    ADIOI_GEN_IwriteStridedColl /* IwriteStridedColl */
 };