add support for PLFS filesystem

This commit was SVN r30370.

config/ompi_check_plfs.m4

@@ -0,0 +1,112 @@
+# -*- shell-script -*-
+#
+# Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
+#                         University Research and Technology
+#                         Corporation.  All rights reserved.
+# Copyright (c) 2004-2005 The University of Tennessee and The University
+#                         of Tennessee Research Foundation.  All rights
+#                         reserved.
+# Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, 
+#                         University of Stuttgart.  All rights reserved.
+# Copyright (c) 2004-2006 The Regents of the University of California.
+#                         All rights reserved.
+# Copyright (c) 2009      Cisco Systems, Inc.  All rights reserved.
+# Copyright (c) 2008-2012 University of Houston. All rights reserved.
+# $COPYRIGHT$
+# 
+# Additional copyrights may follow
+# 
+# $HEADER$
+#
+
+# OMPI_CHECK_PLFS(prefix, [action-if-found], [action-if-not-found])
+# --------------------------------------------------------
+# check if PLFS support can be found.  sets prefix_{CPPFLAGS, 
+# LDFLAGS, LIBS} as needed and runs action-if-found if there is
+# support, otherwise executes action-if-not-found
+AC_DEFUN([OMPI_CHECK_PLFS],[
+
+    check_plfs_CPPFLAGS=
+    check_plfs_LDFLAGS=
+    check_plfs_LIBS=
+
+    check_plfs_save_LIBS="$LIBS" 
+    check_plfs_save_LDFLAGS="$LDFLAGS"
+    check_plfs_save_CPPFLAGS="$CPPFLAGS"
+
+    check_plfs_configuration="none"
+    ompi_check_plfs_happy="yes"
+
+
+    # Get some configuration information
+    AC_ARG_WITH([plfs],
+        [AC_HELP_STRING([--with-plfs(=DIR)],
+             [Build Plfs support, optionally adding DIR/include, DIR/lib, and DIR/lib64 to the search path for headers and libraries])])
+    OMPI_CHECK_WITHDIR([plfs], [$with_plfs], [include/plfs.h])
+
+    AC_ARG_WITH([plfs-libs], 
+        [AC_HELP_STRING([--with-plfs-libs=LIBS],
+                       [Libraries to link with for plfs])])
+
+    temp_with_plfs="$with_plfs"
+    AS_IF([test -z "$with_plfs"],
+          [with_plfs="/usr/local"])
+
+    temp_with_plfs_libs="$with_plfs_libs"
+    AS_IF([test -z "$with_plfs_libs"],
+	[with_plfs_libs="plfs pthread"])
+    
+    # Add correct -I and -L flags
+    AS_IF([test -d "$with_plfs/include"],
+        [check_plfs_CPPFLAGS="-I$with_plfs/include"
+            $1_CPPFLAGS="$check_plfs_CPPFLAGS"
+            CPPFLAGS="$CPPFLAGS $check_plfs_CPPFLAGS"],
+	[ompi_check_plfs_happy="no"])
+    
+    AS_IF([test "$ompi_check_plfs_happy" = "yes"],
+	[AS_IF([test -d "$with_plfs/lib"],
+		[check_plfs_LDFLAGS="-L$with_plfs/lib"
+		    $1_LDFLAGS="$check_plfs_LDFLAGS"
+		    LDFLAGS="$LDFLAGS $check_plfs_LDFLAGS"],
+		[ompi_check_plfs_happy="no"]) 
+    ],[])
+	    
+    # Try to find all the plfs libraries
+    AS_IF([test "$ompi_check_plfs_happy" = "yes"],
+	[ AS_IF([test -n "$with_plfs_libs"]
+		[for lib in $with_plfs_libs ; do
+		    check_plfs_LIBS="$check_plfs_LIBS -l$lib"
+		    done]) 
+		
+	    $1_LIBS="$check_plfs_LIBS"
+	    LIBS="$LIBS $check_plfs_LIBS"
+
+            # check for plfs
+	    AC_CHECK_HEADERS([plfs.h],
+		[AC_MSG_CHECKING([if possible to link PLFS])
+		    AC_LINK_IFELSE(
+			[AC_LANG_PROGRAM(
+				[[#include <plfs.h>]],
+				[[is_plfs_path(NULL);]])],
+			[AC_MSG_RESULT([yes])
+			    ompi_check_plfs_happy="yes"],
+			[AC_MSG_RESULT([no])
+			    ompi_check_plfs_happy="no"])],
+		[ompi_check_plfs_happy="no"])
+    ])
+
+
+    LDFLAGS="$check_plfs_save_LDFLAGS"
+    CPPFLAGS="$check_plfs_save_CPPFLAGS"
+    LIBS="$check_plfs_save_LIBS"
+    AS_IF([test "$ompi_check_plfs_happy" = "yes"],
+          [$2],
+          [AS_IF([test ! -z "$with_plfs" -a "$with_plfs" != "no"],
+                  [echo PLFS support not found])
+              $3])
+
+    with_plfs="$temp_with_plfs"
+    with_plfs_libs="$temp_with_plfs_libs"
+
+])
+

ompi/mca/fbtl/plfs/Makefile.am

@@ -0,0 +1,54 @@
+#
+# Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
+#                         University Research and Technology
+#                         Corporation.  All rights reserved.
+# Copyright (c) 2004-2005 The University of Tennessee and The University
+#                         of Tennessee Research Foundation.  All rights
+#                         reserved.
+# Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, 
+#                         University of Stuttgart.  All rights reserved.
+# Copyright (c) 2004-2005 The Regents of the University of California.
+#                         All rights reserved.
+# Copyright (c) 2008-2011 University of Houston. All rights reserved.
+# $COPYRIGHT$
+# 
+# Additional copyrights may follow
+# 
+# $HEADER$
+#
+
+# Make the output library in this directory, and name it either
+# mca_<type>_<name>.la (for DSO builds) or libmca_<type>_<name>.la
+# (for static builds).
+
+if MCA_BUILD_ompi_fbtl_plfs_DSO
+component_noinst =
+component_install = mca_fbtl_plfs.la
+else
+component_noinst = libmca_fbtl_plfs.la
+component_install =
+endif
+
+# Source files
+
+fbtl_plfs_sources = \
+        fbtl_plfs.h \
+        fbtl_plfs.c \
+        fbtl_plfs_component.c \
+        fbtl_plfs_preadv.c \
+        fbtl_plfs_ipreadv.c \
+        fbtl_plfs_pwritev.c \
+        fbtl_plfs_ipwritev.c
+
+AM_CPPFLAGS = $(fbtl_plfs_CPPFLAGS)
+
+mcacomponentdir = $(pkglibdir)
+mcacomponent_LTLIBRARIES = $(component_install)
+mca_fbtl_plfs_la_SOURCES = $(fbtl_plfs_sources)
+mca_fbtl_plfs_la_LIBADD = $(fbtl_plfs_LIBS)
+mca_fbtl_plfs_la_LDFLAGS = -module -avoid-version $(fbtl_plfs_LDFLAGS)
+
+noinst_LTLIBRARIES = $(component_noinst)
+libmca_fbtl_plfs_la_SOURCES = $(fbtl_plfs_sources)
+libmca_fbtl_plfs_la_LIBADD = $(fbtl_plfs_LIBS)
+libmca_fbtl_plfs_la_LDFLAGS = -module -avoid-version $(fbtl_plfs_LDFLAGS)

ompi/mca/fbtl/plfs/configure.m4

@@ -0,0 +1,42 @@
+# -*- shell-script -*-
+#
+# Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
+#                         University Research and Technology
+#                         Corporation.  All rights reserved.
+# Copyright (c) 2004-2005 The University of Tennessee and The University
+#                         of Tennessee Research Foundation.  All rights
+#                         reserved.
+# Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, 
+#                         University of Stuttgart.  All rights reserved.
+# Copyright (c) 2004-2005 The Regents of the University of California.
+#                         All rights reserved.
+# Copyright (c) 2010      Cisco Systems, Inc.  All rights reserved.
+# Copyright (c) 2008-2014 University of Houston. All rights reserved.
+# $COPYRIGHT$
+# 
+# Additional copyrights may follow
+# 
+# $HEADER$
+#
+
+
+# MCA_fbtl_plfs_CONFIG(action-if-can-compile, 
+#                        [action-if-cant-compile])
+# ------------------------------------------------
+AC_DEFUN([MCA_ompi_fbtl_plfs_CONFIG],[
+    AC_CONFIG_FILES([ompi/mca/fbtl/plfs/Makefile])
+
+    OMPI_CHECK_PLFS([fbtl_plfs],
+                     [fbtl_plfs_happy="yes"],
+                     [fbtl_plfs_happy="no"])
+
+    AS_IF([test "$fbtl_plfs_happy" = "yes"],
+          [$1],
+          [$2])
+
+
+    # substitute in the things needed to build plfs
+    AC_SUBST([fbtl_plfs_CPPFLAGS])
+    AC_SUBST([fbtl_plfs_LDFLAGS])
+    AC_SUBST([fbtl_plfs_LIBS])
+])dnl

ompi/mca/fbtl/plfs/fbtl_plfs.c

@@ -0,0 +1,83 @@
+/*
+ * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2006 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, 
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2008-2014 University of Houston. All rights reserved.
+ * $COPYRIGHT$
+ * 
+ * Additional copyrights may follow
+ * 
+ * $HEADER$
+ *
+ * These symbols are in a file by themselves to provide nice linker
+ * semantics. Since linkers generally pull in symbols by object fules,
+ * keeping these symbols as the only symbols in this file prevents
+ * utility programs such as "ompi_info" from having to import entire
+ * modules just to query their version and parameters
+ */
+
+#include "ompi_config.h"
+#include "mpi.h"
+#include "ompi/mca/fbtl/fbtl.h"
+#include "ompi/mca/fbtl/plfs/fbtl_plfs.h"
+
+/*
+ * *******************************************************************
+ * ************************ actions structure ************************
+ * *******************************************************************
+ */
+static mca_fbtl_base_module_1_0_0_t plfs =  {
+    mca_fbtl_plfs_module_init, /* initalise after being selected */
+    mca_fbtl_plfs_module_finalize, /* close a module on a communicator */
+    mca_fbtl_plfs_preadv,
+    mca_fbtl_plfs_ipreadv,
+    mca_fbtl_plfs_pwritev,
+    mca_fbtl_plfs_ipwritev
+};
+/*
+ * *******************************************************************
+ * ************************* structure ends **************************
+ * *******************************************************************
+ */
+
+int mca_fbtl_plfs_component_init_query(bool enable_progress_threads,
+                                      bool enable_mpi_threads) {
+    /* Nothing to do */
+    return OMPI_SUCCESS;
+}      
+
+struct mca_fbtl_base_module_1_0_0_t *
+mca_fbtl_plfs_component_file_query (mca_io_ompio_file_t *fh, int *priority) {
+   *priority = mca_fbtl_plfs_priority;
+
+   if (PLFS == fh->f_fstype) {
+       if (*priority < 50) {
+           *priority = 50;
+       }
+   }
+   return &plfs;
+}
+
+int mca_fbtl_plfs_component_file_unquery (mca_io_ompio_file_t *file) {    
+   /* This function might be needed for some purposes later. for now it
+    * does not have anything to do since there are no steps which need 
+    * to be undone if this module is not selected */
+
+   return OMPI_SUCCESS;
+}
+
+int mca_fbtl_plfs_module_init (mca_io_ompio_file_t *file) {
+    return OMPI_SUCCESS;
+}
+
+   
+int mca_fbtl_plfs_module_finalize (mca_io_ompio_file_t *file) {
+    return OMPI_SUCCESS;
+}

ompi/mca/fbtl/plfs/fbtl_plfs.h

@@ -0,0 +1,68 @@
+/*
+ * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2006 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, 
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2008-2014 University of Houston. All rights reserved.
+ * $COPYRIGHT$
+ * 
+ * Additional copyrights may follow
+ * 
+ * $HEADER$
+ */
+
+#ifndef MCA_FBTL_PLFS_H
+#define MCA_FBTL_PLFS_H
+
+#include "ompi_config.h"
+#include "opal/mca/mca.h"
+#include "ompi/mca/fbtl/fbtl.h"
+#include "ompi/mca/io/ompio/io_ompio.h"
+#include <plfs.h>
+
+extern int mca_fbtl_plfs_priority;
+
+BEGIN_C_DECLS
+
+int mca_fbtl_plfs_component_init_query(bool enable_progress_threads,
+                                        bool enable_mpi_threads);
+struct mca_fbtl_base_module_1_0_0_t *
+mca_fbtl_plfs_component_file_query (mca_io_ompio_file_t *file, int *priority);
+int mca_fbtl_plfs_component_file_unquery (mca_io_ompio_file_t *file);
+
+int mca_fbtl_plfs_module_init (mca_io_ompio_file_t *file);
+int mca_fbtl_plfs_module_finalize (mca_io_ompio_file_t *file);
+
+OMPI_MODULE_DECLSPEC extern mca_fbtl_base_component_2_0_0_t mca_fbtl_plfs_component;
+/*
+ * ******************************************************************
+ * ********* functions which are implemented in this module *********
+ * ******************************************************************
+ */ 
+
+size_t mca_fbtl_plfs_preadv (mca_io_ompio_file_t *file,
+                              int *sorted);
+size_t mca_fbtl_plfs_pwritev (mca_io_ompio_file_t *file,
+                               int *sorted);
+size_t mca_fbtl_plfs_ipreadv (mca_io_ompio_file_t *file,
+                               int *sorted,
+                               ompi_request_t **request);
+size_t mca_fbtl_plfs_ipwritev (mca_io_ompio_file_t *file,
+                                int *sorted,
+                                ompi_request_t **request);
+
+/*
+ * ******************************************************************
+ * ************ functions implemented in this module end ************
+ * ******************************************************************
+ */ 
+                                     
+END_C_DECLS
+
+#endif /* MCA_FBTL_PLFS_H */

ompi/mca/fbtl/plfs/fbtl_plfs_component.c

@@ -0,0 +1,65 @@
+/*
+ * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2005 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, 
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2008-2014 University of Houston. All rights reserved.
+ * $COPYRIGHT$
+ * 
+ * Additional copyrights may follow
+ * 
+ * $HEADER$
+ *
+ * These symbols are in a file by themselves to provide nice linker
+ * semantics.  Since linkers generally pull in symbols by object
+ * files, keeping these symbols as the only symbols in this file
+ * prevents utility programs such as "ompi_info" from having to import
+ * entire components just to query their version and parameters.
+ */
+
+#include "ompi_config.h"
+#include "fbtl_plfs.h"
+#include "mpi.h"
+
+/*
+ * Public string showing the fbtl plfs component version number
+ */
+const char *mca_fbtl_plfs_component_version_string =
+  "OMPI/MPI plfs FBTL MCA component version " OMPI_VERSION;
+
+int mca_fbtl_plfs_priority = 10;
+
+/*
+ * Instantiate the public struct with all of our public information
+ * and pointers to our public functions in it
+ */
+mca_fbtl_base_component_2_0_0_t mca_fbtl_plfs_component = {
+
+    /* First, the mca_component_t struct containing meta information
+       about the component itself */
+
+    {
+        MCA_FBTL_BASE_VERSION_2_0_0,
+
+        /* Component name and version */
+        "plfs",
+        OMPI_MAJOR_VERSION,
+        OMPI_MINOR_VERSION,
+        OMPI_RELEASE_VERSION,
+        NULL,
+        NULL
+    },
+    {
+        /* This component is checkpointable */
+      MCA_BASE_METADATA_PARAM_CHECKPOINT
+    },
+    mca_fbtl_plfs_component_init_query,      /* get thread level */
+    mca_fbtl_plfs_component_file_query,      /* get priority and actions */
+    mca_fbtl_plfs_component_file_unquery     /* undo what was done by previous function */
+};

ompi/mca/fbtl/plfs/fbtl_plfs_ipreadv.c

@@ -0,0 +1,35 @@
+/*
+ * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2011 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, 
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2008-2014 University of Houston. All rights reserved.
+ * $COPYRIGHT$
+ * 
+ * Additional copyrights may follow
+ * 
+ * $HEADER$
+ */
+
+
+#include "ompi_config.h"
+#include "fbtl_plfs.h"
+
+#include "mpi.h"
+#include "ompi/constants.h"
+#include "ompi/mca/fbtl/fbtl.h"
+
+size_t 
+mca_fbtl_plfs_ipreadv (mca_io_ompio_file_t *file,
+                        int *sorted,
+                        ompi_request_t **request)
+{
+    printf ("PLFS IPREADV\n");
+    return OMPI_ERROR;
+}

ompi/mca/fbtl/plfs/fbtl_plfs_ipwritev.c

@@ -0,0 +1,35 @@
+/*
+ * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2011 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, 
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2008-2014 University of Houston. All rights reserved.
+ * $COPYRIGHT$
+ * 
+ * Additional copyrights may follow
+ * 
+ * $HEADER$
+ */
+
+#include "ompi_config.h"
+#include "fbtl_plfs.h"
+
+#include "mpi.h"
+#include <unistd.h>
+#include "ompi/constants.h"
+#include "ompi/mca/fbtl/fbtl.h"
+
+size_t 
+mca_fbtl_plfs_ipwritev (mca_io_ompio_file_t *fh,
+                         int *sorted,
+                         ompi_request_t **request)
+{
+    printf ("PLFS IPWRITEV\n");
+    return OMPI_ERROR;
+}

ompi/mca/fbtl/plfs/fbtl_plfs_preadv.c

@@ -0,0 +1,201 @@
+/*
+ * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2011 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, 
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2008-2014 University of Houston. All rights reserved.
+ * $COPYRIGHT$
+ * 
+ * Additional copyrights may follow
+ * 
+ * $HEADER$
+ */
+
+#include "ompi_config.h"
+#include "fbtl_plfs.h"
+
+#include "mpi.h"
+#include <unistd.h>
+#include "ompi/constants.h"
+#include "ompi/mca/fbtl/fbtl.h"
+
+size_t 
+mca_fbtl_plfs_preadv (mca_io_ompio_file_t *fh,
+                       int *sorted)
+{
+
+    Plfs_fd *pfd = NULL;
+    plfs_error_t plfs_ret;
+    pfd = fh->f_fs_ptr;
+
+    int i, block=1;
+    struct iovec *iov = NULL;
+    int iov_count = 0;
+    OMPI_MPI_OFFSET_TYPE iov_offset = 0;
+
+    if (NULL == fh->f_io_array) {
+        return OMPI_ERROR;
+    }
+
+    iov = (struct iovec *) malloc 
+        (OMPIO_IOVEC_INITIAL_SIZE * sizeof (struct iovec));
+    if (NULL == iov) {
+        opal_output(1, "OUT OF MEMORY\n");
+        return OMPI_ERR_OUT_OF_RESOURCE;
+    }
+
+    if (NULL != sorted) {
+        for (i=0 ; i<fh->f_num_of_io_entries ; i++) {
+            if (0 == iov_count) {
+                iov[iov_count].iov_base = fh->f_io_array[sorted[i]].memory_address;
+                iov[iov_count].iov_len = fh->f_io_array[sorted[i]].length;
+                iov_offset = (OMPI_MPI_OFFSET_TYPE)(intptr_t)fh->f_io_array[sorted[i]].offset;
+                iov_count ++;
+            }
+
+            if (OMPIO_IOVEC_INITIAL_SIZE*block <= iov_count) {
+                block ++;
+                iov = (struct iovec *)realloc
+                    (iov, OMPIO_IOVEC_INITIAL_SIZE * block *
+                     sizeof(struct iovec));
+                if (NULL == iov) {
+                    opal_output(1, "OUT OF MEMORY\n");
+                    return OMPI_ERR_OUT_OF_RESOURCE;
+                }
+            }
+
+            if (fh->f_num_of_io_entries != i+1) {
+	      if (((OMPI_MPI_OFFSET_TYPE)(intptr_t)fh->f_io_array[sorted[i]].offset + 
+                     (OPAL_PTRDIFF_TYPE)fh->f_io_array[sorted[i]].length) == 
+		  (OMPI_MPI_OFFSET_TYPE)(intptr_t)fh->f_io_array[sorted[i+1]].offset) {                    
+                    iov[iov_count].iov_base = 
+                        fh->f_io_array[sorted[i+1]].memory_address;
+                    iov[iov_count].iov_len = fh->f_io_array[sorted[i+1]].length;
+                    iov_count ++;
+                    continue;
+                }
+            }
+
+	    // Find the total number of bytes to be read.
+	    size_t bytes = 0;
+	    for (int i = 0; i < iov_count; ++i) {
+		bytes += iov[i].iov_len;
+	    }
+
+	    // Allocate a temporary buffer to hold the data
+	    char *buffer;
+	    buffer = (char *) malloc (bytes);
+	    if (buffer == NULL) {
+	        return OMPI_ERROR;
+	    }
+
+	    // Read the data
+	    ssize_t bytes_read;
+
+	    plfs_ret = plfs_read( pfd, buffer, bytes, iov_offset, &bytes_read );
+	    if (PLFS_SUCCESS != plfs_ret) {
+	        opal_output(0, "fbtl_plfs_preadv: Error in plfs_read:\n%s\n", strplfserr(plfs_ret));
+		return OMPI_ERROR;
+	    }
+	    	      
+	    if (bytes_read < 0)
+	        return OMPI_ERROR;
+
+	    // Copy the data from BUFFER into the memory specified by IOV
+	    bytes = bytes_read;
+	    for (int i = 0; i < iov_count; ++i) {
+	        size_t copy = MIN (iov[i].iov_len, bytes);
+		(void) memcpy ((void *) iov[i].iov_base, (void *) buffer, copy);
+		buffer += copy;
+		bytes -= copy;
+		if (bytes == 0) {
+		    break;
+		}
+	    }
+	    iov_count = 0;
+        }
+    }
+
+    else {
+        for (i=0 ; i<fh->f_num_of_io_entries ; i++) {
+            if (0 == iov_count) {
+                iov[iov_count].iov_base = fh->f_io_array[i].memory_address;
+                iov[iov_count].iov_len = fh->f_io_array[i].length;
+                iov_offset = (OMPI_MPI_OFFSET_TYPE)(intptr_t)fh->f_io_array[i].offset;
+                iov_count ++;
+            }
+
+            if (OMPIO_IOVEC_INITIAL_SIZE*block <= iov_count) {
+                block ++;
+                iov = (struct iovec *)realloc
+                    (iov, OMPIO_IOVEC_INITIAL_SIZE * block *
+		     sizeof(struct iovec));
+                if (NULL == iov) {
+                    opal_output(1, "OUT OF MEMORY\n");
+                    return OMPI_ERR_OUT_OF_RESOURCE;
+                }
+            }
+
+            if (fh->f_num_of_io_entries != i+1) {
+	      if (((OMPI_MPI_OFFSET_TYPE)(intptr_t)fh->f_io_array[i].offset + 
+                     (OPAL_PTRDIFF_TYPE)fh->f_io_array[i].length) == 
+		  (OMPI_MPI_OFFSET_TYPE)(intptr_t)fh->f_io_array[i+1].offset) {                    
+                    iov[iov_count].iov_base = 
+                        fh->f_io_array[i+1].memory_address;
+                    iov[iov_count].iov_len = fh->f_io_array[i+1].length;
+                    iov_count ++;
+                    continue;
+                }
+            }
+
+	    // Find the total number of bytes to be read.
+	    size_t bytes = 0;
+	    for (int i = 0; i < iov_count; ++i) {
+		bytes += iov[i].iov_len;
+	    }
+
+	    // Allocate a temporary buffer to hold the data
+	    char *buffer;
+	    buffer = (char *) malloc (bytes);
+	    if (buffer == NULL) {
+	        return OMPI_ERROR;
+	    }
+
+	    // Read the data
+	    ssize_t bytes_read;
+	    plfs_ret = plfs_read( pfd, buffer, bytes, iov_offset, &bytes_read );
+	    if (PLFS_SUCCESS != plfs_ret) {
+	        opal_output(0, "fbtl_plfs_preadv: Error in plfs_read:\n%s\n", strplfserr(plfs_ret));
+		return OMPI_ERROR;
+	    }
+
+	    if (bytes_read < 0)
+	        return OMPI_ERROR;
+
+	    // Copy the data from BUFFER into the memory specified by IOV
+	    bytes = bytes_read;
+	    for (int i = 0; i < iov_count; ++i) {
+	        size_t copy = MIN (iov[i].iov_len, bytes);
+		(void) memcpy ((void *) iov[i].iov_base, (void *) buffer, copy);
+		buffer += copy;
+		bytes -= copy;
+		if (bytes == 0) {
+		    break;
+		}
+	    }
+	    iov_count = 0;
+	}
+    }
+    if (NULL != iov) {
+        free (iov);
+        iov = NULL;
+    }
+
+    return OMPI_SUCCESS;
+}

ompi/mca/fbtl/plfs/fbtl_plfs_pwritev.c

@@ -0,0 +1,197 @@
+/*
+ * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2011 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, 
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2008-2014 University of Houston. All rights reserved.
+ * $COPYRIGHT$
+ * 
+ * Additional copyrights may follow
+ * 
+ * $HEADER$
+ */
+
+
+#include "ompi_config.h"
+#include "fbtl_plfs.h"
+
+#include "mpi.h"
+#include <unistd.h>
+#include "ompi/constants.h"
+#include "ompi/mca/fbtl/fbtl.h"
+
+size_t 
+mca_fbtl_plfs_pwritev (mca_io_ompio_file_t *fh,
+                        int *sorted)
+{
+    Plfs_fd *pfd = NULL;
+    plfs_error_t plfs_ret;
+    pfd = fh->f_fs_ptr;
+
+    int i, block = 1;
+    struct iovec *iov = NULL;
+    int iov_count = 0;
+    OMPI_MPI_OFFSET_TYPE iov_offset = 0;
+
+    if (NULL == fh->f_io_array) {
+        return OMPI_ERROR;
+    }
+
+    iov = (struct iovec *) malloc 
+        (OMPIO_IOVEC_INITIAL_SIZE * sizeof (struct iovec));
+    if (NULL == iov) {
+        opal_output(1, "OUT OF MEMORY\n");
+        return OMPI_ERR_OUT_OF_RESOURCE;
+    }
+
+    if (NULL != sorted) {
+        for (i=0 ; i<fh->f_num_of_io_entries ; i++) {
+            if (0 == iov_count) {
+                iov[iov_count].iov_base = fh->f_io_array[sorted[i]].memory_address;
+                iov[iov_count].iov_len = fh->f_io_array[sorted[i]].length;
+                iov_offset = (OMPI_MPI_OFFSET_TYPE)(intptr_t)fh->f_io_array[sorted[i]].offset;
+                iov_count ++;
+            }
+
+            if (OMPIO_IOVEC_INITIAL_SIZE*block <= iov_count) {
+                block ++;
+                iov = (struct iovec *)realloc
+                    (iov, OMPIO_IOVEC_INITIAL_SIZE * block *
+                     sizeof(struct iovec));
+                if (NULL == iov) {
+                    opal_output(1, "OUT OF MEMORY\n");
+                    return OMPI_ERR_OUT_OF_RESOURCE;
+                }
+            }
+
+            if (fh->f_num_of_io_entries != i+1) {
+	      if (((OMPI_MPI_OFFSET_TYPE)(intptr_t)fh->f_io_array[sorted[i]].offset + 
+                     (OPAL_PTRDIFF_TYPE)fh->f_io_array[sorted[i]].length) == 
+		  (OMPI_MPI_OFFSET_TYPE)(intptr_t)fh->f_io_array[sorted[i+1]].offset) {
+                    iov[iov_count].iov_base = 
+                        fh->f_io_array[sorted[i+1]].memory_address;
+                    iov[iov_count].iov_len = fh->f_io_array[sorted[i+1]].length;
+                    iov_count ++;
+                    continue;
+                }
+            }
+
+	    // Find the total number of bytes to be written.
+	    size_t bytes = 0;
+	    for (int i = 0; i < iov_count; ++i) {
+		bytes += iov[i].iov_len;
+	    }
+
+	    // Allocate a temporary buffer to hold the data
+	    char *buffer;
+	    buffer = (char *) malloc (bytes);
+	    if (buffer == NULL) {
+	        return OMPI_ERROR;
+	    }
+
+	    // Copy the data into BUFFER.
+	    size_t to_copy = bytes;
+	    char *bp = buffer;
+	    for (int i = 0; i < iov_count; ++i) {
+		size_t copy = MIN (iov[i].iov_len, to_copy);
+
+		bp = mempcpy ((void *) bp, (void *) iov[i].iov_base, copy);
+
+		to_copy -= copy;
+		if (to_copy == 0)
+		  break;
+	    }
+	    
+	    // Write the data
+	    ssize_t bytes_written;
+
+	    plfs_ret = plfs_write( pfd, buffer, bytes, iov_offset, 0, &bytes_written );
+	    if (PLFS_SUCCESS != plfs_ret) {
+	        opal_output(0, "fbtl_plfs_pwritev: Error in plfs_write:\n%s\n", strplfserr(plfs_ret));
+		return OMPI_ERROR;
+	    }
+	    iov_count = 0;
+	}
+    }
+    else {
+        for (i=0 ; i<fh->f_num_of_io_entries ; i++) {
+            if (0 == iov_count) {
+                iov[iov_count].iov_base = fh->f_io_array[i].memory_address;
+                iov[iov_count].iov_len = fh->f_io_array[i].length;
+                iov_offset = (OMPI_MPI_OFFSET_TYPE)(intptr_t)fh->f_io_array[i].offset;
+                iov_count ++;
+            }
+
+            if (OMPIO_IOVEC_INITIAL_SIZE*block <= iov_count) {
+                block ++;
+                iov = (struct iovec *)realloc
+                    (iov, OMPIO_IOVEC_INITIAL_SIZE * block *
+                     sizeof(struct iovec));
+                if (NULL == iov) {
+                    opal_output(1, "OUT OF MEMORY\n");
+                    return OMPI_ERR_OUT_OF_RESOURCE;
+                }
+            }
+
+            if (fh->f_num_of_io_entries != i+1) {
+	      if (((OMPI_MPI_OFFSET_TYPE)(intptr_t)fh->f_io_array[i].offset + 
+                     (OPAL_PTRDIFF_TYPE)fh->f_io_array[i].length) == 
+		  (OMPI_MPI_OFFSET_TYPE)(intptr_t)fh->f_io_array[i+1].offset) {
+                    iov[iov_count].iov_base = 
+                        fh->f_io_array[i+1].memory_address;
+                    iov[iov_count].iov_len = fh->f_io_array[i+1].length;
+                    iov_count ++;
+                    continue;
+                }
+            }
+
+	    // Find the total number of bytes to be written.
+	    size_t bytes = 0;
+	    for (int i = 0; i < iov_count; ++i) {
+	        bytes += iov[i].iov_len;
+	    }
+
+	    // Allocate a temporary buffer to hold the data
+	    char *buffer;
+	    buffer = (char *) malloc (bytes);
+	    if (buffer == NULL) {
+	        return OMPI_ERROR;
+	    }
+
+	    // Copy the data into BUFFER.
+	    size_t to_copy = bytes;
+	    char *bp = buffer;
+	    for (int i = 0; i < iov_count; ++i) {
+		size_t copy = MIN (iov[i].iov_len, to_copy);
+		bp = mempcpy ((void *) bp, (void *) iov[i].iov_base, copy);
+		to_copy -= copy;
+		if (to_copy == 0) {
+		  break;
+		}
+	    }
+	    
+	    // Write the data
+	    ssize_t bytes_written;
+
+	    plfs_ret = plfs_write( pfd, buffer, bytes, iov_offset, 0, &bytes_written );
+	    if (PLFS_SUCCESS != plfs_ret) {
+	        opal_output(0, "fbtl_plfs_pwritev: Error in plfs_write:\n%s\n", strplfserr(plfs_ret));
+		return OMPI_ERROR;
+	    }
+	    iov_count = 0;
+	}
+    }
+
+    if (NULL != iov) {
+        free (iov);
+        iov = NULL;
+    }
+
+    return OMPI_SUCCESS;
+}

ompi/mca/fs/plfs/Makefile.am

@@ -0,0 +1,57 @@
+#
+# Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
+#                         University Research and Technology
+#                         Corporation.  All rights reserved.
+# Copyright (c) 2004-2005 The University of Tennessee and The University
+#                         of Tennessee Research Foundation.  All rights
+#                         reserved.
+# Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, 
+#                         University of Stuttgart.  All rights reserved.
+# Copyright (c) 2004-2005 The Regents of the University of California.
+#                         All rights reserved.
+# Copyright (c) 2008-2014 University of Houston. All rights reserved.
+# $COPYRIGHT$
+# 
+# Additional copyrights may follow
+# 
+# $HEADER$
+#
+
+# Make the output library in this directory, and name it either
+# mca_<type>_<name>.la (for DSO builds) or libmca_<type>_<name>.la
+# (for static builds).
+
+if MCA_BUILD_ompi_fs_plfs_DSO
+component_noinst =
+component_install = mca_fs_plfs.la
+else
+component_noinst = libmca_fs_plfs.la
+component_install =
+endif
+
+# Source files
+
+fs_plfs_sources = \
+        fs_plfs.h \
+        fs_plfs.c \
+        fs_plfs_component.c \
+        fs_plfs_file_open.c \
+	fs_plfs_file_close.c \
+	fs_plfs_file_delete.c \
+	fs_plfs_file_sync.c \
+	fs_plfs_file_set_size.c \
+        fs_plfs_file_get_size.c \
+        fs_plfs_file_set_info.c
+
+AM_CPPFLAGS = $(fs_plfs_CPPFLAGS)
+
+mcacomponentdir = $(pkglibdir)
+mcacomponent_LTLIBRARIES = $(component_install)
+mca_fs_plfs_la_SOURCES = $(fs_plfs_sources)
+mca_fs_plfs_la_LIBADD = $(fs_plfs_LIBS)
+mca_fs_plfs_la_LDFLAGS = -module -avoid-version $(fs_plfs_LDFLAGS)
+
+noinst_LTLIBRARIES = $(component_noinst)
+libmca_fs_plfs_la_SOURCES = $(fs_plfs_sources)
+libmca_fs_plfs_la_LIBADD = $(fs_plfs_LIBS)
+libmca_fs_plfs_la_LDFLAGS = -module -avoid-version $(fs_plfs_LDFLAGS)

ompi/mca/fs/plfs/configure.m4

@@ -0,0 +1,41 @@
+# -*- shell-script -*-
+#
+# Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
+#                         University Research and Technology
+#                         Corporation.  All rights reserved.
+# Copyright (c) 2004-2005 The University of Tennessee and The University
+#                         of Tennessee Research Foundation.  All rights
+#                         reserved.
+# Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, 
+#                         University of Stuttgart.  All rights reserved.
+# Copyright (c) 2004-2005 The Regents of the University of California.
+#                         All rights reserved.
+# Copyright (c) 2010      Cisco Systems, Inc.  All rights reserved.
+# Copyright (c) 2008-2014 University of Houston. All rights reserved.
+# $COPYRIGHT$
+# 
+# Additional copyrights may follow
+# 
+# $HEADER$
+#
+
+
+# MCA_fs_plfs_CONFIG(action-if-can-compile, 
+#                        [action-if-cant-compile])
+# ------------------------------------------------
+AC_DEFUN([MCA_ompi_fs_plfs_CONFIG],[
+    AC_CONFIG_FILES([ompi/mca/fs/plfs/Makefile])
+
+    OMPI_CHECK_PLFS([fs_plfs],
+                     [fs_plfs_happy="yes"],
+                     [fs_plfs_happy="no"])
+
+    AS_IF([test "$fs_plfs_happy" = "yes"],
+          [$1],
+          [$2])
+
+    # substitute in the things needed to build plfs
+    AC_SUBST([fs_plfs_CPPFLAGS])
+    AC_SUBST([fs_plfs_LDFLAGS])
+    AC_SUBST([fs_plfs_LIBS])
+])dnl

ompi/mca/fs/plfs/fs_plfs.c

@@ -0,0 +1,152 @@
+/*
+ * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2006 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, 
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2008-2014 University of Houston. All rights reserved.
+ * $COPYRIGHT$
+ * 
+ * Additional copyrights may follow
+ * 
+ * $HEADER$
+ *
+ * These symbols are in a file by themselves to provide nice linker
+ * semantics. Since linkers generally pull in symbols by object fules,
+ * keeping these symbols as the only symbols in this file prevents
+ * utility programs such as "ompi_info" from having to import entire
+ * modules just to query their version and parameters
+ */
+
+#include "ompi_config.h"
+#include "mpi.h"
+#include "ompi/mca/fs/fs.h"
+#include "ompi/mca/fs/base/base.h"
+#include "ompi/mca/fs/plfs/fs_plfs.h"
+
+#ifdef HAVE_SYS_STATFS_H
+#include <sys/statfs.h> /* or <sys/vfs.h> */ 
+#endif
+#ifdef HAVE_SYS_PARAM_H
+#include <sys/param.h>
+#endif
+#ifdef HAVE_SYS_MOUNT_H
+#include <sys/mount.h>
+#endif
+#ifdef HAVE_SYS_STAT_H
+#include <sys/stat.h>
+#endif
+
+#include <sys/ioctl.h>
+#include <plfs.h>
+
+/*
+ * *******************************************************************
+ * ************************ actions structure ************************
+ * *******************************************************************
+ */
+static mca_fs_base_module_1_0_0_t plfs =  {
+    mca_fs_plfs_module_init, /* initalise after being selected */
+    mca_fs_plfs_module_finalize, /* close a module on a communicator */
+    mca_fs_plfs_file_open,
+    mca_fs_plfs_file_close,
+    mca_fs_plfs_file_delete,
+    mca_fs_plfs_file_set_size,
+    mca_fs_plfs_file_get_size,
+    mca_fs_plfs_file_set_info,
+    mca_fs_plfs_file_sync
+};
+/*
+ * *******************************************************************
+ * ************************* structure ends **************************
+ * *******************************************************************
+ */
+
+int mca_fs_plfs_component_init_query(bool enable_progress_threads,
+                                      bool enable_mpi_threads)
+{
+    /* Nothing to do */
+    return OMPI_SUCCESS;
+}      
+
+struct mca_fs_base_module_1_0_0_t *
+mca_fs_plfs_component_file_query (mca_io_ompio_file_t *fh, int *priority)
+{
+    int err;
+    char *dir;
+    struct statfs fsbuf;
+    char *tmp;
+    char wpath[1024];
+
+    /* The code in this function is based on the ADIO FS selection in ROMIO
+     *   Copyright (C) 1997 University of Chicago. 
+     *   See COPYRIGHT notice in top-level directory.
+     */
+
+    *priority = mca_fs_plfs_priority;
+    
+    tmp = strchr (fh->f_filename, ':');
+    if (!tmp) {
+        if (OMPIO_ROOT == fh->f_rank) {
+            do {
+                err = statfs (fh->f_filename, &fsbuf);
+            } while (err && (errno == ESTALE));
+            
+            if (err && (ENOENT == errno)) {
+                mca_fs_base_get_parent_dir (fh->f_filename, &dir);
+                err = statfs (dir, &fsbuf);
+                free (dir);
+            }
+  
+	    getcwd( wpath, sizeof(wpath) );  
+	    if(is_plfs_path(wpath) == 1) {  
+	        fh->f_fstype = PLFS;
+            }
+	}
+	fh->f_comm->c_coll.coll_bcast (&(fh->f_fstype),
+				       1,
+				       MPI_INT,
+				       OMPIO_ROOT,
+				       fh->f_comm,
+				       fh->f_comm->c_coll.coll_bcast_module);
+    }
+    else {
+	if (!strncmp(fh->f_filename, "plfs:", 7) || 
+	    !strncmp(fh->f_filename, "PLFS:", 7)) {
+            fh->f_fstype = PLFS;
+        }
+    }
+    
+    if (PLFS == fh->f_fstype) {
+        if (*priority < 50) {
+	    *priority = 50;
+	    return &plfs;
+       }
+   }
+   return NULL;
+}
+
+int mca_fs_plfs_component_file_unquery (mca_io_ompio_file_t *file)
+{    
+   /* This function might be needed for some purposes later. for now it
+    * does not have anything to do since there are no steps which need 
+    * to be undone if this module is not selected */
+
+   return OMPI_SUCCESS;
+}
+
+int mca_fs_plfs_module_init (mca_io_ompio_file_t *file)
+{
+    return OMPI_SUCCESS;
+}
+
+   
+int mca_fs_plfs_module_finalize (mca_io_ompio_file_t *file) 
+{
+    return OMPI_SUCCESS;
+}

ompi/mca/fs/plfs/fs_plfs.h

@@ -0,0 +1,84 @@
+/*
+ * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2006 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, 
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2008-2014 University of Houston. All rights reserved.
+ * $COPYRIGHT$
+ * 
+ * Additional copyrights may follow
+ * 
+ * $HEADER$
+ */
+
+#ifndef MCA_FS_PLFS_H
+#define MCA_FS_PLFS_H
+
+#include "ompi_config.h"
+#include "opal/mca/mca.h"
+#include "ompi/mca/fs/fs.h"
+#include "ompi/mca/io/ompio/io_ompio.h"
+
+#include <plfs.h>
+
+extern int mca_fs_plfs_priority;
+extern int mca_fs_plfs_num_hostdir;
+
+BEGIN_C_DECLS
+
+int mca_fs_plfs_component_init_query(bool enable_progress_threads,
+                                        bool enable_mpi_threads);
+struct mca_fs_base_module_1_0_0_t *
+mca_fs_plfs_component_file_query (mca_io_ompio_file_t *fh, int *priority);
+int mca_fs_plfs_component_file_unquery (mca_io_ompio_file_t *file);
+
+int mca_fs_plfs_module_init (mca_io_ompio_file_t *file);
+int mca_fs_plfs_module_finalize (mca_io_ompio_file_t *file);
+
+OMPI_MODULE_DECLSPEC extern mca_fs_base_component_2_0_0_t mca_fs_plfs_component;
+/*
+ * ******************************************************************
+ * ********* functions which are implemented in this module *********
+ * ******************************************************************
+ */ 
+
+int mca_fs_plfs_file_open (struct ompi_communicator_t *comm, 
+                             char *filename, 
+                             int amode,
+                             struct ompi_info_t *info, 
+                             mca_io_ompio_file_t *fh);
+
+int mca_fs_plfs_file_close (mca_io_ompio_file_t *fh);
+    
+int mca_fs_plfs_file_delete (char *filename,
+                               struct ompi_info_t *info);
+
+int mca_fs_plfs_file_set_size (mca_io_ompio_file_t *fh,
+                                 OMPI_MPI_OFFSET_TYPE size);
+
+int mca_fs_plfs_file_get_size (mca_io_ompio_file_t *fh, 
+                                 OMPI_MPI_OFFSET_TYPE *size);
+
+int mca_fs_plfs_file_set_info (mca_io_ompio_file_t *fh, 
+                                 struct ompi_info_t *info);
+
+int mca_fs_plfs_file_sync (mca_io_ompio_file_t *fh);
+
+int mca_fs_plfs_file_seek (mca_io_ompio_file_t *fh,
+                             OMPI_MPI_OFFSET_TYPE offset,
+                             int whence);
+/*
+ * ******************************************************************
+ * ************ functions implemented in this module end ************
+ * ******************************************************************
+ */ 
+                                     
+END_C_DECLS
+
+#endif /* MCA_FS_PLFS_H */

ompi/mca/fs/plfs/fs_plfs_component.c

@@ -0,0 +1,86 @@
+/*
+ * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2005 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, 
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2008-2014 University of Houston. All rights reserved.
+ * $COPYRIGHT$
+ * 
+ * Additional copyrights may follow
+ * 
+ * $HEADER$
+ *
+ * These symbols are in a file by themselves to provide nice linker
+ * semantics.  Since linkers generally pull in symbols by object
+ * files, keeping these symbols as the only symbols in this file
+ * prevents utility programs such as "ompi_info" from having to import
+ * entire components just to query their version and parameters.
+ */
+
+#include "ompi_config.h"
+#include "fs_plfs.h"
+#include "mpi.h"
+
+/*
+ * Public string showing the fs plfs component version number
+ */
+const char *mca_fs_plfs_component_version_string =
+  "OMPI/MPI plfs FS MCA component version " OMPI_VERSION;
+
+static int plfs_register(void);
+
+int mca_fs_plfs_priority = 20;
+int mca_fs_plfs_num_hostdir = -1;
+
+/*
+ * Instantiate the public struct with all of our public information
+ * and pointers to our public functions in it
+ */
+mca_fs_base_component_2_0_0_t mca_fs_plfs_component = {
+
+    /* First, the mca_component_t struct containing meta information
+       about the component itself */
+
+    {
+        MCA_FS_BASE_VERSION_2_0_0,
+
+        /* Component name and version */
+        "plfs",
+        OMPI_MAJOR_VERSION,
+        OMPI_MINOR_VERSION,
+        OMPI_RELEASE_VERSION,
+        .mca_register_component_params = plfs_register,
+    },
+    {
+        /* This component is checkpointable */
+      MCA_BASE_METADATA_PARAM_CHECKPOINT
+    },
+    mca_fs_plfs_component_init_query,      /* get thread level */
+    mca_fs_plfs_component_file_query,      /* get priority and actions */
+    mca_fs_plfs_component_file_unquery     /* undo what was done by previous function */
+};
+
+static int
+plfs_register(void)
+{
+    mca_fs_plfs_priority = 20;
+    (void) mca_base_component_var_register(&mca_fs_plfs_component.fsm_version,
+                                           "priority", "Priority of the plfs fs component",
+                                           MCA_BASE_VAR_TYPE_INT, NULL, 0, 0,
+                                           OPAL_INFO_LVL_9,
+                                           MCA_BASE_VAR_SCOPE_READONLY, &mca_fs_plfs_priority);
+    mca_fs_plfs_num_hostdir = -1;
+    (void) mca_base_component_var_register(&mca_fs_plfs_component.fsm_version,
+                                           "num_hostdir", "number of host directories of a file over plfs",
+                                           MCA_BASE_VAR_TYPE_INT, NULL, 0, 0,
+                                           OPAL_INFO_LVL_9,
+                                           MCA_BASE_VAR_SCOPE_READONLY, &mca_fs_plfs_num_hostdir);
+
+    return OMPI_SUCCESS;
+}

ompi/mca/fs/plfs/fs_plfs_file_close.c

@@ -0,0 +1,75 @@
+/*
+ * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2011 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, 
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2008-2014 University of Houston. All rights reserved.
+ * $COPYRIGHT$
+ * 
+ * Additional copyrights may follow
+ * 
+ * $HEADER$
+ */
+
+
+#include "ompi_config.h"
+#include "fs_plfs.h"
+
+#include <fcntl.h>
+#include <unistd.h>
+#include "mpi.h"
+#include "ompi/constants.h"
+#include "ompi/mca/fs/fs.h"
+
+
+/*
+ *	file_close_plfs
+ *
+ *	Function:	- closes a new file
+ *	Accepts:	- file handle
+ *	Returns:	- Success if file closed
+ */
+int
+mca_fs_plfs_file_close (mca_io_ompio_file_t *fh)
+{
+    int flags;
+    plfs_error_t plfs_ret = PLFS_SUCCESS;
+    int amode;
+    char wpath[1024];
+
+    fh->f_comm->c_coll.coll_barrier (fh->f_comm, 
+                                     fh->f_comm->c_coll.coll_barrier_module);
+
+    getcwd( wpath, sizeof(wpath) );
+    sprintf( wpath,"%s/%s",wpath,fh->f_filename );
+    
+    if(-1 == access(fh->f_filename, F_OK)) {
+        return OMPI_ERROR; // file doesn't exist
+    }
+
+    amode = 0;
+    if (fh->f_amode & MPI_MODE_CREATE)
+        amode = amode | O_CREAT;
+    if (fh->f_amode & MPI_MODE_RDONLY)
+        amode = amode | O_RDONLY;
+    if (fh->f_amode & MPI_MODE_WRONLY)
+        amode = amode | O_WRONLY;
+    if (fh->f_amode & MPI_MODE_RDWR)
+        amode = amode | O_RDWR;
+    if (fh->f_amode & MPI_MODE_EXCL) {
+        return OMPI_ERROR;
+    }
+      
+    plfs_ret = plfs_close(fh->f_fs_ptr, 0, 0, amode ,NULL, &flags);    
+    if (PLFS_SUCCESS != plfs_ret) {                                                 
+        opal_output(0, "fs_plfs_file_close: Error in plfs_close:\n%s\n", strplfserr(plfs_ret));
+        return OMPI_ERROR;
+    }
+    return OMPI_SUCCESS;
+}

ompi/mca/fs/plfs/fs_plfs_file_delete.c

@@ -0,0 +1,50 @@
+/*
+ * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2011 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, 
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2008-2014 University of Houston. All rights reserved.
+ * $COPYRIGHT$
+ * 
+ * Additional copyrights may follow
+ * 
+ * $HEADER$
+ */
+
+
+#include "ompi_config.h"
+#include "fs_plfs.h"
+#include <unistd.h>
+
+#include "mpi.h"
+#include "ompi/constants.h"
+#include "ompi/mca/fs/fs.h"
+
+/*
+ *	file_delete_plfs
+ *
+ *	Function:	- deletes a file
+ *	Accepts:	- file name & info
+ *	Returns:	- Success if file closed
+ */
+int
+mca_fs_plfs_file_delete (char* file_name,
+                       struct ompi_info_t *info)
+{
+    plfs_error_t plfs_ret;
+    char wpath[1024];
+    getcwd( wpath, sizeof(wpath) );
+    sprintf( wpath,"%s/%s",wpath,file_name );
+    plfs_ret = plfs_unlink( wpath );
+    if (PLFS_SUCCESS != plfs_ret) {
+        opal_output(0, "fs_plfs_file_delete: Error in plfs_unlink:\n%s\n", strplfserr(plfs_ret));
+	return OMPI_ERROR;
+    }
+    return OMPI_SUCCESS;
+}

ompi/mca/fs/plfs/fs_plfs_file_get_size.c

@@ -0,0 +1,56 @@
+/*
+ * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2011 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, 
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2008-2014 University of Houston. All rights reserved.
+ * $COPYRIGHT$
+ * 
+ * Additional copyrights may follow
+ * 
+ * $HEADER$
+ */
+
+
+#include "ompi_config.h"
+#include "fs_plfs.h"
+
+#include "mpi.h"
+#include "ompi/constants.h"
+#include "ompi/mca/fs/fs.h"
+
+/*
+ *	file_get_size_plfs
+ *
+ *	Function:	- get_size of a file
+ *	Accepts:	- same arguments as MPI_File_get_size()
+ *	Returns:	- Success if size is get
+ */
+int
+mca_fs_plfs_file_get_size (mca_io_ompio_file_t *fh,
+                         OMPI_MPI_OFFSET_TYPE *size)
+{
+    Plfs_fd *pfd = NULL;
+    plfs_error_t plfs_ret;
+    struct stat st;
+    char wpath[1024];
+    int size_only = 1;
+
+    getcwd(wpath, sizeof(wpath));           
+    sprintf(wpath,"%s/%s",wpath,fh->f_filename); 
+
+    plfs_ret = plfs_getattr(pfd, wpath, &st, size_only);
+    if (PLFS_SUCCESS != plfs_ret) {
+        opal_output(0, "fs_plfs_file_get_size: Error in plfs_getattr:\n%s\n", strplfserr(plfs_ret));
+	return OMPI_ERROR;
+    }
+
+    *size = st.st_size;
+    return OMPI_SUCCESS;
+}

ompi/mca/fs/plfs/fs_plfs_file_open.c

@@ -0,0 +1,118 @@
+/*
+ * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2011 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, 
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2008-2014 University of Houston. All rights reserved.
+ * $COPYRIGHT$
+ * 
+ * Additional copyrights may follow
+ * 
+ * $HEADER$
+ */
+
+
+#include "ompi_config.h"
+#include "fs_plfs.h"
+
+#include <fcntl.h>
+#include <unistd.h>
+#include "mpi.h"
+#include "ompi/constants.h"
+#include "ompi/mca/fs/fs.h"
+#include "ompi/communicator/communicator.h"
+#include "ompi/info/info.h"
+
+#include <sys/ioctl.h>
+
+/*
+ *	file_open_plfs
+ *
+ *	Function:	- opens a new file
+ *	Accepts:	- same arguments as MPI_File_open()
+ *	Returns:	- Success if new file handle
+ */
+int
+mca_fs_plfs_file_open (struct ompi_communicator_t *comm, 
+                     char* filename,
+                     int access_mode,
+                     struct ompi_info_t *info,
+                     mca_io_ompio_file_t *fh)
+{
+    int rank;
+    int amode;
+    int old_mask, perm;
+    plfs_error_t plfs_ret;
+    Plfs_fd *pfd = NULL;
+    char wpath[1024];
+    size_t len = sizeof(int);
+    char key[] = "num_hostdirs";
+
+    rank = ompi_comm_rank ( comm );
+
+    getcwd( wpath, sizeof(wpath) );
+    sprintf( wpath,"%s/%s",wpath,filename );
+    
+    if (OMPIO_PERM_NULL == fh->f_perm) {
+        old_mask = umask(022);
+        umask(old_mask);
+        perm = old_mask ^ 0666;
+    }
+    else {
+        perm = fh->f_perm;
+    }
+
+    amode = 0;
+
+    if (access_mode & MPI_MODE_RDONLY)
+        amode = amode | O_RDONLY;
+    if (access_mode & MPI_MODE_WRONLY)
+        amode = amode | O_WRONLY;
+    if (access_mode & MPI_MODE_RDWR)
+        amode = amode | O_RDWR;
+    if (access_mode & MPI_MODE_EXCL) { 
+        if( is_plfs_path(wpath) == 1 ) { //the file already exists
+	    return OMPI_ERROR;
+	}
+    }
+
+    if (0 == rank) {
+        /* MODE_CREATE and MODE_EXCL can only be set by one process */
+        if (access_mode & MPI_MODE_CREATE)
+	    amode = amode | O_CREAT;
+
+	plfs_ret = plfs_open( &pfd, wpath, amode, 0, perm, NULL );
+	fh->f_fs_ptr = pfd;
+    }
+    
+    comm->c_coll.coll_bcast ( &plfs_ret, 1, MPI_INT, 0, comm, comm->c_coll.coll_bcast_module);
+    if ( PLFS_SUCCESS != plfs_ret ) {
+        return OMPI_ERROR;
+    }
+
+    if (0 != rank) {
+        plfs_ret = plfs_open( &pfd, wpath, amode, 0, perm, NULL );
+	if (PLFS_SUCCESS != plfs_ret) {
+	    opal_output(0, "fs_plfs_file_open: Error in plfs_open:\n%s\n", strplfserr(plfs_ret));
+	    return OMPI_ERROR; 
+	}
+	else {
+	    fh->f_fs_ptr = pfd;
+	}
+    }    
+
+    if (mca_fs_plfs_num_hostdir > 0) {
+        plfs_ret = plfs_setxattr( pfd, &mca_fs_plfs_num_hostdir, key, len );
+        if (PLFS_SUCCESS != plfs_ret) {
+	    opal_output(0, "fs_plfs_file_open: Error in plfs_setxattr:\n%s\n", strplfserr(plfs_ret));
+	    return OMPI_ERROR;
+	}
+    }
+    return OMPI_SUCCESS;
+}

ompi/mca/fs/plfs/fs_plfs_file_set_info.c

@@ -0,0 +1,41 @@
+/*
+ * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2011 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, 
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2008-2014 University of Houston. All rights reserved.
+ * $COPYRIGHT$
+ * 
+ * Additional copyrights may follow
+ * 
+ * $HEADER$
+ */
+
+
+#include "ompi_config.h"
+#include "fs_plfs.h"
+
+#include "mpi.h"
+#include "ompi/constants.h"
+#include "ompi/mca/fs/fs.h"
+
+/*
+ *	file_set_info_plfs
+ *
+ *	Function:	- set_info of a file
+ *	Accepts:	- same arguments as MPI_File_set_info()
+ *	Returns:	- Success if info is set
+ */
+int
+mca_fs_plfs_file_set_info (mca_io_ompio_file_t *file_handle,
+				struct ompi_info_t *info)
+{
+    printf ("PLFS SET INFO\n");
+    return OMPI_SUCCESS;
+}

ompi/mca/fs/plfs/fs_plfs_file_set_size.c

@@ -0,0 +1,41 @@
+/*
+ * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2011 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, 
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2008-2011 University of Houston. All rights reserved.
+ * $COPYRIGHT$
+ * 
+ * Additional copyrights may follow
+ * 
+ * $HEADER$
+ */
+
+
+#include "ompi_config.h"
+#include "fs_plfs.h"
+
+#include "mpi.h"
+#include "ompi/constants.h"
+#include "ompi/mca/fs/fs.h"
+
+/*
+ *	file_set_size_plfs
+ *
+ *	Function:	- set_size of a file
+ *	Accepts:	- same arguments as MPI_File_set_size()
+ *	Returns:	- Success if size is set
+ */
+int
+mca_fs_plfs_file_set_size (mca_io_ompio_file_t *file_handle,
+                         OMPI_MPI_OFFSET_TYPE size)
+{
+    printf ("PLFS SET SIZE\n");
+    return OMPI_SUCCESS;
+}

ompi/mca/fs/plfs/fs_plfs_file_sync.c

@@ -0,0 +1,45 @@
+/*
+ * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2011 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, 
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2008-2014 University of Houston. All rights reserved.
+ * $COPYRIGHT$
+ * 
+ * Additional copyrights may follow
+ * 
+ * $HEADER$
+ */
+
+
+#include "ompi_config.h"
+#include "fs_plfs.h"
+
+#include "mpi.h"
+#include "ompi/constants.h"
+#include "ompi/mca/fs/fs.h"
+
+/*
+ *	file_sync_plfs
+ *
+ *	Function:	- closes a new file
+ *	Accepts:	- file handle
+ *	Returns:	- Success if file closed
+ */
+int
+mca_fs_plfs_file_sync (mca_io_ompio_file_t *fh)
+{
+    plfs_error_t plfs_ret;
+    plfs_ret = plfs_sync( fh->f_fs_ptr );
+    if (PLFS_SUCCESS != plfs_ret) {
+        opal_output(0, "fs_plfs_file_sync: Error in plfs_sync:\n%s\n", strplfserr(plfs_ret));
+	return OMPI_ERROR;
+    }
+    return OMPI_SUCCESS;
+}

ompi/mca/io/ompio/io_ompio.h

@@ -94,7 +94,8 @@ enum ompio_fs_type
     NONE = 0,
     UFS = 1,
     PVFS2 = 2,
-    LUSTRE = 3
+    LUSTRE = 3,
+    PLFS = 4
 };
 
 OMPI_DECLSPEC extern mca_io_base_component_2_0_0_t mca_io_ompio_component;