diff --git a/config/ompi_check_plfs.m4 b/config/ompi_check_plfs.m4 new file mode 100644 index 0000000000..35726da056 --- /dev/null +++ b/config/ompi_check_plfs.m4 @@ -0,0 +1,112 @@ +# -*- shell-script -*- +# +# Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana +# University Research and Technology +# Corporation. All rights reserved. +# Copyright (c) 2004-2005 The University of Tennessee and The University +# of Tennessee Research Foundation. All rights +# reserved. +# Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, +# University of Stuttgart. All rights reserved. +# Copyright (c) 2004-2006 The Regents of the University of California. +# All rights reserved. +# Copyright (c) 2009 Cisco Systems, Inc. All rights reserved. +# Copyright (c) 2008-2012 University of Houston. All rights reserved. +# $COPYRIGHT$ +# +# Additional copyrights may follow +# +# $HEADER$ +# + +# OMPI_CHECK_PLFS(prefix, [action-if-found], [action-if-not-found]) +# -------------------------------------------------------- +# check if PLFS support can be found. sets prefix_{CPPFLAGS, +# LDFLAGS, LIBS} as needed and runs action-if-found if there is +# support, otherwise executes action-if-not-found +AC_DEFUN([OMPI_CHECK_PLFS],[ + + check_plfs_CPPFLAGS= + check_plfs_LDFLAGS= + check_plfs_LIBS= + + check_plfs_save_LIBS="$LIBS" + check_plfs_save_LDFLAGS="$LDFLAGS" + check_plfs_save_CPPFLAGS="$CPPFLAGS" + + check_plfs_configuration="none" + ompi_check_plfs_happy="yes" + + + # Get some configuration information + AC_ARG_WITH([plfs], + [AC_HELP_STRING([--with-plfs(=DIR)], + [Build Plfs support, optionally adding DIR/include, DIR/lib, and DIR/lib64 to the search path for headers and libraries])]) + OMPI_CHECK_WITHDIR([plfs], [$with_plfs], [include/plfs.h]) + + AC_ARG_WITH([plfs-libs], + [AC_HELP_STRING([--with-plfs-libs=LIBS], + [Libraries to link with for plfs])]) + + temp_with_plfs="$with_plfs" + AS_IF([test -z "$with_plfs"], + [with_plfs="/usr/local"]) + + temp_with_plfs_libs="$with_plfs_libs" + AS_IF([test -z "$with_plfs_libs"], + [with_plfs_libs="plfs pthread"]) + + # Add correct -I and -L flags + AS_IF([test -d "$with_plfs/include"], + [check_plfs_CPPFLAGS="-I$with_plfs/include" + $1_CPPFLAGS="$check_plfs_CPPFLAGS" + CPPFLAGS="$CPPFLAGS $check_plfs_CPPFLAGS"], + [ompi_check_plfs_happy="no"]) + + AS_IF([test "$ompi_check_plfs_happy" = "yes"], + [AS_IF([test -d "$with_plfs/lib"], + [check_plfs_LDFLAGS="-L$with_plfs/lib" + $1_LDFLAGS="$check_plfs_LDFLAGS" + LDFLAGS="$LDFLAGS $check_plfs_LDFLAGS"], + [ompi_check_plfs_happy="no"]) + ],[]) + + # Try to find all the plfs libraries + AS_IF([test "$ompi_check_plfs_happy" = "yes"], + [ AS_IF([test -n "$with_plfs_libs"] + [for lib in $with_plfs_libs ; do + check_plfs_LIBS="$check_plfs_LIBS -l$lib" + done]) + + $1_LIBS="$check_plfs_LIBS" + LIBS="$LIBS $check_plfs_LIBS" + + # check for plfs + AC_CHECK_HEADERS([plfs.h], + [AC_MSG_CHECKING([if possible to link PLFS]) + AC_LINK_IFELSE( + [AC_LANG_PROGRAM( + [[#include ]], + [[is_plfs_path(NULL);]])], + [AC_MSG_RESULT([yes]) + ompi_check_plfs_happy="yes"], + [AC_MSG_RESULT([no]) + ompi_check_plfs_happy="no"])], + [ompi_check_plfs_happy="no"]) + ]) + + + LDFLAGS="$check_plfs_save_LDFLAGS" + CPPFLAGS="$check_plfs_save_CPPFLAGS" + LIBS="$check_plfs_save_LIBS" + AS_IF([test "$ompi_check_plfs_happy" = "yes"], + [$2], + [AS_IF([test ! -z "$with_plfs" -a "$with_plfs" != "no"], + [echo PLFS support not found]) + $3]) + + with_plfs="$temp_with_plfs" + with_plfs_libs="$temp_with_plfs_libs" + +]) + diff --git a/ompi/mca/fbtl/plfs/Makefile.am b/ompi/mca/fbtl/plfs/Makefile.am new file mode 100644 index 0000000000..f5298871eb --- /dev/null +++ b/ompi/mca/fbtl/plfs/Makefile.am @@ -0,0 +1,54 @@ +# +# Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana +# University Research and Technology +# Corporation. All rights reserved. +# Copyright (c) 2004-2005 The University of Tennessee and The University +# of Tennessee Research Foundation. All rights +# reserved. +# Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, +# University of Stuttgart. All rights reserved. +# Copyright (c) 2004-2005 The Regents of the University of California. +# All rights reserved. +# Copyright (c) 2008-2011 University of Houston. All rights reserved. +# $COPYRIGHT$ +# +# Additional copyrights may follow +# +# $HEADER$ +# + +# Make the output library in this directory, and name it either +# mca__.la (for DSO builds) or libmca__.la +# (for static builds). + +if MCA_BUILD_ompi_fbtl_plfs_DSO +component_noinst = +component_install = mca_fbtl_plfs.la +else +component_noinst = libmca_fbtl_plfs.la +component_install = +endif + +# Source files + +fbtl_plfs_sources = \ + fbtl_plfs.h \ + fbtl_plfs.c \ + fbtl_plfs_component.c \ + fbtl_plfs_preadv.c \ + fbtl_plfs_ipreadv.c \ + fbtl_plfs_pwritev.c \ + fbtl_plfs_ipwritev.c + +AM_CPPFLAGS = $(fbtl_plfs_CPPFLAGS) + +mcacomponentdir = $(pkglibdir) +mcacomponent_LTLIBRARIES = $(component_install) +mca_fbtl_plfs_la_SOURCES = $(fbtl_plfs_sources) +mca_fbtl_plfs_la_LIBADD = $(fbtl_plfs_LIBS) +mca_fbtl_plfs_la_LDFLAGS = -module -avoid-version $(fbtl_plfs_LDFLAGS) + +noinst_LTLIBRARIES = $(component_noinst) +libmca_fbtl_plfs_la_SOURCES = $(fbtl_plfs_sources) +libmca_fbtl_plfs_la_LIBADD = $(fbtl_plfs_LIBS) +libmca_fbtl_plfs_la_LDFLAGS = -module -avoid-version $(fbtl_plfs_LDFLAGS) diff --git a/ompi/mca/fbtl/plfs/configure.m4 b/ompi/mca/fbtl/plfs/configure.m4 new file mode 100644 index 0000000000..85627198da --- /dev/null +++ b/ompi/mca/fbtl/plfs/configure.m4 @@ -0,0 +1,42 @@ +# -*- shell-script -*- +# +# Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana +# University Research and Technology +# Corporation. All rights reserved. +# Copyright (c) 2004-2005 The University of Tennessee and The University +# of Tennessee Research Foundation. All rights +# reserved. +# Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, +# University of Stuttgart. All rights reserved. +# Copyright (c) 2004-2005 The Regents of the University of California. +# All rights reserved. +# Copyright (c) 2010 Cisco Systems, Inc. All rights reserved. +# Copyright (c) 2008-2014 University of Houston. All rights reserved. +# $COPYRIGHT$ +# +# Additional copyrights may follow +# +# $HEADER$ +# + + +# MCA_fbtl_plfs_CONFIG(action-if-can-compile, +# [action-if-cant-compile]) +# ------------------------------------------------ +AC_DEFUN([MCA_ompi_fbtl_plfs_CONFIG],[ + AC_CONFIG_FILES([ompi/mca/fbtl/plfs/Makefile]) + + OMPI_CHECK_PLFS([fbtl_plfs], + [fbtl_plfs_happy="yes"], + [fbtl_plfs_happy="no"]) + + AS_IF([test "$fbtl_plfs_happy" = "yes"], + [$1], + [$2]) + + + # substitute in the things needed to build plfs + AC_SUBST([fbtl_plfs_CPPFLAGS]) + AC_SUBST([fbtl_plfs_LDFLAGS]) + AC_SUBST([fbtl_plfs_LIBS]) +])dnl diff --git a/ompi/mca/fbtl/plfs/fbtl_plfs.c b/ompi/mca/fbtl/plfs/fbtl_plfs.c new file mode 100644 index 0000000000..0fd778da0c --- /dev/null +++ b/ompi/mca/fbtl/plfs/fbtl_plfs.c @@ -0,0 +1,83 @@ +/* + * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana + * University Research and Technology + * Corporation. All rights reserved. + * Copyright (c) 2004-2006 The University of Tennessee and The University + * of Tennessee Research Foundation. All rights + * reserved. + * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, + * University of Stuttgart. All rights reserved. + * Copyright (c) 2004-2005 The Regents of the University of California. + * All rights reserved. + * Copyright (c) 2008-2014 University of Houston. All rights reserved. + * $COPYRIGHT$ + * + * Additional copyrights may follow + * + * $HEADER$ + * + * These symbols are in a file by themselves to provide nice linker + * semantics. Since linkers generally pull in symbols by object fules, + * keeping these symbols as the only symbols in this file prevents + * utility programs such as "ompi_info" from having to import entire + * modules just to query their version and parameters + */ + +#include "ompi_config.h" +#include "mpi.h" +#include "ompi/mca/fbtl/fbtl.h" +#include "ompi/mca/fbtl/plfs/fbtl_plfs.h" + +/* + * ******************************************************************* + * ************************ actions structure ************************ + * ******************************************************************* + */ +static mca_fbtl_base_module_1_0_0_t plfs = { + mca_fbtl_plfs_module_init, /* initalise after being selected */ + mca_fbtl_plfs_module_finalize, /* close a module on a communicator */ + mca_fbtl_plfs_preadv, + mca_fbtl_plfs_ipreadv, + mca_fbtl_plfs_pwritev, + mca_fbtl_plfs_ipwritev +}; +/* + * ******************************************************************* + * ************************* structure ends ************************** + * ******************************************************************* + */ + +int mca_fbtl_plfs_component_init_query(bool enable_progress_threads, + bool enable_mpi_threads) { + /* Nothing to do */ + return OMPI_SUCCESS; +} + +struct mca_fbtl_base_module_1_0_0_t * +mca_fbtl_plfs_component_file_query (mca_io_ompio_file_t *fh, int *priority) { + *priority = mca_fbtl_plfs_priority; + + if (PLFS == fh->f_fstype) { + if (*priority < 50) { + *priority = 50; + } + } + return &plfs; +} + +int mca_fbtl_plfs_component_file_unquery (mca_io_ompio_file_t *file) { + /* This function might be needed for some purposes later. for now it + * does not have anything to do since there are no steps which need + * to be undone if this module is not selected */ + + return OMPI_SUCCESS; +} + +int mca_fbtl_plfs_module_init (mca_io_ompio_file_t *file) { + return OMPI_SUCCESS; +} + + +int mca_fbtl_plfs_module_finalize (mca_io_ompio_file_t *file) { + return OMPI_SUCCESS; +} diff --git a/ompi/mca/fbtl/plfs/fbtl_plfs.h b/ompi/mca/fbtl/plfs/fbtl_plfs.h new file mode 100644 index 0000000000..30988fa768 --- /dev/null +++ b/ompi/mca/fbtl/plfs/fbtl_plfs.h @@ -0,0 +1,68 @@ +/* + * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana + * University Research and Technology + * Corporation. All rights reserved. + * Copyright (c) 2004-2006 The University of Tennessee and The University + * of Tennessee Research Foundation. All rights + * reserved. + * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, + * University of Stuttgart. All rights reserved. + * Copyright (c) 2004-2005 The Regents of the University of California. + * All rights reserved. + * Copyright (c) 2008-2014 University of Houston. All rights reserved. + * $COPYRIGHT$ + * + * Additional copyrights may follow + * + * $HEADER$ + */ + +#ifndef MCA_FBTL_PLFS_H +#define MCA_FBTL_PLFS_H + +#include "ompi_config.h" +#include "opal/mca/mca.h" +#include "ompi/mca/fbtl/fbtl.h" +#include "ompi/mca/io/ompio/io_ompio.h" +#include + +extern int mca_fbtl_plfs_priority; + +BEGIN_C_DECLS + +int mca_fbtl_plfs_component_init_query(bool enable_progress_threads, + bool enable_mpi_threads); +struct mca_fbtl_base_module_1_0_0_t * +mca_fbtl_plfs_component_file_query (mca_io_ompio_file_t *file, int *priority); +int mca_fbtl_plfs_component_file_unquery (mca_io_ompio_file_t *file); + +int mca_fbtl_plfs_module_init (mca_io_ompio_file_t *file); +int mca_fbtl_plfs_module_finalize (mca_io_ompio_file_t *file); + +OMPI_MODULE_DECLSPEC extern mca_fbtl_base_component_2_0_0_t mca_fbtl_plfs_component; +/* + * ****************************************************************** + * ********* functions which are implemented in this module ********* + * ****************************************************************** + */ + +size_t mca_fbtl_plfs_preadv (mca_io_ompio_file_t *file, + int *sorted); +size_t mca_fbtl_plfs_pwritev (mca_io_ompio_file_t *file, + int *sorted); +size_t mca_fbtl_plfs_ipreadv (mca_io_ompio_file_t *file, + int *sorted, + ompi_request_t **request); +size_t mca_fbtl_plfs_ipwritev (mca_io_ompio_file_t *file, + int *sorted, + ompi_request_t **request); + +/* + * ****************************************************************** + * ************ functions implemented in this module end ************ + * ****************************************************************** + */ + +END_C_DECLS + +#endif /* MCA_FBTL_PLFS_H */ diff --git a/ompi/mca/fbtl/plfs/fbtl_plfs_component.c b/ompi/mca/fbtl/plfs/fbtl_plfs_component.c new file mode 100644 index 0000000000..ad97288c90 --- /dev/null +++ b/ompi/mca/fbtl/plfs/fbtl_plfs_component.c @@ -0,0 +1,65 @@ +/* + * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana + * University Research and Technology + * Corporation. All rights reserved. + * Copyright (c) 2004-2005 The University of Tennessee and The University + * of Tennessee Research Foundation. All rights + * reserved. + * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, + * University of Stuttgart. All rights reserved. + * Copyright (c) 2004-2005 The Regents of the University of California. + * All rights reserved. + * Copyright (c) 2008-2014 University of Houston. All rights reserved. + * $COPYRIGHT$ + * + * Additional copyrights may follow + * + * $HEADER$ + * + * These symbols are in a file by themselves to provide nice linker + * semantics. Since linkers generally pull in symbols by object + * files, keeping these symbols as the only symbols in this file + * prevents utility programs such as "ompi_info" from having to import + * entire components just to query their version and parameters. + */ + +#include "ompi_config.h" +#include "fbtl_plfs.h" +#include "mpi.h" + +/* + * Public string showing the fbtl plfs component version number + */ +const char *mca_fbtl_plfs_component_version_string = + "OMPI/MPI plfs FBTL MCA component version " OMPI_VERSION; + +int mca_fbtl_plfs_priority = 10; + +/* + * Instantiate the public struct with all of our public information + * and pointers to our public functions in it + */ +mca_fbtl_base_component_2_0_0_t mca_fbtl_plfs_component = { + + /* First, the mca_component_t struct containing meta information + about the component itself */ + + { + MCA_FBTL_BASE_VERSION_2_0_0, + + /* Component name and version */ + "plfs", + OMPI_MAJOR_VERSION, + OMPI_MINOR_VERSION, + OMPI_RELEASE_VERSION, + NULL, + NULL + }, + { + /* This component is checkpointable */ + MCA_BASE_METADATA_PARAM_CHECKPOINT + }, + mca_fbtl_plfs_component_init_query, /* get thread level */ + mca_fbtl_plfs_component_file_query, /* get priority and actions */ + mca_fbtl_plfs_component_file_unquery /* undo what was done by previous function */ +}; diff --git a/ompi/mca/fbtl/plfs/fbtl_plfs_ipreadv.c b/ompi/mca/fbtl/plfs/fbtl_plfs_ipreadv.c new file mode 100644 index 0000000000..01913b9002 --- /dev/null +++ b/ompi/mca/fbtl/plfs/fbtl_plfs_ipreadv.c @@ -0,0 +1,35 @@ +/* + * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana + * University Research and Technology + * Corporation. All rights reserved. + * Copyright (c) 2004-2011 The University of Tennessee and The University + * of Tennessee Research Foundation. All rights + * reserved. + * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, + * University of Stuttgart. All rights reserved. + * Copyright (c) 2004-2005 The Regents of the University of California. + * All rights reserved. + * Copyright (c) 2008-2014 University of Houston. All rights reserved. + * $COPYRIGHT$ + * + * Additional copyrights may follow + * + * $HEADER$ + */ + + +#include "ompi_config.h" +#include "fbtl_plfs.h" + +#include "mpi.h" +#include "ompi/constants.h" +#include "ompi/mca/fbtl/fbtl.h" + +size_t +mca_fbtl_plfs_ipreadv (mca_io_ompio_file_t *file, + int *sorted, + ompi_request_t **request) +{ + printf ("PLFS IPREADV\n"); + return OMPI_ERROR; +} diff --git a/ompi/mca/fbtl/plfs/fbtl_plfs_ipwritev.c b/ompi/mca/fbtl/plfs/fbtl_plfs_ipwritev.c new file mode 100644 index 0000000000..ae4f252217 --- /dev/null +++ b/ompi/mca/fbtl/plfs/fbtl_plfs_ipwritev.c @@ -0,0 +1,35 @@ +/* + * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana + * University Research and Technology + * Corporation. All rights reserved. + * Copyright (c) 2004-2011 The University of Tennessee and The University + * of Tennessee Research Foundation. All rights + * reserved. + * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, + * University of Stuttgart. All rights reserved. + * Copyright (c) 2004-2005 The Regents of the University of California. + * All rights reserved. + * Copyright (c) 2008-2014 University of Houston. All rights reserved. + * $COPYRIGHT$ + * + * Additional copyrights may follow + * + * $HEADER$ + */ + +#include "ompi_config.h" +#include "fbtl_plfs.h" + +#include "mpi.h" +#include +#include "ompi/constants.h" +#include "ompi/mca/fbtl/fbtl.h" + +size_t +mca_fbtl_plfs_ipwritev (mca_io_ompio_file_t *fh, + int *sorted, + ompi_request_t **request) +{ + printf ("PLFS IPWRITEV\n"); + return OMPI_ERROR; +} diff --git a/ompi/mca/fbtl/plfs/fbtl_plfs_preadv.c b/ompi/mca/fbtl/plfs/fbtl_plfs_preadv.c new file mode 100644 index 0000000000..705e6c28ff --- /dev/null +++ b/ompi/mca/fbtl/plfs/fbtl_plfs_preadv.c @@ -0,0 +1,201 @@ +/* + * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana + * University Research and Technology + * Corporation. All rights reserved. + * Copyright (c) 2004-2011 The University of Tennessee and The University + * of Tennessee Research Foundation. All rights + * reserved. + * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, + * University of Stuttgart. All rights reserved. + * Copyright (c) 2004-2005 The Regents of the University of California. + * All rights reserved. + * Copyright (c) 2008-2014 University of Houston. All rights reserved. + * $COPYRIGHT$ + * + * Additional copyrights may follow + * + * $HEADER$ + */ + +#include "ompi_config.h" +#include "fbtl_plfs.h" + +#include "mpi.h" +#include +#include "ompi/constants.h" +#include "ompi/mca/fbtl/fbtl.h" + +size_t +mca_fbtl_plfs_preadv (mca_io_ompio_file_t *fh, + int *sorted) +{ + + Plfs_fd *pfd = NULL; + plfs_error_t plfs_ret; + pfd = fh->f_fs_ptr; + + int i, block=1; + struct iovec *iov = NULL; + int iov_count = 0; + OMPI_MPI_OFFSET_TYPE iov_offset = 0; + + if (NULL == fh->f_io_array) { + return OMPI_ERROR; + } + + iov = (struct iovec *) malloc + (OMPIO_IOVEC_INITIAL_SIZE * sizeof (struct iovec)); + if (NULL == iov) { + opal_output(1, "OUT OF MEMORY\n"); + return OMPI_ERR_OUT_OF_RESOURCE; + } + + if (NULL != sorted) { + for (i=0 ; if_num_of_io_entries ; i++) { + if (0 == iov_count) { + iov[iov_count].iov_base = fh->f_io_array[sorted[i]].memory_address; + iov[iov_count].iov_len = fh->f_io_array[sorted[i]].length; + iov_offset = (OMPI_MPI_OFFSET_TYPE)(intptr_t)fh->f_io_array[sorted[i]].offset; + iov_count ++; + } + + if (OMPIO_IOVEC_INITIAL_SIZE*block <= iov_count) { + block ++; + iov = (struct iovec *)realloc + (iov, OMPIO_IOVEC_INITIAL_SIZE * block * + sizeof(struct iovec)); + if (NULL == iov) { + opal_output(1, "OUT OF MEMORY\n"); + return OMPI_ERR_OUT_OF_RESOURCE; + } + } + + if (fh->f_num_of_io_entries != i+1) { + if (((OMPI_MPI_OFFSET_TYPE)(intptr_t)fh->f_io_array[sorted[i]].offset + + (OPAL_PTRDIFF_TYPE)fh->f_io_array[sorted[i]].length) == + (OMPI_MPI_OFFSET_TYPE)(intptr_t)fh->f_io_array[sorted[i+1]].offset) { + iov[iov_count].iov_base = + fh->f_io_array[sorted[i+1]].memory_address; + iov[iov_count].iov_len = fh->f_io_array[sorted[i+1]].length; + iov_count ++; + continue; + } + } + + // Find the total number of bytes to be read. + size_t bytes = 0; + for (int i = 0; i < iov_count; ++i) { + bytes += iov[i].iov_len; + } + + // Allocate a temporary buffer to hold the data + char *buffer; + buffer = (char *) malloc (bytes); + if (buffer == NULL) { + return OMPI_ERROR; + } + + // Read the data + ssize_t bytes_read; + + plfs_ret = plfs_read( pfd, buffer, bytes, iov_offset, &bytes_read ); + if (PLFS_SUCCESS != plfs_ret) { + opal_output(0, "fbtl_plfs_preadv: Error in plfs_read:\n%s\n", strplfserr(plfs_ret)); + return OMPI_ERROR; + } + + if (bytes_read < 0) + return OMPI_ERROR; + + // Copy the data from BUFFER into the memory specified by IOV + bytes = bytes_read; + for (int i = 0; i < iov_count; ++i) { + size_t copy = MIN (iov[i].iov_len, bytes); + (void) memcpy ((void *) iov[i].iov_base, (void *) buffer, copy); + buffer += copy; + bytes -= copy; + if (bytes == 0) { + break; + } + } + iov_count = 0; + } + } + + else { + for (i=0 ; if_num_of_io_entries ; i++) { + if (0 == iov_count) { + iov[iov_count].iov_base = fh->f_io_array[i].memory_address; + iov[iov_count].iov_len = fh->f_io_array[i].length; + iov_offset = (OMPI_MPI_OFFSET_TYPE)(intptr_t)fh->f_io_array[i].offset; + iov_count ++; + } + + if (OMPIO_IOVEC_INITIAL_SIZE*block <= iov_count) { + block ++; + iov = (struct iovec *)realloc + (iov, OMPIO_IOVEC_INITIAL_SIZE * block * + sizeof(struct iovec)); + if (NULL == iov) { + opal_output(1, "OUT OF MEMORY\n"); + return OMPI_ERR_OUT_OF_RESOURCE; + } + } + + if (fh->f_num_of_io_entries != i+1) { + if (((OMPI_MPI_OFFSET_TYPE)(intptr_t)fh->f_io_array[i].offset + + (OPAL_PTRDIFF_TYPE)fh->f_io_array[i].length) == + (OMPI_MPI_OFFSET_TYPE)(intptr_t)fh->f_io_array[i+1].offset) { + iov[iov_count].iov_base = + fh->f_io_array[i+1].memory_address; + iov[iov_count].iov_len = fh->f_io_array[i+1].length; + iov_count ++; + continue; + } + } + + // Find the total number of bytes to be read. + size_t bytes = 0; + for (int i = 0; i < iov_count; ++i) { + bytes += iov[i].iov_len; + } + + // Allocate a temporary buffer to hold the data + char *buffer; + buffer = (char *) malloc (bytes); + if (buffer == NULL) { + return OMPI_ERROR; + } + + // Read the data + ssize_t bytes_read; + plfs_ret = plfs_read( pfd, buffer, bytes, iov_offset, &bytes_read ); + if (PLFS_SUCCESS != plfs_ret) { + opal_output(0, "fbtl_plfs_preadv: Error in plfs_read:\n%s\n", strplfserr(plfs_ret)); + return OMPI_ERROR; + } + + if (bytes_read < 0) + return OMPI_ERROR; + + // Copy the data from BUFFER into the memory specified by IOV + bytes = bytes_read; + for (int i = 0; i < iov_count; ++i) { + size_t copy = MIN (iov[i].iov_len, bytes); + (void) memcpy ((void *) iov[i].iov_base, (void *) buffer, copy); + buffer += copy; + bytes -= copy; + if (bytes == 0) { + break; + } + } + iov_count = 0; + } + } + if (NULL != iov) { + free (iov); + iov = NULL; + } + + return OMPI_SUCCESS; +} diff --git a/ompi/mca/fbtl/plfs/fbtl_plfs_pwritev.c b/ompi/mca/fbtl/plfs/fbtl_plfs_pwritev.c new file mode 100644 index 0000000000..4b0789486a --- /dev/null +++ b/ompi/mca/fbtl/plfs/fbtl_plfs_pwritev.c @@ -0,0 +1,197 @@ +/* + * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana + * University Research and Technology + * Corporation. All rights reserved. + * Copyright (c) 2004-2011 The University of Tennessee and The University + * of Tennessee Research Foundation. All rights + * reserved. + * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, + * University of Stuttgart. All rights reserved. + * Copyright (c) 2004-2005 The Regents of the University of California. + * All rights reserved. + * Copyright (c) 2008-2014 University of Houston. All rights reserved. + * $COPYRIGHT$ + * + * Additional copyrights may follow + * + * $HEADER$ + */ + + +#include "ompi_config.h" +#include "fbtl_plfs.h" + +#include "mpi.h" +#include +#include "ompi/constants.h" +#include "ompi/mca/fbtl/fbtl.h" + +size_t +mca_fbtl_plfs_pwritev (mca_io_ompio_file_t *fh, + int *sorted) +{ + Plfs_fd *pfd = NULL; + plfs_error_t plfs_ret; + pfd = fh->f_fs_ptr; + + int i, block = 1; + struct iovec *iov = NULL; + int iov_count = 0; + OMPI_MPI_OFFSET_TYPE iov_offset = 0; + + if (NULL == fh->f_io_array) { + return OMPI_ERROR; + } + + iov = (struct iovec *) malloc + (OMPIO_IOVEC_INITIAL_SIZE * sizeof (struct iovec)); + if (NULL == iov) { + opal_output(1, "OUT OF MEMORY\n"); + return OMPI_ERR_OUT_OF_RESOURCE; + } + + if (NULL != sorted) { + for (i=0 ; if_num_of_io_entries ; i++) { + if (0 == iov_count) { + iov[iov_count].iov_base = fh->f_io_array[sorted[i]].memory_address; + iov[iov_count].iov_len = fh->f_io_array[sorted[i]].length; + iov_offset = (OMPI_MPI_OFFSET_TYPE)(intptr_t)fh->f_io_array[sorted[i]].offset; + iov_count ++; + } + + if (OMPIO_IOVEC_INITIAL_SIZE*block <= iov_count) { + block ++; + iov = (struct iovec *)realloc + (iov, OMPIO_IOVEC_INITIAL_SIZE * block * + sizeof(struct iovec)); + if (NULL == iov) { + opal_output(1, "OUT OF MEMORY\n"); + return OMPI_ERR_OUT_OF_RESOURCE; + } + } + + if (fh->f_num_of_io_entries != i+1) { + if (((OMPI_MPI_OFFSET_TYPE)(intptr_t)fh->f_io_array[sorted[i]].offset + + (OPAL_PTRDIFF_TYPE)fh->f_io_array[sorted[i]].length) == + (OMPI_MPI_OFFSET_TYPE)(intptr_t)fh->f_io_array[sorted[i+1]].offset) { + iov[iov_count].iov_base = + fh->f_io_array[sorted[i+1]].memory_address; + iov[iov_count].iov_len = fh->f_io_array[sorted[i+1]].length; + iov_count ++; + continue; + } + } + + // Find the total number of bytes to be written. + size_t bytes = 0; + for (int i = 0; i < iov_count; ++i) { + bytes += iov[i].iov_len; + } + + // Allocate a temporary buffer to hold the data + char *buffer; + buffer = (char *) malloc (bytes); + if (buffer == NULL) { + return OMPI_ERROR; + } + + // Copy the data into BUFFER. + size_t to_copy = bytes; + char *bp = buffer; + for (int i = 0; i < iov_count; ++i) { + size_t copy = MIN (iov[i].iov_len, to_copy); + + bp = mempcpy ((void *) bp, (void *) iov[i].iov_base, copy); + + to_copy -= copy; + if (to_copy == 0) + break; + } + + // Write the data + ssize_t bytes_written; + + plfs_ret = plfs_write( pfd, buffer, bytes, iov_offset, 0, &bytes_written ); + if (PLFS_SUCCESS != plfs_ret) { + opal_output(0, "fbtl_plfs_pwritev: Error in plfs_write:\n%s\n", strplfserr(plfs_ret)); + return OMPI_ERROR; + } + iov_count = 0; + } + } + else { + for (i=0 ; if_num_of_io_entries ; i++) { + if (0 == iov_count) { + iov[iov_count].iov_base = fh->f_io_array[i].memory_address; + iov[iov_count].iov_len = fh->f_io_array[i].length; + iov_offset = (OMPI_MPI_OFFSET_TYPE)(intptr_t)fh->f_io_array[i].offset; + iov_count ++; + } + + if (OMPIO_IOVEC_INITIAL_SIZE*block <= iov_count) { + block ++; + iov = (struct iovec *)realloc + (iov, OMPIO_IOVEC_INITIAL_SIZE * block * + sizeof(struct iovec)); + if (NULL == iov) { + opal_output(1, "OUT OF MEMORY\n"); + return OMPI_ERR_OUT_OF_RESOURCE; + } + } + + if (fh->f_num_of_io_entries != i+1) { + if (((OMPI_MPI_OFFSET_TYPE)(intptr_t)fh->f_io_array[i].offset + + (OPAL_PTRDIFF_TYPE)fh->f_io_array[i].length) == + (OMPI_MPI_OFFSET_TYPE)(intptr_t)fh->f_io_array[i+1].offset) { + iov[iov_count].iov_base = + fh->f_io_array[i+1].memory_address; + iov[iov_count].iov_len = fh->f_io_array[i+1].length; + iov_count ++; + continue; + } + } + + // Find the total number of bytes to be written. + size_t bytes = 0; + for (int i = 0; i < iov_count; ++i) { + bytes += iov[i].iov_len; + } + + // Allocate a temporary buffer to hold the data + char *buffer; + buffer = (char *) malloc (bytes); + if (buffer == NULL) { + return OMPI_ERROR; + } + + // Copy the data into BUFFER. + size_t to_copy = bytes; + char *bp = buffer; + for (int i = 0; i < iov_count; ++i) { + size_t copy = MIN (iov[i].iov_len, to_copy); + bp = mempcpy ((void *) bp, (void *) iov[i].iov_base, copy); + to_copy -= copy; + if (to_copy == 0) { + break; + } + } + + // Write the data + ssize_t bytes_written; + + plfs_ret = plfs_write( pfd, buffer, bytes, iov_offset, 0, &bytes_written ); + if (PLFS_SUCCESS != plfs_ret) { + opal_output(0, "fbtl_plfs_pwritev: Error in plfs_write:\n%s\n", strplfserr(plfs_ret)); + return OMPI_ERROR; + } + iov_count = 0; + } + } + + if (NULL != iov) { + free (iov); + iov = NULL; + } + + return OMPI_SUCCESS; +} diff --git a/ompi/mca/fs/plfs/Makefile.am b/ompi/mca/fs/plfs/Makefile.am new file mode 100644 index 0000000000..d4128d07d5 --- /dev/null +++ b/ompi/mca/fs/plfs/Makefile.am @@ -0,0 +1,57 @@ +# +# Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana +# University Research and Technology +# Corporation. All rights reserved. +# Copyright (c) 2004-2005 The University of Tennessee and The University +# of Tennessee Research Foundation. All rights +# reserved. +# Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, +# University of Stuttgart. All rights reserved. +# Copyright (c) 2004-2005 The Regents of the University of California. +# All rights reserved. +# Copyright (c) 2008-2014 University of Houston. All rights reserved. +# $COPYRIGHT$ +# +# Additional copyrights may follow +# +# $HEADER$ +# + +# Make the output library in this directory, and name it either +# mca__.la (for DSO builds) or libmca__.la +# (for static builds). + +if MCA_BUILD_ompi_fs_plfs_DSO +component_noinst = +component_install = mca_fs_plfs.la +else +component_noinst = libmca_fs_plfs.la +component_install = +endif + +# Source files + +fs_plfs_sources = \ + fs_plfs.h \ + fs_plfs.c \ + fs_plfs_component.c \ + fs_plfs_file_open.c \ + fs_plfs_file_close.c \ + fs_plfs_file_delete.c \ + fs_plfs_file_sync.c \ + fs_plfs_file_set_size.c \ + fs_plfs_file_get_size.c \ + fs_plfs_file_set_info.c + +AM_CPPFLAGS = $(fs_plfs_CPPFLAGS) + +mcacomponentdir = $(pkglibdir) +mcacomponent_LTLIBRARIES = $(component_install) +mca_fs_plfs_la_SOURCES = $(fs_plfs_sources) +mca_fs_plfs_la_LIBADD = $(fs_plfs_LIBS) +mca_fs_plfs_la_LDFLAGS = -module -avoid-version $(fs_plfs_LDFLAGS) + +noinst_LTLIBRARIES = $(component_noinst) +libmca_fs_plfs_la_SOURCES = $(fs_plfs_sources) +libmca_fs_plfs_la_LIBADD = $(fs_plfs_LIBS) +libmca_fs_plfs_la_LDFLAGS = -module -avoid-version $(fs_plfs_LDFLAGS) diff --git a/ompi/mca/fs/plfs/configure.m4 b/ompi/mca/fs/plfs/configure.m4 new file mode 100644 index 0000000000..7a63540f5c --- /dev/null +++ b/ompi/mca/fs/plfs/configure.m4 @@ -0,0 +1,41 @@ +# -*- shell-script -*- +# +# Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana +# University Research and Technology +# Corporation. All rights reserved. +# Copyright (c) 2004-2005 The University of Tennessee and The University +# of Tennessee Research Foundation. All rights +# reserved. +# Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, +# University of Stuttgart. All rights reserved. +# Copyright (c) 2004-2005 The Regents of the University of California. +# All rights reserved. +# Copyright (c) 2010 Cisco Systems, Inc. All rights reserved. +# Copyright (c) 2008-2014 University of Houston. All rights reserved. +# $COPYRIGHT$ +# +# Additional copyrights may follow +# +# $HEADER$ +# + + +# MCA_fs_plfs_CONFIG(action-if-can-compile, +# [action-if-cant-compile]) +# ------------------------------------------------ +AC_DEFUN([MCA_ompi_fs_plfs_CONFIG],[ + AC_CONFIG_FILES([ompi/mca/fs/plfs/Makefile]) + + OMPI_CHECK_PLFS([fs_plfs], + [fs_plfs_happy="yes"], + [fs_plfs_happy="no"]) + + AS_IF([test "$fs_plfs_happy" = "yes"], + [$1], + [$2]) + + # substitute in the things needed to build plfs + AC_SUBST([fs_plfs_CPPFLAGS]) + AC_SUBST([fs_plfs_LDFLAGS]) + AC_SUBST([fs_plfs_LIBS]) +])dnl diff --git a/ompi/mca/fs/plfs/fs_plfs.c b/ompi/mca/fs/plfs/fs_plfs.c new file mode 100644 index 0000000000..b687003b62 --- /dev/null +++ b/ompi/mca/fs/plfs/fs_plfs.c @@ -0,0 +1,152 @@ +/* + * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana + * University Research and Technology + * Corporation. All rights reserved. + * Copyright (c) 2004-2006 The University of Tennessee and The University + * of Tennessee Research Foundation. All rights + * reserved. + * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, + * University of Stuttgart. All rights reserved. + * Copyright (c) 2004-2005 The Regents of the University of California. + * All rights reserved. + * Copyright (c) 2008-2014 University of Houston. All rights reserved. + * $COPYRIGHT$ + * + * Additional copyrights may follow + * + * $HEADER$ + * + * These symbols are in a file by themselves to provide nice linker + * semantics. Since linkers generally pull in symbols by object fules, + * keeping these symbols as the only symbols in this file prevents + * utility programs such as "ompi_info" from having to import entire + * modules just to query their version and parameters + */ + +#include "ompi_config.h" +#include "mpi.h" +#include "ompi/mca/fs/fs.h" +#include "ompi/mca/fs/base/base.h" +#include "ompi/mca/fs/plfs/fs_plfs.h" + +#ifdef HAVE_SYS_STATFS_H +#include /* or */ +#endif +#ifdef HAVE_SYS_PARAM_H +#include +#endif +#ifdef HAVE_SYS_MOUNT_H +#include +#endif +#ifdef HAVE_SYS_STAT_H +#include +#endif + +#include +#include + +/* + * ******************************************************************* + * ************************ actions structure ************************ + * ******************************************************************* + */ +static mca_fs_base_module_1_0_0_t plfs = { + mca_fs_plfs_module_init, /* initalise after being selected */ + mca_fs_plfs_module_finalize, /* close a module on a communicator */ + mca_fs_plfs_file_open, + mca_fs_plfs_file_close, + mca_fs_plfs_file_delete, + mca_fs_plfs_file_set_size, + mca_fs_plfs_file_get_size, + mca_fs_plfs_file_set_info, + mca_fs_plfs_file_sync +}; +/* + * ******************************************************************* + * ************************* structure ends ************************** + * ******************************************************************* + */ + +int mca_fs_plfs_component_init_query(bool enable_progress_threads, + bool enable_mpi_threads) +{ + /* Nothing to do */ + return OMPI_SUCCESS; +} + +struct mca_fs_base_module_1_0_0_t * +mca_fs_plfs_component_file_query (mca_io_ompio_file_t *fh, int *priority) +{ + int err; + char *dir; + struct statfs fsbuf; + char *tmp; + char wpath[1024]; + + /* The code in this function is based on the ADIO FS selection in ROMIO + * Copyright (C) 1997 University of Chicago. + * See COPYRIGHT notice in top-level directory. + */ + + *priority = mca_fs_plfs_priority; + + tmp = strchr (fh->f_filename, ':'); + if (!tmp) { + if (OMPIO_ROOT == fh->f_rank) { + do { + err = statfs (fh->f_filename, &fsbuf); + } while (err && (errno == ESTALE)); + + if (err && (ENOENT == errno)) { + mca_fs_base_get_parent_dir (fh->f_filename, &dir); + err = statfs (dir, &fsbuf); + free (dir); + } + + getcwd( wpath, sizeof(wpath) ); + if(is_plfs_path(wpath) == 1) { + fh->f_fstype = PLFS; + } + } + fh->f_comm->c_coll.coll_bcast (&(fh->f_fstype), + 1, + MPI_INT, + OMPIO_ROOT, + fh->f_comm, + fh->f_comm->c_coll.coll_bcast_module); + } + else { + if (!strncmp(fh->f_filename, "plfs:", 7) || + !strncmp(fh->f_filename, "PLFS:", 7)) { + fh->f_fstype = PLFS; + } + } + + if (PLFS == fh->f_fstype) { + if (*priority < 50) { + *priority = 50; + return &plfs; + } + } + return NULL; +} + +int mca_fs_plfs_component_file_unquery (mca_io_ompio_file_t *file) +{ + /* This function might be needed for some purposes later. for now it + * does not have anything to do since there are no steps which need + * to be undone if this module is not selected */ + + return OMPI_SUCCESS; +} + +int mca_fs_plfs_module_init (mca_io_ompio_file_t *file) +{ + return OMPI_SUCCESS; +} + + +int mca_fs_plfs_module_finalize (mca_io_ompio_file_t *file) +{ + return OMPI_SUCCESS; +} diff --git a/ompi/mca/fs/plfs/fs_plfs.h b/ompi/mca/fs/plfs/fs_plfs.h new file mode 100644 index 0000000000..0e8504c453 --- /dev/null +++ b/ompi/mca/fs/plfs/fs_plfs.h @@ -0,0 +1,84 @@ +/* + * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana + * University Research and Technology + * Corporation. All rights reserved. + * Copyright (c) 2004-2006 The University of Tennessee and The University + * of Tennessee Research Foundation. All rights + * reserved. + * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, + * University of Stuttgart. All rights reserved. + * Copyright (c) 2004-2005 The Regents of the University of California. + * All rights reserved. + * Copyright (c) 2008-2014 University of Houston. All rights reserved. + * $COPYRIGHT$ + * + * Additional copyrights may follow + * + * $HEADER$ + */ + +#ifndef MCA_FS_PLFS_H +#define MCA_FS_PLFS_H + +#include "ompi_config.h" +#include "opal/mca/mca.h" +#include "ompi/mca/fs/fs.h" +#include "ompi/mca/io/ompio/io_ompio.h" + +#include + +extern int mca_fs_plfs_priority; +extern int mca_fs_plfs_num_hostdir; + +BEGIN_C_DECLS + +int mca_fs_plfs_component_init_query(bool enable_progress_threads, + bool enable_mpi_threads); +struct mca_fs_base_module_1_0_0_t * +mca_fs_plfs_component_file_query (mca_io_ompio_file_t *fh, int *priority); +int mca_fs_plfs_component_file_unquery (mca_io_ompio_file_t *file); + +int mca_fs_plfs_module_init (mca_io_ompio_file_t *file); +int mca_fs_plfs_module_finalize (mca_io_ompio_file_t *file); + +OMPI_MODULE_DECLSPEC extern mca_fs_base_component_2_0_0_t mca_fs_plfs_component; +/* + * ****************************************************************** + * ********* functions which are implemented in this module ********* + * ****************************************************************** + */ + +int mca_fs_plfs_file_open (struct ompi_communicator_t *comm, + char *filename, + int amode, + struct ompi_info_t *info, + mca_io_ompio_file_t *fh); + +int mca_fs_plfs_file_close (mca_io_ompio_file_t *fh); + +int mca_fs_plfs_file_delete (char *filename, + struct ompi_info_t *info); + +int mca_fs_plfs_file_set_size (mca_io_ompio_file_t *fh, + OMPI_MPI_OFFSET_TYPE size); + +int mca_fs_plfs_file_get_size (mca_io_ompio_file_t *fh, + OMPI_MPI_OFFSET_TYPE *size); + +int mca_fs_plfs_file_set_info (mca_io_ompio_file_t *fh, + struct ompi_info_t *info); + +int mca_fs_plfs_file_sync (mca_io_ompio_file_t *fh); + +int mca_fs_plfs_file_seek (mca_io_ompio_file_t *fh, + OMPI_MPI_OFFSET_TYPE offset, + int whence); +/* + * ****************************************************************** + * ************ functions implemented in this module end ************ + * ****************************************************************** + */ + +END_C_DECLS + +#endif /* MCA_FS_PLFS_H */ diff --git a/ompi/mca/fs/plfs/fs_plfs_component.c b/ompi/mca/fs/plfs/fs_plfs_component.c new file mode 100644 index 0000000000..22dd773e23 --- /dev/null +++ b/ompi/mca/fs/plfs/fs_plfs_component.c @@ -0,0 +1,86 @@ +/* + * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana + * University Research and Technology + * Corporation. All rights reserved. + * Copyright (c) 2004-2005 The University of Tennessee and The University + * of Tennessee Research Foundation. All rights + * reserved. + * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, + * University of Stuttgart. All rights reserved. + * Copyright (c) 2004-2005 The Regents of the University of California. + * All rights reserved. + * Copyright (c) 2008-2014 University of Houston. All rights reserved. + * $COPYRIGHT$ + * + * Additional copyrights may follow + * + * $HEADER$ + * + * These symbols are in a file by themselves to provide nice linker + * semantics. Since linkers generally pull in symbols by object + * files, keeping these symbols as the only symbols in this file + * prevents utility programs such as "ompi_info" from having to import + * entire components just to query their version and parameters. + */ + +#include "ompi_config.h" +#include "fs_plfs.h" +#include "mpi.h" + +/* + * Public string showing the fs plfs component version number + */ +const char *mca_fs_plfs_component_version_string = + "OMPI/MPI plfs FS MCA component version " OMPI_VERSION; + +static int plfs_register(void); + +int mca_fs_plfs_priority = 20; +int mca_fs_plfs_num_hostdir = -1; + +/* + * Instantiate the public struct with all of our public information + * and pointers to our public functions in it + */ +mca_fs_base_component_2_0_0_t mca_fs_plfs_component = { + + /* First, the mca_component_t struct containing meta information + about the component itself */ + + { + MCA_FS_BASE_VERSION_2_0_0, + + /* Component name and version */ + "plfs", + OMPI_MAJOR_VERSION, + OMPI_MINOR_VERSION, + OMPI_RELEASE_VERSION, + .mca_register_component_params = plfs_register, + }, + { + /* This component is checkpointable */ + MCA_BASE_METADATA_PARAM_CHECKPOINT + }, + mca_fs_plfs_component_init_query, /* get thread level */ + mca_fs_plfs_component_file_query, /* get priority and actions */ + mca_fs_plfs_component_file_unquery /* undo what was done by previous function */ +}; + +static int +plfs_register(void) +{ + mca_fs_plfs_priority = 20; + (void) mca_base_component_var_register(&mca_fs_plfs_component.fsm_version, + "priority", "Priority of the plfs fs component", + MCA_BASE_VAR_TYPE_INT, NULL, 0, 0, + OPAL_INFO_LVL_9, + MCA_BASE_VAR_SCOPE_READONLY, &mca_fs_plfs_priority); + mca_fs_plfs_num_hostdir = -1; + (void) mca_base_component_var_register(&mca_fs_plfs_component.fsm_version, + "num_hostdir", "number of host directories of a file over plfs", + MCA_BASE_VAR_TYPE_INT, NULL, 0, 0, + OPAL_INFO_LVL_9, + MCA_BASE_VAR_SCOPE_READONLY, &mca_fs_plfs_num_hostdir); + + return OMPI_SUCCESS; +} diff --git a/ompi/mca/fs/plfs/fs_plfs_file_close.c b/ompi/mca/fs/plfs/fs_plfs_file_close.c new file mode 100644 index 0000000000..83109f1e62 --- /dev/null +++ b/ompi/mca/fs/plfs/fs_plfs_file_close.c @@ -0,0 +1,75 @@ +/* + * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana + * University Research and Technology + * Corporation. All rights reserved. + * Copyright (c) 2004-2011 The University of Tennessee and The University + * of Tennessee Research Foundation. All rights + * reserved. + * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, + * University of Stuttgart. All rights reserved. + * Copyright (c) 2004-2005 The Regents of the University of California. + * All rights reserved. + * Copyright (c) 2008-2014 University of Houston. All rights reserved. + * $COPYRIGHT$ + * + * Additional copyrights may follow + * + * $HEADER$ + */ + + +#include "ompi_config.h" +#include "fs_plfs.h" + +#include +#include +#include "mpi.h" +#include "ompi/constants.h" +#include "ompi/mca/fs/fs.h" + + +/* + * file_close_plfs + * + * Function: - closes a new file + * Accepts: - file handle + * Returns: - Success if file closed + */ +int +mca_fs_plfs_file_close (mca_io_ompio_file_t *fh) +{ + int flags; + plfs_error_t plfs_ret = PLFS_SUCCESS; + int amode; + char wpath[1024]; + + fh->f_comm->c_coll.coll_barrier (fh->f_comm, + fh->f_comm->c_coll.coll_barrier_module); + + getcwd( wpath, sizeof(wpath) ); + sprintf( wpath,"%s/%s",wpath,fh->f_filename ); + + if(-1 == access(fh->f_filename, F_OK)) { + return OMPI_ERROR; // file doesn't exist + } + + amode = 0; + if (fh->f_amode & MPI_MODE_CREATE) + amode = amode | O_CREAT; + if (fh->f_amode & MPI_MODE_RDONLY) + amode = amode | O_RDONLY; + if (fh->f_amode & MPI_MODE_WRONLY) + amode = amode | O_WRONLY; + if (fh->f_amode & MPI_MODE_RDWR) + amode = amode | O_RDWR; + if (fh->f_amode & MPI_MODE_EXCL) { + return OMPI_ERROR; + } + + plfs_ret = plfs_close(fh->f_fs_ptr, 0, 0, amode ,NULL, &flags); + if (PLFS_SUCCESS != plfs_ret) { + opal_output(0, "fs_plfs_file_close: Error in plfs_close:\n%s\n", strplfserr(plfs_ret)); + return OMPI_ERROR; + } + return OMPI_SUCCESS; +} diff --git a/ompi/mca/fs/plfs/fs_plfs_file_delete.c b/ompi/mca/fs/plfs/fs_plfs_file_delete.c new file mode 100644 index 0000000000..aad9ec4456 --- /dev/null +++ b/ompi/mca/fs/plfs/fs_plfs_file_delete.c @@ -0,0 +1,50 @@ +/* + * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana + * University Research and Technology + * Corporation. All rights reserved. + * Copyright (c) 2004-2011 The University of Tennessee and The University + * of Tennessee Research Foundation. All rights + * reserved. + * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, + * University of Stuttgart. All rights reserved. + * Copyright (c) 2004-2005 The Regents of the University of California. + * All rights reserved. + * Copyright (c) 2008-2014 University of Houston. All rights reserved. + * $COPYRIGHT$ + * + * Additional copyrights may follow + * + * $HEADER$ + */ + + +#include "ompi_config.h" +#include "fs_plfs.h" +#include + +#include "mpi.h" +#include "ompi/constants.h" +#include "ompi/mca/fs/fs.h" + +/* + * file_delete_plfs + * + * Function: - deletes a file + * Accepts: - file name & info + * Returns: - Success if file closed + */ +int +mca_fs_plfs_file_delete (char* file_name, + struct ompi_info_t *info) +{ + plfs_error_t plfs_ret; + char wpath[1024]; + getcwd( wpath, sizeof(wpath) ); + sprintf( wpath,"%s/%s",wpath,file_name ); + plfs_ret = plfs_unlink( wpath ); + if (PLFS_SUCCESS != plfs_ret) { + opal_output(0, "fs_plfs_file_delete: Error in plfs_unlink:\n%s\n", strplfserr(plfs_ret)); + return OMPI_ERROR; + } + return OMPI_SUCCESS; +} diff --git a/ompi/mca/fs/plfs/fs_plfs_file_get_size.c b/ompi/mca/fs/plfs/fs_plfs_file_get_size.c new file mode 100644 index 0000000000..e27284d569 --- /dev/null +++ b/ompi/mca/fs/plfs/fs_plfs_file_get_size.c @@ -0,0 +1,56 @@ +/* + * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana + * University Research and Technology + * Corporation. All rights reserved. + * Copyright (c) 2004-2011 The University of Tennessee and The University + * of Tennessee Research Foundation. All rights + * reserved. + * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, + * University of Stuttgart. All rights reserved. + * Copyright (c) 2004-2005 The Regents of the University of California. + * All rights reserved. + * Copyright (c) 2008-2014 University of Houston. All rights reserved. + * $COPYRIGHT$ + * + * Additional copyrights may follow + * + * $HEADER$ + */ + + +#include "ompi_config.h" +#include "fs_plfs.h" + +#include "mpi.h" +#include "ompi/constants.h" +#include "ompi/mca/fs/fs.h" + +/* + * file_get_size_plfs + * + * Function: - get_size of a file + * Accepts: - same arguments as MPI_File_get_size() + * Returns: - Success if size is get + */ +int +mca_fs_plfs_file_get_size (mca_io_ompio_file_t *fh, + OMPI_MPI_OFFSET_TYPE *size) +{ + Plfs_fd *pfd = NULL; + plfs_error_t plfs_ret; + struct stat st; + char wpath[1024]; + int size_only = 1; + + getcwd(wpath, sizeof(wpath)); + sprintf(wpath,"%s/%s",wpath,fh->f_filename); + + plfs_ret = plfs_getattr(pfd, wpath, &st, size_only); + if (PLFS_SUCCESS != plfs_ret) { + opal_output(0, "fs_plfs_file_get_size: Error in plfs_getattr:\n%s\n", strplfserr(plfs_ret)); + return OMPI_ERROR; + } + + *size = st.st_size; + return OMPI_SUCCESS; +} diff --git a/ompi/mca/fs/plfs/fs_plfs_file_open.c b/ompi/mca/fs/plfs/fs_plfs_file_open.c new file mode 100644 index 0000000000..a333a477bc --- /dev/null +++ b/ompi/mca/fs/plfs/fs_plfs_file_open.c @@ -0,0 +1,118 @@ +/* + * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana + * University Research and Technology + * Corporation. All rights reserved. + * Copyright (c) 2004-2011 The University of Tennessee and The University + * of Tennessee Research Foundation. All rights + * reserved. + * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, + * University of Stuttgart. All rights reserved. + * Copyright (c) 2004-2005 The Regents of the University of California. + * All rights reserved. + * Copyright (c) 2008-2014 University of Houston. All rights reserved. + * $COPYRIGHT$ + * + * Additional copyrights may follow + * + * $HEADER$ + */ + + +#include "ompi_config.h" +#include "fs_plfs.h" + +#include +#include +#include "mpi.h" +#include "ompi/constants.h" +#include "ompi/mca/fs/fs.h" +#include "ompi/communicator/communicator.h" +#include "ompi/info/info.h" + +#include + +/* + * file_open_plfs + * + * Function: - opens a new file + * Accepts: - same arguments as MPI_File_open() + * Returns: - Success if new file handle + */ +int +mca_fs_plfs_file_open (struct ompi_communicator_t *comm, + char* filename, + int access_mode, + struct ompi_info_t *info, + mca_io_ompio_file_t *fh) +{ + int rank; + int amode; + int old_mask, perm; + plfs_error_t plfs_ret; + Plfs_fd *pfd = NULL; + char wpath[1024]; + size_t len = sizeof(int); + char key[] = "num_hostdirs"; + + rank = ompi_comm_rank ( comm ); + + getcwd( wpath, sizeof(wpath) ); + sprintf( wpath,"%s/%s",wpath,filename ); + + if (OMPIO_PERM_NULL == fh->f_perm) { + old_mask = umask(022); + umask(old_mask); + perm = old_mask ^ 0666; + } + else { + perm = fh->f_perm; + } + + amode = 0; + + if (access_mode & MPI_MODE_RDONLY) + amode = amode | O_RDONLY; + if (access_mode & MPI_MODE_WRONLY) + amode = amode | O_WRONLY; + if (access_mode & MPI_MODE_RDWR) + amode = amode | O_RDWR; + if (access_mode & MPI_MODE_EXCL) { + if( is_plfs_path(wpath) == 1 ) { //the file already exists + return OMPI_ERROR; + } + } + + if (0 == rank) { + /* MODE_CREATE and MODE_EXCL can only be set by one process */ + if (access_mode & MPI_MODE_CREATE) + amode = amode | O_CREAT; + + plfs_ret = plfs_open( &pfd, wpath, amode, 0, perm, NULL ); + fh->f_fs_ptr = pfd; + } + + comm->c_coll.coll_bcast ( &plfs_ret, 1, MPI_INT, 0, comm, comm->c_coll.coll_bcast_module); + if ( PLFS_SUCCESS != plfs_ret ) { + return OMPI_ERROR; + } + + if (0 != rank) { + plfs_ret = plfs_open( &pfd, wpath, amode, 0, perm, NULL ); + if (PLFS_SUCCESS != plfs_ret) { + opal_output(0, "fs_plfs_file_open: Error in plfs_open:\n%s\n", strplfserr(plfs_ret)); + return OMPI_ERROR; + } + else { + fh->f_fs_ptr = pfd; + } + } + + if (mca_fs_plfs_num_hostdir > 0) { + plfs_ret = plfs_setxattr( pfd, &mca_fs_plfs_num_hostdir, key, len ); + if (PLFS_SUCCESS != plfs_ret) { + opal_output(0, "fs_plfs_file_open: Error in plfs_setxattr:\n%s\n", strplfserr(plfs_ret)); + return OMPI_ERROR; + } + } + return OMPI_SUCCESS; +} diff --git a/ompi/mca/fs/plfs/fs_plfs_file_set_info.c b/ompi/mca/fs/plfs/fs_plfs_file_set_info.c new file mode 100644 index 0000000000..3a80b650d7 --- /dev/null +++ b/ompi/mca/fs/plfs/fs_plfs_file_set_info.c @@ -0,0 +1,41 @@ +/* + * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana + * University Research and Technology + * Corporation. All rights reserved. + * Copyright (c) 2004-2011 The University of Tennessee and The University + * of Tennessee Research Foundation. All rights + * reserved. + * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, + * University of Stuttgart. All rights reserved. + * Copyright (c) 2004-2005 The Regents of the University of California. + * All rights reserved. + * Copyright (c) 2008-2014 University of Houston. All rights reserved. + * $COPYRIGHT$ + * + * Additional copyrights may follow + * + * $HEADER$ + */ + + +#include "ompi_config.h" +#include "fs_plfs.h" + +#include "mpi.h" +#include "ompi/constants.h" +#include "ompi/mca/fs/fs.h" + +/* + * file_set_info_plfs + * + * Function: - set_info of a file + * Accepts: - same arguments as MPI_File_set_info() + * Returns: - Success if info is set + */ +int +mca_fs_plfs_file_set_info (mca_io_ompio_file_t *file_handle, + struct ompi_info_t *info) +{ + printf ("PLFS SET INFO\n"); + return OMPI_SUCCESS; +} diff --git a/ompi/mca/fs/plfs/fs_plfs_file_set_size.c b/ompi/mca/fs/plfs/fs_plfs_file_set_size.c new file mode 100644 index 0000000000..8ab6aeb45c --- /dev/null +++ b/ompi/mca/fs/plfs/fs_plfs_file_set_size.c @@ -0,0 +1,41 @@ +/* + * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana + * University Research and Technology + * Corporation. All rights reserved. + * Copyright (c) 2004-2011 The University of Tennessee and The University + * of Tennessee Research Foundation. All rights + * reserved. + * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, + * University of Stuttgart. All rights reserved. + * Copyright (c) 2004-2005 The Regents of the University of California. + * All rights reserved. + * Copyright (c) 2008-2011 University of Houston. All rights reserved. + * $COPYRIGHT$ + * + * Additional copyrights may follow + * + * $HEADER$ + */ + + +#include "ompi_config.h" +#include "fs_plfs.h" + +#include "mpi.h" +#include "ompi/constants.h" +#include "ompi/mca/fs/fs.h" + +/* + * file_set_size_plfs + * + * Function: - set_size of a file + * Accepts: - same arguments as MPI_File_set_size() + * Returns: - Success if size is set + */ +int +mca_fs_plfs_file_set_size (mca_io_ompio_file_t *file_handle, + OMPI_MPI_OFFSET_TYPE size) +{ + printf ("PLFS SET SIZE\n"); + return OMPI_SUCCESS; +} diff --git a/ompi/mca/fs/plfs/fs_plfs_file_sync.c b/ompi/mca/fs/plfs/fs_plfs_file_sync.c new file mode 100644 index 0000000000..c08cba1a25 --- /dev/null +++ b/ompi/mca/fs/plfs/fs_plfs_file_sync.c @@ -0,0 +1,45 @@ +/* + * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana + * University Research and Technology + * Corporation. All rights reserved. + * Copyright (c) 2004-2011 The University of Tennessee and The University + * of Tennessee Research Foundation. All rights + * reserved. + * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, + * University of Stuttgart. All rights reserved. + * Copyright (c) 2004-2005 The Regents of the University of California. + * All rights reserved. + * Copyright (c) 2008-2014 University of Houston. All rights reserved. + * $COPYRIGHT$ + * + * Additional copyrights may follow + * + * $HEADER$ + */ + + +#include "ompi_config.h" +#include "fs_plfs.h" + +#include "mpi.h" +#include "ompi/constants.h" +#include "ompi/mca/fs/fs.h" + +/* + * file_sync_plfs + * + * Function: - closes a new file + * Accepts: - file handle + * Returns: - Success if file closed + */ +int +mca_fs_plfs_file_sync (mca_io_ompio_file_t *fh) +{ + plfs_error_t plfs_ret; + plfs_ret = plfs_sync( fh->f_fs_ptr ); + if (PLFS_SUCCESS != plfs_ret) { + opal_output(0, "fs_plfs_file_sync: Error in plfs_sync:\n%s\n", strplfserr(plfs_ret)); + return OMPI_ERROR; + } + return OMPI_SUCCESS; +} diff --git a/ompi/mca/io/ompio/io_ompio.h b/ompi/mca/io/ompio/io_ompio.h index 1ad6c533ce..2852e7f43a 100644 --- a/ompi/mca/io/ompio/io_ompio.h +++ b/ompi/mca/io/ompio/io_ompio.h @@ -94,7 +94,8 @@ enum ompio_fs_type NONE = 0, UFS = 1, PVFS2 = 2, - LUSTRE = 3 + LUSTRE = 3, + PLFS = 4 }; OMPI_DECLSPEC extern mca_io_base_component_2_0_0_t mca_io_ompio_component;