Merge pull request #1990 from edgargabriel/topic/mt-io

steps towards making file I/O operations thread safe

ompi/file/file.c

@@ -14,6 +14,7 @@
  * Copyright (c) 2009-2012 Cisco Systems, Inc.  All rights reserved.
  * Copyright (c) 2015      Research Organization for Information Science
  *                         and Technology (RIST). All rights reserved.
+ * Copyright (c) 2016      University of Houston. All rights reserved.
  * $COPYRIGHT$
  *
  * Additional copyrights may follow
@@ -31,6 +32,10 @@
 #include "ompi/mca/io/base/base.h"
 #include "ompi/info/info.h"
 
+
+opal_mutex_t ompi_mpi_file_bootstrap_mutex = OPAL_MUTEX_STATIC_INIT;
+
+
 /*
  * Table for Fortran <-> C file handle conversion
  */
@@ -102,6 +107,7 @@ int ompi_file_open(struct ompi_communicator_t *comm, const char *filename,
         return OMPI_ERR_OUT_OF_RESOURCE;
     }
 
+
     /* Save the params */
 
     file->f_comm = comm;
@@ -127,6 +133,9 @@ int ompi_file_open(struct ompi_communicator_t *comm, const char *filename,
         return OMPI_ERR_OUT_OF_RESOURCE;
     }
 
+    /* Create the mutex */
+    OBJ_CONSTRUCT(&file->f_mutex, opal_mutex_t);
+
     /* Select a module and actually open the file */
 
     if (OMPI_SUCCESS != (ret = mca_io_base_file_select(file, NULL))) {
@@ -146,6 +155,9 @@ int ompi_file_open(struct ompi_communicator_t *comm, const char *filename,
  */
 int ompi_file_close(ompi_file_t **file)
 {
+
+    OBJ_DESTRUCT(&(*file)->f_mutex);
+
     (*file)->f_flags |= OMPI_FILE_ISCLOSED;
     OBJ_RELEASE(*file);
     *file = &ompi_mpi_file_null.file;

ompi/file/file.h

@@ -14,6 +14,7 @@
  * Copyright (c) 2009-2012 Cisco Systems, Inc.  All rights reserved.
  * Copyright (c) 2015      Research Organization for Information Science
  *                         and Technology (RIST). All rights reserved.
+ * Copyright (c) 2016      University of Houston. All rights reserved.
  * $COPYRIGHT$
  *
  * Additional copyrights may follow
@@ -78,6 +79,10 @@ struct ompi_file_t {
         indicates what member to look at in the union, below) */
     mca_io_base_version_t f_io_version;
 
+    /** Mutex to be used to protect access to the selected component
+        on a per file-handle basis */
+    opal_mutex_t f_mutex;
+
     /** The selected component (note that this is a union) -- we need
         this to add and remove the component from the list of
         components currently in use by the io framework for

ompi/mca/io/base/io_base_file_select.c

@@ -46,6 +46,7 @@
 #include "ompi/mca/sharedfp/sharedfp.h"
 #include "ompi/mca/sharedfp/base/base.h"
 
+opal_mutex_t ompi_mpi_ompio_bootstrap_mutex = OPAL_MUTEX_STATIC_INIT;
 /*
  * Local types
  */
@@ -201,18 +202,24 @@ int mca_io_base_file_select(ompi_file_t *file,
                  "ompio")) {
         int ret;
 
+        opal_mutex_lock(&ompi_mpi_ompio_bootstrap_mutex);
         if (OMPI_SUCCESS != (ret = mca_base_framework_open(&ompi_fs_base_framework, 0))) {
+            opal_mutex_unlock(&ompi_mpi_ompio_bootstrap_mutex);
             return err;
         }
         if (OMPI_SUCCESS != (ret = mca_base_framework_open(&ompi_fcoll_base_framework, 0))) {
+            opal_mutex_unlock(&ompi_mpi_ompio_bootstrap_mutex);
             return err;
         }
         if (OMPI_SUCCESS != (ret = mca_base_framework_open(&ompi_fbtl_base_framework, 0))) {
+            opal_mutex_unlock(&ompi_mpi_ompio_bootstrap_mutex);
             return err;
         }
         if (OMPI_SUCCESS != (ret = mca_base_framework_open(&ompi_sharedfp_base_framework, 0))) {
+            opal_mutex_unlock(&ompi_mpi_ompio_bootstrap_mutex);
             return err;
         }
+        opal_mutex_unlock(&ompi_mpi_ompio_bootstrap_mutex);
 
         if (OMPI_SUCCESS !=
             (ret = mca_fs_base_find_available(OPAL_ENABLE_PROGRESS_THREADS, 1))) {
@@ -230,6 +237,7 @@ int mca_io_base_file_select(ompi_file_t *file,
             (ret = mca_sharedfp_base_find_available(OPAL_ENABLE_PROGRESS_THREADS, 1))) {
             return err;
         }
+
     }
     /* Finally -- intialize the selected module. */
 

ompi/mca/io/romio314/src/io_romio314_file_open.c

@@ -38,10 +38,10 @@ mca_io_romio314_file_open (ompi_communicator_t *comm,
     mca_io_romio314_data_t *data;
 
     data = (mca_io_romio314_data_t *) fh->f_io_selected_data;
-    OPAL_THREAD_LOCK (&mca_io_romio314_mutex);
+//    OPAL_THREAD_LOCK (&mca_io_romio314_mutex);
     ret = ROMIO_PREFIX(MPI_File_open)(comm, filename, amode, info,
                                       &data->romio_fh);
-    OPAL_THREAD_UNLOCK (&mca_io_romio314_mutex);
+//    OPAL_THREAD_UNLOCK (&mca_io_romio314_mutex);
 
     return ret;
 }

ompi/mpi/c/file_open.c

@@ -15,6 +15,7 @@
  *                         reserved.
  * Copyright (c) 2015      Research Organization for Information Science
  *                         and Technology (RIST). All rights reserved.
+ * Copyright (c) 2016      University of Houston. All rights reserved.
  * $COPYRIGHT$
  *
  * Additional copyrights may follow
@@ -34,6 +35,9 @@
 #include "ompi/mca/io/base/base.h"
 #include "ompi/memchecker.h"
 
+
+extern opal_mutex_t ompi_mpi_file_bootstrap_mutex;
+
 #if OMPI_BUILD_MPI_PROFILING
 #if OPAL_HAVE_WEAK_SYMBOLS
 #pragma weak MPI_File_open = PMPI_File_open
@@ -81,10 +85,20 @@ int MPI_File_open(MPI_Comm comm, const char *filename, int amode,
        and MPI_FILE_DELETE are the only two places that it will be
        initialized). */
 
+    /* For multi-threaded scenarios, initializing the file i/o
+       framework and mca infrastructure needs to be protected
+       by a mutex, similarly to the other frameworks in
+       ompi/runtime/ompi_mpi_init.c
+    */
+
+    opal_mutex_lock(&ompi_mpi_file_bootstrap_mutex);
+
     rc = mca_base_framework_open(&ompi_io_base_framework, 0);
     if (OMPI_SUCCESS != rc) {
+        opal_mutex_unlock(&ompi_mpi_file_bootstrap_mutex);
         return OMPI_ERRHANDLER_INVOKE(MPI_FILE_NULL, rc, FUNC_NAME);
     }
+    opal_mutex_unlock(&ompi_mpi_file_bootstrap_mutex);
 
     OPAL_CR_ENTER_LIBRARY();
 
@@ -96,6 +110,5 @@ int MPI_File_open(MPI_Comm comm, const char *filename, int amode,
     /* Creating the file handle also selects a component to use,
        creates a module, and calls file_open() on the module.  So
        we're good to go. */
-
     OMPI_ERRHANDLER_RETURN(rc, *fh, rc, FUNC_NAME);
 }