fbtl/posix: first cut in adding locking support

Signed-off-by: Edgar Gabriel <egabriel@central.uh.edu>

ompi/mca/fbtl/posix/Makefile.am

@@ -9,7 +9,7 @@
 #                         University of Stuttgart.  All rights reserved.
 # Copyright (c) 2004-2005 The Regents of the University of California.
 #                         All rights reserved.
-# Copyright (c) 2008-2011 University of Houston. All rights reserved.
+# Copyright (c) 2008-2017 University of Houston. All rights reserved.
 # Copyright (c) 2017      IBM Corporation.  All rights reserved.
 # $COPYRIGHT$
 #
@@ -49,4 +49,5 @@ sources = \
         fbtl_posix_preadv.c \
         fbtl_posix_ipreadv.c \
         fbtl_posix_pwritev.c \
-        fbtl_posix_ipwritev.c
+        fbtl_posix_ipwritev.c \
+	fbtl_posix_lock.c

ompi/mca/fbtl/posix/fbtl_posix_lock.c

@@ -0,0 +1,114 @@
+/*
+ * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2011 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2017      University of Houston. All rights reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+
+#include "ompi_config.h"
+#include "fbtl_posix.h"
+
+#include "mpi.h"
+#include <unistd.h>
+#include <sys/uio.h>
+#include <limits.h>
+#include "ompi/constants.h"
+#include "ompi/mca/fbtl/fbtl.h"
+
+
+
+/*
+  op:    can be F_WRLCK or F_RDLCK
+  flags: can be OMPIO_LOCK_ENTIRE_REGION or OMPIO_LOCK_SELECTIVE
+*/
+
+int mca_fbtl_posix_lock ( struct flock *lock, mca_io_ompio_file_t *fh, int op, 
+                          OMPI_MPI_OFFSET_TYPE iov_offset, off_t len, int flags)
+{
+    off_t lmod, bmod;
+
+    lock->l_type   = op;
+    lock->l_whence = SEEK_SET;
+    lock->l_start  =-1;
+    lock->l_len    =-1;
+    if ( fh->f_atomicity ||
+         fh->f_flags & OMPIO_LOCK_ALWAYS ) {
+        /* Need to lock the entire region */
+        lock->l_start = (off_t) iov_offset;
+        lock->l_len   = len;
+    }  
+    else {
+        if ( (fh->f_flags & OMPIO_LOCK_NEVER) ||
+             (fh->f_flags & OMPIO_LOCK_NOT_THIS_OP )){
+            /* OMPIO_LOCK_NEVER:
+                 ompio tells us not to worry about locking. This can be due to three
+                 reasons:
+                 1. user enforced
+                 2. single node job where the locking is handled already in the kernel
+                 3. file view is set to distinct regions such that multiple processes
+                    do not collide on the block level. ( not entirely sure yet how
+                    to check for this except in trivial cases).
+               OMPI_LOCK_NOT_THIS_OP:
+                 will typically be set by fcoll components indicating that the file partitioning
+                 ensures no overlap in blocks.
+            */
+            return 0;
+        }
+        if ( OMPIO_LOCK_ENTIRE_REGION ) {
+            lock->l_start = (off_t) iov_offset;
+            lock->l_len   = len;            
+        }
+        else {
+            /* We only try to lock the first block in the data range if
+               the starting offset is not the starting offset of a file system 
+               block. And the last block in the data range if the offset+len
+               is not equal to the end of a file system block.
+               If we need to lock both beginning + end, we combine 
+               the two into a single lock.
+            */
+            bmod = offset % fh->f_fs_block_size; 
+            if ( !bmod  ) {
+                lock->l_start = (off_t) iov_offset;
+                lock->l_len   = fh->f_fs_block_size - bmod;
+            }
+            lmod = (offset+len-1)%fh->f_fs_block_size;
+            if ( !lmod ) {
+                if ( bmod ) {
+                    lock->l_start = (offset+len-lmod );
+                    lock->l_len   = lmod;
+                }
+                else {
+                    lock->l_len = len;
+                }
+            }
+            if ( -1 == lock->l_start && -1 == lock->l_len ) {
+                /* no need to lock in this instance */
+                return 0;
+            }
+        }
+    }
+
+    return (fcntl ( fh->fd, F_SETLKW, lock));     
+}
+
+int mca_fbtl_posix_unlock ( struct flock *lock, mca_io_ompio_file_t *fh )
+{
+    if ( -1 == lock->l_start && -1 == lock->l_len ) {
+        return 0;
+    }
+    
+    lock->l_type = F_UNLCK;
+    return (fcntl ( fh->fd, F_SETLK, lock));     
+}

ompi/mca/fbtl/posix/fbtl_posix_preadv.c

@@ -9,7 +9,7 @@
  *                         University of Stuttgart.  All rights reserved.
  * Copyright (c) 2004-2005 The Regents of the University of California.
  *                         All rights reserved.
- * Copyright (c) 2008-2014 University of Houston. All rights reserved.
+ * Copyright (c) 2008-2017 University of Houston. All rights reserved.
  * Copyright (c) 2015-2017 Research Organization for Information Science
  *                         and Technology (RIST). All rights reserved.
  * $COPYRIGHT$
@@ -36,6 +36,8 @@ ssize_t mca_fbtl_posix_preadv (mca_io_ompio_file_t *fh )
     int iov_count = 0;
     OMPI_MPI_OFFSET_TYPE iov_offset = 0;
     ssize_t bytes_read=0, ret_code=0;
+    struct flock lock;
+    off_t total_length;
 
     if (NULL == fh->f_io_array) {
         return OMPI_ERROR;
@@ -53,6 +55,7 @@ ssize_t mca_fbtl_posix_preadv (mca_io_ompio_file_t *fh )
 	    iov[iov_count].iov_base = fh->f_io_array[i].memory_address;
 	    iov[iov_count].iov_len = fh->f_io_array[i].length;
 	    iov_offset = (OMPI_MPI_OFFSET_TYPE)(intptr_t)fh->f_io_array[i].offset;
+            total_length = iov[iov_count].iov_len;
 	    iov_count ++;
 	}
 
@@ -75,11 +78,12 @@ ssize_t mca_fbtl_posix_preadv (mca_io_ompio_file_t *fh )
                     iov[iov_count].iov_base =
                         fh->f_io_array[i+1].memory_address;
                     iov[iov_count].iov_len = fh->f_io_array[i+1].length;
+                    total_length += iov[iov_count].iov_len;
                     iov_count ++;
                     continue;
 	    }
 	}
-
+        mca_fbtl_posix_lock ( &lock, fh, F_RDLCK, iov_offset, total_len, OMPIO_LOCK_SELECTIVE ); 
 #if defined(HAVE_PREADV)
 	ret_code = preadv (fh->fd, iov, iov_count, iov_offset);
 	if ( 0 < ret_code ) {
@@ -96,6 +100,7 @@ ssize_t mca_fbtl_posix_preadv (mca_io_ompio_file_t *fh )
 	    bytes_read+=ret_code;
 	}
 #endif
+        mca_fbtl_posix_unlock ( &lock, fh );
 	else if ( ret_code == -1 ) {
             opal_output(1, "readv:%s", strerror(errno));
             free(iov);

ompi/mca/fbtl/posix/fbtl_posix_pwritev.c

@@ -9,7 +9,7 @@
  *                         University of Stuttgart.  All rights reserved.
  * Copyright (c) 2004-2005 The Regents of the University of California.
  *                         All rights reserved.
- * Copyright (c) 2008-2014 University of Houston. All rights reserved.
+ * Copyright (c) 2008-2017 University of Houston. All rights reserved.
  * Copyright (c) 2015-2017 Research Organization for Information Science
  *                         and Technology (RIST). All rights reserved.
  * $COPYRIGHT$
@@ -38,6 +38,8 @@ ssize_t  mca_fbtl_posix_pwritev(mca_io_ompio_file_t *fh )
     int iov_count = 0;
     OMPI_MPI_OFFSET_TYPE iov_offset = 0;
     ssize_t ret_code=0, bytes_written=0;
+    struct flock lock;
+    off_t total_length;
 
     if (NULL == fh->f_io_array) {
         return OMPI_ERROR;
@@ -55,6 +57,7 @@ ssize_t  mca_fbtl_posix_pwritev(mca_io_ompio_file_t *fh )
 	    iov[iov_count].iov_base = fh->f_io_array[i].memory_address;
 	    iov[iov_count].iov_len = fh->f_io_array[i].length;
 	    iov_offset = (OMPI_MPI_OFFSET_TYPE)(intptr_t)fh->f_io_array[i].offset;
+            total_length = iov[iov_count].iov_len;
 	    iov_count ++;
 	}
 
@@ -74,10 +77,10 @@ ssize_t  mca_fbtl_posix_pwritev(mca_io_ompio_file_t *fh )
 		   (ptrdiff_t)fh->f_io_array[i].length) ==
 		  (OMPI_MPI_OFFSET_TYPE)(intptr_t)fh->f_io_array[i+1].offset) &&
 		 (iov_count < IOV_MAX )) {
-		iov[iov_count].iov_base =
+		iov[iov_count].iov_base = fh->f_io_array[i+1].memory_address;
-		    fh->f_io_array[i+1].memory_address;
+		iov[iov_count].iov_len  = fh->f_io_array[i+1].length;
-		iov[iov_count].iov_len = fh->f_io_array[i+1].length;
+                total_length += iov[iov_count].iov_len;
-		iov_count ++;
+                iov_count ++;
 		continue;
 	    }
 	}
@@ -93,6 +96,8 @@ ssize_t  mca_fbtl_posix_pwritev(mca_io_ompio_file_t *fh )
 	  }
 
 	*/
+
+        mca_fbtl_posix_lock ( &lock, fh, F_WRLCK, iov_offset, total_len, OMPIO_LOCK_SELECTIVE ); 
 #if defined (HAVE_PWRITEV) 
 	ret_code = pwritev (fh->fd, iov, iov_count, iov_offset);
 	if ( 0 < ret_code ) {
@@ -110,6 +115,7 @@ ssize_t  mca_fbtl_posix_pwritev(mca_io_ompio_file_t *fh )
 	    bytes_written += ret_code;
 	}
 #endif
+        mca_fbtl_posix_unlock ( &lock, fh );
 	else if (-1 == ret_code ) {
 	    opal_output(1, "writev:%s", strerror(errno));
             free (iov);