common/ompio: rename ompio_cuda* to ompio_buffer*

the infrastructure put in place to manage cuda buffers is actually
a lot more generic than just for cuda buffers. Specifically, we ca
reuse much of the code to implement the external32 data representation.
This commit converts the code from common_ompio_cuda* to
common_ompio_buffer*. There are just very few places where we actually need to keep the OPAL_CUDA_SUPPORT ifdef in place.

Signed-off-by: Edgar Gabriel <egabriel@central.uh.edu>

ompi/mca/common/ompio/Makefile.am

@@ -25,6 +25,7 @@ headers = \
 	common_ompio_aggregators.h \
 	common_ompio_print_queue.h \
 	common_ompio_request.h \
+	common_ompio_buffer.h  \
 	common_ompio.h
 
 sources = \
@@ -34,6 +35,7 @@ sources = \
 	common_ompio_file_open.c   \
 	common_ompio_file_view.c   \
 	common_ompio_file_read.c   \
+	common_ompio_buffer.c      \
 	common_ompio_file_write.c
 
 
@@ -74,10 +76,6 @@ else
 ompidir = $(includedir)
 endif
 
-if OPAL_cuda_support
-headers += common_ompio_cuda.h
-sources += common_ompio_cuda.c
-endif
 
 # These two rules will sym link the "noinst" libtool library filename
 # to the installable libtool library filename in the case where we are

ompi/mca/common/ompio/common_ompio_buffer.c

@@ -9,7 +9,7 @@
  *                          University of Stuttgart.  All rights reserved.
  *  Copyright (c) 2004-2005 The Regents of the University of California.
  *                          All rights reserved.
- *  Copyright (c) 2008-2018 University of Houston. All rights reserved.
+ *  Copyright (c) 2008-2019 University of Houston. All rights reserved.
  *  $COPYRIGHT$
  *
  *  Additional copyrights may follow
@@ -27,18 +27,19 @@
 #include "opal/mca/allocator/allocator.h"
 #include "opal/mca/allocator/base/base.h"
 #include "common_ompio.h"
-#include "common_ompio_cuda.h"
+#include "common_ompio_buffer.h"
 
 
-static opal_mutex_t     mca_common_ompio_cuda_mutex;      /* lock for thread safety */
+static opal_mutex_t     mca_common_ompio_buffer_mutex;      /* lock for thread safety */
 static mca_allocator_base_component_t* mca_common_ompio_allocator_component=NULL;
 static mca_allocator_base_module_t* mca_common_ompio_allocator=NULL;  
 
-static opal_atomic_int32_t  mca_common_ompio_cuda_init = 0;
+static opal_atomic_int32_t  mca_common_ompio_buffer_init = 0;
 static int32_t  mca_common_ompio_pagesize=4096;
-static void* mca_common_ompio_cuda_alloc_seg ( void *ctx, size_t *size );
+static void* mca_common_ompio_buffer_alloc_seg ( void *ctx, size_t *size );
-static void mca_common_ompio_cuda_free_seg ( void *ctx, void *buf );
+static void mca_common_ompio_buffer_free_seg ( void *ctx, void *buf );
 
+#if OPAL_CUDA_SUPPORT
 void mca_common_ompio_check_gpu_buf ( ompio_file_t *fh, const void *buf, int *is_gpu, 
 				      int *is_managed)
 {
@@ -57,8 +58,9 @@ void mca_common_ompio_check_gpu_buf ( ompio_file_t *fh, const void *buf, int *is
     
     return;
 }
+#endif
 
-static void* mca_common_ompio_cuda_alloc_seg ( void*ctx, size_t *size )
+static void* mca_common_ompio_buffer_alloc_seg ( void*ctx, size_t *size )
 {
     char *buf=NULL;
     size_t realsize, numpages;
@@ -67,64 +69,67 @@ static void* mca_common_ompio_cuda_alloc_seg ( void*ctx, size_t *size )
     realsize = numpages * mca_common_ompio_pagesize;
 
     buf = malloc ( realsize);
+#if OPAL_CUDA_SUPPORT
     if ( NULL != buf ) {
         mca_common_cuda_register ( ( char *)buf, realsize, NULL  );
     }
+#endif
     *size = realsize;
     return buf;
 }
 
-static void mca_common_ompio_cuda_free_seg ( void *ctx, void *buf )
+static void mca_common_ompio_buffer_free_seg ( void *ctx, void *buf )
 {
     if ( NULL != buf ) {
+#if OPAL_CUDA_SUPPORT
         mca_common_cuda_unregister ( (char *) buf, NULL );
+#endif
         free ( buf );
     }
     return;
 }
 
-int mca_common_ompio_cuda_alloc_init ( void )
+int mca_common_ompio_buffer_alloc_init ( void )
 {
     bool thread_safe=true;
 
-    if(OPAL_THREAD_ADD_FETCH32(&mca_common_ompio_cuda_init, 1) > 1)
+    if(OPAL_THREAD_ADD_FETCH32(&mca_common_ompio_buffer_init, 1) > 1)
         return OMPI_SUCCESS;
 
     /* initialize static objects */
-    OBJ_CONSTRUCT(&mca_common_ompio_cuda_mutex, opal_mutex_t);
+    OBJ_CONSTRUCT(&mca_common_ompio_buffer_mutex, opal_mutex_t);
 
-    OPAL_THREAD_LOCK (&mca_common_ompio_cuda_mutex );
+    OPAL_THREAD_LOCK (&mca_common_ompio_buffer_mutex );
     /* lookup name of the allocator to use */
     if(NULL == (mca_common_ompio_allocator_component = mca_allocator_component_lookup("basic"))) {
-        OPAL_THREAD_UNLOCK(&mca_common_ompio_cuda_mutex);
+        OPAL_THREAD_UNLOCK(&mca_common_ompio_buffer_mutex);
         return OMPI_ERR_BUFFER;
     }
 
     /* create an instance of the allocator */
     mca_common_ompio_allocator = mca_common_ompio_allocator_component->allocator_init(thread_safe, 
-                                                                                      mca_common_ompio_cuda_alloc_seg, 
+                                                                                      mca_common_ompio_buffer_alloc_seg, 
-                                                                                      mca_common_ompio_cuda_free_seg, 
+                                                                                      mca_common_ompio_buffer_free_seg, 
                                                                                       NULL);
     if(NULL == mca_common_ompio_allocator) {
-        OPAL_THREAD_UNLOCK(&mca_common_ompio_cuda_mutex);
+        OPAL_THREAD_UNLOCK(&mca_common_ompio_buffer_mutex);
         return OMPI_ERR_BUFFER;
     }
 
-//    mca_common_ompio_pagesize = sysconf(_SC_PAGESIZE);
     mca_common_ompio_pagesize = opal_getpagesize();
 
-    OPAL_THREAD_UNLOCK(&mca_common_ompio_cuda_mutex);
+    OPAL_THREAD_UNLOCK(&mca_common_ompio_buffer_mutex);
     return OMPI_SUCCESS;
 }
 
-int mca_common_ompio_cuda_alloc_fini ( void )
+int mca_common_ompio_buffer_alloc_fini ( void )
 {
     if ( NULL != mca_common_ompio_allocator ) {
-        OPAL_THREAD_LOCK (&mca_common_ompio_cuda_mutex);
+        OPAL_THREAD_LOCK (&mca_common_ompio_buffer_mutex);
         mca_common_ompio_allocator->alc_finalize(mca_common_ompio_allocator);
         mca_common_ompio_allocator=NULL;
-        OPAL_THREAD_UNLOCK (&mca_common_ompio_cuda_mutex);
+        OPAL_THREAD_UNLOCK (&mca_common_ompio_buffer_mutex);
-        OBJ_DESTRUCT (&mca_common_ompio_cuda_mutex);
+        OBJ_DESTRUCT (&mca_common_ompio_buffer_mutex);
     }
 
     return OMPI_SUCCESS;
@@ -134,31 +139,31 @@ void *mca_common_ompio_alloc_buf ( ompio_file_t *fh, size_t bufsize )
 {
     char *tmp=NULL;
 
-    if ( !mca_common_ompio_cuda_init ){
+    if ( !mca_common_ompio_buffer_init ){
-        mca_common_ompio_cuda_alloc_init ();
+        mca_common_ompio_buffer_alloc_init ();
     }
     
-    OPAL_THREAD_LOCK (&mca_common_ompio_cuda_mutex);
+    OPAL_THREAD_LOCK (&mca_common_ompio_buffer_mutex);
     tmp = mca_common_ompio_allocator->alc_alloc (mca_common_ompio_allocator,
                                                  bufsize, 0 );
-    OPAL_THREAD_UNLOCK (&mca_common_ompio_cuda_mutex);
+    OPAL_THREAD_UNLOCK (&mca_common_ompio_buffer_mutex);
     return tmp;
 }
 
 void mca_common_ompio_release_buf ( ompio_file_t *fh, void *buf )
 {
 
-    if ( !mca_common_ompio_cuda_init ){
+    if ( !mca_common_ompio_buffer_init ){
         /* Should not happen. You can not release a buf without
         ** having it allocated first. 
         */
         opal_output (1, "error in mca_common_ompio_release_buf: allocator not initialized\n");
     }
 
-    OPAL_THREAD_LOCK (&mca_common_ompio_cuda_mutex);
+    OPAL_THREAD_LOCK (&mca_common_ompio_buffer_mutex);
     mca_common_ompio_allocator->alc_free (mca_common_ompio_allocator,
                                           buf);
-    OPAL_THREAD_UNLOCK (&mca_common_ompio_cuda_mutex);
+    OPAL_THREAD_UNLOCK (&mca_common_ompio_buffer_mutex);
 
     return;
 }

ompi/mca/common/ompio/common_ompio_buffer.h

@@ -10,7 +10,7 @@
  *                         University of Stuttgart.  All rights reserved.
  * Copyright (c) 2004-2005 The Regents of the University of California.
  *                         All rights reserved.
- * Copyright (c) 2008-2018 University of Houston. All rights reserved.
+ * Copyright (c) 2008-2019 University of Houston. All rights reserved.
  * $COPYRIGHT$
  *
  * Additional copyrights may follow
@@ -22,7 +22,7 @@
 #define MCA_COMMON_OMPIO_CUDA_H
 
 
-#define OMPIO_CUDA_PREPARE_BUF(_fh,_buf,_count,_datatype,_tbuf,_convertor,_max_data,_decoded_iov,_iov_count){ \
+#define OMPIO_PREPARE_BUF(_fh,_buf,_count,_datatype,_tbuf,_convertor,_max_data,_decoded_iov,_iov_count){ \
     opal_convertor_clone ( _fh->f_convertor, _convertor, 0);                          \
     opal_convertor_prepare_for_send ( _convertor, &(_datatype->super), _count, _buf );\
     opal_convertor_get_packed_size( _convertor, &_max_data );           \
@@ -40,11 +40,12 @@
     _decoded_iov->iov_len  = _max_data;                                 \
     _iov_count=1;}
 
-
+#if OPAL_CUDA_SUPPORT
 void mca_common_ompio_check_gpu_buf ( ompio_file_t *fh, const void *buf, 
 				      int *is_gpu, int *is_managed);
-int mca_common_ompio_cuda_alloc_init ( void );
+#endif
-int mca_common_ompio_cuda_alloc_fini ( void );
+int mca_common_ompio_buffer_alloc_init ( void );
+int mca_common_ompio_buffer_alloc_fini ( void );
 
 
 void* mca_common_ompio_alloc_buf ( ompio_file_t *fh, size_t bufsize);

ompi/mca/common/ompio/common_ompio_file_read.c

@@ -33,12 +33,10 @@
 
 #include "common_ompio.h"
 #include "common_ompio_request.h"
+#include "common_ompio_buffer.h"
 #include <unistd.h>
 #include <math.h>
 
-#if OPAL_CUDA_SUPPORT
-#include "common_ompio_cuda.h"
-#endif
 
 /* Read and write routines are split into two interfaces.
 **   The

ompi/mca/common/ompio/common_ompio_file_write.c

@@ -31,12 +31,10 @@
 
 #include "common_ompio.h"
 #include "common_ompio_request.h"
+#include "common_ompio_buffer.h"
 #include <unistd.h>
 #include <math.h>
 
-#if OPAL_CUDA_SUPPORT
-#include "common_ompio_cuda.h"
-#endif
 
 int mca_common_ompio_file_write (ompio_file_t *fh,
 			       const void *buf,

ompi/mca/common/ompio/common_ompio_request.c

@@ -10,7 +10,7 @@
  *                         University of Stuttgart.  All rights reserved.
  * Copyright (c) 2004-2005 The Regents of the University of California.
  *                         All rights reserved.
- * Copyright (c) 2008-2018 University of Houston. All rights reserved.
+ * Copyright (c) 2008-2019 University of Houston. All rights reserved.
  * $COPYRIGHT$
  *
  * Additional copyrights may follow
@@ -19,9 +19,7 @@
  */
 
 #include "common_ompio_request.h"
-#if OPAL_CUDA_SUPPORT
+#include "common_ompio_buffer.h"
-#include "common_ompio_cuda.h"
-#endif
 
 static void mca_common_ompio_request_construct(mca_ompio_request_t* req);
 static void mca_common_ompio_request_destruct(mca_ompio_request_t *req);
@@ -37,7 +35,6 @@ opal_list_t mca_common_ompio_pending_requests = {{0}};
 static int mca_common_ompio_request_free ( struct ompi_request_t **req)
 {
     mca_ompio_request_t *ompio_req = ( mca_ompio_request_t *)*req;
-#if OPAL_CUDA_SUPPORT
     if ( NULL != ompio_req->req_tbuf ) {
         if ( MCA_OMPIO_REQUEST_READ == ompio_req->req_type ){
             struct iovec decoded_iov;
@@ -50,7 +47,6 @@ static int mca_common_ompio_request_free ( struct ompi_request_t **req)
         }
         mca_common_ompio_release_buf ( NULL, ompio_req->req_tbuf );
     }
-#endif
     if ( NULL != ompio_req->req_free_fn ) {
         ompio_req->req_free_fn (ompio_req );
     }
@@ -77,10 +73,8 @@ void mca_common_ompio_request_construct(mca_ompio_request_t* req)
     req->req_ompi.req_cancel = mca_common_ompio_request_cancel;
     req->req_ompi.req_type   = OMPI_REQUEST_IO;
     req->req_data            = NULL;
-#if OPAL_CUDA_SUPPORT
     req->req_tbuf            = NULL;
     req->req_size            = 0;
-#endif
     req->req_progress_fn     = NULL;
     req->req_free_fn         = NULL;
 

ompi/mca/common/ompio/common_ompio_request.h

@@ -10,7 +10,7 @@
  *                         University of Stuttgart.  All rights reserved.
  * Copyright (c) 2004-2005 The Regents of the University of California.
  *                         All rights reserved.
- * Copyright (c) 2008-2018 University of Houston. All rights reserved.
+ * Copyright (c) 2008-2019 University of Houston. All rights reserved.
  * Copyright (c) 2018      Research Organization for Information Science
  *                         and Technology (RIST). All rights reserved.
  * $COPYRIGHT$
@@ -52,11 +52,9 @@ struct mca_ompio_request_t {
     mca_ompio_request_type_t                       req_type;
     void                                          *req_data;
     opal_list_item_t                               req_item;
-#if OPAL_CUDA_SUPPORT
     void                                          *req_tbuf;
     size_t                                         req_size;
     opal_convertor_t                          req_convertor;
-#endif
     mca_fbtl_base_module_progress_fn_t      req_progress_fn;
     mca_fbtl_base_module_request_free_fn_t      req_free_fn;
 };