ports/openmpi
ports/openmpi
1
1
Форкнуть 0
Код Релизы Активность

common/ompio: rename ompio_cuda* to ompio_buffer*

Просмотр исходного кода 
the infrastructure put in place to manage cuda buffers is actually
a lot more generic than just for cuda buffers. Specifically, we ca
reuse much of the code to implement the external32 data representation.
This commit converts the code from common_ompio_cuda* to
common_ompio_buffer*. There are just very few places where we actually need to keep the OPAL_CUDA_SUPPORT ifdef in place.

Signed-off-by: Edgar Gabriel <egabriel@central.uh.edu>
Этот коммит содержится в:
Edgar Gabriel 2019-05-20 10:24:31 -05:00
родитель a96efb7620
Коммит 35be18b266
7 изменённых файлов: 47 добавлений и 55 удалений
Показать статистику Скачать Patch файл Скачать Diff файл Показать все файлы Свернуть все файлы

6
ompi/mca/common/ompio/Makefile.am
Просмотреть файл

@ -25,6 +25,7 @@ headers = \
common_ompio_aggregators.h \
common_ompio_print_queue.h \
common_ompio_request.h \
common_ompio_buffer.h \
common_ompio.h
sources = \
@ -34,6 +35,7 @@ sources = \
common_ompio_file_open.c \
common_ompio_file_view.c \
common_ompio_file_read.c \
common_ompio_buffer.c \
common_ompio_file_write.c
@ -74,10 +76,6 @@ else
ompidir = $(includedir)
endif
if OPAL_cuda_support
headers += common_ompio_cuda.h
sources += common_ompio_cuda.c
endif
# These two rules will sym link the "noinst" libtool library filename
# to the installable libtool library filename in the case where we are

63
ompi/mca/common/ompio/common_ompio_cuda.c → ompi/mca/common/ompio/common_ompio_buffer.c
Просмотреть файл

@ -9,7 +9,7 @@
* University of Stuttgart. All rights reserved.
* Copyright (c) 2004-2005 The Regents of the University of California.
* All rights reserved.
* Copyright (c) 2008-2018 University of Houston. All rights reserved.
* Copyright (c) 2008-2019 University of Houston. All rights reserved.
* $COPYRIGHT$
*
* Additional copyrights may follow
@ -27,18 +27,19 @@
#include "opal/mca/allocator/allocator.h"
#include "opal/mca/allocator/base/base.h"
#include "common_ompio.h"
#include "common_ompio_cuda.h"
#include "common_ompio_buffer.h"
static opal_mutex_t mca_common_ompio_cuda_mutex; /* lock for thread safety */
static opal_mutex_t mca_common_ompio_buffer_mutex; /* lock for thread safety */
static mca_allocator_base_component_t* mca_common_ompio_allocator_component=NULL;
static mca_allocator_base_module_t* mca_common_ompio_allocator=NULL;
static opal_atomic_int32_t mca_common_ompio_cuda_init = 0;
static opal_atomic_int32_t mca_common_ompio_buffer_init = 0;
static int32_t mca_common_ompio_pagesize=4096;
static void* mca_common_ompio_cuda_alloc_seg ( void *ctx, size_t *size );
static void mca_common_ompio_cuda_free_seg ( void *ctx, void *buf );
static void* mca_common_ompio_buffer_alloc_seg ( void *ctx, size_t *size );
static void mca_common_ompio_buffer_free_seg ( void *ctx, void *buf );
#if OPAL_CUDA_SUPPORT
void mca_common_ompio_check_gpu_buf ( ompio_file_t *fh, const void *buf, int *is_gpu,
int *is_managed)
{
@ -57,8 +58,9 @@ void mca_common_ompio_check_gpu_buf ( ompio_file_t *fh, const void *buf, int *is
return;
}
#endif
static void* mca_common_ompio_cuda_alloc_seg ( void*ctx, size_t *size )
static void* mca_common_ompio_buffer_alloc_seg ( void*ctx, size_t *size )
{
char *buf=NULL;
size_t realsize, numpages;
@ -67,64 +69,67 @@ static void* mca_common_ompio_cuda_alloc_seg ( void*ctx, size_t *size )
realsize = numpages * mca_common_ompio_pagesize;
buf = malloc ( realsize);
#if OPAL_CUDA_SUPPORT
if ( NULL != buf ) {
mca_common_cuda_register ( ( char *)buf, realsize, NULL );
}
#endif
*size = realsize;
return buf;
}
static void mca_common_ompio_cuda_free_seg ( void *ctx, void *buf )
static void mca_common_ompio_buffer_free_seg ( void *ctx, void *buf )
{
if ( NULL != buf ) {
#if OPAL_CUDA_SUPPORT
mca_common_cuda_unregister ( (char *) buf, NULL );
#endif
free ( buf );
}
return;
}
int mca_common_ompio_cuda_alloc_init ( void )
int mca_common_ompio_buffer_alloc_init ( void )
{
bool thread_safe=true;
if(OPAL_THREAD_ADD_FETCH32(&mca_common_ompio_cuda_init, 1) > 1)
if(OPAL_THREAD_ADD_FETCH32(&mca_common_ompio_buffer_init, 1) > 1)
return OMPI_SUCCESS;
/* initialize static objects */
OBJ_CONSTRUCT(&mca_common_ompio_cuda_mutex, opal_mutex_t);
OBJ_CONSTRUCT(&mca_common_ompio_buffer_mutex, opal_mutex_t);
OPAL_THREAD_LOCK (&mca_common_ompio_cuda_mutex );
OPAL_THREAD_LOCK (&mca_common_ompio_buffer_mutex );
/* lookup name of the allocator to use */
if(NULL == (mca_common_ompio_allocator_component = mca_allocator_component_lookup("basic"))) {
OPAL_THREAD_UNLOCK(&mca_common_ompio_cuda_mutex);
OPAL_THREAD_UNLOCK(&mca_common_ompio_buffer_mutex);
return OMPI_ERR_BUFFER;
}
/* create an instance of the allocator */
mca_common_ompio_allocator = mca_common_ompio_allocator_component->allocator_init(thread_safe,
mca_common_ompio_cuda_alloc_seg,
mca_common_ompio_cuda_free_seg,
mca_common_ompio_buffer_alloc_seg,
mca_common_ompio_buffer_free_seg,
NULL);
if(NULL == mca_common_ompio_allocator) {
OPAL_THREAD_UNLOCK(&mca_common_ompio_cuda_mutex);
OPAL_THREAD_UNLOCK(&mca_common_ompio_buffer_mutex);
return OMPI_ERR_BUFFER;
}
// mca_common_ompio_pagesize = sysconf(_SC_PAGESIZE);
mca_common_ompio_pagesize = opal_getpagesize();
OPAL_THREAD_UNLOCK(&mca_common_ompio_cuda_mutex);
OPAL_THREAD_UNLOCK(&mca_common_ompio_buffer_mutex);
return OMPI_SUCCESS;
}
int mca_common_ompio_cuda_alloc_fini ( void )
int mca_common_ompio_buffer_alloc_fini ( void )
{
if ( NULL != mca_common_ompio_allocator ) {
OPAL_THREAD_LOCK (&mca_common_ompio_cuda_mutex);
OPAL_THREAD_LOCK (&mca_common_ompio_buffer_mutex);
mca_common_ompio_allocator->alc_finalize(mca_common_ompio_allocator);
mca_common_ompio_allocator=NULL;
OPAL_THREAD_UNLOCK (&mca_common_ompio_cuda_mutex);
OBJ_DESTRUCT (&mca_common_ompio_cuda_mutex);
OPAL_THREAD_UNLOCK (&mca_common_ompio_buffer_mutex);
OBJ_DESTRUCT (&mca_common_ompio_buffer_mutex);
}
return OMPI_SUCCESS;
@ -134,31 +139,31 @@ void *mca_common_ompio_alloc_buf ( ompio_file_t *fh, size_t bufsize )
{
char *tmp=NULL;
if ( !mca_common_ompio_cuda_init ){
mca_common_ompio_cuda_alloc_init ();
if ( !mca_common_ompio_buffer_init ){
mca_common_ompio_buffer_alloc_init ();
}
OPAL_THREAD_LOCK (&mca_common_ompio_cuda_mutex);
OPAL_THREAD_LOCK (&mca_common_ompio_buffer_mutex);
tmp = mca_common_ompio_allocator->alc_alloc (mca_common_ompio_allocator,
bufsize, 0 );
OPAL_THREAD_UNLOCK (&mca_common_ompio_cuda_mutex);
OPAL_THREAD_UNLOCK (&mca_common_ompio_buffer_mutex);
return tmp;
}
void mca_common_ompio_release_buf ( ompio_file_t *fh, void *buf )
{
if ( !mca_common_ompio_cuda_init ){
if ( !mca_common_ompio_buffer_init ){
/* Should not happen. You can not release a buf without
** having it allocated first.
*/
opal_output (1, "error in mca_common_ompio_release_buf: allocator not initialized\n");
}
OPAL_THREAD_LOCK (&mca_common_ompio_cuda_mutex);
OPAL_THREAD_LOCK (&mca_common_ompio_buffer_mutex);
mca_common_ompio_allocator->alc_free (mca_common_ompio_allocator,
buf);
OPAL_THREAD_UNLOCK (&mca_common_ompio_cuda_mutex);
OPAL_THREAD_UNLOCK (&mca_common_ompio_buffer_mutex);
return;
}

11
ompi/mca/common/ompio/common_ompio_cuda.h → ompi/mca/common/ompio/common_ompio_buffer.h
Просмотреть файл

@ -10,7 +10,7 @@
* University of Stuttgart. All rights reserved.
* Copyright (c) 2004-2005 The Regents of the University of California.
* All rights reserved.
* Copyright (c) 2008-2018 University of Houston. All rights reserved.
* Copyright (c) 2008-2019 University of Houston. All rights reserved.
* $COPYRIGHT$
*
* Additional copyrights may follow
@ -22,7 +22,7 @@
#define MCA_COMMON_OMPIO_CUDA_H
#define OMPIO_CUDA_PREPARE_BUF(_fh,_buf,_count,_datatype,_tbuf,_convertor,_max_data,_decoded_iov,_iov_count){ \
#define OMPIO_PREPARE_BUF(_fh,_buf,_count,_datatype,_tbuf,_convertor,_max_data,_decoded_iov,_iov_count){ \
opal_convertor_clone ( _fh->f_convertor, _convertor, 0); \
opal_convertor_prepare_for_send ( _convertor, &(_datatype->super), _count, _buf );\
opal_convertor_get_packed_size( _convertor, &_max_data ); \
@ -40,11 +40,12 @@
_decoded_iov->iov_len = _max_data; \
_iov_count=1;}
#if OPAL_CUDA_SUPPORT
void mca_common_ompio_check_gpu_buf ( ompio_file_t *fh, const void *buf,
int *is_gpu, int *is_managed);
int mca_common_ompio_cuda_alloc_init ( void );
int mca_common_ompio_cuda_alloc_fini ( void );
#endif
int mca_common_ompio_buffer_alloc_init ( void );
int mca_common_ompio_buffer_alloc_fini ( void );
void* mca_common_ompio_alloc_buf ( ompio_file_t *fh, size_t bufsize);

4
ompi/mca/common/ompio/common_ompio_file_read.c
Просмотреть файл

@ -33,12 +33,10 @@
#include "common_ompio.h"
#include "common_ompio_request.h"
#include "common_ompio_buffer.h"
#include <unistd.h>
#include <math.h>
#if OPAL_CUDA_SUPPORT
#include "common_ompio_cuda.h"
#endif
/* Read and write routines are split into two interfaces.
** The

4
ompi/mca/common/ompio/common_ompio_file_write.c
Просмотреть файл

@ -31,12 +31,10 @@
#include "common_ompio.h"
#include "common_ompio_request.h"
#include "common_ompio_buffer.h"
#include <unistd.h>
#include <math.h>
#if OPAL_CUDA_SUPPORT
#include "common_ompio_cuda.h"
#endif
int mca_common_ompio_file_write (ompio_file_t *fh,
const void *buf,

10
ompi/mca/common/ompio/common_ompio_request.c
Просмотреть файл

@ -10,7 +10,7 @@
* University of Stuttgart. All rights reserved.
* Copyright (c) 2004-2005 The Regents of the University of California.
* All rights reserved.
* Copyright (c) 2008-2018 University of Houston. All rights reserved.
* Copyright (c) 2008-2019 University of Houston. All rights reserved.
* $COPYRIGHT$
*
* Additional copyrights may follow
@ -19,9 +19,7 @@
*/
#include "common_ompio_request.h"
#if OPAL_CUDA_SUPPORT
#include "common_ompio_cuda.h"
#endif
#include "common_ompio_buffer.h"
static void mca_common_ompio_request_construct(mca_ompio_request_t* req);
static void mca_common_ompio_request_destruct(mca_ompio_request_t *req);
@ -37,7 +35,6 @@ opal_list_t mca_common_ompio_pending_requests = {{0}};
static int mca_common_ompio_request_free ( struct ompi_request_t **req)
{
mca_ompio_request_t *ompio_req = ( mca_ompio_request_t *)*req;
#if OPAL_CUDA_SUPPORT
if ( NULL != ompio_req->req_tbuf ) {
if ( MCA_OMPIO_REQUEST_READ == ompio_req->req_type ){
struct iovec decoded_iov;
@ -50,7 +47,6 @@ static int mca_common_ompio_request_free ( struct ompi_request_t **req)
}
mca_common_ompio_release_buf ( NULL, ompio_req->req_tbuf );
}
#endif
if ( NULL != ompio_req->req_free_fn ) {
ompio_req->req_free_fn (ompio_req );
}
@ -77,10 +73,8 @@ void mca_common_ompio_request_construct(mca_ompio_request_t* req)
req->req_ompi.req_cancel = mca_common_ompio_request_cancel;
req->req_ompi.req_type = OMPI_REQUEST_IO;
req->req_data = NULL;
#if OPAL_CUDA_SUPPORT
req->req_tbuf = NULL;
req->req_size = 0;
#endif
req->req_progress_fn = NULL;
req->req_free_fn = NULL;

4
ompi/mca/common/ompio/common_ompio_request.h
Просмотреть файл

@ -10,7 +10,7 @@
* University of Stuttgart. All rights reserved.
* Copyright (c) 2004-2005 The Regents of the University of California.
* All rights reserved.
* Copyright (c) 2008-2018 University of Houston. All rights reserved.
* Copyright (c) 2008-2019 University of Houston. All rights reserved.
* Copyright (c) 2018 Research Organization for Information Science
* and Technology (RIST). All rights reserved.
* $COPYRIGHT$
@ -52,11 +52,9 @@ struct mca_ompio_request_t {
mca_ompio_request_type_t req_type;
void *req_data;
opal_list_item_t req_item;
#if OPAL_CUDA_SUPPORT
void *req_tbuf;
size_t req_size;
opal_convertor_t req_convertor;
#endif
mca_fbtl_base_module_progress_fn_t req_progress_fn;
mca_fbtl_base_module_request_free_fn_t req_free_fn;
};