From 7861a8c357039383d967a209a2b96d4f108aeea3 Mon Sep 17 00:00:00 2001 From: Edgar Gabriel Date: Tue, 5 Jan 2016 08:14:59 -0600 Subject: [PATCH] revise the logic in the fbtl plfs avoiding the memcpy operation --- ompi/mca/fbtl/plfs/fbtl_plfs_preadv.c | 86 ++---------------------- ompi/mca/fbtl/plfs/fbtl_plfs_pwritev.c | 90 ++------------------------ 2 files changed, 12 insertions(+), 164 deletions(-) diff --git a/ompi/mca/fbtl/plfs/fbtl_plfs_preadv.c b/ompi/mca/fbtl/plfs/fbtl_plfs_preadv.c index 1eefad95be..26e60065a5 100644 --- a/ompi/mca/fbtl/plfs/fbtl_plfs_preadv.c +++ b/ompi/mca/fbtl/plfs/fbtl_plfs_preadv.c @@ -28,74 +28,19 @@ ssize_t mca_fbtl_plfs_preadv (mca_io_ompio_file_t *fh ) { - Plfs_fd *pfd = NULL; + Plfs_fd *pfd = fh->f_fs_ptr; plfs_error_t plfs_ret; - pfd = fh->f_fs_ptr; ssize_t total_bytes_read=0; - - int i, block=1; - struct iovec *iov = NULL; - int iov_count = 0; - OMPI_MPI_OFFSET_TYPE iov_offset = 0; + int i; + ssize_t bytes_read; if (NULL == fh->f_io_array) { return OMPI_ERROR; } - iov = (struct iovec *) malloc - (OMPIO_IOVEC_INITIAL_SIZE * sizeof (struct iovec)); - if (NULL == iov) { - opal_output(1, "OUT OF MEMORY\n"); - return OMPI_ERR_OUT_OF_RESOURCE; - } - for (i=0 ; if_num_of_io_entries ; i++) { - if (0 == iov_count) { - iov[iov_count].iov_base = fh->f_io_array[i].memory_address; - iov[iov_count].iov_len = fh->f_io_array[i].length; - iov_offset = (OMPI_MPI_OFFSET_TYPE)(intptr_t)fh->f_io_array[i].offset; - iov_count ++; - } - - if (OMPIO_IOVEC_INITIAL_SIZE*block <= iov_count) { - block ++; - iov = (struct iovec *)realloc - (iov, OMPIO_IOVEC_INITIAL_SIZE * block * - sizeof(struct iovec)); - if (NULL == iov) { - opal_output(1, "OUT OF MEMORY\n"); - return OMPI_ERR_OUT_OF_RESOURCE; - } - } - - if (fh->f_num_of_io_entries != i+1) { - if (((OMPI_MPI_OFFSET_TYPE)(intptr_t)fh->f_io_array[i].offset + - (OPAL_PTRDIFF_TYPE)fh->f_io_array[i].length) == - (OMPI_MPI_OFFSET_TYPE)(intptr_t)fh->f_io_array[i+1].offset) { - iov[iov_count].iov_base = - fh->f_io_array[i+1].memory_address; - iov[iov_count].iov_len = fh->f_io_array[i+1].length; - iov_count ++; - continue; - } - } - - // Find the total number of bytes to be read. - size_t bytes = 0; - for (int i = 0; i < iov_count; ++i) { - bytes += iov[i].iov_len; - } - - // Allocate a temporary buffer to hold the data - char *buffer; - buffer = (char *) malloc (bytes); - if (buffer == NULL) { - return OMPI_ERROR; - } - - // Read the data - ssize_t bytes_read; - plfs_ret = plfs_read( pfd, buffer, bytes, iov_offset, &bytes_read ); + plfs_ret = plfs_read( pfd, fh->f_io_array[i].memory_address, fh->f_io_array[i].length, + (off_t )fh->f_io_array[i].offset, &bytes_read ); if (PLFS_SUCCESS != plfs_ret) { opal_output(0, "fbtl_plfs_preadv: Error in plfs_read:\n%s\n", strplfserr(plfs_ret)); return OMPI_ERROR; @@ -104,27 +49,6 @@ ssize_t mca_fbtl_plfs_preadv (mca_io_ompio_file_t *fh ) if (bytes_read < 0) return OMPI_ERROR; total_bytes_read += bytes_read; - // Copy the data from BUFFER into the memory specified by IOV - bytes = bytes_read; - for (int i = 0; i < iov_count; ++i) { - size_t copy = MIN (iov[i].iov_len, bytes); - (void) memcpy ((void *) iov[i].iov_base, (void *) buffer, copy); - buffer += copy; - bytes -= copy; - if (bytes == 0) { - break; - } - } - iov_count = 0; - if ( NULL != buffer ) { - free (buffer); - buffer=NULL; - } - } - - if (NULL != iov) { - free (iov); - iov = NULL; } return total_bytes_read; diff --git a/ompi/mca/fbtl/plfs/fbtl_plfs_pwritev.c b/ompi/mca/fbtl/plfs/fbtl_plfs_pwritev.c index 5c79971478..cd63c9db5a 100644 --- a/ompi/mca/fbtl/plfs/fbtl_plfs_pwritev.c +++ b/ompi/mca/fbtl/plfs/fbtl_plfs_pwritev.c @@ -28,102 +28,26 @@ ssize_t mca_fbtl_plfs_pwritev (mca_io_ompio_file_t *fh ) { - Plfs_fd *pfd = NULL; + Plfs_fd *pfd = fh->f_fs_ptr; plfs_error_t plfs_ret; - pfd = fh->f_fs_ptr; ssize_t total_bytes_written=0; - - int i, block = 1; - struct iovec *iov = NULL; - int iov_count = 0; - OMPI_MPI_OFFSET_TYPE iov_offset = 0; + ssize_t bytes_written; + int i; if (NULL == fh->f_io_array) { return OMPI_ERROR; } - iov = (struct iovec *) malloc - (OMPIO_IOVEC_INITIAL_SIZE * sizeof (struct iovec)); - if (NULL == iov) { - opal_output(1, "OUT OF MEMORY\n"); - return OMPI_ERR_OUT_OF_RESOURCE; - } - for (i=0 ; if_num_of_io_entries ; i++) { - if (0 == iov_count) { - iov[iov_count].iov_base = fh->f_io_array[i].memory_address; - iov[iov_count].iov_len = fh->f_io_array[i].length; - iov_offset = (OMPI_MPI_OFFSET_TYPE)(intptr_t)fh->f_io_array[i].offset; - iov_count ++; - } - - if (OMPIO_IOVEC_INITIAL_SIZE*block <= iov_count) { - block ++; - iov = (struct iovec *)realloc - (iov, OMPIO_IOVEC_INITIAL_SIZE * block * - sizeof(struct iovec)); - if (NULL == iov) { - opal_output(1, "OUT OF MEMORY\n"); - return OMPI_ERR_OUT_OF_RESOURCE; - } - } - - if (fh->f_num_of_io_entries != i+1) { - if (((OMPI_MPI_OFFSET_TYPE)(intptr_t)fh->f_io_array[i].offset + - (OPAL_PTRDIFF_TYPE)fh->f_io_array[i].length) == - (OMPI_MPI_OFFSET_TYPE)(intptr_t)fh->f_io_array[i+1].offset) { - iov[iov_count].iov_base = - fh->f_io_array[i+1].memory_address; - iov[iov_count].iov_len = fh->f_io_array[i+1].length; - iov_count ++; - continue; - } - } - - // Find the total number of bytes to be written. - size_t bytes = 0; - for (int i = 0; i < iov_count; ++i) { - bytes += iov[i].iov_len; - } - - // Allocate a temporary buffer to hold the data - char *buffer=NULL; - buffer = (char *) malloc (bytes); - if (buffer == NULL) { - return OMPI_ERROR; - } - - // Copy the data into BUFFER. - size_t to_copy = bytes; - char *bp = buffer; - for (int i = 0; i < iov_count; ++i) { - size_t copy = MIN (iov[i].iov_len, to_copy); - bp = mempcpy ((void *) bp, (void *) iov[i].iov_base, copy); - to_copy -= copy; - if (to_copy == 0) { - break; - } - } - - // Write the data - ssize_t bytes_written; - - plfs_ret = plfs_write( pfd, buffer, bytes, iov_offset, 0, &bytes_written ); + plfs_ret = plfs_write( pfd, fh->f_io_array[i].memory_address, + fh->f_io_array[i].length, + (off_t) fh->f_io_array[i].offset, + fh->f_rank, &bytes_written ); if (PLFS_SUCCESS != plfs_ret) { opal_output(0, "fbtl_plfs_pwritev: Error in plfs_write:\n%s\n", strplfserr(plfs_ret)); return OMPI_ERROR; } total_bytes_written += bytes_written; - iov_count = 0; - if ( NULL != buffer ) { - free ( buffer ); - buffer=NULL; - } - } - - if (NULL != iov) { - free (iov); - iov = NULL; } return total_bytes_written;