207 строки
6.8 KiB
C
207 строки
6.8 KiB
C
|
/*
|
||
|
* Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
|
||
|
* University Research and Technology
|
||
|
* Corporation. All rights reserved.
|
||
|
* Copyright (c) 2004-2005 The University of Tennessee and The University
|
||
|
* of Tennessee Research Foundation. All rights
|
||
|
* reserved.
|
||
|
* Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
|
||
|
* University of Stuttgart. All rights reserved.
|
||
|
* Copyright (c) 2004-2005 The Regents of the University of California.
|
||
|
* All rights reserved.
|
||
|
* Copyright (c) 2013-2016 University of Houston. All rights reserved.
|
||
|
* $COPYRIGHT$
|
||
|
*
|
||
|
* Additional copyrights may follow
|
||
|
*
|
||
|
* $HEADER$
|
||
|
*/
|
||
|
|
||
|
|
||
|
#include "ompi_config.h"
|
||
|
#include "sharedfp_addproc.h"
|
||
|
|
||
|
#include "mpi.h"
|
||
|
#include "ompi/constants.h"
|
||
|
#include "ompi/mca/sharedfp/sharedfp.h"
|
||
|
#include "ompi/mca/sharedfp/base/base.h"
|
||
|
|
||
|
|
||
|
|
||
|
int mca_sharedfp_addproc_iread(mca_io_ompio_file_t *fh,
|
||
|
void *buf,
|
||
|
int count,
|
||
|
ompi_datatype_t *datatype,
|
||
|
MPI_Request * request)
|
||
|
{
|
||
|
int ret = OMPI_SUCCESS;
|
||
|
OMPI_MPI_OFFSET_TYPE offset = 0;
|
||
|
long bytesRequested = 0;
|
||
|
size_t numofBytes;
|
||
|
struct mca_sharedfp_base_data_t *sh = NULL;
|
||
|
|
||
|
if(NULL == fh->f_sharedfp_data){
|
||
|
opal_output(0, "sharedfp_addproc_iread - shared file pointer structure not initialized correctly\n");
|
||
|
return OMPI_ERROR;
|
||
|
}
|
||
|
|
||
|
/* Calculate the number of bytes to write */
|
||
|
opal_datatype_type_size ( &datatype->super ,&numofBytes);
|
||
|
bytesRequested = count * numofBytes;
|
||
|
|
||
|
if ( mca_sharedfp_addproc_verbose ){
|
||
|
opal_output(ompi_sharedfp_base_framework.framework_output,
|
||
|
"mca_sharedfp_addproc_iread: Bytes Requested is %ld\n",bytesRequested);
|
||
|
}
|
||
|
/* Retrieve the shared file data struct */
|
||
|
sh = fh->f_sharedfp_data;
|
||
|
|
||
|
/*Request to the additional process for the offset*/
|
||
|
ret = mca_sharedfp_addproc_request_position(sh,bytesRequested,&offset);
|
||
|
offset /= sh->sharedfh->f_etype_size;
|
||
|
|
||
|
if( OMPI_SUCCESS == ret ){
|
||
|
if ( mca_sharedfp_addproc_verbose ){
|
||
|
opal_output(ompi_sharedfp_base_framework.framework_output,
|
||
|
"mca_sharedfp_addproc_iread: Offset received is %lld\n",offset);
|
||
|
}
|
||
|
/* Read from the file */
|
||
|
ret = mca_common_ompio_file_iread_at ( sh->sharedfh, offset, buf, count, datatype, request);
|
||
|
}
|
||
|
|
||
|
return ret;
|
||
|
}
|
||
|
int mca_sharedfp_addproc_read_ordered_begin(mca_io_ompio_file_t *fh,
|
||
|
void *buf,
|
||
|
int count,
|
||
|
struct ompi_datatype_t *datatype)
|
||
|
{
|
||
|
int ret = OMPI_SUCCESS;
|
||
|
OMPI_MPI_OFFSET_TYPE offset = 0, offsetReceived = 0;
|
||
|
long sendBuff = 0;
|
||
|
long *buff=NULL;
|
||
|
long offsetBuff, bytesRequested = 0;
|
||
|
size_t numofBytes;
|
||
|
int rank, size, i;
|
||
|
struct mca_sharedfp_base_data_t *sh = NULL;
|
||
|
|
||
|
if(NULL == fh->f_sharedfp_data){
|
||
|
opal_output(0, "sharedfp_addproc_read_ordered_begin: shared file pointer "
|
||
|
"structure not initialized correctly\n");
|
||
|
return OMPI_ERROR;
|
||
|
}
|
||
|
|
||
|
if ( true == fh->f_split_coll_in_use ) {
|
||
|
opal_output(0, "Only one split collective I/O operation allowed per "
|
||
|
"file handle at any given point in time!\n");
|
||
|
return MPI_ERR_REQUEST;
|
||
|
}
|
||
|
|
||
|
/*Retrieve the new communicator*/
|
||
|
sh = fh->f_sharedfp_data;
|
||
|
|
||
|
/* Calculate the number of bytes to read*/
|
||
|
opal_datatype_type_size ( &datatype->super, &numofBytes);
|
||
|
sendBuff = count * numofBytes;
|
||
|
|
||
|
/* Get the ranks in the communicator */
|
||
|
rank = ompi_comm_rank ( sh->comm);
|
||
|
size = ompi_comm_size ( sh->comm);
|
||
|
|
||
|
if ( 0 == rank ) {
|
||
|
buff = (long*)malloc(sizeof(OMPI_MPI_OFFSET_TYPE) * size);
|
||
|
if ( NULL == buff )
|
||
|
return OMPI_ERR_OUT_OF_RESOURCE;
|
||
|
}
|
||
|
|
||
|
ret = sh->comm->c_coll.coll_gather( &sendBuff, 1, OMPI_OFFSET_DATATYPE,
|
||
|
buff, 1, OMPI_OFFSET_DATATYPE, 0, sh->comm,
|
||
|
sh->comm->c_coll.coll_gather_module);
|
||
|
if ( OMPI_SUCCESS != ret ) {
|
||
|
goto exit;
|
||
|
}
|
||
|
|
||
|
/* All the counts are present now in the recvBuff.
|
||
|
The size of recvBuff is sizeof_newComm
|
||
|
*/
|
||
|
if ( 0 == rank ) {
|
||
|
for (i = 0; i < size ; i ++) {
|
||
|
if ( mca_sharedfp_addproc_verbose ){
|
||
|
opal_output(ompi_sharedfp_base_framework.framework_output,
|
||
|
"sharedfp_addproc_read_ordered_begin: Buff is %ld\n",buff[i]);
|
||
|
}
|
||
|
bytesRequested += buff[i];
|
||
|
|
||
|
if ( mca_sharedfp_addproc_verbose ){
|
||
|
opal_output(ompi_sharedfp_base_framework.framework_output,
|
||
|
"sharedfp_addproc_read_ordered_begin: Bytes requested are %ld\n",
|
||
|
bytesRequested);
|
||
|
}
|
||
|
}
|
||
|
|
||
|
/* Request the offset to read bytesRequested bytes
|
||
|
** only the root process needs to do the request,
|
||
|
** since the root process will then tell the other
|
||
|
** processes at what offset they should read their
|
||
|
** share of the data.
|
||
|
*/
|
||
|
ret = mca_sharedfp_addproc_request_position(sh,bytesRequested,&offsetReceived);
|
||
|
if( OMPI_SUCCESS != ret ){
|
||
|
goto exit;
|
||
|
}
|
||
|
if ( mca_sharedfp_addproc_verbose ){
|
||
|
opal_output(ompi_sharedfp_base_framework.framework_output,
|
||
|
"sharedfp_addproc_read_ordered_begin: Offset received is %lld\n",
|
||
|
offsetReceived);
|
||
|
}
|
||
|
buff[0] += offsetReceived;
|
||
|
|
||
|
|
||
|
for (i = 1 ; i < size; i++) {
|
||
|
buff[i] += buff[i-1];
|
||
|
}
|
||
|
}
|
||
|
|
||
|
/* Scatter the results to the other processes*/
|
||
|
ret = sh->comm->c_coll.coll_scatter ( buff, 1, OMPI_OFFSET_DATATYPE, &offsetBuff,
|
||
|
1, OMPI_OFFSET_DATATYPE, 0, sh->comm,
|
||
|
sh->comm->c_coll.coll_scatter_module );
|
||
|
if ( OMPI_SUCCESS != ret ) {
|
||
|
goto exit;
|
||
|
}
|
||
|
|
||
|
/*Each process now has its own individual offset in recvBUFF*/
|
||
|
offset = offsetBuff - sendBuff;
|
||
|
offset /= sh->sharedfh->f_etype_size;
|
||
|
|
||
|
if ( mca_sharedfp_addproc_verbose ){
|
||
|
opal_output(ompi_sharedfp_base_framework.framework_output,
|
||
|
"sharedfp_addproc_read_ordered_begin: Offset returned is %lld\n",offset);
|
||
|
}
|
||
|
|
||
|
/* read from the file */
|
||
|
ret = mca_common_ompio_file_iread_at_all(sh->sharedfh,offset,buf,count,datatype,&fh->f_split_coll_req);
|
||
|
fh->f_split_coll_in_use = true;
|
||
|
|
||
|
exit:
|
||
|
if ( NULL != buff ) {
|
||
|
free ( buff );
|
||
|
}
|
||
|
|
||
|
return ret;
|
||
|
|
||
|
}
|
||
|
|
||
|
|
||
|
int mca_sharedfp_addproc_read_ordered_end(mca_io_ompio_file_t *fh,
|
||
|
void *buf,
|
||
|
ompi_status_public_t *status)
|
||
|
{
|
||
|
int ret = OMPI_SUCCESS;
|
||
|
ret = ompi_request_wait ( &fh->f_split_coll_req, status );
|
||
|
|
||
|
/* remove the flag again */
|
||
|
fh->f_split_coll_in_use = false;
|
||
|
return ret;
|
||
|
}
|