sharedfp/all components: revamp internal operations
this commit revamps the internal operations of the sharedfp components. Specifically, it is focused around removing the second file_open operation for shared file pointers. This makes the code more efficient. Because of that, there is no necessity anymore for the sharedfp_lazy_open mca parameter. Signed-off-by: Edgar Gabriel <egabriel@central.uh.edu>
Этот коммит содержится в:
Edgar Gabriel
родитель
a4bfa11425
Коммит
bc0f60dfd9
@ -92,8 +92,6 @@ int mca_common_ompio_file_open (ompi_communicator_t *comm,
|
||||
ompio_fh->f_fstype = NONE;
|
||||
ompio_fh->f_amode = amode;
|
||||
ompio_fh->f_info = info;
|
||||
ompio_fh->f_atomicity = 0;
|
||||
ompio_fh->f_fs_block_size = 4096;
|
||||
|
||||
/* set some function pointers required for fcoll, fbtls and sharedfp modules*/
|
||||
ompio_fh->f_generate_current_file_view=generate_current_file_view_fn;
|
||||
@ -165,19 +163,21 @@ int mca_common_ompio_file_open (ompi_communicator_t *comm,
|
||||
goto fn_fail;
|
||||
}
|
||||
|
||||
if (OMPI_SUCCESS != (ret = mca_fcoll_base_file_select (ompio_fh,
|
||||
NULL))) {
|
||||
opal_output(1, "mca_fcoll_base_file_select() failed\n");
|
||||
goto fn_fail;
|
||||
}
|
||||
/* Set default file view */
|
||||
mca_common_ompio_set_view(ompio_fh,
|
||||
0,
|
||||
&ompi_mpi_byte.dt,
|
||||
&ompi_mpi_byte.dt,
|
||||
"native",
|
||||
info);
|
||||
|
||||
|
||||
if ( true == use_sharedfp ) {
|
||||
/* open the file once more for the shared file pointer if required.
|
||||
** Can be disabled by the user if no shared file pointer operations
|
||||
** are used by his application.
|
||||
*/
|
||||
if ( NULL != ompio_fh->f_sharedfp &&
|
||||
!OMPIO_MCA_GET(ompio_fh, sharedfp_lazy_open) ) {
|
||||
if ( NULL != ompio_fh->f_sharedfp ) {
|
||||
ret = ompio_fh->f_sharedfp->sharedfp_file_open(comm,
|
||||
filename,
|
||||
amode,
|
||||
@ -201,8 +201,7 @@ int mca_common_ompio_file_open (ompi_communicator_t *comm,
|
||||
¤t_size);
|
||||
mca_common_ompio_set_explicit_offset (ompio_fh, current_size);
|
||||
if ( true == use_sharedfp ) {
|
||||
if ( NULL != ompio_fh->f_sharedfp &&
|
||||
!OMPIO_MCA_GET(ompio_fh, sharedfp_lazy_open) ) {
|
||||
if ( NULL != ompio_fh->f_sharedfp ) {
|
||||
shared_fp_base_module = ompio_fh->f_sharedfp;
|
||||
ret = shared_fp_base_module->sharedfp_seek(ompio_fh,current_size, MPI_SEEK_SET);
|
||||
if ( MPI_SUCCESS != ret ) {
|
||||
@ -346,6 +345,7 @@ int mca_common_ompio_file_close (ompio_file_t *ompio_fh)
|
||||
|
||||
if (MPI_DATATYPE_NULL != ompio_fh->f_iov_type) {
|
||||
ompi_datatype_destroy (&ompio_fh->f_iov_type);
|
||||
ompio_fh->f_iov_type=MPI_DATATYPE_NULL;
|
||||
}
|
||||
|
||||
if ( MPI_DATATYPE_NULL != ompio_fh->f_etype ) {
|
||||
@ -409,7 +409,8 @@ int mca_common_ompio_set_file_defaults (ompio_file_t *fh)
|
||||
fh->f_perm = OMPIO_PERM_NULL;
|
||||
fh->f_flags = 0;
|
||||
fh->f_bytes_per_agg = OMPIO_MCA_GET(fh, bytes_per_agg);
|
||||
fh->f_datarep = strdup ("native");
|
||||
fh->f_atomicity = 0;
|
||||
fh->f_fs_block_size = 4096;
|
||||
|
||||
fh->f_offset = 0;
|
||||
fh->f_disp = 0;
|
||||
@ -434,17 +435,10 @@ int mca_common_ompio_set_file_defaults (ompio_file_t *fh)
|
||||
fh->f_stripe_size = 0;
|
||||
/*Decoded iovec of the file-view*/
|
||||
fh->f_decoded_iov = NULL;
|
||||
fh->f_etype = NULL;
|
||||
fh->f_filetype = NULL;
|
||||
fh->f_orig_filetype = NULL;
|
||||
|
||||
mca_common_ompio_set_view(fh,
|
||||
0,
|
||||
&ompi_mpi_byte.dt,
|
||||
&ompi_mpi_byte.dt,
|
||||
"native",
|
||||
fh->f_info);
|
||||
|
||||
fh->f_etype = MPI_DATATYPE_NULL;
|
||||
fh->f_filetype = MPI_DATATYPE_NULL;
|
||||
fh->f_orig_filetype = MPI_DATATYPE_NULL;
|
||||
fh->f_datarep = NULL;
|
||||
|
||||
/*Create a derived datatype for the created iovec */
|
||||
types[0] = &ompi_mpi_long.dt;
|
||||
|
||||
@ -9,7 +9,7 @@
|
||||
* University of Stuttgart. All rights reserved.
|
||||
* Copyright (c) 2004-2005 The Regents of the University of California.
|
||||
* All rights reserved.
|
||||
* Copyright (c) 2008-2017 University of Houston. All rights reserved.
|
||||
* Copyright (c) 2008-2018 University of Houston. All rights reserved.
|
||||
* Copyright (c) 2017-2018 Research Organization for Information Science
|
||||
* and Technology (RIST). All rights reserved.
|
||||
* Copyright (c) 2017 IBM Corporation. All rights reserved.
|
||||
@ -281,6 +281,10 @@ int mca_common_ompio_set_view (ompio_file_t *fh,
|
||||
goto exit;
|
||||
}
|
||||
|
||||
if ( NULL != fh->f_sharedfp ) {
|
||||
ret = fh->f_sharedfp->sharedfp_seek( fh, 0, MPI_SEEK_SET);
|
||||
}
|
||||
|
||||
exit:
|
||||
for( i = 0; i < fh->f_size; i++){
|
||||
free(contg_groups[i].procs_in_contg_group);
|
||||
|
||||
@ -115,12 +115,10 @@ mca_fs_lustre_file_open (struct ompi_communicator_t *comm,
|
||||
errno = 0;
|
||||
if (0 == fh->f_rank) {
|
||||
/* MODE_CREATE and MODE_EXCL can only be set by one process */
|
||||
if ( !(fh->f_flags & OMPIO_SHAREDFP_IS_SET)) {
|
||||
if ( access_mode & MPI_MODE_CREATE )
|
||||
amode = amode | O_CREAT;
|
||||
if (access_mode & MPI_MODE_EXCL)
|
||||
amode = amode | O_EXCL;
|
||||
}
|
||||
|
||||
if ( (fs_lustre_stripe_size>0 || fs_lustre_stripe_width>0) &&
|
||||
( amode&O_CREAT) &&
|
||||
|
||||
@ -76,12 +76,11 @@ mca_fs_ufs_file_open (struct ompi_communicator_t *comm,
|
||||
errno = 0;
|
||||
if ( 0 == rank ) {
|
||||
/* MODE_CREATE and MODE_EXCL can only be set by one process */
|
||||
if ( !(fh->f_flags & OMPIO_SHAREDFP_IS_SET)) {
|
||||
if ( access_mode & MPI_MODE_CREATE )
|
||||
amode = amode | O_CREAT;
|
||||
if (access_mode & MPI_MODE_EXCL)
|
||||
amode = amode | O_EXCL;
|
||||
}
|
||||
|
||||
fh->fd = open (filename, amode, perm);
|
||||
if ( 0 > fh->fd ) {
|
||||
if ( EACCES == errno ) {
|
||||
|
||||
@ -10,7 +10,7 @@
|
||||
* University of Stuttgart. All rights reserved.
|
||||
* Copyright (c) 2004-2005 The Regents of the University of California.
|
||||
* All rights reserved.
|
||||
* Copyright (c) 2008-2016 University of Houston. All rights reserved.
|
||||
* Copyright (c) 2008-2018 University of Houston. All rights reserved.
|
||||
* Copyright (c) 2011-2015 Cisco Systems, Inc. All rights reserved.
|
||||
* Copyright (c) 2012-2013 Inria. All rights reserved.
|
||||
* Copyright (c) 2015-2018 Research Organization for Information Science
|
||||
@ -542,9 +542,6 @@ int mca_io_ompio_get_mca_parameter_value ( char *mca_parameter_name, int name_le
|
||||
else if ( !strncmp ( mca_parameter_name, "grouping_option", name_length )) {
|
||||
return mca_io_ompio_grouping_option;
|
||||
}
|
||||
else if ( !strncmp ( mca_parameter_name, "sharedfp_lazy_open", name_length )) {
|
||||
return mca_io_ompio_sharedfp_lazy_open;
|
||||
}
|
||||
else if ( !strncmp ( mca_parameter_name, "coll_timing_info", name_length )) {
|
||||
return mca_io_ompio_coll_timing_info;
|
||||
}
|
||||
|
||||
@ -48,7 +48,6 @@ extern int mca_io_ompio_cycle_buffer_size;
|
||||
extern int mca_io_ompio_bytes_per_agg;
|
||||
extern int mca_io_ompio_num_aggregators;
|
||||
extern int mca_io_ompio_record_offset_info;
|
||||
extern int mca_io_ompio_sharedfp_lazy_open;
|
||||
extern int mca_io_ompio_grouping_option;
|
||||
extern int mca_io_ompio_max_aggregators_ratio;
|
||||
extern int mca_io_ompio_aggregators_cutoff_threshold;
|
||||
|
||||
@ -39,7 +39,6 @@ int mca_io_ompio_bytes_per_agg = OMPIO_PREALLOC_MAX_BUF_SIZE;
|
||||
int mca_io_ompio_num_aggregators = -1;
|
||||
int mca_io_ompio_record_offset_info = 0;
|
||||
int mca_io_ompio_coll_timing_info = 0;
|
||||
int mca_io_ompio_sharedfp_lazy_open = 0;
|
||||
int mca_io_ompio_max_aggregators_ratio=8;
|
||||
int mca_io_ompio_aggregators_cutoff_threshold=3;
|
||||
int mca_io_ompio_overwrite_amode = 1;
|
||||
@ -194,15 +193,6 @@ static int register_component(void)
|
||||
&mca_io_ompio_num_aggregators);
|
||||
|
||||
|
||||
mca_io_ompio_sharedfp_lazy_open = 0;
|
||||
(void) mca_base_component_var_register(&mca_io_ompio_component.io_version,
|
||||
"sharedfp_lazy_open",
|
||||
"lazy allocation of internal shared file pointer structures",
|
||||
MCA_BASE_VAR_TYPE_INT, NULL, 0, 0,
|
||||
OPAL_INFO_LVL_9,
|
||||
MCA_BASE_VAR_SCOPE_READONLY,
|
||||
&mca_io_ompio_sharedfp_lazy_open);
|
||||
|
||||
mca_io_ompio_grouping_option = 5;
|
||||
(void) mca_base_component_var_register(&mca_io_ompio_component.io_version,
|
||||
"grouping_option",
|
||||
|
||||
@ -65,7 +65,6 @@ int mca_io_ompio_file_set_view (ompi_file_t *fp,
|
||||
int ret=OMPI_SUCCESS;
|
||||
mca_common_ompio_data_t *data;
|
||||
ompio_file_t *fh;
|
||||
ompio_file_t *sh;
|
||||
|
||||
data = (mca_common_ompio_data_t *) fp->f_io_selected_data;
|
||||
|
||||
@ -76,12 +75,6 @@ int mca_io_ompio_file_set_view (ompi_file_t *fp,
|
||||
|
||||
OPAL_THREAD_LOCK(&fp->f_lock);
|
||||
ret = mca_common_ompio_set_view(fh, disp, etype, filetype, datarep, info);
|
||||
|
||||
if ( NULL != fh->f_sharedfp_data) {
|
||||
sh = ((struct mca_sharedfp_base_data_t *)fh->f_sharedfp_data)->sharedfh;
|
||||
ret = mca_common_ompio_set_view(sh, disp, etype, filetype, datarep, info);
|
||||
}
|
||||
|
||||
OPAL_THREAD_UNLOCK(&fp->f_lock);
|
||||
return ret;
|
||||
}
|
||||
|
||||
@ -139,7 +139,7 @@ typedef struct mca_sharedfp_individual_header_record_s{
|
||||
|
||||
mca_sharedfp_individual_header_record* mca_sharedfp_individual_insert_headnode(void);
|
||||
|
||||
int mca_sharedfp_individual_collaborate_data(struct mca_sharedfp_base_data_t *sh);
|
||||
int mca_sharedfp_individual_collaborate_data(struct mca_sharedfp_base_data_t *sh, ompio_file_t *ompio_fh );
|
||||
int mca_sharedfp_individual_get_timestamps_and_reclengths(double **buff, long **rec_length, MPI_Offset **offbuff,struct mca_sharedfp_base_data_t *sh);
|
||||
int mca_sharedfp_individual_create_buff(double **ts,MPI_Offset **off,int totalnodes,int size);
|
||||
int mca_sharedfp_individual_sort_timestamps(double **ts,MPI_Offset **off, int **ranks, int totalnodes);
|
||||
|
||||
@ -9,7 +9,7 @@
|
||||
* University of Stuttgart. All rights reserved.
|
||||
* Copyright (c) 2004-2005 The Regents of the University of California.
|
||||
* All rights reserved.
|
||||
* Copyright (c) 2013-2016 University of Houston. All rights reserved.
|
||||
* Copyright (c) 2013-2018 University of Houston. All rights reserved.
|
||||
* Copyright (c) 2018 Research Organization for Information Science
|
||||
* and Technology (RIST). All rights reserved.
|
||||
* $COPYRIGHT$
|
||||
@ -32,13 +32,11 @@
|
||||
#include <stdlib.h>
|
||||
#include <stdio.h>
|
||||
|
||||
int mca_sharedfp_individual_collaborate_data(struct mca_sharedfp_base_data_t *sh)
|
||||
int mca_sharedfp_individual_collaborate_data(struct mca_sharedfp_base_data_t *sh, ompio_file_t *ompio_fh)
|
||||
{
|
||||
int ret = OMPI_SUCCESS;
|
||||
mca_sharedfp_individual_header_record *headnode = NULL;
|
||||
char *buff=NULL;
|
||||
MPI_Comm comm;
|
||||
int rank, size;
|
||||
int nodesoneachprocess = 0;
|
||||
int idx=0,i=0,j=0, l=0;
|
||||
int *ranks = NULL;
|
||||
@ -53,11 +51,6 @@ int mca_sharedfp_individual_collaborate_data(struct mca_sharedfp_base_data_t *sh
|
||||
ompi_status_public_t status;
|
||||
int recordlength=0;
|
||||
|
||||
comm = sh->comm;
|
||||
|
||||
rank = ompi_comm_rank ( comm );
|
||||
size = ompi_comm_size ( comm );
|
||||
|
||||
headnode = (mca_sharedfp_individual_header_record*)sh->selected_module_data;
|
||||
if ( NULL == headnode) {
|
||||
opal_output(0, "sharedfp_individual_collaborate_data: headnode is NULL but file is open\n");
|
||||
@ -74,12 +67,12 @@ int mca_sharedfp_individual_collaborate_data(struct mca_sharedfp_base_data_t *sh
|
||||
"Nodes of each process = %d\n",nodesoneachprocess);
|
||||
}
|
||||
|
||||
countbuff = (int*)malloc(size * sizeof(int));
|
||||
countbuff = (int*)malloc(ompio_fh->f_size * sizeof(int));
|
||||
if ( NULL == countbuff ) {
|
||||
return OMPI_ERR_OUT_OF_RESOURCE;
|
||||
}
|
||||
|
||||
displ = (int*)malloc(sizeof(int) * size);
|
||||
displ = (int*)malloc(sizeof(int) * ompio_fh->f_size);
|
||||
if ( NULL == displ ) {
|
||||
ret = OMPI_ERR_OUT_OF_RESOURCE;
|
||||
goto exit;
|
||||
@ -93,13 +86,24 @@ int mca_sharedfp_individual_collaborate_data(struct mca_sharedfp_base_data_t *sh
|
||||
goto exit;
|
||||
}
|
||||
|
||||
comm->c_coll->coll_allgather ( &nodesoneachprocess, 1, MPI_INT,
|
||||
countbuff, 1, MPI_INT, comm,
|
||||
comm->c_coll->coll_allgather_module );
|
||||
ret = ompio_fh->f_comm->c_coll->coll_allgather ( &nodesoneachprocess,
|
||||
1,
|
||||
MPI_INT,
|
||||
countbuff,
|
||||
1,
|
||||
MPI_INT,
|
||||
ompio_fh->f_comm,
|
||||
ompio_fh->f_comm->c_coll->coll_allgather_module );
|
||||
|
||||
if ( OMPI_SUCCESS != ret ) {
|
||||
goto exit;
|
||||
}
|
||||
|
||||
|
||||
if ( mca_sharedfp_individual_verbose) {
|
||||
for (i = 0; i < size ; i++) {
|
||||
opal_output(ompi_sharedfp_base_framework.framework_output,"sharedfp_individual_collaborate_data: Countbuff[%d] = %d\n", i, countbuff[i]);
|
||||
for (i = 0; i < ompio_fh->f_size ; i++) {
|
||||
opal_output(ompi_sharedfp_base_framework.framework_output,"sharedfp_individual_collaborate_data: "
|
||||
"Countbuff[%d] = %d\n", i, countbuff[i]);
|
||||
}
|
||||
}
|
||||
|
||||
@ -109,7 +113,7 @@ int mca_sharedfp_individual_collaborate_data(struct mca_sharedfp_base_data_t *sh
|
||||
local_off[0] = 0;
|
||||
}
|
||||
|
||||
for(i = 0; i < size; i++) {
|
||||
for(i = 0; i < ompio_fh->f_size; i++) {
|
||||
displ[i] = totalnodes;
|
||||
if ( mca_sharedfp_individual_verbose ) {
|
||||
opal_output(ompi_sharedfp_base_framework.framework_output,
|
||||
@ -127,27 +131,39 @@ int mca_sharedfp_individual_collaborate_data(struct mca_sharedfp_base_data_t *sh
|
||||
ret = OMPI_ERR_OUT_OF_RESOURCE;
|
||||
goto exit;
|
||||
}
|
||||
for ( l=0, i=0; i<size; i++ ) {
|
||||
for ( l=0, i=0; i< ompio_fh->f_size; i++ ) {
|
||||
for ( j=0; j< countbuff[i]; j++ ) {
|
||||
ranks[l++]=i;
|
||||
}
|
||||
}
|
||||
|
||||
ret = mca_sharedfp_individual_create_buff ( ×tampbuff, &offsetbuff, totalnodes, size);
|
||||
ret = mca_sharedfp_individual_create_buff ( ×tampbuff, &offsetbuff, totalnodes, ompio_fh->f_size);
|
||||
if ( OMPI_SUCCESS != ret ) {
|
||||
goto exit;
|
||||
}
|
||||
|
||||
ret = comm->c_coll->coll_allgatherv ( ind_ts, countbuff[rank], MPI_DOUBLE,
|
||||
timestampbuff, countbuff, displ, MPI_DOUBLE,
|
||||
comm, comm->c_coll->coll_allgatherv_module );
|
||||
ret = ompio_fh->f_comm->c_coll->coll_allgatherv ( ind_ts,
|
||||
countbuff[ompio_fh->f_rank],
|
||||
MPI_DOUBLE,
|
||||
timestampbuff,
|
||||
countbuff,
|
||||
displ,
|
||||
MPI_DOUBLE,
|
||||
ompio_fh->f_comm,
|
||||
ompio_fh->f_comm->c_coll->coll_allgatherv_module );
|
||||
if ( OMPI_SUCCESS != ret ) {
|
||||
goto exit;
|
||||
}
|
||||
|
||||
ret = comm->c_coll->coll_allgatherv ( ind_recordlength, countbuff[rank], OMPI_OFFSET_DATATYPE,
|
||||
offsetbuff, countbuff, displ, OMPI_OFFSET_DATATYPE,
|
||||
comm, comm->c_coll->coll_allgatherv_module );
|
||||
ret = ompio_fh->f_comm->c_coll->coll_allgatherv ( ind_recordlength,
|
||||
countbuff[ompio_fh->f_rank],
|
||||
OMPI_OFFSET_DATATYPE,
|
||||
offsetbuff,
|
||||
countbuff,
|
||||
displ,
|
||||
OMPI_OFFSET_DATATYPE,
|
||||
ompio_fh->f_comm,
|
||||
ompio_fh->f_comm->c_coll->coll_allgatherv_module );
|
||||
if ( OMPI_SUCCESS != ret ) {
|
||||
goto exit;
|
||||
}
|
||||
@ -177,21 +193,27 @@ int mca_sharedfp_individual_collaborate_data(struct mca_sharedfp_base_data_t *sh
|
||||
}
|
||||
|
||||
/*Read from the local data file*/
|
||||
mca_common_ompio_file_read_at ( headnode->datafilehandle,
|
||||
ret = mca_common_ompio_file_read_at ( headnode->datafilehandle,
|
||||
local_off[i], buff, ind_recordlength[i],
|
||||
MPI_BYTE, &status);
|
||||
if ( OMPI_SUCCESS != ret ) {
|
||||
goto exit;
|
||||
}
|
||||
|
||||
idx = mca_sharedfp_individual_getoffset(ind_ts[i],timestampbuff, ranks, rank, totalnodes);
|
||||
idx = mca_sharedfp_individual_getoffset(ind_ts[i],timestampbuff, ranks, ompio_fh->f_rank, totalnodes);
|
||||
|
||||
if ( mca_sharedfp_individual_verbose ) {
|
||||
opal_output(ompi_sharedfp_base_framework.framework_output,
|
||||
"sharedfp_individual_collaborate_data: Process %d writing %ld bytes to main file at position"
|
||||
"%lld (%d)\n", rank, ind_recordlength[i], offsetbuff[idx], idx);
|
||||
"%lld (%d)\n", ompio_fh->f_rank, ind_recordlength[i], offsetbuff[idx], idx);
|
||||
}
|
||||
|
||||
/*Write into main data file*/
|
||||
mca_common_ompio_file_write_at( sh->sharedfh, offsetbuff[idx], buff,
|
||||
ret = mca_common_ompio_file_write_at( ompio_fh, offsetbuff[idx], buff,
|
||||
ind_recordlength[i], MPI_BYTE, &status);
|
||||
if ( OMPI_SUCCESS != ret ) {
|
||||
goto exit;
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
@ -229,7 +251,8 @@ exit:
|
||||
}
|
||||
|
||||
/* Count the number of nodes and create and array of the timestamps*/
|
||||
int mca_sharedfp_individual_get_timestamps_and_reclengths ( double **buff, long **rec_length, MPI_Offset **offbuff,struct mca_sharedfp_base_data_t *sh)
|
||||
int mca_sharedfp_individual_get_timestamps_and_reclengths ( double **buff, long **rec_length,
|
||||
MPI_Offset **offbuff,struct mca_sharedfp_base_data_t *sh)
|
||||
{
|
||||
int num = 0, i= 0, ctr = 0;
|
||||
int ret=OMPI_SUCCESS;
|
||||
@ -277,7 +300,11 @@ int mca_sharedfp_individual_get_timestamps_and_reclengths ( double **buff, long
|
||||
ctr = 0;
|
||||
for (i = 0; i < headnode->numofrecordsonfile ; i++) {
|
||||
|
||||
mca_common_ompio_file_read_at(headnode->metadatafilehandle,metaoffset, &rec, 32, MPI_BYTE,&status);
|
||||
ret = mca_common_ompio_file_read_at(headnode->metadatafilehandle,metaoffset,
|
||||
&rec, 32, MPI_BYTE,&status);
|
||||
if ( OMPI_SUCCESS != ret ) {
|
||||
goto exit;
|
||||
}
|
||||
|
||||
*(*rec_length + ctr) = rec.recordlength;
|
||||
*(*buff + ctr) = rec.timestamp;
|
||||
@ -390,7 +417,8 @@ int mca_sharedfp_individual_sort_timestamps(double **ts, MPI_Offset **off, int
|
||||
}
|
||||
|
||||
|
||||
MPI_Offset mca_sharedfp_individual_assign_globaloffset(MPI_Offset **offsetbuff,int totalnodes,struct mca_sharedfp_base_data_t *sh)
|
||||
MPI_Offset mca_sharedfp_individual_assign_globaloffset(MPI_Offset **offsetbuff,int totalnodes,
|
||||
struct mca_sharedfp_base_data_t *sh)
|
||||
{
|
||||
int i = 0;
|
||||
OMPI_MPI_OFFSET_TYPE temp = 0,prevoffset = 0;
|
||||
|
||||
@ -9,7 +9,7 @@
|
||||
* University of Stuttgart. All rights reserved.
|
||||
* Copyright (c) 2004-2005 The Regents of the University of California.
|
||||
* All rights reserved.
|
||||
* Copyright (c) 2013-2016 University of Houston. All rights reserved.
|
||||
* Copyright (c) 2013-2018 University of Houston. All rights reserved.
|
||||
* Copyright (c) 2015-2018 Research Organization for Information Science
|
||||
* and Technology (RIST). All rights reserved.
|
||||
* Copyright (c) 2016-2017 IBM Corporation. All rights reserved.
|
||||
@ -38,46 +38,24 @@ int mca_sharedfp_individual_file_open (struct ompi_communicator_t *comm,
|
||||
ompio_file_t *fh)
|
||||
{
|
||||
int err = 0;
|
||||
int rank;
|
||||
char * datafilename; /*The array size would change as it is based on the current path*/
|
||||
char * metadatafilename; /*The array size would change as it is based on the current path*/
|
||||
ompio_file_t * datafilehandle;
|
||||
ompio_file_t * metadatafilehandle;
|
||||
mca_sharedfp_individual_header_record* headnode = NULL;
|
||||
struct mca_sharedfp_base_data_t* sh;
|
||||
ompio_file_t * shfileHandle;
|
||||
size_t len=0;
|
||||
|
||||
/*-------------------------------------------------*/
|
||||
/*Open the same file again without shared file pointer*/
|
||||
/*-------------------------------------------------*/
|
||||
shfileHandle = (ompio_file_t *) malloc ( sizeof(ompio_file_t));
|
||||
if ( NULL == shfileHandle ) {
|
||||
opal_output(0, "mca_sharedfp_individual_file_open: unable to allocate memory\n");
|
||||
return OMPI_ERR_OUT_OF_RESOURCE;
|
||||
}
|
||||
|
||||
err = mca_common_ompio_file_open ( comm, filename, amode, info, shfileHandle, false);
|
||||
if ( OMPI_SUCCESS != err ) {
|
||||
opal_output(0, "mca_sharedfp_individual_file_open: Error during file open\n");
|
||||
return err;
|
||||
}
|
||||
shfileHandle->f_fh = fh->f_fh;
|
||||
|
||||
sh = (struct mca_sharedfp_base_data_t*) malloc ( sizeof(struct mca_sharedfp_base_data_t));
|
||||
if ( NULL == sh ){
|
||||
opal_output(0, "mca_sharedfp_individual_file_open: Error, unable to malloc "
|
||||
"f_sharedfp_ptr struct\n");
|
||||
free ( shfileHandle );
|
||||
return OMPI_ERR_OUT_OF_RESOURCE;
|
||||
}
|
||||
|
||||
rank = ompi_comm_rank ( comm );
|
||||
|
||||
/*Populate the sh file structure based on the implementation*/
|
||||
sh->sharedfh = shfileHandle; /* Shared file pointer*/
|
||||
sh->global_offset = 0; /* Global Offset*/
|
||||
sh->comm = comm; /* Communicator*/
|
||||
sh->selected_module_data = NULL;
|
||||
|
||||
/* Assign the metadatalinked list to sh->handle */
|
||||
@ -97,17 +75,15 @@ int mca_sharedfp_individual_file_open (struct ompi_communicator_t *comm,
|
||||
datafilename = (char*)malloc( len );
|
||||
if ( NULL == datafilename ) {
|
||||
opal_output(0, "mca_sharedfp_individual_file_open: unable to allocate memory\n");
|
||||
free ( shfileHandle );
|
||||
free ( sh );
|
||||
return OMPI_ERR_OUT_OF_RESOURCE;
|
||||
}
|
||||
snprintf(datafilename, len, "%s%s%d",filename,".data.",rank);
|
||||
snprintf(datafilename, len, "%s%s%d",filename,".data.",fh->f_rank);
|
||||
|
||||
|
||||
datafilehandle = (ompio_file_t *)malloc(sizeof(ompio_file_t));
|
||||
if ( NULL == datafilehandle ) {
|
||||
opal_output(0, "mca_sharedfp_individual_file_open: unable to allocate memory\n");
|
||||
free ( shfileHandle );
|
||||
free ( sh );
|
||||
free ( datafilename );
|
||||
return OMPI_ERR_OUT_OF_RESOURCE;
|
||||
@ -117,7 +93,6 @@ int mca_sharedfp_individual_file_open (struct ompi_communicator_t *comm,
|
||||
&(MPI_INFO_NULL->super), datafilehandle, false);
|
||||
if ( OMPI_SUCCESS != err) {
|
||||
opal_output(0, "mca_sharedfp_individual_file_open: Error during datafile file open\n");
|
||||
free (shfileHandle );
|
||||
free (sh);
|
||||
free (datafilename);
|
||||
free (datafilehandle);
|
||||
@ -136,18 +111,16 @@ int mca_sharedfp_individual_file_open (struct ompi_communicator_t *comm,
|
||||
/* metadata filename created by appending .metadata.$rank to the original filename*/
|
||||
metadatafilename = (char*) malloc ( len );
|
||||
if ( NULL == metadatafilename ) {
|
||||
free (shfileHandle );
|
||||
free (sh);
|
||||
free (datafilename);
|
||||
free (datafilehandle);
|
||||
opal_output(0, "mca_sharedfp_individual_file_open: Error during memory allocation\n");
|
||||
return OMPI_ERR_OUT_OF_RESOURCE;
|
||||
}
|
||||
snprintf ( metadatafilename, len, "%s%s%d", filename, ".metadata.",rank);
|
||||
snprintf ( metadatafilename, len, "%s%s%d", filename, ".metadata.",fh->f_rank);
|
||||
|
||||
metadatafilehandle = (ompio_file_t *)malloc(sizeof(ompio_file_t));
|
||||
if ( NULL == metadatafilehandle ) {
|
||||
free (shfileHandle );
|
||||
free (sh);
|
||||
free (datafilename);
|
||||
free (datafilehandle);
|
||||
@ -160,7 +133,6 @@ int mca_sharedfp_individual_file_open (struct ompi_communicator_t *comm,
|
||||
&(MPI_INFO_NULL->super), metadatafilehandle, false);
|
||||
if ( OMPI_SUCCESS != err) {
|
||||
opal_output(0, "mca_sharedfp_individual_file_open: Error during metadatafile file open\n");
|
||||
free (shfileHandle );
|
||||
free (sh);
|
||||
free (datafilename);
|
||||
free (datafilehandle);
|
||||
@ -191,16 +163,12 @@ int mca_sharedfp_individual_file_close (ompio_file_t *fh)
|
||||
int err = OMPI_SUCCESS;
|
||||
|
||||
if ( NULL == fh->f_sharedfp_data ){
|
||||
if ( mca_sharedfp_individual_verbose ) {
|
||||
opal_output(ompi_sharedfp_base_framework.framework_output,
|
||||
"sharedfp_inidividual_file_close - shared file pointer structure not initialized\n");
|
||||
}
|
||||
return OMPI_SUCCESS;
|
||||
}
|
||||
sh = fh->f_sharedfp_data;
|
||||
|
||||
/* Merge data from individal files to final output file */
|
||||
err = mca_sharedfp_individual_collaborate_data (sh);
|
||||
err = mca_sharedfp_individual_collaborate_data (sh, fh);
|
||||
|
||||
headnode = (mca_sharedfp_individual_header_record*)(sh->selected_module_data);
|
||||
if (headnode) {
|
||||
@ -229,11 +197,9 @@ int mca_sharedfp_individual_file_close (ompio_file_t *fh)
|
||||
}
|
||||
}
|
||||
|
||||
/* Close the main file opened by this component*/
|
||||
err = mca_common_ompio_file_close(sh->sharedfh);
|
||||
|
||||
/*free shared file pointer data struct*/
|
||||
free(sh);
|
||||
fh->f_sharedfp_data=NULL;
|
||||
|
||||
return err;
|
||||
}
|
||||
|
||||
@ -9,7 +9,7 @@
|
||||
* University of Stuttgart. All rights reserved.
|
||||
* Copyright (c) 2004-2005 The Regents of the University of California.
|
||||
* All rights reserved.
|
||||
* Copyright (c) 2013-2016 University of Houston. All rights reserved.
|
||||
* Copyright (c) 2013-2018 University of Houston. All rights reserved.
|
||||
* Copyright (c) 2015-2018 Research Organization for Information Science
|
||||
* and Technology (RIST). All rights reserved.
|
||||
* $COPYRIGHT$
|
||||
@ -39,24 +39,11 @@ int mca_sharedfp_individual_iwrite(ompio_file_t *fh,
|
||||
OMPI_MPI_OFFSET_TYPE totalbytes = 0;
|
||||
mca_sharedfp_individual_header_record *headnode = NULL;
|
||||
struct mca_sharedfp_base_data_t *sh = NULL;
|
||||
mca_sharedfp_base_module_t * shared_fp_base_module = NULL;
|
||||
|
||||
if(fh->f_sharedfp_data==NULL){
|
||||
if ( mca_sharedfp_individual_verbose ) {
|
||||
opal_output(ompi_sharedfp_base_framework.framework_output,
|
||||
"mca_sharedfp_individual_iwrite: opening the shared file pointer\n");
|
||||
}
|
||||
shared_fp_base_module = fh->f_sharedfp;
|
||||
|
||||
ret = shared_fp_base_module->sharedfp_file_open(fh->f_comm,
|
||||
fh->f_filename,
|
||||
fh->f_amode,
|
||||
fh->f_info,
|
||||
fh);
|
||||
if (ret != OMPI_SUCCESS) {
|
||||
opal_output(0,"mca_sharedfp_individual_iwrite - error opening the shared file pointer\n");
|
||||
return ret;
|
||||
}
|
||||
"mca_sharedfp_individual_iwrite: module not initialized \n");
|
||||
return OMPI_ERROR;
|
||||
}
|
||||
|
||||
/* Calculate the number of bytes of data that needs to be written*/
|
||||
@ -95,7 +82,6 @@ int mca_sharedfp_individual_write_ordered_begin(ompio_file_t *fh,
|
||||
struct ompi_datatype_t *datatype)
|
||||
{
|
||||
int ret = OMPI_SUCCESS;
|
||||
int size = 0, rank = 0;
|
||||
int i = 0;
|
||||
size_t numofbytes = 0;
|
||||
size_t totalbytes = 0;
|
||||
@ -105,35 +91,21 @@ int mca_sharedfp_individual_write_ordered_begin(ompio_file_t *fh,
|
||||
OMPI_MPI_OFFSET_TYPE temp = 0, offset = 0;
|
||||
mca_sharedfp_individual_header_record *headnode = NULL;
|
||||
struct mca_sharedfp_base_data_t *sh = NULL;
|
||||
mca_sharedfp_base_module_t * shared_fp_base_module = NULL;
|
||||
|
||||
if(fh->f_sharedfp_data==NULL){
|
||||
if ( mca_sharedfp_individual_verbose ) {
|
||||
opal_output(ompi_sharedfp_base_framework.framework_output,
|
||||
"sharedfp_individual_write_ordered_begin - opening the shared file pointer\n");
|
||||
}
|
||||
shared_fp_base_module = fh->f_sharedfp;
|
||||
|
||||
ret = shared_fp_base_module->sharedfp_file_open(fh->f_comm,
|
||||
fh->f_filename,
|
||||
fh->f_amode,
|
||||
fh->f_info,
|
||||
fh);
|
||||
if ( OMPI_SUCCESS != ret ) {
|
||||
opal_output(0,"sharedfp_individual_write_ordered_begin - error opening the shared file pointer\n");
|
||||
return ret;
|
||||
}
|
||||
"sharedfp_individual_write_ordered_begin - module not initialized\n");
|
||||
return OMPI_ERROR;
|
||||
}
|
||||
|
||||
if ( true == fh->f_split_coll_in_use ) {
|
||||
opal_output(0, "Only one split collective I/O operation allowed per file handle at any given point in time!\n");
|
||||
opal_output(0, "Only one split collective I/O operation allowed per file handle "
|
||||
"at any given point in time!\n");
|
||||
return MPI_ERR_REQUEST;
|
||||
}
|
||||
|
||||
/*Retrieve the sharedfp data structures*/
|
||||
sh = fh->f_sharedfp_data;
|
||||
rank = ompi_comm_rank ( sh->comm );
|
||||
size = ompi_comm_size ( sh->comm );
|
||||
|
||||
/* Calculate the number of bytes of data that needs to be written*/
|
||||
opal_datatype_type_size ( &datatype->super, &numofbytes);
|
||||
@ -146,50 +118,71 @@ int mca_sharedfp_individual_write_ordered_begin(ompio_file_t *fh,
|
||||
}
|
||||
|
||||
/* Data from all the metadata is combined and written to the main file */
|
||||
ret = mca_sharedfp_individual_collaborate_data ( sh );
|
||||
ret = mca_sharedfp_individual_collaborate_data ( sh, fh );
|
||||
if ( OMPI_SUCCESS != ret) {
|
||||
return ret;
|
||||
}
|
||||
|
||||
if ( 0 == rank ) {
|
||||
offbuff = (OMPI_MPI_OFFSET_TYPE *)malloc ( sizeof(OMPI_MPI_OFFSET_TYPE) * size);
|
||||
if ( 0 == fh->f_rank ) {
|
||||
offbuff = (OMPI_MPI_OFFSET_TYPE *)malloc ( sizeof(OMPI_MPI_OFFSET_TYPE) * fh->f_size);
|
||||
if (NULL == offbuff ) {
|
||||
return OMPI_ERR_OUT_OF_RESOURCE;
|
||||
}
|
||||
}
|
||||
|
||||
/*collect the total bytes to be written*/
|
||||
sh->comm->c_coll->coll_gather ( &totalbytes, 1, OMPI_OFFSET_DATATYPE,
|
||||
offbuff, 1, OMPI_OFFSET_DATATYPE, 0,
|
||||
sh->comm, sh->comm->c_coll->coll_gather_module );
|
||||
ret = fh->f_comm->c_coll->coll_gather ( &totalbytes,
|
||||
1,
|
||||
OMPI_OFFSET_DATATYPE,
|
||||
offbuff,
|
||||
1,
|
||||
OMPI_OFFSET_DATATYPE,
|
||||
0,
|
||||
fh->f_comm,
|
||||
fh->f_comm->c_coll->coll_gather_module );
|
||||
|
||||
if ( 0 == rank ) {
|
||||
if ( OMPI_SUCCESS != ret ) {
|
||||
opal_output(0,"sharedfp_individual_write_ordered_begin: Error in gatherring offsets \n");
|
||||
goto exit;
|
||||
}
|
||||
|
||||
if ( 0 == fh->f_rank ) {
|
||||
prev_offset = offbuff[0];
|
||||
offbuff[0] = sh->global_offset;
|
||||
|
||||
for (i = 1; i < size ; i++){
|
||||
for (i = 1; i < fh->f_size ; i++){
|
||||
temp = offbuff[i];
|
||||
offbuff[i] = offbuff[i - 1] + prev_offset;
|
||||
prev_offset = temp;
|
||||
}
|
||||
|
||||
for (i = 0; i < size; i++){
|
||||
global_offset = offbuff[size - 1] + prev_offset;
|
||||
for (i = 0; i < fh->f_size; i++){
|
||||
global_offset = offbuff[fh->f_size - 1] + prev_offset;
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
/* Scatter the results to the other processes */
|
||||
ret = sh->comm->c_coll->coll_scatter ( offbuff, 1, OMPI_OFFSET_DATATYPE,
|
||||
&offset, 1, OMPI_OFFSET_DATATYPE, 0,
|
||||
sh->comm, sh->comm->c_coll->coll_scatter_module );
|
||||
ret = fh->f_comm->c_coll->coll_scatter ( offbuff,
|
||||
1,
|
||||
OMPI_OFFSET_DATATYPE,
|
||||
&offset,
|
||||
1,
|
||||
OMPI_OFFSET_DATATYPE,
|
||||
0,
|
||||
fh->f_comm,
|
||||
fh->f_comm->c_coll->coll_scatter_module );
|
||||
if ( OMPI_SUCCESS != ret ) {
|
||||
opal_output(0,"sharedfp_individual_write_ordered_begin: Error in scattering offsets \n");
|
||||
goto exit;
|
||||
}
|
||||
|
||||
ret = sh->comm->c_coll->coll_bcast ( &global_offset, 1, OMPI_OFFSET_DATATYPE,
|
||||
0, sh->comm, sh->comm->c_coll->coll_bcast_module );
|
||||
ret = fh->f_comm->c_coll->coll_bcast ( &global_offset,
|
||||
1,
|
||||
OMPI_OFFSET_DATATYPE,
|
||||
0,
|
||||
fh->f_comm,
|
||||
fh->f_comm->c_coll->coll_bcast_module );
|
||||
if ( OMPI_SUCCESS != ret ) {
|
||||
opal_output(0,"sharedfp_individual_write_ordered_begin: Error while bcasting global offset \n");
|
||||
goto exit;
|
||||
@ -198,7 +191,7 @@ int mca_sharedfp_individual_write_ordered_begin(ompio_file_t *fh,
|
||||
sh->global_offset = global_offset;
|
||||
|
||||
/*use file_write_at_all to ensure the order*/
|
||||
ret = mca_common_ompio_file_iwrite_at_all(sh->sharedfh,offset, buf,count,datatype,
|
||||
ret = mca_common_ompio_file_iwrite_at_all(fh, offset, buf, count, datatype,
|
||||
&fh->f_split_coll_req);
|
||||
fh->f_split_coll_in_use = true;
|
||||
if ( OMPI_SUCCESS != ret ) {
|
||||
|
||||
@ -27,9 +27,22 @@
|
||||
#include "ompi/constants.h"
|
||||
#include "ompi/mca/sharedfp/sharedfp.h"
|
||||
|
||||
static int seek_counter=0;
|
||||
|
||||
int mca_sharedfp_individual_seek (ompio_file_t *fh,
|
||||
OMPI_MPI_OFFSET_TYPE offset, int whence)
|
||||
{
|
||||
if ( 0 == seek_counter &&
|
||||
0 == offset &&
|
||||
MPI_SEEK_SET == whence ) {
|
||||
/* This is occuring when setting the default file view. THat is ok.
|
||||
** The component doesn't support however further seek operations.
|
||||
*/
|
||||
|
||||
seek_counter++;
|
||||
return OMPI_SUCCESS;
|
||||
}
|
||||
|
||||
opal_output(0,"mca_sharedfp_individual_seek: NOT IMPLEMENTED\n");
|
||||
return OMPI_ERROR;
|
||||
}
|
||||
|
||||
@ -9,7 +9,7 @@
|
||||
* University of Stuttgart. All rights reserved.
|
||||
* Copyright (c) 2004-2005 The Regents of the University of California.
|
||||
* All rights reserved.
|
||||
* Copyright (c) 2013-2016 University of Houston. All rights reserved.
|
||||
* Copyright (c) 2013-2018 University of Houston. All rights reserved.
|
||||
* Copyright (c) 2015-2018 Research Organization for Information Science
|
||||
* and Technology (RIST). All rights reserved.
|
||||
* $COPYRIGHT$
|
||||
@ -39,23 +39,11 @@ int mca_sharedfp_individual_write (ompio_file_t *fh,
|
||||
size_t totalbytes = 0;
|
||||
mca_sharedfp_individual_header_record *headnode = NULL;
|
||||
struct mca_sharedfp_base_data_t *sh = NULL;
|
||||
mca_sharedfp_base_module_t * shared_fp_base_module = NULL;
|
||||
|
||||
if ( NULL == fh->f_sharedfp_data ) {
|
||||
if ( mca_sharedfp_individual_verbose ) {
|
||||
opal_output(ompi_sharedfp_base_framework.framework_output,"sharedfp_individual_write: opening the shared file pointer file\n");
|
||||
}
|
||||
shared_fp_base_module = fh->f_sharedfp;
|
||||
|
||||
ret = shared_fp_base_module->sharedfp_file_open(fh->f_comm,
|
||||
fh->f_filename,
|
||||
fh->f_amode,
|
||||
fh->f_info,
|
||||
fh);
|
||||
if (ret != OMPI_SUCCESS) {
|
||||
opal_output(0,"sharedfp_individual_write - error opening the shared file pointer\n");
|
||||
return ret;
|
||||
}
|
||||
opal_output(ompi_sharedfp_base_framework.framework_output,
|
||||
"sharedfp_individual_write: module not initialized \n");
|
||||
return OMPI_ERROR;
|
||||
}
|
||||
|
||||
/* Calculate the number of bytes of data that need to be written*/
|
||||
@ -93,7 +81,6 @@ int mca_sharedfp_individual_write_ordered (ompio_file_t *fh,
|
||||
ompi_status_public_t *status)
|
||||
{
|
||||
int ret = OMPI_SUCCESS;
|
||||
int size = 0, rank = 0;
|
||||
int i = 0;
|
||||
size_t numofbytes = 0;
|
||||
size_t totalbytes = 0;
|
||||
@ -103,30 +90,16 @@ int mca_sharedfp_individual_write_ordered (ompio_file_t *fh,
|
||||
OMPI_MPI_OFFSET_TYPE temp = 0, offset = 0;
|
||||
mca_sharedfp_individual_header_record *headnode = NULL;
|
||||
struct mca_sharedfp_base_data_t *sh = NULL;
|
||||
mca_sharedfp_base_module_t * shared_fp_base_module = NULL;
|
||||
|
||||
|
||||
if(fh->f_sharedfp_data==NULL){
|
||||
if ( mca_sharedfp_individual_verbose ) {
|
||||
opal_output(ompi_sharedfp_base_framework.framework_output,
|
||||
"sharedfp_individual_write_ordered - opening the shared file pointer\n");
|
||||
}
|
||||
shared_fp_base_module = fh->f_sharedfp;
|
||||
|
||||
ret = shared_fp_base_module->sharedfp_file_open(fh->f_comm,
|
||||
fh->f_filename,
|
||||
fh->f_amode,
|
||||
fh->f_info,
|
||||
fh);
|
||||
if ( OMPI_SUCCESS != ret ) {
|
||||
opal_output(0,"sharedfp_individual_write_ordered - error opening the shared file pointer\n");
|
||||
return ret;
|
||||
}
|
||||
"sharedfp_individual_write_ordered: module not initialized \n");
|
||||
return OMPI_ERROR;
|
||||
}
|
||||
|
||||
/*Retrieve the sharedfp data structures*/
|
||||
sh = fh->f_sharedfp_data;
|
||||
rank = ompi_comm_rank ( sh->comm );
|
||||
size = ompi_comm_size ( sh->comm );
|
||||
|
||||
/* Calculate the number of bytes of data that needs to be written*/
|
||||
opal_datatype_type_size ( &datatype->super, &numofbytes);
|
||||
@ -139,50 +112,71 @@ int mca_sharedfp_individual_write_ordered (ompio_file_t *fh,
|
||||
}
|
||||
|
||||
/* Data from all the metadata is combined and written to the main file */
|
||||
ret = mca_sharedfp_individual_collaborate_data ( sh );
|
||||
ret = mca_sharedfp_individual_collaborate_data ( sh, fh );
|
||||
if ( OMPI_SUCCESS != ret) {
|
||||
return ret;
|
||||
}
|
||||
|
||||
if ( 0 == rank ) {
|
||||
offbuff = (OMPI_MPI_OFFSET_TYPE *)malloc ( sizeof(OMPI_MPI_OFFSET_TYPE) * size);
|
||||
if ( 0 == fh->f_rank ) {
|
||||
offbuff = (OMPI_MPI_OFFSET_TYPE *)malloc ( sizeof(OMPI_MPI_OFFSET_TYPE) * fh->f_size);
|
||||
if (NULL == offbuff ) {
|
||||
return OMPI_ERR_OUT_OF_RESOURCE;
|
||||
}
|
||||
}
|
||||
|
||||
/*collect the total bytes to be written*/
|
||||
sh->comm->c_coll->coll_gather ( &totalbytes, 1, OMPI_OFFSET_DATATYPE,
|
||||
offbuff, 1, OMPI_OFFSET_DATATYPE, 0,
|
||||
sh->comm, sh->comm->c_coll->coll_gather_module );
|
||||
ret = fh->f_comm->c_coll->coll_gather ( &totalbytes,
|
||||
1,
|
||||
OMPI_OFFSET_DATATYPE,
|
||||
offbuff,
|
||||
1,
|
||||
OMPI_OFFSET_DATATYPE,
|
||||
0,
|
||||
fh->f_comm,
|
||||
fh->f_comm->c_coll->coll_gather_module );
|
||||
|
||||
if ( 0 == rank ) {
|
||||
if ( OMPI_SUCCESS != ret ) {
|
||||
opal_output(0,"sharedfp_individual_write_ordered: Error in gathering offsets \n");
|
||||
goto exit;
|
||||
}
|
||||
|
||||
if ( 0 == fh->f_rank ) {
|
||||
prev_offset = offbuff[0];
|
||||
offbuff[0] = sh->global_offset;
|
||||
|
||||
for (i = 1; i < size ; i++){
|
||||
for (i = 1; i < fh->f_size ; i++){
|
||||
temp = offbuff[i];
|
||||
offbuff[i] = offbuff[i - 1] + prev_offset;
|
||||
prev_offset = temp;
|
||||
}
|
||||
|
||||
for (i = 0; i < size; i++){
|
||||
global_offset = offbuff[size - 1] + prev_offset;
|
||||
for (i = 0; i < fh->f_size; i++){
|
||||
global_offset = offbuff[fh->f_size - 1] + prev_offset;
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
/* Scatter the results to the other processes */
|
||||
ret = sh->comm->c_coll->coll_scatter ( offbuff, 1, OMPI_OFFSET_DATATYPE,
|
||||
&offset, 1, OMPI_OFFSET_DATATYPE, 0,
|
||||
sh->comm, sh->comm->c_coll->coll_scatter_module );
|
||||
ret = fh->f_comm->c_coll->coll_scatter ( offbuff,
|
||||
1,
|
||||
OMPI_OFFSET_DATATYPE,
|
||||
&offset,
|
||||
1,
|
||||
OMPI_OFFSET_DATATYPE,
|
||||
0,
|
||||
fh->f_comm,
|
||||
fh->f_comm->c_coll->coll_scatter_module );
|
||||
if ( OMPI_SUCCESS != ret ) {
|
||||
opal_output(0,"sharedfp_individual_write_ordered: Error in scattering offsets \n");
|
||||
goto exit;
|
||||
}
|
||||
|
||||
ret = sh->comm->c_coll->coll_bcast ( &global_offset, 1, OMPI_OFFSET_DATATYPE,
|
||||
0, sh->comm, sh->comm->c_coll->coll_bcast_module );
|
||||
ret = fh->f_comm->c_coll->coll_bcast ( &global_offset,
|
||||
1,
|
||||
OMPI_OFFSET_DATATYPE,
|
||||
0,
|
||||
fh->f_comm,
|
||||
fh->f_comm->c_coll->coll_bcast_module );
|
||||
if ( OMPI_SUCCESS != ret ) {
|
||||
opal_output(0,"sharedfp_individual_write_ordered: Error while bcasting global offset \n");
|
||||
goto exit;
|
||||
@ -191,7 +185,7 @@ int mca_sharedfp_individual_write_ordered (ompio_file_t *fh,
|
||||
sh->global_offset = global_offset;
|
||||
|
||||
/*use file_write_at_all to ensure the order*/
|
||||
ret = mca_common_ompio_file_write_at_all(sh->sharedfh,offset, buf,count,datatype,status);
|
||||
ret = mca_common_ompio_file_write_at_all(fh, offset, buf,count,datatype,status);
|
||||
if ( OMPI_SUCCESS != ret ) {
|
||||
opal_output(0,"sharedfp_individual_write_ordered: Error while writing the datafile \n");
|
||||
}
|
||||
|
||||
@ -9,7 +9,7 @@
|
||||
* University of Stuttgart. All rights reserved.
|
||||
* Copyright (c) 2004-2005 The Regents of the University of California.
|
||||
* All rights reserved.
|
||||
* Copyright (c) 2013-2017 University of Houston. All rights reserved.
|
||||
* Copyright (c) 2013-2018 University of Houston. All rights reserved.
|
||||
* Copyright (c) 2015-2018 Research Organization for Information Science
|
||||
* and Technology (RIST). All rights reserved.
|
||||
* Copyright (c) 2016-2017 IBM Corporation. All rights reserved.
|
||||
@ -44,48 +44,20 @@ int mca_sharedfp_lockedfile_file_open (struct ompi_communicator_t *comm,
|
||||
{
|
||||
int err = MPI_SUCCESS;
|
||||
char * lockedfilename;
|
||||
int handle, rank;
|
||||
int handle;
|
||||
struct mca_sharedfp_lockedfile_data * module_data = NULL;
|
||||
struct mca_sharedfp_base_data_t* sh;
|
||||
ompio_file_t * shfileHandle, *ompio_fh;
|
||||
mca_common_ompio_data_t *data;
|
||||
|
||||
/*------------------------------------------------------------*/
|
||||
/*Open the same file again without shared file pointer support*/
|
||||
/*------------------------------------------------------------*/
|
||||
shfileHandle = (ompio_file_t *)malloc(sizeof(ompio_file_t));
|
||||
err = mca_common_ompio_file_open(comm,filename,amode,info,shfileHandle,false);
|
||||
if ( OMPI_SUCCESS != err) {
|
||||
opal_output(0, "mca_sharedfp_lockedfile_file_open: Error during file open\n");
|
||||
return err;
|
||||
}
|
||||
shfileHandle->f_fh = fh->f_fh;
|
||||
data = (mca_common_ompio_data_t *) fh->f_fh->f_io_selected_data;
|
||||
ompio_fh = &data->ompio_fh;
|
||||
|
||||
err = mca_common_ompio_set_view (shfileHandle,
|
||||
ompio_fh->f_disp,
|
||||
ompio_fh->f_etype,
|
||||
ompio_fh->f_orig_filetype,
|
||||
ompio_fh->f_datarep,
|
||||
&(MPI_INFO_NULL->super));
|
||||
|
||||
|
||||
/*Memory is allocated here for the sh structure*/
|
||||
sh = (struct mca_sharedfp_base_data_t*)malloc(sizeof(struct mca_sharedfp_base_data_t));
|
||||
if ( NULL == sh){
|
||||
opal_output(0, "mca_sharedfp_lockedfile_file_open: Error, unable to malloc f_sharedfp_ptr struct\n");
|
||||
free ( shfileHandle);
|
||||
opal_output(0, "mca_sharedfp_lockedfile_file_open: Error, unable to malloc f_sharedfp struct\n");
|
||||
return OMPI_ERR_OUT_OF_RESOURCE;
|
||||
}
|
||||
/*Populate the sh file structure based on the implementation*/
|
||||
sh->sharedfh = shfileHandle; /* Shared file pointer*/
|
||||
sh->global_offset = 0; /* Global Offset*/
|
||||
sh->comm = comm; /* Communicator*/
|
||||
sh->selected_module_data = NULL;
|
||||
|
||||
rank = ompi_comm_rank ( sh->comm);
|
||||
|
||||
/*Open a new file which will maintain the pointer for this file open*/
|
||||
if ( mca_sharedfp_lockedfile_verbose ) {
|
||||
opal_output(ompi_sharedfp_base_framework.framework_output,
|
||||
@ -97,7 +69,6 @@ int mca_sharedfp_lockedfile_file_open (struct ompi_communicator_t *comm,
|
||||
if ( NULL == module_data ) {
|
||||
opal_output(ompi_sharedfp_base_framework.framework_output,
|
||||
"mca_sharedfp_lockedfile_file_open: Error, unable to malloc lockedfile_data struct\n");
|
||||
free (shfileHandle);
|
||||
free (sh);
|
||||
return OMPI_ERR_OUT_OF_RESOURCE;
|
||||
}
|
||||
@ -113,7 +84,6 @@ int mca_sharedfp_lockedfile_file_open (struct ompi_communicator_t *comm,
|
||||
size_t filenamelen = strlen(filename) + 16;
|
||||
lockedfilename = (char*)malloc(sizeof(char) * filenamelen);
|
||||
if ( NULL == lockedfilename ) {
|
||||
free (shfileHandle);
|
||||
free (sh);
|
||||
free (module_data);
|
||||
return OMPI_ERR_OUT_OF_RESOURCE;
|
||||
@ -124,7 +94,7 @@ int mca_sharedfp_lockedfile_file_open (struct ompi_communicator_t *comm,
|
||||
/*-------------------------------------------------*/
|
||||
/*Open the lockedfile without shared file pointer */
|
||||
/*-------------------------------------------------*/
|
||||
if ( 0 == rank ) {
|
||||
if ( 0 == comm->c_my_rank ) {
|
||||
OMPI_MPI_OFFSET_TYPE position=0;
|
||||
/*only let main process initialize file pointer,
|
||||
*therefore there is no need to lock the file
|
||||
@ -137,8 +107,7 @@ int mca_sharedfp_lockedfile_file_open (struct ompi_communicator_t *comm,
|
||||
|
||||
handle = open ( lockedfilename, O_RDWR, 0644 );
|
||||
if ( -1 == handle ) {
|
||||
opal_output(0, "[%d]mca_sharedfp_lockedfile_file_open: Error during file open\n", rank);
|
||||
free (shfileHandle);
|
||||
opal_output(0, "[%d]mca_sharedfp_lockedfile_file_open: Error during file open\n", fh->f_rank);
|
||||
free (sh);
|
||||
free(module_data);
|
||||
return OMPI_ERROR;
|
||||
@ -161,13 +130,8 @@ int mca_sharedfp_lockedfile_file_close (ompio_file_t *fh)
|
||||
int err = OMPI_SUCCESS;
|
||||
struct mca_sharedfp_lockedfile_data * module_data = NULL;
|
||||
struct mca_sharedfp_base_data_t *sh;
|
||||
int rank = ompi_comm_rank ( fh->f_comm );
|
||||
|
||||
if ( fh->f_sharedfp_data==NULL){
|
||||
/* Can happen with lazy_open being set */
|
||||
if ( mca_sharedfp_lockedfile_verbose ) {
|
||||
opal_output(0, "sharedfp_lockedfile_file_close - shared file pointer structure not initialized\n");
|
||||
}
|
||||
return OMPI_SUCCESS;
|
||||
}
|
||||
sh = fh->f_sharedfp_data;
|
||||
@ -177,7 +141,7 @@ int mca_sharedfp_lockedfile_file_close (ompio_file_t *fh)
|
||||
/*Close lockedfile handle*/
|
||||
if ( module_data->handle) {
|
||||
close (module_data->handle );
|
||||
if ( 0 == rank ) {
|
||||
if ( 0 == fh->f_rank ) {
|
||||
unlink ( module_data->filename);
|
||||
}
|
||||
}
|
||||
@ -187,9 +151,6 @@ int mca_sharedfp_lockedfile_file_close (ompio_file_t *fh)
|
||||
free ( module_data );
|
||||
}
|
||||
|
||||
/* Close the main file opened by this component*/
|
||||
err = mca_common_ompio_file_close(sh->sharedfh);
|
||||
|
||||
/*free shared file pointer data struct*/
|
||||
free(sh);
|
||||
|
||||
|
||||
@ -9,7 +9,7 @@
|
||||
* University of Stuttgart. All rights reserved.
|
||||
* Copyright (c) 2004-2005 The Regents of the University of California.
|
||||
* All rights reserved.
|
||||
* Copyright (c) 2013-2016 University of Houston. All rights reserved.
|
||||
* Copyright (c) 2013-2018 University of Houston. All rights reserved.
|
||||
* Copyright (c) 2018 Research Organization for Information Science
|
||||
* and Technology (RIST). All rights reserved.
|
||||
* $COPYRIGHT$
|
||||
@ -36,28 +36,15 @@ int mca_sharedfp_lockedfile_iread(ompio_file_t *fh,
|
||||
MPI_Request * request)
|
||||
{
|
||||
int ret = OMPI_SUCCESS;
|
||||
mca_sharedfp_base_module_t * shared_fp_base_module;
|
||||
OMPI_MPI_OFFSET_TYPE offset = 0;
|
||||
long bytesRequested = 0;
|
||||
size_t numofBytes;
|
||||
struct mca_sharedfp_base_data_t *sh = NULL;
|
||||
|
||||
if ( NULL == fh->f_sharedfp_data ) {
|
||||
if ( mca_sharedfp_lockedfile_verbose ) {
|
||||
opal_output(ompi_sharedfp_base_framework.framework_output,
|
||||
"sharedfp_lockedfile_iread: opening the shared file pointer\n");
|
||||
}
|
||||
shared_fp_base_module = fh->f_sharedfp;
|
||||
|
||||
ret = shared_fp_base_module->sharedfp_file_open(fh->f_comm,
|
||||
fh->f_filename,
|
||||
fh->f_amode,
|
||||
fh->f_info,
|
||||
fh);
|
||||
if ( OMPI_SUCCESS != ret ) {
|
||||
opal_output(0,"sharedfp_lockedfile_iread - error opening the shared file pointer\n");
|
||||
return ret;
|
||||
}
|
||||
"sharedfp_lockedfile_iread: module not initialized\n");
|
||||
return OMPI_ERROR;
|
||||
}
|
||||
|
||||
/* Calculate the number of bytes to read */
|
||||
@ -75,7 +62,7 @@ int mca_sharedfp_lockedfile_iread(ompio_file_t *fh,
|
||||
|
||||
/*Request the offset to write bytesRequested bytes*/
|
||||
ret = mca_sharedfp_lockedfile_request_position(sh,bytesRequested,&offset);
|
||||
offset /= sh->sharedfh->f_etype_size;
|
||||
offset /= fh->f_etype_size;
|
||||
|
||||
if ( -1 != ret ) {
|
||||
if ( mca_sharedfp_lockedfile_verbose ) {
|
||||
@ -84,7 +71,7 @@ int mca_sharedfp_lockedfile_iread(ompio_file_t *fh,
|
||||
}
|
||||
|
||||
/* Read the file */
|
||||
ret = mca_common_ompio_file_iread_at(sh->sharedfh,offset,buf,count,datatype,request);
|
||||
ret = mca_common_ompio_file_iread_at(fh,offset,buf,count,datatype,request);
|
||||
}
|
||||
|
||||
return ret;
|
||||
@ -96,7 +83,6 @@ int mca_sharedfp_lockedfile_read_ordered_begin(ompio_file_t *fh,
|
||||
struct ompi_datatype_t *datatype)
|
||||
{
|
||||
int ret = OMPI_SUCCESS;
|
||||
mca_sharedfp_base_module_t * shared_fp_base_module=NULL;
|
||||
OMPI_MPI_OFFSET_TYPE offset = 0;
|
||||
long sendBuff = 0;
|
||||
long *buff=NULL;
|
||||
@ -109,27 +95,16 @@ int mca_sharedfp_lockedfile_read_ordered_begin(ompio_file_t *fh,
|
||||
struct mca_sharedfp_base_data_t *sh = NULL;
|
||||
|
||||
if(fh->f_sharedfp_data==NULL){
|
||||
if ( mca_sharedfp_lockedfile_verbose ) {
|
||||
opal_output(ompi_sharedfp_base_framework.framework_output,
|
||||
"sharedfp_lockedfile_read_ordered_begin: opening the shared file pointer\n");
|
||||
}
|
||||
shared_fp_base_module = fh->f_sharedfp;
|
||||
|
||||
ret = shared_fp_base_module->sharedfp_file_open(fh->f_comm,
|
||||
fh->f_filename,
|
||||
fh->f_amode,
|
||||
fh->f_info,
|
||||
fh);
|
||||
if ( OMPI_SUCCESS != ret ) {
|
||||
opal_output(0,"sharedfp_lockedfile_read_ordered_begin - error opening the shared file pointer\n");
|
||||
return ret;
|
||||
}
|
||||
"sharedfp_lockedfile_read_ordered_begin: module not initialized\n");
|
||||
return OMPI_ERROR;
|
||||
}
|
||||
|
||||
|
||||
if ( true == fh->f_split_coll_in_use ) {
|
||||
opal_output(ompi_sharedfp_base_framework.framework_output,
|
||||
"Only one split collective I/O operation allowed per file handle at any given point in time!\n");
|
||||
"Only one split collective I/O operation allowed per file handle at any "
|
||||
"given point in time!\n");
|
||||
return MPI_ERR_REQUEST;
|
||||
}
|
||||
|
||||
@ -141,8 +116,8 @@ int mca_sharedfp_lockedfile_read_ordered_begin(ompio_file_t *fh,
|
||||
sendBuff = count * numofBytes;
|
||||
|
||||
/* Get the ranks in the communicator */
|
||||
rank = ompi_comm_rank ( sh->comm );
|
||||
size = ompi_comm_size ( sh->comm );
|
||||
rank = ompi_comm_rank ( fh->f_comm );
|
||||
size = ompi_comm_size ( fh->f_comm );
|
||||
|
||||
if ( 0 == rank ) {
|
||||
buff = (long*) malloc (sizeof(long) * size);
|
||||
@ -151,9 +126,9 @@ int mca_sharedfp_lockedfile_read_ordered_begin(ompio_file_t *fh,
|
||||
}
|
||||
}
|
||||
|
||||
ret = sh->comm->c_coll->coll_gather ( &sendBuff, sendcnt, OMPI_OFFSET_DATATYPE, buff, recvcnt,
|
||||
OMPI_OFFSET_DATATYPE, 0, sh->comm,
|
||||
sh->comm->c_coll->coll_gather_module );
|
||||
ret = fh->f_comm->c_coll->coll_gather ( &sendBuff, sendcnt, OMPI_OFFSET_DATATYPE, buff, recvcnt,
|
||||
OMPI_OFFSET_DATATYPE, 0, fh->f_comm,
|
||||
fh->f_comm->c_coll->coll_gather_module );
|
||||
if ( OMPI_SUCCESS != ret ) {
|
||||
goto exit;
|
||||
}
|
||||
@ -191,23 +166,23 @@ int mca_sharedfp_lockedfile_read_ordered_begin(ompio_file_t *fh,
|
||||
}
|
||||
|
||||
/* Scatter the results to the other processes*/
|
||||
ret = sh->comm->c_coll->coll_scatter ( buff, sendcnt, OMPI_OFFSET_DATATYPE,
|
||||
ret = fh->f_comm->c_coll->coll_scatter ( buff, sendcnt, OMPI_OFFSET_DATATYPE,
|
||||
&offsetBuff, recvcnt, OMPI_OFFSET_DATATYPE, 0,
|
||||
sh->comm, sh->comm->c_coll->coll_scatter_module );
|
||||
fh->f_comm, fh->f_comm->c_coll->coll_scatter_module );
|
||||
if ( OMPI_SUCCESS != ret ) {
|
||||
goto exit;
|
||||
}
|
||||
|
||||
/*Each process now has its own individual offset*/
|
||||
offset = offsetBuff - sendBuff;
|
||||
offset /= sh->sharedfh->f_etype_size;
|
||||
offset /= fh->f_etype_size;
|
||||
|
||||
if ( mca_sharedfp_lockedfile_verbose ) {
|
||||
opal_output(ompi_sharedfp_base_framework.framework_output,
|
||||
"sharedfp_lockedfile_read_ordered_begin: Offset returned is %lld\n",offset);
|
||||
}
|
||||
|
||||
ret = mca_common_ompio_file_iread_at_all ( sh->sharedfh, offset, buf, count, datatype, &fh->f_split_coll_req );
|
||||
ret = mca_common_ompio_file_iread_at_all ( fh, offset, buf, count, datatype, &fh->f_split_coll_req );
|
||||
fh->f_split_coll_in_use = true;
|
||||
|
||||
exit:
|
||||
|
||||
@ -9,7 +9,7 @@
|
||||
* University of Stuttgart. All rights reserved.
|
||||
* Copyright (c) 2004-2005 The Regents of the University of California.
|
||||
* All rights reserved.
|
||||
* Copyright (c) 2013-2016 University of Houston. All rights reserved.
|
||||
* Copyright (c) 2013-2018 University of Houston. All rights reserved.
|
||||
* Copyright (c) 2015-2018 Research Organization for Information Science
|
||||
* and Technology (RIST). All rights reserved.
|
||||
* $COPYRIGHT$
|
||||
@ -36,28 +36,15 @@ int mca_sharedfp_lockedfile_iwrite(ompio_file_t *fh,
|
||||
MPI_Request * request)
|
||||
{
|
||||
int ret = OMPI_SUCCESS;
|
||||
mca_sharedfp_base_module_t * shared_fp_base_module;
|
||||
OMPI_MPI_OFFSET_TYPE offset = 0;
|
||||
long bytesRequested = 0;
|
||||
size_t numofBytes;
|
||||
struct mca_sharedfp_base_data_t *sh = NULL;
|
||||
|
||||
if(fh->f_sharedfp_data==NULL){
|
||||
if ( mca_sharedfp_lockedfile_verbose ) {
|
||||
opal_output(ompi_sharedfp_base_framework.framework_output,
|
||||
"sharedfp_lockedfile_iwrite: opening the shared file pointer\n");
|
||||
}
|
||||
shared_fp_base_module = fh->f_sharedfp;
|
||||
|
||||
ret = shared_fp_base_module->sharedfp_file_open(fh->f_comm,
|
||||
fh->f_filename,
|
||||
fh->f_amode,
|
||||
fh->f_info,
|
||||
fh);
|
||||
if ( OMPI_SUCCESS != ret ) {
|
||||
opal_output(0,"sharedfp_lockedfile_iwrite - error opening the shared file pointer\n");
|
||||
return ret;
|
||||
}
|
||||
"sharedfp_lockedfile_iwrite: module not initialized \n");
|
||||
return OMPI_ERROR;
|
||||
}
|
||||
|
||||
/*Calculate the number of bytes to write*/
|
||||
@ -73,7 +60,7 @@ int mca_sharedfp_lockedfile_iwrite(ompio_file_t *fh,
|
||||
|
||||
/*Request the offset to write bytesRequested bytes*/
|
||||
ret = mca_sharedfp_lockedfile_request_position(sh,bytesRequested,&offset);
|
||||
offset /= sh->sharedfh->f_etype_size;
|
||||
offset /= fh->f_etype_size;
|
||||
|
||||
if ( -1 != ret) {
|
||||
if ( mca_sharedfp_lockedfile_verbose ) {
|
||||
@ -82,7 +69,7 @@ int mca_sharedfp_lockedfile_iwrite(ompio_file_t *fh,
|
||||
}
|
||||
|
||||
/* Write to the file */
|
||||
ret = mca_common_ompio_file_iwrite_at(sh->sharedfh,offset,buf,count,datatype,request);
|
||||
ret = mca_common_ompio_file_iwrite_at(fh,offset,buf,count,datatype,request);
|
||||
}
|
||||
|
||||
return ret;
|
||||
@ -94,7 +81,6 @@ int mca_sharedfp_lockedfile_write_ordered_begin(ompio_file_t *fh,
|
||||
struct ompi_datatype_t *datatype)
|
||||
{
|
||||
int ret = OMPI_SUCCESS;
|
||||
mca_sharedfp_base_module_t * shared_fp_base_module=NULL;
|
||||
OMPI_MPI_OFFSET_TYPE offset = 0;
|
||||
long sendBuff = 0;
|
||||
long *buff=NULL;
|
||||
@ -107,26 +93,15 @@ int mca_sharedfp_lockedfile_write_ordered_begin(ompio_file_t *fh,
|
||||
struct mca_sharedfp_base_data_t *sh = NULL;
|
||||
|
||||
if(fh->f_sharedfp_data==NULL){
|
||||
if ( mca_sharedfp_lockedfile_verbose ) {
|
||||
opal_output(ompi_sharedfp_base_framework.framework_output,
|
||||
"sharedfp_lockedfile_write_ordered_begin: opening the shared file pointer\n");
|
||||
}
|
||||
shared_fp_base_module = fh->f_sharedfp;
|
||||
|
||||
ret = shared_fp_base_module->sharedfp_file_open(fh->f_comm,
|
||||
fh->f_filename,
|
||||
fh->f_amode,
|
||||
fh->f_info,
|
||||
fh);
|
||||
if ( OMPI_SUCCESS != ret ) {
|
||||
opal_output(0,"sharedfp_lockedfile_write_ordered_begin - error opening the shared file pointer\n");
|
||||
return ret;
|
||||
}
|
||||
"sharedfp_lockedfile_write_ordered_begin: module not initialized \n");
|
||||
return OMPI_ERROR;
|
||||
}
|
||||
|
||||
|
||||
if ( true == fh->f_split_coll_in_use ) {
|
||||
opal_output(0, "Only one split collective I/O operation allowed per file handle at any given point in time!\n");
|
||||
opal_output(0, "Only one split collective I/O operation allowed per file handle at "
|
||||
"any given point in time!\n");
|
||||
return MPI_ERR_REQUEST;
|
||||
}
|
||||
|
||||
@ -138,8 +113,8 @@ int mca_sharedfp_lockedfile_write_ordered_begin(ompio_file_t *fh,
|
||||
sendBuff = count * numofBytes;
|
||||
|
||||
/* Get the ranks in the communicator */
|
||||
rank = ompi_comm_rank ( sh->comm );
|
||||
size = ompi_comm_size ( sh->comm );
|
||||
rank = ompi_comm_rank ( fh->f_comm );
|
||||
size = ompi_comm_size ( fh->f_comm );
|
||||
|
||||
if ( 0 == rank ) {
|
||||
buff = (long*) malloc (sizeof(long) * size);
|
||||
@ -148,9 +123,15 @@ int mca_sharedfp_lockedfile_write_ordered_begin(ompio_file_t *fh,
|
||||
}
|
||||
}
|
||||
|
||||
ret = sh->comm->c_coll->coll_gather ( &sendBuff, sendcnt, OMPI_OFFSET_DATATYPE, buff, recvcnt,
|
||||
OMPI_OFFSET_DATATYPE, 0, sh->comm,
|
||||
sh->comm->c_coll->coll_gather_module );
|
||||
ret = fh->f_comm->c_coll->coll_gather ( &sendBuff,
|
||||
sendcnt,
|
||||
OMPI_OFFSET_DATATYPE,
|
||||
buff,
|
||||
recvcnt,
|
||||
OMPI_OFFSET_DATATYPE,
|
||||
0,
|
||||
fh->f_comm,
|
||||
fh->f_comm->c_coll->coll_gather_module );
|
||||
if ( OMPI_SUCCESS != ret ) {
|
||||
goto exit;
|
||||
}
|
||||
@ -188,23 +169,29 @@ int mca_sharedfp_lockedfile_write_ordered_begin(ompio_file_t *fh,
|
||||
}
|
||||
|
||||
/* Scatter the results to the other processes*/
|
||||
ret = sh->comm->c_coll->coll_scatter ( buff, sendcnt, OMPI_OFFSET_DATATYPE,
|
||||
&offsetBuff, recvcnt, OMPI_OFFSET_DATATYPE, 0,
|
||||
sh->comm, sh->comm->c_coll->coll_scatter_module );
|
||||
ret = fh->f_comm->c_coll->coll_scatter ( buff,
|
||||
sendcnt,
|
||||
OMPI_OFFSET_DATATYPE,
|
||||
&offsetBuff,
|
||||
recvcnt,
|
||||
OMPI_OFFSET_DATATYPE,
|
||||
0,
|
||||
fh->f_comm,
|
||||
fh->f_comm->c_coll->coll_scatter_module );
|
||||
if ( OMPI_SUCCESS != ret ) {
|
||||
goto exit;
|
||||
}
|
||||
|
||||
/*Each process now has its own individual offset*/
|
||||
offset = offsetBuff - sendBuff;
|
||||
offset /= sh->sharedfh->f_etype_size;
|
||||
offset /= fh->f_etype_size;
|
||||
|
||||
if ( mca_sharedfp_lockedfile_verbose ) {
|
||||
opal_output(ompi_sharedfp_base_framework.framework_output,
|
||||
"sharedfp_lockedfile_write_ordered_begin: Offset returned is %lld\n",offset);
|
||||
}
|
||||
|
||||
ret = mca_common_ompio_file_iwrite_at_all ( sh->sharedfh, offset, buf, count, datatype, &fh->f_split_coll_req );
|
||||
ret = mca_common_ompio_file_iwrite_at_all ( fh, offset, buf, count, datatype, &fh->f_split_coll_req );
|
||||
fh->f_split_coll_in_use = true;
|
||||
|
||||
exit:
|
||||
|
||||
@ -9,7 +9,7 @@
|
||||
* University of Stuttgart. All rights reserved.
|
||||
* Copyright (c) 2004-2005 The Regents of the University of California.
|
||||
* All rights reserved.
|
||||
* Copyright (c) 2013-2016 University of Houston. All rights reserved.
|
||||
* Copyright (c) 2013-2018 University of Houston. All rights reserved.
|
||||
* Copyright (c) 2018 Research Organization for Information Science
|
||||
* and Technology (RIST). All rights reserved.
|
||||
* $COPYRIGHT$
|
||||
@ -32,7 +32,6 @@ int mca_sharedfp_lockedfile_read ( ompio_file_t *fh,
|
||||
void *buf, int count, MPI_Datatype datatype, MPI_Status *status)
|
||||
{
|
||||
int ret = OMPI_SUCCESS;
|
||||
mca_sharedfp_base_module_t * shared_fp_base_module;
|
||||
OMPI_MPI_OFFSET_TYPE offset = 0;
|
||||
long bytesRequested = 0;
|
||||
size_t numofBytes;
|
||||
@ -41,19 +40,9 @@ int mca_sharedfp_lockedfile_read ( ompio_file_t *fh,
|
||||
if ( fh->f_sharedfp_data == NULL ) {
|
||||
if ( mca_sharedfp_lockedfile_verbose ) {
|
||||
opal_output(ompi_sharedfp_base_framework.framework_output,
|
||||
"sharedfp_lockedfile_read: opening the shared file pointer\n");
|
||||
}
|
||||
shared_fp_base_module = fh->f_sharedfp;
|
||||
|
||||
ret = shared_fp_base_module->sharedfp_file_open(fh->f_comm,
|
||||
fh->f_filename,
|
||||
fh->f_amode,
|
||||
fh->f_info,
|
||||
fh);
|
||||
if ( OMPI_SUCCESS != ret ) {
|
||||
opal_output(0,"sharedfp_lockedfile_read - error opening the shared file pointer\n");
|
||||
return ret;
|
||||
"sharedfp_lockedfile_read: module not initialized\n");
|
||||
}
|
||||
return OMPI_ERROR;
|
||||
}
|
||||
|
||||
/* Calculate the number of bytes to read */
|
||||
@ -70,7 +59,7 @@ int mca_sharedfp_lockedfile_read ( ompio_file_t *fh,
|
||||
|
||||
/*Request the offset to write bytesRequested bytes*/
|
||||
ret = mca_sharedfp_lockedfile_request_position(sh,bytesRequested,&offset);
|
||||
offset /= sh->sharedfh->f_etype_size;
|
||||
offset /= fh->f_etype_size;
|
||||
|
||||
if (-1 != ret ) {
|
||||
if ( mca_sharedfp_lockedfile_verbose ) {
|
||||
@ -79,7 +68,7 @@ int mca_sharedfp_lockedfile_read ( ompio_file_t *fh,
|
||||
}
|
||||
|
||||
/* Read the file */
|
||||
ret = mca_common_ompio_file_read_at(sh->sharedfh,offset,buf,count,datatype,status);
|
||||
ret = mca_common_ompio_file_read_at(fh,offset,buf,count,datatype,status);
|
||||
}
|
||||
|
||||
return ret;
|
||||
@ -92,7 +81,6 @@ int mca_sharedfp_lockedfile_read_ordered (ompio_file_t *fh,
|
||||
ompi_status_public_t *status)
|
||||
{
|
||||
int ret = OMPI_SUCCESS;
|
||||
mca_sharedfp_base_module_t * shared_fp_base_module=NULL;
|
||||
OMPI_MPI_OFFSET_TYPE offset = 0;
|
||||
long sendBuff = 0;
|
||||
long *buff=NULL;
|
||||
@ -105,21 +93,9 @@ int mca_sharedfp_lockedfile_read_ordered (ompio_file_t *fh,
|
||||
struct mca_sharedfp_base_data_t *sh = NULL;
|
||||
|
||||
if ( fh->f_sharedfp_data == NULL){
|
||||
if ( mca_sharedfp_lockedfile_verbose ) {
|
||||
opal_output(ompi_sharedfp_base_framework.framework_output,
|
||||
"sharedfp_lockedfile_read_ordered: opening the shared file pointer\n");
|
||||
}
|
||||
shared_fp_base_module = fh->f_sharedfp;
|
||||
|
||||
ret = shared_fp_base_module->sharedfp_file_open(fh->f_comm,
|
||||
fh->f_filename,
|
||||
fh->f_amode,
|
||||
fh->f_info,
|
||||
fh);
|
||||
if ( OMPI_SUCCESS != ret ) {
|
||||
opal_output(0,"sharedfp_lockedfile_read_ordered - error opening the shared file pointer\n");
|
||||
return ret;
|
||||
}
|
||||
"sharedfp_lockedfile_read_ordered: module not initialized\n");
|
||||
return OMPI_ERROR;
|
||||
}
|
||||
|
||||
/*Retrieve the new communicator*/
|
||||
@ -130,8 +106,8 @@ int mca_sharedfp_lockedfile_read_ordered (ompio_file_t *fh,
|
||||
sendBuff = count * numofBytes;
|
||||
|
||||
/* Get the ranks in the communicator */
|
||||
rank = ompi_comm_rank ( sh->comm );
|
||||
size = ompi_comm_size ( sh->comm );
|
||||
rank = ompi_comm_rank ( fh->f_comm );
|
||||
size = ompi_comm_size ( fh->f_comm );
|
||||
|
||||
if ( 0 == rank ) {
|
||||
buff = (long*)malloc(sizeof(long) * size);
|
||||
@ -139,9 +115,9 @@ int mca_sharedfp_lockedfile_read_ordered (ompio_file_t *fh,
|
||||
return OMPI_ERR_OUT_OF_RESOURCE;
|
||||
}
|
||||
|
||||
ret = sh->comm->c_coll->coll_gather ( &sendBuff, sendcnt, OMPI_OFFSET_DATATYPE,
|
||||
ret = fh->f_comm->c_coll->coll_gather ( &sendBuff, sendcnt, OMPI_OFFSET_DATATYPE,
|
||||
buff, recvcnt, OMPI_OFFSET_DATATYPE, 0,
|
||||
sh->comm, sh->comm->c_coll->coll_gather_module );
|
||||
fh->f_comm, fh->f_comm->c_coll->coll_gather_module );
|
||||
if ( OMPI_SUCCESS != ret ) {
|
||||
goto exit;
|
||||
}
|
||||
@ -180,13 +156,13 @@ int mca_sharedfp_lockedfile_read_ordered (ompio_file_t *fh,
|
||||
}
|
||||
|
||||
/* Scatter the results to the other processes*/
|
||||
ret = sh->comm->c_coll->coll_scatter ( buff, sendcnt, OMPI_OFFSET_DATATYPE,
|
||||
ret = fh->f_comm->c_coll->coll_scatter ( buff, sendcnt, OMPI_OFFSET_DATATYPE,
|
||||
&offsetBuff, recvcnt, OMPI_OFFSET_DATATYPE, 0,
|
||||
sh->comm, sh->comm->c_coll->coll_scatter_module );
|
||||
fh->f_comm, fh->f_comm->c_coll->coll_scatter_module );
|
||||
|
||||
/*Each process now has its own individual offset in recvBUFF*/
|
||||
offset = offsetBuff - sendBuff;
|
||||
offset /= sh->sharedfh->f_etype_size;
|
||||
offset /= fh->f_etype_size;
|
||||
|
||||
if ( mca_sharedfp_lockedfile_verbose ) {
|
||||
opal_output(ompi_sharedfp_base_framework.framework_output,
|
||||
@ -194,7 +170,7 @@ int mca_sharedfp_lockedfile_read_ordered (ompio_file_t *fh,
|
||||
}
|
||||
|
||||
/* read to the file */
|
||||
ret = mca_common_ompio_file_read_at_all(sh->sharedfh,offset,buf,count,datatype,status);
|
||||
ret = mca_common_ompio_file_read_at_all(fh,offset,buf,count,datatype,status);
|
||||
|
||||
exit:
|
||||
if ( NULL != buff ) {
|
||||
|
||||
@ -9,7 +9,7 @@
|
||||
* University of Stuttgart. All rights reserved.
|
||||
* Copyright (c) 2004-2005 The Regents of the University of California.
|
||||
* All rights reserved.
|
||||
* Copyright (c) 2013-2016 University of Houston. All rights reserved.
|
||||
* Copyright (c) 2013-2018 University of Houston. All rights reserved.
|
||||
* Copyright (c) 2018 Research Organization for Information Science
|
||||
* and Technology (RIST). All rights reserved.
|
||||
* $COPYRIGHT$
|
||||
@ -35,10 +35,8 @@ int
|
||||
mca_sharedfp_lockedfile_seek (ompio_file_t *fh,
|
||||
OMPI_MPI_OFFSET_TYPE offset, int whence)
|
||||
{
|
||||
int rank;
|
||||
int ret = OMPI_SUCCESS;
|
||||
struct mca_sharedfp_base_data_t *sh = NULL;
|
||||
mca_sharedfp_base_module_t * shared_fp_base_module;
|
||||
struct mca_sharedfp_lockedfile_data * lockedfile_data;
|
||||
int fd_lockedfilehandle;
|
||||
/* flock structure that is used to setup the desired fcntl operation */
|
||||
@ -46,62 +44,47 @@ mca_sharedfp_lockedfile_seek (ompio_file_t *fh,
|
||||
|
||||
if(fh->f_sharedfp_data==NULL){
|
||||
opal_output(ompi_sharedfp_base_framework.framework_output,
|
||||
"sharedfp_lockedfile_seek - opening the shared file pointer\n");
|
||||
shared_fp_base_module = fh->f_sharedfp;
|
||||
|
||||
ret = shared_fp_base_module->sharedfp_file_open(fh->f_comm,
|
||||
fh->f_filename,
|
||||
fh->f_amode,
|
||||
fh->f_info,
|
||||
fh);
|
||||
if (ret != OMPI_SUCCESS) {
|
||||
opal_output(0,"sharedfp_lockedfile_seek - error opening the shared file pointer\n");
|
||||
return ret;
|
||||
}
|
||||
"sharedfp_lockedfile_seek: module not initialized\n");
|
||||
return OMPI_ERROR;
|
||||
}
|
||||
|
||||
sh = fh->f_sharedfp_data;
|
||||
rank = ompi_comm_rank ( sh->comm );
|
||||
|
||||
if( 0 == rank ){
|
||||
if( 0 == fh->f_rank ){
|
||||
if ( MPI_SEEK_SET == whence ){
|
||||
/*don't need to read current value*/
|
||||
if(offset < 0){
|
||||
opal_output(0,"sharedfp_lockedfile_seek - MPI_SEEK_SET, offset must be > 0, got offset=%lld.\n",offset);
|
||||
ret = -1;
|
||||
opal_output(0,"sharedfp_lockedfile_seek - MPI_SEEK_SET, offset must be > 0,"
|
||||
" got offset=%lld.\n",offset);
|
||||
return OMPI_ERROR;
|
||||
}
|
||||
opal_output(ompi_sharedfp_base_framework.framework_output,"MPI_SEEK_SET: new_offset=%lld\n",offset);
|
||||
fflush(stdout);
|
||||
}
|
||||
else if ( MPI_SEEK_CUR == whence){
|
||||
OMPI_MPI_OFFSET_TYPE current_position;
|
||||
int status = mca_sharedfp_lockedfile_get_position(fh,¤t_position);
|
||||
opal_output(ompi_sharedfp_base_framework.framework_output,
|
||||
"MPI_SEEK_CUR: curr=%lld, offset=%lld, call status=%d\n",current_position,offset,status);
|
||||
ret = mca_sharedfp_lockedfile_get_position(fh,¤t_position);
|
||||
if ( OMPI_SUCCESS != ret ) {
|
||||
return OMPI_ERROR;
|
||||
}
|
||||
|
||||
offset = current_position + offset;
|
||||
opal_output(ompi_sharedfp_base_framework.framework_output,
|
||||
"MPI_SEEK_CUR: new_offset=%lld\n",offset);
|
||||
fflush(stdout);
|
||||
if(offset < 0){
|
||||
opal_output(0,"sharedfp_lockedfile_seek - MPI_SEEK_CURE, offset must be > 0, got offset=%lld.\n",offset);
|
||||
ret = -1;
|
||||
opal_output(0,"sharedfp_lockedfile_seek - MPI_SEEK_CUR, offset must be > 0, got offset=%lld.\n",offset);
|
||||
return OMPI_ERROR;
|
||||
}
|
||||
}
|
||||
else if( MPI_SEEK_END == whence ){
|
||||
OMPI_MPI_OFFSET_TYPE end_position=0;
|
||||
mca_common_ompio_file_get_size(sh->sharedfh,&end_position);
|
||||
mca_common_ompio_file_get_size( fh,&end_position);
|
||||
offset = end_position + offset;
|
||||
opal_output(ompi_sharedfp_base_framework.framework_output,
|
||||
"MPI_SEEK_END: file_get_size=%lld\n",end_position);
|
||||
|
||||
if ( offset < 0){
|
||||
opal_output(0,"sharedfp_lockedfile_seek - MPI_SEEK_CUR, offset must be > 0, got offset=%lld.\n",offset);
|
||||
ret = -1;
|
||||
return OMPI_ERROR;
|
||||
}
|
||||
}else{
|
||||
opal_output(0,"sharedfp_lockedfile_seek - whence=%i is not supported\n",whence);
|
||||
ret = -1;
|
||||
return OMPI_ERROR;
|
||||
}
|
||||
|
||||
|
||||
@ -162,6 +145,6 @@ mca_sharedfp_lockedfile_seek (ompio_file_t *fh,
|
||||
}
|
||||
}
|
||||
|
||||
sh->comm->c_coll->coll_barrier ( sh->comm , sh->comm->c_coll->coll_barrier_module );
|
||||
return ret;
|
||||
fh->f_comm->c_coll->coll_barrier ( fh->f_comm , fh->f_comm->c_coll->coll_barrier_module );
|
||||
return OMPI_SUCCESS;
|
||||
}
|
||||
|
||||
@ -9,7 +9,7 @@
|
||||
* University of Stuttgart. All rights reserved.
|
||||
* Copyright (c) 2004-2005 The Regents of the University of California.
|
||||
* All rights reserved.
|
||||
* Copyright (c) 2013-2016 University of Houston. All rights reserved.
|
||||
* Copyright (c) 2013-2018 University of Houston. All rights reserved.
|
||||
* Copyright (c) 2015-2018 Research Organization for Information Science
|
||||
* and Technology (RIST). All rights reserved.
|
||||
* $COPYRIGHT$
|
||||
@ -34,7 +34,6 @@ int mca_sharedfp_lockedfile_write (ompio_file_t *fh,
|
||||
struct ompi_datatype_t *datatype,
|
||||
ompi_status_public_t *status)
|
||||
{
|
||||
mca_sharedfp_base_module_t * shared_fp_base_module;
|
||||
OMPI_MPI_OFFSET_TYPE offset = 0;
|
||||
long bytesRequested = 0;
|
||||
size_t numofBytes;
|
||||
@ -42,21 +41,9 @@ int mca_sharedfp_lockedfile_write (ompio_file_t *fh,
|
||||
int ret = OMPI_SUCCESS;
|
||||
|
||||
if ( NULL == fh->f_sharedfp_data ){
|
||||
if ( mca_sharedfp_lockedfile_verbose ) {
|
||||
opal_output(ompi_sharedfp_base_framework.framework_output,
|
||||
"sharedfp_lockedfile_write - opening the shared file pointer\n");
|
||||
}
|
||||
shared_fp_base_module = fh->f_sharedfp;
|
||||
|
||||
ret = shared_fp_base_module->sharedfp_file_open(fh->f_comm,
|
||||
fh->f_filename,
|
||||
fh->f_amode,
|
||||
fh->f_info,
|
||||
fh);
|
||||
if (ret != OMPI_SUCCESS) {
|
||||
opal_output(0,"sharedfp_lockedfile_write - error opening the shared file pointer\n");
|
||||
return ret;
|
||||
}
|
||||
"sharedfp_lockedfile_write - framework not initialized\n");
|
||||
return OMPI_ERROR;
|
||||
}
|
||||
|
||||
/*Calculate the number of bytes to write*/
|
||||
@ -72,7 +59,7 @@ int mca_sharedfp_lockedfile_write (ompio_file_t *fh,
|
||||
|
||||
/* Request the offset to write bytesRequested bytes */
|
||||
ret = mca_sharedfp_lockedfile_request_position ( sh, bytesRequested, &offset);
|
||||
offset /= sh->sharedfh->f_etype_size;
|
||||
offset /= fh->f_etype_size;
|
||||
|
||||
if (-1 != ret ) {
|
||||
if ( mca_sharedfp_lockedfile_verbose ) {
|
||||
@ -80,7 +67,7 @@ int mca_sharedfp_lockedfile_write (ompio_file_t *fh,
|
||||
"sharedfp_lockedfile_write: Offset received is %lld\n",offset);
|
||||
}
|
||||
/* Write to the file */
|
||||
ret = mca_common_ompio_file_write_at ( sh->sharedfh, offset, buf, count, datatype, status);
|
||||
ret = mca_common_ompio_file_write_at ( fh, offset, buf, count, datatype, status);
|
||||
}
|
||||
|
||||
return ret;
|
||||
@ -93,7 +80,6 @@ int mca_sharedfp_lockedfile_write_ordered (ompio_file_t *fh,
|
||||
ompi_status_public_t *status)
|
||||
{
|
||||
int ret = OMPI_SUCCESS;
|
||||
mca_sharedfp_base_module_t * shared_fp_base_module=NULL;
|
||||
OMPI_MPI_OFFSET_TYPE offset = 0;
|
||||
long sendBuff = 0;
|
||||
long *buff=NULL;
|
||||
@ -107,21 +93,9 @@ int mca_sharedfp_lockedfile_write_ordered (ompio_file_t *fh,
|
||||
struct mca_sharedfp_base_data_t *sh = NULL;
|
||||
|
||||
if( NULL == fh->f_sharedfp_data ) {
|
||||
if ( mca_sharedfp_lockedfile_verbose ) {
|
||||
opal_output(ompi_sharedfp_base_framework.framework_output,
|
||||
"sharedfp_lockedfile_write_ordered - opening the shared file pointer\n");
|
||||
}
|
||||
shared_fp_base_module = fh->f_sharedfp;
|
||||
|
||||
ret = shared_fp_base_module->sharedfp_file_open(fh->f_comm,
|
||||
fh->f_filename,
|
||||
fh->f_amode,
|
||||
fh->f_info,
|
||||
fh);
|
||||
if ( OMPI_SUCCESS != ret ) {
|
||||
opal_output(0,"sharedfp_lockedfile_write_ordered - error opening the shared file pointer\n");
|
||||
return ret;
|
||||
}
|
||||
"sharedfp_lockedfile_write_ordered - framework not initialized\n");
|
||||
return OMPI_ERROR;
|
||||
}
|
||||
|
||||
/*Retrieve the new communicator*/
|
||||
@ -132,8 +106,8 @@ int mca_sharedfp_lockedfile_write_ordered (ompio_file_t *fh,
|
||||
sendBuff = count * numofBytes;
|
||||
|
||||
/* Get the ranks in the communicator */
|
||||
rank = ompi_comm_rank ( sh->comm );
|
||||
size = ompi_comm_size ( sh->comm );
|
||||
rank = ompi_comm_rank ( fh->f_comm );
|
||||
size = ompi_comm_size ( fh->f_comm );
|
||||
|
||||
if ( 0 == rank ) {
|
||||
buff = (long*) malloc (sizeof(long) * size);
|
||||
@ -142,9 +116,15 @@ int mca_sharedfp_lockedfile_write_ordered (ompio_file_t *fh,
|
||||
}
|
||||
}
|
||||
|
||||
ret = sh->comm->c_coll->coll_gather ( &sendBuff, sendcnt, OMPI_OFFSET_DATATYPE, buff, recvcnt,
|
||||
OMPI_OFFSET_DATATYPE, 0, sh->comm,
|
||||
sh->comm->c_coll->coll_gather_module );
|
||||
ret = fh->f_comm->c_coll->coll_gather ( &sendBuff,
|
||||
sendcnt,
|
||||
OMPI_OFFSET_DATATYPE,
|
||||
buff,
|
||||
recvcnt,
|
||||
OMPI_OFFSET_DATATYPE,
|
||||
0,
|
||||
fh->f_comm,
|
||||
fh->f_comm->c_coll->coll_gather_module );
|
||||
if ( OMPI_SUCCESS != ret ) {
|
||||
goto exit;
|
||||
}
|
||||
@ -182,16 +162,22 @@ int mca_sharedfp_lockedfile_write_ordered (ompio_file_t *fh,
|
||||
}
|
||||
|
||||
/* Scatter the results to the other processes*/
|
||||
ret = sh->comm->c_coll->coll_scatter ( buff, sendcnt, OMPI_OFFSET_DATATYPE,
|
||||
&offsetBuff, recvcnt, OMPI_OFFSET_DATATYPE, 0,
|
||||
sh->comm, sh->comm->c_coll->coll_scatter_module );
|
||||
ret = fh->f_comm->c_coll->coll_scatter ( buff,
|
||||
sendcnt,
|
||||
OMPI_OFFSET_DATATYPE,
|
||||
&offsetBuff,
|
||||
recvcnt,
|
||||
OMPI_OFFSET_DATATYPE,
|
||||
0,
|
||||
fh->f_comm,
|
||||
fh->f_comm->c_coll->coll_scatter_module );
|
||||
if ( OMPI_SUCCESS != ret ) {
|
||||
goto exit;
|
||||
}
|
||||
|
||||
/*Each process now has its own individual offset*/
|
||||
offset = offsetBuff - sendBuff;
|
||||
offset /= sh->sharedfh->f_etype_size;
|
||||
offset /= fh->f_etype_size;
|
||||
|
||||
if ( mca_sharedfp_lockedfile_verbose ) {
|
||||
opal_output(ompi_sharedfp_base_framework.framework_output,
|
||||
@ -199,7 +185,7 @@ int mca_sharedfp_lockedfile_write_ordered (ompio_file_t *fh,
|
||||
}
|
||||
|
||||
/* write to the file */
|
||||
ret = mca_common_ompio_file_write_at_all(sh->sharedfh,offset,buf,count,datatype,status);
|
||||
ret = mca_common_ompio_file_write_at_all(fh,offset,buf,count,datatype,status);
|
||||
|
||||
exit:
|
||||
if ( NULL != buff ) {
|
||||
|
||||
@ -10,7 +10,7 @@
|
||||
* University of Stuttgart. All rights reserved.
|
||||
* Copyright (c) 2004-2005 The Regents of the University of California.
|
||||
* All rights reserved.
|
||||
* Copyright (c) 2008-2013 University of Houston. All rights reserved.
|
||||
* Copyright (c) 2008-2018 University of Houston. All rights reserved.
|
||||
* Copyright (c) 2015 Los Alamos National Security, LLC. All rights
|
||||
* reserved.
|
||||
* Copyright (c) 2015-2018 Research Organization for Information Science
|
||||
@ -223,9 +223,7 @@ typedef mca_sharedfp_base_module_1_0_0_t mca_sharedfp_base_module_t;
|
||||
*/
|
||||
struct mca_sharedfp_base_data_t{
|
||||
/* attributes that will be used by all of the sharedfp components */
|
||||
struct ompio_file_t * sharedfh;
|
||||
OMPI_MPI_OFFSET_TYPE global_offset;
|
||||
struct ompi_communicator_t * comm;
|
||||
|
||||
/* attributes that are specific to a component are
|
||||
* combined into a structure that is assigned to this attribute */
|
||||
|
||||
@ -127,7 +127,7 @@ struct mca_sharedfp_sm_data
|
||||
typedef struct mca_sharedfp_sm_data sm_data;
|
||||
|
||||
|
||||
int mca_sharedfp_sm_request_position (struct mca_sharedfp_base_data_t * sh,
|
||||
int mca_sharedfp_sm_request_position (ompio_file_t *fh,
|
||||
int bytes_requested,
|
||||
OMPI_MPI_OFFSET_TYPE * offset);
|
||||
/*
|
||||
|
||||
@ -9,7 +9,7 @@
|
||||
* University of Stuttgart. All rights reserved.
|
||||
* Copyright (c) 2004-2005 The Regents of the University of California.
|
||||
* All rights reserved.
|
||||
* Copyright (c) 2013-2017 University of Houston. All rights reserved.
|
||||
* Copyright (c) 2013-2018 University of Houston. All rights reserved.
|
||||
* Copyright (c) 2013 Intel, Inc. All rights reserved.
|
||||
* Copyright (c) 2015-2018 Research Organization for Information Science
|
||||
* and Technology (RIST). All rights reserved.
|
||||
@ -55,42 +55,14 @@ int mca_sharedfp_sm_file_open (struct ompi_communicator_t *comm,
|
||||
int err = OMPI_SUCCESS;
|
||||
struct mca_sharedfp_base_data_t* sh;
|
||||
struct mca_sharedfp_sm_data * sm_data = NULL;
|
||||
ompio_file_t * shfileHandle, *ompio_fh;
|
||||
char * filename_basename;
|
||||
char * sm_filename;
|
||||
int sm_filename_length;
|
||||
struct mca_sharedfp_sm_offset * sm_offset_ptr;
|
||||
struct mca_sharedfp_sm_offset sm_offset;
|
||||
mca_common_ompio_data_t *data;
|
||||
int sm_fd;
|
||||
int rank;
|
||||
uint32_t comm_cid;
|
||||
|
||||
/*----------------------------------------------------*/
|
||||
/*Open the same file again without shared file pointer*/
|
||||
/*----------------------------------------------------*/
|
||||
shfileHandle = (ompio_file_t *)malloc(sizeof(ompio_file_t));
|
||||
if ( NULL == shfileHandle ) {
|
||||
opal_output(0, "mca_sharedfp_sm_file_open: Error during memory allocation\n");
|
||||
return OMPI_ERR_OUT_OF_RESOURCE;
|
||||
}
|
||||
err = mca_common_ompio_file_open(comm,filename,amode,info,shfileHandle,false);
|
||||
if ( OMPI_SUCCESS != err) {
|
||||
opal_output(0, "mca_sharedfp_sm_file_open: Error during file open\n");
|
||||
free (shfileHandle);
|
||||
return err;
|
||||
}
|
||||
shfileHandle->f_fh = fh->f_fh;
|
||||
data = (mca_common_ompio_data_t *) fh->f_fh->f_io_selected_data;
|
||||
ompio_fh = &data->ompio_fh;
|
||||
|
||||
err = mca_common_ompio_set_view (shfileHandle,
|
||||
ompio_fh->f_disp,
|
||||
ompio_fh->f_etype,
|
||||
ompio_fh->f_orig_filetype,
|
||||
ompio_fh->f_datarep,
|
||||
&(MPI_INFO_NULL->super));
|
||||
|
||||
/*Memory is allocated here for the sh structure*/
|
||||
if ( mca_sharedfp_sm_verbose ) {
|
||||
opal_output(ompi_sharedfp_base_framework.framework_output,
|
||||
@ -99,19 +71,14 @@ int mca_sharedfp_sm_file_open (struct ompi_communicator_t *comm,
|
||||
|
||||
sh = (struct mca_sharedfp_base_data_t*)malloc(sizeof(struct mca_sharedfp_base_data_t));
|
||||
if ( NULL == sh ) {
|
||||
opal_output(0, "mca_sharedfp_sm_file_open: Error, unable to malloc f_sharedfp_ptr struct\n");
|
||||
free(shfileHandle);
|
||||
opal_output(0, "mca_sharedfp_sm_file_open: Error, unable to malloc f_sharedfp struct\n");
|
||||
return OMPI_ERR_OUT_OF_RESOURCE;
|
||||
}
|
||||
|
||||
/*Populate the sh file structure based on the implementation*/
|
||||
sh->sharedfh = shfileHandle; /* Shared file pointer*/
|
||||
sh->global_offset = 0; /* Global Offset*/
|
||||
sh->comm = comm; /* Communicator*/
|
||||
sh->selected_module_data = NULL;
|
||||
|
||||
rank = ompi_comm_rank ( sh->comm );
|
||||
|
||||
/*Open a shared memory segment which will hold the shared file pointer*/
|
||||
if ( mca_sharedfp_sm_verbose ) {
|
||||
opal_output(ompi_sharedfp_base_framework.framework_output,
|
||||
@ -123,7 +90,6 @@ int mca_sharedfp_sm_file_open (struct ompi_communicator_t *comm,
|
||||
if ( NULL == sm_data ){
|
||||
opal_output(0, "mca_sharedfp_sm_file_open: Error, unable to malloc sm_data struct\n");
|
||||
free(sh);
|
||||
free(shfileHandle);
|
||||
return OMPI_ERR_OUT_OF_RESOURCE;
|
||||
}
|
||||
sm_data->sm_filename=NULL;
|
||||
@ -141,7 +107,6 @@ int mca_sharedfp_sm_file_open (struct ompi_communicator_t *comm,
|
||||
opal_output(0, "mca_sharedfp_sm_file_open: Error, unable to malloc sm_filename\n");
|
||||
free(sm_data);
|
||||
free(sh);
|
||||
free(shfileHandle);
|
||||
return OMPI_ERR_OUT_OF_RESOURCE;
|
||||
}
|
||||
|
||||
@ -156,14 +121,13 @@ int mca_sharedfp_sm_file_open (struct ompi_communicator_t *comm,
|
||||
free(sm_filename);
|
||||
free(sm_data);
|
||||
free(sh);
|
||||
free(shfileHandle);
|
||||
return OMPI_ERROR;
|
||||
}
|
||||
|
||||
sm_data->sm_filename = sm_filename;
|
||||
|
||||
/*TODO: is it necessary to write to the file first?*/
|
||||
if( 0 == rank ){
|
||||
if( 0 == fh->f_rank ){
|
||||
memset ( &sm_offset, 0, sizeof (struct mca_sharedfp_sm_offset ));
|
||||
write ( sm_fd, &sm_offset, sizeof(struct mca_sharedfp_sm_offset));
|
||||
}
|
||||
@ -182,7 +146,6 @@ int mca_sharedfp_sm_file_open (struct ompi_communicator_t *comm,
|
||||
free(sm_filename);
|
||||
free(sm_data);
|
||||
free(sh);
|
||||
free(shfileHandle);
|
||||
return OMPI_ERROR;
|
||||
}
|
||||
|
||||
@ -213,7 +176,7 @@ int mca_sharedfp_sm_file_open (struct ompi_communicator_t *comm,
|
||||
fh->f_sharedfp_data = sh;
|
||||
|
||||
/*write initial zero*/
|
||||
if(rank==0){
|
||||
if(fh->f_rank==0){
|
||||
MPI_Offset position=0;
|
||||
|
||||
sem_wait(sm_data->mutex);
|
||||
@ -224,7 +187,6 @@ int mca_sharedfp_sm_file_open (struct ompi_communicator_t *comm,
|
||||
free(sm_filename);
|
||||
free(sm_data);
|
||||
free(sh);
|
||||
free(shfileHandle);
|
||||
munmap(sm_offset_ptr, sizeof(struct mca_sharedfp_sm_offset));
|
||||
err = OMPI_ERROR;
|
||||
}
|
||||
@ -232,7 +194,7 @@ int mca_sharedfp_sm_file_open (struct ompi_communicator_t *comm,
|
||||
comm->c_coll->coll_barrier (comm, comm->c_coll->coll_barrier_module );
|
||||
|
||||
#if defined(HAVE_SEM_OPEN)
|
||||
if ( 0 == rank ) {
|
||||
if ( 0 == fh->f_rank ) {
|
||||
sem_unlink ( sm_data->sem_name);
|
||||
}
|
||||
#endif
|
||||
@ -249,10 +211,6 @@ int mca_sharedfp_sm_file_close (ompio_file_t *fh)
|
||||
struct mca_sharedfp_sm_data * file_data=NULL;
|
||||
|
||||
if( NULL == fh->f_sharedfp_data ){
|
||||
if ( mca_sharedfp_sm_verbose ) {
|
||||
opal_output(ompi_sharedfp_base_framework.framework_output,
|
||||
"sharedfp_sm_file_close: shared file pointer structure not initialized\n");
|
||||
}
|
||||
return OMPI_SUCCESS;
|
||||
}
|
||||
sh = fh->f_sharedfp_data;
|
||||
@ -261,7 +219,7 @@ int mca_sharedfp_sm_file_close (ompio_file_t *fh)
|
||||
* all processes are ready to release the
|
||||
* shared file pointer resources
|
||||
*/
|
||||
sh->comm->c_coll->coll_barrier (sh->comm, sh->comm->c_coll->coll_barrier_module );
|
||||
fh->f_comm->c_coll->coll_barrier (fh->f_comm, fh->f_comm->c_coll->coll_barrier_module );
|
||||
|
||||
file_data = (sm_data*)(sh->selected_module_data);
|
||||
if (file_data) {
|
||||
@ -286,12 +244,8 @@ int mca_sharedfp_sm_file_close (ompio_file_t *fh)
|
||||
free(file_data);
|
||||
}
|
||||
|
||||
/* Close the main file opened by this component*/
|
||||
err = mca_common_ompio_file_close(sh->sharedfh);
|
||||
|
||||
/*free shared file pointer data struct*/
|
||||
free(sh);
|
||||
|
||||
return err;
|
||||
|
||||
}
|
||||
|
||||
@ -9,7 +9,7 @@
|
||||
* University of Stuttgart. All rights reserved.
|
||||
* Copyright (c) 2004-2005 The Regents of the University of California.
|
||||
* All rights reserved.
|
||||
* Copyright (c) 2013 University of Houston. All rights reserved.
|
||||
* Copyright (c) 2013-2018 University of Houston. All rights reserved.
|
||||
* Copyright (c) 2018 Research Organization for Information Science
|
||||
* and Technology (RIST). All rights reserved.
|
||||
* $COPYRIGHT$
|
||||
@ -32,33 +32,15 @@ int
|
||||
mca_sharedfp_sm_get_position(ompio_file_t *fh,
|
||||
OMPI_MPI_OFFSET_TYPE * offset)
|
||||
{
|
||||
int ret = OMPI_SUCCESS;
|
||||
|
||||
struct mca_sharedfp_base_data_t *sh = NULL;
|
||||
mca_sharedfp_base_module_t * shared_fp_base_module = NULL;
|
||||
|
||||
if(fh->f_sharedfp_data==NULL){
|
||||
opal_output(ompi_sharedfp_base_framework.framework_output,
|
||||
"sharedfp_sm_write - opening the shared file pointer\n");
|
||||
shared_fp_base_module = fh->f_sharedfp;
|
||||
|
||||
ret = shared_fp_base_module->sharedfp_file_open(fh->f_comm,
|
||||
fh->f_filename,
|
||||
fh->f_amode,
|
||||
fh->f_info,
|
||||
fh);
|
||||
if (ret != OMPI_SUCCESS) {
|
||||
opal_output(0,"sharedfp_sm_write - error opening the shared file pointer\n");
|
||||
return ret;
|
||||
"sharedfp_sm_write - module not initialized\n");
|
||||
return OMPI_ERROR;
|
||||
}
|
||||
}
|
||||
/*Retrieve the shared file data struct*/
|
||||
sh = fh->f_sharedfp_data;
|
||||
|
||||
/*Requesting the offset to write 0 bytes,
|
||||
*returns the current offset w/o updating it
|
||||
*/
|
||||
ret = mca_sharedfp_sm_request_position(sh,0,offset);
|
||||
|
||||
return ret;
|
||||
return mca_sharedfp_sm_request_position(fh,0,offset);
|
||||
}
|
||||
|
||||
@ -9,7 +9,7 @@
|
||||
* University of Stuttgart. All rights reserved.
|
||||
* Copyright (c) 2004-2005 The Regents of the University of California.
|
||||
* All rights reserved.
|
||||
* Copyright (c) 2013-2016 University of Houston. All rights reserved.
|
||||
* Copyright (c) 2013-2018 University of Houston. All rights reserved.
|
||||
* Copyright (c) 2018 Research Organization for Information Science
|
||||
* and Technology (RIST). All rights reserved.
|
||||
* $COPYRIGHT$
|
||||
@ -38,41 +38,24 @@ int mca_sharedfp_sm_iread(ompio_file_t *fh,
|
||||
OMPI_MPI_OFFSET_TYPE offset = 0;
|
||||
long bytesRequested = 0;
|
||||
size_t numofBytes;
|
||||
struct mca_sharedfp_base_data_t *sh = NULL;
|
||||
mca_sharedfp_base_module_t * shared_fp_base_module = NULL;
|
||||
|
||||
if( NULL == fh->f_sharedfp_data){
|
||||
if ( mca_sharedfp_sm_verbose ) {
|
||||
opal_output(ompi_sharedfp_base_framework.framework_output,
|
||||
"sharedfp_sm_iread: opening the shared file pointer\n");
|
||||
}
|
||||
shared_fp_base_module = fh->f_sharedfp;
|
||||
|
||||
ret = shared_fp_base_module->sharedfp_file_open(fh->f_comm,
|
||||
fh->f_filename,
|
||||
fh->f_amode,
|
||||
fh->f_info,
|
||||
fh);
|
||||
if ( OMPI_SUCCESS != ret ) {
|
||||
opal_output(0,"sharedfp_sm_iread - error opening the shared file pointer\n");
|
||||
return ret;
|
||||
}
|
||||
"sharedfp_sm_iread: module not initialized\n");
|
||||
return OMPI_ERROR;
|
||||
}
|
||||
|
||||
/* Calculate the number of bytes to write */
|
||||
opal_datatype_type_size ( &datatype->super, &numofBytes);
|
||||
bytesRequested = count * numofBytes;
|
||||
|
||||
/* Retrieve the shared file data struct */
|
||||
sh = fh->f_sharedfp_data;
|
||||
|
||||
if ( mca_sharedfp_sm_verbose ) {
|
||||
opal_output(ompi_sharedfp_base_framework.framework_output,
|
||||
"sharedfp_sm_iread: Bytes Requested is %ld\n",bytesRequested);
|
||||
}
|
||||
/*Request the offset to write bytesRequested bytes*/
|
||||
ret = mca_sharedfp_sm_request_position(sh,bytesRequested,&offset);
|
||||
offset /= sh->sharedfh->f_etype_size;
|
||||
ret = mca_sharedfp_sm_request_position(fh,bytesRequested,&offset);
|
||||
offset /= fh->f_etype_size;
|
||||
|
||||
if ( -1 != ret ) {
|
||||
if ( mca_sharedfp_sm_verbose ) {
|
||||
@ -80,7 +63,7 @@ int mca_sharedfp_sm_iread(ompio_file_t *fh,
|
||||
"sharedfp_sm_iread: Offset received is %lld\n",offset);
|
||||
}
|
||||
/* Read the file */
|
||||
ret = mca_common_ompio_file_iread_at(sh->sharedfh,offset,buf,count,datatype,request);
|
||||
ret = mca_common_ompio_file_iread_at(fh,offset,buf,count,datatype,request);
|
||||
}
|
||||
|
||||
return ret;
|
||||
@ -100,54 +83,40 @@ int mca_sharedfp_sm_read_ordered_begin(ompio_file_t *fh,
|
||||
long bytesRequested = 0;
|
||||
int recvcnt = 1, sendcnt = 1;
|
||||
size_t numofBytes;
|
||||
int rank, size, i;
|
||||
struct mca_sharedfp_base_data_t *sh = NULL;
|
||||
mca_sharedfp_base_module_t * shared_fp_base_module = NULL;
|
||||
int i;
|
||||
|
||||
if ( NULL == fh->f_sharedfp_data){
|
||||
if ( mca_sharedfp_sm_verbose ) {
|
||||
opal_output(ompi_sharedfp_base_framework.framework_output,
|
||||
"sharedfp_sm_read_ordered_begin: opening the shared file pointer\n");
|
||||
"sharedfp_sm_read_ordered_begin: module not initialized \n");
|
||||
return OMPI_ERROR;
|
||||
}
|
||||
shared_fp_base_module = fh->f_sharedfp;
|
||||
|
||||
ret = shared_fp_base_module->sharedfp_file_open(fh->f_comm,
|
||||
fh->f_filename,
|
||||
fh->f_amode,
|
||||
fh->f_info,
|
||||
fh);
|
||||
if ( OMPI_SUCCESS != ret ) {
|
||||
opal_output(0,"sharedfp_sm_read_ordered_begin - error opening the shared file pointer\n");
|
||||
return ret;
|
||||
}
|
||||
}
|
||||
if ( true == fh->f_split_coll_in_use ) {
|
||||
opal_output(0,"Only one split collective I/O operation allowed per file handle at any given point in time!\n");
|
||||
opal_output(0,"Only one split collective I/O operation allowed per file "
|
||||
"handle at any given point in time!\n");
|
||||
return MPI_ERR_REQUEST;
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*Retrieve the new communicator*/
|
||||
sh = fh->f_sharedfp_data;
|
||||
|
||||
/* Calculate the number of bytes to read*/
|
||||
opal_datatype_type_size ( &datatype->super, &numofBytes);
|
||||
sendBuff = count * numofBytes;
|
||||
|
||||
/* Get the ranks in the communicator */
|
||||
rank = ompi_comm_rank ( sh->comm );
|
||||
size = ompi_comm_size ( sh->comm );
|
||||
|
||||
if ( 0 == rank ) {
|
||||
buff = (long*)malloc(sizeof(long) * size);
|
||||
if ( 0 == fh->f_rank ) {
|
||||
buff = (long*)malloc(sizeof(long) * fh->f_size);
|
||||
if ( NULL == buff )
|
||||
return OMPI_ERR_OUT_OF_RESOURCE;
|
||||
}
|
||||
|
||||
ret = sh->comm->c_coll->coll_gather ( &sendBuff, sendcnt, OMPI_OFFSET_DATATYPE,
|
||||
buff, recvcnt, OMPI_OFFSET_DATATYPE, 0,
|
||||
sh->comm, sh->comm->c_coll->coll_gather_module );
|
||||
ret = fh->f_comm->c_coll->coll_gather ( &sendBuff,
|
||||
sendcnt,
|
||||
OMPI_OFFSET_DATATYPE,
|
||||
buff,
|
||||
recvcnt,
|
||||
OMPI_OFFSET_DATATYPE,
|
||||
0,
|
||||
fh->f_comm,
|
||||
fh->f_comm->c_coll->coll_gather_module );
|
||||
if( OMPI_SUCCESS != ret){
|
||||
goto exit;
|
||||
}
|
||||
@ -155,8 +124,8 @@ int mca_sharedfp_sm_read_ordered_begin(ompio_file_t *fh,
|
||||
/* All the counts are present now in the recvBuff.
|
||||
** The size of recvBuff is sizeof_newComm
|
||||
*/
|
||||
if ( 0 == rank ) {
|
||||
for (i = 0; i < size ; i ++) {
|
||||
if ( 0 == fh->f_rank ) {
|
||||
for (i = 0; i < fh->f_size ; i ++) {
|
||||
bytesRequested += buff[i];
|
||||
if ( mca_sharedfp_sm_verbose ) {
|
||||
opal_output(ompi_sharedfp_base_framework.framework_output,
|
||||
@ -171,7 +140,7 @@ int mca_sharedfp_sm_read_ordered_begin(ompio_file_t *fh,
|
||||
** processes at what offset they should read their
|
||||
** share of the data.
|
||||
*/
|
||||
ret = mca_sharedfp_sm_request_position(sh,bytesRequested,&offsetReceived);
|
||||
ret = mca_sharedfp_sm_request_position(fh,bytesRequested,&offsetReceived);
|
||||
if( OMPI_SUCCESS != ret){
|
||||
goto exit;
|
||||
}
|
||||
@ -181,22 +150,28 @@ int mca_sharedfp_sm_read_ordered_begin(ompio_file_t *fh,
|
||||
}
|
||||
|
||||
buff[0] += offsetReceived;
|
||||
for (i = 1 ; i < size; i++) {
|
||||
for (i = 1 ; i < fh->f_size; i++) {
|
||||
buff[i] += buff[i-1];
|
||||
}
|
||||
}
|
||||
|
||||
/* Scatter the results to the other processes*/
|
||||
ret = sh->comm->c_coll->coll_scatter ( buff, sendcnt, OMPI_OFFSET_DATATYPE,
|
||||
&offsetBuff, recvcnt, OMPI_OFFSET_DATATYPE, 0,
|
||||
sh->comm, sh->comm->c_coll->coll_scatter_module );
|
||||
ret = fh->f_comm->c_coll->coll_scatter ( buff,
|
||||
sendcnt,
|
||||
OMPI_OFFSET_DATATYPE,
|
||||
&offsetBuff,
|
||||
recvcnt,
|
||||
OMPI_OFFSET_DATATYPE,
|
||||
0,
|
||||
fh->f_comm,
|
||||
fh->f_comm->c_coll->coll_scatter_module );
|
||||
if( OMPI_SUCCESS != ret){
|
||||
goto exit;
|
||||
}
|
||||
|
||||
/*Each process now has its own individual offset in recvBUFF*/
|
||||
offset = offsetBuff - sendBuff;
|
||||
offset /= sh->sharedfh->f_etype_size;
|
||||
offset /= fh->f_etype_size;
|
||||
|
||||
if ( mca_sharedfp_sm_verbose ) {
|
||||
opal_output(ompi_sharedfp_base_framework.framework_output,
|
||||
@ -204,7 +179,7 @@ int mca_sharedfp_sm_read_ordered_begin(ompio_file_t *fh,
|
||||
}
|
||||
|
||||
/* read to the file */
|
||||
ret = mca_common_ompio_file_iread_at_all(sh->sharedfh,offset,buf,count,datatype,
|
||||
ret = mca_common_ompio_file_iread_at_all(fh,offset,buf,count,datatype,
|
||||
&fh->f_split_coll_req);
|
||||
fh->f_split_coll_in_use = true;
|
||||
|
||||
|
||||
@ -9,7 +9,7 @@
|
||||
* University of Stuttgart. All rights reserved.
|
||||
* Copyright (c) 2004-2005 The Regents of the University of California.
|
||||
* All rights reserved.
|
||||
* Copyright (c) 2013-2016 University of Houston. All rights reserved.
|
||||
* Copyright (c) 2013-2018 University of Houston. All rights reserved.
|
||||
* Copyright (c) 2015-2018 Research Organization for Information Science
|
||||
* and Technology (RIST). All rights reserved.
|
||||
* $COPYRIGHT$
|
||||
@ -38,41 +38,24 @@ int mca_sharedfp_sm_iwrite(ompio_file_t *fh,
|
||||
OMPI_MPI_OFFSET_TYPE offset = 0;
|
||||
long bytesRequested = 0;
|
||||
size_t numofBytes;
|
||||
struct mca_sharedfp_base_data_t *sh = NULL;
|
||||
mca_sharedfp_base_module_t * shared_fp_base_module = NULL;
|
||||
|
||||
if( NULL == fh->f_sharedfp_data){
|
||||
if ( mca_sharedfp_sm_verbose ) {
|
||||
opal_output(ompi_sharedfp_base_framework.framework_output,
|
||||
"sharedfp_sm_iwrite - opening the shared file pointer\n");
|
||||
}
|
||||
shared_fp_base_module = fh->f_sharedfp;
|
||||
|
||||
ret = shared_fp_base_module->sharedfp_file_open(fh->f_comm,
|
||||
fh->f_filename,
|
||||
fh->f_amode,
|
||||
fh->f_info,
|
||||
fh);
|
||||
if ( OMPI_SUCCESS != ret ) {
|
||||
opal_output(0,"sharedfp_sm_iwrite - error opening the shared file pointer\n");
|
||||
return ret;
|
||||
}
|
||||
"sharedfp_sm_iwrite - module not initialized\n");
|
||||
return OMPI_ERROR;
|
||||
}
|
||||
|
||||
/* Calculate the number of bytes to write */
|
||||
opal_datatype_type_size ( &datatype->super, &numofBytes);
|
||||
bytesRequested = count * numofBytes;
|
||||
|
||||
/* Retrieve the shared file data struct */
|
||||
sh = fh->f_sharedfp_data;
|
||||
|
||||
if ( mca_sharedfp_sm_verbose ) {
|
||||
opal_output(ompi_sharedfp_base_framework.framework_output,
|
||||
"sharedfp_sm_iwrite: Bytes Requested is %ld\n",bytesRequested);
|
||||
}
|
||||
/* Request the offset to write bytesRequested bytes */
|
||||
ret = mca_sharedfp_sm_request_position(sh,bytesRequested,&offset);
|
||||
offset /= sh->sharedfh->f_etype_size;
|
||||
ret = mca_sharedfp_sm_request_position(fh,bytesRequested,&offset);
|
||||
offset /= fh->f_etype_size;
|
||||
|
||||
if ( -1 != ret ) {
|
||||
if ( mca_sharedfp_sm_verbose ) {
|
||||
@ -80,7 +63,7 @@ int mca_sharedfp_sm_iwrite(ompio_file_t *fh,
|
||||
"sharedfp_sm_iwrite: Offset received is %lld\n",offset);
|
||||
}
|
||||
/* Write to the file */
|
||||
ret = mca_common_ompio_file_iwrite_at(sh->sharedfh,offset,buf,count,datatype,request);
|
||||
ret = mca_common_ompio_file_iwrite_at(fh,offset,buf,count,datatype,request);
|
||||
}
|
||||
|
||||
return ret;
|
||||
@ -101,54 +84,33 @@ int mca_sharedfp_sm_write_ordered_begin(ompio_file_t *fh,
|
||||
long bytesRequested = 0;
|
||||
int recvcnt = 1, sendcnt = 1;
|
||||
size_t numofBytes;
|
||||
int rank, size, i;
|
||||
struct mca_sharedfp_base_data_t *sh = NULL;
|
||||
mca_sharedfp_base_module_t * shared_fp_base_module = NULL;
|
||||
int i;
|
||||
|
||||
if ( NULL == fh->f_sharedfp_data){
|
||||
if ( mca_sharedfp_sm_verbose ) {
|
||||
opal_output(ompi_sharedfp_base_framework.framework_output,
|
||||
"sharedfp_sm_write_ordered_begin: opening the shared file pointer\n");
|
||||
"sharedfp_sm_write_ordered_begin: module not initialized\n");
|
||||
return OMPI_ERROR;
|
||||
}
|
||||
shared_fp_base_module = fh->f_sharedfp;
|
||||
|
||||
ret = shared_fp_base_module->sharedfp_file_open(fh->f_comm,
|
||||
fh->f_filename,
|
||||
fh->f_amode,
|
||||
fh->f_info,
|
||||
fh);
|
||||
if ( OMPI_SUCCESS != ret ) {
|
||||
opal_output(0,"sharedfp_sm_write_ordered_begin - error opening the shared file pointer\n");
|
||||
return ret;
|
||||
}
|
||||
}
|
||||
if ( true == fh->f_split_coll_in_use ) {
|
||||
opal_output(0, "Only one split collective I/O operation allowed per file handle at any given point in time!\n");
|
||||
opal_output(0, "Only one split collective I/O operation allowed per file "
|
||||
"handle at any given point in time!\n");
|
||||
return MPI_ERR_REQUEST;
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*Retrieve the new communicator*/
|
||||
sh = fh->f_sharedfp_data;
|
||||
|
||||
/* Calculate the number of bytes to read*/
|
||||
opal_datatype_type_size ( &datatype->super, &numofBytes);
|
||||
sendBuff = count * numofBytes;
|
||||
|
||||
/* Get the ranks in the communicator */
|
||||
rank = ompi_comm_rank ( sh->comm );
|
||||
size = ompi_comm_size ( sh->comm );
|
||||
|
||||
if ( 0 == rank ) {
|
||||
buff = (long*)malloc(sizeof(long) * size);
|
||||
if ( 0 == fh->f_rank ) {
|
||||
buff = (long*)malloc(sizeof(long) * fh->f_size);
|
||||
if ( NULL == buff )
|
||||
return OMPI_ERR_OUT_OF_RESOURCE;
|
||||
}
|
||||
|
||||
ret = sh->comm->c_coll->coll_gather ( &sendBuff, sendcnt, OMPI_OFFSET_DATATYPE,
|
||||
ret = fh->f_comm->c_coll->coll_gather ( &sendBuff, sendcnt, OMPI_OFFSET_DATATYPE,
|
||||
buff, recvcnt, OMPI_OFFSET_DATATYPE, 0,
|
||||
sh->comm, sh->comm->c_coll->coll_gather_module );
|
||||
fh->f_comm, fh->f_comm->c_coll->coll_gather_module );
|
||||
if( OMPI_SUCCESS != ret){
|
||||
goto exit;
|
||||
}
|
||||
@ -156,8 +118,8 @@ int mca_sharedfp_sm_write_ordered_begin(ompio_file_t *fh,
|
||||
/* All the counts are present now in the recvBuff.
|
||||
** The size of recvBuff is sizeof_newComm
|
||||
*/
|
||||
if ( 0 == rank ) {
|
||||
for (i = 0; i < size ; i ++) {
|
||||
if ( 0 == fh->f_rank ) {
|
||||
for (i = 0; i < fh->f_size ; i ++) {
|
||||
bytesRequested += buff[i];
|
||||
if ( mca_sharedfp_sm_verbose ) {
|
||||
opal_output(ompi_sharedfp_base_framework.framework_output,
|
||||
@ -172,7 +134,7 @@ int mca_sharedfp_sm_write_ordered_begin(ompio_file_t *fh,
|
||||
** processes at what offset they should read their
|
||||
** share of the data.
|
||||
*/
|
||||
ret = mca_sharedfp_sm_request_position(sh,bytesRequested,&offsetReceived);
|
||||
ret = mca_sharedfp_sm_request_position(fh,bytesRequested,&offsetReceived);
|
||||
if( OMPI_SUCCESS != ret){
|
||||
goto exit;
|
||||
}
|
||||
@ -182,22 +144,22 @@ int mca_sharedfp_sm_write_ordered_begin(ompio_file_t *fh,
|
||||
}
|
||||
|
||||
buff[0] += offsetReceived;
|
||||
for (i = 1 ; i < size; i++) {
|
||||
for (i = 1 ; i < fh->f_size; i++) {
|
||||
buff[i] += buff[i-1];
|
||||
}
|
||||
}
|
||||
|
||||
/* Scatter the results to the other processes*/
|
||||
ret = sh->comm->c_coll->coll_scatter ( buff, sendcnt, OMPI_OFFSET_DATATYPE,
|
||||
ret = fh->f_comm->c_coll->coll_scatter ( buff, sendcnt, OMPI_OFFSET_DATATYPE,
|
||||
&offsetBuff, recvcnt, OMPI_OFFSET_DATATYPE, 0,
|
||||
sh->comm, sh->comm->c_coll->coll_scatter_module );
|
||||
fh->f_comm, fh->f_comm->c_coll->coll_scatter_module );
|
||||
if( OMPI_SUCCESS != ret){
|
||||
goto exit;
|
||||
}
|
||||
|
||||
/*Each process now has its own individual offset in recvBUFF*/
|
||||
offset = offsetBuff - sendBuff;
|
||||
offset /= sh->sharedfh->f_etype_size;
|
||||
offset /= fh->f_etype_size;
|
||||
|
||||
if ( mca_sharedfp_sm_verbose ) {
|
||||
opal_output(ompi_sharedfp_base_framework.framework_output,
|
||||
@ -205,7 +167,7 @@ int mca_sharedfp_sm_write_ordered_begin(ompio_file_t *fh,
|
||||
}
|
||||
|
||||
/* read to the file */
|
||||
ret = mca_common_ompio_file_iwrite_at_all(sh->sharedfh,offset,buf,count,datatype,
|
||||
ret = mca_common_ompio_file_iwrite_at_all(fh,offset,buf,count,datatype,
|
||||
&fh->f_split_coll_req);
|
||||
fh->f_split_coll_in_use = true;
|
||||
|
||||
|
||||
@ -9,7 +9,7 @@
|
||||
* University of Stuttgart. All rights reserved.
|
||||
* Copyright (c) 2004-2005 The Regents of the University of California.
|
||||
* All rights reserved.
|
||||
* Copyright (c) 2013-2016 University of Houston. All rights reserved.
|
||||
* Copyright (c) 2013-2018 University of Houston. All rights reserved.
|
||||
* Copyright (c) 2018 Research Organization for Information Science
|
||||
* and Technology (RIST). All rights reserved.
|
||||
* $COPYRIGHT$
|
||||
@ -35,42 +35,25 @@ int mca_sharedfp_sm_read ( ompio_file_t *fh,
|
||||
OMPI_MPI_OFFSET_TYPE offset = 0;
|
||||
long bytesRequested = 0;
|
||||
size_t numofBytes;
|
||||
struct mca_sharedfp_base_data_t *sh = NULL;
|
||||
mca_sharedfp_base_module_t * shared_fp_base_module = NULL;
|
||||
|
||||
if( NULL == fh->f_sharedfp_data){
|
||||
if ( mca_sharedfp_sm_verbose ) {
|
||||
opal_output(ompi_sharedfp_base_framework.framework_output,
|
||||
"sharedfp_sm_read - opening the shared file pointer\n");
|
||||
}
|
||||
shared_fp_base_module = fh->f_sharedfp;
|
||||
|
||||
ret = shared_fp_base_module->sharedfp_file_open(fh->f_comm,
|
||||
fh->f_filename,
|
||||
fh->f_amode,
|
||||
fh->f_info,
|
||||
fh);
|
||||
if ( OMPI_SUCCESS != ret ) {
|
||||
opal_output(0,"sharedfp_sm_read - error opening the shared file pointer\n");
|
||||
return ret;
|
||||
}
|
||||
"sharedfp_sm_read - module not initialized \n");
|
||||
return OMPI_ERROR;
|
||||
}
|
||||
|
||||
/* Calculate the number of bytes to write */
|
||||
opal_datatype_type_size ( &datatype->super, &numofBytes);
|
||||
bytesRequested = count * numofBytes;
|
||||
|
||||
/* Retrieve the shared file data struct */
|
||||
sh = fh->f_sharedfp_data;
|
||||
|
||||
if ( mca_sharedfp_sm_verbose ) {
|
||||
opal_output(ompi_sharedfp_base_framework.framework_output,
|
||||
"sharedfp_sm_read: Bytes Requested is %ld\n",bytesRequested);
|
||||
}
|
||||
|
||||
/*Request the offset to write bytesRequested bytes*/
|
||||
ret = mca_sharedfp_sm_request_position(sh,bytesRequested,&offset);
|
||||
offset /= sh->sharedfh->f_etype_size;
|
||||
ret = mca_sharedfp_sm_request_position(fh,bytesRequested,&offset);
|
||||
offset /= fh->f_etype_size;
|
||||
|
||||
if ( -1 != ret ) {
|
||||
if ( mca_sharedfp_sm_verbose ) {
|
||||
@ -79,7 +62,7 @@ int mca_sharedfp_sm_read ( ompio_file_t *fh,
|
||||
}
|
||||
|
||||
/* Read the file */
|
||||
ret = mca_common_ompio_file_read_at(sh->sharedfh,offset,buf,count,datatype,status);
|
||||
ret = mca_common_ompio_file_read_at(fh,offset,buf,count,datatype,status);
|
||||
}
|
||||
|
||||
return ret;
|
||||
@ -100,48 +83,33 @@ int mca_sharedfp_sm_read_ordered (ompio_file_t *fh,
|
||||
long bytesRequested = 0;
|
||||
int recvcnt = 1, sendcnt = 1;
|
||||
size_t numofBytes;
|
||||
int rank, size, i;
|
||||
struct mca_sharedfp_base_data_t *sh = NULL;
|
||||
mca_sharedfp_base_module_t * shared_fp_base_module = NULL;
|
||||
int i;
|
||||
|
||||
if ( NULL == fh->f_sharedfp_data){
|
||||
if ( mca_sharedfp_sm_verbose ) {
|
||||
opal_output(ompi_sharedfp_base_framework.framework_output,
|
||||
"sharedfp_sm_read_ordered: opening the shared file pointer\n");
|
||||
"sharedfp_sm_read_ordered: module not initialized \n");
|
||||
return OMPI_ERROR;
|
||||
}
|
||||
shared_fp_base_module = fh->f_sharedfp;
|
||||
|
||||
ret = shared_fp_base_module->sharedfp_file_open(fh->f_comm,
|
||||
fh->f_filename,
|
||||
fh->f_amode,
|
||||
fh->f_info,
|
||||
fh);
|
||||
if ( OMPI_SUCCESS != ret ) {
|
||||
opal_output(0,"sharedfp_sm_read_ordered - error opening the shared file pointer\n");
|
||||
return ret;
|
||||
}
|
||||
}
|
||||
|
||||
/*Retrieve the new communicator*/
|
||||
sh = fh->f_sharedfp_data;
|
||||
|
||||
/* Calculate the number of bytes to read*/
|
||||
opal_datatype_type_size ( &datatype->super, &numofBytes);
|
||||
sendBuff = count * numofBytes;
|
||||
|
||||
/* Get the ranks in the communicator */
|
||||
rank = ompi_comm_rank ( sh->comm );
|
||||
size = ompi_comm_size ( sh->comm );
|
||||
|
||||
if ( 0 == rank ) {
|
||||
buff = (long*)malloc(sizeof(long) * size);
|
||||
if ( 0 == fh->f_rank ) {
|
||||
buff = (long*)malloc(sizeof(long) * fh->f_size);
|
||||
if ( NULL == buff )
|
||||
return OMPI_ERR_OUT_OF_RESOURCE;
|
||||
}
|
||||
|
||||
ret = sh->comm->c_coll->coll_gather ( &sendBuff, sendcnt, OMPI_OFFSET_DATATYPE,
|
||||
buff, recvcnt, OMPI_OFFSET_DATATYPE, 0,
|
||||
sh->comm, sh->comm->c_coll->coll_gather_module );
|
||||
ret = fh->f_comm->c_coll->coll_gather ( &sendBuff,
|
||||
sendcnt,
|
||||
OMPI_OFFSET_DATATYPE,
|
||||
buff,
|
||||
recvcnt,
|
||||
OMPI_OFFSET_DATATYPE,
|
||||
0,
|
||||
fh->f_comm,
|
||||
fh->f_comm->c_coll->coll_gather_module );
|
||||
if( OMPI_SUCCESS != ret){
|
||||
goto exit;
|
||||
}
|
||||
@ -149,8 +117,8 @@ int mca_sharedfp_sm_read_ordered (ompio_file_t *fh,
|
||||
/* All the counts are present now in the recvBuff.
|
||||
** The size of recvBuff is sizeof_newComm
|
||||
*/
|
||||
if ( 0 == rank ) {
|
||||
for (i = 0; i < size ; i ++) {
|
||||
if ( 0 == fh->f_rank ) {
|
||||
for (i = 0; i < fh->f_size ; i ++) {
|
||||
bytesRequested += buff[i];
|
||||
if ( mca_sharedfp_sm_verbose ) {
|
||||
opal_output(ompi_sharedfp_base_framework.framework_output,
|
||||
@ -164,7 +132,7 @@ int mca_sharedfp_sm_read_ordered (ompio_file_t *fh,
|
||||
** processes at what offset they should read their
|
||||
** share of the data.
|
||||
*/
|
||||
ret = mca_sharedfp_sm_request_position(sh,bytesRequested,&offsetReceived);
|
||||
ret = mca_sharedfp_sm_request_position(fh,bytesRequested,&offsetReceived);
|
||||
if( OMPI_SUCCESS != ret){
|
||||
goto exit;
|
||||
}
|
||||
@ -174,22 +142,28 @@ int mca_sharedfp_sm_read_ordered (ompio_file_t *fh,
|
||||
}
|
||||
|
||||
buff[0] += offsetReceived;
|
||||
for (i = 1 ; i < size; i++) {
|
||||
for (i = 1 ; i < fh->f_size; i++) {
|
||||
buff[i] += buff[i-1];
|
||||
}
|
||||
}
|
||||
|
||||
/* Scatter the results to the other processes*/
|
||||
ret = sh->comm->c_coll->coll_scatter ( buff, sendcnt, OMPI_OFFSET_DATATYPE,
|
||||
&offsetBuff, recvcnt, OMPI_OFFSET_DATATYPE, 0,
|
||||
sh->comm, sh->comm->c_coll->coll_scatter_module );
|
||||
ret = fh->f_comm->c_coll->coll_scatter ( buff,
|
||||
sendcnt,
|
||||
OMPI_OFFSET_DATATYPE,
|
||||
&offsetBuff,
|
||||
recvcnt,
|
||||
OMPI_OFFSET_DATATYPE,
|
||||
0,
|
||||
fh->f_comm,
|
||||
fh->f_comm->c_coll->coll_scatter_module );
|
||||
if( OMPI_SUCCESS != ret){
|
||||
goto exit;
|
||||
}
|
||||
|
||||
/*Each process now has its own individual offset in recvBUFF*/
|
||||
offset = offsetBuff - sendBuff;
|
||||
offset /= sh->sharedfh->f_etype_size;
|
||||
offset /= fh->f_etype_size;
|
||||
|
||||
if ( mca_sharedfp_sm_verbose ) {
|
||||
opal_output(ompi_sharedfp_base_framework.framework_output,
|
||||
@ -197,7 +171,7 @@ int mca_sharedfp_sm_read_ordered (ompio_file_t *fh,
|
||||
}
|
||||
|
||||
/* read to the file */
|
||||
ret = mca_common_ompio_file_read_at_all(sh->sharedfh,offset,buf,count,datatype,status);
|
||||
ret = mca_common_ompio_file_read_at_all(fh,offset,buf,count,datatype,status);
|
||||
|
||||
exit:
|
||||
if ( NULL != buff ) {
|
||||
|
||||
@ -30,21 +30,24 @@
|
||||
/*use a semaphore to lock the shared memory*/
|
||||
#include <semaphore.h>
|
||||
|
||||
int mca_sharedfp_sm_request_position(struct mca_sharedfp_base_data_t * sh,
|
||||
int mca_sharedfp_sm_request_position(ompio_file_t *fh,
|
||||
int bytes_requested,
|
||||
OMPI_MPI_OFFSET_TYPE *offset)
|
||||
{
|
||||
int ret = OMPI_SUCCESS;
|
||||
OMPI_MPI_OFFSET_TYPE position = 0;
|
||||
OMPI_MPI_OFFSET_TYPE old_offset;
|
||||
struct mca_sharedfp_sm_data * sm_data = sh->selected_module_data;
|
||||
struct mca_sharedfp_sm_data * sm_data = NULL;
|
||||
struct mca_sharedfp_sm_offset * sm_offset_ptr = NULL;
|
||||
int rank = ompi_comm_rank ( sh->comm);
|
||||
struct mca_sharedfp_base_data_t *sh = NULL;
|
||||
|
||||
sh = fh->f_sharedfp_data;
|
||||
sm_data = sh->selected_module_data;
|
||||
|
||||
*offset = 0;
|
||||
if ( mca_sharedfp_sm_verbose ) {
|
||||
opal_output(ompi_sharedfp_base_framework.framework_output,
|
||||
"Aquiring lock, rank=%d...",rank);
|
||||
"Aquiring lock, rank=%d...",fh->f_rank);
|
||||
}
|
||||
|
||||
sm_offset_ptr = sm_data->sm_offset_ptr;
|
||||
@ -55,7 +58,7 @@ int mca_sharedfp_sm_request_position(struct mca_sharedfp_base_data_t * sh,
|
||||
|
||||
if ( mca_sharedfp_sm_verbose ) {
|
||||
opal_output(ompi_sharedfp_base_framework.framework_output,
|
||||
"Succeeded! Acquired sm lock.for rank=%d\n",rank);
|
||||
"Succeeded! Acquired sm lock.for rank=%d\n",fh->f_rank);
|
||||
}
|
||||
|
||||
old_offset=sm_offset_ptr->offset;
|
||||
@ -74,13 +77,13 @@ int mca_sharedfp_sm_request_position(struct mca_sharedfp_base_data_t * sh,
|
||||
|
||||
if ( mca_sharedfp_sm_verbose ) {
|
||||
opal_output(ompi_sharedfp_base_framework.framework_output,
|
||||
"Releasing sm lock...rank=%d",rank);
|
||||
"Releasing sm lock...rank=%d",fh->f_rank);
|
||||
}
|
||||
|
||||
sem_post(sm_data->mutex);
|
||||
if ( mca_sharedfp_sm_verbose ) {
|
||||
opal_output(ompi_sharedfp_base_framework.framework_output,
|
||||
"Released lock! released lock.for rank=%d\n",rank);
|
||||
"Released lock! released lock.for rank=%d\n",fh->f_rank);
|
||||
}
|
||||
|
||||
*offset = old_offset;
|
||||
|
||||
@ -9,7 +9,7 @@
|
||||
* University of Stuttgart. All rights reserved.
|
||||
* Copyright (c) 2004-2005 The Regents of the University of California.
|
||||
* All rights reserved.
|
||||
* Copyright (c) 2013-2016 University of Houston. All rights reserved.
|
||||
* Copyright (c) 2013-2018 University of Houston. All rights reserved.
|
||||
* Copyright (c) 2015 Cisco Systems, Inc. All rights reserved.
|
||||
* Copyright (c) 2018 Research Organization for Information Science
|
||||
* and Technology (RIST). All rights reserved.
|
||||
@ -36,36 +36,22 @@ int
|
||||
mca_sharedfp_sm_seek (ompio_file_t *fh,
|
||||
OMPI_MPI_OFFSET_TYPE offset, int whence)
|
||||
{
|
||||
int rank, status=0;
|
||||
int status=0;
|
||||
OMPI_MPI_OFFSET_TYPE end_position=0;
|
||||
int ret = OMPI_SUCCESS;
|
||||
struct mca_sharedfp_base_data_t *sh = NULL;
|
||||
mca_sharedfp_base_module_t * shared_fp_base_module = NULL;
|
||||
struct mca_sharedfp_sm_data * sm_data = NULL;
|
||||
struct mca_sharedfp_sm_offset * sm_offset_ptr = NULL;
|
||||
|
||||
if( NULL == fh->f_sharedfp_data ) {
|
||||
if ( mca_sharedfp_sm_verbose ) {
|
||||
opal_output(ompi_sharedfp_base_framework.framework_output,
|
||||
"sharedfp_sm_seek: opening the shared file pointer\n");
|
||||
}
|
||||
shared_fp_base_module = fh->f_sharedfp;
|
||||
|
||||
ret = shared_fp_base_module->sharedfp_file_open(fh->f_comm,
|
||||
fh->f_filename,
|
||||
fh->f_amode,
|
||||
fh->f_info,
|
||||
fh);
|
||||
if ( OMPI_SUCCESS != ret ) {
|
||||
opal_output(0,"sharedfp_sm_seek - error opening the shared file pointer\n");
|
||||
return ret;
|
||||
}
|
||||
"sharedfp_sm_seek: module not initialized \n");
|
||||
return OMPI_ERROR;
|
||||
}
|
||||
|
||||
sh = fh->f_sharedfp_data;
|
||||
rank = ompi_comm_rank ( sh->comm );
|
||||
|
||||
if( 0 == rank ){
|
||||
if( 0 == fh->f_rank ){
|
||||
if ( MPI_SEEK_SET == whence){
|
||||
/*no nothing*/
|
||||
if ( offset < 0){
|
||||
@ -97,7 +83,7 @@ mca_sharedfp_sm_seek (ompio_file_t *fh,
|
||||
}
|
||||
else if( MPI_SEEK_END == whence){
|
||||
end_position=0;
|
||||
mca_common_ompio_file_get_size(sh->sharedfh,&end_position);
|
||||
mca_common_ompio_file_get_size(fh,&end_position);
|
||||
|
||||
offset = end_position + offset;
|
||||
if ( mca_sharedfp_sm_verbose ) {
|
||||
@ -125,7 +111,7 @@ mca_sharedfp_sm_seek (ompio_file_t *fh,
|
||||
/*--------------------*/
|
||||
if ( mca_sharedfp_sm_verbose ) {
|
||||
opal_output(ompi_sharedfp_base_framework.framework_output,
|
||||
"sharedfp_sm_seek: Aquiring lock, rank=%d...",rank); fflush(stdout);
|
||||
"sharedfp_sm_seek: Aquiring lock, rank=%d...",fh->f_rank); fflush(stdout);
|
||||
}
|
||||
|
||||
/* Aquire an exclusive lock */
|
||||
@ -135,12 +121,12 @@ mca_sharedfp_sm_seek (ompio_file_t *fh,
|
||||
|
||||
if ( mca_sharedfp_sm_verbose ) {
|
||||
opal_output(ompi_sharedfp_base_framework.framework_output,
|
||||
"sharedfp_sm_seek: Success! Acquired sm lock.for rank=%d\n",rank);
|
||||
"sharedfp_sm_seek: Success! Acquired sm lock.for rank=%d\n",fh->f_rank);
|
||||
}
|
||||
sm_offset_ptr->offset=offset;
|
||||
if ( mca_sharedfp_sm_verbose ) {
|
||||
opal_output(ompi_sharedfp_base_framework.framework_output,
|
||||
"sharedfp_sm_seek: Releasing sm lock...rank=%d",rank); fflush(stdout);
|
||||
"sharedfp_sm_seek: Releasing sm lock...rank=%d",fh->f_rank); fflush(stdout);
|
||||
}
|
||||
sem_post(sm_data->mutex);
|
||||
}
|
||||
@ -148,7 +134,7 @@ mca_sharedfp_sm_seek (ompio_file_t *fh,
|
||||
/* since we are only letting process 0, update the current pointer
|
||||
* all of the other processes need to wait before proceeding.
|
||||
*/
|
||||
sh->comm->c_coll->coll_barrier ( sh->comm, sh->comm->c_coll->coll_barrier_module );
|
||||
fh->f_comm->c_coll->coll_barrier ( fh->f_comm, fh->f_comm->c_coll->coll_barrier_module );
|
||||
|
||||
return ret;
|
||||
}
|
||||
|
||||
@ -9,7 +9,7 @@
|
||||
* University of Stuttgart. All rights reserved.
|
||||
* Copyright (c) 2004-2005 The Regents of the University of California.
|
||||
* All rights reserved.
|
||||
* Copyright (c) 2013-2016 University of Houston. All rights reserved.
|
||||
* Copyright (c) 2013-2018 University of Houston. All rights reserved.
|
||||
* Copyright (c) 2015-2018 Research Organization for Information Science
|
||||
* and Technology (RIST). All rights reserved.
|
||||
* $COPYRIGHT$
|
||||
@ -38,25 +38,11 @@ int mca_sharedfp_sm_write (ompio_file_t *fh,
|
||||
OMPI_MPI_OFFSET_TYPE offset = 0;
|
||||
long bytesRequested = 0;
|
||||
size_t numofBytes;
|
||||
struct mca_sharedfp_base_data_t *sh = NULL;
|
||||
mca_sharedfp_base_module_t * shared_fp_base_module = NULL;
|
||||
|
||||
if( NULL == fh->f_sharedfp_data ){
|
||||
if ( mca_sharedfp_sm_verbose ) {
|
||||
opal_output(ompi_sharedfp_base_framework.framework_output,
|
||||
"sharedfp_sm_write: opening the shared file pointer\n");
|
||||
}
|
||||
shared_fp_base_module = fh->f_sharedfp;
|
||||
|
||||
ret = shared_fp_base_module->sharedfp_file_open(fh->f_comm,
|
||||
fh->f_filename,
|
||||
fh->f_amode,
|
||||
fh->f_info,
|
||||
fh);
|
||||
if ( OMPI_SUCCESS != ret ) {
|
||||
opal_output(0,"sharedfp_sm_write - error opening the shared file pointer\n");
|
||||
return ret;
|
||||
}
|
||||
"sharedfp_sm_write: module not initialized\n");
|
||||
return OMPI_ERROR;
|
||||
}
|
||||
|
||||
/* Calculate the number of bytes to write*/
|
||||
@ -64,7 +50,6 @@ int mca_sharedfp_sm_write (ompio_file_t *fh,
|
||||
bytesRequested = count * numofBytes;
|
||||
|
||||
/*Retrieve the shared file data struct*/
|
||||
sh = fh->f_sharedfp_data;
|
||||
|
||||
if ( mca_sharedfp_sm_verbose ) {
|
||||
opal_output(ompi_sharedfp_base_framework.framework_output,
|
||||
@ -72,8 +57,8 @@ int mca_sharedfp_sm_write (ompio_file_t *fh,
|
||||
}
|
||||
|
||||
/*Request the offset to write bytesRequested bytes*/
|
||||
ret = mca_sharedfp_sm_request_position(sh,bytesRequested,&offset);
|
||||
offset /= sh->sharedfh->f_etype_size;
|
||||
ret = mca_sharedfp_sm_request_position(fh, bytesRequested,&offset);
|
||||
offset /= fh->f_etype_size;
|
||||
if ( -1 != ret ) {
|
||||
if ( mca_sharedfp_sm_verbose ) {
|
||||
opal_output(ompi_sharedfp_base_framework.framework_output,
|
||||
@ -81,7 +66,7 @@ int mca_sharedfp_sm_write (ompio_file_t *fh,
|
||||
}
|
||||
|
||||
/* Write to the file*/
|
||||
ret = mca_common_ompio_file_write_at(sh->sharedfh,offset,buf,count,datatype,status);
|
||||
ret = mca_common_ompio_file_write_at(fh,offset,buf,count,datatype,status);
|
||||
}
|
||||
|
||||
return ret;
|
||||
@ -102,51 +87,27 @@ int mca_sharedfp_sm_write_ordered (ompio_file_t *fh,
|
||||
long bytesRequested = 0;
|
||||
int recvcnt = 1, sendcnt = 1;
|
||||
size_t numofBytes;
|
||||
int rank, size;
|
||||
int i;
|
||||
|
||||
struct mca_sharedfp_base_data_t *sh = NULL;
|
||||
mca_sharedfp_base_module_t * shared_fp_base_module = NULL;
|
||||
|
||||
if( NULL == fh->f_sharedfp_data){
|
||||
if ( mca_sharedfp_sm_verbose ) {
|
||||
opal_output(ompi_sharedfp_base_framework.framework_output,
|
||||
"sharedfp_sm_write_ordered: opening the shared file pointer\n");
|
||||
"sharedfp_sm_write_ordered: module not initialzed \n");
|
||||
return OMPI_ERROR;
|
||||
}
|
||||
shared_fp_base_module = fh->f_sharedfp;
|
||||
|
||||
ret = shared_fp_base_module->sharedfp_file_open(fh->f_comm,
|
||||
fh->f_filename,
|
||||
fh->f_amode,
|
||||
fh->f_info,
|
||||
fh);
|
||||
if ( OMPI_SUCCESS != ret ) {
|
||||
opal_output(0,"sharedfp_sm_write_ordered: error opening the shared file pointer\n");
|
||||
return ret;
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
/*Retrieve the new communicator*/
|
||||
sh = fh->f_sharedfp_data;
|
||||
|
||||
/* Calculate the number of bytes to write*/
|
||||
opal_datatype_type_size ( &datatype->super, &numofBytes);
|
||||
sendBuff = count * numofBytes;
|
||||
|
||||
/* Get the ranks in the communicator */
|
||||
rank = ompi_comm_rank ( sh->comm );
|
||||
size = ompi_comm_size ( sh->comm );
|
||||
|
||||
if ( 0 == rank ) {
|
||||
buff = (long*)malloc(sizeof(long) * size);
|
||||
if ( 0 == fh->f_rank ) {
|
||||
buff = (long*)malloc(sizeof(long) * fh->f_size);
|
||||
if ( NULL == buff )
|
||||
return OMPI_ERR_OUT_OF_RESOURCE;
|
||||
}
|
||||
|
||||
ret = sh->comm->c_coll->coll_gather ( &sendBuff, sendcnt, OMPI_OFFSET_DATATYPE,
|
||||
ret = fh->f_comm->c_coll->coll_gather ( &sendBuff, sendcnt, OMPI_OFFSET_DATATYPE,
|
||||
buff, recvcnt, OMPI_OFFSET_DATATYPE, 0,
|
||||
sh->comm, sh->comm->c_coll->coll_gather_module );
|
||||
fh->f_comm, fh->f_comm->c_coll->coll_gather_module );
|
||||
if ( OMPI_SUCCESS != ret ) {
|
||||
goto exit;
|
||||
}
|
||||
@ -154,8 +115,8 @@ int mca_sharedfp_sm_write_ordered (ompio_file_t *fh,
|
||||
/* All the counts are present now in the recvBuff.
|
||||
** The size of recvBuff is sizeof_newComm
|
||||
*/
|
||||
if ( 0 == rank ) {
|
||||
for (i = 0; i < size ; i ++) {
|
||||
if ( 0 == fh->f_rank ) {
|
||||
for (i = 0; i < fh->f_size ; i ++) {
|
||||
bytesRequested += buff[i];
|
||||
if ( mca_sharedfp_sm_verbose ) {
|
||||
opal_output(ompi_sharedfp_base_framework.framework_output,
|
||||
@ -169,7 +130,7 @@ int mca_sharedfp_sm_write_ordered (ompio_file_t *fh,
|
||||
** processes at what offset they should write their
|
||||
** share of the data.
|
||||
*/
|
||||
ret = mca_sharedfp_sm_request_position(sh,bytesRequested,&offsetReceived);
|
||||
ret = mca_sharedfp_sm_request_position(fh,bytesRequested,&offsetReceived);
|
||||
if( OMPI_SUCCESS != ret){
|
||||
goto exit;
|
||||
}
|
||||
@ -179,15 +140,15 @@ int mca_sharedfp_sm_write_ordered (ompio_file_t *fh,
|
||||
}
|
||||
buff[0] += offsetReceived;
|
||||
|
||||
for (i = 1 ; i < size; i++) {
|
||||
for (i = 1 ; i < fh->f_size; i++) {
|
||||
buff[i] += buff[i-1];
|
||||
}
|
||||
}
|
||||
|
||||
/* Scatter the results to the other processes*/
|
||||
ret = sh->comm->c_coll->coll_scatter ( buff, sendcnt, OMPI_OFFSET_DATATYPE,
|
||||
ret = fh->f_comm->c_coll->coll_scatter ( buff, sendcnt, OMPI_OFFSET_DATATYPE,
|
||||
&offsetBuff, recvcnt, OMPI_OFFSET_DATATYPE, 0,
|
||||
sh->comm, sh->comm->c_coll->coll_scatter_module );
|
||||
fh->f_comm, fh->f_comm->c_coll->coll_scatter_module );
|
||||
|
||||
if ( OMPI_SUCCESS != ret ) {
|
||||
goto exit;
|
||||
@ -195,14 +156,14 @@ int mca_sharedfp_sm_write_ordered (ompio_file_t *fh,
|
||||
|
||||
/* Each process now has its own individual offset */
|
||||
offset = offsetBuff - sendBuff;
|
||||
offset /= sh->sharedfh->f_etype_size;
|
||||
offset /= fh->f_etype_size;
|
||||
|
||||
if ( mca_sharedfp_sm_verbose ) {
|
||||
opal_output(ompi_sharedfp_base_framework.framework_output,
|
||||
"sharedfp_sm_write_ordered: Offset returned is %lld\n",offset);
|
||||
}
|
||||
/* write to the file */
|
||||
ret = mca_common_ompio_file_write_at_all(sh->sharedfh,offset,buf,count,datatype,status);
|
||||
ret = mca_common_ompio_file_write_at_all(fh,offset,buf,count,datatype,status);
|
||||
|
||||
exit:
|
||||
if ( NULL != buff ) {
|
||||
|
||||
Загрузка…
x
Ссылка в новой задаче
Block a user