diff --git a/ompi/mca/sharedfp/addproc/Makefile.am b/ompi/mca/sharedfp/addproc/Makefile.am new file mode 100644 index 0000000000..182ef1a372 --- /dev/null +++ b/ompi/mca/sharedfp/addproc/Makefile.am @@ -0,0 +1,63 @@ +# +# Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana +# University Research and Technology +# Corporation. All rights reserved. +# Copyright (c) 2004-2005 The University of Tennessee and The University +# of Tennessee Research Foundation. All rights +# reserved. +# Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, +# University of Stuttgart. All rights reserved. +# Copyright (c) 2004-2005 The Regents of the University of California. +# All rights reserved. +# Copyright (c) 2013 University of Houston. All rights reserved. +# $COPYRIGHT$ +# +# Additional copyrights may follow +# +# $HEADER$ +# + +# Make the output library in this directory, and name it either +# mca__.la (for DSO builds) or libmca__.la +# (for static builds). + +if MCA_BUILD_ompi_sharedfp_addproc_DSO +component_noinst = +component_install = mca_sharedfp_addproc.la +else +component_noinst = libmca_sharedfp_addproc.la +component_install = +endif + +mcacomponentdir = $(pkglibdir) +mcacomponent_LTLIBRARIES = $(component_install) +mca_sharedfp_addproc_la_SOURCES = $(sources) +mca_sharedfp_addproc_la_LDFLAGS = -module -avoid-version + +noinst_LTLIBRARIES = $(component_noinst) +libmca_sharedfp_addproc_la_SOURCES = $(sources) +libmca_sharedfp_addproc_la_LDFLAGS = -module -avoid-version + +# Source files + +#IMPORTANT: Update here when adding new source code files to the library +sources = \ + sharedfp_addproc.h \ + sharedfp_addproc.c \ + sharedfp_addproc_component.c \ + sharedfp_addproc_seek.c \ + sharedfp_addproc_request_position.c \ + sharedfp_addproc_write.c \ + sharedfp_addproc_iwrite.c \ + sharedfp_addproc_read.c \ + sharedfp_addproc_iread.c \ + sharedfp_addproc_file_open.c + +#The additional process is spawned by executing this executable +bin_PROGRAMS = mca_sharedfp_addproc_control + +mca_sharedfp_addproc_control_SOURCES = \ + sharedfp_addproc_control.h \ + sharedfp_addproc_control.c + +mca_sharedfp_addproc_control_LDADD = $(top_builddir)/ompi/libmpi.la diff --git a/ompi/mca/sharedfp/dummy/sharedfp_dummy.c b/ompi/mca/sharedfp/addproc/sharedfp_addproc.c similarity index 59% rename from ompi/mca/sharedfp/dummy/sharedfp_dummy.c rename to ompi/mca/sharedfp/addproc/sharedfp_addproc.c index 157f511194..3e528dd003 100644 --- a/ompi/mca/sharedfp/dummy/sharedfp_dummy.c +++ b/ompi/mca/sharedfp/addproc/sharedfp_addproc.c @@ -9,7 +9,7 @@ * University of Stuttgart. All rights reserved. * Copyright (c) 2004-2005 The Regents of the University of California. * All rights reserved. - * Copyright (c) 2008-2011 University of Houston. All rights reserved. + * Copyright (c) 2013 University of Houston. All rights reserved. * $COPYRIGHT$ * * Additional copyrights may follow @@ -26,18 +26,31 @@ #include "ompi_config.h" #include "mpi.h" #include "ompi/mca/sharedfp/sharedfp.h" -#include "ompi/mca/sharedfp/dummy/sharedfp_dummy.h" +#include "ompi/mca/sharedfp/addproc/sharedfp_addproc.h" /* * ******************************************************************* * ************************ actions structure ************************ * ******************************************************************* */ -static mca_sharedfp_base_module_1_0_0_t dummy = { - mca_sharedfp_dummy_module_init, /* initalise after being selected */ - mca_sharedfp_dummy_module_finalize, /* close a module on a communicator */ - mca_sharedfp_dummy_update, - mca_sharedfp_dummy_seek + /* IMPORTANT: Update here when adding sharedfp component interface functions*/ +static mca_sharedfp_base_module_1_0_0_t addproc = { + mca_sharedfp_addproc_module_init, /* initalise after being selected */ + mca_sharedfp_addproc_module_finalize, /* close a module on a communicator */ + mca_sharedfp_addproc_seek, + mca_sharedfp_addproc_get_position, + mca_sharedfp_addproc_read, + mca_sharedfp_addproc_read_ordered, + mca_sharedfp_addproc_read_ordered_begin, + mca_sharedfp_addproc_read_ordered_end, + mca_sharedfp_addproc_iread, + mca_sharedfp_addproc_write, + mca_sharedfp_addproc_write_ordered, + mca_sharedfp_addproc_write_ordered_begin, + mca_sharedfp_addproc_write_ordered_end, + mca_sharedfp_addproc_iwrite, + mca_sharedfp_addproc_file_open, + mca_sharedfp_addproc_file_close }; /* * ******************************************************************* @@ -45,7 +58,7 @@ static mca_sharedfp_base_module_1_0_0_t dummy = { * ******************************************************************* */ -int mca_sharedfp_dummy_component_init_query(bool enable_progress_threads, +int mca_sharedfp_addproc_component_init_query(bool enable_progress_threads, bool enable_mpi_threads) { /* Nothing to do */ @@ -54,14 +67,16 @@ int mca_sharedfp_dummy_component_init_query(bool enable_progress_threads, } struct mca_sharedfp_base_module_1_0_0_t * -mca_sharedfp_dummy_component_file_query (int *priority) -{ - *priority = 20; + mca_sharedfp_addproc_component_file_query + (mca_io_ompio_file_t *fh, int *priority) { + *priority = mca_sharedfp_addproc_priority; - return &dummy; + /*test, and update priority*/ + + return &addproc; } -int mca_sharedfp_dummy_component_file_unquery (mca_io_ompio_file_t *file) +int mca_sharedfp_addproc_component_file_unquery (mca_io_ompio_file_t *file) { /* This function might be needed for some purposes later. for now it * does not have anything to do since there are no steps which need @@ -70,13 +85,13 @@ int mca_sharedfp_dummy_component_file_unquery (mca_io_ompio_file_t *file) return OMPI_SUCCESS; } -int mca_sharedfp_dummy_module_init (mca_io_ompio_file_t *file) +int mca_sharedfp_addproc_module_init (mca_io_ompio_file_t *file) { return OMPI_SUCCESS; } -int mca_sharedfp_dummy_module_finalize (mca_io_ompio_file_t *file) +int mca_sharedfp_addproc_module_finalize (mca_io_ompio_file_t *file) { return OMPI_SUCCESS; } diff --git a/ompi/mca/sharedfp/addproc/sharedfp_addproc.h b/ompi/mca/sharedfp/addproc/sharedfp_addproc.h new file mode 100644 index 0000000000..a7fb4dd9b3 --- /dev/null +++ b/ompi/mca/sharedfp/addproc/sharedfp_addproc.h @@ -0,0 +1,164 @@ +/* + * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana + * University Research and Technology + * Corporation. All rights reserved. + * Copyright (c) 2004-2006 The University of Tennessee and The University + * of Tennessee Research Foundation. All rights + * reserved. + * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, + * University of Stuttgart. All rights reserved. + * Copyright (c) 2004-2005 The Regents of the University of California. + * All rights reserved. + * Copyright (c) 2013 University of Houston. All rights reserved. + * $COPYRIGHT$ + * + * Additional copyrights may follow + * + * $HEADER$ + */ + +#ifndef MCA_SHAREDFP_ADDPROC_H +#define MCA_SHAREDFP_ADDPROC_H + +#include "ompi_config.h" +#include "opal/mca/mca.h" +#include "ompi/mca/sharedfp/sharedfp.h" +#include "ompi/mca/io/ompio/io_ompio.h" +#include + +BEGIN_C_DECLS + +int mca_sharedfp_addproc_component_init_query(bool enable_progress_threads, + bool enable_mpi_threads); +struct mca_sharedfp_base_module_1_0_0_t * + mca_sharedfp_addproc_component_file_query (mca_io_ompio_file_t *file, int *priority); +int mca_sharedfp_addproc_component_file_unquery (mca_io_ompio_file_t *file); + +int mca_sharedfp_addproc_module_init (mca_io_ompio_file_t *file); +int mca_sharedfp_addproc_module_finalize (mca_io_ompio_file_t *file); + +extern int mca_sharedfp_addproc_priority; +extern int mca_sharedfp_addproc_verbose; +#if 0 +extern char[MPI_MAX_HOSTNAME_LEN] mca_sharedfp_addproc_control_host; +#endif + +OMPI_MODULE_DECLSPEC extern mca_sharedfp_base_component_2_0_0_t mca_sharedfp_addproc_component; +/* + * ****************************************************************** + * ********* functions which are implemented in this module ********* + * ****************************************************************** + */ +/*IMPORANT: Update here when implementing functions from sharedfp API*/ + +int mca_sharedfp_addproc_seek (mca_io_ompio_file_t *fh, + OMPI_MPI_OFFSET_TYPE offset, int whence); +int mca_sharedfp_addproc_get_position (mca_io_ompio_file_t *fh, + OMPI_MPI_OFFSET_TYPE * offset); +int mca_sharedfp_addproc_file_open (struct ompi_communicator_t *comm, + char* filename, + int amode, + struct ompi_info_t *info, + mca_io_ompio_file_t *fh); +int mca_sharedfp_addproc_file_close (mca_io_ompio_file_t *fh); +int mca_sharedfp_addproc_read (mca_io_ompio_file_t *fh, + void *buf, int count, MPI_Datatype datatype, MPI_Status *status); +int mca_sharedfp_addproc_read_ordered (mca_io_ompio_file_t *fh, + void *buf, int count, struct ompi_datatype_t *datatype, + ompi_status_public_t *status + ); +int mca_sharedfp_addproc_read_ordered_begin (mca_io_ompio_file_t *fh, + void *buf, + int count, + struct ompi_datatype_t *datatype); +int mca_sharedfp_addproc_read_ordered_end (mca_io_ompio_file_t *fh, + void *buf, + ompi_status_public_t *status); +int mca_sharedfp_addproc_iread (mca_io_ompio_file_t *fh, + void *buf, + int count, + struct ompi_datatype_t *datatype, + ompi_request_t **request); +int mca_sharedfp_addproc_write (mca_io_ompio_file_t *fh, + void *buf, + int count, + struct ompi_datatype_t *datatype, + ompi_status_public_t *status); +int mca_sharedfp_addproc_write_ordered (mca_io_ompio_file_t *fh, + void *buf, + int count, + struct ompi_datatype_t *datatype, + ompi_status_public_t *status); +int mca_sharedfp_addproc_write_ordered_begin (mca_io_ompio_file_t *fh, + void *buf, + int count, + struct ompi_datatype_t *datatype); +int mca_sharedfp_addproc_write_ordered_end (mca_io_ompio_file_t *fh, + void *buf, + ompi_status_public_t *status); +int mca_sharedfp_addproc_iwrite (mca_io_ompio_file_t *fh, + void *buf, + int count, + struct ompi_datatype_t *datatype, + ompi_request_t **request); +/****************************************************/ +/*The following are structures and definitions * + * copied over directly from uhio codebase */ +/****************************************************/ + +/*This structure will hang off of the mca_sharedfp_base_data_t's + *selected_module_data attribute + */ +struct mca_sharedfp_addproc_data +{ + MPI_Comm intercom; +}; + +typedef struct mca_sharedfp_addproc_data addproc_data; + + +int mca_sharedfp_addproc_request_position (struct mca_sharedfp_base_data_t * sh, + int bytes_requested, + OMPI_MPI_OFFSET_TYPE * offset); + +#define DO_ACK 0 /* To be set by the Environment Variable*/ +#define REQUEST_TAG 99 +#define ACK_TAG 1 +#define OFFSET_TAG 98 +#define END_TAG 97 + +#define SEEK_END_TAG 91 +#define SEEK_SET_TAG 92 +#define SEEK_CUR_TAG 93 +#define GET_POSITION_TAG 94 + +#define NUM_OF_SPAWNS 1 + +struct list { + + int procNo; + long numBytesArrAddr; + struct list *Next; +}; + +struct Stat { + int tag; + int source; + long* recvBuff; +}; + + +double uhio_shared_gettime(void); + +typedef struct list node; +typedef struct Stat statusStruct; + +/* + * ****************************************************************** + * ************ functions implemented in this module end ************ + * ****************************************************************** + */ + +END_C_DECLS + +#endif /* MCA_SHAREDFP_ADDPROC_H */ diff --git a/ompi/mca/sharedfp/addproc/sharedfp_addproc_component.c b/ompi/mca/sharedfp/addproc/sharedfp_addproc_component.c new file mode 100644 index 0000000000..c02e20c150 --- /dev/null +++ b/ompi/mca/sharedfp/addproc/sharedfp_addproc_component.c @@ -0,0 +1,103 @@ +/* + * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana + * University Research and Technology + * Corporation. All rights reserved. + * Copyright (c) 2004-2005 The University of Tennessee and The University + * of Tennessee Research Foundation. All rights + * reserved. + * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, + * University of Stuttgart. All rights reserved. + * Copyright (c) 2004-2005 The Regents of the University of California. + * All rights reserved. + * Copyright (c) 2013 University of Houston. All rights reserved. + * $COPYRIGHT$ + * + * Additional copyrights may follow + * + * $HEADER$ + * + * These symbols are in a file by themselves to provide nice linker + * semantics. Since linkers generally pull in symbols by object + * files, keeping these symbols as the only symbols in this file + * prevents utility programs such as "ompi_info" from having to import + * entire components just to query their version and parameters. + */ + +#include "ompi_config.h" +#include "sharedfp_addproc.h" +#include "mpi.h" + +/* + * Public string showing the sharedfp addproc component version number + */ +const char *mca_sharedfp_addproc_component_version_string = + "OMPI/MPI addproc SHAREDFP MCA component version " OMPI_VERSION; + +/* + * Global variables + */ +int mca_sharedfp_addproc_priority=1; +int mca_sharedfp_addproc_verbose=0; +#if 0 +char[MPI_MAX_HOSTNAME_LEN] mca_sharedfp_addproc_control_host; +#endif + +static int addproc_register(void); + +/* + * Instantiate the public struct with all of our public information + * and pointers to our public functions in it + */ +mca_sharedfp_base_component_2_0_0_t mca_sharedfp_addproc_component = { + + /* First, the mca_component_t struct containing meta information + about the component itself */ + + { + MCA_SHAREDFP_BASE_VERSION_2_0_0, + + /* Component name and version */ + "addproc", + OMPI_MAJOR_VERSION, + OMPI_MINOR_VERSION, + OMPI_RELEASE_VERSION, + addproc_register, + NULL + }, + { + /* This component is checkpointable */ + MCA_BASE_METADATA_PARAM_CHECKPOINT + }, + mca_sharedfp_addproc_component_init_query, /* get thread level */ + mca_sharedfp_addproc_component_file_query, /* get priority and actions */ + mca_sharedfp_addproc_component_file_unquery /* undo what was done by previous function */ +}; + + +static int addproc_register(void) +{ + mca_sharedfp_addproc_priority = 1; + (void) mca_base_component_var_register(&mca_sharedfp_addproc_component.sharedfpm_version, + "priority", "Priority of the addproc sharedfp component", + MCA_BASE_VAR_TYPE_INT, NULL, 0, 0, + OPAL_INFO_LVL_9, + MCA_BASE_VAR_SCOPE_READONLY, &mca_sharedfp_addproc_priority); + mca_sharedfp_addproc_verbose = 0; + (void) mca_base_component_var_register(&mca_sharedfp_addproc_component.sharedfpm_version, + "verbose", "Verbosity of the addproc sharedfp component", + MCA_BASE_VAR_TYPE_INT, NULL, 0, 0, + OPAL_INFO_LVL_9, + MCA_BASE_VAR_SCOPE_READONLY, &mca_sharedfp_addproc_verbose); + + +#if 0 + memset (mca_sharedfp_addproc_control_host, 0, MPI_MAX_HOSTNAME_LEN); + (void) mca_base_component_var_register(&mca_sharedfp_addproc_component.sharedfpm_version, + "control_host", "Name of the host where to spawn the control process(default:none)", + MCA_BASE_VAR_TYPE_STRING, NULL, 0, 0, + OPAL_INFO_LVL_9, + MCA_BASE_VAR_SCOPE_READONLY, &mca_sharedfp_addproc_control_host); + +#endif + return OMPI_SUCCESS; +} diff --git a/ompi/mca/sharedfp/addproc/sharedfp_addproc_control.c b/ompi/mca/sharedfp/addproc/sharedfp_addproc_control.c new file mode 100644 index 0000000000..3b6b096b53 --- /dev/null +++ b/ompi/mca/sharedfp/addproc/sharedfp_addproc_control.c @@ -0,0 +1,231 @@ +/* + * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana + * University Research and Technology + * Corporation. All rights reserved. + * Copyright (c) 2004-2005 The University of Tennessee and The University + * of Tennessee Research Foundation. All rights + * reserved. + * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, + * University of Stuttgart. All rights reserved. + * Copyright (c) 2004-2005 The Regents of the University of California. + * All rights reserved. + * Copyright (c) 2013 University of Houston. All rights reserved. + * $COPYRIGHT$ + * + * Additional copyrights may follow + * + * $HEADER$ + */ + +#include "sharedfp_addproc_control.h" + +/* #define PRINT_TAG 1 */ +void nodeDelete(node **front, node **rear) +{ + node *delNode; + if ((*front) == NULL && (*rear)==NULL) { + opal_output(1,"The queue is empty\n"); + } + else { + delNode = *front; + if (*front == *rear) { + *rear = NULL; + } + (*front) = (*front)->Next; + + free(delNode); + } + + return; +} + +void nodeInsert(node **front, node **rear, int procNo, long numBytesArrAddr) +{ + node *newNode; + newNode = (node*)malloc(sizeof(node)); + + newNode->Next = NULL; + newNode->procNo = procNo; + newNode->numBytesArrAddr = numBytesArrAddr; + + + if ((*front == NULL) && (*rear == NULL)) { + *front = newNode; + *rear = newNode; +#if 0 + opal_output(1,"Front and rear both NULL\n"); +#endif + fflush(stdout); + } + else { + (*rear)->Next = newNode; + *rear=newNode; +#if 0 + opal_output(1,"Front and rear both not NULL\n"); +#endif + fflush(stdout); + } + + return; +} + +int Check_Request_Offset(int tag_received) +{ +#if 0 + opal_output(1,"Tag received %d\n",tag_received); +#endif + + if (tag_received == REQUEST_TAG) { +#if 0 + opal_output(1,"Return from Check_Request_Offset\n"); +#endif + return 1; + } + + + return 0; +} + +int Check_Acknowledgement(int tag_received) +{ + if (tag_received == ACK_TAG) + return 1; + + return 0; +} + +int End_control_shared_request(int tag_received) +{ + if (tag_received == END_TAG) + return 1; + + + return 0; +} + + +int main(int argc, char **argv) +{ + long recvBuff; + long offsetValue; + long endoffile; + int size; + int tag_received; + int END_FLAG = 0; + + int recvcount = 1; + MPI_Status status; + MPI_Comm parentComm; + static MPI_Offset offset = 0; + + /*statusStruct arr;*/ + + node *rear, *front; + rear = front = NULL; + +#if 0 + opal_output(1,"addproc_control: MPI_INIT\n"); fflush(stdout); +#endif + MPI_Init(&argc,&argv); + +#if 0 + opal_output(1,"addproc_control: MPI_Comm_size\n"); fflush(stdout); +#endif + MPI_Comm_size(MPI_COMM_WORLD,&size); + + + endoffile = 0; + +#if 0 + opal_output(1,"addproc_control: start listening\n"); fflush(stdout); +#endif + while(!END_FLAG) { + + /* Receive request from other processes */ + MPI_Comm_get_parent(&parentComm); + + MPI_Recv(&recvBuff,recvcount,OMPI_OFFSET_DATATYPE,MPI_ANY_SOURCE,MPI_ANY_TAG,parentComm,&status); + tag_received = status.MPI_TAG; + + switch (tag_received) + { + + case REQUEST_TAG: +#if 0 + opal_output(1,"addproc_control: Offset requested by the process %d\n",status.MPI_SOURCE); fflush(stdout); +#endif + /* Insert the node into the linked list */ + nodeInsert(&front,&rear,status.MPI_SOURCE,recvBuff); + break; + case END_TAG: +#if 0 + opal_output(1,"addproc_control: End Control tag received\n"); fflush(stdout); +#endif + END_FLAG = 1; + break; + case SEEK_SET_TAG: + offset = recvBuff; + MPI_Send(&offset,1,OMPI_OFFSET_DATATYPE,status.MPI_SOURCE,SEEK_SET_TAG,parentComm); +#if 0 + opal_output(1,"addproc_control: Seek set tag received\n"); fflush(stdout); +#endif + break; + case SEEK_CUR_TAG: +#if 0 + opal_output(1,"addproc_control: Seek CUR Tag received\n"); fflush(stdout); +#endif + /*set the pointer to the offset*/ + offset += recvBuff; + MPI_Send(&offset,1,OMPI_OFFSET_DATATYPE,status.MPI_SOURCE,SEEK_CUR_TAG,parentComm); + break; + case SEEK_END_TAG: +#if 0 + opal_output(1,"addproc_control: Seek END TAG received\n"); fflush(stdout); +#endif + offset = endoffile; + offset += recvBuff; + MPI_Send(&offset,1,OMPI_OFFSET_DATATYPE,status.MPI_SOURCE,SEEK_END_TAG,parentComm); + break; + case GET_POSITION_TAG: +#if 0 + opal_output(1,"\naddproc_control: Get Position tag received\n"); fflush(stdout); +#endif + /*Send the offset as requested*/ + MPI_Send(&offset,1,OMPI_OFFSET_DATATYPE,status.MPI_SOURCE,GET_POSITION_TAG,parentComm); + break; + default: + opal_output(1,"addproc_control: Unknown tag received\n"); fflush(stdout); + break; + } + + while (front != NULL) { + + offsetValue = offset; + + offset += front->numBytesArrAddr; + + /* Store the end of file */ + if (endoffile < offset) + endoffile = offset; + + + /* MPI_Send to the correct process */ + + MPI_Send(&offsetValue,1,OMPI_OFFSET_DATATYPE, front->procNo, OFFSET_TAG, + parentComm); + nodeDelete(&front,&rear); + + } + + } /* End of while(1) loop */ + +#if 0 + opal_output(1,"addproc_control: finalizing mpi...\n"); fflush(stdout); +#endif + MPI_Finalize(); + +#if 0 + opal_output(1,"addproc_control: Exiting...\n"); +#endif + return 0; +} diff --git a/ompi/mca/sharedfp/addproc/sharedfp_addproc_control.h b/ompi/mca/sharedfp/addproc/sharedfp_addproc_control.h new file mode 100644 index 0000000000..40072e57f4 --- /dev/null +++ b/ompi/mca/sharedfp/addproc/sharedfp_addproc_control.h @@ -0,0 +1,37 @@ +/* + * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana + * University Research and Technology + * Corporation. All rights reserved. + * Copyright (c) 2004-2006 The University of Tennessee and The University + * of Tennessee Research Foundation. All rights + * reserved. + * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, + * University of Stuttgart. All rights reserved. + * Copyright (c) 2004-2005 The Regents of the University of California. + * All rights reserved. + * Copyright (c) 2008 University of Houston. All rights reserved. + * $COPYRIGHT$ + * + * Additional copyrights may follow + * + * $HEADER$ + */ + +#ifndef MCA_SHAREDFP_addproc_control_H +#define MCA_SHAREDFP_addproc_control_H + +#include +#include "mpi.h" +#include "sharedfp_addproc.h" + +BEGIN_C_DECLS + +void nodeDelete(node **front, node **rear); +void nodeInsert(node **front, node **rear, int procNo, long numBytesArrAddr); +int Check_Request_Offset(int tag_received); +int Check_Acknowledgement(int tag_received); +int End_control_shared_request(int tag_received); + +END_C_DECLS + +#endif /* MCA_SHAREDFP_addproc_control_H */ diff --git a/ompi/mca/sharedfp/addproc/sharedfp_addproc_file_open.c b/ompi/mca/sharedfp/addproc/sharedfp_addproc_file_open.c new file mode 100644 index 0000000000..3370b6ac57 --- /dev/null +++ b/ompi/mca/sharedfp/addproc/sharedfp_addproc_file_open.c @@ -0,0 +1,159 @@ +/* + * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana + * University Research and Technology + * Corporation. All rights reserved. + * Copyright (c) 2004-2005 The University of Tennessee and The University + * of Tennessee Research Foundation. All rights + * reserved. + * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, + * University of Stuttgart. All rights reserved. + * Copyright (c) 2004-2005 The Regents of the University of California. + * All rights reserved. + * Copyright (c) 2013 University of Houston. All rights reserved. + * $COPYRIGHT$ + * + * Additional copyrights may follow + * + * $HEADER$ + */ + + +#include "ompi_config.h" +#include "sharedfp_addproc.h" + +#include "mpi.h" +#include "ompi/constants.h" +#include "orte/util/show_help.h" +#include "ompi/mca/sharedfp/sharedfp.h" +#include "ompi/mca/pml/pml.h" + +#include +#include +#include "ompi/mca/sharedfp/base/base.h" + +int mca_sharedfp_addproc_file_open (struct ompi_communicator_t *comm, + char* filename, + int amode, + struct ompi_info_t *info, + mca_io_ompio_file_t *fh) +{ + int ret = OMPI_SUCCESS, err; + int rank; + struct mca_sharedfp_base_data_t* sh; + mca_io_ompio_file_t * shfileHandle; + MPI_Comm newInterComm; + struct mca_sharedfp_addproc_data * addproc_data = NULL; + + /*-------------------------------------------------*/ + /*Open the same file again without shared file pointer*/ + /*-------------------------------------------------*/ + shfileHandle = (mca_io_ompio_file_t *)malloc(sizeof(mca_io_ompio_file_t)); + ret = ompio_io_ompio_file_open(comm,filename,amode,info,shfileHandle,false); + if ( OMPI_SUCCESS != ret) { + printf( "mca_sharedfp_addproc_file_open: Error during file open\n"); + return ret; + } + + /*Memory is allocated here for the sh structure*/ + if ( mca_sharedfp_addproc_verbose ) { + printf( "mca_sharedfp_addproc_file_open: malloc f_sharedfp_ptr struct\n"); + } + sh = (struct mca_sharedfp_base_data_t*)malloc(sizeof(struct mca_sharedfp_base_data_t)); + if ( NULL == sh ){ + printf( "mca_sharedfp_addproc_file_open: Error, unable to malloc f_sharedfp_ptr struct\n"); + return OMPI_ERR_OUT_OF_RESOURCE; + } + + /*Populate the sh file structure based on the implementation*/ + sh->sharedfh = shfileHandle; /* Shared file pointer*/ + sh->global_offset = 0; /* Global Offset*/ + sh->comm = comm; /* Communicator*/ + sh->selected_module_data = NULL; + + rank = ompi_comm_rank ( sh->comm ); + + if ( mca_sharedfp_addproc_verbose ) { + printf( "mca_sharedfp_addproc_file_open: START spawn by rank=%d\n",rank); + } + + /*Spawn a new process which will maintain the offsets for this file open*/ + ret = MPI_Comm_spawn("mca_sharedfp_addproc_control", MPI_ARGV_NULL, 1, MPI_INFO_NULL, + 0, sh->comm, &newInterComm, &err); + if ( OMPI_SUCCESS != ret ) { + printf( "mca_sharedfp_addproc_file_open: error spawning control process ret=%d\n", + ret); + } + + /*If spawning successful*/ + if (newInterComm) { + addproc_data = (struct mca_sharedfp_addproc_data*)malloc(sizeof(struct mca_sharedfp_addproc_data)); + if ( NULL == addproc_data ){ + printf( "mca_sharedfp_addproc_file_open: Error, unable to malloc addproc_data struct\n"); + return OMPI_ERR_OUT_OF_RESOURCE; + } + + /*Store the new Intercommunicator*/ + addproc_data->intercom = newInterComm; + + /*save the addproc data*/ + sh->selected_module_data = addproc_data; + /*remember the shared file handle*/ + fh->f_sharedfp_data = sh; + } + else{ + printf( "mca_sharedfp_addproc_file_open: DONE spawn by rank=%d, errcode[success=%d, err=%d]=%d\n", + rank, MPI_SUCCESS, MPI_ERR_SPAWN, ret); + ret = OMPI_ERROR; + } + + return ret; +} + +int mca_sharedfp_addproc_file_close (mca_io_ompio_file_t *fh) +{ + struct mca_sharedfp_base_data_t *sh=NULL; + int err = OMPI_SUCCESS; + long sendBuff = 0; + int count = 1; + int rank; + struct mca_sharedfp_addproc_data * addproc_data = NULL; + + if ( NULL == fh->f_sharedfp_data){ + /* Can happen with lazy initialization of the sharedfp structures */ + if ( mca_sharedfp_addproc_verbose ) { + printf( "sharedfp_addproc_file_close - shared file pointer structure not initialized\n"); + } + return OMPI_SUCCESS; + } + sh = fh->f_sharedfp_data; + + rank = ompi_comm_rank ( sh->comm ); + + /* Make sure that all processes are ready to release the + ** shared file pointer resources + */ + sh->comm->c_coll.coll_barrier(sh->comm, sh->comm->c_coll.coll_barrier_module ); + + addproc_data = (struct mca_sharedfp_addproc_data*)(sh->selected_module_data); + + if (addproc_data) { + /*tell additional proc to stop listening*/ + if(0 == rank){ + MCA_PML_CALL(send( &sendBuff, count, OMPI_OFFSET_DATATYPE, 0, END_TAG, + MCA_PML_BASE_SEND_STANDARD, addproc_data->intercom)); + } + + /* Free intercommunicator */ + if(addproc_data->intercom){ + ompi_comm_free(&(addproc_data->intercom)); + } + free(addproc_data); + } + + /* Close the main file opened by this component*/ + err = ompio_io_ompio_file_close(sh->sharedfh); + + /*free shared file pointer data struct*/ + free(sh); + return err; +} diff --git a/ompi/mca/sharedfp/addproc/sharedfp_addproc_iread.c b/ompi/mca/sharedfp/addproc/sharedfp_addproc_iread.c new file mode 100644 index 0000000000..6c6f00e209 --- /dev/null +++ b/ompi/mca/sharedfp/addproc/sharedfp_addproc_iread.c @@ -0,0 +1,86 @@ +/* + * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana + * University Research and Technology + * Corporation. All rights reserved. + * Copyright (c) 2004-2005 The University of Tennessee and The University + * of Tennessee Research Foundation. All rights + * reserved. + * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, + * University of Stuttgart. All rights reserved. + * Copyright (c) 2004-2005 The Regents of the University of California. + * All rights reserved. + * Copyright (c) 2013 University of Houston. All rights reserved. + * $COPYRIGHT$ + * + * Additional copyrights may follow + * + * $HEADER$ + */ + + +#include "ompi_config.h" +#include "sharedfp_addproc.h" + +#include "mpi.h" +#include "ompi/constants.h" +#include "orte/util/show_help.h" +#include "ompi/mca/sharedfp/sharedfp.h" + +int mca_sharedfp_addproc_iread(mca_io_ompio_file_t *fh, + void *buf, + int count, + ompi_datatype_t *datatype, + MPI_Request * request) +{ + int ret = OMPI_SUCCESS; + OMPI_MPI_OFFSET_TYPE offset = 0; + long bytesRequested = 0; + size_t numofBytes; + struct mca_sharedfp_base_data_t *sh = NULL; + + if(NULL == fh->f_sharedfp_data){ + opal_output(1, "sharedfp_addproc_iread - shared file pointer structure not initialized correctly\n"); + return OMPI_ERROR; + } + + /* Calculate the number of bytes to write */ + opal_datatype_type_size ( &datatype->super ,&numofBytes); + bytesRequested = count * numofBytes; + + if ( mca_sharedfp_addproc_verbose ){ + printf("mca_sharedfp_addproc_iread: Bytes Requested is %ld\n",bytesRequested); + } + /* Retrieve the shared file data struct */ + sh = fh->f_sharedfp_data; + + /*Request to the additional process for the offset*/ + ret = mca_sharedfp_addproc_request_position(sh,bytesRequested,&offset); + if( OMPI_SUCCESS == ret ){ + if ( mca_sharedfp_addproc_verbose ){ + printf("mca_sharedfp_addproc_iread: Offset received is %lld\n",offset); + } + /* Read from the file */ + ret = ompio_io_ompio_file_iread_at ( sh->sharedfh, offset, buf, count, datatype, request); + } + + return ret; +} +int mca_sharedfp_addproc_read_ordered_begin(mca_io_ompio_file_t *fh, + void *buf, + int count, + struct ompi_datatype_t *datatype) +{ + opal_output(1,"mca_sharedfp_addproc_read_ordered_begin: NOT IMPLEMENTED\n"); + return OMPI_ERROR; + +} + + +int mca_sharedfp_addproc_read_ordered_end(mca_io_ompio_file_t *fh, + void *buf, + ompi_status_public_t *status) +{ + opal_output(1,"mca_sharedfp_addproc_read_ordered_end: NOT IMPLEMENTED\n"); + return OMPI_ERROR; + +} diff --git a/ompi/mca/sharedfp/addproc/sharedfp_addproc_iwrite.c b/ompi/mca/sharedfp/addproc/sharedfp_addproc_iwrite.c new file mode 100644 index 0000000000..c1058a5641 --- /dev/null +++ b/ompi/mca/sharedfp/addproc/sharedfp_addproc_iwrite.c @@ -0,0 +1,85 @@ +/* + * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana + * University Research and Technology + * Corporation. All rights reserved. + * Copyright (c) 2004-2005 The University of Tennessee and The University + * of Tennessee Research Foundation. All rights + * reserved. + * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, + * University of Stuttgart. All rights reserved. + * Copyright (c) 2004-2005 The Regents of the University of California. + * All rights reserved. + * Copyright (c) 2013 University of Houston. All rights reserved. + * $COPYRIGHT$ + * + * Additional copyrights may follow + * + * $HEADER$ + */ + + +#include "ompi_config.h" +#include "sharedfp_addproc.h" + +#include "mpi.h" +#include "ompi/constants.h" +#include "orte/util/show_help.h" +#include "ompi/mca/sharedfp/sharedfp.h" + +int mca_sharedfp_addproc_iwrite(mca_io_ompio_file_t *fh, + void *buf, + int count, + ompi_datatype_t *datatype, + MPI_Request * request) +{ + int ret = OMPI_SUCCESS; + OMPI_MPI_OFFSET_TYPE offset = 0; + long bytesRequested = 0; + size_t numofBytes; + struct mca_sharedfp_base_data_t *sh = NULL; + + if(NULL == fh->f_sharedfp_data){ + opal_output(1, "sharedfp_addproc_iwrite: shared file pointer structure not initialized correctly\n"); + return OMPI_ERROR; + } + + /* Calculate the number of bytes to write */ + opal_datatype_type_size ( &datatype->super, &numofBytes); + bytesRequested = count * numofBytes; + + /* Retrieve the shared file data struct */ + sh = fh->f_sharedfp_data; + + if ( mca_sharedfp_addproc_verbose ){ + printf("sharedfp_addproc_iwrite: Bytes Requested is %ld\n",bytesRequested); + } + /* Request the offset to write bytesRequested bytes */ + ret = mca_sharedfp_addproc_request_position(sh,bytesRequested,&offset); + if ( OMPI_SUCCESS == ret ) { + if ( mca_sharedfp_addproc_verbose ){ + printf("sharedfp_addproc_iwrite: Offset received is %lld\n",offset); + } + /* Write to the file */ + ret = ompio_io_ompio_file_iwrite_at(sh->sharedfh,offset,buf,count,datatype,request); + } + + return ret; +} + +int mca_sharedfp_addproc_write_ordered_begin(mca_io_ompio_file_t *fh, + void *buf, + int count, + struct ompi_datatype_t *datatype) +{ + opal_output(1,"mca_sharedfp_addproc_write_ordered_begin: NOT IMPLEMENTED\n"); + return OMPI_ERROR; +} + + +int mca_sharedfp_addproc_write_ordered_end(mca_io_ompio_file_t *fh, + void *buf, + ompi_status_public_t *status) +{ + opal_output(1,"mca_sharedfp_addproc_write_ordered_end: NOT IMPLEMENTED\n"); + return OMPI_ERROR; +} diff --git a/ompi/mca/sharedfp/addproc/sharedfp_addproc_read.c b/ompi/mca/sharedfp/addproc/sharedfp_addproc_read.c new file mode 100644 index 0000000000..711ea5dd2c --- /dev/null +++ b/ompi/mca/sharedfp/addproc/sharedfp_addproc_read.c @@ -0,0 +1,197 @@ +/* + * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana + * University Research and Technology + * Corporation. All rights reserved. + * Copyright (c) 2004-2005 The University of Tennessee and The University + * of Tennessee Research Foundation. All rights + * reserved. + * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, + * University of Stuttgart. All rights reserved. + * Copyright (c) 2004-2005 The Regents of the University of California. + * All rights reserved. + * Copyright (c) 2013 University of Houston. All rights reserved. + * $COPYRIGHT$ + * + * Additional copyrights may follow + * + * $HEADER$ + */ + + +#include "ompi_config.h" +#include "sharedfp_addproc.h" + +#include "mpi.h" +#include "ompi/constants.h" +#include "orte/util/show_help.h" +#include "ompi/mca/sharedfp/sharedfp.h" + +int mca_sharedfp_addproc_read ( mca_io_ompio_file_t *fh, + void *buf, int count, MPI_Datatype datatype, MPI_Status *status) +{ + int ret = OMPI_SUCCESS; + OMPI_MPI_OFFSET_TYPE offset = 0; + long bytesRequested = 0; + size_t numofBytes; + struct mca_sharedfp_base_data_t *sh = NULL; + mca_sharedfp_base_module_t * shared_fp_base_module = NULL; + + if(NULL == fh->f_sharedfp_data){ + if ( mca_sharedfp_addproc_verbose ) { + printf("sharedfp_addproc_read: opening the shared file pointer file\n"); + } + shared_fp_base_module = fh->f_sharedfp; + + ret = shared_fp_base_module->sharedfp_file_open(fh->f_comm, + fh->f_filename, + fh->f_amode, + fh->f_info, + fh); + if ( OMPI_SUCCESS != ret ) { + opal_output(1,"sharedfp_addproc_read - error opening the shared file pointer\n"); + return ret; + } + } + + /* Calculate the number of bytes to write */ + opal_datatype_type_size ( &datatype->super ,&numofBytes); + bytesRequested = count * numofBytes; + + if ( mca_sharedfp_addproc_verbose ){ + printf("mca_sharedfp_addproc_read: Bytes Requested is %ld\n", bytesRequested); + } + /* Retrieve the shared file data struct */ + sh = fh->f_sharedfp_data; + + /*Request to the additional process for the offset*/ + ret = mca_sharedfp_addproc_request_position(sh,bytesRequested,&offset); + if( OMPI_SUCCESS == ret ){ + if ( mca_sharedfp_addproc_verbose ){ + printf("mca_sharedfp_addproc_read: Offset received is %lld\n",offset); + } + /* Read from the file */ + ret = ompio_io_ompio_file_read_at(sh->sharedfh,offset,buf,count,datatype,status); + } + + return ret; +} + +int mca_sharedfp_addproc_read_ordered (mca_io_ompio_file_t *fh, + void *buf, + int count, + struct ompi_datatype_t *datatype, + ompi_status_public_t *status) +{ + int ret = OMPI_SUCCESS; + OMPI_MPI_OFFSET_TYPE offset = 0, offsetReceived = 0; + long sendBuff = 0; + long *buff=NULL; + long offsetBuff, bytesRequested = 0; + size_t numofBytes; + int rank, size, i; + struct mca_sharedfp_base_data_t *sh = NULL; + mca_sharedfp_base_module_t * shared_fp_base_module = NULL; + + if(NULL == fh->f_sharedfp_data){ + if ( mca_sharedfp_addproc_verbose ) { + printf("sharedfp_addproc_read_ordered: opening the shared file pointer file\n"); + } + shared_fp_base_module = fh->f_sharedfp; + + ret = shared_fp_base_module->sharedfp_file_open(fh->f_comm, + fh->f_filename, + fh->f_amode, + fh->f_info, + fh); + if ( OMPI_SUCCESS != ret ) { + opal_output(1,"sharedfp_addproc_read_ordered - error opening the shared file pointer\n"); + return ret; + } + } + + + /*Retrieve the new communicator*/ + sh = fh->f_sharedfp_data; + + /* Calculate the number of bytes to read*/ + opal_datatype_type_size ( &datatype->super, &numofBytes); + sendBuff = count * numofBytes; + + /* Get the ranks in the communicator */ + rank = ompi_comm_rank ( sh->comm); + size = ompi_comm_size ( sh->comm); + + if ( 0 == rank ) { + buff = (long*)malloc(sizeof(long) * size); + if ( NULL == buff ) + return OMPI_ERR_OUT_OF_RESOURCE; + } + + ret = sh->comm->c_coll.coll_gather( &sendBuff, 1, OMPI_OFFSET_DATATYPE, + buff, 1, OMPI_OFFSET_DATATYPE, 0, sh->comm, + sh->comm->c_coll.coll_gather_module); + if ( OMPI_SUCCESS != ret ) { + goto exit; + } + + /* All the counts are present now in the recvBuff. + The size of recvBuff is sizeof_newComm + */ + if ( 0 == rank ) { + for (i = 0; i < size ; i ++) { + if ( mca_sharedfp_addproc_verbose ){ + printf("sharedfp_addproc_read_ordered: Buff is %ld\n",buff[i]); + } + bytesRequested += buff[i]; + + if ( mca_sharedfp_addproc_verbose ){ + printf("sharedfp_addproc_read_ordered: Bytes requested are %ld\n",bytesRequested); + } + } + + /* Request the offset to read bytesRequested bytes + ** only the root process needs to do the request, + ** since the root process will then tell the other + ** processes at what offset they should read their + ** share of the data. + */ + ret = mca_sharedfp_addproc_request_position(sh,bytesRequested,&offsetReceived); + if( OMPI_SUCCESS != ret ){ + goto exit; + } + if ( mca_sharedfp_addproc_verbose ){ + printf("sharedfp_addproc_read_ordered: Offset received is %lld\n",offsetReceived); + } + buff[0] += offsetReceived; + + + for (i = 1 ; i < size; i++) { + buff[i] += buff[i-1]; + } + } + + /* Scatter the results to the other processes*/ + ret = sh->comm->c_coll.coll_scatter ( buff, 1, OMPI_OFFSET_DATATYPE, &offsetBuff, + 1, OMPI_OFFSET_DATATYPE, 0, sh->comm, + sh->comm->c_coll.coll_scatter_module ); + if ( OMPI_SUCCESS != ret ) { + goto exit; + } + + /*Each process now has its own individual offset in recvBUFF*/ + offset = offsetBuff - sendBuff; + + if ( mca_sharedfp_addproc_verbose ){ + printf("sharedfp_addproc_read_ordered: Offset returned is %lld\n",offset); + } + + /* read from the file */ + ret = ompio_io_ompio_file_read_at_all(sh->sharedfh,offset,buf,count,datatype,status); + +exit: + if ( NULL != buff ) { + free ( buff ); + } + + return ret; +} diff --git a/ompi/mca/sharedfp/addproc/sharedfp_addproc_request_position.c b/ompi/mca/sharedfp/addproc/sharedfp_addproc_request_position.c new file mode 100644 index 0000000000..7ae5183478 --- /dev/null +++ b/ompi/mca/sharedfp/addproc/sharedfp_addproc_request_position.c @@ -0,0 +1,76 @@ +/* + * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana + * University Research and Technology + * Corporation. All rights reserved. + * Copyright (c) 2004-2005 The University of Tennessee and The University + * of Tennessee Research Foundation. All rights + * reserved. + * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, + * University of Stuttgart. All rights reserved. + * Copyright (c) 2004-2005 The Regents of the University of California. + * All rights reserved. + * Copyright (c) 2013 University of Houston. All rights reserved. + * $COPYRIGHT$ + * + * Additional copyrights may follow + * + * $HEADER$ + */ + + +#include "ompi_config.h" +#include "sharedfp_addproc.h" + +#include "mpi.h" +#include "ompi/constants.h" +#include "orte/util/show_help.h" +#include "ompi/mca/pml/pml.h" +#include "ompi/mca/sharedfp/sharedfp.h" + +int mca_sharedfp_addproc_request_position(struct mca_sharedfp_base_data_t * sh, + int bytes_requested, + OMPI_MPI_OFFSET_TYPE *offset) +{ + int ret = OMPI_SUCCESS; + OMPI_MPI_OFFSET_TYPE position = 0; + long sendBuff = bytes_requested ; + int count = 1; + + + struct mca_sharedfp_addproc_data * addproc_data = sh->selected_module_data; + + *offset = 0; + + ret = MCA_PML_CALL(send( &sendBuff, count, OMPI_OFFSET_DATATYPE, 0, REQUEST_TAG, + MCA_PML_BASE_SEND_STANDARD, addproc_data->intercom)); + if ( OMPI_SUCCESS != ret ) { + return ret; + } + ret = MCA_PML_CALL(recv( &position, count, OMPI_OFFSET_DATATYPE, 0, OFFSET_TAG, + addproc_data->intercom, MPI_STATUS_IGNORE)); + + *offset = position; + return ret; +} + +int mca_sharedfp_addproc_get_position(mca_io_ompio_file_t *fh, + OMPI_MPI_OFFSET_TYPE * offset) +{ + int ret = OMPI_SUCCESS; + struct mca_sharedfp_base_data_t *sh = NULL; + + if(NULL == fh->f_sharedfp_data){ + opal_output(1, "sharedfp_addproc_get_position - shared file pointer structure not initialized correctly\n"); + return OMPI_ERROR; + } + + /* Retrieve the shared file data struct*/ + sh = fh->f_sharedfp_data; + + /* Requesting the offset to write 0 bytes, + ** returns the current offset w/o updating it + */ + ret = mca_sharedfp_addproc_request_position(sh, 0, offset); + + return ret; +} diff --git a/ompi/mca/sharedfp/addproc/sharedfp_addproc_seek.c b/ompi/mca/sharedfp/addproc/sharedfp_addproc_seek.c new file mode 100644 index 0000000000..7b0f181dc4 --- /dev/null +++ b/ompi/mca/sharedfp/addproc/sharedfp_addproc_seek.c @@ -0,0 +1,70 @@ +/* + * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana + * University Research and Technology + * Corporation. All rights reserved. + * Copyright (c) 2004-2005 The University of Tennessee and The University + * of Tennessee Research Foundation. All rights + * reserved. + * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, + * University of Stuttgart. All rights reserved. + * Copyright (c) 2004-2005 The Regents of the University of California. + * All rights reserved. + * Copyright (c) 2013 University of Houston. All rights reserved. + * $COPYRIGHT$ + * + * Additional copyrights may follow + * + * $HEADER$ + */ + + +#include "ompi_config.h" +#include "sharedfp_addproc.h" + +#include "mpi.h" +#include "ompi/constants.h" +#include "orte/util/show_help.h" +#include "ompi/mca/pml/pml.h" +#include "ompi/mca/sharedfp/sharedfp.h" + +int +mca_sharedfp_addproc_seek (mca_io_ompio_file_t *fh, + OMPI_MPI_OFFSET_TYPE offset, int whence) +{ + int rank; + int ret = OMPI_SUCCESS; + OMPI_MPI_OFFSET_TYPE position = 0; + struct mca_sharedfp_base_data_t *sh = NULL; + struct mca_sharedfp_addproc_data * addproc_data = sh->selected_module_data; + long buff = 0; + + if(NULL == fh->f_sharedfp_data){ + opal_output(1, "sharedfp_addproc_write_ordered - shared file pointer structure not initialized correctly\n"); + return OMPI_ERROR; + } + + sh = fh->f_sharedfp_data; + rank = ompi_comm_rank ( sh->comm ); + buff = offset; + + + /* This is a collective call, + * only one process needs to communicate with the */ + if(0 == rank){ + ret = MCA_PML_CALL(send ( &buff, 1, OMPI_OFFSET_DATATYPE, 0, whence, + MCA_PML_BASE_SEND_STANDARD, + addproc_data->intercom)); + if ( OMPI_SUCCESS != ret ) { + return OMPI_ERROR; + } + ret = MCA_PML_CALL(recv(&position, 1, OMPI_OFFSET_DATATYPE, 0, whence, + addproc_data->intercom, MPI_STATUS_IGNORE)); + if ( OMPI_SUCCESS != ret ) { + return OMPI_ERROR; + } + + } + ret = sh->comm->c_coll.coll_barrier(sh->comm, sh->comm->c_coll.coll_barrier_module); + + return ret; +} diff --git a/ompi/mca/sharedfp/addproc/sharedfp_addproc_write.c b/ompi/mca/sharedfp/addproc/sharedfp_addproc_write.c new file mode 100644 index 0000000000..8cb526fab6 --- /dev/null +++ b/ompi/mca/sharedfp/addproc/sharedfp_addproc_write.c @@ -0,0 +1,197 @@ +/* + * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana + * University Research and Technology + * Corporation. All rights reserved. + * Copyright (c) 2004-2005 The University of Tennessee and The University + * of Tennessee Research Foundation. All rights + * reserved. + * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, + * University of Stuttgart. All rights reserved. + * Copyright (c) 2004-2005 The Regents of the University of California. + * All rights reserved. + * Copyright (c) 2013 University of Houston. All rights reserved. + * $COPYRIGHT$ + * + * Additional copyrights may follow + * + * $HEADER$ + */ + + +#include "ompi_config.h" +#include "sharedfp_addproc.h" + +#include "mpi.h" +#include "ompi/constants.h" +#include "orte/util/show_help.h" +#include "ompi/mca/sharedfp/sharedfp.h" + +int mca_sharedfp_addproc_write (mca_io_ompio_file_t *fh, + void *buf, + int count, + struct ompi_datatype_t *datatype, + ompi_status_public_t *status) +{ + int ret = OMPI_SUCCESS; + OMPI_MPI_OFFSET_TYPE offset = 0; + long bytesRequested = 0; + size_t numofBytes; + struct mca_sharedfp_base_data_t *sh = NULL; + mca_sharedfp_base_module_t * shared_fp_base_module = NULL; + + if(NULL == fh->f_sharedfp_data){ + if ( mca_sharedfp_addproc_verbose ) { + printf("sharedfp_addproc_write: opening the shared file pointer file\n"); + } + shared_fp_base_module = fh->f_sharedfp; + + ret = shared_fp_base_module->sharedfp_file_open(fh->f_comm, + fh->f_filename, + fh->f_amode, + fh->f_info, + fh); + if ( OMPI_SUCCESS != ret ) { + opal_output(1,"sharedfp_addproc_write - error opening the shared file pointer\n"); + return ret; + } + } + + /* Calculate the number of bytes to write*/ + opal_datatype_type_size ( &datatype->super, &numofBytes); + bytesRequested = count * numofBytes; + + /*Retrieve the shared file data structure */ + sh = fh->f_sharedfp_data; + + if ( mca_sharedfp_addproc_verbose ){ + printf("sharedfp_addproc_write: sharedfp_addproc_write: Bytes Requested is %ld\n",bytesRequested); + } + + /*Request the offset to write bytesRequested bytes*/ + ret = mca_sharedfp_addproc_request_position( sh, bytesRequested, &offset); + if ( OMPI_SUCCESS == ret ) { + if ( mca_sharedfp_addproc_verbose ){ + printf("sharedfp_addproc_write: Offset received is %lld\n",offset); + } + /* Write to the file */ + ret = ompio_io_ompio_file_write_at(sh->sharedfh,offset,buf,count,datatype,status); + } + + return ret; +} + +int mca_sharedfp_addproc_write_ordered (mca_io_ompio_file_t *fh, + void *buf, + int count, + struct ompi_datatype_t *datatype, + ompi_status_public_t *status) +{ + int ret = OMPI_SUCCESS; + OMPI_MPI_OFFSET_TYPE offset = 0, offsetReceived = 0; + long sendBuff = 0; + long *buff=NULL; + long offsetBuff; + long bytesRequested = 0; + int recvcnt = 1, sendcnt = 1; + size_t numofBytes; + int rank, size, i; + struct mca_sharedfp_base_data_t *sh = NULL; + mca_sharedfp_base_module_t * shared_fp_base_module = NULL; + + if ( NULL == fh->f_sharedfp_data){ + if ( mca_sharedfp_addproc_verbose ) { + printf("sharedfp_addproc_write_ordered: opening the shared file pointer\n"); + } + shared_fp_base_module = fh->f_sharedfp; + + ret = shared_fp_base_module->sharedfp_file_open(fh->f_comm, + fh->f_filename, + fh->f_amode, + fh->f_info, + fh); + if ( OMPI_SUCCESS != ret ) { + opal_output(1,"sharedfp_addproc_write_ordered - error opening the shared file pointer\n"); + return ret; + } + } + + /*Retrieve the shared file pointer structure*/ + sh = fh->f_sharedfp_data; + + /* Calculate the number of bytes to write*/ + opal_datatype_type_size ( &datatype->super, &numofBytes); + sendBuff = count * numofBytes; + + /* Get the ranks in the communicator */ + rank = ompi_comm_rank ( sh->comm ); + size = ompi_comm_size ( sh->comm ); + + if ( 0 == rank ) { + buff = (long*)malloc(sizeof(long) * size); + if ( NULL == buff ) + return OMPI_ERR_OUT_OF_RESOURCE; + } + + ret = sh->comm->c_coll.coll_gather ( &sendBuff, sendcnt, OMPI_OFFSET_DATATYPE, buff, + recvcnt, OMPI_OFFSET_DATATYPE, 0, sh->comm, + sh->comm->c_coll.coll_gather_module); + if( OMPI_SUCCESS != ret ){ + goto exit; + } + + /* All the counts are present now in the recvBuff. + The size of recvBuff is sizeof_newComm + */ + if ( 0 == rank ) { + for (i = 0; i < size ; i ++) { + bytesRequested += buff[i]; + + if ( mca_sharedfp_addproc_verbose ){ + printf("sharedfp_addproc_write_ordered: Bytes requested are %ld\n",bytesRequested); + } + } + + /* Request the offset to write bytesRequested bytes + ** only the root process needs to do the request, + ** since the root process will then tell the other + ** processes at what offset they should write their + ** share of the data. + */ + ret = mca_sharedfp_addproc_request_position(sh,bytesRequested,&offsetReceived); + if( OMPI_SUCCESS != ret ){ + goto exit; + } + if ( mca_sharedfp_addproc_verbose ){ + printf("sharedfp_addproc_write_ordered: Offset received is %lld\n",offsetReceived); + } + buff[0] += offsetReceived; + + for (i = 1 ; i < size; i++) { + buff[i] += buff[i-1]; + } + } + + /* Scatter the results to the other processes*/ + ret = sh->comm->c_coll.coll_scatter ( buff, sendcnt, OMPI_OFFSET_DATATYPE, &offsetBuff, + recvcnt, OMPI_OFFSET_DATATYPE, 0, sh->comm, + sh->comm->c_coll.coll_scatter_module ); + if( OMPI_SUCCESS != ret ){ + goto exit; + } + + /*Each process now has its own individual offset in recvBUFF*/ + offset = offsetBuff - sendBuff; + + if ( mca_sharedfp_addproc_verbose ){ + printf("sharedfp_addproc_write_ordered: Offset returned is %lld\n",offset); + } + + /* write to the file */ + ret = ompio_io_ompio_file_write_at_all(sh->sharedfh,offset,buf,count,datatype,status); + +exit: + if ( NULL != buff ) { + free ( buff ); + } + return ret; +} diff --git a/ompi/mca/sharedfp/base/sharedfp_base_file_select.c b/ompi/mca/sharedfp/base/sharedfp_base_file_select.c index c7d5da2441..6bcea16191 100644 --- a/ompi/mca/sharedfp/base/sharedfp_base_file_select.c +++ b/ompi/mca/sharedfp/base/sharedfp_base_file_select.c @@ -9,7 +9,7 @@ * University of Stuttgart. All rights reserved. * Copyright (c) 2004-2005 The Regents of the University of California. * All rights reserved. - * Copyright (c) 2008-2011 University of Houston. All rights reserved. + * Copyright (c) 2008-2013 University of Houston. All rights reserved. * Copyright (c) 2012-2013 Los Alamos National Security, Inc. All rights reserved. * $COPYRIGHT$ * @@ -96,7 +96,7 @@ int mca_sharedfp_base_file_select (struct mca_io_ompio_file_t *file, structure. This is necessary to proceed */ component = (mca_sharedfp_base_component_t *)preferred; - module = component->sharedfpm_file_query (&priority); + module = component->sharedfpm_file_query (file, &priority); if (NULL != module && NULL != module->sharedfp_module_init) { @@ -150,7 +150,7 @@ int mca_sharedfp_base_file_select (struct mca_io_ompio_file_t *file, /* * call the query function and see what it returns */ - module = component->sharedfpm_file_query (&priority); + module = component->sharedfpm_file_query (file, &priority); if (NULL == module || NULL == module->sharedfp_module_init) { diff --git a/ompi/mca/sharedfp/dummy/sharedfp_dummy.h b/ompi/mca/sharedfp/dummy/sharedfp_dummy.h deleted file mode 100644 index d5d427f1b7..0000000000 --- a/ompi/mca/sharedfp/dummy/sharedfp_dummy.h +++ /dev/null @@ -1,61 +0,0 @@ -/* - * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana - * University Research and Technology - * Corporation. All rights reserved. - * Copyright (c) 2004-2006 The University of Tennessee and The University - * of Tennessee Research Foundation. All rights - * reserved. - * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, - * University of Stuttgart. All rights reserved. - * Copyright (c) 2004-2005 The Regents of the University of California. - * All rights reserved. - * Copyright (c) 2008-2011 University of Houston. All rights reserved. - * $COPYRIGHT$ - * - * Additional copyrights may follow - * - * $HEADER$ - */ - -#ifndef MCA_SHAREDFP_DUMMY_H -#define MCA_SHAREDFP_DUMMY_H - -#include "ompi_config.h" -#include "opal/mca/mca.h" -#include "ompi/mca/sharedfp/sharedfp.h" -#include "ompi/mca/io/ompio/io_ompio.h" - - -BEGIN_C_DECLS - -int mca_sharedfp_dummy_component_init_query(bool enable_progress_threads, - bool enable_mpi_threads); -struct mca_sharedfp_base_module_1_0_0_t * -mca_sharedfp_dummy_component_file_query (int *priority); -int mca_sharedfp_dummy_component_file_unquery (mca_io_ompio_file_t *file); - -int mca_sharedfp_dummy_module_init (mca_io_ompio_file_t *file); -int mca_sharedfp_dummy_module_finalize (mca_io_ompio_file_t *file); - -OMPI_MODULE_DECLSPEC extern mca_sharedfp_base_component_2_0_0_t mca_sharedfp_dummy_component; -/* - * ****************************************************************** - * ********* functions which are implemented in this module ********* - * ****************************************************************** - */ - -int mca_sharedfp_dummy_update (mca_io_ompio_file_t *fh, - int num_bytes, - OMPI_MPI_OFFSET_TYPE current_position); -int mca_sharedfp_dummy_seek (mca_io_ompio_file_t *fh, - OMPI_MPI_OFFSET_TYPE position); - -/* - * ****************************************************************** - * ************ functions implemented in this module end ************ - * ****************************************************************** - */ - -END_C_DECLS - -#endif /* MCA_SHAREDFP_DUMMY_H */ diff --git a/ompi/mca/sharedfp/dummy/sharedfp_dummy_component.c b/ompi/mca/sharedfp/dummy/sharedfp_dummy_component.c deleted file mode 100644 index a8109f3264..0000000000 --- a/ompi/mca/sharedfp/dummy/sharedfp_dummy_component.c +++ /dev/null @@ -1,63 +0,0 @@ -/* - * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana - * University Research and Technology - * Corporation. All rights reserved. - * Copyright (c) 2004-2005 The University of Tennessee and The University - * of Tennessee Research Foundation. All rights - * reserved. - * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, - * University of Stuttgart. All rights reserved. - * Copyright (c) 2004-2005 The Regents of the University of California. - * All rights reserved. - * Copyright (c) 2008-2011 University of Houston. All rights reserved. - * $COPYRIGHT$ - * - * Additional copyrights may follow - * - * $HEADER$ - * - * These symbols are in a file by themselves to provide nice linker - * semantics. Since linkers generally pull in symbols by object - * files, keeping these symbols as the only symbols in this file - * prevents utility programs such as "ompi_info" from having to import - * entire components just to query their version and parameters. - */ - -#include "ompi_config.h" -#include "sharedfp_dummy.h" -#include "mpi.h" - -/* - * Public string showing the sharedfp dummy component version number - */ -const char *mca_sharedfp_dummy_component_version_string = - "OMPI/MPI dummy SHAREDFP MCA component version " OMPI_VERSION; - -/* - * Instantiate the public struct with all of our public information - * and pointers to our public functions in it - */ -mca_sharedfp_base_component_2_0_0_t mca_sharedfp_dummy_component = { - - /* First, the mca_component_t struct containing meta information - about the component itself */ - - { - MCA_SHAREDFP_BASE_VERSION_2_0_0, - - /* Component name and version */ - "dummy", - OMPI_MAJOR_VERSION, - OMPI_MINOR_VERSION, - OMPI_RELEASE_VERSION, - NULL, - NULL - }, - { - /* This component is checkpointable */ - MCA_BASE_METADATA_PARAM_CHECKPOINT - }, - mca_sharedfp_dummy_component_init_query, /* get thread level */ - mca_sharedfp_dummy_component_file_query, /* get priority and actions */ - mca_sharedfp_dummy_component_file_unquery /* undo what was done by previous function */ -}; diff --git a/ompi/mca/sharedfp/individual/Makefile.am b/ompi/mca/sharedfp/individual/Makefile.am new file mode 100644 index 0000000000..33fa26ab7d --- /dev/null +++ b/ompi/mca/sharedfp/individual/Makefile.am @@ -0,0 +1,56 @@ +# +# Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana +# University Research and Technology +# Corporation. All rights reserved. +# Copyright (c) 2004-2005 The University of Tennessee and The University +# of Tennessee Research Foundation. All rights +# reserved. +# Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, +# University of Stuttgart. All rights reserved. +# Copyright (c) 2004-2005 The Regents of the University of California. +# All rights reserved. +# Copyright (c) 2008 University of Houston. All rights reserved. +# $COPYRIGHT$ +# +# Additional copyrights may follow +# +# $HEADER$ +# + +# Make the output library in this directory, and name it either +# mca__.la (for DSO builds) or libmca__.la +# (for static builds). + +if MCA_BUILD_ompi_sharedfp_individual_DSO +component_noinst = +component_install = mca_sharedfp_individual.la +else +component_noinst = libmca_sharedfp_individual.la +component_install = +endif + +mcacomponentdir = $(pkglibdir) +mcacomponent_LTLIBRARIES = $(component_install) +mca_sharedfp_individual_la_SOURCES = $(sources) +mca_sharedfp_individual_la_LDFLAGS = -module -avoid-version + +noinst_LTLIBRARIES = $(component_noinst) +libmca_sharedfp_individual_la_SOURCES = $(sources) +libmca_sharedfp_individual_la_LDFLAGS = -module -avoid-version + +# Source files + +#IMPORTANT: Update here when adding new source code files to the library +sources = \ + sharedfp_individual.h \ + sharedfp_individual.c \ + sharedfp_individual_component.c \ + sharedfp_individual_seek.c \ + sharedfp_individual_get_position.c \ + sharedfp_individual_collaborate_data.c \ + sharedfp_individual_write.c \ + sharedfp_individual_iwrite.c \ + sharedfp_individual_read.c \ + sharedfp_individual_insert_metadata.c \ + sharedfp_individual_file_open.c \ + sharedfp_individual_gettime.c diff --git a/ompi/mca/sharedfp/individual/sharedfp_individual.c b/ompi/mca/sharedfp/individual/sharedfp_individual.c new file mode 100644 index 0000000000..83a44df5ff --- /dev/null +++ b/ompi/mca/sharedfp/individual/sharedfp_individual.c @@ -0,0 +1,166 @@ +/* + * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana + * University Research and Technology + * Corporation. All rights reserved. + * Copyright (c) 2004-2006 The University of Tennessee and The University + * of Tennessee Research Foundation. All rights + * reserved. + * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, + * University of Stuttgart. All rights reserved. + * Copyright (c) 2004-2005 The Regents of the University of California. + * All rights reserved. + * Copyright (c) 2013 University of Houston. All rights reserved. + * $COPYRIGHT$ + * + * Additional copyrights may follow + * + * $HEADER$ + * + * These symbols are in a file by themselves to provide nice linker + * semantics. Since linkers generally pull in symbols by object fules, + * keeping these symbols as the only symbols in this file prevents + * utility programs such as "ompi_info" from having to import entire + * modules just to query their version and parameters + */ + +#include "ompi_config.h" +#include "mpi.h" +#include "ompi/mca/sharedfp/sharedfp.h" +#include "ompi/mca/sharedfp/individual/sharedfp_individual.h" + +/* + * ******************************************************************* + * ************************ actions structure ************************ + * ******************************************************************* + */ + /* IMPORTANT: Update here when adding sharedfp component interface functions*/ +static mca_sharedfp_base_module_1_0_0_t individual = { + mca_sharedfp_individual_module_init, /* initalise after being selected */ + mca_sharedfp_individual_module_finalize, /* close a module on a communicator */ + mca_sharedfp_individual_seek, + mca_sharedfp_individual_get_position, + mca_sharedfp_individual_read, + mca_sharedfp_individual_read_ordered, + mca_sharedfp_individual_read_ordered_begin, + mca_sharedfp_individual_read_ordered_end, + mca_sharedfp_individual_iread, + mca_sharedfp_individual_write, + mca_sharedfp_individual_write_ordered, + mca_sharedfp_individual_write_ordered_begin, + mca_sharedfp_individual_write_ordered_end, + mca_sharedfp_individual_iwrite, + mca_sharedfp_individual_file_open, + mca_sharedfp_individual_file_close +}; +/* + * ******************************************************************* + * ************************* structure ends ************************** + * ******************************************************************* + */ + +int mca_sharedfp_individual_component_init_query(bool enable_progress_threads, + bool enable_mpi_threads) +{ + /* Nothing to do */ + + return OMPI_SUCCESS; +} + +struct mca_sharedfp_base_module_1_0_0_t * mca_sharedfp_individual_component_file_query (mca_io_ompio_file_t *fh, int *priority) { + + int amode; + bool wronly_flag=false; + bool relaxed_order_flag=false; + MPI_Info info; + int flag; + int valuelen; + char value[MPI_MAX_INFO_VAL+1]; + *priority = 0; + + /*test, and update priority*/ + /*---------------------------------------------------------*/ + /* 1. Is the file write only? check amode for MPI_MODE_WRONLY */ + amode = fh->f_amode; + if ( amode & MPI_MODE_WRONLY || amode & MPI_MODE_RDWR ) { + wronly_flag=true; + if ( mca_sharedfp_individual_verbose ) { + printf("mca_sharedfp_individual_component_file_query: " + "MPI_MODE_WRONLY[true=%d,false=%d]=%d\n",true,false,wronly_flag); + } + } else { + wronly_flag=false; + if ( mca_sharedfp_individual_verbose ) { + printf("mca_sharedfp_individual_component_file_query: Can not run!, " + "MPI_MODE_WRONLY[true=%d,false=%d]=%d\n",true,false,wronly_flag); + } + } + + /*---------------------------------------------------------*/ + /* 2. Did the user specify MPI_INFO relaxed ordering flag? */ + info = fh->f_info; + if ( info != MPI_INFO_NULL ){ + valuelen = MPI_MAX_INFO_VAL; + ompi_info_get ( info,"OMPIO_SHAREDFP_RELAXED_ORDERING", valuelen, value, &flag); + if ( flag ) { + if ( mca_sharedfp_individual_verbose ) { + printf("mca_sharedfp_individual_component_file_query: " + "OMPIO_SHAREDFP_RELAXED_ORDERING=%s\n",value); + } + /* flag - Returns true if key defined, false if not (boolean). */ + relaxed_order_flag=true; + } + else { + if ( mca_sharedfp_individual_verbose ) { + printf("mca_sharedfp_individual_component_file_query: " + "OMPIO_SHAREDFP_RELAXED_ORDERING MPI_Info key not set. " + "Set this key in order to increase this component's priority value.\n"); + } + } + } + else { + if ( mca_sharedfp_individual_verbose ) { + printf("mca_sharedfp_individual_component_file_query: " + "OMPIO_SHAREDFP_RELAXED_ORDERING MPI_Info key not set, " + "got MPI_INFO_NULL. Set this key in order to increase " + "this component's priority value.\n"); + } + } + + /*For now, this algorithm will not run if the file is not opened write only. + *Setting the OMPIO_SHAREDFP_RELAXED_ORDERING gives this module a higher priority + *otherwise it gets a priority of zero. This means that this module will + *run only if no other module can run + */ + if ( wronly_flag && relaxed_order_flag){ + *priority=mca_sharedfp_individual_priority; + } + else { + *priority=1; + } + + if ( wronly_flag ){ + return &individual; + } + + return NULL; +} + +int mca_sharedfp_individual_component_file_unquery (mca_io_ompio_file_t *file) +{ + /* This function might be needed for some purposes later. for now it + * does not have anything to do since there are no steps which need + * to be undone if this module is not selected */ + + return OMPI_SUCCESS; +} + +int mca_sharedfp_individual_module_init (mca_io_ompio_file_t *file) +{ + return OMPI_SUCCESS; +} + + +int mca_sharedfp_individual_module_finalize (mca_io_ompio_file_t *file) +{ + return OMPI_SUCCESS; +} diff --git a/ompi/mca/sharedfp/individual/sharedfp_individual.h b/ompi/mca/sharedfp/individual/sharedfp_individual.h new file mode 100644 index 0000000000..6ad0cd73f3 --- /dev/null +++ b/ompi/mca/sharedfp/individual/sharedfp_individual.h @@ -0,0 +1,163 @@ +/* + * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana + * University Research and Technology + * Corporation. All rights reserved. + * Copyright (c) 2004-2006 The University of Tennessee and The University + * of Tennessee Research Foundation. All rights + * reserved. + * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, + * University of Stuttgart. All rights reserved. + * Copyright (c) 2004-2005 The Regents of the University of California. + * All rights reserved. + * Copyright (c) 2013 University of Houston. All rights reserved. + * $COPYRIGHT$ + * + * Additional copyrights may follow + * + * $HEADER$ + */ + +#ifndef MCA_SHAREDFP_INDIVIDUAL_H +#define MCA_SHAREDFP_INDIVIDUAL_H + +#include "ompi_config.h" +#include "opal/mca/mca.h" +#include "ompi/mca/sharedfp/sharedfp.h" +#include "ompi/mca/io/ompio/io_ompio.h" + + +BEGIN_C_DECLS + +int mca_sharedfp_individual_component_init_query(bool enable_progress_threads, + bool enable_mpi_threads); +struct mca_sharedfp_base_module_1_0_0_t * + mca_sharedfp_individual_component_file_query (mca_io_ompio_file_t *file, int *priority); +int mca_sharedfp_individual_component_file_unquery (mca_io_ompio_file_t *file); + +int mca_sharedfp_individual_module_init (mca_io_ompio_file_t *file); +int mca_sharedfp_individual_module_finalize (mca_io_ompio_file_t *file); + +extern int mca_sharedfp_individual_priority; +extern int mca_sharedfp_individual_verbose; + +OMPI_MODULE_DECLSPEC extern mca_sharedfp_base_component_2_0_0_t mca_sharedfp_individual_component; +/* + * ****************************************************************** + * ********* functions which are implemented in this module ********* + * ****************************************************************** + */ +/*IMPORANT: Update here when implementing functions from sharedfp API*/ + +int mca_sharedfp_individual_get_position(mca_io_ompio_file_t *fh, + OMPI_MPI_OFFSET_TYPE * offset); +int mca_sharedfp_individual_seek (mca_io_ompio_file_t *fh, + OMPI_MPI_OFFSET_TYPE offset, int whence); +int mca_sharedfp_individual_file_open (struct ompi_communicator_t *comm, + char* filename, + int amode, + struct ompi_info_t *info, + mca_io_ompio_file_t *fh); +int mca_sharedfp_individual_file_close (mca_io_ompio_file_t *fh); +int mca_sharedfp_individual_read (mca_io_ompio_file_t *fh, + void *buf, int count, MPI_Datatype datatype, MPI_Status *status); +int mca_sharedfp_individual_read_ordered (mca_io_ompio_file_t *fh, + void *buf, int count, struct ompi_datatype_t *datatype, + ompi_status_public_t *status); +int mca_sharedfp_individual_read_ordered_begin (mca_io_ompio_file_t *fh, + void *buf, + int count, + struct ompi_datatype_t *datatype); +int mca_sharedfp_individual_read_ordered_end (mca_io_ompio_file_t *fh, + void *buf, + ompi_status_public_t *status); +int mca_sharedfp_individual_iread (mca_io_ompio_file_t *fh, + void *buf, + int count, + struct ompi_datatype_t *datatype, + ompi_request_t **request); +int mca_sharedfp_individual_write (mca_io_ompio_file_t *fh, + void *buf, + int count, + struct ompi_datatype_t *datatype, + ompi_status_public_t *status); +int mca_sharedfp_individual_write_ordered (mca_io_ompio_file_t *fh, + void *buf, + int count, + struct ompi_datatype_t *datatype, + ompi_status_public_t *status); +int mca_sharedfp_individual_write_ordered_begin (mca_io_ompio_file_t *fh, + void *buf, + int count, + struct ompi_datatype_t *datatype); +int mca_sharedfp_individual_write_ordered_end (mca_io_ompio_file_t *fh, + void *buf, + ompi_status_public_t *status); +int mca_sharedfp_individual_iwrite (mca_io_ompio_file_t *fh, + void *buf, + int count, + struct ompi_datatype_t *datatype, + ompi_request_t **request); + +#define OMPI_FILE_WRITE_SHARED 3 +#define MAX_METADATA_RECORDS 1024 + +typedef struct mca_sharedfp_individual_metadata_node_s { + long recordid; /* Denotes the type of function*/ + double timestamp; /* Timestamp*/ + MPI_Offset localposition; /* Offset of the data in the central data file*/ + long recordlength; /* Number of bytes*/ + struct mca_sharedfp_individual_metadata_node_s* next; +} mca_sharedfp_individual_metadata_node; + +struct mca_sharedfp_individual_record2 { + long recordid; /* Denotes the type of function */ + double timestamp; /* Timestamp */ + MPI_Offset localposition; /* Offset of the data in the central data file */ + long recordlength; /* Number of bytes*/ +}; + +/*This structure will hang off of the mca_sharedfp_base_data_t's + *selected_module_data attribute + */ +typedef struct mca_sharedfp_individual_header_record_s{ + int numofrecords; /* Number of records in the linked list*/ + int numofrecordsonfile; /* Number of records in the metadatafile*/ + MPI_Offset datafile_offset; + MPI_Offset metadatafile_offset; + mca_io_ompio_file_t * datafilehandle; + mca_io_ompio_file_t * metadatafilehandle; + char * datafilename; /*for now need to delete this on file close*/ + char * metadatafilename; /*for now need to delete this on file close*/ + MPI_Offset metafile_start_offset; + MPI_Offset datafile_start_offset; + struct mca_sharedfp_individual_metadata_node_s *next; +} mca_sharedfp_individual_header_record; + + +mca_sharedfp_individual_header_record* mca_sharedfp_individual_insert_headnode(void); + +int mca_sharedfp_individual_collaborate_data(struct mca_sharedfp_base_data_t *sh); +int mca_sharedfp_individual_get_timestamps_and_reclengths(double **buff, long **rec_length, MPI_Offset **offbuff,struct mca_sharedfp_base_data_t *sh); +int mca_sharedfp_individual_create_buff(double **ts,MPI_Offset **off,int totalnodes,int size); +int mca_sharedfp_individual_sort_timestamps(double **ts,MPI_Offset **off, int totalnodes); +MPI_Offset mca_sharedfp_individual_assign_globaloffset(MPI_Offset **offsetbuff,int totalnodes,struct mca_sharedfp_base_data_t *sh); +int mca_sharedfp_individual_getoffset(double timestamp, double *ts, int totalnodes); +/*int mca_sharedfp_individual_cleanup(double *ts, int* rnk, MPI_Offset *off);*/ + +int mca_sharedfp_individual_insert_metadata(int functype,long recordlength,struct mca_sharedfp_base_data_t *sh ); +int mca_sharedfp_individual_write_metadata_file(struct mca_sharedfp_base_data_t *sh); +/*MPI_Datatype mca_sharedfp_individual_create_datatype();*/ +/*int mca_sharedfp_individual_compute_highest_globalposition(MPI_Offset* global_off, int size);*/ +/*MPI_Offset mca_sharedfp_individual_get_last_offset(struct mca_sharedfp_file_handle *sh);*/ + +double mca_sharedfp_individual_gettime(void); + +/* + * ****************************************************************** + * ************ functions implemented in this module end ************ + * ****************************************************************** + */ + +END_C_DECLS + +#endif /* MCA_SHAREDFP_INDIVIDUAL_H */ diff --git a/ompi/mca/sharedfp/individual/sharedfp_individual_collaborate_data.c b/ompi/mca/sharedfp/individual/sharedfp_individual_collaborate_data.c new file mode 100644 index 0000000000..622d48d94d --- /dev/null +++ b/ompi/mca/sharedfp/individual/sharedfp_individual_collaborate_data.c @@ -0,0 +1,398 @@ +/* + * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana + * University Research and Technology + * Corporation. All rights reserved. + * Copyright (c) 2004-2005 The University of Tennessee and The University + * of Tennessee Research Foundation. All rights + * reserved. + * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, + * University of Stuttgart. All rights reserved. + * Copyright (c) 2004-2005 The Regents of the University of California. + * All rights reserved. + * Copyright (c) 2013 University of Houston. All rights reserved. + * $COPYRIGHT$ + * + * Additional copyrights may follow + * + * $HEADER$ + */ + + +#include "ompi_config.h" +#include "sharedfp_individual.h" + +#include "mpi.h" +#include "ompi/constants.h" +#include "orte/util/show_help.h" +#include "ompi/mca/sharedfp/sharedfp.h" +#include "ompi/mca/io/ompio/io_ompio.h" + +#include +#include + +int mca_sharedfp_individual_collaborate_data(struct mca_sharedfp_base_data_t *sh) +{ + int ret = OMPI_SUCCESS; + mca_sharedfp_individual_header_record *headnode = NULL; + char *buff=NULL; + MPI_Comm comm; + int rank, size; + int nodesoneachprocess = 0; + int idx = 0,i = 0; + double *timestampbuff = NULL; + OMPI_MPI_OFFSET_TYPE *offsetbuff = NULL; + int *countbuff = NULL; + int *displ = NULL; + double *ind_ts = NULL; + long *ind_recordlength = NULL; + OMPI_MPI_OFFSET_TYPE *local_off = NULL; + int totalnodes = 0; + ompi_status_public_t status; + + comm = sh->comm; + + rank = ompi_comm_rank ( comm ); + size = ompi_comm_size ( comm ); + + headnode = (mca_sharedfp_individual_header_record*)sh->selected_module_data; + if ( NULL == headnode) { + opal_output (1, "sharedfp_individual_collaborate_data: headnode is NULL but file is open\n"); + return OMPI_ERROR; + } + + /* Number of nodes on each process is the sum of records + * on file and records in the linked list + */ + nodesoneachprocess = headnode->numofrecordsonfile + headnode->numofrecords; + + if ( mca_sharedfp_individual_verbose ) { + printf("Nodes of each process = %d\n",nodesoneachprocess); + } + + countbuff = (int*)malloc(size * sizeof(int)); + if ( NULL == countbuff ) { + return OMPI_ERR_OUT_OF_RESOURCE; + } + + displ = (int*)malloc(sizeof(int) * size); + if ( NULL == displ ) { + ret = OMPI_ERR_OUT_OF_RESOURCE; + goto exit; + } + + /* Each process counts the number of nodes + * in its linked list for which global offset */ + ret = mca_sharedfp_individual_get_timestamps_and_reclengths ( &ind_ts, &ind_recordlength, + &local_off, sh ); + if ( OMPI_SUCCESS != ret ) { + goto exit; + } + + comm->c_coll.coll_allgather ( &nodesoneachprocess, 1, MPI_INT, + countbuff, 1, MPI_INT, comm, + comm->c_coll.coll_allgather_module ); + + if ( mca_sharedfp_individual_verbose) { + for (i = 0; i < size ; i++) { + printf("sharedfp_individual_collaborate_data: Countbuff[%d] = %d\n", i, countbuff[i]); + } + } + + if ( nodesoneachprocess == 0) { + ind_ts[0] = 0; + ind_recordlength[0] = 0; + local_off[0] = 0; + } + + for(i = 0; i < size; i++) { + displ[i] = totalnodes; + if ( mca_sharedfp_individual_verbose ) { + printf("sharedfp_individual_collaborate_data: displ[%d] = %d\n",i,displ[i]); + } + totalnodes = totalnodes + countbuff[i]; + } + + if (totalnodes <= 0 ) { + goto exit; + } + + ret = mca_sharedfp_individual_create_buff ( ×tampbuff, &offsetbuff, totalnodes, size); + if ( OMPI_SUCCESS != ret ) { + goto exit; + } + + ret = comm->c_coll.coll_allgatherv ( ind_ts, countbuff[rank], MPI_DOUBLE, + timestampbuff, countbuff, displ, MPI_DOUBLE, + comm, comm->c_coll.coll_allgatherv_module ); + if ( OMPI_SUCCESS != ret ) { + goto exit; + } + + ret = comm->c_coll.coll_allgatherv ( ind_recordlength, countbuff[rank], OMPI_OFFSET_DATATYPE, + offsetbuff, countbuff, displ, OMPI_OFFSET_DATATYPE, + comm, comm->c_coll.coll_allgatherv_module ); + if ( OMPI_SUCCESS != ret ) { + goto exit; + } + + ret = mca_sharedfp_individual_sort_timestamps(×tampbuff, &offsetbuff,totalnodes); + if ( OMPI_SUCCESS != ret ) { + goto exit; + } + + sh->global_offset = mca_sharedfp_individual_assign_globaloffset ( &offsetbuff, totalnodes, sh); + + buff = (char * ) malloc( ind_recordlength[0] * 1.2 ); + if ( NULL == buff ) { + ret = OMPI_ERR_OUT_OF_RESOURCE; + goto exit; + } + + for (i = 0; i < nodesoneachprocess ; i++) { + /*Read from the local data file*/ + ompio_io_ompio_file_read_at ( headnode->datafilehandle, + local_off[i], buff, ind_recordlength[i], + MPI_BYTE, &status); + + idx = mca_sharedfp_individual_getoffset(ind_ts[i],timestampbuff,totalnodes); + + if ( mca_sharedfp_individual_verbose ) { + printf("sharedfp_individual_collaborate_data: Process %d writing %ld bytes to main file \n", + rank,ind_recordlength[i]); + } + + /*Write into main data file*/ + ompio_io_ompio_file_write_at( sh->sharedfh, offsetbuff[idx], buff, + ind_recordlength[i], MPI_BYTE, &status); + + } + +exit: + if ( NULL != countbuff ) { + free ( countbuff ); + } + if ( NULL != displ ) { + free ( displ ); + } + + if( NULL != timestampbuff ){ + free ( timestampbuff ); + } + if ( NULL != offsetbuff ){ + free ( offsetbuff ); + } + if ( NULL != ind_ts ) { + free ( ind_ts ); + } + if ( NULL != ind_recordlength ) { + free ( ind_recordlength ); + } + if ( NULL != local_off ) { + free ( local_off ); + } + if ( NULL != buff ) { + free ( buff ); + } + + return ret; +} + +/* Count the number of nodes and create and array of the timestamps*/ +int mca_sharedfp_individual_get_timestamps_and_reclengths ( double **buff, long **rec_length, MPI_Offset **offbuff,struct mca_sharedfp_base_data_t *sh) +{ + int num = 0, i= 0, ctr = 0; + int ret=OMPI_SUCCESS; + mca_sharedfp_individual_metadata_node *currnode; + mca_sharedfp_individual_header_record *headnode; + OMPI_MPI_OFFSET_TYPE metaoffset = 0; + struct mca_sharedfp_individual_record2 rec; + MPI_Status status; + + headnode = (mca_sharedfp_individual_header_record*)(sh->selected_module_data); + num = ( headnode->numofrecords + headnode->numofrecordsonfile); + currnode = headnode->next; + + if ( mca_sharedfp_individual_verbose ) { + printf("Num is %d\n",num); + } + + if ( 0 == num ) { + *buff = (double*) malloc ( sizeof ( double )); + *rec_length = (long *) malloc ( sizeof ( long )); + *offbuff = (OMPI_MPI_OFFSET_TYPE *)malloc ( sizeof(OMPI_MPI_OFFSET_TYPE) ); + if ( NULL == *buff || NULL == *rec_length || NULL == *offbuff ) { + ret = OMPI_ERR_OUT_OF_RESOURCE; + goto exit; + } + } + else { + *buff = (double* ) malloc(sizeof ( double) * num); + *rec_length = (long *) malloc(sizeof ( long) * num); + *offbuff = (OMPI_MPI_OFFSET_TYPE *) malloc ( sizeof(OMPI_MPI_OFFSET_TYPE) * num); + if ( NULL == *buff || NULL == *rec_length || NULL == *offbuff ) { + ret = OMPI_ERR_OUT_OF_RESOURCE; + goto exit; + } + } + + if ( mca_sharedfp_individual_verbose ) { + printf("sharedfp_individual_get_timestamps_and_reclengths: Numofrecords on file %d\n", + headnode->numofrecordsonfile); + } + + if (headnode->numofrecordsonfile > 0) { + metaoffset = headnode->metafile_start_offset; + ctr = 0; + for (i = 0; i < headnode->numofrecordsonfile ; i++) { + + ompio_io_ompio_file_read_at(headnode->metadatafilehandle,metaoffset, &rec, 32, MPI_BYTE,&status); + + *(*rec_length + ctr) = rec.recordlength; + *(*buff + ctr) = rec.timestamp; + *(*offbuff + ctr) = rec.localposition; + + metaoffset = metaoffset + sizeof(struct mca_sharedfp_individual_record2); + + if ( mca_sharedfp_individual_verbose ) { + printf("sharedfp_individual_get_timestamps_and_reclengths: Ctr = %d\n",ctr); + } + ctr++; + } + + headnode->numofrecordsonfile = 0; + headnode->metafile_start_offset = metaoffset; + + } /* End of if (headnode->numofrecordsonfile > 0) */ + + /* Add the records from the linked list */ + currnode = headnode->next; + while (currnode) { + if ( mca_sharedfp_individual_verbose ) { + printf("Ctr = %d\n",ctr); + } + /* Some error over here..need to check this code again */ + /*while(headnode->next != NULL)*/ + + *(*rec_length + ctr) = currnode->recordlength; + *(*buff + ctr) = currnode->timestamp; + *(*offbuff + ctr) = currnode->localposition; + + ctr = ctr + 1; + + headnode->next = currnode->next; + if ( mca_sharedfp_individual_verbose ) { + printf("sharedfp_individual_get_timestamps_and_reclengths: node deleted from the metadatalinked list\n"); + } + free(currnode); + currnode = headnode->next; + + } /*End of while(currnode) loop*/ + + + /*Reset the numofrecords*/ + headnode->numofrecords = 0; + +exit: + + return ret; +} + +int mca_sharedfp_individual_create_buff(double **ts,MPI_Offset **off,int totalnodes, int size) +{ + + if ( totalnodes) { + *off = (OMPI_MPI_OFFSET_TYPE *) malloc ( totalnodes * sizeof(OMPI_MPI_OFFSET_TYPE)); + if ( NULL == *off ) { + return OMPI_ERR_OUT_OF_RESOURCE; + } + + *ts = (double *) malloc ( totalnodes * sizeof(double) ); + if (NULL == *ts ) { + return OMPI_ERR_OUT_OF_RESOURCE; + } + + } + + return OMPI_SUCCESS; +} + +/*Sort the timestamp buffer*/ +int mca_sharedfp_individual_sort_timestamps(double **ts, MPI_Offset **off, int totalnodes) +{ + + int i = 0; + int j = 0; + int flag = 1; + double tempts = 0.0; + OMPI_MPI_OFFSET_TYPE tempoffset = 0; + + + for (i= 1; (i <= totalnodes)&&(flag) ; i++) { + flag = 0; + for (j = 0; j < (totalnodes - 1); j++) { + if ( *(*ts + j + 1) < *(*ts + j )) { + /*swap timestamp*/ + tempts = *(*ts + j ); + *(*ts + j) = *(*ts + j + 1); + *(*ts + j + 1) = tempts; + + /*swap offset*/ + tempoffset = *(*off + j); + *(*off + j) = *(*off + j + 1); + *(*off + j + 1) = tempoffset; + + flag = 1; + } + } + + } + + return OMPI_SUCCESS; +} + + +MPI_Offset mca_sharedfp_individual_assign_globaloffset(MPI_Offset **offsetbuff,int totalnodes,struct mca_sharedfp_base_data_t *sh) +{ + int i = 0; + OMPI_MPI_OFFSET_TYPE temp = 0,prevoffset = 0; + OMPI_MPI_OFFSET_TYPE global_offset = 0; + + for (i = 0; i < totalnodes; i++) { + temp = *(*offsetbuff + i); + + if (i == 0) { + *(*offsetbuff + i ) = sh->global_offset; + } + else { + *(*offsetbuff + i) = *(*offsetbuff + i - 1) + prevoffset; + } + prevoffset = temp; + } + global_offset = *(*offsetbuff + i - 1) + prevoffset; + + return global_offset; +} + + +int mca_sharedfp_individual_getoffset(double timestamp, double *ts, int totalnodes) +{ + int i = 0; + int notfound = 1; + + while (notfound) { + if (ts[i] == timestamp) + break; + + i++; + + if (i == totalnodes) { + notfound = 0; + } + } + + if (!notfound) { + return -1; + } + + return i; +} diff --git a/ompi/mca/sharedfp/individual/sharedfp_individual_component.c b/ompi/mca/sharedfp/individual/sharedfp_individual_component.c new file mode 100644 index 0000000000..4039d8e395 --- /dev/null +++ b/ompi/mca/sharedfp/individual/sharedfp_individual_component.c @@ -0,0 +1,89 @@ +/* + * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana + * University Research and Technology + * Corporation. All rights reserved. + * Copyright (c) 2004-2005 The University of Tennessee and The University + * of Tennessee Research Foundation. All rights + * reserved. + * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, + * University of Stuttgart. All rights reserved. + * Copyright (c) 2004-2005 The Regents of the University of California. + * All rights reserved. + * Copyright (c) 2013 University of Houston. All rights reserved. + * $COPYRIGHT$ + * + * Additional copyrights may follow + * + * $HEADER$ + * + * These symbols are in a file by themselves to provide nice linker + * semantics. Since linkers generally pull in symbols by object + * files, keeping these symbols as the only symbols in this file + * prevents utility programs such as "ompi_info" from having to import + * entire components just to query their version and parameters. + */ + +#include "ompi_config.h" +#include "sharedfp_individual.h" +#include "mpi.h" + +/* + * Public string showing the sharedfp individual component version number + */ +const char *mca_sharedfp_individual_component_version_string = + "OMPI/MPI individual SHAREDFP MCA component version " OMPI_VERSION; + +/* + * Global variables + */ +int mca_sharedfp_individual_priority=10; +int mca_sharedfp_individual_verbose=0; + +static int individual_register(void); + +/* + * Instantiate the public struct with all of our public information + * and pointers to our public functions in it + */ +mca_sharedfp_base_component_2_0_0_t mca_sharedfp_individual_component = { + + /* First, the mca_component_t struct containing meta information + about the component itself */ + + { + MCA_SHAREDFP_BASE_VERSION_2_0_0, + + /* Component name and version */ + "individual", + OMPI_MAJOR_VERSION, + OMPI_MINOR_VERSION, + OMPI_RELEASE_VERSION, + individual_register, + NULL + }, + { + /* This component is checkpointable */ + MCA_BASE_METADATA_PARAM_CHECKPOINT + }, + mca_sharedfp_individual_component_init_query, /* get thread level */ + mca_sharedfp_individual_component_file_query, /* get priority and actions */ + mca_sharedfp_individual_component_file_unquery /* undo what was done by previous function */ +}; + +static int individual_register(void) +{ + mca_sharedfp_individual_priority = 10; + (void) mca_base_component_var_register(&mca_sharedfp_individual_component.sharedfpm_version, + "priority", "Priority of the individual sharedfp component", + MCA_BASE_VAR_TYPE_INT, NULL, 0, 0, + OPAL_INFO_LVL_9, + MCA_BASE_VAR_SCOPE_READONLY, &mca_sharedfp_individual_priority); + mca_sharedfp_individual_verbose = 0; + (void) mca_base_component_var_register(&mca_sharedfp_individual_component.sharedfpm_version, + "verbose", "Verbosity of the individual sharedfp component", + MCA_BASE_VAR_TYPE_INT, NULL, 0, 0, + OPAL_INFO_LVL_9, + MCA_BASE_VAR_SCOPE_READONLY, &mca_sharedfp_individual_verbose); + + return OMPI_SUCCESS; +} diff --git a/ompi/mca/sharedfp/individual/sharedfp_individual_file_open.c b/ompi/mca/sharedfp/individual/sharedfp_individual_file_open.c new file mode 100644 index 0000000000..c88bd87a4d --- /dev/null +++ b/ompi/mca/sharedfp/individual/sharedfp_individual_file_open.c @@ -0,0 +1,214 @@ +/* + * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana + * University Research and Technology + * Corporation. All rights reserved. + * Copyright (c) 2004-2005 The University of Tennessee and The University + * of Tennessee Research Foundation. All rights + * reserved. + * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, + * University of Stuttgart. All rights reserved. + * Copyright (c) 2004-2005 The Regents of the University of California. + * All rights reserved. + * Copyright (c) 2013 University of Houston. All rights reserved. + * $COPYRIGHT$ + * + * Additional copyrights may follow + * + * $HEADER$ + */ + + +#include "ompi_config.h" +#include "sharedfp_individual.h" + +#include "mpi.h" +#include "ompi/constants.h" +#include "orte/util/show_help.h" +#include "ompi/mca/sharedfp/sharedfp.h" +#include "ompi/mca/sharedfp/base/base.h" +#include "ompi/file/file.h" + + +int mca_sharedfp_individual_file_open (struct ompi_communicator_t *comm, + char* filename, + int amode, + struct ompi_info_t *info, + mca_io_ompio_file_t *fh) +{ + int err = 0; + int rank; + char * datafilename; /*The array size would change as it is based on the current path*/ + char * metadatafilename; /*The array size would change as it is based on the current path*/ + mca_io_ompio_file_t * datafilehandle; + mca_io_ompio_file_t * metadatafilehandle; + mca_sharedfp_individual_header_record* headnode = NULL; + struct mca_sharedfp_base_data_t* sh; + mca_io_ompio_file_t * shfileHandle; + size_t len=0; + + /*-------------------------------------------------*/ + /*Open the same file again without shared file pointer*/ + /*-------------------------------------------------*/ + shfileHandle = (mca_io_ompio_file_t *) malloc ( sizeof(mca_io_ompio_file_t)); + err = ompio_io_ompio_file_open ( comm, filename, amode, info, shfileHandle, false); + if ( OMPI_SUCCESS != err ) { + opal_output(1, "mca_sharedfp_individual_file_open: Error during file open\n"); + return err; + } + + sh = (struct mca_sharedfp_base_data_t*) malloc ( sizeof(struct mca_sharedfp_base_data_t)); + if ( NULL == sh ){ + opal_output(1, "mca_sharedfp_individual_file_open: Error, unable to malloc " + "f_sharedfp_ptr struct\n"); + return OMPI_ERR_OUT_OF_RESOURCE; + } + + rank = ompi_comm_rank ( comm ); + + /*Populate the sh file structure based on the implementation*/ + sh->sharedfh = shfileHandle; /* Shared file pointer*/ + sh->global_offset = 0; /* Global Offset*/ + sh->comm = comm; /* Communicator*/ + sh->selected_module_data = NULL; + + /* Assign the metadatalinked list to sh->handle */ + sh->selected_module_data = (mca_sharedfp_individual_header_record *)mca_sharedfp_individual_insert_headnode(); + + /*--------------------------------------------------------*/ + /* Open an individual data file for each process */ + /* NOTE: Open the data file without shared file pointer */ + /*--------------------------------------------------------*/ + if ( mca_sharedfp_individual_verbose ) { + printf("mca_sharedfp_individual_file_open: open data file.\n"); + } + + /* data filename created by appending .data.$rank to the original filename*/ + len = strlen (filename ) + 64; + datafilename = (char*)malloc( len ); + snprintf(datafilename, len, "%s%s%d",filename,".data.",rank); + + + datafilehandle = (mca_io_ompio_file_t *)malloc(sizeof(mca_io_ompio_file_t)); + err = ompio_io_ompio_file_open(MPI_COMM_SELF, datafilename, + MPI_MODE_RDWR | MPI_MODE_CREATE | MPI_MODE_DELETE_ON_CLOSE, + MPI_INFO_NULL, datafilehandle, false); + if ( OMPI_SUCCESS != err) { + opal_output(1, "mca_sharedfp_individual_file_open: Error during datafile file open\n"); + return err; + } + + /*----------------------------------------------------------*/ + /* Open an individual metadata file for each of the process */ + /* NOTE: Open the meta file without shared file pointer */ + /*----------------------------------------------------------*/ + if ( mca_sharedfp_individual_verbose ) { + printf("mca_sharedfp_individual_file_open: metadata file.\n"); + } + + /* metadata filename created by appending .metadata.$rank to the original filename*/ + metadatafilename = (char*) malloc ( len ); + snprintf ( metadatafilename, len, "%s%s%d", filename, ".metadata.",rank); + + metadatafilehandle = (mca_io_ompio_file_t *)malloc(sizeof(mca_io_ompio_file_t)); + err = ompio_io_ompio_file_open ( MPI_COMM_SELF,metadatafilename, + MPI_MODE_RDWR | MPI_MODE_CREATE | MPI_MODE_DELETE_ON_CLOSE, + MPI_INFO_NULL, metadatafilehandle, false); + if ( OMPI_SUCCESS != err) { + opal_output(1, "mca_sharedfp_individual_file_open: Error during metadatafile file open\n"); + return err; + } + + /*save the datafilehandle and metadatahandle in the sharedfp individual module data structure*/ + headnode = (mca_sharedfp_individual_header_record*)(sh->selected_module_data); + if ( headnode) { + headnode->datafilehandle = datafilehandle; + headnode->metadatafilehandle = metadatafilehandle; + headnode->datafilename = datafilename; + headnode->metadatafilename = metadatafilename; + } + + /*save the sharedfp individual module data structure in the ompio filehandle structure*/ + fh->f_sharedfp_data = sh; + + return err; +} + +int mca_sharedfp_individual_file_close (mca_io_ompio_file_t *fh) +{ + mca_sharedfp_individual_header_record* headnode = NULL; + struct mca_sharedfp_base_data_t *sh; + int err = OMPI_SUCCESS; + + if ( NULL == fh->f_sharedfp_data ){ + if ( mca_sharedfp_individual_verbose ) { + printf("sharedfp_inidividual_file_close - shared file pointer structure not initialized\n"); + } + return OMPI_SUCCESS; + } + sh = fh->f_sharedfp_data; + + /* Merge data from individal files to final output file */ + err = mca_sharedfp_individual_collaborate_data (sh); + + headnode = (mca_sharedfp_individual_header_record*)(sh->selected_module_data); + if (headnode) { + /*Close datafile*/ + if (headnode->datafilehandle) { + /*TODO: properly deal with returned error code*/ + err = ompio_io_ompio_file_close(headnode->datafilehandle); + /* NOTE: No neeed to manually delete the file, + ** the amode should have been set to delete on close + */ + } + if(headnode->datafilename){ + free(headnode->datafilename); + } + + /*Close metadatafile*/ + if (headnode->metadatafilehandle) { + /*TODO: properly deal with returned error code*/ + err = ompio_io_ompio_file_close(headnode->metadatafilehandle); + /* NOTE: No neeed to manually delete the file, + ** the amode should have been set to delete on close + */ + } + if(headnode->metadatafilename){ + free(headnode->metadatafilename); + } + } + + /* Close the main file opened by this component*/ + err = ompio_io_ompio_file_close(sh->sharedfh); + + /*free shared file pointer data struct*/ + free(sh); + + return err; +} + + +mca_sharedfp_individual_header_record* mca_sharedfp_individual_insert_headnode ( void ) +{ + mca_sharedfp_individual_header_record *headnode = NULL; + if ( !headnode ) { + /*Allocate a headnode*/ + headnode = (mca_sharedfp_individual_header_record*)malloc(sizeof(mca_sharedfp_individual_header_record)); + if (!headnode) + return NULL; + } + + headnode->numofrecords = 0; /* No records in the linked list */ + headnode->numofrecordsonfile = 0; /* No records in the metadatafile for this file */ + + headnode->datafile_offset = 0; + headnode->metadatafile_offset = 0; + + headnode->metafile_start_offset = 0; + headnode->datafile_start_offset = 0; + + headnode->metadatafilehandle = 0; + headnode->datafilehandle = 0; + headnode->next = NULL; + + return headnode; +} diff --git a/ompi/mca/sharedfp/dummy/sharedfp_dummy_update.c b/ompi/mca/sharedfp/individual/sharedfp_individual_get_position.c similarity index 66% rename from ompi/mca/sharedfp/dummy/sharedfp_dummy_update.c rename to ompi/mca/sharedfp/individual/sharedfp_individual_get_position.c index 61e2b6e92a..7c77be18a8 100644 --- a/ompi/mca/sharedfp/dummy/sharedfp_dummy_update.c +++ b/ompi/mca/sharedfp/individual/sharedfp_individual_get_position.c @@ -2,34 +2,34 @@ * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana * University Research and Technology * Corporation. All rights reserved. - * Copyright (c) 2004-2011 The University of Tennessee and The University + * Copyright (c) 2004-2005 The University of Tennessee and The University * of Tennessee Research Foundation. All rights * reserved. - * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, + * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, * University of Stuttgart. All rights reserved. * Copyright (c) 2004-2005 The Regents of the University of California. * All rights reserved. - * Copyright (c) 2008-2011 University of Houston. All rights reserved. + * Copyright (c) 2013 University of Houston. All rights reserved. * $COPYRIGHT$ - * + * * Additional copyrights may follow - * + * * $HEADER$ */ #include "ompi_config.h" -#include "sharedfp_dummy.h" +#include "sharedfp_individual.h" #include "mpi.h" #include "ompi/constants.h" +#include "orte/util/show_help.h" #include "ompi/mca/sharedfp/sharedfp.h" int -mca_sharedfp_dummy_update (mca_io_ompio_file_t *fh, - int num_bytes, - OMPI_MPI_OFFSET_TYPE current_position) +mca_sharedfp_individual_get_position(mca_io_ompio_file_t *fh, + OMPI_MPI_OFFSET_TYPE * offset) { - printf ("DUMMY UPDATING\n"); + opal_output(1,"mca_sharedfp_individual_get_position: NOT IMPLEMENTED\n"); return OMPI_SUCCESS; } diff --git a/ompi/mca/sharedfp/individual/sharedfp_individual_gettime.c b/ompi/mca/sharedfp/individual/sharedfp_individual_gettime.c new file mode 100644 index 0000000000..c0ecc67d6d --- /dev/null +++ b/ompi/mca/sharedfp/individual/sharedfp_individual_gettime.c @@ -0,0 +1,29 @@ +#include +#include +#include + +#include "ompi_config.h" +#include "sharedfp_individual.h" + +#include "mpi.h" +#include "ompi/constants.h" +#include "orte/util/show_help.h" +#include "ompi/mca/sharedfp/sharedfp.h" + +#include +#include +#include "mpi.h" + +double mca_sharedfp_individual_gettime(void) +{ + + struct timeval timestamp; + double seconds = 0.0; + double microsec = 0.0; + + gettimeofday(×tamp,NULL); + seconds = (double)timestamp.tv_sec; + microsec = ((double)timestamp.tv_usec)/((double)1000000.0); + + return (seconds+microsec); +} diff --git a/ompi/mca/sharedfp/individual/sharedfp_individual_insert_metadata.c b/ompi/mca/sharedfp/individual/sharedfp_individual_insert_metadata.c new file mode 100644 index 0000000000..c548b91e32 --- /dev/null +++ b/ompi/mca/sharedfp/individual/sharedfp_individual_insert_metadata.c @@ -0,0 +1,145 @@ +/* + * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana + * University Research and Technology + * Corporation. All rights reserved. + * Copyright (c) 2004-2005 The University of Tennessee and The University + * of Tennessee Research Foundation. All rights + * reserved. + * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, + * University of Stuttgart. All rights reserved. + * Copyright (c) 2004-2005 The Regents of the University of California. + * All rights reserved. + * Copyright (c) 2013 University of Houston. All rights reserved. + * $COPYRIGHT$ + * + * Additional copyrights may follow + * + * $HEADER$ + */ + + +#include "ompi_config.h" +#include "sharedfp_individual.h" + +#include "mpi.h" +#include "ompi/constants.h" +#include "orte/util/show_help.h" +#include "ompi/mca/sharedfp/sharedfp.h" + +#include +#include +#include "mpi.h" + +int mca_sharedfp_individual_insert_metadata(int functype,long recordlength,struct mca_sharedfp_base_data_t *sh ) +{ + int ret = OMPI_SUCCESS; + mca_sharedfp_individual_metadata_node *newnode = NULL; + mca_sharedfp_individual_metadata_node *tempnode = NULL; + mca_sharedfp_individual_header_record *headnode = NULL; + + headnode = (mca_sharedfp_individual_header_record*)sh->selected_module_data; + if ( NULL == headnode) { + opal_output (1, "sharedfp_individual_insert_metadat: headnode is NULL but file is open\n"); + return OMPI_ERROR; + } + + + if ( mca_sharedfp_individual_verbose ) { + printf("sharedfp_individual_insert_metadata: Headnode->numofrecords = %d\n", + headnode->numofrecords); + } + /* Check if the maximum limit is reached for the records in the linked list*/ + if (headnode->numofrecords == MAX_METADATA_RECORDS) { + /* Entire linked list is now deleted and a new file*/ + ret = mca_sharedfp_individual_write_metadata_file(sh); + headnode->next = NULL; + } + + /* Allocate a new Node */ + newnode = (mca_sharedfp_individual_metadata_node*)malloc(sizeof(mca_sharedfp_individual_metadata_node)); + if (NULL == newnode) { + opal_output(1,"mca_sharedfp_individual_insert_metadata:Error while allocating new node\n"); + return OMPI_ERR_OUT_OF_RESOURCE; + } + + + /*numofrecords will always carry the number of records present in the metadata linked list*/ + headnode->numofrecords = headnode->numofrecords + 1; + + newnode->recordid = functype; + newnode->timestamp = mca_sharedfp_individual_gettime(); + newnode->localposition = headnode->datafile_offset; /* Datafile offset*/ + newnode->recordlength = recordlength; + newnode->next = NULL; + + + if ( headnode->next == NULL) { + /*headnode allocated but no further metadata node is allocated*/ + headnode->next = newnode; + } + else { + /*We need to append the new node*/ + tempnode = headnode->next; + + while(tempnode->next) { + tempnode = tempnode->next; + } + + tempnode->next = newnode; + } + + return ret; +} + +int mca_sharedfp_individual_write_metadata_file(struct mca_sharedfp_base_data_t *sh) +{ + mca_sharedfp_individual_metadata_node *current = NULL; + struct mca_sharedfp_individual_record2 buff; + mca_sharedfp_individual_header_record *headnode = NULL; + int ret=OMPI_SUCCESS; + MPI_Status status; + + headnode = (mca_sharedfp_individual_header_record*)sh->selected_module_data; + + if (headnode->numofrecordsonfile == 0) + headnode->metadatafile_offset = headnode->metafile_start_offset; + + current = headnode->next; + while (current != NULL) { + /*Read from the linked list*/ + + buff.recordid = current->recordid; + buff.timestamp = current->timestamp; + buff.localposition = current->localposition; + buff.recordlength = current->recordlength; + + if ( mca_sharedfp_individual_verbose ) { + printf("sharedfp_individual_write_metadata_file: Buff recordid %ld\n",buff.recordid); + printf("sharedfp_individual_write_metadata_file: Buff timestamp %f\n", buff.timestamp); + printf("sharedfp_individual_write_metadata_file: Buff localposition %lld\n",buff.localposition); + printf("sharedfp_individual_write_metadata_file: Buff recordlength %ld\n",buff.recordlength); + printf("sharedfp_individual_write_metadata_file: Size of buff %ld\n",sizeof(buff)); + } + + headnode->next = current->next; + free(current); + current = (headnode)->next; + + /*Write to the metadata file*/ + + ret = ompio_io_ompio_file_write_at ( (headnode)->metadatafilehandle, + (headnode)->metadatafile_offset, + &buff,32, MPI_BYTE, &status); + if ( OMPI_SUCCESS != ret ) { + goto exit; + } + (headnode)->numofrecordsonfile = headnode->numofrecordsonfile + 1; + (headnode)->metadatafile_offset = (headnode)->metadatafile_offset + sizeof(buff); + } + + /*All records are being read from the linked list and written to the file*/ + (headnode)->numofrecords = 0; + +exit: + return ret; +} diff --git a/ompi/mca/sharedfp/individual/sharedfp_individual_iwrite.c b/ompi/mca/sharedfp/individual/sharedfp_individual_iwrite.c new file mode 100644 index 0000000000..cf62a773f5 --- /dev/null +++ b/ompi/mca/sharedfp/individual/sharedfp_individual_iwrite.c @@ -0,0 +1,104 @@ +/* + * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana + * University Research and Technology + * Corporation. All rights reserved. + * Copyright (c) 2004-2005 The University of Tennessee and The University + * of Tennessee Research Foundation. All rights + * reserved. + * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, + * University of Stuttgart. All rights reserved. + * Copyright (c) 2004-2005 The Regents of the University of California. + * All rights reserved. + * Copyright (c) 2013 University of Houston. All rights reserved. + * $COPYRIGHT$ + * + * Additional copyrights may follow + * + * $HEADER$ + */ + + +#include "ompi_config.h" +#include "sharedfp_individual.h" + +#include "mpi.h" +#include "ompi/constants.h" +#include "orte/util/show_help.h" +#include "ompi/mca/sharedfp/sharedfp.h" + +int mca_sharedfp_individual_iwrite(mca_io_ompio_file_t *fh, + void *buf, + int count, + ompi_datatype_t *datatype, + MPI_Request * request) +{ + int ret = OMPI_SUCCESS; + size_t numofbytes = 0; + OMPI_MPI_OFFSET_TYPE totalbytes = 0; + mca_sharedfp_individual_header_record *headnode = NULL; + struct mca_sharedfp_base_data_t *sh = NULL; + mca_sharedfp_base_module_t * shared_fp_base_module = NULL; + + if(fh->f_sharedfp_data==NULL){ + if ( mca_sharedfp_individual_verbose ) { + printf("mca_sharedfp_individual_iwrite: opening the shared file pointer\n"); + } + shared_fp_base_module = fh->f_sharedfp; + + ret = shared_fp_base_module->sharedfp_file_open(fh->f_comm, + fh->f_filename, + fh->f_amode, + fh->f_info, + fh); + if (ret != OMPI_SUCCESS) { + opal_output(1,"mca_sharedfp_individual_iwrite - error opening the shared file pointer\n"); + return ret; + } + } + + /* Calculate the number of bytes of data that needs to be written*/ + opal_datatype_type_size ( &datatype->super, &numofbytes); + totalbytes = count * numofbytes; + + sh = fh->f_sharedfp_data; + + headnode = (mca_sharedfp_individual_header_record*)sh->selected_module_data; + if ( NULL == headnode) { + opal_output (1, "sharedfp_individual_iwrite: headnode is NULL but file is open\n"); + return OMPI_ERROR; + } + + /*Insert metadata record into a queue*/ + ret = mca_sharedfp_individual_insert_metadata(OMPI_FILE_WRITE_SHARED,totalbytes,sh); + + + /*Write the data into individual file*/ + ret = ompio_io_ompio_file_iwrite_at ( headnode->datafilehandle, headnode->datafile_offset, + buf, count, datatype, request); + if ( OMPI_SUCCESS != ret ) { + opal_output(1,"sharedfp_individual_iwrite: Error while iwriting the datafile \n"); + return ret; + } + + /* Update the datafileoffset */ + headnode->datafile_offset = headnode->datafile_offset + totalbytes; + + return ret; +} + +int mca_sharedfp_individual_write_ordered_begin(mca_io_ompio_file_t *fh, + void *buf, + int count, + struct ompi_datatype_t *datatype) +{ + opal_output(1,"mca_sharedfp_individual_write_ordered_begin: NOT IMPLEMENTED\n"); + return OMPI_ERROR; +} + +int mca_sharedfp_individual_write_ordered_end(mca_io_ompio_file_t *fh, + void *buf, + ompi_status_public_t *status) +{ + opal_output(1,"mca_sharedfp_individual_write_ordered_end: NOT IMPLEMENTED\n"); + return OMPI_ERROR; +} diff --git a/ompi/mca/sharedfp/individual/sharedfp_individual_read.c b/ompi/mca/sharedfp/individual/sharedfp_individual_read.c new file mode 100644 index 0000000000..f7edccf159 --- /dev/null +++ b/ompi/mca/sharedfp/individual/sharedfp_individual_read.c @@ -0,0 +1,68 @@ +/* + * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana + * University Research and Technology + * Corporation. All rights reserved. + * Copyright (c) 2004-2005 The University of Tennessee and The University + * of Tennessee Research Foundation. All rights + * reserved. + * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, + * University of Stuttgart. All rights reserved. + * Copyright (c) 2004-2005 The Regents of the University of California. + * All rights reserved. + * Copyright (c) 2008 University of Houston. All rights reserved. + * $COPYRIGHT$ + * + * Additional copyrights may follow + * + * $HEADER$ + */ + + +#include "ompi_config.h" +#include "sharedfp_individual.h" + +#include "mpi.h" +#include "ompi/constants.h" +#include "orte/util/show_help.h" +#include "ompi/mca/sharedfp/sharedfp.h" + +int mca_sharedfp_individual_read ( mca_io_ompio_file_t *fh, + void *buf, int count, MPI_Datatype datatype, MPI_Status *status) +{ + opal_output(1,"mca_sharedfp_individual_read: NOT SUPPORTED by this component\n"); + return OMPI_ERROR; +} + +int mca_sharedfp_individual_read_ordered ( mca_io_ompio_file_t *fh, + void *buf, int count, MPI_Datatype datatype, MPI_Status *status) +{ + opal_output(1,"mca_sharedfp_individual_read_ordered: NOT SUPPORTED by this component\n"); + return OMPI_ERROR; +} + +int mca_sharedfp_individual_iread(mca_io_ompio_file_t *fh, + void *buf, + int count, + ompi_datatype_t *datatype, + MPI_Request * request) +{ + opal_output(1,"mca_sharedfp_individual_iread: NOT SUPPORTED by this component\n"); + return OMPI_ERROR; +} + +int mca_sharedfp_individual_read_ordered_begin(mca_io_ompio_file_t *fh, + void *buf, + int count, + struct ompi_datatype_t *datatype) +{ + opal_output(1,"mca_sharedfp_individual_read_ordered_begin: NOT SUPPORTED by this component\n"); + return OMPI_ERROR; +} + +int mca_sharedfp_individual_read_ordered_end(mca_io_ompio_file_t *fh, + void *buf, + ompi_status_public_t *status) +{ + opal_output(1,"mca_sharedfp_individual_read_ordered_end: NOT SUPPORTED by this component\n"); + return OMPI_ERROR; +} diff --git a/ompi/mca/sharedfp/dummy/sharedfp_dummy_seek.c b/ompi/mca/sharedfp/individual/sharedfp_individual_seek.c similarity index 65% rename from ompi/mca/sharedfp/dummy/sharedfp_dummy_seek.c rename to ompi/mca/sharedfp/individual/sharedfp_individual_seek.c index 944d05e09b..f33eace287 100644 --- a/ompi/mca/sharedfp/dummy/sharedfp_dummy_seek.c +++ b/ompi/mca/sharedfp/individual/sharedfp_individual_seek.c @@ -2,14 +2,14 @@ * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana * University Research and Technology * Corporation. All rights reserved. - * Copyright (c) 2004-2011 The University of Tennessee and The University + * Copyright (c) 2004-2005 The University of Tennessee and The University * of Tennessee Research Foundation. All rights * reserved. * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, * University of Stuttgart. All rights reserved. * Copyright (c) 2004-2005 The Regents of the University of California. * All rights reserved. - * Copyright (c) 2008-2011 University of Houston. All rights reserved. + * Copyright (c) 2013 University of Houston. All rights reserved. * $COPYRIGHT$ * * Additional copyrights may follow @@ -19,16 +19,16 @@ #include "ompi_config.h" -#include "sharedfp_dummy.h" +#include "sharedfp_individual.h" #include "mpi.h" #include "ompi/constants.h" +#include "orte/util/show_help.h" #include "ompi/mca/sharedfp/sharedfp.h" -int -mca_sharedfp_dummy_seek (mca_io_ompio_file_t *fh, - OMPI_MPI_OFFSET_TYPE position) +int mca_sharedfp_individual_seek (mca_io_ompio_file_t *fh, + OMPI_MPI_OFFSET_TYPE offset, int whence) { - printf ("DUMMY SEEK\n"); - return OMPI_SUCCESS; + opal_output(1,"mca_sharedfp_individual_seek: NOT IMPLEMENTED\n"); + return OMPI_ERROR; } diff --git a/ompi/mca/sharedfp/individual/sharedfp_individual_write.c b/ompi/mca/sharedfp/individual/sharedfp_individual_write.c new file mode 100644 index 0000000000..ed890ea18d --- /dev/null +++ b/ompi/mca/sharedfp/individual/sharedfp_individual_write.c @@ -0,0 +1,205 @@ +/* + * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana + * University Research and Technology + * Corporation. All rights reserved. + * Copyright (c) 2004-2005 The University of Tennessee and The University + * of Tennessee Research Foundation. All rights + * reserved. + * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, + * University of Stuttgart. All rights reserved. + * Copyright (c) 2004-2005 The Regents of the University of California. + * All rights reserved. + * Copyright (c) 2013 University of Houston. All rights reserved. + * $COPYRIGHT$ + * + * Additional copyrights may follow + * + * $HEADER$ + */ + + +#include "ompi_config.h" +#include "sharedfp_individual.h" + +#include "mpi.h" +#include "ompi/constants.h" +#include "orte/util/show_help.h" +#include "ompi/mca/sharedfp/sharedfp.h" + +int mca_sharedfp_individual_write (mca_io_ompio_file_t *fh, + void *buf, + int count, + struct ompi_datatype_t *datatype, + ompi_status_public_t *status) +{ + int ret = OMPI_SUCCESS; + size_t numofbytes = 0; + size_t totalbytes = 0; + mca_sharedfp_individual_header_record *headnode = NULL; + struct mca_sharedfp_base_data_t *sh = NULL; + mca_sharedfp_base_module_t * shared_fp_base_module = NULL; + + if ( NULL == fh->f_sharedfp_data ) { + if ( mca_sharedfp_individual_verbose ) { + printf("sharedfp_individual_write: opening the shared file pointer file\n"); + } + shared_fp_base_module = fh->f_sharedfp; + + ret = shared_fp_base_module->sharedfp_file_open(fh->f_comm, + fh->f_filename, + fh->f_amode, + fh->f_info, + fh); + if (ret != OMPI_SUCCESS) { + opal_output(1,"sharedfp_individual_write - error opening the shared file pointer\n"); + return ret; + } + } + + /* Calculate the number of bytes of data that need to be written*/ + opal_datatype_type_size ( &datatype->super, &numofbytes); + totalbytes = count * numofbytes; + + /*Retrieve data structure for shared file pointer operations*/ + sh = fh->f_sharedfp_data; + headnode = (mca_sharedfp_individual_header_record*)sh->selected_module_data; + + if (headnode) { + /*Insert metadata record into a queue*/ + mca_sharedfp_individual_insert_metadata(OMPI_FILE_WRITE_SHARED, totalbytes, sh); + + /*Write the data into individual file*/ + ret = ompio_io_ompio_file_write_at ( headnode->datafilehandle, + headnode->datafile_offset, + buf, count, datatype, status); + if ( OMPI_SUCCESS != ret ) { + opal_output(1,"mca_sharedfp_individual_write: Error while writing the datafile \n"); + return -1; + } + + /* Update the datafileoffset*/ + headnode->datafile_offset = headnode->datafile_offset + totalbytes; + } + + return ret; +} + +int mca_sharedfp_individual_write_ordered (mca_io_ompio_file_t *fh, + void *buf, + int count, + struct ompi_datatype_t *datatype, + ompi_status_public_t *status) +{ + int ret = OMPI_SUCCESS; + int size = 0, rank = 0; + int i = 0; + size_t numofbytes = 0; + size_t totalbytes = 0; + OMPI_MPI_OFFSET_TYPE *offbuff=NULL; + OMPI_MPI_OFFSET_TYPE global_offset = 0; + OMPI_MPI_OFFSET_TYPE prev_offset = 0; + OMPI_MPI_OFFSET_TYPE temp = 0, offset = 0; + mca_sharedfp_individual_header_record *headnode = NULL; + struct mca_sharedfp_base_data_t *sh = NULL; + mca_sharedfp_base_module_t * shared_fp_base_module = NULL; + + if(fh->f_sharedfp_data==NULL){ + if ( mca_sharedfp_individual_verbose ) { + printf("sharedfp_individual_write - opening the shared file pointer\n"); + } + shared_fp_base_module = fh->f_sharedfp; + + ret = shared_fp_base_module->sharedfp_file_open(fh->f_comm, + fh->f_filename, + fh->f_amode, + fh->f_info, + fh); + if ( OMPI_SUCCESS != ret ) { + opal_output(1,"sharedfp_individual_write - error opening the shared file pointer\n"); + return ret; + } + } + + /*Retrieve the sharedfp data structures*/ + sh = fh->f_sharedfp_data; + rank = ompi_comm_rank ( sh->comm ); + size = ompi_comm_size ( sh->comm ); + + /* Calculate the number of bytes of data that needs to be written*/ + opal_datatype_type_size ( &datatype->super, &numofbytes); + totalbytes = count * numofbytes; + + headnode = (mca_sharedfp_individual_header_record*)sh->selected_module_data; + if ( NULL == headnode) { + opal_output (1, "sharedfp_individual_write_ordered: headnode is NULL but file is open\n"); + return OMPI_ERROR; + } + + /* Data from all the metadata is combined and written to the main file */ + ret = mca_sharedfp_individual_collaborate_data ( sh ); + if ( OMPI_SUCCESS != ret) { + return ret; + } + + if ( 0 == rank ) { + offbuff = (OMPI_MPI_OFFSET_TYPE *)malloc ( sizeof(OMPI_MPI_OFFSET_TYPE) * size); + if (NULL == offbuff ) { + return OMPI_ERR_OUT_OF_RESOURCE; + } + } + + /*collect the total bytes to be written*/ + sh->comm->c_coll.coll_gather ( &totalbytes, 1, OMPI_OFFSET_DATATYPE, + offbuff, 1, OMPI_OFFSET_DATATYPE, 0, + sh->comm, sh->comm->c_coll.coll_gather_module ); + + if ( 0 == rank ) { + for (i = 0; i < size; i++){ + prev_offset = offbuff[0]; + offbuff[0] = sh->global_offset; + + for (i = 1; i < size ; i++){ + temp = offbuff[i]; + offbuff[i] = offbuff[i - 1] + prev_offset; + prev_offset = temp; + } + + for (i = 0; i < size; i++){ + global_offset = offbuff[size - 1] + prev_offset; + } + + } + } + + + /* Scatter the results to the other processes */ + ret = sh->comm->c_coll.coll_scatter ( offbuff, 1, OMPI_OFFSET_DATATYPE, + &offset, 1, OMPI_OFFSET_DATATYPE, 0, + sh->comm, sh->comm->c_coll.coll_scatter_module ); + if ( OMPI_SUCCESS != ret ) { + opal_output(1,"sharedfp_individual_write_ordered: Error in scattering offsets \n"); + goto exit; + } + + ret = sh->comm->c_coll.coll_bcast ( &global_offset, 1, OMPI_OFFSET_DATATYPE, + 0, sh->comm, sh->comm->c_coll.coll_bcast_module ); + if ( OMPI_SUCCESS != ret ) { + opal_output(1,"sharedfp_individual_write_ordered: Error while bcasting global offset \n"); + goto exit; + } + + sh->global_offset = global_offset; + + /*use file_write_at_all to ensure the order*/ + ret = ompio_io_ompio_file_write_at_all(sh->sharedfh,offset, buf,count,datatype,status); + if ( OMPI_SUCCESS != ret ) { + opal_output(1,"sharedfp_individual_write_ordered: Error while writing the datafile \n"); + } + +exit: + if ( NULL != offbuff ) { + free ( offbuff); + } + + return ret; +} diff --git a/ompi/mca/sharedfp/lockedfile/Makefile.am b/ompi/mca/sharedfp/lockedfile/Makefile.am new file mode 100644 index 0000000000..a072955582 --- /dev/null +++ b/ompi/mca/sharedfp/lockedfile/Makefile.am @@ -0,0 +1,55 @@ +# +# Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana +# University Research and Technology +# Corporation. All rights reserved. +# Copyright (c) 2004-2005 The University of Tennessee and The University +# of Tennessee Research Foundation. All rights +# reserved. +# Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, +# University of Stuttgart. All rights reserved. +# Copyright (c) 2004-2005 The Regents of the University of California. +# All rights reserved. +# Copyright (c) 2008 University of Houston. All rights reserved. +# $COPYRIGHT$ +# +# Additional copyrights may follow +# +# $HEADER$ +# + +# Make the output library in this directory, and name it either +# mca__.la (for DSO builds) or libmca__.la +# (for static builds). + +if MCA_BUILD_ompi_sharedfp_lockedfile_DSO +component_noinst = +component_install = mca_sharedfp_lockedfile.la +else +component_noinst = libmca_sharedfp_lockedfile.la +component_install = +endif + +mcacomponentdir = $(pkglibdir) +mcacomponent_LTLIBRARIES = $(component_install) +mca_sharedfp_lockedfile_la_SOURCES = $(sources) +mca_sharedfp_lockedfile_la_LDFLAGS = -module -avoid-version + +noinst_LTLIBRARIES = $(component_noinst) +libmca_sharedfp_lockedfile_la_SOURCES = $(sources) +libmca_sharedfp_lockedfile_la_LDFLAGS = -module -avoid-version + +# Source files + +#IMPORTANT: Update here when adding new source code files to the library +sources = \ + sharedfp_lockedfile.h \ + sharedfp_lockedfile.c \ + sharedfp_lockedfile_component.c \ + sharedfp_lockedfile_seek.c \ + sharedfp_lockedfile_get_position.c \ + sharedfp_lockedfile_request_position.c \ + sharedfp_lockedfile_write.c \ + sharedfp_lockedfile_iwrite.c \ + sharedfp_lockedfile_read.c \ + sharedfp_lockedfile_iread.c \ + sharedfp_lockedfile_file_open.c diff --git a/ompi/mca/sharedfp/lockedfile/sharedfp_lockedfile.c b/ompi/mca/sharedfp/lockedfile/sharedfp_lockedfile.c new file mode 100644 index 0000000000..b3dcf97e7a --- /dev/null +++ b/ompi/mca/sharedfp/lockedfile/sharedfp_lockedfile.c @@ -0,0 +1,188 @@ +/* + * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana + * University Research and Technology + * Corporation. All rights reserved. + * Copyright (c) 2004-2006 The University of Tennessee and The University + * of Tennessee Research Foundation. All rights + * reserved. + * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, + * University of Stuttgart. All rights reserved. + * Copyright (c) 2004-2005 The Regents of the University of California. + * All rights reserved. + * Copyright (c) 2013 University of Houston. All rights reserved. + * $COPYRIGHT$ + * + * Additional copyrights may follow + * + * $HEADER$ + * + * These symbols are in a file by themselves to provide nice linker + * semantics. Since linkers generally pull in symbols by object fules, + * keeping these symbols as the only symbols in this file prevents + * utility programs such as "ompi_info" from having to import entire + * modules just to query their version and parameters + */ + +#include "ompi_config.h" +#include "mpi.h" +#include "ompi/mca/sharedfp/sharedfp.h" +#include "ompi/mca/sharedfp/lockedfile/sharedfp_lockedfile.h" + +/* included so that we can test file locking availability */ +#include +#include +#include + +/* + * ******************************************************************* + * ************************ actions structure ************************ + * ******************************************************************* + */ + /* IMPORTANT: Update here when adding sharedfp component interface functions*/ +static mca_sharedfp_base_module_1_0_0_t lockedfile = { + mca_sharedfp_lockedfile_module_init, /* initalise after being selected */ + mca_sharedfp_lockedfile_module_finalize, /* close a module on a communicator */ + mca_sharedfp_lockedfile_seek, + mca_sharedfp_lockedfile_get_position, + mca_sharedfp_lockedfile_read, + mca_sharedfp_lockedfile_read_ordered, + mca_sharedfp_lockedfile_read_ordered_begin, + mca_sharedfp_lockedfile_read_ordered_end, + mca_sharedfp_lockedfile_iread, + mca_sharedfp_lockedfile_write, + mca_sharedfp_lockedfile_write_ordered, + mca_sharedfp_lockedfile_write_ordered_begin, + mca_sharedfp_lockedfile_write_ordered_end, + mca_sharedfp_lockedfile_iwrite, + mca_sharedfp_lockedfile_file_open, + mca_sharedfp_lockedfile_file_close +}; +/* + * ******************************************************************* + * ************************* structure ends ************************** + * ******************************************************************* + */ + +int mca_sharedfp_lockedfile_component_init_query(bool enable_progress_threads, + bool enable_mpi_threads) +{ + /* Nothing to do */ + + return OMPI_SUCCESS; +} + +struct mca_sharedfp_base_module_1_0_0_t * mca_sharedfp_lockedfile_component_file_query(mca_io_ompio_file_t *fh, int *priority) { + struct flock lock; + int fd, err; + /*char *filename;*/ + char filename[256]; + int rank; + bool has_file_lock_support=false; + + *priority = mca_sharedfp_lockedfile_priority; + + /*get the rank of this process*/ + rank = ompi_comm_rank ( fh->f_comm); + + /*test, and update priority*/ + /* + * This test tests to see if fcntl returns success when asked to + * establish a file lock. This test is intended for use on file systems + * such as NFS that may not implement file locks. The locked file algorithm makes use + * of file locks to implement the shared file pointer operations, and will not work + * properly if file locks are not available. + * + * This is a simple test and has at least two limitations: + * + * 1. Some implementations of NFS are known to return success for + * setting a file lock when in fact no lock has been set. This + * test will not detect such erroneous implementations of NFS + * + * 2. Some implementations will hang (enter and wait indefinitately) + * within the fcntl call. This test will also hang in that case. + * Under normal conditions, this test should only take a few seconds to + * run. + * + * The test prints a message showing the success or failure of + * setting the file lock if PRINT_TAG is set to true in sharedfp.h. + * It also sets the priority to a non-zero value on success and + * returns NULL on failure. If there is a failure, the system routine + * strerror is interogated in order to print the reason. + */ + + /* Set the filename. */ + /*data filename created by appending .locktest.$rank to the original filename*/ + sprintf(filename,"%s%s%d",fh->f_filename,".locktest.",rank); + + lock.l_type = F_WRLCK; + lock.l_start = 0; + lock.l_whence = SEEK_SET; + lock.l_len = 100; + lock.l_pid = getpid(); + + fd = open(filename, O_RDWR | O_CREAT, 0644); + + if ( -1 == fd ){ + opal_output(1,"mca_sharedfp_lockedfile_component_file_query: error opening file %s", filename); + opal_output(1,"%s\n", strerror(errno)); + has_file_lock_support=false; + } + else{ + err = fcntl(fd, F_SETLKW, &lock); + if ( mca_sharedfp_lockedfile_verbose ) { + printf("mca_sharedfp_lockedfile_component_file_query: returned err=%d, for fd=%d\n",err,fd); + } + + if (err) { + opal_output(1, "mca_sharedfp_lockedfile_component_file_query: Failed to set a file lock on %s\n", filename ); + opal_output(1, "err=%d, errno=%d, EOPNOTSUPP=%d, EINVAL=%d, ENOSYS=%d, EACCES=%d, EAGAIN=%d, EBADF=%d\n", + err,errno,EOPNOTSUPP,EINVAL,ENOSYS,EACCES,EAGAIN,EBADF); + opal_output(1,"%s\n", strerror(errno)); + + if (errno == EACCES || errno == EAGAIN) { + opal_output(1,"errno=EACCES || EAGAIN, Already locked by another process\n"); + } + + } + else { + if ( mca_sharedfp_lockedfile_verbose ) { + printf( "mca_sharedfp_lockedfile_component_file_query: fcntl claims success in setting a file lock on %s\n", filename ); + } + + has_file_lock_support=true; + } + /* printf("err = %d, errno = %d\n", err, errno); */ + close(fd); + unlink( filename ); + } + /**priority=100;*/ + if(has_file_lock_support){ + return &lockedfile; + } + + *priority = 0; + /*module can not run!, return NULL to indicate that we are unable to run*/ + + opal_output(1,"mca_sharedfp_lockedfile_component_file_query: Can not run!, file locking not supported\n"); + return NULL; +} + +int mca_sharedfp_lockedfile_component_file_unquery (mca_io_ompio_file_t *file) +{ + /* This function might be needed for some purposes later. for now it + * does not have anything to do since there are no steps which need + * to be undone if this module is not selected */ + + return OMPI_SUCCESS; +} + +int mca_sharedfp_lockedfile_module_init (mca_io_ompio_file_t *file) +{ + return OMPI_SUCCESS; +} + + +int mca_sharedfp_lockedfile_module_finalize (mca_io_ompio_file_t *file) +{ + return OMPI_SUCCESS; +} diff --git a/ompi/mca/sharedfp/lockedfile/sharedfp_lockedfile.h b/ompi/mca/sharedfp/lockedfile/sharedfp_lockedfile.h new file mode 100644 index 0000000000..c1c4b0e90f --- /dev/null +++ b/ompi/mca/sharedfp/lockedfile/sharedfp_lockedfile.h @@ -0,0 +1,129 @@ +/* + * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana + * University Research and Technology + * Corporation. All rights reserved. + * Copyright (c) 2004-2006 The University of Tennessee and The University + * of Tennessee Research Foundation. All rights + * reserved. + * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, + * University of Stuttgart. All rights reserved. + * Copyright (c) 2004-2005 The Regents of the University of California. + * All rights reserved. + * Copyright (c) 2013 University of Houston. All rights reserved. + * $COPYRIGHT$ + * + * Additional copyrights may follow + * + * $HEADER$ + */ + +#ifndef MCA_SHAREDFP_LOCKEDFILE_H +#define MCA_SHAREDFP_LOCKEDFILE_H + +#include "ompi_config.h" +#include "opal/mca/mca.h" +#include "ompi/mca/sharedfp/sharedfp.h" +#include "ompi/mca/io/ompio/io_ompio.h" + + +BEGIN_C_DECLS + +int mca_sharedfp_lockedfile_component_init_query(bool enable_progress_threads, + bool enable_mpi_threads); +struct mca_sharedfp_base_module_1_0_0_t * + mca_sharedfp_lockedfile_component_file_query (mca_io_ompio_file_t *file, int *priority); +int mca_sharedfp_lockedfile_component_file_unquery (mca_io_ompio_file_t *file); + +int mca_sharedfp_lockedfile_module_init (mca_io_ompio_file_t *file); +int mca_sharedfp_lockedfile_module_finalize (mca_io_ompio_file_t *file); + +extern int mca_sharedfp_lockedfile_priority; +extern int mca_sharedfp_lockedfile_verbose; + +OMPI_MODULE_DECLSPEC extern mca_sharedfp_base_component_2_0_0_t mca_sharedfp_lockedfile_component; +/* + * ****************************************************************** + * ********* functions which are implemented in this module ********* + * ****************************************************************** + */ +/*IMPORANT: Update here when implementing functions from sharedfp API*/ + +int mca_sharedfp_lockedfile_seek (mca_io_ompio_file_t *fh, + OMPI_MPI_OFFSET_TYPE offset, int whence); +int mca_sharedfp_lockedfile_get_position (mca_io_ompio_file_t *fh, + OMPI_MPI_OFFSET_TYPE * offset); +int mca_sharedfp_lockedfile_file_open (struct ompi_communicator_t *comm, + char* filename, + int amode, + struct ompi_info_t *info, + mca_io_ompio_file_t *fh); +int mca_sharedfp_lockedfile_file_close (mca_io_ompio_file_t *fh); +int mca_sharedfp_lockedfile_read (mca_io_ompio_file_t *fh, + void *buf, int count, MPI_Datatype datatype, MPI_Status *status); +int mca_sharedfp_lockedfile_read_ordered (mca_io_ompio_file_t *fh, + void *buf, int count, struct ompi_datatype_t *datatype, + ompi_status_public_t *status + ); +int mca_sharedfp_lockedfile_read_ordered_begin (mca_io_ompio_file_t *fh, + void *buf, + int count, + struct ompi_datatype_t *datatype); +int mca_sharedfp_lockedfile_read_ordered_end (mca_io_ompio_file_t *fh, + void *buf, + ompi_status_public_t *status); +int mca_sharedfp_lockedfile_iread (mca_io_ompio_file_t *fh, + void *buf, + int count, + struct ompi_datatype_t *datatype, + ompi_request_t **request); +int mca_sharedfp_lockedfile_write (mca_io_ompio_file_t *fh, + void *buf, + int count, + struct ompi_datatype_t *datatype, + ompi_status_public_t *status); +int mca_sharedfp_lockedfile_write_ordered (mca_io_ompio_file_t *fh, + void *buf, + int count, + struct ompi_datatype_t *datatype, + ompi_status_public_t *status); +int mca_sharedfp_lockedfile_write_ordered_begin (mca_io_ompio_file_t *fh, + void *buf, + int count, + struct ompi_datatype_t *datatype); +int mca_sharedfp_lockedfile_write_ordered_end (mca_io_ompio_file_t *fh, + void *buf, + ompi_status_public_t *status); +int mca_sharedfp_lockedfile_iwrite (mca_io_ompio_file_t *fh, + void *buf, + int count, + struct ompi_datatype_t *datatype, + ompi_request_t **request); + +/*--------------------------------------------------------------* + *Structures and definitions only for this component + *--------------------------------------------------------------*/ + +/*This structure will hang off of the mca_sharedfp_base_data_t's + *selected_module_data attribute + */ +struct mca_sharedfp_lockedfile_data +{ + int handle; + char* filename; +}; + +typedef struct mca_sharedfp_lockedfile_data lockedfile_data; + + +int mca_sharedfp_lockedfile_request_position (struct mca_sharedfp_base_data_t * sh, + int bytes_requested, + OMPI_MPI_OFFSET_TYPE * offset); +/* + * ****************************************************************** + * ************ functions implemented in this module end ************ + * ****************************************************************** + */ + +END_C_DECLS + +#endif /* MCA_SHAREDFP_LOCKEDFILE_H */ diff --git a/ompi/mca/sharedfp/lockedfile/sharedfp_lockedfile_component.c b/ompi/mca/sharedfp/lockedfile/sharedfp_lockedfile_component.c new file mode 100644 index 0000000000..3e939f5ca8 --- /dev/null +++ b/ompi/mca/sharedfp/lockedfile/sharedfp_lockedfile_component.c @@ -0,0 +1,88 @@ +/* + * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana + * University Research and Technology + * Corporation. All rights reserved. + * Copyright (c) 2004-2005 The University of Tennessee and The University + * of Tennessee Research Foundation. All rights + * reserved. + * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, + * University of Stuttgart. All rights reserved. + * Copyright (c) 2004-2005 The Regents of the University of California. + * All rights reserved. + * Copyright (c) 2013 University of Houston. All rights reserved. + * $COPYRIGHT$ + * + * Additional copyrights may follow + * + * $HEADER$ + * + * These symbols are in a file by themselves to provide nice linker + * semantics. Since linkers generally pull in symbols by object + * files, keeping these symbols as the only symbols in this file + * prevents utility programs such as "ompi_info" from having to import + * entire components just to query their version and parameters. + */ + +#include "ompi_config.h" +#include "sharedfp_lockedfile.h" +#include "mpi.h" + +/* + * Public string showing the sharedfp lockedfile component version number + */ +const char *mca_sharedfp_lockedfile_component_version_string = + "OMPI/MPI lockedfile SHAREDFP MCA component version " OMPI_VERSION; +/* + * Global variables + */ +int mca_sharedfp_lockedfile_priority=10; +int mca_sharedfp_lockedfile_verbose=0; + +static int lockedfile_register(void); + +/* + * Instantiate the public struct with all of our public information + * and pointers to our public functions in it + */ +mca_sharedfp_base_component_2_0_0_t mca_sharedfp_lockedfile_component = { + + /* First, the mca_component_t struct containing meta information + about the component itself */ + + { + MCA_SHAREDFP_BASE_VERSION_2_0_0, + + /* Component name and version */ + "lockedfile", + OMPI_MAJOR_VERSION, + OMPI_MINOR_VERSION, + OMPI_RELEASE_VERSION, + lockedfile_register, + NULL + }, + { + /* This component is checkpointable */ + MCA_BASE_METADATA_PARAM_CHECKPOINT + }, + mca_sharedfp_lockedfile_component_init_query, /* get thread level */ + mca_sharedfp_lockedfile_component_file_query, /* get priority and actions */ + mca_sharedfp_lockedfile_component_file_unquery /* undo what was done by previous function */ +}; + +static int lockedfile_register(void) +{ + mca_sharedfp_lockedfile_priority = 10; + (void) mca_base_component_var_register(&mca_sharedfp_lockedfile_component.sharedfpm_version, + "priority", "Priority of the lockedfile sharedfp component", + MCA_BASE_VAR_TYPE_INT, NULL, 0, 0, + OPAL_INFO_LVL_9, + MCA_BASE_VAR_SCOPE_READONLY, &mca_sharedfp_lockedfile_priority); + mca_sharedfp_lockedfile_verbose = 0; + (void) mca_base_component_var_register(&mca_sharedfp_lockedfile_component.sharedfpm_version, + "verbose", "Verbosity of the lockedfile sharedfp component", + MCA_BASE_VAR_TYPE_INT, NULL, 0, 0, + OPAL_INFO_LVL_9, + MCA_BASE_VAR_SCOPE_READONLY, &mca_sharedfp_lockedfile_verbose); + + return OMPI_SUCCESS; +} diff --git a/ompi/mca/sharedfp/lockedfile/sharedfp_lockedfile_file_open.c b/ompi/mca/sharedfp/lockedfile/sharedfp_lockedfile_file_open.c new file mode 100644 index 0000000000..f91a453cd0 --- /dev/null +++ b/ompi/mca/sharedfp/lockedfile/sharedfp_lockedfile_file_open.c @@ -0,0 +1,161 @@ +/* + * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana + * University Research and Technology + * Corporation. All rights reserved. + * Copyright (c) 2004-2005 The University of Tennessee and The University + * of Tennessee Research Foundation. All rights + * reserved. + * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, + * University of Stuttgart. All rights reserved. + * Copyright (c) 2004-2005 The Regents of the University of California. + * All rights reserved. + * Copyright (c) 2013 University of Houston. All rights reserved. + * $COPYRIGHT$ + * + * Additional copyrights may follow + * + * $HEADER$ + */ + + +#include "ompi_config.h" +#include "sharedfp_lockedfile.h" + +#include "mpi.h" +#include "ompi/constants.h" +#include "orte/util/show_help.h" +#include "ompi/mca/sharedfp/sharedfp.h" +#include "ompi/mca/sharedfp/base/base.h" + +#ifdef HAVE_SYS_STAT_H +#include +#endif +#include + +int mca_sharedfp_lockedfile_file_open (struct ompi_communicator_t *comm, + char* filename, + int amode, + struct ompi_info_t *info, + mca_io_ompio_file_t *fh) +{ + int err = MPI_SUCCESS; + char * lockedfilename; + int handle, rank; + struct mca_sharedfp_lockedfile_data * module_data = NULL; + struct mca_sharedfp_base_data_t* sh; + mca_io_ompio_file_t * shfileHandle; + + /*------------------------------------------------------------*/ + /*Open the same file again without shared file pointer support*/ + /*------------------------------------------------------------*/ + shfileHandle = (mca_io_ompio_file_t *)malloc(sizeof(mca_io_ompio_file_t)); + err = ompio_io_ompio_file_open(comm,filename,amode,info,shfileHandle,false); + if ( OMPI_SUCCESS != err) { + opal_output(1, "mca_sharedfp_lockedfile_file_open: Error during file open\n"); + return err; + } + + /*Memory is allocated here for the sh structure*/ + sh = (struct mca_sharedfp_base_data_t*)malloc(sizeof(struct mca_sharedfp_base_data_t)); + if ( NULL == sh){ + opal_output(1, "mca_sharedfp_lockedfile_file_open: Error, unable to malloc f_sharedfp_ptr struct\n"); + return OMPI_ERR_OUT_OF_RESOURCE; + } + /*Populate the sh file structure based on the implementation*/ + sh->sharedfh = shfileHandle; /* Shared file pointer*/ + sh->global_offset = 0; /* Global Offset*/ + sh->comm = comm; /* Communicator*/ + sh->selected_module_data = NULL; + + rank = ompi_comm_rank ( sh->comm); + + /*Open a new file which will maintain the pointer for this file open*/ + if ( mca_sharedfp_lockedfile_verbose ) { + printf("mca_sharedfp_lockedfile_file_open: open locked file.\n"); + } + + + module_data = (struct mca_sharedfp_lockedfile_data*)malloc(sizeof(struct mca_sharedfp_lockedfile_data)); + if ( NULL == module_data ) { + printf("mca_sharedfp_lockedfile_file_open: Error, unable to malloc lockedfile_data struct\n"); + return OMPI_ERR_OUT_OF_RESOURCE; + } + + lockedfilename = (char*)malloc(sizeof(char) * (strlen(filename) + 64)); + sprintf(lockedfilename,"%s%s",filename,".lockedfile"); + module_data->filename = lockedfilename; + + /*-------------------------------------------------*/ + /*Open the lockedfile without shared file pointer */ + /*-------------------------------------------------*/ + if ( 0 == rank ) { + OMPI_MPI_OFFSET_TYPE position=0; + /*only let main process initialize file pointer, + *therefore there is no need to lock the file + */ + handle = open ( lockedfilename, O_RDWR | O_CREAT, 0644 ); + write ( handle, &position, sizeof(OMPI_MPI_OFFSET_TYPE) ); + close ( handle ); + } + comm->c_coll.coll_barrier ( comm, comm->c_coll.coll_barrier_module ); + + handle = open ( lockedfilename, O_RDWR, 0644 ); + if ( -1 == handle ) { + printf("[%d]mca_sharedfp_lockedfile_file_open: Error during file open\n", rank); + free(module_data); + return OMPI_ERROR; + } + + /*Store the new file handle*/ + module_data->handle = handle; + /* Assign the lockedfile_data to sh->handle*/ + sh->selected_module_data = module_data; + /*remember the shared file handle*/ + fh->f_sharedfp_data = sh; + + comm->c_coll.coll_barrier ( comm, comm->c_coll.coll_barrier_module ); + + return err; +} + +int mca_sharedfp_lockedfile_file_close (mca_io_ompio_file_t *fh) +{ + int err = OMPI_SUCCESS; + struct mca_sharedfp_lockedfile_data * module_data = NULL; + struct mca_sharedfp_base_data_t *sh; + int rank = ompi_comm_rank ( fh->f_comm ); + + if ( fh->f_sharedfp_data==NULL){ + /* Can happen with lazy_open being set */ + if ( mca_sharedfp_lockedfile_verbose ) { + printf("sharedfp_lockedfile_file_close - shared file pointer structure not initialized\n"); + } + return OMPI_SUCCESS; + } + sh = fh->f_sharedfp_data; + + module_data = (lockedfile_data*)(sh->selected_module_data); + if ( module_data) { + /*Close lockedfile handle*/ + if ( module_data->handle) { + close (module_data->handle ); + if ( 0 == rank ) { + unlink ( module_data->filename); + } + } + if ( NULL != module_data->filename ){ + free ( module_data->filename); + } + free ( module_data ); + } + + /* Close the main file opened by this component*/ + err = ompio_io_ompio_file_close(sh->sharedfh); + + /*free shared file pointer data struct*/ + free(sh); + + return err; + +} + diff --git a/ompi/mca/sharedfp/lockedfile/sharedfp_lockedfile_get_position.c b/ompi/mca/sharedfp/lockedfile/sharedfp_lockedfile_get_position.c new file mode 100644 index 0000000000..a8a8572d93 --- /dev/null +++ b/ompi/mca/sharedfp/lockedfile/sharedfp_lockedfile_get_position.c @@ -0,0 +1,62 @@ +/* + * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana + * University Research and Technology + * Corporation. All rights reserved. + * Copyright (c) 2004-2005 The University of Tennessee and The University + * of Tennessee Research Foundation. All rights + * reserved. + * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, + * University of Stuttgart. All rights reserved. + * Copyright (c) 2004-2005 The Regents of the University of California. + * All rights reserved. + * Copyright (c) 2008 University of Houston. All rights reserved. + * $COPYRIGHT$ + * + * Additional copyrights may follow + * + * $HEADER$ + */ + + +#include "ompi_config.h" +#include "sharedfp_lockedfile.h" + +#include "mpi.h" +#include "ompi/constants.h" +#include "orte/util/show_help.h" +#include "ompi/mca/sharedfp/sharedfp.h" + +int +mca_sharedfp_lockedfile_get_position(mca_io_ompio_file_t *fh, + OMPI_MPI_OFFSET_TYPE * offset) +{ + int ret = OMPI_SUCCESS; + mca_sharedfp_base_module_t * shared_fp_base_module; + struct mca_sharedfp_base_data_t *sh = NULL; + opal_output(1,"sharedfp_lockedfile_get_position\n"); + + + if(fh->f_sharedfp_data==NULL){ + opal_output(1,"sharedfp_lockedfile_get_position - opening the shared file pointer\n"); + shared_fp_base_module = fh->f_sharedfp; + + ret = shared_fp_base_module->sharedfp_file_open(fh->f_comm, + fh->f_filename, + fh->f_amode, + fh->f_info, + fh); + if (ret != OMPI_SUCCESS) { + opal_output(1,"sharedfp_lockedfile_write - error opening the shared file pointer\n"); + return ret; + } + } + /*Retrieve the shared file data struct*/ + sh = fh->f_sharedfp_data; + + /*Requesting the offset to write 0 bytes, + *returns the current offset w/o updating it + */ + ret = mca_sharedfp_lockedfile_request_position(sh,0,offset); + + return ret; +} diff --git a/ompi/mca/sharedfp/lockedfile/sharedfp_lockedfile_iread.c b/ompi/mca/sharedfp/lockedfile/sharedfp_lockedfile_iread.c new file mode 100644 index 0000000000..e780f37329 --- /dev/null +++ b/ompi/mca/sharedfp/lockedfile/sharedfp_lockedfile_iread.c @@ -0,0 +1,102 @@ +/* + * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana + * University Research and Technology + * Corporation. All rights reserved. + * Copyright (c) 2004-2005 The University of Tennessee and The University + * of Tennessee Research Foundation. All rights + * reserved. + * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, + * University of Stuttgart. All rights reserved. + * Copyright (c) 2004-2005 The Regents of the University of California. + * All rights reserved. + * Copyright (c) 2013 University of Houston. All rights reserved. + * $COPYRIGHT$ + * + * Additional copyrights may follow + * + * $HEADER$ + */ + + +#include "ompi_config.h" +#include "sharedfp_lockedfile.h" + +#include "mpi.h" +#include "ompi/constants.h" +#include "orte/util/show_help.h" +#include "ompi/mca/sharedfp/sharedfp.h" +#include "ompi/mca/io/ompio/io_ompio.h" + +int mca_sharedfp_lockedfile_iread(mca_io_ompio_file_t *fh, + void *buf, + int count, + ompi_datatype_t *datatype, + MPI_Request * request) +{ + int ret = OMPI_SUCCESS; + mca_sharedfp_base_module_t * shared_fp_base_module; + OMPI_MPI_OFFSET_TYPE offset = 0; + long bytesRequested = 0; + size_t numofBytes; + struct mca_sharedfp_base_data_t *sh = NULL; + + if ( NULL == fh->f_sharedfp_data ) { + if ( mca_sharedfp_lockedfile_verbose ) { + printf("sharedfp_lockedfile_iread: opening the shared file pointer\n"); + } + shared_fp_base_module = fh->f_sharedfp; + + ret = shared_fp_base_module->sharedfp_file_open(fh->f_comm, + fh->f_filename, + fh->f_amode, + fh->f_info, + fh); + if ( OMPI_SUCCESS != ret ) { + opal_output(1,"sharedfp_lockedfile_iread - error opening the shared file pointer\n"); + return ret; + } + } + + /* Calculate the number of bytes to read */ + opal_datatype_type_size ( &datatype->super, &numofBytes); + bytesRequested = count * numofBytes; + + if ( mca_sharedfp_lockedfile_verbose ) { + printf("sharedfp_lockedfile_iread - Bytes Requested is %ld\n",bytesRequested); + } + + + /*Retrieve the shared file data struct*/ + sh = fh->f_sharedfp_data; + + /*Request the offset to write bytesRequested bytes*/ + ret = mca_sharedfp_lockedfile_request_position(sh,bytesRequested,&offset); + if ( -1 != ret ) { + if ( mca_sharedfp_lockedfile_verbose ) { + printf("sharedfp_lockedfile_iread - Offset received is %lld\n",offset); + } + + /* Read the file */ + ret = ompio_io_ompio_file_iread_at(sh->sharedfh,offset,buf,count,datatype,request); + } + + return ret; +} + +int mca_sharedfp_lockedfile_read_ordered_begin(mca_io_ompio_file_t *fh, + void *buf, + int count, + struct ompi_datatype_t *datatype) +{ + opal_output(1,"mca_sharedfp_lockedfile_write_ordered_begin: NOT IMPLEMENTED\n"); + return OMPI_ERROR; +} + + +int mca_sharedfp_lockedfile_read_ordered_end(mca_io_ompio_file_t *fh, + void *buf, + ompi_status_public_t *status) +{ + opal_output(1,"mca_sharedfp_lockedfile_write_ordered_end: NOT IMPLEMENTED\n"); + return OMPI_ERROR; +} diff --git a/ompi/mca/sharedfp/lockedfile/sharedfp_lockedfile_iwrite.c b/ompi/mca/sharedfp/lockedfile/sharedfp_lockedfile_iwrite.c new file mode 100644 index 0000000000..fb11dc00cf --- /dev/null +++ b/ompi/mca/sharedfp/lockedfile/sharedfp_lockedfile_iwrite.c @@ -0,0 +1,101 @@ +/* + * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana + * University Research and Technology + * Corporation. All rights reserved. + * Copyright (c) 2004-2005 The University of Tennessee and The University + * of Tennessee Research Foundation. All rights + * reserved. + * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, + * University of Stuttgart. All rights reserved. + * Copyright (c) 2004-2005 The Regents of the University of California. + * All rights reserved. + * Copyright (c) 2013 University of Houston. All rights reserved. + * $COPYRIGHT$ + * + * Additional copyrights may follow + * + * $HEADER$ + */ + + +#include "ompi_config.h" +#include "sharedfp_lockedfile.h" + +#include "mpi.h" +#include "ompi/constants.h" +#include "orte/util/show_help.h" +#include "ompi/mca/sharedfp/sharedfp.h" +#include "ompi/mca/io/ompio/io_ompio.h" + +int mca_sharedfp_lockedfile_iwrite(mca_io_ompio_file_t *fh, + void *buf, + int count, + ompi_datatype_t *datatype, + MPI_Request * request) +{ + int ret = OMPI_SUCCESS; + mca_sharedfp_base_module_t * shared_fp_base_module; + OMPI_MPI_OFFSET_TYPE offset = 0; + long bytesRequested = 0; + size_t numofBytes; + struct mca_sharedfp_base_data_t *sh = NULL; + + if(fh->f_sharedfp_data==NULL){ + if ( mca_sharedfp_lockedfile_verbose ) { + printf("sharedfp_lockedfile_iwrite: opening the shared file pointer\n"); + } + shared_fp_base_module = fh->f_sharedfp; + + ret = shared_fp_base_module->sharedfp_file_open(fh->f_comm, + fh->f_filename, + fh->f_amode, + fh->f_info, + fh); + if ( OMPI_SUCCESS != ret ) { + opal_output(1,"sharedfp_lockedfile_iwrite - error opening the shared file pointer\n"); + return ret; + } + } + + /*Calculate the number of bytes to write*/ + opal_datatype_type_size ( &datatype->super, &numofBytes); + bytesRequested = count * numofBytes; + if ( mca_sharedfp_lockedfile_verbose ) { + printf("sharedfp_lockedfile_iwrite: Bytes Requested is %ld\n",bytesRequested); + } + + /*Retrieve the shared file data struct*/ + sh = fh->f_sharedfp_data; + + /*Request the offset to write bytesRequested bytes*/ + ret = mca_sharedfp_lockedfile_request_position(sh,bytesRequested,&offset); + if ( -1 != ret) { + if ( mca_sharedfp_lockedfile_verbose ) { + printf("sharedfp_lockedfile_iwrite: Offset received is %lld\n",offset); + } + + /* Write to the file */ + ret = ompio_io_ompio_file_iwrite_at(sh->sharedfh,offset,buf,count,datatype,request); + } + + return ret; +} + +int mca_sharedfp_lockedfile_write_ordered_begin(mca_io_ompio_file_t *fh, + void *buf, + int count, + struct ompi_datatype_t *datatype) +{ + opal_output(1,"mca_sharedfp_lockedfile_write_ordered_begin: NOT IMPLEMENTED\n"); + return OMPI_ERROR; +} + + + +int mca_sharedfp_lockedfile_write_ordered_end(mca_io_ompio_file_t *fh, + void *buf, + ompi_status_public_t *status) +{ + opal_output(1,"mca_sharedfp_lockedfile_write_ordered_end: NOT IMPLEMENTED\n"); + return OMPI_ERROR; +} diff --git a/ompi/mca/sharedfp/lockedfile/sharedfp_lockedfile_read.c b/ompi/mca/sharedfp/lockedfile/sharedfp_lockedfile_read.c new file mode 100644 index 0000000000..9b07cc3720 --- /dev/null +++ b/ompi/mca/sharedfp/lockedfile/sharedfp_lockedfile_read.c @@ -0,0 +1,193 @@ +/* + * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana + * University Research and Technology + * Corporation. All rights reserved. + * Copyright (c) 2004-2005 The University of Tennessee and The University + * of Tennessee Research Foundation. All rights + * reserved. + * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, + * University of Stuttgart. All rights reserved. + * Copyright (c) 2004-2005 The Regents of the University of California. + * All rights reserved. + * Copyright (c) 2013 University of Houston. All rights reserved. + * $COPYRIGHT$ + * + * Additional copyrights may follow + * + * $HEADER$ + */ + + +#include "ompi_config.h" +#include "sharedfp_lockedfile.h" + +#include "mpi.h" +#include "ompi/constants.h" +#include "orte/util/show_help.h" +#include "ompi/mca/sharedfp/sharedfp.h" + +int mca_sharedfp_lockedfile_read ( mca_io_ompio_file_t *fh, + void *buf, int count, MPI_Datatype datatype, MPI_Status *status) +{ + int ret = OMPI_SUCCESS; + mca_sharedfp_base_module_t * shared_fp_base_module; + OMPI_MPI_OFFSET_TYPE offset = 0; + long bytesRequested = 0; + size_t numofBytes; + struct mca_sharedfp_base_data_t *sh = NULL; + + if ( fh->f_sharedfp_data == NULL ) { + if ( mca_sharedfp_lockedfile_verbose ) { + printf("sharedfp_lockedfile_read: opening the shared file pointer\n"); + } + shared_fp_base_module = fh->f_sharedfp; + + ret = shared_fp_base_module->sharedfp_file_open(fh->f_comm, + fh->f_filename, + fh->f_amode, + fh->f_info, + fh); + if ( OMPI_SUCCESS != ret ) { + opal_output(1,"sharedfp_lockedfile_read - error opening the shared file pointer\n"); + return ret; + } + } + + /* Calculate the number of bytes to read */ + opal_datatype_type_size ( &datatype->super, &numofBytes); + bytesRequested = count * numofBytes; + + if ( mca_sharedfp_lockedfile_verbose ) { + printf("sharedfp_lockedfile_read: Bytes Requested is %ld\n",bytesRequested); + } + + /*Retrieve the shared file data struct*/ + sh = fh->f_sharedfp_data; + + /*Request the offset to write bytesRequested bytes*/ + ret = mca_sharedfp_lockedfile_request_position(sh,bytesRequested,&offset); + if (-1 != ret ) { + if ( mca_sharedfp_lockedfile_verbose ) { + printf("sharedfp_lockedfile_read: Offset received is %lld\n",offset); + } + + /* Read the file */ + ret = ompio_io_ompio_file_read_at(sh->sharedfh,offset,buf,count,datatype,status); + } + + return ret; +} + +int mca_sharedfp_lockedfile_read_ordered (mca_io_ompio_file_t *fh, + void *buf, + int count, + struct ompi_datatype_t *datatype, + ompi_status_public_t *status) +{ + int ret = OMPI_SUCCESS; + mca_sharedfp_base_module_t * shared_fp_base_module=NULL; + OMPI_MPI_OFFSET_TYPE offset = 0; + long sendBuff = 0; + long *buff=NULL; + long offsetBuff; + OMPI_MPI_OFFSET_TYPE offsetReceived = 0; + long bytesRequested = 0; + int recvcnt = 1, sendcnt = 1; + size_t numofBytes; + int rank, size, i; + struct mca_sharedfp_base_data_t *sh = NULL; + + if ( fh->f_sharedfp_data == NULL){ + if ( mca_sharedfp_lockedfile_verbose ) { + printf("sharedfp_lockedfile_read_ordered: opening the shared file pointer\n"); + } + shared_fp_base_module = fh->f_sharedfp; + + ret = shared_fp_base_module->sharedfp_file_open(fh->f_comm, + fh->f_filename, + fh->f_amode, + fh->f_info, + fh); + if ( OMPI_SUCCESS != ret ) { + opal_output(1,"sharedfp_lockedfile_read_ordered - error opening the shared file pointer\n"); + return ret; + } + } + + /*Retrieve the new communicator*/ + sh = fh->f_sharedfp_data; + + /* Calculate the number of bytes to read*/ + opal_datatype_type_size ( &datatype->super, &numofBytes ); + sendBuff = count * numofBytes; + + /* Get the ranks in the communicator */ + rank = ompi_comm_rank ( sh->comm ); + size = ompi_comm_size ( sh->comm ); + + if ( 0 == rank ) { + buff = (long*)malloc(sizeof(long) * size); + if ( NULL == buff ) + return OMPI_ERR_OUT_OF_RESOURCE; + } + + ret = sh->comm->c_coll.coll_gather ( &sendBuff, sendcnt, OMPI_OFFSET_DATATYPE, + buff, recvcnt, OMPI_OFFSET_DATATYPE, 0, + sh->comm, sh->comm->c_coll.coll_gather_module ); + if ( OMPI_SUCCESS != ret ) { + goto exit; + } + + /* All the counts are present now in the recvBuff. + The size of recvBuff is sizeof_newComm + */ + if ( 0 == rank ) { + for (i = 0; i < size ; i ++) { + bytesRequested += buff[i]; + if ( mca_sharedfp_lockedfile_verbose ) { + printf("sharedfp_lockedfile_read_ordered: Bytes requested are %ld\n",bytesRequested); + } + } + + /*Request the offset to read bytesRequested bytes + only the root process needs to do the request, + since the root process will then tell the other + processes at what offset they should read their + share of the data. + */ + ret = mca_sharedfp_lockedfile_request_position(sh,bytesRequested,&offsetReceived); + if( OMPI_SUCCESS != ret ){ + goto exit; + } + if ( mca_sharedfp_lockedfile_verbose ) { + printf("sharedfp_lockedfile_read_ordered: Offset received is %lld\n",offsetReceived); + } + buff[0] += offsetReceived; + + for (i = 1 ; i < size; i++) { + buff[i] += buff[i-1]; + } + } + + /* Scatter the results to the other processes*/ + ret = sh->comm->c_coll.coll_scatter ( buff, sendcnt, OMPI_OFFSET_DATATYPE, + &offsetBuff, recvcnt, OMPI_OFFSET_DATATYPE, 0, + sh->comm, sh->comm->c_coll.coll_scatter_module ); + + /*Each process now has its own individual offset in recvBUFF*/ + offset = offsetBuff - sendBuff; + + if ( mca_sharedfp_lockedfile_verbose ) { + printf("sharedfp_lockedfile_read_ordered: Offset returned is %lld\n",offset); + } + + /* read to the file */ + ret = ompio_io_ompio_file_read_at_all(sh->sharedfh,offset,buf,count,datatype,status); + +exit: + if ( NULL != buff ) { + free ( buff ); + } + + return ret; +} diff --git a/ompi/mca/sharedfp/lockedfile/sharedfp_lockedfile_request_position.c b/ompi/mca/sharedfp/lockedfile/sharedfp_lockedfile_request_position.c new file mode 100644 index 0000000000..1c3c2ffa7e --- /dev/null +++ b/ompi/mca/sharedfp/lockedfile/sharedfp_lockedfile_request_position.c @@ -0,0 +1,126 @@ +/* + * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana + * University Research and Technology + * Corporation. All rights reserved. + * Copyright (c) 2004-2005 The University of Tennessee and The University + * of Tennessee Research Foundation. All rights + * reserved. + * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, + * University of Stuttgart. All rights reserved. + * Copyright (c) 2004-2005 The Regents of the University of California. + * All rights reserved. + * Copyright (c) 2013 University of Houston. All rights reserved. + * $COPYRIGHT$ + * + * Additional copyrights may follow + * + * $HEADER$ + */ + + +#include "ompi_config.h" +#include "sharedfp_lockedfile.h" + +#include "mpi.h" +#include "ompi/constants.h" +#include "orte/util/show_help.h" +#include "ompi/mca/sharedfp/sharedfp.h" + +/*Use fcntl to lock the hidden file which stores the current position*/ +#include +#include + +int mca_sharedfp_lockedfile_request_position(struct mca_sharedfp_base_data_t * sh, + int bytes_requested, + OMPI_MPI_OFFSET_TYPE *offset) +{ + int ret = OMPI_SUCCESS; + /*file descriptor for the hidden file that we want to lock*/ + int fd; + /* flock structure that is used to setup the desired fcntl operation */ + struct flock fl; + OMPI_MPI_OFFSET_TYPE position = 0; + /*buffer to use when reading from the locked file*/ + OMPI_MPI_OFFSET_TYPE buf; + /*int count = 1;*/ + + struct mca_sharedfp_lockedfile_data * lockedfile_data = sh->selected_module_data; + int handle = lockedfile_data->handle; + + *offset = 0; + + /* get the saved file descriptor, + * the hidden file was opened during sharedfp_lockedfile_file_open function */ + fd = handle; + + /*set up the flock structure*/ + fl.l_type = F_WRLCK; + fl.l_whence = SEEK_SET; + fl.l_start = 0; + fl.l_len = 0; + fl.l_pid = getpid(); + + + /* Aquire an exclusive lock */ + if (fcntl(fd, F_SETLKW, &fl) == -1) { + printf("sharedfp_lockedfile_request_position: errorr acquiring lock: fcntl(%d,F_SETLKW,&fl)\n",fd); + printf("sharedfp_lockedfile_request_position: error(%i): %s", errno, strerror(errno)); + return OMPI_ERROR; + } + else{ + if ( mca_sharedfp_lockedfile_verbose ) { + printf("sharedfp_lockedfile_request_position: Success: acquired lock.for fd: %d\n",fd); + } + } + + /* read from the file */ + lseek ( fd, 0, SEEK_SET ); + read ( fd, &buf, sizeof(OMPI_MPI_OFFSET_TYPE)); + if ( mca_sharedfp_lockedfile_verbose ) { + printf("sharedfp_lockedfile_request_position: Read last_offset=%lld! ret=%d\n",buf, ret); + } + + /* increment the position */ + position = buf + bytes_requested; + if ( mca_sharedfp_lockedfile_verbose ) { + printf("sharedfp_lockedfile_request_position: old_offset=%lld, bytes_requested=%d, new offset=%lld!\n", + buf,bytes_requested,position); + } + + /* write to the file */ + lseek ( fd, 0, SEEK_SET ); + write ( fd, &position, sizeof(OMPI_MPI_OFFSET_TYPE)); + + /* unlock the file */ + if ( mca_sharedfp_lockedfile_verbose ) { + printf("sharedfp_lockedfile_request_position: Releasing lock..."); + } + + /* NOTE: We thought we could reuse the flock struct + ** and only reset the l_type parameter, but it gave + ** an invalid argument error. On my machine the invalid + ** argument was l_whence. I reset it, but I also decided to + ** reset all of the other parameters to avoid further problems. + */ + fl.l_type = F_UNLCK; /* set to unlock same region */ + fl.l_whence = SEEK_SET; + fl.l_start = 0; + fl.l_len = 0; + fl.l_pid = getpid(); + + if (fcntl(fd, F_SETLK, &fl) == -1) { + printf("sharedfp_lockedfile_request_position:failed to release lock for fd: %d\n",fd); + printf("error(%i): %s", errno, strerror(errno)); + return OMPI_ERROR; + } + else { + if ( mca_sharedfp_lockedfile_verbose ) { + printf("sharedfp_lockedfile_request_position: released lock.for fd: %d\n",fd); + } + } + + /* return position */ + *offset = buf; + + return ret; +} diff --git a/ompi/mca/sharedfp/lockedfile/sharedfp_lockedfile_seek.c b/ompi/mca/sharedfp/lockedfile/sharedfp_lockedfile_seek.c new file mode 100644 index 0000000000..565bd377f2 --- /dev/null +++ b/ompi/mca/sharedfp/lockedfile/sharedfp_lockedfile_seek.c @@ -0,0 +1,167 @@ +/* + * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana + * University Research and Technology + * Corporation. All rights reserved. + * Copyright (c) 2004-2005 The University of Tennessee and The University + * of Tennessee Research Foundation. All rights + * reserved. + * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, + * University of Stuttgart. All rights reserved. + * Copyright (c) 2004-2005 The Regents of the University of California. + * All rights reserved. + * Copyright (c) 2013 University of Houston. All rights reserved. + * $COPYRIGHT$ + * + * Additional copyrights may follow + * + * $HEADER$ + */ + + +#include "ompi_config.h" +#include "sharedfp_lockedfile.h" + +#include "mpi.h" +#include "ompi/constants.h" +#include "orte/util/show_help.h" +#include "ompi/mca/sharedfp/sharedfp.h" + +/*Use fcntl to lock the file which stores the current position*/ +#include + +int +mca_sharedfp_lockedfile_seek (mca_io_ompio_file_t *fh, + OMPI_MPI_OFFSET_TYPE offset, int whence) +{ + int rank; + int ret = OMPI_SUCCESS; + struct mca_sharedfp_base_data_t *sh = NULL; + mca_sharedfp_base_module_t * shared_fp_base_module; + struct mca_sharedfp_lockedfile_data * lockedfile_data; + int fd_lockedfilehandle; + /* flock structure that is used to setup the desired fcntl operation */ + struct flock fl; + + if(fh->f_sharedfp_data==NULL){ + if ( mca_sharedfp_lockedfile_verbose ) { + printf("sharedfp_lockedfile_seek - opening the shared file pointer\n"); + } + shared_fp_base_module = fh->f_sharedfp; + + ret = shared_fp_base_module->sharedfp_file_open(fh->f_comm, + fh->f_filename, + fh->f_amode, + fh->f_info, + fh); + if (ret != OMPI_SUCCESS) { + opal_output(1,"sharedfp_lockedfile_seek - error opening the shared file pointer\n"); + return ret; + } + } + + sh = fh->f_sharedfp_data; + rank = ompi_comm_rank ( sh->comm ); + + if( 0 == rank ){ + if ( MPI_SEEK_SET == whence ){ + /*don't need to read current value*/ + if(offset < 0){ + opal_output(1,"sharedfp_lockedfile_seek - MPI_SEEK_SET, offset must be > 0, got offset=%lld.\n",offset); + ret = -1; + } + opal_output(1,"MPI_SEEK_SET: new_offset=%lld\n",offset); + fflush(stdout); + } + else if ( MPI_SEEK_CUR == whence){ + OMPI_MPI_OFFSET_TYPE current_position; + int status = mca_sharedfp_lockedfile_get_position(fh,¤t_position); + opal_output(1,"MPI_SEEK_CUR: curr=%lld, offset=%lld, call status=%d\n",current_position,offset,status); + + offset = current_position + offset; + opal_output(1,"MPI_SEEK_CUR: new_offset=%lld\n",offset); + fflush(stdout); + if(offset < 0){ + opal_output(1,"sharedfp_lockedfile_seek - MPI_SEEK_CURE, offset must be > 0, got offset=%lld.\n",offset); + ret = -1; + } + } + else if( MPI_SEEK_END == whence ){ + OMPI_MPI_OFFSET_TYPE end_position=0; + ompio_io_ompio_file_get_size(sh->sharedfh,&end_position); + offset = end_position + offset; + if ( mca_sharedfp_lockedfile_verbose ) { + printf("MPI_SEEK_END: file_get_size=%lld\n",end_position); + } + + if ( offset < 0){ + opal_output(1,"sharedfp_lockedfile_seek - MPI_SEEK_CUR, offset must be > 0, got offset=%lld.\n",offset); + ret = -1; + } + }else{ + opal_output(1,"sharedfp_lockedfile_seek - whence=%i is not supported\n",whence); + ret = -1; + } + + + /* Set Shared file pointer */ + lockedfile_data = sh->selected_module_data; + fd_lockedfilehandle = lockedfile_data->handle; + + if ( mca_sharedfp_lockedfile_verbose ) { + printf("sharedfp_lockedfile_seek: Aquiring lock..."); + } + + /* set up the flock structure */ + fl.l_type = F_WRLCK; + fl.l_whence = SEEK_SET; + fl.l_start = 0; + fl.l_len = 0; + fl.l_pid = getpid(); + + /* Aquire an exclusive lock */ + if ( fcntl(fd_lockedfilehandle, F_SETLKW, &fl) == -1) { + opal_output(1, "#####erorr acquiring lock: fcntl(%d,F_SETLKW,&fl)\n",fd_lockedfilehandle); + opal_output(1,"error(%i): %s", errno, strerror(errno)); + return OMPI_ERROR; + } + else{ + if ( mca_sharedfp_lockedfile_verbose ) { + printf("sharedfp_lockedfile_seek: Success! acquired lock.for fd: %d\n",fd_lockedfilehandle); + } + } + + /*-- ----------------- + *write to the file + *-------------------- + */ + lseek ( fd_lockedfilehandle, 0, SEEK_SET); + write ( fd_lockedfilehandle, &offset, sizeof(OMPI_MPI_OFFSET_TYPE)); + + /*------------------- + * unlock the file + *-------------------- + */ + if ( mca_sharedfp_lockedfile_verbose ) { + printf("sharedfp_lockedfile_seek: Releasing lock..."); + } + fl.l_type = F_UNLCK; /* set to unlock same region */ + fl.l_whence = SEEK_SET; + fl.l_start = 0; + fl.l_len = 0; + fl.l_pid = getpid(); + + if (fcntl(fd_lockedfilehandle, F_SETLK, &fl) == -1) { + opal_output(1,"####failed to release lock for fd: %d\n",fd_lockedfilehandle); + opal_output(1,"error(%i): %s", errno, strerror(errno)); + return OMPI_ERROR; + } + else{ + if ( mca_sharedfp_lockedfile_verbose ) { + printf("sharedfp_lockedfile_seek: released lock.for fd: %d\n",fd_lockedfilehandle); + } + } + } + + sh->comm->c_coll.coll_barrier ( sh->comm , sh->comm->c_coll.coll_barrier_module ); + return ret; +} diff --git a/ompi/mca/sharedfp/lockedfile/sharedfp_lockedfile_write.c b/ompi/mca/sharedfp/lockedfile/sharedfp_lockedfile_write.c new file mode 100644 index 0000000000..38269e935f --- /dev/null +++ b/ompi/mca/sharedfp/lockedfile/sharedfp_lockedfile_write.c @@ -0,0 +1,198 @@ +/* + * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana + * University Research and Technology + * Corporation. All rights reserved. + * Copyright (c) 2004-2005 The University of Tennessee and The University + * of Tennessee Research Foundation. All rights + * reserved. + * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, + * University of Stuttgart. All rights reserved. + * Copyright (c) 2004-2005 The Regents of the University of California. + * All rights reserved. + * Copyright (c) 2013 University of Houston. All rights reserved. + * $COPYRIGHT$ + * + * Additional copyrights may follow + * + * $HEADER$ + */ + + +#include "ompi_config.h" +#include "sharedfp_lockedfile.h" + +#include "mpi.h" +#include "ompi/constants.h" +#include "orte/util/show_help.h" +#include "ompi/mca/sharedfp/sharedfp.h" + +int mca_sharedfp_lockedfile_write (mca_io_ompio_file_t *fh, + void *buf, + int count, + struct ompi_datatype_t *datatype, + ompi_status_public_t *status) +{ + mca_sharedfp_base_module_t * shared_fp_base_module; + OMPI_MPI_OFFSET_TYPE offset = 0; + long bytesRequested = 0; + size_t numofBytes; + struct mca_sharedfp_base_data_t *sh = NULL; + int ret = OMPI_SUCCESS; + + if ( NULL == fh->f_sharedfp_data ){ + if ( mca_sharedfp_lockedfile_verbose ) { + printf("sharedfp_lockedfile_write - opening the shared file pointer\n"); + } + shared_fp_base_module = fh->f_sharedfp; + + ret = shared_fp_base_module->sharedfp_file_open(fh->f_comm, + fh->f_filename, + fh->f_amode, + fh->f_info, + fh); + if (ret != OMPI_SUCCESS) { + opal_output(1,"sharedfp_lockedfile_write - error opening the shared file pointer\n"); + return ret; + } + } + + /*Calculate the number of bytes to write*/ + opal_datatype_type_size( &datatype->super, &numofBytes); + bytesRequested = count * numofBytes; + if ( mca_sharedfp_lockedfile_verbose ) { + printf("sharedfp_lockedfile_write: Bytes Requested is %ld\n",bytesRequested); + } + + /*Retrieve the shared file data struct*/ + sh = fh->f_sharedfp_data; + + /* Request the offset to write bytesRequested bytes */ + ret = mca_sharedfp_lockedfile_request_position ( sh, bytesRequested, &offset); + if (-1 != ret ) { + if ( mca_sharedfp_lockedfile_verbose ) { + printf("sharedfp_lockedfile_write: Offset received is %lld\n",offset); + } + /* Write to the file */ + ret = ompio_io_ompio_file_write_at ( sh->sharedfh, offset, buf, count, datatype, status); + } + + return ret; +} + +int mca_sharedfp_lockedfile_write_ordered (mca_io_ompio_file_t *fh, + void *buf, + int count, + struct ompi_datatype_t *datatype, + ompi_status_public_t *status) +{ + int ret = OMPI_SUCCESS; + mca_sharedfp_base_module_t * shared_fp_base_module=NULL; + OMPI_MPI_OFFSET_TYPE offset = 0; + long sendBuff = 0; + long *buff=NULL; + long offsetBuff; + OMPI_MPI_OFFSET_TYPE offsetReceived = 0; + long bytesRequested = 0; + int recvcnt = 1, sendcnt = 1; + size_t numofBytes; + int rank, size, i; + + struct mca_sharedfp_base_data_t *sh = NULL; + + if( NULL == fh->f_sharedfp_data ) { + if ( mca_sharedfp_lockedfile_verbose ) { + printf("sharedfp_lockedfile_write_ordered - opening the shared file pointer\n"); + } + shared_fp_base_module = fh->f_sharedfp; + + ret = shared_fp_base_module->sharedfp_file_open(fh->f_comm, + fh->f_filename, + fh->f_amode, + fh->f_info, + fh); + if ( OMPI_SUCCESS != ret ) { + opal_output(1,"sharedfp_lockedfile_write_ordered - error opening the shared file pointer\n"); + return ret; + } + } + + /*Retrieve the new communicator*/ + sh = fh->f_sharedfp_data; + + /* Calculate the number of bytes to write*/ + opal_datatype_type_size ( &datatype->super, &numofBytes); + sendBuff = count * numofBytes; + + /* Get the ranks in the communicator */ + rank = ompi_comm_rank ( sh->comm ); + size = ompi_comm_size ( sh->comm ); + + if ( 0 == rank ) { + buff = (long*) malloc (sizeof(long) * size); + if ( NULL == buff ) { + return OMPI_ERR_OUT_OF_RESOURCE; + } + } + + ret = sh->comm->c_coll.coll_gather ( &sendBuff, sendcnt, OMPI_OFFSET_DATATYPE, buff, recvcnt, + OMPI_OFFSET_DATATYPE, 0, sh->comm, + sh->comm->c_coll.coll_gather_module ); + if ( OMPI_SUCCESS != ret ) { + goto exit; + } + + /* All the counts are present now in the recvBuff. + The size of recvBuff is sizeof_newComm + */ + if (rank == 0) { + for ( i = 0; i < size ; i ++) { + bytesRequested += buff[i]; + if ( mca_sharedfp_lockedfile_verbose ) { + printf("sharedfp_lockedfile_write_ordered: Bytes requested are %ld\n",bytesRequested); + } + } + + /*Request the offset to write bytesRequested bytes + only the root process needs to do the request, + since the root process will then tell the other + processes at what offset they should write their + share of the data. + */ + ret = mca_sharedfp_lockedfile_request_position(sh,bytesRequested,&offsetReceived); + if ( OMPI_SUCCESS != ret ){ + goto exit; + } + if ( mca_sharedfp_lockedfile_verbose ) { + printf("sharedfp_lockedfile_write_ordered: Offset received is %lld\n",offsetReceived); + } + buff[0] += offsetReceived; + for (i = 1 ; i < size; i++) { + buff[i] += buff[i-1]; + } + } + + /* Scatter the results to the other processes*/ + ret = sh->comm->c_coll.coll_scatter ( buff, sendcnt, OMPI_OFFSET_DATATYPE, + &offsetBuff, recvcnt, OMPI_OFFSET_DATATYPE, 0, + sh->comm, sh->comm->c_coll.coll_scatter_module ); + if ( OMPI_SUCCESS != ret ) { + goto exit; + } + + /*Each process now has its own individual offset*/ + offset = offsetBuff - sendBuff; + + if ( mca_sharedfp_lockedfile_verbose ) { + printf("sharedfp_lockedfile_write_ordered: Offset returned is %lld\n",offset); + } + + /* write to the file */ + ret = ompio_io_ompio_file_write_at_all(sh->sharedfh,offset,buf,count,datatype,status); + +exit: + if ( NULL != buff ) { + free ( buff); + } + + return ret; +} diff --git a/ompi/mca/sharedfp/sharedfp.h b/ompi/mca/sharedfp/sharedfp.h index 4321bfbbed..42bb7af548 100644 --- a/ompi/mca/sharedfp/sharedfp.h +++ b/ompi/mca/sharedfp/sharedfp.h @@ -5,15 +5,15 @@ * Copyright (c) 2004-2005 The University of Tennessee and The University * of Tennessee Research Foundation. All rights * reserved. - * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, + * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, * University of Stuttgart. All rights reserved. * Copyright (c) 2004-2005 The Regents of the University of California. * All rights reserved. - * Copyright (c) 2008-2011 University of Houston. All rights reserved. + * Copyright (c) 2008-2013 University of Houston. All rights reserved. * $COPYRIGHT$ - * + * * Additional copyrights may follow - * + * * $HEADER$ */ @@ -24,17 +24,19 @@ #include "mpi.h" #include "opal/mca/mca.h" #include "opal/mca/base/base.h" +#include "ompi/request/request.h" + BEGIN_C_DECLS struct mca_io_ompio_file_t; - +struct ompi_file_t; /* * Macro for use in components that are of type coll */ #define MCA_SHAREDFP_BASE_VERSION_2_0_0 \ MCA_BASE_VERSION_2_0_0, \ - "sharedfp", 2, 0, 0 + "sharedfp", 2, 0, 0 /* * This framework abstracts out operations of the shared filepointer @@ -67,24 +69,26 @@ struct mca_io_ompio_file_t; */ typedef int (*mca_sharedfp_base_component_init_query_1_0_0_fn_t) - (bool enable_progress_threads, - bool enable_mpi_threads); +(bool enable_progress_threads, + bool enable_mpi_threads); -typedef struct mca_sharedfp_base_module_1_0_0_t * -(*mca_sharedfp_base_component_file_query_1_0_0_fn_t) (int *priority); +typedef struct mca_sharedfp_base_module_1_0_0_t * + (*mca_sharedfp_base_component_file_query_1_0_0_fn_t) + (struct mca_io_ompio_file_t *file, int *priority); typedef int (*mca_sharedfp_base_component_file_unquery_1_0_0_fn_t) - (struct mca_io_ompio_file_t *file); + (struct mca_io_ompio_file_t *file); /* * ****************** component struct ****************************** * Structure for sharedfp v2.0.0 components.This is chained to MCA v2.0.0 * ****************** component struct ****************************** */ + struct mca_sharedfp_base_component_2_0_0_t { mca_base_component_t sharedfpm_version; mca_base_component_data_t sharedfpm_data; - + mca_sharedfp_base_component_init_query_1_0_0_fn_t sharedfpm_init_query; mca_sharedfp_base_component_file_query_1_0_0_fn_t sharedfpm_file_query; mca_sharedfp_base_component_file_unquery_1_0_0_fn_t sharedfpm_file_unquery; @@ -106,11 +110,71 @@ typedef int (*mca_sharedfp_base_module_init_1_0_0_fn_t) typedef int (*mca_sharedfp_base_module_finalize_1_0_0_fn_t) (struct mca_io_ompio_file_t *file); -typedef int (*mca_sharedfp_base_module_update_fn_t)( - struct mca_io_ompio_file_t *fh, int num_bytes, - OMPI_MPI_OFFSET_TYPE current_position); +/* SHAREDFP function definitions */ +/* IMPORTANT: Update here when adding sharedfp component interface functions*/ typedef int (*mca_sharedfp_base_module_seek_fn_t)( - struct mca_io_ompio_file_t *fh, OMPI_MPI_OFFSET_TYPE position); + struct mca_io_ompio_file_t *fh, OMPI_MPI_OFFSET_TYPE offset, int whence); +typedef int (*mca_sharedfp_base_module_get_position_fn_t)( + struct mca_io_ompio_file_t *fh, OMPI_MPI_OFFSET_TYPE * offset); +typedef int (*mca_sharedfp_base_module_write_fn_t)( + struct mca_io_ompio_file_t *fh, + void *buf, + int count, + struct ompi_datatype_t *datatype, + ompi_status_public_t *status); +typedef int (*mca_sharedfp_base_module_write_ordered_fn_t)( + struct mca_io_ompio_file_t *fh, + void *buf, + int count, + struct ompi_datatype_t *datatype, + ompi_status_public_t *status); +typedef int (*mca_sharedfp_base_module_write_ordered_begin_fn_t)( + struct mca_io_ompio_file_t *fh, + void *buf, + int count, + struct ompi_datatype_t *datatype); +typedef int (*mca_sharedfp_base_module_write_ordered_end_fn_t)( + struct mca_io_ompio_file_t *fh, + void *buf, + ompi_status_public_t *status); +typedef int (*mca_sharedfp_base_module_iwrite_fn_t)( + struct mca_io_ompio_file_t *fh, + void *buf, + int count, + struct ompi_datatype_t *datatype, + ompi_request_t ** request); +typedef int (*mca_sharedfp_base_module_read_fn_t)( + struct mca_io_ompio_file_t *fh, + void *buf, + int count, + struct ompi_datatype_t *datatype, + ompi_status_public_t *status); +typedef int (*mca_sharedfp_base_module_read_ordered_fn_t)( + struct mca_io_ompio_file_t *fh, + void *buf, + int count, + struct ompi_datatype_t *datatype, + ompi_status_public_t *status); +typedef int (*mca_sharedfp_base_module_iread_fn_t)( + struct mca_io_ompio_file_t *fh, + void *buf, + int count, + struct ompi_datatype_t *datatype, + ompi_request_t ** request); +typedef int (*mca_sharedfp_base_module_read_ordered_begin_fn_t)( + struct mca_io_ompio_file_t *fh, + void *buf, + int count, + struct ompi_datatype_t *datatype); +typedef int (*mca_sharedfp_base_module_read_ordered_end_fn_t)( + struct mca_io_ompio_file_t *fh, + void *buf, + ompi_status_public_t *status); +typedef int (*mca_sharedfp_base_module_file_open_fn_t)( + struct ompi_communicator_t *comm, char *filename, int amode, + struct ompi_info_t *info, struct mca_io_ompio_file_t *fh); +typedef int (*mca_sharedfp_base_module_file_close_fn_t)(struct mca_io_ompio_file_t *fh); + /* * *********************************************************************** @@ -125,14 +189,45 @@ struct mca_sharedfp_base_module_1_0_0_t { */ mca_sharedfp_base_module_init_1_0_0_fn_t sharedfp_module_init; mca_sharedfp_base_module_finalize_1_0_0_fn_t sharedfp_module_finalize; - + /* SHAREDFP function pointers */ - mca_sharedfp_base_module_update_fn_t sharedfp_update; + /* IMPORTANT: Update here when adding sharedfp component interface functions*/ mca_sharedfp_base_module_seek_fn_t sharedfp_seek; + mca_sharedfp_base_module_get_position_fn_t sharedfp_get_position; + mca_sharedfp_base_module_read_fn_t sharedfp_read; + mca_sharedfp_base_module_read_ordered_fn_t sharedfp_read_ordered; + mca_sharedfp_base_module_read_ordered_begin_fn_t sharedfp_read_ordered_begin; + mca_sharedfp_base_module_read_ordered_end_fn_t sharedfp_read_ordered_end; + mca_sharedfp_base_module_iread_fn_t sharedfp_iread; + mca_sharedfp_base_module_write_fn_t sharedfp_write; + mca_sharedfp_base_module_write_ordered_fn_t sharedfp_write_ordered; + mca_sharedfp_base_module_write_ordered_begin_fn_t sharedfp_write_ordered_begin; + mca_sharedfp_base_module_write_ordered_end_fn_t sharedfp_write_ordered_end; + mca_sharedfp_base_module_iwrite_fn_t sharedfp_iwrite; + mca_sharedfp_base_module_file_open_fn_t sharedfp_file_open; + mca_sharedfp_base_module_file_close_fn_t sharedfp_file_close; }; typedef struct mca_sharedfp_base_module_1_0_0_t mca_sharedfp_base_module_1_0_0_t; typedef mca_sharedfp_base_module_1_0_0_t mca_sharedfp_base_module_t; + +/* This structure keeps all of the data needed by a sharedfp module. + * This structure is assigned to the ompio file handle's + * 'f_sharedfp_data' attribute during the call to the file_open function. + */ +struct mca_sharedfp_base_data_t{ + /* attributes that will be used by all of the sharedfp components */ + struct mca_io_ompio_file_t * sharedfh; + OMPI_MPI_OFFSET_TYPE global_offset; + struct ompi_communicator_t * comm; + + /* attributes that are specific to a component are + * combined into a structure that is assigned to this attribute */ + void *selected_module_data; +}; + +/**************************************/ + END_C_DECLS #endif /* OMPI_MCA_SHAREDFP_H */ diff --git a/ompi/mca/sharedfp/dummy/Makefile.am b/ompi/mca/sharedfp/sm/Makefile.am similarity index 59% rename from ompi/mca/sharedfp/dummy/Makefile.am rename to ompi/mca/sharedfp/sm/Makefile.am index 91319edaca..470c3c0112 100644 --- a/ompi/mca/sharedfp/dummy/Makefile.am +++ b/ompi/mca/sharedfp/sm/Makefile.am @@ -9,7 +9,7 @@ # University of Stuttgart. All rights reserved. # Copyright (c) 2004-2005 The Regents of the University of California. # All rights reserved. -# Copyright (c) 2008-2011 University of Houston. All rights reserved. +# Copyright (c) 2008 University of Houston. All rights reserved. # $COPYRIGHT$ # # Additional copyrights may follow @@ -21,28 +21,35 @@ # mca__.la (for DSO builds) or libmca__.la # (for static builds). -if MCA_BUILD_ompi_sharedfp_dummy_DSO +if MCA_BUILD_ompi_sharedfp_sm_DSO component_noinst = -component_install = mca_sharedfp_dummy.la +component_install = mca_sharedfp_sm.la else -component_noinst = libmca_sharedfp_dummy.la +component_noinst = libmca_sharedfp_sm.la component_install = endif mcacomponentdir = $(pkglibdir) mcacomponent_LTLIBRARIES = $(component_install) -mca_sharedfp_dummy_la_SOURCES = $(sources) -mca_sharedfp_dummy_la_LDFLAGS = -module -avoid-version +mca_sharedfp_sm_la_SOURCES = $(sources) +mca_sharedfp_sm_la_LDFLAGS = -module -avoid-version noinst_LTLIBRARIES = $(component_noinst) -libmca_sharedfp_dummy_la_SOURCES = $(sources) -libmca_sharedfp_dummy_la_LDFLAGS = -module -avoid-version +libmca_sharedfp_sm_la_SOURCES = $(sources) +libmca_sharedfp_sm_la_LDFLAGS = -module -avoid-version # Source files +#IMPORTANT: Update here when adding new source code files to the library sources = \ - sharedfp_dummy.h \ - sharedfp_dummy.c \ - sharedfp_dummy_component.c \ - sharedfp_dummy_update.c \ - sharedfp_dummy_seek.c + sharedfp_sm.h \ + sharedfp_sm.c \ + sharedfp_sm_component.c \ + sharedfp_sm_seek.c \ + sharedfp_sm_get_position.c \ + sharedfp_sm_request_position.c \ + sharedfp_sm_write.c \ + sharedfp_sm_iwrite.c \ + sharedfp_sm_read.c \ + sharedfp_sm_iread.c \ + sharedfp_sm_file_open.c diff --git a/ompi/mca/sharedfp/sm/sharedfp_sm.c b/ompi/mca/sharedfp/sm/sharedfp_sm.c new file mode 100644 index 0000000000..b6c33a688b --- /dev/null +++ b/ompi/mca/sharedfp/sm/sharedfp_sm.c @@ -0,0 +1,116 @@ +/* + * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana + * University Research and Technology + * Corporation. All rights reserved. + * Copyright (c) 2004-2006 The University of Tennessee and The University + * of Tennessee Research Foundation. All rights + * reserved. + * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, + * University of Stuttgart. All rights reserved. + * Copyright (c) 2004-2005 The Regents of the University of California. + * All rights reserved. + * Copyright (c) 2008 University of Houston. All rights reserved. + * $COPYRIGHT$ + * + * Additional copyrights may follow + * + * $HEADER$ + * + * These symbols are in a file by themselves to provide nice linker + * semantics. Since linkers generally pull in symbols by object fules, + * keeping these symbols as the only symbols in this file prevents + * utility programs such as "ompi_info" from having to import entire + * modules just to query their version and parameters + */ + +#include "ompi_config.h" +#include "mpi.h" +#include "ompi/mca/sharedfp/sharedfp.h" +#include "ompi/mca/sharedfp/sm/sharedfp_sm.h" + +/* + * ******************************************************************* + * ************************ actions structure ************************ + * ******************************************************************* + */ + /* IMPORTANT: Update here when adding sharedfp component interface functions*/ +static mca_sharedfp_base_module_1_0_0_t sm = { + mca_sharedfp_sm_module_init, /* initalise after being selected */ + mca_sharedfp_sm_module_finalize, /* close a module on a communicator */ + mca_sharedfp_sm_seek, + mca_sharedfp_sm_get_position, + mca_sharedfp_sm_read, + mca_sharedfp_sm_read_ordered, + mca_sharedfp_sm_read_ordered_begin, + mca_sharedfp_sm_read_ordered_end, + mca_sharedfp_sm_iread, + mca_sharedfp_sm_write, + mca_sharedfp_sm_write_ordered, + mca_sharedfp_sm_write_ordered_begin, + mca_sharedfp_sm_write_ordered_end, + mca_sharedfp_sm_iwrite, + mca_sharedfp_sm_file_open, + mca_sharedfp_sm_file_close +}; +/* + * ******************************************************************* + * ************************* structure ends ************************** + * ******************************************************************* + */ + +int mca_sharedfp_sm_component_init_query(bool enable_progress_threads, + bool enable_mpi_threads) +{ + /* Nothing to do */ + + return OMPI_SUCCESS; +} + +struct mca_sharedfp_base_module_1_0_0_t * mca_sharedfp_sm_component_file_query(mca_io_ompio_file_t *fh, int *priority) +{ + int i; + ompi_proc_t *proc; + ompi_communicator_t * comm = fh->f_comm; + int size = ompi_comm_size(comm); + + *priority = 0; + + /* test, and update priority. All processes have to be + ** on a single node. + ** original test copied from mca/coll/sm/coll_sm_module.c: + */ + ompi_group_t *group = comm->c_local_group; + + for (i = 0; i < size; ++i) { + proc = ompi_group_peer_lookup(group,i); + if (!OPAL_PROC_ON_LOCAL_NODE(proc->proc_flags)){ + opal_output(1,"mca_sharedfp_sm_component_file_query: Disqualifying myself: (%d/%s) " + "not all processes are on the same node.", + comm->c_contextid, comm->c_name); + return NULL; + } + } + /* This module can run */ + *priority = mca_sharedfp_sm_priority; + return &sm; +} + +int mca_sharedfp_sm_component_file_unquery (mca_io_ompio_file_t *file) +{ + /* This function might be needed for some purposes later. for now it + * does not have anything to do since there are no steps which need + * to be undone if this module is not selected */ + + return OMPI_SUCCESS; +} + +int mca_sharedfp_sm_module_init (mca_io_ompio_file_t *file) +{ + return OMPI_SUCCESS; +} + + +int mca_sharedfp_sm_module_finalize (mca_io_ompio_file_t *file) +{ + return OMPI_SUCCESS; +} diff --git a/ompi/mca/sharedfp/sm/sharedfp_sm.h b/ompi/mca/sharedfp/sm/sharedfp_sm.h new file mode 100644 index 0000000000..acae03dc8d --- /dev/null +++ b/ompi/mca/sharedfp/sm/sharedfp_sm.h @@ -0,0 +1,134 @@ +/* + * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana + * University Research and Technology + * Corporation. All rights reserved. + * Copyright (c) 2004-2006 The University of Tennessee and The University + * of Tennessee Research Foundation. All rights + * reserved. + * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, + * University of Stuttgart. All rights reserved. + * Copyright (c) 2004-2005 The Regents of the University of California. + * All rights reserved. + * Copyright (c) 2008 University of Houston. All rights reserved. + * $COPYRIGHT$ + * + * Additional copyrights may follow + * + * $HEADER$ + */ + +#ifndef MCA_SHAREDFP_sm_H +#define MCA_SHAREDFP_sm_H + +#include "ompi_config.h" +#include "opal/mca/mca.h" +#include "ompi/mca/sharedfp/sharedfp.h" +#include "ompi/mca/io/ompio/io_ompio.h" +#include + +BEGIN_C_DECLS + +int mca_sharedfp_sm_component_init_query(bool enable_progress_threads, + bool enable_mpi_threads); +struct mca_sharedfp_base_module_1_0_0_t * + mca_sharedfp_sm_component_file_query (mca_io_ompio_file_t *file, int *priority); +int mca_sharedfp_sm_component_file_unquery (mca_io_ompio_file_t *file); + +int mca_sharedfp_sm_module_init (mca_io_ompio_file_t *file); +int mca_sharedfp_sm_module_finalize (mca_io_ompio_file_t *file); + +extern int mca_sharedfp_sm_priority; +extern int mca_sharedfp_sm_verbose; + +OMPI_MODULE_DECLSPEC extern mca_sharedfp_base_component_2_0_0_t mca_sharedfp_sm_component; +/* + * ****************************************************************** + * ********* functions which are implemented in this module ********* + * ****************************************************************** + */ +/*IMPORANT: Update here when implementing functions from sharedfp API*/ +int mca_sharedfp_sm_seek (mca_io_ompio_file_t *fh, + OMPI_MPI_OFFSET_TYPE offset, int whence); +int mca_sharedfp_sm_get_position (mca_io_ompio_file_t *fh, + OMPI_MPI_OFFSET_TYPE * offset); +int mca_sharedfp_sm_file_open (struct ompi_communicator_t *comm, + char* filename, + int amode, + struct ompi_info_t *info, + mca_io_ompio_file_t *fh); +int mca_sharedfp_sm_file_close (mca_io_ompio_file_t *fh); +int mca_sharedfp_sm_read (mca_io_ompio_file_t *fh, + void *buf, int count, MPI_Datatype datatype, MPI_Status *status); +int mca_sharedfp_sm_read_ordered (mca_io_ompio_file_t *fh, + void *buf, int count, struct ompi_datatype_t *datatype, + ompi_status_public_t *status + ); +int mca_sharedfp_sm_read_ordered_begin (mca_io_ompio_file_t *fh, + void *buf, + int count, + struct ompi_datatype_t *datatype); +int mca_sharedfp_sm_read_ordered_end (mca_io_ompio_file_t *fh, + void *buf, + ompi_status_public_t *status); +int mca_sharedfp_sm_iread (mca_io_ompio_file_t *fh, + void *buf, + int count, + struct ompi_datatype_t *datatype, + ompi_request_t **request); +int mca_sharedfp_sm_write (mca_io_ompio_file_t *fh, + void *buf, + int count, + struct ompi_datatype_t *datatype, + ompi_status_public_t *status); +int mca_sharedfp_sm_write_ordered (mca_io_ompio_file_t *fh, + void *buf, + int count, + struct ompi_datatype_t *datatype, + ompi_status_public_t *status); +int mca_sharedfp_sm_write_ordered_begin (mca_io_ompio_file_t *fh, + void *buf, + int count, + struct ompi_datatype_t *datatype); +int mca_sharedfp_sm_write_ordered_end (mca_io_ompio_file_t *fh, + void *buf, + ompi_status_public_t *status); +int mca_sharedfp_sm_iwrite (mca_io_ompio_file_t *fh, + void *buf, + int count, + struct ompi_datatype_t *datatype, + ompi_request_t **request); + +/*--------------------------------------------------------------* + *Structures and definitions only for this component + *--------------------------------------------------------------*/ + +struct sm_offset{ + sem_t mutex; /* the mutex: a Posix memory-based semaphore */ + long long offset; /* and the shared file pointer offset */ +} sm_offset; + +/*This structure will hang off of the mca_sharedfp_base_data_t's + *selected_module_data attribute + */ +struct mca_sharedfp_sm_data +{ + struct sm_offset * sm_offset_ptr; + /*save filename so that we can remove the file on close*/ + char * sm_filename; +}; + +typedef struct mca_sharedfp_sm_data sm_data; + + +int mca_sharedfp_sm_request_position (struct mca_sharedfp_base_data_t * sh, + int bytes_requested, + OMPI_MPI_OFFSET_TYPE * offset); +/* + * ****************************************************************** + * ************ functions implemented in this module end ************ + * ****************************************************************** + */ + +END_C_DECLS + +#endif /* MCA_SHAREDFP_sm_H */ diff --git a/ompi/mca/sharedfp/sm/sharedfp_sm_component.c b/ompi/mca/sharedfp/sm/sharedfp_sm_component.c new file mode 100644 index 0000000000..9cb998fd9b --- /dev/null +++ b/ompi/mca/sharedfp/sm/sharedfp_sm_component.c @@ -0,0 +1,88 @@ +/* + * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana + * University Research and Technology + * Corporation. All rights reserved. + * Copyright (c) 2004-2005 The University of Tennessee and The University + * of Tennessee Research Foundation. All rights + * reserved. + * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, + * University of Stuttgart. All rights reserved. + * Copyright (c) 2004-2005 The Regents of the University of California. + * All rights reserved. + * Copyright (c) 2013 University of Houston. All rights reserved. + * $COPYRIGHT$ + * + * Additional copyrights may follow + * + * $HEADER$ + * + * These symbols are in a file by themselves to provide nice linker + * semantics. Since linkers generally pull in symbols by object + * files, keeping these symbols as the only symbols in this file + * prevents utility programs such as "ompi_info" from having to import + * entire components just to query their version and parameters. + */ + +#include "ompi_config.h" +#include "sharedfp_sm.h" +#include "mpi.h" + +/* + * Public string showing the sharedfp sm component version number + */ +const char *mca_sharedfp_sm_component_version_string = + "OMPI/MPI sm SHAREDFP MCA component version " OMPI_VERSION; +/* + * Global variables + */ +int mca_sharedfp_sm_priority=10; +int mca_sharedfp_sm_verbose=0; + +static int sm_register(void); + +/* + * Instantiate the public struct with all of our public information + * and pointers to our public functions in it + */ +mca_sharedfp_base_component_2_0_0_t mca_sharedfp_sm_component = { + + /* First, the mca_component_t struct containing meta information + about the component itself */ + + { + MCA_SHAREDFP_BASE_VERSION_2_0_0, + + /* Component name and version */ + "sm", + OMPI_MAJOR_VERSION, + OMPI_MINOR_VERSION, + OMPI_RELEASE_VERSION, + sm_register, + NULL + }, + { + /* This component is checkpointable */ + MCA_BASE_METADATA_PARAM_CHECKPOINT + }, + mca_sharedfp_sm_component_init_query, /* get thread level */ + mca_sharedfp_sm_component_file_query, /* get priority and actions */ + mca_sharedfp_sm_component_file_unquery /* undo what was done by previous function */ +}; + +static int sm_register(void) +{ + mca_sharedfp_sm_priority = 10; + (void) mca_base_component_var_register(&mca_sharedfp_sm_component.sharedfpm_version, + "priority", "Priority of the sm sharedfp component", + MCA_BASE_VAR_TYPE_INT, NULL, 0, 0, + OPAL_INFO_LVL_9, + MCA_BASE_VAR_SCOPE_READONLY, &mca_sharedfp_sm_priority); + mca_sharedfp_sm_verbose = 0; + (void) mca_base_component_var_register(&mca_sharedfp_sm_component.sharedfpm_version, + "verbose", "Verbosity of the sm sharedfp component", + MCA_BASE_VAR_TYPE_INT, NULL, 0, 0, + OPAL_INFO_LVL_9, + MCA_BASE_VAR_SCOPE_READONLY, &mca_sharedfp_sm_verbose); + + return OMPI_SUCCESS; +} diff --git a/ompi/mca/sharedfp/sm/sharedfp_sm_file_open.c b/ompi/mca/sharedfp/sm/sharedfp_sm_file_open.c new file mode 100644 index 0000000000..932d30fdd7 --- /dev/null +++ b/ompi/mca/sharedfp/sm/sharedfp_sm_file_open.c @@ -0,0 +1,223 @@ +/* + * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana + * University Research and Technology + * Corporation. All rights reserved. + * Copyright (c) 2004-2005 The University of Tennessee and The University + * of Tennessee Research Foundation. All rights + * reserved. + * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, + * University of Stuttgart. All rights reserved. + * Copyright (c) 2004-2005 The Regents of the University of California. + * All rights reserved. + * Copyright (c) 2013 University of Houston. All rights reserved. + * $COPYRIGHT$ + * + * Additional copyrights may follow + * + * $HEADER$ + */ + + +#include "ompi_config.h" +#include "sharedfp_sm.h" + +#include "mpi.h" +#include "ompi/constants.h" +#include "orte/util/show_help.h" +#include "ompi/mca/sharedfp/sharedfp.h" +#include "ompi/mca/sharedfp/base/base.h" + +#include +#include + +int mca_sharedfp_sm_file_open (struct ompi_communicator_t *comm, + char* filename, + int amode, + struct ompi_info_t *info, + mca_io_ompio_file_t *fh) +{ + int err = OMPI_SUCCESS; + struct mca_sharedfp_base_data_t* sh; + struct mca_sharedfp_sm_data * sm_data = NULL; + mca_io_ompio_file_t * shfileHandle; + char * filename_basename; + char * sm_filename; + struct sm_offset * sm_offset_ptr; + int sm_fd; + int rank; + + /*----------------------------------------------------*/ + /*Open the same file again without shared file pointer*/ + /*----------------------------------------------------*/ + shfileHandle = (mca_io_ompio_file_t *)malloc(sizeof(mca_io_ompio_file_t)); + err = ompio_io_ompio_file_open(comm,filename,amode,info,shfileHandle,false); + if ( OMPI_SUCCESS != err) { + opal_output(1, "mca_sharedfp_sm_file_open: Error during file open\n"); + return err; + } + + /*Memory is allocated here for the sh structure*/ + if ( mca_sharedfp_sm_verbose ) { + printf( "mca_sharedfp_sm_file_open: malloc f_sharedfp_ptr struct\n"); + } + + sh = (struct mca_sharedfp_base_data_t*)malloc(sizeof(struct mca_sharedfp_base_data_t)); + if ( NULL == sh ) { + opal_output(1, "mca_sharedfp_sm_file_open: Error, unable to malloc f_sharedfp_ptr struct\n"); + free(shfileHandle); + return OMPI_ERR_OUT_OF_RESOURCE; + } + + /*Populate the sh file structure based on the implementation*/ + sh->sharedfh = shfileHandle; /* Shared file pointer*/ + sh->global_offset = 0; /* Global Offset*/ + sh->comm = comm; /* Communicator*/ + sh->selected_module_data = NULL; + + rank = ompi_comm_rank ( sh->comm ); + + /*Open a shared memory segment which will hold the shared file pointer*/ + if ( mca_sharedfp_sm_verbose ) { + printf( "mca_sharedfp_sm_file_open: allocatge shared memory segment.\n"); + } + + + sm_data = (struct mca_sharedfp_sm_data*) malloc ( sizeof(struct mca_sharedfp_sm_data)); + if ( NULL == sm_data ){ + opal_output(1, "mca_sharedfp_sm_file_open: Error, unable to malloc sm_data struct\n"); + free(sh); + free(shfileHandle); + return OMPI_ERR_OUT_OF_RESOURCE; + } + sm_data->sm_filename=NULL; + + + /* the shared memory segment is identified opening a file + ** and then mapping it to memory + ** For sharedfp we also want to put the file backed shared memory into the tmp directory + ** TODO: properly name the file so that different jobs can run on the same system w/o + ** overwriting each other, e.g. orte_process_info.proc_session_dir + */ + /*sprintf(sm_filename,"%s%s",filename,".sm");*/ + filename_basename = basename(filename); + sm_filename = (char*) malloc( sizeof(char) * (strlen(filename_basename)+64) ); + if (NULL == sm_filename) { + free(sh); + free(shfileHandle); + return OMPI_ERR_OUT_OF_RESOURCE; + } + sprintf(sm_filename,"/tmp/OMPIO_sharedfp_sm_%s%s",filename_basename,".sm"); + + /* open shared memory file, initialize to 0, map into memory */ + sm_fd = open(sm_filename, O_RDWR | O_CREAT, + S_IRUSR | S_IWUSR | S_IRGRP | S_IROTH); + if ( sm_fd == -1){ + /*error opening file*/ + printf("mca_sharedfp_sm_file_open: Error, unable to open file for mmap: %s\n",sm_filename); + free(sh); + free(shfileHandle); + return OMPI_ERROR; + } + + sm_data->sm_filename = sm_filename; + + /*TODO: is it necessary to write to the file first?*/ + if( 0 == rank ){ + write(sm_fd, &sm_offset, sizeof(struct sm_offset)); + } + comm->c_coll.coll_barrier (comm, comm->c_coll.coll_barrier_module ); + + /*the file has been written to, now we can map*/ + sm_offset_ptr = mmap(NULL, sizeof(struct sm_offset), PROT_READ | PROT_WRITE, + MAP_SHARED, sm_fd, 0); + + close(sm_fd); + + if ( sm_offset_ptr==MAP_FAILED){ + err = OMPI_ERROR; + printf("mca_sharedfp_sm_file_open: Error, unable to mmap file: %s\n",sm_filename); + printf("%s\n", strerror(errno)); + free(sh); + free(shfileHandle); + return OMPI_ERROR; + } + + /* Initialize semaphore so that is shared between processes. */ + /* the semaphore is shared by keeping it in the shared memory segment */ + if(sem_init(&sm_offset_ptr->mutex, 1, 1) != -1){ + /*If opening was successful*/ + /*Store the new file handle*/ + sm_data->sm_offset_ptr = sm_offset_ptr; + /* Assign the sm_data to sh->selected_module_data*/ + sh->selected_module_data = sm_data; + /*remember the shared file handle*/ + fh->f_sharedfp_data = sh; + + /*write initial zero*/ + if(rank==0){ + MPI_Offset position=0; + sem_wait(&sm_offset_ptr->mutex); + sm_offset_ptr->offset=position; + sem_post(&sm_offset_ptr->mutex); + } + }else{ + free(sm_data); + free(sh); + free(shfileHandle); + err = OMPI_ERROR; + } + + comm->c_coll.coll_barrier (comm, comm->c_coll.coll_barrier_module ); + + return err; +} + +int mca_sharedfp_sm_file_close (mca_io_ompio_file_t *fh) +{ + int err = OMPI_SUCCESS; + /*sharedfp data structure*/ + struct mca_sharedfp_base_data_t *sh=NULL; + /*sharedfp sm module data structure*/ + struct mca_sharedfp_sm_data * file_data=NULL; + + if( NULL == fh->f_sharedfp_data ){ + if ( mca_sharedfp_sm_verbose ) { + printf("sharedfp_sm_file_close: shared file pointer structure not initialized\n"); + } + return OMPI_SUCCESS; + } + sh = fh->f_sharedfp_data; + + /* Use an MPI Barrier in order to make sure that + * all processes are ready to release the + * shared file pointer resources + */ + sh->comm->c_coll.coll_barrier (sh->comm, sh->comm->c_coll.coll_barrier_module ); + + file_data = (sm_data*)(sh->selected_module_data); + if (file_data) { + /*Close sm handle*/ + if (file_data->sm_offset_ptr) { + /* destroy semaphore */ + sem_destroy(&file_data->sm_offset_ptr->mutex); + /*Release the shared memory segment.*/ + munmap(file_data->sm_offset_ptr,sizeof(struct sm_offset)); + /*Q: Do we need to delete the file? */ + remove(file_data->sm_filename); + } + /*free our sm data structure*/ + if(file_data->sm_filename){ + free(file_data->sm_filename); + } + free(file_data); + } + + /* Close the main file opened by this component*/ + err = ompio_io_ompio_file_close(sh->sharedfh); + + /*free shared file pointer data struct*/ + free(sh); + + return err; + +} diff --git a/ompi/mca/sharedfp/sm/sharedfp_sm_get_position.c b/ompi/mca/sharedfp/sm/sharedfp_sm_get_position.c new file mode 100644 index 0000000000..d2816dcc61 --- /dev/null +++ b/ompi/mca/sharedfp/sm/sharedfp_sm_get_position.c @@ -0,0 +1,61 @@ +/* + * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana + * University Research and Technology + * Corporation. All rights reserved. + * Copyright (c) 2004-2005 The University of Tennessee and The University + * of Tennessee Research Foundation. All rights + * reserved. + * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, + * University of Stuttgart. All rights reserved. + * Copyright (c) 2004-2005 The Regents of the University of California. + * All rights reserved. + * Copyright (c) 2013 University of Houston. All rights reserved. + * $COPYRIGHT$ + * + * Additional copyrights may follow + * + * $HEADER$ + */ + + +#include "ompi_config.h" +#include "sharedfp_sm.h" + +#include "mpi.h" +#include "ompi/constants.h" +#include "orte/util/show_help.h" +#include "ompi/mca/sharedfp/sharedfp.h" + +int +mca_sharedfp_sm_get_position(mca_io_ompio_file_t *fh, + OMPI_MPI_OFFSET_TYPE * offset) +{ + int ret = OMPI_SUCCESS; + + struct mca_sharedfp_base_data_t *sh = NULL; + mca_sharedfp_base_module_t * shared_fp_base_module = NULL; + + if(fh->f_sharedfp_data==NULL){ + opal_output(1,"sharedfp_sm_write - opening the shared file pointer\n"); + shared_fp_base_module = fh->f_sharedfp; + + ret = shared_fp_base_module->sharedfp_file_open(fh->f_comm, + fh->f_filename, + fh->f_amode, + fh->f_info, + fh); + if (ret != OMPI_SUCCESS) { + opal_output(1,"sharedfp_sm_write - error opening the shared file pointer\n"); + return ret; + } + } + /*Retrieve the shared file data struct*/ + sh = fh->f_sharedfp_data; + + /*Requesting the offset to write 0 bytes, + *returns the current offset w/o updating it + */ + ret = mca_sharedfp_sm_request_position(sh,0,offset); + + return ret; +} diff --git a/ompi/mca/sharedfp/sm/sharedfp_sm_iread.c b/ompi/mca/sharedfp/sm/sharedfp_sm_iread.c new file mode 100644 index 0000000000..84ec0e8e9d --- /dev/null +++ b/ompi/mca/sharedfp/sm/sharedfp_sm_iread.c @@ -0,0 +1,99 @@ +/* + * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana + * University Research and Technology + * Corporation. All rights reserved. + * Copyright (c) 2004-2005 The University of Tennessee and The University + * of Tennessee Research Foundation. All rights + * reserved. + * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, + * University of Stuttgart. All rights reserved. + * Copyright (c) 2004-2005 The Regents of the University of California. + * All rights reserved. + * Copyright (c) 2013 University of Houston. All rights reserved. + * $COPYRIGHT$ + * + * Additional copyrights may follow + * + * $HEADER$ + */ + + +#include "ompi_config.h" +#include "sharedfp_sm.h" + +#include "mpi.h" +#include "ompi/constants.h" +#include "orte/util/show_help.h" +#include "ompi/mca/sharedfp/sharedfp.h" + +int mca_sharedfp_sm_iread(mca_io_ompio_file_t *fh, + void *buf, + int count, + ompi_datatype_t *datatype, + MPI_Request * request) +{ + int ret = OMPI_SUCCESS; + OMPI_MPI_OFFSET_TYPE offset = 0; + long bytesRequested = 0; + size_t numofBytes; + struct mca_sharedfp_base_data_t *sh = NULL; + mca_sharedfp_base_module_t * shared_fp_base_module = NULL; + + if( NULL == fh->f_sharedfp_data){ + if ( mca_sharedfp_sm_verbose ) { + printf("sharedfp_sm_iread: opening the shared file pointer\n"); + } + shared_fp_base_module = fh->f_sharedfp; + + ret = shared_fp_base_module->sharedfp_file_open(fh->f_comm, + fh->f_filename, + fh->f_amode, + fh->f_info, + fh); + if ( OMPI_SUCCESS != ret ) { + opal_output(1,"sharedfp_sm_iread - error opening the shared file pointer\n"); + return ret; + } + } + + /* Calculate the number of bytes to write */ + opal_datatype_type_size ( &datatype->super, &numofBytes); + bytesRequested = count * numofBytes; + + /* Retrieve the shared file data struct */ + sh = fh->f_sharedfp_data; + + if ( mca_sharedfp_sm_verbose ) { + printf("sharedfp_sm_iread: Bytes Requested is %ld\n",bytesRequested); + } + /*Request the offset to write bytesRequested bytes*/ + ret = mca_sharedfp_sm_request_position(sh,bytesRequested,&offset); + + if ( -1 != ret ) { + if ( mca_sharedfp_sm_verbose ) { + printf("sharedfp_sm_iread: Offset received is %lld\n",offset); + } + /* Read the file */ + ret = ompio_io_ompio_file_iread_at(sh->sharedfh,offset,buf,count,datatype,request); + } + + return ret; +} + +int mca_sharedfp_sm_read_ordered_begin(mca_io_ompio_file_t *fh, + void *buf, + int count, + struct ompi_datatype_t *datatype) +{ + opal_output(1,"mca_sharedfp_sm_read_ordered_begin: NOT IMPLEMENTED\n"); + return OMPI_ERROR; +} + + +int mca_sharedfp_sm_read_ordered_end(mca_io_ompio_file_t *fh, + void *buf, + ompi_status_public_t *status) +{ + opal_output(1,"mca_sharedfp_sm_read_ordered_end: NOT IMPLEMENTED\n"); + return OMPI_ERROR; +} diff --git a/ompi/mca/sharedfp/sm/sharedfp_sm_iwrite.c b/ompi/mca/sharedfp/sm/sharedfp_sm_iwrite.c new file mode 100644 index 0000000000..7f15d0ab36 --- /dev/null +++ b/ompi/mca/sharedfp/sm/sharedfp_sm_iwrite.c @@ -0,0 +1,100 @@ +/* + * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana + * University Research and Technology + * Corporation. All rights reserved. + * Copyright (c) 2004-2005 The University of Tennessee and The University + * of Tennessee Research Foundation. All rights + * reserved. + * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, + * University of Stuttgart. All rights reserved. + * Copyright (c) 2004-2005 The Regents of the University of California. + * All rights reserved. + * Copyright (c) 2013 University of Houston. All rights reserved. + * $COPYRIGHT$ + * + * Additional copyrights may follow + * + * $HEADER$ + */ + + +#include "ompi_config.h" +#include "sharedfp_sm.h" + +#include "mpi.h" +#include "ompi/constants.h" +#include "orte/util/show_help.h" +#include "ompi/mca/sharedfp/sharedfp.h" + +int mca_sharedfp_sm_iwrite(mca_io_ompio_file_t *fh, + void *buf, + int count, + ompi_datatype_t *datatype, + MPI_Request * request) +{ + int ret = OMPI_SUCCESS; + OMPI_MPI_OFFSET_TYPE offset = 0; + long bytesRequested = 0; + size_t numofBytes; + struct mca_sharedfp_base_data_t *sh = NULL; + mca_sharedfp_base_module_t * shared_fp_base_module = NULL; + + if( NULL == fh->f_sharedfp_data){ + if ( mca_sharedfp_sm_verbose ) { + printf("sharedfp_sm_iwrite - opening the shared file pointer\n"); + } + shared_fp_base_module = fh->f_sharedfp; + + ret = shared_fp_base_module->sharedfp_file_open(fh->f_comm, + fh->f_filename, + fh->f_amode, + fh->f_info, + fh); + if ( OMPI_SUCCESS != ret ) { + opal_output(1,"sharedfp_sm_iwrite - error opening the shared file pointer\n"); + return ret; + } + } + + /* Calculate the number of bytes to write */ + opal_datatype_type_size ( &datatype->super, &numofBytes); + bytesRequested = count * numofBytes; + + /* Retrieve the shared file data struct */ + sh = fh->f_sharedfp_data; + + if ( mca_sharedfp_sm_verbose ) { + printf("sharedfp_sm_iwrite: Bytes Requested is %ld\n",bytesRequested); + } + /* Request the offset to write bytesRequested bytes */ + ret = mca_sharedfp_sm_request_position(sh,bytesRequested,&offset); + + if ( -1 != ret ) { + if ( mca_sharedfp_sm_verbose ) { + printf("sharedfp_sm_iwrite: Offset received is %lld\n",offset); + } + /* Write to the file */ + ret = ompio_io_ompio_file_iwrite_at(sh->sharedfh,offset,buf,count,datatype,request); + } + + return ret; + +} + +int mca_sharedfp_sm_write_ordered_begin(mca_io_ompio_file_t *fh, + void *buf, + int count, + struct ompi_datatype_t *datatype) +{ + opal_output(1,"mca_sharedfp_sm_write_ordered_begin: NOT IMPLEMENTED\n"); + return OMPI_ERROR; +} + + +int mca_sharedfp_sm_write_ordered_end(mca_io_ompio_file_t *fh, + void *buf, + ompi_status_public_t *status) +{ + opal_output(1,"mca_sharedfp_sm_write_ordered_end: NOT IMPLEMENTED\n"); + return OMPI_ERROR; +} diff --git a/ompi/mca/sharedfp/sm/sharedfp_sm_read.c b/ompi/mca/sharedfp/sm/sharedfp_sm_read.c new file mode 100644 index 0000000000..406031e780 --- /dev/null +++ b/ompi/mca/sharedfp/sm/sharedfp_sm_read.c @@ -0,0 +1,196 @@ +/* + * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana + * University Research and Technology + * Corporation. All rights reserved. + * Copyright (c) 2004-2005 The University of Tennessee and The University + * of Tennessee Research Foundation. All rights + * reserved. + * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, + * University of Stuttgart. All rights reserved. + * Copyright (c) 2004-2005 The Regents of the University of California. + * All rights reserved. + * Copyright (c) 2013 University of Houston. All rights reserved. + * $COPYRIGHT$ + * + * Additional copyrights may follow + * + * $HEADER$ + */ + + +#include "ompi_config.h" +#include "sharedfp_sm.h" + +#include "mpi.h" +#include "ompi/constants.h" +#include "orte/util/show_help.h" +#include "ompi/mca/sharedfp/sharedfp.h" + +int mca_sharedfp_sm_read ( mca_io_ompio_file_t *fh, + void *buf, int count, MPI_Datatype datatype, MPI_Status *status) +{ + int ret = OMPI_SUCCESS; + OMPI_MPI_OFFSET_TYPE offset = 0; + long bytesRequested = 0; + size_t numofBytes; + struct mca_sharedfp_base_data_t *sh = NULL; + mca_sharedfp_base_module_t * shared_fp_base_module = NULL; + + if( NULL == fh->f_sharedfp_data){ + if ( mca_sharedfp_sm_verbose ) { + printf("sharedfp_sm_read - opening the shared file pointer\n"); + } + shared_fp_base_module = fh->f_sharedfp; + + ret = shared_fp_base_module->sharedfp_file_open(fh->f_comm, + fh->f_filename, + fh->f_amode, + fh->f_info, + fh); + if ( OMPI_SUCCESS != ret ) { + opal_output(1,"sharedfp_sm_read - error opening the shared file pointer\n"); + return ret; + } + } + + /* Calculate the number of bytes to write */ + opal_datatype_type_size ( &datatype->super, &numofBytes); + bytesRequested = count * numofBytes; + + /* Retrieve the shared file data struct */ + sh = fh->f_sharedfp_data; + + if ( mca_sharedfp_sm_verbose ) { + printf("sharedfp_sm_read: Bytes Requested is %ld\n",bytesRequested); + } + + /*Request the offset to write bytesRequested bytes*/ + ret = mca_sharedfp_sm_request_position(sh,bytesRequested,&offset); + + if ( -1 != ret ) { + if ( mca_sharedfp_sm_verbose ) { + printf("sharedfp_sm_read: Offset received is %lld\n",offset); + } + + /* Read the file */ + ret = ompio_io_ompio_file_read_at(sh->sharedfh,offset,buf,count,datatype,status); + } + + return ret; +} + +int mca_sharedfp_sm_read_ordered (mca_io_ompio_file_t *fh, + void *buf, + int count, + struct ompi_datatype_t *datatype, + ompi_status_public_t *status) +{ + int ret = OMPI_SUCCESS; + OMPI_MPI_OFFSET_TYPE offset = 0; + long sendBuff = 0; + long *buff=NULL; + long offsetBuff; + OMPI_MPI_OFFSET_TYPE offsetReceived = 0; + long bytesRequested = 0; + int recvcnt = 1, sendcnt = 1; + size_t numofBytes; + int rank, size, i; + struct mca_sharedfp_base_data_t *sh = NULL; + mca_sharedfp_base_module_t * shared_fp_base_module = NULL; + + if ( NULL == fh->f_sharedfp_data){ + if ( mca_sharedfp_sm_verbose ) { + printf("sharedfp_sm_read_ordered: opening the shared file pointer\n"); + } + shared_fp_base_module = fh->f_sharedfp; + + ret = shared_fp_base_module->sharedfp_file_open(fh->f_comm, + fh->f_filename, + fh->f_amode, + fh->f_info, + fh); + if ( OMPI_SUCCESS != ret ) { + opal_output(1,"sharedfp_sm_read_ordered - error opening the shared file pointer\n"); + return ret; + } + } + + /*Retrieve the new communicator*/ + sh = fh->f_sharedfp_data; + + /* Calculate the number of bytes to read*/ + opal_datatype_type_size ( &datatype->super, &numofBytes); + sendBuff = count * numofBytes; + + /* Get the ranks in the communicator */ + rank = ompi_comm_rank ( sh->comm ); + size = ompi_comm_size ( sh->comm ); + + if ( 0 == rank ) { + buff = (long*)malloc(sizeof(long) * size); + if ( NULL == buff ) + return OMPI_ERR_OUT_OF_RESOURCE; + } + + ret = sh->comm->c_coll.coll_gather ( &sendBuff, sendcnt, OMPI_OFFSET_DATATYPE, + buff, recvcnt, OMPI_OFFSET_DATATYPE, 0, + sh->comm, sh->comm->c_coll.coll_gather_module ); + if( OMPI_SUCCESS != ret){ + goto exit; + } + + /* All the counts are present now in the recvBuff. + ** The size of recvBuff is sizeof_newComm + */ + if ( 0 == rank ) { + for (i = 0; i < size ; i ++) { + bytesRequested += buff[i]; + if ( mca_sharedfp_sm_verbose ) { + printf("mca_sharedfp_sm_read_ordered: Bytes requested are %ld\n",bytesRequested); + } + } + + /* Request the offset to read bytesRequested bytes + ** only the root process needs to do the request, + ** since the root process will then tell the other + ** processes at what offset they should read their + ** share of the data. + */ + ret = mca_sharedfp_sm_request_position(sh,bytesRequested,&offsetReceived); + if( OMPI_SUCCESS != ret){ + goto exit; + } + if ( mca_sharedfp_sm_verbose ) { + printf("mca_sharedfp_sm_read_ordered: Offset received is %lld\n",offsetReceived); + } + + buff[0] += offsetReceived; + for (i = 1 ; i < size; i++) { + buff[i] += buff[i-1]; + } + } + + /* Scatter the results to the other processes*/ + ret = sh->comm->c_coll.coll_scatter ( buff, sendcnt, OMPI_OFFSET_DATATYPE, + &offsetBuff, recvcnt, OMPI_OFFSET_DATATYPE, 0, + sh->comm, sh->comm->c_coll.coll_scatter_module ); + if( OMPI_SUCCESS != ret){ + goto exit; + } + + /*Each process now has its own individual offset in recvBUFF*/ + offset = offsetBuff - sendBuff; + if ( mca_sharedfp_sm_verbose ) { + printf("mca_sharedfp_sm_read_ordered: Offset returned is %lld\n",offset); + } + + /* read to the file */ + ret = ompio_io_ompio_file_read_at_all(sh->sharedfh,offset,buf,count,datatype,status); + +exit: + if ( NULL != buff ) { + free ( buff ); + } + + return ret; +} diff --git a/ompi/mca/sharedfp/sm/sharedfp_sm_request_position.c b/ompi/mca/sharedfp/sm/sharedfp_sm_request_position.c new file mode 100644 index 0000000000..2559bcffe3 --- /dev/null +++ b/ompi/mca/sharedfp/sm/sharedfp_sm_request_position.c @@ -0,0 +1,80 @@ +/* + * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana + * University Research and Technology + * Corporation. All rights reserved. + * Copyright (c) 2004-2005 The University of Tennessee and The University + * of Tennessee Research Foundation. All rights + * reserved. + * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, + * University of Stuttgart. All rights reserved. + * Copyright (c) 2004-2005 The Regents of the University of California. + * All rights reserved. + * Copyright (c) 2013 University of Houston. All rights reserved. + * $COPYRIGHT$ + * + * Additional copyrights may follow + * + * $HEADER$ + */ + + +#include "ompi_config.h" +#include "sharedfp_sm.h" + +#include "mpi.h" +#include "ompi/constants.h" +#include "orte/util/show_help.h" +#include "ompi/mca/sharedfp/sharedfp.h" + +/*use a semaphore to lock the shared memory*/ +#include + +int mca_sharedfp_sm_request_position(struct mca_sharedfp_base_data_t * sh, + int bytes_requested, + OMPI_MPI_OFFSET_TYPE *offset) +{ + int ret = OMPI_SUCCESS; + OMPI_MPI_OFFSET_TYPE position = 0; + OMPI_MPI_OFFSET_TYPE old_offset; + struct mca_sharedfp_sm_data * sm_data = sh->selected_module_data; + struct sm_offset * sm_offset_ptr = NULL; + int rank = ompi_comm_rank ( sh->comm); + + *offset = 0; + if ( mca_sharedfp_sm_verbose ) { + printf("Aquiring lock, rank=%d...",rank); + } + + sm_offset_ptr = sm_data->sm_offset_ptr; + + /* Aquire an exclusive lock */ + sem_wait(&sm_offset_ptr->mutex); + + if ( mca_sharedfp_sm_verbose ) { + printf("Succeeded! Acquired sm lock.for rank=%d\n",rank); + } + + old_offset=sm_offset_ptr->offset; + if ( mca_sharedfp_sm_verbose ) { + printf("Read last_offset=%lld!\n",old_offset); + } + + position = old_offset + bytes_requested; + if ( mca_sharedfp_sm_verbose ) { + printf("old_offset=%lld, bytes_requested=%d, new offset=%lld!\n",old_offset,bytes_requested,position); + } + sm_offset_ptr->offset=position; + + + if ( mca_sharedfp_sm_verbose ) { + printf("Releasing sm lock...rank=%d",rank); + } + sem_post(&sm_offset_ptr->mutex); + if ( mca_sharedfp_sm_verbose ) { + printf("Released lock! released lock.for rank=%d\n",rank); + } + + *offset = old_offset; + + return ret; +} diff --git a/ompi/mca/sharedfp/sm/sharedfp_sm_seek.c b/ompi/mca/sharedfp/sm/sharedfp_sm_seek.c new file mode 100644 index 0000000000..9496390449 --- /dev/null +++ b/ompi/mca/sharedfp/sm/sharedfp_sm_seek.c @@ -0,0 +1,142 @@ +/* + * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana + * University Research and Technology + * Corporation. All rights reserved. + * Copyright (c) 2004-2005 The University of Tennessee and The University + * of Tennessee Research Foundation. All rights + * reserved. + * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, + * University of Stuttgart. All rights reserved. + * Copyright (c) 2004-2005 The Regents of the University of California. + * All rights reserved. + * Copyright (c) 2013 University of Houston. All rights reserved. + * $COPYRIGHT$ + * + * Additional copyrights may follow + * + * $HEADER$ + */ + + +#include "ompi_config.h" +#include "sharedfp_sm.h" + +#include "mpi.h" +#include "ompi/constants.h" +#include "orte/util/show_help.h" +#include "ompi/mca/sharedfp/sharedfp.h" + +/*use a semaphore to lock the shared memory location*/ +#include + +int +mca_sharedfp_sm_seek (mca_io_ompio_file_t *fh, + OMPI_MPI_OFFSET_TYPE offset, int whence) +{ + int rank, status=0; + OMPI_MPI_OFFSET_TYPE end_position=0; + int ret = OMPI_SUCCESS; + struct mca_sharedfp_base_data_t *sh = NULL; + mca_sharedfp_base_module_t * shared_fp_base_module = NULL; + struct mca_sharedfp_sm_data * sm_data = NULL; + struct sm_offset * sm_offset_ptr = NULL; + + if( NULL == fh->f_sharedfp_data ) { + if ( mca_sharedfp_sm_verbose ) { + printf("sharedfp_sm_seek: opening the shared file pointer\n"); + } + shared_fp_base_module = fh->f_sharedfp; + + ret = shared_fp_base_module->sharedfp_file_open(fh->f_comm, + fh->f_filename, + fh->f_amode, + fh->f_info, + fh); + if ( OMPI_SUCCESS != ret ) { + opal_output(1,"sharedfp_sm_seek - error opening the shared file pointer\n"); + return ret; + } + } + + sh = fh->f_sharedfp_data; + rank = ompi_comm_rank ( sh->comm ); + + if( 0 == rank ){ + if ( MPI_SEEK_SET == whence){ + /*no nothing*/ + if ( offset < 0){ + opal_output(1,"sharedfp_sm_seek - MPI_SEEK_SET, offset must be > 0, got offset=%lld.\n",offset); + ret = -1; + } + if ( mca_sharedfp_sm_verbose ) { + printf("sharedfp_sm_seek: MPI_SEEK_SET new_offset=%lld\n",offset); + } + } + else if( MPI_SEEK_CUR == whence){ + OMPI_MPI_OFFSET_TYPE current_position; + ret = mca_sharedfp_sm_get_position ( fh, ¤t_position); + if ( mca_sharedfp_sm_verbose ) { + printf("sharedfp_sm_seek: MPI_SEEK_CUR: curr=%lld, offset=%lld, call status=%d\n", + current_position,offset,status); + } + offset = current_position + offset; + if ( mca_sharedfp_sm_verbose ) { + printf("sharedfp_sm_seek: MPI_SEEK_CUR: new_offset=%lld\n",offset); + } + if(offset < 0){ + opal_output(1,"sharedfp_sm_seek - MPI_SEEK_CURE, offset must be > 0, got offset=%lld.\n",offset); + ret = -1; + } + } + else if( MPI_SEEK_END == whence){ + end_position=0; + ompio_io_ompio_file_get_size(sh->sharedfh,&end_position); + + offset = end_position + offset; + if ( mca_sharedfp_sm_verbose ) { + printf("sharedfp_sm_seek: MPI_SEEK_END: file_get_size=%lld\n",end_position); + } + if(offset < 0){ + opal_output(1,"sharedfp_sm_seek - MPI_SEEK_CUR, offset must be > 0, got offset=%lld.\n",offset); + ret = -1; + } + } + else { + opal_output(1,"sharedfp_sm_seek - whence=%i is not supported\n",whence); + ret = -1; + } + + /*-----------------------------------------------------*/ + /* Set Shared file pointer */ + /*-----------------------------------------------------*/ + sm_data = sh->selected_module_data; + sm_offset_ptr = sm_data->sm_offset_ptr; + + /*-------------------*/ + /*lock the file */ + /*--------------------*/ + if ( mca_sharedfp_sm_verbose ) { + printf("sharedfp_sm_seek: Aquiring lock, rank=%d...",rank); fflush(stdout); + } + + /* Aquire an exclusive lock */ + sm_offset_ptr = sm_data->sm_offset_ptr; + sem_wait(&sm_offset_ptr->mutex); + + if ( mca_sharedfp_sm_verbose ) { + printf("sharedfp_sm_seek: Success! Acquired sm lock.for rank=%d\n",rank); + } + sm_offset_ptr->offset=offset; + if ( mca_sharedfp_sm_verbose ) { + printf("sharedfp_sm_seek: Releasing sm lock...rank=%d",rank); fflush(stdout); + } + sem_post(&sm_offset_ptr->mutex); + } + + /* since we are only letting process 0, update the current pointer + * all of the other processes need to wait before proceeding. + */ + sh->comm->c_coll.coll_barrier ( sh->comm, sh->comm->c_coll.coll_barrier_module ); + + return ret; +} diff --git a/ompi/mca/sharedfp/sm/sharedfp_sm_write.c b/ompi/mca/sharedfp/sm/sharedfp_sm_write.c new file mode 100644 index 0000000000..7f9348caca --- /dev/null +++ b/ompi/mca/sharedfp/sm/sharedfp_sm_write.c @@ -0,0 +1,202 @@ +/* + * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana + * University Research and Technology + * Corporation. All rights reserved. + * Copyright (c) 2004-2005 The University of Tennessee and The University + * of Tennessee Research Foundation. All rights + * reserved. + * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, + * University of Stuttgart. All rights reserved. + * Copyright (c) 2004-2005 The Regents of the University of California. + * All rights reserved. + * Copyright (c) 2013 University of Houston. All rights reserved. + * $COPYRIGHT$ + * + * Additional copyrights may follow + * + * $HEADER$ + */ + + +#include "ompi_config.h" +#include "sharedfp_sm.h" + +#include "mpi.h" +#include "ompi/constants.h" +#include "orte/util/show_help.h" +#include "ompi/mca/sharedfp/sharedfp.h" + +int mca_sharedfp_sm_write (mca_io_ompio_file_t *fh, + void *buf, + int count, + struct ompi_datatype_t *datatype, + ompi_status_public_t *status) +{ + int ret = OMPI_SUCCESS; + OMPI_MPI_OFFSET_TYPE offset = 0; + long bytesRequested = 0; + size_t numofBytes; + struct mca_sharedfp_base_data_t *sh = NULL; + mca_sharedfp_base_module_t * shared_fp_base_module = NULL; + + if( NULL == fh->f_sharedfp_data ){ + if ( mca_sharedfp_sm_verbose ) { + printf("sharedfp_sm_write: opening the shared file pointer\n"); + } + shared_fp_base_module = fh->f_sharedfp; + + ret = shared_fp_base_module->sharedfp_file_open(fh->f_comm, + fh->f_filename, + fh->f_amode, + fh->f_info, + fh); + if ( OMPI_SUCCESS != ret ) { + opal_output(1,"sharedfp_sm_write - error opening the shared file pointer\n"); + return ret; + } + } + + /* Calculate the number of bytes to write*/ + opal_datatype_type_size ( &datatype->super, &numofBytes); + bytesRequested = count * numofBytes; + + /*Retrieve the shared file data struct*/ + sh = fh->f_sharedfp_data; + + if ( mca_sharedfp_sm_verbose ) { + printf("sharedfp_sm_write: Requested is %ld\n",bytesRequested); + } + + /*Request the offset to write bytesRequested bytes*/ + ret = mca_sharedfp_sm_request_position(sh,bytesRequested,&offset); + if ( -1 != ret ) { + if ( mca_sharedfp_sm_verbose ) { + printf("sharedfp_sm_write: fset received is %lld\n",offset); + } + + /* Write to the file*/ + ret = ompio_io_ompio_file_write_at(sh->sharedfh,offset,buf,count,datatype,status); + } + + return ret; +} + +int mca_sharedfp_sm_write_ordered (mca_io_ompio_file_t *fh, + void *buf, + int count, + struct ompi_datatype_t *datatype, + ompi_status_public_t *status) +{ + int ret = OMPI_SUCCESS; + OMPI_MPI_OFFSET_TYPE offset = 0; + long sendBuff = 0; + long *buff=NULL; + long offsetBuff; + OMPI_MPI_OFFSET_TYPE offsetReceived = 0; + long bytesRequested = 0; + int recvcnt = 1, sendcnt = 1; + size_t numofBytes; + int rank, size; + int i; + + struct mca_sharedfp_base_data_t *sh = NULL; + mca_sharedfp_base_module_t * shared_fp_base_module = NULL; + + if( NULL == fh->f_sharedfp_data){ + if ( mca_sharedfp_sm_verbose ) { + printf("sharedfp_sm_write_ordered: opening the shared file pointer\n"); + } + shared_fp_base_module = fh->f_sharedfp; + + ret = shared_fp_base_module->sharedfp_file_open(fh->f_comm, + fh->f_filename, + fh->f_amode, + fh->f_info, + fh); + if ( OMPI_SUCCESS != ret ) { + opal_output(1,"sharedfp_sm_write_ordered: error opening the shared file pointer\n"); + return ret; + } + } + + + /*Retrieve the new communicator*/ + sh = fh->f_sharedfp_data; + + /* Calculate the number of bytes to write*/ + opal_datatype_type_size ( &datatype->super, &numofBytes); + sendBuff = count * numofBytes; + + /* Get the ranks in the communicator */ + rank = ompi_comm_rank ( sh->comm ); + size = ompi_comm_size ( sh->comm ); + + if ( 0 == rank ) { + buff = (long*)malloc(sizeof(long) * size); + if ( NULL == buff ) + return OMPI_ERR_OUT_OF_RESOURCE; + } + + ret = sh->comm->c_coll.coll_gather ( &sendBuff, sendcnt, OMPI_OFFSET_DATATYPE, + buff, recvcnt, OMPI_OFFSET_DATATYPE, 0, + sh->comm, sh->comm->c_coll.coll_gather_module ); + if ( OMPI_SUCCESS != ret ) { + goto exit; + } + + /* All the counts are present now in the recvBuff. + ** The size of recvBuff is sizeof_newComm + */ + if ( 0 == rank ) { + for (i = 0; i < size ; i ++) { + bytesRequested += buff[i]; + if ( mca_sharedfp_sm_verbose ) { + printf("sharedfp_sm_write_ordered: Bytes requested are %ld\n",bytesRequested); + } + } + + /* Request the offset to write bytesRequested bytes + ** only the root process needs to do the request, + ** since the root process will then tell the other + ** processes at what offset they should write their + ** share of the data. + */ + ret = mca_sharedfp_sm_request_position(sh,bytesRequested,&offsetReceived); + if( OMPI_SUCCESS != ret){ + goto exit; + } + if ( mca_sharedfp_sm_verbose ) { + printf("sharedfp_sm_write_ordered: Offset received is %lld\n",offsetReceived); + } + buff[0] += offsetReceived; + + for (i = 1 ; i < size; i++) { + buff[i] += buff[i-1]; + } + } + + /* Scatter the results to the other processes*/ + ret = sh->comm->c_coll.coll_scatter ( buff, sendcnt, OMPI_OFFSET_DATATYPE, + &offsetBuff, recvcnt, OMPI_OFFSET_DATATYPE, 0, + sh->comm, sh->comm->c_coll.coll_scatter_module ); + + if ( OMPI_SUCCESS != ret ) { + goto exit; + } + + /* Each process now has its own individual offset */ + offset = offsetBuff - sendBuff; + + if ( mca_sharedfp_sm_verbose ) { + printf("sharedfp_sm_write_ordered: Offset returned is %lld\n",offset); + } + /* write to the file */ + ret = ompio_io_ompio_file_write_at_all(sh->sharedfh,offset,buf,count,datatype,status); + +exit: + if ( NULL != buff ) { + free ( buff ); + } + + return ret; +}