diff --git a/ompi/mca/common/ompio/Makefile.am b/ompi/mca/common/ompio/Makefile.am index aa7112cb03..6eda4644b8 100644 --- a/ompi/mca/common/ompio/Makefile.am +++ b/ompi/mca/common/ompio/Makefile.am @@ -11,7 +11,7 @@ # All rights reserved. # Copyright (c) 2008-2016 University of Houston. All rights reserved. # Copyright (c) 2016 IBM Corporation. All rights reserved. -# Copyright (c) 2017 Research Organization for Information Science +# Copyright (c) 2017-2018 Research Organization for Information Science # and Technology (RIST). All rights reserved. # # $COPYRIGHT$ @@ -22,11 +22,13 @@ # headers = \ + common_ompio_aggregators.h \ common_ompio_print_queue.h \ common_ompio_request.h \ common_ompio.h sources = \ + common_ompio_aggregators.c \ common_ompio_print_queue.c \ common_ompio_request.c \ common_ompio_file_open.c \ diff --git a/ompi/mca/common/ompio/common_ompio.h b/ompi/mca/common/ompio/common_ompio.h index 2edb9d280c..5a844ca522 100644 --- a/ompi/mca/common/ompio/common_ompio.h +++ b/ompi/mca/common/ompio/common_ompio.h @@ -11,6 +11,8 @@ * Copyright (c) 2004-2005 The Regents of the University of California. * All rights reserved. * Copyright (c) 2008-2016 University of Houston. All rights reserved. + * Copyright (c) 2018 Research Organization for Information Science + * and Technology (RIST). All rights reserved. * $COPYRIGHT$ * * Additional copyrights may follow @@ -21,8 +23,212 @@ #ifndef MCA_COMMON_OMPIO_H #define MCA_COMMON_OMPIO_H -#include "ompi/mca/common/ompio/common_ompio_print_queue.h" -#include "ompi/mca/io/ompio/io_ompio.h" +#include + +#include "mpi.h" +#include "opal/class/opal_list.h" +#include "ompi/errhandler/errhandler.h" +#include "opal/threads/mutex.h" +#include "ompi/file/file.h" +#include "ompi/mca/io/io.h" +#include "ompi/mca/fs/fs.h" +#include "ompi/mca/fcoll/fcoll.h" +#include "ompi/mca/fbtl/fbtl.h" +#include "ompi/mca/sharedfp/sharedfp.h" +#include "ompi/communicator/communicator.h" +#include "ompi/info/info.h" +#include "opal/datatype/opal_convertor.h" +#include "ompi/datatype/ompi_datatype.h" +#include "ompi/request/request.h" + +#define OMPIO_MIN(a, b) (((a) < (b)) ? (a) : (b)) +#define OMPIO_MAX(a, b) (((a) < (b)) ? (b) : (a)) + +#define OMPIO_MCA_GET(fh, name) ((fh)->f_get_mca_parameter_value(#name, strlen(#name)+1)) + +/* + * Flags + */ +#define OMPIO_CONTIGUOUS_MEMORY 0x00000001 +#define OMPIO_UNIFORM_FVIEW 0x00000002 +#define OMPIO_FILE_IS_OPEN 0x00000004 +#define OMPIO_FILE_VIEW_IS_SET 0x00000008 +#define OMPIO_CONTIGUOUS_FVIEW 0x00000010 +#define OMPIO_AGGREGATOR_IS_SET 0x00000020 +#define OMPIO_SHAREDFP_IS_SET 0x00000040 +#define OMPIO_LOCK_ENTIRE_FILE 0x00000080 +#define OMPIO_LOCK_NEVER 0x00000100 +#define OMPIO_LOCK_NOT_THIS_OP 0x00000200 + + +#define OMPIO_ROOT 0 + +/*AGGREGATOR GROUPING DECISIONS*/ +#define OMPIO_MERGE 1 +#define OMPIO_SPLIT 2 +#define OMPIO_RETAIN 3 + +#define DATA_VOLUME 1 +#define UNIFORM_DISTRIBUTION 2 +#define CONTIGUITY 3 +#define OPTIMIZE_GROUPING 4 +#define SIMPLE 5 +#define NO_REFINEMENT 6 +#define SIMPLE_PLUS 7 + +#define OMPIO_LOCK_ENTIRE_REGION 10 +#define OMPIO_LOCK_SELECTIVE 11 + +#define OMPIO_FCOLL_WANT_TIME_BREAKDOWN 0 +#define MCA_IO_DEFAULT_FILE_VIEW_SIZE 4*1024*1024 + +#define OMPIO_UNIFORM_DIST_THRESHOLD 0.5 +#define OMPIO_CONTG_THRESHOLD 1048576 +#define OMPIO_CONTG_FACTOR 8 +#define OMPIO_DEFAULT_STRIPE_SIZE 1048576 +#define OMPIO_PROCS_PER_GROUP_TAG 0 +#define OMPIO_PROCS_IN_GROUP_TAG 1 +#define OMPIO_MERGE_THRESHOLD 0.5 + +#define OMPIO_PERM_NULL -1 +#define OMPIO_IOVEC_INITIAL_SIZE 100 + +enum ompio_fs_type +{ + NONE = 0, + UFS = 1, + PVFS2 = 2, + LUSTRE = 3, + PLFS = 4 +}; + +typedef struct mca_io_ompio_io_array_t { + void *memory_address; + /* we need that of type OMPI_MPI_OFFSET_TYPE */ + void *offset; + size_t length; + /*mca_io_ompio_server_t io_server;*/ +} mca_io_ompio_io_array_t; + + +typedef struct mca_io_ompio_access_array_t{ + OMPI_MPI_OFFSET_TYPE *offsets; + int *lens; + MPI_Aint *mem_ptrs; + int count; +} mca_io_ompio_access_array_t; + + +/* forward declaration to keep the compiler happy. */ +struct mca_io_ompio_file_t; +typedef int (*mca_io_ompio_generate_current_file_view_fn_t) (struct mca_io_ompio_file_t *fh, + size_t max_data, + struct iovec **f_iov, + int *iov_count); + +/* functions to retrieve the number of aggregators and the size of the + temporary buffer on aggregators from the fcoll modules */ +typedef int (*mca_io_ompio_get_mca_parameter_value_fn_t) ( char *mca_parameter_name, int name_length ); + + +struct mca_common_ompio_print_queue; + +/** + * Back-end structure for MPI_File + */ +struct mca_io_ompio_file_t { + /* General parameters */ + int fd; + struct ompi_file_t *f_fh; /* pointer back to the file_t structure */ + OMPI_MPI_OFFSET_TYPE f_offset; /* byte offset of current position */ + OMPI_MPI_OFFSET_TYPE f_disp; /* file_view displacement */ + int f_rank; + int f_size; + int f_amode; + int f_perm; + ompi_communicator_t *f_comm; + const char *f_filename; + char *f_datarep; + opal_convertor_t *f_convertor; + opal_info_t *f_info; + int32_t f_flags; + void *f_fs_ptr; + int f_fs_block_size; + int f_atomicity; + size_t f_stripe_size; + int f_stripe_count; + size_t f_cc_size; + int f_bytes_per_agg; + enum ompio_fs_type f_fstype; + ompi_request_t *f_split_coll_req; + bool f_split_coll_in_use; + /* Place for selected sharedfp module to hang it's data. + Note: Neither f_sharedfp nor f_sharedfp_component seemed appropriate for this. + */ + void *f_sharedfp_data; + + + /* File View parameters */ + struct iovec *f_decoded_iov; + uint32_t f_iov_count; + ompi_datatype_t *f_iov_type; + size_t f_position_in_file_view; /* in bytes */ + size_t f_total_bytes; /* total bytes read/written within 1 Fview*/ + int f_index_in_file_view; + ptrdiff_t f_view_extent; + size_t f_view_size; + ompi_datatype_t *f_etype; + ompi_datatype_t *f_filetype; + ompi_datatype_t *f_orig_filetype; /* the fileview passed by the user to us */ + size_t f_etype_size; + + /* contains IO requests that needs to be read/written */ + mca_io_ompio_io_array_t *f_io_array; + int f_num_of_io_entries; + + /* Hooks for modules to hang things */ + mca_base_component_t *f_fs_component; + mca_base_component_t *f_fcoll_component; + mca_base_component_t *f_fbtl_component; + mca_base_component_t *f_sharedfp_component; + + /* structure of function pointers */ + mca_fs_base_module_t *f_fs; + mca_fcoll_base_module_t *f_fcoll; + mca_fbtl_base_module_t *f_fbtl; + mca_sharedfp_base_module_t *f_sharedfp; + + /* Timing information */ + struct mca_common_ompio_print_queue *f_coll_write_time; + struct mca_common_ompio_print_queue *f_coll_read_time; + + /*initial list of aggregators and groups*/ + int *f_init_aggr_list; + int f_init_num_aggrs; + int f_init_procs_per_group; + int *f_init_procs_in_group; + + /* final of aggregators and groups*/ + int *f_aggr_list; + int f_num_aggrs; + int *f_procs_in_group; + int f_procs_per_group; + + /* internal ompio functions required by fbtl and fcoll */ + mca_io_ompio_generate_current_file_view_fn_t f_generate_current_file_view; + + mca_io_ompio_get_mca_parameter_value_fn_t f_get_mca_parameter_value; +}; +typedef struct mca_io_ompio_file_t mca_io_ompio_file_t; + +struct mca_io_ompio_data_t { + mca_io_ompio_file_t ompio_fh; +}; +typedef struct mca_io_ompio_data_t mca_io_ompio_data_t; + + +#include "common_ompio_print_queue.h" +#include "common_ompio_aggregators.h" OMPI_DECLSPEC int mca_common_ompio_file_write (mca_io_ompio_file_t *fh, const void *buf, int count, struct ompi_datatype_t *datatype, @@ -79,6 +285,9 @@ OMPI_DECLSPEC int mca_common_ompio_file_open (ompi_communicator_t *comm, const c int amode, opal_info_t *info, mca_io_ompio_file_t *ompio_fh, bool use_sharedfp); +int mca_common_ompio_file_delete (const char *filename, + struct opal_info_t *info); + OMPI_DECLSPEC int mca_common_ompio_file_close (mca_io_ompio_file_t *ompio_fh); OMPI_DECLSPEC int mca_common_ompio_file_get_size (mca_io_ompio_file_t *ompio_fh, OMPI_MPI_OFFSET_TYPE *size); OMPI_DECLSPEC int mca_common_ompio_file_get_position (mca_io_ompio_file_t *fh,OMPI_MPI_OFFSET_TYPE *offset); @@ -89,4 +298,18 @@ OMPI_DECLSPEC int mca_common_ompio_set_view (mca_io_ompio_file_t *fh, OMPI_MPI_ opal_info_t *info); +/* + * Function that takes in a datatype and buffer, and decodes that datatype + * into an iovec using the convertor_raw function + */ +OMPI_DECLSPEC int mca_common_ompio_decode_datatype (struct mca_io_ompio_file_t *fh, + struct ompi_datatype_t *datatype, + int count, + const void *buf, + size_t *max_data, + struct iovec **iov, + uint32_t *iov_count); + +OMPI_DECLSPEC int mca_common_ompio_set_callbacks(mca_io_ompio_generate_current_file_view_fn_t generate_current_file_view, + mca_io_ompio_get_mca_parameter_value_fn_t get_mca_parameter_value); #endif /* MCA_COMMON_OMPIO_H */ diff --git a/ompi/mca/io/ompio/io_ompio_aggregators.c b/ompi/mca/common/ompio/common_ompio_aggregators.c similarity index 83% rename from ompi/mca/io/ompio/io_ompio_aggregators.c rename to ompi/mca/common/ompio/common_ompio_aggregators.c index af7b74c01b..303f5a119b 100644 --- a/ompi/mca/io/ompio/io_ompio_aggregators.c +++ b/ompi/mca/common/ompio/common_ompio_aggregators.c @@ -39,7 +39,7 @@ #include #include -#include "io_ompio.h" +#include "common_ompio.h" /* ** This file contains all the functionality related to determing the number of aggregators @@ -53,14 +53,13 @@ ** of the access pattern */ - static double cost_calc (int P, int P_agg, size_t Data_proc, size_t coll_buffer, int dim ); #define DIM1 1 #define DIM2 2 -int mca_io_ompio_simple_grouping(mca_io_ompio_file_t *fh, - int *num_groups_out, - mca_io_ompio_contg *contg_groups) +int mca_common_ompio_simple_grouping(mca_io_ompio_file_t *fh, + int *num_groups_out, + mca_common_ompio_contg *contg_groups) { int num_groups=1; @@ -103,7 +102,7 @@ int mca_io_ompio_simple_grouping(mca_io_ompio_file_t *fh, ** for mca_io_ompio_max_aggregators will decrease the maximum number of aggregators ** allowed for the given no. of processes. */ - dtime_threshold = (double) mca_io_ompio_aggregators_cutoff_threshold / 100.0; + dtime_threshold = (double) OMPIO_MCA_GET(fh, aggregators_cutoff_threshold) / 100.0; /* Determine whether to use the formula for 1-D or 2-D data decomposition. Anything ** that is not 1-D is assumed to be 2-D in this version @@ -176,20 +175,20 @@ int mca_io_ompio_simple_grouping(mca_io_ompio_file_t *fh, #endif /* Cap the maximum number of aggregators.*/ - if ( num_groups > (fh->f_size/mca_io_ompio_max_aggregators_ratio)) { - num_groups = (fh->f_size/mca_io_ompio_max_aggregators_ratio); + if ( num_groups > (fh->f_size/OMPIO_MCA_GET(fh, max_aggregators_ratio))) { + num_groups = (fh->f_size/OMPIO_MCA_GET(fh, max_aggregators_ratio)); } if ( 1 >= num_groups ) { num_groups = 1; } *num_groups_out = num_groups; - return mca_io_ompio_forced_grouping ( fh, num_groups, contg_groups); + return mca_common_ompio_forced_grouping ( fh, num_groups, contg_groups); } -int mca_io_ompio_forced_grouping ( mca_io_ompio_file_t *fh, - int num_groups, - mca_io_ompio_contg *contg_groups) +int mca_common_ompio_forced_grouping ( mca_io_ompio_file_t *fh, + int num_groups, + mca_common_ompio_contg *contg_groups) { int group_size = fh->f_size / num_groups; int rest = fh->f_size % num_groups; @@ -225,9 +224,9 @@ int mca_io_ompio_forced_grouping ( mca_io_ompio_file_t *fh, return OMPI_SUCCESS; } -int mca_io_ompio_fview_based_grouping(mca_io_ompio_file_t *fh, - int *num_groups, - mca_io_ompio_contg *contg_groups) +int mca_common_ompio_fview_based_grouping(mca_io_ompio_file_t *fh, + int *num_groups, + mca_common_ompio_contg *contg_groups) { int k = 0; @@ -322,9 +321,9 @@ exit: return ret; } -int mca_io_ompio_cart_based_grouping(mca_io_ompio_file_t *ompio_fh, - int *num_groups, - mca_io_ompio_contg *contg_groups) +int mca_common_ompio_cart_based_grouping(mca_io_ompio_file_t *ompio_fh, + int *num_groups, + mca_common_ompio_contg *contg_groups) { int k = 0; int g=0; @@ -449,9 +448,9 @@ exit: -int mca_io_ompio_finalize_initial_grouping(mca_io_ompio_file_t *fh, - int num_groups, - mca_io_ompio_contg *contg_groups) +int mca_common_ompio_finalize_initial_grouping(mca_io_ompio_file_t *fh, + int num_groups, + mca_common_ompio_contg *contg_groups) { int z = 0; @@ -502,9 +501,9 @@ int mca_io_ompio_finalize_initial_grouping(mca_io_ompio_file_t *fh, ** of aggregators. */ -int mca_io_ompio_set_aggregator_props (struct mca_io_ompio_file_t *fh, - int num_aggregators, - size_t bytes_per_proc) +int mca_common_ompio_set_aggregator_props (struct mca_io_ompio_file_t *fh, + int num_aggregators, + size_t bytes_per_proc) { int j; int ret=OMPI_SUCCESS; @@ -512,10 +511,10 @@ int mca_io_ompio_set_aggregator_props (struct mca_io_ompio_file_t *fh, fh->f_flags |= OMPIO_AGGREGATOR_IS_SET; if ( (-1 == num_aggregators) && - ((SIMPLE != mca_io_ompio_grouping_option && - NO_REFINEMENT != mca_io_ompio_grouping_option && - SIMPLE_PLUS != mca_io_ompio_grouping_option ))) { - ret = mca_io_ompio_create_groups(fh,bytes_per_proc); + ((SIMPLE != OMPIO_MCA_GET(fh, grouping_option) && + NO_REFINEMENT != OMPIO_MCA_GET(fh, grouping_option) && + SIMPLE_PLUS != OMPIO_MCA_GET(fh, grouping_option) ))) { + ret = mca_common_ompio_create_groups(fh,bytes_per_proc); } else { fh->f_procs_per_group = fh->f_init_procs_per_group; @@ -544,8 +543,11 @@ int mca_io_ompio_set_aggregator_props (struct mca_io_ompio_file_t *fh, -int mca_io_ompio_create_groups(mca_io_ompio_file_t *fh, - size_t bytes_per_proc) +/*****************************************************************************************************/ +/*****************************************************************************************************/ +/*****************************************************************************************************/ +int mca_common_ompio_create_groups(mca_io_ompio_file_t *fh, + size_t bytes_per_proc) { int is_aggregator = 0; @@ -561,46 +563,46 @@ int mca_io_ompio_create_groups(mca_io_ompio_file_t *fh, OMPI_MPI_OFFSET_TYPE bytes_per_group = 0; OMPI_MPI_OFFSET_TYPE *aggr_bytes_per_group = NULL; - ret = mca_io_ompio_prepare_to_group(fh, - &start_offsets_lens, - &end_offsets, - &aggr_bytes_per_group, - &bytes_per_group, - &decision_list, - bytes_per_proc, - &is_aggregator, - &ompio_grouping_flag); + ret = mca_common_ompio_prepare_to_group(fh, + &start_offsets_lens, + &end_offsets, + &aggr_bytes_per_group, + &bytes_per_group, + &decision_list, + bytes_per_proc, + &is_aggregator, + &ompio_grouping_flag); if ( OMPI_SUCCESS != ret ) { - opal_output (1, "mca_io_ompio_create_groups: error in mca_io_ompio_prepare_to_group\n"); + opal_output (1, "mca_common_ompio_create_groups: error in mca_common_ompio_prepare_to_group\n"); goto exit; } switch(ompio_grouping_flag){ case OMPIO_SPLIT: - ret = mca_io_ompio_split_initial_groups(fh, - start_offsets_lens, - end_offsets, - bytes_per_group); + ret = mca_common_ompio_split_initial_groups(fh, + start_offsets_lens, + end_offsets, + bytes_per_group); break; case OMPIO_MERGE: - ret = mca_io_ompio_merge_initial_groups(fh, - aggr_bytes_per_group, - decision_list, - is_aggregator); + ret = mca_common_ompio_merge_initial_groups(fh, + aggr_bytes_per_group, + decision_list, + is_aggregator); break; case OMPIO_RETAIN: - ret = mca_io_ompio_retain_initial_groups(fh); + ret = mca_common_ompio_retain_initial_groups(fh); break; } if ( OMPI_SUCCESS != ret ) { - opal_output (1, "mca_io_ompio_create_groups: error in subroutine called within switch statement\n"); + opal_output (1, "mca_common_ompio_create_groups: error in subroutine called within switch statement\n"); goto exit; } @@ -618,13 +620,13 @@ int mca_io_ompio_create_groups(mca_io_ompio_file_t *fh, fh->f_comm, fh->f_comm->c_coll->coll_allreduce_module); if ( OMPI_SUCCESS != ret ) { - opal_output (1, "mca_io_ompio_create_groups: error in allreduce\n"); + opal_output (1, "mca_common_ompio_create_groups: error in allreduce\n"); goto exit; } tmp_final_aggrs =(int*) malloc ( fh->f_size *sizeof(int)); if ( NULL == tmp_final_aggrs ) { - opal_output(1,"mca_io_ompio_create_groups: could not allocate memory\n"); + opal_output(1,"mca_common_ompio_create_groups: could not allocate memory\n"); goto exit; } ret = fh->f_comm->c_coll->coll_allgather (&final_aggr, @@ -636,7 +638,7 @@ int mca_io_ompio_create_groups(mca_io_ompio_file_t *fh, fh->f_comm, fh->f_comm->c_coll->coll_allgather_module); if ( OMPI_SUCCESS != ret ) { - opal_output (1, "mca_io_ompio_create_groups: error in allreduce\n"); + opal_output (1, "mca_common_ompio_create_groups: error in allreduce\n"); goto exit; } @@ -645,7 +647,7 @@ int mca_io_ompio_create_groups(mca_io_ompio_file_t *fh, fh->f_num_aggrs = final_num_aggrs; fh->f_aggr_list = (int*) malloc (fh->f_num_aggrs * sizeof(int)); if ( NULL == fh->f_aggr_list ) { - opal_output(1,"mca_io_ompio_create_groups: could not allocate memory\n"); + opal_output(1,"mca_common_ompio_create_groups: could not allocate memory\n"); goto exit; } @@ -679,10 +681,10 @@ exit: return ret; } -int mca_io_ompio_merge_initial_groups(mca_io_ompio_file_t *fh, - OMPI_MPI_OFFSET_TYPE *aggr_bytes_per_group, - int *decision_list, - int is_aggregator){ +int mca_common_ompio_merge_initial_groups(mca_io_ompio_file_t *fh, + OMPI_MPI_OFFSET_TYPE *aggr_bytes_per_group, + int *decision_list, + int is_aggregator){ OMPI_MPI_OFFSET_TYPE sum_bytes = 0; MPI_Request *sendreqs = NULL; @@ -711,13 +713,13 @@ int mca_io_ompio_merge_initial_groups(mca_io_ompio_file_t *fh, break; } else if((decision_list[i] == OMPIO_MERGE) && - (sum_bytes <= mca_io_ompio_bytes_per_agg)){ + (sum_bytes <= OMPIO_MCA_GET(fh, bytes_per_agg))){ sum_bytes = sum_bytes + aggr_bytes_per_group[i]; decision_list[i] = merge_pair_flag; i++; } else if((decision_list[i] == OMPIO_MERGE) && - (sum_bytes >= mca_io_ompio_bytes_per_agg)){ + (sum_bytes >= OMPIO_MCA_GET(fh, bytes_per_agg))){ if(decision_list[i+1] == OMPIO_MERGE){ merge_pair_flag++; decision_list[i] = merge_pair_flag; @@ -772,10 +774,10 @@ int mca_io_ompio_merge_initial_groups(mca_io_ompio_file_t *fh, for( j = 0 ; j < end-start+1 ;j++){ if(fh->f_rank == merge_aggrs[j]){ - ret = mca_io_ompio_merge_groups(fh, merge_aggrs, - end-start+1); + ret = mca_common_ompio_merge_groups(fh, merge_aggrs, + end-start+1); if ( OMPI_SUCCESS != ret ) { - opal_output (1, "mca_io_ompio_merge_initial_groups: error in mca_io_ompio_merge_groups\n"); + opal_output (1, "mca_common_ompio_merge_initial_groups: error in mca_common_ompio_merge_groups\n"); free ( merge_aggrs ); return ret; } @@ -813,7 +815,7 @@ int mca_io_ompio_merge_initial_groups(mca_io_ompio_file_t *fh, fh->f_comm, sendreqs + r++)); if ( OMPI_SUCCESS != ret ) { - opal_output (1, "mca_io_ompio_merge_initial_groups: error in Isend\n"); + opal_output (1, "mca_common_ompio_merge_initial_groups: error in Isend\n"); goto exit; } //new aggregator sends distribution of process to all its new members @@ -826,7 +828,7 @@ int mca_io_ompio_merge_initial_groups(mca_io_ompio_file_t *fh, fh->f_comm, sendreqs + r++)); if ( OMPI_SUCCESS != ret ) { - opal_output (1, "mca_io_ompio_merge_initial_groups: error in Isend 2\n"); + opal_output (1, "mca_common_ompio_merge_initial_groups: error in Isend 2\n"); goto exit; } @@ -843,7 +845,7 @@ int mca_io_ompio_merge_initial_groups(mca_io_ompio_file_t *fh, fh->f_comm, MPI_STATUS_IGNORE)); if ( OMPI_SUCCESS != ret ) { - opal_output (1, "mca_io_ompio_merge_initial_groups: error in Recv\n"); + opal_output (1, "mca_common_ompio_merge_initial_groups: error in Recv\n"); return ret; } @@ -861,7 +863,7 @@ int mca_io_ompio_merge_initial_groups(mca_io_ompio_file_t *fh, fh->f_comm, MPI_STATUS_IGNORE)); if ( OMPI_SUCCESS != ret ) { - opal_output (1, "mca_io_ompio_merge_initial_groups: error in Recv 2\n"); + opal_output (1, "mca_common_ompio_merge_initial_groups: error in Recv 2\n"); return ret; } @@ -879,10 +881,10 @@ exit: return ret; } -int mca_io_ompio_split_initial_groups(mca_io_ompio_file_t *fh, - OMPI_MPI_OFFSET_TYPE *start_offsets_lens, - OMPI_MPI_OFFSET_TYPE *end_offsets, - OMPI_MPI_OFFSET_TYPE bytes_per_group){ +int mca_common_ompio_split_initial_groups(mca_io_ompio_file_t *fh, + OMPI_MPI_OFFSET_TYPE *start_offsets_lens, + OMPI_MPI_OFFSET_TYPE *end_offsets, + OMPI_MPI_OFFSET_TYPE bytes_per_group){ int size_new_group = 0; @@ -895,24 +897,24 @@ int mca_io_ompio_split_initial_groups(mca_io_ompio_file_t *fh, OMPI_MPI_OFFSET_TYPE max_cci = 0; OMPI_MPI_OFFSET_TYPE min_cci = 0; - size_new_group = ceil ((float)mca_io_ompio_bytes_per_agg * fh->f_init_procs_per_group/ bytes_per_group); + size_new_group = ceil ((float)OMPIO_MCA_GET(fh, bytes_per_agg) * fh->f_init_procs_per_group/ bytes_per_group); size_old_group = fh->f_init_procs_per_group; - ret = mca_io_ompio_split_a_group(fh, - start_offsets_lens, - end_offsets, - size_new_group, - &max_cci, - &min_cci, - &num_groups, - &size_smallest_group); + ret = mca_common_ompio_split_a_group(fh, + start_offsets_lens, + end_offsets, + size_new_group, + &max_cci, + &min_cci, + &num_groups, + &size_smallest_group); if (OMPI_SUCCESS != ret ) { - opal_output (1, "mca_io_ompio_split_initial_groups: error in mca_io_ompio_split_a_group\n"); + opal_output (1, "mca_common_ompio_split_initial_groups: error in mca_common_ompio_split_a_group\n"); return ret; } - switch(mca_io_ompio_grouping_option){ + switch(OMPIO_MCA_GET(fh, grouping_option)){ case DATA_VOLUME: //Just use size as returned by split group size_last_group = size_smallest_group; @@ -944,16 +946,16 @@ int mca_io_ompio_split_initial_groups(mca_io_ompio_file_t *fh, (size_new_group < size_old_group)){ size_new_group = floor( (float) (size_new_group + size_old_group ) / 2 ); - ret = mca_io_ompio_split_a_group(fh, - start_offsets_lens, - end_offsets, - size_new_group, - &max_cci, - &min_cci, - &num_groups, - &size_smallest_group); + ret = mca_common_ompio_split_a_group(fh, + start_offsets_lens, + end_offsets, + size_new_group, + &max_cci, + &min_cci, + &num_groups, + &size_smallest_group); if (OMPI_SUCCESS != ret ) { - opal_output (1, "mca_io_ompio_split_initial_groups: error in mca_io_ompio_split_a_group 2\n"); + opal_output (1, "mca_common_ompio_split_initial_groups: error in mca_common_ompio_split_a_group 2\n"); return ret; } } @@ -972,16 +974,16 @@ int mca_io_ompio_split_initial_groups(mca_io_ompio_file_t *fh, //monitor the previous iteration //break if it has not changed. size_new_group = ceil( (float) (size_new_group + size_old_group ) / 2 ); - ret = mca_io_ompio_split_a_group(fh, - start_offsets_lens, - end_offsets, - size_new_group, - &max_cci, - &min_cci, - &num_groups, - &size_smallest_group); + ret = mca_common_ompio_split_a_group(fh, + start_offsets_lens, + end_offsets, + size_new_group, + &max_cci, + &min_cci, + &num_groups, + &size_smallest_group); if (OMPI_SUCCESS != ret ) { - opal_output (1, "mca_io_ompio_split_initial_groups: error in mca_io_ompio_split_a_group 3\n"); + opal_output (1, "mca_common_ompio_split_initial_groups: error in mca_common_ompio_split_a_group 3\n"); return ret; } } @@ -1010,13 +1012,13 @@ int mca_io_ompio_split_initial_groups(mca_io_ompio_file_t *fh, break; } - ret = mca_io_ompio_finalize_split(fh, size_new_group, size_last_group); + ret = mca_common_ompio_finalize_split(fh, size_new_group, size_last_group); return ret; } -int mca_io_ompio_retain_initial_groups(mca_io_ompio_file_t *fh){ +int mca_common_ompio_retain_initial_groups(mca_io_ompio_file_t *fh){ int i = 0; @@ -1034,9 +1036,9 @@ int mca_io_ompio_retain_initial_groups(mca_io_ompio_file_t *fh){ return OMPI_SUCCESS; } -int mca_io_ompio_merge_groups(mca_io_ompio_file_t *fh, - int *merge_aggrs, - int num_merge_aggrs) +int mca_common_ompio_merge_groups(mca_io_ompio_file_t *fh, + int *merge_aggrs, + int num_merge_aggrs) { int i = 0; int *sizes_old_group; @@ -1123,14 +1125,14 @@ exit: -int mca_io_ompio_split_a_group(mca_io_ompio_file_t *fh, - OMPI_MPI_OFFSET_TYPE *start_offsets_lens, - OMPI_MPI_OFFSET_TYPE *end_offsets, - int size_new_group, - OMPI_MPI_OFFSET_TYPE *max_cci, - OMPI_MPI_OFFSET_TYPE *min_cci, - int *num_groups, - int *size_smallest_group) +int mca_common_ompio_split_a_group(mca_io_ompio_file_t *fh, + OMPI_MPI_OFFSET_TYPE *start_offsets_lens, + OMPI_MPI_OFFSET_TYPE *end_offsets, + int size_new_group, + OMPI_MPI_OFFSET_TYPE *max_cci, + OMPI_MPI_OFFSET_TYPE *min_cci, + int *num_groups, + int *size_smallest_group) { OMPI_MPI_OFFSET_TYPE *cci = NULL; @@ -1184,9 +1186,9 @@ int mca_io_ompio_split_a_group(mca_io_ompio_file_t *fh, return OMPI_SUCCESS; } -int mca_io_ompio_finalize_split(mca_io_ompio_file_t *fh, - int size_new_group, - int size_last_group) +int mca_common_ompio_finalize_split(mca_io_ompio_file_t *fh, + int size_new_group, + int size_last_group) { //based on new group and last group finalize f_procs_per_group and f_procs_in_group @@ -1238,15 +1240,15 @@ int mca_io_ompio_finalize_split(mca_io_ompio_file_t *fh, return OMPI_SUCCESS; } -int mca_io_ompio_prepare_to_group(mca_io_ompio_file_t *fh, - OMPI_MPI_OFFSET_TYPE **start_offsets_lens, - OMPI_MPI_OFFSET_TYPE **end_offsets, // need it? - OMPI_MPI_OFFSET_TYPE **aggr_bytes_per_group, - OMPI_MPI_OFFSET_TYPE *bytes_per_group, - int **decision_list, - size_t bytes_per_proc, - int *is_aggregator, - int *ompio_grouping_flag) +int mca_common_ompio_prepare_to_group(mca_io_ompio_file_t *fh, + OMPI_MPI_OFFSET_TYPE **start_offsets_lens, + OMPI_MPI_OFFSET_TYPE **end_offsets, // need it? + OMPI_MPI_OFFSET_TYPE **aggr_bytes_per_group, + OMPI_MPI_OFFSET_TYPE *bytes_per_group, + int **decision_list, + size_t bytes_per_proc, + int *is_aggregator, + int *ompio_grouping_flag) { OMPI_MPI_OFFSET_TYPE start_offset_len[3] = {0}; @@ -1291,7 +1293,7 @@ int mca_io_ompio_prepare_to_group(mca_io_ompio_file_t *fh, fh->f_init_procs_per_group, fh->f_comm); if ( OMPI_SUCCESS != ret ) { - opal_output (1, "mca_io_ompio_prepare_to_grou[: error in ompi_fcoll_base_coll_allgather_array\n"); + opal_output (1, "mca_common_ompio_prepare_to_group: error in ompi_fcoll_base_coll_allgather_array\n"); goto exit; } end_offsets_tmp = (OMPI_MPI_OFFSET_TYPE* )malloc (fh->f_init_procs_per_group * sizeof(OMPI_MPI_OFFSET_TYPE)); @@ -1347,19 +1349,19 @@ int mca_io_ompio_prepare_to_group(mca_io_ompio_file_t *fh, fh->f_init_num_aggrs, fh->f_comm); if ( OMPI_SUCCESS != ret ) { - opal_output (1, "mca_io_ompio_prepare_to_group: error in ompi_fcoll_base_coll_allgather_array 2\n"); + opal_output (1, "mca_common_ompio_prepare_to_group: error in ompi_fcoll_base_coll_allgather_array 2\n"); free(decision_list_tmp); goto exit; } for( i = 0; i < fh->f_init_num_aggrs; i++){ if((size_t)(aggr_bytes_per_group_tmp[i])> - (size_t)mca_io_ompio_bytes_per_agg){ + (size_t)OMPIO_MCA_GET(fh, bytes_per_agg)){ decision_list_tmp[i] = OMPIO_SPLIT; split_count++; } else if((size_t)(aggr_bytes_per_group_tmp[i])< - (size_t)mca_io_ompio_bytes_per_agg){ + (size_t)OMPIO_MCA_GET(fh, bytes_per_agg)){ decision_list_tmp[i] = OMPIO_MERGE; merge_count++; } diff --git a/ompi/mca/common/ompio/common_ompio_aggregators.h b/ompi/mca/common/ompio/common_ompio_aggregators.h new file mode 100644 index 0000000000..8f2c6e579a --- /dev/null +++ b/ompi/mca/common/ompio/common_ompio_aggregators.h @@ -0,0 +1,111 @@ +/* -*- Mode: C; c-basic-offset:4 ; -*- */ +/* + * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana + * University Research and Technology + * Corporation. All rights reserved. + * Copyright (c) 2004-2013 The University of Tennessee and The University + * of Tennessee Research Foundation. All rights + * reserved. + * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, + * University of Stuttgart. All rights reserved. + * Copyright (c) 2004-2005 The Regents of the University of California. + * All rights reserved. + * Copyright (c) 2008-2016 University of Houston. All rights reserved. + * Copyright (c) 2018 Research Organization for Information Science + * and Technology (RIST). All rights reserved. + * + * $COPYRIGHT$ + * + * Additional copyrights may follow + * + * $HEADER$ + */ + + +#ifndef MCA_COMMON_OMPIO_AGGREGATORS_H +#define MCA_COMMON_OMPIO_AGGREGATORS_H + + +/*AGGREGATOR GROUPING DECISIONS*/ +#define OMPIO_MERGE 1 +#define OMPIO_SPLIT 2 +#define OMPIO_RETAIN 3 + +typedef struct { + int ndims; + int *dims; + int *periods; + int *coords; + int reorder; +} mca_io_ompio_cart_topo_components; + + +typedef struct{ + OMPI_MPI_OFFSET_TYPE contg_chunk_size; + int *procs_in_contg_group; + int procs_per_contg_group; +} mca_common_ompio_contg; + + +/*Aggregator selection methods*/ +OMPI_DECLSPEC int mca_common_ompio_set_aggregator_props (struct mca_io_ompio_file_t *fh, + int num_aggregators, + size_t bytes_per_proc); + +int mca_common_ompio_forced_grouping ( mca_io_ompio_file_t *fh, + int num_groups, + mca_common_ompio_contg *contg_groups); + +int mca_common_ompio_cart_based_grouping(mca_io_ompio_file_t *ompio_fh, int *num_groups, + mca_common_ompio_contg *contg_groups); + +int mca_common_ompio_fview_based_grouping(mca_io_ompio_file_t *fh, int *num_groups, + mca_common_ompio_contg *contg_groups); + +int mca_common_ompio_simple_grouping(mca_io_ompio_file_t *fh, int *num_groups, + mca_common_ompio_contg *contg_groups); + +int mca_common_ompio_finalize_initial_grouping(mca_io_ompio_file_t *fh, int num_groups, + mca_common_ompio_contg *contg_groups); + +int mca_common_ompio_create_groups(mca_io_ompio_file_t *fh, size_t bytes_per_proc); + +int mca_common_ompio_prepare_to_group(mca_io_ompio_file_t *fh, + OMPI_MPI_OFFSET_TYPE **start_offsets_lens, + OMPI_MPI_OFFSET_TYPE **end_offsets, + OMPI_MPI_OFFSET_TYPE **aggr_bytes_per_group, + OMPI_MPI_OFFSET_TYPE *bytes_per_group, + int **decision_list, + size_t bytes_per_proc, + int *is_aggregator, + int *ompio_grouping_flag); + +int mca_common_ompio_retain_initial_groups(mca_io_ompio_file_t *fh); + + +int mca_common_ompio_split_initial_groups(mca_io_ompio_file_t *fh, + OMPI_MPI_OFFSET_TYPE *start_offsets_lens, + OMPI_MPI_OFFSET_TYPE *end_offsets, + OMPI_MPI_OFFSET_TYPE bytes_per_group); + + +int mca_common_ompio_split_a_group(mca_io_ompio_file_t *fh, + OMPI_MPI_OFFSET_TYPE *start_offsets_lens, + OMPI_MPI_OFFSET_TYPE *end_offsets, + int size_new_group, + OMPI_MPI_OFFSET_TYPE *max_cci, + OMPI_MPI_OFFSET_TYPE *min_cci, + int *num_groups, int *size_smallest_group); + +int mca_common_ompio_finalize_split(mca_io_ompio_file_t *fh, int size_new_group, + int size_last_group); + +int mca_common_ompio_merge_initial_groups(mca_io_ompio_file_t *fh, + OMPI_MPI_OFFSET_TYPE *aggr_bytes_per_group, + int *decision_list, int is_aggregator); + +int mca_common_ompio_merge_groups(mca_io_ompio_file_t *fh, int *merge_aggrs, + int num_merge_aggrs); + + +#endif /* MCA_COMMON_AGGREGATORS_H */ diff --git a/ompi/mca/common/ompio/common_ompio_file_open.c b/ompi/mca/common/ompio/common_ompio_file_open.c index 47786c60c7..5f4f7811f6 100644 --- a/ompi/mca/common/ompio/common_ompio_file_open.c +++ b/ompi/mca/common/ompio/common_ompio_file_open.c @@ -10,9 +10,9 @@ * Copyright (c) 2004-2005 The Regents of the University of California. * All rights reserved. * Copyright (c) 2008-2017 University of Houston. All rights reserved. - * Copyright (c) 2015-2017 Research Organization for Information Science + * Copyright (c) 2015-2018 Research Organization for Information Science * and Technology (RIST). All rights reserved. - * Copyright (c) 2016 Cisco Systems, Inc. All rights reserved. + * Copyright (c) 2016 Cisco Systems, Inc. All rights reserved. * Copyright (c) 2017 IBM Corporation. All rights reserved. * $COPYRIGHT$ * @@ -41,6 +41,9 @@ #include "common_ompio.h" #include "ompi/mca/topo/topo.h" +static mca_io_ompio_generate_current_file_view_fn_t generate_current_file_view_fn; +static mca_io_ompio_get_mca_parameter_value_fn_t get_mca_parameter_value_fn; + int mca_common_ompio_file_open (ompi_communicator_t *comm, const char *filename, int amode, @@ -92,6 +95,10 @@ int mca_common_ompio_file_open (ompi_communicator_t *comm, ompio_fh->f_atomicity = 0; ompio_fh->f_fs_block_size = 4096; + /* set some function pointers required for fcoll, fbtls and sharedfp modules*/ + ompio_fh->f_generate_current_file_view=generate_current_file_view_fn; + ompio_fh->f_get_mca_parameter_value=get_mca_parameter_value_fn; + mca_common_ompio_set_file_defaults (ompio_fh); ompio_fh->f_filename = filename; @@ -102,16 +109,10 @@ int mca_common_ompio_file_open (ompi_communicator_t *comm, mca_common_ompio_initialize_print_queue(&ompio_fh->f_coll_write_time); mca_common_ompio_initialize_print_queue(&ompio_fh->f_coll_read_time); - /* set some function pointers required for fcoll, fbtls and sharedfp modules*/ - ompio_fh->f_decode_datatype=ompi_io_ompio_decode_datatype; - ompio_fh->f_generate_current_file_view=ompi_io_ompio_generate_current_file_view; - - ompio_fh->f_get_mca_parameter_value=mca_io_ompio_get_mca_parameter_value; - ompio_fh->f_set_aggregator_props=mca_io_ompio_set_aggregator_props; - /* This fix is needed for data seiving to work with two-phase collective I/O */ - if ( mca_io_ompio_overwrite_amode && !(amode & MPI_MODE_SEQUENTIAL) ) { + if ( OMPIO_MCA_GET(ompio_fh, overwrite_amode) && !(amode & MPI_MODE_SEQUENTIAL) ) { + if ((amode & MPI_MODE_WRONLY)){ amode -= MPI_MODE_WRONLY; amode += MPI_MODE_RDWR; @@ -176,7 +177,7 @@ int mca_common_ompio_file_open (ompi_communicator_t *comm, ** are used by his application. */ if ( NULL != ompio_fh->f_sharedfp && - !mca_io_ompio_sharedfp_lazy_open ) { + !OMPIO_MCA_GET(ompio_fh, sharedfp_lazy_open) ) { ret = ompio_fh->f_sharedfp->sharedfp_file_open(comm, filename, amode, @@ -201,7 +202,7 @@ int mca_common_ompio_file_open (ompi_communicator_t *comm, mca_common_ompio_set_explicit_offset (ompio_fh, current_size); if ( true == use_sharedfp ) { if ( NULL != ompio_fh->f_sharedfp && - !mca_io_ompio_sharedfp_lazy_open ) { + !OMPIO_MCA_GET(ompio_fh, sharedfp_lazy_open) ) { shared_fp_base_module = ompio_fh->f_sharedfp; ret = shared_fp_base_module->sharedfp_seek(ompio_fh,current_size, MPI_SEEK_SET); if ( MPI_SUCCESS != ret ) { @@ -241,7 +242,7 @@ int mca_common_ompio_file_close (mca_io_ompio_file_t *ompio_fh) } - if(mca_io_ompio_coll_timing_info){ + if(OMPIO_MCA_GET(ompio_fh, coll_timing_info)){ strcpy (name, "WRITE"); if (!mca_common_ompio_empty_print_queue(ompio_fh->f_coll_write_time)){ ret = mca_common_ompio_print_time_info(ompio_fh->f_coll_write_time, @@ -278,7 +279,7 @@ int mca_common_ompio_file_close (mca_io_ompio_file_t *ompio_fh) ret = ompio_fh->f_fs->fs_file_close (ompio_fh); } if ( delete_flag ) { - ret = mca_io_ompio_file_delete ( ompio_fh->f_filename, &(MPI_INFO_NULL->super) ); + ret = mca_common_ompio_file_delete ( ompio_fh->f_filename, &(MPI_INFO_NULL->super) ); } if ( NULL != ompio_fh->f_fs ) { @@ -407,7 +408,7 @@ int mca_common_ompio_set_file_defaults (mca_io_ompio_file_t *fh) fh->f_io_array = NULL; fh->f_perm = OMPIO_PERM_NULL; fh->f_flags = 0; - fh->f_bytes_per_agg = mca_io_ompio_bytes_per_agg; + fh->f_bytes_per_agg = OMPIO_MCA_GET(fh, bytes_per_agg); fh->f_datarep = strdup ("native"); fh->f_offset = 0; @@ -472,3 +473,153 @@ int mca_common_ompio_set_file_defaults (mca_io_ompio_file_t *fh) } +int mca_common_ompio_file_delete (const char *filename, + struct opal_info_t *info) +{ + int ret = OMPI_SUCCESS; + + /* No locking required for file_delete according to my understanding. + One thread will succeed, the other ones silently ignore the + error that the file is already deleted. + */ + ret = unlink(filename); + + if (0 > ret ) { + if ( ENOENT == errno ) { + return MPI_ERR_NO_SUCH_FILE; + } else { + opal_output (0, "mca_common_ompio_file_delete: Could not remove file %s errno = %d %s\n", filename, + errno, strerror(errno)); + return MPI_ERR_ACCESS; + } + } + + return OMPI_SUCCESS; +} + +int mca_common_ompio_decode_datatype (struct mca_io_ompio_file_t *fh, + ompi_datatype_t *datatype, + int count, + const void *buf, + size_t *max_data, + struct iovec **iov, + uint32_t *iovec_count) +{ + + + + opal_convertor_t convertor; + size_t remaining_length = 0; + uint32_t i; + uint32_t temp_count; + struct iovec *temp_iov=NULL; + size_t temp_data; + + + opal_convertor_clone (fh->f_convertor, &convertor, 0); + + if (OMPI_SUCCESS != opal_convertor_prepare_for_send (&convertor, + &(datatype->super), + count, + buf)) { + opal_output (1, "Cannot attach the datatype to a convertor\n"); + return OMPI_ERROR; + } + + if ( 0 == datatype->super.size ) { + *max_data = 0; + *iovec_count = 0; + *iov = NULL; + return OMPI_SUCCESS; + } + + remaining_length = count * datatype->super.size; + + temp_count = OMPIO_IOVEC_INITIAL_SIZE; + temp_iov = (struct iovec*)malloc(temp_count * sizeof(struct iovec)); + if (NULL == temp_iov) { + opal_output (1, "OUT OF MEMORY\n"); + return OMPI_ERR_OUT_OF_RESOURCE; + } + + while (0 == opal_convertor_raw(&convertor, + temp_iov, + &temp_count, + &temp_data)) { +#if 0 + printf ("%d: New raw extraction (iovec_count = %d, max_data = %lu)\n", + fh->f_rank,temp_count, (unsigned long)temp_data); + for (i = 0; i < temp_count; i++) { + printf ("%d: \t{%p, %lu}\n",fh->f_rank, + temp_iov[i].iov_base, + (unsigned long)temp_iov[i].iov_len); + } +#endif + + *iovec_count = *iovec_count + temp_count; + *max_data = *max_data + temp_data; + *iov = (struct iovec *) realloc (*iov, *iovec_count * sizeof(struct iovec)); + if (NULL == *iov) { + opal_output(1, "OUT OF MEMORY\n"); + free(temp_iov); + return OMPI_ERR_OUT_OF_RESOURCE; + } + for (i=0 ; if_rank,temp_count, temp_data); + for (i = 0; i < temp_count; i++) { + printf ("%d: \t offset[%d]: %ld; length[%d]: %ld\n", fh->f_rank,i,temp_iov[i].iov_base, i,temp_iov[i].iov_len); + } +#endif + *iovec_count = *iovec_count + temp_count; + *max_data = *max_data + temp_data; + if ( temp_count > 0 ) { + *iov = (struct iovec *) realloc (*iov, *iovec_count * sizeof(struct iovec)); + if (NULL == *iov) { + opal_output(1, "OUT OF MEMORY\n"); + free(temp_iov); + return OMPI_ERR_OUT_OF_RESOURCE; + } + } + for (i=0 ; if_rank) { + printf ("%d Entries: \n",*iovec_count); + for (i=0 ; i<*iovec_count ; i++) { + printf ("\t{%p, %d}\n", + (*iov)[i].iov_base, + (*iov)[i].iov_len); + } + } +#endif + if (remaining_length != 0) { + printf( "Not all raw description was been extracted (%lu bytes missing)\n", + (unsigned long) remaining_length ); + } + + free (temp_iov); + + return OMPI_SUCCESS; +} + +int mca_common_ompio_set_callbacks(mca_io_ompio_generate_current_file_view_fn_t generate_current_file_view, + mca_io_ompio_get_mca_parameter_value_fn_t get_mca_parameter_value) +{ + generate_current_file_view_fn = generate_current_file_view; + get_mca_parameter_value_fn = get_mca_parameter_value; + return OMPI_SUCCESS; +} diff --git a/ompi/mca/common/ompio/common_ompio_file_read.c b/ompi/mca/common/ompio/common_ompio_file_read.c index fcbe6d1e47..4bebf3e1d9 100644 --- a/ompi/mca/common/ompio/common_ompio_file_read.c +++ b/ompi/mca/common/ompio/common_ompio_file_read.c @@ -10,6 +10,8 @@ * Copyright (c) 2004-2005 The Regents of the University of California. * All rights reserved. * Copyright (c) 2008-2018 University of Houston. All rights reserved. + * Copyright (c) 2018 Research Organization for Information Science + * and Technology (RIST). All rights reserved. * $COPYRIGHT$ * * Additional copyrights may follow @@ -31,7 +33,6 @@ #include "common_ompio.h" #include "common_ompio_request.h" -#include "ompi/mca/io/ompio/io_ompio.h" #include "math.h" #include @@ -85,19 +86,19 @@ int mca_common_ompio_file_read (mca_io_ompio_file_t *fh, return ret; } - ompi_io_ompio_decode_datatype (fh, - datatype, - count, - buf, - &max_data, - &decoded_iov, - &iov_count); + mca_common_ompio_decode_datatype (fh, + datatype, + count, + buf, + &max_data, + &decoded_iov, + &iov_count); - if ( -1 == mca_io_ompio_cycle_buffer_size ) { + if ( -1 == OMPIO_MCA_GET(fh, cycle_buffer_size) ) { bytes_per_cycle = max_data; } else { - bytes_per_cycle = mca_io_ompio_cycle_buffer_size; + bytes_per_cycle = OMPIO_MCA_GET(fh, cycle_buffer_size); } cycles = ceil((float)max_data/bytes_per_cycle); @@ -207,13 +208,13 @@ int mca_common_ompio_file_iread (mca_io_ompio_file_t *fh, int i = 0; /* index into the decoded iovec of the buffer */ int j = 0; /* index into the file vie iovec */ - ompi_io_ompio_decode_datatype (fh, - datatype, - count, - buf, - &max_data, - &decoded_iov, - &iov_count); + mca_common_ompio_decode_datatype (fh, + datatype, + count, + buf, + &max_data, + &decoded_iov, + &iov_count); // Non-blocking operations have to occur in a single cycle j = fh->f_index_in_file_view; diff --git a/ompi/mca/common/ompio/common_ompio_file_view.c b/ompi/mca/common/ompio/common_ompio_file_view.c index 9486b66c80..d348480fa8 100644 --- a/ompi/mca/common/ompio/common_ompio_file_view.c +++ b/ompi/mca/common/ompio/common_ompio_file_view.c @@ -10,7 +10,7 @@ * Copyright (c) 2004-2005 The Regents of the University of California. * All rights reserved. * Copyright (c) 2008-2017 University of Houston. All rights reserved. - * Copyright (c) 2017 Research Organization for Information Science + * Copyright (c) 2017-2018 Research Organization for Information Science * and Technology (RIST). All rights reserved. * Copyright (c) 2017 IBM Corporation. All rights reserved. * $COPYRIGHT$ @@ -27,6 +27,7 @@ #include #include "common_ompio.h" +#include "common_ompio_aggregators.h" #include "ompi/mca/fcoll/base/base.h" #include "ompi/mca/topo/topo.h" @@ -65,7 +66,7 @@ int mca_common_ompio_set_view (mca_io_ompio_file_t *fh, int i, flag; int num_groups = 0; int num_cb_nodes=-1; - mca_io_ompio_contg *contg_groups=NULL; + mca_common_ompio_contg *contg_groups=NULL; size_t ftype_size; ptrdiff_t ftype_extent, lb, ub; @@ -123,13 +124,13 @@ int mca_common_ompio_set_view (mca_io_ompio_file_t *fh, fh->f_index_in_file_view=0; fh->f_position_in_file_view=0; - ompi_io_ompio_decode_datatype (fh, - newfiletype, - 1, - NULL, - &max_data, - &fh->f_decoded_iov, - &fh->f_iov_count); + mca_common_ompio_decode_datatype (fh, + newfiletype, + 1, + NULL, + &max_data, + &fh->f_decoded_iov, + &fh->f_iov_count); opal_datatype_get_extent(&newfiletype->super, &lb, &fh->f_view_extent); opal_datatype_type_ub (&newfiletype->super, &ub); @@ -141,7 +142,7 @@ int mca_common_ompio_set_view (mca_io_ompio_file_t *fh, ompi_datatype_duplicate (newfiletype, &fh->f_filetype); - if( SIMPLE_PLUS == mca_io_ompio_grouping_option ) { + if( SIMPLE_PLUS == OMPIO_MCA_GET(fh, grouping_option) ) { fh->f_cc_size = get_contiguous_chunk_size (fh, 1); } else { @@ -155,7 +156,7 @@ int mca_common_ompio_set_view (mca_io_ompio_file_t *fh, } } - contg_groups = (mca_io_ompio_contg*) calloc ( 1, fh->f_size * sizeof(mca_io_ompio_contg)); + contg_groups = (mca_common_ompio_contg*) calloc ( 1, fh->f_size * sizeof(mca_common_ompio_contg)); if (NULL == contg_groups) { opal_output (1, "OUT OF MEMORY\n"); return OMPI_ERR_OUT_OF_RESOURCE; @@ -188,9 +189,9 @@ int mca_common_ompio_set_view (mca_io_ompio_file_t *fh, } - if ( -1 != mca_io_ompio_num_aggregators || -1 != num_cb_nodes) { + if ( -1 != OMPIO_MCA_GET(fh, num_aggregators) || -1 != num_cb_nodes) { /* The user requested a particular number of aggregators */ - num_groups = mca_io_ompio_num_aggregators; + num_groups = OMPIO_MCA_GET(fh, num_aggregators); if ( -1 != num_cb_nodes ) { /* A hint through an MPI Info object trumps an mca parameter value */ num_groups = num_cb_nodes; @@ -198,14 +199,14 @@ int mca_common_ompio_set_view (mca_io_ompio_file_t *fh, if ( num_groups > fh->f_size ) { num_groups = fh->f_size; } - mca_io_ompio_forced_grouping ( fh, num_groups, contg_groups); + mca_common_ompio_forced_grouping ( fh, num_groups, contg_groups); } else { - if ( SIMPLE != mca_io_ompio_grouping_option && - SIMPLE_PLUS != mca_io_ompio_grouping_option ) { - ret = mca_io_ompio_fview_based_grouping(fh, - &num_groups, - contg_groups); + if ( SIMPLE != OMPIO_MCA_GET(fh, grouping_option) && + SIMPLE_PLUS != OMPIO_MCA_GET(fh, grouping_option) ) { + ret = mca_common_ompio_fview_based_grouping(fh, + &num_groups, + contg_groups); if ( OMPI_SUCCESS != ret ) { opal_output(1, "mca_common_ompio_set_view: mca_io_ompio_fview_based_grouping failed\n"); goto exit; @@ -221,9 +222,9 @@ int mca_common_ompio_set_view (mca_io_ompio_file_t *fh, goto exit; } if ( ndims > 1 ) { - ret = mca_io_ompio_cart_based_grouping( fh, - &num_groups, - contg_groups); + ret = mca_common_ompio_cart_based_grouping( fh, + &num_groups, + contg_groups); if (OMPI_SUCCESS != ret ) { opal_output(1, "mca_common_ompio_set_view: mca_io_ompio_cart_based_grouping failed\n"); goto exit; @@ -233,9 +234,9 @@ int mca_common_ompio_set_view (mca_io_ompio_file_t *fh, } if ( !done ) { - ret = mca_io_ompio_simple_grouping(fh, - &num_groups, - contg_groups); + ret = mca_common_ompio_simple_grouping(fh, + &num_groups, + contg_groups); if ( OMPI_SUCCESS != ret ){ opal_output(1, "mca_common_ompio_set_view: mca_io_ompio_simple_grouping failed\n"); goto exit; @@ -259,9 +260,9 @@ int mca_common_ompio_set_view (mca_io_ompio_file_t *fh, } #endif - ret = mca_io_ompio_finalize_initial_grouping(fh, - num_groups, - contg_groups); + ret = mca_common_ompio_finalize_initial_grouping(fh, + num_groups, + contg_groups); if ( OMPI_SUCCESS != ret ) { opal_output(1, "mca_common_ompio_set_view: mca_io_ompio_finalize_initial_grouping failed\n"); goto exit; diff --git a/ompi/mca/common/ompio/common_ompio_file_write.c b/ompi/mca/common/ompio/common_ompio_file_write.c index 07361dfa67..d3cb9a66a4 100644 --- a/ompi/mca/common/ompio/common_ompio_file_write.c +++ b/ompi/mca/common/ompio/common_ompio_file_write.c @@ -10,7 +10,7 @@ * Copyright (c) 2004-2005 The Regents of the University of California. * All rights reserved. * Copyright (c) 2008-2018 University of Houston. All rights reserved. - * Copyright (c) 2015-2017 Research Organization for Information Science + * Copyright (c) 2015-2018 Research Organization for Information Science * and Technology (RIST). All rights reserved. * $COPYRIGHT$ * @@ -31,7 +31,6 @@ #include "common_ompio.h" #include "common_ompio_request.h" -#include "ompi/mca/io/ompio/io_ompio.h" #include "math.h" #include @@ -62,19 +61,19 @@ int mca_common_ompio_file_write (mca_io_ompio_file_t *fh, return ret; } - ompi_io_ompio_decode_datatype (fh, - datatype, - count, - buf, - &max_data, - &decoded_iov, - &iov_count); + mca_common_ompio_decode_datatype (fh, + datatype, + count, + buf, + &max_data, + &decoded_iov, + &iov_count); - if ( -1 == mca_io_ompio_cycle_buffer_size ) { + if ( -1 == OMPIO_MCA_GET(fh, cycle_buffer_size) ) { bytes_per_cycle = max_data; } else { - bytes_per_cycle = mca_io_ompio_cycle_buffer_size; + bytes_per_cycle = OMPIO_MCA_GET(fh, cycle_buffer_size); } cycles = ceil((float)max_data/bytes_per_cycle); @@ -177,13 +176,13 @@ int mca_common_ompio_file_iwrite (mca_io_ompio_file_t *fh, int i = 0; /* index into the decoded iovec of the buffer */ int j = 0; /* index into the file vie iovec */ - ompi_io_ompio_decode_datatype (fh, - datatype, - count, - buf, - &max_data, - &decoded_iov, - &iov_count); + mca_common_ompio_decode_datatype (fh, + datatype, + count, + buf, + &max_data, + &decoded_iov, + &iov_count); j = fh->f_index_in_file_view; /* Non blocking operations have to occur in a single cycle */ diff --git a/ompi/mca/common/ompio/common_ompio_print_queue.c b/ompi/mca/common/ompio/common_ompio_print_queue.c index 30b3e549e9..7cb7ef3e1d 100644 --- a/ompi/mca/common/ompio/common_ompio_print_queue.c +++ b/ompi/mca/common/ompio/common_ompio_print_queue.c @@ -11,6 +11,8 @@ * Copyright (c) 2004-2005 The Regents of the University of California. * All rights reserved. * Copyright (c) 2008-2016 University of Houston. All rights reserved. + * Copyright (c) 2018 Research Organization for Information Science + * and Technology (RIST). All rights reserved. * * $COPYRIGHT$ * @@ -25,7 +27,6 @@ #include "ompi/datatype/ompi_datatype.h" #include "ompi/mca/common/ompio/common_ompio.h" -#include "ompi/mca/io/ompio/io_ompio.h" /* Print queue related function implementations */ diff --git a/ompi/mca/common/ompio/common_ompio_print_queue.h b/ompi/mca/common/ompio/common_ompio_print_queue.h index ba74e30a55..dbdcdcc659 100644 --- a/ompi/mca/common/ompio/common_ompio_print_queue.h +++ b/ompi/mca/common/ompio/common_ompio_print_queue.h @@ -11,7 +11,7 @@ * Copyright (c) 2004-2005 The Regents of the University of California. * All rights reserved. * Copyright (c) 2008-2016 University of Houston. All rights reserved. - * Copyright (c) 2015 Research Organization for Information Science + * Copyright (c) 2015-2018 Research Organization for Information Science * and Technology (RIST). All rights reserved. * $COPYRIGHT$ * @@ -26,7 +26,6 @@ #include "mpi.h" -OMPI_DECLSPEC extern int mca_io_ompio_coll_timing_info; struct mca_io_ompio_file_t; #define MCA_COMMON_OMPIO_QUEUESIZE 2048 diff --git a/ompi/mca/common/ompio/common_ompio_request.h b/ompi/mca/common/ompio/common_ompio_request.h index 2a66afab0d..50508e99e1 100644 --- a/ompi/mca/common/ompio/common_ompio_request.h +++ b/ompi/mca/common/ompio/common_ompio_request.h @@ -11,6 +11,8 @@ * Copyright (c) 2004-2005 The Regents of the University of California. * All rights reserved. * Copyright (c) 2008-2018 University of Houston. All rights reserved. + * Copyright (c) 2018 Research Organization for Information Science + * and Technology (RIST). All rights reserved. * $COPYRIGHT$ * * Additional copyrights may follow @@ -24,7 +26,7 @@ #include "ompi_config.h" #include "ompi/request/request.h" #include "ompi/mca/fbtl/fbtl.h" -#include "ompi/mca/io/ompio/io_ompio.h" +#include "common_ompio.h" BEGIN_C_DECLS diff --git a/ompi/mca/fcoll/dynamic/Makefile.am b/ompi/mca/fcoll/dynamic/Makefile.am index 6b77394ec6..603fa55df7 100644 --- a/ompi/mca/fcoll/dynamic/Makefile.am +++ b/ompi/mca/fcoll/dynamic/Makefile.am @@ -12,6 +12,8 @@ # Copyright (c) 2008-2015 University of Houston. All rights reserved. # Copyright (c) 2012 Cisco Systems, Inc. All rights reserved. # Copyright (c) 2017 IBM Corporation. All rights reserved. +# Copyright (c) 2018 Research Organization for Information Science +# and Technology (RIST). All rights reserved. # $COPYRIGHT$ # # Additional copyrights may follow @@ -42,7 +44,7 @@ mcacomponentdir = $(ompilibdir) mcacomponent_LTLIBRARIES = $(component_install) mca_fcoll_dynamic_la_SOURCES = $(sources) mca_fcoll_dynamic_la_LDFLAGS = -module -avoid-version -mca_fcoll_dynamic_la_LIBADD = $(top_builddir)/ompi/lib@OMPI_LIBMPI_NAME@.la +mca_fcoll_dynamic_la_LIBADD = $(OMPI_TOP_BUILDDIR)/ompi/mca/common/ompio/libmca_common_ompio.la noinst_LTLIBRARIES = $(component_noinst) libmca_fcoll_dynamic_la_SOURCES =$(sources) diff --git a/ompi/mca/fcoll/dynamic/fcoll_dynamic_file_read_all.c b/ompi/mca/fcoll/dynamic/fcoll_dynamic_file_read_all.c index 249100054b..b20dccaa0f 100644 --- a/ompi/mca/fcoll/dynamic/fcoll_dynamic_file_read_all.c +++ b/ompi/mca/fcoll/dynamic/fcoll_dynamic_file_read_all.c @@ -10,7 +10,7 @@ * Copyright (c) 2004-2005 The Regents of the University of California. * All rights reserved. * Copyright (c) 2008-2015 University of Houston. All rights reserved. - * Copyright (c) 2017 Research Organization for Information Science + * Copyright (c) 2017-2018 Research Organization for Information Science * and Technology (RIST). All rights reserved. * Copyright (c) 2017 IBM Corporation. All rights reserved. * $COPYRIGHT$ @@ -27,7 +27,7 @@ #include "ompi/constants.h" #include "ompi/mca/fcoll/fcoll.h" #include "ompi/mca/fcoll/base/fcoll_base_coll_array.h" -#include "ompi/mca/io/ompio/io_ompio.h" +#include "ompi/mca/common/ompio/common_ompio.h" #include "ompi/mca/io/io.h" #include "math.h" #include "ompi/mca/pml/pml.h" @@ -125,13 +125,13 @@ mca_fcoll_dynamic_file_read_all (mca_io_ompio_file_t *fh, if (! recvbuf_is_contiguous ) { - ret = fh->f_decode_datatype ((struct mca_io_ompio_file_t *)fh, - datatype, - count, - buf, - &max_data, - &decoded_iov, - &iov_count); + ret = mca_common_ompio_decode_datatype ((struct mca_io_ompio_file_t *)fh, + datatype, + count, + buf, + &max_data, + &decoded_iov, + &iov_count); if (OMPI_SUCCESS != ret){ goto exit; } @@ -149,9 +149,9 @@ mca_fcoll_dynamic_file_read_all (mca_io_ompio_file_t *fh, ret = OMPI_ERROR; goto exit; } - ret = fh->f_set_aggregator_props ((struct mca_io_ompio_file_t *) fh, - dynamic_num_io_procs, - max_data); + ret = mca_common_ompio_set_aggregator_props ((struct mca_io_ompio_file_t *) fh, + dynamic_num_io_procs, + max_data); if (OMPI_SUCCESS != ret){ goto exit; } diff --git a/ompi/mca/fcoll/dynamic/fcoll_dynamic_file_write_all.c b/ompi/mca/fcoll/dynamic/fcoll_dynamic_file_write_all.c index 8d43188991..f16792f793 100644 --- a/ompi/mca/fcoll/dynamic/fcoll_dynamic_file_write_all.c +++ b/ompi/mca/fcoll/dynamic/fcoll_dynamic_file_write_all.c @@ -10,7 +10,7 @@ * Copyright (c) 2004-2005 The Regents of the University of California. * All rights reserved. * Copyright (c) 2008-2015 University of Houston. All rights reserved. - * Copyright (c) 2015-2017 Research Organization for Information Science + * Copyright (c) 2015-2018 Research Organization for Information Science * and Technology (RIST). All rights reserved. * Copyright (c) 2017 IBM Corporation. All rights reserved. * $COPYRIGHT$ @@ -27,7 +27,7 @@ #include "ompi/constants.h" #include "ompi/mca/fcoll/fcoll.h" #include "ompi/mca/fcoll/base/fcoll_base_coll_array.h" -#include "ompi/mca/io/ompio/io_ompio.h" +#include "ompi/mca/common/ompio/common_ompio.h" #include "ompi/mca/io/io.h" #include "math.h" #include "ompi/mca/pml/pml.h" @@ -127,13 +127,13 @@ mca_fcoll_dynamic_file_write_all (mca_io_ompio_file_t *fh, if (! sendbuf_is_contiguous ) { - ret = fh->f_decode_datatype ((struct mca_io_ompio_file_t *) fh, - datatype, - count, - buf, - &max_data, - &decoded_iov, - &iov_count); + ret = mca_common_ompio_decode_datatype ((struct mca_io_ompio_file_t *) fh, + datatype, + count, + buf, + &max_data, + &decoded_iov, + &iov_count); if (OMPI_SUCCESS != ret ){ goto exit; } @@ -151,9 +151,9 @@ mca_fcoll_dynamic_file_write_all (mca_io_ompio_file_t *fh, ret = OMPI_ERROR; goto exit; } - ret = fh->f_set_aggregator_props ((struct mca_io_ompio_file_t *) fh, - dynamic_num_io_procs, - max_data); + ret = mca_common_ompio_set_aggregator_props ((struct mca_io_ompio_file_t *) fh, + dynamic_num_io_procs, + max_data); if (OMPI_SUCCESS != ret){ goto exit; diff --git a/ompi/mca/fcoll/dynamic_gen2/Makefile.am b/ompi/mca/fcoll/dynamic_gen2/Makefile.am index 052e34fc50..a4e91e5cd7 100644 --- a/ompi/mca/fcoll/dynamic_gen2/Makefile.am +++ b/ompi/mca/fcoll/dynamic_gen2/Makefile.am @@ -10,8 +10,10 @@ # Copyright (c) 2004-2005 The Regents of the University of California. # All rights reserved. # Copyright (c) 2008-2015 University of Houston. All rights reserved. -# Copyright (c) 2012 Cisco Systems, Inc. All rights reserved. +# Copyright (c) 2012 Cisco Systems, Inc. All rights reserved. # Copyright (c) 2017 IBM Corporation. All rights reserved. +# Copyright (c) 2018 Research Organization for Information Science +# and Technology (RIST). All rights reserved. # $COPYRIGHT$ # # Additional copyrights may follow @@ -42,7 +44,7 @@ mcacomponentdir = $(ompilibdir) mcacomponent_LTLIBRARIES = $(component_install) mca_fcoll_dynamic_gen2_la_SOURCES = $(sources) mca_fcoll_dynamic_gen2_la_LDFLAGS = -module -avoid-version -mca_fcoll_dynamic_gen2_la_LIBADD = $(top_builddir)/ompi/lib@OMPI_LIBMPI_NAME@.la +mca_fcoll_dynamic_gen2_la_LIBADD = $(OMPI_TOP_BUILDDIR)/ompi/mca/common/ompio/libmca_common_ompio.la noinst_LTLIBRARIES = $(component_noinst) libmca_fcoll_dynamic_gen2_la_SOURCES =$(sources) diff --git a/ompi/mca/fcoll/dynamic_gen2/fcoll_dynamic_gen2_file_read_all.c b/ompi/mca/fcoll/dynamic_gen2/fcoll_dynamic_gen2_file_read_all.c index 95465cc401..d3ed82050c 100644 --- a/ompi/mca/fcoll/dynamic_gen2/fcoll_dynamic_gen2_file_read_all.c +++ b/ompi/mca/fcoll/dynamic_gen2/fcoll_dynamic_gen2_file_read_all.c @@ -10,7 +10,7 @@ * Copyright (c) 2004-2005 The Regents of the University of California. * All rights reserved. * Copyright (c) 2008-2015 University of Houston. All rights reserved. - * Copyright (c) 2017 Research Organization for Information Science + * Copyright (c) 2017-2018 Research Organization for Information Science * and Technology (RIST). All rights reserved. * Copyright (c) 2017 IBM Corporation. All rights reserved. * $COPYRIGHT$ @@ -27,7 +27,7 @@ #include "ompi/constants.h" #include "ompi/mca/fcoll/fcoll.h" #include "ompi/mca/fcoll/base/fcoll_base_coll_array.h" -#include "ompi/mca/io/ompio/io_ompio.h" +#include "ompi/mca/common/ompio/common_ompio.h" #include "ompi/mca/io/io.h" #include "math.h" #include "ompi/mca/pml/pml.h" @@ -125,13 +125,13 @@ mca_fcoll_dynamic_gen2_file_read_all (mca_io_ompio_file_t *fh, if (! recvbuf_is_contiguous ) { - ret = fh->f_decode_datatype ((struct mca_io_ompio_file_t *)fh, - datatype, - count, - buf, - &max_data, - &decoded_iov, - &iov_count); + ret = mca_common_ompio_decode_datatype ((struct mca_io_ompio_file_t *)fh, + datatype, + count, + buf, + &max_data, + &decoded_iov, + &iov_count); if (OMPI_SUCCESS != ret){ goto exit; } @@ -149,9 +149,9 @@ mca_fcoll_dynamic_gen2_file_read_all (mca_io_ompio_file_t *fh, ret = OMPI_ERROR; goto exit; } - ret = fh->f_set_aggregator_props ((struct mca_io_ompio_file_t *) fh, - dynamic_gen2_num_io_procs, - max_data); + ret = mca_common_ompio_set_aggregator_props ((struct mca_io_ompio_file_t *) fh, + dynamic_gen2_num_io_procs, + max_data); if (OMPI_SUCCESS != ret){ goto exit; } diff --git a/ompi/mca/fcoll/dynamic_gen2/fcoll_dynamic_gen2_file_write_all.c b/ompi/mca/fcoll/dynamic_gen2/fcoll_dynamic_gen2_file_write_all.c index 91b242f967..fe3f8c8f84 100644 --- a/ompi/mca/fcoll/dynamic_gen2/fcoll_dynamic_gen2_file_write_all.c +++ b/ompi/mca/fcoll/dynamic_gen2/fcoll_dynamic_gen2_file_write_all.c @@ -10,7 +10,7 @@ * Copyright (c) 2004-2005 The Regents of the University of California. * All rights reserved. * Copyright (c) 2008-2016 University of Houston. All rights reserved. - * Copyright (c) 2015-2017 Research Organization for Information Science + * Copyright (c) 2015-2018 Research Organization for Information Science * and Technology (RIST). All rights reserved. * Copyright (c) 2017 IBM Corporation. All rights reserved. * Copyright (c) 2018 Cisco Systems, Inc. All rights reserved @@ -28,7 +28,7 @@ #include "ompi/constants.h" #include "ompi/mca/fcoll/fcoll.h" #include "ompi/mca/fcoll/base/fcoll_base_coll_array.h" -#include "ompi/mca/io/ompio/io_ompio.h" +#include "ompi/mca/common/ompio/common_ompio.h" #include "ompi/mca/io/io.h" #include "math.h" #include "ompi/mca/pml/pml.h" @@ -168,13 +168,13 @@ int mca_fcoll_dynamic_gen2_file_write_all (mca_io_ompio_file_t *fh, the user requested */ bytes_per_cycle =bytes_per_cycle/2; - ret = fh->f_decode_datatype ((struct mca_io_ompio_file_t *) fh, - datatype, - count, - buf, - &max_data, - &decoded_iov, - &iov_count); + ret = mca_common_ompio_decode_datatype ((struct mca_io_ompio_file_t *) fh, + datatype, + count, + buf, + &max_data, + &decoded_iov, + &iov_count); if (OMPI_SUCCESS != ret ){ goto exit; } diff --git a/ompi/mca/fcoll/individual/Makefile.am b/ompi/mca/fcoll/individual/Makefile.am index 21438356bc..437bd498cf 100644 --- a/ompi/mca/fcoll/individual/Makefile.am +++ b/ompi/mca/fcoll/individual/Makefile.am @@ -10,8 +10,10 @@ # Copyright (c) 2004-2005 The Regents of the University of California. # All rights reserved. # Copyright (c) 2008-2015 University of Houston. All rights reserved. -# Copyright (c) 2012 Cisco Systems, Inc. All rights reserved. +# Copyright (c) 2012 Cisco Systems, Inc. All rights reserved. # Copyright (c) 2017 IBM Corporation. All rights reserved. +# Copyright (c) 2018 Research Organization for Information Science +# and Technology (RIST). All rights reserved. # $COPYRIGHT$ # # Additional copyrights may follow @@ -42,7 +44,7 @@ mcacomponentdir = $(ompilibdir) mcacomponent_LTLIBRARIES = $(component_install) mca_fcoll_individual_la_SOURCES = $(sources) mca_fcoll_individual_la_LDFLAGS = -module -avoid-version -mca_fcoll_individual_la_LIBADD = $(top_builddir)/ompi/lib@OMPI_LIBMPI_NAME@.la +mca_fcoll_individual_la_LIBADD = $(OMPI_TOP_BUILDDIR)/ompi/mca/common/ompio/libmca_common_ompio.la noinst_LTLIBRARIES = $(component_noinst) libmca_fcoll_individual_la_SOURCES =$(sources) diff --git a/ompi/mca/fcoll/two_phase/Makefile.am b/ompi/mca/fcoll/two_phase/Makefile.am index 154d9a32e9..b62fdec8c2 100644 --- a/ompi/mca/fcoll/two_phase/Makefile.am +++ b/ompi/mca/fcoll/two_phase/Makefile.am @@ -10,8 +10,10 @@ # Copyright (c) 2004-2005 The Regents of the University of California. # All rights reserved. # Copyright (c) 2008-2015 University of Houston. All rights reserved. -# Copyright (c) 2012 Cisco Systems, Inc. All rights reserved. +# Copyright (c) 2012 Cisco Systems, Inc. All rights reserved. # Copyright (c) 2017 IBM Corporation. All rights reserved. +# Copyright (c) 2018 Research Organization for Information Science +# and Technology (RIST). All rights reserved. # $COPYRIGHT$ # # Additional copyrights may follow @@ -43,7 +45,7 @@ mcacomponentdir = $(ompilibdir) mcacomponent_LTLIBRARIES = $(component_install) mca_fcoll_two_phase_la_SOURCES = $(sources) mca_fcoll_two_phase_la_LDFLAGS = -module -avoid-version -mca_fcoll_two_phase_la_LIBADD = $(top_builddir)/ompi/lib@OMPI_LIBMPI_NAME@.la +mca_fcoll_two_phase_la_LIBADD = $(OMPI_TOP_BUILDDIR)/ompi/mca/common/ompio/libmca_common_ompio.la noinst_LTLIBRARIES = $(component_noinst) libmca_fcoll_two_phase_la_SOURCES =$(sources) diff --git a/ompi/mca/fcoll/two_phase/fcoll_two_phase_file_read_all.c b/ompi/mca/fcoll/two_phase/fcoll_two_phase_file_read_all.c index 3bc932cff0..fe536c7af6 100644 --- a/ompi/mca/fcoll/two_phase/fcoll_two_phase_file_read_all.c +++ b/ompi/mca/fcoll/two_phase/fcoll_two_phase_file_read_all.c @@ -14,7 +14,7 @@ * Copyright (c) 2015 Cisco Systems, Inc. All rights reserved. * Copyright (c) 2015 Los Alamos National Security, LLC. All rights * reserved. - * Copyright (c) 2017 Research Organization for Information Science + * Copyright (c) 2017-2018 Research Organization for Information Science * and Technology (RIST). All rights reserved. * $COPYRIGHT$ * @@ -29,7 +29,7 @@ #include "ompi/constants.h" #include "ompi/communicator/communicator.h" #include "ompi/mca/fcoll/fcoll.h" -#include "ompi/mca/io/ompio/io_ompio.h" +#include "ompi/mca/common/ompio/common_ompio.h" #include "ompi/mca/io/io.h" #include "opal/mca/base/base.h" #include "math.h" @@ -150,13 +150,13 @@ mca_fcoll_two_phase_file_read_all (mca_io_ompio_file_t *fh, // } if (! (fh->f_flags & OMPIO_CONTIGUOUS_MEMORY)) { - ret = fh->f_decode_datatype ((struct mca_io_ompio_file_t *)fh, - datatype, - count, - buf, - &max_data, - &temp_iov, - &iov_count); + ret = mca_common_ompio_decode_datatype ((struct mca_io_ompio_file_t *)fh, + datatype, + count, + buf, + &max_data, + &temp_iov, + &iov_count); if (OMPI_SUCCESS != ret ){ goto exit; } @@ -192,9 +192,9 @@ mca_fcoll_two_phase_file_read_all (mca_io_ompio_file_t *fh, goto exit; } if (-1 == two_phase_num_io_procs ){ - ret = fh->f_set_aggregator_props ((struct mca_io_ompio_file_t *)fh, - two_phase_num_io_procs, - max_data); + ret = mca_common_ompio_set_aggregator_props ((struct mca_io_ompio_file_t *)fh, + two_phase_num_io_procs, + max_data); if (OMPI_SUCCESS != ret){ goto exit; } diff --git a/ompi/mca/fcoll/two_phase/fcoll_two_phase_file_write_all.c b/ompi/mca/fcoll/two_phase/fcoll_two_phase_file_write_all.c index fc598e31f0..5e8efb9557 100644 --- a/ompi/mca/fcoll/two_phase/fcoll_two_phase_file_write_all.c +++ b/ompi/mca/fcoll/two_phase/fcoll_two_phase_file_write_all.c @@ -11,7 +11,7 @@ * Copyright (c) 2004-2005 The Regents of the University of California. * All rights reserved. * Copyright (c) 2008-2014 University of Houston. All rights reserved. - * Copyright (c) 2015-2017 Research Organization for Information Science + * Copyright (c) 2015-2018 Research Organization for Information Science * and Technology (RIST). All rights reserved. * Copyright (c) 2015-2016 Los Alamos National Security, LLC. All rights * reserved. @@ -29,7 +29,7 @@ #include "ompi/constants.h" #include "ompi/communicator/communicator.h" #include "ompi/mca/fcoll/fcoll.h" -#include "ompi/mca/io/ompio/io_ompio.h" +#include "ompi/mca/common/ompio/common_ompio.h" #include "ompi/mca/io/io.h" #include "opal/mca/base/base.h" #include "math.h" @@ -180,13 +180,13 @@ mca_fcoll_two_phase_file_write_all (mca_io_ompio_file_t *fh, if (! (fh->f_flags & OMPIO_CONTIGUOUS_MEMORY)) { - ret = fh->f_decode_datatype ((struct mca_io_ompio_file_t *)fh, - datatype, - count, - buf, - &max_data, - &temp_iov, - &iov_count); + ret = mca_common_ompio_decode_datatype ((struct mca_io_ompio_file_t *)fh, + datatype, + count, + buf, + &max_data, + &temp_iov, + &iov_count); if (OMPI_SUCCESS != ret ){ goto exit; } @@ -228,9 +228,9 @@ mca_fcoll_two_phase_file_write_all (mca_io_ompio_file_t *fh, goto exit; } if(-1 == two_phase_num_io_procs){ - ret = fh->f_set_aggregator_props ((struct mca_io_ompio_file_t *)fh, - two_phase_num_io_procs, - max_data); + ret = mca_common_ompio_set_aggregator_props ((struct mca_io_ompio_file_t *)fh, + two_phase_num_io_procs, + max_data); if ( OMPI_SUCCESS != ret){ goto exit; } diff --git a/ompi/mca/fcoll/two_phase/fcoll_two_phase_support_fns.c b/ompi/mca/fcoll/two_phase/fcoll_two_phase_support_fns.c index 13c60c23b0..ef153806dc 100644 --- a/ompi/mca/fcoll/two_phase/fcoll_two_phase_support_fns.c +++ b/ompi/mca/fcoll/two_phase/fcoll_two_phase_support_fns.c @@ -11,7 +11,7 @@ * Copyright (c) 2004-2005 The Regents of the University of California. * All rights reserved. * Copyright (c) 2008-2011 University of Houston. All rights reserved. - * Copyright (c) 2014 Research Organization for Information Science + * Copyright (c) 2014-2018 Research Organization for Information Science * and Technology (RIST). All rights reserved. * Copyright (c) 2015 Los Alamos National Security, LLC. All rights * reserved. @@ -28,7 +28,7 @@ #include "mpi.h" #include "ompi/constants.h" #include "ompi/mca/fcoll/fcoll.h" -#include "ompi/mca/io/ompio/io_ompio.h" +#include "ompi/mca/common/ompio/common_ompio.h" #include "ompi/mca/io/io.h" #include "opal/mca/base/base.h" #include "math.h" diff --git a/ompi/mca/fcoll/vulcan/Makefile.am b/ompi/mca/fcoll/vulcan/Makefile.am index 0684396a18..63f1ad9da2 100644 --- a/ompi/mca/fcoll/vulcan/Makefile.am +++ b/ompi/mca/fcoll/vulcan/Makefile.am @@ -11,6 +11,8 @@ # All rights reserved. # Copyright (c) 2008-2018 University of Houston. All rights reserved. # Copyright (c) 2012 Cisco Systems, Inc. All rights reserved. +# Copyright (c) 2018 Research Organization for Information Science +# and Technology (RIST). All rights reserved. # $COPYRIGHT$ # # Additional copyrights may follow @@ -41,6 +43,7 @@ mcacomponentdir = $(ompilibdir) mcacomponent_LTLIBRARIES = $(component_install) mca_fcoll_vulcan_la_SOURCES = $(sources) mca_fcoll_vulcan_la_LDFLAGS = -module -avoid-version +mca_fcoll_vulcan_la_LIBADD = $(OMPI_TOP_BUILDDIR)/ompi/mca/common/ompio/libmca_common_ompio.la noinst_LTLIBRARIES = $(component_noinst) libmca_fcoll_vulcan_la_SOURCES =$(sources) diff --git a/ompi/mca/fcoll/vulcan/fcoll_vulcan_file_read_all.c b/ompi/mca/fcoll/vulcan/fcoll_vulcan_file_read_all.c index 77934408c8..0bee1baaee 100644 --- a/ompi/mca/fcoll/vulcan/fcoll_vulcan_file_read_all.c +++ b/ompi/mca/fcoll/vulcan/fcoll_vulcan_file_read_all.c @@ -10,7 +10,7 @@ * Copyright (c) 2004-2005 The Regents of the University of California. * All rights reserved. * Copyright (c) 2008-2017 University of Houston. All rights reserved. - * Copyright (c) 2017 Research Organization for Information Science + * Copyright (c) 2017-2018 Research Organization for Information Science * and Technology (RIST). All rights reserved. * $COPYRIGHT$ * @@ -26,7 +26,7 @@ #include "ompi/constants.h" #include "ompi/mca/fcoll/fcoll.h" #include "ompi/mca/fcoll/base/fcoll_base_coll_array.h" -#include "ompi/mca/io/ompio/io_ompio.h" +#include "ompi/mca/common/ompio/common_ompio.h" #include "ompi/mca/io/io.h" #include "math.h" #include "ompi/mca/pml/pml.h" @@ -124,13 +124,13 @@ mca_fcoll_vulcan_file_read_all (mca_io_ompio_file_t *fh, if (! recvbuf_is_contiguous ) { - ret = fh->f_decode_datatype ((struct mca_io_ompio_file_t *)fh, - datatype, - count, - buf, - &max_data, - &decoded_iov, - &iov_count); + ret = mca_common_ompio_decode_datatype ((struct mca_io_ompio_file_t *)fh, + datatype, + count, + buf, + &max_data, + &decoded_iov, + &iov_count); if (OMPI_SUCCESS != ret){ goto exit; } @@ -149,9 +149,9 @@ mca_fcoll_vulcan_file_read_all (mca_io_ompio_file_t *fh, goto exit; } - ret = fh->f_set_aggregator_props ((struct mca_io_ompio_file_t *) fh, - vulcan_num_io_procs, - max_data); + ret = mca_common_ompio_set_aggregator_props ((struct mca_io_ompio_file_t *) fh, + vulcan_num_io_procs, + max_data); if (OMPI_SUCCESS != ret){ goto exit; } diff --git a/ompi/mca/fcoll/vulcan/fcoll_vulcan_file_write_all.c b/ompi/mca/fcoll/vulcan/fcoll_vulcan_file_write_all.c index 5730116311..d75d92e728 100644 --- a/ompi/mca/fcoll/vulcan/fcoll_vulcan_file_write_all.c +++ b/ompi/mca/fcoll/vulcan/fcoll_vulcan_file_write_all.c @@ -10,7 +10,7 @@ * Copyright (c) 2004-2005 The Regents of the University of California. * All rights reserved. * Copyright (c) 2008-2018 University of Houston. All rights reserved. - * Copyright (c) 2015-2017 Research Organization for Information Science + * Copyright (c) 2015-2018 Research Organization for Information Science * and Technology (RIST). All rights reserved. * $COPYRIGHT$ * @@ -26,7 +26,7 @@ #include "ompi/constants.h" #include "ompi/mca/fcoll/fcoll.h" #include "ompi/mca/fcoll/base/fcoll_base_coll_array.h" -#include "ompi/mca/io/ompio/io_ompio.h" +#include "ompi/mca/common/ompio/common_ompio.h" #include "ompi/mca/io/io.h" #include "ompi/mca/common/ompio/common_ompio_request.h" #include "math.h" @@ -184,13 +184,13 @@ int mca_fcoll_vulcan_file_write_all (mca_io_ompio_file_t *fh, bytes_per_cycle =bytes_per_cycle/2; write_chunksize = bytes_per_cycle; - ret = fh->f_decode_datatype ((struct mca_io_ompio_file_t *) fh, - datatype, - count, - buf, - &max_data, - &decoded_iov, - &iov_count); + ret = mca_common_ompio_decode_datatype ((struct mca_io_ompio_file_t *) fh, + datatype, + count, + buf, + &max_data, + &decoded_iov, + &iov_count); if (OMPI_SUCCESS != ret ){ goto exit; } @@ -1620,7 +1620,7 @@ int mca_fcoll_vulcan_get_configuration (mca_io_ompio_file_t *fh, int num_io_proc size_t max_data) { int i, ret; - ret = fh->f_set_aggregator_props (fh, num_io_procs, max_data); + ret = mca_common_ompio_set_aggregator_props (fh, num_io_procs, max_data); /* Note: as of this version of the vulcan component, we are not using yet the num_groups parameter to split the aggregators (and processes) into diff --git a/ompi/mca/io/ompio/Makefile.am b/ompi/mca/io/ompio/Makefile.am index 8e742a3bb6..9cf3950cf6 100644 --- a/ompi/mca/io/ompio/Makefile.am +++ b/ompi/mca/io/ompio/Makefile.am @@ -11,6 +11,8 @@ # All rights reserved. # Copyright (c) 2008-2012 University of Houston. All rights reserved. # Copyright (c) 2016-2017 IBM Corporation. All rights reserved. +# Copyright (c) 2018 Research Organization for Information Science +# and Technology (RIST). All rights reserved. # $COPYRIGHT$ # # Additional copyrights may follow @@ -34,8 +36,7 @@ mcacomponentdir = $(ompilibdir) mcacomponent_LTLIBRARIES = $(component_install) mca_io_ompio_la_SOURCES = $(headers) $(sources) mca_io_ompio_la_LDFLAGS = -module -avoid-version -mca_io_ompio_la_LIBADD = $(top_builddir)/ompi/lib@OMPI_LIBMPI_NAME@.la \ - $(io_ompio_LIBS) \ +mca_io_ompio_la_LIBADD = $(io_ompio_LIBS) \ $(OMPI_TOP_BUILDDIR)/ompi/mca/common/ompio/libmca_common_ompio.la noinst_LTLIBRARIES = $(component_noinst) @@ -45,12 +46,10 @@ libmca_io_ompio_la_LDFLAGS = -module -avoid-version # Source files headers = \ - io_ompio.h \ - io_ompio_aggregators.h + io_ompio.h sources = \ io_ompio.c \ - io_ompio_aggregators.c \ io_ompio_component.c \ io_ompio_module.c \ io_ompio_file_set_view.c \ diff --git a/ompi/mca/io/ompio/io_ompio.c b/ompi/mca/io/ompio/io_ompio.c index 05188ad764..0d15c742d8 100644 --- a/ompi/mca/io/ompio/io_ompio.c +++ b/ompi/mca/io/ompio/io_ompio.c @@ -13,7 +13,7 @@ * Copyright (c) 2008-2016 University of Houston. All rights reserved. * Copyright (c) 2011-2015 Cisco Systems, Inc. All rights reserved. * Copyright (c) 2012-2013 Inria. All rights reserved. - * Copyright (c) 2015-2017 Research Organization for Information Science + * Copyright (c) 2015-2018 Research Organization for Information Science * and Technology (RIST). All rights reserved. * $COPYRIGHT$ * @@ -415,127 +415,6 @@ int ompi_io_ompio_generate_current_file_view (struct mca_io_ompio_file_t *fh, } -int ompi_io_ompio_decode_datatype (struct mca_io_ompio_file_t *fh, - ompi_datatype_t *datatype, - int count, - const void *buf, - size_t *max_data, - struct iovec **iov, - uint32_t *iovec_count) -{ - - - - opal_convertor_t convertor; - size_t remaining_length = 0; - uint32_t i; - uint32_t temp_count; - struct iovec *temp_iov=NULL; - size_t temp_data; - - - opal_convertor_clone (fh->f_convertor, &convertor, 0); - - if (OMPI_SUCCESS != opal_convertor_prepare_for_send (&convertor, - &(datatype->super), - count, - buf)) { - opal_output (1, "Cannot attach the datatype to a convertor\n"); - return OMPI_ERROR; - } - - if ( 0 == datatype->super.size ) { - *max_data = 0; - *iovec_count = 0; - *iov = NULL; - return OMPI_SUCCESS; - } - - remaining_length = count * datatype->super.size; - - temp_count = OMPIO_IOVEC_INITIAL_SIZE; - temp_iov = (struct iovec*)malloc(temp_count * sizeof(struct iovec)); - if (NULL == temp_iov) { - opal_output (1, "OUT OF MEMORY\n"); - return OMPI_ERR_OUT_OF_RESOURCE; - } - - while (0 == opal_convertor_raw(&convertor, - temp_iov, - &temp_count, - &temp_data)) { -#if 0 - printf ("%d: New raw extraction (iovec_count = %d, max_data = %lu)\n", - fh->f_rank,temp_count, (unsigned long)temp_data); - for (i = 0; i < temp_count; i++) { - printf ("%d: \t{%p, %lu}\n",fh->f_rank, - temp_iov[i].iov_base, - (unsigned long)temp_iov[i].iov_len); - } -#endif - - *iovec_count = *iovec_count + temp_count; - *max_data = *max_data + temp_data; - *iov = (struct iovec *) realloc (*iov, *iovec_count * sizeof(struct iovec)); - if (NULL == *iov) { - opal_output(1, "OUT OF MEMORY\n"); - free(temp_iov); - return OMPI_ERR_OUT_OF_RESOURCE; - } - for (i=0 ; if_rank,temp_count, temp_data); - for (i = 0; i < temp_count; i++) { - printf ("%d: \t offset[%d]: %ld; length[%d]: %ld\n", fh->f_rank,i,temp_iov[i].iov_base, i,temp_iov[i].iov_len); - } -#endif - *iovec_count = *iovec_count + temp_count; - *max_data = *max_data + temp_data; - if ( temp_count > 0 ) { - *iov = (struct iovec *) realloc (*iov, *iovec_count * sizeof(struct iovec)); - if (NULL == *iov) { - opal_output(1, "OUT OF MEMORY\n"); - free(temp_iov); - return OMPI_ERR_OUT_OF_RESOURCE; - } - } - for (i=0 ; if_rank) { - printf ("%d Entries: \n",*iovec_count); - for (i=0 ; i<*iovec_count ; i++) { - printf ("\t{%p, %d}\n", - (*iov)[i].iov_base, - (*iov)[i].iov_len); - } - } -#endif - if (remaining_length != 0) { - printf( "Not all raw description was been extracted (%lu bytes missing)\n", - (unsigned long) remaining_length ); - } - - free (temp_iov); - - return OMPI_SUCCESS; -} - - - int ompi_io_ompio_sort_offlen (mca_io_ompio_offlen_array_t *io_array, int num_entries, int *sorted){ @@ -660,6 +539,15 @@ int mca_io_ompio_get_mca_parameter_value ( char *mca_parameter_name, int name_le else if ( !strncmp ( mca_parameter_name, "aggregators_cutoff_threshold", name_length )) { return mca_io_ompio_aggregators_cutoff_threshold; } + else if ( !strncmp ( mca_parameter_name, "grouping_option", name_length )) { + return mca_io_ompio_grouping_option; + } + else if ( !strncmp ( mca_parameter_name, "sharedfp_lazy_open", name_length )) { + return mca_io_ompio_sharedfp_lazy_open; + } + else if ( !strncmp ( mca_parameter_name, "coll_timing_info", name_length )) { + return mca_io_ompio_coll_timing_info; + } else { opal_output (1, "Error in mca_io_ompio_get_mca_parameter_value: unknown parameter name"); } diff --git a/ompi/mca/io/ompio/io_ompio.h b/ompi/mca/io/ompio/io_ompio.h index 457346dc41..f4a4c0a020 100644 --- a/ompi/mca/io/ompio/io_ompio.h +++ b/ompi/mca/io/ompio/io_ompio.h @@ -11,7 +11,7 @@ * Copyright (c) 2004-2005 The Regents of the University of California. * All rights reserved. * Copyright (c) 2008-2018 University of Houston. All rights reserved. - * Copyright (c) 2015-2017 Research Organization for Information Science + * Copyright (c) 2015-2018 Research Organization for Information Science * and Technology (RIST). All rights reserved. * Copyright (c) 2016-2017 IBM Corporation. All rights reserved. * $COPYRIGHT$ @@ -42,6 +42,7 @@ #include "ompi/datatype/ompi_datatype.h" #include "ompi/request/request.h" +#include "ompi/mca/common/ompio/common_ompio.h" extern int mca_io_ompio_cycle_buffer_size; extern int mca_io_ompio_bytes_per_agg; @@ -55,36 +56,13 @@ extern int mca_io_ompio_overwrite_amode; OMPI_DECLSPEC extern int mca_io_ompio_coll_timing_info; -/* - * Flags - */ -#define OMPIO_CONTIGUOUS_MEMORY 0x00000001 -#define OMPIO_UNIFORM_FVIEW 0x00000002 -#define OMPIO_FILE_IS_OPEN 0x00000004 -#define OMPIO_FILE_VIEW_IS_SET 0x00000008 -#define OMPIO_CONTIGUOUS_FVIEW 0x00000010 -#define OMPIO_AGGREGATOR_IS_SET 0x00000020 -#define OMPIO_SHAREDFP_IS_SET 0x00000040 -#define OMPIO_LOCK_ENTIRE_FILE 0x00000080 -#define OMPIO_LOCK_NEVER 0x00000100 -#define OMPIO_LOCK_NOT_THIS_OP 0x00000200 - - #define QUEUESIZE 2048 -#define MCA_IO_DEFAULT_FILE_VIEW_SIZE 4*1024*1024 -#define OMPIO_FCOLL_WANT_TIME_BREAKDOWN 0 - -#define OMPIO_MIN(a, b) (((a) < (b)) ? (a) : (b)) -#define OMPIO_MAX(a, b) (((a) < (b)) ? (b) : (a)) /* * General values */ #define OMPIO_PREALLOC_MAX_BUF_SIZE 33554432 #define OMPIO_DEFAULT_CYCLE_BUF_SIZE 536870912 -#define OMPIO_PERM_NULL -1 -#define OMPIO_IOVEC_INITIAL_SIZE 100 -#define OMPIO_ROOT 0 #define OMPIO_MAX_NAME 100 #define OMPIO_TAG_GATHER -100 #define OMPIO_TAG_GATHERV -101 @@ -104,45 +82,8 @@ OMPI_DECLSPEC extern int mca_io_ompio_coll_timing_info; /*---------------------------*/ - -/*AGGREGATOR GROUPING DECISIONS*/ -#define OMPIO_MERGE 1 -#define OMPIO_SPLIT 2 -#define OMPIO_RETAIN 3 - -#define DATA_VOLUME 1 -#define UNIFORM_DISTRIBUTION 2 -#define CONTIGUITY 3 -#define OPTIMIZE_GROUPING 4 -#define SIMPLE 5 -#define NO_REFINEMENT 6 -#define SIMPLE_PLUS 7 - -#define OMPIO_UNIFORM_DIST_THRESHOLD 0.5 -#define OMPIO_CONTG_THRESHOLD 1048576 -#define OMPIO_CONTG_FACTOR 8 -#define OMPIO_DEFAULT_STRIPE_SIZE 1048576 -#define OMPIO_PROCS_PER_GROUP_TAG 0 -#define OMPIO_PROCS_IN_GROUP_TAG 1 -#define OMPIO_MERGE_THRESHOLD 0.5 - - -#define OMPIO_LOCK_ENTIRE_REGION 10 -#define OMPIO_LOCK_SELECTIVE 11 - -/*---------------------------*/ - BEGIN_C_DECLS -enum ompio_fs_type -{ - NONE = 0, - UFS = 1, - PVFS2 = 2, - LUSTRE = 3, - PLFS = 4 -}; - OMPI_DECLSPEC extern mca_io_base_component_2_0_0_t mca_io_ompio_component; /* * global variables, instantiated in module.c @@ -151,22 +92,6 @@ extern opal_mutex_t mca_io_ompio_mutex; extern mca_io_base_module_2_0_0_t mca_io_ompio_module; OMPI_DECLSPEC extern mca_io_base_component_2_0_0_t mca_io_ompio_component; -typedef struct mca_io_ompio_io_array_t { - void *memory_address; - /* we need that of type OMPI_MPI_OFFSET_TYPE */ - void *offset; - size_t length; - /*mca_io_ompio_server_t io_server;*/ -} mca_io_ompio_io_array_t; - - -typedef struct mca_io_ompio_access_array_t{ - OMPI_MPI_OFFSET_TYPE *offsets; - int *lens; - MPI_Aint *mem_ptrs; - int count; -} mca_io_ompio_access_array_t; - /*Used in extracting offset adj-matrix*/ typedef struct mca_io_ompio_offlen_array_t{ OMPI_MPI_OFFSET_TYPE offset; @@ -175,150 +100,13 @@ typedef struct mca_io_ompio_offlen_array_t{ }mca_io_ompio_offlen_array_t; -/* - * Function that takes in a datatype and buffer, and decodes that datatype - * into an iovec using the convertor_raw function - */ - -/* forward declaration to keep the compiler happy. */ -struct mca_io_ompio_file_t; -typedef int (*mca_io_ompio_decode_datatype_fn_t) (struct mca_io_ompio_file_t *fh, - struct ompi_datatype_t *datatype, - int count, - const void *buf, - size_t *max_data, - struct iovec **iov, - uint32_t *iov_count); -typedef int (*mca_io_ompio_generate_current_file_view_fn_t) (struct mca_io_ompio_file_t *fh, - size_t max_data, - struct iovec **f_iov, - int *iov_count); - -/* functions to retrieve the number of aggregators and the size of the - temporary buffer on aggregators from the fcoll modules */ -typedef int (*mca_io_ompio_get_mca_parameter_value_fn_t) ( char *mca_parameter_name, int name_length ); -typedef int (*mca_io_ompio_set_aggregator_props_fn_t) (struct mca_io_ompio_file_t *fh, - int num_aggregators, - size_t bytes_per_proc); - - -struct mca_common_ompio_print_queue; - -/** - * Back-end structure for MPI_File - */ -struct mca_io_ompio_file_t { - /* General parameters */ - int fd; - struct ompi_file_t *f_fh; /* pointer back to the file_t structure */ - OMPI_MPI_OFFSET_TYPE f_offset; /* byte offset of current position */ - OMPI_MPI_OFFSET_TYPE f_disp; /* file_view displacement */ - int f_rank; - int f_size; - int f_amode; - int f_perm; - ompi_communicator_t *f_comm; - const char *f_filename; - char *f_datarep; - opal_convertor_t *f_convertor; - opal_info_t *f_info; - int32_t f_flags; - void *f_fs_ptr; - int f_fs_block_size; - int f_atomicity; - size_t f_stripe_size; - int f_stripe_count; - size_t f_cc_size; - int f_bytes_per_agg; - enum ompio_fs_type f_fstype; - ompi_request_t *f_split_coll_req; - bool f_split_coll_in_use; - /* Place for selected sharedfp module to hang it's data. - Note: Neither f_sharedfp nor f_sharedfp_component seemed appropriate for this. - */ - void *f_sharedfp_data; - - - /* File View parameters */ - struct iovec *f_decoded_iov; - uint32_t f_iov_count; - ompi_datatype_t *f_iov_type; - size_t f_position_in_file_view; /* in bytes */ - size_t f_total_bytes; /* total bytes read/written within 1 Fview*/ - int f_index_in_file_view; - ptrdiff_t f_view_extent; - size_t f_view_size; - ompi_datatype_t *f_etype; - ompi_datatype_t *f_filetype; - ompi_datatype_t *f_orig_filetype; /* the fileview passed by the user to us */ - size_t f_etype_size; - - /* contains IO requests that needs to be read/written */ - mca_io_ompio_io_array_t *f_io_array; - int f_num_of_io_entries; - - /* Hooks for modules to hang things */ - mca_base_component_t *f_fs_component; - mca_base_component_t *f_fcoll_component; - mca_base_component_t *f_fbtl_component; - mca_base_component_t *f_sharedfp_component; - - /* structure of function pointers */ - mca_fs_base_module_t *f_fs; - mca_fcoll_base_module_t *f_fcoll; - mca_fbtl_base_module_t *f_fbtl; - mca_sharedfp_base_module_t *f_sharedfp; - - /* Timing information */ - struct mca_common_ompio_print_queue *f_coll_write_time; - struct mca_common_ompio_print_queue *f_coll_read_time; - - /*initial list of aggregators and groups*/ - int *f_init_aggr_list; - int f_init_num_aggrs; - int f_init_procs_per_group; - int *f_init_procs_in_group; - - /* final of aggregators and groups*/ - int *f_aggr_list; - int f_num_aggrs; - int *f_procs_in_group; - int f_procs_per_group; - - /* internal ompio functions required by fbtl and fcoll */ - mca_io_ompio_decode_datatype_fn_t f_decode_datatype; - mca_io_ompio_generate_current_file_view_fn_t f_generate_current_file_view; - - mca_io_ompio_get_mca_parameter_value_fn_t f_get_mca_parameter_value; - mca_io_ompio_set_aggregator_props_fn_t f_set_aggregator_props; -}; -typedef struct mca_io_ompio_file_t mca_io_ompio_file_t; - -struct mca_io_ompio_data_t { - mca_io_ompio_file_t ompio_fh; -}; -typedef struct mca_io_ompio_data_t mca_io_ompio_data_t; - - #include "ompi/mca/common/ompio/common_ompio.h" -#include "io_ompio_aggregators.h" +#include "ompi/mca/common/ompio/common_ompio_aggregators.h" /* functions to retrieve the number of aggregators and the size of the temporary buffer on aggregators from the fcoll modules */ OMPI_DECLSPEC int mca_io_ompio_get_mca_parameter_value ( char *mca_parameter_name, int name_length); -/* - * Function that takes in a datatype and buffer, and decodes that datatype - * into an iovec using the convertor_raw function - */ -OMPI_DECLSPEC int ompi_io_ompio_decode_datatype (struct mca_io_ompio_file_t *fh, - struct ompi_datatype_t *datatype, - int count, - const void *buf, - size_t *max_data, - struct iovec **iov, - uint32_t *iov_count); - /* * Function that sorts an io_array according to the offset by filling * up an array of the indices into the array (HEAP SORT) @@ -363,8 +151,6 @@ int mca_io_ompio_file_open (struct ompi_communicator_t *comm, struct opal_info_t *info, struct ompi_file_t *fh); int mca_io_ompio_file_close (struct ompi_file_t *fh); -int mca_io_ompio_file_delete (const char *filename, - struct opal_info_t *info); int mca_io_ompio_file_set_size (struct ompi_file_t *fh, OMPI_MPI_OFFSET_TYPE size); int mca_io_ompio_file_preallocate (struct ompi_file_t *fh, diff --git a/ompi/mca/io/ompio/io_ompio_aggregators.h b/ompi/mca/io/ompio/io_ompio_aggregators.h deleted file mode 100644 index 3815a16d66..0000000000 --- a/ompi/mca/io/ompio/io_ompio_aggregators.h +++ /dev/null @@ -1,109 +0,0 @@ -/* -*- Mode: C; c-basic-offset:4 ; -*- */ -/* - * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana - * University Research and Technology - * Corporation. All rights reserved. - * Copyright (c) 2004-2013 The University of Tennessee and The University - * of Tennessee Research Foundation. All rights - * reserved. - * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, - * University of Stuttgart. All rights reserved. - * Copyright (c) 2004-2005 The Regents of the University of California. - * All rights reserved. - * Copyright (c) 2008-2016 University of Houston. All rights reserved. - * - * $COPYRIGHT$ - * - * Additional copyrights may follow - * - * $HEADER$ - */ - - -#ifndef MCA_IO_OMPIO_AGGREGATORS_H -#define MCA_IO_OMPIO_AGGREGATORS_H - - -/*AGGREGATOR GROUPING DECISIONS*/ -#define OMPIO_MERGE 1 -#define OMPIO_SPLIT 2 -#define OMPIO_RETAIN 3 - -typedef struct { - int ndims; - int *dims; - int *periods; - int *coords; - int reorder; -} mca_io_ompio_cart_topo_components; - - -typedef struct{ - OMPI_MPI_OFFSET_TYPE contg_chunk_size; - int *procs_in_contg_group; - int procs_per_contg_group; -} mca_io_ompio_contg; - - - -/*Aggregator selection methods*/ -OMPI_DECLSPEC int mca_io_ompio_set_aggregator_props (struct mca_io_ompio_file_t *fh, - int num_aggregators, - size_t bytes_per_proc); - -int mca_io_ompio_forced_grouping ( mca_io_ompio_file_t *fh, - int num_groups, - mca_io_ompio_contg *contg_groups); - -int mca_io_ompio_cart_based_grouping(mca_io_ompio_file_t *ompio_fh, int *num_groups, - mca_io_ompio_contg *contg_groups); - -int mca_io_ompio_fview_based_grouping(mca_io_ompio_file_t *fh, int *num_groups, - mca_io_ompio_contg *contg_groups); -int mca_io_ompio_simple_grouping(mca_io_ompio_file_t *fh, int *num_groups, - mca_io_ompio_contg *contg_groups); - -int mca_io_ompio_finalize_initial_grouping(mca_io_ompio_file_t *fh, int num_groups, - mca_io_ompio_contg *contg_groups); - -int mca_io_ompio_create_groups(mca_io_ompio_file_t *fh, size_t bytes_per_proc); - -int mca_io_ompio_prepare_to_group(mca_io_ompio_file_t *fh, - OMPI_MPI_OFFSET_TYPE **start_offsets_lens, - OMPI_MPI_OFFSET_TYPE **end_offsets, - OMPI_MPI_OFFSET_TYPE **aggr_bytes_per_group, - OMPI_MPI_OFFSET_TYPE *bytes_per_group, - int **decision_list, - size_t bytes_per_proc, - int *is_aggregator, - int *ompio_grouping_flag); - -int mca_io_ompio_retain_initial_groups(mca_io_ompio_file_t *fh); - - -int mca_io_ompio_split_initial_groups(mca_io_ompio_file_t *fh, - OMPI_MPI_OFFSET_TYPE *start_offsets_lens, - OMPI_MPI_OFFSET_TYPE *end_offsets, - OMPI_MPI_OFFSET_TYPE bytes_per_group); - - -int mca_io_ompio_split_a_group(mca_io_ompio_file_t *fh, - OMPI_MPI_OFFSET_TYPE *start_offsets_lens, - OMPI_MPI_OFFSET_TYPE *end_offsets, - int size_new_group, - OMPI_MPI_OFFSET_TYPE *max_cci, - OMPI_MPI_OFFSET_TYPE *min_cci, - int *num_groups, int *size_smallest_group); - -int mca_io_ompio_finalize_split(mca_io_ompio_file_t *fh, int size_new_group, - int size_last_group); - -int mca_io_ompio_merge_initial_groups(mca_io_ompio_file_t *fh, - OMPI_MPI_OFFSET_TYPE *aggr_bytes_per_group, - int *decision_list, int is_aggregator); - -int mca_io_ompio_merge_groups(mca_io_ompio_file_t *fh, int *merge_aggrs, - int num_merge_aggrs); - - -#endif diff --git a/ompi/mca/io/ompio/io_ompio_component.c b/ompi/mca/io/ompio/io_ompio_component.c index 41a477828c..bdacd3a266 100644 --- a/ompi/mca/io/ompio/io_ompio_component.c +++ b/ompi/mca/io/ompio/io_ompio_component.c @@ -13,7 +13,7 @@ * Copyright (c) 2008-2018 University of Houston. All rights reserved. * Copyright (c) 2015 Los Alamos National Security, LLC. All rights * reserved. - * Copyright (c) 2015 Research Organization for Information Science + * Copyright (c) 2015-2018 Research Organization for Information Science * and Technology (RIST). All rights reserved. * Copyright (c) 2016-2017 IBM Corporation. All rights reserved. * $COPYRIGHT$ @@ -260,7 +260,8 @@ static int open_component(void) mca_common_ompio_request_init (); - return OMPI_SUCCESS; + return mca_common_ompio_set_callbacks(ompi_io_ompio_generate_current_file_view, + mca_io_ompio_get_mca_parameter_value); } @@ -367,7 +368,7 @@ static int delete_select(const char *filename, struct opal_info_t *info, int ret; OPAL_THREAD_LOCK (&mca_io_ompio_mutex); - ret = mca_io_ompio_file_delete (filename, info); + ret = mca_common_ompio_file_delete (filename, info); OPAL_THREAD_UNLOCK (&mca_io_ompio_mutex); return ret; diff --git a/ompi/mca/io/ompio/io_ompio_file_open.c b/ompi/mca/io/ompio/io_ompio_file_open.c index 6c71805720..d9b1d1a695 100644 --- a/ompi/mca/io/ompio/io_ompio_file_open.c +++ b/ompi/mca/io/ompio/io_ompio_file_open.c @@ -10,9 +10,9 @@ * Copyright (c) 2004-2005 The Regents of the University of California. * All rights reserved. * Copyright (c) 2008-2016 University of Houston. All rights reserved. - * Copyright (c) 2015-2017 Research Organization for Information Science + * Copyright (c) 2015-2018 Research Organization for Information Science * and Technology (RIST). All rights reserved. - * Copyright (c) 2016 Cisco Systems, Inc. All rights reserved. + * Copyright (c) 2016 Cisco Systems, Inc. All rights reserved. * Copyright (c) 2016-2017 IBM Corporation. All rights reserved. * $COPYRIGHT$ * @@ -103,30 +103,6 @@ int mca_io_ompio_file_close (ompi_file_t *fh) return ret; } -int mca_io_ompio_file_delete (const char *filename, - struct opal_info_t *info) -{ - int ret = OMPI_SUCCESS; - - /* No locking required for file_delete according to my understanding. - One thread will succeed, the other ones silently ignore the - error that the file is already deleted. - */ - ret = unlink(filename); - - if (0 > ret ) { - if ( ENOENT == errno ) { - return MPI_ERR_NO_SUCH_FILE; - } else { - opal_output (0, "mca_io_ompio_file_delete: Could not remove file %s errno = %d %s\n", filename, - errno, strerror(errno)); - return MPI_ERR_ACCESS; - } - } - - return OMPI_SUCCESS; -} - int mca_io_ompio_file_preallocate (ompi_file_t *fh, OMPI_MPI_OFFSET_TYPE diskspace) { diff --git a/ompi/mca/sharedfp/individual/Makefile.am b/ompi/mca/sharedfp/individual/Makefile.am index d0a4ed34ba..acb17a7081 100644 --- a/ompi/mca/sharedfp/individual/Makefile.am +++ b/ompi/mca/sharedfp/individual/Makefile.am @@ -11,6 +11,8 @@ # All rights reserved. # Copyright (c) 2008 University of Houston. All rights reserved. # Copyright (c) 2017 IBM Corporation. All rights reserved. +# Copyright (c) 2018 Research Organization for Information Science +# and Technology (RIST). All rights reserved. # $COPYRIGHT$ # # Additional copyrights may follow @@ -34,7 +36,7 @@ mcacomponentdir = $(ompilibdir) mcacomponent_LTLIBRARIES = $(component_install) mca_sharedfp_individual_la_SOURCES = $(sources) mca_sharedfp_individual_la_LDFLAGS = -module -avoid-version -mca_sharedfp_individual_la_LIBADD = $(top_builddir)/ompi/lib@OMPI_LIBMPI_NAME@.la +mca_sharedfp_individual_la_LIBADD = $(OMPI_TOP_BUILDDIR)/ompi/mca/common/ompio/libmca_common_ompio.la noinst_LTLIBRARIES = $(component_noinst) libmca_sharedfp_individual_la_SOURCES = $(sources) diff --git a/ompi/mca/sharedfp/individual/sharedfp_individual_collaborate_data.c b/ompi/mca/sharedfp/individual/sharedfp_individual_collaborate_data.c index 1fc2a388d9..3b44b48505 100644 --- a/ompi/mca/sharedfp/individual/sharedfp_individual_collaborate_data.c +++ b/ompi/mca/sharedfp/individual/sharedfp_individual_collaborate_data.c @@ -10,6 +10,8 @@ * Copyright (c) 2004-2005 The Regents of the University of California. * All rights reserved. * Copyright (c) 2013-2016 University of Houston. All rights reserved. + * Copyright (c) 2018 Research Organization for Information Science + * and Technology (RIST). All rights reserved. * $COPYRIGHT$ * * Additional copyrights may follow @@ -26,7 +28,6 @@ #include "ompi/mca/sharedfp/sharedfp.h" #include "ompi/mca/sharedfp/base/base.h" #include "ompi/mca/common/ompio/common_ompio.h" -#include "ompi/mca/io/ompio/io_ompio.h" #include #include diff --git a/ompi/mca/sharedfp/lockedfile/Makefile.am b/ompi/mca/sharedfp/lockedfile/Makefile.am index b0151c5612..8777844675 100644 --- a/ompi/mca/sharedfp/lockedfile/Makefile.am +++ b/ompi/mca/sharedfp/lockedfile/Makefile.am @@ -11,6 +11,8 @@ # All rights reserved. # Copyright (c) 2008 University of Houston. All rights reserved. # Copyright (c) 2017 IBM Corporation. All rights reserved. +# Copyright (c) 2018 Research Organization for Information Science +# and Technology (RIST). All rights reserved. # $COPYRIGHT$ # # Additional copyrights may follow @@ -34,7 +36,7 @@ mcacomponentdir = $(ompilibdir) mcacomponent_LTLIBRARIES = $(component_install) mca_sharedfp_lockedfile_la_SOURCES = $(sources) mca_sharedfp_lockedfile_la_LDFLAGS = -module -avoid-version -mca_sharedfp_lockedfile_la_LIBADD = $(top_builddir)/ompi/lib@OMPI_LIBMPI_NAME@.la +mca_sharedfp_lockedfile_la_LIBADD = $(OMPI_TOP_BUILDDIR)/ompi/mca/common/ompio/libmca_common_ompio.la noinst_LTLIBRARIES = $(component_noinst) libmca_sharedfp_lockedfile_la_SOURCES = $(sources) diff --git a/ompi/mca/sharedfp/lockedfile/sharedfp_lockedfile_iread.c b/ompi/mca/sharedfp/lockedfile/sharedfp_lockedfile_iread.c index 190573ab6f..e23ee6eb11 100644 --- a/ompi/mca/sharedfp/lockedfile/sharedfp_lockedfile_iread.c +++ b/ompi/mca/sharedfp/lockedfile/sharedfp_lockedfile_iread.c @@ -10,6 +10,8 @@ * Copyright (c) 2004-2005 The Regents of the University of California. * All rights reserved. * Copyright (c) 2013-2016 University of Houston. All rights reserved. + * Copyright (c) 2018 Research Organization for Information Science + * and Technology (RIST). All rights reserved. * $COPYRIGHT$ * * Additional copyrights may follow @@ -25,7 +27,7 @@ #include "ompi/constants.h" #include "ompi/mca/sharedfp/sharedfp.h" #include "ompi/mca/sharedfp/base/base.h" -#include "ompi/mca/io/ompio/io_ompio.h" +#include "ompi/mca/common/ompio/common_ompio.h" int mca_sharedfp_lockedfile_iread(mca_io_ompio_file_t *fh, void *buf, diff --git a/ompi/mca/sharedfp/lockedfile/sharedfp_lockedfile_iwrite.c b/ompi/mca/sharedfp/lockedfile/sharedfp_lockedfile_iwrite.c index bf9fb801be..dbc64909cc 100644 --- a/ompi/mca/sharedfp/lockedfile/sharedfp_lockedfile_iwrite.c +++ b/ompi/mca/sharedfp/lockedfile/sharedfp_lockedfile_iwrite.c @@ -10,7 +10,7 @@ * Copyright (c) 2004-2005 The Regents of the University of California. * All rights reserved. * Copyright (c) 2013-2016 University of Houston. All rights reserved. - * Copyright (c) 2015 Research Organization for Information Science + * Copyright (c) 2015-2018 Research Organization for Information Science * and Technology (RIST). All rights reserved. * $COPYRIGHT$ * @@ -27,7 +27,7 @@ #include "ompi/constants.h" #include "ompi/mca/sharedfp/sharedfp.h" #include "ompi/mca/sharedfp/base/base.h" -#include "ompi/mca/io/ompio/io_ompio.h" +#include "ompi/mca/common/ompio/common_ompio.h" int mca_sharedfp_lockedfile_iwrite(mca_io_ompio_file_t *fh, const void *buf, diff --git a/ompi/mca/sharedfp/sm/Makefile.am b/ompi/mca/sharedfp/sm/Makefile.am index 3553cb80c5..62e0ab43f4 100644 --- a/ompi/mca/sharedfp/sm/Makefile.am +++ b/ompi/mca/sharedfp/sm/Makefile.am @@ -11,6 +11,8 @@ # All rights reserved. # Copyright (c) 2008 University of Houston. All rights reserved. # Copyright (c) 2017 IBM Corporation. All rights reserved. +# Copyright (c) 2018 Research Organization for Information Science +# and Technology (RIST). All rights reserved. # $COPYRIGHT$ # # Additional copyrights may follow @@ -34,7 +36,7 @@ mcacomponentdir = $(ompilibdir) mcacomponent_LTLIBRARIES = $(component_install) mca_sharedfp_sm_la_SOURCES = $(sources) mca_sharedfp_sm_la_LDFLAGS = -module -avoid-version -mca_sharedfp_sm_la_LIBADD = $(top_builddir)/ompi/lib@OMPI_LIBMPI_NAME@.la +mca_sharedfp_sm_la_LIBADD = $(OMPI_TOP_BUILDDIR)/ompi/mca/common/ompio/libmca_common_ompio.la noinst_LTLIBRARIES = $(component_noinst) libmca_sharedfp_sm_la_SOURCES = $(sources)