2015-04-18 18:36:05 +03:00
/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
2005-08-31 05:43:48 +04:00
/*
2007-03-17 02:11:45 +03:00
* Copyright ( c ) 2004 - 2007 The Trustees of Indiana University and Indiana
2005-11-05 22:57:48 +03:00
* University Research and Technology
* Corporation . All rights reserved .
2015-02-25 02:49:23 +03:00
* Copyright ( c ) 2004 - 2015 The University of Tennessee and The University
2005-11-05 22:57:48 +03:00
* of Tennessee Research Foundation . All rights
* reserved .
2015-02-25 02:49:23 +03:00
* Copyright ( c ) 2004 - 2005 High Performance Computing Center Stuttgart ,
2005-08-31 05:43:48 +04:00
* University of Stuttgart . All rights reserved .
* Copyright ( c ) 2004 - 2005 The Regents of the University of California .
* All rights reserved .
2008-05-07 06:31:24 +04:00
* Copyright ( c ) 2008 Sun Microsystems , Inc . All rights reserved .
2008-07-29 02:40:57 +04:00
* Copyright ( c ) 2008 Cisco Systems , Inc . All rights reserved .
2015-04-18 18:36:05 +03:00
* Copyright ( c ) 2015 Los Alamos National Security , LLC . All rights
* reserved .
2015-05-08 03:19:02 +03:00
* Copyright ( c ) 2015 Research Organization for Information Science
* and Technology ( RIST ) . All rights reserved .
2005-08-31 05:43:48 +04:00
* $ COPYRIGHT $
2015-06-24 06:59:57 +03:00
*
2005-08-31 05:43:48 +04:00
* Additional copyrights may follow
2015-06-24 06:59:57 +03:00
*
2005-08-31 05:43:48 +04:00
* $ HEADER $
*
* These symbols are in a file by themselves to provide nice linker
* semantics . Since linkers generally pull in symbols by object
* files , keeping these symbols as the only symbols in this file
* prevents utility programs such as " ompi_info " from having to import
* entire components just to query their version and parameters .
*/
# include "ompi_config.h"
2009-02-14 05:26:12 +03:00
# include "opal/util/output.h"
2005-08-31 05:43:48 +04:00
# include "coll_tuned.h"
# include "mpi.h"
2006-02-12 04:33:29 +03:00
# include "ompi/mca/coll/coll.h"
2005-08-31 05:43:48 +04:00
# include "coll_tuned.h"
2009-08-15 01:06:23 +04:00
# include "coll_tuned_dynamic_file.h"
2005-11-10 04:19:27 +03:00
2005-08-31 05:43:48 +04:00
/*
* Public string showing the coll ompi_tuned component version number
*/
2005-12-22 16:49:33 +03:00
const char * ompi_coll_tuned_component_version_string =
2007-08-19 07:37:49 +04:00
" Open MPI tuned collective MCA component version " OMPI_VERSION ;
2005-08-31 05:43:48 +04:00
/*
* Global variable
*/
2005-12-22 16:49:33 +03:00
int ompi_coll_tuned_stream = - 1 ;
int ompi_coll_tuned_priority = 30 ;
2013-03-28 01:09:41 +04:00
bool ompi_coll_tuned_use_dynamic_rules = false ;
2005-12-22 16:49:33 +03:00
char * ompi_coll_tuned_dynamic_rules_filename = ( char * ) NULL ;
int ompi_coll_tuned_init_tree_fanout = 4 ;
int ompi_coll_tuned_init_chain_fanout = 4 ;
2007-02-20 07:25:00 +03:00
int ompi_coll_tuned_init_max_requests = 128 ;
2014-09-18 12:07:51 +04:00
int ompi_coll_tuned_alltoall_small_msg = 200 ;
int ompi_coll_tuned_alltoall_intermediate_msg = 3000 ;
2005-10-25 01:33:48 +04:00
/* forced alogrithm variables */
2006-04-20 03:42:06 +04:00
/* indices for the MCA parameters */
2015-05-08 03:19:02 +03:00
coll_tuned_force_algorithm_mca_param_indices_t ompi_coll_tuned_forced_params [ COLLCOUNT ] = { { 0 } } ;
2006-04-20 03:42:06 +04:00
/* max algorithm values */
2015-05-08 03:19:02 +03:00
int ompi_coll_tuned_forced_max_algorithms [ COLLCOUNT ] = { 0 } ;
2005-10-25 07:55:58 +04:00
2005-08-31 05:43:48 +04:00
/*
* Local function
*/
2013-03-28 01:09:41 +04:00
static int tuned_register ( void ) ;
2005-08-31 05:43:48 +04:00
static int tuned_open ( void ) ;
2006-02-01 23:50:14 +03:00
static int tuned_close ( void ) ;
2005-08-31 05:43:48 +04:00
/*
* Instantiate the public struct with all of our public information
* and pointers to our public functions in it
*/
2006-02-01 23:50:14 +03:00
mca_coll_tuned_component_t mca_coll_tuned_component = {
2008-07-29 02:40:57 +04:00
/* First, fill in the super */
2015-06-24 06:59:57 +03:00
{
2006-10-18 06:00:46 +04:00
/* First, the mca_component_t struct containing meta information
about the component itself */
2015-04-18 18:36:05 +03:00
. collm_version = {
2008-07-29 02:40:57 +04:00
MCA_COLL_BASE_VERSION_2_0_0 ,
2005-08-31 05:43:48 +04:00
2006-10-18 06:00:46 +04:00
/* Component name and version */
2015-04-18 18:36:05 +03:00
. mca_component_name = " tuned " ,
MCA_BASE_MAKE_VERSION ( component , OMPI_MAJOR_VERSION , OMPI_MINOR_VERSION ,
OMPI_RELEASE_VERSION ) ,
2005-08-31 05:43:48 +04:00
2006-10-18 06:00:46 +04:00
/* Component open and close functions */
2015-04-18 18:36:05 +03:00
. mca_open_component = tuned_open ,
. mca_close_component = tuned_close ,
. mca_register_component_params = tuned_register ,
2006-10-18 06:00:46 +04:00
} ,
2015-04-18 18:36:05 +03:00
. collm_data = {
2007-03-17 02:11:45 +03:00
/* The component is checkpoint ready */
MCA_BASE_METADATA_PARAM_CHECKPOINT
2006-10-18 06:00:46 +04:00
} ,
2006-02-01 23:50:14 +03:00
2006-10-18 06:00:46 +04:00
/* Initialization / querying functions */
2006-02-01 23:50:14 +03:00
2015-04-18 18:36:05 +03:00
. collm_init_query = ompi_coll_tuned_init_query ,
. collm_comm_query = ompi_coll_tuned_comm_query ,
2006-10-18 06:00:46 +04:00
} ,
2006-02-01 23:50:14 +03:00
/* priority of the module */
0 ,
2006-10-18 06:00:46 +04:00
/* Tuned component specific information */
2015-06-24 06:59:57 +03:00
NULL /* ompi_coll_alg_rule_t ptr */
2005-08-31 05:43:48 +04:00
} ;
2013-03-28 01:09:41 +04:00
static int tuned_register ( void )
{
2015-06-24 06:59:57 +03:00
/* Use a low priority, but allow other components to be lower */
2013-03-28 01:09:41 +04:00
ompi_coll_tuned_priority = 30 ;
( void ) mca_base_component_var_register ( & mca_coll_tuned_component . super . collm_version ,
" priority " , " Priority of the tuned coll component " ,
MCA_BASE_VAR_TYPE_INT , NULL , 0 , 0 ,
OPAL_INFO_LVL_6 ,
MCA_BASE_VAR_SCOPE_READONLY ,
& ompi_coll_tuned_priority ) ;
/* some initial guesses at topology parameters */
ompi_coll_tuned_init_tree_fanout = 4 ;
( void ) mca_base_component_var_register ( & mca_coll_tuned_component . super . collm_version ,
" init_tree_fanout " ,
" Inital fanout used in the tree topologies for each communicator. This is only an initial guess, if a tuned collective needs a different fanout for an operation, it build it dynamically. This parameter is only for the first guess and might save a little time " ,
MCA_BASE_VAR_TYPE_INT , NULL , 0 , 0 ,
OPAL_INFO_LVL_6 ,
MCA_BASE_VAR_SCOPE_READONLY ,
& ompi_coll_tuned_init_tree_fanout ) ;
ompi_coll_tuned_init_chain_fanout = 4 ;
( void ) mca_base_component_var_register ( & mca_coll_tuned_component . super . collm_version ,
" init_chain_fanout " ,
" Inital fanout used in the chain (fanout followed by pipeline) topologies for each communicator. This is only an initial guess, if a tuned collective needs a different fanout for an operation, it build it dynamically. This parameter is only for the first guess and might save a little time " ,
MCA_BASE_VAR_TYPE_INT , NULL , 0 , 0 ,
OPAL_INFO_LVL_6 ,
MCA_BASE_VAR_SCOPE_READONLY ,
& ompi_coll_tuned_init_chain_fanout ) ;
2014-09-18 12:07:51 +04:00
ompi_coll_tuned_alltoall_small_msg = 200 ;
2014-09-15 16:34:21 +04:00
( void ) mca_base_component_var_register ( & mca_coll_tuned_component . super . collm_version ,
" alltoall_small_msg " ,
" threshold (if supported) to decide if small MSGs alltoall algorithm will be used " ,
MCA_BASE_VAR_TYPE_INT , NULL , 0 , 0 ,
OPAL_INFO_LVL_6 ,
MCA_BASE_VAR_SCOPE_READONLY ,
& ompi_coll_tuned_alltoall_small_msg ) ;
2014-09-18 12:07:51 +04:00
ompi_coll_tuned_alltoall_intermediate_msg = 3000 ;
2014-09-15 16:34:21 +04:00
( void ) mca_base_component_var_register ( & mca_coll_tuned_component . super . collm_version ,
" alltoall_intermediate_msg " ,
" threshold (if supported) to decide if intermediate MSGs alltoall algorithm will be used " ,
MCA_BASE_VAR_TYPE_INT , NULL , 0 , 0 ,
OPAL_INFO_LVL_6 ,
MCA_BASE_VAR_SCOPE_READONLY ,
& ompi_coll_tuned_alltoall_intermediate_msg ) ;
2013-03-28 01:09:41 +04:00
ompi_coll_tuned_use_dynamic_rules = false ;
( void ) mca_base_component_var_register ( & mca_coll_tuned_component . super . collm_version ,
" use_dynamic_rules " ,
" Switch used to decide if we use static (compiled/if statements) or dynamic (built at runtime) decision function rules " ,
MCA_BASE_VAR_TYPE_BOOL , NULL , 0 , 0 ,
OPAL_INFO_LVL_6 ,
MCA_BASE_VAR_SCOPE_READONLY ,
& ompi_coll_tuned_use_dynamic_rules ) ;
ompi_coll_tuned_dynamic_rules_filename = NULL ;
( void ) mca_base_component_var_register ( & mca_coll_tuned_component . super . collm_version ,
" dynamic_rules_filename " ,
" Filename of configuration file that contains the dynamic (@runtime) decision function rules " ,
MCA_BASE_VAR_TYPE_STRING , NULL , 0 , 0 ,
OPAL_INFO_LVL_6 ,
MCA_BASE_VAR_SCOPE_READONLY ,
& ompi_coll_tuned_dynamic_rules_filename ) ;
/* register forced params */
ompi_coll_tuned_allreduce_intra_check_forced_init ( & ompi_coll_tuned_forced_params [ ALLREDUCE ] ) ;
ompi_coll_tuned_alltoall_intra_check_forced_init ( & ompi_coll_tuned_forced_params [ ALLTOALL ] ) ;
ompi_coll_tuned_allgather_intra_check_forced_init ( & ompi_coll_tuned_forced_params [ ALLGATHER ] ) ;
ompi_coll_tuned_allgatherv_intra_check_forced_init ( & ompi_coll_tuned_forced_params [ ALLGATHERV ] ) ;
ompi_coll_tuned_alltoallv_intra_check_forced_init ( & ompi_coll_tuned_forced_params [ ALLTOALLV ] ) ;
ompi_coll_tuned_barrier_intra_check_forced_init ( & ompi_coll_tuned_forced_params [ BARRIER ] ) ;
ompi_coll_tuned_bcast_intra_check_forced_init ( & ompi_coll_tuned_forced_params [ BCAST ] ) ;
ompi_coll_tuned_reduce_intra_check_forced_init ( & ompi_coll_tuned_forced_params [ REDUCE ] ) ;
ompi_coll_tuned_reduce_scatter_intra_check_forced_init ( & ompi_coll_tuned_forced_params [ REDUCESCATTER ] ) ;
ompi_coll_tuned_gather_intra_check_forced_init ( & ompi_coll_tuned_forced_params [ GATHER ] ) ;
ompi_coll_tuned_scatter_intra_check_forced_init ( & ompi_coll_tuned_forced_params [ SCATTER ] ) ;
return OMPI_SUCCESS ;
}
2005-08-31 05:43:48 +04:00
static int tuned_open ( void )
{
2009-08-15 01:06:23 +04:00
int rc ;
2009-05-07 00:11:28 +04:00
# if OPAL_ENABLE_DEBUG
2006-11-10 22:54:09 +03:00
{
int param ;
2005-08-31 05:43:48 +04:00
2013-03-28 01:09:41 +04:00
param = mca_base_var_find ( " ompi " , " coll " , " base " , " verbose " ) ;
2006-11-10 22:54:09 +03:00
if ( param > = 0 ) {
2013-03-28 01:09:41 +04:00
const int * verbose = NULL ;
mca_base_var_get_value ( param , & verbose , NULL , NULL ) ;
if ( verbose & & verbose [ 0 ] > 0 ) {
2008-06-09 18:53:58 +04:00
ompi_coll_tuned_stream = opal_output_open ( NULL ) ;
2006-11-10 22:54:09 +03:00
}
}
}
2009-05-07 00:11:28 +04:00
# endif /* OPAL_ENABLE_DEBUG */
2006-11-10 22:54:09 +03:00
2006-04-20 03:42:06 +04:00
/* now check that the user hasn't overrode any of the decision functions if dynamic rules are enabled */
/* the user can redo this before every comm dup/create if they like */
2005-10-25 07:55:58 +04:00
/* this is useful for benchmarking and user knows best tuning */
2006-04-20 03:42:06 +04:00
/* as this is the component we only lookup the indicies of the mca params */
/* the actual values are looked up during comm create via module init */
2015-06-24 06:59:57 +03:00
2005-10-25 07:55:58 +04:00
/* intra functions first */
2006-11-10 22:54:09 +03:00
/* if dynamic rules allowed then look up dynamic rules config filename, else we leave it an empty filename (NULL) */
/* by default DISABLE dynamic rules and instead use fixed [if based] rules */
2006-04-20 03:42:06 +04:00
if ( ompi_coll_tuned_use_dynamic_rules ) {
2009-08-15 01:06:23 +04:00
if ( ompi_coll_tuned_dynamic_rules_filename ) {
2015-06-24 06:59:57 +03:00
OPAL_OUTPUT ( ( ompi_coll_tuned_stream , " coll:tuned:component_open Reading collective rules file [%s] " ,
2009-08-15 01:06:23 +04:00
ompi_coll_tuned_dynamic_rules_filename ) ) ;
rc = ompi_coll_tuned_read_rules_config_file ( ompi_coll_tuned_dynamic_rules_filename ,
& ( mca_coll_tuned_component . all_base_rules ) , COLLCOUNT ) ;
if ( rc > = 0 ) {
OPAL_OUTPUT ( ( ompi_coll_tuned_stream , " coll:tuned:module_open Read %d valid rules \n " , rc ) ) ;
} else {
OPAL_OUTPUT ( ( ompi_coll_tuned_stream , " coll:tuned:module_open Reading collective rules file failed \n " ) ) ;
mca_coll_tuned_component . all_base_rules = NULL ;
}
}
2006-04-20 03:42:06 +04:00
}
2005-10-25 07:55:58 +04:00
2008-06-09 18:53:58 +04:00
OPAL_OUTPUT ( ( ompi_coll_tuned_stream , " coll:tuned:component_open: done! " ) ) ;
2005-08-31 05:43:48 +04:00
return OMPI_SUCCESS ;
}
2006-02-01 23:50:14 +03:00
/* here we should clean up state stored on the component */
/* i.e. alg table and dynamic changable rules if allocated etc */
static int tuned_close ( void )
{
2008-06-09 18:53:58 +04:00
OPAL_OUTPUT ( ( ompi_coll_tuned_stream , " coll:tuned:component_close: called " ) ) ;
2006-02-01 23:50:14 +03:00
/* dealloc alg table if allocated */
/* dealloc dynamic changable rules if allocated */
2008-06-09 18:53:58 +04:00
OPAL_OUTPUT ( ( ompi_coll_tuned_stream , " coll:tuned:component_close: done! " ) ) ;
2006-02-01 23:50:14 +03:00
2009-08-15 01:06:23 +04:00
if ( NULL ! = mca_coll_tuned_component . all_base_rules ) {
ompi_coll_tuned_free_all_rules ( mca_coll_tuned_component . all_base_rules , COLLCOUNT ) ;
mca_coll_tuned_component . all_base_rules = NULL ;
}
2006-02-01 23:50:14 +03:00
return OMPI_SUCCESS ;
}
2007-08-19 07:37:49 +04:00
static void
mca_coll_tuned_module_construct ( mca_coll_tuned_module_t * module )
{
2015-02-15 22:48:08 +03:00
mca_coll_tuned_module_t * tuned_module = ( mca_coll_tuned_module_t * ) module ;
2015-06-24 06:59:57 +03:00
2015-02-15 22:48:08 +03:00
for ( int i = 0 ; i < COLLCOUNT ; i + + ) {
tuned_module - > user_forced [ i ] . algorithm = 0 ;
tuned_module - > com_rules [ i ] = NULL ;
2007-08-19 07:37:49 +04:00
}
}
2015-02-15 22:48:08 +03:00
OBJ_CLASS_INSTANCE ( mca_coll_tuned_module_t , mca_coll_base_module_t ,
mca_coll_tuned_module_construct , NULL ) ;