cf377db823
Features: - Support for an override parameter file (openmpi-mca-param-override.conf). Variable values in this file can not be overridden by any file or environment value. - Support for boolean, unsigned, and unsigned long long variables. - Support for true/false values. - Support for enumerations on integer variables. - Support for MPIT scope, verbosity, and binding. - Support for command line source. - Support for setting variable source via the environment using OMPI_MCA_SOURCE_<var name>=source (either command or file:filename) - Cleaner API. - Support for variable groups (equivalent to MPIT categories). Notes: - Variables must be created with a backing store (char **, int *, or bool *) that must live at least as long as the variable. - Creating a variable with the MCA_BASE_VAR_FLAG_SETTABLE enables the use of mca_base_var_set_value() to change the value. - String values are duplicated when the variable is registered. It is up to the caller to free the original value if necessary. The new value will be freed by the mca_base_var system and must not be freed by the user. - Variables with constant scope may not be settable. - Variable groups (and all associated variables) are deregistered when the component is closed or the component repository item is freed. This prevents a segmentation fault from accessing a variable after its component is unloaded. - After some discussion we decided we should remove the automatic registration of component priority variables. Few component actually made use of this feature. - The enumerator interface was updated to be general enough to handle future uses of the interface. - The code to generate ompi_info output has been moved into the MCA variable system. See mca_base_var_dump(). opal: update core and components to mca_base_var system orte: update core and components to mca_base_var system ompi: update core and components to mca_base_var system This commit also modifies the rmaps framework. The following variables were moved from ppr and lama: rmaps_base_pernode, rmaps_base_n_pernode, rmaps_base_n_persocket. Both lama and ppr create synonyms for these variables. This commit was SVN r28236.
315 строки
13 KiB
C
315 строки
13 KiB
C
/*
|
|
* Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
|
|
* University Research and Technology
|
|
* Corporation. All rights reserved.
|
|
* Copyright (c) 2004-2005 The University of Tennessee and The University
|
|
* of Tennessee Research Foundation. All rights
|
|
* reserved.
|
|
* Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
|
|
* University of Stuttgart. All rights reserved.
|
|
* Copyright (c) 2004-2005 The Regents of the University of California.
|
|
* All rights reserved.
|
|
* Copyright (c) 2006 Cisco Systems, Inc. All rights reserved.
|
|
* Copyright (c) 2007-2008 Sun Microsystems, Inc. All rights reserved.
|
|
* $COPYRIGHT$
|
|
*
|
|
* Additional copyrights may follow
|
|
*
|
|
* $HEADER$
|
|
*/
|
|
|
|
#include "ompi_config.h"
|
|
#include "opal/mca/base/base.h"
|
|
#include "btl_udapl.h"
|
|
#include "btl_udapl_mca.h"
|
|
#include <string.h>
|
|
|
|
/*
|
|
* Utility routine for string parameter registration.
|
|
*
|
|
* @param param_name (IN) MCA parameter name
|
|
* @param param_desc (IN) MCA parameter description
|
|
* @param default_value (IN) MCA parameter default value
|
|
* @param out_value (OUT) value of MCA parameter; either default,
|
|
* or value as determined from typical
|
|
* MCA parameter setting methods
|
|
* @param flags (IN) MCA parameter boundary flag
|
|
* @return OMPI_SUCCESS or OMPI_ERR_BAD_PARAM
|
|
*/
|
|
static inline int mca_btl_udapl_reg_string(const char* param_name,
|
|
const char* param_desc,
|
|
const char* default_value,
|
|
char **storage, int flags)
|
|
{
|
|
*storage = default_value;
|
|
(void) mca_base_component_var_register(&mca_btl_udapl_component.super.btl_version, param_name,
|
|
param_desc, MCA_BASE_VAR_TYPE_STRING, NULL, 0, 0,
|
|
OPAL_INFO_LVL_9,
|
|
MCA_BASE_VAR_SCOPE_READONLY, storage);
|
|
|
|
if (NULL == *storage && !((flags & REGSTR_EMPTY_OK) == REGSTR_EMPTY_OK)) {
|
|
BTL_ERROR(("ERROR: MCA Parameter %s : Value (NULL) out of range : "
|
|
"Default value (%s)\n \t Parameter Description : %s",
|
|
param_name, default_value, param_desc));
|
|
return OMPI_ERR_BAD_PARAM;
|
|
}
|
|
|
|
if ((flags & REGSTR_EMPTY_NOT_OK) && 0 == strlen(*storage)) {
|
|
BTL_ERROR(("ERROR: MCA Parameter %s : Value (%s) out of range : "
|
|
"Default value (%s)\n \t Parameter Description : %s",
|
|
param_name, *storage, default_value, param_desc));
|
|
return OMPI_ERR_BAD_PARAM;
|
|
}
|
|
|
|
return OMPI_SUCCESS;
|
|
}
|
|
|
|
|
|
/*
|
|
* Utility routine for integer parameter registration.
|
|
*
|
|
* @param param_name (IN) MCA parameter name
|
|
* @param param_desc (IN) MCA parameter description
|
|
* @param default_value (IN) MCA parameter default value
|
|
* @param out_value (OUT) value of MCA parameter; either default,
|
|
* or value as determined from typical
|
|
* MCA parameter setting methods
|
|
* @param flags (IN) MCA parameter boundary flag
|
|
* @return OMPI_SUCCESS or OMPI_ERR_BAD_PARAM
|
|
*/
|
|
static inline int mca_btl_udapl_reg_int(const char* param_name,
|
|
const char* param_desc,
|
|
int default_value, int *storage,
|
|
int flags)
|
|
{
|
|
*storage = default_value;
|
|
(void) mca_base_component_var_register(&mca_btl_udapl_component.super.btl_version, param_name,
|
|
param_desc, MCA_BASE_VAR_TYPE_INT, NULL, 0, 0,
|
|
OPAL_INFO_LVL_9,
|
|
MCA_BASE_VAR_SCOPE_READONLY, storage);
|
|
|
|
if ((flags & REGINT_NEG_ONE_OK) && -1 == *storage) {
|
|
return OMPI_SUCCESS;
|
|
}
|
|
|
|
if (((flags & REGINT_GE_ZERO) && *storage < 0) ||
|
|
((flags & REGINT_GE_ONE) && *storage < 1) ||
|
|
((flags & REGINT_NONZERO) && 0 == *storage)) {
|
|
BTL_ERROR(("ERROR: MCA Parameter %s : Value (%d) out of range : "
|
|
"Default value (%d)\n \t Parameter Description : %s\n",
|
|
param_name, *storage, default_value, param_desc));
|
|
return OMPI_ERR_BAD_PARAM;
|
|
}
|
|
|
|
return OMPI_SUCCESS;
|
|
}
|
|
|
|
|
|
/*
|
|
* Register and check all MCA parameters
|
|
*
|
|
* @return OMPI_SUCCESS or OMPI_ERR_BAD_PARAM
|
|
*/
|
|
int mca_btl_udapl_register_mca_params(void)
|
|
{
|
|
int rc, tmp_rc;
|
|
|
|
rc = OMPI_SUCCESS;
|
|
|
|
/* register uDAPL component parameters */
|
|
CHECK_PARAM_REGISTER_RETURN_VALUE(mca_btl_udapl_reg_int("free_list_num",
|
|
"Initial size of free lists (must be >= 1).",
|
|
8,
|
|
&mca_btl_udapl_component.udapl_free_list_num,
|
|
REGINT_GE_ONE), tmp_rc, rc);
|
|
|
|
CHECK_PARAM_REGISTER_RETURN_VALUE(mca_btl_udapl_reg_int("free_list_max",
|
|
"Maximum size of free lists "
|
|
"(-1 = infinite, otherwise must be >= 1).",
|
|
-1,
|
|
&mca_btl_udapl_component.udapl_free_list_max,
|
|
REGINT_NEG_ONE_OK | REGINT_GE_ONE), tmp_rc, rc);
|
|
|
|
CHECK_PARAM_REGISTER_RETURN_VALUE(mca_btl_udapl_reg_int("free_list_inc",
|
|
"Increment size of free lists (must be >= 1).",
|
|
8,
|
|
&mca_btl_udapl_component.udapl_free_list_inc,
|
|
REGINT_GE_ONE), tmp_rc, rc);
|
|
|
|
CHECK_PARAM_REGISTER_RETURN_VALUE(mca_btl_udapl_reg_string("mpool",
|
|
"Name of the memory pool to be used.",
|
|
"grdma",
|
|
&mca_btl_udapl_component.udapl_mpool_name,
|
|
REGSTR_EMPTY_NOT_OK), tmp_rc, rc);
|
|
|
|
CHECK_PARAM_REGISTER_RETURN_VALUE(mca_btl_udapl_reg_int("max_modules",
|
|
"Maximum number of supported HCAs.",
|
|
8,
|
|
&mca_btl_udapl_component.udapl_max_btls,
|
|
REGINT_GE_ONE), tmp_rc, rc);
|
|
|
|
CHECK_PARAM_REGISTER_RETURN_VALUE(mca_btl_udapl_reg_int("num_recvs",
|
|
"Total number of receive buffers to keep posted "
|
|
"per endpoint (must be >= 1).",
|
|
8,
|
|
&mca_btl_udapl_component.udapl_num_recvs,
|
|
REGINT_GE_ONE), tmp_rc, rc);
|
|
|
|
CHECK_PARAM_REGISTER_RETURN_VALUE(mca_btl_udapl_reg_int("num_sends",
|
|
"Maximum number of sends to post on an endpoint "
|
|
"(must be >= 1).",
|
|
7,
|
|
&mca_btl_udapl_component.udapl_num_sends,
|
|
REGINT_GE_ONE), tmp_rc, rc);
|
|
|
|
CHECK_PARAM_REGISTER_RETURN_VALUE(mca_btl_udapl_reg_int("sr_win",
|
|
"Window size at which point an explicit "
|
|
"credit message will be generated (must be >= 1).",
|
|
4,
|
|
&mca_btl_udapl_component.udapl_sr_win,
|
|
REGINT_GE_ONE), tmp_rc, rc);
|
|
|
|
CHECK_PARAM_REGISTER_RETURN_VALUE(mca_btl_udapl_reg_int("use_eager_rdma",
|
|
"Use of RDMA for small messages : "
|
|
"1 = default, use RDMA for small messages; "
|
|
"0 = do not use RDMA for small messages. ",
|
|
1,
|
|
&mca_btl_udapl_component.udapl_use_eager_rdma,
|
|
REGINT_GE_ZERO), tmp_rc, rc);
|
|
|
|
CHECK_PARAM_REGISTER_RETURN_VALUE(mca_btl_udapl_reg_int("eager_rdma_num",
|
|
"Number of RDMA buffers to allocate "
|
|
"for small messages (must be >= 1).",
|
|
32,
|
|
&mca_btl_udapl_component.udapl_eager_rdma_num,
|
|
REGINT_GE_ONE), tmp_rc, rc);
|
|
|
|
CHECK_PARAM_REGISTER_RETURN_VALUE(mca_btl_udapl_reg_int("max_eager_rdma_peers",
|
|
"Maximum number of peers allowed to use "
|
|
"RDMA for short messages (independently RDMA will "
|
|
"still be used for large messages, (must be >= 0; "
|
|
"if zero then RDMA will not be used for short messages).",
|
|
16,
|
|
&mca_btl_udapl_component.udapl_max_eager_rdma_peers,
|
|
REGINT_GE_ZERO), tmp_rc, rc);
|
|
|
|
CHECK_PARAM_REGISTER_RETURN_VALUE(mca_btl_udapl_reg_int("eager_rdma_win",
|
|
"Window size at which point an explicit "
|
|
"credit message will be generated (must be >= 1).",
|
|
28,
|
|
&mca_btl_udapl_component.udapl_eager_rdma_win,
|
|
REGINT_GE_ONE), tmp_rc, rc);
|
|
|
|
CHECK_PARAM_REGISTER_RETURN_VALUE(mca_btl_udapl_reg_int("timeout",
|
|
"Connection timeout, in microseconds.",
|
|
MCA_BTL_UDAPL_CONN_TIMEOUT_DEFAULT,
|
|
&mca_btl_udapl_component.udapl_timeout,
|
|
REGINT_GE_ONE), tmp_rc, rc);
|
|
|
|
CHECK_PARAM_REGISTER_RETURN_VALUE(mca_btl_udapl_reg_int("conn_priv_data",
|
|
"Use connect private data to establish connections "
|
|
"(not supported by all uDAPL implementations).",
|
|
0,
|
|
&mca_btl_udapl_component.udapl_conn_priv_data,
|
|
REGINT_GE_ZERO), tmp_rc, rc);
|
|
|
|
CHECK_PARAM_REGISTER_RETURN_VALUE(mca_btl_udapl_reg_int("async_events",
|
|
"The asynchronous event queue will only be "
|
|
"checked after entering progress this number of times.",
|
|
100000000,
|
|
&mca_btl_udapl_component.udapl_async_events,
|
|
REGINT_GE_ONE), tmp_rc, rc);
|
|
|
|
CHECK_PARAM_REGISTER_RETURN_VALUE(mca_btl_udapl_reg_int("buffer_alignment",
|
|
"Preferred communication buffer alignment, "
|
|
"in bytes (must be >= 1).",
|
|
DAT_OPTIMAL_ALIGNMENT,
|
|
&mca_btl_udapl_component.udapl_buffer_alignment,
|
|
REGINT_GE_ONE), tmp_rc, rc);
|
|
|
|
CHECK_PARAM_REGISTER_RETURN_VALUE(mca_btl_udapl_reg_string("if_include",
|
|
"Comma-delimited list of interfaces to be included "
|
|
"(e.g. \"ibd0,ibd1 or OpenIB-cma,OpenIB-cma-1\"; empty value means "
|
|
"to use all interfaces found). Mutually exclusive with "
|
|
"btl_udapl_if_exclude.",
|
|
NULL, &mca_btl_udapl_component.if_include,
|
|
REGSTR_EMPTY_OK), tmp_rc, rc);
|
|
|
|
CHECK_PARAM_REGISTER_RETURN_VALUE(mca_btl_udapl_reg_string("if_exclude",
|
|
"Comma-delimited list of interfaces to be excluded from use "
|
|
"(e.g. \"ibd0,ibd1 or OpenIB-cma,OpenIB-cma-1\"; empty value means "
|
|
"not to exclude any). Mutually exclusive with btl_udapl_if_include.",
|
|
NULL, &mca_btl_udapl_component.if_exclude,
|
|
REGSTR_EMPTY_OK), tmp_rc, rc);
|
|
|
|
CHECK_PARAM_REGISTER_RETURN_VALUE(mca_btl_udapl_reg_int("verbose",
|
|
"Verbosity level of the uDAPL BTL (-1 thru 100)",
|
|
VERBOSE_SHOW_HELP,
|
|
&(mca_btl_udapl_component.udapl_verbosity),
|
|
REGINT_NEG_ONE_OK), tmp_rc, rc);
|
|
|
|
CHECK_PARAM_REGISTER_RETURN_VALUE(mca_btl_udapl_reg_int("compare_subnet",
|
|
"By default uDAPL BTL will compare subnets using netmask to "
|
|
"determine if an interface is reachable. Setting this parameter to "
|
|
"0 will essentially turn this comparison off and the uDAPL BTL will "
|
|
"assume all uDAPL interfaces are reachable (0 or 1, default==1).",
|
|
1,
|
|
&(mca_btl_udapl_component.udapl_compare_subnet),
|
|
REGINT_GE_ZERO), tmp_rc, rc);
|
|
|
|
/* register uDAPL module parameters */
|
|
CHECK_PARAM_REGISTER_RETURN_VALUE(mca_btl_udapl_reg_int("async_evd_qlen",
|
|
"The asynchronous event dispatcher queue length.",
|
|
MCA_BTL_UDAPL_ASYNC_EVD_QLEN_DEFAULT,
|
|
(int*)&mca_btl_udapl_module.udapl_async_evd_qlen,
|
|
REGINT_GE_ONE), tmp_rc, rc);
|
|
|
|
CHECK_PARAM_REGISTER_RETURN_VALUE(mca_btl_udapl_reg_int("conn_evd_qlen",
|
|
"The connection event dispatcher queue length is "
|
|
"a function of the number of connections expected.",
|
|
MCA_BTL_UDAPL_CONN_EVD_QLEN_DEFAULT,
|
|
(int*)&mca_btl_udapl_module.udapl_conn_evd_qlen,
|
|
REGINT_GE_ONE), tmp_rc, rc);
|
|
|
|
CHECK_PARAM_REGISTER_RETURN_VALUE(mca_btl_udapl_reg_int("dto_evd_qlen",
|
|
"The data transfer operation event dispatcher queue length is "
|
|
"a function of the number of connections as well as the "
|
|
"maximum number of outstanding data transfer operations.",
|
|
MCA_BTL_UDAPL_DTO_EVD_QLEN_DEFAULT,
|
|
(int*)&mca_btl_udapl_module.udapl_dto_evd_qlen,
|
|
REGINT_GE_ONE), tmp_rc, rc);
|
|
|
|
CHECK_PARAM_REGISTER_RETURN_VALUE(mca_btl_udapl_reg_int("max_request_dtos",
|
|
"Maximum number of outstanding "
|
|
"submitted sends and rdma operations per endpoint, (see Section "
|
|
"6.6.6 of uDAPL Spec.).",
|
|
MCA_BTL_UDAPL_MAX_REQUEST_DTOS_DEFAULT,
|
|
(int*)&mca_btl_udapl_module.udapl_max_request_dtos,
|
|
REGINT_GE_ONE), tmp_rc, rc);
|
|
|
|
CHECK_PARAM_REGISTER_RETURN_VALUE(mca_btl_udapl_reg_int("max_recv_dtos",
|
|
"Maximum number of outstanding "
|
|
"submitted receive operations per endpoint, (see Section "
|
|
"6.6.6 of uDAPL Spec.).",
|
|
MCA_BTL_UDAPL_MAX_RECV_DTOS_DEFAULT,
|
|
(int*)&mca_btl_udapl_module.udapl_max_recv_dtos,
|
|
REGINT_GE_ONE), tmp_rc, rc);
|
|
|
|
mca_btl_udapl_module.super.btl_exclusivity =
|
|
MCA_BTL_EXCLUSIVITY_DEFAULT - 10;
|
|
mca_btl_udapl_module.super.btl_eager_limit = 8*1024;
|
|
mca_btl_udapl_module.super.btl_rndv_eager_limit = 8*1024;
|
|
mca_btl_udapl_module.super.btl_max_send_size = 64*1024;
|
|
mca_btl_udapl_module.super.btl_rdma_pipeline_send_length = 512*1024;
|
|
mca_btl_udapl_module.super.btl_rdma_pipeline_frag_size = 128 * 1024;
|
|
mca_btl_udapl_module.super.btl_min_rdma_pipeline_size = 0;
|
|
mca_btl_udapl_module.super.btl_flags = MCA_BTL_FLAGS_PUT | MCA_BTL_FLAGS_SEND;
|
|
mca_btl_udapl_module.super.btl_bandwidth = 225;
|
|
mca_btl_udapl_module.super.btl_latency = 0;
|
|
|
|
mca_btl_base_param_register(&mca_btl_udapl_component.super.btl_version,
|
|
&mca_btl_udapl_module.super);
|
|
|
|
return rc;
|
|
}
|