ports/openmpi
ports/openmpi
1
1
Форкнуть 0
Код Релизы Активность

Fix for SEP when num local procs is greater than available contexts

Просмотр исходного кода 
For cases when the number of local processes is greater than the number of
available contexts, the SEP initialization phase would calculate the number of
contexts to provision for each rank to be 0 and would eventually crash.

Fix the issue here by using regular endpoints in the event the number of local
processes is more than available contexts. This fixes issue #6182.

Signed-off-by: Aravind Gopalakrishnan <Aravind.Gopalakrishnan@intel.com>
(cherry picked from commit e5e19dfcf7618185bdc89f1e91506a9bf15355b1)
Signed-off-by: Brian Barrett <bbarrett@amazon.com>
Этот коммит содержится в:
Aravind Gopalakrishnan 2018-12-12 10:53:56 -08:00 коммит произвёл Brian Barrett
родитель 4b293d3823
Коммит 3858b51d11
3 изменённых файлов: 26 добавлений и 21 удалений
Показать статистику Скачать Patch файл Скачать Diff файл Показать все файлы Свернуть все файлы

4
ompi/mca/mtl/ofi/mtl_ofi.h
Просмотреть файл

@ -1407,7 +1407,7 @@ ompi_mtl_ofi_add_comm(struct mca_mtl_base_module_t *mtl,
struct ompi_communicator_t *comm)
{
int ret;
mca_mtl_ofi_ep_type ep_type = (false == ompi_mtl_ofi.sep_supported) ?
mca_mtl_ofi_ep_type ep_type = (0 == ompi_mtl_ofi.enable_sep) ?
OFI_REGULAR_EP : OFI_SCALABLE_EP;
/*
@ -1438,7 +1438,7 @@ ompi_mtl_ofi_del_comm(struct mca_mtl_base_module_t *mtl,
struct ompi_communicator_t *comm)
{
int ret = OMPI_SUCCESS;
mca_mtl_ofi_ep_type ep_type = (false == ompi_mtl_ofi.sep_supported) ?
mca_mtl_ofi_ep_type ep_type = (0 == ompi_mtl_ofi.enable_sep) ?
OFI_REGULAR_EP : OFI_SCALABLE_EP;
/*

39
ompi/mca/mtl/ofi/mtl_ofi_component.c
Просмотреть файл

@ -472,13 +472,6 @@ static int ompi_mtl_ofi_init_sep(struct fi_info *prov)
int ret = OMPI_SUCCESS, num_ofi_ctxts;
struct fi_av_attr av_attr = {0};
ompi_mtl_ofi.max_ctx_cnt = (prov->domain_attr->max_ep_tx_ctx <
prov->domain_attr->max_ep_rx_ctx) ?
prov->domain_attr->max_ep_tx_ctx :
prov->domain_attr->max_ep_rx_ctx;
/* Provision enough contexts to service all ranks in a node */
ompi_mtl_ofi.max_ctx_cnt /= (1 + ompi_process_info.num_local_peers);
prov->ep_attr->tx_ctx_cnt = prov->ep_attr->rx_ctx_cnt =
ompi_mtl_ofi.max_ctx_cnt;
@ -600,15 +593,14 @@ static mca_mtl_base_module_t*
ompi_mtl_ofi_component_init(bool enable_progress_threads,
bool enable_mpi_threads)
{
int ret, fi_version;
int ret, fi_version, num_local_ranks;
int ofi_tag_leading_zeros, ofi_tag_bits_for_cid;
struct fi_info *hints;
struct fi_info *providers = NULL;
struct fi_info *prov = NULL;
struct fi_info *prov_cq_data = NULL;
char ep_name[FI_NAME_MAX] = {0};
size_t namelen;
int ofi_tag_leading_zeros;
int ofi_tag_bits_for_cid;
/**
* Hints to filter providers
@ -773,10 +765,10 @@ ompi_mtl_ofi_component_init(bool enable_progress_threads,
ompi_mtl_ofi.num_peers = 0;
/* Check if Scalable Endpoints can be enabled for the provider */
ompi_mtl_ofi.sep_supported = false;
ompi_mtl_ofi.enable_sep = 0;
if ((prov->domain_attr->max_ep_tx_ctx > 1) ||
(prov->domain_attr->max_ep_rx_ctx > 1)) {
ompi_mtl_ofi.sep_supported = true;
ompi_mtl_ofi.enable_sep = 1;
opal_output_verbose(1, ompi_mtl_base_framework.framework_output,
"%s:%d: Scalable EP supported in %s provider. Enabling in MTL.\n",
__FILE__, __LINE__, prov->fabric_attr->prov_name);
@ -785,7 +777,7 @@ ompi_mtl_ofi_component_init(bool enable_progress_threads,
/*
* Scalable Endpoints is required for Thread Grouping feature
*/
if (!ompi_mtl_ofi.sep_supported && ompi_mtl_ofi.thread_grouping) {
if (!ompi_mtl_ofi.enable_sep && ompi_mtl_ofi.thread_grouping) {
opal_show_help("help-mtl-ofi.txt", "SEP unavailable", true,
prov->fabric_attr->prov_name,
ompi_process_info.nodename, __FILE__, __LINE__,
@ -848,7 +840,20 @@ ompi_mtl_ofi_component_init(bool enable_progress_threads,
* vectors, completion counters or event queues etc, and enabled.
* See man fi_endpoint for more details.
*/
if (true == ompi_mtl_ofi.sep_supported) {
ompi_mtl_ofi.max_ctx_cnt = (prov->domain_attr->max_ep_tx_ctx <
prov->domain_attr->max_ep_rx_ctx) ?
prov->domain_attr->max_ep_tx_ctx :
prov->domain_attr->max_ep_rx_ctx;
num_local_ranks = 1 + ompi_process_info.num_local_peers;
if (ompi_mtl_ofi.max_ctx_cnt <= num_local_ranks) {
ompi_mtl_ofi.enable_sep = 0;
}
if (1 == ompi_mtl_ofi.enable_sep) {
/* Provision enough contexts to service all ranks in a node */
ompi_mtl_ofi.max_ctx_cnt /= num_local_ranks;
ret = ompi_mtl_ofi_init_sep(prov);
} else {
ret = ompi_mtl_ofi_init_regular_ep(prov);
@ -926,7 +931,7 @@ error:
if (ompi_mtl_ofi.av) {
(void) fi_close((fid_t)ompi_mtl_ofi.av);
}
if ((false == ompi_mtl_ofi.sep_supported) &&
if ((0 == ompi_mtl_ofi.enable_sep) &&
ompi_mtl_ofi.ofi_ctxt != NULL &&
ompi_mtl_ofi.ofi_ctxt[0].cq) {
/* Check if CQ[0] was created for non-SEP case and close if needed */
@ -964,9 +969,9 @@ ompi_mtl_ofi_finalize(struct mca_mtl_base_module_t *mtl)
goto finalize_err;
}
if (false == ompi_mtl_ofi.sep_supported) {
if (0 == ompi_mtl_ofi.enable_sep) {
/*
* CQ[0] is bound to SEP object when SEP is not supported by a
* CQ[0] is bound to SEP object Nwhen SEP is not supported by a
* provider. OFI spec requires that we close the Endpoint that is bound
* to the CQ before closing the CQ itself. So, for the non-SEP case, we
* handle the closing of CQ[0] here.

4
ompi/mca/mtl/ofi/mtl_ofi_types.h
Просмотреть файл

@ -71,8 +71,8 @@ typedef struct mca_mtl_ofi_module_t {
/* MCA parameter for Thread grouping feature */
int thread_grouping;
/* Boolen value to indicate if provider supports Scalable EP or not */
bool sep_supported;
/* If SEP is used by OFI MTL */
int enable_sep;
/* Numbers of bits used for rx contexts */
int rx_ctx_bits;