ports/openmpi
ports/openmpi
1
1
Форкнуть 0
Код Релизы Активность

btl/openib: fix regression in XRC support

Просмотр исходного кода 
Commit open-mpi/ompi@400af6c52d
introduced a regression in XRC support. The commit reversed the
ordering of shared receive queue (SRQ) and completion queue (CQ)
completion. CQ creation must always preceed SRQ creation when using
XRC as the CQs are needed to create the SRQs. This commit fixes the
ordering so that CQs are always created before SRQs.

Signed-off-by: Nathan Hjelm <hjelmn@lanl.gov>
Этот коммит содержится в:
Nathan Hjelm 2016-02-16 16:42:50 -07:00
родитель 7de01b347c
Коммит 123a39ac3c
2 изменённых файлов: 38 добавлений и 28 удалений
Показать статистику Скачать Patch файл Скачать Diff файл Показать все файлы Свернуть все файлы

63
opal/mca/btl/openib/btl_openib.c
Просмотреть файл

@ -12,7 +12,7 @@
* All rights reserved.
* Copyright (c) 2007-2013 Cisco Systems, Inc. All rights reserved.
* Copyright (c) 2006-2015 Mellanox Technologies. All rights reserved.
* Copyright (c) 2006-2015 Los Alamos National Security, LLC. All rights
* Copyright (c) 2006-2016 Los Alamos National Security, LLC. All rights
* reserved.
* Copyright (c) 2006-2007 Voltaire All rights reserved.
* Copyright (c) 2008-2012 Oracle and/or its affiliates. All rights reserved.
@ -403,6 +403,8 @@ static int create_srq(mca_btl_openib_module_t *openib_btl)
}
}
openib_btl->srqs_created = true;
return OPAL_SUCCESS;
}
@ -410,7 +412,7 @@ static int openib_btl_prepare(struct mca_btl_openib_module_t* openib_btl)
{
int rc = OPAL_SUCCESS;
opal_mutex_lock(&openib_btl->ib_lock);
if (0 == openib_btl->num_peers &&
if (!openib_btl->srqs_created &&
(mca_btl_openib_component.num_srq_qps > 0 ||
mca_btl_openib_component.num_xrc_qps > 0)) {
rc = create_srq(openib_btl);
@ -420,17 +422,12 @@ static int openib_btl_prepare(struct mca_btl_openib_module_t* openib_btl)
}
static int openib_btl_size_queues(struct mca_btl_openib_module_t* openib_btl, size_t nprocs)
static int openib_btl_size_queues(struct mca_btl_openib_module_t* openib_btl)
{
uint32_t send_cqes, recv_cqes;
int rc = OPAL_SUCCESS, qp;
mca_btl_openib_device_t *device = openib_btl->device;
if( 0 == nprocs){
/* nothing to do */
return OPAL_SUCCESS;
}
opal_mutex_lock(&openib_btl->ib_lock);
/* figure out reasonable sizes for completion queues */
for(qp = 0; qp < mca_btl_openib_component.num_qps; qp++) {
@ -439,7 +436,7 @@ static int openib_btl_size_queues(struct mca_btl_openib_module_t* openib_btl, si
recv_cqes = mca_btl_openib_component.qp_infos[qp].rd_num;
} else {
send_cqes = (mca_btl_openib_component.qp_infos[qp].rd_num +
mca_btl_openib_component.qp_infos[qp].u.pp_qp.rd_rsv) * nprocs;
mca_btl_openib_component.qp_infos[qp].u.pp_qp.rd_rsv) * openib_btl->num_peers;
recv_cqes = send_cqes;
}
@ -459,7 +456,6 @@ static int openib_btl_size_queues(struct mca_btl_openib_module_t* openib_btl, si
goto out;
}
openib_btl->num_peers += nprocs;
out:
opal_mutex_unlock(&openib_btl->ib_lock);
return rc;
@ -1032,10 +1028,14 @@ int mca_btl_openib_add_procs(
return rc;
}
rc = openib_btl_prepare(openib_btl);
if (OPAL_SUCCESS != rc) {
BTL_ERROR(("could not prepare openib btl structure for usel"));
return rc;
if (0 == openib_btl->num_peers) {
/* ensure completion queues are created before attempting to
* make a loop-back queue pair */
rc = openib_btl_size_queues(openib_btl);
if (OPAL_SUCCESS != rc) {
BTL_ERROR(("error creating cqs"));
return rc;
}
}
/* prepare all proc's and account them properly */
@ -1084,10 +1084,20 @@ int mca_btl_openib_add_procs(
}
}
/* account this procs if need */
rc = openib_btl_size_queues(openib_btl, nprocs_new);
if (nprocs_new) {
OPAL_THREAD_ADD32(&openib_btl->num_peers, nprocs_new);
/* adjust cq sizes given the new procs */
rc = openib_btl_size_queues (openib_btl);
if (OPAL_SUCCESS != rc) {
BTL_ERROR(("error creating cqs"));
return rc;
}
}
rc = openib_btl_prepare (openib_btl);
if (OPAL_SUCCESS != rc) {
BTL_ERROR(("error creating cqs"));
BTL_ERROR(("could not prepare openib btl module for use"));
return rc;
}
@ -1160,10 +1170,8 @@ struct mca_btl_base_endpoint_t *mca_btl_openib_get_ep (struct mca_btl_base_modul
{
mca_btl_openib_module_t *openib_btl = (mca_btl_openib_module_t *) btl;
volatile mca_btl_base_endpoint_t *endpoint = NULL;
int local_port_cnt = 0, btl_rank, rc;
mca_btl_openib_proc_t *ib_proc;
int rc;
int local_port_cnt = 0, btl_rank;
size_t nprocs_new = 0;
rc = prepare_device_for_use (openib_btl->device);
if (OPAL_SUCCESS != rc) {
@ -1171,12 +1179,6 @@ struct mca_btl_base_endpoint_t *mca_btl_openib_get_ep (struct mca_btl_base_modul
return NULL;
}
rc = openib_btl_prepare(openib_btl);
if (OPAL_SUCCESS != rc) {
BTL_ERROR(("could not prepare openib btl structure for use"));
return NULL;
}
if (NULL == (ib_proc = mca_btl_openib_proc_get_locked(proc))) {
/* if we don't have connection info for this process, it's
* okay because some other method might be able to reach it,
@ -1193,7 +1195,8 @@ struct mca_btl_base_endpoint_t *mca_btl_openib_get_ep (struct mca_btl_base_modul
/* this is a new process to this openib btl
* account this procs if need */
rc = openib_btl_size_queues(openib_btl, nprocs_new);
OPAL_THREAD_ADD32(&openib_btl->num_peers, 1);
rc = openib_btl_size_queues(openib_btl);
if (OPAL_SUCCESS != rc) {
BTL_ERROR(("error creating cqs"));
return NULL;
@ -1218,6 +1221,12 @@ struct mca_btl_base_endpoint_t *mca_btl_openib_get_ep (struct mca_btl_base_modul
return NULL;
}
rc = openib_btl_prepare(openib_btl);
if (OPAL_SUCCESS != rc) {
BTL_ERROR(("could not prepare openib btl structure for use"));
goto exit;
}
for (size_t j = 0 ; j < ib_proc->proc_endpoint_count ; ++j) {
endpoint = ib_proc->proc_endpoints[j];
if (endpoint->endpoint_btl == openib_btl) {

3
opal/mca/btl/openib/btl_openib.h
Просмотреть файл

@ -12,7 +12,7 @@
* All rights reserved.
* Copyright (c) 2006-2011 Cisco Systems, Inc. All rights reserved.
* Copyright (c) 2006-2009 Mellanox Technologies. All rights reserved.
* Copyright (c) 2006-2015 Los Alamos National Security, LLC. All rights
* Copyright (c) 2006-2016 Los Alamos National Security, LLC. All rights
* reserved.
* Copyright (c) 2006-2007 Voltaire All rights reserved.
* Copyright (c) 2009-2010 Oracle and/or its affiliates. All rights reserved.
@ -458,6 +458,7 @@ struct mca_btl_openib_module_t {
mca_btl_base_module_t super;
bool btl_inited;
bool srqs_created;
/** Common information about all ports */
mca_btl_openib_modex_message_t port_info;