1
1

Fixes trac:3152: slightly more advanced than the patch on the ticket:

* If the MCA param btl_openib_cq_size is set to 0 (which is the
   default), use the device CQ max size. Otherwise, use the MCA param
   value (and never adjust it again).
 * Remove the CQ size adjustment code. Since we default to max CQ
   size, there really isn't much point in having it any more. I think
   people setting an absolute CQ size is going to be rare, so let's
   not do anything fancy with it.
 * If the MCA param value is larger than what the device supports,
   print a warning (only once per process) and default to using the
   device max
 * Add a BTL_VERBOSE displaying which CQ size we used

This commit was SVN r26730.

The following Trac tickets were found above:
  Ticket 3152 --> https://svn.open-mpi.org/trac/ompi/ticket/3152
Этот коммит содержится в:
Jeff Squyres 2012-07-03 16:49:59 +00:00
родитель 1244f1f93a
Коммит 5315c91baf
3 изменённых файлов: 56 добавлений и 47 удалений

Просмотреть файл

@ -165,27 +165,52 @@ static inline struct ibv_cq *create_cq_compat(struct ibv_context *context,
#endif
}
static int adjust_cq(mca_btl_openib_device_t *device, const int cq)
static int create_cq(mca_btl_openib_device_t *device, const int cq)
{
uint32_t cq_size = device->cq_size[cq];
if (NULL == device->ib_cq[cq]) {
uint32_t cq_size = (uint32_t) device->ib_dev_attr.max_cqe;
/* make sure we don't exceed the maximum CQ size and that we
* don't size the queue smaller than otherwise requested
*/
if(cq_size < mca_btl_openib_component.ib_cq_size[cq])
cq_size = mca_btl_openib_component.ib_cq_size[cq];
/* Was the MCA param set? */
if (mca_btl_openib_component.ib_cq_size[cq] > 0) {
if(cq_size > (uint32_t)device->ib_dev_attr.max_cqe)
cq_size = device->ib_dev_attr.max_cqe;
/* If the MCA param is <= the max device CQ size, use it.
Otherwise, warn and keep it set to the device max. */
if (mca_btl_openib_component.ib_cq_size[cq] <=
device->ib_dev_attr.max_cqe) {
cq_size = mca_btl_openib_component.ib_cq_size[cq];
} else {
static bool showed_already = false;
if(NULL == device->ib_cq[cq]) {
device->ib_cq[cq] = create_cq_compat(device->ib_dev_context, cq_size,
/* Since we usually create multiple CQs in each
process (on each device), put some protection to
only show this message once per process. */
if (!showed_already) {
orte_show_help("help-mpi-btl-openib.txt",
"cq length too large", true,
orte_process_info.nodename,
ibv_get_device_name(device->ib_dev),
device->ib_dev_attr.vendor_id,
device->ib_dev_attr.vendor_part_id,
mca_btl_openib_component.ib_cq_size[cq],
device->ib_dev_attr.max_cqe);
}
showed_already = true;
/* No need to adjust cq_size -- it's already set to
the device max */
}
}
BTL_VERBOSE(("creating CQ with %u CQ entries (max supported: %u)\n",
cq_size, device->ib_dev_attr.max_cqe));
device->ib_cq[cq] = create_cq_compat(device->ib_dev_context,
cq_size,
#if OMPI_ENABLE_PROGRESS_THREADS == 1
device, device->ib_channel,
device, device->ib_channel,
#else
NULL, NULL,
NULL, NULL,
#endif
0);
0);
if (NULL == device->ib_cq[cq]) {
mca_btl_openib_show_init_error(__FILE__, __LINE__, "ibv_create_cq",
@ -213,18 +238,6 @@ static int adjust_cq(mca_btl_openib_device_t *device, const int cq)
OPAL_THREAD_UNLOCK(&device->device_lock);
#endif
}
#ifdef HAVE_IBV_RESIZE_CQ
else if (cq_size > mca_btl_openib_component.ib_cq_size[cq]){
int rc;
rc = ibv_resize_cq(device->ib_cq[cq], cq_size);
/* For ConnectX the resize CQ is not implemented and verbs returns -ENOSYS
* but should return ENOSYS. So it is reason for abs */
if(rc && ENOSYS != abs(rc)) {
BTL_ERROR(("cannot resize completion queue, error: %d", rc));
return OMPI_ERROR;
}
}
#endif
return OMPI_SUCCESS;
}
@ -389,26 +402,12 @@ static int mca_btl_openib_size_queues(struct mca_btl_openib_module_t* openib_btl
int rc = OMPI_SUCCESS, qp;
mca_btl_openib_device_t *device = openib_btl->device;
/* figure out reasonable sizes for completion queues */
for(qp = 0; qp < mca_btl_openib_component.num_qps; qp++) {
if(BTL_OPENIB_QP_TYPE_SRQ(qp)) {
send_cqes = mca_btl_openib_component.qp_infos[qp].u.srq_qp.sd_max;
recv_cqes = mca_btl_openib_component.qp_infos[qp].rd_num;
} else {
send_cqes = (mca_btl_openib_component.qp_infos[qp].rd_num +
mca_btl_openib_component.qp_infos[qp].u.pp_qp.rd_rsv) * nprocs;
recv_cqes = send_cqes;
}
openib_btl->device->cq_size[qp_cq_prio(qp)] += recv_cqes;
openib_btl->device->cq_size[BTL_OPENIB_LP_CQ] += send_cqes;
}
rc = adjust_cq(device, BTL_OPENIB_HP_CQ);
rc = create_cq(device, BTL_OPENIB_HP_CQ);
if (OMPI_SUCCESS != rc) {
goto out;
}
rc = adjust_cq(device, BTL_OPENIB_LP_CQ);
rc = create_cq(device, BTL_OPENIB_LP_CQ);
if (OMPI_SUCCESS != rc) {
goto out;
}

Просмотреть файл

@ -248,11 +248,9 @@ int btl_openib_register_mca_params(void)
REGINT_GE_ONE));
CHECK(reg_int("cq_size", "ib_cq_size",
"Minimum size of the OpenFabrics completion queue "
"(CQs are automatically sized based on the number "
"of peer MPI processes; this value determines the "
"*minimum* size of all CQs)",
1000, &ival, REGINT_GE_ONE));
"Size of the OpenFabrics completion queue. "
"0 means to use the device's max CQ size.",
0, &ival, REGINT_GE_ZERO));
mca_btl_openib_component.ib_cq_size[BTL_OPENIB_LP_CQ] =
mca_btl_openib_component.ib_cq_size[BTL_OPENIB_HP_CQ] = (uint32_t) ival;

Просмотреть файл

@ -689,3 +689,15 @@ device).
Use "ibv_devinfo -v" on the local host to see the GID table of this
device.
#
[cq length too large]
The mca_btl_openib_cq_size MCA parameter was set to a value larger
than what a local device supports. You can use the command
"ibv_devinfo -v | grep max_cqe" to find out a device's max CQ length.
Open MPI will default to using the device's max CQ length.
Local host: %s
Local adapter: %s (vendor 0x%x, part ID %d)
Requested CQ length: %u
Max supported CQ length: %u