ports/openmpi
ports/openmpi
1
1
Форкнуть 0
Код Релизы Активность

More common/verbs improvements:

Просмотр исходного кода 
* Add OMPI_COMMON_VERBS_FLAGS_NOT_RC, which looks for a device	that
   does ''not'' support RC
 * Add ompi_common_verbs_find_max_inline(), and	remove that code from
   the openib BTL component

This commit was SVN r27393.
Этот коммит содержится в:
Jeff Squyres 2012-10-03 00:57:39 +00:00
родитель 82171eef9d
Коммит 8c369224bf
5 изменённых файлов: 137 добавлений и 64 удалений
Показать статистику Скачать Patch файл Скачать Diff файл Показать все файлы Свернуть все файлы

65
ompi/mca/btl/openib/btl_openib_component.c
Просмотреть файл

@ -1756,66 +1756,13 @@ static int init_one_device(opal_list_t *btl_list, struct ibv_device* ib_dev)
}
}
}
/* Horrible. :-( Per the thread starting here:
http://lists.openfabrics.org/pipermail/general/2008-June/051822.html,
we can't rely on the value reported by the device to determine
the maximum max_inline_data value. So we have to search by
looping over max_inline_data values and trying to make dummy
QPs. Yuck! */
/* If we don't have a set max inline data size, search for it */
if (need_search) {
struct ibv_qp *qp;
struct ibv_cq *cq;
struct ibv_qp_init_attr init_attr;
uint32_t max_inline_data;
/* Make a dummy CQ */
#if OMPI_IBV_CREATE_CQ_ARGS == 3
cq = ibv_create_cq(device->ib_dev_context, 1, NULL);
#else
cq = ibv_create_cq(device->ib_dev_context, 1, NULL, NULL, 0);
#endif
if (NULL == cq) {
orte_show_help("help-mpi-btl-openib.txt", "init-fail-create-q",
true, orte_process_info.nodename,
__FILE__, __LINE__, "ibv_create_cq",
strerror(errno), errno,
ibv_get_device_name(device->ib_dev));
ret = OMPI_ERR_NOT_AVAILABLE;
goto error;
}
/* Setup the QP attributes */
memset(&init_attr, 0, sizeof(init_attr));
init_attr.qp_type = IBV_QPT_RC;
init_attr.send_cq = cq;
init_attr.recv_cq = cq;
init_attr.srq = 0;
init_attr.cap.max_send_sge = 1;
init_attr.cap.max_recv_sge = 1;
init_attr.cap.max_recv_wr = 1;
/* Loop over max_inline_data values; just check powers of 2 --
that's good enough */
init_attr.cap.max_inline_data = max_inline_data = 1 << 20;
while (max_inline_data > 0) {
qp = ibv_create_qp(device->ib_pd, &init_attr);
if (NULL != qp) {
break;
}
max_inline_data >>= 1;
init_attr.cap.max_inline_data = max_inline_data;
}
/* Did we find it? */
if (NULL != qp) {
device->max_inline_data = max_inline_data;
ibv_destroy_qp(qp);
} else {
device->max_inline_data = 0;
}
/* Destroy the temp CQ */
ibv_destroy_cq(cq);
ompi_common_verbs_find_max_inline(device->ib_dev,
device->ib_dev_context,
device->ib_pd,
&device->max_inline_data);
}
/* Should we use RDMA for short / eager messages? First check MCA

1
ompi/mca/common/verbs/Makefile.am
Просмотреть файл

@ -17,6 +17,7 @@ headers = \
sources = \
common_verbs_basics.c \
common_verbs_devlist.c \
common_verbs_find_max_inline.c \
common_verbs_find_ports.c \
common_verbs_mca.c \
common_verbs_port.c \

20
ompi/mca/common/verbs/common_verbs.h
Просмотреть файл

@ -75,14 +75,15 @@ OBJ_CLASS_DECLARATION(ompi_common_verbs_port_item_t);
enum {
OMPI_COMMON_VERBS_FLAGS_RC = 0x1,
OMPI_COMMON_VERBS_FLAGS_UD = 0x2,
OMPI_COMMON_VERBS_FLAGS_TRANSPORT_IB = 0x4,
OMPI_COMMON_VERBS_FLAGS_TRANSPORT_IWARP = 0x8,
OMPI_COMMON_VERBS_FLAGS_NOT_RC = 0x2,
OMPI_COMMON_VERBS_FLAGS_UD = 0x4,
OMPI_COMMON_VERBS_FLAGS_TRANSPORT_IB = 0x8,
OMPI_COMMON_VERBS_FLAGS_TRANSPORT_IWARP = 0x10,
/* Note that these 2 link layer flags will only be useful if
defined(HAVE_IBV_LINK_LAYER_ETHERNET). Otherwise, they will be
ignored. */
OMPI_COMMON_VERBS_FLAGS_LINK_LAYER_IB = 0x10,
OMPI_COMMON_VERBS_FLAGS_LINK_LAYER_ETHERNET = 0x20,
OMPI_COMMON_VERBS_FLAGS_LINK_LAYER_IB = 0x20,
OMPI_COMMON_VERBS_FLAGS_LINK_LAYER_ETHERNET = 0x40,
OMPI_COMMON_VERBS_FLAGS_MAX
};
@ -140,6 +141,15 @@ ompi_common_verbs_port_bw(struct ibv_port_attr *port_attr,
OMPI_DECLSPEC int
ompi_common_verbs_mtu(struct ibv_port_attr *port_attr);
/*
* Find the max_inline_data value for a given device
*/
OMPI_DECLSPEC int
ompi_common_verbs_find_max_inline(struct ibv_device *device,
struct ibv_context *context,
struct ibv_pd *pd,
uint32_t *max_inline_arg);
/*
* Test a device to see if it can handle a specific QP type (RC and/or
* UD). Will return the logical AND if multiple types are specified

110
ompi/mca/common/verbs/common_verbs_find_max_inline.c Обычный файл
Просмотреть файл

@ -0,0 +1,110 @@
/*
* Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
* University Research and Technology
* Corporation. All rights reserved.
* Copyright (c) 2004-2008 The University of Tennessee and The University
* of Tennessee Research Foundation. All rights
* reserved.
* Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
* University of Stuttgart. All rights reserved.
* Copyright (c) 2004-2005 The Regents of the University of California.
* All rights reserved.
* Copyright (c) 2006-2012 Cisco Systems, Inc. All rights reserved.
* Copyright (c) 2006-2012 Mellanox Technologies. All rights reserved.
* Copyright (c) 2006-2007 Los Alamos National Security, LLC. All rights
* reserved.
* Copyright (c) 2006-2007 Voltaire All rights reserved.
* Copyright (c) 2009 Sun Microsystems, Inc. All rights reserved.
* Copyright (c) 2009-2012 Oak Ridge National Laboratory. All rights reserved.
* $COPYRIGHT$
*
* Additional copyrights may follow
*
* $HEADER$
*/
#include "ompi_config.h"
#include <stdio.h>
#include <string.h>
#include <infiniband/verbs.h>
#include <stdint.h>
#include "opal_stdint.h"
#include "opal/types.h"
#include "opal/util/output.h"
#include "opal/util/argv.h"
#include "opal/class/opal_object.h"
#include "orte/util/show_help.h"
#include "orte/util/proc_info.h"
#include "orte/runtime/orte_globals.h"
#include "ompi/constants.h"
#include "common_verbs.h"
/* Horrible. :-( Per the thread starting here:
http://lists.openfabrics.org/pipermail/general/2008-June/051822.html,
we can't rely on the value reported by the device to determine the
maximum max_inline_data value. So we have to search by looping
over max_inline_data values and trying to make dummy QPs. Yuck! */
int ompi_common_verbs_find_max_inline(struct ibv_device *device,
struct ibv_context *context,
struct ibv_pd *pd,
uint32_t *max_inline_arg)
{
int ret;
struct ibv_qp *qp;
struct ibv_cq *cq;
struct ibv_qp_init_attr init_attr;
uint32_t max_inline_data;
*max_inline_arg = 0;
/* Make a dummy CQ */
#if OMPI_IBV_CREATE_CQ_ARGS == 3
cq = ibv_create_cq(context, 1, NULL);
#else
cq = ibv_create_cq(context, 1, NULL, NULL, 0);
#endif
if (NULL == cq) {
orte_show_help("help-mpi-btl-openib.txt", "init-fail-create-q",
true, orte_process_info.nodename,
__FILE__, __LINE__, "ibv_create_cq",
strerror(errno), errno,
ibv_get_device_name(device));
return OMPI_ERR_NOT_AVAILABLE;
}
/* Setup the QP attributes */
memset(&init_attr, 0, sizeof(init_attr));
init_attr.qp_type = IBV_QPT_RC;
init_attr.send_cq = cq;
init_attr.recv_cq = cq;
init_attr.srq = 0;
init_attr.cap.max_send_sge = 1;
init_attr.cap.max_recv_sge = 1;
init_attr.cap.max_recv_wr = 1;
/* Loop over max_inline_data values; just check powers of 2 --
that's good enough */
init_attr.cap.max_inline_data = max_inline_data = 1 << 20;
ret = OMPI_ERR_NOT_FOUND;
while (max_inline_data > 0) {
qp = ibv_create_qp(pd, &init_attr);
if (NULL != qp) {
*max_inline_arg = max_inline_data;
ibv_destroy_qp(qp);
ret = OMPI_SUCCESS;
break;
}
max_inline_data >>= 1;
init_attr.cap.max_inline_data = max_inline_data;
}
/* Destroy the temp CQ */
ibv_destroy_cq(cq);
return ret;
}

5
ompi/mca/common/verbs/common_verbs_qp_type.c
Просмотреть файл

@ -83,6 +83,11 @@ int ompi_common_verbs_qp_test(struct ibv_context *device_context, int flags)
rc = OMPI_ERR_NOT_SUPPORTED;
goto out;
}
if (flags & OMPI_COMMON_VERBS_FLAGS_NOT_RC &&
make_qp(pd, cq, IBV_QPT_RC)) {
rc = OMPI_ERR_TYPE_MISMATCH;
goto out;
}
if (flags & OMPI_COMMON_VERBS_FLAGS_UD &&
!make_qp(pd, cq, IBV_QPT_UD)) {
rc = OMPI_ERR_NOT_SUPPORTED;