ports/openmpi
ports/openmpi
1
1
Форкнуть 0
Код Релизы Активность

opal/common: remove stale common components

Просмотр исходного кода 
The verbs and verbs_usnic components are now no longer necessary / no
longer used anywhere in the code base.

Signed-off-by: Jeff Squyres <jsquyres@cisco.com>
Этот коммит содержится в:
Jeff Squyres 2019-01-10 09:38:14 -08:00
родитель 3e82449dbe
Коммит 3f4af8e51c
19 изменённых файлов: 0 добавлений и 1782 удалений
Показать статистику Скачать Patch файл Скачать Diff файл Показать все файлы Свернуть все файлы

1
VERSION
Просмотреть файл

@ -114,4 +114,3 @@ libmca_opal_common_ofi_so_version=0:0:0
libmca_opal_common_sm_so_version=0:0:0
libmca_opal_common_ucx_so_version=0:0:0
libmca_opal_common_ugni_so_version=0:0:0
libmca_opal_common_verbs_so_version=0:0:0

1
configure.ac
Просмотреть файл

@ -158,7 +158,6 @@ AC_SUBST(libmca_opal_common_ofi_so_version)
AC_SUBST(libmca_opal_common_cuda_so_version)
AC_SUBST(libmca_opal_common_sm_so_version)
AC_SUBST(libmca_opal_common_ugni_so_version)
AC_SUBST(libmca_opal_common_verbs_so_version)
AC_SUBST(libmca_orte_common_alps_so_version)
AC_SUBST(libmca_ompi_common_ompio_so_version)
AC_SUBST(libmca_ompi_common_monitoring_so_version)

83
opal/mca/common/verbs/Makefile.am
Просмотреть файл

@ -1,83 +0,0 @@
#
# Copyright (c) 2009-2012 Mellanox Technologies. All rights reserved.
# Copyright (c) 2009-2012 Oak Ridge National Laboratory. All rights reserved.
# Copyright (c) 2012-2015 Cisco Systems, Inc. All rights reserved.
# $COPYRIGHT$
#
# Additional copyrights may follow
#
# $HEADER$
#
AM_CPPFLAGS = $(common_verbs_CPPFLAGS)
headers = \
common_verbs.h
sources = \
common_verbs_basics.c \
common_verbs_devlist.c \
common_verbs_find_max_inline.c \
common_verbs_find_ports.c \
common_verbs_mca.c \
common_verbs_port.c \
common_verbs_qp_type.c
dist_opaldata_DATA = \
help-opal-common-verbs.txt
# To simplify components that link to this library, we will *always*
# have an output libtool library named libmca_<type>_<name>.la -- even
# for case 2) described above (i.e., so there's no conditional logic
# necessary in component Makefile.am's that link to this library).
# Hence, if we're creating a noinst version of this library (i.e.,
# case 2), we sym link it to the libmca_<type>_<name>.la name
# (libtool will do the Right Things under the covers). See the
# all-local and clean-local rules, below, for how this is effected.
lib_LTLIBRARIES =
noinst_LTLIBRARIES =
comp_inst = lib@OPAL_LIB_PREFIX@mca_common_verbs.la
comp_noinst = lib@OPAL_LIB_PREFIX@mca_common_verbs_noinst.la
if MCA_BUILD_opal_common_verbs_DSO
lib_LTLIBRARIES += $(comp_inst)
else
noinst_LTLIBRARIES += $(comp_noinst)
endif
lib@OPAL_LIB_PREFIX@mca_common_verbs_la_SOURCES = $(headers) $(sources)
lib@OPAL_LIB_PREFIX@mca_common_verbs_la_CPPFLAGS = $(common_verbs_CPPFLAGS)
lib@OPAL_LIB_PREFIX@mca_common_verbs_la_LDFLAGS = \
-version-info $(libmca_opal_common_verbs_so_version) \
$(common_verbs_LDFLAGS)
lib@OPAL_LIB_PREFIX@mca_common_verbs_la_LIBADD = $(common_verbs_LIBS)
lib@OPAL_LIB_PREFIX@mca_common_verbs_noinst_la_SOURCES = $(headers) $(sources)
# Conditionally install the header files
if WANT_INSTALL_HEADERS
opaldir = $(opalincludedir)/opal/mca/common/verbs
opal_HEADERS = $(headers)
else
opaldir = $(includedir)
endif
# These two rules will sym link the "noinst" libtool library filename
# to the installable libtool library filename in the case where we are
# compiling this component statically (case 2), described above).
V=0
OMPI_V_LN_SCOMP = $(ompi__v_LN_SCOMP_$V)
ompi__v_LN_SCOMP_ = $(ompi__v_LN_SCOMP_$AM_DEFAULT_VERBOSITY)
ompi__v_LN_SCOMP_0 = @echo " LN_S " `basename $(comp_inst)`;
all-local:
$(OMPI_V_LN_SCOMP) if test -z "$(lib_LTLIBRARIES)"; then \
rm -f "$(comp_inst)"; \
$(LN_S) "$(comp_noinst)" "$(comp_inst)"; \
fi
clean-local:
if test -z "$(lib_LTLIBRARIES)"; then \
rm -f "$(comp_inst)"; \
fi

186
opal/mca/common/verbs/common_verbs.h
Просмотреть файл

@ -1,186 +0,0 @@
/*
* Copyright (c) 2009-2012 Mellanox Technologies. All rights reserved.
* All rights reserved.
* Copyright (c) 2009-2012 Oak Ridge National Laboratory. All rights reserved.
* Copyright (c) 2012-2015 Cisco Systems, Inc. All rights reserved.
* Copyright (c) 2014 The University of Tennessee and The University
* of Tennessee Research Foundation. All rights
* reserved.
* $COPYRIGHT$
*
* Additional copyrights may follow
*
* $HEADER$
*/
#ifndef _COMMON_OFAUTILS_H_
#define _COMMON_OFAUTILS_H_
#include "opal_config.h"
#include <stdint.h>
#include <infiniband/verbs.h>
#include "opal/mca/mca.h"
#include <infiniband/verbs.h>
#include "opal/class/opal_list.h"
BEGIN_C_DECLS
/*
* common_verbs_devlist.c
*/
OPAL_DECLSPEC struct ibv_device **opal_ibv_get_device_list(int *num_devs);
OPAL_DECLSPEC void opal_ibv_free_device_list(struct ibv_device **ib_devs);
/*
* common_verbs_mca.c
*/
extern bool opal_common_verbs_warn_nonexistent_if;
extern int opal_common_verbs_want_fork_support;
OPAL_DECLSPEC void opal_common_verbs_mca_register(mca_base_component_t *component);
/*
* common_verbs_basics.c
*/
bool opal_common_verbs_check_basics(void);
/*
* common_verbs_find_ports.c
*/
typedef struct opal_common_verbs_device_item_t {
opal_object_t super;
struct ibv_device *device;
char *device_name;
struct ibv_context *context;
struct ibv_device_attr device_attr;
/** This field defaults to true, meaning that the destructor for
opal_common_verbs_device_item_t will invoke ibv_close_device()
on the context. An upper layer can reset this field to false,
however, indicating that the destructor should *not* invoke
ibv_close_device() (e.g., if the upper layer has copied the
context and is using it). */
bool destructor_free_context;
} opal_common_verbs_device_item_t;
OBJ_CLASS_DECLARATION(opal_common_verbs_device_item_t);
typedef struct opal_common_verbs_port_item_t {
opal_list_item_t super;
opal_common_verbs_device_item_t *device;
uint8_t port_num;
struct ibv_port_attr port_attr;
} opal_common_verbs_port_item_t;
OBJ_CLASS_DECLARATION(opal_common_verbs_port_item_t);
enum {
OPAL_COMMON_VERBS_FLAGS_RC = 0x1,
OPAL_COMMON_VERBS_FLAGS_NOT_RC = 0x2,
OPAL_COMMON_VERBS_FLAGS_UD = 0x4,
OPAL_COMMON_VERBS_FLAGS_TRANSPORT_IB = 0x8,
OPAL_COMMON_VERBS_FLAGS_TRANSPORT_IWARP = 0x10,
/* Note that these 2 link layer flags will only be useful if
defined(HAVE_IBV_LINK_LAYER_ETHERNET). Otherwise, they will be
ignored. */
OPAL_COMMON_VERBS_FLAGS_LINK_LAYER_IB = 0x80,
OPAL_COMMON_VERBS_FLAGS_LINK_LAYER_ETHERNET = 0x100,
OPAL_COMMON_VERBS_FLAGS_MAX
};
/**
* Find a list of ibv_device ports that match a specific criteria.
*
* @param if_include (IN): comma-delimited list of interfaces to use
* @param if_exclude (IN): comma-delimited list of interfaces to NOT use
* @param flags (IN): bit flags
* @param verbose_stream (IN): stream to send opal_output_verbose messages to
*
* The ports will adhere to the if_include / if_exclude lists (only
* one can be specified). The lists are comma-delimited tokens in one
* of two forms:
*
* interface_name
* interface_name:port
*
* Hence, a if_include list could be the follwing: "mlx4_0,mthca0:1".
*
* The flags provide logical OR behavior -- a port will be included if
* it includes any of the capabilities/characteristics listed in the
* flags.
*
* Note that if the verbose_stream is >=0, output will be sent to that
* stream with a verbose level of 5.
*
* A valid list will always be returned. It will contain zero or more
* opal_common_verbs_port_item_t items. Each item can be individually
* OBJ_RELEASE'd; the destructor will take care of cleaning up the
* linked opal_common_verbs_device_item_t properly (i.e., when all
* port_items referring to it have been freed).
*/
OPAL_DECLSPEC opal_list_t *
opal_common_verbs_find_ports(const char *if_include,
const char *if_exclude,
int flags,
int verbose_stream);
/*
* Trivial function to compute the bandwidth on an ibv_port.
*
* Will return OPAL_ERR_NOT_FOUND if it can't figure out the bandwidth
* (and the bandwidth parameter value will be undefined). Otherwise,
* will return OPAL_SUCCESS and set bandwidth to an appropriate value.
*/
OPAL_DECLSPEC int
opal_common_verbs_port_bw(struct ibv_port_attr *port_attr,
uint32_t *bandwidth);
/*
* Trivial function to switch on the verbs MTU enum and return a
* numeric value.
*/
OPAL_DECLSPEC int
opal_common_verbs_mtu(struct ibv_port_attr *port_attr);
/*
* Find the max_inline_data value for a given device
*/
OPAL_DECLSPEC int
opal_common_verbs_find_max_inline(struct ibv_device *device,
struct ibv_context *context,
struct ibv_pd *pd,
uint32_t *max_inline_arg);
/*
* Test a device to see if it can handle a specific QP type (RC and/or
* UD). Will return the logical AND if multiple types are specified
* (e.g., if (RC|UD) are in flags, then will return OPAL_SUCCESS only
* if *both* types can be created on the device).
*
* Flags can be the logical OR of OPAL_COMMON_VERBS_FLAGS_RC and/or
* OPAL_COMMON_VERBS_FLAGS_UD. All other values are ignored.
*/
OPAL_DECLSPEC int opal_common_verbs_qp_test(struct ibv_context *device_context,
int flags);
/*
* ibv_fork_init testing - if fork support is requested then ibv_fork_init
* should be called right at the beginning of the verbs initialization flow, before ibv_create_* call.
*
* Known limitations:
* If ibv_fork_init is called after ibv_create_* functions - it will have no effect.
* OMPI initializes verbs many times during initialization in the following verbs components:
* oob/ud, btl/openib, mtl/mxm, pml/yalla, oshmem/ikrit, ompi/mca/coll/{fca,hcoll}
*
* So, ibv_fork_init should be called once, in the beginning of the init flow of every verb component
* to proper request fork support.
*
*/
int opal_common_verbs_fork_test(void);
END_C_DECLS
#endif

109
opal/mca/common/verbs/common_verbs_basics.c
Просмотреть файл

@ -1,109 +0,0 @@
/*
* Copyright (c) 2012-2016 Cisco Systems, Inc. All rights reserved.
*
* Copyright (c) 2018 Amazon.com, Inc. or its affiliates. All Rights reserved.
* $COPYRIGHT$
*
* Additional copyrights may follow
*
* $HEADER$
*/
#include "opal_config.h"
#include <stdio.h>
#ifdef HAVE_SYS_TYPES_H
#include <sys/types.h>
#endif
#ifdef HAVE_SYS_STAT_H
#include <sys/stat.h>
#endif
#ifdef HAVE_UNISTD_H
#include <unistd.h>
#endif
#if OPAL_COMMON_VERBS_USNIC_HAPPY
#include "opal/mca/common/verbs_usnic/common_verbs_usnic.h"
#endif
/* This is crummy, but <infiniband/driver.h> doesn't work on all
platforms with all compilers. Specifically, trying to include it
on RHEL4U3 with the PGI 32 bit compiler will cause problems because
certain 64 bit types are not defined. Per advice from Roland D.,
just include the one prototype that we need in this case
(ibv_get_sysfs_path()). */
#include <infiniband/verbs.h>
#ifdef HAVE_INFINIBAND_DRIVER_H
#include <infiniband/driver.h>
#else
const char *ibv_get_sysfs_path(void);
#endif
#include "common_verbs.h"
#include "opal/runtime/opal_params.h"
#include "opal/util/show_help.h"
#include "opal/util/proc.h"
#include "opal/util/printf.h"
/***********************************************************************/
bool opal_common_verbs_check_basics(void)
{
#if defined(__linux__)
int rc;
char *file;
struct stat s;
/* Check to see if $sysfsdir/class/infiniband/ exists */
opal_asprintf(&file, "%s/class/infiniband", ibv_get_sysfs_path());
if (NULL == file) {
return false;
}
rc = stat(file, &s);
free(file);
if (0 != rc || !S_ISDIR(s.st_mode)) {
return false;
}
#endif
/* It exists and is a directory -- good enough */
return true;
}
int opal_common_verbs_fork_test(void)
{
int ret = OPAL_SUCCESS;
/* Make sure that ibv_fork_init() is the first ibv_* function to
be invoked in this process. */
#ifdef HAVE_IBV_FORK_INIT
if (0 != opal_common_verbs_want_fork_support) {
/* Check if fork support is requested by the user */
if (0 != ibv_fork_init()) {
/* If the opal_common_verbs_want_fork_support MCA
* parameter is >0 but the call to ibv_fork_init() failed,
* then return an error code.
*/
if (opal_common_verbs_want_fork_support > 0) {
opal_show_help("help-opal-common-verbs.txt",
"ibv_fork_init fail", true,
opal_proc_local_get()->proc_hostname, errno,
strerror(errno));
ret = OPAL_ERROR;
}
}
}
#endif
#if OPAL_COMMON_VERBS_USNIC_HAPPY
/* Now register any necessary fake libibverbs drivers. We
piggyback loading these fake drivers on the fork test because
they must be loaded before ibv_get_device_list() is invoked.
Note that this routine is in a different common component (see
comments over there for an explanation why). */
opal_common_verbs_usnic_register_fake_drivers();
#endif
return ret;
}

95
opal/mca/common/verbs/common_verbs_devlist.c
Просмотреть файл

@ -1,95 +0,0 @@
/* -*- Mode: C; c-basic-offset:4 ; -*- */
/*
* Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
* University Research and Technology
* Corporation. All rights reserved.
* Copyright (c) 2004-2008 The University of Tennessee and The University
* of Tennessee Research Foundation. All rights
* reserved.
* Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
* University of Stuttgart. All rights reserved.
* Copyright (c) 2004-2005 The Regents of the University of California.
* All rights reserved.
* Copyright (c) 2006-2012 Cisco Systems, Inc. All rights reserved.
* Copyright (c) 2006-2012 Mellanox Technologies. All rights reserved.
* Copyright (c) 2006-2007 Los Alamos National Security, LLC. All rights
* reserved.
* Copyright (c) 2006-2007 Voltaire All rights reserved.
* Copyright (c) 2009 Sun Microsystems, Inc. All rights reserved.
* Copyright (c) 2009-2012 Oak Ridge National Laboratory. All rights reserved.
* $COPYRIGHT$
*
* Additional copyrights may follow
*
* $HEADER$
*/
#include "opal_config.h"
#include <infiniband/verbs.h>
/* This is crummy, but <infiniband/driver.h> doesn't work on all
platforms with all compilers. Specifically, trying to include it
on RHEL4U3 with the PGI 32 bit compiler will cause problems because
certain 64 bit types are not defined. Per advice from Roland D.,
just include the one prototype that we need in this case
(ibv_get_sysfs_path()). */
#ifdef HAVE_INFINIBAND_DRIVER_H
#include <infiniband/driver.h>
#else
const char *ibv_get_sysfs_path(void);
#endif
#include "opal/util/output.h"
#include "common_verbs.h"
/*
* Portable wrapper around ibv_get_device_list() / ibv_get_devices().
*/
struct ibv_device **opal_ibv_get_device_list(int *num_devs)
{
struct ibv_device **ib_devs;
#ifdef HAVE_IBV_GET_DEVICE_LIST
ib_devs = ibv_get_device_list(num_devs);
#else
struct dlist *dev_list;
struct ibv_device *ib_dev;
*num_devs = 0;
/* Determine the number of device's available on the host */
dev_list = ibv_get_devices();
if (NULL == dev_list) {
return NULL;
}
dlist_start(dev_list);
dlist_for_each_data(dev_list, ib_dev, struct ibv_device)
(*num_devs)++;
/* Allocate space for the ib devices */
ib_devs = (struct ibv_device**)malloc(*num_devs * sizeof(struct ibv_dev*));
if (NULL == ib_devs) {
*num_devs = 0;
opal_output(0, "Failed malloc: %s:%d", __FILE__, __LINE__);
return NULL;
}
dlist_start(dev_list);
dlist_for_each_data(dev_list, ib_dev, struct ibv_device)
*(++ib_devs) = ib_dev;
#endif
return ib_devs;
}
void opal_ibv_free_device_list(struct ibv_device **ib_devs)
{
#ifdef HAVE_IBV_GET_DEVICE_LIST
ibv_free_device_list(ib_devs);
#else
free(ib_devs);
#endif
}

108
opal/mca/common/verbs/common_verbs_find_max_inline.c
Просмотреть файл

@ -1,108 +0,0 @@
/*
* Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
* University Research and Technology
* Corporation. All rights reserved.
* Copyright (c) 2004-2008 The University of Tennessee and The University
* of Tennessee Research Foundation. All rights
* reserved.
* Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
* University of Stuttgart. All rights reserved.
* Copyright (c) 2004-2005 The Regents of the University of California.
* All rights reserved.
* Copyright (c) 2006-2013 Cisco Systems, Inc. All rights reserved.
* Copyright (c) 2006-2012 Mellanox Technologies. All rights reserved.
* Copyright (c) 2006-2007 Los Alamos National Security, LLC. All rights
* reserved.
* Copyright (c) 2006-2007 Voltaire All rights reserved.
* Copyright (c) 2009 Sun Microsystems, Inc. All rights reserved.
* Copyright (c) 2009-2012 Oak Ridge National Laboratory. All rights reserved.
* $COPYRIGHT$
*
* Additional copyrights may follow
*
* $HEADER$
*/
#include "opal_config.h"
#include <stdio.h>
#include <string.h>
#include <infiniband/verbs.h>
#include <stdint.h>
#include "opal_stdint.h"
#include "opal/types.h"
#include "opal/util/output.h"
#include "opal/util/argv.h"
#include "opal/class/opal_object.h"
#include "opal/util/show_help.h"
#include "opal/util/proc.h"
#include "opal/constants.h"
#include "common_verbs.h"
/* Horrible. :-( Per the thread starting here:
http://lists.openfabrics.org/pipermail/general/2008-June/051822.html,
we can't rely on the value reported by the device to determine the
maximum max_inline_data value. So we have to search by looping
over max_inline_data values and trying to make dummy QPs. Yuck! */
int opal_common_verbs_find_max_inline(struct ibv_device *device,
struct ibv_context *context,
struct ibv_pd *pd,
uint32_t *max_inline_arg)
{
int ret;
struct ibv_qp *qp;
struct ibv_cq *cq;
struct ibv_qp_init_attr init_attr;
uint32_t max_inline_data;
*max_inline_arg = 0;
/* Make a dummy CQ */
#if OPAL_IBV_CREATE_CQ_ARGS == 3
cq = ibv_create_cq(context, 1, NULL);
#else
cq = ibv_create_cq(context, 1, NULL, NULL, 0);
#endif
if (NULL == cq) {
opal_show_help("help-mpi-btl-openib.txt", "init-fail-create-q",
true, opal_proc_local_get()->proc_hostname,
__FILE__, __LINE__, "ibv_create_cq",
strerror(errno), errno,
ibv_get_device_name(device));
return OPAL_ERR_NOT_AVAILABLE;
}
/* Setup the QP attributes */
memset(&init_attr, 0, sizeof(init_attr));
init_attr.qp_type = IBV_QPT_RC;
init_attr.send_cq = cq;
init_attr.recv_cq = cq;
init_attr.srq = 0;
init_attr.cap.max_send_sge = 1;
init_attr.cap.max_recv_sge = 1;
init_attr.cap.max_recv_wr = 1;
/* Loop over max_inline_data values; just check powers of 2 --
that's good enough */
init_attr.cap.max_inline_data = max_inline_data = 1 << 20;
ret = OPAL_ERR_NOT_FOUND;
while (max_inline_data > 0) {
qp = ibv_create_qp(pd, &init_attr);
if (NULL != qp) {
*max_inline_arg = max_inline_data;
ibv_destroy_qp(qp);
ret = OPAL_SUCCESS;
break;
}
max_inline_data >>= 1;
init_attr.cap.max_inline_data = max_inline_data;
}
/* Destroy the temp CQ */
ibv_destroy_cq(cq);
return ret;
}

505
opal/mca/common/verbs/common_verbs_find_ports.c
Просмотреть файл

@ -1,505 +0,0 @@
/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
/*
* Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
* University Research and Technology
* Corporation. All rights reserved.
* Copyright (c) 2004-2014 The University of Tennessee and The University
* of Tennessee Research Foundation. All rights
* reserved.
* Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
* University of Stuttgart. All rights reserved.
* Copyright (c) 2004-2005 The Regents of the University of California.
* All rights reserved.
* Copyright (c) 2006-2014 Cisco Systems, Inc. All rights reserved.
* Copyright (c) 2006-2012 Mellanox Technologies. All rights reserved.
* Copyright (c) 2006-2015 Los Alamos National Security, LLC. All rights
* reserved.
* Copyright (c) 2006-2007 Voltaire All rights reserved.
* Copyright (c) 2009 Sun Microsystems, Inc. All rights reserved.
* Copyright (c) 2009-2012 Oak Ridge National Laboratory. All rights reserved.
* $COPYRIGHT$
*
* Additional copyrights may follow
*
* $HEADER$
*/
#include "opal_config.h"
#include <stdio.h>
#include <string.h>
#include <infiniband/verbs.h>
#include <stdint.h>
#include "opal_stdint.h"
#include "opal/types.h"
#include "opal/util/output.h"
#include "opal/util/argv.h"
#include "opal/class/opal_object.h"
#include "opal/util/show_help.h"
#include "opal/util/proc.h"
#include "opal/constants.h"
#include "common_verbs.h"
/***********************************************************************/
static void device_item_construct(opal_common_verbs_device_item_t *di)
{
di->device = NULL;
di->device_name = NULL;
di->context = NULL;
di->destructor_free_context = true;
memset(&di->device_attr, 0, sizeof(di->device_attr));
}
static void device_item_destruct(opal_common_verbs_device_item_t *di)
{
if (NULL != di->device_name) {
free(di->device_name);
}
/* Only free the context if a) the device is open, and b) the
upper layer didn't tell us not to */
if (NULL != di->context && di->destructor_free_context) {
ibv_close_device(di->context);
}
/* Zero out all the fields */
device_item_construct(di);
}
OBJ_CLASS_INSTANCE(opal_common_verbs_device_item_t,
opal_object_t,
device_item_construct,
device_item_destruct);
/***********************************************************************/
static void port_item_construct(opal_common_verbs_port_item_t *pi)
{
pi->device = NULL;
pi->port_num = 0;
memset(&pi->port_attr, 0, sizeof(pi->port_attr));
}
static void port_item_destruct(opal_common_verbs_port_item_t *pi)
{
OBJ_RELEASE(pi->device);
/* Zero out all the fields */
port_item_construct(pi);
}
OBJ_CLASS_INSTANCE(opal_common_verbs_port_item_t,
opal_list_item_t,
port_item_construct,
port_item_destruct);
/***********************************************************************/
/*
* Given a list of include or exclude items (never both), determine
* whether we want the current port or not.
*/
static bool want_this_port(char **include_list, char **exclude_list,
opal_common_verbs_device_item_t *di, int port)
{
int i;
char name[1024];
/* If we have no include or exclude list, then we unconditionally
want the port */
if (NULL == include_list && NULL == exclude_list) {
return true;
}
/* Search the include list */
if (NULL != include_list) {
for (i = 0; NULL != include_list[i]; ++i) {
/* First check if we can find the naked device name */
if (strcmp(di->device_name, include_list[i]) == 0) {
return true;
}
/* Now check for the specific port number */
snprintf(name, sizeof(name), "%s:%d", di->device_name, port);
if (strcmp(name, include_list[i]) == 0) {
return true;
}
}
/* Didn't find it. So we don't want it. */
return false;
}
/* Search the exclude list */
else {
for (i = 0; NULL != exclude_list[i]; ++i) {
/* First check if we can find the naked device name */
if (strcmp(di->device_name, exclude_list[i]) == 0) {
return false;
}
/* Now check for the specific port number */
snprintf(name, sizeof(name), "%s:%d", di->device_name, port);
if (strcmp(name, exclude_list[i]) == 0) {
return false;
}
}
/* Didn't find it. So we want it. */
return true;
}
/* Will never get here */
}
/***********************************************************************/
#if HAVE_DECL_IBV_LINK_LAYER_ETHERNET
static const char *link_layer_to_str(int link_type)
{
switch(link_type) {
case IBV_LINK_LAYER_INFINIBAND: return "IB";
case IBV_LINK_LAYER_ETHERNET: return "IWARP";
case IBV_LINK_LAYER_UNSPECIFIED:
default: return "unspecified";
}
}
#endif
/***********************************************************************/
static void check_sanity(char ***if_sanity_list, const char *dev_name, int port)
{
int i;
char tmp[BUFSIZ], **list;
const char *compare;
if (NULL == if_sanity_list || NULL == *if_sanity_list) {
return;
}
list = *if_sanity_list;
/* A match is found if:
- "dev_name" is in the list and port == -1, or
- "dev_name:port" is in the list
If a match is found, remove that entry from the list. */
memset(tmp, 0, sizeof(tmp));
if (port > 0) {
snprintf(tmp, sizeof(tmp) - 1, "%s:%d", dev_name, port);
compare = tmp;
} else {
compare = dev_name;
}
for (i = 0; NULL != list[i]; ++i) {
if (0 == strcmp(list[i], compare)) {
int count = opal_argv_count(list);
opal_argv_delete(&count, &list, i, 1);
*if_sanity_list = list;
--i;
}
}
}
/***********************************************************************/
/*
* Find a list of ibv_ports matching a set of criteria.
*/
opal_list_t *opal_common_verbs_find_ports(const char *if_include,
const char *if_exclude,
int flags,
int stream)
{
int32_t num_devs;
struct ibv_device **devices;
struct ibv_device *device;
struct ibv_context *device_context;
struct ibv_device_attr device_attr;
struct ibv_port_attr port_attr;
char **if_include_list = NULL, **if_exclude_list = NULL, **if_sanity_list = NULL;
opal_common_verbs_device_item_t *di;
opal_common_verbs_port_item_t *pi;
int rc;
uint32_t j;
opal_list_t *port_list = NULL;
bool want;
/* Sanity check the include/exclude params */
if (NULL != if_include && NULL != if_exclude) {
return NULL;
}
/* Query all the IBV devices on the machine. Use an ompi
compatibility function, because how to get this list changed
over the history of the IBV API. */
devices = opal_ibv_get_device_list(&num_devs);
if (0 == num_devs) {
opal_output_verbose(5, stream, "no verbs interfaces found");
return NULL;
}
opal_output_verbose(5, stream, "found %d verbs interface%s",
num_devs, (num_devs != 1) ? "s" : "");
/* Allocate a list to fill */
port_list = OBJ_NEW(opal_list_t);
if (NULL == port_list) {
return NULL;
}
if (NULL != if_include) {
opal_output_verbose(5, stream, "finding verbs interfaces, including %s",
if_include);
if_include_list = opal_argv_split(if_include, ',');
if_sanity_list = opal_argv_copy(if_include_list);
} else if (NULL != if_exclude) {
opal_output_verbose(5, stream, "finding verbs interfaces, excluding %s",
if_exclude);
if_exclude_list = opal_argv_split(if_exclude, ',');
if_sanity_list = opal_argv_copy(if_exclude_list);
}
/* Now loop through all the devices. Get the attributes for each
port on each device to see if they match our selection
criteria. */
for (int32_t i = 0; (int32_t) i < num_devs; ++i) {
/* See if this device is on the include/exclude sanity check
list. If it is, remove it from the sanity check list
(i.e., we should end up with an empty list at the end if
all entries in the sanity check list exist) */
device = devices[i];
check_sanity(&if_sanity_list, ibv_get_device_name(device), -1);
opal_output_verbose(5, stream, "examining verbs interface: %s",
ibv_get_device_name(device));
device_context = ibv_open_device(device);
if (NULL == device_context) {
opal_show_help("help-opal-common-verbs.txt",
"ibv_open_device fail", true,
opal_proc_local_get()->proc_hostname,
ibv_get_device_name(device),
errno, strerror(errno));
goto err_free_port_list;
}
if (ibv_query_device(device_context, &device_attr)){
opal_show_help("help-opal-common-verbs.txt",
"ibv_query_device fail", true,
opal_proc_local_get()->proc_hostname,
ibv_get_device_name(device),
errno, strerror(errno));
goto err_free_port_list;
}
/* Now that we have the attributes of this device, remove all
ports of this device from the sanity check list. Note that
IBV ports are indexed from 1, not 0. */
for (j = 1; j <= device_attr.phys_port_cnt; j++) {
check_sanity(&if_sanity_list, ibv_get_device_name(device), j);
}
/* Check the device-specific flags to see if we want this
device */
want = false;
if (flags & OPAL_COMMON_VERBS_FLAGS_TRANSPORT_IB &&
IBV_TRANSPORT_IB == device->transport_type) {
opal_output_verbose(5, stream, "verbs interface %s has right type (IB)",
ibv_get_device_name(device));
want = true;
}
if (flags & OPAL_COMMON_VERBS_FLAGS_TRANSPORT_IWARP &&
IBV_TRANSPORT_IWARP == device->transport_type) {
opal_output_verbose(5, stream, "verbs interface %s has right type (IWARP)",
ibv_get_device_name(device));
want = true;
}
/* Check for RC or UD QP support */
if (flags & OPAL_COMMON_VERBS_FLAGS_RC) {
rc = opal_common_verbs_qp_test(device_context, flags);
if (OPAL_SUCCESS == rc) {
want = true;
opal_output_verbose(5, stream,
"verbs interface %s supports RC QPs",
ibv_get_device_name(device));
} else {
opal_output_verbose(5, stream,
"verbs interface %s failed to make RC QP",
ibv_get_device_name(device));
}
}
if (flags & OPAL_COMMON_VERBS_FLAGS_UD) {
rc = opal_common_verbs_qp_test(device_context, flags);
if (OPAL_SUCCESS == rc) {
want = true;
opal_output_verbose(5, stream,
"verbs interface %s supports UD QPs",
ibv_get_device_name(device));
} else if (OPAL_ERR_TYPE_MISMATCH == rc) {
opal_output_verbose(5, stream,
"verbs interface %s made an RC QP! we don't want RC-capable devices",
ibv_get_device_name(device));
} else {
opal_output_verbose(5, stream,
"verbs interface %s failed to make UD QP",
ibv_get_device_name(device));
}
}
/* If we didn't want it, go to the next device */
if (!want) {
continue;
}
/* Make a device_item_t to hold the device information */
di = OBJ_NEW(opal_common_verbs_device_item_t);
if (NULL == di) {
goto err_free_port_list;
}
di->device = device;
di->context = device_context;
di->device_attr = device_attr;
di->device_name = strdup(ibv_get_device_name(device));
/* Note IBV ports are 1 based (not 0 based) */
for (j = 1; j <= device_attr.phys_port_cnt; j++) {
/* If we don't want this port (based on if_include /
if_exclude lists), skip it */
if (!want_this_port(if_include_list, if_exclude_list, di, j)) {
opal_output_verbose(5, stream, "verbs interface %s:%d: rejected by include/exclude",
ibv_get_device_name(device), j);
continue;
}
/* Query the port */
if (ibv_query_port(device_context, (uint8_t) j, &port_attr)) {
opal_show_help("help-opal-common-verbs.txt",
"ibv_query_port fail", true,
opal_proc_local_get()->proc_hostname,
ibv_get_device_name(device),
errno, strerror(errno));
goto err_free_port_list;
}
/* We definitely only want ACTIVE ports */
if (IBV_PORT_ACTIVE != port_attr.state) {
opal_output_verbose(5, stream, "verbs interface %s:%d: not ACTIVE",
ibv_get_device_name(device), j);
continue;
}
/* Check the port-specific flags to see if we want this
port */
want = false;
if (0 == flags) {
want = true;
}
if ((flags & (OPAL_COMMON_VERBS_FLAGS_LINK_LAYER_IB |
OPAL_COMMON_VERBS_FLAGS_LINK_LAYER_ETHERNET)) ==
(OPAL_COMMON_VERBS_FLAGS_LINK_LAYER_IB |
OPAL_COMMON_VERBS_FLAGS_LINK_LAYER_ETHERNET)) {
/* If they specified both link layers, then we want this port */
want = true;
} else if ((flags & (OPAL_COMMON_VERBS_FLAGS_LINK_LAYER_IB |
OPAL_COMMON_VERBS_FLAGS_LINK_LAYER_ETHERNET)) == 0) {
/* If they specified neither link layer, then we want this port */
want = true;
}
#if HAVE_DECL_IBV_LINK_LAYER_ETHERNET
else if (flags & OPAL_COMMON_VERBS_FLAGS_LINK_LAYER_IB) {
if (IBV_LINK_LAYER_INFINIBAND == port_attr.link_layer) {
want = true;
} else {
opal_output_verbose(5, stream, "verbs interface %s:%d has wrong link layer (has %s, want IB)",
ibv_get_device_name(device), j,
link_layer_to_str(port_attr.link_layer));
}
} else if (flags & OPAL_COMMON_VERBS_FLAGS_LINK_LAYER_ETHERNET) {
if (IBV_LINK_LAYER_ETHERNET == port_attr.link_layer) {
want = true;
} else {
opal_output_verbose(5, stream, "verbs interface %s:%d has wrong link layer (has %s, want Ethernet)",
ibv_get_device_name(device), j,
link_layer_to_str(port_attr.link_layer));
}
}
#endif
if (!want) {
continue;
}
/* If we got this far, we want the port. Make an item for it. */
pi = OBJ_NEW(opal_common_verbs_port_item_t);
if (NULL == pi) {
goto err_free_port_list;
}
pi->device = di;
pi->port_num = j;
pi->port_attr = port_attr;
OBJ_RETAIN(di);
/* Add the port item to the list */
opal_list_append(port_list, &pi->super);
opal_output_verbose(5, stream, "found acceptable verbs interface %s:%d",
ibv_get_device_name(device), j);
}
/* We're done with the device; if some ports are using it, its
ref count will be > 0, and therefore the device won't be
deleted here. */
OBJ_RELEASE(di);
}
/* Sanity check that the devices specified in the if_include /
if_exclude lists actually existed. If this is true, then the
sanity list will now be empty. If there are still items left
on the list, then they didn't exist. Bad. Print a warning (if
the warning is not disabled). */
if (0 != opal_argv_count(if_sanity_list)) {
if (opal_common_verbs_warn_nonexistent_if) {
char *str = opal_argv_join(if_sanity_list, ',');
opal_show_help("help-opal-common-verbs.txt", "nonexistent port",
true, opal_proc_local_get()->proc_hostname,
((NULL != if_include) ? "in" : "ex"), str);
free(str);
/* Only warn once per process */
opal_common_verbs_warn_nonexistent_if = false;
}
}
if (NULL != if_sanity_list) {
opal_argv_free(if_sanity_list);
}
opal_argv_free(if_include_list);
opal_argv_free(if_exclude_list);
/* All done! */
opal_ibv_free_device_list(devices);
return port_list;
err_free_port_list:
OPAL_LIST_RELEASE(port_list);
opal_ibv_free_device_list(devices);
if (NULL != if_sanity_list) {
opal_argv_free(if_sanity_list);
}
opal_argv_free(if_include_list);
opal_argv_free(if_exclude_list);
return NULL;
}

60
opal/mca/common/verbs/common_verbs_mca.c
Просмотреть файл

@ -1,60 +0,0 @@
/*
* Copyright (c) 2012 Cisco Systems, Inc. All rights reserved.
*
* $COPYRIGHT$
*
* Additional copyrights may follow
*
* $HEADER$
*/
#include "opal_config.h"
#include "common_verbs.h"
#include "opal/mca/base/mca_base_var.h"
/***********************************************************************/
static bool registered = false;
static int warn_nonexistent_if_index = -1;
bool opal_common_verbs_warn_nonexistent_if = true;
int opal_common_verbs_want_fork_support = -1;
static void register_internal(void)
{
opal_common_verbs_warn_nonexistent_if = true;
warn_nonexistent_if_index =
mca_base_var_register("opal", "opal_common", "verbs", "warn_nonexistent_if",
"Warn if non-existent devices and/or ports are specified in device include/exclude MCA parameters "
"(0 = do not warn; any other value = warn)",
MCA_BASE_VAR_TYPE_BOOL, NULL, 0, MCA_BASE_VAR_FLAG_SETTABLE,
OPAL_INFO_LVL_9, MCA_BASE_VAR_SCOPE_LOCAL,
&opal_common_verbs_warn_nonexistent_if);
/* A depreacated synonym */
mca_base_var_register_synonym(warn_nonexistent_if_index, "ompi", "ompi_common",
"verbs", "warn_nonexistent_if", MCA_BASE_VAR_SYN_FLAG_DEPRECATED);
mca_base_var_register("opal", "opal_common", "verbs", "want_fork_support",
"Whether fork support is desired or not "
"(negative = try to enable fork support, but continue even "
"if it is not available, 0 = do not enable fork support, "
"positive = try to enable fork support and fail if it is not available)",
MCA_BASE_VAR_TYPE_INT, NULL, 0, MCA_BASE_VAR_FLAG_SETTABLE,
OPAL_INFO_LVL_8, MCA_BASE_VAR_SCOPE_ALL_EQ,
&opal_common_verbs_want_fork_support);
registered = true;
}
void opal_common_verbs_mca_register(mca_base_component_t *component)
{
if (!registered) {
register_internal();
}
/* Make synonym for the common_verbs MCA params. */
mca_base_var_register_synonym(warn_nonexistent_if_index, "ompi", component->mca_type_name,
component->mca_component_name, "warn_nonexistent_if", 0);
}

120
opal/mca/common/verbs/common_verbs_port.c
Просмотреть файл

@ -1,120 +0,0 @@
/*
* Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
* University Research and Technology
* Corporation. All rights reserved.
* Copyright (c) 2004-2011 The University of Tennessee and The University
* of Tennessee Research Foundation. All rights
* reserved.
* Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
* University of Stuttgart. All rights reserved.
* Copyright (c) 2004-2005 The Regents of the University of California.
* All rights reserved.
* Copyright (c) 2006-2012 Cisco Systems, Inc. All rights reserved.
* Copyright (c) 2006-2009 Mellanox Technologies. All rights reserved.
* Copyright (c) 2006-2012 Los Alamos National Security, LLC. All rights
* reserved.
* Copyright (c) 2006-2007 Voltaire All rights reserved.
* Copyright (c) 2009-2012 Oracle and/or its affiliates. All rights reserved.
* Copyright (c) 2011 NVIDIA Corporation. All rights reserved.
* Copyright (c) 2012 Oak Ridge National Laboratory. All rights reserved
*
* $COPYRIGHT$
*
* Additional copyrights may follow
*
* $HEADER$
*/
#include "opal_config.h"
#include "opal/constants.h"
#include <infiniband/verbs.h>
#include "common_verbs.h"
int opal_common_verbs_port_bw(struct ibv_port_attr *port_attr,
uint32_t *bandwidth)
{
*bandwidth = 0;
/* To calculate the bandwidth available on this port, we have to
look up the values corresponding to port->active_speed and
port->active_width. These are enums corresponding to the IB
spec. Overall forumula to get the true link speed is 8/10 or
64/66 of the reported speed (depends on the coding that is
being used for the particular speed) times the number of
links. */
switch (port_attr->active_speed) {
case 1:
/* SDR: 2.5 Gbps * 0.8, in megabits */
*bandwidth = 2000;
break;
case 2:
/* DDR: 5 Gbps * 0.8, in megabits */
*bandwidth = 4000;
break;
case 4:
/* QDR: 10 Gbps * 0.8, in megabits */
*bandwidth = 8000;
break;
case 8:
/* FDR10: 10.3125 Gbps * 64/66, in megabits */
*bandwidth = 10000;
break;
case 16:
/* FDR: 14.0625 Gbps * 64/66, in megabits */
*bandwidth = 13636;
break;
case 32:
/* EDR: 25.78125 Gbps * 64/66, in megabits */
*bandwidth = 25000;
break;
case 64:
/* HDR: 50Gbps * 64/66, in megabits */
*bandwidth = 50000;
break;
default:
/* Who knows? */
return OPAL_ERR_NOT_FOUND;
}
switch (port_attr->active_width) {
case 1:
/* 1x */
/* unity */
break;
case 2:
/* 4x */
*bandwidth *= 4;
break;
case 4:
/* 8x */
*bandwidth *= 8;
break;
case 8:
/* 12x */
*bandwidth *= 12;
break;
default:
/* Who knows? */
return OPAL_ERR_NOT_FOUND;
}
return OPAL_SUCCESS;
}
int opal_common_verbs_mtu(struct ibv_port_attr *port_attr)
{
if (NULL == port_attr) {
return 0;
}
switch(port_attr->active_mtu) {
case IBV_MTU_256: return 256;
case IBV_MTU_512: return 512;
case IBV_MTU_1024: return 1024;
case IBV_MTU_2048: return 2048;
case IBV_MTU_4096: return 4096;
default: return 0;
}
}

104
opal/mca/common/verbs/common_verbs_qp_type.c
Просмотреть файл

@ -1,104 +0,0 @@
/*
* Copyright (c) 2012-2013 Cisco Systems, Inc. All rights reserved.
*
* $COPYRIGHT$
*
* Additional copyrights may follow
*
* $HEADER$
*/
#include "opal_config.h"
#include "opal/constants.h"
#include <stdio.h>
#include <string.h>
#include <infiniband/verbs.h>
#include "common_verbs.h"
/*
* It seems you can't probe a device / port to see if it supports a
* specific type of QP. You just have to try to make it and see if it
* works. This is a short helper function to try to make a QP of a
* specific type and return whether it worked.
*/
static bool make_qp(struct ibv_pd *pd, struct ibv_cq *cq, enum ibv_qp_type type)
{
struct ibv_qp_init_attr qpia;
struct ibv_qp *qp;
memset(&qpia, 0, sizeof(qpia));
qpia.qp_context = NULL;
qpia.send_cq = cq;
qpia.recv_cq = cq;
qpia.srq = NULL;
qpia.cap.max_send_wr = 1;
qpia.cap.max_recv_wr = 1;
qpia.cap.max_send_sge = 1;
qpia.cap.max_recv_sge = 1;
qpia.cap.max_inline_data = 0;
qpia.qp_type = type;
qpia.sq_sig_all = 0;
qp = ibv_create_qp(pd, &qpia);
if (NULL != qp) {
ibv_destroy_qp(qp);
return true;
}
return false;
}
int opal_common_verbs_qp_test(struct ibv_context *device_context, int flags)
{
int rc = OPAL_SUCCESS;
struct ibv_pd *pd = NULL;
struct ibv_cq *cq = NULL;
/* Bozo check */
if (NULL == device_context ||
(0 == (flags & (OPAL_COMMON_VERBS_FLAGS_RC | OPAL_COMMON_VERBS_FLAGS_UD)))) {
return OPAL_ERR_BAD_PARAM;
}
/* Try to make both the PD and CQ */
pd = ibv_alloc_pd(device_context);
if (NULL == pd) {
return OPAL_ERR_OUT_OF_RESOURCE;
}
cq = ibv_create_cq(device_context, 2, NULL, NULL, 0);
if (NULL == cq) {
rc = OPAL_ERR_OUT_OF_RESOURCE;
goto out;
}
/* Now try to make the QP(s) of the desired type(s) */
if (flags & OPAL_COMMON_VERBS_FLAGS_RC &&
!make_qp(pd, cq, IBV_QPT_RC)) {
rc = OPAL_ERR_NOT_SUPPORTED;
goto out;
}
if (flags & OPAL_COMMON_VERBS_FLAGS_NOT_RC &&
make_qp(pd, cq, IBV_QPT_RC)) {
rc = OPAL_ERR_TYPE_MISMATCH;
goto out;
}
if (flags & OPAL_COMMON_VERBS_FLAGS_UD &&
!make_qp(pd, cq, IBV_QPT_UD)) {
rc = OPAL_ERR_NOT_SUPPORTED;
goto out;
}
out:
/* Free the PD and/or CQ */
if (NULL != pd) {
ibv_dealloc_pd(pd);
}
if (NULL != cq) {
ibv_destroy_cq(cq);
}
return rc;
}

41
opal/mca/common/verbs/configure.m4
Просмотреть файл

@ -1,41 +0,0 @@
# -*- shell-script -*-
#
# Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
# University Research and Technology
# Corporation. All rights reserved.
# Copyright (c) 2004-2005 The University of Tennessee and The University
# of Tennessee Research Foundation. All rights
# reserved.
# Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
# University of Stuttgart. All rights reserved.
# Copyright (c) 2004-2005 The Regents of the University of California.
# All rights reserved.
# Copyright (c) 2007-2012 Cisco Systems, Inc. All rights reserved.
# Copyright (c) 2009-2012 Mellanox Technologies. All rights reserved.
# Copyright (c) 2009-2012 Oak Ridge National Laboratory. All rights reserved.
# $COPYRIGHT$
#
# Additional copyrights may follow
#
# $HEADER$
#
# MCA_opal_common_verbs_CONFIG([action-if-can-compile],
# [action-if-cant-compile])
# ------------------------------------------------
AC_DEFUN([MCA_opal_common_verbs_CONFIG],[
AC_CONFIG_FILES([opal/mca/common/verbs/Makefile])
common_verbs_happy="no"
OPAL_CHECK_OPENFABRICS([common_verbs],
[common_verbs_happy="yes"])
AS_IF([test "$common_verbs_happy" = "yes"],
[$1],
[$2])
# substitute in the things needed to build openib
AC_SUBST([common_verbs_CFLAGS])
AC_SUBST([common_verbs_CPPFLAGS])
AC_SUBST([common_verbs_LDFLAGS])
AC_SUBST([common_verbs_LIBS])
])dnl

54
opal/mca/common/verbs/help-opal-common-verbs.txt
Просмотреть файл

@ -1,54 +0,0 @@
#
# Copyright (c) 2012-2014 Cisco Systems, Inc. All rights reserved.
#
# $COPYRIGHT$
#
# Additional copyrights may follow
#
# $HEADER$
#
[ibv_open_device fail]
Open MPI failed to open an OpenFabrics device. This is an unusual
error; the system reported the OpenFabrics device as being present,
but then later failed to access it successfully. This usually
indicates either a misconfiguration or a failed OpenFabrics hardware
device.
All OpenFabrics support has been disabled in this MPI process; your
job may or may not continue.
Hostname: %s
Device name: %s
Error (%d): %s
#
[ibv_query_device fail]
Open MPI failed to query an OpenFabrics device. This is an unusual
error; the system reported the OpenFabrics device as being present,
Open MPI was able to open it succesfully, but then later failed to
query it successfully. This usually indicates either a
misconfiguration or a failed OpenFabrics hardware device.
All OpenFabrics support has been disabled in this MPI process; your
job may or may not continue.
Hostname: %s
Device name: %s
Error (%d): %s
#
[nonexistent port]
WARNING: One or more nonexistent OpenFabrics devices/ports were
specified:
Host: %s
MCA parameter: ompi_common_verbs_%sclude
Nonexistent entities: %s
These entities will be ignored. You can disable this warning by
setting the ompi_common_verbs_warn_nonexistent_if MCA parameter to 0.
#
[ibv_fork_init fail]
Fork support was requested but the library call ibv_fork_init() failed.
Hostname: %s
Error (%d): %s
#

7
opal/mca/common/verbs/owner.txt
Просмотреть файл

@ -1,7 +0,0 @@
#
# owner/status file
# owner: institution that is responsible for this package
# status: e.g. active, maintenance, unmaintained
#
owner: MELLANOX
status: maintenance

40
opal/mca/common/verbs_usnic/Makefile.am
Просмотреть файл

@ -1,40 +0,0 @@
#
# Copyright (c) 2009-2012 Mellanox Technologies. All rights reserved.
# Copyright (c) 2009-2012 Oak Ridge National Laboratory. All rights reserved.
# Copyright (c) 2012-2015 Cisco Systems, Inc. All rights reserved.
# $COPYRIGHT$
#
# Additional copyrights may follow
#
# $HEADER$
#
headers = common_verbs_usnic.h
sources = common_verbs_usnic_fake.c
# This component is always linked statically. It has code that is
# registered as a driver for libibverbs. There is no corresponding
# *un*register API in libibverbs, so this code can never be dlclosed.
# And therefore it must be in the libopen-pal library, not a DSO or
# dependent library.
noinst_LTLIBRARIES = lib@OPAL_LIB_PREFIX@mca_common_verbs_usnic.la
lib@OPAL_LIB_PREFIX@mca_common_verbs_usnic_la_SOURCES = \
$(headers) $(sources)
lib@OPAL_LIB_PREFIX@mca_common_verbs_usnic_la_CPPFLAGS = \
$(common_verbs_usnic_CPPFLAGS)
lib@OPAL_LIB_PREFIX@mca_common_verbs_usnic_la_LDFLAGS = \
$(common_verbs_usnic_LDFLAGS)
lib@OPAL_LIB_PREFIX@mca_common_verbs_usnic_la_LIBADD = \
$(common_verbs_usnic_LIBS)
# Conditionally install the header files
if WANT_INSTALL_HEADERS
opaldir = $(opalincludedir)/opal/mca/common/verbs_usnic
opal_HEADERS = $(headers)
else
opaldir = $(includedir)
endif

27
opal/mca/common/verbs_usnic/common_verbs_usnic.h
Просмотреть файл

@ -1,27 +0,0 @@
/*
* Copyright (c) 2015 Cisco Systems, Inc. All rights reserved.
* $COPYRIGHT$
*
* Additional copyrights may follow
*
* $HEADER$
*/
#ifndef _COMMON_VERBS_USNIC_H_
#define _COMMON_VERBS_USNIC_H_
#include "opal_config.h"
#include <stdint.h>
#include <infiniband/verbs.h>
BEGIN_C_DECLS
/*
* Register fake verbs drivers
*/
void opal_common_verbs_usnic_register_fake_drivers(void);
END_C_DECLS
#endif

135
opal/mca/common/verbs_usnic/common_verbs_usnic_fake.c
Просмотреть файл

@ -1,135 +0,0 @@
/*
* Copyright (c) 2015 Cisco Systems, Inc. All rights reserved.
*
* $COPYRIGHT$
*
* Additional copyrights may follow
*
* $HEADER$
*/
/*
* The code in this file prevents spurious libibverbs warnings on
* stderr about devices that it doesn't recognize.
*
* Specifically, Cisco usNIC devices are exposed through the Linux
* InfiniBand kernel interface (i.e., they show up in
* /sys/class/infiniband). However, the userspace side of these
* drivers is not exposed through libibverbs (i.e., there is no
* libibverbs provider/plugin for usNIC). Therefore, when
* ibv_get_device_list() is invoked, libibverbs cannot find a plugin
* for usnic devices. This causes libibverbs to emit a spurious
* warning message on stderr.
*
* To avoid these extra stderr warnings, we insert a fake usnic verbs
* libibverbs provider that safely squelches these warnings.
*
* More specifically: the userspace side of usNIC is exposed through
* libfabric; we don't need libibverbs warnings about not being able
* to find a usnic driver.
*
* Note: this code is statically linked into libopen-pal. It is
* registered via ibv_register_driver(), and there is no corresponding
* *un*register IBV API. Hence, we cannot allow this code to be
* dlclosed (e.g., if it is a DSO or a dependent common library) -- it
* must be in libopen-pal itself, which will stay resident in the MPI
* application.
*/
#include "opal_config.h"
#include <stdio.h>
#include <sys/types.h>
#include <dirent.h>
#include <string.h>
#include <infiniband/verbs.h>
#ifdef HAVE_INFINIBAND_DRIVER_H
#include <infiniband/driver.h>
#endif
#include "common_verbs_usnic.h"
/***********************************************************************/
#define PCI_VENDOR_ID_CISCO (0x1137)
static struct ibv_context *fake_alloc_context(struct ibv_device *ibdev,
int cmd_fd)
{
/* Nothing to do here */
return NULL;
}
static void fake_free_context(struct ibv_context *ibctx)
{
/* Nothing to do here */
}
/* Put just enough in here to convince libibverbs that this is a valid
device, and a little extra just in case someone looks at this
struct in a debugger. */
static struct ibv_device fake_dev = {
.ops = {
.alloc_context = fake_alloc_context,
.free_context = fake_free_context
},
.name = "fake ibv_device inserted by Open MPI for non-verbs devices"
};
static struct ibv_device *fake_driver_init(const char *uverbs_sys_path,
int abi_version)
{
char value[8];
int vendor;
/* This function should only be invoked for
/sys/class/infiniband/usnic_X devices, but double check just to
be absolutely sure: read the vendor ID and ensure that it is
Cisco. */
if (ibv_read_sysfs_file(uverbs_sys_path, "device/vendor",
value, sizeof(value)) < 0) {
return NULL;
}
if (sscanf(value, "%i", &vendor) != 1) {
return NULL;
}
if (vendor == PCI_VENDOR_ID_CISCO) {
return &fake_dev;
}
/* We didn't find a device that we want to support */
return NULL;
}
void opal_common_verbs_usnic_register_fake_drivers(void)
{
/* No need to do this more than once */
static bool already_done = false;
if (already_done) {
return;
}
already_done = true;
/* If there are any usnic devices, then register a fake driver */
DIR *class_dir;
class_dir = opendir("/sys/class/infiniband");
if (NULL == class_dir) {
return;
}
bool found = false;
struct dirent *dent;
while ((dent = readdir(class_dir)) != NULL) {
if (strncmp(dent->d_name, "usnic_", 6) == 0) {
found = true;
break;
}
}
closedir(class_dir);
if (found) {
ibv_register_driver("usnic_verbs", fake_driver_init);
}
}

99
opal/mca/common/verbs_usnic/configure.m4
Просмотреть файл

@ -1,99 +0,0 @@
# -*- shell-script -*-
#
# Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
# University Research and Technology
# Corporation. All rights reserved.
# Copyright (c) 2004-2005 The University of Tennessee and The University
# of Tennessee Research Foundation. All rights
# reserved.
# Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
# University of Stuttgart. All rights reserved.
# Copyright (c) 2004-2005 The Regents of the University of California.
# All rights reserved.
# Copyright (c) 2007-2016 Cisco Systems, Inc. All rights reserved.
# Copyright (c) 2009-2012 Mellanox Technologies. All rights reserved.
# Copyright (c) 2009-2012 Oak Ridge National Laboratory. All rights reserved.
# $COPYRIGHT$
#
# Additional copyrights may follow
#
# $HEADER$
#
#
# This component is a workaround to a bug in libibverbs that prints a
# dire warning that usNIC devices are not supported (of course not --
# usNIC devices provide functionality through libfabric, not
# libibverbs). This component was written before a better workaround
# was created: a "no op" libibverbs plugin for usNIC devices
# (https://github.com/cisco/libusnic_verbs, and is also available in
# binary form on cisco.com).
#
# Hence, this component no longer builds by default. It's still
# available if a user specifically asks for it (e.g., if they do not
# want to install the "no op" libibverbs plugin), but it's not the
# default. This component also has the side-effect of making
# libopen-pal.so depend on libibverbs.so, which can be annoying for
# packagers (which is another reason it isn't built by default any
# more).
#
# This component must be linked statically into libopen-pal because it
# registers a provider for libibverbs at run time, and there's no
# libibverbs API to *un*register a plugin. Hence, we can't allow this
# code to be dlclosed/removed from the process. Hence: it must be
# compiled statically into libopen-pal.
#
AC_DEFUN([MCA_opal_common_verbs_usnic_COMPILE_MODE], [
AC_MSG_CHECKING([for MCA component $2:$3 compile mode])
$4="static"
AC_MSG_RESULT([$$4])
])
# MCA_opal_common_verbs_usnic_CONFIG([action-if-can-compile],
# [action-if-cant-compile])
# ------------------------------------------------
AC_DEFUN([MCA_opal_common_verbs_usnic_CONFIG],[
AC_CONFIG_FILES([opal/mca/common/verbs_usnic/Makefile])
common_verbs_usnic_happy=0
AC_ARG_WITH(verbs-usnic,
AC_HELP_STRING([--with-verbs-usnic],
[Add support in Open MPI to defeat a seemingly dire warning message from libibverbs that Cisco usNIC devices are not supported. This support is not compiled by default because you can also avoid this libibverbs bug by installing the libibverbs_usnic "no no" plugin, available from https://github.com/cisco/libusnic_verbs or in binary form from cisco.com]))
AS_IF([test "$with_verbs_usnic" = "yes"],
[common_verbs_usnic_happy=1])
AS_IF([test $common_verbs_usnic_happy -eq 1],
[OPAL_CHECK_OPENFABRICS([common_verbs_usnic],
[common_verbs_usnic_happy=1],
[common_verbs_usnic_happy=0])
])
AS_IF([test $common_verbs_usnic_happy -eq 1],
[AC_CHECK_MEMBER([struct ibv_device.ops],
[],
[AC_MSG_WARN([--with-verbs-usnic specified, but the verbs.h does not])
AC_MSG_WARN([have the required member fields. It is highly likely])
AC_MSG_WARN([that you do not need --with-verbs-usnic. Try configuring])
AC_MSG_WARN([and building Open MPI without it; if you get warnings])
AC_MSG_WARN([about usnic IB devices anyway, please let us know.])
AC_MSG_WARN([Since you asked for --with-verbs-usnic and we cannot])
AC_MSG_WARN([deliver it, configure will now abort.])
AC_MSG_ERROR([Cannot continue])
],
[#include <infiniband/verbs.h>])
])
AC_DEFINE_UNQUOTED([OPAL_COMMON_VERBS_USNIC_HAPPY],
[$common_verbs_usnic_happy],
[Whether the common/usnic_verbs component is being built or not])
AS_IF([test $common_verbs_usnic_happy -eq 1],
[$1],
[$2])
# substitute in the things needed to build openib
AC_SUBST([common_verbs_usnic_CPPFLAGS])
AC_SUBST([common_verbs_usnic_LDFLAGS])
AC_SUBST([common_verbs_usnic_LIBS])
])dnl

7
opal/mca/common/verbs_usnic/owner.txt
Просмотреть файл

@ -1,7 +0,0 @@
#
# owner/status file
# owner: institution that is responsible for this package
# status: e.g. active, maintenance, unmaintained
#
owner: Cisco
status: maintenance