c42ab8ea37
Commit from a long-standing Mercurial tree that ended up incorporating a lot of things: * A few fixes for CPC interface changes in all the CPCs * Attempts (but not yet finished) to fix shutdown problems in the IB CM CPC * #1319: add CTS support (i.e., initiator guarantees to send first message; automatically activated for iWARP over the RDMA CM CPC) * Some variable and function renamings to make this be generic (e.g., alloc_credit_frag became alloc_control_frag) * CPCs no longer post receive buffers; they only post a single receive buffer for the CTS if they use CTS. Instead, the main BTL now posts the main sets of receive buffers. * CPCs allocate a CTS buffer only if they're about to make a connection * RDMA CM improvements: * Use threaded mode openib fd monitoring to wait for for RDMA CM events * Synchronize endpoint finalization and disconnection between main thread and service thread to avoid/fix some race conditions * Converted several structs to be OBJs so that we can use reference counting to know when to invoke destructors * Make some new OBJ's have opal_list_item_t's as their base, thereby eliminating the need for the local list_item_t type * Renamed many variables to be internally consistent * Centralize the decision in an inline function as to whether this process or the remote process is supposed to be the initiator * Add oodles of OPAL_OUTPUT statements for debugging (hard-wired to output stream -1; to be activated by developers if they want/need them) * Use rdma_create_qp() instead of ibv_create_qp() * openib fd monitoring improvements: * Renamed a bunch of functions and variables to be a little more obvious as to their true function * Use pipes to communicate between main thread and service thread * Add ability for main thread to invoke a function back on the service thread * Ensure to set initiator_depth and responder_resources properly, but putting max_qp_rd_ataom and ma_qp_init_rd_atom in the modex (see rdma_connect(3)) * Ensure to set the source IP address in rdma_resolve() to ensure that we select the correct OpenFabrics source port * Make new MCA param: openib_btl_connect_rdmacm_resolve_timeout * Other improvements: * btl_openib_device_type MCA param: can be "iw" or "ib" or "all" (or "infiniband" or "iwarp") * Somewhat improved error handling * Bunches of spelling fixes in comments, VERBOSE, and OUTPUT statements * Oodles of little coding style fixes * Changed shutdown ordering of btl; the device is now an OBJ with ref counting for destruction * Added some more show_help error messages * Change configury to only build IBCM / RDMACM if we have threads (because we need a progress thread) This commit was SVN r19686. The following Trac tickets were found above: Ticket 1210 --> https://svn.open-mpi.org/trac/ompi/ticket/1210
87 строки
3.1 KiB
Bash
87 строки
3.1 KiB
Bash
# -*- shell-script -*-
|
|
#
|
|
# Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
|
|
# University Research and Technology
|
|
# Corporation. All rights reserved.
|
|
# Copyright (c) 2004-2005 The University of Tennessee and The University
|
|
# of Tennessee Research Foundation. All rights
|
|
# reserved.
|
|
# Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
|
|
# University of Stuttgart. All rights reserved.
|
|
# Copyright (c) 2004-2005 The Regents of the University of California.
|
|
# All rights reserved.
|
|
# Copyright (c) 2007 Cisco, Inc. All rights reserved.
|
|
# Copyright (c) 2008 Mellanox Technologies. All rights reserved.
|
|
# $COPYRIGHT$
|
|
#
|
|
# Additional copyrights may follow
|
|
#
|
|
# $HEADER$
|
|
#
|
|
|
|
# MCA_btl_openib_POST_CONFIG([should_build])
|
|
# ------------------------------------------
|
|
AC_DEFUN([MCA_btl_openib_POST_CONFIG], [
|
|
AM_CONDITIONAL([MCA_btl_openib_have_xrc], [test $1 -eq 1 -a "x$btl_openib_have_xrc" = "x1"])
|
|
AM_CONDITIONAL([MCA_btl_openib_have_rdmacm], [test $1 -eq 1 -a "x$btl_openib_have_rdmacm" = "x1"])
|
|
AM_CONDITIONAL([MCA_btl_openib_have_ibcm], [test $1 -eq 1 -a "x$btl_openib_have_ibcm" = "x1"])
|
|
])
|
|
|
|
|
|
# MCA_btl_openib_CONFIG([action-if-can-compile],
|
|
# [action-if-cant-compile])
|
|
# ------------------------------------------------
|
|
AC_DEFUN([MCA_btl_openib_CONFIG],[
|
|
OMPI_VAR_SCOPE_PUSH([cpcs have_threads])
|
|
cpcs="oob"
|
|
|
|
OMPI_CHECK_OPENIB([btl_openib],
|
|
[btl_openib_happy="yes"],
|
|
[btl_openib_happy="no"])
|
|
|
|
AS_IF([test "$btl_openib_happy" = "yes"],
|
|
[btl_openib_WRAPPER_EXTRA_LDFLAGS="$btl_openib_LDFLAGS"
|
|
btl_openib_WRAPPER_EXTRA_LIBS="$btl_openib_LIBS"
|
|
|
|
# With the new openib flags, look for ibv_fork_init
|
|
LDFLAGS_save="$LDFLAGS"
|
|
LIBS_save="$LIBS"
|
|
LDFLAGS="$LDFLAGS $btl_openib_LDFLAGS"
|
|
LIBS="$LIBS $btl_openib_LIBS"
|
|
AC_CHECK_FUNCS([ibv_fork_init])
|
|
LDFLAGS="$LDFLAGS_save"
|
|
LIBS="$LIBS_save"
|
|
$1],
|
|
[$2])
|
|
|
|
AC_MSG_CHECKING([for thread support (needed for ibcm/rdmacm)])
|
|
have_threads=`echo $THREAD_TYPE | awk '{ print [$]1 }'`
|
|
if test "x$have_threads" = "x"; then
|
|
have_threads=none
|
|
fi
|
|
AC_MSG_RESULT([$have_threads])
|
|
|
|
AS_IF([test "$btl_openib_happy" = "yes"],
|
|
[if test "x$btl_openib_have_xrc" = "x1"; then
|
|
cpcs="$cpcs xoob"
|
|
fi
|
|
if test "x$btl_openib_have_rdmacm" = "x1" -a \
|
|
"$have_threads" != "none"; then
|
|
cpcs="$cpcs rdmacm"
|
|
fi
|
|
if test "x$btl_openib_have_ibcm" = "x1" -a \
|
|
"$have_threads" != "none"; then
|
|
cpcs="$cpcs ibcm"
|
|
fi
|
|
AC_MSG_CHECKING([which openib btl cpcs will be built])
|
|
AC_MSG_RESULT([$cpcs])])
|
|
|
|
# substitute in the things needed to build openib
|
|
AC_SUBST([btl_openib_CFLAGS])
|
|
AC_SUBST([btl_openib_CPPFLAGS])
|
|
AC_SUBST([btl_openib_LDFLAGS])
|
|
AC_SUBST([btl_openib_LIBS])
|
|
|
|
OMPI_VAR_SCOPE_POP
|
|
])dnl
|