e4c4e7df5e
In order to have an effect, ibv_fork_init should be called in the beginning of the verbs initialization flow - before the calls to the ibv_create_qp and ibv_create_cq verbs. These functions are called from the oob/ud code and by the time the other verbs components (btl openib, pml yalla, ...) call ibv_fork_init, it's too late. This commit forces the call to ibv_fork_init (if it's requested) right at the beginning of all the components that are using verbs. (ibv_fork_init() can be safely called multiple times) This commit also removes the btl_openib_want_fork_support mca parameter and adds a new mca parameter instead - opal_verbs_want_fork_support. Through this new parameter, fork support may be requested for ALL components. The default value for this parameter is set to 1. Before this commit the btl_openib_want_fork_support parameter didn't provide fork support for the openib btl if its value was set to 1. (because when openib called ibv_fork_init, it was already after the calls to ibv_create_* in oob/ud and thereofre it failed).
95 строки
2.5 KiB
C
95 строки
2.5 KiB
C
/*
|
|
* Copyright (c) 2012 Cisco Systems, Inc. All rights reserved.
|
|
*
|
|
* $COPYRIGHT$
|
|
*
|
|
* Additional copyrights may follow
|
|
*
|
|
* $HEADER$
|
|
*/
|
|
|
|
#include "opal_config.h"
|
|
|
|
#include <stdio.h>
|
|
#ifdef HAVE_SYS_TYPES_H
|
|
#include <sys/types.h>
|
|
#endif
|
|
#ifdef HAVE_SYS_STAT_H
|
|
#include <sys/stat.h>
|
|
#endif
|
|
#ifdef HAVE_UNISTD_H
|
|
#include <unistd.h>
|
|
#endif
|
|
|
|
/* This is crummy, but <infiniband/driver.h> doesn't work on all
|
|
platforms with all compilers. Specifically, trying to include it
|
|
on RHEL4U3 with the PGI 32 bit compiler will cause problems because
|
|
certain 64 bit types are not defined. Per advice from Roland D.,
|
|
just include the one prototype that we need in this case
|
|
(ibv_get_sysfs_path()). */
|
|
#include <infiniband/verbs.h>
|
|
#ifdef HAVE_INFINIBAND_DRIVER_H
|
|
#include <infiniband/driver.h>
|
|
#else
|
|
const char *ibv_get_sysfs_path(void);
|
|
#endif
|
|
|
|
#include "common_verbs.h"
|
|
#include "opal/runtime/opal_params.h"
|
|
#include "opal/util/show_help.h"
|
|
#include "opal/util/proc.h"
|
|
|
|
/***********************************************************************/
|
|
|
|
bool opal_common_verbs_check_basics(void)
|
|
{
|
|
#if defined(__linux__)
|
|
int rc;
|
|
char *file;
|
|
struct stat s;
|
|
|
|
/* Check to see if $sysfsdir/class/infiniband/ exists */
|
|
asprintf(&file, "%s/class/infiniband", ibv_get_sysfs_path());
|
|
if (NULL == file) {
|
|
return false;
|
|
}
|
|
rc = stat(file, &s);
|
|
free(file);
|
|
if (0 != rc || !S_ISDIR(s.st_mode)) {
|
|
return false;
|
|
}
|
|
#endif
|
|
|
|
/* It exists and is a directory -- good enough */
|
|
return true;
|
|
}
|
|
|
|
int opal_common_verbs_fork_test(void)
|
|
{
|
|
/* Make sure that ibv_fork_init is called before the calls to other memory registering verbs,
|
|
* which will be called after this function */
|
|
#ifdef HAVE_IBV_FORK_INIT
|
|
if (0 != opal_verbs_want_fork_support) {
|
|
/* Check if fork support is requested by the user */
|
|
if (0 != ibv_fork_init()) {
|
|
/* If the opal_want_fork_support MCA parameter is >0 but
|
|
* the call to ibv_fork_init() failed, then return an error code.
|
|
*/
|
|
if (opal_verbs_want_fork_support > 0) {
|
|
opal_show_help("help-opal-common-verbs.txt",
|
|
"ibv_fork_init fail", true,
|
|
opal_proc_local_get()->proc_hostname, errno,
|
|
strerror(errno));
|
|
return OPAL_ERROR;
|
|
}
|
|
} else {
|
|
return OPAL_SUCCESS;
|
|
}
|
|
} else {
|
|
return OPAL_SUCCESS;
|
|
}
|
|
#endif
|
|
return OPAL_SUCCESS;
|
|
}
|
|
|