ports/openmpi
ports/openmpi
1
1
Форкнуть 0
Код Релизы Активность

Remove stale and unused component

Просмотр исходного кода
Этот коммит содержится в:
Ralph Castain 2015-03-30 09:56:06 -07:00
родитель 0c553c2693
Коммит 79b90a54b6
15 изменённых файлов: 0 добавлений и 5013 удалений
Показать статистику Скачать Patch файл Скачать Diff файл Показать все файлы Свернуть все файлы

0
opal/mca/common/ofacm/.opal_ignore
Просмотреть файл

83
opal/mca/common/ofacm/Makefile.am
Просмотреть файл

@ -1,83 +0,0 @@
#
# Copyright (c) 2009-2012 Mellanox Technologies. All rights reserved.
# Copyright (c) 2009-2014 Cisco Systems, Inc. All rights reserved.
# Copyright (c) 2009-2012 Oak Ridge National Laboratory. All rights reserved.
# $COPYRIGHT$
#
# Additional copyrights may follow
#
# $HEADER$
#
AM_CPPFLAGS = $(common_ofacm_CPPFLAGS)
dist_opaldata_DATA = \
help-mpi-common-ofacm-base.txt \
help-mpi-common-ofacm-oob.txt
headers = \
base.h \
common_ofacm_oob.h \
common_ofacm_empty.h \
connect.h
sources = \
common_ofacm_base.c \
common_ofacm_oob.c \
common_ofacm_empty.c
# If we have XRC support, build XOOB connection module
if MCA_common_ofacm_have_xrc
sources += \
common_ofacm_xoob.c \
common_ofacm_xoob.h
endif
# See opal/mca/common/sm/Makefile.am for an explanation of the
# LTLIBRARIES values listed below.
lib_LTLIBRARIES =
noinst_LTLIBRARIES =
comp_inst = lib@OPAL_LIB_PREFIX@mca_common_ofacm.la
comp_noinst = lib@OPAL_LIB_PREFIX@mca_common_ofacm_noinst.la
if MCA_BUILD_opal_common_ofacm_DSO
lib_LTLIBRARIES += $(comp_inst)
else
noinst_LTLIBRARIES += $(comp_noinst)
endif
lib@OPAL_LIB_PREFIX@mca_common_ofacm_la_SOURCES = $(headers) $(sources)
lib@OPAL_LIB_PREFIX@mca_common_ofacm_la_CPPFLAGS = $(common_ofacm_CPPFLAGS)
lib@OPAL_LIB_PREFIX@mca_common_ofacm_la_LDFLAGS = \
-version-info $(libmca_common_ofacm_so_version) \
$(common_ofacm_LDFLAGS)
lib@OPAL_LIB_PREFIX@mca_common_ofacm_la_LIBADD = $(common_ofacm_LIBS)
lib@OPAL_LIB_PREFIX@mca_common_ofacm_noinst_la_SOURCES = $(headers) $(sources)
# Conditionally install the header files
if WANT_INSTALL_HEADERS
opaldir = $(opalincludedir)/opal/mca/common/ofacm
opal_HEADERS = $(headers)
else
opaldir = $(includedir)
endif
# See opal/mca/common/sm/Makefile.am for an explanation of the
# the *-local rules, below.
V=0
OMPI_V_LN_SCOMP = $(ompi__v_LN_SCOMP_$V)
ompi__v_LN_SCOMP_ = $(ompi__v_LN_SCOMP_$AM_DEFAULT_VERBOSITY)
ompi__v_LN_SCOMP_0 = @echo " LN_S " `basename $(comp_inst)`;
all-local:
$(OMPI_V_LN_SCOMP) if test -z "$(lib_LTLIBRARIES)"; then \
rm -f "$(comp_inst)"; \
$(LN_S) "$(comp_noinst)" "$(comp_inst)"; \
fi
clean-local:
if test -z "$(lib_LTLIBRARIES)"; then \
rm -f "$(comp_inst)"; \
fi

190
opal/mca/common/ofacm/base.h
Просмотреть файл

@ -1,190 +0,0 @@
/*
* Copyright (c) 2007-2008 Cisco Systems, Inc. All rights reserved.
* Copyright (c) 2009-2012 Mellanox Technologies. All rights reserved.
*
* Copyright (c) 2009-2012 Oak Ridge National Laboratory. All rights reserved.
* $COPYRIGHT$
*
* Additional copyrights may follow
*
* $HEADER$
*/
#ifndef OPAL_COMMON_OFACM_BASE_H
#define OPAL_COMMON_OFACM_BASE_H
#include "opal_config.h"
#include <stdio.h>
#include <stdarg.h>
#include "connect.h"
BEGIN_C_DECLS
#define HAVE_XRC (1 == OPAL_HAVE_CONNECTX_XRC)
extern int opal_common_ofacm_base_output;
extern int opal_common_ofacm_base_verbose; /* disabled by default */
/* File for sl data produced only for a 3D-Torus Cluster */
extern char* opal_common_ofacm_three_dim_torus;
static inline int opal_common_ofacm_base_err(const char* fmt, ...)
{
va_list list;
int ret;
va_start(list, fmt);
ret = vfprintf(stderr, fmt, list);
va_end(list);
return ret;
}
#define OFACM_ERROR(args) \
do { \
opal_common_ofacm_base_err("[%s]%s[%s:%d:%s] ", \
opal_proc_local_get()->proc_hostname, \
OPAL_NAME_PRINT(OPAL_PROC_MY_NAME), \
__FILE__, __LINE__, __func__); \
opal_common_ofacm_base_err args; \
opal_common_ofacm_base_err("\n"); \
} while(0);
#if OPAL_ENABLE_DEBUG
#define OFACM_VERBOSE(args) \
do { \
if(opal_common_ofacm_base_verbose > 0) { \
opal_common_ofacm_base_err("[%s]%s[%s:%d:%s] ",\
opal_proc_local_get()->proc_hostname, \
OPAL_NAME_PRINT(OPAL_PROC_MY_NAME), \
__FILE__, __LINE__, __func__); \
opal_common_ofacm_base_err args; \
opal_common_ofacm_base_err("\n"); \
} \
} while(0);
#else
#define OFACM_VERBOSE(args)
#endif
/*
* PUBLIC functions
* ****************
*/
/*
* Open function
*/
OPAL_DECLSPEC int opal_common_ofacm_base_register(mca_base_component_t *base);
/*
* Query CPCs to see if they want to run on a specific port.
* Input:
* port - port information
* Output:
* cpcs - list of availible cpcs
* num_cpcs - number of cpcs
*/
OPAL_DECLSPEC int opal_common_ofacm_base_select_for_local_port
(opal_common_ofacm_base_dev_desc_t *dev,
opal_common_ofacm_base_module_t ***cpcs, int *num_cpcs);
/*
* Select function
* Input:
* local_cpcs - local cpc modules
* num_local_cpcs - number of local cpc modules
* remote_cpc_data - cpc information from remote peer
* remote_cpc_data_count - num of remote information from remote peer
* Output:
* ret_local_cpc - matched cpc module
* ret_remote_cpc_data - matched remote cpc data
*/
OPAL_DECLSPEC int opal_common_ofacm_base_find_match
(opal_common_ofacm_base_module_t **local_cpcs, int num_local_cpcs,
opal_common_ofacm_base_module_data_t *remote_cpc_data, int remote_cpc_data_count,
opal_common_ofacm_base_module_t **ret_local_cpc,
opal_common_ofacm_base_module_data_t **ret_remote_cpc_data);
/*
* Find a CPC's index so that we can send it in the modex
*/
OPAL_DECLSPEC int opal_common_ofacm_base_get_cpc_index
(opal_common_ofacm_base_component_t *cpc);
/*
* Start a new connection to an endpoint
*/
OPAL_DECLSPEC int opal_common_ofacm_base_start_connect
(struct opal_common_ofacm_base_local_connection_context_t *context);
/*
* Component-wide CPC finalize
*/
OPAL_DECLSPEC void opal_common_ofacm_base_finalize(void);
/*
* Component-wide CPC init
*/
OPAL_DECLSPEC int opal_common_ofacm_base_init(void);
/*
* Lookup a CPC by its index (received from the modex)
*/
OPAL_DECLSPEC opal_common_ofacm_base_component_t *
opal_common_ofacm_base_get_cpc_byindex(uint8_t index);
/*
* PRIVATE functions (called only by cpcs)
* ***************************************
*/
/*
* Proc initialization function
*/
void opal_common_ofacm_base_proc_setup
(opal_common_ofacm_base_proc_t *proc,
opal_common_ofacm_base_local_connection_context_t *context,
opal_proc_t *proc_opal);
/*
* Context initialization function
*/
int opal_common_ofacm_base_context_init
(opal_common_ofacm_base_local_connection_context_t *context,
opal_common_ofacm_base_module_t *cpc,
opal_common_ofacm_base_context_connect_cb_fn_t connect_cb,
opal_common_ofacm_base_context_error_cb_fn_t error_cb,
opal_common_ofacm_base_context_prepare_recv_cb_fn_t prepare_recv_cb,
opal_common_ofacm_base_proc_t *proc,
opal_common_ofacm_base_qp_config_t *qp_config,
struct ibv_pd *pd, uint64_t subnet_id, int cpc_type,
uint16_t lid, uint16_t rem_lid,
int32_t user_context_index, void *user_context);
/*
* Remote context initialization.
* Returns operation status
*/
int opal_common_ofacm_base_remote_context_init
(opal_common_ofacm_base_remote_connection_context_t *context,
int num_qps, int num_srqs);
/* Find OFACM proc on specific component */
opal_common_ofacm_base_proc_t* opal_common_ofacm_base_find_proc
(opal_common_ofacm_base_component_t *component, opal_proc_t *proc);
#if 0
/*
* Allocate a CTS frag
*/
int opal_common_ofacm_base_alloc_cts(
struct mca_btl_base_endpoint_t *endpoint);
/*
* Free a CTS frag
*/
int opal_common_ofacm_base_free_cts(
struct mca_btl_base_endpoint_t *endpoint);
#endif
END_C_DECLS
#endif

699
opal/mca/common/ofacm/common_ofacm_base.c
Просмотреть файл

@ -1,699 +0,0 @@
/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
/*
* Copyright (c) 2007-2013 Cisco Systems, Inc. All rights reserved.
* Copyright (c) 2007-2012 Mellanox Technologies. All rights reserved.
*
* Copyright (c) 2009-2012 Oak Ridge National Laboratory. All rights reserved.
* Copyright (c) 2013 Intel, Inc. All rights reserved
* Copyright (c) 2014 Research Organization for Information Science
* and Technology (RIST). All rights reserved.
* Copyright (c) 2015 Los Alamos National Security, LLC. All rights
* reserved.
* $COPYRIGHT$
*
* Additional copyrights may follow
*
* $HEADER$
*/
#include "opal_config.h"
#include <string.h>
#include "base.h"
#include "common_ofacm_oob.h"
#include "common_ofacm_empty.h"
#if HAVE_XRC
#include "common_ofacm_xoob.h"
#endif
#include "opal/constants.h"
#include "opal/class/opal_list.h"
#include "opal/util/argv.h"
#include "opal/util/output.h"
#include "opal/util/show_help.h"
#include "infiniband/verbs.h"
static opal_common_ofacm_base_component_t **available = NULL;
static int num_available = 0;
static char *opal_common_ofacm_cpc_include;
static char *opal_common_ofacm_cpc_exclude;
/* Global variables */
int opal_common_ofacm_base_verbose = 0; /* disabled by default */
char* opal_common_ofacm_three_dim_torus = NULL;
bool cpc_explicitly_defined = false;
int opal_common_ofacm_base_output = 1;
bool opal_common_ofacm_base_register_was_called = false;
bool opal_common_ofacm_base_init_was_called = false;
/*
* Array of all possible connection functions
*/
static opal_common_ofacm_base_component_t *all[] = {
&opal_common_ofacm_oob,
/* Always have an entry here so that the CP indexes will always be
the same: if XRC is not available, use the "empty" CPC */
#if HAVE_XRC
&opal_common_ofacm_xoob,
#else
&opal_common_ofacm_empty,
#endif
NULL
};
static void ofacm_base_proc_contructor (opal_common_ofacm_base_proc_t *proc)
{
proc->proc_opal = NULL;
OBJ_CONSTRUCT(&proc->all_contexts, opal_list_t);
}
static void ofacm_base_proc_destructor (opal_common_ofacm_base_proc_t *proc)
{
OBJ_DESTRUCT(&proc->all_contexts);
}
void opal_common_ofacm_base_proc_setup(opal_common_ofacm_base_proc_t *proc,
opal_common_ofacm_base_local_connection_context_t *context,
opal_proc_t *proc_opal)
{
if (NULL == proc->proc_opal) {
/* first init for the proc, lets set ompi proc */
proc->proc_opal = proc_opal;
}
/* put the context on the proc list */
opal_list_append(&proc->all_contexts, (opal_list_item_t *)context);
}
OBJ_CLASS_INSTANCE(opal_common_ofacm_base_proc_t,
opal_list_item_t,
ofacm_base_proc_contructor,
ofacm_base_proc_destructor);
/* Constructors / Destructors */
static void base_local_context_constructor
(opal_common_ofacm_base_local_connection_context_t *context)
{
context->proc = NULL;
context->state = MCA_COMMON_OFACM_CLOSED;
context->subnet_id = 0;
context->lid = 0;
context->num_of_qps = 0;
context->init_attr = NULL;
context->attr = NULL;
context->ib_pd = NULL;
context->qps = NULL;
context->user_context = NULL;
context->initiator = 0;
context->index = 0;
context->xrc_recv_qp_num = 0;
/* remote info we will allocate and fill during qp creation */
memset(&context->remote_info, 0, sizeof(context->remote_info));
OBJ_CONSTRUCT(&context->context_lock, opal_mutex_t);
}
static void base_local_context_destructor
(opal_common_ofacm_base_local_connection_context_t *context)
{
/* Release remote data arrays */
if (NULL != context->remote_info.rem_qps) {
free(context->remote_info.rem_qps);
}
if (NULL != context->remote_info.rem_srqs) {
free(context->remote_info.rem_srqs);
}
OBJ_DESTRUCT(&context->context_lock);
}
OBJ_CLASS_INSTANCE(opal_common_ofacm_base_local_connection_context_t,
opal_list_item_t,
base_local_context_constructor,
base_local_context_destructor);
int opal_common_ofacm_base_context_init(opal_common_ofacm_base_local_connection_context_t *context,
opal_common_ofacm_base_module_t *cpc,
opal_common_ofacm_base_context_connect_cb_fn_t connect_cb,
opal_common_ofacm_base_context_error_cb_fn_t error_cb,
opal_common_ofacm_base_context_prepare_recv_cb_fn_t prepare_recv_cb,
opal_common_ofacm_base_proc_t *proc,
opal_common_ofacm_base_qp_config_t *qp_config,
struct ibv_pd *pd, uint64_t subnet_id, int cpc_type,
uint16_t lid, uint16_t rem_lid,
int32_t user_context_index, void *user_context)
{
context->proc = proc;
context->cpc = cpc;
context->subnet_id = subnet_id;
context->cpc_type = cpc_type;
context->lid = lid;
context->rem_lid = rem_lid;
context->num_of_qps = qp_config->num_qps;
/* If upper layer defines the QPs we do not want to overwrite it */
if (NULL == context->qps) {
context->qps = calloc(context->num_of_qps, sizeof(opal_common_ofacm_base_qp_t));
if(NULL == context->qps) {
OFACM_ERROR(("Failed to allocate memory for qps"));
return OPAL_ERR_OUT_OF_RESOURCE;
}
}
context->num_of_srqs = qp_config->num_srqs;
context->srq_num = qp_config->srq_num;
context->init_attr = qp_config->init_attr;
context->attr = qp_config->attr;
context->custom_init_attr_mask = qp_config->init_attr_mask;
context->custom_rtr_attr_mask = qp_config->rtr_attr_mask;
context->custom_rts_attr_mask = qp_config->rts_attr_mask;
context->ib_pd = pd;
context->connect_cb = connect_cb;
context->error_cb = error_cb;
context->prepare_recv_cb = prepare_recv_cb ;
context->index = user_context_index;
context->user_context = user_context;
return OPAL_SUCCESS;
}
int opal_common_ofacm_base_remote_context_init(opal_common_ofacm_base_remote_connection_context_t *context,
int num_qps, int num_srqs)
{
context->rem_qps = (opal_common_ofacm_base_rem_qp_info_t *)
calloc(num_qps, sizeof(opal_common_ofacm_base_rem_qp_info_t));
if (NULL == context->rem_qps) {
return OPAL_ERROR;
}
context->rem_srqs = (opal_common_ofacm_base_rem_srq_info_t *)
calloc(num_qps, sizeof(opal_common_ofacm_base_rem_srq_info_t));
if (NULL == context->rem_srqs) {
return OPAL_ERROR;
}
return OPAL_SUCCESS;
}
opal_common_ofacm_base_proc_t* opal_common_ofacm_base_find_proc
(opal_common_ofacm_base_component_t *component, opal_proc_t *proc)
{
opal_common_ofacm_base_proc_t *ret = NULL;
opal_list_item_t *item;
opal_list_t *list = &component->all_procs;
for (item = opal_list_get_first(list);
item != opal_list_get_end(list);
item = opal_list_get_next(item)) {
if (proc == ((opal_common_ofacm_base_proc_t *)item)->proc_opal){
ret = (opal_common_ofacm_base_proc_t *)item;
}
}
return ret;
}
/*
* Register MCA parameters
*/
int opal_common_ofacm_base_register(mca_base_component_t *base)
{
int i, j, save;
char **temp = NULL, *string = NULL, *all_cpc_names = NULL;
if (opal_common_ofacm_base_register_was_called) {
return OPAL_SUCCESS;
}
opal_common_ofacm_base_register_was_called = true;
/* Make an MCA parameter to select which connect module to use */
for (i = 0; NULL != all[i]; ++i) {
/* The CPC name "empty" is reserved for "fake" CPC modules */
if (0 != strcmp(all[i]->cbc_name, "empty")) {
opal_argv_append_nosize(&temp, all[i]->cbc_name);
}
}
all_cpc_names = opal_argv_join(temp, ',');
opal_argv_free(temp);
asprintf(&string,
"Method used to select OpenFabrics connections (valid values: %s)",
all_cpc_names);
opal_common_ofacm_cpc_include = NULL;
(void) mca_base_component_var_register(base, "ofacm_cpc_include", string,
MCA_BASE_VAR_TYPE_STRING, NULL, 0, 0,
OPAL_INFO_LVL_9,
MCA_BASE_VAR_SCOPE_READONLY,
&opal_common_ofacm_cpc_include);
free(string);
asprintf(&string,
"Method used to exclude OpenFabrics connections (valid values: %s)",
all_cpc_names);
opal_common_ofacm_cpc_exclude = NULL;
(void) mca_base_component_var_register(base, "ofacm_cpc_exclude", string,
MCA_BASE_VAR_TYPE_STRING, NULL, 0, 0,
OPAL_INFO_LVL_9,
MCA_BASE_VAR_SCOPE_READONLY,
&opal_common_ofacm_cpc_exclude);
free(string);
/* Register the name of the file containing the fabric's Service Levels (SL) */
opal_common_ofacm_three_dim_torus = NULL;
(void) mca_base_var_register("ompi", "common", "ofacm", "three_dim_torus",
"The name of the file contating Service Level (SL) data for 3D-Torus cluster",
MCA_BASE_VAR_TYPE_STRING, NULL, 0, 0,
OPAL_INFO_LVL_9,
MCA_BASE_VAR_SCOPE_READONLY,
&opal_common_ofacm_three_dim_torus);
opal_common_ofacm_base_verbose = 0;
(void) mca_base_var_register("ompi", "common", "ofacm", "base_verbose",
"Verbosity level of the OFACM framework",
MCA_BASE_VAR_TYPE_INT, NULL, 0, 0,
OPAL_INFO_LVL_9,
MCA_BASE_VAR_SCOPE_READONLY,
&opal_common_ofacm_base_verbose);
/* Parse the if_[in|ex]clude paramters to come up with a list of
CPCs that are available */
available = calloc(1, sizeof(all));
/* If we have an "include" list, then find all those CPCs and put
them in available[] */
if (NULL != opal_common_ofacm_cpc_include) {
cpc_explicitly_defined = true;
temp = opal_argv_split(opal_common_ofacm_cpc_include, ',');
for (save = j = 0; NULL != temp[j]; ++j) {
for (i = 0; NULL != all[i]; ++i) {
if (0 == strcmp(temp[j], all[i]->cbc_name)) {
OFACM_VERBOSE(("include: saving %s", all[i]->cbc_name));
available[save++] = all[i];
++num_available;
break;
}
}
if (NULL == all[i]) {
opal_show_help("help-mpi-common-ofacm-base.txt",
"cpc name not found", true,
"include", opal_proc_local_get()->proc_hostname,
"include", opal_common_ofacm_cpc_include, temp[j],
all_cpc_names);
opal_argv_free(temp);
free(all_cpc_names);
return OPAL_ERR_NOT_FOUND;
}
}
opal_argv_free(temp);
}
/* Otherwise, if we have an "exclude" list, take all the CPCs that
are not in that list and put them in available[] */
else if (NULL != opal_common_ofacm_cpc_exclude) {
cpc_explicitly_defined = true;
temp = opal_argv_split(opal_common_ofacm_cpc_exclude, ',');
/* First: error check -- ensure that all the names are valid */
for (j = 0; NULL != temp[j]; ++j) {
for (i = 0; NULL != all[i]; ++i) {
if (0 == strcmp(temp[j], all[i]->cbc_name)) {
break;
}
}
if (NULL == all[i]) {
opal_show_help("help-mpi-common-ofacm-base.txt",
"cpc name not found", true,
"exclude", opal_proc_local_get()->proc_hostname,
"exclude", opal_common_ofacm_cpc_exclude, temp[j],
all_cpc_names);
opal_argv_free(temp);
free(all_cpc_names);
return OPAL_ERR_NOT_FOUND;
}
}
/* Now do the exclude */
for (save = i = 0; NULL != all[i]; ++i) {
for (j = 0; NULL != temp[j]; ++j) {
if (0 == strcmp(temp[j], all[i]->cbc_name)) {
break;
}
}
if (NULL == temp[j]) {
OFACM_VERBOSE(("exclude: saving %s", all[i]->cbc_name));
available[save++] = all[i];
++num_available;
}
}
opal_argv_free(temp);
}
/* If there's no include/exclude list, copy all[] into available[] */
else {
OFACM_VERBOSE(("no include or exclude: saving all"));
memcpy(available, all, sizeof(all));
num_available = (sizeof(all) /
sizeof(opal_common_ofacm_base_module_t *)) - 1;
}
/* Call the register function on all the CPCs so that they may
setup any MCA params specific to the connection type */
for (i = 0; NULL != available[i]; ++i) {
if (NULL != available[i]->cbc_register) {
available[i]->cbc_register();
}
}
return OPAL_SUCCESS;
}
/*
* Called once during openib BTL component initialization to allow CPC
* components to initialize.
*/
int opal_common_ofacm_base_init(void)
{
int i, rc;
if (opal_common_ofacm_base_init_was_called) {
return OPAL_SUCCESS;
}
opal_common_ofacm_base_init_was_called = true;
/* Call each available CPC component's open function, if it has
one. If the CPC component open function returns OPAL_SUCCESS,
keep it. If it returns ERR_NOT_SUPPORTED, remove it from the
available[] array. If it returns something else, return that
error upward. */
for (i = num_available = 0; NULL != available[i]; ++i) {
if (NULL == available[i]->cbc_init) {
available[num_available++] = available[i];
OFACM_VERBOSE(("found available cpc (NULL init): %s",
all[i]->cbc_name));
continue;
}
rc = available[i]->cbc_init();
if (OPAL_SUCCESS == rc) {
available[num_available++] = available[i];
OFACM_VERBOSE(("found available cpc (SUCCESS init): %s",
all[i]->cbc_name));
continue;
} else if (OPAL_ERR_NOT_SUPPORTED == rc) {
continue;
} else {
return rc;
}
}
available[num_available] = NULL;
return (num_available > 0) ? OPAL_SUCCESS : OPAL_ERR_NOT_AVAILABLE;
}
/*
* Find all the CPCs that are eligible for a single local port (i.e.,
* openib module).
*/
int opal_common_ofacm_base_select_for_local_port(opal_common_ofacm_base_dev_desc_t *dev,
opal_common_ofacm_base_module_t ***cpcs, int *num_cpcs)
{
char *msg = NULL;
int i, rc, cpc_index, len;
opal_common_ofacm_base_module_t **tmp_cpcs;
tmp_cpcs = calloc(num_available,
sizeof(opal_common_ofacm_base_module_t *));
if (NULL == tmp_cpcs) {
return OPAL_ERR_OUT_OF_RESOURCE;
}
/* Go through all available CPCs and query them to see if they
want to run on this module. If they do, save them to a running
array. */
for (len = 1, i = 0; NULL != available[i]; ++i) {
len += strlen(available[i]->cbc_name) + 2;
}
msg = malloc(len);
if (NULL == msg) {
return OPAL_ERR_OUT_OF_RESOURCE;
}
msg[0] = '\0';
for (cpc_index = i = 0; NULL != available[i]; ++i) {
if (i > 0) {
strcat(msg, ", ");
}
strcat(msg, available[i]->cbc_name);
rc = available[i]->cbc_query(dev, &tmp_cpcs[cpc_index]);
if (OPAL_ERR_NOT_SUPPORTED == rc || OPAL_ERR_UNREACH == rc) {
continue;
} else if (OPAL_SUCCESS != rc) {
free(tmp_cpcs);
free(msg);
return rc;
}
OFACM_VERBOSE(("match cpc for local port: %s",
available[i]->cbc_name));
/* If the CPC wants to use the CTS protocol, check to ensure
that QP 0 is PP; if it's not, we can't use this CPC (or the
CTS protocol) */
/* Pasha: Wrong place to check qp type, should be moved to CMs
if (cpcs[cpc_index]->cbm_uses_cts &&
!BTL_OPENIB_QP_TYPE_PP(0)) {
OFACM_VERBOSE(("this CPC only supports when the first btl_openib_receive_queues QP is a PP QP"));
continue;
}
*/
/* This CPC has indicated that it wants to run on this openib
BTL module. Woo hoo! */
++cpc_index;
}
/* If we got an empty array, then no CPCs were eligible. Doh! */
if (0 == cpc_index) {
opal_show_help("help-mpi-common-ofacm-base.txt",
"no cpcs for port", true,
opal_proc_local_get()->proc_hostname,
ibv_get_device_name(dev->ib_dev),
msg);
free(tmp_cpcs);
free(msg);
return OPAL_ERR_NOT_SUPPORTED;
}
free(msg);
/* We got at least one eligible CPC; save the array into the
module's port_info */
*num_cpcs = cpc_index;
*cpcs = tmp_cpcs;
return OPAL_SUCCESS;
}
/*
* This function is invoked when determining whether we have a CPC in
* common with a specific remote port. We already know that the
* subnet ID is the same between a specific local port and the target
* remote port; now we need to know if we can find a CPC in common
* between the two.
*
* If yes, be sure to find the *same* CPC on both sides. We know
* which CPCs are available on each side, and we know the priorities
* that were assigned on both sides. So find a CPC that is common to
* both sides and has the highest overall priority (between both
* sides).
*
* Return the matching CPC, or NULL if not found.
*/
int
opal_common_ofacm_base_find_match(opal_common_ofacm_base_module_t **local_cpcs, int num_local_cpcs,
opal_common_ofacm_base_module_data_t *remote_cpc_data, int remote_cpc_data_count,
opal_common_ofacm_base_module_t **ret_local_cpc,
opal_common_ofacm_base_module_data_t **ret_remote_cpc_data)
{
int i, j, max = -1;
opal_common_ofacm_base_module_t *local_cpc, *local_selected = NULL;
opal_common_ofacm_base_module_data_t *local_cpcd, *remote_cpcd,
*remote_selected = NULL;
/* Iterate over all the CPCs on the local module */
for (i = 0; i < num_local_cpcs; ++i) {
local_cpc = local_cpcs[i];
local_cpcd = &(local_cpc->data);
/* Iterate over all the CPCs on the remote port */
for (j = 0; j < remote_cpc_data_count; ++j) {
remote_cpcd = &(remote_cpc_data[j]);
/* Are the components the same? */
if (local_cpcd->cbm_component == remote_cpcd->cbm_component) {
/* If so, update the max priority found so far */
if (max < local_cpcd->cbm_priority) {
max = local_cpcd->cbm_priority;
local_selected = local_cpc;
remote_selected = remote_cpcd;
}
if (max < remote_cpcd->cbm_priority) {
max = remote_cpcd->cbm_priority;
local_selected = local_cpc;
remote_selected = remote_cpcd;
}
}
}
}
/* All done! */
if (NULL != local_selected) {
*ret_local_cpc = local_selected;
*ret_remote_cpc_data = remote_selected;
OFACM_VERBOSE(("find_match: found match!"));
return OPAL_SUCCESS;
} else {
OFACM_VERBOSE(("find_match: did NOT find match!"));
return OPAL_ERR_NOT_FOUND;
}
}
/*
* Lookup a CPC component's index in the all[] array so that we can
* send it int the modex
*/
int opal_common_ofacm_base_get_cpc_index(opal_common_ofacm_base_component_t *cpc)
{
int i;
for (i = 0; NULL != all[i]; ++i) {
if (all[i] == cpc) {
return i;
}
}
/* Not found */
return -1;
}
/*
* Lookup a CPC by its index (received from the modex)
*/
opal_common_ofacm_base_component_t *
opal_common_ofacm_base_get_cpc_byindex(uint8_t index)
{
return (index >= (sizeof(all) /
sizeof(opal_common_ofacm_base_module_t *))) ?
NULL : all[index];
}
/*
* This function we never call from BTL - so it is no reason to expose it
* in base.
*/
#if 0
int opal_common_ofacm_base_alloc_cts(mca_btl_base_endpoint_t *endpoint)
{
opal_free_list_item_t *fli;
int length = sizeof(mca_btl_openib_header_t) +
sizeof(mca_btl_openib_header_coalesced_t) +
sizeof(mca_btl_openib_control_header_t) +
sizeof(mca_btl_openib_footer_t) +
mca_btl_openib_component.qp_infos[mca_btl_openib_component.credits_qp].size;
/* Explicitly don't use the mpool registration */
fli = &(endpoint->endpoint_cts_frag.super.super.base.super);
fli->registration = NULL;
fli->ptr = malloc(length);
if (NULL == fli->ptr) {
BTL_ERROR(("malloc failed"));
return OPAL_ERR_OUT_OF_RESOURCE;
}
endpoint->endpoint_cts_mr =
ibv_reg_mr(endpoint->endpoint_btl->device->ib_pd,
fli->ptr, length,
IBV_ACCESS_LOCAL_WRITE | IBV_ACCESS_REMOTE_WRITE |
IBV_ACCESS_REMOTE_READ);
OPAL_OUTPUT((-1, "registered memory %p, length %d", fli->ptr, length));
if (NULL == endpoint->endpoint_cts_mr) {
free(fli->ptr);
BTL_ERROR(("Failed to reg mr!"));
return OPAL_ERR_OUT_OF_RESOURCE;
}
/* Copy the lkey where it needs to go */
endpoint->endpoint_cts_frag.super.sg_entry.lkey =
endpoint->endpoint_cts_frag.super.super.segment.seg_key.key32[0] =
endpoint->endpoint_cts_mr->lkey;
endpoint->endpoint_cts_frag.super.sg_entry.length = length;
/* Construct the rest of the recv_frag_t */
OBJ_CONSTRUCT(&(endpoint->endpoint_cts_frag), mca_btl_openib_recv_frag_t);
endpoint->endpoint_cts_frag.super.super.base.order =
mca_btl_openib_component.credits_qp;
endpoint->endpoint_cts_frag.super.endpoint = endpoint;
OPAL_OUTPUT((-1, "Got a CTS frag for peer %s, addr %p, length %d, lkey %d",
(NULL == endpoint->endpoint_proc->proc_opal->proc_hostname) ?
"unknown" : endpoint->endpoint_proc->proc_opal->proc_hostname,
(void*) endpoint->endpoint_cts_frag.super.sg_entry.addr,
endpoint->endpoint_cts_frag.super.sg_entry.length,
endpoint->endpoint_cts_frag.super.sg_entry.lkey));
return OPAL_SUCCESS;
}
#endif
/* This function is needed for CTS packet release on completion..
* and it is bad idea...it is 2 possible solutions:
* - make the send operation blocking (simple and not optimal).
* - rdmacm should add own progress function (best but not trivial).
*/
#if 0
int opal_common_ofacm_base_free_cts(mca_btl_base_endpoint_t *endpoint)
{
if (NULL != endpoint->endpoint_cts_mr) {
ibv_dereg_mr(endpoint->endpoint_cts_mr);
endpoint->endpoint_cts_mr = NULL;
}
if (NULL != endpoint->endpoint_cts_frag.super.super.base.super.ptr) {
free(endpoint->endpoint_cts_frag.super.super.base.super.ptr);
endpoint->endpoint_cts_frag.super.super.base.super.ptr = NULL;
OPAL_OUTPUT((-1, "Freeing CTS frag"));
}
return OPAL_SUCCESS;
}
#endif
/*
* Called to start a connection
*/
int opal_common_ofacm_base_start_connect(
opal_common_ofacm_base_local_connection_context_t *context)
{
#if 0
/* If the CPC uses the CTS protocol, provide a frag buffer for the
CPC to post. Must allocate these frags up here in the main
thread because the FREE_LIST_WAIT is not thread safe. */
if (cpc->cbm_uses_cts) {
int rc;
rc = opal_common_ofacm_base_alloc_cts(context);
if (OPAL_SUCCESS != rc) {
return rc;
}
}
#endif
return context->cpc->cbm_start_connect(context);
}
/*
* Called during openib btl component close
*/
void opal_common_ofacm_base_finalize(void)
{
int i;
if (NULL != available) {
for (i = 0; NULL != available[i]; ++i) {
if (NULL != available[i]->cbc_finalize) {
available[i]->cbc_finalize();
}
}
free(available);
available = NULL;
}
}

48
opal/mca/common/ofacm/common_ofacm_empty.c
Просмотреть файл

@ -1,48 +0,0 @@
/*
* Copyright (c) 2008 Cisco Systems, Inc. All rights reserved.
*
* Copyright (c) 2009-2012 Oak Ridge National Laboratory. All rights reserved.
* Copyright (c) 2009-2012 Mellanox Technologies. All rights reserved.
* $COPYRIGHT$
*
* Additional copyrights may follow
*
* $HEADER$
*/
#include "opal_config.h"
#include "base.h"
#include "connect.h"
#include "opal/constants.h"
static void empty_component_register(void);
static int empty_component_init(void);
static int empty_component_query(opal_common_ofacm_base_dev_desc_t *dev,
opal_common_ofacm_base_module_t **cpc);
opal_common_ofacm_base_component_t opal_common_ofacm_empty = {
"empty",
empty_component_register,
empty_component_init,
empty_component_query,
NULL
};
static void empty_component_register(void)
{
/* Nothing to do */
}
static int empty_component_init(void)
{
/* Never let this CPC run */
return OPAL_ERR_NOT_SUPPORTED;
}
static int empty_component_query(opal_common_ofacm_base_dev_desc_t *dev,
opal_common_ofacm_base_module_t **cpc)
{
/* Never let this CPC run */
return OPAL_ERR_NOT_SUPPORTED;
}

22
opal/mca/common/ofacm/common_ofacm_empty.h
Просмотреть файл

@ -1,22 +0,0 @@
/*
* Copyright (c) 2007-2008 Cisco Systems, Inc. All rights reserved.
*
* Copyright (c) 2009-2012 Oak Ridge National Laboratory. All rights reserved.
* Copyright (c) 2009-2012 Mellanox Technologies. All rights reserved.
* $COPYRIGHT$
*
* Additional copyrights may follow
*
* $HEADER$
*/
#ifndef BTL_OPENIB_CONNECT_EMPTY_H
#define BTL_OPENIB_CONNECT_EMPTY_H
#include "opal_config.h"
#include "connect.h"
extern opal_common_ofacm_base_component_t opal_common_ofacm_empty;
#endif

1672
opal/mca/common/ofacm/common_ofacm_oob.c
Просмотреть файл

Разница между файлами не показана из-за своего большого размера Загрузить разницу

20
opal/mca/common/ofacm/common_ofacm_oob.h
Просмотреть файл

@ -1,20 +0,0 @@
/*
* Copyright (c) 2007-2008 Cisco Systems, Inc. All rights reserved.
* Copyright (c) 2009-2012 Mellanox Technologies. All rights reserved.
*
* Copyright (c) 2009-2012 Oak Ridge National Laboratory. All rights reserved.
* $COPYRIGHT$
*
* Additional copyrights may follow
*
* $HEADER$
*/
#ifndef COMMON_OFACM_OOB_H
#define COMMON_OFACM_OOB_H
#include "connect.h"
extern opal_common_ofacm_base_component_t opal_common_ofacm_oob;
#endif

1539
opal/mca/common/ofacm/common_ofacm_xoob.c
Просмотреть файл

Разница между файлами не показана из-за своего большого размера Загрузить разницу

69
opal/mca/common/ofacm/common_ofacm_xoob.h
Просмотреть файл

@ -1,69 +0,0 @@
/*
* Copyright (c) 2007-2012 Mellanox Technologies. All rights reserved.
* Copyright (c) 2008 Cisco Systems, Inc. All rights reserved.
*
* Copyright (c) 2009-2012 Oak Ridge National Laboratory. All rights reserved.
* $COPYRIGHT$
*
* Additional copyrights may follow
*
* $HEADER$
*/
#ifndef COMMON_OFACM_XOOB_H
#define COMMON_OFACM_XOOB_H
#include "opal/class/opal_hash_table.h"
#include "connect.h"
extern opal_common_ofacm_base_component_t opal_common_ofacm_xoob;
typedef enum {
XOOB_ADDR_CONNECTING = 100,
XOOB_ADDR_CONNECTED,
XOOB_ADDR_CLOSED
} opal_common_ofacm_ib_addr_state_t;
struct ofacm_ib_address_t {
opal_list_item_t super;
void *key; /* the key with size 80bit - [subnet(64) LID(16bit)] */
uint64_t subnet_id; /* caching subnet_id */
uint16_t lid; /* caching lid */
opal_list_t pending_contexts; /* list of endpoints that use this ib_address */
struct opal_common_ofacm_base_qp_t *qps; /* pointer to qp that will be used
for communication with the
destination */
uint32_t remote_xrc_rcv_qp_num; /* remote xrc qp number */
opal_mutex_t addr_lock; /* protection */
opal_common_ofacm_ib_addr_state_t status; /* ib port status */
};
typedef struct ofacm_ib_address_t
ofacm_ib_address_t;
struct opal_common_ofacm_xoob_local_connection_context_t {
opal_common_ofacm_base_local_connection_context_t super;
ofacm_ib_address_t *addr;
uint32_t xrc_recv_qp_num; /* in xrc we will use it as recv qp */
uint32_t xrc_recv_psn;
};
typedef struct opal_common_ofacm_xoob_local_connection_context_t
opal_common_ofacm_xoob_local_connection_context_t;
OPAL_DECLSPEC OBJ_CLASS_DECLARATION(opal_common_ofacm_xoob_local_connection_context_t);
struct opal_common_ofacm_xoob_module_t {
opal_common_ofacm_base_module_t super;
opal_hash_table_t ib_addr_table; /**< used only for xrc.hash-table that
keeps table of all lids/subnets */
};
typedef struct opal_common_ofacm_xoob_module_t
opal_common_ofacm_xoob_module_t;
struct pending_context_t {
opal_list_item_t super;
opal_common_ofacm_xoob_local_connection_context_t *xcontext;
};
typedef struct pending_context_t
pending_context_t;
OPAL_DECLSPEC OBJ_CLASS_DECLARATION(pending_context_t);
#endif

61
opal/mca/common/ofacm/configure.m4
Просмотреть файл

@ -1,61 +0,0 @@
# -*- shell-script -*-
#
# Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
# University Research and Technology
# Corporation. All rights reserved.
# Copyright (c) 2004-2005 The University of Tennessee and The University
# of Tennessee Research Foundation. All rights
# reserved.
# Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
# University of Stuttgart. All rights reserved.
# Copyright (c) 2004-2005 The Regents of the University of California.
# All rights reserved.
# Copyright (c) 2007-2009 Cisco Systems, Inc. All rights reserved.
# Copyright (c) 2009-2012 Mellanox Technologies. All rights reserved.
# Copyright (c) 2009-2012 Oak Ridge National Laboratory. All rights reserved.
# $COPYRIGHT$
#
# Additional copyrights may follow
#
# $HEADER$
#
# MCA_opal_common_ofacm_CONFIG([should_build])
# ------------------------------------------
AC_DEFUN([MCA_opal_common_ofacm_POST_CONFIG], [
AM_CONDITIONAL([MCA_common_ofacm_have_xrc], [test $1 -eq 1 -a "x$common_ofacm_have_xrc" = "x1"])
])
# MCA_opal_common_ofacm_CONFIG([action-if-can-compile],
# [action-if-cant-compile])
# ------------------------------------------------
AC_DEFUN([MCA_opal_common_ofacm_CONFIG],[
AC_CONFIG_FILES([opal/mca/common/ofacm/Makefile])
OPAL_VAR_SCOPE_PUSH([modules ofacm_have_threads])
modules="oob"
common_ofacm_happy="no"
OPAL_CHECK_OPENFABRICS([common_ofacm],
[common_ofacm_happy="yes"
OPAL_CHECK_OPENFABRICS_CM([common_ofacm])])
AS_IF([test "$common_ofacm_happy" = "yes"],
[$1],
[$2])
AS_IF([test "$common_ofacm_happy" = "yes"],
[if test "x$common_ofacm_have_xrc" = "x1"; then
modules="$modules xoob"
fi
AC_MSG_CHECKING([which OpenFabrics CM modules will be built])
AC_MSG_RESULT([$modules])])
# substitute in the things needed to build openib
AC_SUBST([common_ofacm_CFLAGS])
AC_SUBST([common_ofacm_CPPFLAGS])
AC_SUBST([common_ofacm_LDFLAGS])
AC_SUBST([common_ofacm_LIBS])
OPAL_VAR_SCOPE_POP
])dnl

542
opal/mca/common/ofacm/connect.h
Просмотреть файл

@ -1,542 +0,0 @@
/*
* Copyright (c) 2007-2008 Cisco Systems, Inc. All rights reserved.
* Copyright (c) 2009 Mellanox Technogies, Inc. All rights reserved.
*
* Copyright (c) 2009-2012 Oak Ridge National Laboratory. All rights reserved.
* Copyright (c) 2009-2012 Mellanox Technologies. All rights reserved.
* $COPYRIGHT$
*
* Additional copyrights may follow
*
* $HEADER$
*/
/**
* @file
*
* This interface is designed to hide the back-end details of how IB
* RC connections are made from the rest of the openib BTL. There are
* module-like instances of the implemented functionality (dlopen and
* friends are not used, but all the functionality is accessed through
* struct's of function pointers, so you can swap between multiple
* different implementations at run time, just like real components).
* Hence, these entities are referred to as "Connect
* Pseudo-Components" (CPCs).
*
* The CPCs are referenced by their names (e.g., "oob", "rdma_cm").
*
* CPCs are split into components and modules, similar to all other
* MCA frameworks in this code base.
*
* Before diving into the CPC interface, let's discuss some
* terminology and mappings of data structures:
*
* - a BTL module represents a network port (in the case of the openib
* BTL, a LID)
* - a CPC module represents one way to make connections to a BTL module
* - hence, a BTL module has potentially multiple CPC modules
* associated with it
* - an endpoint represnts a connection between a local BTL module and
* a remote BTL module (in the openib BTL, because of BSRQ, an
* endpoint can contain multiple QPs)
* - when an endpoint is created, one of the CPC modules associated
* with the local BTL is selected and associated with the endpoint
* (obviously, it is a CPC module that is common between the local
* and remote BTL modules)
* - endpoints may be created and destroyed during the MPI job
* - endpoints are created lazily, during the first communication
* between two peers
* - endpoints are destroyed when two MPI processes become
* disconnected (e.g., MPI-2 dynamics or MPI_FINALIZE)
* - hence, BTL modules and CPC modules outlive endpoints.
* Specifically, BTL modules and CPC modules live from MPI_INIT to
* MPI_FINALIZE. endpoints come and go as MPI semantics demand it.
* - therefore, CPC modules need to cache information on endpoints that
* are specific to that connection.
*
* Component interface:
*
* - component_register(): The openib BTL's component_open() function
* calls the connect_base_register() function, which scans all
* compiled-in CPC's. If they have component_register() functions,
* they are called (component_register() functions are only allowed to
* register MCA parameters).
*
* NOTE: The connect_base_register() function will process the
* btl_openib_cpc_include and btl_openib_cpc_exclude MCA parameters
* and automatically include/exclude CPCs as relevant. If a CPC is
* excluded, none of its other interface functions will be invoked for
* the duration of the process.
*
* - component_init(): The openib BTL's component_init() function
* calls connect_base_init(), which will invoke this query function on
* each CPC to see if it wants to run at all. CPCs can gracefully
* remove themselves from consideration in this process by returning
* OPAL_ERR_NOT_SUPPORTED.
*
* - component_query(): The openib BTL's init_one_port() calls the
* connect_base_select_for_local_port() function, which, for each LID
* on that port, calls the component_query() function on every
* available CPC on that LID. This function is intended to see if a
* CPC can run on a sepcific openib BTL module (i.e., LID). If it
* can, the CPC is supposed to create a CPC module that is specific to
* that BTL/LID and return it. If it cannot, it should return
* OPAL_ERR_NOT_SUPPORTED and be gracefully skipped for this
* OpenFabrics port.
*
* component_finalize(): The openib BTL's component_close() function
* calls connect_base_finalize(), which, in turn, calls the
* component_finalize() function on all available CPCs. Note that all
* CPC modules will have been finalized by this point; the CPC
* component_finalize() function is a chance for the CPC to clean up
* any component-specific resources.
*
* Module interface:
*
* cbm_component member: A pointer pointing to the single, global
* instance of the CPC component. This member is used for creating a
* unique index representing the modules' component so that it can be
* shared with remote peer processes.
*
* cbm_priority member: An integer between 0 and 100, inclusive,
* representing the priority of this CPC.
*
* cbm_modex_message member: A pointer to a blob buffer that will be
* included in the modex message for this port for this CPC (it is
* assumed that this blob is a) only understandable by the
* corresponding CPC in the peer process, and b) contains specific
* addressing/contact information for *this* port's CPC module).
*
* cbm_modex_message_len member: The length of the cbm_modex_message
* blob, in bytes.
*
* cbm_endpoint_init(): Called during endpoint creation, allowing a
* CPC module to cache information on the endpoint. A pointer to the
* endpoint's CPC module is already cached on the endpoint.
*
* cbm_start_connect(): initiate a connection to a remote peer. The
* CPC is responsible for setting itself up for asyncronous operation
* for progressing the outgoing connection request.
*
* cbm_endpoint_finalize(): Called during the endpoint destrouction,
* allowing the CPC module to destroy anything that it cached on the
* endpoint.
*
* cbm_finalize(): shut down all asynchronous handling and clean up
* any state that was setup for this CPC module/BTL. Some CPCs setup
* asynchronous support on a per-HCA/NIC basis (vs. per-port/LID). It
* is the reponsibility of the CPC to figure out such issues (e.g.,
* via reference counting) -- there is no notification from the
* upper-level BTL about when an entire HCA/NIC is no longer being
* used. There is only this function, which tells when a specific
* CPC/BTL module is no longer being used.
*
* cbm_uses_cts: a bool that indicates whether the CPC will use the
* CTS protocol or not.
* - if true: the CPC will post the fragment on
* endpoint->endpoint_cts_frag as a receive buffer and will *not*
* call opal_btl_openib_post_recvs().
* - if false: the CPC will call opal_btl_openib_post_recvs() before
* calling opal_btl_openib_cpc_complete().
*
* There are two functions in the main openib BTL that the CPC may
* call:
*
* - opal_btl_openib_post_recvs(endpoint): once a QP is locally
* connected to the remote side (but we don't know if the remote side
* is connected to us yet), this function is invoked to post buffers
* on the QP, setup credits for the endpoint, etc. This function is
* *only* invoked if the CPC's cbm_uses_cts is false.
*
* - opal_btl_openib_cpc_complete(endpoint): once that a CPC knows
* that a QP is connected on *both* sides, this function is invoked to
* tell the main openib BTL "ok, you can use this connection now."
* (e.g., the main openib BTL will either invoke the CTS protocol or
* start sending out fragments that were queued while the connection
* was establishing, etc.).
*/
#ifndef OPAL_COMMON_OFACM_CONNECT_H
#define OPAL_COMMON_OFACM_CONNECT_H
/* System includes */
#include <infiniband/verbs.h>
#include "opal/threads/mutex.h"
#include "opal/class/opal_list.h"
#include "opal/util/proc.h"
BEGIN_C_DECLS
#define BCF_MAX_NAME 64
/**
* Must forward declare these structs to avoid include file loops.
*/
/**
* This is struct is defined below
*/
struct opal_common_ofacm_base_module_t;
/* special capabilities */
#define OPAL_COMMON_OFACM_XRC_ONLY 1
#define OPAL_COMMON_OFACM_IWARP_ONLY 1 << 1
/**
* State of OFACM connection.
*/
typedef enum {
/* Defines the state in which this BTL instance
* has started the process of connection */
MCA_COMMON_OFACM_CONNECTING,
/* Waiting for ack from endpoint */
MCA_COMMON_OFACM_CONNECT_ACK,
/*Waiting for final connection ACK from endpoint */
MCA_COMMON_OFACM_WAITING_ACK,
/* Connected ... both sender & receiver have
* buffers associated with this connection */
MCA_COMMON_OFACM_CONNECTED,
/* Connection is closed, there are no resources
* associated with this */
MCA_COMMON_OFACM_CLOSED,
/* Maximum number of retries have been used.
* Report failure on send to upper layer */
MCA_COMMON_OFACM_FAILED,
/* We found is useful to have one more
* state that maybe utilized for user needs */
MCA_COMMON_OFACM_USER_CUSTOM
} opal_common_ofacm_connection_state_t;
typedef enum {
MCA_COMMON_OFACM_BTL = 0,
MCA_COMMON_OFACM_COLL = 100
} opal_common_ofacm_type;
typedef struct opal_common_ofacm_base_dev_desc_t {
struct ibv_device* ib_dev; /* device */
struct ibv_context* ib_dev_context; /* device context */
int capabilities; /* Special capabilities like: XRC, Iwarp, etc.. */
} opal_common_ofacm_base_dev_desc_t;
/* QPs configuration container that should be filled by
* upper layer, for example - btl */
typedef struct opal_common_ofacm_base_qp_config_t {
int num_qps;
int num_srqs;
struct ibv_qp_init_attr *init_attr;
struct ibv_qp_attr *attr;
uint32_t *srq_num;
uint32_t *init_attr_mask;
uint32_t *rtr_attr_mask;
uint32_t *rts_attr_mask;
} opal_common_ofacm_base_qp_config_t;
/* QP base data */
typedef struct opal_common_ofacm_base_qp_t {
struct ibv_qp *lcl_qp;
size_t ib_inline_max; /**< max size of IB inline send */
uint32_t lcl_psn;
int32_t sd_wqe; /**< number of available send wqe entries */
int users;
opal_mutex_t lock;
} opal_common_ofacm_base_qp_t;
/* Remote QP info */
typedef struct opal_common_ofacm_base_rem_qp_info_t {
uint32_t rem_qp_num;
/* Remote QP number */
uint32_t rem_psn;
/* Remote processes port sequence number */
} opal_common_ofacm_base_rem_qp_info_t;
/* Remote SRQ info */
typedef struct opal_common_ofacm_base_rem_srq_info_t {
/* Remote SRQ number */
uint32_t rem_srq_num;
} opal_common_ofacm_base_rem_srq_info_t;
/* Remote connection context */
typedef struct opal_common_ofacm_base_remote_connection_context_t {
opal_object_t super;
/* Local identifier of the remote process */
uint16_t rem_lid;
/* subnet id of remote process */
uint64_t rem_subnet_id;
/* MTU of remote process */
uint32_t rem_mtu; /* TBD: not sure that i need this one */
/* index of remote endpoint in endpoint array */
uint32_t rem_index; /* TBD: the index we use as immidiate data */
/* Remote QPs */
opal_common_ofacm_base_rem_qp_info_t *rem_qps;
/* Remote xrc_srq info, used only with XRC connections */
opal_common_ofacm_base_rem_srq_info_t *rem_srqs;
} opal_common_ofacm_base_remote_connection_context_t;
typedef struct opal_common_ofacm_base_proc_t {
opal_list_item_t super;
opal_proc_t *proc_opal; /* target proc */
opal_list_t all_contexts; /* list of all contexts connected to
this endpoint*/
} opal_common_ofacm_base_proc_t;
OPAL_DECLSPEC OBJ_CLASS_DECLARATION(opal_common_ofacm_base_proc_t);
/* Connection call back function that is called on connection setup */
typedef void (*opal_common_ofacm_base_context_connect_cb_fn_t)(void *);
/* Connection call back function that is called on context error */
typedef void (*opal_common_ofacm_base_context_error_cb_fn_t)(void *);
/* Prepare recive call back function that is when recv side should be prepared,
* for example recv packet prepost */
typedef int (*opal_common_ofacm_base_context_prepare_recv_cb_fn_t)(void *);
/* Basic connection context
* ========================
* The initial connection contxet is created during endpoint initialazation call.
* Each CPC will return opal_common_ofacm_base_local_connection_context_t that
* is based on CPC connection context.
*
* As Input for context creation user must provide:
* ================================================
* number of QPs
* qp init atributes
* qp standart attribute
* pointer to protection domain
* pointer to user context (for example pointer to endpoint in case of btl)
*/
typedef struct opal_common_ofacm_base_local_connection_context_t {
opal_list_item_t super;
struct opal_common_ofacm_base_proc_t *proc; /* target proc */
struct opal_common_ofacm_base_module_t *cpc; /* Pointer to context cpc */
opal_common_ofacm_connection_state_t state; /* Connection context status */
uint64_t subnet_id; /* caching subnet_id */
int cpc_type; /* connection manager family: openib, coll, etc..*/
uint16_t lid; /* caching lid */
uint16_t rem_lid; /* remote lid */
uint8_t num_of_qps; /* Number of qps that we want to open */
struct opal_common_ofacm_base_qp_t *qps; /* qps data */
uint8_t num_of_srqs; /* Number of qps that we want to open */
uint32_t *srq_num; /* srq numbers for recv on this context */
struct ibv_qp_init_attr *init_attr; /* list of initial attr for each qp */
struct ibv_qp_attr *attr; /* qp attributes */
struct ibv_pd* ib_pd; /* protection domain */
uint32_t *custom_init_attr_mask; /* in additional to standard attr_mask we want allow to user
specify special custom masks for init */
uint32_t *custom_rtr_attr_mask; /* in additional to standard attr_mask we want allow to user
specify special custom masks for rtr */
uint32_t *custom_rts_attr_mask; /* in additional to standard attr_mask we want allow to user
specify special custom masks for rts */
void *user_context; /* back pointer to endpoint */
opal_common_ofacm_base_context_connect_cb_fn_t connect_cb; /* Connection callback function */
opal_common_ofacm_base_context_error_cb_fn_t error_cb; /* Error callback function */
opal_common_ofacm_base_context_prepare_recv_cb_fn_t prepare_recv_cb; /* Prepare recv side
(prepost) callback function */
/* TBD: Need to check when we can update the index. I think during endpoint creation we do not
* have it. It mean that BTL should some how to update it later ...*/
int32_t index; /* user context index */
bool initiator; /* initiator of connection ? */
opal_common_ofacm_base_remote_connection_context_t remote_info; /* data about remote side of this
connection*/
uint32_t xrc_recv_qp_num ; /* in xrc we will use it as recv qp */
opal_mutex_t context_lock; /* protection */
} opal_common_ofacm_base_local_connection_context_t;
OPAL_DECLSPEC OBJ_CLASS_DECLARATION(opal_common_ofacm_base_local_connection_context_t);
/* Constructor and destructor are located in common_ofacm_base.c */
/************************************************************************/
/**
* Function to register MCA params in the connect functions. It
* returns no value, so it cannot fail.
*/
typedef void (*opal_common_ofacm_base_component_register_fn_t)(void);
/**
* This function is invoked once by the openib BTL component during
* startup. It is intended to have CPC component-wide startup.
*
* Return value:
*
* - OPAL_SUCCESS: this CPC component will be used in selection during
* this process.
*
* - OPAL_ERR_NOT_SUPPORTED: this CPC component will be silently
* ignored in this process.
*
* - Other OPAL_ERR_* values: the error will be propagated upwards,
* likely causing a fatal error (and/or the openib BTL component
* being ignored).
*/
typedef int (*opal_common_ofacm_base_component_init_fn_t)(void);
/**
* Query the CPC to see if it wants to run on a specific port (i.e., a
* specific BTL module). If the component init function previously
* returned OPAL_SUCCESS, this function is invoked once per BTL module
* creation (i.e., for each port found by an MPI process). If this
* CPC wants to be used on this BTL module, it returns a CPC module
* that is specific to this BTL module.
*
* The BTL module in question is passed to the function; all of its
* attributes can be used to query to see if it's eligible for this
* CPC.
*
* If it is eligible, the CPC is responsible for creating a
* corresponding CPC module, filling in all the relevant fields on the
* modules, and for setting itself up to run (per above) and returning
* a CPC module (this is effectively the "module_init" function).
* Note that the module priority must be between 0 and 100
* (inclusive). When multiple CPCs are eligible for a single module,
* the CPC with the highest priority will be used.
*
* Return value:
*
* - OPAL_SUCCESS if this CPC is eligible for and was able to be setup
* for this BTL module. It is assumed that the CPC is now completely
* setup to run on this openib module (per description above).
*
* - OPAL_ERR_NOT_SUPPORTED if this CPC cannot support this BTL
* module. This is not an error; it's just the CPC saying "sorry, I
* cannot support this BTL module."
*
* - Other OPAL_ERR_* code: an error occurred.
*/
typedef int (*opal_common_ofacm_base_func_component_query_t)
(struct opal_common_ofacm_base_dev_desc_t *dev,
struct opal_common_ofacm_base_module_t **cpc);
/**
* This function is invoked once by the openib BTL component during
* shutdown. It is intended to have CPC component-wide shutdown.
*/
typedef int (*opal_common_ofacm_base_component_finalize_fn_t)(void);
/**
* CPC component struct
*/
typedef struct opal_common_ofacm_base_component_t {
/** Name of this set of connection functions */
char cbc_name[BCF_MAX_NAME];
/** Register function. Can be NULL. */
opal_common_ofacm_base_component_register_fn_t cbc_register;
/** CPC component init function. Can be NULL. */
opal_common_ofacm_base_component_init_fn_t cbc_init;
/** Query the CPC component to get a CPC module corresponding to
an openib BTL module. Cannot be NULL. */
opal_common_ofacm_base_func_component_query_t cbc_query;
/** CPC component finalize function. Can be NULL. */
opal_common_ofacm_base_component_finalize_fn_t cbc_finalize;
/** All connection contexts that are using this CPC **/
opal_list_t all_procs;
} opal_common_ofacm_base_component_t;
/************************************************************************/
/**
* Function called when an endpoint has been created and has been
* associated with a CPC.
*/
typedef opal_common_ofacm_base_local_connection_context_t*
(*opal_common_ofacm_base_module_endpoint_init_fn_t)
(opal_proc_t *proc,
opal_common_ofacm_base_qp_config_t *qp_config, struct ibv_pd *pd,
uint64_t subnet_id, int cpc_type, uint16_t lid, uint16_t rem_lid,
int32_t user_context_index, void *user_context,
struct opal_common_ofacm_base_module_t *cpc,
opal_common_ofacm_base_context_connect_cb_fn_t connect_cb,
opal_common_ofacm_base_context_error_cb_fn_t error_cb,
opal_common_ofacm_base_context_prepare_recv_cb_fn_t prepare_recv_cb);
/**
* Function to initiate a connection to a remote process.
*/
typedef int (*opal_common_ofacm_base_module_start_connect_fn_t)
(struct opal_common_ofacm_base_local_connection_context_t *context);
/**
* Function called when an endpoint is being destroyed.
*/
typedef int (*opal_common_ofacm_base_module_endpoint_finalize_fn_t)
(struct opal_common_ofacm_base_local_connection_context_t *context);
/**
* Function to finalize the CPC module. It is called once when the
* CPC module's corresponding openib BTL module is being finalized.
*/
typedef int (*opal_common_ofacm_base_module_finalize_fn_t)(void);
/**
* Error callback that is called by cpc module on error.
* The callback should be set on upper layer (for example BTL)
*/
typedef int (*opal_common_ofacm_base_module_error_cb_fn_t)(void *);
/**
* Meta data about a CPC module. This is in a standalone struct
* because it is used in both the CPC module struct and the
* openib_btl_proc_t struct to hold information received from the
* modex.
*/
typedef struct opal_common_ofacm_base_module_data_t {
/** Pointer back to the component. Used by the base and openib
btl to calculate this module's index for the modex. */
opal_common_ofacm_base_component_t *cbm_component;
/** Priority of the CPC module (must be >=0 and <=100) */
uint8_t cbm_priority;
/** Blob that the CPC wants to include in the openib modex message
for a specific port, or NULL if the CPC does not want to
include a message in the modex. */
void *cbm_modex_message;
/** Length of the cbm_modex_message blob (0 if
cbm_modex_message==NULL). The message is intended to be short
(because the size of the modex broadcast is a function of
sum(cbm_modex_message_len[i]) for
i=(0...total_num_ports_in_MPI_job) -- e.g., IBCM imposes its
own [very short] limits (per IBTA volume 1, chapter 12). */
uint8_t cbm_modex_message_len;
} opal_common_ofacm_base_module_data_t;
/**
* Struct for holding CPC module and associated meta data
*/
typedef struct opal_common_ofacm_base_module_t {
/** Meta data about the module */
opal_common_ofacm_base_module_data_t data;
/** Endpoint initialization function */
opal_common_ofacm_base_module_endpoint_init_fn_t cbm_endpoint_init;
/** Connect function */
opal_common_ofacm_base_module_start_connect_fn_t cbm_start_connect;
/** Endpoint finalization function */
opal_common_ofacm_base_module_endpoint_finalize_fn_t cbm_endpoint_finalize;
/** Finalize the cpc module */
opal_common_ofacm_base_module_finalize_fn_t cbm_finalize;
/** Whether this module will use the CTS protocol or not. This
directly states whether this module will call
mca_btl_openib_endpoint_post_recvs() or not: true = this
module will *not* call _post_recvs() and instead will post the
receive buffer provided at endpoint->endpoint_cts_frag on qp
0. */
bool cbm_uses_cts;
} opal_common_ofacm_base_module_t;
END_C_DECLS
#endif

41
opal/mca/common/ofacm/help-mpi-common-ofacm-base.txt
Просмотреть файл

@ -1,41 +0,0 @@
# -*- text -*-
#
# Copyright (c) 2008 Cisco Systems, Inc. All rights reserved.
# Copyright (c) 2009-2012 Oak Ridge National Laboratory. All rights reserved.
# Copyright (c) 2009-2012 Mellanox Technologies. All rights reserved.
# $COPYRIGHT$
#
# Additional copyrights may follow
#
# $HEADER$
#
# This is the US/English help file for Open MPI's OpenFabrics IB CPC
# support.
#
[no cpcs for port]
No OpenFabrics connection schemes reported that they were able to be
used on a specific port. As such, the openib BTL (OpenFabrics
support) will be disabled for this port.
Local host: %s
Local device: %s
CPCs attempted: %s
#
[cpc name not found]
An invalid CPC name was specified via the btl_openib_cpc_%s MCA
parameter.
Local host: %s
btl_openib_cpc_%s value: %s
Invalid name: %s
All possible valid names: %s
#
[inline truncated]
WARNING: The btl_openib_max_inline_data MCA parameter was used to
specify how much inline data should be used, but a device reduced this
value. This is not an error; it simply means that your run will use
a smaller inline data value than was requested.
Local host: %s
Requested value: %d
Value used by device: %d

20
opal/mca/common/ofacm/help-mpi-common-ofacm-oob.txt
Просмотреть файл

@ -1,20 +0,0 @@
# -*- text -*-
#
# Copyright (c) 2009-2012 Mellanox Technologies. All rights reserved.
# Copyright (c) 2009-2012 Oak Ridge National Laboratory. All rights reserved.
# $COPYRIGHT$
#
# Additional copyrights may follow
#
# $HEADER$
#
[ofacm oob fatal error]
The OOB OpenFabrics Connection Manager module tried to raise fatal error,
but failed.
Local host: %s
Source file: %s
Source line: %d
Your job is now going to abort, sorry.
#

7
opal/mca/common/ofacm/owner.txt
Просмотреть файл

@ -1,7 +0,0 @@
#
# owner/status file
# owner: institution that is responsible for this package
# status: e.g. active, maintenance, unmaintained
#
owner: nobody
status: dead?