ports/openmpi
ports/openmpi
1
1
Форкнуть 0
Код Релизы Активность

Adding support for APM over different ports

Просмотр исходного кода 
This commit was SVN r17521.
Этот коммит содержится в:
Pavel Shamis 2008-02-20 13:44:05 +00:00
родитель 09afc36f5f
Коммит a0d12a9c92
8 изменённых файлов: 187 добавлений и 46 удалений
Показать статистику Скачать Patch файл Скачать Diff файл Показать все файлы Свернуть все файлы

9
ompi/mca/btl/openib/btl_openib.h
Просмотреть файл

@ -169,7 +169,8 @@ struct mca_btl_openib_component_t {
int32_t max_eager_rdma;
uint32_t btls_per_lid;
uint32_t max_lmc;
int32_t apm;
int32_t apm_lmc;
int32_t apm_ports;
uint32_t buffer_alignment; /**< Preferred communication buffer alignment in Bytes (must be power of two) */
#if OMPI_HAVE_THREADS
int32_t fatal_counter; /**< Counts number on fatal events that we got on all hcas */
@ -229,9 +230,8 @@ struct mca_btl_openib_port_info {
uint8_t padding[4];
#endif
uint64_t subnet_id;
#if HAVE_XRC
uint16_t lid; /* used only in xrc */
#endif
uint16_t apm_lid; /* the lid is used for APM to different port */
char *cpclist;
};
typedef struct mca_btl_openib_port_info mca_btl_openib_port_info_t;
@ -285,6 +285,7 @@ typedef struct mca_btl_openib_hca_t {
uint8_t use_eager_rdma;
uint8_t btls; /** < number of btls using this HCA */
opal_pointer_array_t *endpoints;
opal_pointer_array_t *hca_btls;
uint16_t hp_cq_polls;
uint16_t eager_rdma_polls;
bool pollme;
@ -335,7 +336,7 @@ struct mca_btl_openib_module_t {
uint16_t pkey_index;
struct ibv_port_attr ib_port_attr;
uint16_t lid; /**< lid that is actually used (for LMC) */
uint16_t apm_lmc_max; /**< the maximal lmc that can be used for apm */
int apm_port; /**< Alternative port that may be used for APM */
uint8_t src_path_bits; /**< offset from base lid (for LMC) */
int32_t num_peers;

136
ompi/mca/btl/openib/btl_openib_async.c
Просмотреть файл

@ -27,6 +27,8 @@
#include "btl_openib.h"
#include "btl_openib_mca.h"
#include "btl_openib_async.h"
#include "btl_openib_proc.h"
#include "btl_openib_endpoint.h"
struct mca_btl_openib_async_poll {
int active_poll_size;
@ -90,6 +92,33 @@ static const char *openib_event_to_str (enum ibv_event_type event)
return "UNKNOWN";
}
}
/* QP to endpoint */
static mca_btl_openib_endpoint_t * qp2endpoint(struct ibv_qp *qp, mca_btl_openib_hca_t *hca)
{
mca_btl_openib_endpoint_t *ep;
int ep_i, qp_i;
for(ep_i = 0; ep_i < opal_pointer_array_get_size(hca->endpoints); ep_i++) {
ep = opal_pointer_array_get_item(hca->endpoints, ep_i);
for(qp_i = 0; qp_i < mca_btl_openib_component.num_qps; qp_i++) {
if (qp == ep->qps[qp_i].qp->lcl_qp)
return ep;
}
}
return NULL;
}
/* XRC recive QP to endpoint */
static mca_btl_openib_endpoint_t * xrc_qp2endpoint(uint32_t qp_num, mca_btl_openib_hca_t *hca)
{
mca_btl_openib_endpoint_t *ep;
int ep_i;
for(ep_i = 0; ep_i < opal_pointer_array_get_size(hca->endpoints); ep_i++) {
ep = opal_pointer_array_get_item(hca->endpoints, ep_i);
if (qp_num == ep->xrc_recv_qp_num)
return ep;
}
return NULL;
}
/* Function inits mca_btl_openib_async_poll */
static int btl_openib_async_poll_init(struct mca_btl_openib_async_poll *hcas_poll)
@ -194,6 +223,7 @@ static int btl_openib_async_hcah(struct mca_btl_openib_async_poll *hcas_poll, in
if (mca_btl_openib_component.openib_btls[j]->hca->ib_dev_context->async_fd ==
hcas_poll->async_pollfd[index].fd ) {
hca = mca_btl_openib_component.openib_btls[j]->hca;
break;
}
}
if (NULL != hca) {
@ -219,15 +249,16 @@ static int btl_openib_async_hcah(struct mca_btl_openib_async_poll *hcas_poll, in
#endif
switch(event_type) {
case IBV_EVENT_PATH_MIG:
if (0 != mca_btl_openib_component.apm) {
BTL_ERROR(("APM: Alternative path migration reported."));
BTL_ERROR(("Alternative path migration event reported"));
if (APM_ENABLED) {
BTL_ERROR(("Trying to find additional path..."));
if (!xrc_event)
mca_btl_openib_load_apm(event.element.qp,
mca_btl_openib_component.openib_btls[j]);
qp2endpoint(event.element.qp, hca));
#if HAVE_XRC
else
mca_btl_openib_load_apm_xrc_rcv(event.element.xrc_qp_num,
mca_btl_openib_component.openib_btls[j]);
xrc_qp2endpoint(event.element.xrc_qp_num, hca));
#endif
}
break;
@ -345,55 +376,126 @@ static void apm_update_attr(struct ibv_qp_attr *attr, enum ibv_qp_attr_mask *mas
{
*mask = IBV_QP_ALT_PATH|IBV_QP_PATH_MIG_STATE;
attr->alt_ah_attr.dlid = attr->ah_attr.dlid + 1;
attr->alt_ah_attr.src_path_bits = attr->ah_attr.src_path_bits + 1;
attr->alt_ah_attr.static_rate = attr->ah_attr.static_rate;
attr->alt_ah_attr.sl = attr->ah_attr.sl;
attr->alt_pkey_index = attr->pkey_index;
attr->alt_port_num = attr->port_num;
attr->alt_timeout = attr->timeout;
attr->path_mig_state = IBV_MIG_REARM;
BTL_VERBOSE(("New APM LMC loaded: alt_src_port:%d, dlid: %d, src_bits %d, old_src_bits: %d, old_dlid %d",
attr->alt_port_num, attr->alt_ah_attr.dlid,
attr->alt_ah_attr.src_path_bits, attr->ah_attr.src_path_bits, attr->ah_attr.dlid));
}
static int apm_update_port(mca_btl_openib_endpoint_t *ep,
struct ibv_qp_attr *attr, enum ibv_qp_attr_mask *mask)
{
size_t port_i;
uint16_t apm_lid = 0;
if (attr->port_num == ep->endpoint_btl->apm_port) {
/* all ports were used */
BTL_ERROR(("APM: already all ports were used port_num %d apm_port %d",
attr->port_num, ep->endpoint_btl->apm_port));
return OMPI_ERROR;
}
/* looking for alternatve lid on remote site */
for(port_i = 0; port_i < ep->endpoint_proc->proc_port_count; port_i++) {
if (ep->endpoint_proc->proc_ports[port_i].lid == attr->ah_attr.dlid - mca_btl_openib_component.apm_lmc) {
apm_lid = ep->endpoint_proc->proc_ports[port_i].apm_lid;
}
}
if (0 == apm_lid) {
/* APM was disabled on one of site ? */
BTL_VERBOSE(("APM: Was disabled ? dlid %d %d %d", attr->ah_attr.dlid, attr->ah_attr.src_path_bits, ep->endpoint_btl->src_path_bits));
return OMPI_ERROR;
}
/* We guess cthat the LMC is the same on all ports */
attr->alt_ah_attr.static_rate = attr->ah_attr.static_rate;
attr->alt_ah_attr.sl = attr->ah_attr.sl;
attr->alt_pkey_index = attr->pkey_index;
attr->alt_timeout = attr->timeout;
attr->path_mig_state = IBV_MIG_REARM;
*mask = IBV_QP_ALT_PATH|IBV_QP_PATH_MIG_STATE;
attr->alt_port_num = ep->endpoint_btl->apm_port;
attr->alt_ah_attr.src_path_bits = ep->endpoint_btl->src_path_bits;
attr->alt_ah_attr.dlid = apm_lid;
BTL_VERBOSE(("New APM port loaded: alt_src_port:%d, dlid: %d, src_bits: %d:%d, old_dlid %d",
attr->alt_port_num, attr->alt_ah_attr.dlid,
attr->ah_attr.src_path_bits, attr->alt_ah_attr.src_path_bits,
attr->ah_attr.dlid));
return OMPI_SUCCESS;
}
/* Load new dlid to the QP */
void mca_btl_openib_load_apm(struct ibv_qp *qp, struct mca_btl_openib_module_t *btl)
void mca_btl_openib_load_apm(struct ibv_qp *qp, mca_btl_openib_endpoint_t *ep)
{
struct ibv_qp_init_attr qp_init_attr;
struct ibv_qp_attr attr;
enum ibv_qp_attr_mask mask = 0;
struct mca_btl_openib_module_t *btl;
BTL_VERBOSE(("APM: Loading alternative path"));
assert (NULL != ep);
btl = ep->endpoint_btl;
if (ibv_query_qp(qp, &attr, mask, &qp_init_attr))
BTL_ERROR(("Failed to ibv_query_qp, qp num: %d", qp->qp_num));
if (attr.ah_attr.src_path_bits - btl->src_path_bits < btl->apm_lmc_max) {
attr.alt_ah_attr.src_path_bits = attr.ah_attr.src_path_bits + 1;
if (mca_btl_openib_component.apm_lmc &&
attr.ah_attr.src_path_bits - btl->src_path_bits < mca_btl_openib_component.apm_lmc) {
BTL_VERBOSE(("APM LMC: src: %d btl_src: %d lmc_max: %d",
attr.ah_attr.src_path_bits,
btl->src_path_bits,
mca_btl_openib_component.apm_lmc));
apm_update_attr(&attr, &mask);
} else {
BTL_ERROR(("Failed to load alternative path, all %d were used",
attr.ah_attr.src_path_bits - btl->src_path_bits));
if (mca_btl_openib_component.apm_ports) {
/* Try to migrate to next port */
if (OMPI_SUCCESS != apm_update_port(ep, &attr, &mask))
return;
} else {
BTL_ERROR(("Failed to load alternative path, all %d were used",
attr.ah_attr.src_path_bits - btl->src_path_bits));
}
}
apm_update_attr(&attr, &mask);
if (ibv_modify_qp(qp, &attr, mask))
BTL_ERROR(("Failed to ibv_query_qp, qp num: %d", qp->qp_num));
BTL_ERROR(("Failed to ibv_query_qp, qp num: %p, errno says: %s (%d)"
,qp->qp_num ,strerror(errno), errno));
}
void mca_btl_openib_load_apm_xrc_rcv(uint32_t qp_num, struct mca_btl_openib_module_t *btl)
void mca_btl_openib_load_apm_xrc_rcv(uint32_t qp_num, mca_btl_openib_endpoint_t *ep)
{
#if HAVE_XRC
struct ibv_qp_init_attr qp_init_attr;
struct ibv_qp_attr attr;
enum ibv_qp_attr_mask mask = 0;
struct mca_btl_openib_module_t *btl;
BTL_VERBOSE(("APM XRC: Loading alternative path"));
assert (NULL != ep);
btl = ep->endpoint_btl;
if (ibv_query_xrc_rcv_qp(btl->hca->xrc_domain, qp_num, &attr, mask, &qp_init_attr))
BTL_ERROR(("Failed to ibv_query_qp, qp num: %d", qp_num));
if (attr.ah_attr.src_path_bits - btl->src_path_bits < btl->apm_lmc_max) {
attr.alt_ah_attr.src_path_bits = attr.ah_attr.src_path_bits + 1;
if (mca_btl_openib_component.apm_lmc &&
attr.ah_attr.src_path_bits - btl->src_path_bits < mca_btl_openib_component.apm_lmc) {
apm_update_attr(&attr, &mask);
} else {
BTL_ERROR(("Failed to load alternative path, all %d were used",
attr.ah_attr.src_path_bits - btl->src_path_bits));
if (mca_btl_openib_component.apm_ports) {
/* Try to migrate to next port */
if (OMPI_SUCCESS != apm_update_port(ep, &attr, &mask))
return;
} else {
BTL_ERROR(("Failed to load alternative path, all %d were used",
attr.ah_attr.src_path_bits - btl->src_path_bits));
}
}
apm_update_attr(&attr, &mask);
ibv_modify_xrc_rcv_qp(btl->hca->xrc_domain, qp_num, &attr, mask);
/* Maybe the qp already was modified by other process - ignoring error */

8
ompi/mca/btl/openib/btl_openib_async.h
Просмотреть файл

@ -11,8 +11,12 @@
#ifndef MCA_BTL_OPENIB_ASYNC_H
#define MCA_BTL_OPENIB_ASYNC_H
#include "btl_openib_endpoint.h"
void* btl_openib_async_thread(void *one_hca);
void mca_btl_openib_load_apm(struct ibv_qp *qp, struct mca_btl_openib_module_t *btl);
void mca_btl_openib_load_apm_xrc_rcv(uint32_t qp_num, struct mca_btl_openib_module_t *btl);
void mca_btl_openib_load_apm(struct ibv_qp *qp, mca_btl_openib_endpoint_t *ep);
void mca_btl_openib_load_apm_xrc_rcv(uint32_t qp_num, mca_btl_openib_endpoint_t *ep);
#define APM_ENABLED (0 != mca_btl_openib_component.apm_lmc || 0 != mca_btl_openib_component.apm_ports)
#endif

58
ompi/mca/btl/openib/btl_openib_component.c
Просмотреть файл

@ -402,19 +402,20 @@ static int init_one_port(opal_list_t *btl_list, mca_btl_openib_hca_t *hca,
#if OMPI_HAVE_THREADS
/* APM support */
if (lmc > 1){
if (-1 == mca_btl_openib_component.apm) {
if (-1 == mca_btl_openib_component.apm_lmc) {
lmc_step = lmc;
} else if (0 == lmc % (mca_btl_openib_component.apm + 1)) {
lmc_step = mca_btl_openib_component.apm + 1;
mca_btl_openib_component.apm_lmc = lmc - 1;
} else if (0 == lmc % (mca_btl_openib_component.apm_lmc + 1)) {
lmc_step = mca_btl_openib_component.apm_lmc + 1;
} else {
opal_show_help("help-mpi-btl-openib.txt", "apm with wrong lmc",true,
mca_btl_openib_component.apm, lmc);
mca_btl_openib_component.apm_lmc, lmc);
return OMPI_ERROR;
}
} else {
if (mca_btl_openib_component.apm) {
if (mca_btl_openib_component.apm_lmc) {
/* Disable apm and report warning */
mca_btl_openib_component.apm = 0;
mca_btl_openib_component.apm_lmc = 0;
opal_show_help("help-mpi-btl-openib.txt", "apm without lmc",true);
}
}
@ -441,18 +442,14 @@ static int init_one_port(opal_list_t *btl_list, mca_btl_openib_hca_t *hca,
openib_btl->port_num = (uint8_t) port_num;
openib_btl->pkey_index = pkey_index;
openib_btl->lid = lid;
openib_btl->apm_lmc_max = lmc_step ;
openib_btl->apm_port = 0;
openib_btl->src_path_bits = lid - ib_port_attr->lid;
/* store the subnet for multi-nic support */
openib_btl->port_info.subnet_id = subnet_id;
openib_btl->port_info.mtu = hca->mtu;
#if HAVE_XRC
/* This code is protected with ifdef because we don't want to send
* extra bytes during OOB */
if(MCA_BTL_XRC_ENABLED) {
openib_btl->port_info.lid = lid;
}
#endif
openib_btl->port_info.lid = lid;
rc = ompi_btl_openib_connect_base_query(&openib_btl->port_info.cpclist, hca);
if (OMPI_SUCCESS != rc) {
continue;
@ -549,6 +546,7 @@ static int init_one_port(opal_list_t *btl_list, mca_btl_openib_hca_t *hca,
}
}
opal_list_append(btl_list, (opal_list_item_t*) ib_selected);
opal_pointer_array_add(hca->hca_btls, (void*) openib_btl);
hca->btls++;
++mca_btl_openib_component.ib_num_btls;
if (-1 != mca_btl_openib_component.ib_max_btls &&
@ -868,6 +866,22 @@ static bool inline is_credit_message(const mca_btl_openib_recv_frag_t *frag)
(MCA_BTL_OPENIB_CONTROL_CREDITS == chdr->type);
}
static void init_apm_port(mca_btl_openib_hca_t *hca, int port, uint16_t lid)
{
int index;
struct mca_btl_openib_module_t *btl;
for(index = 0; index < hca->btls; index++) {
btl = opal_pointer_array_get_item(hca->hca_btls, index);
/* Ok, we already have btl for the fist port,
* second one will be used for APM */
btl->apm_port = port;
btl->port_info.apm_lid = lid + btl->src_path_bits;
mca_btl_openib_component.apm_ports++;
BTL_VERBOSE(("APM-PORT: Setting alternative port - %d, lid - %d"
,port ,lid));
}
}
static int init_one_hca(opal_list_t *btl_list, struct ibv_device* ib_dev)
{
struct mca_mpool_base_resources_t mpool_resources;
@ -885,6 +899,11 @@ static int init_one_hca(opal_list_t *btl_list, struct ibv_device* ib_dev)
hca->ib_dev = ib_dev;
hca->ib_dev_context = ibv_open_device(ib_dev);
hca->hca_btls = OBJ_NEW(opal_pointer_array_t);
if (OPAL_SUCCESS != opal_pointer_array_init(hca->hca_btls, 2, INT_MAX, 2)) {
BTL_ERROR(("Failed to initialize hca_btls array: %s:%d\n", __FILE__, __LINE__));
return OMPI_ERR_OUT_OF_RESOURCE;
}
if(NULL == hca->ib_dev_context){
BTL_ERROR(("error obtaining device context for %s errno says %s\n",
@ -1033,13 +1052,15 @@ static int init_one_hca(opal_list_t *btl_list, struct ibv_device* ib_dev)
ibv_get_device_name(ib_dev), i, strerror(errno)));
break;
}
if(IBV_PORT_ACTIVE == ib_port_attr.state){
if(IBV_PORT_ACTIVE == ib_port_attr.state) {
if (mca_btl_openib_component.apm_ports && hca->btls > 0) {
init_apm_port(hca, i, ib_port_attr.lid);
break;
}
if (0 == mca_btl_openib_component.ib_pkey_val) {
ret = init_one_port(btl_list, hca, i, mca_btl_openib_component.ib_pkey_ix,
&ib_port_attr);
}
else {
} else {
uint16_t pkey,j;
for (j=0; j < hca->ib_dev_attr.max_pkeys; j++) {
ibv_query_pkey(hca->ib_dev_context, i, j, &pkey);
@ -1063,6 +1084,11 @@ static int init_one_hca(opal_list_t *btl_list, struct ibv_device* ib_dev)
/* If we made a BTL, we're done. Otherwise, fall through and
destroy everything */
if (hca->btls > 0) {
/* if apm was enabled it should be > 1 */
if (1 == mca_btl_openib_component.apm_ports) {
opal_show_help("help-mpi-btl-openib.txt", "apm not enough ports", true);
mca_btl_openib_component.apm_ports = 0;
}
ret = prepare_hca_for_use(hca);
if(OMPI_SUCCESS == ret)
return OMPI_SUCCESS;

6
ompi/mca/btl/openib/btl_openib_endpoint.c
Просмотреть файл

@ -551,15 +551,15 @@ void mca_btl_openib_endpoint_connected(mca_btl_openib_endpoint_t *endpoint)
/* Run over all qps and load alternative path */
#if OMPI_HAVE_THREADS
if (0 != mca_btl_openib_component.apm) {
if (APM_ENABLED) {
int i;
if (MCA_BTL_XRC_ENABLED) {
if (master) {
mca_btl_openib_load_apm(endpoint->ib_addr->qp->lcl_qp, endpoint->endpoint_btl);
mca_btl_openib_load_apm(endpoint->ib_addr->qp->lcl_qp, endpoint);
}
} else {
for(i = 0; i < mca_btl_openib_component.num_qps; i++) {
mca_btl_openib_load_apm(endpoint->qps[i].qp->lcl_qp, endpoint->endpoint_btl);
mca_btl_openib_load_apm(endpoint->qps[i].qp->lcl_qp, endpoint);
}
}
}

8
ompi/mca/btl/openib/btl_openib_mca.c
Просмотреть файл

@ -403,10 +403,14 @@ int btl_openib_register_mca_params(void)
mca_btl_openib_component.max_lmc = (uint32_t) ival;
#if OMPI_HAVE_THREADS
CHECK(reg_int("enable_apm", "Maximum number of alterative paths for each HCA port "
CHECK(reg_int("enable_apm_over_lmc", "Maximum number of alterative paths for each HCA port "
"(must be >= -1, where 0 = disable apm, -1 = all availible alternative paths )",
0, &ival, REGINT_NEG_ONE_OK|REGINT_GE_ZERO));
mca_btl_openib_component.apm = (uint32_t) ival;
mca_btl_openib_component.apm_lmc = (uint32_t) ival;
CHECK(reg_int("enable_apm_over_ports", "Enable alterative path migration over different ports of the same HCA"
"(must be >= 0, where 0 = disable apm over ports , 1 = enable apm over ports of the same hca )",
0, &ival, REGINT_GE_ZERO));
mca_btl_openib_component.apm_ports = (uint32_t) ival;
CHECK(reg_int("use_async_event_thread",
"If nonzero, use the thread that will handle InfiniBand asyncihronous events ",

4
ompi/mca/btl/openib/connect/btl_openib_connect_xoob.c
Просмотреть файл

@ -570,8 +570,8 @@ static int xoob_recv_qp_create(mca_btl_openib_endpoint_t *endpoint, mca_btl_open
endpoint->xrc_recv_qp_num, strerror(ret), ret));
return OMPI_ERROR;
}
if (0 != mca_btl_openib_component.apm) {
mca_btl_openib_load_apm_xrc_rcv(endpoint->xrc_recv_qp_num, openib_btl);
if (APM_ENABLED) {
mca_btl_openib_load_apm_xrc_rcv(endpoint->xrc_recv_qp_num, endpoint);
}
return OMPI_SUCCESS;

4
ompi/mca/btl/openib/help-mpi-btl-openib.txt
Просмотреть файл

@ -444,3 +444,7 @@ APM support will be disabled.
#
[apm with wrong lmc]
Can not provide %d alternative paths with LMC bit configured to %d.
#
[apm not enough ports]
WARNING: For APM over ports ompi require at least 2 active ports and only single
active port was found. Disabling APM over ports