diff --git a/ompi/mca/btl/openib/btl_openib.c b/ompi/mca/btl/openib/btl_openib.c index ca78de7615..13b750c07b 100644 --- a/ompi/mca/btl/openib/btl_openib.c +++ b/ompi/mca/btl/openib/btl_openib.c @@ -11,7 +11,7 @@ * Copyright (c) 2004-2005 The Regents of the University of California. * All rights reserved. * Copyright (c) 2007 Cisco Systems, Inc. All rights reserved. - * Copyright (c) 2006-2008 Mellanox Technologies. All rights reserved. + * Copyright (c) 2006-2009 Mellanox Technologies. All rights reserved. * Copyright (c) 2006-2007 Los Alamos National Security, LLC. All rights * reserved. * Copyright (c) 2006-2007 Voltaire All rights reserved. @@ -638,7 +638,7 @@ mca_btl_base_descriptor_t* mca_btl_openib_alloc( if(mca_btl_openib_component.use_message_coalescing && (flags & MCA_BTL_DES_FLAGS_BTL_OWNERSHIP)) { int prio = !(flags & MCA_BTL_DES_FLAGS_PRIORITY); - sfrag = check_coalescing(&ep->qps[qp].qp->pending_frags[prio], + sfrag = check_coalescing(&ep->qps[qp].no_wqe_pending_frags[prio], &ep->qps[qp].qp->lock, ep, size); if(NULL == sfrag) { diff --git a/ompi/mca/btl/openib/btl_openib_component.c b/ompi/mca/btl/openib/btl_openib_component.c index 7d49effda5..b7dd8ca389 100644 --- a/ompi/mca/btl/openib/btl_openib_component.c +++ b/ompi/mca/btl/openib/btl_openib_component.c @@ -11,7 +11,7 @@ * Copyright (c) 2004-2005 The Regents of the University of California. * All rights reserved. * Copyright (c) 2006-2008 Cisco Systems, Inc. All rights reserved. - * Copyright (c) 2006-2008 Mellanox Technologies. All rights reserved. + * Copyright (c) 2006-2009 Mellanox Technologies. All rights reserved. * Copyright (c) 2006-2007 Los Alamos National Security, LLC. All rights * reserved. * Copyright (c) 2006-2007 Voltaire All rights reserved. @@ -2733,14 +2733,12 @@ progress_pending_frags_wqe(mca_btl_base_endpoint_t *ep, const int qpn) OPAL_THREAD_LOCK(&ep->endpoint_lock); for(i = 0; i < 2; i++) { while(qp->sd_wqe > 0) { - mca_btl_base_endpoint_t *ep; - OPAL_THREAD_LOCK(&qp->lock); - frag = opal_list_remove_first(&qp->pending_frags[i]); - OPAL_THREAD_UNLOCK(&qp->lock); + mca_btl_base_endpoint_t *tmp_ep; + frag = opal_list_remove_first(&ep->qps[qpn].no_wqe_pending_frags[i]); if(NULL == frag) break; - ep = to_com_frag(frag)->endpoint; - mca_btl_openib_endpoint_post_send(ep, to_send_frag(frag)); + tmp_ep = to_com_frag(frag)->endpoint; + mca_btl_openib_endpoint_post_send(tmp_ep, to_send_frag(frag)); } } OPAL_THREAD_UNLOCK(&ep->endpoint_lock); diff --git a/ompi/mca/btl/openib/btl_openib_endpoint.c b/ompi/mca/btl/openib/btl_openib_endpoint.c index 66b688b52b..a6ad33a037 100644 --- a/ompi/mca/btl/openib/btl_openib_endpoint.c +++ b/ompi/mca/btl/openib/btl_openib_endpoint.c @@ -14,7 +14,7 @@ * Copyright (c) 2006-2007 Los Alamos National Security, LLC. All rights * reserved. * Copyright (c) 2006-2007 Voltaire All rights reserved. - * Copyright (c) 2006-2008 Mellanox Technologies, Inc. All rights reserved. + * Copyright (c) 2006-2009 Mellanox Technologies, Inc. All rights reserved. * * $COPYRIGHT$ * @@ -120,10 +120,8 @@ static inline int acruire_wqe(mca_btl_openib_endpoint_t *ep, if(qp_get_wqe(ep, qp) < 0) { qp_put_wqe(ep, qp); - OPAL_THREAD_LOCK(&ep->qps[qp].qp->lock); - opal_list_append(&ep->qps[qp].qp->pending_frags[prio], + opal_list_append(&ep->qps[qp].no_wqe_pending_frags[prio], (opal_list_item_t *)frag); - OPAL_THREAD_UNLOCK(&ep->qps[qp].qp->lock); return OMPI_ERR_OUT_OF_RESOURCE; } @@ -281,8 +279,6 @@ static mca_btl_openib_qp_t *endpoint_alloc_qp(void) return NULL; } - OBJ_CONSTRUCT(&qp->pending_frags[0], opal_list_t); - OBJ_CONSTRUCT(&qp->pending_frags[1], opal_list_t); OBJ_CONSTRUCT(&qp->lock, opal_mutex_t); return qp; @@ -347,8 +343,12 @@ static void endpoint_init_qp(mca_btl_base_endpoint_t *ep, const int qp) ep_qp->rd_credit_send_lock = 0; ep_qp->credit_frag = NULL; + OBJ_CONSTRUCT(&ep_qp->no_wqe_pending_frags[0], opal_list_t); + OBJ_CONSTRUCT(&ep_qp->no_wqe_pending_frags[1], opal_list_t); + OBJ_CONSTRUCT(&ep_qp->pending_frags[0], opal_list_t); OBJ_CONSTRUCT(&ep_qp->pending_frags[1], opal_list_t); + switch(BTL_OPENIB_QP_TYPE(qp)) { case MCA_BTL_OPENIB_PP_QP: endpoint_init_qp_pp(ep_qp, qp); @@ -500,16 +500,17 @@ static void mca_btl_openib_endpoint_destruct(mca_btl_base_endpoint_t* endpoint) OBJ_DESTRUCT(&endpoint->qps[qp].pending_frags[0]); OBJ_DESTRUCT(&endpoint->qps[qp].pending_frags[1]); + MCA_BTL_OPENIB_CLEAN_PENDING_FRAGS( + &endpoint->qps[qp].no_wqe_pending_frags[0]); + MCA_BTL_OPENIB_CLEAN_PENDING_FRAGS( + &endpoint->qps[qp].no_wqe_pending_frags[1]); + OBJ_DESTRUCT(&endpoint->qps[qp].no_wqe_pending_frags[0]); + OBJ_DESTRUCT(&endpoint->qps[qp].no_wqe_pending_frags[1]); + + if(--endpoint->qps[qp].qp->users != 0) continue; - MCA_BTL_OPENIB_CLEAN_PENDING_FRAGS( - &endpoint->qps[qp].qp->pending_frags[0]); - MCA_BTL_OPENIB_CLEAN_PENDING_FRAGS( - &endpoint->qps[qp].qp->pending_frags[1]); - OBJ_DESTRUCT(&endpoint->qps[qp].qp->pending_frags[0]); - OBJ_DESTRUCT(&endpoint->qps[qp].qp->pending_frags[1]); - if(endpoint->qps[qp].qp->lcl_qp != NULL) if(ibv_destroy_qp(endpoint->qps[qp].qp->lcl_qp)) BTL_ERROR(("Failed to destroy QP:%d\n", qp)); diff --git a/ompi/mca/btl/openib/btl_openib_endpoint.h b/ompi/mca/btl/openib/btl_openib_endpoint.h index 2e45d0e031..2bdf7873f1 100644 --- a/ompi/mca/btl/openib/btl_openib_endpoint.h +++ b/ompi/mca/btl/openib/btl_openib_endpoint.h @@ -13,7 +13,7 @@ * Copyright (c) 2006-2007 Los Alamos National Security, LLC. All rights * reserved. * Copyright (c) 2006-2007 Voltaire All rights reserved. - * Copyright (c) 2007-2008 Mellanox Technologies. All rights reserved. + * Copyright (c) 2007-2009 Mellanox Technologies. All rights reserved. * $COPYRIGHT$ * * Additional copyrights may follow @@ -126,8 +126,6 @@ typedef struct mca_btl_openib_qp_t { struct ibv_qp *lcl_qp; uint32_t lcl_psn; int32_t sd_wqe; /**< number of available send wqe entries */ - opal_list_t pending_frags[2]; /**< put fragments here if there is no wqe - available */ int users; opal_mutex_t lock; } mca_btl_openib_qp_t; @@ -136,6 +134,8 @@ typedef struct mca_btl_openib_endpoint_qp_t { mca_btl_openib_qp_t *qp; opal_list_t pending_frags[2]; /**< put fragment here if there is no credits available */ + opal_list_t no_wqe_pending_frags[2]; /**< put fragments here if there is no wqe + available */ int32_t rd_credit_send_lock; /**< Lock credit send fragment */ mca_btl_openib_send_control_frag_t *credit_frag; size_t ib_inline_max; /**< max size of inline send*/ diff --git a/ompi/mca/btl/openib/connect/btl_openib_connect_xoob.c b/ompi/mca/btl/openib/connect/btl_openib_connect_xoob.c index de03ce9121..dae3f33968 100644 --- a/ompi/mca/btl/openib/connect/btl_openib_connect_xoob.c +++ b/ompi/mca/btl/openib/connect/btl_openib_connect_xoob.c @@ -1,5 +1,5 @@ /* - * Copyright (c) 2007-2008 Mellanox Technologies. All rights reserved. + * Copyright (c) 2007-2009 Mellanox Technologies. All rights reserved. * Copyright (c) 2008 Cisco Systems, Inc. All rights reserved. * * $COPYRIGHT$ @@ -409,7 +409,7 @@ static int xoob_send_qp_create (mca_btl_base_endpoint_t* endpoint) if (qp_init_attr.cap.max_inline_data < req_inline) { endpoint->qps[0].ib_inline_max = qp_init_attr.cap.max_inline_data; orte_show_help("help-mpi-btl-openib-cpc-base.txt", - "inline truncated", true, orte_process_info.nodename, + "inline truncated", orte_process_info.nodename, ibv_get_device_name(openib_btl->device->ib_dev), req_inline, qp_init_attr.cap.max_inline_data); } else { @@ -715,8 +715,8 @@ static void xoob_restart_connect(mca_btl_base_endpoint_t *endpoint) endpoint->ib_addr->status = MCA_BTL_IB_ADDR_CLOSED; /* destroy the qp */ /* the reciver site was alredy closed so all pending list must be clean ! */ - assert (opal_list_is_empty(&endpoint->ib_addr->qp->pending_frags[0])); - assert (opal_list_is_empty(&endpoint->ib_addr->qp->pending_frags[1])); + assert (opal_list_is_empty(&endpoint->qps->no_wqe_pending_frags[0])); + assert (opal_list_is_empty(&endpoint->qps->no_wqe_pending_frags[1])); if(ibv_destroy_qp(endpoint->qps[0].qp->lcl_qp)) BTL_ERROR(("Failed to destroy QP")); case MCA_BTL_IB_ADDR_CLOSED: @@ -954,13 +954,6 @@ static int xoob_component_query(mca_btl_openib_module_t *openib_btl, ibv_get_device_name(openib_btl->device->ib_dev)); return OMPI_ERR_NOT_SUPPORTED; } - - /* Print warning and switch off coalescing mode (ticket #1693)*/ - if (mca_btl_openib_component.use_message_coalescing) { - orte_show_help("help-mpi-btl-openib-cpc-base.txt", - "bug #1693", true, orte_process_info.nodename); - mca_btl_openib_component.use_message_coalescing = 0; - } *cpc = malloc(sizeof(ompi_btl_openib_connect_base_module_t)); if (NULL == *cpc) { diff --git a/ompi/mca/btl/openib/connect/help-mpi-btl-openib-cpc-base.txt b/ompi/mca/btl/openib/connect/help-mpi-btl-openib-cpc-base.txt index 23571f9e1d..640494fa3d 100644 --- a/ompi/mca/btl/openib/connect/help-mpi-btl-openib-cpc-base.txt +++ b/ompi/mca/btl/openib/connect/help-mpi-btl-openib-cpc-base.txt @@ -38,19 +38,3 @@ a smaller inline data value than was requested. Local device: %s Requested value: %d Value used by device: %d -# -[bug #1693] -WARNING: ConnectX XRC support was enabled together with coalescing -mode. This combination is not supported in this version of Open MPI -because it may cause random seg faults (please see Open MPI bug ticket -#1693 https://svn.open-mpi.org/trac/ompi/ticket/1693 for more -details). Message coalescing has been disabled in this job to prevent -catastrophic failure. - -You can silence this warning either by disabling XRC (i.e., not specifying -an "X" queue in btl_openib_receive_queues) or disabling message -coalescing by setting the btl_openib_use_message_coalescing MCA -parameter to 0. - - Local host: %s -#