From f46548e6912dd79e135d982ae4ee64ab9387570f Mon Sep 17 00:00:00 2001 From: Galen Shipman Date: Sun, 2 Oct 2005 18:58:57 +0000 Subject: [PATCH] Add SRQ support to OpenIB btl, removed old mca param - not used.. This commit was SVN r7585. --- ompi/mca/btl/mvapi/btl_mvapi.h | 3 - ompi/mca/btl/mvapi/btl_mvapi_component.c | 4 +- ompi/mca/btl/openib/btl_openib.c | 75 +++++++++++++++++--- ompi/mca/btl/openib/btl_openib.h | 81 ++++++++++++++++++++-- ompi/mca/btl/openib/btl_openib_component.c | 48 +++++++++---- ompi/mca/btl/openib/btl_openib_endpoint.c | 58 +++++++++++++--- 6 files changed, 227 insertions(+), 42 deletions(-) diff --git a/ompi/mca/btl/mvapi/btl_mvapi.h b/ompi/mca/btl/mvapi/btl_mvapi.h index ad0efbe686..7ea9629544 100644 --- a/ompi/mca/btl/mvapi/btl_mvapi.h +++ b/ompi/mca/btl/mvapi/btl_mvapi.h @@ -82,9 +82,6 @@ struct mca_btl_mvapi_component_t { opal_mutex_t ib_lock; /**< lock for accessing module state */ - int ib_mem_registry_hints_log_size; - /**< log2 size of hints hash array used by memory registry */ - char* ib_mpool_name; /**< name of ib memory pool */ diff --git a/ompi/mca/btl/mvapi/btl_mvapi_component.c b/ompi/mca/btl/mvapi/btl_mvapi_component.c index 1de18afac5..9ab0a80cfb 100644 --- a/ompi/mca/btl/mvapi/btl_mvapi_component.c +++ b/ompi/mca/btl/mvapi/btl_mvapi_component.c @@ -121,8 +121,6 @@ int mca_btl_mvapi_component_open(void) mca_btl_mvapi_param_register_int ("free_list_max", -1); mca_btl_mvapi_component.ib_free_list_inc = mca_btl_mvapi_param_register_int ("free_list_inc", 32); - mca_btl_mvapi_component.ib_mem_registry_hints_log_size = - mca_btl_mvapi_param_register_int ("hints_log_size", 8); mca_btl_mvapi_component.ib_mpool_name = mca_btl_mvapi_param_register_string("mpool", "mvapi"); mca_btl_mvapi_component.ib_rr_buf_max = @@ -180,7 +178,7 @@ int mca_btl_mvapi_component_open(void) mca_btl_mvapi_component.ib_src_path_bits = mca_btl_mvapi_param_register_int("ib_src_path_bits", 0); - + mca_btl_mvapi_component.rd_per_peer = mca_btl_mvapi_param_register_int("rd_per_peer", 16); diff --git a/ompi/mca/btl/openib/btl_openib.c b/ompi/mca/btl/openib/btl_openib.c index 11b4b40db0..9917fbb765 100644 --- a/ompi/mca/btl/openib/btl_openib.c +++ b/ompi/mca/btl/openib/btl_openib.c @@ -32,7 +32,7 @@ #include "mca/mpool/openib/mpool_openib.h" #include #include - +#include mca_btl_openib_module_t mca_btl_openib_module = { { &mca_btl_openib_component.super, @@ -118,7 +118,14 @@ int mca_btl_openib_add_procs( OPAL_THREAD_UNLOCK(&ib_proc->proc_lock); peers[i] = ib_peer; } - +#ifdef OMPI_MCA_BTL_OPENIB_HAVE_SRQ + if( 0 == openib_btl->num_peers ) { + openib_btl->num_peers += nprocs; + if(mca_btl_openib_component.use_srq) { + openib_btl->rd_buf_max = mca_btl_openib_component.ib_rr_buf_max + log2(nprocs) * mca_btl_openib_component.rd_per_peer; + } + } +#endif return OMPI_SUCCESS; } @@ -148,10 +155,10 @@ int mca_btl_openib_register( mca_btl_openib_module_t* openib_btl = (mca_btl_openib_module_t*) btl; - OPAL_THREAD_LOCK(&ib->btl.ib_lock); + OPAL_THREAD_LOCK(&openib->ib_lock); openib_btl->ib_reg[tag].cbfunc = cbfunc; openib_btl->ib_reg[tag].cbdata = cbdata; - OPAL_THREAD_UNLOCK(&ib->btl.ib_lock); + OPAL_THREAD_UNLOCK(&openib->ib_lock); return OMPI_SUCCESS; } @@ -361,7 +368,7 @@ mca_btl_base_descriptor_t* mca_btl_openib_prepare_src( } iov.iov_len = max_data; - iov.iov_base = (unsigned char*) (frag->segment.seg_addr.lval + reserve); + iov.iov_base = (unsigned char*) (unsigned long) (frag->segment.seg_addr.lval + reserve); rc = ompi_convertor_pack(convertor, &iov, &iov_count, &max_data, &free_after); *size = max_data; @@ -567,6 +574,7 @@ int mca_btl_openib_put( mca_btl_base_module_t* btl, { struct ibv_send_wr* bad_wr; mca_btl_openib_frag_t* frag = (mca_btl_openib_frag_t*) descriptor; + mca_btl_openib_module_t* openib_btl = (mca_btl_openib_module_t*) btl; frag->endpoint = endpoint; frag->wr_desc.sr_desc.opcode = IBV_WR_RDMA_WRITE; frag->wr_desc.sr_desc.send_flags = IBV_SEND_SIGNALED; @@ -588,9 +596,17 @@ int mca_btl_openib_put( mca_btl_base_module_t* btl, return OMPI_ERROR; } - MCA_BTL_OPENIB_ENDPOINT_POST_RR_HIGH(endpoint, 1); - MCA_BTL_OPENIB_ENDPOINT_POST_RR_LOW(endpoint, 1); - +#ifdef OMPI_MCA_BTL_OPENIB_HAVE_SRQ + if(mca_btl_openib_component.use_srq) { + MCA_BTL_OPENIB_POST_SRR_HIGH(openib_btl, 1); + MCA_BTL_OPENIB_POST_SRR_LOW(openib_btl, 1); + } else { +#endif + MCA_BTL_OPENIB_ENDPOINT_POST_RR_HIGH(endpoint, 1); + MCA_BTL_OPENIB_ENDPOINT_POST_RR_LOW(endpoint, 1); +#ifdef OMPI_MCA_BTL_OPENIB_HAVE_SRQ + } +#endif return OMPI_SUCCESS; } @@ -606,6 +622,7 @@ int mca_btl_openib_get( mca_btl_base_module_t* btl, { struct ibv_send_wr* bad_wr; mca_btl_openib_frag_t* frag = (mca_btl_openib_frag_t*) descriptor; + mca_btl_openib_module_t* openib_btl = (mca_btl_openib_module_t*) btl; frag->endpoint = endpoint; frag->wr_desc.sr_desc.opcode = IBV_WR_RDMA_READ; frag->wr_desc.sr_desc.send_flags = IBV_SEND_SIGNALED; @@ -627,8 +644,18 @@ int mca_btl_openib_get( mca_btl_base_module_t* btl, return OMPI_ERROR; } - MCA_BTL_OPENIB_ENDPOINT_POST_RR_HIGH(endpoint, 1); - MCA_BTL_OPENIB_ENDPOINT_POST_RR_LOW(endpoint, 1); + +#ifdef OMPI_MCA_BTL_OPENIB_HAVE_SRQ + if(mca_btl_openib_component.use_srq) { + MCA_BTL_OPENIB_POST_SRR_HIGH(openib_btl, 1); + MCA_BTL_OPENIB_POST_SRR_LOW(openib_btl, 1); + } else { +#endif + MCA_BTL_OPENIB_ENDPOINT_POST_RR_HIGH(endpoint, 1); + MCA_BTL_OPENIB_ENDPOINT_POST_RR_LOW(endpoint, 1); +#ifdef OMPI_MCA_BTL_OPENIB_HAVE_SRQ + } +#endif return OMPI_SUCCESS; @@ -655,7 +682,35 @@ int mca_btl_openib_module_init(mca_btl_openib_module_t *openib_btl) strerror(errno))); return OMPI_ERROR; } + +#ifdef OMPI_MCA_BTL_OPENIB_HAVE_SRQ + if(mca_btl_openib_component.use_srq) { + + struct ibv_srq_init_attr attr; + attr.attr.max_wr = mca_btl_openib_component.ib_wq_size; + attr.attr.max_sge = mca_btl_openib_component.ib_sg_list_size; + openib_btl->srr_posted_high = 0; + openib_btl->srr_posted_low = 0; + + openib_btl->srq_high = ibv_create_srq(openib_btl->ib_pd, &attr); + if(NULL == openib_btl->srq_high) { + BTL_ERROR(("error in ibv_create_srq\n")); + return OMPI_ERROR; + } + + openib_btl->srq_low = ibv_create_srq(openib_btl->ib_pd, &attr); + if(NULL == openib_btl->srq_high) { + BTL_ERROR(("error in ibv_create_srq\n")); + return OMPI_ERROR; + } + + } else { + openib_btl->srq_high = NULL; + openib_btl->srq_low = NULL; + } +#endif + /* Create the low and high priority queue pairs */ #if OMPI_MCA_BTL_OPENIB_IBV_CREATE_CQ_ARGS == 3 openib_btl->ib_cq_low = diff --git a/ompi/mca/btl/openib/btl_openib.h b/ompi/mca/btl/openib/btl_openib.h index bae00ccb98..f9b3d4dee5 100644 --- a/ompi/mca/btl/openib/btl_openib.h +++ b/ompi/mca/btl/openib/btl_openib.h @@ -78,9 +78,6 @@ struct mca_btl_openib_component_t { opal_mutex_t ib_lock; /**< lock for accessing module state */ - int ib_mem_registry_hints_log_size; - /**< log2 size of hints hash array used by memory registry */ - char* ib_mpool_name; /**< name of ib memory pool */ @@ -94,7 +91,8 @@ struct mca_btl_openib_component_t { size_t max_send_size; uint32_t leave_pinned; uint32_t reg_mru_len; - + uint32_t use_srq; + uint32_t ib_cq_size; /**< Max outstanding CQE on the CQ */ uint32_t ib_wq_size; /**< Max outstanding WR on the WQ */ uint32_t ib_sg_list_size; /**< Max scatter/gather descriptor entries on the WQ*/ @@ -110,8 +108,10 @@ struct mca_btl_openib_component_t { uint32_t ib_service_level; uint32_t ib_static_rate; uint32_t ib_src_path_bits; + uint32_t rd_per_peer; /* number of receive descriptors to post per log2(peers) + in SRQ mode */ + - }; typedef struct mca_btl_openib_component_t mca_btl_openib_component_t; extern mca_btl_openib_component_t mca_btl_openib_component; @@ -157,6 +157,14 @@ struct mca_btl_openib_module_t { bool poll_cq; +#ifdef OMPI_MCA_BTL_OPENIB_HAVE_SRQ + struct ibv_srq *srq_high; + struct ibv_srq *srq_low; + uint32_t srr_posted_high; + uint32_t srr_posted_low; +#endif + uint32_t num_peers; + uint32_t rd_buf_max; }; typedef struct mca_btl_openib_module_t mca_btl_openib_module_t; @@ -389,6 +397,69 @@ extern void mca_btl_openib_send_frag_return( int mca_btl_openib_module_init(mca_btl_openib_module_t* openib_btl); +#ifdef OMPI_MCA_BTL_OPENIB_HAVE_SRQ + +#define MCA_BTL_OPENIB_POST_SRR_HIGH(openib_btl, additional) \ +{ \ + OPAL_THREAD_LOCK(&openib_btl->ib_lock); \ + if(openib_btl->srr_posted_high <= mca_btl_openib_component.ib_rr_buf_min+additional && \ + openib_btl->srr_posted_high < mca_btl_openib_component.ib_rr_buf_max){ \ + MCA_BTL_OPENIB_POST_SRR_SUB(mca_btl_openib_component.ib_rr_buf_max - \ + openib_btl->srr_posted_high, \ + openib_btl, \ + &openib_btl->recv_free_eager, \ + &openib_btl->srr_posted_high, \ + openib_btl->srq_high); \ + } \ + OPAL_THREAD_UNLOCK(&openib_btl->ib_lock); \ +} + +#define MCA_BTL_OPENIB_POST_SRR_LOW(openib_btl, additional) \ +{ \ + OPAL_THREAD_LOCK(&openib_btl->ib_lock); \ + if(openib_btl->srr_posted_low <= mca_btl_openib_component.ib_rr_buf_min+additional && \ + openib_btl->srr_posted_low < mca_btl_openib_component.ib_rr_buf_max){ \ + MCA_BTL_OPENIB_POST_SRR_SUB(mca_btl_openib_component.ib_rr_buf_max - \ + openib_btl->srr_posted_low, \ + openib_btl, \ + &openib_btl->recv_free_max, \ + &openib_btl->srr_posted_low, \ + openib_btl->srq_low); \ + } \ + OPAL_THREAD_UNLOCK(&openib_btl->ib_lock); \ +} + + +#define MCA_BTL_OPENIB_POST_SRR_SUB(cnt, \ + openib_btl, \ + frag_list, \ + srr_posted, \ + srq) \ +{\ + do { \ + uint32_t i; \ + opal_list_item_t* item = NULL; \ + mca_btl_openib_frag_t* frag = NULL; \ + struct ibv_recv_wr *bad_wr; \ + int32_t rc; \ + for(i = 0; i < cnt; i++) { \ + OMPI_FREE_LIST_WAIT(frag_list, item, rc); \ + frag = (mca_btl_openib_frag_t*) item; \ + frag->sg_entry.length = frag->size + \ + ((unsigned char*) frag->segment.seg_addr.pval- \ + (unsigned char*) frag->hdr); \ + if(ibv_post_srq_recv(srq, &frag->wr_desc.rr_desc, &bad_wr)) { \ + BTL_ERROR(("error posting receive descriptors to shared receive queue: %s",\ + strerror(errno))); \ + return OMPI_ERROR; \ + }\ + }\ + OPAL_THREAD_ADD32(srr_posted, cnt); \ + } while(0);\ +} + +#endif + #if defined(c_plusplus) || defined(__cplusplus) } #endif diff --git a/ompi/mca/btl/openib/btl_openib_component.c b/ompi/mca/btl/openib/btl_openib_component.c index 39085bfc44..7a97ba4582 100644 --- a/ompi/mca/btl/openib/btl_openib_component.c +++ b/ompi/mca/btl/openib/btl_openib_component.c @@ -35,7 +35,7 @@ #include "datatype/convertor.h" -#include "mca/mpool/mvapi/mpool_mvapi.h" +#include "mca/mpool/mpool.h" #include #include #include @@ -124,8 +124,6 @@ int mca_btl_openib_component_open(void) mca_btl_openib_param_register_int ("free_list_max", 1024); mca_btl_openib_component.ib_free_list_inc = mca_btl_openib_param_register_int ("free_list_inc", 32); - mca_btl_openib_component.ib_mem_registry_hints_log_size = - mca_btl_openib_param_register_int ("hints_log_size", 8); mca_btl_openib_component.ib_mpool_name = mca_btl_openib_param_register_string("mpool", "openib"); mca_btl_openib_component.ib_rr_buf_max = @@ -134,6 +132,8 @@ int mca_btl_openib_component_open(void) mca_btl_openib_param_register_int("rr_buf_min", 8); mca_btl_openib_component.reg_mru_len = mca_btl_openib_param_register_int("reg_mru_len", 16); + mca_btl_openib_component.use_srq = + mca_btl_openib_param_register_int("use_srq", 0); mca_btl_openib_component.ib_cq_size = mca_btl_openib_param_register_int("ib_cq_size", @@ -181,7 +181,10 @@ int mca_btl_openib_component_open(void) mca_btl_openib_component.ib_src_path_bits = mca_btl_openib_param_register_int("ib_src_path_bits", 0); - + mca_btl_openib_component.rd_per_peer = + mca_btl_openib_param_register_int("rd_per_peer", + 16); + mca_btl_openib_module.super.btl_exclusivity = @@ -229,7 +232,7 @@ int mca_btl_openib_component_close(void) /* - * Register MVAPI port information. The MCA framework + * Register OPENIB port information. The MCA framework * will make this available to all peers. */ @@ -573,7 +576,7 @@ int mca_btl_openib_component_progress() /* Process a RECV */ BTL_VERBOSE(("Got an recv on the completion queue")); - frag = (mca_btl_openib_frag_t*) (void*) wc.wr_id; + frag = (mca_btl_openib_frag_t*) (void*) (unsigned long) wc.wr_id; endpoint = (mca_btl_openib_endpoint_t*) frag->endpoint; frag->rc=OMPI_SUCCESS; frag->segment.seg_len = @@ -588,8 +591,17 @@ int mca_btl_openib_component_progress() &frag->base, openib_btl->ib_reg[frag->hdr->tag].cbdata); - OPAL_THREAD_ADD32(&endpoint->rr_posted_high, -1); - MCA_BTL_OPENIB_ENDPOINT_POST_RR_HIGH(((mca_btl_openib_frag_t*)wc.wr_id)->endpoint, 0); +#ifdef OMPI_MCA_BTL_OPENIB_HAVE_SRQ + if(mca_btl_openib_component.use_srq) { + OPAL_THREAD_ADD32(&openib_btl->srr_posted_high, -1); + MCA_BTL_OPENIB_POST_SRR_HIGH(openib_btl, 0); + } else { +#endif + OPAL_THREAD_ADD32(&endpoint->rr_posted_high, -1); + MCA_BTL_OPENIB_ENDPOINT_POST_RR_HIGH(((mca_btl_openib_frag_t*) (void*) (unsigned long) wc.wr_id)->endpoint, 0); +#ifdef OMPI_MCA_BTL_OPENIB_HAVE_SRQ + } +#endif OMPI_FREE_LIST_RETURN(&(openib_btl->recv_free_eager), (opal_list_item_t*) frag); count++; break; @@ -599,7 +611,7 @@ int mca_btl_openib_component_progress() case IBV_WC_SEND : /* Process a completed send or rdma write*/ - frag = (mca_btl_openib_frag_t*) wc.wr_id; + frag = (mca_btl_openib_frag_t*) (void*) (unsigned long) wc.wr_id; frag->rc = OMPI_SUCCESS; frag->base.des_cbfunc(&openib_btl->super, frag->endpoint, &frag->base, frag->rc); count++; @@ -632,7 +644,7 @@ int mca_btl_openib_component_progress() case IBV_WC_RECV: /* process a recv completion (this should only occur for a send not an rdma) */ BTL_VERBOSE(( "Got a recv completion")); - frag = (mca_btl_openib_frag_t*) wc.wr_id; + frag = (mca_btl_openib_frag_t*) (void*) (unsigned long) wc.wr_id; endpoint = (mca_btl_openib_endpoint_t*) frag->endpoint; frag->rc=OMPI_SUCCESS; @@ -647,8 +659,18 @@ int mca_btl_openib_component_progress() &frag->base, openib_btl->ib_reg[frag->hdr->tag].cbdata); - OPAL_THREAD_ADD32(&endpoint->rr_posted_low, -1); - MCA_BTL_OPENIB_ENDPOINT_POST_RR_LOW(((mca_btl_openib_frag_t*)wc.wr_id)->endpoint, 0); +#ifdef OMPI_MCA_BTL_OPENIB_HAVE_SRQ + if(mca_btl_openib_component.use_srq) { + OPAL_THREAD_ADD32(&openib_btl->srr_posted_low, -1); + MCA_BTL_OPENIB_POST_SRR_LOW(openib_btl, 0); + } else { +#endif + OPAL_THREAD_ADD32(&endpoint->rr_posted_low, -1); + MCA_BTL_OPENIB_ENDPOINT_POST_RR_LOW(((mca_btl_openib_frag_t*) (void*) + (unsigned long)wc.wr_id)->endpoint, 0); +#ifdef OMPI_MCA_BTL_OPENIB_HAVE_SRQ + } +#endif OMPI_FREE_LIST_RETURN(&(openib_btl->recv_free_max), (opal_list_item_t*) frag); count++; break; @@ -658,7 +680,7 @@ int mca_btl_openib_component_progress() case IBV_WC_SEND : /* Process a completed send */ - frag = (mca_btl_openib_frag_t*) wc.wr_id; + frag = (mca_btl_openib_frag_t*) (void*) (unsigned long) wc.wr_id; frag->rc = OMPI_SUCCESS; frag->base.des_cbfunc(&openib_btl->super, frag->endpoint, &frag->base, frag->rc); count++; diff --git a/ompi/mca/btl/openib/btl_openib_endpoint.c b/ompi/mca/btl/openib/btl_openib_endpoint.c index d0dbbe5c1d..7ff2725dcf 100644 --- a/ompi/mca/btl/openib/btl_openib_endpoint.c +++ b/ompi/mca/btl/openib/btl_openib_endpoint.c @@ -40,6 +40,9 @@ int mca_btl_openib_endpoint_create_qp( mca_btl_openib_module_t* openib_btl, struct ibv_pd* pd, struct ibv_cq* cq, +#if OMPI_MCA_BTL_OPENIB_HAVE_SRQ + struct ibv_srq* srq, +#endif struct ibv_qp_attr* qp_attr, struct ibv_qp** qp ); @@ -95,8 +98,18 @@ static inline int mca_btl_openib_endpoint_post_send(mca_btl_openib_module_t* ope BTL_ERROR(("error posting send request errno says %s", strerror(errno))); return OMPI_ERROR; } - MCA_BTL_OPENIB_ENDPOINT_POST_RR_HIGH(endpoint, 1); - MCA_BTL_OPENIB_ENDPOINT_POST_RR_LOW(endpoint, 1); + +#ifdef OMPI_MCA_BTL_OPENIB_HAVE_SRQ + if(mca_btl_openib_component.use_srq) { + MCA_BTL_OPENIB_POST_SRR_HIGH(openib_btl, 1); + MCA_BTL_OPENIB_POST_SRR_LOW(openib_btl, 1); + } else { +#endif + MCA_BTL_OPENIB_ENDPOINT_POST_RR_HIGH(endpoint, 1); + MCA_BTL_OPENIB_ENDPOINT_POST_RR_LOW(endpoint, 1); +#ifdef OMPI_MCA_BTL_OPENIB_HAVE_SRQ + } +#endif return OMPI_SUCCESS; } @@ -268,6 +281,9 @@ static int mca_btl_openib_endpoint_start_connect(mca_btl_base_endpoint_t* endpoi if(OMPI_SUCCESS != (rc = mca_btl_openib_endpoint_create_qp(openib_btl, openib_btl->ib_pd, openib_btl->ib_cq_high, +#ifdef OMPI_MCA_BTL_OPENIB_HAVE_SRQ + openib_btl->srq_high, +#endif endpoint->lcl_qp_attr_high, &endpoint->lcl_qp_high))) { BTL_ERROR(("error creating queue pair, error code %d", rc)); @@ -280,7 +296,10 @@ static int mca_btl_openib_endpoint_start_connect(mca_btl_base_endpoint_t* endpoi if(OMPI_SUCCESS != (rc = mca_btl_openib_endpoint_create_qp(openib_btl, openib_btl->ib_pd, openib_btl->ib_cq_low, - endpoint->lcl_qp_attr_low, +#ifdef OMPI_MCA_BTL_OPENIB_HAVE_SRQ + openib_btl->srq_low, +#endif + endpoint->lcl_qp_attr_low, &endpoint->lcl_qp_low))) { BTL_ERROR(("error creating queue pair, error code %d", rc)); return rc; @@ -316,6 +335,10 @@ static int mca_btl_openib_endpoint_reply_start_connect(mca_btl_openib_endpoint_t if(OMPI_SUCCESS != (rc = mca_btl_openib_endpoint_create_qp(openib_btl, openib_btl->ib_pd, openib_btl->ib_cq_high, +#ifdef OMPI_MCA_BTL_OPENIB_HAVE_SRQ + openib_btl->srq_high, +#endif + endpoint->lcl_qp_attr_high, &endpoint->lcl_qp_high))) { BTL_ERROR(("error creating queue pair, error code %d", rc)); @@ -328,6 +351,10 @@ static int mca_btl_openib_endpoint_reply_start_connect(mca_btl_openib_endpoint_t if(OMPI_SUCCESS != (rc = mca_btl_openib_endpoint_create_qp(openib_btl, openib_btl->ib_pd, openib_btl->ib_cq_low, +#ifdef OMPI_MCA_BTL_OPENIB_HAVE_SRQ + openib_btl->srq_low, +#endif + endpoint->lcl_qp_attr_low, &endpoint->lcl_qp_low))) { BTL_ERROR(("error creating queue pair, error code %d", rc)); @@ -728,9 +755,18 @@ int mca_btl_openib_endpoint_connect( if(rc != OMPI_SUCCESS) { return rc; } - - MCA_BTL_OPENIB_ENDPOINT_POST_RR_HIGH(endpoint, 0); - MCA_BTL_OPENIB_ENDPOINT_POST_RR_LOW(endpoint, 0); + +#ifdef OMPI_MCA_BTL_OPENIB_HAVE_SRQ + if(mca_btl_openib_component.use_srq) { + MCA_BTL_OPENIB_POST_SRR_HIGH(openib_btl, 1); + MCA_BTL_OPENIB_POST_SRR_LOW(openib_btl, 1); + } else { +#endif + MCA_BTL_OPENIB_ENDPOINT_POST_RR_HIGH(endpoint, 1); + MCA_BTL_OPENIB_ENDPOINT_POST_RR_LOW(endpoint, 1); +#ifdef OMPI_MCA_BTL_OPENIB_HAVE_SRQ + } +#endif return OMPI_SUCCESS; } @@ -745,6 +781,9 @@ int mca_btl_openib_endpoint_create_qp( mca_btl_openib_module_t* openib_btl, struct ibv_pd* pd, struct ibv_cq* cq, +#ifdef OMPI_MCA_BTL_OPENIB_HAVE_SRQ + struct ibv_srq* srq, +#endif struct ibv_qp_attr* qp_attr, struct ibv_qp** qp ) @@ -762,8 +801,11 @@ int mca_btl_openib_endpoint_create_qp( qp_init_attr.cap.max_send_sge = mca_btl_openib_component.ib_sg_list_size; qp_init_attr.cap.max_recv_sge = mca_btl_openib_component.ib_sg_list_size; qp_init_attr.qp_type = IBV_QPT_RC; - - +#ifdef OMPI_MCA_BTL_OPENIB_HAVE_SRQ + if(mca_btl_openib_component.use_srq) { + qp_init_attr.srq = srq; + } +#endif my_qp = ibv_create_qp(pd, &qp_init_attr); if(NULL == my_qp) {