From 3d027e57a8bfecba3be9b0701ee2e0b7ad0c1268 Mon Sep 17 00:00:00 2001 From: Brian Barrett Date: Sat, 17 Jun 2006 21:32:25 +0000 Subject: [PATCH] * fix for ticket #141. If we are going to shortcut out of polling the send/receive queues if there is something available in the short message rdma queues, then we have to poll *ALL* the rdma queues before exiting, or we aren't fair about frag reception and fall into degenerate matching cases. This commit was SVN r10410. --- ompi/mca/btl/mvapi/btl_mvapi_component.c | 13 +++++++------ ompi/mca/btl/openib/btl_openib_component.c | 21 +++++++++++---------- 2 files changed, 18 insertions(+), 16 deletions(-) diff --git a/ompi/mca/btl/mvapi/btl_mvapi_component.c b/ompi/mca/btl/mvapi/btl_mvapi_component.c index 5266336887..5edf044483 100644 --- a/ompi/mca/btl/mvapi/btl_mvapi_component.c +++ b/ompi/mca/btl/mvapi/btl_mvapi_component.c @@ -727,10 +727,8 @@ int mca_btl_mvapi_component_progress( void ) mca_btl_mvapi_frag_t* frag; mca_btl_mvapi_endpoint_t* endpoint; - /* Poll for completions */ + /* Poll for RDMA completions - if any succeed, we don't process the slower queues */ for(i = 0; i < mca_btl_mvapi_component.ib_num_btls; i++) { - VAPI_ret_t ret; - VAPI_wc_desc_t comp; mca_btl_mvapi_module_t* mvapi_btl = &mca_btl_mvapi_component.mvapi_btls[i]; OPAL_THREAD_LOCK(&mvapi_btl->eager_rdma_lock); @@ -774,10 +772,13 @@ int mca_btl_mvapi_component_progress( void ) } else OPAL_THREAD_UNLOCK(&endpoint->eager_rdma_local.lock); } + } + if (count) return count; - if(count) - break; - + for(i = 0; i < mca_btl_mvapi_component.ib_num_btls; i++) { + VAPI_ret_t ret; + VAPI_wc_desc_t comp; + mca_btl_mvapi_module_t* mvapi_btl = &mca_btl_mvapi_component.mvapi_btls[i]; /* we have two completion queues, one for "high" priority and one for "low". * we will check the high priority and process them until there are none left. * note that low priority messages are only processed one per progress call. diff --git a/ompi/mca/btl/openib/btl_openib_component.c b/ompi/mca/btl/openib/btl_openib_component.c index bd7a7b4ff1..0c9545368c 100644 --- a/ompi/mca/btl/openib/btl_openib_component.c +++ b/ompi/mca/btl/openib/btl_openib_component.c @@ -812,16 +812,9 @@ int mca_btl_openib_component_progress() mca_btl_openib_frag_t* frag; mca_btl_openib_endpoint_t* endpoint; - /* Poll for completions */ + /* Poll for RDMA completions - if any succeed, we don't process the slower queues */ for(i = 0; i < mca_btl_openib_component.ib_num_btls; i++) { - - struct ibv_wc wc; mca_btl_openib_module_t* openib_btl = &mca_btl_openib_component.openib_btls[i]; - - /* we have two completion queues, one for "high" priority and one for "low". - * we will check the high priority and process them until there are none left. - * note that low priority messages are only processed one per progress call. - */ OPAL_THREAD_LOCK(&openib_btl->eager_rdma_lock); c = openib_btl->eager_rdma_buffers_count; @@ -864,9 +857,17 @@ int mca_btl_openib_component_progress() } else OPAL_THREAD_UNLOCK(&endpoint->eager_rdma_local.lock); } + } + if(count) return count; - if(count) - break; + for(i = 0; i < mca_btl_openib_component.ib_num_btls; i++) { + struct ibv_wc wc; + mca_btl_openib_module_t* openib_btl = &mca_btl_openib_component.openib_btls[i]; + + /* we have two completion queues, one for "high" priority and one for "low". + * we will check the high priority and process them until there are none left. + * note that low priority messages are only processed one per progress call. + */ ne=ibv_poll_cq(openib_btl->ib_cq_hp, 1, &wc ); if(ne < 0 ){