diff --git a/src/mca/pml/teg/src/pml_teg_isend.c b/src/mca/pml/teg/src/pml_teg_isend.c index 718d09a9b8..3ba96aa028 100644 --- a/src/mca/pml/teg/src/pml_teg_isend.c +++ b/src/mca/pml/teg/src/pml_teg_isend.c @@ -117,3 +117,4 @@ int mca_pml_teg_send(void *buf, MCA_PML_TEG_FREE((ompi_request_t **) & sendreq); return OMPI_SUCCESS; } + diff --git a/src/mca/pml/teg/src/pml_teg_ptl.h b/src/mca/pml/teg/src/pml_teg_ptl.h index 3d7eb80338..ce1c909769 100644 --- a/src/mca/pml/teg/src/pml_teg_ptl.h +++ b/src/mca/pml/teg/src/pml_teg_ptl.h @@ -2,10 +2,13 @@ #define _MCA_PML_BASE_PTL_ #include "mca/ptl/ptl.h" +#include "threads/condition.h" + struct mca_pml_base_ptl_t { ompi_list_t ptl_cache; /**< cache of send requests */ size_t ptl_cache_size; /**< maximum size of cache */ + size_t ptl_cache_alloc; /**< current number of allocated items */ ompi_mutex_t ptl_cache_lock; /**< lock for queue access */ struct mca_ptl_base_module_t* ptl; /**< back pointer to ptl */ }; diff --git a/src/mca/pml/teg/src/pml_teg_sendreq.h b/src/mca/pml/teg/src/pml_teg_sendreq.h index 732bec06ca..2b351dba15 100644 --- a/src/mca/pml/teg/src/pml_teg_sendreq.h +++ b/src/mca/pml/teg/src/pml_teg_sendreq.h @@ -35,16 +35,29 @@ OMPI_THREAD_LOCK(&ptl_base->ptl_cache_lock); \ sendreq = (mca_pml_base_send_request_t*) \ ompi_list_remove_first(&ptl_base->ptl_cache); \ - OMPI_THREAD_UNLOCK(&ptl_base->ptl_cache_lock); \ if(NULL != sendreq) { \ + OMPI_THREAD_UNLOCK(&ptl_base->ptl_cache_lock); \ rc = OMPI_SUCCESS; \ - } else { \ + } else if (ptl_base->ptl_cache_alloc < ptl_base->ptl_cache_size) { \ mca_ptl_base_module_t* ptl = ptl_base->ptl; \ ompi_list_item_t* item; \ OMPI_FREE_LIST_WAIT(&mca_pml_teg.teg_send_requests, item, rc); \ sendreq = (mca_pml_base_send_request_t*)item; \ sendreq->req_ptl = ptl; \ - ptl->ptl_request_init(ptl, sendreq); \ + if(ptl->ptl_request_init(ptl, sendreq) == OMPI_SUCCESS) { \ + sendreq->req_cached = true; \ + ptl_base->ptl_cache_alloc++; \ + } \ + OMPI_THREAD_UNLOCK(&ptl_base->ptl_cache_lock); \ + } else { \ + /* \ + * take a request from the global pool \ + */ \ + ompi_list_item_t* item; \ + OMPI_THREAD_UNLOCK(&ptl_base->ptl_cache_lock); \ + OMPI_FREE_LIST_WAIT(&mca_pml_teg.teg_send_requests, item, rc); \ + sendreq = (mca_pml_base_send_request_t*)item; \ + sendreq->req_ptl = ptl_proc->ptl; \ } \ \ /* otherwise - take the allocation from the global list */ \ @@ -59,33 +72,26 @@ } -#define MCA_PML_TEG_SEND_REQUEST_RETURN(request) \ +#define MCA_PML_TEG_SEND_REQUEST_RETURN(sendreq) \ { \ - mca_ptl_base_module_t* ptl = sendreq->req_ptl; \ + mca_ptl_base_module_t* ptl = (sendreq)->req_ptl; \ mca_pml_base_ptl_t* ptl_base = ptl->ptl_base; \ \ /* Decrement reference count on communicator. */ \ - OBJ_RELEASE(request->req_base.req_comm); \ + OBJ_RELEASE((sendreq)->req_base.req_comm); \ \ /* \ * If there is a cache associated with the ptl - first attempt \ * to return the send descriptor to the cache. \ */ \ - if(NULL != ptl->ptl_base) { \ + if(NULL != ptl->ptl_base && (sendreq)->req_cached) { \ OMPI_THREAD_LOCK(&ptl_base->ptl_cache_lock); \ - if(ompi_list_get_size(&ptl_base->ptl_cache) >= ptl_base->ptl_cache_size) {\ - /* if cache limit is exceeded - return to global pool */ \ - ptl->ptl_request_fini(ptl, sendreq); \ - OMPI_FREE_LIST_RETURN(&mca_pml_teg.teg_send_requests, \ - (ompi_list_item_t*)sendreq); \ - } else { \ - ompi_list_prepend(&ptl_base->ptl_cache, \ - (ompi_list_item_t*)sendreq); \ - } \ + ompi_list_prepend(&ptl_base->ptl_cache, \ + (ompi_list_item_t*)sendreq); \ OMPI_THREAD_UNLOCK(&ptl_base->ptl_cache_lock); \ } else { \ OMPI_FREE_LIST_RETURN( \ - &mca_pml_teg.teg_send_requests, (ompi_list_item_t*)request); \ + &mca_pml_teg.teg_send_requests, (ompi_list_item_t*)sendreq); \ } \ }