diff --git a/src/mca/ptl/portals/src/ptl_portals.c b/src/mca/ptl/portals/src/ptl_portals.c index f77fdc181a..ae6cfccb4c 100644 --- a/src/mca/ptl/portals/src/ptl_portals.c +++ b/src/mca/ptl/portals/src/ptl_portals.c @@ -140,14 +140,16 @@ mca_ptl_portals_module_enable(struct mca_ptl_portals_module_t *ptl, /* BWB - not really sure how - would have to track a lot more data... */ } else { /* only do all the hard stuff if we haven't created the queue */ - if (ptl->frag_eq_handle != PTL_EQ_NONE) return OMPI_SUCCESS; + if (ptl->frag_eq_handles[MCA_PTL_PORTALS_EQ_FRAGS] != PTL_EQ_NONE) { + return OMPI_SUCCESS; + } /* create an event queue, then the match entries for the match entries */ ret = PtlEQAlloc(ptl->ni_handle, ptl->event_queue_size, PTL_EQ_HANDLER_NONE, - &(ptl->frag_eq_handle)); + &(ptl->frag_eq_handles[MCA_PTL_PORTALS_EQ_FRAGS])); if (ret != PTL_OK) { ompi_output(mca_ptl_portals_component.portals_output, "Failed to allocate event queue: %d", ret); diff --git a/src/mca/ptl/portals/src/ptl_portals.h b/src/mca/ptl/portals/src/ptl_portals.h index e37e358098..a4f81af6ca 100644 --- a/src/mca/ptl/portals/src/ptl_portals.h +++ b/src/mca/ptl/portals/src/ptl_portals.h @@ -81,6 +81,9 @@ struct mca_ptl_portals_component_t { typedef struct mca_ptl_portals_component_t mca_ptl_portals_component_t; +#define MCA_PTL_PORTALS_EQ_FRAGS 0 +#define MCA_PTL_PORTALS_EQ_SIZE 1 + struct mca_ptl_portals_module_t { /* base PTL module interface */ mca_ptl_base_module_t super; @@ -93,7 +96,7 @@ struct mca_ptl_portals_module_t { int event_queue_size; /* frag receive event queue */ - ptl_handle_eq_t frag_eq_handle; + ptl_handle_eq_t frag_eq_handles[MCA_PTL_PORTALS_EQ_SIZE]; /* our portals network interface */ ptl_handle_ni_t ni_handle; @@ -108,7 +111,6 @@ typedef struct mca_ptl_portals_module_t mca_ptl_portals_module_t; struct mca_ptl_portals_recv_frag_t; struct mca_ptl_portals_send_frag_t; - /* * Component functions (ptl_portals_component.c) */ diff --git a/src/mca/ptl/portals/src/ptl_portals_component.c b/src/mca/ptl/portals/src/ptl_portals_component.c index 5c1ce2f2da..73460933ec 100644 --- a/src/mca/ptl/portals/src/ptl_portals_component.c +++ b/src/mca/ptl/portals/src/ptl_portals_component.c @@ -105,6 +105,8 @@ param_register_int(const char* param_name, int mca_ptl_portals_component_open(void) { + int i; + /* initialize state */ mca_ptl_portals_component.portals_num_modules = 0; mca_ptl_portals_component.portals_modules = NULL; @@ -169,7 +171,10 @@ mca_ptl_portals_component_open(void) ompi_output_open(&portals_output_stream); /* fill in remaining defaults for module data */ - mca_ptl_portals_module.frag_eq_handle = PTL_EQ_NONE; + for (i = 0 ; i < MCA_PTL_PORTALS_EQ_SIZE ; ++i) { + mca_ptl_portals_module.frag_eq_handles[i] = PTL_EQ_NONE; + } + mca_ptl_portals_module.ni_handle = PTL_INVALID_HANDLE; mca_ptl_portals_module.dropped = 0; @@ -303,7 +308,8 @@ mca_ptl_portals_component_progress(mca_ptl_tstamp_t tstamp) int which; int ret; - if (module->frag_eq_handle == PTL_EQ_NONE) continue; + if (module->frag_eq_handles[MCA_PTL_PORTALS_EQ_SIZE - 1] == + PTL_EQ_NONE) continue; /* they are all initialized at once */ #if OMPI_ENABLE_DEBUG /* BWB - this is going to kill performance */ @@ -318,8 +324,8 @@ mca_ptl_portals_component_progress(mca_ptl_tstamp_t tstamp) } #endif - ret = PtlEQPoll(&(module->frag_eq_handle), - 1, /* number of eq handles */ + ret = PtlEQPoll(module->frag_eq_handles, + MCA_PTL_PORTALS_EQ_SIZE, /* number of eq handles */ (int) tstamp, &ev, &which); @@ -337,7 +343,7 @@ mca_ptl_portals_component_progress(mca_ptl_tstamp_t tstamp) } /* only one place we can have an event */ - assert(which == 0); + assert(which == MCA_PTL_PORTALS_EQ_FRAGS); #if PTL_PORTALS_HAVE_EVENT_UNLINK /* not everyone has UNLINK. Use it only to print the event, diff --git a/src/mca/ptl/portals/src/ptl_portals_recv.c b/src/mca/ptl/portals/src/ptl_portals_recv.c index 6dbc4087d2..9666198b6f 100644 --- a/src/mca/ptl/portals/src/ptl_portals_recv.c +++ b/src/mca/ptl/portals/src/ptl_portals_recv.c @@ -69,7 +69,7 @@ ptl_portals_post_recv_md(struct mca_ptl_portals_module_t *ptl, void *data_ptr) md.max_size = ptl->super.ptl_first_frag_size; md.options = PTL_MD_OP_PUT | PTL_MD_MAX_SIZE; md.user_ptr = NULL; - md.eq_handle = ptl->frag_eq_handle; + md.eq_handle = ptl->frag_eq_handles[MCA_PTL_PORTALS_EQ_FRAGS]; ret = PtlMDAttach(me_handle, md, diff --git a/src/mca/ptl/portals/src/ptl_portals_send.h b/src/mca/ptl/portals/src/ptl_portals_send.h index 03ec489a4f..18b31be858 100644 --- a/src/mca/ptl/portals/src/ptl_portals_send.h +++ b/src/mca/ptl/portals/src/ptl_portals_send.h @@ -54,7 +54,7 @@ mca_ptl_portals_send_frag(struct mca_ptl_portals_module_t *ptl, md.max_size = 0; md.options = PTL_MD_IOVEC; /* BWB - can we optimize? */ md.user_ptr = sendfrag; - md.eq_handle = ptl->frag_eq_handle; + md.eq_handle = ptl->frag_eq_handles[MCA_PTL_PORTALS_EQ_FRAGS]; /* make a free-floater */ ret = PtlMDBind(ptl->ni_handle,