From 09057fe3112098663375e637792addc42df288ad Mon Sep 17 00:00:00 2001 From: Brian Barrett Date: Sun, 15 May 2005 21:05:00 +0000 Subject: [PATCH] * massive cleanup of debugging output to make it much easier to match messages * use different event queues for send / recv, part of moving towards dealing with dropped fragments This commit was SVN r5719. --- src/mca/ptl/portals/configure.stub | 32 +++++----- src/mca/ptl/portals/src/ptl_portals.c | 24 ++++---- src/mca/ptl/portals/src/ptl_portals.h | 11 ++-- .../ptl/portals/src/ptl_portals_compat_utcp.c | 17 ++++++ .../ptl/portals/src/ptl_portals_component.c | 51 +++++++++------- src/mca/ptl/portals/src/ptl_portals_recv.c | 23 +++---- src/mca/ptl/portals/src/ptl_portals_recv.h | 14 +++++ src/mca/ptl/portals/src/ptl_portals_send.c | 60 ++++++++++++++----- src/mca/ptl/portals/src/ptl_portals_send.h | 4 +- 9 files changed, 157 insertions(+), 79 deletions(-) diff --git a/src/mca/ptl/portals/configure.stub b/src/mca/ptl/portals/configure.stub index 00d629dd8f..9465a3357c 100644 --- a/src/mca/ptl/portals/configure.stub +++ b/src/mca/ptl/portals/configure.stub @@ -160,46 +160,50 @@ AC_DEFUN([MCA_CONFIGURE_STUB],[ [Portals table id to use for retransmit request queue]) MCA_PTL_PORTALS_CONFIG_VAL([debug-level], - [PTL_PORTALS_DEFAULT_DEBUG_LEVEL], [99], - [Default debugging level for portals ptl]) + [PTL_PORTALS_DEFAULT_DEBUG_LEVEL], [100], + [debugging level for portals ptl]) MCA_PTL_PORTALS_CONFIG_VAL([request-cache-size], [PTL_PORTALS_DEFAULT_REQUEST_CACHE_SIZE], [1], - [Default request cache size for portals ptl]) + [request cache size for portals ptl]) MCA_PTL_PORTALS_CONFIG_VAL([first-frag-size], [PTL_PORTALS_DEFAULT_FIRST_FRAG_SIZE], [16384], - [Default first frag size for portals ptl]) + [first frag size for portals ptl]) MCA_PTL_PORTALS_CONFIG_VAL([first-frag-num-entries], [PTL_PORTALS_DEFAULT_FIRST_FRAG_NUM_ENTRIES], [3], - [Default number of memory descriptors for first fragments]) + [number of memory descriptors for first fragments]) MCA_PTL_PORTALS_CONFIG_VAL([first-frag-entry-size], [PTL_PORTALS_DEFAULT_FIRST_FRAG_ENTRY_SIZE], [2098152], - [Default size of memory associeted with first fag md]) + [size of memory associeted with first fag md]) - MCA_PTL_PORTALS_CONFIG_VAL([event-queue-size], - [PTL_PORTALS_DEFAULT_EVENT_QUEUE_SIZE], [512], - [Default size of event queue for first frag mds]) + MCA_PTL_PORTALS_CONFIG_VAL([recv-queue-size], + [PTL_PORTALS_DEFAULT_RECV_QUEUE_SIZE], [512], + [size of event queue for receiving frags]) + + MCA_PTL_PORTALS_CONFIG_VAL([send-queue-size], + [PTL_PORTALS_DEFAULT_SEND_QUEUE_SIZE], [128], + [Max number of send fragmenst pending]) MCA_PTL_PORTALS_CONFIG_VAL([rndv-frag-min-size], [PTL_PORTALS_DEFAULT_RNDV_FRAG_MIN_SIZE], [0], - [Default minimum size of rndv fragments]) + [minimum size of rndv fragments]) MCA_PTL_PORTALS_CONFIG_VAL([rndv-frag-max-size], [PTL_PORTALS_DEFAULT_RNDV_FRAG_MAX_SIZE], [16384], - [Default maximum size of rndv fragments]) + [maximum size of rndv fragments]) MCA_PTL_PORTALS_CONFIG_VAL([free-list-init-num], [PTL_PORTALS_DEFAULT_FREE_LIST_INIT_NUM], [256], - [Default starting size of free lists]) + [starting size of free lists]) MCA_PTL_PORTALS_CONFIG_VAL([free-list-max-num], [PTL_PORTALS_DEFAULT_FREE_LIST_MAX_NUM], [-1], - [Default maximum size of free lists]) + [maximum size of free lists]) MCA_PTL_PORTALS_CONFIG_VAL([free-list-inc-num], [PTL_PORTALS_DEFAULT_FREE_LIST_inc_NUM], [256], - [Default grow size for freelists]) + [grow size for freelists]) # # Save extra compiler/linker flags so that they can be added in diff --git a/src/mca/ptl/portals/src/ptl_portals.c b/src/mca/ptl/portals/src/ptl_portals.c index ae6cfccb4c..3c343824a4 100644 --- a/src/mca/ptl/portals/src/ptl_portals.c +++ b/src/mca/ptl/portals/src/ptl_portals.c @@ -140,20 +140,22 @@ mca_ptl_portals_module_enable(struct mca_ptl_portals_module_t *ptl, /* BWB - not really sure how - would have to track a lot more data... */ } else { /* only do all the hard stuff if we haven't created the queue */ - if (ptl->frag_eq_handles[MCA_PTL_PORTALS_EQ_FRAGS] != PTL_EQ_NONE) { + if (ptl->eq_handles[MCA_PTL_PORTALS_EQ_SIZE - 1] != PTL_EQ_NONE) { return OMPI_SUCCESS; } /* create an event queue, then the match entries for the match entries */ - ret = PtlEQAlloc(ptl->ni_handle, - ptl->event_queue_size, - PTL_EQ_HANDLER_NONE, - &(ptl->frag_eq_handles[MCA_PTL_PORTALS_EQ_FRAGS])); - if (ret != PTL_OK) { - ompi_output(mca_ptl_portals_component.portals_output, - "Failed to allocate event queue: %d", ret); - return OMPI_ERROR; + for (i = 0 ; i < MCA_PTL_PORTALS_EQ_SIZE ; ++i) { + ret = PtlEQAlloc(ptl->ni_handle, + ptl->eq_sizes[i], + PTL_EQ_HANDLER_NONE, + &(ptl->eq_handles[i])); + if (ret != PTL_OK) { + ompi_output(mca_ptl_portals_component.portals_output, + "Failed to allocate event queue: %d", ret); + return OMPI_ERROR; + } } for (i = 0 ; i < ptl->first_frag_num_entries ; ++i) { @@ -176,9 +178,11 @@ mca_ptl_portals_finalize(struct mca_ptl_base_module_t *ptl_base) ret = PtlNIFini(ptl->ni_handle); if (PTL_OK != ret) { ompi_output_verbose(20, mca_ptl_portals_component.portals_output, - "PtlNIFini returned %d\n", ret); + "PtlNIFini returned %d", ret); return OMPI_ERROR; } + ompi_output_verbose(20, mca_ptl_portals_component.portals_output, + "successfully finalized module"); return OMPI_SUCCESS; } diff --git a/src/mca/ptl/portals/src/ptl_portals.h b/src/mca/ptl/portals/src/ptl_portals.h index a4f81af6ca..1162bdda94 100644 --- a/src/mca/ptl/portals/src/ptl_portals.h +++ b/src/mca/ptl/portals/src/ptl_portals.h @@ -81,8 +81,9 @@ struct mca_ptl_portals_component_t { typedef struct mca_ptl_portals_component_t mca_ptl_portals_component_t; -#define MCA_PTL_PORTALS_EQ_FRAGS 0 -#define MCA_PTL_PORTALS_EQ_SIZE 1 +#define MCA_PTL_PORTALS_EQ_RECV 0 +#define MCA_PTL_PORTALS_EQ_SEND 1 +#define MCA_PTL_PORTALS_EQ_SIZE 2 struct mca_ptl_portals_module_t { /* base PTL module interface */ @@ -92,11 +93,11 @@ struct mca_ptl_portals_module_t { int first_frag_num_entries; /* size of each md for first frags */ int first_frag_entry_size; - /* size for event queue */ - int event_queue_size; + /* size for event queue */ + int eq_sizes[MCA_PTL_PORTALS_EQ_SIZE]; /* frag receive event queue */ - ptl_handle_eq_t frag_eq_handles[MCA_PTL_PORTALS_EQ_SIZE]; + ptl_handle_eq_t eq_handles[MCA_PTL_PORTALS_EQ_SIZE]; /* our portals network interface */ ptl_handle_ni_t ni_handle; diff --git a/src/mca/ptl/portals/src/ptl_portals_compat_utcp.c b/src/mca/ptl/portals/src/ptl_portals_compat_utcp.c index 975b12d0aa..3af1741da5 100644 --- a/src/mca/ptl/portals/src/ptl_portals_compat_utcp.c +++ b/src/mca/ptl/portals/src/ptl_portals_compat_utcp.c @@ -30,6 +30,8 @@ #include "ptl_portals.h" #include "ptl_portals_compat.h" +#include + /* how's this for source code diving? - find private method for getting interface */ extern unsigned int utcp_my_nid(const char *if_str); @@ -43,9 +45,20 @@ mca_ptl_portals_init(mca_ptl_portals_component_t *comp) { ptl_process_id_t info; int ret; +#if 0 + FILE *output; + char *tmp; + asprintf(&tmp, "portals.%d", getpid()); + output = fopen(tmp, "w"); + free(tmp); + + utcp_lib_out = output; + utcp_api_out = output; +#else utcp_lib_out = stderr; utcp_api_out = stderr; +#endif info.nid = htonl(utcp_my_nid(mca_ptl_portals_component.portals_ifname)); info.pid = htonl((ptl_pid_t) getpid()); @@ -197,6 +210,10 @@ mca_ptl_portals_add_procs_compat(struct mca_ptl_portals_module_t* ptl, return OMPI_ERR_FATAL; } +#if 0 + PtlNIDebug(ptl->ni_handle, PTL_DBG_ALL | PTL_DBG_NI_ALL); +#endif + return OMPI_SUCCESS; } diff --git a/src/mca/ptl/portals/src/ptl_portals_component.c b/src/mca/ptl/portals/src/ptl_portals_component.c index 73460933ec..00d82e718a 100644 --- a/src/mca/ptl/portals/src/ptl_portals_component.c +++ b/src/mca/ptl/portals/src/ptl_portals_component.c @@ -62,17 +62,17 @@ mca_ptl_portals_component_t mca_ptl_portals_component = { static ompi_output_stream_t portals_output_stream = { - true, - 0, - 0, - 0, - NULL, - NULL, - false, - true, - false, - false, - NULL + true, /* is debugging */ + 0, /* verbose level */ + 0, /* want syslog */ + 0, /* syslog priority */ + NULL, /* syslog ident */ + NULL, /* prefix */ + false, /* want stdout */ + false, /* want stderr */ + true, /* want file */ + false, /* file append */ + "ptl-portals" /* file suffix */ }; @@ -159,9 +159,13 @@ mca_ptl_portals_component_open(void) mca_ptl_portals_module.first_frag_entry_size = param_register_int("first_frag_entry_size", PTL_PORTALS_DEFAULT_FIRST_FRAG_ENTRY_SIZE); - mca_ptl_portals_module.event_queue_size = - param_register_int("event_queue_size", - PTL_PORTALS_DEFAULT_EVENT_QUEUE_SIZE); + + mca_ptl_portals_module.eq_sizes[MCA_PTL_PORTALS_EQ_RECV] = + param_register_int("recv_queue_size", + PTL_PORTALS_DEFAULT_RECV_QUEUE_SIZE); + mca_ptl_portals_module.eq_sizes[MCA_PTL_PORTALS_EQ_SEND] = + (param_register_int("send_queue_size", + PTL_PORTALS_DEFAULT_SEND_QUEUE_SIZE)) * 3; /* finish with objects */ asprintf(&(portals_output_stream.lds_prefix), @@ -172,7 +176,7 @@ mca_ptl_portals_component_open(void) /* fill in remaining defaults for module data */ for (i = 0 ; i < MCA_PTL_PORTALS_EQ_SIZE ; ++i) { - mca_ptl_portals_module.frag_eq_handles[i] = PTL_EQ_NONE; + mca_ptl_portals_module.eq_handles[i] = PTL_EQ_NONE; } mca_ptl_portals_module.ni_handle = PTL_INVALID_HANDLE; @@ -299,6 +303,7 @@ mca_ptl_portals_component_progress(mca_ptl_tstamp_t tstamp) { int num_progressed = 0; size_t i; + tstamp = 10; for (i = 0 ; i < mca_ptl_portals_component.portals_num_modules ; ++i) { struct mca_ptl_portals_module_t *module = @@ -308,7 +313,7 @@ mca_ptl_portals_component_progress(mca_ptl_tstamp_t tstamp) int which; int ret; - if (module->frag_eq_handles[MCA_PTL_PORTALS_EQ_SIZE - 1] == + if (module->eq_handles[MCA_PTL_PORTALS_EQ_SIZE - 1] == PTL_EQ_NONE) continue; /* they are all initialized at once */ #if OMPI_ENABLE_DEBUG @@ -324,7 +329,7 @@ mca_ptl_portals_component_progress(mca_ptl_tstamp_t tstamp) } #endif - ret = PtlEQPoll(module->frag_eq_handles, + ret = PtlEQPoll(module->eq_handles, MCA_PTL_PORTALS_EQ_SIZE, /* number of eq handles */ (int) tstamp, &ev, @@ -339,25 +344,23 @@ mca_ptl_portals_component_progress(mca_ptl_tstamp_t tstamp) continue; } else if (PTL_EQ_DROPPED == ret) { ompi_output_verbose(10, mca_ptl_portals_component.portals_output, - "event queue entries were dropped"); + "*** Event queue entries were dropped"); } - /* only one place we can have an event */ - assert(which == MCA_PTL_PORTALS_EQ_FRAGS); - #if PTL_PORTALS_HAVE_EVENT_UNLINK /* not everyone has UNLINK. Use it only to print the event, so we can make sure we properly re-initialize the ones that need to be re-initialized */ if (PTL_EVENT_UNLINK == ev.type) { - ompi_output_verbose(100, mca_ptl_portals_component.portals_output, - "unlink event occurred"); + OMPI_OUTPUT_VERBOSE((100, mca_ptl_portals_component.portals_output, + "unlink event occurred")); continue; } #endif if (ev.md.user_ptr == NULL) { /* no fragment associated with it - it's a receive */ + assert(which == MCA_PTL_PORTALS_EQ_RECV); mca_ptl_portals_process_recv_event(module, &ev); } else { /* there's a fragment associated with it - choose based on @@ -365,8 +368,10 @@ mca_ptl_portals_component_progress(mca_ptl_tstamp_t tstamp) mca_ptl_base_frag_t *frag = (mca_ptl_base_frag_t*) ev.md.user_ptr; if (frag->frag_type == MCA_PTL_FRAGMENT_SEND) { + assert(which == MCA_PTL_PORTALS_EQ_SEND); mca_ptl_portals_process_send_event(&ev); } else { + assert(which == MCA_PTL_PORTALS_EQ_RECV); mca_ptl_portals_process_recv_event(module, &ev); } } diff --git a/src/mca/ptl/portals/src/ptl_portals_recv.c b/src/mca/ptl/portals/src/ptl_portals_recv.c index 9666198b6f..029924db80 100644 --- a/src/mca/ptl/portals/src/ptl_portals_recv.c +++ b/src/mca/ptl/portals/src/ptl_portals_recv.c @@ -69,7 +69,7 @@ ptl_portals_post_recv_md(struct mca_ptl_portals_module_t *ptl, void *data_ptr) md.max_size = ptl->super.ptl_first_frag_size; md.options = PTL_MD_OP_PUT | PTL_MD_MAX_SIZE; md.user_ptr = NULL; - md.eq_handle = ptl->frag_eq_handles[MCA_PTL_PORTALS_EQ_FRAGS]; + md.eq_handle = ptl->eq_handles[MCA_PTL_PORTALS_EQ_RECV]; ret = PtlMDAttach(me_handle, md, @@ -94,12 +94,12 @@ mca_ptl_portals_process_recv_event(struct mca_ptl_portals_module_t *ptl, int ret; if (ev->type == PTL_EVENT_PUT_START) { - OMPI_OUTPUT_VERBOSE((100, mca_ptl_portals_component.portals_output, - "PUT_START event received (%ld)", ev->link)); + OMPI_OUTPUT_VERBOSE((101, mca_ptl_portals_component.portals_output, + "starting to receive message", ev->link)); } else if (ev->type == PTL_EVENT_PUT_END) { mca_ptl_base_header_t *hdr; - OMPI_OUTPUT_VERBOSE((100, mca_ptl_portals_component.portals_output, + OMPI_OUTPUT_VERBOSE((101, mca_ptl_portals_component.portals_output, "message %ld received, start: %p, mlength: %lld," " offset: %lld", ev->link, ev->md.start, ev->mlength, ev->offset)); @@ -129,21 +129,24 @@ mca_ptl_portals_process_recv_event(struct mca_ptl_portals_module_t *ptl, { mca_ptl_portals_send_frag_t *sendfrag; mca_ptl_base_send_request_t *sendreq; - OMPI_OUTPUT_VERBOSE((100, - mca_ptl_portals_component.portals_output, - "received ack for request %p", - hdr->hdr_ack.hdr_dst_match)); sendfrag = hdr->hdr_ack.hdr_src_ptr.pval; sendreq = sendfrag->frag_send.frag_request; sendreq->req_peer_match = hdr->hdr_ack.hdr_dst_match; + + OMPI_OUTPUT_VERBOSE((100, + mca_ptl_portals_component.portals_output, + "received ack for recv request %p (msg %d)", + hdr->hdr_ack.hdr_dst_match, + sendreq->req_send.req_base.req_sequence)); + mca_ptl_portals_complete_send_event(sendfrag); } break; default: ompi_output(mca_ptl_portals_component.portals_output, - "unable to deal with header of type %d", + "*** unable to deal with header of type %d", hdr->hdr_common.hdr_type); break; } @@ -160,7 +163,7 @@ mca_ptl_portals_process_recv_event(struct mca_ptl_portals_module_t *ptl, } else { ompi_output_verbose(10, mca_ptl_portals_component.portals_output, - "unknown event: %d (%ld)", + "*** unknown event: %d (%ld)", ev->type, ev->link); } diff --git a/src/mca/ptl/portals/src/ptl_portals_recv.h b/src/mca/ptl/portals/src/ptl_portals_recv.h index 63e2dee401..792270f77d 100644 --- a/src/mca/ptl/portals/src/ptl_portals_recv.h +++ b/src/mca/ptl/portals/src/ptl_portals_recv.h @@ -89,6 +89,12 @@ mca_ptl_portals_process_first_frag(struct mca_ptl_portals_module_t *ptl, recvfrag = mca_ptl_portals_recv_get_frag(ptl, hdr, ev, header_size); if (NULL == recvfrag) return OMPI_ERR_TEMP_OUT_OF_RESOURCE; + OMPI_OUTPUT_VERBOSE((100, mca_ptl_portals_component.portals_output, + "recving first frag of size %d for msg %d from %lu", + recvfrag->frag_size, + (int) hdr->hdr_match.hdr_msg_seq, + ev->initiator.pid)); + recvfrag->frag_recv.frag_request = NULL; ptl->super.ptl_match(&ptl->super, &recvfrag->frag_recv, &hdr->hdr_match); @@ -141,6 +147,14 @@ mca_ptl_portals_process_frag_frag(struct mca_ptl_portals_module_t *ptl, &bytes_delivered, &free_after ); } + OMPI_OUTPUT_VERBOSE((100, mca_ptl_portals_component.portals_output, + "recving secnd frag of size %d for msg %d, offset %lld from %lu, %p", + recvfrag->frag_size, + (int) hdr->hdr_match.hdr_msg_seq, + hdr->hdr_frag.hdr_frag_offset, + ev->initiator.pid, + request)); + /* update request status */ ptl->super.ptl_recv_progress(&ptl->super, request, diff --git a/src/mca/ptl/portals/src/ptl_portals_send.c b/src/mca/ptl/portals/src/ptl_portals_send.c index 7efe2650a3..e6abfe7065 100644 --- a/src/mca/ptl/portals/src/ptl_portals_send.c +++ b/src/mca/ptl/portals/src/ptl_portals_send.c @@ -64,10 +64,6 @@ mca_ptl_portals_send(struct mca_ptl_base_module_t *ptl_base, mca_ptl_base_header_t* hdr; int ret; - OMPI_OUTPUT_VERBOSE((100, mca_ptl_portals_component.portals_output, - "mca_ptl_portals_send to %lu, %lu", - peer_id->nid, peer_id->pid)); - if (sendreq->req_cached && offset == 0) { sendfrag = (mca_ptl_portals_send_frag_t*)(sendreq+1); } else { @@ -148,6 +144,12 @@ mca_ptl_portals_send(struct mca_ptl_base_module_t *ptl_base, sendfrag->frag_vector[0].iov_len = sizeof(mca_ptl_base_rendezvous_header_t); } + OMPI_OUTPUT_VERBOSE((100, mca_ptl_portals_component.portals_output, + "sending first frag of size %d for msg %lld to %lu", + sendfrag->frag_send.frag_base.frag_size, + sendreq->req_send.req_base.req_sequence, + peer_id->pid)); + } else { hdr->hdr_common.hdr_type = MCA_PTL_HDR_TYPE_FRAG; sendfrag->frag_vector[0].iov_len = sizeof(mca_ptl_base_frag_header_t); @@ -156,9 +158,13 @@ mca_ptl_portals_send(struct mca_ptl_base_module_t *ptl_base, hdr->hdr_frag.hdr_frag_length = sendfrag->frag_send.frag_base.frag_size; hdr->hdr_frag.hdr_dst_ptr = sendreq->req_peer_match; - ompi_output_verbose(100, mca_ptl_portals_component.portals_output, - "sending frag for request %p", - hdr->hdr_frag.hdr_dst_ptr); + OMPI_OUTPUT_VERBOSE((100, mca_ptl_portals_component.portals_output, + "sending secnd frag of size %d for msg %lld, offset %lld to %lu, %p", + sendfrag->frag_send.frag_base.frag_size, + sendreq->req_send.req_base.req_sequence, + hdr->hdr_frag.hdr_frag_offset, + peer_id->pid, + sendreq->req_peer_match)); sendfrag->frag_send.frag_base.frag_size = size; } @@ -188,26 +194,50 @@ mca_ptl_portals_process_send_event(ptl_event_t *ev) if (ev->type == PTL_EVENT_SEND_START) { OMPI_OUTPUT_VERBOSE((100, mca_ptl_portals_component.portals_output, - "ptl event send start for msg %d, length: %d", - (int) hdr->hdr_match.hdr_msg_seq, - (int) ev->mlength)); + "ptl event send start for msg %d", + (int) hdr->hdr_match.hdr_msg_seq)); } else if (ev->type == PTL_EVENT_SEND_END) { OMPI_OUTPUT_VERBOSE((100, mca_ptl_portals_component.portals_output, "ptl event send end for msg %d", (int) hdr->hdr_match.hdr_msg_seq)); } else if (ev->type == PTL_EVENT_ACK) { - OMPI_OUTPUT_VERBOSE((100, mca_ptl_portals_component.portals_output, - "ptl event ack for msg %d", - (int) hdr->hdr_match.hdr_msg_seq)); if (frag->frag_send.frag_request == NULL) { + OMPI_OUTPUT_VERBOSE((100, mca_ptl_portals_component.portals_output, + "done sending ack for recv request %p to %lu", + hdr->hdr_ack.hdr_dst_match.pval, + ev->initiator.pid)); + assert(MCA_PTL_HDR_TYPE_ACK == hdr->hdr_common.hdr_type); + /* if request is NULL, it's an ACK - just return the frag to the pool */ OMPI_FREE_LIST_RETURN(&mca_ptl_portals_component.portals_send_frags, (ompi_list_item_t*) frag); } else { + bool frag_ack; + +#if OMPI_ENABLE_DEBUG + if (MCA_PTL_HDR_TYPE_MATCH == hdr->hdr_common.hdr_type || + MCA_PTL_HDR_TYPE_RNDV == hdr->hdr_common.hdr_type) { + OMPI_OUTPUT_VERBOSE((100, mca_ptl_portals_component.portals_output, + "done sending first frag for msg %d to %lu", + (int) hdr->hdr_match.hdr_msg_seq, + ev->initiator.pid)); + } else if (MCA_PTL_HDR_TYPE_FRAG == hdr->hdr_common.hdr_type) { + OMPI_OUTPUT_VERBOSE((100, mca_ptl_portals_component.portals_output, + "done sending secnd frag to req %p, offset %lld", + hdr->hdr_frag.hdr_dst_ptr.pval, + hdr->hdr_frag.hdr_frag_offset)); + } else { + ompi_output(mca_ptl_portals_component.portals_output, + "unexpected send event hdr type: %d. aborting", + hdr->hdr_common.hdr_type); + abort(); + } +#endif + /* it's a completion of a data fragment */ - bool frag_ack = (hdr->hdr_common.hdr_flags & MCA_PTL_FLAGS_ACK) ? + frag_ack = (hdr->hdr_common.hdr_flags & MCA_PTL_FLAGS_ACK) ? true : false; if (frag_ack == false) { @@ -222,7 +252,7 @@ mca_ptl_portals_process_send_event(ptl_event_t *ev) PtlMDUnlink(ev->md_handle); } else { ompi_output_verbose(10, mca_ptl_portals_component.portals_output, - "unknown event for msg %d: %d", + "*** Unknown event for msg %d: %d", (int) hdr->hdr_match.hdr_msg_seq, ev->type); } diff --git a/src/mca/ptl/portals/src/ptl_portals_send.h b/src/mca/ptl/portals/src/ptl_portals_send.h index 18b31be858..1f0e68d0b8 100644 --- a/src/mca/ptl/portals/src/ptl_portals_send.h +++ b/src/mca/ptl/portals/src/ptl_portals_send.h @@ -54,7 +54,7 @@ mca_ptl_portals_send_frag(struct mca_ptl_portals_module_t *ptl, md.max_size = 0; md.options = PTL_MD_IOVEC; /* BWB - can we optimize? */ md.user_ptr = sendfrag; - md.eq_handle = ptl->frag_eq_handles[MCA_PTL_PORTALS_EQ_FRAGS]; + md.eq_handle = ptl->eq_handles[MCA_PTL_PORTALS_EQ_SEND]; /* make a free-floater */ ret = PtlMDBind(ptl->ni_handle, @@ -133,7 +133,7 @@ mca_ptl_portals_send_ack(struct mca_ptl_portals_module_t *ptl, sendfrag->frag_vector[0].iov_len = sizeof(mca_ptl_base_ack_header_t); OMPI_OUTPUT_VERBOSE((100, mca_ptl_portals_component.portals_output, - "sending ack for request %p", request)); + "sending ack for recv request %p", request)); return mca_ptl_portals_send_frag(ptl, sendfrag); }