ports/openmpi
ports/openmpi
1
1
Форкнуть 0
Код Релизы Активность

* massive cleanup of debugging output to make it much easier to match

Просмотр исходного кода 
messages
* use different event queues for send / recv, part of moving towards dealing
  with dropped fragments

This commit was SVN r5719.
Этот коммит содержится в:
Brian Barrett 2005-05-15 21:05:00 +00:00
родитель ac7b97a0d9
Коммит 09057fe311
9 изменённых файлов: 157 добавлений и 79 удалений
Показать статистику Скачать Patch файл Скачать Diff файл Показать все файлы Свернуть все файлы

32
src/mca/ptl/portals/configure.stub
Просмотреть файл

@ -160,46 +160,50 @@ AC_DEFUN([MCA_CONFIGURE_STUB],[
[Portals table id to use for retransmit request queue])
MCA_PTL_PORTALS_CONFIG_VAL([debug-level],
[PTL_PORTALS_DEFAULT_DEBUG_LEVEL], [99],
[Default debugging level for portals ptl])
[PTL_PORTALS_DEFAULT_DEBUG_LEVEL], [100],
[debugging level for portals ptl])
MCA_PTL_PORTALS_CONFIG_VAL([request-cache-size],
[PTL_PORTALS_DEFAULT_REQUEST_CACHE_SIZE], [1],
[Default request cache size for portals ptl])
[request cache size for portals ptl])
MCA_PTL_PORTALS_CONFIG_VAL([first-frag-size],
[PTL_PORTALS_DEFAULT_FIRST_FRAG_SIZE], [16384],
[Default first frag size for portals ptl])
[first frag size for portals ptl])
MCA_PTL_PORTALS_CONFIG_VAL([first-frag-num-entries],
[PTL_PORTALS_DEFAULT_FIRST_FRAG_NUM_ENTRIES], [3],
[Default number of memory descriptors for first fragments])
[number of memory descriptors for first fragments])
MCA_PTL_PORTALS_CONFIG_VAL([first-frag-entry-size],
[PTL_PORTALS_DEFAULT_FIRST_FRAG_ENTRY_SIZE], [2098152],
[Default size of memory associeted with first fag md])
[size of memory associeted with first fag md])
MCA_PTL_PORTALS_CONFIG_VAL([event-queue-size],
[PTL_PORTALS_DEFAULT_EVENT_QUEUE_SIZE], [512],
[Default size of event queue for first frag mds])
MCA_PTL_PORTALS_CONFIG_VAL([recv-queue-size],
[PTL_PORTALS_DEFAULT_RECV_QUEUE_SIZE], [512],
[size of event queue for receiving frags])
MCA_PTL_PORTALS_CONFIG_VAL([send-queue-size],
[PTL_PORTALS_DEFAULT_SEND_QUEUE_SIZE], [128],
[Max number of send fragmenst pending])
MCA_PTL_PORTALS_CONFIG_VAL([rndv-frag-min-size],
[PTL_PORTALS_DEFAULT_RNDV_FRAG_MIN_SIZE], [0],
[Default minimum size of rndv fragments])
[minimum size of rndv fragments])
MCA_PTL_PORTALS_CONFIG_VAL([rndv-frag-max-size],
[PTL_PORTALS_DEFAULT_RNDV_FRAG_MAX_SIZE], [16384],
[Default maximum size of rndv fragments])
[maximum size of rndv fragments])
MCA_PTL_PORTALS_CONFIG_VAL([free-list-init-num],
[PTL_PORTALS_DEFAULT_FREE_LIST_INIT_NUM], [256],
[Default starting size of free lists])
[starting size of free lists])
MCA_PTL_PORTALS_CONFIG_VAL([free-list-max-num],
[PTL_PORTALS_DEFAULT_FREE_LIST_MAX_NUM], [-1],
[Default maximum size of free lists])
[maximum size of free lists])
MCA_PTL_PORTALS_CONFIG_VAL([free-list-inc-num],
[PTL_PORTALS_DEFAULT_FREE_LIST_inc_NUM], [256],
[Default grow size for freelists])
[grow size for freelists])
#
# Save extra compiler/linker flags so that they can be added in

24
src/mca/ptl/portals/src/ptl_portals.c
Просмотреть файл

@ -140,20 +140,22 @@ mca_ptl_portals_module_enable(struct mca_ptl_portals_module_t *ptl,
/* BWB - not really sure how - would have to track a lot more data... */
} else {
/* only do all the hard stuff if we haven't created the queue */
if (ptl->frag_eq_handles[MCA_PTL_PORTALS_EQ_FRAGS] != PTL_EQ_NONE) {
if (ptl->eq_handles[MCA_PTL_PORTALS_EQ_SIZE - 1] != PTL_EQ_NONE) {
return OMPI_SUCCESS;
}
/* create an event queue, then the match entries for the match
entries */
ret = PtlEQAlloc(ptl->ni_handle,
ptl->event_queue_size,
PTL_EQ_HANDLER_NONE,
&(ptl->frag_eq_handles[MCA_PTL_PORTALS_EQ_FRAGS]));
if (ret != PTL_OK) {
ompi_output(mca_ptl_portals_component.portals_output,
"Failed to allocate event queue: %d", ret);
return OMPI_ERROR;
for (i = 0 ; i < MCA_PTL_PORTALS_EQ_SIZE ; ++i) {
ret = PtlEQAlloc(ptl->ni_handle,
ptl->eq_sizes[i],
PTL_EQ_HANDLER_NONE,
&(ptl->eq_handles[i]));
if (ret != PTL_OK) {
ompi_output(mca_ptl_portals_component.portals_output,
"Failed to allocate event queue: %d", ret);
return OMPI_ERROR;
}
}
for (i = 0 ; i < ptl->first_frag_num_entries ; ++i) {
@ -176,9 +178,11 @@ mca_ptl_portals_finalize(struct mca_ptl_base_module_t *ptl_base)
ret = PtlNIFini(ptl->ni_handle);
if (PTL_OK != ret) {
ompi_output_verbose(20, mca_ptl_portals_component.portals_output,
"PtlNIFini returned %d\n", ret);
"PtlNIFini returned %d", ret);
return OMPI_ERROR;
}
ompi_output_verbose(20, mca_ptl_portals_component.portals_output,
"successfully finalized module");
return OMPI_SUCCESS;
}

11
src/mca/ptl/portals/src/ptl_portals.h
Просмотреть файл

@ -81,8 +81,9 @@ struct mca_ptl_portals_component_t {
typedef struct mca_ptl_portals_component_t mca_ptl_portals_component_t;
#define MCA_PTL_PORTALS_EQ_FRAGS 0
#define MCA_PTL_PORTALS_EQ_SIZE 1
#define MCA_PTL_PORTALS_EQ_RECV 0
#define MCA_PTL_PORTALS_EQ_SEND 1
#define MCA_PTL_PORTALS_EQ_SIZE 2
struct mca_ptl_portals_module_t {
/* base PTL module interface */
@ -92,11 +93,11 @@ struct mca_ptl_portals_module_t {
int first_frag_num_entries;
/* size of each md for first frags */
int first_frag_entry_size;
/* size for event queue */
int event_queue_size;
/* size for event queue */
int eq_sizes[MCA_PTL_PORTALS_EQ_SIZE];
/* frag receive event queue */
ptl_handle_eq_t frag_eq_handles[MCA_PTL_PORTALS_EQ_SIZE];
ptl_handle_eq_t eq_handles[MCA_PTL_PORTALS_EQ_SIZE];
/* our portals network interface */
ptl_handle_ni_t ni_handle;

17
src/mca/ptl/portals/src/ptl_portals_compat_utcp.c
Просмотреть файл

@ -30,6 +30,8 @@
#include "ptl_portals.h"
#include "ptl_portals_compat.h"
#include <p3api/debug.h>
/* how's this for source code diving? - find private method for
getting interface */
extern unsigned int utcp_my_nid(const char *if_str);
@ -43,9 +45,20 @@ mca_ptl_portals_init(mca_ptl_portals_component_t *comp)
{
ptl_process_id_t info;
int ret;
#if 0
FILE *output;
char *tmp;
asprintf(&tmp, "portals.%d", getpid());
output = fopen(tmp, "w");
free(tmp);
utcp_lib_out = output;
utcp_api_out = output;
#else
utcp_lib_out = stderr;
utcp_api_out = stderr;
#endif
info.nid = htonl(utcp_my_nid(mca_ptl_portals_component.portals_ifname));
info.pid = htonl((ptl_pid_t) getpid());
@ -197,6 +210,10 @@ mca_ptl_portals_add_procs_compat(struct mca_ptl_portals_module_t* ptl,
return OMPI_ERR_FATAL;
}
#if 0
PtlNIDebug(ptl->ni_handle, PTL_DBG_ALL | PTL_DBG_NI_ALL);
#endif
return OMPI_SUCCESS;
}

51
src/mca/ptl/portals/src/ptl_portals_component.c
Просмотреть файл

@ -62,17 +62,17 @@ mca_ptl_portals_component_t mca_ptl_portals_component = {
static ompi_output_stream_t portals_output_stream = {
true,
0,
0,
0,
NULL,
NULL,
false,
true,
false,
false,
NULL
true, /* is debugging */
0, /* verbose level */
0, /* want syslog */
0, /* syslog priority */
NULL, /* syslog ident */
NULL, /* prefix */
false, /* want stdout */
false, /* want stderr */
true, /* want file */
false, /* file append */
"ptl-portals" /* file suffix */
};
@ -159,9 +159,13 @@ mca_ptl_portals_component_open(void)
mca_ptl_portals_module.first_frag_entry_size =
param_register_int("first_frag_entry_size",
PTL_PORTALS_DEFAULT_FIRST_FRAG_ENTRY_SIZE);
mca_ptl_portals_module.event_queue_size =
param_register_int("event_queue_size",
PTL_PORTALS_DEFAULT_EVENT_QUEUE_SIZE);
mca_ptl_portals_module.eq_sizes[MCA_PTL_PORTALS_EQ_RECV] =
param_register_int("recv_queue_size",
PTL_PORTALS_DEFAULT_RECV_QUEUE_SIZE);
mca_ptl_portals_module.eq_sizes[MCA_PTL_PORTALS_EQ_SEND] =
(param_register_int("send_queue_size",
PTL_PORTALS_DEFAULT_SEND_QUEUE_SIZE)) * 3;
/* finish with objects */
asprintf(&(portals_output_stream.lds_prefix),
@ -172,7 +176,7 @@ mca_ptl_portals_component_open(void)
/* fill in remaining defaults for module data */
for (i = 0 ; i < MCA_PTL_PORTALS_EQ_SIZE ; ++i) {
mca_ptl_portals_module.frag_eq_handles[i] = PTL_EQ_NONE;
mca_ptl_portals_module.eq_handles[i] = PTL_EQ_NONE;
}
mca_ptl_portals_module.ni_handle = PTL_INVALID_HANDLE;
@ -299,6 +303,7 @@ mca_ptl_portals_component_progress(mca_ptl_tstamp_t tstamp)
{
int num_progressed = 0;
size_t i;
tstamp = 10;
for (i = 0 ; i < mca_ptl_portals_component.portals_num_modules ; ++i) {
struct mca_ptl_portals_module_t *module =
@ -308,7 +313,7 @@ mca_ptl_portals_component_progress(mca_ptl_tstamp_t tstamp)
int which;
int ret;
if (module->frag_eq_handles[MCA_PTL_PORTALS_EQ_SIZE - 1] ==
if (module->eq_handles[MCA_PTL_PORTALS_EQ_SIZE - 1] ==
PTL_EQ_NONE) continue; /* they are all initialized at once */
#if OMPI_ENABLE_DEBUG
@ -324,7 +329,7 @@ mca_ptl_portals_component_progress(mca_ptl_tstamp_t tstamp)
}
#endif
ret = PtlEQPoll(module->frag_eq_handles,
ret = PtlEQPoll(module->eq_handles,
MCA_PTL_PORTALS_EQ_SIZE, /* number of eq handles */
(int) tstamp,
&ev,
@ -339,25 +344,23 @@ mca_ptl_portals_component_progress(mca_ptl_tstamp_t tstamp)
continue;
} else if (PTL_EQ_DROPPED == ret) {
ompi_output_verbose(10, mca_ptl_portals_component.portals_output,
"event queue entries were dropped");
"*** Event queue entries were dropped");
}
/* only one place we can have an event */
assert(which == MCA_PTL_PORTALS_EQ_FRAGS);
#if PTL_PORTALS_HAVE_EVENT_UNLINK
/* not everyone has UNLINK. Use it only to print the event,
so we can make sure we properly re-initialize the ones that
need to be re-initialized */
if (PTL_EVENT_UNLINK == ev.type) {
ompi_output_verbose(100, mca_ptl_portals_component.portals_output,
"unlink event occurred");
OMPI_OUTPUT_VERBOSE((100, mca_ptl_portals_component.portals_output,
"unlink event occurred"));
continue;
}
#endif
if (ev.md.user_ptr == NULL) {
/* no fragment associated with it - it's a receive */
assert(which == MCA_PTL_PORTALS_EQ_RECV);
mca_ptl_portals_process_recv_event(module, &ev);
} else {
/* there's a fragment associated with it - choose based on
@ -365,8 +368,10 @@ mca_ptl_portals_component_progress(mca_ptl_tstamp_t tstamp)
mca_ptl_base_frag_t *frag =
(mca_ptl_base_frag_t*) ev.md.user_ptr;
if (frag->frag_type == MCA_PTL_FRAGMENT_SEND) {
assert(which == MCA_PTL_PORTALS_EQ_SEND);
mca_ptl_portals_process_send_event(&ev);
} else {
assert(which == MCA_PTL_PORTALS_EQ_RECV);
mca_ptl_portals_process_recv_event(module, &ev);
}
}

23
src/mca/ptl/portals/src/ptl_portals_recv.c
Просмотреть файл

@ -69,7 +69,7 @@ ptl_portals_post_recv_md(struct mca_ptl_portals_module_t *ptl, void *data_ptr)
md.max_size = ptl->super.ptl_first_frag_size;
md.options = PTL_MD_OP_PUT | PTL_MD_MAX_SIZE;
md.user_ptr = NULL;
md.eq_handle = ptl->frag_eq_handles[MCA_PTL_PORTALS_EQ_FRAGS];
md.eq_handle = ptl->eq_handles[MCA_PTL_PORTALS_EQ_RECV];
ret = PtlMDAttach(me_handle,
md,
@ -94,12 +94,12 @@ mca_ptl_portals_process_recv_event(struct mca_ptl_portals_module_t *ptl,
int ret;
if (ev->type == PTL_EVENT_PUT_START) {
OMPI_OUTPUT_VERBOSE((100, mca_ptl_portals_component.portals_output,
"PUT_START event received (%ld)", ev->link));
OMPI_OUTPUT_VERBOSE((101, mca_ptl_portals_component.portals_output,
"starting to receive message", ev->link));
} else if (ev->type == PTL_EVENT_PUT_END) {
mca_ptl_base_header_t *hdr;
OMPI_OUTPUT_VERBOSE((100, mca_ptl_portals_component.portals_output,
OMPI_OUTPUT_VERBOSE((101, mca_ptl_portals_component.portals_output,
"message %ld received, start: %p, mlength: %lld,"
" offset: %lld",
ev->link, ev->md.start, ev->mlength, ev->offset));
@ -129,21 +129,24 @@ mca_ptl_portals_process_recv_event(struct mca_ptl_portals_module_t *ptl,
{
mca_ptl_portals_send_frag_t *sendfrag;
mca_ptl_base_send_request_t *sendreq;
OMPI_OUTPUT_VERBOSE((100,
mca_ptl_portals_component.portals_output,
"received ack for request %p",
hdr->hdr_ack.hdr_dst_match));
sendfrag = hdr->hdr_ack.hdr_src_ptr.pval;
sendreq = sendfrag->frag_send.frag_request;
sendreq->req_peer_match = hdr->hdr_ack.hdr_dst_match;
OMPI_OUTPUT_VERBOSE((100,
mca_ptl_portals_component.portals_output,
"received ack for recv request %p (msg %d)",
hdr->hdr_ack.hdr_dst_match,
sendreq->req_send.req_base.req_sequence));
mca_ptl_portals_complete_send_event(sendfrag);
}
break;
default:
ompi_output(mca_ptl_portals_component.portals_output,
"unable to deal with header of type %d",
"*** unable to deal with header of type %d",
hdr->hdr_common.hdr_type);
break;
}
@ -160,7 +163,7 @@ mca_ptl_portals_process_recv_event(struct mca_ptl_portals_module_t *ptl,
} else {
ompi_output_verbose(10, mca_ptl_portals_component.portals_output,
"unknown event: %d (%ld)",
"*** unknown event: %d (%ld)",
ev->type, ev->link);
}

14
src/mca/ptl/portals/src/ptl_portals_recv.h
Просмотреть файл

@ -89,6 +89,12 @@ mca_ptl_portals_process_first_frag(struct mca_ptl_portals_module_t *ptl,
recvfrag = mca_ptl_portals_recv_get_frag(ptl, hdr, ev, header_size);
if (NULL == recvfrag) return OMPI_ERR_TEMP_OUT_OF_RESOURCE;
OMPI_OUTPUT_VERBOSE((100, mca_ptl_portals_component.portals_output,
"recving first frag of size %d for msg %d from %lu",
recvfrag->frag_size,
(int) hdr->hdr_match.hdr_msg_seq,
ev->initiator.pid));
recvfrag->frag_recv.frag_request = NULL;
ptl->super.ptl_match(&ptl->super, &recvfrag->frag_recv,
&hdr->hdr_match);
@ -141,6 +147,14 @@ mca_ptl_portals_process_frag_frag(struct mca_ptl_portals_module_t *ptl,
&bytes_delivered, &free_after );
}
OMPI_OUTPUT_VERBOSE((100, mca_ptl_portals_component.portals_output,
"recving secnd frag of size %d for msg %d, offset %lld from %lu, %p",
recvfrag->frag_size,
(int) hdr->hdr_match.hdr_msg_seq,
hdr->hdr_frag.hdr_frag_offset,
ev->initiator.pid,
request));
/* update request status */
ptl->super.ptl_recv_progress(&ptl->super,
request,

60
src/mca/ptl/portals/src/ptl_portals_send.c
Просмотреть файл

@ -64,10 +64,6 @@ mca_ptl_portals_send(struct mca_ptl_base_module_t *ptl_base,
mca_ptl_base_header_t* hdr;
int ret;
OMPI_OUTPUT_VERBOSE((100, mca_ptl_portals_component.portals_output,
"mca_ptl_portals_send to %lu, %lu",
peer_id->nid, peer_id->pid));
if (sendreq->req_cached && offset == 0) {
sendfrag = (mca_ptl_portals_send_frag_t*)(sendreq+1);
} else {
@ -148,6 +144,12 @@ mca_ptl_portals_send(struct mca_ptl_base_module_t *ptl_base,
sendfrag->frag_vector[0].iov_len = sizeof(mca_ptl_base_rendezvous_header_t);
}
OMPI_OUTPUT_VERBOSE((100, mca_ptl_portals_component.portals_output,
"sending first frag of size %d for msg %lld to %lu",
sendfrag->frag_send.frag_base.frag_size,
sendreq->req_send.req_base.req_sequence,
peer_id->pid));
} else {
hdr->hdr_common.hdr_type = MCA_PTL_HDR_TYPE_FRAG;
sendfrag->frag_vector[0].iov_len = sizeof(mca_ptl_base_frag_header_t);
@ -156,9 +158,13 @@ mca_ptl_portals_send(struct mca_ptl_base_module_t *ptl_base,
hdr->hdr_frag.hdr_frag_length = sendfrag->frag_send.frag_base.frag_size;
hdr->hdr_frag.hdr_dst_ptr = sendreq->req_peer_match;
ompi_output_verbose(100, mca_ptl_portals_component.portals_output,
"sending frag for request %p",
hdr->hdr_frag.hdr_dst_ptr);
OMPI_OUTPUT_VERBOSE((100, mca_ptl_portals_component.portals_output,
"sending secnd frag of size %d for msg %lld, offset %lld to %lu, %p",
sendfrag->frag_send.frag_base.frag_size,
sendreq->req_send.req_base.req_sequence,
hdr->hdr_frag.hdr_frag_offset,
peer_id->pid,
sendreq->req_peer_match));
sendfrag->frag_send.frag_base.frag_size = size;
}
@ -188,26 +194,50 @@ mca_ptl_portals_process_send_event(ptl_event_t *ev)
if (ev->type == PTL_EVENT_SEND_START) {
OMPI_OUTPUT_VERBOSE((100, mca_ptl_portals_component.portals_output,
"ptl event send start for msg %d, length: %d",
(int) hdr->hdr_match.hdr_msg_seq,
(int) ev->mlength));
"ptl event send start for msg %d",
(int) hdr->hdr_match.hdr_msg_seq));
} else if (ev->type == PTL_EVENT_SEND_END) {
OMPI_OUTPUT_VERBOSE((100, mca_ptl_portals_component.portals_output,
"ptl event send end for msg %d",
(int) hdr->hdr_match.hdr_msg_seq));
} else if (ev->type == PTL_EVENT_ACK) {
OMPI_OUTPUT_VERBOSE((100, mca_ptl_portals_component.portals_output,
"ptl event ack for msg %d",
(int) hdr->hdr_match.hdr_msg_seq));
if (frag->frag_send.frag_request == NULL) {
OMPI_OUTPUT_VERBOSE((100, mca_ptl_portals_component.portals_output,
"done sending ack for recv request %p to %lu",
hdr->hdr_ack.hdr_dst_match.pval,
ev->initiator.pid));
assert(MCA_PTL_HDR_TYPE_ACK == hdr->hdr_common.hdr_type);
/* if request is NULL, it's an ACK - just return the frag
to the pool */
OMPI_FREE_LIST_RETURN(&mca_ptl_portals_component.portals_send_frags,
(ompi_list_item_t*) frag);
} else {
bool frag_ack;
#if OMPI_ENABLE_DEBUG
if (MCA_PTL_HDR_TYPE_MATCH == hdr->hdr_common.hdr_type ||
MCA_PTL_HDR_TYPE_RNDV == hdr->hdr_common.hdr_type) {
OMPI_OUTPUT_VERBOSE((100, mca_ptl_portals_component.portals_output,
"done sending first frag for msg %d to %lu",
(int) hdr->hdr_match.hdr_msg_seq,
ev->initiator.pid));
} else if (MCA_PTL_HDR_TYPE_FRAG == hdr->hdr_common.hdr_type) {
OMPI_OUTPUT_VERBOSE((100, mca_ptl_portals_component.portals_output,
"done sending secnd frag to req %p, offset %lld",
hdr->hdr_frag.hdr_dst_ptr.pval,
hdr->hdr_frag.hdr_frag_offset));
} else {
ompi_output(mca_ptl_portals_component.portals_output,
"unexpected send event hdr type: %d. aborting",
hdr->hdr_common.hdr_type);
abort();
}
#endif
/* it's a completion of a data fragment */
bool frag_ack = (hdr->hdr_common.hdr_flags & MCA_PTL_FLAGS_ACK) ?
frag_ack = (hdr->hdr_common.hdr_flags & MCA_PTL_FLAGS_ACK) ?
true : false;
if (frag_ack == false) {
@ -222,7 +252,7 @@ mca_ptl_portals_process_send_event(ptl_event_t *ev)
PtlMDUnlink(ev->md_handle);
} else {
ompi_output_verbose(10, mca_ptl_portals_component.portals_output,
"unknown event for msg %d: %d",
"*** Unknown event for msg %d: %d",
(int) hdr->hdr_match.hdr_msg_seq, ev->type);
}

4
src/mca/ptl/portals/src/ptl_portals_send.h
Просмотреть файл

@ -54,7 +54,7 @@ mca_ptl_portals_send_frag(struct mca_ptl_portals_module_t *ptl,
md.max_size = 0;
md.options = PTL_MD_IOVEC; /* BWB - can we optimize? */
md.user_ptr = sendfrag;
md.eq_handle = ptl->frag_eq_handles[MCA_PTL_PORTALS_EQ_FRAGS];
md.eq_handle = ptl->eq_handles[MCA_PTL_PORTALS_EQ_SEND];
/* make a free-floater */
ret = PtlMDBind(ptl->ni_handle,
@ -133,7 +133,7 @@ mca_ptl_portals_send_ack(struct mca_ptl_portals_module_t *ptl,
sendfrag->frag_vector[0].iov_len = sizeof(mca_ptl_base_ack_header_t);
OMPI_OUTPUT_VERBOSE((100, mca_ptl_portals_component.portals_output,
"sending ack for request %p", request));
"sending ack for recv request %p", request));
return mca_ptl_portals_send_frag(ptl, sendfrag);
}