From faf3d438bbb4f2042ea515fb2f12d4695441b532 Mon Sep 17 00:00:00 2001 From: Brian Barrett Date: Thu, 5 May 2005 19:53:50 +0000 Subject: [PATCH] * Put all the source files in src/, as it was getting really annoying having everything in one directory. Still have only one Makefile, so it shouldn't change build time at all * Now that I finally understand the header system for data, refactor a little bit of the code to match what really should be happening * start of a hacked up send() - puts the data for a 0 byte message on the other side, and all the pointers are where i think they should be. So my plan of attack will work. But I think I'm going to have to use iovecs instead of memcpy() real soon now. This commit was SVN r5610. --- src/mca/ptl/portals/Makefile.am | 22 +-- src/mca/ptl/portals/configure.params | 2 +- src/mca/ptl/portals/ptl_portals_sendfrag.c | 26 --- src/mca/ptl/portals/{ => src}/ptl_portals.c | 160 +++--------------- src/mca/ptl/portals/{ => src}/ptl_portals.h | 12 +- .../portals/{ => src}/ptl_portals_compat.h | 0 .../{ => src}/ptl_portals_compat_redstorm.c | 0 .../{ => src}/ptl_portals_compat_utcp.c | 0 .../portals/{ => src}/ptl_portals_component.c | 13 +- src/mca/ptl/portals/src/ptl_portals_send.c | 131 ++++++++++++++ .../ptl_portals_send.h} | 0 .../ptl/portals/{ => src}/ptl_portals_stubs.c | 19 --- 12 files changed, 184 insertions(+), 201 deletions(-) delete mode 100644 src/mca/ptl/portals/ptl_portals_sendfrag.c rename src/mca/ptl/portals/{ => src}/ptl_portals.c (55%) rename src/mca/ptl/portals/{ => src}/ptl_portals.h (97%) rename src/mca/ptl/portals/{ => src}/ptl_portals_compat.h (100%) rename src/mca/ptl/portals/{ => src}/ptl_portals_compat_redstorm.c (100%) rename src/mca/ptl/portals/{ => src}/ptl_portals_compat_utcp.c (100%) rename src/mca/ptl/portals/{ => src}/ptl_portals_component.c (94%) create mode 100644 src/mca/ptl/portals/src/ptl_portals_send.c rename src/mca/ptl/portals/{ptl_portals_sendfrag.h => src/ptl_portals_send.h} (100%) rename src/mca/ptl/portals/{ => src}/ptl_portals_stubs.c (71%) diff --git a/src/mca/ptl/portals/Makefile.am b/src/mca/ptl/portals/Makefile.am index b605178412..34fd4a47ba 100644 --- a/src/mca/ptl/portals/Makefile.am +++ b/src/mca/ptl/portals/Makefile.am @@ -31,18 +31,20 @@ component_install = endif EXTRA_DIST = \ - ptl_portals_compat_utcp.c \ - ptl_portals_compat_redstorm.c + src/ptl_portals_compat_utcp.c \ + src/ptl_portals_compat_redstorm.c portals_SOURCES = \ - ptl_portals.h \ - ptl_portals_compat.h \ - ptl_portals_sendfrag.h \ - ptl_portals.c \ - ptl_portals_component.c \ - ptl_portals_stubs.c \ - ptl_portals_compat_utcp.c \ - ptl_portals_sendfrag.c + src/ptl_portals.h \ + src/ptl_portals_compat.h \ + src/ptl_portals_send.h \ + src/ptl_portals_recv.h \ + src/ptl_portals.c \ + src/ptl_portals_component.c \ + src/ptl_portals_stubs.c \ + src/ptl_portals_compat_utcp.c \ + src/ptl_portals_send.c \ + src/ptl_portals_recv.c mcacomponentdir = $(libdir)/openmpi diff --git a/src/mca/ptl/portals/configure.params b/src/mca/ptl/portals/configure.params index 4b5bdcd1ec..8f416a26e3 100644 --- a/src/mca/ptl/portals/configure.params +++ b/src/mca/ptl/portals/configure.params @@ -17,6 +17,6 @@ # Specific to this module -PARAM_INIT_FILE=ptl_portals.h +PARAM_INIT_FILE=src/ptl_portals.h PARAM_CONFIG_HEADER_FILE="portals_config.h" PARAM_CONFIG_FILES="Makefile" diff --git a/src/mca/ptl/portals/ptl_portals_sendfrag.c b/src/mca/ptl/portals/ptl_portals_sendfrag.c deleted file mode 100644 index 28125dd609..0000000000 --- a/src/mca/ptl/portals/ptl_portals_sendfrag.c +++ /dev/null @@ -1,26 +0,0 @@ -/* - * Copyright (c) 2004-2005 The Trustees of Indiana University. - * All rights reserved. - * Copyright (c) 2004-2005 The Trustees of the University of Tennessee. - * All rights reserved. - * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, - * University of Stuttgart. All rights reserved. - * Copyright (c) 2004-2005 The Regents of the University of California. - * All rights reserved. - * $COPYRIGHT$ - * - * Additional copyrights may follow - * - * $HEADER$ - */ - -#include "ompi_config.h" -#include "portals_config.h" - -#include "ptl_portals.h" -#include "ptl_portals_compat.h" -#include "ptl_portals_sendfrag.h" - -OBJ_CLASS_INSTANCE(mca_ptl_portals_send_frag_t, - mca_ptl_base_send_frag_t, - NULL, NULL); diff --git a/src/mca/ptl/portals/ptl_portals.c b/src/mca/ptl/portals/src/ptl_portals.c similarity index 55% rename from src/mca/ptl/portals/ptl_portals.c rename to src/mca/ptl/portals/src/ptl_portals.c index bb17b0427f..76123d1729 100644 --- a/src/mca/ptl/portals/ptl_portals.c +++ b/src/mca/ptl/portals/src/ptl_portals.c @@ -29,7 +29,8 @@ #include "ptl_portals.h" #include "ptl_portals_compat.h" -#include "ptl_portals_sendfrag.h" +#include "ptl_portals_send.h" +#include "ptl_portals_recv.h" mca_ptl_portals_module_t mca_ptl_portals_module = { { @@ -145,7 +146,7 @@ mca_ptl_portals_module_enable(struct mca_ptl_portals_module_t *ptl, ret = PtlEQAlloc(ptl->ni_handle, ptl->first_frag_queue_size, PTL_EQ_HANDLER_NONE, - &(ptl->frag_receive_eq_handle)); + &(ptl->frag_eq_handle)); if (ret != PTL_OK) { ompi_output(mca_ptl_portals_component.portals_output, "Failed to allocate event queue: %d", ret); @@ -153,10 +154,10 @@ mca_ptl_portals_module_enable(struct mca_ptl_portals_module_t *ptl, } ompi_output_verbose(100, mca_ptl_portals_component.portals_output, "allocated event queue: %d", - ptl->frag_receive_eq_handle); + ptl->frag_eq_handle); for (i = 0 ; i < ptl->first_frag_num_entries ; ++i) { - ret = ptl_portals_new_frag_entry(ptl); + ret = ptl_portals_post_recv_md(ptl); if (OMPI_SUCCESS != ret) return ret; ptl->frag_queues_created = true; } @@ -166,139 +167,6 @@ mca_ptl_portals_module_enable(struct mca_ptl_portals_module_t *ptl, } -int -ptl_portals_new_frag_entry(struct mca_ptl_portals_module_t *ptl) -{ - ptl_handle_me_t me_handle; - ptl_handle_md_t md_handle; - ptl_md_t md; - void *mem; - int ret; - ptl_process_id_t proc = { PTL_NID_ANY, PTL_PID_ANY }; - - /* create match entry */ - ret = PtlMEAttach(ptl->ni_handle, - PTL_PORTALS_FRAG_TABLE_ID, - proc, - 0, /* match bits */ - 0, /* ignore bits */ - PTL_UNLINK, - PTL_INS_AFTER, - &me_handle); - if (PTL_OK != ret) return OMPI_ERROR; - - /* and some memory */ - mem = malloc(ptl->first_frag_entry_size); - if (NULL == mem) { - PtlMEUnlink(me_handle); - return OMPI_ERR_TEMP_OUT_OF_RESOURCE; - } - - /* and the memory descriptor */ - md.start = mem; - md.length = ptl->first_frag_entry_size; - md.threshold = PTL_MD_THRESH_INF; - md.max_size = md.length - ptl->super.ptl_first_frag_size; - md.options = PTL_MD_OP_PUT | PTL_MD_MAX_SIZE; - md.user_ptr = NULL; - md.eq_handle = ptl->frag_receive_eq_handle; - - ret = PtlMDAttach(me_handle, - md, - PTL_UNLINK, - &md_handle); - if (PTL_OK != ret) { - PtlMEUnlink(me_handle); - return OMPI_ERROR; - } - - ompi_output_verbose(50, mca_ptl_portals_component.portals_output, - "new receive buffer posted"); - - return OMPI_SUCCESS; -} - - -int -mca_ptl_portals_send(struct mca_ptl_base_module_t *ptl_base, - struct mca_ptl_base_peer_t *ptl_peer, - struct mca_pml_base_send_request_t *sendreq, - size_t offset, size_t size, int flags) -{ - mca_ptl_portals_module_t* ptl = (mca_ptl_portals_module_t*) ptl_base; - ptl_process_id_t *peer_id = (ptl_process_id_t*) ptl_peer; - mca_ptl_portals_send_frag_t* sendfrag; - mca_ptl_base_header_t* hdr; - int ret; - ptl_md_t md; - - ompi_output_verbose(100, mca_ptl_portals_component.portals_output, - "mca_ptl_portals_send to %lu, %lu", - peer_id->nid, peer_id->pid); - - if (sendreq->req_cached) { - sendfrag = (mca_ptl_portals_send_frag_t*)(sendreq+1); - } else { - ompi_output(mca_ptl_portals_component.portals_output, - "request not cached - not implemented."); - return OMPI_ERROR; - } - - /* initialize convertor */ - if (size > 0) { - ompi_output(mca_ptl_portals_component.portals_output, - "request size > 0, not implemented"); - return OMPI_ERROR; - } else { - sendfrag->frag_send.frag_base.frag_addr = NULL; - sendfrag->frag_send.frag_base.frag_size = 0; - } - - /* setup message header */ - hdr = &sendfrag->frag_send.frag_base.frag_header; - if(offset == 0) { - hdr->hdr_common.hdr_flags = flags; - hdr->hdr_match.hdr_contextid = sendreq->req_base.req_comm->c_contextid; - hdr->hdr_match.hdr_src = sendreq->req_base.req_comm->c_my_rank; - hdr->hdr_match.hdr_dst = sendreq->req_base.req_peer; - hdr->hdr_match.hdr_tag = sendreq->req_base.req_tag; - hdr->hdr_match.hdr_msg_length = sendreq->req_bytes_packed; - hdr->hdr_match.hdr_msg_seq = sendreq->req_base.req_sequence; - } else { - ompi_output(mca_ptl_portals_component.portals_output, - "offset > 0, not implemented"); - return OMPI_ERROR; - } - - /* fragment state */ -#if 0 - sendfrag->frag_send.frag_base.frag_owner = &ptl_peer->peer_ptl->super; -#endif - sendfrag->frag_send.frag_request = sendreq; -#if 0 - sendfrag->frag_send.frag_base.frag_peer = ptl_peer; -#endif - - - /* must update the offset after actual fragment size is determined - * before attempting to send the fragment - */ - mca_pml_base_send_request_offset(sendreq, - sendfrag->frag_send.frag_base.frag_size); -#if 0 - md.start = mem; - md.length = ptl->first_frag_entry_size; - md.threshold = PTL_MD_THRESH_INF; - md.max_size = md.length - ptl->super.ptl_first_frag_size; - md.options = PTL_MD_OP_PUT | PTL_MD_MAX_SIZE; - md.user_ptr = NULL; - md.eq_handle = ptl->frag_receive_eq_handle; -#endif - - return OMPI_ERROR; -} - - int mca_ptl_portals_finalize(struct mca_ptl_base_module_t *ptl_base) { @@ -315,3 +183,21 @@ mca_ptl_portals_finalize(struct mca_ptl_base_module_t *ptl_base) return OMPI_SUCCESS; } + + +int +mca_ptl_portals_request_init(struct mca_ptl_base_module_t *ptl, + struct mca_pml_base_send_request_t *req) +{ + OBJ_CONSTRUCT(req + 1, mca_ptl_portals_send_frag_t); + return OMPI_SUCCESS; +} + + +void +mca_ptl_portals_request_fini(struct mca_ptl_base_module_t *ptl, + struct mca_pml_base_send_request_t *req) +{ + OBJ_DESTRUCT(req + 1); +} + diff --git a/src/mca/ptl/portals/ptl_portals.h b/src/mca/ptl/portals/src/ptl_portals.h similarity index 97% rename from src/mca/ptl/portals/ptl_portals.h rename to src/mca/ptl/portals/src/ptl_portals.h index 6d579db284..ddb56b46d7 100644 --- a/src/mca/ptl/portals/ptl_portals.h +++ b/src/mca/ptl/portals/src/ptl_portals.h @@ -60,6 +60,15 @@ struct mca_ptl_portals_component_t { /** List of currently available modules */ struct mca_ptl_portals_module_t **portals_modules; + /** free list of portals send fragments */ + ompi_free_list_t portals_send_frags; + + /** free list of portals recv fragments */ + ompi_free_list_t portals_recv_frags; + + /** queue of pending sends */ + ompi_list_t portals_pending_acks; + /** lock for accessing component */ ompi_mutex_t portals_lock; }; @@ -157,7 +166,7 @@ struct mca_ptl_portals_module_t { /* frag receive data */ bool frag_queues_created; /* frag receive event queue */ - ptl_handle_eq_t frag_receive_eq_handle; + ptl_handle_eq_t frag_eq_handle; /** our portals network interface */ ptl_handle_ni_t ni_handle; @@ -353,6 +362,5 @@ extern int mca_ptl_portals_send( extern int mca_ptl_portals_module_enable(struct mca_ptl_portals_module_t *ptl, int value); -extern int ptl_portals_new_frag_entry(struct mca_ptl_portals_module_t *ptl); #endif diff --git a/src/mca/ptl/portals/ptl_portals_compat.h b/src/mca/ptl/portals/src/ptl_portals_compat.h similarity index 100% rename from src/mca/ptl/portals/ptl_portals_compat.h rename to src/mca/ptl/portals/src/ptl_portals_compat.h diff --git a/src/mca/ptl/portals/ptl_portals_compat_redstorm.c b/src/mca/ptl/portals/src/ptl_portals_compat_redstorm.c similarity index 100% rename from src/mca/ptl/portals/ptl_portals_compat_redstorm.c rename to src/mca/ptl/portals/src/ptl_portals_compat_redstorm.c diff --git a/src/mca/ptl/portals/ptl_portals_compat_utcp.c b/src/mca/ptl/portals/src/ptl_portals_compat_utcp.c similarity index 100% rename from src/mca/ptl/portals/ptl_portals_compat_utcp.c rename to src/mca/ptl/portals/src/ptl_portals_compat_utcp.c diff --git a/src/mca/ptl/portals/ptl_portals_component.c b/src/mca/ptl/portals/src/ptl_portals_component.c similarity index 94% rename from src/mca/ptl/portals/ptl_portals_component.c rename to src/mca/ptl/portals/src/ptl_portals_component.c index 084cc3a8c0..f4401f64c7 100644 --- a/src/mca/ptl/portals/ptl_portals_component.c +++ b/src/mca/ptl/portals/src/ptl_portals_component.c @@ -115,6 +115,9 @@ mca_ptl_portals_component_open(void) mca_ptl_portals_component.portals_modules = NULL; /* initialize objects */ + OBJ_CONSTRUCT(&mca_ptl_portals_component.portals_send_frags, ompi_free_list_t); + OBJ_CONSTRUCT(&mca_ptl_portals_component.portals_recv_frags, ompi_free_list_t); + OBJ_CONSTRUCT(&mca_ptl_portals_component.portals_pending_acks, ompi_list_t); OBJ_CONSTRUCT(&mca_ptl_portals_component.portals_lock, ompi_mutex_t); /* register portals module parameters */ @@ -176,6 +179,9 @@ mca_ptl_portals_component_close(void) /* release resources */ OBJ_DESTRUCT(&mca_ptl_portals_component.portals_lock); + OBJ_DESTRUCT(&mca_ptl_portals_component.portals_recv_frags); + OBJ_DESTRUCT(&mca_ptl_portals_component.portals_pending_acks); + OBJ_DESTRUCT(&mca_ptl_portals_component.portals_lock); if (NULL != mca_ptl_portals_component.portals_ifname) { free(mca_ptl_portals_component.portals_ifname); @@ -270,7 +276,7 @@ mca_ptl_portals_component_progress(mca_ptl_tstamp_t tstamp) if (! module->frag_queues_created) continue; - ret = PtlEQGet(module->frag_receive_eq_handle, &my_event); + ret = PtlEQGet(module->frag_eq_handle, &my_event); if (PTL_EQ_EMPTY == ret) { continue; } else if (!(PTL_OK == ret || PTL_EQ_DROPPED == ret)) { @@ -281,11 +287,6 @@ mca_ptl_portals_component_progress(mca_ptl_tstamp_t tstamp) ompi_output_verbose(20, mca_ptl_portals_component.portals_output, "Progress found dropped packets"); } - - ompi_output_verbose(100, mca_ptl_portals_component.portals_output, - "my_event: %d, %d, %d, %d %d %d", - my_event.type, my_event.rlength, my_event.offset, - my_event.link, my_event.ni_fail_type, my_event.sequence); num_progressed++; } diff --git a/src/mca/ptl/portals/src/ptl_portals_send.c b/src/mca/ptl/portals/src/ptl_portals_send.c new file mode 100644 index 0000000000..1333410055 --- /dev/null +++ b/src/mca/ptl/portals/src/ptl_portals_send.c @@ -0,0 +1,131 @@ +/* + * Copyright (c) 2004-2005 The Trustees of Indiana University. + * All rights reserved. + * Copyright (c) 2004-2005 The Trustees of the University of Tennessee. + * All rights reserved. + * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, + * University of Stuttgart. All rights reserved. + * Copyright (c) 2004-2005 The Regents of the University of California. + * All rights reserved. + * $COPYRIGHT$ + * + * Additional copyrights may follow + * + * $HEADER$ + */ + +#include "ompi_config.h" +#include "portals_config.h" + +#include "ptl_portals.h" +#include "ptl_portals_compat.h" +#include "ptl_portals_send.h" +#include "mca/pml/pml.h" +#include "mca/pml/base/pml_base_sendreq.h" + +OBJ_CLASS_INSTANCE(mca_ptl_portals_send_frag_t, + mca_ptl_base_send_frag_t, + NULL, NULL); + + +int +mca_ptl_portals_send(struct mca_ptl_base_module_t *ptl_base, + struct mca_ptl_base_peer_t *ptl_peer, + struct mca_pml_base_send_request_t *sendreq, + size_t offset, size_t size, int flags) +{ + mca_ptl_portals_module_t* ptl = (mca_ptl_portals_module_t*) ptl_base; + ptl_process_id_t *peer_id = (ptl_process_id_t*) ptl_peer; + mca_ptl_portals_send_frag_t* sendfrag; + mca_ptl_base_header_t* hdr; + int ret; + ptl_md_t md; + ptl_handle_md_t md_handle; + + ompi_output_verbose(100, mca_ptl_portals_component.portals_output, + "mca_ptl_portals_send to %lu, %lu", + peer_id->nid, peer_id->pid); + + if (sendreq->req_cached) { + sendfrag = (mca_ptl_portals_send_frag_t*)(sendreq+1); + } else { + ompi_list_item_t *item; + OMPI_FREE_LIST_GET(&mca_ptl_portals_component.portals_send_frags, + item, ret); + if (NULL == item) return ret; + sendfrag = (mca_ptl_portals_send_frag_t *) item; + } + + /* initialize convertor */ + if (size > 0) { + ompi_output(mca_ptl_portals_component.portals_output, + "request size > 0, not implemented"); + return OMPI_ERROR; + } else { + sendfrag->frag_send.frag_base.frag_addr = NULL; + sendfrag->frag_send.frag_base.frag_size = 0; + } + + /* setup message header */ + hdr = &sendfrag->frag_send.frag_base.frag_header; + if(offset == 0) { + hdr->hdr_common.hdr_flags = flags; + hdr->hdr_match.hdr_contextid = sendreq->req_base.req_comm->c_contextid; + hdr->hdr_match.hdr_src = sendreq->req_base.req_comm->c_my_rank; + hdr->hdr_match.hdr_dst = sendreq->req_base.req_peer; + hdr->hdr_match.hdr_tag = sendreq->req_base.req_tag; + hdr->hdr_match.hdr_msg_length = sendreq->req_bytes_packed; + hdr->hdr_match.hdr_msg_seq = sendreq->req_base.req_sequence; + } else { + ompi_output(mca_ptl_portals_component.portals_output, + "offset > 0, not implemented"); + return OMPI_ERROR; + } + + /* fragment state */ + sendfrag->frag_send.frag_base.frag_owner = ptl_base; + sendfrag->frag_send.frag_request = sendreq; + sendfrag->frag_send.frag_base.frag_peer = ptl_peer; + + + /* must update the offset after actual fragment size is determined + * before attempting to send the fragment + */ + mca_pml_base_send_request_offset(sendreq, + sendfrag->frag_send.frag_base.frag_size); + + md.start = hdr; + md.length = sizeof(mca_ptl_base_header_t); + md.threshold = 2; /* we do a put, we get out of here */ + md.max_size = 0; + md.options = 0; + md.user_ptr = 123; + md.eq_handle = ptl->frag_eq_handle; + + /* make a free-floater */ + ret = PtlMDBind(ptl->ni_handle, + md, + PTL_UNLINK, + &md_handle); + if (ret != PTL_OK) { + ompi_output(mca_ptl_portals_component.portals_output, + "PtlMDBind failed with error %d", ret); + return OMPI_ERROR; + } + + ret = PtlPut(md_handle, + PTL_NO_ACK_REQ, + *((ptl_process_id_t*) ptl_peer), + PTL_PORTALS_FRAG_TABLE_ID, + 0, /* ac_index */ + 0, /* match bits */ + 0, /* remote offset - not used */ + 321); /* hdr_data - not used */ + if (ret != PTL_OK) { + ompi_output(mca_ptl_portals_component.portals_output, + "PtlPut failed with error %d", ret); + return OMPI_ERROR; + } + + return OMPI_SUCCESS; +} diff --git a/src/mca/ptl/portals/ptl_portals_sendfrag.h b/src/mca/ptl/portals/src/ptl_portals_send.h similarity index 100% rename from src/mca/ptl/portals/ptl_portals_sendfrag.h rename to src/mca/ptl/portals/src/ptl_portals_send.h diff --git a/src/mca/ptl/portals/ptl_portals_stubs.c b/src/mca/ptl/portals/src/ptl_portals_stubs.c similarity index 71% rename from src/mca/ptl/portals/ptl_portals_stubs.c rename to src/mca/ptl/portals/src/ptl_portals_stubs.c index 0af8bb2789..2e30ac607f 100644 --- a/src/mca/ptl/portals/ptl_portals_stubs.c +++ b/src/mca/ptl/portals/src/ptl_portals_stubs.c @@ -31,25 +31,6 @@ -int -mca_ptl_portals_request_init(struct mca_ptl_base_module_t *ptl, - struct mca_pml_base_send_request_t *req) -{ - ompi_output(mca_ptl_portals_component.portals_output, - "unimplemented function mca_ptl_request_init"); - return OMPI_SUCCESS; -} - - -void -mca_ptl_portals_request_fini(struct mca_ptl_base_module_t *ptl, - struct mca_pml_base_send_request_t *req) -{ - ompi_output(mca_ptl_portals_component.portals_output, - "unimplemented function mca_ptl_request_fini"); - return; -} - void mca_ptl_portals_matched(struct mca_ptl_base_module_t *ptl, struct mca_ptl_base_recv_frag_t *frag)