* Put all the source files in src/, as it was getting really annoying having
everything in one directory. Still have only one Makefile, so it shouldn't change build time at all * Now that I finally understand the header system for data, refactor a little bit of the code to match what really should be happening * start of a hacked up send() - puts the data for a 0 byte message on the other side, and all the pointers are where i think they should be. So my plan of attack will work. But I think I'm going to have to use iovecs instead of memcpy() real soon now. This commit was SVN r5610.
Этот коммит содержится в:
Brian Barrett
родитель
863eac2df8
Коммит
faf3d438bb
@ -31,18 +31,20 @@ component_install =
|
||||
endif
|
||||
|
||||
EXTRA_DIST = \
|
||||
ptl_portals_compat_utcp.c \
|
||||
ptl_portals_compat_redstorm.c
|
||||
src/ptl_portals_compat_utcp.c \
|
||||
src/ptl_portals_compat_redstorm.c
|
||||
|
||||
portals_SOURCES = \
|
||||
ptl_portals.h \
|
||||
ptl_portals_compat.h \
|
||||
ptl_portals_sendfrag.h \
|
||||
ptl_portals.c \
|
||||
ptl_portals_component.c \
|
||||
ptl_portals_stubs.c \
|
||||
ptl_portals_compat_utcp.c \
|
||||
ptl_portals_sendfrag.c
|
||||
src/ptl_portals.h \
|
||||
src/ptl_portals_compat.h \
|
||||
src/ptl_portals_send.h \
|
||||
src/ptl_portals_recv.h \
|
||||
src/ptl_portals.c \
|
||||
src/ptl_portals_component.c \
|
||||
src/ptl_portals_stubs.c \
|
||||
src/ptl_portals_compat_utcp.c \
|
||||
src/ptl_portals_send.c \
|
||||
src/ptl_portals_recv.c
|
||||
|
||||
|
||||
mcacomponentdir = $(libdir)/openmpi
|
||||
|
||||
@ -17,6 +17,6 @@
|
||||
|
||||
# Specific to this module
|
||||
|
||||
PARAM_INIT_FILE=ptl_portals.h
|
||||
PARAM_INIT_FILE=src/ptl_portals.h
|
||||
PARAM_CONFIG_HEADER_FILE="portals_config.h"
|
||||
PARAM_CONFIG_FILES="Makefile"
|
||||
|
||||
@ -1,26 +0,0 @@
|
||||
/*
|
||||
* Copyright (c) 2004-2005 The Trustees of Indiana University.
|
||||
* All rights reserved.
|
||||
* Copyright (c) 2004-2005 The Trustees of the University of Tennessee.
|
||||
* All rights reserved.
|
||||
* Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
|
||||
* University of Stuttgart. All rights reserved.
|
||||
* Copyright (c) 2004-2005 The Regents of the University of California.
|
||||
* All rights reserved.
|
||||
* $COPYRIGHT$
|
||||
*
|
||||
* Additional copyrights may follow
|
||||
*
|
||||
* $HEADER$
|
||||
*/
|
||||
|
||||
#include "ompi_config.h"
|
||||
#include "portals_config.h"
|
||||
|
||||
#include "ptl_portals.h"
|
||||
#include "ptl_portals_compat.h"
|
||||
#include "ptl_portals_sendfrag.h"
|
||||
|
||||
OBJ_CLASS_INSTANCE(mca_ptl_portals_send_frag_t,
|
||||
mca_ptl_base_send_frag_t,
|
||||
NULL, NULL);
|
||||
@ -29,7 +29,8 @@
|
||||
|
||||
#include "ptl_portals.h"
|
||||
#include "ptl_portals_compat.h"
|
||||
#include "ptl_portals_sendfrag.h"
|
||||
#include "ptl_portals_send.h"
|
||||
#include "ptl_portals_recv.h"
|
||||
|
||||
mca_ptl_portals_module_t mca_ptl_portals_module = {
|
||||
{
|
||||
@ -145,7 +146,7 @@ mca_ptl_portals_module_enable(struct mca_ptl_portals_module_t *ptl,
|
||||
ret = PtlEQAlloc(ptl->ni_handle,
|
||||
ptl->first_frag_queue_size,
|
||||
PTL_EQ_HANDLER_NONE,
|
||||
&(ptl->frag_receive_eq_handle));
|
||||
&(ptl->frag_eq_handle));
|
||||
if (ret != PTL_OK) {
|
||||
ompi_output(mca_ptl_portals_component.portals_output,
|
||||
"Failed to allocate event queue: %d", ret);
|
||||
@ -153,10 +154,10 @@ mca_ptl_portals_module_enable(struct mca_ptl_portals_module_t *ptl,
|
||||
}
|
||||
ompi_output_verbose(100, mca_ptl_portals_component.portals_output,
|
||||
"allocated event queue: %d",
|
||||
ptl->frag_receive_eq_handle);
|
||||
ptl->frag_eq_handle);
|
||||
|
||||
for (i = 0 ; i < ptl->first_frag_num_entries ; ++i) {
|
||||
ret = ptl_portals_new_frag_entry(ptl);
|
||||
ret = ptl_portals_post_recv_md(ptl);
|
||||
if (OMPI_SUCCESS != ret) return ret;
|
||||
ptl->frag_queues_created = true;
|
||||
}
|
||||
@ -166,139 +167,6 @@ mca_ptl_portals_module_enable(struct mca_ptl_portals_module_t *ptl,
|
||||
}
|
||||
|
||||
|
||||
int
|
||||
ptl_portals_new_frag_entry(struct mca_ptl_portals_module_t *ptl)
|
||||
{
|
||||
ptl_handle_me_t me_handle;
|
||||
ptl_handle_md_t md_handle;
|
||||
ptl_md_t md;
|
||||
void *mem;
|
||||
int ret;
|
||||
ptl_process_id_t proc = { PTL_NID_ANY, PTL_PID_ANY };
|
||||
|
||||
/* create match entry */
|
||||
ret = PtlMEAttach(ptl->ni_handle,
|
||||
PTL_PORTALS_FRAG_TABLE_ID,
|
||||
proc,
|
||||
0, /* match bits */
|
||||
0, /* ignore bits */
|
||||
PTL_UNLINK,
|
||||
PTL_INS_AFTER,
|
||||
&me_handle);
|
||||
if (PTL_OK != ret) return OMPI_ERROR;
|
||||
|
||||
/* and some memory */
|
||||
mem = malloc(ptl->first_frag_entry_size);
|
||||
if (NULL == mem) {
|
||||
PtlMEUnlink(me_handle);
|
||||
return OMPI_ERR_TEMP_OUT_OF_RESOURCE;
|
||||
}
|
||||
|
||||
/* and the memory descriptor */
|
||||
md.start = mem;
|
||||
md.length = ptl->first_frag_entry_size;
|
||||
md.threshold = PTL_MD_THRESH_INF;
|
||||
md.max_size = md.length - ptl->super.ptl_first_frag_size;
|
||||
md.options = PTL_MD_OP_PUT | PTL_MD_MAX_SIZE;
|
||||
md.user_ptr = NULL;
|
||||
md.eq_handle = ptl->frag_receive_eq_handle;
|
||||
|
||||
ret = PtlMDAttach(me_handle,
|
||||
md,
|
||||
PTL_UNLINK,
|
||||
&md_handle);
|
||||
if (PTL_OK != ret) {
|
||||
PtlMEUnlink(me_handle);
|
||||
return OMPI_ERROR;
|
||||
}
|
||||
|
||||
ompi_output_verbose(50, mca_ptl_portals_component.portals_output,
|
||||
"new receive buffer posted");
|
||||
|
||||
return OMPI_SUCCESS;
|
||||
}
|
||||
|
||||
|
||||
int
|
||||
mca_ptl_portals_send(struct mca_ptl_base_module_t *ptl_base,
|
||||
struct mca_ptl_base_peer_t *ptl_peer,
|
||||
struct mca_pml_base_send_request_t *sendreq,
|
||||
size_t offset, size_t size, int flags)
|
||||
{
|
||||
mca_ptl_portals_module_t* ptl = (mca_ptl_portals_module_t*) ptl_base;
|
||||
ptl_process_id_t *peer_id = (ptl_process_id_t*) ptl_peer;
|
||||
mca_ptl_portals_send_frag_t* sendfrag;
|
||||
mca_ptl_base_header_t* hdr;
|
||||
int ret;
|
||||
ptl_md_t md;
|
||||
|
||||
ompi_output_verbose(100, mca_ptl_portals_component.portals_output,
|
||||
"mca_ptl_portals_send to %lu, %lu",
|
||||
peer_id->nid, peer_id->pid);
|
||||
|
||||
if (sendreq->req_cached) {
|
||||
sendfrag = (mca_ptl_portals_send_frag_t*)(sendreq+1);
|
||||
} else {
|
||||
ompi_output(mca_ptl_portals_component.portals_output,
|
||||
"request not cached - not implemented.");
|
||||
return OMPI_ERROR;
|
||||
}
|
||||
|
||||
/* initialize convertor */
|
||||
if (size > 0) {
|
||||
ompi_output(mca_ptl_portals_component.portals_output,
|
||||
"request size > 0, not implemented");
|
||||
return OMPI_ERROR;
|
||||
} else {
|
||||
sendfrag->frag_send.frag_base.frag_addr = NULL;
|
||||
sendfrag->frag_send.frag_base.frag_size = 0;
|
||||
}
|
||||
|
||||
/* setup message header */
|
||||
hdr = &sendfrag->frag_send.frag_base.frag_header;
|
||||
if(offset == 0) {
|
||||
hdr->hdr_common.hdr_flags = flags;
|
||||
hdr->hdr_match.hdr_contextid = sendreq->req_base.req_comm->c_contextid;
|
||||
hdr->hdr_match.hdr_src = sendreq->req_base.req_comm->c_my_rank;
|
||||
hdr->hdr_match.hdr_dst = sendreq->req_base.req_peer;
|
||||
hdr->hdr_match.hdr_tag = sendreq->req_base.req_tag;
|
||||
hdr->hdr_match.hdr_msg_length = sendreq->req_bytes_packed;
|
||||
hdr->hdr_match.hdr_msg_seq = sendreq->req_base.req_sequence;
|
||||
} else {
|
||||
ompi_output(mca_ptl_portals_component.portals_output,
|
||||
"offset > 0, not implemented");
|
||||
return OMPI_ERROR;
|
||||
}
|
||||
|
||||
/* fragment state */
|
||||
#if 0
|
||||
sendfrag->frag_send.frag_base.frag_owner = &ptl_peer->peer_ptl->super;
|
||||
#endif
|
||||
sendfrag->frag_send.frag_request = sendreq;
|
||||
#if 0
|
||||
sendfrag->frag_send.frag_base.frag_peer = ptl_peer;
|
||||
#endif
|
||||
|
||||
|
||||
/* must update the offset after actual fragment size is determined
|
||||
* before attempting to send the fragment
|
||||
*/
|
||||
mca_pml_base_send_request_offset(sendreq,
|
||||
sendfrag->frag_send.frag_base.frag_size);
|
||||
#if 0
|
||||
md.start = mem;
|
||||
md.length = ptl->first_frag_entry_size;
|
||||
md.threshold = PTL_MD_THRESH_INF;
|
||||
md.max_size = md.length - ptl->super.ptl_first_frag_size;
|
||||
md.options = PTL_MD_OP_PUT | PTL_MD_MAX_SIZE;
|
||||
md.user_ptr = NULL;
|
||||
md.eq_handle = ptl->frag_receive_eq_handle;
|
||||
#endif
|
||||
|
||||
return OMPI_ERROR;
|
||||
}
|
||||
|
||||
|
||||
int
|
||||
mca_ptl_portals_finalize(struct mca_ptl_base_module_t *ptl_base)
|
||||
{
|
||||
@ -315,3 +183,21 @@ mca_ptl_portals_finalize(struct mca_ptl_base_module_t *ptl_base)
|
||||
|
||||
return OMPI_SUCCESS;
|
||||
}
|
||||
|
||||
|
||||
int
|
||||
mca_ptl_portals_request_init(struct mca_ptl_base_module_t *ptl,
|
||||
struct mca_pml_base_send_request_t *req)
|
||||
{
|
||||
OBJ_CONSTRUCT(req + 1, mca_ptl_portals_send_frag_t);
|
||||
return OMPI_SUCCESS;
|
||||
}
|
||||
|
||||
|
||||
void
|
||||
mca_ptl_portals_request_fini(struct mca_ptl_base_module_t *ptl,
|
||||
struct mca_pml_base_send_request_t *req)
|
||||
{
|
||||
OBJ_DESTRUCT(req + 1);
|
||||
}
|
||||
|
||||
@ -60,6 +60,15 @@ struct mca_ptl_portals_component_t {
|
||||
/** List of currently available modules */
|
||||
struct mca_ptl_portals_module_t **portals_modules;
|
||||
|
||||
/** free list of portals send fragments */
|
||||
ompi_free_list_t portals_send_frags;
|
||||
|
||||
/** free list of portals recv fragments */
|
||||
ompi_free_list_t portals_recv_frags;
|
||||
|
||||
/** queue of pending sends */
|
||||
ompi_list_t portals_pending_acks;
|
||||
|
||||
/** lock for accessing component */
|
||||
ompi_mutex_t portals_lock;
|
||||
};
|
||||
@ -157,7 +166,7 @@ struct mca_ptl_portals_module_t {
|
||||
/* frag receive data */
|
||||
bool frag_queues_created;
|
||||
/* frag receive event queue */
|
||||
ptl_handle_eq_t frag_receive_eq_handle;
|
||||
ptl_handle_eq_t frag_eq_handle;
|
||||
|
||||
/** our portals network interface */
|
||||
ptl_handle_ni_t ni_handle;
|
||||
@ -353,6 +362,5 @@ extern int mca_ptl_portals_send(
|
||||
|
||||
extern int mca_ptl_portals_module_enable(struct mca_ptl_portals_module_t *ptl,
|
||||
int value);
|
||||
extern int ptl_portals_new_frag_entry(struct mca_ptl_portals_module_t *ptl);
|
||||
|
||||
#endif
|
||||
@ -115,6 +115,9 @@ mca_ptl_portals_component_open(void)
|
||||
mca_ptl_portals_component.portals_modules = NULL;
|
||||
|
||||
/* initialize objects */
|
||||
OBJ_CONSTRUCT(&mca_ptl_portals_component.portals_send_frags, ompi_free_list_t);
|
||||
OBJ_CONSTRUCT(&mca_ptl_portals_component.portals_recv_frags, ompi_free_list_t);
|
||||
OBJ_CONSTRUCT(&mca_ptl_portals_component.portals_pending_acks, ompi_list_t);
|
||||
OBJ_CONSTRUCT(&mca_ptl_portals_component.portals_lock, ompi_mutex_t);
|
||||
|
||||
/* register portals module parameters */
|
||||
@ -176,6 +179,9 @@ mca_ptl_portals_component_close(void)
|
||||
|
||||
/* release resources */
|
||||
OBJ_DESTRUCT(&mca_ptl_portals_component.portals_lock);
|
||||
OBJ_DESTRUCT(&mca_ptl_portals_component.portals_recv_frags);
|
||||
OBJ_DESTRUCT(&mca_ptl_portals_component.portals_pending_acks);
|
||||
OBJ_DESTRUCT(&mca_ptl_portals_component.portals_lock);
|
||||
|
||||
if (NULL != mca_ptl_portals_component.portals_ifname) {
|
||||
free(mca_ptl_portals_component.portals_ifname);
|
||||
@ -270,7 +276,7 @@ mca_ptl_portals_component_progress(mca_ptl_tstamp_t tstamp)
|
||||
|
||||
if (! module->frag_queues_created) continue;
|
||||
|
||||
ret = PtlEQGet(module->frag_receive_eq_handle, &my_event);
|
||||
ret = PtlEQGet(module->frag_eq_handle, &my_event);
|
||||
if (PTL_EQ_EMPTY == ret) {
|
||||
continue;
|
||||
} else if (!(PTL_OK == ret || PTL_EQ_DROPPED == ret)) {
|
||||
@ -281,11 +287,6 @@ mca_ptl_portals_component_progress(mca_ptl_tstamp_t tstamp)
|
||||
ompi_output_verbose(20, mca_ptl_portals_component.portals_output,
|
||||
"Progress found dropped packets");
|
||||
}
|
||||
|
||||
ompi_output_verbose(100, mca_ptl_portals_component.portals_output,
|
||||
"my_event: %d, %d, %d, %d %d %d",
|
||||
my_event.type, my_event.rlength, my_event.offset,
|
||||
my_event.link, my_event.ni_fail_type, my_event.sequence);
|
||||
num_progressed++;
|
||||
}
|
||||
|
||||
131
src/mca/ptl/portals/src/ptl_portals_send.c
Обычный файл
131
src/mca/ptl/portals/src/ptl_portals_send.c
Обычный файл
@ -0,0 +1,131 @@
|
||||
/*
|
||||
* Copyright (c) 2004-2005 The Trustees of Indiana University.
|
||||
* All rights reserved.
|
||||
* Copyright (c) 2004-2005 The Trustees of the University of Tennessee.
|
||||
* All rights reserved.
|
||||
* Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
|
||||
* University of Stuttgart. All rights reserved.
|
||||
* Copyright (c) 2004-2005 The Regents of the University of California.
|
||||
* All rights reserved.
|
||||
* $COPYRIGHT$
|
||||
*
|
||||
* Additional copyrights may follow
|
||||
*
|
||||
* $HEADER$
|
||||
*/
|
||||
|
||||
#include "ompi_config.h"
|
||||
#include "portals_config.h"
|
||||
|
||||
#include "ptl_portals.h"
|
||||
#include "ptl_portals_compat.h"
|
||||
#include "ptl_portals_send.h"
|
||||
#include "mca/pml/pml.h"
|
||||
#include "mca/pml/base/pml_base_sendreq.h"
|
||||
|
||||
OBJ_CLASS_INSTANCE(mca_ptl_portals_send_frag_t,
|
||||
mca_ptl_base_send_frag_t,
|
||||
NULL, NULL);
|
||||
|
||||
|
||||
int
|
||||
mca_ptl_portals_send(struct mca_ptl_base_module_t *ptl_base,
|
||||
struct mca_ptl_base_peer_t *ptl_peer,
|
||||
struct mca_pml_base_send_request_t *sendreq,
|
||||
size_t offset, size_t size, int flags)
|
||||
{
|
||||
mca_ptl_portals_module_t* ptl = (mca_ptl_portals_module_t*) ptl_base;
|
||||
ptl_process_id_t *peer_id = (ptl_process_id_t*) ptl_peer;
|
||||
mca_ptl_portals_send_frag_t* sendfrag;
|
||||
mca_ptl_base_header_t* hdr;
|
||||
int ret;
|
||||
ptl_md_t md;
|
||||
ptl_handle_md_t md_handle;
|
||||
|
||||
ompi_output_verbose(100, mca_ptl_portals_component.portals_output,
|
||||
"mca_ptl_portals_send to %lu, %lu",
|
||||
peer_id->nid, peer_id->pid);
|
||||
|
||||
if (sendreq->req_cached) {
|
||||
sendfrag = (mca_ptl_portals_send_frag_t*)(sendreq+1);
|
||||
} else {
|
||||
ompi_list_item_t *item;
|
||||
OMPI_FREE_LIST_GET(&mca_ptl_portals_component.portals_send_frags,
|
||||
item, ret);
|
||||
if (NULL == item) return ret;
|
||||
sendfrag = (mca_ptl_portals_send_frag_t *) item;
|
||||
}
|
||||
|
||||
/* initialize convertor */
|
||||
if (size > 0) {
|
||||
ompi_output(mca_ptl_portals_component.portals_output,
|
||||
"request size > 0, not implemented");
|
||||
return OMPI_ERROR;
|
||||
} else {
|
||||
sendfrag->frag_send.frag_base.frag_addr = NULL;
|
||||
sendfrag->frag_send.frag_base.frag_size = 0;
|
||||
}
|
||||
|
||||
/* setup message header */
|
||||
hdr = &sendfrag->frag_send.frag_base.frag_header;
|
||||
if(offset == 0) {
|
||||
hdr->hdr_common.hdr_flags = flags;
|
||||
hdr->hdr_match.hdr_contextid = sendreq->req_base.req_comm->c_contextid;
|
||||
hdr->hdr_match.hdr_src = sendreq->req_base.req_comm->c_my_rank;
|
||||
hdr->hdr_match.hdr_dst = sendreq->req_base.req_peer;
|
||||
hdr->hdr_match.hdr_tag = sendreq->req_base.req_tag;
|
||||
hdr->hdr_match.hdr_msg_length = sendreq->req_bytes_packed;
|
||||
hdr->hdr_match.hdr_msg_seq = sendreq->req_base.req_sequence;
|
||||
} else {
|
||||
ompi_output(mca_ptl_portals_component.portals_output,
|
||||
"offset > 0, not implemented");
|
||||
return OMPI_ERROR;
|
||||
}
|
||||
|
||||
/* fragment state */
|
||||
sendfrag->frag_send.frag_base.frag_owner = ptl_base;
|
||||
sendfrag->frag_send.frag_request = sendreq;
|
||||
sendfrag->frag_send.frag_base.frag_peer = ptl_peer;
|
||||
|
||||
|
||||
/* must update the offset after actual fragment size is determined
|
||||
* before attempting to send the fragment
|
||||
*/
|
||||
mca_pml_base_send_request_offset(sendreq,
|
||||
sendfrag->frag_send.frag_base.frag_size);
|
||||
|
||||
md.start = hdr;
|
||||
md.length = sizeof(mca_ptl_base_header_t);
|
||||
md.threshold = 2; /* we do a put, we get out of here */
|
||||
md.max_size = 0;
|
||||
md.options = 0;
|
||||
md.user_ptr = 123;
|
||||
md.eq_handle = ptl->frag_eq_handle;
|
||||
|
||||
/* make a free-floater */
|
||||
ret = PtlMDBind(ptl->ni_handle,
|
||||
md,
|
||||
PTL_UNLINK,
|
||||
&md_handle);
|
||||
if (ret != PTL_OK) {
|
||||
ompi_output(mca_ptl_portals_component.portals_output,
|
||||
"PtlMDBind failed with error %d", ret);
|
||||
return OMPI_ERROR;
|
||||
}
|
||||
|
||||
ret = PtlPut(md_handle,
|
||||
PTL_NO_ACK_REQ,
|
||||
*((ptl_process_id_t*) ptl_peer),
|
||||
PTL_PORTALS_FRAG_TABLE_ID,
|
||||
0, /* ac_index */
|
||||
0, /* match bits */
|
||||
0, /* remote offset - not used */
|
||||
321); /* hdr_data - not used */
|
||||
if (ret != PTL_OK) {
|
||||
ompi_output(mca_ptl_portals_component.portals_output,
|
||||
"PtlPut failed with error %d", ret);
|
||||
return OMPI_ERROR;
|
||||
}
|
||||
|
||||
return OMPI_SUCCESS;
|
||||
}
|
||||
@ -31,25 +31,6 @@
|
||||
|
||||
|
||||
|
||||
int
|
||||
mca_ptl_portals_request_init(struct mca_ptl_base_module_t *ptl,
|
||||
struct mca_pml_base_send_request_t *req)
|
||||
{
|
||||
ompi_output(mca_ptl_portals_component.portals_output,
|
||||
"unimplemented function mca_ptl_request_init");
|
||||
return OMPI_SUCCESS;
|
||||
}
|
||||
|
||||
|
||||
void
|
||||
mca_ptl_portals_request_fini(struct mca_ptl_base_module_t *ptl,
|
||||
struct mca_pml_base_send_request_t *req)
|
||||
{
|
||||
ompi_output(mca_ptl_portals_component.portals_output,
|
||||
"unimplemented function mca_ptl_request_fini");
|
||||
return;
|
||||
}
|
||||
|
||||
void
|
||||
mca_ptl_portals_matched(struct mca_ptl_base_module_t *ptl,
|
||||
struct mca_ptl_base_recv_frag_t *frag)
|
||||
Загрузка…
x
Ссылка в новой задаче
Block a user