rip out a bunch of code that didn't work and really sucked and was only there
to try to get some numbers that I couldn't actually get. So back to the restart point. This commit was SVN r9914.
Этот коммит содержится в:
Brian Barrett
родитель
f2a6e63d82
Коммит
db03ca0cc0
@ -98,9 +98,6 @@ ompi_pml_portals_add_procs(struct ompi_proc_t** procs, size_t nprocs)
|
||||
static bool done_init = false;
|
||||
int ret;
|
||||
|
||||
opal_output_verbose(100, ompi_pml_portals.portals_output,
|
||||
"pml_add_procs called with %d procs\n", nprocs);
|
||||
|
||||
if (0 == nprocs) return OMPI_SUCCESS;
|
||||
|
||||
/* allocate space for our pml information */
|
||||
@ -111,75 +108,11 @@ ompi_pml_portals_add_procs(struct ompi_proc_t** procs, size_t nprocs)
|
||||
|
||||
ompi_pml_portals_add_procs_compat(procs, nprocs);
|
||||
|
||||
opal_output_verbose(100, ompi_pml_portals.portals_output,
|
||||
"proc list:");
|
||||
for (i = 0 ; i < nprocs ; ++i) {
|
||||
ompi_pml_portals_proc_t *ptlproc = (ompi_pml_portals_proc_t*) procs[i]->proc_pml;
|
||||
opal_output_verbose(100, ompi_pml_portals.portals_output,
|
||||
" procs[%d] = %u, %u",
|
||||
i, ptlproc->proc_id.nid, ptlproc->proc_id.pid);
|
||||
}
|
||||
|
||||
if (!done_init) {
|
||||
ptl_md_t md;
|
||||
ptl_handle_md_t md_h;
|
||||
ptl_process_id_t anyproc;
|
||||
uint64_t match_bits = 0;
|
||||
|
||||
opal_output_verbose(10, ompi_pml_portals.portals_output,
|
||||
"running initialization");
|
||||
|
||||
/* setup our event queues */
|
||||
ret = PtlEQAlloc(ompi_pml_portals.portals_ni_h,
|
||||
3, /* BWB - fix me */
|
||||
PTL_EQ_HANDLER_NONE,
|
||||
&(ompi_pml_portals.portals_blocking_send_queue));
|
||||
assert(ret == PTL_OK);
|
||||
|
||||
ret = PtlEQAlloc(ompi_pml_portals.portals_ni_h,
|
||||
3, /* BWB - fix me */
|
||||
PTL_EQ_HANDLER_NONE,
|
||||
&(ompi_pml_portals.portals_blocking_receive_queue));
|
||||
assert(ret == PTL_OK);
|
||||
|
||||
ret = PtlEQAlloc(ompi_pml_portals.portals_ni_h,
|
||||
1024, /* BWB - fix me */
|
||||
PTL_EQ_HANDLER_NONE,
|
||||
&(ompi_pml_portals.portals_unexpected_receive_queue));
|
||||
assert(ret == PTL_OK);
|
||||
|
||||
/* create unexpected message match entry */
|
||||
anyproc.nid = PTL_NID_ANY;
|
||||
anyproc.pid = PTL_PID_ANY;
|
||||
|
||||
/* unexpected message match entry should receive from anyone,
|
||||
so ignore bits are all 1 */
|
||||
ret = PtlMEAttach(ompi_pml_portals.portals_ni_h,
|
||||
PML_PTLS_INDEX_RECV,
|
||||
anyproc,
|
||||
match_bits,
|
||||
~match_bits,
|
||||
PTL_RETAIN,
|
||||
PTL_INS_AFTER,
|
||||
&(ompi_pml_portals.portals_unexpected_me_h));
|
||||
assert(ret == PTL_OK);
|
||||
|
||||
md.start = NULL;
|
||||
md.length = 0;
|
||||
md.threshold = PTL_MD_THRESH_INF;
|
||||
md.max_size = 0;
|
||||
md.options = (PTL_MD_OP_PUT | PTL_MD_TRUNCATE | PTL_MD_ACK_DISABLE | PTL_MD_EVENT_START_DISABLE);
|
||||
md.eq_handle = ompi_pml_portals.portals_unexpected_receive_queue;
|
||||
|
||||
ret = PtlMDAttach(ompi_pml_portals.portals_unexpected_me_h,
|
||||
md,
|
||||
PTL_RETAIN,
|
||||
&md_h);
|
||||
assert(ret == PTL_OK);
|
||||
|
||||
opal_output_verbose(10, ompi_pml_portals.portals_output,
|
||||
"unexpected me: %ld",
|
||||
ompi_pml_portals.portals_unexpected_me_h);
|
||||
/* do all initialization that should happen *after* PtlNIInit
|
||||
here. Can't do it before this because we need the modex to
|
||||
wire everything up in the utcp case */
|
||||
|
||||
done_init = true;
|
||||
}
|
||||
@ -193,9 +126,6 @@ ompi_pml_portals_del_procs(struct ompi_proc_t** procs, size_t nprocs)
|
||||
{
|
||||
size_t i;
|
||||
|
||||
opal_output_verbose(100, ompi_pml_portals.portals_output,
|
||||
"pml_del_procs called with %d procs\n", nprocs);
|
||||
|
||||
if (0 == nprocs) return OMPI_SUCCESS;
|
||||
|
||||
/* allocate space for our pml information */
|
||||
|
||||
@ -65,21 +65,6 @@ struct ompi_pml_portals_t {
|
||||
|
||||
/* our portals network interface */
|
||||
ptl_handle_ni_t portals_ni_h;
|
||||
|
||||
/* blocking send event queue */
|
||||
ptl_handle_eq_t portals_blocking_send_queue;
|
||||
ompi_convertor_t portals_blocking_send_convertor;
|
||||
|
||||
/* blocking receive event queue */
|
||||
ptl_handle_eq_t portals_blocking_receive_queue;
|
||||
ompi_convertor_t portals_blocking_receive_convertor;
|
||||
|
||||
/* unexpected receive event queue */
|
||||
ptl_handle_eq_t portals_unexpected_receive_queue;
|
||||
|
||||
ptl_handle_me_t portals_unexpected_me_h;
|
||||
|
||||
opal_list_t portals_unexpected_events;
|
||||
};
|
||||
typedef struct ompi_pml_portals_t ompi_pml_portals_t;
|
||||
extern ompi_pml_portals_t ompi_pml_portals;
|
||||
@ -97,78 +82,13 @@ extern opal_class_t ompi_pml_portals_proc_t_class;
|
||||
|
||||
|
||||
|
||||
/*
|
||||
* Portals match info
|
||||
*/
|
||||
|
||||
#define PML_PTLS_READY 0x8000000000000000ULL
|
||||
#define PML_PTLS_LONG 0x4000000000000000ULL
|
||||
#define PML_PTLS_SHORT 0x2000000000000000ULL
|
||||
|
||||
#define PML_PTLS_PROT_MASK 0xE000000000000000ULL
|
||||
#define PML_PTLS_CTX_MASK 0x1FFF000000000000ULL
|
||||
#define PML_PTLS_SOURCE_MASK 0x0000FFFF00000000ULL
|
||||
#define PML_PTLS_TAG_MASK 0x000000007FFFFFFFULL
|
||||
|
||||
#define PML_PTLS_RECV_BITS(match_bits, ignore_bits, ctxid, src, tag) \
|
||||
{ \
|
||||
match_bits = 0; \
|
||||
ignore_bits = PML_PTLS_PROT_MASK; \
|
||||
\
|
||||
match_bits = ctxid; \
|
||||
match_bits = (match_bits << 16); \
|
||||
\
|
||||
if (src == MPI_ANY_SOURCE) { \
|
||||
match_bits = (match_bits << 32); \
|
||||
ignore_bits |= PML_PTLS_SOURCE_MASK; \
|
||||
} else { \
|
||||
match_bits |= src; \
|
||||
match_bits = (match_bits << 32); \
|
||||
} \
|
||||
\
|
||||
if (tag == MPI_ANY_TAG) { \
|
||||
ignore_bits |= PML_PTLS_TAG_MASK; \
|
||||
} else { \
|
||||
match_bits |= (0x00000000FFFFFFFFULL & tag); \
|
||||
} \
|
||||
} \
|
||||
|
||||
|
||||
#define PML_PTLS_SEND_BITS(match_bits, ctxid, src, tag) \
|
||||
{ \
|
||||
match_bits = ctxid; \
|
||||
match_bits = (match_bits << 16); \
|
||||
match_bits |= src; \
|
||||
match_bits = (match_bits << 32); \
|
||||
match_bits |= (0x00000000FFFFFFFFULL & tag); \
|
||||
}
|
||||
|
||||
#define PML_PTLS_IS_LONG(match_bits) (match_bits & PML_PTLS_LONG)
|
||||
#define PML_PTLS_GET_SOURCE(match_bits, src) \
|
||||
{ \
|
||||
src = (int)((PML_PTLS_SOURCE_MASK & match_bits) >> 32); \
|
||||
}
|
||||
|
||||
#define PML_PTLS_GET_TAG(match_bits, tag) \
|
||||
{ \
|
||||
tag = (int)(PML_PTLS_TAG_MASK & match_bits); \
|
||||
}
|
||||
|
||||
#define PML_PTLS_GET_CONTEXT(match_bits, ctxid) \
|
||||
{ \
|
||||
ctxid = (int)((PML_PTLS_CTX_MASK & match_bits) >> 48); \
|
||||
}
|
||||
|
||||
|
||||
/* table indexes */
|
||||
#define PML_PTLS_INDEX_RECV (OMPI_PML_PORTALS_STARTING_TABLE_ID + 0)
|
||||
#define PML_PTLS_INDEX_READ (OMPI_PML_PORTALS_STARTING_TABLE_ID + 1)
|
||||
#define PML_PTLS_INDEX_ACK (OMPI_PML_PORTALS_STARTING_TABLE_ID + 2)
|
||||
|
||||
/*
|
||||
* PML interface functions.
|
||||
*/
|
||||
|
||||
/* PML interface functions */
|
||||
extern int ompi_pml_portals_add_procs(struct ompi_proc_t **procs, size_t nprocs);
|
||||
extern int ompi_pml_portals_del_procs(struct ompi_proc_t **procs, size_t nprocs);
|
||||
|
||||
|
||||
@ -144,17 +144,6 @@ ompi_pml_portals_component_init(int* priority,
|
||||
return NULL;
|
||||
}
|
||||
|
||||
OBJ_CONSTRUCT(&(ompi_pml_portals.portals_unexpected_events),
|
||||
opal_list_t);
|
||||
|
||||
OBJ_CONSTRUCT(&ompi_pml_portals.portals_blocking_send_convertor,
|
||||
ompi_convertor_t);
|
||||
OBJ_CONSTRUCT(&ompi_pml_portals.portals_blocking_receive_convertor,
|
||||
ompi_convertor_t);
|
||||
|
||||
opal_output_verbose(20, ompi_pml_portals.portals_output,
|
||||
"successfully initialized portals pml");
|
||||
|
||||
return &ompi_pml_portals.super;
|
||||
}
|
||||
|
||||
@ -162,15 +151,8 @@ ompi_pml_portals_component_init(int* priority,
|
||||
static int
|
||||
ompi_pml_portals_component_fini(void)
|
||||
{
|
||||
PtlEQFree(ompi_pml_portals.portals_unexpected_receive_queue);
|
||||
PtlEQFree(ompi_pml_portals.portals_blocking_receive_queue);
|
||||
PtlEQFree(ompi_pml_portals.portals_blocking_send_queue);
|
||||
|
||||
PtlNIFini(ompi_pml_portals.portals_ni_h);
|
||||
|
||||
OBJ_DESTRUCT(&ompi_pml_portals.portals_blocking_send_convertor);
|
||||
OBJ_DESTRUCT(&ompi_pml_portals.portals_blocking_receive_convertor);
|
||||
|
||||
opal_output_verbose(20, ompi_pml_portals.portals_output,
|
||||
"successfully finalized portals pml");
|
||||
|
||||
|
||||
@ -1,118 +0,0 @@
|
||||
/*
|
||||
* Copyright (c) 2004-2006 The Trustees of Indiana University and Indiana
|
||||
* University Research and Technology
|
||||
* Corporation. All rights reserved.
|
||||
* All rights reserved.
|
||||
* $COPYRIGHT$
|
||||
*
|
||||
* Additional copyrights may follow
|
||||
*
|
||||
* $HEADER$
|
||||
*/
|
||||
|
||||
#ifndef PML_PORTALS_DATATYPE_H
|
||||
#define PML_PORTALS_DATATYPE_H
|
||||
|
||||
|
||||
static inline int
|
||||
ompi_pml_portals_prepare_md_send(ompi_convertor_t *convertor,
|
||||
ptl_md_t *md,
|
||||
int *free_after)
|
||||
{
|
||||
struct iovec iov;
|
||||
uint32_t iov_count = 1;
|
||||
int32_t conv_free_after;
|
||||
size_t bufsize;
|
||||
|
||||
ompi_convertor_get_packed_size(convertor, &bufsize);
|
||||
iov.iov_len = bufsize;
|
||||
if (0 == ompi_convertor_need_buffers(convertor)) {
|
||||
iov.iov_base = NULL;
|
||||
*free_after = 0;
|
||||
} else {
|
||||
/* BWB - try to use the iovec option here */
|
||||
|
||||
iov.iov_base = malloc(bufsize);
|
||||
if (NULL == iov.iov_base) {
|
||||
return OMPI_ERR_OUT_OF_RESOURCE;
|
||||
}
|
||||
*free_after = 1;
|
||||
}
|
||||
|
||||
ompi_convertor_pack(convertor, &iov, &iov_count, &bufsize,
|
||||
&conv_free_after);
|
||||
|
||||
md->start = iov.iov_base;
|
||||
md->length = iov.iov_len;
|
||||
md->options = 0;
|
||||
|
||||
return OMPI_SUCCESS;
|
||||
}
|
||||
|
||||
|
||||
static inline int
|
||||
ompi_pml_portals_free_md_send(ptl_md_t *md, int free_after)
|
||||
{
|
||||
if (free_after) {
|
||||
free(md->start);
|
||||
}
|
||||
|
||||
return OMPI_SUCCESS;
|
||||
}
|
||||
|
||||
|
||||
static inline int
|
||||
ompi_pml_portals_prepare_md_recv(ompi_convertor_t *convertor,
|
||||
ptl_md_t *md,
|
||||
int *free_after)
|
||||
{
|
||||
struct iovec iov;
|
||||
size_t bufsize;
|
||||
long lb;
|
||||
|
||||
ompi_convertor_get_packed_size(convertor, &bufsize);
|
||||
iov.iov_len = bufsize;
|
||||
if (0 == ompi_convertor_need_buffers(convertor)) {
|
||||
ompi_ddt_type_lb(convertor->pDesc, &lb);
|
||||
iov.iov_base = convertor->pBaseBuf + lb + convertor->bConverted;
|
||||
*free_after = 0;
|
||||
} else {
|
||||
/* BWB - try to use the iovec option here */
|
||||
|
||||
iov.iov_base = malloc(bufsize);
|
||||
if (NULL == iov.iov_base) {
|
||||
return OMPI_ERR_OUT_OF_RESOURCE;
|
||||
}
|
||||
*free_after = 1;
|
||||
}
|
||||
|
||||
md->start = iov.iov_base;
|
||||
md->length = iov.iov_len;
|
||||
md->options = 0;
|
||||
|
||||
return OMPI_SUCCESS;
|
||||
}
|
||||
|
||||
static inline int
|
||||
ompi_pml_portals_free_md_recv(ompi_convertor_t *convertor,
|
||||
ptl_md_t *md,
|
||||
int free_after)
|
||||
{
|
||||
uint32_t iov_count = 1;
|
||||
size_t max_data;
|
||||
struct iovec iov;
|
||||
|
||||
iov.iov_len = md->length;
|
||||
iov.iov_base = md->start;
|
||||
|
||||
if (free_after) {
|
||||
/* need to unpack into user buffer */
|
||||
ompi_convertor_unpack(convertor, &iov, &iov_count,
|
||||
&max_data, &free_after);
|
||||
free(md->start);
|
||||
}
|
||||
|
||||
return OMPI_SUCCESS;
|
||||
}
|
||||
|
||||
#endif
|
||||
@ -25,44 +25,6 @@
|
||||
#include "ompi/datatype/convertor.h"
|
||||
#include "pml_portals_datatype.h"
|
||||
|
||||
|
||||
struct pml_portals_recv_info_t {
|
||||
opal_list_item_t super;
|
||||
ptl_event_t ev;
|
||||
};
|
||||
typedef struct pml_portals_recv_info_t pml_portals_recv_info_t;
|
||||
OBJ_CLASS_INSTANCE(pml_portals_recv_info_t, opal_list_item_t, NULL, NULL);
|
||||
|
||||
|
||||
static int
|
||||
get_data(ptl_event_t ev, ptl_md_t md, ompi_convertor_t *convertor)
|
||||
{
|
||||
ptl_handle_md_t md_h;
|
||||
|
||||
OPAL_OUTPUT_VERBOSE((100, ompi_pml_portals.portals_output,
|
||||
"calling get_data for %ld", ev.hdr_data));
|
||||
|
||||
/* create the floating md */
|
||||
md.threshold = 1;
|
||||
md.options = PTL_MD_EVENT_START_DISABLE;
|
||||
md.eq_handle = ompi_pml_portals.portals_blocking_receive_queue;
|
||||
|
||||
PtlMDBind(ompi_pml_portals.portals_ni_h, md,
|
||||
PTL_RETAIN, &md_h);
|
||||
|
||||
PtlGet(md_h, ev.initiator, PML_PTLS_INDEX_READ,
|
||||
0, ev.hdr_data, 0);
|
||||
|
||||
PtlEQWait(ompi_pml_portals.portals_blocking_receive_queue, &ev);
|
||||
assert(ev.type == PTL_EVENT_REPLY_END);
|
||||
OPAL_OUTPUT_VERBOSE((100, ompi_pml_portals.portals_output,
|
||||
"get: REPLY_END event received"));
|
||||
|
||||
PtlMDUnlink(md_h);
|
||||
|
||||
return OMPI_SUCCESS;
|
||||
}
|
||||
|
||||
int
|
||||
ompi_pml_portals_irecv_init(void *addr,
|
||||
size_t count,
|
||||
@ -98,169 +60,15 @@ ompi_pml_portals_recv(void *buf,
|
||||
struct ompi_communicator_t *comm,
|
||||
ompi_status_public_t * status)
|
||||
{
|
||||
uint64_t ignore_bits, match_bits;
|
||||
opal_list_item_t *list_item;
|
||||
int ret, free_after;
|
||||
ptl_md_t md, new_md;
|
||||
ptl_handle_md_t md_h;
|
||||
ptl_handle_me_t me_h;
|
||||
ptl_process_id_t portals_proc;
|
||||
ompi_pml_portals_proc_t *pml_portals_proc =
|
||||
(ompi_pml_portals_proc_t*) comm->c_pml_procs[src];
|
||||
ptl_event_t ev;
|
||||
|
||||
OPAL_OUTPUT_VERBOSE((90, ompi_pml_portals.portals_output,
|
||||
"receive called from %d", src));
|
||||
|
||||
/* BWB - fix me - need some way of finding source in ANY_SOURCE case */
|
||||
ompi_convertor_copy_and_prepare_for_send(comm->c_pml_procs[comm->c_my_rank]->proc_ompi->proc_convertor,
|
||||
datatype,
|
||||
count,
|
||||
buf,
|
||||
0,
|
||||
&ompi_pml_portals.portals_blocking_receive_convertor);
|
||||
ptl_process_id_t source;
|
||||
|
||||
if (MPI_ANY_SOURCE == src) {
|
||||
portals_proc.nid = PTL_NID_ANY;
|
||||
portals_proc.pid = PTL_PID_ANY;
|
||||
source.nid = PTL_NID_ANY;
|
||||
source.pid = PTL_PID_ANY;
|
||||
} else {
|
||||
portals_proc = pml_portals_proc->proc_id;
|
||||
}
|
||||
ompi_pml_portals_prepare_md_recv(&ompi_pml_portals.portals_blocking_receive_convertor, &md, &free_after);
|
||||
|
||||
PML_PTLS_RECV_BITS(match_bits, ignore_bits, comm->c_contextid, src, tag);
|
||||
|
||||
/* first, make sure it's not in the queue of processed unexpected msgs */
|
||||
list_item = opal_list_get_first(&(ompi_pml_portals.portals_unexpected_events));
|
||||
while (list_item != opal_list_get_end(&(ompi_pml_portals.portals_unexpected_events))) {
|
||||
opal_list_item_t *next = opal_list_get_next(list_item);
|
||||
pml_portals_recv_info_t * info = (pml_portals_recv_info_t*) list_item;
|
||||
|
||||
if ((info->ev.match_bits & ~ignore_bits) == match_bits) {
|
||||
/* we have a match... */
|
||||
OPAL_OUTPUT_VERBOSE((100, ompi_pml_portals.portals_output,
|
||||
"recv: event waiting in queue\n"));
|
||||
get_data(ev, md, &ompi_pml_portals.portals_blocking_receive_convertor);
|
||||
opal_list_remove_item(&(ompi_pml_portals.portals_unexpected_events),
|
||||
list_item);
|
||||
OBJ_RELEASE(list_item);
|
||||
goto cleanup;
|
||||
} else {
|
||||
OPAL_OUTPUT_VERBOSE((100, ompi_pml_portals.portals_output,
|
||||
"recv: ignoring data: %lx", info->ev.match_bits));
|
||||
}
|
||||
list_item = next;
|
||||
source = ((ompi_pml_portals_proc_t*) comm->c_pml_procs[src])->proc_id;
|
||||
}
|
||||
|
||||
restart_search:
|
||||
/* now check the unexpected event queue */
|
||||
while (true) {
|
||||
int ret = PtlEQGet(ompi_pml_portals.portals_unexpected_receive_queue,
|
||||
&ev);
|
||||
if (PTL_OK == ret) {
|
||||
if ((ev.match_bits & ~ignore_bits) == match_bits) {
|
||||
/* we have a match... */
|
||||
OPAL_OUTPUT_VERBOSE((100, ompi_pml_portals.portals_output,
|
||||
"recv: event waiting in portals\n"));
|
||||
get_data(ev, md, &ompi_pml_portals.portals_blocking_receive_convertor);
|
||||
goto cleanup;
|
||||
} else {
|
||||
pml_portals_recv_info_t *item = OBJ_NEW(pml_portals_recv_info_t);
|
||||
OPAL_OUTPUT_VERBOSE((100, ompi_pml_portals.portals_output,
|
||||
"recv: ignoring data: %lx", ev.match_bits));
|
||||
item->ev = ev;
|
||||
opal_list_append(&(ompi_pml_portals.portals_unexpected_events),
|
||||
&(item->super));
|
||||
}
|
||||
} else if (PTL_EQ_EMPTY == ret) {
|
||||
break;
|
||||
} else {
|
||||
abort();
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
/* now post a receive */
|
||||
OPAL_OUTPUT_VERBOSE((100, ompi_pml_portals.portals_output,
|
||||
"receive from: %u, %u\n", portals_proc.nid, portals_proc.pid));
|
||||
OPAL_OUTPUT_VERBOSE((100, ompi_pml_portals.portals_output,
|
||||
"receive match bits: %lx, %lx\n", match_bits, ignore_bits));
|
||||
PtlMEInsert(ompi_pml_portals.portals_unexpected_me_h,
|
||||
portals_proc,
|
||||
match_bits,
|
||||
ignore_bits,
|
||||
PTL_UNLINK,
|
||||
PTL_INS_BEFORE,
|
||||
&me_h);
|
||||
|
||||
md.threshold = 0;
|
||||
md.options |= (PTL_MD_OP_PUT | PTL_MD_EVENT_START_DISABLE);
|
||||
md.eq_handle = ompi_pml_portals.portals_blocking_receive_queue;
|
||||
PtlMDAttach(me_h, md, PTL_RETAIN, &md_h);
|
||||
|
||||
/* now try to make active */
|
||||
new_md = md;
|
||||
new_md.threshold = 3;
|
||||
#if 0
|
||||
OPAL_OUTPUT_VERBOSE((100, ompi_pml_portals.portals_output,
|
||||
" calling PtlMDUpdate(\n"));
|
||||
OPAL_OUTPUT_VERBOSE((100, ompi_pml_portals.portals_output,
|
||||
" md_handle = %d\n",md_h));
|
||||
OPAL_OUTPUT_VERBOSE((100, ompi_pml_portals.portals_output,
|
||||
" old md =>\n"));
|
||||
OPAL_OUTPUT_VERBOSE((100, ompi_pml_portals.portals_output,
|
||||
" start = %p\n",md.start));
|
||||
OPAL_OUTPUT_VERBOSE((100, ompi_pml_portals.portals_output,
|
||||
" length = %d\n",md.length));
|
||||
OPAL_OUTPUT_VERBOSE((100, ompi_pml_portals.portals_output,
|
||||
" threshold = %d\n",md.threshold));
|
||||
OPAL_OUTPUT_VERBOSE((100, ompi_pml_portals.portals_output,
|
||||
" options = %d\n",md.options));
|
||||
OPAL_OUTPUT_VERBOSE((100, ompi_pml_portals.portals_output,
|
||||
" user_ptr = %p\n",md.user_ptr));
|
||||
OPAL_OUTPUT_VERBOSE((100, ompi_pml_portals.portals_output,
|
||||
" eventq = %d\n",md.eq_handle));
|
||||
OPAL_OUTPUT_VERBOSE((100, ompi_pml_portals.portals_output,
|
||||
" new md =>\n"));
|
||||
OPAL_OUTPUT_VERBOSE((100, ompi_pml_portals.portals_output,
|
||||
" start = %p\n",new_md.start));
|
||||
OPAL_OUTPUT_VERBOSE((100, ompi_pml_portals.portals_output,
|
||||
" length = %d\n",new_md.length));
|
||||
OPAL_OUTPUT_VERBOSE((100, ompi_pml_portals.portals_output,
|
||||
" threshold = %d\n",new_md.threshold));
|
||||
OPAL_OUTPUT_VERBOSE((100, ompi_pml_portals.portals_output,
|
||||
" options = %d\n",new_md.options));
|
||||
OPAL_OUTPUT_VERBOSE((100, ompi_pml_portals.portals_output,
|
||||
" user_ptr = %p\n",new_md.user_ptr));
|
||||
OPAL_OUTPUT_VERBOSE((100, ompi_pml_portals.portals_output,
|
||||
" eventq = %d\n",new_md.eq_handle));
|
||||
OPAL_OUTPUT_VERBOSE((100, ompi_pml_portals.portals_output,
|
||||
" test eventq = %d )\n",ompi_pml_portals.portals_unexpected_receive_queue));
|
||||
#endif
|
||||
|
||||
ret = PtlMDUpdate(md_h, &md, &new_md, ompi_pml_portals.portals_unexpected_receive_queue);
|
||||
OPAL_OUTPUT_VERBOSE((100, ompi_pml_portals.portals_output,
|
||||
"recv: Update ret: %d\n", ret));
|
||||
if (ret == PTL_MD_NO_UPDATE) {
|
||||
OPAL_OUTPUT_VERBOSE((100, ompi_pml_portals.portals_output,
|
||||
"recv: no update :(\n"));
|
||||
/* a message has arrived since we searched - look again */
|
||||
PtlMDUnlink(md_h);
|
||||
if (free_after) { free(md.start); }
|
||||
goto restart_search;
|
||||
}
|
||||
|
||||
/* wait for our completion event */
|
||||
PtlEQWait(ompi_pml_portals.portals_blocking_receive_queue, &ev);
|
||||
assert(ev.type == PTL_EVENT_PUT_END);
|
||||
OPAL_OUTPUT_VERBOSE((90, ompi_pml_portals.portals_output,
|
||||
"recv: PUT_END event received"));
|
||||
|
||||
PtlMDUnlink(md_h);
|
||||
|
||||
cleanup:
|
||||
ompi_pml_portals_free_md_recv(&ompi_pml_portals.portals_blocking_receive_convertor, &md, free_after);
|
||||
|
||||
return OMPI_SUCCESS;
|
||||
return OMPI_ERR_NOT_IMPLEMENTED;
|
||||
}
|
||||
|
||||
|
||||
@ -21,7 +21,6 @@
|
||||
#include "ompi/datatype/datatype.h"
|
||||
#include "ompi/communicator/communicator.h"
|
||||
#include "ompi/datatype/convertor.h"
|
||||
#include "pml_portals_datatype.h"
|
||||
|
||||
int
|
||||
ompi_pml_portals_isend_init(void* buf,
|
||||
@ -60,73 +59,8 @@ ompi_pml_portals_send(void *buf,
|
||||
mca_pml_base_send_mode_t sendmode,
|
||||
ompi_communicator_t* comm)
|
||||
{
|
||||
int ret, free_after;
|
||||
static int msg_count = 1;
|
||||
uint64_t match_bits;
|
||||
ptl_process_id_t target = ((ompi_pml_portals_proc_t*) comm->c_pml_procs[dst])->proc_id;
|
||||
|
||||
ptl_md_t md;
|
||||
ptl_handle_me_t me_h;
|
||||
ptl_handle_md_t md_h;
|
||||
ptl_event_t ev;
|
||||
ompi_pml_portals_proc_t *portals_proc =
|
||||
(ompi_pml_portals_proc_t*) comm->c_pml_procs[dst];
|
||||
|
||||
if (MCA_PML_BASE_SEND_SYNCHRONOUS == sendmode) abort();
|
||||
|
||||
ompi_convertor_copy_and_prepare_for_send(comm->c_pml_procs[dst]->proc_ompi->proc_convertor,
|
||||
datatype,
|
||||
count,
|
||||
buf,
|
||||
0,
|
||||
&ompi_pml_portals.portals_blocking_send_convertor);
|
||||
PtlMEAttach(ompi_pml_portals.portals_ni_h,
|
||||
PML_PTLS_INDEX_READ,
|
||||
portals_proc->proc_id,
|
||||
msg_count,
|
||||
0,
|
||||
PTL_UNLINK,
|
||||
PTL_INS_AFTER,
|
||||
&me_h);
|
||||
|
||||
ompi_pml_portals_prepare_md_send(&ompi_pml_portals.portals_blocking_send_convertor,
|
||||
&md, &free_after);
|
||||
md.threshold = 100;
|
||||
md.options |= (PTL_MD_OP_GET | PTL_MD_EVENT_START_DISABLE);
|
||||
md.eq_handle = ompi_pml_portals.portals_blocking_send_queue;
|
||||
PtlMDAttach(me_h, md, PTL_RETAIN, &md_h);
|
||||
PML_PTLS_SEND_BITS(match_bits, comm->c_contextid, comm->c_my_rank, tag);
|
||||
|
||||
OPAL_OUTPUT_VERBOSE((100, ompi_pml_portals.portals_output,
|
||||
"send to: %u, %u\n",
|
||||
portals_proc->proc_id.nid, portals_proc->proc_id.pid));
|
||||
OPAL_OUTPUT_VERBOSE((100, ompi_pml_portals.portals_output,
|
||||
"send match bits: %lx\n", match_bits));
|
||||
PtlPut(md_h, PTL_ACK_REQ, portals_proc->proc_id,
|
||||
PML_PTLS_INDEX_RECV, 0,
|
||||
match_bits, 0, msg_count);
|
||||
|
||||
msg_count++;
|
||||
|
||||
/* our send end event */
|
||||
ret = PtlEQWait(ompi_pml_portals.portals_blocking_send_queue, &ev);
|
||||
assert(ret == PTL_OK);
|
||||
assert(ev.type == PTL_EVENT_SEND_END);
|
||||
OPAL_OUTPUT_VERBOSE((100, ompi_pml_portals.portals_output,
|
||||
"send: SEND_END event received"));
|
||||
|
||||
/* our ack / get event */
|
||||
ret = PtlEQWait(ompi_pml_portals.portals_blocking_send_queue, &ev);
|
||||
assert(ret == PTL_OK);
|
||||
assert((ev.type == PTL_EVENT_ACK) || (ev.type == PTL_EVENT_GET_END));
|
||||
OPAL_OUTPUT_VERBOSE((100, ompi_pml_portals.portals_output,
|
||||
"send: GET_END/ACK event received"));
|
||||
|
||||
ompi_pml_portals_free_md_send(&md, free_after);
|
||||
|
||||
PtlMDUnlink(md_h);
|
||||
|
||||
ompi_convertor_cleanup(&ompi_pml_portals.portals_blocking_send_convertor);
|
||||
|
||||
return OMPI_SUCCESS;
|
||||
return OMPI_ERR_NOT_IMPLEMENTED;
|
||||
}
|
||||
|
||||
|
||||
Загрузка…
Ссылка в новой задаче
Block a user