ports/openmpi
ports/openmpi
1
1
Форкнуть 0
Код Релизы Активность

Bring Portals4 updates back into the trunk

Просмотр исходного кода 
This commit was SVN r24154.
Этот коммит содержится в:
Brian Barrett 2010-12-07 20:11:25 +00:00
родитель 2dc5cbb483
Коммит a26fadb26e
14 изменённых файлов: 1399 добавлений и 297 удалений
Показать статистику Скачать Patch файл Скачать Diff файл Показать все файлы Свернуть все файлы

12
ompi/mca/mtl/portals4/Makefile.am
Просмотреть файл

@ -10,6 +10,7 @@
# Copyright (c) 2004-2005 The Regents of the University of California.
# All rights reserved.
# Copyright (c) 2010 Cisco Systems, Inc. All rights reserved.
# Copyright (c) 2010 Sandia National Laboratories. All rights reserved.
# $COPYRIGHT$
#
# Additional copyrights may follow
@ -32,14 +33,16 @@ component_install =
endif
local_sources = \
mtl_portals4_component.c \
mtl_portals4.c \
mtl_portals4.h \
mtl_portals4_component.c \
mtl_portals4_endpoint.h \
mtl_portals4.h \
mtl_portals4_probe.c \
mtl_portals4_recv.c \
mtl_portals4_recv_short.c \
mtl_portals4_recv_short.h \
mtl_portals4_request.h \
mtl_portals4_send.c \
mtl_portals4_probe.c
mtl_portals4_send.c
mcacomponentdir = $(pkglibdir)
mcacomponent_LTLIBRARIES = $(component_install)
@ -52,4 +55,3 @@ noinst_LTLIBRARIES = $(component_noinst)
libmca_mtl_portals4_la_SOURCES = $(local_sources)
libmca_mtl_portals4_la_LIBADD = $(mtl_portals4_LIBS)
libmca_mtl_portals4_la_LDFLAGS = -module -avoid-version $(mtl_portals4_LDFLAGS)

1
ompi/mca/mtl/portals4/configure.m4
Просмотреть файл

@ -11,6 +11,7 @@
# Copyright (c) 2004-2005 The Regents of the University of California.
# All rights reserved.
# Copyright (c) 2010 Cisco Systems, Inc. All rights reserved.
# Copyright (c) 2010 Sandia National Laboratories. All rights reserved.
# $COPYRIGHT$
#
# Additional copyrights may follow

288
ompi/mca/mtl/portals4/mtl_portals4.c
Просмотреть файл

@ -1,5 +1,15 @@
/*
* Copyright (c) 2010 Sandia National Laboratories. All rights reserved.
* Copyright (c) 2004-2006 The Trustees of Indiana University and Indiana
* University Research and Technology
* Corporation. All rights reserved.
* Copyright (c) 2004-2005 The University of Tennessee and The University
* of Tennessee Research Foundation. All rights
* reserved.
* Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
* University of Stuttgart. All rights reserved.
* Copyright (c) 2004-2005 The Regents of the University of California.
* All rights reserved.
* Copyright (c) 2010 Sandia National Laboratories. All rights reserved.
* $COPYRIGHT$
*
* Additional copyrights may follow
@ -13,10 +23,12 @@
#include <portals4_runtime.h>
#include "ompi/mca/mtl/mtl.h"
#include "opal/class/opal_list.h"
#include "mtl_portals4.h"
#include "mtl_portals4_endpoint.h"
#include "mtl_portals4_request.h"
#include "mtl_portals4_recv_short.h"
mca_mtl_portals4_module_t ompi_mtl_portals4 = {
{
@ -29,7 +41,7 @@ mca_mtl_portals4_module_t ompi_mtl_portals4 = {
ompi_mtl_portals4_del_procs,
ompi_mtl_portals4_finalize,
ompi_mtl_portals4_send,
NULL, /* send */
ompi_mtl_portals4_isend,
ompi_mtl_portals4_irecv,
ompi_mtl_portals4_iprobe,
@ -41,27 +53,152 @@ mca_mtl_portals4_module_t ompi_mtl_portals4 = {
int
ompi_mtl_portals4_add_procs(struct mca_mtl_base_module_t *mtl,
size_t nprocs,
struct ompi_proc_t** procs,
struct mca_mtl_base_endpoint_t **mtl_peer_data)
size_t nprocs,
struct ompi_proc_t** procs,
struct mca_mtl_base_endpoint_t **mtl_peer_data)
{
mca_mtl_portals4_module_t *mtl_portals4 = (mca_mtl_portals4_module_t*) mtl;
int i;
struct runtime_proc_t *world_procs;
int world_size = runtime_get_size();
runtime_get_nidpid_map(&world_procs);
if (world_size != (int) nprocs) return OMPI_ERROR;
mtl_portals4->endpoints = malloc(sizeof(struct mca_mtl_base_endpoint_t) * nprocs);
if (NULL == mtl_portals4->endpoints) {
return OMPI_ERROR;
int ret;
ptl_md_t md;
ptl_me_t me;
size_t i;
struct runtime_proc_t *ptlprocs;
int nptlprocs;
ptl_pt_index_t pt;
/* create event queue */
ret = PtlEQAlloc(ompi_mtl_portals4.ni_h,
ompi_mtl_portals4.queue_size,
&(ompi_mtl_portals4.eq_h));
if (PTL_OK != ret) {
opal_output(ompi_mtl_base_output,
"%s:%d: PtlEQAlloc failed: %d\n",
__FILE__, __LINE__, ret);
return ompi_mtl_portals4_get_error(ret);
}
for (i = 0 ; i < (int) nprocs ; ++i) {
mtl_peer_data[i] = &(mtl_portals4->endpoints[i]);
mtl_peer_data[i]->ptl_proc.phys.nid = world_procs[i].nid;
mtl_peer_data[i]->ptl_proc.phys.pid = world_procs[i].pid;
/* Create portal table entries */
ret = PtlPTAlloc(ompi_mtl_portals4.ni_h,
PTL_PT_FLOWCTRL,
ompi_mtl_portals4.eq_h,
PTL_SEND_TABLE_ID,
&pt);
if (PTL_OK != ret) {
PtlEQFree(ompi_mtl_portals4.eq_h);
opal_output(ompi_mtl_base_output,
"%s:%d: PtlPTAlloc failed: %d\n",
__FILE__, __LINE__, ret);
return ompi_mtl_portals4_get_error(ret);
}
ret = PtlPTAlloc(ompi_mtl_portals4.ni_h,
PTL_PT_FLOWCTRL,
ompi_mtl_portals4.eq_h,
PTL_READ_TABLE_ID,
&pt);
if (PTL_OK != ret) {
PtlPTFree(ompi_mtl_portals4.ni_h, PTL_SEND_TABLE_ID);
PtlEQFree(ompi_mtl_portals4.eq_h);
opal_output(ompi_mtl_base_output,
"%s:%d: PtlPTAlloc failed: %d\n",
__FILE__, __LINE__, ret);
return ompi_mtl_portals4_get_error(ret);
}
/* bind zero-length md for sending acks */
md.start = NULL;
md.length = 0;
md.options = 0;
md.eq_handle = PTL_EQ_NONE;
md.ct_handle = PTL_CT_NONE;
ret = PtlMDBind(ompi_mtl_portals4.ni_h,
&md,
&ompi_mtl_portals4.zero_md_h );
if (PTL_OK != ret) {
PtlPTFree(ompi_mtl_portals4.ni_h, PTL_READ_TABLE_ID);
PtlPTFree(ompi_mtl_portals4.ni_h, PTL_SEND_TABLE_ID);
PtlEQFree(ompi_mtl_portals4.eq_h);
opal_output(ompi_mtl_base_output,
"%s:%d: PtlMDBind failed: %d\n",
__FILE__, __LINE__, ret);
return ompi_mtl_portals4_get_error(ret);
}
/* Handle long overflows */
me.start = NULL;
me.length = 0;
me.ct_handle = PTL_CT_NONE;
me.min_free = 0;
me.ac_id.uid = PTL_UID_ANY;
me.options = PTL_ME_OP_PUT;
me.match_id.phys.nid = PTL_NID_ANY;
me.match_id.phys.pid = PTL_PID_ANY;
me.match_bits = PTL_LONG_MSG;
me.ignore_bits = PTL_CONTEXT_MASK | PTL_SOURCE_MASK | PTL_TAG_MASK;
ompi_mtl_portals4.long_overflow_request.event_callback =
ompi_mtl_portals4_recv_progress;
ret = PtlMEAppend(ompi_mtl_portals4.ni_h,
PTL_SEND_TABLE_ID,
&me,
PTL_OVERFLOW,
&ompi_mtl_portals4.long_overflow_request,
&ompi_mtl_portals4.long_overflow_me_h);
if (PTL_OK != ret) {
PtlMDRelease(ompi_mtl_portals4.zero_md_h);
PtlPTFree(ompi_mtl_portals4.ni_h, PTL_READ_TABLE_ID);
PtlPTFree(ompi_mtl_portals4.ni_h, PTL_SEND_TABLE_ID);
PtlEQFree(ompi_mtl_portals4.eq_h);
opal_output(ompi_mtl_base_output,
"%s:%d: PtlMEAppend failed: %d\n",
__FILE__, __LINE__, ret);
return ompi_mtl_portals4_get_error(ret);
}
/* attach short unex recv blocks */
ret = ompi_mtl_portals4_recv_short_init((mca_mtl_portals4_module_t*) mtl);
if (OMPI_SUCCESS != ret) {
PtlMEUnlink(ompi_mtl_portals4.long_overflow_me_h);
PtlMDRelease(ompi_mtl_portals4.zero_md_h);
PtlPTFree(ompi_mtl_portals4.ni_h, PTL_READ_TABLE_ID);
PtlPTFree(ompi_mtl_portals4.ni_h, PTL_SEND_TABLE_ID);
PtlEQFree(ompi_mtl_portals4.eq_h);
opal_output(ompi_mtl_base_output,
"%s:%d: short receive block initialization failed: %d\n",
__FILE__, __LINE__, ret);
return ret;
}
/* activate progress callback */
opal_progress_register(ompi_mtl_portals4_progress);
/* Get the list of ptl_process_id_t from the runtime and copy into structure */
nptlprocs = runtime_get_nidpid_map(&ptlprocs);
if ((size_t)nptlprocs != nprocs) {
PtlMEUnlink(ompi_mtl_portals4.long_overflow_me_h);
PtlMDRelease(ompi_mtl_portals4.zero_md_h);
PtlPTFree(ompi_mtl_portals4.ni_h, PTL_READ_TABLE_ID);
PtlPTFree(ompi_mtl_portals4.ni_h, PTL_SEND_TABLE_ID);
PtlEQFree(ompi_mtl_portals4.eq_h);
opal_output(ompi_mtl_base_output,
"%s:%d: nptlprocs != nprocs: %d\n",
__FILE__, __LINE__, ret);
return OMPI_ERR_NOT_SUPPORTED;
}
for (i = 0 ; i < nprocs ; ++i) {
mtl_peer_data[i] = malloc(sizeof(struct mca_mtl_base_endpoint_t));
if (NULL == mtl_peer_data[i]) {
PtlMEUnlink(ompi_mtl_portals4.long_overflow_me_h);
PtlMDRelease(ompi_mtl_portals4.zero_md_h);
PtlPTFree(ompi_mtl_portals4.ni_h, PTL_READ_TABLE_ID);
PtlPTFree(ompi_mtl_portals4.ni_h, PTL_SEND_TABLE_ID);
PtlEQFree(ompi_mtl_portals4.eq_h);
opal_output(ompi_mtl_base_output,
"%s:%d: malloc failed: %d\n",
__FILE__, __LINE__, ret);
return OMPI_ERROR;
}
mtl_peer_data[i]->ptl_proc.phys.nid = ptlprocs[i].nid;
mtl_peer_data[i]->ptl_proc.phys.pid = ptlprocs[i].pid;
}
return OMPI_SUCCESS;
@ -70,10 +207,18 @@ ompi_mtl_portals4_add_procs(struct mca_mtl_base_module_t *mtl,
int
ompi_mtl_portals4_del_procs(struct mca_mtl_base_module_t *mtl,
size_t nprocs,
struct ompi_proc_t** procs,
struct mca_mtl_base_endpoint_t **mtl_peer_data)
size_t nprocs,
struct ompi_proc_t** procs,
struct mca_mtl_base_endpoint_t **mtl_peer_data)
{
size_t i;
for (i = 0 ; i < nprocs ; ++i) {
if (NULL != mtl_peer_data[i]) {
free(mtl_peer_data[i]);
}
}
return OMPI_SUCCESS;
}
@ -81,8 +226,103 @@ ompi_mtl_portals4_del_procs(struct mca_mtl_base_module_t *mtl,
int
ompi_mtl_portals4_finalize(struct mca_mtl_base_module_t *mtl)
{
PtlNIFini(ompi_mtl_portals4.ptl_ni_h);
opal_progress_unregister(ompi_mtl_portals4_progress);
while (0 != ompi_mtl_portals4_progress()) { }
ompi_mtl_portals4_recv_short_fini(&ompi_mtl_portals4);
PtlMEUnlink(ompi_mtl_portals4.long_overflow_me_h);
PtlMDRelease(ompi_mtl_portals4.zero_md_h);
PtlEQFree(ompi_mtl_portals4.eq_h);
PtlPTFree(ompi_mtl_portals4.ni_h, PTL_READ_TABLE_ID);
PtlPTFree(ompi_mtl_portals4.ni_h, PTL_SEND_TABLE_ID);
PtlNIFini(ompi_mtl_portals4.ni_h);
PtlFini();
return OMPI_SUCCESS;
}
int
ompi_mtl_portals4_progress(void)
{
int count = 0, ret;
ptl_event_t ev;
ompi_mtl_portals4_request_t *ptl_request;
while (true) {
ret = PtlEQGet(ompi_mtl_portals4.eq_h, &ev);
if (PTL_OK == ret) {
OPAL_OUTPUT_VERBOSE((ompi_mtl_base_output, 50,
"Found event of type %d\n", ev.type));
switch (ev.type) {
case PTL_EVENT_GET:
case PTL_EVENT_PUT:
case PTL_EVENT_PUT_OVERFLOW:
case PTL_EVENT_ATOMIC:
case PTL_EVENT_ATOMIC_OVERFLOW:
if (NULL != ev.user_ptr) {
ptl_request = ev.user_ptr;
ret = ptl_request->event_callback(&ev, ptl_request);
if (OMPI_SUCCESS != ret) {
opal_output(ompi_mtl_base_output,
"Error returned from target event callback: %d", ret);
abort();
}
}
break;
case PTL_EVENT_REPLY:
case PTL_EVENT_SEND:
case PTL_EVENT_ACK:
if (NULL != ev.user_ptr) {
ptl_request = ev.user_ptr;
ret = ptl_request->event_callback(&ev, ptl_request);
if (OMPI_SUCCESS != ret) {
opal_output(ompi_mtl_base_output,
"Error returned from initiator event callback: %d", ret);
abort();
}
}
break;
case PTL_EVENT_DROPPED:
case PTL_EVENT_PT_DISABLED:
/* do stuff - flow control */
opal_output(ompi_mtl_base_output, "Unhandled flow control event.");
abort();
break;
case PTL_EVENT_AUTO_UNLINK:
break;
case PTL_EVENT_AUTO_FREE:
if (OMPI_SUCCESS != (ret = ompi_mtl_portals4_recv_short_block_repost(&ev))) {
opal_output(ompi_mtl_base_output,
"Error returned from PTL_EVENT_FREE callback: %d", ret);
abort();
}
break;
case PTL_EVENT_PROBE:
if (NULL != ev.user_ptr) {
ptl_request = ev.user_ptr;
ret = ptl_request->event_callback(&ev, ptl_request);
if (OMPI_SUCCESS != ret) {
opal_output(ompi_mtl_base_output,
"Error returned from target event callback: %d", ret);
abort();
}
}
break;
default:
opal_output(ompi_mtl_base_output,
"Unknown event type %d (error: %d)", (int)ev.type, ret);
abort();
}
} else if (PTL_EQ_EMPTY == ret) {
break;
} else {
opal_output(ompi_mtl_base_output,
"Error returned from PtlEQGet: %d", ret);
abort();
}
}
return count;
}

121
ompi/mca/mtl/portals4/mtl_portals4.h
Просмотреть файл

@ -1,5 +1,15 @@
/*
* Copyright (c) 2010 Sandia National Laboratories. All rights reserved.
* Copyright (c) 2004-2006 The Trustees of Indiana University and Indiana
* University Research and Technology
* Corporation. All rights reserved.
* Copyright (c) 2004-2007 The University of Tennessee and The University
* of Tennessee Research Foundation. All rights
* reserved.
* Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
* University of Stuttgart. All rights reserved.
* Copyright (c) 2004-2005 The Regents of the University of California.
* All rights reserved.
* Copyright (c) 2010 Sandia National Laboratories. All rights reserved.
* $COPYRIGHT$
*
* Additional copyrights may follow
@ -7,20 +17,42 @@
* $HEADER$
*/
#ifndef MTL_PORTALS4_H
#define MTL_PORTALS4_H
#ifndef MTL_PORTALS_H_HAS_BEEN_INCLUDED
#define MTL_PORTALS_H_HAS_BEEN_INCLUDED
#include <portals4.h>
#include "ompi_config.h"
#include "opal/class/opal_list.h"
#include "ompi/class/ompi_free_list.h"
#include "ompi/mca/mtl/mtl.h"
#include "ompi/mca/mtl/base/base.h"
#include "opal/datatype/opal_convertor.h"
#include "mtl_portals4_endpoint.h"
#include "mtl_portals4_request.h"
BEGIN_C_DECLS
struct mca_mtl_portals4_module_t {
mca_mtl_base_module_t base;
ptl_handle_ni_t ptl_ni_h;
mca_mtl_base_endpoint_t *endpoints;
/* configuration */
size_t eager_limit;
size_t recv_short_size;
int recv_short_num;
int queue_size;
/* global handles */
ptl_handle_ni_t ni_h;
ptl_handle_eq_t eq_h;
/* for zero-length sends and acks */
ptl_handle_md_t zero_md_h;
/* long message receive overflow */
ptl_handle_me_t long_overflow_me_h;
ompi_mtl_portals4_request_t long_overflow_request;
opal_list_t recv_short_blocks;
};
typedef struct mca_mtl_portals4_module_t mca_mtl_portals4_module_t;
@ -28,6 +60,9 @@ extern mca_mtl_portals4_module_t ompi_mtl_portals4;
OMPI_DECLSPEC extern mca_mtl_base_component_2_0_0_t mca_mtl_portals4_component;
#define PTL_SEND_TABLE_ID 2
#define PTL_READ_TABLE_ID 4
/* match/ignore bit manipulation
*
@ -37,6 +72,7 @@ OMPI_DECLSPEC extern mca_mtl_base_component_2_0_0_t mca_mtl_portals4_component;
* | | | |
* +---- protocol
*/
#define PTL_PROTOCOL_MASK 0xF000000000000000ULL
#define PTL_CONTEXT_MASK 0x0FFF000000000000ULL
#define PTL_SOURCE_MASK 0x0000FFFF00000000ULL
@ -53,37 +89,37 @@ OMPI_DECLSPEC extern mca_mtl_base_component_2_0_0_t mca_mtl_portals4_component;
/* send posting */
#define PTL_SET_SEND_BITS(match_bits, contextid, source, tag, type) \
{ \
match_bits = contextid; \
match_bits = (match_bits << 16); \
match_bits |= source; \
match_bits = (match_bits << 32); \
match_bits |= (PTL_TAG_MASK & tag) | type; \
}
{ \
match_bits = contextid; \
match_bits = (match_bits << 16); \
match_bits |= source; \
match_bits = (match_bits << 32); \
match_bits |= (PTL_TAG_MASK & tag) | type; \
}
/* receive posting */
#define PTL_SET_RECV_BITS(match_bits, ignore_bits, contextid, source, tag) \
{ \
match_bits = 0; \
ignore_bits = PTL_PROTOCOL_IGNR; \
\
match_bits = contextid; \
match_bits = (match_bits << 16); \
\
if (MPI_ANY_SOURCE == source) { \
match_bits = (match_bits << 32); \
ignore_bits |= PTL_SOURCE_IGNR; \
} else { \
match_bits |= source; \
match_bits = (match_bits << 32); \
} \
\
if (MPI_ANY_TAG == tag) { \
ignore_bits |= PTL_TAG_IGNR; \
} else { \
match_bits |= (PTL_TAG_MASK & tag); \
} \
}
{ \
match_bits = 0; \
ignore_bits = PTL_PROTOCOL_IGNR; \
\
match_bits = contextid; \
match_bits = (match_bits << 16); \
\
if (MPI_ANY_SOURCE == source) { \
match_bits = (match_bits << 32); \
ignore_bits |= PTL_SOURCE_IGNR; \
} else { \
match_bits |= source; \
match_bits = (match_bits << 32); \
} \
\
if (MPI_ANY_TAG == tag) { \
ignore_bits |= PTL_TAG_IGNR; \
} else { \
match_bits |= (PTL_TAG_MASK & tag); \
} \
}
#define PTL_IS_SHORT_MSG(match_bits) \
(0 != (PTL_SHORT_MSG & match_bits))
@ -91,8 +127,8 @@ OMPI_DECLSPEC extern mca_mtl_base_component_2_0_0_t mca_mtl_portals4_component;
(0 != (PTL_LONG_MSG & match_bits))
#define PTL_IS_READY_MSG(match_bits) \
(0 != (PTL_READY_MSG & match_bits))
#define PTL_IS_SYNC_MSG(event) \
(0 != event.hdr_data)
#define PTL_IS_SYNC_MSG(ev) \
(0 != ev->hdr_data)
#define PTL_GET_TAG(match_bits) ((int)(match_bits & PTL_TAG_MASK))
#define PTL_GET_SOURCE(match_bits) ((int)((match_bits & PTL_SOURCE_MASK) >> 32))
@ -110,13 +146,6 @@ extern int ompi_mtl_portals4_del_procs(struct mca_mtl_base_module_t* mtl,
struct ompi_proc_t** procs,
struct mca_mtl_base_endpoint_t **mtl_peer_data);
extern int ompi_mtl_portals4_send(struct mca_mtl_base_module_t* mtl,
struct ompi_communicator_t* comm,
int dest,
int tag,
struct opal_convertor_t *convertor,
mca_pml_base_send_mode_t mode);
extern int ompi_mtl_portals4_isend(struct mca_mtl_base_module_t* mtl,
struct ompi_communicator_t* comm,
int dest,
@ -144,6 +173,10 @@ extern int ompi_mtl_portals4_cancel(struct mca_mtl_base_module_t* mtl,
mca_mtl_request_t *mtl_request,
int flag);
extern int ompi_mtl_portals4_progress(void);
extern int ompi_mtl_portals4_get_error(int ptl_error);
END_C_DECLS
#endif /* MTL_PORTALS4_H_HAS_BEEN_INCLUDED */
#endif /* MTL_PORTALS_H_HAS_BEEN_INCLUDED */

172
ompi/mca/mtl/portals4/mtl_portals4_component.c
Просмотреть файл

@ -1,5 +1,15 @@
/*
* Copyright (c) 2010 Sandia National Laboratories. All rights reserved.
* Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
* University Research and Technology
* Corporation. All rights reserved.
* Copyright (c) 2004-2005 The University of Tennessee and The University
* of Tennessee Research Foundation. All rights
* reserved.
* Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
* University of Stuttgart. All rights reserved.
* Copyright (c) 2004-2005 The Regents of the University of California.
* All rights reserved.
* Copyright (c) 2010 Sandia National Laboratories. All rights reserved.
* $COPYRIGHT$
*
* Additional copyrights may follow
@ -9,34 +19,43 @@
#include "ompi_config.h"
#include "opal/mca/event/event.h"
#include "opal/util/output.h"
#include "opal/mca/base/mca_base_param.h"
#include "mtl_portals4.h"
#include "mtl_portals4_request.h"
static int ompi_mtl_portals4_component_open(void);
static int ompi_mtl_portals4_component_close(void);
static mca_mtl_base_module_t* ompi_mtl_portals4_component_init(bool enable_progress_threads,
bool enable_mpi_threads);
static mca_mtl_base_module_t* ompi_mtl_portals4_component_init(
bool enable_progress_threads, bool enable_mpi_threads);
mca_mtl_base_component_2_0_0_t mca_mtl_portals4_component = {
{
MCA_MTL_BASE_VERSION_2_0_0,
"portals4", /* MCA component name */
OMPI_MAJOR_VERSION, /* MCA component major version */
OMPI_MINOR_VERSION, /* MCA component minor version */
OMPI_RELEASE_VERSION, /* MCA component release version */
ompi_mtl_portals4_component_open, /* component open */
ompi_mtl_portals4_component_close /* component close */
},
{
/* The component is not checkpoint ready */
MCA_BASE_METADATA_PARAM_NONE
},
ompi_mtl_portals4_component_init, /* component init */
/* First, the mca_base_component_t struct containing meta
* information about the component itself */
{
MCA_MTL_BASE_VERSION_2_0_0,
"portals4", /* MCA component name */
OMPI_MAJOR_VERSION, /* MCA component major version */
OMPI_MINOR_VERSION, /* MCA component minor version */
OMPI_RELEASE_VERSION, /* MCA component release version */
ompi_mtl_portals4_component_open, /* component open */
ompi_mtl_portals4_component_close /* component close */
},
{
/* The component is not checkpoint ready */
MCA_BASE_METADATA_PARAM_NONE
},
ompi_mtl_portals4_component_init, /* component init */
};
static int
ompi_mtl_portals4_component_open(void)
{
@ -46,9 +65,43 @@ ompi_mtl_portals4_component_open(void)
sizeof(ompi_mtl_portals4_request_t) -
sizeof(struct mca_mtl_request_t);
ompi_mtl_portals4.ptl_ni_h = PTL_INVALID_HANDLE;
mca_base_param_reg_int(&mca_mtl_portals4_component.mtl_version,
"eager_limit",
"Cross-over point from eager to rendezvous sends",
false,
false,
32 * 1024,
&tmp);
ompi_mtl_portals4.eager_limit = tmp;
return OMPI_SUCCESS;
mca_base_param_reg_int(&mca_mtl_portals4_component.mtl_version,
"short_recv_num",
"Number of short message receive blocks",
false,
false,
8,
&ompi_mtl_portals4.recv_short_num);
mca_base_param_reg_int(&mca_mtl_portals4_component.mtl_version,
"short_recv_size",
"Size of short message receive blocks",
false,
false,
1024 * 1024,
&tmp);
ompi_mtl_portals4.recv_short_size = tmp;
mca_base_param_reg_int(&mca_mtl_portals4_component.mtl_version,
"queue_size",
"Size of the event queue in entries",
false,
false,
1024,
&ompi_mtl_portals4.queue_size);
ompi_mtl_portals4.ni_h = PTL_INVALID_HANDLE;
return ompi_mtl_portals4_get_error(PtlInit());
}
@ -63,15 +116,82 @@ static mca_mtl_base_module_t*
ompi_mtl_portals4_component_init(bool enable_progress_threads,
bool enable_mpi_threads)
{
if (PTL_OK != PtlInit()) {
return NULL;
}
if (PTL_OK != PtlNIInit(PTL_IFACE_DEFAULT, PTL_NI_MATCHING|PTL_NI_PHYSICAL,
PTL_PID_ANY, NULL, NULL, 0, NULL, NULL,
&ompi_mtl_portals4.ptl_ni_h)) {
if (PTL_OK != PtlNIInit(PTL_IFACE_DEFAULT,
PTL_NI_PHYSICAL | PTL_NI_MATCHING,
PTL_PID_ANY,
NULL,
NULL,
0,
NULL,
NULL,
&ompi_mtl_portals4.ni_h)) {
return NULL;
}
return &ompi_mtl_portals4.base;
}
int
ompi_mtl_portals4_get_error(int ptl_error)
{
int ret;
switch (ptl_error) {
case PTL_OK:
ret = OMPI_SUCCESS;
break;
case PTL_ARG_INVALID:
ret = OMPI_ERR_BAD_PARAM;
break;
case PTL_CT_NONE_REACHED:
ret = OMPI_ERR_TIMEOUT;
break;
case PTL_EQ_DROPPED:
ret = OMPI_ERR_OUT_OF_RESOURCE;
break;
case PTL_EQ_EMPTY:
ret = OMPI_ERR_TEMP_OUT_OF_RESOURCE;
break;
case PTL_FAIL:
ret = OMPI_ERROR;
break;
case PTL_IN_USE:
ret = OMPI_ERR_RESOURCE_BUSY;
break;
case PTL_INTERRUPTED:
ret = OMPI_ERR_RESOURCE_BUSY;
break;
case PTL_LIST_TOO_LONG:
ret = OMPI_ERR_OUT_OF_RESOURCE;
break;
case PTL_NI_NOT_LOGICAL:
ret = OMPI_ERR_FATAL;
break;
case PTL_NO_INIT:
ret = OMPI_ERR_FATAL;
break;
case PTL_NO_SPACE:
ret = OMPI_ERR_OUT_OF_RESOURCE;
break;
case PTL_PID_IN_USE:
ret = OMPI_ERR_BAD_PARAM;
break;
case PTL_PT_FULL:
ret = OMPI_ERR_OUT_OF_RESOURCE;
break;
case PTL_PT_EQ_NEEDED:
ret = OMPI_ERR_FATAL;
break;
case PTL_PT_IN_USE:
ret = OMPI_ERR_RESOURCE_BUSY;
break;
case PTL_SIZE_INVALID:
ret = OMPI_ERR_BAD_PARAM;
break;
default:
ret = OMPI_ERROR;
}
return ret;
}

18
ompi/mca/mtl/portals4/mtl_portals4_endpoint.h
Просмотреть файл

@ -1,5 +1,15 @@
/*
* Copyright (c) 2010 Sandia National Laboratories. All rights reserved.
* Copyright (c) 2004-2006 The Trustees of Indiana University and Indiana
* University Research and Technology
* Corporation. All rights reserved.
* Copyright (c) 2004-2005 The University of Tennessee and The University
* of Tennessee Research Foundation. All rights
* reserved.
* Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
* University of Stuttgart. All rights reserved.
* Copyright (c) 2004-2005 The Regents of the University of California.
* All rights reserved.
* Copyright (c) 2010 Sandia National Laboratories. All rights reserved.
* $COPYRIGHT$
*
* Additional copyrights may follow
@ -7,10 +17,8 @@
* $HEADER$
*/
#ifndef OMPI_MTL_PORTALS4_ENDPOINT_H
#define OMPI_MTL_PORTALS4_ENDPOINT_H
#include <portals4.h>
#ifndef OMPI_MTL_PORTALS_ENDPOINT_H
#define OMPI_MTL_PORTALS_ENDPOINT_H
struct mca_mtl_base_endpoint_t {
ptl_process_t ptl_proc;

21
ompi/mca/mtl/portals4/mtl_portals4_probe.c
Просмотреть файл

@ -1,5 +1,15 @@
/*
* Copyright (c) 2010 Sandia National Laboratories. All rights reserved.
* Copyright (c) 2004-2006 The Trustees of Indiana University and Indiana
* University Research and Technology
* Corporation. All rights reserved.
* Copyright (c) 2004-2010 The University of Tennessee and The University
* of Tennessee Research Foundation. All rights
* reserved.
* Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
* University of Stuttgart. All rights reserved.
* Copyright (c) 2004-2005 The Regents of the University of California.
* All rights reserved.
* Copyright (c) 2010 Sandia National Laboratories. All rights reserved.
* $COPYRIGHT$
*
* Additional copyrights may follow
@ -8,8 +18,9 @@
*/
#include "ompi_config.h"
#include "ompi/communicator/communicator.h"
#include "mtl_portals4.h"
#include "mtl_portals4_request.h"
int
ompi_mtl_portals4_iprobe(struct mca_mtl_base_module_t* mtl,
@ -19,5 +30,9 @@ ompi_mtl_portals4_iprobe(struct mca_mtl_base_module_t* mtl,
int *flag,
struct ompi_status_public_t *status)
{
return OMPI_SUCCESS;
/* BWB: FIX ME: implement */
opal_output(ompi_mtl_base_output,
"%s:%d: iprobe failed: not implemented\n",
__FILE__, __LINE__);
return OMPI_ERR_NOT_IMPLEMENTED;
}

294
ompi/mca/mtl/portals4/mtl_portals4_recv.c
Просмотреть файл

@ -1,5 +1,15 @@
/*
* Copyright (c) 2010 Sandia National Laboratories. All rights reserved.
* Copyright (c) 2004-2006 The Trustees of Indiana University and Indiana
* University Research and Technology
* Corporation. All rights reserved.
* Copyright (c) 2004-2010 The University of Tennessee and The University
* of Tennessee Research Foundation. All rights
* reserved.
* Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
* University of Stuttgart. All rights reserved.
* Copyright (c) 2004-2005 The Regents of the University of California.
* All rights reserved.
* Copyright (c) 2010 Sandia National Laboratories. All rights reserved.
* $COPYRIGHT$
*
* Additional copyrights may follow
@ -7,9 +17,226 @@
* $HEADER$
*/
#include "ompi_config.h"
#include "opal/class/opal_list.h"
#include "ompi/communicator/communicator.h"
#include "ompi/datatype/ompi_datatype.h"
#include "opal/datatype/opal_convertor.h"
#include "ompi/mca/mtl/base/base.h"
#include "ompi/mca/mtl/base/mtl_base_datatype.h"
#include "mtl_portals4.h"
#include "mtl_portals4_endpoint.h"
#include "mtl_portals4_request.h"
#include "mtl_portals4_recv_short.h"
/* called when a receive should be progressed */
int
ompi_mtl_portals4_recv_progress(ptl_event_t *ev,
ompi_mtl_portals4_request_t* ptl_request)
{
int ret;
switch (ev->type) {
case PTL_EVENT_PUT:
if (ev->ni_fail_type == PTL_NI_OK) {
/* make sure the data is in the right place */
ret = ompi_mtl_datatype_unpack(ptl_request->convertor,
ev->start,
ev->mlength);
if (OMPI_SUCCESS != ret) {
opal_output_verbose(ompi_mtl_base_output, 1,
"%s:%d: ompi_mtl_datatype_unpack failed: %d",
__FILE__, __LINE__, ret);
ptl_request->super.ompi_req->req_status.MPI_ERROR = ret;
}
/* set the status */
ptl_request->super.ompi_req->req_status.MPI_SOURCE =
PTL_GET_SOURCE(ev->match_bits);
ptl_request->super.ompi_req->req_status.MPI_TAG =
PTL_GET_TAG(ev->match_bits);
if (ev->rlength > ev->mlength) {
ptl_request->super.ompi_req->req_status.MPI_ERROR = MPI_ERR_TRUNCATE;
}
ptl_request->super.ompi_req->req_status._ucount =
ev->mlength;
} else {
opal_output_verbose(ompi_mtl_base_output, 1,
"%s:%d: recv(PTL_EVENT_PUT) ni_fail_type: %d",
__FILE__, __LINE__, ret);
ptl_request->super.ompi_req->req_status.MPI_ERROR = OMPI_ERROR;
}
ptl_request->super.completion_callback(&ptl_request->super);
break;
case PTL_EVENT_REPLY:
if (ev->ni_fail_type == PTL_NI_OK) {
/* make sure the data is in the right place */
ret = ompi_mtl_datatype_unpack(ptl_request->convertor,
ev->start,
ev->mlength);
if (OMPI_SUCCESS != ret) {
opal_output_verbose(ompi_mtl_base_output, 1,
"%s:%d: ompi_mtl_datatype_unpack failed: %d",
__FILE__, __LINE__, ret);
ptl_request->super.ompi_req->req_status.MPI_ERROR = ret;
}
/* set the status - most of this filled in right after issuing
the PtlGet */
ptl_request->super.ompi_req->req_status._ucount =
ev->mlength;
} else {
opal_output_verbose(ompi_mtl_base_output, 1,
"%s:%d: recv(PTL_EVENT_REPLY) ni_fail_type: %d",
__FILE__, __LINE__, ret);
ptl_request->super.ompi_req->req_status.MPI_ERROR = OMPI_ERROR;
}
PtlMDRelease(ptl_request->md_h);
ptl_request->super.completion_callback(&ptl_request->super);
break;
case PTL_EVENT_PUT_OVERFLOW:
/* overflow case. Short messages have the buffer stashed
somewhere. Long messages left in buffer at the source */
if (PTL_IS_SHORT_MSG(ev->match_bits)) {
if (ev->ni_fail_type == PTL_NI_OK) {
ptl_request->super.ompi_req->req_status.MPI_SOURCE =
PTL_GET_SOURCE(ev->match_bits);
ptl_request->super.ompi_req->req_status.MPI_TAG =
PTL_GET_TAG(ev->match_bits);
if (ev->rlength > ptl_request->delivery_len) {
ptl_request->super.ompi_req->req_status.MPI_ERROR = MPI_ERR_TRUNCATE;
}
ptl_request->super.ompi_req->req_status._ucount =
ev->mlength;
if (ev->mlength > 0) {
struct iovec iov;
uint32_t iov_count = 1;
size_t max_data;
iov.iov_base = (char*) ev->start;
iov.iov_len = ev->mlength;
max_data = iov.iov_len;
ret = opal_convertor_unpack(ptl_request->convertor,
&iov, &iov_count,
&max_data );
if (OMPI_SUCCESS != ret) {
opal_output_verbose(ompi_mtl_base_output, 1,
"%s:%d: opal_convertor_unpack failed: %d",
__FILE__, __LINE__, ret);
if (NULL != ptl_request->buffer_ptr) free(ptl_request->buffer_ptr);
ptl_request->super.ompi_req->req_status.MPI_ERROR = ret;
ptl_request->super.completion_callback(&ptl_request->super);
return OMPI_SUCCESS;
}
}
if (NULL != ptl_request->buffer_ptr) free(ptl_request->buffer_ptr);
/* if it's a sync, send the ack */
if (PTL_IS_SYNC_MSG(ev)) {
ret = PtlPut(ompi_mtl_portals4.zero_md_h,
0,
0,
PTL_NO_ACK_REQ,
ev->initiator,
PTL_READ_TABLE_ID,
ev->hdr_data,
0,
NULL,
0);
if (PTL_OK != ret) {
opal_output_verbose(ompi_mtl_base_output, 1,
"%s:%d: PtlPut failed: %d",
__FILE__, __LINE__, ret);
ptl_request->super.ompi_req->req_status.MPI_ERROR =
ompi_mtl_portals4_get_error(ret);;
ptl_request->super.completion_callback(&ptl_request->super);
return OMPI_SUCCESS;
}
}
} else {
opal_output_verbose(ompi_mtl_base_output, 1,
"%s:%d: recv(PTL_EVENT_PUT_OVERFLOW) ni_fail_type: %d",
__FILE__, __LINE__, ret);
ptl_request->super.ompi_req->req_status.MPI_ERROR = OMPI_ERROR;
}
ptl_request->super.completion_callback(&ptl_request->super);
} else {
ptl_md_t md;
if (ev->ni_fail_type == PTL_NI_OK) {
/* set the status */
ptl_request->super.ompi_req->req_status.MPI_SOURCE =
PTL_GET_SOURCE(ev->match_bits);
ptl_request->super.ompi_req->req_status.MPI_TAG =
PTL_GET_TAG(ev->match_bits);
if (ev->rlength > ptl_request->delivery_len) {
ptl_request->super.ompi_req->req_status.MPI_ERROR = MPI_ERR_TRUNCATE;
}
md.start = ptl_request->delivery_ptr;
md.length = (ev->rlength > ptl_request->delivery_len) ?
ptl_request->delivery_len : ev->rlength;
md.options = 0;
md.eq_handle = ompi_mtl_portals4.eq_h;
md.ct_handle = PTL_CT_NONE;
ret = PtlMDBind(ompi_mtl_portals4.ni_h,
&md,
&ptl_request->md_h);
if (PTL_OK != ret) {
if (NULL != ptl_request->buffer_ptr) free(ptl_request->buffer_ptr);
opal_output_verbose(ompi_mtl_base_output, 1,
"%s:%d: PtlMDBind failed: %d",
__FILE__, __LINE__, ret);
ptl_request->super.ompi_req->req_status.MPI_ERROR =
ompi_mtl_portals4_get_error(ret);;
ptl_request->super.completion_callback(&ptl_request->super);
return OMPI_SUCCESS;
}
ret = PtlGet(ompi_mtl_portals4.ni_h,
0,
md.length,
ev->initiator,
PTL_READ_TABLE_ID,
ev->hdr_data,
ptl_request,
0);
if (PTL_OK != ret) {
PtlMDRelease(ptl_request->md_h);
if (NULL != ptl_request->buffer_ptr) free(ptl_request->buffer_ptr);
opal_output_verbose(ompi_mtl_base_output, 1,
"%s:%d: PtlGet failed: %d",
__FILE__, __LINE__, ret);
ptl_request->super.ompi_req->req_status.MPI_ERROR =
ompi_mtl_portals4_get_error(ret);;
ptl_request->super.completion_callback(&ptl_request->super);
return OMPI_SUCCESS;
}
} else {
opal_output_verbose(ompi_mtl_base_output, 1,
"%s:%d: recv(PTL_EVENT_PUT_OVERFLOW) ni_fail_type: %d",
__FILE__, __LINE__, ret);
ptl_request->super.ompi_req->req_status.MPI_ERROR = OMPI_ERROR;
ptl_request->super.completion_callback(&ptl_request->super);
}
}
break;
default:
opal_output(ompi_mtl_base_output,
"Unhandled receive callback with event type %d",
ev->type);
return OMPI_ERROR;
}
return OMPI_SUCCESS;
}
int
ompi_mtl_portals4_irecv(struct mca_mtl_base_module_t* mtl,
@ -19,6 +246,67 @@ ompi_mtl_portals4_irecv(struct mca_mtl_base_module_t* mtl,
struct opal_convertor_t *convertor,
mca_mtl_request_t *mtl_request)
{
return OMPI_SUCCESS;
}
ptl_match_bits_t match_bits, ignore_bits;
int ret = OMPI_SUCCESS;
ptl_process_t remote_proc;
mca_mtl_base_endpoint_t *endpoint = NULL;
ompi_mtl_portals4_request_t *ptl_request =
(ompi_mtl_portals4_request_t*) mtl_request;
void *start;
size_t length;
bool free_after;
ptl_me_t me;
if (MPI_ANY_SOURCE == src) {
remote_proc.phys.nid = PTL_NID_ANY;
remote_proc.phys.pid = PTL_PID_ANY;
} else {
ompi_proc_t* ompi_proc = ompi_comm_peer_lookup( comm, src );
endpoint = (mca_mtl_base_endpoint_t*) ompi_proc->proc_pml;
remote_proc = endpoint->ptl_proc;
}
PTL_SET_RECV_BITS(match_bits, ignore_bits, comm->c_contextid,
src, tag);
ret = ompi_mtl_datatype_recv_buf(convertor, &start, &length, &free_after);
if (OMPI_SUCCESS != ret) {
opal_output_verbose(ompi_mtl_base_output, 1,
"%s:%d: PtlMEAppend failed: %d",
__FILE__, __LINE__, ret);
return ret;
}
ptl_request->event_callback = ompi_mtl_portals4_recv_progress;
ptl_request->buffer_ptr = (free_after) ? start : NULL;
ptl_request->convertor = convertor;
ptl_request->delivery_ptr = start;
ptl_request->delivery_len = length;
ptl_request->super.ompi_req->req_status.MPI_ERROR = OMPI_SUCCESS;
me.start = start;
me.length = length;
me.ct_handle = PTL_CT_NONE;
me.min_free = 0;
me.ac_id.uid = PTL_UID_ANY;
me.options = PTL_ME_OP_PUT | PTL_ME_USE_ONCE | PTL_ME_EVENT_UNLINK_DISABLE;
me.match_id = remote_proc;
me.match_bits = match_bits;
me.ignore_bits = ignore_bits;
ret = PtlMEAppend(ompi_mtl_portals4.ni_h,
PTL_SEND_TABLE_ID,
&me,
PTL_PRIORITY_LIST,
ptl_request,
&ptl_request->me_h);
if (PTL_OK != ret) {
if (NULL != ptl_request->buffer_ptr) free(ptl_request->buffer_ptr);
opal_output_verbose(ompi_mtl_base_output, 1,
"%s:%d: PtlMEAppend failed: %d",
__FILE__, __LINE__, ret);
return ompi_mtl_portals4_get_error(ret);
}
return OMPI_SUCCESS;
}

142
ompi/mca/mtl/portals4/mtl_portals4_recv_short.c Обычный файл
Просмотреть файл

@ -0,0 +1,142 @@
/*
* Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
* University Research and Technology
* Corporation. All rights reserved.
* Copyright (c) 2004-2005 The University of Tennessee and The University
* of Tennessee Research Foundation. All rights
* reserved.
* Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
* University of Stuttgart. All rights reserved.
* Copyright (c) 2004-2005 The Regents of the University of California.
* All rights reserved.
* Copyright (c) 2010 Sandia National Laboratories. All rights reserved.
* $COPYRIGHT$
*
* Additional copyrights may follow
*
* $HEADER$
*/
#include "ompi_config.h"
#include "ompi/constants.h"
#include "mtl_portals4.h"
#include "mtl_portals4_recv_short.h"
OBJ_CLASS_INSTANCE(ompi_mtl_portals4_recv_short_block_t,
opal_list_item_t,
NULL, NULL);
static ompi_mtl_portals4_recv_short_block_t*
ompi_mtl_portals4_recv_short_block_init(mca_mtl_portals4_module_t *mtl)
{
ompi_mtl_portals4_recv_short_block_t *block;
block = OBJ_NEW(ompi_mtl_portals4_recv_short_block_t);
block->mtl = mtl;
block->start = malloc(mtl->recv_short_size);
if (block->start == NULL) return NULL;
block->me_h = PTL_INVALID_HANDLE;
return block;
}
static int
ompi_mtl_portals4_recv_short_block_free(ompi_mtl_portals4_recv_short_block_t *block)
{
if (PTL_INVALID_HANDLE != block->me_h) {
PtlMEUnlink(block->me_h);
block->me_h = PTL_INVALID_HANDLE;
}
if (NULL != block->start) {
free(block->start);
block->start = NULL;
}
OBJ_RELEASE(block);
return OMPI_SUCCESS;
}
static inline int
ompi_mtl_portals4_activate_block(ompi_mtl_portals4_recv_short_block_t *block)
{
ptl_match_bits_t match_bits = PTL_SHORT_MSG;
ptl_match_bits_t ignore_bits = PTL_CONTEXT_MASK | PTL_SOURCE_MASK | PTL_TAG_MASK;
ptl_me_t me;
int ret;
me.start = block->start;
me.length = block->mtl->recv_short_size;
me.ct_handle = PTL_CT_NONE;
me.min_free = block->mtl->eager_limit;
me.ac_id.uid = PTL_UID_ANY;
me.options = PTL_ME_OP_PUT | PTL_ME_MANAGE_LOCAL | PTL_ME_NO_TRUNCATE |
PTL_ME_MAY_ALIGN | PTL_ME_ACK_DISABLE | PTL_ME_EVENT_COMM_DISABLE;
me.match_id.phys.nid = PTL_NID_ANY;
me.match_id.phys.pid = PTL_PID_ANY;
me.match_bits = match_bits;
me.ignore_bits = ignore_bits;
ret = PtlMEAppend(block->mtl->ni_h,
PTL_SEND_TABLE_ID,
&me,
PTL_OVERFLOW,
block,
&block->me_h);
return (ret == PTL_OK) ? OMPI_SUCCESS : ompi_mtl_portals4_get_error(ret);
}
int
ompi_mtl_portals4_recv_short_block_repost(ptl_event_t *ev)
{
return ompi_mtl_portals4_activate_block(ev->user_ptr);
}
int
ompi_mtl_portals4_recv_short_init(mca_mtl_portals4_module_t *mtl)
{
int i;
OBJ_CONSTRUCT(&(mtl->recv_short_blocks), opal_list_t);
/* create the recv blocks */
for (i = 0 ; i < mtl->recv_short_num ; ++i) {
ompi_mtl_portals4_recv_short_block_t *block =
ompi_mtl_portals4_recv_short_block_init(mtl);
if (NULL == block) {
return OMPI_ERR_OUT_OF_RESOURCE;
}
opal_list_append(&(mtl->recv_short_blocks),
(opal_list_item_t*) block);
ompi_mtl_portals4_activate_block(block);
}
return OMPI_SUCCESS;
}
int
ompi_mtl_portals4_recv_short_fini(mca_mtl_portals4_module_t *mtl)
{
opal_list_item_t *item;
while (NULL != (item = opal_list_remove_first(&mtl->recv_short_blocks))) {
ompi_mtl_portals4_recv_short_block_t *block =
(ompi_mtl_portals4_recv_short_block_t*) item;
ompi_mtl_portals4_recv_short_block_free(block);
}
return OMPI_SUCCESS;
}

45
ompi/mca/mtl/portals4/mtl_portals4_recv_short.h Обычный файл
Просмотреть файл

@ -0,0 +1,45 @@
/*
* Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
* University Research and Technology
* Corporation. All rights reserved.
* Copyright (c) 2004-2006 The University of Tennessee and The University
* of Tennessee Research Foundation. All rights
* reserved.
* Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
* University of Stuttgart. All rights reserved.
* Copyright (c) 2004-2005 The Regents of the University of California.
* All rights reserved.
* Copyright (c) 2010 Sandia National Laboratories. All rights reserved.
* $COPYRIGHT$
*
* Additional copyrights may follow
*
* $HEADER$
*/
#ifndef OMPI_MTL_PORTALS_RECV_SHORT_H
#define OMPI_MTL_PORTALS_RECV_SHORT_H
struct ompi_mtl_portals4_recv_short_block_t {
opal_list_item_t base;
mca_mtl_portals4_module_t *mtl;
void *start;
ptl_handle_me_t me_h;
};
typedef struct ompi_mtl_portals4_recv_short_block_t ompi_mtl_portals4_recv_short_block_t;
OBJ_CLASS_DECLARATION(ompi_mtl_portals4_recv_short_block_t);
extern int
ompi_mtl_portals4_recv_short_init(mca_mtl_portals4_module_t *mtl);
extern int
ompi_mtl_portals4_recv_short_fini(mca_mtl_portals4_module_t *mtl);
extern int
ompi_mtl_portals4_recv_short_block_repost(ptl_event_t *ev);
extern int
ompi_mtl_portals4_recv_progress(ptl_event_t *ev,
ompi_mtl_portals4_request_t* ptl_request);
#endif /* OMPI_MTL_PORTALS_RECV_SHORT_H */

25
ompi/mca/mtl/portals4/mtl_portals4_request.h
Просмотреть файл

@ -1,5 +1,15 @@
/*
* Copyright (c) 2010 Sandia National Laboratories. All rights reserved.
* Copyright (c) 2004-2006 The Trustees of Indiana University and Indiana
* University Research and Technology
* Corporation. All rights reserved.
* Copyright (c) 2004-2005 The University of Tennessee and The University
* of Tennessee Research Foundation. All rights
* reserved.
* Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
* University of Stuttgart. All rights reserved.
* Copyright (c) 2004-2005 The Regents of the University of California.
* All rights reserved.
* Copyright (c) 2010 Sandia National Laboratories. All rights reserved.
* $COPYRIGHT$
*
* Additional copyrights may follow
@ -7,13 +17,22 @@
* $HEADER$
*/
#ifndef OMPI_MTL_PORTALS4_REQUEST_H
#define OMPI_MTL_PORTALS4_REQUEST_H
#ifndef OMPI_MTL_PORTALS_REQUEST_H
#define OMPI_MTL_PORTALS_REQUEST_H
#include "opal/datatype/opal_convertor.h"
#include "ompi/mca/mtl/mtl.h"
struct ompi_mtl_portals4_request_t {
struct mca_mtl_request_t super;
int (*event_callback)(ptl_event_t *ev, struct ompi_mtl_portals4_request_t*);
void *buffer_ptr; /* send and receive side */
ptl_handle_md_t md_h; /* send and receive side */
ptl_handle_me_t me_h; /* send and receive side */
int event_count; /* send side */
struct opal_convertor_t *convertor; /* recv side */
void *delivery_ptr; /* recv side */
size_t delivery_len; /* recv side */
};
typedef struct ompi_mtl_portals4_request_t ompi_mtl_portals4_request_t;

383
ompi/mca/mtl/portals4/mtl_portals4_send.c
Просмотреть файл

@ -1,5 +1,15 @@
/*
* Copyright (c) 2010 Sandia National Laboratories. All rights reserved.
* Copyright (c) 2004-2006 The Trustees of Indiana University and Indiana
* University Research and Technology
* Corporation. All rights reserved.
* Copyright (c) 2004-2005 The University of Tennessee and The University
* of Tennessee Research Foundation. All rights
* reserved.
* Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
* University of Stuttgart. All rights reserved.
* Copyright (c) 2004-2005 The Regents of the University of California.
* All rights reserved.
* Copyright (c) 2010 Sandia National Laboratories. All rights reserved.
* $COPYRIGHT$
*
* Additional copyrights may follow
@ -9,21 +19,312 @@
#include "ompi_config.h"
#include "ompi/communicator/communicator.h"
#include "opal/datatype/opal_convertor.h"
#include "ompi/mca/mtl/base/base.h"
#include "ompi/mca/mtl/base/mtl_base_datatype.h"
#include "mtl_portals4.h"
#include "mtl_portals4_request.h"
#include "mtl_portals4_endpoint.h"
int
ompi_mtl_portals4_send(struct mca_mtl_base_module_t* mtl,
struct ompi_communicator_t* comm,
int dest,
int tag,
struct opal_convertor_t *convertor,
mca_pml_base_send_mode_t mode)
/* called when no ack is necessary */
static int
ompi_mtl_portals4_short_callback(ptl_event_t *ev, ompi_mtl_portals4_request_t *ptl_request)
{
assert(ev->type == PTL_EVENT_SEND);
assert(NULL != ptl_request->super.ompi_req);
if (ev->ni_fail_type != PTL_NI_OK) {
opal_output_verbose(ompi_mtl_base_output, 1,
"%s:%d: short send callback ni_fail_type: %d",
__FILE__, __LINE__, ev->ni_fail_type);
ptl_request->super.ompi_req->req_status.MPI_ERROR = OMPI_ERROR;
}
if (NULL != ptl_request->buffer_ptr) {
free(ptl_request->buffer_ptr);
}
PtlMDRelease(ptl_request->md_h);
ptl_request->super.completion_callback(&ptl_request->super);
return OMPI_SUCCESS;
}
/* called when send should wait for an ack or get */
static int
ompi_mtl_portals4_long_callback(ptl_event_t *ev, struct ompi_mtl_portals4_request_t* ptl_request)
{
assert(ev->type == PTL_EVENT_SEND || ev->type == PTL_EVENT_ACK || ev->type == PTL_EVENT_GET);
assert(NULL != ptl_request->super.ompi_req);
if (ev->ni_fail_type != PTL_NI_OK) {
opal_output_verbose(ompi_mtl_base_output, 1,
"%s:%d: long send callback ni_fail_type: %d",
__FILE__, __LINE__, ev->ni_fail_type);
ptl_request->super.ompi_req->req_status.MPI_ERROR = OMPI_ERROR;
}
/* we only receive an ack if the message was received into an
expected message. Otherwise, we don't get an ack, but mark
completion when the message was pulled (long message). */
if ( ++(ptl_request->event_count) == 2 ) {
if (NULL != ptl_request->buffer_ptr) {
free(ptl_request->buffer_ptr);
}
PtlMDRelease(ptl_request->md_h);
ptl_request->super.completion_callback(&ptl_request->super);
}
/* received an ack - unlink the me */
if (ev->type == PTL_EVENT_ACK) {
PtlMEUnlink(ptl_request->me_h);
}
return OMPI_SUCCESS;
}
/* called when sync send should wait for an ack or put */
static int
ompi_mtl_portals4_sync_callback(ptl_event_t *ev, struct ompi_mtl_portals4_request_t* ptl_request)
{
assert(ev->type == PTL_EVENT_SEND || ev->type == PTL_EVENT_ACK || ev->type == PTL_EVENT_PUT);
assert(NULL != ptl_request->super.ompi_req);
if (ev->ni_fail_type != PTL_NI_OK) {
opal_output_verbose(ompi_mtl_base_output, 1,
"%s:%d: sync send callback ni_fail_type: %d",
__FILE__, __LINE__, ev->ni_fail_type);
ptl_request->super.ompi_req->req_status.MPI_ERROR = OMPI_ERROR;
}
/* we only receive an ack if the message was received into an
expected message. Otherwise, we don't get an ack, but mark
completion when a zero-length put arrrives. */
if ( ++(ptl_request->event_count) == 2 ) {
if (NULL != ptl_request->buffer_ptr) {
free(ptl_request->buffer_ptr);
}
PtlMDRelease(ptl_request->md_h);
ptl_request->super.completion_callback(&ptl_request->super);
}
/* received an ack - unlink the me */
if (ev->type == PTL_EVENT_ACK) {
PtlMEUnlink(ptl_request->me_h);
}
return OMPI_SUCCESS;
}
static int
ompi_mtl_portals4_short_isend(mca_pml_base_send_mode_t mode, void *start, int length,
int contextid, int localrank, int tag, ptl_process_t dest,
ompi_mtl_portals4_request_t *ptl_request )
{
int ret;
ptl_match_bits_t mode_bits;
ptl_match_bits_t match_bits;
ptl_md_t md;
ptl_request->event_callback = ompi_mtl_portals4_short_callback;
mode_bits = (MCA_PML_BASE_SEND_READY != mode) ? PTL_SHORT_MSG : PTL_READY_MSG;
PTL_SET_SEND_BITS(match_bits, contextid, localrank, tag, mode_bits);
md.start = start;
md.length = length;
md.options = 0;
md.eq_handle = ompi_mtl_portals4.eq_h;
md.ct_handle = PTL_CT_NONE;
ret = PtlMDBind(ompi_mtl_portals4.ni_h,
&md,
&ptl_request->md_h);
if (PTL_OK != ret) {
opal_output_verbose(ompi_mtl_base_output, 1,
"%s:%d: PtlMDBind failed: %d",
__FILE__, __LINE__, ret);
return ompi_mtl_portals4_get_error(ret);
}
ret = PtlPut(ptl_request->md_h,
0,
length,
PTL_NO_ACK_REQ,
dest,
PTL_SEND_TABLE_ID,
match_bits,
0,
ptl_request,
0);
if (PTL_OK != ret) {
opal_output_verbose(ompi_mtl_base_output, 1,
"%s:%d: PtlPut failed: %d",
__FILE__, __LINE__, ret);
PtlMDRelease(ptl_request->md_h);
return ompi_mtl_portals4_get_error(ret);
}
return OMPI_SUCCESS;
}
static int
ompi_mtl_portals4_long_isend( void *start, int length, int contextid, int localrank, int tag,
ptl_process_t dest, ompi_mtl_portals4_request_t *ptl_request )
{
int ret;
ptl_match_bits_t match_bits;
ptl_md_t md;
ptl_me_t me;
ptl_request->event_callback = ompi_mtl_portals4_long_callback;
PTL_SET_SEND_BITS(match_bits, contextid, localrank, tag, PTL_LONG_MSG);
md.start = start;
md.length = length;
md.options = 0;
md.eq_handle = ompi_mtl_portals4.eq_h;
md.ct_handle = PTL_CT_NONE;
ret = PtlMDBind(ompi_mtl_portals4.ni_h,
&md,
&ptl_request->md_h);
if (PTL_OK != ret) {
opal_output_verbose(ompi_mtl_base_output, 1,
"%s:%d: PtlMDBind failed: %d",
__FILE__, __LINE__, ret);
return ompi_mtl_portals4_get_error(ret);
}
me.start = start;
me.length = length;
me.ct_handle = PTL_CT_NONE;
me.min_free = 0;
me.ac_id.uid = PTL_UID_ANY;
me.options = PTL_ME_OP_GET | PTL_ME_USE_ONCE;
me.match_id = dest;
me.match_bits = (ptl_match_bits_t)(uintptr_t)ptl_request;
me.ignore_bits = 0;
ret = PtlMEAppend(ompi_mtl_portals4.ni_h,
PTL_READ_TABLE_ID,
&me,
PTL_PRIORITY_LIST,
ptl_request,
&ptl_request->me_h);
if (PTL_OK != ret) {
opal_output_verbose(ompi_mtl_base_output, 1,
"%s:%d: PtlMEAppend failed: %d",
__FILE__, __LINE__, ret);
PtlMDRelease(ptl_request->md_h);
return ompi_mtl_portals4_get_error(ret);
}
ret = PtlPut(ptl_request->md_h,
0,
length,
PTL_ACK_REQ,
dest,
PTL_SEND_TABLE_ID,
match_bits,
0,
ptl_request,
(ptl_hdr_data_t)(uintptr_t)ptl_request);
if (PTL_OK != ret) {
opal_output_verbose(ompi_mtl_base_output, 1,
"%s:%d: PtlPut failed: %d",
__FILE__, __LINE__, ret);
PtlMEUnlink(ptl_request->me_h);
PtlMDRelease(ptl_request->md_h);
return ompi_mtl_portals4_get_error(ret);
}
return OMPI_SUCCESS;
}
static int
ompi_mtl_portals4_sync_isend( void *start, int length, int contextid, int localrank, int tag,
ptl_process_t dest, ompi_mtl_portals4_request_t *ptl_request )
{
int ret;
ptl_match_bits_t match_bits;
ptl_md_t md;
ptl_me_t me;
ptl_request->event_callback = ompi_mtl_portals4_sync_callback;
PTL_SET_SEND_BITS(match_bits, contextid, localrank, tag, PTL_SHORT_MSG);
md.start = start;
md.length = length;
md.options = 0;
md.eq_handle = ompi_mtl_portals4.eq_h;
md.ct_handle = PTL_CT_NONE;
ret = PtlMDBind(ompi_mtl_portals4.ni_h,
&md,
&ptl_request->md_h);
if (PTL_OK != ret) {
opal_output_verbose(ompi_mtl_base_output, 1,
"%s:%d: PtlMDBind failed: %d",
__FILE__, __LINE__, ret);
return ompi_mtl_portals4_get_error(ret);
}
me.start = NULL;
me.length = 0;
me.ct_handle = PTL_CT_NONE;
me.min_free = 0;
me.ac_id.uid = PTL_UID_ANY;
me.options = PTL_ME_OP_PUT | PTL_ME_USE_ONCE;
me.match_id = dest;
me.match_bits = (ptl_match_bits_t)(uintptr_t)ptl_request;
me.ignore_bits = 0;
ret = PtlMEAppend(ompi_mtl_portals4.ni_h,
PTL_READ_TABLE_ID,
&me,
PTL_PRIORITY_LIST,
ptl_request,
&ptl_request->me_h);
if (PTL_OK != ret) {
opal_output_verbose(ompi_mtl_base_output, 1,
"%s:%d: PtlMEAppend failed: %d",
__FILE__, __LINE__, ret);
PtlMDRelease(ptl_request->md_h);
return ompi_mtl_portals4_get_error(ret);
}
ret = PtlPut(ptl_request->md_h,
0,
length,
PTL_ACK_REQ,
dest,
PTL_SEND_TABLE_ID,
match_bits,
0,
ptl_request,
(ptl_hdr_data_t)(uintptr_t)ptl_request);
if (PTL_OK != ret) {
opal_output_verbose(ompi_mtl_base_output, 1,
"%s:%d: PtlPut failed: %d",
__FILE__, __LINE__, ret);
PtlMEUnlink(ptl_request->me_h);
PtlMDRelease(ptl_request->md_h);
return ompi_mtl_portals4_get_error(ret);
}
return OMPI_SUCCESS;
}
int
ompi_mtl_portals4_isend(struct mca_mtl_base_module_t* mtl,
struct ompi_communicator_t* comm,
@ -34,6 +335,68 @@ ompi_mtl_portals4_isend(struct mca_mtl_base_module_t* mtl,
bool blocking,
mca_mtl_request_t *mtl_request)
{
return OMPI_SUCCESS;
}
int ret = OMPI_SUCCESS;
ompi_proc_t *ompi_proc = ompi_comm_peer_lookup( comm, dest );
mca_mtl_base_endpoint_t *endpoint = (mca_mtl_base_endpoint_t*)ompi_proc->proc_pml;
ompi_mtl_portals4_request_t *ptl_request = (ompi_mtl_portals4_request_t*)mtl_request;
void *start;
size_t length;
bool free_after;
ret = ompi_mtl_datatype_pack(convertor, &start, &length, &free_after);
if (OMPI_SUCCESS != ret) return ret;
ptl_request->buffer_ptr = (free_after) ? start : NULL;
ptl_request->event_count = 0;
ptl_request->super.ompi_req->req_status.MPI_ERROR = OMPI_SUCCESS;
switch (mode) {
case MCA_PML_BASE_SEND_STANDARD:
case MCA_PML_BASE_SEND_READY:
case MCA_PML_BASE_SEND_BUFFERED:
if ((length <= ompi_mtl_portals4.eager_limit) ||
(MCA_PML_BASE_SEND_READY == mode)) {
ret = ompi_mtl_portals4_short_isend(mode,
start,
length,
comm->c_contextid,
comm->c_my_rank,
tag,
endpoint->ptl_proc,
ptl_request);
break;
}
/* long standard send case falls through */
case MCA_PML_BASE_SEND_SYNCHRONOUS:
if (length <= ompi_mtl_portals4.eager_limit) {
ret = ompi_mtl_portals4_sync_isend(start,
length,
comm->c_contextid,
comm->c_my_rank,
tag,
endpoint->ptl_proc,
ptl_request);
} else {
/* if we got this far, we're either a standard or synchronous long send */
ret = ompi_mtl_portals4_long_isend(start,
length,
comm->c_contextid,
comm->c_my_rank,
tag,
endpoint->ptl_proc,
ptl_request);
}
break;
default:
opal_output(fileno(stderr),"Unexpected msg type %dn", mode);
ret = OMPI_ERR_NOT_SUPPORTED;
}
if (OMPI_SUCCESS != ret && NULL != ptl_request->buffer_ptr) {
free(ptl_request->buffer_ptr);
}
return ret;
}

115
ompi/mca/mtl/portals4/mtl_portals4_send_short.c
Просмотреть файл

@ -1,115 +0,0 @@
/*
* Copyright (c) 2004-2006 The Trustees of Indiana University and Indiana
* University Research and Technology
* Corporation. All rights reserved.
* Copyright (c) 2004-2005 The University of Tennessee and The University
* of Tennessee Research Foundation. All rights
* reserved.
* Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
* University of Stuttgart. All rights reserved.
* Copyright (c) 2004-2005 The Regents of the University of California.
* All rights reserved.
* $COPYRIGHT$
*
* Additional copyrights may follow
*
* $HEADER$
*/
#include "ompi_config.h"
#include "opal/util/output.h"
#include "mtl_portals.h"
#include "mtl_portals_request.h"
#include "mtl_portals_send_short.h"
static ompi_mtl_portals_request_t ptl_short_request;
/* short send callback */
static int
ompi_mtl_portals_short_callback(ptl_event_t *ev, ompi_mtl_portals_request_t *ptl_request)
{
switch (ev->type) {
case PTL_EVENT_SEND_END:
ompi_mtl_portals_free_short_buf(ev->offset);
break;
default:
opal_output(fileno(stderr)," Unexpected event type %d in ompi_mtl_portals_short_callback()\n",ev->type);
abort();
}
return OMPI_SUCCESS;
}
/* initialize short copy blocks */
void
ompi_mtl_portals_short_setup()
{
int ret;
int i;
if ((ompi_mtl_portals.ptl_num_copy_blocks > 0) && (ompi_mtl_portals.ptl_copy_block_len > 0)) {
ompi_mtl_portals.ptl_short_md.length = ompi_mtl_portals.ptl_num_copy_blocks *
ompi_mtl_portals.ptl_copy_block_len;
ompi_mtl_portals.ptl_short_md.start = malloc(ompi_mtl_portals.ptl_short_md.length);
if (NULL == ompi_mtl_portals.ptl_short_md.start ) {
ompi_mtl_portals.ptl_num_copy_blocks = 0;
return;
}
ompi_mtl_portals.ptl_short_md.threshold = PTL_MD_THRESH_INF;
ompi_mtl_portals.ptl_short_md.max_size = 0;
ompi_mtl_portals.ptl_short_md.options = PTL_MD_EVENT_START_DISABLE;
ompi_mtl_portals.ptl_short_md.user_ptr = &ptl_short_request;
ompi_mtl_portals.ptl_short_md.eq_handle = ompi_mtl_portals.ptl_eq_h;
ret = PtlMDBind(ompi_mtl_portals.ptl_ni_h,
ompi_mtl_portals.ptl_short_md,
PTL_RETAIN,
&ompi_mtl_portals.ptl_short_md_h);
if (PTL_OK != ret) {
free(ompi_mtl_portals.ptl_short_md.start);
ompi_mtl_portals.ptl_num_copy_blocks = 0;
return;
}
ptl_short_request.event_callback = ompi_mtl_portals_short_callback;
ompi_mtl_portals.ptl_copy_block_free_list = malloc(ompi_mtl_portals.ptl_num_copy_blocks * sizeof(int));
if (NULL == ompi_mtl_portals.ptl_copy_block_free_list) {
free(ompi_mtl_portals.ptl_short_md.start);
ompi_mtl_portals.ptl_num_copy_blocks = 0;
return;
}
for (i=0; i<ompi_mtl_portals.ptl_num_copy_blocks; i++) {
ompi_mtl_portals.ptl_copy_block_free_list[i] = i;
}
ompi_mtl_portals.ptl_copy_block_first_free = 0;
}
}
/* free short resources */
void
ompi_mtl_portals_short_cleanup()
{
if (ompi_mtl_portals.ptl_num_copy_blocks > 0) {
free(ompi_mtl_portals.ptl_short_md.start);
free(ompi_mtl_portals.ptl_copy_block_free_list);
ompi_mtl_portals.ptl_num_copy_blocks = 0;
}
}

59
ompi/mca/mtl/portals4/mtl_portals4_send_short.h
Просмотреть файл

@ -1,59 +0,0 @@
/*
* Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
* University Research and Technology
* Corporation. All rights reserved.
* Copyright (c) 2004-2006 The University of Tennessee and The University
* of Tennessee Research Foundation. All rights
* reserved.
* Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
* University of Stuttgart. All rights reserved.
* Copyright (c) 2004-2005 The Regents of the University of California.
* All rights reserved.
* $COPYRIGHT$
*
* Additional copyrights may follow
*
* $HEADER$
*/
#ifndef OMPI_MTL_PORTALS_SEND_SHORT_H
#define OMPI_MTL_PORTALS_SEND_SHORT_H
extern void ompi_mtl_portals_short_setup(void);
extern void ompi_mtl_portals_short_cleanup(void);
static inline int
ompi_mtl_portals_alloc_short_buf(void)
{
int buf_num;
while ( ompi_mtl_portals.ptl_copy_block_first_free == ompi_mtl_portals.ptl_num_copy_blocks ) {
ompi_mtl_portals_progress();
}
buf_num = ompi_mtl_portals.ptl_copy_block_free_list[ompi_mtl_portals.ptl_copy_block_first_free++];
assert((buf_num >= 0) && (buf_num < ompi_mtl_portals.ptl_num_copy_blocks));
return buf_num;
}
static inline void
ompi_mtl_portals_free_short_buf( int offset )
{
int buf_num;
buf_num = offset / ompi_mtl_portals.ptl_copy_block_len;
assert((buf_num >= 0) && (buf_num < ompi_mtl_portals.ptl_num_copy_blocks));
ompi_mtl_portals.ptl_copy_block_first_free--;
assert(ompi_mtl_portals.ptl_copy_block_first_free >= 0);
ompi_mtl_portals.ptl_copy_block_free_list[ompi_mtl_portals.ptl_copy_block_first_free] = buf_num;
}
#endif /* OMPI_MTL_PORTALS_SEND_SHORT_H */