ports/openmpi
ports/openmpi
1
1
Форкнуть 0
Код Релизы Активность

Remove Portals 3.3 support

Просмотр исходного кода 
This commit was SVN r27656.
Этот коммит содержится в:
Brian Barrett 2012-12-06 20:11:27 +00:00
родитель c00e6a7abf
Коммит 702451111b
35 изменённых файлов: 0 добавлений и 6475 удалений
Показать статистику Скачать Patch файл Скачать Diff файл Показать все файлы Свернуть все файлы

60
ompi/mca/btl/portals/Makefile.am
Просмотреть файл

@ -1,60 +0,0 @@
#
# Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
# University Research and Technology
# Corporation. All rights reserved.
# Copyright (c) 2004-2005 The University of Tennessee and The University
# of Tennessee Research Foundation. All rights
# reserved.
# Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
# University of Stuttgart. All rights reserved.
# Copyright (c) 2004-2005 The Regents of the University of California.
# All rights reserved.
# Copyright (c) 2010 Cisco Systems, Inc. All rights reserved.
# $COPYRIGHT$
#
# Additional copyrights may follow
#
# $HEADER$
#
# Make the output library in this directory, and name it either
# mca_<type>_<name>.la (for DSO builds) or libmca_<type>_<name>.la
# (for static builds).
if MCA_BUILD_ompi_btl_portals_DSO
component_noinst =
component_install = mca_btl_portals.la
else
component_noinst = libmca_btl_portals.la
component_install =
endif
portals_SOURCES = \
btl_portals.h \
btl_portals_endpoint.h \
btl_portals_frag.h \
btl_portals_send.h \
btl_portals_recv.h \
btl_portals.c \
btl_portals_component.c \
btl_portals_frag.c \
btl_portals_send.c \
btl_portals_recv.c \
btl_portals_rdma.c
AM_CPPFLAGS = $(btl_portals_CPPFLAGS)
mcacomponentdir = $(pkglibdir)
mcacomponent_LTLIBRARIES = $(component_install)
mca_btl_portals_la_SOURCES = $(portals_SOURCES)
nodist_mca_btl_portals_la_SOURCES = $(portals_nodist_SOURCES)
mca_btl_portals_la_LIBADD = \
$(btl_portals_LIBS) \
$(top_ompi_builddir)/ompi/mca/common/portals/libmca_common_portals.la
mca_btl_portals_la_LDFLAGS = -module -avoid-version $(btl_portals_LDFLAGS)
noinst_LTLIBRARIES = $(component_noinst)
libmca_btl_portals_la_SOURCES = $(portals_SOURCES)
nodist_libmca_btl_portals_la_SOURCES = $(portals_nodist_SOURCES)
libmca_btl_portals_la_LIBADD = $(btl_portals_LIBS)
libmca_btl_portals_la_LDFLAGS = -module -avoid-version $(btl_portals_LDFLAGS)

569
ompi/mca/btl/portals/btl_portals.c
Просмотреть файл

@ -1,569 +0,0 @@
/*
* Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
* University Research and Technology
* Corporation. All rights reserved.
* Copyright (c) 2004-2008 The University of Tennessee and The University
* of Tennessee Research Foundation. All rights
* reserved.
* Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
* University of Stuttgart. All rights reserved.
* Copyright (c) 2004-2005 The Regents of the University of California.
* All rights reserved.
* Copyright (c) 2008 UT-Battelle, LLC. All rights reserved.
* Copyright (c) 2012 Los Alamos National Security, LLC. All rights
* reserved.
* $COPYRIGHT$
*
* Additional copyrights may follow
*
* $HEADER$
*/
#include "ompi_config.h"
#include <sys/types.h>
#include <unistd.h>
#include <stdio.h>
#include <errno.h>
#include <inttypes.h>
#include "opal/class/opal_bitmap.h"
#include "ompi/constants.h"
#include "ompi/mca/btl/btl.h"
#include "opal/datatype/opal_convertor.h"
#include "btl_portals.h"
#include "btl_portals_endpoint.h"
#include "btl_portals_recv.h"
#include "btl_portals_frag.h"
mca_btl_portals_module_t mca_btl_portals_module = {
{
&mca_btl_portals_component.super,
/* NOTE: All these default values are set in
component_open() */
0, /* max size of first frag */
0, /* min send size */
0, /* max send size */
0, /* btl_rdma_pipeline_send_length */
0, /* btl_rdma_pipeline_frag_size */
0, /* btl_min_rdma_pipeline_size */
0, /* exclusivity - higher than sm, lower than self */
0, /* latency */
0, /* bandwidth */
0, /* btl flags */
0, /* btl segment size */
mca_btl_portals_add_procs,
mca_btl_portals_del_procs,
NULL,
mca_btl_portals_finalize,
mca_btl_portals_alloc,
mca_btl_portals_free,
mca_btl_portals_prepare_src,
mca_btl_portals_prepare_dst,
mca_btl_portals_send,
NULL, /* mca_btl_portals_sendi, */
mca_btl_portals_put,
mca_btl_portals_get,
mca_btl_base_dump,
NULL, /* mpool */
NULL, /* register error */
NULL
},
};
int
mca_btl_portals_add_procs(struct mca_btl_base_module_t* btl_base,
size_t nprocs, struct ompi_proc_t **procs,
struct mca_btl_base_endpoint_t** peers,
opal_bitmap_t* reachable)
{
int ret;
struct ompi_proc_t *curr_proc = NULL;
ptl_process_id_t *portals_procs = NULL;
size_t i;
unsigned long distance;
bool need_activate = false;
bool accel;
assert(&mca_btl_portals_module == (mca_btl_portals_module_t*) btl_base);
opal_output_verbose(50, mca_btl_portals_component.portals_output,
"Adding %d procs (%d)", (int) nprocs,
(int) mca_btl_portals_module.portals_num_procs);
/* if we havne't already, get our network handle */
if (mca_btl_portals_module.portals_ni_h == PTL_INVALID_HANDLE) {
ret = ompi_common_portals_ni_initialize(&mca_btl_portals_module.portals_ni_h, &accel);
if (OMPI_SUCCESS != ret) return ret;
}
portals_procs = malloc(nprocs * sizeof(ptl_process_id_t));
ret = ompi_common_portals_get_procs(nprocs, procs, portals_procs);
if (OMPI_SUCCESS != ret) return ret;
if (0 == mca_btl_portals_module.portals_num_procs) {
need_activate = true;
}
/* loop through all procs, setting our reachable flag */
for (i= 0; i < nprocs ; ++i) {
curr_proc = procs[i];
/* portals doesn't support heterogeneous yet... */
if (ompi_proc_local()->proc_arch != curr_proc->proc_arch) {
continue;
}
peers[i] = malloc(sizeof(mca_btl_base_endpoint_t));
if (NULL == peers[i]) return OMPI_ERROR;
*((mca_btl_base_endpoint_t*) peers[i]) = portals_procs[i];
/* accelerated doesn't support PtlNIDist() */
if (accel == false) {
/* make sure we can reach the process - this is supposed to be
a cheap-ish operation */
ret = PtlNIDist(mca_btl_portals_module.portals_ni_h,
portals_procs[i],
&distance);
if (ret != PTL_OK) {
opal_output_verbose(10, mca_btl_portals_component.portals_output,
"Could not find distance to process %d", (int) i);
continue;
}
}
OPAL_THREAD_ADD32(&mca_btl_portals_module.portals_num_procs, 1);
/* and here we can reach */
opal_bitmap_set_bit(reachable, i);
}
if (NULL != portals_procs) free(portals_procs);
if (need_activate && mca_btl_portals_module.portals_num_procs > 0) {
/* create eqs */
int i;
opal_output_verbose(50, mca_btl_portals_component.portals_output,
"Enabling progress");
for (i = 0 ; i < OMPI_BTL_PORTALS_EQ_SIZE ; ++i) {
int ptl_ret = PtlEQAlloc(mca_btl_portals_module.portals_ni_h,
mca_btl_portals_module.portals_eq_sizes[i],
PTL_EQ_HANDLER_NONE,
&(mca_btl_portals_module.portals_eq_handles[i]));
if (PTL_OK != ptl_ret) {
opal_output(mca_btl_portals_component.portals_output,
"Error creating EQ %d: %d", i, ptl_ret);
/* BWB - better error code? */
return OMPI_ERROR;
}
}
ret = mca_btl_portals_recv_enable(&mca_btl_portals_module);
/* fill in send memory descriptor */
mca_btl_portals_module.md_send.start = NULL;
mca_btl_portals_module.md_send.length = 0;
mca_btl_portals_module.md_send.threshold = PTL_MD_THRESH_INF;
mca_btl_portals_module.md_send.max_size = 0;
mca_btl_portals_module.md_send.options = PTL_MD_EVENT_START_DISABLE;
mca_btl_portals_module.md_send.user_ptr = NULL;
mca_btl_portals_module.md_send.eq_handle =
mca_btl_portals_module.portals_eq_handles[OMPI_BTL_PORTALS_EQ_SEND];
} else {
ret = OMPI_SUCCESS;
}
return ret;
}
int
mca_btl_portals_del_procs(struct mca_btl_base_module_t *btl_base,
size_t nprocs,
struct ompi_proc_t **procs,
struct mca_btl_base_endpoint_t **peers)
{
size_t i = 0;
int ret = OMPI_SUCCESS;
assert(&mca_btl_portals_module == (mca_btl_portals_module_t*) btl_base);
opal_output_verbose(50, mca_btl_portals_component.portals_output,
"Removing %d procs (%d)", (int) nprocs,
(int) mca_btl_portals_module.portals_num_procs);
for (i = 0 ; i < nprocs ; ++i) {
free(peers[i]);
OPAL_THREAD_ADD32(&mca_btl_portals_module.portals_num_procs, -1);
}
if (0 == mca_btl_portals_module.portals_num_procs) {
int i;
opal_output_verbose(50, mca_btl_portals_component.portals_output,
"Disabling progress");
ret = mca_btl_portals_recv_disable(&mca_btl_portals_module);
/* destroy eqs */
for (i = 0 ; i < OMPI_BTL_PORTALS_EQ_SIZE ; ++i) {
int ptl_ret = PtlEQFree(mca_btl_portals_module.portals_eq_handles[i]);
if (PTL_OK != ptl_ret) {
opal_output(mca_btl_portals_component.portals_output,
"Error freeing EQ %d: %d", i, ptl_ret);
}
}
} else {
ret = OMPI_SUCCESS;
}
return ret;
}
mca_btl_base_descriptor_t*
mca_btl_portals_alloc(struct mca_btl_base_module_t* btl_base,
struct mca_btl_base_endpoint_t* endpoint,
uint8_t order,
size_t size,
uint32_t flags)
{
int rc;
mca_btl_portals_frag_t* frag;
assert(&mca_btl_portals_module == (mca_btl_portals_module_t*) btl_base);
if (size <= mca_btl_portals_module.super.btl_eager_limit) {
OMPI_BTL_PORTALS_FRAG_ALLOC_EAGER(&mca_btl_portals_module, frag, rc);
if (OMPI_SUCCESS != rc) return NULL;
frag->segments[0].base.seg_len = size;
} else {
OMPI_BTL_PORTALS_FRAG_ALLOC_MAX(&mca_btl_portals_module, frag, rc);
if (OMPI_SUCCESS != rc) return NULL;
frag->segments[0].base.seg_len =
size <= mca_btl_portals_module.super.btl_max_send_size ?
size : mca_btl_portals_module.super.btl_max_send_size ;
}
frag->base.des_src_cnt = 1;
frag->base.des_flags = flags | MCA_BTL_DES_SEND_ALWAYS_CALLBACK;
frag->base.order = MCA_BTL_NO_ORDER;
return &frag->base;
}
int
mca_btl_portals_free(struct mca_btl_base_module_t* btl_base,
mca_btl_base_descriptor_t* des)
{
mca_btl_portals_frag_t* frag = (mca_btl_portals_frag_t*) des;
assert(&mca_btl_portals_module == (mca_btl_portals_module_t*) btl_base);
if (BTL_PORTALS_FRAG_TYPE_EAGER == frag->type) {
/* don't ever unlink eager frags */
OMPI_BTL_PORTALS_FRAG_RETURN_EAGER(&mca_btl_portals_module.super, frag);
} else if (BTL_PORTALS_FRAG_TYPE_MAX == frag->type) {
if (frag->md_h != PTL_INVALID_HANDLE) {
PtlMDUnlink(frag->md_h);
frag->md_h = PTL_INVALID_HANDLE;
}
OMPI_BTL_PORTALS_FRAG_RETURN_MAX(&mca_btl_portals_module.super, frag);
} else if (BTL_PORTALS_FRAG_TYPE_USER == frag->type) {
if (frag->md_h != PTL_INVALID_HANDLE) {
PtlMDUnlink(frag->md_h);
frag->md_h = PTL_INVALID_HANDLE;
}
OPAL_THREAD_ADD32(&mca_btl_portals_module.portals_outstanding_ops, -1);
OMPI_BTL_PORTALS_FRAG_RETURN_USER(&mca_btl_portals_module.super, frag);
} else {
return OMPI_ERR_BAD_PARAM;
}
return OMPI_SUCCESS;
}
mca_btl_base_descriptor_t*
mca_btl_portals_prepare_src(struct mca_btl_base_module_t* btl_base,
struct mca_btl_base_endpoint_t* peer,
mca_mpool_base_registration_t* registration,
struct opal_convertor_t* convertor,
uint8_t order,
size_t reserve,
size_t* size,
uint32_t flags)
{
mca_btl_portals_frag_t* frag;
size_t max_data = *size;
struct iovec iov;
uint32_t iov_count = 1;
int ret;
assert(&mca_btl_portals_module == (mca_btl_portals_module_t*) btl_base);
if (0 != reserve || 0 != opal_convertor_need_buffers(convertor)) {
frag = (mca_btl_portals_frag_t*)
mca_btl_portals_alloc(btl_base, peer, MCA_BTL_NO_ORDER, max_data + reserve, flags);
if (NULL == frag) {
return NULL;
}
if (max_data + reserve > frag->size) {
max_data = frag->size - reserve;
}
iov.iov_len = max_data;
iov.iov_base = (unsigned char*) frag->segments[0].base.seg_addr.pval + reserve;
ret = opal_convertor_pack(convertor, &iov, &iov_count,
&max_data );
*size = max_data;
if ( ret < 0 ) {
return NULL;
}
frag->segments[0].base.seg_len = max_data + reserve;
frag->base.des_src_cnt = 1;
} else {
/* no need to pack - rdma operation out of user's buffer */
ptl_md_t md;
ptl_handle_me_t me_h;
/* reserve space in the event queue for rdma operations immediately */
while (OPAL_THREAD_ADD32(&mca_btl_portals_module.portals_outstanding_ops, 1) >
mca_btl_portals_module.portals_max_outstanding_ops) {
OPAL_THREAD_ADD32(&mca_btl_portals_module.portals_outstanding_ops, -1);
mca_btl_portals_component_progress();
}
OMPI_BTL_PORTALS_FRAG_ALLOC_USER(&mca_btl_portals_module.super, frag, ret);
if(NULL == frag){
OPAL_THREAD_ADD32(&mca_btl_portals_module.portals_outstanding_ops, -1);
return NULL;
}
iov.iov_len = max_data;
iov.iov_base = NULL;
opal_convertor_pack(convertor, &iov, &iov_count, &max_data );
frag->segments[0].base.seg_len = max_data;
frag->segments[0].base.seg_addr.pval = iov.iov_base;
frag->segments[0].key = OPAL_THREAD_ADD64(&(mca_btl_portals_module.portals_rdma_key), 1);
frag->base.des_src_cnt = 1;
/* either a put or get. figure out which later */
OPAL_OUTPUT_VERBOSE((90, mca_btl_portals_component.portals_output,
"rdma src posted for frag 0x%lx, callback 0x%lx, bits %"PRIu64", flags say %d" ,
(unsigned long) frag,
(unsigned long) frag->base.des_cbfunc,
frag->segments[0].key, flags));
/* create a match entry */
ret = PtlMEAttach(mca_btl_portals_module.portals_ni_h,
OMPI_BTL_PORTALS_RDMA_TABLE_ID,
*((mca_btl_base_endpoint_t*) peer),
frag->segments[0].key, /* match */
0, /* ignore */
PTL_UNLINK,
PTL_INS_AFTER,
&me_h);
if (PTL_OK != ret) {
opal_output(mca_btl_portals_component.portals_output,
"Error creating rdma src ME: %d", ret);
OMPI_BTL_PORTALS_FRAG_RETURN_USER(&mca_btl_portals_module.super, frag);
OPAL_THREAD_ADD32(&mca_btl_portals_module.portals_outstanding_ops, -1);
return NULL;
}
/* setup the memory descriptor */
md.start = frag->segments[0].base.seg_addr.pval;
md.length = frag->segments[0].base.seg_len;
md.threshold = PTL_MD_THRESH_INF;
md.max_size = 0;
md.options = PTL_MD_OP_PUT | PTL_MD_OP_GET | PTL_MD_EVENT_START_DISABLE;
md.user_ptr = frag; /* keep a pointer to ourselves */
md.eq_handle = mca_btl_portals_module.portals_eq_handles[OMPI_BTL_PORTALS_EQ_SEND];
ret = PtlMDAttach(me_h,
md,
PTL_UNLINK,
&(frag->md_h));
if (PTL_OK != ret) {
opal_output(mca_btl_portals_component.portals_output,
"Error creating rdma src MD: %d", ret);
PtlMEUnlink(me_h);
OMPI_BTL_PORTALS_FRAG_RETURN_USER(&mca_btl_portals_module.super, frag);
OPAL_THREAD_ADD32(&mca_btl_portals_module.portals_outstanding_ops, -1);
return NULL;
}
}
frag->base.des_src = frag->segments;
frag->base.des_dst = NULL;
frag->base.des_dst_cnt = 0;
frag->base.des_flags = flags | MCA_BTL_DES_SEND_ALWAYS_CALLBACK;
frag->base.order = MCA_BTL_NO_ORDER;
return &frag->base;
}
mca_btl_base_descriptor_t*
mca_btl_portals_prepare_dst(struct mca_btl_base_module_t* btl_base,
struct mca_btl_base_endpoint_t* peer,
mca_mpool_base_registration_t* registration,
struct opal_convertor_t* convertor,
uint8_t order,
size_t reserve,
size_t* size,
uint32_t flags)
{
mca_btl_portals_frag_t* frag;
ptl_md_t md;
ptl_handle_me_t me_h;
int ret;
assert(&mca_btl_portals_module == (mca_btl_portals_module_t*) btl_base);
/* reserve space in the event queue for rdma operations immediately */
while (OPAL_THREAD_ADD32(&mca_btl_portals_module.portals_outstanding_ops, 1) >
mca_btl_portals_module.portals_max_outstanding_ops) {
OPAL_THREAD_ADD32(&mca_btl_portals_module.portals_outstanding_ops, -1);
mca_btl_portals_component_progress();
}
OMPI_BTL_PORTALS_FRAG_ALLOC_USER(&mca_btl_portals_module.super, frag, ret);
if(NULL == frag) {
OPAL_THREAD_ADD32(&mca_btl_portals_module.portals_outstanding_ops, -1);
return NULL;
}
frag->segments[0].base.seg_len = *size;
opal_convertor_get_current_pointer( convertor, (void**)&(frag->segments[0].base.seg_addr.pval) );
frag->segments[0].key = OPAL_THREAD_ADD64(&(mca_btl_portals_module.portals_rdma_key), 1);
frag->base.des_src = NULL;
frag->base.des_src_cnt = 0;
frag->base.des_dst = frag->segments;
frag->base.des_dst_cnt = 1;
frag->base.des_flags = flags | MCA_BTL_DES_SEND_ALWAYS_CALLBACK;
OPAL_OUTPUT_VERBOSE((90, mca_btl_portals_component.portals_output,
"rdma dest posted for frag 0x%lx, callback 0x%lx, bits %" PRIu64 " flags %d",
(unsigned long) frag,
(unsigned long) frag->base.des_cbfunc,
frag->segments[0].key;
flags));
/* create a match entry */
ret = PtlMEAttach(mca_btl_portals_module.portals_ni_h,
OMPI_BTL_PORTALS_RDMA_TABLE_ID,
*((mca_btl_base_endpoint_t*) peer),
frag->segments[0].key, /* match */
0, /* ignore */
PTL_UNLINK,
PTL_INS_AFTER,
&me_h);
if (PTL_OK != ret) {
opal_output(mca_btl_portals_component.portals_output,
"Error creating rdma dest ME: %d", ret);
OPAL_THREAD_ADD32(&mca_btl_portals_module.portals_outstanding_ops, -1);
OMPI_BTL_PORTALS_FRAG_RETURN_USER(&mca_btl_portals_module.super, frag);
return NULL;
}
/* setup the memory descriptor. */
md.start = frag->segments[0].base.seg_addr.pval;
md.length = frag->segments[0].base.seg_len;
md.threshold = PTL_MD_THRESH_INF;
md.max_size = 0;
md.options = PTL_MD_OP_PUT | PTL_MD_OP_GET | PTL_MD_EVENT_START_DISABLE;
md.user_ptr = frag; /* keep a pointer to ourselves */
md.eq_handle = mca_btl_portals_module.portals_eq_handles[OMPI_BTL_PORTALS_EQ_SEND];
ret = PtlMDAttach(me_h,
md,
PTL_UNLINK,
&(frag->md_h));
if (PTL_OK != ret) {
opal_output(mca_btl_portals_component.portals_output,
"Error creating rdma dest MD: %d", ret);
PtlMEUnlink(me_h);
OPAL_THREAD_ADD32(&mca_btl_portals_module.portals_outstanding_ops, -1);
OMPI_BTL_PORTALS_FRAG_RETURN_USER(&mca_btl_portals_module.super, frag);
return NULL;
}
frag->base.order = MCA_BTL_NO_ORDER;
return &frag->base;
}
int
mca_btl_portals_finalize(struct mca_btl_base_module_t *btl_base)
{
int ret;
assert(&mca_btl_portals_module == (mca_btl_portals_module_t*) btl_base);
OPAL_OUTPUT_VERBOSE((90, mca_btl_portals_component.portals_output,
"in mca_btl_portals_finalize"));
/* sanity check */
assert(mca_btl_portals_module.portals_outstanding_ops >= 0);
/* finalize all communication */
while (mca_btl_portals_module.portals_outstanding_ops > 0) {
OPAL_OUTPUT_VERBOSE((90, mca_btl_portals_component.portals_output,
"portals_outstanding_ops: %d",
mca_btl_portals_module.portals_outstanding_ops));
mca_btl_portals_component_progress();
}
if (mca_btl_portals_module.portals_num_procs != 0) {
int i;
ret = mca_btl_portals_recv_disable(&mca_btl_portals_module);
/* destroy eqs */
for (i = 0 ; i < OMPI_BTL_PORTALS_EQ_SIZE ; ++i) {
int ptl_ret = PtlEQFree(mca_btl_portals_module.portals_eq_handles[i]);
if (PTL_OK != ptl_ret) {
#if (OMPI_PORTALS_CRAYXT3 || OMPI_PORTALS_CRAYXT3_MODEX)
if (i != OMPI_BTL_PORTALS_EQ_SEND && PTL_EQ_IN_USE != ptl_ret) {
/* The PML isn't great about cleaning up after itself.
Ignore related errors. */
#endif
opal_output(mca_btl_portals_component.portals_output,
"Error freeing EQ %d: %d", i, ptl_ret);
#if (OMPI_PORTALS_CRAYXT3 || OMPI_PORTALS_CRAYXT3_MODEX)
}
#endif
}
}
}
OBJ_DESTRUCT(&mca_btl_portals_module.portals_recv_blocks);
OBJ_DESTRUCT(&mca_btl_portals_module.portals_recv_frag);
OBJ_DESTRUCT(&mca_btl_portals_module.portals_frag_eager);
OBJ_DESTRUCT(&mca_btl_portals_module.portals_frag_max);
OBJ_DESTRUCT(&mca_btl_portals_module.portals_frag_user);
ompi_common_portals_ni_finalize();
ompi_common_portals_finalize();
opal_output_verbose(20, mca_btl_portals_component.portals_output,
"successfully finalized module");
return OMPI_SUCCESS;
}

234
ompi/mca/btl/portals/btl_portals.h
Просмотреть файл

@ -1,234 +0,0 @@
/*
* Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
* University Research and Technology
* Corporation. All rights reserved.
* Copyright (c) 2004-2009 The University of Tennessee and The University
* of Tennessee Research Foundation. All rights
* reserved.
* Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
* University of Stuttgart. All rights reserved.
* Copyright (c) 2004-2005 The Regents of the University of California.
* All rights reserved.
* Copyright (c) 2008 UT-Battelle, LLC. All rights reserved.
* $COPYRIGHT$
*
* Additional copyrights may follow
*
* $HEADER$
*/
/*
* @file
*/
#ifndef OMPI_BTL_PORTALS_H
#define OMPI_BTL_PORTALS_H
#include "ompi_config.h"
#include "ompi/mca/common/portals/common_portals.h"
#include "ompi/mca/btl/btl.h"
#include "ompi/mca/btl/base/base.h"
#include "ompi/class/ompi_free_list.h"
#include "btl_portals_endpoint.h"
#include "btl_portals_frag.h"
/*
* Portals BTL component.
*/
struct mca_btl_portals_component_t {
/* base BTL component */
mca_btl_base_component_2_0_0_t super;
/* output channel for debugging. Value settings when using
* output_verbose:
*
* - 0 : critical user information
* - 10: general execution diagnostic information
* - 20: initialization / shutdown diagnostic information
* - 30: basic debugging information
* - 90: useful only to developers
* - 100: lots and lots of performance impacting output
*/
int portals_output;
/* initial size of free lists */
int portals_free_list_init_num;
/* max size of free lists */
int portals_free_list_max_num;
/* numer of elements to grow free lists */
int portals_free_list_inc_num;
/* number of eager fragments */
int portals_free_list_eager_max_num;
/* shall I use portals to send to thyself? */
int portals_support_self;
/* do I need a portals ACK? */
int portals_need_ack;
};
typedef struct mca_btl_portals_component_t mca_btl_portals_component_t;
#define OMPI_BTL_PORTALS_EQ_SEND 0
#define OMPI_BTL_PORTALS_EQ_RECV 1
#define OMPI_BTL_PORTALS_EQ_SIZE 2
struct mca_btl_portals_module_t {
/* base BTL module interface */
mca_btl_base_module_t super;
/* number of processes we're actively connected to. Needed to
know when to do activation / shutdown */
int32_t portals_num_procs;
/* fragment free lists */
ompi_free_list_t portals_frag_eager;
ompi_free_list_t portals_frag_max;
ompi_free_list_t portals_frag_user;
/* incoming send message receive memory descriptors */
int portals_recv_mds_num;
int portals_recv_mds_size;
opal_list_t portals_recv_blocks;
/* frag for receive callbacks */
mca_btl_portals_frag_recv_t portals_recv_frag;
/* event queues. Keep sends on own eq, since we can't control
space for the ack otherwise */
int portals_eq_sizes[OMPI_BTL_PORTALS_EQ_SIZE];
ptl_handle_eq_t portals_eq_handles[OMPI_BTL_PORTALS_EQ_SIZE];
/* "reject" entry for recv match list */
ptl_handle_me_t portals_recv_reject_me_h;
/* number outstanding sends and local rdma */
volatile int32_t portals_outstanding_ops;
int32_t portals_max_outstanding_ops;
/* sends queued until there's time to send */
opal_list_t portals_queued_sends;
/* key to use for next rdma operation */
volatile int64_t portals_rdma_key;
/* our portals network interface */
ptl_handle_ni_t portals_ni_h;
/* number of dropped messages */
ptl_sr_value_t portals_sr_dropped;
/* descriptors for send */
ptl_md_t md_send;
};
typedef struct mca_btl_portals_module_t mca_btl_portals_module_t;
/*
* Component functions (btl_portals_component.c)
*/
int mca_btl_portals_component_open(void);
int mca_btl_portals_component_close(void);
mca_btl_base_module_t** mca_btl_portals_component_init(int *num_btls,
bool has_progress_threads,
bool has_mpi_threads);
int mca_btl_portals_component_progress(void);
/*
* Compatibility functions (btl_portals_compat_{}.c)
*
* Not part of the BTL interface. Need to be implemented for every
* version of Portals
*/
int mca_btl_portals_init_compat(mca_btl_portals_component_t *comp);
/* 4th argument is a ptl_peers array, as that's what we'll get back
from many of the access functions... */
int mca_btl_portals_add_procs_compat(mca_btl_portals_module_t* btl,
size_t nprocs, struct ompi_proc_t **procs,
ptl_process_id_t **ptl_peers);
/*
* Module configuration functions (btl_portals.c)
*/
int mca_btl_portals_finalize(struct mca_btl_base_module_t* btl_base);
int mca_btl_portals_add_procs(struct mca_btl_base_module_t* btl_base,
size_t nprocs,
struct ompi_proc_t **procs,
struct mca_btl_base_endpoint_t** peers,
opal_bitmap_t* reachable);
int mca_btl_portals_del_procs(struct mca_btl_base_module_t* btl_base,
size_t nprocs,
struct ompi_proc_t **procs,
struct mca_btl_base_endpoint_t** peers);
mca_btl_base_descriptor_t*
mca_btl_portals_alloc(struct mca_btl_base_module_t* btl_base,
struct mca_btl_base_endpoint_t* endpoint,
uint8_t order,
size_t size,
uint32_t flags);
int mca_btl_portals_free(struct mca_btl_base_module_t* btl_base,
mca_btl_base_descriptor_t* des);
mca_btl_base_descriptor_t*
mca_btl_portals_prepare_src(struct mca_btl_base_module_t* btl_base,
struct mca_btl_base_endpoint_t* peer,
mca_mpool_base_registration_t* registration,
struct opal_convertor_t* convertor,
uint8_t order,
size_t reserve,
size_t* size,
uint32_t flags);
mca_btl_base_descriptor_t*
mca_btl_portals_prepare_dst(struct mca_btl_base_module_t* btl_base,
struct mca_btl_base_endpoint_t* peer,
mca_mpool_base_registration_t* registration,
struct opal_convertor_t* convertor,
uint8_t order,
size_t reserve,
size_t* size,
uint32_t flags);
int mca_btl_portals_send(struct mca_btl_base_module_t* btl_base,
struct mca_btl_base_endpoint_t* btl_peer,
struct mca_btl_base_descriptor_t* descriptor,
mca_btl_base_tag_t tag);
int mca_btl_portals_sendi(struct mca_btl_base_module_t* btl_base,
struct mca_btl_base_endpoint_t* endpoint,
struct opal_convertor_t* convertor,
void* header,
size_t header_size,
size_t payload_size,
uint8_t order,
uint32_t flags,
mca_btl_base_tag_t tag,
mca_btl_base_descriptor_t** des);
int mca_btl_portals_put(struct mca_btl_base_module_t* btl_base,
struct mca_btl_base_endpoint_t* btl_peer,
struct mca_btl_base_descriptor_t* decriptor);
int mca_btl_portals_get(struct mca_btl_base_module_t* btl_base,
struct mca_btl_base_endpoint_t* btl_peer,
struct mca_btl_base_descriptor_t* decriptor);
/*
* global structures
*/
OMPI_MODULE_DECLSPEC extern mca_btl_portals_component_t mca_btl_portals_component;
extern mca_btl_portals_module_t mca_btl_portals_module;
#endif

678
ompi/mca/btl/portals/btl_portals_component.c
Просмотреть файл

@ -1,678 +0,0 @@
/*
* Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
* University Research and Technology
* Corporation. All rights reserved.
* Copyright (c) 2004-2005 The University of Tennessee and The University
* of Tennessee Research Foundation. All rights
* reserved.
* Copyright (c) 2004-2008 High Performance Computing Center Stuttgart,
* University of Stuttgart. All rights reserved.
* Copyright (c) 2004-2005 The Regents of the University of California.
* All rights reserved.
* Copyright (c) 2008 UT-Battelle, LLC. All rights reserved.
* $COPYRIGHT$
*
* Additional copyrights may follow
*
* $HEADER$
*/
#include "ompi_config.h"
#include <sys/types.h>
#include <unistd.h>
#include <limits.h>
#include "ompi/constants.h"
#include "opal/mca/base/mca_base_param.h"
#include "ompi/mca/common/portals/common_portals.h"
#include "btl_portals.h"
#include "btl_portals_frag.h"
#include "btl_portals_send.h"
#include "btl_portals_recv.h"
mca_btl_portals_component_t mca_btl_portals_component = {
{
/* First, the mca_base_module_t struct containing meta
information about the module itself */
{
MCA_BTL_BASE_VERSION_2_0_0,
"portals", /* MCA module name */
OMPI_MAJOR_VERSION, /* MCA module major version */
OMPI_MINOR_VERSION, /* MCA module minor version */
OMPI_RELEASE_VERSION, /* MCA module release version */
mca_btl_portals_component_open, /* module open */
mca_btl_portals_component_close /* module close */
},
{
/* The component is not checkpoint ready */
MCA_BASE_METADATA_PARAM_NONE
},
mca_btl_portals_component_init,
mca_btl_portals_component_progress,
}
};
static opal_output_stream_t portals_output_stream;
int
mca_btl_portals_component_open(void)
{
int i;
ompi_common_portals_register_mca();
/*
* get configured state for component
*/
/* start up debugging output */
OBJ_CONSTRUCT(&portals_output_stream, opal_output_stream_t);
portals_output_stream.lds_is_debugging = true;
portals_output_stream.lds_want_stdout = true;
portals_output_stream.lds_file_suffix = "btl-portals";
mca_base_param_reg_int(&mca_btl_portals_component.super.btl_version,
"debug_level",
"Debugging verbosity (0 - 100)",
false,
false,
0,
&(portals_output_stream.lds_verbose_level));
asprintf(&(portals_output_stream.lds_prefix),
"btl: portals (%s): ", ompi_common_portals_nodeid());
mca_btl_portals_component.portals_output =
opal_output_open(&portals_output_stream);
mca_base_param_reg_int(&mca_btl_portals_component.super.btl_version,
"free_list_init_num",
"Initial number of elements to initialize in free lists",
false,
false,
16,
&(mca_btl_portals_component.portals_free_list_init_num));
mca_base_param_reg_int(&mca_btl_portals_component.super.btl_version,
"free_list_max_num",
"Max number of elements to initialize in free lists",
false,
false,
1024,
&(mca_btl_portals_component.portals_free_list_max_num));
mca_base_param_reg_int(&mca_btl_portals_component.super.btl_version,
"free_list_inc_num",
"Increment count for free lists",
false,
false,
16,
&(mca_btl_portals_component.portals_free_list_inc_num));
mca_base_param_reg_int(&mca_btl_portals_component.super.btl_version,
"eager_frag_limit",
"Maximum number of pre-pinned eager fragments",
false,
false,
32,
&(mca_btl_portals_component.portals_free_list_eager_max_num));
mca_base_param_reg_int(&mca_btl_portals_component.super.btl_version,
"support_self",
"Use portals for send to self",
false,
false,
1, /* default to true.. */
&(mca_btl_portals_component.portals_support_self));
mca_base_param_reg_int(&mca_btl_portals_component.super.btl_version,
"needs_ack",
"Require a portals level ACK",
false,
false,
1, /* default to true.. */
&(mca_btl_portals_component.portals_need_ack));
/*
* fill default module state
*/
mca_btl_portals_module.super.btl_exclusivity = 60;
mca_btl_portals_module.super.btl_eager_limit = 32 * 1024;
mca_btl_portals_module.super.btl_rndv_eager_limit = 32 * 1024;
mca_btl_portals_module.super.btl_max_send_size = 64 * 1024;
mca_btl_portals_module.super.btl_rdma_pipeline_send_length = 64 * 1024;
mca_btl_portals_module.super.btl_rdma_pipeline_frag_size = INT_MAX;
mca_btl_portals_module.super.btl_min_rdma_pipeline_size = 0;
mca_btl_portals_module.super.btl_flags =
MCA_BTL_FLAGS_RDMA |
MCA_BTL_FLAGS_RDMA_MATCHED;
mca_btl_portals_module.super.btl_seg_size = sizeof (mca_btl_portals_segment_t);
mca_btl_portals_module.super.btl_bandwidth = 1000;
mca_btl_portals_module.super.btl_latency = 0;
mca_btl_base_param_register(&mca_btl_portals_component.super.btl_version,
&mca_btl_portals_module.super);
/* send in place actually increases our latency because we have to
hold on to the buffer until we're done with it, rather than
copy and send. So don't use it for now. */
mca_btl_portals_module.portals_num_procs = 0;
for (i = 0 ; i < OMPI_BTL_PORTALS_EQ_SIZE ; ++i) {
mca_btl_portals_module.portals_eq_sizes[i] = 0;
mca_btl_portals_module.portals_eq_handles[i] = PTL_EQ_NONE;
}
/* eq handles will be created when the module is instantiated.
Set sizes here */
mca_base_param_reg_int(&mca_btl_portals_component.super.btl_version,
"eq_recv_size",
"Size of the receive event queue",
false,
false,
16 * 1024,
&(mca_btl_portals_module.portals_eq_sizes[OMPI_BTL_PORTALS_EQ_RECV]));
mca_base_param_reg_int(&mca_btl_portals_component.super.btl_version,
"max_pending_ops",
"Maximum number of pending send/rdma frags",
false,
false,
8 * 1024,
&(mca_btl_portals_module.portals_max_outstanding_ops));
/* ops_pending * 2 for end, ack */
mca_btl_portals_module.portals_eq_sizes[OMPI_BTL_PORTALS_EQ_SEND] =
mca_btl_portals_module.portals_max_outstanding_ops * 2;
mca_btl_portals_module.portals_recv_reject_me_h = PTL_INVALID_HANDLE;
mca_base_param_reg_int(&mca_btl_portals_component.super.btl_version,
"recv_md_num",
"Number of send frag receive descriptors",
false,
false,
3,
&(mca_btl_portals_module.portals_recv_mds_num));
mca_base_param_reg_int(&mca_btl_portals_component.super.btl_version,
"recv_md_size",
"Size of send frag receive descriptors",
false,
false,
10 * 1024 * 1024,
&(mca_btl_portals_module.portals_recv_mds_size));
mca_btl_portals_module.portals_ni_h = PTL_INVALID_HANDLE;
mca_btl_portals_module.portals_sr_dropped = 0;
mca_btl_portals_module.portals_outstanding_ops = 0;
mca_btl_portals_module.portals_rdma_key = 1;
return OMPI_SUCCESS;
}
int
mca_btl_portals_component_close(void)
{
/* release resources */
if (NULL != portals_output_stream.lds_prefix) {
free(portals_output_stream.lds_prefix);
}
/* close debugging stream */
opal_output_close(mca_btl_portals_component.portals_output);
mca_btl_portals_component.portals_output = -1;
return OMPI_SUCCESS;
}
mca_btl_base_module_t**
mca_btl_portals_component_init(int *num_btls,
bool enable_progress_threads,
bool enable_mpi_threads)
{
mca_btl_base_module_t ** btls = malloc(sizeof(mca_btl_base_module_t*));
bool accel;
btls[0] = (mca_btl_base_module_t*) &mca_btl_portals_module;
if (enable_progress_threads || enable_mpi_threads) {
opal_output_verbose(20, mca_btl_portals_component.portals_output,
"disabled because threads enabled");
return NULL;
}
/* initialize portals btl. note that this is in the compat code because
it's fairly non-portable between implementations */
if (OMPI_SUCCESS != ompi_common_portals_initialize(&mca_btl_portals_module.portals_ni_h, &accel)) {
opal_output_verbose(20, mca_btl_portals_component.portals_output,
"disabled because compatibility init failed");
return NULL;
}
OBJ_CONSTRUCT(&(mca_btl_portals_module.portals_frag_eager), ompi_free_list_t);
OBJ_CONSTRUCT(&(mca_btl_portals_module.portals_frag_max), ompi_free_list_t);
OBJ_CONSTRUCT(&(mca_btl_portals_module.portals_frag_user), ompi_free_list_t);
/* eager frags */
ompi_free_list_init_new(&(mca_btl_portals_module.portals_frag_eager),
sizeof(mca_btl_portals_frag_eager_t) +
mca_btl_portals_module.super.btl_eager_limit,
opal_cache_line_size,
OBJ_CLASS(mca_btl_portals_frag_eager_t),
0,opal_cache_line_size,
mca_btl_portals_component.portals_free_list_init_num,
mca_btl_portals_component.portals_free_list_eager_max_num,
mca_btl_portals_component.portals_free_list_inc_num,
NULL);
/* send frags */
ompi_free_list_init_new(&(mca_btl_portals_module.portals_frag_max),
sizeof(mca_btl_portals_frag_max_t) +
mca_btl_portals_module.super.btl_max_send_size,
opal_cache_line_size,
OBJ_CLASS(mca_btl_portals_frag_max_t),
0,opal_cache_line_size,
mca_btl_portals_component.portals_free_list_init_num,
mca_btl_portals_component.portals_free_list_max_num,
mca_btl_portals_component.portals_free_list_inc_num,
NULL);
/* user frags */
ompi_free_list_init_new(&(mca_btl_portals_module.portals_frag_user),
sizeof(mca_btl_portals_frag_user_t),
opal_cache_line_size,
OBJ_CLASS(mca_btl_portals_frag_user_t),
0,opal_cache_line_size,
mca_btl_portals_component.portals_free_list_init_num,
mca_btl_portals_component.portals_free_list_max_num,
mca_btl_portals_component.portals_free_list_inc_num,
NULL);
/* recv frags */
OBJ_CONSTRUCT(&(mca_btl_portals_module.portals_recv_frag),
mca_btl_portals_frag_recv_t);
/* receive block list */
OBJ_CONSTRUCT(&(mca_btl_portals_module.portals_recv_blocks), opal_list_t);
/* list for send requests that have to be delayed */
OBJ_CONSTRUCT(&(mca_btl_portals_module.portals_queued_sends),
opal_list_t);
*num_btls = 1;
opal_output_verbose(20, mca_btl_portals_component.portals_output,
"initialized Portals module");
return btls;
}
int
mca_btl_portals_component_progress(void)
{
int num_progressed = 0;
int ret, which, btl_ownership;
static ptl_event_t ev;
mca_btl_portals_frag_t *frag = NULL;
mca_btl_portals_recv_block_t *block = NULL;
mca_btl_base_tag_t tag;
mca_btl_base_segment_t seg[2];
if (0 == mca_btl_portals_module.portals_num_procs) {
return 0;
}
while (true) {
ret = PtlEQPoll(mca_btl_portals_module.portals_eq_handles,
OMPI_BTL_PORTALS_EQ_SIZE,
0, /* timeout */
&ev, /* event structure to update */
&which); /* which queue the event came from - we don't care */
switch (ret) {
case PTL_OK:
frag = ev.md.user_ptr;
btl_ownership = (frag->base.des_flags & MCA_BTL_DES_FLAGS_BTL_OWNERSHIP);
num_progressed++;
switch (ev.type) {
case PTL_EVENT_GET_START:
/* generated on source (target) when a get from memory starts */
OPAL_OUTPUT_VERBOSE((90, mca_btl_portals_component.portals_output,
"PTL_EVENT_GET_START for 0x%lx, %d",
(unsigned long) frag, (int) ev.hdr_data));
break;
case PTL_EVENT_GET_END:
/* generated on source (target) when a get from memory ends */
OPAL_OUTPUT_VERBOSE((90, mca_btl_portals_component.portals_output,
"PTL_EVENT_GET_END for 0x%lx, %d, flags %d",
(unsigned long) frag, (int) ev.hdr_data,
frag->base.des_flags));
if( btl_ownership ) {
OPAL_OUTPUT_VERBOSE((90, mca_btl_portals_component.portals_output,
"in PTL_EVENT_GET_END received a frag with btl_ownership!"));
mca_btl_portals_free(&mca_btl_portals_module.super,
&frag->base);
}
break;
case PTL_EVENT_PUT_START:
tag = ((unsigned char*) (&ev.hdr_data))[7];
/* generated on destination (target) when a put into memory starts */
OPAL_OUTPUT_VERBOSE((90, mca_btl_portals_component.portals_output,
"PTL_EVENT_PUT_START for 0x%lx, %d",
(unsigned long) frag, (int) tag));
#if OPAL_ENABLE_DEBUG
if (ev.ni_fail_type != PTL_NI_OK) {
opal_output(mca_btl_portals_component.portals_output,
"Failure to start event\n");
return OMPI_ERROR;
}
#endif
/* if it's a pending unexpected receive, do book keeping. */
if (tag < MCA_BTL_TAG_MAX) {
block = ev.md.user_ptr;
OPAL_THREAD_ADD32(&(block->pending), 1);
}
break;
case PTL_EVENT_PUT_END:
tag = ((unsigned char*) (&ev.hdr_data))[7];
/* generated on destination (target) when a put into memory ends */
OPAL_OUTPUT_VERBOSE((90, mca_btl_portals_component.portals_output,
"PTL_EVENT_PUT_END for 0x%lx, %d",
(unsigned long) frag, (int) tag));
#if OPAL_ENABLE_DEBUG
if (ev.ni_fail_type != PTL_NI_OK) {
opal_output(mca_btl_portals_component.portals_output,
"Failure to end event\n");
mca_btl_portals_return_block_part(&mca_btl_portals_module,
block);
return OMPI_ERROR;
}
#endif
/* if it's an unexpected receive, do book keeping and send to PML */
if (tag < MCA_BTL_TAG_MAX) {
block = ev.md.user_ptr;
frag = &mca_btl_portals_module.portals_recv_frag;
if(ev.match_bits) {
uint8_t header_size = ((uint8_t*) (&ev.hdr_data))[6];
memcpy(frag->data, &ev.match_bits, header_size > 8 ? 8 : header_size);
if(header_size > 8) {
memcpy(frag->data+8, &ev.hdr_data, header_size - 8);
}
OPAL_OUTPUT_VERBOSE((90, mca_btl_portals_component.portals_output,"received %x %x %x %x %x %x %x %x %x %x : header_size %x : tag %x \n",
frag->data[0],
frag->data[1],
frag->data[2],
frag->data[3],
frag->data[4],
frag->data[5],
frag->data[6],
frag->data[7],
frag->data[8],
frag->data[9],
header_size,
tag
));
OPAL_OUTPUT_VERBOSE((90, mca_btl_portals_component.portals_output,"received %d bytes \n", (int) ev.mlength));
frag->base.des_dst = seg;
seg[0].seg_addr.pval = &frag->data;
seg[0].seg_len = header_size;
if(ev.mlength) {
seg[1].seg_addr.pval = ((((char*) ev.md.start) + ev.offset));
seg[1].seg_len = ev.mlength;
frag->base.des_dst_cnt = 2;
} else {
frag->base.des_dst_cnt = 1;
}
} else {
/* if we ever make this thread hot, need to do
something with the receive fragments */
seg[0].seg_addr.pval = (((char*) ev.md.start) + ev.offset);
seg[0].seg_len = ev.mlength;
OPAL_OUTPUT_VERBOSE((90, mca_btl_portals_component.portals_output,
"received send fragment 0x%lx (thresh: %d, length %d)",
(unsigned long) frag,
ev.md.threshold, (int) ev.mlength));
frag->base.des_dst_cnt = 1;
}
if (ev.md.length - (ev.offset + ev.mlength) < (ptl_size_t) ev.md.max_size ||
ev.md.threshold == 1) {
/* the block is full. It's deactivated automagically, but we
can't start it up again until everyone is done with it.
The actual reactivation and all that will happen after the
free completes the last operation... */
OPAL_OUTPUT_VERBOSE((90, mca_btl_portals_component.portals_output,
"marking block 0x%lx as full",
(unsigned long) block->start));
block->full = true;
}
/* NTH: is it ok to overwrite this. All callbacks should expect base segments */
frag->base.des_dst = seg;
mca_btl_base_active_message_trigger[tag].cbfunc(
&mca_btl_portals_module.super,
tag,
&frag->base,
mca_btl_base_active_message_trigger[tag].cbdata);
mca_btl_portals_return_block_part(&mca_btl_portals_module, block);
}
break;
case PTL_EVENT_REPLY_START:
/* generated on destination (origin) when a get starts
returning data */
OPAL_OUTPUT_VERBOSE((90, mca_btl_portals_component.portals_output,
"PTL_EVENT_REPLY_START for 0x%lx, %d",
(unsigned long) frag, (int) ev.hdr_data));
break;
case PTL_EVENT_REPLY_END:
/* generated on destination (origin) when a get is
done returning data */
OPAL_OUTPUT_VERBOSE((90, mca_btl_portals_component.portals_output,
"PTL_EVENT_REPLY_END for 0x%lx",
(unsigned long) frag));
frag->base.des_cbfunc(&mca_btl_portals_module.super,
frag->endpoint,
&frag->base,
OMPI_SUCCESS);
if( btl_ownership ) {
OPAL_OUTPUT_VERBOSE((90, mca_btl_portals_component.portals_output,
"in PTL_EVENT_REPLY_END received a frag with btl_ownership!"));
mca_btl_portals_free(&mca_btl_portals_module.super,
&frag->base);
}
break;
case PTL_EVENT_SEND_START:
/* generated on source (origin) when put starts sending */
#if OPAL_ENABLE_DEBUG
OPAL_OUTPUT_VERBOSE((90, mca_btl_portals_component.portals_output,
"PTL_EVENT_SEND_START for 0x%lx, %d",
(unsigned long) frag, (int) ev.hdr_data));
if (ev.ni_fail_type != PTL_NI_OK) {
opal_output(mca_btl_portals_component.portals_output,
"Failure to start send event\n");
frag->base.des_cbfunc(&mca_btl_portals_module.super,
frag->endpoint,
&frag->base,
OMPI_ERROR);
if( btl_ownership ) {
mca_btl_portals_free(&mca_btl_portals_module.super,
&frag->base);
}
}
#endif
break;
case PTL_EVENT_SEND_END:
/* generated on source (origin) when put stops sending */
#if OPAL_ENABLE_DEBUG
OPAL_OUTPUT_VERBOSE((90, mca_btl_portals_component.portals_output,
"PTL_EVENT_SEND_END for 0x%lx, %d",
(unsigned long) frag, (int) ev.hdr_data));
if (ev.ni_fail_type != PTL_NI_OK) {
opal_output(mca_btl_portals_component.portals_output,
"Failure to end send event\n");
if( MCA_BTL_DES_SEND_ALWAYS_CALLBACK & frag->base.des_flags ){
frag->base.des_cbfunc(&mca_btl_portals_module.super,
frag->endpoint,
&frag->base,
OMPI_ERROR);
}
if( btl_ownership ) {
mca_btl_portals_free(&mca_btl_portals_module.super,
&frag->base);
}
}
#endif
if(!mca_btl_portals_component.portals_need_ack) {
/* my part's done, in portals we trust! */
if( MCA_BTL_DES_SEND_ALWAYS_CALLBACK & frag->base.des_flags ){
frag->base.des_cbfunc(&mca_btl_portals_module.super,
frag->endpoint,
&frag->base,
OMPI_SUCCESS);
}
if( btl_ownership ) {
mca_btl_portals_free(&mca_btl_portals_module.super,
&frag->base);
}
if (0 != frag->size) {
OPAL_THREAD_ADD32(&mca_btl_portals_module.portals_outstanding_ops,
-1);
MCA_BTL_PORTALS_PROGRESS_QUEUED_SENDS();
}
}
break;
case PTL_EVENT_ACK:
/* ack that a put as completed on other side */
/* ACK for either send or RDMA put. Either way, we
just call the callback function on goodness.
Requeue the put on badness */
OPAL_OUTPUT_VERBOSE((90, mca_btl_portals_component.portals_output,
"PTL_EVENT_ACK for 0x%lx",
(unsigned long) frag));
#if OPAL_ENABLE_DEBUG
if(!mca_btl_portals_component.portals_need_ack) {
opal_output(mca_btl_portals_component.portals_output,
"Received PTL_EVENT_ACK but ACK's are disabled!\n");
abort();
}
if (ev.ni_fail_type != PTL_NI_OK) {
opal_output(mca_btl_portals_component.portals_output,
"Failure to ack event\n");
if( MCA_BTL_DES_SEND_ALWAYS_CALLBACK & frag->base.des_flags ){
frag->base.des_cbfunc(&mca_btl_portals_module.super,
frag->endpoint,
&frag->base,
OMPI_ERROR);
}
if( btl_ownership ) {
mca_btl_portals_free(&mca_btl_portals_module.super,
&frag->base);
}
} else
#endif
if ((ev.rlength != ev.mlength) || (ev.rlength == 0)) {
/* other side received message but truncated to 0.
This should only happen for unexpected
messages, and only when the other side has no
buffer space available for receiving */
if (ev.rlength == 0)
opal_output_verbose(10, mca_btl_portals_component.portals_output,
"PTL_EVENT_ACK: ev.rlength=%lu ev.mlength=%lu: requeueing request",
(unsigned long)ev.rlength, (unsigned long)ev.mlength);
else
opal_output_verbose(50, mca_btl_portals_component.portals_output,
"message was dropped. Trying again");
mca_btl_portals_send(&mca_btl_portals_module.super,
frag->endpoint,
&frag->base,
frag->hdr.tag);
} else {
/* other side received the message. should have
received entire thing */
/* let the PML know we're done */
if( MCA_BTL_DES_SEND_ALWAYS_CALLBACK & frag->base.des_flags ) {
frag->base.des_cbfunc(&mca_btl_portals_module.super,
frag->endpoint,
&frag->base,
OMPI_SUCCESS);
}
if( btl_ownership ) {
mca_btl_portals_free(&mca_btl_portals_module.super,
&frag->base);
}
}
if (0 != frag->size) {
OPAL_THREAD_ADD32(&mca_btl_portals_module.portals_outstanding_ops,
-1);
MCA_BTL_PORTALS_PROGRESS_QUEUED_SENDS();
}
break;
default:
break;
}
break;
case PTL_EQ_EMPTY:
/* there's nothing in the queue. This is actually the
common case, so the easiest way to make the compiler
emit something that doesn't completely blow here is to
just to go back to a good old goto */
goto done;
break;
case PTL_EQ_DROPPED:
/* not sure how we could deal with this more gracefully */
opal_output(mca_btl_portals_component.portals_output,
"WARNING: EQ events dropped. Too many messages pending.");
opal_output(mca_btl_portals_component.portals_output,
"WARNING: Giving up in dispair");
abort();
break;
default:
opal_output(mca_btl_portals_component.portals_output,
"WARNING: Error in PtlEQPoll (%d). This shouldn't happen",
ret);
abort();
break;
}
}
done:
return num_progressed;
}

37
ompi/mca/btl/portals/btl_portals_endpoint.h
Просмотреть файл

@ -1,37 +0,0 @@
/*
* Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
* University Research and Technology
* Corporation. All rights reserved.
* Copyright (c) 2004-2005 The University of Tennessee and The University
* of Tennessee Research Foundation. All rights
* reserved.
* Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
* University of Stuttgart. All rights reserved.
* Copyright (c) 2004-2005 The Regents of the University of California.
* All rights reserved.
* $COPYRIGHT$
*
* Additional copyrights may follow
*
* $HEADER$
*/
#ifndef MCA_BTL_PORTALS_ENDPOINT_H
#define MCA_BTL_PORTALS_ENDPOINT_H
#include "btl_portals.h"
BEGIN_C_DECLS
/**
* An abstraction that represents a connection to a endpoint process.
* An instance of mca_btl_base_endpoint_t is associated w/ each process
* and BTL pair at startup. However, connections to the endpoint
* are established dynamically on an as-needed basis:
*/
typedef ptl_process_id_t mca_btl_base_endpoint_t;
typedef mca_btl_base_endpoint_t mca_btl_portals_endpoint_t;
END_C_DECLS
#endif /* MCA_BTL_PORTALS_ENDPOINT_H */

124
ompi/mca/btl/portals/btl_portals_frag.c
Просмотреть файл

@ -1,124 +0,0 @@
/*
* Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
* University Research and Technology
* Corporation. All rights reserved.
* Copyright (c) 2004-2005 The University of Tennessee and The University
* of Tennessee Research Foundation. All rights
* reserved.
* Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
* University of Stuttgart. All rights reserved.
* Copyright (c) 2004-2005 The Regents of the University of California.
* All rights reserved.
* Copyright (c) 2008 UT-Battelle, LLC. All rights reserved.
* $COPYRIGHT$
*
* Additional copyrights may follow
*
* $HEADER$
*/
#include "ompi_config.h"
#include "btl_portals.h"
#include "btl_portals_frag.h"
static void
mca_btl_portals_frag_common_send_constructor(mca_btl_portals_frag_t* frag)
{
frag->base.des_flags = 0;
frag->base.des_dst = 0;
frag->base.des_dst_cnt = 0;
frag->base.des_src = frag->segments;
frag->base.des_src_cnt = 2;
frag->segments[0].base.seg_addr.pval = frag + 1;
frag->segments[0].base.seg_len = frag->size;
frag->segments[0].key = 0;
frag->md_h = PTL_INVALID_HANDLE;
}
static void
mca_btl_portals_frag_eager_constructor(mca_btl_portals_frag_t* frag)
{
frag->size = mca_btl_portals_module.super.btl_eager_limit;
mca_btl_portals_frag_common_send_constructor(frag);
frag->type = BTL_PORTALS_FRAG_TYPE_EAGER;
}
static void
mca_btl_portals_frag_eager_destructor(mca_btl_portals_frag_t* frag)
{
if (PTL_INVALID_HANDLE == frag->md_h) {
PtlMDUnlink(frag->md_h);
frag->md_h = PTL_INVALID_HANDLE;
}
}
static void
mca_btl_portals_frag_max_constructor(mca_btl_portals_frag_t* frag)
{
frag->size = mca_btl_portals_module.super.btl_max_send_size;
mca_btl_portals_frag_common_send_constructor(frag);
frag->type = BTL_PORTALS_FRAG_TYPE_MAX;
}
static void
mca_btl_portals_frag_user_constructor(mca_btl_portals_frag_t* frag)
{
frag->base.des_flags = 0;
frag->base.des_dst = 0;
frag->base.des_dst_cnt = 0;
frag->base.des_src = 0;
frag->base.des_src_cnt = 0;
frag->size = 0;
frag->type = BTL_PORTALS_FRAG_TYPE_USER;
}
static void
mca_btl_portals_frag_recv_constructor(mca_btl_portals_frag_t* frag)
{
frag->base.des_flags = 0;
frag->base.des_dst = &frag->segments[0];
frag->base.des_dst_cnt = 1;
frag->base.des_src = NULL;
frag->base.des_src_cnt = 0;
frag->size = 0;
}
OBJ_CLASS_INSTANCE(
mca_btl_portals_frag_t,
mca_btl_base_descriptor_t,
NULL,
NULL);
OBJ_CLASS_INSTANCE(
mca_btl_portals_frag_eager_t,
mca_btl_base_descriptor_t,
mca_btl_portals_frag_eager_constructor,
mca_btl_portals_frag_eager_destructor);
OBJ_CLASS_INSTANCE(
mca_btl_portals_frag_max_t,
mca_btl_base_descriptor_t,
mca_btl_portals_frag_max_constructor,
NULL);
OBJ_CLASS_INSTANCE(
mca_btl_portals_frag_user_t,
mca_btl_base_descriptor_t,
mca_btl_portals_frag_user_constructor,
NULL);
OBJ_CLASS_INSTANCE(
mca_btl_portals_frag_recv_t,
mca_btl_base_descriptor_t,
mca_btl_portals_frag_recv_constructor,
NULL);

126
ompi/mca/btl/portals/btl_portals_frag.h
Просмотреть файл

@ -1,126 +0,0 @@
/*
* Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
* University Research and Technology
* Corporation. All rights reserved.
* Copyright (c) 2004-2006 The University of Tennessee and The University
* of Tennessee Research Foundation. All rights
* reserved.
* Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
* University of Stuttgart. All rights reserved.
* Copyright (c) 2004-2005 The Regents of the University of California.
* All rights reserved.
* Copyright (c) 2011-2012 Los Alamos National Security, LLC.
* All rights reserved.
* $COPYRIGHT$
*
* Additional copyrights may follow
*
* $HEADER$
*/
#ifndef OMPI_BTL_PORTALS_FRAG_H
#define OMPI_BTL_PORTALS_FRAG_H
BEGIN_C_DECLS
struct mca_btl_portals_segment_t {
mca_btl_base_segment_t base;
ptl_match_bits_t key;
};
typedef struct mca_btl_portals_segment_t mca_btl_portals_segment_t;
/**
* Portals send fragment derived type
*/
struct mca_btl_portals_frag_t {
mca_btl_base_descriptor_t base;
mca_btl_portals_segment_t segments[1];
/* needed for retransmit case */
struct mca_btl_base_endpoint_t *endpoint;
/* needed for retransmit case */
mca_btl_base_header_t hdr;
/* handle to use for communication */
ptl_handle_md_t md_h;
/* size of the allocated memory region -- not the amount of data
we need to send */
size_t size;
enum { BTL_PORTALS_FRAG_TYPE_EAGER,
BTL_PORTALS_FRAG_TYPE_MAX,
BTL_PORTALS_FRAG_TYPE_USER } type;
unsigned char data[16];
};
typedef struct mca_btl_portals_frag_t mca_btl_portals_frag_t;
OBJ_CLASS_DECLARATION(mca_btl_portals_frag_t);
typedef struct mca_btl_portals_frag_t mca_btl_portals_frag_eager_t;
OBJ_CLASS_DECLARATION(mca_btl_portals_frag_eager_t);
typedef struct mca_btl_portals_frag_t mca_btl_portals_frag_max_t;
OBJ_CLASS_DECLARATION(mca_btl_portals_frag_max_t);
typedef struct mca_btl_portals_frag_t mca_btl_portals_frag_user_t;
OBJ_CLASS_DECLARATION(mca_btl_portals_frag_user_t);
typedef struct mca_btl_portals_frag_t mca_btl_portals_frag_recv_t;
OBJ_CLASS_DECLARATION(mca_btl_portals_frag_recv_t);
/*
* Macros to allocate/return descriptors from module specific
* free list(s).
*/
#define OMPI_BTL_PORTALS_FRAG_ALLOC_EAGER(btl_macro, frag, rc) \
{ \
\
ompi_free_list_item_t *item; \
OMPI_FREE_LIST_GET(&((mca_btl_portals_module_t*)btl_macro)->portals_frag_eager, item, rc); \
frag = (mca_btl_portals_frag_t*) item; \
if (rc == OMPI_ERR_TEMP_OUT_OF_RESOURCE) { \
OMPI_BTL_PORTALS_FRAG_ALLOC_MAX(btl_macro, frag, rc); \
} \
}
#define OMPI_BTL_PORTALS_FRAG_RETURN_EAGER(btl_macro, frag) \
{ \
assert(BTL_PORTALS_FRAG_TYPE_EAGER == frag->type); \
OMPI_FREE_LIST_RETURN(&((mca_btl_portals_module_t*)btl_macro)->portals_frag_eager, \
(ompi_free_list_item_t*)(frag)); \
}
#define OMPI_BTL_PORTALS_FRAG_ALLOC_MAX(btl_macro, frag, rc) \
{ \
\
ompi_free_list_item_t *item_macro; \
OMPI_FREE_LIST_GET(&((mca_btl_portals_module_t*)btl_macro)->portals_frag_max, item_macro, rc); \
frag = (mca_btl_portals_frag_t*) item_macro; \
}
#define OMPI_BTL_PORTALS_FRAG_RETURN_MAX(btl_macro, frag) \
{ \
assert(BTL_PORTALS_FRAG_TYPE_MAX == frag->type); \
OMPI_FREE_LIST_RETURN(&((mca_btl_portals_module_t*)btl_macro)->portals_frag_max, \
(ompi_free_list_item_t*)(frag)); \
}
#define OMPI_BTL_PORTALS_FRAG_ALLOC_USER(btl_macro, frag, rc) \
{ \
ompi_free_list_item_t *item; \
OMPI_FREE_LIST_WAIT(&((mca_btl_portals_module_t*)btl_macro)->portals_frag_user, item, rc); \
frag = (mca_btl_portals_frag_t*) item; \
}
#define OMPI_BTL_PORTALS_FRAG_RETURN_USER(btl_macro, frag) \
{ \
assert(BTL_PORTALS_FRAG_TYPE_USER == frag->type); \
OMPI_FREE_LIST_RETURN(&((mca_btl_portals_module_t*)btl_macro)->portals_frag_user, \
(ompi_free_list_item_t*)(frag)); \
}
END_C_DECLS
#endif

103
ompi/mca/btl/portals/btl_portals_rdma.c
Просмотреть файл

@ -1,103 +0,0 @@
/*
* Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
* University Research and Technology
* Corporation. All rights reserved.
* Copyright (c) 2004-2005 The University of Tennessee and The University
* of Tennessee Research Foundation. All rights
* reserved.
* Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
* University of Stuttgart. All rights reserved.
* Copyright (c) 2004-2005 The Regents of the University of California.
* All rights reserved.
* Copyright (c) 2008 UT-Battelle, LLC. All rights reserved.
* $COPYRIGHT$
*
* Additional copyrights may follow
*
* $HEADER$
*/
#include "ompi_config.h"
#include <inttypes.h>
#include "ompi/constants.h"
#include "btl_portals.h"
#include "btl_portals_frag.h"
int
mca_btl_portals_put(struct mca_btl_base_module_t* btl_base,
struct mca_btl_base_endpoint_t* btl_peer,
struct mca_btl_base_descriptor_t* descriptor)
{
mca_btl_portals_segment_t *dst_seg = (mca_btl_portals_segment_t *) descriptor->des_dst;
mca_btl_portals_frag_t *frag = (mca_btl_portals_frag_t*) descriptor;
int ret;
unsigned char hdr_data[8];
OPAL_OUTPUT_VERBOSE((90, mca_btl_portals_component.portals_output,
"PtlPut (rdma) fragment %lx, bits %" PRIx64,
(unsigned long) frag, dst_seg->key));
assert(&mca_btl_portals_module == (mca_btl_portals_module_t*) btl_base);
assert(frag->md_h != PTL_INVALID_HANDLE);
frag->endpoint = btl_peer;
hdr_data[7] = frag->hdr.tag = MCA_BTL_TAG_MAX;
/* setup the send */
assert(1 == frag->base.des_src_cnt);
ret = PtlPut(frag->md_h,
(mca_btl_portals_component.portals_need_ack ? PTL_ACK_REQ : PTL_NO_ACK_REQ),
*((mca_btl_base_endpoint_t*) btl_peer),
OMPI_BTL_PORTALS_RDMA_TABLE_ID,
0, /* ac_index - not used*/
dst_seg->key, /* match bits */
0, /* remote offset - not used */
*((ptl_hdr_data_t*) hdr_data)); /* hdr_data: tag */
if (ret != PTL_OK) {
opal_output(mca_btl_portals_component.portals_output,
"PtlPut failed with error %d", ret);
return OMPI_ERROR;
}
return OMPI_SUCCESS;
}
int
mca_btl_portals_get(struct mca_btl_base_module_t* btl_base,
struct mca_btl_base_endpoint_t* btl_peer,
struct mca_btl_base_descriptor_t* descriptor)
{
mca_btl_portals_segment_t *src_seg = (mca_btl_portals_segment_t *) descriptor->des_src;
mca_btl_portals_frag_t *frag = (mca_btl_portals_frag_t*) descriptor;
int ret;
OPAL_OUTPUT_VERBOSE((90, mca_btl_portals_component.portals_output,
"PtlGet (rdma) fragment %lx, bits %" PRIx64,
(unsigned long) frag, src_seg->key));
assert(&mca_btl_portals_module == (mca_btl_portals_module_t*) btl_base);
assert(frag->md_h != PTL_INVALID_HANDLE);
frag->endpoint = btl_peer;
frag->hdr.tag = MCA_BTL_TAG_MAX;
ret = PtlGet(frag->md_h,
*((mca_btl_base_endpoint_t*) btl_peer),
OMPI_BTL_PORTALS_RDMA_TABLE_ID,
0, /* ac_index - not used*/
src_seg->key, /* match bits */
0); /* remote offset - not used */
if (ret != PTL_OK) {
opal_output(mca_btl_portals_component.portals_output,
"PtlGet failed with error %d", ret);
return OMPI_ERROR;
}
return OMPI_SUCCESS;
}

165
ompi/mca/btl/portals/btl_portals_recv.c
Просмотреть файл

@ -1,165 +0,0 @@
/*
* Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
* University Research and Technology
* Corporation. All rights reserved.
* Copyright (c) 2004-2005 The University of Tennessee and The University
* of Tennessee Research Foundation. All rights
* reserved.
* Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
* University of Stuttgart. All rights reserved.
* Copyright (c) 2004-2005 The Regents of the University of California.
* All rights reserved.
* $COPYRIGHT$
*
* Additional copyrights may follow
*
* $HEADER$
*/
#include "ompi_config.h"
#include "ompi/constants.h"
#include "btl_portals.h"
#include "btl_portals_recv.h"
#include "btl_portals_frag.h"
OBJ_CLASS_INSTANCE(mca_btl_portals_recv_block_t,
opal_list_item_t,
NULL, NULL);
int
mca_btl_portals_recv_enable(mca_btl_portals_module_t *btl)
{
ptl_md_t md;
ptl_handle_md_t md_h;
ptl_process_id_t any_proc = {PTL_NID_ANY, PTL_PID_ANY};
int ret;
int i;
uint64_t ignore_bits = ~((uint64_t) 0);
/* create the reject entry */
md.start = NULL;
md.length = 0;
md.threshold = PTL_MD_THRESH_INF;
md.max_size = 0;
md.options = PTL_MD_TRUNCATE | PTL_MD_OP_PUT;
md.user_ptr = NULL;
md.eq_handle = PTL_EQ_NONE;
opal_output_verbose(90, mca_btl_portals_component.portals_output,
"About to create reject entry");
ret = PtlMEAttach(btl->portals_ni_h,
OMPI_BTL_PORTALS_SEND_TABLE_ID,
any_proc,
0, /* match */
ignore_bits, /* ignore */
PTL_RETAIN,
PTL_INS_BEFORE,
&(btl->portals_recv_reject_me_h));
if (PTL_OK != ret) {
opal_output(mca_btl_portals_component.portals_output,
"Error creating recv reject ME: %d", ret);
return OMPI_ERROR;
}
ret = PtlMDAttach(btl->portals_recv_reject_me_h,
md,
PTL_RETAIN,
&md_h);
if (PTL_OK != ret) {
opal_output(mca_btl_portals_component.portals_output,
"Error attaching recv reject MD: %d", ret);
mca_btl_portals_recv_disable(btl);
return OMPI_ERROR;
}
/* create the recv blocks */
for (i = 0 ; i < btl->portals_recv_mds_num ; ++i) {
mca_btl_portals_recv_block_t *block =
mca_btl_portals_recv_block_init(btl);
if (NULL == block) {
mca_btl_portals_recv_disable(btl);
return OMPI_ERROR;
}
opal_list_append(&(btl->portals_recv_blocks),
(opal_list_item_t*) block);
mca_btl_portals_activate_block(block);
}
return OMPI_SUCCESS;
}
int
mca_btl_portals_recv_disable(mca_btl_portals_module_t *btl)
{
opal_list_item_t *item;
if (opal_list_get_size(&btl->portals_recv_blocks) > 0) {
while (NULL !=
(item = opal_list_remove_first(&btl->portals_recv_blocks))) {
mca_btl_portals_recv_block_t *block =
(mca_btl_portals_recv_block_t*) item;
mca_btl_portals_recv_block_free(block);
}
}
if (PTL_INVALID_HANDLE != btl->portals_recv_reject_me_h) {
/* destroy the reject entry */
PtlMEUnlink(btl->portals_recv_reject_me_h);
btl->portals_recv_reject_me_h = PTL_INVALID_HANDLE;
}
return OMPI_SUCCESS;
}
mca_btl_portals_recv_block_t*
mca_btl_portals_recv_block_init(mca_btl_portals_module_t *btl)
{
mca_btl_portals_recv_block_t *block;
block = OBJ_NEW(mca_btl_portals_recv_block_t);
block->btl = btl;
block->length = btl->portals_recv_mds_size;
block->start = malloc(block->length);
if (block->start == NULL) return NULL;
block->me_h = PTL_INVALID_HANDLE;
block->md_h = PTL_INVALID_HANDLE;
block->full = false;
block->pending = 0;
return block;
}
int
mca_btl_portals_recv_block_free(mca_btl_portals_recv_block_t *block)
{
/* need to clear out the md */
while (block->pending != 0) {
mca_btl_portals_component_progress();
}
if (PTL_INVALID_HANDLE != block->md_h) {
PtlMDUnlink(block->md_h);
block->md_h = PTL_INVALID_HANDLE;
}
if (NULL != block->start) {
free(block->start);
block->start = NULL;
}
block->length = 0;
block->full = false;
return OMPI_SUCCESS;
}

142
ompi/mca/btl/portals/btl_portals_recv.h
Просмотреть файл

@ -1,142 +0,0 @@
/*
* Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
* University Research and Technology
* Corporation. All rights reserved.
* Copyright (c) 2004-2005 The University of Tennessee and The University
* of Tennessee Research Foundation. All rights
* reserved.
* Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
* University of Stuttgart. All rights reserved.
* Copyright (c) 2004-2005 The Regents of the University of California.
* All rights reserved.
* $COPYRIGHT$
*
* Additional copyrights may follow
*
* $HEADER$
*/
#ifndef OMPI_BTL_PORTALS_RECV_H
#define OMPI_BTL_PORTALS_RECV_H
#include "btl_portals_frag.h"
struct mca_btl_portals_recv_block_t {
opal_list_item_t base;
mca_btl_portals_module_t *btl;
void *start;
size_t length;
ptl_handle_me_t me_h;
ptl_handle_md_t md_h;
volatile bool full;
volatile int32_t pending;
};
typedef struct mca_btl_portals_recv_block_t mca_btl_portals_recv_block_t;
OBJ_CLASS_DECLARATION(mca_btl_portals_recv_block_t);
int mca_btl_portals_recv_enable(mca_btl_portals_module_t *btl);
int mca_btl_portals_recv_disable(mca_btl_portals_module_t *btl);
/**
* Create a block of memory for receiving send messages. Must call
* activate_block on the returned block of memory before it will be
* active with the POrtals library
*
* Module lock must be held before calling this function
*/
mca_btl_portals_recv_block_t*
mca_btl_portals_recv_block_init(mca_btl_portals_module_t *btl);
/**
* Free a block of memory. Will remove the match entry, then progress
* Portals until the pending count is returned to 0. Will then free
* all resources associated with block.
*
* Module lock must be held before calling this function
*/
int mca_btl_portals_recv_block_free(mca_btl_portals_recv_block_t *block);
/**
* activate a block. Blocks that are full (have gone inactive) can be
* re-activated with this call. There is no need to hold the lock
* before calling this function
*/
static inline int
mca_btl_portals_activate_block(mca_btl_portals_recv_block_t *block)
{
int ret;
ptl_process_id_t any_proc = { PTL_NID_ANY, PTL_PID_ANY };
ptl_md_t md;
uint64_t ignore_bits = ~((uint64_t) 0);
/* if we have pending operations, something very, very, very bad
has happened... */
assert(block->pending == 0);
if (NULL == block->start) return OMPI_ERROR;
/* create match entry */
ret = PtlMEInsert(block->btl->portals_recv_reject_me_h,
any_proc,
0, /* match bits */
ignore_bits, /* ignore bits */
PTL_UNLINK,
PTL_INS_BEFORE,
&(block->me_h));
if (PTL_OK != ret) return OMPI_ERROR;
/* and the memory descriptor */
md.start = block->start;
md.length = block->length;
/* try to throttle incoming sends so that we don't overrun the incoming
queue size */
md.threshold = mca_btl_portals_module.portals_eq_sizes[OMPI_BTL_PORTALS_EQ_RECV] /
(mca_btl_portals_module.portals_recv_mds_num * 2);
md.max_size = block->btl->super.btl_max_send_size;
md.options = PTL_MD_OP_PUT | PTL_MD_MAX_SIZE;
md.user_ptr = block;
md.eq_handle = block->btl->portals_eq_handles[OMPI_BTL_PORTALS_EQ_RECV];
block->pending = 0;
block->full = false;
/* make sure that everyone sees the update on full value */
opal_atomic_mb();
ret = PtlMDAttach(block->me_h,
md,
PTL_UNLINK,
&(block->md_h));
if (PTL_OK != ret) {
PtlMEUnlink(block->me_h);
return OMPI_ERROR;
}
return OMPI_SUCCESS;
}
static inline void
mca_btl_portals_return_block_part(mca_btl_portals_module_t *btl,
mca_btl_portals_recv_block_t *block)
{
int ret;
OPAL_THREAD_ADD32(&(block->pending), -1);
if (block->full == true) {
if (block->pending == 0) {
ret = mca_btl_portals_activate_block(block);
if (OMPI_SUCCESS != ret) {
/* BWB - now what? */
}
}
}
}
#endif /* OMPI_BTL_PORTALS_RECV_H */

258
ompi/mca/btl/portals/btl_portals_send.c
Просмотреть файл

@ -1,258 +0,0 @@
/*
* Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
* University Research and Technology
* Corporation. All rights reserved.
* Copyright (c) 2004-2005 The University of Tennessee and The University
* of Tennessee Research Foundation. All rights
* reserved.
* Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
* University of Stuttgart. All rights reserved.
* Copyright (c) 2004-2005 The Regents of the University of California.
* All rights reserved.
* Copyright (c) 2008 UT-Battelle, LLC. All rights reserved.
* Copyright (c) 2012 Los Alamos National Security, LLC. All rights
* reserved.
* $COPYRIGHT$
*
* Additional copyrights may follow
*
* $HEADER$
*/
#include "ompi_config.h"
#include <assert.h>
#include "ompi/constants.h"
#include "opal/datatype/opal_convertor.h"
#include "btl_portals.h"
#include "btl_portals_send.h"
int
mca_btl_portals_send(struct mca_btl_base_module_t* btl_base,
struct mca_btl_base_endpoint_t* endpoint,
struct mca_btl_base_descriptor_t* descriptor,
mca_btl_base_tag_t tag)
{
mca_btl_portals_frag_t *frag = (mca_btl_portals_frag_t*) descriptor;
int ret;
unsigned char hdr_data[8];
assert(&mca_btl_portals_module == (mca_btl_portals_module_t*) btl_base);
frag->endpoint = endpoint;
hdr_data[7] = frag->hdr.tag = tag;
OPAL_OUTPUT_VERBOSE((90, mca_btl_portals_component.portals_output,
"PtlPut (send) fragment %lx",
(unsigned long) frag));
if (OPAL_THREAD_ADD32(&mca_btl_portals_module.portals_outstanding_ops, 1) >
mca_btl_portals_module.portals_max_outstanding_ops) {
/* no space - queue and continute */
opal_output_verbose(50, mca_btl_portals_component.portals_output,
"no space for message 0x%lx. Adding to back of queue",
(unsigned long) frag);
OPAL_THREAD_ADD32(&mca_btl_portals_module.portals_outstanding_ops, -1);
opal_list_append(&(mca_btl_portals_module.portals_queued_sends),
(opal_list_item_t*) frag);
}
if (frag->md_h == PTL_INVALID_HANDLE) {
/* setup the send - always describe entire fragment */
mca_btl_portals_module.md_send.start = frag->segments[0].base.seg_addr.pval;
mca_btl_portals_module.md_send.length =
0 == frag->size ? frag->segments[0].base.seg_len : frag->size;
#if OPAL_ENABLE_DEBUG
mca_btl_portals_module.md_send.options =
PTL_MD_EVENT_START_DISABLE;
#else
/* optimized build, we can get rid of the END event */
/* if we are using portals ACK's for completion */
mca_btl_portals_module.md_send.options =
(PTL_MD_EVENT_START_DISABLE |
(mca_btl_portals_component.portals_need_ack ? PTL_MD_EVENT_END_DISABLE : 0));
#endif
mca_btl_portals_module.md_send.user_ptr = frag; /* keep a pointer to ourselves */
/* make a free-floater */
ret = PtlMDBind(mca_btl_portals_module.portals_ni_h,
mca_btl_portals_module.md_send,
PTL_UNLINK,
&frag->md_h);
if (ret != PTL_OK) {
opal_output(mca_btl_portals_component.portals_output,
"PtlMDBind failed with error %d", ret);
return OMPI_ERROR;
}
}
OPAL_OUTPUT_VERBOSE((90, mca_btl_portals_component.portals_output,
"fragment info:\n"
"\tstart: 0x%lx\n"
"\tlen: %d",
(unsigned long) frag->segments[0].base.seg_addr.pval,
frag->segments[0].base.seg_len));
ret = PtlPutRegion(frag->md_h, /* memory descriptor */
0, /* fragment offset */
frag->segments[0].base.seg_len, /* fragment length */
(mca_btl_portals_component.portals_need_ack ? PTL_ACK_REQ : PTL_NO_ACK_REQ),
*((mca_btl_base_endpoint_t*) endpoint),
OMPI_BTL_PORTALS_SEND_TABLE_ID,
0, /* ac_index - not used */
0, /* match bits */
0, /* remote offset - not used */
*((ptl_hdr_data_t*) hdr_data)); /* hdr_data: tag */
if (ret != PTL_OK) {
opal_output(mca_btl_portals_component.portals_output,
"send: PtlPut failed with error %d", ret);
return OMPI_ERROR;
}
return OMPI_SUCCESS;
}
int mca_btl_portals_sendi(struct mca_btl_base_module_t* btl_base,
struct mca_btl_base_endpoint_t* endpoint,
struct opal_convertor_t* convertor,
void* header,
size_t header_size,
size_t payload_size,
uint8_t order,
uint32_t flags,
mca_btl_base_tag_t tag,
mca_btl_base_descriptor_t** des)
{
mca_btl_portals_frag_t *frag = NULL;
struct iovec iov;
unsigned int iov_count;
unsigned char match_bits[8];
unsigned char hdr_data[8];
int ret;
size_t max_data;
opal_output(0, "mca_btl_portals_sendi status is incomplete");
abort();
assert(&mca_btl_portals_module == (mca_btl_portals_module_t*) btl_base);
*des = NULL;
if (OPAL_THREAD_ADD32(&mca_btl_portals_module.portals_outstanding_ops, 1) >
mca_btl_portals_module.portals_max_outstanding_ops) {
/* no space - queue and continue */
opal_output_verbose(50, mca_btl_portals_component.portals_output,
"no resources left for send inline");
OPAL_THREAD_ADD32(&mca_btl_portals_module.portals_outstanding_ops, -1);
*des = mca_btl_portals_alloc(btl_base, endpoint, order,
payload_size + header_size, flags);
return OMPI_ERR_RESOURCE_BUSY;
} else if(14 < header_size) {
/* header is too big */
*des = mca_btl_portals_alloc(btl_base, endpoint, order,
payload_size + header_size, flags);
OPAL_THREAD_ADD32(&mca_btl_portals_module.portals_outstanding_ops, -1);
return OMPI_ERR_RESOURCE_BUSY;
}
assert (payload_size <= mca_btl_portals_module.super.btl_eager_limit);
OMPI_BTL_PORTALS_FRAG_ALLOC_EAGER(&mca_btl_portals_module, frag, ret);
if (OMPI_SUCCESS != ret) {
OPAL_THREAD_ADD32(&mca_btl_portals_module.portals_outstanding_ops, -1);
return OMPI_ERR_RESOURCE_BUSY;
}
frag->segments[0].base.seg_len = payload_size;
frag->base.des_src_cnt = 1;
frag->base.des_flags = flags;
frag->base.order = MCA_BTL_NO_ORDER;
frag->endpoint = endpoint;
if(payload_size) {
/* pack the data into the supplied buffer */
iov.iov_base = (IOVBASE_TYPE*)((unsigned char*)frag->segments[0].base.seg_addr.pval);
iov.iov_len = max_data = payload_size;
iov_count = 1;
(void)opal_convertor_pack( convertor,
&iov, &iov_count, &max_data);
assert(max_data == payload_size);
}
if(header_size) {
memcpy(match_bits, header, header_size > 8 ? 8 : header_size);
}
if(header_size > 8 ) {
memcpy(hdr_data, ((unsigned char*) header) + 8, header_size - 8);
}
hdr_data[6] = header_size;
hdr_data[7] = frag->hdr.tag = tag;
OPAL_OUTPUT_VERBOSE((90, mca_btl_portals_component.portals_output,
"PtlPut (send) fragment %lx",
(unsigned long) frag));
if (frag->md_h == PTL_INVALID_HANDLE) {
/* setup the send - always describe entire fragment */
mca_btl_portals_module.md_send.start = frag->segments[0].base.seg_addr.pval;
mca_btl_portals_module.md_send.length =
0 == frag->size ? frag->segments[0].base.seg_len : frag->size;
#if OPAL_ENABLE_DEBUG
mca_btl_portals_module.md_send.options =
PTL_MD_EVENT_START_DISABLE;
#else
/* optimized build, we can get rid of the END event */
/* if we are using portals ACK's for completion */
mca_btl_portals_module.md_send.options =
(PTL_MD_EVENT_START_DISABLE |
(mca_btl_portals_component.portals_need_ack ? PTL_MD_EVENT_END_DISABLE : 0));
#endif
mca_btl_portals_module.md_send.user_ptr = frag; /* keep a pointer to ourselves */
/* make a free-floater */
ret = PtlMDBind(mca_btl_portals_module.portals_ni_h,
mca_btl_portals_module.md_send,
PTL_UNLINK,
&frag->md_h);
if (ret != PTL_OK) {
opal_output(mca_btl_portals_component.portals_output,
"PtlMDBind failed with error %d", ret);
return OMPI_ERROR;
}
}
OPAL_OUTPUT_VERBOSE((90, mca_btl_portals_component.portals_output,
"fragment info:\n"
"\tstart: 0x%lx\n"
"\tlen: %d",
(unsigned long) frag->segments[0].base.seg_addr.lval,
frag->segments[0].seg_len));
ret = PtlPutRegion(frag->md_h, /* memory descriptor */
0, /* fragment offset */
frag->segments[0].base.seg_len, /* fragment length */
(mca_btl_portals_component.portals_need_ack ? PTL_ACK_REQ : PTL_NO_ACK_REQ),
*((mca_btl_base_endpoint_t*) endpoint),
OMPI_BTL_PORTALS_SEND_TABLE_ID,
0, /* ac_index - not used */
*((ptl_match_bits_t*) match_bits), /* match bits */
0, /* remote offset - not used */
*((ptl_hdr_data_t*) hdr_data)); /* hdr_data: tag */
if (ret != PTL_OK) {
opal_output(mca_btl_portals_component.portals_output,
"send: PtlPut failed with error %d", ret);
return OMPI_ERROR;
}
return OMPI_SUCCESS;
}

43
ompi/mca/btl/portals/btl_portals_send.h
Просмотреть файл

@ -1,43 +0,0 @@
/*
* Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
* University Research and Technology
* Corporation. All rights reserved.
* Copyright (c) 2004-2005 The University of Tennessee and The University
* of Tennessee Research Foundation. All rights
* reserved.
* Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
* University of Stuttgart. All rights reserved.
* Copyright (c) 2004-2005 The Regents of the University of California.
* All rights reserved.
* $COPYRIGHT$
*
* Additional copyrights may follow
*
* $HEADER$
*/
#ifndef OMPI_BTL_PORTALS_SEND_H
#define OMPI_BTL_PORTALS_SEND_H
#include "btl_portals_frag.h"
#define MCA_BTL_PORTALS_PROGRESS_QUEUED_SENDS() \
if ((0 != opal_list_get_size(&(mca_btl_portals_module.portals_queued_sends))) && \
(mca_btl_portals_module.portals_outstanding_ops < \
mca_btl_portals_module.portals_max_outstanding_ops)) { \
mca_btl_portals_frag_t *qfrag = (mca_btl_portals_frag_t*) \
opal_list_remove_first(&(mca_btl_portals_module.portals_queued_sends)); \
OPAL_OUTPUT_VERBOSE((90, mca_btl_portals_component.portals_output, \
"retransmit for frag 0x%lx, 0x%lx", \
(unsigned long) qfrag, \
(unsigned long) qfrag->base.des_cbfunc)); \
return mca_btl_portals_send(&mca_btl_portals_module.super, \
qfrag->endpoint, \
&(qfrag->base), \
qfrag->hdr.tag); \
} \
return OMPI_SUCCESS;
#endif /* OMPI_BTL_PORTALS_SEND_H */

48
ompi/mca/btl/portals/configure.m4
Просмотреть файл

@ -1,48 +0,0 @@
# -*- shell-script -*-
#
# Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
# University Research and Technology
# Corporation. All rights reserved.
# Copyright (c) 2004-2005 The University of Tennessee and The University
# of Tennessee Research Foundation. All rights
# reserved.
# Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
# University of Stuttgart. All rights reserved.
# Copyright (c) 2004-2005 The Regents of the University of California.
# All rights reserved.
# Copyright (c) 2010 Cisco Systems, Inc. All rights reserved.
# $COPYRIGHT$
#
# Additional copyrights may follow
#
# $HEADER$
#
# MCA_btl_portals_CONFIG(action-if-can-compile,
# [action-if-cant-compile])
# ------------------------------------------------
AC_DEFUN([MCA_ompi_btl_portals_CONFIG],[
AC_CONFIG_FILES([ompi/mca/btl/portals/Makefile])
OMPI_CHECK_PORTALS([btl_portals],
[btl_portals_happy="yes"],
[btl_portals_happy="no"])
AS_IF([test "$btl_portals_happy" = "yes"],
[btl_portals_WRAPPER_EXTRA_LDFLAGS="$btl_portals_LDFLAGS"
btl_portals_WRAPPER_EXTRA_LIBS="$btl_portals_LIBS"
$1],
[$2])
AC_CHECK_HEADERS([catamount/cnos_mpi_os.h], [],
[AC_CHECK_HEADERS([cnos_mpi_os.h], [], [$2],
[AC_INCLUDES_DEFAULT])],
[AC_INCLUDES_DEFAULT])
# substitute in the things needed to build portals
AC_SUBST([btl_portals_CPPFLAGS])
AC_SUBST([btl_portals_LDFLAGS])
AC_SUBST([btl_portals_LIBS])
])dnl

112
ompi/mca/common/portals/Makefile.am
Просмотреть файл

@ -1,112 +0,0 @@
#
# Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
# University Research and Technology
# Corporation. All rights reserved.
# Copyright (c) 2004-2005 The University of Tennessee and The University
# of Tennessee Research Foundation. All rights
# reserved.
# Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
# University of Stuttgart. All rights reserved.
# Copyright (c) 2004-2005 The Regents of the University of California.
# All rights reserved.
# Copyright (c) 2009-2010 Cisco Systems, Inc. All rights reserved.
# $COPYRIGHT$
#
# Additional copyrights may follow
#
# $HEADER$
#
# A word of explanation...
#
# This library is linked against various MCA components because all
# shared-memory based components (e.g., mpool, ptl, etc.) need to
# share some common code and data. There's two cases:
#
# 1. libmca_common_portals.la is a shared library. By linking that shared
# library to all components that need it, the OS linker will
# automatically load it into the process as necessary, and there will
# only be one copy (i.e., all the components will share *one* copy of
# the code and data).
# 2. libmca_common_portals.la is a static library. In this case, it
# will be rolled up into the top-level libmpi.la. It will also be
# rolled into each component, but then the component will also be
# rolled up into the upper-level libmpi.la. Libtool sorts this all
# out and it all works out in the end.
#
# Note that building this common component statically and linking
# against other dynamic components is *not* supported!
AM_CPPFLAGS = $(common_portals_CPPFLAGS)
# Header files
headers = \
common_portals.h
# Source files
sources = \
common_portals.c
EXTRA_DIST = common_portals_crayxt3.c \
common_portals_utcp.c \
common_portals_cray_xt_modex.c
# As per above, we'll either have an installable or noinst result.
# The installable one should follow the same MCA prefix naming rules
# (i.e., libmca_<type>_<name>.la). The noinst one can be named
# whatever it wants, although libmca_<type>_<name>_noinst.la is
# recommended.
# To simplify components that link to this library, we will *always*
# have an output libtool library named libmca_<type>_<name>.la -- even
# for case 2) described above (i.e., so there's no conditional logic
# necessary in component Makefile.am's that link to this library).
# Hence, if we're creating a noinst version of this library (i.e.,
# case 2), we sym link it to the libmca_<type>_<name>.la name
# (libtool will do the Right Things under the covers). See the
# all-local and clean-local rules, below, for how this is effected.
lib_LTLIBRARIES =
noinst_LTLIBRARIES =
comp_inst = libmca_common_portals.la
comp_noinst = libmca_common_portals_noinst.la
if MCA_BUILD_ompi_common_portals_DSO
lib_LTLIBRARIES += $(comp_inst)
else
noinst_LTLIBRARIES += $(comp_noinst)
endif
libmca_common_portals_la_SOURCES = $(headers) $(sources)
libmca_common_portals_la_LDFLAGS = -version-info $(libmca_common_portals_so_version) $(common_portals_LDFLAGS)
libmca_common_portals_la_LIBADD = $(common_portals_LIBS)
libmca_common_portals_noinst_la_SOURCES = $(libmca_common_portals_la_SOURCES)
libmca_common_portals_noinst_la_LDFLAGS = $(common_portals_LDFLAGS)
libmca_common_portals_noinst_la_LIBADD = $(common_portals_LIBS)
# Conditionally install the header files
if WANT_INSTALL_HEADERS
ompidir = $(includedir)/openmpi/$(subdir)
ompi_HEADERS = $(headers)
endif
# These two rules will sym link the "noinst" libtool library filename
# to the installable libtool library filename in the case where we are
# compiling this component statically (case 2), described above).
all-local:
if test -z "$(lib_LTLIBRARIES)"; then \
rm -f "$(comp_inst)"; \
$(LN_S) "$(comp_noinst)" "$(comp_inst)"; \
fi
clean-local:
if test -z "$(lib_LTLIBRARIES)"; then \
rm -f "$(comp_inst)"; \
fi

123
ompi/mca/common/portals/common_portals.c
Просмотреть файл

@ -1,123 +0,0 @@
/*
* Copyright (c) 2004-2006 The Trustees of Indiana University and Indiana
* University Research and Technology
* Corporation. All rights reserved.
* Copyright (c) 2004-2005 The University of Tennessee and The University
* of Tennessee Research Foundation. All rights
* reserved.
* Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
* University of Stuttgart. All rights reserved.
* Copyright (c) 2004-2006 The Regents of the University of California.
* All rights reserved.
* $COPYRIGHT$
*
* Additional copyrights may follow
*
* $HEADER$
*/
#include "ompi_config.h"
#include "ompi/constants.h"
#include "common_portals.h"
#if OMPI_PORTALS_UTCP
#include "common_portals_utcp.c"
#elif OMPI_PORTALS_CRAYXT3
#include "common_portals_crayxt3.c"
#elif OMPI_PORTALS_CRAYXT3_MODEX
#include "common_portals_cray_xt_modex.c"
#else
#error "Unknown Portals library configuration"
#endif
int
ompi_common_portals_error_ptl_to_ompi(int ptl_error)
{
int ret;
switch (ptl_error) {
case PTL_OK:
ret = OMPI_SUCCESS;
break;
case PTL_AC_INDEX_INVALID:
ret = OMPI_ERR_BAD_PARAM;
break;
case PTL_EQ_DROPPED:
ret = OMPI_ERR_OUT_OF_RESOURCE;
break;
case PTL_EQ_INVALID:
ret = OMPI_ERR_BAD_PARAM;
break;
case PTL_FAIL:
ret = OMPI_ERROR;
break;
case PTL_HANDLE_INVALID:
ret = OMPI_ERR_BAD_PARAM;
break;
case PTL_IFACE_INVALID:
ret = OMPI_ERR_BAD_PARAM;
break;
case PTL_MD_ILLEGAL:
ret = OMPI_ERR_BAD_PARAM;
break;
case PTL_MD_INVALID:
ret = OMPI_ERR_BAD_PARAM;
break;
case PTL_MD_IN_USE:
ret = OMPI_ERR_RESOURCE_BUSY;
break;
case PTL_ME_INVALID:
ret = OMPI_ERR_BAD_PARAM;
break;
case PTL_ME_IN_USE:
ret = OMPI_ERR_RESOURCE_BUSY;
break;
case PTL_ME_LIST_TOO_LONG:
ret = OMPI_ERR_OUT_OF_RESOURCE;
break;
case PTL_NI_INVALID:
ret = OMPI_ERR_BAD_PARAM;
break;
case PTL_NO_INIT:
ret = OMPI_ERR_BAD_PARAM;
break;
case PTL_NO_SPACE:
ret = OMPI_ERR_OUT_OF_RESOURCE;
break;
case PTL_PID_INVALID:
ret = OMPI_ERR_BAD_PARAM;
break;
case PTL_PROCESS_INVALID:
ret = OMPI_ERR_BAD_PARAM;
break;
case PTL_PT_FULL:
ret = OMPI_ERR_OUT_OF_RESOURCE;
break;
case PTL_PT_INDEX_INVALID:
ret = OMPI_ERR_BAD_PARAM;
break;
case PTL_SEGV:
ret = OMPI_ERR_VALUE_OUT_OF_BOUNDS;
break;
case PTL_SR_INDEX_INVALID:
ret = OMPI_ERR_BAD_PARAM;
break;
#if !(OMPI_PORTALS_CRAYXT3 || OMPI_PORTALS_CRAYXT3_MODEX)
case PTL_UNKNOWN_ERROR:
ret = OMPI_ERROR;
break;
#endif
default:
ret = OMPI_ERROR;
}
return ret;
}

163
ompi/mca/common/portals/common_portals.h
Просмотреть файл

@ -1,163 +0,0 @@
/*
* Copyright (c) 2004-2006 The Trustees of Indiana University and Indiana
* University Research and Technology
* Corporation. All rights reserved.
* Copyright (c) 2004-2005 The University of Tennessee and The University
* of Tennessee Research Foundation. All rights
* reserved.
* Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
* University of Stuttgart. All rights reserved.
* Copyright (c) 2004-2006 The Regents of the University of California.
* All rights reserved.
* $COPYRIGHT$
*
* Additional copyrights may follow
*
* $HEADER$
*/
#ifndef OMPI_MCA_COMMON_PORTALS_H
#define OMPI_MCA_COMMON_PORTALS_H
#if OMPI_PORTALS_UTCP
#include <portals3.h>
#include <stdio.h>
#include <p3nal_utcp.h>
#include <p3rt/p3rt.h>
#include <p3api/debug.h>
#define OMPI_BTL_PORTALS_SEND_TABLE_ID 0
#define OMPI_BTL_PORTALS_RDMA_TABLE_ID 1
#define OMPI_MTL_PORTALS_SEND_TABLE_ID 2
#define OMPI_MTL_PORTALS_READ_TABLE_ID 3
#define OMPI_MTL_PORTALS_ACK_TABLE_ID 4
#elif (OMPI_PORTALS_CRAYXT3 || OMPI_PORTALS_CRAYXT3_MODEX)
#include <portals/portals3.h>
#define PTL_EQ_HANDLER_NONE NULL
/* Cray's definition, differs from the spec */
#define PTL_NO_ACK_REQ PTL_NOACK_REQ
#define OMPI_BTL_PORTALS_SEND_TABLE_ID 30
#define OMPI_BTL_PORTALS_RDMA_TABLE_ID 31
#define OMPI_MTL_PORTALS_SEND_TABLE_ID 32
#define OMPI_MTL_PORTALS_READ_TABLE_ID 33
#define OMPI_MTL_PORTALS_ACK_TABLE_ID 34
#else
#error "Unknown Portals library configuration"
#endif
#include "ompi/proc/proc.h"
/**
* Simple identifier for identifying node/process
*
* Get a string representing a simple way to identify the node/rank of
* the current process. Currently returns the rank in the job on the
* XT-3 or the hostname/pid on the reference implementation.
*
* \note Caller is responsible for calling free() on the returned
* string.
*/
char* ompi_common_portals_nodeid(void);
/**
* Register MCA parameters for Portals code
*
* Register MCA parameters for Portals common code. This should be
* called during component open so that parameters are available to
* omp_info and the like. This call will not intiailize the Portals
* interface or cause any communication.
*
* @retval OMPI_SUCCESS
*/
int ompi_common_portals_register_mca(void);
/**
* Initialize compatibility code
*
* Initialize Portals compatibility code. A best effort is made to
* initialize Portals (with PtlInit() and PtlNIInit(), although this
* may not be possible if use of the modex is required to setup the
* network (as is the case with the utcp reference implementation).
*
* @param ni_handle (OUT) network interface handle
* @param bool (OUT) true if using accelerated Portals, false otherwise
*
* @retval OMPI_SUCCESS Portals successfully initialized
* @retval OMPI_ERR_NOT_AVAILABLE Portals could not be initialized
*/
int ompi_common_portals_initialize(ptl_handle_ni_t *ni_handle, bool *accel);
/**
* Initialize network interface
*
* Initialize the portals network interface. The initializization may
* actually have happened in ompi_common_portals_initialize(), but
* this will return the network interface handle. This function may
* require some information shared by the modex, so should only be
* called after the modex data is available.
*
* @param ni_handle (OUT) network interface handle
* @param bool (OUT) true if using accelerated Portals, false otherwise
*
* @retval OMPI_SUCCESS Portals network interface successfully initialized
* @retval OMPI_ERROR Something bad happened
*/
int ompi_common_portals_ni_initialize(ptl_handle_ni_t *ni_handle, bool *accel);
/**
* Get process_id_t array for proc list
*
* Get ptl_process_id_t array for proc list
*
* @param nprocs (IN) Number of procs in proc list
* @param procs (IN) List of OMPI procs
* @param portals_procs (OUT) array of ptl_process_id_t
* structures associated with OMPI procs
*
* @retval OMPI_SUCCESS All went well
* @retval OMPI_ERROR All went poorly
*/
int ompi_common_portals_get_procs(size_t nprocs,
struct ompi_proc_t **procs,
ptl_process_id_t *portals_procs);
/**
* Shut down Portals network interface
*
* Shut down Portals network devince , including calling PtlNIFini()
* if appropriate. The common code will reference count so that it is
* safe for each component that calls
* ompi_component_portals_ni_initialize() to call
* ompi_common_portals_ni_finalize()
*/
int ompi_common_portals_ni_finalize(void);
/**
* Shut down Portals
*
* Shut down Portals, including calling PtlFini() if appropriate. The
* common code will reference count so that it is safe for each
* component that calls ompi_component_portals_initialize() to call
* ompi_common_portals_finalize()
*/
int ompi_common_portals_finalize(void);
int ompi_common_portals_error_ptl_to_ompi(int ptl_error);
#endif /* OMPI_MCA_COMMON_PORTALS_H */

158
ompi/mca/common/portals/common_portals_cray_xt_modex.c
Просмотреть файл

@ -1,158 +0,0 @@
/*
* Copyright (c) 2004-2006 The Trustees of Indiana University and Indiana
* University Research and Technology
* Corporation. All rights reserved.
* Copyright (c) 2004-2005 The University of Tennessee and The University
* of Tennessee Research Foundation. All rights
* reserved.
* Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
* University of Stuttgart. All rights reserved.
* Copyright (c) 2004-2006 The Regents of the University of California.
* All rights reserved.
* Copyright (c) 2008 UT-Battelle, LLC. All rights reserved.
* $COPYRIGHT$
*
* Additional copyrights may follow
*
* $HEADER$
*/
#include "ompi_config.h"
#include "common_portals.h"
#include "ompi/constants.h"
#include "opal/mca/base/base.h"
#include "opal/util/output.h"
#include "ompi/runtime/ompi_module_exchange.h"
static mca_base_component_t portals_component = {
MCA_BASE_VERSION_2_0_0,
"common",
MCA_BASE_VERSION_2_0_0,
"portals",
MCA_BASE_VERSION_2_0_0,
NULL,
NULL
};
char *
ompi_common_portals_nodeid(void)
{
char *ret;
asprintf(&ret, "CNL:");
return ret;
}
int
ompi_common_portals_register_mca(void)
{
return OMPI_SUCCESS;
}
int
ompi_common_portals_initialize(ptl_handle_ni_t *ni_handle, bool *accel)
{
int ret, max_interfaces;
ptl_process_id_t ptl_process_id;
ptl_interface_t ni_iface = PTL_IFACE_DEFAULT;
*accel = false;
/*
* If we use the YOD launcher we can use the default interface
* otherwise we need to use the SeaStar Bridged interface (for CNL/APRUN)
*/
ni_iface = IFACE_FROM_BRIDGE_AND_NALID(PTL_BRIDGE_UK,PTL_IFACE_SS);
/*
* Initialize Portals interface
*/
ret = PtlInit(&max_interfaces);
if (PTL_OK != ret) {
opal_output(0, "PtlInit failed, returning %d\n", ret);
return OMPI_ERR_NOT_AVAILABLE;
}
/*
* Initialize a network device
*/
ret = PtlNIInit(ni_iface, /* interface to initialize */
PTL_PID_ANY, /* let library assign our pid */
NULL, /* no desired limits */
NULL, /* actual limits */
ni_handle /* our interface handle */
);
if (PTL_OK != ret && PTL_IFACE_DUP != ret) {
opal_output(0, "PtlNIInit failed, returning %d (%s : %d)\n",
ret, __FILE__, __LINE__);
return OMPI_ERROR;
}
ret = PtlGetId(*ni_handle ,&ptl_process_id);
if(PTL_OK != ret) {
opal_output(0, "PtlGetId failed, returning %d\n", ret);
return OMPI_ERROR;
}
/* publish my nid/pid info */
ret = ompi_modex_send(&portals_component,
&ptl_process_id, sizeof(ptl_process_id_t));
if (OMPI_SUCCESS != ret) {
return ret;
}
return OMPI_SUCCESS;
}
int
ompi_common_portals_ni_initialize(ptl_handle_ni_t *ni_handle, bool *accel)
{
return OMPI_SUCCESS;
}
int
ompi_common_portals_get_procs(size_t nprocs,
struct ompi_proc_t **procs,
ptl_process_id_t *portals_procs)
{
size_t i, size;
int ret;
ptl_process_id_t *ptl_process_id;
for (i = 0 ; i < nprocs ; ++i) {
ret = ompi_modex_recv(&portals_component,
procs[i], (void**) &ptl_process_id, &size);
if (OMPI_SUCCESS != ret) {
opal_output(0, "ompi_modex_recv failed: %d", ret);
return ret;
} else if (sizeof(ptl_process_id_t) != size) {
opal_output(0, "ompi_modex_recv returned size %d, expected %d",
(int) size, (int) sizeof(ptl_process_id_t));
return OMPI_ERROR;
}
portals_procs[i] = *ptl_process_id;
}
return OMPI_SUCCESS;
}
int
ompi_common_portals_ni_finalize(void)
{
return OMPI_SUCCESS;
}
int
ompi_common_portals_finalize(void)
{
return OMPI_SUCCESS;
}

183
ompi/mca/common/portals/common_portals_crayxt3.c
Просмотреть файл

@ -1,183 +0,0 @@
/*
* Copyright (c) 2004-2006 The Trustees of Indiana University and Indiana
* University Research and Technology
* Corporation. All rights reserved.
* Copyright (c) 2004-2005 The University of Tennessee and The University
* of Tennessee Research Foundation. All rights
* reserved.
* Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
* University of Stuttgart. All rights reserved.
* Copyright (c) 2004-2006 The Regents of the University of California.
* All rights reserved.
* $COPYRIGHT$
*
* Additional copyrights may follow
*
* $HEADER$
*/
#include "ompi_config.h"
#if defined(HAVE_CNOS_MPI_OS_H)
# include "cnos_mpi_os.h"
#elif defined(HAVE_CATAMOUNT_CNOS_MPI_OS_H)
# include "catamount/cnos_mpi_os.h"
#endif
#include "ompi/constants.h"
#include "ompi/proc/proc.h"
#include "opal/mca/base/base.h"
#include "opal/util/output.h"
static bool use_accelerated;
char *
ompi_common_portals_nodeid(void)
{
char *ret;
asprintf(&ret, "%5d", cnos_get_rank());
return ret;
}
int
ompi_common_portals_register_mca(void)
{
return OMPI_SUCCESS;
}
int
ompi_common_portals_initialize(ptl_handle_ni_t *ni_handle, bool *accel)
{
int ret, max_interfaces;
/*
* Initialize Portals interface
*/
ret = PtlInit(&max_interfaces);
if (PTL_OK != ret) {
opal_output(0, "%5d: PtlInit failed, returning %d\n",
cnos_get_rank(), ret);
return OMPI_ERR_NOT_AVAILABLE;
}
return OMPI_SUCCESS;
}
int
ompi_common_portals_ni_initialize(ptl_handle_ni_t *ni_handle, bool *accel)
{
ptl_interface_t ni_iface = PTL_IFACE_DEFAULT;
int max_interfaces;
int launcher;
int ret;
int tmp = 0;
#if defined(CRAY_ACCEL)
mca_base_param_reg_int_name("mca",
"use_accelerated_portals",
"Use Accelerated Portals",
false,
false,
0,
&tmp);
#endif
*accel = use_accelerated = (tmp == 0) ? false : true;
launcher = cnos_launcher();
/*
* If we use the YOD launcher we can use the default interface
* otherwise we need to use the SeaStar Bridged interface (for CNL/APRUN)
*/
if( launcher != CNOS_LAUNCHER_YOD ) {
ni_iface = IFACE_FROM_BRIDGE_AND_NALID(PTL_BRIDGE_UK,PTL_IFACE_SS);
}
#if defined(CRAY_ACCEL)
else if (use_accelerated == true) {
ni_iface = CRAY_ACCEL;
}
#endif
/*
* Initialize Portals interface
*/
ret = PtlInit(&max_interfaces);
if (PTL_OK != ret) {
opal_output(0, "%5d: PtlInit failed, returning %d\n",
cnos_get_rank(), ret);
return OMPI_ERR_NOT_AVAILABLE;
}
/*
* Initialize a network device
*/
ret = PtlNIInit(ni_iface, /* interface to initialize */
PTL_PID_ANY, /* let library assign our pid */
NULL, /* no desired limits */
NULL, /* actual limits */
ni_handle /* our interface handle */
);
if (PTL_OK != ret && PTL_IFACE_DUP != ret) {
opal_output(0, "%5d: PtlNIInit failed, returning %d (%s : %d)\n",
cnos_get_rank(), ret, __FILE__, __LINE__);
return OMPI_ERROR;
}
return OMPI_SUCCESS;
}
int
ompi_common_portals_get_procs(size_t nprocs,
struct ompi_proc_t **procs,
ptl_process_id_t *portals_procs)
{
int nptl_procs = 0;
cnos_nidpid_map_t *map;
int pid_space_offset = 0;
int i;
/*
* assumption that the vpid of the process name is the rank in the
* nidpid map. THis will not be true if someone changes the sds
* component...
*/
nptl_procs = cnos_get_nidpid_map(&map);
if (nptl_procs <= 0) {
opal_output(0, "%5d: cnos_get_nidpid_map() returned %d",
cnos_get_rank(), nptl_procs);
return OMPI_ERR_FATAL;
}
#if defined(CRAY_ACCEL)
pid_space_offset = (use_accelerated == true) ? ACCEL_PTLS_PID_SPACE_OFFSET : 0;
#endif
for (i = 0 ; i < nprocs ; ++i) {
size_t idx = (size_t) procs[i]->proc_name.vpid;
if (idx >= nptl_procs) return OMPI_ERR_NOT_FOUND;
portals_procs[i].nid = map[idx].nid;
portals_procs[i].pid = map[idx].pid + pid_space_offset;
}
return OMPI_SUCCESS;
}
int
ompi_common_portals_ni_finalize(void)
{
return OMPI_SUCCESS;
}
int
ompi_common_portals_finalize(void)
{
return OMPI_SUCCESS;
}

340
ompi/mca/common/portals/common_portals_utcp.c
Просмотреть файл

@ -1,340 +0,0 @@
/*
* Copyright (c) 2004-2006 The Trustees of Indiana University and Indiana
* University Research and Technology
* Corporation. All rights reserved.
* Copyright (c) 2004-2005 The University of Tennessee and The University
* of Tennessee Research Foundation. All rights
* reserved.
* Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
* University of Stuttgart. All rights reserved.
* Copyright (c) 2004-2006 The Regents of the University of California.
* All rights reserved.
* $COPYRIGHT$
*
* Additional copyrights may follow
*
* $HEADER$
*/
#include <unistd.h>
#include <stdio.h>
#include <p3api/debug.h>
#include "opal/mca/mca.h"
#include "opal/util/output.h"
#include "opal/mca/base/mca_base_param.h"
#include "ompi/proc/proc.h"
#include "ompi/constants.h"
#include "ompi/runtime/ompi_module_exchange.h"
#ifdef __APPLE__
static char *ptl_ifname = "en0";
FILE *p3_out = stderr;
#else
static char *ptl_ifname = "eth0";
#endif
/* how's this for source code diving? - find private method for
getting interface */
extern int p3tcp_my_nid(const char *if_str, unsigned int *nid);
static volatile int32_t usage_count = 0;
static volatile int32_t ni_usage_count = 0;
static bool setup_utcp_params = true;
static bool init_called = false;
static ptl_handle_ni_t active_ni_h = PTL_INVALID_HANDLE;
static mca_base_component_t portals_component = {
MCA_BASE_VERSION_2_0_0,
"common",
MCA_BASE_VERSION_2_0_0,
"portals",
MCA_BASE_VERSION_2_0_0,
NULL,
NULL
};
char *
ompi_common_portals_nodeid(void)
{
char *ret;
asprintf(&ret, "%5d", getpid());
return ret;
}
int
ompi_common_portals_register_mca(void)
{
mca_base_param_reg_string(&portals_component,
"ifname",
"Interface name to use for communication",
false,
false,
ptl_ifname,
&ptl_ifname);
return OMPI_SUCCESS;
}
int
ompi_common_portals_initialize(ptl_handle_ni_t *ni_handle, bool *accel)
{
int ret;
ptl_process_id_t info;
if (OPAL_THREAD_ADD32(&usage_count, 1) > 1) return OMPI_SUCCESS;
/* if the environment variables for the utcp implementation are
already set, assume the user is running without the full Open
RTE and is doing RTE testing for a more tightly-coupled
platform (like, say, Red Storm). Otherwise, be nice and use
the modex to setup everything for the user */
if (NULL == getenv("PTL_MY_RID")) {
setup_utcp_params = true;
} else {
setup_utcp_params = false;
}
if (setup_utcp_params) {
/* Find our contact information and post to registry. Don't
initialize Portals until we have everyone's contact
information. */
unsigned int nid;
p3tcp_my_nid(ptl_ifname, &nid);
info.nid = htonl(nid);
info.pid = htonl((ptl_pid_t) getpid());
} else {
/* Initialize Portals and publish our assigned contact
information */
int max_interfaces;
unsigned int nptl_procs, rank;
ret = PtlInit(&max_interfaces);
if (PTL_OK != ret) {
opal_output(0, "%5d: PtlInit failed, returning %d\n",
getpid(), ret);
return OMPI_ERR_NOT_AVAILABLE;
}
init_called = true;
/* tell the UTCP runtime code to read the env variables */
PtlSetRank(PTL_INVALID_HANDLE, -1, -1);
/* Initialize a network device */
ret = PtlNIInit(PTL_IFACE_DEFAULT, /* interface to initialize */
PTL_PID_ANY, /* let library assign our pid */
NULL, /* no desired limits */
NULL, /* no need to have limits around */
&active_ni_h /* our interface handle */
);
if (PTL_OK != ret) {
opal_output(0, "%5d: PtlNIInit failed, returning %d\n",
getpid(), ret);
return OMPI_ERR_FATAL;
}
ret = PtlGetRank(active_ni_h, &rank, &nptl_procs);
if (ret != PTL_OK) {
opal_output(0, "%5d, PtlGetRank() returned %d",
getpid(), ret);
return OMPI_ERR_FATAL;
}
ret = PtlGetRankId(active_ni_h, rank, &info);
if (ret != PTL_OK) {
opal_output(0, "%5d, PtlGetRank(rank=%d) returned %d",
getpid(), rank, ret);
return OMPI_ERR_FATAL;
}
}
ret = ompi_modex_send(&portals_component,
&info, sizeof(ptl_process_id_t));
if (OMPI_SUCCESS != ret) {
return ret;
}
return OMPI_SUCCESS;
}
int
ompi_common_portals_ni_initialize(ptl_handle_ni_t *ni_handle, bool *accel)
{
int ret;
*accel = false;
OPAL_THREAD_ADD32(&ni_usage_count, 1);
if (PTL_INVALID_HANDLE != active_ni_h) {
*ni_handle = active_ni_h;
return OMPI_SUCCESS;
}
if (setup_utcp_params) {
ompi_proc_t **procs;
int my_rid = 0;
ptl_process_id_t *info;
char *nidmap = NULL, *pidmap = NULL;
char *nid_str, *pid_str;
size_t map_size = 0;
size_t nprocs, size, i;
char *tmp;
ompi_proc_t* proc_self = ompi_proc_local();
int max_interfaces;
/* get our world */
procs = ompi_proc_world(&nprocs);
map_size = nprocs * 12 + 1; /* 12 is max length of long in decimal */
nidmap = malloc(map_size);
pidmap = malloc(map_size);
nid_str = malloc(12 + 1);
pid_str = malloc(12 + 1);
if (NULL == nidmap || NULL == pidmap ||
NULL == nid_str || NULL == pid_str)
return OMPI_ERROR;
for (i = 0 ; i < nprocs ; ++i) {
if (proc_self == procs[i]) my_rid = i;
ret = ompi_modex_recv(&portals_component,
procs[i], (void**) &info, &size);
if (OMPI_SUCCESS != ret) {
opal_output(0, "%5d: ompi_modex_recv failed: %d",
getpid(), ret);
return ret;
} else if (sizeof(ptl_process_id_t) != size) {
opal_output(0, "%5d: ompi_modex_recv returned size %d, expected %d",
getpid(), size, sizeof(ptl_process_id_t));
return OMPI_ERROR;
}
if (i == 0) {
snprintf(nidmap, map_size, "%u", ntohl(info->nid));
snprintf(pidmap, map_size, "%u", ntohl(info->pid));
} else {
snprintf(nid_str, 12 + 1, ":%u", ntohl(info->nid));
snprintf(pid_str, 12 + 1, ":%u", ntohl(info->pid));
strncat(nidmap, nid_str, 12);
strncat(pidmap, pid_str, 12);
}
free(info);
}
asprintf(&tmp, "PTL_MY_RID=%u", my_rid);
putenv(tmp);
asprintf(&tmp, "PTL_NIDMAP=%s", nidmap);
putenv(tmp);
asprintf(&tmp, "PTL_PIDMAP=%s", pidmap);
putenv(tmp);
asprintf(&tmp, "PTL_IFACE=%s", ptl_ifname);
putenv(tmp);
free(pidmap);
free(nidmap);
free(pid_str);
free(nid_str);
/*
* Initialize Portals
*/
ret = PtlInit(&max_interfaces);
if (PTL_OK != ret) {
opal_output(0, "%5d: PtlInit failed, returning %d\n",
getpid(), ret);
return OMPI_ERR_NOT_AVAILABLE;
}
init_called = true;
/* tell the UTCP runtime code to read the env variables */
PtlSetRank(PTL_INVALID_HANDLE, -1, -1);
/* Initialize a network device */
ret = PtlNIInit(PTL_IFACE_DEFAULT, /* interface to initialize */
PTL_PID_ANY, /* let library assign our pid */
NULL, /* no desired limits */
NULL, /* no need to have limits around */
&active_ni_h /* our interface handle */
);
if (PTL_OK != ret) {
opal_output(0, "%5d: PtlNIInit failed, returning %d\n",
getpid(), ret);
return OMPI_ERR_FATAL;
}
*ni_handle = active_ni_h;
return OMPI_SUCCESS;
}
/* shouldn't ever be able to get here */
return OMPI_ERROR;
}
int
ompi_common_portals_get_procs(size_t nprocs,
struct ompi_proc_t **procs,
ptl_process_id_t *portals_procs)
{
size_t i, size;
int ret;
ptl_process_id_t *info;
for (i = 0 ; i < nprocs ; ++i) {
ret = ompi_modex_recv(&portals_component,
procs[i], (void**) &info, &size);
if (OMPI_SUCCESS != ret) {
opal_output(0, "%5d: ompi_modex_recv failed: %d",
getpid(), ret);
return ret;
} else if (sizeof(ptl_process_id_t) != size) {
opal_output(0, "%5d: ompi_modex_recv returned size %d, expected %d",
getpid(), size, sizeof(ptl_process_id_t));
return OMPI_ERROR;
}
portals_procs[i].nid = ntohl(info->nid);
portals_procs[i].pid = ntohl(info->pid);
}
return OMPI_SUCCESS;
}
int
ompi_common_portals_ni_finalize(void)
{
if (OPAL_THREAD_ADD32(&ni_usage_count, -1) <= 0) {
if (PTL_INVALID_HANDLE != active_ni_h) {
if (PTL_OK != PtlNIFini(active_ni_h)) {
active_ni_h = PTL_INVALID_HANDLE;
return OMPI_ERROR;
}
active_ni_h = PTL_INVALID_HANDLE;
}
}
return OMPI_SUCCESS;
}
int
ompi_common_portals_finalize(void)
{
if (OPAL_THREAD_ADD32(&usage_count, -1) <= 0) {
if (init_called) {
PtlFini();
}
}
return OMPI_SUCCESS;
}

79
ompi/mca/common/portals/configure.m4
Просмотреть файл

@ -1,79 +0,0 @@
# -*- shell-script -*-
#
# Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
# University Research and Technology
# Corporation. All rights reserved.
# Copyright (c) 2004-2005 The University of Tennessee and The University
# of Tennessee Research Foundation. All rights
# reserved.
# Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
# University of Stuttgart. All rights reserved.
# Copyright (c) 2004-2005 The Regents of the University of California.
# All rights reserved.
# Copyright (c) 2010 Cisco Systems, Inc. All rights reserved.
# $COPYRIGHT$
#
# Additional copyrights may follow
#
# $HEADER$
#
# MCA_common_portals_CONFIG(action-if-can-compile,
# [action-if-cant-compile])
# ------------------------------------------------
AC_DEFUN([MCA_ompi_common_portals_CONFIG],[
AC_CONFIG_FILES([ompi/mca/common/portals/Makefile])
OMPI_CHECK_PORTALS([common_portals],
[common_portals_happy="yes"],
[common_portals_happy="no"])
if test "$common_portals_happy" = "yes" -a "$with_portals_config" = "utcp" ; then
# Portals interface
symbols="PtlInit PtlFini PtlNIInit"
symbols="$symbols PtlNIFini PtlNIStatus PtlNIDist PtlNIHandle"
symbols="$symbols PtlGetUid PtlGetId PtlGetJid "
symbols="$symbols PtlMEAttach PtlMEAttachAny PtlMEInsert PtlMEUnlink"
symbols="$symbols PtlMDAttach PtlMDBind PtlMDUnlink PtlMDUpdate"
symbols="$symbols PtlEQAlloc PtlEQFree PtlEQGet PtlEQWait"
symbols="$symbols PtlEQPoll PtlACEntry"
symbols="$symbols PtlPut PtlPutRegion PtlGet PtlGetRegon PtlGetPut"
# Portals reference implementation debugging stuff
symbols="$symbols PtlMEDump PtlNIEqDump PtlTblDump PtlNIDebug"
symbols="$symbols PtlErrorStr PtlEventKindStr PtlNIFailStr"
# Portals reference implementation RTE interface
symbols="$symbols PtlGetNIDMap PtlGetPIDMap PtlGetRank PtlGetRankId"
symbols="$symbols PtlSetJID PtlSetNIDMap PtlSetPIDMap PtlSetRank"
flags=
for symbol in $symbols ; do
case $host in
*-darwin*)
flags="$flags -Wl,-u,=_$symbol"
;;
*-linux*)
flags="$flags -Wl,-undefined=$symbol"
;;
esac
done
OMPI_LIBMPI_EXTRA_LDFLAGS="$common_portals_LDFLAGS $flags"
OMPI_LIBMPI_EXTRA_LIBS="$common_portals_LIBS"
commmon_portals_LIBS=
fi
AS_IF([test "$common_portals_happy" = "yes"],
[common_portals_WRAPPER_EXTRA_LDFLAGS="$common_portals_LDFLAGS"
common_portals_WRAPPER_EXTRA_LIBS="$common_portals_LIBS"
$1],
[$2])
# substitute in the things needed to build portals
AC_SUBST([common_portals_CPPFLAGS])
AC_SUBST([common_portals_LDFLAGS])
AC_SUBST([common_portals_LIBS])
])dnl

61
ompi/mca/mtl/portals/Makefile.am
Просмотреть файл

@ -1,61 +0,0 @@
#
# Copyright (c) 2004-2006 The Trustees of Indiana University and Indiana
# University Research and Technology
# Corporation. All rights reserved.
# Copyright (c) 2004-2005 The University of Tennessee and The University
# of Tennessee Research Foundation. All rights
# reserved.
# Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
# University of Stuttgart. All rights reserved.
# Copyright (c) 2004-2005 The Regents of the University of California.
# All rights reserved.
# Copyright (c) 2010 Cisco Systems, Inc. All rights reserved.
# $COPYRIGHT$
#
# Additional copyrights may follow
#
# $HEADER$
#
# Make the output library in this directory, and name it either
# mca_<type>_<name>.la (for DSO builds) or libmca_<type>_<name>.la
# (for static builds).
AM_CPPFLAGS = $(mtl_portals_CPPFLAGS)
if MCA_BUILD_ompi_mtl_portals_DSO
component_noinst =
component_install = mca_mtl_portals.la
else
component_noinst = libmca_mtl_portals.la
component_install =
endif
local_sources = \
mtl_portals_component.c \
mtl_portals.c \
mtl_portals.h \
mtl_portals_endpoint.h \
mtl_portals_recv.c \
mtl_portals_recv.h \
mtl_portals_recv_short.h \
mtl_portals_recv_short.c \
mtl_portals_request.h \
mtl_portals_send_short.h \
mtl_portals_send_short.c \
mtl_portals_send.c \
mtl_portals_probe.c
mcacomponentdir = $(pkglibdir)
mcacomponent_LTLIBRARIES = $(component_install)
mca_mtl_portals_la_SOURCES = $(local_sources)
mca_mtl_portals_la_LIBADD = \
$(mtl_portals_LIBS) \
$(top_ompi_builddir)/ompi/mca/common/portals/libmca_common_portals.la
mca_mtl_portals_la_LDFLAGS = -module -avoid-version $(mtl_portals_LDFLAGS)
noinst_LTLIBRARIES = $(component_noinst)
libmca_mtl_portals_la_SOURCES = $(local_sources)
libmca_mtl_portals_la_LIBADD = $(mtl_portals_LIBS)
libmca_mtl_portals_la_LDFLAGS = -module -avoid-version $(mtl_portals_LDFLAGS)

41
ompi/mca/mtl/portals/configure.m4
Просмотреть файл

@ -1,41 +0,0 @@
# -*- shell-script -*-
#
# Copyright (c) 2004-2006 The Trustees of Indiana University and Indiana
# University Research and Technology
# Corporation. All rights reserved.
# Copyright (c) 2004-2005 The University of Tennessee and The University
# of Tennessee Research Foundation. All rights
# reserved.
# Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
# University of Stuttgart. All rights reserved.
# Copyright (c) 2004-2005 The Regents of the University of California.
# All rights reserved.
# Copyright (c) 2010 Cisco Systems, Inc. All rights reserved.
# $COPYRIGHT$
#
# Additional copyrights may follow
#
# $HEADER$
#
# MCA_mtl_portals_CONFIG(action-if-can-compile,
# [action-if-cant-compile])
# ------------------------------------------------
AC_DEFUN([MCA_ompi_mtl_portals_CONFIG],[
AC_CONFIG_FILES([ompi/mca/mtl/portals/Makefile])
OMPI_CHECK_PORTALS([mtl_portals],
[mtl_portals_happy="yes"],
[mtl_portals_happy="no"])
AS_IF([test "$mtl_portals_happy" = "yes"],
[mtl_portals_WRAPPER_EXTRA_LDFLAGS="$mtl_portals_LDFLAGS"
mtl_portals_WRAPPER_EXTRA_LIBS="$mtl_portals_LIBS"
$1],
[$2])
# substitute in the things needed to build portals
AC_SUBST([mtl_portals_CPPFLAGS])
AC_SUBST([mtl_portals_LDFLAGS])
AC_SUBST([mtl_portals_LIBS])
])dnl

406
ompi/mca/mtl/portals/mtl_portals.c
Просмотреть файл

@ -1,406 +0,0 @@
/*
* Copyright (c) 2004-2006 The Trustees of Indiana University and Indiana
* University Research and Technology
* Corporation. All rights reserved.
* Copyright (c) 2004-2005 The University of Tennessee and The University
* of Tennessee Research Foundation. All rights
* reserved.
* Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
* University of Stuttgart. All rights reserved.
* Copyright (c) 2004-2005 The Regents of the University of California.
* All rights reserved.
* $COPYRIGHT$
*
* Additional copyrights may follow
*
* $HEADER$
*/
#include "ompi_config.h"
#include "ompi/mca/mtl/mtl.h"
#include "opal/class/opal_list.h"
#include "mtl_portals.h"
#include "mtl_portals_endpoint.h"
#include "mtl_portals_request.h"
#include "mtl_portals_recv.h"
#include "mtl_portals_recv_short.h"
#include "mtl_portals_send_short.h"
static ptl_handle_md_t send_catchall_md_h;
static ptl_handle_md_t ack_catchall_md_h;
static ptl_handle_md_t read_catchall_md_h;
static ptl_handle_md_t unex_long_md_h;
static ompi_mtl_portals_request_t catchall_send_request;
static ompi_mtl_portals_request_t catchall_ack_request;
static ompi_mtl_portals_request_t catchall_read_request;
mca_mtl_portals_module_t ompi_mtl_portals = {
{
8191, /* max cid - 2^13 - 1 */
(1UL << 30), /* max tag value - must allow negatives */
0, /* request reserve space */
0, /* flags */
ompi_mtl_portals_add_procs,
ompi_mtl_portals_del_procs,
ompi_mtl_portals_finalize,
ompi_mtl_portals_send,
ompi_mtl_portals_isend,
ompi_mtl_portals_irecv,
ompi_mtl_portals_iprobe,
ompi_mtl_portals_imrecv,
ompi_mtl_portals_improbe,
ompi_mtl_portals_cancel,
ompi_mtl_portals_add_comm,
ompi_mtl_portals_del_comm
}
};
/* BWB - fix me - this should be an ompi_free_list_item_t */
OBJ_CLASS_INSTANCE(ompi_mtl_portals_event_t, opal_list_item_t,
NULL, NULL);
/* catchall callback */
static int
ompi_mtl_portals_catchall_callback(ptl_event_t *ev,
ompi_mtl_portals_request_t *ptl_request)
{
if (ptl_request == &catchall_send_request) {
opal_output(fileno(stderr), "ERROR - received catchall event on send queue");
} else if (ptl_request == &catchall_ack_request) {
opal_output(fileno(stderr), "ERROR - received catchall event on ack queue");
} else if (ptl_request == &catchall_read_request) {
opal_output(fileno(stderr), "ERROR - received catchall event on read queue");
} else {
opal_output(fileno(stderr), "ERROR - received catchall event of unknown origin");
}
abort();
return OMPI_ERROR;
}
int
ompi_mtl_portals_add_procs(struct mca_mtl_base_module_t *mtl,
size_t nprocs,
struct ompi_proc_t** procs,
struct mca_mtl_base_endpoint_t **mtl_peer_data)
{
int ret = OMPI_SUCCESS;
ptl_process_id_t *portals_procs = NULL;
ptl_md_t md;
size_t i;
ptl_match_bits_t match_bits;
ptl_match_bits_t ignore_bits;
ptl_process_id_t anyid = { PTL_NID_ANY, PTL_PID_ANY };
bool accel;
assert(mtl == &ompi_mtl_portals.base);
/* if we haven't already initialized the network, do so now. We
delay until add_procs because if we want the automatic runtime
environment setup the common code does for the utcp
implementation, we can't do it until modex information can be
received. */
if (PTL_INVALID_HANDLE == ompi_mtl_portals.ptl_ni_h) {
ret = ompi_common_portals_ni_initialize(&(ompi_mtl_portals.ptl_ni_h), &accel);
if (OMPI_SUCCESS != ret) goto cleanup;
}
/* event queue for expected events */
ret = PtlEQAlloc(ompi_mtl_portals.ptl_ni_h,
ompi_mtl_portals.ptl_expected_queue_size,
PTL_EQ_HANDLER_NONE,
&(ompi_mtl_portals.ptl_eq_h));
assert(ret == PTL_OK);
/* event queue for unexpected receives */
ret = PtlEQAlloc(ompi_mtl_portals.ptl_ni_h,
ompi_mtl_portals.ptl_unexpected_queue_size,
PTL_EQ_HANDLER_NONE,
&(ompi_mtl_portals.ptl_unex_eq_h));
assert(ret == PTL_OK);
/* empty event queue for PtlMEMDPost() */
ret = PtlEQAlloc(ompi_mtl_portals.ptl_ni_h,
1,
PTL_EQ_HANDLER_NONE,
&(ompi_mtl_portals.ptl_empty_eq_h));
assert(ret == PTL_OK);
/* attach the long unex msg buffer */
match_bits = PTL_LONG_MSG;
ignore_bits = ~(PTL_LONG_MSG);
ret = PtlMEAttach(ompi_mtl_portals.ptl_ni_h,
OMPI_MTL_PORTALS_SEND_TABLE_ID,
anyid,
match_bits,
ignore_bits,
PTL_RETAIN,
PTL_INS_AFTER,
&(ompi_mtl_portals.ptl_unex_long_me_h));
assert(ret == PTL_OK);
md.start = NULL;
md.length = 0;
md.threshold = PTL_MD_THRESH_INF;
md.max_size = 0;
md.options = PTL_MD_OP_PUT | PTL_MD_TRUNCATE | PTL_MD_ACK_DISABLE;
md.eq_handle = ompi_mtl_portals.ptl_unex_eq_h;
md.user_ptr = NULL;
ret = PtlMDAttach(ompi_mtl_portals.ptl_unex_long_me_h,
md,
PTL_RETAIN,
&unex_long_md_h);
assert(ret == PTL_OK);
/* attach catchalls to the send, ack, and read portals */
md.eq_handle = ompi_mtl_portals.ptl_eq_h;
/* catchall for the send portal */
catchall_send_request.event_callback = ompi_mtl_portals_catchall_callback;
md.user_ptr = &catchall_send_request;
ret = PtlMEMDPost(ompi_mtl_portals.ptl_ni_h,
ompi_mtl_portals.ptl_unex_long_me_h,
anyid,
0,
~0,
PTL_RETAIN,
PTL_INS_AFTER,
md,
PTL_UNLINK,
&(ompi_mtl_portals.ptl_send_catchall_me_h),
&send_catchall_md_h,
ompi_mtl_portals.ptl_empty_eq_h);
assert(ret == PTL_OK);
/* catchall for ack portal */
catchall_ack_request.event_callback = ompi_mtl_portals_catchall_callback;
md.user_ptr = &catchall_ack_request;
ret = PtlMEAttach(ompi_mtl_portals.ptl_ni_h,
OMPI_MTL_PORTALS_ACK_TABLE_ID,
anyid,
0,
~0,
PTL_RETAIN,
PTL_INS_AFTER,
&(ompi_mtl_portals.ptl_ack_catchall_me_h));
assert(ret == PTL_OK);
ret = PtlMDAttach(ompi_mtl_portals.ptl_ack_catchall_me_h,
md,
PTL_UNLINK,
&ack_catchall_md_h);
assert(ret == PTL_OK);
/* catchall for read portal */
catchall_read_request.event_callback = ompi_mtl_portals_catchall_callback;
md.user_ptr = &catchall_read_request;
ret = PtlMEAttach(ompi_mtl_portals.ptl_ni_h,
OMPI_MTL_PORTALS_READ_TABLE_ID,
anyid,
0,
~0,
PTL_RETAIN,
PTL_INS_AFTER,
&(ompi_mtl_portals.ptl_read_catchall_me_h));
assert(ret == PTL_OK);
ret = PtlMDAttach(ompi_mtl_portals.ptl_read_catchall_me_h,
md,
PTL_RETAIN,
&read_catchall_md_h);
assert(ret == PTL_OK);
/* attach short unex recv blocks */
ret = ompi_mtl_portals_recv_short_enable((mca_mtl_portals_module_t*) mtl);
opal_progress_register(ompi_mtl_portals_progress);
/* bind zero-length md for sending zero-length msgs and acks */
md.start = NULL;
md.length = 0;
md.threshold = PTL_MD_THRESH_INF;
md.max_size = 0;
md.options = PTL_MD_EVENT_START_DISABLE | PTL_MD_EVENT_END_DISABLE;
md.user_ptr = NULL;
md.eq_handle = PTL_EQ_NONE;
ret = PtlMDBind(ompi_mtl_portals.ptl_ni_h,
md,
PTL_RETAIN,
&ompi_mtl_portals.ptl_zero_md_h );
assert(ret == PTL_OK);
/* set up the short copy blocks */
ompi_mtl_portals_short_setup();
/* get the list of ptl_process_id_t structures for the given proc
structures. If the Portals runtime environment supports
comm_spawn, we'll be able to support it as well. */
portals_procs = malloc(sizeof(ptl_process_id_t) * nprocs);
if (NULL == portals_procs) goto cleanup;
ret = ompi_common_portals_get_procs(nprocs, procs, portals_procs);
if (OMPI_SUCCESS != ret) goto cleanup;
/* copy the ptl_process_id_t information into our per-proc data
store */
for (i = 0 ; i < nprocs ; ++i) {
mtl_peer_data[i] = malloc(sizeof(struct mca_mtl_base_endpoint_t));
if (NULL == mtl_peer_data[i]) goto cleanup;
mtl_peer_data[i]->ptl_proc.nid = portals_procs[i].nid;
mtl_peer_data[i]->ptl_proc.pid = portals_procs[i].pid;
}
cleanup:
if (NULL != portals_procs) free(portals_procs);
return ret;
}
int
ompi_mtl_portals_del_procs(struct mca_mtl_base_module_t *mtl,
size_t nprocs,
struct ompi_proc_t** procs,
struct mca_mtl_base_endpoint_t **mtl_peer_data)
{
size_t i;
assert(mtl == &ompi_mtl_portals.base);
for (i = 0 ; i < nprocs ; ++i) {
if (NULL != mtl_peer_data[i]) {
free(mtl_peer_data[i]);
}
}
ompi_mtl_portals_recv_short_disable((mca_mtl_portals_module_t *) mtl);
ompi_mtl_portals_short_cleanup();
(void)PtlMDUnlink(ompi_mtl_portals.ptl_zero_md_h);
(void)PtlMDUnlink(send_catchall_md_h);
(void)PtlMDUnlink(ack_catchall_md_h);
(void)PtlMDUnlink(read_catchall_md_h);
(void)PtlMDUnlink(unex_long_md_h);
(void)PtlMEUnlink(ompi_mtl_portals.ptl_unex_long_me_h);
(void)PtlMEUnlink(ompi_mtl_portals.ptl_send_catchall_me_h);
(void)PtlMEUnlink(ompi_mtl_portals.ptl_ack_catchall_me_h);
(void)PtlMEUnlink(ompi_mtl_portals.ptl_read_catchall_me_h);
return OMPI_SUCCESS;
}
int
ompi_mtl_portals_finalize(struct mca_mtl_base_module_t *mtl)
{
assert(mtl == &ompi_mtl_portals.base);
/* Don't try to wait for things to finish if we've never initialized */
if (PTL_INVALID_HANDLE != ompi_mtl_portals.ptl_ni_h) {
ptl_event_t ev;
int ret;
opal_progress_unregister(ompi_mtl_portals_progress);
/* Before progressing remaining events, check whether we don't get PTL_EQ_INVALID */
ret = PtlEQPeek(ompi_mtl_portals.ptl_eq_h, &ev);
if (PTL_EQ_INVALID != ret) {
while (0 != ompi_mtl_portals_progress()) { }
}
}
ompi_common_portals_ni_finalize();
ompi_common_portals_finalize();
return OMPI_SUCCESS;
}
int
ompi_mtl_portals_cancel(struct mca_mtl_base_module_t* mtl,
mca_mtl_request_t *mtl_request,
int flag)
{
return OMPI_SUCCESS;
}
int
ompi_mtl_portals_add_comm(struct mca_mtl_base_module_t *mtl,
struct ompi_communicator_t *comm)
{
return OMPI_SUCCESS;
}
int
ompi_mtl_portals_del_comm(struct mca_mtl_base_module_t *mtl,
struct ompi_communicator_t *comm)
{
return OMPI_SUCCESS;
}
int
ompi_mtl_portals_progress(void)
{
int count = 0, ret;
ptl_event_t ev;
ompi_mtl_portals_request_t *ptl_request;
while (true) {
ret = PtlEQGet(ompi_mtl_portals.ptl_eq_h, &ev);
if (PTL_OK == ret) {
if (ev.type == PTL_EVENT_UNLINK) continue;
if (NULL != ev.md.user_ptr) {
ptl_request = ev.md.user_ptr;
ret = ptl_request->event_callback(&ev, ptl_request);
if (OMPI_SUCCESS != ret) {
opal_output(0, " Error returned from the event callback. Error code - %d \n",ret);
abort();
}
}
} else if (PTL_EQ_EMPTY == ret) {
break;
} else {
opal_output(0, " Error returned from PtlEQGet. Error code - %d \n",ret);
abort();
}
}
/* clean out the unexpected event queue too */
if (ompi_mtl_portals.ptl_aggressive_polling == true) {
while ( NULL != ompi_mtl_portals_search_unex_events(0, ~0, true) );
}
return count;
}

249
ompi/mca/mtl/portals/mtl_portals.h
Просмотреть файл

@ -1,249 +0,0 @@
/*
* Copyright (c) 2004-2006 The Trustees of Indiana University and Indiana
* University Research and Technology
* Corporation. All rights reserved.
* Copyright (c) 2004-2007 The University of Tennessee and The University
* of Tennessee Research Foundation. All rights
* reserved.
* Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
* University of Stuttgart. All rights reserved.
* Copyright (c) 2004-2005 The Regents of the University of California.
* All rights reserved.
* $COPYRIGHT$
*
* Additional copyrights may follow
*
* $HEADER$
*/
#ifndef MTL_PORTALS_H_HAS_BEEN_INCLUDED
#define MTL_PORTALS_H_HAS_BEEN_INCLUDED
#include "ompi_config.h"
#include "opal/class/opal_list.h"
#include "ompi/class/ompi_free_list.h"
#include "ompi/mca/mtl/mtl.h"
#include "ompi/mca/mtl/base/base.h"
#include "opal/datatype/opal_convertor.h"
#include "ompi/mca/common/portals/common_portals.h"
BEGIN_C_DECLS
struct mca_mtl_portals_module_t {
mca_mtl_base_module_t base;
ptl_handle_ni_t ptl_ni_h;
size_t eager_limit;
ptl_handle_eq_t ptl_eq_h;
ptl_handle_eq_t ptl_unex_eq_h;
/* long unex msgs - insert posted recvs before this */
ptl_handle_me_t ptl_unex_long_me_h;
/* send catchall - insert short unex buffers before this */
ptl_handle_me_t ptl_send_catchall_me_h;
/* catchall for ack portal */
ptl_handle_me_t ptl_ack_catchall_me_h;
/* catchall for read portal */
ptl_handle_me_t ptl_read_catchall_me_h;
/* for zero-length sends and acks */
ptl_handle_md_t ptl_zero_md_h;
ompi_free_list_t event_fl;
int ptl_recv_short_mds_num;
int ptl_recv_short_mds_size;
opal_list_t ptl_recv_short_blocks;
opal_list_t unexpected_messages;
int ptl_expected_queue_size;
int ptl_unexpected_queue_size;
/* for send-side copy blocks */
ptl_md_t ptl_short_md;
ptl_handle_md_t ptl_short_md_h;
int ptl_num_copy_blocks;
ptl_size_t ptl_copy_block_len;
int *ptl_copy_block_free_list;
int ptl_copy_block_first_free;
/* empty event queue for PtlMEMDPost() */
ptl_handle_eq_t ptl_empty_eq_h;
/* turn off aggressive polling of the unex msg event queue */
bool ptl_aggressive_polling;
/* use rendezvous for long messages */
bool ptl_use_rendezvous;
/* output channel for debugging */
int portals_output;
};
typedef struct mca_mtl_portals_module_t mca_mtl_portals_module_t;
extern mca_mtl_portals_module_t ompi_mtl_portals;
OMPI_DECLSPEC mca_mtl_base_component_2_0_0_t mca_mtl_portals_component;
struct ompi_mtl_portals_event_t {
struct ompi_free_list_item_t super;
ptl_event_t ev;
bool is_complete;
};
typedef struct ompi_mtl_portals_event_t ompi_mtl_portals_event_t;
OBJ_CLASS_DECLARATION(ompi_mtl_portals_event_t);
/* match/ignore bit manipulation
*
* 0123 4567 01234567 01234567 01234567 01234567 01234567 01234567 01234567
* | | |
* ^ | context id | source | message tag
* | | | |
* +---- protocol
*/
#define PTL_PROTOCOL_MASK 0xF000000000000000ULL
#define PTL_CONTEXT_MASK 0x0FFF000000000000ULL
#define PTL_SOURCE_MASK 0x0000FFFF00000000ULL
#define PTL_TAG_MASK 0x00000000FFFFFFFFULL
#define PTL_PROTOCOL_IGNR PTL_PROTOCOL_MASK
#define PTL_CONTEXT_IGNR PTL_CONTEXT_MASK
#define PTL_SOURCE_IGNR PTL_SOURCE_MASK
#define PTL_TAG_IGNR 0x000000007FFFFFFFULL
#define PTL_SHORT_MSG 0x1000000000000000ULL
#define PTL_LONG_MSG 0x2000000000000000ULL
#define PTL_READY_MSG 0x4000000000000000ULL
/* send posting */
#define PTL_SET_SEND_BITS(match_bits, contextid, source, tag, type) \
{ \
match_bits = contextid; \
match_bits = (match_bits << 16); \
match_bits |= source; \
match_bits = (match_bits << 32); \
match_bits |= (PTL_TAG_MASK & tag) | type; \
}
/* receive posting */
#define PTL_SET_RECV_BITS(match_bits, ignore_bits, contextid, source, tag) \
{ \
match_bits = 0; \
ignore_bits = PTL_PROTOCOL_IGNR; \
\
match_bits = contextid; \
match_bits = (match_bits << 16); \
\
if (MPI_ANY_SOURCE == source) { \
match_bits = (match_bits << 32); \
ignore_bits |= PTL_SOURCE_IGNR; \
} else { \
match_bits |= source; \
match_bits = (match_bits << 32); \
} \
\
if (MPI_ANY_TAG == tag) { \
ignore_bits |= PTL_TAG_IGNR; \
} else { \
match_bits |= (PTL_TAG_MASK & tag); \
} \
}
#define PTL_IS_SHORT_MSG(match_bits) \
(0 != (PTL_SHORT_MSG & match_bits))
#define PTL_IS_LONG_MSG(match_bits) \
(0 != (PTL_LONG_MSG & match_bits))
#define PTL_IS_READY_MSG(match_bits) \
(0 != (PTL_READY_MSG & match_bits))
#define PTL_IS_SYNC_MSG(event) \
(0 != event.hdr_data)
#define PTL_GET_TAG(match_bits) ((int)(match_bits & PTL_TAG_MASK))
#define PTL_GET_SOURCE(match_bits) ((int)((match_bits & PTL_SOURCE_MASK) >> 32))
/* MTL interface functions */
extern int ompi_mtl_portals_finalize(struct mca_mtl_base_module_t *mtl);
extern int ompi_mtl_portals_add_procs(struct mca_mtl_base_module_t* mtl,
size_t nprocs,
struct ompi_proc_t** procs,
struct mca_mtl_base_endpoint_t **mtl_peer_data);
extern int ompi_mtl_portals_del_procs(struct mca_mtl_base_module_t* mtl,
size_t nprocs,
struct ompi_proc_t** procs,
struct mca_mtl_base_endpoint_t **mtl_peer_data);
extern int ompi_mtl_portals_send(struct mca_mtl_base_module_t* mtl,
struct ompi_communicator_t* comm,
int dest,
int tag,
struct opal_convertor_t *convertor,
mca_pml_base_send_mode_t mode);
extern int ompi_mtl_portals_isend(struct mca_mtl_base_module_t* mtl,
struct ompi_communicator_t* comm,
int dest,
int tag,
struct opal_convertor_t *convertor,
mca_pml_base_send_mode_t mode,
bool blocking,
mca_mtl_request_t *mtl_request);
extern int ompi_mtl_portals_irecv(struct mca_mtl_base_module_t* mtl,
struct ompi_communicator_t *comm,
int src,
int tag,
struct opal_convertor_t *convertor,
mca_mtl_request_t *mtl_request);
extern int ompi_mtl_portals_iprobe(struct mca_mtl_base_module_t* mtl,
struct ompi_communicator_t *comm,
int src,
int tag,
int *flag,
struct ompi_status_public_t *status);
extern int ompi_mtl_portals_imrecv(struct mca_mtl_base_module_t* mtl,
struct opal_convertor_t *convertor,
struct ompi_message_t **message,
struct mca_mtl_request_t *mtl_request);
extern int ompi_mtl_portals_improbe(struct mca_mtl_base_module_t *mtl,
struct ompi_communicator_t *comm,
int src,
int tag,
int *matched,
struct ompi_message_t **message,
struct ompi_status_public_t *status);
extern int ompi_mtl_portals_cancel(struct mca_mtl_base_module_t* mtl,
mca_mtl_request_t *mtl_request,
int flag);
extern int ompi_mtl_portals_add_comm(struct mca_mtl_base_module_t *mtl,
struct ompi_communicator_t *comm);
extern int ompi_mtl_portals_del_comm(struct mca_mtl_base_module_t *mtl,
struct ompi_communicator_t *comm);
extern int ompi_mtl_portals_progress(void);
END_C_DECLS
#endif /* MTL_PORTALS_H_HAS_BEEN_INCLUDED */

205
ompi/mca/mtl/portals/mtl_portals_component.c
Просмотреть файл

@ -1,205 +0,0 @@
/*
* Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
* University Research and Technology
* Corporation. All rights reserved.
* Copyright (c) 2004-2005 The University of Tennessee and The University
* of Tennessee Research Foundation. All rights
* reserved.
* Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
* University of Stuttgart. All rights reserved.
* Copyright (c) 2004-2005 The Regents of the University of California.
* All rights reserved.
* $COPYRIGHT$
*
* Additional copyrights may follow
*
* $HEADER$
*/
#include "ompi_config.h"
#include "opal/mca/event/event.h"
#include "opal/util/output.h"
#include "opal/mca/base/mca_base_param.h"
#include "ompi/mca/common/portals/common_portals.h"
#include "mtl_portals.h"
#include "mtl_portals_request.h"
static int ompi_mtl_portals_component_open(void);
static int ompi_mtl_portals_component_close(void);
static mca_mtl_base_module_t* ompi_mtl_portals_component_init(
bool enable_progress_threads, bool enable_mpi_threads);
mca_mtl_base_component_2_0_0_t mca_mtl_portals_component = {
/* First, the mca_base_component_t struct containing meta
* information about the component itself */
{
MCA_MTL_BASE_VERSION_2_0_0,
"portals", /* MCA component name */
OMPI_MAJOR_VERSION, /* MCA component major version */
OMPI_MINOR_VERSION, /* MCA component minor version */
OMPI_RELEASE_VERSION, /* MCA component release version */
ompi_mtl_portals_component_open, /* component open */
ompi_mtl_portals_component_close /* component close */
},
{
/* The component is not checkpoint ready */
MCA_BASE_METADATA_PARAM_NONE
},
ompi_mtl_portals_component_init, /* component init */
};
static opal_output_stream_t mtl_portals_output_stream;
static int
ompi_mtl_portals_component_open(void)
{
int tmp;
ompi_common_portals_register_mca();
ompi_mtl_portals.base.mtl_request_size =
sizeof(ompi_mtl_portals_request_t) -
sizeof(struct mca_mtl_request_t);
mca_base_param_reg_int(&mca_mtl_portals_component.mtl_version,
"eager_limit",
"Cross-over point from eager to rendezvous sends",
false,
false,
128 * 1024,
&tmp);
ompi_mtl_portals.eager_limit = tmp;
mca_base_param_reg_int(&mca_mtl_portals_component.mtl_version,
"short_recv_mds_num",
"Number of short message receive blocks",
false,
false,
3,
&ompi_mtl_portals.ptl_recv_short_mds_num);
mca_base_param_reg_int(&mca_mtl_portals_component.mtl_version,
"short_recv_mds_size",
"Size of short message receive blocks",
false,
false,
15 * 1024 * 1024,
&ompi_mtl_portals.ptl_recv_short_mds_size);
OBJ_CONSTRUCT(&mtl_portals_output_stream, opal_output_stream_t);
mtl_portals_output_stream.lds_is_debugging = true;
mtl_portals_output_stream.lds_want_stdout = true;
mtl_portals_output_stream.lds_file_suffix = "btl-portals";
mca_base_param_reg_int(&mca_mtl_portals_component.mtl_version,
"debug_level",
"Debugging verbosity (0 - 100)",
false,
false,
0,
&(mtl_portals_output_stream.lds_verbose_level));
asprintf(&(mtl_portals_output_stream.lds_prefix),
"btl: portals (%s): ", ompi_common_portals_nodeid());
ompi_mtl_portals.portals_output =
opal_output_open(&mtl_portals_output_stream);
ompi_mtl_portals.ptl_ni_h = PTL_INVALID_HANDLE;
mca_base_param_reg_int(&mca_mtl_portals_component.mtl_version,
"expected_queue_size",
"Size of the expected receive queue in bytes",
false,
false,
1024,
&ompi_mtl_portals.ptl_expected_queue_size);
mca_base_param_reg_int(&mca_mtl_portals_component.mtl_version,
"unexpected_queue_size",
"Size of the unexpected receive queue in bytes",
false,
false,
1024,
&ompi_mtl_portals.ptl_unexpected_queue_size);
mca_base_param_reg_int(&mca_mtl_portals_component.mtl_version,
"num_copy_blocks",
"Number of short message copy blocks",
false,
false,
256,
&ompi_mtl_portals.ptl_num_copy_blocks);
mca_base_param_reg_int(&mca_mtl_portals_component.mtl_version,
"copy_block_len",
"Length (in bytes) of each short message copy block",
false,
false,
8192,
&tmp);
ompi_mtl_portals.ptl_copy_block_len = tmp;
mca_base_param_reg_int(&mca_mtl_portals_component.mtl_version,
"aggressive_polling",
"Turn off aggressive polling of unexpected messages",
false,
false,
1,
&tmp);
ompi_mtl_portals.ptl_aggressive_polling = (tmp == 0) ? false : true;
mca_base_param_reg_int(&mca_mtl_portals_component.mtl_version,
"use_rendezvous",
"Use a rendezvous protocol for long messages",
false,
false,
0,
&tmp);
ompi_mtl_portals.ptl_use_rendezvous = ((tmp == 0) ? false : true);
return OMPI_SUCCESS;
}
static int
ompi_mtl_portals_component_close(void)
{
return OMPI_SUCCESS;
}
static mca_mtl_base_module_t*
ompi_mtl_portals_component_init(bool enable_progress_threads,
bool enable_mpi_threads)
{
bool accel;
/* we don't run with no stinkin' threads */
if (enable_progress_threads || enable_mpi_threads) return NULL;
/* initialize our interface */
if (OMPI_SUCCESS != ompi_common_portals_initialize(&(ompi_mtl_portals.ptl_ni_h), &accel)) {
return NULL;
}
OBJ_CONSTRUCT(&ompi_mtl_portals.event_fl, ompi_free_list_t);
ompi_free_list_init_new(&ompi_mtl_portals.event_fl,
sizeof(ompi_mtl_portals_event_t),
opal_cache_line_size,
OBJ_CLASS(ompi_mtl_portals_event_t),
0,opal_cache_line_size,
1, -1, 1, NULL);
OBJ_CONSTRUCT(&ompi_mtl_portals.ptl_recv_short_blocks, opal_list_t);
OBJ_CONSTRUCT(&ompi_mtl_portals.unexpected_messages, opal_list_t);
return &ompi_mtl_portals.base;
}

27
ompi/mca/mtl/portals/mtl_portals_endpoint.h
Просмотреть файл

@ -1,27 +0,0 @@
/*
* Copyright (c) 2004-2006 The Trustees of Indiana University and Indiana
* University Research and Technology
* Corporation. All rights reserved.
* Copyright (c) 2004-2005 The University of Tennessee and The University
* of Tennessee Research Foundation. All rights
* reserved.
* Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
* University of Stuttgart. All rights reserved.
* Copyright (c) 2004-2005 The Regents of the University of California.
* All rights reserved.
* $COPYRIGHT$
*
* Additional copyrights may follow
*
* $HEADER$
*/
#ifndef OMPI_MTL_PORTALS_ENDPOINT_H
#define OMPI_MTL_PORTALS_ENDPOINT_H
struct mca_mtl_base_endpoint_t {
ptl_process_id_t ptl_proc;
};
typedef struct mca_mtl_base_endpoint_t mca_mtl_base_endpoint_t;
#endif

71
ompi/mca/mtl/portals/mtl_portals_probe.c
Просмотреть файл

@ -1,71 +0,0 @@
/*
* Copyright (c) 2004-2006 The Trustees of Indiana University and Indiana
* University Research and Technology
* Corporation. All rights reserved.
* Copyright (c) 2004-2010 The University of Tennessee and The University
* of Tennessee Research Foundation. All rights
* reserved.
* Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
* University of Stuttgart. All rights reserved.
* Copyright (c) 2004-2005 The Regents of the University of California.
* All rights reserved.
* $COPYRIGHT$
*
* Additional copyrights may follow
*
* $HEADER$
*/
#include "ompi_config.h"
#include "ompi/communicator/communicator.h"
#include "mtl_portals.h"
#include "mtl_portals_request.h"
#include "mtl_portals_recv.h"
int
ompi_mtl_portals_iprobe(struct mca_mtl_base_module_t* mtl,
struct ompi_communicator_t *comm,
int src,
int tag,
int *flag,
struct ompi_status_public_t *status)
{
ptl_match_bits_t match_bits;
ptl_match_bits_t ignore_bits;
ompi_mtl_portals_event_t *recv_event = NULL;
PTL_SET_RECV_BITS(match_bits, ignore_bits, comm->c_contextid, src, tag);
/* first, check the queue of processed unexpected messages */
recv_event = ompi_mtl_portals_search_unex_q(match_bits, ignore_bits, true);
if (NULL == recv_event) {
/* check for new events */
recv_event = ompi_mtl_portals_search_unex_events(match_bits, ignore_bits, true);
}
if ( NULL != recv_event ) {
/* found it */
*flag = 1;
status->MPI_SOURCE = PTL_GET_SOURCE(recv_event->ev.match_bits);
status->MPI_TAG = PTL_GET_TAG(recv_event->ev.match_bits);
status->_ucount = recv_event->ev.rlength;
status->MPI_ERROR = OMPI_SUCCESS;
} else {
*flag = 0;
}
return OMPI_SUCCESS;
}
int
ompi_mtl_portals_improbe(struct mca_mtl_base_module_t *mtl,
struct ompi_communicator_t *comm,
int src,
int tag,
int *matched,
struct ompi_message_t **message,
struct ompi_status_public_t *status)
{
return OMPI_ERR_NOT_IMPLEMENTED;
}

573
ompi/mca/mtl/portals/mtl_portals_recv.c
Просмотреть файл

@ -1,573 +0,0 @@
/*
* Copyright (c) 2004-2006 The Trustees of Indiana University and Indiana
* University Research and Technology
* Corporation. All rights reserved.
* Copyright (c) 2004-2010 The University of Tennessee and The University
* of Tennessee Research Foundation. All rights
* reserved.
* Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
* University of Stuttgart. All rights reserved.
* Copyright (c) 2004-2005 The Regents of the University of California.
* All rights reserved.
* $COPYRIGHT$
*
* Additional copyrights may follow
*
* $HEADER$
*/
#include "ompi_config.h"
#include "opal/class/opal_list.h"
#include "ompi/communicator/communicator.h"
#include "ompi/datatype/ompi_datatype.h"
#include "opal/datatype/opal_convertor.h"
#include "ompi/mca/mtl/base/base.h"
#include "ompi/mca/mtl/base/mtl_base_datatype.h"
#include "mtl_portals.h"
#include "mtl_portals_endpoint.h"
#include "mtl_portals_request.h"
#include "mtl_portals_recv.h"
#include "mtl_portals_recv_short.h"
#define CHECK_MATCH(incoming_bits, match_bits, ignore_bits) \
(((incoming_bits ^ match_bits) & ~ignore_bits) == 0)
static int
ompi_mtl_portals_recv_progress(ptl_event_t *, struct ompi_mtl_portals_request_t* );
static int
ompi_mtl_portals_rendezvous_get(ptl_event_t *ev,
ompi_mtl_portals_request_t *ptl_request)
{
ptl_md_t md;
ptl_handle_md_t md_h;
int ret;
md.start = ev->md.start;
md.length = ev->md.length;
md.threshold = 2; /* send and reply */
md.options = PTL_MD_EVENT_START_DISABLE;
md.user_ptr = ptl_request;
md.eq_handle = ompi_mtl_portals.ptl_eq_h;
ret = PtlMDBind(ompi_mtl_portals.ptl_ni_h, md, PTL_UNLINK, &md_h);
if (PTL_OK != ret) {
opal_output(fileno(stderr)," Error returned from PtlMDBind(). Error code - %d \n",ret);
abort();
}
ptl_request->is_complete = false;
ptl_request->event_callback = ompi_mtl_portals_recv_progress;
ret = PtlGet(md_h,
ev->initiator,
OMPI_MTL_PORTALS_READ_TABLE_ID,
0,
ev->hdr_data,
0);
if (PTL_OK != ret) {
opal_output(fileno(stderr)," Error returned from PtlGet. Error code - %d \n",ret);
abort();
}
/* stay here until the reply comes */
while (ptl_request->is_complete == false) {
ompi_mtl_portals_progress();
}
return OMPI_SUCCESS;
}
/* called when a receive should be progressed */
static int
ompi_mtl_portals_recv_progress(ptl_event_t *ev,
struct ompi_mtl_portals_request_t* ptl_request)
{
int ret;
switch (ev->type) {
case PTL_EVENT_PUT_END:
if (PTL_IS_LONG_MSG(ev->match_bits) && (ompi_mtl_portals.ptl_use_rendezvous == true)) {
/* get the data */
ret = ompi_mtl_portals_rendezvous_get(ev, ptl_request);
if ( OMPI_SUCCESS != ret ) {
opal_output(fileno(stderr)," Error returned from ompi_mtl_portals_rendezvous_get(). Error code - %d \n",ret);
return ret;
}
}
/* make sure the data is in the right place */
ret = ompi_mtl_datatype_unpack(ptl_request->convertor,
ev->md.start, ev->mlength);
if (OMPI_SUCCESS != ret) return ret;
/* set the status */
ptl_request->super.ompi_req->req_status.MPI_SOURCE =
PTL_GET_SOURCE(ev->match_bits);
ptl_request->super.ompi_req->req_status.MPI_TAG =
PTL_GET_TAG(ev->match_bits);
ptl_request->super.ompi_req->req_status.MPI_ERROR =
(ev->rlength > ev->mlength) ?
MPI_ERR_TRUNCATE : MPI_SUCCESS;
ptl_request->super.ompi_req->req_status._ucount =
ev->mlength;
OPAL_OUTPUT_VERBOSE((50, ompi_mtl_base_output,
"recv complete: 0x%016llx\n", ev->match_bits));
ptl_request->super.completion_callback(&ptl_request->super);
ptl_request->is_complete = true;
break;
case PTL_EVENT_REPLY_END:
/* make sure the data is in the right place */
ret = ompi_mtl_datatype_unpack(ptl_request->convertor, ev->md.start, ev->mlength);
if (OMPI_SUCCESS != ret) return ret;
/* set the status - most of this filled in right after issuing
the PtlGet*/
ptl_request->super.ompi_req->req_status._ucount =
ev->mlength;
OPAL_OUTPUT_VERBOSE((50, ompi_mtl_base_output,
"recv complete: 0x%016llx\n", ev->match_bits));
ptl_request->super.completion_callback(&ptl_request->super);
ptl_request->is_complete = true;
break;
default:
break;
}
return OMPI_SUCCESS;
}
static int
ompi_mtl_portals_get_data(ompi_mtl_portals_event_t *recv_event,
struct opal_convertor_t *convertor,
ompi_mtl_portals_request_t *ptl_request)
{
int ret;
ptl_md_t md;
ptl_handle_md_t md_h;
size_t buflen;
if (PTL_IS_SHORT_MSG(recv_event->ev.match_bits)) {
/* the buffer is sitting in the short message queue */
struct iovec iov;
uint32_t iov_count = 1;
size_t max_data;
ompi_mtl_portals_recv_short_block_t *block =
recv_event->ev.md.user_ptr;
iov.iov_base = (((char*) recv_event->ev.md.start) + recv_event->ev.offset);
iov.iov_len = recv_event->ev.mlength;
max_data = iov.iov_len;
/* see if this message filled the receive block */
if (recv_event->ev.md.length - (recv_event->ev.offset +
recv_event->ev.mlength) <
(ptl_size_t) recv_event->ev.md.max_size) {
block->full = true;
}
/* pull out the data */
if (iov.iov_len > 0) {
ret = opal_convertor_unpack(convertor, &iov, &iov_count,
&max_data );
if (0 > ret) return ret;
}
/* if synchronous, return an ack */
if (PTL_IS_SYNC_MSG(recv_event->ev)) {
md.length = 0;
md.start = (((char*) recv_event->ev.md.start) + recv_event->ev.offset);
md.threshold = 1; /* send */
md.options = PTL_MD_EVENT_START_DISABLE;
md.user_ptr = NULL;
md.eq_handle = ompi_mtl_portals.ptl_eq_h;
ret = PtlMDBind(ompi_mtl_portals.ptl_ni_h, md,
PTL_UNLINK, &md_h);
if (PTL_OK != ret) {
opal_output(fileno(stderr)," Error returned from PtlMDBind. Error code - %d \n",ret);
abort();
}
OPAL_OUTPUT_VERBOSE((50, ompi_mtl_base_output,
"acking recv: 0x%016llx\n",
recv_event->ev.match_bits));
ret = PtlPut(md_h,
PTL_NO_ACK_REQ,
recv_event->ev.initiator,
OMPI_MTL_PORTALS_ACK_TABLE_ID,
0,
recv_event->ev.hdr_data,
0,
0);
if (PTL_OK != ret) {
opal_output(fileno(stderr)," Error returned from PtlPut. Error code - %d \n",ret);
abort();
}
}
/* finished with our buffer space */
ompi_mtl_portals_return_block_part(&ompi_mtl_portals, block);
opal_convertor_get_packed_size(convertor, &buflen);
ptl_request->super.ompi_req->req_status.MPI_SOURCE =
PTL_GET_SOURCE(recv_event->ev.match_bits);
ptl_request->super.ompi_req->req_status.MPI_TAG =
PTL_GET_TAG(recv_event->ev.match_bits);
ptl_request->super.ompi_req->req_status.MPI_ERROR =
(recv_event->ev.rlength > buflen) ?
MPI_ERR_TRUNCATE : MPI_SUCCESS;
ptl_request->super.ompi_req->req_status._ucount =
recv_event->ev.mlength;
OPAL_OUTPUT_VERBOSE((50, ompi_mtl_base_output,
"recv complete: 0x%016llx\n",
recv_event->ev.match_bits));
ptl_request->super.completion_callback(&ptl_request->super);
ptl_request->is_complete = true;
} else {
ret = ompi_mtl_datatype_recv_buf(convertor, &md.start, &buflen,
&ptl_request->free_after);
if (OMPI_SUCCESS != ret) {
opal_output(fileno(stderr)," Error returned from ompi_mtl_datatype_recv_buf. Error code - %d \n",ret);
abort();
}
md.length = (recv_event->ev.rlength > buflen) ? buflen : recv_event->ev.rlength;
md.threshold = 2; /* send and get */
md.options = PTL_MD_EVENT_START_DISABLE;
md.user_ptr = ptl_request;
md.eq_handle = ompi_mtl_portals.ptl_eq_h;
/* retain because it's unclear how many events we'll get here.
Some implementations give just the REPLY, others give SEND
and REPLY */
ret = PtlMDBind(ompi_mtl_portals.ptl_ni_h, md,
PTL_RETAIN, &md_h);
if (PTL_OK != ret) {
opal_output(fileno(stderr)," Error returned from ompi_mtl_datatype_recv_buf. Error code - %d \n",ret);
abort();
}
ptl_request->event_callback = ompi_mtl_portals_recv_progress;
ret = PtlGet(md_h,
recv_event->ev.initiator,
OMPI_MTL_PORTALS_READ_TABLE_ID,
0,
recv_event->ev.hdr_data,
0);
if (PTL_OK != ret) {
opal_output(fileno(stderr)," Error returned from PtlGet. Error code - %d \n",ret);
abort();
}
ptl_request->super.ompi_req->req_status.MPI_SOURCE =
PTL_GET_SOURCE(recv_event->ev.match_bits);
ptl_request->super.ompi_req->req_status.MPI_TAG =
PTL_GET_TAG(recv_event->ev.match_bits);
ptl_request->super.ompi_req->req_status.MPI_ERROR =
(recv_event->ev.rlength > buflen) ?
MPI_ERR_TRUNCATE : MPI_SUCCESS;
}
return OMPI_SUCCESS;
}
static void
ompi_mtl_portals_match_up_put_end(ptl_seq_t link)
{
opal_list_item_t *list_item;
/* match up a PUT_END event with its corresponding PUT_START event */
list_item = opal_list_get_first(&ompi_mtl_portals.unexpected_messages);
while (list_item != opal_list_get_end(&ompi_mtl_portals.unexpected_messages)) {
opal_list_item_t *next_item = opal_list_get_next(list_item);
ompi_mtl_portals_event_t *recv_event = (ompi_mtl_portals_event_t*) list_item;
if (recv_event->ev.link == link) {
recv_event->is_complete = true;
return;
}
list_item = next_item;
}
/* should never get here */
opal_output(fileno(stderr)," ompi_mtl_portals_match_up_put_end failed \n");
abort();
}
static void
ompi_mtl_portals_wait_for_put_end(ptl_seq_t link)
{
ptl_event_t ev;
int ret;
/* wait for a PUT_END event that matches the message we're looking for */
while (true) {
ret = PtlEQWait(ompi_mtl_portals.ptl_unex_eq_h,&ev);
if (PTL_OK == ret) {
if (PTL_EVENT_PUT_START == ev.type) {
ompi_free_list_item_t *item;
ompi_mtl_portals_event_t *recv_event;
OMPI_FREE_LIST_GET(&ompi_mtl_portals.event_fl, item, ret);
recv_event = (ompi_mtl_portals_event_t*) item;
recv_event->ev = ev;
recv_event->is_complete = false;
opal_list_append(&(ompi_mtl_portals.unexpected_messages),
(opal_list_item_t*) recv_event);
if (PTL_IS_SHORT_MSG(recv_event->ev.match_bits)) {
ompi_mtl_portals_recv_short_block_t *block =
recv_event->ev.md.user_ptr;
OPAL_THREAD_ADD32(&block->pending, 1);
}
} else if (PTL_EVENT_PUT_END == ev.type) {
if (link == ev.link) {
/* the one we want */
return;
}
/* otherwise match it up */
ompi_mtl_portals_match_up_put_end(ev.link);
} else {
opal_output(fileno(stderr)," Unrecognised event type - %d - ompi_mtl_portals_wait_for_put_end : %d \n",ev.type,ret);
abort();
}
} else {
opal_output(fileno(stderr)," Error returned in ompi_mtl_portals_wait_for_put_end from PtlEQWait : %d \n",ret);
abort();
}
}
}
ompi_mtl_portals_event_t*
ompi_mtl_portals_search_unex_events(ptl_match_bits_t match_bits,
ptl_match_bits_t ignore_bits,
bool probe)
{
ptl_event_t ev;
int ret;
/* check to see if there are any events in the unexpected event queue */
while (true) {
ret = PtlEQGet(ompi_mtl_portals.ptl_unex_eq_h,&ev);
if (PTL_OK == ret) {
if (PTL_EVENT_PUT_START == ev.type) {
ompi_free_list_item_t *item;
ompi_mtl_portals_event_t *recv_event;
OMPI_FREE_LIST_GET(&ompi_mtl_portals.event_fl, item, ret);
recv_event = (ompi_mtl_portals_event_t*) item;
recv_event->ev = ev;
recv_event->is_complete = false;
if (PTL_IS_SHORT_MSG(recv_event->ev.match_bits)) {
ompi_mtl_portals_recv_short_block_t *block =
recv_event->ev.md.user_ptr;
OPAL_THREAD_ADD32(&block->pending, 1);
}
if (CHECK_MATCH(recv_event->ev.match_bits, match_bits, ignore_bits)) {
/* the one we want */
if (probe == false) {
ompi_mtl_portals_wait_for_put_end(recv_event->ev.link);
} else {
/* just probing, so add it to the unex list */
opal_list_append(&(ompi_mtl_portals.unexpected_messages),
(opal_list_item_t*) recv_event);
}
return recv_event;
} else {
/* not the one we want, so add it to the unex list */
opal_list_append(&(ompi_mtl_portals.unexpected_messages),
(opal_list_item_t*) recv_event);
}
} else if (PTL_EVENT_PUT_END == ev.type) {
/* can't be the one we want */
ompi_mtl_portals_match_up_put_end(ev.link);
} else {
opal_output(fileno(stderr)," Unrecognised event type - %d - ompi_mtl_portals_search_unex_events : %d \n",ev.type,ret);
abort();
}
} else if (PTL_EQ_EMPTY == ret) {
break;
} else {
opal_output(fileno(stderr)," Error returned in ompi_mtl_portals_search_unex_events from PtlEQWait : %d \n",ret);
abort();
}
}
return NULL;
}
ompi_mtl_portals_event_t*
ompi_mtl_portals_search_unex_q(ptl_match_bits_t match_bits,
ptl_match_bits_t ignore_bits,
bool probe)
{
opal_list_item_t *list_item;
ompi_mtl_portals_event_t *recv_event = NULL;
/* check the queue of processed unexpected messages */
list_item = opal_list_get_first(&ompi_mtl_portals.unexpected_messages);
while (list_item != opal_list_get_end(&ompi_mtl_portals.unexpected_messages)) {
opal_list_item_t *next_item = opal_list_get_next(list_item);
recv_event = (ompi_mtl_portals_event_t*) list_item;
if (CHECK_MATCH(recv_event->ev.match_bits, match_bits, ignore_bits)) {
/* we have a match... */
if ( probe == false ) {
if ( false == recv_event->is_complete) {
/* wait for put end event */
ompi_mtl_portals_wait_for_put_end(recv_event->ev.link);
}
opal_list_remove_item(&(ompi_mtl_portals.unexpected_messages),
list_item);
}
return recv_event;
}
list_item = next_item;
}
/* didn't find it */
return NULL;
}
int
ompi_mtl_portals_irecv(struct mca_mtl_base_module_t* mtl,
struct ompi_communicator_t *comm,
int src,
int tag,
struct opal_convertor_t *convertor,
mca_mtl_request_t *mtl_request)
{
ptl_match_bits_t match_bits, ignore_bits;
ptl_md_t md;
ptl_handle_md_t md_h;
ptl_handle_me_t me_h;
int ret;
ptl_process_id_t remote_proc;
mca_mtl_base_endpoint_t *endpoint = NULL;
ompi_mtl_portals_request_t *ptl_request =
(ompi_mtl_portals_request_t*) mtl_request;
ompi_mtl_portals_event_t *recv_event = NULL;
size_t buflen;
bool did_once = false;
ptl_request->convertor = convertor;
if (MPI_ANY_SOURCE == src) {
remote_proc.nid = PTL_NID_ANY;
remote_proc.pid = PTL_PID_ANY;
} else {
ompi_proc_t* ompi_proc = ompi_comm_peer_lookup( comm, src );
endpoint = (mca_mtl_base_endpoint_t*) ompi_proc->proc_pml;
remote_proc = endpoint->ptl_proc;
}
PTL_SET_RECV_BITS(match_bits, ignore_bits, comm->c_contextid,
src, tag);
OPAL_OUTPUT_VERBOSE((50, ompi_mtl_base_output,
"recv bits: 0x%016llx 0x%016llx\n",
match_bits, ignore_bits));
/* first, check the queue of processed unexpected messages */
recv_event = ompi_mtl_portals_search_unex_q(match_bits, ignore_bits, false);
if (NULL != recv_event) {
/* found it */
ret = ompi_mtl_portals_get_data(recv_event, convertor, ptl_request);
OMPI_FREE_LIST_RETURN(&ompi_mtl_portals.event_fl,
(ompi_free_list_item_t*)recv_event);
goto cleanup;
} else {
restart_search:
/* check unexpected events */
recv_event = ompi_mtl_portals_search_unex_events(match_bits, ignore_bits, false);
if (NULL != recv_event) {
/* found it */
ret = ompi_mtl_portals_get_data(recv_event, convertor, ptl_request);
OMPI_FREE_LIST_RETURN(&ompi_mtl_portals.event_fl,
(ompi_free_list_item_t*)recv_event);
goto cleanup;
}
}
/* didn't find it, now post the receive */
if ( false == did_once ) {
ret = ompi_mtl_datatype_recv_buf(convertor, &md.start, &buflen,
&ptl_request->free_after);
if (OMPI_SUCCESS != ret) {
opal_output(fileno(stderr)," Error returned from ompi_mtl_datatype_recv_buf(). Error code - %d \n",ret);
abort();
}
did_once = true;
}
md.length = buflen;
md.threshold = 1;
md.options = PTL_MD_OP_PUT | PTL_MD_TRUNCATE | PTL_MD_EVENT_START_DISABLE;
md.user_ptr = ptl_request;
md.eq_handle = ompi_mtl_portals.ptl_eq_h;
ret = PtlMEMDPost(ompi_mtl_portals.ptl_ni_h,
ompi_mtl_portals.ptl_unex_long_me_h,
remote_proc,
match_bits,
ignore_bits,
PTL_UNLINK,
PTL_INS_BEFORE,
md,
PTL_UNLINK,
&me_h,
&md_h,
ompi_mtl_portals.ptl_unex_eq_h);
if (ret == PTL_MD_NO_UPDATE) {
/* a message has arrived since we searched - look again */
goto restart_search;
} else if( PTL_OK != ret ) {
return ompi_common_portals_error_ptl_to_ompi(ret);
}
ptl_request->event_callback = ompi_mtl_portals_recv_progress;
return OMPI_SUCCESS;
cleanup:
if ((did_once == true) && (ptl_request->free_after)) {
free(md.start);
}
return ret;
}
int
ompi_mtl_portals_imrecv(struct mca_mtl_base_module_t* mtl,
struct opal_convertor_t *convertor,
struct ompi_message_t **message,
struct mca_mtl_request_t *mtl_request)
{
return OMPI_ERR_NOT_IMPLEMENTED;
}

32
ompi/mca/mtl/portals/mtl_portals_recv.h
Просмотреть файл

@ -1,32 +0,0 @@
/*
* Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
* University Research and Technology
* Corporation. All rights reserved.
* Copyright (c) 2004-2006 The University of Tennessee and The University
* of Tennessee Research Foundation. All rights
* reserved.
* Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
* University of Stuttgart. All rights reserved.
* Copyright (c) 2004-2005 The Regents of the University of California.
* All rights reserved.
* $COPYRIGHT$
*
* Additional copyrights may follow
*
* $HEADER$
*/
#ifndef OMPI_MTL_PORTALS_RECV_H
#define OMPI_MTL_PORTALS_RECV_H
extern ompi_mtl_portals_event_t*
ompi_mtl_portals_search_unex_events(ptl_match_bits_t match_bits,
ptl_match_bits_t ignore_bits,
bool probe);
extern ompi_mtl_portals_event_t*
ompi_mtl_portals_search_unex_q(ptl_match_bits_t match_bits,
ptl_match_bits_t ignore_bits,
bool probe);
#endif /* OMPI_MTL_PORTALS_RECV_SHORT_H */

114
ompi/mca/mtl/portals/mtl_portals_recv_short.c
Просмотреть файл

@ -1,114 +0,0 @@
/*
* Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
* University Research and Technology
* Corporation. All rights reserved.
* Copyright (c) 2004-2005 The University of Tennessee and The University
* of Tennessee Research Foundation. All rights
* reserved.
* Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
* University of Stuttgart. All rights reserved.
* Copyright (c) 2004-2005 The Regents of the University of California.
* All rights reserved.
* $COPYRIGHT$
*
* Additional copyrights may follow
*
* $HEADER$
*/
#include "ompi_config.h"
#include "ompi/constants.h"
#include "mtl_portals.h"
#include "mtl_portals_recv_short.h"
OBJ_CLASS_INSTANCE(ompi_mtl_portals_recv_short_block_t,
opal_list_item_t,
NULL, NULL);
int
ompi_mtl_portals_recv_short_enable(mca_mtl_portals_module_t *mtl)
{
int i;
/* create the recv blocks */
for (i = 0 ; i < mtl->ptl_recv_short_mds_num ; ++i) {
ompi_mtl_portals_recv_short_block_t *block =
ompi_mtl_portals_recv_short_block_init(mtl);
if (NULL == block) {
ompi_mtl_portals_recv_short_disable(mtl);
return OMPI_ERROR;
}
opal_list_append(&(mtl->ptl_recv_short_blocks),
(opal_list_item_t*) block);
ompi_mtl_portals_activate_block(block);
}
return OMPI_SUCCESS;
}
int
ompi_mtl_portals_recv_short_disable(mca_mtl_portals_module_t *mtl)
{
opal_list_item_t *item;
if (opal_list_get_size(&mtl->ptl_recv_short_blocks) > 0) {
while (NULL !=
(item = opal_list_remove_first(&mtl->ptl_recv_short_blocks))) {
ompi_mtl_portals_recv_short_block_t *block =
(ompi_mtl_portals_recv_short_block_t*) item;
ompi_mtl_portals_recv_short_block_free(block);
}
}
return OMPI_SUCCESS;
}
ompi_mtl_portals_recv_short_block_t*
ompi_mtl_portals_recv_short_block_init(mca_mtl_portals_module_t *mtl)
{
ompi_mtl_portals_recv_short_block_t *block;
block = OBJ_NEW(ompi_mtl_portals_recv_short_block_t);
block->mtl = mtl;
block->length = mtl->ptl_recv_short_mds_size;
block->start = malloc(block->length);
if (block->start == NULL) return NULL;
block->me_h = PTL_INVALID_HANDLE;
block->md_h = PTL_INVALID_HANDLE;
block->full = false;
block->pending = 0;
return block;
}
int
ompi_mtl_portals_recv_short_block_free(ompi_mtl_portals_recv_short_block_t *block)
{
/* need to clear out the md */
while (block->pending != 0) {
ompi_mtl_portals_progress();
}
if (PTL_INVALID_HANDLE != block->md_h) {
PtlMDUnlink(block->md_h);
block->md_h = PTL_INVALID_HANDLE;
}
if (NULL != block->start) {
free(block->start);
block->start = NULL;
}
block->length = 0;
block->full = false;
return OMPI_SUCCESS;
}

134
ompi/mca/mtl/portals/mtl_portals_recv_short.h
Просмотреть файл

@ -1,134 +0,0 @@
/*
* Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
* University Research and Technology
* Corporation. All rights reserved.
* Copyright (c) 2004-2006 The University of Tennessee and The University
* of Tennessee Research Foundation. All rights
* reserved.
* Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
* University of Stuttgart. All rights reserved.
* Copyright (c) 2004-2005 The Regents of the University of California.
* All rights reserved.
* $COPYRIGHT$
*
* Additional copyrights may follow
*
* $HEADER$
*/
#ifndef OMPI_MTL_PORTALS_RECV_SHORT_H
#define OMPI_MTL_PORTALS_RECV_SHORT_H
struct ompi_mtl_portals_recv_short_block_t {
opal_list_item_t base;
mca_mtl_portals_module_t *mtl;
void *start;
size_t length;
ptl_handle_me_t me_h;
ptl_handle_md_t md_h;
volatile bool full;
volatile int32_t pending;
};
typedef struct ompi_mtl_portals_recv_short_block_t ompi_mtl_portals_recv_short_block_t;
OBJ_CLASS_DECLARATION(ompi_mtl_portals_recv_short_block_t);
extern int
ompi_mtl_portals_recv_short_enable(mca_mtl_portals_module_t *mtl);
extern int
ompi_mtl_portals_recv_short_disable(mca_mtl_portals_module_t *mtl);
/**
* Create a block of memory for receiving send messages. Must call
* activate_block on the returned block of memory before it will be
* active with the POrtals library
*
* Module lock must be held before calling this function
*/
extern ompi_mtl_portals_recv_short_block_t*
ompi_mtl_portals_recv_short_block_init(mca_mtl_portals_module_t *mtl);
/**
* Free a block of memory. Will remove the match entry, then progress
* Portals until the pending count is returned to 0. Will then free
* all resources associated with block.
*
* Module lock must be held before calling this function
*/
extern int
ompi_mtl_portals_recv_short_block_free(ompi_mtl_portals_recv_short_block_t *block);
/**
* activate a block. Blocks that are full (have gone inactive) can be
* re-activated with this call. There is no need to hold the lock
* before calling this function
*/
static inline int
ompi_mtl_portals_activate_block(ompi_mtl_portals_recv_short_block_t *block)
{
int ret;
ptl_process_id_t any_proc = { PTL_NID_ANY, PTL_PID_ANY };
ptl_md_t md;
uint64_t match_bits = PTL_SHORT_MSG;
uint64_t ignore_bits = PTL_CONTEXT_MASK | PTL_SOURCE_MASK | PTL_TAG_MASK;
/* if we have pending operations, something very, very, very bad
has happened... */
assert(block->pending == 0);
if (NULL == block->start) return OMPI_ERROR;
md.start = block->start;
md.length = block->length;
md.threshold = PTL_MD_THRESH_INF;
md.max_size = block->mtl->eager_limit;
md.options = PTL_MD_OP_PUT | PTL_MD_MAX_SIZE | PTL_MD_ACK_DISABLE;
md.user_ptr = block;
md.eq_handle = block->mtl->ptl_unex_eq_h;
block->pending = 0;
block->full = false;
/* make sure that everyone sees the update on full value */
opal_atomic_mb();
ret = PtlMEMDPost(ompi_mtl_portals.ptl_ni_h,
ompi_mtl_portals.ptl_send_catchall_me_h,
any_proc,
match_bits,
ignore_bits,
PTL_UNLINK,
PTL_INS_BEFORE,
md,
PTL_UNLINK,
&(block->me_h),
&(block->md_h),
ompi_mtl_portals.ptl_empty_eq_h);
if (PTL_OK != ret) return OMPI_ERROR;
return OMPI_SUCCESS;
}
static inline void
ompi_mtl_portals_return_block_part(mca_mtl_portals_module_t *mtl,
ompi_mtl_portals_recv_short_block_t *block)
{
int ret;
OPAL_THREAD_ADD32(&(block->pending), -1);
if (block->full == true) {
if (block->pending == 0) {
ret = ompi_mtl_portals_activate_block(block);
if (OMPI_SUCCESS != ret) {
/* BWB - now what? */
}
}
}
}
#endif /* OMPI_MTL_PORTALS_RECV_SHORT_H */

37
ompi/mca/mtl/portals/mtl_portals_request.h
Просмотреть файл

@ -1,37 +0,0 @@
/*
* Copyright (c) 2004-2006 The Trustees of Indiana University and Indiana
* University Research and Technology
* Corporation. All rights reserved.
* Copyright (c) 2004-2005 The University of Tennessee and The University
* of Tennessee Research Foundation. All rights
* reserved.
* Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
* University of Stuttgart. All rights reserved.
* Copyright (c) 2004-2005 The Regents of the University of California.
* All rights reserved.
* $COPYRIGHT$
*
* Additional copyrights may follow
*
* $HEADER$
*/
#ifndef OMPI_MTL_PORTALS_REQUEST_H
#define OMPI_MTL_PORTALS_REQUEST_H
#include "opal/datatype/opal_convertor.h"
#include "ompi/mca/mtl/mtl.h"
struct ompi_mtl_portals_request_t {
struct mca_mtl_request_t super;
bool free_after;
struct opal_convertor_t *convertor;
volatile bool is_complete;
int event_count;
int (*event_callback)(ptl_event_t *ev, struct ompi_mtl_portals_request_t*);
};
typedef struct ompi_mtl_portals_request_t ompi_mtl_portals_request_t;
#endif

606
ompi/mca/mtl/portals/mtl_portals_send.c
Просмотреть файл

@ -1,606 +0,0 @@
/*
* Copyright (c) 2004-2006 The Trustees of Indiana University and Indiana
* University Research and Technology
* Corporation. All rights reserved.
* Copyright (c) 2004-2005 The University of Tennessee and The University
* of Tennessee Research Foundation. All rights
* reserved.
* Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
* University of Stuttgart. All rights reserved.
* Copyright (c) 2004-2005 The Regents of the University of California.
* All rights reserved.
* $COPYRIGHT$
*
* Additional copyrights may follow
*
* $HEADER$
*/
#include "ompi_config.h"
#include "ompi/communicator/communicator.h"
#include "opal/datatype/opal_convertor.h"
#include "ompi/mca/mtl/base/base.h"
#include "ompi/mca/mtl/base/mtl_base_datatype.h"
#include "mtl_portals.h"
#include "mtl_portals_request.h"
#include "mtl_portals_endpoint.h"
#include "mtl_portals_send_short.h"
/* send event callback functions */
/* called when no ack is necessary */
static int
ompi_mtl_portals_medium_callback(ptl_event_t *ev, ompi_mtl_portals_request_t *ptl_request)
{
switch (ev->type) {
case PTL_EVENT_SEND_END:
if (ptl_request->free_after) {
free(ev->md.start);
}
OPAL_OUTPUT_VERBOSE((50, ompi_mtl_base_output,
"send complete: 0x%016llx\n",
ev->match_bits));
ptl_request->is_complete = true;
if ( NULL != ptl_request->super.ompi_req ) {
ptl_request->super.ompi_req->req_status.MPI_ERROR = OMPI_SUCCESS;
ptl_request->super.completion_callback(&ptl_request->super);
}
break;
default:
opal_output(fileno(stderr)," Unexpected event type %d in ompi_mtl_portals_medium_callback()\n",ev->type);
/* abort(); */
}
return OMPI_SUCCESS;
}
/* called when send should wait for an ack or get */
static int
ompi_mtl_portals_long_callback(ptl_event_t *ev, struct ompi_mtl_portals_request_t* ptl_request)
{
switch (ev->type) {
case PTL_EVENT_SEND_END:
case PTL_EVENT_ACK:
case PTL_EVENT_GET_END:
/* we only receive an ack if the message was received into an
expected message. Otherwise, we don't get an ack, but mark
completion when the message was pulled (long message). */
if ( ++(ptl_request->event_count) == 2 ) {
if (ptl_request->free_after) {
free(ev->md.start);
}
OPAL_OUTPUT_VERBOSE((50, ompi_mtl_base_output,
"send complete: 0x%016llx\n",
ev->match_bits));
ptl_request->is_complete = true;
if ( NULL != ptl_request->super.ompi_req ) {
ptl_request->super.ompi_req->req_status.MPI_ERROR = OMPI_SUCCESS;
ptl_request->super.completion_callback(&ptl_request->super);
}
}
break;
default:
opal_output(fileno(stderr)," Unexpected event type %d in ompi_mtl_portals_long_callback()\n",ev->type);
abort();
}
return OMPI_SUCCESS;
}
/* called for a rendezvous long send */
static int
ompi_mtl_portals_long_rendezvous_callback(ptl_event_t *ev, struct ompi_mtl_portals_request_t* ptl_request)
{
switch (ev->type) {
case PTL_EVENT_GET_END:
if (ptl_request->free_after) {
free(ev->md.start);
}
OPAL_OUTPUT_VERBOSE((50, ompi_mtl_base_output,
"send complete: 0x%016llx\n",
ev->match_bits));
ptl_request->is_complete = true;
if ( NULL != ptl_request->super.ompi_req ) {
ptl_request->super.ompi_req->req_status.MPI_ERROR = OMPI_SUCCESS;
ptl_request->super.completion_callback(&ptl_request->super);
}
break;
default:
opal_output(fileno(stderr)," Unexpected event type %d in ompi_mtl_portals_long_callback()\n",ev->type);
abort();
}
return OMPI_SUCCESS;
}
/* called when sync send should wait for an ack or put */
static int
ompi_mtl_portals_sync_callback(ptl_event_t *ev, struct ompi_mtl_portals_request_t* ptl_request)
{
switch (ev->type) {
case PTL_EVENT_SEND_END:
case PTL_EVENT_ACK:
case PTL_EVENT_PUT_END:
/* we only receive an ack if the message was received into an
expected message. Otherwise, we don't get an ack, but mark
completion when a zero-length put arrrives. */
if ( ++(ptl_request->event_count) == 2 ) {
if (ptl_request->free_after) {
free(ev->md.start);
}
OPAL_OUTPUT_VERBOSE((50, ompi_mtl_base_output,
"send complete: 0x%016llx\n",
ev->match_bits));
ptl_request->is_complete = true;
if ( NULL != ptl_request->super.ompi_req ) {
ptl_request->super.ompi_req->req_status.MPI_ERROR = OMPI_SUCCESS;
ptl_request->super.completion_callback(&ptl_request->super);
}
}
break;
default:
opal_output(fileno(stderr)," Unexpected event type %d in ompi_mtl_portals_sync_callback()\n",ev->type);
abort();
}
return OMPI_SUCCESS;
}
/* internal send functions */
static int
ompi_mtl_portals_zero_send(mca_pml_base_send_mode_t mode, int contextid, int localrank,
int tag, ptl_process_id_t dest)
{
int ret;
ptl_match_bits_t match_bits;
ptl_match_bits_t mode_bits;
mode_bits = (MCA_PML_BASE_SEND_READY != mode) ? PTL_SHORT_MSG : PTL_READY_MSG;
PTL_SET_SEND_BITS(match_bits, contextid, localrank, tag, mode_bits);
ret = PtlPut(ompi_mtl_portals.ptl_zero_md_h,
PTL_NO_ACK_REQ,
dest,
OMPI_MTL_PORTALS_SEND_TABLE_ID,
0,
match_bits,
0,
0 );
if (PTL_OK != ret) {
return ompi_common_portals_error_ptl_to_ompi(ret);
}
return OMPI_SUCCESS;
}
static int
ompi_mtl_portals_short_send(mca_pml_base_send_mode_t mode, void *start, int length,
int contextid, int localrank, int tag, ptl_process_id_t dest)
{
int ret;
ptl_match_bits_t match_bits;
ptl_size_t offset;
void *copyblock;
ptl_match_bits_t mode_bits;
mode_bits = (MCA_PML_BASE_SEND_READY != mode) ? PTL_SHORT_MSG : PTL_READY_MSG;
PTL_SET_SEND_BITS(match_bits, contextid, localrank, tag, mode_bits);
offset = ompi_mtl_portals_alloc_short_buf() * ompi_mtl_portals.ptl_copy_block_len;
copyblock = (char *)ompi_mtl_portals.ptl_short_md.start + offset;
memcpy(copyblock, start, length);
ret = PtlPutRegion(ompi_mtl_portals.ptl_short_md_h,
offset,
length,
PTL_NO_ACK_REQ,
dest,
OMPI_MTL_PORTALS_SEND_TABLE_ID,
0,
match_bits,
0,
0);
if (PTL_OK != ret ) {
return ompi_common_portals_error_ptl_to_ompi(ret);
}
return OMPI_SUCCESS;
}
static int
ompi_mtl_portals_medium_isend( mca_pml_base_send_mode_t mode, void *start, int length,
int contextid, int localrank, int tag, ptl_process_id_t dest,
ompi_mtl_portals_request_t *ptl_request )
{
int ret;
ptl_match_bits_t match_bits;
ptl_md_t md;
ptl_handle_md_t md_h;
ptl_match_bits_t mode_bits;
mode_bits = (MCA_PML_BASE_SEND_READY != mode) ? PTL_SHORT_MSG : PTL_READY_MSG;
PTL_SET_SEND_BITS(match_bits, contextid, localrank, tag, mode_bits);
md.start = start;
md.length = length;
md.threshold = 1; /* send end */
md.options = PTL_MD_EVENT_START_DISABLE;
md.user_ptr = ptl_request;
md.eq_handle = ompi_mtl_portals.ptl_eq_h;
ret = PtlMDBind(ompi_mtl_portals.ptl_ni_h,
md,
PTL_UNLINK,
&md_h);
if (PTL_OK != ret) {
if (ptl_request->free_after) free(start);
return ompi_common_portals_error_ptl_to_ompi(ret);
}
ret = PtlPut(md_h,
PTL_NO_ACK_REQ,
dest,
OMPI_MTL_PORTALS_SEND_TABLE_ID,
0,
match_bits,
0,
0);
if (PTL_OK != ret) {
PtlMDUnlink(md_h);
if (ptl_request->free_after) free(start);
return ompi_common_portals_error_ptl_to_ompi(ret);
}
ptl_request->is_complete = false;
ptl_request->event_callback = ompi_mtl_portals_medium_callback;
ptl_request->event_count = 0;
return OMPI_SUCCESS;
}
static int
ompi_mtl_portals_long_isend( void *start, int length, int contextid, int localrank, int tag,
ptl_process_id_t dest, ompi_mtl_portals_request_t *ptl_request )
{
int ret;
ptl_match_bits_t match_bits;
ptl_md_t md;
ptl_handle_md_t md_h;
ptl_handle_me_t me_h;
PTL_SET_SEND_BITS(match_bits, contextid, localrank, tag, PTL_LONG_MSG);
md.start = start;
md.length = length;
if (ompi_mtl_portals.ptl_use_rendezvous == true) {
md.threshold = 1; /* get event */
} else {
md.threshold = 2; /* sent event, ack or get event */
}
md.options = PTL_MD_OP_GET | PTL_MD_EVENT_START_DISABLE;
md.user_ptr = ptl_request;
md.eq_handle = ompi_mtl_portals.ptl_eq_h;
ret = PtlMEMDPost(ompi_mtl_portals.ptl_ni_h,
ompi_mtl_portals.ptl_read_catchall_me_h,
dest,
(ptl_match_bits_t)(uintptr_t)ptl_request,
0,
PTL_UNLINK,
PTL_INS_BEFORE,
md,
PTL_UNLINK,
&me_h,
&md_h,
ompi_mtl_portals.ptl_empty_eq_h);
if (PTL_OK != ret) {
if (ptl_request->free_after) free(start);
return ompi_common_portals_error_ptl_to_ompi(ret);
}
if (ompi_mtl_portals.ptl_use_rendezvous == false) {
ret = PtlPut(md_h,
PTL_ACK_REQ,
dest,
OMPI_MTL_PORTALS_SEND_TABLE_ID,
0,
match_bits,
0,
(ptl_hdr_data_t)(uintptr_t)ptl_request);
ptl_request->event_callback = ompi_mtl_portals_long_callback;
} else {
/* just send a zero-length message */
ret = PtlPut(ompi_mtl_portals.ptl_zero_md_h,
PTL_NO_ACK_REQ,
dest,
OMPI_MTL_PORTALS_SEND_TABLE_ID,
0,
match_bits,
0,
(ptl_hdr_data_t)(uintptr_t)ptl_request);
ptl_request->event_callback = ompi_mtl_portals_long_rendezvous_callback;
}
if (PTL_OK != ret) {
PtlMEUnlink(me_h);
if (ptl_request->free_after) free(start);
return ompi_common_portals_error_ptl_to_ompi(ret);
}
ptl_request->is_complete = false;
ptl_request->event_count = 0;
return OMPI_SUCCESS;
}
static int
ompi_mtl_portals_sync_isend( void *start, int length, int contextid, int localrank, int tag,
ptl_process_id_t dest, ompi_mtl_portals_request_t *ptl_request )
{
int ret;
ptl_match_bits_t match_bits;
ptl_md_t md;
ptl_handle_md_t md_h;
ptl_handle_me_t me_h;
PTL_SET_SEND_BITS(match_bits, contextid, localrank, tag, PTL_SHORT_MSG);
md.start = start;
md.length = length;
md.threshold = 2; /* send, {ack, get} */
md.options = PTL_MD_OP_PUT | PTL_MD_EVENT_START_DISABLE;
md.user_ptr = ptl_request;
md.eq_handle = ompi_mtl_portals.ptl_eq_h;
ret = PtlMEMDPost(ompi_mtl_portals.ptl_ni_h,
ompi_mtl_portals.ptl_ack_catchall_me_h,
dest,
(ptl_match_bits_t)(uintptr_t)ptl_request,
0,
PTL_UNLINK,
PTL_INS_BEFORE,
md,
PTL_UNLINK,
&me_h,
&md_h,
ompi_mtl_portals.ptl_empty_eq_h);
if (PTL_OK != ret) {
if (ptl_request->free_after) free(start);
return ompi_common_portals_error_ptl_to_ompi(ret);
}
ret = PtlPut(md_h,
PTL_ACK_REQ,
dest,
OMPI_MTL_PORTALS_SEND_TABLE_ID,
0,
match_bits,
0,
(ptl_hdr_data_t)(uintptr_t)ptl_request);
if (PTL_OK != ret) {
PtlMEUnlink(me_h);
if (ptl_request->free_after) free(start);
return ompi_common_portals_error_ptl_to_ompi(ret);
}
ptl_request->is_complete = false;
ptl_request->event_callback = ompi_mtl_portals_sync_callback;
ptl_request->event_count = 0;
return OMPI_SUCCESS;
}
/* We use a macro for this since the send code is identical for blocking and non-blocking
sends, but we want both to be as fast as possible. We define it here since it doesn't
get used anywhere else. */
#define PTL_SEND_CODE \
{ \
switch (mode) { \
case MCA_PML_BASE_SEND_STANDARD: \
case MCA_PML_BASE_SEND_READY: \
case MCA_PML_BASE_SEND_BUFFERED: \
\
if (0 == length) { \
\
ret = ompi_mtl_portals_zero_send(mode, \
comm->c_contextid, \
comm->c_my_rank, \
tag, \
endpoint->ptl_proc); \
if (OMPI_SUCCESS != ret) return ret; \
\
ptl_request->is_complete = true; \
\
break; \
\
} else if (length <= ompi_mtl_portals.ptl_copy_block_len) { \
\
ret = ompi_mtl_portals_short_send(mode, \
start, \
length, \
comm->c_contextid, \
comm->c_my_rank, \
tag, \
endpoint->ptl_proc); \
if (OMPI_SUCCESS != ret) return ret; \
\
ptl_request->is_complete = true; \
\
break; \
\
} else if ((length <= ompi_mtl_portals.eager_limit) || \
(MCA_PML_BASE_SEND_READY == mode)) { \
\
ret = ompi_mtl_portals_medium_isend(mode, \
start, \
length, \
comm->c_contextid, \
comm->c_my_rank, \
tag, \
endpoint->ptl_proc, \
ptl_request ); \
if (OMPI_SUCCESS != ret) return ret; \
\
break; \
\
} \
\
/* long standard send case falls through */ \
\
case MCA_PML_BASE_SEND_SYNCHRONOUS: \
\
if (length <= ompi_mtl_portals.eager_limit) { \
\
ret = ompi_mtl_portals_sync_isend(start, \
length, \
comm->c_contextid, \
comm->c_my_rank, \
tag, \
endpoint->ptl_proc, \
ptl_request ); \
if (OMPI_SUCCESS != ret) return ret; \
\
} else { \
/* if we got this far, we're either a standard or synchronous long send */ \
ret = ompi_mtl_portals_long_isend(start, \
length, \
comm->c_contextid, \
comm->c_my_rank, \
tag, \
endpoint->ptl_proc, \
ptl_request ); \
if (OMPI_SUCCESS != ret) return ret; \
\
} \
\
break; \
\
default: \
opal_output(fileno(stderr),"Unexpected msg type %d\n", mode); \
abort(); \
\
} \
}
/* external send functions */
int
ompi_mtl_portals_send(struct mca_mtl_base_module_t* mtl,
struct ompi_communicator_t* comm,
int dest,
int tag,
struct opal_convertor_t *convertor,
mca_pml_base_send_mode_t mode)
{
int ret;
ompi_proc_t *ompi_proc = ompi_comm_peer_lookup( comm, dest );
mca_mtl_base_endpoint_t *endpoint = (mca_mtl_base_endpoint_t*) ompi_proc->proc_pml;
void *start;
size_t length;
ompi_mtl_portals_request_t mtl_request;
ompi_mtl_portals_request_t *ptl_request = (ompi_mtl_portals_request_t*)&mtl_request;
assert(mtl == &ompi_mtl_portals.base);
ret = ompi_mtl_datatype_pack(convertor, &start, &length,
&(ptl_request->free_after));
if (OMPI_SUCCESS != ret) return ret;
ptl_request->is_complete = false;
ptl_request->super.ompi_req = NULL;
PTL_SEND_CODE;
/* wait for send to complete */
while (ptl_request->is_complete == false) {
ompi_mtl_portals_progress();
}
return OMPI_SUCCESS;
}
int
ompi_mtl_portals_isend(struct mca_mtl_base_module_t* mtl,
struct ompi_communicator_t* comm,
int dest,
int tag,
struct opal_convertor_t *convertor,
mca_pml_base_send_mode_t mode,
bool blocking,
mca_mtl_request_t *mtl_request)
{
int ret;
ompi_proc_t *ompi_proc = ompi_comm_peer_lookup( comm, dest );
mca_mtl_base_endpoint_t *endpoint = (mca_mtl_base_endpoint_t*)ompi_proc->proc_pml;
ompi_mtl_portals_request_t *ptl_request = (ompi_mtl_portals_request_t*)mtl_request;
void *start;
size_t length;
assert(mtl == &ompi_mtl_portals.base);
ret = ompi_mtl_datatype_pack(convertor, &start, &length,
&(ptl_request->free_after));
if (OMPI_SUCCESS != ret) return ret;
ptl_request->is_complete = false;
PTL_SEND_CODE;
if (ptl_request->is_complete == true ) {
ptl_request->super.ompi_req->req_status.MPI_ERROR = OMPI_SUCCESS;
ptl_request->super.completion_callback(&ptl_request->super);
}
return OMPI_SUCCESS;
}

115
ompi/mca/mtl/portals/mtl_portals_send_short.c
Просмотреть файл

@ -1,115 +0,0 @@
/*
* Copyright (c) 2004-2006 The Trustees of Indiana University and Indiana
* University Research and Technology
* Corporation. All rights reserved.
* Copyright (c) 2004-2005 The University of Tennessee and The University
* of Tennessee Research Foundation. All rights
* reserved.
* Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
* University of Stuttgart. All rights reserved.
* Copyright (c) 2004-2005 The Regents of the University of California.
* All rights reserved.
* $COPYRIGHT$
*
* Additional copyrights may follow
*
* $HEADER$
*/
#include "ompi_config.h"
#include "opal/util/output.h"
#include "mtl_portals.h"
#include "mtl_portals_request.h"
#include "mtl_portals_send_short.h"
static ompi_mtl_portals_request_t ptl_short_request;
/* short send callback */
static int
ompi_mtl_portals_short_callback(ptl_event_t *ev, ompi_mtl_portals_request_t *ptl_request)
{
switch (ev->type) {
case PTL_EVENT_SEND_END:
ompi_mtl_portals_free_short_buf(ev->offset);
break;
default:
opal_output(fileno(stderr)," Unexpected event type %d in ompi_mtl_portals_short_callback()\n",ev->type);
abort();
}
return OMPI_SUCCESS;
}
/* initialize short copy blocks */
void
ompi_mtl_portals_short_setup()
{
int ret;
int i;
if ((ompi_mtl_portals.ptl_num_copy_blocks > 0) && (ompi_mtl_portals.ptl_copy_block_len > 0)) {
ompi_mtl_portals.ptl_short_md.length = ompi_mtl_portals.ptl_num_copy_blocks *
ompi_mtl_portals.ptl_copy_block_len;
ompi_mtl_portals.ptl_short_md.start = malloc(ompi_mtl_portals.ptl_short_md.length);
if (NULL == ompi_mtl_portals.ptl_short_md.start ) {
ompi_mtl_portals.ptl_num_copy_blocks = 0;
return;
}
ompi_mtl_portals.ptl_short_md.threshold = PTL_MD_THRESH_INF;
ompi_mtl_portals.ptl_short_md.max_size = 0;
ompi_mtl_portals.ptl_short_md.options = PTL_MD_EVENT_START_DISABLE;
ompi_mtl_portals.ptl_short_md.user_ptr = &ptl_short_request;
ompi_mtl_portals.ptl_short_md.eq_handle = ompi_mtl_portals.ptl_eq_h;
ret = PtlMDBind(ompi_mtl_portals.ptl_ni_h,
ompi_mtl_portals.ptl_short_md,
PTL_RETAIN,
&ompi_mtl_portals.ptl_short_md_h);
if (PTL_OK != ret) {
free(ompi_mtl_portals.ptl_short_md.start);
ompi_mtl_portals.ptl_num_copy_blocks = 0;
return;
}
ptl_short_request.event_callback = ompi_mtl_portals_short_callback;
ompi_mtl_portals.ptl_copy_block_free_list = malloc(ompi_mtl_portals.ptl_num_copy_blocks * sizeof(int));
if (NULL == ompi_mtl_portals.ptl_copy_block_free_list) {
free(ompi_mtl_portals.ptl_short_md.start);
ompi_mtl_portals.ptl_num_copy_blocks = 0;
return;
}
for (i=0; i<ompi_mtl_portals.ptl_num_copy_blocks; i++) {
ompi_mtl_portals.ptl_copy_block_free_list[i] = i;
}
ompi_mtl_portals.ptl_copy_block_first_free = 0;
}
}
/* free short resources */
void
ompi_mtl_portals_short_cleanup()
{
if (ompi_mtl_portals.ptl_num_copy_blocks > 0) {
free(ompi_mtl_portals.ptl_short_md.start);
free(ompi_mtl_portals.ptl_copy_block_free_list);
ompi_mtl_portals.ptl_num_copy_blocks = 0;
}
}

59
ompi/mca/mtl/portals/mtl_portals_send_short.h
Просмотреть файл

@ -1,59 +0,0 @@
/*
* Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
* University Research and Technology
* Corporation. All rights reserved.
* Copyright (c) 2004-2006 The University of Tennessee and The University
* of Tennessee Research Foundation. All rights
* reserved.
* Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
* University of Stuttgart. All rights reserved.
* Copyright (c) 2004-2005 The Regents of the University of California.
* All rights reserved.
* $COPYRIGHT$
*
* Additional copyrights may follow
*
* $HEADER$
*/
#ifndef OMPI_MTL_PORTALS_SEND_SHORT_H
#define OMPI_MTL_PORTALS_SEND_SHORT_H
extern void ompi_mtl_portals_short_setup(void);
extern void ompi_mtl_portals_short_cleanup(void);
static inline int
ompi_mtl_portals_alloc_short_buf(void)
{
int buf_num;
while ( ompi_mtl_portals.ptl_copy_block_first_free == ompi_mtl_portals.ptl_num_copy_blocks ) {
ompi_mtl_portals_progress();
}
buf_num = ompi_mtl_portals.ptl_copy_block_free_list[ompi_mtl_portals.ptl_copy_block_first_free++];
assert((buf_num >= 0) && (buf_num < ompi_mtl_portals.ptl_num_copy_blocks));
return buf_num;
}
static inline void
ompi_mtl_portals_free_short_buf( int offset )
{
int buf_num;
buf_num = offset / ompi_mtl_portals.ptl_copy_block_len;
assert((buf_num >= 0) && (buf_num < ompi_mtl_portals.ptl_num_copy_blocks));
ompi_mtl_portals.ptl_copy_block_first_free--;
assert(ompi_mtl_portals.ptl_copy_block_first_free >= 0);
ompi_mtl_portals.ptl_copy_block_free_list[ompi_mtl_portals.ptl_copy_block_first_free] = buf_num;
}
#endif /* OMPI_MTL_PORTALS_SEND_SHORT_H */