ports/openmpi
ports/openmpi
1
1
Форкнуть 0
Код Релизы Активность

Update common_portals to allow using portals interconnect with a modex rather

Просмотр исходного кода 
than relying on cnos to get the nid/pid map. 

This commit was SVN r17588.
Этот коммит содержится в:
Galen Shipman 2008-02-25 19:17:21 +00:00
родитель b4bbb70bb7
Коммит 44003a41f2
12 изменённых файлов: 165 добавлений и 131 удалений
Показать статистику Скачать Patch файл Скачать Diff файл Показать все файлы Свернуть все файлы

2
LICENSE
Просмотреть файл

@ -23,6 +23,8 @@ Copyright (c) 2006-2007 Sun Microsystems, Inc. All rights reserved.
Use is subject to license terms.
Copyright (c) 2006-2007 The University of Houston. All rights reserved.
Copyright (c) 2006 Myricom, Inc. All rights reserved.
Copyright (c) 2007-2008 UT-Battelle, LLC. All rights reserved.
$COPYRIGHT$
Additional copyrights may follow

16
config/ompi_check_portals.m4
Просмотреть файл

@ -43,7 +43,7 @@ AC_DEFUN([OMPI_CHECK_PORTALS],[
AC_ARG_WITH([portals-config],
AC_HELP_STRING([--with-portals-config],
[configuration to use for Portals support.
One of "utcp", "xt3". (default: utcp)]))
One of "utcp", "xt3", "xt3-modex". (default: utcp)]))
AC_ARG_WITH([portals-libs],
[AC_HELP_STRING([--with-portals-libs=LIBS],
[Libraries to link with for portals])])
@ -53,9 +53,12 @@ AC_DEFUN([OMPI_CHECK_PORTALS],[
with_portals_config="utcp"
elif test "$with_portals_config" = "redstorm" ; then
with_portals_config="xt3"
elif test "$with_portals_config" = "cnl_modex" ; then
with_portals_config="xt3-modex"
fi
OMPI_PORTALS_UTCP=0
OMPI_PORTALS_CRAYXT3=0
OMPI_PORTALS_CRAYXT3_MODEX=0
case "$with_portals_config" in
"utcp")
OMPI_PORTALS_UTCP=1
@ -71,6 +74,13 @@ AC_DEFUN([OMPI_CHECK_PORTALS],[
check_portals_header_prefix="portals/"
AC_MSG_RESULT([Cray XT3])
;;
"xt3-modex")
OMPI_PORTALS_CRAYXT3_MODEX=1
OMPI_PORTALS_HAVE_EVENT_UNLINK=0
check_portals_LIBS=
check_portals_header_prefix="portals/"
AC_MSG_RESULT([Cray XT3 - Modex])
;;
*)
# ok to call ERROR here - the user specified something invalid.
# that should be brought to his attention
@ -84,9 +94,13 @@ AC_DEFUN([OMPI_CHECK_PORTALS],[
AC_DEFINE_UNQUOTED([OMPI_PORTALS_UTCP], [$OMPI_PORTALS_UTCP],
[Use the UTCP reference implementation of Portals])
AC_DEFINE_UNQUOTED([OMPI_PORTALS_CRAYXT3], [$OMPI_PORTALS_CRAYXT3],
[Use the Cray XT-3 implementation of Portals])
AC_DEFINE_UNQUOTED([OMPI_PORTALS_CRAYXT3_MODEX], [$OMPI_PORTALS_CRAYXT3_MODEX],
[Use the Cray XT-3 implementation of Portals using Modex])
# Add correct -I and -L flags
AS_IF([test -n "$with_portals"],
[AS_IF([test -d "$with_portals/include"],

2
contrib/platform/cray_xt_cnl_romio
Просмотреть файл

@ -4,7 +4,7 @@ enable_shared=no
with_threads=no
enable_pretty_print_stacktrace=no
enable_dlopen=no
with_portals_config=redstorm
with_portals_config=cnl_modex
with_memory_manager=none
enable_mca_no_build=maffinity-first_use,maffinity-libnuma,paffinity-linux,pls-rsh,pml-dr,filem-rsh,grpcomm-cnos,pls-cnos,rmgr-cnos,rml-cnos,routed-cnos,sds-portals_utcp,sds-cnos
enable_heterogeneous=no

4
ompi/mca/btl/portals/btl_portals.c
Просмотреть файл

@ -526,14 +526,14 @@ mca_btl_portals_finalize(struct mca_btl_base_module_t *btl_base)
for (i = 0 ; i < OMPI_BTL_PORTALS_EQ_SIZE ; ++i) {
int ptl_ret = PtlEQFree(mca_btl_portals_module.portals_eq_handles[i]);
if (PTL_OK != ptl_ret) {
#if OMPI_PORTALS_CRAYXT3
#if (OMPI_PORTALS_CRAYXT3 || OMPI_PORTALS_CRAYXT3_MODEX)
if (i != OMPI_BTL_PORTALS_EQ_SEND && PTL_EQ_IN_USE != ptl_ret) {
/* The PML isn't great about cleaning up after itself.
Ignore related errors. */
#endif
opal_output(mca_btl_portals_component.portals_output,
"Error freeing EQ %d: %d", i, ptl_ret);
#if OMPI_PORTALS_CRAYXT3
#if (OMPI_PORTALS_CRAYXT3 || OMPI_PORTALS_CRAYXT3_MODEX)
}
#endif
}

3
ompi/mca/btl/portals/btl_portals.h
Просмотреть файл

@ -61,6 +61,9 @@ struct mca_btl_portals_component_t {
/* number of eager fragments */
int portals_free_list_eager_max_num;
/* shall I use portals to send to thyself? */
int portals_support_self;
};
typedef struct mca_btl_portals_component_t mca_btl_portals_component_t;

11
ompi/mca/btl/portals/btl_portals_component.c
Просмотреть файл

@ -125,6 +125,14 @@ mca_btl_portals_component_open(void)
32,
&(mca_btl_portals_component.portals_free_list_eager_max_num));
mca_base_param_reg_int(&mca_btl_portals_component.super.btl_version,
"support_self",
"Use portals for send to self",
false,
false,
1, /* default to true.. */
&(mca_btl_portals_component.portals_support_self));
/*
* fill default module state
*/
@ -222,6 +230,7 @@ mca_btl_portals_component_init(int *num_btls,
bool enable_mpi_threads)
{
mca_btl_base_module_t ** btls = malloc(sizeof(mca_btl_base_module_t*));
bool accel;
btls[0] = (mca_btl_base_module_t*) &mca_btl_portals_module;
if (enable_progress_threads || enable_mpi_threads) {
@ -232,7 +241,7 @@ mca_btl_portals_component_init(int *num_btls,
/* initialize portals btl. note that this is in the compat code because
it's fairly non-portable between implementations */
if (OMPI_SUCCESS != ompi_common_portals_initialize()) {
if (OMPI_SUCCESS != ompi_common_portals_initialize(&mca_btl_portals_module.portals_ni_h, &accel)) {
opal_output_verbose(20, mca_btl_portals_component.portals_output,
"disabled because compatibility init failed");
return NULL;

5
ompi/mca/common/portals/common_portals.c
Просмотреть файл

@ -29,6 +29,9 @@
#include "common_portals_crayxt3.c"
#elif OMPI_PORTALS_CRAYXT3_MODEX
#include "common_portals_cray_xt_modex.c"
#else
#error "Unknown Portals library configuration"
@ -107,7 +110,7 @@ ompi_common_portals_error_ptl_to_ompi(int ptl_error)
case PTL_SR_INDEX_INVALID:
ret = OMPI_ERR_BAD_PARAM;
break;
#if !OMPI_PORTALS_CRAYXT3
#if !(OMPI_PORTALS_CRAYXT3 || OMPI_PORTALS_CRAYXT3_MODEX)
case PTL_UNKNOWN_ERROR:
ret = OMPI_ERROR;
break;

12
ompi/mca/common/portals/common_portals.h
Просмотреть файл

@ -34,8 +34,7 @@
#define OMPI_MTL_PORTALS_READ_TABLE_ID 3
#define OMPI_MTL_PORTALS_ACK_TABLE_ID 4
#elif OMPI_PORTALS_CRAYXT3
#elif (OMPI_PORTALS_CRAYXT3 || OMPI_PORTALS_CRAYXT3_MODEX)
#include <portals/portals3.h>
#define PTL_EQ_HANDLER_NONE NULL
/* Cray's definition, differs from the spec */
@ -91,11 +90,13 @@ int ompi_common_portals_register_mca(void);
* may not be possible if use of the modex is required to setup the
* network (as is the case with the utcp reference implementation).
*
* @param ni_handle (OUT) network interface handle
* @param bool (OUT) true if using accelerated Portals, false otherwise
*
* @retval OMPI_SUCCESS Portals successfully initialized
* @retval OMPI_ERR_NOT_AVAILABLE Portals could not be initialized
*/
int ompi_common_portals_initialize(void);
int ompi_common_portals_initialize(ptl_handle_ni_t *ni_handle, bool *accel);
/**
* Initialize network interface
@ -112,8 +113,7 @@ int ompi_common_portals_initialize(void);
* @retval OMPI_SUCCESS Portals network interface successfully initialized
* @retval OMPI_ERROR Something bad happened
*/
int ompi_common_portals_ni_initialize(ptl_handle_ni_t *ni_handle,
bool *accel);
int ompi_common_portals_ni_initialize(ptl_handle_ni_t *ni_handle, bool *accel);
/**

2
ompi/mca/common/portals/common_portals_crayxt3.c
Просмотреть файл

@ -42,7 +42,7 @@ ompi_common_portals_register_mca(void)
int
ompi_common_portals_initialize(void)
ompi_common_portals_initialize(ptl_handle_ni_t *ni_handle, bool *accel)
{
int ret, max_interfaces;

2
ompi/mca/common/portals/common_portals_utcp.c
Просмотреть файл

@ -80,7 +80,7 @@ ompi_common_portals_register_mca(void)
int
ompi_common_portals_initialize(void)
ompi_common_portals_initialize(ptl_handle_ni_t *ni_handle, bool *accel)
{
int ret;
ptl_process_id_t info;

262
ompi/mca/mtl/portals/mtl_portals.c
Просмотреть файл

@ -86,6 +86,9 @@ ompi_mtl_portals_add_procs(struct mca_mtl_base_module_t *mtl,
ptl_process_id_t *portals_procs = NULL;
ptl_md_t md;
size_t i;
ptl_match_bits_t match_bits;
ptl_match_bits_t ignore_bits;
ptl_process_id_t anyid = { PTL_NID_ANY, PTL_PID_ANY };
bool accel;
assert(mtl == &ompi_mtl_portals.base);
@ -95,142 +98,141 @@ ompi_mtl_portals_add_procs(struct mca_mtl_base_module_t *mtl,
environment setup the common code does for the utcp
implementation, we can't do it until modex information can be
received. */
if (PTL_INVALID_HANDLE == ompi_mtl_portals.ptl_ni_h) {
ptl_match_bits_t match_bits;
ptl_match_bits_t ignore_bits;
ptl_process_id_t anyid = { PTL_NID_ANY, PTL_PID_ANY };
ret = ompi_common_portals_ni_initialize(&(ompi_mtl_portals.ptl_ni_h), &accel);
if (OMPI_SUCCESS != ret) goto cleanup;
/* event queue for expected events */
ret = PtlEQAlloc(ompi_mtl_portals.ptl_ni_h,
ompi_mtl_portals.ptl_expected_queue_size,
PTL_EQ_HANDLER_NONE,
&(ompi_mtl_portals.ptl_eq_h));
assert(ret == PTL_OK);
/* event queue for unexpected receives */
ret = PtlEQAlloc(ompi_mtl_portals.ptl_ni_h,
ompi_mtl_portals.ptl_unexpected_queue_size,
PTL_EQ_HANDLER_NONE,
&(ompi_mtl_portals.ptl_unex_eq_h));
assert(ret == PTL_OK);
/* empty event queue for PtlMEMDPost() */
ret = PtlEQAlloc(ompi_mtl_portals.ptl_ni_h,
1,
PTL_EQ_HANDLER_NONE,
&(ompi_mtl_portals.ptl_empty_eq_h));
assert(ret == PTL_OK);
/* attach the long unex msg buffer */
match_bits = PTL_LONG_MSG;
ignore_bits = ~(PTL_LONG_MSG);
ret = PtlMEAttach(ompi_mtl_portals.ptl_ni_h,
OMPI_MTL_PORTALS_SEND_TABLE_ID,
anyid,
match_bits,
ignore_bits,
PTL_RETAIN,
PTL_INS_AFTER,
&(ompi_mtl_portals.ptl_unex_long_me_h));
assert(ret == PTL_OK);
md.start = NULL;
md.length = 0;
md.threshold = PTL_MD_THRESH_INF;
md.max_size = 0;
md.options = PTL_MD_OP_PUT | PTL_MD_TRUNCATE | PTL_MD_ACK_DISABLE;
md.eq_handle = ompi_mtl_portals.ptl_unex_eq_h;
md.user_ptr = NULL;
ret = PtlMDAttach(ompi_mtl_portals.ptl_unex_long_me_h,
md,
PTL_RETAIN,
&unex_long_md_h);
assert(ret == PTL_OK);
/* attach catchalls to the send, ack, and read portals */
catchall_request.event_callback = ompi_mtl_portals_catchall_callback;
md.eq_handle = ompi_mtl_portals.ptl_eq_h;
md.user_ptr = &catchall_request;
/* catchall for the send portal */
ret = PtlMEMDPost(ompi_mtl_portals.ptl_ni_h,
ompi_mtl_portals.ptl_unex_long_me_h,
anyid,
0,
~0,
PTL_RETAIN,
PTL_INS_AFTER,
md,
PTL_UNLINK,
&(ompi_mtl_portals.ptl_send_catchall_me_h),
&send_catchall_md_h,
ompi_mtl_portals.ptl_empty_eq_h);
assert(ret == PTL_OK);
/* catchall for ack portal */
ret = PtlMEAttach(ompi_mtl_portals.ptl_ni_h,
OMPI_MTL_PORTALS_ACK_TABLE_ID,
anyid,
0,
~0,
PTL_RETAIN,
PTL_INS_AFTER,
&(ompi_mtl_portals.ptl_ack_catchall_me_h));
assert(ret == PTL_OK);
ret = PtlMDAttach(ompi_mtl_portals.ptl_ack_catchall_me_h,
md,
PTL_UNLINK,
&ack_catchall_md_h);
assert(ret == PTL_OK);
/* catchall for read portal */
ret = PtlMEAttach(ompi_mtl_portals.ptl_ni_h,
OMPI_MTL_PORTALS_READ_TABLE_ID,
anyid,
0,
~0,
PTL_RETAIN,
PTL_INS_AFTER,
&(ompi_mtl_portals.ptl_read_catchall_me_h));
assert(ret == PTL_OK);
ret = PtlMDAttach(ompi_mtl_portals.ptl_read_catchall_me_h,
md,
PTL_RETAIN,
&read_catchall_md_h);
assert(ret == PTL_OK);
/* attach short unex recv blocks */
ret = ompi_mtl_portals_recv_short_enable((mca_mtl_portals_module_t*) mtl);
opal_progress_register(ompi_mtl_portals_progress);
/* bind zero-length md for sending zero-length msgs and acks */
md.start = NULL;
md.length = 0;
md.threshold = PTL_MD_THRESH_INF;
md.max_size = 0;
md.options = PTL_MD_EVENT_START_DISABLE | PTL_MD_EVENT_END_DISABLE;
md.user_ptr = NULL;
md.eq_handle = PTL_EQ_NONE;
ret = PtlMDBind(ompi_mtl_portals.ptl_ni_h,
md,
PTL_RETAIN,
&ompi_mtl_portals.ptl_zero_md_h );
assert(ret == PTL_OK);
/* set up the short copy blocks */
ompi_mtl_portals_short_setup();
}
/* event queue for expected events */
ret = PtlEQAlloc(ompi_mtl_portals.ptl_ni_h,
ompi_mtl_portals.ptl_expected_queue_size,
PTL_EQ_HANDLER_NONE,
&(ompi_mtl_portals.ptl_eq_h));
assert(ret == PTL_OK);
/* event queue for unexpected receives */
ret = PtlEQAlloc(ompi_mtl_portals.ptl_ni_h,
ompi_mtl_portals.ptl_unexpected_queue_size,
PTL_EQ_HANDLER_NONE,
&(ompi_mtl_portals.ptl_unex_eq_h));
assert(ret == PTL_OK);
/* empty event queue for PtlMEMDPost() */
ret = PtlEQAlloc(ompi_mtl_portals.ptl_ni_h,
1,
PTL_EQ_HANDLER_NONE,
&(ompi_mtl_portals.ptl_empty_eq_h));
assert(ret == PTL_OK);
/* attach the long unex msg buffer */
match_bits = PTL_LONG_MSG;
ignore_bits = ~(PTL_LONG_MSG);
ret = PtlMEAttach(ompi_mtl_portals.ptl_ni_h,
OMPI_MTL_PORTALS_SEND_TABLE_ID,
anyid,
match_bits,
ignore_bits,
PTL_RETAIN,
PTL_INS_AFTER,
&(ompi_mtl_portals.ptl_unex_long_me_h));
assert(ret == PTL_OK);
md.start = NULL;
md.length = 0;
md.threshold = PTL_MD_THRESH_INF;
md.max_size = 0;
md.options = PTL_MD_OP_PUT | PTL_MD_TRUNCATE | PTL_MD_ACK_DISABLE;
md.eq_handle = ompi_mtl_portals.ptl_unex_eq_h;
md.user_ptr = NULL;
ret = PtlMDAttach(ompi_mtl_portals.ptl_unex_long_me_h,
md,
PTL_RETAIN,
&unex_long_md_h);
assert(ret == PTL_OK);
/* attach catchalls to the send, ack, and read portals */
catchall_request.event_callback = ompi_mtl_portals_catchall_callback;
md.eq_handle = ompi_mtl_portals.ptl_eq_h;
md.user_ptr = &catchall_request;
/* catchall for the send portal */
ret = PtlMEMDPost(ompi_mtl_portals.ptl_ni_h,
ompi_mtl_portals.ptl_unex_long_me_h,
anyid,
0,
~0,
PTL_RETAIN,
PTL_INS_AFTER,
md,
PTL_UNLINK,
&(ompi_mtl_portals.ptl_send_catchall_me_h),
&send_catchall_md_h,
ompi_mtl_portals.ptl_empty_eq_h);
assert(ret == PTL_OK);
/* catchall for ack portal */
ret = PtlMEAttach(ompi_mtl_portals.ptl_ni_h,
OMPI_MTL_PORTALS_ACK_TABLE_ID,
anyid,
0,
~0,
PTL_RETAIN,
PTL_INS_AFTER,
&(ompi_mtl_portals.ptl_ack_catchall_me_h));
assert(ret == PTL_OK);
ret = PtlMDAttach(ompi_mtl_portals.ptl_ack_catchall_me_h,
md,
PTL_UNLINK,
&ack_catchall_md_h);
assert(ret == PTL_OK);
/* catchall for read portal */
ret = PtlMEAttach(ompi_mtl_portals.ptl_ni_h,
OMPI_MTL_PORTALS_READ_TABLE_ID,
anyid,
0,
~0,
PTL_RETAIN,
PTL_INS_AFTER,
&(ompi_mtl_portals.ptl_read_catchall_me_h));
assert(ret == PTL_OK);
ret = PtlMDAttach(ompi_mtl_portals.ptl_read_catchall_me_h,
md,
PTL_RETAIN,
&read_catchall_md_h);
assert(ret == PTL_OK);
/* attach short unex recv blocks */
ret = ompi_mtl_portals_recv_short_enable((mca_mtl_portals_module_t*) mtl);
opal_progress_register(ompi_mtl_portals_progress);
/* bind zero-length md for sending zero-length msgs and acks */
md.start = NULL;
md.length = 0;
md.threshold = PTL_MD_THRESH_INF;
md.max_size = 0;
md.options = PTL_MD_EVENT_START_DISABLE | PTL_MD_EVENT_END_DISABLE;
md.user_ptr = NULL;
md.eq_handle = PTL_EQ_NONE;
ret = PtlMDBind(ompi_mtl_portals.ptl_ni_h,
md,
PTL_RETAIN,
&ompi_mtl_portals.ptl_zero_md_h );
assert(ret == PTL_OK);
/* set up the short copy blocks */
ompi_mtl_portals_short_setup();
/* get the list of ptl_process_id_t structures for the given proc
structures. If the Portals runtime environment supports
comm_spawn, we'll be able to support it as well. */

3
ompi/mca/mtl/portals/mtl_portals_component.c
Просмотреть файл

@ -166,11 +166,12 @@ static mca_mtl_base_module_t*
ompi_mtl_portals_component_init(bool enable_progress_threads,
bool enable_mpi_threads)
{
bool accel;
/* we don't run with no stinkin' threads */
if (enable_progress_threads || enable_mpi_threads) return NULL;
/* initialize our interface */
if (OMPI_SUCCESS != ompi_common_portals_initialize()) {
if (OMPI_SUCCESS != ompi_common_portals_initialize(&(ompi_mtl_portals.ptl_ni_h), &accel)) {
return NULL;
}