diff --git a/LICENSE b/LICENSE index 022a2d461d..ec2bd2fb72 100644 --- a/LICENSE +++ b/LICENSE @@ -23,6 +23,8 @@ Copyright (c) 2006-2007 Sun Microsystems, Inc. All rights reserved. Use is subject to license terms. Copyright (c) 2006-2007 The University of Houston. All rights reserved. Copyright (c) 2006 Myricom, Inc. All rights reserved. +Copyright (c) 2007-2008 UT-Battelle, LLC. All rights reserved. + $COPYRIGHT$ Additional copyrights may follow diff --git a/config/ompi_check_portals.m4 b/config/ompi_check_portals.m4 index caa5aefa5f..f0e1cf95e0 100644 --- a/config/ompi_check_portals.m4 +++ b/config/ompi_check_portals.m4 @@ -43,7 +43,7 @@ AC_DEFUN([OMPI_CHECK_PORTALS],[ AC_ARG_WITH([portals-config], AC_HELP_STRING([--with-portals-config], [configuration to use for Portals support. - One of "utcp", "xt3". (default: utcp)])) + One of "utcp", "xt3", "xt3-modex". (default: utcp)])) AC_ARG_WITH([portals-libs], [AC_HELP_STRING([--with-portals-libs=LIBS], [Libraries to link with for portals])]) @@ -53,9 +53,12 @@ AC_DEFUN([OMPI_CHECK_PORTALS],[ with_portals_config="utcp" elif test "$with_portals_config" = "redstorm" ; then with_portals_config="xt3" + elif test "$with_portals_config" = "cnl_modex" ; then + with_portals_config="xt3-modex" fi OMPI_PORTALS_UTCP=0 OMPI_PORTALS_CRAYXT3=0 + OMPI_PORTALS_CRAYXT3_MODEX=0 case "$with_portals_config" in "utcp") OMPI_PORTALS_UTCP=1 @@ -71,6 +74,13 @@ AC_DEFUN([OMPI_CHECK_PORTALS],[ check_portals_header_prefix="portals/" AC_MSG_RESULT([Cray XT3]) ;; + "xt3-modex") + OMPI_PORTALS_CRAYXT3_MODEX=1 + OMPI_PORTALS_HAVE_EVENT_UNLINK=0 + check_portals_LIBS= + check_portals_header_prefix="portals/" + AC_MSG_RESULT([Cray XT3 - Modex]) + ;; *) # ok to call ERROR here - the user specified something invalid. # that should be brought to his attention @@ -84,8 +94,12 @@ AC_DEFUN([OMPI_CHECK_PORTALS],[ AC_DEFINE_UNQUOTED([OMPI_PORTALS_UTCP], [$OMPI_PORTALS_UTCP], [Use the UTCP reference implementation of Portals]) + AC_DEFINE_UNQUOTED([OMPI_PORTALS_CRAYXT3], [$OMPI_PORTALS_CRAYXT3], [Use the Cray XT-3 implementation of Portals]) + + AC_DEFINE_UNQUOTED([OMPI_PORTALS_CRAYXT3_MODEX], [$OMPI_PORTALS_CRAYXT3_MODEX], + [Use the Cray XT-3 implementation of Portals using Modex]) # Add correct -I and -L flags AS_IF([test -n "$with_portals"], diff --git a/contrib/platform/cray_xt_cnl_romio b/contrib/platform/cray_xt_cnl_romio index 07365001ac..1cb85bacd6 100755 --- a/contrib/platform/cray_xt_cnl_romio +++ b/contrib/platform/cray_xt_cnl_romio @@ -4,7 +4,7 @@ enable_shared=no with_threads=no enable_pretty_print_stacktrace=no enable_dlopen=no -with_portals_config=redstorm +with_portals_config=cnl_modex with_memory_manager=none enable_mca_no_build=maffinity-first_use,maffinity-libnuma,paffinity-linux,pls-rsh,pml-dr,filem-rsh,grpcomm-cnos,pls-cnos,rmgr-cnos,rml-cnos,routed-cnos,sds-portals_utcp,sds-cnos enable_heterogeneous=no diff --git a/ompi/mca/btl/portals/btl_portals.c b/ompi/mca/btl/portals/btl_portals.c index 9f6d2ff431..ab56f18a8a 100644 --- a/ompi/mca/btl/portals/btl_portals.c +++ b/ompi/mca/btl/portals/btl_portals.c @@ -526,14 +526,14 @@ mca_btl_portals_finalize(struct mca_btl_base_module_t *btl_base) for (i = 0 ; i < OMPI_BTL_PORTALS_EQ_SIZE ; ++i) { int ptl_ret = PtlEQFree(mca_btl_portals_module.portals_eq_handles[i]); if (PTL_OK != ptl_ret) { -#if OMPI_PORTALS_CRAYXT3 +#if (OMPI_PORTALS_CRAYXT3 || OMPI_PORTALS_CRAYXT3_MODEX) if (i != OMPI_BTL_PORTALS_EQ_SEND && PTL_EQ_IN_USE != ptl_ret) { /* The PML isn't great about cleaning up after itself. Ignore related errors. */ #endif opal_output(mca_btl_portals_component.portals_output, "Error freeing EQ %d: %d", i, ptl_ret); -#if OMPI_PORTALS_CRAYXT3 +#if (OMPI_PORTALS_CRAYXT3 || OMPI_PORTALS_CRAYXT3_MODEX) } #endif } diff --git a/ompi/mca/btl/portals/btl_portals.h b/ompi/mca/btl/portals/btl_portals.h index 3c69602d70..70efdff069 100644 --- a/ompi/mca/btl/portals/btl_portals.h +++ b/ompi/mca/btl/portals/btl_portals.h @@ -61,6 +61,9 @@ struct mca_btl_portals_component_t { /* number of eager fragments */ int portals_free_list_eager_max_num; + + /* shall I use portals to send to thyself? */ + int portals_support_self; }; typedef struct mca_btl_portals_component_t mca_btl_portals_component_t; diff --git a/ompi/mca/btl/portals/btl_portals_component.c b/ompi/mca/btl/portals/btl_portals_component.c index d15446c4cc..35aeec4ac4 100644 --- a/ompi/mca/btl/portals/btl_portals_component.c +++ b/ompi/mca/btl/portals/btl_portals_component.c @@ -125,6 +125,14 @@ mca_btl_portals_component_open(void) 32, &(mca_btl_portals_component.portals_free_list_eager_max_num)); + mca_base_param_reg_int(&mca_btl_portals_component.super.btl_version, + "support_self", + "Use portals for send to self", + false, + false, + 1, /* default to true.. */ + &(mca_btl_portals_component.portals_support_self)); + /* * fill default module state */ @@ -222,6 +230,7 @@ mca_btl_portals_component_init(int *num_btls, bool enable_mpi_threads) { mca_btl_base_module_t ** btls = malloc(sizeof(mca_btl_base_module_t*)); + bool accel; btls[0] = (mca_btl_base_module_t*) &mca_btl_portals_module; if (enable_progress_threads || enable_mpi_threads) { @@ -232,7 +241,7 @@ mca_btl_portals_component_init(int *num_btls, /* initialize portals btl. note that this is in the compat code because it's fairly non-portable between implementations */ - if (OMPI_SUCCESS != ompi_common_portals_initialize()) { + if (OMPI_SUCCESS != ompi_common_portals_initialize(&mca_btl_portals_module.portals_ni_h, &accel)) { opal_output_verbose(20, mca_btl_portals_component.portals_output, "disabled because compatibility init failed"); return NULL; diff --git a/ompi/mca/common/portals/common_portals.c b/ompi/mca/common/portals/common_portals.c index 07f7c4bdbe..421ba6510d 100644 --- a/ompi/mca/common/portals/common_portals.c +++ b/ompi/mca/common/portals/common_portals.c @@ -29,6 +29,9 @@ #include "common_portals_crayxt3.c" +#elif OMPI_PORTALS_CRAYXT3_MODEX +#include "common_portals_cray_xt_modex.c" + #else #error "Unknown Portals library configuration" @@ -107,7 +110,7 @@ ompi_common_portals_error_ptl_to_ompi(int ptl_error) case PTL_SR_INDEX_INVALID: ret = OMPI_ERR_BAD_PARAM; break; -#if !OMPI_PORTALS_CRAYXT3 +#if !(OMPI_PORTALS_CRAYXT3 || OMPI_PORTALS_CRAYXT3_MODEX) case PTL_UNKNOWN_ERROR: ret = OMPI_ERROR; break; diff --git a/ompi/mca/common/portals/common_portals.h b/ompi/mca/common/portals/common_portals.h index c2fda609d3..f4bcf4313b 100644 --- a/ompi/mca/common/portals/common_portals.h +++ b/ompi/mca/common/portals/common_portals.h @@ -34,8 +34,7 @@ #define OMPI_MTL_PORTALS_READ_TABLE_ID 3 #define OMPI_MTL_PORTALS_ACK_TABLE_ID 4 -#elif OMPI_PORTALS_CRAYXT3 - +#elif (OMPI_PORTALS_CRAYXT3 || OMPI_PORTALS_CRAYXT3_MODEX) #include #define PTL_EQ_HANDLER_NONE NULL /* Cray's definition, differs from the spec */ @@ -91,11 +90,13 @@ int ompi_common_portals_register_mca(void); * may not be possible if use of the modex is required to setup the * network (as is the case with the utcp reference implementation). * + * @param ni_handle (OUT) network interface handle + * @param bool (OUT) true if using accelerated Portals, false otherwise + * * @retval OMPI_SUCCESS Portals successfully initialized * @retval OMPI_ERR_NOT_AVAILABLE Portals could not be initialized */ -int ompi_common_portals_initialize(void); - +int ompi_common_portals_initialize(ptl_handle_ni_t *ni_handle, bool *accel); /** * Initialize network interface @@ -112,8 +113,7 @@ int ompi_common_portals_initialize(void); * @retval OMPI_SUCCESS Portals network interface successfully initialized * @retval OMPI_ERROR Something bad happened */ -int ompi_common_portals_ni_initialize(ptl_handle_ni_t *ni_handle, - bool *accel); +int ompi_common_portals_ni_initialize(ptl_handle_ni_t *ni_handle, bool *accel); /** diff --git a/ompi/mca/common/portals/common_portals_crayxt3.c b/ompi/mca/common/portals/common_portals_crayxt3.c index ecf52d0d77..e7ab9ab61a 100644 --- a/ompi/mca/common/portals/common_portals_crayxt3.c +++ b/ompi/mca/common/portals/common_portals_crayxt3.c @@ -42,7 +42,7 @@ ompi_common_portals_register_mca(void) int -ompi_common_portals_initialize(void) +ompi_common_portals_initialize(ptl_handle_ni_t *ni_handle, bool *accel) { int ret, max_interfaces; diff --git a/ompi/mca/common/portals/common_portals_utcp.c b/ompi/mca/common/portals/common_portals_utcp.c index 74bd151e25..9a392151d5 100644 --- a/ompi/mca/common/portals/common_portals_utcp.c +++ b/ompi/mca/common/portals/common_portals_utcp.c @@ -80,7 +80,7 @@ ompi_common_portals_register_mca(void) int -ompi_common_portals_initialize(void) +ompi_common_portals_initialize(ptl_handle_ni_t *ni_handle, bool *accel) { int ret; ptl_process_id_t info; diff --git a/ompi/mca/mtl/portals/mtl_portals.c b/ompi/mca/mtl/portals/mtl_portals.c index 2393766c98..dc470c69ff 100644 --- a/ompi/mca/mtl/portals/mtl_portals.c +++ b/ompi/mca/mtl/portals/mtl_portals.c @@ -86,8 +86,11 @@ ompi_mtl_portals_add_procs(struct mca_mtl_base_module_t *mtl, ptl_process_id_t *portals_procs = NULL; ptl_md_t md; size_t i; + ptl_match_bits_t match_bits; + ptl_match_bits_t ignore_bits; + ptl_process_id_t anyid = { PTL_NID_ANY, PTL_PID_ANY }; bool accel; - + assert(mtl == &ompi_mtl_portals.base); /* if we havne't already initialized the network, do so now. We @@ -95,141 +98,140 @@ ompi_mtl_portals_add_procs(struct mca_mtl_base_module_t *mtl, environment setup the common code does for the utcp implementation, we can't do it until modex information can be received. */ + if (PTL_INVALID_HANDLE == ompi_mtl_portals.ptl_ni_h) { - ptl_match_bits_t match_bits; - ptl_match_bits_t ignore_bits; - ptl_process_id_t anyid = { PTL_NID_ANY, PTL_PID_ANY }; - + ret = ompi_common_portals_ni_initialize(&(ompi_mtl_portals.ptl_ni_h), &accel); if (OMPI_SUCCESS != ret) goto cleanup; + } + + /* event queue for expected events */ + ret = PtlEQAlloc(ompi_mtl_portals.ptl_ni_h, + ompi_mtl_portals.ptl_expected_queue_size, + PTL_EQ_HANDLER_NONE, + &(ompi_mtl_portals.ptl_eq_h)); + assert(ret == PTL_OK); + + /* event queue for unexpected receives */ + ret = PtlEQAlloc(ompi_mtl_portals.ptl_ni_h, + ompi_mtl_portals.ptl_unexpected_queue_size, + PTL_EQ_HANDLER_NONE, + &(ompi_mtl_portals.ptl_unex_eq_h)); + assert(ret == PTL_OK); - /* event queue for expected events */ - ret = PtlEQAlloc(ompi_mtl_portals.ptl_ni_h, - ompi_mtl_portals.ptl_expected_queue_size, - PTL_EQ_HANDLER_NONE, - &(ompi_mtl_portals.ptl_eq_h)); - assert(ret == PTL_OK); + /* empty event queue for PtlMEMDPost() */ + ret = PtlEQAlloc(ompi_mtl_portals.ptl_ni_h, + 1, + PTL_EQ_HANDLER_NONE, + &(ompi_mtl_portals.ptl_empty_eq_h)); + assert(ret == PTL_OK); - /* event queue for unexpected receives */ - ret = PtlEQAlloc(ompi_mtl_portals.ptl_ni_h, - ompi_mtl_portals.ptl_unexpected_queue_size, - PTL_EQ_HANDLER_NONE, - &(ompi_mtl_portals.ptl_unex_eq_h)); - assert(ret == PTL_OK); + /* attach the long unex msg buffer */ + match_bits = PTL_LONG_MSG; + ignore_bits = ~(PTL_LONG_MSG); - /* empty event queue for PtlMEMDPost() */ - ret = PtlEQAlloc(ompi_mtl_portals.ptl_ni_h, - 1, - PTL_EQ_HANDLER_NONE, - &(ompi_mtl_portals.ptl_empty_eq_h)); - assert(ret == PTL_OK); + ret = PtlMEAttach(ompi_mtl_portals.ptl_ni_h, + OMPI_MTL_PORTALS_SEND_TABLE_ID, + anyid, + match_bits, + ignore_bits, + PTL_RETAIN, + PTL_INS_AFTER, + &(ompi_mtl_portals.ptl_unex_long_me_h)); + assert(ret == PTL_OK); - /* attach the long unex msg buffer */ - match_bits = PTL_LONG_MSG; - ignore_bits = ~(PTL_LONG_MSG); + md.start = NULL; + md.length = 0; + md.threshold = PTL_MD_THRESH_INF; + md.max_size = 0; + md.options = PTL_MD_OP_PUT | PTL_MD_TRUNCATE | PTL_MD_ACK_DISABLE; + md.eq_handle = ompi_mtl_portals.ptl_unex_eq_h; + md.user_ptr = NULL; - ret = PtlMEAttach(ompi_mtl_portals.ptl_ni_h, - OMPI_MTL_PORTALS_SEND_TABLE_ID, - anyid, - match_bits, - ignore_bits, - PTL_RETAIN, - PTL_INS_AFTER, - &(ompi_mtl_portals.ptl_unex_long_me_h)); - assert(ret == PTL_OK); + ret = PtlMDAttach(ompi_mtl_portals.ptl_unex_long_me_h, + md, + PTL_RETAIN, + &unex_long_md_h); + assert(ret == PTL_OK); - md.start = NULL; - md.length = 0; - md.threshold = PTL_MD_THRESH_INF; - md.max_size = 0; - md.options = PTL_MD_OP_PUT | PTL_MD_TRUNCATE | PTL_MD_ACK_DISABLE; - md.eq_handle = ompi_mtl_portals.ptl_unex_eq_h; - md.user_ptr = NULL; + /* attach catchalls to the send, ack, and read portals */ + catchall_request.event_callback = ompi_mtl_portals_catchall_callback; + md.eq_handle = ompi_mtl_portals.ptl_eq_h; + md.user_ptr = &catchall_request; - ret = PtlMDAttach(ompi_mtl_portals.ptl_unex_long_me_h, - md, - PTL_RETAIN, - &unex_long_md_h); - assert(ret == PTL_OK); + /* catchall for the send portal */ + ret = PtlMEMDPost(ompi_mtl_portals.ptl_ni_h, + ompi_mtl_portals.ptl_unex_long_me_h, + anyid, + 0, + ~0, + PTL_RETAIN, + PTL_INS_AFTER, + md, + PTL_UNLINK, + &(ompi_mtl_portals.ptl_send_catchall_me_h), + &send_catchall_md_h, + ompi_mtl_portals.ptl_empty_eq_h); + assert(ret == PTL_OK); - /* attach catchalls to the send, ack, and read portals */ - catchall_request.event_callback = ompi_mtl_portals_catchall_callback; - md.eq_handle = ompi_mtl_portals.ptl_eq_h; - md.user_ptr = &catchall_request; + /* catchall for ack portal */ + ret = PtlMEAttach(ompi_mtl_portals.ptl_ni_h, + OMPI_MTL_PORTALS_ACK_TABLE_ID, + anyid, + 0, + ~0, + PTL_RETAIN, + PTL_INS_AFTER, + &(ompi_mtl_portals.ptl_ack_catchall_me_h)); + assert(ret == PTL_OK); - /* catchall for the send portal */ - ret = PtlMEMDPost(ompi_mtl_portals.ptl_ni_h, - ompi_mtl_portals.ptl_unex_long_me_h, - anyid, - 0, - ~0, - PTL_RETAIN, - PTL_INS_AFTER, - md, - PTL_UNLINK, - &(ompi_mtl_portals.ptl_send_catchall_me_h), - &send_catchall_md_h, - ompi_mtl_portals.ptl_empty_eq_h); - assert(ret == PTL_OK); + ret = PtlMDAttach(ompi_mtl_portals.ptl_ack_catchall_me_h, + md, + PTL_UNLINK, + &ack_catchall_md_h); + assert(ret == PTL_OK); - /* catchall for ack portal */ - ret = PtlMEAttach(ompi_mtl_portals.ptl_ni_h, - OMPI_MTL_PORTALS_ACK_TABLE_ID, - anyid, - 0, - ~0, - PTL_RETAIN, - PTL_INS_AFTER, - &(ompi_mtl_portals.ptl_ack_catchall_me_h)); - assert(ret == PTL_OK); + /* catchall for read portal */ + ret = PtlMEAttach(ompi_mtl_portals.ptl_ni_h, + OMPI_MTL_PORTALS_READ_TABLE_ID, + anyid, + 0, + ~0, + PTL_RETAIN, + PTL_INS_AFTER, + &(ompi_mtl_portals.ptl_read_catchall_me_h)); + assert(ret == PTL_OK); - ret = PtlMDAttach(ompi_mtl_portals.ptl_ack_catchall_me_h, - md, - PTL_UNLINK, - &ack_catchall_md_h); - assert(ret == PTL_OK); + ret = PtlMDAttach(ompi_mtl_portals.ptl_read_catchall_me_h, + md, + PTL_RETAIN, + &read_catchall_md_h); + assert(ret == PTL_OK); - /* catchall for read portal */ - ret = PtlMEAttach(ompi_mtl_portals.ptl_ni_h, - OMPI_MTL_PORTALS_READ_TABLE_ID, - anyid, - 0, - ~0, - PTL_RETAIN, - PTL_INS_AFTER, - &(ompi_mtl_portals.ptl_read_catchall_me_h)); - assert(ret == PTL_OK); + /* attach short unex recv blocks */ + ret = ompi_mtl_portals_recv_short_enable((mca_mtl_portals_module_t*) mtl); - ret = PtlMDAttach(ompi_mtl_portals.ptl_read_catchall_me_h, - md, - PTL_RETAIN, - &read_catchall_md_h); - assert(ret == PTL_OK); + opal_progress_register(ompi_mtl_portals_progress); - /* attach short unex recv blocks */ - ret = ompi_mtl_portals_recv_short_enable((mca_mtl_portals_module_t*) mtl); + /* bind zero-length md for sending zero-length msgs and acks */ + md.start = NULL; + md.length = 0; + md.threshold = PTL_MD_THRESH_INF; + md.max_size = 0; + md.options = PTL_MD_EVENT_START_DISABLE | PTL_MD_EVENT_END_DISABLE; + md.user_ptr = NULL; + md.eq_handle = PTL_EQ_NONE; - opal_progress_register(ompi_mtl_portals_progress); + ret = PtlMDBind(ompi_mtl_portals.ptl_ni_h, + md, + PTL_RETAIN, + &ompi_mtl_portals.ptl_zero_md_h ); + assert(ret == PTL_OK); - /* bind zero-length md for sending zero-length msgs and acks */ - md.start = NULL; - md.length = 0; - md.threshold = PTL_MD_THRESH_INF; - md.max_size = 0; - md.options = PTL_MD_EVENT_START_DISABLE | PTL_MD_EVENT_END_DISABLE; - md.user_ptr = NULL; - md.eq_handle = PTL_EQ_NONE; + /* set up the short copy blocks */ + ompi_mtl_portals_short_setup(); - ret = PtlMDBind(ompi_mtl_portals.ptl_ni_h, - md, - PTL_RETAIN, - &ompi_mtl_portals.ptl_zero_md_h ); - assert(ret == PTL_OK); - - /* set up the short copy blocks */ - ompi_mtl_portals_short_setup(); - - } + /* get the list of ptl_process_id_t structures for the given proc structures. If the Portals runtime environment supports diff --git a/ompi/mca/mtl/portals/mtl_portals_component.c b/ompi/mca/mtl/portals/mtl_portals_component.c index 35822fcdf4..744d4b8828 100644 --- a/ompi/mca/mtl/portals/mtl_portals_component.c +++ b/ompi/mca/mtl/portals/mtl_portals_component.c @@ -166,11 +166,12 @@ static mca_mtl_base_module_t* ompi_mtl_portals_component_init(bool enable_progress_threads, bool enable_mpi_threads) { + bool accel; /* we don't run with no stinkin' threads */ if (enable_progress_threads || enable_mpi_threads) return NULL; /* initialize our interface */ - if (OMPI_SUCCESS != ompi_common_portals_initialize()) { + if (OMPI_SUCCESS != ompi_common_portals_initialize(&(ompi_mtl_portals.ptl_ni_h), &accel)) { return NULL; }