ports/openmpi
ports/openmpi
1
1
Форкнуть 0
Код Релизы Активность

fix #1829 : udapl btl support for relaxed ordering

Просмотр исходного кода 
This commit was SVN r20772.
Этот коммит содержится в:
Donald Kerr 2009-03-13 01:01:00 +00:00
родитель 04585ed8e3
Коммит ef55aae401
6 изменённых файлов: 60 добавлений и 24 удалений
Показать статистику Скачать Patch файл Скачать Diff файл Показать все файлы Свернуть все файлы

52
ompi/mca/btl/udapl/btl_udapl.c
Просмотреть файл

@ -89,13 +89,20 @@ static int udapl_reg_mr(void *reg_data, void *base, size_t size,
DAT_VLEN dat_size;
DAT_VADDR dat_addr;
int rc;
DAT_MEM_TYPE lmr_mem_type = DAT_MEM_TYPE_VIRTUAL;
region.for_va = base;
udapl_reg->lmr_triplet.virtual_address = (DAT_VADDR)(uintptr_t)base;
udapl_reg->lmr_triplet.segment_length = size;
udapl_reg->lmr = NULL;
rc = dat_lmr_create(btl->udapl_ia, DAT_MEM_TYPE_VIRTUAL, region, size,
#if HAVE_UDAPL_DAT_MEM_TYPE_SO_VIRTUAL
if (reg->flags & MCA_MPOOL_FLAGS_SO_MEM) {
lmr_mem_type = DAT_MEM_TYPE_SO_VIRTUAL;
}
#endif
rc = dat_lmr_create(btl->udapl_ia, lmr_mem_type, region, size,
btl->udapl_pz, DAT_MEM_PRIV_ALL_FLAG, &udapl_reg->lmr,
&udapl_reg->lmr_triplet.lmr_context, &udapl_reg->rmr_context,
&dat_size, &dat_addr);
@ -156,30 +163,32 @@ mca_btl_udapl_init(DAT_NAME_PTR ia_name, mca_btl_udapl_module_t* btl)
#if defined(__SVR4) && defined(__sun)
if (strcmp(major, "DAT_INVALID_PARAMETER") == 0 &&
strcmp(minor, "DAT_INVALID_RO_COOKIE") == 0) {
/* Some platforms that Solaris runs on implement the PCI
* standard for relaxed ordering(RO). Using RDMA with
* polling on a memory location as the uDAPL (and openib
* BTL by the way) BTL does for short messages with
* relaxed ordering could potentially produce silent data
* corruption. For this reason we need to detect systems
* which support relaxed ordering and turn off RDMA for
* short messages. The uDAPL standard does not provide a
* way to inform users of this scenario so Sun has
* implemented the following. If a platform supports
* relaxed ordering when the interface name is passed into
* the dat_ia_open() call, the call will return
* DAT_INVALID_PARAMETER and DAT_INVALID_RO_COOKIE.
* DAT_INVALID_RO_COOKIE is not part of the uDAPL standard
* at this time. The only way to open this interface is to
* prefix the following cookie "RO_AWARE_" to the ia name
* that was retreived from the dat registry.
/* Some platforms that Solaris runs on implement the PCI
* standard for relaxed ordering(RO). Using RDMA with
* polling on a memory location as the uDAPL (and openib
* by the way) BTL does for short messages with
* relaxed ordering could potentially produce silent data
* corruption. For this reason we need to take extra
* steps and this is accomplished by setting
* "ro_aware_system = 1" and handling as required.
*
* The uDAPL standard does not provide an interface to
* inform users of this scenario so Sun has implemented the
* following: If a platform supports relaxed ordering
* when the interface name is passed into the
* dat_ia_open() call, the call will return
* DAT_INVALID_PARAMETER and DAT_INVALID_RO_COOKIE.
* DAT_INVALID_RO_COOKIE is not part of the uDAPL standard
* at this time. The only way to open this interface is
* to prefix the following cookie "RO_AWARE_" to the ia
* name that was retreived from the dat registry.
*
* Example: ia_name = "ib0", new expected name will be
* "RO_AWARE_ib0".
*
* Here, since our first ia open attempt failed in the
* predetermined way, add the cookie and try to open again.
**/
* standard way, add the cookie and try to open again.
*/
DAT_NAME_PTR ro_ia_name;
/* prefix relaxed order cookie to ia_name */
@ -202,8 +211,7 @@ mca_btl_udapl_init(DAT_NAME_PTR ia_name, mca_btl_udapl_module_t* btl)
(const char**)&minor);
if (DAT_SUCCESS == rc) {
/* do not allow RDMA for short messages */
mca_btl_udapl_component.udapl_use_eager_rdma = 0;
mca_btl_udapl_component.ro_aware_system = 1;
free(ro_ia_name);
} else {
BTL_UDAPL_VERBOSE_HELP(VERBOSE_SHOW_HELP,

1
ompi/mca/btl/udapl/btl_udapl.h
Просмотреть файл

@ -95,6 +95,7 @@ struct mca_btl_udapl_component_t {
char *if_exclude;
char **if_exclude_list;
char **if_list; /* used for checking entries not found */
int32_t ro_aware_system; /* default 0; 1 if relaxed ordered platform */
};
typedef struct mca_btl_udapl_component_t mca_btl_udapl_component_t;

3
ompi/mca/btl/udapl/btl_udapl_component.c
Просмотреть файл

@ -12,7 +12,7 @@
* All rights reserved.
* Copyright (c) 2006 Sandia National Laboratories. All rights
* reserved.
* Copyright (c) 2007-2008 Sun Microsystems, Inc. All rights reserved.
* Copyright (c) 2007-2009 Sun Microsystems, Inc. All rights reserved.
* Copyright (c) 2007-2008 Cisco Systems, Inc. All rights reserved.
*
* $COPYRIGHT$
@ -192,6 +192,7 @@ int mca_btl_udapl_component_open(void)
/* initialize state */
mca_btl_udapl_component.udapl_num_btls=0;
mca_btl_udapl_component.udapl_btls=NULL;
mca_btl_udapl_component.ro_aware_system=0;
/* initialize objects */
OBJ_CONSTRUCT(&mca_btl_udapl_component.udapl_procs, opal_list_t);

8
ompi/mca/btl/udapl/btl_udapl_endpoint.c
Просмотреть файл

@ -1142,6 +1142,7 @@ void mca_btl_udapl_endpoint_connect_eager_rdma(
char* alloc_ptr;
size_t size_plus_align;
int i;
uint32_t flags = MCA_MPOOL_FLAGS_CACHE_BYPASS;
mca_btl_udapl_module_t* udapl_btl = endpoint->endpoint_btl;
OPAL_THREAD_LOCK(&endpoint->endpoint_eager_rdma_local.lock);
@ -1172,10 +1173,15 @@ void mca_btl_udapl_endpoint_connect_eager_rdma(
mca_btl_udapl_component.udapl_eager_frag_size,
DAT_OPTIMAL_ALIGNMENT, size_t);
/* set flags value accordingly if ro aware */
if (mca_btl_udapl_component.ro_aware_system) {
flags |= MCA_MPOOL_FLAGS_SO_MEM;
}
/* create and register memory for all rdma segments */
buf = udapl_btl->super.btl_mpool->mpool_alloc(udapl_btl->super.btl_mpool,
(size_plus_align * mca_btl_udapl_component.udapl_eager_rdma_num),
0, 0,
0, flags,
(mca_mpool_base_registration_t**)&endpoint->endpoint_eager_rdma_local.reg);
if(!buf)

18
ompi/mca/btl/udapl/configure.m4
Просмотреть файл

@ -10,6 +10,7 @@
# University of Stuttgart. All rights reserved.
# Copyright (c) 2004-2005 The Regents of the University of California.
# All rights reserved.
# Copyright (c) 2009 Sun Microsystems, Inc. All rights reserved.
# $COPYRIGHT$
#
# Additional copyrights may follow
@ -39,6 +40,23 @@ AC_DEFUN([MCA_btl_udapl_CONFIG],[
[AC_MSG_WARN([Removed -pedantic from CFLAGS for
uDAPL component because the uDAPL headers are not fully ISO C])])
# Test for uDAPL relaxed ordered specific symbols
AS_IF([test "$btl_udapl_happy" = "yes"],
[AC_MSG_CHECKING(for uDAPL relax order aware)
AC_TRY_COMPILE([#include <dat/udat.h>],
[DAT_RETURN dr = DAT_INVALID_RO_COOKIE;
DAT_MEM_TYPE dmt = DAT_MEM_TYPE_SO_VIRTUAL;],
[AC_MSG_RESULT(yes)
btl_udapl_ro_aware=1],
[AC_MSG_RESULT(no)
btl_udapl_ro_aware=0])
AC_DEFINE_UNQUOTED([HAVE_UDAPL_DAT_INVALID_RO_COOKIE],
[$btl_udapl_ro_aware],
[uDAPL DAT_INVALID_RO_COOKIE check])
AC_DEFINE_UNQUOTED([HAVE_UDAPL_DAT_MEM_TYPE_SO_VIRTUAL],
[$btl_udapl_ro_aware],
[uDAPL DAT_MEM_TYPE_SO_VIRTUAL check])])
# substitute in the things needed to build udapl
AC_SUBST([btl_udapl_CFLAGS])
AC_SUBST([btl_udapl_CPPFLAGS])

2
ompi/mca/mpool/mpool.h
Просмотреть файл

@ -10,6 +10,7 @@
* University of Stuttgart. All rights reserved.
* Copyright (c) 2004-2005 The Regents of the University of California.
* All rights reserved.
* Copyright (c) 2009 Sun Microsystems, Inc. All rights reserved.
* $COPYRIGHT$
*
* Additional copyrights may follow
@ -34,6 +35,7 @@
#define MCA_MPOOL_FLAGS_PERSIST 0x2
#define MCA_MPOOL_FLAGS_MPI_ALLOC_MEM 0x4
#define MCA_MPOOL_FLAGS_INVALID 0x8
#define MCA_MPOOL_FLAGS_SO_MEM 0x10
struct mca_mpool_base_resources_t;