ports/openmpi
ports/openmpi
1
1
Форкнуть 0
Код Релизы Активность

Remove the proc_pml and proc_bml fields from ompi_proc_t and replace with a

Просмотр исходного кода 
configure-time dynamic allocation of flags.  The net result for platforms
which only support BTL-based communication is a reduction of 8*nprocs bytes
per process.  Platforms which support both MTLs and BTLs will not see
a space reduction, but will now be able to safely run both the MTL and BTL
side-by-side, which will prove useful.

This commit was SVN r29100.
Этот коммит содержится в:
Brian Barrett 2013-08-30 16:54:55 +00:00
родитель 18962d296b
Коммит 16a1166884
58 изменённых файлов: 506 добавлений и 282 удалений
Показать статистику Скачать Patch файл Скачать Diff файл Показать все файлы Свернуть все файлы

46
config/ompi_endpoint_tag.m4 Обычный файл
Просмотреть файл

@ -0,0 +1,46 @@
# -*- shell-script -*-
#
# Copyright (c) 2013 Sandia National Laboratories. All rights reserved.
# $COPYRIGHT$
#
# Additional copyrights may follow
#
# $HEADER$
#
AC_DEFUN([OMPI_REQUIRE_ENDPOINT_TAG_INIT], [
ompi_endpoint_tag_counter=0
])
# OMPI_REQUIRE_ENDPOINT_TAG(tag_name)
# -----------------------------------
# This component requires an endpoint tag (storage space in the
# ompi_proc_t structure) for a tag named tag_name. If tag_name
# already has a tag index, this function is a no-op.
AC_DEFUN([OMPI_REQUIRE_ENDPOINT_TAG], [
AC_REQUIRE([OMPI_REQUIRE_ENDPOINT_TAG_INIT])
AC_MSG_CHECKING([for index in endpoint array for tag $1])
AS_IF([test -z "$OMPI_PROC_ENDPOINT_TAG_$1"],
[OMPI_PROC_ENDPOINT_TAG_$1=$ompi_endpoint_tag_counter
AC_DEFINE_UNQUOTED([OMPI_PROC_ENDPOINT_TAG_$1], [$OMPI_PROC_ENDPOINT_TAG_$1],
[Index into endpoint array for $1])
ompi_endpoint_tag_counter=`expr $ompi_endpoint_tag_counter + 1`])
AC_MSG_RESULT([$OMPI_PROC_ENDPOINT_TAG_$1])
])
AC_DEFUN([OMPI_REQUIRE_ENDPOINT_TAG_FINI], [
dnl AC_ARG_ENABLE([dynamic-endpoint-information],
dnl [AC_HELP_STRING([--disable-dynamic-endpoint-information],
dnl [Disable support for dynamic endpoint information storage. This saves a considerable amount of memory for large processes, but makes loading outside components which require endpoint storage impossible.])])
dnl AS_IF([test "$enable_endpoint_information" != "no"],
dnl [OMPI_REQUIRE_ENDPOINT_TAG(DYNAMIC)])
AC_MSG_CHECKING([for size of endpoint array])
AS_IF([test -z "$ompi_endpoint_tag_counter" -o "$ompi_endpoint_tag_counter" = "0"],
[AC_MSG_ERROR([Endpoint index count is 0. This means no MPI communication would be possible. Aborting.])])
AC_MSG_RESULT([$ompi_endpoint_tag_counter])
AC_DEFINE_UNQUOTED([OMPI_PROC_ENDPOINT_TAG_MAX], [$ompi_endpoint_tag_counter],
[Maximum number of endpoint entries to be attached to an ompi_proc_t])
])

2
configure.ac
Просмотреть файл

@ -1068,6 +1068,8 @@ AC_MSG_RESULT([$ompi_subdir_args])
OMPI_MCA
m4_ifdef([project_ompi], [OMPI_REQUIRE_ENDPOINT_TAG_FINI])
# checkpoint results
AC_CACHE_SAVE

17
ompi/mca/bml/r2/bml_r2.c
Просмотреть файл

@ -164,7 +164,7 @@ static int mca_bml_r2_add_procs( size_t nprocs,
struct ompi_proc_t* proc = procs[p_index];
OBJ_RETAIN(proc);
if(NULL != proc->proc_bml) {
if(NULL != proc->proc_endpoints[OMPI_PROC_ENDPOINT_TAG_BML]) {
continue; /* go to the next proc */
}
/* Allocate the new_procs on demand */
@ -217,7 +217,8 @@ static int mca_bml_r2_add_procs( size_t nprocs,
for( p = 0; p < n_new_procs; p++ ) {
if(opal_bitmap_is_set_bit(reachable, p)) {
ompi_proc_t *proc = new_procs[p];
mca_bml_base_endpoint_t * bml_endpoint = (mca_bml_base_endpoint_t*) proc->proc_bml;
mca_bml_base_endpoint_t * bml_endpoint =
(mca_bml_base_endpoint_t*) proc->proc_endpoints[OMPI_PROC_ENDPOINT_TAG_BML];
mca_bml_base_btl_t* bml_btl;
size_t size;
@ -237,7 +238,7 @@ static int mca_bml_r2_add_procs( size_t nprocs,
mca_bml_base_btl_array_reserve(&bml_endpoint->btl_rdma, mca_bml_r2.num_btl_modules);
bml_endpoint->btl_max_send_size = -1;
bml_endpoint->btl_proc = proc;
proc->proc_bml = bml_endpoint;
proc->proc_endpoints[OMPI_PROC_ENDPOINT_TAG_BML] = bml_endpoint;
bml_endpoint->btl_flags_or = 0;
}
@ -309,7 +310,8 @@ static int mca_bml_r2_add_procs( size_t nprocs,
/* iterate back through procs and compute metrics for registered r2s */
for(p=0; p<n_new_procs; p++) {
ompi_proc_t *proc = new_procs[p];
mca_bml_base_endpoint_t* bml_endpoint = (mca_bml_base_endpoint_t*) proc->proc_bml;
mca_bml_base_endpoint_t* bml_endpoint =
(mca_bml_base_endpoint_t*) proc->proc_endpoints[OMPI_PROC_ENDPOINT_TAG_BML];
double total_bandwidth = 0;
uint32_t latency = 0xffffffff;
size_t n_index;
@ -395,7 +397,7 @@ static int mca_bml_r2_add_procs( size_t nprocs,
for(p=0; p<n_new_procs; p++) {
ompi_proc_t *proc = new_procs[p];
if (NULL == proc->proc_bml) {
if (NULL == proc->proc_endpoints[OMPI_PROC_ENDPOINT_TAG_BML]) {
if (NULL == unreach_proc) {
unreach_proc = proc;
}
@ -449,7 +451,8 @@ static int mca_bml_r2_del_procs(size_t nprocs,
for(p = 0; p < n_del_procs; p++) {
ompi_proc_t *proc = del_procs[p];
mca_bml_base_endpoint_t* bml_endpoint = (mca_bml_base_endpoint_t*) proc->proc_bml;
mca_bml_base_endpoint_t* bml_endpoint =
(mca_bml_base_endpoint_t*) proc->proc_endpoints[OMPI_PROC_ENDPOINT_TAG_BML];
size_t f_index, f_size;
size_t n_index, n_size;
@ -520,7 +523,7 @@ static inline int bml_r2_remove_btl_progress(mca_btl_base_module_t* btl)
static int mca_bml_r2_del_proc_btl(ompi_proc_t* proc, mca_btl_base_module_t* btl)
{
mca_bml_base_endpoint_t* ep = (mca_bml_base_endpoint_t*)proc->proc_bml;
mca_bml_base_endpoint_t* ep = (mca_bml_base_endpoint_t*)proc->proc_endpoints[OMPI_PROC_ENDPOINT_TAG_BML];
mca_bml_base_btl_t* bml_btl;
mca_btl_base_module_t* ep_btl;
double total_bandwidth = 0;

12
ompi/mca/bml/r2/bml_r2_ft.c
Просмотреть файл

@ -190,9 +190,9 @@ int mca_bml_r2_ft_event(int state)
mca_bml_r2.btls_added = false;
for(p = 0; p < (int)num_procs; ++p) {
if( NULL != procs[p]->proc_bml) {
OBJ_RELEASE(procs[p]->proc_bml);
procs[p]->proc_bml = NULL;
if( NULL != procs[p]->proc_endpoints[OMPI_PROC_ENDPOINT_TAG_BML]) {
OBJ_RELEASE((mca_bml_base_endpoint_t*) procs[p]->proc_endpoints[OMPI_PROC_ENDPOINT_TAG_BML]);
procs[p]->proc_endpoints[OMPI_PROC_ENDPOINT_TAG_BML] = NULL;
}
OBJ_RELEASE(procs[p]);
@ -285,9 +285,9 @@ int mca_bml_r2_ft_event(int state)
mca_bml_r2.btls_added = false;
for(p = 0; p < (int)num_procs; ++p) {
if( NULL != procs[p]->proc_bml) {
OBJ_RELEASE(procs[p]->proc_bml);
procs[p]->proc_bml = NULL;
if( NULL != procs[p]->proc_endpoints[OMPI_PROC_ENDPOINT_TAG_BML]) {
OBJ_RELEASE((mca_bml_base_endpoint_t*) procs[p]->proc_endpoints[OMPI_PROC_ENDPOINT_TAG_BML]);
procs[p]->proc_endpoints[OMPI_PROC_ENDPOINT_TAG_BML] = NULL;
}
OBJ_RELEASE(procs[p]);

26
ompi/mca/bml/r2/configure.m4 Обычный файл
Просмотреть файл

@ -0,0 +1,26 @@
# -*- shell-script -*-
#
# Copyright (c) 2013 Sandia National Laboratories. All rights reserved.
# $COPYRIGHT$
#
# Additional copyrights may follow
#
# $HEADER$
#
# MCA_ompi_bml_r2_POST_CONFIG(will_build)
# ----------------------------------------
# Only require the tag if we're actually going to be built, since bml
# is one of the ones frequently disabled for large installs.
AC_DEFUN([MCA_ompi_bml_r2_POST_CONFIG], [
AS_IF([test "$1" = "1"], [OMPI_REQUIRE_ENDPOINT_TAG([BML])])
])dnl
# MCA_ompi_bml_r2_CONFIG(action-if-can-compile,
# [action-if-cant-compile])
# ------------------------------------------------
# We can always build, unless we were explicitly disabled.
AC_DEFUN([MCA_ompi_bml_r2_CONFIG],[
AC_CONFIG_FILES([ompi/mca/bml/r2/Makefile])
[$1]
])dnl

29
ompi/mca/btl/portals4/btl_portals4.c
Просмотреть файл

@ -115,11 +115,25 @@ mca_btl_portals4_add_procs(struct mca_btl_base_module_t* btl_base,
return OMPI_ERROR;
}
btl_peer_data[i] = malloc(sizeof(mca_btl_base_endpoint_t));
if (NULL == btl_peer_data[i]) return OMPI_ERROR;
btl_peer_data[i]->ptl_proc = *id;
if (NULL == procs[i]->proc_endpoints[OMPI_PROC_ENDPOINT_TAG_PORTALS4]) {
btl_peer_data[i] = malloc(sizeof(mca_btl_base_endpoint_t));
if (NULL == btl_peer_data[i]) return OMPI_ERROR;
btl_peer_data[i]->ptl_proc = *id;
procs[i]->proc_endpoints[OMPI_PROC_ENDPOINT_TAG_PORTALS4] = btl_peer_data[i];
OPAL_OUTPUT_VERBOSE((90, ompi_btl_base_framework.framework_output, "add_procs: nid=%x pid=%x\n", id->phys.nid, id->phys.pid));
OPAL_OUTPUT_VERBOSE((90, ompi_btl_base_framework.framework_output,
"add_procs: nid=%x pid=%x\n", id->phys.nid, id->phys.pid));
} else {
ptl_process_t *proc = (ptl_process_t*) procs[i]->proc_endpoints[OMPI_PROC_ENDPOINT_TAG_PORTALS4];
if (proc->phys.nid != id->phys.nid ||
proc->phys.pid != id->phys.pid) {
opal_output_verbose(1, ompi_btl_base_framework.framework_output,
"%s:%d: existing peer and modex peer don't match\n",
__FILE__, __LINE__);
return OMPI_ERROR;
}
btl_peer_data[i] = (mca_btl_base_endpoint_t*) proc;
}
OPAL_THREAD_ADD32(&mca_btl_portals4_module.portals_num_procs, 1);
/* and here we can reach */
@ -147,8 +161,13 @@ mca_btl_portals4_del_procs(struct mca_btl_base_module_t *btl,
"mca_btl_portals4_del_procs: Removing %d procs (%d)", (int) nprocs,
(int) mca_btl_portals4_module.portals_num_procs);
/* See comment in btl_portals4_endpoint.h about why we look at the
portals4 entry in proc_endpoints instead of the peer_data */
for (i = 0 ; i < nprocs ; ++i) {
free(btl_peer_data[i]);
if (NULL != procs[i]->proc_endpoints[OMPI_PROC_ENDPOINT_TAG_PORTALS4]) {
free(procs[i]->proc_endpoints[OMPI_PROC_ENDPOINT_TAG_PORTALS4]);
procs[i]->proc_endpoints[OMPI_PROC_ENDPOINT_TAG_PORTALS4] = NULL;
}
OPAL_THREAD_ADD32(&mca_btl_portals4_module.portals_num_procs, -1);
}

9
ompi/mca/btl/portals4/btl_portals4_endpoint.h
Просмотреть файл

@ -29,6 +29,15 @@ BEGIN_C_DECLS
* An instance of mca_btl_base_endpoint_t is associated w/ each process
* and BTL pair at startup. However, connections to the endpoint
* are established dynamically on an as-needed basis:
*
* The MTL, OSC, and COLL components expect the ptl_process_t to be
* hanging off the PORTALS4 tag in the proc_endpoints. That's not
* entirely convenient for the BTL, since we get an endpoint, not an
* ompi_proc_t. So we store the ptl_process_t in both places. Since
* the btl_base_endpoint_t is just a ptl_process_t, we use the same
* storage for both. During tear-down, it's entirely possible that
* the MTL is going to free the PORTALS4 memory, so we need to be
* careful during del_procs.
*/
struct mca_btl_base_endpoint_t {
ptl_process_t ptl_proc;

10
ompi/mca/btl/portals4/configure.m4
Просмотреть файл

@ -19,6 +19,16 @@
# $HEADER$
#
# MCA_ompi_btl_portals4_POST_CONFIG(will_build)
# ----------------------------------------
# Unlike most of the BTLs, we need to register an endpoint tag so that
# we can get our endpoint information from the same place as all the
# other Portals components (one-sided, mtl, coll, etc.). See comment
# in btl_portals4_endpoint.h for how the pieces fit together.
AC_DEFUN([MCA_ompi_btl_portals4_POST_CONFIG], [
AS_IF([test "$1" = "1"], [OMPI_REQUIRE_ENDPOINT_TAG([PORTALS4])])
])dnl
# MCA_btl_portals4_CONFIG(action-if-can-compile,
# [action-if-cant-compile])
# ------------------------------------------------

14
ompi/mca/coll/hierarch/coll_hierarch.c
Просмотреть файл

@ -101,12 +101,8 @@ mca_coll_hierarch_comm_query(struct ompi_communicator_t *comm, int *priority )
return NULL;
}
/* This module only works with ob1 pml respectively btl's. Opt out
of we use cm/mtl's. */
if ( strcmp ( mca_pml_base_selected_component.pmlm_version.mca_component_name, "ob1")) {
return NULL;
}
/* This module only works when the BTLs are alive. If they aren't, time to exit. */
if (!mca_bml_base_inited()) return NULL;
size = ompi_comm_size(comm);
if (size < 3) {
@ -611,6 +607,7 @@ mca_coll_hierarch_checkfor_component ( struct ompi_communicator_t *comm,
struct mca_bml_base_btl_array_t *bml_btl_array=NULL;
mca_bml_base_btl_t *bml_btl=NULL;
mca_btl_base_component_t *btl=NULL;
mca_bml_base_endpoint_t *endpoint;
int i, size, rc;
@ -647,11 +644,12 @@ mca_coll_hierarch_checkfor_component ( struct ompi_communicator_t *comm,
continue;
}
endpoint = (mca_bml_base_endpoint_t*) procs[i]->proc_endpoints[OMPI_PROC_ENDPOINT_TAG_BML];
if ( use_rdma ) {
bml_btl_array = &(procs[i]->proc_bml->btl_rdma);
bml_btl_array = &(endpoint->btl_rdma);
}
else {
bml_btl_array = &(procs[i]->proc_bml->btl_send);
bml_btl_array = &(endpoint->btl_send);
}
bml_btl = mca_bml_base_btl_array_get_index ( bml_btl_array, 0 );
btl = bml_btl->btl->btl_component;

27
ompi/mca/coll/hierarch/configure.m4 Обычный файл
Просмотреть файл

@ -0,0 +1,27 @@
# -*- shell-script -*-
#
# Copyright (c) 2013 Sandia National Laboratories. All rights reserved.
# $COPYRIGHT$
#
# Additional copyrights may follow
#
# $HEADER$
#
# MCA_ompi_coll_hierarch_POST_CONFIG(will_build)
# ----------------------------------------
# The hierarch coll requires a BML endpoint tag to compile, so require it.
# Require in POST_CONFIG instead of CONFIG so that we only require it
# if we're not disabled.
AC_DEFUN([MCA_ompi_coll_hierarch_POST_CONFIG], [
AS_IF([test "$1" = "1"], [OMPI_REQUIRE_ENDPOINT_TAG([BML])])
])dnl
# MCA_ompi_coll_hierarch_CONFIG(action-if-can-compile,
# [action-if-cant-compile])
# ------------------------------------------------
# We can always build, unless we were explicitly disabled.
AC_DEFUN([MCA_ompi_coll_hierarch_CONFIG],[
AC_CONFIG_FILES([ompi/mca/coll/hierarch/Makefile])
[$1]
])dnl

4
ompi/mca/coll/portals4/coll_portals4.h
Просмотреть файл

@ -94,9 +94,7 @@ static inline ptl_process_t
ompi_coll_portals4_get_peer(struct ompi_communicator_t *comm, int rank)
{
ompi_proc_t *proc = ompi_comm_peer_lookup(comm, rank);
mca_mtl_base_endpoint_t * endpoint =
(mca_mtl_base_endpoint_t*) proc->proc_pml;
return endpoint->ptl_proc;
return *((ptl_process_t*) proc->proc_endpoints[OMPI_PROC_ENDPOINT_TAG_PORTALS4]);
}
static inline int

7
ompi/mca/coll/portals4/configure.m4
Просмотреть файл

@ -8,6 +8,13 @@
# $HEADER$
#
# MCA_ompi_coll_portals4_POST_CONFIG(will_build)
# ----------------------------------------
# Only require the tag if we're actually going to be built
AC_DEFUN([MCA_ompi_coll_portals4_POST_CONFIG], [
AS_IF([test "$1" = "1"], [OMPI_REQUIRE_ENDPOINT_TAG([PORTALS4])])
])dnl
# MCA_coll_portals4_CONFIG(action-if-can-compile,
# [action-if-cant-compile])
# ------------------------------------------------

26
ompi/mca/mtl/mtl.h
Просмотреть файл

@ -46,8 +46,6 @@ struct opal_convertor_t;
struct mca_mtl_base_module_t;
struct mca_mtl_base_endpoint_t;
struct mca_mtl_request_t {
/** pointer to associated ompi_request_t */
struct ompi_request_t *ompi_req;
@ -129,19 +127,15 @@ typedef int (*mca_mtl_base_module_finalize_fn_t)(struct mca_mtl_base_module_t* m
* peer process.
*
* It is an error for a proc to not be reachable by the given MTL, and
* an error should be returned if that case is detected. The PML
* provides the MTL the option to return a pointer to a data structure
* defined by the MTL that is passed in with all communication
* functions. The array of procinfo pointers will be allocated by the
* PML, but it is up to the MTL module to create the memory for the
* procinfo structure itself. The procinfo structure is opaque to the
* PML and is only used internally by the MTL.
* an error should be returned if that case is detected. If a MTL
* requires per-endpoint data, it must handle storage, either using a
* static endpoint tag (MTL is the default tag that should generally
* be used) or a dynamic endpoint tag (although it should be noted
* that OMPI can be built without dynamic endpoint tag support).
*
* @param mtl (IN) MTL module
* @param nprocs (IN) Number of processes
* @param procs (IN) Set of processes
* @param endpoint (OUT) Array of (optional) mca_mtl_base_procinfo_t
* structures, one per proc in procs
*
* @retval OMPI_SUCCESS successfully connected to processes
* @retval other failure during setup
@ -149,8 +143,7 @@ typedef int (*mca_mtl_base_module_finalize_fn_t)(struct mca_mtl_base_module_t* m
typedef int (*mca_mtl_base_module_add_procs_fn_t)(
struct mca_mtl_base_module_t* mtl,
size_t nprocs,
struct ompi_proc_t** procs,
struct mca_mtl_base_endpoint_t **mtl_peer_data);
struct ompi_proc_t** procs);
/**
@ -158,7 +151,9 @@ typedef int (*mca_mtl_base_module_add_procs_fn_t)(
*
* When the process list changes, the PML notifies the MTL of the
* change, to provide the opportunity to cleanup or release any
* resources associated with the peer.
* resources associated with the peer. The MTL is responsible for
* releasing any memory associated with the endpoint data it may have
* stored during add_procs().
*
* @param mtl (IN) MTL module
* @param nprocs (IN) Number of processes
@ -170,8 +165,7 @@ typedef int (*mca_mtl_base_module_add_procs_fn_t)(
typedef int (*mca_mtl_base_module_del_procs_fn_t)(
struct mca_mtl_base_module_t* mtl,
size_t nprocs,
struct ompi_proc_t** procs,
struct mca_mtl_base_endpoint_t **mtl_peer_data);
struct ompi_proc_t** procs);
/**

11
ompi/mca/mtl/mx/configure.m4
Просмотреть файл

@ -11,6 +11,7 @@
# Copyright (c) 2004-2005 The Regents of the University of California.
# All rights reserved.
# Copyright (c) 2010 Cisco Systems, Inc. All rights reserved.
# Copyright (c) 2013 Sandia National Laboratories. All rights reserved.
# $COPYRIGHT$
#
# Additional copyrights may follow
@ -18,6 +19,12 @@
# $HEADER$
#
# MCA_ompi_mtl_mx_POST_CONFIG(will_build)
# ----------------------------------------
# Only require the tag if we're actually going to be built
AC_DEFUN([MCA_ompi_mtl_mx_POST_CONFIG], [
AS_IF([test "$1" = "1"], [OMPI_REQUIRE_ENDPOINT_TAG([MTL])])
])dnl
# MCA_mtl_mx_CONFIG([action-if-can-compile],
# [action-if-cant-compile])
@ -26,8 +33,8 @@ AC_DEFUN([MCA_ompi_mtl_mx_CONFIG],[
AC_CONFIG_FILES([ompi/mca/mtl/mx/Makefile])
OMPI_CHECK_MX([mtl_mx],
[mtl_mx_happy="yes"],
[mtl_mx_happy="no"])
[mtl_mx_happy="yes"],
[mtl_mx_happy="no"])
AS_IF([test "$mtl_mx_happy" = "yes"],
[$1],

14
ompi/mca/mtl/mx/mtl_mx.c
Просмотреть файл

@ -33,13 +33,11 @@
static int
ompi_mtl_mx_add_procs(struct mca_mtl_base_module_t *mtl,
size_t nprocs,
struct ompi_proc_t** procs,
struct mca_mtl_base_endpoint_t **mtl_peer_data);
struct ompi_proc_t** procs);
static int
ompi_mtl_mx_del_procs(struct mca_mtl_base_module_t *mtl,
size_t nprocs,
struct ompi_proc_t** procs,
struct mca_mtl_base_endpoint_t **mtl_peer_data);
struct ompi_proc_t** procs);
static int
ompi_mtl_mx_add_comm(struct mca_mtl_base_module_t *mtl,
@ -158,8 +156,7 @@ ompi_mtl_mx_finalize(struct mca_mtl_base_module_t* mtl) {
int
ompi_mtl_mx_add_procs(struct mca_mtl_base_module_t *mtl,
size_t nprocs,
struct ompi_proc_t** procs,
struct mca_mtl_base_endpoint_t **mtl_peer_data)
struct ompi_proc_t** procs)
{
int i;
assert(mtl == &ompi_mtl_mx.super);
@ -170,7 +167,7 @@ ompi_mtl_mx_add_procs(struct mca_mtl_base_module_t *mtl,
if(NULL == mtl_mx_endpoint) {
return OMPI_ERROR;
}
mtl_peer_data[i] = (struct mca_mtl_base_endpoint_t*)
procs[i]->proc_endpoints[OMPI_PROC_ENDPOINT_TAG_MTL] =
mtl_mx_endpoint;
}
@ -191,8 +188,7 @@ ompi_mtl_mx_add_procs(struct mca_mtl_base_module_t *mtl,
int
ompi_mtl_mx_del_procs(struct mca_mtl_base_module_t *mtl,
size_t nprocs,
struct ompi_proc_t** procs,
struct mca_mtl_base_endpoint_t **mtl_peer_data)
struct ompi_proc_t** procs)
{
return OMPI_SUCCESS;
}

7
ompi/mca/mtl/mx/mtl_mx_endpoint.h
Просмотреть файл

@ -42,12 +42,12 @@ typedef struct mca_mtl_mx_addr_t mca_mtl_mx_addr_t;
/**
* An abstraction that represents a connection to a endpoint process.
* An instance of mca_mtl_base_endpoint_t is associated w/ each process
* An instance of mca_mtl_mx_endpoint_t is associated w/ each process
* and MTL pair at startup. However, connections to the endpoint
* are established dynamically on an as-needed basis:
*/
struct mca_mtl_base_endpoint_t {
struct mca_mtl_mx_endpoint_t {
opal_list_item_t super;
struct mca_mtl_mx_module_t* mtl_mx_module;
@ -61,8 +61,7 @@ struct mca_mtl_base_endpoint_t {
};
typedef struct mca_mtl_base_endpoint_t mca_mtl_base_endpoint_t;
typedef mca_mtl_base_endpoint_t mca_mtl_mx_endpoint_t;
typedef struct mca_mtl_mx_endpoint_t mca_mtl_mx_endpoint_t;
OBJ_CLASS_DECLARATION(mca_mtl_mx_endpoint);
mca_mtl_mx_endpoint_t* mca_mtl_mx_endpoint_create(ompi_proc_t*);

4
ompi/mca/mtl/mx/mtl_mx_send.c
Просмотреть файл

@ -43,7 +43,7 @@ ompi_mtl_mx_send(struct mca_mtl_base_module_t* mtl,
mx_status_t mx_status;
uint32_t result;
ompi_proc_t* ompi_proc = ompi_comm_peer_lookup( comm, dest );
mca_mtl_mx_endpoint_t* mx_endpoint = (mca_mtl_mx_endpoint_t*) ompi_proc->proc_pml;
mca_mtl_mx_endpoint_t* mx_endpoint = (mca_mtl_mx_endpoint_t*) ompi_proc->proc_endpoints[OMPI_PROC_ENDPOINT_TAG_MTL];
char* where;
assert(mtl == &ompi_mtl_mx.super);
@ -139,7 +139,7 @@ ompi_mtl_mx_isend(struct mca_mtl_base_module_t* mtl,
mca_mtl_mx_request_t * mtl_mx_request = (mca_mtl_mx_request_t*) mtl_request;
size_t length;
ompi_proc_t* ompi_proc = ompi_comm_peer_lookup( comm, dest );
mca_mtl_mx_endpoint_t* mx_endpoint = (mca_mtl_mx_endpoint_t*) ompi_proc->proc_pml;
mca_mtl_mx_endpoint_t* mx_endpoint = (mca_mtl_mx_endpoint_t*) ompi_proc->proc_endpoints[OMPI_PROC_ENDPOINT_TAG_MTL];
char* where;
assert(mtl == &ompi_mtl_mx.super);

25
ompi/mca/mtl/mxm/configure.m4
Просмотреть файл

@ -1,11 +1,20 @@
/*
* Copyright (C) Mellanox Technologies Ltd. 2001-2011. ALL RIGHTS RESERVED.
* $COPYRIGHT$
*
* Additional copyrights may follow
*
* $HEADER$
*/
# -*- shell-script -*-
#
# Copyright (C) Mellanox Technologies Ltd. 2001-2011. ALL RIGHTS RESERVED.
# Copyright (c) 2013 Sandia National Laboratories. All rights reserved.
# $COPYRIGHT$
#
# Additional copyrights may follow
#
# $HEADER$
#
# MCA_ompi_mtl_mxm_POST_CONFIG(will_build)
# ----------------------------------------
# Only require the tag if we're actually going to be built
AC_DEFUN([MCA_ompi_mtl_mxm_POST_CONFIG], [
AS_IF([test "$1" = "1"], [OMPI_REQUIRE_ENDPOINT_TAG([MTL])])
])dnl
# MCA_mtl_mxm_CONFIG([action-if-can-compile],
# [action-if-cant-compile])

27
ompi/mca/mtl/mxm/mtl_mxm.c
Просмотреть файл

@ -444,8 +444,7 @@ int ompi_mtl_mxm_finalize(struct mca_mtl_base_module_t* mtl)
}
int ompi_mtl_mxm_add_procs(struct mca_mtl_base_module_t *mtl, size_t nprocs,
struct ompi_proc_t** procs, /*const*/
struct mca_mtl_base_endpoint_t **mtl_peer_data)
struct ompi_proc_t** procs)
{
#if MXM_API < MXM_VERSION(2,0)
ompi_mtl_mxm_ep_conn_info_t *ep_info;
@ -457,6 +456,7 @@ int ompi_mtl_mxm_add_procs(struct mca_mtl_base_module_t *mtl, size_t nprocs,
mxm_error_t err;
size_t i;
int rc;
mca_mtl_mxm_endpoint_t *endpoint;
assert(mtl == &ompi_mtl_mxm.super);
@ -489,14 +489,15 @@ int ompi_mtl_mxm_add_procs(struct mca_mtl_base_module_t *mtl, size_t nprocs,
conn_reqs[i].ptl_addr[MXM_PTL_SHM] = (struct sockaddr *)&(ep_info[i].ptl_addr[MXM_PTL_SHM]);
conn_reqs[i].ptl_addr[MXM_PTL_RDMA] = (struct sockaddr *)&(ep_info[i].ptl_addr[MXM_PTL_RDMA]);
#else
mtl_peer_data[i] = (mca_mtl_mxm_endpoint_t *) OBJ_NEW(mca_mtl_mxm_endpoint_t);
mtl_peer_data[i]->mtl_mxm_module = &ompi_mtl_mxm;
err = mxm_ep_connect(ompi_mtl_mxm.ep, ep_address, &mtl_peer_data[i]->mxm_conn);
endpoint = OBJ_NEW(mca_mtl_mxm_endpoint_t);
endpoint->mtl_mxm_module = &ompi_mtl_mxm;
err = mxm_ep_connect(ompi_mtl_mxm.ep, ep_address, &endpoint->mxm_conn);
if (err != MXM_OK) {
MXM_ERROR("MXM returned connect error: %s\n", mxm_error_string(err));
rc = OMPI_ERROR;
goto bail;
}
procs[i]->proc_endpoints[OMPI_PROC_ENDPOINT_TAG_MTL] = endpoint;
#endif
free(ep_address);
}
@ -521,9 +522,10 @@ int ompi_mtl_mxm_add_procs(struct mca_mtl_base_module_t *mtl, size_t nprocs,
/* Save returned connections */
for (i = 0; i < nprocs; ++i) {
mtl_peer_data[i] = (mca_mtl_mxm_endpoint_t *) OBJ_NEW(mca_mtl_mxm_endpoint_t);
mtl_peer_data[i]->mtl_mxm_module = &ompi_mtl_mxm;
mtl_peer_data[i]->mxm_conn = conn_reqs[i].conn;
endpoint = OBJ_NEW(mca_mtl_mxm_endpoint_t);
endpoint->mtl_mxm_module = &ompi_mtl_mxm;
endpoint->mxm_conn = conn_reqs[i].conn;
procs[i]->proc_endpoints[OMPI_PROC_ENDPOINT_TAG_MTL] = endpoint;
}
#endif
rc = OMPI_SUCCESS;
@ -537,14 +539,15 @@ bail:
}
int ompi_mtl_mxm_del_procs(struct mca_mtl_base_module_t *mtl, size_t nprocs,
struct ompi_proc_t** procs,
struct mca_mtl_base_endpoint_t **mtl_peer_data)
struct ompi_proc_t** procs)
{
size_t i;
for (i = 0; i < nprocs; ++i) {
mxm_ep_disconnect(mtl_peer_data[i]->mxm_conn);
OBJ_RELEASE(mtl_peer_data[i]);
mca_mtl_mxm_endpoint_t *endpoint = (mca_mtl_mxm_endpoint_t*)
procs[i]->proc_endpoints[OMPI_PROC_ENDPOINT_TAG_MTL];
mxm_ep_disconnect(endpoint->mxm_conn);
OBJ_RELEASE(endpoint);
}
return OMPI_SUCCESS;
}

6
ompi/mca/mtl/mxm/mtl_mxm.h
Просмотреть файл

@ -37,12 +37,10 @@ BEGIN_C_DECLS
/* MTL interface functions */
extern int ompi_mtl_mxm_add_procs(struct mca_mtl_base_module_t* mtl,
size_t nprocs, struct ompi_proc_t** procs,
struct mca_mtl_base_endpoint_t **mtl_peer_data);
size_t nprocs, struct ompi_proc_t** procs);
extern int ompi_mtl_mxm_del_procs(struct mca_mtl_base_module_t* mtl,
size_t nprocs, struct ompi_proc_t** procs,
struct mca_mtl_base_endpoint_t **mtl_peer_data);
size_t nprocs, struct ompi_proc_t** procs);
extern int ompi_mtl_mxm_send(struct mca_mtl_base_module_t* mtl,
struct ompi_communicator_t* comm, int dest, int tag,

7
ompi/mca/mtl/mxm/mtl_mxm_endpoint.h
Просмотреть файл

@ -19,12 +19,12 @@ OBJ_CLASS_DECLARATION(mca_mtl_mxm_endpoint_t);
/**
* An abstraction that represents a connection to a endpoint process.
* An instance of mca_mtl_base_endpoint_t is associated w/ each process
* An instance of mca_mtl_mxm_endpoint_t is associated w/ each process
* and MTL pair at startup. However, connections to the endpoint
* are established dynamically on an as-needed basis:
*/
struct mca_mtl_base_endpoint_t {
struct mca_mtl_mxm_endpoint_t {
opal_list_item_t super;
struct mca_mtl_mxm_module_t* mtl_mxm_module;
@ -34,8 +34,7 @@ struct mca_mtl_base_endpoint_t {
/**< MXM Connection handle*/
};
typedef struct mca_mtl_base_endpoint_t mca_mtl_base_endpoint_t;
typedef mca_mtl_base_endpoint_t mca_mtl_mxm_endpoint_t;
typedef struct mca_mtl_mxm_endpoint_t mca_mtl_mxm_endpoint_t;
OBJ_CLASS_DECLARATION(mca_mtl_mxm_endpoint);
END_C_DECLS

2
ompi/mca/mtl/mxm/mtl_mxm_types.h
Просмотреть файл

@ -64,7 +64,7 @@ OMPI_DECLSPEC mca_mtl_mxm_component_t mca_mtl_mxm_component;
static inline mxm_conn_h ompi_mtl_mxm_conn_lookup(struct ompi_communicator_t* comm, int rank) {
ompi_proc_t* ompi_proc = ompi_comm_peer_lookup(comm, rank);
mca_mtl_mxm_endpoint_t *endpoint = (mca_mtl_mxm_endpoint_t*) ompi_proc->proc_pml;
mca_mtl_mxm_endpoint_t *endpoint = (mca_mtl_mxm_endpoint_t*) ompi_proc->proc_endpoints[OMPI_PROC_ENDPOINT_TAG_MTL];
return endpoint->mxm_conn;
}

1
ompi/mca/mtl/portals4/Makefile.am
Просмотреть файл

@ -37,7 +37,6 @@ local_sources = \
mtl_portals4.h \
mtl_portals4_cancel.c \
mtl_portals4_component.c \
mtl_portals4_endpoint.h \
mtl_portals4_message.c \
mtl_portals4_message.h \
mtl_portals4_probe.c \

9
ompi/mca/mtl/portals4/configure.m4
Просмотреть файл

@ -11,7 +11,7 @@
# Copyright (c) 2004-2005 The Regents of the University of California.
# All rights reserved.
# Copyright (c) 2010 Cisco Systems, Inc. All rights reserved.
# Copyright (c) 2010 Sandia National Laboratories. All rights reserved.
# Copyright (c) 2013 Sandia National Laboratories. All rights reserved.
# $COPYRIGHT$
#
# Additional copyrights may follow
@ -19,6 +19,13 @@
# $HEADER$
#
# MCA_ompi_mtl_portals4_POST_CONFIG(will_build)
# ----------------------------------------
# Only require the tag if we're actually going to be built
AC_DEFUN([MCA_ompi_mtl_portals4_POST_CONFIG], [
AS_IF([test "$1" = "1"], [OMPI_REQUIRE_ENDPOINT_TAG([PORTALS4])])
])dnl
# MCA_mtl_portals4_CONFIG(action-if-can-compile,
# [action-if-cant-compile])
# ------------------------------------------------

50
ompi/mca/mtl/portals4/mtl_portals4.c
Просмотреть файл

@ -27,7 +27,6 @@
#include "ompi/runtime/ompi_module_exchange.h"
#include "mtl_portals4.h"
#include "mtl_portals4_endpoint.h"
#include "mtl_portals4_recv_short.h"
extern mca_mtl_base_component_2_0_0_t mca_mtl_portals4_component;
@ -60,15 +59,14 @@ mca_mtl_portals4_module_t ompi_mtl_portals4 = {
int
ompi_mtl_portals4_add_procs(struct mca_mtl_base_module_t *mtl,
size_t nprocs,
struct ompi_proc_t** procs,
struct mca_mtl_base_endpoint_t **mtl_peer_data)
struct ompi_proc_t** procs)
{
int ret, me;
size_t i;
/* Get the list of ptl_process_id_t from the runtime and copy into structure */
for (i = 0 ; i < nprocs ; ++i) {
ptl_process_t *id;
ptl_process_t *modex_id;
size_t size;
if( procs[i] == ompi_proc_local_proc ) {
@ -85,16 +83,8 @@ ompi_mtl_portals4_add_procs(struct mca_mtl_base_module_t *mtl,
return OMPI_ERR_NOT_SUPPORTED;
}
mtl_peer_data[i] = malloc(sizeof(struct mca_mtl_base_endpoint_t));
if (NULL == mtl_peer_data[i]) {
opal_output_verbose(1, ompi_mtl_base_framework.framework_output,
"%s:%d: malloc failed: %d\n",
__FILE__, __LINE__, ret);
return OMPI_ERR_OUT_OF_RESOURCE;
}
ret = ompi_modex_recv(&mca_mtl_portals4_component.mtl_version,
procs[i], (void**) &id, &size);
procs[i], (void**) &modex_id, &size);
if (OMPI_SUCCESS != ret) {
opal_output_verbose(1, ompi_mtl_base_framework.framework_output,
"%s:%d: ompi_modex_recv failed: %d\n",
@ -106,12 +96,32 @@ ompi_mtl_portals4_add_procs(struct mca_mtl_base_module_t *mtl,
__FILE__, __LINE__, ret);
return OMPI_ERR_BAD_PARAM;
}
mtl_peer_data[i]->ptl_proc = *id;
if (NULL == procs[i]->proc_endpoints[OMPI_PROC_ENDPOINT_TAG_PORTALS4]) {
ptl_process_t *peer_id;
peer_id = malloc(sizeof(ptl_process_t));
if (NULL == peer_id) {
opal_output_verbose(1, ompi_mtl_base_framework.framework_output,
"%s:%d: malloc failed: %d\n",
__FILE__, __LINE__, ret);
return OMPI_ERR_OUT_OF_RESOURCE;
}
*peer_id = *modex_id;
procs[i]->proc_endpoints[OMPI_PROC_ENDPOINT_TAG_PORTALS4] = peer_id;
} else {
ptl_process_t *proc = (ptl_process_t*) procs[i]->proc_endpoints[OMPI_PROC_ENDPOINT_TAG_PORTALS4];
if (proc->phys.nid != modex_id->phys.nid ||
proc->phys.pid != modex_id->phys.pid) {
opal_output_verbose(1, ompi_mtl_base_framework.framework_output,
"%s:%d: existing peer and modex peer don't match\n",
__FILE__, __LINE__);
return OMPI_ERROR;
}
}
}
#if OMPI_MTL_PORTALS4_FLOW_CONTROL
ret = ompi_mtl_portals4_flowctl_add_procs(me, nprocs, mtl_peer_data);
ret = ompi_mtl_portals4_flowctl_add_procs(me, nprocs, procs);
if (OMPI_SUCCESS != ret) {
opal_output_verbose(1, ompi_mtl_base_framework.framework_output,
"%s:%d: flowctl_add_procs failed: %d\n",
@ -127,14 +137,14 @@ ompi_mtl_portals4_add_procs(struct mca_mtl_base_module_t *mtl,
int
ompi_mtl_portals4_del_procs(struct mca_mtl_base_module_t *mtl,
size_t nprocs,
struct ompi_proc_t** procs,
struct mca_mtl_base_endpoint_t **mtl_peer_data)
struct ompi_proc_t** procs)
{
size_t i;
for (i = 0 ; i < nprocs ; ++i) {
if (NULL != mtl_peer_data[i]) {
free(mtl_peer_data[i]);
if (NULL != procs[i]->proc_endpoints[OMPI_PROC_ENDPOINT_TAG_PORTALS4]) {
free(procs[i]->proc_endpoints[OMPI_PROC_ENDPOINT_TAG_PORTALS4]);
procs[i]->proc_endpoints[OMPI_PROC_ENDPOINT_TAG_PORTALS4] = NULL;
}
}

6
ompi/mca/mtl/portals4/mtl_portals4.h
Просмотреть файл

@ -269,13 +269,11 @@ extern int ompi_mtl_portals4_finalize(struct mca_mtl_base_module_t *mtl);
extern int ompi_mtl_portals4_add_procs(struct mca_mtl_base_module_t* mtl,
size_t nprocs,
struct ompi_proc_t** procs,
struct mca_mtl_base_endpoint_t **mtl_peer_data);
struct ompi_proc_t** procs);
extern int ompi_mtl_portals4_del_procs(struct mca_mtl_base_module_t* mtl,
size_t nprocs,
struct ompi_proc_t** procs,
struct mca_mtl_base_endpoint_t **mtl_peer_data);
struct ompi_proc_t** procs);
extern int ompi_mtl_portals4_send(struct mca_mtl_base_module_t* mtl,
struct ompi_communicator_t* comm,

16
ompi/mca/mtl/portals4/mtl_portals4_flowctl.c
Просмотреть файл

@ -9,9 +9,10 @@
#include "ompi_config.h"
#include "ompi/proc/proc.h"
#include "mtl_portals4.h"
#include "mtl_portals4_flowctl.h"
#include "mtl_portals4_endpoint.h"
#include "mtl_portals4_recv_short.h"
OBJ_CLASS_INSTANCE(ompi_mtl_portals4_pending_request_t, opal_free_list_item_t,
@ -227,7 +228,7 @@ ompi_mtl_portals4_flowctl_fini(void)
int
ompi_mtl_portals4_flowctl_add_procs(size_t me,
size_t npeers,
struct mca_mtl_base_endpoint_t **peers)
struct ompi_proc_t **procs)
{
int i;
@ -239,21 +240,24 @@ ompi_mtl_portals4_flowctl_add_procs(size_t me,
}
ompi_mtl_portals4.flowctl.num_procs = npeers;
ompi_mtl_portals4.flowctl.root = peers[0]->ptl_proc;
ompi_mtl_portals4.flowctl.root =
*((ptl_process_t*) procs[0]->proc_endpoints[OMPI_PROC_ENDPOINT_TAG_PORTALS4]);
if (0 == me) {
ompi_mtl_portals4.flowctl.i_am_root = true;
} else {
ompi_mtl_portals4.flowctl.i_am_root = false;
ompi_mtl_portals4.flowctl.parent =
peers[(me - 1) / 2]->ptl_proc;
*((ptl_process_t*) procs[(me - 1) / 2]->proc_endpoints[OMPI_PROC_ENDPOINT_TAG_PORTALS4]);
}
ompi_mtl_portals4.flowctl.me = peers[me]->ptl_proc;
ompi_mtl_portals4.flowctl.me =
*((ptl_process_t*) procs[me]->proc_endpoints[OMPI_PROC_ENDPOINT_TAG_PORTALS4]);
for (i = 0 ; i < 2 ; ++i) {
size_t tmp = (2 * me) + i + 1;
if (tmp < npeers) {
ompi_mtl_portals4.flowctl.num_children++;
ompi_mtl_portals4.flowctl.children[i] = peers[tmp]->ptl_proc;
ompi_mtl_portals4.flowctl.children[i] =
*((ptl_process_t*) procs[tmp]->proc_endpoints[OMPI_PROC_ENDPOINT_TAG_PORTALS4]);
}
}

4
ompi/mca/mtl/portals4/mtl_portals4_flowctl.h
Просмотреть файл

@ -26,7 +26,7 @@ struct ompi_mtl_portals4_pending_request_t {
int tag;
int my_rank;
int fc_notified;
struct mca_mtl_base_endpoint_t *endpoint;
ptl_process_t *proc;
struct ompi_mtl_portals4_isend_request_t *ptl_request;
};
typedef struct ompi_mtl_portals4_pending_request_t ompi_mtl_portals4_pending_request_t;
@ -91,7 +91,7 @@ int ompi_mtl_portals4_flowctl_fini(void);
int ompi_mtl_portals4_flowctl_add_procs(size_t me,
size_t npeers,
struct mca_mtl_base_endpoint_t **peers);
struct ompi_proc_t **procs);
int ompi_mtl_portals4_flowctl_trigger(void);

9
ompi/mca/mtl/portals4/mtl_portals4_probe.c
Просмотреть файл

@ -23,7 +23,6 @@
#include "mtl_portals4.h"
#include "mtl_portals4_request.h"
#include "mtl_portals4_endpoint.h"
#include "mtl_portals4_message.h"
static int
@ -63,7 +62,6 @@ ompi_mtl_portals4_iprobe(struct mca_mtl_base_module_t* mtl,
struct ompi_status_public_t *status)
{
struct ompi_mtl_portals4_probe_request_t request;
mca_mtl_base_endpoint_t *endpoint = NULL;
ptl_me_t me;
ptl_process_t remote_proc;
ptl_match_bits_t match_bits, ignore_bits;
@ -74,8 +72,7 @@ ompi_mtl_portals4_iprobe(struct mca_mtl_base_module_t* mtl,
remote_proc.phys.pid = PTL_PID_ANY;
} else {
ompi_proc_t* ompi_proc = ompi_comm_peer_lookup( comm, src );
endpoint = (mca_mtl_base_endpoint_t*) ompi_proc->proc_pml;
remote_proc = endpoint->ptl_proc;
remote_proc = *((ptl_process_t*) ompi_proc->proc_endpoints[OMPI_PROC_ENDPOINT_TAG_PORTALS4]);
}
MTL_PORTALS4_SET_RECV_BITS(match_bits, ignore_bits, comm->c_contextid,
@ -133,7 +130,6 @@ ompi_mtl_portals4_improbe(struct mca_mtl_base_module_t *mtl,
struct ompi_status_public_t *status)
{
struct ompi_mtl_portals4_probe_request_t request;
mca_mtl_base_endpoint_t *endpoint = NULL;
ptl_me_t me;
ptl_process_t remote_proc;
ptl_match_bits_t match_bits, ignore_bits;
@ -148,8 +144,7 @@ ompi_mtl_portals4_improbe(struct mca_mtl_base_module_t *mtl,
remote_proc.phys.pid = PTL_PID_ANY;
} else {
ompi_proc_t* ompi_proc = ompi_comm_peer_lookup( comm, src );
endpoint = (mca_mtl_base_endpoint_t*) ompi_proc->proc_pml;
remote_proc = endpoint->ptl_proc;
remote_proc = *((ptl_process_t*) ompi_proc->proc_endpoints[OMPI_PROC_ENDPOINT_TAG_PORTALS4]);
}
MTL_PORTALS4_SET_RECV_BITS(match_bits, ignore_bits, comm->c_contextid,

5
ompi/mca/mtl/portals4/mtl_portals4_recv.c
Просмотреть файл

@ -29,7 +29,6 @@
#include "ompi/message/message.h"
#include "mtl_portals4.h"
#include "mtl_portals4_endpoint.h"
#include "mtl_portals4_request.h"
#include "mtl_portals4_recv_short.h"
#include "mtl_portals4_message.h"
@ -335,7 +334,6 @@ ompi_mtl_portals4_irecv(struct mca_mtl_base_module_t* mtl,
ptl_match_bits_t match_bits, ignore_bits;
int ret = OMPI_SUCCESS;
ptl_process_t remote_proc;
mca_mtl_base_endpoint_t *endpoint = NULL;
ompi_mtl_portals4_recv_request_t *ptl_request =
(ompi_mtl_portals4_recv_request_t*) mtl_request;
void *start;
@ -348,8 +346,7 @@ ompi_mtl_portals4_irecv(struct mca_mtl_base_module_t* mtl,
remote_proc.phys.pid = PTL_PID_ANY;
} else {
ompi_proc_t* ompi_proc = ompi_comm_peer_lookup( comm, src );
endpoint = (mca_mtl_base_endpoint_t*) ompi_proc->proc_pml;
remote_proc = endpoint->ptl_proc;
remote_proc = *((ptl_process_t*) ompi_proc->proc_endpoints[OMPI_PROC_ENDPOINT_TAG_PORTALS4]);
}
MTL_PORTALS4_SET_RECV_BITS(match_bits, ignore_bits, comm->c_contextid,

34
ompi/mca/mtl/portals4/mtl_portals4_send.c
Просмотреть файл

@ -26,7 +26,6 @@
#include "mtl_portals4.h"
#include "mtl_portals4_request.h"
#include "mtl_portals4_endpoint.h"
#if OMPI_MTL_PORTALS4_FLOW_CONTROL
#include "mtl_portals4_flowctl.h"
#endif
@ -175,7 +174,7 @@ static inline int
ompi_mtl_portals4_short_isend(mca_pml_base_send_mode_t mode,
void *start, int length, int contextid, int tag,
int localrank,
mca_mtl_base_endpoint_t *endpoint,
ptl_process_t *proc,
ompi_mtl_portals4_isend_request_t *ptl_request)
{
int ret;
@ -202,7 +201,7 @@ ompi_mtl_portals4_short_isend(mca_pml_base_send_mode_t mode,
PTL_ME_USE_ONCE |
PTL_ME_EVENT_LINK_DISABLE |
PTL_ME_EVENT_UNLINK_DISABLE;
me.match_id = endpoint->ptl_proc;
me.match_id = *proc;
me.match_bits = hdr_data;
me.ignore_bits = 0;
@ -242,7 +241,7 @@ ompi_mtl_portals4_short_isend(mca_pml_base_send_mode_t mode,
(ptl_size_t) ((char*) start - (char*) base),
length,
PTL_ACK_REQ,
endpoint->ptl_proc,
*proc,
ompi_mtl_portals4.recv_idx,
match_bits,
0,
@ -264,7 +263,7 @@ ompi_mtl_portals4_short_isend(mca_pml_base_send_mode_t mode,
static inline int
ompi_mtl_portals4_long_isend(void *start, int length, int contextid, int tag,
int localrank,
mca_mtl_base_endpoint_t *endpoint,
ptl_process_t *proc,
ompi_mtl_portals4_isend_request_t *ptl_request)
{
int ret;
@ -290,7 +289,7 @@ ompi_mtl_portals4_long_isend(void *start, int length, int contextid, int tag,
PTL_ME_USE_ONCE |
PTL_ME_EVENT_LINK_DISABLE |
PTL_ME_EVENT_UNLINK_DISABLE;
me.match_id = endpoint->ptl_proc;
me.match_id = *proc;
me.match_bits = hdr_data;
me.ignore_bits = 0;
@ -320,7 +319,7 @@ ompi_mtl_portals4_long_isend(void *start, int length, int contextid, int tag,
(ptl_size_t) ((char*) start - (char*) base),
put_length,
PTL_ACK_REQ,
endpoint->ptl_proc,
*proc,
ompi_mtl_portals4.recv_idx,
match_bits,
0,
@ -368,7 +367,7 @@ ompi_mtl_portals4_pending_list_progress()
pending->contextid,
pending->tag,
pending->my_rank,
pending->endpoint,
pending->proc,
pending->ptl_request);
} else {
ret = ompi_mtl_portals4_long_isend(pending->start,
@ -376,7 +375,7 @@ ompi_mtl_portals4_pending_list_progress()
pending->contextid,
pending->tag,
pending->my_rank,
pending->endpoint,
pending->proc,
pending->ptl_request);
}
if (OPAL_UNLIKELY(OMPI_SUCCESS != ret)) {
@ -403,11 +402,10 @@ ompi_mtl_portals4_send_start(struct mca_mtl_base_module_t* mtl,
size_t length;
bool free_after;
ompi_proc_t *ompi_proc = ompi_comm_peer_lookup(comm, dest);
mca_mtl_base_endpoint_t *endpoint =
(mca_mtl_base_endpoint_t*) ompi_proc->proc_pml;
ptl_process_t *proc = (ptl_process_t*) ompi_proc->proc_endpoints[OMPI_PROC_ENDPOINT_TAG_PORTALS4];
#if OMPI_MTL_PORTALS4_FLOW_CONTROL
opal_free_list_item_t *item;
ompi_mtl_portals4_pending_request_t *pending;
opal_free_list_item_t *item;
ompi_mtl_portals4_pending_request_t *pending;
#endif
ret = ompi_mtl_datatype_pack(convertor, &start, &length, &free_after);
@ -420,8 +418,8 @@ ompi_mtl_portals4_send_start(struct mca_mtl_base_module_t* mtl,
OPAL_OUTPUT_VERBOSE((50, ompi_mtl_base_framework.framework_output,
"Send %lu to %x,%x of length %d\n",
ptl_request->opcount,
endpoint->ptl_proc.phys.nid,
endpoint->ptl_proc.phys.pid,
proc->phys.nid,
proc->phys.pid,
(int)length));
#if OMPI_MTL_PORTALS4_FLOW_CONTROL
@ -437,7 +435,7 @@ ompi_mtl_portals4_send_start(struct mca_mtl_base_module_t* mtl,
pending->tag = tag;
pending->my_rank = comm->c_my_rank;
pending->fc_notified = 0;
pending->endpoint = endpoint;
pending->proc = proc;
pending->ptl_request = ptl_request;
if (OPAL_THREAD_ADD32(&ompi_mtl_portals4.flowctl.send_slots, -1) < 0) {
@ -469,7 +467,7 @@ ompi_mtl_portals4_send_start(struct mca_mtl_base_module_t* mtl,
comm->c_contextid,
tag,
comm->c_my_rank,
endpoint,
proc,
ptl_request);
} else {
ret = ompi_mtl_portals4_long_isend(start,
@ -477,7 +475,7 @@ ompi_mtl_portals4_send_start(struct mca_mtl_base_module_t* mtl,
comm->c_contextid,
tag,
comm->c_my_rank,
endpoint,
proc,
ptl_request);
}

7
ompi/mca/mtl/psm/configure.m4
Просмотреть файл

@ -11,6 +11,7 @@
# Copyright (c) 2004-2005 The Regents of the University of California.
# All rights reserved.
# Copyright (c) 2010 Cisco Systems, Inc. All rights reserved.
# Copyright (c) 2013 Sandia National Laboratories. All rights reserved.
# $COPYRIGHT$
#
# Additional copyrights may follow
@ -18,6 +19,12 @@
# $HEADER$
#
# MCA_ompi_mtl_psm_POST_CONFIG(will_build)
# ----------------------------------------
# Only require the tag if we're actually going to be built
AC_DEFUN([MCA_ompi_mtl_psm_POST_CONFIG], [
AS_IF([test "$1" = "1"], [OMPI_REQUIRE_ENDPOINT_TAG([MTL])])
])dnl
# MCA_mtl_psm_CONFIG([action-if-can-compile],
# [action-if-cant-compile])

13
ompi/mca/mtl/psm/mtl_psm.c
Просмотреть файл

@ -248,8 +248,7 @@ ompi_mtl_psm_connect_error_msg(psm_error_t err)
int
ompi_mtl_psm_add_procs(struct mca_mtl_base_module_t *mtl,
size_t nprocs,
struct ompi_proc_t** procs, /*const*/
struct mca_mtl_base_endpoint_t **mtl_peer_data)
struct ompi_proc_t** procs)
{
int i,j;
int rc;
@ -332,10 +331,11 @@ ompi_mtl_psm_add_procs(struct mca_mtl_base_module_t *mtl,
/* Fill in endpoint data */
for (i = 0; i < (int) nprocs; i++) {
mtl_peer_data[i] =
mca_mtl_psm_endpoint_t *endpoint =
(mca_mtl_psm_endpoint_t *) OBJ_NEW(mca_mtl_psm_endpoint_t);
mtl_peer_data[i]->peer_epid = epids_in[i];
mtl_peer_data[i]->peer_addr = epaddrs_out[i];
endpoint->peer_epid = epids_in[i];
endpoint->peer_addr = epaddrs_out[i];
procs[i]->proc_endpoints[OMPI_PROC_ENDPOINT_TAG_MTL] = endpoint;
}
rc = OMPI_SUCCESS;
@ -358,8 +358,7 @@ bail:
int
ompi_mtl_psm_del_procs(struct mca_mtl_base_module_t *mtl,
size_t nprocs,
struct ompi_proc_t** procs,
struct mca_mtl_base_endpoint_t **mtl_peer_data)
struct ompi_proc_t** procs)
{
return OMPI_SUCCESS;
}

6
ompi/mca/mtl/psm/mtl_psm.h
Просмотреть файл

@ -33,13 +33,11 @@ BEGIN_C_DECLS
/* MTL interface functions */
extern int ompi_mtl_psm_add_procs(struct mca_mtl_base_module_t* mtl,
size_t nprocs,
struct ompi_proc_t** procs,
struct mca_mtl_base_endpoint_t **mtl_peer_data);
struct ompi_proc_t** procs);
extern int ompi_mtl_psm_del_procs(struct mca_mtl_base_module_t* mtl,
size_t nprocs,
struct ompi_proc_t** procs,
struct mca_mtl_base_endpoint_t **mtl_peer_data);
struct ompi_proc_t** procs);
int
ompi_mtl_psm_send(struct mca_mtl_base_module_t* mtl,

7
ompi/mca/mtl/psm/mtl_psm_endpoint.h
Просмотреть файл

@ -33,12 +33,12 @@ OBJ_CLASS_DECLARATION(mca_mtl_psm_endpoint_t);
/**
* An abstraction that represents a connection to a endpoint process.
* An instance of mca_mtl_base_endpoint_t is associated w/ each process
* An instance of mca_mtl_psm_endpoint_t is associated w/ each process
* and MTL pair at startup. However, connections to the endpoint
* are established dynamically on an as-needed basis:
*/
struct mca_mtl_base_endpoint_t {
struct mca_mtl_psm_endpoint_t {
opal_list_item_t super;
struct mca_mtl_psm_module_t* mtl_psm_module;
@ -51,8 +51,7 @@ struct mca_mtl_base_endpoint_t {
/**< The connected endpoint handle*/
};
typedef struct mca_mtl_base_endpoint_t mca_mtl_base_endpoint_t;
typedef mca_mtl_base_endpoint_t mca_mtl_psm_endpoint_t;
typedef struct mca_mtl_psm_endpoint_t mca_mtl_psm_endpoint_t;
OBJ_CLASS_DECLARATION(mca_mtl_psm_endpoint);
END_C_DECLS

4
ompi/mca/mtl/psm/mtl_psm_send.c
Просмотреть файл

@ -42,7 +42,7 @@ ompi_mtl_psm_send(struct mca_mtl_base_module_t* mtl,
int ret;
size_t length;
ompi_proc_t* ompi_proc = ompi_comm_peer_lookup( comm, dest );
mca_mtl_psm_endpoint_t* psm_endpoint = (mca_mtl_psm_endpoint_t*) ompi_proc->proc_pml;
mca_mtl_psm_endpoint_t* psm_endpoint = (mca_mtl_psm_endpoint_t*) ompi_proc->proc_endpoints[OMPI_PROC_ENDPOINT_TAG_MTL];
assert(mtl == &ompi_mtl_psm.super);
@ -94,7 +94,7 @@ ompi_mtl_psm_isend(struct mca_mtl_base_module_t* mtl,
mca_mtl_psm_request_t * mtl_psm_request = (mca_mtl_psm_request_t*) mtl_request;
size_t length;
ompi_proc_t* ompi_proc = ompi_comm_peer_lookup( comm, dest );
mca_mtl_psm_endpoint_t* psm_endpoint = (mca_mtl_psm_endpoint_t*)ompi_proc->proc_pml;
mca_mtl_psm_endpoint_t* psm_endpoint = (mca_mtl_psm_endpoint_t*)ompi_proc->proc_endpoints[OMPI_PROC_ENDPOINT_TAG_MTL];
assert(mtl == &ompi_mtl_psm.super);

26
ompi/mca/osc/rdma/configure.m4 Обычный файл
Просмотреть файл

@ -0,0 +1,26 @@
# -*- shell-script -*-
#
# Copyright (c) 2013 Sandia National Laboratories. All rights reserved.
# $COPYRIGHT$
#
# Additional copyrights may follow
#
# $HEADER$
#
# MCA_ompi_osc_rdma_POST_CONFIG(will_build)
# ----------------------------------------
# Only require the tag if we're actually going to be built, since bml
# is one of the ones frequently disabled for large installs.
AC_DEFUN([MCA_ompi_osc_rdma_POST_CONFIG], [
AS_IF([test "$1" = "1"], [OMPI_REQUIRE_ENDPOINT_TAG([BML])])
])dnl
# MCA_ompi_osc_rdma_CONFIG(action-if-can-compile,
# [action-if-cant-compile])
# ------------------------------------------------
# We can always build, unless we were explicitly disabled.
AC_DEFUN([MCA_ompi_osc_rdma_CONFIG],[
AC_CONFIG_FILES([ompi/mca/osc/rdma/Makefile])
[$1]
])dnl

4
ompi/mca/osc/rdma/osc_rdma_component.c
Просмотреть файл

@ -863,7 +863,7 @@ component_fragment_cb(struct mca_btl_base_module_t *btl,
/* find the bml_btl */
proc = ompi_comm_peer_lookup(module->m_comm, origin);
endpoint = (mca_bml_base_endpoint_t*) proc->proc_bml;
endpoint = (mca_bml_base_endpoint_t*) proc->proc_endpoints[OMPI_PROC_ENDPOINT_TAG_BML];
bml_btl = mca_bml_base_btl_array_find(&endpoint->btl_rdma, btl);
if (NULL == bml_btl) {
opal_output(ompi_osc_base_framework.framework_output,
@ -1173,7 +1173,7 @@ setup_rdma(ompi_osc_rdma_module_t *module)
ompi_proc_t *proc = ompi_comm_peer_lookup(module->m_comm, i);
ompi_osc_rdma_peer_info_t *peer_info = &module->m_peer_info[i];
mca_bml_base_endpoint_t *endpoint =
(mca_bml_base_endpoint_t*) proc->proc_bml;
(mca_bml_base_endpoint_t*) proc->proc_endpoints[OMPI_PROC_ENDPOINT_TAG_BML];
int num_avail =
mca_bml_base_btl_array_get_size(&endpoint->btl_rdma);
size_t j, size;

6
ompi/mca/osc/rdma/osc_rdma_data_move.c
Просмотреть файл

@ -463,7 +463,7 @@ ompi_osc_rdma_sendreq_send(ompi_osc_rdma_module_t *module,
}
/* get a buffer... */
endpoint = (mca_bml_base_endpoint_t*) sendreq->req_target_proc->proc_bml;
endpoint = (mca_bml_base_endpoint_t*) sendreq->req_target_proc->proc_endpoints[OMPI_PROC_ENDPOINT_TAG_BML];
bml_btl = mca_bml_base_btl_array_get_next(&endpoint->btl_eager);
btl = bml_btl->btl;
mca_bml_base_alloc(bml_btl, &descriptor, MCA_BTL_NO_ORDER,
@ -727,7 +727,7 @@ ompi_osc_rdma_replyreq_send(ompi_osc_rdma_module_t *module,
size_t written_data = 0;
/* Get a BTL and a fragment to go with it */
endpoint = (mca_bml_base_endpoint_t*) replyreq->rep_origin_proc->proc_bml;
endpoint = (mca_bml_base_endpoint_t*) replyreq->rep_origin_proc->proc_endpoints[OMPI_PROC_ENDPOINT_TAG_BML];
bml_btl = mca_bml_base_btl_array_get_next(&endpoint->btl_eager);
mca_bml_base_alloc(bml_btl, &descriptor, MCA_BTL_NO_ORDER,
bml_btl->btl->btl_eager_limit, MCA_BTL_DES_FLAGS_PRIORITY | MCA_BTL_DES_SEND_ALWAYS_CALLBACK);
@ -1326,7 +1326,7 @@ ompi_osc_rdma_control_send(ompi_osc_rdma_module_t *module,
ompi_osc_rdma_control_header_t *header = NULL;
/* Get a BTL and a fragment to go with it */
endpoint = (mca_bml_base_endpoint_t*) proc->proc_bml;
endpoint = (mca_bml_base_endpoint_t*) proc->proc_endpoints[OMPI_PROC_ENDPOINT_TAG_BML];
bml_btl = mca_bml_base_btl_array_get_next(&endpoint->btl_eager);
mca_bml_base_alloc(bml_btl, &descriptor, MCA_BTL_NO_ORDER,
sizeof(ompi_osc_rdma_control_header_t),

27
ompi/mca/pml/bfo/configure.m4 Обычный файл
Просмотреть файл

@ -0,0 +1,27 @@
# -*- shell-script -*-
#
# Copyright (c) 2013 Sandia National Laboratories. All rights reserved.
# $COPYRIGHT$
#
# Additional copyrights may follow
#
# $HEADER$
#
# MCA_ompi_pml_bfo_POST_CONFIG(will_build)
# ----------------------------------------
# The BFO PML requires a BML endpoint tag to compile, so require it.
# Require in POST_CONFIG instead of CONFIG so that we only require it
# if we're not disabled.
AC_DEFUN([MCA_ompi_pml_bfo_POST_CONFIG], [
AS_IF([test "$1" = "1"], [OMPI_REQUIRE_ENDPOINT_TAG([BML])])
])dnl
# MCA_ompi_pml_bfo_CONFIG(action-if-can-compile,
# [action-if-cant-compile])
# ------------------------------------------------
# We can always build, unless we were explicitly disabled.
AC_DEFUN([MCA_ompi_pml_bfo_CONFIG],[
AC_CONFIG_FILES([ompi/mca/pml/bfo/Makefile])
[$1]
])dnl

13
ompi/mca/pml/bfo/pml_bfo.c
Просмотреть файл

@ -301,18 +301,11 @@ int mca_pml_bfo_add_procs(ompi_proc_t** procs, size_t nprocs)
{
opal_bitmap_t reachable;
int rc;
size_t i;
opal_list_item_t *item;
if(nprocs == 0)
return OMPI_SUCCESS;
/* we don't have any endpoint data we need to cache on the
ompi_proc_t, so set proc_pml to NULL */
for (i = 0 ; i < nprocs ; ++i) {
procs[i]->proc_pml = NULL;
}
OBJ_CONSTRUCT(&reachable, opal_bitmap_t);
rc = opal_bitmap_init(&reachable, (int)nprocs);
if(OMPI_SUCCESS != rc)
@ -452,7 +445,7 @@ int mca_pml_bfo_dump(struct ompi_communicator_t* comm, int verbose)
/* iterate through all procs on communicator */
for( i = 0; i < (int)pml_comm->num_procs; i++ ) {
mca_pml_bfo_comm_proc_t* proc = &pml_comm->procs[i];
mca_bml_base_endpoint_t* ep = (mca_bml_base_endpoint_t*)proc->ompi_proc->proc_bml;
mca_bml_base_endpoint_t* ep = (mca_bml_base_endpoint_t*)proc->ompi_proc->proc_endpoints[OMPI_PROC_ENDPOINT_TAG_BML];
size_t n;
opal_output(0, "[Rank %d]\n", i);
@ -567,8 +560,10 @@ void mca_pml_bfo_process_pending_packets(mca_bml_base_btl_t* bml_btl)
pckt->bml_btl->btl == bml_btl->btl) {
send_dst = pckt->bml_btl;
} else {
mca_bml_base_endpoint_t* endpoint =
(mca_bml_base_endpoint_t*) pckt->proc->proc_endpoints[OMPI_PROC_ENDPOINT_TAG_BML];
send_dst = mca_bml_base_btl_array_find(
&pckt->proc->proc_bml->btl_eager, bml_btl->btl);
&endpoint->btl_eager, bml_btl->btl);
}
if(NULL == send_dst) {
OPAL_THREAD_LOCK(&mca_pml_bfo.lock);

24
ompi/mca/pml/bfo/pml_bfo_failover.c
Просмотреть файл

@ -281,7 +281,7 @@ void mca_pml_bfo_repost_fin(struct mca_btl_base_descriptor_t* des) {
mca_bml_base_btl_t* bml_btl;
proc = (ompi_proc_t*) des->des_cbdata;
bml_endpoint = (mca_bml_base_endpoint_t*) proc->proc_bml;
bml_endpoint = (mca_bml_base_endpoint_t*) proc->proc_endpoints[OMPI_PROC_ENDPOINT_TAG_BML];
hdr = (mca_pml_bfo_fin_hdr_t*)des->des_src->seg_addr.pval;
opal_output_verbose(20, mca_pml_bfo_output,
@ -665,7 +665,7 @@ void mca_pml_bfo_send_request_rndvrestartnotify(mca_pml_bfo_send_request_t* send
int rc;
mca_bml_base_btl_t* bml_btl;
ompi_proc_t* proc = (ompi_proc_t*)sendreq->req_send.req_base.req_proc;
mca_bml_base_endpoint_t* bml_endpoint = (mca_bml_base_endpoint_t*) proc->proc_bml;
mca_bml_base_endpoint_t* bml_endpoint = (mca_bml_base_endpoint_t*) proc->proc_endpoints[OMPI_PROC_ENDPOINT_TAG_BML];
/* If this message is not a repost, then update the sequence number. */
if (!repost) {
@ -811,7 +811,7 @@ void mca_pml_bfo_send_request_restart(mca_pml_bfo_send_request_t* sendreq,
/* This code here is essentially the same is mca_pml_bfo_send_request_start()
* but with a few modifications since we are restarting the request, not
* starting entirely from scratch. */
endpoint = (mca_bml_base_endpoint_t*)sendreq->req_send.req_base.req_proc->proc_bml;
endpoint = (mca_bml_base_endpoint_t*)sendreq->req_send.req_base.req_proc->proc_endpoints[OMPI_PROC_ENDPOINT_TAG_BML];
sendreq->req_endpoint = endpoint;
sendreq->req_state = 0;
sendreq->req_lock = 0;
@ -1040,7 +1040,7 @@ void mca_pml_bfo_recv_request_recverrnotify(mca_pml_bfo_recv_request_t* recvreq,
mca_btl_base_descriptor_t* des;
mca_pml_bfo_restart_hdr_t* restart;
ompi_proc_t* proc = (ompi_proc_t*)recvreq->req_recv.req_base.req_proc;
mca_bml_base_endpoint_t* bml_endpoint = (mca_bml_base_endpoint_t*) proc->proc_bml;
mca_bml_base_endpoint_t* bml_endpoint = (mca_bml_base_endpoint_t*) proc->proc_endpoints[OMPI_PROC_ENDPOINT_TAG_BML];
mca_bml_base_btl_t* bml_btl;
int rc;
@ -1112,7 +1112,7 @@ void mca_pml_bfo_recv_request_rndvrestartack(mca_pml_bfo_recv_request_t* recvreq
mca_btl_base_descriptor_t* des;
mca_pml_bfo_restart_hdr_t* restart;
ompi_proc_t* proc = (ompi_proc_t*)recvreq->req_recv.req_base.req_proc;
mca_bml_base_endpoint_t* bml_endpoint = (mca_bml_base_endpoint_t*) proc->proc_bml;
mca_bml_base_endpoint_t* bml_endpoint = (mca_bml_base_endpoint_t*) proc->proc_endpoints[OMPI_PROC_ENDPOINT_TAG_BML];
mca_bml_base_btl_t* bml_btl;
int rc;
@ -1210,7 +1210,7 @@ void mca_pml_bfo_recv_request_rndvrestartnack(mca_btl_base_descriptor_t* olddes,
}
hdr = (mca_pml_bfo_restart_hdr_t*)segments->seg_addr.pval;
bml_endpoint = ompi_proc->proc_bml;
bml_endpoint = ompi_proc->proc_endpoints[OMPI_PROC_ENDPOINT_TAG_BML];
assert(0 != bml_endpoint->btl_eager.arr_size);
bml_btl = mca_bml_base_btl_array_get_next(&bml_endpoint->btl_eager);
@ -1383,7 +1383,7 @@ void mca_pml_bfo_map_out_btl(struct mca_btl_base_module_t* btl,
bool remove = false;
int i;
ep = (mca_bml_base_endpoint_t*)errproc->proc_bml;
ep = (mca_bml_base_endpoint_t*)errproc->proc_endpoints[OMPI_PROC_ENDPOINT_TAG_BML];
/* The bml_del_proc_btl function does not indicate if it
* actually removed a btl, so let me check up front. This is
@ -1539,7 +1539,7 @@ static void mca_pml_bfo_error_pending_packets(mca_btl_base_module_t* btl,
* not pointing at a stale bml. We do not really care
* which BML it is pointing at as long as it is valid.
* In either case, then put entry back on the list. */
if (pckt->proc->proc_bml == ep) {
if (pckt->proc->proc_endpoints[OMPI_PROC_ENDPOINT_TAG_BML] == ep) {
opal_output_verbose(15, mca_pml_bfo_output,
"INFO: Found matching pckt on pckt_pending list, adjusting bml");
pckt->bml_btl = mca_bml_base_btl_array_get_next(&ep->btl_eager);
@ -1667,7 +1667,7 @@ static void mca_pml_bfo_error_pending_packets(mca_btl_base_module_t* btl,
break;
proc = (ompi_proc_t*)sendreq->req_send.req_base.req_proc;
bml_endpoint = (mca_bml_base_endpoint_t*) proc->proc_bml;
bml_endpoint = (mca_bml_base_endpoint_t*) proc->proc_endpoints[OMPI_PROC_ENDPOINT_TAG_BML];
/* Check to see if it matches our endpoint. If it does not,
* then just put it back on the list as there is nothing
@ -1734,7 +1734,7 @@ static void mca_pml_bfo_error_pending_packets(mca_btl_base_module_t* btl,
break;
proc = (ompi_proc_t*)recvreq->req_recv.req_base.req_proc;
bml_endpoint = (mca_bml_base_endpoint_t*) proc->proc_bml;
bml_endpoint = (mca_bml_base_endpoint_t*) proc->proc_endpoints[OMPI_PROC_ENDPOINT_TAG_BML];
if (bml_endpoint != ep) {
OPAL_THREAD_LOCK(&mca_pml_bfo.lock);
@ -2033,7 +2033,7 @@ void mca_pml_bfo_find_recvreq_eager_bml_btl(mca_bml_base_btl_t** bml_btl,
{
if ((*bml_btl)->btl != btl) {
ompi_proc_t *proc = (ompi_proc_t*)recvreq->req_recv.req_base.req_proc;
mca_bml_base_endpoint_t* bml_endpoint = (mca_bml_base_endpoint_t*) proc->proc_bml;
mca_bml_base_endpoint_t* bml_endpoint = (mca_bml_base_endpoint_t*) proc->proc_endpoints[OMPI_PROC_ENDPOINT_TAG_BML];
opal_output_verbose(25, mca_pml_bfo_output,
"%s completion: BML does not match BTL, find it back, "
@ -2063,7 +2063,7 @@ void mca_pml_bfo_find_recvreq_rdma_bml_btl(mca_bml_base_btl_t** bml_btl,
{
if ((*bml_btl)->btl != btl) {
ompi_proc_t *proc = (ompi_proc_t*)recvreq->req_recv.req_base.req_proc;
mca_bml_base_endpoint_t* bml_endpoint = (mca_bml_base_endpoint_t*) proc->proc_bml;
mca_bml_base_endpoint_t* bml_endpoint = (mca_bml_base_endpoint_t*) proc->proc_endpoints[OMPI_PROC_ENDPOINT_TAG_BML];
opal_output_verbose(25, mca_pml_bfo_output,
"%s completion: BML does not match BTL, find it back, "

2
ompi/mca/pml/bfo/pml_bfo_failover.h
Просмотреть файл

@ -334,7 +334,7 @@ extern void mca_pml_bfo_recv_frag_callback_recverrnotify( mca_btl_base_module_t
#define MCA_PML_BFO_CHECK_EAGER_BML_BTL_ON_FIN_COMPLETION(bml_btl, btl, des) \
if (bml_btl->btl != btl) { \
ompi_proc_t *proc = (ompi_proc_t*) des->des_cbdata; \
mca_bml_base_endpoint_t* bml_endpoint = (mca_bml_base_endpoint_t*) proc->proc_bml; \
mca_bml_base_endpoint_t* bml_endpoint = (mca_bml_base_endpoint_t*) proc->proc_endpoints[OMPI_PROC_ENDPOINT_TAG_BML]; \
bml_btl = mca_bml_base_btl_array_find(&bml_endpoint->btl_eager, btl); \
}
#define MCA_PML_BFO_CHECK_SENDREQ_EAGER_BML_BTL(bml_btl, btl, sendreq, type) \

4
ompi/mca/pml/bfo/pml_bfo_recvreq.c
Просмотреть файл

@ -283,7 +283,7 @@ static int mca_pml_bfo_recv_request_ack(
ompi_proc_t* proc = (ompi_proc_t*)recvreq->req_recv.req_base.req_proc;
mca_bml_base_endpoint_t* bml_endpoint = NULL;
bml_endpoint = (mca_bml_base_endpoint_t*) proc->proc_bml;
bml_endpoint = (mca_bml_base_endpoint_t*) proc->proc_endpoints[OMPI_PROC_ENDPOINT_TAG_BML];
/* by default copy everything */
recvreq->req_send_offset = bytes_received;
@ -559,7 +559,7 @@ void mca_pml_bfo_recv_request_progress_rget( mca_pml_bfo_recv_request_t* recvreq
}
/* lookup bml datastructures */
bml_endpoint = (mca_bml_base_endpoint_t*)recvreq->req_recv.req_base.req_proc->proc_bml;
bml_endpoint = (mca_bml_base_endpoint_t*)recvreq->req_recv.req_base.req_proc->proc_endpoints[OMPI_PROC_ENDPOINT_TAG_BML];
assert (btl->btl_seg_size * hdr->hdr_seg_cnt <= sizeof (frag->rdma_segs));

2
ompi/mca/pml/bfo/pml_bfo_recvreq.h
Просмотреть файл

@ -418,7 +418,7 @@ static inline int mca_pml_bfo_recv_request_ack_send(ompi_proc_t* proc,
size_t i;
mca_bml_base_btl_t* bml_btl;
mca_bml_base_endpoint_t* endpoint =
(mca_bml_base_endpoint_t*)proc->proc_bml;
(mca_bml_base_endpoint_t*)proc->proc_endpoints[OMPI_PROC_ENDPOINT_TAG_BML];
for(i = 0; i < mca_bml_base_btl_array_get_size(&endpoint->btl_eager); i++) {
bml_btl = mca_bml_base_btl_array_get_next(&endpoint->btl_eager);

2
ompi/mca/pml/bfo/pml_bfo_sendreq.h
Просмотреть файл

@ -427,7 +427,7 @@ mca_pml_bfo_send_request_start( mca_pml_bfo_send_request_t* sendreq )
{
mca_pml_bfo_comm_t* comm = sendreq->req_send.req_base.req_comm->c_pml_comm;
mca_bml_base_endpoint_t* endpoint = (mca_bml_base_endpoint_t*)
sendreq->req_send.req_base.req_proc->proc_bml;
sendreq->req_send.req_base.req_proc->proc_endpoints[OMPI_PROC_ENDPOINT_TAG_BML];
size_t i;
if( OPAL_UNLIKELY(endpoint == NULL) ) {

44
ompi/mca/pml/cm/pml_cm.c
Просмотреть файл

@ -112,8 +112,6 @@ int
mca_pml_cm_add_procs(struct ompi_proc_t** procs, size_t nprocs)
{
int ret;
size_t i;
struct mca_mtl_base_endpoint_t **endpoints;
#if OPAL_ENABLE_HETEROGENEOUS_SUPPORT
for (i = 0 ; i < nprocs ; ++i) {
@ -130,27 +128,8 @@ mca_pml_cm_add_procs(struct ompi_proc_t** procs, size_t nprocs)
return ret;
}
endpoints = (struct mca_mtl_base_endpoint_t**)malloc(nprocs * sizeof(struct mca_mtl_base_endpoint_t*));
if (NULL == endpoints) return OMPI_ERROR;
#if OPAL_ENABLE_DEBUG
for (i = 0 ; i < nprocs ; ++i) {
endpoints[i] = NULL;
}
#endif
ret = OMPI_MTL_CALL(add_procs(ompi_mtl, nprocs, procs, endpoints));
if (OMPI_SUCCESS != ret) {
free(endpoints);
return ret;
}
for (i = 0 ; i < nprocs ; ++i) {
procs[i]->proc_pml = (struct mca_pml_endpoint_t*) endpoints[i];
}
free(endpoints);
return OMPI_SUCCESS;
ret = OMPI_MTL_CALL(add_procs(ompi_mtl, nprocs, procs));
return ret;
}
@ -158,24 +137,9 @@ int
mca_pml_cm_del_procs(struct ompi_proc_t** procs, size_t nprocs)
{
int ret;
size_t i;
struct mca_mtl_base_endpoint_t **endpoints;
endpoints = (struct mca_mtl_base_endpoint_t**)malloc(nprocs * sizeof(struct mca_mtl_base_endpoint_t*));
if (NULL == endpoints) return OMPI_ERROR;
for (i = 0 ; i < nprocs ; ++i) {
endpoints[i] = (struct mca_mtl_base_endpoint_t*) procs[i]->proc_pml;
}
ret = OMPI_MTL_CALL(del_procs(ompi_mtl, nprocs, procs, endpoints));
if (OMPI_SUCCESS != ret) {
free(endpoints);
return ret;
}
free(endpoints);
return OMPI_SUCCESS;
ret = OMPI_MTL_CALL(del_procs(ompi_mtl, nprocs, procs));
return ret;
}

27
ompi/mca/pml/ob1/configure.m4 Обычный файл
Просмотреть файл

@ -0,0 +1,27 @@
# -*- shell-script -*-
#
# Copyright (c) 2013 Sandia National Laboratories. All rights reserved.
# $COPYRIGHT$
#
# Additional copyrights may follow
#
# $HEADER$
#
# MCA_ompi_pml_ob1_POST_CONFIG(will_build)
# ----------------------------------------
# The OB1 PML requires a BML endpoint tag to compile, so require it.
# Require in POST_CONFIG instead of CONFIG so that we only require it
# if we're not disabled.
AC_DEFUN([MCA_ompi_pml_ob1_POST_CONFIG], [
AS_IF([test "$1" = "1"], [OMPI_REQUIRE_ENDPOINT_TAG([BML])])
])dnl
# MCA_ompi_pml_ob1_CONFIG(action-if-can-compile,
# [action-if-cant-compile])
# ------------------------------------------------
# We can always build, unless we were explicitly disabled.
AC_DEFUN([MCA_ompi_pml_ob1_CONFIG],[
AC_CONFIG_FILES([ompi/mca/pml/ob1/Makefile])
[$1]
])dnl

13
ompi/mca/pml/ob1/pml_ob1.c
Просмотреть файл

@ -306,18 +306,11 @@ int mca_pml_ob1_add_procs(ompi_proc_t** procs, size_t nprocs)
{
opal_bitmap_t reachable;
int rc;
size_t i;
opal_list_item_t *item;
if(nprocs == 0)
return OMPI_SUCCESS;
/* we don't have any endpoint data we need to cache on the
ompi_proc_t, so set proc_pml to NULL */
for (i = 0 ; i < nprocs ; ++i) {
procs[i]->proc_pml = NULL;
}
OBJ_CONSTRUCT(&reachable, opal_bitmap_t);
rc = opal_bitmap_init(&reachable, (int)nprocs);
if(OMPI_SUCCESS != rc)
@ -555,7 +548,7 @@ int mca_pml_ob1_dump(struct ompi_communicator_t* comm, int verbose)
/* iterate through all procs on communicator */
for( i = 0; i < (int)pml_comm->num_procs; i++ ) {
mca_pml_ob1_comm_proc_t* proc = &pml_comm->procs[i];
mca_bml_base_endpoint_t* ep = (mca_bml_base_endpoint_t*)proc->ompi_proc->proc_bml;
mca_bml_base_endpoint_t* ep = (mca_bml_base_endpoint_t*)proc->ompi_proc->proc_endpoints[OMPI_PROC_ENDPOINT_TAG_BML];
size_t n;
opal_output(0, "[Rank %d] expected_seq %d ompi_proc %p send_seq %d\n",
@ -662,8 +655,10 @@ void mca_pml_ob1_process_pending_packets(mca_bml_base_btl_t* bml_btl)
pckt->bml_btl->btl == bml_btl->btl) {
send_dst = pckt->bml_btl;
} else {
mca_bml_base_endpoint_t* endpoint =
(mca_bml_base_endpoint_t*) pckt->proc->proc_endpoints[OMPI_PROC_ENDPOINT_TAG_BML];
send_dst = mca_bml_base_btl_array_find(
&pckt->proc->proc_bml->btl_eager, bml_btl->btl);
&endpoint->btl_eager, bml_btl->btl);
}
if(NULL == send_dst) {
OPAL_THREAD_LOCK(&mca_pml_ob1.lock);

6
ompi/mca/pml/ob1/pml_ob1_cuda.c
Просмотреть файл

@ -150,7 +150,7 @@ int mca_pml_ob1_cuda_need_buffers(void * rreq,
{
mca_pml_ob1_recv_request_t* recvreq = (mca_pml_ob1_recv_request_t*)rreq;
mca_bml_base_endpoint_t* bml_endpoint =
(mca_bml_base_endpoint_t*)recvreq->req_recv.req_base.req_proc->proc_bml;
(mca_bml_base_endpoint_t*)recvreq->req_recv.req_base.req_proc->proc_endpoints[OMPI_PROC_ENDPOINT_TAG_BML];
mca_bml_base_btl_t *bml_btl = mca_bml_base_btl_array_find(&bml_endpoint->btl_send, btl);
if ((recvreq->req_recv.req_base.req_convertor.flags & CONVERTOR_CUDA) &&
@ -180,11 +180,11 @@ void mca_pml_ob1_cuda_add_ipc_support(struct mca_btl_base_module_t* btl, int32_t
int i;
assert(NULL != errproc);
assert(NULL != errproc->proc_bml);
assert(NULL != errproc->proc_endpoints[OMPI_PROC_ENDPOINT_TAG_BML]);
if (NULL != btlinfo) {
btl_verbose_stream = *(int *)btlinfo;
}
ep = (mca_bml_base_endpoint_t*)errproc->proc_bml;
ep = (mca_bml_base_endpoint_t*)errproc->proc_endpoints[OMPI_PROC_ENDPOINT_TAG_BML];
/* Find the corresponding bml and adjust the flag to support CUDA get */
for( i = 0; i < (int)ep->btl_send.arr_size; i++ ) {

4
ompi/mca/pml/ob1/pml_ob1_recvreq.c
Просмотреть файл

@ -255,7 +255,7 @@ static int mca_pml_ob1_recv_request_ack(
ompi_proc_t* proc = (ompi_proc_t*)recvreq->req_recv.req_base.req_proc;
mca_bml_base_endpoint_t* bml_endpoint = NULL;
bml_endpoint = (mca_bml_base_endpoint_t*) proc->proc_bml;
bml_endpoint = (mca_bml_base_endpoint_t*) proc->proc_endpoints[OMPI_PROC_ENDPOINT_TAG_BML];
/* by default copy everything */
recvreq->req_send_offset = bytes_received;
@ -648,7 +648,7 @@ void mca_pml_ob1_recv_request_progress_rget( mca_pml_ob1_recv_request_t* recvreq
}
/* lookup bml datastructures */
bml_endpoint = (mca_bml_base_endpoint_t*)recvreq->req_recv.req_base.req_proc->proc_bml;
bml_endpoint = (mca_bml_base_endpoint_t*)recvreq->req_recv.req_base.req_proc->proc_endpoints[OMPI_PROC_ENDPOINT_TAG_BML];
rdma_bml = mca_bml_base_btl_array_find(&bml_endpoint->btl_rdma, btl);
#if OMPI_CUDA_SUPPORT

2
ompi/mca/pml/ob1/pml_ob1_recvreq.h
Просмотреть файл

@ -410,7 +410,7 @@ static inline int mca_pml_ob1_recv_request_ack_send(ompi_proc_t* proc,
size_t i;
mca_bml_base_btl_t* bml_btl;
mca_bml_base_endpoint_t* endpoint =
(mca_bml_base_endpoint_t*)proc->proc_bml;
(mca_bml_base_endpoint_t*)proc->proc_endpoints[OMPI_PROC_ENDPOINT_TAG_BML];
for(i = 0; i < mca_bml_base_btl_array_get_size(&endpoint->btl_eager); i++) {
bml_btl = mca_bml_base_btl_array_get_next(&endpoint->btl_eager);

2
ompi/mca/pml/ob1/pml_ob1_sendreq.h
Просмотреть файл

@ -428,7 +428,7 @@ mca_pml_ob1_send_request_start( mca_pml_ob1_send_request_t* sendreq )
{
mca_pml_ob1_comm_t* comm = sendreq->req_send.req_base.req_comm->c_pml_comm;
mca_bml_base_endpoint_t* endpoint = (mca_bml_base_endpoint_t*)
sendreq->req_send.req_base.req_proc->proc_bml;
sendreq->req_send.req_base.req_proc->proc_endpoints[OMPI_PROC_ENDPOINT_TAG_BML];
size_t i;
if( OPAL_UNLIKELY(endpoint == NULL) ) {

14
ompi/mca/pml/pml.h
Просмотреть файл

@ -73,20 +73,6 @@ BEGIN_C_DECLS
*/
typedef uint64_t mca_pml_sequence_t;
/**
* Base PML endpoint structure
*
* Base PML structure for caching endpoint information on a proc. A
* pointer to an mca_pml_endpoint_t is maintained on each ompi_proc_t,
* in the proc_pml field, to provide per-process cache information.
* The data is opaque to the active PML -- no other subsystem will
* attempt to access the information in the cache.
*
* The PML is responsible for allocation and deallocation of the
* endpoint data during pml_add_procs and pml_del_procs.
*/
struct mca_pml_endpoint_t;
struct ompi_proc_t;
typedef enum {

27
ompi/mca/sbgp/p2p/configure.m4 Обычный файл
Просмотреть файл

@ -0,0 +1,27 @@
# -*- shell-script -*-
#
# Copyright (c) 2013 Sandia National Laboratories. All rights reserved.
# $COPYRIGHT$
#
# Additional copyrights may follow
#
# $HEADER$
#
# MCA_ompi_sbgp_p2p_POST_CONFIG(will_build)
# ----------------------------------------
# The p2p sbgp requires a BML endpoint tag to compile, so require it.
# Require in POST_CONFIG instead of CONFIG so that we only require it
# if we're not disabled.
AC_DEFUN([MCA_ompi_sbgp_p2p_POST_CONFIG], [
AS_IF([test "$1" = "1"], [OMPI_REQUIRE_ENDPOINT_TAG([BML])])
])dnl
# MCA_ompi_sbgp_p2p_CONFIG(action-if-can-compile,
# [action-if-cant-compile])
# ------------------------------------------------
# We can always build, unless we were explicitly disabled.
AC_DEFUN([MCA_ompi_sbgp_p2p_CONFIG],[
AC_CONFIG_FILES([ompi/mca/sbgp/p2p/Makefile])
[$1]
])dnl

14
ompi/mca/sbgp/p2p/sbgp_p2p_component.c
Просмотреть файл

@ -169,15 +169,18 @@ static mca_sbgp_base_module_t * mca_sbgp_p2p_select_procs(struct ompi_proc_t **
/* count the number of ranks in the group */
cnt=0;
for( proc=0 ; proc < n_procs_in ; proc++) {
mca_bml_base_endpoint_t* endpoint =
(mca_bml_base_endpoint_t*) procs[proc]->proc_endpoints[OMPI_PROC_ENDPOINT_TAG_BML];
if(my_rank == proc ) {
cnt++;
continue;
}
/* loop over btls */
for( i_btl=0 ; i_btl < (int) mca_bml_base_btl_array_get_size(&(procs[proc]->proc_bml->btl_eager)) ; i_btl++ ) {
for( i_btl=0 ; i_btl < (int) mca_bml_base_btl_array_get_size(&(endpoint->btl_eager)) ; i_btl++ ) {
if(key) {
/* I am checking for specific btl */
if( strcmp(procs[proc]->proc_bml->btl_eager.bml_btls[i_btl].btl->btl_component->btl_version.mca_component_name,key)) {
if( strcmp(endpoint->btl_eager.bml_btls[i_btl].btl->btl_component->btl_version.mca_component_name,key)) {
cnt++;
break;
@ -206,6 +209,9 @@ fflush(stderr);
cnt=0;
for( proc=0 ; proc < n_procs_in ; proc++) {
mca_bml_base_endpoint_t* endpoint =
(mca_bml_base_endpoint_t*) procs[proc]->proc_endpoints[OMPI_PROC_ENDPOINT_TAG_BML];
if(my_rank == proc ) {
module->super.group_list[cnt]=proc;
cnt++;
@ -214,11 +220,11 @@ fflush(stderr);
/* loop over btls */
for( i_btl=0 ;
i_btl < (int) mca_bml_base_btl_array_get_size(&(procs[proc]->proc_bml->btl_eager)) ;
i_btl < (int) mca_bml_base_btl_array_get_size(&(endpoint->btl_eager)) ;
i_btl++ ) {
if(key) {
/* I am checking for specific btl */
if( strcmp(procs[proc]->proc_bml->btl_eager.bml_btls[i_btl].btl->
if( strcmp(endpoint->btl_eager.bml_btls[i_btl].btl->
btl_component->btl_version.mca_component_name,key)) {
module->super.group_list[cnt]=proc;
cnt++;

4
ompi/proc/proc.c
Просмотреть файл

@ -23,6 +23,7 @@
#include "ompi_config.h"
#include <string.h>
#include <strings.h>
#include "ompi/constants.h"
#include "opal/datatype/opal_convertor.h"
@ -54,8 +55,7 @@ OBJ_CLASS_INSTANCE(
void ompi_proc_construct(ompi_proc_t* proc)
{
proc->proc_bml = NULL;
proc->proc_pml = NULL;
bzero(proc->proc_endpoints, sizeof(proc->proc_endpoints));
/* By default all processors are supposedly having the same architecture as me. Thus,
* by default we run in a homogeneous environment. Later, when the RTE can tell us

18
ompi/proc/proc.h
Просмотреть файл

@ -51,16 +51,26 @@ BEGIN_C_DECLS
*
* Remote Open MPI process structure. Each process contains exactly
* one ompi_proc_t structure for each remote process it knows about.
*
* Each proc entry has an array of endpoint data associated with it.
* The size of this array, and its entries, is unique to a particular
* build of Open MPI. As the endpoint list (or index values) are
* local to a process, this does not negatively impact heterogeneous
* builds. If a component or framework requires a tag index, it
* should call OMPI_REQUIRE_ENDPOINT_TAG(<name>). Requests which
* share the same name will have the same value, allowing
* cross-component sharing of endpoint data. The tag may be referenced
* by the pre-processor define OMPI_PROC_ENDPOINT_TAG_<name>. Adding
* a tag increases the memory consumed by Open MPI, so should only be done
* if unavoidable.
*/
struct ompi_proc_t {
/** allow proc to be placed on a list */
opal_list_item_t super;
/** this process' name */
ompi_process_name_t proc_name;
/** PML specific proc data */
struct mca_pml_endpoint_t* proc_pml;
/** BML specific proc data */
struct mca_bml_base_endpoint_t* proc_bml;
/* endpoint data */
void *proc_endpoints[OMPI_PROC_ENDPOINT_TAG_MAX];
/** architecture of this process */
uint32_t proc_arch;
/** flags for this proc */