btl/vader: rename vader -> sm
Now that the old sm btl has been gone for some time there was a request to rename vader to sm. This commit does just that (reluctantly). An alias has been generated so specifying vader in the btl selection variable or specifying vader parameters will continue to work. Signed-off-by: Nathan Hjelm <hjelmn@google.com>
Этот коммит содержится в:
Nathan Hjelm
Jeff Squyres
родитель
9fae5bfdf3
Коммит
9d8f634044
31
README
31
README
@ -594,7 +594,6 @@ MPI Functionality and Features
|
||||
- tcp
|
||||
- ugni
|
||||
- usnic
|
||||
- vader (shared memory)
|
||||
|
||||
Currently, MPI File operations are not thread safe even if MPI is
|
||||
initialized for MPI_THREAD_MULTIPLE support.
|
||||
@ -675,8 +674,8 @@ Network Support
|
||||
- SMCUDA
|
||||
- Cisco usNIC
|
||||
- uGNI (Cray Gemini, Aries)
|
||||
- vader (XPMEM, Linux CMA, Linux KNEM, and copy-in/copy-out shared
|
||||
memory)
|
||||
- shared memory (XPMEM, Linux CMA, Linux KNEM, and
|
||||
copy-in/copy-out shared memory)
|
||||
|
||||
- "cm" supports a smaller number of networks (and they cannot be
|
||||
used together), but may provide better overall MPI performance:
|
||||
@ -785,16 +784,16 @@ Network Support
|
||||
v2.6.15 with libibverbs v1.1 or later (first released as part of
|
||||
OFED v1.2), per restrictions imposed by the OFED network stack.
|
||||
|
||||
- Linux "knem" support is used when the "vader" or "sm" (shared
|
||||
memory) BTLs are compiled with knem support (see the --with-knem
|
||||
configure option) and the knem Linux module is loaded in the running
|
||||
kernel. If the knem Linux kernel module is not loaded, the knem
|
||||
support is (by default) silently deactivated during Open MPI jobs.
|
||||
- Linux "knem" support is used when the "sm" (shared memory) BTL is
|
||||
compiled with knem support (see the --with-knem configure option)
|
||||
and the knem Linux module is loaded in the running kernel. If the
|
||||
knem Linux kernel module is not loaded, the knem support is (by
|
||||
default) silently deactivated during Open MPI jobs.
|
||||
|
||||
See https://knem.gforge.inria.fr/ for details on Knem.
|
||||
|
||||
- Linux Cross-Memory Attach (CMA) or XPMEM is used by the vader
|
||||
shared-memory BTL when the CMA/XPMEM libraries are installedm,
|
||||
- Linux Cross-Memory Attach (CMA) or XPMEM is used by the "sm" shared
|
||||
memory BTL when the CMA/XPMEM libraries are installedm,
|
||||
respectively. Linux CMA and XPMEM are similar (but different)
|
||||
mechanisms for Open MPI to utilize single-copy semantics for shared
|
||||
memory.
|
||||
@ -2154,14 +2153,14 @@ comma-delimited list to the "btl" MCA parameter:
|
||||
|
||||
shell$ mpirun --mca btl tcp,self hello_world_mpi
|
||||
|
||||
To add shared memory support, add "vader" into the command-delimited
|
||||
list (list order does not matter):
|
||||
To add shared memory support, add "sm" into the command-delimited list
|
||||
(list order does not matter):
|
||||
|
||||
shell$ mpirun --mca btl tcp,vader,self hello_world_mpi
|
||||
shell$ mpirun --mca btl tcp,sm,self hello_world_mpi
|
||||
|
||||
(there is an "sm" shared memory BTL, too, but "vader" is a newer
|
||||
generation of shared memory support; by default, "vader" will be used
|
||||
instead of "sm")
|
||||
(there used to be a "vader" BTL for shared memory support; it was
|
||||
renamed to "sm" in Open MPI v5.0.0, but the alias "vader" still works
|
||||
as well)
|
||||
|
||||
To specifically deactivate a specific component, the comma-delimited
|
||||
list can be prepended with a "^" to negate it:
|
||||
|
||||
@ -16,6 +16,7 @@
|
||||
* and Technology (RIST). All rights reserved.
|
||||
* Copyright (c) 2016-2018 Los Alamos National Security, LLC. All rights
|
||||
* reserved.
|
||||
* Copyright (c) 2020 Google, LLC. All rights reserved.
|
||||
* $COPYRIGHT$
|
||||
*
|
||||
* Additional copyrights may follow
|
||||
@ -30,6 +31,7 @@
|
||||
#include "opal/mca/mca.h"
|
||||
#include "opal/util/output.h"
|
||||
#include "opal/mca/base/base.h"
|
||||
#include "opal/mca/base/mca_base_alias.h"
|
||||
#include "opal/mca/btl/btl.h"
|
||||
#include "opal/mca/btl/base/base.h"
|
||||
|
||||
@ -154,6 +156,29 @@ static int mca_btl_base_register(mca_base_register_flag_t flags)
|
||||
(void) mca_base_var_enum_register("opal", "btl", "base", "btl_atomic_flags",
|
||||
&mca_btl_base_atomic_enum);
|
||||
|
||||
/* Note that we break abstraction rules here by listing two
|
||||
specific BTLs here in the base. This is necessary, however,
|
||||
due to extraordinary circumstances:
|
||||
|
||||
1. In Open MPI v5.0.0, we want to rename the "vader" BTL to be
|
||||
"sm", because the "old" sm BTL is long gone, and the "vader"
|
||||
name is not very user-friendly.
|
||||
|
||||
2. The MCA aliasing mechanism was therefore created for v5.0.0.
|
||||
Both the component itself and all of its MCA vars are
|
||||
aliased.
|
||||
|
||||
3. However -- at least as currently implemented -- by the time
|
||||
individual components are registered, it's too late to make
|
||||
aliases. Hence, if we want to preserve the name "vader" for
|
||||
some sembalance of backwards compatibility (and we do!), we
|
||||
have to register "vader" as an "alias for "sm" up here in
|
||||
the BTL base, before any BTL components are registered.
|
||||
|
||||
This is why we tolerate this abstraction break up here in the
|
||||
BTL component base. */
|
||||
(void) mca_base_alias_register ("opal", "btl", "sm", "vader", MCA_BASE_ALIAS_FLAG_NONE);
|
||||
|
||||
return OPAL_SUCCESS;
|
||||
}
|
||||
|
||||
|
||||
@ -18,6 +18,7 @@
|
||||
* Copyright (c) 2015 Research Organization for Information Science
|
||||
* and Technology (RIST). All rights reserved.
|
||||
* Copyright (c) 2020 Intel, Inc. All rights reserved.
|
||||
* Copyright (c) 2020 Google, LLC. All rights reserved.
|
||||
* $COPYRIGHT$
|
||||
*
|
||||
* Additional copyrights may follow
|
||||
@ -190,7 +191,7 @@ typedef uint8_t mca_btl_base_tag_t;
|
||||
#define MCA_BTL_TAG_IB (MCA_BTL_TAG_BTL + 0)
|
||||
#define MCA_BTL_TAG_UDAPL (MCA_BTL_TAG_BTL + 1)
|
||||
#define MCA_BTL_TAG_SMCUDA (MCA_BTL_TAG_BTL + 2)
|
||||
#define MCA_BTL_TAG_VADER (MCA_BTL_TAG_BTL + 3)
|
||||
#define MCA_BTL_TAG_SM (MCA_BTL_TAG_BTL + 3)
|
||||
|
||||
/* prefered protocol */
|
||||
#define MCA_BTL_FLAGS_SEND 0x0001
|
||||
|
||||
@ -13,6 +13,7 @@
|
||||
# Copyright (c) 2011-2014 Los Alamos National Security, LLC. All rights
|
||||
# reserved.
|
||||
# Copyright (c) 2017 IBM Corporation. All rights reserved.
|
||||
# Copyright (c) 2020 Google, LLC. All rights reserved.
|
||||
# $COPYRIGHT$
|
||||
#
|
||||
# Additional copyrights may follow
|
||||
@ -20,50 +21,50 @@
|
||||
# $HEADER$
|
||||
#
|
||||
|
||||
AM_CPPFLAGS = $(btl_vader_CPPFLAGS)
|
||||
AM_CPPFLAGS = $(btl_sm_CPPFLAGS)
|
||||
|
||||
dist_opaldata_DATA = help-btl-vader.txt
|
||||
dist_opaldata_DATA = help-btl-sm.txt
|
||||
|
||||
libmca_btl_vader_la_sources = \
|
||||
btl_vader_module.c \
|
||||
btl_vader.h \
|
||||
btl_vader_component.c \
|
||||
btl_vader_endpoint.h \
|
||||
btl_vader_fifo.h \
|
||||
btl_vader_frag.c \
|
||||
btl_vader_frag.h \
|
||||
btl_vader_send.c \
|
||||
btl_vader_sendi.c \
|
||||
btl_vader_fbox.h \
|
||||
btl_vader_get.c \
|
||||
btl_vader_put.c \
|
||||
btl_vader_xpmem.c \
|
||||
btl_vader_xpmem.h \
|
||||
btl_vader_knem.c \
|
||||
btl_vader_knem.h \
|
||||
btl_vader_sc_emu.c \
|
||||
btl_vader_atomic.c
|
||||
libmca_btl_sm_la_sources = \
|
||||
btl_sm_module.c \
|
||||
btl_sm.h \
|
||||
btl_sm_component.c \
|
||||
btl_sm_endpoint.h \
|
||||
btl_sm_fifo.h \
|
||||
btl_sm_frag.c \
|
||||
btl_sm_frag.h \
|
||||
btl_sm_send.c \
|
||||
btl_sm_sendi.c \
|
||||
btl_sm_fbox.h \
|
||||
btl_sm_get.c \
|
||||
btl_sm_put.c \
|
||||
btl_sm_xpmem.c \
|
||||
btl_sm_xpmem.h \
|
||||
btl_sm_knem.c \
|
||||
btl_sm_knem.h \
|
||||
btl_sm_sc_emu.c \
|
||||
btl_sm_atomic.c
|
||||
|
||||
# Make the output library in this directory, and name it either
|
||||
# mca_<type>_<name>.la (for DSO builds) or libmca_<type>_<name>.la
|
||||
# (for static builds).
|
||||
|
||||
if MCA_BUILD_opal_btl_vader_DSO
|
||||
if MCA_BUILD_opal_btl_sm_DSO
|
||||
component_noinst =
|
||||
component_install = mca_btl_vader.la
|
||||
component_install = mca_btl_sm.la
|
||||
else
|
||||
component_noinst = libmca_btl_vader.la
|
||||
component_noinst = libmca_btl_sm.la
|
||||
component_install =
|
||||
endif
|
||||
|
||||
mcacomponentdir = $(opallibdir)
|
||||
mcacomponent_LTLIBRARIES = $(component_install)
|
||||
mca_btl_vader_la_SOURCES = $(libmca_btl_vader_la_sources)
|
||||
mca_btl_vader_la_LDFLAGS = -module -avoid-version $(btl_vader_LDFLAGS)
|
||||
mca_btl_vader_la_LIBADD = $(top_builddir)/opal/lib@OPAL_LIB_PREFIX@open-pal.la \
|
||||
$(btl_vader_LIBS)
|
||||
mca_btl_sm_la_SOURCES = $(libmca_btl_sm_la_sources)
|
||||
mca_btl_sm_la_LDFLAGS = -module -avoid-version $(btl_sm_LDFLAGS)
|
||||
mca_btl_sm_la_LIBADD = $(top_builddir)/opal/lib@OPAL_LIB_PREFIX@open-pal.la \
|
||||
$(btl_sm_LIBS)
|
||||
|
||||
noinst_LTLIBRARIES = $(component_noinst)
|
||||
libmca_btl_vader_la_SOURCES = $(libmca_btl_vader_la_sources)
|
||||
libmca_btl_vader_la_LIBADD = $(btl_vader_LIBS)
|
||||
libmca_btl_vader_la_LDFLAGS = -module -avoid-version $(btl_vader_LDFLAGS)
|
||||
libmca_btl_sm_la_SOURCES = $(libmca_btl_sm_la_sources)
|
||||
libmca_btl_sm_la_LIBADD = $(btl_sm_LIBS)
|
||||
libmca_btl_sm_la_LDFLAGS = -module -avoid-version $(btl_sm_LDFLAGS)
|
||||
329
opal/mca/btl/sm/btl_sm.h
Обычный файл
329
opal/mca/btl/sm/btl_sm.h
Обычный файл
@ -0,0 +1,329 @@
|
||||
/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
|
||||
/*
|
||||
* Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
|
||||
* University Research and Technology
|
||||
* Corporation. All rights reserved.
|
||||
* Copyright (c) 2004-2009 The University of Tennessee and The University
|
||||
* of Tennessee Research Foundation. All rights
|
||||
* reserved.
|
||||
* Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
|
||||
* University of Stuttgart. All rights reserved.
|
||||
* Copyright (c) 2004-2005 The Regents of the University of California.
|
||||
* All rights reserved.
|
||||
* Copyright (c) 2006-2007 Voltaire. All rights reserved.
|
||||
* Copyright (c) 2009-2010 Cisco Systems, Inc. All rights reserved.
|
||||
* Copyright (c) 2010-2018 Los Alamos National Security, LLC. All rights
|
||||
* reserved.
|
||||
* Copyright (c) 2015 Mellanox Technologies. All rights reserved.
|
||||
* Copyright (c) 2018 Triad National Security, LLC. All rights
|
||||
* reserved.
|
||||
* Copyright (c) 2020 Google, LLC. All rights reserved.
|
||||
*
|
||||
* Copyright (c) 2019-2020 Intel, Inc. All rights reserved.
|
||||
* $COPYRIGHT$
|
||||
*
|
||||
* Additional copyrights may follow
|
||||
*
|
||||
* $HEADER$
|
||||
*/
|
||||
/**
|
||||
* @file
|
||||
*/
|
||||
#ifndef MCA_BTL_SM_H
|
||||
#define MCA_BTL_SM_H
|
||||
|
||||
#include "opal_config.h"
|
||||
|
||||
#include <stddef.h>
|
||||
#include <stdlib.h>
|
||||
#include <string.h>
|
||||
|
||||
# include <stdint.h>
|
||||
#ifdef HAVE_SCHED_H
|
||||
# include <sched.h>
|
||||
#endif /* HAVE_SCHED_H */
|
||||
#ifdef HAVE_UNISTD_H
|
||||
# include <unistd.h>
|
||||
#endif /* HAVE_UNISTD_H */
|
||||
|
||||
#include "opal/mca/shmem/base/base.h"
|
||||
|
||||
#include "opal/class/opal_free_list.h"
|
||||
#include "opal/sys/atomic.h"
|
||||
#include "opal/mca/btl/btl.h"
|
||||
#include "opal/mca/rcache/rcache.h"
|
||||
#include "opal/mca/rcache/base/rcache_base_vma.h"
|
||||
#include "opal/mca/btl/base/base.h"
|
||||
#include "opal/mca/rcache/rcache.h"
|
||||
#include "opal/mca/rcache/base/base.h"
|
||||
#include "opal/mca/btl/base/btl_base_error.h"
|
||||
#include "opal/mca/mpool/base/base.h"
|
||||
#include "opal/util/proc.h"
|
||||
#include "btl_sm_endpoint.h"
|
||||
|
||||
#include "opal/mca/pmix/pmix-internal.h"
|
||||
|
||||
#include "btl_sm_xpmem.h"
|
||||
#include "btl_sm_knem.h"
|
||||
|
||||
BEGIN_C_DECLS
|
||||
|
||||
#define min(a,b) ((a) < (b) ? (a) : (b))
|
||||
|
||||
/*
|
||||
* Shared Memory resource managment
|
||||
*/
|
||||
|
||||
struct sm_fifo_t;
|
||||
|
||||
/*
|
||||
* Modex data
|
||||
*/
|
||||
union sm_modex_t {
|
||||
#if OPAL_BTL_SM_HAVE_XPMEM
|
||||
struct sm_modex_xpmem_t {
|
||||
xpmem_segid_t seg_id;
|
||||
void *segment_base;
|
||||
uintptr_t address_max;
|
||||
} xpmem;
|
||||
#endif
|
||||
struct sm_modex_other_t {
|
||||
ino_t user_ns_id;
|
||||
int seg_ds_size;
|
||||
/* seg_ds needs to be the last element */
|
||||
opal_shmem_ds_t seg_ds;
|
||||
} other;
|
||||
};
|
||||
|
||||
/**
|
||||
* Single copy mechanisms
|
||||
*/
|
||||
enum {
|
||||
MCA_BTL_SM_XPMEM = 0,
|
||||
MCA_BTL_SM_CMA = 1,
|
||||
MCA_BTL_SM_KNEM = 2,
|
||||
MCA_BTL_SM_NONE = 3,
|
||||
MCA_BTL_SM_EMUL = 4,
|
||||
};
|
||||
|
||||
/**
|
||||
* Shared Memory (SM) BTL module.
|
||||
*/
|
||||
struct mca_btl_sm_component_t {
|
||||
mca_btl_base_component_3_0_0_t super; /**< base BTL component */
|
||||
int sm_free_list_num; /**< initial size of free lists */
|
||||
int sm_free_list_max; /**< maximum size of free lists */
|
||||
int sm_free_list_inc; /**< number of elements to alloc when growing free lists */
|
||||
#if OPAL_BTL_SM_HAVE_XPMEM
|
||||
xpmem_segid_t my_seg_id; /**< this rank's xpmem segment id */
|
||||
uintptr_t my_address_max; /**< largest address */
|
||||
mca_rcache_base_vma_module_t *vma_module; /**< registration cache for xpmem segments */
|
||||
#endif
|
||||
opal_shmem_ds_t seg_ds; /**< this rank's shared memory segment (when not using xpmem) */
|
||||
|
||||
opal_mutex_t lock; /**< lock to protect concurrent updates to this structure's members */
|
||||
char *my_segment; /**< this rank's base pointer */
|
||||
size_t segment_size; /**< size of my_segment */
|
||||
int32_t num_smp_procs; /**< current number of smp procs on this host */
|
||||
opal_atomic_int32_t local_rank; /**< current rank index at add_procs() time */
|
||||
opal_free_list_t sm_frags_eager; /**< free list of sm send frags */
|
||||
opal_free_list_t sm_frags_max_send; /**< free list of sm max send frags (large fragments) */
|
||||
opal_free_list_t sm_frags_user; /**< free list of small inline frags */
|
||||
opal_free_list_t sm_fboxes; /**< free list of available fast-boxes */
|
||||
|
||||
unsigned int fbox_threshold; /**< number of sends required before we setup a send fast box for a peer */
|
||||
unsigned int fbox_max; /**< maximum number of send fast boxes to allocate */
|
||||
unsigned int fbox_size; /**< size of each peer fast box allocation */
|
||||
|
||||
int single_copy_mechanism; /**< single copy mechanism to use */
|
||||
|
||||
int memcpy_limit; /**< Limit where we switch from memmove to memcpy */
|
||||
int log_attach_align; /**< Log of the alignment for xpmem segments */
|
||||
unsigned int max_inline_send; /**< Limit for copy-in-copy-out fragments */
|
||||
|
||||
mca_btl_base_endpoint_t *endpoints; /**< array of local endpoints (one for each local peer including myself) */
|
||||
mca_btl_base_endpoint_t **fbox_in_endpoints; /**< array of fast box in endpoints */
|
||||
unsigned int num_fbox_in_endpoints; /**< number of fast boxes to poll */
|
||||
struct sm_fifo_t *my_fifo; /**< pointer to the local fifo */
|
||||
|
||||
opal_list_t pending_endpoints; /**< list of endpoints with pending fragments */
|
||||
opal_list_t pending_fragments; /**< fragments pending remote completion */
|
||||
|
||||
char *backing_directory; /**< directory to place shared memory backing files */
|
||||
|
||||
/* knem stuff */
|
||||
#if OPAL_BTL_SM_HAVE_KNEM
|
||||
unsigned int knem_dma_min; /**< minimum size to enable DMA for knem transfers (0 disables) */
|
||||
#endif
|
||||
mca_mpool_base_module_t *mpool;
|
||||
};
|
||||
typedef struct mca_btl_sm_component_t mca_btl_sm_component_t;
|
||||
OPAL_MODULE_DECLSPEC extern mca_btl_sm_component_t mca_btl_sm_component;
|
||||
|
||||
/**
|
||||
* SM BTL Interface
|
||||
*/
|
||||
struct mca_btl_sm_t {
|
||||
mca_btl_base_module_t super; /**< base BTL interface */
|
||||
bool btl_inited; /**< flag indicating if btl has been inited */
|
||||
mca_btl_base_module_error_cb_fn_t error_cb;
|
||||
#if OPAL_BTL_SM_HAVE_KNEM
|
||||
int knem_fd;
|
||||
|
||||
/* registration cache */
|
||||
mca_rcache_base_module_t *knem_rcache;
|
||||
#endif
|
||||
};
|
||||
typedef struct mca_btl_sm_t mca_btl_sm_t;
|
||||
OPAL_MODULE_DECLSPEC extern mca_btl_sm_t mca_btl_sm;
|
||||
|
||||
/* number of peers on the node (not including self) */
|
||||
#define MCA_BTL_SM_NUM_LOCAL_PEERS opal_process_info.num_local_peers
|
||||
|
||||
/* local rank in the group */
|
||||
#define MCA_BTL_SM_LOCAL_RANK opal_process_info.my_local_rank
|
||||
|
||||
/* memcpy is faster at larger sizes but is undefined if the
|
||||
pointers are aliased (TODO -- readd alias check) */
|
||||
static inline void sm_memmove (void *dst, void *src, size_t size)
|
||||
{
|
||||
if (size >= (size_t) mca_btl_sm_component.memcpy_limit) {
|
||||
memcpy (dst, src, size);
|
||||
} else {
|
||||
memmove (dst, src, size);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Initiate a send to the peer.
|
||||
*
|
||||
* @param btl (IN) BTL module
|
||||
* @param peer (IN) BTL peer addressing
|
||||
*/
|
||||
int mca_btl_sm_send(struct mca_btl_base_module_t *btl,
|
||||
struct mca_btl_base_endpoint_t *endpoint,
|
||||
struct mca_btl_base_descriptor_t *descriptor,
|
||||
mca_btl_base_tag_t tag);
|
||||
|
||||
/**
|
||||
* Initiate an inline send to the peer.
|
||||
*
|
||||
* @param btl (IN) BTL module
|
||||
* @param peer (IN) BTL peer addressing
|
||||
*/
|
||||
int mca_btl_sm_sendi (struct mca_btl_base_module_t *btl,
|
||||
struct mca_btl_base_endpoint_t *endpoint,
|
||||
struct opal_convertor_t *convertor,
|
||||
void *header, size_t header_size,
|
||||
size_t payload_size, uint8_t order,
|
||||
uint32_t flags, mca_btl_base_tag_t tag,
|
||||
mca_btl_base_descriptor_t **descriptor);
|
||||
|
||||
/**
|
||||
* Initiate an synchronous put.
|
||||
*
|
||||
* @param btl (IN) BTL module
|
||||
* @param endpoint (IN) BTL addressing information
|
||||
* @param descriptor (IN) Description of the data to be transferred
|
||||
*/
|
||||
#if OPAL_BTL_SM_HAVE_XPMEM
|
||||
int mca_btl_sm_put_xpmem (mca_btl_base_module_t *btl, mca_btl_base_endpoint_t *endpoint, void *local_address,
|
||||
uint64_t remote_address, mca_btl_base_registration_handle_t *local_handle,
|
||||
mca_btl_base_registration_handle_t *remote_handle, size_t size, int flags,
|
||||
int order, mca_btl_base_rdma_completion_fn_t cbfunc, void *cbcontext, void *cbdata);
|
||||
#endif
|
||||
|
||||
#if OPAL_BTL_SM_HAVE_CMA
|
||||
int mca_btl_sm_put_cma (mca_btl_base_module_t *btl, mca_btl_base_endpoint_t *endpoint, void *local_address,
|
||||
uint64_t remote_address, mca_btl_base_registration_handle_t *local_handle,
|
||||
mca_btl_base_registration_handle_t *remote_handle, size_t size, int flags,
|
||||
int order, mca_btl_base_rdma_completion_fn_t cbfunc, void *cbcontext, void *cbdata);
|
||||
#endif
|
||||
|
||||
#if OPAL_BTL_SM_HAVE_KNEM
|
||||
int mca_btl_sm_put_knem (mca_btl_base_module_t *btl, mca_btl_base_endpoint_t *endpoint, void *local_address,
|
||||
uint64_t remote_address, mca_btl_base_registration_handle_t *local_handle,
|
||||
mca_btl_base_registration_handle_t *remote_handle, size_t size, int flags,
|
||||
int order, mca_btl_base_rdma_completion_fn_t cbfunc, void *cbcontext, void *cbdata);
|
||||
#endif
|
||||
|
||||
int mca_btl_sm_put_sc_emu (mca_btl_base_module_t *btl, mca_btl_base_endpoint_t *endpoint, void *local_address,
|
||||
uint64_t remote_address, mca_btl_base_registration_handle_t *local_handle,
|
||||
mca_btl_base_registration_handle_t *remote_handle, size_t size, int flags,
|
||||
int order, mca_btl_base_rdma_completion_fn_t cbfunc, void *cbcontext, void *cbdata);
|
||||
|
||||
/**
|
||||
* Initiate an synchronous get.
|
||||
*
|
||||
* @param btl (IN) BTL module
|
||||
* @param endpoint (IN) BTL addressing information
|
||||
* @param descriptor (IN) Description of the data to be transferred
|
||||
*/
|
||||
#if OPAL_BTL_SM_HAVE_XPMEM
|
||||
int mca_btl_sm_get_xpmem (mca_btl_base_module_t *btl, mca_btl_base_endpoint_t *endpoint, void *local_address,
|
||||
uint64_t remote_address, mca_btl_base_registration_handle_t *local_handle,
|
||||
mca_btl_base_registration_handle_t *remote_handle, size_t size, int flags,
|
||||
int order, mca_btl_base_rdma_completion_fn_t cbfunc, void *cbcontext, void *cbdata);
|
||||
#endif
|
||||
|
||||
#if OPAL_BTL_SM_HAVE_CMA
|
||||
int mca_btl_sm_get_cma (mca_btl_base_module_t *btl, mca_btl_base_endpoint_t *endpoint, void *local_address,
|
||||
uint64_t remote_address, mca_btl_base_registration_handle_t *local_handle,
|
||||
mca_btl_base_registration_handle_t *remote_handle, size_t size, int flags,
|
||||
int order, mca_btl_base_rdma_completion_fn_t cbfunc, void *cbcontext, void *cbdata);
|
||||
#endif
|
||||
|
||||
#if OPAL_BTL_SM_HAVE_KNEM
|
||||
int mca_btl_sm_get_knem (mca_btl_base_module_t *btl, mca_btl_base_endpoint_t *endpoint, void *local_address,
|
||||
uint64_t remote_address, mca_btl_base_registration_handle_t *local_handle,
|
||||
mca_btl_base_registration_handle_t *remote_handle, size_t size, int flags,
|
||||
int order, mca_btl_base_rdma_completion_fn_t cbfunc, void *cbcontext, void *cbdata);
|
||||
#endif
|
||||
|
||||
ino_t mca_btl_sm_get_user_ns_id(void);
|
||||
|
||||
int mca_btl_sm_get_sc_emu (mca_btl_base_module_t *btl, mca_btl_base_endpoint_t *endpoint, void *local_address,
|
||||
uint64_t remote_address, mca_btl_base_registration_handle_t *local_handle,
|
||||
mca_btl_base_registration_handle_t *remote_handle, size_t size, int flags,
|
||||
int order, mca_btl_base_rdma_completion_fn_t cbfunc, void *cbcontext, void *cbdata);
|
||||
|
||||
int mca_btl_sm_emu_aop (struct mca_btl_base_module_t *btl, struct mca_btl_base_endpoint_t *endpoint,
|
||||
uint64_t remote_address, mca_btl_base_registration_handle_t *remote_handle,
|
||||
mca_btl_base_atomic_op_t op, uint64_t operand, int flags, int order,
|
||||
mca_btl_base_rdma_completion_fn_t cbfunc, void *cbcontext, void *cbdata);
|
||||
|
||||
int mca_btl_sm_emu_afop (struct mca_btl_base_module_t *btl, struct mca_btl_base_endpoint_t *endpoint,
|
||||
void *local_address, uint64_t remote_address, mca_btl_base_registration_handle_t *local_handle,
|
||||
mca_btl_base_registration_handle_t *remote_handle, mca_btl_base_atomic_op_t op,
|
||||
uint64_t operand, int flags, int order, mca_btl_base_rdma_completion_fn_t cbfunc,
|
||||
void *cbcontext, void *cbdata);
|
||||
|
||||
int mca_btl_sm_emu_acswap (struct mca_btl_base_module_t *btl, struct mca_btl_base_endpoint_t *endpoint,
|
||||
void *local_address, uint64_t remote_address, mca_btl_base_registration_handle_t *local_handle,
|
||||
mca_btl_base_registration_handle_t *remote_handle, uint64_t compare, uint64_t value, int flags,
|
||||
int order, mca_btl_base_rdma_completion_fn_t cbfunc, void *cbcontext, void *cbdata);
|
||||
|
||||
void mca_btl_sm_sc_emu_init (void);
|
||||
|
||||
/**
|
||||
* Allocate a segment.
|
||||
*
|
||||
* @param btl (IN) BTL module
|
||||
* @param size (IN) Request segment size.
|
||||
*/
|
||||
mca_btl_base_descriptor_t* mca_btl_sm_alloc (struct mca_btl_base_module_t* btl,
|
||||
struct mca_btl_base_endpoint_t* endpoint,
|
||||
uint8_t order, size_t size, uint32_t flags);
|
||||
|
||||
/**
|
||||
* Return a segment allocated by this BTL.
|
||||
*
|
||||
* @param btl (IN) BTL module
|
||||
* @param segment (IN) Allocated segment.
|
||||
*/
|
||||
int mca_btl_sm_free (struct mca_btl_base_module_t *btl, mca_btl_base_descriptor_t *des);
|
||||
|
||||
|
||||
END_C_DECLS
|
||||
|
||||
#endif
|
||||
49
opal/mca/btl/sm/btl_sm_atomic.c
Обычный файл
49
opal/mca/btl/sm/btl_sm_atomic.c
Обычный файл
@ -0,0 +1,49 @@
|
||||
/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
|
||||
/*
|
||||
* Copyright (c) 2010-2017 Los Alamos National Security, LLC. All rights
|
||||
* reserved.
|
||||
* Copyright (c) 2019-2020 Google, Inc. All rights reserved.
|
||||
* $COPYRIGHT$
|
||||
*
|
||||
* Additional copyrights may follow
|
||||
*
|
||||
* $HEADER$
|
||||
*/
|
||||
|
||||
#include "opal_config.h"
|
||||
|
||||
#include "btl_sm.h"
|
||||
#include "btl_sm_frag.h"
|
||||
#include "btl_sm_endpoint.h"
|
||||
#include "btl_sm_xpmem.h"
|
||||
|
||||
int mca_btl_sm_emu_aop (struct mca_btl_base_module_t *btl, struct mca_btl_base_endpoint_t *endpoint,
|
||||
uint64_t remote_address, mca_btl_base_registration_handle_t *remote_handle,
|
||||
mca_btl_base_atomic_op_t op, uint64_t operand, int flags, int order,
|
||||
mca_btl_base_rdma_completion_fn_t cbfunc, void *cbcontext, void *cbdata)
|
||||
{
|
||||
size_t size = (flags & MCA_BTL_ATOMIC_FLAG_32BIT) ? 4 : 8;
|
||||
return mca_btl_sm_rdma_frag_start (btl, endpoint, MCA_BTL_SM_OP_ATOMIC, operand, 0, op, order, flags,
|
||||
size, NULL, remote_address, cbfunc, cbcontext, cbdata);
|
||||
}
|
||||
|
||||
int mca_btl_sm_emu_afop (struct mca_btl_base_module_t *btl, struct mca_btl_base_endpoint_t *endpoint,
|
||||
void *local_address, uint64_t remote_address, mca_btl_base_registration_handle_t *local_handle,
|
||||
mca_btl_base_registration_handle_t *remote_handle, mca_btl_base_atomic_op_t op,
|
||||
uint64_t operand, int flags, int order, mca_btl_base_rdma_completion_fn_t cbfunc,
|
||||
void *cbcontext, void *cbdata)
|
||||
{
|
||||
size_t size = (flags & MCA_BTL_ATOMIC_FLAG_32BIT) ? 4 : 8;
|
||||
return mca_btl_sm_rdma_frag_start (btl, endpoint, MCA_BTL_SM_OP_ATOMIC, operand, 0, op, order, flags,
|
||||
size, local_address, remote_address, cbfunc, cbcontext, cbdata);
|
||||
}
|
||||
|
||||
int mca_btl_sm_emu_acswap (struct mca_btl_base_module_t *btl, struct mca_btl_base_endpoint_t *endpoint,
|
||||
void *local_address, uint64_t remote_address, mca_btl_base_registration_handle_t *local_handle,
|
||||
mca_btl_base_registration_handle_t *remote_handle, uint64_t compare, uint64_t value, int flags,
|
||||
int order, mca_btl_base_rdma_completion_fn_t cbfunc, void *cbcontext, void *cbdata)
|
||||
{
|
||||
size_t size = (flags & MCA_BTL_ATOMIC_FLAG_32BIT) ? 4 : 8;
|
||||
return mca_btl_sm_rdma_frag_start (btl, endpoint, MCA_BTL_SM_OP_CSWAP, compare, value, 0, order,
|
||||
flags, size, local_address, remote_address, cbfunc, cbcontext, cbdata);
|
||||
}
|
||||
@ -36,11 +36,11 @@
|
||||
#include "opal/mca/threads/mutex.h"
|
||||
#include "opal/mca/btl/base/btl_base_error.h"
|
||||
|
||||
#include "btl_vader.h"
|
||||
#include "btl_vader_frag.h"
|
||||
#include "btl_vader_fifo.h"
|
||||
#include "btl_vader_fbox.h"
|
||||
#include "btl_vader_xpmem.h"
|
||||
#include "btl_sm.h"
|
||||
#include "btl_sm_frag.h"
|
||||
#include "btl_sm_fifo.h"
|
||||
#include "btl_sm_fbox.h"
|
||||
#include "btl_sm_xpmem.h"
|
||||
|
||||
#ifdef HAVE_SYS_STAT_H
|
||||
#include <sys/stat.h>
|
||||
@ -58,112 +58,112 @@
|
||||
#define MAP_ANONYMOUS MAP_ANON
|
||||
#endif
|
||||
|
||||
static int mca_btl_vader_component_progress (void);
|
||||
static int mca_btl_vader_component_open(void);
|
||||
static int mca_btl_vader_component_close(void);
|
||||
static int mca_btl_vader_component_register(void);
|
||||
static mca_btl_base_module_t** mca_btl_vader_component_init(int *num_btls,
|
||||
bool enable_progress_threads,
|
||||
bool enable_mpi_threads);
|
||||
static int mca_btl_sm_component_progress (void);
|
||||
static int mca_btl_sm_component_open(void);
|
||||
static int mca_btl_sm_component_close(void);
|
||||
static int mca_btl_sm_component_register(void);
|
||||
static mca_btl_base_module_t** mca_btl_sm_component_init(int *num_btls,
|
||||
bool enable_progress_threads,
|
||||
bool enable_mpi_threads);
|
||||
|
||||
/* This enumeration is in order of preference */
|
||||
static mca_base_var_enum_value_t single_copy_mechanisms[] = {
|
||||
#if OPAL_BTL_VADER_HAVE_XPMEM
|
||||
{.value = MCA_BTL_VADER_XPMEM, .string = "xpmem"},
|
||||
#if OPAL_BTL_SM_HAVE_XPMEM
|
||||
{.value = MCA_BTL_SM_XPMEM, .string = "xpmem"},
|
||||
#endif
|
||||
#if OPAL_BTL_VADER_HAVE_CMA
|
||||
{.value = MCA_BTL_VADER_CMA, .string = "cma"},
|
||||
#if OPAL_BTL_SM_HAVE_CMA
|
||||
{.value = MCA_BTL_SM_CMA, .string = "cma"},
|
||||
#endif
|
||||
#if OPAL_BTL_VADER_HAVE_KNEM
|
||||
{.value = MCA_BTL_VADER_KNEM, .string = "knem"},
|
||||
#if OPAL_BTL_SM_HAVE_KNEM
|
||||
{.value = MCA_BTL_SM_KNEM, .string = "knem"},
|
||||
#endif
|
||||
{.value = MCA_BTL_VADER_EMUL, .string = "emulated"},
|
||||
{.value = MCA_BTL_VADER_NONE, .string = "none"},
|
||||
{.value = 0, .string = NULL}
|
||||
{.value = MCA_BTL_SM_EMUL, .string = "emulated"},
|
||||
{.value = MCA_BTL_SM_NONE, .string = "none"},
|
||||
{.value = 0, .string = NULL}
|
||||
};
|
||||
|
||||
/*
|
||||
* Shared Memory (VADER) component instance.
|
||||
* Shared Memory (SM) component instance.
|
||||
*/
|
||||
mca_btl_vader_component_t mca_btl_vader_component = {
|
||||
.super = {
|
||||
/* First, the mca_base_component_t struct containing meta information
|
||||
about the component itself */
|
||||
.btl_version = {
|
||||
MCA_BTL_DEFAULT_VERSION("vader"),
|
||||
.mca_open_component = mca_btl_vader_component_open,
|
||||
.mca_close_component = mca_btl_vader_component_close,
|
||||
.mca_register_component_params = mca_btl_vader_component_register,
|
||||
},
|
||||
.btl_data = {
|
||||
/* The component is checkpoint ready */
|
||||
.param_field = MCA_BASE_METADATA_PARAM_CHECKPOINT
|
||||
},
|
||||
mca_btl_sm_component_t mca_btl_sm_component = {
|
||||
.super = {
|
||||
/* First, the mca_base_component_t struct containing meta information
|
||||
about the component itself */
|
||||
.btl_version = {
|
||||
MCA_BTL_DEFAULT_VERSION("sm"),
|
||||
.mca_open_component = mca_btl_sm_component_open,
|
||||
.mca_close_component = mca_btl_sm_component_close,
|
||||
.mca_register_component_params = mca_btl_sm_component_register,
|
||||
},
|
||||
.btl_data = {
|
||||
/* The component is checkpoint ready */
|
||||
.param_field = MCA_BASE_METADATA_PARAM_CHECKPOINT
|
||||
},
|
||||
|
||||
.btl_init = mca_btl_vader_component_init,
|
||||
.btl_progress = mca_btl_vader_component_progress,
|
||||
.btl_init = mca_btl_sm_component_init,
|
||||
.btl_progress = mca_btl_sm_component_progress,
|
||||
} /* end super */
|
||||
};
|
||||
|
||||
static int mca_btl_vader_component_register (void)
|
||||
static int mca_btl_sm_component_register (void)
|
||||
{
|
||||
mca_base_var_enum_t *new_enum;
|
||||
|
||||
(void) mca_base_var_group_component_register(&mca_btl_vader_component.super.btl_version,
|
||||
(void) mca_base_var_group_component_register(&mca_btl_sm_component.super.btl_version,
|
||||
"Enhanced shared memory byte transport later");
|
||||
|
||||
/* register VADER component variables */
|
||||
mca_btl_vader_component.vader_free_list_num = 8;
|
||||
(void) mca_base_component_var_register(&mca_btl_vader_component.super.btl_version,
|
||||
/* register SM component variables */
|
||||
mca_btl_sm_component.sm_free_list_num = 8;
|
||||
(void) mca_base_component_var_register(&mca_btl_sm_component.super.btl_version,
|
||||
"free_list_num", "Initial number of fragments "
|
||||
"to allocate for shared memory communication.",
|
||||
MCA_BASE_VAR_TYPE_INT, NULL, 0,
|
||||
MCA_BASE_VAR_FLAG_SETTABLE, OPAL_INFO_LVL_9,
|
||||
MCA_BASE_VAR_SCOPE_LOCAL,
|
||||
&mca_btl_vader_component.vader_free_list_num);
|
||||
mca_btl_vader_component.vader_free_list_max = 512;
|
||||
(void) mca_base_component_var_register(&mca_btl_vader_component.super.btl_version,
|
||||
&mca_btl_sm_component.sm_free_list_num);
|
||||
mca_btl_sm_component.sm_free_list_max = 512;
|
||||
(void) mca_base_component_var_register(&mca_btl_sm_component.super.btl_version,
|
||||
"free_list_max", "Maximum number of fragments "
|
||||
"to allocate for shared memory communication.",
|
||||
MCA_BASE_VAR_TYPE_INT, NULL, 0,
|
||||
MCA_BASE_VAR_FLAG_SETTABLE, OPAL_INFO_LVL_9,
|
||||
MCA_BASE_VAR_SCOPE_LOCAL,
|
||||
&mca_btl_vader_component.vader_free_list_max);
|
||||
mca_btl_vader_component.vader_free_list_inc = 64;
|
||||
(void) mca_base_component_var_register(&mca_btl_vader_component.super.btl_version,
|
||||
&mca_btl_sm_component.sm_free_list_max);
|
||||
mca_btl_sm_component.sm_free_list_inc = 64;
|
||||
(void) mca_base_component_var_register(&mca_btl_sm_component.super.btl_version,
|
||||
"free_list_inc", "Number of fragments to create "
|
||||
"on each allocation.", MCA_BASE_VAR_TYPE_INT, NULL, 0,
|
||||
MCA_BASE_VAR_FLAG_SETTABLE, OPAL_INFO_LVL_9,
|
||||
MCA_BASE_VAR_SCOPE_LOCAL,
|
||||
&mca_btl_vader_component.vader_free_list_inc);
|
||||
&mca_btl_sm_component.sm_free_list_inc);
|
||||
|
||||
mca_btl_vader_component.memcpy_limit = 524288;
|
||||
(void) mca_base_component_var_register(&mca_btl_vader_component.super.btl_version,
|
||||
mca_btl_sm_component.memcpy_limit = 524288;
|
||||
(void) mca_base_component_var_register(&mca_btl_sm_component.super.btl_version,
|
||||
"memcpy_limit", "Message size to switch from using "
|
||||
"memove to memcpy. The relative speed of these two "
|
||||
"routines can vary by size.", MCA_BASE_VAR_TYPE_INT,
|
||||
NULL, 0, MCA_BASE_VAR_FLAG_SETTABLE, OPAL_INFO_LVL_5,
|
||||
MCA_BASE_VAR_SCOPE_LOCAL,
|
||||
&mca_btl_vader_component.memcpy_limit);
|
||||
#if OPAL_BTL_VADER_HAVE_XPMEM
|
||||
mca_btl_vader_component.log_attach_align = 21;
|
||||
(void) mca_base_component_var_register(&mca_btl_vader_component.super.btl_version,
|
||||
&mca_btl_sm_component.memcpy_limit);
|
||||
#if OPAL_BTL_SM_HAVE_XPMEM
|
||||
mca_btl_sm_component.log_attach_align = 21;
|
||||
(void) mca_base_component_var_register(&mca_btl_sm_component.super.btl_version,
|
||||
"log_align", "Log base 2 of the alignment to use for xpmem "
|
||||
"segments (default: 21, minimum: 12, maximum: 25)",
|
||||
MCA_BASE_VAR_TYPE_INT, NULL, 0,
|
||||
MCA_BASE_VAR_FLAG_SETTABLE, OPAL_INFO_LVL_5,
|
||||
MCA_BASE_VAR_SCOPE_LOCAL,
|
||||
&mca_btl_vader_component.log_attach_align);
|
||||
&mca_btl_sm_component.log_attach_align);
|
||||
#endif
|
||||
|
||||
#if OPAL_BTL_VADER_HAVE_XPMEM && 64 == MCA_BTL_VADER_BITNESS
|
||||
mca_btl_vader_component.segment_size = 1 << 24;
|
||||
#if OPAL_BTL_SM_HAVE_XPMEM && 64 == MCA_BTL_SM_BITNESS
|
||||
mca_btl_sm_component.segment_size = 1 << 24;
|
||||
#else
|
||||
mca_btl_vader_component.segment_size = 1 << 22;
|
||||
mca_btl_sm_component.segment_size = 1 << 22;
|
||||
#endif
|
||||
(void) mca_base_component_var_register(&mca_btl_vader_component.super.btl_version,
|
||||
(void) mca_base_component_var_register(&mca_btl_sm_component.super.btl_version,
|
||||
"segment_size", "Maximum size of all shared "
|
||||
#if OPAL_BTL_VADER_HAVE_XPMEM && 64 == MCA_BTL_VADER_BITNESS
|
||||
#if OPAL_BTL_SM_HAVE_XPMEM && 64 == MCA_BTL_SM_BITNESS
|
||||
"memory buffers (default: 16M)",
|
||||
#else
|
||||
"memory buffers (default: 4M)",
|
||||
@ -171,130 +171,130 @@ static int mca_btl_vader_component_register (void)
|
||||
MCA_BASE_VAR_TYPE_INT, NULL, 0,
|
||||
MCA_BASE_VAR_FLAG_SETTABLE, OPAL_INFO_LVL_5,
|
||||
MCA_BASE_VAR_SCOPE_LOCAL,
|
||||
&mca_btl_vader_component.segment_size);
|
||||
&mca_btl_sm_component.segment_size);
|
||||
|
||||
mca_btl_vader_component.max_inline_send = 256;
|
||||
(void) mca_base_component_var_register(&mca_btl_vader_component.super.btl_version,
|
||||
mca_btl_sm_component.max_inline_send = 256;
|
||||
(void) mca_base_component_var_register(&mca_btl_sm_component.super.btl_version,
|
||||
"max_inline_send", "Maximum size to transfer "
|
||||
"using copy-in copy-out semantics",
|
||||
MCA_BASE_VAR_TYPE_UNSIGNED_INT, NULL, 0,
|
||||
MCA_BASE_VAR_FLAG_SETTABLE, OPAL_INFO_LVL_5,
|
||||
MCA_BASE_VAR_SCOPE_LOCAL,
|
||||
&mca_btl_vader_component.max_inline_send);
|
||||
&mca_btl_sm_component.max_inline_send);
|
||||
|
||||
mca_btl_vader_component.fbox_threshold = 16;
|
||||
(void) mca_base_component_var_register(&mca_btl_vader_component.super.btl_version,
|
||||
mca_btl_sm_component.fbox_threshold = 16;
|
||||
(void) mca_base_component_var_register(&mca_btl_sm_component.super.btl_version,
|
||||
"fbox_threshold", "Number of sends required "
|
||||
"before an eager send buffer is setup for a peer "
|
||||
"(default: 16)", MCA_BASE_VAR_TYPE_UNSIGNED_INT, NULL,
|
||||
0, MCA_BASE_VAR_FLAG_SETTABLE, OPAL_INFO_LVL_5,
|
||||
MCA_BASE_VAR_SCOPE_LOCAL, &mca_btl_vader_component.fbox_threshold);
|
||||
MCA_BASE_VAR_SCOPE_LOCAL, &mca_btl_sm_component.fbox_threshold);
|
||||
|
||||
mca_btl_vader_component.fbox_max = 32;
|
||||
(void) mca_base_component_var_register(&mca_btl_vader_component.super.btl_version,
|
||||
mca_btl_sm_component.fbox_max = 32;
|
||||
(void) mca_base_component_var_register(&mca_btl_sm_component.super.btl_version,
|
||||
"fbox_max", "Maximum number of eager send buffers "
|
||||
"to allocate (default: 32)", MCA_BASE_VAR_TYPE_UNSIGNED_INT,
|
||||
NULL, 0, MCA_BASE_VAR_FLAG_SETTABLE, OPAL_INFO_LVL_5,
|
||||
MCA_BASE_VAR_SCOPE_LOCAL, &mca_btl_vader_component.fbox_max);
|
||||
MCA_BASE_VAR_SCOPE_LOCAL, &mca_btl_sm_component.fbox_max);
|
||||
|
||||
mca_btl_vader_component.fbox_size = 4096;
|
||||
(void) mca_base_component_var_register(&mca_btl_vader_component.super.btl_version,
|
||||
mca_btl_sm_component.fbox_size = 4096;
|
||||
(void) mca_base_component_var_register(&mca_btl_sm_component.super.btl_version,
|
||||
"fbox_size", "Size of per-peer fast transfer buffers (default: 4k)",
|
||||
MCA_BASE_VAR_TYPE_UNSIGNED_INT, NULL, 0, MCA_BASE_VAR_FLAG_SETTABLE,
|
||||
OPAL_INFO_LVL_5, MCA_BASE_VAR_SCOPE_LOCAL, &mca_btl_vader_component.fbox_size);
|
||||
OPAL_INFO_LVL_5, MCA_BASE_VAR_SCOPE_LOCAL, &mca_btl_sm_component.fbox_size);
|
||||
|
||||
(void) mca_base_var_enum_create ("btl_vader_single_copy_mechanisms", single_copy_mechanisms, &new_enum);
|
||||
(void) mca_base_var_enum_create ("btl_sm_single_copy_mechanisms", single_copy_mechanisms, &new_enum);
|
||||
|
||||
/* Default to the best available mechanism (see the enumerator for ordering) */
|
||||
mca_btl_vader_component.single_copy_mechanism = single_copy_mechanisms[0].value;
|
||||
(void) mca_base_component_var_register(&mca_btl_vader_component.super.btl_version,
|
||||
mca_btl_sm_component.single_copy_mechanism = single_copy_mechanisms[0].value;
|
||||
(void) mca_base_component_var_register(&mca_btl_sm_component.super.btl_version,
|
||||
"single_copy_mechanism", "Single copy mechanism to use (defaults to best available)",
|
||||
MCA_BASE_VAR_TYPE_INT, new_enum, 0, MCA_BASE_VAR_FLAG_SETTABLE,
|
||||
OPAL_INFO_LVL_3, MCA_BASE_VAR_SCOPE_GROUP, &mca_btl_vader_component.single_copy_mechanism);
|
||||
OPAL_INFO_LVL_3, MCA_BASE_VAR_SCOPE_GROUP, &mca_btl_sm_component.single_copy_mechanism);
|
||||
OBJ_RELEASE(new_enum);
|
||||
|
||||
if (0 == access ("/dev/shm", W_OK)) {
|
||||
mca_btl_vader_component.backing_directory = "/dev/shm";
|
||||
mca_btl_sm_component.backing_directory = "/dev/shm";
|
||||
} else {
|
||||
mca_btl_vader_component.backing_directory = opal_process_info.job_session_dir;
|
||||
mca_btl_sm_component.backing_directory = opal_process_info.job_session_dir;
|
||||
}
|
||||
(void) mca_base_component_var_register (&mca_btl_vader_component.super.btl_version, "backing_directory",
|
||||
(void) mca_base_component_var_register (&mca_btl_sm_component.super.btl_version, "backing_directory",
|
||||
"Directory to place backing files for shared memory communication. "
|
||||
"This directory should be on a local filesystem such as /tmp or "
|
||||
"/dev/shm (default: (linux) /dev/shm, (others) session directory)",
|
||||
MCA_BASE_VAR_TYPE_STRING, NULL, 0, 0, OPAL_INFO_LVL_3,
|
||||
MCA_BASE_VAR_SCOPE_READONLY, &mca_btl_vader_component.backing_directory);
|
||||
MCA_BASE_VAR_SCOPE_READONLY, &mca_btl_sm_component.backing_directory);
|
||||
|
||||
|
||||
#if OPAL_BTL_VADER_HAVE_KNEM
|
||||
#if OPAL_BTL_SM_HAVE_KNEM
|
||||
/* Currently disabling DMA mode by default; it's not clear that this is useful in all applications and architectures. */
|
||||
mca_btl_vader_component.knem_dma_min = 0;
|
||||
(void) mca_base_component_var_register(&mca_btl_vader_component.super.btl_version, "knem_dma_min",
|
||||
mca_btl_sm_component.knem_dma_min = 0;
|
||||
(void) mca_base_component_var_register(&mca_btl_sm_component.super.btl_version, "knem_dma_min",
|
||||
"Minimum message size (in bytes) to use the knem DMA mode; "
|
||||
"ignored if knem does not support DMA mode (0 = do not use the "
|
||||
"knem DMA mode, default: 0)", MCA_BASE_VAR_TYPE_UNSIGNED_INT,
|
||||
NULL, 0, 0, OPAL_INFO_LVL_9, MCA_BASE_VAR_SCOPE_READONLY,
|
||||
&mca_btl_vader_component.knem_dma_min);
|
||||
&mca_btl_sm_component.knem_dma_min);
|
||||
#endif
|
||||
|
||||
mca_btl_vader.super.btl_exclusivity = MCA_BTL_EXCLUSIVITY_HIGH;
|
||||
mca_btl_sm.super.btl_exclusivity = MCA_BTL_EXCLUSIVITY_HIGH;
|
||||
|
||||
if (MCA_BTL_VADER_XPMEM == mca_btl_vader_component.single_copy_mechanism) {
|
||||
mca_btl_vader.super.btl_eager_limit = 32 * 1024;
|
||||
mca_btl_vader.super.btl_rndv_eager_limit = mca_btl_vader.super.btl_eager_limit;
|
||||
mca_btl_vader.super.btl_max_send_size = mca_btl_vader.super.btl_eager_limit;
|
||||
mca_btl_vader.super.btl_min_rdma_pipeline_size = INT_MAX;
|
||||
if (MCA_BTL_SM_XPMEM == mca_btl_sm_component.single_copy_mechanism) {
|
||||
mca_btl_sm.super.btl_eager_limit = 32 * 1024;
|
||||
mca_btl_sm.super.btl_rndv_eager_limit = mca_btl_sm.super.btl_eager_limit;
|
||||
mca_btl_sm.super.btl_max_send_size = mca_btl_sm.super.btl_eager_limit;
|
||||
mca_btl_sm.super.btl_min_rdma_pipeline_size = INT_MAX;
|
||||
} else {
|
||||
mca_btl_vader.super.btl_eager_limit = 4 * 1024;
|
||||
mca_btl_vader.super.btl_rndv_eager_limit = 32 * 1024;
|
||||
mca_btl_vader.super.btl_max_send_size = 32 * 1024;
|
||||
mca_btl_vader.super.btl_min_rdma_pipeline_size = INT_MAX;
|
||||
mca_btl_sm.super.btl_eager_limit = 4 * 1024;
|
||||
mca_btl_sm.super.btl_rndv_eager_limit = 32 * 1024;
|
||||
mca_btl_sm.super.btl_max_send_size = 32 * 1024;
|
||||
mca_btl_sm.super.btl_min_rdma_pipeline_size = INT_MAX;
|
||||
}
|
||||
|
||||
mca_btl_vader.super.btl_rdma_pipeline_send_length = mca_btl_vader.super.btl_eager_limit;
|
||||
mca_btl_vader.super.btl_rdma_pipeline_frag_size = mca_btl_vader.super.btl_eager_limit;
|
||||
mca_btl_sm.super.btl_rdma_pipeline_send_length = mca_btl_sm.super.btl_eager_limit;
|
||||
mca_btl_sm.super.btl_rdma_pipeline_frag_size = mca_btl_sm.super.btl_eager_limit;
|
||||
|
||||
#if OPAL_HAVE_ATOMIC_MATH_64
|
||||
mca_btl_vader.super.btl_flags = MCA_BTL_FLAGS_SEND_INPLACE | MCA_BTL_FLAGS_SEND | MCA_BTL_FLAGS_RDMA |
|
||||
mca_btl_sm.super.btl_flags = MCA_BTL_FLAGS_SEND_INPLACE | MCA_BTL_FLAGS_SEND | MCA_BTL_FLAGS_RDMA |
|
||||
MCA_BTL_FLAGS_ATOMIC_OPS | MCA_BTL_FLAGS_ATOMIC_FOPS;
|
||||
|
||||
mca_btl_vader.super.btl_atomic_flags = MCA_BTL_ATOMIC_SUPPORTS_ADD | MCA_BTL_ATOMIC_SUPPORTS_AND |
|
||||
mca_btl_sm.super.btl_atomic_flags = MCA_BTL_ATOMIC_SUPPORTS_ADD | MCA_BTL_ATOMIC_SUPPORTS_AND |
|
||||
MCA_BTL_ATOMIC_SUPPORTS_OR | MCA_BTL_ATOMIC_SUPPORTS_XOR | MCA_BTL_ATOMIC_SUPPORTS_CSWAP |
|
||||
MCA_BTL_ATOMIC_SUPPORTS_GLOB | MCA_BTL_ATOMIC_SUPPORTS_SWAP;
|
||||
#if OPAL_HAVE_ATOMIC_MATH_32
|
||||
mca_btl_vader.super.btl_atomic_flags |= MCA_BTL_ATOMIC_SUPPORTS_32BIT;
|
||||
mca_btl_sm.super.btl_atomic_flags |= MCA_BTL_ATOMIC_SUPPORTS_32BIT;
|
||||
#endif /* OPAL_HAVE_ATOMIC_MATH_32 */
|
||||
|
||||
#if OPAL_HAVE_ATOMIC_MIN_64
|
||||
mca_btl_vader.super.btl_atomic_flags |= MCA_BTL_ATOMIC_SUPPORTS_MIN;
|
||||
mca_btl_sm.super.btl_atomic_flags |= MCA_BTL_ATOMIC_SUPPORTS_MIN;
|
||||
#endif /* OPAL_HAVE_ATOMIC_MIN_64 */
|
||||
|
||||
#if OPAL_HAVE_ATOMIC_MAX_64
|
||||
mca_btl_vader.super.btl_atomic_flags |= MCA_BTL_ATOMIC_SUPPORTS_MAX;
|
||||
mca_btl_sm.super.btl_atomic_flags |= MCA_BTL_ATOMIC_SUPPORTS_MAX;
|
||||
#endif /* OPAL_HAVE_ATOMIC_MAX_64 */
|
||||
|
||||
#else
|
||||
mca_btl_vader.super.btl_flags = MCA_BTL_FLAGS_SEND_INPLACE | MCA_BTL_FLAGS_SEND | MCA_BTL_FLAGS_RDMA;
|
||||
mca_btl_sm.super.btl_flags = MCA_BTL_FLAGS_SEND_INPLACE | MCA_BTL_FLAGS_SEND | MCA_BTL_FLAGS_RDMA;
|
||||
#endif /* OPAL_HAVE_ATOMIC_MATH_64 */
|
||||
|
||||
if (MCA_BTL_VADER_NONE != mca_btl_vader_component.single_copy_mechanism) {
|
||||
if (MCA_BTL_SM_NONE != mca_btl_sm_component.single_copy_mechanism) {
|
||||
/* True single copy mechanisms should provide better bandwidth */
|
||||
mca_btl_vader.super.btl_bandwidth = 40000; /* Mbs */
|
||||
mca_btl_sm.super.btl_bandwidth = 40000; /* Mbs */
|
||||
} else {
|
||||
mca_btl_vader.super.btl_bandwidth = 10000; /* Mbs */
|
||||
mca_btl_sm.super.btl_bandwidth = 10000; /* Mbs */
|
||||
}
|
||||
|
||||
mca_btl_vader.super.btl_get = mca_btl_vader_get_sc_emu;
|
||||
mca_btl_vader.super.btl_put = mca_btl_vader_put_sc_emu;
|
||||
mca_btl_vader.super.btl_atomic_op = mca_btl_vader_emu_aop;
|
||||
mca_btl_vader.super.btl_atomic_fop = mca_btl_vader_emu_afop;
|
||||
mca_btl_vader.super.btl_atomic_cswap = mca_btl_vader_emu_acswap;
|
||||
mca_btl_sm.super.btl_get = mca_btl_sm_get_sc_emu;
|
||||
mca_btl_sm.super.btl_put = mca_btl_sm_put_sc_emu;
|
||||
mca_btl_sm.super.btl_atomic_op = mca_btl_sm_emu_aop;
|
||||
mca_btl_sm.super.btl_atomic_fop = mca_btl_sm_emu_afop;
|
||||
mca_btl_sm.super.btl_atomic_cswap = mca_btl_sm_emu_acswap;
|
||||
|
||||
mca_btl_vader.super.btl_latency = 1; /* Microsecs */
|
||||
mca_btl_sm.super.btl_latency = 1; /* Microsecs */
|
||||
|
||||
/* Call the BTL based to register its MCA params */
|
||||
mca_btl_base_param_register(&mca_btl_vader_component.super.btl_version,
|
||||
&mca_btl_vader.super);
|
||||
mca_btl_base_param_register(&mca_btl_sm_component.super.btl_version,
|
||||
&mca_btl_sm.super);
|
||||
|
||||
return OPAL_SUCCESS;
|
||||
}
|
||||
@ -304,18 +304,18 @@ static int mca_btl_vader_component_register (void)
|
||||
* component parameters.
|
||||
*/
|
||||
|
||||
static int mca_btl_vader_component_open(void)
|
||||
static int mca_btl_sm_component_open(void)
|
||||
{
|
||||
/* initialize objects */
|
||||
OBJ_CONSTRUCT(&mca_btl_vader_component.vader_frags_eager, opal_free_list_t);
|
||||
OBJ_CONSTRUCT(&mca_btl_vader_component.vader_frags_user, opal_free_list_t);
|
||||
OBJ_CONSTRUCT(&mca_btl_vader_component.vader_frags_max_send, opal_free_list_t);
|
||||
OBJ_CONSTRUCT(&mca_btl_vader_component.vader_fboxes, opal_free_list_t);
|
||||
OBJ_CONSTRUCT(&mca_btl_vader_component.lock, opal_mutex_t);
|
||||
OBJ_CONSTRUCT(&mca_btl_vader_component.pending_endpoints, opal_list_t);
|
||||
OBJ_CONSTRUCT(&mca_btl_vader_component.pending_fragments, opal_list_t);
|
||||
#if OPAL_BTL_VADER_HAVE_KNEM
|
||||
mca_btl_vader.knem_fd = -1;
|
||||
OBJ_CONSTRUCT(&mca_btl_sm_component.sm_frags_eager, opal_free_list_t);
|
||||
OBJ_CONSTRUCT(&mca_btl_sm_component.sm_frags_user, opal_free_list_t);
|
||||
OBJ_CONSTRUCT(&mca_btl_sm_component.sm_frags_max_send, opal_free_list_t);
|
||||
OBJ_CONSTRUCT(&mca_btl_sm_component.sm_fboxes, opal_free_list_t);
|
||||
OBJ_CONSTRUCT(&mca_btl_sm_component.lock, opal_mutex_t);
|
||||
OBJ_CONSTRUCT(&mca_btl_sm_component.pending_endpoints, opal_list_t);
|
||||
OBJ_CONSTRUCT(&mca_btl_sm_component.pending_fragments, opal_list_t);
|
||||
#if OPAL_BTL_SM_HAVE_KNEM
|
||||
mca_btl_sm.knem_fd = -1;
|
||||
#endif
|
||||
|
||||
return OPAL_SUCCESS;
|
||||
@ -326,41 +326,41 @@ static int mca_btl_vader_component_open(void)
|
||||
* component cleanup - sanity checking of queue lengths
|
||||
*/
|
||||
|
||||
static int mca_btl_vader_component_close(void)
|
||||
static int mca_btl_sm_component_close(void)
|
||||
{
|
||||
OBJ_DESTRUCT(&mca_btl_vader_component.vader_frags_eager);
|
||||
OBJ_DESTRUCT(&mca_btl_vader_component.vader_frags_user);
|
||||
OBJ_DESTRUCT(&mca_btl_vader_component.vader_frags_max_send);
|
||||
OBJ_DESTRUCT(&mca_btl_vader_component.vader_fboxes);
|
||||
OBJ_DESTRUCT(&mca_btl_vader_component.lock);
|
||||
OBJ_DESTRUCT(&mca_btl_vader_component.pending_endpoints);
|
||||
OBJ_DESTRUCT(&mca_btl_vader_component.pending_fragments);
|
||||
OBJ_DESTRUCT(&mca_btl_sm_component.sm_frags_eager);
|
||||
OBJ_DESTRUCT(&mca_btl_sm_component.sm_frags_user);
|
||||
OBJ_DESTRUCT(&mca_btl_sm_component.sm_frags_max_send);
|
||||
OBJ_DESTRUCT(&mca_btl_sm_component.sm_fboxes);
|
||||
OBJ_DESTRUCT(&mca_btl_sm_component.lock);
|
||||
OBJ_DESTRUCT(&mca_btl_sm_component.pending_endpoints);
|
||||
OBJ_DESTRUCT(&mca_btl_sm_component.pending_fragments);
|
||||
|
||||
if (MCA_BTL_VADER_XPMEM == mca_btl_vader_component.single_copy_mechanism &&
|
||||
NULL != mca_btl_vader_component.my_segment) {
|
||||
munmap (mca_btl_vader_component.my_segment, mca_btl_vader_component.segment_size);
|
||||
if (MCA_BTL_SM_XPMEM == mca_btl_sm_component.single_copy_mechanism &&
|
||||
NULL != mca_btl_sm_component.my_segment) {
|
||||
munmap (mca_btl_sm_component.my_segment, mca_btl_sm_component.segment_size);
|
||||
}
|
||||
|
||||
mca_btl_vader_component.my_segment = NULL;
|
||||
mca_btl_sm_component.my_segment = NULL;
|
||||
|
||||
#if OPAL_BTL_VADER_HAVE_KNEM
|
||||
mca_btl_vader_knem_fini ();
|
||||
#if OPAL_BTL_SM_HAVE_KNEM
|
||||
mca_btl_sm_knem_fini ();
|
||||
#endif
|
||||
|
||||
if (mca_btl_vader_component.mpool) {
|
||||
mca_btl_vader_component.mpool->mpool_finalize (mca_btl_vader_component.mpool);
|
||||
mca_btl_vader_component.mpool = NULL;
|
||||
if (mca_btl_sm_component.mpool) {
|
||||
mca_btl_sm_component.mpool->mpool_finalize (mca_btl_sm_component.mpool);
|
||||
mca_btl_sm_component.mpool = NULL;
|
||||
}
|
||||
|
||||
return OPAL_SUCCESS;
|
||||
}
|
||||
|
||||
/*
|
||||
* mca_btl_vader_parse_proc_ns_user() tries to get the user namespace ID
|
||||
* mca_btl_sm_parse_proc_ns_user() tries to get the user namespace ID
|
||||
* of the current process.
|
||||
* Returns the ID of the user namespace. In the case of an error '0' is returned.
|
||||
*/
|
||||
ino_t mca_btl_vader_get_user_ns_id(void)
|
||||
ino_t mca_btl_sm_get_user_ns_id(void)
|
||||
{
|
||||
struct stat buf;
|
||||
|
||||
@ -374,23 +374,23 @@ ino_t mca_btl_vader_get_user_ns_id(void)
|
||||
|
||||
return buf.st_ino;
|
||||
}
|
||||
static int mca_btl_base_vader_modex_send (void)
|
||||
static int mca_btl_base_sm_modex_send (void)
|
||||
{
|
||||
union vader_modex_t modex;
|
||||
union sm_modex_t modex;
|
||||
int modex_size, rc;
|
||||
|
||||
#if OPAL_BTL_VADER_HAVE_XPMEM
|
||||
if (MCA_BTL_VADER_XPMEM == mca_btl_vader_component.single_copy_mechanism) {
|
||||
modex.xpmem.seg_id = mca_btl_vader_component.my_seg_id;
|
||||
modex.xpmem.segment_base = mca_btl_vader_component.my_segment;
|
||||
modex.xpmem.address_max = mca_btl_vader_component.my_address_max;
|
||||
#if OPAL_BTL_SM_HAVE_XPMEM
|
||||
if (MCA_BTL_SM_XPMEM == mca_btl_sm_component.single_copy_mechanism) {
|
||||
modex.xpmem.seg_id = mca_btl_sm_component.my_seg_id;
|
||||
modex.xpmem.segment_base = mca_btl_sm_component.my_segment;
|
||||
modex.xpmem.address_max = mca_btl_sm_component.my_address_max;
|
||||
|
||||
modex_size = sizeof (modex.xpmem);
|
||||
} else {
|
||||
#endif
|
||||
modex.other.seg_ds_size = opal_shmem_sizeof_shmem_ds (&mca_btl_vader_component.seg_ds);
|
||||
memmove (&modex.other.seg_ds, &mca_btl_vader_component.seg_ds, modex.other.seg_ds_size);
|
||||
modex.other.user_ns_id = mca_btl_vader_get_user_ns_id();
|
||||
modex.other.seg_ds_size = opal_shmem_sizeof_shmem_ds (&mca_btl_sm_component.seg_ds);
|
||||
memmove (&modex.other.seg_ds, &mca_btl_sm_component.seg_ds, modex.other.seg_ds_size);
|
||||
modex.other.user_ns_id = mca_btl_sm_get_user_ns_id();
|
||||
/*
|
||||
* If modex.other.user_ns_id is '0' something did not work out
|
||||
* during user namespace detection. Assuming there are no
|
||||
@ -399,55 +399,55 @@ static int mca_btl_base_vader_modex_send (void)
|
||||
*/
|
||||
modex_size = sizeof (modex.other);
|
||||
|
||||
#if OPAL_BTL_VADER_HAVE_XPMEM
|
||||
#if OPAL_BTL_SM_HAVE_XPMEM
|
||||
}
|
||||
#endif
|
||||
|
||||
OPAL_MODEX_SEND(rc, PMIX_LOCAL,
|
||||
&mca_btl_vader_component.super.btl_version, &modex, modex_size);
|
||||
&mca_btl_sm_component.super.btl_version, &modex, modex_size);
|
||||
|
||||
return rc;
|
||||
}
|
||||
|
||||
#if OPAL_BTL_VADER_HAVE_XPMEM || OPAL_BTL_VADER_HAVE_CMA || OPAL_BTL_VADER_HAVE_KNEM
|
||||
static void mca_btl_vader_select_next_single_copy_mechanism (void)
|
||||
#if OPAL_BTL_SM_HAVE_XPMEM || OPAL_BTL_SM_HAVE_CMA || OPAL_BTL_SM_HAVE_KNEM
|
||||
static void mca_btl_sm_select_next_single_copy_mechanism (void)
|
||||
{
|
||||
for (int i = 0 ; single_copy_mechanisms[i].value != MCA_BTL_VADER_NONE ; ++i) {
|
||||
if (single_copy_mechanisms[i].value == mca_btl_vader_component.single_copy_mechanism) {
|
||||
mca_btl_vader_component.single_copy_mechanism = single_copy_mechanisms[i+1].value;
|
||||
for (int i = 0 ; single_copy_mechanisms[i].value != MCA_BTL_SM_NONE ; ++i) {
|
||||
if (single_copy_mechanisms[i].value == mca_btl_sm_component.single_copy_mechanism) {
|
||||
mca_btl_sm_component.single_copy_mechanism = single_copy_mechanisms[i+1].value;
|
||||
return;
|
||||
}
|
||||
}
|
||||
}
|
||||
#endif
|
||||
|
||||
static void mca_btl_vader_check_single_copy (void)
|
||||
static void mca_btl_sm_check_single_copy (void)
|
||||
{
|
||||
#if OPAL_BTL_VADER_HAVE_XPMEM || OPAL_BTL_VADER_HAVE_CMA || OPAL_BTL_VADER_HAVE_KNEM
|
||||
int initial_mechanism = mca_btl_vader_component.single_copy_mechanism;
|
||||
#if OPAL_BTL_SM_HAVE_XPMEM || OPAL_BTL_SM_HAVE_CMA || OPAL_BTL_SM_HAVE_KNEM
|
||||
int initial_mechanism = mca_btl_sm_component.single_copy_mechanism;
|
||||
#endif
|
||||
|
||||
/* single-copy emulation is always used to support AMO's right now */
|
||||
mca_btl_vader_sc_emu_init ();
|
||||
mca_btl_sm_sc_emu_init ();
|
||||
|
||||
#if OPAL_BTL_VADER_HAVE_XPMEM
|
||||
if (MCA_BTL_VADER_XPMEM == mca_btl_vader_component.single_copy_mechanism) {
|
||||
#if OPAL_BTL_SM_HAVE_XPMEM
|
||||
if (MCA_BTL_SM_XPMEM == mca_btl_sm_component.single_copy_mechanism) {
|
||||
/* try to create an xpmem segment for the entire address space */
|
||||
int rc = mca_btl_vader_xpmem_init ();
|
||||
int rc = mca_btl_sm_xpmem_init ();
|
||||
if (OPAL_SUCCESS != rc) {
|
||||
if (MCA_BTL_VADER_XPMEM == initial_mechanism) {
|
||||
opal_show_help("help-btl-vader.txt", "xpmem-make-failed",
|
||||
if (MCA_BTL_SM_XPMEM == initial_mechanism) {
|
||||
opal_show_help("help-btl-sm.txt", "xpmem-make-failed",
|
||||
true, opal_process_info.nodename, errno,
|
||||
strerror(errno));
|
||||
}
|
||||
|
||||
mca_btl_vader_select_next_single_copy_mechanism ();
|
||||
mca_btl_sm_select_next_single_copy_mechanism ();
|
||||
}
|
||||
}
|
||||
#endif
|
||||
|
||||
#if OPAL_BTL_VADER_HAVE_CMA
|
||||
if (MCA_BTL_VADER_CMA == mca_btl_vader_component.single_copy_mechanism) {
|
||||
#if OPAL_BTL_SM_HAVE_CMA
|
||||
if (MCA_BTL_SM_CMA == mca_btl_sm_component.single_copy_mechanism) {
|
||||
/* Check if we have the proper permissions for CMA */
|
||||
char buffer = '0';
|
||||
bool cma_happy = false;
|
||||
@ -476,63 +476,63 @@ static void mca_btl_vader_check_single_copy (void)
|
||||
}
|
||||
|
||||
if (!cma_happy) {
|
||||
mca_btl_vader_select_next_single_copy_mechanism ();
|
||||
mca_btl_sm_select_next_single_copy_mechanism ();
|
||||
|
||||
if (MCA_BTL_VADER_CMA == initial_mechanism) {
|
||||
opal_show_help("help-btl-vader.txt", "cma-permission-denied",
|
||||
if (MCA_BTL_SM_CMA == initial_mechanism) {
|
||||
opal_show_help("help-btl-sm.txt", "cma-permission-denied",
|
||||
true, opal_process_info.nodename);
|
||||
}
|
||||
} else {
|
||||
/* ptrace_scope will allow CMA */
|
||||
mca_btl_vader.super.btl_get = mca_btl_vader_get_cma;
|
||||
mca_btl_vader.super.btl_put = mca_btl_vader_put_cma;
|
||||
mca_btl_sm.super.btl_get = mca_btl_sm_get_cma;
|
||||
mca_btl_sm.super.btl_put = mca_btl_sm_put_cma;
|
||||
}
|
||||
}
|
||||
#endif
|
||||
|
||||
#if OPAL_BTL_VADER_HAVE_KNEM
|
||||
if (MCA_BTL_VADER_KNEM == mca_btl_vader_component.single_copy_mechanism) {
|
||||
/* mca_btl_vader_knem_init will set the appropriate get/put functions */
|
||||
int rc = mca_btl_vader_knem_init ();
|
||||
#if OPAL_BTL_SM_HAVE_KNEM
|
||||
if (MCA_BTL_SM_KNEM == mca_btl_sm_component.single_copy_mechanism) {
|
||||
/* mca_btl_sm_knem_init will set the appropriate get/put functions */
|
||||
int rc = mca_btl_sm_knem_init ();
|
||||
if (OPAL_SUCCESS != rc) {
|
||||
if (MCA_BTL_VADER_KNEM == initial_mechanism) {
|
||||
opal_show_help("help-btl-vader.txt", "knem requested but not available",
|
||||
if (MCA_BTL_SM_KNEM == initial_mechanism) {
|
||||
opal_show_help("help-btl-sm.txt", "knem requested but not available",
|
||||
true, opal_process_info.nodename);
|
||||
}
|
||||
|
||||
/* disable single copy */
|
||||
mca_btl_vader_select_next_single_copy_mechanism ();
|
||||
mca_btl_sm_select_next_single_copy_mechanism ();
|
||||
}
|
||||
}
|
||||
#endif
|
||||
|
||||
if (MCA_BTL_VADER_NONE == mca_btl_vader_component.single_copy_mechanism) {
|
||||
mca_btl_vader.super.btl_flags &= ~MCA_BTL_FLAGS_RDMA;
|
||||
mca_btl_vader.super.btl_get = NULL;
|
||||
mca_btl_vader.super.btl_put = NULL;
|
||||
if (MCA_BTL_SM_NONE == mca_btl_sm_component.single_copy_mechanism) {
|
||||
mca_btl_sm.super.btl_flags &= ~MCA_BTL_FLAGS_RDMA;
|
||||
mca_btl_sm.super.btl_get = NULL;
|
||||
mca_btl_sm.super.btl_put = NULL;
|
||||
}
|
||||
}
|
||||
|
||||
/*
|
||||
* VADER component initialization
|
||||
* SM component initialization
|
||||
*/
|
||||
static mca_btl_base_module_t **mca_btl_vader_component_init (int *num_btls,
|
||||
bool enable_progress_threads,
|
||||
bool enable_mpi_threads)
|
||||
static mca_btl_base_module_t **mca_btl_sm_component_init (int *num_btls,
|
||||
bool enable_progress_threads,
|
||||
bool enable_mpi_threads)
|
||||
{
|
||||
mca_btl_vader_component_t *component = &mca_btl_vader_component;
|
||||
mca_btl_sm_component_t *component = &mca_btl_sm_component;
|
||||
mca_btl_base_module_t **btls = NULL;
|
||||
int rc;
|
||||
|
||||
*num_btls = 0;
|
||||
|
||||
/* disable if there are no local peers */
|
||||
if (0 == MCA_BTL_VADER_NUM_LOCAL_PEERS) {
|
||||
BTL_VERBOSE(("No peers to communicate with. Disabling vader."));
|
||||
if (0 == MCA_BTL_SM_NUM_LOCAL_PEERS) {
|
||||
BTL_VERBOSE(("No peers to communicate with. Disabling sm."));
|
||||
return NULL;
|
||||
}
|
||||
|
||||
#if OPAL_BTL_VADER_HAVE_XPMEM
|
||||
#if OPAL_BTL_SM_HAVE_XPMEM
|
||||
/* limit segment alignment to be between 4k and 16M */
|
||||
if (component->log_attach_align < 12) {
|
||||
component->log_attach_align = 12;
|
||||
@ -551,10 +551,10 @@ static mca_btl_base_module_t **mca_btl_vader_component_init (int *num_btls,
|
||||
component->segment_size = (2 << 20);
|
||||
}
|
||||
|
||||
component->fbox_size = (component->fbox_size + MCA_BTL_VADER_FBOX_ALIGNMENT_MASK) & ~MCA_BTL_VADER_FBOX_ALIGNMENT_MASK;
|
||||
component->fbox_size = (component->fbox_size + MCA_BTL_SM_FBOX_ALIGNMENT_MASK) & ~MCA_BTL_SM_FBOX_ALIGNMENT_MASK;
|
||||
|
||||
if (component->segment_size > (1ul << MCA_BTL_VADER_OFFSET_BITS)) {
|
||||
component->segment_size = 2ul << MCA_BTL_VADER_OFFSET_BITS;
|
||||
if (component->segment_size > (1ul << MCA_BTL_SM_OFFSET_BITS)) {
|
||||
component->segment_size = 2ul << MCA_BTL_SM_OFFSET_BITS;
|
||||
}
|
||||
|
||||
/* no fast boxes allocated initially */
|
||||
@ -562,13 +562,13 @@ static mca_btl_base_module_t **mca_btl_vader_component_init (int *num_btls,
|
||||
|
||||
component->local_rank = 0;
|
||||
|
||||
mca_btl_vader_check_single_copy ();
|
||||
mca_btl_sm_check_single_copy ();
|
||||
|
||||
if (MCA_BTL_VADER_XPMEM != mca_btl_vader_component.single_copy_mechanism) {
|
||||
if (MCA_BTL_SM_XPMEM != mca_btl_sm_component.single_copy_mechanism) {
|
||||
char *sm_file;
|
||||
|
||||
rc = opal_asprintf(&sm_file, "%s" OPAL_PATH_SEP "vader_segment.%s.%x.%d", mca_btl_vader_component.backing_directory,
|
||||
opal_process_info.nodename, OPAL_PROC_MY_NAME.jobid, MCA_BTL_VADER_LOCAL_RANK);
|
||||
rc = opal_asprintf(&sm_file, "%s" OPAL_PATH_SEP "sm_segment.%s.%x.%d", mca_btl_sm_component.backing_directory,
|
||||
opal_process_info.nodename, OPAL_PROC_MY_NAME.jobid, MCA_BTL_SM_LOCAL_RANK);
|
||||
if (0 > rc) {
|
||||
free (btls);
|
||||
return NULL;
|
||||
@ -600,9 +600,9 @@ static mca_btl_base_module_t **mca_btl_vader_component_init (int *num_btls,
|
||||
}
|
||||
|
||||
/* initialize my fifo */
|
||||
vader_fifo_init ((struct vader_fifo_t *) component->my_segment);
|
||||
sm_fifo_init ((struct sm_fifo_t *) component->my_segment);
|
||||
|
||||
rc = mca_btl_base_vader_modex_send ();
|
||||
rc = mca_btl_base_sm_modex_send ();
|
||||
if (OPAL_SUCCESS != rc) {
|
||||
BTL_VERBOSE(("Error sending modex"));
|
||||
goto failed;
|
||||
@ -611,15 +611,15 @@ static mca_btl_base_module_t **mca_btl_vader_component_init (int *num_btls,
|
||||
*num_btls = 1;
|
||||
|
||||
/* get pointer to the btls */
|
||||
btls[0] = (mca_btl_base_module_t *) &mca_btl_vader;
|
||||
btls[0] = (mca_btl_base_module_t *) &mca_btl_sm;
|
||||
|
||||
/* set flag indicating btl not inited */
|
||||
mca_btl_vader.btl_inited = false;
|
||||
mca_btl_sm.btl_inited = false;
|
||||
|
||||
return btls;
|
||||
failed:
|
||||
#if OPAL_BTL_VADER_HAVE_XPMEM
|
||||
if (MCA_BTL_VADER_XPMEM == mca_btl_vader_component.single_copy_mechanism) {
|
||||
failed:
|
||||
#if OPAL_BTL_SM_HAVE_XPMEM
|
||||
if (MCA_BTL_SM_XPMEM == mca_btl_sm_component.single_copy_mechanism) {
|
||||
munmap (component->my_segment, component->segment_size);
|
||||
} else
|
||||
#endif
|
||||
@ -632,14 +632,14 @@ failed:
|
||||
return NULL;
|
||||
}
|
||||
|
||||
void mca_btl_vader_poll_handle_frag (mca_btl_vader_hdr_t *hdr, struct mca_btl_base_endpoint_t *endpoint)
|
||||
void mca_btl_sm_poll_handle_frag (mca_btl_sm_hdr_t *hdr, struct mca_btl_base_endpoint_t *endpoint)
|
||||
{
|
||||
mca_btl_base_segment_t segments[2];
|
||||
mca_btl_base_descriptor_t frag = {.des_segments = segments, .des_segment_count = 1};
|
||||
const mca_btl_active_message_callback_t *reg;
|
||||
|
||||
if (hdr->flags & MCA_BTL_VADER_FLAG_COMPLETE) {
|
||||
mca_btl_vader_frag_complete (hdr->frag);
|
||||
if (hdr->flags & MCA_BTL_SM_FLAG_COMPLETE) {
|
||||
mca_btl_sm_frag_complete (hdr->frag);
|
||||
return;
|
||||
}
|
||||
|
||||
@ -647,46 +647,46 @@ void mca_btl_vader_poll_handle_frag (mca_btl_vader_hdr_t *hdr, struct mca_btl_ba
|
||||
segments[0].seg_addr.pval = (void *) (hdr + 1);
|
||||
segments[0].seg_len = hdr->len;
|
||||
|
||||
if (hdr->flags & MCA_BTL_VADER_FLAG_SINGLE_COPY) {
|
||||
if (hdr->flags & MCA_BTL_SM_FLAG_SINGLE_COPY) {
|
||||
mca_rcache_base_registration_t *xpmem_reg;
|
||||
|
||||
xpmem_reg = vader_get_registation (endpoint, hdr->sc_iov.iov_base,
|
||||
hdr->sc_iov.iov_len, 0,
|
||||
&segments[1].seg_addr.pval);
|
||||
xpmem_reg = sm_get_registation (endpoint, hdr->sc_iov.iov_base,
|
||||
hdr->sc_iov.iov_len, 0,
|
||||
&segments[1].seg_addr.pval);
|
||||
assert (NULL != xpmem_reg);
|
||||
|
||||
segments[1].seg_len = hdr->sc_iov.iov_len;
|
||||
frag.des_segment_count = 2;
|
||||
|
||||
/* recv upcall */
|
||||
reg->cbfunc(&mca_btl_vader.super, hdr->tag, &frag, reg->cbdata);
|
||||
vader_return_registration (xpmem_reg, endpoint);
|
||||
reg->cbfunc(&mca_btl_sm.super, hdr->tag, &frag, reg->cbdata);
|
||||
sm_return_registration (xpmem_reg, endpoint);
|
||||
} else {
|
||||
reg->cbfunc(&mca_btl_vader.super, hdr->tag, &frag, reg->cbdata);
|
||||
reg->cbfunc(&mca_btl_sm.super, hdr->tag, &frag, reg->cbdata);
|
||||
}
|
||||
|
||||
if (OPAL_UNLIKELY(MCA_BTL_VADER_FLAG_SETUP_FBOX & hdr->flags)) {
|
||||
mca_btl_vader_endpoint_setup_fbox_recv (endpoint, relative2virtual(hdr->fbox_base));
|
||||
mca_btl_vader_component.fbox_in_endpoints[mca_btl_vader_component.num_fbox_in_endpoints++] = endpoint;
|
||||
if (OPAL_UNLIKELY(MCA_BTL_SM_FLAG_SETUP_FBOX & hdr->flags)) {
|
||||
mca_btl_sm_endpoint_setup_fbox_recv (endpoint, relative2virtual(hdr->fbox_base));
|
||||
mca_btl_sm_component.fbox_in_endpoints[mca_btl_sm_component.num_fbox_in_endpoints++] = endpoint;
|
||||
}
|
||||
|
||||
hdr->flags = MCA_BTL_VADER_FLAG_COMPLETE;
|
||||
vader_fifo_write_back (hdr, endpoint);
|
||||
hdr->flags = MCA_BTL_SM_FLAG_COMPLETE;
|
||||
sm_fifo_write_back (hdr, endpoint);
|
||||
}
|
||||
|
||||
static int mca_btl_vader_poll_fifo (void)
|
||||
static int mca_btl_sm_poll_fifo (void)
|
||||
{
|
||||
struct mca_btl_base_endpoint_t *endpoint;
|
||||
mca_btl_vader_hdr_t *hdr;
|
||||
mca_btl_sm_hdr_t *hdr;
|
||||
|
||||
/* poll the fifo until it is empty or a limit has been hit (8 is arbitrary) */
|
||||
for (int fifo_count = 0 ; fifo_count < 31 ; ++fifo_count) {
|
||||
hdr = vader_fifo_read (mca_btl_vader_component.my_fifo, &endpoint);
|
||||
hdr = sm_fifo_read (mca_btl_sm_component.my_fifo, &endpoint);
|
||||
if (NULL == hdr) {
|
||||
return fifo_count;
|
||||
}
|
||||
|
||||
mca_btl_vader_poll_handle_frag (hdr, endpoint);
|
||||
mca_btl_sm_poll_handle_frag (hdr, endpoint);
|
||||
}
|
||||
|
||||
return 1;
|
||||
@ -695,14 +695,14 @@ static int mca_btl_vader_poll_fifo (void)
|
||||
/**
|
||||
* Progress pending messages on an endpoint
|
||||
*
|
||||
* @param ep (IN) Vader BTL endpoint
|
||||
* @param ep (IN) Sm BTL endpoint
|
||||
*
|
||||
* This is called with the component lock held so the component lock does
|
||||
* not need to be aquired before modifying the pending_endpoints list.
|
||||
*/
|
||||
static void mca_btl_vader_progress_waiting (mca_btl_base_endpoint_t *ep)
|
||||
static void mca_btl_sm_progress_waiting (mca_btl_base_endpoint_t *ep)
|
||||
{
|
||||
mca_btl_vader_frag_t *frag, *next;
|
||||
mca_btl_sm_frag_t *frag, *next;
|
||||
int ret = 1;
|
||||
|
||||
if (OPAL_UNLIKELY(NULL == ep)) {
|
||||
@ -710,8 +710,8 @@ static void mca_btl_vader_progress_waiting (mca_btl_base_endpoint_t *ep)
|
||||
}
|
||||
|
||||
OPAL_THREAD_LOCK(&ep->pending_frags_lock);
|
||||
OPAL_LIST_FOREACH_SAFE(frag, next, &ep->pending_frags, mca_btl_vader_frag_t) {
|
||||
ret = vader_fifo_write_ep (frag->hdr, ep);
|
||||
OPAL_LIST_FOREACH_SAFE(frag, next, &ep->pending_frags, mca_btl_sm_frag_t) {
|
||||
ret = sm_fifo_write_ep (frag->hdr, ep);
|
||||
if (!ret) {
|
||||
OPAL_THREAD_UNLOCK(&ep->pending_frags_lock);
|
||||
return;
|
||||
@ -721,7 +721,7 @@ static void mca_btl_vader_progress_waiting (mca_btl_base_endpoint_t *ep)
|
||||
}
|
||||
|
||||
ep->waiting = false;
|
||||
opal_list_remove_item (&mca_btl_vader_component.pending_endpoints, &ep->super);
|
||||
opal_list_remove_item (&mca_btl_sm_component.pending_endpoints, &ep->super);
|
||||
|
||||
OPAL_THREAD_UNLOCK(&ep->pending_frags_lock);
|
||||
}
|
||||
@ -729,26 +729,26 @@ static void mca_btl_vader_progress_waiting (mca_btl_base_endpoint_t *ep)
|
||||
/**
|
||||
* Progress pending messages on all waiting endpoints
|
||||
*
|
||||
* @param ep (IN) Vader BTL endpoint
|
||||
* @param ep (IN) Sm BTL endpoint
|
||||
*/
|
||||
static void mca_btl_vader_progress_endpoints (void)
|
||||
static void mca_btl_sm_progress_endpoints (void)
|
||||
{
|
||||
mca_btl_base_endpoint_t *ep, *next;
|
||||
int count;
|
||||
|
||||
count = opal_list_get_size (&mca_btl_vader_component.pending_endpoints);
|
||||
count = opal_list_get_size (&mca_btl_sm_component.pending_endpoints);
|
||||
if (OPAL_LIKELY(0 == count)) {
|
||||
return;
|
||||
}
|
||||
|
||||
OPAL_THREAD_LOCK(&mca_btl_vader_component.lock);
|
||||
OPAL_LIST_FOREACH_SAFE(ep, next, &mca_btl_vader_component.pending_endpoints, mca_btl_base_endpoint_t) {
|
||||
mca_btl_vader_progress_waiting (ep);
|
||||
OPAL_THREAD_LOCK(&mca_btl_sm_component.lock);
|
||||
OPAL_LIST_FOREACH_SAFE(ep, next, &mca_btl_sm_component.pending_endpoints, mca_btl_base_endpoint_t) {
|
||||
mca_btl_sm_progress_waiting (ep);
|
||||
}
|
||||
OPAL_THREAD_UNLOCK(&mca_btl_vader_component.lock);
|
||||
OPAL_THREAD_UNLOCK(&mca_btl_sm_component.lock);
|
||||
}
|
||||
|
||||
static int mca_btl_vader_component_progress (void)
|
||||
static int mca_btl_sm_component_progress (void)
|
||||
{
|
||||
static opal_atomic_int32_t lock = 0;
|
||||
int count = 0;
|
||||
@ -760,18 +760,18 @@ static int mca_btl_vader_component_progress (void)
|
||||
}
|
||||
|
||||
/* check for messages in fast boxes */
|
||||
if (mca_btl_vader_component.num_fbox_in_endpoints) {
|
||||
count = mca_btl_vader_check_fboxes ();
|
||||
if (mca_btl_sm_component.num_fbox_in_endpoints) {
|
||||
count = mca_btl_sm_check_fboxes ();
|
||||
}
|
||||
|
||||
mca_btl_vader_progress_endpoints ();
|
||||
mca_btl_sm_progress_endpoints ();
|
||||
|
||||
if (VADER_FIFO_FREE == mca_btl_vader_component.my_fifo->fifo_head) {
|
||||
if (SM_FIFO_FREE == mca_btl_sm_component.my_fifo->fifo_head) {
|
||||
lock = 0;
|
||||
return count;
|
||||
}
|
||||
|
||||
count += mca_btl_vader_poll_fifo ();
|
||||
count += mca_btl_sm_poll_fifo ();
|
||||
opal_atomic_mb ();
|
||||
lock = 0;
|
||||
|
||||
@ -15,6 +15,7 @@
|
||||
* reserved.
|
||||
* Copyright (c) 2018 Triad National Security, LLC. All rights
|
||||
* reserved.
|
||||
* Copyright (c) 2020 Google, LLC. All rights reserved.
|
||||
* $COPYRIGHT$
|
||||
*
|
||||
* Additional copyrights may follow
|
||||
@ -25,16 +26,16 @@
|
||||
* @file
|
||||
*/
|
||||
|
||||
#ifndef MCA_BTL_VADER_ENDPOINT_H
|
||||
#define MCA_BTL_VADER_ENDPOINT_H
|
||||
#ifndef MCA_BTL_SM_ENDPOINT_H
|
||||
#define MCA_BTL_SM_ENDPOINT_H
|
||||
|
||||
#include "opal_config.h"
|
||||
#include "btl_vader_xpmem.h"
|
||||
#include "btl_sm_xpmem.h"
|
||||
|
||||
#define MCA_BTL_VADER_FBOX_ALIGNMENT 32
|
||||
#define MCA_BTL_VADER_FBOX_ALIGNMENT_MASK (MCA_BTL_VADER_FBOX_ALIGNMENT - 1)
|
||||
#define MCA_BTL_SM_FBOX_ALIGNMENT 32
|
||||
#define MCA_BTL_SM_FBOX_ALIGNMENT_MASK (MCA_BTL_SM_FBOX_ALIGNMENT - 1)
|
||||
|
||||
struct vader_fifo_t;
|
||||
struct sm_fifo_t;
|
||||
|
||||
/**
|
||||
* An abstraction that represents a connection to a endpoint process.
|
||||
@ -42,7 +43,7 @@ struct vader_fifo_t;
|
||||
* and BTL pair at startup.
|
||||
*/
|
||||
|
||||
struct mca_btl_vader_fbox_t;
|
||||
struct mca_btl_sm_fbox_t;
|
||||
|
||||
typedef struct mca_btl_base_endpoint_t {
|
||||
opal_list_item_t super;
|
||||
@ -69,13 +70,13 @@ typedef struct mca_btl_base_endpoint_t {
|
||||
char *segment_base; /**< start of the peer's segment (in the address space
|
||||
* of this process) */
|
||||
|
||||
struct vader_fifo_t *fifo; /**< */
|
||||
struct sm_fifo_t *fifo; /**< */
|
||||
|
||||
opal_mutex_t lock; /**< lock to protect endpoint structures from concurrent
|
||||
* access */
|
||||
|
||||
union {
|
||||
#if OPAL_BTL_VADER_HAVE_XPMEM
|
||||
#if OPAL_BTL_SM_HAVE_XPMEM
|
||||
struct {
|
||||
xpmem_apid_t apid; /**< xpmem apid for remote peer */
|
||||
uintptr_t address_max; /**< largest address that can be attached */
|
||||
@ -92,35 +93,35 @@ typedef struct mca_btl_base_endpoint_t {
|
||||
bool waiting; /**< endpoint is on the component wait list */
|
||||
} mca_btl_base_endpoint_t;
|
||||
|
||||
typedef mca_btl_base_endpoint_t mca_btl_vader_endpoint_t;
|
||||
typedef mca_btl_base_endpoint_t mca_btl_sm_endpoint_t;
|
||||
|
||||
OBJ_CLASS_DECLARATION(mca_btl_vader_endpoint_t);
|
||||
OBJ_CLASS_DECLARATION(mca_btl_sm_endpoint_t);
|
||||
|
||||
static inline void mca_btl_vader_endpoint_setup_fbox_recv (struct mca_btl_base_endpoint_t *endpoint, void *base)
|
||||
static inline void mca_btl_sm_endpoint_setup_fbox_recv (struct mca_btl_base_endpoint_t *endpoint, void *base)
|
||||
{
|
||||
endpoint->fbox_in.startp = (uint32_t *) base;
|
||||
endpoint->fbox_in.start = MCA_BTL_VADER_FBOX_ALIGNMENT;
|
||||
endpoint->fbox_in.start = MCA_BTL_SM_FBOX_ALIGNMENT;
|
||||
endpoint->fbox_in.seq = 0;
|
||||
opal_atomic_wmb ();
|
||||
endpoint->fbox_in.buffer = base;
|
||||
}
|
||||
|
||||
static inline void mca_btl_vader_endpoint_setup_fbox_send (struct mca_btl_base_endpoint_t *endpoint, opal_free_list_item_t *fbox)
|
||||
static inline void mca_btl_sm_endpoint_setup_fbox_send (struct mca_btl_base_endpoint_t *endpoint, opal_free_list_item_t *fbox)
|
||||
{
|
||||
void *base = fbox->ptr;
|
||||
|
||||
endpoint->fbox_out.start = MCA_BTL_VADER_FBOX_ALIGNMENT;
|
||||
endpoint->fbox_out.end = MCA_BTL_VADER_FBOX_ALIGNMENT;
|
||||
endpoint->fbox_out.start = MCA_BTL_SM_FBOX_ALIGNMENT;
|
||||
endpoint->fbox_out.end = MCA_BTL_SM_FBOX_ALIGNMENT;
|
||||
endpoint->fbox_out.startp = (uint32_t *) base;
|
||||
endpoint->fbox_out.startp[0] = MCA_BTL_VADER_FBOX_ALIGNMENT;
|
||||
endpoint->fbox_out.startp[0] = MCA_BTL_SM_FBOX_ALIGNMENT;
|
||||
endpoint->fbox_out.seq = 0;
|
||||
endpoint->fbox_out.fbox = fbox;
|
||||
|
||||
/* zero out the first header in the fast box */
|
||||
memset ((char *) base + MCA_BTL_VADER_FBOX_ALIGNMENT, 0, MCA_BTL_VADER_FBOX_ALIGNMENT);
|
||||
memset ((char *) base + MCA_BTL_SM_FBOX_ALIGNMENT, 0, MCA_BTL_SM_FBOX_ALIGNMENT);
|
||||
|
||||
opal_atomic_wmb ();
|
||||
endpoint->fbox_out.buffer = base;
|
||||
}
|
||||
|
||||
#endif /* MCA_BTL_VADER_ENDPOINT_H */
|
||||
#endif /* MCA_BTL_SM_ENDPOINT_H */
|
||||
@ -4,6 +4,7 @@
|
||||
* reserved.
|
||||
* Copyright (c) 2018 Triad National Security, LLC. All rights
|
||||
* reserved.
|
||||
* Copyright (c) 2020 Google, LLC. All rights reserveed.
|
||||
* $COPYRIGHT$
|
||||
*
|
||||
* Additional copyrights may follow
|
||||
@ -11,14 +12,14 @@
|
||||
* $HEADER$
|
||||
*/
|
||||
|
||||
#if !defined(MCA_BTL_VADER_FBOX_H)
|
||||
#define MCA_BTL_VADER_FBOX_H
|
||||
#if !defined(MCA_BTL_SM_FBOX_H)
|
||||
#define MCA_BTL_SM_FBOX_H
|
||||
|
||||
#include "btl_vader.h"
|
||||
#include "btl_sm.h"
|
||||
|
||||
#define MCA_BTL_VADER_POLL_COUNT 31
|
||||
#define MCA_BTL_SM_POLL_COUNT 31
|
||||
|
||||
typedef union mca_btl_vader_fbox_hdr_t {
|
||||
typedef union mca_btl_sm_fbox_hdr_t {
|
||||
struct {
|
||||
/* NTH: on 32-bit platforms loading/unloading the header may be completed
|
||||
* in multiple instructions. To ensure that seq is never loaded before tag
|
||||
@ -36,12 +37,12 @@ typedef union mca_btl_vader_fbox_hdr_t {
|
||||
uint32_t value1;
|
||||
} data_i32;
|
||||
uint64_t ival;
|
||||
} mca_btl_vader_fbox_hdr_t;
|
||||
} mca_btl_sm_fbox_hdr_t;
|
||||
|
||||
#define MCA_BTL_VADER_FBOX_HDR(x) ((mca_btl_vader_fbox_hdr_t *) (x))
|
||||
#define MCA_BTL_SM_FBOX_HDR(x) ((mca_btl_sm_fbox_hdr_t *) (x))
|
||||
|
||||
#define MCA_BTL_VADER_FBOX_OFFSET_MASK 0x7fffffff
|
||||
#define MCA_BTL_VADER_FBOX_HB_MASK 0x80000000
|
||||
#define MCA_BTL_SM_FBOX_OFFSET_MASK 0x7fffffff
|
||||
#define MCA_BTL_SM_FBOX_HB_MASK 0x80000000
|
||||
|
||||
/* if the two offsets are equal and the high bit matches the buffer is empty else the buffer is full.
|
||||
* note that start will never be end - 1 so this simplified conditional will always produce the correct
|
||||
@ -49,14 +50,14 @@ typedef union mca_btl_vader_fbox_hdr_t {
|
||||
#define BUFFER_FREE(s,e,hbm,size) (((s + !hbm) > (e)) ? (s) - (e) : (size - (e)))
|
||||
|
||||
/** macro for checking if the high bit is set */
|
||||
#define MCA_BTL_VADER_FBOX_OFFSET_HBS(v) (!!((v) & MCA_BTL_VADER_FBOX_HB_MASK))
|
||||
#define MCA_BTL_SM_FBOX_OFFSET_HBS(v) (!!((v) & MCA_BTL_SM_FBOX_HB_MASK))
|
||||
|
||||
void mca_btl_vader_poll_handle_frag (mca_btl_vader_hdr_t *hdr, mca_btl_base_endpoint_t *ep);
|
||||
void mca_btl_sm_poll_handle_frag (mca_btl_sm_hdr_t *hdr, mca_btl_base_endpoint_t *ep);
|
||||
|
||||
static inline void mca_btl_vader_fbox_set_header (mca_btl_vader_fbox_hdr_t *hdr, uint16_t tag,
|
||||
uint16_t seq, uint32_t size)
|
||||
static inline void mca_btl_sm_fbox_set_header (mca_btl_sm_fbox_hdr_t *hdr, uint16_t tag,
|
||||
uint16_t seq, uint32_t size)
|
||||
{
|
||||
mca_btl_vader_fbox_hdr_t tmp = {.data = {.tag = tag, .seq = seq, .size = size}};
|
||||
mca_btl_sm_fbox_hdr_t tmp = {.data = {.tag = tag, .seq = seq, .size = size}};
|
||||
/* clear out existing tag/seq */
|
||||
hdr->data_i32.value1 = 0;
|
||||
opal_atomic_wmb ();
|
||||
@ -65,20 +66,20 @@ static inline void mca_btl_vader_fbox_set_header (mca_btl_vader_fbox_hdr_t *hdr,
|
||||
hdr->data_i32.value1 = tmp.data_i32.value1;
|
||||
}
|
||||
|
||||
static inline mca_btl_vader_fbox_hdr_t mca_btl_vader_fbox_read_header (mca_btl_vader_fbox_hdr_t *hdr)
|
||||
static inline mca_btl_sm_fbox_hdr_t mca_btl_sm_fbox_read_header (mca_btl_sm_fbox_hdr_t *hdr)
|
||||
{
|
||||
mca_btl_vader_fbox_hdr_t tmp = {.data_i32 = {.value1 = hdr->data_i32.value1}};;
|
||||
mca_btl_sm_fbox_hdr_t tmp = {.data_i32 = {.value1 = hdr->data_i32.value1}};;
|
||||
opal_atomic_rmb ();
|
||||
tmp.data_i32.value0 = hdr->data_i32.value0;
|
||||
return tmp;
|
||||
}
|
||||
|
||||
/* attempt to reserve a contiguous segment from the remote ep */
|
||||
static inline bool mca_btl_vader_fbox_sendi (mca_btl_base_endpoint_t *ep, unsigned char tag,
|
||||
void * restrict header, const size_t header_size,
|
||||
void * restrict payload, const size_t payload_size)
|
||||
static inline bool mca_btl_sm_fbox_sendi (mca_btl_base_endpoint_t *ep, unsigned char tag,
|
||||
void * restrict header, const size_t header_size,
|
||||
void * restrict payload, const size_t payload_size)
|
||||
{
|
||||
const unsigned int fbox_size = mca_btl_vader_component.fbox_size;
|
||||
const unsigned int fbox_size = mca_btl_sm_component.fbox_size;
|
||||
size_t size = header_size + payload_size;
|
||||
unsigned int start, end, buffer_free;
|
||||
size_t data_size = size;
|
||||
@ -93,16 +94,16 @@ static inline bool mca_btl_vader_fbox_sendi (mca_btl_base_endpoint_t *ep, unsign
|
||||
OPAL_THREAD_LOCK(&ep->lock);
|
||||
|
||||
/* the high bit helps determine if the buffer is empty or full */
|
||||
hbs = MCA_BTL_VADER_FBOX_OFFSET_HBS(ep->fbox_out.end);
|
||||
hbm = MCA_BTL_VADER_FBOX_OFFSET_HBS(ep->fbox_out.start) == hbs;
|
||||
hbs = MCA_BTL_SM_FBOX_OFFSET_HBS(ep->fbox_out.end);
|
||||
hbm = MCA_BTL_SM_FBOX_OFFSET_HBS(ep->fbox_out.start) == hbs;
|
||||
|
||||
/* read current start and end offsets and check for free space */
|
||||
start = ep->fbox_out.start & MCA_BTL_VADER_FBOX_OFFSET_MASK;
|
||||
end = ep->fbox_out.end & MCA_BTL_VADER_FBOX_OFFSET_MASK;
|
||||
start = ep->fbox_out.start & MCA_BTL_SM_FBOX_OFFSET_MASK;
|
||||
end = ep->fbox_out.end & MCA_BTL_SM_FBOX_OFFSET_MASK;
|
||||
buffer_free = BUFFER_FREE(start, end, hbm, fbox_size);
|
||||
|
||||
/* need space for the fragment + the header */
|
||||
size = (size + sizeof (mca_btl_vader_fbox_hdr_t) + MCA_BTL_VADER_FBOX_ALIGNMENT_MASK) & ~MCA_BTL_VADER_FBOX_ALIGNMENT_MASK;
|
||||
size = (size + sizeof (mca_btl_sm_fbox_hdr_t) + MCA_BTL_SM_FBOX_ALIGNMENT_MASK) & ~MCA_BTL_SM_FBOX_ALIGNMENT_MASK;
|
||||
|
||||
dst = ep->fbox_out.buffer + end;
|
||||
|
||||
@ -115,8 +116,8 @@ static inline bool mca_btl_vader_fbox_sendi (mca_btl_base_endpoint_t *ep, unsign
|
||||
start = ep->fbox_out.start = ep->fbox_out.startp[0];
|
||||
|
||||
/* recalculate how much buffer space is available */
|
||||
start &= MCA_BTL_VADER_FBOX_OFFSET_MASK;
|
||||
hbm = MCA_BTL_VADER_FBOX_OFFSET_HBS(ep->fbox_out.start) == hbs;
|
||||
start &= MCA_BTL_SM_FBOX_OFFSET_MASK;
|
||||
hbm = MCA_BTL_SM_FBOX_OFFSET_HBS(ep->fbox_out.start) == hbs;
|
||||
buffer_free = BUFFER_FREE(start, end, hbm, fbox_size);
|
||||
|
||||
opal_atomic_rmb ();
|
||||
@ -126,10 +127,10 @@ static inline bool mca_btl_vader_fbox_sendi (mca_btl_base_endpoint_t *ep, unsign
|
||||
if (OPAL_UNLIKELY(buffer_free > 0 && buffer_free < size && start <= end)) {
|
||||
BTL_VERBOSE(("message will not fit in remaining buffer space. skipping to beginning"));
|
||||
|
||||
mca_btl_vader_fbox_set_header (MCA_BTL_VADER_FBOX_HDR(dst), 0xff, ep->fbox_out.seq++,
|
||||
buffer_free - sizeof (mca_btl_vader_fbox_hdr_t));
|
||||
mca_btl_sm_fbox_set_header (MCA_BTL_SM_FBOX_HDR(dst), 0xff, ep->fbox_out.seq++,
|
||||
buffer_free - sizeof (mca_btl_sm_fbox_hdr_t));
|
||||
|
||||
end = MCA_BTL_VADER_FBOX_ALIGNMENT;
|
||||
end = MCA_BTL_SM_FBOX_ALIGNMENT;
|
||||
/* toggle the high bit */
|
||||
hbs = !hbs;
|
||||
/* toggle the high bit match */
|
||||
@ -148,7 +149,7 @@ static inline bool mca_btl_vader_fbox_sendi (mca_btl_base_endpoint_t *ep, unsign
|
||||
BTL_VERBOSE(("writing fragment of size %u to offset %u {start: 0x%x, end: 0x%x (hbs: %d)} of peer's buffer. free = %u",
|
||||
(unsigned int) size, end, start, end, hbs, buffer_free));
|
||||
|
||||
data = dst + sizeof (mca_btl_vader_fbox_hdr_t);
|
||||
data = dst + sizeof (mca_btl_sm_fbox_hdr_t);
|
||||
|
||||
memcpy (data, header, header_size);
|
||||
if (payload) {
|
||||
@ -162,13 +163,13 @@ static inline bool mca_btl_vader_fbox_sendi (mca_btl_base_endpoint_t *ep, unsign
|
||||
/* toggle the high bit */
|
||||
hbs = !hbs;
|
||||
/* reset the end pointer to the beginning of the buffer */
|
||||
end = MCA_BTL_VADER_FBOX_ALIGNMENT;
|
||||
end = MCA_BTL_SM_FBOX_ALIGNMENT;
|
||||
} else if (buffer_free > size) {
|
||||
MCA_BTL_VADER_FBOX_HDR(ep->fbox_out.buffer + end)->ival = 0;
|
||||
MCA_BTL_SM_FBOX_HDR(ep->fbox_out.buffer + end)->ival = 0;
|
||||
}
|
||||
|
||||
/* write out part of the header now. the tag will be written when the data is available */
|
||||
mca_btl_vader_fbox_set_header (MCA_BTL_VADER_FBOX_HDR(dst), tag, ep->fbox_out.seq++, data_size);
|
||||
mca_btl_sm_fbox_set_header (MCA_BTL_SM_FBOX_HDR(dst), tag, ep->fbox_out.seq++, data_size);
|
||||
|
||||
/* align the buffer */
|
||||
ep->fbox_out.end = ((uint32_t) hbs << 31) | end;
|
||||
@ -178,21 +179,21 @@ static inline bool mca_btl_vader_fbox_sendi (mca_btl_base_endpoint_t *ep, unsign
|
||||
return true;
|
||||
}
|
||||
|
||||
static inline bool mca_btl_vader_check_fboxes (void)
|
||||
static inline bool mca_btl_sm_check_fboxes (void)
|
||||
{
|
||||
const unsigned int fbox_size = mca_btl_vader_component.fbox_size;
|
||||
const unsigned int fbox_size = mca_btl_sm_component.fbox_size;
|
||||
bool processed = false;
|
||||
|
||||
for (unsigned int i = 0 ; i < mca_btl_vader_component.num_fbox_in_endpoints ; ++i) {
|
||||
mca_btl_base_endpoint_t *ep = mca_btl_vader_component.fbox_in_endpoints[i];
|
||||
unsigned int start = ep->fbox_in.start & MCA_BTL_VADER_FBOX_OFFSET_MASK;
|
||||
for (unsigned int i = 0 ; i < mca_btl_sm_component.num_fbox_in_endpoints ; ++i) {
|
||||
mca_btl_base_endpoint_t *ep = mca_btl_sm_component.fbox_in_endpoints[i];
|
||||
unsigned int start = ep->fbox_in.start & MCA_BTL_SM_FBOX_OFFSET_MASK;
|
||||
|
||||
/* save the current high bit state */
|
||||
bool hbs = MCA_BTL_VADER_FBOX_OFFSET_HBS(ep->fbox_in.start);
|
||||
bool hbs = MCA_BTL_SM_FBOX_OFFSET_HBS(ep->fbox_in.start);
|
||||
int poll_count;
|
||||
|
||||
for (poll_count = 0 ; poll_count <= MCA_BTL_VADER_POLL_COUNT ; ++poll_count) {
|
||||
const mca_btl_vader_fbox_hdr_t hdr = mca_btl_vader_fbox_read_header (MCA_BTL_VADER_FBOX_HDR(ep->fbox_in.buffer + start));
|
||||
for (poll_count = 0 ; poll_count <= MCA_BTL_SM_POLL_COUNT ; ++poll_count) {
|
||||
const mca_btl_sm_fbox_hdr_t hdr = mca_btl_sm_fbox_read_header (MCA_BTL_SM_FBOX_HDR(ep->fbox_in.buffer + start));
|
||||
|
||||
/* check for a valid tag a sequence number */
|
||||
if (0 == hdr.data.tag || hdr.data.seq != ep->fbox_in.seq) {
|
||||
@ -223,18 +224,18 @@ static inline bool mca_btl_vader_check_fboxes (void)
|
||||
segment.seg_addr.pval = (void *) (ep->fbox_in.buffer + start + sizeof (hdr));
|
||||
|
||||
/* call the registered callback function */
|
||||
reg->cbfunc(&mca_btl_vader.super, hdr.data.tag, &desc, reg->cbdata);
|
||||
reg->cbfunc(&mca_btl_sm.super, hdr.data.tag, &desc, reg->cbdata);
|
||||
} else if (OPAL_LIKELY(0xfe == hdr.data.tag)) {
|
||||
/* process fragment header */
|
||||
fifo_value_t *value = (fifo_value_t *)(ep->fbox_in.buffer + start + sizeof (hdr));
|
||||
mca_btl_vader_hdr_t *hdr = relative2virtual(*value);
|
||||
mca_btl_vader_poll_handle_frag (hdr, ep);
|
||||
mca_btl_sm_hdr_t *hdr = relative2virtual(*value);
|
||||
mca_btl_sm_poll_handle_frag (hdr, ep);
|
||||
}
|
||||
|
||||
start = (start + hdr.data.size + sizeof (hdr) + MCA_BTL_VADER_FBOX_ALIGNMENT_MASK) & ~MCA_BTL_VADER_FBOX_ALIGNMENT_MASK;
|
||||
start = (start + hdr.data.size + sizeof (hdr) + MCA_BTL_SM_FBOX_ALIGNMENT_MASK) & ~MCA_BTL_SM_FBOX_ALIGNMENT_MASK;
|
||||
if (OPAL_UNLIKELY(fbox_size == start)) {
|
||||
/* jump to the beginning of the buffer */
|
||||
start = MCA_BTL_VADER_FBOX_ALIGNMENT;
|
||||
start = MCA_BTL_SM_FBOX_ALIGNMENT;
|
||||
/* toggle the high bit */
|
||||
hbs = !hbs;
|
||||
}
|
||||
@ -255,22 +256,22 @@ static inline bool mca_btl_vader_check_fboxes (void)
|
||||
return processed;
|
||||
}
|
||||
|
||||
static inline void mca_btl_vader_try_fbox_setup (mca_btl_base_endpoint_t *ep, mca_btl_vader_hdr_t *hdr)
|
||||
static inline void mca_btl_sm_try_fbox_setup (mca_btl_base_endpoint_t *ep, mca_btl_sm_hdr_t *hdr)
|
||||
{
|
||||
if (OPAL_UNLIKELY(NULL == ep->fbox_out.buffer && mca_btl_vader_component.fbox_threshold == OPAL_THREAD_ADD_FETCH_SIZE_T (&ep->send_count, 1))) {
|
||||
/* protect access to mca_btl_vader_component.segment_offset */
|
||||
OPAL_THREAD_LOCK(&mca_btl_vader_component.lock);
|
||||
if (OPAL_UNLIKELY(NULL == ep->fbox_out.buffer && mca_btl_sm_component.fbox_threshold == OPAL_THREAD_ADD_FETCH_SIZE_T (&ep->send_count, 1))) {
|
||||
/* protect access to mca_btl_sm_component.segment_offset */
|
||||
OPAL_THREAD_LOCK(&mca_btl_sm_component.lock);
|
||||
|
||||
/* verify the remote side will accept another fbox */
|
||||
if (0 <= opal_atomic_add_fetch_32 (&ep->fifo->fbox_available, -1)) {
|
||||
opal_free_list_item_t *fbox = opal_free_list_get (&mca_btl_vader_component.vader_fboxes);
|
||||
opal_free_list_item_t *fbox = opal_free_list_get (&mca_btl_sm_component.sm_fboxes);
|
||||
|
||||
if (NULL != fbox) {
|
||||
/* zero out the fast box */
|
||||
memset (fbox->ptr, 0, mca_btl_vader_component.fbox_size);
|
||||
mca_btl_vader_endpoint_setup_fbox_send (ep, fbox);
|
||||
memset (fbox->ptr, 0, mca_btl_sm_component.fbox_size);
|
||||
mca_btl_sm_endpoint_setup_fbox_send (ep, fbox);
|
||||
|
||||
hdr->flags |= MCA_BTL_VADER_FLAG_SETUP_FBOX;
|
||||
hdr->flags |= MCA_BTL_SM_FLAG_SETUP_FBOX;
|
||||
hdr->fbox_base = virtual2relative((char *) ep->fbox_out.buffer);
|
||||
} else {
|
||||
opal_atomic_add_fetch_32 (&ep->fifo->fbox_available, 1);
|
||||
@ -279,8 +280,8 @@ static inline void mca_btl_vader_try_fbox_setup (mca_btl_base_endpoint_t *ep, mc
|
||||
opal_atomic_wmb ();
|
||||
}
|
||||
|
||||
OPAL_THREAD_UNLOCK(&mca_btl_vader_component.lock);
|
||||
OPAL_THREAD_UNLOCK(&mca_btl_sm_component.lock);
|
||||
}
|
||||
}
|
||||
|
||||
#endif /* !defined(MCA_BTL_VADER_FBOX_H) */
|
||||
#endif /* !defined(MCA_BTL_SM_FBOX_H) */
|
||||
@ -14,6 +14,7 @@
|
||||
* Copyright (c) 2009-2010 Cisco Systems, Inc. All rights reserved.
|
||||
* Copyright (c) 2010-2018 Los Alamos National Security, LLC.
|
||||
* All rights reserved.
|
||||
* Copyright (c) 2020 Google, LLC. All rights reserved.
|
||||
* $COPYRIGHT$
|
||||
*
|
||||
* Additional copyrights may follow
|
||||
@ -23,33 +24,33 @@
|
||||
/**
|
||||
* @file
|
||||
*/
|
||||
#ifndef MCA_BTL_VADER_FIFO_H
|
||||
#define MCA_BTL_VADER_FIFO_H
|
||||
#ifndef MCA_BTL_SM_FIFO_H
|
||||
#define MCA_BTL_SM_FIFO_H
|
||||
|
||||
#include "btl_vader.h"
|
||||
#include "btl_vader_endpoint.h"
|
||||
#include "btl_vader_frag.h"
|
||||
#include "btl_sm.h"
|
||||
#include "btl_sm_endpoint.h"
|
||||
#include "btl_sm_frag.h"
|
||||
|
||||
#define vader_item_compare_exchange(x, y, z) opal_atomic_compare_exchange_strong_ptr ((opal_atomic_intptr_t *) (x), (intptr_t *) (y), (intptr_t) (z))
|
||||
#define sm_item_compare_exchange(x, y, z) opal_atomic_compare_exchange_strong_ptr ((opal_atomic_intptr_t *) (x), (intptr_t *) (y), (intptr_t) (z))
|
||||
|
||||
#if SIZEOF_VOID_P == 8
|
||||
#define vader_item_swap(x, y) opal_atomic_swap_64((opal_atomic_int64_t *)(x), (int64_t)(y))
|
||||
#define sm_item_swap(x, y) opal_atomic_swap_64((opal_atomic_int64_t *)(x), (int64_t)(y))
|
||||
|
||||
#define MCA_BTL_VADER_OFFSET_MASK 0xffffffffll
|
||||
#define MCA_BTL_VADER_OFFSET_BITS 32
|
||||
#define MCA_BTL_VADER_BITNESS 64
|
||||
#define MCA_BTL_SM_OFFSET_MASK 0xffffffffll
|
||||
#define MCA_BTL_SM_OFFSET_BITS 32
|
||||
#define MCA_BTL_SM_BITNESS 64
|
||||
#else
|
||||
#define vader_item_swap(x, y) opal_atomic_swap_32((opal_atomic_int32_t *)(x), (int32_t)(y))
|
||||
#define sm_item_swap(x, y) opal_atomic_swap_32((opal_atomic_int32_t *)(x), (int32_t)(y))
|
||||
|
||||
#define MCA_BTL_VADER_OFFSET_MASK 0x00ffffffl
|
||||
#define MCA_BTL_VADER_OFFSET_BITS 24
|
||||
#define MCA_BTL_VADER_BITNESS 32
|
||||
#define MCA_BTL_SM_OFFSET_MASK 0x00ffffffl
|
||||
#define MCA_BTL_SM_OFFSET_BITS 24
|
||||
#define MCA_BTL_SM_BITNESS 32
|
||||
#endif
|
||||
|
||||
typedef opal_atomic_intptr_t atomic_fifo_value_t;
|
||||
typedef intptr_t fifo_value_t;
|
||||
|
||||
#define VADER_FIFO_FREE ((fifo_value_t)-2)
|
||||
#define SM_FIFO_FREE ((fifo_value_t)-2)
|
||||
|
||||
/*
|
||||
* Shared Memory FIFOs
|
||||
@ -66,14 +67,14 @@ typedef intptr_t fifo_value_t;
|
||||
*/
|
||||
|
||||
/* lock free fifo */
|
||||
typedef struct vader_fifo_t {
|
||||
typedef struct sm_fifo_t {
|
||||
atomic_fifo_value_t fifo_head;
|
||||
atomic_fifo_value_t fifo_tail;
|
||||
opal_atomic_int32_t fbox_available;
|
||||
} vader_fifo_t;
|
||||
} sm_fifo_t;
|
||||
|
||||
/* large enough to ensure the fifo is on its own cache line */
|
||||
#define MCA_BTL_VADER_FIFO_SIZE 128
|
||||
#define MCA_BTL_SM_FIFO_SIZE 128
|
||||
|
||||
/***
|
||||
* One or more FIFO components may be a pointer that must be
|
||||
@ -87,23 +88,23 @@ typedef struct vader_fifo_t {
|
||||
/* This only works for finding the relative address for a pointer within my_segment */
|
||||
static inline fifo_value_t virtual2relative (char *addr)
|
||||
{
|
||||
return (fifo_value_t) ((intptr_t) (addr - mca_btl_vader_component.my_segment)) | ((fifo_value_t)MCA_BTL_VADER_LOCAL_RANK << MCA_BTL_VADER_OFFSET_BITS);
|
||||
return (fifo_value_t) ((intptr_t) (addr - mca_btl_sm_component.my_segment)) | ((fifo_value_t)MCA_BTL_SM_LOCAL_RANK << MCA_BTL_SM_OFFSET_BITS);
|
||||
}
|
||||
|
||||
static inline fifo_value_t virtual2relativepeer (struct mca_btl_base_endpoint_t *endpoint, char *addr)
|
||||
{
|
||||
return (fifo_value_t) ((intptr_t) (addr - endpoint->segment_base)) | ((fifo_value_t)endpoint->peer_smp_rank << MCA_BTL_VADER_OFFSET_BITS);
|
||||
return (fifo_value_t) ((intptr_t) (addr - endpoint->segment_base)) | ((fifo_value_t)endpoint->peer_smp_rank << MCA_BTL_SM_OFFSET_BITS);
|
||||
}
|
||||
|
||||
static inline void *relative2virtual (fifo_value_t offset)
|
||||
{
|
||||
return (void *)(intptr_t)((offset & MCA_BTL_VADER_OFFSET_MASK) + mca_btl_vader_component.endpoints[offset >> MCA_BTL_VADER_OFFSET_BITS].segment_base);
|
||||
return (void *)(intptr_t)((offset & MCA_BTL_SM_OFFSET_MASK) + mca_btl_sm_component.endpoints[offset >> MCA_BTL_SM_OFFSET_BITS].segment_base);
|
||||
}
|
||||
|
||||
#include "btl_vader_fbox.h"
|
||||
#include "btl_sm_fbox.h"
|
||||
|
||||
/**
|
||||
* vader_fifo_read:
|
||||
* sm_fifo_read:
|
||||
*
|
||||
* @brief reads a single fragment from a local fifo
|
||||
*
|
||||
@ -114,12 +115,12 @@ static inline void *relative2virtual (fifo_value_t offset)
|
||||
*
|
||||
* This function does not currently support multiple readers.
|
||||
*/
|
||||
static inline mca_btl_vader_hdr_t *vader_fifo_read (vader_fifo_t *fifo, struct mca_btl_base_endpoint_t **ep)
|
||||
static inline mca_btl_sm_hdr_t *sm_fifo_read (sm_fifo_t *fifo, struct mca_btl_base_endpoint_t **ep)
|
||||
{
|
||||
mca_btl_vader_hdr_t *hdr;
|
||||
mca_btl_sm_hdr_t *hdr;
|
||||
fifo_value_t value;
|
||||
|
||||
if (VADER_FIFO_FREE == fifo->fifo_head) {
|
||||
if (SM_FIFO_FREE == fifo->fifo_head) {
|
||||
return NULL;
|
||||
}
|
||||
|
||||
@ -127,18 +128,18 @@ static inline mca_btl_vader_hdr_t *vader_fifo_read (vader_fifo_t *fifo, struct m
|
||||
|
||||
value = fifo->fifo_head;
|
||||
|
||||
*ep = &mca_btl_vader_component.endpoints[value >> MCA_BTL_VADER_OFFSET_BITS];
|
||||
hdr = (mca_btl_vader_hdr_t *) relative2virtual (value);
|
||||
*ep = &mca_btl_sm_component.endpoints[value >> MCA_BTL_SM_OFFSET_BITS];
|
||||
hdr = (mca_btl_sm_hdr_t *) relative2virtual (value);
|
||||
|
||||
fifo->fifo_head = VADER_FIFO_FREE;
|
||||
fifo->fifo_head = SM_FIFO_FREE;
|
||||
|
||||
assert (hdr->next != value);
|
||||
|
||||
if (OPAL_UNLIKELY(VADER_FIFO_FREE == hdr->next)) {
|
||||
if (OPAL_UNLIKELY(SM_FIFO_FREE == hdr->next)) {
|
||||
opal_atomic_rmb();
|
||||
|
||||
if (!vader_item_compare_exchange (&fifo->fifo_tail, &value, VADER_FIFO_FREE)) {
|
||||
while (VADER_FIFO_FREE == hdr->next) {
|
||||
if (!sm_item_compare_exchange (&fifo->fifo_tail, &value, SM_FIFO_FREE)) {
|
||||
while (SM_FIFO_FREE == hdr->next) {
|
||||
opal_atomic_rmb ();
|
||||
}
|
||||
|
||||
@ -152,29 +153,29 @@ static inline mca_btl_vader_hdr_t *vader_fifo_read (vader_fifo_t *fifo, struct m
|
||||
return hdr;
|
||||
}
|
||||
|
||||
static inline void vader_fifo_init (vader_fifo_t *fifo)
|
||||
static inline void sm_fifo_init (sm_fifo_t *fifo)
|
||||
{
|
||||
/* due to a compiler bug in Oracle C 5.15 the following line was broken into two. Not
|
||||
* ideal but oh well. See #5814 */
|
||||
/* fifo->fifo_head = fifo->fifo_tail = VADER_FIFO_FREE; */
|
||||
fifo->fifo_head = VADER_FIFO_FREE;
|
||||
fifo->fifo_tail = VADER_FIFO_FREE;
|
||||
fifo->fbox_available = mca_btl_vader_component.fbox_max;
|
||||
mca_btl_vader_component.my_fifo = fifo;
|
||||
/* fifo->fifo_head = fifo->fifo_tail = SM_FIFO_FREE; */
|
||||
fifo->fifo_head = SM_FIFO_FREE;
|
||||
fifo->fifo_tail = SM_FIFO_FREE;
|
||||
fifo->fbox_available = mca_btl_sm_component.fbox_max;
|
||||
mca_btl_sm_component.my_fifo = fifo;
|
||||
}
|
||||
|
||||
static inline void vader_fifo_write (vader_fifo_t *fifo, fifo_value_t value)
|
||||
static inline void sm_fifo_write (sm_fifo_t *fifo, fifo_value_t value)
|
||||
{
|
||||
fifo_value_t prev;
|
||||
|
||||
opal_atomic_wmb ();
|
||||
prev = vader_item_swap (&fifo->fifo_tail, value);
|
||||
prev = sm_item_swap (&fifo->fifo_tail, value);
|
||||
opal_atomic_rmb ();
|
||||
|
||||
assert (prev != value);
|
||||
|
||||
if (OPAL_LIKELY(VADER_FIFO_FREE != prev)) {
|
||||
mca_btl_vader_hdr_t *hdr = (mca_btl_vader_hdr_t *) relative2virtual (prev);
|
||||
if (OPAL_LIKELY(SM_FIFO_FREE != prev)) {
|
||||
mca_btl_sm_hdr_t *hdr = (mca_btl_sm_hdr_t *) relative2virtual (prev);
|
||||
hdr->next = value;
|
||||
} else {
|
||||
fifo->fifo_head = value;
|
||||
@ -184,7 +185,7 @@ static inline void vader_fifo_write (vader_fifo_t *fifo, fifo_value_t value)
|
||||
}
|
||||
|
||||
/**
|
||||
* vader_fifo_write_ep:
|
||||
* sm_fifo_write_ep:
|
||||
*
|
||||
* @brief write a frag (relative to this process' base) to another rank's fifo
|
||||
*
|
||||
@ -194,37 +195,37 @@ static inline void vader_fifo_write (vader_fifo_t *fifo, fifo_value_t value)
|
||||
* This function is used to send a fragment to a remote peer. {hdr} must belong
|
||||
* to the current process.
|
||||
*/
|
||||
static inline bool vader_fifo_write_ep (mca_btl_vader_hdr_t *hdr, struct mca_btl_base_endpoint_t *ep)
|
||||
static inline bool sm_fifo_write_ep (mca_btl_sm_hdr_t *hdr, struct mca_btl_base_endpoint_t *ep)
|
||||
{
|
||||
fifo_value_t rhdr = virtual2relative ((char *) hdr);
|
||||
if (ep->fbox_out.buffer) {
|
||||
/* if there is a fast box for this peer then use the fast box to send the fragment header.
|
||||
* this is done to ensure fragment ordering */
|
||||
opal_atomic_wmb ();
|
||||
return mca_btl_vader_fbox_sendi (ep, 0xfe, &rhdr, sizeof (rhdr), NULL, 0);
|
||||
return mca_btl_sm_fbox_sendi (ep, 0xfe, &rhdr, sizeof (rhdr), NULL, 0);
|
||||
}
|
||||
mca_btl_vader_try_fbox_setup (ep, hdr);
|
||||
hdr->next = VADER_FIFO_FREE;
|
||||
vader_fifo_write (ep->fifo, rhdr);
|
||||
mca_btl_sm_try_fbox_setup (ep, hdr);
|
||||
hdr->next = SM_FIFO_FREE;
|
||||
sm_fifo_write (ep->fifo, rhdr);
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
/**
|
||||
* vader_fifo_write_back:
|
||||
* sm_fifo_write_back:
|
||||
*
|
||||
* @brief write a frag (relative to the remote process' base) to the remote fifo
|
||||
*
|
||||
* @param[in] hdr - fragment header to write
|
||||
* @param[in] ep - endpoint the fragment belongs to
|
||||
*
|
||||
* This function is used to return a fragment to the sending process. It differs from vader_fifo_write_ep
|
||||
* This function is used to return a fragment to the sending process. It differs from sm_fifo_write_ep
|
||||
* in that it uses the {ep} to produce the relative address.
|
||||
*/
|
||||
static inline void vader_fifo_write_back (mca_btl_vader_hdr_t *hdr, struct mca_btl_base_endpoint_t *ep)
|
||||
static inline void sm_fifo_write_back (mca_btl_sm_hdr_t *hdr, struct mca_btl_base_endpoint_t *ep)
|
||||
{
|
||||
hdr->next = VADER_FIFO_FREE;
|
||||
vader_fifo_write(ep->fifo, virtual2relativepeer (ep, (char *) hdr));
|
||||
hdr->next = SM_FIFO_FREE;
|
||||
sm_fifo_write(ep->fifo, virtual2relativepeer (ep, (char *) hdr));
|
||||
}
|
||||
|
||||
#endif /* MCA_BTL_VADER_FIFO_H */
|
||||
#endif /* MCA_BTL_SM_FIFO_H */
|
||||
@ -15,6 +15,7 @@
|
||||
* reserved.
|
||||
* Copyright (c) 2018 Triad National Security, LLC. All rights
|
||||
* reserved.
|
||||
* Copyright (c) 2020 Google, LLC. All rights reserved.
|
||||
* $COPYRIGHT$
|
||||
*
|
||||
* Additional copyrights may follow
|
||||
@ -24,12 +25,12 @@
|
||||
|
||||
#include "opal_config.h"
|
||||
|
||||
#include "btl_vader.h"
|
||||
#include "btl_vader_frag.h"
|
||||
#include "btl_sm.h"
|
||||
#include "btl_sm_frag.h"
|
||||
|
||||
static inline void mca_btl_vader_frag_constructor (mca_btl_vader_frag_t *frag)
|
||||
static inline void mca_btl_sm_frag_constructor (mca_btl_sm_frag_t *frag)
|
||||
{
|
||||
frag->hdr = (mca_btl_vader_hdr_t*)frag->base.super.ptr;
|
||||
frag->hdr = (mca_btl_sm_hdr_t*)frag->base.super.ptr;
|
||||
if(frag->hdr != NULL) {
|
||||
frag->hdr->frag = frag;
|
||||
frag->hdr->flags = 0;
|
||||
@ -40,9 +41,9 @@ static inline void mca_btl_vader_frag_constructor (mca_btl_vader_frag_t *frag)
|
||||
frag->base.des_segment_count = 1;
|
||||
}
|
||||
|
||||
int mca_btl_vader_frag_init (opal_free_list_item_t *item, void *ctx)
|
||||
int mca_btl_sm_frag_init (opal_free_list_item_t *item, void *ctx)
|
||||
{
|
||||
mca_btl_vader_frag_t *frag = (mca_btl_vader_frag_t *) item;
|
||||
mca_btl_sm_frag_t *frag = (mca_btl_sm_frag_t *) item;
|
||||
|
||||
/* Set the list element here so we don't have to set it on the critical path */
|
||||
frag->my_list = (opal_free_list_t *) ctx;
|
||||
@ -50,5 +51,5 @@ int mca_btl_vader_frag_init (opal_free_list_item_t *item, void *ctx)
|
||||
return OPAL_SUCCESS;
|
||||
}
|
||||
|
||||
OBJ_CLASS_INSTANCE(mca_btl_vader_frag_t, mca_btl_base_descriptor_t,
|
||||
mca_btl_vader_frag_constructor, NULL);
|
||||
OBJ_CLASS_INSTANCE(mca_btl_sm_frag_t, mca_btl_base_descriptor_t,
|
||||
mca_btl_sm_frag_constructor, NULL);
|
||||
@ -14,7 +14,7 @@
|
||||
* Copyright (c) 2009 Cisco Systems, Inc. All rights reserved.
|
||||
* Copyright (c) 2011-2015 Los Alamos National Security, LLC. All rights
|
||||
* reserved.
|
||||
* Copyright (c) 2019 Google, Inc. All rights reserved.
|
||||
* Copyright (c) 2019-2020 Google, Inc. All rights reserved.
|
||||
* $COPYRIGHT$
|
||||
*
|
||||
* Additional copyrights may follow
|
||||
@ -22,49 +22,49 @@
|
||||
* $HEADER$
|
||||
*/
|
||||
|
||||
#ifndef MCA_BTL_VADER_SEND_FRAG_H
|
||||
#define MCA_BTL_VADER_SEND_FRAG_H
|
||||
#ifndef MCA_BTL_SM_SEND_FRAG_H
|
||||
#define MCA_BTL_SM_SEND_FRAG_H
|
||||
|
||||
#include "opal_config.h"
|
||||
|
||||
enum {
|
||||
MCA_BTL_VADER_FLAG_INLINE = 0,
|
||||
MCA_BTL_VADER_FLAG_SINGLE_COPY = 1,
|
||||
MCA_BTL_VADER_FLAG_COMPLETE = 2,
|
||||
MCA_BTL_VADER_FLAG_SETUP_FBOX = 4,
|
||||
MCA_BTL_SM_FLAG_INLINE = 0,
|
||||
MCA_BTL_SM_FLAG_SINGLE_COPY = 1,
|
||||
MCA_BTL_SM_FLAG_COMPLETE = 2,
|
||||
MCA_BTL_SM_FLAG_SETUP_FBOX = 4,
|
||||
};
|
||||
|
||||
struct mca_btl_vader_frag_t;
|
||||
struct mca_btl_vader_fbox_t;
|
||||
struct mca_btl_sm_frag_t;
|
||||
struct mca_btl_sm_fbox_t;
|
||||
|
||||
enum mca_btl_vader_sc_emu_type_t {
|
||||
MCA_BTL_VADER_OP_PUT,
|
||||
MCA_BTL_VADER_OP_GET,
|
||||
MCA_BTL_VADER_OP_ATOMIC,
|
||||
MCA_BTL_VADER_OP_CSWAP,
|
||||
enum mca_btl_sm_sc_emu_type_t {
|
||||
MCA_BTL_SM_OP_PUT,
|
||||
MCA_BTL_SM_OP_GET,
|
||||
MCA_BTL_SM_OP_ATOMIC,
|
||||
MCA_BTL_SM_OP_CSWAP,
|
||||
};
|
||||
typedef enum mca_btl_vader_sc_emu_type_t mca_btl_vader_sc_emu_type_t;
|
||||
typedef enum mca_btl_sm_sc_emu_type_t mca_btl_sm_sc_emu_type_t;
|
||||
|
||||
struct mca_btl_vader_sc_emu_hdr_t {
|
||||
mca_btl_vader_sc_emu_type_t type;
|
||||
struct mca_btl_sm_sc_emu_hdr_t {
|
||||
mca_btl_sm_sc_emu_type_t type;
|
||||
uint64_t addr;
|
||||
mca_btl_base_atomic_op_t op;
|
||||
int flags;
|
||||
int64_t operand[2];
|
||||
};
|
||||
typedef struct mca_btl_vader_sc_emu_hdr_t mca_btl_vader_sc_emu_hdr_t;
|
||||
typedef struct mca_btl_sm_sc_emu_hdr_t mca_btl_sm_sc_emu_hdr_t;
|
||||
|
||||
/**
|
||||
* FIFO fragment header
|
||||
*/
|
||||
struct mca_btl_vader_hdr_t {
|
||||
struct mca_btl_sm_hdr_t {
|
||||
/** next item in fifo. many peers may touch this */
|
||||
volatile intptr_t next;
|
||||
/** pointer back the the fragment */
|
||||
struct mca_btl_vader_frag_t *frag;
|
||||
struct mca_btl_sm_frag_t *frag;
|
||||
/** tag associated with this fragment (used to lookup callback) */
|
||||
mca_btl_base_tag_t tag;
|
||||
/** vader send flags (inline, complete, setup fbox, etc) */
|
||||
/** sm send flags (inline, complete, setup fbox, etc) */
|
||||
uint8_t flags;
|
||||
/** length of data following this header */
|
||||
int32_t len;
|
||||
@ -73,12 +73,12 @@ struct mca_btl_vader_hdr_t {
|
||||
/** if the fragment indicates to setup a fast box the base is stored here */
|
||||
intptr_t fbox_base;
|
||||
};
|
||||
typedef struct mca_btl_vader_hdr_t mca_btl_vader_hdr_t;
|
||||
typedef struct mca_btl_sm_hdr_t mca_btl_sm_hdr_t;
|
||||
|
||||
/**
|
||||
* shared memory send fragment derived type.
|
||||
*/
|
||||
struct mca_btl_vader_frag_t {
|
||||
struct mca_btl_sm_frag_t {
|
||||
/** base object */
|
||||
mca_btl_base_descriptor_t base;
|
||||
/** storage for segment data (max 2) */
|
||||
@ -86,11 +86,11 @@ struct mca_btl_vader_frag_t {
|
||||
/** endpoint this fragment is active on */
|
||||
struct mca_btl_base_endpoint_t *endpoint;
|
||||
/** fragment header (in the shared memory region) */
|
||||
mca_btl_vader_hdr_t *hdr;
|
||||
mca_btl_sm_hdr_t *hdr;
|
||||
/** free list this fragment was allocated within */
|
||||
opal_free_list_t *my_list;
|
||||
/** rdma callback data */
|
||||
struct mca_btl_vader_rdma_cbdata_t {
|
||||
struct mca_btl_sm_rdma_cbdata_t {
|
||||
void *local_address;
|
||||
uint64_t remote_address;
|
||||
mca_btl_base_rdma_completion_fn_t cbfunc;
|
||||
@ -101,11 +101,11 @@ struct mca_btl_vader_frag_t {
|
||||
} rdma;
|
||||
};
|
||||
|
||||
typedef struct mca_btl_vader_frag_t mca_btl_vader_frag_t;
|
||||
typedef struct mca_btl_sm_frag_t mca_btl_sm_frag_t;
|
||||
|
||||
static inline int mca_btl_vader_frag_alloc (mca_btl_vader_frag_t **frag, opal_free_list_t *list,
|
||||
struct mca_btl_base_endpoint_t *endpoint) {
|
||||
*frag = (mca_btl_vader_frag_t *) opal_free_list_get (list);
|
||||
static inline int mca_btl_sm_frag_alloc (mca_btl_sm_frag_t **frag, opal_free_list_t *list,
|
||||
struct mca_btl_base_endpoint_t *endpoint) {
|
||||
*frag = (mca_btl_sm_frag_t *) opal_free_list_get (list);
|
||||
if (OPAL_LIKELY(NULL != *frag)) {
|
||||
(*frag)->endpoint = endpoint;
|
||||
}
|
||||
@ -113,7 +113,7 @@ static inline int mca_btl_vader_frag_alloc (mca_btl_vader_frag_t **frag, opal_fr
|
||||
return OPAL_SUCCESS;
|
||||
}
|
||||
|
||||
static inline void mca_btl_vader_frag_return (mca_btl_vader_frag_t *frag)
|
||||
static inline void mca_btl_sm_frag_return (mca_btl_sm_frag_t *frag)
|
||||
{
|
||||
if (frag->hdr) {
|
||||
frag->hdr->flags = 0;
|
||||
@ -125,40 +125,40 @@ static inline void mca_btl_vader_frag_return (mca_btl_vader_frag_t *frag)
|
||||
opal_free_list_return (frag->my_list, (opal_free_list_item_t *)frag);
|
||||
}
|
||||
|
||||
OBJ_CLASS_DECLARATION(mca_btl_vader_frag_t);
|
||||
OBJ_CLASS_DECLARATION(mca_btl_sm_frag_t);
|
||||
|
||||
#define MCA_BTL_VADER_FRAG_ALLOC_EAGER(frag, endpoint) \
|
||||
mca_btl_vader_frag_alloc (&(frag), &mca_btl_vader_component.vader_frags_eager, endpoint)
|
||||
#define MCA_BTL_SM_FRAG_ALLOC_EAGER(frag, endpoint) \
|
||||
mca_btl_sm_frag_alloc (&(frag), &mca_btl_sm_component.sm_frags_eager, endpoint)
|
||||
|
||||
#define MCA_BTL_VADER_FRAG_ALLOC_MAX(frag, endpoint) \
|
||||
mca_btl_vader_frag_alloc (&(frag), &mca_btl_vader_component.vader_frags_max_send, endpoint)
|
||||
#define MCA_BTL_SM_FRAG_ALLOC_MAX(frag, endpoint) \
|
||||
mca_btl_sm_frag_alloc (&(frag), &mca_btl_sm_component.sm_frags_max_send, endpoint)
|
||||
|
||||
#define MCA_BTL_VADER_FRAG_ALLOC_USER(frag, endpoint) \
|
||||
mca_btl_vader_frag_alloc (&(frag), &mca_btl_vader_component.vader_frags_user, endpoint)
|
||||
#define MCA_BTL_SM_FRAG_ALLOC_USER(frag, endpoint) \
|
||||
mca_btl_sm_frag_alloc (&(frag), &mca_btl_sm_component.sm_frags_user, endpoint)
|
||||
|
||||
#define MCA_BTL_VADER_FRAG_RETURN(frag) mca_btl_vader_frag_return(frag)
|
||||
#define MCA_BTL_SM_FRAG_RETURN(frag) mca_btl_sm_frag_return(frag)
|
||||
|
||||
|
||||
static inline void mca_btl_vader_frag_complete (mca_btl_vader_frag_t *frag) {
|
||||
static inline void mca_btl_sm_frag_complete (mca_btl_sm_frag_t *frag) {
|
||||
/* save the descriptor flags since the callback is allowed to free the frag */
|
||||
int des_flags = frag->base.des_flags;
|
||||
|
||||
if (OPAL_UNLIKELY(MCA_BTL_DES_SEND_ALWAYS_CALLBACK & des_flags)) {
|
||||
/* completion callback */
|
||||
frag->base.des_cbfunc (&mca_btl_vader.super, frag->endpoint, &frag->base, OPAL_SUCCESS);
|
||||
frag->base.des_cbfunc (&mca_btl_sm.super, frag->endpoint, &frag->base, OPAL_SUCCESS);
|
||||
}
|
||||
|
||||
if (OPAL_LIKELY(des_flags & MCA_BTL_DES_FLAGS_BTL_OWNERSHIP)) {
|
||||
MCA_BTL_VADER_FRAG_RETURN(frag);
|
||||
MCA_BTL_SM_FRAG_RETURN(frag);
|
||||
}
|
||||
}
|
||||
|
||||
int mca_btl_vader_frag_init (opal_free_list_item_t *item, void *ctx);
|
||||
int mca_btl_sm_frag_init (opal_free_list_item_t *item, void *ctx);
|
||||
|
||||
static inline void mca_btl_vader_rdma_frag_advance (mca_btl_base_module_t *btl, mca_btl_base_endpoint_t *endpoint,
|
||||
mca_btl_vader_frag_t *frag, int status)
|
||||
static inline void mca_btl_sm_rdma_frag_advance (mca_btl_base_module_t *btl, mca_btl_base_endpoint_t *endpoint,
|
||||
mca_btl_sm_frag_t *frag, int status)
|
||||
{
|
||||
mca_btl_vader_sc_emu_hdr_t *hdr = (mca_btl_vader_sc_emu_hdr_t *) frag->segments[0].seg_addr.pval;
|
||||
mca_btl_sm_sc_emu_hdr_t *hdr = (mca_btl_sm_sc_emu_hdr_t *) frag->segments[0].seg_addr.pval;
|
||||
mca_btl_base_rdma_completion_fn_t cbfunc = frag->rdma.cbfunc;
|
||||
size_t hdr_size = sizeof (*hdr);
|
||||
size_t len = frag->rdma.sent ? frag->segments[0].seg_len - hdr_size : 0;
|
||||
@ -167,9 +167,9 @@ static inline void mca_btl_vader_rdma_frag_advance (mca_btl_base_module_t *btl,
|
||||
void *data = (void *) (hdr + 1);
|
||||
|
||||
if (frag->rdma.sent) {
|
||||
if (MCA_BTL_VADER_OP_GET == hdr->type) {
|
||||
if (MCA_BTL_SM_OP_GET == hdr->type) {
|
||||
memcpy (frag->rdma.local_address, data, len);
|
||||
} else if ((MCA_BTL_VADER_OP_ATOMIC == hdr->type || MCA_BTL_VADER_OP_CSWAP == hdr->type) &&
|
||||
} else if ((MCA_BTL_SM_OP_ATOMIC == hdr->type || MCA_BTL_SM_OP_CSWAP == hdr->type) &&
|
||||
frag->rdma.local_address) {
|
||||
if (8 == len) {
|
||||
*((int64_t *) frag->rdma.local_address) = hdr->operand[0];
|
||||
@ -180,55 +180,55 @@ static inline void mca_btl_vader_rdma_frag_advance (mca_btl_base_module_t *btl,
|
||||
}
|
||||
|
||||
if (frag->rdma.remaining) {
|
||||
size_t packet_size = (frag->rdma.remaining + hdr_size) <= mca_btl_vader.super.btl_max_send_size ?
|
||||
frag->rdma.remaining : mca_btl_vader.super.btl_max_send_size - hdr_size;
|
||||
size_t packet_size = (frag->rdma.remaining + hdr_size) <= mca_btl_sm.super.btl_max_send_size ?
|
||||
frag->rdma.remaining : mca_btl_sm.super.btl_max_send_size - hdr_size;
|
||||
|
||||
/* advance the local and remote pointers */
|
||||
frag->rdma.local_address = (void *)((uintptr_t) frag->rdma.local_address + len);
|
||||
frag->rdma.remote_address += len;
|
||||
|
||||
if (MCA_BTL_VADER_OP_PUT == hdr->type) {
|
||||
if (MCA_BTL_SM_OP_PUT == hdr->type) {
|
||||
/* copy the next block into the fragment buffer */
|
||||
memcpy ((void *) (hdr + 1), frag->rdma.local_address, packet_size);
|
||||
}
|
||||
|
||||
hdr->addr = frag->rdma.remote_address;
|
||||
/* clear out the complete flag before sending the fragment again */
|
||||
frag->hdr->flags &= ~MCA_BTL_VADER_FLAG_COMPLETE;
|
||||
frag->hdr->flags &= ~MCA_BTL_SM_FLAG_COMPLETE;
|
||||
frag->segments[0].seg_len = packet_size + sizeof (*hdr);
|
||||
frag->rdma.sent += packet_size;
|
||||
frag->rdma.remaining -= packet_size;
|
||||
|
||||
/* send is always successful */
|
||||
(void) mca_btl_vader_send (btl, endpoint, &frag->base, MCA_BTL_TAG_VADER);
|
||||
(void) mca_btl_sm_send (btl, endpoint, &frag->base, MCA_BTL_TAG_SM);
|
||||
return;
|
||||
}
|
||||
|
||||
/* return the fragment before calling the callback */
|
||||
MCA_BTL_VADER_FRAG_RETURN(frag);
|
||||
MCA_BTL_SM_FRAG_RETURN(frag);
|
||||
cbfunc (btl, endpoint, (void *)((uintptr_t) frag->rdma.local_address - frag->rdma.sent), NULL,
|
||||
context, cbdata, status);
|
||||
}
|
||||
|
||||
static inline int
|
||||
mca_btl_vader_rdma_frag_start (mca_btl_base_module_t *btl, mca_btl_base_endpoint_t *endpoint, int type,
|
||||
uint64_t operand1, uint64_t operand2, mca_btl_base_atomic_op_t op, int order,
|
||||
int flags, size_t size, void *local_address, int64_t remote_address,
|
||||
mca_btl_base_rdma_completion_fn_t cbfunc, void *cbcontext, void *cbdata)
|
||||
mca_btl_sm_rdma_frag_start (mca_btl_base_module_t *btl, mca_btl_base_endpoint_t *endpoint, int type,
|
||||
uint64_t operand1, uint64_t operand2, mca_btl_base_atomic_op_t op, int order,
|
||||
int flags, size_t size, void *local_address, int64_t remote_address,
|
||||
mca_btl_base_rdma_completion_fn_t cbfunc, void *cbcontext, void *cbdata)
|
||||
{
|
||||
mca_btl_vader_sc_emu_hdr_t *hdr;
|
||||
mca_btl_sm_sc_emu_hdr_t *hdr;
|
||||
size_t hdr_size = sizeof (*hdr);
|
||||
size_t packet_size = (size + hdr_size) <= mca_btl_vader.super.btl_max_send_size ? size :
|
||||
mca_btl_vader.super.btl_max_send_size - hdr_size;
|
||||
mca_btl_vader_frag_t *frag;
|
||||
size_t packet_size = (size + hdr_size) <= mca_btl_sm.super.btl_max_send_size ? size :
|
||||
mca_btl_sm.super.btl_max_send_size - hdr_size;
|
||||
mca_btl_sm_frag_t *frag;
|
||||
|
||||
frag = (mca_btl_vader_frag_t *) mca_btl_vader_alloc (btl, endpoint, order, packet_size + hdr_size,
|
||||
MCA_BTL_DES_SEND_ALWAYS_CALLBACK);
|
||||
frag = (mca_btl_sm_frag_t *) mca_btl_sm_alloc (btl, endpoint, order, packet_size + hdr_size,
|
||||
MCA_BTL_DES_SEND_ALWAYS_CALLBACK);
|
||||
if (OPAL_UNLIKELY(NULL == frag)) {
|
||||
return OPAL_ERR_OUT_OF_RESOURCE;
|
||||
}
|
||||
|
||||
frag->base.des_cbfunc = (mca_btl_base_completion_fn_t) mca_btl_vader_rdma_frag_advance;
|
||||
frag->base.des_cbfunc = (mca_btl_base_completion_fn_t) mca_btl_sm_rdma_frag_advance;
|
||||
frag->rdma.local_address = local_address;
|
||||
frag->rdma.remote_address = remote_address;
|
||||
frag->rdma.cbfunc = cbfunc;
|
||||
@ -237,7 +237,7 @@ mca_btl_vader_rdma_frag_start (mca_btl_base_module_t *btl, mca_btl_base_endpoint
|
||||
frag->rdma.remaining = size;
|
||||
frag->rdma.sent = 0;
|
||||
|
||||
hdr = (mca_btl_vader_sc_emu_hdr_t *) frag->segments[0].seg_addr.pval;
|
||||
hdr = (mca_btl_sm_sc_emu_hdr_t *) frag->segments[0].seg_addr.pval;
|
||||
|
||||
hdr->type = type;
|
||||
hdr->addr = remote_address;
|
||||
@ -246,8 +246,8 @@ mca_btl_vader_rdma_frag_start (mca_btl_base_module_t *btl, mca_btl_base_endpoint
|
||||
hdr->operand[0] = operand1;
|
||||
hdr->operand[1] = operand2;
|
||||
|
||||
mca_btl_vader_rdma_frag_advance (btl, endpoint, frag, OPAL_SUCCESS);
|
||||
mca_btl_sm_rdma_frag_advance (btl, endpoint, frag, OPAL_SUCCESS);
|
||||
return OPAL_SUCCESS;
|
||||
}
|
||||
|
||||
#endif /* MCA_BTL_VADER_SEND_FRAG_H */
|
||||
#endif /* MCA_BTL_SM_SEND_FRAG_H */
|
||||
@ -14,12 +14,12 @@
|
||||
|
||||
#include "opal_config.h"
|
||||
|
||||
#include "btl_vader.h"
|
||||
#include "btl_vader_frag.h"
|
||||
#include "btl_vader_endpoint.h"
|
||||
#include "btl_vader_xpmem.h"
|
||||
#include "btl_sm.h"
|
||||
#include "btl_sm_frag.h"
|
||||
#include "btl_sm_endpoint.h"
|
||||
#include "btl_sm_xpmem.h"
|
||||
|
||||
#if OPAL_BTL_VADER_HAVE_CMA
|
||||
#if OPAL_BTL_SM_HAVE_CMA
|
||||
#include <sys/uio.h>
|
||||
|
||||
#if OPAL_CMA_NEED_SYSCALL_DEFS
|
||||
@ -36,8 +36,8 @@
|
||||
* @param endpoint (IN) BTL addressing information
|
||||
* @param descriptor (IN) Description of the data to be transferred
|
||||
*/
|
||||
#if OPAL_BTL_VADER_HAVE_XPMEM
|
||||
int mca_btl_vader_get_xpmem (mca_btl_base_module_t *btl, mca_btl_base_endpoint_t *endpoint, void *local_address,
|
||||
#if OPAL_BTL_SM_HAVE_XPMEM
|
||||
int mca_btl_sm_get_xpmem (mca_btl_base_module_t *btl, mca_btl_base_endpoint_t *endpoint, void *local_address,
|
||||
uint64_t remote_address, mca_btl_base_registration_handle_t *local_handle,
|
||||
mca_btl_base_registration_handle_t *remote_handle, size_t size, int flags,
|
||||
int order, mca_btl_base_rdma_completion_fn_t cbfunc, void *cbcontext, void *cbdata)
|
||||
@ -49,14 +49,14 @@ int mca_btl_vader_get_xpmem (mca_btl_base_module_t *btl, mca_btl_base_endpoint_t
|
||||
(void) local_handle;
|
||||
(void) remote_handle;
|
||||
|
||||
reg = vader_get_registation (endpoint, (void *)(intptr_t) remote_address, size, 0, &rem_ptr);
|
||||
reg = sm_get_registation (endpoint, (void *)(intptr_t) remote_address, size, 0, &rem_ptr);
|
||||
if (OPAL_UNLIKELY(NULL == rem_ptr)) {
|
||||
return OPAL_ERROR;
|
||||
}
|
||||
|
||||
vader_memmove (local_address, rem_ptr, size);
|
||||
sm_memmove (local_address, rem_ptr, size);
|
||||
|
||||
vader_return_registration (reg, endpoint);
|
||||
sm_return_registration (reg, endpoint);
|
||||
|
||||
/* always call the callback function */
|
||||
cbfunc (btl, endpoint, local_address, local_handle, cbcontext, cbdata, OPAL_SUCCESS);
|
||||
@ -65,8 +65,8 @@ int mca_btl_vader_get_xpmem (mca_btl_base_module_t *btl, mca_btl_base_endpoint_t
|
||||
}
|
||||
#endif
|
||||
|
||||
#if OPAL_BTL_VADER_HAVE_CMA
|
||||
int mca_btl_vader_get_cma (mca_btl_base_module_t *btl, mca_btl_base_endpoint_t *endpoint, void *local_address,
|
||||
#if OPAL_BTL_SM_HAVE_CMA
|
||||
int mca_btl_sm_get_cma (mca_btl_base_module_t *btl, mca_btl_base_endpoint_t *endpoint, void *local_address,
|
||||
uint64_t remote_address, mca_btl_base_registration_handle_t *local_handle,
|
||||
mca_btl_base_registration_handle_t *remote_handle, size_t size, int flags,
|
||||
int order, mca_btl_base_rdma_completion_fn_t cbfunc, void *cbcontext, void *cbdata)
|
||||
@ -111,8 +111,8 @@ int mca_btl_vader_get_cma (mca_btl_base_module_t *btl, mca_btl_base_endpoint_t *
|
||||
}
|
||||
#endif
|
||||
|
||||
#if OPAL_BTL_VADER_HAVE_KNEM
|
||||
int mca_btl_vader_get_knem (mca_btl_base_module_t *btl, mca_btl_base_endpoint_t *endpoint, void *local_address,
|
||||
#if OPAL_BTL_SM_HAVE_KNEM
|
||||
int mca_btl_sm_get_knem (mca_btl_base_module_t *btl, mca_btl_base_endpoint_t *endpoint, void *local_address,
|
||||
uint64_t remote_address, mca_btl_base_registration_handle_t *local_handle,
|
||||
mca_btl_base_registration_handle_t *remote_handle, size_t size, int flags,
|
||||
int order, mca_btl_base_rdma_completion_fn_t cbfunc, void *cbcontext, void *cbdata)
|
||||
@ -134,15 +134,15 @@ int mca_btl_vader_get_knem (mca_btl_base_module_t *btl, mca_btl_base_endpoint_t
|
||||
/* Use the DMA flag if knem supports it *and* the segment length
|
||||
* is greater than the cutoff. Not that if DMA is not supported
|
||||
* or the user specified 0 for knem_dma_min the knem_dma_min was
|
||||
* set to UINT_MAX in mca_btl_vader_knem_init. */
|
||||
if (mca_btl_vader_component.knem_dma_min <= size) {
|
||||
* set to UINT_MAX in mca_btl_sm_knem_init. */
|
||||
if (mca_btl_sm_component.knem_dma_min <= size) {
|
||||
icopy.flags = KNEM_FLAG_DMA;
|
||||
}
|
||||
/* synchronous flags only, no need to specify icopy.async_status_index */
|
||||
|
||||
/* When the ioctl returns, the transfer is done and we can invoke
|
||||
the btl callback and return the frag */
|
||||
if (OPAL_UNLIKELY(0 != ioctl (mca_btl_vader.knem_fd, KNEM_CMD_INLINE_COPY, &icopy))) {
|
||||
if (OPAL_UNLIKELY(0 != ioctl (mca_btl_sm.knem_fd, KNEM_CMD_INLINE_COPY, &icopy))) {
|
||||
return OPAL_ERROR;
|
||||
}
|
||||
|
||||
@ -157,15 +157,15 @@ int mca_btl_vader_get_knem (mca_btl_base_module_t *btl, mca_btl_base_endpoint_t
|
||||
}
|
||||
#endif
|
||||
|
||||
int mca_btl_vader_get_sc_emu (mca_btl_base_module_t *btl, mca_btl_base_endpoint_t *endpoint, void *local_address,
|
||||
int mca_btl_sm_get_sc_emu (mca_btl_base_module_t *btl, mca_btl_base_endpoint_t *endpoint, void *local_address,
|
||||
uint64_t remote_address, mca_btl_base_registration_handle_t *local_handle,
|
||||
mca_btl_base_registration_handle_t *remote_handle, size_t size, int flags,
|
||||
int order, mca_btl_base_rdma_completion_fn_t cbfunc, void *cbcontext, void *cbdata)
|
||||
{
|
||||
if (size > mca_btl_vader.super.btl_get_limit) {
|
||||
if (size > mca_btl_sm.super.btl_get_limit) {
|
||||
return OPAL_ERR_NOT_AVAILABLE;
|
||||
}
|
||||
|
||||
return mca_btl_vader_rdma_frag_start (btl, endpoint, MCA_BTL_VADER_OP_GET, 0, 0, 0, order, flags, size,
|
||||
local_address, remote_address, cbfunc, cbcontext, cbdata);
|
||||
return mca_btl_sm_rdma_frag_start (btl, endpoint, MCA_BTL_SM_OP_GET, 0, 0, 0, order, flags, size,
|
||||
local_address, remote_address, cbfunc, cbcontext, cbdata);
|
||||
}
|
||||
@ -9,9 +9,9 @@
|
||||
* $HEADER$
|
||||
*/
|
||||
|
||||
#include "btl_vader.h"
|
||||
#include "btl_sm.h"
|
||||
|
||||
#if OPAL_BTL_VADER_HAVE_KNEM
|
||||
#if OPAL_BTL_SM_HAVE_KNEM
|
||||
|
||||
#include <stdio.h>
|
||||
#include <fcntl.h>
|
||||
@ -20,12 +20,12 @@
|
||||
|
||||
#include "opal/util/show_help.h"
|
||||
|
||||
OBJ_CLASS_INSTANCE(mca_btl_vader_registration_handle_t, mca_rcache_base_registration_t, NULL, NULL);
|
||||
OBJ_CLASS_INSTANCE(mca_btl_sm_registration_handle_t, mca_rcache_base_registration_t, NULL, NULL);
|
||||
|
||||
static int mca_btl_vader_knem_reg (void *reg_data, void *base, size_t size,
|
||||
static int mca_btl_sm_knem_reg (void *reg_data, void *base, size_t size,
|
||||
mca_rcache_base_registration_t *reg)
|
||||
{
|
||||
mca_btl_vader_registration_handle_t *knem_reg = (mca_btl_vader_registration_handle_t *) reg;
|
||||
mca_btl_sm_registration_handle_t *knem_reg = (mca_btl_sm_registration_handle_t *) reg;
|
||||
struct knem_cmd_create_region knem_cr;
|
||||
struct knem_cmd_param_iovec knem_iov;
|
||||
|
||||
@ -44,9 +44,9 @@ static int mca_btl_vader_knem_reg (void *reg_data, void *base, size_t size,
|
||||
knem_cr.protection |= PROT_READ;
|
||||
}
|
||||
|
||||
/* Vader will explicitly destroy this cookie */
|
||||
/* Sm will explicitly destroy this cookie */
|
||||
knem_cr.flags = 0;
|
||||
if (OPAL_UNLIKELY(ioctl(mca_btl_vader.knem_fd, KNEM_CMD_CREATE_REGION, &knem_cr) < 0)) {
|
||||
if (OPAL_UNLIKELY(ioctl(mca_btl_sm.knem_fd, KNEM_CMD_CREATE_REGION, &knem_cr) < 0)) {
|
||||
return OPAL_ERROR;
|
||||
}
|
||||
|
||||
@ -56,27 +56,27 @@ static int mca_btl_vader_knem_reg (void *reg_data, void *base, size_t size,
|
||||
return OPAL_SUCCESS;
|
||||
}
|
||||
|
||||
static int mca_btl_vader_knem_dereg (void *reg_data, mca_rcache_base_registration_t *reg)
|
||||
static int mca_btl_sm_knem_dereg (void *reg_data, mca_rcache_base_registration_t *reg)
|
||||
{
|
||||
mca_btl_vader_registration_handle_t *knem_reg = (mca_btl_vader_registration_handle_t *) reg;
|
||||
mca_btl_sm_registration_handle_t *knem_reg = (mca_btl_sm_registration_handle_t *) reg;
|
||||
|
||||
/* NTH: explicity ignore the return code. Don't care about this cookie anymore anyway. */
|
||||
(void) ioctl(mca_btl_vader.knem_fd, KNEM_CMD_DESTROY_REGION, &knem_reg->btl_handle.cookie);
|
||||
(void) ioctl(mca_btl_sm.knem_fd, KNEM_CMD_DESTROY_REGION, &knem_reg->btl_handle.cookie);
|
||||
|
||||
return OPAL_SUCCESS;
|
||||
}
|
||||
|
||||
static mca_btl_base_registration_handle_t *
|
||||
mca_btl_vader_register_mem_knem (struct mca_btl_base_module_t* btl,
|
||||
mca_btl_sm_register_mem_knem (struct mca_btl_base_module_t* btl,
|
||||
struct mca_btl_base_endpoint_t *endpoint,
|
||||
void *base, size_t size, uint32_t flags)
|
||||
{
|
||||
mca_btl_vader_t *vader_module = (mca_btl_vader_t *) btl;
|
||||
mca_btl_vader_registration_handle_t *reg = NULL;
|
||||
mca_btl_sm_t *sm_module = (mca_btl_sm_t *) btl;
|
||||
mca_btl_sm_registration_handle_t *reg = NULL;
|
||||
int access_flags = flags & MCA_BTL_REG_FLAG_ACCESS_ANY;
|
||||
int rc;
|
||||
|
||||
rc = vader_module->knem_rcache->rcache_register (vader_module->knem_rcache, base, size, 0,
|
||||
rc = sm_module->knem_rcache->rcache_register (sm_module->knem_rcache, base, size, 0,
|
||||
access_flags,
|
||||
(mca_rcache_base_registration_t **) ®);
|
||||
if (OPAL_UNLIKELY(OPAL_SUCCESS != rc)) {
|
||||
@ -87,41 +87,41 @@ mca_btl_vader_register_mem_knem (struct mca_btl_base_module_t* btl,
|
||||
}
|
||||
|
||||
static int
|
||||
mca_btl_vader_deregister_mem_knem (struct mca_btl_base_module_t *btl, struct mca_btl_base_registration_handle_t *handle)
|
||||
mca_btl_sm_deregister_mem_knem (struct mca_btl_base_module_t *btl, struct mca_btl_base_registration_handle_t *handle)
|
||||
{
|
||||
mca_btl_vader_t *vader_module = (mca_btl_vader_t *) btl;
|
||||
mca_btl_vader_registration_handle_t *reg =
|
||||
(mca_btl_vader_registration_handle_t *)((intptr_t) handle - offsetof (mca_btl_vader_registration_handle_t, btl_handle));
|
||||
mca_btl_sm_t *sm_module = (mca_btl_sm_t *) btl;
|
||||
mca_btl_sm_registration_handle_t *reg =
|
||||
(mca_btl_sm_registration_handle_t *)((intptr_t) handle - offsetof (mca_btl_sm_registration_handle_t, btl_handle));
|
||||
|
||||
vader_module->knem_rcache->rcache_deregister (vader_module->knem_rcache, ®->base);
|
||||
sm_module->knem_rcache->rcache_deregister (sm_module->knem_rcache, ®->base);
|
||||
|
||||
return OPAL_SUCCESS;
|
||||
}
|
||||
|
||||
int mca_btl_vader_knem_init (void)
|
||||
int mca_btl_sm_knem_init (void)
|
||||
{
|
||||
mca_rcache_base_resources_t rcache_resources = {
|
||||
.cache_name = "vader", .reg_data = NULL,
|
||||
.sizeof_reg = sizeof (mca_btl_vader_registration_handle_t),
|
||||
.register_mem = mca_btl_vader_knem_reg,
|
||||
.deregister_mem = mca_btl_vader_knem_dereg
|
||||
.cache_name = "sm", .reg_data = NULL,
|
||||
.sizeof_reg = sizeof (mca_btl_sm_registration_handle_t),
|
||||
.register_mem = mca_btl_sm_knem_reg,
|
||||
.deregister_mem = mca_btl_sm_knem_dereg
|
||||
};
|
||||
struct knem_cmd_info knem_info;
|
||||
int rc;
|
||||
|
||||
/* Open the knem device. Try to print a helpful message if we
|
||||
fail to open it. */
|
||||
mca_btl_vader.knem_fd = open("/dev/knem", O_RDWR);
|
||||
if (mca_btl_vader.knem_fd < 0) {
|
||||
mca_btl_sm.knem_fd = open("/dev/knem", O_RDWR);
|
||||
if (mca_btl_sm.knem_fd < 0) {
|
||||
if (EACCES == errno) {
|
||||
struct stat sbuf;
|
||||
if (0 != stat("/dev/knem", &sbuf)) {
|
||||
sbuf.st_mode = 0;
|
||||
}
|
||||
opal_show_help("help-btl-vader.txt", "knem permission denied",
|
||||
opal_show_help("help-btl-sm.txt", "knem permission denied",
|
||||
true, opal_process_info.nodename, sbuf.st_mode);
|
||||
} else {
|
||||
opal_show_help("help-btl-vader.txt", "knem fail open",
|
||||
opal_show_help("help-btl-sm.txt", "knem fail open",
|
||||
true, opal_process_info.nodename, errno,
|
||||
strerror(errno));
|
||||
}
|
||||
@ -133,67 +133,67 @@ int mca_btl_vader_knem_init (void)
|
||||
/* Check that the ABI if kernel module running is the same
|
||||
* as what we were compiled against. */
|
||||
memset (&knem_info, 0, sizeof (knem_info));
|
||||
rc = ioctl(mca_btl_vader.knem_fd, KNEM_CMD_GET_INFO, &knem_info);
|
||||
rc = ioctl(mca_btl_sm.knem_fd, KNEM_CMD_GET_INFO, &knem_info);
|
||||
if (rc < 0) {
|
||||
opal_show_help("help-btl-vader.txt", "knem get ABI fail",
|
||||
opal_show_help("help-btl-sm.txt", "knem get ABI fail",
|
||||
true, opal_process_info.nodename, errno,
|
||||
strerror(errno));
|
||||
break;
|
||||
}
|
||||
|
||||
if (KNEM_ABI_VERSION != knem_info.abi) {
|
||||
opal_show_help("help-btl-vader.txt", "knem ABI mismatch",
|
||||
opal_show_help("help-btl-sm.txt", "knem ABI mismatch",
|
||||
true, opal_process_info.nodename, KNEM_ABI_VERSION,
|
||||
knem_info.abi);
|
||||
break;
|
||||
}
|
||||
|
||||
if (!(mca_btl_vader_component.knem_dma_min && (knem_info.features & KNEM_FEATURE_DMA))) {
|
||||
if (!(mca_btl_sm_component.knem_dma_min && (knem_info.features & KNEM_FEATURE_DMA))) {
|
||||
/* disable DMA */
|
||||
mca_btl_vader_component.knem_dma_min = UINT_MAX;
|
||||
mca_btl_sm_component.knem_dma_min = UINT_MAX;
|
||||
}
|
||||
|
||||
/* TODO: add async support */
|
||||
|
||||
/* knem set up successfully */
|
||||
mca_btl_vader.super.btl_get = mca_btl_vader_get_knem;
|
||||
mca_btl_vader.super.btl_put = mca_btl_vader_put_knem;
|
||||
mca_btl_sm.super.btl_get = mca_btl_sm_get_knem;
|
||||
mca_btl_sm.super.btl_put = mca_btl_sm_put_knem;
|
||||
|
||||
/* knem requires registration */
|
||||
mca_btl_vader.super.btl_register_mem = mca_btl_vader_register_mem_knem;
|
||||
mca_btl_vader.super.btl_deregister_mem = mca_btl_vader_deregister_mem_knem;
|
||||
mca_btl_vader.super.btl_registration_handle_size = sizeof (mca_btl_base_registration_handle_t);
|
||||
mca_btl_sm.super.btl_register_mem = mca_btl_sm_register_mem_knem;
|
||||
mca_btl_sm.super.btl_deregister_mem = mca_btl_sm_deregister_mem_knem;
|
||||
mca_btl_sm.super.btl_registration_handle_size = sizeof (mca_btl_base_registration_handle_t);
|
||||
|
||||
mca_btl_vader.knem_rcache = mca_rcache_base_module_create ("grdma", NULL,
|
||||
mca_btl_sm.knem_rcache = mca_rcache_base_module_create ("grdma", NULL,
|
||||
&rcache_resources);
|
||||
if (NULL == mca_btl_vader.knem_rcache) {
|
||||
if (NULL == mca_btl_sm.knem_rcache) {
|
||||
return OPAL_ERR_OUT_OF_RESOURCE;
|
||||
}
|
||||
|
||||
return OPAL_SUCCESS;
|
||||
} while (0);
|
||||
|
||||
mca_btl_vader_knem_fini ();
|
||||
mca_btl_sm_knem_fini ();
|
||||
|
||||
return OPAL_ERR_NOT_AVAILABLE;;
|
||||
}
|
||||
|
||||
int mca_btl_vader_knem_fini (void)
|
||||
int mca_btl_sm_knem_fini (void)
|
||||
{
|
||||
if (-1 != mca_btl_vader.knem_fd) {
|
||||
close (mca_btl_vader.knem_fd);
|
||||
mca_btl_vader.knem_fd = -1;
|
||||
if (-1 != mca_btl_sm.knem_fd) {
|
||||
close (mca_btl_sm.knem_fd);
|
||||
mca_btl_sm.knem_fd = -1;
|
||||
}
|
||||
|
||||
if (mca_btl_vader.knem_rcache) {
|
||||
(void) mca_rcache_base_module_destroy (mca_btl_vader.knem_rcache);
|
||||
mca_btl_vader.knem_rcache = NULL;
|
||||
if (mca_btl_sm.knem_rcache) {
|
||||
(void) mca_rcache_base_module_destroy (mca_btl_sm.knem_rcache);
|
||||
mca_btl_sm.knem_rcache = NULL;
|
||||
}
|
||||
|
||||
return OPAL_SUCCESS;
|
||||
}
|
||||
|
||||
int mca_btl_vader_knem_progress (void)
|
||||
int mca_btl_sm_knem_progress (void)
|
||||
{
|
||||
/* NTH: does nothing until async support is added */
|
||||
return OPAL_SUCCESS;
|
||||
@ -9,10 +9,10 @@
|
||||
* $HEADER$
|
||||
*/
|
||||
|
||||
#if !defined(BTL_VADER_KNEM_H)
|
||||
#define BTL_VADER_KNEM_H
|
||||
#if !defined(BTL_SM_KNEM_H)
|
||||
#define BTL_SM_KNEM_H
|
||||
|
||||
#if OPAL_BTL_VADER_HAVE_KNEM
|
||||
#if OPAL_BTL_SM_HAVE_KNEM
|
||||
|
||||
#include <knem_io.h>
|
||||
#include <sys/mman.h>
|
||||
@ -23,16 +23,16 @@ struct mca_btl_base_registration_handle_t {
|
||||
intptr_t base_addr;
|
||||
};
|
||||
|
||||
struct mca_btl_vader_registration_handle_t {
|
||||
struct mca_btl_sm_registration_handle_t {
|
||||
mca_rcache_base_registration_t base;
|
||||
mca_btl_base_registration_handle_t btl_handle;
|
||||
};
|
||||
typedef struct mca_btl_vader_registration_handle_t mca_btl_vader_registration_handle_t;
|
||||
typedef struct mca_btl_sm_registration_handle_t mca_btl_sm_registration_handle_t;
|
||||
|
||||
int mca_btl_vader_knem_init (void);
|
||||
int mca_btl_vader_knem_fini (void);
|
||||
int mca_btl_vader_knem_progress (void);
|
||||
int mca_btl_sm_knem_init (void);
|
||||
int mca_btl_sm_knem_fini (void);
|
||||
int mca_btl_sm_knem_progress (void);
|
||||
|
||||
#endif /* OPAL_BTL_VADER_HAVE_KNEM */
|
||||
#endif /* OPAL_BTL_SM_HAVE_KNEM */
|
||||
|
||||
#endif /* defined(BTL_VADER_KNEM_H) */
|
||||
#endif /* defined(BTL_SM_KNEM_H) */
|
||||
@ -30,24 +30,24 @@
|
||||
#include "opal_config.h"
|
||||
#include "opal/util/show_help.h"
|
||||
|
||||
#include "btl_vader.h"
|
||||
#include "btl_vader_endpoint.h"
|
||||
#include "btl_vader_fifo.h"
|
||||
#include "btl_vader_fbox.h"
|
||||
#include "btl_vader_xpmem.h"
|
||||
#include "btl_sm.h"
|
||||
#include "btl_sm_endpoint.h"
|
||||
#include "btl_sm_fifo.h"
|
||||
#include "btl_sm_fbox.h"
|
||||
#include "btl_sm_xpmem.h"
|
||||
|
||||
#include <string.h>
|
||||
|
||||
static int vader_del_procs (struct mca_btl_base_module_t *btl,
|
||||
static int sm_del_procs (struct mca_btl_base_module_t *btl,
|
||||
size_t nprocs, struct opal_proc_t **procs,
|
||||
struct mca_btl_base_endpoint_t **peers);
|
||||
|
||||
static int vader_register_error_cb (struct mca_btl_base_module_t* btl,
|
||||
static int sm_register_error_cb (struct mca_btl_base_module_t* btl,
|
||||
mca_btl_base_module_error_cb_fn_t cbfunc);
|
||||
|
||||
static int vader_finalize (struct mca_btl_base_module_t* btl);
|
||||
static int sm_finalize (struct mca_btl_base_module_t* btl);
|
||||
|
||||
static struct mca_btl_base_descriptor_t *vader_prepare_src (
|
||||
static struct mca_btl_base_descriptor_t *sm_prepare_src (
|
||||
struct mca_btl_base_module_t *btl,
|
||||
struct mca_btl_base_endpoint_t *endpoint,
|
||||
struct opal_convertor_t *convertor,
|
||||
@ -57,27 +57,27 @@ static struct mca_btl_base_descriptor_t *vader_prepare_src (
|
||||
uint32_t flags
|
||||
);
|
||||
|
||||
static int vader_add_procs(struct mca_btl_base_module_t* btl,
|
||||
static int sm_add_procs(struct mca_btl_base_module_t* btl,
|
||||
size_t nprocs, struct opal_proc_t **procs,
|
||||
struct mca_btl_base_endpoint_t** peers,
|
||||
struct opal_bitmap_t* reachability);
|
||||
|
||||
static int vader_ft_event (int state);
|
||||
static int sm_ft_event (int state);
|
||||
|
||||
mca_btl_vader_t mca_btl_vader = {
|
||||
mca_btl_sm_t mca_btl_sm = {
|
||||
{
|
||||
&mca_btl_vader_component.super,
|
||||
.btl_add_procs = vader_add_procs,
|
||||
.btl_del_procs = vader_del_procs,
|
||||
.btl_finalize = vader_finalize,
|
||||
.btl_alloc = mca_btl_vader_alloc,
|
||||
.btl_free = mca_btl_vader_free,
|
||||
.btl_prepare_src = vader_prepare_src,
|
||||
.btl_send = mca_btl_vader_send,
|
||||
.btl_sendi = mca_btl_vader_sendi,
|
||||
&mca_btl_sm_component.super,
|
||||
.btl_add_procs = sm_add_procs,
|
||||
.btl_del_procs = sm_del_procs,
|
||||
.btl_finalize = sm_finalize,
|
||||
.btl_alloc = mca_btl_sm_alloc,
|
||||
.btl_free = mca_btl_sm_free,
|
||||
.btl_prepare_src = sm_prepare_src,
|
||||
.btl_send = mca_btl_sm_send,
|
||||
.btl_sendi = mca_btl_sm_sendi,
|
||||
.btl_dump = mca_btl_base_dump,
|
||||
.btl_register_error = vader_register_error_cb,
|
||||
.btl_ft_event = vader_ft_event
|
||||
.btl_register_error = sm_register_error_cb,
|
||||
.btl_ft_event = sm_ft_event
|
||||
}
|
||||
};
|
||||
|
||||
@ -89,23 +89,23 @@ mca_btl_vader_t mca_btl_vader = {
|
||||
*/
|
||||
|
||||
/* coverity[+kill] */
|
||||
static void vader_btl_exit(mca_btl_vader_t *btl)
|
||||
static void sm_btl_exit(mca_btl_sm_t *btl)
|
||||
{
|
||||
if (NULL != btl && NULL != btl->error_cb) {
|
||||
btl->error_cb(&btl->super, MCA_BTL_ERROR_FLAGS_FATAL,
|
||||
(opal_proc_t*) opal_proc_local_get(),
|
||||
"The vader BTL is aborting the MPI job (via PML error callback).");
|
||||
"The sm BTL is aborting the MPI job (via PML error callback).");
|
||||
}
|
||||
|
||||
/* If the PML error callback returns (or if there wasn't one), just exit. Shrug. */
|
||||
fprintf(stderr, "*** The Open MPI vader BTL is aborting the MPI job (via exit(3)).\n");
|
||||
fprintf(stderr, "*** The Open MPI sm BTL is aborting the MPI job (via exit(3)).\n");
|
||||
fflush(stderr);
|
||||
exit(1);
|
||||
}
|
||||
|
||||
static int vader_btl_first_time_init(mca_btl_vader_t *vader_btl, int n)
|
||||
static int sm_btl_first_time_init(mca_btl_sm_t *sm_btl, int n)
|
||||
{
|
||||
mca_btl_vader_component_t *component = &mca_btl_vader_component;
|
||||
mca_btl_sm_component_t *component = &mca_btl_sm_component;
|
||||
int rc;
|
||||
|
||||
/* generate the endpoints */
|
||||
@ -121,16 +121,16 @@ static int vader_btl_first_time_init(mca_btl_vader_t *vader_btl, int n)
|
||||
return OPAL_ERR_OUT_OF_RESOURCE;
|
||||
}
|
||||
|
||||
component->mpool = mca_mpool_basic_create ((void *) (component->my_segment + MCA_BTL_VADER_FIFO_SIZE),
|
||||
(unsigned long) (mca_btl_vader_component.segment_size - MCA_BTL_VADER_FIFO_SIZE), 64);
|
||||
component->mpool = mca_mpool_basic_create ((void *) (component->my_segment + MCA_BTL_SM_FIFO_SIZE),
|
||||
(unsigned long) (mca_btl_sm_component.segment_size - MCA_BTL_SM_FIFO_SIZE), 64);
|
||||
if (NULL == component->mpool) {
|
||||
free (component->endpoints);
|
||||
return OPAL_ERR_OUT_OF_RESOURCE;
|
||||
}
|
||||
|
||||
rc = opal_free_list_init (&component->vader_fboxes, sizeof (opal_free_list_item_t), 8,
|
||||
OBJ_CLASS(opal_free_list_item_t), mca_btl_vader_component.fbox_size,
|
||||
opal_cache_line_size, 0, mca_btl_vader_component.fbox_max, 4,
|
||||
rc = opal_free_list_init (&component->sm_fboxes, sizeof (opal_free_list_item_t), 8,
|
||||
OBJ_CLASS(opal_free_list_item_t), mca_btl_sm_component.fbox_size,
|
||||
opal_cache_line_size, 0, mca_btl_sm_component.fbox_max, 4,
|
||||
component->mpool, 0, NULL, NULL, NULL);
|
||||
if (OPAL_SUCCESS != rc) {
|
||||
return rc;
|
||||
@ -138,55 +138,55 @@ static int vader_btl_first_time_init(mca_btl_vader_t *vader_btl, int n)
|
||||
|
||||
/* initialize fragment descriptor free lists */
|
||||
/* initialize free list for small send and inline fragments */
|
||||
rc = opal_free_list_init (&component->vader_frags_user,
|
||||
sizeof(mca_btl_vader_frag_t),
|
||||
opal_cache_line_size, OBJ_CLASS(mca_btl_vader_frag_t),
|
||||
mca_btl_vader_component.max_inline_send + sizeof (mca_btl_vader_frag_t),
|
||||
opal_cache_line_size, component->vader_free_list_num,
|
||||
component->vader_free_list_max,
|
||||
component->vader_free_list_inc,
|
||||
component->mpool, 0, NULL, mca_btl_vader_frag_init,
|
||||
&component->vader_frags_user);
|
||||
rc = opal_free_list_init (&component->sm_frags_user,
|
||||
sizeof(mca_btl_sm_frag_t),
|
||||
opal_cache_line_size, OBJ_CLASS(mca_btl_sm_frag_t),
|
||||
mca_btl_sm_component.max_inline_send + sizeof (mca_btl_sm_frag_t),
|
||||
opal_cache_line_size, component->sm_free_list_num,
|
||||
component->sm_free_list_max,
|
||||
component->sm_free_list_inc,
|
||||
component->mpool, 0, NULL, mca_btl_sm_frag_init,
|
||||
&component->sm_frags_user);
|
||||
if (OPAL_SUCCESS != rc) {
|
||||
return rc;
|
||||
}
|
||||
|
||||
/* initialize free list for buffered send fragments */
|
||||
rc = opal_free_list_init (&component->vader_frags_eager,
|
||||
sizeof (mca_btl_vader_frag_t),
|
||||
opal_cache_line_size, OBJ_CLASS(mca_btl_vader_frag_t),
|
||||
mca_btl_vader.super.btl_eager_limit + sizeof (mca_btl_vader_frag_t),
|
||||
opal_cache_line_size, component->vader_free_list_num,
|
||||
component->vader_free_list_max,
|
||||
component->vader_free_list_inc,
|
||||
component->mpool, 0, NULL, mca_btl_vader_frag_init,
|
||||
&component->vader_frags_eager);
|
||||
rc = opal_free_list_init (&component->sm_frags_eager,
|
||||
sizeof (mca_btl_sm_frag_t),
|
||||
opal_cache_line_size, OBJ_CLASS(mca_btl_sm_frag_t),
|
||||
mca_btl_sm.super.btl_eager_limit + sizeof (mca_btl_sm_frag_t),
|
||||
opal_cache_line_size, component->sm_free_list_num,
|
||||
component->sm_free_list_max,
|
||||
component->sm_free_list_inc,
|
||||
component->mpool, 0, NULL, mca_btl_sm_frag_init,
|
||||
&component->sm_frags_eager);
|
||||
if (OPAL_SUCCESS != rc) {
|
||||
return rc;
|
||||
}
|
||||
|
||||
if (MCA_BTL_VADER_XPMEM != mca_btl_vader_component.single_copy_mechanism) {
|
||||
if (MCA_BTL_SM_XPMEM != mca_btl_sm_component.single_copy_mechanism) {
|
||||
/* initialize free list for buffered send fragments */
|
||||
rc = opal_free_list_init (&component->vader_frags_max_send,
|
||||
sizeof (mca_btl_vader_frag_t),
|
||||
opal_cache_line_size, OBJ_CLASS(mca_btl_vader_frag_t),
|
||||
mca_btl_vader.super.btl_max_send_size + sizeof (mca_btl_vader_frag_t),
|
||||
opal_cache_line_size, component->vader_free_list_num,
|
||||
component->vader_free_list_max,
|
||||
component->vader_free_list_inc,
|
||||
component->mpool, 0, NULL, mca_btl_vader_frag_init,
|
||||
&component->vader_frags_max_send);
|
||||
rc = opal_free_list_init (&component->sm_frags_max_send,
|
||||
sizeof (mca_btl_sm_frag_t),
|
||||
opal_cache_line_size, OBJ_CLASS(mca_btl_sm_frag_t),
|
||||
mca_btl_sm.super.btl_max_send_size + sizeof (mca_btl_sm_frag_t),
|
||||
opal_cache_line_size, component->sm_free_list_num,
|
||||
component->sm_free_list_max,
|
||||
component->sm_free_list_inc,
|
||||
component->mpool, 0, NULL, mca_btl_sm_frag_init,
|
||||
&component->sm_frags_max_send);
|
||||
if (OPAL_SUCCESS != rc) {
|
||||
return rc;
|
||||
}
|
||||
}
|
||||
|
||||
/* set flag indicating btl has been inited */
|
||||
vader_btl->btl_inited = true;
|
||||
sm_btl->btl_inited = true;
|
||||
|
||||
#if OPAL_BTL_VADER_HAVE_XPMEM
|
||||
if (MCA_BTL_VADER_XPMEM == mca_btl_vader_component.single_copy_mechanism) {
|
||||
mca_btl_vader_component.vma_module = mca_rcache_base_vma_module_alloc ();
|
||||
#if OPAL_BTL_SM_HAVE_XPMEM
|
||||
if (MCA_BTL_SM_XPMEM == mca_btl_sm_component.single_copy_mechanism) {
|
||||
mca_btl_sm_component.vma_module = mca_rcache_base_vma_module_alloc ();
|
||||
}
|
||||
#endif
|
||||
|
||||
@ -194,18 +194,18 @@ static int vader_btl_first_time_init(mca_btl_vader_t *vader_btl, int n)
|
||||
}
|
||||
|
||||
|
||||
static int init_vader_endpoint (struct mca_btl_base_endpoint_t *ep, struct opal_proc_t *proc, int remote_rank) {
|
||||
mca_btl_vader_component_t *component = &mca_btl_vader_component;
|
||||
union vader_modex_t *modex;
|
||||
static int init_sm_endpoint (struct mca_btl_base_endpoint_t *ep, struct opal_proc_t *proc, int remote_rank) {
|
||||
mca_btl_sm_component_t *component = &mca_btl_sm_component;
|
||||
union sm_modex_t *modex;
|
||||
ino_t my_user_ns_id;
|
||||
size_t msg_size;
|
||||
int rc;
|
||||
|
||||
OBJ_CONSTRUCT(ep, mca_btl_vader_endpoint_t);
|
||||
OBJ_CONSTRUCT(ep, mca_btl_sm_endpoint_t);
|
||||
|
||||
ep->peer_smp_rank = remote_rank;
|
||||
|
||||
if (remote_rank != MCA_BTL_VADER_LOCAL_RANK) {
|
||||
if (remote_rank != MCA_BTL_SM_LOCAL_RANK) {
|
||||
OPAL_MODEX_RECV_IMMEDIATE(rc, &component->super.btl_version,
|
||||
&proc->proc_name, (void **) &modex, &msg_size);
|
||||
if (OPAL_SUCCESS != rc) {
|
||||
@ -213,12 +213,12 @@ static int init_vader_endpoint (struct mca_btl_base_endpoint_t *ep, struct opal_
|
||||
}
|
||||
|
||||
/* attach to the remote segment */
|
||||
#if OPAL_BTL_VADER_HAVE_XPMEM
|
||||
if (MCA_BTL_VADER_XPMEM == mca_btl_vader_component.single_copy_mechanism) {
|
||||
#if OPAL_BTL_SM_HAVE_XPMEM
|
||||
if (MCA_BTL_SM_XPMEM == mca_btl_sm_component.single_copy_mechanism) {
|
||||
/* always use xpmem if it is available */
|
||||
ep->segment_data.xpmem.apid = xpmem_get (modex->xpmem.seg_id, XPMEM_RDWR, XPMEM_PERMIT_MODE, (void *) 0666);
|
||||
ep->segment_data.xpmem.address_max = modex->xpmem.address_max;
|
||||
(void) vader_get_registation (ep, modex->xpmem.segment_base, mca_btl_vader_component.segment_size,
|
||||
(void) sm_get_registation (ep, modex->xpmem.segment_base, mca_btl_sm_component.segment_size,
|
||||
MCA_RCACHE_FLAGS_PERSIST, (void **) &ep->segment_base);
|
||||
} else {
|
||||
#endif
|
||||
@ -235,12 +235,12 @@ static int init_vader_endpoint (struct mca_btl_base_endpoint_t *ep, struct opal_
|
||||
return OPAL_ERROR;
|
||||
}
|
||||
|
||||
if (MCA_BTL_VADER_CMA == mca_btl_vader_component.single_copy_mechanism) {
|
||||
my_user_ns_id = mca_btl_vader_get_user_ns_id();
|
||||
if (MCA_BTL_SM_CMA == mca_btl_sm_component.single_copy_mechanism) {
|
||||
my_user_ns_id = mca_btl_sm_get_user_ns_id();
|
||||
if (my_user_ns_id != modex->other.user_ns_id) {
|
||||
mca_base_var_source_t source;
|
||||
int vari;
|
||||
rc = mca_base_var_find_by_name("btl_vader_single_copy_mechanism", &vari);
|
||||
rc = mca_base_var_find_by_name("btl_sm_single_copy_mechanism", &vari);
|
||||
if (OPAL_ERROR == rc) {
|
||||
return OPAL_ERROR;
|
||||
}
|
||||
@ -254,28 +254,28 @@ static int init_vader_endpoint (struct mca_btl_base_endpoint_t *ep, struct opal_
|
||||
* for processes running in different user namespaces even if
|
||||
* all involved user IDs are mapped to the same user ID.
|
||||
*
|
||||
* Fallback to MCA_BTL_VADER_EMUL.
|
||||
* Fallback to MCA_BTL_SM_EMUL.
|
||||
*/
|
||||
if (MCA_BASE_VAR_SOURCE_DEFAULT != source) {
|
||||
/* If CMA has been explicitly selected we want to error out */
|
||||
opal_show_help("help-btl-vader.txt", "cma-different-user-namespace-error",
|
||||
opal_show_help("help-btl-sm.txt", "cma-different-user-namespace-error",
|
||||
true, opal_process_info.nodename);
|
||||
vader_btl_exit(&mca_btl_vader);
|
||||
sm_btl_exit(&mca_btl_sm);
|
||||
}
|
||||
/*
|
||||
* If CMA has been selected because it is the default or
|
||||
* some fallback, this falls back even further.
|
||||
*/
|
||||
opal_show_help("help-btl-vader.txt", "cma-different-user-namespace-warning",
|
||||
opal_show_help("help-btl-sm.txt", "cma-different-user-namespace-warning",
|
||||
true, opal_process_info.nodename);
|
||||
mca_btl_vader_component.single_copy_mechanism = MCA_BTL_VADER_EMUL;
|
||||
mca_btl_vader.super.btl_get = mca_btl_vader_get_sc_emu;
|
||||
mca_btl_vader.super.btl_put = mca_btl_vader_put_sc_emu;
|
||||
mca_btl_vader.super.btl_put_limit = mca_btl_vader.super.btl_max_send_size - sizeof (mca_btl_vader_sc_emu_hdr_t);
|
||||
mca_btl_vader.super.btl_get_limit = mca_btl_vader.super.btl_max_send_size - sizeof (mca_btl_vader_sc_emu_hdr_t);
|
||||
mca_btl_sm_component.single_copy_mechanism = MCA_BTL_SM_EMUL;
|
||||
mca_btl_sm.super.btl_get = mca_btl_sm_get_sc_emu;
|
||||
mca_btl_sm.super.btl_put = mca_btl_sm_put_sc_emu;
|
||||
mca_btl_sm.super.btl_put_limit = mca_btl_sm.super.btl_max_send_size - sizeof (mca_btl_sm_sc_emu_hdr_t);
|
||||
mca_btl_sm.super.btl_get_limit = mca_btl_sm.super.btl_max_send_size - sizeof (mca_btl_sm_sc_emu_hdr_t);
|
||||
}
|
||||
}
|
||||
#if OPAL_BTL_VADER_HAVE_XPMEM
|
||||
#if OPAL_BTL_SM_HAVE_XPMEM
|
||||
}
|
||||
#endif
|
||||
OBJ_CONSTRUCT(&ep->lock, opal_mutex_t);
|
||||
@ -286,12 +286,12 @@ static int init_vader_endpoint (struct mca_btl_base_endpoint_t *ep, struct opal_
|
||||
ep->segment_base = component->my_segment;
|
||||
}
|
||||
|
||||
ep->fifo = (struct vader_fifo_t *) ep->segment_base;
|
||||
ep->fifo = (struct sm_fifo_t *) ep->segment_base;
|
||||
|
||||
return OPAL_SUCCESS;
|
||||
}
|
||||
|
||||
static int fini_vader_endpoint (struct mca_btl_base_endpoint_t *ep)
|
||||
static int fini_sm_endpoint (struct mca_btl_base_endpoint_t *ep)
|
||||
{
|
||||
/* check if the endpoint is initialized. avoids a double-destruct */
|
||||
if (ep->fifo) {
|
||||
@ -315,13 +315,13 @@ static int fini_vader_endpoint (struct mca_btl_base_endpoint_t *ep)
|
||||
*
|
||||
*/
|
||||
|
||||
static int vader_add_procs (struct mca_btl_base_module_t* btl,
|
||||
static int sm_add_procs (struct mca_btl_base_module_t* btl,
|
||||
size_t nprocs, struct opal_proc_t **procs,
|
||||
struct mca_btl_base_endpoint_t **peers,
|
||||
opal_bitmap_t *reachability)
|
||||
{
|
||||
mca_btl_vader_component_t *component = &mca_btl_vader_component;
|
||||
mca_btl_vader_t *vader_btl = (mca_btl_vader_t *) btl;
|
||||
mca_btl_sm_component_t *component = &mca_btl_sm_component;
|
||||
mca_btl_sm_t *sm_btl = (mca_btl_sm_t *) btl;
|
||||
const opal_proc_t *my_proc;
|
||||
int rc = OPAL_SUCCESS;
|
||||
|
||||
@ -333,12 +333,13 @@ static int vader_add_procs (struct mca_btl_base_module_t* btl,
|
||||
}
|
||||
|
||||
/* jump out if there's not someone we can talk to */
|
||||
if (1 > MCA_BTL_VADER_NUM_LOCAL_PEERS) {
|
||||
if (1 > MCA_BTL_SM_NUM_LOCAL_PEERS) {
|
||||
return OPAL_SUCCESS;
|
||||
}
|
||||
|
||||
if (!vader_btl->btl_inited) {
|
||||
rc = vader_btl_first_time_init (vader_btl, 1 + MCA_BTL_VADER_NUM_LOCAL_PEERS);
|
||||
|
||||
if (!sm_btl->btl_inited) {
|
||||
rc = sm_btl_first_time_init (sm_btl, 1 + MCA_BTL_SM_NUM_LOCAL_PEERS);
|
||||
if (rc != OPAL_SUCCESS) {
|
||||
return rc;
|
||||
}
|
||||
@ -365,7 +366,7 @@ static int vader_add_procs (struct mca_btl_base_module_t* btl,
|
||||
int rank = opal_atomic_fetch_add_32(&component -> local_rank, 1);
|
||||
|
||||
peers[proc] = component->endpoints + rank;
|
||||
rc = init_vader_endpoint (peers[proc], procs[proc], rank);
|
||||
rc = init_sm_endpoint (peers[proc], procs[proc], rank);
|
||||
if (OPAL_SUCCESS != rc) {
|
||||
break;
|
||||
}
|
||||
@ -384,13 +385,13 @@ static int vader_add_procs (struct mca_btl_base_module_t* btl,
|
||||
*
|
||||
*/
|
||||
|
||||
static int vader_del_procs(struct mca_btl_base_module_t *btl,
|
||||
static int sm_del_procs(struct mca_btl_base_module_t *btl,
|
||||
size_t nprocs, struct opal_proc_t **procs,
|
||||
struct mca_btl_base_endpoint_t **peers)
|
||||
{
|
||||
for (size_t i = 0 ; i < nprocs ; ++i) {
|
||||
if (peers[i]) {
|
||||
fini_vader_endpoint (peers[i]);
|
||||
fini_sm_endpoint (peers[i]);
|
||||
peers[i] = NULL;
|
||||
}
|
||||
}
|
||||
@ -412,35 +413,35 @@ static int vader_del_procs(struct mca_btl_base_module_t *btl,
|
||||
*
|
||||
*/
|
||||
|
||||
static int vader_finalize(struct mca_btl_base_module_t *btl)
|
||||
static int sm_finalize(struct mca_btl_base_module_t *btl)
|
||||
{
|
||||
mca_btl_vader_component_t *component = &mca_btl_vader_component;
|
||||
mca_btl_vader_t *vader_btl = (mca_btl_vader_t *) btl;
|
||||
mca_btl_sm_component_t *component = &mca_btl_sm_component;
|
||||
mca_btl_sm_t *sm_btl = (mca_btl_sm_t *) btl;
|
||||
|
||||
if (!vader_btl->btl_inited) {
|
||||
if (!sm_btl->btl_inited) {
|
||||
return OPAL_SUCCESS;
|
||||
}
|
||||
|
||||
for (int i = 0 ; i < (int)(1 + MCA_BTL_VADER_NUM_LOCAL_PEERS) ; ++i) {
|
||||
fini_vader_endpoint (component->endpoints + i);
|
||||
for (int i = 0 ; i < (int)(1 + MCA_BTL_SM_NUM_LOCAL_PEERS) ; ++i) {
|
||||
fini_sm_endpoint (component->endpoints + i);
|
||||
}
|
||||
|
||||
free (component->endpoints);
|
||||
component->endpoints = NULL;
|
||||
|
||||
vader_btl->btl_inited = false;
|
||||
sm_btl->btl_inited = false;
|
||||
|
||||
free (component->fbox_in_endpoints);
|
||||
component->fbox_in_endpoints = NULL;
|
||||
|
||||
if (MCA_BTL_VADER_XPMEM != mca_btl_vader_component.single_copy_mechanism) {
|
||||
opal_shmem_unlink (&mca_btl_vader_component.seg_ds);
|
||||
opal_shmem_segment_detach (&mca_btl_vader_component.seg_ds);
|
||||
if (MCA_BTL_SM_XPMEM != mca_btl_sm_component.single_copy_mechanism) {
|
||||
opal_shmem_unlink (&mca_btl_sm_component.seg_ds);
|
||||
opal_shmem_segment_detach (&mca_btl_sm_component.seg_ds);
|
||||
}
|
||||
|
||||
#if OPAL_BTL_VADER_HAVE_XPMEM
|
||||
if (NULL != mca_btl_vader_component.vma_module) {
|
||||
OBJ_RELEASE(mca_btl_vader_component.vma_module);
|
||||
#if OPAL_BTL_SM_HAVE_XPMEM
|
||||
if (NULL != mca_btl_sm_component.vma_module) {
|
||||
OBJ_RELEASE(mca_btl_sm_component.vma_module);
|
||||
}
|
||||
#endif
|
||||
|
||||
@ -455,10 +456,10 @@ static int vader_finalize(struct mca_btl_base_module_t *btl)
|
||||
* @param cbfunc (IN) function to call on error
|
||||
* @return Status indicating if cleanup was successful
|
||||
*/
|
||||
static int vader_register_error_cb(struct mca_btl_base_module_t* btl,
|
||||
static int sm_register_error_cb(struct mca_btl_base_module_t* btl,
|
||||
mca_btl_base_module_error_cb_fn_t cbfunc)
|
||||
{
|
||||
((mca_btl_vader_t *)btl)->error_cb = cbfunc;
|
||||
((mca_btl_sm_t *)btl)->error_cb = cbfunc;
|
||||
return OPAL_SUCCESS;
|
||||
}
|
||||
|
||||
@ -468,19 +469,19 @@ static int vader_register_error_cb(struct mca_btl_base_module_t* btl,
|
||||
* @param btl (IN) BTL module
|
||||
* @param size (IN) Request segment size.
|
||||
*/
|
||||
mca_btl_base_descriptor_t *mca_btl_vader_alloc(struct mca_btl_base_module_t *btl,
|
||||
mca_btl_base_descriptor_t *mca_btl_sm_alloc(struct mca_btl_base_module_t *btl,
|
||||
struct mca_btl_base_endpoint_t *endpoint,
|
||||
uint8_t order, size_t size, uint32_t flags)
|
||||
{
|
||||
mca_btl_vader_frag_t *frag = NULL;
|
||||
mca_btl_sm_frag_t *frag = NULL;
|
||||
|
||||
if (size <= (size_t) mca_btl_vader_component.max_inline_send) {
|
||||
(void) MCA_BTL_VADER_FRAG_ALLOC_USER(frag, endpoint);
|
||||
} else if (size <= mca_btl_vader.super.btl_eager_limit) {
|
||||
(void) MCA_BTL_VADER_FRAG_ALLOC_EAGER(frag, endpoint);
|
||||
} else if (MCA_BTL_VADER_XPMEM != mca_btl_vader_component.single_copy_mechanism &&
|
||||
size <= mca_btl_vader.super.btl_max_send_size) {
|
||||
(void) MCA_BTL_VADER_FRAG_ALLOC_MAX(frag, endpoint);
|
||||
if (size <= (size_t) mca_btl_sm_component.max_inline_send) {
|
||||
(void) MCA_BTL_SM_FRAG_ALLOC_USER(frag, endpoint);
|
||||
} else if (size <= mca_btl_sm.super.btl_eager_limit) {
|
||||
(void) MCA_BTL_SM_FRAG_ALLOC_EAGER(frag, endpoint);
|
||||
} else if (MCA_BTL_SM_XPMEM != mca_btl_sm_component.single_copy_mechanism &&
|
||||
size <= mca_btl_sm.super.btl_max_send_size) {
|
||||
(void) MCA_BTL_SM_FRAG_ALLOC_MAX(frag, endpoint);
|
||||
}
|
||||
|
||||
if (OPAL_LIKELY(frag != NULL)) {
|
||||
@ -499,9 +500,9 @@ mca_btl_base_descriptor_t *mca_btl_vader_alloc(struct mca_btl_base_module_t *btl
|
||||
* @param btl (IN) BTL module
|
||||
* @param segment (IN) Allocated segment.
|
||||
*/
|
||||
int mca_btl_vader_free (struct mca_btl_base_module_t *btl, mca_btl_base_descriptor_t *des)
|
||||
int mca_btl_sm_free (struct mca_btl_base_module_t *btl, mca_btl_base_descriptor_t *des)
|
||||
{
|
||||
MCA_BTL_VADER_FRAG_RETURN((mca_btl_vader_frag_t *) des);
|
||||
MCA_BTL_SM_FRAG_RETURN((mca_btl_sm_frag_t *) des);
|
||||
|
||||
return OPAL_SUCCESS;
|
||||
}
|
||||
@ -511,14 +512,14 @@ int mca_btl_vader_free (struct mca_btl_base_module_t *btl, mca_btl_base_descript
|
||||
*
|
||||
* @param btl (IN) BTL module
|
||||
*/
|
||||
static struct mca_btl_base_descriptor_t *vader_prepare_src (struct mca_btl_base_module_t *btl,
|
||||
static struct mca_btl_base_descriptor_t *sm_prepare_src (struct mca_btl_base_module_t *btl,
|
||||
struct mca_btl_base_endpoint_t *endpoint,
|
||||
struct opal_convertor_t *convertor,
|
||||
uint8_t order, size_t reserve, size_t *size,
|
||||
uint32_t flags)
|
||||
{
|
||||
const size_t total_size = reserve + *size;
|
||||
mca_btl_vader_frag_t *frag;
|
||||
mca_btl_sm_frag_t *frag;
|
||||
void *data_ptr;
|
||||
int rc;
|
||||
|
||||
@ -531,11 +532,11 @@ static struct mca_btl_base_descriptor_t *vader_prepare_src (struct mca_btl_base_
|
||||
struct iovec iov;
|
||||
|
||||
/* non-contiguous data requires the convertor */
|
||||
if (MCA_BTL_VADER_XPMEM != mca_btl_vader_component.single_copy_mechanism &&
|
||||
total_size > mca_btl_vader.super.btl_eager_limit) {
|
||||
(void) MCA_BTL_VADER_FRAG_ALLOC_MAX(frag, endpoint);
|
||||
if (MCA_BTL_SM_XPMEM != mca_btl_sm_component.single_copy_mechanism &&
|
||||
total_size > mca_btl_sm.super.btl_eager_limit) {
|
||||
(void) MCA_BTL_SM_FRAG_ALLOC_MAX(frag, endpoint);
|
||||
} else
|
||||
(void) MCA_BTL_VADER_FRAG_ALLOC_EAGER(frag, endpoint);
|
||||
(void) MCA_BTL_SM_FRAG_ALLOC_EAGER(frag, endpoint);
|
||||
|
||||
if (OPAL_UNLIKELY(NULL == frag)) {
|
||||
return NULL;
|
||||
@ -548,31 +549,31 @@ static struct mca_btl_base_descriptor_t *vader_prepare_src (struct mca_btl_base_
|
||||
|
||||
rc = opal_convertor_pack (convertor, &iov, &iov_count, size);
|
||||
if (OPAL_UNLIKELY(rc < 0)) {
|
||||
MCA_BTL_VADER_FRAG_RETURN(frag);
|
||||
MCA_BTL_SM_FRAG_RETURN(frag);
|
||||
return NULL;
|
||||
}
|
||||
|
||||
frag->segments[0].seg_len = *size + reserve;
|
||||
} else {
|
||||
if (MCA_BTL_VADER_XPMEM != mca_btl_vader_component.single_copy_mechanism) {
|
||||
if (OPAL_LIKELY(total_size <= mca_btl_vader.super.btl_eager_limit)) {
|
||||
(void) MCA_BTL_VADER_FRAG_ALLOC_EAGER(frag, endpoint);
|
||||
if (MCA_BTL_SM_XPMEM != mca_btl_sm_component.single_copy_mechanism) {
|
||||
if (OPAL_LIKELY(total_size <= mca_btl_sm.super.btl_eager_limit)) {
|
||||
(void) MCA_BTL_SM_FRAG_ALLOC_EAGER(frag, endpoint);
|
||||
} else {
|
||||
(void) MCA_BTL_VADER_FRAG_ALLOC_MAX(frag, endpoint);
|
||||
(void) MCA_BTL_SM_FRAG_ALLOC_MAX(frag, endpoint);
|
||||
}
|
||||
} else
|
||||
(void) MCA_BTL_VADER_FRAG_ALLOC_USER(frag, endpoint);
|
||||
(void) MCA_BTL_SM_FRAG_ALLOC_USER(frag, endpoint);
|
||||
|
||||
if (OPAL_UNLIKELY(NULL == frag)) {
|
||||
return NULL;
|
||||
}
|
||||
|
||||
#if OPAL_BTL_VADER_HAVE_XPMEM
|
||||
#if OPAL_BTL_SM_HAVE_XPMEM
|
||||
/* use xpmem to send this segment if it is above the max inline send size */
|
||||
if (OPAL_UNLIKELY(MCA_BTL_VADER_XPMEM == mca_btl_vader_component.single_copy_mechanism &&
|
||||
total_size > (size_t) mca_btl_vader_component.max_inline_send)) {
|
||||
if (OPAL_UNLIKELY(MCA_BTL_SM_XPMEM == mca_btl_sm_component.single_copy_mechanism &&
|
||||
total_size > (size_t) mca_btl_sm_component.max_inline_send)) {
|
||||
/* single copy send */
|
||||
frag->hdr->flags = MCA_BTL_VADER_FLAG_SINGLE_COPY;
|
||||
frag->hdr->flags = MCA_BTL_SM_FLAG_SINGLE_COPY;
|
||||
|
||||
/* set up single copy io vector */
|
||||
frag->hdr->sc_iov.iov_base = data_ptr;
|
||||
@ -587,7 +588,7 @@ static struct mca_btl_base_descriptor_t *vader_prepare_src (struct mca_btl_base_
|
||||
/* NTH: the covertor adds some latency so we bypass it here */
|
||||
memcpy ((void *)((uintptr_t)frag->segments[0].seg_addr.pval + reserve), data_ptr, *size);
|
||||
frag->segments[0].seg_len = total_size;
|
||||
#if OPAL_BTL_VADER_HAVE_XPMEM
|
||||
#if OPAL_BTL_SM_HAVE_XPMEM
|
||||
}
|
||||
#endif
|
||||
}
|
||||
@ -603,12 +604,12 @@ static struct mca_btl_base_descriptor_t *vader_prepare_src (struct mca_btl_base_
|
||||
* @param state Checkpoint Stae
|
||||
* @return OPAL_SUCCESS or failure status
|
||||
*/
|
||||
static int vader_ft_event (int state)
|
||||
static int sm_ft_event (int state)
|
||||
{
|
||||
return OPAL_SUCCESS;
|
||||
}
|
||||
|
||||
static void mca_btl_vader_endpoint_constructor (mca_btl_vader_endpoint_t *ep)
|
||||
static void mca_btl_sm_endpoint_constructor (mca_btl_sm_endpoint_t *ep)
|
||||
{
|
||||
OBJ_CONSTRUCT(&ep->pending_frags, opal_list_t);
|
||||
OBJ_CONSTRUCT(&ep->pending_frags_lock, opal_mutex_t);
|
||||
@ -616,17 +617,17 @@ static void mca_btl_vader_endpoint_constructor (mca_btl_vader_endpoint_t *ep)
|
||||
ep->fbox_out.fbox = NULL;
|
||||
}
|
||||
|
||||
#if OPAL_BTL_VADER_HAVE_XPMEM
|
||||
#if OPAL_BTL_SM_HAVE_XPMEM
|
||||
#endif
|
||||
|
||||
static void mca_btl_vader_endpoint_destructor (mca_btl_vader_endpoint_t *ep)
|
||||
static void mca_btl_sm_endpoint_destructor (mca_btl_sm_endpoint_t *ep)
|
||||
{
|
||||
OBJ_DESTRUCT(&ep->pending_frags);
|
||||
OBJ_DESTRUCT(&ep->pending_frags_lock);
|
||||
|
||||
#if OPAL_BTL_VADER_HAVE_XPMEM
|
||||
if (MCA_BTL_VADER_XPMEM == mca_btl_vader_component.single_copy_mechanism) {
|
||||
mca_btl_vader_xpmem_cleanup_endpoint (ep);
|
||||
#if OPAL_BTL_SM_HAVE_XPMEM
|
||||
if (MCA_BTL_SM_XPMEM == mca_btl_sm_component.single_copy_mechanism) {
|
||||
mca_btl_sm_xpmem_cleanup_endpoint (ep);
|
||||
} else
|
||||
#endif
|
||||
if (ep->segment_data.other.seg_ds) {
|
||||
@ -643,7 +644,7 @@ static void mca_btl_vader_endpoint_destructor (mca_btl_vader_endpoint_t *ep)
|
||||
opal_shmem_segment_detach (&seg_ds);
|
||||
}
|
||||
if (ep->fbox_out.fbox) {
|
||||
opal_free_list_return (&mca_btl_vader_component.vader_fboxes, ep->fbox_out.fbox);
|
||||
opal_free_list_return (&mca_btl_sm_component.sm_fboxes, ep->fbox_out.fbox);
|
||||
}
|
||||
|
||||
ep->fbox_in.buffer = ep->fbox_out.buffer = NULL;
|
||||
@ -652,4 +653,4 @@ static void mca_btl_vader_endpoint_destructor (mca_btl_vader_endpoint_t *ep)
|
||||
ep->fifo = NULL;
|
||||
}
|
||||
|
||||
OBJ_CLASS_INSTANCE(mca_btl_vader_endpoint_t, opal_list_item_t, mca_btl_vader_endpoint_constructor, mca_btl_vader_endpoint_destructor);
|
||||
OBJ_CLASS_INSTANCE(mca_btl_sm_endpoint_t, opal_list_item_t, mca_btl_sm_endpoint_constructor, mca_btl_sm_endpoint_destructor);
|
||||
@ -14,12 +14,12 @@
|
||||
|
||||
#include "opal_config.h"
|
||||
|
||||
#include "btl_vader.h"
|
||||
#include "btl_vader_frag.h"
|
||||
#include "btl_vader_endpoint.h"
|
||||
#include "btl_vader_xpmem.h"
|
||||
#include "btl_sm.h"
|
||||
#include "btl_sm_frag.h"
|
||||
#include "btl_sm_endpoint.h"
|
||||
#include "btl_sm_xpmem.h"
|
||||
|
||||
#if OPAL_BTL_VADER_HAVE_CMA
|
||||
#if OPAL_BTL_SM_HAVE_CMA
|
||||
#include <sys/uio.h>
|
||||
|
||||
#if OPAL_CMA_NEED_SYSCALL_DEFS
|
||||
@ -35,8 +35,8 @@
|
||||
* @param endpoint (IN) BTL addressing information
|
||||
* @param descriptor (IN) Description of the data to be transferred
|
||||
*/
|
||||
#if OPAL_BTL_VADER_HAVE_XPMEM
|
||||
int mca_btl_vader_put_xpmem (mca_btl_base_module_t *btl, mca_btl_base_endpoint_t *endpoint, void *local_address,
|
||||
#if OPAL_BTL_SM_HAVE_XPMEM
|
||||
int mca_btl_sm_put_xpmem (mca_btl_base_module_t *btl, mca_btl_base_endpoint_t *endpoint, void *local_address,
|
||||
uint64_t remote_address, mca_btl_base_registration_handle_t *local_handle,
|
||||
mca_btl_base_registration_handle_t *remote_handle, size_t size, int flags,
|
||||
int order, mca_btl_base_rdma_completion_fn_t cbfunc, void *cbcontext, void *cbdata)
|
||||
@ -44,14 +44,14 @@ int mca_btl_vader_put_xpmem (mca_btl_base_module_t *btl, mca_btl_base_endpoint_t
|
||||
mca_rcache_base_registration_t *reg;
|
||||
void *rem_ptr;
|
||||
|
||||
reg = vader_get_registation (endpoint, (void *)(intptr_t) remote_address, size, 0, &rem_ptr);
|
||||
reg = sm_get_registation (endpoint, (void *)(intptr_t) remote_address, size, 0, &rem_ptr);
|
||||
if (OPAL_UNLIKELY(NULL == reg)) {
|
||||
return OPAL_ERROR;
|
||||
}
|
||||
|
||||
vader_memmove (rem_ptr, local_address, size);
|
||||
sm_memmove (rem_ptr, local_address, size);
|
||||
|
||||
vader_return_registration (reg, endpoint);
|
||||
sm_return_registration (reg, endpoint);
|
||||
|
||||
/* always call the callback function */
|
||||
cbfunc (btl, endpoint, local_address, local_handle, cbcontext, cbdata, OPAL_SUCCESS);
|
||||
@ -60,8 +60,8 @@ int mca_btl_vader_put_xpmem (mca_btl_base_module_t *btl, mca_btl_base_endpoint_t
|
||||
}
|
||||
#endif
|
||||
|
||||
#if OPAL_BTL_VADER_HAVE_CMA
|
||||
int mca_btl_vader_put_cma (mca_btl_base_module_t *btl, mca_btl_base_endpoint_t *endpoint, void *local_address,
|
||||
#if OPAL_BTL_SM_HAVE_CMA
|
||||
int mca_btl_sm_put_cma (mca_btl_base_module_t *btl, mca_btl_base_endpoint_t *endpoint, void *local_address,
|
||||
uint64_t remote_address, mca_btl_base_registration_handle_t *local_handle,
|
||||
mca_btl_base_registration_handle_t *remote_handle, size_t size, int flags,
|
||||
int order, mca_btl_base_rdma_completion_fn_t cbfunc, void *cbcontext, void *cbdata)
|
||||
@ -70,7 +70,7 @@ int mca_btl_vader_put_cma (mca_btl_base_module_t *btl, mca_btl_base_endpoint_t *
|
||||
struct iovec dst_iov = {.iov_base = (void *)(intptr_t) remote_address, .iov_len = size};
|
||||
ssize_t ret;
|
||||
|
||||
/* This should not be needed, see the rationale in mca_btl_vader_get_cma() */
|
||||
/* This should not be needed, see the rationale in mca_btl_sm_get_cma() */
|
||||
do {
|
||||
ret = process_vm_writev (endpoint->segment_data.other.seg_ds->seg_cpid, &src_iov, 1, &dst_iov, 1, 0);
|
||||
if (0 > ret) {
|
||||
@ -90,8 +90,8 @@ int mca_btl_vader_put_cma (mca_btl_base_module_t *btl, mca_btl_base_endpoint_t *
|
||||
}
|
||||
#endif
|
||||
|
||||
#if OPAL_BTL_VADER_HAVE_KNEM
|
||||
int mca_btl_vader_put_knem (mca_btl_base_module_t *btl, mca_btl_base_endpoint_t *endpoint, void *local_address,
|
||||
#if OPAL_BTL_SM_HAVE_KNEM
|
||||
int mca_btl_sm_put_knem (mca_btl_base_module_t *btl, mca_btl_base_endpoint_t *endpoint, void *local_address,
|
||||
uint64_t remote_address, mca_btl_base_registration_handle_t *local_handle,
|
||||
mca_btl_base_registration_handle_t *remote_handle, size_t size, int flags,
|
||||
int order, mca_btl_base_rdma_completion_fn_t cbfunc, void *cbcontext, void *cbdata)
|
||||
@ -113,15 +113,15 @@ int mca_btl_vader_put_knem (mca_btl_base_module_t *btl, mca_btl_base_endpoint_t
|
||||
/* Use the DMA flag if knem supports it *and* the segment length
|
||||
* is greater than the cutoff. Not that if DMA is not supported
|
||||
* or the user specified 0 for knem_dma_min the knem_dma_min was
|
||||
* set to UINT_MAX in mca_btl_vader_knem_init. */
|
||||
if (mca_btl_vader_component.knem_dma_min <= size) {
|
||||
* set to UINT_MAX in mca_btl_sm_knem_init. */
|
||||
if (mca_btl_sm_component.knem_dma_min <= size) {
|
||||
icopy.flags = KNEM_FLAG_DMA;
|
||||
}
|
||||
/* synchronous flags only, no need to specify icopy.async_status_index */
|
||||
|
||||
/* When the ioctl returns, the transfer is done and we can invoke
|
||||
the btl callback and return the frag */
|
||||
if (OPAL_UNLIKELY(0 != ioctl (mca_btl_vader.knem_fd, KNEM_CMD_INLINE_COPY, &icopy))) {
|
||||
if (OPAL_UNLIKELY(0 != ioctl (mca_btl_sm.knem_fd, KNEM_CMD_INLINE_COPY, &icopy))) {
|
||||
return OPAL_ERROR;
|
||||
}
|
||||
|
||||
@ -139,15 +139,15 @@ int mca_btl_vader_put_knem (mca_btl_base_module_t *btl, mca_btl_base_endpoint_t
|
||||
/**
|
||||
* @brief Provides an emulated put path which uses copy-in copy-out with shared memory buffers
|
||||
*/
|
||||
int mca_btl_vader_put_sc_emu (mca_btl_base_module_t *btl, mca_btl_base_endpoint_t *endpoint, void *local_address,
|
||||
int mca_btl_sm_put_sc_emu (mca_btl_base_module_t *btl, mca_btl_base_endpoint_t *endpoint, void *local_address,
|
||||
uint64_t remote_address, mca_btl_base_registration_handle_t *local_handle,
|
||||
mca_btl_base_registration_handle_t *remote_handle, size_t size, int flags,
|
||||
int order, mca_btl_base_rdma_completion_fn_t cbfunc, void *cbcontext, void *cbdata)
|
||||
{
|
||||
if (size > mca_btl_vader.super.btl_put_limit) {
|
||||
if (size > mca_btl_sm.super.btl_put_limit) {
|
||||
return OPAL_ERR_NOT_AVAILABLE;
|
||||
}
|
||||
|
||||
return mca_btl_vader_rdma_frag_start (btl, endpoint, MCA_BTL_VADER_OP_PUT, 0, 0, 0, order, flags, size,
|
||||
local_address, remote_address, cbfunc, cbcontext, cbdata);
|
||||
return mca_btl_sm_rdma_frag_start (btl, endpoint, MCA_BTL_SM_OP_PUT, 0, 0, 0, order, flags, size,
|
||||
local_address, remote_address, cbfunc, cbcontext, cbdata);
|
||||
}
|
||||
@ -9,11 +9,11 @@
|
||||
* $HEADER$
|
||||
*/
|
||||
|
||||
#include "btl_vader.h"
|
||||
#include "btl_vader_frag.h"
|
||||
#include "btl_sm.h"
|
||||
#include "btl_sm_frag.h"
|
||||
|
||||
#if OPAL_HAVE_ATOMIC_MATH_64
|
||||
static void mca_btl_vader_sc_emu_atomic_64 (int64_t *operand, opal_atomic_int64_t *addr, mca_btl_base_atomic_op_t op)
|
||||
static void mca_btl_sm_sc_emu_atomic_64 (int64_t *operand, opal_atomic_int64_t *addr, mca_btl_base_atomic_op_t op)
|
||||
{
|
||||
int64_t result = 0;
|
||||
|
||||
@ -52,7 +52,7 @@ static void mca_btl_vader_sc_emu_atomic_64 (int64_t *operand, opal_atomic_int64_
|
||||
#endif
|
||||
|
||||
#if OPAL_HAVE_ATOMIC_MATH_32
|
||||
static void mca_btl_vader_sc_emu_atomic_32 (int32_t *operand, opal_atomic_int32_t *addr, mca_btl_base_atomic_op_t op)
|
||||
static void mca_btl_sm_sc_emu_atomic_32 (int32_t *operand, opal_atomic_int32_t *addr, mca_btl_base_atomic_op_t op)
|
||||
{
|
||||
int32_t result = 0;
|
||||
|
||||
@ -90,27 +90,27 @@ static void mca_btl_vader_sc_emu_atomic_32 (int32_t *operand, opal_atomic_int32_
|
||||
}
|
||||
#endif
|
||||
|
||||
static void mca_btl_vader_sc_emu_rdma (mca_btl_base_module_t *btl, mca_btl_base_tag_t tag, mca_btl_base_descriptor_t *desc, void *ctx)
|
||||
static void mca_btl_sm_sc_emu_rdma (mca_btl_base_module_t *btl, mca_btl_base_tag_t tag, mca_btl_base_descriptor_t *desc, void *ctx)
|
||||
{
|
||||
mca_btl_vader_sc_emu_hdr_t *hdr = (mca_btl_vader_sc_emu_hdr_t *) desc->des_segments[0].seg_addr.pval;
|
||||
mca_btl_sm_sc_emu_hdr_t *hdr = (mca_btl_sm_sc_emu_hdr_t *) desc->des_segments[0].seg_addr.pval;
|
||||
size_t size = desc->des_segments[0].seg_len - sizeof (*hdr);
|
||||
void *data = (void *)(hdr + 1);
|
||||
|
||||
switch (hdr->type) {
|
||||
case MCA_BTL_VADER_OP_PUT:
|
||||
case MCA_BTL_SM_OP_PUT:
|
||||
memcpy ((void *) hdr->addr, data, size);
|
||||
break;
|
||||
case MCA_BTL_VADER_OP_GET:
|
||||
case MCA_BTL_SM_OP_GET:
|
||||
memcpy (data, (void *) hdr->addr, size);
|
||||
break;
|
||||
#if OPAL_HAVE_ATOMIC_MATH_64
|
||||
case MCA_BTL_VADER_OP_ATOMIC:
|
||||
case MCA_BTL_SM_OP_ATOMIC:
|
||||
if (!(hdr->flags & MCA_BTL_ATOMIC_FLAG_32BIT)) {
|
||||
mca_btl_vader_sc_emu_atomic_64 (hdr->operand, (void *) hdr->addr, hdr->op);
|
||||
mca_btl_sm_sc_emu_atomic_64 (hdr->operand, (void *) hdr->addr, hdr->op);
|
||||
#if OPAL_HAVE_ATOMIC_MATH_32
|
||||
} else {
|
||||
int32_t tmp = (int32_t) hdr->operand[0];
|
||||
mca_btl_vader_sc_emu_atomic_32 (&tmp, (void *) hdr->addr, hdr->op);
|
||||
mca_btl_sm_sc_emu_atomic_32 (&tmp, (void *) hdr->addr, hdr->op);
|
||||
hdr->operand[0] = tmp;
|
||||
#else
|
||||
} else {
|
||||
@ -121,7 +121,7 @@ static void mca_btl_vader_sc_emu_rdma (mca_btl_base_module_t *btl, mca_btl_base_
|
||||
break;
|
||||
#endif /* OPAL_HAVE_ATOMIC_MATH_64 */
|
||||
#if OPAL_HAVE_ATOMIC_MATH_64
|
||||
case MCA_BTL_VADER_OP_CSWAP:
|
||||
case MCA_BTL_SM_OP_CSWAP:
|
||||
if (!(hdr->flags & MCA_BTL_ATOMIC_FLAG_32BIT)) {
|
||||
opal_atomic_compare_exchange_strong_64 ((opal_atomic_int64_t *) hdr->addr, &hdr->operand[0], hdr->operand[1]);
|
||||
#if OPAL_HAVE_ATOMIC_MATH_32
|
||||
@ -139,8 +139,8 @@ static void mca_btl_vader_sc_emu_rdma (mca_btl_base_module_t *btl, mca_btl_base_
|
||||
}
|
||||
}
|
||||
|
||||
void mca_btl_vader_sc_emu_init (void)
|
||||
void mca_btl_sm_sc_emu_init (void)
|
||||
{
|
||||
mca_btl_base_active_message_trigger[MCA_BTL_TAG_VADER].cbfunc = mca_btl_vader_sc_emu_rdma;
|
||||
mca_btl_base_active_message_trigger[MCA_BTL_TAG_VADER].cbdata = NULL;
|
||||
mca_btl_base_active_message_trigger[MCA_BTL_TAG_SM].cbfunc = mca_btl_sm_sc_emu_rdma;
|
||||
mca_btl_base_active_message_trigger[MCA_BTL_TAG_SM].cbdata = NULL;
|
||||
}
|
||||
@ -23,10 +23,10 @@
|
||||
|
||||
#include "opal_config.h"
|
||||
|
||||
#include "btl_vader.h"
|
||||
#include "btl_vader_frag.h"
|
||||
#include "btl_vader_fifo.h"
|
||||
#include "btl_vader_fbox.h"
|
||||
#include "btl_sm.h"
|
||||
#include "btl_sm_frag.h"
|
||||
#include "btl_sm_fifo.h"
|
||||
#include "btl_sm_fbox.h"
|
||||
|
||||
/**
|
||||
* Initiate a send to the peer.
|
||||
@ -34,12 +34,12 @@
|
||||
* @param btl (IN) BTL module
|
||||
* @param peer (IN) BTL peer addressing
|
||||
*/
|
||||
int mca_btl_vader_send (struct mca_btl_base_module_t *btl,
|
||||
int mca_btl_sm_send (struct mca_btl_base_module_t *btl,
|
||||
struct mca_btl_base_endpoint_t *endpoint,
|
||||
struct mca_btl_base_descriptor_t *descriptor,
|
||||
mca_btl_base_tag_t tag)
|
||||
{
|
||||
mca_btl_vader_frag_t *frag = (mca_btl_vader_frag_t *) descriptor;
|
||||
mca_btl_sm_frag_t *frag = (mca_btl_sm_frag_t *) descriptor;
|
||||
const size_t total_size = frag->segments[0].seg_len;
|
||||
|
||||
/* in order to work around a long standing ob1 bug (see #3845) we have to always
|
||||
@ -52,14 +52,14 @@ int mca_btl_vader_send (struct mca_btl_base_module_t *btl,
|
||||
frag->hdr->tag = tag;
|
||||
|
||||
/* post the relative address of the descriptor into the peer's fifo */
|
||||
if (opal_list_get_size (&endpoint->pending_frags) || !vader_fifo_write_ep (frag->hdr, endpoint)) {
|
||||
if (opal_list_get_size (&endpoint->pending_frags) || !sm_fifo_write_ep (frag->hdr, endpoint)) {
|
||||
frag->base.des_flags |= MCA_BTL_DES_SEND_ALWAYS_CALLBACK;
|
||||
OPAL_THREAD_LOCK(&endpoint->pending_frags_lock);
|
||||
opal_list_append (&endpoint->pending_frags, (opal_list_item_t *) frag);
|
||||
if (!endpoint->waiting) {
|
||||
OPAL_THREAD_LOCK(&mca_btl_vader_component.lock);
|
||||
opal_list_append (&mca_btl_vader_component.pending_endpoints, &endpoint->super);
|
||||
OPAL_THREAD_UNLOCK(&mca_btl_vader_component.lock);
|
||||
OPAL_THREAD_LOCK(&mca_btl_sm_component.lock);
|
||||
opal_list_append (&mca_btl_sm_component.pending_endpoints, &endpoint->super);
|
||||
OPAL_THREAD_UNLOCK(&mca_btl_sm_component.lock);
|
||||
endpoint->waiting = true;
|
||||
}
|
||||
OPAL_THREAD_UNLOCK(&endpoint->pending_frags_lock);
|
||||
@ -69,7 +69,7 @@ int mca_btl_vader_send (struct mca_btl_base_module_t *btl,
|
||||
return OPAL_SUCCESS;
|
||||
|
||||
#if 0
|
||||
if ((frag->hdr->flags & MCA_BTL_VADER_FLAG_SINGLE_COPY) ||
|
||||
if ((frag->hdr->flags & MCA_BTL_SM_FLAG_SINGLE_COPY) ||
|
||||
!(frag->base.des_flags & MCA_BTL_DES_FLAGS_BTL_OWNERSHIP)) {
|
||||
frag->base.des_flags |= MCA_BTL_DES_SEND_ALWAYS_CALLBACK;
|
||||
|
||||
@ -25,11 +25,11 @@
|
||||
|
||||
#include "opal_config.h"
|
||||
|
||||
#include "btl_vader.h"
|
||||
#include "btl_vader_frag.h"
|
||||
#include "btl_vader_fifo.h"
|
||||
#include "btl_sm.h"
|
||||
#include "btl_sm_frag.h"
|
||||
#include "btl_sm_fifo.h"
|
||||
|
||||
#include "btl_vader_fbox.h"
|
||||
#include "btl_sm_fbox.h"
|
||||
|
||||
/**
|
||||
* Initiate an inline send to the peer.
|
||||
@ -37,7 +37,7 @@
|
||||
* @param btl (IN) BTL module
|
||||
* @param peer (IN) BTL peer addressing
|
||||
*/
|
||||
int mca_btl_vader_sendi (struct mca_btl_base_module_t *btl,
|
||||
int mca_btl_sm_sendi (struct mca_btl_base_module_t *btl,
|
||||
struct mca_btl_base_endpoint_t *endpoint,
|
||||
struct opal_convertor_t *convertor,
|
||||
void *header, size_t header_size,
|
||||
@ -45,7 +45,7 @@ int mca_btl_vader_sendi (struct mca_btl_base_module_t *btl,
|
||||
uint32_t flags, mca_btl_base_tag_t tag,
|
||||
mca_btl_base_descriptor_t **descriptor)
|
||||
{
|
||||
mca_btl_vader_frag_t *frag;
|
||||
mca_btl_sm_frag_t *frag;
|
||||
void *data_ptr = NULL;
|
||||
size_t length;
|
||||
|
||||
@ -63,14 +63,14 @@ int mca_btl_vader_sendi (struct mca_btl_base_module_t *btl,
|
||||
}
|
||||
|
||||
if (!(payload_size && opal_convertor_need_buffers (convertor)) &&
|
||||
mca_btl_vader_fbox_sendi (endpoint, tag, header, header_size, data_ptr, payload_size)) {
|
||||
mca_btl_sm_fbox_sendi (endpoint, tag, header, header_size, data_ptr, payload_size)) {
|
||||
return OPAL_SUCCESS;
|
||||
}
|
||||
|
||||
length = header_size + payload_size;
|
||||
|
||||
/* allocate a fragment, giving up if we can't get one */
|
||||
frag = (mca_btl_vader_frag_t *) mca_btl_vader_alloc (btl, endpoint, order, length,
|
||||
frag = (mca_btl_sm_frag_t *) mca_btl_sm_alloc (btl, endpoint, order, length,
|
||||
flags | MCA_BTL_DES_FLAGS_BTL_OWNERSHIP);
|
||||
if (OPAL_UNLIKELY(NULL == frag)) {
|
||||
if (descriptor) {
|
||||
@ -104,11 +104,11 @@ int mca_btl_vader_sendi (struct mca_btl_base_module_t *btl,
|
||||
}
|
||||
|
||||
/* write the fragment pointer to peer's the FIFO. the progress function will return the fragment */
|
||||
if (!vader_fifo_write_ep (frag->hdr, endpoint)) {
|
||||
if (!sm_fifo_write_ep (frag->hdr, endpoint)) {
|
||||
if (descriptor) {
|
||||
*descriptor = &frag->base;
|
||||
} else {
|
||||
mca_btl_vader_free (btl, &frag->base);
|
||||
mca_btl_sm_free (btl, &frag->base);
|
||||
}
|
||||
return OPAL_ERR_OUT_OF_RESOURCE;
|
||||
}
|
||||
@ -13,17 +13,17 @@
|
||||
* $HEADER$
|
||||
*/
|
||||
|
||||
#include "btl_vader.h"
|
||||
#include "btl_sm.h"
|
||||
|
||||
#include "opal/include/opal/align.h"
|
||||
#include "opal/mca/memchecker/base/base.h"
|
||||
|
||||
#if OPAL_BTL_VADER_HAVE_XPMEM
|
||||
#if OPAL_BTL_SM_HAVE_XPMEM
|
||||
|
||||
int mca_btl_vader_xpmem_init (void)
|
||||
int mca_btl_sm_xpmem_init (void)
|
||||
{
|
||||
/* Any attachment that goes past the Linux TASK_SIZE will always fail. To prevent this we need to
|
||||
* determine the value of TASK_SIZE. On x86_64 the value was hard-coded in vader to be
|
||||
* determine the value of TASK_SIZE. On x86_64 the value was hard-coded in sm to be
|
||||
* 0x7ffffffffffful but this approach does not work with AARCH64 (and possibly other architectures).
|
||||
* Since there is really no way to directly determine the value we can (in all cases?) look through
|
||||
* the mapping for this process to determine what the largest address is. This should be the top
|
||||
@ -57,48 +57,48 @@ int mca_btl_vader_xpmem_init (void)
|
||||
}
|
||||
|
||||
/* save the calcuated maximum */
|
||||
mca_btl_vader_component.my_address_max = address_max - 1;
|
||||
mca_btl_sm_component.my_address_max = address_max - 1;
|
||||
|
||||
/* it is safe to use XPMEM_MAXADDR_SIZE here (which is always (size_t)-1 even though
|
||||
* it is not safe for attach */
|
||||
mca_btl_vader_component.my_seg_id = xpmem_make (0, XPMEM_MAXADDR_SIZE, XPMEM_PERMIT_MODE,
|
||||
mca_btl_sm_component.my_seg_id = xpmem_make (0, XPMEM_MAXADDR_SIZE, XPMEM_PERMIT_MODE,
|
||||
(void *)0666);
|
||||
if (-1 == mca_btl_vader_component.my_seg_id) {
|
||||
if (-1 == mca_btl_sm_component.my_seg_id) {
|
||||
return OPAL_ERR_NOT_AVAILABLE;
|
||||
}
|
||||
|
||||
mca_btl_vader.super.btl_get = mca_btl_vader_get_xpmem;
|
||||
mca_btl_vader.super.btl_put = mca_btl_vader_put_xpmem;
|
||||
mca_btl_sm.super.btl_get = mca_btl_sm_get_xpmem;
|
||||
mca_btl_sm.super.btl_put = mca_btl_sm_put_xpmem;
|
||||
|
||||
return OPAL_SUCCESS;
|
||||
}
|
||||
|
||||
struct vader_check_reg_ctx_t {
|
||||
struct sm_check_reg_ctx_t {
|
||||
mca_btl_base_endpoint_t *ep;
|
||||
mca_rcache_base_registration_t **reg;
|
||||
uintptr_t base;
|
||||
uintptr_t bound;
|
||||
};
|
||||
typedef struct vader_check_reg_ctx_t vader_check_reg_ctx_t;
|
||||
typedef struct sm_check_reg_ctx_t sm_check_reg_ctx_t;
|
||||
|
||||
static int vader_check_reg (mca_rcache_base_registration_t *reg, void *ctx)
|
||||
static int sm_check_reg (mca_rcache_base_registration_t *reg, void *ctx)
|
||||
{
|
||||
vader_check_reg_ctx_t *vader_ctx = (vader_check_reg_ctx_t *) ctx;
|
||||
sm_check_reg_ctx_t *sm_ctx = (sm_check_reg_ctx_t *) ctx;
|
||||
|
||||
if ((intptr_t) reg->alloc_base != vader_ctx->ep->peer_smp_rank) {
|
||||
if ((intptr_t) reg->alloc_base != sm_ctx->ep->peer_smp_rank) {
|
||||
/* ignore this registration */
|
||||
return OPAL_SUCCESS;
|
||||
}
|
||||
|
||||
vader_ctx->reg[0] = reg;
|
||||
sm_ctx->reg[0] = reg;
|
||||
|
||||
if (vader_ctx->bound <= (uintptr_t) reg->bound && vader_ctx->base >= (uintptr_t) reg->base) {
|
||||
if (sm_ctx->bound <= (uintptr_t) reg->bound && sm_ctx->base >= (uintptr_t) reg->base) {
|
||||
if (0 == opal_atomic_fetch_add_32 (®->ref_count, 1)) {
|
||||
/* registration is being deleted by a thread in vader_return_registration. the
|
||||
/* registration is being deleted by a thread in sm_return_registration. the
|
||||
* VMA tree implementation will block in mca_rcache_delete until we finish
|
||||
* iterating over the VMA tree so it is safe to just ignore this registration
|
||||
* and continue. */
|
||||
vader_ctx->reg[0] = NULL;
|
||||
sm_ctx->reg[0] = NULL;
|
||||
return OPAL_SUCCESS;
|
||||
}
|
||||
return 1;
|
||||
@ -106,7 +106,7 @@ static int vader_check_reg (mca_rcache_base_registration_t *reg, void *ctx)
|
||||
|
||||
if (MCA_RCACHE_FLAGS_INVALID & opal_atomic_fetch_or_32(®->flags, MCA_RCACHE_FLAGS_INVALID)) {
|
||||
/* another thread has already marked this registration as invalid. ignore and continue. */
|
||||
vader_ctx->reg[0] = NULL;
|
||||
sm_ctx->reg[0] = NULL;
|
||||
return OPAL_SUCCESS;
|
||||
}
|
||||
|
||||
@ -116,9 +116,9 @@ static int vader_check_reg (mca_rcache_base_registration_t *reg, void *ctx)
|
||||
return 2;
|
||||
}
|
||||
|
||||
void vader_return_registration (mca_rcache_base_registration_t *reg, struct mca_btl_base_endpoint_t *ep)
|
||||
void sm_return_registration (mca_rcache_base_registration_t *reg, struct mca_btl_base_endpoint_t *ep)
|
||||
{
|
||||
mca_rcache_base_vma_module_t *vma_module = mca_btl_vader_component.vma_module;
|
||||
mca_rcache_base_vma_module_t *vma_module = mca_btl_sm_component.vma_module;
|
||||
int32_t ref_count;
|
||||
|
||||
ref_count = opal_atomic_add_fetch_32 (®->ref_count, -1);
|
||||
@ -137,13 +137,13 @@ void vader_return_registration (mca_rcache_base_registration_t *reg, struct mca_
|
||||
|
||||
/* look up the remote pointer in the peer rcache and attach if
|
||||
* necessary */
|
||||
mca_rcache_base_registration_t *vader_get_registation (struct mca_btl_base_endpoint_t *ep, void *rem_ptr,
|
||||
mca_rcache_base_registration_t *sm_get_registation (struct mca_btl_base_endpoint_t *ep, void *rem_ptr,
|
||||
size_t size, int flags, void **local_ptr)
|
||||
{
|
||||
mca_rcache_base_vma_module_t *vma_module = mca_btl_vader_component.vma_module;
|
||||
uint64_t attach_align = 1 << mca_btl_vader_component.log_attach_align;
|
||||
mca_rcache_base_vma_module_t *vma_module = mca_btl_sm_component.vma_module;
|
||||
uint64_t attach_align = 1 << mca_btl_sm_component.log_attach_align;
|
||||
mca_rcache_base_registration_t *reg = NULL;
|
||||
vader_check_reg_ctx_t check_ctx = {.ep = ep, .reg = ®};
|
||||
sm_check_reg_ctx_t check_ctx = {.ep = ep, .reg = ®};
|
||||
xpmem_addr_t xpmem_addr;
|
||||
uintptr_t base, bound;
|
||||
int rc;
|
||||
@ -158,11 +158,11 @@ mca_rcache_base_registration_t *vader_get_registation (struct mca_btl_base_endpo
|
||||
check_ctx.bound = bound;
|
||||
|
||||
/* several segments may match the base pointer */
|
||||
rc = mca_rcache_base_vma_iterate (vma_module, (void *) base, bound - base, true, vader_check_reg, &check_ctx);
|
||||
rc = mca_rcache_base_vma_iterate (vma_module, (void *) base, bound - base, true, sm_check_reg, &check_ctx);
|
||||
if (2 == rc) {
|
||||
bound = bound < (uintptr_t) reg->bound ? (uintptr_t) reg->bound : bound;
|
||||
base = base > (uintptr_t) reg->base ? (uintptr_t) reg->base : base;
|
||||
vader_return_registration(reg, ep);
|
||||
sm_return_registration(reg, ep);
|
||||
reg = NULL;
|
||||
}
|
||||
|
||||
@ -204,14 +204,14 @@ mca_rcache_base_registration_t *vader_get_registation (struct mca_btl_base_endpo
|
||||
return reg;
|
||||
}
|
||||
|
||||
struct vader_cleanup_reg_ctx {
|
||||
mca_btl_vader_endpoint_t *ep;
|
||||
struct sm_cleanup_reg_ctx {
|
||||
mca_btl_sm_endpoint_t *ep;
|
||||
opal_list_t *registrations;
|
||||
};
|
||||
|
||||
static int mca_btl_vader_endpoint_xpmem_rcache_cleanup (mca_rcache_base_registration_t *reg, void *ctx)
|
||||
static int mca_btl_sm_endpoint_xpmem_rcache_cleanup (mca_rcache_base_registration_t *reg, void *ctx)
|
||||
{
|
||||
struct vader_cleanup_reg_ctx *cleanup_ctx = (struct vader_cleanup_reg_ctx *) ctx;
|
||||
struct sm_cleanup_reg_ctx *cleanup_ctx = (struct sm_cleanup_reg_ctx *) ctx;
|
||||
if ((intptr_t) reg->alloc_base == cleanup_ctx->ep->peer_smp_rank) {
|
||||
opal_list_append(cleanup_ctx->registrations, ®->super.super);
|
||||
}
|
||||
@ -219,21 +219,21 @@ static int mca_btl_vader_endpoint_xpmem_rcache_cleanup (mca_rcache_base_registra
|
||||
return OPAL_SUCCESS;
|
||||
}
|
||||
|
||||
void mca_btl_vader_xpmem_cleanup_endpoint (struct mca_btl_base_endpoint_t *ep)
|
||||
void mca_btl_sm_xpmem_cleanup_endpoint (struct mca_btl_base_endpoint_t *ep)
|
||||
{
|
||||
mca_rcache_base_registration_t *reg;
|
||||
opal_list_t registrations;
|
||||
struct vader_cleanup_reg_ctx cleanup_ctx = {.ep = ep, .registrations = ®istrations};
|
||||
struct sm_cleanup_reg_ctx cleanup_ctx = {.ep = ep, .registrations = ®istrations};
|
||||
|
||||
OBJ_CONSTRUCT(®istrations, opal_list_t);
|
||||
|
||||
/* clean out the registration cache */
|
||||
(void) mca_rcache_base_vma_iterate (mca_btl_vader_component.vma_module,
|
||||
(void) mca_rcache_base_vma_iterate (mca_btl_sm_component.vma_module,
|
||||
NULL, (size_t) -1, true,
|
||||
mca_btl_vader_endpoint_xpmem_rcache_cleanup,
|
||||
mca_btl_sm_endpoint_xpmem_rcache_cleanup,
|
||||
(void *) &cleanup_ctx);
|
||||
while (NULL != (reg = (mca_rcache_base_registration_t *) opal_list_remove_first(®istrations))) {
|
||||
vader_return_registration (reg, ep);
|
||||
sm_return_registration (reg, ep);
|
||||
}
|
||||
OBJ_DESTRUCT(®istrations);
|
||||
|
||||
@ -243,4 +243,4 @@ void mca_btl_vader_xpmem_cleanup_endpoint (struct mca_btl_base_endpoint_t *ep)
|
||||
}
|
||||
}
|
||||
|
||||
#endif /* OPAL_BTL_VADER_HAVE_XPMEM */
|
||||
#endif /* OPAL_BTL_SM_HAVE_XPMEM */
|
||||
@ -11,10 +11,10 @@
|
||||
* $HEADER$
|
||||
*/
|
||||
|
||||
#if !defined(MCA_BTL_VADER_XPMEM_H)
|
||||
#define MCA_BTL_VADER_XPMEM_H
|
||||
#if !defined(MCA_BTL_SM_XPMEM_H)
|
||||
#define MCA_BTL_SM_XPMEM_H
|
||||
|
||||
#if OPAL_BTL_VADER_HAVE_XPMEM
|
||||
#if OPAL_BTL_SM_HAVE_XPMEM
|
||||
|
||||
#if defined(HAVE_XPMEM_H)
|
||||
#include <xpmem.h>
|
||||
@ -35,17 +35,17 @@
|
||||
|
||||
struct mca_btl_base_endpoint_t;
|
||||
|
||||
int mca_btl_vader_xpmem_init (void);
|
||||
int mca_btl_sm_xpmem_init (void);
|
||||
|
||||
mca_rcache_base_registration_t *vader_get_registation (struct mca_btl_base_endpoint_t *endpoint, void *rem_ptr,
|
||||
mca_rcache_base_registration_t *sm_get_registation (struct mca_btl_base_endpoint_t *endpoint, void *rem_ptr,
|
||||
size_t size, int flags, void **local_ptr);
|
||||
|
||||
void vader_return_registration (mca_rcache_base_registration_t *reg, struct mca_btl_base_endpoint_t *endpoint);
|
||||
void mca_btl_vader_xpmem_cleanup_endpoint (struct mca_btl_base_endpoint_t *ep);
|
||||
void sm_return_registration (mca_rcache_base_registration_t *reg, struct mca_btl_base_endpoint_t *endpoint);
|
||||
void mca_btl_sm_xpmem_cleanup_endpoint (struct mca_btl_base_endpoint_t *ep);
|
||||
|
||||
#else
|
||||
|
||||
static inline mca_rcache_base_registration_t *vader_get_registation (struct mca_btl_base_endpoint_t *endpoint, void *rem_ptr,
|
||||
static inline mca_rcache_base_registration_t *sm_get_registation (struct mca_btl_base_endpoint_t *endpoint, void *rem_ptr,
|
||||
size_t size, int flags, void **local_ptr)
|
||||
{
|
||||
(void) endpoint;
|
||||
@ -56,12 +56,12 @@ static inline mca_rcache_base_registration_t *vader_get_registation (struct mca_
|
||||
return NULL;
|
||||
}
|
||||
|
||||
static inline void vader_return_registration (mca_rcache_base_registration_t *reg, struct mca_btl_base_endpoint_t *endpoint)
|
||||
static inline void sm_return_registration (mca_rcache_base_registration_t *reg, struct mca_btl_base_endpoint_t *endpoint)
|
||||
{
|
||||
(void) reg;
|
||||
(void) endpoint;
|
||||
}
|
||||
|
||||
#endif /* OPAL_BTL_VADER_HAVE_XPMEM */
|
||||
#endif /* OPAL_BTL_SM_HAVE_XPMEM */
|
||||
|
||||
#endif
|
||||
53
opal/mca/btl/sm/configure.m4
Обычный файл
53
opal/mca/btl/sm/configure.m4
Обычный файл
@ -0,0 +1,53 @@
|
||||
# -*- shell-script -*-
|
||||
#
|
||||
# Copyright (c) 2009 The University of Tennessee and The University
|
||||
# of Tennessee Research Foundation. All rights
|
||||
# reserved.
|
||||
# Copyright (c) 2009-2016 Cisco Systems, Inc. All rights reserved.
|
||||
# Copyright (c) 2011-2014 Los Alamos National Security, LLC. All rights
|
||||
# reserved.
|
||||
# Copyright (c) 2015 Research Organization for Information Science
|
||||
# and Technology (RIST). All rights reserved.
|
||||
# $COPYRIGHT$
|
||||
#
|
||||
# Additional copyrights may follow
|
||||
#
|
||||
# $HEADER$
|
||||
#
|
||||
|
||||
# MCA_btl_sm_CONFIG([action-if-can-compile],
|
||||
# [action-if-cant-compile])
|
||||
# ------------------------------------------------
|
||||
AC_DEFUN([MCA_opal_btl_sm_CONFIG],[
|
||||
AC_CONFIG_FILES([opal/mca/btl/sm/Makefile])
|
||||
|
||||
OPAL_VAR_SCOPE_PUSH([btl_sm_xpmem_happy btl_sm_cma_happy btl_sm_knem_happy])
|
||||
|
||||
# Check for single-copy APIs
|
||||
|
||||
OPAL_CHECK_XPMEM([btl_sm], [btl_sm_xpmem_happy=1], [btl_sm_xpmem_happy=0])
|
||||
OPAL_CHECK_KNEM([btl_sm], [btl_sm_knem_happy=1],[btl_sm_knem_happy=0])
|
||||
OPAL_CHECK_CMA([btl_sm], [AC_CHECK_HEADER([sys/prctl.h]) btl_sm_cma_happy=1], [btl_sm_cma_happy=0])
|
||||
|
||||
AC_DEFINE_UNQUOTED([OPAL_BTL_SM_HAVE_XPMEM], [$btl_sm_xpmem_happy],
|
||||
[If XPMEM support can be enabled within sm])
|
||||
|
||||
AC_DEFINE_UNQUOTED([OPAL_BTL_SM_HAVE_CMA], [$btl_sm_cma_happy],
|
||||
[If CMA support can be enabled within sm])
|
||||
|
||||
AC_DEFINE_UNQUOTED([OPAL_BTL_SM_HAVE_KNEM], [$btl_sm_knem_happy],
|
||||
[If KNEM support can be enabled within sm])
|
||||
|
||||
OPAL_VAR_SCOPE_POP
|
||||
|
||||
# always happy
|
||||
[$1]
|
||||
|
||||
OPAL_SUMMARY_ADD([[Transports]],[[Shared memory/copy in+copy out]],[$1],[yes])
|
||||
|
||||
# substitute in the things needed to build with XPMEM support
|
||||
AC_SUBST([btl_sm_CFLAGS])
|
||||
AC_SUBST([btl_sm_CPPFLAGS])
|
||||
AC_SUBST([btl_sm_LDFLAGS])
|
||||
AC_SUBST([btl_sm_LIBS])
|
||||
])dnl
|
||||
@ -17,7 +17,7 @@
|
||||
# This is the US/English help file for Open MPI's shared memory support.
|
||||
#
|
||||
[sys call fail]
|
||||
A system call failed during vader shared memory BTL initialization
|
||||
A system call failed during sm shared memory BTL initialization
|
||||
that should not have. It is likely that your MPI job will now either
|
||||
abort or experience performance degradation.
|
||||
|
||||
@ -25,13 +25,13 @@ abort or experience performance degradation.
|
||||
Error: %s (errno %d)
|
||||
#
|
||||
[no locality]
|
||||
WARNING: Missing locality information required for vader shared memory
|
||||
WARNING: Missing locality information required for sm shared memory
|
||||
BTL initialization. Continuing without shared memory support.
|
||||
#
|
||||
[knem permission denied]
|
||||
WARING: Open MPI failed to open the /dev/knem device due to a
|
||||
permissions problem. Please check with your system administrator to
|
||||
get the permissions fixed, or set the btl_vader_single_copy_mechanism
|
||||
get the permissions fixed, or set the btl_sm_single_copy_mechanism
|
||||
MCA variable to none to silence this warning and run without knem
|
||||
support.
|
||||
|
||||
@ -41,10 +41,10 @@ support.
|
||||
[knem fail open]
|
||||
WARNING: Open MPI failed to open the /dev/knem device due to a local
|
||||
error. Please check with your system administrator to get the problem
|
||||
fixed, or set the btl_vader_single_copy_mechanism MCA variable to none
|
||||
fixed, or set the btl_sm_single_copy_mechanism MCA variable to none
|
||||
to silence this warning and run without knem support.
|
||||
|
||||
The vader shared memory BTL will fall back on another single-copy
|
||||
The sm shared memory BTL will fall back on another single-copy
|
||||
mechanism if one is available. This may result in lower performance.
|
||||
|
||||
Local host: %s
|
||||
@ -54,10 +54,10 @@ mechanism if one is available. This may result in lower performance.
|
||||
WARNING: Open MPI failed to retrieve the ABI version from the
|
||||
/dev/knem device due to a local error. This usually indicates an
|
||||
error in your knem installation; please check with your system
|
||||
administrator, or set the btl_vader_single_copy_mechanism MCA variable
|
||||
administrator, or set the btl_sm_single_copy_mechanism MCA variable
|
||||
to none to silence this warning and run without knem support.
|
||||
|
||||
The vader shared memory BTL will fall back on another single-copy
|
||||
The sm shared memory BTL will fall back on another single-copy
|
||||
mechanism if one is available. This may result in lower performance.
|
||||
|
||||
Local host: %s
|
||||
@ -68,11 +68,11 @@ WARNING: Open MPI was compiled with support for one version of the
|
||||
knem kernel module, but it discovered a different version running in
|
||||
/dev/knem. Open MPI needs to be installed with support for the same
|
||||
version of knem as is in the running Linux kernel. Please check with
|
||||
your system administrator, or set the btl_vader_single_copy_mechanism
|
||||
your system administrator, or set the btl_sm_single_copy_mechanism
|
||||
MCA variable to none to silence this warning and run without knem
|
||||
support.
|
||||
|
||||
The vader shared memory BTL will fall back on another single-copy
|
||||
The sm shared memory BTL will fall back on another single-copy
|
||||
mechanism if one is available. This may result in lower performance.
|
||||
|
||||
Local host: %s
|
||||
@ -82,10 +82,10 @@ mechanism if one is available. This may result in lower performance.
|
||||
[knem mmap fail]
|
||||
Open MPI failed to map support from the knem Linux kernel module; this
|
||||
shouldn't happen. Please check with your system administrator, or set
|
||||
the btl_vader_single_copy_mechanism MCA variable to none to silence
|
||||
the btl_sm_single_copy_mechanism MCA variable to none to silence
|
||||
this warning and run without knem support.
|
||||
|
||||
The vader shared memory BTL will fall back on another single-copy
|
||||
The sm shared memory BTL will fall back on another single-copy
|
||||
mechanism if one is available. This may result in lower performance.
|
||||
|
||||
Local host: %s
|
||||
@ -95,10 +95,10 @@ mechanism if one is available. This may result in lower performance.
|
||||
[knem init error]
|
||||
Open MPI encountered an error during the knem initialization. Please
|
||||
check with your system administrator, or set the
|
||||
btl_vader_single_copy_mechanism MCA variable to none to silence this
|
||||
btl_sm_single_copy_mechanism MCA variable to none to silence this
|
||||
warning and run without knem support.
|
||||
|
||||
The vader shared memory BTL will fall back on another single-copy
|
||||
The sm shared memory BTL will fall back on another single-copy
|
||||
mechanism if one is available. This may result in lower performance.
|
||||
|
||||
Local host: %s
|
||||
@ -107,40 +107,40 @@ mechanism if one is available. This may result in lower performance.
|
||||
#
|
||||
[knem requested but not available]
|
||||
WARNING: Linux kernel knem support was requested via the
|
||||
btl_vader_single_copy_mechanism MCA parameter, but Knem support was either not
|
||||
btl_sm_single_copy_mechanism MCA parameter, but Knem support was either not
|
||||
compiled into this Open MPI installation, or Knem support was unable
|
||||
to be activated in this process.
|
||||
|
||||
The vader BTL will fall back on another single-copy mechanism if one
|
||||
The sm BTL will fall back on another single-copy mechanism if one
|
||||
is available. This may result in lower performance.
|
||||
|
||||
Local host: %s
|
||||
#
|
||||
[cma-permission-denied]
|
||||
WARNING: Linux kernel CMA support was requested via the
|
||||
btl_vader_single_copy_mechanism MCA variable, but CMA support is
|
||||
btl_sm_single_copy_mechanism MCA variable, but CMA support is
|
||||
not available due to restrictive ptrace settings.
|
||||
|
||||
The vader shared memory BTL will fall back on another single-copy
|
||||
The sm shared memory BTL will fall back on another single-copy
|
||||
mechanism if one is available. This may result in lower performance.
|
||||
|
||||
Local host: %s
|
||||
#
|
||||
[cma-different-user-namespace-error]
|
||||
ERROR: Linux kernel CMA support was requested via the
|
||||
btl_vader_single_copy_mechanism MCA variable, but CMA support is
|
||||
btl_sm_single_copy_mechanism MCA variable, but CMA support is
|
||||
not available due to different user namespaces.
|
||||
|
||||
Your MPI job will abort now. Please select another value for
|
||||
btl_vader_single_copy_mechanism.
|
||||
btl_sm_single_copy_mechanism.
|
||||
|
||||
Local host: %s
|
||||
#
|
||||
[cma-different-user-namespace-warning]
|
||||
WARNING: The default btl_vader_single_copy_mechanism CMA is
|
||||
WARNING: The default btl_sm_single_copy_mechanism CMA is
|
||||
not available due to different user namespaces.
|
||||
|
||||
The vader shared memory BTL will fall back on another single-copy
|
||||
The sm shared memory BTL will fall back on another single-copy
|
||||
mechanism if one is available. This may result in lower performance.
|
||||
|
||||
Local host: %s
|
||||
@ -149,7 +149,7 @@ mechanism if one is available. This may result in lower performance.
|
||||
WARNING: Could not generate an xpmem segment id for this process'
|
||||
address space.
|
||||
|
||||
The vader shared memory BTL will fall back on another single-copy
|
||||
The sm shared memory BTL will fall back on another single-copy
|
||||
mechanism if one is available. This may result in lower performance.
|
||||
|
||||
Local host: %s
|
||||
@ -1,329 +0,0 @@
|
||||
/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
|
||||
/*
|
||||
* Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
|
||||
* University Research and Technology
|
||||
* Corporation. All rights reserved.
|
||||
* Copyright (c) 2004-2009 The University of Tennessee and The University
|
||||
* of Tennessee Research Foundation. All rights
|
||||
* reserved.
|
||||
* Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
|
||||
* University of Stuttgart. All rights reserved.
|
||||
* Copyright (c) 2004-2005 The Regents of the University of California.
|
||||
* All rights reserved.
|
||||
* Copyright (c) 2006-2007 Voltaire. All rights reserved.
|
||||
* Copyright (c) 2009-2010 Cisco Systems, Inc. All rights reserved.
|
||||
* Copyright (c) 2010-2018 Los Alamos National Security, LLC. All rights
|
||||
* reserved.
|
||||
* Copyright (c) 2015 Mellanox Technologies. All rights reserved.
|
||||
* Copyright (c) 2018 Triad National Security, LLC. All rights
|
||||
* reserved.
|
||||
* Copyright (c) 2020 Google, LLC. All rights reserved.
|
||||
*
|
||||
* Copyright (c) 2019-2020 Intel, Inc. All rights reserved.
|
||||
* $COPYRIGHT$
|
||||
*
|
||||
* Additional copyrights may follow
|
||||
*
|
||||
* $HEADER$
|
||||
*/
|
||||
/**
|
||||
* @file
|
||||
*/
|
||||
#ifndef MCA_BTL_VADER_H
|
||||
#define MCA_BTL_VADER_H
|
||||
|
||||
#include "opal_config.h"
|
||||
|
||||
#include <stddef.h>
|
||||
#include <stdlib.h>
|
||||
#include <string.h>
|
||||
|
||||
# include <stdint.h>
|
||||
#ifdef HAVE_SCHED_H
|
||||
# include <sched.h>
|
||||
#endif /* HAVE_SCHED_H */
|
||||
#ifdef HAVE_UNISTD_H
|
||||
# include <unistd.h>
|
||||
#endif /* HAVE_UNISTD_H */
|
||||
|
||||
#include "opal/mca/shmem/base/base.h"
|
||||
|
||||
#include "opal/class/opal_free_list.h"
|
||||
#include "opal/sys/atomic.h"
|
||||
#include "opal/mca/btl/btl.h"
|
||||
#include "opal/mca/rcache/rcache.h"
|
||||
#include "opal/mca/rcache/base/rcache_base_vma.h"
|
||||
#include "opal/mca/btl/base/base.h"
|
||||
#include "opal/mca/rcache/rcache.h"
|
||||
#include "opal/mca/rcache/base/base.h"
|
||||
#include "opal/mca/btl/base/btl_base_error.h"
|
||||
#include "opal/mca/mpool/base/base.h"
|
||||
#include "opal/util/proc.h"
|
||||
#include "btl_vader_endpoint.h"
|
||||
|
||||
#include "opal/mca/pmix/pmix-internal.h"
|
||||
|
||||
#include "btl_vader_xpmem.h"
|
||||
#include "btl_vader_knem.h"
|
||||
|
||||
BEGIN_C_DECLS
|
||||
|
||||
#define min(a,b) ((a) < (b) ? (a) : (b))
|
||||
|
||||
/*
|
||||
* Shared Memory resource managment
|
||||
*/
|
||||
|
||||
struct vader_fifo_t;
|
||||
|
||||
/*
|
||||
* Modex data
|
||||
*/
|
||||
union vader_modex_t {
|
||||
#if OPAL_BTL_VADER_HAVE_XPMEM
|
||||
struct vader_modex_xpmem_t {
|
||||
xpmem_segid_t seg_id;
|
||||
void *segment_base;
|
||||
uintptr_t address_max;
|
||||
} xpmem;
|
||||
#endif
|
||||
struct vader_modex_other_t {
|
||||
ino_t user_ns_id;
|
||||
int seg_ds_size;
|
||||
/* seg_ds needs to be the last element */
|
||||
opal_shmem_ds_t seg_ds;
|
||||
} other;
|
||||
};
|
||||
|
||||
/**
|
||||
* Single copy mechanisms
|
||||
*/
|
||||
enum {
|
||||
MCA_BTL_VADER_XPMEM = 0,
|
||||
MCA_BTL_VADER_CMA = 1,
|
||||
MCA_BTL_VADER_KNEM = 2,
|
||||
MCA_BTL_VADER_NONE = 3,
|
||||
MCA_BTL_VADER_EMUL = 4,
|
||||
};
|
||||
|
||||
/**
|
||||
* Shared Memory (VADER) BTL module.
|
||||
*/
|
||||
struct mca_btl_vader_component_t {
|
||||
mca_btl_base_component_3_0_0_t super; /**< base BTL component */
|
||||
int vader_free_list_num; /**< initial size of free lists */
|
||||
int vader_free_list_max; /**< maximum size of free lists */
|
||||
int vader_free_list_inc; /**< number of elements to alloc when growing free lists */
|
||||
#if OPAL_BTL_VADER_HAVE_XPMEM
|
||||
xpmem_segid_t my_seg_id; /**< this rank's xpmem segment id */
|
||||
uintptr_t my_address_max; /**< largest address */
|
||||
mca_rcache_base_vma_module_t *vma_module; /**< registration cache for xpmem segments */
|
||||
#endif
|
||||
opal_shmem_ds_t seg_ds; /**< this rank's shared memory segment (when not using xpmem) */
|
||||
|
||||
opal_mutex_t lock; /**< lock to protect concurrent updates to this structure's members */
|
||||
char *my_segment; /**< this rank's base pointer */
|
||||
size_t segment_size; /**< size of my_segment */
|
||||
int32_t num_smp_procs; /**< current number of smp procs on this host */
|
||||
opal_atomic_int32_t local_rank; /**< current rank index at add_procs() time */
|
||||
opal_free_list_t vader_frags_eager; /**< free list of vader send frags */
|
||||
opal_free_list_t vader_frags_max_send; /**< free list of vader max send frags (large fragments) */
|
||||
opal_free_list_t vader_frags_user; /**< free list of small inline frags */
|
||||
opal_free_list_t vader_fboxes; /**< free list of available fast-boxes */
|
||||
|
||||
unsigned int fbox_threshold; /**< number of sends required before we setup a send fast box for a peer */
|
||||
unsigned int fbox_max; /**< maximum number of send fast boxes to allocate */
|
||||
unsigned int fbox_size; /**< size of each peer fast box allocation */
|
||||
|
||||
int single_copy_mechanism; /**< single copy mechanism to use */
|
||||
|
||||
int memcpy_limit; /**< Limit where we switch from memmove to memcpy */
|
||||
int log_attach_align; /**< Log of the alignment for xpmem segments */
|
||||
unsigned int max_inline_send; /**< Limit for copy-in-copy-out fragments */
|
||||
|
||||
mca_btl_base_endpoint_t *endpoints; /**< array of local endpoints (one for each local peer including myself) */
|
||||
mca_btl_base_endpoint_t **fbox_in_endpoints; /**< array of fast box in endpoints */
|
||||
unsigned int num_fbox_in_endpoints; /**< number of fast boxes to poll */
|
||||
struct vader_fifo_t *my_fifo; /**< pointer to the local fifo */
|
||||
|
||||
opal_list_t pending_endpoints; /**< list of endpoints with pending fragments */
|
||||
opal_list_t pending_fragments; /**< fragments pending remote completion */
|
||||
|
||||
char *backing_directory; /**< directory to place shared memory backing files */
|
||||
|
||||
/* knem stuff */
|
||||
#if OPAL_BTL_VADER_HAVE_KNEM
|
||||
unsigned int knem_dma_min; /**< minimum size to enable DMA for knem transfers (0 disables) */
|
||||
#endif
|
||||
mca_mpool_base_module_t *mpool;
|
||||
};
|
||||
typedef struct mca_btl_vader_component_t mca_btl_vader_component_t;
|
||||
OPAL_MODULE_DECLSPEC extern mca_btl_vader_component_t mca_btl_vader_component;
|
||||
|
||||
/**
|
||||
* VADER BTL Interface
|
||||
*/
|
||||
struct mca_btl_vader_t {
|
||||
mca_btl_base_module_t super; /**< base BTL interface */
|
||||
bool btl_inited; /**< flag indicating if btl has been inited */
|
||||
mca_btl_base_module_error_cb_fn_t error_cb;
|
||||
#if OPAL_BTL_VADER_HAVE_KNEM
|
||||
int knem_fd;
|
||||
|
||||
/* registration cache */
|
||||
mca_rcache_base_module_t *knem_rcache;
|
||||
#endif
|
||||
};
|
||||
typedef struct mca_btl_vader_t mca_btl_vader_t;
|
||||
OPAL_MODULE_DECLSPEC extern mca_btl_vader_t mca_btl_vader;
|
||||
|
||||
/* number of peers on the node (not including self) */
|
||||
#define MCA_BTL_VADER_NUM_LOCAL_PEERS opal_process_info.num_local_peers
|
||||
|
||||
/* local rank in the group */
|
||||
#define MCA_BTL_VADER_LOCAL_RANK opal_process_info.my_local_rank
|
||||
|
||||
/* memcpy is faster at larger sizes but is undefined if the
|
||||
pointers are aliased (TODO -- readd alias check) */
|
||||
static inline void vader_memmove (void *dst, void *src, size_t size)
|
||||
{
|
||||
if (size >= (size_t) mca_btl_vader_component.memcpy_limit) {
|
||||
memcpy (dst, src, size);
|
||||
} else {
|
||||
memmove (dst, src, size);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Initiate a send to the peer.
|
||||
*
|
||||
* @param btl (IN) BTL module
|
||||
* @param peer (IN) BTL peer addressing
|
||||
*/
|
||||
int mca_btl_vader_send(struct mca_btl_base_module_t *btl,
|
||||
struct mca_btl_base_endpoint_t *endpoint,
|
||||
struct mca_btl_base_descriptor_t *descriptor,
|
||||
mca_btl_base_tag_t tag);
|
||||
|
||||
/**
|
||||
* Initiate an inline send to the peer.
|
||||
*
|
||||
* @param btl (IN) BTL module
|
||||
* @param peer (IN) BTL peer addressing
|
||||
*/
|
||||
int mca_btl_vader_sendi (struct mca_btl_base_module_t *btl,
|
||||
struct mca_btl_base_endpoint_t *endpoint,
|
||||
struct opal_convertor_t *convertor,
|
||||
void *header, size_t header_size,
|
||||
size_t payload_size, uint8_t order,
|
||||
uint32_t flags, mca_btl_base_tag_t tag,
|
||||
mca_btl_base_descriptor_t **descriptor);
|
||||
|
||||
/**
|
||||
* Initiate an synchronous put.
|
||||
*
|
||||
* @param btl (IN) BTL module
|
||||
* @param endpoint (IN) BTL addressing information
|
||||
* @param descriptor (IN) Description of the data to be transferred
|
||||
*/
|
||||
#if OPAL_BTL_VADER_HAVE_XPMEM
|
||||
int mca_btl_vader_put_xpmem (mca_btl_base_module_t *btl, mca_btl_base_endpoint_t *endpoint, void *local_address,
|
||||
uint64_t remote_address, mca_btl_base_registration_handle_t *local_handle,
|
||||
mca_btl_base_registration_handle_t *remote_handle, size_t size, int flags,
|
||||
int order, mca_btl_base_rdma_completion_fn_t cbfunc, void *cbcontext, void *cbdata);
|
||||
#endif
|
||||
|
||||
#if OPAL_BTL_VADER_HAVE_CMA
|
||||
int mca_btl_vader_put_cma (mca_btl_base_module_t *btl, mca_btl_base_endpoint_t *endpoint, void *local_address,
|
||||
uint64_t remote_address, mca_btl_base_registration_handle_t *local_handle,
|
||||
mca_btl_base_registration_handle_t *remote_handle, size_t size, int flags,
|
||||
int order, mca_btl_base_rdma_completion_fn_t cbfunc, void *cbcontext, void *cbdata);
|
||||
#endif
|
||||
|
||||
#if OPAL_BTL_VADER_HAVE_KNEM
|
||||
int mca_btl_vader_put_knem (mca_btl_base_module_t *btl, mca_btl_base_endpoint_t *endpoint, void *local_address,
|
||||
uint64_t remote_address, mca_btl_base_registration_handle_t *local_handle,
|
||||
mca_btl_base_registration_handle_t *remote_handle, size_t size, int flags,
|
||||
int order, mca_btl_base_rdma_completion_fn_t cbfunc, void *cbcontext, void *cbdata);
|
||||
#endif
|
||||
|
||||
int mca_btl_vader_put_sc_emu (mca_btl_base_module_t *btl, mca_btl_base_endpoint_t *endpoint, void *local_address,
|
||||
uint64_t remote_address, mca_btl_base_registration_handle_t *local_handle,
|
||||
mca_btl_base_registration_handle_t *remote_handle, size_t size, int flags,
|
||||
int order, mca_btl_base_rdma_completion_fn_t cbfunc, void *cbcontext, void *cbdata);
|
||||
|
||||
/**
|
||||
* Initiate an synchronous get.
|
||||
*
|
||||
* @param btl (IN) BTL module
|
||||
* @param endpoint (IN) BTL addressing information
|
||||
* @param descriptor (IN) Description of the data to be transferred
|
||||
*/
|
||||
#if OPAL_BTL_VADER_HAVE_XPMEM
|
||||
int mca_btl_vader_get_xpmem (mca_btl_base_module_t *btl, mca_btl_base_endpoint_t *endpoint, void *local_address,
|
||||
uint64_t remote_address, mca_btl_base_registration_handle_t *local_handle,
|
||||
mca_btl_base_registration_handle_t *remote_handle, size_t size, int flags,
|
||||
int order, mca_btl_base_rdma_completion_fn_t cbfunc, void *cbcontext, void *cbdata);
|
||||
#endif
|
||||
|
||||
#if OPAL_BTL_VADER_HAVE_CMA
|
||||
int mca_btl_vader_get_cma (mca_btl_base_module_t *btl, mca_btl_base_endpoint_t *endpoint, void *local_address,
|
||||
uint64_t remote_address, mca_btl_base_registration_handle_t *local_handle,
|
||||
mca_btl_base_registration_handle_t *remote_handle, size_t size, int flags,
|
||||
int order, mca_btl_base_rdma_completion_fn_t cbfunc, void *cbcontext, void *cbdata);
|
||||
#endif
|
||||
|
||||
#if OPAL_BTL_VADER_HAVE_KNEM
|
||||
int mca_btl_vader_get_knem (mca_btl_base_module_t *btl, mca_btl_base_endpoint_t *endpoint, void *local_address,
|
||||
uint64_t remote_address, mca_btl_base_registration_handle_t *local_handle,
|
||||
mca_btl_base_registration_handle_t *remote_handle, size_t size, int flags,
|
||||
int order, mca_btl_base_rdma_completion_fn_t cbfunc, void *cbcontext, void *cbdata);
|
||||
#endif
|
||||
|
||||
ino_t mca_btl_vader_get_user_ns_id(void);
|
||||
|
||||
int mca_btl_vader_get_sc_emu (mca_btl_base_module_t *btl, mca_btl_base_endpoint_t *endpoint, void *local_address,
|
||||
uint64_t remote_address, mca_btl_base_registration_handle_t *local_handle,
|
||||
mca_btl_base_registration_handle_t *remote_handle, size_t size, int flags,
|
||||
int order, mca_btl_base_rdma_completion_fn_t cbfunc, void *cbcontext, void *cbdata);
|
||||
|
||||
int mca_btl_vader_emu_aop (struct mca_btl_base_module_t *btl, struct mca_btl_base_endpoint_t *endpoint,
|
||||
uint64_t remote_address, mca_btl_base_registration_handle_t *remote_handle,
|
||||
mca_btl_base_atomic_op_t op, uint64_t operand, int flags, int order,
|
||||
mca_btl_base_rdma_completion_fn_t cbfunc, void *cbcontext, void *cbdata);
|
||||
|
||||
int mca_btl_vader_emu_afop (struct mca_btl_base_module_t *btl, struct mca_btl_base_endpoint_t *endpoint,
|
||||
void *local_address, uint64_t remote_address, mca_btl_base_registration_handle_t *local_handle,
|
||||
mca_btl_base_registration_handle_t *remote_handle, mca_btl_base_atomic_op_t op,
|
||||
uint64_t operand, int flags, int order, mca_btl_base_rdma_completion_fn_t cbfunc,
|
||||
void *cbcontext, void *cbdata);
|
||||
|
||||
int mca_btl_vader_emu_acswap (struct mca_btl_base_module_t *btl, struct mca_btl_base_endpoint_t *endpoint,
|
||||
void *local_address, uint64_t remote_address, mca_btl_base_registration_handle_t *local_handle,
|
||||
mca_btl_base_registration_handle_t *remote_handle, uint64_t compare, uint64_t value, int flags,
|
||||
int order, mca_btl_base_rdma_completion_fn_t cbfunc, void *cbcontext, void *cbdata);
|
||||
|
||||
void mca_btl_vader_sc_emu_init (void);
|
||||
|
||||
/**
|
||||
* Allocate a segment.
|
||||
*
|
||||
* @param btl (IN) BTL module
|
||||
* @param size (IN) Request segment size.
|
||||
*/
|
||||
mca_btl_base_descriptor_t* mca_btl_vader_alloc (struct mca_btl_base_module_t* btl,
|
||||
struct mca_btl_base_endpoint_t* endpoint,
|
||||
uint8_t order, size_t size, uint32_t flags);
|
||||
|
||||
/**
|
||||
* Return a segment allocated by this BTL.
|
||||
*
|
||||
* @param btl (IN) BTL module
|
||||
* @param segment (IN) Allocated segment.
|
||||
*/
|
||||
int mca_btl_vader_free (struct mca_btl_base_module_t *btl, mca_btl_base_descriptor_t *des);
|
||||
|
||||
|
||||
END_C_DECLS
|
||||
|
||||
#endif
|
||||
@ -1,49 +0,0 @@
|
||||
/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
|
||||
/*
|
||||
* Copyright (c) 2010-2017 Los Alamos National Security, LLC. All rights
|
||||
* reserved.
|
||||
* Copyright (c) 2019 Google, Inc. All rights reserved.
|
||||
* $COPYRIGHT$
|
||||
*
|
||||
* Additional copyrights may follow
|
||||
*
|
||||
* $HEADER$
|
||||
*/
|
||||
|
||||
#include "opal_config.h"
|
||||
|
||||
#include "btl_vader.h"
|
||||
#include "btl_vader_frag.h"
|
||||
#include "btl_vader_endpoint.h"
|
||||
#include "btl_vader_xpmem.h"
|
||||
|
||||
int mca_btl_vader_emu_aop (struct mca_btl_base_module_t *btl, struct mca_btl_base_endpoint_t *endpoint,
|
||||
uint64_t remote_address, mca_btl_base_registration_handle_t *remote_handle,
|
||||
mca_btl_base_atomic_op_t op, uint64_t operand, int flags, int order,
|
||||
mca_btl_base_rdma_completion_fn_t cbfunc, void *cbcontext, void *cbdata)
|
||||
{
|
||||
size_t size = (flags & MCA_BTL_ATOMIC_FLAG_32BIT) ? 4 : 8;
|
||||
return mca_btl_vader_rdma_frag_start (btl, endpoint, MCA_BTL_VADER_OP_ATOMIC, operand, 0, op, order, flags,
|
||||
size, NULL, remote_address, cbfunc, cbcontext, cbdata);
|
||||
}
|
||||
|
||||
int mca_btl_vader_emu_afop (struct mca_btl_base_module_t *btl, struct mca_btl_base_endpoint_t *endpoint,
|
||||
void *local_address, uint64_t remote_address, mca_btl_base_registration_handle_t *local_handle,
|
||||
mca_btl_base_registration_handle_t *remote_handle, mca_btl_base_atomic_op_t op,
|
||||
uint64_t operand, int flags, int order, mca_btl_base_rdma_completion_fn_t cbfunc,
|
||||
void *cbcontext, void *cbdata)
|
||||
{
|
||||
size_t size = (flags & MCA_BTL_ATOMIC_FLAG_32BIT) ? 4 : 8;
|
||||
return mca_btl_vader_rdma_frag_start (btl, endpoint, MCA_BTL_VADER_OP_ATOMIC, operand, 0, op, order, flags,
|
||||
size, local_address, remote_address, cbfunc, cbcontext, cbdata);
|
||||
}
|
||||
|
||||
int mca_btl_vader_emu_acswap (struct mca_btl_base_module_t *btl, struct mca_btl_base_endpoint_t *endpoint,
|
||||
void *local_address, uint64_t remote_address, mca_btl_base_registration_handle_t *local_handle,
|
||||
mca_btl_base_registration_handle_t *remote_handle, uint64_t compare, uint64_t value, int flags,
|
||||
int order, mca_btl_base_rdma_completion_fn_t cbfunc, void *cbcontext, void *cbdata)
|
||||
{
|
||||
size_t size = (flags & MCA_BTL_ATOMIC_FLAG_32BIT) ? 4 : 8;
|
||||
return mca_btl_vader_rdma_frag_start (btl, endpoint, MCA_BTL_VADER_OP_CSWAP, compare, value, 0, order,
|
||||
flags, size, local_address, remote_address, cbfunc, cbcontext, cbdata);
|
||||
}
|
||||
@ -1,53 +0,0 @@
|
||||
# -*- shell-script -*-
|
||||
#
|
||||
# Copyright (c) 2009 The University of Tennessee and The University
|
||||
# of Tennessee Research Foundation. All rights
|
||||
# reserved.
|
||||
# Copyright (c) 2009-2016 Cisco Systems, Inc. All rights reserved.
|
||||
# Copyright (c) 2011-2014 Los Alamos National Security, LLC. All rights
|
||||
# reserved.
|
||||
# Copyright (c) 2015 Research Organization for Information Science
|
||||
# and Technology (RIST). All rights reserved.
|
||||
# $COPYRIGHT$
|
||||
#
|
||||
# Additional copyrights may follow
|
||||
#
|
||||
# $HEADER$
|
||||
#
|
||||
|
||||
# MCA_btl_vader_CONFIG([action-if-can-compile],
|
||||
# [action-if-cant-compile])
|
||||
# ------------------------------------------------
|
||||
AC_DEFUN([MCA_opal_btl_vader_CONFIG],[
|
||||
AC_CONFIG_FILES([opal/mca/btl/vader/Makefile])
|
||||
|
||||
OPAL_VAR_SCOPE_PUSH([btl_vader_xpmem_happy btl_vader_cma_happy btl_vader_knem_happy])
|
||||
|
||||
# Check for single-copy APIs
|
||||
|
||||
OPAL_CHECK_XPMEM([btl_vader], [btl_vader_xpmem_happy=1], [btl_vader_xpmem_happy=0])
|
||||
OPAL_CHECK_KNEM([btl_vader], [btl_vader_knem_happy=1],[btl_vader_knem_happy=0])
|
||||
OPAL_CHECK_CMA([btl_vader], [AC_CHECK_HEADER([sys/prctl.h]) btl_vader_cma_happy=1], [btl_vader_cma_happy=0])
|
||||
|
||||
AC_DEFINE_UNQUOTED([OPAL_BTL_VADER_HAVE_XPMEM], [$btl_vader_xpmem_happy],
|
||||
[If XPMEM support can be enabled within vader])
|
||||
|
||||
AC_DEFINE_UNQUOTED([OPAL_BTL_VADER_HAVE_CMA], [$btl_vader_cma_happy],
|
||||
[If CMA support can be enabled within vader])
|
||||
|
||||
AC_DEFINE_UNQUOTED([OPAL_BTL_VADER_HAVE_KNEM], [$btl_vader_knem_happy],
|
||||
[If KNEM support can be enabled within vader])
|
||||
|
||||
OPAL_VAR_SCOPE_POP
|
||||
|
||||
# always happy
|
||||
[$1]
|
||||
|
||||
OPAL_SUMMARY_ADD([[Transports]],[[Shared memory/copy in+copy out]],[$1],[yes])
|
||||
|
||||
# substitute in the things needed to build with XPMEM support
|
||||
AC_SUBST([btl_vader_CFLAGS])
|
||||
AC_SUBST([btl_vader_CPPFLAGS])
|
||||
AC_SUBST([btl_vader_LDFLAGS])
|
||||
AC_SUBST([btl_vader_LIBS])
|
||||
])dnl
|
||||
Загрузка…
x
Ссылка в новой задаче
Block a user