Per RFC, remove the udapl BTL.
This commit was SVN r29400.
Этот коммит содержится в:
Jeff Squyres
родитель
9389592e05
Коммит
66dadbe1e7
@ -1,63 +0,0 @@
|
|||||||
#
|
|
||||||
# Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
|
|
||||||
# University Research and Technology
|
|
||||||
# Corporation. All rights reserved.
|
|
||||||
# Copyright (c) 2004-2005 The University of Tennessee and The University
|
|
||||||
# of Tennessee Research Foundation. All rights
|
|
||||||
# reserved.
|
|
||||||
# Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
|
|
||||||
# University of Stuttgart. All rights reserved.
|
|
||||||
# Copyright (c) 2004-2005 The Regents of the University of California.
|
|
||||||
# All rights reserved.
|
|
||||||
# Copyright (c) 2007 Sun Microsystems, Inc. All rights reserved.
|
|
||||||
#
|
|
||||||
# Copyright (c) 2010 Cisco Systems, Inc. All rights reserved.
|
|
||||||
# $COPYRIGHT$
|
|
||||||
#
|
|
||||||
# Additional copyrights may follow
|
|
||||||
#
|
|
||||||
# $HEADER$
|
|
||||||
#
|
|
||||||
|
|
||||||
CFLAGS = $(btl_udapl_CFLAGS)
|
|
||||||
AM_CPPFLAGS = $(btl_udapl_CPPFLAGS) -DPKGDATADIR=\"$(pkgdatadir)\"
|
|
||||||
|
|
||||||
dist_pkgdata_DATA = \
|
|
||||||
help-mpi-btl-udapl.txt
|
|
||||||
|
|
||||||
udapl_sources = \
|
|
||||||
btl_udapl.c \
|
|
||||||
btl_udapl.h \
|
|
||||||
btl_udapl_component.c \
|
|
||||||
btl_udapl_eager_rdma.h \
|
|
||||||
btl_udapl_endpoint.c \
|
|
||||||
btl_udapl_endpoint.h \
|
|
||||||
btl_udapl_frag.c \
|
|
||||||
btl_udapl_frag.h \
|
|
||||||
btl_udapl_mca.c \
|
|
||||||
btl_udapl_mca.h \
|
|
||||||
btl_udapl_proc.c \
|
|
||||||
btl_udapl_proc.h
|
|
||||||
|
|
||||||
# Make the output library in this directory, and name it either
|
|
||||||
# mca_<type>_<name>.la (for DSO builds) or libmca_<type>_<name>.la
|
|
||||||
# (for static builds).
|
|
||||||
|
|
||||||
if MCA_BUILD_ompi_btl_udapl_DSO
|
|
||||||
component_noinst =
|
|
||||||
component_install = mca_btl_udapl.la
|
|
||||||
else
|
|
||||||
component_noinst = libmca_btl_udapl.la
|
|
||||||
component_install =
|
|
||||||
endif
|
|
||||||
|
|
||||||
mcacomponentdir = $(pkglibdir)
|
|
||||||
mcacomponent_LTLIBRARIES = $(component_install)
|
|
||||||
mca_btl_udapl_la_SOURCES = $(udapl_sources)
|
|
||||||
mca_btl_udapl_la_LIBADD = $(btl_udapl_LIBS)
|
|
||||||
mca_btl_udapl_la_LDFLAGS = -module -avoid-version $(btl_udapl_LDFLAGS)
|
|
||||||
|
|
||||||
noinst_LTLIBRARIES = $(component_noinst)
|
|
||||||
libmca_btl_udapl_la_SOURCES = $(udapl_sources)
|
|
||||||
libmca_btl_udapl_la_LIBADD = $(btl_udapl_LIBS)
|
|
||||||
libmca_btl_udapl_la_LDFLAGS = -module -avoid-version $(btl_udapl_LDFLAGS)
|
|
||||||
Разница между файлами не показана из-за своего большого размера
Загрузить разницу
@ -1,468 +0,0 @@
|
|||||||
/* -*- Mode: C; c-basic-offset:4 ; -*- */
|
|
||||||
/*
|
|
||||||
* Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
|
|
||||||
* University Research and Technology
|
|
||||||
* Corporation. All rights reserved.
|
|
||||||
* Copyright (c) 2004-2009 The University of Tennessee and The University
|
|
||||||
* of Tennessee Research Foundation. All rights
|
|
||||||
* reserved.
|
|
||||||
* Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
|
|
||||||
* University of Stuttgart. All rights reserved.
|
|
||||||
* Copyright (c) 2004-2005 The Regents of the University of California.
|
|
||||||
* All rights reserved.
|
|
||||||
* Copyright (c) 2007-2009 Sun Microsystems, Inc. All rights reserved.
|
|
||||||
* Copyright (c) 2007 Cisco Systems, Inc. All rights reserved.
|
|
||||||
*
|
|
||||||
* $COPYRIGHT$
|
|
||||||
*
|
|
||||||
* Additional copyrights may follow
|
|
||||||
*
|
|
||||||
* $HEADER$
|
|
||||||
*/
|
|
||||||
/**
|
|
||||||
* @file
|
|
||||||
*/
|
|
||||||
#ifndef MCA_BTL_UDAPL_H
|
|
||||||
#define MCA_BTL_UDAPL_H
|
|
||||||
|
|
||||||
#include "ompi_config.h"
|
|
||||||
#include <sys/types.h>
|
|
||||||
#include <string.h>
|
|
||||||
#include <dat/udat.h>
|
|
||||||
|
|
||||||
/* Open MPI includes */
|
|
||||||
#include "opal/class/opal_pointer_array.h"
|
|
||||||
#include "opal/mca/event/event.h"
|
|
||||||
#include "ompi/class/ompi_free_list.h"
|
|
||||||
#include "ompi/mca/btl/btl.h"
|
|
||||||
#include "ompi/mca/btl/base/base.h"
|
|
||||||
#include "ompi/mca/btl/base/btl_base_error.h"
|
|
||||||
#include "ompi/mca/mpool/mpool.h"
|
|
||||||
#include "ompi/mca/btl/btl.h"
|
|
||||||
#include "btl_udapl_endpoint.h"
|
|
||||||
|
|
||||||
BEGIN_C_DECLS
|
|
||||||
|
|
||||||
|
|
||||||
/**
|
|
||||||
* uDAPL BTL component.
|
|
||||||
*/
|
|
||||||
|
|
||||||
struct mca_btl_udapl_component_t {
|
|
||||||
mca_btl_base_component_2_0_0_t super; /**< base BTL component */
|
|
||||||
|
|
||||||
int32_t udapl_verbosity; /**< report out level, see
|
|
||||||
"Report Out from uDAPL BTL" below for details. */
|
|
||||||
size_t udapl_num_btls; /**< number of hcas available to the uDAPL component */
|
|
||||||
unsigned int udapl_max_btls; /**< maximum number of supported hcas */
|
|
||||||
struct mca_btl_udapl_module_t **udapl_btls; /**< array of available BTL modules */
|
|
||||||
int32_t udapl_num_recvs; /**< number of recv buffers to keep posted */
|
|
||||||
int32_t udapl_num_sends; /**< number of sends to post on endpoint */
|
|
||||||
int32_t udapl_sr_win; /**< number of fragments recieved before
|
|
||||||
returning credits to sender */
|
|
||||||
unsigned int udapl_timeout; /**< connection timeout, in microseconds */
|
|
||||||
size_t udapl_eager_frag_size;
|
|
||||||
size_t udapl_max_frag_size;
|
|
||||||
size_t udapl_eager_rdma_frag_size; /* size of the rdma fragement including data
|
|
||||||
* payload space
|
|
||||||
*/
|
|
||||||
|
|
||||||
int udapl_free_list_num; /**< initial size of free lists */
|
|
||||||
int udapl_free_list_max; /**< maximum size of free lists */
|
|
||||||
int udapl_free_list_inc; /**< number of elements to alloc when growing */
|
|
||||||
int32_t udapl_use_eager_rdma; /**< turn rdma for small msgs on/off */
|
|
||||||
int32_t udapl_eager_rdma_num; /**< number of rdma buffers allocated
|
|
||||||
for short messages */
|
|
||||||
int32_t udapl_max_eager_rdma_peers; /**< maximum number of peers allowed to
|
|
||||||
use RDMA for short messages (cap)
|
|
||||||
*/
|
|
||||||
int32_t udapl_eager_rdma_win; /**< number of eager RDMA fragments
|
|
||||||
recieved before returning credits to
|
|
||||||
sender */
|
|
||||||
int32_t udapl_conn_priv_data; /**< use connect priv data for proc data */
|
|
||||||
int32_t udapl_async_events; /**< dequeue asynchronous events */
|
|
||||||
int32_t udapl_buffer_alignment; /**< preferred communication buffer alignment, in bytes */
|
|
||||||
opal_list_t udapl_procs; /**< list of udapl proc structures */
|
|
||||||
opal_mutex_t udapl_lock; /**< lock for accessing module state */
|
|
||||||
char* udapl_mpool_name; /**< name of memory pool */
|
|
||||||
int32_t udapl_compare_subnet;/**< whether to compare with netmask or not */
|
|
||||||
char *if_include;
|
|
||||||
char **if_include_list;
|
|
||||||
char *if_exclude;
|
|
||||||
char **if_exclude_list;
|
|
||||||
char **if_list; /* used for checking entries not found */
|
|
||||||
int32_t ro_aware_system; /* default 0; 1 if relaxed ordered platform */
|
|
||||||
};
|
|
||||||
typedef struct mca_btl_udapl_component_t mca_btl_udapl_component_t;
|
|
||||||
|
|
||||||
OMPI_MODULE_DECLSPEC extern mca_btl_udapl_component_t mca_btl_udapl_component;
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
/**
|
|
||||||
* BTL Module Interface
|
|
||||||
*/
|
|
||||||
struct mca_btl_udapl_module_t {
|
|
||||||
mca_btl_base_module_t super; /**< base BTL interface */
|
|
||||||
mca_btl_udapl_addr_t udapl_addr;
|
|
||||||
|
|
||||||
/* uDAPL interface and other handles */
|
|
||||||
DAT_IA_HANDLE udapl_ia;
|
|
||||||
DAT_PZ_HANDLE udapl_pz;
|
|
||||||
DAT_PSP_HANDLE udapl_psp;
|
|
||||||
DAT_IA_ATTR udapl_ia_attr;
|
|
||||||
|
|
||||||
/* event dispatchers - async, data transfer, connection negotiation */
|
|
||||||
DAT_EVD_HANDLE udapl_evd_async;
|
|
||||||
DAT_EVD_HANDLE udapl_evd_dto;
|
|
||||||
DAT_EVD_HANDLE udapl_evd_conn;
|
|
||||||
DAT_EP_PARAM udapl_ep_param;
|
|
||||||
|
|
||||||
/* free list of fragment descriptors */
|
|
||||||
ompi_free_list_t udapl_frag_eager;
|
|
||||||
ompi_free_list_t udapl_frag_eager_recv;
|
|
||||||
ompi_free_list_t udapl_frag_max;
|
|
||||||
ompi_free_list_t udapl_frag_max_recv;
|
|
||||||
ompi_free_list_t udapl_frag_user;
|
|
||||||
ompi_free_list_t udapl_frag_control;
|
|
||||||
|
|
||||||
opal_mutex_t udapl_lock; /* lock for accessing module state */
|
|
||||||
opal_mutex_t udapl_eager_rdma_lock; /* eager rdma lock */
|
|
||||||
int32_t udapl_eager_rdma_endpoint_count; /* count of the number of
|
|
||||||
* endpoints in
|
|
||||||
* udapl_eager_rdma_endpoints
|
|
||||||
*/
|
|
||||||
opal_pointer_array_t *udapl_eager_rdma_endpoints; /* array of endpoints
|
|
||||||
* with eager rdma
|
|
||||||
* connections
|
|
||||||
*/
|
|
||||||
int32_t udapl_async_events;
|
|
||||||
int32_t udapl_connect_inprogress;
|
|
||||||
int32_t udapl_num_peers;
|
|
||||||
|
|
||||||
/* module specific limits */
|
|
||||||
int udapl_async_evd_qlen;
|
|
||||||
int udapl_conn_evd_qlen;
|
|
||||||
int udapl_dto_evd_qlen;
|
|
||||||
int udapl_max_request_dtos; /**< maximum number of outstanding consumer
|
|
||||||
submitted sends and rdma operations, see
|
|
||||||
section 6.6.6 of uDAPL Spec */
|
|
||||||
int udapl_max_recv_dtos; /**< maximum number of outstanding consumer
|
|
||||||
submitted recv operations, see section
|
|
||||||
6.6.6 of uDAPL Spec */
|
|
||||||
uint32_t udapl_if_mask; /**< netmask value btl module */
|
|
||||||
};
|
|
||||||
typedef struct mca_btl_udapl_module_t mca_btl_udapl_module_t;
|
|
||||||
extern mca_btl_udapl_module_t mca_btl_udapl_module;
|
|
||||||
|
|
||||||
struct mca_btl_udapl_reg_t {
|
|
||||||
mca_mpool_base_registration_t base;
|
|
||||||
DAT_LMR_HANDLE lmr; /* local memory region (LMR) */
|
|
||||||
DAT_LMR_TRIPLET lmr_triplet; /* LMR triplet - context, address, length */
|
|
||||||
DAT_RMR_CONTEXT rmr_context; /* remote memory region context handle */
|
|
||||||
|
|
||||||
};
|
|
||||||
typedef struct mca_btl_udapl_reg_t mca_btl_udapl_reg_t;
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Report Out from uDAPL BTL
|
|
||||||
*
|
|
||||||
* - BTL_ERROR() : Use to report out errors from uDAPL BTL. These are
|
|
||||||
* critical errors which will most likely cause the program to fail so
|
|
||||||
* this message should always be reported to the user. Defined in
|
|
||||||
* btl/base/btl_base_error.h.
|
|
||||||
* Example:
|
|
||||||
* dat_strerror(rc, (const char**)&major, (const char**)&minor);
|
|
||||||
* BTL_ERROR(("ERROR: %s %s %s\n", "dat_cr_accept", major, minor));
|
|
||||||
*
|
|
||||||
* - BTL_UDAPL_VERBOSE_OUTPUT() : Use to output different levels
|
|
||||||
* of verbosity to the user. See Note below.
|
|
||||||
* Example:
|
|
||||||
* BTL_UDAPL_VERBOSE_OUTPUT(VERBOSE_DIAGNOSE,
|
|
||||||
* ("WARNING: don't %s", "jump"));
|
|
||||||
*
|
|
||||||
* - BTL_UDAPL_VERBOSE_HELP() : Use output information as defined in
|
|
||||||
* uDAPL BTL help file (help-mpi-btl-udapl.txt). See Note below.
|
|
||||||
* Example:
|
|
||||||
* BTL_UDAPL_VERBOSE_HELP(VERBOSE_SHOW_HELP,
|
|
||||||
* ("help-mpi-btl-udapl.txt",
|
|
||||||
* "invalid num rdma segments", true, 22));
|
|
||||||
*
|
|
||||||
* Note : - Verbose levels are defined below. These levels are
|
|
||||||
* controlled by the mca parameter "btl_udapl_verbose".
|
|
||||||
* The verbose level is set to 10 by default so that critical
|
|
||||||
* error and useful help information will appear. Which ever value
|
|
||||||
* this param is set to, those messages as well as any lower level
|
|
||||||
* verbose messages will be reported.
|
|
||||||
* - Setting "btl_udapl_verbose" to "-1" will turn off all
|
|
||||||
* messages reported by the use of BTL_UDAPL_VERBOSE_*().
|
|
||||||
* - These macros should not be used in a critical path as they
|
|
||||||
* are always included in the compiled code.
|
|
||||||
* - These macros rely on the use of paranthesis around the "args"
|
|
||||||
* value.
|
|
||||||
*
|
|
||||||
* Values used with BTL_UDAPL_VERBOSE_*():
|
|
||||||
*
|
|
||||||
* - 0: critical user information; should always be reported;
|
|
||||||
* on by default
|
|
||||||
* - 10: useful help messages that would be reported from
|
|
||||||
* "help-mpi-btl-udapl.txt"; accessed from
|
|
||||||
* BTL_UDAPL_VERBOSE_HELP(); on by default
|
|
||||||
* - 20: general execution diagnostic information;
|
|
||||||
* may be useful to user or btl developer
|
|
||||||
* - 30: basic debugging/diagnostic information
|
|
||||||
* - 90: useful only to developers
|
|
||||||
* - 100: other components do not appear to go beyond 100 for verbose
|
|
||||||
* levels so noting here as the max for future reference
|
|
||||||
*/
|
|
||||||
#define VERBOSE_CRITICAL 0
|
|
||||||
#define VERBOSE_SHOW_HELP 10
|
|
||||||
#define VERBOSE_INFORM 20
|
|
||||||
#define VERBOSE_DIAGNOSE 30
|
|
||||||
#define VERBOSE_DEVELOPER 90
|
|
||||||
|
|
||||||
#define BTL_UDAPL_VERBOSE_OUTPUT(verbose_level, args) \
|
|
||||||
do { \
|
|
||||||
if (verbose_level <= mca_btl_udapl_component.udapl_verbosity) { \
|
|
||||||
BTL_OUTPUT(args); \
|
|
||||||
} \
|
|
||||||
} while(0);
|
|
||||||
|
|
||||||
#define BTL_UDAPL_VERBOSE_HELP(verbose_level, args) \
|
|
||||||
do { \
|
|
||||||
if (verbose_level <= mca_btl_udapl_component.udapl_verbosity) { \
|
|
||||||
opal_show_help args; \
|
|
||||||
} \
|
|
||||||
} while(0);
|
|
||||||
|
|
||||||
|
|
||||||
/*
|
|
||||||
* Report a uDAPL error - for debugging
|
|
||||||
*/
|
|
||||||
|
|
||||||
#if OPAL_ENABLE_DEBUG
|
|
||||||
extern void mca_btl_udapl_error(DAT_RETURN ret, char* str);
|
|
||||||
|
|
||||||
#define MCA_BTL_UDAPL_ERROR(ret, str) \
|
|
||||||
mca_btl_udapl_error((ret), (str));
|
|
||||||
|
|
||||||
#else
|
|
||||||
#define MCA_BTL_UDAPL_ERROR(ret, str)
|
|
||||||
#endif
|
|
||||||
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Register uDAPL component parameters with the MCA framework
|
|
||||||
*/
|
|
||||||
extern int mca_btl_udapl_component_open(void);
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Any final cleanup before being unloaded.
|
|
||||||
*/
|
|
||||||
extern int mca_btl_udapl_component_close(void);
|
|
||||||
|
|
||||||
/**
|
|
||||||
* uDAPL component initialization.
|
|
||||||
*
|
|
||||||
* @param num_btl_modules (OUT) Number of BTLs returned in BTL array.
|
|
||||||
* @param allow_multi_user_threads (OUT) Flag indicating wether BTL supports user threads (TRUE)
|
|
||||||
* @param have_hidden_threads (OUT) Flag indicating wether BTL uses threads (TRUE)
|
|
||||||
*/
|
|
||||||
extern mca_btl_base_module_t** mca_btl_udapl_component_init(
|
|
||||||
int *num_btl_modules,
|
|
||||||
bool allow_multi_user_threads,
|
|
||||||
bool have_hidden_threads
|
|
||||||
);
|
|
||||||
|
|
||||||
|
|
||||||
/**
|
|
||||||
* uDAPL component progress.
|
|
||||||
*/
|
|
||||||
|
|
||||||
extern int mca_btl_udapl_component_progress(void);
|
|
||||||
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Initialize resources for a new BTL/uDAPL IA
|
|
||||||
*
|
|
||||||
* @param ia_name Name of uDAPL interface adapter
|
|
||||||
* @param btl BTL instance.
|
|
||||||
* @return OMPI_SUCCESS or error status on failure.
|
|
||||||
*/
|
|
||||||
|
|
||||||
extern int mca_btl_udapl_init(
|
|
||||||
DAT_NAME_PTR ia_name,
|
|
||||||
struct mca_btl_udapl_module_t* btl
|
|
||||||
);
|
|
||||||
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Cleanup any resources held by the BTL.
|
|
||||||
*
|
|
||||||
* @param btl BTL instance.
|
|
||||||
* @return OMPI_SUCCESS or error status on failure.
|
|
||||||
*/
|
|
||||||
|
|
||||||
extern int mca_btl_udapl_finalize(
|
|
||||||
struct mca_btl_base_module_t* btl
|
|
||||||
);
|
|
||||||
|
|
||||||
|
|
||||||
/**
|
|
||||||
* PML->BTL notification of change in the process list.
|
|
||||||
*
|
|
||||||
* @param btl (IN)
|
|
||||||
* @param nprocs (IN) Number of processes
|
|
||||||
* @param procs (IN) Set of processes
|
|
||||||
* @param peers (OUT) Set of (optional) peer addressing info.
|
|
||||||
* @param peers (IN/OUT) Set of processes that are reachable via this BTL.
|
|
||||||
* @return OMPI_SUCCESS or error status on failure.
|
|
||||||
*
|
|
||||||
*/
|
|
||||||
|
|
||||||
extern int mca_btl_udapl_add_procs(
|
|
||||||
struct mca_btl_base_module_t* btl,
|
|
||||||
size_t nprocs,
|
|
||||||
struct ompi_proc_t **procs,
|
|
||||||
struct mca_btl_base_endpoint_t** peers,
|
|
||||||
opal_bitmap_t* reachable
|
|
||||||
);
|
|
||||||
|
|
||||||
/**
|
|
||||||
* PML->BTL notification of change in the process list.
|
|
||||||
*
|
|
||||||
* @param btl (IN) BTL instance
|
|
||||||
* @param nproc (IN) Number of processes.
|
|
||||||
* @param procs (IN) Set of processes.
|
|
||||||
* @param peers (IN) Set of peer data structures.
|
|
||||||
* @return Status indicating if cleanup was successful
|
|
||||||
*
|
|
||||||
*/
|
|
||||||
|
|
||||||
extern int mca_btl_udapl_del_procs(
|
|
||||||
struct mca_btl_base_module_t* btl,
|
|
||||||
size_t nprocs,
|
|
||||||
struct ompi_proc_t **procs,
|
|
||||||
struct mca_btl_base_endpoint_t** peers
|
|
||||||
);
|
|
||||||
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Initiate an asynchronous send.
|
|
||||||
*
|
|
||||||
* @param btl (IN) BTL module
|
|
||||||
* @param endpoint (IN) BTL addressing information
|
|
||||||
* @param descriptor (IN) Description of the data to be transferred
|
|
||||||
* @param tag (IN) The tag value used to notify the peer.
|
|
||||||
*/
|
|
||||||
|
|
||||||
extern int mca_btl_udapl_send(
|
|
||||||
struct mca_btl_base_module_t* btl,
|
|
||||||
struct mca_btl_base_endpoint_t* btl_peer,
|
|
||||||
struct mca_btl_base_descriptor_t* descriptor,
|
|
||||||
mca_btl_base_tag_t tag
|
|
||||||
);
|
|
||||||
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Initiate an asynchronous put.
|
|
||||||
*
|
|
||||||
* @param btl (IN) BTL module
|
|
||||||
* @param endpoint (IN) BTL addressing information
|
|
||||||
* @param descriptor (IN) Description of the data to be transferred
|
|
||||||
*/
|
|
||||||
|
|
||||||
extern int mca_btl_udapl_put(
|
|
||||||
struct mca_btl_base_module_t* btl,
|
|
||||||
struct mca_btl_base_endpoint_t* btl_peer,
|
|
||||||
struct mca_btl_base_descriptor_t* decriptor
|
|
||||||
);
|
|
||||||
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Initiate an asynchronous get.
|
|
||||||
*
|
|
||||||
* @param btl (IN) BTL module
|
|
||||||
* @param endpoint (IN) BTL addressing information
|
|
||||||
* @param descriptor (IN) Description of the data to be transferred
|
|
||||||
*/
|
|
||||||
|
|
||||||
extern int mca_btl_udapl_get(
|
|
||||||
struct mca_btl_base_module_t* btl,
|
|
||||||
struct mca_btl_base_endpoint_t* btl_peer,
|
|
||||||
struct mca_btl_base_descriptor_t* decriptor
|
|
||||||
);
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Allocate a descriptor with a segment of the requested size.
|
|
||||||
* Note that the BTL layer may choose to return a smaller size
|
|
||||||
* if it cannot support the request.
|
|
||||||
*
|
|
||||||
* @param btl (IN) BTL module
|
|
||||||
* @param size (IN) Request segment size.
|
|
||||||
*/
|
|
||||||
|
|
||||||
extern mca_btl_base_descriptor_t* mca_btl_udapl_alloc(
|
|
||||||
struct mca_btl_base_module_t* btl,
|
|
||||||
struct mca_btl_base_endpoint_t* endpoint,
|
|
||||||
uint8_t order,
|
|
||||||
size_t size,
|
|
||||||
uint32_t flags);
|
|
||||||
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Return a segment allocated by this BTL.
|
|
||||||
*
|
|
||||||
* @param btl (IN) BTL module
|
|
||||||
* @param descriptor (IN) Allocated descriptor.
|
|
||||||
*/
|
|
||||||
|
|
||||||
extern int mca_btl_udapl_free(
|
|
||||||
struct mca_btl_base_module_t* btl,
|
|
||||||
mca_btl_base_descriptor_t* des);
|
|
||||||
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Prepare a descriptor for send/rdma using the supplied
|
|
||||||
* convertor. If the convertor references data that is contigous,
|
|
||||||
* the descriptor may simply point to the user buffer. Otherwise,
|
|
||||||
* this routine is responsible for allocating buffer space and
|
|
||||||
* packing if required.
|
|
||||||
*
|
|
||||||
* @param btl (IN) BTL module
|
|
||||||
* @param endpoint (IN) BTL peer addressing
|
|
||||||
* @param convertor (IN) Data type convertor
|
|
||||||
* @param reserve (IN) Additional bytes requested by upper layer to precede user data
|
|
||||||
* @param size (IN/OUT) Number of bytes to prepare (IN), number of bytes actually prepared (OUT)
|
|
||||||
*/
|
|
||||||
|
|
||||||
mca_btl_base_descriptor_t* mca_btl_udapl_prepare_src(
|
|
||||||
struct mca_btl_base_module_t* btl,
|
|
||||||
struct mca_btl_base_endpoint_t* peer,
|
|
||||||
struct mca_mpool_base_registration_t*,
|
|
||||||
struct opal_convertor_t* convertor,
|
|
||||||
uint8_t order,
|
|
||||||
size_t reserve,
|
|
||||||
size_t* size,
|
|
||||||
uint32_t flags
|
|
||||||
);
|
|
||||||
|
|
||||||
extern mca_btl_base_descriptor_t* mca_btl_udapl_prepare_dst(
|
|
||||||
struct mca_btl_base_module_t* btl,
|
|
||||||
struct mca_btl_base_endpoint_t* peer,
|
|
||||||
struct mca_mpool_base_registration_t*,
|
|
||||||
struct opal_convertor_t* convertor,
|
|
||||||
uint8_t order,
|
|
||||||
size_t reserve,
|
|
||||||
size_t* size,
|
|
||||||
uint32_t flags);
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Fault Tolerance Event Notification Function
|
|
||||||
* @param state Checkpoint Stae
|
|
||||||
* @return OMPI_SUCCESS or failure status
|
|
||||||
*/
|
|
||||||
int mca_btl_udapl_ft_event(int state);
|
|
||||||
|
|
||||||
END_C_DECLS
|
|
||||||
#endif
|
|
||||||
Разница между файлами не показана из-за своего большого размера
Загрузить разницу
@ -1,105 +0,0 @@
|
|||||||
/*
|
|
||||||
* Copyright (c) 2006 Sun Microsystems, Inc. All rights reserved.
|
|
||||||
*
|
|
||||||
* $COPYRIGHT$
|
|
||||||
*
|
|
||||||
* Additional copyrights may follow
|
|
||||||
*
|
|
||||||
* $HEADER$
|
|
||||||
*/
|
|
||||||
|
|
||||||
#ifndef MCA_BTL_UDAPL_EAGER_RDMA_H
|
|
||||||
#define MCA_BTL_UDAPL_EAGER_RDMA_H
|
|
||||||
|
|
||||||
/* Open MPI includes */
|
|
||||||
#include "ompi/mca/btl/udapl/btl_udapl_endpoint.h"
|
|
||||||
|
|
||||||
|
|
||||||
BEGIN_C_DECLS
|
|
||||||
|
|
||||||
/*
|
|
||||||
* Describe endpoint local memory region.
|
|
||||||
*/
|
|
||||||
struct mca_btl_udapl_eager_rdma_local_t {
|
|
||||||
ompi_ptr_t base; /**< points to fragment structures */
|
|
||||||
struct mca_btl_udapl_reg_t* reg;
|
|
||||||
uint8_t head; /**< RDMA buffer to poll */
|
|
||||||
int32_t credits; /**< number of local rdma buffers ready to be reclaimed,
|
|
||||||
reused. Initially equal to 0. */
|
|
||||||
opal_mutex_t lock; /**< protect access to RDMA buffer */
|
|
||||||
};
|
|
||||||
typedef struct mca_btl_udapl_eager_rdma_local_t mca_btl_udapl_eager_rdma_local_t;
|
|
||||||
|
|
||||||
/*
|
|
||||||
* Describe endpoint remote memory region.
|
|
||||||
*/
|
|
||||||
struct mca_btl_udapl_eager_rdma_remote_t {
|
|
||||||
ompi_ptr_t base; /**< points to start of data region, not
|
|
||||||
fragment structures */
|
|
||||||
DAT_RMR_CONTEXT rkey; /**< key required to access remote memory */
|
|
||||||
uint8_t head; /**< RDMA buffer to use */
|
|
||||||
int32_t tokens; /**< number of available rdma buffers, initially equal
|
|
||||||
to mca parameter eager_rdma_num */
|
|
||||||
opal_mutex_t lock; /**< protect access to RDMA buffer */
|
|
||||||
};
|
|
||||||
typedef struct mca_btl_udapl_eager_rdma_remote_t mca_btl_udapl_eager_rdma_remote_t;
|
|
||||||
|
|
||||||
/*
|
|
||||||
* Encapsulate data that describes a remote memory region.
|
|
||||||
*/
|
|
||||||
struct mca_btl_udapl_eager_rdma_connect_t {
|
|
||||||
mca_btl_udapl_control_header_t control;
|
|
||||||
uint32_t rkey;
|
|
||||||
ompi_ptr_t rdma_start;
|
|
||||||
};
|
|
||||||
typedef struct mca_btl_udapl_eager_rdma_connect_t mca_btl_udapl_eager_rdma_connect_t;
|
|
||||||
|
|
||||||
/*
|
|
||||||
* Encapsulate data that describes rdma credit information.
|
|
||||||
*/
|
|
||||||
struct mca_btl_udapl_eager_rdma_credit_t {
|
|
||||||
mca_btl_udapl_control_header_t control;
|
|
||||||
uint32_t credits;
|
|
||||||
};
|
|
||||||
typedef struct mca_btl_udapl_eager_rdma_credit_t mca_btl_udapl_eager_rdma_credit_t;
|
|
||||||
|
|
||||||
#define EAGER_RDMA_BUFFER_AVAILABLE (0)
|
|
||||||
#define EAGER_RDMA_BUFFER_IN_USE (0xff)
|
|
||||||
|
|
||||||
#define MCA_BTL_UDAPL_RDMA_FRAG_IN_USE(F) do { \
|
|
||||||
*(volatile uint8_t*) ((char*)(F) + \
|
|
||||||
(mca_btl_udapl_component.udapl_eager_rdma_frag_size - \
|
|
||||||
(sizeof(mca_btl_udapl_footer_t)))); \
|
|
||||||
} while (0)
|
|
||||||
|
|
||||||
#define MCA_BTL_UDAPL_RDMA_FRAG_ASSIGN_IN_USE(F) do { \
|
|
||||||
*(volatile uint8_t*) ((char*)(F) + \
|
|
||||||
(mca_btl_udapl_component.udapl_eager_rdma_frag_size- \
|
|
||||||
(sizeof(mca_btl_udapl_footer_t)))) = EAGER_RDMA_BUFFER_IN_USE; \
|
|
||||||
} while (0)
|
|
||||||
|
|
||||||
#define MCA_BTL_UDAPL_RDMA_FRAG_ASSIGN_AVAILABLE(F) do { \
|
|
||||||
*(volatile uint8_t*) ((char*)(F) + \
|
|
||||||
(mca_btl_udapl_component.udapl_eager_rdma_frag_size - \
|
|
||||||
(sizeof(mca_btl_udapl_footer_t)))) = EAGER_RDMA_BUFFER_AVAILABLE; \
|
|
||||||
} while (0)
|
|
||||||
|
|
||||||
/* Retrieve the rdma fragment at location I */
|
|
||||||
#define MCA_BTL_UDAPL_GET_LOCAL_RDMA_FRAG(E, I) \
|
|
||||||
(mca_btl_udapl_frag_t*) \
|
|
||||||
((char*)(E)->endpoint_eager_rdma_local.base.pval + \
|
|
||||||
(I) * sizeof(mca_btl_udapl_frag_eager_rdma_t))
|
|
||||||
|
|
||||||
/*
|
|
||||||
* Increment the index I by one while not exceeding the total number of
|
|
||||||
* available eager rdma fragments
|
|
||||||
*/
|
|
||||||
#define MCA_BTL_UDAPL_RDMA_NEXT_INDEX(I) do { \
|
|
||||||
(I) = ((I) + 1); \
|
|
||||||
if((I) == \
|
|
||||||
mca_btl_udapl_component.udapl_eager_rdma_num) \
|
|
||||||
(I) = 0; \
|
|
||||||
} while (0)
|
|
||||||
|
|
||||||
END_C_DECLS
|
|
||||||
#endif
|
|
||||||
Разница между файлами не показана из-за своего большого размера
Загрузить разницу
@ -1,218 +0,0 @@
|
|||||||
/*
|
|
||||||
* Copyright (c) 2004-2006 The Trustees of Indiana University and Indiana
|
|
||||||
* University Research and Technology
|
|
||||||
* Corporation. All rights reserved.
|
|
||||||
* Copyright (c) 2004-2005 The University of Tennessee and The University
|
|
||||||
* of Tennessee Research Foundation. All rights
|
|
||||||
* reserved.
|
|
||||||
* Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
|
|
||||||
* University of Stuttgart. All rights reserved.
|
|
||||||
* Copyright (c) 2004-2005 The Regents of the University of California.
|
|
||||||
* All rights reserved.
|
|
||||||
* Copyright (c) 2006 Sandia National Laboratories. All rights
|
|
||||||
* reserved.
|
|
||||||
* Copyright (c) 2006-2009 Sun Microsystems, Inc. All rights reserved.
|
|
||||||
*
|
|
||||||
* $COPYRIGHT$
|
|
||||||
*
|
|
||||||
* Additional copyrights may follow
|
|
||||||
*
|
|
||||||
* $HEADER$
|
|
||||||
*/
|
|
||||||
|
|
||||||
#ifndef MCA_BTL_UDAPL_ENDPOINT_H
|
|
||||||
#define MCA_BTL_UDAPL_ENDPOINT_H
|
|
||||||
|
|
||||||
#include "opal/class/opal_list.h"
|
|
||||||
#include "opal/mca/event/event.h"
|
|
||||||
#include "ompi/mca/btl/base/btl_base_error.h"
|
|
||||||
#include "btl_udapl_frag.h"
|
|
||||||
#include "btl_udapl.h"
|
|
||||||
#include "btl_udapl_eager_rdma.h"
|
|
||||||
|
|
||||||
BEGIN_C_DECLS
|
|
||||||
|
|
||||||
|
|
||||||
#define BTL_UDAPL_TOKEN_AVAIL(E, C, T) \
|
|
||||||
do { \
|
|
||||||
(T) = 0; \
|
|
||||||
if ( (E)->endpoint_lwqe_tokens[(C)] > 0 && \
|
|
||||||
((E)->endpoint_sr_tokens[(C)] + \
|
|
||||||
(((C) == BTL_UDAPL_EAGER_CONNECTION)?(E)->endpoint_eager_rdma_remote.tokens:0)) \
|
|
||||||
) { \
|
|
||||||
(T) = 1; \
|
|
||||||
} \
|
|
||||||
} while (0)
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Structure used to publish uDAPL id information to peers.
|
|
||||||
*/
|
|
||||||
struct mca_btl_udapl_addr_t {
|
|
||||||
DAT_CONN_QUAL port;
|
|
||||||
DAT_SOCK_ADDR addr;
|
|
||||||
bool inuse;
|
|
||||||
};
|
|
||||||
typedef struct mca_btl_udapl_addr_t mca_btl_udapl_addr_t;
|
|
||||||
|
|
||||||
|
|
||||||
/**
|
|
||||||
* State of uDAPL endpoint connection.
|
|
||||||
*/
|
|
||||||
|
|
||||||
typedef enum {
|
|
||||||
MCA_BTL_UDAPL_CONN_EAGER,
|
|
||||||
MCA_BTL_UDAPL_CONN_MAX,
|
|
||||||
MCA_BTL_UDAPL_CONNECTED,
|
|
||||||
MCA_BTL_UDAPL_CLOSED,
|
|
||||||
MCA_BTL_UDAPL_FAILED
|
|
||||||
} mca_btl_udapl_endpoint_state_t;
|
|
||||||
|
|
||||||
/*
|
|
||||||
* Establish a name for the 2 connections opened per peer
|
|
||||||
*/
|
|
||||||
typedef enum {
|
|
||||||
BTL_UDAPL_EAGER_CONNECTION,
|
|
||||||
BTL_UDAPL_MAX_CONNECTION,
|
|
||||||
BTL_UDAPL_NUM_CONNECTION
|
|
||||||
} mca_btl_udapl_endpoint_conn_t;
|
|
||||||
|
|
||||||
/*
|
|
||||||
* Encapsulate data that describes sendrecv credit information.
|
|
||||||
*/
|
|
||||||
struct mca_btl_udapl_sr_credit_t {
|
|
||||||
mca_btl_udapl_control_header_t control;
|
|
||||||
uint32_t credits;
|
|
||||||
int connection; /* 0 == BTL_UDAPL_EAGER_CONNECTION;
|
|
||||||
1 == BTL_UDAPL_MAX_CONNECTION */
|
|
||||||
};
|
|
||||||
typedef struct mca_btl_udapl_sr_credit_t mca_btl_udapl_sr_credit_t;
|
|
||||||
|
|
||||||
/**
|
|
||||||
* An abstraction that represents a connection to a endpoint process.
|
|
||||||
* An instance of mca_btl_base_endpoint_t is associated w/ each process
|
|
||||||
* and BTL pair at startup. However, connections to the endpoint
|
|
||||||
* are established dynamically on an as-needed basis:
|
|
||||||
*/
|
|
||||||
|
|
||||||
struct mca_btl_base_endpoint_t {
|
|
||||||
opal_list_item_t super;
|
|
||||||
|
|
||||||
struct mca_btl_udapl_module_t* endpoint_btl;
|
|
||||||
/**< BTL instance that created this connection */
|
|
||||||
|
|
||||||
struct mca_btl_udapl_proc_t* endpoint_proc;
|
|
||||||
/**< proc structure corresponding to endpoint */
|
|
||||||
|
|
||||||
mca_btl_udapl_endpoint_state_t endpoint_state;
|
|
||||||
/**< current state of the endpoint connection */
|
|
||||||
|
|
||||||
opal_list_t endpoint_eager_frags;
|
|
||||||
opal_list_t endpoint_max_frags;
|
|
||||||
/**< pending send frags on this endpoint */
|
|
||||||
|
|
||||||
int32_t endpoint_eager_sends;
|
|
||||||
int32_t endpoint_max_sends;
|
|
||||||
/**< number of sends that may be posted */
|
|
||||||
|
|
||||||
int32_t endpoint_sr_tokens[BTL_UDAPL_NUM_CONNECTION];
|
|
||||||
/**< number of sends that may be posted */
|
|
||||||
|
|
||||||
int32_t endpoint_sr_credits[BTL_UDAPL_NUM_CONNECTION];
|
|
||||||
/**< number of recvs that are now available */
|
|
||||||
|
|
||||||
int32_t endpoint_lwqe_tokens[BTL_UDAPL_NUM_CONNECTION];
|
|
||||||
/**< number of local work queue credits available (combination of
|
|
||||||
posted sends and rdma writes allowed per endpoint */
|
|
||||||
|
|
||||||
int32_t endpoint_connection_seq;
|
|
||||||
/**< sequence number of sendrecv message for the connection est */
|
|
||||||
|
|
||||||
int32_t endpoint_connections_completed;
|
|
||||||
/**< count of completed connections for priv data connection est. */
|
|
||||||
|
|
||||||
opal_mutex_t endpoint_lock;
|
|
||||||
/**< lock for concurrent access to endpoint state */
|
|
||||||
|
|
||||||
mca_btl_udapl_addr_t endpoint_addr;
|
|
||||||
/**< remote address on the other side of this endpoint */
|
|
||||||
|
|
||||||
DAT_EP_HANDLE endpoint_eager;
|
|
||||||
DAT_EP_HANDLE endpoint_max;
|
|
||||||
/**< uDAPL endpoint handle */
|
|
||||||
|
|
||||||
int32_t endpoint_eager_rdma_index;
|
|
||||||
/**< index into array of endpoints with RDMA buffers */
|
|
||||||
mca_btl_udapl_eager_rdma_local_t endpoint_eager_rdma_local;
|
|
||||||
/**< info about local RDMA buffer */
|
|
||||||
mca_btl_udapl_eager_rdma_remote_t endpoint_eager_rdma_remote;
|
|
||||||
/**< info about remote RDMA buffer */
|
|
||||||
};
|
|
||||||
|
|
||||||
typedef struct mca_btl_base_endpoint_t mca_btl_base_endpoint_t;
|
|
||||||
typedef mca_btl_base_endpoint_t mca_btl_udapl_endpoint_t;
|
|
||||||
|
|
||||||
OBJ_CLASS_DECLARATION(mca_btl_udapl_endpoint_t);
|
|
||||||
|
|
||||||
|
|
||||||
/*
|
|
||||||
* Start sending data on an endpoint.
|
|
||||||
*/
|
|
||||||
|
|
||||||
int mca_btl_udapl_endpoint_send(mca_btl_base_endpoint_t* endpoint,
|
|
||||||
mca_btl_udapl_frag_t* frag);
|
|
||||||
|
|
||||||
/*
|
|
||||||
* Set up OOB recv callback.
|
|
||||||
*/
|
|
||||||
|
|
||||||
void mca_btl_udapl_endpoint_post_oob_recv(void);
|
|
||||||
|
|
||||||
/*
|
|
||||||
* Finish establishing a connection
|
|
||||||
*/
|
|
||||||
|
|
||||||
int mca_btl_udapl_endpoint_finish_connect(struct mca_btl_udapl_module_t* btl,
|
|
||||||
mca_btl_udapl_addr_t* addr,
|
|
||||||
int32_t* seq,
|
|
||||||
DAT_EP_HANDLE endpoint);
|
|
||||||
|
|
||||||
/*
|
|
||||||
* Send number of eager rdma credits
|
|
||||||
*/
|
|
||||||
int mca_btl_udapl_endpoint_send_eager_rdma_credits(mca_btl_base_endpoint_t* endpoint);
|
|
||||||
|
|
||||||
/*
|
|
||||||
* Establish uDAPL endpoint parameters
|
|
||||||
*/
|
|
||||||
int mca_btl_udapl_endpoint_get_params(struct mca_btl_udapl_module_t* btl,
|
|
||||||
DAT_EP_PARAM* ep_param);
|
|
||||||
|
|
||||||
/*
|
|
||||||
* Create uDAPL endpoint
|
|
||||||
*/
|
|
||||||
int mca_btl_udapl_endpoint_create(struct mca_btl_udapl_module_t* btl,
|
|
||||||
DAT_EP_HANDLE* udapl_endpoint);
|
|
||||||
|
|
||||||
/*
|
|
||||||
* Send number of send recv credits
|
|
||||||
*/
|
|
||||||
int mca_btl_udapl_endpoint_send_sr_credits(mca_btl_base_endpoint_t* endpoint,
|
|
||||||
const int connection);
|
|
||||||
|
|
||||||
/*
|
|
||||||
* Handle the established DAT endpoint when private data is in use
|
|
||||||
*/
|
|
||||||
int mca_btl_udapl_endpoint_pd_established_conn(
|
|
||||||
struct mca_btl_udapl_module_t* btl,
|
|
||||||
DAT_EP_HANDLE established_ep);
|
|
||||||
|
|
||||||
/*
|
|
||||||
* Utility routine. Search list of endpoints to find one that matches
|
|
||||||
* the given address.
|
|
||||||
*/
|
|
||||||
mca_btl_udapl_endpoint_t* mca_btl_udapl_find_endpoint_address_match(
|
|
||||||
struct mca_btl_udapl_module_t* btl,
|
|
||||||
mca_btl_udapl_addr_t addr);
|
|
||||||
|
|
||||||
END_C_DECLS
|
|
||||||
#endif
|
|
||||||
@ -1,134 +0,0 @@
|
|||||||
/*
|
|
||||||
* Copyright (c) 2004-2006 The Trustees of Indiana University and Indiana
|
|
||||||
* University Research and Technology
|
|
||||||
* Corporation. All rights reserved.
|
|
||||||
* Copyright (c) 2004-2005 The University of Tennessee and The University
|
|
||||||
* of Tennessee Research Foundation. All rights
|
|
||||||
* reserved.
|
|
||||||
* Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
|
|
||||||
* University of Stuttgart. All rights reserved.
|
|
||||||
* Copyright (c) 2004-2005 The Regents of the University of California.
|
|
||||||
* All rights reserved.
|
|
||||||
* Copyright (c) 2006 Sandia National Laboratories. All rights
|
|
||||||
* reserved.
|
|
||||||
* Copyright (c) 2006 Sun Microsystems, Inc. All rights reserved.
|
|
||||||
* $COPYRIGHT$
|
|
||||||
*
|
|
||||||
* Additional copyrights may follow
|
|
||||||
*
|
|
||||||
* $HEADER$
|
|
||||||
*/
|
|
||||||
|
|
||||||
#include "btl_udapl.h"
|
|
||||||
#include "btl_udapl_frag.h"
|
|
||||||
|
|
||||||
|
|
||||||
static void mca_btl_udapl_frag_common_constructor(mca_btl_udapl_frag_t* frag)
|
|
||||||
{
|
|
||||||
mca_btl_udapl_reg_t* reg =
|
|
||||||
(mca_btl_udapl_reg_t*)frag->base.super.registration;
|
|
||||||
|
|
||||||
#if OPAL_ENABLE_DEBUG
|
|
||||||
frag->base.des_src = NULL;
|
|
||||||
frag->base.des_src_cnt = 0;
|
|
||||||
frag->base.des_dst = NULL;
|
|
||||||
frag->base.des_dst_cnt = 0;
|
|
||||||
frag->base.des_flags = 0;
|
|
||||||
#endif
|
|
||||||
|
|
||||||
frag->registration = reg;
|
|
||||||
frag->segment.base.seg_addr.pval = (unsigned char*)frag->base.super.ptr;
|
|
||||||
frag->ftr = NULL;
|
|
||||||
|
|
||||||
/* Don't understand why yet, but there are cases where reg is NULL -
|
|
||||||
that is, this memory has not been registered. So be careful not
|
|
||||||
to dereference a NULL pointer. */
|
|
||||||
if(NULL != reg) {
|
|
||||||
/* Save the LMR context so we can set up LMR subset triplets later */
|
|
||||||
frag->triplet.lmr_context = reg->lmr_triplet.lmr_context;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
static void mca_btl_udapl_frag_eager_constructor(mca_btl_udapl_frag_t* frag)
|
|
||||||
{
|
|
||||||
frag->segment.base.seg_len = mca_btl_udapl_module.super.btl_eager_limit;
|
|
||||||
frag->size = mca_btl_udapl_component.udapl_eager_frag_size;
|
|
||||||
mca_btl_udapl_frag_common_constructor(frag);
|
|
||||||
}
|
|
||||||
|
|
||||||
static void mca_btl_udapl_frag_max_constructor(mca_btl_udapl_frag_t* frag)
|
|
||||||
{
|
|
||||||
frag->segment.base.seg_len = mca_btl_udapl_module.super.btl_max_send_size;
|
|
||||||
frag->size = mca_btl_udapl_component.udapl_max_frag_size;
|
|
||||||
mca_btl_udapl_frag_common_constructor(frag);
|
|
||||||
}
|
|
||||||
|
|
||||||
static void mca_btl_udapl_frag_user_constructor(mca_btl_udapl_frag_t* frag)
|
|
||||||
{
|
|
||||||
mca_btl_udapl_frag_common_constructor(frag);
|
|
||||||
frag->segment.base.seg_len = 0;
|
|
||||||
frag->segment.base.seg_addr.pval = NULL;
|
|
||||||
frag->ftr = NULL;
|
|
||||||
frag->size = 0;
|
|
||||||
frag->registration = NULL;
|
|
||||||
}
|
|
||||||
|
|
||||||
static void mca_btl_udapl_frag_eager_rdma_constructor(mca_btl_udapl_frag_t* frag)
|
|
||||||
{
|
|
||||||
mca_btl_udapl_frag_eager_constructor(frag);
|
|
||||||
frag->segment.base.seg_len = mca_btl_udapl_module.super.btl_eager_limit;
|
|
||||||
frag->size = mca_btl_udapl_component.udapl_eager_frag_size;
|
|
||||||
frag->rdma_ftr = (mca_btl_udapl_rdma_footer_t *)
|
|
||||||
((char *)(frag->segment.base.seg_addr.pval) +
|
|
||||||
frag->size -
|
|
||||||
sizeof(mca_btl_udapl_rdma_footer_t));
|
|
||||||
frag->rdma_ftr->active=0;
|
|
||||||
}
|
|
||||||
|
|
||||||
static void mca_btl_udapl_frag_common_destructor(mca_btl_udapl_frag_t* frag)
|
|
||||||
{
|
|
||||||
#if OPAL_ENABLE_DEBUG
|
|
||||||
frag->ftr = NULL;
|
|
||||||
frag->size = 0;
|
|
||||||
frag->registration = NULL;
|
|
||||||
frag->segment.base.seg_len = 0;
|
|
||||||
frag->segment.base.seg_addr.pval = NULL;
|
|
||||||
|
|
||||||
frag->base.des_src = NULL;
|
|
||||||
frag->base.des_src_cnt = 0;
|
|
||||||
frag->base.des_dst = NULL;
|
|
||||||
frag->base.des_dst_cnt = 0;
|
|
||||||
frag->base.des_flags = 0;
|
|
||||||
#endif
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
OBJ_CLASS_INSTANCE(
|
|
||||||
mca_btl_udapl_frag_t,
|
|
||||||
mca_btl_base_descriptor_t,
|
|
||||||
NULL,
|
|
||||||
NULL);
|
|
||||||
|
|
||||||
OBJ_CLASS_INSTANCE(
|
|
||||||
mca_btl_udapl_frag_eager_t,
|
|
||||||
mca_btl_base_descriptor_t,
|
|
||||||
mca_btl_udapl_frag_eager_constructor,
|
|
||||||
mca_btl_udapl_frag_common_destructor);
|
|
||||||
|
|
||||||
OBJ_CLASS_INSTANCE(
|
|
||||||
mca_btl_udapl_frag_max_t,
|
|
||||||
mca_btl_base_descriptor_t,
|
|
||||||
mca_btl_udapl_frag_max_constructor,
|
|
||||||
mca_btl_udapl_frag_common_destructor);
|
|
||||||
|
|
||||||
OBJ_CLASS_INSTANCE(
|
|
||||||
mca_btl_udapl_frag_user_t,
|
|
||||||
mca_btl_base_descriptor_t,
|
|
||||||
mca_btl_udapl_frag_user_constructor,
|
|
||||||
NULL);
|
|
||||||
|
|
||||||
OBJ_CLASS_INSTANCE(
|
|
||||||
mca_btl_udapl_frag_eager_rdma_t,
|
|
||||||
mca_btl_base_descriptor_t,
|
|
||||||
mca_btl_udapl_frag_eager_rdma_constructor,
|
|
||||||
mca_btl_udapl_frag_common_destructor);
|
|
||||||
@ -1,203 +0,0 @@
|
|||||||
/*
|
|
||||||
* Copyright (c) 2004-2006 The Trustees of Indiana University and Indiana
|
|
||||||
* University Research and Technology
|
|
||||||
* Corporation. All rights reserved.
|
|
||||||
* Copyright (c) 2004-2013 The University of Tennessee and The University
|
|
||||||
* of Tennessee Research Foundation. All rights
|
|
||||||
* reserved.
|
|
||||||
* Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
|
|
||||||
* University of Stuttgart. All rights reserved.
|
|
||||||
* Copyright (c) 2004-2005 The Regents of the University of California.
|
|
||||||
* All rights reserved.
|
|
||||||
* Copyright (c) 2006 Sandia National Laboratories. All rights
|
|
||||||
* reserved.
|
|
||||||
* Copyright (c) 2007-2009 Sun Microsystems, Inc. All rights reserved.
|
|
||||||
*
|
|
||||||
* $COPYRIGHT$
|
|
||||||
*
|
|
||||||
* Additional copyrights may follow
|
|
||||||
*
|
|
||||||
* $HEADER$
|
|
||||||
*/
|
|
||||||
|
|
||||||
#ifndef MCA_BTL_UDAPL_FRAG_H
|
|
||||||
#define MCA_BTL_UDAPL_FRAG_H
|
|
||||||
|
|
||||||
|
|
||||||
#define MCA_BTL_UDAPL_FRAG_ALIGN (8)
|
|
||||||
#include "ompi_config.h"
|
|
||||||
|
|
||||||
BEGIN_C_DECLS
|
|
||||||
|
|
||||||
typedef enum {
|
|
||||||
MCA_BTL_UDAPL_SEND,
|
|
||||||
MCA_BTL_UDAPL_RECV,
|
|
||||||
MCA_BTL_UDAPL_PUT,
|
|
||||||
MCA_BTL_UDAPL_GET,
|
|
||||||
MCA_BTL_UDAPL_CONN_RECV,
|
|
||||||
MCA_BTL_UDAPL_CONN_SEND,
|
|
||||||
MCA_BTL_UDAPL_RDMA_WRITE,
|
|
||||||
MCA_BTL_UDAPL_FRAG_EAGER_RDMA,
|
|
||||||
MCA_BTL_UDAPL_IGNORE
|
|
||||||
} mca_btl_udapl_frag_type_t;
|
|
||||||
|
|
||||||
typedef enum {
|
|
||||||
MCA_BTL_UDAPL_CONTROL_NOOP,
|
|
||||||
MCA_BTL_UDAPL_CONTROL_RDMA_CONNECT,
|
|
||||||
MCA_BTL_UDAPL_CONTROL_RDMA_CREDIT,
|
|
||||||
MCA_BTL_UDAPL_CONTROL_SR_CREDIT
|
|
||||||
} mca_btl_udapl_control_t;
|
|
||||||
|
|
||||||
/* Control message header */
|
|
||||||
struct mca_btl_udapl_control_header_t {
|
|
||||||
mca_btl_udapl_control_t type;
|
|
||||||
};
|
|
||||||
typedef struct mca_btl_udapl_control_header_t mca_btl_udapl_control_header_t;
|
|
||||||
|
|
||||||
/**
|
|
||||||
* uDAPL btl footer.
|
|
||||||
* This is put after the payload packet so the PML header can be aligned.
|
|
||||||
* Must be aligned on MCA_BTL_UDAPL_FRAG_ALIGN byte boundary.
|
|
||||||
*/
|
|
||||||
struct mca_btl_udapl_footer_t {
|
|
||||||
mca_btl_base_tag_t tag;
|
|
||||||
};
|
|
||||||
typedef struct mca_btl_udapl_footer_t mca_btl_udapl_footer_t;
|
|
||||||
|
|
||||||
/**
|
|
||||||
* uDAPL BTL rdma footer.
|
|
||||||
* This is used in addtion to the uDAPL BTL footer. The two are seperate to
|
|
||||||
* allow for any padding that may be required between the two.
|
|
||||||
*/
|
|
||||||
struct mca_btl_udapl_rdma_footer_t {
|
|
||||||
uint32_t size;
|
|
||||||
volatile uint8_t active;/* 0 = not in use; 1 = data is available to be
|
|
||||||
* received; this should always be the last entry
|
|
||||||
* in this structure
|
|
||||||
*/
|
|
||||||
char pad[3]; /* pad out be aligned on MCA_BTL_UDAPL_FRAG_ALIGN byte boundary */
|
|
||||||
};
|
|
||||||
typedef struct mca_btl_udapl_rdma_footer_t mca_btl_udapl_rdma_footer_t;
|
|
||||||
|
|
||||||
struct mca_btl_udapl_segment_t {
|
|
||||||
mca_btl_base_segment_t base;
|
|
||||||
DAT_RMR_CONTEXT context;
|
|
||||||
};
|
|
||||||
typedef struct mca_btl_udapl_segment_t mca_btl_udapl_segment_t;
|
|
||||||
|
|
||||||
/**
|
|
||||||
* uDAPL fragment derived type.
|
|
||||||
*/
|
|
||||||
struct mca_btl_udapl_frag_t {
|
|
||||||
mca_btl_base_descriptor_t base;
|
|
||||||
mca_btl_udapl_segment_t segment;
|
|
||||||
|
|
||||||
struct mca_btl_udapl_module_t* btl;
|
|
||||||
struct mca_btl_base_endpoint_t* endpoint;
|
|
||||||
DAT_LMR_TRIPLET triplet;
|
|
||||||
struct mca_btl_udapl_reg_t* registration;
|
|
||||||
|
|
||||||
mca_btl_udapl_footer_t* ftr;
|
|
||||||
mca_btl_udapl_rdma_footer_t* rdma_ftr;
|
|
||||||
size_t size;
|
|
||||||
mca_btl_udapl_frag_type_t type;
|
|
||||||
uint32_t pad; /* Padding the structure to be evenly divisble by MCA_BTL_UDAPL_FRAG_ALIGN */
|
|
||||||
};
|
|
||||||
typedef struct mca_btl_udapl_frag_t mca_btl_udapl_frag_t;
|
|
||||||
OBJ_CLASS_DECLARATION(mca_btl_udapl_frag_t);
|
|
||||||
|
|
||||||
|
|
||||||
typedef struct mca_btl_udapl_frag_t mca_btl_udapl_frag_eager_t;
|
|
||||||
OBJ_CLASS_DECLARATION(mca_btl_udapl_frag_eager_t);
|
|
||||||
|
|
||||||
typedef struct mca_btl_udapl_frag_t mca_btl_udapl_frag_max_t;
|
|
||||||
OBJ_CLASS_DECLARATION(mca_btl_udapl_frag_max_t);
|
|
||||||
|
|
||||||
typedef struct mca_btl_udapl_frag_t mca_btl_udapl_frag_user_t;
|
|
||||||
OBJ_CLASS_DECLARATION(mca_btl_udapl_frag_user_t);
|
|
||||||
|
|
||||||
typedef struct mca_btl_udapl_frag_t mca_btl_udapl_frag_eager_rdma_t;
|
|
||||||
OBJ_CLASS_DECLARATION(mca_btl_udapl_frag_eager_rdma_t);
|
|
||||||
|
|
||||||
|
|
||||||
/*
|
|
||||||
* Macros to allocate/return descriptors from module specific
|
|
||||||
* free list(s).
|
|
||||||
*/
|
|
||||||
|
|
||||||
#define MCA_BTL_UDAPL_FRAG_ALLOC_EAGER(btl, frag) \
|
|
||||||
{ \
|
|
||||||
ompi_free_list_item_t *item; \
|
|
||||||
OMPI_FREE_LIST_GET_MT(&((mca_btl_udapl_module_t*)btl)->udapl_frag_eager, item); \
|
|
||||||
frag = (mca_btl_udapl_frag_t*) item; \
|
|
||||||
}
|
|
||||||
|
|
||||||
#define MCA_BTL_UDAPL_FRAG_RETURN_EAGER(btl, frag) \
|
|
||||||
{ \
|
|
||||||
OMPI_FREE_LIST_RETURN_MT(&((mca_btl_udapl_module_t*)btl)->udapl_frag_eager, \
|
|
||||||
(ompi_free_list_item_t*)(frag)); \
|
|
||||||
}
|
|
||||||
|
|
||||||
#define MCA_BTL_UDAPL_FRAG_ALLOC_EAGER_RECV(btl, frag) \
|
|
||||||
{ \
|
|
||||||
ompi_free_list_item_t *item; \
|
|
||||||
OMPI_FREE_LIST_GET_MT(&((mca_btl_udapl_module_t*)btl)->udapl_frag_eager_recv, item); \
|
|
||||||
frag = (mca_btl_udapl_frag_t*) item; \
|
|
||||||
}
|
|
||||||
|
|
||||||
#define MCA_BTL_UDAPL_FRAG_ALLOC_MAX(btl, frag) \
|
|
||||||
{ \
|
|
||||||
ompi_free_list_item_t *item; \
|
|
||||||
OMPI_FREE_LIST_GET_MT(&((mca_btl_udapl_module_t*)btl)->udapl_frag_max, item); \
|
|
||||||
frag = (mca_btl_udapl_frag_t*) item; \
|
|
||||||
}
|
|
||||||
|
|
||||||
#define MCA_BTL_UDAPL_FRAG_RETURN_MAX(btl, frag) \
|
|
||||||
{ \
|
|
||||||
OMPI_FREE_LIST_RETURN_MT(&((mca_btl_udapl_module_t*)btl)->udapl_frag_max, \
|
|
||||||
(ompi_free_list_item_t*)(frag)); \
|
|
||||||
}
|
|
||||||
|
|
||||||
#define MCA_BTL_UDAPL_FRAG_ALLOC_MAX_RECV(btl, frag) \
|
|
||||||
{ \
|
|
||||||
ompi_free_list_item_t *item; \
|
|
||||||
OMPI_FREE_LIST_GET_MT(&((mca_btl_udapl_module_t*)btl)->udapl_frag_max_recv, item); \
|
|
||||||
frag = (mca_btl_udapl_frag_t*) item; \
|
|
||||||
}
|
|
||||||
|
|
||||||
#define MCA_BTL_UDAPL_FRAG_ALLOC_USER(btl, frag) \
|
|
||||||
{ \
|
|
||||||
ompi_free_list_item_t *item; \
|
|
||||||
OMPI_FREE_LIST_GET_MT(&((mca_btl_udapl_module_t*)btl)->udapl_frag_user, item); \
|
|
||||||
frag = (mca_btl_udapl_frag_t*) item; \
|
|
||||||
}
|
|
||||||
|
|
||||||
#define MCA_BTL_UDAPL_FRAG_RETURN_USER(btl, frag) \
|
|
||||||
{ \
|
|
||||||
OMPI_FREE_LIST_RETURN_MT(&((mca_btl_udapl_module_t*)btl)->udapl_frag_user, \
|
|
||||||
(ompi_free_list_item_t*)(frag)); \
|
|
||||||
}
|
|
||||||
|
|
||||||
#define MCA_BTL_UDAPL_FRAG_ALLOC_CONTROL(btl, frag) \
|
|
||||||
{ \
|
|
||||||
ompi_free_list_item_t *item; \
|
|
||||||
OMPI_FREE_LIST_GET_MT(&((mca_btl_udapl_module_t*)btl)->udapl_frag_control, item); \
|
|
||||||
frag = (mca_btl_udapl_frag_t*) item; \
|
|
||||||
}
|
|
||||||
|
|
||||||
#define MCA_BTL_UDAPL_FRAG_RETURN_CONTROL(btl, frag) \
|
|
||||||
{ \
|
|
||||||
OMPI_FREE_LIST_RETURN_MT(&((mca_btl_udapl_module_t*)btl)->udapl_frag_control, \
|
|
||||||
(ompi_free_list_item_t*)(frag)); \
|
|
||||||
}
|
|
||||||
|
|
||||||
/*
|
|
||||||
* Calculate the pad value P required to align the given size S
|
|
||||||
*/
|
|
||||||
#define MCA_BTL_UDAPL_FRAG_CALC_ALIGNMENT_PAD(P,S) do { \
|
|
||||||
(P) = ((S) % MCA_BTL_UDAPL_FRAG_ALIGN) == 0 ? \
|
|
||||||
0 : (MCA_BTL_UDAPL_FRAG_ALIGN - ((S) % MCA_BTL_UDAPL_FRAG_ALIGN)); \
|
|
||||||
} while (0);
|
|
||||||
|
|
||||||
END_C_DECLS
|
|
||||||
#endif
|
|
||||||
@ -1,314 +0,0 @@
|
|||||||
/*
|
|
||||||
* Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
|
|
||||||
* University Research and Technology
|
|
||||||
* Corporation. All rights reserved.
|
|
||||||
* Copyright (c) 2004-2005 The University of Tennessee and The University
|
|
||||||
* of Tennessee Research Foundation. All rights
|
|
||||||
* reserved.
|
|
||||||
* Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
|
|
||||||
* University of Stuttgart. All rights reserved.
|
|
||||||
* Copyright (c) 2004-2005 The Regents of the University of California.
|
|
||||||
* All rights reserved.
|
|
||||||
* Copyright (c) 2006 Cisco Systems, Inc. All rights reserved.
|
|
||||||
* Copyright (c) 2007-2008 Sun Microsystems, Inc. All rights reserved.
|
|
||||||
* $COPYRIGHT$
|
|
||||||
*
|
|
||||||
* Additional copyrights may follow
|
|
||||||
*
|
|
||||||
* $HEADER$
|
|
||||||
*/
|
|
||||||
|
|
||||||
#include "ompi_config.h"
|
|
||||||
#include "opal/mca/base/base.h"
|
|
||||||
#include "btl_udapl.h"
|
|
||||||
#include "btl_udapl_mca.h"
|
|
||||||
#include <string.h>
|
|
||||||
|
|
||||||
/*
|
|
||||||
* Utility routine for string parameter registration.
|
|
||||||
*
|
|
||||||
* @param param_name (IN) MCA parameter name
|
|
||||||
* @param param_desc (IN) MCA parameter description
|
|
||||||
* @param default_value (IN) MCA parameter default value
|
|
||||||
* @param out_value (OUT) value of MCA parameter; either default,
|
|
||||||
* or value as determined from typical
|
|
||||||
* MCA parameter setting methods
|
|
||||||
* @param flags (IN) MCA parameter boundary flag
|
|
||||||
* @return OMPI_SUCCESS or OMPI_ERR_BAD_PARAM
|
|
||||||
*/
|
|
||||||
static inline int mca_btl_udapl_reg_string(const char* param_name,
|
|
||||||
const char* param_desc,
|
|
||||||
const char* default_value,
|
|
||||||
char **storage, int flags)
|
|
||||||
{
|
|
||||||
*storage = default_value;
|
|
||||||
(void) mca_base_component_var_register(&mca_btl_udapl_component.super.btl_version, param_name,
|
|
||||||
param_desc, MCA_BASE_VAR_TYPE_STRING, NULL, 0, 0,
|
|
||||||
OPAL_INFO_LVL_9,
|
|
||||||
MCA_BASE_VAR_SCOPE_READONLY, storage);
|
|
||||||
|
|
||||||
if (NULL == *storage && !((flags & REGSTR_EMPTY_OK) == REGSTR_EMPTY_OK)) {
|
|
||||||
BTL_ERROR(("ERROR: MCA Parameter %s : Value (NULL) out of range : "
|
|
||||||
"Default value (%s)\n \t Parameter Description : %s",
|
|
||||||
param_name, default_value, param_desc));
|
|
||||||
return OMPI_ERR_BAD_PARAM;
|
|
||||||
}
|
|
||||||
|
|
||||||
if ((flags & REGSTR_EMPTY_NOT_OK) && 0 == strlen(*storage)) {
|
|
||||||
BTL_ERROR(("ERROR: MCA Parameter %s : Value (%s) out of range : "
|
|
||||||
"Default value (%s)\n \t Parameter Description : %s",
|
|
||||||
param_name, *storage, default_value, param_desc));
|
|
||||||
return OMPI_ERR_BAD_PARAM;
|
|
||||||
}
|
|
||||||
|
|
||||||
return OMPI_SUCCESS;
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
/*
|
|
||||||
* Utility routine for integer parameter registration.
|
|
||||||
*
|
|
||||||
* @param param_name (IN) MCA parameter name
|
|
||||||
* @param param_desc (IN) MCA parameter description
|
|
||||||
* @param default_value (IN) MCA parameter default value
|
|
||||||
* @param out_value (OUT) value of MCA parameter; either default,
|
|
||||||
* or value as determined from typical
|
|
||||||
* MCA parameter setting methods
|
|
||||||
* @param flags (IN) MCA parameter boundary flag
|
|
||||||
* @return OMPI_SUCCESS or OMPI_ERR_BAD_PARAM
|
|
||||||
*/
|
|
||||||
static inline int mca_btl_udapl_reg_int(const char* param_name,
|
|
||||||
const char* param_desc,
|
|
||||||
int default_value, int *storage,
|
|
||||||
int flags)
|
|
||||||
{
|
|
||||||
*storage = default_value;
|
|
||||||
(void) mca_base_component_var_register(&mca_btl_udapl_component.super.btl_version, param_name,
|
|
||||||
param_desc, MCA_BASE_VAR_TYPE_INT, NULL, 0, 0,
|
|
||||||
OPAL_INFO_LVL_9,
|
|
||||||
MCA_BASE_VAR_SCOPE_READONLY, storage);
|
|
||||||
|
|
||||||
if ((flags & REGINT_NEG_ONE_OK) && -1 == *storage) {
|
|
||||||
return OMPI_SUCCESS;
|
|
||||||
}
|
|
||||||
|
|
||||||
if (((flags & REGINT_GE_ZERO) && *storage < 0) ||
|
|
||||||
((flags & REGINT_GE_ONE) && *storage < 1) ||
|
|
||||||
((flags & REGINT_NONZERO) && 0 == *storage)) {
|
|
||||||
BTL_ERROR(("ERROR: MCA Parameter %s : Value (%d) out of range : "
|
|
||||||
"Default value (%d)\n \t Parameter Description : %s\n",
|
|
||||||
param_name, *storage, default_value, param_desc));
|
|
||||||
return OMPI_ERR_BAD_PARAM;
|
|
||||||
}
|
|
||||||
|
|
||||||
return OMPI_SUCCESS;
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
/*
|
|
||||||
* Register and check all MCA parameters
|
|
||||||
*
|
|
||||||
* @return OMPI_SUCCESS or OMPI_ERR_BAD_PARAM
|
|
||||||
*/
|
|
||||||
int mca_btl_udapl_register_mca_params(void)
|
|
||||||
{
|
|
||||||
int rc, tmp_rc;
|
|
||||||
|
|
||||||
rc = OMPI_SUCCESS;
|
|
||||||
|
|
||||||
/* register uDAPL component parameters */
|
|
||||||
CHECK_PARAM_REGISTER_RETURN_VALUE(mca_btl_udapl_reg_int("free_list_num",
|
|
||||||
"Initial size of free lists (must be >= 1).",
|
|
||||||
8,
|
|
||||||
&mca_btl_udapl_component.udapl_free_list_num,
|
|
||||||
REGINT_GE_ONE), tmp_rc, rc);
|
|
||||||
|
|
||||||
CHECK_PARAM_REGISTER_RETURN_VALUE(mca_btl_udapl_reg_int("free_list_max",
|
|
||||||
"Maximum size of free lists "
|
|
||||||
"(-1 = infinite, otherwise must be >= 1).",
|
|
||||||
-1,
|
|
||||||
&mca_btl_udapl_component.udapl_free_list_max,
|
|
||||||
REGINT_NEG_ONE_OK | REGINT_GE_ONE), tmp_rc, rc);
|
|
||||||
|
|
||||||
CHECK_PARAM_REGISTER_RETURN_VALUE(mca_btl_udapl_reg_int("free_list_inc",
|
|
||||||
"Increment size of free lists (must be >= 1).",
|
|
||||||
8,
|
|
||||||
&mca_btl_udapl_component.udapl_free_list_inc,
|
|
||||||
REGINT_GE_ONE), tmp_rc, rc);
|
|
||||||
|
|
||||||
CHECK_PARAM_REGISTER_RETURN_VALUE(mca_btl_udapl_reg_string("mpool",
|
|
||||||
"Name of the memory pool to be used.",
|
|
||||||
"grdma",
|
|
||||||
&mca_btl_udapl_component.udapl_mpool_name,
|
|
||||||
REGSTR_EMPTY_NOT_OK), tmp_rc, rc);
|
|
||||||
|
|
||||||
CHECK_PARAM_REGISTER_RETURN_VALUE(mca_btl_udapl_reg_int("max_modules",
|
|
||||||
"Maximum number of supported HCAs.",
|
|
||||||
8,
|
|
||||||
&mca_btl_udapl_component.udapl_max_btls,
|
|
||||||
REGINT_GE_ONE), tmp_rc, rc);
|
|
||||||
|
|
||||||
CHECK_PARAM_REGISTER_RETURN_VALUE(mca_btl_udapl_reg_int("num_recvs",
|
|
||||||
"Total number of receive buffers to keep posted "
|
|
||||||
"per endpoint (must be >= 1).",
|
|
||||||
8,
|
|
||||||
&mca_btl_udapl_component.udapl_num_recvs,
|
|
||||||
REGINT_GE_ONE), tmp_rc, rc);
|
|
||||||
|
|
||||||
CHECK_PARAM_REGISTER_RETURN_VALUE(mca_btl_udapl_reg_int("num_sends",
|
|
||||||
"Maximum number of sends to post on an endpoint "
|
|
||||||
"(must be >= 1).",
|
|
||||||
7,
|
|
||||||
&mca_btl_udapl_component.udapl_num_sends,
|
|
||||||
REGINT_GE_ONE), tmp_rc, rc);
|
|
||||||
|
|
||||||
CHECK_PARAM_REGISTER_RETURN_VALUE(mca_btl_udapl_reg_int("sr_win",
|
|
||||||
"Window size at which point an explicit "
|
|
||||||
"credit message will be generated (must be >= 1).",
|
|
||||||
4,
|
|
||||||
&mca_btl_udapl_component.udapl_sr_win,
|
|
||||||
REGINT_GE_ONE), tmp_rc, rc);
|
|
||||||
|
|
||||||
CHECK_PARAM_REGISTER_RETURN_VALUE(mca_btl_udapl_reg_int("use_eager_rdma",
|
|
||||||
"Use of RDMA for small messages : "
|
|
||||||
"1 = default, use RDMA for small messages; "
|
|
||||||
"0 = do not use RDMA for small messages. ",
|
|
||||||
1,
|
|
||||||
&mca_btl_udapl_component.udapl_use_eager_rdma,
|
|
||||||
REGINT_GE_ZERO), tmp_rc, rc);
|
|
||||||
|
|
||||||
CHECK_PARAM_REGISTER_RETURN_VALUE(mca_btl_udapl_reg_int("eager_rdma_num",
|
|
||||||
"Number of RDMA buffers to allocate "
|
|
||||||
"for small messages (must be >= 1).",
|
|
||||||
32,
|
|
||||||
&mca_btl_udapl_component.udapl_eager_rdma_num,
|
|
||||||
REGINT_GE_ONE), tmp_rc, rc);
|
|
||||||
|
|
||||||
CHECK_PARAM_REGISTER_RETURN_VALUE(mca_btl_udapl_reg_int("max_eager_rdma_peers",
|
|
||||||
"Maximum number of peers allowed to use "
|
|
||||||
"RDMA for short messages (independently RDMA will "
|
|
||||||
"still be used for large messages, (must be >= 0; "
|
|
||||||
"if zero then RDMA will not be used for short messages).",
|
|
||||||
16,
|
|
||||||
&mca_btl_udapl_component.udapl_max_eager_rdma_peers,
|
|
||||||
REGINT_GE_ZERO), tmp_rc, rc);
|
|
||||||
|
|
||||||
CHECK_PARAM_REGISTER_RETURN_VALUE(mca_btl_udapl_reg_int("eager_rdma_win",
|
|
||||||
"Window size at which point an explicit "
|
|
||||||
"credit message will be generated (must be >= 1).",
|
|
||||||
28,
|
|
||||||
&mca_btl_udapl_component.udapl_eager_rdma_win,
|
|
||||||
REGINT_GE_ONE), tmp_rc, rc);
|
|
||||||
|
|
||||||
CHECK_PARAM_REGISTER_RETURN_VALUE(mca_btl_udapl_reg_int("timeout",
|
|
||||||
"Connection timeout, in microseconds.",
|
|
||||||
MCA_BTL_UDAPL_CONN_TIMEOUT_DEFAULT,
|
|
||||||
&mca_btl_udapl_component.udapl_timeout,
|
|
||||||
REGINT_GE_ONE), tmp_rc, rc);
|
|
||||||
|
|
||||||
CHECK_PARAM_REGISTER_RETURN_VALUE(mca_btl_udapl_reg_int("conn_priv_data",
|
|
||||||
"Use connect private data to establish connections "
|
|
||||||
"(not supported by all uDAPL implementations).",
|
|
||||||
0,
|
|
||||||
&mca_btl_udapl_component.udapl_conn_priv_data,
|
|
||||||
REGINT_GE_ZERO), tmp_rc, rc);
|
|
||||||
|
|
||||||
CHECK_PARAM_REGISTER_RETURN_VALUE(mca_btl_udapl_reg_int("async_events",
|
|
||||||
"The asynchronous event queue will only be "
|
|
||||||
"checked after entering progress this number of times.",
|
|
||||||
100000000,
|
|
||||||
&mca_btl_udapl_component.udapl_async_events,
|
|
||||||
REGINT_GE_ONE), tmp_rc, rc);
|
|
||||||
|
|
||||||
CHECK_PARAM_REGISTER_RETURN_VALUE(mca_btl_udapl_reg_int("buffer_alignment",
|
|
||||||
"Preferred communication buffer alignment, "
|
|
||||||
"in bytes (must be >= 1).",
|
|
||||||
DAT_OPTIMAL_ALIGNMENT,
|
|
||||||
&mca_btl_udapl_component.udapl_buffer_alignment,
|
|
||||||
REGINT_GE_ONE), tmp_rc, rc);
|
|
||||||
|
|
||||||
CHECK_PARAM_REGISTER_RETURN_VALUE(mca_btl_udapl_reg_string("if_include",
|
|
||||||
"Comma-delimited list of interfaces to be included "
|
|
||||||
"(e.g. \"ibd0,ibd1 or OpenIB-cma,OpenIB-cma-1\"; empty value means "
|
|
||||||
"to use all interfaces found). Mutually exclusive with "
|
|
||||||
"btl_udapl_if_exclude.",
|
|
||||||
NULL, &mca_btl_udapl_component.if_include,
|
|
||||||
REGSTR_EMPTY_OK), tmp_rc, rc);
|
|
||||||
|
|
||||||
CHECK_PARAM_REGISTER_RETURN_VALUE(mca_btl_udapl_reg_string("if_exclude",
|
|
||||||
"Comma-delimited list of interfaces to be excluded from use "
|
|
||||||
"(e.g. \"ibd0,ibd1 or OpenIB-cma,OpenIB-cma-1\"; empty value means "
|
|
||||||
"not to exclude any). Mutually exclusive with btl_udapl_if_include.",
|
|
||||||
NULL, &mca_btl_udapl_component.if_exclude,
|
|
||||||
REGSTR_EMPTY_OK), tmp_rc, rc);
|
|
||||||
|
|
||||||
CHECK_PARAM_REGISTER_RETURN_VALUE(mca_btl_udapl_reg_int("verbose",
|
|
||||||
"Verbosity level of the uDAPL BTL (-1 thru 100)",
|
|
||||||
VERBOSE_SHOW_HELP,
|
|
||||||
&(mca_btl_udapl_component.udapl_verbosity),
|
|
||||||
REGINT_NEG_ONE_OK), tmp_rc, rc);
|
|
||||||
|
|
||||||
CHECK_PARAM_REGISTER_RETURN_VALUE(mca_btl_udapl_reg_int("compare_subnet",
|
|
||||||
"By default uDAPL BTL will compare subnets using netmask to "
|
|
||||||
"determine if an interface is reachable. Setting this parameter to "
|
|
||||||
"0 will essentially turn this comparison off and the uDAPL BTL will "
|
|
||||||
"assume all uDAPL interfaces are reachable (0 or 1, default==1).",
|
|
||||||
1,
|
|
||||||
&(mca_btl_udapl_component.udapl_compare_subnet),
|
|
||||||
REGINT_GE_ZERO), tmp_rc, rc);
|
|
||||||
|
|
||||||
/* register uDAPL module parameters */
|
|
||||||
CHECK_PARAM_REGISTER_RETURN_VALUE(mca_btl_udapl_reg_int("async_evd_qlen",
|
|
||||||
"The asynchronous event dispatcher queue length.",
|
|
||||||
MCA_BTL_UDAPL_ASYNC_EVD_QLEN_DEFAULT,
|
|
||||||
(int*)&mca_btl_udapl_module.udapl_async_evd_qlen,
|
|
||||||
REGINT_GE_ONE), tmp_rc, rc);
|
|
||||||
|
|
||||||
CHECK_PARAM_REGISTER_RETURN_VALUE(mca_btl_udapl_reg_int("conn_evd_qlen",
|
|
||||||
"The connection event dispatcher queue length is "
|
|
||||||
"a function of the number of connections expected.",
|
|
||||||
MCA_BTL_UDAPL_CONN_EVD_QLEN_DEFAULT,
|
|
||||||
(int*)&mca_btl_udapl_module.udapl_conn_evd_qlen,
|
|
||||||
REGINT_GE_ONE), tmp_rc, rc);
|
|
||||||
|
|
||||||
CHECK_PARAM_REGISTER_RETURN_VALUE(mca_btl_udapl_reg_int("dto_evd_qlen",
|
|
||||||
"The data transfer operation event dispatcher queue length is "
|
|
||||||
"a function of the number of connections as well as the "
|
|
||||||
"maximum number of outstanding data transfer operations.",
|
|
||||||
MCA_BTL_UDAPL_DTO_EVD_QLEN_DEFAULT,
|
|
||||||
(int*)&mca_btl_udapl_module.udapl_dto_evd_qlen,
|
|
||||||
REGINT_GE_ONE), tmp_rc, rc);
|
|
||||||
|
|
||||||
CHECK_PARAM_REGISTER_RETURN_VALUE(mca_btl_udapl_reg_int("max_request_dtos",
|
|
||||||
"Maximum number of outstanding "
|
|
||||||
"submitted sends and rdma operations per endpoint, (see Section "
|
|
||||||
"6.6.6 of uDAPL Spec.).",
|
|
||||||
MCA_BTL_UDAPL_MAX_REQUEST_DTOS_DEFAULT,
|
|
||||||
(int*)&mca_btl_udapl_module.udapl_max_request_dtos,
|
|
||||||
REGINT_GE_ONE), tmp_rc, rc);
|
|
||||||
|
|
||||||
CHECK_PARAM_REGISTER_RETURN_VALUE(mca_btl_udapl_reg_int("max_recv_dtos",
|
|
||||||
"Maximum number of outstanding "
|
|
||||||
"submitted receive operations per endpoint, (see Section "
|
|
||||||
"6.6.6 of uDAPL Spec.).",
|
|
||||||
MCA_BTL_UDAPL_MAX_RECV_DTOS_DEFAULT,
|
|
||||||
(int*)&mca_btl_udapl_module.udapl_max_recv_dtos,
|
|
||||||
REGINT_GE_ONE), tmp_rc, rc);
|
|
||||||
|
|
||||||
mca_btl_udapl_module.super.btl_exclusivity =
|
|
||||||
MCA_BTL_EXCLUSIVITY_DEFAULT - 10;
|
|
||||||
mca_btl_udapl_module.super.btl_eager_limit = 8*1024;
|
|
||||||
mca_btl_udapl_module.super.btl_rndv_eager_limit = 8*1024;
|
|
||||||
mca_btl_udapl_module.super.btl_max_send_size = 64*1024;
|
|
||||||
mca_btl_udapl_module.super.btl_rdma_pipeline_send_length = 512*1024;
|
|
||||||
mca_btl_udapl_module.super.btl_rdma_pipeline_frag_size = 128 * 1024;
|
|
||||||
mca_btl_udapl_module.super.btl_min_rdma_pipeline_size = 0;
|
|
||||||
mca_btl_udapl_module.super.btl_flags = MCA_BTL_FLAGS_PUT | MCA_BTL_FLAGS_SEND;
|
|
||||||
mca_btl_udapl_module.super.btl_bandwidth = 225;
|
|
||||||
mca_btl_udapl_module.super.btl_latency = 0;
|
|
||||||
|
|
||||||
mca_btl_base_param_register(&mca_btl_udapl_component.super.btl_version,
|
|
||||||
&mca_btl_udapl_module.super);
|
|
||||||
|
|
||||||
return rc;
|
|
||||||
}
|
|
||||||
@ -1,63 +0,0 @@
|
|||||||
/*
|
|
||||||
* Copyright (c) 2006 Cisco Systems, Inc. All rights reserved.
|
|
||||||
* Copyright (c) 2007 Sun Microsystems, Inc. All rights reserved.
|
|
||||||
* $COPYRIGHT$
|
|
||||||
*
|
|
||||||
* Additional copyrights may follow
|
|
||||||
*
|
|
||||||
* $HEADER$
|
|
||||||
*/
|
|
||||||
|
|
||||||
#ifndef MCA_BTL_UDAPL_MCA_H
|
|
||||||
#define MCA_BTL_UDAPL_MCA_H
|
|
||||||
|
|
||||||
BEGIN_C_DECLS
|
|
||||||
|
|
||||||
/* Define Integer Boundaries */
|
|
||||||
#define REGINT_NEG_ONE_OK 0x01 /* value = -1 is valid */
|
|
||||||
#define REGINT_GE_ZERO 0x02 /* value >= 0 is valid */
|
|
||||||
#define REGINT_GE_ONE 0x04 /* value > 1 is valid */
|
|
||||||
#define REGINT_NONZERO 0x08 /* value != 0 is valid */
|
|
||||||
|
|
||||||
/* Define String Boundaries */
|
|
||||||
#define REGSTR_EMPTY_OK 0x01 /* empty string is valid */
|
|
||||||
#define REGSTR_EMPTY_NOT_OK 0x02 /* empty string is not valid */
|
|
||||||
|
|
||||||
/* Define default parameter values that need to be known beyond the
|
|
||||||
* initial setting; for example, if a parameter is tuned dynamically
|
|
||||||
* by the BTL it would not be advisable to do so if the user has
|
|
||||||
* modified the default.
|
|
||||||
*/
|
|
||||||
#define MCA_BTL_UDAPL_ASYNC_EVD_QLEN_DEFAULT 256
|
|
||||||
#define MCA_BTL_UDAPL_CONN_EVD_QLEN_DEFAULT 256
|
|
||||||
#define MCA_BTL_UDAPL_DTO_EVD_QLEN_DEFAULT 256
|
|
||||||
#define MCA_BTL_UDAPL_CONN_TIMEOUT_DEFAULT 10000000
|
|
||||||
#define MCA_BTL_UDAPL_CONN_TIMEOUT_INC 200000 /* connection timeout
|
|
||||||
* is in microseconds;
|
|
||||||
* this incremental
|
|
||||||
* value is equivalent
|
|
||||||
* to .2 seconds
|
|
||||||
*/
|
|
||||||
#define MCA_BTL_UDAPL_CONN_TIMEOUT_MAX 2147483647
|
|
||||||
#define MCA_BTL_UDAPL_MAX_RECV_DTOS_DEFAULT 8
|
|
||||||
#define MCA_BTL_UDAPL_MAX_REQUEST_DTOS_DEFAULT 76
|
|
||||||
#define MCA_BTL_UDAPL_NUM_RECVS_DEFAULT 8
|
|
||||||
|
|
||||||
|
|
||||||
#define CHECK_PARAM_REGISTER_RETURN_VALUE(expr, tmp_rc, rc) \
|
|
||||||
{ \
|
|
||||||
tmp_rc = (expr); \
|
|
||||||
if (OMPI_SUCCESS != tmp_rc) { \
|
|
||||||
rc = tmp_rc; \
|
|
||||||
} \
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Function to register MCA params and check for sane values
|
|
||||||
*/
|
|
||||||
|
|
||||||
int mca_btl_udapl_register_mca_params(void);
|
|
||||||
|
|
||||||
|
|
||||||
END_C_DECLS
|
|
||||||
#endif
|
|
||||||
@ -1,312 +0,0 @@
|
|||||||
/*
|
|
||||||
* Copyright (c) 2004-2006 The Trustees of Indiana University and Indiana
|
|
||||||
* University Research and Technology
|
|
||||||
* Corporation. All rights reserved.
|
|
||||||
* Copyright (c) 2004-2011 The University of Tennessee and The University
|
|
||||||
* of Tennessee Research Foundation. All rights
|
|
||||||
* reserved.
|
|
||||||
* Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
|
|
||||||
* University of Stuttgart. All rights reserved.
|
|
||||||
* Copyright (c) 2004-2005 The Regents of the University of California.
|
|
||||||
* All rights reserved.
|
|
||||||
* Copyright (c) 2006 Sandia National Laboratories. All rights
|
|
||||||
* reserved.
|
|
||||||
* Copyright (c) 2008 Sun Microsystems, Inc. All rights reserved.
|
|
||||||
* Copyright (c) 2013 Intel, Inc. All rights reserved
|
|
||||||
* $COPYRIGHT$
|
|
||||||
*
|
|
||||||
* Additional copyrights may follow
|
|
||||||
*
|
|
||||||
* $HEADER$
|
|
||||||
*/
|
|
||||||
|
|
||||||
#include "ompi_config.h"
|
|
||||||
|
|
||||||
#include "ompi/runtime/ompi_module_exchange.h"
|
|
||||||
#include "opal/util/net.h"
|
|
||||||
#include "btl_udapl.h"
|
|
||||||
#include "btl_udapl_endpoint.h"
|
|
||||||
#include "btl_udapl_proc.h"
|
|
||||||
|
|
||||||
static void mca_btl_udapl_proc_construct(mca_btl_udapl_proc_t* proc);
|
|
||||||
static void mca_btl_udapl_proc_destruct(mca_btl_udapl_proc_t* proc);
|
|
||||||
|
|
||||||
OBJ_CLASS_INSTANCE(mca_btl_udapl_proc_t,
|
|
||||||
opal_list_item_t, mca_btl_udapl_proc_construct,
|
|
||||||
mca_btl_udapl_proc_destruct);
|
|
||||||
|
|
||||||
void mca_btl_udapl_proc_construct(mca_btl_udapl_proc_t* udapl_proc)
|
|
||||||
{
|
|
||||||
udapl_proc->proc_ompi = 0;
|
|
||||||
udapl_proc->proc_addr_count = 0;
|
|
||||||
udapl_proc->proc_endpoints = 0;
|
|
||||||
udapl_proc->proc_endpoint_count = 0;
|
|
||||||
OBJ_CONSTRUCT(&udapl_proc->proc_lock, opal_mutex_t);
|
|
||||||
|
|
||||||
/* add to list of all proc instance */
|
|
||||||
OPAL_THREAD_LOCK(&mca_btl_udapl_component.udapl_lock);
|
|
||||||
opal_list_append(&mca_btl_udapl_component.udapl_procs, &udapl_proc->super);
|
|
||||||
OPAL_THREAD_UNLOCK(&mca_btl_udapl_component.udapl_lock);
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
/*
|
|
||||||
* Cleanup uDAPL proc instance
|
|
||||||
*/
|
|
||||||
|
|
||||||
void mca_btl_udapl_proc_destruct(mca_btl_udapl_proc_t* udapl_proc)
|
|
||||||
{
|
|
||||||
/* remove from list of all proc instances */
|
|
||||||
OPAL_THREAD_LOCK(&mca_btl_udapl_component.udapl_lock);
|
|
||||||
opal_list_remove_item(&mca_btl_udapl_component.udapl_procs, &udapl_proc->super);
|
|
||||||
OPAL_THREAD_UNLOCK(&mca_btl_udapl_component.udapl_lock);
|
|
||||||
|
|
||||||
/* release resources */
|
|
||||||
if(NULL != udapl_proc->proc_endpoints) {
|
|
||||||
free(udapl_proc->proc_endpoints);
|
|
||||||
}
|
|
||||||
OBJ_DESTRUCT(&udapl_proc->proc_lock);
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
/*
|
|
||||||
* Look for an existing uDAPL process instances based on the associated
|
|
||||||
* ompi_proc_t instance.
|
|
||||||
*/
|
|
||||||
static mca_btl_udapl_proc_t* mca_btl_udapl_proc_lookup_ompi(ompi_proc_t* ompi_proc)
|
|
||||||
{
|
|
||||||
mca_btl_udapl_proc_t* udapl_proc;
|
|
||||||
|
|
||||||
OPAL_THREAD_LOCK(&mca_btl_udapl_component.udapl_lock);
|
|
||||||
|
|
||||||
for(udapl_proc = (mca_btl_udapl_proc_t*)
|
|
||||||
opal_list_get_first(&mca_btl_udapl_component.udapl_procs);
|
|
||||||
udapl_proc != (mca_btl_udapl_proc_t*)
|
|
||||||
opal_list_get_end(&mca_btl_udapl_component.udapl_procs);
|
|
||||||
udapl_proc = (mca_btl_udapl_proc_t*)opal_list_get_next(udapl_proc)) {
|
|
||||||
|
|
||||||
if(udapl_proc->proc_ompi == ompi_proc) {
|
|
||||||
OPAL_THREAD_UNLOCK(&mca_btl_udapl_component.udapl_lock);
|
|
||||||
return udapl_proc;
|
|
||||||
}
|
|
||||||
|
|
||||||
}
|
|
||||||
|
|
||||||
OPAL_THREAD_UNLOCK(&mca_btl_udapl_component.udapl_lock);
|
|
||||||
|
|
||||||
return NULL;
|
|
||||||
}
|
|
||||||
|
|
||||||
/*
|
|
||||||
* Create a uDAPL process structure. There is a one-to-one correspondence
|
|
||||||
* between a ompi_proc_t and a mca_btl_udapl_proc_t instance. We cache
|
|
||||||
* additional data (specifically the list of mca_btl_udapl_endpoint_t instances,
|
|
||||||
* and published addresses) associated w/ a given destination on this
|
|
||||||
* datastructure.
|
|
||||||
*/
|
|
||||||
|
|
||||||
mca_btl_udapl_proc_t* mca_btl_udapl_proc_create(ompi_proc_t* ompi_proc)
|
|
||||||
{
|
|
||||||
mca_btl_udapl_proc_t* udapl_proc = NULL;
|
|
||||||
size_t size;
|
|
||||||
int rc;
|
|
||||||
|
|
||||||
/* Check if we have already created a uDAPL proc
|
|
||||||
* structure for this ompi process */
|
|
||||||
udapl_proc = mca_btl_udapl_proc_lookup_ompi(ompi_proc);
|
|
||||||
if(udapl_proc != NULL) {
|
|
||||||
return udapl_proc;
|
|
||||||
}
|
|
||||||
|
|
||||||
/* create a new udapl proc out of the ompi_proc ... */
|
|
||||||
udapl_proc = OBJ_NEW(mca_btl_udapl_proc_t);
|
|
||||||
udapl_proc->proc_endpoint_count = 0;
|
|
||||||
udapl_proc->proc_ompi = ompi_proc;
|
|
||||||
|
|
||||||
/* query for the peer address info */
|
|
||||||
rc = ompi_modex_recv(
|
|
||||||
&mca_btl_udapl_component.super.btl_version,
|
|
||||||
ompi_proc,
|
|
||||||
(void*)&udapl_proc->proc_addrs,
|
|
||||||
&size);
|
|
||||||
if(OMPI_SUCCESS != rc) {
|
|
||||||
BTL_UDAPL_VERBOSE_OUTPUT(VERBOSE_CRITICAL,
|
|
||||||
("ompi_modex_recv failed for peer %s",
|
|
||||||
OMPI_NAME_PRINT(&ompi_proc->proc_name)));
|
|
||||||
OBJ_RELEASE(udapl_proc);
|
|
||||||
return NULL;
|
|
||||||
}
|
|
||||||
|
|
||||||
if((size % sizeof(mca_btl_udapl_addr_t)) != 0) {
|
|
||||||
BTL_UDAPL_VERBOSE_OUTPUT(VERBOSE_CRITICAL,
|
|
||||||
("invalid udapl address for peer %s",
|
|
||||||
OMPI_NAME_PRINT(&ompi_proc->proc_name)));
|
|
||||||
OBJ_RELEASE(udapl_proc);
|
|
||||||
return NULL;
|
|
||||||
}
|
|
||||||
|
|
||||||
udapl_proc->proc_addr_count = size/sizeof(mca_btl_udapl_addr_t);
|
|
||||||
if (0 == udapl_proc->proc_addr_count) {
|
|
||||||
udapl_proc->proc_endpoints = NULL;
|
|
||||||
} else {
|
|
||||||
udapl_proc->proc_endpoints = (mca_btl_base_endpoint_t**)
|
|
||||||
malloc(udapl_proc->proc_addr_count * sizeof(mca_btl_base_endpoint_t*));
|
|
||||||
}
|
|
||||||
if(NULL == udapl_proc->proc_endpoints) {
|
|
||||||
OBJ_RELEASE(udapl_proc);
|
|
||||||
return NULL;
|
|
||||||
}
|
|
||||||
return udapl_proc;
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
/*
|
|
||||||
* Find an address on the peer_process which matches stated criteria
|
|
||||||
* to the udapl btl module address information. Return in peer_addr_idx
|
|
||||||
* the index to the peer_process address that matches the btl module
|
|
||||||
* address. Where match criteria is:
|
|
||||||
* - the address in not already in use
|
|
||||||
* - compare addresses using netmask, the netmask value can be modified with
|
|
||||||
* "--mca btl_udapl_if_mask"
|
|
||||||
*
|
|
||||||
* Note: since this is called from mca_btl_udapl_proc_insert() it
|
|
||||||
* is assumed that the process lock is locked when entered.
|
|
||||||
*
|
|
||||||
* @param udapl_btl (IN) BTL module
|
|
||||||
* @param peer_process (IN) BTL peer process
|
|
||||||
* @param peer_addr_idx(IN/OUT) Index of address on peer_process
|
|
||||||
* which matches the udapl_btl address data.
|
|
||||||
* On success should be >= 0.
|
|
||||||
* @return OMPI_SUCCESS or error status on failure
|
|
||||||
*/
|
|
||||||
static int mca_btl_udapl_proc_address_match(
|
|
||||||
mca_btl_udapl_module_t* udapl_btl,
|
|
||||||
mca_btl_udapl_proc_t* peer_proc,
|
|
||||||
int* peer_addr_idx)
|
|
||||||
{
|
|
||||||
int i;
|
|
||||||
struct sockaddr *saddr;
|
|
||||||
struct sockaddr_in *btl_addr;
|
|
||||||
struct sockaddr_in *peer_addr;
|
|
||||||
char btl_addr_string[INET_ADDRSTRLEN];
|
|
||||||
char peer_addr_string[INET_ADDRSTRLEN];
|
|
||||||
|
|
||||||
*peer_addr_idx = MCA_BTL_UDAPL_INVALID_PEER_ADDR_IDX;
|
|
||||||
|
|
||||||
/* use generic address to find address family */
|
|
||||||
saddr = (struct sockaddr *)&(udapl_btl->udapl_addr.addr);
|
|
||||||
|
|
||||||
if (saddr->sa_family == AF_INET) {
|
|
||||||
|
|
||||||
btl_addr = (struct sockaddr_in *)saddr;
|
|
||||||
|
|
||||||
/* Loop thru peer process addresses looking for match.
|
|
||||||
* Match criteria:
|
|
||||||
* - address should not be "inuse"
|
|
||||||
* - both udapl btl module and peer address should be on
|
|
||||||
* the same subnet (compare with if_mask value)
|
|
||||||
*/
|
|
||||||
for(i = 0; i < (int) peer_proc->proc_addr_count; i++) {
|
|
||||||
|
|
||||||
peer_addr =
|
|
||||||
(struct sockaddr_in *)&(peer_proc->proc_addrs[i].addr);
|
|
||||||
|
|
||||||
if (VERBOSE_INFORM <=
|
|
||||||
mca_btl_udapl_component.udapl_verbosity) {
|
|
||||||
|
|
||||||
/* retrieve udapl btl and peer address string for reporting */
|
|
||||||
inet_ntop(AF_INET, (void *) &btl_addr->sin_addr,
|
|
||||||
btl_addr_string, INET_ADDRSTRLEN);
|
|
||||||
inet_ntop(AF_INET, (void *) &peer_addr->sin_addr,
|
|
||||||
peer_addr_string, INET_ADDRSTRLEN);
|
|
||||||
}
|
|
||||||
|
|
||||||
if ((false == peer_proc->proc_addrs[i].inuse) &&
|
|
||||||
(opal_net_samenetwork((struct sockaddr *)btl_addr,
|
|
||||||
(struct sockaddr *)peer_addr, udapl_btl->udapl_if_mask))) {
|
|
||||||
|
|
||||||
/* capture index of remote address where match found */
|
|
||||||
*peer_addr_idx = i;
|
|
||||||
|
|
||||||
/* mark this address as now being used */
|
|
||||||
peer_proc->proc_addrs[i].inuse = true;
|
|
||||||
|
|
||||||
/* report what address was found to match */
|
|
||||||
BTL_UDAPL_VERBOSE_OUTPUT(VERBOSE_INFORM,
|
|
||||||
("uDAPL BTL module(%s) matched %s",
|
|
||||||
btl_addr_string, peer_addr_string));
|
|
||||||
break;
|
|
||||||
} else {
|
|
||||||
/* peer address already used by another udapl btl
|
|
||||||
* module or netmask check not successful so skip
|
|
||||||
*/
|
|
||||||
BTL_UDAPL_VERBOSE_OUTPUT(VERBOSE_INFORM,
|
|
||||||
("uDAPL BTL module(%s) either skipped because it "
|
|
||||||
"is already in use or match criteria not successful "
|
|
||||||
"for peer address %s",
|
|
||||||
btl_addr_string, peer_addr_string));
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
} else {
|
|
||||||
/* current uDAPL BTL only supports IPv4 */
|
|
||||||
BTL_UDAPL_VERBOSE_HELP(VERBOSE_SHOW_HELP,
|
|
||||||
("help-mpi-btl-udapl.txt", "IPv4 only",
|
|
||||||
true, ompi_process_info.nodename));
|
|
||||||
return OMPI_ERROR;
|
|
||||||
}
|
|
||||||
|
|
||||||
if (MCA_BTL_UDAPL_INVALID_PEER_ADDR_IDX == *peer_addr_idx) {
|
|
||||||
BTL_UDAPL_VERBOSE_HELP(VERBOSE_SHOW_HELP,
|
|
||||||
("help-mpi-btl-udapl.txt", "no network match",
|
|
||||||
true, btl_addr_string, ompi_process_info.nodename,
|
|
||||||
(NULL == peer_proc->proc_ompi->proc_hostname) ?
|
|
||||||
"unknown" : peer_proc->proc_ompi->proc_hostname));
|
|
||||||
return OMPI_ERR_OUT_OF_RESOURCE;
|
|
||||||
}
|
|
||||||
|
|
||||||
return OMPI_SUCCESS;
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
/*
|
|
||||||
* Note that this routine must be called with the lock on the process
|
|
||||||
* already held. Insert a btl instance into the proc array and assign
|
|
||||||
* it an address.
|
|
||||||
*/
|
|
||||||
int mca_btl_udapl_proc_insert(
|
|
||||||
mca_btl_udapl_proc_t* udapl_proc,
|
|
||||||
mca_btl_base_endpoint_t* udapl_endpoint)
|
|
||||||
{
|
|
||||||
int peer_address_idx;
|
|
||||||
mca_btl_udapl_module_t* udapl_btl = udapl_endpoint->endpoint_btl;
|
|
||||||
|
|
||||||
/* Check so as not to create more endpoints than addresses.
|
|
||||||
* Example: If one node has 3 btl modules and another only has 2,
|
|
||||||
* this check prevents the node with 3 btl modules from
|
|
||||||
* overloading the other, i.e. only 2 possible connections will
|
|
||||||
* be possible.
|
|
||||||
*/
|
|
||||||
if (udapl_proc->proc_endpoint_count > udapl_proc->proc_addr_count)
|
|
||||||
return OMPI_ERR_OUT_OF_RESOURCE;
|
|
||||||
|
|
||||||
/* Find an endpoint on the udapl process of interest that matches
|
|
||||||
* the endpoint information of the current udapl btl module
|
|
||||||
*/
|
|
||||||
if (OMPI_SUCCESS !=
|
|
||||||
mca_btl_udapl_proc_address_match(udapl_btl, udapl_proc,
|
|
||||||
&peer_address_idx)) {
|
|
||||||
/* no address on peer proc met criteria */
|
|
||||||
return OMPI_ERR_OUT_OF_RESOURCE;
|
|
||||||
}
|
|
||||||
|
|
||||||
/* insert into endpoint array */
|
|
||||||
udapl_endpoint->endpoint_proc = udapl_proc;
|
|
||||||
udapl_endpoint->endpoint_addr =
|
|
||||||
udapl_proc->proc_addrs[peer_address_idx];
|
|
||||||
|
|
||||||
udapl_proc->proc_endpoints[udapl_proc->proc_endpoint_count] = udapl_endpoint;
|
|
||||||
udapl_proc->proc_endpoint_count++;
|
|
||||||
return OMPI_SUCCESS;
|
|
||||||
}
|
|
||||||
|
|
||||||
@ -1,67 +0,0 @@
|
|||||||
/*
|
|
||||||
* Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
|
|
||||||
* University Research and Technology
|
|
||||||
* Corporation. All rights reserved.
|
|
||||||
* Copyright (c) 2004-2011 The University of Tennessee and The University
|
|
||||||
* of Tennessee Research Foundation. All rights
|
|
||||||
* reserved.
|
|
||||||
* Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
|
|
||||||
* University of Stuttgart. All rights reserved.
|
|
||||||
* Copyright (c) 2004-2005 The Regents of the University of California.
|
|
||||||
* All rights reserved.
|
|
||||||
* Copyright (c) 2008 Sun Microsystems, Inc. All rights reserved.
|
|
||||||
* $COPYRIGHT$
|
|
||||||
*
|
|
||||||
* Additional copyrights may follow
|
|
||||||
*
|
|
||||||
* $HEADER$
|
|
||||||
*/
|
|
||||||
|
|
||||||
#ifndef MCA_BTL_UDAPL_PROC_H
|
|
||||||
#define MCA_BTL_UDAPL_PROC_H
|
|
||||||
|
|
||||||
#include "opal/class/opal_object.h"
|
|
||||||
#include "ompi/proc/proc.h"
|
|
||||||
#include "btl_udapl.h"
|
|
||||||
#include "btl_udapl_endpoint.h"
|
|
||||||
|
|
||||||
BEGIN_C_DECLS
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Represents the state of a remote process and the set of addresses
|
|
||||||
* that it exports. Also cache an instance of mca_btl_base_endpoint_t for
|
|
||||||
* each
|
|
||||||
* BTL instance that attempts to open a connection to the process.
|
|
||||||
*/
|
|
||||||
struct mca_btl_udapl_proc_t {
|
|
||||||
opal_list_item_t super;
|
|
||||||
/**< allow proc to be placed on a list */
|
|
||||||
|
|
||||||
ompi_proc_t *proc_ompi;
|
|
||||||
/**< pointer to corresponding ompi_proc_t */
|
|
||||||
|
|
||||||
struct mca_btl_udapl_addr_t* proc_addrs;
|
|
||||||
/**< array of addresses exported by peer */
|
|
||||||
|
|
||||||
size_t proc_addr_count;
|
|
||||||
/**< number of addresses published by peer */
|
|
||||||
|
|
||||||
struct mca_btl_base_endpoint_t **proc_endpoints;
|
|
||||||
/**< array of endpoints that have been created to access this proc */
|
|
||||||
|
|
||||||
size_t proc_endpoint_count;
|
|
||||||
/**< number of endpoints */
|
|
||||||
|
|
||||||
opal_mutex_t proc_lock;
|
|
||||||
/**< lock to protect against concurrent access to proc state */
|
|
||||||
};
|
|
||||||
typedef struct mca_btl_udapl_proc_t mca_btl_udapl_proc_t;
|
|
||||||
OBJ_CLASS_DECLARATION(mca_btl_udapl_proc_t);
|
|
||||||
|
|
||||||
#define MCA_BTL_UDAPL_INVALID_PEER_ADDR_IDX -1
|
|
||||||
|
|
||||||
mca_btl_udapl_proc_t* mca_btl_udapl_proc_create(ompi_proc_t* ompi_proc);
|
|
||||||
int mca_btl_udapl_proc_insert(mca_btl_udapl_proc_t*, mca_btl_base_endpoint_t*);
|
|
||||||
|
|
||||||
END_C_DECLS
|
|
||||||
#endif
|
|
||||||
@ -1,62 +0,0 @@
|
|||||||
# -*- shell-script -*-
|
|
||||||
#
|
|
||||||
# Copyright (c) 2004-2006 The Trustees of Indiana University and Indiana
|
|
||||||
# University Research and Technology
|
|
||||||
# Corporation. All rights reserved.
|
|
||||||
# Copyright (c) 2004-2005 The University of Tennessee and The University
|
|
||||||
# of Tennessee Research Foundation. All rights
|
|
||||||
# reserved.
|
|
||||||
# Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
|
|
||||||
# University of Stuttgart. All rights reserved.
|
|
||||||
# Copyright (c) 2004-2005 The Regents of the University of California.
|
|
||||||
# All rights reserved.
|
|
||||||
# Copyright (c) 2009 Sun Microsystems, Inc. All rights reserved.
|
|
||||||
# Copyright (c) 2010 Cisco Systems, Inc. All rights reserved.
|
|
||||||
# $COPYRIGHT$
|
|
||||||
#
|
|
||||||
# Additional copyrights may follow
|
|
||||||
#
|
|
||||||
# $HEADER$
|
|
||||||
#
|
|
||||||
|
|
||||||
|
|
||||||
# MCA_btl_udapl_CONFIG([action-if-can-compile],
|
|
||||||
# [action-if-cant-compile])
|
|
||||||
# ------------------------------------------------
|
|
||||||
AC_DEFUN([MCA_ompi_btl_udapl_CONFIG],[
|
|
||||||
AC_CONFIG_FILES([ompi/mca/btl/udapl/Makefile])
|
|
||||||
|
|
||||||
OMPI_CHECK_UDAPL([btl_udapl],
|
|
||||||
[btl_udapl_happy="yes"],
|
|
||||||
[btl_udapl_happy="no"])
|
|
||||||
|
|
||||||
AS_IF([test "$btl_udapl_happy" = "yes"],
|
|
||||||
[$1],
|
|
||||||
[$2])
|
|
||||||
|
|
||||||
# Borrowed from MVAPI BTL - a data structure in the uDAPL headers
|
|
||||||
# is not fully ISO C. Remove -pedantic to silence a warning.
|
|
||||||
btl_udapl_CFLAGS="`echo $CFLAGS | sed 's/-pedantic//g'`"
|
|
||||||
AS_IF([test "$btl_udapl_CFLAGS" != "$CFLAGS" -a "$btl_udapl_happy" = "yes"],
|
|
||||||
[AC_MSG_WARN([Removed -pedantic from CFLAGS for
|
|
||||||
uDAPL component because the uDAPL headers are not fully ISO C])])
|
|
||||||
|
|
||||||
# Test for uDAPL relaxed ordered specific symbols
|
|
||||||
AS_IF([test "$btl_udapl_happy" = "yes"],
|
|
||||||
[AC_MSG_CHECKING(for uDAPL DAT_MEM_TYPE_SO_VIRTUAL)
|
|
||||||
AC_TRY_COMPILE([#include <dat/udat.h>],
|
|
||||||
[DAT_MEM_TYPE dmt = DAT_MEM_TYPE_SO_VIRTUAL;],
|
|
||||||
[AC_MSG_RESULT(yes)
|
|
||||||
btl_udapl_ro_aware=1],
|
|
||||||
[AC_MSG_RESULT(no)
|
|
||||||
btl_udapl_ro_aware=0])
|
|
||||||
AC_DEFINE_UNQUOTED([HAVE_DAT_MEM_TYPE_SO_VIRTUAL],
|
|
||||||
[$btl_udapl_ro_aware],
|
|
||||||
[uDAPL DAT_MEM_TYPE_SO_VIRTUAL check])])
|
|
||||||
|
|
||||||
# substitute in the things needed to build udapl
|
|
||||||
AC_SUBST([btl_udapl_CFLAGS])
|
|
||||||
AC_SUBST([btl_udapl_CPPFLAGS])
|
|
||||||
AC_SUBST([btl_udapl_LDFLAGS])
|
|
||||||
AC_SUBST([btl_udapl_LIBS])
|
|
||||||
])dnl
|
|
||||||
@ -1,139 +0,0 @@
|
|||||||
# -*- text -*-
|
|
||||||
#
|
|
||||||
# Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
|
|
||||||
# University Research and Technology
|
|
||||||
# Corporation. All rights reserved.
|
|
||||||
# Copyright (c) 2004-2005 The University of Tennessee and The University
|
|
||||||
# of Tennessee Research Foundation. All rights
|
|
||||||
# reserved.
|
|
||||||
# Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
|
|
||||||
# University of Stuttgart. All rights reserved.
|
|
||||||
# Copyright (c) 2004-2006 The Regents of the University of California.
|
|
||||||
# All rights reserved.
|
|
||||||
# Copyright (c) 2007-2008 Sun Microsystems, Inc. All rights reserved.
|
|
||||||
#
|
|
||||||
# $COPYRIGHT$
|
|
||||||
#
|
|
||||||
# Additional copyrights may follow
|
|
||||||
#
|
|
||||||
# $HEADER$
|
|
||||||
#
|
|
||||||
# This is the US/English general help file for Open MPI.
|
|
||||||
#
|
|
||||||
[invalid num rdma segments]
|
|
||||||
WARNING: MCA parameter [btl_udapl_eager_rdma_num = %d] is not valid.
|
|
||||||
RDMA will not be used for short messages. Try setting to positive
|
|
||||||
value, e.g. 16.
|
|
||||||
#
|
|
||||||
[use default endpoint params]
|
|
||||||
WARNING: Using default uDAPL endpoint parameters not those that
|
|
||||||
would have been modified by MCA parameters.
|
|
||||||
#
|
|
||||||
[optimal buffer alignment mismatch]
|
|
||||||
WARNING: DAT_OPTIMAL_ALIGNMENT = %d : BTL buffer_alignment = %d.
|
|
||||||
The BTL buffer_alignment value may not be optimal. If all nodes
|
|
||||||
report the same DAT_OPTIMAL_ALIGNMENT value and this differs from
|
|
||||||
BTL buffer_alignment then setting "--mca btl_udapl_buffer_alignment
|
|
||||||
%d" may improve performance.
|
|
||||||
#
|
|
||||||
[max_recv_dtos too low]
|
|
||||||
WARNING: The MCA parameter max_recv_dtos has been modified to a value,
|
|
||||||
%d, that is insufficient. This value must be greater than or equal to
|
|
||||||
num_recvs, %d. The uDAPL BTL will adjust to allow the program to
|
|
||||||
proceed.
|
|
||||||
#
|
|
||||||
[max_request_dtos too low]
|
|
||||||
WARNING: The MCA parameter max_request_dtos has been modified to a
|
|
||||||
value, %d, which may not be sufficient. Try setting max_request_dtos
|
|
||||||
to %d if program fails.
|
|
||||||
#
|
|
||||||
[max_recv_dtos system max]
|
|
||||||
WARNING: The MCA parameter max_recv_dtos is trying to be set to,
|
|
||||||
%d, which is larger than allowable so the value will be set to maximum
|
|
||||||
allowed, %d.
|
|
||||||
#
|
|
||||||
[max_request_dtos system max]
|
|
||||||
WARNING: The MCA parameter max_request_dtos is trying to be set to,
|
|
||||||
%d, which is larger than allowable so the value will be set to maximum
|
|
||||||
allowed, %d.
|
|
||||||
#
|
|
||||||
[evd_qlen adapter max]
|
|
||||||
WARNING: The MCA parameter %s is trying to be set to %d,
|
|
||||||
which is larger than allowable so the value will be set to maximum
|
|
||||||
allowed, %d.
|
|
||||||
#
|
|
||||||
[evd_qlen too low]
|
|
||||||
WARNING: The MCA parameter %s has been modified to a value,
|
|
||||||
%d, which may not be sufficient. Try setting %s to %d if
|
|
||||||
program fails.
|
|
||||||
#
|
|
||||||
[connection timeout low]
|
|
||||||
WARNING: The MCA parameter %s has been modified to a value,
|
|
||||||
%d, which may not be sufficient. Try setting %s to %d if
|
|
||||||
program fails.
|
|
||||||
#
|
|
||||||
[dat_lmr_create DAT_INSUFFICIENT_RESOURCES]
|
|
||||||
WARNING: The uDAPL BTL is not able to register memory. Possibly out of
|
|
||||||
allowed privileged memory (i.e. memory that can be pinned). Increasing
|
|
||||||
the allowed privileged memory may alleviate this issue.
|
|
||||||
#
|
|
||||||
[dat_ia_open fail]
|
|
||||||
WARNING: Failed to open "%s" [%s:%s].
|
|
||||||
This may be a real error or it may be an invalid entry in the uDAPL
|
|
||||||
Registry which is contained in the dat.conf file. Contact your local
|
|
||||||
System Administrator to confirm the availability of the interfaces in
|
|
||||||
the dat.conf file.
|
|
||||||
#
|
|
||||||
[specified include and exclude]
|
|
||||||
ERROR: You have specified both the btl_udapl_if_include and
|
|
||||||
btl_udapl_if_exclude MCA parameters. These two parameters are
|
|
||||||
mutually exclusive; you can only specify one or the other.
|
|
||||||
|
|
||||||
For reference, the values that you specified are:
|
|
||||||
|
|
||||||
btl_udapl_if_include: %s
|
|
||||||
btl_udapl_if_exclude: %s
|
|
||||||
#
|
|
||||||
[nonexistent entry]
|
|
||||||
WARNING: One or more nonexistent interfaces were specified:
|
|
||||||
|
|
||||||
Host: %s
|
|
||||||
MCA parameter: btl_udapl_if_%sclude
|
|
||||||
Nonexistent entities: %s
|
|
||||||
|
|
||||||
These entities will be ignored.
|
|
||||||
#
|
|
||||||
[IPv4 only]
|
|
||||||
WARNING: uDAPL BTL only supports IPv4 addressing at this time.
|
|
||||||
Something other than an IPv4 address was detected on %s.
|
|
||||||
#
|
|
||||||
[no network match]
|
|
||||||
WARNING: Interface %s on node %s not able to find matching
|
|
||||||
interface on peer node %s. Could be that the interfaces are on
|
|
||||||
different subnets or there are fewer available uDAPL interfaces on peer.
|
|
||||||
#
|
|
||||||
[interface not found]
|
|
||||||
WARNING: Host %s, not able to determine interface name for
|
|
||||||
address %s. Will attempt to continue, assuming all addresses to
|
|
||||||
peer are reachable.
|
|
||||||
#
|
|
||||||
[netmask not found]
|
|
||||||
WARNING: Host %s, not able to determine netmask for address
|
|
||||||
%s. Will attempt to continue assuming all addresses to
|
|
||||||
peer are reachable.
|
|
||||||
#
|
|
||||||
[relaxed order support]
|
|
||||||
WARNING: While attempting to open interface %s the system reported
|
|
||||||
DAT_INVALID_RO_COOKIE. This indicates the currrent system supports
|
|
||||||
relaxed ordering. An attempt will be made to open the interface using
|
|
||||||
the following modified interface name %s. Open MPI must not use RDMA
|
|
||||||
for short eager messages in this scenario. Therefore, if opened
|
|
||||||
successfully RDMA will not be used for short eager messages. This will
|
|
||||||
negatively impact short message latency.
|
|
||||||
#
|
|
||||||
[dat_ia_open fail RO]
|
|
||||||
WARNING: Failed to open "%s" [%s:%s].
|
|
||||||
Attempted to call dat_ia_open() on an interface that has been prefixed
|
|
||||||
with "RO_AWARE_" after first trying to open %s and failed with
|
|
||||||
DAT_INVALID_RO_COOKIE.
|
|
||||||
#
|
|
||||||
Загрузка…
x
Ссылка в новой задаче
Block a user