ports/openmpi
ports/openmpi
1
1
Форкнуть 0
Код Релизы Активность

Per RFC, remove the udapl BTL.

Просмотр исходного кода 
This commit was SVN r29400.
Этот коммит содержится в:
Jeff Squyres 2013-10-08 15:18:59 +00:00
родитель 9389592e05
Коммит 66dadbe1e7
15 изменённых файлов: 0 добавлений и 6423 удалений
Показать статистику Скачать Patch файл Скачать Diff файл Показать все файлы Свернуть все файлы

63
ompi/mca/btl/udapl/Makefile.am
Просмотреть файл

@ -1,63 +0,0 @@
#
# Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
# University Research and Technology
# Corporation. All rights reserved.
# Copyright (c) 2004-2005 The University of Tennessee and The University
# of Tennessee Research Foundation. All rights
# reserved.
# Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
# University of Stuttgart. All rights reserved.
# Copyright (c) 2004-2005 The Regents of the University of California.
# All rights reserved.
# Copyright (c) 2007 Sun Microsystems, Inc. All rights reserved.
#
# Copyright (c) 2010 Cisco Systems, Inc. All rights reserved.
# $COPYRIGHT$
#
# Additional copyrights may follow
#
# $HEADER$
#
CFLAGS = $(btl_udapl_CFLAGS)
AM_CPPFLAGS = $(btl_udapl_CPPFLAGS) -DPKGDATADIR=\"$(pkgdatadir)\"
dist_pkgdata_DATA = \
help-mpi-btl-udapl.txt
udapl_sources = \
btl_udapl.c \
btl_udapl.h \
btl_udapl_component.c \
btl_udapl_eager_rdma.h \
btl_udapl_endpoint.c \
btl_udapl_endpoint.h \
btl_udapl_frag.c \
btl_udapl_frag.h \
btl_udapl_mca.c \
btl_udapl_mca.h \
btl_udapl_proc.c \
btl_udapl_proc.h
# Make the output library in this directory, and name it either
# mca_<type>_<name>.la (for DSO builds) or libmca_<type>_<name>.la
# (for static builds).
if MCA_BUILD_ompi_btl_udapl_DSO
component_noinst =
component_install = mca_btl_udapl.la
else
component_noinst = libmca_btl_udapl.la
component_install =
endif
mcacomponentdir = $(pkglibdir)
mcacomponent_LTLIBRARIES = $(component_install)
mca_btl_udapl_la_SOURCES = $(udapl_sources)
mca_btl_udapl_la_LIBADD = $(btl_udapl_LIBS)
mca_btl_udapl_la_LDFLAGS = -module -avoid-version $(btl_udapl_LDFLAGS)
noinst_LTLIBRARIES = $(component_noinst)
libmca_btl_udapl_la_SOURCES = $(udapl_sources)
libmca_btl_udapl_la_LIBADD = $(btl_udapl_LIBS)
libmca_btl_udapl_la_LDFLAGS = -module -avoid-version $(btl_udapl_LDFLAGS)

1344
ompi/mca/btl/udapl/btl_udapl.c
Просмотреть файл

Разница между файлами не показана из-за своего большого размера Загрузить разницу

468
ompi/mca/btl/udapl/btl_udapl.h
Просмотреть файл

@ -1,468 +0,0 @@
/* -*- Mode: C; c-basic-offset:4 ; -*- */
/*
* Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
* University Research and Technology
* Corporation. All rights reserved.
* Copyright (c) 2004-2009 The University of Tennessee and The University
* of Tennessee Research Foundation. All rights
* reserved.
* Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
* University of Stuttgart. All rights reserved.
* Copyright (c) 2004-2005 The Regents of the University of California.
* All rights reserved.
* Copyright (c) 2007-2009 Sun Microsystems, Inc. All rights reserved.
* Copyright (c) 2007 Cisco Systems, Inc. All rights reserved.
*
* $COPYRIGHT$
*
* Additional copyrights may follow
*
* $HEADER$
*/
/**
* @file
*/
#ifndef MCA_BTL_UDAPL_H
#define MCA_BTL_UDAPL_H
#include "ompi_config.h"
#include <sys/types.h>
#include <string.h>
#include <dat/udat.h>
/* Open MPI includes */
#include "opal/class/opal_pointer_array.h"
#include "opal/mca/event/event.h"
#include "ompi/class/ompi_free_list.h"
#include "ompi/mca/btl/btl.h"
#include "ompi/mca/btl/base/base.h"
#include "ompi/mca/btl/base/btl_base_error.h"
#include "ompi/mca/mpool/mpool.h"
#include "ompi/mca/btl/btl.h"
#include "btl_udapl_endpoint.h"
BEGIN_C_DECLS
/**
* uDAPL BTL component.
*/
struct mca_btl_udapl_component_t {
mca_btl_base_component_2_0_0_t super; /**< base BTL component */
int32_t udapl_verbosity; /**< report out level, see
"Report Out from uDAPL BTL" below for details. */
size_t udapl_num_btls; /**< number of hcas available to the uDAPL component */
unsigned int udapl_max_btls; /**< maximum number of supported hcas */
struct mca_btl_udapl_module_t **udapl_btls; /**< array of available BTL modules */
int32_t udapl_num_recvs; /**< number of recv buffers to keep posted */
int32_t udapl_num_sends; /**< number of sends to post on endpoint */
int32_t udapl_sr_win; /**< number of fragments recieved before
returning credits to sender */
unsigned int udapl_timeout; /**< connection timeout, in microseconds */
size_t udapl_eager_frag_size;
size_t udapl_max_frag_size;
size_t udapl_eager_rdma_frag_size; /* size of the rdma fragement including data
* payload space
*/
int udapl_free_list_num; /**< initial size of free lists */
int udapl_free_list_max; /**< maximum size of free lists */
int udapl_free_list_inc; /**< number of elements to alloc when growing */
int32_t udapl_use_eager_rdma; /**< turn rdma for small msgs on/off */
int32_t udapl_eager_rdma_num; /**< number of rdma buffers allocated
for short messages */
int32_t udapl_max_eager_rdma_peers; /**< maximum number of peers allowed to
use RDMA for short messages (cap)
*/
int32_t udapl_eager_rdma_win; /**< number of eager RDMA fragments
recieved before returning credits to
sender */
int32_t udapl_conn_priv_data; /**< use connect priv data for proc data */
int32_t udapl_async_events; /**< dequeue asynchronous events */
int32_t udapl_buffer_alignment; /**< preferred communication buffer alignment, in bytes */
opal_list_t udapl_procs; /**< list of udapl proc structures */
opal_mutex_t udapl_lock; /**< lock for accessing module state */
char* udapl_mpool_name; /**< name of memory pool */
int32_t udapl_compare_subnet;/**< whether to compare with netmask or not */
char *if_include;
char **if_include_list;
char *if_exclude;
char **if_exclude_list;
char **if_list; /* used for checking entries not found */
int32_t ro_aware_system; /* default 0; 1 if relaxed ordered platform */
};
typedef struct mca_btl_udapl_component_t mca_btl_udapl_component_t;
OMPI_MODULE_DECLSPEC extern mca_btl_udapl_component_t mca_btl_udapl_component;
/**
* BTL Module Interface
*/
struct mca_btl_udapl_module_t {
mca_btl_base_module_t super; /**< base BTL interface */
mca_btl_udapl_addr_t udapl_addr;
/* uDAPL interface and other handles */
DAT_IA_HANDLE udapl_ia;
DAT_PZ_HANDLE udapl_pz;
DAT_PSP_HANDLE udapl_psp;
DAT_IA_ATTR udapl_ia_attr;
/* event dispatchers - async, data transfer, connection negotiation */
DAT_EVD_HANDLE udapl_evd_async;
DAT_EVD_HANDLE udapl_evd_dto;
DAT_EVD_HANDLE udapl_evd_conn;
DAT_EP_PARAM udapl_ep_param;
/* free list of fragment descriptors */
ompi_free_list_t udapl_frag_eager;
ompi_free_list_t udapl_frag_eager_recv;
ompi_free_list_t udapl_frag_max;
ompi_free_list_t udapl_frag_max_recv;
ompi_free_list_t udapl_frag_user;
ompi_free_list_t udapl_frag_control;
opal_mutex_t udapl_lock; /* lock for accessing module state */
opal_mutex_t udapl_eager_rdma_lock; /* eager rdma lock */
int32_t udapl_eager_rdma_endpoint_count; /* count of the number of
* endpoints in
* udapl_eager_rdma_endpoints
*/
opal_pointer_array_t *udapl_eager_rdma_endpoints; /* array of endpoints
* with eager rdma
* connections
*/
int32_t udapl_async_events;
int32_t udapl_connect_inprogress;
int32_t udapl_num_peers;
/* module specific limits */
int udapl_async_evd_qlen;
int udapl_conn_evd_qlen;
int udapl_dto_evd_qlen;
int udapl_max_request_dtos; /**< maximum number of outstanding consumer
submitted sends and rdma operations, see
section 6.6.6 of uDAPL Spec */
int udapl_max_recv_dtos; /**< maximum number of outstanding consumer
submitted recv operations, see section
6.6.6 of uDAPL Spec */
uint32_t udapl_if_mask; /**< netmask value btl module */
};
typedef struct mca_btl_udapl_module_t mca_btl_udapl_module_t;
extern mca_btl_udapl_module_t mca_btl_udapl_module;
struct mca_btl_udapl_reg_t {
mca_mpool_base_registration_t base;
DAT_LMR_HANDLE lmr; /* local memory region (LMR) */
DAT_LMR_TRIPLET lmr_triplet; /* LMR triplet - context, address, length */
DAT_RMR_CONTEXT rmr_context; /* remote memory region context handle */
};
typedef struct mca_btl_udapl_reg_t mca_btl_udapl_reg_t;
/**
* Report Out from uDAPL BTL
*
* - BTL_ERROR() : Use to report out errors from uDAPL BTL. These are
* critical errors which will most likely cause the program to fail so
* this message should always be reported to the user. Defined in
* btl/base/btl_base_error.h.
* Example:
* dat_strerror(rc, (const char**)&major, (const char**)&minor);
* BTL_ERROR(("ERROR: %s %s %s\n", "dat_cr_accept", major, minor));
*
* - BTL_UDAPL_VERBOSE_OUTPUT() : Use to output different levels
* of verbosity to the user. See Note below.
* Example:
* BTL_UDAPL_VERBOSE_OUTPUT(VERBOSE_DIAGNOSE,
* ("WARNING: don't %s", "jump"));
*
* - BTL_UDAPL_VERBOSE_HELP() : Use output information as defined in
* uDAPL BTL help file (help-mpi-btl-udapl.txt). See Note below.
* Example:
* BTL_UDAPL_VERBOSE_HELP(VERBOSE_SHOW_HELP,
* ("help-mpi-btl-udapl.txt",
* "invalid num rdma segments", true, 22));
*
* Note : - Verbose levels are defined below. These levels are
* controlled by the mca parameter "btl_udapl_verbose".
* The verbose level is set to 10 by default so that critical
* error and useful help information will appear. Which ever value
* this param is set to, those messages as well as any lower level
* verbose messages will be reported.
* - Setting "btl_udapl_verbose" to "-1" will turn off all
* messages reported by the use of BTL_UDAPL_VERBOSE_*().
* - These macros should not be used in a critical path as they
* are always included in the compiled code.
* - These macros rely on the use of paranthesis around the "args"
* value.
*
* Values used with BTL_UDAPL_VERBOSE_*():
*
* - 0: critical user information; should always be reported;
* on by default
* - 10: useful help messages that would be reported from
* "help-mpi-btl-udapl.txt"; accessed from
* BTL_UDAPL_VERBOSE_HELP(); on by default
* - 20: general execution diagnostic information;
* may be useful to user or btl developer
* - 30: basic debugging/diagnostic information
* - 90: useful only to developers
* - 100: other components do not appear to go beyond 100 for verbose
* levels so noting here as the max for future reference
*/
#define VERBOSE_CRITICAL 0
#define VERBOSE_SHOW_HELP 10
#define VERBOSE_INFORM 20
#define VERBOSE_DIAGNOSE 30
#define VERBOSE_DEVELOPER 90
#define BTL_UDAPL_VERBOSE_OUTPUT(verbose_level, args) \
do { \
if (verbose_level <= mca_btl_udapl_component.udapl_verbosity) { \
BTL_OUTPUT(args); \
} \
} while(0);
#define BTL_UDAPL_VERBOSE_HELP(verbose_level, args) \
do { \
if (verbose_level <= mca_btl_udapl_component.udapl_verbosity) { \
opal_show_help args; \
} \
} while(0);
/*
* Report a uDAPL error - for debugging
*/
#if OPAL_ENABLE_DEBUG
extern void mca_btl_udapl_error(DAT_RETURN ret, char* str);
#define MCA_BTL_UDAPL_ERROR(ret, str) \
mca_btl_udapl_error((ret), (str));
#else
#define MCA_BTL_UDAPL_ERROR(ret, str)
#endif
/**
* Register uDAPL component parameters with the MCA framework
*/
extern int mca_btl_udapl_component_open(void);
/**
* Any final cleanup before being unloaded.
*/
extern int mca_btl_udapl_component_close(void);
/**
* uDAPL component initialization.
*
* @param num_btl_modules (OUT) Number of BTLs returned in BTL array.
* @param allow_multi_user_threads (OUT) Flag indicating wether BTL supports user threads (TRUE)
* @param have_hidden_threads (OUT) Flag indicating wether BTL uses threads (TRUE)
*/
extern mca_btl_base_module_t** mca_btl_udapl_component_init(
int *num_btl_modules,
bool allow_multi_user_threads,
bool have_hidden_threads
);
/**
* uDAPL component progress.
*/
extern int mca_btl_udapl_component_progress(void);
/**
* Initialize resources for a new BTL/uDAPL IA
*
* @param ia_name Name of uDAPL interface adapter
* @param btl BTL instance.
* @return OMPI_SUCCESS or error status on failure.
*/
extern int mca_btl_udapl_init(
DAT_NAME_PTR ia_name,
struct mca_btl_udapl_module_t* btl
);
/**
* Cleanup any resources held by the BTL.
*
* @param btl BTL instance.
* @return OMPI_SUCCESS or error status on failure.
*/
extern int mca_btl_udapl_finalize(
struct mca_btl_base_module_t* btl
);
/**
* PML->BTL notification of change in the process list.
*
* @param btl (IN)
* @param nprocs (IN) Number of processes
* @param procs (IN) Set of processes
* @param peers (OUT) Set of (optional) peer addressing info.
* @param peers (IN/OUT) Set of processes that are reachable via this BTL.
* @return OMPI_SUCCESS or error status on failure.
*
*/
extern int mca_btl_udapl_add_procs(
struct mca_btl_base_module_t* btl,
size_t nprocs,
struct ompi_proc_t **procs,
struct mca_btl_base_endpoint_t** peers,
opal_bitmap_t* reachable
);
/**
* PML->BTL notification of change in the process list.
*
* @param btl (IN) BTL instance
* @param nproc (IN) Number of processes.
* @param procs (IN) Set of processes.
* @param peers (IN) Set of peer data structures.
* @return Status indicating if cleanup was successful
*
*/
extern int mca_btl_udapl_del_procs(
struct mca_btl_base_module_t* btl,
size_t nprocs,
struct ompi_proc_t **procs,
struct mca_btl_base_endpoint_t** peers
);
/**
* Initiate an asynchronous send.
*
* @param btl (IN) BTL module
* @param endpoint (IN) BTL addressing information
* @param descriptor (IN) Description of the data to be transferred
* @param tag (IN) The tag value used to notify the peer.
*/
extern int mca_btl_udapl_send(
struct mca_btl_base_module_t* btl,
struct mca_btl_base_endpoint_t* btl_peer,
struct mca_btl_base_descriptor_t* descriptor,
mca_btl_base_tag_t tag
);
/**
* Initiate an asynchronous put.
*
* @param btl (IN) BTL module
* @param endpoint (IN) BTL addressing information
* @param descriptor (IN) Description of the data to be transferred
*/
extern int mca_btl_udapl_put(
struct mca_btl_base_module_t* btl,
struct mca_btl_base_endpoint_t* btl_peer,
struct mca_btl_base_descriptor_t* decriptor
);
/**
* Initiate an asynchronous get.
*
* @param btl (IN) BTL module
* @param endpoint (IN) BTL addressing information
* @param descriptor (IN) Description of the data to be transferred
*/
extern int mca_btl_udapl_get(
struct mca_btl_base_module_t* btl,
struct mca_btl_base_endpoint_t* btl_peer,
struct mca_btl_base_descriptor_t* decriptor
);
/**
* Allocate a descriptor with a segment of the requested size.
* Note that the BTL layer may choose to return a smaller size
* if it cannot support the request.
*
* @param btl (IN) BTL module
* @param size (IN) Request segment size.
*/
extern mca_btl_base_descriptor_t* mca_btl_udapl_alloc(
struct mca_btl_base_module_t* btl,
struct mca_btl_base_endpoint_t* endpoint,
uint8_t order,
size_t size,
uint32_t flags);
/**
* Return a segment allocated by this BTL.
*
* @param btl (IN) BTL module
* @param descriptor (IN) Allocated descriptor.
*/
extern int mca_btl_udapl_free(
struct mca_btl_base_module_t* btl,
mca_btl_base_descriptor_t* des);
/**
* Prepare a descriptor for send/rdma using the supplied
* convertor. If the convertor references data that is contigous,
* the descriptor may simply point to the user buffer. Otherwise,
* this routine is responsible for allocating buffer space and
* packing if required.
*
* @param btl (IN) BTL module
* @param endpoint (IN) BTL peer addressing
* @param convertor (IN) Data type convertor
* @param reserve (IN) Additional bytes requested by upper layer to precede user data
* @param size (IN/OUT) Number of bytes to prepare (IN), number of bytes actually prepared (OUT)
*/
mca_btl_base_descriptor_t* mca_btl_udapl_prepare_src(
struct mca_btl_base_module_t* btl,
struct mca_btl_base_endpoint_t* peer,
struct mca_mpool_base_registration_t*,
struct opal_convertor_t* convertor,
uint8_t order,
size_t reserve,
size_t* size,
uint32_t flags
);
extern mca_btl_base_descriptor_t* mca_btl_udapl_prepare_dst(
struct mca_btl_base_module_t* btl,
struct mca_btl_base_endpoint_t* peer,
struct mca_mpool_base_registration_t*,
struct opal_convertor_t* convertor,
uint8_t order,
size_t reserve,
size_t* size,
uint32_t flags);
/**
* Fault Tolerance Event Notification Function
* @param state Checkpoint Stae
* @return OMPI_SUCCESS or failure status
*/
int mca_btl_udapl_ft_event(int state);
END_C_DECLS
#endif

1223
ompi/mca/btl/udapl/btl_udapl_component.c
Просмотреть файл

Разница между файлами не показана из-за своего большого размера Загрузить разницу

105
ompi/mca/btl/udapl/btl_udapl_eager_rdma.h
Просмотреть файл

@ -1,105 +0,0 @@
/*
* Copyright (c) 2006 Sun Microsystems, Inc. All rights reserved.
*
* $COPYRIGHT$
*
* Additional copyrights may follow
*
* $HEADER$
*/
#ifndef MCA_BTL_UDAPL_EAGER_RDMA_H
#define MCA_BTL_UDAPL_EAGER_RDMA_H
/* Open MPI includes */
#include "ompi/mca/btl/udapl/btl_udapl_endpoint.h"
BEGIN_C_DECLS
/*
* Describe endpoint local memory region.
*/
struct mca_btl_udapl_eager_rdma_local_t {
ompi_ptr_t base; /**< points to fragment structures */
struct mca_btl_udapl_reg_t* reg;
uint8_t head; /**< RDMA buffer to poll */
int32_t credits; /**< number of local rdma buffers ready to be reclaimed,
reused. Initially equal to 0. */
opal_mutex_t lock; /**< protect access to RDMA buffer */
};
typedef struct mca_btl_udapl_eager_rdma_local_t mca_btl_udapl_eager_rdma_local_t;
/*
* Describe endpoint remote memory region.
*/
struct mca_btl_udapl_eager_rdma_remote_t {
ompi_ptr_t base; /**< points to start of data region, not
fragment structures */
DAT_RMR_CONTEXT rkey; /**< key required to access remote memory */
uint8_t head; /**< RDMA buffer to use */
int32_t tokens; /**< number of available rdma buffers, initially equal
to mca parameter eager_rdma_num */
opal_mutex_t lock; /**< protect access to RDMA buffer */
};
typedef struct mca_btl_udapl_eager_rdma_remote_t mca_btl_udapl_eager_rdma_remote_t;
/*
* Encapsulate data that describes a remote memory region.
*/
struct mca_btl_udapl_eager_rdma_connect_t {
mca_btl_udapl_control_header_t control;
uint32_t rkey;
ompi_ptr_t rdma_start;
};
typedef struct mca_btl_udapl_eager_rdma_connect_t mca_btl_udapl_eager_rdma_connect_t;
/*
* Encapsulate data that describes rdma credit information.
*/
struct mca_btl_udapl_eager_rdma_credit_t {
mca_btl_udapl_control_header_t control;
uint32_t credits;
};
typedef struct mca_btl_udapl_eager_rdma_credit_t mca_btl_udapl_eager_rdma_credit_t;
#define EAGER_RDMA_BUFFER_AVAILABLE (0)
#define EAGER_RDMA_BUFFER_IN_USE (0xff)
#define MCA_BTL_UDAPL_RDMA_FRAG_IN_USE(F) do { \
*(volatile uint8_t*) ((char*)(F) + \
(mca_btl_udapl_component.udapl_eager_rdma_frag_size - \
(sizeof(mca_btl_udapl_footer_t)))); \
} while (0)
#define MCA_BTL_UDAPL_RDMA_FRAG_ASSIGN_IN_USE(F) do { \
*(volatile uint8_t*) ((char*)(F) + \
(mca_btl_udapl_component.udapl_eager_rdma_frag_size- \
(sizeof(mca_btl_udapl_footer_t)))) = EAGER_RDMA_BUFFER_IN_USE; \
} while (0)
#define MCA_BTL_UDAPL_RDMA_FRAG_ASSIGN_AVAILABLE(F) do { \
*(volatile uint8_t*) ((char*)(F) + \
(mca_btl_udapl_component.udapl_eager_rdma_frag_size - \
(sizeof(mca_btl_udapl_footer_t)))) = EAGER_RDMA_BUFFER_AVAILABLE; \
} while (0)
/* Retrieve the rdma fragment at location I */
#define MCA_BTL_UDAPL_GET_LOCAL_RDMA_FRAG(E, I) \
(mca_btl_udapl_frag_t*) \
((char*)(E)->endpoint_eager_rdma_local.base.pval + \
(I) * sizeof(mca_btl_udapl_frag_eager_rdma_t))
/*
* Increment the index I by one while not exceeding the total number of
* available eager rdma fragments
*/
#define MCA_BTL_UDAPL_RDMA_NEXT_INDEX(I) do { \
(I) = ((I) + 1); \
if((I) == \
mca_btl_udapl_component.udapl_eager_rdma_num) \
(I) = 0; \
} while (0)
END_C_DECLS
#endif

1708
ompi/mca/btl/udapl/btl_udapl_endpoint.c
Просмотреть файл

Разница между файлами не показана из-за своего большого размера Загрузить разницу

218
ompi/mca/btl/udapl/btl_udapl_endpoint.h
Просмотреть файл

@ -1,218 +0,0 @@
/*
* Copyright (c) 2004-2006 The Trustees of Indiana University and Indiana
* University Research and Technology
* Corporation. All rights reserved.
* Copyright (c) 2004-2005 The University of Tennessee and The University
* of Tennessee Research Foundation. All rights
* reserved.
* Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
* University of Stuttgart. All rights reserved.
* Copyright (c) 2004-2005 The Regents of the University of California.
* All rights reserved.
* Copyright (c) 2006 Sandia National Laboratories. All rights
* reserved.
* Copyright (c) 2006-2009 Sun Microsystems, Inc. All rights reserved.
*
* $COPYRIGHT$
*
* Additional copyrights may follow
*
* $HEADER$
*/
#ifndef MCA_BTL_UDAPL_ENDPOINT_H
#define MCA_BTL_UDAPL_ENDPOINT_H
#include "opal/class/opal_list.h"
#include "opal/mca/event/event.h"
#include "ompi/mca/btl/base/btl_base_error.h"
#include "btl_udapl_frag.h"
#include "btl_udapl.h"
#include "btl_udapl_eager_rdma.h"
BEGIN_C_DECLS
#define BTL_UDAPL_TOKEN_AVAIL(E, C, T) \
do { \
(T) = 0; \
if ( (E)->endpoint_lwqe_tokens[(C)] > 0 && \
((E)->endpoint_sr_tokens[(C)] + \
(((C) == BTL_UDAPL_EAGER_CONNECTION)?(E)->endpoint_eager_rdma_remote.tokens:0)) \
) { \
(T) = 1; \
} \
} while (0)
/**
* Structure used to publish uDAPL id information to peers.
*/
struct mca_btl_udapl_addr_t {
DAT_CONN_QUAL port;
DAT_SOCK_ADDR addr;
bool inuse;
};
typedef struct mca_btl_udapl_addr_t mca_btl_udapl_addr_t;
/**
* State of uDAPL endpoint connection.
*/
typedef enum {
MCA_BTL_UDAPL_CONN_EAGER,
MCA_BTL_UDAPL_CONN_MAX,
MCA_BTL_UDAPL_CONNECTED,
MCA_BTL_UDAPL_CLOSED,
MCA_BTL_UDAPL_FAILED
} mca_btl_udapl_endpoint_state_t;
/*
* Establish a name for the 2 connections opened per peer
*/
typedef enum {
BTL_UDAPL_EAGER_CONNECTION,
BTL_UDAPL_MAX_CONNECTION,
BTL_UDAPL_NUM_CONNECTION
} mca_btl_udapl_endpoint_conn_t;
/*
* Encapsulate data that describes sendrecv credit information.
*/
struct mca_btl_udapl_sr_credit_t {
mca_btl_udapl_control_header_t control;
uint32_t credits;
int connection; /* 0 == BTL_UDAPL_EAGER_CONNECTION;
1 == BTL_UDAPL_MAX_CONNECTION */
};
typedef struct mca_btl_udapl_sr_credit_t mca_btl_udapl_sr_credit_t;
/**
* An abstraction that represents a connection to a endpoint process.
* An instance of mca_btl_base_endpoint_t is associated w/ each process
* and BTL pair at startup. However, connections to the endpoint
* are established dynamically on an as-needed basis:
*/
struct mca_btl_base_endpoint_t {
opal_list_item_t super;
struct mca_btl_udapl_module_t* endpoint_btl;
/**< BTL instance that created this connection */
struct mca_btl_udapl_proc_t* endpoint_proc;
/**< proc structure corresponding to endpoint */
mca_btl_udapl_endpoint_state_t endpoint_state;
/**< current state of the endpoint connection */
opal_list_t endpoint_eager_frags;
opal_list_t endpoint_max_frags;
/**< pending send frags on this endpoint */
int32_t endpoint_eager_sends;
int32_t endpoint_max_sends;
/**< number of sends that may be posted */
int32_t endpoint_sr_tokens[BTL_UDAPL_NUM_CONNECTION];
/**< number of sends that may be posted */
int32_t endpoint_sr_credits[BTL_UDAPL_NUM_CONNECTION];
/**< number of recvs that are now available */
int32_t endpoint_lwqe_tokens[BTL_UDAPL_NUM_CONNECTION];
/**< number of local work queue credits available (combination of
posted sends and rdma writes allowed per endpoint */
int32_t endpoint_connection_seq;
/**< sequence number of sendrecv message for the connection est */
int32_t endpoint_connections_completed;
/**< count of completed connections for priv data connection est. */
opal_mutex_t endpoint_lock;
/**< lock for concurrent access to endpoint state */
mca_btl_udapl_addr_t endpoint_addr;
/**< remote address on the other side of this endpoint */
DAT_EP_HANDLE endpoint_eager;
DAT_EP_HANDLE endpoint_max;
/**< uDAPL endpoint handle */
int32_t endpoint_eager_rdma_index;
/**< index into array of endpoints with RDMA buffers */
mca_btl_udapl_eager_rdma_local_t endpoint_eager_rdma_local;
/**< info about local RDMA buffer */
mca_btl_udapl_eager_rdma_remote_t endpoint_eager_rdma_remote;
/**< info about remote RDMA buffer */
};
typedef struct mca_btl_base_endpoint_t mca_btl_base_endpoint_t;
typedef mca_btl_base_endpoint_t mca_btl_udapl_endpoint_t;
OBJ_CLASS_DECLARATION(mca_btl_udapl_endpoint_t);
/*
* Start sending data on an endpoint.
*/
int mca_btl_udapl_endpoint_send(mca_btl_base_endpoint_t* endpoint,
mca_btl_udapl_frag_t* frag);
/*
* Set up OOB recv callback.
*/
void mca_btl_udapl_endpoint_post_oob_recv(void);
/*
* Finish establishing a connection
*/
int mca_btl_udapl_endpoint_finish_connect(struct mca_btl_udapl_module_t* btl,
mca_btl_udapl_addr_t* addr,
int32_t* seq,
DAT_EP_HANDLE endpoint);
/*
* Send number of eager rdma credits
*/
int mca_btl_udapl_endpoint_send_eager_rdma_credits(mca_btl_base_endpoint_t* endpoint);
/*
* Establish uDAPL endpoint parameters
*/
int mca_btl_udapl_endpoint_get_params(struct mca_btl_udapl_module_t* btl,
DAT_EP_PARAM* ep_param);
/*
* Create uDAPL endpoint
*/
int mca_btl_udapl_endpoint_create(struct mca_btl_udapl_module_t* btl,
DAT_EP_HANDLE* udapl_endpoint);
/*
* Send number of send recv credits
*/
int mca_btl_udapl_endpoint_send_sr_credits(mca_btl_base_endpoint_t* endpoint,
const int connection);
/*
* Handle the established DAT endpoint when private data is in use
*/
int mca_btl_udapl_endpoint_pd_established_conn(
struct mca_btl_udapl_module_t* btl,
DAT_EP_HANDLE established_ep);
/*
* Utility routine. Search list of endpoints to find one that matches
* the given address.
*/
mca_btl_udapl_endpoint_t* mca_btl_udapl_find_endpoint_address_match(
struct mca_btl_udapl_module_t* btl,
mca_btl_udapl_addr_t addr);
END_C_DECLS
#endif

134
ompi/mca/btl/udapl/btl_udapl_frag.c
Просмотреть файл

@ -1,134 +0,0 @@
/*
* Copyright (c) 2004-2006 The Trustees of Indiana University and Indiana
* University Research and Technology
* Corporation. All rights reserved.
* Copyright (c) 2004-2005 The University of Tennessee and The University
* of Tennessee Research Foundation. All rights
* reserved.
* Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
* University of Stuttgart. All rights reserved.
* Copyright (c) 2004-2005 The Regents of the University of California.
* All rights reserved.
* Copyright (c) 2006 Sandia National Laboratories. All rights
* reserved.
* Copyright (c) 2006 Sun Microsystems, Inc. All rights reserved.
* $COPYRIGHT$
*
* Additional copyrights may follow
*
* $HEADER$
*/
#include "btl_udapl.h"
#include "btl_udapl_frag.h"
static void mca_btl_udapl_frag_common_constructor(mca_btl_udapl_frag_t* frag)
{
mca_btl_udapl_reg_t* reg =
(mca_btl_udapl_reg_t*)frag->base.super.registration;
#if OPAL_ENABLE_DEBUG
frag->base.des_src = NULL;
frag->base.des_src_cnt = 0;
frag->base.des_dst = NULL;
frag->base.des_dst_cnt = 0;
frag->base.des_flags = 0;
#endif
frag->registration = reg;
frag->segment.base.seg_addr.pval = (unsigned char*)frag->base.super.ptr;
frag->ftr = NULL;
/* Don't understand why yet, but there are cases where reg is NULL -
that is, this memory has not been registered. So be careful not
to dereference a NULL pointer. */
if(NULL != reg) {
/* Save the LMR context so we can set up LMR subset triplets later */
frag->triplet.lmr_context = reg->lmr_triplet.lmr_context;
}
}
static void mca_btl_udapl_frag_eager_constructor(mca_btl_udapl_frag_t* frag)
{
frag->segment.base.seg_len = mca_btl_udapl_module.super.btl_eager_limit;
frag->size = mca_btl_udapl_component.udapl_eager_frag_size;
mca_btl_udapl_frag_common_constructor(frag);
}
static void mca_btl_udapl_frag_max_constructor(mca_btl_udapl_frag_t* frag)
{
frag->segment.base.seg_len = mca_btl_udapl_module.super.btl_max_send_size;
frag->size = mca_btl_udapl_component.udapl_max_frag_size;
mca_btl_udapl_frag_common_constructor(frag);
}
static void mca_btl_udapl_frag_user_constructor(mca_btl_udapl_frag_t* frag)
{
mca_btl_udapl_frag_common_constructor(frag);
frag->segment.base.seg_len = 0;
frag->segment.base.seg_addr.pval = NULL;
frag->ftr = NULL;
frag->size = 0;
frag->registration = NULL;
}
static void mca_btl_udapl_frag_eager_rdma_constructor(mca_btl_udapl_frag_t* frag)
{
mca_btl_udapl_frag_eager_constructor(frag);
frag->segment.base.seg_len = mca_btl_udapl_module.super.btl_eager_limit;
frag->size = mca_btl_udapl_component.udapl_eager_frag_size;
frag->rdma_ftr = (mca_btl_udapl_rdma_footer_t *)
((char *)(frag->segment.base.seg_addr.pval) +
frag->size -
sizeof(mca_btl_udapl_rdma_footer_t));
frag->rdma_ftr->active=0;
}
static void mca_btl_udapl_frag_common_destructor(mca_btl_udapl_frag_t* frag)
{
#if OPAL_ENABLE_DEBUG
frag->ftr = NULL;
frag->size = 0;
frag->registration = NULL;
frag->segment.base.seg_len = 0;
frag->segment.base.seg_addr.pval = NULL;
frag->base.des_src = NULL;
frag->base.des_src_cnt = 0;
frag->base.des_dst = NULL;
frag->base.des_dst_cnt = 0;
frag->base.des_flags = 0;
#endif
}
OBJ_CLASS_INSTANCE(
mca_btl_udapl_frag_t,
mca_btl_base_descriptor_t,
NULL,
NULL);
OBJ_CLASS_INSTANCE(
mca_btl_udapl_frag_eager_t,
mca_btl_base_descriptor_t,
mca_btl_udapl_frag_eager_constructor,
mca_btl_udapl_frag_common_destructor);
OBJ_CLASS_INSTANCE(
mca_btl_udapl_frag_max_t,
mca_btl_base_descriptor_t,
mca_btl_udapl_frag_max_constructor,
mca_btl_udapl_frag_common_destructor);
OBJ_CLASS_INSTANCE(
mca_btl_udapl_frag_user_t,
mca_btl_base_descriptor_t,
mca_btl_udapl_frag_user_constructor,
NULL);
OBJ_CLASS_INSTANCE(
mca_btl_udapl_frag_eager_rdma_t,
mca_btl_base_descriptor_t,
mca_btl_udapl_frag_eager_rdma_constructor,
mca_btl_udapl_frag_common_destructor);

203
ompi/mca/btl/udapl/btl_udapl_frag.h
Просмотреть файл

@ -1,203 +0,0 @@
/*
* Copyright (c) 2004-2006 The Trustees of Indiana University and Indiana
* University Research and Technology
* Corporation. All rights reserved.
* Copyright (c) 2004-2013 The University of Tennessee and The University
* of Tennessee Research Foundation. All rights
* reserved.
* Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
* University of Stuttgart. All rights reserved.
* Copyright (c) 2004-2005 The Regents of the University of California.
* All rights reserved.
* Copyright (c) 2006 Sandia National Laboratories. All rights
* reserved.
* Copyright (c) 2007-2009 Sun Microsystems, Inc. All rights reserved.
*
* $COPYRIGHT$
*
* Additional copyrights may follow
*
* $HEADER$
*/
#ifndef MCA_BTL_UDAPL_FRAG_H
#define MCA_BTL_UDAPL_FRAG_H
#define MCA_BTL_UDAPL_FRAG_ALIGN (8)
#include "ompi_config.h"
BEGIN_C_DECLS
typedef enum {
MCA_BTL_UDAPL_SEND,
MCA_BTL_UDAPL_RECV,
MCA_BTL_UDAPL_PUT,
MCA_BTL_UDAPL_GET,
MCA_BTL_UDAPL_CONN_RECV,
MCA_BTL_UDAPL_CONN_SEND,
MCA_BTL_UDAPL_RDMA_WRITE,
MCA_BTL_UDAPL_FRAG_EAGER_RDMA,
MCA_BTL_UDAPL_IGNORE
} mca_btl_udapl_frag_type_t;
typedef enum {
MCA_BTL_UDAPL_CONTROL_NOOP,
MCA_BTL_UDAPL_CONTROL_RDMA_CONNECT,
MCA_BTL_UDAPL_CONTROL_RDMA_CREDIT,
MCA_BTL_UDAPL_CONTROL_SR_CREDIT
} mca_btl_udapl_control_t;
/* Control message header */
struct mca_btl_udapl_control_header_t {
mca_btl_udapl_control_t type;
};
typedef struct mca_btl_udapl_control_header_t mca_btl_udapl_control_header_t;
/**
* uDAPL btl footer.
* This is put after the payload packet so the PML header can be aligned.
* Must be aligned on MCA_BTL_UDAPL_FRAG_ALIGN byte boundary.
*/
struct mca_btl_udapl_footer_t {
mca_btl_base_tag_t tag;
};
typedef struct mca_btl_udapl_footer_t mca_btl_udapl_footer_t;
/**
* uDAPL BTL rdma footer.
* This is used in addtion to the uDAPL BTL footer. The two are seperate to
* allow for any padding that may be required between the two.
*/
struct mca_btl_udapl_rdma_footer_t {
uint32_t size;
volatile uint8_t active;/* 0 = not in use; 1 = data is available to be
* received; this should always be the last entry
* in this structure
*/
char pad[3]; /* pad out be aligned on MCA_BTL_UDAPL_FRAG_ALIGN byte boundary */
};
typedef struct mca_btl_udapl_rdma_footer_t mca_btl_udapl_rdma_footer_t;
struct mca_btl_udapl_segment_t {
mca_btl_base_segment_t base;
DAT_RMR_CONTEXT context;
};
typedef struct mca_btl_udapl_segment_t mca_btl_udapl_segment_t;
/**
* uDAPL fragment derived type.
*/
struct mca_btl_udapl_frag_t {
mca_btl_base_descriptor_t base;
mca_btl_udapl_segment_t segment;
struct mca_btl_udapl_module_t* btl;
struct mca_btl_base_endpoint_t* endpoint;
DAT_LMR_TRIPLET triplet;
struct mca_btl_udapl_reg_t* registration;
mca_btl_udapl_footer_t* ftr;
mca_btl_udapl_rdma_footer_t* rdma_ftr;
size_t size;
mca_btl_udapl_frag_type_t type;
uint32_t pad; /* Padding the structure to be evenly divisble by MCA_BTL_UDAPL_FRAG_ALIGN */
};
typedef struct mca_btl_udapl_frag_t mca_btl_udapl_frag_t;
OBJ_CLASS_DECLARATION(mca_btl_udapl_frag_t);
typedef struct mca_btl_udapl_frag_t mca_btl_udapl_frag_eager_t;
OBJ_CLASS_DECLARATION(mca_btl_udapl_frag_eager_t);
typedef struct mca_btl_udapl_frag_t mca_btl_udapl_frag_max_t;
OBJ_CLASS_DECLARATION(mca_btl_udapl_frag_max_t);
typedef struct mca_btl_udapl_frag_t mca_btl_udapl_frag_user_t;
OBJ_CLASS_DECLARATION(mca_btl_udapl_frag_user_t);
typedef struct mca_btl_udapl_frag_t mca_btl_udapl_frag_eager_rdma_t;
OBJ_CLASS_DECLARATION(mca_btl_udapl_frag_eager_rdma_t);
/*
* Macros to allocate/return descriptors from module specific
* free list(s).
*/
#define MCA_BTL_UDAPL_FRAG_ALLOC_EAGER(btl, frag) \
{ \
ompi_free_list_item_t *item; \
OMPI_FREE_LIST_GET_MT(&((mca_btl_udapl_module_t*)btl)->udapl_frag_eager, item); \
frag = (mca_btl_udapl_frag_t*) item; \
}
#define MCA_BTL_UDAPL_FRAG_RETURN_EAGER(btl, frag) \
{ \
OMPI_FREE_LIST_RETURN_MT(&((mca_btl_udapl_module_t*)btl)->udapl_frag_eager, \
(ompi_free_list_item_t*)(frag)); \
}
#define MCA_BTL_UDAPL_FRAG_ALLOC_EAGER_RECV(btl, frag) \
{ \
ompi_free_list_item_t *item; \
OMPI_FREE_LIST_GET_MT(&((mca_btl_udapl_module_t*)btl)->udapl_frag_eager_recv, item); \
frag = (mca_btl_udapl_frag_t*) item; \
}
#define MCA_BTL_UDAPL_FRAG_ALLOC_MAX(btl, frag) \
{ \
ompi_free_list_item_t *item; \
OMPI_FREE_LIST_GET_MT(&((mca_btl_udapl_module_t*)btl)->udapl_frag_max, item); \
frag = (mca_btl_udapl_frag_t*) item; \
}
#define MCA_BTL_UDAPL_FRAG_RETURN_MAX(btl, frag) \
{ \
OMPI_FREE_LIST_RETURN_MT(&((mca_btl_udapl_module_t*)btl)->udapl_frag_max, \
(ompi_free_list_item_t*)(frag)); \
}
#define MCA_BTL_UDAPL_FRAG_ALLOC_MAX_RECV(btl, frag) \
{ \
ompi_free_list_item_t *item; \
OMPI_FREE_LIST_GET_MT(&((mca_btl_udapl_module_t*)btl)->udapl_frag_max_recv, item); \
frag = (mca_btl_udapl_frag_t*) item; \
}
#define MCA_BTL_UDAPL_FRAG_ALLOC_USER(btl, frag) \
{ \
ompi_free_list_item_t *item; \
OMPI_FREE_LIST_GET_MT(&((mca_btl_udapl_module_t*)btl)->udapl_frag_user, item); \
frag = (mca_btl_udapl_frag_t*) item; \
}
#define MCA_BTL_UDAPL_FRAG_RETURN_USER(btl, frag) \
{ \
OMPI_FREE_LIST_RETURN_MT(&((mca_btl_udapl_module_t*)btl)->udapl_frag_user, \
(ompi_free_list_item_t*)(frag)); \
}
#define MCA_BTL_UDAPL_FRAG_ALLOC_CONTROL(btl, frag) \
{ \
ompi_free_list_item_t *item; \
OMPI_FREE_LIST_GET_MT(&((mca_btl_udapl_module_t*)btl)->udapl_frag_control, item); \
frag = (mca_btl_udapl_frag_t*) item; \
}
#define MCA_BTL_UDAPL_FRAG_RETURN_CONTROL(btl, frag) \
{ \
OMPI_FREE_LIST_RETURN_MT(&((mca_btl_udapl_module_t*)btl)->udapl_frag_control, \
(ompi_free_list_item_t*)(frag)); \
}
/*
* Calculate the pad value P required to align the given size S
*/
#define MCA_BTL_UDAPL_FRAG_CALC_ALIGNMENT_PAD(P,S) do { \
(P) = ((S) % MCA_BTL_UDAPL_FRAG_ALIGN) == 0 ? \
0 : (MCA_BTL_UDAPL_FRAG_ALIGN - ((S) % MCA_BTL_UDAPL_FRAG_ALIGN)); \
} while (0);
END_C_DECLS
#endif

314
ompi/mca/btl/udapl/btl_udapl_mca.c
Просмотреть файл

@ -1,314 +0,0 @@
/*
* Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
* University Research and Technology
* Corporation. All rights reserved.
* Copyright (c) 2004-2005 The University of Tennessee and The University
* of Tennessee Research Foundation. All rights
* reserved.
* Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
* University of Stuttgart. All rights reserved.
* Copyright (c) 2004-2005 The Regents of the University of California.
* All rights reserved.
* Copyright (c) 2006 Cisco Systems, Inc. All rights reserved.
* Copyright (c) 2007-2008 Sun Microsystems, Inc. All rights reserved.
* $COPYRIGHT$
*
* Additional copyrights may follow
*
* $HEADER$
*/
#include "ompi_config.h"
#include "opal/mca/base/base.h"
#include "btl_udapl.h"
#include "btl_udapl_mca.h"
#include <string.h>
/*
* Utility routine for string parameter registration.
*
* @param param_name (IN) MCA parameter name
* @param param_desc (IN) MCA parameter description
* @param default_value (IN) MCA parameter default value
* @param out_value (OUT) value of MCA parameter; either default,
* or value as determined from typical
* MCA parameter setting methods
* @param flags (IN) MCA parameter boundary flag
* @return OMPI_SUCCESS or OMPI_ERR_BAD_PARAM
*/
static inline int mca_btl_udapl_reg_string(const char* param_name,
const char* param_desc,
const char* default_value,
char **storage, int flags)
{
*storage = default_value;
(void) mca_base_component_var_register(&mca_btl_udapl_component.super.btl_version, param_name,
param_desc, MCA_BASE_VAR_TYPE_STRING, NULL, 0, 0,
OPAL_INFO_LVL_9,
MCA_BASE_VAR_SCOPE_READONLY, storage);
if (NULL == *storage && !((flags & REGSTR_EMPTY_OK) == REGSTR_EMPTY_OK)) {
BTL_ERROR(("ERROR: MCA Parameter %s : Value (NULL) out of range : "
"Default value (%s)\n \t Parameter Description : %s",
param_name, default_value, param_desc));
return OMPI_ERR_BAD_PARAM;
}
if ((flags & REGSTR_EMPTY_NOT_OK) && 0 == strlen(*storage)) {
BTL_ERROR(("ERROR: MCA Parameter %s : Value (%s) out of range : "
"Default value (%s)\n \t Parameter Description : %s",
param_name, *storage, default_value, param_desc));
return OMPI_ERR_BAD_PARAM;
}
return OMPI_SUCCESS;
}
/*
* Utility routine for integer parameter registration.
*
* @param param_name (IN) MCA parameter name
* @param param_desc (IN) MCA parameter description
* @param default_value (IN) MCA parameter default value
* @param out_value (OUT) value of MCA parameter; either default,
* or value as determined from typical
* MCA parameter setting methods
* @param flags (IN) MCA parameter boundary flag
* @return OMPI_SUCCESS or OMPI_ERR_BAD_PARAM
*/
static inline int mca_btl_udapl_reg_int(const char* param_name,
const char* param_desc,
int default_value, int *storage,
int flags)
{
*storage = default_value;
(void) mca_base_component_var_register(&mca_btl_udapl_component.super.btl_version, param_name,
param_desc, MCA_BASE_VAR_TYPE_INT, NULL, 0, 0,
OPAL_INFO_LVL_9,
MCA_BASE_VAR_SCOPE_READONLY, storage);
if ((flags & REGINT_NEG_ONE_OK) && -1 == *storage) {
return OMPI_SUCCESS;
}
if (((flags & REGINT_GE_ZERO) && *storage < 0) ||
((flags & REGINT_GE_ONE) && *storage < 1) ||
((flags & REGINT_NONZERO) && 0 == *storage)) {
BTL_ERROR(("ERROR: MCA Parameter %s : Value (%d) out of range : "
"Default value (%d)\n \t Parameter Description : %s\n",
param_name, *storage, default_value, param_desc));
return OMPI_ERR_BAD_PARAM;
}
return OMPI_SUCCESS;
}
/*
* Register and check all MCA parameters
*
* @return OMPI_SUCCESS or OMPI_ERR_BAD_PARAM
*/
int mca_btl_udapl_register_mca_params(void)
{
int rc, tmp_rc;
rc = OMPI_SUCCESS;
/* register uDAPL component parameters */
CHECK_PARAM_REGISTER_RETURN_VALUE(mca_btl_udapl_reg_int("free_list_num",
"Initial size of free lists (must be >= 1).",
8,
&mca_btl_udapl_component.udapl_free_list_num,
REGINT_GE_ONE), tmp_rc, rc);
CHECK_PARAM_REGISTER_RETURN_VALUE(mca_btl_udapl_reg_int("free_list_max",
"Maximum size of free lists "
"(-1 = infinite, otherwise must be >= 1).",
-1,
&mca_btl_udapl_component.udapl_free_list_max,
REGINT_NEG_ONE_OK | REGINT_GE_ONE), tmp_rc, rc);
CHECK_PARAM_REGISTER_RETURN_VALUE(mca_btl_udapl_reg_int("free_list_inc",
"Increment size of free lists (must be >= 1).",
8,
&mca_btl_udapl_component.udapl_free_list_inc,
REGINT_GE_ONE), tmp_rc, rc);
CHECK_PARAM_REGISTER_RETURN_VALUE(mca_btl_udapl_reg_string("mpool",
"Name of the memory pool to be used.",
"grdma",
&mca_btl_udapl_component.udapl_mpool_name,
REGSTR_EMPTY_NOT_OK), tmp_rc, rc);
CHECK_PARAM_REGISTER_RETURN_VALUE(mca_btl_udapl_reg_int("max_modules",
"Maximum number of supported HCAs.",
8,
&mca_btl_udapl_component.udapl_max_btls,
REGINT_GE_ONE), tmp_rc, rc);
CHECK_PARAM_REGISTER_RETURN_VALUE(mca_btl_udapl_reg_int("num_recvs",
"Total number of receive buffers to keep posted "
"per endpoint (must be >= 1).",
8,
&mca_btl_udapl_component.udapl_num_recvs,
REGINT_GE_ONE), tmp_rc, rc);
CHECK_PARAM_REGISTER_RETURN_VALUE(mca_btl_udapl_reg_int("num_sends",
"Maximum number of sends to post on an endpoint "
"(must be >= 1).",
7,
&mca_btl_udapl_component.udapl_num_sends,
REGINT_GE_ONE), tmp_rc, rc);
CHECK_PARAM_REGISTER_RETURN_VALUE(mca_btl_udapl_reg_int("sr_win",
"Window size at which point an explicit "
"credit message will be generated (must be >= 1).",
4,
&mca_btl_udapl_component.udapl_sr_win,
REGINT_GE_ONE), tmp_rc, rc);
CHECK_PARAM_REGISTER_RETURN_VALUE(mca_btl_udapl_reg_int("use_eager_rdma",
"Use of RDMA for small messages : "
"1 = default, use RDMA for small messages; "
"0 = do not use RDMA for small messages. ",
1,
&mca_btl_udapl_component.udapl_use_eager_rdma,
REGINT_GE_ZERO), tmp_rc, rc);
CHECK_PARAM_REGISTER_RETURN_VALUE(mca_btl_udapl_reg_int("eager_rdma_num",
"Number of RDMA buffers to allocate "
"for small messages (must be >= 1).",
32,
&mca_btl_udapl_component.udapl_eager_rdma_num,
REGINT_GE_ONE), tmp_rc, rc);
CHECK_PARAM_REGISTER_RETURN_VALUE(mca_btl_udapl_reg_int("max_eager_rdma_peers",
"Maximum number of peers allowed to use "
"RDMA for short messages (independently RDMA will "
"still be used for large messages, (must be >= 0; "
"if zero then RDMA will not be used for short messages).",
16,
&mca_btl_udapl_component.udapl_max_eager_rdma_peers,
REGINT_GE_ZERO), tmp_rc, rc);
CHECK_PARAM_REGISTER_RETURN_VALUE(mca_btl_udapl_reg_int("eager_rdma_win",
"Window size at which point an explicit "
"credit message will be generated (must be >= 1).",
28,
&mca_btl_udapl_component.udapl_eager_rdma_win,
REGINT_GE_ONE), tmp_rc, rc);
CHECK_PARAM_REGISTER_RETURN_VALUE(mca_btl_udapl_reg_int("timeout",
"Connection timeout, in microseconds.",
MCA_BTL_UDAPL_CONN_TIMEOUT_DEFAULT,
&mca_btl_udapl_component.udapl_timeout,
REGINT_GE_ONE), tmp_rc, rc);
CHECK_PARAM_REGISTER_RETURN_VALUE(mca_btl_udapl_reg_int("conn_priv_data",
"Use connect private data to establish connections "
"(not supported by all uDAPL implementations).",
0,
&mca_btl_udapl_component.udapl_conn_priv_data,
REGINT_GE_ZERO), tmp_rc, rc);
CHECK_PARAM_REGISTER_RETURN_VALUE(mca_btl_udapl_reg_int("async_events",
"The asynchronous event queue will only be "
"checked after entering progress this number of times.",
100000000,
&mca_btl_udapl_component.udapl_async_events,
REGINT_GE_ONE), tmp_rc, rc);
CHECK_PARAM_REGISTER_RETURN_VALUE(mca_btl_udapl_reg_int("buffer_alignment",
"Preferred communication buffer alignment, "
"in bytes (must be >= 1).",
DAT_OPTIMAL_ALIGNMENT,
&mca_btl_udapl_component.udapl_buffer_alignment,
REGINT_GE_ONE), tmp_rc, rc);
CHECK_PARAM_REGISTER_RETURN_VALUE(mca_btl_udapl_reg_string("if_include",
"Comma-delimited list of interfaces to be included "
"(e.g. \"ibd0,ibd1 or OpenIB-cma,OpenIB-cma-1\"; empty value means "
"to use all interfaces found). Mutually exclusive with "
"btl_udapl_if_exclude.",
NULL, &mca_btl_udapl_component.if_include,
REGSTR_EMPTY_OK), tmp_rc, rc);
CHECK_PARAM_REGISTER_RETURN_VALUE(mca_btl_udapl_reg_string("if_exclude",
"Comma-delimited list of interfaces to be excluded from use "
"(e.g. \"ibd0,ibd1 or OpenIB-cma,OpenIB-cma-1\"; empty value means "
"not to exclude any). Mutually exclusive with btl_udapl_if_include.",
NULL, &mca_btl_udapl_component.if_exclude,
REGSTR_EMPTY_OK), tmp_rc, rc);
CHECK_PARAM_REGISTER_RETURN_VALUE(mca_btl_udapl_reg_int("verbose",
"Verbosity level of the uDAPL BTL (-1 thru 100)",
VERBOSE_SHOW_HELP,
&(mca_btl_udapl_component.udapl_verbosity),
REGINT_NEG_ONE_OK), tmp_rc, rc);
CHECK_PARAM_REGISTER_RETURN_VALUE(mca_btl_udapl_reg_int("compare_subnet",
"By default uDAPL BTL will compare subnets using netmask to "
"determine if an interface is reachable. Setting this parameter to "
"0 will essentially turn this comparison off and the uDAPL BTL will "
"assume all uDAPL interfaces are reachable (0 or 1, default==1).",
1,
&(mca_btl_udapl_component.udapl_compare_subnet),
REGINT_GE_ZERO), tmp_rc, rc);
/* register uDAPL module parameters */
CHECK_PARAM_REGISTER_RETURN_VALUE(mca_btl_udapl_reg_int("async_evd_qlen",
"The asynchronous event dispatcher queue length.",
MCA_BTL_UDAPL_ASYNC_EVD_QLEN_DEFAULT,
(int*)&mca_btl_udapl_module.udapl_async_evd_qlen,
REGINT_GE_ONE), tmp_rc, rc);
CHECK_PARAM_REGISTER_RETURN_VALUE(mca_btl_udapl_reg_int("conn_evd_qlen",
"The connection event dispatcher queue length is "
"a function of the number of connections expected.",
MCA_BTL_UDAPL_CONN_EVD_QLEN_DEFAULT,
(int*)&mca_btl_udapl_module.udapl_conn_evd_qlen,
REGINT_GE_ONE), tmp_rc, rc);
CHECK_PARAM_REGISTER_RETURN_VALUE(mca_btl_udapl_reg_int("dto_evd_qlen",
"The data transfer operation event dispatcher queue length is "
"a function of the number of connections as well as the "
"maximum number of outstanding data transfer operations.",
MCA_BTL_UDAPL_DTO_EVD_QLEN_DEFAULT,
(int*)&mca_btl_udapl_module.udapl_dto_evd_qlen,
REGINT_GE_ONE), tmp_rc, rc);
CHECK_PARAM_REGISTER_RETURN_VALUE(mca_btl_udapl_reg_int("max_request_dtos",
"Maximum number of outstanding "
"submitted sends and rdma operations per endpoint, (see Section "
"6.6.6 of uDAPL Spec.).",
MCA_BTL_UDAPL_MAX_REQUEST_DTOS_DEFAULT,
(int*)&mca_btl_udapl_module.udapl_max_request_dtos,
REGINT_GE_ONE), tmp_rc, rc);
CHECK_PARAM_REGISTER_RETURN_VALUE(mca_btl_udapl_reg_int("max_recv_dtos",
"Maximum number of outstanding "
"submitted receive operations per endpoint, (see Section "
"6.6.6 of uDAPL Spec.).",
MCA_BTL_UDAPL_MAX_RECV_DTOS_DEFAULT,
(int*)&mca_btl_udapl_module.udapl_max_recv_dtos,
REGINT_GE_ONE), tmp_rc, rc);
mca_btl_udapl_module.super.btl_exclusivity =
MCA_BTL_EXCLUSIVITY_DEFAULT - 10;
mca_btl_udapl_module.super.btl_eager_limit = 8*1024;
mca_btl_udapl_module.super.btl_rndv_eager_limit = 8*1024;
mca_btl_udapl_module.super.btl_max_send_size = 64*1024;
mca_btl_udapl_module.super.btl_rdma_pipeline_send_length = 512*1024;
mca_btl_udapl_module.super.btl_rdma_pipeline_frag_size = 128 * 1024;
mca_btl_udapl_module.super.btl_min_rdma_pipeline_size = 0;
mca_btl_udapl_module.super.btl_flags = MCA_BTL_FLAGS_PUT | MCA_BTL_FLAGS_SEND;
mca_btl_udapl_module.super.btl_bandwidth = 225;
mca_btl_udapl_module.super.btl_latency = 0;
mca_btl_base_param_register(&mca_btl_udapl_component.super.btl_version,
&mca_btl_udapl_module.super);
return rc;
}

63
ompi/mca/btl/udapl/btl_udapl_mca.h
Просмотреть файл

@ -1,63 +0,0 @@
/*
* Copyright (c) 2006 Cisco Systems, Inc. All rights reserved.
* Copyright (c) 2007 Sun Microsystems, Inc. All rights reserved.
* $COPYRIGHT$
*
* Additional copyrights may follow
*
* $HEADER$
*/
#ifndef MCA_BTL_UDAPL_MCA_H
#define MCA_BTL_UDAPL_MCA_H
BEGIN_C_DECLS
/* Define Integer Boundaries */
#define REGINT_NEG_ONE_OK 0x01 /* value = -1 is valid */
#define REGINT_GE_ZERO 0x02 /* value >= 0 is valid */
#define REGINT_GE_ONE 0x04 /* value > 1 is valid */
#define REGINT_NONZERO 0x08 /* value != 0 is valid */
/* Define String Boundaries */
#define REGSTR_EMPTY_OK 0x01 /* empty string is valid */
#define REGSTR_EMPTY_NOT_OK 0x02 /* empty string is not valid */
/* Define default parameter values that need to be known beyond the
* initial setting; for example, if a parameter is tuned dynamically
* by the BTL it would not be advisable to do so if the user has
* modified the default.
*/
#define MCA_BTL_UDAPL_ASYNC_EVD_QLEN_DEFAULT 256
#define MCA_BTL_UDAPL_CONN_EVD_QLEN_DEFAULT 256
#define MCA_BTL_UDAPL_DTO_EVD_QLEN_DEFAULT 256
#define MCA_BTL_UDAPL_CONN_TIMEOUT_DEFAULT 10000000
#define MCA_BTL_UDAPL_CONN_TIMEOUT_INC 200000 /* connection timeout
* is in microseconds;
* this incremental
* value is equivalent
* to .2 seconds
*/
#define MCA_BTL_UDAPL_CONN_TIMEOUT_MAX 2147483647
#define MCA_BTL_UDAPL_MAX_RECV_DTOS_DEFAULT 8
#define MCA_BTL_UDAPL_MAX_REQUEST_DTOS_DEFAULT 76
#define MCA_BTL_UDAPL_NUM_RECVS_DEFAULT 8
#define CHECK_PARAM_REGISTER_RETURN_VALUE(expr, tmp_rc, rc) \
{ \
tmp_rc = (expr); \
if (OMPI_SUCCESS != tmp_rc) { \
rc = tmp_rc; \
} \
}
/**
* Function to register MCA params and check for sane values
*/
int mca_btl_udapl_register_mca_params(void);
END_C_DECLS
#endif

312
ompi/mca/btl/udapl/btl_udapl_proc.c
Просмотреть файл

@ -1,312 +0,0 @@
/*
* Copyright (c) 2004-2006 The Trustees of Indiana University and Indiana
* University Research and Technology
* Corporation. All rights reserved.
* Copyright (c) 2004-2011 The University of Tennessee and The University
* of Tennessee Research Foundation. All rights
* reserved.
* Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
* University of Stuttgart. All rights reserved.
* Copyright (c) 2004-2005 The Regents of the University of California.
* All rights reserved.
* Copyright (c) 2006 Sandia National Laboratories. All rights
* reserved.
* Copyright (c) 2008 Sun Microsystems, Inc. All rights reserved.
* Copyright (c) 2013 Intel, Inc. All rights reserved
* $COPYRIGHT$
*
* Additional copyrights may follow
*
* $HEADER$
*/
#include "ompi_config.h"
#include "ompi/runtime/ompi_module_exchange.h"
#include "opal/util/net.h"
#include "btl_udapl.h"
#include "btl_udapl_endpoint.h"
#include "btl_udapl_proc.h"
static void mca_btl_udapl_proc_construct(mca_btl_udapl_proc_t* proc);
static void mca_btl_udapl_proc_destruct(mca_btl_udapl_proc_t* proc);
OBJ_CLASS_INSTANCE(mca_btl_udapl_proc_t,
opal_list_item_t, mca_btl_udapl_proc_construct,
mca_btl_udapl_proc_destruct);
void mca_btl_udapl_proc_construct(mca_btl_udapl_proc_t* udapl_proc)
{
udapl_proc->proc_ompi = 0;
udapl_proc->proc_addr_count = 0;
udapl_proc->proc_endpoints = 0;
udapl_proc->proc_endpoint_count = 0;
OBJ_CONSTRUCT(&udapl_proc->proc_lock, opal_mutex_t);
/* add to list of all proc instance */
OPAL_THREAD_LOCK(&mca_btl_udapl_component.udapl_lock);
opal_list_append(&mca_btl_udapl_component.udapl_procs, &udapl_proc->super);
OPAL_THREAD_UNLOCK(&mca_btl_udapl_component.udapl_lock);
}
/*
* Cleanup uDAPL proc instance
*/
void mca_btl_udapl_proc_destruct(mca_btl_udapl_proc_t* udapl_proc)
{
/* remove from list of all proc instances */
OPAL_THREAD_LOCK(&mca_btl_udapl_component.udapl_lock);
opal_list_remove_item(&mca_btl_udapl_component.udapl_procs, &udapl_proc->super);
OPAL_THREAD_UNLOCK(&mca_btl_udapl_component.udapl_lock);
/* release resources */
if(NULL != udapl_proc->proc_endpoints) {
free(udapl_proc->proc_endpoints);
}
OBJ_DESTRUCT(&udapl_proc->proc_lock);
}
/*
* Look for an existing uDAPL process instances based on the associated
* ompi_proc_t instance.
*/
static mca_btl_udapl_proc_t* mca_btl_udapl_proc_lookup_ompi(ompi_proc_t* ompi_proc)
{
mca_btl_udapl_proc_t* udapl_proc;
OPAL_THREAD_LOCK(&mca_btl_udapl_component.udapl_lock);
for(udapl_proc = (mca_btl_udapl_proc_t*)
opal_list_get_first(&mca_btl_udapl_component.udapl_procs);
udapl_proc != (mca_btl_udapl_proc_t*)
opal_list_get_end(&mca_btl_udapl_component.udapl_procs);
udapl_proc = (mca_btl_udapl_proc_t*)opal_list_get_next(udapl_proc)) {
if(udapl_proc->proc_ompi == ompi_proc) {
OPAL_THREAD_UNLOCK(&mca_btl_udapl_component.udapl_lock);
return udapl_proc;
}
}
OPAL_THREAD_UNLOCK(&mca_btl_udapl_component.udapl_lock);
return NULL;
}
/*
* Create a uDAPL process structure. There is a one-to-one correspondence
* between a ompi_proc_t and a mca_btl_udapl_proc_t instance. We cache
* additional data (specifically the list of mca_btl_udapl_endpoint_t instances,
* and published addresses) associated w/ a given destination on this
* datastructure.
*/
mca_btl_udapl_proc_t* mca_btl_udapl_proc_create(ompi_proc_t* ompi_proc)
{
mca_btl_udapl_proc_t* udapl_proc = NULL;
size_t size;
int rc;
/* Check if we have already created a uDAPL proc
* structure for this ompi process */
udapl_proc = mca_btl_udapl_proc_lookup_ompi(ompi_proc);
if(udapl_proc != NULL) {
return udapl_proc;
}
/* create a new udapl proc out of the ompi_proc ... */
udapl_proc = OBJ_NEW(mca_btl_udapl_proc_t);
udapl_proc->proc_endpoint_count = 0;
udapl_proc->proc_ompi = ompi_proc;
/* query for the peer address info */
rc = ompi_modex_recv(
&mca_btl_udapl_component.super.btl_version,
ompi_proc,
(void*)&udapl_proc->proc_addrs,
&size);
if(OMPI_SUCCESS != rc) {
BTL_UDAPL_VERBOSE_OUTPUT(VERBOSE_CRITICAL,
("ompi_modex_recv failed for peer %s",
OMPI_NAME_PRINT(&ompi_proc->proc_name)));
OBJ_RELEASE(udapl_proc);
return NULL;
}
if((size % sizeof(mca_btl_udapl_addr_t)) != 0) {
BTL_UDAPL_VERBOSE_OUTPUT(VERBOSE_CRITICAL,
("invalid udapl address for peer %s",
OMPI_NAME_PRINT(&ompi_proc->proc_name)));
OBJ_RELEASE(udapl_proc);
return NULL;
}
udapl_proc->proc_addr_count = size/sizeof(mca_btl_udapl_addr_t);
if (0 == udapl_proc->proc_addr_count) {
udapl_proc->proc_endpoints = NULL;
} else {
udapl_proc->proc_endpoints = (mca_btl_base_endpoint_t**)
malloc(udapl_proc->proc_addr_count * sizeof(mca_btl_base_endpoint_t*));
}
if(NULL == udapl_proc->proc_endpoints) {
OBJ_RELEASE(udapl_proc);
return NULL;
}
return udapl_proc;
}
/*
* Find an address on the peer_process which matches stated criteria
* to the udapl btl module address information. Return in peer_addr_idx
* the index to the peer_process address that matches the btl module
* address. Where match criteria is:
* - the address in not already in use
* - compare addresses using netmask, the netmask value can be modified with
* "--mca btl_udapl_if_mask"
*
* Note: since this is called from mca_btl_udapl_proc_insert() it
* is assumed that the process lock is locked when entered.
*
* @param udapl_btl (IN) BTL module
* @param peer_process (IN) BTL peer process
* @param peer_addr_idx(IN/OUT) Index of address on peer_process
* which matches the udapl_btl address data.
* On success should be >= 0.
* @return OMPI_SUCCESS or error status on failure
*/
static int mca_btl_udapl_proc_address_match(
mca_btl_udapl_module_t* udapl_btl,
mca_btl_udapl_proc_t* peer_proc,
int* peer_addr_idx)
{
int i;
struct sockaddr *saddr;
struct sockaddr_in *btl_addr;
struct sockaddr_in *peer_addr;
char btl_addr_string[INET_ADDRSTRLEN];
char peer_addr_string[INET_ADDRSTRLEN];
*peer_addr_idx = MCA_BTL_UDAPL_INVALID_PEER_ADDR_IDX;
/* use generic address to find address family */
saddr = (struct sockaddr *)&(udapl_btl->udapl_addr.addr);
if (saddr->sa_family == AF_INET) {
btl_addr = (struct sockaddr_in *)saddr;
/* Loop thru peer process addresses looking for match.
* Match criteria:
* - address should not be "inuse"
* - both udapl btl module and peer address should be on
* the same subnet (compare with if_mask value)
*/
for(i = 0; i < (int) peer_proc->proc_addr_count; i++) {
peer_addr =
(struct sockaddr_in *)&(peer_proc->proc_addrs[i].addr);
if (VERBOSE_INFORM <=
mca_btl_udapl_component.udapl_verbosity) {
/* retrieve udapl btl and peer address string for reporting */
inet_ntop(AF_INET, (void *) &btl_addr->sin_addr,
btl_addr_string, INET_ADDRSTRLEN);
inet_ntop(AF_INET, (void *) &peer_addr->sin_addr,
peer_addr_string, INET_ADDRSTRLEN);
}
if ((false == peer_proc->proc_addrs[i].inuse) &&
(opal_net_samenetwork((struct sockaddr *)btl_addr,
(struct sockaddr *)peer_addr, udapl_btl->udapl_if_mask))) {
/* capture index of remote address where match found */
*peer_addr_idx = i;
/* mark this address as now being used */
peer_proc->proc_addrs[i].inuse = true;
/* report what address was found to match */
BTL_UDAPL_VERBOSE_OUTPUT(VERBOSE_INFORM,
("uDAPL BTL module(%s) matched %s",
btl_addr_string, peer_addr_string));
break;
} else {
/* peer address already used by another udapl btl
* module or netmask check not successful so skip
*/
BTL_UDAPL_VERBOSE_OUTPUT(VERBOSE_INFORM,
("uDAPL BTL module(%s) either skipped because it "
"is already in use or match criteria not successful "
"for peer address %s",
btl_addr_string, peer_addr_string));
}
}
} else {
/* current uDAPL BTL only supports IPv4 */
BTL_UDAPL_VERBOSE_HELP(VERBOSE_SHOW_HELP,
("help-mpi-btl-udapl.txt", "IPv4 only",
true, ompi_process_info.nodename));
return OMPI_ERROR;
}
if (MCA_BTL_UDAPL_INVALID_PEER_ADDR_IDX == *peer_addr_idx) {
BTL_UDAPL_VERBOSE_HELP(VERBOSE_SHOW_HELP,
("help-mpi-btl-udapl.txt", "no network match",
true, btl_addr_string, ompi_process_info.nodename,
(NULL == peer_proc->proc_ompi->proc_hostname) ?
"unknown" : peer_proc->proc_ompi->proc_hostname));
return OMPI_ERR_OUT_OF_RESOURCE;
}
return OMPI_SUCCESS;
}
/*
* Note that this routine must be called with the lock on the process
* already held. Insert a btl instance into the proc array and assign
* it an address.
*/
int mca_btl_udapl_proc_insert(
mca_btl_udapl_proc_t* udapl_proc,
mca_btl_base_endpoint_t* udapl_endpoint)
{
int peer_address_idx;
mca_btl_udapl_module_t* udapl_btl = udapl_endpoint->endpoint_btl;
/* Check so as not to create more endpoints than addresses.
* Example: If one node has 3 btl modules and another only has 2,
* this check prevents the node with 3 btl modules from
* overloading the other, i.e. only 2 possible connections will
* be possible.
*/
if (udapl_proc->proc_endpoint_count > udapl_proc->proc_addr_count)
return OMPI_ERR_OUT_OF_RESOURCE;
/* Find an endpoint on the udapl process of interest that matches
* the endpoint information of the current udapl btl module
*/
if (OMPI_SUCCESS !=
mca_btl_udapl_proc_address_match(udapl_btl, udapl_proc,
&peer_address_idx)) {
/* no address on peer proc met criteria */
return OMPI_ERR_OUT_OF_RESOURCE;
}
/* insert into endpoint array */
udapl_endpoint->endpoint_proc = udapl_proc;
udapl_endpoint->endpoint_addr =
udapl_proc->proc_addrs[peer_address_idx];
udapl_proc->proc_endpoints[udapl_proc->proc_endpoint_count] = udapl_endpoint;
udapl_proc->proc_endpoint_count++;
return OMPI_SUCCESS;
}

67
ompi/mca/btl/udapl/btl_udapl_proc.h
Просмотреть файл

@ -1,67 +0,0 @@
/*
* Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
* University Research and Technology
* Corporation. All rights reserved.
* Copyright (c) 2004-2011 The University of Tennessee and The University
* of Tennessee Research Foundation. All rights
* reserved.
* Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
* University of Stuttgart. All rights reserved.
* Copyright (c) 2004-2005 The Regents of the University of California.
* All rights reserved.
* Copyright (c) 2008 Sun Microsystems, Inc. All rights reserved.
* $COPYRIGHT$
*
* Additional copyrights may follow
*
* $HEADER$
*/
#ifndef MCA_BTL_UDAPL_PROC_H
#define MCA_BTL_UDAPL_PROC_H
#include "opal/class/opal_object.h"
#include "ompi/proc/proc.h"
#include "btl_udapl.h"
#include "btl_udapl_endpoint.h"
BEGIN_C_DECLS
/**
* Represents the state of a remote process and the set of addresses
* that it exports. Also cache an instance of mca_btl_base_endpoint_t for
* each
* BTL instance that attempts to open a connection to the process.
*/
struct mca_btl_udapl_proc_t {
opal_list_item_t super;
/**< allow proc to be placed on a list */
ompi_proc_t *proc_ompi;
/**< pointer to corresponding ompi_proc_t */
struct mca_btl_udapl_addr_t* proc_addrs;
/**< array of addresses exported by peer */
size_t proc_addr_count;
/**< number of addresses published by peer */
struct mca_btl_base_endpoint_t **proc_endpoints;
/**< array of endpoints that have been created to access this proc */
size_t proc_endpoint_count;
/**< number of endpoints */
opal_mutex_t proc_lock;
/**< lock to protect against concurrent access to proc state */
};
typedef struct mca_btl_udapl_proc_t mca_btl_udapl_proc_t;
OBJ_CLASS_DECLARATION(mca_btl_udapl_proc_t);
#define MCA_BTL_UDAPL_INVALID_PEER_ADDR_IDX -1
mca_btl_udapl_proc_t* mca_btl_udapl_proc_create(ompi_proc_t* ompi_proc);
int mca_btl_udapl_proc_insert(mca_btl_udapl_proc_t*, mca_btl_base_endpoint_t*);
END_C_DECLS
#endif

62
ompi/mca/btl/udapl/configure.m4
Просмотреть файл

@ -1,62 +0,0 @@
# -*- shell-script -*-
#
# Copyright (c) 2004-2006 The Trustees of Indiana University and Indiana
# University Research and Technology
# Corporation. All rights reserved.
# Copyright (c) 2004-2005 The University of Tennessee and The University
# of Tennessee Research Foundation. All rights
# reserved.
# Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
# University of Stuttgart. All rights reserved.
# Copyright (c) 2004-2005 The Regents of the University of California.
# All rights reserved.
# Copyright (c) 2009 Sun Microsystems, Inc. All rights reserved.
# Copyright (c) 2010 Cisco Systems, Inc. All rights reserved.
# $COPYRIGHT$
#
# Additional copyrights may follow
#
# $HEADER$
#
# MCA_btl_udapl_CONFIG([action-if-can-compile],
# [action-if-cant-compile])
# ------------------------------------------------
AC_DEFUN([MCA_ompi_btl_udapl_CONFIG],[
AC_CONFIG_FILES([ompi/mca/btl/udapl/Makefile])
OMPI_CHECK_UDAPL([btl_udapl],
[btl_udapl_happy="yes"],
[btl_udapl_happy="no"])
AS_IF([test "$btl_udapl_happy" = "yes"],
[$1],
[$2])
# Borrowed from MVAPI BTL - a data structure in the uDAPL headers
# is not fully ISO C. Remove -pedantic to silence a warning.
btl_udapl_CFLAGS="`echo $CFLAGS | sed 's/-pedantic//g'`"
AS_IF([test "$btl_udapl_CFLAGS" != "$CFLAGS" -a "$btl_udapl_happy" = "yes"],
[AC_MSG_WARN([Removed -pedantic from CFLAGS for
uDAPL component because the uDAPL headers are not fully ISO C])])
# Test for uDAPL relaxed ordered specific symbols
AS_IF([test "$btl_udapl_happy" = "yes"],
[AC_MSG_CHECKING(for uDAPL DAT_MEM_TYPE_SO_VIRTUAL)
AC_TRY_COMPILE([#include <dat/udat.h>],
[DAT_MEM_TYPE dmt = DAT_MEM_TYPE_SO_VIRTUAL;],
[AC_MSG_RESULT(yes)
btl_udapl_ro_aware=1],
[AC_MSG_RESULT(no)
btl_udapl_ro_aware=0])
AC_DEFINE_UNQUOTED([HAVE_DAT_MEM_TYPE_SO_VIRTUAL],
[$btl_udapl_ro_aware],
[uDAPL DAT_MEM_TYPE_SO_VIRTUAL check])])
# substitute in the things needed to build udapl
AC_SUBST([btl_udapl_CFLAGS])
AC_SUBST([btl_udapl_CPPFLAGS])
AC_SUBST([btl_udapl_LDFLAGS])
AC_SUBST([btl_udapl_LIBS])
])dnl

139
ompi/mca/btl/udapl/help-mpi-btl-udapl.txt
Просмотреть файл

@ -1,139 +0,0 @@
# -*- text -*-
#
# Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
# University Research and Technology
# Corporation. All rights reserved.
# Copyright (c) 2004-2005 The University of Tennessee and The University
# of Tennessee Research Foundation. All rights
# reserved.
# Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
# University of Stuttgart. All rights reserved.
# Copyright (c) 2004-2006 The Regents of the University of California.
# All rights reserved.
# Copyright (c) 2007-2008 Sun Microsystems, Inc. All rights reserved.
#
# $COPYRIGHT$
#
# Additional copyrights may follow
#
# $HEADER$
#
# This is the US/English general help file for Open MPI.
#
[invalid num rdma segments]
WARNING: MCA parameter [btl_udapl_eager_rdma_num = %d] is not valid.
RDMA will not be used for short messages. Try setting to positive
value, e.g. 16.
#
[use default endpoint params]
WARNING: Using default uDAPL endpoint parameters not those that
would have been modified by MCA parameters.
#
[optimal buffer alignment mismatch]
WARNING: DAT_OPTIMAL_ALIGNMENT = %d : BTL buffer_alignment = %d.
The BTL buffer_alignment value may not be optimal. If all nodes
report the same DAT_OPTIMAL_ALIGNMENT value and this differs from
BTL buffer_alignment then setting "--mca btl_udapl_buffer_alignment
%d" may improve performance.
#
[max_recv_dtos too low]
WARNING: The MCA parameter max_recv_dtos has been modified to a value,
%d, that is insufficient. This value must be greater than or equal to
num_recvs, %d. The uDAPL BTL will adjust to allow the program to
proceed.
#
[max_request_dtos too low]
WARNING: The MCA parameter max_request_dtos has been modified to a
value, %d, which may not be sufficient. Try setting max_request_dtos
to %d if program fails.
#
[max_recv_dtos system max]
WARNING: The MCA parameter max_recv_dtos is trying to be set to,
%d, which is larger than allowable so the value will be set to maximum
allowed, %d.
#
[max_request_dtos system max]
WARNING: The MCA parameter max_request_dtos is trying to be set to,
%d, which is larger than allowable so the value will be set to maximum
allowed, %d.
#
[evd_qlen adapter max]
WARNING: The MCA parameter %s is trying to be set to %d,
which is larger than allowable so the value will be set to maximum
allowed, %d.
#
[evd_qlen too low]
WARNING: The MCA parameter %s has been modified to a value,
%d, which may not be sufficient. Try setting %s to %d if
program fails.
#
[connection timeout low]
WARNING: The MCA parameter %s has been modified to a value,
%d, which may not be sufficient. Try setting %s to %d if
program fails.
#
[dat_lmr_create DAT_INSUFFICIENT_RESOURCES]
WARNING: The uDAPL BTL is not able to register memory. Possibly out of
allowed privileged memory (i.e. memory that can be pinned). Increasing
the allowed privileged memory may alleviate this issue.
#
[dat_ia_open fail]
WARNING: Failed to open "%s" [%s:%s].
This may be a real error or it may be an invalid entry in the uDAPL
Registry which is contained in the dat.conf file. Contact your local
System Administrator to confirm the availability of the interfaces in
the dat.conf file.
#
[specified include and exclude]
ERROR: You have specified both the btl_udapl_if_include and
btl_udapl_if_exclude MCA parameters. These two parameters are
mutually exclusive; you can only specify one or the other.
For reference, the values that you specified are:
btl_udapl_if_include: %s
btl_udapl_if_exclude: %s
#
[nonexistent entry]
WARNING: One or more nonexistent interfaces were specified:
Host: %s
MCA parameter: btl_udapl_if_%sclude
Nonexistent entities: %s
These entities will be ignored.
#
[IPv4 only]
WARNING: uDAPL BTL only supports IPv4 addressing at this time.
Something other than an IPv4 address was detected on %s.
#
[no network match]
WARNING: Interface %s on node %s not able to find matching
interface on peer node %s. Could be that the interfaces are on
different subnets or there are fewer available uDAPL interfaces on peer.
#
[interface not found]
WARNING: Host %s, not able to determine interface name for
address %s. Will attempt to continue, assuming all addresses to
peer are reachable.
#
[netmask not found]
WARNING: Host %s, not able to determine netmask for address
%s. Will attempt to continue assuming all addresses to
peer are reachable.
#
[relaxed order support]
WARNING: While attempting to open interface %s the system reported
DAT_INVALID_RO_COOKIE. This indicates the currrent system supports
relaxed ordering. An attempt will be made to open the interface using
the following modified interface name %s. Open MPI must not use RDMA
for short eager messages in this scenario. Therefore, if opened
successfully RDMA will not be used for short eager messages. This will
negatively impact short message latency.
#
[dat_ia_open fail RO]
WARNING: Failed to open "%s" [%s:%s].
Attempted to call dat_ia_open() on an interface that has been prefixed
with "RO_AWARE_" after first trying to open %s and failed with
DAT_INVALID_RO_COOKIE.
#