6310ce955c
- the registration array is now global instead of one by BTL. - each framework have to declare the entries in the registration array reserved. Then it have to define the internal way of sharing (or not) these entries between all components. As an example, the PML will not share as there is only one active PML at any moment, while the BTLs will have to. The tag is 8 bits long, the first 3 are reserved for the framework while the remaining 5 are use internally by each framework. - The registration function is optional. If a BTL do not provide such function, nothing happens. However, in the case where such function is provided in the BTL structure, it will be called by the BML, when a tag is registered. Now, it's time for the second step... Converting OB1 from a switch based PML to an active message one. This commit was SVN r17140.
337 строки
9.9 KiB
C
337 строки
9.9 KiB
C
/*
|
|
* Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
|
|
* University Research and Technology
|
|
* Corporation. All rights reserved.
|
|
* Copyright (c) 2004-2008 The University of Tennessee and The University
|
|
* of Tennessee Research Foundation. All rights
|
|
* reserved.
|
|
* Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
|
|
* University of Stuttgart. All rights reserved.
|
|
* Copyright (c) 2004-2005 The Regents of the University of California.
|
|
* All rights reserved.
|
|
* $COPYRIGHT$
|
|
*
|
|
* Additional copyrights may follow
|
|
*
|
|
* $HEADER$
|
|
*/
|
|
/**
|
|
* @file
|
|
*/
|
|
#ifndef MCA_BTL_SCTP_H
|
|
#define MCA_BTL_SCTP_H
|
|
|
|
/* Standard system includes */
|
|
#include "ompi_config.h"
|
|
#ifdef HAVE_SYS_TYPES_H
|
|
#include <sys/types.h>
|
|
#endif
|
|
#ifdef HAVE_SYS_SOCKET_H
|
|
#include <sys/socket.h>
|
|
#endif
|
|
#ifdef HAVE_NETINET_IN_H
|
|
#include <netinet/in.h>
|
|
#endif
|
|
|
|
/* Open MPI includes */
|
|
#include "opal/event/event.h"
|
|
#include "opal/util/output.h"
|
|
#include "ompi/class/ompi_bitmap.h"
|
|
#include "ompi/class/ompi_free_list.h"
|
|
#include "ompi/mca/pml/pml.h"
|
|
#include "ompi/mca/btl/btl.h"
|
|
#include "ompi/mca/btl/base/base.h"
|
|
#include "ompi/mca/mpool/mpool.h"
|
|
#include "ompi/mca/btl/btl.h"
|
|
#include "opal/class/opal_hash_table.h"
|
|
|
|
/* For the assocID -> proc mapping, do we use opal_hash_table (0)
|
|
* or our homemade array (1)?
|
|
*/
|
|
#define MCA_BTL_SCTP_DONT_USE_HASH 1
|
|
|
|
#if 1
|
|
/* if you do not want to see these debug messages */
|
|
#define SCTP_BTL_ERROR(args)
|
|
#else
|
|
#define SCTP_BTL_ERROR(args) BTL_ERROR(args)
|
|
#endif
|
|
|
|
#define MCA_BTL_SCTP_STATISTICS 0
|
|
#if defined(c_plusplus) || defined(__cplusplus)
|
|
extern "C" {
|
|
#endif
|
|
|
|
|
|
/**
|
|
* SCTP BTL component.
|
|
*/
|
|
|
|
struct mca_btl_sctp_component_t {
|
|
mca_btl_base_component_1_0_1_t super; /**< base BTL component */
|
|
uint32_t sctp_num_btls; /**< number of hcas available to the SCTP component */
|
|
struct mca_btl_sctp_module_t **sctp_btls; /**< array of available BTL modules */
|
|
struct mca_btl_sctp_proc_t* sctp_local; /**< local proc struct */
|
|
int sctp_free_list_num; /**< initial size of free lists */
|
|
int sctp_free_list_max; /**< maximum size of free lists */
|
|
int sctp_free_list_inc; /**< number of elements to alloc when growing free lists */
|
|
int sctp_endpoint_cache; /**< amount of cache on each endpoint */
|
|
opal_hash_table_t sctp_procs; /**< hash table of sctp proc structures */
|
|
#if MCA_BTL_SCTP_DONT_USE_HASH
|
|
#else
|
|
opal_hash_table_t sctp_assocID_hash; /**< hash table of procs keyed on assocIDs */
|
|
#endif
|
|
opal_list_t sctp_events; /**< list of pending sctp events */
|
|
opal_mutex_t sctp_lock; /**< lock for accessing module state */
|
|
opal_event_t sctp_recv_event; /**< recv event for listen socket */
|
|
int sctp_listen_sd; /**< listen socket for incoming connection requests */
|
|
unsigned short sctp_listen_port; /**< listen port */
|
|
char* sctp_if_include; /**< comma seperated list of interface to include */
|
|
char* sctp_if_exclude; /**< comma seperated list of interface to exclude */
|
|
int sctp_sndbuf; /**< socket sndbuf size */
|
|
int sctp_rcvbuf; /**< socket rcvbuf size */
|
|
int sctp_use_nodelay; /**< SCTP_NODELAY value */
|
|
int sctp_if_11; /**< Are we going 1 to 1? 1 to many is default. */
|
|
|
|
/* free list of fragment descriptors */
|
|
ompi_free_list_t sctp_frag_eager;
|
|
ompi_free_list_t sctp_frag_max;
|
|
ompi_free_list_t sctp_frag_user;
|
|
};
|
|
typedef struct mca_btl_sctp_component_t mca_btl_sctp_component_t;
|
|
|
|
OMPI_MODULE_DECLSPEC extern mca_btl_sctp_component_t mca_btl_sctp_component;
|
|
|
|
/**
|
|
* BTL Module Interface
|
|
*/
|
|
struct mca_btl_sctp_module_t {
|
|
mca_btl_base_module_t super; /**< base BTL interface */
|
|
int sctp_ifindex; /**< PTL interface index */
|
|
struct sockaddr_in sctp_ifaddr; /**< PTL interface address */
|
|
struct sockaddr_in sctp_ifmask; /**< PTL interface netmask */
|
|
opal_list_t sctp_endpoints;
|
|
#if MCA_BTL_SCTP_STATISTICS
|
|
size_t sctp_bytes_sent;
|
|
size_t sctp_bytes_recv;
|
|
size_t sctp_send_handler;
|
|
#endif
|
|
};
|
|
typedef struct mca_btl_sctp_module_t mca_btl_sctp_module_t;
|
|
extern mca_btl_sctp_module_t mca_btl_sctp_module;
|
|
|
|
#if defined(__WINDOWS__)
|
|
#define CLOSE_THE_SOCKET(socket) closesocket(socket)
|
|
#else
|
|
#define CLOSE_THE_SOCKET(socket) close(socket)
|
|
#endif /* defined(__WINDOWS__) */
|
|
|
|
/**
|
|
* Register SCTP component parameters with the MCA framework
|
|
*/
|
|
extern int mca_btl_sctp_component_open(void);
|
|
|
|
/**
|
|
* Any final cleanup before being unloaded.
|
|
*/
|
|
extern int mca_btl_sctp_component_close(void);
|
|
|
|
/**
|
|
* SCTP component initialization.
|
|
*
|
|
* @param num_btl_modules (OUT) Number of BTLs returned in BTL array.
|
|
* @param allow_multi_user_threads (OUT) Flag indicating wether BTL supports user threads (TRUE)
|
|
* @param have_hidden_threads (OUT) Flag indicating wether BTL uses threads (TRUE)
|
|
*/
|
|
extern mca_btl_base_module_t** mca_btl_sctp_component_init(
|
|
int *num_btl_modules,
|
|
bool allow_multi_user_threads,
|
|
bool have_hidden_threads
|
|
);
|
|
|
|
|
|
|
|
/**
|
|
* SCTP component control.
|
|
*/
|
|
int mca_btl_sctp_component_control(
|
|
int param,
|
|
void* value,
|
|
size_t size
|
|
);
|
|
|
|
|
|
/**
|
|
* SCTP component progress.
|
|
*/
|
|
extern int mca_btl_sctp_component_progress(void);
|
|
|
|
|
|
|
|
/**
|
|
* Cleanup any resources held by the BTL.
|
|
*
|
|
* @param btl BTL instance.
|
|
* @return OMPI_SUCCESS or error status on failure.
|
|
*/
|
|
|
|
extern int mca_btl_sctp_finalize(
|
|
struct mca_btl_base_module_t* btl
|
|
);
|
|
|
|
|
|
/**
|
|
* PML->BTL notification of change in the process list.
|
|
*
|
|
* @param btl (IN)
|
|
* @param nprocs (IN) Number of processes
|
|
* @param procs (IN) Set of processes
|
|
* @param peers (OUT) Set of (optional) peer addressing info.
|
|
* @param peers (IN/OUT) Set of processes that are reachable via this BTL.
|
|
* @return OMPI_SUCCESS or error status on failure.
|
|
*
|
|
*/
|
|
|
|
extern int mca_btl_sctp_add_procs(
|
|
struct mca_btl_base_module_t* btl,
|
|
size_t nprocs,
|
|
struct ompi_proc_t **procs,
|
|
struct mca_btl_base_endpoint_t** peers,
|
|
ompi_bitmap_t* reachable
|
|
);
|
|
|
|
/**
|
|
* PML->BTL notification of change in the process list.
|
|
*
|
|
* @param btl (IN) BTL instance
|
|
* @param nproc (IN) Number of processes.
|
|
* @param procs (IN) Set of processes.
|
|
* @param peers (IN) Set of peer data structures.
|
|
* @return Status indicating if cleanup was successful
|
|
*
|
|
*/
|
|
|
|
extern int mca_btl_sctp_del_procs(
|
|
struct mca_btl_base_module_t* btl,
|
|
size_t nprocs,
|
|
struct ompi_proc_t **procs,
|
|
struct mca_btl_base_endpoint_t** peers
|
|
);
|
|
|
|
|
|
/**
|
|
* Initiate an asynchronous send.
|
|
*
|
|
* @param btl (IN) BTL module
|
|
* @param endpoint (IN) BTL addressing information
|
|
* @param descriptor (IN) Description of the data to be transfered
|
|
* @param tag (IN) The tag value used to notify the peer.
|
|
*/
|
|
|
|
extern int mca_btl_sctp_send(
|
|
struct mca_btl_base_module_t* btl,
|
|
struct mca_btl_base_endpoint_t* btl_peer,
|
|
struct mca_btl_base_descriptor_t* descriptor,
|
|
mca_btl_base_tag_t tag
|
|
);
|
|
|
|
|
|
/**
|
|
* Initiate an asynchronous put.
|
|
*
|
|
* @param btl (IN) BTL module
|
|
* @param endpoint (IN) BTL addressing information
|
|
* @param descriptor (IN) Description of the data to be transferred
|
|
*/
|
|
|
|
extern int mca_btl_sctp_put(
|
|
struct mca_btl_base_module_t* btl,
|
|
struct mca_btl_base_endpoint_t* btl_peer,
|
|
struct mca_btl_base_descriptor_t* decriptor
|
|
);
|
|
|
|
|
|
/**
|
|
* Initiate an asynchronous get.
|
|
*
|
|
* @param btl (IN) BTL module
|
|
* @param endpoint (IN) BTL addressing information
|
|
* @param descriptor (IN) Description of the data to be transferred
|
|
*/
|
|
|
|
extern int mca_btl_sctp_get(
|
|
struct mca_btl_base_module_t* btl,
|
|
struct mca_btl_base_endpoint_t* btl_peer,
|
|
struct mca_btl_base_descriptor_t* decriptor
|
|
);
|
|
|
|
/**
|
|
* Allocate a descriptor with a segment of the requested size.
|
|
* Note that the BTL layer may choose to return a smaller size
|
|
* if it cannot support the request.
|
|
*
|
|
* @param btl (IN) BTL module
|
|
* @param size (IN) Request segment size.
|
|
*/
|
|
|
|
extern mca_btl_base_descriptor_t* mca_btl_sctp_alloc(
|
|
struct mca_btl_base_module_t* btl,
|
|
struct mca_btl_base_endpoint_t* endpoint,
|
|
uint8_t order,
|
|
size_t size,
|
|
uint32_t flags);
|
|
|
|
|
|
/**
|
|
* Return a segment allocated by this BTL.
|
|
*
|
|
* @param btl (IN) BTL module
|
|
* @param descriptor (IN) Allocated descriptor.
|
|
*/
|
|
|
|
extern int mca_btl_sctp_free(
|
|
struct mca_btl_base_module_t* btl,
|
|
mca_btl_base_descriptor_t* des);
|
|
|
|
|
|
/**
|
|
* Prepare a descriptor for send/rdma using the supplied
|
|
* convertor. If the convertor references data that is contigous,
|
|
* the descriptor may simply point to the user buffer. Otherwise,
|
|
* this routine is responsible for allocating buffer space and
|
|
* packing if required.
|
|
*
|
|
* @param btl (IN) BTL module
|
|
* @param endpoint (IN) BTL peer addressing
|
|
* @param convertor (IN) Data type convertor
|
|
* @param reserve (IN) Additional bytes requested by upper layer to precede user data
|
|
* @param size (IN/OUT) Number of bytes to prepare (IN), number of bytes actually prepared (OUT)
|
|
*/
|
|
|
|
mca_btl_base_descriptor_t* mca_btl_sctp_prepare_src(
|
|
struct mca_btl_base_module_t* btl,
|
|
struct mca_btl_base_endpoint_t* peer,
|
|
struct mca_mpool_base_registration_t*,
|
|
struct ompi_convertor_t* convertor,
|
|
uint8_t order,
|
|
size_t reserve,
|
|
size_t* size,
|
|
uint32_t flags
|
|
);
|
|
|
|
extern mca_btl_base_descriptor_t* mca_btl_sctp_prepare_dst(
|
|
struct mca_btl_base_module_t* btl,
|
|
struct mca_btl_base_endpoint_t* peer,
|
|
struct mca_mpool_base_registration_t*,
|
|
struct ompi_convertor_t* convertor,
|
|
uint8_t order,
|
|
size_t reserve,
|
|
size_t* size,
|
|
uint32_t flags);
|
|
|
|
|
|
#if defined(c_plusplus) || defined(__cplusplus)
|
|
}
|
|
#endif
|
|
#endif
|