251 строка
9.4 KiB
C
251 строка
9.4 KiB
C
/* -*- Mode: C; c-basic-offset:4 ; -*- */
|
|
|
|
/*
|
|
* Copyright (c) 2004-2005 The Trustees of Indiana University.
|
|
* All rights reserved.
|
|
* Copyright (c) 2004-2005 The Trustees of the University of Tennessee.
|
|
* All rights reserved.
|
|
* Copyright (c) 2004 The Ohio State University.
|
|
* All rights reserved.
|
|
* Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
|
|
* University of Stuttgart. All rights reserved.
|
|
* Copyright (c) 2004-2005 The Regents of the University of California.
|
|
* All rights reserved.
|
|
* $COPYRIGHT$
|
|
*
|
|
* Additional copyrights may follow
|
|
*
|
|
* $HEADER$
|
|
*/
|
|
/**
|
|
* @file
|
|
*/
|
|
#ifndef MCA_PTL_GM_H
|
|
#define MCA_PTL_GM_H
|
|
|
|
#include "ompi_config.h"
|
|
#include "gm_config.h"
|
|
#include "class/ompi_free_list.h"
|
|
#include "mca/ptl/ptl.h"
|
|
#include "ptl_gm_priv.h"
|
|
#include "ptl_gm_peer.h"
|
|
|
|
#define MCA_PTL_GM_STATISTICS 0
|
|
#define MAX_RECV_TOKENS 256
|
|
#define PTL_GM_ADMIN_SEND_TOKENS 0
|
|
#define PTL_GM_ADMIN_RECV_TOKENS 0
|
|
#define GM_SIZE 30
|
|
#define NUM_RECV_FRAGS 256
|
|
#define MCA_PTL_GM_FRAG_CACHED
|
|
#if defined(c_plusplus) || defined(__cplusplus)
|
|
extern "C" {
|
|
#endif
|
|
|
|
/**
|
|
* GM PTL component
|
|
*/
|
|
struct mca_ptl_gm_component_t {
|
|
mca_ptl_base_component_1_0_0_t super; /**< base PTL module */
|
|
struct mca_ptl_gm_module_t **gm_ptl_modules; /**< array of available PTL modules */
|
|
size_t gm_num_ptl_modules; /**< number of ptls actually used */
|
|
size_t gm_max_ptl_modules; /**< maximum number of ptls - available */
|
|
uint32_t gm_max_port_number; /**< maximum number of ports by board */
|
|
uint32_t gm_max_boards_number; /**< maximum number of boards on the node */
|
|
uint32_t gm_free_list_num; /**< initial size of free lists */
|
|
uint32_t gm_free_list_max; /**< maximum size of free lists */
|
|
uint32_t gm_free_list_inc; /**< number of elements to alloc when growing free lists */
|
|
uint32_t gm_segment_size; /**< size of the allocated segment */
|
|
uint32_t gm_eager_limit; /**< number of bytes before the rendez-vous protocol. If the
|
|
**< size of the message is less than this number then GM
|
|
**< use a eager protocol. */
|
|
#if OMPI_MCA_PTL_GM_HAVE_RDMA_GET
|
|
uint32_t gm_rndv_burst_limit;
|
|
uint32_t gm_rdma_frag_size; /**< maximum fragment size used to transfer data over RDMA */
|
|
#endif /* OMPI_MCA_PTL_GM_HAVE_RDMA_GET */
|
|
char* gm_port_name; /**< the name used to get the port */
|
|
|
|
struct mca_ptl_gm_proc_t* gm_local;
|
|
opal_list_t gm_procs;
|
|
opal_list_t gm_send_req;
|
|
ompi_free_list_t gm_unexpected_frags_data;
|
|
|
|
opal_mutex_t gm_lock; /**< lock for accessing module state */
|
|
};
|
|
|
|
typedef struct mca_ptl_gm_component_t mca_ptl_gm_component_t;
|
|
extern mca_ptl_gm_component_t mca_ptl_gm_component;
|
|
|
|
/**
|
|
* GM PTL Interface
|
|
*/
|
|
struct mca_ptl_gm_module_t {
|
|
mca_ptl_base_module_t super; /**< base PTL module interface */
|
|
struct gm_port *gm_port;
|
|
mca_ptl_gm_addr_t local_addr;
|
|
unsigned int num_send_tokens;
|
|
unsigned int num_recv_tokens;
|
|
unsigned int max_send_tokens;
|
|
unsigned int max_recv_tokens;
|
|
void* gm_send_dma_memory; /**< pointer to the send DMA registered memory attached to the PTL */
|
|
void* gm_recv_dma_memory; /**< pointer to the recv DMA registered memory attached to the PTL */
|
|
struct mca_ptl_gm_send_frag_t* gm_send_fragments;
|
|
struct mca_ptl_gm_recv_frag_t* gm_recv_fragments;
|
|
|
|
ompi_free_list_t gm_send_frags;
|
|
ompi_free_list_t gm_send_dma_frags;
|
|
ompi_free_list_t gm_recv_frags_free;
|
|
opal_list_t gm_send_frags_queue;
|
|
opal_list_t gm_pending_acks;
|
|
opal_list_t gm_recv_outstanding_queue;
|
|
|
|
opal_thread_t thread;
|
|
#if MCA_PTL_GM_STATISTICS
|
|
size_t ptl_bytes_sent;
|
|
size_t ptl_bytes_recv;
|
|
#endif /* MCA_PTL_GM_STATISTICS */
|
|
};
|
|
|
|
typedef struct mca_ptl_gm_module_t mca_ptl_gm_module_t;
|
|
extern mca_ptl_gm_module_t mca_ptl_gm_module;
|
|
|
|
/**
|
|
* Register GM module parameters with the MCA framework
|
|
*/
|
|
extern int mca_ptl_gm_component_open (void);
|
|
|
|
/**
|
|
* Any final cleanup before being unloaded.
|
|
*/
|
|
extern int mca_ptl_gm_component_close (void);
|
|
|
|
/**
|
|
* GM module initialization.
|
|
*
|
|
* @param num_ptls (OUT) Number of PTLs returned in PTL array.
|
|
* @param allow_multi_user_threads (OUT) Flag indicating wether PTL supports user threads (TRUE)
|
|
* @param have_hidden_threads (OUT) Flag indicating wether PTL uses threads (TRUE)
|
|
*/
|
|
extern mca_ptl_base_module_t **mca_ptl_gm_component_init (int *num_ptl_modules,
|
|
bool enable_progress_threads,
|
|
bool enable_mpi_threads);
|
|
|
|
/**
|
|
* GM module control.
|
|
*/
|
|
extern int mca_ptl_gm_component_control (int param, void *value, size_t size);
|
|
|
|
/**
|
|
* GM module progress.
|
|
*/
|
|
extern int mca_ptl_gm_component_progress (mca_ptl_tstamp_t tstamp);
|
|
|
|
/**
|
|
* GM put
|
|
*/
|
|
extern int mca_ptl_gm_put( struct mca_ptl_base_module_t *ptl,
|
|
struct mca_ptl_base_peer_t *ptl_peer,
|
|
struct mca_ptl_base_send_request_t *sendreq,
|
|
size_t offset, size_t size, int flags);
|
|
|
|
/**
|
|
* GM get
|
|
*/
|
|
extern int mca_ptl_gm_get (struct mca_ptl_base_module_t *ptl,
|
|
struct mca_ptl_base_peer_t *ptl_peer,
|
|
struct mca_ptl_base_recv_request_t *sendreq,
|
|
size_t offset, size_t size, int flags);
|
|
|
|
/**
|
|
* PML->PTL notification of change in the process list.
|
|
*
|
|
* @param ptl (IN)
|
|
* @param nprocs (IN) Number of processes
|
|
* @param procs (IN) Set of processes
|
|
* @param peers (OUT) Set of (optional) peer addressing info.
|
|
* @param peers (IN/OUT) Set of processes that are reachable via this PTL.
|
|
* @return OMPI_SUCCESS or error status on failure.
|
|
*
|
|
*/
|
|
extern int mca_ptl_gm_add_procs (struct mca_ptl_base_module_t *ptl,
|
|
size_t nprocs,
|
|
struct ompi_proc_t **procs,
|
|
struct mca_ptl_base_peer_t **peers,
|
|
struct ompi_bitmap_t * reachable);
|
|
|
|
/**
|
|
* PML->PTL notification of change in the process list.
|
|
*
|
|
* @param ptl (IN) PTL instance
|
|
* @param nproc (IN) Number of processes.
|
|
* @param procs (IN) Set of processes.
|
|
* @param peers (IN) Set of peer data structures.
|
|
* @return Status indicating if cleanup was successful
|
|
*
|
|
*/
|
|
extern int mca_ptl_gm_del_procs( struct mca_ptl_base_module_t *ptl,
|
|
size_t nprocs,
|
|
struct ompi_proc_t **procs,
|
|
struct mca_ptl_base_peer_t **peers );
|
|
|
|
/**
|
|
* PML->PTL Allocate a send request from the PTL modules free list.
|
|
*
|
|
* @param ptl (IN) PTL instance
|
|
* @param request (OUT) Pointer to allocated request.
|
|
* @return Status indicating if allocation was successful.
|
|
*
|
|
*/
|
|
extern int mca_ptl_gm_request_init( struct mca_ptl_base_module_t* ptl,
|
|
struct mca_ptl_base_send_request_t* req);
|
|
|
|
/**
|
|
*
|
|
*/
|
|
extern void mca_ptl_gm_request_fini( struct mca_ptl_base_module_t *ptl,
|
|
struct mca_ptl_base_send_request_t* req);
|
|
|
|
/**
|
|
* PML->PTL Notification that a receive fragment has been matched.
|
|
*
|
|
* @param ptl (IN) PTL instance
|
|
* @param recv_frag (IN) Receive fragment
|
|
*
|
|
*/
|
|
extern void mca_ptl_gm_matched (struct mca_ptl_base_module_t *ptl,
|
|
struct mca_ptl_base_recv_frag_t *frag);
|
|
|
|
/**
|
|
*
|
|
*/
|
|
extern int mca_ptl_gm_finalize (struct mca_ptl_base_module_t *ptl);
|
|
|
|
/**
|
|
* Internally allocate memory for the unexpected messages. We will manage a list
|
|
* of such buffers in order to avoid too many memory allocations.
|
|
*/
|
|
extern char* gm_get_local_buffer( void );
|
|
extern void gm_release_local_buffer( char* ptr );
|
|
|
|
union mca_ptl_base_header_t;
|
|
void mca_ptl_gm_dump_header( char* str, union mca_ptl_base_header_t* hdr );
|
|
|
|
#if OMPI_ENABLE_DEBUG
|
|
#include "opal/class/opal_list.h"
|
|
/* If debug is enabled we have to work around the item validity checks. */
|
|
#define OMPI_GM_FREE_LIST_RETURN( LIST, ITEM ) \
|
|
do { \
|
|
(ITEM)->opal_list_item_refcount = 0; \
|
|
(ITEM)->opal_list_item_belong_to = NULL; \
|
|
(ITEM)->super.cls_init_file_name = __FILE__; \
|
|
(ITEM)->super.cls_init_lineno = __LINE__; \
|
|
OMPI_FREE_LIST_RETURN( (LIST), (ITEM) ); \
|
|
} while(0)
|
|
#else
|
|
#define OMPI_GM_FREE_LIST_RETURN OMPI_FREE_LIST_RETURN
|
|
#endif /* OMPI_ENABLE_DEBUG */
|
|
|
|
#if defined(c_plusplus) || defined(__cplusplus)
|
|
}
|
|
#endif
|
|
#endif
|