/* -*- Mode: C; c-basic-offset:4 ; -*- */ /* * Copyright (c) 2004-2005 The Trustees of Indiana University. * All rights reserved. * Copyright (c) 2004-2005 The Trustees of the University of Tennessee. * All rights reserved. * Copyright (c) 2004 The Ohio State University. * All rights reserved. * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, * University of Stuttgart. All rights reserved. * Copyright (c) 2004-2005 The Regents of the University of California. * All rights reserved. * $COPYRIGHT$ * * Additional copyrights may follow * * $HEADER$ */ /** * @file */ #ifndef MCA_PTL_GM_H #define MCA_PTL_GM_H #include "ompi_config.h" #include "gm_config.h" #include "class/ompi_free_list.h" #include "mca/ptl/ptl.h" #include "ptl_gm_priv.h" #include "ptl_gm_peer.h" #define MCA_PTL_GM_STATISTICS 0 #define MAX_RECV_TOKENS 256 #define PTL_GM_ADMIN_SEND_TOKENS 0 #define PTL_GM_ADMIN_RECV_TOKENS 0 #define GM_SIZE 30 #define NUM_RECV_FRAGS 256 #define MCA_PTL_GM_FRAG_CACHED #if defined(c_plusplus) || defined(__cplusplus) extern "C" { #endif /** * GM PTL component */ struct mca_ptl_gm_component_t { mca_ptl_base_component_1_0_0_t super; /**< base PTL module */ struct mca_ptl_gm_module_t **gm_ptl_modules; /**< array of available PTL modules */ size_t gm_num_ptl_modules; /**< number of ptls actually used */ size_t gm_max_ptl_modules; /**< maximum number of ptls - available */ uint32_t gm_max_port_number; /**< maximum number of ports by board */ uint32_t gm_max_boards_number; /**< maximum number of boards on the node */ uint32_t gm_free_list_num; /**< initial size of free lists */ uint32_t gm_free_list_max; /**< maximum size of free lists */ uint32_t gm_free_list_inc; /**< number of elements to alloc when growing free lists */ uint32_t gm_segment_size; /**< size of the allocated segment */ uint32_t gm_eager_limit; /**< number of bytes before the rendez-vous protocol. If the **< size of the message is less than this number then GM **< use a eager protocol. */ #if OMPI_MCA_PTL_GM_HAVE_RDMA_GET uint32_t gm_rndv_burst_limit; uint32_t gm_rdma_frag_size; /**< maximum fragment size used to transfer data over RDMA */ #endif /* OMPI_MCA_PTL_GM_HAVE_RDMA_GET */ char* gm_port_name; /**< the name used to get the port */ struct mca_ptl_gm_proc_t* gm_local; opal_list_t gm_procs; opal_list_t gm_send_req; ompi_free_list_t gm_unexpected_frags_data; opal_mutex_t gm_lock; /**< lock for accessing module state */ }; typedef struct mca_ptl_gm_component_t mca_ptl_gm_component_t; extern mca_ptl_gm_component_t mca_ptl_gm_component; /** * GM PTL Interface */ struct mca_ptl_gm_module_t { mca_ptl_base_module_t super; /**< base PTL module interface */ struct gm_port *gm_port; mca_ptl_gm_addr_t local_addr; unsigned int num_send_tokens; unsigned int num_recv_tokens; unsigned int max_send_tokens; unsigned int max_recv_tokens; void* gm_send_dma_memory; /**< pointer to the send DMA registered memory attached to the PTL */ void* gm_recv_dma_memory; /**< pointer to the recv DMA registered memory attached to the PTL */ struct mca_ptl_gm_send_frag_t* gm_send_fragments; struct mca_ptl_gm_recv_frag_t* gm_recv_fragments; ompi_free_list_t gm_send_frags; ompi_free_list_t gm_send_dma_frags; ompi_free_list_t gm_recv_frags_free; opal_list_t gm_send_frags_queue; opal_list_t gm_pending_acks; opal_list_t gm_recv_outstanding_queue; opal_thread_t thread; #if MCA_PTL_GM_STATISTICS size_t ptl_bytes_sent; size_t ptl_bytes_recv; #endif /* MCA_PTL_GM_STATISTICS */ }; typedef struct mca_ptl_gm_module_t mca_ptl_gm_module_t; extern mca_ptl_gm_module_t mca_ptl_gm_module; /** * Register GM module parameters with the MCA framework */ extern int mca_ptl_gm_component_open (void); /** * Any final cleanup before being unloaded. */ extern int mca_ptl_gm_component_close (void); /** * GM module initialization. * * @param num_ptls (OUT) Number of PTLs returned in PTL array. * @param allow_multi_user_threads (OUT) Flag indicating wether PTL supports user threads (TRUE) * @param have_hidden_threads (OUT) Flag indicating wether PTL uses threads (TRUE) */ extern mca_ptl_base_module_t **mca_ptl_gm_component_init (int *num_ptl_modules, bool enable_progress_threads, bool enable_mpi_threads); /** * GM module control. */ extern int mca_ptl_gm_component_control (int param, void *value, size_t size); /** * GM module progress. */ extern int mca_ptl_gm_component_progress (mca_ptl_tstamp_t tstamp); /** * GM put */ extern int mca_ptl_gm_put( struct mca_ptl_base_module_t *ptl, struct mca_ptl_base_peer_t *ptl_peer, struct mca_ptl_base_send_request_t *sendreq, size_t offset, size_t size, int flags); /** * GM get */ extern int mca_ptl_gm_get (struct mca_ptl_base_module_t *ptl, struct mca_ptl_base_peer_t *ptl_peer, struct mca_ptl_base_recv_request_t *sendreq, size_t offset, size_t size, int flags); /** * PML->PTL notification of change in the process list. * * @param ptl (IN) * @param nprocs (IN) Number of processes * @param procs (IN) Set of processes * @param peers (OUT) Set of (optional) peer addressing info. * @param peers (IN/OUT) Set of processes that are reachable via this PTL. * @return OMPI_SUCCESS or error status on failure. * */ extern int mca_ptl_gm_add_procs (struct mca_ptl_base_module_t *ptl, size_t nprocs, struct ompi_proc_t **procs, struct mca_ptl_base_peer_t **peers, struct ompi_bitmap_t * reachable); /** * PML->PTL notification of change in the process list. * * @param ptl (IN) PTL instance * @param nproc (IN) Number of processes. * @param procs (IN) Set of processes. * @param peers (IN) Set of peer data structures. * @return Status indicating if cleanup was successful * */ extern int mca_ptl_gm_del_procs( struct mca_ptl_base_module_t *ptl, size_t nprocs, struct ompi_proc_t **procs, struct mca_ptl_base_peer_t **peers ); /** * PML->PTL Allocate a send request from the PTL modules free list. * * @param ptl (IN) PTL instance * @param request (OUT) Pointer to allocated request. * @return Status indicating if allocation was successful. * */ extern int mca_ptl_gm_request_init( struct mca_ptl_base_module_t* ptl, struct mca_ptl_base_send_request_t* req); /** * */ extern void mca_ptl_gm_request_fini( struct mca_ptl_base_module_t *ptl, struct mca_ptl_base_send_request_t* req); /** * PML->PTL Notification that a receive fragment has been matched. * * @param ptl (IN) PTL instance * @param recv_frag (IN) Receive fragment * */ extern void mca_ptl_gm_matched (struct mca_ptl_base_module_t *ptl, struct mca_ptl_base_recv_frag_t *frag); /** * */ extern int mca_ptl_gm_finalize (struct mca_ptl_base_module_t *ptl); /** * Internally allocate memory for the unexpected messages. We will manage a list * of such buffers in order to avoid too many memory allocations. */ extern char* gm_get_local_buffer( void ); extern void gm_release_local_buffer( char* ptr ); union mca_ptl_base_header_t; void mca_ptl_gm_dump_header( char* str, union mca_ptl_base_header_t* hdr ); #if OMPI_ENABLE_DEBUG #include "opal/class/opal_list.h" /* If debug is enabled we have to work around the item validity checks. */ #define OMPI_GM_FREE_LIST_RETURN( LIST, ITEM ) \ do { \ (ITEM)->opal_list_item_refcount = 0; \ (ITEM)->opal_list_item_belong_to = NULL; \ (ITEM)->super.cls_init_file_name = __FILE__; \ (ITEM)->super.cls_init_lineno = __LINE__; \ OMPI_FREE_LIST_RETURN( (LIST), (ITEM) ); \ } while(0) #else #define OMPI_GM_FREE_LIST_RETURN OMPI_FREE_LIST_RETURN #endif /* OMPI_ENABLE_DEBUG */ #if defined(c_plusplus) || defined(__cplusplus) } #endif #endif