/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */ /* * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana * University Research and Technology * Corporation. All rights reserved. * Copyright (c) 2004-2009 The University of Tennessee and The University * of Tennessee Research Foundation. All rights * reserved. * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, * University of Stuttgart. All rights reserved. * Copyright (c) 2004-2005 The Regents of the University of California. * All rights reserved. * Copyright (c) 2015 Los Alamos National Security, LLC. All rights * reserved. * $COPYRIGHT$ * * Additional copyrights may follow * * $HEADER$ */ /** * @file */ #ifndef MCA_BTL_TEMPLATE_H #define MCA_BTL_TEMPLATE_H #include "opal_config.h" #include #include /* Open MPI includes */ #include "opal/mca/event/event.h" #include "opal/mca/btl/btl.h" #include "opal/mca/btl/base/base.h" #include "opal/mca/mpool/mpool.h" BEGIN_C_DECLS #define MCA_BTL_HAS_MPOOL 1 /** * Infiniband (TEMPLATE) BTL component. */ struct mca_btl_template_component_t { mca_btl_base_component_3_0_0_t super; /**< base BTL component */ uint32_t template_num_btls; /**< number of hcas available to the TEMPLATE component */ struct mca_btl_template_module_t *template_btls; /**< array of available BTL modules */ int template_free_list_num; /**< initial size of free lists */ int template_free_list_max; /**< maximum size of free lists */ int template_free_list_inc; /**< number of elements to alloc when growing free lists */ opal_list_t template_procs; /**< list of template proc structures */ opal_mutex_t template_lock; /**< lock for accessing module state */ char* template_mpool_name; /**< name of memory pool */ bool leave_pinned; /**< pin memory on first use and leave pinned */ }; typedef struct mca_btl_template_component_t mca_btl_template_component_t; OPAL_MODULE_DECLSPEC extern mca_btl_template_component_t mca_btl_template_component; /** * BTL Module Interface */ struct mca_btl_template_module_t { mca_btl_base_module_t super; /**< base BTL interface */ /* free list of fragment descriptors */ opal_free_list_t template_frag_eager; opal_free_list_t template_frag_max; opal_free_list_t template_frag_user; /* lock for accessing module state */ opal_mutex_t template_lock; #if MCA_BTL_HAS_MPOOL struct mca_mpool_base_module_t* template_mpool; #endif }; typedef struct mca_btl_template_module_t mca_btl_template_module_t; extern mca_btl_template_module_t mca_btl_template_module; /** * TEMPLATE component initialization. * * @param num_btl_modules (OUT) Number of BTLs returned in BTL array. * @param allow_multi_user_threads (OUT) Flag indicating wether BTL supports user threads (TRUE) * @param have_hidden_threads (OUT) Flag indicating wether BTL uses threads (TRUE) */ extern mca_btl_base_module_t** mca_btl_template_component_init( int *num_btl_modules, bool allow_multi_user_threads, bool have_hidden_threads ); /** * TEMPLATE component progress. */ extern int mca_btl_template_component_progress(void); /** * Cleanup any resources held by the BTL. * * @param btl BTL instance. * @return OPAL_SUCCESS or error status on failure. */ extern int mca_btl_template_finalize( struct mca_btl_base_module_t* btl ); /** * PML->BTL notification of change in the process list. * * @param btl (IN) * @param nprocs (IN) Number of processes * @param procs (IN) Set of processes * @param peers (OUT) Set of (optional) peer addressing info. * @param peers (IN/OUT) Set of processes that are reachable via this BTL. * @return OPAL_SUCCESS or error status on failure. * */ extern int mca_btl_template_add_procs( struct mca_btl_base_module_t* btl, size_t nprocs, struct opal_proc_t **procs, struct mca_btl_base_endpoint_t** peers, opal_bitmap_t* reachable ); /** * PML->BTL notification of change in the process list. * * @param btl (IN) BTL instance * @param nproc (IN) Number of processes. * @param procs (IN) Set of processes. * @param peers (IN) Set of peer data structures. * @return Status indicating if cleanup was successful * */ extern int mca_btl_template_del_procs( struct mca_btl_base_module_t* btl, size_t nprocs, struct opal_proc_t **procs, struct mca_btl_base_endpoint_t** peers ); /** * Initiate an asynchronous send. * * @param btl (IN) BTL module * @param endpoint (IN) BTL addressing information * @param descriptor (IN) Description of the data to be transfered * @param tag (IN) The tag value used to notify the peer. */ extern int mca_btl_template_send( struct mca_btl_base_module_t* btl, struct mca_btl_base_endpoint_t* btl_peer, struct mca_btl_base_descriptor_t* descriptor, mca_btl_base_tag_t tag ); /** * Initiate an asynchronous put. * Completion Semantics: if this function returns a 1 then the operation * is complete. a return of OPAL_SUCCESS indicates * the put operation has been queued with the * network. the local_handle can not be deregistered * until all outstanding operations on that handle * have been completed. * * @param btl (IN) BTL module * @param endpoint (IN) BTL addressing information * @param local_address (IN) Local address to put from (registered) * @param remote_address (IN) Remote address to put to (registered remotely) * @param local_handle (IN) Registration handle for region containing * (local_address, local_address + size) * @param remote_handle (IN) Remote registration handle for region containing * (remote_address, remote_address + size) * @param size (IN) Number of bytes to put * @param flags (IN) Flags for this put operation * @param order (IN) Ordering * @param cbfunc (IN) Function to call on completion (if queued) * @param cbcontext (IN) Context for the callback * @param cbdata (IN) Data for callback * * @retval OPAL_SUCCESS The descriptor was successfully queued for a put * @retval OPAL_ERROR The descriptor was NOT successfully queued for a put * @retval OPAL_ERR_OUT_OF_RESOURCE Insufficient resources to queue the put * operation. Try again later * @retval OPAL_ERR_NOT_AVAILABLE Put can not be performed due to size or * alignment restrictions. */ int mca_btl_template_put (struct mca_btl_base_module_t *btl, struct mca_btl_base_endpoint_t *endpoint, void *local_address, uint64_t remote_address, struct mca_btl_base_registration_handle_t *local_handle, struct mca_btl_base_registration_handle_t *remote_handle, size_t size, int flags, int order, mca_btl_base_rdma_completion_fn_t cbfunc, void *cbcontext, void *cbdata); /** * Initiate an asynchronous get. * Completion Semantics: if this function returns a 1 then the operation * is complete. a return of OPAL_SUCCESS indicates * the get operation has been queued with the * network. the local_handle can not be deregistered * until all outstanding operations on that handle * have been completed. * * @param btl (IN) BTL module * @param endpoint (IN) BTL addressing information * @param local_address (IN) Local address to put from (registered) * @param remote_address (IN) Remote address to put to (registered remotely) * @param local_handle (IN) Registration handle for region containing * (local_address, local_address + size) * @param remote_handle (IN) Remote registration handle for region containing * (remote_address, remote_address + size) * @param size (IN) Number of bytes to put * @param flags (IN) Flags for this put operation * @param order (IN) Ordering * @param cbfunc (IN) Function to call on completion (if queued) * @param cbcontext (IN) Context for the callback * @param cbdata (IN) Data for callback * * @retval OPAL_SUCCESS The descriptor was successfully queued for a put * @retval OPAL_ERROR The descriptor was NOT successfully queued for a put * @retval OPAL_ERR_OUT_OF_RESOURCE Insufficient resources to queue the put * operation. Try again later * @retval OPAL_ERR_NOT_AVAILABLE Put can not be performed due to size or * alignment restrictions. */ int mca_btl_template_get (struct mca_btl_base_module_t *btl, struct mca_btl_base_endpoint_t *endpoint, void *local_address, uint64_t remote_address, struct mca_btl_base_registration_handle_t *local_handle, struct mca_btl_base_registration_handle_t *remote_handle, size_t size, int flags, int order, mca_btl_base_rdma_completion_fn_t cbfunc, void *cbcontext, void *cbdata); /** * @brief Register a memory region for put/get/atomic operations. * * @param btl (IN) BTL module * @param endpoint(IN) BTL addressing information (or NULL for all endpoints) * @param base (IN) Pointer to start of region * @param size (IN) Size of region * @param flags (IN) Flags indicating what operation will be performed. Valid * values are MCA_BTL_DES_FLAGS_PUT, MCA_BTL_DES_FLAGS_GET, * and MCA_BTL_DES_FLAGS_ATOMIC * * @returns a memory registration handle valid for both local and remote operations * @returns NULL if the region could not be registered * * This function registers the specified region with the hardware for use with * the btl_put, btl_get, btl_atomic_cas, btl_atomic_op, and btl_atomic_fop * functions. Care should be taken to not hold an excessive number of registrations * as they may use limited system/NIC resources. */ struct mca_btl_base_registration_handle_t *mca_btl_template_register_mem ( struct mca_btl_base_module_t* btl, struct mca_btl_base_endpoint_t *endpoint, void *base, size_t size, uint32_t flags); /** * @brief Deregister a memory region * * @param btl (IN) BTL module region was registered with * @param handle (IN) BTL registration handle to deregister * * This function deregisters the memory region associated with the specified handle. Care * should be taken to not perform any RDMA or atomic operation on this memory region * after it is deregistered. It is erroneous to specify a memory handle associated with * a remote node. */ int mca_btl_template_deregister_mem (struct mca_btl_base_module_t* btl, struct mca_btl_base_registration_handle_t *handle); /** * Register a callback function that is called on receipt * of a fragment. * * @param btl (IN) BTL module * @return Status indicating if registration was successful * */ extern int mca_btl_template_register( struct mca_btl_base_module_t* btl, mca_btl_base_tag_t tag, mca_btl_base_module_recv_cb_fn_t cbfunc, void* cbdata); /** * Allocate a descriptor with a segment of the requested size. * Note that the BTL layer may choose to return a smaller size * if it cannot support the request. * * @param btl (IN) BTL module * @param size (IN) Request segment size. */ extern mca_btl_base_descriptor_t* mca_btl_template_alloc( struct mca_btl_base_module_t* btl, struct mca_btl_base_endpoint_t* endpoint, uint8_t order, size_t size, uint32_t flags); /** * Return a segment allocated by this BTL. * * @param btl (IN) BTL module * @param descriptor (IN) Allocated descriptor. */ extern int mca_btl_template_free( struct mca_btl_base_module_t* btl, mca_btl_base_descriptor_t* des); /** * Prepare a descriptor for send/rdma using the supplied * convertor. If the convertor references data that is contigous, * the descriptor may simply point to the user buffer. Otherwise, * this routine is responsible for allocating buffer space and * packing if required. * * @param btl (IN) BTL module * @param endpoint (IN) BTL peer addressing * @param convertor (IN) Data type convertor * @param reserve (IN) Additional bytes requested by upper layer to precede user data * @param size (IN/OUT) Number of bytes to prepare (IN), number of bytes actually prepared (OUT) */ mca_btl_base_descriptor_t* mca_btl_template_prepare_src( struct mca_btl_base_module_t* btl, struct mca_btl_base_endpoint_t* peer, struct opal_convertor_t* convertor, uint8_t order, size_t reserve, size_t* size, uint32_t flags ); /** * Fault Tolerance Event Notification Function * @param state Checkpoint Stae * @return OPAL_SUCCESS or failure status */ int mca_btl_template_ft_event(int state); END_C_DECLS #endif