9ffac85650
With Open MPI 5.0, the decision was made to stop building 3rd-party packages, such as Libevent, HWLOC, PMIx, and PRRTE as MCA components and instead 1) start relying on external libraries whenever possible and 2) Open MPI builds the 3rd party libraries (if needed) as independent libraries, rather than linked into libopen-pal. This patch moves libevent from an MCA framework to a stand-alone library built outside of OPAL. A wrapper in opal/util is provided to minimize the unnecessary changes in the rest of the code. When using the internal Libevent, it will be installed as a stand-alone libevent.a, instead of bundled in OPAL. Any pre-installed version of Libevent at or after 2.0.21 is preferred over the internal version. Signed-off-by: Brian Barrett <bbarrett@amazon.com>
380 строки
13 KiB
C
380 строки
13 KiB
C
/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
|
|
/*
|
|
* Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
|
|
* University Research and Technology
|
|
* Corporation. All rights reserved.
|
|
* Copyright (c) 2004-2009 The University of Tennessee and The University
|
|
* of Tennessee Research Foundation. All rights
|
|
* reserved.
|
|
* Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
|
|
* University of Stuttgart. All rights reserved.
|
|
* Copyright (c) 2004-2005 The Regents of the University of California.
|
|
* All rights reserved.
|
|
* Copyright (c) 2015 Los Alamos National Security, LLC. All rights
|
|
* reserved.
|
|
* Copyright (c) 2020 Google, LLC. All rights reserved.
|
|
* Copyright (c) 2020 Amazon.com, Inc. or its affiliates.
|
|
* All Rights reserved.
|
|
* $COPYRIGHT$
|
|
*
|
|
* Additional copyrights may follow
|
|
*
|
|
* $HEADER$
|
|
*/
|
|
/**
|
|
* @file
|
|
*/
|
|
#ifndef MCA_BTL_TEMPLATE_H
|
|
#define MCA_BTL_TEMPLATE_H
|
|
|
|
#include "opal_config.h"
|
|
#include <sys/types.h>
|
|
#include <string.h>
|
|
|
|
/* Open MPI includes */
|
|
#include "opal/util/event.h"
|
|
#include "opal/mca/btl/btl.h"
|
|
#include "opal/mca/btl/base/base.h"
|
|
#include "opal/mca/mpool/mpool.h"
|
|
|
|
BEGIN_C_DECLS
|
|
|
|
#define MCA_BTL_HAS_MPOOL 1
|
|
|
|
/**
|
|
* Infiniband (TEMPLATE) BTL component.
|
|
*/
|
|
|
|
struct mca_btl_template_component_t {
|
|
mca_btl_base_component_3_0_0_t super; /**< base BTL component */
|
|
|
|
uint32_t template_num_btls;
|
|
/**< number of hcas available to the TEMPLATE component */
|
|
|
|
struct mca_btl_template_module_t *template_btls;
|
|
/**< array of available BTL modules */
|
|
|
|
int template_free_list_num;
|
|
/**< initial size of free lists */
|
|
|
|
int template_free_list_max;
|
|
/**< maximum size of free lists */
|
|
|
|
int template_free_list_inc;
|
|
/**< number of elements to alloc when growing free lists */
|
|
|
|
opal_list_t template_procs;
|
|
/**< list of template proc structures */
|
|
|
|
opal_mutex_t template_lock;
|
|
/**< lock for accessing module state */
|
|
|
|
char* template_mpool_name;
|
|
/**< name of memory pool */
|
|
|
|
bool leave_pinned;
|
|
/**< pin memory on first use and leave pinned */
|
|
};
|
|
typedef struct mca_btl_template_component_t mca_btl_template_component_t;
|
|
|
|
OPAL_MODULE_DECLSPEC extern mca_btl_template_component_t mca_btl_template_component;
|
|
|
|
/**
|
|
* BTL Module Interface
|
|
*/
|
|
struct mca_btl_template_module_t {
|
|
mca_btl_base_module_t super; /**< base BTL interface */
|
|
|
|
/* free list of fragment descriptors */
|
|
opal_free_list_t template_frag_eager;
|
|
opal_free_list_t template_frag_max;
|
|
opal_free_list_t template_frag_user;
|
|
|
|
/* lock for accessing module state */
|
|
opal_mutex_t template_lock;
|
|
|
|
#if MCA_BTL_HAS_MPOOL
|
|
struct mca_mpool_base_module_t* template_mpool;
|
|
#endif
|
|
};
|
|
typedef struct mca_btl_template_module_t mca_btl_template_module_t;
|
|
extern mca_btl_template_module_t mca_btl_template_module;
|
|
|
|
|
|
/**
|
|
* TEMPLATE component initialization.
|
|
*
|
|
* @param num_btl_modules (OUT) Number of BTLs returned in BTL array.
|
|
* @param allow_multi_user_threads (OUT) Flag indicating wether BTL supports user threads (TRUE)
|
|
* @param have_hidden_threads (OUT) Flag indicating wether BTL uses threads (TRUE)
|
|
*/
|
|
extern mca_btl_base_module_t** mca_btl_template_component_init(
|
|
int *num_btl_modules,
|
|
bool allow_multi_user_threads,
|
|
bool have_hidden_threads
|
|
);
|
|
|
|
|
|
/**
|
|
* TEMPLATE component progress.
|
|
*/
|
|
extern int mca_btl_template_component_progress(void);
|
|
|
|
|
|
|
|
/**
|
|
* Cleanup any resources held by the BTL.
|
|
*
|
|
* @param btl BTL instance.
|
|
* @return OPAL_SUCCESS or error status on failure.
|
|
*/
|
|
|
|
extern int mca_btl_template_finalize(
|
|
struct mca_btl_base_module_t* btl
|
|
);
|
|
|
|
|
|
/**
|
|
* PML->BTL notification of change in the process list.
|
|
*
|
|
* @param btl (IN)
|
|
* @param nprocs (IN) Number of processes
|
|
* @param procs (IN) Set of processes
|
|
* @param peers (OUT) Set of (optional) peer addressing info.
|
|
* @param peers (IN/OUT) Set of processes that are reachable via this BTL.
|
|
* @return OPAL_SUCCESS or error status on failure.
|
|
*
|
|
*/
|
|
|
|
extern int mca_btl_template_add_procs(
|
|
struct mca_btl_base_module_t* btl,
|
|
size_t nprocs,
|
|
struct opal_proc_t **procs,
|
|
struct mca_btl_base_endpoint_t** peers,
|
|
opal_bitmap_t* reachable
|
|
);
|
|
|
|
/**
|
|
* PML->BTL notification of change in the process list.
|
|
*
|
|
* @param btl (IN) BTL instance
|
|
* @param nproc (IN) Number of processes.
|
|
* @param procs (IN) Set of processes.
|
|
* @param peers (IN) Set of peer data structures.
|
|
* @return Status indicating if cleanup was successful
|
|
*
|
|
*/
|
|
|
|
extern int mca_btl_template_del_procs(
|
|
struct mca_btl_base_module_t* btl,
|
|
size_t nprocs,
|
|
struct opal_proc_t **procs,
|
|
struct mca_btl_base_endpoint_t** peers
|
|
);
|
|
|
|
|
|
/**
|
|
* Initiate an asynchronous send.
|
|
*
|
|
* @param btl (IN) BTL module
|
|
* @param endpoint (IN) BTL addressing information
|
|
* @param descriptor (IN) Description of the data to be transfered
|
|
* @param tag (IN) The tag value used to notify the peer.
|
|
*/
|
|
|
|
extern int mca_btl_template_send(
|
|
struct mca_btl_base_module_t* btl,
|
|
struct mca_btl_base_endpoint_t* btl_peer,
|
|
struct mca_btl_base_descriptor_t* descriptor,
|
|
mca_btl_base_tag_t tag
|
|
);
|
|
|
|
|
|
/**
|
|
* Initiate an asynchronous put.
|
|
* Completion Semantics: if this function returns a 1 then the operation
|
|
* is complete. a return of OPAL_SUCCESS indicates
|
|
* the put operation has been queued with the
|
|
* network. the local_handle can not be deregistered
|
|
* until all outstanding operations on that handle
|
|
* have been completed.
|
|
*
|
|
* @param btl (IN) BTL module
|
|
* @param endpoint (IN) BTL addressing information
|
|
* @param local_address (IN) Local address to put from (registered)
|
|
* @param remote_address (IN) Remote address to put to (registered remotely)
|
|
* @param local_handle (IN) Registration handle for region containing
|
|
* (local_address, local_address + size)
|
|
* @param remote_handle (IN) Remote registration handle for region containing
|
|
* (remote_address, remote_address + size)
|
|
* @param size (IN) Number of bytes to put
|
|
* @param flags (IN) Flags for this put operation
|
|
* @param order (IN) Ordering
|
|
* @param cbfunc (IN) Function to call on completion (if queued)
|
|
* @param cbcontext (IN) Context for the callback
|
|
* @param cbdata (IN) Data for callback
|
|
*
|
|
* @retval OPAL_SUCCESS The descriptor was successfully queued for a put
|
|
* @retval OPAL_ERROR The descriptor was NOT successfully queued for a put
|
|
* @retval OPAL_ERR_OUT_OF_RESOURCE Insufficient resources to queue the put
|
|
* operation. Try again later
|
|
* @retval OPAL_ERR_NOT_AVAILABLE Put can not be performed due to size or
|
|
* alignment restrictions.
|
|
*/
|
|
int mca_btl_template_put (struct mca_btl_base_module_t *btl,
|
|
struct mca_btl_base_endpoint_t *endpoint, void *local_address,
|
|
uint64_t remote_address, struct mca_btl_base_registration_handle_t *local_handle,
|
|
struct mca_btl_base_registration_handle_t *remote_handle, size_t size, int flags,
|
|
int order, mca_btl_base_rdma_completion_fn_t cbfunc, void *cbcontext, void *cbdata);
|
|
|
|
/**
|
|
* Initiate an asynchronous get.
|
|
* Completion Semantics: if this function returns a 1 then the operation
|
|
* is complete. a return of OPAL_SUCCESS indicates
|
|
* the get operation has been queued with the
|
|
* network. the local_handle can not be deregistered
|
|
* until all outstanding operations on that handle
|
|
* have been completed.
|
|
*
|
|
* @param btl (IN) BTL module
|
|
* @param endpoint (IN) BTL addressing information
|
|
* @param local_address (IN) Local address to put from (registered)
|
|
* @param remote_address (IN) Remote address to put to (registered remotely)
|
|
* @param local_handle (IN) Registration handle for region containing
|
|
* (local_address, local_address + size)
|
|
* @param remote_handle (IN) Remote registration handle for region containing
|
|
* (remote_address, remote_address + size)
|
|
* @param size (IN) Number of bytes to put
|
|
* @param flags (IN) Flags for this put operation
|
|
* @param order (IN) Ordering
|
|
* @param cbfunc (IN) Function to call on completion (if queued)
|
|
* @param cbcontext (IN) Context for the callback
|
|
* @param cbdata (IN) Data for callback
|
|
*
|
|
* @retval OPAL_SUCCESS The descriptor was successfully queued for a put
|
|
* @retval OPAL_ERROR The descriptor was NOT successfully queued for a put
|
|
* @retval OPAL_ERR_OUT_OF_RESOURCE Insufficient resources to queue the put
|
|
* operation. Try again later
|
|
* @retval OPAL_ERR_NOT_AVAILABLE Put can not be performed due to size or
|
|
* alignment restrictions.
|
|
*/
|
|
int mca_btl_template_get (struct mca_btl_base_module_t *btl,
|
|
struct mca_btl_base_endpoint_t *endpoint, void *local_address,
|
|
uint64_t remote_address, struct mca_btl_base_registration_handle_t *local_handle,
|
|
struct mca_btl_base_registration_handle_t *remote_handle, size_t size, int flags,
|
|
int order, mca_btl_base_rdma_completion_fn_t cbfunc, void *cbcontext, void *cbdata);
|
|
|
|
/**
|
|
* @brief Register a memory region for put/get/atomic operations.
|
|
*
|
|
* @param btl (IN) BTL module
|
|
* @param endpoint(IN) BTL addressing information (or NULL for all endpoints)
|
|
* @param base (IN) Pointer to start of region
|
|
* @param size (IN) Size of region
|
|
* @param flags (IN) Flags indicating what memory access level is requested.
|
|
* See opal/mca/rcache/rcache.h for valid access flags.
|
|
*
|
|
* @returns a memory registration handle valid for both local and remote operations
|
|
* @returns NULL if the region could not be registered
|
|
*
|
|
* This function registers the specified region with the hardware for use with
|
|
* the btl_put, btl_get, btl_atomic_cas, btl_atomic_op, and btl_atomic_fop
|
|
* functions. Care should be taken to not hold an excessive number of registrations
|
|
* as they may use limited system/NIC resources.
|
|
*/
|
|
struct mca_btl_base_registration_handle_t *mca_btl_template_register_mem (
|
|
struct mca_btl_base_module_t* btl, struct mca_btl_base_endpoint_t *endpoint, void *base,
|
|
size_t size, uint32_t flags);
|
|
|
|
/**
|
|
* @brief Deregister a memory region
|
|
*
|
|
* @param btl (IN) BTL module region was registered with
|
|
* @param handle (IN) BTL registration handle to deregister
|
|
*
|
|
* This function deregisters the memory region associated with the specified handle. Care
|
|
* should be taken to not perform any RDMA or atomic operation on this memory region
|
|
* after it is deregistered. It is erroneous to specify a memory handle associated with
|
|
* a remote node.
|
|
*/
|
|
int mca_btl_template_deregister_mem (struct mca_btl_base_module_t* btl,
|
|
struct mca_btl_base_registration_handle_t *handle);
|
|
|
|
/**
|
|
* Register a callback function that is called on receipt
|
|
* of a fragment.
|
|
*
|
|
* @param btl (IN) BTL module
|
|
* @return Status indicating if registration was successful
|
|
*
|
|
*/
|
|
|
|
extern int mca_btl_template_register(
|
|
struct mca_btl_base_module_t* btl,
|
|
mca_btl_base_tag_t tag,
|
|
mca_btl_base_module_recv_cb_fn_t cbfunc,
|
|
void* cbdata);
|
|
|
|
/**
|
|
* Allocate a descriptor with a segment of the requested size.
|
|
* Note that the BTL layer may choose to return a smaller size
|
|
* if it cannot support the request.
|
|
*
|
|
* @param btl (IN) BTL module
|
|
* @param size (IN) Request segment size.
|
|
*/
|
|
|
|
extern mca_btl_base_descriptor_t* mca_btl_template_alloc(
|
|
struct mca_btl_base_module_t* btl,
|
|
struct mca_btl_base_endpoint_t* endpoint,
|
|
uint8_t order,
|
|
size_t size,
|
|
uint32_t flags);
|
|
|
|
|
|
/**
|
|
* Return a segment allocated by this BTL.
|
|
*
|
|
* @param btl (IN) BTL module
|
|
* @param descriptor (IN) Allocated descriptor.
|
|
*/
|
|
|
|
extern int mca_btl_template_free(
|
|
struct mca_btl_base_module_t* btl,
|
|
mca_btl_base_descriptor_t* des);
|
|
|
|
|
|
/**
|
|
* Prepare a descriptor for send/rdma using the supplied
|
|
* convertor. If the convertor references data that is contigous,
|
|
* the descriptor may simply point to the user buffer. Otherwise,
|
|
* this routine is responsible for allocating buffer space and
|
|
* packing if required.
|
|
*
|
|
* @param btl (IN) BTL module
|
|
* @param endpoint (IN) BTL peer addressing
|
|
* @param convertor (IN) Data type convertor
|
|
* @param reserve (IN) Additional bytes requested by upper layer to precede user data
|
|
* @param size (IN/OUT) Number of bytes to prepare (IN), number of bytes actually prepared (OUT)
|
|
*/
|
|
|
|
mca_btl_base_descriptor_t* mca_btl_template_prepare_src(
|
|
struct mca_btl_base_module_t* btl,
|
|
struct mca_btl_base_endpoint_t* peer,
|
|
struct opal_convertor_t* convertor,
|
|
uint8_t order,
|
|
size_t reserve,
|
|
size_t* size,
|
|
uint32_t flags
|
|
);
|
|
|
|
/**
|
|
* Fault Tolerance Event Notification Function
|
|
* @param state Checkpoint Stae
|
|
* @return OPAL_SUCCESS or failure status
|
|
*/
|
|
int mca_btl_template_ft_event(int state);
|
|
|
|
END_C_DECLS
|
|
#endif
|