806 строки
26 KiB
C
806 строки
26 KiB
C
|
/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
|
||
|
|
/*
|
||
|
|
* Copyright (c) 2009-2012 Oak Ridge National Laboratory. All rights reserved.
|
||
|
|
* Copyright (c) 2009-2012 Mellanox Technologies. All rights reserved.
|
||
|
|
* Copyright (c) 2013-2015 Los Alamos National Security, LLC. All rights
|
||
|
|
* reserved.
|
||
|
|
* Copyright (c) 2015 Research Organization for Information Science
|
||
|
|
* and Technology (RIST). All rights reserved.
|
||
|
|
* $COPYRIGHT$
|
||
|
|
*
|
||
|
|
* Additional copyrights may follow
|
||
|
|
*
|
||
|
|
* $HEADER$
|
||
|
|
*/
|
||
|
|
|
||
|
|
#ifndef MCA_BCOL_H
|
||
|
|
#define MCA_BCOL_H
|
||
|
|
|
||
|
|
#include "ompi_config.h"
|
||
|
|
#include "opal/class/opal_list.h"
|
||
|
|
#include "ompi/mca/mca.h"
|
||
|
|
#include "ompi/mca/coll/coll.h"
|
||
|
|
#include "opal/mca/mpool/mpool.h"
|
||
|
|
#include "ompi/mca/sbgp/sbgp.h"
|
||
|
|
#include "ompi/datatype/ompi_datatype.h"
|
||
|
|
#include "ompi/op/op.h"
|
||
|
|
#include "ompi/include/ompi/constants.h"
|
||
|
|
#include "ompi/patterns/net/netpatterns_knomial_tree.h"
|
||
|
|
|
||
|
|
#include "opal/util/show_help.h"
|
||
|
|
|
||
|
|
#include <limits.h>
|
||
|
|
|
||
|
|
#if defined(c_plusplus) || defined(__cplusplus)
|
||
|
|
extern "C" {
|
||
|
|
#endif
|
||
|
|
|
||
|
|
/* Forward declaration - please do not remove it */
|
||
|
|
struct ml_buffers_t;
|
||
|
|
|
||
|
|
struct mca_bcol_base_coll_fn_comm_attributes_t;
|
||
|
|
struct mca_bcol_base_coll_fn_invoke_attributes_t;
|
||
|
|
struct mca_bcol_base_coll_fn_desc_t;
|
||
|
|
|
||
|
|
#define NUM_MSG_RANGES 5
|
||
|
|
#define MSG_RANGE_INITIAL (1024)*12
|
||
|
|
#define MSG_RANGE_INC 10
|
||
|
|
#define BCOL_THRESHOLD_UNLIMITED (INT_MAX)
|
||
|
|
/* Maximum size of a bcol's header. This allows us to correctly calculate the message
|
||
|
|
* thresholds. If the header of any bcol exceeds this value then increase this one
|
||
|
|
* to match. */
|
||
|
|
#define BCOL_HEADER_MAX 96
|
||
|
|
|
||
|
|
#define BCOL_HEAD_ALIGN 32 /* will turn into an MCA parameter after debug */
|
||
|
|
|
||
|
|
/*
|
||
|
|
* Functions supported
|
||
|
|
*/
|
||
|
|
enum bcol_coll {
|
||
|
|
/* blocking functions */
|
||
|
|
BCOL_ALLGATHER,
|
||
|
|
BCOL_ALLGATHERV,
|
||
|
|
BCOL_ALLREDUCE,
|
||
|
|
BCOL_ALLTOALL,
|
||
|
|
BCOL_ALLTOALLV,
|
||
|
|
BCOL_ALLTOALLW,
|
||
|
|
BCOL_BARRIER,
|
||
|
|
BCOL_BCAST,
|
||
|
|
BCOL_EXSCAN,
|
||
|
|
BCOL_GATHER,
|
||
|
|
BCOL_GATHERV,
|
||
|
|
BCOL_REDUCE,
|
||
|
|
BCOL_REDUCE_SCATTER,
|
||
|
|
BCOL_SCAN,
|
||
|
|
BCOL_SCATTER,
|
||
|
|
BCOL_SCATTERV,
|
||
|
|
BCOL_FANIN,
|
||
|
|
BCOL_FANOUT,
|
||
|
|
|
||
|
|
/* nonblocking functions */
|
||
|
|
BCOL_IALLGATHER,
|
||
|
|
BCOL_IALLGATHERV,
|
||
|
|
BCOL_IALLREDUCE,
|
||
|
|
BCOL_IALLTOALL,
|
||
|
|
BCOL_IALLTOALLV,
|
||
|
|
BCOL_IALLTOALLW,
|
||
|
|
BCOL_IBARRIER,
|
||
|
|
BCOL_IBCAST,
|
||
|
|
BCOL_IEXSCAN,
|
||
|
|
BCOL_IGATHER,
|
||
|
|
BCOL_IGATHERV,
|
||
|
|
BCOL_IREDUCE,
|
||
|
|
BCOL_IREDUCE_SCATTER,
|
||
|
|
BCOL_ISCAN,
|
||
|
|
BCOL_ISCATTER,
|
||
|
|
BCOL_ISCATTERV,
|
||
|
|
BCOL_IFANIN,
|
||
|
|
BCOL_IFANOUT,
|
||
|
|
|
||
|
|
BCOL_SYNC,
|
||
|
|
/* New function - needed for intermediate steps */
|
||
|
|
BCOL_REDUCE_TO_LEADER,
|
||
|
|
BCOL_NUM_OF_FUNCTIONS
|
||
|
|
};
|
||
|
|
typedef enum bcol_coll bcol_coll;
|
||
|
|
|
||
|
|
typedef enum bcol_elem_type {
|
||
|
|
BCOL_SINGLE_ELEM_TYPE,
|
||
|
|
BCOL_MULTI_ELEM_TYPE,
|
||
|
|
BCOL_NUM_OF_ELEM_TYPES
|
||
|
|
} bcol_elem_type;
|
||
|
|
|
||
|
|
typedef int (*mca_bcol_base_module_coll_support_all_types_fn_t)(bcol_coll coll_name);
|
||
|
|
typedef int (*mca_bcol_base_module_coll_support_fn_t)(int op, int dtype, bcol_elem_type elem_num);
|
||
|
|
|
||
|
|
/*
|
||
|
|
* Collective function status
|
||
|
|
*/
|
||
|
|
enum {
|
||
|
|
BCOL_FN_NOT_STARTED = (OMPI_ERR_MAX - 1),
|
||
|
|
BCOL_FN_STARTED = (OMPI_ERR_MAX - 2),
|
||
|
|
BCOL_FN_COMPLETE = (OMPI_ERR_MAX - 3)
|
||
|
|
};
|
||
|
|
|
||
|
|
|
||
|
|
|
||
|
|
/**
|
||
|
|
* Collective component initialization
|
||
|
|
*
|
||
|
|
* Initialize the given collective component. This function should
|
||
|
|
* initialize any component-level. data. It will be called exactly
|
||
|
|
* once during MPI_INIT.
|
||
|
|
*
|
||
|
|
* @note The component framework is not lazily opened, so attempts
|
||
|
|
* should be made to minimze the amount of memory allocated during
|
||
|
|
* this function.
|
||
|
|
*
|
||
|
|
* @param[in] enable_progress_threads True if the component needs to
|
||
|
|
* support progress threads
|
||
|
|
* @param[in] enable_mpi_threads True if the component needs to
|
||
|
|
* support MPI_THREAD_MULTIPLE
|
||
|
|
*
|
||
|
|
* @retval OMPI_SUCCESS Component successfully initialized
|
||
|
|
* @retval ORTE_ERROR An unspecified error occurred
|
||
|
|
*/
|
||
|
|
typedef int (*mca_bcol_base_component_init_query_fn_t)
|
||
|
|
(bool enable_progress_threads, bool enable_mpi_threads);
|
||
|
|
|
||
|
|
/**
|
||
|
|
* Query whether a component is available for the given sub-group
|
||
|
|
*
|
||
|
|
* Query whether the component is available for the given
|
||
|
|
* sub-group. If the component is available, an array of pointers should be
|
||
|
|
* allocated and returned (with refcount at 1). The module will not
|
||
|
|
* be used for collective operations until module_enable() is called
|
||
|
|
* on the module, but may be destroyed (via OBJ_RELEASE) either before
|
||
|
|
* or after module_enable() is called. If the module needs to release
|
||
|
|
* resources obtained during query(), it should do so in the module
|
||
|
|
* destructor.
|
||
|
|
*
|
||
|
|
* A component may provide NULL to this function to indicate it does
|
||
|
|
* not wish to run or return an error during module_enable().
|
||
|
|
*
|
||
|
|
* @note The communicator is available for point-to-point
|
||
|
|
* communication, but other functionality is not available during this
|
||
|
|
* phase of initialization.
|
||
|
|
*
|
||
|
|
* @param[in] sbgp Pointer to sub-group module.
|
||
|
|
* @param[out] priority Priority setting for component on
|
||
|
|
* this communicator
|
||
|
|
* @param[out] num_modules Number of modules that where generated
|
||
|
|
* for the sub-group module.
|
||
|
|
*
|
||
|
|
* @returns An array of pointer to an initialized modules structures if the component can
|
||
|
|
* provide a modules with the requested functionality or NULL if the
|
||
|
|
* component should not be used on the given communicator.
|
||
|
|
*/
|
||
|
|
typedef struct mca_bcol_base_module_t **(*mca_bcol_base_component_comm_query_fn_t)
|
||
|
|
(mca_sbgp_base_module_t *sbgp, int *num_modules);
|
||
|
|
|
||
|
|
|
||
|
|
typedef int (*mca_bcol_barrier_init_fn_t)(struct mca_bcol_base_module_t *bcol_module,
|
||
|
|
mca_sbgp_base_module_t *sbgp_module);
|
||
|
|
|
||
|
|
|
||
|
|
|
||
|
|
/*
|
||
|
|
* Macro for use in modules that are of type btl v2.0.0
|
||
|
|
*/
|
||
|
|
#define MCA_BCOL_BASE_VERSION_2_0_0 \
|
||
|
|
OMPI_MCA_BASE_VERSION_2_1_0("bcol", 2, 0, 0)
|
||
|
|
|
||
|
|
|
||
|
|
/* This is really an abstarction violation, but is the easiest way to get
|
||
|
|
* started. For memory management we need to know what bcol components
|
||
|
|
* have compatible memory management schemes. Such compatibility can
|
||
|
|
* be used to eliminate memory copies between levels in the collective
|
||
|
|
* operation hierarchy, by having the output buffer of one level be the
|
||
|
|
* input buffer to the next level
|
||
|
|
*/
|
||
|
|
|
||
|
|
enum {
|
||
|
|
BCOL_SHARED_MEMORY_UMA=0,
|
||
|
|
BCOL_SHARED_MEMORY_SOCKET,
|
||
|
|
BCOL_POINT_TO_POINT,
|
||
|
|
BCOL_IB_OFFLOAD,
|
||
|
|
BCOL_SIZE
|
||
|
|
};
|
||
|
|
|
||
|
|
OMPI_DECLSPEC extern int bcol_mpool_compatibility[BCOL_SIZE][BCOL_SIZE];
|
||
|
|
OMPI_DECLSPEC extern int bcol_mpool_index[BCOL_SIZE][BCOL_SIZE];
|
||
|
|
|
||
|
|
/* what are the input parameters ? too many void * pointers here */
|
||
|
|
typedef int (*bcol_register_mem_fn_t)(void *context_data, void *base,
|
||
|
|
size_t size, void **reg_desc);
|
||
|
|
/* deregistration function */
|
||
|
|
typedef int (*bcol_deregister_mem_fn_t)(void *context_data, void *reg_desc);
|
||
|
|
|
||
|
|
/* Bcol network context definition */
|
||
|
|
struct bcol_base_network_context_t {
|
||
|
|
opal_object_t super;
|
||
|
|
/* Context id - defined by upper layer, ML */
|
||
|
|
int context_id;
|
||
|
|
/* Any context information that bcol what to use */
|
||
|
|
void *context_data;
|
||
|
|
|
||
|
|
/* registration function */
|
||
|
|
bcol_register_mem_fn_t register_memory_fn;
|
||
|
|
/* deregistration function */
|
||
|
|
bcol_deregister_mem_fn_t deregister_memory_fn;
|
||
|
|
};
|
||
|
|
typedef struct bcol_base_network_context_t bcol_base_network_context_t;
|
||
|
|
OMPI_DECLSPEC OBJ_CLASS_DECLARATION(bcol_base_network_context_t);
|
||
|
|
|
||
|
|
/*
|
||
|
|
*primitive function types
|
||
|
|
*/
|
||
|
|
|
||
|
|
/* bcast */
|
||
|
|
enum {
|
||
|
|
/* small data function */
|
||
|
|
BCOL_BCAST_SMALL_DATA,
|
||
|
|
|
||
|
|
/* small data - dynamic decision making supported */
|
||
|
|
BCOL_BCAST_SMALL_DATA_DYNAMIC,
|
||
|
|
|
||
|
|
/* number of functions */
|
||
|
|
BCOL_NUM_BCAST_FUNCTIONS
|
||
|
|
};
|
||
|
|
|
||
|
|
|
||
|
|
/**
|
||
|
|
* BCOL instance.
|
||
|
|
*/
|
||
|
|
|
||
|
|
/* no limit on fragment size - this supports using user buffers rather
|
||
|
|
* than library buffers
|
||
|
|
*/
|
||
|
|
#define FRAG_SIZE_NO_LIMIT -1
|
||
|
|
|
||
|
|
/* forward declaration */
|
||
|
|
struct coll_bcol_collective_description_t;
|
||
|
|
|
||
|
|
struct mca_bcol_base_component_2_0_0_t {
|
||
|
|
|
||
|
|
/** Base component description */
|
||
|
|
mca_base_component_t bcol_version;
|
||
|
|
|
||
|
|
/** Component initialization function */
|
||
|
|
mca_bcol_base_component_init_query_fn_t collm_init_query;
|
||
|
|
|
||
|
|
/** Query whether component is useable for given communicator */
|
||
|
|
mca_bcol_base_component_comm_query_fn_t collm_comm_query;
|
||
|
|
|
||
|
|
/** If bcol supports all possible data types */
|
||
|
|
mca_bcol_base_module_coll_support_fn_t coll_support;
|
||
|
|
|
||
|
|
/** If bcol supports all possible data types for given collective operation */
|
||
|
|
mca_bcol_base_module_coll_support_all_types_fn_t coll_support_all_types;
|
||
|
|
|
||
|
|
/** Use this flag to prevent init_query multiple calls
|
||
|
|
in case we have the same bcol more than on a single level */
|
||
|
|
bool init_done;
|
||
|
|
|
||
|
|
/** If collective calls with bcols of this type need to be ordered */
|
||
|
|
bool need_ordering;
|
||
|
|
|
||
|
|
/** MCA parameter: Priority of this component */
|
||
|
|
int priority;
|
||
|
|
|
||
|
|
/** Bcast function pointers */
|
||
|
|
struct coll_bcol_collective_description_t *
|
||
|
|
bcast_functions[BCOL_NUM_BCAST_FUNCTIONS];
|
||
|
|
|
||
|
|
/** Number of network contexts - need this for resource management */
|
||
|
|
int n_net_contexts;
|
||
|
|
|
||
|
|
/** List of network contexts */
|
||
|
|
bcol_base_network_context_t **network_contexts;
|
||
|
|
|
||
|
|
/*
|
||
|
|
* Fragmentation support
|
||
|
|
*/
|
||
|
|
|
||
|
|
/** Minimum fragement size */
|
||
|
|
int min_frag_size;
|
||
|
|
|
||
|
|
/** Maximum fragment size */
|
||
|
|
int max_frag_size;
|
||
|
|
|
||
|
|
/** Supports direct use of user-buffers */
|
||
|
|
bool can_use_user_buffers;
|
||
|
|
};
|
||
|
|
typedef struct mca_bcol_base_component_2_0_0_t mca_bcol_base_component_2_0_0_t;
|
||
|
|
typedef struct mca_bcol_base_component_2_0_0_t mca_bcol_base_component_t;
|
||
|
|
OMPI_DECLSPEC OBJ_CLASS_DECLARATION(mca_bcol_base_component_t);
|
||
|
|
|
||
|
|
/* forward declaration */
|
||
|
|
struct mca_coll_ml_descriptor_t;
|
||
|
|
struct mca_bcol_base_payload_buffer_desc_t;
|
||
|
|
struct mca_bcol_base_route_info_t;
|
||
|
|
|
||
|
|
typedef struct {
|
||
|
|
int order_num; /* Seq num of collective fragment */
|
||
|
|
int bcols_started; /* How many bcols need ordering have been started */
|
||
|
|
int n_fns_need_ordering; /* The number of functions are called for bcols need ordering */
|
||
|
|
} mca_bcol_base_order_info_t;
|
||
|
|
|
||
|
|
/* structure that encapsultes information propagated amongst multiple
|
||
|
|
* fragments whereby completing the entire ensemble of fragments is
|
||
|
|
* necessary in order to complete the entire collective
|
||
|
|
*/
|
||
|
|
struct bcol_fragment_descriptor_t {
|
||
|
|
/* start iterator */
|
||
|
|
int head;
|
||
|
|
/* end iterator */
|
||
|
|
int tail;
|
||
|
|
/* current iteration */
|
||
|
|
int start_iter;
|
||
|
|
/* number of full iterations this frag */
|
||
|
|
int num_iter;
|
||
|
|
/* end iter */
|
||
|
|
int end_iter;
|
||
|
|
};
|
||
|
|
typedef struct bcol_fragment_descriptor_t bcol_fragment_descriptor_t;
|
||
|
|
|
||
|
|
struct bcol_function_args_t {
|
||
|
|
/* full message sequence number */
|
||
|
|
int64_t sequence_num;
|
||
|
|
/* full message descriptor - single copy of fragment invariant
|
||
|
|
* parameters */
|
||
|
|
/* Pasha: We don need this one for new flow - remove it */
|
||
|
|
struct mca_coll_ml_descriptor_t *full_message_descriptor;
|
||
|
|
struct mca_bcol_base_route_info_t *root_route;
|
||
|
|
/* function status */
|
||
|
|
int function_status;
|
||
|
|
/* root, for rooted operations */
|
||
|
|
int root;
|
||
|
|
/* input buffer */
|
||
|
|
const void *sbuf;
|
||
|
|
void *rbuf;
|
||
|
|
const void *userbuf;
|
||
|
|
struct mca_bcol_base_payload_buffer_desc_t *src_desc;
|
||
|
|
struct mca_bcol_base_payload_buffer_desc_t *dst_desc;
|
||
|
|
/* ml buffer size */
|
||
|
|
uint32_t buffer_size;
|
||
|
|
/* index of buffer in ml payload cache */
|
||
|
|
int buffer_index;
|
||
|
|
int count;
|
||
|
|
struct ompi_datatype_t *dtype;
|
||
|
|
struct ompi_op_t *op;
|
||
|
|
int sbuf_offset;
|
||
|
|
int rbuf_offset;
|
||
|
|
/* for bcol opaque data */
|
||
|
|
void *bcol_opaque_data;
|
||
|
|
/* An output argument that will be used by BCOL function to tell ML that the result of the BCOL is in rbuf */
|
||
|
|
bool result_in_rbuf;
|
||
|
|
bool root_flag; /* True if the rank is root of operation */
|
||
|
|
bool need_dt_support; /* will trigger alternate code path for some colls */
|
||
|
|
int status; /* Used for non-blocking collective completion */
|
||
|
|
uint32_t frag_size; /* fragment size for large messages */
|
||
|
|
int hier_factor; /* factor used when bcast is invoked as a service function back down
|
||
|
|
* the tree in allgather for example, the pacl_len is not the actual
|
||
|
|
* len of the data needing bcasting
|
||
|
|
*/
|
||
|
|
mca_bcol_base_order_info_t order_info;
|
||
|
|
bcol_fragment_descriptor_t frag_info;
|
||
|
|
|
||
|
|
};
|
||
|
|
|
||
|
|
struct mca_bcol_base_route_info_t {
|
||
|
|
int level;
|
||
|
|
int rank;
|
||
|
|
};
|
||
|
|
typedef struct mca_bcol_base_route_info_t mca_bcol_base_route_info_t;
|
||
|
|
|
||
|
|
struct mca_bcol_base_lmngr_block_t {
|
||
|
|
opal_list_item_t super;
|
||
|
|
struct mca_coll_ml_lmngr_t *lmngr;
|
||
|
|
void* base_addr;
|
||
|
|
};
|
||
|
|
typedef struct mca_bcol_base_lmngr_block_t mca_bcol_base_lmngr_block_t;
|
||
|
|
OBJ_CLASS_DECLARATION(mca_bcol_base_lmngr_block_t);
|
||
|
|
|
||
|
|
struct mca_bcol_base_memory_block_desc_t {
|
||
|
|
|
||
|
|
/* memory block for payload buffers */
|
||
|
|
struct mca_bcol_base_lmngr_block_t *block;
|
||
|
|
|
||
|
|
/* Address offset in bytes -- Indicates free memory in the block */
|
||
|
|
uint64_t block_addr_offset;
|
||
|
|
|
||
|
|
/* size of the memory block */
|
||
|
|
size_t size_block;
|
||
|
|
|
||
|
|
/* number of memory banks */
|
||
|
|
uint32_t num_banks;
|
||
|
|
|
||
|
|
/* number of buffers per bank */
|
||
|
|
uint32_t num_buffers_per_bank;
|
||
|
|
|
||
|
|
/* size of a payload buffer */
|
||
|
|
uint32_t size_buffer;
|
||
|
|
|
||
|
|
/* pointer to buffer descriptors initialized */
|
||
|
|
struct mca_bcol_base_payload_buffer_desc_t *buffer_descs;
|
||
|
|
|
||
|
|
/* index of the next free buffer in the block */
|
||
|
|
uint64_t next_free_buffer;
|
||
|
|
|
||
|
|
uint32_t *bank_release_counters;
|
||
|
|
|
||
|
|
/* Counter that defines what bank should be synchronized next
|
||
|
|
* since collectives could be completed out of order, we have to make
|
||
|
|
* sure that memory synchronization collectives started in order ! */
|
||
|
|
int memsync_counter;
|
||
|
|
|
||
|
|
/* This arrays of flags used to signal that the bank is ready for recycling */
|
||
|
|
bool *ready_for_memsync;
|
||
|
|
|
||
|
|
/* This flags monitors if bank is open for usage. Usually we expect that user
|
||
|
|
* will do the check only on buffer-zero allocation */
|
||
|
|
bool *bank_is_busy;
|
||
|
|
|
||
|
|
};
|
||
|
|
|
||
|
|
/* convenience typedef */
|
||
|
|
typedef struct mca_bcol_base_memory_block_desc_t mca_bcol_base_memory_block_desc_t;
|
||
|
|
|
||
|
|
typedef void (*mca_bcol_base_release_buff_fn_t)(struct mca_bcol_base_memory_block_desc_t *ml_memblock, uint32_t buff_id);
|
||
|
|
|
||
|
|
struct mca_bcol_base_payload_buffer_desc_t {
|
||
|
|
void *base_data_addr; /* buffer address */
|
||
|
|
void *data_addr; /* buffer address + header offset */
|
||
|
|
uint64_t generation_number; /* my generation */
|
||
|
|
uint64_t bank_index; /* my bank */
|
||
|
|
uint64_t buffer_index; /* my buff index */
|
||
|
|
};
|
||
|
|
/* convenience typedef */
|
||
|
|
typedef struct mca_bcol_base_payload_buffer_desc_t mca_bcol_base_payload_buffer_desc_t;
|
||
|
|
|
||
|
|
|
||
|
|
|
||
|
|
|
||
|
|
|
||
|
|
|
||
|
|
typedef struct bcol_function_args_t bcol_function_args_t;
|
||
|
|
|
||
|
|
|
||
|
|
/* The collective operation is defined by a series of collective operations
|
||
|
|
* invoked through a function pointer. Each function may be different,
|
||
|
|
* so will store the arguments in a struct and pass a pointer to the struct,
|
||
|
|
* and use this as a way to hide the different function signatures.
|
||
|
|
*
|
||
|
|
* @param[in] input_args Structure with function arguments
|
||
|
|
* @param[in] bcol_desc Component specific paremeters
|
||
|
|
* @param[out] status return status of the function
|
||
|
|
* MCA_BCOL_COMPLETE - function completed
|
||
|
|
* MCA_BCOL_IN_PROGRESS - function incomplete
|
||
|
|
*
|
||
|
|
* @retval OMPI_SUCCESS successful completion
|
||
|
|
* @retval OMPI_ERROR function returned error
|
||
|
|
*/
|
||
|
|
/* forward declaration */
|
||
|
|
struct mca_bcol_base_module_t;
|
||
|
|
|
||
|
|
/* collective function prototype - all functions have the same interface
|
||
|
|
* so that we can call them via a function pointer */
|
||
|
|
struct mca_bcol_base_function_t;
|
||
|
|
typedef int (*mca_bcol_base_module_collective_fn_primitives_t)
|
||
|
|
(bcol_function_args_t *input_args, struct mca_bcol_base_function_t *const_args);
|
||
|
|
|
||
|
|
typedef int (*mca_bcol_base_module_collective_init_fn_primitives_t)
|
||
|
|
(struct mca_bcol_base_module_t *bcol_module);
|
||
|
|
|
||
|
|
/**
|
||
|
|
* function to query for collctive function attributes
|
||
|
|
*
|
||
|
|
* @param attribute (IN) the attribute of interest
|
||
|
|
* @param algorithm_parameters (OUT) the value of attribute for this
|
||
|
|
* function. If this attribute is not supported,
|
||
|
|
* OMPI_ERR_NOT_FOUND is returned.
|
||
|
|
*/
|
||
|
|
typedef int (*mca_bcol_get_collective_attributes)(int attribute,
|
||
|
|
void *algorithm_parameters);
|
||
|
|
|
||
|
|
/* data structure for tracking the relevant data needed for ml level
|
||
|
|
* algorithm construction (e.g., function selection), initialization, and
|
||
|
|
* usage.
|
||
|
|
*/
|
||
|
|
struct coll_bcol_collective_description_t {
|
||
|
|
/* collective initiation function - first functin called */
|
||
|
|
mca_bcol_base_module_collective_fn_primitives_t coll_fn;
|
||
|
|
|
||
|
|
/* collective progress function - first functin called */
|
||
|
|
mca_bcol_base_module_collective_fn_primitives_t progress_fn;
|
||
|
|
|
||
|
|
/* collective progress function - first functin called */
|
||
|
|
mca_bcol_get_collective_attributes get_attributes;
|
||
|
|
|
||
|
|
/* attributes supported - bit map */
|
||
|
|
uint64_t attribute;
|
||
|
|
|
||
|
|
};
|
||
|
|
typedef struct coll_bcol_collective_description_t
|
||
|
|
coll_bcol_collective_description_t;
|
||
|
|
|
||
|
|
/* collective operation attributes */
|
||
|
|
enum {
|
||
|
|
/* supports dynamic decisions - e.g., do not need to have the collective
|
||
|
|
* operation fully defined before it can be started
|
||
|
|
*/
|
||
|
|
BCOL_ATTRIBUTE_DYNAMIC,
|
||
|
|
|
||
|
|
/* number of attributes */
|
||
|
|
BCOL_NUM_ATTRIBUTES
|
||
|
|
};
|
||
|
|
|
||
|
|
/* For rooted collectives,
|
||
|
|
* does the algorithm knows its data source ?
|
||
|
|
*/
|
||
|
|
enum {
|
||
|
|
DATA_SRC_KNOWN=0,
|
||
|
|
DATA_SRC_UNKNOWN,
|
||
|
|
DATA_SRC_TYPES
|
||
|
|
};
|
||
|
|
|
||
|
|
enum {
|
||
|
|
BLOCKING,
|
||
|
|
NON_BLOCKING
|
||
|
|
};
|
||
|
|
/* gvm For selection logic */
|
||
|
|
struct mca_bcol_base_coll_fn_comm_attributes_t {
|
||
|
|
int bcoll_type;
|
||
|
|
int comm_size_min;
|
||
|
|
int comm_size_max;
|
||
|
|
int data_src;
|
||
|
|
int waiting_semantics;
|
||
|
|
};
|
||
|
|
|
||
|
|
typedef struct mca_bcol_base_coll_fn_comm_attributes_t
|
||
|
|
mca_bcol_base_coll_fn_comm_attributes_t;
|
||
|
|
|
||
|
|
struct mca_bcol_base_coll_fn_invoke_attributes_t {
|
||
|
|
int bcol_msg_min;
|
||
|
|
int bcol_msg_max;
|
||
|
|
uint64_t datatype_bitmap; /* Max is OMPI_DATATYPE_MAX_PREDEFINED defined to be 45 */
|
||
|
|
uint32_t op_types_bitmap; /* bit map of optypes supported */
|
||
|
|
};
|
||
|
|
|
||
|
|
typedef struct mca_bcol_base_coll_fn_invoke_attributes_t
|
||
|
|
mca_bcol_base_coll_fn_invoke_attributes_t;
|
||
|
|
|
||
|
|
struct mca_bcol_base_coll_fn_desc_t {
|
||
|
|
opal_list_item_t super;
|
||
|
|
struct mca_bcol_base_coll_fn_comm_attributes_t *comm_attr;
|
||
|
|
struct mca_bcol_base_coll_fn_invoke_attributes_t *inv_attr;
|
||
|
|
mca_bcol_base_module_collective_fn_primitives_t coll_fn;
|
||
|
|
mca_bcol_base_module_collective_fn_primitives_t progress_fn;
|
||
|
|
};
|
||
|
|
|
||
|
|
typedef struct mca_bcol_base_coll_fn_desc_t mca_bcol_base_coll_fn_desc_t;
|
||
|
|
OBJ_CLASS_DECLARATION(mca_bcol_base_coll_fn_desc_t);
|
||
|
|
|
||
|
|
/* end selection logic */
|
||
|
|
|
||
|
|
typedef int (*mca_bcol_base_module_collective_init_fn_t)
|
||
|
|
(struct mca_bcol_base_module_t *bcol_module,
|
||
|
|
mca_sbgp_base_module_t *sbgp_module);
|
||
|
|
|
||
|
|
/* per communicator memory initialization function */
|
||
|
|
typedef int (*mca_bcol_module_mem_init)(struct ml_buffers_t *registered_buffers,
|
||
|
|
mca_bcol_base_component_t *module);
|
||
|
|
|
||
|
|
/* Initialize memory block - ml_memory_block initialization interface function
|
||
|
|
*
|
||
|
|
* Invoked at the ml level, used to pass bcol specific registration information
|
||
|
|
* for the "ml_memory_block"
|
||
|
|
*
|
||
|
|
* @param[in] ml_memory_block Pointer to the ml_memory_block. This struct
|
||
|
|
* contains bcol specific registration information and a call back function
|
||
|
|
* used for resource recycling.
|
||
|
|
*
|
||
|
|
* @param[in] reg_data bcol specific registration data.
|
||
|
|
*
|
||
|
|
* @returns On Success: OMPI_SUCCESS
|
||
|
|
* On Failure: OMPI_ERROR
|
||
|
|
*
|
||
|
|
*/
|
||
|
|
/*typedef int (*mca_bcol_base_init_memory_fn_t)
|
||
|
|
(struct mca_bcol_base_memory_block_desc_t *ml_block, void *reg_data);*/
|
||
|
|
|
||
|
|
typedef int (*mca_bcol_base_init_memory_fn_t)
|
||
|
|
(struct mca_bcol_base_memory_block_desc_t *payload_block,
|
||
|
|
uint32_t data_offset,
|
||
|
|
struct mca_bcol_base_module_t *bcol,
|
||
|
|
void *reg_data);
|
||
|
|
|
||
|
|
typedef int (*mca_common_allgather_init_fn_t)
|
||
|
|
(struct mca_bcol_base_module_t *bcol_module);
|
||
|
|
|
||
|
|
typedef void (*mca_bcol_base_set_thresholds_fn_t)
|
||
|
|
(struct mca_bcol_base_module_t *bcol_module);
|
||
|
|
|
||
|
|
enum {
|
||
|
|
MCA_BCOL_BASE_ZERO_COPY = 1,
|
||
|
|
MCA_BCOL_BASE_NO_ML_BUFFER_FOR_LARGE_MSG = 1 << 1,
|
||
|
|
MCA_BCOL_BASE_NO_ML_BUFFER_FOR_BARRIER = 1 << 2
|
||
|
|
};
|
||
|
|
|
||
|
|
/* base module */
|
||
|
|
struct mca_bcol_base_module_t {
|
||
|
|
/* base coll component */
|
||
|
|
opal_object_t super;
|
||
|
|
|
||
|
|
/* bcol component (Pasha: Do we really need cache the component?)*/
|
||
|
|
mca_bcol_base_component_t *bcol_component;
|
||
|
|
|
||
|
|
/* network context that is used by this bcol
|
||
|
|
only one context per bcol is allowed */
|
||
|
|
bcol_base_network_context_t *network_context;
|
||
|
|
|
||
|
|
/* We are going to use the context index a lot,
|
||
|
|
int order to decrease number of dereferences
|
||
|
|
bcol->network_context->index
|
||
|
|
we are caching the value on bcol */
|
||
|
|
int context_index;
|
||
|
|
|
||
|
|
/* Set of flags that describe features supported by bcol */
|
||
|
|
uint64_t supported_mode;
|
||
|
|
|
||
|
|
/* per communicator memory initialization function */
|
||
|
|
mca_bcol_module_mem_init init_module;
|
||
|
|
|
||
|
|
/* sub-grouping module partner */
|
||
|
|
mca_sbgp_base_module_t *sbgp_partner_module;
|
||
|
|
|
||
|
|
/* size of subgroup - cache this, so can have access when
|
||
|
|
* sbgp_partner_module no longer existes */
|
||
|
|
int size_of_subgroup;
|
||
|
|
|
||
|
|
/* sequence number offset - want to make sure that we start
|
||
|
|
* id'ing collectives with id 0, so we can have simple
|
||
|
|
* resource management.
|
||
|
|
*/
|
||
|
|
int64_t squence_number_offset;
|
||
|
|
|
||
|
|
|
||
|
|
/* number of times to poll for operation completion before
|
||
|
|
* breaking out of a non-blocking collective operation
|
||
|
|
*/
|
||
|
|
int n_poll_loops;
|
||
|
|
|
||
|
|
/* size of header that will go in data buff, should not include
|
||
|
|
* any info regarding alignment, let the ml level handle this
|
||
|
|
*/
|
||
|
|
uint32_t header_size;
|
||
|
|
|
||
|
|
|
||
|
|
/* Each bcol is assigned a unique value
|
||
|
|
* see if we can get away with 16-bit id
|
||
|
|
*/
|
||
|
|
int16_t bcol_id;
|
||
|
|
|
||
|
|
/*FIXME:
|
||
|
|
* Since mca_bcol_base_module_t is the only parameter which will be passed
|
||
|
|
* into the bcol_basesmuma_bcast_init(), add the flag to indicate whether
|
||
|
|
* the hdl-based algorithms will get enabled.
|
||
|
|
*/
|
||
|
|
bool use_hdl;
|
||
|
|
/*
|
||
|
|
* Collective function pointers
|
||
|
|
*/
|
||
|
|
/* changing function signature - will replace bcol_functions */
|
||
|
|
mca_bcol_base_module_collective_fn_primitives_t bcol_function_table[BCOL_NUM_OF_FUNCTIONS];
|
||
|
|
|
||
|
|
/* Tables hold pointers to functions */
|
||
|
|
mca_bcol_base_module_collective_init_fn_primitives_t bcol_function_init_table[BCOL_NUM_OF_FUNCTIONS];
|
||
|
|
opal_list_t bcol_fns_table[BCOL_NUM_OF_FUNCTIONS];
|
||
|
|
struct mca_bcol_base_coll_fn_desc_t*
|
||
|
|
filtered_fns_table[DATA_SRC_TYPES][2][BCOL_NUM_OF_FUNCTIONS][NUM_MSG_RANGES+1][OMPI_OP_NUM_OF_TYPES][OMPI_DATATYPE_MAX_PREDEFINED];
|
||
|
|
|
||
|
|
/*
|
||
|
|
* Bcol interface function to pass bcol specific
|
||
|
|
* info and memory recycling call back
|
||
|
|
*/
|
||
|
|
mca_bcol_base_init_memory_fn_t bcol_memory_init;
|
||
|
|
|
||
|
|
/*
|
||
|
|
* netpatterns interface function, would like to invoke this on
|
||
|
|
* on the ml level
|
||
|
|
*/
|
||
|
|
mca_common_allgather_init_fn_t k_nomial_tree;
|
||
|
|
/* Each bcol caches a list which describes how many ranks
|
||
|
|
* are "below" each rank in this bcol
|
||
|
|
*/
|
||
|
|
int *list_n_connected;
|
||
|
|
|
||
|
|
/* offsets for scatter/gather */
|
||
|
|
int hier_scather_offset;
|
||
|
|
|
||
|
|
/* Small message threshold for each collective */
|
||
|
|
int small_message_thresholds[BCOL_NUM_OF_FUNCTIONS];
|
||
|
|
|
||
|
|
/* Set small_message_thresholds array */
|
||
|
|
mca_bcol_base_set_thresholds_fn_t set_small_msg_thresholds;
|
||
|
|
|
||
|
|
/* Pointer to the order counter on the upper layer,
|
||
|
|
used if the bcol needs to be ordered */
|
||
|
|
int *next_inorder;
|
||
|
|
};
|
||
|
|
typedef struct mca_bcol_base_module_t mca_bcol_base_module_t;
|
||
|
|
OMPI_DECLSPEC OBJ_CLASS_DECLARATION(mca_bcol_base_module_t);
|
||
|
|
|
||
|
|
/* function description */
|
||
|
|
struct mca_bcol_base_function_t {
|
||
|
|
int fn_idx;
|
||
|
|
/* module */
|
||
|
|
struct mca_bcol_base_module_t *bcol_module;
|
||
|
|
|
||
|
|
/*
|
||
|
|
* The following two parameters are used for bcol modules
|
||
|
|
* that want to do some optimizations based on the fact that
|
||
|
|
* n functions from the same bcol module are called in a row.
|
||
|
|
* For example, in the iboffload case, on the first call one
|
||
|
|
* will want to initialize the MWR, and start to instantiate
|
||
|
|
* it, but only post it at the end of the last call.
|
||
|
|
* The index of this function in a sequence of consecutive
|
||
|
|
* functions from the same bcol
|
||
|
|
*/
|
||
|
|
int index_in_consecutive_same_bcol_calls;
|
||
|
|
|
||
|
|
/* number of times functions from this bcol are
|
||
|
|
* called in order
|
||
|
|
*/
|
||
|
|
int n_of_this_type_in_a_row;
|
||
|
|
|
||
|
|
/*
|
||
|
|
* number of times functions from this module are called in the
|
||
|
|
* collective operation.
|
||
|
|
*/
|
||
|
|
int n_of_this_type_in_collective;
|
||
|
|
int index_of_this_type_in_collective;
|
||
|
|
};
|
||
|
|
typedef struct mca_bcol_base_function_t mca_bcol_base_function_t;
|
||
|
|
|
||
|
|
|
||
|
|
|
||
|
|
|
||
|
|
struct mca_bcol_base_descriptor_t {
|
||
|
|
opal_free_list_item_t super;
|
||
|
|
/* Vasily: will be described in the future */
|
||
|
|
};
|
||
|
|
typedef struct mca_bcol_base_descriptor_t mca_bcol_base_descriptor_t;
|
||
|
|
|
||
|
|
static inline __opal_attribute_always_inline__ size_t
|
||
|
|
mca_bcol_base_get_buff_length(ompi_datatype_t *dtype, int count)
|
||
|
|
{
|
||
|
|
ptrdiff_t lb, extent;
|
||
|
|
ompi_datatype_get_extent(dtype, &lb, &extent);
|
||
|
|
|
||
|
|
return (size_t) (extent * count);
|
||
|
|
}
|
||
|
|
|
||
|
|
#define MCA_BCOL_CHECK_ORDER(module, bcol_function_args) \
|
||
|
|
do { \
|
||
|
|
if (*((module)->next_inorder) != \
|
||
|
|
(bcol_function_args)->order_info.order_num) { \
|
||
|
|
return BCOL_FN_NOT_STARTED; \
|
||
|
|
} \
|
||
|
|
} while (0);
|
||
|
|
|
||
|
|
#define MCA_BCOL_UPDATE_ORDER_COUNTER(module, order_info) \
|
||
|
|
do { \
|
||
|
|
(order_info)->bcols_started++; \
|
||
|
|
if ((order_info)->n_fns_need_ordering == \
|
||
|
|
(order_info)->bcols_started) { \
|
||
|
|
++(*((module)->next_inorder)); \
|
||
|
|
} \
|
||
|
|
} while (0);
|
||
|
|
|
||
|
|
#if defined(c_plusplus) || defined(__cplusplus)
|
||
|
|
}
|
||
|
|
#endif
|
||
|
|
#endif /* MCA_BCOL_H */
|