219 строки
7.3 KiB
C
219 строки
7.3 KiB
C
/*
|
|
* Copyright (c) 2004-2005 The Trustees of Indiana University.
|
|
* All rights reserved.
|
|
* Copyright (c) 2004-2005 The Trustees of the University of Tennessee.
|
|
* All rights reserved.
|
|
* Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
|
|
* University of Stuttgart. All rights reserved.
|
|
* Copyright (c) 2004-2005 The Regents of the University of California.
|
|
* All rights reserved.
|
|
* $COPYRIGHT$
|
|
*
|
|
* Additional copyrights may follow
|
|
*
|
|
* $HEADER$
|
|
*/
|
|
/*
|
|
* @file
|
|
*/
|
|
#ifndef OMPI_BTL_PORTALS_H
|
|
#define OMPI_BTL_PORTALS_H
|
|
|
|
#include "btl_portals_compat.h"
|
|
|
|
#include "ompi/mca/pml/pml.h"
|
|
#include "ompi/mca/btl/btl.h"
|
|
#include "ompi/mca/btl/base/base.h"
|
|
#include "class/ompi_bitmap.h"
|
|
#include "class/ompi_free_list.h"
|
|
#include "class/orte_proc_table.h"
|
|
|
|
#include "btl_portals_endpoint.h"
|
|
|
|
#define OMPI_BTL_PORTALS_SEND_TABLE_ID (OMPI_BTL_PORTALS_STARTING_TABLE_ID + 0)
|
|
#define OMPI_BTL_PORTALS_RDMA_TABLE_ID (OMPI_BTL_PORTALS_STARTING_TABLE_ID + 1)
|
|
|
|
/*
|
|
* Portals BTL component.
|
|
*/
|
|
struct mca_btl_portals_component_t {
|
|
/* base BTL component */
|
|
mca_btl_base_component_1_0_0_t super;
|
|
|
|
/* output channel for debugging. Value settings when using
|
|
* output_verbose:
|
|
*
|
|
* - 0 : critical user information
|
|
* - 10: general execution diagnostic information
|
|
* - 20: initialization / shutdown diagnostic information
|
|
* - 30: basic debugging information
|
|
* - 90: useful only to developers
|
|
* - 100: lots and lots of performance impacting output
|
|
*/
|
|
int portals_output;
|
|
|
|
#if OMPI_BTL_PORTALS_UTCP
|
|
/* ethernet interface to use - only has meaning with utcp
|
|
reference */
|
|
char *portals_ifname;
|
|
#endif
|
|
|
|
/* initial size of free lists */
|
|
int portals_free_list_init_num;
|
|
/* max size of free lists */
|
|
int portals_free_list_max_num;
|
|
/* numer of elements to grow free lists */
|
|
int portals_free_list_inc_num;
|
|
};
|
|
typedef struct mca_btl_portals_component_t mca_btl_portals_component_t;
|
|
|
|
|
|
#define OMPI_BTL_PORTALS_EQ_SEND 0
|
|
#define OMPI_BTL_PORTALS_EQ 1
|
|
#define OMPI_BTL_PORTALS_EQ_SIZE 2
|
|
|
|
struct mca_btl_portals_module_t {
|
|
/* base BTL module interface */
|
|
mca_btl_base_module_t super;
|
|
|
|
/* registered callbacks */
|
|
mca_btl_base_recv_reg_t portals_reg[MCA_BTL_TAG_MAX];
|
|
|
|
/* number of processes we're actively connected to. Needed to
|
|
know when to do activation / shutdown */
|
|
int32_t portals_num_procs;
|
|
|
|
/* fragment free lists */
|
|
ompi_free_list_t portals_frag_eager;
|
|
ompi_free_list_t portals_frag_max;
|
|
ompi_free_list_t portals_frag_user;
|
|
ompi_free_list_t portals_frag_recv;
|
|
|
|
/* incoming send message receive memory descriptors */
|
|
int portals_recv_mds_num;
|
|
int portals_recv_mds_size;
|
|
opal_list_t portals_recv_blocks;
|
|
|
|
/* event queues. Keep sends on own eq, since we can't control
|
|
space for the ack otherwise */
|
|
int portals_eq_sizes[OMPI_BTL_PORTALS_EQ_SIZE];
|
|
ptl_handle_eq_t portals_eq_handles[OMPI_BTL_PORTALS_EQ_SIZE];
|
|
|
|
/* "reject" entry for recv match list */
|
|
ptl_handle_me_t portals_recv_reject_me_h;
|
|
|
|
/* number outstanding sends */
|
|
volatile int32_t portals_outstanding_sends;
|
|
int32_t portals_max_outstanding_sends;
|
|
|
|
/* queued sends */
|
|
opal_list_t portals_queued_sends;
|
|
|
|
/* key to use for next rdma operation */
|
|
volatile int64_t portals_rdma_key;
|
|
|
|
/* our portals network interface */
|
|
ptl_handle_ni_t portals_ni_h;
|
|
|
|
/* number of dropped messages */
|
|
ptl_sr_value_t portals_sr_dropped;
|
|
|
|
/* descriptors for send */
|
|
ptl_md_t md_send;
|
|
};
|
|
typedef struct mca_btl_portals_module_t mca_btl_portals_module_t;
|
|
|
|
/*
|
|
* Component functions (btl_portals_component.c)
|
|
*/
|
|
int mca_btl_portals_component_open(void);
|
|
int mca_btl_portals_component_close(void);
|
|
|
|
|
|
mca_btl_base_module_t** mca_btl_portals_component_init(int *num_btls,
|
|
bool has_progress_threads,
|
|
bool has_mpi_threads);
|
|
|
|
int mca_btl_portals_component_progress(void);
|
|
|
|
/*
|
|
* Compatibility functions (btl_portals_compat_{}.c)
|
|
*
|
|
* Not part of the BTL interface. Need to be implemented for every
|
|
* version of Portals
|
|
*/
|
|
int mca_btl_portals_init_compat(mca_btl_portals_component_t *comp);
|
|
|
|
/* 4th argument is a ptl_peers array, as that's what we'll get back
|
|
from many of the access functions... */
|
|
int mca_btl_portals_add_procs_compat(mca_btl_portals_module_t* btl,
|
|
size_t nprocs, struct ompi_proc_t **procs,
|
|
ptl_process_id_t **ptl_peers);
|
|
|
|
/*
|
|
* Module configuration functions (btl_portals.c)
|
|
*/
|
|
int mca_btl_portals_finalize(struct mca_btl_base_module_t* btl_base);
|
|
|
|
int mca_btl_portals_add_procs(struct mca_btl_base_module_t* btl_base,
|
|
size_t nprocs,
|
|
struct ompi_proc_t **procs,
|
|
struct mca_btl_base_endpoint_t** peers,
|
|
ompi_bitmap_t* reachable);
|
|
|
|
int mca_btl_portals_del_procs(struct mca_btl_base_module_t* btl_base,
|
|
size_t nprocs,
|
|
struct ompi_proc_t **procs,
|
|
struct mca_btl_base_endpoint_t** peers);
|
|
|
|
int mca_btl_portals_register(struct mca_btl_base_module_t* btl_base,
|
|
mca_btl_base_tag_t tag,
|
|
mca_btl_base_module_recv_cb_fn_t cbfunc,
|
|
void* cbdata);
|
|
|
|
mca_btl_base_descriptor_t*
|
|
mca_btl_portals_alloc(struct mca_btl_base_module_t* btl_base,
|
|
size_t size);
|
|
|
|
int mca_btl_portals_free(struct mca_btl_base_module_t* btl_base,
|
|
mca_btl_base_descriptor_t* des);
|
|
|
|
mca_btl_base_descriptor_t*
|
|
mca_btl_portals_prepare_src(struct mca_btl_base_module_t* btl_base,
|
|
struct mca_btl_base_endpoint_t* peer,
|
|
mca_mpool_base_registration_t* registration,
|
|
struct ompi_convertor_t* convertor,
|
|
size_t reserve,
|
|
size_t* size);
|
|
|
|
mca_btl_base_descriptor_t*
|
|
mca_btl_portals_prepare_dst(struct mca_btl_base_module_t* btl_base,
|
|
struct mca_btl_base_endpoint_t* peer,
|
|
mca_mpool_base_registration_t* registration,
|
|
struct ompi_convertor_t* convertor,
|
|
size_t reserve,
|
|
size_t* size);
|
|
|
|
int mca_btl_portals_send(struct mca_btl_base_module_t* btl_base,
|
|
struct mca_btl_base_endpoint_t* btl_peer,
|
|
struct mca_btl_base_descriptor_t* descriptor,
|
|
mca_btl_base_tag_t tag);
|
|
|
|
int mca_btl_portals_put(struct mca_btl_base_module_t* btl_base,
|
|
struct mca_btl_base_endpoint_t* btl_peer,
|
|
struct mca_btl_base_descriptor_t* decriptor);
|
|
|
|
|
|
int mca_btl_portals_get(struct mca_btl_base_module_t* btl_base,
|
|
struct mca_btl_base_endpoint_t* btl_peer,
|
|
struct mca_btl_base_descriptor_t* decriptor);
|
|
|
|
|
|
/*
|
|
* global structures
|
|
*/
|
|
extern mca_btl_portals_component_t mca_btl_portals_component;
|
|
extern mca_btl_portals_module_t mca_btl_portals_module;
|
|
|
|
#endif
|