ports/openmpi
ports/openmpi
1
1
Форкнуть 0
Код Релизы Активность

* start cleaning up output statements

Просмотр исходного кода 
* start refactoring duplicate code into inline functions (probably will
  have to become macros, but not until debugging is done)
* general code cleanup

This commit was SVN r5706.
Этот коммит содержится в:
Brian Barrett 2005-05-13 02:54:06 +00:00
родитель fdfe457578
Коммит a7fd494448
7 изменённых файлов: 247 добавлений и 456 удалений
Показать статистику Скачать Patch файл Скачать Diff файл Показать все файлы Свернуть все файлы

12
src/mca/ptl/portals/src/ptl_portals.c
Просмотреть файл

@ -78,14 +78,15 @@ mca_ptl_portals_add_procs(struct mca_ptl_base_module_t* ptl,
ptl_process_id_t *portals_procs; ptl_process_id_t *portals_procs;
size_t i; size_t i;
unsigned long distance; unsigned long distance;
struct mca_ptl_portals_module_t *myptl = (struct mca_ptl_portals_module_t*) ptl; struct mca_ptl_portals_module_t *myptl =
(struct mca_ptl_portals_module_t*) ptl;
/* make sure our environment is fully initialized. At end of this /* make sure our environment is fully initialized. At end of this
call, we have a working network handle on our module and call, we have a working network handle on our module and
portals_procs will have the portals process identifier for each portals_procs will have the portals process identifier for each
proc (ordered, in theory) */ proc (ordered, in theory) */
ret = mca_ptl_portals_add_procs_compat((struct mca_ptl_portals_module_t*) ptl, ret = mca_ptl_portals_add_procs_compat(myptl, nprocs, procs,
nprocs, procs, &portals_procs); &portals_procs);
if (OMPI_SUCCESS != ret) return ret; if (OMPI_SUCCESS != ret) return ret;
/* loop through all procs, setting our reachable flag */ /* loop through all procs, setting our reachable flag */
@ -152,9 +153,6 @@ mca_ptl_portals_module_enable(struct mca_ptl_portals_module_t *ptl,
"Failed to allocate event queue: %d", ret); "Failed to allocate event queue: %d", ret);
return OMPI_ERROR; return OMPI_ERROR;
} }
ompi_output_verbose(100, mca_ptl_portals_component.portals_output,
"allocated event queue: %d",
ptl->frag_eq_handle);
for (i = 0 ; i < ptl->first_frag_num_entries ; ++i) { for (i = 0 ; i < ptl->first_frag_num_entries ; ++i) {
ret = ptl_portals_post_recv_md(ptl, NULL); ret = ptl_portals_post_recv_md(ptl, NULL);
@ -175,7 +173,7 @@ mca_ptl_portals_finalize(struct mca_ptl_base_module_t *ptl_base)
ret = PtlNIFini(ptl->ni_handle); ret = PtlNIFini(ptl->ni_handle);
if (PTL_OK != ret) { if (PTL_OK != ret) {
ompi_output_verbose(90, mca_ptl_portals_component.portals_output, ompi_output_verbose(20, mca_ptl_portals_component.portals_output,
"PtlNIFini returned %d\n", ret); "PtlNIFini returned %d\n", ret);
return OMPI_ERROR; return OMPI_ERROR;
} }

393
src/mca/ptl/portals/src/ptl_portals.h
Просмотреть файл

@ -13,7 +13,7 @@
* *
* $HEADER$ * $HEADER$
*/ */
/** /*
* @file * @file
*/ */
#ifndef MCA_PTL_PORTALS_H #ifndef MCA_PTL_PORTALS_H
@ -27,140 +27,63 @@
#include "class/ompi_free_list.h" #include "class/ompi_free_list.h"
#include "class/ompi_proc_table.h" #include "class/ompi_proc_table.h"
/* need this type for the compat header */
struct mca_ptl_portals_component_t;
typedef struct mca_ptl_portals_component_t mca_ptl_portals_component_t;
/*
/**
* Portals PTL component. * Portals PTL component.
*/ */
struct mca_ptl_portals_component_t { struct mca_ptl_portals_component_t {
/** base PTL component */ /* base PTL component */
mca_ptl_base_component_1_0_0_t super; mca_ptl_base_component_1_0_0_t super;
/** output channel for debugging. Value settings when using /* output channel for debugging. Value settings when using
* output_verbose: * output_verbose:
* *
* - 0 : critical user information * - 0 : critical user information
* - 10: initialization / shutdown diagnostic information * - 10: general execution diagnostic information
* - 20: general execution diagnostic information * - 20: initialization / shutdown diagnostic information
* - 30: basic debugging information
* - 90: useful only to developers * - 90: useful only to developers
* - 100: lots and lots of performance impacting output
*/ */
int portals_output; int portals_output;
#if PTL_PORTALS_UTCP #if PTL_PORTALS_UTCP
/** ethernet interface to use - only has meaning with utcp /* ethernet interface to use - only has meaning with utcp
reference */ reference */
char *portals_ifname; char *portals_ifname;
#endif #endif
/** Number of currently active portals modules */ /* Number of currently active portals modules. We assume these
never change between init and finalize, so these aren't thread
locked */
uint32_t portals_num_modules; uint32_t portals_num_modules;
/** List of currently available modules */ /* List of currently available modules */
struct mca_ptl_portals_module_t **portals_modules; struct mca_ptl_portals_module_t **portals_modules;
/** initial size of free lists */ /* initial size of free lists */
int portals_free_list_init_num; int portals_free_list_init_num;
/** max size of free lists */ /* max size of free lists */
int portals_free_list_max_num; int portals_free_list_max_num;
/** numer of elements to grow free lists */ /* numer of elements to grow free lists */
int portals_free_list_inc_num; int portals_free_list_inc_num;
/** free list of portals send fragments */ /* free list of portals send fragments */
ompi_free_list_t portals_send_frags; ompi_free_list_t portals_send_frags;
/** free list of portals recv fragments */ /* free list of portals recv fragments */
ompi_free_list_t portals_recv_frags; ompi_free_list_t portals_recv_frags;
/** queue of pending sends */ /* queue of pending sends */
ompi_list_t portals_pending_acks; ompi_list_t portals_pending_acks;
/** lock for accessing component */ /* lock for accessing component */
ompi_mutex_t portals_lock; ompi_mutex_t portals_lock;
}; };
typedef struct mca_ptl_portals_component_t mca_ptl_portals_component_t;
struct mca_ptl_portals_recv_frag_t;
struct mca_ptl_portals_send_frag_t;
extern mca_ptl_portals_component_t mca_ptl_portals_component;
/**
* Register Portals module parameters with the MCA framework
*/
extern int mca_ptl_portals_component_open(void);
/**
* Any final cleanup before being unloaded.
*/
extern int mca_ptl_portals_component_close(void);
/**
* MCA->PTL Intializes the PTL component and creates specific PTL
* module(s).
*
* @param num_ptls (OUT) Returns the number of ptl instances created, or 0
* if the transport is not available.
*
* @param allow_multi_user_threads (OUT) Indicated wether this component can
* run at MPI_THREAD_MULTIPLE or not.
*
* @param have_hidden_threads (OUT) Whether this component uses
* hidden threads (e.g., progress threads) or not.
*
* @return Array of pointers to PTL modules, or NULL if the transport
* is not available.
*
* During component initialization, the PTL component should discover
* the physical devices that are available for the given transport,
* and create a PTL instance to represent each device. Any addressing
* information required by peers to reach the device should be published
* during this function via the mca_base_modex_send() interface.
*
*/
extern mca_ptl_base_module_t** mca_ptl_portals_component_init(
int *num_ptls,
bool enable_progress_threads,
bool enable_mpi_threads
);
/**
* MCA->PTL Called to dynamically change a component parameter.
*
* @param flag (IN) Parameter to change.
* @param value (IN) Optional parameter value.
*
* @return OMPI_SUCCESS or error code on failure.
*
* The only supported parameter is currently MCA_PTL_ENABLE,
* which can be used by the PML to enable/disable forwarding
* by the PTL.
*/
extern int mca_ptl_portals_component_control(
int param,
void* value,
size_t size
);
/**
* MCA->PTL Called to progress outstanding requests for
* non-threaded polling environments.
*
* @param tstamp Current time.
* @return OMPI_SUCCESS or error code on failure.
*/
extern int mca_ptl_portals_component_progress(
mca_ptl_tstamp_t tstamp
);
/**
* Portals PTL module.
*/
struct mca_ptl_portals_module_t { struct mca_ptl_portals_module_t {
mca_ptl_base_module_t super; /**< base PTL module interface */ /* base PTL module interface */
mca_ptl_base_module_t super;
/* number of mds for first frags */ /* number of mds for first frags */
int first_frag_num_entries; int first_frag_num_entries;
@ -172,203 +95,95 @@ struct mca_ptl_portals_module_t {
/* frag receive event queue */ /* frag receive event queue */
ptl_handle_eq_t frag_eq_handle; ptl_handle_eq_t frag_eq_handle;
/** our portals network interface */ /* our portals network interface */
ptl_handle_ni_t ni_handle; ptl_handle_ni_t ni_handle;
/** the limits returned from PtlNIInit for interface */ /* the limits returned from PtlNIInit for interface */
ptl_ni_limits_t limits; ptl_ni_limits_t limits;
/** number of dropped messages */ /* number of dropped messages */
ptl_sr_value_t dropped; ptl_sr_value_t dropped;
}; };
typedef struct mca_ptl_portals_module_t mca_ptl_portals_module_t; typedef struct mca_ptl_portals_module_t mca_ptl_portals_module_t;
struct mca_ptl_portals_recv_frag_t;
struct mca_ptl_portals_send_frag_t;
/*
* Component functions (ptl_portals_component.c)
*/
int mca_ptl_portals_component_open(void);
int mca_ptl_portals_component_close(void);
mca_ptl_base_module_t** mca_ptl_portals_component_init(int *num_ptls,
bool has_progress_threads,
bool has_mpi_threads);
int mca_ptl_portals_component_control(int param,
void* value,
size_t size);
int mca_ptl_portals_component_progress(mca_ptl_tstamp_t tstamp);
/*
* Compatibility functions (ptl_portals_compat_{}.c)
*
* Need to be implemented for every version of Portals
*/
int mca_ptl_portals_init(mca_ptl_portals_component_t *comp);
int mca_ptl_portals_add_procs_compat(mca_ptl_portals_module_t* ptl,
size_t nprocs, struct ompi_proc_t **procs,
ptl_process_id_t **portals_procs);
/*
* Module configuration functions (ptl_portals.c)
*/
int mca_ptl_portals_finalize(struct mca_ptl_base_module_t* ptl);
int mca_ptl_portals_add_procs(struct mca_ptl_base_module_t* ptl,
size_t nprocs,
struct ompi_proc_t **procs,
struct mca_ptl_base_peer_t** peers,
ompi_bitmap_t* reachable);
int mca_ptl_portals_del_procs(struct mca_ptl_base_module_t* ptl,
size_t nprocs,
struct ompi_proc_t **procs,
struct mca_ptl_base_peer_t** peers);
int mca_ptl_portals_module_enable(struct mca_ptl_portals_module_t *ptl,
int value);
int mca_ptl_portals_request_init(struct mca_ptl_base_module_t* ptl,
struct mca_ptl_base_send_request_t* req);
void mca_ptl_portals_request_fini(struct mca_ptl_base_module_t* ptl,
struct mca_ptl_base_send_request_t* req);
/*
* Communication functions (ptl_portals_{send,recv}.c)
*/
void mca_ptl_portals_matched(struct mca_ptl_base_module_t* ptl,
struct mca_ptl_base_recv_frag_t* frag);
int mca_ptl_portals_send(struct mca_ptl_base_module_t* ptl,
struct mca_ptl_base_peer_t* ptl_peer,
struct mca_ptl_base_send_request_t*,
size_t offset,
size_t size,
int flags);
int mca_ptl_portals_process_send_event(ptl_event_t *ev);
/*
* global structures
*/
extern mca_ptl_portals_component_t mca_ptl_portals_component;
/* don't use, except as base for creating module instances */
extern mca_ptl_portals_module_t mca_ptl_portals_module; extern mca_ptl_portals_module_t mca_ptl_portals_module;
/**
* Create/initialize the PORTALS PTL modules.
* @return OMPI_SUCCESS or error status on failure.
*/
extern int mca_ptl_portals_module_init(void);
/**
* Cleanup any resources held by the PTL.
*
* @param ptl PTL instance.
* @return OMPI_SUCCESS or error status on failure.
*/
extern int mca_ptl_portals_finalize(
struct mca_ptl_base_module_t* ptl
);
/**
* PML->PTL notification of change in the process list.
*
* @param ptl (IN) PTL instance
* @param nprocs (IN) Number of processes
* @param procs (IN) Set of processes
* @param peer (OUT) Set of (optional) mca_ptl_base_peer_t instances returned by PTL. * @param reachable (OUT) Bitmask indicating set of peer processes that are reachable by this PTL.
* @return OMPI_SUCCESS or error status on failure.
*
* The mca_ptl_base_module_add_procs_fn_t() is called by the PML to
* determine the set of PTLs that should be used to reach each process.
* Any addressing information exported by the peer via the mca_base_modex_send()
* function should be available during this call via the corresponding
* mca_base_modex_recv() function. The PTL may utilize this information to
* determine reachability of each peer process.
*
* For each process that is reachable by the PTL, the bit corresponding to the index
* into the proc array (nprocs) should be set in the reachable bitmask. The PML
* provides the PTL the option to return a pointer to a data structure defined
* by the PTL that is returned to the PTL on subsequent calls to the PTL data
* transfer functions (e.g ptl_send). This may be used by the PTL to cache any addressing
* or connection information (e.g. TCP socket, IP queue pair).
*/
extern int mca_ptl_portals_add_procs(
struct mca_ptl_base_module_t* ptl,
size_t nprocs,
struct ompi_proc_t **procs,
struct mca_ptl_base_peer_t** peers,
ompi_bitmap_t* reachable
);
/**
* PML->PTL notification of change to the process list.
*
* @param ptl (IN) PTL instance
* @param nprocs (IN) Number of processes
* @param proc (IN) Set of processes
* @param peer (IN) Set of peer addressing information.
* @return Status indicating if cleanup was successful
*
* When the process list changes, the PML notifies the PTL of the
* change, to provide the opportunity to cleanup or release any
* resources associated with the peer.
*/
extern int mca_ptl_portals_del_procs(
struct mca_ptl_base_module_t* ptl,
size_t nprocs,
struct ompi_proc_t **procs,
struct mca_ptl_base_peer_t** peers
);
/**
* PML->PTL Initialize a send request for use by the PTL.
*
* @param ptl (IN) PTL instance
* @param request (IN) Pointer to allocated request.
*
* To reduce latency (number of required allocations), the PML allocates up
* to ptl_cache_bytes of additional space contigous w/ the base send request.
* This space may be used by the PTL for additional control information (e.g.
* first fragment descriptor).
*
* The ptl_request_init() function is called by the PML when requests are
* allocated to the PTLs cache. These requests will be cached by the PML
* on completion and re-used by the same PTL w/out additional calls to
* ptl_request_init().
*
* If the cache size is exceeded, the PML may pass requests to ptl_send/ptl_put
* that have been taken from the global pool and have not been initialized by the
* PTL. These requests will have the req_cached attribute set to false.
*
*/
extern int mca_ptl_portals_request_init(
struct mca_ptl_base_module_t* ptl,
struct mca_ptl_base_send_request_t*
);
/**
* PML->PTL Cleanup any resources that may have been associated with the
* request by the PTL.
*
* @param ptl (IN) PTL instance
* @param request (IN) Pointer to allocated request.
*
* The ptl_request_fini function is called when the PML removes a request
* from the PTLs cache (due to resource constraints). This routine provides
* the PTL the chance to cleanup/release any resources cached on the send
* descriptor by the PTL.
*/
extern void mca_ptl_portals_request_fini(
struct mca_ptl_base_module_t* ptl,
struct mca_ptl_base_send_request_t*
);
/**
* PML->PTL Notification from the PML to the PTL that a receive
* has been posted and matched against the indicated fragment.
*
* @param ptl (IN) PTL instance
* @param recv_frag Matched fragment
*
* The ptl_matched() function is called by the PML when a fragment
* is matched to a posted receive. This may occur during a call to
* ptl_match() if the receive is matched, or at a later point in time
* when a matching receive is posted.
*
* When this routine is called, the PTL is responsible for generating
* an acknowledgment to the peer if the MCA_PTL_FLAGS_ACK
* bit is set in the original fragment header. Additionally, the PTL
* is responsible for transferring any data associated with the fragment
* into the users buffer utilizing the datatype engine, and notifying
* the PML that the fragment has completed via the ptl_recv_progress()
* function.
*/
extern void mca_ptl_portals_matched(
struct mca_ptl_base_module_t* ptl,
struct mca_ptl_base_recv_frag_t* frag
);
/**
* PML->PTL Initiate a send to the peer.
*
* @param ptl (IN) PTL instance
* @param ptl_base_peer (IN) PTL peer addressing
* @param request (IN) Send request
* @param offset Current offset into packed/contiguous buffer.
* @param size (IN) Number of bytes PML is requesting PTL to deliver,
* @param flags (IN) Flags that should be passed to the peer via the message header.
* @param request (OUT) OMPI_SUCCESS if the PTL was able to queue one or more fragments
*
* The PML implements a rendevouz protocol, with up to the PTL threshold
* (ptl_first_frag_size) bytes of the message sent in eager send mode. The ptl_send()
* function is called by the PML to initiate the send of the first message fragment.
*
* The PTL is responsible for updating the current data offset (req_offset) in the
* request to reflect the actual number of bytes fragmented. This may be less than
* the requested size, due to resource constraints or datatype alighnment/offset. If
* an acknowledgment is required, the MCA_PTL_FLAGS_ACK bit will be set in the
* flags parameter. In this case, the PTL should not call ptl_send_progress() function
* to indicate completion of the fragment until the ack is received. For all other
* fragments ptl_send_progress() may be called based on local completion semantics.
*/
extern int mca_ptl_portals_send(
struct mca_ptl_base_module_t* ptl,
struct mca_ptl_base_peer_t* ptl_peer,
struct mca_ptl_base_send_request_t*,
size_t offset,
size_t size,
int flags
);
extern int mca_ptl_portals_module_enable(struct mca_ptl_portals_module_t *ptl,
int value);
#endif #endif

6
src/mca/ptl/portals/src/ptl_portals_compat.h
Просмотреть файл

@ -34,10 +34,4 @@
#endif #endif
int mca_ptl_portals_init(mca_ptl_portals_component_t *comp);
int mca_ptl_portals_add_procs_compat(mca_ptl_portals_module_t* ptl,
size_t nprocs, struct ompi_proc_t **procs,
ptl_process_id_t **portals_procs);
#endif /* PTL_PORTALS_NAL_H */ #endif /* PTL_PORTALS_NAL_H */

157
src/mca/ptl/portals/src/ptl_portals_component.c
Просмотреть файл

@ -28,10 +28,6 @@
#include "ptl_portals_send.h" #include "ptl_portals_send.h"
/*
* The portals component
*/
mca_ptl_portals_component_t mca_ptl_portals_component = { mca_ptl_portals_component_t mca_ptl_portals_component = {
{ {
/* First, the mca_base_module_t struct containing meta /* First, the mca_base_module_t struct containing meta
@ -81,7 +77,7 @@ static ompi_output_stream_t portals_output_stream = {
static inline char* static inline char*
mca_ptl_portals_param_register_string(const char* param_name, param_register_string(const char* param_name,
const char* default_value) const char* default_value)
{ {
char *param_value; char *param_value;
@ -94,7 +90,7 @@ mca_ptl_portals_param_register_string(const char* param_name,
static inline int static inline int
mca_ptl_portals_param_register_int(const char* param_name, param_register_int(const char* param_name,
int default_value) int default_value)
{ {
int id = mca_base_param_register_int("ptl", "portals", param_name, int id = mca_base_param_register_int("ptl", "portals", param_name,
@ -105,10 +101,7 @@ mca_ptl_portals_param_register_int(const char* param_name,
} }
/*
* Called by MCA framework to open the module, registers
* module parameters.
*/
int int
mca_ptl_portals_component_open(void) mca_ptl_portals_component_open(void)
{ {
@ -117,63 +110,65 @@ mca_ptl_portals_component_open(void)
mca_ptl_portals_component.portals_modules = NULL; mca_ptl_portals_component.portals_modules = NULL;
/* initialize objects */ /* initialize objects */
OBJ_CONSTRUCT(&mca_ptl_portals_component.portals_send_frags, ompi_free_list_t); OBJ_CONSTRUCT(&mca_ptl_portals_component.portals_send_frags,
OBJ_CONSTRUCT(&mca_ptl_portals_component.portals_recv_frags, ompi_free_list_t); ompi_free_list_t);
OBJ_CONSTRUCT(&mca_ptl_portals_component.portals_pending_acks, ompi_list_t); OBJ_CONSTRUCT(&mca_ptl_portals_component.portals_recv_frags,
OBJ_CONSTRUCT(&mca_ptl_portals_component.portals_lock, ompi_mutex_t); ompi_free_list_t);
OBJ_CONSTRUCT(&mca_ptl_portals_component.portals_pending_acks,
ompi_list_t);
OBJ_CONSTRUCT(&mca_ptl_portals_component.portals_lock,
ompi_mutex_t);
/* register portals module parameters */ /* register portals module parameters */
#if PTL_PORTALS_UTCP #if PTL_PORTALS_UTCP
mca_ptl_portals_component.portals_ifname = mca_ptl_portals_component.portals_ifname =
mca_ptl_portals_param_register_string("ifname", "eth0"); param_register_string("ifname", "eth0");
#endif #endif
portals_output_stream.lds_verbose_level = portals_output_stream.lds_verbose_level =
mca_ptl_portals_param_register_int("debug_level", param_register_int("debug_level",
PTL_PORTALS_DEFAULT_DEBUG_LEVEL); PTL_PORTALS_DEFAULT_DEBUG_LEVEL);
mca_ptl_portals_component.portals_free_list_init_num = mca_ptl_portals_component.portals_free_list_init_num =
mca_ptl_portals_param_register_int("free_list_init_num", param_register_int("free_list_init_num",
PTL_PORTALS_DEFAULT_FREE_LIST_INIT_NUM); PTL_PORTALS_DEFAULT_FREE_LIST_INIT_NUM);
mca_ptl_portals_component.portals_free_list_max_num = mca_ptl_portals_component.portals_free_list_max_num =
mca_ptl_portals_param_register_int("free_list_max_num", param_register_int("free_list_max_num",
PTL_PORTALS_DEFAULT_FREE_LIST_MAX_NUM); PTL_PORTALS_DEFAULT_FREE_LIST_MAX_NUM);
mca_ptl_portals_component.portals_free_list_inc_num = mca_ptl_portals_component.portals_free_list_inc_num =
mca_ptl_portals_param_register_int("free_list_inc_num", param_register_int("free_list_inc_num",
PTL_PORTALS_DEFAULT_FREE_LIST_inc_NUM); PTL_PORTALS_DEFAULT_FREE_LIST_inc_NUM);
mca_ptl_portals_module.super.ptl_cache_size = mca_ptl_portals_module.super.ptl_cache_size =
mca_ptl_portals_param_register_int("request_cache_size", param_register_int("request_cache_size",
PTL_PORTALS_DEFAULT_REQUEST_CACHE_SIZE); PTL_PORTALS_DEFAULT_REQUEST_CACHE_SIZE);
mca_ptl_portals_module.super.ptl_first_frag_size = mca_ptl_portals_module.super.ptl_first_frag_size =
mca_ptl_portals_param_register_int("first_frag_size", param_register_int("first_frag_size",
PTL_PORTALS_DEFAULT_FIRST_FRAG_SIZE); PTL_PORTALS_DEFAULT_FIRST_FRAG_SIZE);
mca_ptl_portals_module.super.ptl_min_frag_size = mca_ptl_portals_module.super.ptl_min_frag_size =
mca_ptl_portals_param_register_int("rndv_frag_min_size", param_register_int("rndv_frag_min_size",
PTL_PORTALS_DEFAULT_RNDV_FRAG_MIN_SIZE); PTL_PORTALS_DEFAULT_RNDV_FRAG_MIN_SIZE);
mca_ptl_portals_module.super.ptl_max_frag_size = mca_ptl_portals_module.super.ptl_max_frag_size =
mca_ptl_portals_param_register_int("rndv_frag_max_size", param_register_int("rndv_frag_max_size",
PTL_PORTALS_DEFAULT_RNDV_FRAG_MAX_SIZE); PTL_PORTALS_DEFAULT_RNDV_FRAG_MAX_SIZE);
mca_ptl_portals_module.first_frag_num_entries = mca_ptl_portals_module.first_frag_num_entries =
mca_ptl_portals_param_register_int("first_frag_num_entries", param_register_int("first_frag_num_entries",
PTL_PORTALS_DEFAULT_FIRST_FRAG_NUM_ENTRIES); PTL_PORTALS_DEFAULT_FIRST_FRAG_NUM_ENTRIES);
mca_ptl_portals_module.first_frag_entry_size = mca_ptl_portals_module.first_frag_entry_size =
mca_ptl_portals_param_register_int("first_frag_entry_size", param_register_int("first_frag_entry_size",
PTL_PORTALS_DEFAULT_FIRST_FRAG_ENTRY_SIZE); PTL_PORTALS_DEFAULT_FIRST_FRAG_ENTRY_SIZE);
mca_ptl_portals_module.first_frag_queue_size = mca_ptl_portals_module.first_frag_queue_size =
mca_ptl_portals_param_register_int("first_frag_queue_size", param_register_int("first_frag_queue_size",
PTL_PORTALS_DEFAULT_FIRST_FRAG_QUEUE_SIZE); PTL_PORTALS_DEFAULT_FIRST_FRAG_QUEUE_SIZE);
/* finish with objects */ /* finish with objects */
asprintf(&(portals_output_stream.lds_prefix), "ptl_portals (%5d): ", getpid()); asprintf(&(portals_output_stream.lds_prefix),
"ptl: portals (%5d): ", getpid());
mca_ptl_portals_component.portals_output = mca_ptl_portals_component.portals_output =
ompi_output_open(&portals_output_stream); ompi_output_open(&portals_output_stream);
ompi_output_verbose(90, mca_ptl_portals_component.portals_output, /* fill in remaining defaults for module data */
"mca_ptl_portals_component_open()");
/* fill in defaults for module data */
mca_ptl_portals_module.frag_eq_handle = PTL_EQ_NONE; mca_ptl_portals_module.frag_eq_handle = PTL_EQ_NONE;
mca_ptl_portals_module.ni_handle = PTL_INVALID_HANDLE; mca_ptl_portals_module.ni_handle = PTL_INVALID_HANDLE;
mca_ptl_portals_module.dropped = 0; mca_ptl_portals_module.dropped = 0;
@ -182,18 +177,11 @@ mca_ptl_portals_component_open(void)
} }
/*
* module cleanup - sanity checking of queue lengths
*/
int int
mca_ptl_portals_component_close(void) mca_ptl_portals_component_close(void)
{ {
ompi_output_verbose(90, mca_ptl_portals_component.portals_output,
"mca_ptl_portals_component_close()");
/* finalize interface? */
/* print out debugging if anything is pending */ /* print out debugging if anything is pending */
/* BWB - implement me, if possible */
/* release resources */ /* release resources */
OBJ_DESTRUCT(&mca_ptl_portals_component.portals_lock); OBJ_DESTRUCT(&mca_ptl_portals_component.portals_lock);
@ -209,13 +197,13 @@ mca_ptl_portals_component_close(void)
free(portals_output_stream.lds_prefix); free(portals_output_stream.lds_prefix);
} }
ompi_output_close(mca_ptl_portals_component.portals_output);
mca_ptl_portals_component.portals_output = -1;
return OMPI_SUCCESS; return OMPI_SUCCESS;
} }
/*
* portals module initialization.
*/
mca_ptl_base_module_t** mca_ptl_base_module_t**
mca_ptl_portals_component_init(int *num_ptls, mca_ptl_portals_component_init(int *num_ptls,
bool enable_progress_threads, bool enable_progress_threads,
@ -224,11 +212,11 @@ mca_ptl_portals_component_init(int *num_ptls,
mca_ptl_base_module_t** ptls; mca_ptl_base_module_t** ptls;
*num_ptls = 0; *num_ptls = 0;
ompi_output_verbose(90, mca_ptl_portals_component.portals_output, if (enable_progress_threads) {
"mca_ptl_portals_component_init()"); ompi_output_verbose(20, mca_ptl_portals_component.portals_output,
"disabled because progress threads enabled");
/* BWB - no support for threads */ return NULL;
if (enable_progress_threads || enable_mpi_threads) return NULL; }
ompi_free_list_init(&mca_ptl_portals_component.portals_send_frags, ompi_free_list_init(&mca_ptl_portals_component.portals_send_frags,
sizeof(mca_ptl_portals_send_frag_t), sizeof(mca_ptl_portals_send_frag_t),
@ -249,7 +237,8 @@ mca_ptl_portals_component_init(int *num_ptls,
/* initialize portals ptl. note that this is in the compat code because /* initialize portals ptl. note that this is in the compat code because
it's fairly non-portable between implementations */ it's fairly non-portable between implementations */
if (OMPI_SUCCESS != mca_ptl_portals_init(&mca_ptl_portals_component)) { if (OMPI_SUCCESS != mca_ptl_portals_init(&mca_ptl_portals_component)) {
/* error message should already be displayed */ ompi_output_verbose(20, mca_ptl_portals_component.portals_output,
"disabled because compatibility init failed");
return NULL; return NULL;
} }
@ -263,32 +252,35 @@ mca_ptl_portals_component_init(int *num_ptls,
mca_ptl_portals_component.portals_num_modules * mca_ptl_portals_component.portals_num_modules *
sizeof(mca_ptl_base_module_t*)); sizeof(mca_ptl_base_module_t*));
*num_ptls = mca_ptl_portals_component.portals_num_modules; *num_ptls = mca_ptl_portals_component.portals_num_modules;
ompi_output_verbose(20, mca_ptl_portals_component.portals_output,
"initialized %d modules",
*num_ptls);
return ptls; return ptls;
} }
/*
* Portals module control
*/
int int
mca_ptl_portals_component_control(int param, void* value, size_t size) mca_ptl_portals_component_control(int param, void* value, size_t size)
{ {
uint32_t i; uint32_t i;
int ret = OMPI_SUCCESS; int ret = OMPI_SUCCESS;
ompi_output_verbose(100, mca_ptl_portals_component.portals_output, ompi_output_verbose(30, mca_ptl_portals_component.portals_output,
"mca_ptl_portals_component_control(%d, %d)", "component control: %d, %d",
param, (*(int*) value)); param, (*(int*) value));
switch(param) { switch(param) {
case MCA_PTL_ENABLE: case MCA_PTL_ENABLE:
OMPI_THREAD_LOCK(&mca_ptl_portals_component.portals_lock); for (i = 0 ;
for (i = 0 ; i < mca_ptl_portals_component.portals_num_modules ; ++i) { i < mca_ptl_portals_component.portals_num_modules ;
ret = mca_ptl_portals_module_enable(mca_ptl_portals_component.portals_modules[i], ++i) {
*(int*)value); ret = mca_ptl_portals_module_enable(
mca_ptl_portals_component.portals_modules[i],
*(int*)value);
if (ret != OMPI_SUCCESS) break; if (ret != OMPI_SUCCESS) break;
} }
OMPI_THREAD_UNLOCK(&mca_ptl_portals_component.portals_lock);
break; break;
default: default:
break; break;
@ -297,25 +289,23 @@ mca_ptl_portals_component_control(int param, void* value, size_t size)
} }
/*
* Portals module progress.
*/
int int
mca_ptl_portals_component_progress(mca_ptl_tstamp_t tstamp) mca_ptl_portals_component_progress(mca_ptl_tstamp_t tstamp)
{ {
int num_progressed = 0; int num_progressed = 0;
size_t i; size_t i;
int ret;
int which;
for (i = 0 ; i < mca_ptl_portals_component.portals_num_modules ; ++i) { for (i = 0 ; i < mca_ptl_portals_component.portals_num_modules ; ++i) {
struct mca_ptl_portals_module_t *module = struct mca_ptl_portals_module_t *module =
mca_ptl_portals_component.portals_modules[i]; mca_ptl_portals_component.portals_modules[i];
ptl_event_t ev; ptl_event_t ev;
ptl_sr_value_t numdropped; ptl_sr_value_t numdropped;
int which;
int ret;
if (module->frag_eq_handle == PTL_EQ_NONE) continue; if (module->frag_eq_handle == PTL_EQ_NONE) continue;
#if OMPI_ENABLE_DEBUG
/* BWB - this is going to kill performance */ /* BWB - this is going to kill performance */
PtlNIStatus(module->ni_handle, PtlNIStatus(module->ni_handle,
PTL_SR_DROP_COUNT, PTL_SR_DROP_COUNT,
@ -326,6 +316,7 @@ mca_ptl_portals_component_progress(mca_ptl_tstamp_t tstamp)
module->dropped, numdropped); module->dropped, numdropped);
module->dropped = numdropped; module->dropped = numdropped;
} }
#endif
ret = PtlEQPoll(&(module->frag_eq_handle), ret = PtlEQPoll(&(module->frag_eq_handle),
1, /* number of eq handles */ 1, /* number of eq handles */
@ -336,33 +327,35 @@ mca_ptl_portals_component_progress(mca_ptl_tstamp_t tstamp)
/* nothing to see here - move along */ /* nothing to see here - move along */
continue; continue;
} else if (!(PTL_OK == ret || PTL_EQ_DROPPED == ret)) { } else if (!(PTL_OK == ret || PTL_EQ_DROPPED == ret)) {
/* BWB - we need to figure out what to do here - this is not /* BWB - how can we report errors? */
supposed to happen */
ompi_output(mca_ptl_portals_component.portals_output, ompi_output(mca_ptl_portals_component.portals_output,
"Error calling PtlEQGet: %d", ret); "Error calling PtlEQGet: %d", ret);
continue; continue;
} else if (PTL_EQ_DROPPED == ret) { } else if (PTL_EQ_DROPPED == ret) {
/* BWB - drop events should be handled already, but nice to know ompi_output_verbose(10, mca_ptl_portals_component.portals_output,
they happened. */ "event queue entries were dropped");
ompi_output_verbose(20, mca_ptl_portals_component.portals_output,
"Progress found dropped packets");
} }
/* only one place we can have an event */ /* only one place we can have an event */
assert(which == 0); assert(which == 0);
#if PTL_PORTALS_HAVE_EVENT_UNLINK #if PTL_PORTALS_HAVE_EVENT_UNLINK
/* not everyone has UNLINK. Use it only to print the event,
so we can make sure we properly re-initialize the ones that
need to be re-initialized */
if (PTL_EVENT_UNLINK == ev.type) { if (PTL_EVENT_UNLINK == ev.type) {
ompi_output_verbose(100, mca_ptl_portals_component.portals_output, ompi_output_verbose(100, mca_ptl_portals_component.portals_output,
"unlink event occurred"); "unlink event occurred");
continue; continue;
} }
#endif #endif
if (ev.md.user_ptr == NULL) { if (ev.md.user_ptr == NULL) {
/* no request associated with it - it's a receive */ /* no fragment associated with it - it's a receive */
mca_ptl_portals_process_recv_event(module, &ev); mca_ptl_portals_process_recv_event(module, &ev);
} else { } else {
/* there's a request associated with it */ /* there's a fragment associated with it - choose based on
frag type */
mca_ptl_base_frag_t *frag = mca_ptl_base_frag_t *frag =
(mca_ptl_base_frag_t*) ev.md.user_ptr; (mca_ptl_base_frag_t*) ev.md.user_ptr;
if (frag->frag_type == MCA_PTL_FRAGMENT_SEND) { if (frag->frag_type == MCA_PTL_FRAGMENT_SEND) {

21
src/mca/ptl/portals/src/ptl_portals_recv.c
Просмотреть файл

@ -242,23 +242,10 @@ mca_ptl_portals_process_recv_event(struct mca_ptl_portals_module_t *ptl,
sendreq->req_peer_match = hdr->hdr_ack.hdr_dst_match; sendreq->req_peer_match = hdr->hdr_ack.hdr_dst_match;
ompi_output_verbose(100, mca_ptl_portals_component.portals_output, OMPI_OUTPUT_VERBOSE((100, mca_ptl_portals_component.portals_output,
"received ack for request %p", "received ack for request %p",
hdr->hdr_ack.hdr_dst_match); hdr->hdr_ack.hdr_dst_match));
mca_ptl_portals_complete_send_event(sendfrag);
sendfrag->frag_send.frag_base.frag_owner->
ptl_send_progress(sendfrag->frag_send.frag_base.frag_owner,
sendfrag->frag_send.frag_request,
sendfrag->frag_send.frag_base.frag_size);
/* return frag to freelist if not part of request */
if (sendfrag->frag_send.frag_request->req_cached == false) {
if (sendfrag->frag_send.frag_base.frag_addr == NULL) {
free(sendfrag->frag_send.frag_base.frag_addr);
}
OMPI_FREE_LIST_RETURN(&mca_ptl_portals_component.portals_send_frags,
(ompi_list_item_t*) sendfrag);
}
} }
break; break;

55
src/mca/ptl/portals/src/ptl_portals_send.c
Просмотреть файл

@ -188,56 +188,39 @@ mca_ptl_portals_process_send_event(ptl_event_t *ev)
&(frag->frag_send.frag_base.frag_header); &(frag->frag_send.frag_base.frag_header);
if (ev->type == PTL_EVENT_SEND_START) { if (ev->type == PTL_EVENT_SEND_START) {
ompi_output_verbose(100, mca_ptl_portals_component.portals_output, OMPI_OUTPUT_VERBOSE((100, mca_ptl_portals_component.portals_output,
"SEND_START event for msg %d, length: %d", "ptl event send start for msg %d, length: %d",
(int) hdr->hdr_match.hdr_msg_seq, (int) hdr->hdr_match.hdr_msg_seq,
(int) ev->mlength); (int) ev->mlength));
} else if (ev->type == PTL_EVENT_SEND_END) { } else if (ev->type == PTL_EVENT_SEND_END) {
ompi_output_verbose(100, mca_ptl_portals_component.portals_output, OMPI_OUTPUT_VERBOSE((100, mca_ptl_portals_component.portals_output,
"SEND_END event for msg %d", "ptl event send end for msg %d",
(int) hdr->hdr_match.hdr_msg_seq); (int) hdr->hdr_match.hdr_msg_seq));
} else if (ev->type == PTL_EVENT_ACK) { } else if (ev->type == PTL_EVENT_ACK) {
ompi_output_verbose(100, mca_ptl_portals_component.portals_output, OMPI_OUTPUT_VERBOSE((100, mca_ptl_portals_component.portals_output,
"ACK event for msg %d", "ptl event ack for msg %d",
(int) hdr->hdr_match.hdr_msg_seq); (int) hdr->hdr_match.hdr_msg_seq));
/* discard ACKs for acks */
if (frag->frag_send.frag_request == NULL) { if (frag->frag_send.frag_request == NULL) {
/* if request is NULL, it's an ACK - just return the frag
to the pool */
OMPI_FREE_LIST_RETURN(&mca_ptl_portals_component.portals_send_frags, OMPI_FREE_LIST_RETURN(&mca_ptl_portals_component.portals_send_frags,
(ompi_list_item_t*) frag); (ompi_list_item_t*) frag);
} else { } else {
bool frag_ack; /* it's a completion of a data fragment */
bool frag_ack = (hdr->hdr_common.hdr_flags & MCA_PTL_FLAGS_ACK) ?
true : false;
frag_ack = (hdr->hdr_common.hdr_flags & MCA_PTL_FLAGS_ACK) ? true : false;
if (frag_ack == false) { if (frag_ack == false) {
/* this frag is done! */ /* data frag is done and we aren't waiting on an ack.
complete it. if waiting for an ack, will be
/* let the PML know */ completed when process_recv_event sees an ack */
frag->frag_send.frag_base.frag_owner-> mca_ptl_portals_complete_send_event(frag);
ptl_send_progress(frag->frag_send.frag_base.frag_owner,
frag->frag_send.frag_request,
frag->frag_send.frag_base.frag_size);
/* return frag to freelist if not part of request */
if (frag->frag_send.frag_request->req_cached == false ||
frag->frag_send.frag_base.frag_header.hdr_common.hdr_type == MCA_PTL_HDR_TYPE_FRAG) {
if (frag->free_data) {
free(frag->frag_vector[1].iov_base);
}
OMPI_FREE_LIST_RETURN(&mca_ptl_portals_component.portals_send_frags,
(ompi_list_item_t*) frag);
}
} else {
/* need to wait for the ack... */
;
} }
} }
#if 0
/* unlink memory descriptor */ /* unlink memory descriptor */
PtlMDUnlink(ev->md_handle); PtlMDUnlink(ev->md_handle);
#endif
} else { } else {
ompi_output_verbose(10, mca_ptl_portals_component.portals_output, ompi_output_verbose(10, mca_ptl_portals_component.portals_output,
"unknown event for msg %d: %d", "unknown event for msg %d: %d",

57
src/mca/ptl/portals/src/ptl_portals_send.h
Просмотреть файл

@ -20,6 +20,7 @@
#include "mca/ptl/base/ptl_base_sendfrag.h" #include "mca/ptl/base/ptl_base_sendfrag.h"
#include "mca/ptl/base/ptl_base_recvfrag.h" #include "mca/ptl/base/ptl_base_recvfrag.h"
#include "mca/ptl/base/ptl_base_sendreq.h"
#include "ptl_portals_recv.h" #include "ptl_portals_recv.h"
#if defined(c_plusplus) || defined(__cplusplus) #if defined(c_plusplus) || defined(__cplusplus)
@ -38,9 +39,6 @@ extern "C" {
} }
#endif #endif
extern int mca_ptl_portals_process_send_event(ptl_event_t *ev);
static inline int static inline int
mca_ptl_portals_send_frag(struct mca_ptl_portals_module_t *ptl, mca_ptl_portals_send_frag(struct mca_ptl_portals_module_t *ptl,
mca_ptl_portals_send_frag_t* sendfrag) mca_ptl_portals_send_frag_t* sendfrag)
@ -80,11 +78,11 @@ mca_ptl_portals_send_frag(struct mca_ptl_portals_module_t *ptl,
if (ret != PTL_OK) { if (ret != PTL_OK) {
ompi_output(mca_ptl_portals_component.portals_output, ompi_output(mca_ptl_portals_component.portals_output,
"PtlPut failed with error %d", ret); "PtlPut failed with error %d", ret);
PtlMDUnlink(md_handle);
return OMPI_ERROR; return OMPI_ERROR;
} }
return OMPI_SUCCESS; return OMPI_SUCCESS;
} }
@ -101,12 +99,10 @@ mca_ptl_portals_send_ack(struct mca_ptl_portals_module_t *ptl,
/* get a fragment */ /* get a fragment */
OMPI_FREE_LIST_GET(&mca_ptl_portals_component.portals_send_frags, OMPI_FREE_LIST_GET(&mca_ptl_portals_component.portals_send_frags,
item, ret); item, ret);
if (NULL == item) { if (NULL == item) return ret;
/* BWB - fix me */
return ret;
}
sendfrag = (mca_ptl_portals_send_frag_t *) item; sendfrag = (mca_ptl_portals_send_frag_t *) item;
/* no payload */
sendfrag->frag_vector[1].iov_base = NULL; sendfrag->frag_vector[1].iov_base = NULL;
sendfrag->frag_vector[1].iov_len = 0; sendfrag->frag_vector[1].iov_len = 0;
@ -116,27 +112,52 @@ mca_ptl_portals_send_ack(struct mca_ptl_portals_module_t *ptl,
hdr->hdr_ack.hdr_common.hdr_type = MCA_PTL_HDR_TYPE_ACK; hdr->hdr_ack.hdr_common.hdr_type = MCA_PTL_HDR_TYPE_ACK;
hdr->hdr_ack.hdr_common.hdr_flags = 0; hdr->hdr_ack.hdr_common.hdr_flags = 0;
hdr->hdr_ack.hdr_src_ptr = recvfrag->frag_recv.frag_base.frag_header.hdr_rndv.hdr_src_ptr; #if OMPI_ENABLE_MEM_DEBUG
hdr->hdr_ack.hdr_dst_match.lval = 0; /* for VALGRIND/PURIFY - REPLACE WITH MACRO */ hdr->hdr_ack.hdr_dst_match.lval = 0;
hdr->hdr_ack.hdr_dst_addr.lval = 0;
#endif
hdr->hdr_ack.hdr_src_ptr =
recvfrag->frag_recv.frag_base.frag_header.hdr_rndv.hdr_src_ptr;
hdr->hdr_ack.hdr_dst_match.pval = request; hdr->hdr_ack.hdr_dst_match.pval = request;
hdr->hdr_ack.hdr_dst_addr.lval = 0; /* for VALGRIND/PURIFY - REPLACE WITH MACRO */
hdr->hdr_ack.hdr_dst_addr.pval = request->req_recv.req_base.req_addr; hdr->hdr_ack.hdr_dst_addr.pval = request->req_recv.req_base.req_addr;
hdr->hdr_ack.hdr_dst_size = request->req_recv.req_bytes_packed; hdr->hdr_ack.hdr_dst_size = request->req_recv.req_bytes_packed;
/* can ignore most of the fragment, but need to make sure the
request is NULL so that process_send_event knows it's an ack
completing */
sendfrag->frag_send.frag_request = NULL; sendfrag->frag_send.frag_request = NULL;
sendfrag->frag_send.frag_base.frag_peer = (struct mca_ptl_base_peer_t*) &(recvfrag->frag_source); sendfrag->frag_send.frag_base.frag_peer =
sendfrag->frag_send.frag_base.frag_owner = &ptl->super; (struct mca_ptl_base_peer_t*) &(recvfrag->frag_source);
sendfrag->frag_send.frag_base.frag_addr = NULL;
sendfrag->frag_send.frag_base.frag_size = 0;
sendfrag->frag_vector[0].iov_len = sizeof(mca_ptl_base_ack_header_t); sendfrag->frag_vector[0].iov_len = sizeof(mca_ptl_base_ack_header_t);
ompi_output_verbose(100, mca_ptl_portals_component.portals_output, OMPI_OUTPUT_VERBOSE((100, mca_ptl_portals_component.portals_output,
"sending ack for request %p", request); "sending ack for request %p", request));
return mca_ptl_portals_send_frag(ptl, sendfrag); return mca_ptl_portals_send_frag(ptl, sendfrag);
} }
static inline void
mca_ptl_portals_complete_send_event(mca_ptl_portals_send_frag_t* frag)
{
frag->frag_send.frag_base.frag_owner->
ptl_send_progress(frag->frag_send.frag_base.frag_owner,
frag->frag_send.frag_request,
frag->frag_send.frag_base.frag_size);
/* return frag to freelist if not part of request */
if (frag->frag_send.frag_request->req_cached == false ||
frag->frag_send.frag_base.frag_header.hdr_common.hdr_type ==
MCA_PTL_HDR_TYPE_FRAG) {
if (frag->free_data) {
free(frag->frag_vector[1].iov_base);
}
OMPI_FREE_LIST_RETURN(&mca_ptl_portals_component.portals_send_frags,
(ompi_list_item_t*) frag);
}
}
#endif /* MCA_PTL_PORTALS_SENDFRAG_H_ */ #endif /* MCA_PTL_PORTALS_SENDFRAG_H_ */