diff --git a/src/mca/mpi/pml/pml.h b/src/mca/mpi/pml/pml.h index 114cc69cee..5c60c4701f 100644 --- a/src/mca/mpi/pml/pml.h +++ b/src/mca/mpi/pml/pml.h @@ -6,7 +6,6 @@ #define MCA_PML_H #include "lam_config.h" - #include "lam/lam.h" #include "lam/lfc/list.h" #include "mpi/communicator/communicator.h" @@ -17,20 +16,11 @@ /* - * PML module functions - */ - -typedef struct mca_pml_t * (*mca_pml_base_init_fn_t)(int* priority, int* min_thread, int* max_thread); -typedef int (*mca_pml_base_fini_fn_t)(void); - -/* - * PML types + * PML module types */ struct mca_ptl_t; -typedef uint64_t mca_pml_base_tstamp_t; - typedef enum { MCA_PML_BASE_SEND_STANDARD, MCA_PML_BASE_SEND_BUFFERED, @@ -45,9 +35,33 @@ typedef struct mca_pml_base_status_t mca_pml_base_status_t; #define LAM_ANY_TAG MPI_ANY_TAG +/** + * MCA->PML Called by MCA framework to initialize the module. + * + * @param priority (OUT) Relative priority or ranking used by MCA to selected a module. + * @param thread_min (OUT) Minimum thread level supported by the module. + * @param thread_max (OUT) Maximum thread level supported by the module. + */ + +typedef struct mca_pml_t * (*mca_pml_base_module_init_fn_t)( + int* priority, + int* min_thread, + int* max_thread); + +/** + * PML module version and interface functions. + */ + +struct mca_pml_base_module_1_0_0_t { + mca_base_module_t pmlm_version; + mca_base_module_data_1_0_0_t pmlm_data; + mca_pml_base_module_init_fn_t pmlm_init; +}; +typedef struct mca_pml_base_module_1_0_0_t mca_pml_base_module_1_0_0_t; + /* - * PML interface functions + * PML instance interface functions and datatype */ typedef int (*mca_pml_base_add_comm_fn_t)(struct lam_communicator_t*); @@ -55,7 +69,8 @@ typedef int (*mca_pml_base_del_comm_fn_t)(struct lam_communicator_t*); typedef int (*mca_pml_base_add_procs_fn_t)(struct lam_proc_t **procs, size_t nprocs); typedef int (*mca_pml_base_del_procs_fn_t)(struct lam_proc_t **procs, size_t nprocs); typedef int (*mca_pml_base_add_ptls_fn_t)(struct mca_ptl_t **ptls, size_t nptls); -typedef int (*mca_pml_base_progress_fn_t)(mca_pml_base_tstamp_t); +typedef int (*mca_pml_base_fini_fn_t)(void); +typedef int (*mca_pml_base_progress_fn_t)(void); typedef int (*mca_pml_base_irecv_init_fn_t)( void *buf, @@ -116,21 +131,9 @@ typedef int (*mca_pml_base_wait_fn_t)( mca_pml_base_status_t* status ); -/* - * PML module definition. - */ -struct mca_pml_base_module_1_0_0_t { - mca_base_module_t pmlm_version; - mca_base_module_data_1_0_0_t pmlm_data; - mca_pml_base_init_fn_t pmlm_init; -}; -typedef struct mca_pml_base_module_1_0_0_t mca_pml_base_module_1_0_0_t; - - -/* - * Struct that represents the common state and interface functions - * provided by a PML. +/** + * PML instance interface functions. */ struct mca_pml_t { diff --git a/src/mca/mpi/pml/teg/src/Makefile.am b/src/mca/mpi/pml/teg/src/Makefile.am index a1ae0bf3c8..4616eed29f 100644 --- a/src/mca/mpi/pml/teg/src/Makefile.am +++ b/src/mca/mpi/pml/teg/src/Makefile.am @@ -18,6 +18,7 @@ libmca_pml_teg_la_SOURCES = \ pml_teg_isend.c \ pml_teg_proc.c \ pml_teg_proc.h \ + pml_teg_progress.c \ pml_teg_sendreq.c \ pml_teg_sendreq.h \ pml_ptl_array.c \ diff --git a/src/mca/mpi/pml/teg/src/pml_teg.c b/src/mca/mpi/pml/teg/src/pml_teg.c index 531819cdb7..6e98b54a84 100644 --- a/src/mca/mpi/pml/teg/src/pml_teg.c +++ b/src/mca/mpi/pml/teg/src/pml_teg.c @@ -69,7 +69,7 @@ mca_pml_teg_t mca_pml_teg = { * long description */ -int mca_pml_teg_open(lam_cmd_line_t* cmd_line) +int mca_pml_teg_open(void) { return LAM_SUCCESS; } diff --git a/src/mca/mpi/pml/teg/src/pml_teg.h b/src/mca/mpi/pml/teg/src/pml_teg.h index b8519c36ec..be0cdf21ae 100644 --- a/src/mca/mpi/pml/teg/src/pml_teg.h +++ b/src/mca/mpi/pml/teg/src/pml_teg.h @@ -22,9 +22,8 @@ */ struct mca_pml_teg_t { - mca_pml_t super; /**< comment */ + mca_pml_t super; - /* available ptls */ mca_ptl_base_module_t **teg_ptl_modules; size_t teg_num_ptl_modules; @@ -47,10 +46,7 @@ extern mca_pml_teg_t mca_pml_teg; extern mca_pml_base_module_1_0_0_t mca_pml_teg_module_1_0_0_0; -extern int mca_pml_teg_open( - lam_cmd_line_t* cmd_line -); - +extern int mca_pml_teg_open(void); extern int mca_pml_teg_close(void); extern mca_pml_t* mca_pml_teg_init( @@ -133,9 +129,7 @@ extern int mca_pml_teg_irecv( struct lam_request_t **request ); -extern int mca_pml_teg_progress( - mca_pml_base_tstamp_t tstamp -); +extern int mca_pml_teg_progress(void); extern int mca_pml_teg_start( lam_request_t* request diff --git a/src/mca/mpi/pml/teg/src/pml_teg_progress.c b/src/mca/mpi/pml/teg/src/pml_teg_progress.c new file mode 100644 index 0000000000..7625ed1e75 --- /dev/null +++ b/src/mca/mpi/pml/teg/src/pml_teg_progress.c @@ -0,0 +1,27 @@ +#include "pml_teg.h" +#include "pml_teg_sendreq.h" + + +int mca_pml_teg_progress(void) +{ + THREAD_LOCK(&mca_pml_teg.teg_lock); + mca_ptl_base_send_request_t* req; + for(req = (mca_ptl_base_send_request_t*)lam_list_get_first(&mca_pml_teg.teg_incomplete_sends); + req != (mca_ptl_base_send_request_t*)lam_list_get_end(&mca_pml_teg.teg_incomplete_sends); + req = (mca_ptl_base_send_request_t*)lam_list_get_next(req)) { + + bool complete; + int rc = mca_pml_teg_send_request_schedule(req, &complete); + if(rc != LAM_SUCCESS) { + THREAD_UNLOCK(&mca_pml_teg.teg_lock); + return rc; + } + if(complete) { + req = (mca_ptl_base_send_request_t*)lam_list_remove( + &mca_pml_teg.teg_incomplete_sends, (lam_list_item_t*)req); + } + } + THREAD_UNLOCK(&mca_pml_teg.teg_lock); + return LAM_SUCCESS; +} + diff --git a/src/mca/mpi/pml/teg/src/pml_teg_sendreq.c b/src/mca/mpi/pml/teg/src/pml_teg_sendreq.c index 93c1f4ef51..467f6dca8c 100644 --- a/src/mca/mpi/pml/teg/src/pml_teg_sendreq.c +++ b/src/mca/mpi/pml/teg/src/pml_teg_sendreq.c @@ -76,33 +76,3 @@ int mca_pml_teg_send_request_schedule(mca_ptl_base_send_request_t* req, bool* co return LAM_SUCCESS; } - -/* - * Check for queued messages that need to be scheduled. - * - */ - -int mca_pml_teg_send_request_progress(void) -{ - THREAD_LOCK(&mca_pml_teg.teg_lock); - mca_ptl_base_send_request_t* req; - for(req = (mca_ptl_base_send_request_t*)lam_list_get_first(&mca_pml_teg.teg_incomplete_sends); - req != (mca_ptl_base_send_request_t*)lam_list_get_end(&mca_pml_teg.teg_incomplete_sends); - req = (mca_ptl_base_send_request_t*)lam_list_get_next(req)) { - - bool complete; - int rc = mca_pml_teg_send_request_schedule(req, &complete); - if(rc != LAM_SUCCESS) { - THREAD_UNLOCK(&mca_pml_teg.teg_lock); - return rc; - } - if(complete) { - req = (mca_ptl_base_send_request_t*)lam_list_remove( - &mca_pml_teg.teg_incomplete_sends, (lam_list_item_t*)req); - } - } - THREAD_UNLOCK(&mca_pml_teg.teg_lock); - return LAM_SUCCESS; -} - - diff --git a/src/mca/mpi/pml/teg/src/pml_teg_sendreq.h b/src/mca/mpi/pml/teg/src/pml_teg_sendreq.h index 302df5ce59..fed4ec8d25 100644 --- a/src/mca/mpi/pml/teg/src/pml_teg_sendreq.h +++ b/src/mca/mpi/pml/teg/src/pml_teg_sendreq.h @@ -5,12 +5,12 @@ #ifndef LAM_PML_TEG_SEND_REQUEST_H #define LAM_PML_TEG_SEND_REQUEST_H +#include "pml_teg_proc.h" #include "mca/mpi/ptl/base/ptl_base_sendreq.h" #include "mca/mpi/ptl/base/ptl_base_sendfrag.h" int mca_pml_teg_send_request_schedule(mca_ptl_base_send_request_t* req, bool* complete); -int mca_pml_teg_send_request_progress(void); static inline int mca_pml_teg_send_request_alloc( @@ -27,7 +27,7 @@ static inline int mca_pml_teg_send_request_alloc( int rc = ptl->ptl_request_alloc(ptl,sendreq); if(rc != LAM_SUCCESS) return rc; - (*sendreq)->req_first_ptl = ptl; + (*sendreq)->req_owner = ptl; return LAM_SUCCESS; } @@ -35,17 +35,23 @@ static inline int mca_pml_teg_send_request_alloc( static inline int mca_pml_teg_send_request_start( mca_ptl_base_send_request_t* req) { - mca_ptl_t* ptl = req->req_first_ptl; + mca_ptl_t* ptl = req->req_owner; size_t first_fragment_size = ptl->ptl_frag_first_size; - - // queue for pending acknowledgment - THREAD_SCOPED_LOCK(&mca_pml_teg.teg_lock, - lam_list_append(&mca_pml_teg.teg_pending_acks, (lam_list_item_t*)req)); + int rc; + bool complete; // start the first fragment if(req->req_length < first_fragment_size) first_fragment_size = req->req_length; - return req->req_first_ptl->ptl_send(ptl, req, first_fragment_size); + rc = ptl->ptl_send(ptl, req, first_fragment_size, &complete); + if(rc != LAM_SUCCESS) + return rc; + + // if incomplete queue to retry later + if(complete == false) { + THREAD_SCOPED_LOCK(&mca_pml_teg.teg_lock, + lam_list_append(&mca_pml_teg.teg_incomplete_sends, (lam_list_item_t*)req)); + } } diff --git a/src/mca/mpi/ptl/base/ptl_base_fragment.h b/src/mca/mpi/ptl/base/ptl_base_fragment.h index a279a04df1..b68ad2570a 100644 --- a/src/mca/mpi/ptl/base/ptl_base_fragment.h +++ b/src/mca/mpi/ptl/base/ptl_base_fragment.h @@ -16,7 +16,7 @@ extern lam_class_info_t mca_ptl_base_frag_cls; struct mca_ptl_base_frag_t { lam_list_item_t super; mca_ptl_base_reliable_hdr_t frag_header; - struct mca_ptl_t* frag_ptl; + struct mca_ptl_t* frag_owner; /**< PTL that allocated this fragment */ }; typedef struct mca_ptl_base_frag_t mca_ptl_base_frag_t; diff --git a/src/mca/mpi/ptl/base/ptl_base_sendreq.h b/src/mca/mpi/ptl/base/ptl_base_sendreq.h index 909f1cc0d4..2f28f557e8 100644 --- a/src/mca/mpi/ptl/base/ptl_base_sendreq.h +++ b/src/mca/mpi/ptl/base/ptl_base_sendreq.h @@ -14,6 +14,9 @@ struct mca_ptl_base_send_request_t { /* request object - common data structure for use by wait/test */ mca_pml_base_request_t super; + /* allow send request to be placed on ack list */ + lam_list_item_t req_ack_item; + /* pointer to user data */ void *req_data; /* size of send/recv in bytes */ @@ -28,15 +31,12 @@ struct mca_ptl_base_send_request_t { bool req_clear_to_send; /* type of send */ mca_pml_base_send_mode_t req_send_mode; - /* time at which watchdog timer expires */ - mca_pml_base_tstamp_t req_time_out; /* sequence number for MPI pt-2-pt ordering */ mca_ptl_base_sequence_t req_msg_sequence_number; /* queue of fragments that are waiting to be acknowledged */ mca_ptl_base_queue_t req_unacked_frags; - /* first ptl/fragment descriptor */ - struct mca_ptl_t* req_first_ptl; - struct mca_ptl_base_send_frag_t* req_first_frag; + /* PTL that allocated this descriptor */ + struct mca_ptl_t* req_owner; }; typedef struct mca_ptl_base_send_request_t mca_ptl_base_send_request_t; diff --git a/src/mca/mpi/ptl/ptl.h b/src/mca/mpi/ptl/ptl.h index 992b338628..9638dd5bf8 100644 --- a/src/mca/mpi/ptl/ptl.h +++ b/src/mca/mpi/ptl/ptl.h @@ -28,20 +28,52 @@ typedef uint64_t mca_ptl_base_sequence_t; typedef uint64_t mca_ptl_base_tstamp_t; typedef lam_list_t mca_ptl_base_queue_t; +/* + * PTL module interface functions and datatype. + */ + /** - * Intializes the PTL module and creates specific PTL instance(s). + * MCA->PTL Intializes the PTL module and creates specific PTL instance(s). * * @param num_ptls (OUT) Returns the number of ptl instances created. * @param thread_min (OUT) Minimum thread level supported by the PTL. * @param thread_max (OUT) Maximum thread level supported by the PTL. * @return Array of pointers to PTL instances. */ -typedef struct mca_ptl_t** (*mca_ptl_base_init_fn_t)( +typedef struct mca_ptl_t** (*mca_ptl_base_module_init_fn_t)( int* num_ptls, int *thread_min, int *thread_max ); + +/** + * PML->PTL Progresses outstanding requests in each PTL module. + * + * @param timstamp (IN) The current time - used for retransmisstion timers. + */ +typedef void (*mca_ptl_base_module_progress_fn_t)( + mca_ptl_base_tstamp_t timestamp +); + +/** + * PTL module version and interface functions. + */ + +struct mca_ptl_base_module_1_0_0_t { + mca_base_module_t ptlm_version; + mca_base_module_data_1_0_0_t ptlm_data; + mca_ptl_base_module_init_fn_t ptlm_init; + mca_ptl_base_module_progress_fn_t ptlm_progress; +}; +typedef struct mca_ptl_base_module_1_0_0_t mca_ptl_base_module_1_0_0_t; +typedef struct mca_ptl_base_module_1_0_0_t mca_ptl_base_module_t; + + +/* + * PTL instance interface functions and datatype. + */ + /** * PML->PTL notification of change in the process list. * @@ -57,44 +89,36 @@ typedef int (*mca_ptl_base_add_procs_fn_t)( ); /** - * Called by the MCA to cleanup any resources held by the PTL. + * MCA->PTL Clean up any resources held by PTL instance before the module is unloaded. + * + * @param ptl (IN) The PTL module instance that is being unloaded. */ typedef int (*mca_ptl_base_fini_fn_t)( - struct mca_ptl_t* + struct mca_ptl_t* ptl ); +/** + * PML->PTL notification of change in the process list. + * + * @param ptl (IN) + * @param procs (IN) + * @param nprocs (IN) + * @return + */ typedef int (*mca_ptl_base_request_alloc_fn_t)( struct mca_ptl_t* ptl, - struct mca_ptl_base_send_request_t** + struct mca_ptl_base_send_request_t** send_request ); typedef int (*mca_ptl_base_send_fn_t)( struct mca_ptl_t* ptl, - struct mca_ptl_base_send_request_t*, - size_t size + struct mca_ptl_base_send_request_t* send_request, + size_t size, + bool* complete ); -typedef int (*mca_ptl_base_progress_fn_t)( - struct mca_ptl_t*, - mca_pml_base_tstamp_t -); - -/* - * Struct used to pass PTL module information from the each PTL - * instance back to the MCA framework. - */ - -struct mca_ptl_base_module_1_0_0_t { - mca_base_module_t ptlm_version; - mca_base_module_data_1_0_0_t ptlm_data; - mca_ptl_base_init_fn_t ptlm_init; -}; -typedef struct mca_ptl_base_module_1_0_0_t mca_ptl_base_module_1_0_0_t; -typedef struct mca_ptl_base_module_1_0_0_t mca_ptl_base_module_t; - /** - * Struct that represents the common state and interface functions - * provided by a PTL. + * PTL instance interface functions and common state. */ struct mca_ptl_t { @@ -112,7 +136,6 @@ struct mca_ptl_t { mca_ptl_base_add_procs_fn_t ptl_add_procs; mca_ptl_base_fini_fn_t ptl_fini; mca_ptl_base_send_fn_t ptl_send; - mca_ptl_base_progress_fn_t ptl_progress; mca_ptl_base_request_alloc_fn_t ptl_request_alloc; }; @@ -126,7 +149,7 @@ typedef struct mca_ptl_t mca_ptl_t; extern "C" { #endif int mca_ptl_base_close(void); - int mca_ptl_base_open(lam_cmd_line_t *cmd); + int mca_ptl_base_open(void); int mca_ptl_base_query(void); int mca_ptl_base_init(void); #if defined(c_plusplus) || defined(__cplusplus)