From 5e47dafca6707b7b54d395ab4b9b6ed28bbcb45a Mon Sep 17 00:00:00 2001 From: Jeff Squyres Date: Sun, 30 Jan 2005 18:42:37 +0000 Subject: [PATCH] Slightly change the coll comm query API -- pass in a **data to allow the component to cache data that is necessary to do the query (e.g., if you need to try to allocate resources in order to determine selectability). This data is given back during unquery() and is cached on comm->c_coll_selected_data if the module is selected (before calling comm_init). This commit was SVN r4251. --- src/mca/coll/base/coll_base_comm_select.c | 66 +++++++++++++++-------- src/mca/coll/basic/coll_basic.h | 4 +- src/mca/coll/basic/coll_basic_module.c | 26 ++------- src/mca/coll/coll.h | 13 ++++- src/mca/coll/self/coll_self.h | 4 +- src/mca/coll/self/coll_self_component.c | 4 +- src/mca/coll/self/coll_self_module.c | 3 +- 7 files changed, 67 insertions(+), 53 deletions(-) diff --git a/src/mca/coll/base/coll_base_comm_select.c b/src/mca/coll/base/coll_base_comm_select.c index d89bcbefcc..a8a28a9e03 100644 --- a/src/mca/coll/base/coll_base_comm_select.c +++ b/src/mca/coll/base/coll_base_comm_select.c @@ -51,14 +51,16 @@ static mca_coll_base_module_1_0_0_t null_module = { * Local types */ struct avail_coll_t { - ompi_list_item_t super; + ompi_list_item_t super; - int ac_priority; - const mca_coll_base_component_1_0_0_t *ac_component; - const mca_coll_base_module_1_0_0_t *ac_module; + int ac_priority; + const mca_coll_base_component_1_0_0_t *ac_component; + const mca_coll_base_module_1_0_0_t *ac_module; + struct mca_coll_base_comm_t *ac_data; }; typedef struct avail_coll_t avail_coll_t; + /* * Local functions */ @@ -67,19 +69,24 @@ static ompi_list_t *check_components(ompi_list_t *components, char **names, int num_names); static int check_one_component(ompi_communicator_t *comm, const mca_base_component_t *component, - const mca_coll_base_module_1_0_0_t **module); + const mca_coll_base_module_1_0_0_t **module, + struct mca_coll_base_comm_t **data); static int query(const mca_base_component_t *component, ompi_communicator_t *comm, int *priority, - const mca_coll_base_module_1_0_0_t **module); + const mca_coll_base_module_1_0_0_t **module, + struct mca_coll_base_comm_t **data); static int query_1_0_0(const mca_coll_base_component_1_0_0_t *coll_component, ompi_communicator_t *comm, int *priority, - const mca_coll_base_module_1_0_0_t **module); + const mca_coll_base_module_1_0_0_t **module, + struct mca_coll_base_comm_t **data); static void unquery(const mca_coll_base_component_1_0_0_t *coll_component, - ompi_communicator_t *comm); + ompi_communicator_t *comm, + struct mca_coll_base_comm_t *data); static void unquery_1_0_0(const mca_coll_base_component_1_0_0_t *coll_component, - ompi_communicator_t *comm); + ompi_communicator_t *comm, + struct mca_coll_base_comm_t *data); static int module_init(const mca_coll_base_module_1_0_0_t *module, ompi_communicator_t *comm); @@ -114,6 +121,7 @@ int mca_coll_base_comm_select(ompi_communicator_t *comm, ompi_list_item_t *item; const mca_coll_base_component_1_0_0_t *selected_component, *component; const mca_coll_base_module_1_0_0_t *selected_module; + struct mca_coll_base_comm_t *selected_data; /* Announce */ @@ -232,11 +240,13 @@ int mca_coll_base_comm_select(ompi_communicator_t *comm, avail = (avail_coll_t *) item; selected_component = avail->ac_component; selected_module = avail->ac_module; + selected_data = avail->ac_data; OBJ_RELEASE(avail); } else { using_basic = true; selected_component = mca_coll_base_basic_component; selected_module = comm->c_coll_basic_module; + selected_data = comm->c_coll_basic_data; } #else /* JMS CONTINUE HERE */ @@ -251,7 +261,7 @@ int mca_coll_base_comm_select(ompi_communicator_t *comm, item = ompi_list_remove_first(selectable)) { avail = (avail_coll_t *) item; component = avail->ac_component; - unquery(component, comm); + unquery(component, comm, avail->ac_data); OBJ_RELEASE(avail); } OBJ_RELEASE(selectable); @@ -262,6 +272,7 @@ int mca_coll_base_comm_select(ompi_communicator_t *comm, it. */ comm->c_coll_selected_component = selected_component; + comm->c_coll_selected_data = selected_data; if (!using_basic) { comm->c_coll = *selected_module; replace_null_with_basic(comm); @@ -309,6 +320,7 @@ static ompi_list_t *check_components(ompi_list_t *components, bool want_to_check; ompi_list_t *selectable; avail_coll_t *avail, *avail2; + struct mca_coll_base_comm_t *data; /* Make a list of the components that query successfully */ @@ -341,7 +353,7 @@ static ompi_list_t *check_components(ompi_list_t *components, so */ if (want_to_check) { - priority = check_one_component(comm, component, &module); + priority = check_one_component(comm, component, &module, &data); if (priority > 0) { /* We have a component that indicated that it wants to run by @@ -351,6 +363,7 @@ static ompi_list_t *check_components(ompi_list_t *components, avail->ac_priority = priority; avail->ac_component = (mca_coll_base_component_1_0_0_t *) component; avail->ac_module = module; + avail->ac_data = data; /* Put this item on the list in priority order (highest priority first). Should it go first? */ @@ -398,12 +411,13 @@ static ompi_list_t *check_components(ompi_list_t *components, */ static int check_one_component(ompi_communicator_t *comm, const mca_base_component_t *component, - const mca_coll_base_module_1_0_0_t **module) + const mca_coll_base_module_1_0_0_t **module, + struct mca_coll_base_comm_t **data) { int err; int priority = -1; - err = query(component, comm, &priority, module); + err = query(component, comm, &priority, module, data); if (OMPI_SUCCESS == err) { priority = (priority < 100) ? priority : 100; @@ -432,7 +446,8 @@ static int check_one_component(ompi_communicator_t *comm, */ static int query(const mca_base_component_t *component, ompi_communicator_t *comm, - int *priority, const mca_coll_base_module_1_0_0_t **module) + int *priority, const mca_coll_base_module_1_0_0_t **module, + struct mca_coll_base_comm_t **data) { /* coll v1.0.0 */ @@ -443,7 +458,7 @@ static int query(const mca_base_component_t *component, const mca_coll_base_component_1_0_0_t *coll100 = (mca_coll_base_component_1_0_0_t *) component; - return query_1_0_0(coll100, comm, priority, module); + return query_1_0_0(coll100, comm, priority, module, data); } /* Unknown coll API version -- return error */ @@ -454,13 +469,14 @@ static int query(const mca_base_component_t *component, static int query_1_0_0(const mca_coll_base_component_1_0_0_t *component, ompi_communicator_t *comm, int *priority, - const mca_coll_base_module_1_0_0_t **module) + const mca_coll_base_module_1_0_0_t **module, + struct mca_coll_base_comm_t **data) { const mca_coll_base_module_1_0_0_t *ret; /* There's currently no need for conversion */ - ret = component->collm_comm_query(comm, priority); + ret = component->collm_comm_query(comm, priority, data); if (NULL != ret) { *module = ret; return OMPI_SUCCESS; @@ -475,7 +491,8 @@ static int query_1_0_0(const mca_coll_base_component_1_0_0_t *component, **************************************************************************/ static void unquery(const mca_coll_base_component_1_0_0_t *component, - ompi_communicator_t *comm) + ompi_communicator_t *comm, + struct mca_coll_base_comm_t *data) { if (1 == component->collm_version.mca_major_version && 0 == component->collm_version.mca_minor_version && @@ -483,7 +500,7 @@ static void unquery(const mca_coll_base_component_1_0_0_t *component, const mca_coll_base_component_1_0_0_t *coll100 = (mca_coll_base_component_1_0_0_t *) component; - unquery_1_0_0(coll100, comm); + unquery_1_0_0(coll100, comm, data); } /* There's no way to have a version that we don't recognize here -- @@ -492,9 +509,12 @@ static void unquery(const mca_coll_base_component_1_0_0_t *component, static void unquery_1_0_0(const mca_coll_base_component_1_0_0_t *component, - ompi_communicator_t *comm) + ompi_communicator_t *comm, + struct mca_coll_base_comm_t *data) { - component->collm_comm_unquery(comm); + if (NULL != component->collm_comm_unquery) { + component->collm_comm_unquery(comm, data); + } } @@ -536,16 +556,18 @@ static int query_basic(ompi_communicator_t *comm) { int ret; int priority; + struct mca_coll_base_comm_t *data; ret = OMPI_SUCCESS; if (NULL == comm->c_coll_basic_module) { ret = query((mca_base_component_t *) mca_coll_base_basic_component, comm, - &priority, &comm->c_coll_basic_module); + &priority, &comm->c_coll_basic_module, &data); if (ret != OMPI_SUCCESS) { comm->c_coll_basic_module = NULL; return ret; } + comm->c_coll_basic_data = data; ret = module_init(comm->c_coll_basic_module, comm); } return ret; diff --git a/src/mca/coll/basic/coll_basic.h b/src/mca/coll/basic/coll_basic.h index a4d62b2b02..f4a7f91c15 100644 --- a/src/mca/coll/basic/coll_basic.h +++ b/src/mca/coll/basic/coll_basic.h @@ -45,8 +45,8 @@ OMPI_COMP_EXPORT extern int mca_coll_basic_priority_param; int mca_coll_basic_init_query(bool *allow_multi_user_threads, bool *have_hidden_threads); const struct mca_coll_base_module_1_0_0_t * - mca_coll_basic_comm_query(struct ompi_communicator_t *comm, int *priority); - int mca_coll_basic_comm_unquery(struct ompi_communicator_t *comm); + mca_coll_basic_comm_query(struct ompi_communicator_t *comm, int *priority, + struct mca_coll_base_comm_t **data); const struct mca_coll_base_module_1_0_0_t * mca_coll_basic_module_init(struct ompi_communicator_t *comm); diff --git a/src/mca/coll/basic/coll_basic_module.c b/src/mca/coll/basic/coll_basic_module.c index 5f19cec67c..bfdc3a9b5d 100644 --- a/src/mca/coll/basic/coll_basic_module.c +++ b/src/mca/coll/basic/coll_basic_module.c @@ -142,7 +142,8 @@ int mca_coll_basic_init_query(bool *allow_multi_user_threads, * priority we want to return. */ const mca_coll_base_module_1_0_0_t * -mca_coll_basic_comm_query(struct ompi_communicator_t *comm, int *priority) +mca_coll_basic_comm_query(struct ompi_communicator_t *comm, int *priority, + struct mca_coll_base_comm_t **data) { if (OMPI_SUCCESS != mca_base_param_lookup_int(mca_coll_basic_priority_param, priority)) { @@ -153,12 +154,8 @@ mca_coll_basic_comm_query(struct ompi_communicator_t *comm, int *priority) algorithms. */ if (OMPI_COMM_IS_INTER(comm)) { - /* Intercommunicators */ return &inter_linear; } else { - - /* Intracommunicators */ - if (ompi_comm_size(comm) <= mca_coll_base_crossover) { return &intra_linear; } else { @@ -181,19 +178,13 @@ mca_coll_basic_module_init(struct ompi_communicator_t *comm) /* Allocate the data that hangs off the communicator */ - comm->c_coll_basic_data = NULL; - if (OMPI_COMM_IS_INTER(comm)) { - /* Intercommunicators */ - /* JMS Continue here */ size = ompi_comm_remote_size(comm); } else { - /* Intracommunicators */ - /* JMS Continue here */ size = ompi_comm_size(comm); } data = malloc(sizeof(struct mca_coll_base_comm_t) + - (sizeof(ompi_request_t *) * size * 2)); + (sizeof(ompi_request_t *) * size * 2)); if (NULL == data) { return NULL; @@ -201,9 +192,6 @@ mca_coll_basic_module_init(struct ompi_communicator_t *comm) data->mccb_reqs = (ompi_request_t **) (data + 1); data->mccb_num_reqs = size * 2; - /* Initialize the communicator */ - - /* All done */ comm->c_coll_basic_data = data; @@ -228,14 +216,6 @@ int mca_coll_basic_module_finalize(struct ompi_communicator_t *comm) comm->c_coll_basic_data->mccb_num_reqs = 0; #endif - if (OMPI_COMM_IS_INTER(comm)) { - /* Intercommunicators */ - /* JMS Continue here */ - } else { - /* Intracommunicators */ - /* JMS Continue here */ - } - /* All done */ free(comm->c_coll_basic_data); diff --git a/src/mca/coll/coll.h b/src/mca/coll/coll.h index 0e92737139..37f419f525 100644 --- a/src/mca/coll/coll.h +++ b/src/mca/coll/coll.h @@ -22,6 +22,13 @@ #include "mca/base/base.h" +/* + * Forward declaration + */ + +struct mca_coll_base_comm_t; + + /* * Coll component function typedefs */ @@ -30,9 +37,11 @@ typedef int (*mca_coll_base_component_init_query_fn_t) (bool *allow_multi_user_threads, bool *have_hidden_threads); typedef const struct mca_coll_base_module_1_0_0_t * (*mca_coll_base_component_comm_query_1_0_0_fn_t) - (struct ompi_communicator_t *comm, int *priority); + (struct ompi_communicator_t *comm, int *priority, + struct mca_coll_base_comm_t **data); typedef int (*mca_coll_base_component_comm_unquery_fn_t) - (struct ompi_communicator_t *comm); + (struct ompi_communicator_t *comm, + struct mca_coll_base_comm_t *data); /* * Coll module function typedefs diff --git a/src/mca/coll/self/coll_self.h b/src/mca/coll/self/coll_self.h index e0b02b02e6..7bbc5a5a04 100644 --- a/src/mca/coll/self/coll_self.h +++ b/src/mca/coll/self/coll_self.h @@ -45,8 +45,8 @@ OMPI_COMP_EXPORT extern int mca_coll_self_priority_param; int mca_coll_self_init_query(bool *allow_multi_user_threads, bool *have_hidden_threads); const struct mca_coll_base_module_1_0_0_t * - mca_coll_self_comm_query(struct ompi_communicator_t *comm, int *priority); - int mca_coll_self_comm_unquery(struct ompi_communicator_t *comm); + mca_coll_self_comm_query(struct ompi_communicator_t *comm, int *priority, + struct mca_coll_base_comm_t **data); const struct mca_coll_base_module_1_0_0_t * mca_coll_self_module_init(struct ompi_communicator_t *comm); diff --git a/src/mca/coll/self/coll_self_component.c b/src/mca/coll/self/coll_self_component.c index 6040d54514..710e94328d 100644 --- a/src/mca/coll/self/coll_self_component.c +++ b/src/mca/coll/self/coll_self_component.c @@ -42,6 +42,7 @@ int mca_coll_self_priority_param = -1; */ static int self_open(void); + /* * Instantiate the public struct with all of our public information * and pointers to our public functions in it @@ -83,7 +84,7 @@ const mca_coll_base_component_1_0_0_t mca_coll_self_component = { mca_coll_self_init_query, mca_coll_self_comm_query, - NULL + NULL, }; @@ -97,3 +98,4 @@ static int self_open(void) return OMPI_SUCCESS; } + diff --git a/src/mca/coll/self/coll_self_module.c b/src/mca/coll/self/coll_self_module.c index 034b5c0665..30b8882ee7 100644 --- a/src/mca/coll/self/coll_self_module.c +++ b/src/mca/coll/self/coll_self_module.c @@ -78,7 +78,8 @@ int mca_coll_self_init_query(bool *allow_multi_user_threads, * priority we want to return. */ const mca_coll_base_module_1_0_0_t * -mca_coll_self_comm_query(struct ompi_communicator_t *comm, int *priority) +mca_coll_self_comm_query(struct ompi_communicator_t *comm, int *priority, + struct mca_coll_base_comm_t **data) { /* We only work on intracommunicators of size 1 */