2004-11-20 22:12:43 +03:00
|
|
|
/*
|
2005-11-05 22:57:48 +03:00
|
|
|
* Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
|
|
|
|
* University Research and Technology
|
|
|
|
* Corporation. All rights reserved.
|
|
|
|
* Copyright (c) 2004-2005 The University of Tennessee and The University
|
|
|
|
* of Tennessee Research Foundation. All rights
|
|
|
|
* reserved.
|
2005-09-01 05:07:30 +04:00
|
|
|
* Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
|
2004-11-28 23:09:25 +03:00
|
|
|
* University of Stuttgart. All rights reserved.
|
2005-03-24 15:43:37 +03:00
|
|
|
* Copyright (c) 2004-2005 The Regents of the University of California.
|
|
|
|
* All rights reserved.
|
2004-11-22 04:38:40 +03:00
|
|
|
* $COPYRIGHT$
|
2005-09-01 05:07:30 +04:00
|
|
|
*
|
2004-11-22 04:38:40 +03:00
|
|
|
* Additional copyrights may follow
|
2005-09-01 05:07:30 +04:00
|
|
|
*
|
2004-11-20 22:12:43 +03:00
|
|
|
* $HEADER$
|
|
|
|
*/
|
|
|
|
/** @file:
|
|
|
|
*
|
|
|
|
* The Open MPI general purpose registry - implementation.
|
|
|
|
*
|
|
|
|
*/
|
|
|
|
|
|
|
|
/*
|
|
|
|
* includes
|
|
|
|
*/
|
|
|
|
|
2005-03-14 23:57:21 +03:00
|
|
|
#include "orte_config.h"
|
|
|
|
|
2006-02-12 04:33:29 +03:00
|
|
|
#include "orte/orte_constants.h"
|
2005-09-09 22:27:17 +04:00
|
|
|
#include "opal/util/trace.h"
|
2005-03-14 23:57:21 +03:00
|
|
|
|
2005-08-01 20:38:15 +04:00
|
|
|
#include "orte/mca/errmgr/errmgr.h"
|
|
|
|
#include "orte/class/orte_pointer_array.h"
|
|
|
|
#include "orte/mca/gpr/proxy/gpr_proxy.h"
|
2004-11-20 22:12:43 +03:00
|
|
|
|
2005-03-14 23:57:21 +03:00
|
|
|
int
|
2006-08-15 23:54:10 +04:00
|
|
|
orte_gpr_proxy_enter_subscription(orte_std_cntr_t cnt, orte_gpr_subscription_t **subscriptions)
|
2004-11-20 22:12:43 +03:00
|
|
|
{
|
2005-03-14 23:57:21 +03:00
|
|
|
orte_gpr_proxy_subscriber_t *sub;
|
2006-08-15 23:54:10 +04:00
|
|
|
orte_std_cntr_t i;
|
2005-09-01 05:07:30 +04:00
|
|
|
|
2005-09-09 22:27:17 +04:00
|
|
|
OPAL_TRACE(2);
|
|
|
|
|
2005-03-14 23:57:21 +03:00
|
|
|
for (i=0; i < cnt; i++) {
|
|
|
|
sub = OBJ_NEW(orte_gpr_proxy_subscriber_t);
|
|
|
|
if (NULL == sub) {
|
|
|
|
ORTE_ERROR_LOG(ORTE_ERR_OUT_OF_RESOURCE);
|
|
|
|
return ORTE_ERR_OUT_OF_RESOURCE;
|
|
|
|
}
|
2005-08-01 20:38:15 +04:00
|
|
|
if (NULL != subscriptions[i]->name) {
|
|
|
|
sub->name = strdup(subscriptions[i]->name);
|
|
|
|
}
|
2005-03-14 23:57:21 +03:00
|
|
|
sub->callback = subscriptions[i]->cbfunc;
|
|
|
|
sub->user_tag = subscriptions[i]->user_tag;
|
2005-09-01 05:07:30 +04:00
|
|
|
if (0 > orte_pointer_array_add(&sub->index, orte_gpr_proxy_globals.subscriptions, sub)) {
|
2005-03-14 23:57:21 +03:00
|
|
|
ORTE_ERROR_LOG(ORTE_ERR_OUT_OF_RESOURCE);
|
|
|
|
return ORTE_ERR_OUT_OF_RESOURCE;
|
|
|
|
}
|
2005-07-18 22:49:00 +04:00
|
|
|
sub->id = orte_gpr_proxy_globals.num_subs;
|
2005-06-24 20:59:37 +04:00
|
|
|
subscriptions[i]->id = sub->id;
|
|
|
|
(orte_gpr_proxy_globals.num_subs)++;
|
Well, we are getting closer to resolving the comm_spawn problem. For the benefit of those that haven't been in the midst of this discussion, the problem is that this is the first case where the process starting a set of processes has not been mpirun and is not guaranteed to be alive throughout the lifetime of the spawned processes. This sounds simple, but actually has some profound impacts.
Most of this checkin consists of more debugging stuff. Hopefully, you won't see any printf's that aren't protected by debug flags - if you do, let me know and I'll take them out with my apologies.
Outside of debugging, the biggest change was a revamp of the shutdown process. For several reasons, we had chosen to have all processes "wait" for a shutdown message before exiting. This message is typically generated by mpirun, but in the case of comm_spawn we needed to do something else. We have decided that the best way to solve this problem is to:
(a) replace the shutdown message (which needed to be generated by somebody - usually mpirun) with an oob_barrier call. This still requires that the rank 0 process be alive. However, we terminate all processes if one abnormally terminates anyway, so this isn't a problem (with the standard or our implementation); and
(b) have the state-of-health monitoring subsystem issue the call to cleanup the job from the registry. Since the state-of-health subsystem isn't available yet, we have temporarily assigned that responsibility to the rank 0 process. Once the state-of-health subsystem is available, we will have it monitor the job for all-processes-complete and then it can tell the registry to cleanup the job (i.e., remove all data relating to this job).
Hope that helps a little. I'll put all this into the design docs soon.
This commit was SVN r3754.
2004-12-09 00:44:41 +03:00
|
|
|
}
|
2005-09-01 05:07:30 +04:00
|
|
|
|
2005-03-14 23:57:21 +03:00
|
|
|
return ORTE_SUCCESS;
|
2004-11-20 22:12:43 +03:00
|
|
|
}
|
|
|
|
|
|
|
|
|
2005-03-14 23:57:21 +03:00
|
|
|
int
|
2006-08-15 23:54:10 +04:00
|
|
|
orte_gpr_proxy_enter_trigger(orte_std_cntr_t cnt, orte_gpr_trigger_t **trigs)
|
2004-11-20 22:12:43 +03:00
|
|
|
{
|
2005-09-01 05:07:30 +04:00
|
|
|
orte_gpr_proxy_trigger_t *trig, **tptr;
|
2006-08-15 23:54:10 +04:00
|
|
|
orte_std_cntr_t i, j, k;
|
2005-09-01 05:07:30 +04:00
|
|
|
|
2005-09-09 22:27:17 +04:00
|
|
|
OPAL_TRACE(2);
|
|
|
|
|
2005-06-24 20:59:37 +04:00
|
|
|
for (i=0; i < cnt; i++) {
|
2005-09-01 05:07:30 +04:00
|
|
|
/* If the provided trigger has a name, see if it already is on
|
|
|
|
* the local trigger list. If so, then check to see if we
|
|
|
|
* already defined a return point for it and/or if this trigger
|
|
|
|
* doesn't - in either of those two cases, we ignore the
|
|
|
|
* trigger and just use the existing entry
|
|
|
|
*/
|
|
|
|
if (NULL != trigs[i]->name) {
|
|
|
|
tptr = (orte_gpr_proxy_trigger_t**)(orte_gpr_proxy_globals.triggers)->addr;
|
|
|
|
for (j=0, k=0; k < orte_gpr_proxy_globals.num_trigs &&
|
|
|
|
j < (orte_gpr_proxy_globals.triggers)->size; j++) {
|
|
|
|
if (NULL != tptr[j]) {
|
|
|
|
k++;
|
2006-02-07 06:32:36 +03:00
|
|
|
if (NULL != tptr[j]->name &&
|
|
|
|
0 == strcmp(tptr[j]->name, trigs[i]->name)) {
|
2005-09-01 05:07:30 +04:00
|
|
|
/* same name - trigger is already on list */
|
|
|
|
if (NULL != tptr[j]->callback || NULL == trigs[i]->cbfunc) {
|
|
|
|
/* ignore these cases */
|
|
|
|
trig = tptr[j];
|
|
|
|
goto MOVEON;
|
|
|
|
}
|
|
|
|
/* reach here if either the prior trigger didn't provide
|
|
|
|
* a callback, and the new one provides one. In this
|
|
|
|
* case, we update the existing trigger callback and then
|
|
|
|
* move on
|
|
|
|
*/
|
|
|
|
tptr[j]->callback = trigs[i]->cbfunc;
|
|
|
|
trig = tptr[j];
|
|
|
|
goto MOVEON;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
/* either the trigger doesn't have a name, OR it did, but it isn't
|
|
|
|
* already on the list - add it to the list now
|
|
|
|
*/
|
2005-07-18 22:49:00 +04:00
|
|
|
trig = OBJ_NEW(orte_gpr_proxy_trigger_t);
|
|
|
|
if (NULL == trig) {
|
|
|
|
ORTE_ERROR_LOG(ORTE_ERR_OUT_OF_RESOURCE);
|
|
|
|
return ORTE_ERR_OUT_OF_RESOURCE;
|
|
|
|
}
|
2005-08-01 20:38:15 +04:00
|
|
|
if (NULL != trigs[i]->name) {
|
|
|
|
trig->name = strdup(trigs[i]->name);
|
|
|
|
}
|
2005-07-18 22:49:00 +04:00
|
|
|
/* ensure that the proper routing flag is set
|
|
|
|
* in the action field to match the trigger callback
|
|
|
|
* function
|
|
|
|
*/
|
|
|
|
if (NULL != trigs[i]->cbfunc) {
|
|
|
|
trigs[i]->action = trigs[i]->action |
|
|
|
|
ORTE_GPR_TRIG_ROUTE_DATA_THRU_ME;
|
2005-06-24 20:59:37 +04:00
|
|
|
} else {
|
2005-07-18 22:49:00 +04:00
|
|
|
trigs[i]->action = trigs[i]->action &
|
|
|
|
~ORTE_GPR_TRIG_ROUTE_DATA_THRU_ME;
|
|
|
|
}
|
|
|
|
trig->callback = trigs[i]->cbfunc;
|
|
|
|
trig->user_tag = trigs[i]->user_tag;
|
2005-09-01 05:07:30 +04:00
|
|
|
if (0 > orte_pointer_array_add(&trig->index, orte_gpr_proxy_globals.triggers, trig)) {
|
2005-06-24 20:59:37 +04:00
|
|
|
ORTE_ERROR_LOG(ORTE_ERR_OUT_OF_RESOURCE);
|
|
|
|
return ORTE_ERR_OUT_OF_RESOURCE;
|
|
|
|
}
|
2005-07-18 22:49:00 +04:00
|
|
|
trig->id = orte_gpr_proxy_globals.num_trigs;
|
|
|
|
(orte_gpr_proxy_globals.num_trigs)++;
|
2005-09-01 05:07:30 +04:00
|
|
|
MOVEON:
|
|
|
|
trigs[i]->id = trig->id;
|
Well, we are getting closer to resolving the comm_spawn problem. For the benefit of those that haven't been in the midst of this discussion, the problem is that this is the first case where the process starting a set of processes has not been mpirun and is not guaranteed to be alive throughout the lifetime of the spawned processes. This sounds simple, but actually has some profound impacts.
Most of this checkin consists of more debugging stuff. Hopefully, you won't see any printf's that aren't protected by debug flags - if you do, let me know and I'll take them out with my apologies.
Outside of debugging, the biggest change was a revamp of the shutdown process. For several reasons, we had chosen to have all processes "wait" for a shutdown message before exiting. This message is typically generated by mpirun, but in the case of comm_spawn we needed to do something else. We have decided that the best way to solve this problem is to:
(a) replace the shutdown message (which needed to be generated by somebody - usually mpirun) with an oob_barrier call. This still requires that the rank 0 process be alive. However, we terminate all processes if one abnormally terminates anyway, so this isn't a problem (with the standard or our implementation); and
(b) have the state-of-health monitoring subsystem issue the call to cleanup the job from the registry. Since the state-of-health subsystem isn't available yet, we have temporarily assigned that responsibility to the rank 0 process. Once the state-of-health subsystem is available, we will have it monitor the job for all-processes-complete and then it can tell the registry to cleanup the job (i.e., remove all data relating to this job).
Hope that helps a little. I'll put all this into the design docs soon.
This commit was SVN r3754.
2004-12-09 00:44:41 +03:00
|
|
|
}
|
2005-09-01 05:07:30 +04:00
|
|
|
|
2005-03-14 23:57:21 +03:00
|
|
|
return ORTE_SUCCESS;
|
2004-11-20 22:12:43 +03:00
|
|
|
}
|
|
|
|
|
|
|
|
|
2005-06-24 20:59:37 +04:00
|
|
|
int
|
2005-09-01 05:07:30 +04:00
|
|
|
orte_gpr_proxy_remove_subscription(orte_gpr_proxy_subscriber_t *sub)
|
2004-11-20 22:12:43 +03:00
|
|
|
{
|
2006-08-15 23:54:10 +04:00
|
|
|
orte_std_cntr_t index;
|
2005-09-01 05:07:30 +04:00
|
|
|
|
2005-09-09 22:27:17 +04:00
|
|
|
OPAL_TRACE(2);
|
|
|
|
|
2005-09-01 05:07:30 +04:00
|
|
|
if (NULL == sub) {
|
|
|
|
ORTE_ERROR_LOG(ORTE_ERR_BAD_PARAM);
|
|
|
|
return ORTE_ERR_BAD_PARAM;
|
2004-11-20 22:12:43 +03:00
|
|
|
}
|
2005-09-01 05:07:30 +04:00
|
|
|
|
|
|
|
index = sub->index;
|
|
|
|
OBJ_RELEASE(sub);
|
|
|
|
orte_pointer_array_set_item(orte_gpr_proxy_globals.subscriptions, index, NULL);
|
|
|
|
|
2005-03-14 23:57:21 +03:00
|
|
|
return ORTE_SUCCESS;
|
2004-11-20 22:12:43 +03:00
|
|
|
}
|
|
|
|
|
2005-07-18 22:49:00 +04:00
|
|
|
int
|
2005-09-01 05:07:30 +04:00
|
|
|
orte_gpr_proxy_remove_trigger(orte_gpr_proxy_trigger_t *trig)
|
2005-07-18 22:49:00 +04:00
|
|
|
{
|
2006-08-15 23:54:10 +04:00
|
|
|
orte_std_cntr_t index;
|
2005-09-01 05:07:30 +04:00
|
|
|
|
2005-09-09 22:27:17 +04:00
|
|
|
OPAL_TRACE(2);
|
|
|
|
|
2005-09-01 05:07:30 +04:00
|
|
|
if (NULL == trig) {
|
|
|
|
ORTE_ERROR_LOG(ORTE_ERR_BAD_PARAM);
|
|
|
|
return ORTE_ERR_BAD_PARAM;
|
2005-07-18 22:49:00 +04:00
|
|
|
}
|
2005-09-01 05:07:30 +04:00
|
|
|
|
|
|
|
index = trig->index;
|
|
|
|
OBJ_RELEASE(trig);
|
|
|
|
orte_pointer_array_set_item(orte_gpr_proxy_globals.triggers, index, NULL);
|
|
|
|
|
2005-07-18 22:49:00 +04:00
|
|
|
return ORTE_SUCCESS;
|
|
|
|
}
|
|
|
|
|