2004-01-15 06:08:25 +00:00
|
|
|
/*
|
2004-11-22 01:38:40 +00:00
|
|
|
* Copyright (c) 2004-2005 The Trustees of Indiana University.
|
|
|
|
|
* All rights reserved.
|
|
|
|
|
* Copyright (c) 2004-2005 The Trustees of the University of Tennessee.
|
|
|
|
|
* All rights reserved.
|
2004-11-28 20:09:25 +00:00
|
|
|
* Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
|
|
|
|
|
* University of Stuttgart. All rights reserved.
|
2004-11-22 01:38:40 +00:00
|
|
|
* $COPYRIGHT$
|
|
|
|
|
*
|
|
|
|
|
* Additional copyrights may follow
|
|
|
|
|
*
|
2004-01-15 06:08:25 +00:00
|
|
|
* $HEADER$
|
|
|
|
|
*/
|
|
|
|
|
|
2004-06-07 15:33:53 +00:00
|
|
|
#include "ompi_config.h"
|
2004-01-15 06:08:25 +00:00
|
|
|
|
2004-03-18 21:35:28 +00:00
|
|
|
#include "include/constants.h"
|
2004-01-15 06:08:25 +00:00
|
|
|
#include "mpi.h"
|
2004-09-16 09:22:28 +00:00
|
|
|
#include "event/event.h"
|
2004-03-17 18:45:16 +00:00
|
|
|
#include "group/group.h"
|
2004-08-29 09:05:14 +00:00
|
|
|
#include "errhandler/errcode.h"
|
|
|
|
|
#include "errhandler/errclass.h"
|
2004-04-20 22:38:22 +00:00
|
|
|
#include "communicator/communicator.h"
|
|
|
|
|
#include "datatype/datatype.h"
|
|
|
|
|
#include "op/op.h"
|
2004-08-14 01:56:05 +00:00
|
|
|
#include "file/file.h"
|
2004-08-12 16:56:24 +00:00
|
|
|
#include "info/info.h"
|
2004-11-17 02:30:07 +00:00
|
|
|
#include "util/proc_info.h"
|
2004-03-17 18:45:16 +00:00
|
|
|
#include "runtime/runtime.h"
|
2004-10-28 15:40:46 +00:00
|
|
|
#include "runtime/ompi_progress.h"
|
2004-09-28 01:38:16 +00:00
|
|
|
#include "runtime/ompi_rte_wait.h"
|
2004-11-05 07:52:30 +00:00
|
|
|
#include "attribute/attribute.h"
|
2004-08-14 01:56:05 +00:00
|
|
|
|
2004-03-17 18:45:16 +00:00
|
|
|
#include "mca/base/base.h"
|
2004-10-14 20:50:06 +00:00
|
|
|
#include "mca/base/mca_base_module_exchange.h"
|
2004-03-17 18:45:16 +00:00
|
|
|
#include "mca/ptl/ptl.h"
|
|
|
|
|
#include "mca/ptl/base/base.h"
|
|
|
|
|
#include "mca/pml/pml.h"
|
|
|
|
|
#include "mca/pml/base/base.h"
|
|
|
|
|
#include "mca/coll/coll.h"
|
|
|
|
|
#include "mca/coll/base/base.h"
|
2004-08-14 01:56:05 +00:00
|
|
|
#include "mca/topo/topo.h"
|
|
|
|
|
#include "mca/topo/base/base.h"
|
|
|
|
|
#include "mca/io/io.h"
|
|
|
|
|
#include "mca/io/base/base.h"
|
2004-09-16 08:38:24 +00:00
|
|
|
#include "mca/oob/base/base.h"
|
2004-09-23 14:34:35 +00:00
|
|
|
#include "mca/ns/base/base.h"
|
2004-11-20 19:12:43 +00:00
|
|
|
#include "mca/gpr/base/base.h"
|
2005-01-15 13:20:26 +00:00
|
|
|
#include "mca/mpool/base/base.h"
|
2004-01-29 19:40:22 +00:00
|
|
|
|
2004-01-15 06:08:25 +00:00
|
|
|
|
2004-06-07 15:33:53 +00:00
|
|
|
int ompi_mpi_finalize(void)
|
2004-01-15 06:08:25 +00:00
|
|
|
{
|
2004-11-20 19:12:43 +00:00
|
|
|
int ret;
|
|
|
|
|
ompi_rte_process_status_t my_status;
|
Well, we are getting closer to resolving the comm_spawn problem. For the benefit of those that haven't been in the midst of this discussion, the problem is that this is the first case where the process starting a set of processes has not been mpirun and is not guaranteed to be alive throughout the lifetime of the spawned processes. This sounds simple, but actually has some profound impacts.
Most of this checkin consists of more debugging stuff. Hopefully, you won't see any printf's that aren't protected by debug flags - if you do, let me know and I'll take them out with my apologies.
Outside of debugging, the biggest change was a revamp of the shutdown process. For several reasons, we had chosen to have all processes "wait" for a shutdown message before exiting. This message is typically generated by mpirun, but in the case of comm_spawn we needed to do something else. We have decided that the best way to solve this problem is to:
(a) replace the shutdown message (which needed to be generated by somebody - usually mpirun) with an oob_barrier call. This still requires that the rank 0 process be alive. However, we terminate all processes if one abnormally terminates anyway, so this isn't a problem (with the standard or our implementation); and
(b) have the state-of-health monitoring subsystem issue the call to cleanup the job from the registry. Since the state-of-health subsystem isn't available yet, we have temporarily assigned that responsibility to the rank 0 process. Once the state-of-health subsystem is available, we will have it monitor the job for all-processes-complete and then it can tell the registry to cleanup the job (i.e., remove all data relating to this job).
Hope that helps a little. I'll put all this into the design docs soon.
This commit was SVN r3754.
2004-12-08 21:44:41 +00:00
|
|
|
int my_rank;
|
|
|
|
|
mca_ns_base_jobid_t my_jobid;
|
2004-01-15 06:08:25 +00:00
|
|
|
|
2004-11-20 19:12:43 +00:00
|
|
|
ompi_mpi_finalized = true;
|
2004-10-28 15:40:46 +00:00
|
|
|
#if OMPI_HAVE_THREADS == 0
|
2005-01-18 17:32:54 +00:00
|
|
|
ompi_progress_events(OMPI_EVLOOP_NONBLOCK);
|
2004-10-28 15:40:46 +00:00
|
|
|
#endif
|
2004-01-29 19:40:22 +00:00
|
|
|
|
2004-11-20 19:12:43 +00:00
|
|
|
/* begin recording compound command */
|
|
|
|
|
ompi_registry.begin_compound_cmd();
|
|
|
|
|
|
|
|
|
|
/* Set process status to "terminating"*/
|
Well, we are getting closer to resolving the comm_spawn problem. For the benefit of those that haven't been in the midst of this discussion, the problem is that this is the first case where the process starting a set of processes has not been mpirun and is not guaranteed to be alive throughout the lifetime of the spawned processes. This sounds simple, but actually has some profound impacts.
Most of this checkin consists of more debugging stuff. Hopefully, you won't see any printf's that aren't protected by debug flags - if you do, let me know and I'll take them out with my apologies.
Outside of debugging, the biggest change was a revamp of the shutdown process. For several reasons, we had chosen to have all processes "wait" for a shutdown message before exiting. This message is typically generated by mpirun, but in the case of comm_spawn we needed to do something else. We have decided that the best way to solve this problem is to:
(a) replace the shutdown message (which needed to be generated by somebody - usually mpirun) with an oob_barrier call. This still requires that the rank 0 process be alive. However, we terminate all processes if one abnormally terminates anyway, so this isn't a problem (with the standard or our implementation); and
(b) have the state-of-health monitoring subsystem issue the call to cleanup the job from the registry. Since the state-of-health subsystem isn't available yet, we have temporarily assigned that responsibility to the rank 0 process. Once the state-of-health subsystem is available, we will have it monitor the job for all-processes-complete and then it can tell the registry to cleanup the job (i.e., remove all data relating to this job).
Hope that helps a little. I'll put all this into the design docs soon.
This commit was SVN r3754.
2004-12-08 21:44:41 +00:00
|
|
|
my_rank = ompi_comm_rank(&ompi_mpi_comm_world);
|
|
|
|
|
my_status.rank = (int32_t)my_rank;
|
2004-11-30 16:27:32 +00:00
|
|
|
my_status.local_pid = (int32_t)ompi_process_info.pid;
|
|
|
|
|
my_status.nodename = strdup(ompi_system_info.nodename);
|
2004-11-20 19:12:43 +00:00
|
|
|
my_status.status_key = OMPI_PROC_TERMINATING;
|
|
|
|
|
my_status.exit_code = 0;
|
|
|
|
|
if (OMPI_SUCCESS != (ret = ompi_rte_set_process_status(&my_status, ompi_rte_get_self()))) {
|
|
|
|
|
return ret;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
/* execute the compound command - no return data requested
|
|
|
|
|
*/
|
|
|
|
|
ompi_registry.exec_compound_cmd(OMPI_REGISTRY_NO_RETURN_REQUESTED);
|
|
|
|
|
|
|
|
|
|
/* wait for all processes to reach same state */
|
Well, we are getting closer to resolving the comm_spawn problem. For the benefit of those that haven't been in the midst of this discussion, the problem is that this is the first case where the process starting a set of processes has not been mpirun and is not guaranteed to be alive throughout the lifetime of the spawned processes. This sounds simple, but actually has some profound impacts.
Most of this checkin consists of more debugging stuff. Hopefully, you won't see any printf's that aren't protected by debug flags - if you do, let me know and I'll take them out with my apologies.
Outside of debugging, the biggest change was a revamp of the shutdown process. For several reasons, we had chosen to have all processes "wait" for a shutdown message before exiting. This message is typically generated by mpirun, but in the case of comm_spawn we needed to do something else. We have decided that the best way to solve this problem is to:
(a) replace the shutdown message (which needed to be generated by somebody - usually mpirun) with an oob_barrier call. This still requires that the rank 0 process be alive. However, we terminate all processes if one abnormally terminates anyway, so this isn't a problem (with the standard or our implementation); and
(b) have the state-of-health monitoring subsystem issue the call to cleanup the job from the registry. Since the state-of-health subsystem isn't available yet, we have temporarily assigned that responsibility to the rank 0 process. Once the state-of-health subsystem is available, we will have it monitor the job for all-processes-complete and then it can tell the registry to cleanup the job (i.e., remove all data relating to this job).
Hope that helps a little. I'll put all this into the design docs soon.
This commit was SVN r3754.
2004-12-08 21:44:41 +00:00
|
|
|
mca_oob_barrier();
|
2004-12-02 13:28:10 +00:00
|
|
|
|
Well, we are getting closer to resolving the comm_spawn problem. For the benefit of those that haven't been in the midst of this discussion, the problem is that this is the first case where the process starting a set of processes has not been mpirun and is not guaranteed to be alive throughout the lifetime of the spawned processes. This sounds simple, but actually has some profound impacts.
Most of this checkin consists of more debugging stuff. Hopefully, you won't see any printf's that aren't protected by debug flags - if you do, let me know and I'll take them out with my apologies.
Outside of debugging, the biggest change was a revamp of the shutdown process. For several reasons, we had chosen to have all processes "wait" for a shutdown message before exiting. This message is typically generated by mpirun, but in the case of comm_spawn we needed to do something else. We have decided that the best way to solve this problem is to:
(a) replace the shutdown message (which needed to be generated by somebody - usually mpirun) with an oob_barrier call. This still requires that the rank 0 process be alive. However, we terminate all processes if one abnormally terminates anyway, so this isn't a problem (with the standard or our implementation); and
(b) have the state-of-health monitoring subsystem issue the call to cleanup the job from the registry. Since the state-of-health subsystem isn't available yet, we have temporarily assigned that responsibility to the rank 0 process. Once the state-of-health subsystem is available, we will have it monitor the job for all-processes-complete and then it can tell the registry to cleanup the job (i.e., remove all data relating to this job).
Hope that helps a little. I'll put all this into the design docs soon.
This commit was SVN r3754.
2004-12-08 21:44:41 +00:00
|
|
|
/* need the following code to cleanup the job in the registry.
|
|
|
|
|
* once the state-of-health monitoring system is available, we will
|
|
|
|
|
* have that system perform this function. until then, we will have the
|
|
|
|
|
* rank 0 process do it.
|
|
|
|
|
*/
|
|
|
|
|
if (0 == my_rank) {
|
|
|
|
|
my_jobid = ompi_name_server.get_jobid(ompi_rte_get_self());
|
|
|
|
|
ompi_rte_job_shutdown(my_jobid);
|
|
|
|
|
}
|
|
|
|
|
|
2004-11-20 19:12:43 +00:00
|
|
|
/* Shut down any bindings-specific issues: C++, F77, F90 (may or
|
|
|
|
|
may not be necessary...?) */
|
|
|
|
|
|
|
|
|
|
/* Free communication objects */
|
|
|
|
|
|
|
|
|
|
/* free window resources */
|
|
|
|
|
|
|
|
|
|
/* free file resources */
|
|
|
|
|
if (OMPI_SUCCESS != (ret = ompi_file_finalize())) {
|
|
|
|
|
return ret;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
/* free communicator resources */
|
|
|
|
|
if (OMPI_SUCCESS != (ret = ompi_comm_finalize())) {
|
|
|
|
|
return ret;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
/* free requests */
|
|
|
|
|
if (OMPI_SUCCESS != (ret = ompi_request_finalize())) {
|
|
|
|
|
return ret;
|
|
|
|
|
}
|
|
|
|
|
|
2004-12-02 13:28:10 +00:00
|
|
|
/* Now that all MPI objects dealing with communications are gone,
|
|
|
|
|
shut down MCA types having to do with communications */
|
|
|
|
|
if (OMPI_SUCCESS != (ret = mca_ptl_base_close())) {
|
|
|
|
|
return ret;
|
|
|
|
|
}
|
|
|
|
|
if (OMPI_SUCCESS != (ret = mca_pml_base_close())) {
|
|
|
|
|
return ret;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
2004-11-20 19:12:43 +00:00
|
|
|
/* Free secondary resources */
|
|
|
|
|
|
|
|
|
|
/* free attr resources */
|
|
|
|
|
if (OMPI_SUCCESS != (ret = ompi_attr_finalize())) {
|
|
|
|
|
return ret;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
/* free group resources */
|
|
|
|
|
if (OMPI_SUCCESS != (ret = ompi_group_finalize())) {
|
|
|
|
|
return ret;
|
|
|
|
|
}
|
|
|
|
|
|
2004-12-02 13:28:10 +00:00
|
|
|
/* free proc resources */
|
|
|
|
|
if ( OMPI_SUCCESS != (ret = ompi_proc_finalize())) {
|
|
|
|
|
return ret;
|
|
|
|
|
}
|
|
|
|
|
|
2004-11-20 19:12:43 +00:00
|
|
|
/* free internal error resources */
|
|
|
|
|
if (OMPI_SUCCESS != (ret = ompi_errcode_intern_finalize())) {
|
|
|
|
|
return ret;
|
|
|
|
|
}
|
2004-08-29 09:05:14 +00:00
|
|
|
|
2004-11-20 19:12:43 +00:00
|
|
|
/* free error class resources */
|
|
|
|
|
if (OMPI_SUCCESS != (ret = ompi_errclass_finalize())) {
|
|
|
|
|
return ret;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
/* free error code resources */
|
|
|
|
|
if (OMPI_SUCCESS != (ret = ompi_mpi_errcode_finalize())) {
|
|
|
|
|
return ret;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
/* free errhandler resources */
|
|
|
|
|
if (OMPI_SUCCESS != (ret = ompi_errhandler_finalize())) {
|
|
|
|
|
return ret;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
/* Free all other resources */
|
|
|
|
|
|
|
|
|
|
/* free op resources */
|
|
|
|
|
if (OMPI_SUCCESS != (ret = ompi_op_finalize())) {
|
|
|
|
|
return ret;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
/* free ddt resources */
|
|
|
|
|
if (OMPI_SUCCESS != (ret = ompi_ddt_finalize())) {
|
|
|
|
|
return ret;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
/* free info resources */
|
|
|
|
|
if (OMPI_SUCCESS != (ret = ompi_info_finalize())) {
|
|
|
|
|
return ret;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
/* free module exchange resources */
|
|
|
|
|
if (OMPI_SUCCESS != (ret = mca_base_modex_finalize())) {
|
|
|
|
|
return ret;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
/* Close down MCA modules */
|
|
|
|
|
|
2005-01-04 15:43:26 +00:00
|
|
|
/* io is opened lazily, so it's only necessary to close it if it
|
|
|
|
|
was actually opened */
|
|
|
|
|
|
|
|
|
|
if (mca_io_base_components_opened_valid ||
|
|
|
|
|
mca_io_base_components_available_valid) {
|
|
|
|
|
if (OMPI_SUCCESS != (ret = mca_io_base_close())) {
|
|
|
|
|
return ret;
|
|
|
|
|
}
|
2004-11-20 19:12:43 +00:00
|
|
|
}
|
|
|
|
|
if (OMPI_SUCCESS != (ret = mca_topo_base_close())) {
|
|
|
|
|
return ret;
|
|
|
|
|
}
|
|
|
|
|
if (OMPI_SUCCESS != (ret = mca_coll_base_close())) {
|
|
|
|
|
return ret;
|
|
|
|
|
}
|
2005-01-15 13:20:26 +00:00
|
|
|
if (OMPI_SUCCESS != (ret = mca_mpool_base_close())) {
|
|
|
|
|
return ret;
|
|
|
|
|
}
|
2004-11-20 19:12:43 +00:00
|
|
|
|
|
|
|
|
/* Leave the RTE */
|
|
|
|
|
|
|
|
|
|
if (OMPI_SUCCESS != (ret = ompi_rte_finalize())) {
|
|
|
|
|
return ret;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
/* Close down the MCA */
|
|
|
|
|
|
|
|
|
|
if (OMPI_SUCCESS != (ret = mca_base_close())) {
|
|
|
|
|
return ret;
|
|
|
|
|
}
|
2004-08-31 09:37:28 +00:00
|
|
|
|
2004-11-20 19:12:43 +00:00
|
|
|
/* Leave OMPI land */
|
2004-08-29 09:05:14 +00:00
|
|
|
|
2004-11-20 19:12:43 +00:00
|
|
|
if (OMPI_SUCCESS != (ret = ompi_finalize())) {
|
|
|
|
|
return ret;
|
|
|
|
|
}
|
2004-08-29 09:05:14 +00:00
|
|
|
|
2004-11-20 19:12:43 +00:00
|
|
|
/* All done */
|
2004-01-15 06:08:25 +00:00
|
|
|
|
2004-11-20 19:12:43 +00:00
|
|
|
return MPI_SUCCESS;
|
2004-01-15 06:08:25 +00:00
|
|
|
}
|