/* * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana * University Research and Technology * Corporation. All rights reserved. * Copyright (c) 2004-2005 The University of Tennessee and The University * of Tennessee Research Foundation. All rights * reserved. * Copyright (c) 2004-2008 High Performance Computing Center Stuttgart, * University of Stuttgart. All rights reserved. * Copyright (c) 2004-2005 The Regents of the University of California. * All rights reserved. * Copyright (c) 2006-2007 University of Houston. All rights reserved. * Copyright (c) 2006-2007 Cisco Systems, Inc. All rights reserved. * $COPYRIGHT$ * * Additional copyrights may follow * * $HEADER$ */ #include "ompi_config.h" #include "ompi/mpi/c/bindings.h" #include "ompi/mca/pml/pml.h" #include "ompi/communicator/communicator.h" #include "ompi/request/request.h" #include "ompi/include/ompi/memchecker.h" #if OMPI_HAVE_WEAK_SYMBOLS && OMPI_PROFILING_DEFINES #pragma weak MPI_Intercomm_create = PMPI_Intercomm_create #endif #if OMPI_PROFILING_DEFINES #include "ompi/mpi/c/profile/defines.h" #endif static const char FUNC_NAME[] = "MPI_Intercomm_create"; int MPI_Intercomm_create(MPI_Comm local_comm, int local_leader, MPI_Comm bridge_comm, int remote_leader, int tag, MPI_Comm *newintercomm) { int local_size=0, local_rank=0; int lleader=0, rleader=0; ompi_communicator_t *newcomp=NULL; struct ompi_proc_t **rprocs=NULL; int rc=0, rsize=0; ompi_proc_t **proc_list=NULL; int j; ompi_group_t *new_group_pointer; MEMCHECKER( memchecker_comm(local_comm); memchecker_comm(bridge_comm); ); OPAL_CR_TEST_CHECKPOINT_READY(); if ( MPI_PARAM_CHECK ) { OMPI_ERR_INIT_FINALIZE(FUNC_NAME); if ( ompi_comm_invalid ( local_comm ) || ( local_comm->c_flags & OMPI_COMM_INTER ) ) return OMPI_ERRHANDLER_INVOKE ( MPI_COMM_WORLD, MPI_ERR_COMM, FUNC_NAME); if ( NULL == newintercomm ) return OMPI_ERRHANDLER_INVOKE ( local_comm, MPI_ERR_ARG, FUNC_NAME); /* if ( tag < 0 || tag > MPI_TAG_UB ) return OMPI_ERRHANDLER_INVOKE ( local_comm, MPI_ERR_ARG, FUNC_NAME); */ } local_size = ompi_comm_size ( local_comm ); local_rank = ompi_comm_rank ( local_comm ); lleader = local_leader; rleader = remote_leader; if ( MPI_PARAM_CHECK ) { if ( (0 > local_leader) || (local_leader >= local_size) ) return OMPI_ERRHANDLER_INVOKE ( local_comm, MPI_ERR_ARG, FUNC_NAME); /* remember that the remote_leader and bridge_comm arguments just have to be valid at the local_leader */ if ( local_rank == local_leader ) { if ( ompi_comm_invalid ( bridge_comm ) || (bridge_comm->c_flags & OMPI_COMM_INTER) ) { return OMPI_ERRHANDLER_INVOKE ( local_comm, MPI_ERR_COMM, FUNC_NAME); } if ( (remote_leader < 0) || (remote_leader >= ompi_comm_size(bridge_comm))) { return OMPI_ERRHANDLER_INVOKE ( local_comm, MPI_ERR_ARG, FUNC_NAME); } } /* if ( local_rank == local_leader ) */ } if ( local_rank == local_leader ) { MPI_Request req; /* local leader exchange group sizes lists */ rc = MCA_PML_CALL(irecv(&rsize, 1, MPI_INT, rleader, tag, bridge_comm, &req)); if ( rc != MPI_SUCCESS ) { goto err_exit; } rc = MCA_PML_CALL(send (&local_size, 1, MPI_INT, rleader, tag, MCA_PML_BASE_SEND_STANDARD, bridge_comm)); if ( rc != MPI_SUCCESS ) { goto err_exit; } rc = ompi_request_wait_all ( 1, &req, MPI_STATUS_IGNORE); if ( rc != MPI_SUCCESS ) { goto err_exit; } } /* bcast size and list of remote processes to all processes in local_comm */ rc = local_comm->c_coll.coll_bcast ( &rsize, 1, MPI_INT, lleader, local_comm, local_comm->c_coll.coll_bcast_module); if ( rc != MPI_SUCCESS ) { goto err_exit; } rprocs = ompi_comm_get_rprocs ( local_comm, bridge_comm, lleader, remote_leader, tag, rsize ); if ( NULL == rprocs ) { goto err_exit; } if ( MPI_PARAM_CHECK ) { if(OMPI_GROUP_IS_DENSE(local_comm->c_local_group)) { rc = ompi_comm_overlapping_groups(local_comm->c_local_group->grp_proc_count, local_comm->c_local_group->grp_proc_pointers, rsize, rprocs); } else { proc_list = (ompi_proc_t **) calloc (local_comm->c_local_group->grp_proc_count, sizeof (ompi_proc_t *)); for(j=0 ; jc_local_group->grp_proc_count ; j++) { proc_list[j] = ompi_group_peer_lookup(local_comm->c_local_group,j); } rc = ompi_comm_overlapping_groups(local_comm->c_local_group->grp_proc_count, proc_list, rsize, rprocs); } if ( OMPI_SUCCESS != rc ) { goto err_exit; } } new_group_pointer=ompi_group_allocate(rsize); if( NULL == new_group_pointer ) { return MPI_ERR_GROUP; } /* put group elements in the list */ for (j = 0; j < rsize; j++) { new_group_pointer->grp_proc_pointers[j] = rprocs[j]; } ompi_group_increment_proc_count(new_group_pointer); rc = ompi_comm_set ( &newcomp, /* new comm */ local_comm, /* old comm */ local_comm->c_local_group->grp_proc_count, /* local_size */ NULL, /* local_procs*/ rsize, /* remote_size */ NULL, /* remote_procs */ NULL, /* attrs */ local_comm->error_handler, /* error handler*/ NULL, /* topo mpodule */ local_comm->c_local_group, /* local group */ new_group_pointer /* remote group */ ); if ( NULL == newcomp ) { rc = MPI_ERR_INTERN; goto err_exit; } if ( MPI_SUCCESS != rc ) { goto err_exit; } ompi_group_decrement_proc_count (new_group_pointer); OBJ_RELEASE(new_group_pointer); new_group_pointer = MPI_GROUP_NULL; /* Determine context id. It is identical to f_2_c_handle */ rc = ompi_comm_nextcid ( newcomp, /* new comm */ local_comm, /* old comm */ bridge_comm, /* bridge comm */ &lleader, /* local leader */ &rleader, /* remote_leader */ OMPI_COMM_CID_INTRA_BRIDGE, /* mode */ -1 ); /* send_first */ if ( MPI_SUCCESS != rc ) { goto err_exit; } /* activate comm and init coll-module */ rc = ompi_comm_activate ( newcomp, /* new comm */ local_comm, /* old comm */ bridge_comm, /* bridge comm */ &lleader, /* local leader */ &rleader, /* remote_leader */ OMPI_COMM_CID_INTRA_BRIDGE, /* mode */ -1, /* send_first */ 0); /* sync_flag */ if ( MPI_SUCCESS != rc ) { goto err_exit; } err_exit: if ( NULL != rprocs ) { free ( rprocs ); } if ( NULL != proc_list ) { free ( proc_list ); } if ( OMPI_SUCCESS != rc ) { *newintercomm = MPI_COMM_NULL; return OMPI_ERRHANDLER_INVOKE(local_comm, MPI_ERR_INTERN, FUNC_NAME); } *newintercomm = newcomp; return MPI_SUCCESS; }