From 8b2a285f8f5f7a1bbe631df1ef2970be189b76e6 Mon Sep 17 00:00:00 2001 From: Brian Barrett Date: Sun, 22 Jan 2006 04:56:18 +0000 Subject: [PATCH] * update reference implementation support to match latest release This commit was SVN r8783. --- .../mca/btl/portals/btl_portals_compat_utcp.c | 244 +++++++----------- ompi/mca/btl/portals/configure.m4 | 2 +- 2 files changed, 95 insertions(+), 151 deletions(-) diff --git a/ompi/mca/btl/portals/btl_portals_compat_utcp.c b/ompi/mca/btl/portals/btl_portals_compat_utcp.c index 4b85a50467..93d8f858d5 100644 --- a/ompi/mca/btl/portals/btl_portals_compat_utcp.c +++ b/ompi/mca/btl/portals/btl_portals_compat_utcp.c @@ -48,7 +48,6 @@ static bool use_modex = true; int mca_btl_portals_init_compat(mca_btl_portals_component_t *comp) { - ptl_process_id_t info; int ret, max_interfaces; #if 0 /* send all the portals internal debug to a file or stderr */ FILE *output; @@ -60,9 +59,6 @@ mca_btl_portals_init_compat(mca_btl_portals_component_t *comp) utcp_lib_out = output; utcp_api_out = output; -#else - utcp_lib_out = stderr; - utcp_api_out = stderr; #endif /* if the environment variables for the utcp implementation are @@ -71,49 +67,68 @@ mca_btl_portals_init_compat(mca_btl_portals_component_t *comp) platform (like, say, Red Storm). Otherwise, be nice and use the modex to setup everything for the user */ if (NULL == getenv("PTL_MY_RID")) { + char *iface_name; use_modex = true; + asprintf(&iface_name, "PTL_IFACE=%s", + mca_btl_portals_component.portals_ifname); + putenv(iface_name); } else { use_modex = false; } + /* Initialize Portals interface */ + ret = PtlInit(&max_interfaces); + if (PTL_OK != ret) { + opal_output_verbose(10, mca_btl_portals_component.portals_output, + "PtlInit failed, returning %d", ret); + return OMPI_ERR_FATAL; + } + + /* Initialize a network device */ + ret = PtlNIInit(PTL_IFACE_DEFAULT, /* interface to initialize */ + PTL_PID_ANY, /* let library assign our pid */ + NULL, /* no desired limits */ + NULL, /* no need to have limits around */ + &mca_btl_portals_module.portals_ni_h /* our interface handle */ + ); + if (PTL_OK != ret) { + opal_output_verbose(10, mca_btl_portals_component.portals_output, + "PtlNIInit failed, returning %d", ret); + return OMPI_ERR_FATAL; + } + if (use_modex) { + ptl_process_id_t proc_id; + + ret = PtlGetId(mca_btl_portals_module.portals_ni_h, &proc_id); + if (PTL_OK != ret) { + opal_output_verbose(10, mca_btl_portals_component.portals_output, + "PtlGetId failed, returning %d", ret); + return OMPI_ERR_FATAL; + } + /* post our contact info in the registry */ - info.nid = htonl(utcp_my_nid(mca_btl_portals_component.portals_ifname)); - info.pid = htonl((ptl_pid_t) getpid()); + proc_id.nid = htonl(proc_id.nid); + proc_id.pid = htonl(proc_id.pid); + opal_output_verbose(100, mca_btl_portals_component.portals_output, - "contact info: %u, %u", ntohl(info.nid), - ntohl(info.pid)); + "contact info: %u, %u", ntohl(proc_id.nid), + ntohl(proc_id.pid)); ret = mca_pml_base_modex_send(&mca_btl_portals_component.super.btl_version, - &info, sizeof(ptl_process_id_t)); + &proc_id, sizeof(ptl_process_id_t)); if (OMPI_SUCCESS != ret) { opal_output_verbose(10, mca_btl_portals_component.portals_output, "mca_pml_base_modex_send failed: %d", ret); return ret; } } else { - /* - * Initialize Portals interface - */ - ret = PtlInit(&max_interfaces); + /* Initialize the NID/PID data, per the 3.3.2 p3rt api */ + ret = PtlSetRank(mca_btl_portals_module.portals_ni_h, -1, -1); if (PTL_OK != ret) { opal_output_verbose(10, mca_btl_portals_component.portals_output, - "PtlInit failed, returning %d\n", ret); - return OMPI_ERR_FATAL; - } - - /* - * Initialize a network device - */ - ret = PtlNIInit(PTL_IFACE_DEFAULT, /* interface to initialize */ - PTL_PID_ANY, /* let library assign our pid */ - NULL, /* no desired limits */ - NULL, /* no need to have limits around */ - &mca_btl_portals_module.portals_ni_h /* our interface handle */ - ); - if (PTL_OK != ret) { - opal_output_verbose(10, mca_btl_portals_component.portals_output, - "PtlNIInit failed, returning %d\n", ret); + "PtlSetRank(handle, -1, -1) failed, returning %d", + ret); return OMPI_ERR_FATAL; } } @@ -128,45 +143,27 @@ mca_btl_portals_add_procs_compat(struct mca_btl_portals_module_t* btl, ptl_process_id_t **portals_procs) { int ret; + unsigned int nptl_procs, rank, i; if (use_modex) { int my_rid; - ptl_process_id_t *info; - char *nidmap = NULL; - char *pidmap = NULL; - char *nid_str; - char *pid_str; - const size_t map_size = nprocs * 12 + 1; /* 12 is max length of long in decimal */ - size_t size, i; - char *tmp; + ptl_process_id_t *proc_id; + ptl_nid_t *nidmap = NULL; + ptl_pid_t *pidmap = NULL; + size_t j, size; ompi_proc_t* proc_self = ompi_proc_local(); - int max_interfaces; - /* - * Do all the NID/PID map setup - */ - /* each nid is a int, so need 10 there, plus the : */ - nidmap = malloc(map_size); - pidmap = malloc(map_size); - nid_str = malloc(12 + 1); - pid_str = malloc(12 + 1); - if (NULL == nidmap || NULL == pidmap || - NULL == nid_str || NULL == pid_str) - return OMPI_ERROR; + /* Create and set the NID / PID maps as needed */ - /* get space for the portals procs list */ - *portals_procs = calloc(nprocs, sizeof(ptl_process_id_t)); - if (NULL == *portals_procs) { - opal_output_verbose(10, mca_btl_portals_component.portals_output, - "calloc(nprocs, sizeof(ptl_process_id_t)) failed"); - return OMPI_ERR_TEMP_OUT_OF_RESOURCE; - } - - for (i = 0 ; i < nprocs ; ++i) { - if (proc_self == procs[i]) my_rid = i; + nidmap = malloc(sizeof(ptl_nid_t) * nprocs); + pidmap = malloc(sizeof(ptl_pid_t) * nprocs); + if (NULL == nidmap || NULL == pidmap) return OMPI_ERROR; + + for (j = 0 ; j < nprocs ; ++j) { + if (proc_self == procs[j]) my_rid = j; ret = mca_pml_base_modex_recv(&mca_btl_portals_component.super.btl_version, - procs[i], (void**) &info, &size); + procs[j], (void**) &proc_id, &size); if (OMPI_SUCCESS != ret) { opal_output_verbose(10, mca_btl_portals_component.portals_output, "mca_pml_base_modex_recv failed: %d", ret); @@ -178,108 +175,55 @@ mca_btl_portals_add_procs_compat(struct mca_btl_portals_module_t* btl, return OMPI_ERROR; } - if (i == 0) { - snprintf(nidmap, map_size, "%u", ntohl(info->nid)); - snprintf(pidmap, map_size, "%u", ntohl(info->pid)); - } else { - snprintf(nid_str, 12 + 1, ":%u", ntohl(info->nid)); - snprintf(pid_str, 12 + 1, ":%u", ntohl(info->pid)); - strncat(nidmap, nid_str, 12); - strncat(pidmap, pid_str, 12); - } + nidmap[j] = ntohl(proc_id->nid); + pidmap[j] = ntohl(proc_id->pid); - /* update my local array of proc structs */ - (*portals_procs)[i].nid = ntohl(info->nid); - (*portals_procs)[i].pid = ntohl(info->pid); - - free(info); + free(proc_id); } - opal_output_verbose(100, mca_btl_portals_component.portals_output, - "my rid: %u", my_rid); - opal_output_verbose(100, mca_btl_portals_component.portals_output, - "nid map: %s", nidmap); - opal_output_verbose(100, mca_btl_portals_component.portals_output, - "pid map: %s", pidmap); - opal_output_verbose(100, mca_btl_portals_component.portals_output, - "iface: %s", - mca_btl_portals_component.portals_ifname); - - asprintf(&tmp, "PTL_MY_RID=%u", my_rid); - putenv(tmp); - asprintf(&tmp, "PTL_NIDMAP=%s", nidmap); - putenv(tmp); - asprintf(&tmp, "PTL_PIDMAP=%s", pidmap); - putenv(tmp); - asprintf(&tmp, "PTL_IFACE=%s", mca_btl_portals_component.portals_ifname); - putenv(tmp); + PtlSetRank(mca_btl_portals_module.portals_ni_h, + my_rid, (unsigned) nprocs); + PtlSetNIDMap(mca_btl_portals_module.portals_ni_h, + nidmap, (unsigned) nprocs); + PtlSetPIDMap(mca_btl_portals_module.portals_ni_h, + pidmap, (unsigned) nprocs); free(pidmap); free(nidmap); - free(pid_str); - free(nid_str); + } - /* - * Initialize Portals - */ - ret = PtlInit(&max_interfaces); + ret = PtlGetRank(mca_btl_portals_module.portals_ni_h, &rank, &nptl_procs); + if (ret != PTL_OK) { + opal_output_verbose(10, mca_btl_portals_component.portals_output, + "PtlGetRank() returned %d", ret); + return OMPI_ERR_FATAL; + } else if (nptl_procs != nprocs) { + opal_output_verbose(10, mca_btl_portals_component.portals_output, + "nptl_procs != nprocs (%d, %d)", nptl_procs, + nprocs); + return OMPI_ERR_FATAL; + } + + /* create enough space for all the proc info structs */ + *portals_procs = calloc(nprocs, sizeof(ptl_process_id_t)); + if (NULL == *portals_procs) { + opal_output_verbose(10, mca_btl_portals_component.portals_output, + "calloc(nprocs, sizeof(ptl_process_id_t)) failed"); + return OMPI_ERR_TEMP_OUT_OF_RESOURCE; + } + /* fill in all the proc info structs */ + for (i = 0 ; i < nprocs ; ++i) { + ret = PtlGetRankId(mca_btl_portals_module.portals_ni_h, + i, &((*portals_procs)[i])); if (PTL_OK != ret) { - opal_output_verbose(10, mca_btl_portals_component.portals_output, - "PtlInit failed, returning %d\n", ret); + opal_output_verbose(10, + mca_btl_portals_component.portals_output, + "PtlGetRankId(%d) failed: %d\n", i, ret); return OMPI_ERR_FATAL; } - - ret = PtlNIInit(PTL_IFACE_DEFAULT, /* interface to initialize */ - PTL_PID_ANY, /* let library assign our pid */ - NULL, /* no desired limits */ - NULL, /* save our limits somewhere */ - &(mca_btl_portals_module.portals_ni_h) /* our interface handle */ - ); - if (PTL_OK != ret) { - opal_output_verbose(10, mca_btl_portals_component.portals_output, - "PtlNIInit failed, returning %d\n", ret); - return OMPI_ERR_FATAL; - } - } else { /* use_modex */ - unsigned int nptl_procs, rank, i; - - /* - * FIXME - XXX - FIXME - * BWB - implicit assumption that cnos procs list will match our - * procs list. Don't know what to do about that... - */ - ret = PtlGetRank(&rank, &nptl_procs); - if (ret != PTL_OK) { - opal_output_verbose(10, mca_btl_portals_component.portals_output, - "PtlGetRank() returned %d", ret); - return OMPI_ERR_FATAL; - } else if (nptl_procs != nprocs) { - opal_output_verbose(10, mca_btl_portals_component.portals_output, - "nptl_procs != nprocs (%d, %d)", nptl_procs, - nprocs); - return OMPI_ERR_FATAL; - } - - /* create enough space for all the proc info structs */ - *portals_procs = calloc(nprocs, sizeof(ptl_process_id_t)); - if (NULL == *portals_procs) { - opal_output_verbose(10, mca_btl_portals_component.portals_output, - "calloc(nprocs, sizeof(ptl_process_id_t)) failed"); - return OMPI_ERR_TEMP_OUT_OF_RESOURCE; - } - /* fill in all the proc info structs */ - for (i = 0 ; i < nprocs ; ++i) { - ret = PtlGetRankId(i, &((*portals_procs)[i])); - if (PTL_OK != ret) { - opal_output_verbose(10, - mca_btl_portals_component.portals_output, - "PtlGetRankId(%d) failed: %d\n", i, ret); - return OMPI_ERR_FATAL; - } - } } -#if 0 +#if 1 PtlNIDebug(mca_btl_portals_module.portals_ni_h, PTL_DBG_ALL); #endif diff --git a/ompi/mca/btl/portals/configure.m4 b/ompi/mca/btl/portals/configure.m4 index 4aac8e27ca..a94d509f2f 100644 --- a/ompi/mca/btl/portals/configure.m4 +++ b/ompi/mca/btl/portals/configure.m4 @@ -114,7 +114,7 @@ AC_DEFUN([MCA_btl_portals_CONFIG_PLATFORM], [ "utcp") BTL_PORTALS_UTCP=1 BTL_PORTALS_HAVE_EVENT_UNLINK=1 - btl_portals_LIBS="-lutcpapi -lutcplib -lp3api -lp3lib -lp3rt" + btl_portals_LIBS="-lp3utcp -lp3api -lp3lib -lp3rt -lp3utcp" btl_portals_compat="utcp" btl_portals_header_prefix= btl_portals_starting_table_id=0