ports/openmpi
ports/openmpi
1
1
Форкнуть 0
Код Релизы Активность

Modify the daemon cmd processor to relay and then process the cmd locally. We couldn't do this before due to the daemon's needing to update contact info prior to doing the relay. However, the new routed system plus the inclusion of the nidmap in the launch message now makes this possible.

Просмотр исходного кода 
It is a small launch performance improvement as now we relay the launch cmd across to the next daemon before taking the time to launch our own local procs. Still, it does allow more parallel operations during the launch procedure.

This commit was SVN r20104.
Этот коммит содержится в:
Ralph Castain 2008-12-10 19:18:36 +00:00
родитель 67ae66326c
Коммит 9d7cb82bba
3 изменённых файлов: 120 добавлений и 101 удалений
Показать статистику Скачать Patch файл Скачать Diff файл Показать все файлы Свернуть все файлы

134
orte/mca/odls/base/odls_base_default_fns.c
Просмотреть файл

@ -87,6 +87,7 @@ int orte_odls_base_default_get_add_procs_data(opal_buffer_t *data,
int8_t flag;
int8_t *tmp;
orte_vpid_t i;
orte_daemon_cmd_flag_t command;
/* get the job data pointer */
if (NULL == (jdata = orte_get_job_data_object(job))) {
@ -150,6 +151,15 @@ int orte_odls_base_default_get_add_procs_data(opal_buffer_t *data,
}
OBJ_RELEASE(wireup);
/* insert an "add-procs" command here so we can cleanly process it on the
* other end
*/
command = ORTE_DAEMON_ADD_LOCAL_PROCS;
if (ORTE_SUCCESS != (rc = opal_dss.pack(data, &command, 1, ORTE_DAEMON_CMD))) {
ORTE_ERROR_LOG(rc);
return rc;
}
/* are we co-locating debugger daemons? */
if (NULL != orte_debugger_daemon) {
orte_app_context_t **apps;
@ -279,6 +289,75 @@ int orte_odls_base_default_get_add_procs_data(opal_buffer_t *data,
return ORTE_SUCCESS;
}
int orte_odls_base_default_update_daemon_info(opal_buffer_t *data)
{
opal_buffer_t wireup;
opal_byte_object_t *bo;
int rc;
orte_std_cntr_t cnt;
int32_t numbytes;
/* extract the byte object holding the daemonmap */
cnt=1;
if (ORTE_SUCCESS != (rc = opal_dss.unpack(data, &bo, &cnt, OPAL_BYTE_OBJECT))) {
ORTE_ERROR_LOG(rc);
return rc;
}
/* retain a copy for downloading to child processes */
if (NULL != orte_odls_globals.dmap) {
free(orte_odls_globals.dmap->bytes);
free(orte_odls_globals.dmap);
orte_odls_globals.dmap = NULL;
}
opal_dss.copy((void**)&orte_odls_globals.dmap, bo, OPAL_BYTE_OBJECT);
/* update our local nidmap, if required - the decode function
* knows what to do - it will also free the bytes in the bo
*/
OPAL_OUTPUT_VERBOSE((5, orte_odls_globals.output,
"%s odls:update:daemon:info updating nidmap",
ORTE_NAME_PRINT(ORTE_PROC_MY_NAME)));
if (ORTE_SUCCESS != (rc = orte_ess.update_nidmap(bo))) {
ORTE_ERROR_LOG(rc);
return rc;
}
/* update the routing tree */
if (ORTE_SUCCESS != (rc = orte_routed.update_routing_tree())) {
ORTE_ERROR_LOG(rc);
return rc;
}
/* unpack the #bytes of daemon wireup info in the message */
cnt=1;
if (ORTE_SUCCESS != (rc = opal_dss.unpack(data, &numbytes, &cnt, OPAL_INT32))) {
ORTE_ERROR_LOG(rc);
return rc;
}
/* any bytes there? */
if (0 < numbytes) {
/* unpack the byte object */
cnt=1;
if (ORTE_SUCCESS != (rc = opal_dss.unpack(data, &bo, &cnt, OPAL_BYTE_OBJECT))) {
ORTE_ERROR_LOG(rc);
return rc;
}
/* load it into a buffer */
OBJ_CONSTRUCT(&wireup, opal_buffer_t);
opal_dss.load(&wireup, bo->bytes, bo->size);
/* pass it for processing */
if (ORTE_SUCCESS != (rc = orte_routed.init_routes(ORTE_PROC_MY_NAME->jobid, &wireup))) {
ORTE_ERROR_LOG(rc);
OBJ_DESTRUCT(&wireup);
return rc;
}
/* done with the buffer - dump it */
OBJ_DESTRUCT(&wireup);
}
return ORTE_SUCCESS;
}
int orte_odls_base_default_construct_child_list(opal_buffer_t *data,
orte_jobid_t *job)
{
@ -288,9 +367,7 @@ int orte_odls_base_default_construct_child_list(opal_buffer_t *data,
orte_std_cntr_t cnt;
orte_process_name_t proc, daemon;
orte_odls_job_t *jobdat;
opal_buffer_t wireup;
opal_byte_object_t *bo;
int32_t numbytes;
opal_buffer_t alert;
opal_list_item_t *item;
orte_namelist_t *nm;
@ -313,59 +390,6 @@ int orte_odls_base_default_construct_child_list(opal_buffer_t *data,
/* set the default values since they may not be included in the data */
*job = ORTE_JOBID_INVALID;
/* extract the byte object holding the daemonmap */
cnt=1;
if (ORTE_SUCCESS != (rc = opal_dss.unpack(data, &bo, &cnt, OPAL_BYTE_OBJECT))) {
ORTE_ERROR_LOG(rc);
goto REPORT_ERROR;
}
/* retain a copy for downloading to child processes */
opal_dss.copy((void**)&orte_odls_globals.dmap, bo, OPAL_BYTE_OBJECT);
/* update our local nidmap, if required - the decode function
* knows what to do - it will also free the bytes in the bo
*/
OPAL_OUTPUT_VERBOSE((5, orte_odls_globals.output,
"%s odls:construct updating nidmap",
ORTE_NAME_PRINT(ORTE_PROC_MY_NAME)));
if (ORTE_SUCCESS != (rc = orte_ess.update_nidmap(bo))) {
ORTE_ERROR_LOG(rc);
goto REPORT_ERROR;
}
/* update the routing tree */
if (ORTE_SUCCESS != (rc = orte_routed.update_routing_tree())) {
ORTE_ERROR_LOG(rc);
goto REPORT_ERROR;
}
/* unpack the #bytes of daemon wireup info in the message */
cnt=1;
if (ORTE_SUCCESS != (rc = opal_dss.unpack(data, &numbytes, &cnt, OPAL_INT32))) {
ORTE_ERROR_LOG(rc);
goto REPORT_ERROR;
}
/* any bytes there? */
if (0 < numbytes) {
/* unpack the byte object */
cnt=1;
if (ORTE_SUCCESS != (rc = opal_dss.unpack(data, &bo, &cnt, OPAL_BYTE_OBJECT))) {
ORTE_ERROR_LOG(rc);
goto REPORT_ERROR;
}
/* load it into a buffer */
OBJ_CONSTRUCT(&wireup, opal_buffer_t);
opal_dss.load(&wireup, bo->bytes, bo->size);
/* pass it for processing */
if (ORTE_SUCCESS != (rc = orte_routed.init_routes(ORTE_PROC_MY_NAME->jobid, &wireup))) {
ORTE_ERROR_LOG(rc);
OBJ_DESTRUCT(&wireup);
goto REPORT_ERROR;
}
/* done with the buffer - dump it */
OBJ_DESTRUCT(&wireup);
}
/* unpack the flag - are we co-locating debugger daemons? */
cnt=1;
if (ORTE_SUCCESS != (rc = opal_dss.unpack(data, &flag, &cnt, OPAL_INT8))) {

3
orte/mca/odls/base/odls_private.h
Просмотреть файл

@ -125,6 +125,9 @@ ORTE_DECLSPEC int
orte_odls_base_default_get_add_procs_data(opal_buffer_t *data,
orte_jobid_t job);
ORTE_DECLSPEC int
orte_odls_base_default_update_daemon_info(opal_buffer_t *data);
ORTE_DECLSPEC int
orte_odls_base_default_construct_child_list(opal_buffer_t *data,
orte_jobid_t *job);

84
orte/orted/orted_comm.c
Просмотреть файл

@ -71,6 +71,8 @@
#include "orte/mca/plm/base/plm_private.h"
#include "orte/mca/routed/routed.h"
#include "orte/mca/odls/base/odls_private.h"
#include "orte/runtime/runtime.h"
#include "orte/runtime/orte_globals.h"
#include "orte/runtime/orte_wait.h"
@ -80,9 +82,6 @@
/*
* Globals
*/
static bool relay_is_required;
static bool exit_after_relay;
static int process_commands(orte_process_name_t* sender,
opal_buffer_t *buffer,
orte_rml_tag_t tag);
@ -98,12 +97,9 @@ static void send_callback(int status, orte_process_name_t *peer,
OBJ_RELEASE(buf);
}
static void send_relay(int fd, short event, void *data)
static void send_relay(opal_buffer_t *buf, orte_jobid_t target_job, orte_rml_tag_t tag)
{
orte_message_event_t *mev = (orte_message_event_t*)data;
opal_buffer_t *buffer=NULL;
orte_rml_tag_t tag = mev->tag;
orte_jobid_t target_job = mev->sender.jobid;
opal_buffer_t *buffer = NULL;
opal_list_t recips;
opal_list_item_t *item;
orte_namelist_t *nm;
@ -149,17 +145,17 @@ static void send_relay(int fd, short event, void *data)
* recipients can correctly process it
*/
n = 1;
if (ORTE_SUCCESS != (ret = opal_dss.unpack(mev->buffer, &command, &n, ORTE_DAEMON_CMD))) {
if (ORTE_SUCCESS != (ret = opal_dss.unpack(buf, &command, &n, ORTE_DAEMON_CMD))) {
ORTE_ERROR_LOG(ret);
goto CLEANUP;
}
n = 1;
if (ORTE_SUCCESS != (ret = opal_dss.unpack(mev->buffer, &job, &n, ORTE_JOBID))) {
if (ORTE_SUCCESS != (ret = opal_dss.unpack(buf, &job, &n, ORTE_JOBID))) {
ORTE_ERROR_LOG(ret);
goto CLEANUP;
}
n = 1;
if (ORTE_SUCCESS != (ret = opal_dss.unpack(mev->buffer, &msg_tag, &n, ORTE_RML_TAG))) {
if (ORTE_SUCCESS != (ret = opal_dss.unpack(buf, &msg_tag, &n, ORTE_RML_TAG))) {
ORTE_ERROR_LOG(ret);
goto CLEANUP;
}
@ -167,26 +163,26 @@ static void send_relay(int fd, short event, void *data)
if (ORTE_RML_TAG_DAEMON != tag) {
/* remove the message_local_procs cmd data */
n = 1;
if (ORTE_SUCCESS != (ret = opal_dss.unpack(mev->buffer, &command, &n, ORTE_DAEMON_CMD))) {
if (ORTE_SUCCESS != (ret = opal_dss.unpack(buf, &command, &n, ORTE_DAEMON_CMD))) {
ORTE_ERROR_LOG(ret);
goto CLEANUP;
}
n = 1;
if (ORTE_SUCCESS != (ret = opal_dss.unpack(mev->buffer, &job, &n, ORTE_JOBID))) {
if (ORTE_SUCCESS != (ret = opal_dss.unpack(buf, &job, &n, ORTE_JOBID))) {
ORTE_ERROR_LOG(ret);
goto CLEANUP;
}
n = 1;
if (ORTE_SUCCESS != (ret = opal_dss.unpack(mev->buffer, &msg_tag, &n, ORTE_RML_TAG))) {
if (ORTE_SUCCESS != (ret = opal_dss.unpack(buf, &msg_tag, &n, ORTE_RML_TAG))) {
ORTE_ERROR_LOG(ret);
goto CLEANUP;
}
}
buffer = OBJ_NEW(opal_buffer_t);
opal_dss.copy_payload(buffer, mev->buffer);
opal_dss.copy_payload(buffer, buf);
} else {
/* buffer is already setup - just point to it */
buffer = mev->buffer;
buffer = buf;
/* tag needs to be set to daemon_tag */
tag = ORTE_RML_TAG_DAEMON;
}
@ -253,14 +249,9 @@ static void send_relay(int fd, short event, void *data)
CLEANUP:
/* cleanup */
OBJ_DESTRUCT(&recips);
if (NULL != buffer && buffer != mev->buffer) {
if (NULL != buffer && buffer != buf) {
OBJ_RELEASE(buffer);
}
OBJ_RELEASE(mev);
/* see if we need to exit */
if (exit_after_relay) {
orte_trigger_event(&orte_exit);
}
}
void orte_daemon_recv(int status, orte_process_name_t* sender,
@ -295,7 +286,7 @@ static int wait_time=1;
void orte_daemon_cmd_processor(int fd, short event, void *data)
{
orte_message_event_t *mev = (orte_message_event_t*)data;
orte_process_name_t *sender = &(mev->sender), target;
orte_process_name_t *sender = &(mev->sender);
opal_buffer_t *buffer = mev->buffer;
orte_rml_tag_t tag = mev->tag, target_tag;
orte_jobid_t job;
@ -378,22 +369,33 @@ void orte_daemon_cmd_processor(int fd, short event, void *data)
ORTE_ERROR_LOG(ret);
goto CLEANUP;
}
/* let the send_relay function know the target jobid */
target.jobid = job;
target.vpid = ORTE_VPID_INVALID; /* irrelevant, but better than random */
/* save this buffer location */
save = buffer->unpack_ptr;
/* rewind the buffer so we can relay it correctly */
/* unpack the command that will actually be executed */
n = 1;
if (ORTE_SUCCESS != (ret = opal_dss.unpack(buffer, &command, &n, ORTE_DAEMON_CMD))) {
ORTE_ERROR_LOG(ret);
goto CLEANUP;
}
/* is this an add-procs cmd? */
if (ORTE_DAEMON_ADD_LOCAL_PROCS == command) {
/* yes - then it contains daemon update info - process it */
if (ORTE_SUCCESS != (ret = orte_odls_base_default_update_daemon_info(buffer))) {
ORTE_ERROR_LOG(ret);
goto CLEANUP;
}
/* flag this location */
save = buffer->unpack_ptr;
}
/* rewind the buffer to the beginning */
buffer->unpack_ptr = unpack_ptr;
/* setup an event to actually perform the relay */
ORTE_MESSAGE_EVENT(&target, buffer, target_tag, send_relay);
/* flag that a relay is required */
relay_is_required = true;
/* rewind the buffer to the right place for processing */
/* do the relay */
send_relay(buffer, job, target_tag);
/* rewind the buffer to the right place for processing the cmd */
buffer->unpack_ptr = save;
} else {
/* flag that a relay is -not- required */
relay_is_required = false;
/* rewind the buffer so we can process it correctly */
buffer->unpack_ptr = unpack_ptr;
}
@ -669,12 +671,7 @@ static int process_commands(orte_process_name_t* sender,
orte_rml.send_buffer(ORTE_PROC_MY_HNP, &ack, ORTE_RML_TAG_PLM, 0);
OBJ_DESTRUCT(&ack);
}
/* check to see if we need to relay messages */
if (relay_is_required) {
exit_after_relay = true;
} else {
orte_trigger_event(&orte_exit);
}
orte_trigger_event(&orte_exit);
return ORTE_SUCCESS;
break;
@ -712,12 +709,7 @@ static int process_commands(orte_process_name_t* sender,
*/
return ORTE_SUCCESS;
}
/* check to see if we need to relay messages */
if (relay_is_required) {
exit_after_relay = true;
} else {
orte_trigger_event(&orte_exit);
}
orte_trigger_event(&orte_exit);
return ORTE_SUCCESS;
break;