2c9be59b37
This new MTL runs over PSM2 for Omni Path. PSM2 is a descendant of PSM with changes to support more ranks and some MPI-3 features like mprobe. PSM2 will only support Omni Path networks; PSM only supports True Scale. Likewise, the existing PSM MTL will continue to be maintained for True Scale, while the PSM2 MTL is developed and maintained for Omni Path.
135 строки
3.7 KiB
C
135 строки
3.7 KiB
C
/*
|
|
* Copyright (c) 2004-2006 The Trustees of Indiana University and Indiana
|
|
* University Research and Technology
|
|
* Corporation. All rights reserved.
|
|
* Copyright (c) 2004-2010 The University of Tennessee and The University
|
|
* of Tennessee Research Foundation. All rights
|
|
* reserved.
|
|
* Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
|
|
* University of Stuttgart. All rights reserved.
|
|
* Copyright (c) 2004-2006 The Regents of the University of California.
|
|
* All rights reserved.
|
|
* Copyright (c) 2006 QLogic Corporation. All rights reserved.
|
|
* Copyright (c) 2006-2007 Los Alamos National Security, LLC. All rights
|
|
* reserved.
|
|
* Copyright (c) 2013-2015 Intel, Inc. All rights reserved
|
|
* $COPYRIGHT$
|
|
*
|
|
* Additional copyrights may follow
|
|
*
|
|
* $HEADER$
|
|
*/
|
|
|
|
#include "ompi_config.h"
|
|
#include "mtl_psm2.h"
|
|
#include "mtl_psm2_types.h"
|
|
#include "psm2.h"
|
|
#include "ompi/communicator/communicator.h"
|
|
#include "ompi/message/message.h"
|
|
|
|
|
|
int ompi_mtl_psm2_iprobe(struct mca_mtl_base_module_t* mtl,
|
|
struct ompi_communicator_t *comm,
|
|
int src,
|
|
int tag,
|
|
int *flag,
|
|
struct ompi_status_public_t *status)
|
|
{
|
|
psm_mq_tag_t mqtag, tagsel;
|
|
psm_mq_status2_t mqstat;
|
|
psm_error_t err;
|
|
|
|
PSM_MAKE_TAGSEL(src, tag, comm->c_contextid, mqtag, tagsel);
|
|
|
|
err = psm_mq_iprobe2(ompi_mtl_psm2.mq,
|
|
PSM_MQ_ANY_ADDR, &mqtag, &tagsel, &mqstat);
|
|
if (err == PSM_OK) {
|
|
*flag = 1;
|
|
if(MPI_STATUS_IGNORE != status) {
|
|
status->MPI_SOURCE = mqstat.msg_tag.tag2;
|
|
status->MPI_TAG = mqstat.msg_tag.tag1;
|
|
status->_ucount = mqstat.nbytes;
|
|
|
|
switch (mqstat.error_code) {
|
|
case PSM_OK:
|
|
status->MPI_ERROR = OMPI_SUCCESS;
|
|
break;
|
|
case PSM_MQ_TRUNCATION:
|
|
status->MPI_ERROR = MPI_ERR_TRUNCATE;
|
|
break;
|
|
default:
|
|
status->MPI_ERROR = MPI_ERR_INTERN;
|
|
}
|
|
}
|
|
|
|
return OMPI_SUCCESS;
|
|
}
|
|
else if (err == PSM_MQ_INCOMPLETE) {
|
|
*flag = 0;
|
|
return OMPI_SUCCESS;
|
|
}
|
|
else
|
|
return OMPI_ERROR;
|
|
}
|
|
|
|
|
|
int
|
|
ompi_mtl_psm2_improbe(struct mca_mtl_base_module_t *mtl,
|
|
struct ompi_communicator_t *comm,
|
|
int src,
|
|
int tag,
|
|
int *matched,
|
|
struct ompi_message_t **message,
|
|
struct ompi_status_public_t *status)
|
|
{
|
|
struct ompi_message_t* msg;
|
|
psm_mq_tag_t mqtag, tagsel;
|
|
psm_mq_status2_t mqstat;
|
|
psm_mq_req_t mqreq;
|
|
psm_error_t err;
|
|
|
|
PSM_MAKE_TAGSEL(src, tag, comm->c_contextid, mqtag, tagsel);
|
|
|
|
err = psm_mq_improbe2(ompi_mtl_psm2.mq,
|
|
PSM_MQ_ANY_ADDR, &mqtag, &tagsel, &mqreq, &mqstat);
|
|
if (err == PSM_OK) {
|
|
|
|
if(MPI_STATUS_IGNORE != status) {
|
|
status->MPI_SOURCE = mqstat.msg_tag.tag2;
|
|
status->MPI_TAG = mqstat.msg_tag.tag1;
|
|
status->_ucount = mqstat.nbytes;
|
|
|
|
switch (mqstat.error_code) {
|
|
case PSM_OK:
|
|
status->MPI_ERROR = OMPI_SUCCESS;
|
|
break;
|
|
case PSM_MQ_TRUNCATION:
|
|
status->MPI_ERROR = MPI_ERR_TRUNCATE;
|
|
break;
|
|
default:
|
|
status->MPI_ERROR = MPI_ERR_INTERN;
|
|
}
|
|
}
|
|
|
|
msg = ompi_message_alloc();
|
|
if(NULL == msg) {
|
|
return OMPI_ERR_OUT_OF_RESOURCE;
|
|
}
|
|
|
|
msg->comm = comm;
|
|
msg->req_ptr = mqreq;
|
|
msg->peer = mqstat.msg_tag.tag2;
|
|
msg->count = mqstat.nbytes;
|
|
|
|
*message = msg;
|
|
*matched = 1;
|
|
return OMPI_SUCCESS;
|
|
} else if(err == PSM_MQ_INCOMPLETE) {
|
|
*matched = 0;
|
|
*message = MPI_MESSAGE_NULL;
|
|
return OMPI_SUCCESS;
|
|
} else {
|
|
return OMPI_ERROR;
|
|
}
|
|
}
|