ports/openmpi
ports/openmpi
1
1
Fork 0
Code Releases Activity

Add new PSM2 MTL.

Browse Source 
This new MTL runs over PSM2 for Omni Path.  PSM2 is a descendant of PSM
with changes to support more ranks and some MPI-3 features like mprobe.

PSM2 will only support Omni Path networks; PSM only supports True Scale.
Likewise, the existing PSM MTL will continue to be maintained for True
Scale, while the PSM2 MTL is developed and maintained for Omni Path.
This commit is contained in:
Andrew Friedley 2014-05-27 08:09:27 -07:00
parent 74cf969561
commit 2c9be59b37
17 changed files with 1707 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

72
config/ompi_check_psm2.m4 Normal file
View File

@ -0,0 +1,72 @@
# -*- shell-script -*-
#
# Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
# University Research and Technology
# Corporation. All rights reserved.
# Copyright (c) 2004-2005 The University of Tennessee and The University
# of Tennessee Research Foundation. All rights
# reserved.
# Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
# University of Stuttgart. All rights reserved.
# Copyright (c) 2004-2006 The Regents of the University of California.
# All rights reserved.
# Copyright (c) 2006 QLogic Corp. All rights reserved.
# Copyright (c) 2009 Cisco Systems, Inc. All rights reserved.
# Copyright (c) 2014 Intel Corporation. All rights reserved.
# $COPYRIGHT$
#
# Additional copyrights may follow
#
# $HEADER$
#
# OMPI_CHECK_PSM2(prefix, [action-if-found], [action-if-not-found])
# --------------------------------------------------------
# check if PSM2 support can be found. sets prefix_{CPPFLAGS,
# LDFLAGS, LIBS} as needed and runs action-if-found if there is
# support, otherwise executes action-if-not-found
AC_DEFUN([OMPI_CHECK_PSM2],[
AC_ARG_WITH([psm2],
[AC_HELP_STRING([--with-psm2(=DIR)],
[Build PSM2 (Intel PSM2) support, optionally adding DIR/include, DIR/lib, and DIR/lib64 to the search path for headers and libraries])])
OPAL_CHECK_WITHDIR([psm2], [$with_psm2], [include/psm2.h])
AC_ARG_WITH([psm2-libdir],
[AC_HELP_STRING([--with-psm2-libdir=DIR],
[Search for PSM (Intel PSM2) libraries in DIR])])
OPAL_CHECK_WITHDIR([psm2-libdir], [$with_psm2_libdir], [libpsm2.*])
ompi_check_psm2_$1_save_CPPFLAGS="$CPPFLAGS"
ompi_check_psm2_$1_save_LDFLAGS="$LDFLAGS"
ompi_check_psm2_$1_save_LIBS="$LIBS"
AS_IF([test "$with_psm2" != "no"],
[AS_IF([test ! -z "$with_psm2" -a "$with_psm2" != "yes"],
[ompi_check_psm2_dir="$with_psm2"])
AS_IF([test ! -z "$with_psm2_libdir" -a "$with_psm2_libdir" != "yes"],
[ompi_check_psm2_libdir="$with_psm2_libdir"])
OPAL_CHECK_PACKAGE([$1],
[psm2.h],
[psm2],
[psm_mq_irecv2],
[],
[$ompi_check_psm2_dir],
[$ompi_check_psm2_libdir],
[ompi_check_psm2_happy="yes"],
[ompi_check_psm2_happy="no"])],
[ompi_check_psm2_happy="no"])
CPPFLAGS="$ompi_check_psm2_$1_save_CPPFLAGS"
LDFLAGS="$ompi_check_psm2_$1_save_LDFLAGS"
LIBS="$ompi_check_psm2_$1_save_LIBS"
AS_IF([test "$ompi_check_psm2_happy" = "yes" -a "$enable_progress_threads" = "yes"],
[AC_MSG_WARN([PSM2 driver does not currently support progress threads. Disabling MTL.])
ompi_check_psm2_happy="no"])
AS_IF([test "$ompi_check_psm2_happy" = "yes"],
[$2],
[AS_IF([test ! -z "$with_psm2" -a "$with_psm2" != "no"],
[AC_MSG_ERROR([PSM2 support requested but not found. Aborting])])
$3])
])

61
ompi/mca/mtl/psm2/Makefile.am Normal file
View File

@ -0,0 +1,61 @@
#
# Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
# University Research and Technology
# Corporation. All rights reserved.
# Copyright (c) 2004-2005 The University of Tennessee and The University
# of Tennessee Research Foundation. All rights
# reserved.
# Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
# University of Stuttgart. All rights reserved.
# Copyright (c) 2004-2006 The Regents of the University of California.
# All rights reserved.
# Copyright (c) 2010 Cisco Systems, Inc. All rights reserved.
# Copyright (c) 2015 Intel, Inc. All rights reserved
# $COPYRIGHT$
#
# Additional copyrights may follow
#
# $HEADER$
#
EXTRA_DIST = post_configure.sh
AM_CPPFLAGS = $(mtl_psm2_CPPFLAGS)
dist_ompidata_DATA = help-mtl-psm.txt
mtl_psm2_sources = \
mtl_psm2.c \
mtl_psm2.h \
mtl_psm2_cancel.c \
mtl_psm2_component.c \
mtl_psm2_endpoint.c \
mtl_psm2_endpoint.h \
mtl_psm2_probe.c \
mtl_psm2_recv.c \
mtl_psm2_request.h \
mtl_psm2_send.c \
mtl_psm2_types.h
# Make the output library in this directory, and name it either
# mca_<type>_<name>.la (for DSO builds) or libmca_<type>_<name>.la
# (for static builds).
if MCA_BUILD_ompi_mtl_psm2_DSO
component_noinst =
component_install = mca_mtl_psm2.la
else
component_noinst = libmca_mtl_psm2.la
component_install =
endif
mcacomponentdir = $(ompilibdir)
mcacomponent_LTLIBRARIES = $(component_install)
mca_mtl_psm2_la_SOURCES = $(mtl_psm2_sources)
mca_mtl_psm2_la_LIBADD = $(mtl_psm2_LIBS)
mca_mtl_psm2_la_LDFLAGS = -module -avoid-version $(mtl_psm2_LDFLAGS)
noinst_LTLIBRARIES = $(component_noinst)
libmca_mtl_psm2_la_SOURCES = $(mtl_psm2_sources)
libmca_mtl_psm2_la_LIBADD = $(mtl_psm2_LIBS)
libmca_mtl_psm2_la_LDFLAGS = -module -avoid-version $(mtl_psm2_LDFLAGS)

49
ompi/mca/mtl/psm2/configure.m4 Normal file
View File

@ -0,0 +1,49 @@
# -*- shell-script -*-
#
# Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
# University Research and Technology
# Corporation. All rights reserved.
# Copyright (c) 2004-2005 The University of Tennessee and The University
# of Tennessee Research Foundation. All rights
# reserved.
# Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
# University of Stuttgart. All rights reserved.
# Copyright (c) 2004-2005 The Regents of the University of California.
# All rights reserved.
# Copyright (c) 2010 Cisco Systems, Inc. All rights reserved.
# Copyright (c) 2013 Sandia National Laboratories. All rights reserved.
# Copyright (c) 2014 Intel Corporation. All rights reserved.
# $COPYRIGHT$
#
# Additional copyrights may follow
#
# $HEADER$
#
# MCA_ompi_mtl_psm2_POST_CONFIG(will_build)
# ----------------------------------------
# Only require the tag if we're actually going to be built
AC_DEFUN([MCA_ompi_mtl_psm2_POST_CONFIG], [
AS_IF([test "$1" = "1"], [OMPI_REQUIRE_ENDPOINT_TAG([MTL])])
])dnl
# MCA_mtl_psm2_CONFIG([action-if-can-compile],
# [action-if-cant-compile])
# ------------------------------------------------
AC_DEFUN([MCA_ompi_mtl_psm2_CONFIG],[
AC_CONFIG_FILES([ompi/mca/mtl/psm2/Makefile])
OMPI_CHECK_PSM2([mtl_psm2],
[mtl_psm2_happy="yes"],
[mtl_psm2_happy="no"])
AS_IF([test "$mtl_psm2_happy" = "yes"],
[$1],
[$2])
# substitute in the things needed to build psm2
AC_SUBST([mtl_psm2_CFLAGS])
AC_SUBST([mtl_psm2_CPPFLAGS])
AC_SUBST([mtl_psm2_LDFLAGS])
AC_SUBST([mtl_psm2_LIBS])
])dnl

43
ompi/mca/mtl/psm2/help-mtl-psm.txt Normal file
View File

@ -0,0 +1,43 @@
# -*- text -*-
#
# Copyright (C) 2009. QLogic Corporation. All rights reserved.
# $COPYRIGHT$
#
# Additional copyrights may follow
#
# $HEADER$
#
[psm init]
Initialization of PSM library failed.
Error: %s
#
[debug level]
Unable to set PSM debug level.
Error: %s
#
[unable to open endpoint]
PSM was unable to open an endpoint. Please make sure that the network link is
active on the node and the hardware is functioning.
Error: %s
#
[no uuid present]
Error obtaining unique transport key from ORTE (orte_precondition_transports %s
the environment).
Local host: %s
#
[error polling network]
Error %s occurred in attempting to make network progress (psm_mq_ipeek).
#
[error posting receive]
Unable to post application receive buffer (psm_mq_irecv or psm_mq_imrecv).
Error: %s
Buffer: %p
Length: %d
#
[path query mechanism unknown]
Unknown path record query mechanism %s. Supported mechanisms are %s.

463
ompi/mca/mtl/psm2/mtl_psm2.c Normal file
View File

@ -0,0 +1,463 @@
/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
/*
* Copyright (c) 2004-2006 The Trustees of Indiana University and Indiana
* University Research and Technology
* Corporation. All rights reserved.
* Copyright (c) 2004-2010 The University of Tennessee and The University
* of Tennessee Research Foundation. All rights
* reserved.
* Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
* University of Stuttgart. All rights reserved.
* Copyright (c) 2004-2006 The Regents of the University of California.
* All rights reserved.
* Copyright (c) 2006 QLogic Corporation. All rights reserved.
* Copyright (c) 2013-2015 Intel, Inc. All rights reserved
* Copyright (c) 2014 Los Alamos National Security, LLC. All rights
* reserved.
* $COPYRIGHT$
*
* Additional copyrights may follow
*
* $HEADER$
*/
#include "ompi_config.h"
#include "opal/mca/pmix/pmix.h"
#include "ompi/mca/mtl/mtl.h"
#include "ompi/mca/mtl/base/mtl_base_datatype.h"
#include "opal/util/show_help.h"
#include "ompi/proc/proc.h"
#include "mtl_psm2.h"
#include "mtl_psm2_types.h"
#include "mtl_psm2_endpoint.h"
#include "mtl_psm2_request.h"
mca_mtl_psm2_module_t ompi_mtl_psm2 = {
.super = {
/* NTH: PSM supports 16 bit context ids */
.mtl_max_contextid = (1UL << 16) - 1,
.mtl_max_tag = (1UL << 30), /* must allow negatives */
.mtl_add_procs = ompi_mtl_psm2_add_procs,
.mtl_del_procs = ompi_mtl_psm2_del_procs,
.mtl_finalize = ompi_mtl_psm2_finalize,
.mtl_send = ompi_mtl_psm2_send,
.mtl_isend = ompi_mtl_psm2_isend,
.mtl_irecv = ompi_mtl_psm2_irecv,
.mtl_iprobe = ompi_mtl_psm2_iprobe,
.mtl_imrecv = ompi_mtl_psm2_imrecv,
.mtl_improbe = ompi_mtl_psm2_improbe,
.mtl_cancel = ompi_mtl_psm2_cancel,
.mtl_add_comm = ompi_mtl_psm2_add_comm,
.mtl_del_comm = ompi_mtl_psm2_del_comm
}
};
static
psm_error_t
ompi_mtl_psm2_errhandler(psm_ep_t ep, const psm_error_t error,
const char *error_string, psm_error_token_t token)
{
switch (error) {
/* We don't want PSM to default to exiting when the following errors occur */
case PSM_EP_DEVICE_FAILURE:
case PSM_EP_NO_DEVICE:
case PSM_EP_NO_PORTS_AVAIL:
case PSM_EP_NO_NETWORK:
case PSM_EP_INVALID_UUID_KEY:
opal_show_help("help-mtl-psm.txt",
"unable to open endpoint", true,
psm_error_get_string(error));
break;
/* We can't handle any other errors than the ones above */
default:
opal_output(0, "Open MPI detected an unexpected PSM error in opening "
"an endpoint: %s\n", error_string);
return psm_error_defer(token);
break;
}
return error;
}
int ompi_mtl_psm2_progress( void );
int ompi_mtl_psm2_module_init(int local_rank, int num_local_procs) {
psm_error_t err;
psm_ep_t ep; /* endpoint handle */
psm_mq_t mq;
psm_epid_t epid; /* unique lid+port identifier */
psm_uuid_t unique_job_key;
struct psm_ep_open_opts ep_opt;
unsigned long long *uu = (unsigned long long *) unique_job_key;
char *generated_key;
char env_string[256];
int rc;
generated_key = getenv("OMPI_MCA_orte_precondition_transports");
memset(uu, 0, sizeof(psm_uuid_t));
if (!generated_key || (strlen(generated_key) != 33) ||
sscanf(generated_key, "%016llx-%016llx", &uu[0], &uu[1]) != 2)
{
opal_show_help("help-mtl-psm.txt",
"no uuid present", true,
generated_key ? "could not be parsed from" :
"not present in", ompi_process_info.nodename);
return OMPI_ERROR;
}
/* Handle our own errors for opening endpoints */
psm_error_register_handler(ompi_mtl_psm2.ep, ompi_mtl_psm2_errhandler);
/* Setup MPI_LOCALRANKID and MPI_LOCALNRANKS so PSM can allocate hardware
* contexts correctly.
*/
snprintf(env_string, sizeof(env_string), "%d", local_rank);
setenv("MPI_LOCALRANKID", env_string, 0);
snprintf(env_string, sizeof(env_string), "%d", num_local_procs);
setenv("MPI_LOCALNRANKS", env_string, 0);
/* Setup the endpoint options. */
psm_ep_open_opts_get_defaults(&ep_opt);
ep_opt.timeout = ompi_mtl_psm2.connect_timeout * 1e9;
ep_opt.affinity = PSM_EP_OPEN_AFFINITY_SKIP; /* do not let PSM set affinity */
/* Open PSM endpoint */
err = psm_ep_open(unique_job_key, &ep_opt, &ep, &epid);
if (err) {
opal_show_help("help-mtl-psm.txt",
"unable to open endpoint", true,
psm_error_get_string(err));
return OMPI_ERROR;
}
/* Future errors are handled by the default error handler */
psm_error_register_handler(ompi_mtl_psm2.ep, PSM_ERRHANDLER_DEFAULT);
err = psm_mq_init(ep,
0xffff000000000000ULL,
NULL,
0,
&mq);
if (err) {
opal_show_help("help-mtl-psm.txt",
"psm init", true,
psm_error_get_string(err));
return OMPI_ERROR;
}
ompi_mtl_psm2.ep = ep;
ompi_mtl_psm2.epid = epid;
ompi_mtl_psm2.mq = mq;
OPAL_MODEX_SEND(rc, PMIX_SYNC_REQD, PMIX_GLOBAL,
&mca_mtl_psm2_component.super.mtl_version,
&ompi_mtl_psm2.epid,
sizeof(psm_epid_t));
if (OMPI_SUCCESS != rc) {
opal_output(0, "Open MPI couldn't send PSM2 epid to head node process");
return OMPI_ERROR;
}
/* register the psm progress function */
opal_progress_register(ompi_mtl_psm2_progress);
return OMPI_SUCCESS;
}
int
ompi_mtl_psm2_finalize(struct mca_mtl_base_module_t* mtl) {
psm_error_t err;
opal_progress_unregister(ompi_mtl_psm2_progress);
/* free resources */
err = psm_mq_finalize(ompi_mtl_psm2.mq);
if (err) {
opal_output(0, "Error in psm_mq_finalize (error %s)\n",
psm_error_get_string(err));
return OMPI_ERROR;
}
err = psm_ep_close(ompi_mtl_psm2.ep, PSM_EP_CLOSE_GRACEFUL, 1*1e9);
if (err) {
opal_output(0, "Error in psm_ep_close (error %s)\n",
psm_error_get_string(err));
return OMPI_ERROR;
}
err = psm_finalize();
if (err) {
opal_output(0, "Error in psm_finalize (error %s)\n",
psm_error_get_string(err));
return OMPI_ERROR;
}
return OMPI_SUCCESS;
}
static
const char *
ompi_mtl_psm2_connect_error_msg(psm_error_t err)
{
switch (err) { /* See if we expect the error */
case PSM_EPID_UNREACHABLE:
case PSM_EPID_INVALID_NODE:
case PSM_EPID_INVALID_MTU:
case PSM_EPID_INVALID_UUID_KEY:
case PSM_EPID_INVALID_VERSION:
case PSM_EPID_INVALID_CONNECT:
return psm_error_get_string(err);
break;
case PSM_EPID_UNKNOWN:
return "Connect status could not be determined "
"because of other errors";
default:
return NULL;
}
}
#ifndef min
# define min(a,b) ((a) < (b) ? (a) : (b))
#endif
#ifndef max
# define max(a,b) ((a) > (b) ? (a) : (b))
#endif
int
ompi_mtl_psm2_add_procs(struct mca_mtl_base_module_t *mtl,
size_t nprocs,
struct ompi_proc_t** procs)
{
int i,j;
int rc;
psm_epid_t *epids_in = NULL;
int *mask_in = NULL;
psm_epid_t *epid;
psm_epaddr_t *epaddrs_out = NULL;
psm_error_t *errs_out = NULL, err;
size_t size;
int proc_errors[PSM_ERROR_LAST] = { 0 };
int timeout_in_secs;
assert(mtl == &ompi_mtl_psm2.super);
rc = OMPI_ERR_OUT_OF_RESOURCE;
errs_out = (psm_error_t *) malloc(nprocs * sizeof(psm_error_t));
if (errs_out == NULL) {
goto bail;
}
epids_in = (psm_epid_t *) malloc(nprocs * sizeof(psm_epid_t));
if (epids_in == NULL) {
goto bail;
}
mask_in = (int *) malloc(nprocs * sizeof(int));
if (mask_in == NULL) {
goto bail;
}
epaddrs_out = (psm_epaddr_t *) malloc(nprocs * sizeof(psm_epaddr_t));
if (epaddrs_out == NULL) {
goto bail;
}
rc = OMPI_SUCCESS;
/* Get the epids for all the processes from modex */
for (i = 0; i < (int) nprocs; i++) {
if (NULL != procs[i]->proc_endpoints[OMPI_PROC_ENDPOINT_TAG_MTL]) {
/* Already connected: don't connect again */
mask_in[i] = 0;
continue;
}
OPAL_MODEX_RECV(rc, &mca_mtl_psm2_component.super.mtl_version,
&procs[i]->super, (void**)&epid, &size);
if (rc != OMPI_SUCCESS || size != sizeof(psm_epid_t)) {
return OMPI_ERROR;
}
epids_in[i] = *epid;
mask_in[i] = 1;
}
timeout_in_secs = max(ompi_mtl_psm2.connect_timeout, 0.5 * nprocs);
psm_error_register_handler(ompi_mtl_psm2.ep, PSM_ERRHANDLER_NOP);
err = psm_ep_connect(ompi_mtl_psm2.ep,
nprocs,
epids_in,
mask_in,
errs_out,
epaddrs_out,
timeout_in_secs * 1e9);
if (err) {
char *errstr = (char *) ompi_mtl_psm2_connect_error_msg(err);
if (errstr == NULL) {
opal_output(0, "PSM returned unhandled/unknown connect error: %s\n",
psm_error_get_string(err));
}
for (i = 0; i < (int) nprocs; i++) {
if (0 == mask_in[i]) {
continue;
}
psm_error_t thiserr = errs_out[i];
errstr = (char *) ompi_mtl_psm2_connect_error_msg(thiserr);
if (proc_errors[thiserr] == 0) {
proc_errors[thiserr] = 1;
opal_output(0, "PSM EP connect error (%s):",
errstr ? errstr : "unknown connect error");
for (j = 0; j < (int) nprocs; j++) {
if (errs_out[j] == thiserr) {
opal_output(0, " %s", (NULL == procs[j]->super.proc_hostname) ?
"unknown" : procs[j]->super.proc_hostname);
}
}
opal_output(0, "\n");
}
}
rc = OMPI_ERROR;
}
else {
/* Default error handling is enabled, errors will not be returned to
* user. PSM prints the error and the offending endpoint's hostname
* and exits with -1 */
psm_error_register_handler(ompi_mtl_psm2.ep, PSM_ERRHANDLER_DEFAULT);
/* Fill in endpoint data */
for (i = 0; i < (int) nprocs; i++) {
if (0 == mask_in[i]) {
continue;
}
mca_mtl_psm2_endpoint_t *endpoint =
(mca_mtl_psm2_endpoint_t *) OBJ_NEW(mca_mtl_psm2_endpoint_t);
endpoint->peer_epid = epids_in[i];
endpoint->peer_addr = epaddrs_out[i];
procs[i]->proc_endpoints[OMPI_PROC_ENDPOINT_TAG_MTL] = endpoint;
}
rc = OMPI_SUCCESS;
}
bail:
if (epids_in != NULL) {
free(epids_in);
}
if (mask_in != NULL) {
free(mask_in);
}
if (errs_out != NULL) {
free(errs_out);
}
if (epaddrs_out != NULL) {
free(epaddrs_out);
}
return rc;
}
int
ompi_mtl_psm2_del_procs(struct mca_mtl_base_module_t *mtl,
size_t nprocs,
struct ompi_proc_t** procs)
{
return OMPI_SUCCESS;
}
int
ompi_mtl_psm2_add_comm(struct mca_mtl_base_module_t *mtl,
struct ompi_communicator_t *comm)
{
return OMPI_SUCCESS;
}
int
ompi_mtl_psm2_del_comm(struct mca_mtl_base_module_t *mtl,
struct ompi_communicator_t *comm)
{
return OMPI_SUCCESS;
}
int ompi_mtl_psm2_progress( void ) {
psm_error_t err;
mca_mtl_psm2_request_t* mtl_psm2_request;
psm_mq_status2_t psm_status;
psm_mq_req_t req;
int completed = 1;
do {
err = psm_mq_ipeek2(ompi_mtl_psm2.mq, &req, NULL);
if (err == PSM_MQ_INCOMPLETE) {
return completed;
} else if (err != PSM_OK) {
goto error;
}
completed++;
err = psm_mq_test2(&req, &psm_status);
if (err != PSM_OK) {
goto error;
}
mtl_psm2_request = (mca_mtl_psm2_request_t*) psm_status.context;
if (mtl_psm2_request->type == OMPI_mtl_psm2_IRECV) {
mtl_psm2_request->super.ompi_req->req_status.MPI_SOURCE =
psm_status.msg_tag.tag2;
mtl_psm2_request->super.ompi_req->req_status.MPI_TAG =
psm_status.msg_tag.tag1;
mtl_psm2_request->super.ompi_req->req_status._ucount =
psm_status.nbytes;
ompi_mtl_datatype_unpack(mtl_psm2_request->convertor,
mtl_psm2_request->buf,
psm_status.msg_length);
}
if(mtl_psm2_request->type == OMPI_mtl_psm2_ISEND) {
if (mtl_psm2_request->free_after) {
free(mtl_psm2_request->buf);
}
}
switch (psm_status.error_code) {
case PSM_OK:
mtl_psm2_request->super.ompi_req->req_status.MPI_ERROR =
OMPI_SUCCESS;
break;
case PSM_MQ_TRUNCATION:
mtl_psm2_request->super.ompi_req->req_status.MPI_ERROR =
MPI_ERR_TRUNCATE;
break;
default:
mtl_psm2_request->super.ompi_req->req_status.MPI_ERROR =
MPI_ERR_INTERN;
}
mtl_psm2_request->super.completion_callback(&mtl_psm2_request->super);
}
while (1);
error:
opal_show_help("help-mtl-psm.txt",
"error polling network", true,
psm_error_get_string(err));
return 1;
}

106
ompi/mca/mtl/psm2/mtl_psm2.h Normal file
View File

@ -0,0 +1,106 @@
/*
* Copyright (c) 2004-2006 The Trustees of Indiana University and Indiana
* University Research and Technology
* Corporation. All rights reserved.
* Copyright (c) 2004-2005 The University of Tennessee and The University
* of Tennessee Research Foundation. All rights
* reserved.
* Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
* University of Stuttgart. All rights reserved.
* Copyright (c) 2004-2006 The Regents of the University of California.
* All rights reserved.
* Copyright (c) 2006 QLogic Corporation. All rights reserved.
* Copyright (c) 2015 Intel, Inc. All rights reserved
* $COPYRIGHT$
*
* Additional copyrights may follow
*
* $HEADER$
*/
#ifndef MTL_PSM2_H_HAS_BEEN_INCLUDED
#define MTL_PSM2_H_HAS_BEEN_INCLUDED
#include "ompi/mca/pml/pml.h"
#include "ompi/mca/mtl/mtl.h"
#include "ompi/mca/mtl/base/base.h"
#include "opal/datatype/opal_convertor.h"
#include <psm2.h>
#include <psm2_mq.h>
BEGIN_C_DECLS
/* MTL interface functions */
extern int ompi_mtl_psm2_add_procs(struct mca_mtl_base_module_t* mtl,
size_t nprocs,
struct ompi_proc_t** procs);
extern int ompi_mtl_psm2_del_procs(struct mca_mtl_base_module_t* mtl,
size_t nprocs,
struct ompi_proc_t** procs);
int
ompi_mtl_psm2_send(struct mca_mtl_base_module_t* mtl,
struct ompi_communicator_t* comm,
int dest,
int tag,
struct opal_convertor_t *convertor,
mca_pml_base_send_mode_t mode);
extern int ompi_mtl_psm2_isend(struct mca_mtl_base_module_t* mtl,
struct ompi_communicator_t* comm,
int dest,
int tag,
struct opal_convertor_t *convertor,
mca_pml_base_send_mode_t mode,
bool blocking,
mca_mtl_request_t * mtl_request);
extern int ompi_mtl_psm2_irecv(struct mca_mtl_base_module_t* mtl,
struct ompi_communicator_t *comm,
int src,
int tag,
struct opal_convertor_t *convertor,
struct mca_mtl_request_t *mtl_request);
extern int ompi_mtl_psm2_iprobe(struct mca_mtl_base_module_t* mtl,
struct ompi_communicator_t *comm,
int src,
int tag,
int *flag,
struct ompi_status_public_t *status);
extern int ompi_mtl_psm2_imrecv(struct mca_mtl_base_module_t* mtl,
struct opal_convertor_t *convertor,
struct ompi_message_t **message,
struct mca_mtl_request_t *mtl_request);
extern int ompi_mtl_psm2_improbe(struct mca_mtl_base_module_t *mtl,
struct ompi_communicator_t *comm,
int src,
int tag,
int *matched,
struct ompi_message_t **message,
struct ompi_status_public_t *status);
extern int ompi_mtl_psm2_cancel(struct mca_mtl_base_module_t* mtl,
struct mca_mtl_request_t *mtl_request,
int flag);
extern int ompi_mtl_psm2_add_comm(struct mca_mtl_base_module_t *mtl,
struct ompi_communicator_t *comm);
extern int ompi_mtl_psm2_del_comm(struct mca_mtl_base_module_t *mtl,
struct ompi_communicator_t *comm);
extern int ompi_mtl_psm2_finalize(struct mca_mtl_base_module_t* mtl);
int ompi_mtl_psm2_module_init(int local_rank, int num_local_procs);
END_C_DECLS
#endif /* MTL_PSM2_H_HAS_BEEN_INCLUDED */

55
ompi/mca/mtl/psm2/mtl_psm2_cancel.c Normal file
View File

@ -0,0 +1,55 @@
/*
* Copyright (c) 2004-2006 The Trustees of Indiana University and Indiana
* University Research and Technology
* Corporation. All rights reserved.
* Copyright (c) 2004-2005 The University of Tennessee and The University
* of Tennessee Research Foundation. All rights
* reserved.
* Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
* University of Stuttgart. All rights reserved.
* Copyright (c) 2004-2006 The Regents of the University of California.
* All rights reserved.
* Copyright (c) 2006 QLogic Corporation. All rights reserved.
* Copyright (c) 2015 Intel, Inc. All rights reserved
* $COPYRIGHT$
*
* Additional copyrights may follow
*
* $HEADER$
*/
#include "ompi_config.h"
#include "mtl_psm2.h"
#include "mtl_psm2_request.h"
int ompi_mtl_psm2_cancel(struct mca_mtl_base_module_t* mtl,
struct mca_mtl_request_t *mtl_request,
int flag) {
psm_error_t err;
psm_mq_status_t status;
mca_mtl_psm2_request_t *mtl_psm2_request =
(mca_mtl_psm2_request_t*) mtl_request;
/* PSM does not support canceling sends */
if(OMPI_mtl_psm2_ISEND == mtl_psm2_request->type) {
return OMPI_SUCCESS;
}
err = psm_mq_cancel(&mtl_psm2_request->psm_request);
if(PSM_OK == err) {
err = psm_mq_test(&mtl_psm2_request->psm_request, &status);
if(PSM_OK == err) {
mtl_request->ompi_req->req_status._cancelled = true;
mtl_psm2_request->super.completion_callback(&mtl_psm2_request->super);
return OMPI_SUCCESS;
} else {
return OMPI_ERROR;
}
} else if(PSM_MQ_INCOMPLETE == err) {
return OMPI_SUCCESS;
}
return OMPI_ERROR;
}

219
ompi/mca/mtl/psm2/mtl_psm2_component.c Normal file
View File

@ -0,0 +1,219 @@
/*
* Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
* University Research and Technology
* Corporation. All rights reserved.
* Copyright (c) 2004-2005 The University of Tennessee and The University
* of Tennessee Research Foundation. All rights
* reserved.
* Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
* University of Stuttgart. All rights reserved.
* Copyright (c) 2004-2005 The Regents of the University of California.
* All rights reserved.
* Copyright (c) 2006-2010 QLogic Corporation. All rights reserved.
* Copyright (c) 2012-2013 Los Alamos National Security, LLC.
* All rights reserved.
* Copyright (c) 2013-2015 Intel, Inc. All rights reserved
* $COPYRIGHT$
*
* Additional copyrights may follow
*
* $HEADER$
*/
#include "ompi_config.h"
#include "opal/mca/event/event.h"
#include "opal/util/output.h"
#include "opal/util/show_help.h"
#include "ompi/proc/proc.h"
#include "mtl_psm2.h"
#include "mtl_psm2_types.h"
#include "mtl_psm2_request.h"
#include "psm.h"
#include <sys/types.h>
#include <sys/stat.h>
#include <unistd.h>
static int param_priority;
static int ompi_mtl_psm2_component_open(void);
static int ompi_mtl_psm2_component_close(void);
static int ompi_mtl_psm2_component_query(mca_base_module_t **module, int *priority);
static int ompi_mtl_psm2_component_register(void);
static mca_mtl_base_module_t* ompi_mtl_psm2_component_init( bool enable_progress_threads,
bool enable_mpi_threads );
mca_mtl_psm2_component_t mca_mtl_psm2_component = {
{
/* First, the mca_base_component_t struct containing meta
* information about the component itself */
.mtl_version = {
MCA_MTL_BASE_VERSION_2_0_0,
.mca_component_name = "psm2",
MCA_BASE_MAKE_VERSION(component, OMPI_MAJOR_VERSION, OMPI_MINOR_VERSION,
OMPI_RELEASE_VERSION),
.mca_open_component = ompi_mtl_psm2_component_open,
.mca_close_component = ompi_mtl_psm2_component_close,
.mca_query_component = ompi_mtl_psm2_component_query,
.mca_register_component_params = ompi_mtl_psm2_component_register,
},
.mtl_data = {
/* The component is not checkpoint ready */
MCA_BASE_METADATA_PARAM_NONE
},
.mtl_init = ompi_mtl_psm2_component_init,
}
};
static int
ompi_mtl_psm2_component_register(void)
{
ompi_mtl_psm2.connect_timeout = 180;
(void) mca_base_component_var_register(&mca_mtl_psm2_component.super.mtl_version,
"connect_timeout",
"PSM connection timeout value in seconds",
MCA_BASE_VAR_TYPE_INT, NULL, 0, 0,
OPAL_INFO_LVL_9,
MCA_BASE_VAR_SCOPE_READONLY,
&ompi_mtl_psm2.connect_timeout);
param_priority = 120;
(void) mca_base_component_var_register (&mca_mtl_psm2_component.super.mtl_version,
"priority", "Priority of the PSM2 MTL component",
MCA_BASE_VAR_TYPE_INT, NULL, 0, 0,
OPAL_INFO_LVL_9,
MCA_BASE_VAR_SCOPE_READONLY,
&param_priority);
return OMPI_SUCCESS;
}
static int
ompi_mtl_psm2_component_open(void)
{
struct stat st;
/* Component available only if Omni-Path hardware is present */
if (0 == stat("/dev/hfi1", &st)) {
return OMPI_SUCCESS;
}
else {
return OPAL_ERR_NOT_AVAILABLE;
}
}
static int
ompi_mtl_psm2_component_query(mca_base_module_t **module, int *priority)
{
/*
* if we get here it means that PSM is available so give high priority
*/
*priority = param_priority;
*module = (mca_base_module_t *)&ompi_mtl_psm2.super;
return OMPI_SUCCESS;
}
static int
ompi_mtl_psm2_component_close(void)
{
return OMPI_SUCCESS;
}
static int
get_num_total_procs(int *out_ntp)
{
*out_ntp = (int)ompi_process_info.num_procs;
return OMPI_SUCCESS;
}
static int
get_num_local_procs(int *out_nlp)
{
/* num_local_peers does not include us in
* its calculation, so adjust for that */
*out_nlp = (int)(1 + ompi_process_info.num_local_peers);
return OMPI_SUCCESS;
}
static int
get_local_rank(int *out_rank)
{
ompi_node_rank_t my_node_rank;
*out_rank = 0;
if (OMPI_NODE_RANK_INVALID == (my_node_rank =
ompi_process_info.my_node_rank)) {
return OMPI_ERROR;
}
*out_rank = (int)my_node_rank;
return OMPI_SUCCESS;
}
static mca_mtl_base_module_t *
ompi_mtl_psm2_component_init(bool enable_progress_threads,
bool enable_mpi_threads)
{
psm_error_t err;
int verno_major = PSM_VERNO_MAJOR;
int verno_minor = PSM_VERNO_MINOR;
int local_rank = -1, num_local_procs = 0;
int num_total_procs = 0;
/* Compute the total number of processes on this host and our local rank
* on that node. We need to provide PSM with these values so it can
* allocate hardware contexts appropriately across processes.
*/
if (OMPI_SUCCESS != get_num_local_procs(&num_local_procs)) {
opal_output(0, "Cannot determine number of local processes. "
"Cannot continue.\n");
return NULL;
}
if (OMPI_SUCCESS != get_local_rank(&local_rank)) {
opal_output(0, "Cannot determine local rank. Cannot continue.\n");
return NULL;
}
if (OMPI_SUCCESS != get_num_total_procs(&num_total_procs)) {
opal_output(0, "Cannot determine total number of processes. "
"Cannot continue.\n");
return NULL;
}
err = psm_error_register_handler(NULL /* no ep */,
PSM_ERRHANDLER_NOP);
if (err) {
opal_output(0, "Error in psm_error_register_handler (error %s)\n",
psm_error_get_string(err));
return NULL;
}
if (num_local_procs == num_total_procs) {
setenv("PSM_DEVICES", "self,shm", 0);
}
err = psm_init(&verno_major, &verno_minor);
if (err) {
opal_show_help("help-mtl-psm.txt",
"psm init", true,
psm_error_get_string(err));
return NULL;
}
/* Complete PSM initialization */
ompi_mtl_psm2_module_init(local_rank, num_local_procs);
ompi_mtl_psm2.super.mtl_request_size =
sizeof(mca_mtl_psm2_request_t) -
sizeof(struct mca_mtl_request_t);
return &ompi_mtl_psm2.super;
}

54
ompi/mca/mtl/psm2/mtl_psm2_endpoint.c Normal file
View File

@ -0,0 +1,54 @@
/*
* Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
* University Research and Technology
* Corporation. All rights reserved.
* Copyright (c) 2004-2005 The University of Tennessee and The University
* of Tennessee Research Foundation. All rights
* reserved.
* Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
* University of Stuttgart. All rights reserved.
* Copyright (c) 2004-2006 The Regents of the University of California.
* All rights reserved.
* Copyright (c) 2006 QLogic Corporation. All rights reserved.
* Copyright (c) 2015 Intel, Inc. All rights reserved
* $COPYRIGHT$
*
* Additional copyrights may follow
*
* $HEADER$
*/
#include "ompi_config.h"
#include <sys/time.h>
#include <time.h>
#include "ompi/types.h"
#include "mtl_psm2.h"
#include "mtl_psm2_types.h"
#include "mtl_psm2_endpoint.h"
/*
* Initialize state of the endpoint instance.
*
*/
static void mca_mtl_psm2_endpoint_construct(mca_mtl_psm2_endpoint_t* endpoint)
{
endpoint->mtl_psm2_module = NULL;
}
/*
* Destroy a endpoint
*
*/
static void mca_mtl_psm2_endpoint_destruct(mca_mtl_psm2_endpoint_t* endpoint)
{
}
OBJ_CLASS_INSTANCE(
mca_mtl_psm2_endpoint_t,
opal_list_item_t,
mca_mtl_psm2_endpoint_construct,
mca_mtl_psm2_endpoint_destruct);

59
ompi/mca/mtl/psm2/mtl_psm2_endpoint.h Normal file
View File

@ -0,0 +1,59 @@
/*
* Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
* University Research and Technology
* Corporation. All rights reserved.
* Copyright (c) 2004-2005 The University of Tennessee and The University
* of Tennessee Research Foundation. All rights
* reserved.
* Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
* University of Stuttgart. All rights reserved.
* Copyright (c) 2004-2006 The Regents of the University of California.
* All rights reserved.
* Copyright (c) 2006 QLogic Corporation. All rights reserved.
* Copyright (c) 2015 Intel, Inc. All rights reserved
* $COPYRIGHT$
*
* Additional copyrights may follow
*
* $HEADER$
*/
#ifndef MCA_MTL_PSM2_ENDPOINT_H
#define MCA_MTL_PSM2_ENDPOINT_H
#include "opal/class/opal_list.h"
#include "opal/mca/event/event.h"
#include "ompi/mca/mtl/mtl.h"
#include "mtl_psm2.h"
#include "psm2.h"
BEGIN_C_DECLS
OBJ_CLASS_DECLARATION(mca_mtl_psm2_endpoint_t);
/**
* An abstraction that represents a connection to a endpoint process.
* An instance of mca_mtl_psm2_endpoint_t is associated w/ each process
* and MTL pair at startup. However, connections to the endpoint
* are established dynamically on an as-needed basis:
*/
struct mca_mtl_psm2_endpoint_t {
opal_list_item_t super;
struct mca_mtl_psm2_module_t* mtl_psm2_module;
/**< MTL instance that created this connection */
psm_epid_t peer_epid;
/**< The unique epid for the opened port */
psm_epaddr_t peer_addr;
/**< The connected endpoint handle*/
};
typedef struct mca_mtl_psm2_endpoint_t mca_mtl_psm2_endpoint_t;
OBJ_CLASS_DECLARATION(mca_mtl_psm2_endpoint);
END_C_DECLS
#endif

134
ompi/mca/mtl/psm2/mtl_psm2_probe.c Normal file
View File

@ -0,0 +1,134 @@
/*
* Copyright (c) 2004-2006 The Trustees of Indiana University and Indiana
* University Research and Technology
* Corporation. All rights reserved.
* Copyright (c) 2004-2010 The University of Tennessee and The University
* of Tennessee Research Foundation. All rights
* reserved.
* Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
* University of Stuttgart. All rights reserved.
* Copyright (c) 2004-2006 The Regents of the University of California.
* All rights reserved.
* Copyright (c) 2006 QLogic Corporation. All rights reserved.
* Copyright (c) 2006-2007 Los Alamos National Security, LLC. All rights
* reserved.
* Copyright (c) 2013-2015 Intel, Inc. All rights reserved
* $COPYRIGHT$
*
* Additional copyrights may follow
*
* $HEADER$
*/
#include "ompi_config.h"
#include "mtl_psm2.h"
#include "mtl_psm2_types.h"
#include "psm2.h"
#include "ompi/communicator/communicator.h"
#include "ompi/message/message.h"
int ompi_mtl_psm2_iprobe(struct mca_mtl_base_module_t* mtl,
struct ompi_communicator_t *comm,
int src,
int tag,
int *flag,
struct ompi_status_public_t *status)
{
psm_mq_tag_t mqtag, tagsel;
psm_mq_status2_t mqstat;
psm_error_t err;
PSM_MAKE_TAGSEL(src, tag, comm->c_contextid, mqtag, tagsel);
err = psm_mq_iprobe2(ompi_mtl_psm2.mq,
PSM_MQ_ANY_ADDR, &mqtag, &tagsel, &mqstat);
if (err == PSM_OK) {
*flag = 1;
if(MPI_STATUS_IGNORE != status) {
status->MPI_SOURCE = mqstat.msg_tag.tag2;
status->MPI_TAG = mqstat.msg_tag.tag1;
status->_ucount = mqstat.nbytes;
switch (mqstat.error_code) {
case PSM_OK:
status->MPI_ERROR = OMPI_SUCCESS;
break;
case PSM_MQ_TRUNCATION:
status->MPI_ERROR = MPI_ERR_TRUNCATE;
break;
default:
status->MPI_ERROR = MPI_ERR_INTERN;
}
}
return OMPI_SUCCESS;
}
else if (err == PSM_MQ_INCOMPLETE) {
*flag = 0;
return OMPI_SUCCESS;
}
else
return OMPI_ERROR;
}
int
ompi_mtl_psm2_improbe(struct mca_mtl_base_module_t *mtl,
struct ompi_communicator_t *comm,
int src,
int tag,
int *matched,
struct ompi_message_t **message,
struct ompi_status_public_t *status)
{
struct ompi_message_t* msg;
psm_mq_tag_t mqtag, tagsel;
psm_mq_status2_t mqstat;
psm_mq_req_t mqreq;
psm_error_t err;
PSM_MAKE_TAGSEL(src, tag, comm->c_contextid, mqtag, tagsel);
err = psm_mq_improbe2(ompi_mtl_psm2.mq,
PSM_MQ_ANY_ADDR, &mqtag, &tagsel, &mqreq, &mqstat);
if (err == PSM_OK) {
if(MPI_STATUS_IGNORE != status) {
status->MPI_SOURCE = mqstat.msg_tag.tag2;
status->MPI_TAG = mqstat.msg_tag.tag1;
status->_ucount = mqstat.nbytes;
switch (mqstat.error_code) {
case PSM_OK:
status->MPI_ERROR = OMPI_SUCCESS;
break;
case PSM_MQ_TRUNCATION:
status->MPI_ERROR = MPI_ERR_TRUNCATE;
break;
default:
status->MPI_ERROR = MPI_ERR_INTERN;
}
}
msg = ompi_message_alloc();
if(NULL == msg) {
return OMPI_ERR_OUT_OF_RESOURCE;
}
msg->comm = comm;
msg->req_ptr = mqreq;
msg->peer = mqstat.msg_tag.tag2;
msg->count = mqstat.nbytes;
*message = msg;
*matched = 1;
return OMPI_SUCCESS;
} else if(err == PSM_MQ_INCOMPLETE) {
*matched = 0;
*message = MPI_MESSAGE_NULL;
return OMPI_SUCCESS;
} else {
return OMPI_ERROR;
}
}

124
ompi/mca/mtl/psm2/mtl_psm2_recv.c Normal file
View File

@ -0,0 +1,124 @@
/*
* Copyright (c) 2004-2006 The Trustees of Indiana University and Indiana
* University Research and Technology
* Corporation. All rights reserved.
* Copyright (c) 2004-2005 The University of Tennessee and The University
* of Tennessee Research Foundation. All rights
* reserved.
* Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
* University of Stuttgart. All rights reserved.
* Copyright (c) 2004-2006 The Regents of the University of California.
* All rights reserved.
* Copyright (c) 2006 QLogic Corporation. All rights reserved.
* Copyright (c) 2013-2015 Intel, Inc. All rights reserved
* $COPYRIGHT$
*
* Additional copyrights may follow
*
* $HEADER$
*/
#include "ompi_config.h"
#include "ompi/communicator/communicator.h"
#include "ompi/message/message.h"
#include "opal/datatype/opal_convertor.h"
#include "ompi/mca/mtl/base/mtl_base_datatype.h"
#include "opal/util/show_help.h"
#include "mtl_psm2.h"
#include "mtl_psm2_types.h"
#include "mtl_psm2_request.h"
int
ompi_mtl_psm2_irecv(struct mca_mtl_base_module_t* mtl,
struct ompi_communicator_t *comm,
int src,
int tag,
struct opal_convertor_t *convertor,
struct mca_mtl_request_t *mtl_request)
{
int ret;
psm_error_t err;
mca_mtl_psm2_request_t * mtl_psm2_request = (mca_mtl_psm2_request_t*) mtl_request;
psm_mq_tag_t mqtag;
psm_mq_tag_t tagsel;
size_t length;
ret = ompi_mtl_datatype_recv_buf(convertor,
&mtl_psm2_request->buf,
&length,
&mtl_psm2_request->free_after);
if (OMPI_SUCCESS != ret) return ret;
mtl_psm2_request->length = length;
mtl_psm2_request->convertor = convertor;
mtl_psm2_request->type = OMPI_mtl_psm2_IRECV;
PSM_MAKE_TAGSEL(src, tag, comm->c_contextid, mqtag, tagsel);
err = psm_mq_irecv2(ompi_mtl_psm2.mq,
PSM_MQ_ANY_ADDR,
&mqtag,
&tagsel,
0,
mtl_psm2_request->buf,
length,
mtl_psm2_request,
&mtl_psm2_request->psm_request);
if (err) {
opal_show_help("help-mtl-psm.txt",
"error posting receive", true,
psm_error_get_string(err),
mtl_psm2_request->buf, length);
return OMPI_ERROR;
}
return OMPI_SUCCESS;
}
int
ompi_mtl_psm2_imrecv(struct mca_mtl_base_module_t* mtl,
struct opal_convertor_t *convertor,
struct ompi_message_t **message,
struct mca_mtl_request_t *mtl_request)
{
mca_mtl_psm2_request_t *mtl_psm2_request =
(mca_mtl_psm2_request_t*) mtl_request;
size_t length;
psm_error_t err;
int ret;
mtl_psm2_request->psm_request =
(psm_mq_req_t)(*message)->req_ptr;
ret = ompi_mtl_datatype_recv_buf(convertor,
&mtl_psm2_request->buf,
&length,
&mtl_psm2_request->free_after);
if (OMPI_SUCCESS != ret) return ret;
mtl_psm2_request->length = length;
mtl_psm2_request->convertor = convertor;
mtl_psm2_request->type = OMPI_mtl_psm2_IRECV;
err = psm_mq_imrecv(ompi_mtl_psm2.mq, 0,
mtl_psm2_request->buf, length, mtl_psm2_request,
&mtl_psm2_request->psm_request);
if(err) {
opal_show_help("help-mtl-psm.txt",
"error posting receive", true,
psm_error_get_string(err),
mtl_psm2_request->buf, length);
return OMPI_ERROR;
}
*message = MPI_MESSAGE_NULL;
return OMPI_SUCCESS;
}

44
ompi/mca/mtl/psm2/mtl_psm2_request.h Normal file
View File

@ -0,0 +1,44 @@
/*
* Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
* University Research and Technology
* Corporation. All rights reserved.
* Copyright (c) 2004-2005 The University of Tennessee and The University
* of Tennessee Research Foundation. All rights
* reserved.
* Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
* University of Stuttgart. All rights reserved.
* Copyright (c) 2004-2005 The Regents of the University of California.
* All rights reserved.
* Copyright (c) 2006 QLogic Corporation. All rights reserved.
* Copyright (c) 2015 Intel, Inc. All rights reserved
* $COPYRIGHT$
*
* Additional copyrights may follow
*
* $HEADER$
*/
#ifndef OMPI_MTL_PSM2_REQUEST_H
#define OMPI_MTL_PSM2_REQUEST_H
#include "opal/datatype/opal_convertor.h"
typedef enum {
OMPI_mtl_psm2_ISEND,
OMPI_mtl_psm2_IRECV
} mca_mtl_psm2_request_type_t;
struct mca_mtl_psm2_request_t {
struct mca_mtl_request_t super;
mca_mtl_psm2_request_type_t type;
psm_mq_req_t psm_request;
/* psm_segment_t psm_segment[1]; */
void *buf;
size_t length;
struct opal_convertor_t *convertor;
bool free_after;
};
typedef struct mca_mtl_psm2_request_t mca_mtl_psm2_request_t;
#endif

129
ompi/mca/mtl/psm2/mtl_psm2_send.c Normal file
View File

@ -0,0 +1,129 @@
/*
* Copyright (c) 2004-2006 The Trustees of Indiana University and Indiana
* University Research and Technology
* Corporation. All rights reserved.
* Copyright (c) 2004-2005 The University of Tennessee and The University
* of Tennessee Research Foundation. All rights
* reserved.
* Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
* University of Stuttgart. All rights reserved.
* Copyright (c) 2004-2006 The Regents of the University of California.
* All rights reserved.
* Copyright (c) 2006 QLogic Corporation. All rights reserved.
* Copyright (c) 2013-2015 Intel, Inc. All rights reserved
* $COPYRIGHT$
*
* Additional copyrights may follow
*
* $HEADER$
*/
#include "ompi_config.h"
#include "ompi/mca/pml/pml.h"
#include "ompi/communicator/communicator.h"
#include "opal/datatype/opal_convertor.h"
#include "mtl_psm2.h"
#include "mtl_psm2_types.h"
#include "mtl_psm2_request.h"
#include "ompi/mca/mtl/base/mtl_base_datatype.h"
int
ompi_mtl_psm2_send(struct mca_mtl_base_module_t* mtl,
struct ompi_communicator_t* comm,
int dest,
int tag,
struct opal_convertor_t *convertor,
mca_pml_base_send_mode_t mode)
{
psm_error_t err;
mca_mtl_psm2_request_t mtl_psm2_request;
psm_mq_tag_t mqtag;
uint32_t flags = 0;
int ret;
size_t length;
ompi_proc_t* ompi_proc = ompi_comm_peer_lookup( comm, dest );
mca_mtl_psm2_endpoint_t* psm_endpoint = (mca_mtl_psm2_endpoint_t*) ompi_proc->proc_endpoints[OMPI_PROC_ENDPOINT_TAG_MTL];
assert(mtl == &ompi_mtl_psm2.super);
PSM_MAKE_MQTAG(comm->c_contextid, comm->c_my_rank, tag, mqtag);
ret = ompi_mtl_datatype_pack(convertor,
&mtl_psm2_request.buf,
&length,
&mtl_psm2_request.free_after);
mtl_psm2_request.length = length;
mtl_psm2_request.convertor = convertor;
mtl_psm2_request.type = OMPI_mtl_psm2_ISEND;
if (OMPI_SUCCESS != ret) return ret;
if (mode == MCA_PML_BASE_SEND_SYNCHRONOUS)
flags |= PSM_MQ_FLAG_SENDSYNC;
err = psm_mq_send2(ompi_mtl_psm2.mq,
psm_endpoint->peer_addr,
flags,
&mqtag,
mtl_psm2_request.buf,
length);
if (mtl_psm2_request.free_after) {
free(mtl_psm2_request.buf);
}
return err == PSM_OK ? OMPI_SUCCESS : OMPI_ERROR;
}
int
ompi_mtl_psm2_isend(struct mca_mtl_base_module_t* mtl,
struct ompi_communicator_t* comm,
int dest,
int tag,
struct opal_convertor_t *convertor,
mca_pml_base_send_mode_t mode,
bool blocking,
mca_mtl_request_t * mtl_request)
{
psm_error_t psm_error;
psm_mq_tag_t mqtag;
uint32_t flags = 0;
int ret;
mca_mtl_psm2_request_t * mtl_psm2_request = (mca_mtl_psm2_request_t*) mtl_request;
size_t length;
ompi_proc_t* ompi_proc = ompi_comm_peer_lookup( comm, dest );
mca_mtl_psm2_endpoint_t* psm_endpoint = (mca_mtl_psm2_endpoint_t*)ompi_proc->proc_endpoints[OMPI_PROC_ENDPOINT_TAG_MTL];
assert(mtl == &ompi_mtl_psm2.super);
PSM_MAKE_MQTAG(comm->c_contextid, comm->c_my_rank, tag, mqtag);
ret = ompi_mtl_datatype_pack(convertor,
&mtl_psm2_request->buf,
&length,
&mtl_psm2_request->free_after);
mtl_psm2_request->length= length;
mtl_psm2_request->convertor = convertor;
mtl_psm2_request->type = OMPI_mtl_psm2_ISEND;
if (OMPI_SUCCESS != ret) return ret;
if (mode == MCA_PML_BASE_SEND_SYNCHRONOUS)
flags |= PSM_MQ_FLAG_SENDSYNC;
psm_error = psm_mq_isend2(ompi_mtl_psm2.mq,
psm_endpoint->peer_addr,
flags,
&mqtag,
mtl_psm2_request->buf,
length,
mtl_psm2_request,
&mtl_psm2_request->psm_request);
return psm_error == PSM_OK ? OMPI_SUCCESS : OMPI_ERROR;
}

93
ompi/mca/mtl/psm2/mtl_psm2_types.h Executable file
View File

@ -0,0 +1,93 @@
/*
* Copyright (c) 2004-2006 The Trustees of Indiana University and Indiana
* University Research and Technology
* Corporation. All rights reserved.
* Copyright (c) 2004-2007 The University of Tennessee and The University
* of Tennessee Research Foundation. All rights
* reserved.
* Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
* University of Stuttgart. All rights reserved.
* Copyright (c) 2004-2006 The Regents of the University of California.
* All rights reserved.
* Copyright (c) 2006 QLogic Corporation. All rights reserved.
* Copyright (c) 2011 Los Alamos National Security, LLC.
* All rights reserved.
* Copyright (c) 2013-2015 Intel, Inc. All rights reserved
* $COPYRIGHT$
*
* Additional copyrights may follow
*
* $HEADER$
*/
#ifndef MTL_PSM2_TYPES_H_HAS_BEEN_INCLUDED
#define MTL_PSM2_TYPES_H_HAS_BEEN_INCLUDED
#include "ompi_config.h"
#include "mtl_psm2.h"
#include "ompi/communicator/communicator.h"
#include "ompi/mca/mtl/mtl.h"
#include "ompi/mca/mtl/base/base.h"
#include "mtl_psm2_endpoint.h"
#include "psm2.h"
BEGIN_C_DECLS
/**
* MTL Module Interface
*/
struct mca_mtl_psm2_module_t {
mca_mtl_base_module_t super; /**< base MTL interface */
int32_t connect_timeout;
psm_ep_t ep;
psm_mq_t mq;
psm_epid_t epid;
psm_epaddr_t epaddr;
};
typedef struct mca_mtl_psm2_module_t mca_mtl_psm2_module_t;
extern mca_mtl_psm2_module_t ompi_mtl_psm2;
struct mca_mtl_psm2_component_t {
mca_mtl_base_component_2_0_0_t super; /**< base MTL component */
};
typedef struct mca_mtl_psm2_component_t mca_mtl_psm2_component_t;
OMPI_DECLSPEC extern mca_mtl_psm2_component_t mca_mtl_psm2_component;
#define PSM_MAKE_MQTAG(ctxt,rank,utag,tag) \
do { \
(tag).tag0 = ctxt; \
(tag).tag1 = utag; \
(tag).tag2 = rank; \
} while (0)
#define PSM_MAKE_TAGSEL(user_rank, user_tag, user_ctxt, tag, _tagsel) \
do { \
(tag).tag0 = user_ctxt; \
(tag).tag1 = user_tag; \
(tag).tag2 = user_rank; \
(_tagsel).tag0 = 0xffffffffULL; \
(_tagsel).tag1 = 0xffffffffULL; \
(_tagsel).tag2 = 0xffffffffULL; \
if((user_tag) == MPI_ANY_TAG) \
{ \
(_tagsel).tag1 = 0x80000000ULL; \
(tag).tag1 = 0x00000000ULL; \
} \
if((user_rank) == MPI_ANY_SOURCE) \
{ \
(_tagsel).tag2 = 0x00000000ULL; \
} \
} while (0)
END_C_DECLS
#endif /* MTL_PSM2_TYPES_H_HAS_BEEN_INCLUDED */

1
ompi/mca/mtl/psm2/post_configure.sh Normal file
View File

@ -0,0 +1 @@
DIRECT_CALL_HEADER="ompi/mca/mtl/psm2/mtl_psm.h"

1
ompi/mca/pml/cm/pml_cm_component.c
View File

@ -156,6 +156,7 @@ mca_pml_cm_component_init(int* priority,
*priority = -1;
return NULL;
} else if((strcmp(ompi_mtl_base_selected_component->mtl_version.mca_component_name, "psm") == 0) ||
(strcmp(ompi_mtl_base_selected_component->mtl_version.mca_component_name, "psm2") == 0) ||
(strcmp(ompi_mtl_base_selected_component->mtl_version.mca_component_name, "mxm") == 0) ||
(strcmp(ompi_mtl_base_selected_component->mtl_version.mca_component_name, "ofi") == 0) ||
(strcmp(ompi_mtl_base_selected_component->mtl_version.mca_component_name, "portals4") == 0)) {