ports/openmpi
ports/openmpi
1
1
Форкнуть 0
Код Релизы Активность

Merge pull request #650 from afriedle-intel/afriedle-psm2-mtl

Просмотр исходного кода 
Add new PSM2 MTL.
Этот коммит содержится в:
Andrew Friedley 2015-06-22 13:00:25 -07:00
родитель 74cf969561 2c9be59b37
Коммит a5cfbdd7fa
17 изменённых файлов: 1707 добавлений и 0 удалений
Показать статистику Скачать Patch файл Скачать Diff файл Показать все файлы Свернуть все файлы

72
config/ompi_check_psm2.m4 Обычный файл
Просмотреть файл

@ -0,0 +1,72 @@
# -*- shell-script -*-
#
# Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
# University Research and Technology
# Corporation. All rights reserved.
# Copyright (c) 2004-2005 The University of Tennessee and The University
# of Tennessee Research Foundation. All rights
# reserved.
# Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
# University of Stuttgart. All rights reserved.
# Copyright (c) 2004-2006 The Regents of the University of California.
# All rights reserved.
# Copyright (c) 2006 QLogic Corp. All rights reserved.
# Copyright (c) 2009 Cisco Systems, Inc. All rights reserved.
# Copyright (c) 2014 Intel Corporation. All rights reserved.
# $COPYRIGHT$
#
# Additional copyrights may follow
#
# $HEADER$
#
# OMPI_CHECK_PSM2(prefix, [action-if-found], [action-if-not-found])
# --------------------------------------------------------
# check if PSM2 support can be found. sets prefix_{CPPFLAGS,
# LDFLAGS, LIBS} as needed and runs action-if-found if there is
# support, otherwise executes action-if-not-found
AC_DEFUN([OMPI_CHECK_PSM2],[
AC_ARG_WITH([psm2],
[AC_HELP_STRING([--with-psm2(=DIR)],
[Build PSM2 (Intel PSM2) support, optionally adding DIR/include, DIR/lib, and DIR/lib64 to the search path for headers and libraries])])
OPAL_CHECK_WITHDIR([psm2], [$with_psm2], [include/psm2.h])
AC_ARG_WITH([psm2-libdir],
[AC_HELP_STRING([--with-psm2-libdir=DIR],
[Search for PSM (Intel PSM2) libraries in DIR])])
OPAL_CHECK_WITHDIR([psm2-libdir], [$with_psm2_libdir], [libpsm2.*])
ompi_check_psm2_$1_save_CPPFLAGS="$CPPFLAGS"
ompi_check_psm2_$1_save_LDFLAGS="$LDFLAGS"
ompi_check_psm2_$1_save_LIBS="$LIBS"
AS_IF([test "$with_psm2" != "no"],
[AS_IF([test ! -z "$with_psm2" -a "$with_psm2" != "yes"],
[ompi_check_psm2_dir="$with_psm2"])
AS_IF([test ! -z "$with_psm2_libdir" -a "$with_psm2_libdir" != "yes"],
[ompi_check_psm2_libdir="$with_psm2_libdir"])
OPAL_CHECK_PACKAGE([$1],
[psm2.h],
[psm2],
[psm_mq_irecv2],
[],
[$ompi_check_psm2_dir],
[$ompi_check_psm2_libdir],
[ompi_check_psm2_happy="yes"],
[ompi_check_psm2_happy="no"])],
[ompi_check_psm2_happy="no"])
CPPFLAGS="$ompi_check_psm2_$1_save_CPPFLAGS"
LDFLAGS="$ompi_check_psm2_$1_save_LDFLAGS"
LIBS="$ompi_check_psm2_$1_save_LIBS"
AS_IF([test "$ompi_check_psm2_happy" = "yes" -a "$enable_progress_threads" = "yes"],
[AC_MSG_WARN([PSM2 driver does not currently support progress threads. Disabling MTL.])
ompi_check_psm2_happy="no"])
AS_IF([test "$ompi_check_psm2_happy" = "yes"],
[$2],
[AS_IF([test ! -z "$with_psm2" -a "$with_psm2" != "no"],
[AC_MSG_ERROR([PSM2 support requested but not found. Aborting])])
$3])
])

61
ompi/mca/mtl/psm2/Makefile.am Обычный файл
Просмотреть файл

@ -0,0 +1,61 @@
#
# Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
# University Research and Technology
# Corporation. All rights reserved.
# Copyright (c) 2004-2005 The University of Tennessee and The University
# of Tennessee Research Foundation. All rights
# reserved.
# Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
# University of Stuttgart. All rights reserved.
# Copyright (c) 2004-2006 The Regents of the University of California.
# All rights reserved.
# Copyright (c) 2010 Cisco Systems, Inc. All rights reserved.
# Copyright (c) 2015 Intel, Inc. All rights reserved
# $COPYRIGHT$
#
# Additional copyrights may follow
#
# $HEADER$
#
EXTRA_DIST = post_configure.sh
AM_CPPFLAGS = $(mtl_psm2_CPPFLAGS)
dist_ompidata_DATA = help-mtl-psm.txt
mtl_psm2_sources = \
mtl_psm2.c \
mtl_psm2.h \
mtl_psm2_cancel.c \
mtl_psm2_component.c \
mtl_psm2_endpoint.c \
mtl_psm2_endpoint.h \
mtl_psm2_probe.c \
mtl_psm2_recv.c \
mtl_psm2_request.h \
mtl_psm2_send.c \
mtl_psm2_types.h
# Make the output library in this directory, and name it either
# mca_<type>_<name>.la (for DSO builds) or libmca_<type>_<name>.la
# (for static builds).
if MCA_BUILD_ompi_mtl_psm2_DSO
component_noinst =
component_install = mca_mtl_psm2.la
else
component_noinst = libmca_mtl_psm2.la
component_install =
endif
mcacomponentdir = $(ompilibdir)
mcacomponent_LTLIBRARIES = $(component_install)
mca_mtl_psm2_la_SOURCES = $(mtl_psm2_sources)
mca_mtl_psm2_la_LIBADD = $(mtl_psm2_LIBS)
mca_mtl_psm2_la_LDFLAGS = -module -avoid-version $(mtl_psm2_LDFLAGS)
noinst_LTLIBRARIES = $(component_noinst)
libmca_mtl_psm2_la_SOURCES = $(mtl_psm2_sources)
libmca_mtl_psm2_la_LIBADD = $(mtl_psm2_LIBS)
libmca_mtl_psm2_la_LDFLAGS = -module -avoid-version $(mtl_psm2_LDFLAGS)

49
ompi/mca/mtl/psm2/configure.m4 Обычный файл
Просмотреть файл

@ -0,0 +1,49 @@
# -*- shell-script -*-
#
# Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
# University Research and Technology
# Corporation. All rights reserved.
# Copyright (c) 2004-2005 The University of Tennessee and The University
# of Tennessee Research Foundation. All rights
# reserved.
# Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
# University of Stuttgart. All rights reserved.
# Copyright (c) 2004-2005 The Regents of the University of California.
# All rights reserved.
# Copyright (c) 2010 Cisco Systems, Inc. All rights reserved.
# Copyright (c) 2013 Sandia National Laboratories. All rights reserved.
# Copyright (c) 2014 Intel Corporation. All rights reserved.
# $COPYRIGHT$
#
# Additional copyrights may follow
#
# $HEADER$
#
# MCA_ompi_mtl_psm2_POST_CONFIG(will_build)
# ----------------------------------------
# Only require the tag if we're actually going to be built
AC_DEFUN([MCA_ompi_mtl_psm2_POST_CONFIG], [
AS_IF([test "$1" = "1"], [OMPI_REQUIRE_ENDPOINT_TAG([MTL])])
])dnl
# MCA_mtl_psm2_CONFIG([action-if-can-compile],
# [action-if-cant-compile])
# ------------------------------------------------
AC_DEFUN([MCA_ompi_mtl_psm2_CONFIG],[
AC_CONFIG_FILES([ompi/mca/mtl/psm2/Makefile])
OMPI_CHECK_PSM2([mtl_psm2],
[mtl_psm2_happy="yes"],
[mtl_psm2_happy="no"])
AS_IF([test "$mtl_psm2_happy" = "yes"],
[$1],
[$2])
# substitute in the things needed to build psm2
AC_SUBST([mtl_psm2_CFLAGS])
AC_SUBST([mtl_psm2_CPPFLAGS])
AC_SUBST([mtl_psm2_LDFLAGS])
AC_SUBST([mtl_psm2_LIBS])
])dnl

43
ompi/mca/mtl/psm2/help-mtl-psm.txt Обычный файл
Просмотреть файл

@ -0,0 +1,43 @@
# -*- text -*-
#
# Copyright (C) 2009. QLogic Corporation. All rights reserved.
# $COPYRIGHT$
#
# Additional copyrights may follow
#
# $HEADER$
#
[psm init]
Initialization of PSM library failed.
Error: %s
#
[debug level]
Unable to set PSM debug level.
Error: %s
#
[unable to open endpoint]
PSM was unable to open an endpoint. Please make sure that the network link is
active on the node and the hardware is functioning.
Error: %s
#
[no uuid present]
Error obtaining unique transport key from ORTE (orte_precondition_transports %s
the environment).
Local host: %s
#
[error polling network]
Error %s occurred in attempting to make network progress (psm_mq_ipeek).
#
[error posting receive]
Unable to post application receive buffer (psm_mq_irecv or psm_mq_imrecv).
Error: %s
Buffer: %p
Length: %d
#
[path query mechanism unknown]
Unknown path record query mechanism %s. Supported mechanisms are %s.

463
ompi/mca/mtl/psm2/mtl_psm2.c Обычный файл
Просмотреть файл

@ -0,0 +1,463 @@
/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
/*
* Copyright (c) 2004-2006 The Trustees of Indiana University and Indiana
* University Research and Technology
* Corporation. All rights reserved.
* Copyright (c) 2004-2010 The University of Tennessee and The University
* of Tennessee Research Foundation. All rights
* reserved.
* Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
* University of Stuttgart. All rights reserved.
* Copyright (c) 2004-2006 The Regents of the University of California.
* All rights reserved.
* Copyright (c) 2006 QLogic Corporation. All rights reserved.
* Copyright (c) 2013-2015 Intel, Inc. All rights reserved
* Copyright (c) 2014 Los Alamos National Security, LLC. All rights
* reserved.
* $COPYRIGHT$
*
* Additional copyrights may follow
*
* $HEADER$
*/
#include "ompi_config.h"
#include "opal/mca/pmix/pmix.h"
#include "ompi/mca/mtl/mtl.h"
#include "ompi/mca/mtl/base/mtl_base_datatype.h"
#include "opal/util/show_help.h"
#include "ompi/proc/proc.h"
#include "mtl_psm2.h"
#include "mtl_psm2_types.h"
#include "mtl_psm2_endpoint.h"
#include "mtl_psm2_request.h"
mca_mtl_psm2_module_t ompi_mtl_psm2 = {
.super = {
/* NTH: PSM supports 16 bit context ids */
.mtl_max_contextid = (1UL << 16) - 1,
.mtl_max_tag = (1UL << 30), /* must allow negatives */
.mtl_add_procs = ompi_mtl_psm2_add_procs,
.mtl_del_procs = ompi_mtl_psm2_del_procs,
.mtl_finalize = ompi_mtl_psm2_finalize,
.mtl_send = ompi_mtl_psm2_send,
.mtl_isend = ompi_mtl_psm2_isend,
.mtl_irecv = ompi_mtl_psm2_irecv,
.mtl_iprobe = ompi_mtl_psm2_iprobe,
.mtl_imrecv = ompi_mtl_psm2_imrecv,
.mtl_improbe = ompi_mtl_psm2_improbe,
.mtl_cancel = ompi_mtl_psm2_cancel,
.mtl_add_comm = ompi_mtl_psm2_add_comm,
.mtl_del_comm = ompi_mtl_psm2_del_comm
}
};
static
psm_error_t
ompi_mtl_psm2_errhandler(psm_ep_t ep, const psm_error_t error,
const char *error_string, psm_error_token_t token)
{
switch (error) {
/* We don't want PSM to default to exiting when the following errors occur */
case PSM_EP_DEVICE_FAILURE:
case PSM_EP_NO_DEVICE:
case PSM_EP_NO_PORTS_AVAIL:
case PSM_EP_NO_NETWORK:
case PSM_EP_INVALID_UUID_KEY:
opal_show_help("help-mtl-psm.txt",
"unable to open endpoint", true,
psm_error_get_string(error));
break;
/* We can't handle any other errors than the ones above */
default:
opal_output(0, "Open MPI detected an unexpected PSM error in opening "
"an endpoint: %s\n", error_string);
return psm_error_defer(token);
break;
}
return error;
}
int ompi_mtl_psm2_progress( void );
int ompi_mtl_psm2_module_init(int local_rank, int num_local_procs) {
psm_error_t err;
psm_ep_t ep; /* endpoint handle */
psm_mq_t mq;
psm_epid_t epid; /* unique lid+port identifier */
psm_uuid_t unique_job_key;
struct psm_ep_open_opts ep_opt;
unsigned long long *uu = (unsigned long long *) unique_job_key;
char *generated_key;
char env_string[256];
int rc;
generated_key = getenv("OMPI_MCA_orte_precondition_transports");
memset(uu, 0, sizeof(psm_uuid_t));
if (!generated_key || (strlen(generated_key) != 33) ||
sscanf(generated_key, "%016llx-%016llx", &uu[0], &uu[1]) != 2)
{
opal_show_help("help-mtl-psm.txt",
"no uuid present", true,
generated_key ? "could not be parsed from" :
"not present in", ompi_process_info.nodename);
return OMPI_ERROR;
}
/* Handle our own errors for opening endpoints */
psm_error_register_handler(ompi_mtl_psm2.ep, ompi_mtl_psm2_errhandler);
/* Setup MPI_LOCALRANKID and MPI_LOCALNRANKS so PSM can allocate hardware
* contexts correctly.
*/
snprintf(env_string, sizeof(env_string), "%d", local_rank);
setenv("MPI_LOCALRANKID", env_string, 0);
snprintf(env_string, sizeof(env_string), "%d", num_local_procs);
setenv("MPI_LOCALNRANKS", env_string, 0);
/* Setup the endpoint options. */
psm_ep_open_opts_get_defaults(&ep_opt);
ep_opt.timeout = ompi_mtl_psm2.connect_timeout * 1e9;
ep_opt.affinity = PSM_EP_OPEN_AFFINITY_SKIP; /* do not let PSM set affinity */
/* Open PSM endpoint */
err = psm_ep_open(unique_job_key, &ep_opt, &ep, &epid);
if (err) {
opal_show_help("help-mtl-psm.txt",
"unable to open endpoint", true,
psm_error_get_string(err));
return OMPI_ERROR;
}
/* Future errors are handled by the default error handler */
psm_error_register_handler(ompi_mtl_psm2.ep, PSM_ERRHANDLER_DEFAULT);
err = psm_mq_init(ep,
0xffff000000000000ULL,
NULL,
0,
&mq);
if (err) {
opal_show_help("help-mtl-psm.txt",
"psm init", true,
psm_error_get_string(err));
return OMPI_ERROR;
}
ompi_mtl_psm2.ep = ep;
ompi_mtl_psm2.epid = epid;
ompi_mtl_psm2.mq = mq;
OPAL_MODEX_SEND(rc, PMIX_SYNC_REQD, PMIX_GLOBAL,
&mca_mtl_psm2_component.super.mtl_version,
&ompi_mtl_psm2.epid,
sizeof(psm_epid_t));
if (OMPI_SUCCESS != rc) {
opal_output(0, "Open MPI couldn't send PSM2 epid to head node process");
return OMPI_ERROR;
}
/* register the psm progress function */
opal_progress_register(ompi_mtl_psm2_progress);
return OMPI_SUCCESS;
}
int
ompi_mtl_psm2_finalize(struct mca_mtl_base_module_t* mtl) {
psm_error_t err;
opal_progress_unregister(ompi_mtl_psm2_progress);
/* free resources */
err = psm_mq_finalize(ompi_mtl_psm2.mq);
if (err) {
opal_output(0, "Error in psm_mq_finalize (error %s)\n",
psm_error_get_string(err));
return OMPI_ERROR;
}
err = psm_ep_close(ompi_mtl_psm2.ep, PSM_EP_CLOSE_GRACEFUL, 1*1e9);
if (err) {
opal_output(0, "Error in psm_ep_close (error %s)\n",
psm_error_get_string(err));
return OMPI_ERROR;
}
err = psm_finalize();
if (err) {
opal_output(0, "Error in psm_finalize (error %s)\n",
psm_error_get_string(err));
return OMPI_ERROR;
}
return OMPI_SUCCESS;
}
static
const char *
ompi_mtl_psm2_connect_error_msg(psm_error_t err)
{
switch (err) { /* See if we expect the error */
case PSM_EPID_UNREACHABLE:
case PSM_EPID_INVALID_NODE:
case PSM_EPID_INVALID_MTU:
case PSM_EPID_INVALID_UUID_KEY:
case PSM_EPID_INVALID_VERSION:
case PSM_EPID_INVALID_CONNECT:
return psm_error_get_string(err);
break;
case PSM_EPID_UNKNOWN:
return "Connect status could not be determined "
"because of other errors";
default:
return NULL;
}
}
#ifndef min
# define min(a,b) ((a) < (b) ? (a) : (b))
#endif
#ifndef max
# define max(a,b) ((a) > (b) ? (a) : (b))
#endif
int
ompi_mtl_psm2_add_procs(struct mca_mtl_base_module_t *mtl,
size_t nprocs,
struct ompi_proc_t** procs)
{
int i,j;
int rc;
psm_epid_t *epids_in = NULL;
int *mask_in = NULL;
psm_epid_t *epid;
psm_epaddr_t *epaddrs_out = NULL;
psm_error_t *errs_out = NULL, err;
size_t size;
int proc_errors[PSM_ERROR_LAST] = { 0 };
int timeout_in_secs;
assert(mtl == &ompi_mtl_psm2.super);
rc = OMPI_ERR_OUT_OF_RESOURCE;
errs_out = (psm_error_t *) malloc(nprocs * sizeof(psm_error_t));
if (errs_out == NULL) {
goto bail;
}
epids_in = (psm_epid_t *) malloc(nprocs * sizeof(psm_epid_t));
if (epids_in == NULL) {
goto bail;
}
mask_in = (int *) malloc(nprocs * sizeof(int));
if (mask_in == NULL) {
goto bail;
}
epaddrs_out = (psm_epaddr_t *) malloc(nprocs * sizeof(psm_epaddr_t));
if (epaddrs_out == NULL) {
goto bail;
}
rc = OMPI_SUCCESS;
/* Get the epids for all the processes from modex */
for (i = 0; i < (int) nprocs; i++) {
if (NULL != procs[i]->proc_endpoints[OMPI_PROC_ENDPOINT_TAG_MTL]) {
/* Already connected: don't connect again */
mask_in[i] = 0;
continue;
}
OPAL_MODEX_RECV(rc, &mca_mtl_psm2_component.super.mtl_version,
&procs[i]->super, (void**)&epid, &size);
if (rc != OMPI_SUCCESS || size != sizeof(psm_epid_t)) {
return OMPI_ERROR;
}
epids_in[i] = *epid;
mask_in[i] = 1;
}
timeout_in_secs = max(ompi_mtl_psm2.connect_timeout, 0.5 * nprocs);
psm_error_register_handler(ompi_mtl_psm2.ep, PSM_ERRHANDLER_NOP);
err = psm_ep_connect(ompi_mtl_psm2.ep,
nprocs,
epids_in,
mask_in,
errs_out,
epaddrs_out,
timeout_in_secs * 1e9);
if (err) {
char *errstr = (char *) ompi_mtl_psm2_connect_error_msg(err);
if (errstr == NULL) {
opal_output(0, "PSM returned unhandled/unknown connect error: %s\n",
psm_error_get_string(err));
}
for (i = 0; i < (int) nprocs; i++) {
if (0 == mask_in[i]) {
continue;
}
psm_error_t thiserr = errs_out[i];
errstr = (char *) ompi_mtl_psm2_connect_error_msg(thiserr);
if (proc_errors[thiserr] == 0) {
proc_errors[thiserr] = 1;
opal_output(0, "PSM EP connect error (%s):",
errstr ? errstr : "unknown connect error");
for (j = 0; j < (int) nprocs; j++) {
if (errs_out[j] == thiserr) {
opal_output(0, " %s", (NULL == procs[j]->super.proc_hostname) ?
"unknown" : procs[j]->super.proc_hostname);
}
}
opal_output(0, "\n");
}
}
rc = OMPI_ERROR;
}
else {
/* Default error handling is enabled, errors will not be returned to
* user. PSM prints the error and the offending endpoint's hostname
* and exits with -1 */
psm_error_register_handler(ompi_mtl_psm2.ep, PSM_ERRHANDLER_DEFAULT);
/* Fill in endpoint data */
for (i = 0; i < (int) nprocs; i++) {
if (0 == mask_in[i]) {
continue;
}
mca_mtl_psm2_endpoint_t *endpoint =
(mca_mtl_psm2_endpoint_t *) OBJ_NEW(mca_mtl_psm2_endpoint_t);
endpoint->peer_epid = epids_in[i];
endpoint->peer_addr = epaddrs_out[i];
procs[i]->proc_endpoints[OMPI_PROC_ENDPOINT_TAG_MTL] = endpoint;
}
rc = OMPI_SUCCESS;
}
bail:
if (epids_in != NULL) {
free(epids_in);
}
if (mask_in != NULL) {
free(mask_in);
}
if (errs_out != NULL) {
free(errs_out);
}
if (epaddrs_out != NULL) {
free(epaddrs_out);
}
return rc;
}
int
ompi_mtl_psm2_del_procs(struct mca_mtl_base_module_t *mtl,
size_t nprocs,
struct ompi_proc_t** procs)
{
return OMPI_SUCCESS;
}
int
ompi_mtl_psm2_add_comm(struct mca_mtl_base_module_t *mtl,
struct ompi_communicator_t *comm)
{
return OMPI_SUCCESS;
}
int
ompi_mtl_psm2_del_comm(struct mca_mtl_base_module_t *mtl,
struct ompi_communicator_t *comm)
{
return OMPI_SUCCESS;
}
int ompi_mtl_psm2_progress( void ) {
psm_error_t err;
mca_mtl_psm2_request_t* mtl_psm2_request;
psm_mq_status2_t psm_status;
psm_mq_req_t req;
int completed = 1;
do {
err = psm_mq_ipeek2(ompi_mtl_psm2.mq, &req, NULL);
if (err == PSM_MQ_INCOMPLETE) {
return completed;
} else if (err != PSM_OK) {
goto error;
}
completed++;
err = psm_mq_test2(&req, &psm_status);
if (err != PSM_OK) {
goto error;
}
mtl_psm2_request = (mca_mtl_psm2_request_t*) psm_status.context;
if (mtl_psm2_request->type == OMPI_mtl_psm2_IRECV) {
mtl_psm2_request->super.ompi_req->req_status.MPI_SOURCE =
psm_status.msg_tag.tag2;
mtl_psm2_request->super.ompi_req->req_status.MPI_TAG =
psm_status.msg_tag.tag1;
mtl_psm2_request->super.ompi_req->req_status._ucount =
psm_status.nbytes;
ompi_mtl_datatype_unpack(mtl_psm2_request->convertor,
mtl_psm2_request->buf,
psm_status.msg_length);
}
if(mtl_psm2_request->type == OMPI_mtl_psm2_ISEND) {
if (mtl_psm2_request->free_after) {
free(mtl_psm2_request->buf);
}
}
switch (psm_status.error_code) {
case PSM_OK:
mtl_psm2_request->super.ompi_req->req_status.MPI_ERROR =
OMPI_SUCCESS;
break;
case PSM_MQ_TRUNCATION:
mtl_psm2_request->super.ompi_req->req_status.MPI_ERROR =
MPI_ERR_TRUNCATE;
break;
default:
mtl_psm2_request->super.ompi_req->req_status.MPI_ERROR =
MPI_ERR_INTERN;
}
mtl_psm2_request->super.completion_callback(&mtl_psm2_request->super);
}
while (1);
error:
opal_show_help("help-mtl-psm.txt",
"error polling network", true,
psm_error_get_string(err));
return 1;
}

106
ompi/mca/mtl/psm2/mtl_psm2.h Обычный файл
Просмотреть файл

@ -0,0 +1,106 @@
/*
* Copyright (c) 2004-2006 The Trustees of Indiana University and Indiana
* University Research and Technology
* Corporation. All rights reserved.
* Copyright (c) 2004-2005 The University of Tennessee and The University
* of Tennessee Research Foundation. All rights
* reserved.
* Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
* University of Stuttgart. All rights reserved.
* Copyright (c) 2004-2006 The Regents of the University of California.
* All rights reserved.
* Copyright (c) 2006 QLogic Corporation. All rights reserved.
* Copyright (c) 2015 Intel, Inc. All rights reserved
* $COPYRIGHT$
*
* Additional copyrights may follow
*
* $HEADER$
*/
#ifndef MTL_PSM2_H_HAS_BEEN_INCLUDED
#define MTL_PSM2_H_HAS_BEEN_INCLUDED
#include "ompi/mca/pml/pml.h"
#include "ompi/mca/mtl/mtl.h"
#include "ompi/mca/mtl/base/base.h"
#include "opal/datatype/opal_convertor.h"
#include <psm2.h>
#include <psm2_mq.h>
BEGIN_C_DECLS
/* MTL interface functions */
extern int ompi_mtl_psm2_add_procs(struct mca_mtl_base_module_t* mtl,
size_t nprocs,
struct ompi_proc_t** procs);
extern int ompi_mtl_psm2_del_procs(struct mca_mtl_base_module_t* mtl,
size_t nprocs,
struct ompi_proc_t** procs);
int
ompi_mtl_psm2_send(struct mca_mtl_base_module_t* mtl,
struct ompi_communicator_t* comm,
int dest,
int tag,
struct opal_convertor_t *convertor,
mca_pml_base_send_mode_t mode);
extern int ompi_mtl_psm2_isend(struct mca_mtl_base_module_t* mtl,
struct ompi_communicator_t* comm,
int dest,
int tag,
struct opal_convertor_t *convertor,
mca_pml_base_send_mode_t mode,
bool blocking,
mca_mtl_request_t * mtl_request);
extern int ompi_mtl_psm2_irecv(struct mca_mtl_base_module_t* mtl,
struct ompi_communicator_t *comm,
int src,
int tag,
struct opal_convertor_t *convertor,
struct mca_mtl_request_t *mtl_request);
extern int ompi_mtl_psm2_iprobe(struct mca_mtl_base_module_t* mtl,
struct ompi_communicator_t *comm,
int src,
int tag,
int *flag,
struct ompi_status_public_t *status);
extern int ompi_mtl_psm2_imrecv(struct mca_mtl_base_module_t* mtl,
struct opal_convertor_t *convertor,
struct ompi_message_t **message,
struct mca_mtl_request_t *mtl_request);
extern int ompi_mtl_psm2_improbe(struct mca_mtl_base_module_t *mtl,
struct ompi_communicator_t *comm,
int src,
int tag,
int *matched,
struct ompi_message_t **message,
struct ompi_status_public_t *status);
extern int ompi_mtl_psm2_cancel(struct mca_mtl_base_module_t* mtl,
struct mca_mtl_request_t *mtl_request,
int flag);
extern int ompi_mtl_psm2_add_comm(struct mca_mtl_base_module_t *mtl,
struct ompi_communicator_t *comm);
extern int ompi_mtl_psm2_del_comm(struct mca_mtl_base_module_t *mtl,
struct ompi_communicator_t *comm);
extern int ompi_mtl_psm2_finalize(struct mca_mtl_base_module_t* mtl);
int ompi_mtl_psm2_module_init(int local_rank, int num_local_procs);
END_C_DECLS
#endif /* MTL_PSM2_H_HAS_BEEN_INCLUDED */

55
ompi/mca/mtl/psm2/mtl_psm2_cancel.c Обычный файл
Просмотреть файл

@ -0,0 +1,55 @@
/*
* Copyright (c) 2004-2006 The Trustees of Indiana University and Indiana
* University Research and Technology
* Corporation. All rights reserved.
* Copyright (c) 2004-2005 The University of Tennessee and The University
* of Tennessee Research Foundation. All rights
* reserved.
* Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
* University of Stuttgart. All rights reserved.
* Copyright (c) 2004-2006 The Regents of the University of California.
* All rights reserved.
* Copyright (c) 2006 QLogic Corporation. All rights reserved.
* Copyright (c) 2015 Intel, Inc. All rights reserved
* $COPYRIGHT$
*
* Additional copyrights may follow
*
* $HEADER$
*/
#include "ompi_config.h"
#include "mtl_psm2.h"
#include "mtl_psm2_request.h"
int ompi_mtl_psm2_cancel(struct mca_mtl_base_module_t* mtl,
struct mca_mtl_request_t *mtl_request,
int flag) {
psm_error_t err;
psm_mq_status_t status;
mca_mtl_psm2_request_t *mtl_psm2_request =
(mca_mtl_psm2_request_t*) mtl_request;
/* PSM does not support canceling sends */
if(OMPI_mtl_psm2_ISEND == mtl_psm2_request->type) {
return OMPI_SUCCESS;
}
err = psm_mq_cancel(&mtl_psm2_request->psm_request);
if(PSM_OK == err) {
err = psm_mq_test(&mtl_psm2_request->psm_request, &status);
if(PSM_OK == err) {
mtl_request->ompi_req->req_status._cancelled = true;
mtl_psm2_request->super.completion_callback(&mtl_psm2_request->super);
return OMPI_SUCCESS;
} else {
return OMPI_ERROR;
}
} else if(PSM_MQ_INCOMPLETE == err) {
return OMPI_SUCCESS;
}
return OMPI_ERROR;
}

219
ompi/mca/mtl/psm2/mtl_psm2_component.c Обычный файл
Просмотреть файл

@ -0,0 +1,219 @@
/*
* Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
* University Research and Technology
* Corporation. All rights reserved.
* Copyright (c) 2004-2005 The University of Tennessee and The University
* of Tennessee Research Foundation. All rights
* reserved.
* Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
* University of Stuttgart. All rights reserved.
* Copyright (c) 2004-2005 The Regents of the University of California.
* All rights reserved.
* Copyright (c) 2006-2010 QLogic Corporation. All rights reserved.
* Copyright (c) 2012-2013 Los Alamos National Security, LLC.
* All rights reserved.
* Copyright (c) 2013-2015 Intel, Inc. All rights reserved
* $COPYRIGHT$
*
* Additional copyrights may follow
*
* $HEADER$
*/
#include "ompi_config.h"
#include "opal/mca/event/event.h"
#include "opal/util/output.h"
#include "opal/util/show_help.h"
#include "ompi/proc/proc.h"
#include "mtl_psm2.h"
#include "mtl_psm2_types.h"
#include "mtl_psm2_request.h"
#include "psm.h"
#include <sys/types.h>
#include <sys/stat.h>
#include <unistd.h>
static int param_priority;
static int ompi_mtl_psm2_component_open(void);
static int ompi_mtl_psm2_component_close(void);
static int ompi_mtl_psm2_component_query(mca_base_module_t **module, int *priority);
static int ompi_mtl_psm2_component_register(void);
static mca_mtl_base_module_t* ompi_mtl_psm2_component_init( bool enable_progress_threads,
bool enable_mpi_threads );
mca_mtl_psm2_component_t mca_mtl_psm2_component = {
{
/* First, the mca_base_component_t struct containing meta
* information about the component itself */
.mtl_version = {
MCA_MTL_BASE_VERSION_2_0_0,
.mca_component_name = "psm2",
MCA_BASE_MAKE_VERSION(component, OMPI_MAJOR_VERSION, OMPI_MINOR_VERSION,
OMPI_RELEASE_VERSION),
.mca_open_component = ompi_mtl_psm2_component_open,
.mca_close_component = ompi_mtl_psm2_component_close,
.mca_query_component = ompi_mtl_psm2_component_query,
.mca_register_component_params = ompi_mtl_psm2_component_register,
},
.mtl_data = {
/* The component is not checkpoint ready */
MCA_BASE_METADATA_PARAM_NONE
},
.mtl_init = ompi_mtl_psm2_component_init,
}
};
static int
ompi_mtl_psm2_component_register(void)
{
ompi_mtl_psm2.connect_timeout = 180;
(void) mca_base_component_var_register(&mca_mtl_psm2_component.super.mtl_version,
"connect_timeout",
"PSM connection timeout value in seconds",
MCA_BASE_VAR_TYPE_INT, NULL, 0, 0,
OPAL_INFO_LVL_9,
MCA_BASE_VAR_SCOPE_READONLY,
&ompi_mtl_psm2.connect_timeout);
param_priority = 120;
(void) mca_base_component_var_register (&mca_mtl_psm2_component.super.mtl_version,
"priority", "Priority of the PSM2 MTL component",
MCA_BASE_VAR_TYPE_INT, NULL, 0, 0,
OPAL_INFO_LVL_9,
MCA_BASE_VAR_SCOPE_READONLY,
&param_priority);
return OMPI_SUCCESS;
}
static int
ompi_mtl_psm2_component_open(void)
{
struct stat st;
/* Component available only if Omni-Path hardware is present */
if (0 == stat("/dev/hfi1", &st)) {
return OMPI_SUCCESS;
}
else {
return OPAL_ERR_NOT_AVAILABLE;
}
}
static int
ompi_mtl_psm2_component_query(mca_base_module_t **module, int *priority)
{
/*
* if we get here it means that PSM is available so give high priority
*/
*priority = param_priority;
*module = (mca_base_module_t *)&ompi_mtl_psm2.super;
return OMPI_SUCCESS;
}
static int
ompi_mtl_psm2_component_close(void)
{
return OMPI_SUCCESS;
}
static int
get_num_total_procs(int *out_ntp)
{
*out_ntp = (int)ompi_process_info.num_procs;
return OMPI_SUCCESS;
}
static int
get_num_local_procs(int *out_nlp)
{
/* num_local_peers does not include us in
* its calculation, so adjust for that */
*out_nlp = (int)(1 + ompi_process_info.num_local_peers);
return OMPI_SUCCESS;
}
static int
get_local_rank(int *out_rank)
{
ompi_node_rank_t my_node_rank;
*out_rank = 0;
if (OMPI_NODE_RANK_INVALID == (my_node_rank =
ompi_process_info.my_node_rank)) {
return OMPI_ERROR;
}
*out_rank = (int)my_node_rank;
return OMPI_SUCCESS;
}
static mca_mtl_base_module_t *
ompi_mtl_psm2_component_init(bool enable_progress_threads,
bool enable_mpi_threads)
{
psm_error_t err;
int verno_major = PSM_VERNO_MAJOR;
int verno_minor = PSM_VERNO_MINOR;
int local_rank = -1, num_local_procs = 0;
int num_total_procs = 0;
/* Compute the total number of processes on this host and our local rank
* on that node. We need to provide PSM with these values so it can
* allocate hardware contexts appropriately across processes.
*/
if (OMPI_SUCCESS != get_num_local_procs(&num_local_procs)) {
opal_output(0, "Cannot determine number of local processes. "
"Cannot continue.\n");
return NULL;
}
if (OMPI_SUCCESS != get_local_rank(&local_rank)) {
opal_output(0, "Cannot determine local rank. Cannot continue.\n");
return NULL;
}
if (OMPI_SUCCESS != get_num_total_procs(&num_total_procs)) {
opal_output(0, "Cannot determine total number of processes. "
"Cannot continue.\n");
return NULL;
}
err = psm_error_register_handler(NULL /* no ep */,
PSM_ERRHANDLER_NOP);
if (err) {
opal_output(0, "Error in psm_error_register_handler (error %s)\n",
psm_error_get_string(err));
return NULL;
}
if (num_local_procs == num_total_procs) {
setenv("PSM_DEVICES", "self,shm", 0);
}
err = psm_init(&verno_major, &verno_minor);
if (err) {
opal_show_help("help-mtl-psm.txt",
"psm init", true,
psm_error_get_string(err));
return NULL;
}
/* Complete PSM initialization */
ompi_mtl_psm2_module_init(local_rank, num_local_procs);
ompi_mtl_psm2.super.mtl_request_size =
sizeof(mca_mtl_psm2_request_t) -
sizeof(struct mca_mtl_request_t);
return &ompi_mtl_psm2.super;
}

54
ompi/mca/mtl/psm2/mtl_psm2_endpoint.c Обычный файл
Просмотреть файл

@ -0,0 +1,54 @@
/*
* Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
* University Research and Technology
* Corporation. All rights reserved.
* Copyright (c) 2004-2005 The University of Tennessee and The University
* of Tennessee Research Foundation. All rights
* reserved.
* Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
* University of Stuttgart. All rights reserved.
* Copyright (c) 2004-2006 The Regents of the University of California.
* All rights reserved.
* Copyright (c) 2006 QLogic Corporation. All rights reserved.
* Copyright (c) 2015 Intel, Inc. All rights reserved
* $COPYRIGHT$
*
* Additional copyrights may follow
*
* $HEADER$
*/
#include "ompi_config.h"
#include <sys/time.h>
#include <time.h>
#include "ompi/types.h"
#include "mtl_psm2.h"
#include "mtl_psm2_types.h"
#include "mtl_psm2_endpoint.h"
/*
* Initialize state of the endpoint instance.
*
*/
static void mca_mtl_psm2_endpoint_construct(mca_mtl_psm2_endpoint_t* endpoint)
{
endpoint->mtl_psm2_module = NULL;
}
/*
* Destroy a endpoint
*
*/
static void mca_mtl_psm2_endpoint_destruct(mca_mtl_psm2_endpoint_t* endpoint)
{
}
OBJ_CLASS_INSTANCE(
mca_mtl_psm2_endpoint_t,
opal_list_item_t,
mca_mtl_psm2_endpoint_construct,
mca_mtl_psm2_endpoint_destruct);

59
ompi/mca/mtl/psm2/mtl_psm2_endpoint.h Обычный файл
Просмотреть файл

@ -0,0 +1,59 @@
/*
* Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
* University Research and Technology
* Corporation. All rights reserved.
* Copyright (c) 2004-2005 The University of Tennessee and The University
* of Tennessee Research Foundation. All rights
* reserved.
* Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
* University of Stuttgart. All rights reserved.
* Copyright (c) 2004-2006 The Regents of the University of California.
* All rights reserved.
* Copyright (c) 2006 QLogic Corporation. All rights reserved.
* Copyright (c) 2015 Intel, Inc. All rights reserved
* $COPYRIGHT$
*
* Additional copyrights may follow
*
* $HEADER$
*/
#ifndef MCA_MTL_PSM2_ENDPOINT_H
#define MCA_MTL_PSM2_ENDPOINT_H
#include "opal/class/opal_list.h"
#include "opal/mca/event/event.h"
#include "ompi/mca/mtl/mtl.h"
#include "mtl_psm2.h"
#include "psm2.h"
BEGIN_C_DECLS
OBJ_CLASS_DECLARATION(mca_mtl_psm2_endpoint_t);
/**
* An abstraction that represents a connection to a endpoint process.
* An instance of mca_mtl_psm2_endpoint_t is associated w/ each process
* and MTL pair at startup. However, connections to the endpoint
* are established dynamically on an as-needed basis:
*/
struct mca_mtl_psm2_endpoint_t {
opal_list_item_t super;
struct mca_mtl_psm2_module_t* mtl_psm2_module;
/**< MTL instance that created this connection */
psm_epid_t peer_epid;
/**< The unique epid for the opened port */
psm_epaddr_t peer_addr;
/**< The connected endpoint handle*/
};
typedef struct mca_mtl_psm2_endpoint_t mca_mtl_psm2_endpoint_t;
OBJ_CLASS_DECLARATION(mca_mtl_psm2_endpoint);
END_C_DECLS
#endif

134
ompi/mca/mtl/psm2/mtl_psm2_probe.c Обычный файл
Просмотреть файл

@ -0,0 +1,134 @@
/*
* Copyright (c) 2004-2006 The Trustees of Indiana University and Indiana
* University Research and Technology
* Corporation. All rights reserved.
* Copyright (c) 2004-2010 The University of Tennessee and The University
* of Tennessee Research Foundation. All rights
* reserved.
* Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
* University of Stuttgart. All rights reserved.
* Copyright (c) 2004-2006 The Regents of the University of California.
* All rights reserved.
* Copyright (c) 2006 QLogic Corporation. All rights reserved.
* Copyright (c) 2006-2007 Los Alamos National Security, LLC. All rights
* reserved.
* Copyright (c) 2013-2015 Intel, Inc. All rights reserved
* $COPYRIGHT$
*
* Additional copyrights may follow
*
* $HEADER$
*/
#include "ompi_config.h"
#include "mtl_psm2.h"
#include "mtl_psm2_types.h"
#include "psm2.h"
#include "ompi/communicator/communicator.h"
#include "ompi/message/message.h"
int ompi_mtl_psm2_iprobe(struct mca_mtl_base_module_t* mtl,
struct ompi_communicator_t *comm,
int src,
int tag,
int *flag,
struct ompi_status_public_t *status)
{
psm_mq_tag_t mqtag, tagsel;
psm_mq_status2_t mqstat;
psm_error_t err;
PSM_MAKE_TAGSEL(src, tag, comm->c_contextid, mqtag, tagsel);
err = psm_mq_iprobe2(ompi_mtl_psm2.mq,
PSM_MQ_ANY_ADDR, &mqtag, &tagsel, &mqstat);
if (err == PSM_OK) {
*flag = 1;
if(MPI_STATUS_IGNORE != status) {
status->MPI_SOURCE = mqstat.msg_tag.tag2;
status->MPI_TAG = mqstat.msg_tag.tag1;
status->_ucount = mqstat.nbytes;
switch (mqstat.error_code) {
case PSM_OK:
status->MPI_ERROR = OMPI_SUCCESS;
break;
case PSM_MQ_TRUNCATION:
status->MPI_ERROR = MPI_ERR_TRUNCATE;
break;
default:
status->MPI_ERROR = MPI_ERR_INTERN;
}
}
return OMPI_SUCCESS;
}
else if (err == PSM_MQ_INCOMPLETE) {
*flag = 0;
return OMPI_SUCCESS;
}
else
return OMPI_ERROR;
}
int
ompi_mtl_psm2_improbe(struct mca_mtl_base_module_t *mtl,
struct ompi_communicator_t *comm,
int src,
int tag,
int *matched,
struct ompi_message_t **message,
struct ompi_status_public_t *status)
{
struct ompi_message_t* msg;
psm_mq_tag_t mqtag, tagsel;
psm_mq_status2_t mqstat;
psm_mq_req_t mqreq;
psm_error_t err;
PSM_MAKE_TAGSEL(src, tag, comm->c_contextid, mqtag, tagsel);
err = psm_mq_improbe2(ompi_mtl_psm2.mq,
PSM_MQ_ANY_ADDR, &mqtag, &tagsel, &mqreq, &mqstat);
if (err == PSM_OK) {
if(MPI_STATUS_IGNORE != status) {
status->MPI_SOURCE = mqstat.msg_tag.tag2;
status->MPI_TAG = mqstat.msg_tag.tag1;
status->_ucount = mqstat.nbytes;
switch (mqstat.error_code) {
case PSM_OK:
status->MPI_ERROR = OMPI_SUCCESS;
break;
case PSM_MQ_TRUNCATION:
status->MPI_ERROR = MPI_ERR_TRUNCATE;
break;
default:
status->MPI_ERROR = MPI_ERR_INTERN;
}
}
msg = ompi_message_alloc();
if(NULL == msg) {
return OMPI_ERR_OUT_OF_RESOURCE;
}
msg->comm = comm;
msg->req_ptr = mqreq;
msg->peer = mqstat.msg_tag.tag2;
msg->count = mqstat.nbytes;
*message = msg;
*matched = 1;
return OMPI_SUCCESS;
} else if(err == PSM_MQ_INCOMPLETE) {
*matched = 0;
*message = MPI_MESSAGE_NULL;
return OMPI_SUCCESS;
} else {
return OMPI_ERROR;
}
}

124
ompi/mca/mtl/psm2/mtl_psm2_recv.c Обычный файл
Просмотреть файл

@ -0,0 +1,124 @@
/*
* Copyright (c) 2004-2006 The Trustees of Indiana University and Indiana
* University Research and Technology
* Corporation. All rights reserved.
* Copyright (c) 2004-2005 The University of Tennessee and The University
* of Tennessee Research Foundation. All rights
* reserved.
* Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
* University of Stuttgart. All rights reserved.
* Copyright (c) 2004-2006 The Regents of the University of California.
* All rights reserved.
* Copyright (c) 2006 QLogic Corporation. All rights reserved.
* Copyright (c) 2013-2015 Intel, Inc. All rights reserved
* $COPYRIGHT$
*
* Additional copyrights may follow
*
* $HEADER$
*/
#include "ompi_config.h"
#include "ompi/communicator/communicator.h"
#include "ompi/message/message.h"
#include "opal/datatype/opal_convertor.h"
#include "ompi/mca/mtl/base/mtl_base_datatype.h"
#include "opal/util/show_help.h"
#include "mtl_psm2.h"
#include "mtl_psm2_types.h"
#include "mtl_psm2_request.h"
int
ompi_mtl_psm2_irecv(struct mca_mtl_base_module_t* mtl,
struct ompi_communicator_t *comm,
int src,
int tag,
struct opal_convertor_t *convertor,
struct mca_mtl_request_t *mtl_request)
{
int ret;
psm_error_t err;
mca_mtl_psm2_request_t * mtl_psm2_request = (mca_mtl_psm2_request_t*) mtl_request;
psm_mq_tag_t mqtag;
psm_mq_tag_t tagsel;
size_t length;
ret = ompi_mtl_datatype_recv_buf(convertor,
&mtl_psm2_request->buf,
&length,
&mtl_psm2_request->free_after);
if (OMPI_SUCCESS != ret) return ret;
mtl_psm2_request->length = length;
mtl_psm2_request->convertor = convertor;
mtl_psm2_request->type = OMPI_mtl_psm2_IRECV;
PSM_MAKE_TAGSEL(src, tag, comm->c_contextid, mqtag, tagsel);
err = psm_mq_irecv2(ompi_mtl_psm2.mq,
PSM_MQ_ANY_ADDR,
&mqtag,
&tagsel,
0,
mtl_psm2_request->buf,
length,
mtl_psm2_request,
&mtl_psm2_request->psm_request);
if (err) {
opal_show_help("help-mtl-psm.txt",
"error posting receive", true,
psm_error_get_string(err),
mtl_psm2_request->buf, length);
return OMPI_ERROR;
}
return OMPI_SUCCESS;
}
int
ompi_mtl_psm2_imrecv(struct mca_mtl_base_module_t* mtl,
struct opal_convertor_t *convertor,
struct ompi_message_t **message,
struct mca_mtl_request_t *mtl_request)
{
mca_mtl_psm2_request_t *mtl_psm2_request =
(mca_mtl_psm2_request_t*) mtl_request;
size_t length;
psm_error_t err;
int ret;
mtl_psm2_request->psm_request =
(psm_mq_req_t)(*message)->req_ptr;
ret = ompi_mtl_datatype_recv_buf(convertor,
&mtl_psm2_request->buf,
&length,
&mtl_psm2_request->free_after);
if (OMPI_SUCCESS != ret) return ret;
mtl_psm2_request->length = length;
mtl_psm2_request->convertor = convertor;
mtl_psm2_request->type = OMPI_mtl_psm2_IRECV;
err = psm_mq_imrecv(ompi_mtl_psm2.mq, 0,
mtl_psm2_request->buf, length, mtl_psm2_request,
&mtl_psm2_request->psm_request);
if(err) {
opal_show_help("help-mtl-psm.txt",
"error posting receive", true,
psm_error_get_string(err),
mtl_psm2_request->buf, length);
return OMPI_ERROR;
}
*message = MPI_MESSAGE_NULL;
return OMPI_SUCCESS;
}

44
ompi/mca/mtl/psm2/mtl_psm2_request.h Обычный файл
Просмотреть файл

@ -0,0 +1,44 @@
/*
* Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
* University Research and Technology
* Corporation. All rights reserved.
* Copyright (c) 2004-2005 The University of Tennessee and The University
* of Tennessee Research Foundation. All rights
* reserved.
* Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
* University of Stuttgart. All rights reserved.
* Copyright (c) 2004-2005 The Regents of the University of California.
* All rights reserved.
* Copyright (c) 2006 QLogic Corporation. All rights reserved.
* Copyright (c) 2015 Intel, Inc. All rights reserved
* $COPYRIGHT$
*
* Additional copyrights may follow
*
* $HEADER$
*/
#ifndef OMPI_MTL_PSM2_REQUEST_H
#define OMPI_MTL_PSM2_REQUEST_H
#include "opal/datatype/opal_convertor.h"
typedef enum {
OMPI_mtl_psm2_ISEND,
OMPI_mtl_psm2_IRECV
} mca_mtl_psm2_request_type_t;
struct mca_mtl_psm2_request_t {
struct mca_mtl_request_t super;
mca_mtl_psm2_request_type_t type;
psm_mq_req_t psm_request;
/* psm_segment_t psm_segment[1]; */
void *buf;
size_t length;
struct opal_convertor_t *convertor;
bool free_after;
};
typedef struct mca_mtl_psm2_request_t mca_mtl_psm2_request_t;
#endif

129
ompi/mca/mtl/psm2/mtl_psm2_send.c Обычный файл
Просмотреть файл

@ -0,0 +1,129 @@
/*
* Copyright (c) 2004-2006 The Trustees of Indiana University and Indiana
* University Research and Technology
* Corporation. All rights reserved.
* Copyright (c) 2004-2005 The University of Tennessee and The University
* of Tennessee Research Foundation. All rights
* reserved.
* Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
* University of Stuttgart. All rights reserved.
* Copyright (c) 2004-2006 The Regents of the University of California.
* All rights reserved.
* Copyright (c) 2006 QLogic Corporation. All rights reserved.
* Copyright (c) 2013-2015 Intel, Inc. All rights reserved
* $COPYRIGHT$
*
* Additional copyrights may follow
*
* $HEADER$
*/
#include "ompi_config.h"
#include "ompi/mca/pml/pml.h"
#include "ompi/communicator/communicator.h"
#include "opal/datatype/opal_convertor.h"
#include "mtl_psm2.h"
#include "mtl_psm2_types.h"
#include "mtl_psm2_request.h"
#include "ompi/mca/mtl/base/mtl_base_datatype.h"
int
ompi_mtl_psm2_send(struct mca_mtl_base_module_t* mtl,
struct ompi_communicator_t* comm,
int dest,
int tag,
struct opal_convertor_t *convertor,
mca_pml_base_send_mode_t mode)
{
psm_error_t err;
mca_mtl_psm2_request_t mtl_psm2_request;
psm_mq_tag_t mqtag;
uint32_t flags = 0;
int ret;
size_t length;
ompi_proc_t* ompi_proc = ompi_comm_peer_lookup( comm, dest );
mca_mtl_psm2_endpoint_t* psm_endpoint = (mca_mtl_psm2_endpoint_t*) ompi_proc->proc_endpoints[OMPI_PROC_ENDPOINT_TAG_MTL];
assert(mtl == &ompi_mtl_psm2.super);
PSM_MAKE_MQTAG(comm->c_contextid, comm->c_my_rank, tag, mqtag);
ret = ompi_mtl_datatype_pack(convertor,
&mtl_psm2_request.buf,
&length,
&mtl_psm2_request.free_after);
mtl_psm2_request.length = length;
mtl_psm2_request.convertor = convertor;
mtl_psm2_request.type = OMPI_mtl_psm2_ISEND;
if (OMPI_SUCCESS != ret) return ret;
if (mode == MCA_PML_BASE_SEND_SYNCHRONOUS)
flags |= PSM_MQ_FLAG_SENDSYNC;
err = psm_mq_send2(ompi_mtl_psm2.mq,
psm_endpoint->peer_addr,
flags,
&mqtag,
mtl_psm2_request.buf,
length);
if (mtl_psm2_request.free_after) {
free(mtl_psm2_request.buf);
}
return err == PSM_OK ? OMPI_SUCCESS : OMPI_ERROR;
}
int
ompi_mtl_psm2_isend(struct mca_mtl_base_module_t* mtl,
struct ompi_communicator_t* comm,
int dest,
int tag,
struct opal_convertor_t *convertor,
mca_pml_base_send_mode_t mode,
bool blocking,
mca_mtl_request_t * mtl_request)
{
psm_error_t psm_error;
psm_mq_tag_t mqtag;
uint32_t flags = 0;
int ret;
mca_mtl_psm2_request_t * mtl_psm2_request = (mca_mtl_psm2_request_t*) mtl_request;
size_t length;
ompi_proc_t* ompi_proc = ompi_comm_peer_lookup( comm, dest );
mca_mtl_psm2_endpoint_t* psm_endpoint = (mca_mtl_psm2_endpoint_t*)ompi_proc->proc_endpoints[OMPI_PROC_ENDPOINT_TAG_MTL];
assert(mtl == &ompi_mtl_psm2.super);
PSM_MAKE_MQTAG(comm->c_contextid, comm->c_my_rank, tag, mqtag);
ret = ompi_mtl_datatype_pack(convertor,
&mtl_psm2_request->buf,
&length,
&mtl_psm2_request->free_after);
mtl_psm2_request->length= length;
mtl_psm2_request->convertor = convertor;
mtl_psm2_request->type = OMPI_mtl_psm2_ISEND;
if (OMPI_SUCCESS != ret) return ret;
if (mode == MCA_PML_BASE_SEND_SYNCHRONOUS)
flags |= PSM_MQ_FLAG_SENDSYNC;
psm_error = psm_mq_isend2(ompi_mtl_psm2.mq,
psm_endpoint->peer_addr,
flags,
&mqtag,
mtl_psm2_request->buf,
length,
mtl_psm2_request,
&mtl_psm2_request->psm_request);
return psm_error == PSM_OK ? OMPI_SUCCESS : OMPI_ERROR;
}

93
ompi/mca/mtl/psm2/mtl_psm2_types.h Исполняемый файл
Просмотреть файл

@ -0,0 +1,93 @@
/*
* Copyright (c) 2004-2006 The Trustees of Indiana University and Indiana
* University Research and Technology
* Corporation. All rights reserved.
* Copyright (c) 2004-2007 The University of Tennessee and The University
* of Tennessee Research Foundation. All rights
* reserved.
* Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
* University of Stuttgart. All rights reserved.
* Copyright (c) 2004-2006 The Regents of the University of California.
* All rights reserved.
* Copyright (c) 2006 QLogic Corporation. All rights reserved.
* Copyright (c) 2011 Los Alamos National Security, LLC.
* All rights reserved.
* Copyright (c) 2013-2015 Intel, Inc. All rights reserved
* $COPYRIGHT$
*
* Additional copyrights may follow
*
* $HEADER$
*/
#ifndef MTL_PSM2_TYPES_H_HAS_BEEN_INCLUDED
#define MTL_PSM2_TYPES_H_HAS_BEEN_INCLUDED
#include "ompi_config.h"
#include "mtl_psm2.h"
#include "ompi/communicator/communicator.h"
#include "ompi/mca/mtl/mtl.h"
#include "ompi/mca/mtl/base/base.h"
#include "mtl_psm2_endpoint.h"
#include "psm2.h"
BEGIN_C_DECLS
/**
* MTL Module Interface
*/
struct mca_mtl_psm2_module_t {
mca_mtl_base_module_t super; /**< base MTL interface */
int32_t connect_timeout;
psm_ep_t ep;
psm_mq_t mq;
psm_epid_t epid;
psm_epaddr_t epaddr;
};
typedef struct mca_mtl_psm2_module_t mca_mtl_psm2_module_t;
extern mca_mtl_psm2_module_t ompi_mtl_psm2;
struct mca_mtl_psm2_component_t {
mca_mtl_base_component_2_0_0_t super; /**< base MTL component */
};
typedef struct mca_mtl_psm2_component_t mca_mtl_psm2_component_t;
OMPI_DECLSPEC extern mca_mtl_psm2_component_t mca_mtl_psm2_component;
#define PSM_MAKE_MQTAG(ctxt,rank,utag,tag) \
do { \
(tag).tag0 = ctxt; \
(tag).tag1 = utag; \
(tag).tag2 = rank; \
} while (0)
#define PSM_MAKE_TAGSEL(user_rank, user_tag, user_ctxt, tag, _tagsel) \
do { \
(tag).tag0 = user_ctxt; \
(tag).tag1 = user_tag; \
(tag).tag2 = user_rank; \
(_tagsel).tag0 = 0xffffffffULL; \
(_tagsel).tag1 = 0xffffffffULL; \
(_tagsel).tag2 = 0xffffffffULL; \
if((user_tag) == MPI_ANY_TAG) \
{ \
(_tagsel).tag1 = 0x80000000ULL; \
(tag).tag1 = 0x00000000ULL; \
} \
if((user_rank) == MPI_ANY_SOURCE) \
{ \
(_tagsel).tag2 = 0x00000000ULL; \
} \
} while (0)
END_C_DECLS
#endif /* MTL_PSM2_TYPES_H_HAS_BEEN_INCLUDED */

1
ompi/mca/mtl/psm2/post_configure.sh Обычный файл
Просмотреть файл

@ -0,0 +1 @@
DIRECT_CALL_HEADER="ompi/mca/mtl/psm2/mtl_psm.h"

1
ompi/mca/pml/cm/pml_cm_component.c
Просмотреть файл

@ -156,6 +156,7 @@ mca_pml_cm_component_init(int* priority,
*priority = -1;
return NULL;
} else if((strcmp(ompi_mtl_base_selected_component->mtl_version.mca_component_name, "psm") == 0) ||
(strcmp(ompi_mtl_base_selected_component->mtl_version.mca_component_name, "psm2") == 0) ||
(strcmp(ompi_mtl_base_selected_component->mtl_version.mca_component_name, "mxm") == 0) ||
(strcmp(ompi_mtl_base_selected_component->mtl_version.mca_component_name, "ofi") == 0) ||
(strcmp(ompi_mtl_base_selected_component->mtl_version.mca_component_name, "portals4") == 0)) {