ports/openmpi
ports/openmpi
1
1
Форкнуть 0
Код Релизы Активность

removed teg/ptls

Просмотр исходного кода 
This commit was SVN r9115.
Этот коммит содержится в:
Tim Woodall 2006-02-22 21:23:24 +00:00
родитель e58b758031
Коммит 08b3bad09d
159 изменённых файлов: 0 добавлений и 24666 удалений
Показать статистику Скачать Patch файл Скачать Diff файл Показать все файлы Свернуть все файлы

67
ompi/mca/pml/teg/Makefile.am
Просмотреть файл

@ -1,67 +0,0 @@
#
# Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
# University Research and Technology
# Corporation. All rights reserved.
# Copyright (c) 2004-2005 The University of Tennessee and The University
# of Tennessee Research Foundation. All rights
# reserved.
# Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
# University of Stuttgart. All rights reserved.
# Copyright (c) 2004-2005 The Regents of the University of California.
# All rights reserved.
# $COPYRIGHT$
#
# Additional copyrights may follow
#
# $HEADER$
#
teg_sources = \
pml_teg.c \
pml_teg.h \
pml_teg_cancel.c \
pml_teg_component.c \
pml_teg_component.h \
pml_teg_iprobe.c \
pml_teg_irecv.c \
pml_teg_isend.c \
pml_teg_ptl.c \
pml_teg_ptl.h \
pml_teg_proc.c \
pml_teg_proc.h \
pml_teg_progress.c \
pml_teg_recvfrag.c \
pml_teg_recvfrag.h \
pml_teg_recvreq.c \
pml_teg_recvreq.h \
pml_teg_sendreq.c \
pml_teg_sendreq.h \
pml_teg_start.c \
pml_ptl_array.c \
pml_ptl_array.h
if OMPI_BUILD_pml_teg_DSO
component_noinst =
component_install = mca_pml_teg.la
else
component_noinst = libmca_pml_teg.la
component_install =
endif
mcacomponentdir = $(libdir)/openmpi
mcacomponent_LTLIBRARIES = $(component_install)
mca_pml_teg_la_SOURCES = $(teg_sources)
mca_pml_teg_la_LIBADD = \
$(top_ompi_builddir)/ompi/libmpi.la \
$(top_ompi_builddir)/orte/liborte.la \
$(top_ompi_builddir)/opal/libopal.la
mca_pml_teg_la_LDFLAGS = -module -avoid-version
noinst_LTLIBRARIES = $(component_noinst)
libmca_pml_teg_la_SOURCES = $(teg_sources)
libmca_pml_teg_la_LIBADD =
libmca_pml_teg_la_LDFLAGS = -module -avoid-version

24
ompi/mca/pml/teg/configure.params
Просмотреть файл

@ -1,24 +0,0 @@
# -*- shell-script -*-
#
# Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
# University Research and Technology
# Corporation. All rights reserved.
# Copyright (c) 2004-2005 The University of Tennessee and The University
# of Tennessee Research Foundation. All rights
# reserved.
# Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
# University of Stuttgart. All rights reserved.
# Copyright (c) 2004-2005 The Regents of the University of California.
# All rights reserved.
# $COPYRIGHT$
#
# Additional copyrights may follow
#
# $HEADER$
#
# Specific to this module
PARAM_INIT_FILE=pml_teg.c
PARAM_CONFIG_HEADER_FILE="teg_config.h"
PARAM_CONFIG_FILES="Makefile"

63
ompi/mca/pml/teg/pml_ptl_array.c
Просмотреть файл

@ -1,63 +0,0 @@
/*
* Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
* University Research and Technology
* Corporation. All rights reserved.
* Copyright (c) 2004-2005 The University of Tennessee and The University
* of Tennessee Research Foundation. All rights
* reserved.
* Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
* University of Stuttgart. All rights reserved.
* Copyright (c) 2004-2005 The Regents of the University of California.
* All rights reserved.
* $COPYRIGHT$
*
* Additional copyrights may follow
*
* $HEADER$
*/
#include "ompi_config.h"
#include <string.h>
#include "ompi/mca/pml/pml.h"
#include "pml_ptl_array.h"
#include "ompi/constants.h"
static void mca_ptl_array_construct(mca_ptl_array_t* array)
{
array->ptl_procs = 0;
array->ptl_size = 0;
array->ptl_index = 0;
array->ptl_reserve = 0;
}
static void mca_ptl_array_destruct(mca_ptl_array_t* array)
{
if(array->ptl_procs != 0)
free(array->ptl_procs);
}
OBJ_CLASS_INSTANCE(
mca_pml_teg_ptl_array_t,
opal_object_t,
mca_ptl_array_construct,
mca_ptl_array_destruct
);
int mca_ptl_array_reserve(mca_ptl_array_t* array, size_t size)
{
mca_ptl_proc_t *procs;
if(array->ptl_reserve >= size)
return OMPI_SUCCESS;
procs = (mca_ptl_proc_t *)realloc(array->ptl_procs, sizeof(mca_ptl_proc_t)*size);
if(NULL == procs)
return OMPI_ERR_OUT_OF_RESOURCE;
array->ptl_procs = procs;
array->ptl_reserve = size;
memset(array->ptl_procs+array->ptl_size, 0, (size-array->ptl_size)*sizeof(mca_ptl_proc_t));
return OMPI_SUCCESS;
}

148
ompi/mca/pml/teg/pml_ptl_array.h
Просмотреть файл

@ -1,148 +0,0 @@
/*
* Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
* University Research and Technology
* Corporation. All rights reserved.
* Copyright (c) 2004-2005 The University of Tennessee and The University
* of Tennessee Research Foundation. All rights
* reserved.
* Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
* University of Stuttgart. All rights reserved.
* Copyright (c) 2004-2005 The Regents of the University of California.
* All rights reserved.
* $COPYRIGHT$
*
* Additional copyrights may follow
*
* $HEADER$
*/
/**
* @file
*/
#ifndef OMPI_PTL_ARRAY_H
#define OMPI_PTL_ARRAY_H
#include "opal/util/output.h"
#include "ompi/mca/ptl/ptl.h"
#if defined(c_plusplus) || defined(__cplusplus)
extern "C" {
#endif
extern opal_class_t mca_pml_teg_ptl_array_t_class;
/**
* A data structure associated with a ompi_proc_t that caches
* addressing/scheduling attributes for a specific PTL instance
* that can be used to reach the process.
*/
struct mca_ptl_proc_t {
int ptl_weight; /**< PTL weight for scheduling */
struct mca_ptl_base_peer_t* ptl_peer; /**< PTL addressing info */
struct mca_pml_base_ptl_t* ptl_base; /**< PML specific PTL info */
mca_ptl_base_module_t *ptl; /**< PTL module */
};
typedef struct mca_ptl_proc_t mca_ptl_proc_t;
/**
* A dynamically growable array of mca_ptl_proc_t instances.
* Maintains an index into the array that is used for round-robin
* scheduling across contents.
*/
struct mca_ptl_array_t {
opal_object_t super;
mca_ptl_proc_t* ptl_procs; /**< array of ptl procs */
size_t ptl_size; /**< number available */
size_t ptl_reserve; /**< size of allocated ptl_proc array */
size_t ptl_index; /**< last used index*/
};
typedef struct mca_ptl_array_t mca_ptl_array_t;
typedef struct mca_ptl_array_t mca_pml_teg_ptl_array_t;
/**
* If required, reallocate (grow) the array to the indicate size.
*
* @param array (IN)
* @param size (IN)
*/
int mca_ptl_array_reserve(mca_ptl_array_t*, size_t);
static inline size_t mca_ptl_array_get_size(mca_ptl_array_t* array)
{
return array->ptl_size;
}
/**
* Grow the array if required, and set the size.
*
* @param array (IN)
* @param size (IN)
*/
static inline void mca_ptl_array_set_size(mca_ptl_array_t* array, size_t size)
{
if(array->ptl_size > array->ptl_reserve)
mca_ptl_array_reserve(array, size);
array->ptl_size = size;
}
/**
* Grow the array size by one and return the item at that index.
*
* @param array (IN)
*/
static inline mca_ptl_proc_t* mca_ptl_array_insert(mca_ptl_array_t* array)
{
#if OMPI_ENABLE_DEBUG
if(array->ptl_size >= array->ptl_reserve) {
opal_output(0, "mca_ptl_array_insert: invalid array index %d >= %d",
array->ptl_size, array->ptl_reserve);
return 0;
}
#endif
return &array->ptl_procs[array->ptl_size++];
}
/**
* Return an array item at the specified index.
*
* @param array (IN)
* @param index (IN)
*/
static inline mca_ptl_proc_t* mca_ptl_array_get_index(mca_ptl_array_t* array, size_t index)
{
#if OMPI_ENABLE_DEBUG
if(index >= array->ptl_size) {
opal_output(0, "mca_ptl_array_get_index: invalid array index %d >= %d",
index, array->ptl_size);
return 0;
}
#endif
return &array->ptl_procs[index];
}
/**
* Return the next LRU index in the array.
*
* @param array (IN)
* @param index (IN)
*/
static inline mca_ptl_proc_t* mca_ptl_array_get_next(mca_ptl_array_t* array)
{
mca_ptl_proc_t* ptl_proc;
#if OMPI_ENABLE_DEBUG
if(array->ptl_size == 0) {
opal_output(0, "mca_ptl_array_get_next: invalid array size");
return 0;
}
#endif
ptl_proc = &array->ptl_procs[array->ptl_index++];
if(array->ptl_index == array->ptl_size)
array->ptl_index = 0;
return ptl_proc;
}
#if defined(c_plusplus) || defined(__cplusplus)
}
#endif
#endif

454
ompi/mca/pml/teg/pml_teg.c
Просмотреть файл

@ -1,454 +0,0 @@
/*
* Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
* University Research and Technology
* Corporation. All rights reserved.
* Copyright (c) 2004-2005 The University of Tennessee and The University
* of Tennessee Research Foundation. All rights
* reserved.
* Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
* University of Stuttgart. All rights reserved.
* Copyright (c) 2004-2005 The Regents of the University of California.
* All rights reserved.
* $COPYRIGHT$
*
* Additional copyrights may follow
*
* $HEADER$
*/
#include "ompi_config.h"
#include <stdlib.h>
#include <string.h>
#include "ompi/class/ompi_bitmap.h"
#include "ompi/mca/pml/pml.h"
#include "ompi/mca/ptl/ptl.h"
#include "ompi/mca/ptl/base/base.h"
#include "ompi/mca/ptl/base/ptl_base_comm.h"
#include "ompi/mca/ptl/base/ptl_base_header.h"
#include "ompi/mca/ptl/base/ptl_base_recvfrag.h"
#include "ompi/mca/ptl/base/ptl_base_sendfrag.h"
#include "pml_teg.h"
#include "pml_teg_component.h"
#include "pml_teg_proc.h"
#include "pml_teg_ptl.h"
#include "pml_teg_recvreq.h"
#include "pml_teg_sendreq.h"
#include "pml_teg_recvfrag.h"
mca_pml_teg_t mca_pml_teg = {
{
mca_pml_teg_add_procs,
mca_pml_teg_del_procs,
mca_pml_teg_enable,
mca_pml_teg_progress,
mca_pml_teg_add_comm,
mca_pml_teg_del_comm,
mca_pml_teg_irecv_init,
mca_pml_teg_irecv,
mca_pml_teg_recv,
mca_pml_teg_isend_init,
mca_pml_teg_isend,
mca_pml_teg_send,
mca_pml_teg_iprobe,
mca_pml_teg_probe,
mca_pml_teg_start,
32768,
(0x7fffffff) /* XXX should be INT_MAX, as in ob1 */
}
};
int mca_pml_teg_add_comm(ompi_communicator_t* comm)
{
/* allocate pml specific comm data */
mca_pml_ptl_comm_t* pml_comm = OBJ_NEW(mca_pml_ptl_comm_t);
if (NULL == pml_comm) {
return OMPI_ERR_OUT_OF_RESOURCE;
}
mca_pml_ptl_comm_init_size(pml_comm, comm->c_remote_group->grp_proc_count);
comm->c_pml_comm = pml_comm;
return OMPI_SUCCESS;
}
int mca_pml_teg_del_comm(ompi_communicator_t* comm)
{
OBJ_RELEASE(comm->c_pml_comm);
comm->c_pml_comm = NULL; /* make sure it's set to NULL */
return OMPI_SUCCESS;
}
static int ptl_exclusivity_compare(const void* arg1, const void* arg2)
{
mca_ptl_base_module_t* ptl1 = *(struct mca_ptl_base_module_t**)arg1;
mca_ptl_base_module_t* ptl2 = *(struct mca_ptl_base_module_t**)arg2;
if( ptl1->ptl_exclusivity > ptl2->ptl_exclusivity ) {
return -1;
} else if (ptl1->ptl_exclusivity == ptl2->ptl_exclusivity ) {
return 0;
} else {
return 1;
}
}
static int mca_pml_teg_add_ptls(void)
{
/* build an array of ptls and ptl modules */
mca_ptl_base_selected_module_t* selected_ptl;
size_t num_ptls = opal_list_get_size(&mca_ptl_base_modules_initialized);
size_t cache_bytes = 0;
mca_pml_teg.teg_num_ptl_modules = 0;
mca_pml_teg.teg_num_ptl_progress = 0;
mca_pml_teg.teg_num_ptl_components = 0;
mca_pml_teg.teg_ptl_modules = (mca_ptl_base_module_t **)malloc(sizeof(mca_ptl_base_module_t*) * num_ptls);
mca_pml_teg.teg_ptl_progress = (mca_ptl_base_component_progress_fn_t*)malloc(sizeof(mca_ptl_base_component_progress_fn_t) * num_ptls);
mca_pml_teg.teg_ptl_components = (mca_ptl_base_component_t **)malloc(sizeof(mca_ptl_base_component_t*) * num_ptls);
if (NULL == mca_pml_teg.teg_ptl_modules ||
NULL == mca_pml_teg.teg_ptl_progress ||
NULL == mca_pml_teg.teg_ptl_components) {
return OMPI_ERR_OUT_OF_RESOURCE;
}
for(selected_ptl = (mca_ptl_base_selected_module_t*)
opal_list_get_first(&mca_ptl_base_modules_initialized);
selected_ptl != (mca_ptl_base_selected_module_t*)
opal_list_get_end(&mca_ptl_base_modules_initialized);
selected_ptl = (mca_ptl_base_selected_module_t*)opal_list_get_next(selected_ptl)) {
mca_ptl_base_module_t *ptl = selected_ptl->pbsm_module;
size_t i;
mca_pml_teg.teg_ptl_modules[mca_pml_teg.teg_num_ptl_modules++] = ptl;
for(i=0; i < mca_pml_teg.teg_num_ptl_components; i++) {
if(mca_pml_teg.teg_ptl_components[i] == ptl->ptl_component) {
break;
}
}
if(i == mca_pml_teg.teg_num_ptl_components) {
mca_pml_teg.teg_ptl_components[mca_pml_teg.teg_num_ptl_components++] = ptl->ptl_component;
}
/*
*setup ptl
*/
/* set pointer to fragment matching logic routine, if this
* not already set by the ptl
*/
if(NULL == ptl->ptl_match) {
ptl->ptl_match = mca_pml_teg_recv_frag_match;
}
ptl->ptl_send_progress = mca_pml_teg_send_request_progress;
ptl->ptl_recv_progress = mca_pml_teg_recv_request_progress;
ptl->ptl_stack = ptl;
ptl->ptl_base = NULL;
/* find maximum required size for cache */
if(ptl->ptl_cache_bytes > cache_bytes) {
cache_bytes = ptl->ptl_cache_bytes;
}
}
/* setup send fragments based on largest required send request */
ompi_free_list_init( &mca_pml_teg.teg_send_requests,
sizeof(mca_pml_teg_send_request_t) + cache_bytes,
OBJ_CLASS(mca_pml_teg_send_request_t),
mca_pml_teg.teg_free_list_num,
mca_pml_teg.teg_free_list_max,
mca_pml_teg.teg_free_list_inc,
NULL );
/* sort ptl list by exclusivity */
qsort(mca_pml_teg.teg_ptl_modules, mca_pml_teg.teg_num_ptl_modules, sizeof(struct mca_ptl_t*), ptl_exclusivity_compare);
return OMPI_SUCCESS;
}
/*
* Called by the base PML in order to notify the PMLs about their selected status. After the init pass,
* the base module will choose one PML (depending on informations provided by the init function) and then
* it will call the pml_enable function with true (for the selected one) and with false for all the
* others. The selected one can then pass control information through to all PTL modules.
*/
int mca_pml_teg_enable(bool enable)
{
size_t i;
int value = enable;
/* If I'm not selected then prepare for close */
if( false == enable ) return OMPI_SUCCESS;
/* recv requests */
ompi_free_list_init( &mca_pml_teg.teg_recv_requests,
sizeof(mca_pml_teg_recv_request_t),
OBJ_CLASS(mca_pml_teg_recv_request_t),
mca_pml_teg.teg_free_list_num,
mca_pml_teg.teg_free_list_max,
mca_pml_teg.teg_free_list_inc,
NULL );
/* Grab all the PTLs and prepare them */
mca_pml_teg_add_ptls();
/* and now notify them about the status */
for(i=0; i < mca_pml_teg.teg_num_ptl_components; i++) {
if(NULL != mca_pml_teg.teg_ptl_components[i]->ptlm_control) {
int rc = mca_pml_teg.teg_ptl_components[i]->ptlm_control(MCA_PTL_ENABLE,&value,sizeof(value));
if(rc != OMPI_SUCCESS)
return rc;
}
}
return OMPI_SUCCESS;
}
/*
* For each proc setup a datastructure that indicates the PTLs
* that can be used to reach the destination.
*
*/
int mca_pml_teg_add_procs(ompi_proc_t** procs, size_t nprocs)
{
size_t p;
ompi_bitmap_t reachable;
struct mca_ptl_base_peer_t** ptl_peers = NULL;
int rc;
size_t p_index;
if(nprocs == 0)
return OMPI_SUCCESS;
OBJ_CONSTRUCT(&reachable, ompi_bitmap_t);
rc = ompi_bitmap_init(&reachable, nprocs);
if(OMPI_SUCCESS != rc)
return rc;
/* attempt to add all procs to each ptl */
ptl_peers = (struct mca_ptl_base_peer_t **)malloc(nprocs * sizeof(struct mca_ptl_base_peer_t*));
for(p_index = 0; p_index < mca_pml_teg.teg_num_ptl_modules; p_index++) {
mca_ptl_base_module_t* ptl = mca_pml_teg.teg_ptl_modules[p_index];
int ptl_inuse = 0;
/* if the ptl can reach the destination proc it sets the
* corresponding bit (proc index) in the reachable bitmap
* and can return addressing information for each proc
* that is passed back to the ptl on data transfer calls
*/
ompi_bitmap_clear_all_bits(&reachable);
memset(ptl_peers, 0, nprocs * sizeof(struct mca_ptl_base_peer_t*));
rc = ptl->ptl_add_procs(ptl, nprocs, procs, ptl_peers, &reachable);
if(OMPI_SUCCESS != rc) {
free(ptl_peers);
return rc;
}
/* for each proc that is reachable - add the ptl to the procs array(s) */
for(p=0; p < nprocs; p++) {
ompi_proc_t *proc;
mca_pml_teg_proc_t* proc_pml;
mca_ptl_proc_t* proc_ptl;
size_t size;
if( !ompi_bitmap_is_set_bit(&reachable, p) ) continue;
proc = procs[p];
proc_pml = (mca_pml_teg_proc_t*) proc->proc_pml;
/* this ptl can be used */
ptl_inuse++;
/* initialize each proc */
if(NULL == proc_pml) {
/* allocate pml specific proc data */
proc_pml = OBJ_NEW(mca_pml_teg_proc_t);
if (NULL == proc_pml) {
opal_output(0, "mca_pml_teg_add_procs: unable to allocate resources");
free(ptl_peers);
return OMPI_ERR_OUT_OF_RESOURCE;
}
/* preallocate space in array for max number of ptls */
mca_ptl_array_reserve(&proc_pml->proc_ptl_first, mca_pml_teg.teg_num_ptl_modules);
mca_ptl_array_reserve(&proc_pml->proc_ptl_next, mca_pml_teg.teg_num_ptl_modules);
proc_pml->base.proc_ompi = proc;
proc->proc_pml = (mca_pml_proc_t*) proc_pml;
}
/* dont allow an additional PTL with a lower exclusivity ranking */
size = mca_ptl_array_get_size(&proc_pml->proc_ptl_next);
if(size > 0) {
proc_ptl = mca_ptl_array_get_index(&proc_pml->proc_ptl_next, size-1);
/* skip this ptl if the exclusivity is less than the previous */
if(proc_ptl->ptl->ptl_exclusivity > ptl->ptl_exclusivity) {
if(ptl_peers[p] != NULL) {
ptl->ptl_del_procs(ptl, 1, &proc, &ptl_peers[p]);
}
continue;
}
}
/* cache the ptl on the proc */
proc_ptl = mca_ptl_array_insert(&proc_pml->proc_ptl_next);
proc_ptl->ptl = ptl;
proc_ptl->ptl_peer = ptl_peers[p];
proc_ptl->ptl_weight = 0;
proc_pml->proc_ptl_flags |= ptl->ptl_flags;
}
if(ptl_inuse > 0 && NULL != ptl->ptl_component->ptlm_progress) {
size_t p;
bool found = false;
for(p=0; p < mca_pml_teg.teg_num_ptl_progress; p++) {
if(mca_pml_teg.teg_ptl_progress[p] == ptl->ptl_component->ptlm_progress) {
found = true;
break;
}
}
if(found == false) {
mca_pml_teg.teg_ptl_progress[mca_pml_teg.teg_num_ptl_progress] =
ptl->ptl_component->ptlm_progress;
mca_pml_teg.teg_num_ptl_progress++;
}
}
}
free(ptl_peers);
/* iterate back through procs and compute metrics for registered ptls */
for(p=0; p<nprocs; p++) {
ompi_proc_t *proc = procs[p];
mca_pml_teg_proc_t* proc_pml = (mca_pml_teg_proc_t*) proc->proc_pml;
double total_bandwidth = 0;
uint32_t latency = 0;
size_t n_index;
size_t n_size;
/* skip over procs w/ no ptls registered */
if(NULL == proc_pml)
continue;
/* (1) determine the total bandwidth available across all ptls
* note that we need to do this here, as we may already have ptls configured
* (2) determine the highest priority ranking for latency
*/
n_size = mca_ptl_array_get_size(&proc_pml->proc_ptl_next);
for(n_index = 0; n_index < n_size; n_index++) {
struct mca_ptl_proc_t* proc_ptl = mca_ptl_array_get_index(&proc_pml->proc_ptl_next, n_index);
struct mca_ptl_base_module_t* ptl = proc_ptl->ptl;
total_bandwidth += proc_ptl->ptl->ptl_bandwidth;
if(ptl->ptl_latency > latency)
latency = ptl->ptl_latency;
}
/* (1) set the weight of each ptl as a percentage of overall bandwidth
* (2) copy all ptl instances at the highest priority ranking into the
* list of ptls used for first fragments
*/
for(n_index = 0; n_index < n_size; n_index++) {
struct mca_ptl_proc_t* proc_ptl = mca_ptl_array_get_index(&proc_pml->proc_ptl_next, n_index);
struct mca_ptl_base_module_t *ptl = proc_ptl->ptl;
double weight;
/* compute weighting factor for this ptl */
if(ptl->ptl_bandwidth)
weight = proc_ptl->ptl->ptl_bandwidth / total_bandwidth;
else
weight = 1.0 / n_size;
proc_ptl->ptl_weight = (int)(weight * 100);
/*
* save/create ptl extension for use by pml
*/
proc_ptl->ptl_base = ptl->ptl_base;
if (NULL == proc_ptl->ptl_base &&
ptl->ptl_cache_bytes > 0 &&
NULL != ptl->ptl_request_init &&
NULL != ptl->ptl_request_fini) {
mca_pml_base_ptl_t* ptl_base = OBJ_NEW(mca_pml_base_ptl_t);
ptl_base->ptl = ptl;
ptl_base->ptl_cache_size = ptl->ptl_cache_size;
proc_ptl->ptl_base = ptl->ptl_base = ptl_base;
}
/* check to see if this ptl is already in the array of ptls used for first
* fragments - if not add it.
*/
if(ptl->ptl_latency == latency) {
struct mca_ptl_proc_t* proc_new = mca_ptl_array_insert(&proc_pml->proc_ptl_first);
*proc_new = *proc_ptl;
}
}
}
return OMPI_SUCCESS;
}
/*
* iterate through each proc and notify any PTLs associated
* with the proc that it is/has gone away
*/
int mca_pml_teg_del_procs(ompi_proc_t** procs, size_t nprocs)
{
size_t p;
int rc;
for(p = 0; p < nprocs; p++) {
ompi_proc_t *proc = procs[p];
mca_pml_teg_proc_t* proc_pml = (mca_pml_teg_proc_t*) proc->proc_pml;
size_t f_index, f_size;
size_t n_index, n_size;
/* notify each ptl that the proc is going away */
f_size = mca_ptl_array_get_size(&proc_pml->proc_ptl_first);
for(f_index = 0; f_index < f_size; f_index++) {
mca_ptl_proc_t* ptl_proc = mca_ptl_array_get_index(&proc_pml->proc_ptl_first, f_index);
mca_ptl_base_module_t* ptl = ptl_proc->ptl;
rc = ptl->ptl_del_procs(ptl, 1, &proc, &ptl_proc->ptl_peer);
if(OMPI_SUCCESS != rc) {
return rc;
}
/* remove this from next array so that we dont call it twice w/
* the same address pointer
*/
n_size = mca_ptl_array_get_size(&proc_pml->proc_ptl_first);
for(n_index = 0; n_index < n_size; n_index++) {
mca_ptl_proc_t* next_proc = mca_ptl_array_get_index(&proc_pml->proc_ptl_next, n_index);
if(next_proc->ptl == ptl) {
memset(next_proc, 0, sizeof(mca_ptl_proc_t));
break;
}
}
}
/* notify each ptl that was not in the array of ptls for first fragments */
n_size = mca_ptl_array_get_size(&proc_pml->proc_ptl_next);
for(n_index = 0; n_index < n_size; n_index++) {
mca_ptl_proc_t* ptl_proc = mca_ptl_array_get_index(&proc_pml->proc_ptl_first, n_index);
mca_ptl_base_module_t* ptl = ptl_proc->ptl;
if (ptl != 0) {
rc = ptl->ptl_del_procs(ptl,1,&proc,&ptl_proc->ptl_peer);
if(OMPI_SUCCESS != rc)
return rc;
}
}
/* do any required cleanup */
OBJ_RELEASE(proc_pml);
proc->proc_pml = NULL;
}
return OMPI_SUCCESS;
}
int mca_pml_teg_component_fini(void)
{
/* FIX */
return OMPI_SUCCESS;
}

263
ompi/mca/pml/teg/pml_teg.h
Просмотреть файл

@ -1,263 +0,0 @@
/*
* Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
* University Research and Technology
* Corporation. All rights reserved.
* Copyright (c) 2004-2005 The University of Tennessee and The University
* of Tennessee Research Foundation. All rights
* reserved.
* Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
* University of Stuttgart. All rights reserved.
* Copyright (c) 2004-2005 The Regents of the University of California.
* All rights reserved.
* $COPYRIGHT$
*
* Additional copyrights may follow
*
* $HEADER$
*/
/**
* @file
*/
#ifndef MCA_PML_TEG_H
#define MCA_PML_TEG_H
#include "opal/threads/threads.h"
#include "opal/threads/condition.h"
#include "ompi/class/ompi_free_list.h"
#include "opal/util/cmd_line.h"
#include "ompi/request/request.h"
#include "ompi/mca/pml/pml.h"
#include "ompi/mca/pml/base/pml_base_request.h"
#include "ompi/mca/pml/base/pml_base_bsend.h"
#include "ompi/mca/ptl/base/ptl_base_sendreq.h"
#include "ompi/mca/ptl/ptl.h"
#if defined(c_plusplus) || defined(__cplusplus)
extern "C" {
#endif
/**
* TEG PML module
*/
struct mca_pml_teg_t {
mca_pml_base_module_t super;
mca_ptl_base_component_t **teg_ptl_components;
size_t teg_num_ptl_components;
mca_ptl_base_module_t** teg_ptl_modules;
size_t teg_num_ptl_modules;
mca_ptl_base_component_progress_fn_t* teg_ptl_progress;
size_t teg_num_ptl_progress;
opal_list_t teg_procs;
opal_mutex_t teg_lock;
int teg_priority;
int teg_free_list_num; /* initial size of free list */
int teg_free_list_max; /* maximum size of free list */
int teg_free_list_inc; /* number of elements to grow free list */
int teg_poll_iterations; /* number of iterations to poll for completion */
/* free list of requests */
ompi_free_list_t teg_send_requests;
ompi_free_list_t teg_recv_requests;
/* list of pending send requests */
opal_list_t teg_send_pending;
};
typedef struct mca_pml_teg_t mca_pml_teg_t;
extern mca_pml_teg_t mca_pml_teg;
/*
* PML module functions.
*/
extern int mca_pml_teg_component_open(void);
extern int mca_pml_teg_component_close(void);
extern mca_pml_base_module_t* mca_pml_teg_component_init(
int *priority,
bool enable_progress_threads,
bool enable_mpi_threads
);
extern int mca_pml_teg_component_fini(void);
/*
* PML interface functions.
*/
extern int mca_pml_teg_add_comm(
struct ompi_communicator_t* comm
);
extern int mca_pml_teg_del_comm(
struct ompi_communicator_t* comm
);
extern int mca_pml_teg_add_procs(
struct ompi_proc_t **procs,
size_t nprocs
);
extern int mca_pml_teg_del_procs(
struct ompi_proc_t **procs,
size_t nprocs
);
extern int mca_pml_teg_enable(
bool enable
);
extern int mca_pml_teg_progress(void);
extern int mca_pml_teg_iprobe(
int dst,
int tag,
struct ompi_communicator_t* comm,
int *matched,
ompi_status_public_t* status
);
extern int mca_pml_teg_probe(
int dst,
int tag,
struct ompi_communicator_t* comm,
ompi_status_public_t* status
);
extern int mca_pml_teg_cancelled(
ompi_request_t* request,
int *flag
);
extern int mca_pml_teg_isend_init(
void *buf,
size_t count,
struct ompi_datatype_t *datatype,
int dst,
int tag,
mca_pml_base_send_mode_t mode,
struct ompi_communicator_t* comm,
struct ompi_request_t **request
);
extern int mca_pml_teg_isend(
void *buf,
size_t count,
struct ompi_datatype_t *datatype,
int dst,
int tag,
mca_pml_base_send_mode_t mode,
struct ompi_communicator_t* comm,
struct ompi_request_t **request
);
extern int mca_pml_teg_send(
void *buf,
size_t count,
struct ompi_datatype_t *datatype,
int dst,
int tag,
mca_pml_base_send_mode_t mode,
struct ompi_communicator_t* comm
);
extern int mca_pml_teg_irecv_init(
void *buf,
size_t count,
struct ompi_datatype_t *datatype,
int src,
int tag,
struct ompi_communicator_t* comm,
struct ompi_request_t **request
);
extern int mca_pml_teg_irecv(
void *buf,
size_t count,
struct ompi_datatype_t *datatype,
int src,
int tag,
struct ompi_communicator_t* comm,
struct ompi_request_t **request
);
extern int mca_pml_teg_recv(
void *buf,
size_t count,
struct ompi_datatype_t *datatype,
int src,
int tag,
struct ompi_communicator_t* comm,
ompi_status_public_t* status
);
extern int mca_pml_teg_progress(void);
extern int mca_pml_teg_start(
size_t count,
ompi_request_t** requests
);
#if defined(c_plusplus) || defined(__cplusplus)
}
#endif
#define MCA_PML_TEG_FINI(request) \
{ \
mca_pml_base_request_t* pml_request = *(mca_pml_base_request_t**)(request); \
if(pml_request->req_persistent) { \
if(pml_request->req_free_called) { \
MCA_PML_TEG_FREE(request); \
} else { \
pml_request->req_ompi.req_state = OMPI_REQUEST_INACTIVE; \
} \
} else { \
MCA_PML_TEG_FREE(request); \
} \
}
#define MCA_PML_TEG_FREE(request) \
{ \
mca_pml_base_request_t* pml_request = *(mca_pml_base_request_t**)(request); \
pml_request->req_free_called = true; \
if( pml_request->req_pml_complete == true) \
{ \
switch(pml_request->req_type) { \
case MCA_PML_REQUEST_SEND: \
{ \
mca_pml_teg_send_request_t* sendreq = (mca_pml_teg_send_request_t*)pml_request; \
while(sendreq->req_lock > 0); \
if(sendreq->req_send.req_send_mode == MCA_PML_BASE_SEND_BUFFERED) { \
mca_pml_base_bsend_request_fini((ompi_request_t*)sendreq); \
} \
MCA_PML_TEG_SEND_REQUEST_RETURN(sendreq); \
break; \
} \
case MCA_PML_REQUEST_RECV: \
{ \
mca_pml_teg_recv_request_t* recvreq = (mca_pml_teg_recv_request_t*)pml_request; \
MCA_PML_TEG_RECV_REQUEST_RETURN(recvreq); \
break; \
} \
default: \
break; \
} \
} \
*(request) = MPI_REQUEST_NULL; \
}
#endif

30
ompi/mca/pml/teg/pml_teg_cancel.c
Просмотреть файл

@ -1,30 +0,0 @@
/* -*- Mode: C; c-basic-offset:4 ; -*- */
/*
* Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
* University Research and Technology
* Corporation. All rights reserved.
* Copyright (c) 2004-2005 The University of Tennessee and The University
* of Tennessee Research Foundation. All rights
* reserved.
* Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
* University of Stuttgart. All rights reserved.
* Copyright (c) 2004-2005 The Regents of the University of California.
* All rights reserved.
* $COPYRIGHT$
*
* Additional copyrights may follow
*
* $HEADER$
*/
#include "ompi_config.h"
#include "pml_teg.h"
int mca_pml_teg_cancelled(ompi_request_t* request, int* flag)
{
if(NULL != flag)
*flag = (true == request->req_status._cancelled ? 1 : 0);
return OMPI_SUCCESS;
}

158
ompi/mca/pml/teg/pml_teg_component.c
Просмотреть файл

@ -1,158 +0,0 @@
/*
* Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
* University Research and Technology
* Corporation. All rights reserved.
* Copyright (c) 2004-2005 The University of Tennessee and The University
* of Tennessee Research Foundation. All rights
* reserved.
* Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
* University of Stuttgart. All rights reserved.
* Copyright (c) 2004-2005 The Regents of the University of California.
* All rights reserved.
* $COPYRIGHT$
*
* Additional copyrights may follow
*
* $HEADER$
*/
#include "ompi_config.h"
#include "opal/event/event.h"
#include "mpi.h"
#include "ompi/mca/pml/pml.h"
#include "ompi/mca/ptl/ptl.h"
#include "opal/mca/base/mca_base_param.h"
#include "ompi/mca/pml/base/pml_base_bsend.h"
#include "ompi/mca/ptl/base/base.h"
#include "pml_teg.h"
#include "pml_teg_proc.h"
#include "pml_teg_sendreq.h"
#include "pml_teg_recvreq.h"
mca_pml_base_component_1_0_0_t mca_pml_teg_component = {
/* First, the mca_base_component_t struct containing meta
information about the component itself */
{
/* Indicate that we are a pml v1.0.0 component (which also implies
a specific MCA version) */
MCA_PML_BASE_VERSION_1_0_0,
"teg", /* MCA component name */
OMPI_MAJOR_VERSION, /* MCA component major version */
OMPI_MINOR_VERSION, /* MCA component minor version */
OMPI_RELEASE_VERSION, /* MCA component release version */
mca_pml_teg_component_open, /* component open */
mca_pml_teg_component_close /* component close */
},
/* Next the MCA v1.0.0 component meta data */
{
/* Whether the component is checkpointable or not */
false
},
mca_pml_teg_component_init, /* component init */
mca_pml_teg_component_fini /* component finalize */
};
static inline int mca_pml_teg_param_register_int(
const char* param_name,
int default_value)
{
int id = mca_base_param_register_int("pml","teg",param_name,NULL,default_value);
int param_value = default_value;
mca_base_param_lookup_int(id,&param_value);
return param_value;
}
int mca_pml_teg_component_open(void)
{
OBJ_CONSTRUCT(&mca_pml_teg.teg_lock, opal_mutex_t);
OBJ_CONSTRUCT(&mca_pml_teg.teg_send_requests, ompi_free_list_t);
OBJ_CONSTRUCT(&mca_pml_teg.teg_recv_requests, ompi_free_list_t);
OBJ_CONSTRUCT(&mca_pml_teg.teg_procs, opal_list_t);
OBJ_CONSTRUCT(&mca_pml_teg.teg_send_pending, opal_list_t);
mca_pml_teg.teg_ptl_components = NULL;
mca_pml_teg.teg_num_ptl_components = 0;
mca_pml_teg.teg_ptl_modules = NULL;
mca_pml_teg.teg_num_ptl_modules = 0;
mca_pml_teg.teg_ptl_progress = NULL;
mca_pml_teg.teg_num_ptl_progress = 0;
mca_pml_teg.teg_free_list_num =
mca_pml_teg_param_register_int("free_list_num", 4);
mca_pml_teg.teg_free_list_max =
mca_pml_teg_param_register_int("free_list_max", -1);
mca_pml_teg.teg_free_list_inc =
mca_pml_teg_param_register_int("free_list_inc", 64);
mca_pml_teg.teg_poll_iterations =
mca_pml_teg_param_register_int("poll_iterations", 100000);
mca_pml_teg.teg_priority =
mca_pml_teg_param_register_int("priority", 0);
return mca_ptl_base_open();
}
int mca_pml_teg_component_close(void)
{
int rc;
/* I was not enabled */
if( NULL == mca_pml_teg.teg_ptl_components )
return OMPI_SUCCESS;
if(OMPI_SUCCESS != (rc = mca_ptl_base_close()))
return rc;
if(NULL != mca_pml_teg.teg_ptl_components) {
free(mca_pml_teg.teg_ptl_components);
mca_pml_teg.teg_ptl_components = NULL;
}
if(NULL != mca_pml_teg.teg_ptl_modules) {
free(mca_pml_teg.teg_ptl_modules);
mca_pml_teg.teg_ptl_modules = NULL;
}
if(NULL != mca_pml_teg.teg_ptl_progress) {
free(mca_pml_teg.teg_ptl_progress);
mca_pml_teg.teg_ptl_progress = NULL;
}
OBJ_DESTRUCT(&mca_pml_teg.teg_send_pending);
OBJ_DESTRUCT(&mca_pml_teg.teg_send_requests);
OBJ_DESTRUCT(&mca_pml_teg.teg_recv_requests);
OBJ_DESTRUCT(&mca_pml_teg.teg_procs);
OBJ_DESTRUCT(&mca_pml_teg.teg_lock);
return OMPI_SUCCESS;
}
mca_pml_base_module_t* mca_pml_teg_component_init( int* priority,
bool enable_progress_threads,
bool enable_mpi_threads )
{
int rc;
*priority = mca_pml_teg.teg_priority;
/* buffered send */
if(OMPI_SUCCESS != mca_pml_base_bsend_init(enable_mpi_threads)) {
opal_output(0, "mca_pml_teg_component_init: mca_pml_bsend_init failed\n");
return NULL;
}
rc = mca_ptl_base_select( enable_progress_threads, enable_mpi_threads );
if( rc != OMPI_SUCCESS )
return NULL;
return &mca_pml_teg.super;
}

29
ompi/mca/pml/teg/pml_teg_component.h
Просмотреть файл

@ -1,29 +0,0 @@
/*
* Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
* University Research and Technology
* Corporation. All rights reserved.
* Copyright (c) 2004-2005 The University of Tennessee and The University
* of Tennessee Research Foundation. All rights
* reserved.
* Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
* University of Stuttgart. All rights reserved.
* $COPYRIGHT$
*
* Additional copyrights may follow
*
* $HEADER$
*/
/**
* @file
*/
#ifndef MCA_PML_TEG_COMPONENT_H
#define MCA_PML_TEG_COMPONENT_H
/*
* PML module functions.
*/
OMPI_COMP_EXPORT extern mca_pml_base_component_1_0_0_t mca_pml_teg_component;
#endif

95
ompi/mca/pml/teg/pml_teg_iprobe.c
Просмотреть файл

@ -1,95 +0,0 @@
/*
* Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
* University Research and Technology
* Corporation. All rights reserved.
* Copyright (c) 2004-2005 The University of Tennessee and The University
* of Tennessee Research Foundation. All rights
* reserved.
* Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
* University of Stuttgart. All rights reserved.
* Copyright (c) 2004-2005 The Regents of the University of California.
* All rights reserved.
* $COPYRIGHT$
*
* Additional copyrights may follow
*
* $HEADER$
*/
#include "ompi_config.h"
#include "ompi/request/request.h"
#include "pml_teg_recvreq.h"
int mca_pml_teg_iprobe(int src,
int tag,
struct ompi_communicator_t *comm,
int *matched, ompi_status_public_t * status)
{
int rc;
mca_ptl_base_recv_request_t recvreq;
OBJ_CONSTRUCT( &(recvreq), mca_ptl_base_recv_request_t );
recvreq.req_recv.req_base.req_ompi.req_type = OMPI_REQUEST_PML;
recvreq.req_recv.req_base.req_type = MCA_PML_REQUEST_IPROBE;
MCA_PML_TEG_RECV_REQUEST_INIT(&recvreq, NULL, 0, &ompi_mpi_char, src, tag, comm, true);
*matched = 0;
if ((rc = mca_pml_teg_recv_request_start(&recvreq)) == OMPI_SUCCESS) {
if( recvreq.req_recv.req_base.req_ompi.req_complete == true ) {
if( NULL != status ) {
*status = recvreq.req_recv.req_base.req_ompi.req_status;
}
*matched = 1;
} else {
/* we are supposed to progress ... */
opal_progress();
}
}
MCA_PML_BASE_RECV_REQUEST_FINI((&recvreq.req_recv));
return rc;
}
int mca_pml_teg_probe(int src,
int tag,
struct ompi_communicator_t *comm,
ompi_status_public_t * status)
{
int rc;
mca_ptl_base_recv_request_t recvreq;
OBJ_CONSTRUCT( &(recvreq), mca_ptl_base_recv_request_t );
recvreq.req_recv.req_base.req_ompi.req_type = OMPI_REQUEST_PML;
recvreq.req_recv.req_base.req_type = MCA_PML_REQUEST_PROBE;
MCA_PML_TEG_RECV_REQUEST_INIT(&recvreq, NULL, 0, &ompi_mpi_char, src, tag, comm, true);
if ((rc = mca_pml_teg_recv_request_start(&recvreq)) != OMPI_SUCCESS) {
MCA_PML_BASE_RECV_REQUEST_FINI((&recvreq.req_recv));
return rc;
}
if (recvreq.req_recv.req_base.req_ompi.req_complete == false) {
/* give up and sleep until completion */
if (opal_using_threads()) {
opal_mutex_lock(&ompi_request_lock);
ompi_request_waiting++;
while (recvreq.req_recv.req_base.req_ompi.req_complete == false)
opal_condition_wait(&ompi_request_cond, &ompi_request_lock);
ompi_request_waiting--;
opal_mutex_unlock(&ompi_request_lock);
} else {
ompi_request_waiting++;
while (recvreq.req_recv.req_base.req_ompi.req_complete == false)
opal_condition_wait(&ompi_request_cond, &ompi_request_lock);
ompi_request_waiting--;
}
}
if (NULL != status) {
*status = recvreq.req_recv.req_base.req_ompi.req_status;
}
MCA_PML_BASE_RECV_REQUEST_FINI(&recvreq.req_recv);
return OMPI_SUCCESS;
}

119
ompi/mca/pml/teg/pml_teg_irecv.c
Просмотреть файл

@ -1,119 +0,0 @@
/*
* Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
* University Research and Technology
* Corporation. All rights reserved.
* Copyright (c) 2004-2005 The University of Tennessee and The University
* of Tennessee Research Foundation. All rights
* reserved.
* Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
* University of Stuttgart. All rights reserved.
* Copyright (c) 2004-2005 The Regents of the University of California.
* All rights reserved.
* $COPYRIGHT$
*
* Additional copyrights may follow
*
* $HEADER$
*/
#include "ompi_config.h"
#include "ompi/request/request.h"
#include "pml_teg_recvreq.h"
int mca_pml_teg_irecv_init(void *addr,
size_t count,
struct ompi_datatype_t * datatype,
int src,
int tag,
struct ompi_communicator_t *comm,
struct ompi_request_t **request)
{
int rc;
mca_ptl_base_recv_request_t *recvreq;
MCA_PML_TEG_RECV_REQUEST_ALLOC(recvreq, rc);
if (NULL == recvreq)
return rc;
MCA_PML_TEG_RECV_REQUEST_INIT(recvreq,
addr,
count, datatype, src, tag, comm, true);
*request = (ompi_request_t *) recvreq;
return OMPI_SUCCESS;
}
int mca_pml_teg_irecv(void *addr,
size_t count,
struct ompi_datatype_t * datatype,
int src,
int tag,
struct ompi_communicator_t *comm,
struct ompi_request_t **request)
{
int rc;
mca_ptl_base_recv_request_t *recvreq;
MCA_PML_TEG_RECV_REQUEST_ALLOC(recvreq, rc);
if (NULL == recvreq)
return rc;
MCA_PML_TEG_RECV_REQUEST_INIT(recvreq,
addr,
count, datatype, src, tag, comm, false);
if ((rc = mca_pml_teg_recv_request_start(recvreq)) != OMPI_SUCCESS) {
MCA_PML_TEG_RECV_REQUEST_RETURN(recvreq);
return rc;
}
*request = (ompi_request_t *) recvreq;
return OMPI_SUCCESS;
}
int mca_pml_teg_recv(void *addr,
size_t count,
struct ompi_datatype_t * datatype,
int src,
int tag,
struct ompi_communicator_t *comm,
ompi_status_public_t * status)
{
int rc;
mca_ptl_base_recv_request_t *recvreq;
MCA_PML_TEG_RECV_REQUEST_ALLOC(recvreq, rc);
if (NULL == recvreq)
return rc;
MCA_PML_TEG_RECV_REQUEST_INIT(recvreq,
addr,
count, datatype, src, tag, comm, false);
if ((rc = mca_pml_teg_recv_request_start(recvreq)) != OMPI_SUCCESS) {
goto recv_finish;
}
if (recvreq->req_recv.req_base.req_ompi.req_complete == false) {
/* give up and sleep until completion */
if (opal_using_threads()) {
opal_mutex_lock(&ompi_request_lock);
ompi_request_waiting++;
while (recvreq->req_recv.req_base.req_ompi.req_complete == false)
opal_condition_wait(&ompi_request_cond, &ompi_request_lock);
ompi_request_waiting--;
opal_mutex_unlock(&ompi_request_lock);
} else {
ompi_request_waiting++;
while (recvreq->req_recv.req_base.req_ompi.req_complete == false)
opal_condition_wait(&ompi_request_cond, &ompi_request_lock);
ompi_request_waiting--;
}
}
recv_finish:
if (NULL != status) { /* return status */
*status = recvreq->req_recv.req_base.req_ompi.req_status;
}
MCA_PML_TEG_RECV_REQUEST_RETURN(recvreq);
return recvreq->req_recv.req_base.req_ompi.req_status.MPI_ERROR;
}

130
ompi/mca/pml/teg/pml_teg_isend.c
Просмотреть файл

@ -1,130 +0,0 @@
/*
* Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
* University Research and Technology
* Corporation. All rights reserved.
* Copyright (c) 2004-2005 The University of Tennessee and The University
* of Tennessee Research Foundation. All rights
* reserved.
* Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
* University of Stuttgart. All rights reserved.
* Copyright (c) 2004-2005 The Regents of the University of California.
* All rights reserved.
* $COPYRIGHT$
*
* Additional copyrights may follow
*
* $HEADER$
*/
#include "ompi_config.h"
#include "pml_teg.h"
#include "pml_teg_proc.h"
#include "pml_teg_sendreq.h"
#include "pml_teg_recvreq.h"
int mca_pml_teg_isend_init(void *buf,
size_t count,
struct ompi_datatype_t * datatype,
int dst,
int tag,
mca_pml_base_send_mode_t sendmode,
ompi_communicator_t * comm,
ompi_request_t ** request)
{
int rc;
mca_pml_teg_send_request_t *sendreq;
MCA_PML_TEG_SEND_REQUEST_ALLOC(comm, dst, sendreq, rc);
if (rc != OMPI_SUCCESS)
return rc;
MCA_PML_TEG_SEND_REQUEST_INIT(sendreq,
buf,
count,
datatype,
dst, tag,
comm, sendmode, true);
*request = (ompi_request_t *) sendreq;
return OMPI_SUCCESS;
}
int mca_pml_teg_isend(void *buf,
size_t count,
struct ompi_datatype_t * datatype,
int dst,
int tag,
mca_pml_base_send_mode_t sendmode,
ompi_communicator_t * comm,
ompi_request_t ** request)
{
int rc;
mca_pml_teg_send_request_t *sendreq;
MCA_PML_TEG_SEND_REQUEST_ALLOC(comm, dst, sendreq, rc);
if (rc != OMPI_SUCCESS)
return rc;
MCA_PML_TEG_SEND_REQUEST_INIT(sendreq,
buf,
count,
datatype,
dst, tag,
comm, sendmode, false);
MCA_PML_TEG_SEND_REQUEST_START(sendreq, rc);
*request = (ompi_request_t *) sendreq;
return rc;
}
int mca_pml_teg_send(void *buf,
size_t count,
struct ompi_datatype_t * datatype,
int dst,
int tag,
mca_pml_base_send_mode_t sendmode,
ompi_communicator_t * comm)
{
int rc;
mca_pml_teg_send_request_t *sendreq;
MCA_PML_TEG_SEND_REQUEST_ALLOC(comm, dst, sendreq, rc);
if (rc != OMPI_SUCCESS)
return rc;
MCA_PML_TEG_SEND_REQUEST_INIT(sendreq,
buf,
count,
datatype,
dst, tag,
comm, sendmode, false);
MCA_PML_TEG_SEND_REQUEST_START(sendreq, rc);
if (rc != OMPI_SUCCESS) {
MCA_PML_TEG_FREE((ompi_request_t **) & sendreq);
return rc;
}
if (sendreq->req_send.req_base.req_ompi.req_complete == false) {
/* give up and sleep until completion */
if (opal_using_threads()) {
opal_mutex_lock(&ompi_request_lock);
ompi_request_waiting++;
while (sendreq->req_send.req_base.req_ompi.req_complete == false)
opal_condition_wait(&ompi_request_cond, &ompi_request_lock);
ompi_request_waiting--;
opal_mutex_unlock(&ompi_request_lock);
} else {
ompi_request_waiting++;
while (sendreq->req_send.req_base.req_ompi.req_complete == false)
opal_condition_wait(&ompi_request_cond, &ompi_request_lock);
ompi_request_waiting--;
}
}
/* return request to pool */
MCA_PML_TEG_FREE((ompi_request_t **) & sendreq);
return OMPI_SUCCESS;
}

58
ompi/mca/pml/teg/pml_teg_proc.c
Просмотреть файл

@ -1,58 +0,0 @@
/*
* Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
* University Research and Technology
* Corporation. All rights reserved.
* Copyright (c) 2004-2005 The University of Tennessee and The University
* of Tennessee Research Foundation. All rights
* reserved.
* Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
* University of Stuttgart. All rights reserved.
* Copyright (c) 2004-2005 The Regents of the University of California.
* All rights reserved.
* $COPYRIGHT$
*
* Additional copyrights may follow
*
* $HEADER$
*/
#include "ompi_config.h"
#include "opal/sys/atomic.h"
#include "pml_teg.h"
#include "pml_teg_proc.h"
#include "pml_ptl_array.h"
static void mca_pml_teg_proc_construct(mca_pml_teg_proc_t* proc)
{
proc->base.proc_ompi = NULL;
proc->proc_ptl_flags = 0;
OBJ_CONSTRUCT(&proc->base.proc_lock, opal_mutex_t);
OBJ_CONSTRUCT(&proc->proc_ptl_first, mca_pml_teg_ptl_array_t);
OBJ_CONSTRUCT(&proc->proc_ptl_next, mca_pml_teg_ptl_array_t);
OPAL_THREAD_LOCK(&mca_pml_teg.teg_lock);
opal_list_append(&mca_pml_teg.teg_procs, (opal_list_item_t*)proc);
OPAL_THREAD_UNLOCK(&mca_pml_teg.teg_lock);
}
static void mca_pml_teg_proc_destruct(mca_pml_teg_proc_t* proc)
{
OPAL_THREAD_LOCK(&mca_pml_teg.teg_lock);
opal_list_remove_item(&mca_pml_teg.teg_procs, (opal_list_item_t*)proc);
OPAL_THREAD_UNLOCK(&mca_pml_teg.teg_lock);
OBJ_DESTRUCT(&proc->base.proc_lock);
OBJ_DESTRUCT(&proc->proc_ptl_first);
OBJ_DESTRUCT(&proc->proc_ptl_next);
}
OBJ_CLASS_INSTANCE(
mca_pml_teg_proc_t,
opal_list_item_t,
mca_pml_teg_proc_construct,
mca_pml_teg_proc_destruct
);

106
ompi/mca/pml/teg/pml_teg_proc.h
Просмотреть файл

@ -1,106 +0,0 @@
/*
* Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
* University Research and Technology
* Corporation. All rights reserved.
* Copyright (c) 2004-2005 The University of Tennessee and The University
* of Tennessee Research Foundation. All rights
* reserved.
* Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
* University of Stuttgart. All rights reserved.
* Copyright (c) 2004-2005 The Regents of the University of California.
* All rights reserved.
* $COPYRIGHT$
*
* Additional copyrights may follow
*
* $HEADER$
*/
/**
* @file
*/
#ifndef MCA_PML_PROC_H
#define MCA_PML_PROC_H
#include "opal/threads/mutex.h"
#include "ompi/communicator/communicator.h"
#include "ompi/group/group.h"
#include "ompi/proc/proc.h"
#include "pml_ptl_array.h"
#if defined(c_plusplus) || defined(__cplusplus)
extern "C" {
#endif
/**
* Structure associated w/ ompi_proc_t that contains data specific
* to the PML. Note that this name is not PML specific.
*/
struct mca_pml_teg_proc_t {
mca_pml_proc_t base;
mca_ptl_array_t proc_ptl_first; /**< array of ptls to use for first fragments */
mca_ptl_array_t proc_ptl_next; /**< array of ptls to use for remaining fragments */
uint32_t proc_ptl_flags; /**< aggregate ptl flags */
};
typedef struct mca_pml_teg_proc_t mca_pml_teg_proc_t;
OMPI_COMP_EXPORT extern opal_class_t mca_pml_teg_proc_t_class;
/**
* Return the mca_pml_proc_t instance cached in the communicators local group.
*
* @param comm Communicator
* @param rank Peer rank
* @return mca_pml_proc_t instance
*/
static inline mca_pml_proc_t* mca_pml_teg_proc_lookup_local(ompi_communicator_t* comm, int rank)
{
ompi_proc_t* proc = comm->c_local_group->grp_proc_pointers[rank];
return proc->proc_pml;
}
/**
* Return the mca_pml_proc_t instance cached on the communicators remote group.
*
* @param comm Communicator
* @param rank Peer rank
* @return mca_pml_proc_t instance
*/
static inline mca_pml_proc_t* mca_pml_teg_proc_lookup_remote(ompi_communicator_t* comm, int rank)
{
ompi_proc_t* proc = comm->c_remote_group->grp_proc_pointers[rank];
return proc->proc_pml;
}
/**
* Return the mca_ptl_peer_t instance corresponding to the process/ptl combination.
*
* @param comm Communicator
* @param rank Peer rank
* @return mca_pml_proc_t instance
*/
static inline struct mca_ptl_base_peer_t* mca_pml_teg_proc_lookup_remote_peer(
ompi_communicator_t* comm,
int rank,
struct mca_ptl_base_module_t* ptl)
{
ompi_proc_t* proc = comm->c_remote_group->grp_proc_pointers[rank];
mca_pml_teg_proc_t* proc_pml =(mca_pml_teg_proc_t*) proc->proc_pml;
size_t i, size = mca_ptl_array_get_size(&proc_pml->proc_ptl_first);
mca_ptl_proc_t* proc_ptl = proc_pml->proc_ptl_first.ptl_procs;
for(i = 0; i < size; i++) {
if(proc_ptl->ptl == ptl) {
return proc_ptl->ptl_peer;
}
proc_ptl++;
}
return NULL;
}
#if defined(c_plusplus) || defined(__cplusplus)
}
#endif
#endif

42
ompi/mca/pml/teg/pml_teg_progress.c
Просмотреть файл

@ -1,42 +0,0 @@
/*
* Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
* University Research and Technology
* Corporation. All rights reserved.
* Copyright (c) 2004-2005 The University of Tennessee and The University
* of Tennessee Research Foundation. All rights
* reserved.
* Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
* University of Stuttgart. All rights reserved.
* Copyright (c) 2004-2005 The Regents of the University of California.
* All rights reserved.
* $COPYRIGHT$
*
* Additional copyrights may follow
*
* $HEADER$
*/
#include "ompi_config.h"
#include "pml_teg.h"
#include "pml_teg_sendreq.h"
int mca_pml_teg_progress(void)
{
mca_ptl_tstamp_t tstamp = 0;
size_t i;
int count = 0;
/*
* Progress each of the PTL modules
*/
for(i=0; i<mca_pml_teg.teg_num_ptl_progress; i++) {
int rc = mca_pml_teg.teg_ptl_progress[i](tstamp);
if(rc > 0) {
count += rc;
}
}
return count;
}

45
ompi/mca/pml/teg/pml_teg_ptl.c
Просмотреть файл

@ -1,45 +0,0 @@
/*
* Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
* University Research and Technology
* Corporation. All rights reserved.
* Copyright (c) 2004-2005 The University of Tennessee and The University
* of Tennessee Research Foundation. All rights
* reserved.
* Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
* University of Stuttgart. All rights reserved.
* Copyright (c) 2004-2005 The Regents of the University of California.
* All rights reserved.
* $COPYRIGHT$
*
* Additional copyrights may follow
*
* $HEADER$
*/
#include "ompi_config.h"
#include "pml_teg_ptl.h"
static void mca_pml_base_ptl_construct(mca_pml_base_ptl_t* ptl)
{
OBJ_CONSTRUCT(&ptl->ptl_cache, opal_list_t);
OBJ_CONSTRUCT(&ptl->ptl_cache_lock, opal_mutex_t);
ptl->ptl = NULL;
ptl->ptl_cache_size = 0;
ptl->ptl_cache_alloc = 0;
}
static void mca_pml_base_ptl_destruct(mca_pml_base_ptl_t* ptl)
{
OBJ_DESTRUCT(&ptl->ptl_cache);
OBJ_DESTRUCT(&ptl->ptl_cache_lock);
}
OBJ_CLASS_INSTANCE(
mca_pml_base_ptl_t,
opal_list_t,
mca_pml_base_ptl_construct,
mca_pml_base_ptl_destruct
);

46
ompi/mca/pml/teg/pml_teg_ptl.h
Просмотреть файл

@ -1,46 +0,0 @@
/*
* Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
* University Research and Technology
* Corporation. All rights reserved.
* Copyright (c) 2004-2005 The University of Tennessee and The University
* of Tennessee Research Foundation. All rights
* reserved.
* Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
* University of Stuttgart. All rights reserved.
* Copyright (c) 2004-2005 The Regents of the University of California.
* All rights reserved.
* $COPYRIGHT$
*
* Additional copyrights may follow
*
* $HEADER$
*/
#ifndef _MCA_PML_BASE_PTL_
#define _MCA_PML_BASE_PTL_
#include "ompi/mca/pml/pml.h"
#include "ompi/mca/ptl/ptl.h"
#include "opal/threads/condition.h"
#if defined(c_plusplus) || defined(__cplusplus)
extern "C" {
#endif
struct mca_pml_base_ptl_t {
opal_list_t ptl_cache; /**< cache of send requests */
size_t ptl_cache_size; /**< maximum size of cache */
size_t ptl_cache_alloc; /**< current number of allocated items */
opal_mutex_t ptl_cache_lock; /**< lock for queue access */
struct mca_ptl_base_module_t* ptl; /**< back pointer to ptl */
};
typedef struct mca_pml_base_ptl_t mca_pml_base_ptl_t;
OBJ_CLASS_DECLARATION(mca_pml_base_ptl_t);
#if defined(c_plusplus) || defined(__cplusplus)
}
#endif
#endif

93
ompi/mca/pml/teg/pml_teg_recvfrag.c
Просмотреть файл

@ -1,93 +0,0 @@
/*
* Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
* University Research and Technology
* Corporation. All rights reserved.
* Copyright (c) 2004-2005 The University of Tennessee and The University
* of Tennessee Research Foundation. All rights
* reserved.
* Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
* University of Stuttgart. All rights reserved.
* Copyright (c) 2004-2005 The Regents of the University of California.
* All rights reserved.
* $COPYRIGHT$
*
* Additional copyrights may follow
*
* $HEADER$
*/
/**
* @file
*/
#include "ompi_config.h"
#include "ompi/mca/pml/pml.h"
#include "pml_teg_recvfrag.h"
#include "pml_teg_proc.h"
OMPI_DECLSPEC extern opal_class_t mca_ptl_base_recv_frag_t_class;
/**
* Called by the PTL to match attempt a match for new fragments.
*
* @param ptl (IN) The PTL pointer
* @param frag (IN) Receive fragment descriptor.
* @param header (IN) Header corresponding to the receive fragment.
* @return OMPI_SUCCESS or error status on failure.
*/
bool mca_pml_teg_recv_frag_match(
mca_ptl_base_module_t* ptl,
mca_ptl_base_recv_frag_t* frag,
mca_ptl_base_match_header_t* header)
{
bool matched;
bool matches = false;
opal_list_t matched_frags;
if((matched = mca_ptl_base_match(header, frag, &matched_frags, &matches)) == false) {
frag = (matches ? (mca_ptl_base_recv_frag_t*)opal_list_remove_first(&matched_frags) : NULL);
}
while(NULL != frag) {
mca_ptl_base_module_t* ptl = frag->frag_base.frag_owner;
mca_ptl_base_recv_request_t *request = frag->frag_request;
mca_ptl_base_match_header_t *header = &frag->frag_base.frag_header.hdr_match;
/*
* Initialize request status.
*/
/* TODO request->req_recv.req_bytes_packed = header->hdr_msg_length; */
request->req_recv.req_base.req_ompi.req_status.MPI_SOURCE = header->hdr_src;
request->req_recv.req_base.req_ompi.req_status.MPI_TAG = header->hdr_tag;
/*
* If probe - signal request is complete - but don't notify PTL
*/
if(request->req_recv.req_base.req_type == MCA_PML_REQUEST_PROBE) {
ptl->ptl_recv_progress( ptl,
request,
header->hdr_msg_length,
header->hdr_msg_length );
matched = mca_pml_teg_recv_frag_match( ptl, frag, header );
} else {
/* if required - setup pointer to ptls peer */
if (NULL == frag->frag_base.frag_peer) {
frag->frag_base.frag_peer = mca_pml_teg_proc_lookup_remote_peer(
request->req_recv.req_base.req_comm,header->hdr_src,ptl);
}
MCA_PML_TEG_RECV_MATCHED( ptl, frag );
};
/* process any additional fragments that arrived out of order */
frag = (matches ? (mca_ptl_base_recv_frag_t*)opal_list_remove_first(&matched_frags) : NULL);
};
return matched;
}

64
ompi/mca/pml/teg/pml_teg_recvfrag.h
Просмотреть файл

@ -1,64 +0,0 @@
/*
* Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
* University Research and Technology
* Corporation. All rights reserved.
* Copyright (c) 2004-2005 The University of Tennessee and The University
* of Tennessee Research Foundation. All rights
* reserved.
* Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
* University of Stuttgart. All rights reserved.
* Copyright (c) 2004-2005 The Regents of the University of California.
* All rights reserved.
* $COPYRIGHT$
*
* Additional copyrights may follow
*
* $HEADER$
*/
/**
* @file
*/
#ifndef MCA_PML_TEG_RECVFRAG_H
#define MCA_PML_TEG_RECVFRAG_H
#include "ompi/mca/ptl/ptl.h"
#include "ompi/mca/pml/base/pml_base_recvreq.h"
#include "ompi/mca/ptl/base/ptl_base_recvfrag.h"
/**
* Called by the PTL to match attempt a match for new fragments.
*
* @param ptl (IN) The PTL pointer
* @param frag (IN) Receive fragment descriptor.
* @param header (IN) Header corresponding to the receive fragment.
* @return OMPI_SUCCESS or error status on failure.
*/
bool mca_pml_teg_recv_frag_match(
mca_ptl_base_module_t* ptl,
mca_ptl_base_recv_frag_t* frag,
mca_ptl_base_match_header_t* header
);
#define MCA_PML_TEG_RECV_MATCHED( ptl, frag ) \
do { \
mca_pml_base_recv_request_t* _request = (mca_pml_base_recv_request_t*)(frag)->frag_request; \
/* Now that we have the sender we can create the convertor. Additionally, we know */ \
/* that the required convertor should start at the position zero as we just match */ \
/* the first fragment. */ \
if( 0 != (_request)->req_bytes_packed ) { \
(_request)->req_base.req_proc = ompi_comm_peer_lookup( \
(_request)->req_base.req_comm, \
frag->frag_base.frag_header.hdr_match.hdr_src); \
ompi_convertor_copy_and_prepare_for_recv( \
(_request)->req_base.req_proc->proc_convertor, \
(_request)->req_base.req_datatype, \
(_request)->req_base.req_count, \
(_request)->req_base.req_addr, \
&((_request)->req_convertor) ); \
} \
ptl->ptl_matched( (ptl), (frag) ); /* notify ptl of match */ \
} while (0)
#endif

282
ompi/mca/pml/teg/pml_teg_recvreq.c
Просмотреть файл

@ -1,282 +0,0 @@
/*
* Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
* University Research and Technology
* Corporation. All rights reserved.
* Copyright (c) 2004-2005 The University of Tennessee and The University
* of Tennessee Research Foundation. All rights
* reserved.
* Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
* University of Stuttgart. All rights reserved.
* Copyright (c) 2004-2005 The Regents of the University of California.
* All rights reserved.
* $COPYRIGHT$
*
* Additional copyrights may follow
*
* $HEADER$
*/
#include "ompi_config.h"
#include "ompi/mca/pml/pml.h"
#include "ompi/mca/ptl/ptl.h"
#include "ompi/mca/ptl/base/ptl_base_comm.h"
#include "pml_teg_recvreq.h"
#include "pml_teg_recvfrag.h"
#include "pml_teg_sendreq.h"
static mca_ptl_base_recv_frag_t* mca_pml_teg_recv_request_match_specific_proc(
mca_ptl_base_recv_request_t* request, int proc);
static int mca_pml_teg_recv_request_fini(struct ompi_request_t** request)
{
MCA_PML_TEG_FINI(request);
return OMPI_SUCCESS;
}
static int mca_pml_teg_recv_request_free(struct ompi_request_t** request)
{
MCA_PML_TEG_FREE(request);
return OMPI_SUCCESS;
}
static int mca_pml_teg_recv_request_cancel(struct ompi_request_t* request, int complete)
{
mca_pml_base_request_t* teg_request = (mca_pml_base_request_t*)request;
ompi_communicator_t* ompi_comm = teg_request->req_comm;
mca_pml_ptl_comm_t* pml_comm = (mca_pml_ptl_comm_t*)ompi_comm->c_pml_comm;
if( true == request->req_complete ) { /* way to late to cancel this one */
return OMPI_SUCCESS;
}
/* The rest should be protected behind the match logic lock */
OPAL_THREAD_LOCK(&pml_comm->c_matching_lock);
if( OMPI_ANY_TAG == request->req_status.MPI_TAG ) { /* the match have not been already done */
if( teg_request->req_peer == OMPI_ANY_SOURCE ) {
opal_list_remove_item( &(pml_comm->c_wild_receives),
(opal_list_item_t*)request );
} else {
opal_list_remove_item( pml_comm->c_specific_receives + teg_request->req_peer,
(opal_list_item_t*)request );
}
}
OPAL_THREAD_UNLOCK(&pml_comm->c_matching_lock);
request->req_status._cancelled = true;
request->req_complete = true; /* mark it as completed so all the test/wait functions
* on this particular request will finish */
/* Now we have a problem if we are in a multi-threaded environment. We shou ld
* broadcast the condition on the request in order to allow the other threa ds
* to complete their test/wait functions.
*/
ompi_request_completed++;
if(ompi_request_waiting) {
opal_condition_broadcast(&ompi_request_cond);
}
return OMPI_SUCCESS;
}
static void mca_pml_teg_recv_request_construct(mca_ptl_base_recv_request_t* request)
{
request->req_recv.req_base.req_type = MCA_PML_REQUEST_RECV;
request->req_recv.req_base.req_ompi.req_fini = mca_pml_teg_recv_request_fini;
request->req_recv.req_base.req_ompi.req_free = mca_pml_teg_recv_request_free;
request->req_recv.req_base.req_ompi.req_cancel = mca_pml_teg_recv_request_cancel;
}
static void mca_pml_teg_recv_request_destruct(mca_ptl_base_recv_request_t* request)
{
}
OBJ_CLASS_INSTANCE(
mca_pml_teg_recv_request_t,
mca_ptl_base_recv_request_t,
mca_pml_teg_recv_request_construct,
mca_pml_teg_recv_request_destruct);
/*
* Update the recv request status to reflect the number of bytes
* received and actually delivered to the application.
*/
void mca_pml_teg_recv_request_progress(
struct mca_ptl_base_module_t* ptl,
mca_ptl_base_recv_request_t* req,
size_t bytes_received,
size_t bytes_delivered)
{
OPAL_THREAD_LOCK(&ompi_request_lock);
req->req_bytes_received += bytes_received;
req->req_bytes_delivered += bytes_delivered;
if (req->req_bytes_received >= req->req_recv.req_bytes_packed) {
/* initialize request status */
req->req_recv.req_base.req_ompi.req_status._count = req->req_bytes_delivered;
req->req_recv.req_base.req_pml_complete = true;
req->req_recv.req_base.req_ompi.req_complete = true;
ompi_request_completed++;
if(ompi_request_waiting) {
opal_condition_broadcast(&ompi_request_cond);
}
}
OPAL_THREAD_UNLOCK(&ompi_request_lock);
}
/*
* This routine is used to match a posted receive when the source process
* is specified.
*/
void mca_pml_teg_recv_request_match_specific(mca_ptl_base_recv_request_t* request)
{
ompi_communicator_t *comm = request->req_recv.req_base.req_comm;
mca_pml_ptl_comm_t* pml_comm = comm->c_pml_comm;
int req_peer = request->req_recv.req_base.req_peer;
mca_ptl_base_recv_frag_t* frag;
/* check for a specific match */
OPAL_THREAD_LOCK(&pml_comm->c_matching_lock);
/* assign sequence number */
request->req_recv.req_base.req_sequence = pml_comm->c_recv_seq++;
if (opal_list_get_size(&pml_comm->c_unexpected_frags[req_peer]) > 0 &&
(frag = mca_pml_teg_recv_request_match_specific_proc(request, req_peer)) != NULL) {
mca_ptl_base_module_t* ptl = frag->frag_base.frag_owner;
/* setup pointer to ptls peer */
if(NULL == frag->frag_base.frag_peer)
frag->frag_base.frag_peer = mca_pml_teg_proc_lookup_remote_peer(comm,req_peer,ptl);
OPAL_THREAD_UNLOCK(&pml_comm->c_matching_lock);
if( !((MCA_PML_REQUEST_IPROBE == request->req_recv.req_base.req_type) ||
(MCA_PML_REQUEST_PROBE == request->req_recv.req_base.req_type)) ) {
MCA_PML_TEG_RECV_MATCHED( ptl, frag );
}
return; /* match found */
}
/* We didn't find any matches. Record this irecv so we can match
* it when the message comes in.
*/
if(request->req_recv.req_base.req_type != MCA_PML_REQUEST_IPROBE) {
opal_list_append(pml_comm->c_specific_receives+req_peer, (opal_list_item_t*)request);
}
OPAL_THREAD_UNLOCK(&pml_comm->c_matching_lock);
}
/*
* this routine is used to try and match a wild posted receive - where
* wild is determined by the value assigned to the source process
*/
void mca_pml_teg_recv_request_match_wild(mca_ptl_base_recv_request_t* request)
{
ompi_communicator_t *comm = request->req_recv.req_base.req_comm;
mca_pml_ptl_comm_t* pml_comm = comm->c_pml_comm;
int proc_count = comm->c_remote_group->grp_proc_count;
int proc;
/*
* Loop over all the outstanding messages to find one that matches.
* There is an outer loop over lists of messages from each
* process, then an inner loop over the messages from the
* process.
*/
OPAL_THREAD_LOCK(&pml_comm->c_matching_lock);
/* assign sequence number */
request->req_recv.req_base.req_sequence = pml_comm->c_recv_seq++;
for (proc = 0; proc < proc_count; proc++) {
mca_ptl_base_recv_frag_t* frag;
/* continue if no frags to match */
if (opal_list_get_size(&pml_comm->c_unexpected_frags[proc]) == 0)
continue;
/* loop over messages from the current proc */
if ((frag = mca_pml_teg_recv_request_match_specific_proc(request, proc)) != NULL) {
mca_ptl_base_module_t* ptl = frag->frag_base.frag_owner;
/* if required - setup pointer to ptls peer */
if(NULL == frag->frag_base.frag_peer)
frag->frag_base.frag_peer = mca_pml_teg_proc_lookup_remote_peer(comm,proc,ptl);
OPAL_THREAD_UNLOCK(&pml_comm->c_matching_lock);
if( !((MCA_PML_REQUEST_IPROBE == request->req_recv.req_base.req_type) ||
(MCA_PML_REQUEST_PROBE == request->req_recv.req_base.req_type)) ) {
MCA_PML_TEG_RECV_MATCHED( ptl, frag );
}
return; /* match found */
}
}
/* We didn't find any matches. Record this irecv so we can match to
* it when the message comes in.
*/
if(request->req_recv.req_base.req_type != MCA_PML_REQUEST_IPROBE)
opal_list_append(&pml_comm->c_wild_receives, (opal_list_item_t*)request);
OPAL_THREAD_UNLOCK(&pml_comm->c_matching_lock);
}
/*
* this routine tries to match a posted receive. If a match is found,
* it places the request in the appropriate matched receive list.
*/
static mca_ptl_base_recv_frag_t* mca_pml_teg_recv_request_match_specific_proc(
mca_ptl_base_recv_request_t* request, int proc)
{
mca_pml_ptl_comm_t *pml_comm = request->req_recv.req_base.req_comm->c_pml_comm;
opal_list_t* unexpected_frags = pml_comm->c_unexpected_frags+proc;
mca_ptl_base_recv_frag_t* frag;
mca_ptl_base_match_header_t* header;
int tag = request->req_recv.req_base.req_tag;
if( OMPI_ANY_TAG == tag ) {
for (frag = (mca_ptl_base_recv_frag_t*)opal_list_get_first(unexpected_frags);
frag != (mca_ptl_base_recv_frag_t*)opal_list_get_end(unexpected_frags);
frag = (mca_ptl_base_recv_frag_t*)opal_list_get_next(frag)) {
header = &(frag->frag_base.frag_header.hdr_match);
/* check first frag - we assume that process matching has been done already */
if( header->hdr_tag >= 0 ) {
goto find_fragment;
}
}
} else {
for (frag = (mca_ptl_base_recv_frag_t*)opal_list_get_first(unexpected_frags);
frag != (mca_ptl_base_recv_frag_t*)opal_list_get_end(unexpected_frags);
frag = (mca_ptl_base_recv_frag_t*)opal_list_get_next(frag)) {
header = &(frag->frag_base.frag_header.hdr_match);
/* check first frag - we assume that process matching has been done already */
if ( tag == header->hdr_tag ) {
/* we assume that the tag is correct from MPI point of view (ie. >= 0 ) */
goto find_fragment;
}
}
}
return NULL;
find_fragment:
request->req_recv.req_bytes_packed = header->hdr_msg_length;
request->req_recv.req_base.req_ompi.req_status.MPI_TAG = header->hdr_tag;
request->req_recv.req_base.req_ompi.req_status.MPI_SOURCE = header->hdr_src;
if( !((MCA_PML_REQUEST_IPROBE == request->req_recv.req_base.req_type) ||
(MCA_PML_REQUEST_PROBE == request->req_recv.req_base.req_type)) ) {
opal_list_remove_item(unexpected_frags, (opal_list_item_t*)frag);
frag->frag_request = request;
} else {
/* it's a probe, therefore report it's completion */
mca_pml_teg_recv_request_progress( NULL, request, header->hdr_msg_length, header->hdr_msg_length );
}
return frag;
}

154
ompi/mca/pml/teg/pml_teg_recvreq.h
Просмотреть файл

@ -1,154 +0,0 @@
/*
* Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
* University Research and Technology
* Corporation. All rights reserved.
* Copyright (c) 2004-2005 The University of Tennessee and The University
* of Tennessee Research Foundation. All rights
* reserved.
* Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
* University of Stuttgart. All rights reserved.
* Copyright (c) 2004-2005 The Regents of the University of California.
* All rights reserved.
* $COPYRIGHT$
*
* Additional copyrights may follow
*
* $HEADER$
*/
/**
* @file
*/
#ifndef OMPI_PML_TEG_RECV_REQUEST_H
#define OMPI_PML_TEG_RECV_REQUEST_H
#include "pml_teg.h"
#include "pml_teg_proc.h"
#include "ompi/mca/ptl/base/ptl_base_recvreq.h"
#include "ompi/mca/ptl/base/ptl_base_recvfrag.h"
#if defined(c_plusplus) || defined(__cplusplus)
extern "C" {
#endif
typedef mca_ptl_base_recv_request_t mca_pml_teg_recv_request_t;
OBJ_CLASS_DECLARATION(mca_pml_teg_recv_request_t);
/**
* Allocate a recv request from the modules free list.
*
* @param rc (OUT) OMPI_SUCCESS or error status on failure.
* @return Receive request.
*/
#define MCA_PML_TEG_RECV_REQUEST_ALLOC(recvreq, rc) \
do { \
opal_list_item_t* item; \
OMPI_FREE_LIST_GET(&mca_pml_teg.teg_recv_requests, item, rc); \
recvreq = (mca_ptl_base_recv_request_t*)item; \
} while(0)
/**
* Initialize a recv request.
*/
#define MCA_PML_TEG_RECV_REQUEST_INIT( \
request, \
addr, \
count, \
datatype, \
src, \
tag, \
comm, \
persistent) \
{ \
MCA_PML_BASE_RECV_REQUEST_INIT( \
(&(request)->req_recv), \
addr, \
count, \
datatype, \
src, \
tag, \
comm, \
persistent \
); \
}
/**
* Return a recv request to the modules free list.
*
* @param request (IN) Receive request.
*/
#define MCA_PML_TEG_RECV_REQUEST_RETURN(request) \
do { \
MCA_PML_BASE_RECV_REQUEST_FINI( &request->req_recv ); \
OMPI_FREE_LIST_RETURN(&mca_pml_teg.teg_recv_requests, (opal_list_item_t*)request); \
} while(0)
/**
* Attempt to match the request against the unexpected fragment list
* for all source ranks w/in the communicator.
*
* @param request (IN) Request to match.
*/
void mca_pml_teg_recv_request_match_wild(mca_ptl_base_recv_request_t* request);
/**
* Attempt to match the request against the unexpected fragment list
* for a specific source rank.
*
* @param request (IN) Request to match.
*/
void mca_pml_teg_recv_request_match_specific(mca_ptl_base_recv_request_t* request);
/**
* Start an initialized request.
*
* @param request Receive request.
* @return OMPI_SUCESS or error status on failure.
*/
static inline int mca_pml_teg_recv_request_start(mca_ptl_base_recv_request_t* request)
{
/* init/re-init the request */
request->req_bytes_received = 0;
request->req_bytes_delivered = 0;
request->req_recv.req_base.req_pml_complete = false;
request->req_recv.req_base.req_ompi.req_complete = false;
request->req_recv.req_base.req_ompi.req_state = OMPI_REQUEST_ACTIVE;
/* always set the req_status.MPI_TAG to ANY_TAG before starting the request. This field
* is used on the cancel part in order to find out if the request has been matched or not.
*/
request->req_recv.req_base.req_ompi.req_status.MPI_TAG = OMPI_ANY_TAG;
request->req_recv.req_base.req_ompi.req_status.MPI_ERROR = OMPI_SUCCESS;
request->req_recv.req_base.req_ompi.req_status._cancelled = 0;
/* attempt to match posted recv */
if(request->req_recv.req_base.req_peer == OMPI_ANY_SOURCE) {
mca_pml_teg_recv_request_match_wild(request);
} else {
mca_pml_teg_recv_request_match_specific(request);
}
return OMPI_SUCCESS;
}
/**
* Update status of a recv request based on the completion status of
* the receive fragment.
*
* @param ptl (IN) The PTL pointer.
* @param request (IN) Receive request.
* @param bytes_received (IN) Bytes received from peer.
* @param bytes_delivered (IN) Bytes delivered to application.
*/
void mca_pml_teg_recv_request_progress(
struct mca_ptl_base_module_t* ptl,
mca_ptl_base_recv_request_t* request,
size_t bytes_received,
size_t bytes_delivered
);
#if defined(c_plusplus) || defined(__cplusplus)
}
#endif
#endif

220
ompi/mca/pml/teg/pml_teg_sendreq.c
Просмотреть файл

@ -1,220 +0,0 @@
/*
* Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
* University Research and Technology
* Corporation. All rights reserved.
* Copyright (c) 2004-2005 The University of Tennessee and The University
* of Tennessee Research Foundation. All rights
* reserved.
* Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
* University of Stuttgart. All rights reserved.
* Copyright (c) 2004-2005 The Regents of the University of California.
* All rights reserved.
* $COPYRIGHT$
*
* Additional copyrights may follow
*
* $HEADER$
*/
/*%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%*/
#include "ompi_config.h"
#include "ompi/constants.h"
#include "ompi/mca/pml/pml.h"
#include "ompi/mca/ptl/ptl.h"
#include "pml_teg.h"
#include "pml_teg_proc.h"
#include "pml_teg_sendreq.h"
#include "pml_teg_recvreq.h"
static int mca_pml_teg_send_request_fini(struct ompi_request_t** request)
{
MCA_PML_TEG_FINI(request);
return OMPI_SUCCESS;
}
static int mca_pml_teg_send_request_free(struct ompi_request_t** request)
{
MCA_PML_TEG_FREE(request);
return OMPI_SUCCESS;
}
static int mca_pml_teg_send_request_cancel(struct ompi_request_t* request, int complete)
{
/* we dont cancel send requests by now */
return OMPI_SUCCESS;
}
static void mca_pml_teg_send_request_construct(mca_pml_teg_send_request_t* req)
{
req->req_cached = false;
req->req_send.req_base.req_ompi.req_fini = mca_pml_teg_send_request_fini;
req->req_send.req_base.req_ompi.req_free = mca_pml_teg_send_request_free;
req->req_send.req_base.req_ompi.req_cancel = mca_pml_teg_send_request_cancel;
}
static void mca_pml_teg_send_request_destruct(mca_pml_teg_send_request_t* req)
{
}
OBJ_CLASS_INSTANCE(
mca_pml_teg_send_request_t,
mca_ptl_base_send_request_t,
mca_pml_teg_send_request_construct,
mca_pml_teg_send_request_destruct);
/**
* Schedule message delivery across potentially multiple PTLs.
*
* @param request (IN) Request to schedule
* @return status Error status
*
*/
int mca_pml_teg_send_request_schedule(mca_ptl_base_send_request_t* req)
{
ompi_proc_t *proc;
mca_pml_teg_proc_t* proc_pml;
int send_count = 0;
size_t bytes_remaining;
size_t num_ptl_avail;
size_t num_ptl;
/*
* Only allow one thread in this routine for a given request.
* However, we cannot block callers on a mutex, so simply keep track
* of the number of times the routine has been called and run through
* the scheduling logic once for every call.
*/
if(OPAL_THREAD_ADD32(&req->req_lock,1) == 1) {
proc = ompi_comm_peer_lookup(req->req_send.req_base.req_comm, req->req_send.req_base.req_peer);
proc_pml = (mca_pml_teg_proc_t*) proc->proc_pml;
do {
/* allocate remaining bytes to PTLs */
bytes_remaining = req->req_send.req_bytes_packed - req->req_offset;
num_ptl_avail = proc_pml->proc_ptl_next.ptl_size;
num_ptl = 0;
while(bytes_remaining > 0 && num_ptl++ < num_ptl_avail) {
mca_ptl_proc_t* ptl_proc = mca_ptl_array_get_next(&proc_pml->proc_ptl_next);
mca_ptl_base_module_t* ptl = ptl_proc->ptl;
int rc;
/* if this is the last PTL that is available to use, or the number of
* bytes remaining in the message is less than the PTLs minimum fragment
* size, then go ahead and give the rest of the message to this PTL.
*/
size_t bytes_to_frag;
if(num_ptl == num_ptl_avail || bytes_remaining < ptl->ptl_min_frag_size) {
bytes_to_frag = bytes_remaining;
/* otherwise attempt to give the PTL a percentage of the message
* based on a weighting factor. for simplicity calculate this as
* a percentage of the overall message length (regardless of amount
* previously assigned)
*/
} else {
bytes_to_frag = (ptl_proc->ptl_weight * bytes_remaining) / 100;
}
/* makes sure that we don't exceed ptl_max_frag_size */
if(ptl->ptl_max_frag_size != 0 && bytes_to_frag > ptl->ptl_max_frag_size)
bytes_to_frag = ptl->ptl_max_frag_size;
rc = ptl->ptl_put(ptl, ptl_proc->ptl_peer, req, req->req_offset, bytes_to_frag, 0);
if(rc == OMPI_SUCCESS) {
send_count++;
bytes_remaining = req->req_send.req_bytes_packed - req->req_offset;
}
}
/* unable to complete send - queue for later */
if(send_count == 0) {
OPAL_THREAD_LOCK(&mca_pml_teg.teg_lock);
opal_list_append(&mca_pml_teg.teg_send_pending, (opal_list_item_t*)req);
OPAL_THREAD_UNLOCK(&mca_pml_teg.teg_lock);
req->req_lock = 0;
return OMPI_ERR_OUT_OF_RESOURCE;
}
/* fragments completed while scheduling - so retry */
} while(OPAL_THREAD_ADD32(&req->req_lock,-1) > 0);
/* free the request if completed while in the scheduler */
if (req->req_send.req_base.req_free_called && req->req_send.req_base.req_pml_complete) {
MCA_PML_TEG_FREE((ompi_request_t**)&req);
}
}
return OMPI_SUCCESS;
}
/**
* Update the status of the send request to reflect the number of bytes
* "actually" sent (and acknowledged). This should be called by the
* lower layer PTL after the fragment is actually delivered and has been
* acknowledged (if required). Note that this routine should NOT be called
* directly by the PTL, a function pointer is setup on the PTL at init to
* enable upcalls into the PML w/out directly linking to a specific PML
* implementation.
*/
void mca_pml_teg_send_request_progress(
struct mca_ptl_base_module_t* ptl,
mca_ptl_base_send_request_t* req,
size_t bytes_sent)
{
bool schedule = false;
OPAL_THREAD_LOCK(&ompi_request_lock);
req->req_bytes_sent += bytes_sent;
if (req->req_bytes_sent >= req->req_send.req_bytes_packed) {
req->req_send.req_base.req_pml_complete = true;
if (req->req_send.req_base.req_ompi.req_complete == false) {
req->req_send.req_base.req_ompi.req_status.MPI_SOURCE = req->req_send.req_base.req_comm->c_my_rank;
req->req_send.req_base.req_ompi.req_status.MPI_TAG = req->req_send.req_base.req_tag;
req->req_send.req_base.req_ompi.req_status.MPI_ERROR = OMPI_SUCCESS;
req->req_send.req_base.req_ompi.req_status._count = req->req_bytes_sent;
req->req_send.req_base.req_ompi.req_complete = true;
ompi_request_completed++;
if(ompi_request_waiting) {
opal_condition_broadcast(&ompi_request_cond);
}
} else if(req->req_send.req_base.req_free_called) {
/* don't free the request if in the scheduler */
if(req->req_lock == 0) {
MCA_PML_TEG_FREE((ompi_request_t**)&req);
}
} else if (req->req_send.req_send_mode == MCA_PML_BASE_SEND_BUFFERED) {
mca_pml_base_bsend_request_fini((ompi_request_t*)req);
}
/* test to see if we have scheduled the entire request */
} else if (req->req_offset < req->req_send.req_bytes_packed) {
schedule = true;
}
OPAL_THREAD_UNLOCK(&ompi_request_lock);
/* schedule remaining fragments of this request */
if(schedule) {
mca_pml_teg_send_request_schedule(req);
}
/* check for pending requests that need to be progressed */
while(opal_list_get_size(&mca_pml_teg.teg_send_pending) != 0) {
OPAL_THREAD_LOCK(&mca_pml_teg.teg_lock);
req = (mca_ptl_base_send_request_t*)opal_list_remove_first(&mca_pml_teg.teg_send_pending);
OPAL_THREAD_UNLOCK(&mca_pml_teg.teg_lock);
if(req == NULL)
break;
if(mca_pml_teg_send_request_schedule(req) != OMPI_SUCCESS)
break;
}
}

215
ompi/mca/pml/teg/pml_teg_sendreq.h
Просмотреть файл

@ -1,215 +0,0 @@
/*
* Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
* University Research and Technology
* Corporation. All rights reserved.
* Copyright (c) 2004-2005 The University of Tennessee and The University
* of Tennessee Research Foundation. All rights
* reserved.
* Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
* University of Stuttgart. All rights reserved.
* Copyright (c) 2004-2005 The Regents of the University of California.
* All rights reserved.
* $COPYRIGHT$
*
* Additional copyrights may follow
*
* $HEADER$
*/
/**
* @file
*/
#ifndef OMPI_PML_TEG_SEND_REQUEST_H
#define OMPI_PML_TEG_SEND_REQUEST_H
#include "ompi/mca/ptl/ptl.h"
#include "ompi/mca/ptl/base/ptl_base_sendreq.h"
#include "ompi/mca/ptl/base/ptl_base_sendfrag.h"
#include "ompi/mca/ptl/base/ptl_base_comm.h"
#include "pml_teg_proc.h"
#include "pml_teg_ptl.h"
#if defined(c_plusplus) || defined(__cplusplus)
extern "C" {
#endif
typedef mca_ptl_base_send_request_t mca_pml_teg_send_request_t;
OBJ_CLASS_DECLARATION(mca_pml_teg_send_request_t);
#define MCA_PML_TEG_SEND_REQUEST_ALLOC( \
comm, \
dst, \
sendreq, \
rc) \
{ \
mca_pml_teg_proc_t *proc = \
(mca_pml_teg_proc_t*) mca_pml_teg_proc_lookup_remote(comm,dst); \
mca_ptl_proc_t* ptl_proc; \
mca_pml_base_ptl_t* ptl_base; \
\
if(NULL == proc) { \
return OMPI_ERR_OUT_OF_RESOURCE; \
} \
OPAL_THREAD_SCOPED_LOCK(&proc->base.proc_lock, \
(ptl_proc = mca_ptl_array_get_next(&proc->proc_ptl_first))); \
ptl_base = ptl_proc->ptl_base; \
/* \
* check to see if there is a cache of send requests associated with \
* this ptl - if so try the allocation from there. \
*/ \
if(NULL != ptl_base) { \
OPAL_THREAD_LOCK(&ptl_base->ptl_cache_lock); \
sendreq = (mca_pml_teg_send_request_t*) \
opal_list_remove_first(&ptl_base->ptl_cache); \
if(NULL != sendreq) { \
OPAL_THREAD_UNLOCK(&ptl_base->ptl_cache_lock); \
rc = OMPI_SUCCESS; \
} else if (ptl_base->ptl_cache_alloc < ptl_base->ptl_cache_size) { \
/* \
* allocate an additional request to the cache \
*/ \
mca_ptl_base_module_t* ptl = ptl_base->ptl; \
opal_list_item_t* item; \
OMPI_FREE_LIST_WAIT(&mca_pml_teg.teg_send_requests, item, rc); \
sendreq = (mca_pml_teg_send_request_t*)item; \
sendreq->req_ptl = ptl; \
if(ptl->ptl_request_init(ptl, sendreq) == OMPI_SUCCESS) { \
sendreq->req_cached = true; \
ptl_base->ptl_cache_alloc++; \
} \
OPAL_THREAD_UNLOCK(&ptl_base->ptl_cache_lock); \
} else { \
/* \
* take a request from the global pool \
*/ \
opal_list_item_t* item; \
OPAL_THREAD_UNLOCK(&ptl_base->ptl_cache_lock); \
OMPI_FREE_LIST_WAIT(&mca_pml_teg.teg_send_requests, item, rc); \
sendreq = (mca_pml_teg_send_request_t*)item; \
sendreq->req_ptl = ptl_proc->ptl; \
} \
\
/* otherwise - take the allocation from the global list */ \
} else { \
opal_list_item_t* item; \
OMPI_FREE_LIST_WAIT(&mca_pml_teg.teg_send_requests, item, rc); \
sendreq = (mca_pml_teg_send_request_t*)item; \
sendreq->req_ptl = ptl_proc->ptl; \
} \
/* update request to point to current peer */ \
sendreq->req_peer = ptl_proc->ptl_peer; \
sendreq->req_send.req_base.req_proc = proc->base.proc_ompi; \
}
#define MCA_PML_TEG_SEND_REQUEST_INIT( request, \
addr, \
count, \
datatype, \
peer, \
tag, \
comm, \
mode, \
persistent) \
{ \
MCA_PML_BASE_SEND_REQUEST_INIT((&request->req_send), \
addr, \
count, \
datatype, \
peer, \
tag, \
comm, \
mode, \
persistent \
); \
}
#define MCA_PML_TEG_SEND_REQUEST_RETURN(sendreq) \
{ \
mca_ptl_base_module_t* ptl = (sendreq)->req_ptl; \
mca_pml_base_ptl_t* ptl_base = ptl->ptl_base; \
\
/* Let the base handle the reference counts */ \
MCA_PML_BASE_SEND_REQUEST_FINI((&sendreq->req_send)); \
\
/* \
* If there is a cache associated with the ptl - first attempt \
* to return the send descriptor to the cache. \
*/ \
if(NULL != ptl->ptl_base && (sendreq)->req_cached) { \
OPAL_THREAD_LOCK(&ptl_base->ptl_cache_lock); \
opal_list_prepend(&ptl_base->ptl_cache, \
(opal_list_item_t*)sendreq); \
OPAL_THREAD_UNLOCK(&ptl_base->ptl_cache_lock); \
} else { \
OMPI_FREE_LIST_RETURN( \
&mca_pml_teg.teg_send_requests, (opal_list_item_t*)sendreq); \
} \
}
/**
* Start a send request.
*/
#define MCA_PML_TEG_SEND_REQUEST_START(req, rc) \
{ \
mca_ptl_base_module_t* ptl = req->req_ptl; \
size_t first_fragment_size = ptl->ptl_first_frag_size; \
int flags; \
\
req->req_lock = 0; \
req->req_bytes_sent = 0; \
req->req_peer_match.lval = 0; \
req->req_peer_addr.lval = 0; \
req->req_peer_size = 0; \
req->req_offset = 0; \
req->req_send.req_base.req_pml_complete = false; \
req->req_send.req_base.req_ompi.req_complete = false; \
req->req_send.req_base.req_ompi.req_state = OMPI_REQUEST_ACTIVE; \
req->req_send.req_base.req_sequence = mca_pml_ptl_comm_send_sequence( \
req->req_send.req_base.req_comm->c_pml_comm, req->req_send.req_base.req_peer); \
\
/* handle buffered send */ \
if(req->req_send.req_send_mode == MCA_PML_BASE_SEND_BUFFERED) { \
mca_pml_base_bsend_request_start(&req->req_send.req_base.req_ompi); \
} \
\
/* start the first fragment */ \
if (first_fragment_size == 0 || \
req->req_send.req_bytes_packed <= first_fragment_size) { \
first_fragment_size = req->req_send.req_bytes_packed; \
flags = (req->req_send.req_send_mode == MCA_PML_BASE_SEND_SYNCHRONOUS) ? \
MCA_PTL_FLAGS_ACK : 0; \
} else { \
/* require match for first fragment of a multi-fragment */ \
flags = MCA_PTL_FLAGS_ACK; \
} \
rc = ptl->ptl_send(ptl, req->req_peer, req, 0, first_fragment_size, \
flags); \
}
/**
* Schedule any data that was not delivered in the first fragment
* across the available PTLs.
*/
int mca_pml_teg_send_request_schedule(mca_ptl_base_send_request_t* req);
/**
* Update the request to reflect the number of bytes delivered. If this
* was the first fragment - schedule the rest of the data.
*/
void mca_pml_teg_send_request_progress(
struct mca_ptl_base_module_t* ptl,
mca_ptl_base_send_request_t* send_request,
size_t bytes_sent
);
#if defined(c_plusplus) || defined(__cplusplus)
}
#endif
#endif

124
ompi/mca/pml/teg/pml_teg_start.c
Просмотреть файл

@ -1,124 +0,0 @@
/*
* Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
* University Research and Technology
* Corporation. All rights reserved.
* Copyright (c) 2004-2005 The University of Tennessee and The University
* of Tennessee Research Foundation. All rights
* reserved.
* Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
* University of Stuttgart. All rights reserved.
* Copyright (c) 2004-2005 The Regents of the University of California.
* All rights reserved.
* $COPYRIGHT$
*
* Additional copyrights may follow
*
* $HEADER$
*/
#include "ompi_config.h"
#include "pml_teg.h"
#include "pml_teg_recvreq.h"
#include "pml_teg_sendreq.h"
int mca_pml_teg_start(size_t count, ompi_request_t** requests)
{
int rc;
size_t i;
for(i=0; i<count; i++) {
mca_pml_base_request_t *pml_request = (mca_pml_base_request_t*)requests[i];
if(NULL == pml_request)
continue;
/* If the persistent request is currently active - obtain the
* request lock and verify the status is incomplete. if the
* pml layer has not completed the request - mark the request
* as free called - so that it will be freed when the request
* completes - and create a new request.
*/
switch(pml_request->req_ompi.req_state) {
case OMPI_REQUEST_INACTIVE:
if(pml_request->req_pml_complete == true)
break;
/* otherwise fall through */
case OMPI_REQUEST_ACTIVE: {
ompi_request_t *request;
OPAL_THREAD_LOCK(&ompi_request_lock);
if (pml_request->req_pml_complete == false) {
/* free request after it completes */
pml_request->req_free_called = true;
} else {
/* can reuse the existing request */
OPAL_THREAD_UNLOCK(&ompi_request_lock);
break;
}
/* allocate a new request */
switch(pml_request->req_type) {
case MCA_PML_REQUEST_SEND: {
mca_pml_base_send_mode_t sendmode =
((mca_pml_base_send_request_t*)pml_request)->req_send_mode;
rc = mca_pml_teg_isend_init(
pml_request->req_addr,
pml_request->req_count,
pml_request->req_datatype,
pml_request->req_peer,
pml_request->req_tag,
sendmode,
pml_request->req_comm,
&request);
break;
}
case MCA_PML_REQUEST_RECV:
rc = mca_pml_teg_irecv_init(
pml_request->req_addr,
pml_request->req_count,
pml_request->req_datatype,
pml_request->req_peer,
pml_request->req_tag,
pml_request->req_comm,
&request);
break;
default:
rc = OMPI_ERR_REQUEST;
break;
}
OPAL_THREAD_UNLOCK(&ompi_request_lock);
if(OMPI_SUCCESS != rc)
return rc;
pml_request = (mca_pml_base_request_t*)request;
requests[i] = request;
break;
}
default:
return OMPI_ERR_REQUEST;
}
/* start the request */
switch(pml_request->req_type) {
case MCA_PML_REQUEST_SEND:
{
mca_ptl_base_send_request_t* sendreq = (mca_ptl_base_send_request_t*)pml_request;
MCA_PML_TEG_SEND_REQUEST_START(sendreq, rc);
if(rc != OMPI_SUCCESS)
return rc;
break;
}
case MCA_PML_REQUEST_RECV:
{
mca_ptl_base_recv_request_t* recvreq = (mca_ptl_base_recv_request_t*)pml_request;
if((rc = mca_pml_teg_recv_request_start(recvreq)) != OMPI_SUCCESS)
return rc;
break;
}
default:
return OMPI_ERR_REQUEST;
}
}
return OMPI_SUCCESS;
}

1
ompi/mca/pml/teg/post_configure.sh
Просмотреть файл

@ -1 +0,0 @@
DIRECT_CALL_HEADER="mca/pml/teg/pml_teg.h"

54
ompi/mca/pml/uniq/Makefile.am
Просмотреть файл

@ -1,54 +0,0 @@
#
# Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
# University Research and Technology
# Corporation. All rights reserved.
# Copyright (c) 2004-2005 The University of Tennessee and The University
# of Tennessee Research Foundation. All rights
# reserved.
# Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
# University of Stuttgart. All rights reserved.
# Copyright (c) 2004-2005 The Regents of the University of California.
# All rights reserved.
# $COPYRIGHT$
#
# Additional copyrights may follow
#
# $HEADER$
#
# Use the top-level Makefile.options
# Make the output library in this directory, and name it either
# mca_<type>_<name>.la (for DSO builds) or libmca_<type>_<name>.la
# (for static builds).
if OMPI_BUILD_pml_uniq_DSO
component_noinst =
component_install = mca_pml_uniq.la
else
component_noinst = libmca_pml_uniq.la
component_install =
endif
mcacomponentdir = $(libdir)/openmpi
mcacomponent_LTLIBRARIES = $(component_install)
mca_pml_uniq_la_SOURCES = $(pml_uniq_la_sources)
mca_pml_uniq_la_LIBADD = \
$(top_ompi_builddir)/ompi/libmpi.la \
$(top_ompi_builddir)/orte/liborte.la \
$(top_ompi_builddir)/opal/libopal.la
mca_pml_uniq_la_LDFLAGS = -module -avoid-version
noinst_LTLIBRARIES = $(component_noinst)
libmca_pml_uniq_la_SOURCES = $(pml_uniq_la_sources)
libmca_pml_uniq_la_LIBADD =
libmca_pml_uniq_la_LDFLAGS = -module -avoid-version
pml_uniq_la_sources = pml_uniq.c pml_uniq.h pml_uniq_cancel.c pml_uniq_component.c \
pml_uniq_component.h pml_uniq_iprobe.c pml_uniq_irecv.c pml_uniq_isend.c \
pml_uniq_ptl.c pml_uniq_ptl.h pml_uniq_proc.c pml_uniq_proc.h pml_uniq_progress.c \
pml_uniq_recvfrag.c pml_uniq_recvfrag.h pml_uniq_recvreq.c pml_uniq_recvreq.h \
pml_uniq_sendreq.c pml_uniq_sendreq.h pml_uniq_start.c

24
ompi/mca/pml/uniq/configure.params
Просмотреть файл

@ -1,24 +0,0 @@
# -*- shell-script -*-
#
# Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
# University Research and Technology
# Corporation. All rights reserved.
# Copyright (c) 2004-2005 The University of Tennessee and The University
# of Tennessee Research Foundation. All rights
# reserved.
# Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
# University of Stuttgart. All rights reserved.
# Copyright (c) 2004-2005 The Regents of the University of California.
# All rights reserved.
# $COPYRIGHT$
#
# Additional copyrights may follow
#
# $HEADER$
#
# Specific to this module
PARAM_INIT_FILE=pml_uniq.c
PARAM_CONFIG_HEADER_FILE="uniq_config.h"
PARAM_CONFIG_FILES="Makefile"

386
ompi/mca/pml/uniq/pml_uniq.c
Просмотреть файл

@ -1,386 +0,0 @@
/* -*- Mode: C; c-basic-offset:4 ; -*- */
/*
* Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
* University Research and Technology
* Corporation. All rights reserved.
* Copyright (c) 2004-2005 The University of Tennessee and The University
* of Tennessee Research Foundation. All rights
* reserved.
* Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
* University of Stuttgart. All rights reserved.
* Copyright (c) 2004-2005 The Regents of the University of California.
* All rights reserved.
* $COPYRIGHT$
*
* Additional copyrights may follow
*
* $HEADER$
*/
#include "ompi_config.h"
#include <stdlib.h>
#include <string.h>
#include "ompi/class/ompi_bitmap.h"
#include "ompi/mca/pml/pml.h"
#include "ompi/mca/ptl/ptl.h"
#include "ompi/mca/ptl/base/base.h"
#include "ompi/mca/ptl/base/ptl_base_comm.h"
#include "ompi/mca/ptl/base/ptl_base_header.h"
#include "ompi/mca/ptl/base/ptl_base_recvfrag.h"
#include "ompi/mca/ptl/base/ptl_base_sendfrag.h"
#include "pml_uniq.h"
#include "pml_uniq_component.h"
#include "pml_uniq_proc.h"
#include "pml_uniq_ptl.h"
#include "pml_uniq_recvreq.h"
#include "pml_uniq_sendreq.h"
#include "pml_uniq_recvfrag.h"
mca_pml_uniq_t mca_pml_uniq = {
{
mca_pml_uniq_add_procs,
mca_pml_uniq_del_procs,
mca_pml_uniq_enable,
mca_pml_uniq_progress,
mca_pml_uniq_add_comm,
mca_pml_uniq_del_comm,
mca_pml_uniq_irecv_init,
mca_pml_uniq_irecv,
mca_pml_uniq_recv,
mca_pml_uniq_isend_init,
mca_pml_uniq_isend,
mca_pml_uniq_send,
mca_pml_uniq_iprobe,
mca_pml_uniq_probe,
mca_pml_uniq_start,
32768,
(0x7fffffff) /* XXX should be INT_MAX, as in ob1 */
}
};
int mca_pml_uniq_add_comm(ompi_communicator_t* comm)
{
/* allocate pml specific comm data */
mca_pml_ptl_comm_t* pml_comm = OBJ_NEW(mca_pml_ptl_comm_t);
if (NULL == pml_comm) {
return OMPI_ERR_OUT_OF_RESOURCE;
}
mca_pml_ptl_comm_init_size(pml_comm, comm->c_remote_group->grp_proc_count);
comm->c_pml_comm = pml_comm;
return OMPI_SUCCESS;
}
int mca_pml_uniq_del_comm(ompi_communicator_t* comm)
{
OBJ_RELEASE(comm->c_pml_comm);
comm->c_pml_comm = NULL; /* make sure it's set to NULL */
return OMPI_SUCCESS;
}
static int ptl_exclusivity_compare(const void* arg1, const void* arg2)
{
mca_ptl_base_module_t* ptl1 = *(struct mca_ptl_base_module_t**)arg1;
mca_ptl_base_module_t* ptl2 = *(struct mca_ptl_base_module_t**)arg2;
if( ptl1->ptl_exclusivity > ptl2->ptl_exclusivity ) {
return -1;
} else if (ptl1->ptl_exclusivity == ptl2->ptl_exclusivity ) {
return 0;
} else {
return 1;
}
}
static int mca_pml_uniq_add_ptls(void)
{
/* build an array of ptls and ptl modules */
mca_ptl_base_selected_module_t* selected_ptl;
size_t num_ptls = opal_list_get_size(&mca_ptl_base_modules_initialized);
size_t cache_bytes = 0;
mca_pml_uniq.uniq_num_ptl_modules = 0;
mca_pml_uniq.uniq_num_ptl_progress = 0;
mca_pml_uniq.uniq_num_ptl_components = 0;
mca_pml_uniq.uniq_ptl_modules = (mca_ptl_base_module_t **)malloc(sizeof(mca_ptl_base_module_t*) * num_ptls);
mca_pml_uniq.uniq_ptl_progress = (mca_ptl_base_component_progress_fn_t*)malloc(sizeof(mca_ptl_base_component_progress_fn_t) * num_ptls);
mca_pml_uniq.uniq_ptl_components = (mca_ptl_base_component_t **)malloc(sizeof(mca_ptl_base_component_t*) * num_ptls);
if (NULL == mca_pml_uniq.uniq_ptl_modules ||
NULL == mca_pml_uniq.uniq_ptl_progress ||
NULL == mca_pml_uniq.uniq_ptl_components) {
return OMPI_ERR_OUT_OF_RESOURCE;
}
for(selected_ptl = (mca_ptl_base_selected_module_t*)
opal_list_get_first(&mca_ptl_base_modules_initialized);
selected_ptl != (mca_ptl_base_selected_module_t*)
opal_list_get_end(&mca_ptl_base_modules_initialized);
selected_ptl = (mca_ptl_base_selected_module_t*)opal_list_get_next(selected_ptl)) {
mca_ptl_base_module_t *ptl = selected_ptl->pbsm_module;
size_t i;
mca_pml_uniq.uniq_ptl_modules[mca_pml_uniq.uniq_num_ptl_modules++] = ptl;
for(i=0; i < mca_pml_uniq.uniq_num_ptl_components; i++) {
if(mca_pml_uniq.uniq_ptl_components[i] == ptl->ptl_component) {
break;
}
}
if(i == mca_pml_uniq.uniq_num_ptl_components) {
mca_pml_uniq.uniq_ptl_components[mca_pml_uniq.uniq_num_ptl_components++] = ptl->ptl_component;
}
/*
*setup ptl
*/
/* set pointer to fragment matching logic routine, if this
* not already set by the ptl
*/
if(NULL == ptl->ptl_match) {
ptl->ptl_match = mca_pml_uniq_recv_frag_match;
}
ptl->ptl_send_progress = mca_pml_uniq_send_request_progress;
ptl->ptl_recv_progress = mca_pml_uniq_recv_request_progress;
ptl->ptl_stack = ptl;
ptl->ptl_base = NULL;
/* find maximum required size for cache */
if(ptl->ptl_cache_bytes > cache_bytes) {
cache_bytes = ptl->ptl_cache_bytes;
}
}
/* setup send fragments based on largest required send request */
ompi_free_list_init( &mca_pml_uniq.uniq_send_requests,
sizeof(mca_pml_uniq_send_request_t) + cache_bytes,
OBJ_CLASS(mca_pml_uniq_send_request_t),
mca_pml_uniq.uniq_free_list_num,
mca_pml_uniq.uniq_free_list_max,
mca_pml_uniq.uniq_free_list_inc,
NULL );
/* sort ptl list by exclusivity */
qsort(mca_pml_uniq.uniq_ptl_modules, mca_pml_uniq.uniq_num_ptl_modules, sizeof(struct mca_ptl_t*), ptl_exclusivity_compare);
return OMPI_SUCCESS;
}
/*
* Called by the base PML in order to notify the PMLs about their selected status. After the init pass,
* the base module will choose one PML (depending on informations provided by the init function) and then
* it will call the pml_enable function with true (for the selected one) and with false for all the
* others. The selected one can then pass control information through to all PTL modules.
*/
int mca_pml_uniq_enable(bool enable)
{
size_t i;
int value = enable;
/* If I'm not selected then prepare for close */
if( false == enable ) return OMPI_SUCCESS;
/* recv requests */
ompi_free_list_init( &mca_pml_uniq.uniq_recv_requests,
sizeof(mca_pml_uniq_recv_request_t),
OBJ_CLASS(mca_pml_uniq_recv_request_t),
mca_pml_uniq.uniq_free_list_num,
mca_pml_uniq.uniq_free_list_max,
mca_pml_uniq.uniq_free_list_inc,
NULL );
/* Grab all the PTLs and prepare them */
mca_pml_uniq_add_ptls();
/* and now notify them about the status */
for(i=0; i < mca_pml_uniq.uniq_num_ptl_components; i++) {
if(NULL != mca_pml_uniq.uniq_ptl_components[i]->ptlm_control) {
int rc = mca_pml_uniq.uniq_ptl_components[i]->ptlm_control(MCA_PTL_ENABLE,&value,sizeof(value));
if(rc != OMPI_SUCCESS)
return rc;
}
}
return OMPI_SUCCESS;
}
/*
* For each proc setup a datastructure that indicates the PTLs
* that can be used to reach the destination.
*
*/
int mca_pml_uniq_add_procs(ompi_proc_t** procs, size_t nprocs)
{
size_t p;
ompi_bitmap_t reachable;
struct mca_ptl_base_peer_t** ptl_peers = NULL;
int rc;
size_t p_index;
if(nprocs == 0)
return OMPI_SUCCESS;
OBJ_CONSTRUCT(&reachable, ompi_bitmap_t);
rc = ompi_bitmap_init(&reachable, nprocs);
if(OMPI_SUCCESS != rc)
return rc;
/* attempt to add all procs to each ptl */
ptl_peers = (struct mca_ptl_base_peer_t **)malloc(nprocs * sizeof(struct mca_ptl_base_peer_t*));
for(p_index = 0; p_index < mca_pml_uniq.uniq_num_ptl_modules; p_index++) {
mca_ptl_base_module_t* ptl = mca_pml_uniq.uniq_ptl_modules[p_index];
int ptl_inuse = 0;
/* if the ptl can reach the destination proc it sets the
* corresponding bit (proc index) in the reachable bitmap
* and can return addressing information for each proc
* that is passed back to the ptl on data transfer calls
*/
ompi_bitmap_clear_all_bits(&reachable);
memset(ptl_peers, 0, nprocs * sizeof(struct mca_ptl_base_peer_t*));
rc = ptl->ptl_add_procs(ptl, nprocs, procs, ptl_peers, &reachable);
if(OMPI_SUCCESS != rc) {
free(ptl_peers);
return rc;
}
/* for each proc that is reachable - add the ptl to the procs array(s) */
for(p=0; p < nprocs; p++) {
ompi_proc_t *proc;
mca_pml_uniq_proc_t* proc_pml;
if( !ompi_bitmap_is_set_bit(&reachable, p) ) continue;
proc = procs[p];
proc_pml = (mca_pml_uniq_proc_t*) proc->proc_pml;
/* this ptl can be used */
ptl_inuse++;
/* initialize each proc */
if(NULL == proc_pml) {
/* allocate pml specific proc data */
proc_pml = OBJ_NEW(mca_pml_uniq_proc_t);
if (NULL == proc_pml) {
opal_output(0, "mca_pml_uniq_add_procs: unable to allocate resources");
free(ptl_peers);
return OMPI_ERR_OUT_OF_RESOURCE;
}
proc_pml->base.proc_ompi = proc;
proc->proc_pml = (mca_pml_proc_t*) proc_pml;
/* it's the first PTL so add it to both first and next */
proc_pml->proc_ptl_flags |= ptl->ptl_flags;
if (NULL == ptl->ptl_base &&
ptl->ptl_cache_bytes > 0 &&
NULL != ptl->ptl_request_init &&
NULL != ptl->ptl_request_fini) {
mca_pml_uniq_ptl_t* ptl_base = OBJ_NEW(mca_pml_uniq_ptl_t);
ptl_base->ptl = ptl;
ptl_base->ptl_cache_size = ptl->ptl_cache_size;
ptl->ptl_base = (struct mca_pml_base_ptl_t*)ptl_base;
}
proc_pml->proc_ptl_first.ptl_base = (mca_pml_base_ptl_t*)ptl->ptl_base;
proc_pml->proc_ptl_first.ptl_peer = ptl_peers[p];
proc_pml->proc_ptl_first.ptl = ptl;
#if PML_UNIQ_ACCEPT_NEXT_PTL
proc_pml->proc_ptl_next.ptl_base = (mca_pml_base_ptl_t*)ptl->ptl_base;
proc_pml->proc_ptl_next.ptl_peer = ptl_peers[p];
proc_pml->proc_ptl_next.ptl = ptl;
#endif /* PML_UNIQ_ACCEPT_NEXT_PTL */
} else {
/* choose the best for first and next. For the first look at the latency when
* for the next at the maximum bandwidth.
*/
opal_output( 0, "Not yet done dude !!!" );
#if PML_UNIQ_ACCEPT_NEXT_PTL
#endif /* PML_UNIQ_ACCEPT_NEXT_PTL */
}
/* dont allow an additional PTL with a lower exclusivity ranking */
if( NULL != proc_pml->proc_ptl_first.ptl ) {
/* skip this ptl if the exclusivity is less than the previous */
if( proc_pml->proc_ptl_first.ptl->ptl_exclusivity > ptl->ptl_exclusivity ) {
if(ptl_peers[p] != NULL) {
ptl->ptl_del_procs(ptl, 1, &proc, &ptl_peers[p]);
}
continue;
}
}
proc_pml->proc_ptl_flags |= ptl->ptl_flags;
}
if(ptl_inuse > 0 && NULL != ptl->ptl_component->ptlm_progress) {
size_t p;
bool found = false;
for(p=0; p < mca_pml_uniq.uniq_num_ptl_progress; p++) {
if(mca_pml_uniq.uniq_ptl_progress[p] == ptl->ptl_component->ptlm_progress) {
found = true;
break;
}
}
if(found == false) {
mca_pml_uniq.uniq_ptl_progress[mca_pml_uniq.uniq_num_ptl_progress] =
ptl->ptl_component->ptlm_progress;
mca_pml_uniq.uniq_num_ptl_progress++;
}
}
}
free(ptl_peers);
return OMPI_SUCCESS;
}
/*
* iterate through each proc and notify any PTLs associated
* with the proc that it is/has gone away
*/
int mca_pml_uniq_del_procs(ompi_proc_t** procs, size_t nprocs)
{
size_t p;
int rc;
for(p = 0; p < nprocs; p++) {
ompi_proc_t *proc = procs[p];
mca_pml_uniq_proc_t* proc_pml = (mca_pml_uniq_proc_t*) proc->proc_pml;
mca_ptl_proc_t* ptl_proc;
mca_ptl_base_module_t* ptl;
/* If the PTL used for the first fragment and the one use for the others is not
* the same then we have to remove the processor from both of them.
*/
ptl_proc = &(proc_pml->proc_ptl_first);
ptl = ptl_proc->ptl;
rc = ptl->ptl_del_procs( ptl, 1, &proc, &ptl_proc->ptl_peer );
if( OMPI_SUCCESS != rc ) {
return rc;
}
#if PML_UNIQ_ACCEPT_NEXT_PTL
if( proc_pml->proc_ptl_first.ptl != proc_pml->proc_ptl_next.ptl ) {
ptl_proc = &(proc_pml->proc_ptl_next);
ptl = ptl_proc->ptl;
rc = ptl->ptl_del_procs( ptl, 1, &proc, &ptl_proc->ptl_peer );
if( OMPI_SUCCESS != rc ) {
return rc;
}
}
#endif /* PML_UNIQ_ACCEPT_NEXT_PTL */
/* do any required cleanup */
OBJ_RELEASE(proc_pml);
proc->proc_pml = NULL;
}
return OMPI_SUCCESS;
}
int mca_pml_uniq_component_fini(void)
{
/* FIX */
return OMPI_SUCCESS;
}

258
ompi/mca/pml/uniq/pml_uniq.h
Просмотреть файл

@ -1,258 +0,0 @@
/*
* Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
* University Research and Technology
* Corporation. All rights reserved.
* Copyright (c) 2004-2005 The University of Tennessee and The University
* of Tennessee Research Foundation. All rights
* reserved.
* Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
* University of Stuttgart. All rights reserved.
* Copyright (c) 2004-2005 The Regents of the University of California.
* All rights reserved.
* $COPYRIGHT$
*
* Additional copyrights may follow
*
* $HEADER$
*/
/**
* @file
*/
#ifndef MCA_PML_UNIQ_H
#define MCA_PML_UNIQ_H
#include "opal/threads/threads.h"
#include "opal/threads/condition.h"
#include "ompi/class/ompi_free_list.h"
#include "opal/util/cmd_line.h"
#include "ompi/request/request.h"
#include "ompi/mca/pml/pml.h"
#include "ompi/mca/pml/base/pml_base_request.h"
#include "ompi/mca/pml/base/pml_base_bsend.h"
#include "ompi/mca/pml/base/pml_base_sendreq.h"
#include "ompi/mca/ptl/ptl.h"
#if defined(c_plusplus) || defined(__cplusplus)
extern "C" {
#endif
/**
* UNIQ PML module
*/
struct mca_pml_uniq_t {
mca_pml_base_module_t super;
mca_ptl_base_component_t **uniq_ptl_components;
size_t uniq_num_ptl_components;
mca_ptl_base_module_t** uniq_ptl_modules;
size_t uniq_num_ptl_modules;
mca_ptl_base_component_progress_fn_t* uniq_ptl_progress;
size_t uniq_num_ptl_progress;
opal_list_t uniq_procs;
opal_mutex_t uniq_lock;
int uniq_free_list_num; /* initial size of free list */
int uniq_free_list_max; /* maximum size of free list */
int uniq_free_list_inc; /* number of elements to grow free list */
int uniq_poll_iterations; /* number of iterations to poll for completion */
int uniq_priority; /* the PML priority */
/* free list of requests */
ompi_free_list_t uniq_send_requests;
ompi_free_list_t uniq_recv_requests;
/* list of pending send requests */
opal_list_t uniq_send_pending;
};
typedef struct mca_pml_uniq_t mca_pml_uniq_t;
extern mca_pml_uniq_t mca_pml_uniq;
/*
* PML module functions.
*/
extern int mca_pml_uniq_component_open(void);
extern int mca_pml_uniq_component_close(void);
extern mca_pml_base_module_t* mca_pml_uniq_component_init(
int *priority,
bool enable_progress_threads,
bool enable_mpi_threads
);
extern int mca_pml_uniq_component_fini(void);
/*
* PML interface functions.
*/
extern int mca_pml_uniq_add_comm(
struct ompi_communicator_t* comm
);
extern int mca_pml_uniq_del_comm(
struct ompi_communicator_t* comm
);
extern int mca_pml_uniq_add_procs(
struct ompi_proc_t **procs,
size_t nprocs
);
extern int mca_pml_uniq_del_procs(
struct ompi_proc_t **procs,
size_t nprocs
);
extern int mca_pml_uniq_enable(
bool enable
);
extern int mca_pml_uniq_progress(void);
extern int mca_pml_uniq_iprobe(
int dst,
int tag,
struct ompi_communicator_t* comm,
int *matched,
ompi_status_public_t* status
);
extern int mca_pml_uniq_probe(
int dst,
int tag,
struct ompi_communicator_t* comm,
ompi_status_public_t* status
);
extern int mca_pml_uniq_cancelled(
ompi_request_t* request,
int *flag
);
extern int mca_pml_uniq_isend_init(
void *buf,
size_t count,
struct ompi_datatype_t *datatype,
int dst,
int tag,
mca_pml_base_send_mode_t mode,
struct ompi_communicator_t* comm,
struct ompi_request_t **request
);
extern int mca_pml_uniq_isend(
void *buf,
size_t count,
struct ompi_datatype_t *datatype,
int dst,
int tag,
mca_pml_base_send_mode_t mode,
struct ompi_communicator_t* comm,
struct ompi_request_t **request
);
extern int mca_pml_uniq_send(
void *buf,
size_t count,
struct ompi_datatype_t *datatype,
int dst,
int tag,
mca_pml_base_send_mode_t mode,
struct ompi_communicator_t* comm
);
extern int mca_pml_uniq_irecv_init(
void *buf,
size_t count,
struct ompi_datatype_t *datatype,
int src,
int tag,
struct ompi_communicator_t* comm,
struct ompi_request_t **request
);
extern int mca_pml_uniq_irecv(
void *buf,
size_t count,
struct ompi_datatype_t *datatype,
int src,
int tag,
struct ompi_communicator_t* comm,
struct ompi_request_t **request
);
extern int mca_pml_uniq_recv(
void *buf,
size_t count,
struct ompi_datatype_t *datatype,
int src,
int tag,
struct ompi_communicator_t* comm,
ompi_status_public_t* status
);
extern int mca_pml_uniq_progress(void);
extern int mca_pml_uniq_start(
size_t count,
ompi_request_t** requests
);
#if defined(c_plusplus) || defined(__cplusplus)
}
#endif
#define MCA_PML_UNIQ_FREE(request) \
{ \
mca_pml_base_request_t* pml_request = *(mca_pml_base_request_t**)(request); \
pml_request->req_free_called = true; \
if( pml_request->req_pml_complete == true) \
{ \
switch(pml_request->req_type) { \
case MCA_PML_REQUEST_SEND: \
{ \
mca_ptl_base_send_request_t* sendreq = (mca_ptl_base_send_request_t*)pml_request; \
while(sendreq->req_lock > 0); \
if(sendreq->req_send.req_send_mode == MCA_PML_BASE_SEND_BUFFERED) { \
mca_pml_base_bsend_request_fini((ompi_request_t*)sendreq); \
} \
MCA_PML_UNIQ_SEND_REQUEST_RETURN(sendreq); \
break; \
} \
case MCA_PML_REQUEST_RECV: \
{ \
mca_ptl_base_recv_request_t* recvreq = (mca_ptl_base_recv_request_t*)pml_request; \
MCA_PML_UNIQ_RECV_REQUEST_RETURN(recvreq); \
break; \
} \
default: \
break; \
} \
} \
*(request) = MPI_REQUEST_NULL; \
}
#define MCA_PML_UNIQ_FINI(request) \
{ \
mca_pml_base_request_t* pml_request = *(mca_pml_base_request_t**)(request); \
if( (pml_request->req_persistent) && !(pml_request->req_free_called) ) { \
pml_request->req_ompi.req_state = OMPI_REQUEST_INACTIVE; \
} else { \
MCA_PML_UNIQ_FREE(request); \
} \
}
#endif

30
ompi/mca/pml/uniq/pml_uniq_cancel.c
Просмотреть файл

@ -1,30 +0,0 @@
/* -*- Mode: C; c-basic-offset:4 ; -*- */
/*
* Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
* University Research and Technology
* Corporation. All rights reserved.
* Copyright (c) 2004-2005 The University of Tennessee and The University
* of Tennessee Research Foundation. All rights
* reserved.
* Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
* University of Stuttgart. All rights reserved.
* Copyright (c) 2004-2005 The Regents of the University of California.
* All rights reserved.
* $COPYRIGHT$
*
* Additional copyrights may follow
*
* $HEADER$
*/
#include "ompi_config.h"
#include "pml_uniq.h"
int mca_pml_uniq_cancelled(ompi_request_t* request, int* flag)
{
if(NULL != flag)
*flag = (true == request->req_status._cancelled ? 1 : 0);
return OMPI_SUCCESS;
}

159
ompi/mca/pml/uniq/pml_uniq_component.c
Просмотреть файл

@ -1,159 +0,0 @@
/*
* Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
* University Research and Technology
* Corporation. All rights reserved.
* Copyright (c) 2004-2005 The University of Tennessee and The University
* of Tennessee Research Foundation. All rights
* reserved.
* Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
* University of Stuttgart. All rights reserved.
* Copyright (c) 2004-2005 The Regents of the University of California.
* All rights reserved.
* $COPYRIGHT$
*
* Additional copyrights may follow
*
* $HEADER$
*/
#include "ompi_config.h"
#include "opal/event/event.h"
#include "mpi.h"
#include "ompi/mca/pml/pml.h"
#include "ompi/mca/ptl/ptl.h"
#include "ompi/mca/ptl/base/base.h"
#include "opal/mca/base/mca_base_param.h"
#include "ompi/mca/pml/base/pml_base_bsend.h"
#include "pml_uniq.h"
#include "pml_uniq_proc.h"
#include "pml_uniq_sendreq.h"
#include "pml_uniq_recvreq.h"
mca_pml_base_component_1_0_0_t mca_pml_uniq_component = {
/* First, the mca_base_component_t struct containing meta
information about the component itself */
{
/* Indicate that we are a pml v1.0.0 component (which also implies
a specific MCA version) */
MCA_PML_BASE_VERSION_1_0_0,
"uniq", /* MCA component name */
OMPI_MAJOR_VERSION, /* MCA component major version */
OMPI_MINOR_VERSION, /* MCA component minor version */
OMPI_RELEASE_VERSION, /* MCA component release version */
mca_pml_uniq_component_open, /* component open */
mca_pml_uniq_component_close /* component close */
},
/* Next the MCA v1.0.0 component meta data */
{
/* Whether the component is checkpointable or not */
false
},
mca_pml_uniq_component_init, /* component init */
mca_pml_uniq_component_fini /* component finalize */
};
static inline int mca_pml_uniq_param_register_int(
const char* param_name,
int default_value)
{
int id = mca_base_param_register_int("pml","uniq",param_name,NULL,default_value);
int param_value = default_value;
mca_base_param_lookup_int(id,&param_value);
return param_value;
}
int mca_pml_uniq_component_open(void)
{
OBJ_CONSTRUCT(&mca_pml_uniq.uniq_lock, opal_mutex_t);
OBJ_CONSTRUCT(&mca_pml_uniq.uniq_send_requests, ompi_free_list_t);
OBJ_CONSTRUCT(&mca_pml_uniq.uniq_recv_requests, ompi_free_list_t);
OBJ_CONSTRUCT(&mca_pml_uniq.uniq_procs, opal_list_t);
OBJ_CONSTRUCT(&mca_pml_uniq.uniq_send_pending, opal_list_t);
mca_pml_uniq.uniq_ptl_components = NULL;
mca_pml_uniq.uniq_num_ptl_components = 0;
mca_pml_uniq.uniq_ptl_modules = NULL;
mca_pml_uniq.uniq_num_ptl_modules = 0;
mca_pml_uniq.uniq_ptl_progress = NULL;
mca_pml_uniq.uniq_num_ptl_progress = 0;
mca_pml_uniq.uniq_free_list_num =
mca_pml_uniq_param_register_int("free_list_num", 256);
mca_pml_uniq.uniq_free_list_max =
mca_pml_uniq_param_register_int("free_list_max", -1);
mca_pml_uniq.uniq_free_list_inc =
mca_pml_uniq_param_register_int("free_list_inc", 256);
mca_pml_uniq.uniq_poll_iterations =
mca_pml_uniq_param_register_int("poll_iterations", 100000);
mca_pml_uniq.uniq_priority =
mca_pml_uniq_param_register_int("priority", 0);
return mca_ptl_base_open();
}
int mca_pml_uniq_component_close(void)
{
int rc;
if( NULL == mca_pml_uniq.uniq_ptl_components ) /* I was not enabled */
return OMPI_SUCCESS;
if( OMPI_SUCCESS != (rc = mca_ptl_base_close()) )
return rc;
if(NULL != mca_pml_uniq.uniq_ptl_components) {
free(mca_pml_uniq.uniq_ptl_components);
mca_pml_uniq.uniq_ptl_components = NULL;
}
mca_pml_uniq.uniq_num_ptl_components = 0;
if(NULL != mca_pml_uniq.uniq_ptl_modules) {
free(mca_pml_uniq.uniq_ptl_modules);
mca_pml_uniq.uniq_ptl_modules = NULL;
}
mca_pml_uniq.uniq_num_ptl_modules = 0;
if(NULL != mca_pml_uniq.uniq_ptl_progress) {
free(mca_pml_uniq.uniq_ptl_progress);
mca_pml_uniq.uniq_ptl_progress = NULL;
}
OBJ_DESTRUCT(&mca_pml_uniq.uniq_send_pending);
OBJ_DESTRUCT(&mca_pml_uniq.uniq_send_requests);
OBJ_DESTRUCT(&mca_pml_uniq.uniq_recv_requests);
OBJ_DESTRUCT(&mca_pml_uniq.uniq_procs);
OBJ_DESTRUCT(&mca_pml_uniq.uniq_lock);
return OMPI_SUCCESS;
}
mca_pml_base_module_t* mca_pml_uniq_component_init(int* priority,
bool enable_progress_threads,
bool enable_mpi_threads)
{
int rc;
*priority = mca_pml_uniq.uniq_priority;
/* buffered send */
if( OMPI_SUCCESS != mca_pml_base_bsend_init(enable_mpi_threads) ) {
opal_output(0, "mca_pml_uniq_component_init: mca_pml_bsend_init failed\n");
return NULL;
}
rc = mca_ptl_base_select( enable_progress_threads, enable_mpi_threads );
if( rc != OMPI_SUCCESS )
return NULL;
return &mca_pml_uniq.super;
}

29
ompi/mca/pml/uniq/pml_uniq_component.h
Просмотреть файл

@ -1,29 +0,0 @@
/*
* Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
* University Research and Technology
* Corporation. All rights reserved.
* Copyright (c) 2004-2005 The University of Tennessee and The University
* of Tennessee Research Foundation. All rights
* reserved.
* Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
* University of Stuttgart. All rights reserved.
* $COPYRIGHT$
*
* Additional copyrights may follow
*
* $HEADER$
*/
/**
* @file
*/
#ifndef MCA_PML_UNIQ_COMPONENT_H
#define MCA_PML_UNIQ_COMPONENT_H
/*
* PML module functions.
*/
OMPI_COMP_EXPORT extern mca_pml_base_component_1_0_0_t mca_pml_uniq_component;
#endif

95
ompi/mca/pml/uniq/pml_uniq_iprobe.c
Просмотреть файл

@ -1,95 +0,0 @@
/*
* Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
* University Research and Technology
* Corporation. All rights reserved.
* Copyright (c) 2004-2005 The University of Tennessee and The University
* of Tennessee Research Foundation. All rights
* reserved.
* Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
* University of Stuttgart. All rights reserved.
* Copyright (c) 2004-2005 The Regents of the University of California.
* All rights reserved.
* $COPYRIGHT$
*
* Additional copyrights may follow
*
* $HEADER$
*/
#include "ompi_config.h"
#include "ompi/request/request.h"
#include "pml_uniq_recvreq.h"
int mca_pml_uniq_iprobe(int src,
int tag,
struct ompi_communicator_t *comm,
int *matched, ompi_status_public_t * status)
{
int rc;
mca_ptl_base_recv_request_t recvreq;
OBJ_CONSTRUCT( &(recvreq), mca_ptl_base_recv_request_t );
recvreq.req_recv.req_base.req_ompi.req_type = OMPI_REQUEST_PML;
recvreq.req_recv.req_base.req_type = MCA_PML_REQUEST_IPROBE;
MCA_PML_UNIQ_RECV_REQUEST_INIT(&recvreq, NULL, 0, &ompi_mpi_char, src, tag, comm, true);
*matched = 0;
if ((rc = mca_pml_uniq_recv_request_start(&recvreq)) == OMPI_SUCCESS) {
if( recvreq.req_recv.req_base.req_ompi.req_complete == true ) {
if( NULL != status ) {
*status = recvreq.req_recv.req_base.req_ompi.req_status;
}
*matched = 1;
} else {
/* we are supposed to progress ... */
opal_progress();
}
}
MCA_PML_BASE_RECV_REQUEST_FINI( &recvreq.req_recv );
return rc;
}
int mca_pml_uniq_probe(int src,
int tag,
struct ompi_communicator_t *comm,
ompi_status_public_t * status)
{
int rc;
mca_ptl_base_recv_request_t recvreq;
OBJ_CONSTRUCT( &(recvreq), mca_ptl_base_recv_request_t );
recvreq.req_recv.req_base.req_ompi.req_type = OMPI_REQUEST_PML;
recvreq.req_recv.req_base.req_type = MCA_PML_REQUEST_PROBE;
MCA_PML_UNIQ_RECV_REQUEST_INIT(&recvreq, NULL, 0, &ompi_mpi_char, src, tag, comm, true);
if ((rc = mca_pml_uniq_recv_request_start(&recvreq)) != OMPI_SUCCESS) {
MCA_PML_BASE_RECV_REQUEST_FINI( &recvreq.req_recv );
return rc;
}
if (recvreq.req_recv.req_base.req_ompi.req_complete == false) {
/* give up and sleep until completion */
if (opal_using_threads()) {
opal_mutex_lock(&ompi_request_lock);
ompi_request_waiting++;
while (recvreq.req_recv.req_base.req_ompi.req_complete == false)
opal_condition_wait(&ompi_request_cond, &ompi_request_lock);
ompi_request_waiting--;
opal_mutex_unlock(&ompi_request_lock);
} else {
ompi_request_waiting++;
while (recvreq.req_recv.req_base.req_ompi.req_complete == false)
opal_condition_wait(&ompi_request_cond, &ompi_request_lock);
ompi_request_waiting--;
}
}
if (NULL != status) {
*status = recvreq.req_recv.req_base.req_ompi.req_status;
}
MCA_PML_BASE_RECV_REQUEST_FINI( &recvreq.req_recv );
return OMPI_SUCCESS;
}

119
ompi/mca/pml/uniq/pml_uniq_irecv.c
Просмотреть файл

@ -1,119 +0,0 @@
/*
* Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
* University Research and Technology
* Corporation. All rights reserved.
* Copyright (c) 2004-2005 The University of Tennessee and The University
* of Tennessee Research Foundation. All rights
* reserved.
* Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
* University of Stuttgart. All rights reserved.
* Copyright (c) 2004-2005 The Regents of the University of California.
* All rights reserved.
* $COPYRIGHT$
*
* Additional copyrights may follow
*
* $HEADER$
*/
#include "ompi_config.h"
#include "ompi/request/request.h"
#include "pml_uniq_recvreq.h"
int mca_pml_uniq_irecv_init(void *addr,
size_t count,
struct ompi_datatype_t * datatype,
int src,
int tag,
struct ompi_communicator_t *comm,
struct ompi_request_t **request)
{
int rc;
mca_ptl_base_recv_request_t *recvreq;
MCA_PML_UNIQ_RECV_REQUEST_ALLOC(recvreq, rc);
if (NULL == recvreq)
return rc;
MCA_PML_UNIQ_RECV_REQUEST_INIT(recvreq,
addr,
count, datatype, src, tag, comm, true);
*request = (ompi_request_t *) recvreq;
return OMPI_SUCCESS;
}
int mca_pml_uniq_irecv(void *addr,
size_t count,
struct ompi_datatype_t * datatype,
int src,
int tag,
struct ompi_communicator_t *comm,
struct ompi_request_t **request)
{
int rc;
mca_ptl_base_recv_request_t *recvreq;
MCA_PML_UNIQ_RECV_REQUEST_ALLOC(recvreq, rc);
if (NULL == recvreq)
return rc;
MCA_PML_UNIQ_RECV_REQUEST_INIT(recvreq,
addr,
count, datatype, src, tag, comm, false);
if ((rc = mca_pml_uniq_recv_request_start(recvreq)) != OMPI_SUCCESS) {
MCA_PML_UNIQ_RECV_REQUEST_RETURN(recvreq);
return rc;
}
*request = (ompi_request_t *) recvreq;
return OMPI_SUCCESS;
}
int mca_pml_uniq_recv(void *addr,
size_t count,
struct ompi_datatype_t * datatype,
int src,
int tag,
struct ompi_communicator_t *comm,
ompi_status_public_t * status)
{
int rc;
mca_ptl_base_recv_request_t *recvreq;
MCA_PML_UNIQ_RECV_REQUEST_ALLOC(recvreq, rc);
if (NULL == recvreq)
return rc;
MCA_PML_UNIQ_RECV_REQUEST_INIT(recvreq,
addr,
count, datatype, src, tag, comm, false);
if ((rc = mca_pml_uniq_recv_request_start(recvreq)) != OMPI_SUCCESS) {
goto recv_finish;
}
if (recvreq->req_recv.req_base.req_ompi.req_complete == false) {
/* give up and sleep until completion */
if (opal_using_threads()) {
opal_mutex_lock(&ompi_request_lock);
ompi_request_waiting++;
while (recvreq->req_recv.req_base.req_ompi.req_complete == false)
opal_condition_wait(&ompi_request_cond, &ompi_request_lock);
ompi_request_waiting--;
opal_mutex_unlock(&ompi_request_lock);
} else {
ompi_request_waiting++;
while (recvreq->req_recv.req_base.req_ompi.req_complete == false)
opal_condition_wait(&ompi_request_cond, &ompi_request_lock);
ompi_request_waiting--;
}
}
recv_finish:
if (NULL != status) { /* return status */
*status = recvreq->req_recv.req_base.req_ompi.req_status;
}
MCA_PML_UNIQ_RECV_REQUEST_RETURN(recvreq);
return recvreq->req_recv.req_base.req_ompi.req_status.MPI_ERROR;
}

130
ompi/mca/pml/uniq/pml_uniq_isend.c
Просмотреть файл

@ -1,130 +0,0 @@
/*
* Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
* University Research and Technology
* Corporation. All rights reserved.
* Copyright (c) 2004-2005 The University of Tennessee and The University
* of Tennessee Research Foundation. All rights
* reserved.
* Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
* University of Stuttgart. All rights reserved.
* Copyright (c) 2004-2005 The Regents of the University of California.
* All rights reserved.
* $COPYRIGHT$
*
* Additional copyrights may follow
*
* $HEADER$
*/
#include "ompi_config.h"
#include "pml_uniq.h"
#include "pml_uniq_proc.h"
#include "pml_uniq_sendreq.h"
#include "pml_uniq_recvreq.h"
int mca_pml_uniq_isend_init(void *buf,
size_t count,
struct ompi_datatype_t * datatype,
int dst,
int tag,
mca_pml_base_send_mode_t sendmode,
ompi_communicator_t * comm,
ompi_request_t ** request)
{
int rc;
mca_ptl_base_send_request_t *sendreq;
MCA_PML_UNIQ_SEND_REQUEST_ALLOC(comm, dst, sendreq, rc);
if (rc != OMPI_SUCCESS)
return rc;
MCA_PML_UNIQ_SEND_REQUEST_INIT(sendreq,
buf,
count,
datatype,
dst, tag,
comm, sendmode, true);
*request = (ompi_request_t *) sendreq;
return OMPI_SUCCESS;
}
int mca_pml_uniq_isend(void *buf,
size_t count,
struct ompi_datatype_t * datatype,
int dst,
int tag,
mca_pml_base_send_mode_t sendmode,
ompi_communicator_t * comm,
ompi_request_t ** request)
{
int rc;
mca_ptl_base_send_request_t *sendreq;
MCA_PML_UNIQ_SEND_REQUEST_ALLOC(comm, dst, sendreq, rc);
if (rc != OMPI_SUCCESS)
return rc;
MCA_PML_UNIQ_SEND_REQUEST_INIT(sendreq,
buf,
count,
datatype,
dst, tag,
comm, sendmode, false);
MCA_PML_UNIQ_SEND_REQUEST_START(sendreq, rc);
*request = (ompi_request_t *) sendreq;
return rc;
}
int mca_pml_uniq_send(void *buf,
size_t count,
struct ompi_datatype_t * datatype,
int dst,
int tag,
mca_pml_base_send_mode_t sendmode,
ompi_communicator_t * comm)
{
int rc;
mca_ptl_base_send_request_t *sendreq;
MCA_PML_UNIQ_SEND_REQUEST_ALLOC(comm, dst, sendreq, rc);
if (rc != OMPI_SUCCESS)
return rc;
MCA_PML_UNIQ_SEND_REQUEST_INIT(sendreq,
buf,
count,
datatype,
dst, tag,
comm, sendmode, false);
MCA_PML_UNIQ_SEND_REQUEST_START(sendreq, rc);
if (rc != OMPI_SUCCESS) {
MCA_PML_UNIQ_FREE((ompi_request_t **) & sendreq);
return rc;
}
if (sendreq->req_send.req_base.req_ompi.req_complete == false) {
/* give up and sleep until completion */
if (opal_using_threads()) {
opal_mutex_lock(&ompi_request_lock);
ompi_request_waiting++;
while (sendreq->req_send.req_base.req_ompi.req_complete == false)
opal_condition_wait(&ompi_request_cond, &ompi_request_lock);
ompi_request_waiting--;
opal_mutex_unlock(&ompi_request_lock);
} else {
ompi_request_waiting++;
while (sendreq->req_send.req_base.req_ompi.req_complete == false)
opal_condition_wait(&ompi_request_cond, &ompi_request_lock);
ompi_request_waiting--;
}
}
/* return request to pool */
MCA_PML_UNIQ_FREE((ompi_request_t **) & sendreq);
return OMPI_SUCCESS;
}

61
ompi/mca/pml/uniq/pml_uniq_proc.c
Просмотреть файл

@ -1,61 +0,0 @@
/*
* Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
* University Research and Technology
* Corporation. All rights reserved.
* Copyright (c) 2004-2005 The University of Tennessee and The University
* of Tennessee Research Foundation. All rights
* reserved.
* Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
* University of Stuttgart. All rights reserved.
* Copyright (c) 2004-2005 The Regents of the University of California.
* All rights reserved.
* $COPYRIGHT$
*
* Additional copyrights may follow
*
* $HEADER$
*/
#include "ompi_config.h"
#include "opal/sys/atomic.h"
#include "pml_uniq.h"
#include "pml_uniq_proc.h"
static void mca_pml_uniq_proc_construct(mca_pml_uniq_proc_t* proc)
{
proc->base.proc_ompi = NULL;
proc->proc_ptl_flags = 0;
OBJ_CONSTRUCT(&proc->base.proc_lock, opal_mutex_t);
proc->proc_ptl_first.ptl_peer = NULL;
proc->proc_ptl_first.ptl_base = NULL;
proc->proc_ptl_first.ptl = NULL;
#if PML_UNIQ_ACCEPT_NEXT_PTL
proc->proc_ptl_next.ptl_peer = NULL;
proc->proc_ptl_next.ptl_base = NULL;
proc->proc_ptl_next.ptl = NULL;
#endif /* PML_UNIQ_ACCEPT_NEXT_PTL */
OPAL_THREAD_LOCK(&mca_pml_uniq.uniq_lock);
opal_list_append(&mca_pml_uniq.uniq_procs, (opal_list_item_t*)proc);
OPAL_THREAD_UNLOCK(&mca_pml_uniq.uniq_lock);
}
static void mca_pml_uniq_proc_destruct(mca_pml_uniq_proc_t* proc)
{
OPAL_THREAD_LOCK(&mca_pml_uniq.uniq_lock);
opal_list_remove_item(&mca_pml_uniq.uniq_procs, (opal_list_item_t*)proc);
OPAL_THREAD_UNLOCK(&mca_pml_uniq.uniq_lock);
OBJ_DESTRUCT(&proc->base.proc_lock);
}
OBJ_CLASS_INSTANCE(
mca_pml_uniq_proc_t,
opal_list_item_t,
mca_pml_uniq_proc_construct,
mca_pml_uniq_proc_destruct
);

124
ompi/mca/pml/uniq/pml_uniq_proc.h
Просмотреть файл

@ -1,124 +0,0 @@
/*
* Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
* University Research and Technology
* Corporation. All rights reserved.
* Copyright (c) 2004-2005 The University of Tennessee and The University
* of Tennessee Research Foundation. All rights
* reserved.
* Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
* University of Stuttgart. All rights reserved.
* Copyright (c) 2004-2005 The Regents of the University of California.
* All rights reserved.
* $COPYRIGHT$
*
* Additional copyrights may follow
*
* $HEADER$
*/
/**
* @file
*/
#ifndef MCA_PML_PROC_H
#define MCA_PML_PROC_H
#include "opal/threads/mutex.h"
#include "ompi/communicator/communicator.h"
#include "ompi/group/group.h"
#include "ompi/proc/proc.h"
/* This define has to move outside of this file. Maybe on some configuration file.
* Anyway by for, for the debugging purpose, here it's quite a safe place.
*/
#define PML_UNIQ_ACCEPT_NEXT_PTL 0
#if defined(c_plusplus) || defined(__cplusplus)
extern "C" {
#endif
/**
* A data structure associated with a ompi_proc_t that caches
* addressing/scheduling attributes for a specific PTL instance
* that can be used to reach the process.
*/
struct mca_ptl_proc_t {
struct mca_ptl_base_peer_t* ptl_peer; /**< PTL addressing info */
struct mca_pml_uniq_ptl_t* ptl_base; /**< PML specific PTL info */
mca_ptl_base_module_t *ptl; /**< PTL module */
};
typedef struct mca_ptl_proc_t mca_ptl_proc_t;
/**
* Structure associated w/ ompi_proc_t that contains data specific
* to the PML. Note that this name is not PML specific.
*/
struct mca_pml_uniq_proc_t {
mca_pml_proc_t base;
mca_ptl_proc_t proc_ptl_first; /**< ptl for the first fragment */
#if PML_UNIQ_ACCEPT_NEXT_PTL
mca_ptl_proc_t proc_ptl_next; /**< ptl for the remaining fragments */
#endif /* PML_UNIQ_ACCEPT_NEXT_PTL */
uint32_t proc_ptl_flags; /**< aggregate ptl flags */
};
typedef struct mca_pml_uniq_proc_t mca_pml_uniq_proc_t;
OMPI_COMP_EXPORT extern opal_class_t mca_pml_uniq_proc_t_class;
/**
* Return the mca_pml_proc_t instance cached in the communicators local group.
*
* @param comm Communicator
* @param rank Peer rank
* @return mca_pml_proc_t instance
*/
static inline mca_pml_proc_t* mca_pml_uniq_proc_lookup_local(ompi_communicator_t* comm, int rank)
{
ompi_proc_t* proc = comm->c_local_group->grp_proc_pointers[rank];
return proc->proc_pml;
}
/**
* Return the mca_pml_proc_t instance cached on the communicators remote group.
*
* @param comm Communicator
* @param rank Peer rank
* @return mca_pml_proc_t instance
*/
static inline mca_pml_proc_t* mca_pml_uniq_proc_lookup_remote(ompi_communicator_t* comm, int rank)
{
ompi_proc_t* proc = comm->c_remote_group->grp_proc_pointers[rank];
return proc->proc_pml;
}
/**
* Return the mca_ptl_peer_t instance corresponding to the process/ptl combination.
*
* @param comm Communicator
* @param rank Peer rank
* @return mca_pml_proc_t instance
*/
static inline struct mca_ptl_base_peer_t*
mca_pml_uniq_proc_lookup_remote_peer( ompi_communicator_t* comm,
int rank,
struct mca_ptl_base_module_t* ptl)
{
ompi_proc_t* proc = comm->c_remote_group->grp_proc_pointers[rank];
mca_pml_uniq_proc_t* proc_pml = (mca_pml_uniq_proc_t*) proc->proc_pml;
if( proc_pml->proc_ptl_first.ptl == ptl )
return proc_pml->proc_ptl_first.ptl_peer;
#if PML_UNIQ_ACCEPT_NEXT_PTL
if( proc_pml->proc_ptl_next.ptl == ptl )
return proc_pml->proc_ptl_next.ptl_peer;
#endif /* PML_UNIQ_ACCEPT_NEXT_PTL */
return NULL;
}
#if defined(c_plusplus) || defined(__cplusplus)
}
#endif
#endif

42
ompi/mca/pml/uniq/pml_uniq_progress.c
Просмотреть файл

@ -1,42 +0,0 @@
/*
* Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
* University Research and Technology
* Corporation. All rights reserved.
* Copyright (c) 2004-2005 The University of Tennessee and The University
* of Tennessee Research Foundation. All rights
* reserved.
* Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
* University of Stuttgart. All rights reserved.
* Copyright (c) 2004-2005 The Regents of the University of California.
* All rights reserved.
* $COPYRIGHT$
*
* Additional copyrights may follow
*
* $HEADER$
*/
#include "ompi_config.h"
#include "pml_uniq.h"
#include "pml_uniq_sendreq.h"
int mca_pml_uniq_progress(void)
{
mca_ptl_tstamp_t tstamp = 0;
size_t i;
int count = 0;
/*
* Progress each of the PTL modules
*/
for(i=0; i<mca_pml_uniq.uniq_num_ptl_progress; i++) {
int rc = mca_pml_uniq.uniq_ptl_progress[i](tstamp);
if(rc > 0) {
count += rc;
}
}
return count;
}

44
ompi/mca/pml/uniq/pml_uniq_ptl.c
Просмотреть файл

@ -1,44 +0,0 @@
/*
* Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
* University Research and Technology
* Corporation. All rights reserved.
* Copyright (c) 2004-2005 The University of Tennessee and The University
* of Tennessee Research Foundation. All rights
* reserved.
* Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
* University of Stuttgart. All rights reserved.
* Copyright (c) 2004-2005 The Regents of the University of California.
* All rights reserved.
* $COPYRIGHT$
*
* Additional copyrights may follow
*
* $HEADER$
*/
#include "ompi_config.h"
#include "pml_uniq_ptl.h"
static void mca_pml_uniq_ptl_construct(mca_pml_uniq_ptl_t* ptl)
{
OBJ_CONSTRUCT(&ptl->ptl_cache, opal_list_t);
OBJ_CONSTRUCT(&ptl->ptl_cache_lock, opal_mutex_t);
ptl->ptl = NULL;
ptl->ptl_cache_size = 0;
ptl->ptl_cache_alloc = 0;
}
static void mca_pml_uniq_ptl_destruct(mca_pml_uniq_ptl_t* ptl)
{
OBJ_DESTRUCT(&ptl->ptl_cache);
OBJ_DESTRUCT(&ptl->ptl_cache_lock);
}
OBJ_CLASS_INSTANCE(
mca_pml_uniq_ptl_t,
opal_list_t,
mca_pml_uniq_ptl_construct,
mca_pml_uniq_ptl_destruct
);

47
ompi/mca/pml/uniq/pml_uniq_ptl.h
Просмотреть файл

@ -1,47 +0,0 @@
/*
* Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
* University Research and Technology
* Corporation. All rights reserved.
* Copyright (c) 2004-2005 The University of Tennessee and The University
* of Tennessee Research Foundation. All rights
* reserved.
* Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
* University of Stuttgart. All rights reserved.
* Copyright (c) 2004-2005 The Regents of the University of California.
* All rights reserved.
* $COPYRIGHT$
*
* Additional copyrights may follow
*
* $HEADER$
*/
#ifndef _MCA_PML_BASE_PTL_
#define _MCA_PML_BASE_PTL_
#include "ompi/mca/pml/pml.h"
#include "ompi/mca/ptl/ptl.h"
#include "opal/threads/condition.h"
#if defined(c_plusplus) || defined(__cplusplus)
extern "C" {
#endif
struct mca_pml_uniq_ptl_t {
opal_list_t ptl_cache; /**< cache of send requests */
size_t ptl_cache_size; /**< maximum size of cache */
size_t ptl_cache_alloc; /**< current number of allocated items */
opal_mutex_t ptl_cache_lock; /**< lock for queue access */
struct mca_ptl_base_module_t* ptl; /**< back pointer to ptl */
};
typedef struct mca_pml_uniq_ptl_t mca_pml_uniq_ptl_t;
typedef struct mca_pml_uniq_ptl_t mca_pml_base_ptl_t;
OBJ_CLASS_DECLARATION(mca_pml_uniq_ptl_t);
#if defined(c_plusplus) || defined(__cplusplus)
}
#endif
#endif

92
ompi/mca/pml/uniq/pml_uniq_recvfrag.c
Просмотреть файл

@ -1,92 +0,0 @@
/*
* Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
* University Research and Technology
* Corporation. All rights reserved.
* Copyright (c) 2004-2005 The University of Tennessee and The University
* of Tennessee Research Foundation. All rights
* reserved.
* Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
* University of Stuttgart. All rights reserved.
* Copyright (c) 2004-2005 The Regents of the University of California.
* All rights reserved.
* $COPYRIGHT$
*
* Additional copyrights may follow
*
* $HEADER$
*/
/**
* @file
*/
#include "ompi_config.h"
#include "ompi/mca/pml/pml.h"
#include "pml_uniq_recvfrag.h"
#include "pml_uniq_proc.h"
OMPI_DECLSPEC extern opal_class_t mca_ptl_base_recv_frag_t_class;
/**
* Called by the PTL to match attempt a match for new fragments.
*
* @param ptl (IN) The PTL pointer
* @param frag (IN) Receive fragment descriptor.
* @param header (IN) Header corresponding to the receive fragment.
* @return OMPI_SUCCESS or error status on failure.
*/
bool mca_pml_uniq_recv_frag_match(
mca_ptl_base_module_t* ptl,
mca_ptl_base_recv_frag_t* frag,
mca_ptl_base_match_header_t* header)
{
bool matched;
bool matches = false;
opal_list_t matched_frags;
if((matched = mca_ptl_base_match(header, frag, &matched_frags, &matches)) == false) {
frag = (matches ? (mca_ptl_base_recv_frag_t*)opal_list_remove_first(&matched_frags) : NULL);
}
while(NULL != frag) {
mca_ptl_base_module_t* ptl = frag->frag_base.frag_owner;
mca_ptl_base_recv_request_t *request = frag->frag_request;
mca_ptl_base_match_header_t *header = &frag->frag_base.frag_header.hdr_match;
/*
* Initialize request status.
*/
request->req_recv.req_bytes_packed = header->hdr_msg_length;
request->req_recv.req_base.req_ompi.req_status.MPI_SOURCE = header->hdr_src;
request->req_recv.req_base.req_ompi.req_status.MPI_TAG = header->hdr_tag;
/*
* If probe - signal request is complete - but don't notify PTL
*/
if(request->req_recv.req_base.req_type == MCA_PML_REQUEST_PROBE) {
ptl->ptl_recv_progress( ptl,
request,
header->hdr_msg_length,
header->hdr_msg_length );
matched = mca_pml_uniq_recv_frag_match( ptl, frag, header );
} else {
/* if required - setup pointer to ptls peer */
if (NULL == frag->frag_base.frag_peer) {
frag->frag_base.frag_peer = mca_pml_uniq_proc_lookup_remote_peer(request->req_recv.req_base.req_comm,header->hdr_src,ptl);
}
MCA_PML_UNIQ_RECV_MATCHED( ptl, frag );
};
/* process any additional fragments that arrived out of order */
frag = (matches ? (mca_ptl_base_recv_frag_t*)opal_list_remove_first(&matched_frags) : NULL);
};
return matched;
}

64
ompi/mca/pml/uniq/pml_uniq_recvfrag.h
Просмотреть файл

@ -1,64 +0,0 @@
/*
* Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
* University Research and Technology
* Corporation. All rights reserved.
* Copyright (c) 2004-2005 The University of Tennessee and The University
* of Tennessee Research Foundation. All rights
* reserved.
* Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
* University of Stuttgart. All rights reserved.
* Copyright (c) 2004-2005 The Regents of the University of California.
* All rights reserved.
* $COPYRIGHT$
*
* Additional copyrights may follow
*
* $HEADER$
*/
/**
* @file
*/
#ifndef MCA_PML_UNIQ_RECVFRAG_H
#define MCA_PML_UNIQ_RECVFRAG_H
#include "ompi/mca/ptl/ptl.h"
#include "ompi/mca/pml/base/pml_base_recvreq.h"
#include "ompi/mca/ptl/base/ptl_base_recvfrag.h"
#include "ompi/datatype/convertor.h"
/**
* Called by the PTL to match attempt a match for new fragments.
*
* @param ptl (IN) The PTL pointer
* @param frag (IN) Receive fragment descriptor.
* @param header (IN) Header corresponding to the receive fragment.
* @return OMPI_SUCCESS or error status on failure.
*/
bool mca_pml_uniq_recv_frag_match(
mca_ptl_base_module_t* ptl,
mca_ptl_base_recv_frag_t* frag,
mca_ptl_base_match_header_t* header
);
#define MCA_PML_UNIQ_RECV_MATCHED( ptl, frag ) \
do { \
mca_pml_base_recv_request_t* _request = (mca_pml_base_recv_request_t*)(frag)->frag_request; \
/* Now that we have the sender we can create the convertor. Additionally, we know */ \
/* that the required convertor should start at the position zero as we just match */ \
/* the first fragment. */ \
if( 0 != (_request)->req_bytes_packed ) { \
(_request)->req_base.req_proc = ompi_comm_peer_lookup( \
(_request)->req_base.req_comm, \
frag->frag_base.frag_header.hdr_match.hdr_src); \
ompi_convertor_copy_and_prepare_for_recv( \
(_request)->req_base.req_proc->proc_convertor, \
(_request)->req_base.req_datatype, \
(_request)->req_base.req_count, \
(_request)->req_base.req_addr, \
&((_request)->req_convertor) ); \
} \
ptl->ptl_matched( (ptl), (frag) ); /* notify ptl of match */ \
} while (0)
#endif

284
ompi/mca/pml/uniq/pml_uniq_recvreq.c
Просмотреть файл

@ -1,284 +0,0 @@
/*
* Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
* University Research and Technology
* Corporation. All rights reserved.
* Copyright (c) 2004-2005 The University of Tennessee and The University
* of Tennessee Research Foundation. All rights
* reserved.
* Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
* University of Stuttgart. All rights reserved.
* Copyright (c) 2004-2005 The Regents of the University of California.
* All rights reserved.
* $COPYRIGHT$
*
* Additional copyrights may follow
*
* $HEADER$
*/
#include "ompi_config.h"
#include "ompi/mca/pml/pml.h"
#include "ompi/mca/ptl/ptl.h"
#include "ompi/mca/ptl/base/ptl_base_comm.h"
#include "pml_uniq_recvreq.h"
#include "pml_uniq_sendreq.h"
#include "pml_uniq_recvfrag.h"
static mca_ptl_base_recv_frag_t* mca_pml_uniq_recv_request_match_specific_proc(
mca_ptl_base_recv_request_t* request, int proc);
static int mca_pml_uniq_recv_request_fini(struct ompi_request_t** request)
{
MCA_PML_UNIQ_FINI(request);
return OMPI_SUCCESS;
}
static int mca_pml_uniq_recv_request_free(struct ompi_request_t** request)
{
MCA_PML_UNIQ_FREE(request);
return OMPI_SUCCESS;
}
static int mca_pml_uniq_recv_request_cancel(struct ompi_request_t* request, int complete)
{
mca_pml_base_request_t* uniq_request = (mca_pml_base_request_t*)request;
ompi_communicator_t* ompi_comm = uniq_request->req_comm;
mca_pml_ptl_comm_t* pml_comm = (mca_pml_ptl_comm_t*)ompi_comm->c_pml_comm;
if( true == request->req_complete ) { /* way to late to cancel this one */
return OMPI_SUCCESS;
}
/* The rest should be protected behind the match logic lock */
OPAL_THREAD_LOCK(&pml_comm->c_matching_lock);
if( OMPI_ANY_TAG == request->req_status.MPI_TAG ) { /* the match have not been already done */
if( uniq_request->req_peer == OMPI_ANY_SOURCE ) {
opal_list_remove_item( &(pml_comm->c_wild_receives),
(opal_list_item_t*)request );
} else {
opal_list_remove_item( pml_comm->c_specific_receives + uniq_request->req_peer,
(opal_list_item_t*)request );
}
}
OPAL_THREAD_UNLOCK(&pml_comm->c_matching_lock);
request->req_status._cancelled = true;
request->req_complete = true; /* mark it as completed so all the test/wait functions
* on this particular request will finish */
/* Now we have a problem if we are in a multi-threaded environment. We should
* broadcast the condition on the request in order to allow the other threads
* to complete their test/wait functions.
*/
ompi_request_completed++;
if(ompi_request_waiting) {
opal_condition_broadcast(&ompi_request_cond);
}
return OMPI_SUCCESS;
}
static void mca_pml_uniq_recv_request_construct(mca_pml_base_recv_request_t* request)
{
request->req_base.req_type = MCA_PML_REQUEST_RECV;
request->req_base.req_ompi.req_fini = mca_pml_uniq_recv_request_fini;
request->req_base.req_ompi.req_free = mca_pml_uniq_recv_request_free;
request->req_base.req_ompi.req_cancel = mca_pml_uniq_recv_request_cancel;
}
static void mca_pml_uniq_recv_request_destruct(mca_pml_base_recv_request_t* request)
{
}
OBJ_CLASS_INSTANCE(
mca_pml_uniq_recv_request_t,
mca_pml_base_recv_request_t,
mca_pml_uniq_recv_request_construct,
mca_pml_uniq_recv_request_destruct);
/*
* Update the recv request status to reflect the number of bytes
* received and actually delivered to the application.
*/
void mca_pml_uniq_recv_request_progress(
struct mca_ptl_base_module_t* ptl,
mca_ptl_base_recv_request_t* req,
size_t bytes_received,
size_t bytes_delivered)
{
OPAL_THREAD_LOCK(&ompi_request_lock);
req->req_bytes_received += bytes_received;
req->req_bytes_delivered += bytes_delivered;
if (req->req_bytes_received >= req->req_recv.req_bytes_packed) {
/* initialize request status */
req->req_recv.req_base.req_ompi.req_status._count = req->req_bytes_delivered;
req->req_recv.req_base.req_pml_complete = true;
req->req_recv.req_base.req_ompi.req_complete = true;
ompi_request_completed++;
if(ompi_request_waiting) {
opal_condition_broadcast(&ompi_request_cond);
}
}
OPAL_THREAD_UNLOCK(&ompi_request_lock);
}
/*
* This routine is used to match a posted receive when the source process
* is specified.
*/
void mca_pml_uniq_recv_request_match_specific(mca_ptl_base_recv_request_t* request)
{
ompi_communicator_t *comm = request->req_recv.req_base.req_comm;
mca_pml_ptl_comm_t* pml_comm = comm->c_pml_comm;
int req_peer = request->req_recv.req_base.req_peer;
mca_ptl_base_recv_frag_t* frag;
/* check for a specific match */
OPAL_THREAD_LOCK(&pml_comm->c_matching_lock);
/* assign sequence number */
request->req_recv.req_base.req_sequence = pml_comm->c_recv_seq++;
if (opal_list_get_size(&pml_comm->c_unexpected_frags[req_peer]) > 0 &&
(frag = mca_pml_uniq_recv_request_match_specific_proc(request, req_peer)) != NULL) {
mca_ptl_base_module_t* ptl = frag->frag_base.frag_owner;
/* setup pointer to ptls peer */
if(NULL == frag->frag_base.frag_peer)
frag->frag_base.frag_peer = mca_pml_uniq_proc_lookup_remote_peer(comm,req_peer,ptl);
OPAL_THREAD_UNLOCK(&pml_comm->c_matching_lock);
if( !((MCA_PML_REQUEST_IPROBE == request->req_recv.req_base.req_type) ||
(MCA_PML_REQUEST_PROBE == request->req_recv.req_base.req_type)) ) {
MCA_PML_UNIQ_RECV_MATCHED( ptl, frag );
}
return; /* match found */
}
/* We didn't find any matches. Record this irecv so we can match
* it when the message comes in.
*/
if(request->req_recv.req_base.req_type != MCA_PML_REQUEST_IPROBE) {
opal_list_append(pml_comm->c_specific_receives+req_peer, (opal_list_item_t*)request);
}
OPAL_THREAD_UNLOCK(&pml_comm->c_matching_lock);
}
/*
* this routine is used to try and match a wild posted receive - where
* wild is determined by the value assigned to the source process
*/
void mca_pml_uniq_recv_request_match_wild(mca_ptl_base_recv_request_t* request)
{
ompi_communicator_t *comm = request->req_recv.req_base.req_comm;
mca_pml_ptl_comm_t* pml_comm = comm->c_pml_comm;
int proc_count = comm->c_remote_group->grp_proc_count;
int proc;
/*
* Loop over all the outstanding messages to find one that matches.
* There is an outer loop over lists of messages from each
* process, then an inner loop over the messages from the
* process.
*/
OPAL_THREAD_LOCK(&pml_comm->c_matching_lock);
/* assign sequence number */
request->req_recv.req_base.req_sequence = pml_comm->c_recv_seq++;
for (proc = 0; proc < proc_count; proc++) {
mca_ptl_base_recv_frag_t* frag;
/* continue if no frags to match */
if (opal_list_get_size(&pml_comm->c_unexpected_frags[proc]) == 0)
continue;
/* loop over messages from the current proc */
if ((frag = mca_pml_uniq_recv_request_match_specific_proc(request, proc)) != NULL) {
mca_ptl_base_module_t* ptl = frag->frag_base.frag_owner;
/* if required - setup pointer to ptls peer */
if(NULL == frag->frag_base.frag_peer)
frag->frag_base.frag_peer = mca_pml_uniq_proc_lookup_remote_peer(comm,proc,ptl);
OPAL_THREAD_UNLOCK(&pml_comm->c_matching_lock);
if( !((MCA_PML_REQUEST_IPROBE == request->req_recv.req_base.req_type) ||
(MCA_PML_REQUEST_PROBE == request->req_recv.req_base.req_type)) ) {
MCA_PML_UNIQ_RECV_MATCHED( ptl, frag );
}
return; /* match found */
}
}
/* We didn't find any matches. Record this irecv so we can match to
* it when the message comes in.
*/
if(request->req_recv.req_base.req_type != MCA_PML_REQUEST_IPROBE)
opal_list_append(&pml_comm->c_wild_receives, (opal_list_item_t*)request);
OPAL_THREAD_UNLOCK(&pml_comm->c_matching_lock);
}
/*
* this routine tries to match a posted receive. If a match is found,
* it places the request in the appropriate matched receive list.
*/
static mca_ptl_base_recv_frag_t* mca_pml_uniq_recv_request_match_specific_proc(
mca_ptl_base_recv_request_t* request, int proc)
{
mca_pml_ptl_comm_t *pml_comm = request->req_recv.req_base.req_comm->c_pml_comm;
opal_list_t* unexpected_frags = pml_comm->c_unexpected_frags+proc;
mca_ptl_base_recv_frag_t* frag;
mca_ptl_base_match_header_t* header;
int tag = request->req_recv.req_base.req_tag;
if( OMPI_ANY_TAG == tag ) {
for (frag = (mca_ptl_base_recv_frag_t*)opal_list_get_first(unexpected_frags);
frag != (mca_ptl_base_recv_frag_t*)opal_list_get_end(unexpected_frags);
frag = (mca_ptl_base_recv_frag_t*)opal_list_get_next(frag)) {
header = &(frag->frag_base.frag_header.hdr_match);
/* check first frag - we assume that process matching has been done already */
if( header->hdr_tag >= 0 ) {
goto find_fragment;
}
}
} else {
for (frag = (mca_ptl_base_recv_frag_t*)opal_list_get_first(unexpected_frags);
frag != (mca_ptl_base_recv_frag_t*)opal_list_get_end(unexpected_frags);
frag = (mca_ptl_base_recv_frag_t*)opal_list_get_next(frag)) {
header = &(frag->frag_base.frag_header.hdr_match);
/* check first frag - we assume that process matching has been done already */
if ( tag == header->hdr_tag ) {
/* we assume that the tag is correct from MPI point of view (ie. >= 0 ) */
goto find_fragment;
}
}
}
return NULL;
find_fragment:
request->req_recv.req_bytes_packed = header->hdr_msg_length;
request->req_recv.req_base.req_ompi.req_status.MPI_TAG = header->hdr_tag;
request->req_recv.req_base.req_ompi.req_status.MPI_SOURCE = header->hdr_src;
if( !((MCA_PML_REQUEST_IPROBE == request->req_recv.req_base.req_type) ||
(MCA_PML_REQUEST_PROBE == request->req_recv.req_base.req_type)) ) {
opal_list_remove_item(unexpected_frags, (opal_list_item_t*)frag);
frag->frag_request = request;
} else {
/* it's a probe, therefore report it's completion */
mca_pml_uniq_recv_request_progress( NULL, request, header->hdr_msg_length, header->hdr_msg_length );
}
return frag;
}

152
ompi/mca/pml/uniq/pml_uniq_recvreq.h
Просмотреть файл

@ -1,152 +0,0 @@
/*
* Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
* University Research and Technology
* Corporation. All rights reserved.
* Copyright (c) 2004-2005 The University of Tennessee and The University
* of Tennessee Research Foundation. All rights
* reserved.
* Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
* University of Stuttgart. All rights reserved.
* Copyright (c) 2004-2005 The Regents of the University of California.
* All rights reserved.
* $COPYRIGHT$
*
* Additional copyrights may follow
*
* $HEADER$
*/
/**
* @file
*/
#ifndef OMPI_PML_UNIQ_RECV_REQUEST_H
#define OMPI_PML_UNIQ_RECV_REQUEST_H
#include "pml_uniq.h"
#include "pml_uniq_proc.h"
#include "ompi/mca/ptl/base/ptl_base_recvreq.h"
#include "ompi/mca/ptl/base/ptl_base_recvfrag.h"
#if defined(c_plusplus) || defined(__cplusplus)
extern "C" {
#endif
typedef mca_pml_base_recv_request_t mca_pml_uniq_recv_request_t;
OBJ_CLASS_DECLARATION(mca_pml_uniq_recv_request_t);
/**
* Allocate a recv request from the modules free list.
*
* @param rc (OUT) OMPI_SUCCESS or error status on failure.
* @return Receive request.
*/
#define MCA_PML_UNIQ_RECV_REQUEST_ALLOC(recvreq, rc) \
do { \
opal_list_item_t* item; \
OMPI_FREE_LIST_GET(&mca_pml_uniq.uniq_recv_requests, item, rc); \
recvreq = (mca_ptl_base_recv_request_t*)item; \
} while(0)
/**
* Initialize a recv request.
*/
#define MCA_PML_UNIQ_RECV_REQUEST_INIT( \
request, \
addr, \
count, \
datatype, \
src, \
tag, \
comm, \
persistent) \
{ \
MCA_PML_BASE_RECV_REQUEST_INIT( \
(&(request)->req_recv), \
addr, \
count, \
datatype, \
src, \
tag, \
comm, \
persistent \
); \
}
/**
* Return a recv request to the modules free list.
*
* @param request (IN) Receive request.
*/
#define MCA_PML_UNIQ_RECV_REQUEST_RETURN(request) \
do { \
MCA_PML_BASE_RECV_REQUEST_FINI( &((request)->req_recv) ); \
OMPI_FREE_LIST_RETURN(&mca_pml_uniq.uniq_recv_requests, (opal_list_item_t*)(request)); \
} while(0)
/**
* Attempt to match the request against the unexpected fragment list
* for all source ranks w/in the communicator.
*
* @param request (IN) Request to match.
*/
void mca_pml_uniq_recv_request_match_wild(mca_ptl_base_recv_request_t* request);
/**
* Attempt to match the request against the unexpected fragment list
* for a specific source rank.
*
* @param request (IN) Request to match.
*/
void mca_pml_uniq_recv_request_match_specific(mca_ptl_base_recv_request_t* request);
/**
* Start an initialized request.
*
* @param request Receive request.
* @return OMPI_SUCESS or error status on failure.
*/
static inline int mca_pml_uniq_recv_request_start(mca_ptl_base_recv_request_t* request)
{
/* init/re-init the request */
request->req_bytes_received = 0;
request->req_bytes_delivered = 0;
request->req_recv.req_base.req_pml_complete = false;
request->req_recv.req_base.req_ompi.req_complete = false;
request->req_recv.req_base.req_ompi.req_state = OMPI_REQUEST_ACTIVE;
/* always set the req_status.MPI_TAG to ANY_TAG before starting the request. This field
* is used on the cancel part in order to find out if the request has been matched or not.
*/
request->req_recv.req_base.req_ompi.req_status.MPI_TAG = OMPI_ANY_TAG;
request->req_recv.req_base.req_ompi.req_status.MPI_ERROR = OMPI_SUCCESS;
request->req_recv.req_base.req_ompi.req_status._cancelled = 0;
/* attempt to match posted recv */
if(request->req_recv.req_base.req_peer == OMPI_ANY_SOURCE) {
mca_pml_uniq_recv_request_match_wild(request);
} else {
mca_pml_uniq_recv_request_match_specific(request);
}
return OMPI_SUCCESS;
}
/**
* Update status of a recv request based on the completion status of
* the receive fragment.
*
* @param ptl (IN) The PTL pointer.
* @param request (IN) Receive request.
* @param bytes_received (IN) Bytes received from peer.
* @param bytes_delivered (IN) Bytes delivered to application.
*/
void mca_pml_uniq_recv_request_progress(
struct mca_ptl_base_module_t* ptl,
mca_ptl_base_recv_request_t* request,
size_t bytes_received,
size_t bytes_delivered
);
#if defined(c_plusplus) || defined(__cplusplus)
}
#endif
#endif /* OMPI_PML_UNIQ_RECV_REQUEST_H */

197
ompi/mca/pml/uniq/pml_uniq_sendreq.c
Просмотреть файл

@ -1,197 +0,0 @@
/*
* Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
* University Research and Technology
* Corporation. All rights reserved.
* Copyright (c) 2004-2005 The University of Tennessee and The University
* of Tennessee Research Foundation. All rights
* reserved.
* Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
* University of Stuttgart. All rights reserved.
* Copyright (c) 2004-2005 The Regents of the University of California.
* All rights reserved.
* $COPYRIGHT$
*
* Additional copyrights may follow
*
* $HEADER$
*/
/*%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%*/
#include "ompi_config.h"
#include "ompi/constants.h"
#include "ompi/mca/pml/pml.h"
#include "ompi/mca/ptl/ptl.h"
#include "pml_uniq.h"
#include "pml_uniq_proc.h"
#include "pml_uniq_sendreq.h"
#include "pml_uniq_recvreq.h"
static int mca_pml_uniq_send_request_fini(struct ompi_request_t** request)
{
MCA_PML_UNIQ_FINI(request);
return OMPI_SUCCESS;
}
static int mca_pml_uniq_send_request_free(struct ompi_request_t** request)
{
MCA_PML_UNIQ_FREE(request);
return OMPI_SUCCESS;
}
static int mca_pml_uniq_send_request_cancel(struct ompi_request_t* request, int complete)
{
/* we dont cancel send requests by now */
return OMPI_SUCCESS;
}
static void mca_pml_uniq_send_request_construct(mca_pml_base_send_request_t* req)
{
req->req_base.req_type = MCA_PML_REQUEST_SEND;
req->req_base.req_ompi.req_fini = mca_pml_uniq_send_request_fini;
req->req_base.req_ompi.req_free = mca_pml_uniq_send_request_free;
req->req_base.req_ompi.req_cancel = mca_pml_uniq_send_request_cancel;
}
static void mca_pml_uniq_send_request_destruct(mca_pml_base_send_request_t* req)
{
}
OBJ_CLASS_INSTANCE(
mca_pml_uniq_send_request_t,
mca_pml_base_send_request_t,
mca_pml_uniq_send_request_construct,
mca_pml_uniq_send_request_destruct);
/**
* Schedule message delivery across potentially multiple PTLs.
*
* @param request (IN) Request to schedule
* @return status Error status
*
*/
int mca_pml_uniq_send_request_schedule(mca_ptl_base_send_request_t* req)
{
int rc;
size_t bytes_remaining;
/*
* Only allow one thread in this routine for a given request.
* However, we cannot block callers on a mutex, so simply keep track
* of the number of times the routine has been called and run through
* the scheduling logic once for every call.
*/
if(OPAL_THREAD_ADD32(&req->req_lock,1) == 1) {
mca_pml_uniq_proc_t* proc_pml = (mca_pml_uniq_proc_t*)
mca_pml_uniq_proc_lookup_remote( req->req_send.req_base.req_comm,
req->req_send.req_base.req_peer );
#if PML_UNIQ_ACCEPT_NEXT_PTL
mca_ptl_proc_t* ptl_proc = &(proc_pml->proc_ptl_next);
#else
mca_ptl_proc_t* ptl_proc = &(proc_pml->proc_ptl_first);
#endif /* PML_UNIQ_ACCEPT_NEXT_PTL */
mca_ptl_base_module_t* ptl = ptl_proc->ptl;
/* allocate remaining bytes to PTLs */
bytes_remaining = req->req_send.req_bytes_packed - req->req_offset;
/* The rest of the message will be scheduled over the same PTL (the one in the next field). We try
* to be PTL friendly here so we will respect the maximum size accepted by the PTL.
*/
if( bytes_remaining > ptl->ptl_max_frag_size) {
bytes_remaining = ptl->ptl_max_frag_size;
}
rc = ptl->ptl_put(ptl, ptl_proc->ptl_peer, req, req->req_offset, bytes_remaining, 0);
if(rc == OMPI_SUCCESS) {
bytes_remaining = req->req_send.req_bytes_packed - req->req_offset;
} else { /* unable to complete send - queue for later */
OPAL_THREAD_LOCK(&mca_pml_uniq.uniq_lock);
opal_list_append(&mca_pml_uniq.uniq_send_pending, (opal_list_item_t*)req);
OPAL_THREAD_UNLOCK(&mca_pml_uniq.uniq_lock);
req->req_lock = 0;
return OMPI_ERR_OUT_OF_RESOURCE;
}
OPAL_THREAD_ADD32(&req->req_lock,-1);
/* free the request if completed while in the scheduler */
if (req->req_send.req_base.req_free_called && req->req_send.req_base.req_pml_complete) {
MCA_PML_UNIQ_FREE((ompi_request_t**)&req);
}
return OMPI_SUCCESS;
}
OPAL_THREAD_ADD32(&req->req_lock,-1);
return OMPI_SUCCESS;
}
/**
* Update the status of the send request to reflect the number of bytes
* "actually" sent (and acknowledged). This should be called by the
* lower layer PTL after the fragment is actually delivered and has been
* acknowledged (if required). Note that this routine should NOT be called
* directly by the PTL, a function pointer is setup on the PTL at init to
* enable upcalls into the PML w/out directly linking to a specific PML
* implementation.
*/
void mca_pml_uniq_send_request_progress(
struct mca_ptl_base_module_t* ptl,
mca_ptl_base_send_request_t* req,
size_t bytes_sent)
{
bool schedule = false;
OPAL_THREAD_LOCK(&ompi_request_lock);
req->req_bytes_sent += bytes_sent;
if (req->req_bytes_sent >= req->req_send.req_bytes_packed) {
req->req_send.req_base.req_pml_complete = true;
if (req->req_send.req_base.req_ompi.req_complete == false) {
req->req_send.req_base.req_ompi.req_status.MPI_SOURCE = req->req_send.req_base.req_comm->c_my_rank;
req->req_send.req_base.req_ompi.req_status.MPI_TAG = req->req_send.req_base.req_tag;
req->req_send.req_base.req_ompi.req_status.MPI_ERROR = OMPI_SUCCESS;
req->req_send.req_base.req_ompi.req_status._count = req->req_bytes_sent;
req->req_send.req_base.req_ompi.req_complete = true;
ompi_request_completed++;
if(ompi_request_waiting) {
opal_condition_broadcast(&ompi_request_cond);
}
} else if(req->req_send.req_base.req_free_called) {
/* don't free the request if in the scheduler */
if(req->req_lock == 0) {
MCA_PML_UNIQ_FREE((ompi_request_t**)&req);
}
} else if (req->req_send.req_send_mode == MCA_PML_BASE_SEND_BUFFERED) {
mca_pml_base_bsend_request_fini((ompi_request_t*)req);
}
/* test to see if we have scheduled the entire request */
} else if (req->req_offset < req->req_send.req_bytes_packed) {
schedule = true;
}
OPAL_THREAD_UNLOCK(&ompi_request_lock);
/* schedule remaining fragments of this request */
if(schedule) {
mca_pml_uniq_send_request_schedule(req);
}
/* check for pending requests that need to be progressed */
while(opal_list_get_size(&mca_pml_uniq.uniq_send_pending) != 0) {
OPAL_THREAD_LOCK(&mca_pml_uniq.uniq_lock);
req = (mca_ptl_base_send_request_t*)opal_list_remove_first(&mca_pml_uniq.uniq_send_pending);
OPAL_THREAD_UNLOCK(&mca_pml_uniq.uniq_lock);
if(req == NULL)
break;
if(mca_pml_uniq_send_request_schedule(req) != OMPI_SUCCESS)
break;
}
}

209
ompi/mca/pml/uniq/pml_uniq_sendreq.h
Просмотреть файл

@ -1,209 +0,0 @@
/*
* Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
* University Research and Technology
* Corporation. All rights reserved.
* Copyright (c) 2004-2005 The University of Tennessee and The University
* of Tennessee Research Foundation. All rights
* reserved.
* Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
* University of Stuttgart. All rights reserved.
* Copyright (c) 2004-2005 The Regents of the University of California.
* All rights reserved.
* $COPYRIGHT$
*
* Additional copyrights may follow
*
* $HEADER$
*/
/**
* @file
*/
#ifndef OMPI_PML_UNIQ_SEND_REQUEST_H
#define OMPI_PML_UNIQ_SEND_REQUEST_H
#include "ompi/mca/ptl/ptl.h"
#include "ompi/mca/ptl/base/ptl_base_sendreq.h"
#include "ompi/mca/ptl/base/ptl_base_sendfrag.h"
#include "ompi/mca/ptl/base/ptl_base_comm.h"
#include "pml_uniq_proc.h"
#include "pml_uniq_ptl.h"
#if defined(c_plusplus) || defined(__cplusplus)
extern "C" {
#endif
typedef mca_pml_base_send_request_t mca_pml_uniq_send_request_t;
OBJ_CLASS_DECLARATION(mca_pml_uniq_send_request_t);
#define MCA_PML_UNIQ_SEND_REQUEST_ALLOC( \
comm, \
dst, \
sendreq, \
rc) \
{ \
mca_pml_uniq_proc_t *proc = \
(mca_pml_uniq_proc_t*) mca_pml_uniq_proc_lookup_remote(comm,dst); \
mca_pml_uniq_ptl_t* ptl_base; \
\
if(NULL == proc) { \
return OMPI_ERR_OUT_OF_RESOURCE; \
} \
ptl_base = proc->proc_ptl_first.ptl_base; \
/* \
* check to see if there is a cache of send requests associated with \
* this ptl - if so try the allocation from there. \
*/ \
if(NULL != ptl_base) { \
OPAL_THREAD_LOCK(&ptl_base->ptl_cache_lock); \
sendreq = (mca_ptl_base_send_request_t*) \
opal_list_remove_first(&ptl_base->ptl_cache); \
if(NULL != sendreq) { \
OPAL_THREAD_UNLOCK(&ptl_base->ptl_cache_lock); \
rc = OMPI_SUCCESS; \
} else if (ptl_base->ptl_cache_alloc < ptl_base->ptl_cache_size) { \
/* \
* allocate an additional request to the cache \
*/ \
mca_ptl_base_module_t* ptl = ptl_base->ptl; \
opal_list_item_t* item; \
OMPI_FREE_LIST_WAIT(&mca_pml_uniq.uniq_send_requests, item, rc); \
sendreq = (mca_ptl_base_send_request_t*)item; \
sendreq->req_ptl = ptl; \
if(ptl->ptl_request_init(ptl, sendreq) == OMPI_SUCCESS) { \
sendreq->req_cached = true; \
ptl_base->ptl_cache_alloc++; \
} \
OPAL_THREAD_UNLOCK(&ptl_base->ptl_cache_lock); \
} else { \
/* \
* take a request from the global pool \
*/ \
opal_list_item_t* item; \
OPAL_THREAD_UNLOCK(&ptl_base->ptl_cache_lock); \
OMPI_FREE_LIST_WAIT(&mca_pml_uniq.uniq_send_requests, item, rc); \
sendreq = (mca_ptl_base_send_request_t*)item; \
sendreq->req_ptl = proc->proc_ptl_first.ptl; \
} \
\
/* otherwise - take the allocation from the global list */ \
} else { \
opal_list_item_t* item; \
OMPI_FREE_LIST_WAIT(&mca_pml_uniq.uniq_send_requests, item, rc); \
sendreq = (mca_ptl_base_send_request_t*)item; \
sendreq->req_ptl = proc->proc_ptl_first.ptl; \
} \
/* update request to point to current peer */ \
sendreq->req_peer = proc->proc_ptl_first.ptl_peer; \
sendreq->req_send.req_base.req_proc = proc->base.proc_ompi; \
}
#define MCA_PML_UNIQ_SEND_REQUEST_INIT( request, \
addr, \
count, \
datatype, \
peer, \
tag, \
comm, \
mode, \
persistent) \
{ \
MCA_PML_BASE_SEND_REQUEST_INIT((&request->req_send), \
addr, \
count, \
datatype, \
peer, \
tag, \
comm, \
mode, \
persistent \
); \
}
#define MCA_PML_UNIQ_SEND_REQUEST_RETURN(sendreq) \
{ \
mca_ptl_base_module_t* ptl = (sendreq)->req_ptl; \
mca_pml_uniq_ptl_t* ptl_base = (mca_pml_uniq_ptl_t*)ptl->ptl_base; \
\
/* Let the base handle the reference counts */ \
MCA_PML_BASE_SEND_REQUEST_FINI( &((sendreq)->req_send) ); \
\
/* \
* If there is a cache associated with the ptl - first attempt \
* to return the send descriptor to the cache. \
*/ \
if(NULL != ptl->ptl_base && (sendreq)->req_cached) { \
OPAL_THREAD_LOCK(&ptl_base->ptl_cache_lock); \
opal_list_prepend(&ptl_base->ptl_cache, \
(opal_list_item_t*)sendreq); \
OPAL_THREAD_UNLOCK(&ptl_base->ptl_cache_lock); \
} else { \
OMPI_FREE_LIST_RETURN( \
&mca_pml_uniq.uniq_send_requests, (opal_list_item_t*)(sendreq)); \
} \
}
/**
* Start a send request.
*/
#define MCA_PML_UNIQ_SEND_REQUEST_START(req, rc) \
{ \
mca_ptl_base_module_t* ptl = req->req_ptl; \
size_t first_fragment_size = ptl->ptl_first_frag_size; \
int flags; \
\
req->req_offset = 0; \
req->req_lock = 0; \
req->req_bytes_sent = 0; \
req->req_peer_match.lval = 0; \
req->req_peer_addr.lval = 0; \
req->req_peer_size = 0; \
req->req_send.req_base.req_pml_complete = false; \
req->req_send.req_base.req_ompi.req_complete = false; \
req->req_send.req_base.req_ompi.req_state = OMPI_REQUEST_ACTIVE; \
req->req_send.req_base.req_sequence = mca_pml_ptl_comm_send_sequence( \
req->req_send.req_base.req_comm->c_pml_comm, req->req_send.req_base.req_peer); \
\
/* handle buffered send */ \
if(req->req_send.req_send_mode == MCA_PML_BASE_SEND_BUFFERED) { \
mca_pml_base_bsend_request_start(&req->req_send.req_base.req_ompi); \
} \
\
/* start the first fragment */ \
if (first_fragment_size == 0 || \
req->req_send.req_bytes_packed <= first_fragment_size) { \
first_fragment_size = req->req_send.req_bytes_packed; \
flags = (req->req_send.req_send_mode == MCA_PML_BASE_SEND_SYNCHRONOUS) ? \
MCA_PTL_FLAGS_ACK : 0; \
} else { \
/* require match for first fragment of a multi-fragment */ \
flags = MCA_PTL_FLAGS_ACK; \
} \
rc = ptl->ptl_send(ptl, req->req_peer, req, 0, first_fragment_size, \
flags); \
}
/**
* Schedule any data that was not delivered in the first fragment
* across the available PTLs.
*/
int mca_pml_uniq_send_request_schedule(mca_ptl_base_send_request_t* req);
/**
* Update the request to reflect the number of bytes delivered. If this
* was the first fragment - schedule the rest of the data.
*/
void mca_pml_uniq_send_request_progress(
struct mca_ptl_base_module_t* ptl,
mca_ptl_base_send_request_t* send_request,
size_t bytes_sent
);
#if defined(c_plusplus) || defined(__cplusplus)
}
#endif
#endif

124
ompi/mca/pml/uniq/pml_uniq_start.c
Просмотреть файл

@ -1,124 +0,0 @@
/*
* Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
* University Research and Technology
* Corporation. All rights reserved.
* Copyright (c) 2004-2005 The University of Tennessee and The University
* of Tennessee Research Foundation. All rights
* reserved.
* Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
* University of Stuttgart. All rights reserved.
* Copyright (c) 2004-2005 The Regents of the University of California.
* All rights reserved.
* $COPYRIGHT$
*
* Additional copyrights may follow
*
* $HEADER$
*/
#include "ompi_config.h"
#include "pml_uniq.h"
#include "pml_uniq_recvreq.h"
#include "pml_uniq_sendreq.h"
int mca_pml_uniq_start(size_t count, ompi_request_t** requests)
{
int rc;
size_t i;
for(i=0; i<count; i++) {
mca_pml_base_request_t *pml_request = (mca_pml_base_request_t*)requests[i];
if(NULL == pml_request)
continue;
/* If the persistent request is currently active - obtain the
* request lock and verify the status is incomplete. if the
* pml layer has not completed the request - mark the request
* as free called - so that it will be freed when the request
* completes - and create a new request.
*/
switch(pml_request->req_ompi.req_state) {
case OMPI_REQUEST_INACTIVE:
if(pml_request->req_pml_complete == true)
break;
/* otherwise fall through */
case OMPI_REQUEST_ACTIVE: {
ompi_request_t *request;
OPAL_THREAD_LOCK(&ompi_request_lock);
if (pml_request->req_pml_complete == false) {
/* free request after it completes */
pml_request->req_free_called = true;
} else {
/* can reuse the existing request */
OPAL_THREAD_UNLOCK(&ompi_request_lock);
break;
}
/* allocate a new request */
switch(pml_request->req_type) {
case MCA_PML_REQUEST_SEND: {
mca_pml_base_send_mode_t sendmode =
((mca_pml_base_send_request_t*)pml_request)->req_send_mode;
rc = mca_pml_uniq_isend_init(
pml_request->req_addr,
pml_request->req_count,
pml_request->req_datatype,
pml_request->req_peer,
pml_request->req_tag,
sendmode,
pml_request->req_comm,
&request);
break;
}
case MCA_PML_REQUEST_RECV:
rc = mca_pml_uniq_irecv_init(
pml_request->req_addr,
pml_request->req_count,
pml_request->req_datatype,
pml_request->req_peer,
pml_request->req_tag,
pml_request->req_comm,
&request);
break;
default:
rc = OMPI_ERR_REQUEST;
break;
}
OPAL_THREAD_UNLOCK(&ompi_request_lock);
if(OMPI_SUCCESS != rc)
return rc;
pml_request = (mca_pml_base_request_t*)request;
requests[i] = request;
break;
}
default:
return OMPI_ERR_REQUEST;
}
/* start the request */
switch(pml_request->req_type) {
case MCA_PML_REQUEST_SEND:
{
mca_ptl_base_send_request_t* sendreq = (mca_ptl_base_send_request_t*)pml_request;
MCA_PML_UNIQ_SEND_REQUEST_START(sendreq, rc);
if(rc != OMPI_SUCCESS)
return rc;
break;
}
case MCA_PML_REQUEST_RECV:
{
mca_ptl_base_recv_request_t* recvreq = (mca_ptl_base_recv_request_t*)pml_request;
if((rc = mca_pml_uniq_recv_request_start(recvreq)) != OMPI_SUCCESS)
return rc;
break;
}
default:
return OMPI_ERR_REQUEST;
}
}
return OMPI_SUCCESS;
}

41
ompi/mca/ptl/Makefile.am
Просмотреть файл

@ -1,41 +0,0 @@
#
# Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
# University Research and Technology
# Corporation. All rights reserved.
# Copyright (c) 2004-2005 The University of Tennessee and The University
# of Tennessee Research Foundation. All rights
# reserved.
# Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
# University of Stuttgart. All rights reserved.
# Copyright (c) 2004-2005 The Regents of the University of California.
# All rights reserved.
# $COPYRIGHT$
#
# Additional copyrights may follow
#
# $HEADER$
#
# main library setup
noinst_LTLIBRARIES = libmca_ptl.la
libmca_ptl_la_SOURCES =
# header setup
nobase_ompi_HEADERS =
# local files
headers = ptl.h
libmca_ptl_la_SOURCES += $(headers)
# Conditionally install the header files
if WANT_INSTALL_HEADERS
nobase_ompi_HEADERS += $(headers)
ompidir = $(includedir)/openmpi/ompi/mca/ptl
else
ompidir = $(includedir)
endif
include base/Makefile.am
distclean-local:
rm -f base/static-components.h

40
ompi/mca/ptl/base/Makefile.am
Просмотреть файл

@ -1,40 +0,0 @@
#
# Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
# University Research and Technology
# Corporation. All rights reserved.
# Copyright (c) 2004-2005 The University of Tennessee and The University
# of Tennessee Research Foundation. All rights
# reserved.
# Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
# University of Stuttgart. All rights reserved.
# Copyright (c) 2004-2005 The Regents of the University of California.
# All rights reserved.
# $COPYRIGHT$
#
# Additional copyrights may follow
#
# $HEADER$
#
headers += \
base/base.h \
base/ptl_base_comm.h \
base/ptl_base_fragment.h \
base/ptl_base_header.h \
base/ptl_base_match.h \
base/ptl_base_recvfrag.h \
base/ptl_base_recvreq.h \
base/ptl_base_sendfrag.h \
base/ptl_base_sendreq.h
libmca_ptl_la_SOURCES += \
base/ptl_base_close.c \
base/ptl_base_comm.c \
base/ptl_base_fragment.c \
base/ptl_base_match.c \
base/ptl_base_open.c \
base/ptl_base_recvfrag.c \
base/ptl_base_recvreq.c \
base/ptl_base_select.c \
base/ptl_base_sendfrag.c \
base/ptl_base_sendreq.c

66
ompi/mca/ptl/base/base.h
Просмотреть файл

@ -1,66 +0,0 @@
/*
* Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
* University Research and Technology
* Corporation. All rights reserved.
* Copyright (c) 2004-2005 The University of Tennessee and The University
* of Tennessee Research Foundation. All rights
* reserved.
* Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
* University of Stuttgart. All rights reserved.
* Copyright (c) 2004-2005 The Regents of the University of California.
* All rights reserved.
* $COPYRIGHT$
*
* Additional copyrights may follow
*
* $HEADER$
*/
/**
* @file
*/
#ifndef MCA_PTL_BASE_H
#define MCA_PTL_BASE_H
#include "ompi_config.h"
#include "opal/class/opal_list.h"
#include "opal/mca/mca.h"
#include "ompi/mca/ptl/ptl.h"
#if defined(c_plusplus) || defined(__cplusplus)
extern "C" {
#endif
struct mca_ptl_base_selected_module_t {
opal_list_item_t super;
mca_ptl_base_component_t *pbsm_component;
mca_ptl_base_module_t *pbsm_module;
};
typedef struct mca_ptl_base_selected_module_t mca_ptl_base_selected_module_t;
/*
* Global functions for MCA: overall PTL open and close
*/
OMPI_DECLSPEC int mca_ptl_base_open(void);
OMPI_DECLSPEC int mca_ptl_base_select(bool enable_progress_threads,
bool enable_mpi_threads);
OMPI_DECLSPEC int mca_ptl_base_close(void);
/*
* Globals
*/
OMPI_DECLSPEC extern int mca_ptl_base_output;
OMPI_DECLSPEC extern char* mca_ptl_base_include;
OMPI_DECLSPEC extern char* mca_ptl_base_exclude;
OMPI_DECLSPEC extern opal_list_t mca_ptl_base_components_opened;
OMPI_DECLSPEC extern opal_list_t mca_ptl_base_components_initialized;
OMPI_DECLSPEC extern opal_list_t mca_ptl_base_modules_initialized;
#if defined(c_plusplus) || defined(__cplusplus)
}
#endif
#endif /* MCA_PTL_BASE_H */

84
ompi/mca/ptl/base/ptl_base_close.c
Просмотреть файл

@ -1,84 +0,0 @@
/*
* Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
* University Research and Technology
* Corporation. All rights reserved.
* Copyright (c) 2004-2005 The University of Tennessee and The University
* of Tennessee Research Foundation. All rights
* reserved.
* Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
* University of Stuttgart. All rights reserved.
* Copyright (c) 2004-2005 The Regents of the University of California.
* All rights reserved.
* $COPYRIGHT$
*
* Additional copyrights may follow
*
* $HEADER$
*/
#include "ompi_config.h"
#include <stdio.h>
#include "ompi/constants.h"
#include "opal/event/event.h"
#include "opal/mca/mca.h"
#include "opal/mca/base/base.h"
#include "ompi/mca/pml/pml.h"
#include "ompi/mca/ptl/ptl.h"
#include "ompi/mca/ptl/base/base.h"
extern int mca_ptl_base_open_called;
int mca_ptl_base_close(void)
{
opal_list_item_t *item;
mca_ptl_base_selected_module_t *sm;
if( 0 == mca_ptl_base_open_called ) return OMPI_ERROR;
mca_ptl_base_open_called = 0;
/* disable event processing while cleaning up ptls */
opal_event_disable();
/* Finalize all the ptl components and free their list items */
for (item = opal_list_remove_first(&mca_ptl_base_modules_initialized);
NULL != item;
item = opal_list_remove_first(&mca_ptl_base_modules_initialized)) {
sm = (mca_ptl_base_selected_module_t *) item;
/* Blatently ignore the return code (what would we do to recover,
anyway? This component is going away, so errors don't matter
anymore) */
sm->pbsm_module->ptl_finalize(sm->pbsm_module);
free(sm);
}
/* Close all remaining opened components (may be one if this is a
OMPI RTE program, or [possibly] multiple if this is ompi_info) */
if (0 != opal_list_get_size(&mca_ptl_base_components_initialized)) {
mca_base_components_close(mca_ptl_base_output,
&mca_ptl_base_components_initialized, NULL);
}
OBJ_DESTRUCT( &mca_ptl_base_components_initialized );
OBJ_DESTRUCT( &mca_ptl_base_components_opened );
/* cleanup */
if( NULL != mca_ptl_base_include ) {
free(mca_ptl_base_include);
mca_ptl_base_include = NULL;
}
if( NULL != mca_ptl_base_exclude ) {
free(mca_ptl_base_exclude);
mca_ptl_base_exclude = NULL;
}
/* restore event processing */
opal_event_enable();
/* All done */
return OMPI_SUCCESS;
}

103
ompi/mca/ptl/base/ptl_base_comm.c
Просмотреть файл

@ -1,103 +0,0 @@
/*
* Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
* University Research and Technology
* Corporation. All rights reserved.
* Copyright (c) 2004-2005 The University of Tennessee and The University
* of Tennessee Research Foundation. All rights
* reserved.
* Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
* University of Stuttgart. All rights reserved.
* Copyright (c) 2004-2005 The Regents of the University of California.
* All rights reserved.
* $COPYRIGHT$
*
* Additional copyrights may follow
*
* $HEADER$
*/
#include "ompi_config.h"
#include <string.h>
#include "ompi/constants.h"
#include "ompi/mca/pml/pml.h"
#include "ompi/mca/ptl/base/ptl_base_comm.h"
static void mca_pml_ptl_comm_construct(mca_pml_ptl_comm_t* comm);
static void mca_pml_ptl_comm_destruct(mca_pml_ptl_comm_t* comm);
opal_class_t mca_pml_ptl_comm_t_class = {
"mca_pml_ptl_comm_t",
OBJ_CLASS(opal_object_t),
(opal_construct_t)mca_pml_ptl_comm_construct,
(opal_destruct_t)mca_pml_ptl_comm_destruct
};
static void mca_pml_ptl_comm_construct(mca_pml_ptl_comm_t* comm)
{
OBJ_CONSTRUCT(&comm->c_wild_receives, opal_list_t);
OBJ_CONSTRUCT(&comm->c_matching_lock, opal_mutex_t);
comm->c_recv_seq = 0;
}
static void mca_pml_ptl_comm_destruct(mca_pml_ptl_comm_t* comm)
{
free(comm->c_msg_seq);
free(comm->c_next_msg_seq);
free(comm->c_unexpected_frags);
free(comm->c_frags_cant_match);
free(comm->c_specific_receives);
OBJ_DESTRUCT(&comm->c_wild_receives);
OBJ_DESTRUCT(&comm->c_matching_lock);
}
int mca_pml_ptl_comm_init_size(mca_pml_ptl_comm_t* comm, size_t size)
{
size_t i;
/* send message sequence-number support - sender side */
comm->c_msg_seq = malloc(sizeof(uint32_t) * size);
if(NULL == comm->c_msg_seq)
return OMPI_ERR_OUT_OF_RESOURCE;
memset(comm->c_msg_seq, 0, sizeof(uint32_t) * size);
/* send message sequence-number support - receiver side */
comm->c_next_msg_seq = malloc(sizeof(uint16_t) * size);
if(NULL == comm->c_next_msg_seq)
return OMPI_ERR_OUT_OF_RESOURCE;
memset(comm->c_next_msg_seq, 0, sizeof(uint16_t) * size);
/* unexpected fragments queues */
comm->c_unexpected_frags = malloc(sizeof(opal_list_t) * size);
if(NULL == comm->c_unexpected_frags)
return OMPI_ERR_OUT_OF_RESOURCE;
for(i=0; i<size; i++) {
opal_list_t* object = comm->c_unexpected_frags+i;
OBJ_CONSTRUCT(object, opal_list_t);
}
/* out-of-order fragments queues */
comm->c_frags_cant_match = malloc(sizeof(opal_list_t) * size);
if(NULL == comm->c_frags_cant_match)
return OMPI_ERR_OUT_OF_RESOURCE;
for(i=0; i<size; i++) {
opal_list_t* object = comm->c_frags_cant_match+i;
OBJ_CONSTRUCT(object, opal_list_t);
}
/* queues of unmatched specific (source process specified) receives */
comm->c_specific_receives = malloc(sizeof(opal_list_t) * size);
if(NULL == comm->c_specific_receives)
return OMPI_ERR_OUT_OF_RESOURCE;
for(i=0; i<size; i++) {
opal_list_t *object = comm->c_specific_receives+i;
OBJ_CONSTRUCT(object, opal_list_t);
}
return OMPI_SUCCESS;
}

79
ompi/mca/ptl/base/ptl_base_comm.h
Просмотреть файл

@ -1,79 +0,0 @@
/*
* Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
* University Research and Technology
* Corporation. All rights reserved.
* Copyright (c) 2004-2005 The University of Tennessee and The University
* of Tennessee Research Foundation. All rights
* reserved.
* Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
* University of Stuttgart. All rights reserved.
* Copyright (c) 2004-2005 The Regents of the University of California.
* All rights reserved.
* $COPYRIGHT$
*
* Additional copyrights may follow
*
* $HEADER$
*/
/**
* @file
*/
#ifndef MCA_PML_COMM_H
#define MCA_PML_COMM_H
#include "opal/threads/mutex.h"
#include "opal/threads/condition.h"
#include "ompi/mca/ptl/ptl.h"
#include "opal/class/opal_list.h"
#if defined(c_plusplus) || defined(__cplusplus)
extern "C" {
#endif
OMPI_DECLSPEC extern opal_class_t mca_pml_ptl_comm_t_class;
/**
* Cached on ompi_communicator_t to hold queues/state
* used by the PML<->PTL interface for matching logic.
*/
struct mca_pml_comm_t {
opal_object_t super;
uint32_t *c_msg_seq; /**< send message sequence number - sender side */
uint16_t *c_next_msg_seq; /**< send message sequence number - receiver side */
mca_ptl_sequence_t c_recv_seq; /**< recv request sequence number - receiver side */
opal_mutex_t c_matching_lock; /**< matching lock */
opal_list_t *c_unexpected_frags; /**< unexpected fragment queues */
opal_list_t *c_frags_cant_match; /**< out-of-order fragment queues */
opal_list_t *c_specific_receives; /**< queues of unmatched specific (source process specified) receives */
opal_list_t c_wild_receives; /**< queue of unmatched wild (source process not specified) receives */
};
typedef struct mca_pml_comm_t mca_pml_ptl_comm_t;
/**
* Initialize an instance of mca_pml_ptl_comm_t based on the communicator size.
*
* @param comm Instance of mca_pml_ptl_comm_t
* @param size Size of communicator
* @return OMPI_SUCCESS or error status on failure.
*/
OMPI_DECLSPEC extern int mca_pml_ptl_comm_init_size(mca_pml_ptl_comm_t* comm, size_t size);
/**
* Obtain the next sequence number (MPI) for a given destination rank.
*
* @param comm Instance of mca_pml_ptl_comm_t
* @param dst Rank of destination.
* @return Next available sequence number.
*/
static inline mca_ptl_sequence_t mca_pml_ptl_comm_send_sequence(mca_pml_ptl_comm_t* comm, int dst)
{
volatile int32_t *msg_seq = (volatile int32_t*)(comm->c_msg_seq+dst);
return (mca_ptl_sequence_t)OPAL_THREAD_ADD32(msg_seq, 1)-1;
}
#if defined(c_plusplus) || defined(__cplusplus)
}
#endif
#endif

43
ompi/mca/ptl/base/ptl_base_fragment.c
Просмотреть файл

@ -1,43 +0,0 @@
/*
* Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
* University Research and Technology
* Corporation. All rights reserved.
* Copyright (c) 2004-2005 The University of Tennessee and The University
* of Tennessee Research Foundation. All rights
* reserved.
* Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
* University of Stuttgart. All rights reserved.
* Copyright (c) 2004-2005 The Regents of the University of California.
* All rights reserved.
* $COPYRIGHT$
*
* Additional copyrights may follow
*
* $HEADER$
*/
/*%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%*/
#include "ompi_config.h"
#include "opal/class/opal_list.h"
#include "ompi/mca/ptl/base/ptl_base_fragment.h"
static void mca_ptl_base_frag_construct(mca_ptl_base_frag_t* frag);
static void mca_ptl_base_frag_destruct(mca_ptl_base_frag_t* frag);
opal_class_t mca_ptl_base_frag_t_class = {
"mca_ptl_base_frag_t",
OBJ_CLASS(opal_list_item_t),
(opal_construct_t) mca_ptl_base_frag_construct,
(opal_destruct_t) mca_ptl_base_frag_destruct
};
static void mca_ptl_base_frag_construct(mca_ptl_base_frag_t* frag)
{
OBJ_CONSTRUCT(&frag->frag_convertor, ompi_convertor_t);
}
static void mca_ptl_base_frag_destruct(mca_ptl_base_frag_t* frag)
{
}

66
ompi/mca/ptl/base/ptl_base_fragment.h
Просмотреть файл

@ -1,66 +0,0 @@
/*
* Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
* University Research and Technology
* Corporation. All rights reserved.
* Copyright (c) 2004-2005 The University of Tennessee and The University
* of Tennessee Research Foundation. All rights
* reserved.
* Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
* University of Stuttgart. All rights reserved.
* Copyright (c) 2004-2005 The Regents of the University of California.
* All rights reserved.
* $COPYRIGHT$
*
* Additional copyrights may follow
*
* $HEADER$
*/
/**
* @file
*/
#ifndef MCA_PTL_BASE_FRAGMENT_H
#define MCA_PTL_BASE_FRAGMENT_H
#include "opal/class/opal_list.h"
#include "ompi/mca/pml/pml.h"
#include "ompi/mca/ptl/ptl.h"
#include "ompi/datatype/datatype.h"
#include "ompi/datatype/convertor.h"
#include "ompi/mca/ptl/base/ptl_base_header.h"
#if defined(c_plusplus) || defined(__cplusplus)
extern "C" {
#endif
/**
* Type of fragment
*/
typedef enum {
MCA_PTL_FRAGMENT_SEND,
MCA_PTL_FRAGMENT_RECV
} mca_ptl_base_frag_type_t;
/**
* Base type for fragment descriptors.
*/
struct mca_ptl_base_frag_t {
opal_list_item_t super; /**< allow the fragment to be placed on a list */
mca_ptl_base_header_t frag_header; /**< header used for fragment matching */
struct mca_ptl_base_module_t* frag_owner; /**< PTL that allocated this fragment */
struct mca_ptl_base_peer_t* frag_peer; /**< PTL specific addressing info */
void *frag_addr; /**< pointer into request buffer at fragment offset */
size_t frag_size; /**< number of bytes available in request buffer */
mca_ptl_base_frag_type_t frag_type; /**< fragment derived type */
ompi_convertor_t frag_convertor; /**< datatype convertor for fragment packing/unpacking */
};
typedef struct mca_ptl_base_frag_t mca_ptl_base_frag_t;
OMPI_DECLSPEC OBJ_CLASS_DECLARATION(mca_ptl_base_frag_t);
#if defined(c_plusplus) || defined(__cplusplus)
}
#endif
#endif

226
ompi/mca/ptl/base/ptl_base_header.h
Просмотреть файл

@ -1,226 +0,0 @@
/*
* Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
* University Research and Technology
* Corporation. All rights reserved.
* Copyright (c) 2004-2005 The University of Tennessee and The University
* of Tennessee Research Foundation. All rights
* reserved.
* Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
* University of Stuttgart. All rights reserved.
* Copyright (c) 2004-2005 The Regents of the University of California.
* All rights reserved.
* $COPYRIGHT$
*
* Additional copyrights may follow
*
* $HEADER$
*/
/**
* @file
*/
#ifndef MCA_PTL_BASE_HEADER_H
#define MCA_PTL_BASE_HEADER_H
#include "ompi_config.h"
#include "ompi/mca/ptl/ptl.h"
#ifdef HAVE_SYS_TYPES_H
#include <sys/types.h>
#endif
#ifdef HAVE_NETINET_IN_H
#include <netinet/in.h>
#endif
#define MCA_PTL_HDR_TYPE_MATCH 1
#define MCA_PTL_HDR_TYPE_RNDV 2
#define MCA_PTL_HDR_TYPE_FRAG 3
#define MCA_PTL_HDR_TYPE_ACK 4
#define MCA_PTL_HDR_TYPE_NACK 5
#define MCA_PTL_HDR_TYPE_GET 6
#define MCA_PTL_HDR_TYPE_FIN 7
#define MCA_PTL_HDR_TYPE_FIN_ACK 8
#define MCA_PTL_HDR_TYPE_MAX 9
#define MCA_PTL_FLAGS_ACK 1 /* is an ack required */
#define MCA_PTL_FLAGS_NBO 2 /* is the header in network byte order */
/*
* Convert a 64 bit value to network byte order.
*/
static inline uint64_t hton64(uint64_t val)
{
union { uint64_t ll;
uint32_t l[2];
} w, r;
/* platform already in network byte order? */
if(htonl(1) == 1L)
return val;
w.ll = val;
r.l[0] = htonl(w.l[1]);
r.l[1] = htonl(w.l[0]);
return r.ll;
}
/*
* Convert a 64 bit value from network to host byte order.
*/
static inline uint64_t ntoh64(uint64_t val)
{
union { uint64_t ll;
uint32_t l[2];
} w, r;
/* platform already in network byte order? */
if(htonl(1) == 1L)
return val;
w.ll = val;
r.l[0] = ntohl(w.l[1]);
r.l[1] = ntohl(w.l[0]);
return r.ll;
}
/**
* Common header attributes - must be first element in each header type
*/
struct mca_ptl_base_common_header_t {
uint8_t hdr_type; /**< type of envelope */
uint8_t hdr_flags; /**< flags indicating how fragment should be processed */
};
typedef struct mca_ptl_base_common_header_t mca_ptl_base_common_header_t;
#define MCA_PTL_BASE_COMMON_HDR_NTOH(h)
#define MCA_PTL_BASE_COMMON_HDR_HTON(h)
/**
* Header definition for the first fragment, contains the
* attributes required to match the corresponding posted receive.
*/
struct mca_ptl_base_match_header_t {
mca_ptl_base_common_header_t hdr_common; /**< common attributes */
uint16_t hdr_contextid; /**< communicator index */
int32_t hdr_src; /**< source rank */
int32_t hdr_dst; /**< destination rank */
int32_t hdr_tag; /**< user tag */
uint64_t hdr_msg_length; /**< message length */
uint16_t hdr_msg_seq; /**< message sequence number */
};
typedef struct mca_ptl_base_match_header_t mca_ptl_base_match_header_t;
#define MCA_PTL_BASE_MATCH_HDR_NTOH(h) \
do { \
MCA_PTL_BASE_COMMON_HDR_NTOH((h).hdr_common); \
(h).hdr_contextid = ntohs((h).hdr_contextid); \
(h).hdr_src = ntohl((h).hdr_src); \
(h).hdr_dst = ntohl((h).hdr_dst); \
(h).hdr_tag = ntohl((h).hdr_tag); \
(h).hdr_msg_length = ntoh64((h).hdr_msg_length); \
(h).hdr_msg_seq = ntohs((h).hdr_msg_seq); \
} while (0)
#define MCA_PTL_BASE_MATCH_HDR_HTON(h) \
do { \
MCA_PTL_BASE_COMMON_HDR_HTON((h).hdr_common); \
(h).hdr_contextid = htons((h).hdr_contextid); \
(h).hdr_src = htonl((h).hdr_src); \
(h).hdr_dst = htonl((h).hdr_dst); \
(h).hdr_tag = htonl((h).hdr_tag); \
(h).hdr_msg_length = hton64((h).hdr_msg_length); \
(h).hdr_msg_seq = htons((h).hdr_msg_seq); \
} while (0)
/**
* Header definition for the first fragment when an acknowledgment
* is required. This could be the first fragment of a large message
* or a short message that requires an ack (synchronous).
*/
struct mca_ptl_base_rendezvous_header_t {
mca_ptl_base_match_header_t hdr_match;
uint64_t hdr_frag_length; /**< fragment length */
ompi_ptr_t hdr_src_ptr; /**< pointer to source fragment - returned in ack */
};
typedef struct mca_ptl_base_rendezvous_header_t mca_ptl_base_rendezvous_header_t;
#define MCA_PTL_BASE_RNDV_HDR_NTOH(h) \
do { \
MCA_PTL_BASE_MATCH_HDR_NTOH((h).hdr_match); \
(h).hdr_frag_length = ntoh64((h).hdr_frag_length); \
} while (0)
#define MCA_PTL_BASE_RNDV_HDR_HTON(h) \
do { \
MCA_PTL_BASE_MATCH_HDR_HTON((h).hdr_match); \
(h).hdr_frag_length = hton64((h).hdr_frag_length); \
} while (0)
/**
* Header for subsequent fragments.
*/
struct mca_ptl_base_frag_header_t {
mca_ptl_base_common_header_t hdr_common; /**< common attributes */
uint64_t hdr_frag_length; /**< fragment length */
uint64_t hdr_frag_offset; /**< offset into message */
ompi_ptr_t hdr_src_ptr; /**< pointer to source fragment */
ompi_ptr_t hdr_dst_ptr; /**< pointer to matched receive */
};
typedef struct mca_ptl_base_frag_header_t mca_ptl_base_frag_header_t;
#define MCA_PTL_BASE_FRAG_HDR_NTOH(h) \
do { \
MCA_PTL_BASE_COMMON_HDR_NTOH((h).hdr_common); \
(h).hdr_frag_length = ntoh64((h).hdr_frag_length); \
(h).hdr_frag_offset = ntoh64((h).hdr_frag_offset); \
} while (0)
#define MCA_PTL_BASE_FRAG_HDR_HTON(h) \
do { \
MCA_PTL_BASE_COMMON_HDR_HTON((h).hdr_common); \
(h).hdr_frag_length = hton64((h).hdr_frag_length); \
(h).hdr_frag_offset = hton64((h).hdr_frag_offset); \
} while (0)
/**
* Header used to acknowledgment outstanding fragment(s).
*/
struct mca_ptl_base_ack_header_t {
mca_ptl_base_common_header_t hdr_common; /**< common attributes */
ompi_ptr_t hdr_src_ptr; /**< source fragment */
ompi_ptr_t hdr_dst_match; /**< matched receive request */
ompi_ptr_t hdr_dst_addr; /**< posted receive buffer */
uint64_t hdr_dst_size; /**< size of posted buffer */
/* sequence range? */
};
typedef struct mca_ptl_base_ack_header_t mca_ptl_base_ack_header_t;
#define MCA_PTL_BASE_ACK_HDR_NTOH(h) \
do { \
MCA_PTL_BASE_COMMON_HDR_NTOH(h.hdr_common); \
(h).hdr_dst_size = ntoh64((h).hdr_dst_size); \
} while (0)
#define MCA_PTL_BASE_ACK_HDR_HTON(h) \
do { \
MCA_PTL_BASE_COMMON_HDR_HTON((h).hdr_common); \
(h).hdr_dst_size = hton64((h).hdr_dst_size); \
} while (0)
/**
* Union of defined header types.
*/
union mca_ptl_base_header_t {
mca_ptl_base_common_header_t hdr_common;
mca_ptl_base_match_header_t hdr_match;
mca_ptl_base_rendezvous_header_t hdr_rndv;
mca_ptl_base_frag_header_t hdr_frag;
mca_ptl_base_ack_header_t hdr_ack;
};
typedef union mca_ptl_base_header_t mca_ptl_base_header_t;
#endif

674
ompi/mca/ptl/base/ptl_base_match.c
Просмотреть файл

@ -1,674 +0,0 @@
/** @file */
/*
* Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
* University Research and Technology
* Corporation. All rights reserved.
* Copyright (c) 2004-2005 The University of Tennessee and The University
* of Tennessee Research Foundation. All rights
* reserved.
* Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
* University of Stuttgart. All rights reserved.
* Copyright (c) 2004-2005 The Regents of the University of California.
* All rights reserved.
* $COPYRIGHT$
*
* Additional copyrights may follow
*
* $HEADER$
*/
#include "ompi_config.h"
#include <stdio.h>
#include "opal/class/opal_list.h"
#include "opal/threads/mutex.h"
#include "ompi/constants.h"
#include "ompi/communicator/communicator.h"
#include "ompi/mca/pml/pml.h"
#include "ompi/mca/ptl/ptl.h"
#include "ompi/mca/ptl/base/ptl_base_comm.h"
#include "ompi/mca/ptl/base/ptl_base_recvfrag.h"
#include "ompi/mca/ptl/base/ptl_base_header.h"
#include "ompi/mca/ptl/base/ptl_base_match.h"
/**
* Try and match the incoming message fragment to the list of
* "wild" receives
*
* @param frag_header Matching data from recived fragment (IN)
*
* @param pml_comm Pointer to the communicator structure used for
* matching purposes. (IN)
*
* @return Matched receive
*
* This routine assumes that the appropriate matching locks are
* set by the upper level routine.
*/
#define MCA_PTL_BASE_CHECK_WILD_RECEIVES_FOR_MATCH(frag_header,pml_comm,return_match) \
do { \
/* local parameters */ \
opal_list_t* wild_receives = &pml_comm->c_wild_receives; \
mca_ptl_base_recv_request_t *wild_recv; \
int frag_tag,recv_tag; \
\
/* initialization */ \
frag_tag=frag_header->hdr_tag; \
\
/* \
* Loop over the wild irecvs - no need to lock, the upper level \
* locking is protecting from having other threads trying to \
* change this list. \
*/ \
for(wild_recv = (mca_ptl_base_recv_request_t *) \
opal_list_get_first(wild_receives); \
wild_recv != (mca_ptl_base_recv_request_t *) \
opal_list_get_end(wild_receives); \
wild_recv = (mca_ptl_base_recv_request_t *) \
((opal_list_item_t *)wild_recv)->opal_list_next) { \
\
recv_tag = wild_recv->req_recv.req_base.req_tag; \
if ( \
/* exact tag match */ \
(frag_tag == recv_tag) || \
/* wild tag match - negative tags (except for \
* OMPI_ANY_TAG) are reserved for internal use, and will \
* not be matched with OMPI_ANY_TAG */ \
( (recv_tag == OMPI_ANY_TAG) && (0 <= frag_tag) ) ) \
\
{ \
/* \
* Mark that this is the matching irecv, and go to process it. \
*/ \
return_match = wild_recv; \
\
/* remove this irecv from the postd wild ireceive list */ \
opal_list_remove_item(wild_receives, \
(opal_list_item_t *)wild_recv); \
\
/* found match - no need to continue */ \
break; \
} \
} \
} while(0)
/**
* Try and match the incoming message fragment to the list of
* "specific" receives
*
* @param frag_header Matching data from recived fragment (IN)
*
* @param pml_comm Pointer to the communicator structure used for
* matching purposes. (IN)
*
* @return Matched receive
*
* This routine assumes that the appropriate matching locks are
* set by the upper level routine.
*/
#define MCA_PTL_BASE_CHECK_SPECIFIC_RECEIVES_FOR_MATCH(frag_header, pml_comm, return_match) \
do { \
/* local variables */ \
opal_list_t* specific_receives = (pml_comm->c_specific_receives)+frag_src; \
mca_ptl_base_recv_request_t *specific_recv; \
int frag_src,recv_tag,frag_tag; \
\
/* initialization */ \
frag_src = frag_header->hdr_src; \
frag_tag=frag_header->hdr_tag; \
\
/* \
* Loop over the specific irecvs. \
*/ \
for(specific_recv = (mca_ptl_base_recv_request_t *) \
opal_list_get_first(specific_receives); \
specific_recv != (mca_ptl_base_recv_request_t *) \
opal_list_get_end(specific_receives); \
specific_recv = (mca_ptl_base_recv_request_t *) \
((opal_list_item_t *)specific_recv)->opal_list_next) { \
/* \
* Check for a match \
*/ \
recv_tag = specific_recv->req_recv.req_base.req_tag; \
if ( (frag_tag == recv_tag) || \
( (recv_tag == OMPI_ANY_TAG) && (0 <= frag_tag) ) ) { \
\
/* \
* Match made \
*/ \
return_match = specific_recv; \
\
/* remove descriptor from posted specific ireceive list */ \
opal_list_remove_item(specific_receives, \
(opal_list_item_t *)specific_recv); \
\
break; \
} \
} \
} while(0)
/**
* Try and match the incoming message fragment to the list of
* "wild" receives and "specific" receives. Used when both types
* of receives have been posted, i.e. when we need to coordinate
* between multiple lists to make sure ordered delivery occurs.
*
* @param frag_header Matching data from recived fragment (IN)
*
* @param pml_comm Pointer to the communicator structure used for
* matching purposes. (IN)
*
* @return Matched receive
*
* This routine assumes that the appropriate matching locks are
* set by the upper level routine.
*/
#define MCA_PTL_BASE_CHECK_SPECIFIC_AND_WILD_RECEIVES_FOR_MATCH( \
frag_header, pml_comm, return_match) \
do { \
/* local variables */ \
mca_ptl_base_recv_request_t *specific_recv, *wild_recv; \
mca_ptl_sequence_t wild_recv_seq, specific_recv_seq; \
int frag_src,frag_tag, wild_recv_tag, specific_recv_tag; \
\
/* initialization */ \
frag_src = frag_header->hdr_src; \
frag_tag=frag_header->hdr_tag; \
\
/* \
* We know that when this is called, both specific and wild irecvs \
* have been posted. \
*/ \
specific_recv = (mca_ptl_base_recv_request_t *) \
opal_list_get_first((pml_comm->c_specific_receives)+frag_src); \
wild_recv = (mca_ptl_base_recv_request_t *) \
opal_list_get_first(&(pml_comm->c_wild_receives)); \
\
specific_recv_seq = specific_recv->req_recv.req_base.req_sequence; \
wild_recv_seq = wild_recv->req_recv.req_base.req_sequence; \
\
while (true) { \
if (wild_recv_seq < specific_recv_seq) { \
/* \
* wild recv is earlier than the specific one. \
*/ \
/* \
* try and match \
*/ \
wild_recv_tag = wild_recv->req_recv.req_base.req_tag; \
if ( (frag_tag == wild_recv_tag) || \
( (wild_recv_tag == OMPI_ANY_TAG) && (0 <= frag_tag) ) ) { \
/* \
* Match made \
*/ \
return_match=wild_recv; \
\
/* remove this recv from the wild receive queue */ \
opal_list_remove_item(&(pml_comm->c_wild_receives), \
(opal_list_item_t *)wild_recv); \
break; \
} \
\
/* \
* No match, go to the next. \
*/ \
wild_recv=(mca_ptl_base_recv_request_t *) \
((opal_list_item_t *)wild_recv)->opal_list_next; \
\
/* \
* If that was the last wild one, just look at the \
* rest of the specific ones. \
*/ \
if (wild_recv == (mca_ptl_base_recv_request_t *) \
opal_list_get_end(&(pml_comm->c_wild_receives)) ) \
{ \
MCA_PTL_BASE_CHECK_SPECIFIC_RECEIVES_FOR_MATCH(frag_header, pml_comm, return_match); \
break; \
} \
\
/* \
* Get the sequence number for this recv, and go \
* back to the top of the loop. \
*/ \
wild_recv_seq = wild_recv->req_recv.req_base.req_sequence; \
\
} else { \
/* \
* specific recv is earlier than the wild one. \
*/ \
specific_recv_tag=specific_recv->req_recv.req_base.req_tag; \
if ( (frag_tag == specific_recv_tag) || \
( (specific_recv_tag == OMPI_ANY_TAG) && (0<=frag_tag)) ) \
{ \
/* \
* Match made \
*/ \
return_match = specific_recv; \
/* remove descriptor from specific receive list */ \
opal_list_remove_item((pml_comm->c_specific_receives)+frag_src, \
(opal_list_item_t *)specific_recv); \
break; \
} \
\
/* \
* No match, go on to the next specific irecv. \
*/ \
specific_recv = (mca_ptl_base_recv_request_t *) \
((opal_list_item_t *)specific_recv)->opal_list_next; \
\
/* \
* If that was the last specific irecv, process the \
* rest of the wild ones. \
*/ \
if (specific_recv == (mca_ptl_base_recv_request_t *) \
opal_list_get_end((pml_comm->c_specific_receives)+frag_src) ) \
{ \
MCA_PTL_BASE_CHECK_WILD_RECEIVES_FOR_MATCH(frag_header, pml_comm, return_match); \
break; \
} \
/* \
* Get the sequence number for this recv, and go \
* back to the top of the loop. \
*/ \
specific_recv_seq = specific_recv->req_recv.req_base.req_sequence; \
} \
} \
} while(0)
/*
* Specialized matching routines for internal use only.
*/
static bool mca_ptl_base_check_cantmatch_for_match(
opal_list_t *additional_matches,
mca_pml_ptl_comm_t *pml_comm, int frag_src);
/**
* RCS/CTS receive side matching
*
* @param frag_header list of parameters needed for matching
* This list is also embeded in frag_desc,
* but this allows to save a memory copy when
* a match is made in this routine. (IN)
* @param frag_desc pointer to receive fragment which we want
* to match (IN/OUT). If a match is not made,
* frag_header is copied to frag_desc.
* @param match_made parameter indicating if we matched frag_desc/
* frag_header (OUT)
* @param additional_matches if a match is made with frag_desc, we
* may be able to match fragments that previously
* have arrived out-of-order. If this is the
* case, the associated fragment descriptors are
* put on this list for further processing. (OUT)
*
* @return OMPI error code
*
* This routine is used to try and match a newly arrived message fragment
* to pre-posted receives. The following assumptions are made
* - fragments are received out of order
* - for long messages, e.g. more than one fragment, a RTS/CTS algorithm
* is used.
* - 2nd and greater fragments include a receive descriptor pointer
* - fragments may be dropped
* - fragments may be corrupt
* - this routine may be called simultaneously by more than one thread
*/
bool mca_ptl_base_match(
mca_ptl_base_match_header_t *frag_header,
mca_ptl_base_recv_frag_t *frag_desc,
opal_list_t *additional_matches,
bool* additional_match)
{
/* local variables */
uint16_t next_msg_seq_expected, frag_msg_seq;
ompi_communicator_t *comm_ptr;
mca_ptl_base_recv_request_t *matched_receive = NULL;
mca_pml_ptl_comm_t *pml_comm;
int frag_src;
bool match_made=false;
/* communicator pointer */
comm_ptr=ompi_comm_lookup(frag_header->hdr_contextid);
pml_comm=(mca_pml_ptl_comm_t *)comm_ptr->c_pml_comm;
/* source sequence number */
frag_msg_seq = frag_header->hdr_msg_seq;
/* get fragment communicator source rank */
frag_src = frag_header->hdr_src;
/* get next expected message sequence number - if threaded
* run, lock to make sure that if another thread is processing
* a frag from the same message a match is made only once.
* Also, this prevents other posted receives (for a pair of
* end points) from being processed, and potentially "loosing"
* the fragment.
*/
OPAL_THREAD_LOCK(&pml_comm->c_matching_lock);
/* get sequence number of next message that can be processed */
next_msg_seq_expected = (uint16_t)*((pml_comm->c_next_msg_seq)+frag_src);
if (frag_msg_seq == next_msg_seq_expected) {
/*
* This is the sequence number we were expecting,
* so we can try matching it to already posted
* receives.
*/
/* We're now expecting the next sequence number. */
(pml_comm->c_next_msg_seq[frag_src])++;
/*
* figure out what sort of matching logic to use, if need to
* look only at "specific" receives, or "wild" receives,
* or if we need to traverse both sets at the same time.
*/
if (opal_list_get_size((pml_comm->c_specific_receives)+frag_src) == 0 ){
/*
* There are only wild irecvs, so specialize the algorithm.
*/
MCA_PTL_BASE_CHECK_WILD_RECEIVES_FOR_MATCH(frag_header, pml_comm, matched_receive);
} else if (opal_list_get_size(&(pml_comm->c_wild_receives)) == 0 ) {
/*
* There are only specific irecvs, so specialize the algorithm.
*/
MCA_PTL_BASE_CHECK_SPECIFIC_RECEIVES_FOR_MATCH(frag_header, pml_comm, matched_receive);
} else {
/*
* There are some of each.
*/
MCA_PTL_BASE_CHECK_SPECIFIC_AND_WILD_RECEIVES_FOR_MATCH(frag_header, pml_comm, matched_receive);
}
/* if match found, process data */
if (matched_receive) {
/* set flag indicating the input fragment was matched */
match_made = true;
/* associate the receive descriptor with the fragment descriptor */
frag_desc->frag_request=matched_receive;
/* set lenght of incoming message */
matched_receive->req_recv.req_bytes_packed = frag_header->hdr_msg_length;
/*
* update delivered sequence number information, if needed.
*/
if( (matched_receive->req_recv.req_base.req_type == MCA_PML_REQUEST_PROBE) ) {
/* Match a probe, rollback the next expected sequence number */
(pml_comm->c_next_msg_seq[frag_src])--;
}
} else {
/* if no match found, place on unexpected queue */
opal_list_append( ((pml_comm->c_unexpected_frags)+frag_src),
(opal_list_item_t *)frag_desc );
}
/*
* Now that new message has arrived, check to see if
* any fragments on the c_c_frags_cant_match list
* may now be used to form new matchs
*/
if (0 < opal_list_get_size((pml_comm->c_frags_cant_match)+frag_src)) {
*additional_match = mca_ptl_base_check_cantmatch_for_match(additional_matches,pml_comm,frag_src);
}
} else {
/*
* This message comes after the next expected, so it
* is ahead of sequence. Save it for later.
*/
opal_list_append( ((pml_comm->c_frags_cant_match)+frag_src),
(opal_list_item_t *)frag_desc);
}
OPAL_THREAD_UNLOCK(&pml_comm->c_matching_lock);
return match_made;
}
/**
* Scan the list of frags that came in ahead of time to see if any
* can be processed at this time. If they can, try and match the
* frags.
*
* @param additional_matches List to hold new matches with fragments
* from the c_frags_cant_match list. (IN/OUT)
*
* @param pml_comm Pointer to the communicator structure used for
* matching purposes. (IN)
*
* This routine assumes that the appropriate matching locks are
* set by the upper level routine.
*/
static bool mca_ptl_base_check_cantmatch_for_match(opal_list_t *additional_matches,
mca_pml_ptl_comm_t *pml_comm, int frag_src)
{
/* local parameters */
int match_found;
uint16_t next_msg_seq_expected, frag_seq;
mca_ptl_base_recv_frag_t *frag_desc;
mca_ptl_base_recv_request_t *matched_receive = NULL;
bool match_made = false;
/*
* Loop over all the out of sequence messages. No ordering is assumed
* in the c_frags_cant_match list.
*/
match_found = 1;
while ((0 < opal_list_get_size((pml_comm->c_frags_cant_match)+frag_src)) &&
match_found) {
/* initialize match flag for this search */
match_found = 0;
/* get sequence number of next message that can be processed */
next_msg_seq_expected = *((pml_comm->c_next_msg_seq)+frag_src);
/* search the list for a fragment from the send with sequence
* number next_msg_seq_expected
*/
for(frag_desc = (mca_ptl_base_recv_frag_t *)
opal_list_get_first((pml_comm->c_frags_cant_match)+frag_src);
frag_desc != (mca_ptl_base_recv_frag_t *)
opal_list_get_end((pml_comm->c_frags_cant_match)+frag_src);
frag_desc = (mca_ptl_base_recv_frag_t *)
opal_list_get_next(frag_desc))
{
/*
* If the message has the next expected seq from that proc...
*/
frag_seq=frag_desc->frag_base.frag_header.hdr_match.hdr_msg_seq;
if (frag_seq == next_msg_seq_expected) {
mca_ptl_base_match_header_t* frag_header =
&frag_desc->frag_base.frag_header.hdr_match;
/* We're now expecting the next sequence number. */
(pml_comm->c_next_msg_seq[frag_src])++;
/* signal that match was made */
match_found = 1;
/*
* remove frag_desc from list
*/
opal_list_remove_item((pml_comm->c_frags_cant_match)+frag_src,
(opal_list_item_t *)frag_desc);
/*
* figure out what sort of matching logic to use, if need to
* look only at "specific" receives, or "wild" receives,
* or if we need to traverse both sets at the same time.
*/
frag_src = frag_header->hdr_src;
if (opal_list_get_size((pml_comm->c_specific_receives)+frag_src) == 0 ) {
/*
* There are only wild irecvs, so specialize the algorithm.
*/
MCA_PTL_BASE_CHECK_WILD_RECEIVES_FOR_MATCH(frag_header, pml_comm, matched_receive);
} else if (opal_list_get_size(&(pml_comm->c_wild_receives)) == 0 ) {
/*
* There are only specific irecvs, so specialize the algorithm.
*/
MCA_PTL_BASE_CHECK_SPECIFIC_RECEIVES_FOR_MATCH(frag_header, pml_comm, matched_receive);
} else {
/*
* There are some of each.
*/
MCA_PTL_BASE_CHECK_SPECIFIC_AND_WILD_RECEIVES_FOR_MATCH(frag_header, pml_comm, matched_receive);
}
/* if match found, process data */
if (matched_receive) {
/* associate the receive descriptor with the fragment
* descriptor */
frag_desc->frag_request=matched_receive;
/* add this fragment descriptor to the list of
* descriptors to be processed later
*/
if(match_made == false) {
match_made = true;
OBJ_CONSTRUCT(additional_matches, opal_list_t);
}
opal_list_append(additional_matches, (opal_list_item_t *)frag_desc);
} else {
/* if no match found, place on unexpected queue */
opal_list_append( ((pml_comm->c_unexpected_frags)+frag_src),
(opal_list_item_t *)frag_desc);
}
/* c_frags_cant_match is not an ordered list, so exit loop
* and re-start search for next sequence number */
break;
} /* end if (frag_seq == next_msg_seq_expected) */
} /* end for (frag_desc) loop */
} /* end while loop */
return match_made;
}
/**
* RCS/CTS receive side matching
*
* @param frag_header list of parameters needed for matching
* This list is also embeded in frag_desc,
* but this allows to save a memory copy when
* a match is made in this routine. (IN)
* @param frag_desc pointer to receive fragment which we want
* to match (IN/OUT). If a match is not made,
* frag_header is copied to frag_desc.
* @param match_made parameter indicating if we matched frag_desc/
* frag_header (OUT)
* @return indication if match was made or not.
*
* This routine is used to try and match a newly arrived message fragment
* to pre-posted receives. The following assumptions are made
* - fragments are received in order, so no explicit sequence
* tracking is needed.
* - for long messages, e.g. more than one fragment, a RTS/CTS algorithm
* is used.
* - 2nd and greater fragments include a receive descriptor pointer
* - this routine may be called simoultaneously by more than one thread
*
* On return, if match is made:
* neither the fragment, nor the matched receive descriptor
* are on any list
* if match is not made:
* The fragment is placed on the unexpected fragment list
*/
bool mca_ptl_base_match_in_order_network_delivery(
mca_ptl_base_match_header_t *frag_header,
struct mca_ptl_base_recv_frag_t *frag_desc)
{
/* local variables */
ompi_communicator_t *comm_ptr;
mca_ptl_base_recv_request_t *matched_receive = NULL;
mca_pml_ptl_comm_t *pml_comm;
int frag_src;
bool match_made=false;
/* communicator pointer */
comm_ptr=ompi_comm_lookup(frag_header->hdr_contextid);
pml_comm=(mca_pml_ptl_comm_t *)comm_ptr->c_pml_comm;
/* get fragment communicator source rank */
frag_src = frag_header->hdr_src;
/* get next expected message sequence number - if threaded
* run, lock to make sure that if another thread is processing
* a frag from the same message a match is made only once.
* Also, this prevents other posted receives (for a pair of
* end points) from being processed, and potentially "loosing"
* the fragment.
*/
OPAL_THREAD_LOCK(&pml_comm->c_matching_lock);
/*
* figure out what sort of matching logic to use, if need to
* look only at "specific" receives, or "wild" receives,
* or if we need to traverse both sets at the same time.
*/
if (opal_list_get_size((pml_comm->c_specific_receives)+frag_src) == 0 ){
/*
* There are only wild irecvs, so specialize the algorithm.
*/
MCA_PTL_BASE_CHECK_WILD_RECEIVES_FOR_MATCH(frag_header, pml_comm, matched_receive);
} else if (opal_list_get_size(&(pml_comm->c_wild_receives)) == 0 ) {
/*
* There are only specific irecvs, so specialize the algorithm.
*/
MCA_PTL_BASE_CHECK_SPECIFIC_RECEIVES_FOR_MATCH(frag_header, pml_comm, matched_receive);
} else {
/*
* There are some of each.
*/
MCA_PTL_BASE_CHECK_SPECIFIC_AND_WILD_RECEIVES_FOR_MATCH(frag_header, pml_comm, matched_receive);
}
/* if match found, process data */
if (matched_receive) {
/* set flag indicating the input fragment was matched */
match_made=true;
/* associate the receive descriptor with the fragment descriptor */
frag_desc->frag_request=matched_receive;
/* set lenght of incoming message */
matched_receive->req_recv.req_bytes_packed=frag_header->hdr_msg_length;
} else {
/* if no match found, place on unexpected queue */
opal_list_append( ((pml_comm->c_unexpected_frags)+frag_src),
(opal_list_item_t *)frag_desc);
}
OPAL_THREAD_UNLOCK(&pml_comm->c_matching_lock);
return match_made;
}

77
ompi/mca/ptl/base/ptl_base_match.h
Просмотреть файл

@ -1,77 +0,0 @@
/*
* Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
* University Research and Technology
* Corporation. All rights reserved.
* Copyright (c) 2004-2005 The University of Tennessee and The University
* of Tennessee Research Foundation. All rights
* reserved.
* Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
* University of Stuttgart. All rights reserved.
* Copyright (c) 2004-2005 The Regents of the University of California.
* All rights reserved.
* $COPYRIGHT$
*
* Additional copyrights may follow
*
* $HEADER$
*/
/**
* @file
*/
#ifndef MCA_PTL_BASE_MATCH_H
#define MCA_PTL_BASE_MATCH_H
#if defined(c_plusplus) || defined(__cplusplus)
extern "C" {
#endif
struct mca_ptl_base_recv_frag_t;
/**
* RCS/CTS receive side matching
* Match incoming fragments against posted receives. Out of order
* delivery.
*
* @param frag_header (IN) Header of received fragment.
* @param frag_desc (IN) Received fragment descriptor.
* @param match_made (OUT) Flag indicating wether a match was made.
* @param additional_matches (OUT) List of additional matches
* @return OMPI_SUCCESS or error status on failure.
*/
OMPI_DECLSPEC bool mca_ptl_base_match(
mca_ptl_base_match_header_t *frag_header,
struct mca_ptl_base_recv_frag_t *frag_desc,
opal_list_t *additional_matches,
bool* additional_matched);
/**
* RCS/CTS receive side matching
*
* @param frag_header list of parameters needed for matching
* This list is also embeded in frag_desc,
* but this allows to save a memory copy when
* a match is made in this routine. (IN)
* @param frag_desc pointer to receive fragment which we want
* to match (IN/OUT). If a match is not made,
* frag_header is copied to frag_desc.
* @param match_made parameter indicating if we matched frag_desc/
* frag_header (OUT)
* @return indication if match was made or not.
*
* This routine is used to try and match a newly arrived message fragment
* to pre-posted receives. The following assumptions are made
* - fragments are received in order, so no explicit sequence
* tracking is needed.
* - for long messages, e.g. more than one fragment, a RTS/CTS algorithm
* is used.
* - 2nd and greater fragments include a receive descriptor pointer
* - this routine may be called simoultaneously by more than one thread
*/
OMPI_DECLSPEC bool mca_ptl_base_match_in_order_network_delivery(
mca_ptl_base_match_header_t *frag_header,
struct mca_ptl_base_recv_frag_t *frag_desc);
#if defined(c_plusplus) || defined(__cplusplus)
}
#endif
#endif /* MCA_PTL_BASE_MATCH_H */

86
ompi/mca/ptl/base/ptl_base_open.c
Просмотреть файл

@ -1,86 +0,0 @@
/*
* Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
* University Research and Technology
* Corporation. All rights reserved.
* Copyright (c) 2004-2005 The University of Tennessee and The University
* of Tennessee Research Foundation. All rights
* reserved.
* Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
* University of Stuttgart. All rights reserved.
* Copyright (c) 2004-2005 The Regents of the University of California.
* All rights reserved.
* $COPYRIGHT$
*
* Additional copyrights may follow
*
* $HEADER$
*/
#include "ompi_config.h"
#include <stdio.h>
#include "opal/mca/mca.h"
#include "opal/mca/base/base.h"
#include "opal/mca/base/mca_base_param.h"
#include "ompi/constants.h"
#include "ompi/mca/pml/pml.h"
#include "ompi/mca/ptl/ptl.h"
#include "ompi/mca/ptl/base/base.h"
/*
* The following file was created by configure. It contains extern
* statements and the definition of an array of pointers to each
* component's public mca_base_component_t struct.
*/
#include "ompi/mca/ptl/base/static-components.h"
/*
* Global variables
*/
int mca_ptl_base_output = -1;
char* mca_ptl_base_include = NULL;
char* mca_ptl_base_exclude = NULL;
opal_list_t mca_ptl_base_components_opened;
opal_list_t mca_ptl_base_components_initialized;
opal_list_t mca_ptl_base_modules_initialized;
int mca_ptl_base_open_called = 0;
/**
* Function for finding and opening either all MCA components, or the one
* that was specifically requested via a MCA parameter.
*/
int mca_ptl_base_open(void)
{
if( 0 != mca_ptl_base_open_called ) return OMPI_SUCCESS;
mca_ptl_base_open_called = 1;
/* Open up all available components */
if (OMPI_SUCCESS !=
mca_base_components_open("ptl", 0, mca_ptl_base_static_components,
&mca_ptl_base_components_opened, true)) {
return OMPI_ERROR;
}
/* Initialize the list containing all the PTL's where the init function has been called */
OBJ_CONSTRUCT( &mca_ptl_base_components_initialized, opal_list_t );
/* Initialize the list so that in mca_ptl_base_close(), we can
iterate over it (even if it's empty, as in the case of
ompi_info) */
OBJ_CONSTRUCT(&mca_ptl_base_modules_initialized, opal_list_t);
/* register parameters */
mca_base_param_lookup_string(
mca_base_param_register_string("ptl","base","include",NULL,NULL), &mca_ptl_base_include);
mca_base_param_lookup_string(
mca_base_param_register_string("ptl","base","exclude",NULL,NULL), &mca_ptl_base_exclude);
/* All done */
return OMPI_SUCCESS;
}

47
ompi/mca/ptl/base/ptl_base_recvfrag.c
Просмотреть файл

@ -1,47 +0,0 @@
/*
* Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
* University Research and Technology
* Corporation. All rights reserved.
* Copyright (c) 2004-2005 The University of Tennessee and The University
* of Tennessee Research Foundation. All rights
* reserved.
* Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
* University of Stuttgart. All rights reserved.
* Copyright (c) 2004-2005 The Regents of the University of California.
* All rights reserved.
* $COPYRIGHT$
*
* Additional copyrights may follow
*
* $HEADER$
*/
/*%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%*/
#include "ompi_config.h"
#include "ompi/mca/pml/pml.h"
#include "ompi/mca/ptl/ptl.h"
#include "ompi/mca/ptl/base/ptl_base_recvfrag.h"
#include "ompi/mca/ptl/base/ptl_base_match.h"
static void mca_ptl_base_recv_frag_construct(mca_ptl_base_recv_frag_t* frag);
static void mca_ptl_base_recv_frag_destruct(mca_ptl_base_recv_frag_t* frag);
OBJ_CLASS_INSTANCE(
mca_ptl_base_recv_frag_t,
mca_ptl_base_frag_t,
mca_ptl_base_recv_frag_construct,
mca_ptl_base_recv_frag_destruct
);
void mca_ptl_base_recv_frag_construct(mca_ptl_base_recv_frag_t* frag)
{
frag->frag_base.frag_type = MCA_PTL_FRAGMENT_RECV;
}
void mca_ptl_base_recv_frag_destruct(mca_ptl_base_recv_frag_t* frag)
{
}

49
ompi/mca/ptl/base/ptl_base_recvfrag.h
Просмотреть файл

@ -1,49 +0,0 @@
/*
* Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
* University Research and Technology
* Corporation. All rights reserved.
* Copyright (c) 2004-2005 The University of Tennessee and The University
* of Tennessee Research Foundation. All rights
* reserved.
* Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
* University of Stuttgart. All rights reserved.
* Copyright (c) 2004-2005 The Regents of the University of California.
* All rights reserved.
* $COPYRIGHT$
*
* Additional copyrights may follow
*
* $HEADER$
*/
/**
* @file
*/
#ifndef MCA_PTL_BASE_RECVFRAG_H
#define MCA_PTL_BASE_RECVFRAG_H
#include "ompi/mca/ptl/ptl.h"
#include "ompi/mca/ptl/base/ptl_base_recvreq.h"
#include "ompi/mca/ptl/base/ptl_base_fragment.h"
#include "ompi/mca/ptl/base/ptl_base_match.h"
#if defined(c_plusplus) || defined(__cplusplus)
extern "C" {
#endif
OMPI_DECLSPEC extern opal_class_t mca_ptl_base_recv_frag_t_class;
/**
* Base type for receive fragment descriptors.
*/
struct mca_ptl_base_recv_frag_t {
mca_ptl_base_frag_t frag_base; /**< base fragment descriptor */
mca_ptl_base_recv_request_t *frag_request; /**< matched posted receive */
bool frag_is_buffered; /**< does fragment need to be unpacked into users buffer */
};
typedef struct mca_ptl_base_recv_frag_t mca_ptl_base_recv_frag_t;
#if defined(c_plusplus) || defined(__cplusplus)
}
#endif
#endif

10
ompi/mca/ptl/base/ptl_base_recvreq.c
Просмотреть файл

@ -1,10 +0,0 @@
#include "ompi/mca/ptl/base/ptl_base_recvreq.h"
OBJ_CLASS_INSTANCE(
mca_ptl_base_recv_request_t,
mca_pml_base_recv_request_t,
NULL,
NULL
);

49
ompi/mca/ptl/base/ptl_base_recvreq.h
Просмотреть файл

@ -1,49 +0,0 @@
/*
* Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
* University Research and Technology
* Corporation. All rights reserved.
* Copyright (c) 2004-2005 The University of Tennessee and The University
* of Tennessee Research Foundation. All rights
* reserved.
* Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
* University of Stuttgart. All rights reserved.
* Copyright (c) 2004-2005 The Regents of the University of California.
* All rights reserved.
* $COPYRIGHT$
*
* Additional copyrights may follow
*
* $HEADER$
*/
/**
* @file
*/
#ifndef MCA_PTL_BASE_RECV_REQUEST_H
#define MCA_PTL_BASE_RECV_REQUEST_H
#include "ompi/mca/ptl/ptl.h"
#include "ompi/mca/pml/base/pml_base_recvreq.h"
#if defined(c_plusplus) || defined(__cplusplus)
extern "C" {
#endif
/**
* Base type for recv requests
*/
struct mca_ptl_base_recv_request_t {
mca_pml_base_recv_request_t req_recv;
size_t req_bytes_received; /**< number of bytes received from network */
size_t req_bytes_delivered; /**< number of bytes delivered to user */
};
typedef struct mca_ptl_base_recv_request_t mca_ptl_base_recv_request_t;
OMPI_DECLSPEC OBJ_CLASS_DECLARATION(mca_ptl_base_recv_request_t);
#if defined(c_plusplus) || defined(__cplusplus)
}
#endif
#endif

152
ompi/mca/ptl/base/ptl_base_select.c
Просмотреть файл

@ -1,152 +0,0 @@
/*
* Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
* University Research and Technology
* Corporation. All rights reserved.
* Copyright (c) 2004-2005 The University of Tennessee and The University
* of Tennessee Research Foundation. All rights
* reserved.
* Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
* University of Stuttgart. All rights reserved.
* Copyright (c) 2004-2005 The Regents of the University of California.
* All rights reserved.
* $COPYRIGHT$
*
* Additional copyrights may follow
*
* $HEADER$
*/
#include "ompi_config.h"
#include "opal/util/argv.h"
#include "opal/util/show_help.h"
#include "opal/util/output.h"
#include "opal/mca/mca.h"
#include "opal/mca/base/base.h"
#include "orte/runtime/runtime.h"
#include "ompi/constants.h"
#include "ompi/mca/pml/pml.h"
#include "ompi/mca/ptl/ptl.h"
#include "ompi/mca/ptl/base/base.h"
/**
* Function for weeding out ptl components that don't want to run.
*
* Call the init function on all available components to find out if
* they want to run. Select all components that don't fail. Failing
* components will be closed and unloaded. The selected modules will
* be returned to the caller in a opal_list_t.
*/
int mca_ptl_base_select(bool enable_progress_threads,
bool enable_mpi_threads)
{
int i, num_ptls;
opal_list_item_t *item;
opal_list_t* useless = OBJ_NEW(opal_list_t);
mca_base_component_list_item_t *cli;
mca_ptl_base_component_t *component;
mca_ptl_base_module_t **modules;
mca_ptl_base_selected_module_t *sm;
char** include = opal_argv_split(mca_ptl_base_include, ',');
char** exclude = opal_argv_split(mca_ptl_base_exclude, ',');
/* Traverse the list of opened modules; call their init
functions. */
while( NULL != (item = opal_list_remove_first(&mca_ptl_base_components_opened)) ) {
bool keep_me = true;
cli = (mca_base_component_list_item_t *) item;
component = (mca_ptl_base_component_t *) cli->cli_component;
/* if there is an include list - item must be in the list to be included */
if ( NULL != include ) {
char** argv = include;
keep_me = false; /* all PTLs not in the include list cannot be selected */
while(argv && *argv) {
if(strcmp(component->ptlm_version.mca_component_name,*argv) == 0) {
keep_me = true;
break;
}
argv++;
}
/* otherwise - check the exclude list to see if this item has been specifically excluded */
} else if ( NULL != exclude ) {
char** argv = exclude;
keep_me = true; /* all PTL's not in except list should be keeped */
while(argv && *argv) {
if(strcmp(component->ptlm_version.mca_component_name,*argv) == 0) {
keep_me = false;
break;
}
argv++;
}
}
if( keep_me == false) {
opal_list_append( useless, item );
continue;
}
opal_output_verbose(10, mca_ptl_base_output,
"select: initializing %s component %s",
component->ptlm_version.mca_type_name,
component->ptlm_version.mca_component_name);
if (NULL == component->ptlm_init) {
opal_output_verbose(10, mca_ptl_base_output,
"select: no init function; ignoring component");
opal_list_append( useless, item );
continue;
} else {
modules = component->ptlm_init(&num_ptls, enable_progress_threads,
enable_mpi_threads);
/* If the component didn't initialize, remove it from the opened
list and remove it from the component repository */
if (NULL == modules) {
opal_output_verbose( 10, mca_ptl_base_output,
"select: %s PTL init returned failure",
component->ptlm_version.mca_component_name);
opal_list_append( useless, item );
continue;
}
/* Otherwise, it initialized properly. Save it. */
else {
opal_output_verbose(10, mca_ptl_base_output,
"select: init returned success");
opal_list_append( &mca_ptl_base_components_initialized, item );
for (i = 0; i < num_ptls; ++i) {
sm = malloc(sizeof(mca_ptl_base_selected_module_t));
if (NULL == sm) {
return OMPI_ERR_OUT_OF_RESOURCE;
}
OBJ_CONSTRUCT(sm, opal_list_item_t);
sm->pbsm_component = component;
sm->pbsm_module = modules[i];
opal_list_append(&mca_ptl_base_modules_initialized,
(opal_list_item_t*) sm);
}
free(modules);
}
}
}
/* All useless components have to be cleanly removed */
mca_base_components_close( mca_ptl_base_output, useless, NULL );
OBJ_RELEASE( useless );
opal_argv_free( include );
opal_argv_free( exclude );
/* Finished querying all components. Check for the bozo case. */
if (0 == opal_list_get_size(&mca_ptl_base_modules_initialized)) {
opal_show_help("help-mca-base.txt", "find-available:none-found", true,
"ptl");
orte_abort(1, "");
}
/* All done */
return OMPI_SUCCESS;
}

44
ompi/mca/ptl/base/ptl_base_sendfrag.c
Просмотреть файл

@ -1,44 +0,0 @@
/*
* Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
* University Research and Technology
* Corporation. All rights reserved.
* Copyright (c) 2004-2005 The University of Tennessee and The University
* of Tennessee Research Foundation. All rights
* reserved.
* Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
* University of Stuttgart. All rights reserved.
* Copyright (c) 2004-2005 The Regents of the University of California.
* All rights reserved.
* $COPYRIGHT$
*
* Additional copyrights may follow
*
* $HEADER$
*/
#include "ompi_config.h"
#include "ompi/mca/pml/pml.h"
#include "ompi/mca/ptl/base/ptl_base_sendfrag.h"
static void mca_ptl_base_send_frag_construct(mca_ptl_base_send_frag_t* frag);
static void mca_ptl_base_send_frag_destruct(mca_ptl_base_send_frag_t* frag);
opal_class_t mca_ptl_base_send_frag_t_class = {
"mca_ptl_base_send_frag_t",
OBJ_CLASS(mca_ptl_base_frag_t),
(opal_construct_t) mca_ptl_base_send_frag_construct,
(opal_destruct_t) mca_ptl_base_send_frag_destruct
};
static void mca_ptl_base_send_frag_construct(mca_ptl_base_send_frag_t* frag)
{
frag->frag_base.frag_type = MCA_PTL_FRAGMENT_SEND;
}
static void mca_ptl_base_send_frag_destruct(mca_ptl_base_send_frag_t* frag)
{
}

46
ompi/mca/ptl/base/ptl_base_sendfrag.h
Просмотреть файл

@ -1,46 +0,0 @@
/*
* Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
* University Research and Technology
* Corporation. All rights reserved.
* Copyright (c) 2004-2005 The University of Tennessee and The University
* of Tennessee Research Foundation. All rights
* reserved.
* Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
* University of Stuttgart. All rights reserved.
* Copyright (c) 2004-2005 The Regents of the University of California.
* All rights reserved.
* $COPYRIGHT$
*
* Additional copyrights may follow
*
* $HEADER$
*/
/**
* @file
*/
#ifndef MCA_PTL_BASE_SEND_FRAG_H
#define MCA_PTL_BASE_SEND_FRAG_H
#include "ompi/mca/ptl/ptl.h"
#include "ompi/mca/ptl/base/ptl_base_fragment.h"
#if defined(c_plusplus) || defined(__cplusplus)
extern "C" {
#endif
OMPI_DECLSPEC extern opal_class_t mca_ptl_base_send_frag_t_class;
/**
* Base type for send fragment descriptors
*/
struct mca_ptl_base_send_frag_t {
mca_ptl_base_frag_t frag_base; /**< base fragment descriptor */
struct mca_ptl_base_send_request_t *frag_request; /**< pointer to send request */
};
typedef struct mca_ptl_base_send_frag_t mca_ptl_base_send_frag_t;
#if defined(c_plusplus) || defined(__cplusplus)
}
#endif
#endif

10
ompi/mca/ptl/base/ptl_base_sendreq.c
Просмотреть файл

@ -1,10 +0,0 @@
#include "ompi/mca/ptl/base/ptl_base_sendreq.h"
OBJ_CLASS_INSTANCE(
mca_ptl_base_send_request_t,
mca_pml_base_send_request_t,
NULL,
NULL
);

71
ompi/mca/ptl/base/ptl_base_sendreq.h
Просмотреть файл

@ -1,71 +0,0 @@
/*
* Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
* University Research and Technology
* Corporation. All rights reserved.
* Copyright (c) 2004-2005 The University of Tennessee and The University
* of Tennessee Research Foundation. All rights
* reserved.
* Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
* University of Stuttgart. All rights reserved.
* Copyright (c) 2004-2005 The Regents of the University of California.
* All rights reserved.
* $COPYRIGHT$
*
* Additional copyrights may follow
*
* $HEADER$
*/
/**
* @file
*/
#ifndef MCA_PTL_BASE_SEND_REQUEST_H
#define MCA_PTL_BASE_SEND_REQUEST_H
#include "ompi/mca/ptl/ptl.h"
#include "ompi/mca/pml/base/pml_base_sendreq.h"
#if defined(c_plusplus) || defined(__cplusplus)
extern "C" {
#endif
/**
* Base type for send requests
*/
struct mca_ptl_base_send_request_t {
mca_pml_base_send_request_t req_send;
size_t req_offset; /**< number of bytes that have been scheduled */
size_t req_bytes_sent; /**< number of bytes that have been sent */
ompi_ptr_t req_peer_match; /**< matched receive at peer */
ompi_ptr_t req_peer_addr; /**< peers remote buffer address */
uint64_t req_peer_size; /**< size of peers remote buffer */
bool req_cached; /**< has this request been obtained from the ptls cache */
volatile int32_t req_lock; /**< lock used by the scheduler */
struct mca_ptl_base_module_t* req_ptl; /**< ptl allocated for first fragment */
struct mca_ptl_base_peer_t* req_peer; /**< peer associated w/ this ptl */
};
typedef struct mca_ptl_base_send_request_t mca_ptl_base_send_request_t;
OMPI_DECLSPEC OBJ_CLASS_DECLARATION(mca_ptl_base_send_request_t);
/**
* Atomically increase the request offset.
*
* @param request (IN) Send request.
* @param offset (IN) Increment.
*/
static inline void mca_ptl_base_send_request_offset(
mca_ptl_base_send_request_t* request,
size_t offset)
{
OPAL_THREAD_ADD_SIZE_T((&request->req_offset), offset);
}
#if defined(c_plusplus) || defined(__cplusplus)
}
#endif
#endif

73
ompi/mca/ptl/gm/Makefile.am
Просмотреть файл

@ -1,73 +0,0 @@
#
# Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
# University Research and Technology
# Corporation. All rights reserved.
# Copyright (c) 2004-2005 The University of Tennessee and The University
# of Tennessee Research Foundation. All rights
# reserved.
# Copyright (c) 2004 The Ohio State University.
# All rights reserved.
# Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
# University of Stuttgart. All rights reserved.
# Copyright (c) 2004-2005 The Regents of the University of California.
# All rights reserved.
# $COPYRIGHT$
#
# Additional copyrights may follow
#
# $HEADER$
#
# Use the top-level Makefile.options
include $(top_ompi_srcdir)/config/Makefile.options
AM_CPPFLAGS = $(ptl_gm_CPPFLAGS)
# Make the output library in this directory, and name it either
# mca_<type>_<name>.la (for DSO builds) or libmca_<type>_<name>.la
# (for static builds).
#if OMPI_ENABLE_GM_CACHE
# additional_source = ptl_gm_regcache.c
#else
# additional_source =
#endif
libmca_ptl_gm_la_sources = $(additional_source) \
ptl_gm.c \
ptl_gm.h \
ptl_gm_component.c \
ptl_gm_peer.h \
ptl_gm_priv.h \
ptl_gm_priv.c \
ptl_gm_proc.c \
ptl_gm_proc.h \
ptl_gm_sendfrag.c \
ptl_gm_sendfrag.h \
ptl_gm_memory.c
if OMPI_BUILD_ptl_gm_DSO
component_noinst =
component_install = mca_ptl_gm.la
else
component_noinst = libmca_ptl_gm.la
component_install =
endif
mcacomponentdir = $(libdir)/openmpi
mcacomponent_LTLIBRARIES = $(component_install)
mca_ptl_gm_la_SOURCES = $(libmca_ptl_gm_la_sources)
mca_ptl_gm_la_LIBADD = \
$(ptl_gm_LIBS) \
$(top_ompi_builddir)/ompi/libmpi.la \
$(top_ompi_builddir)/orte/liborte.la \
$(top_ompi_builddir)/opal/libopal.la
mca_ptl_gm_la_LDFLAGS = -module -avoid-version $(ptl_gm_LDFLAGS)
noinst_LTLIBRARIES = $(component_noinst)
libmca_ptl_gm_la_SOURCES = $(libmca_ptl_gm_la_sources)
libmca_ptl_gm_la_LIBADD = $(ptl_gm_LIBS)
libmca_ptl_gm_la_LDFLAGS = -module -avoid-version $(ptl_gm_LDFLAGS)

43
ompi/mca/ptl/gm/configure.m4
Просмотреть файл

@ -1,43 +0,0 @@
# -*- shell-script -*-
#
# Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
# University Research and Technology
# Corporation. All rights reserved.
# Copyright (c) 2004-2005 The University of Tennessee and The University
# of Tennessee Research Foundation. All rights
# reserved.
# Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
# University of Stuttgart. All rights reserved.
# Copyright (c) 2004-2005 The Regents of the University of California.
# All rights reserved.
# $COPYRIGHT$
#
# Additional copyrights may follow
#
# $HEADER$
#
# MCA_ptl_gm_CONFIG([action-if-can-compile],
# [action-if-cant-compile])
# ------------------------------------------------
AC_DEFUN([MCA_ptl_gm_CONFIG],[
OMPI_CHECK_GM([ptl_gm],
[ptl_gm_happy="yes"],
[ptl_gm_happy="no"])
AS_IF([test "$ptl_gm_happy" = "yes"],
[ptl_gm_WRAPPER_EXTRA_LDFLAGS="$ptl_gm_LDFLAGS"
ptl_gm_WRAPPER_EXTRA_LIBS="$ptl_gm_LIBS"
$1],
[$2])
# substitute in the things needed to build gm
AC_SUBST([ptl_gm_CFLAGS])
AC_SUBST([ptl_gm_CPPFLAGS])
AC_SUBST([ptl_gm_LDFLAGS])
AC_SUBST([ptl_gm_LIBS])
# Define it for internal use.
AC_DEFINE_UNQUOTED(OMPI_MCA_PTL_GM_CACHE_ENABLE, 0,
[Whether we want the internal GM cache to be activated.])
])dnl

25
ompi/mca/ptl/gm/configure.params
Просмотреть файл

@ -1,25 +0,0 @@
# -*- shell-script -*-
#
# Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
# University Research and Technology
# Corporation. All rights reserved.
# Copyright (c) 2004-2005 The University of Tennessee and The University
# of Tennessee Research Foundation. All rights
# reserved.
# Copyright (c) 2004 The Ohio State University.
# All rights reserved.
# Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
# University of Stuttgart. All rights reserved.
# Copyright (c) 2004-2005 The Regents of the University of California.
# All rights reserved.
# $COPYRIGHT$
#
# Additional copyrights may follow
#
# $HEADER$
#
# Specific to this module
PARAM_INIT_FILE=ptl_gm.c
PARAM_CONFIG_FILES="Makefile"

414
ompi/mca/ptl/gm/ptl_gm.c
Просмотреть файл

@ -1,414 +0,0 @@
/* -*- Mode: C; c-basic-offset:4 ; -*- */
/*
* Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
* University Research and Technology
* Corporation. All rights reserved.
* Copyright (c) 2004-2005 The University of Tennessee and The University
* of Tennessee Research Foundation. All rights
* reserved.
* Copyright (c) 2004 The Ohio State University.
* All rights reserved.
* Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
* University of Stuttgart. All rights reserved.
* Copyright (c) 2004-2005 The Regents of the University of California.
* All rights reserved.
* $COPYRIGHT$
*
* Additional copyrights may follow
*
* $HEADER$
*/
#include "ompi_config.h"
#include <string.h>
#include "ompi/class/ompi_bitmap.h"
#include "opal/util/output.h"
#include "orte/util/proc_info.h"
#include "orte/mca/ns/ns.h"
#include "ompi/mca/ptl/ptl.h"
#include "ompi/mca/ptl/base/ptl_base_header.h"
#include "ptl_gm.h"
#include "ptl_gm_proc.h"
#include "ptl_gm_priv.h"
#include "ptl_gm_peer.h"
#include "ptl_gm_sendfrag.h"
#include "ompi/proc/proc.h"
mca_ptl_gm_module_t mca_ptl_gm_module = {
{
&mca_ptl_gm_component.super,
1, /* max size of request cache */
sizeof(mca_ptl_gm_send_frag_t), /* bytes required by ptl for a request */
0, /* max size of first fragment */
0, /* min fragment size */
0, /* max fragment size */
0, /* exclusivity */
50, /* latency */
0, /* bandwidth */
MCA_PTL_PUT, /* ptl flags */
/* collection of interfaces */
mca_ptl_gm_add_procs,
mca_ptl_gm_del_procs,
mca_ptl_gm_finalize,
mca_ptl_gm_peer_send,
mca_ptl_gm_put,
mca_ptl_gm_get,
mca_ptl_gm_matched,
mca_ptl_gm_request_init,
mca_ptl_gm_request_fini,
NULL,
NULL,
NULL
}
};
OBJ_CLASS_INSTANCE (mca_ptl_gm_send_request_t,
mca_ptl_base_send_request_t, NULL, NULL);
OBJ_CLASS_INSTANCE (mca_ptl_gm_peer_t, opal_list_item_t, NULL, NULL);
int
mca_ptl_gm_add_procs (struct mca_ptl_base_module_t *ptl,
size_t nprocs,
struct ompi_proc_t **orte_procs,
struct mca_ptl_base_peer_t **peers,
ompi_bitmap_t * reachable)
{
uint32_t i, j, num_peer_ptls = 1;
struct ompi_proc_t *orte_proc;
mca_ptl_gm_proc_t *ptl_proc;
mca_ptl_gm_peer_t *ptl_peer;
ompi_proc_t* local_proc = ompi_proc_local();
for (i = 0; i < nprocs; i++) {
orte_proc = orte_procs[i];
if( orte_proc == local_proc ) continue;
ptl_proc = mca_ptl_gm_proc_create ((mca_ptl_gm_module_t *) ptl, orte_proc);
if (NULL == ptl_proc) {
opal_output( 0, "[%s:%d] cannot allocate memory for the GM module", __FILE__, __LINE__ );
continue;
}
OPAL_THREAD_LOCK (&ptl_proc->proc_lock);
if (ptl_proc->proc_addr_count == ptl_proc->proc_peer_count) {
OPAL_THREAD_UNLOCK (&ptl_proc->proc_lock);
opal_output( 0, "[%s:%d] modex exchange failed for GM module", __FILE__, __LINE__ );
continue;
}
ptl_peer = NULL; /* force it to NULL before looping through the ptls */
/* TODO: make this extensible to multiple nics */
for( j = 0; j < num_peer_ptls; j++ ) {
ptl_peer = OBJ_NEW (mca_ptl_gm_peer_t);
if (NULL == ptl_peer) {
OPAL_THREAD_UNLOCK (&ptl_proc->proc_lock);
opal_output( 0, "[%s:%d] cannot allocate memory for one of the GM ptl", __FILE__, __LINE__ );
continue;
}
ptl_peer->peer_ptl = (mca_ptl_gm_module_t *) ptl;
ptl_peer->peer_proc = ptl_proc;
ptl_peer->peer_addr.port_id = ptl_proc->proc_addrs->port_id;
#if GM_API_VERSION > 0x200
ptl_peer->peer_addr.global_id = ptl_proc->proc_addrs->global_id;
if (GM_SUCCESS != gm_global_id_to_node_id(((mca_ptl_gm_module_t *) ptl)->gm_port,
ptl_proc->proc_addrs[j].global_id,
&(ptl_peer->peer_addr.local_id))) {
opal_output( 0, "[%s:%d] error in converting global to local id \n",
__FILE__, __LINE__ );
OBJ_RELEASE( ptl_peer );
assert( NULL == ptl_peer );
continue;
}
#else
strncpy( ptl_peer->peer_addr.global_id, ptl_proc->proc_addrs->global_id, GM_MAX_HOST_NAME_LEN );
ptl_peer->peer_addr.local_id = gm_host_name_to_node_id( ((mca_ptl_gm_module_t *) ptl)->gm_port,
ptl_proc->proc_addrs[j].global_id );
if( GM_NO_SUCH_NODE_ID == ptl_peer->peer_addr.local_id ) {
opal_output( 0, "Unable to convert the remote host name (%s) to a host id",
ptl_proc->proc_addrs[j].global_id );
OBJ_RELEASE( ptl_peer );
assert( NULL == ptl_peer );
continue;
}
#endif /* GM_API_VERSION > 0x200 */
ptl_proc->peer_arr[ptl_proc->proc_peer_count] = ptl_peer;
ptl_proc->proc_peer_count++;
ompi_bitmap_set_bit (reachable, i); /* set the bit again and again */
}
OPAL_THREAD_UNLOCK (&ptl_proc->proc_lock);
peers[i] = (struct mca_ptl_base_peer_t*)ptl_peer;
}
return OMPI_SUCCESS;
}
/*
*
*/
int
mca_ptl_gm_del_procs (struct mca_ptl_base_module_t *ptl,
size_t nprocs,
struct ompi_proc_t **procs,
struct mca_ptl_base_peer_t **peers)
{
size_t i;
for (i = 0; i < nprocs; i++) {
OBJ_RELEASE (peers[i]);
}
return OMPI_SUCCESS;
}
/*
*
*/
int
mca_ptl_gm_finalize (struct mca_ptl_base_module_t *base_ptl)
{
uint32_t index;
mca_ptl_gm_module_t* ptl = (mca_ptl_gm_module_t*)base_ptl;
for( index = 0; index < mca_ptl_gm_component.gm_num_ptl_modules; index++ ) {
if( mca_ptl_gm_component.gm_ptl_modules[index] == ptl ) {
mca_ptl_gm_component.gm_ptl_modules[index] = NULL;
break;
}
}
if( index == mca_ptl_gm_component.gm_num_ptl_modules ) {
opal_output( 0, "%p is not a GM PTL !!!\n", (void*)base_ptl );
return OMPI_ERROR;
}
/* we should do the same things as in the init step in reverse order.
* First we shutdown all threads if there are any.
*/
#if OMPI_HAVE_POSIX_THREADS
if( 0 != ptl->thread.t_handle ) {
void* thread_return;
pthread_cancel( ptl->thread.t_handle );
opal_thread_join( &(ptl->thread), &thread_return );
}
#endif /* OMPI_HAVE_POSIX_THREADS */
/* Closing each port require several steps. As there is no way to cancel all
* already posted messages we start by unregistering all memory and then close
* the port. After we can release all internal data.
*/
if( ptl->gm_send_dma_memory != NULL ) {
gm_dma_free( ptl->gm_port, ptl->gm_send_dma_memory );
ptl->gm_send_dma_memory = NULL;
}
if( ptl->gm_recv_dma_memory != NULL ) {
gm_dma_free( ptl->gm_port, ptl->gm_recv_dma_memory );
ptl->gm_recv_dma_memory = NULL;
}
/* Now close the port if one is open */
if( ptl->gm_port != NULL ) {
gm_close( ptl->gm_port );
ptl->gm_port = NULL;
}
/* And now release all internal ressources. */
OBJ_DESTRUCT( &(ptl->gm_send_frags) );
if( ptl->gm_send_fragments != NULL ) {
free( ptl->gm_send_fragments );
ptl->gm_send_fragments = NULL;
}
OBJ_DESTRUCT( &(ptl->gm_recv_frags_free) );
if( ptl->gm_recv_fragments != NULL ) {
free( ptl->gm_recv_fragments );
ptl->gm_recv_fragments = NULL;
}
/* These are supposed to be empty by now */
OBJ_DESTRUCT( &(ptl->gm_send_frags_queue) );
OBJ_DESTRUCT( &(ptl->gm_pending_acks) );
OBJ_DESTRUCT( &(ptl->gm_recv_outstanding_queue) );
/* And finally release the PTL itself */
free( ptl );
return OMPI_SUCCESS;
}
int
mca_ptl_gm_request_init( struct mca_ptl_base_module_t *ptl,
struct mca_ptl_base_send_request_t *request )
{
#if 0
mca_ptl_gm_send_frag_t *frag;
struct mca_ptl_gm_send_request_t *req;
frag = mca_ptl_gm_alloc_send_frag(ptl, request);
if (NULL == frag) {
opal_output(0,"[%s:%d] Unable to allocate a gm send fragment\n");
return OMPI_ERR_OUT_OF_RESOURCE;
} else {
req = (mca_ptl_gm_send_request_t *)request;
req->req_frag = frag;
frag->status = 0; /*MCA_PTL_GM_FRAG_CACHED;*/
}
return OMPI_SUCCESS;
#endif
return OMPI_SUCCESS;
}
/*
*
*/
void
mca_ptl_gm_request_fini (struct mca_ptl_base_module_t *ptl,
struct mca_ptl_base_send_request_t *request)
{
#if 0
mca_ptl_gm_send_frag_t *frag;
frag = ((mca_ptl_gm_send_request_t *)request)->req_frag;
OMPI_FREE_LIST_RETURN(&(((mca_ptl_gm_module_t *)ptl)->gm_send_frags),
(opal_list_item_t *)frag);
frag->status = 0;
#endif
OBJ_DESTRUCT(request+1);
}
/*
* Initiate a put
*/
int
mca_ptl_gm_put (struct mca_ptl_base_module_t *ptl,
struct mca_ptl_base_peer_t *ptl_peer,
struct mca_ptl_base_send_request_t *sendreq,
size_t offset, size_t size, int flags)
{
int rc;
mca_ptl_gm_send_frag_t *putfrag;
rc = mca_ptl_gm_put_frag_init( &putfrag,
(mca_ptl_gm_peer_t*)ptl_peer, (mca_ptl_gm_module_t*)ptl,
sendreq, offset, &size, flags );
rc = mca_ptl_gm_peer_send_continue( (mca_ptl_gm_peer_t *)ptl_peer, putfrag,
sendreq, offset, &size, flags );
return OMPI_SUCCESS;
}
/*
* initiate a get.
*/
int
mca_ptl_gm_get (struct mca_ptl_base_module_t *ptl,
struct mca_ptl_base_peer_t *ptl_base_peer,
struct mca_ptl_base_recv_request_t *request,
size_t offset, size_t size, int flags)
{
return OMPI_SUCCESS;
}
static void mca_ptl_gm_basic_ack_callback( struct gm_port* port, void* context, gm_status_t status )
{
mca_ptl_gm_module_t* gm_ptl;
mca_ptl_base_header_t* header;
header = (mca_ptl_base_header_t*)context;
gm_ptl = (mca_ptl_gm_module_t*)header->hdr_ack.hdr_dst_addr.pval;
OMPI_GM_FREE_LIST_RETURN( &(gm_ptl->gm_send_dma_frags), ((opal_list_item_t*)header) );
/* release the send token */
opal_atomic_add( &(gm_ptl->num_send_tokens), 1 );
}
/* A posted receive has been matched - if required send an
* ack back to the peer and process the fragment.
*/
void
mca_ptl_gm_matched( mca_ptl_base_module_t* ptl,
mca_ptl_base_recv_frag_t* frag )
{
mca_ptl_base_recv_request_t *request;
mca_ptl_base_header_t *hdr;
int32_t rc;
mca_ptl_gm_module_t *gm_ptl;
mca_ptl_gm_recv_frag_t *recv_frag;
mca_ptl_gm_peer_t* peer;
struct iovec iov = { NULL, 0 };
gm_ptl = (mca_ptl_gm_module_t *)ptl;
request = frag->frag_request;
recv_frag = (mca_ptl_gm_recv_frag_t *)frag;
peer = (mca_ptl_gm_peer_t*)recv_frag->frag_recv.frag_base.frag_peer;
if( frag->frag_base.frag_header.hdr_common.hdr_flags & MCA_PTL_FLAGS_ACK ) { /* need to send an ack back */
opal_list_item_t *item;
OMPI_FREE_LIST_WAIT( &(gm_ptl->gm_send_dma_frags), item, rc );
if( NULL == item ) {
opal_output(0,"[%s:%d] unable to alloc a gm fragment\n", __FILE__,__LINE__);
OPAL_THREAD_LOCK (&mca_ptl_gm_component.gm_lock);
opal_list_append (&mca_ptl_gm_module.gm_pending_acks, (opal_list_item_t *)frag);
OPAL_THREAD_UNLOCK (&mca_ptl_gm_component.gm_lock);
} else {
opal_atomic_sub( &(gm_ptl->num_send_tokens), 1 );
assert( gm_ptl->num_send_tokens >= 0 );
hdr = (mca_ptl_base_header_t*)item;
hdr->hdr_ack.hdr_common.hdr_type = MCA_PTL_HDR_TYPE_ACK;
hdr->hdr_ack.hdr_common.hdr_flags = frag->frag_base.frag_header.hdr_common.hdr_flags;
hdr->hdr_ack.hdr_src_ptr = frag->frag_base.frag_header.hdr_rndv.hdr_src_ptr;
hdr->hdr_ack.hdr_dst_match.lval = 0L;
hdr->hdr_ack.hdr_dst_match.pval = request;
hdr->hdr_ack.hdr_dst_addr.lval = 0L;
hdr->hdr_ack.hdr_dst_addr.pval = ptl; /* local use */
hdr->hdr_ack.hdr_dst_size = request->req_recv.req_bytes_packed;
gm_send_with_callback( ((mca_ptl_gm_module_t*)ptl)->gm_port, hdr,
GM_SIZE, sizeof(mca_ptl_base_ack_header_t),
GM_LOW_PRIORITY,
peer->peer_addr.local_id,
peer->peer_addr.port_id,
mca_ptl_gm_basic_ack_callback,
(void *)hdr );
}
}
if( frag->frag_base.frag_size > 0 ) {
ompi_convertor_t* convertor;
uint32_t out_size;
int32_t freeAfter;
size_t max_data;
iov.iov_len = recv_frag->attached_data_length;
/* Here we expect that frag_addr is the begin of the buffer header included */
iov.iov_base = frag->frag_base.frag_addr;
convertor = &(request->req_recv.req_convertor);
out_size = 1;
max_data = iov.iov_len;
rc = ompi_convertor_unpack( convertor, &(iov), &out_size, &max_data, &freeAfter );
assert( rc >= 0 );
recv_frag->frag_bytes_processed += max_data;
}
/* update progress*/
ptl->ptl_recv_progress( ptl, request, iov.iov_len, iov.iov_len );
/* Now update the status of the fragment */
if( ((mca_ptl_gm_recv_frag_t*)frag)->have_allocated_buffer == true ) {
mca_ptl_gm_release_local_buffer( ((mca_ptl_gm_recv_frag_t*)frag)->frag_recv.frag_base.frag_addr );
((mca_ptl_gm_recv_frag_t*)frag)->have_allocated_buffer = false;
}
/* I'm done with this fragment. Return it to the free list */
OMPI_FREE_LIST_RETURN( &(gm_ptl->gm_recv_frags_free), (opal_list_item_t*)frag );
}

251
ompi/mca/ptl/gm/ptl_gm.h
Просмотреть файл

@ -1,251 +0,0 @@
/* -*- Mode: C; c-basic-offset:4 ; -*- */
/*
* Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
* University Research and Technology
* Corporation. All rights reserved.
* Copyright (c) 2004-2005 The University of Tennessee and The University
* of Tennessee Research Foundation. All rights
* reserved.
* Copyright (c) 2004 The Ohio State University.
* All rights reserved.
* Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
* University of Stuttgart. All rights reserved.
* Copyright (c) 2004-2005 The Regents of the University of California.
* All rights reserved.
* $COPYRIGHT$
*
* Additional copyrights may follow
*
* $HEADER$
*/
/**
* @file
*/
#ifndef MCA_PTL_GM_H
#define MCA_PTL_GM_H
#include "ompi_config.h"
#include "ompi/class/ompi_free_list.h"
#include "ompi/mca/ptl/ptl.h"
#include "ptl_gm_priv.h"
#include "ptl_gm_peer.h"
#define MCA_PTL_GM_STATISTICS 0
#define MAX_RECV_TOKENS 256
#define PTL_GM_ADMIN_SEND_TOKENS 0
#define PTL_GM_ADMIN_RECV_TOKENS 0
#define GM_SIZE 30
#define NUM_RECV_FRAGS 256
#define MCA_PTL_GM_FRAG_CACHED
#if defined(c_plusplus) || defined(__cplusplus)
extern "C" {
#endif
/**
* GM PTL component
*/
struct mca_ptl_gm_component_t {
mca_ptl_base_component_1_0_0_t super; /**< base PTL module */
struct mca_ptl_gm_module_t **gm_ptl_modules; /**< array of available PTL modules */
size_t gm_num_ptl_modules; /**< number of ptls actually used */
size_t gm_max_ptl_modules; /**< maximum number of ptls - available */
uint32_t gm_max_port_number; /**< maximum number of ports by board */
uint32_t gm_max_boards_number; /**< maximum number of boards on the node */
uint32_t gm_free_list_num; /**< initial size of free lists */
uint32_t gm_free_list_max; /**< maximum size of free lists */
uint32_t gm_free_list_inc; /**< number of elements to alloc when growing free lists */
uint32_t gm_segment_size; /**< size of the allocated segment */
uint32_t gm_eager_limit; /**< number of bytes before the rendez-vous protocol. If the
**< size of the message is less than this number then GM
**< use a eager protocol. */
#if OMPI_MCA_PTL_GM_HAVE_RDMA_GET
uint32_t gm_rndv_burst_limit;
uint32_t gm_rdma_frag_size; /**< maximum fragment size used to transfer data over RDMA */
#endif /* OMPI_MCA_PTL_GM_HAVE_RDMA_GET */
char* gm_port_name; /**< the name used to get the port */
struct mca_ptl_gm_proc_t* gm_local;
opal_list_t gm_procs;
opal_list_t gm_send_req;
ompi_free_list_t gm_unexpected_frags_data;
opal_mutex_t gm_lock; /**< lock for accessing module state */
};
typedef struct mca_ptl_gm_component_t mca_ptl_gm_component_t;
extern mca_ptl_gm_component_t mca_ptl_gm_component;
/**
* GM PTL Interface
*/
struct mca_ptl_gm_module_t {
mca_ptl_base_module_t super; /**< base PTL module interface */
struct gm_port *gm_port;
mca_ptl_gm_addr_t local_addr;
unsigned int num_send_tokens;
unsigned int num_recv_tokens;
unsigned int max_send_tokens;
unsigned int max_recv_tokens;
void* gm_send_dma_memory; /**< pointer to the send DMA registered memory attached to the PTL */
void* gm_recv_dma_memory; /**< pointer to the recv DMA registered memory attached to the PTL */
struct mca_ptl_gm_send_frag_t* gm_send_fragments;
struct mca_ptl_gm_recv_frag_t* gm_recv_fragments;
ompi_free_list_t gm_send_frags;
ompi_free_list_t gm_send_dma_frags;
ompi_free_list_t gm_recv_frags_free;
opal_list_t gm_send_frags_queue;
opal_list_t gm_pending_acks;
opal_list_t gm_recv_outstanding_queue;
opal_thread_t thread;
#if MCA_PTL_GM_STATISTICS
size_t ptl_bytes_sent;
size_t ptl_bytes_recv;
#endif /* MCA_PTL_GM_STATISTICS */
};
typedef struct mca_ptl_gm_module_t mca_ptl_gm_module_t;
extern mca_ptl_gm_module_t mca_ptl_gm_module;
/**
* Register GM module parameters with the MCA framework
*/
extern int mca_ptl_gm_component_open (void);
/**
* Any final cleanup before being unloaded.
*/
extern int mca_ptl_gm_component_close (void);
/**
* GM module initialization.
*
* @param num_ptls (OUT) Number of PTLs returned in PTL array.
* @param allow_multi_user_threads (OUT) Flag indicating wether PTL supports user threads (TRUE)
* @param have_hidden_threads (OUT) Flag indicating wether PTL uses threads (TRUE)
*/
extern mca_ptl_base_module_t **mca_ptl_gm_component_init (int *num_ptl_modules,
bool enable_progress_threads,
bool enable_mpi_threads);
/**
* GM module control.
*/
extern int mca_ptl_gm_component_control (int param, void *value, size_t size);
/**
* GM module progress.
*/
extern int mca_ptl_gm_component_progress (mca_ptl_tstamp_t tstamp);
/**
* GM put
*/
extern int mca_ptl_gm_put( struct mca_ptl_base_module_t *ptl,
struct mca_ptl_base_peer_t *ptl_peer,
struct mca_ptl_base_send_request_t *sendreq,
size_t offset, size_t size, int flags);
/**
* GM get
*/
extern int mca_ptl_gm_get (struct mca_ptl_base_module_t *ptl,
struct mca_ptl_base_peer_t *ptl_peer,
struct mca_ptl_base_recv_request_t *sendreq,
size_t offset, size_t size, int flags);
/**
* PML->PTL notification of change in the process list.
*
* @param ptl (IN)
* @param nprocs (IN) Number of processes
* @param procs (IN) Set of processes
* @param peers (OUT) Set of (optional) peer addressing info.
* @param peers (IN/OUT) Set of processes that are reachable via this PTL.
* @return OMPI_SUCCESS or error status on failure.
*
*/
extern int mca_ptl_gm_add_procs (struct mca_ptl_base_module_t *ptl,
size_t nprocs,
struct ompi_proc_t **procs,
struct mca_ptl_base_peer_t **peers,
struct ompi_bitmap_t * reachable);
/**
* PML->PTL notification of change in the process list.
*
* @param ptl (IN) PTL instance
* @param nproc (IN) Number of processes.
* @param procs (IN) Set of processes.
* @param peers (IN) Set of peer data structures.
* @return Status indicating if cleanup was successful
*
*/
extern int mca_ptl_gm_del_procs( struct mca_ptl_base_module_t *ptl,
size_t nprocs,
struct ompi_proc_t **procs,
struct mca_ptl_base_peer_t **peers );
/**
* PML->PTL Allocate a send request from the PTL modules free list.
*
* @param ptl (IN) PTL instance
* @param request (OUT) Pointer to allocated request.
* @return Status indicating if allocation was successful.
*
*/
extern int mca_ptl_gm_request_init( struct mca_ptl_base_module_t* ptl,
struct mca_ptl_base_send_request_t* req);
/**
*
*/
extern void mca_ptl_gm_request_fini( struct mca_ptl_base_module_t *ptl,
struct mca_ptl_base_send_request_t* req);
/**
* PML->PTL Notification that a receive fragment has been matched.
*
* @param ptl (IN) PTL instance
* @param recv_frag (IN) Receive fragment
*
*/
extern void mca_ptl_gm_matched (struct mca_ptl_base_module_t *ptl,
struct mca_ptl_base_recv_frag_t *frag);
/**
*
*/
extern int mca_ptl_gm_finalize (struct mca_ptl_base_module_t *ptl);
/**
* Internally allocate memory for the unexpected messages. We will manage a list
* of such buffers in order to avoid too many memory allocations.
*/
extern char* mca_ptl_gm_get_local_buffer( void );
extern void mca_ptl_gm_release_local_buffer( char* ptr );
union mca_ptl_base_header_t;
void mca_ptl_gm_dump_header( char* str, union mca_ptl_base_header_t* hdr );
#if OMPI_ENABLE_DEBUG
#include "opal/class/opal_list.h"
/* If debug is enabled we have to work around the item validity checks. */
#define OMPI_GM_FREE_LIST_RETURN( LIST, ITEM ) \
do { \
(ITEM)->opal_list_item_refcount = 0; \
(ITEM)->opal_list_item_belong_to = NULL; \
(ITEM)->super.cls_init_file_name = __FILE__; \
(ITEM)->super.cls_init_lineno = __LINE__; \
OMPI_FREE_LIST_RETURN( (LIST), (ITEM) ); \
} while(0)
#else
#define OMPI_GM_FREE_LIST_RETURN OMPI_FREE_LIST_RETURN
#endif /* OMPI_ENABLE_DEBUG */
#if defined(c_plusplus) || defined(__cplusplus)
}
#endif
#endif

600
ompi/mca/ptl/gm/ptl_gm_component.c
Просмотреть файл

@ -1,600 +0,0 @@
/* -*- Mode: C; c-basic-offset:4 ; -*- */
/*
* Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
* University Research and Technology
* Corporation. All rights reserved.
* Copyright (c) 2004-2005 The University of Tennessee and The University
* of Tennessee Research Foundation. All rights
* reserved.
* Copyright (c) 2004 The Ohio State University.
* All rights reserved.
* Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
* University of Stuttgart. All rights reserved.
* Copyright (c) 2004-2005 The Regents of the University of California.
* All rights reserved.
* $COPYRIGHT$
*
* Additional copyrights may follow
*
* $HEADER$
*/
#include "ompi_config.h"
#include "ompi/constants.h"
#include "opal/util/output.h"
#include "ompi/mca/ptl/ptl.h"
#include "ompi/mca/pml/base/pml_base_module_exchange.h"
#include "ptl_gm.h"
#include "ptl_gm_priv.h"
#include "ptl_gm_peer.h"
#include "ptl_gm_sendfrag.h"
mca_ptl_gm_component_t mca_ptl_gm_component = {
{
/* First, the mca_base_component_t struct containing meta information
about the component itself */
{
/* Indicate that we are a pml v1.0.0 component (which also implies a
specific MCA version) */
MCA_PTL_BASE_VERSION_1_0_0,
"gm", /* MCA component name */
OMPI_MAJOR_VERSION, /* MCA component major version */
OMPI_MINOR_VERSION, /* MCA component minor version */
OMPI_RELEASE_VERSION, /* MCA component release version */
mca_ptl_gm_component_open, /* component open */
mca_ptl_gm_component_close /* component close */
}
,
/* Next the MCA v1.0.0 component meta data */
{
/* Whether the component is checkpointable or not */
false
},
mca_ptl_gm_component_init,
mca_ptl_gm_component_control,
mca_ptl_gm_component_progress
}
};
/*
* utility routines for parameter registration
*/
static inline char *
mca_ptl_gm_param_register_string( const char *param_name,
const char *default_value )
{
char *param_value;
int id = mca_base_param_register_string( "ptl", "gm", param_name, NULL,
default_value) ;
mca_base_param_lookup_string (id, &param_value);
return param_value;
}
static inline int
mca_ptl_gm_param_register_int( const char *param_name, int default_value )
{
int id =
mca_base_param_register_int ("ptl", "gm", param_name, NULL,
default_value);
int param_value = default_value;
mca_base_param_lookup_int (id, &param_value);
return param_value;
}
/*
* Called by MCA framework to open the module, registers
* module parameters.
*/
int
mca_ptl_gm_component_open(void)
{
uint32_t default_first_frag_size;
/* initialize state */
mca_ptl_gm_component.gm_ptl_modules = NULL;
mca_ptl_gm_component.gm_num_ptl_modules = 0;
/* initialize objects */
OBJ_CONSTRUCT (&mca_ptl_gm_component.gm_lock, opal_mutex_t);
OBJ_CONSTRUCT (&mca_ptl_gm_component.gm_procs, opal_list_t);
OBJ_CONSTRUCT (&mca_ptl_gm_component.gm_send_req, opal_list_t);
/* register GM component parameters */
mca_ptl_gm_component.gm_port_name =
mca_ptl_gm_param_register_string( "port_name", "OMPI_GM" );
mca_ptl_gm_component.gm_max_port_number =
mca_ptl_gm_param_register_int ("max_ports_number", 16 );
mca_ptl_gm_component.gm_max_boards_number =
mca_ptl_gm_param_register_int ("max_boards_number", 4 );
mca_ptl_gm_component.gm_max_ptl_modules =
mca_ptl_gm_param_register_int( "max_ptl_modules", 1 );
mca_ptl_gm_component.gm_segment_size =
mca_ptl_gm_param_register_int( "segment_size", 32 * 1024 );
default_first_frag_size = mca_ptl_gm_component.gm_segment_size - sizeof(mca_ptl_base_rendezvous_header_t);
mca_ptl_gm_module.super.ptl_first_frag_size =
mca_ptl_gm_param_register_int ("first_frag_size", default_first_frag_size );
/* the first_frag_size should be always less than the gm_segment_size by at least the
* header sizeof.
*/
if( mca_ptl_gm_module.super.ptl_first_frag_size > default_first_frag_size ) {
mca_ptl_gm_module.super.ptl_first_frag_size = default_first_frag_size;
}
mca_ptl_gm_module.super.ptl_min_frag_size =
mca_ptl_gm_param_register_int ("min_frag_size", 64 * 1024);
mca_ptl_gm_module.super.ptl_max_frag_size =
mca_ptl_gm_param_register_int ("max_frag_size", 256 * 1024 * 1024);
/* Parameters setting the message limits. */
mca_ptl_gm_component.gm_eager_limit =
mca_ptl_gm_param_register_int( "eager_limit", 128 * 1024 );
#if OMPI_MCA_PTL_GM_HAVE_RDMA_GET
mca_ptl_gm_component.gm_rndv_burst_limit =
mca_ptl_gm_param_register_int( "rndv_burst_limit", 512 * 1024 );
mca_ptl_gm_component.gm_rdma_frag_size =
mca_ptl_gm_param_register_int ("rdma_frag_size", 128 * 1024);
#endif /* OMPI_MCA_PTL_GM_HAVE_RDMA_GET */
mca_ptl_gm_component.gm_free_list_num =
mca_ptl_gm_param_register_int ("free_list_num", 256);
mca_ptl_gm_component.gm_free_list_inc =
mca_ptl_gm_param_register_int ("free_list_inc", 32);
return OMPI_SUCCESS;
}
/*
* component close
*/
int mca_ptl_gm_component_close (void)
{
uint32_t index;
mca_ptl_base_module_t* ptl;
for( index = 0; index < mca_ptl_gm_component.gm_num_ptl_modules; index++ ) {
ptl = (mca_ptl_base_module_t*)mca_ptl_gm_component.gm_ptl_modules[index];
if( NULL != ptl )
ptl->ptl_finalize( ptl );
}
mca_ptl_gm_component.gm_num_ptl_modules = 0;
if (NULL != mca_ptl_gm_component.gm_ptl_modules)
free (mca_ptl_gm_component.gm_ptl_modules);
OBJ_DESTRUCT( &mca_ptl_gm_component.gm_procs );
OBJ_DESTRUCT( &mca_ptl_gm_component.gm_send_req );
OBJ_DESTRUCT( &mca_ptl_gm_component.gm_lock );
return OMPI_SUCCESS;
}
/*
* Create a ptl instance and add to components list.
*/
static int
mca_ptl_gm_create( mca_ptl_gm_module_t** pptl )
{
mca_ptl_gm_module_t *ptl;
ptl = (mca_ptl_gm_module_t *)malloc( sizeof(mca_ptl_gm_module_t) );
if (NULL == ptl) {
opal_output( 0, " ran out of resource to allocate ptl_instance \n" );
return OMPI_ERR_OUT_OF_RESOURCE;
}
/* copy the basic informations in the new PTL */
memcpy (ptl, &mca_ptl_gm_module, sizeof(mca_ptl_gm_module_t) );
#if OMPI_HAVE_POSIX_THREADS
ptl->thread.t_handle = (pthread_t)-1;
#endif /* OMPI_HAVE_POSIX_THREADS */
*pptl = ptl;
return OMPI_SUCCESS;
}
/*
* Register GM component addressing information. The MCA framework
* will make this available to all peers.
*/
static int
mca_ptl_gm_module_store_data_toexchange (void)
{
int rc;
size_t i;
size_t size;
mca_ptl_gm_addr_t *addrs;
size = mca_ptl_gm_component.gm_num_ptl_modules * sizeof (mca_ptl_gm_addr_t);
addrs = (mca_ptl_gm_addr_t *)malloc (size);
if (NULL == addrs) {
return OMPI_ERR_OUT_OF_RESOURCE;
}
for (i = 0; i < mca_ptl_gm_component.gm_num_ptl_modules; i++) {
mca_ptl_gm_module_t *ptl = mca_ptl_gm_component.gm_ptl_modules[i];
addrs[i].local_id = ptl->local_addr.local_id;
#if GM_API_VERSION > 0x200
addrs[i].global_id = ptl->local_addr.global_id;
#else
strncpy( addrs[i].global_id, ptl->local_addr.global_id, GM_MAX_HOST_NAME_LEN );
#endif /* GM_API_VERSION > 0x200 */
addrs[i].port_id = ptl->local_addr.port_id;
}
rc = mca_pml_base_modex_send (&mca_ptl_gm_component.super.ptlm_version, addrs, size);
free (addrs);
return rc;
}
#if OMPI_HAVE_POSIX_THREADS
static void*
mca_ptl_gm_thread_progress( opal_thread_t* thread )
{
gm_recv_event_t *event;
mca_ptl_gm_module_t* ptl = thread->t_arg;
/* This thread enter in a cancel enabled state */
pthread_setcancelstate( PTHREAD_CANCEL_ENABLE, NULL );
pthread_setcanceltype( PTHREAD_CANCEL_ASYNCHRONOUS, NULL );
while(1) {
event = gm_blocking_receive(ptl->gm_port);
if( GM_NO_RECV_EVENT != gm_ntohc(event->recv.type) )
mca_ptl_gm_analyze_recv_event( ptl, event );
}
return PTHREAD_CANCELED;
}
#endif /* OMPI_HAVE_POSIX_THREADS */
/* Scan all ports on the boards. As it's difficult to find the total number of boards
* we use a predefined maximum.
* Return the number of discovered boards where opening a port was a succesfull operation.
*/
static int32_t
mca_ptl_gm_discover_boards( mca_ptl_gm_module_t** pptl,
uint32_t max_ptls, uint32_t max_boards, uint32_t max_port )
{
uint32_t board_no, port_no, index = 0, local_id;
struct gm_port* gm_port;
#if GM_API_VERSION > 0x200
uint32_t global_id;
#else
char global_id[GM_MAX_HOST_NAME_LEN];
#endif /* GM_API_VERSION > 0x200 */
for( board_no = 0; board_no < max_boards; board_no++ ) {
/* open the first available gm port for this board */
for( port_no = 2; port_no < max_port; port_no++ ) {
if (3 == port_no) {
continue; /* port 0,1,3 reserved */
} else if (GM_SUCCESS ==
gm_open(&gm_port, board_no, port_no,
mca_ptl_gm_component.gm_port_name,
OMPI_MCA_PTL_GM_API_VERSION) ) {
break;
}
}
if( port_no == max_port ) {
continue;
}
/* Get node local Id */
if( GM_SUCCESS != gm_get_node_id( gm_port, &local_id) ) {
opal_output (0, " failure to get local_id \n");
continue;
}
/* Gather an unique id for the node */
#if GM_API_VERSION > 0x200
if (GM_SUCCESS != gm_node_id_to_global_id( gm_port, local_id, &global_id) ) {
opal_output (0, " Error: Unable to get my GM global unique id \n");
continue;
}
#else
{
if( GM_SUCCESS != gm_get_host_name( gm_port, global_id ) ) {
opal_output( 0, "Error: Unable to get the GM host name\n" );
continue;
}
}
#endif /* GM_API_VERSION > 0x200 */
/* Create the ptl. If fail return the number of already created */
if( OMPI_SUCCESS != mca_ptl_gm_create( &(pptl[index]) ) ) {
return index;
}
pptl[index]->gm_port = gm_port;
pptl[index]->local_addr.port_id = port_no;
pptl[index]->local_addr.local_id = local_id;
#if GM_API_VERSION > 0x200
pptl[index]->local_addr.global_id = global_id;
#else
strncpy( pptl[index]->local_addr.global_id, global_id, GM_MAX_HOST_NAME_LEN );
#endif /* GM_API_VERSION > 0x200 */
/* everything is OK let's mark it as usable and go to the next one */
if( (++index) >= max_ptls ) {
break;
}
}
return index;
}
static int
mca_ptl_gm_init_sendrecv (mca_ptl_gm_module_t * ptl)
{
uint32_t i;
mca_ptl_gm_send_frag_t *sfragment;
mca_ptl_gm_recv_frag_t *free_rfragment;
ptl->num_send_tokens = gm_num_send_tokens (ptl->gm_port);
ptl->max_send_tokens = ptl->num_send_tokens;
ptl->num_send_tokens -= PTL_GM_ADMIN_SEND_TOKENS;
ptl->num_recv_tokens = gm_num_receive_tokens (ptl->gm_port);
ptl->max_recv_tokens = ptl->num_recv_tokens;
ptl->num_recv_tokens -= PTL_GM_ADMIN_RECV_TOKENS;
/****************SEND****************************/
/* construct a list of send fragments */
OBJ_CONSTRUCT (&(ptl->gm_send_frags), ompi_free_list_t);
OBJ_CONSTRUCT (&(ptl->gm_send_dma_frags), ompi_free_list_t);
OBJ_CONSTRUCT (&(ptl->gm_send_frags_queue), opal_list_t);
/* We need a free list just to handle the send fragment that we provide.
* Just to make sure that we dont waste memory, we dont allow this list to
* grow anymore.
*/
ompi_free_list_init( &(ptl->gm_send_frags),
sizeof (mca_ptl_gm_send_frag_t),
OBJ_CLASS (mca_ptl_gm_send_frag_t),
0, /* do not allocate any items I'll provide them */
0, /* maximum number of list allocated elements will be zero */
0,
NULL ); /* not using mpool */
/* A free list containing all DMA allocate memory.
* This free list does not have the right to allocate any new item
* as they should be allocated with a special GM function.
*/
ompi_free_list_init( &(ptl->gm_send_dma_frags),
mca_ptl_gm_component.gm_segment_size,
OBJ_CLASS (opal_list_item_t),
0, /* do not allocate any items I'll provide them */
0, /* maximum number of list allocated elements will be zero */
0,
NULL ); /* not using mpool */
/* allocate the elements */
sfragment = (mca_ptl_gm_send_frag_t *)calloc( ptl->num_send_tokens, sizeof(mca_ptl_gm_send_frag_t) );
ptl->gm_send_fragments = sfragment;
/* allocate the registered memory */
ptl->gm_send_dma_memory = gm_dma_malloc( ptl->gm_port,
(mca_ptl_gm_component.gm_segment_size * ptl->num_send_tokens) + GM_PAGE_LEN );
if( NULL == ptl->gm_send_dma_memory ) {
opal_output( 0, "unable to allocate registered memory\n" );
return OMPI_ERR_OUT_OF_RESOURCE;
}
for (i = 0; i < ptl->num_send_tokens; i++) {
sfragment->send_buf = NULL;
OMPI_GM_FREE_LIST_RETURN( &(ptl->gm_send_frags), (opal_list_item_t*)sfragment );
OMPI_GM_FREE_LIST_RETURN( &(ptl->gm_send_dma_frags),
(opal_list_item_t*)((char*)ptl->gm_send_dma_memory +
i * mca_ptl_gm_component.gm_segment_size) );
sfragment++;
}
/*****************RECEIVE*****************************/
/* allow remote memory access */
if( GM_SUCCESS != gm_allow_remote_memory_access (ptl->gm_port) ) {
opal_output (0, "unable to allow remote memory access\n");
}
OBJ_CONSTRUCT (&(ptl->gm_recv_outstanding_queue), opal_list_t);
/* construct the list of recv fragments free */
OBJ_CONSTRUCT (&(ptl->gm_recv_frags_free), ompi_free_list_t);
ompi_free_list_init( &(ptl->gm_recv_frags_free),
sizeof (mca_ptl_gm_recv_frag_t),
OBJ_CLASS (mca_ptl_gm_recv_frag_t),
0, /* by default I will provide all items */
ptl->num_recv_tokens * 10, /* the maximum number of items in the free list */
ptl->num_recv_tokens, /* if it need to allocate some more */
NULL );
/* allocate the elements */
free_rfragment = (mca_ptl_gm_recv_frag_t *)
calloc( ptl->num_recv_tokens, sizeof(mca_ptl_gm_recv_frag_t) );
ptl->gm_recv_fragments = free_rfragment;
/*allocate the registered memory */
ptl->gm_recv_dma_memory =
gm_dma_malloc( ptl->gm_port, (mca_ptl_gm_component.gm_segment_size * ptl->num_recv_tokens) + GM_PAGE_LEN );
if( NULL == ptl->gm_recv_dma_memory ) {
opal_output( 0, "unable to allocate registered memory for receive\n" );
return OMPI_ERR_OUT_OF_RESOURCE;
}
for( i = 0; i < 2; i++ ) {
OMPI_GM_FREE_LIST_RETURN( &(ptl->gm_recv_frags_free), (opal_list_item_t *)free_rfragment );
free_rfragment++;
gm_provide_receive_buffer( ptl->gm_port, (char*)ptl->gm_recv_dma_memory + i * mca_ptl_gm_component.gm_segment_size,
GM_SIZE, GM_HIGH_PRIORITY );
}
for( i = 2; i < ptl->num_recv_tokens; i++ ) {
OMPI_GM_FREE_LIST_RETURN( &(ptl->gm_recv_frags_free), (opal_list_item_t *)free_rfragment );
free_rfragment++;
gm_provide_receive_buffer( ptl->gm_port, (char*)ptl->gm_recv_dma_memory + i * mca_ptl_gm_component.gm_segment_size,
GM_SIZE, GM_LOW_PRIORITY );
}
OBJ_CONSTRUCT( &(ptl->gm_pending_acks), opal_list_t );
return OMPI_SUCCESS;
}
static int
mca_ptl_gm_init( mca_ptl_gm_component_t * gm )
{
uint32_t index;
mca_ptl_gm_module_t* ptl;
uint32_t save_counter;
/* let's try to find if GM is available */
if( GM_SUCCESS != gm_init() ) {
opal_output( 0, "[%s:%d] error in initializing the gm library\n", __FILE__, __LINE__ );
return OMPI_ERR_OUT_OF_RESOURCE;
}
/* First discover all available boards. For each board we will create a unique PTL */
mca_ptl_gm_component.gm_ptl_modules = calloc( mca_ptl_gm_component.gm_max_ptl_modules,
sizeof (mca_ptl_gm_module_t *));
if (NULL == mca_ptl_gm_component.gm_ptl_modules) {
opal_output( 0, "[%s:%d] error in initializing the gm PTL's.\n", __FILE__, __LINE__ );
return OMPI_ERR_OUT_OF_RESOURCE;
}
mca_ptl_gm_component.gm_num_ptl_modules =
mca_ptl_gm_discover_boards( mca_ptl_gm_component.gm_ptl_modules,
mca_ptl_gm_component.gm_max_ptl_modules,
mca_ptl_gm_component.gm_max_boards_number,
mca_ptl_gm_component.gm_max_port_number );
/* In the case when we are in a multi-threaded environment each
* PTL will have its own thread. At this point all structures are
* correctly initialized, each thread will grab one and use it.
*/
for( index = 0; index < mca_ptl_gm_component.gm_num_ptl_modules; index++ ) {
ptl = mca_ptl_gm_component.gm_ptl_modules[index];
/* Now prepost some received and allocate some sends. After
* this step the PTL is fully initialized.
*/
if( OMPI_SUCCESS != mca_ptl_gm_init_sendrecv( ptl ) )
break;
if( opal_using_threads() ) {
#if OMPI_HAVE_POSIX_THREADS
ptl->thread.t_run = (opal_thread_fn_t)mca_ptl_gm_thread_progress;
ptl->thread.t_arg = (void*)ptl;
#endif /* OMPI_HAVE_POSIX_THREADS */
if( OMPI_SUCCESS != opal_thread_start( &(ptl->thread) ) ) {
break;
}
}
}
save_counter = index;
/* If we are unable to start all the required threads we update the total
* number of threads and call finalize for the others PTLs.
*/
for( ; index < mca_ptl_gm_component.gm_num_ptl_modules; index++ ) {
mca_ptl_base_module_t* ptl = (mca_ptl_base_module_t*)mca_ptl_gm_component.gm_ptl_modules[index];
ptl->ptl_finalize( ptl );
}
mca_ptl_gm_component.gm_num_ptl_modules = save_counter;
/* A free list containing all memory used for keep data for unexpected requests. */
OBJ_CONSTRUCT( &(mca_ptl_gm_component.gm_unexpected_frags_data), ompi_free_list_t );
ompi_free_list_init( &(mca_ptl_gm_component.gm_unexpected_frags_data),
mca_ptl_gm_component.gm_segment_size,
OBJ_CLASS (opal_list_item_t),
16, /* keep is small in the begining */
128, /* maximum number of list elements */
16, /* Number of elements to grow by per allocation */
NULL ); /* not using mpool */
#if OMPI_MCA_PTL_GM_CACHE_ENABLE
mca_ptl_gm_regcache_init();
#endif /* OMPI_MCA_PTL_GM_CACHE_ENABLE */
return (mca_ptl_gm_component.gm_num_ptl_modules > 0 ? OMPI_SUCCESS : OMPI_ERR_OUT_OF_RESOURCE);
}
/*
* Initialize the GM component,
* check how many boards are available and open ports on them.
*/
mca_ptl_base_module_t **
mca_ptl_gm_component_init (int *num_ptl_modules,
bool enable_progress_threads,
bool enable_mpi_threads)
{
mca_ptl_base_module_t **ptls;
*num_ptl_modules = 0;
if (OMPI_SUCCESS != mca_ptl_gm_init (&mca_ptl_gm_component)) {
/*opal_output( 0, "[%s:%d] error in initializing gm state and PTL's. (%d PTL's)\n",
__FILE__, __LINE__, mca_ptl_gm_component.gm_num_ptl_modules );*/
return NULL;
}
/* publish GM parameters with the MCA framework */
if (OMPI_SUCCESS != mca_ptl_gm_module_store_data_toexchange ())
return 0;
/* return array of PTLs */
ptls = (mca_ptl_base_module_t**) malloc (
mca_ptl_gm_component.gm_num_ptl_modules * sizeof(mca_ptl_base_module_t *));
if (NULL == ptls) {
return NULL;
}
memcpy (ptls, mca_ptl_gm_component.gm_ptl_modules,
mca_ptl_gm_component.gm_num_ptl_modules * sizeof(mca_ptl_gm_module_t *));
*num_ptl_modules = mca_ptl_gm_component.gm_num_ptl_modules;
return ptls;
}
/*
* GM module control
*/
int
mca_ptl_gm_component_control (int param, void *value, size_t size)
{
return OMPI_SUCCESS;
}
char* mca_ptl_gm_get_local_buffer( void )
{
opal_list_item_t* item;
int rc;
OMPI_FREE_LIST_WAIT( &(mca_ptl_gm_component.gm_unexpected_frags_data), item, rc );
return (char*)item;
}
void mca_ptl_gm_release_local_buffer( char* ptr )
{
OMPI_GM_FREE_LIST_RETURN( &(mca_ptl_gm_component.gm_unexpected_frags_data), (opal_list_item_t*)ptr );
}
/*
* GM module progress.
*/
int
mca_ptl_gm_component_progress (mca_ptl_tstamp_t tstamp)
{
uint32_t i;
gm_recv_event_t *event;
mca_ptl_gm_module_t *ptl;
for( i = 0; i < mca_ptl_gm_component.gm_num_ptl_modules;) {
ptl = mca_ptl_gm_component.gm_ptl_modules[i];
event = gm_receive(ptl->gm_port);
/* If there are no receive events just skip the function call */
if( GM_NO_RECV_EVENT != gm_ntohc(event->recv.type) ) {
if( 1 == mca_ptl_gm_analyze_recv_event( ptl, event ) ) {
/* we try to empty the GM event queue */
continue;
}
}
i++;
}
return OMPI_SUCCESS;
}

52
ompi/mca/ptl/gm/ptl_gm_memory.c
Просмотреть файл

@ -1,52 +0,0 @@
/* -*- Mode: C; c-basic-offset:4 ; -*- */
/*
* Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
* University Research and Technology
* Corporation. All rights reserved.
* Copyright (c) 2004-2005 The University of Tennessee and The University
* of Tennessee Research Foundation. All rights
* reserved.
* Copyright (c) 2004 The Ohio State University.
* All rights reserved.
* Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
* University of Stuttgart. All rights reserved.
* Copyright (c) 2004-2005 The Regents of the University of California.
* All rights reserved.
* $COPYRIGHT$
*
* Additional copyrights may follow
*
* $HEADER$
*/
#include "ompi_config.h"
#include "ptl_gm.h"
#include "ptl_gm_priv.h"
gm_status_t mca_ptl_gm_register_memory(struct gm_port *port, void *ptr, unsigned len)
{
#if OMPI_MCA_PTL_GM_SUPPORT_REGISTERING
#if OMPI_MCA_PTL_GM_CACHE_ENABLE
gmpi_use_interval( port, (gm_up_t)ptr, len );
return GM_SUCCESS;
#else
return gm_register_memory( port, ptr, len );
#endif /* OMPI_MCA_PTL_GM_CACHE_ENABLE */
#else
return GM_FAILURE;
#endif /* OMPI_MCA_PTL_GM_SUPPORT_REGISTERING */
}
gm_status_t mca_ptl_gm_deregister_memory( struct gm_port *port, void *ptr, unsigned len )
{
#if OMPI_MCA_PTL_GM_SUPPORT_REGISTERING
#if OMPI_MCA_PTL_GM_CACHE_ENABLE
return gmpi_unuse_interval( port, (gm_up_t)ptr, len );
#else
return gm_deregister_memory( port, ptr, len );
#endif /* OMPI_MCA_PTL_GM_CACHE_ENABLE */
#else
return GM_FAILURE;
#endif /* OMPI_MCA_PTL_GM_SUPPORT_REGISTERING */
}

72
ompi/mca/ptl/gm/ptl_gm_peer.h
Просмотреть файл

@ -1,72 +0,0 @@
/* -*- Mode: C; c-basic-offset:4 ; -*- */
/*
* Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
* University Research and Technology
* Corporation. All rights reserved.
* Copyright (c) 2004-2005 The University of Tennessee and The University
* of Tennessee Research Foundation. All rights
* reserved.
* Copyright (c) 2004 The Ohio State University.
* All rights reserved.
* Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
* University of Stuttgart. All rights reserved.
* Copyright (c) 2004-2005 The Regents of the University of California.
* All rights reserved.
* $COPYRIGHT$
*
* Additional copyrights may follow
*
* $HEADER$
*/
/**
* @file
*/
#ifndef MCA_PTL_GM_PEER_H
#define MCA_PTL_GM_PEER_H
#include "opal/class/opal_list.h"
#include "ompi/types.h"
#if defined(c_plusplus) || defined(__cplusplus)
extern "C" {
#endif
/**
* Structure used to publish GM id information to peers.
*/
struct mca_ptl_gm_addr_t {
#if GM_API_VERSION > 0x200
unsigned int global_id;
#else
char global_id[GM_MAX_HOST_NAME_LEN];
#endif /* GM_API_VERSION > 0x200 */
unsigned int local_id;
unsigned int port_id;
};
typedef struct mca_ptl_gm_addr_t mca_ptl_gm_addr_t;
/**
* An abstraction that represents a connection to a peer process.
*/
struct mca_ptl_gm_peer_t {
opal_list_item_t super;
struct mca_ptl_gm_module_t* peer_ptl;
struct mca_ptl_gm_proc_t* peer_proc;
struct mca_ptl_gm_addr_t peer_addr; /**< address of peer */
int num_credits;
int max_credits;
int resending;
int num_resend;
bool get_started;
};
typedef struct mca_ptl_gm_peer_t mca_ptl_gm_peer_t;
OBJ_CLASS_DECLARATION(mca_ptl_gm_peer_t);
#if defined(c_plusplus) || defined(__cplusplus)
}
#endif
#endif

1005
ompi/mca/ptl/gm/ptl_gm_priv.c
Просмотреть файл

Разница между файлами не показана из-за своего большого размера Загрузить разницу

68
ompi/mca/ptl/gm/ptl_gm_priv.h
Просмотреть файл

@ -1,68 +0,0 @@
/* -*- Mode: C; c-basic-offset:4 ; -*- */
/*
* Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
* University Research and Technology
* Corporation. All rights reserved.
* Copyright (c) 2004-2005 The University of Tennessee and The University
* of Tennessee Research Foundation. All rights
* reserved.
* Copyright (c) 2004 The Ohio State University.
* All rights reserved.
* Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
* University of Stuttgart. All rights reserved.
* Copyright (c) 2004-2005 The Regents of the University of California.
* All rights reserved.
* $COPYRIGHT$
*
* Additional copyrights may follow
*
* $HEADER$
*/
#include "ompi_config.h"
#include "gm.h"
struct mca_ptl_gm_send_frag_t;
struct mca_ptl_gm_peer_t;
struct mca_ptl_gm_module_t;
/* Pinning down memory pages is a costly operation. We can avoid it by using a LRU list
* of pinned down memory, managed inside the GM PTL.
*/
gm_status_t mca_ptl_gm_register_memory( struct gm_port *port, void *ptr, unsigned len );
gm_status_t mca_ptl_gm_deregister_memory( struct gm_port *port, void *ptr, unsigned len );
#if OMPI_MCA_PTL_GM_CACHE_ENABLE
void mca_ptl_gm_regcache_init(void);
unsigned int gmpi_use_interval(struct gm_port *gmpi_gm_port, gm_up_t start, unsigned int length);
gm_status_t gmpi_unuse_interval(struct gm_port *gmpi_gm_port, gm_up_t start, unsigned int length);
void gmpi_clear_interval(struct gm_port *gmpi_gm_port, gm_up_t start, unsigned int length);
void gmpi_clear_all_intervals(void);
#endif /* OMPI_MCA_PTL_GM_CACHE_ENABLE */
/* Some flags that have to go in the header hdr_common.hdr_flags field */
#define PTL_FLAG_GM_HAS_FRAGMENT 0x04
#define PTL_FLAG_GM_LAST_FRAGMENT 0x08
#define PTL_FLAG_GM_REQUIRE_LOCK 0x10
/* Internal flags for handling long messages */
#define GM_PTL_REGISTER_MEMORY 0x01
#define GM_PTL_SEND_MESSAGE 0x02
int mca_ptl_gm_analyze_recv_event( struct mca_ptl_gm_module_t* ptl, gm_recv_event_t* event );
void mca_ptl_gm_outstanding_recv( struct mca_ptl_gm_module_t *ptl);
int mca_ptl_gm_peer_send( struct mca_ptl_base_module_t* ptl,
struct mca_ptl_base_peer_t* ptl_base_peer,
struct mca_ptl_base_send_request_t *sendreq,
size_t offset,
size_t size,
int flags );
int
mca_ptl_gm_peer_send_continue( struct mca_ptl_gm_peer_t *ptl_peer,
struct mca_ptl_gm_send_frag_t *fragment,
struct mca_ptl_base_send_request_t *sendreq,
size_t offset,
size_t *size,
int flags );

184
ompi/mca/ptl/gm/ptl_gm_proc.c
Просмотреть файл

@ -1,184 +0,0 @@
/* -*- Mode: C; c-basic-offset:4 ; -*- */
/*
* Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
* University Research and Technology
* Corporation. All rights reserved.
* Copyright (c) 2004-2005 The University of Tennessee and The University
* of Tennessee Research Foundation. All rights
* reserved.
* Copyright (c) 2004 The Ohio State University.
* All rights reserved.
* Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
* University of Stuttgart. All rights reserved.
* Copyright (c) 2004-2005 The Regents of the University of California.
* All rights reserved.
* $COPYRIGHT$
*
* Additional copyrights may follow
*
* $HEADER$
*/
#include "ompi_config.h"
#include <string.h>
#include "opal/sys/atomic.h"
#include "opal/class/opal_hash_table.h"
#include "ompi/mca/pml/base/pml_base_module_exchange.h"
#include "ompi/proc/proc.h"
#include "opal/util/output.h"
#include "ptl_gm.h"
#include "ptl_gm_peer.h"
#include "ptl_gm_proc.h"
#include "ptl_gm_priv.h"
static void mca_ptl_gm_proc_construct (mca_ptl_gm_proc_t * proc);
static void mca_ptl_gm_proc_destruct (mca_ptl_gm_proc_t * proc);
static mca_ptl_gm_proc_t *mca_ptl_gm_proc_lookup_ompi (ompi_proc_t *
ompi_proc);
opal_class_t mca_ptl_gm_proc_t_class = {
"mca_ptl_gm_proc_t",
OBJ_CLASS (opal_list_item_t),
(opal_construct_t) mca_ptl_gm_proc_construct,
(opal_destruct_t) mca_ptl_gm_proc_destruct
};
/**
* Initialize gm proc instance
*/
void
mca_ptl_gm_proc_construct (mca_ptl_gm_proc_t * proc)
{
proc->proc_ompi = NULL;
proc->proc_addrs = NULL;
proc->proc_addr_count = 0;
proc->peer_arr = NULL;
proc->proc_peer_count = 0;
OBJ_CONSTRUCT (&proc->proc_lock, opal_mutex_t);
/* add to list of all proc instance */
OPAL_THREAD_LOCK (&mca_ptl_gm_component.gm_lock);
opal_list_append (&mca_ptl_gm_component.gm_procs, &proc->super);
OPAL_THREAD_UNLOCK (&mca_ptl_gm_component.gm_lock);
return;
}
/*
* Cleanup gm proc instance
*/
void
mca_ptl_gm_proc_destruct (mca_ptl_gm_proc_t * proc)
{
/* remove from list of all proc instances */
OPAL_THREAD_LOCK (&mca_ptl_gm_component.gm_lock);
opal_list_remove_item (&mca_ptl_gm_component.gm_procs, &proc->super);
OPAL_THREAD_UNLOCK (&mca_ptl_gm_component.gm_lock);
/* release resources */
if (NULL != proc->peer_arr)
free (proc->peer_arr);
return;
}
/*
* Create a GM process structure. There is a one-to-one correspondence
* between a ompi_proc_t and a mca_ptl_gm_proc_t instance.
* We cache additional data (specifically the list
* of mca_ptl_gm_peer_t instances, and publiched
* addresses) associated w/ a given destination on this datastructure.
*/
mca_ptl_gm_proc_t *
mca_ptl_gm_proc_create (mca_ptl_gm_module_t * ptl, ompi_proc_t * ompi_proc)
{
int rc;
size_t size;
mca_ptl_gm_proc_t *ptl_proc;
ptl_proc = mca_ptl_gm_proc_lookup_ompi (ompi_proc);
if (ptl_proc != NULL)
{
return ptl_proc;
}
/* only gm ptl opened */
ptl_proc = OBJ_NEW (mca_ptl_gm_proc_t);
ptl_proc->proc_ompi = ompi_proc;
/* Extract exposed addresses from remote proc */
rc = mca_pml_base_modex_recv (&mca_ptl_gm_component.super.ptlm_version,
ompi_proc, (void **) &ptl_proc->proc_addrs,
&size);
if (rc != OMPI_SUCCESS) {
opal_output (0,
"[%s:%d] mca_pml_base_modex_recv failed to recv data \n",
__FILE__, __LINE__);
OBJ_RELEASE (ptl_proc);
return NULL;
}
if (0 != (size % sizeof (mca_ptl_gm_addr_t))) {
opal_output (0, "[%s:%d] invalid received data size %d\n",
__FILE__, __LINE__, size);
return NULL;
}
ptl_proc->proc_addr_count = size / sizeof (mca_ptl_gm_addr_t);
/* allocate space for peer array - one for each exported address */
ptl_proc->peer_arr = (mca_ptl_gm_peer_t **)
malloc (ptl_proc->proc_addr_count * sizeof (mca_ptl_gm_peer_t *));
if (NULL == ptl_proc->peer_arr) {
OBJ_RELEASE (ptl_proc);
opal_output (0, "[%s:%d] unable to allocate peer procs \n"
__FILE__, __LINE__);
return NULL;
}
if(NULL == mca_ptl_gm_component.gm_local &&
ompi_proc == ompi_proc_local() ) {
mca_ptl_gm_component.gm_local = ptl_proc;
}
return ptl_proc;
}
/*
* Look for an existing GM process instances based on the associated
* ompi_proc_t instance.
*/
static mca_ptl_gm_proc_t *
mca_ptl_gm_proc_lookup_ompi (ompi_proc_t * ompi_proc)
{
mca_ptl_gm_proc_t *gm_proc;
OPAL_THREAD_LOCK (&mca_ptl_gm_component.gm_lock);
gm_proc = (mca_ptl_gm_proc_t *)
opal_list_get_first (&mca_ptl_gm_component.gm_procs);
for (; gm_proc != (mca_ptl_gm_proc_t *)
opal_list_get_end (&mca_ptl_gm_component.gm_procs);
gm_proc = (mca_ptl_gm_proc_t *) opal_list_get_next (gm_proc)) {
if (gm_proc->proc_ompi == ompi_proc) {
OPAL_THREAD_UNLOCK (&mca_ptl_gm_component.gm_lock);
return gm_proc;
}
}
OPAL_THREAD_UNLOCK (&mca_ptl_gm_component.gm_lock);
return NULL;
}

57
ompi/mca/ptl/gm/ptl_gm_proc.h
Просмотреть файл

@ -1,57 +0,0 @@
/* -*- Mode: C; c-basic-offset:4 ; -*- */
/*
* Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
* University Research and Technology
* Corporation. All rights reserved.
* Copyright (c) 2004-2005 The University of Tennessee and The University
* of Tennessee Research Foundation. All rights
* reserved.
* Copyright (c) 2004 The Ohio State University.
* All rights reserved.
* Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
* University of Stuttgart. All rights reserved.
* Copyright (c) 2004-2005 The Regents of the University of California.
* All rights reserved.
* $COPYRIGHT$
*
* Additional copyrights may follow
*
* $HEADER$
*/
/**
* @file
*/
#ifndef MCA_PTL_GM_PROC_H
#define MCA_PTL_GM_PROC_H
#include "opal/class/opal_object.h"
#include "orte/mca/ns/ns_types.h"
#include "ptl_gm.h"
#if defined(c_plusplus) || defined(__cplusplus)
extern "C" {
#endif
extern opal_class_t mca_ptl_gm_proc_t_class;
struct mca_ptl_gm_proc_t {
opal_list_item_t super; /**< allow proc to be placed on a list */
struct ompi_proc_t *proc_ompi; /**< pointer to corresponding orte_process_name_t */
struct mca_ptl_gm_addr_t *proc_addrs; /**< array of addresses published by peer */
opal_mutex_t proc_lock; /**< lock to protect against concurrent access to proc state */
size_t proc_peer_count;
size_t proc_addr_count;
struct mca_ptl_gm_peer_t **peer_arr;
orte_process_name_t proc_guid;
};
typedef struct mca_ptl_gm_proc_t mca_ptl_gm_proc_t;
mca_ptl_gm_proc_t *mca_ptl_gm_proc_create (mca_ptl_gm_module_t * ptl,
struct ompi_proc_t* orte_proc);
mca_ptl_gm_proc_t *mca_ptl_gm_proc_lookup (void *guid, size_t size);
#if defined(c_plusplus) || defined(__cplusplus)
}
#endif
#endif

534
ompi/mca/ptl/gm/ptl_gm_regcache.c
Просмотреть файл

@ -1,534 +0,0 @@
/*************************************************************************
* Myricom MPICH-GM ch_gm backend *
* Copyright (c) 2001 by Myricom, Inc. *
* All rights reserved. *
*************************************************************************/
#include "ompi_config.h"
#include "ptl_gm.h"
#include "ptl_gm_priv.h"
#if OMPI_MCA_PTL_GM_CACHE_ENABLE
#include <stdio.h>
#include <assert.h>
#include <stdlib.h>
#define GM_DISABLE_REGISTRATION 0
#define GMPI_ENABLE_REG_CACHE 1
#define GMPI_DEBUG_REG_CACHE_PRINT0(A)
#define GMPI_DEBUG_REG_CACHE_PRINT1(A,B,C)
#define GMPI_DEBUG_REG_CACHE_PRINT2(A,B,C,D,E)
#define GMPI_DEBUG_REGISTRATION_USE_SEGMENT(ADDR, LENGTH)
#define GMPI_DEBUG_DMA_MEMORY_USE(SIZE)
#define GMPI_DEBUG_DMA_MEMORY_ACQUIRE(ADDR, LENGTH)
#define GMPI_DEBUG_DMA_MEMORY_RELEASE(ADDR, LENGTH)
#define GMPI_DEBUG_DMA_MEMORY_UNUSE(SIZE)
#define GMPI_DEBUG_REGISTRATION_CLEAR_SEGMENT(ADDR, LENGTH)
#define GMPI_DEBUG_REGISTRATION_UNUSE_SEGMENT(ADDR, LENGTH)
#define GMPI_DEBUG_REGISTRATION_CLEAR_ALL_SEGMENTS()
#define gmpi_debug_assert assert
#define gmpi_malloc_assert(A, B, C) assert( NULL != (A) )
#define gmpi_abort(ID) abort()
#if GM_DISABLE_REGISTRATION
void
gmpi_clear_interval(gm_up_t start, unsigned int length)
{
GMPI_DEBUG_REG_CACHE_PRINT2("Clear_interval", "start",
start, "length", length);
}
void
gmpi_clear_all_intervals(void)
{
GMPI_DEBUG_REG_CACHE_PRINT0("Clear_all_intervals");
}
#else
#if GMPI_ENABLE_REG_CACHE
typedef struct _entry
{
gm_up_t addr;
struct _entry * prev;
struct _entry * next;
unsigned int refcount;
} regcache_entry;
static struct gm_hash * regcache_hash = NULL;
static struct gm_lookaside * regcache_lookaside = NULL;
static regcache_entry * regcache_head = NULL;
static regcache_entry * regcache_tail = NULL;
void
mca_ptl_gm_regcache_init(void)
{
gmpi_debug_assert(GM_PAGE_LEN != 0);
if (regcache_hash == NULL)
{
regcache_hash = gm_create_hash(gm_hash_compare_ptrs,
gm_hash_hash_ptr, 0, 0,
4096, 0);
regcache_lookaside = gm_create_lookaside(sizeof(regcache_entry),
4096);
}
gmpi_malloc_assert(regcache_hash,
"mca_ptl_gm_regcache_init",
"gm_create_hash: regcache page hash");
gmpi_malloc_assert(regcache_lookaside,
"mca_ptl_gm_regcache_init",
"gm_create_lookaside: regcache entries list");
}
static void
gmpi_regcache_deregister(struct gm_port *gmpi_gm_port, void * addr, unsigned int pages)
{
if (pages > 0)
{
gm_deregister_memory(gmpi_gm_port, addr, GM_PAGE_LEN*pages);
GMPI_DEBUG_DMA_MEMORY_RELEASE((gm_up_t)addr,GM_PAGE_LEN*pages);
}
}
static void
gmpi_regcache_garbage_collector(struct gm_port *gmpi_gm_port, unsigned int required)
{
regcache_entry * entry_ptr, * next_entry;
unsigned int count = 0;
gm_up_t batch_addr = 0;
unsigned int batch_pages = 0;
GMPI_DEBUG_REG_CACHE_PRINT1("Garbage_collector start", "required", required);
entry_ptr = regcache_head;
while ((count < required) && (entry_ptr != NULL))
{
if (entry_ptr->refcount == 0)
{
gm_hash_remove(regcache_hash, (void *)entry_ptr->addr);
if (batch_addr == 0)
{
batch_addr = entry_ptr->addr;
batch_pages++;
}
else
{
if (entry_ptr->addr == batch_addr+batch_pages*GM_PAGE_LEN)
{
batch_pages++;
}
else
{
gmpi_regcache_deregister(gmpi_gm_port, (void *)batch_addr, batch_pages);
batch_addr = entry_ptr->addr;
batch_pages = 1;
}
}
count++;
next_entry = entry_ptr->next;
if (regcache_head == entry_ptr)
regcache_head = next_entry;
else
entry_ptr->prev->next = next_entry;
if (regcache_tail == entry_ptr)
regcache_tail = entry_ptr->prev;
else
entry_ptr->next->prev = entry_ptr->prev;
gm_lookaside_free(entry_ptr);
entry_ptr = next_entry;
}
else
{
entry_ptr = entry_ptr->next;
}
}
if (batch_addr)
{
gmpi_regcache_deregister(gmpi_gm_port, (void *)batch_addr, batch_pages);
}
GMPI_DEBUG_REG_CACHE_PRINT2("Garbage_collector stop", "required",
required, "count", count);
}
static unsigned int
gmpi_regcache_register(struct gm_port *gmpi_gm_port, void * addr, unsigned int pages)
{
unsigned int i;
regcache_entry * entry_ptr;
gm_status_t status;
GMPI_DEBUG_REG_CACHE_PRINT2("Regcache_register", "addr",
(gm_up_t)addr, "len", pages*GM_PAGE_LEN);
if (gm_register_memory(gmpi_gm_port, addr, GM_PAGE_LEN*pages) != GM_SUCCESS)
{
GMPI_DEBUG_REG_CACHE_PRINT0("Regcache_register - using GC");
gmpi_regcache_garbage_collector(gmpi_gm_port, 4096);
if (gm_register_memory(gmpi_gm_port, addr, GM_PAGE_LEN*pages)
!= GM_SUCCESS)
{
GMPI_DEBUG_REG_CACHE_PRINT2("Register_memory failed", "start",
addr, "length", GM_PAGE_LEN*pages);
return 0;
}
}
GMPI_DEBUG_DMA_MEMORY_ACQUIRE((gm_up_t)addr,GM_PAGE_LEN*pages);
for (i=0; i<pages; i++)
{
entry_ptr = (regcache_entry *)gm_lookaside_alloc(regcache_lookaside);
gmpi_malloc_assert(entry_ptr,
"gmpi_regcache_register",
"gm_lookaside_alloc: regcache entry");
if (regcache_head == NULL)
{
regcache_head = entry_ptr;
}
else
{
regcache_tail->next = entry_ptr;
}
entry_ptr->prev = regcache_tail;
entry_ptr->next = NULL;
regcache_tail = entry_ptr;
entry_ptr->refcount = 1;
GMPI_DEBUG_DMA_MEMORY_USE(GM_PAGE_LEN);
entry_ptr->addr = (gm_up_t)addr + i*GM_PAGE_LEN;
status = gm_hash_insert(regcache_hash,
(void *)(entry_ptr->addr),
(void *)(entry_ptr));
if (status != GM_SUCCESS)
{
fprintf(stderr, "[%d]: gm_hash_insert failure in "
"gmpi_regcache_register: out of memory\n", 0 );
gmpi_abort (0);
}
}
return 1;
}
unsigned int
gmpi_use_interval(struct gm_port *gmpi_gm_port, gm_up_t start, unsigned int length)
{
gm_up_t addr, end, batch_addr;
unsigned int batch_pages;
regcache_entry * entry_ptr;
GMPI_DEBUG_REG_CACHE_PRINT2("Use_interval", "start",
start, "len", length);
if (length == 0)
{
return 0;
}
GMPI_DEBUG_REGISTRATION_USE_SEGMENT(start, length);
addr = start & ~(GM_PAGE_LEN-1);
end = start + length;
batch_addr = 0;
batch_pages = 0;
while (addr < end)
{
entry_ptr = (regcache_entry *)gm_hash_find(regcache_hash, (void *)addr);
if (entry_ptr == NULL)
{
if (batch_addr == 0)
{
batch_addr = addr;
}
batch_pages++;
}
else
{
if (entry_ptr->refcount == 0)
{
GMPI_DEBUG_DMA_MEMORY_USE(GM_PAGE_LEN);
}
entry_ptr->refcount++;
if (batch_addr != 0)
{
GMPI_DEBUG_REG_CACHE_PRINT2("Use_interval batch", "batch_addr",
batch_addr, "batch_pages",
batch_pages);
if (gmpi_regcache_register(gmpi_gm_port, (void *)batch_addr, batch_pages) == 0)
{
entry_ptr->refcount--;
if (entry_ptr->refcount == 0)
{
GMPI_DEBUG_DMA_MEMORY_UNUSE(GM_PAGE_LEN);
}
if (batch_addr > start)
{
return (batch_addr-start);
}
else
{
return 0;
}
}
batch_addr = 0;
batch_pages = 0;
/* move the entry to the end of the list (LRU policy) */
if (entry_ptr != regcache_tail)
{
if (entry_ptr == regcache_head)
{
gmpi_debug_assert(entry_ptr->next != NULL);
entry_ptr->next->prev = NULL;
regcache_head = entry_ptr->next;
}
else
{
gmpi_debug_assert(entry_ptr->prev != NULL);
gmpi_debug_assert(entry_ptr->next != NULL);
entry_ptr->prev->next = entry_ptr->next;
entry_ptr->next->prev = entry_ptr->prev;
}
entry_ptr->next = NULL;
entry_ptr->prev = regcache_tail;
regcache_tail->next = entry_ptr;
regcache_tail = entry_ptr;
}
}
}
addr += GM_PAGE_LEN;
}
if (batch_addr != 0)
{
if (gmpi_regcache_register(gmpi_gm_port, (void *)batch_addr, batch_pages) == 0)
{
if (batch_addr > start)
{
return (batch_addr-start);
}
else
{
return 0;
}
}
}
return length;
}
gm_status_t
gmpi_unuse_interval(struct gm_port *gmpi_gm_port, gm_up_t start, unsigned int length)
{
gm_up_t addr, end;
regcache_entry * entry_ptr;
GMPI_DEBUG_REG_CACHE_PRINT2("Unuse_interval", "start",
start, "length", length);
if (length == 0)
{
return GM_SUCCESS;
}
GMPI_DEBUG_REGISTRATION_UNUSE_SEGMENT(start, length);
addr = start & ~(GM_PAGE_LEN-1);
end = start + length;
while (addr < end)
{
entry_ptr = (regcache_entry *)gm_hash_find(regcache_hash, (void *)addr);
gmpi_debug_assert(entry_ptr != NULL);
gmpi_debug_assert(entry_ptr->refcount > 0);
entry_ptr->refcount--;
if (entry_ptr->refcount == 0)
{
GMPI_DEBUG_DMA_MEMORY_UNUSE(GM_PAGE_LEN);
}
addr += GM_PAGE_LEN;
}
return GM_SUCCESS;
}
void
gmpi_clear_interval(struct gm_port *gmpi_gm_port, gm_up_t start, unsigned int length)
{
gm_up_t addr, end, batch_addr;
unsigned int batch_pages;
regcache_entry * entry_ptr;
GMPI_DEBUG_REG_CACHE_PRINT2("Clear_interval", "start",
start, "length", length);
GMPI_DEBUG_REGISTRATION_CLEAR_SEGMENT(start, length);
if (regcache_hash != NULL)
{
addr = start & ~(GM_PAGE_LEN-1);
end = start + length;
batch_addr = 0;
batch_pages = 0;
while (addr < end)
{
entry_ptr = (regcache_entry *)gm_hash_find(regcache_hash,
(void *)addr);
if (entry_ptr != NULL)
{
gmpi_debug_assert(entry_ptr->refcount == 0);
if (entry_ptr->refcount > 0)
{
GMPI_DEBUG_DMA_MEMORY_UNUSE(GM_PAGE_LEN);
}
gm_hash_remove(regcache_hash, (void *)addr);
if (batch_addr == 0)
batch_addr = addr;
batch_pages++;
if (regcache_head == entry_ptr)
regcache_head = entry_ptr->next;
else
entry_ptr->prev->next = entry_ptr->next;
if (regcache_tail == entry_ptr)
regcache_tail = entry_ptr->prev;
else
entry_ptr->next->prev = entry_ptr->prev;
gm_lookaside_free(entry_ptr);
}
else
{
if (batch_addr != 0)
{
gmpi_regcache_deregister(gmpi_gm_port, (void *)batch_addr, batch_pages);
batch_addr = 0;
batch_pages = 0;
}
}
addr += GM_PAGE_LEN;
}
if (batch_addr != 0)
gmpi_regcache_deregister(gmpi_gm_port, (void *)batch_addr, batch_pages);
}
}
void
gmpi_clear_all_intervals(void)
{
struct gm_hash *old_regcache_hash;
GMPI_DEBUG_REG_CACHE_PRINT0("Clear_all_intervals");
GMPI_DEBUG_REGISTRATION_CLEAR_ALL_SEGMENTS();
if (regcache_hash != NULL)
{
old_regcache_hash = regcache_hash;
regcache_hash = NULL;
gm_destroy_hash (old_regcache_hash);
gm_destroy_lookaside (regcache_lookaside);
}
}
#else /* NO_REG_CACHE */
unsigned int
gmpi_use_interval(struct gm_port *gmpi_gm_port, gm_up_t start, unsigned int length)
{
gm_status_c cc;
GMPI_DEBUG_REG_CACHE_PRINT2("Use_interval", "start",
start, "len", length);
if (length == 0)
{
return 0;
}
GMPI_DEBUG_REGISTRATION_USE_SEGMENT(start, length);
if ((cc = gm_register_memory(gmpi_gm_port, (void*)start, length)) != GM_SUCCESS)
{
GMPI_DEBUG_REG_CACHE_PRINT2("Use_interval no_regcache: register failed",
"start", start, "len", length);
return 0;
}
GMPI_DEBUG_DMA_MEMORY_USE(length);
GMPI_DEBUG_DMA_MEMORY_ACQUIRE(start,length);
return length;
}
gm_status_t
gmpi_unuse_interval(struct gm_port *gmpi_gm_port, gm_up_t start, unsigned int length)
{
gm_status_t cc;
GMPI_DEBUG_REG_CACHE_PRINT2("Unuse_interval", "start",
start, "length", length);
if (length == 0)
{
return;
}
GMPI_DEBUG_REGISTRATION_UNUSE_SEGMENT(start, length);
cc = gm_deregister_memory(gmpi_gm_port, (void*)start, length);
gmpi_debug_assert(cc == GM_SUCCESS);
GMPI_DEBUG_DMA_MEMORY_UNUSE(length);
GMPI_DEBUG_DMA_MEMORY_RELEASE(start,length);
return cc;
}
void
mca_ptl_gm_regcache_init(void)
{
;
}
void
gmpi_clear_interval(gm_up_t start, unsigned int length)
{
GMPI_DEBUG_REG_CACHE_PRINT2("Clear_interval", "start",
start, "length", length);
GMPI_DEBUG_REGISTRATION_CLEAR_SEGMENT(start, length);
}
void
gmpi_clear_all_intervals(void)
{
GMPI_DEBUG_REG_CACHE_PRINT0("Clear_all_intervals");
GMPI_DEBUG_REGISTRATION_CLEAR_ALL_SEGMENTS();
}
#endif
#endif
#endif /* OMPI_MCA_PTL_GM_CACHE_ENABLE */

113
ompi/mca/ptl/gm/ptl_gm_sendfrag.c
Просмотреть файл

@ -1,113 +0,0 @@
/* -*- Mode: C; c-basic-offset:4 ; -*- */
/*
* Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
* University Research and Technology
* Corporation. All rights reserved.
* Copyright (c) 2004-2005 The University of Tennessee and The University
* of Tennessee Research Foundation. All rights
* reserved.
* Copyright (c) 2004 The Ohio State University.
* All rights reserved.
* Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
* University of Stuttgart. All rights reserved.
* Copyright (c) 2004-2005 The Regents of the University of California.
* All rights reserved.
* $COPYRIGHT$
*
* Additional copyrights may follow
*
* $HEADER$
*/
#include "ompi_config.h"
#include "ompi/types.h"
#include "ompi/datatype/datatype.h"
#include "ptl_gm.h"
#include "ptl_gm_sendfrag.h"
#include "ptl_gm_priv.h"
/*
* send fragment constructor/destructors.
*/
OBJ_CLASS_INSTANCE(mca_ptl_gm_send_frag_t,
mca_ptl_base_send_frag_t,
NULL, NULL);
/* It's not yet clear for me what's the best solution here. Block
* until we get a free request or allocate a new one. The fist case
* allow us to never take care of the gm allocated DMA buffer as all
* send fragments already have one attached, but it can stop the
* application progression. The second case require special cases: we
* should set the data in the header inside the fragment and later
* when we get some free fragments with DMA memory attached we should
* put the header back there, and send it.
*
* I will implement the first case and add the second one in my TODO
* list.
*/
mca_ptl_gm_send_frag_t*
mca_ptl_gm_alloc_send_frag( struct mca_ptl_gm_module_t* ptl,
struct mca_ptl_base_send_request_t* sendreq )
{
opal_list_item_t* item;
mca_ptl_gm_send_frag_t* frag;
int32_t rc;
/* first get a gm_send_frag */
OMPI_FREE_LIST_GET( &(ptl->gm_send_frags), item, rc );
frag = (mca_ptl_gm_send_frag_t*)item;
/* And then get some DMA memory to put the data */
OMPI_FREE_LIST_WAIT( &(ptl->gm_send_dma_frags), item, rc );
opal_atomic_sub( &(ptl->num_send_tokens), 1 );
assert( ptl->num_send_tokens >= 0 );
frag->send_buf = (void*)item;
frag->frag_send.frag_request = sendreq;
frag->frag_send.frag_base.frag_owner = (struct mca_ptl_base_module_t*)ptl;
frag->frag_send.frag_base.frag_addr = sendreq->req_send.req_addr;
frag->frag_bytes_processed = 0;
frag->frag_bytes_validated = 0;
frag->status = -1;
frag->type = PUT;
ompi_ptl_gm_init_pipeline( &(frag->pipeline) );
return frag;
}
int mca_ptl_gm_put_frag_init( struct mca_ptl_gm_send_frag_t** putfrag,
struct mca_ptl_gm_peer_t* ptl_peer,
struct mca_ptl_gm_module_t* gm_ptl,
struct mca_ptl_base_send_request_t* sendreq,
size_t offset, size_t* size, int flags )
{
ompi_convertor_t* convertor;
mca_ptl_gm_send_frag_t* frag;
frag = mca_ptl_gm_alloc_send_frag( gm_ptl, sendreq ); /*alloc_put_frag */
frag->frag_send.frag_base.frag_peer = (struct mca_ptl_base_peer_t*)ptl_peer;
frag->frag_send.frag_base.frag_size = *size;
frag->frag_offset = offset;
if( (*size) > 0 ) {
convertor = &(frag->frag_send.frag_base.frag_convertor);
/* GM use the default parameters for the convertor without any special memory
* allocation function. We have to call the prepare_for_send in order to
* initialize the missing parameters of the convertor.
*/
ompi_convertor_clone_with_position( &(sendreq->req_send.req_convertor), convertor, 1,
&offset );
}
*putfrag = frag;
return OMPI_SUCCESS;
}
/*
* recv fragment constructor/destructors.
*/
OBJ_CLASS_INSTANCE(mca_ptl_gm_recv_frag_t,
mca_ptl_base_recv_frag_t,
NULL, NULL);

219
ompi/mca/ptl/gm/ptl_gm_sendfrag.h
Просмотреть файл

@ -1,219 +0,0 @@
/* -*- Mode: C; c-basic-offset:4 ; -*- */
/*
* Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
* University Research and Technology
* Corporation. All rights reserved.
* Copyright (c) 2004-2005 The University of Tennessee and The University
* of Tennessee Research Foundation. All rights
* reserved.
* Copyright (c) 2004 The Ohio State University.
* All rights reserved.
* Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
* University of Stuttgart. All rights reserved.
* Copyright (c) 2004-2005 The Regents of the University of California.
* All rights reserved.
* $COPYRIGHT$
*
* Additional copyrights may follow
*
* $HEADER$
*/
/**
* @file
*/
#ifndef MCA_PTL_GM_SEND_FRAG_H
#define MCA_PTL_GM_SEND_FRAG_H
#include "ompi/mca/ptl/base/ptl_base_sendreq.h"
#include "ompi/mca/ptl/base/ptl_base_sendfrag.h"
#include "ompi/mca/ptl/base/ptl_base_recvfrag.h"
#include "ompi/communicator/communicator.h"
#define MATCH 0
#define FRAG 1
#define ACK 2
#define PUT 3
/* depth of the GM internal pipeline */
#define GM_PIPELINE_DEPTH 3
#define PTL_GM_PIPELINE_EMPTY 0x0000
#define PTL_GM_PIPELINE_DEREGISTER 0x0001
#define PTL_GM_PIPELINE_REGISTER 0x0002
#define PTL_GM_PIPELINE_REMOTE 0x0004
#define PTL_GM_PIPELINE_TRANSFERT (PTL_GM_PIPELINE_REGISTER | PTL_GM_PIPELINE_REMOTE)
#define PTL_GM_PIPELINE_HAS_INTERNAL_BUFFERS 0x0008
#if defined(c_plusplus) || defined(__cplusplus)
extern "C" {
#endif
OBJ_CLASS_DECLARATION (mca_ptl_gm_send_frag_t);
OBJ_CLASS_DECLARATION (mca_ptl_gm_recv_frag_t);
/* specific header for GM rendezvous protocol. It will be filled up by the sender
* and should be able to hold a pointer to the last registered memory location.
*/
struct mca_ptl_gm_frag_header_t {
mca_ptl_base_frag_header_t hdr_frag;
ompi_ptr_t registered_memory;
};
typedef struct mca_ptl_gm_frag_header_t mca_ptl_gm_frag_header_t;
struct mca_ptl_gm_pipeline_line_t {
uint16_t flags;
uint16_t hdr_flags;
uint64_t length;
uint64_t offset;
ompi_ptr_t local_memory;
ompi_ptr_t remote_memory;
};
typedef struct mca_ptl_gm_pipeline_line_t mca_ptl_gm_pipeline_line_t;
struct mca_ptl_gm_pipeline_info_t {
mca_ptl_gm_pipeline_line_t lines[GM_PIPELINE_DEPTH];
uint32_t pos_register;
uint32_t pos_remote;
uint32_t pos_deregister;
uint32_t pos_transfert;
};
typedef struct mca_ptl_gm_pipeline_info_t mca_ptl_gm_pipeline_info_t;
struct mca_ptl_gm_peer_t;
/**
* GM send fragment derived type.
*/
struct mca_ptl_gm_send_frag_t {
mca_ptl_base_send_frag_t frag_send; /**< base send fragment descriptor */
void* send_buf;
ompi_ptr_t* registered_buf;
uint64_t frag_bytes_processed; /**< data sended so far */
uint64_t frag_bytes_validated; /**< amount of data for which we receive an ack */
uint64_t frag_offset; /**< initial offset of the fragment as specified by the upper level */
mca_ptl_gm_pipeline_info_t pipeline; /**< storing the information about the status
* of the pipeline for long messages. */
int status;
uint32_t type;
};
typedef struct mca_ptl_gm_send_frag_t mca_ptl_gm_send_frag_t;
struct mca_ptl_gm_recv_frag_t {
mca_ptl_base_recv_frag_t frag_recv;
uint64_t frag_bytes_processed;
uint64_t frag_bytes_validated; /**< amount of data for which we receive an ack */
uint64_t frag_offset;
mca_ptl_gm_pipeline_info_t pipeline; /**< storing the information about the status of
* the pipeline for long messages. */
uint32_t type;
bool matched;
bool have_allocated_buffer;
uint32_t attached_data_length;
};
typedef struct mca_ptl_gm_recv_frag_t mca_ptl_gm_recv_frag_t;
mca_ptl_gm_send_frag_t *
mca_ptl_gm_alloc_send_frag( struct mca_ptl_gm_module_t* ptl,
struct mca_ptl_base_send_request_t* sendreq );
int
mca_ptl_gm_put_frag_init( struct mca_ptl_gm_send_frag_t** sendfrag,
struct mca_ptl_gm_peer_t * ptl_peer,
struct mca_ptl_gm_module_t *ptl,
struct mca_ptl_base_send_request_t * sendreq,
size_t offset,
size_t* size,
int flags );
#define OMPI_FREE_LIST_TRY_GET(fl, item) \
{ \
item = NULL; \
if(opal_using_threads()) { \
if( opal_mutex_trylock( &((fl)->fl_lock)) ) { \
/* We get the lock. Now let's remove one of the elements */ \
item = opal_list_remove_first(&((fl)->super)); \
opal_mutex_unlock(&((fl)->fl_lock)); \
} \
} else { \
item = opal_list_remove_first(&((fl)->super)); \
} \
}
static inline int
mca_ptl_gm_init_header_rndv( mca_ptl_base_header_t *hdr,
struct mca_ptl_base_send_request_t * sendreq,
int flags )
{
hdr->hdr_common.hdr_flags = flags;
hdr->hdr_common.hdr_type = MCA_PTL_HDR_TYPE_RNDV;
hdr->hdr_rndv.hdr_match.hdr_contextid = sendreq->req_send.req_base.req_comm->c_contextid;
hdr->hdr_rndv.hdr_match.hdr_src = sendreq->req_send.req_base.req_comm->c_my_rank;
hdr->hdr_rndv.hdr_match.hdr_dst = sendreq->req_send.req_base.req_peer;
hdr->hdr_rndv.hdr_match.hdr_tag = sendreq->req_send.req_base.req_tag;
hdr->hdr_rndv.hdr_match.hdr_msg_length = sendreq->req_send.req_bytes_packed;
hdr->hdr_rndv.hdr_match.hdr_msg_seq = sendreq->req_send.req_base.req_sequence;
hdr->hdr_rndv.hdr_src_ptr.lval = 0L;
hdr->hdr_rndv.hdr_src_ptr.pval = sendreq;
return OMPI_SUCCESS;
}
static inline int
mca_ptl_gm_init_header_frag( struct mca_ptl_gm_send_frag_t* sendfrag,
struct mca_ptl_gm_peer_t * ptl_peer,
struct mca_ptl_base_send_request_t * sendreq,
size_t offset,
size_t* size,
int flags )
{
mca_ptl_base_header_t *hdr = (mca_ptl_base_header_t *)sendfrag->send_buf;
hdr->hdr_common.hdr_flags = flags;
hdr->hdr_common.hdr_type = MCA_PTL_HDR_TYPE_FRAG;
hdr->hdr_frag.hdr_frag_offset = offset;
hdr->hdr_frag.hdr_frag_length = *size;
hdr->hdr_frag.hdr_src_ptr.lval = 0;
hdr->hdr_frag.hdr_src_ptr.pval = sendfrag; /* pointer to the frag */
hdr->hdr_frag.hdr_dst_ptr = sendreq->req_peer_match;
sendfrag->type = FRAG;
return OMPI_SUCCESS;
}
static inline void ompi_ptl_gm_init_pipeline( mca_ptl_gm_pipeline_info_t* pipeline )
{
int i;
pipeline->pos_register = 0;
pipeline->pos_remote = 0;
pipeline->pos_deregister = 0;
pipeline->pos_transfert = 0;
for( i = 0; i < GM_PIPELINE_DEPTH; i++ )
pipeline->lines[i].flags = 0;
}
static inline mca_ptl_gm_recv_frag_t*
mca_ptl_gm_alloc_recv_frag( struct mca_ptl_base_module_t *ptl )
{
int rc;
opal_list_item_t* item;
mca_ptl_gm_recv_frag_t* frag;
OMPI_FREE_LIST_GET( &(((mca_ptl_gm_module_t *)ptl)->gm_recv_frags_free), item, rc );
frag = (mca_ptl_gm_recv_frag_t*)item;
frag->frag_recv.frag_base.frag_owner = (struct mca_ptl_base_module_t*)ptl;
frag->frag_bytes_processed = 0;
frag->frag_bytes_validated = 0;
frag->frag_offset = 0;
ompi_ptl_gm_init_pipeline( &(frag->pipeline) );
return frag;
}
#if defined(c_plusplus) || defined(__cplusplus)
}
#endif
#endif

54
ompi/mca/ptl/mx/Makefile.am
Просмотреть файл

@ -1,54 +0,0 @@
#
# Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
# University Research and Technology
# Corporation. All rights reserved.
# Copyright (c) 2004-2005 The University of Tennessee and The University
# of Tennessee Research Foundation. All rights
# reserved.
# Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
# University of Stuttgart. All rights reserved.
# Copyright (c) 2004-2005 The Regents of the University of California.
# All rights reserved.
# $COPYRIGHT$
#
# Additional copyrights may follow
#
# $HEADER$
#
# Use the top-level Makefile.options
AM_CPPFLAGS = $(ptl_mx_CPPFLAGS)
# Make the output library in this directory, and name it either
# mca_<type>_<name>.la (for DSO builds) or libmca_<type>_<name>.la
# (for static builds).
if OMPI_BUILD_ptl_mx_DSO
component_noinst =
component_install = mca_ptl_mx.la
else
component_noinst = libmca_ptl_mx.la
component_install =
endif
ptl_mx_SOURCES = ptl_mx.c ptl_mx.h ptl_mx_component.c ptl_mx_module.c ptl_mx_module.h \
ptl_mx_peer.c ptl_mx_peer.h ptl_mx_proc.c ptl_mx_proc.h \
ptl_mx_recvfrag.c ptl_mx_recvfrag.h ptl_mx_sendfrag.c ptl_mx_sendfrag.h
mcacomponentdir = $(libdir)/openmpi
mcacomponent_LTLIBRARIES = $(component_install)
mca_ptl_mx_la_SOURCES = $(ptl_mx_SOURCES)
mca_ptl_mx_la_LIBADD = \
$(ptl_mx_LIBS) \
$(top_ompi_builddir)/ompi/libmpi.la \
$(top_ompi_builddir)/orte/liborte.la \
$(top_ompi_builddir)/opal/libopal.la
mca_ptl_mx_la_LDFLAGS = -module -avoid-version $(ptl_mx_LDFLAGS)
noinst_LTLIBRARIES = $(component_noinst)
libmca_ptl_mx_la_SOURCES = $(ptl_mx_SOURCES)
libmca_ptl_mx_la_LIBADD = $(ptl_mx_LIBS)
libmca_ptl_mx_la_LDFLAGS = -module -avoid-version $(ptl_mx_LDFLAGS)

69
ompi/mca/ptl/mx/configure.m4
Просмотреть файл

@ -1,69 +0,0 @@
# -*- shell-script -*-
#
# Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
# University Research and Technology
# Corporation. All rights reserved.
# Copyright (c) 2004-2005 The University of Tennessee and The University
# of Tennessee Research Foundation. All rights
# reserved.
# Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
# University of Stuttgart. All rights reserved.
# Copyright (c) 2004-2005 The Regents of the University of California.
# All rights reserved.
# $COPYRIGHT$
#
# Additional copyrights may follow
#
# $HEADER$
#
# MCA_ptl_mx_CONFIG([action-if-can-compile],
# [action-if-cant-compile])
# ------------------------------------------------
AC_DEFUN([MCA_ptl_mx_CONFIG],[
OMPI_CHECK_MX([ptl_mx],
[ptl_mx_happy="yes"],
[ptl_mx_happy="no"])
if [ test "$ptl_mx_happy" = "yes" ]; then
#
# Save a copy of the flags
#
ompi_check_mx_callback_CPPFLAGS="$CPPFLAGS"
ompi_check_mx_callback_LDFLAGS="$LDFLAGS"
ompi_check_mx_callback_LIBS="$LIBS"
#
# Set the value allowing MX compilation
#
CPPFLAGS="$CPPFLAGS $ptl_mx_CPPFLAGS"
LDFLAGS="$LDFLAGS $ptl_mx_LDFLAGS"
LIBS="$LIBS $ptl_mx_LIBS"
AC_MSG_CHECKING([for a MX version with mx_register_match_callback])
AC_TRY_COMPILE([#include <myriexpress.h>],
[mx_register_match_callback(0, 0, 0);],
[ptl_mx_happy="yes"],
[ptl_mx_happy="no"])
AC_MSG_RESULT([$ptl_mx_happy])
#
# Restore the original flags
#
CPPFLAGS="$ompi_check_mx_callback_CPPFLAGS"
LDFLAGS="$ompi_check_mx_callback_LDFLAGS"
LIBS="$ompi_check_mx_callback_LIBS"
fi
AS_IF([test "$ptl_mx_happy" = "yes"],
[ptl_mx_WRAPPER_EXTRA_LDFLAGS="$ptl_mx_LDFLAGS"
ptl_mx_WRAPPER_EXTRA_LIBS="$ptl_mx_LIBS"
$1],
[$2])
# substitute in the things needed to build mx
AC_SUBST([ptl_mx_CFLAGS])
AC_SUBST([ptl_mx_CPPFLAGS])
AC_SUBST([ptl_mx_LDFLAGS])
AC_SUBST([ptl_mx_LIBS])
])dnl

23
ompi/mca/ptl/mx/configure.params
Просмотреть файл

@ -1,23 +0,0 @@
# -*- shell-script -*-
#
# Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
# University Research and Technology
# Corporation. All rights reserved.
# Copyright (c) 2004-2005 The University of Tennessee and The University
# of Tennessee Research Foundation. All rights
# reserved.
# Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
# University of Stuttgart. All rights reserved.
# Copyright (c) 2004-2005 The Regents of the University of California.
# All rights reserved.
# $COPYRIGHT$
#
# Additional copyrights may follow
#
# $HEADER$
#
# Specific to this module
PARAM_INIT_FILE=ptl_mx.c
PARAM_CONFIG_FILES="Makefile"

499
ompi/mca/ptl/mx/ptl_mx.c
Просмотреть файл

@ -1,499 +0,0 @@
/*
* Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
* University Research and Technology
* Corporation. All rights reserved.
* Copyright (c) 2004-2005 The University of Tennessee and The University
* of Tennessee Research Foundation. All rights
* reserved.
* Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
* University of Stuttgart. All rights reserved.
* Copyright (c) 2004-2005 The Regents of the University of California.
* All rights reserved.
* $COPYRIGHT$
*
* Additional copyrights may follow
*
* $HEADER$
*/
#include "ompi_config.h"
#include <errno.h>
#include <unistd.h>
#include <string.h>
#include "ompi/constants.h"
#include "opal/util/argv.h"
#include "opal/util/output.h"
#include "ompi/mca/pml/pml.h"
#include "ompi/mca/ptl/ptl.h"
#include "ompi/communicator/communicator.h"
#include "ptl_mx.h"
#include "ptl_mx_peer.h"
#include "ptl_mx_sendfrag.h"
#include "ptl_mx_recvfrag.h"
mca_ptl_mx_module_t mca_ptl_mx_module = {
{
&mca_ptl_mx_component.super,
16, /* ptl_cache_size */
sizeof(mca_ptl_mx_send_frag_t), /* ptl_cache_bytes */
(32 * 1024) - sizeof(mca_ptl_base_header_t), /* ptl_frag_first_size */
0, /* ptl_frag_min_size */
-1, /* ptl_frag_max_size */
0, /* ptl_exclusivity */
0, /* ptl_latency */
0, /* ptl_bandwidth */
MCA_PTL_PUT, /* ptl flags */
mca_ptl_mx_add_procs,
mca_ptl_mx_del_procs,
mca_ptl_mx_finalize,
mca_ptl_mx_send,
mca_ptl_mx_send_continue,
NULL, /* get */
mca_ptl_mx_matched, /* matched */
mca_ptl_mx_request_init,
mca_ptl_mx_request_fini,
NULL, /* match */
NULL,
NULL
}
};
/**
* Allocate memory for use by the convert.
*/
static void *mca_ptl_mx_alloc( size_t *size, void* user )
{
return malloc(*size);
}
/**
* PML->PTL Initialize a send request for use by the PTL.
*
* @param ptl (IN) PTL instance
* @param request (IN) Pointer to allocated request.
*
* To reduce latency (number of required allocations), the PML allocates up
* to ptl_cache_bytes of additional space contigous w/ the base send request.
* This space may be used by the PTL for additional control information (e.g.
* first fragment descriptor).
*
* The ptl_request_init() function is called by the PML when requests are
* allocated to the PTLs cache. These requests will be cached by the PML
* on completion and re-used by the same PTL w/out additional calls to
* ptl_request_init().
*
* If the cache size is exceeded, the PML may pass requests to ptl_send/ptl_put
* that have been taken from the global pool and have not been initialized by the
* PTL. These requests will have the req_cached attribute set to false.
*
*/
int mca_ptl_mx_request_init(struct mca_ptl_base_module_t* ptl, mca_ptl_base_send_request_t* request)
{
OBJ_CONSTRUCT(request+1, mca_ptl_mx_send_frag_t);
return OMPI_SUCCESS;
}
/**
* PML->PTL Cleanup any resources that may have been associated with the
* request by the PTL.
*
* @param ptl (IN) PTL instance
* @param request (IN) Pointer to allocated request.
*
* The ptl_request_fini function is called when the PML removes a request
* from the PTLs cache (due to resource constraints). This routine provides
* the PTL the chance to cleanup/release any resources cached on the send
* descriptor by the PTL.
*/
void mca_ptl_mx_request_fini(struct mca_ptl_base_module_t* ptl, mca_ptl_base_send_request_t* request)
{
OBJ_DESTRUCT(request+1);
}
/**
* PML->PTL Initiate a send to the peer.
*
* @param ptl (IN) PTL instance
* @param ptl_base_peer (IN) PTL peer addressing
* @param request (IN) Send request
* @param offset Current offset into packed/contiguous buffer.
* @param size (IN) Number of bytes PML is requesting PTL to deliver,
* @param flags (IN) Flags that should be passed to the peer via the message header.
* @param request (OUT) OMPI_SUCCESS if the PTL was able to queue one or more fragments
*
* The PML implements a rendevouz protocol, with up to the PTL threshold
* (ptl_first_frag_size) bytes of the message sent in eager send mode. The ptl_send()
* function is called by the PML to initiate the send of the first message fragment.
*
* The PTL is responsible for updating the current data offset (req_offset) in the
* request to reflect the actual number of bytes fragmented. This may be less than
* the requested size, due to resource constraints or datatype alighnment/offset. If
* an acknowledgment is required, the MCA_PTL_FLAGS_ACK bit will be set in the
* flags parameter. In this case, the PTL should not call ptl_send_progress() function
* to indicate completion of the fragment until the ack is received. For all other
* fragments ptl_send_progress() may be called based on local completion semantics.
*/
int mca_ptl_mx_send(
struct mca_ptl_base_module_t* ptl,
struct mca_ptl_base_peer_t* ptl_peer,
struct mca_ptl_base_send_request_t* sendreq,
size_t offset,
size_t size,
int flags)
{
mca_ptl_mx_module_t* mx_ptl = (mca_ptl_mx_module_t*)ptl;
mca_ptl_mx_send_frag_t* sendfrag;
mca_ptl_base_header_t* hdr;
mx_segment_t *segments;
mx_return_t mx_return;
ompi_ptr_t match;
int rc;
if (sendreq->req_cached) {
sendfrag = (mca_ptl_mx_send_frag_t*)(sendreq+1);
} else {
opal_list_item_t* item;
OMPI_FREE_LIST_GET(&mca_ptl_mx_component.mx_send_frags, item, rc);
if(NULL == (sendfrag = (mca_ptl_mx_send_frag_t*)item))
return rc;
}
/* setup iovec */
sendfrag->frag_progress = 0;
sendfrag->frag_free = 0;
/* initialize convertor */
if(size > 0) {
ompi_convertor_t *convertor;
struct iovec iov;
uint32_t iov_count;
size_t max_data;
int rc;
convertor = &sendreq->req_send.req_convertor;
ompi_convertor_personalize( convertor, 0, &offset, mca_ptl_mx_alloc, NULL );
/* if data is contigous convertor will return an offset
* into users buffer - otherwise will return an allocated buffer
* that holds the packed data
*/
iov.iov_base = NULL;
iov.iov_len = size;
iov_count = 1;
max_data = size;
if((rc = ompi_convertor_pack( convertor,
&iov,
&iov_count,
&max_data,
&(sendfrag->frag_free))) < 0) {
return OMPI_ERROR;
}
sendfrag->frag_segments[1].segment_ptr = iov.iov_base;
sendfrag->frag_segments[1].segment_length = iov.iov_len;
sendfrag->frag_send.frag_base.frag_addr = iov.iov_base;
sendfrag->frag_send.frag_base.frag_size = iov.iov_len;
} else {
sendfrag->frag_send.frag_base.frag_addr = NULL;
sendfrag->frag_send.frag_base.frag_size = 0;
}
/* setup message header */
hdr = &sendfrag->frag_send.frag_base.frag_header;
/* first fragment - need to try and match at the receiver */
if(offset == 0) {
hdr->hdr_common.hdr_flags = flags;
hdr->hdr_match.hdr_contextid = sendreq->req_send.req_base.req_comm->c_contextid;
hdr->hdr_match.hdr_src = sendreq->req_send.req_base.req_comm->c_my_rank;
hdr->hdr_match.hdr_dst = sendreq->req_send.req_base.req_peer;
hdr->hdr_match.hdr_tag = sendreq->req_send.req_base.req_tag;
hdr->hdr_match.hdr_msg_length = sendreq->req_send.req_bytes_packed;
hdr->hdr_match.hdr_msg_seq = sendreq->req_send.req_base.req_sequence;
/* for the first 32K - send header for matching + data */
segments = sendfrag->frag_segments;
if(sendfrag->frag_send.frag_base.frag_size > 0) {
sendfrag->frag_segment_count = 2;
} else {
sendfrag->frag_segment_count = 1;
}
/* if an acknoweldgment is not required - can get by with a shorter header */
if((flags & MCA_PTL_FLAGS_ACK) == 0) {
hdr->hdr_common.hdr_type = MCA_PTL_HDR_TYPE_MATCH;
sendfrag->frag_segments[0].segment_length = sizeof(mca_ptl_base_match_header_t);
match.lval = MCA_PTL_HDR_TYPE_MATCH;
/* convert header to network byte order if required */
if(ptl_peer->peer_nbo) {
hdr->hdr_common.hdr_flags |= MCA_PTL_FLAGS_NBO;
MCA_PTL_BASE_MATCH_HDR_HTON(hdr->hdr_match);
}
} else {
hdr->hdr_common.hdr_type = MCA_PTL_HDR_TYPE_RNDV;
hdr->hdr_rndv.hdr_frag_length = sendfrag->frag_send.frag_base.frag_size;
hdr->hdr_rndv.hdr_src_ptr.lval = 0; /* for VALGRIND/PURIFY - REPLACE WITH MACRO */
hdr->hdr_rndv.hdr_src_ptr.pval = sendfrag;
sendfrag->frag_segments[0].segment_length = sizeof(mca_ptl_base_rendezvous_header_t);
match.lval = MCA_PTL_HDR_TYPE_RNDV;
/* convert header to network byte order if required */
if(ptl_peer->peer_nbo) {
hdr->hdr_common.hdr_flags |= MCA_PTL_FLAGS_NBO;
MCA_PTL_BASE_RNDV_HDR_HTON(hdr->hdr_rndv);
}
}
/* non-zero offset - fragment of a previously started message */
} else {
hdr->hdr_common.hdr_type = MCA_PTL_HDR_TYPE_FRAG;
hdr->hdr_common.hdr_flags = flags;
hdr->hdr_frag.hdr_frag_offset = offset;
hdr->hdr_frag.hdr_frag_length = sendfrag->frag_send.frag_base.frag_size;
hdr->hdr_frag.hdr_src_ptr.lval = 0; /* for VALGRIND/PURIFY - REPLACE WITH MACRO */
hdr->hdr_frag.hdr_src_ptr.pval = sendfrag;
hdr->hdr_frag.hdr_dst_ptr = sendreq->req_peer_match;
match.sval.uval = sendreq->req_peer_match.ival;
match.sval.lval = offset;
/* dont send a header for after the first 32K - MX currently doesn't
* support DMA of more than one segment.
*/
segments = sendfrag->frag_segments+1;
sendfrag->frag_segment_count = 1;
/* convert header to network byte order if required */
if(ptl_peer->peer_nbo) {
hdr->hdr_common.hdr_flags |= MCA_PTL_FLAGS_NBO;
MCA_PTL_BASE_FRAG_HDR_HTON(hdr->hdr_frag);
}
}
/* fragment state */
sendfrag->frag_send.frag_base.frag_owner = &ptl_peer->peer_ptl->super;
sendfrag->frag_send.frag_request = sendreq;
sendfrag->frag_send.frag_base.frag_peer = ptl_peer;
/* must update the offset after actual fragment size is determined
* before attempting to send the fragment
*/
mca_ptl_base_send_request_offset(sendreq,
sendfrag->frag_send.frag_base.frag_size);
/* start the fragment */
mx_return = mx_isend( mx_ptl->mx_endpoint,
segments,
sendfrag->frag_segment_count,
ptl_peer->peer_addr,
match.lval,
sendfrag,
&sendfrag->frag_request );
if(mx_return != MX_SUCCESS) {
opal_output(0, "mca_ptl_mx_send: mx_isend() failed with return value=%d\n", mx_return);
return OMPI_ERROR;
}
return OMPI_SUCCESS;
}
/**
* PML->PTL Initiate a send to the peer.
*
* @param ptl (IN) PTL instance
* @param ptl_base_peer (IN) PTL peer addressing
* @param request (IN) Send request
* @param offset Current offset into packed/contiguous buffer.
* @param size (IN) Number of bytes PML is requesting PTL to deliver,
* @param flags (IN) Flags that should be passed to the peer via the message header.
* @param request (OUT) OMPI_SUCCESS if the PTL was able to queue one or more fragments
*
* Continue sending fragments of a large message to the peer.
*/
int mca_ptl_mx_send_continue(
struct mca_ptl_base_module_t* ptl,
struct mca_ptl_base_peer_t* ptl_peer,
struct mca_ptl_base_send_request_t* sendreq,
size_t offset,
size_t size,
int flags)
{
mca_ptl_mx_module_t* mx_ptl = (mca_ptl_mx_module_t*)ptl;
mca_ptl_mx_send_frag_t* sendfrag;
mx_return_t mx_return;
ompi_ptr_t match;
ompi_convertor_t *convertor;
struct iovec iov;
uint32_t iov_count;
size_t max_data;
int rc;
/* allocate fragment */
MCA_PTL_MX_SEND_FRAG_ALLOC(sendfrag, rc);
if(rc != OMPI_SUCCESS) {
return rc;
}
sendfrag->frag_free = 0;
/* initialize convertor */
convertor = &sendfrag->frag_send.frag_base.frag_convertor;
ompi_convertor_clone( &(sendreq->req_send.req_convertor), convertor, 1 );
ompi_convertor_personalize( convertor, 0, &offset, mca_ptl_mx_alloc, NULL );
/* if data is contigous convertor will return an offset
* into users buffer - otherwise will return an allocated buffer
* that holds the packed data
*/
iov.iov_base = NULL;
iov.iov_len = size;
iov_count = 1;
max_data = size;
if((rc = ompi_convertor_pack( convertor,
&iov,
&iov_count,
&max_data,
&(sendfrag->frag_free))) < 0) {
return OMPI_ERROR;
}
sendfrag->frag_segments[0].segment_length = sizeof(mca_ptl_base_frag_header_t);
sendfrag->frag_segments[1].segment_ptr = iov.iov_base;
sendfrag->frag_segments[1].segment_length = iov.iov_len;
sendfrag->frag_segment_count = 1;
sendfrag->frag_send.frag_base.frag_addr = iov.iov_base;
sendfrag->frag_send.frag_base.frag_header.hdr_common.hdr_type = MCA_PTL_HDR_TYPE_FRAG;
sendfrag->frag_send.frag_base.frag_header.hdr_common.hdr_flags = 0;
sendfrag->frag_send.frag_base.frag_header.hdr_frag.hdr_frag_length = iov.iov_len;
sendfrag->frag_send.frag_base.frag_header.hdr_frag.hdr_frag_offset = offset;
/* fragment state */
sendfrag->frag_send.frag_base.frag_owner = &ptl_peer->peer_ptl->super;
sendfrag->frag_send.frag_request = sendreq;
sendfrag->frag_send.frag_base.frag_size = size;
sendfrag->frag_send.frag_base.frag_peer = ptl_peer;
sendfrag->frag_progress = 0;
/* must update the offset after actual fragment size is determined
* before attempting to send the fragment
*/
mca_ptl_base_send_request_offset(sendreq, size);
/* start the fragment */
match.sval.uval = sendreq->req_peer_match.ival;
match.sval.lval = offset;
mx_return = mx_isend(
mx_ptl->mx_endpoint,
sendfrag->frag_segments+1,
1,
sendfrag->frag_send.frag_base.frag_peer->peer_addr,
match.lval,
sendfrag,
&sendfrag->frag_request);
if(mx_return != MX_SUCCESS) {
opal_output(0, "mca_ptl_mx_send: mx_isend() failed with return value=%d\n", mx_return);
return OMPI_ERROR;
}
return OMPI_SUCCESS;
}
/**
* PML->PTL Notification from the PML to the PTL that a receive
* has been posted and matched against the indicated fragment.
*
* @param ptl (IN) PTL instance
* @param recv_frag Matched fragment
*
* The ptl_matched() function is called by the PML when a fragment
* is matched to a posted receive. This may occur during a call to
* ptl_match() if the receive is matched, or at a later point in time
* when a matching receive is posted.
*
* When this routine is called, the PTL is responsible for generating
* an acknowledgment to the peer if the MCA_PTL_FLAGS_ACK
* bit is set in the original fragment header. Additionally, the PTL
* is responsible for transferring any data associated with the fragment
* into the users buffer utilizing the datatype engine, and notifying
* the PML that the fragment has completed via the ptl_recv_progress()
* function.
*/
void mca_ptl_mx_matched(
mca_ptl_base_module_t* ptl,
mca_ptl_base_recv_frag_t* frag)
{
mca_ptl_base_header_t* hdr = &frag->frag_base.frag_header;
mca_ptl_base_recv_request_t* request = frag->frag_request;
mca_ptl_mx_module_t* mx_ptl = (mca_ptl_mx_module_t*)ptl;
mca_ptl_mx_recv_frag_t* mx_frag = (mca_ptl_mx_recv_frag_t*)frag;
size_t bytes_delivered = mx_frag->frag_size;
bool ack_pending = false;
/* generate an acknowledgment if required */
if(hdr->hdr_common.hdr_flags & MCA_PTL_FLAGS_ACK) {
int rc;
mca_ptl_mx_send_frag_t* ack;
MCA_PTL_MX_SEND_FRAG_ALLOC(ack, rc);
if(NULL == ack) {
OPAL_THREAD_LOCK(&mca_ptl_mx_component.mx_lock);
ack_pending = true;
opal_list_append(&mca_ptl_mx_component.mx_pending_acks, (opal_list_item_t*)frag);
OPAL_THREAD_UNLOCK(&mca_ptl_mx_component.mx_lock);
} else {
mx_return_t mx_return;
MCA_PTL_MX_SEND_FRAG_INIT_ACK(ack, ptl, mx_frag);
if(hdr->hdr_common.hdr_flags & MCA_PTL_FLAGS_NBO) {
MCA_PTL_BASE_ACK_HDR_HTON(ack->frag_send.frag_base.frag_header.hdr_ack);
}
/* start the fragment */
mx_return = mx_isend(
mx_ptl->mx_endpoint,
ack->frag_segments,
ack->frag_segment_count,
ack->frag_send.frag_base.frag_peer->peer_addr,
MCA_PTL_HDR_TYPE_ACK,
ack,
&ack->frag_request);
if(mx_return != MX_SUCCESS) {
opal_output(0, "mca_ptl_mx_matched: mx_isend() failed with return value=%d\n", mx_return);
OPAL_THREAD_LOCK(&mca_ptl_mx_component.mx_lock);
ack_pending = true;
opal_list_append(&mca_ptl_mx_component.mx_pending_acks, (opal_list_item_t*)frag);
OPAL_THREAD_UNLOCK(&mca_ptl_mx_component.mx_lock);
}
}
}
/* copy data into users buffer */
if(mx_frag->frag_size > 0) {
struct iovec iov;
uint32_t iov_count = 1;
int free_after = 0;
ompi_convertor_t* convertor = &(request->req_recv.req_convertor);
/* we do not attach a memory allocation function so the personalization of
* the convertor is not necessary.
*/
iov.iov_base = mx_frag->frag_data;
iov.iov_len = mx_frag->frag_size;
ompi_convertor_unpack(convertor, &iov, &iov_count, &bytes_delivered, &free_after );
}
/* update request status */
ptl->ptl_recv_progress( ptl, request,
mx_frag->frag_size, bytes_delivered);
/* release resources */
if(ack_pending == false)
MCA_PTL_MX_RECV_FRAG_RETURN(mx_frag);
}

360
ompi/mca/ptl/mx/ptl_mx.h
Просмотреть файл

@ -1,360 +0,0 @@
/*
* Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
* University Research and Technology
* Corporation. All rights reserved.
* Copyright (c) 2004-2005 The University of Tennessee and The University
* of Tennessee Research Foundation. All rights
* reserved.
* Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
* University of Stuttgart. All rights reserved.
* Copyright (c) 2004-2005 The Regents of the University of California.
* All rights reserved.
* $COPYRIGHT$
*
* Additional copyrights may follow
*
* $HEADER$
*/
/**
* @file
*/
#ifndef MCA_PTL_MX_H
#define MCA_PTL_MX_H
#include "ompi_config.h"
#include <myriexpress.h>
#include "ompi/mca/pml/pml.h"
#include "ompi/mca/ptl/ptl.h"
#include "ompi/class/ompi_bitmap.h"
#include "ompi/class/ompi_free_list.h"
#include "orte/class/orte_proc_table.h"
#define MCA_PTL_MX_STATISTICS 0
/**
* Myricom MX PTL component.
*/
struct mca_ptl_mx_component_t {
mca_ptl_base_component_1_0_0_t super; /**< base PTL component */
int mx_free_list_num; /**< initial size of free lists */
int mx_free_list_max; /**< maximum size of free lists */
int mx_free_list_inc; /**< number of elements to growing free lists by */
int mx_prepost; /**< number of preposted recvs */
int mx_debug; /**< debug level */
uint32_t mx_filter; /**< filter assigned to application */
uint32_t mx_num_ptls; /**< number of MX NICs available to app */
uint32_t mx_max_ptls; /**< max number of MX NICs to use */
struct mca_ptl_mx_module_t** mx_ptls; /**< array of available PTL modules */
ompi_free_list_t mx_send_frags; /**< free list of mx send fragments */
ompi_free_list_t mx_recv_frags; /**< free list of mx recv fragments */
opal_hash_table_t mx_procs; /**< hash table of procs */
opal_list_t mx_pending_acks; /**< queue of pending sends */
opal_mutex_t mx_lock; /**< lock for accessing module state */
};
typedef struct mca_ptl_mx_component_t mca_ptl_mx_component_t;
struct mca_ptl_mx_recv_frag_t;
struct mca_ptl_mx_send_frag_t;
extern mca_ptl_mx_component_t mca_ptl_mx_component;
/**
* Register MX module parameters with the MCA framework
*/
extern int mca_ptl_mx_component_open(void);
/**
* Any final cleanup before being unloaded.
*/
extern int mca_ptl_mx_component_close(void);
/**
* MCA->PTL Intializes the PTL component and creates specific PTL
* module(s).
*
* @param num_ptls (OUT) Returns the number of ptl instances created, or 0
* if the transport is not available.
*
* @param allow_multi_user_threads (OUT) Indicated wether this component can
* run at MPI_THREAD_MULTIPLE or not.
*
* @param have_hidden_threads (OUT) Whether this component uses
* hidden threads (e.g., progress threads) or not.
*
* @return Array of pointers to PTL modules, or NULL if the transport
* is not available.
*
* During component initialization, the PTL component should discover
* the physical devices that are available for the given transport,
* and create a PTL instance to represent each device. Any addressing
* information required by peers to reach the device should be published
* during this function via the mca_pml_base_modex_send() interface.
*
*/
extern mca_ptl_base_module_t** mca_ptl_mx_component_init(
int *num_ptls,
bool allow_multi_user_threads,
bool have_hidden_threads );
/**
* MCA->PTL Called to dynamically change a component parameter.
*
* @param flag (IN) Parameter to change.
* @param value (IN) Optional parameter value.
*
* @return OMPI_SUCCESS or error code on failure.
*
* The only supported parameter is currently MCA_PTL_ENABLE,
* which can be used by the PML to enable/disable forwarding
* by the PTL.
*/
extern int mca_ptl_mx_component_control(
int param,
void* value,
size_t size
);
/**
* MCA->PTL Called to progress outstanding requests for
* non-threaded polling environments.
*
* @param tstamp Current time.
* @return OMPI_SUCCESS or error code on failure.
*/
extern int mca_ptl_mx_component_progress(
mca_ptl_tstamp_t tstamp
);
/**
* Myricom MX PTL module.
*/
struct mca_ptl_mx_module_t {
mca_ptl_base_module_t super; /**< base PTL module interface */
opal_list_t mx_peers; /**< list of peers */
bool mx_enabled; /**< flag to indicate if endpoint enabled */
mx_endpoint_t mx_endpoint; /**< endpoint */
mx_endpoint_addr_t mx_endpoint_addr; /**< endpoint address */
volatile int32_t mx_recvs_posted; /**< count of posted match fragments */
#if OMPI_ENABLE_PROGRESS_THREADS
opal_thread_t mx_thread; /**< thread for progressing outstanding requests */
#endif
};
typedef struct mca_ptl_mx_module_t mca_ptl_mx_module_t;
extern mca_ptl_mx_module_t mca_ptl_mx_module;
/**
* Create/initialize the MX PTL modules.
* @return OMPI_SUCCESS or error status on failure.
*/
extern int mca_ptl_mx_module_init(void);
/**
* Cleanup any resources held by the PTL.
*
* @param ptl PTL instance.
* @return OMPI_SUCCESS or error status on failure.
*/
extern int mca_ptl_mx_finalize(
struct mca_ptl_base_module_t* ptl
);
/**
* PML->PTL notification of change in the process list.
*
* @param ptl (IN) PTL instance
* @param nprocs (IN) Number of processes
* @param procs (IN) Set of processes
* @param peer (OUT) Set of (optional) mca_ptl_base_peer_t instances returned by PTL. * @param reachable (OUT) Bitmask indicating set of peer processes that are reachable by this PTL.
* @return OMPI_SUCCESS or error status on failure.
*
* The mca_ptl_base_module_add_procs_fn_t() is called by the PML to
* determine the set of PTLs that should be used to reach each process.
* Any addressing information exported by the peer via the mca_pml_base_modex_send()
* function should be available during this call via the corresponding
* mca_pml_base_modex_recv() function. The PTL may utilize this information to
* determine reachability of each peer process.
*
* For each process that is reachable by the PTL, the bit corresponding to the index
* into the proc array (nprocs) should be set in the reachable bitmask. The PML
* provides the PTL the option to return a pointer to a data structure defined
* by the PTL that is returned to the PTL on subsequent calls to the PTL data
* transfer functions (e.g ptl_send). This may be used by the PTL to cache any addressing
* or connection information (e.g. TCP socket, IP queue pair).
*/
extern int mca_ptl_mx_add_procs(
struct mca_ptl_base_module_t* ptl,
size_t nprocs,
struct ompi_proc_t **procs,
struct mca_ptl_base_peer_t** peers,
ompi_bitmap_t* reachable
);
/**
* PML->PTL notification of change to the process list.
*
* @param ptl (IN) PTL instance
* @param nprocs (IN) Number of processes
* @param proc (IN) Set of processes
* @param peer (IN) Set of peer addressing information.
* @return Status indicating if cleanup was successful
*
* When the process list changes, the PML notifies the PTL of the
* change, to provide the opportunity to cleanup or release any
* resources associated with the peer.
*/
extern int mca_ptl_mx_del_procs(
struct mca_ptl_base_module_t* ptl,
size_t nprocs,
struct ompi_proc_t **procs,
struct mca_ptl_base_peer_t** peers
);
/**
* PML->PTL Initialize a send request for use by the PTL.
*
* @param ptl (IN) PTL instance
* @param request (IN) Pointer to allocated request.
*
* To reduce latency (number of required allocations), the PML allocates up
* to ptl_cache_bytes of additional space contigous w/ the base send request.
* This space may be used by the PTL for additional control information (e.g.
* first fragment descriptor).
*
* The ptl_request_init() function is called by the PML when requests are
* allocated to the PTLs cache. These requests will be cached by the PML
* on completion and re-used by the same PTL w/out additional calls to
* ptl_request_init().
*
* If the cache size is exceeded, the PML may pass requests to ptl_send/ptl_put
* that have been taken from the global pool and have not been initialized by the
* PTL. These requests will have the req_cached attribute set to false.
*
*/
extern int mca_ptl_mx_request_init(
struct mca_ptl_base_module_t* ptl,
struct mca_ptl_base_send_request_t*
);
/**
* PML->PTL Cleanup any resources that may have been associated with the
* request by the PTL.
*
* @param ptl (IN) PTL instance
* @param request (IN) Pointer to allocated request.
*
* The ptl_request_fini function is called when the PML removes a request
* from the PTLs cache (due to resource constraints). This routine provides
* the PTL the chance to cleanup/release any resources cached on the send
* descriptor by the PTL.
*/
extern void mca_ptl_mx_request_fini(
struct mca_ptl_base_module_t* ptl,
struct mca_ptl_base_send_request_t*
);
/**
* PML->PTL Notification from the PML to the PTL that a receive
* has been posted and matched against the indicated fragment.
*
* @param ptl (IN) PTL instance
* @param recv_frag Matched fragment
*
* The ptl_matched() function is called by the PML when a fragment
* is matched to a posted receive. This may occur during a call to
* ptl_match() if the receive is matched, or at a later point in time
* when a matching receive is posted.
*
* When this routine is called, the PTL is responsible for generating
* an acknowledgment to the peer if the MCA_PTL_FLAGS_ACK
* bit is set in the original fragment header. Additionally, the PTL
* is responsible for transferring any data associated with the fragment
* into the users buffer utilizing the datatype engine, and notifying
* the PML that the fragment has completed via the ptl_recv_progress()
* function.
*/
extern void mca_ptl_mx_matched(
struct mca_ptl_base_module_t* ptl,
struct mca_ptl_base_recv_frag_t* frag
);
/**
* PML->PTL Initiate a send to the peer.
*
* @param ptl (IN) PTL instance
* @param ptl_base_peer (IN) PTL peer addressing
* @param request (IN) Send request
* @param offset Current offset into packed/contiguous buffer.
* @param size (IN) Number of bytes PML is requesting PTL to deliver,
* @param flags (IN) Flags that should be passed to the peer via the message header.
* @param request (OUT) OMPI_SUCCESS if the PTL was able to queue one or more fragments
*
* The PML implements a rendevouz protocol, with up to the PTL threshold
* (ptl_first_frag_size) bytes of the message sent in eager send mode. The ptl_send()
* function is called by the PML to initiate the send of the first message fragment.
*
* The PTL is responsible for updating the current data offset (req_offset) in the
* request to reflect the actual number of bytes fragmented. This may be less than
* the requested size, due to resource constraints or datatype alighnment/offset. If
* an acknowledgment is required, the MCA_PTL_FLAGS_ACK bit will be set in the
* flags parameter. In this case, the PTL should not call ptl_send_progress() function
* to indicate completion of the fragment until the ack is received. For all other
* fragments ptl_send_progress() may be called based on local completion semantics.
*/
extern int mca_ptl_mx_send(
struct mca_ptl_base_module_t* ptl,
struct mca_ptl_base_peer_t* ptl_peer,
struct mca_ptl_base_send_request_t*,
size_t offset,
size_t size,
int flags
);
/**
* PML->PTL Continue sending fragments of a large message.
*
* @param ptl (IN) PTL instance
* @param ptl_base_peer (IN) PTL peer addressing
* @param request (IN) Send request
* @param offset Current offset into packed/contiguous buffer.
* @param size (IN) Number of bytes PML is requesting PTL to deliver,
* @param flags (IN) Flags that should be passed to the peer via the message header.
* @param request (OUT) OMPI_SUCCESS if the PTL was able to queue one or more fragments
*
*/
extern int mca_ptl_mx_send_continue(
struct mca_ptl_base_module_t* ptl,
struct mca_ptl_base_peer_t* ptl_peer,
struct mca_ptl_base_send_request_t*,
size_t offset,
size_t size,
int flags
);
#define HAVE_MX_ICOMPLETED 0
#if HAVE_MX_ICOMPLETED
extern mx_return_t mx_icompleted(mx_endpoint_t endpoint, mx_status_t *status, uint32_t *result);
#endif
#endif

307
ompi/mca/ptl/mx/ptl_mx_component.c
Просмотреть файл

@ -1,307 +0,0 @@
/*
* Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
* University Research and Technology
* Corporation. All rights reserved.
* Copyright (c) 2004-2005 The University of Tennessee and The University
* of Tennessee Research Foundation. All rights
* reserved.
* Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
* University of Stuttgart. All rights reserved.
* Copyright (c) 2004-2005 The Regents of the University of California.
* All rights reserved.
* $COPYRIGHT$
*
* Additional copyrights may follow
*
* $HEADER$
*/
#include "ompi_config.h"
#include "ompi/constants.h"
#include "opal/util/output.h"
#include "opal/threads/threads.h"
#include "opal/mca/base/mca_base_param.h"
#include "ptl_mx.h"
#include "ptl_mx_module.h"
#include "ptl_mx_peer.h"
/*
* The MX component
*/
mca_ptl_mx_component_t mca_ptl_mx_component = {
{
/* First, the mca_base_module_t struct containing meta
information about the module itself */
{
/* Indicate that we are a pml v1.0.0 module (which also
implies a specific MCA version) */
MCA_PTL_BASE_VERSION_1_0_0,
"mx", /* MCA module name */
OMPI_MAJOR_VERSION, /* MCA module major version */
OMPI_MINOR_VERSION, /* MCA module minor version */
OMPI_RELEASE_VERSION, /* MCA module release version */
mca_ptl_mx_component_open, /* module open */
mca_ptl_mx_component_close /* module close */
},
/* Next the MCA v1.0.0 module meta data */
{
/* Whether the module is checkpointable or not */
false
},
mca_ptl_mx_component_init,
mca_ptl_mx_component_control,
mca_ptl_mx_component_progress,
}
};
/*
* utility routines for parameter registration
*/
static inline char* mca_ptl_mx_param_register_string(
const char* param_name,
const char* default_value)
{
char *param_value;
int id = mca_base_param_register_string("ptl","mx",param_name,NULL,default_value);
mca_base_param_lookup_string(id, &param_value);
return param_value;
}
static inline int mca_ptl_mx_param_register_int(
const char* param_name,
int default_value)
{
int id = mca_base_param_register_int("ptl","mx",param_name,NULL,default_value);
int param_value = default_value;
mca_base_param_lookup_int(id,&param_value);
return param_value;
}
/*
* Called by MCA framework to open the module, registers
* module parameters.
*/
int mca_ptl_mx_component_open(void)
{
/* initialize state */
mca_ptl_mx_component.mx_ptls = NULL;
mca_ptl_mx_component.mx_num_ptls = 0;
/* register MX module parameters */
mca_ptl_mx_component.mx_filter =
(uint32_t)mca_ptl_mx_param_register_int("filter", 0xdeadbeef);
mca_ptl_mx_component.mx_prepost =
mca_ptl_mx_param_register_int("prepost", 1);
mca_ptl_mx_component.mx_debug =
mca_ptl_mx_param_register_int("debug", 0);
mca_ptl_mx_component.mx_free_list_num =
mca_ptl_mx_param_register_int("free_list_num", 256);
mca_ptl_mx_component.mx_free_list_max =
mca_ptl_mx_param_register_int("free_list_max", -1);
mca_ptl_mx_component.mx_free_list_inc =
mca_ptl_mx_param_register_int("free_list_inc", 256);
mca_ptl_mx_component.mx_max_ptls =
(uint32_t)mca_ptl_mx_param_register_int("num_nics", -1);
mca_ptl_mx_module.super.ptl_exclusivity =
mca_ptl_mx_param_register_int("exclusivity", 0);
mca_ptl_mx_module.super.ptl_first_frag_size =
mca_ptl_mx_param_register_int("first_frag_size",
(32*1024) - sizeof(mca_ptl_base_header_t));
mca_ptl_mx_module.super.ptl_min_frag_size =
mca_ptl_mx_param_register_int("min_frag_size", 32*1024);
mca_ptl_mx_module.super.ptl_max_frag_size =
mca_ptl_mx_param_register_int("max_frag_size", -1);
return OMPI_SUCCESS;
}
/*
* module cleanup - sanity checking of queue lengths
*/
int mca_ptl_mx_component_close(void)
{
if( NULL == mca_ptl_mx_component.mx_ptls )
return OMPI_SUCCESS;
mx_finalize();
#if OMPI_ENABLE_DEBUG
if (mca_ptl_mx_component.mx_send_frags.fl_num_allocated &&
mca_ptl_mx_component.mx_send_frags.fl_num_allocated !=
mca_ptl_mx_component.mx_send_frags.super.opal_list_length) {
opal_output(0, "mx send frags: %d allocated %d returned\n",
mca_ptl_mx_component.mx_send_frags.fl_num_allocated,
mca_ptl_mx_component.mx_send_frags.super.opal_list_length);
}
/* allow for pre-posted receives */
if (mca_ptl_mx_component.mx_recv_frags.fl_num_allocated &&
mca_ptl_mx_component.mx_recv_frags.fl_num_allocated - 3 >
mca_ptl_mx_component.mx_recv_frags.super.opal_list_length) {
opal_output(0, "mx recv frags: %d allocated %d returned\n",
mca_ptl_mx_component.mx_recv_frags.fl_num_allocated,
mca_ptl_mx_component.mx_recv_frags.super.opal_list_length);
}
#endif
/* release resources */
OBJ_DESTRUCT(&mca_ptl_mx_component.mx_send_frags);
OBJ_DESTRUCT(&mca_ptl_mx_component.mx_recv_frags);
OBJ_DESTRUCT(&mca_ptl_mx_component.mx_procs);
OBJ_DESTRUCT(&mca_ptl_mx_component.mx_pending_acks);
OBJ_DESTRUCT(&mca_ptl_mx_component.mx_lock);
return OMPI_SUCCESS;
}
/*
* MX module initialization.
*/
mca_ptl_base_module_t** mca_ptl_mx_component_init(
int *num_ptls,
bool enable_progress_threads,
bool enable_mpi_threads)
{
mca_ptl_base_module_t** ptls;
*num_ptls = 0;
/* initialize objects */
OBJ_CONSTRUCT(&mca_ptl_mx_component.mx_send_frags, ompi_free_list_t);
OBJ_CONSTRUCT(&mca_ptl_mx_component.mx_recv_frags, ompi_free_list_t);
OBJ_CONSTRUCT(&mca_ptl_mx_component.mx_procs, opal_hash_table_t);
OBJ_CONSTRUCT(&mca_ptl_mx_component.mx_pending_acks, opal_list_t);
OBJ_CONSTRUCT(&mca_ptl_mx_component.mx_lock, opal_mutex_t);
ompi_free_list_init( &mca_ptl_mx_component.mx_send_frags,
sizeof(mca_ptl_mx_send_frag_t),
OBJ_CLASS(mca_ptl_mx_send_frag_t),
mca_ptl_mx_component.mx_free_list_num,
mca_ptl_mx_component.mx_free_list_max,
mca_ptl_mx_component.mx_free_list_inc,
NULL ); /* use default allocator */
ompi_free_list_init( &mca_ptl_mx_component.mx_recv_frags,
sizeof(mca_ptl_mx_recv_frag_t),
OBJ_CLASS(mca_ptl_mx_recv_frag_t),
mca_ptl_mx_component.mx_free_list_num,
mca_ptl_mx_component.mx_free_list_max,
mca_ptl_mx_component.mx_free_list_inc,
NULL ); /* use default allocator */
/* intialize process hash table */
opal_hash_table_init( &mca_ptl_mx_component.mx_procs, 256 );
/* initialize mx ptls */
if(OMPI_SUCCESS != mca_ptl_mx_module_init())
return NULL;
/* allocate and return a copy of the ptl array */
ptls = malloc( mca_ptl_mx_component.mx_num_ptls * sizeof(mca_ptl_base_module_t*) );
if(NULL == ptls)
return NULL;
memcpy( ptls, mca_ptl_mx_component.mx_ptls,
mca_ptl_mx_component.mx_num_ptls*sizeof(mca_ptl_mx_module_t*) );
*num_ptls = mca_ptl_mx_component.mx_num_ptls;
return ptls;
}
/*
* MX module control
*/
int mca_ptl_mx_component_control(int param, void* value, size_t size)
{
switch(param) {
case MCA_PTL_ENABLE:
if(*(int*)value) {
mca_ptl_mx_enable();
} else
mca_ptl_mx_disable();
break;
default:
break;
}
return OMPI_SUCCESS;
}
/*
* MX module progress.
*/
int mca_ptl_mx_component_progress(mca_ptl_tstamp_t tstamp)
{
int num_progressed = 0;
size_t i;
for(i=0; i<mca_ptl_mx_component.mx_num_ptls; i++) {
mca_ptl_mx_module_t* ptl = mca_ptl_mx_component.mx_ptls[i];
mx_status_t mx_status;
mx_return_t mx_return;
uint32_t mx_result = 0;
#if HAVE_MX_ICOMPLETED == 0
mx_request_t mx_request;
if(ptl->mx_recvs_posted == 0) {
OPAL_THREAD_ADD32(&ptl->mx_recvs_posted,1);
MCA_PTL_MX_POST(ptl,MCA_PTL_HDR_TYPE_MATCH,sizeof(mca_ptl_base_match_header_t));
}
mx_return = mx_ipeek(
ptl->mx_endpoint,
&mx_request,
&mx_result);
if(mx_return != MX_SUCCESS) {
opal_output(0, "mca_ptl_mx_component_progress: mx_ipeek() failed with status %d\n",
mx_return);
return OMPI_ERROR;
}
if(mx_result == 0) {
continue;
}
mx_return = mx_test(
ptl->mx_endpoint,
&mx_request,
&mx_status,
&mx_result);
if(mx_return == MX_SUCCESS) {
MCA_PTL_MX_PROGRESS(ptl, mx_status);
} else {
opal_output(0, "mca_ptl_mx_progress: mx_test() failed with status=%dn",
mx_return);
}
num_progressed++;
#else
/* pre-post receive */
if(ptl->mx_recvs_posted == 0) {
OPAL_THREAD_ADD32(&ptl->mx_recvs_posted,1);
MCA_PTL_MX_POST(ptl,MCA_PTL_HDR_TYPE_MATCH,sizeof(mca_ptl_base_match_header_t));
}
/* poll for completion */
mx_return = mx_icompleted(
ptl->mx_endpoint,
&mx_status,
&mx_result);
if(mx_return != MX_SUCCESS) {
opal_output(0, "mca_ptl_mx_component_progress: mx_ipeek() failed with status %d\n",
mx_return);
return OMPI_ERROR;
}
if(mx_result > 0) {
MCA_PTL_MX_PROGRESS(ptl, mx_status);
}
num_progressed++;
#endif
}
return num_progressed;
}

459
ompi/mca/ptl/mx/ptl_mx_module.c
Просмотреть файл

@ -1,459 +0,0 @@
/*
* Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
* University Research and Technology
* Corporation. All rights reserved.
* Copyright (c) 2004-2005 The University of Tennessee and The University
* of Tennessee Research Foundation. All rights
* reserved.
* Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
* University of Stuttgart. All rights reserved.
* Copyright (c) 2004-2005 The Regents of the University of California.
* All rights reserved.
* $COPYRIGHT$
*
* Additional copyrights may follow
*
* $HEADER$
*/
#include "ptl_mx.h"
#include "ompi/constants.h"
#include "ompi/mca/pml/base/pml_base_module_exchange.h"
#include "ompi/communicator/communicator.h"
#include "opal/util/output.h"
#include "ptl_mx_peer.h"
#include "ptl_mx_proc.h"
#include "ptl_mx_module.h"
static mca_ptl_mx_module_t* mca_ptl_mx_create(uint64_t addr);
static void* mca_ptl_mx_mem_alloc( size_t* size, void* userdata )
{
return malloc(*size);
}
/**
* Initialize MX PTL modules
*/
int mca_ptl_mx_module_init(void)
{
size_t size;
uint32_t i;
int rc;
uint64_t *nic_addrs;
mca_ptl_mx_endpoint_t *endpoint_addrs;
mx_return_t status;
/* intialize MX library */
if(MX_SUCCESS != (status = mx_init())) {
opal_output(0, "mca_ptl_mx_init: mx_init() failed with status=%d\n", status);
return OMPI_ERROR;
}
/* Do not abort on errors */
mx_set_error_handler(MX_ERRORS_RETURN);
/* determine the number of NICs */
if((status = mx_get_info( NULL, MX_NIC_COUNT, NULL, 0,
&mca_ptl_mx_component.mx_num_ptls, sizeof(uint32_t))) != MX_SUCCESS) {
opal_output(0, "mca_ptl_mx_init: mx_get_info(MX_NIC_COUNT) failed with status=%d\n", status);
return OMPI_ERROR;
}
/* determine the NIC ids */
size = sizeof(uint64_t) * (mca_ptl_mx_component.mx_num_ptls+1);
if(NULL == (nic_addrs = (uint64_t*)malloc(size)))
return OMPI_ERR_OUT_OF_RESOURCE;
if( (status = mx_get_info( NULL, MX_NIC_IDS, NULL, 0,
nic_addrs, size)) != MX_SUCCESS) {
free(nic_addrs);
return OMPI_ERROR;
}
/* check for limit on number of ptls */
if(mca_ptl_mx_component.mx_num_ptls > mca_ptl_mx_component.mx_max_ptls)
mca_ptl_mx_component.mx_num_ptls = mca_ptl_mx_component.mx_max_ptls;
/* allocate an array of pointers to ptls */
mca_ptl_mx_component.mx_ptls = (mca_ptl_mx_module_t**)malloc(
sizeof(mca_ptl_mx_module_t*) * mca_ptl_mx_component.mx_num_ptls);
if(NULL == mca_ptl_mx_component.mx_ptls) {
opal_output(0, "mca_ptl_mx_init: malloc() failed\n");
return OMPI_ERR_OUT_OF_RESOURCE;
}
/* create a ptl for each NIC */
for( i = 0; i < mca_ptl_mx_component.mx_num_ptls; i++ ) {
mca_ptl_mx_module_t* ptl = mca_ptl_mx_create(nic_addrs[i]);
if(NULL == ptl) {
return OMPI_ERROR;
}
mca_ptl_mx_component.mx_ptls[i] = ptl;
}
free(nic_addrs);
/* post local endpoint addresses */
size = mca_ptl_mx_component.mx_num_ptls * sizeof(mca_ptl_mx_endpoint_t);
endpoint_addrs = (mca_ptl_mx_endpoint_t*)malloc(size);
if(NULL == endpoint_addrs) {
opal_output(0, "mca_ptl_mx_module_init: malloc() failed\n");
return OMPI_ERR_OUT_OF_RESOURCE;
}
for( i = 0; i < mca_ptl_mx_component.mx_num_ptls; i++ ) {
mca_ptl_mx_module_t* ptl = mca_ptl_mx_component.mx_ptls[i];
mx_decompose_endpoint_addr( ptl->mx_endpoint_addr,
&(endpoint_addrs[i].nic_id), &(endpoint_addrs[i].endpoint_id) );
}
if((rc = mca_pml_base_modex_send( &mca_ptl_mx_component.super.ptlm_version,
endpoint_addrs, size )) != OMPI_SUCCESS )
return rc;
return OMPI_SUCCESS;
}
/**
* Thread to progress outstanding requests.
*/
#if OMPI_ENABLE_PROGRESS_THREADS
static void* mca_ptl_mx_thread(opal_object_t *arg)
{
opal_thread_t* thr = (opal_thread_t*)arg;
mca_ptl_mx_module_t* ptl = thr->t_arg;
while(ptl->mx_enabled) {
mx_request_t mx_request;
mx_return_t mx_return;
mx_status_t mx_status;
uint32_t mx_result;
/* block waiting for status */
mx_return = mx_peek(
ptl->mx_endpoint,
UINT_MAX,
&mx_request,
&mx_result);
if(mx_return == MX_TIMEOUT)
continue;
else if(ptl->mx_enabled == false)
break;
else if(mx_return != MX_SUCCESS) {
opal_output(0, "mca_ptl_mx_thread: mx_probe() failed with status %d\n",
mx_return);
break;
}
/* dispatch completed requests */
mx_return = mx_test(
ptl->mx_endpoint,
&mx_request,
&mx_status,
&mx_result);
if(mx_return == MX_SUCCESS) {
MCA_PTL_MX_PROGRESS(ptl, mx_status);
} else {
opal_output(0, "mca_ptl_mx_progress: mx_test() failed with status=%dn",
mx_return);
}
/* pre-post receive */
if(ptl->mx_recvs_posted == 0) {
OPAL_THREAD_ADD32(&ptl->mx_recvs_posted,1);
MCA_PTL_MX_POST(ptl,MCA_PTL_HDR_TYPE_MATCH,sizeof(mca_ptl_base_match_header_t));
}
}
return NULL;
}
#endif
/*
* Callback on a match.
*
*/
static void mca_ptl_mx_match(void* context, uint64_t match_value, int size)
{
mca_ptl_mx_module_t* ptl = (mca_ptl_mx_module_t*)context;
mca_ptl_base_recv_request_t* request;
mca_ptl_mx_recv_frag_t *frag;
mx_return_t mx_return;
ompi_ptr_t match;
size_t offset;
ompi_proc_t* proc;
ompi_convertor_t* convertor;
int rc;
/* use of the header type as the match value */
if(match_value <= MCA_PTL_HDR_TYPE_MAX)
return;
/* otherwise extract request pointer and offset */
match.lval = match_value;
request = (mca_ptl_base_recv_request_t*)match.pval;
offset = match.sval.lval;
proc = ompi_comm_peer_lookup(request->req_recv.req_base.req_comm,
request->req_recv.req_base.req_ompi.req_status.MPI_SOURCE);
/* allocate a fragment for receive */
MCA_PTL_MX_RECV_FRAG_ALLOC(frag, rc);
if(rc != OMPI_SUCCESS) {
opal_output(0, "mca_ptl_mx_match: unable to allocate resources.\n");
return;
}
frag->frag_size = size;
frag->frag_recv.frag_request = request;
frag->frag_recv.frag_base.frag_peer = NULL;
frag->frag_recv.frag_base.frag_owner = &ptl->super;
frag->frag_recv.frag_base.frag_size = frag->frag_size;
frag->frag_recv.frag_base.frag_header.hdr_common.hdr_type =
MCA_PTL_HDR_TYPE_FRAG;
frag->frag_recv.frag_base.frag_header.hdr_common.hdr_flags = 0;
convertor = &frag->frag_recv.frag_base.frag_convertor;
ompi_convertor_clone( &(request->req_recv.req_convertor),
convertor, 1 );
ompi_convertor_personalize( convertor, 0, &offset, mca_ptl_mx_mem_alloc, NULL );
/* non-contiguous - allocate buffer for receive */
if( 1 == ompi_convertor_need_buffers( convertor ) ||
request->req_recv.req_bytes_packed < offset + frag->frag_size ) {
/* TODO - use a fixed fragment size for non-contigous and convert
* this to a free-list of buffers.
*/
frag->frag_recv.frag_is_buffered = true;
frag->frag_recv.frag_base.frag_addr = malloc(frag->frag_size);
if( NULL == frag->frag_recv.frag_base.frag_addr ) {
opal_output(0, "mca_ptl_mx_match: unable to allocate buffer (%d)\n", frag->frag_size);
MCA_PTL_MX_RECV_FRAG_RETURN(frag);
return;
}
/* check for sending more than receiving */
if( offset > request->req_recv.req_bytes_packed ) {
frag->frag_recv.frag_base.frag_size = 0;
} else if (offset + frag->frag_size > request->req_recv.req_bytes_packed ) {
frag->frag_recv.frag_base.frag_size = request->req_recv.req_bytes_packed - offset;
}
/* calculate offset into users buffer */
} else {
frag->frag_recv.frag_base.frag_addr = ((unsigned char*)request->req_recv.req_base.req_addr) + offset;
frag->frag_recv.frag_is_buffered = false;
}
/* dont receive a header */
frag->frag_segment_count = 1;
frag->frag_segments[1].segment_ptr = frag->frag_recv.frag_base.frag_addr;
frag->frag_segments[1].segment_length = frag->frag_size;
mx_return = mx_irecv(
ptl->mx_endpoint,
frag->frag_segments+1,
frag->frag_segment_count,
match_value,
MX_MATCH_MASK_NONE,
frag,
&frag->frag_request);
if(mx_return != MX_SUCCESS) {
opal_output(0, "mca_ptl_mx_match: mx_irecv() failed with status=%dn", mx_return);
MCA_PTL_MX_RECV_FRAG_RETURN(frag);
}
}
/*
* Create and intialize an MX PTL module, where each module
* represents a specific NIC.
*/
static mca_ptl_mx_module_t* mca_ptl_mx_create(uint64_t addr)
{
mca_ptl_mx_module_t* ptl = malloc(sizeof(mca_ptl_mx_module_t));
mx_return_t status;
uint32_t nic_id;
if(NULL == ptl) return NULL;
status = mx_nic_id_to_board_number( addr, &nic_id );
if( MX_SUCCESS != status ) {
return NULL;
}
/* copy over default settings */
memcpy(ptl, &mca_ptl_mx_module, sizeof(mca_ptl_mx_module_t));
OBJ_CONSTRUCT(&ptl->mx_peers, opal_list_t);
/* open local endpoint */
status = mx_open_endpoint( nic_id, MX_ANY_ENDPOINT,
mca_ptl_mx_component.mx_filter,
NULL, 0, &ptl->mx_endpoint);
if(status != MX_SUCCESS) {
opal_output(0, "mca_ptl_mx_init: mx_open_endpoint() failed with status=%d\n", status);
mca_ptl_mx_finalize(&ptl->super);
return NULL;
}
/* query the endpoint address */
if((status = mx_get_endpoint_addr( ptl->mx_endpoint,
&ptl->mx_endpoint_addr)) != MX_SUCCESS) {
opal_output(0, "mca_ptl_mx_init: mx_get_endpoint_addr() failed with status=%d\n", status);
mca_ptl_mx_finalize(&ptl->super);
return NULL;
}
/* prepost a receive buffer */
ptl->mx_recvs_posted = 1;
MCA_PTL_MX_POST(ptl, MCA_PTL_HDR_TYPE_MATCH, sizeof(mca_ptl_base_match_header_t));
MCA_PTL_MX_POST(ptl, MCA_PTL_HDR_TYPE_RNDV, sizeof(mca_ptl_base_rendezvous_header_t));
MCA_PTL_MX_POST(ptl, MCA_PTL_HDR_TYPE_ACK, sizeof(mca_ptl_base_ack_header_t));
/* register a callback function for matching */
mx_register_match_callback(ptl->mx_endpoint, mca_ptl_mx_match, ptl);
return ptl;
}
void mca_ptl_mx_enable()
{
size_t i;
for(i=0; i<mca_ptl_mx_component.mx_num_ptls; i++) {
mca_ptl_mx_module_t* ptl = mca_ptl_mx_component.mx_ptls[i];
ptl->mx_enabled = true;
#if OMPI_ENABLE_PROGRESS_THREADS
/* create a thread to progress requests */
OBJ_CONSTRUCT(&ptl->mx_thread, opal_thread_t);
ptl->mx_thread.t_run = mca_ptl_mx_thread;
ptl->mx_thread.t_arg = ptl;
if(opal_thread_start(&ptl->mx_thread) != OMPI_SUCCESS) {
opal_output(0, "mca_ptl_mx_create: unable to start progress thread.\n");
return;
}
#endif
}
}
void mca_ptl_mx_disable(void)
{
size_t i;
for(i=0; i<mca_ptl_mx_component.mx_num_ptls; i++) {
mca_ptl_mx_module_t* ptl = mca_ptl_mx_component.mx_ptls[i];
ptl->mx_enabled = false;
#if OMPI_ENABLE_PROGRESS_THREADS
mx_wakeup(ptl->mx_endpoint);
opal_thread_join(&ptl->mx_thread, NULL);
#endif
}
}
/*
* Cleanup PTL resources.
*/
int mca_ptl_mx_finalize(struct mca_ptl_base_module_t* ptl)
{
mca_ptl_mx_module_t* mx_ptl = (mca_ptl_mx_module_t*)ptl;
mx_ptl->mx_enabled = false;
#if OMPI_ENABLE_PROGRESS_THREADS
mx_wakeup(mx_ptl->mx_endpoint);
opal_thread_join(&mx_ptl->mx_thread, NULL);
#endif
mx_close_endpoint(mx_ptl->mx_endpoint);
mx_ptl->mx_endpoint = NULL;
free(mx_ptl);
return OMPI_SUCCESS;
}
/**
* PML->PTL notification of addition to the process list.
*
* @param ptl (IN)
* @param nprocs (IN) Number of processes
* @param procs (IN) Set of processes
* @param peers (OUT) Set of (optional) peer addressing info.
* @param peers (IN/OUT) Set of processes that are reachable via this PTL.
* @return OMPI_SUCCESS or error status on failure.
*
*/
int mca_ptl_mx_add_procs(
struct mca_ptl_base_module_t* ptl,
size_t nprocs,
struct ompi_proc_t **procs,
struct mca_ptl_base_peer_t** peers,
ompi_bitmap_t* reachable)
{
ompi_proc_t* proc_self = ompi_proc_local();
mca_ptl_mx_module_t* ptl_mx = (mca_ptl_mx_module_t*)ptl;
size_t n;
for( n = 0; n < nprocs; n++ ) {
int rc;
mca_ptl_mx_proc_t *ptl_proc;
mca_ptl_mx_peer_t* ptl_peer;
/* Dont let mx register for self */
if( proc_self == procs[n] ) continue;
if((ptl_proc = mca_ptl_mx_proc_create(procs[n])) == NULL)
return OMPI_ERR_OUT_OF_RESOURCE;
/* peer doesn't export enough addresses */
OPAL_THREAD_LOCK(&ptl_proc->proc_lock);
if(ptl_proc->proc_peer_count == ptl_proc->proc_addr_count) {
OPAL_THREAD_UNLOCK(&ptl_proc->proc_lock);
continue;
}
/* The ptl_proc datastructure is shared by all MX PTL instances that are trying
* to reach this destination. Cache the peer instance on the proc.
*/
ptl_peer = OBJ_NEW(mca_ptl_mx_peer_t);
if(NULL == ptl_peer) {
OPAL_THREAD_UNLOCK(&ptl_proc->proc_lock);
return OMPI_ERR_OUT_OF_RESOURCE;
}
ptl_peer->peer_ptl = ptl_mx;
rc = mca_ptl_mx_proc_insert(ptl_proc, ptl_peer);
if(rc != OMPI_SUCCESS) {
OBJ_RELEASE(ptl_peer);
OPAL_THREAD_UNLOCK(&ptl_proc->proc_lock);
return rc;
}
/* do we need to convert to/from network byte order */
if(procs[n]->proc_arch != proc_self->proc_arch) {
ptl_peer->peer_nbo = true;
}
/* set peer as reachable */
ompi_bitmap_set_bit(reachable, n);
OPAL_THREAD_UNLOCK(&ptl_proc->proc_lock);
peers[n] = ptl_peer;
opal_list_append(&ptl_mx->mx_peers, (opal_list_item_t*)ptl_peer);
}
return OMPI_SUCCESS;
}
/**
* PML->PTL notification of change in the process list.
*
* @param ptl (IN) PTL instance
* @param nproc (IN) Number of processes.
* @param procs (IN) Set of processes.
* @param peers (IN) Set of peer data structures.
* @return Status indicating if cleanup was successful
*
*/
int mca_ptl_mx_del_procs(
struct mca_ptl_base_module_t* ptl,
size_t nprocs,
struct ompi_proc_t **proc,
struct mca_ptl_base_peer_t** ptl_peer)
{
return OMPI_SUCCESS;
}

142
ompi/mca/ptl/mx/ptl_mx_module.h
Просмотреть файл

@ -1,142 +0,0 @@
/*
* Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
* University Research and Technology
* Corporation. All rights reserved.
* Copyright (c) 2004-2005 The University of Tennessee and The University
* of Tennessee Research Foundation. All rights
* reserved.
* Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
* University of Stuttgart. All rights reserved.
* Copyright (c) 2004-2005 The Regents of the University of California.
* All rights reserved.
* $COPYRIGHT$
*
* Additional copyrights may follow
*
* $HEADER$
*/
/**
* @file
*/
#ifndef MCA_PTL_MX_MODULE_H
#define MCA_PTL_MX_MODULE_H
#include "ompi_config.h"
#include "ptl_mx.h"
#include "ptl_mx_recvfrag.h"
#include "ptl_mx_sendfrag.h"
/**
* Post a receive for short messages (<32 K)
*/
#define MCA_PTL_MX_POST(ptl, match, header_size) \
do { \
mca_ptl_mx_recv_frag_t *frag; \
mx_return_t mx_return; \
int rc; \
\
MCA_PTL_MX_RECV_FRAG_ALLOC(frag, rc); \
if(rc != OMPI_SUCCESS) { \
opal_output(0, "mca_ptl_mx_match: unable to allocate resources.\n"); \
break; \
} \
frag->frag_size = 0; \
frag->frag_recv.frag_base.frag_owner = &ptl->super; \
frag->frag_recv.frag_base.frag_peer = NULL; \
frag->frag_recv.frag_base.frag_size = 0; \
frag->frag_recv.frag_base.frag_addr = frag->frag_data; \
frag->frag_recv.frag_is_buffered = true; \
frag->frag_recv.frag_request = NULL; \
frag->frag_segments[0].segment_length = header_size; \
frag->frag_segments[1].segment_ptr = frag->frag_data; \
frag->frag_segments[1].segment_length = sizeof(frag->frag_data); \
frag->frag_segment_count = 2; \
mx_return = mx_irecv( \
ptl->mx_endpoint, \
frag->frag_segments, \
frag->frag_segment_count, \
match, \
MX_MATCH_MASK_NONE, \
frag, \
&frag->frag_request); \
if(mx_return != MX_SUCCESS) { \
opal_output(0, "mca_ptl_mx_match: mx_irecv() failed with status=%dn", mx_return); \
MCA_PTL_MX_RECV_FRAG_RETURN(frag); \
} \
} while(0)
/**
* Routine to process complete request(s).
*/
#define MCA_PTL_MX_PROGRESS(ptl, mx_status) \
do { \
mca_ptl_base_frag_t* frag; \
frag = (mca_ptl_base_frag_t*)mx_status.context; \
switch(frag->frag_type) { \
case MCA_PTL_FRAGMENT_SEND: \
{ \
mca_ptl_mx_send_frag_t* sendfrag = (mca_ptl_mx_send_frag_t*)frag; \
MCA_PTL_MX_SEND_FRAG_PROGRESS(sendfrag); \
break; \
} \
case MCA_PTL_FRAGMENT_RECV: \
{ \
mca_ptl_mx_recv_frag_t* recvfrag = (mca_ptl_mx_recv_frag_t*)frag; \
mca_ptl_base_header_t* hdr = \
&recvfrag->frag_recv.frag_base.frag_header; \
switch(hdr->hdr_common.hdr_type) { \
case MCA_PTL_HDR_TYPE_MATCH: \
{ \
recvfrag->frag_size = hdr->hdr_match.hdr_msg_length; \
MCA_PTL_MX_RECV_FRAG_MATCH(recvfrag,hdr); \
OPAL_THREAD_ADD32(&ptl->mx_recvs_posted, -1); \
break; \
} \
case MCA_PTL_HDR_TYPE_RNDV: \
{ \
recvfrag->frag_size = hdr->hdr_rndv.hdr_frag_length; \
MCA_PTL_MX_RECV_FRAG_RNDV(recvfrag,hdr); \
MCA_PTL_MX_POST(ptl, MCA_PTL_HDR_TYPE_RNDV, \
sizeof(mca_ptl_base_rendezvous_header_t)); \
break; \
} \
case MCA_PTL_HDR_TYPE_FRAG: \
{ \
MCA_PTL_MX_RECV_FRAG_FRAG(recvfrag); \
break; \
} \
case MCA_PTL_HDR_TYPE_ACK: \
{ \
mca_ptl_mx_send_frag_t* sendfrag; \
mca_ptl_base_send_request_t* sendreq; \
sendfrag = (mca_ptl_mx_send_frag_t*) \
hdr->hdr_ack.hdr_src_ptr.pval; \
sendreq = sendfrag->frag_send.frag_request; \
sendreq->req_peer_match = hdr->hdr_ack.hdr_dst_match; \
MCA_PTL_MX_SEND_FRAG_PROGRESS(sendfrag); \
MCA_PTL_MX_RECV_FRAG_RETURN(recvfrag); \
MCA_PTL_MX_POST(ptl, MCA_PTL_HDR_TYPE_ACK, \
sizeof(mca_ptl_base_ack_header_t)); \
break; \
} \
} \
break; \
} \
default: \
{ \
opal_output(0, "mca_ptl_mx_progress: invalid request type: %d", \
frag->frag_type); \
break; \
} \
} \
} while(0)
void mca_ptl_mx_enable(void);
void mca_ptl_mx_disable(void);
#endif

49
ompi/mca/ptl/mx/ptl_mx_peer.c
Просмотреть файл

@ -1,49 +0,0 @@
/*
* Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
* University Research and Technology
* Corporation. All rights reserved.
* Copyright (c) 2004-2005 The University of Tennessee and The University
* of Tennessee Research Foundation. All rights
* reserved.
* Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
* University of Stuttgart. All rights reserved.
* Copyright (c) 2004-2005 The Regents of the University of California.
* All rights reserved.
* $COPYRIGHT$
*
* Additional copyrights may follow
*
* $HEADER$
*/
#include "ompi_config.h"
#include "ptl_mx.h"
#include "ptl_mx_peer.h"
static void mca_ptl_mx_peer_construct(mca_ptl_base_peer_t* ptl_peer);
static void mca_ptl_mx_peer_destruct(mca_ptl_base_peer_t* ptl_peer);
OBJ_CLASS_INSTANCE(
mca_ptl_mx_peer_t,
opal_list_item_t,
mca_ptl_mx_peer_construct,
mca_ptl_mx_peer_destruct);
/*
* Initialize state of the peer instance.
*/
static void mca_ptl_mx_peer_construct(mca_ptl_base_peer_t* ptl_peer)
{
ptl_peer->peer_ptl = NULL;
ptl_peer->peer_nbo = false;
}
/*
* Cleanup any resources held by the peer.
*/
static void mca_ptl_mx_peer_destruct(mca_ptl_base_peer_t* ptl_peer)
{
}

44
ompi/mca/ptl/mx/ptl_mx_peer.h
Просмотреть файл

@ -1,44 +0,0 @@
/*
* Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
* University Research and Technology
* Corporation. All rights reserved.
* Copyright (c) 2004-2005 The University of Tennessee and The University
* of Tennessee Research Foundation. All rights
* reserved.
* Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
* University of Stuttgart. All rights reserved.
* Copyright (c) 2004-2005 The Regents of the University of California.
* All rights reserved.
* $COPYRIGHT$
*
* Additional copyrights may follow
*
* $HEADER$
*/
/**
* @file
*/
#ifndef MCA_PTL_MX_PEER_H
#define MCA_PTL_MX_PEER_H
#include "ompi_config.h"
#include <myriexpress.h>
/**
* An abstraction that represents a a peer process.
*/
struct mca_ptl_base_peer_t {
opal_list_item_t peer_item;
mx_endpoint_addr_t peer_addr;
struct mca_ptl_mx_module_t* peer_ptl;
struct mca_ptl_mx_proc_t* peer_proc;
bool peer_nbo;
};
typedef struct mca_ptl_base_peer_t mca_ptl_base_peer_t;
typedef struct mca_ptl_base_peer_t mca_ptl_mx_peer_t;
OBJ_CLASS_DECLARATION(mca_ptl_mx_peer_t);
#endif

203
ompi/mca/ptl/mx/ptl_mx_proc.c
Просмотреть файл

@ -1,203 +0,0 @@
/*
* Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
* University Research and Technology
* Corporation. All rights reserved.
* Copyright (c) 2004-2005 The University of Tennessee and The University
* of Tennessee Research Foundation. All rights
* reserved.
* Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
* University of Stuttgart. All rights reserved.
* Copyright (c) 2004-2005 The Regents of the University of California.
* All rights reserved.
* $COPYRIGHT$
*
* Additional copyrights may follow
*
* $HEADER$
*/
#include "ompi_config.h"
#include <string.h>
#include "opal/sys/atomic.h"
#include "orte/class/orte_proc_table.h"
#include "ompi/mca/pml/base/pml_base_module_exchange.h"
#include "opal/util/output.h"
#include "ptl_mx.h"
#include "ptl_mx_peer.h"
#include "ptl_mx_proc.h"
static void mca_ptl_mx_proc_construct(mca_ptl_mx_proc_t* proc);
static void mca_ptl_mx_proc_destruct(mca_ptl_mx_proc_t* proc);
OBJ_CLASS_INSTANCE(
mca_ptl_mx_proc_t,
opal_list_item_t,
mca_ptl_mx_proc_construct,
mca_ptl_mx_proc_destruct
);
/**
* Initialize mx proc instance
*/
void mca_ptl_mx_proc_construct(mca_ptl_mx_proc_t* proc)
{
proc->proc_ompi = NULL;
proc->proc_addrs = NULL;
proc->proc_addr_count = 0;
proc->proc_peers = NULL;
proc->proc_peer_count = 0;
OBJ_CONSTRUCT(&proc->proc_lock, opal_mutex_t);
}
/*
* Cleanup mx proc instance
*/
void mca_ptl_mx_proc_destruct(mca_ptl_mx_proc_t* proc)
{
/* remove from list of all proc instances */
OPAL_THREAD_LOCK(&mca_ptl_mx_component.mx_lock);
orte_hash_table_remove_proc(&mca_ptl_mx_component.mx_procs, &proc->proc_name);
OPAL_THREAD_UNLOCK(&mca_ptl_mx_component.mx_lock);
/* release resources */
if(NULL != proc->proc_peers)
free(proc->proc_peers);
}
/*
* Create a MX process structure. There is a one-to-one correspondence
* between a ompi_proc_t and a mca_ptl_mx_proc_t instance. We cache additional
* data (specifically the list of mca_ptl_mx_peer_t instances, and published
* addresses) associated w/ a given destination on this datastructure.
*/
mca_ptl_mx_proc_t* mca_ptl_mx_proc_create(ompi_proc_t* ompi_proc)
{
int rc;
size_t size;
mca_ptl_mx_proc_t* ptl_proc;
OPAL_THREAD_LOCK(&mca_ptl_mx_component.mx_lock);
ptl_proc = (mca_ptl_mx_proc_t*)orte_hash_table_get_proc(
&mca_ptl_mx_component.mx_procs, &ompi_proc->proc_name);
if(NULL != ptl_proc) {
OPAL_THREAD_UNLOCK(&mca_ptl_mx_component.mx_lock);
return ptl_proc;
}
ptl_proc = OBJ_NEW(mca_ptl_mx_proc_t);
if(NULL == ptl_proc)
return NULL;
ptl_proc->proc_ompi = ompi_proc;
ptl_proc->proc_name = ompi_proc->proc_name;
/* add to hash table of all proc instance */
orte_hash_table_set_proc(
&mca_ptl_mx_component.mx_procs,
&ptl_proc->proc_name,
ptl_proc);
OPAL_THREAD_UNLOCK(&mca_ptl_mx_component.mx_lock);
/* lookup mx parameters exported by this proc */
rc = mca_pml_base_modex_recv(
&mca_ptl_mx_component.super.ptlm_version,
ompi_proc,
(void**)&ptl_proc->proc_addrs,
&size);
if(rc != OMPI_SUCCESS) {
opal_output(0, "mca_ptl_mx_proc_create: mca_pml_base_modex_recv: failed with return value=%d", rc);
OBJ_RELEASE(ptl_proc);
return NULL;
}
if(0 != (size % sizeof(mca_ptl_mx_endpoint_t))) {
opal_output(0, "mca_ptl_mx_proc_create: mca_pml_base_modex_recv: invalid size %d\n", size);
return NULL;
}
ptl_proc->proc_addr_count = size / sizeof(mca_ptl_mx_endpoint_t);
/* allocate space for peer array - one for each exported address */
ptl_proc->proc_peers = (mca_ptl_mx_peer_t**)
malloc(ptl_proc->proc_addr_count * sizeof(mca_ptl_base_peer_t*));
if(NULL == ptl_proc->proc_peers) {
OBJ_RELEASE(ptl_proc);
return NULL;
}
return ptl_proc;
}
/*
* Look for an existing MX process instance based on the globally unique
* process identifier.
*/
mca_ptl_mx_proc_t* mca_ptl_mx_proc_lookup(const orte_process_name_t *name)
{
mca_ptl_mx_proc_t* proc;
OPAL_THREAD_LOCK(&mca_ptl_mx_component.mx_lock);
proc = (mca_ptl_mx_proc_t*)orte_hash_table_get_proc(
&mca_ptl_mx_component.mx_procs, name);
OPAL_THREAD_UNLOCK(&mca_ptl_mx_component.mx_lock);
return proc;
}
/*
* Note that this routine must be called with the lock on the process already
* held. Insert a ptl instance into the proc array and assign it an address.
*/
int mca_ptl_mx_proc_insert(mca_ptl_mx_proc_t* ptl_proc, mca_ptl_base_peer_t* ptl_peer)
{
mx_return_t mx_status;
mca_ptl_mx_endpoint_t* remote = &(ptl_proc->proc_addrs[ptl_proc->proc_peer_count]);
int num_retry = 0;
/* insert into peer array */
ptl_peer->peer_proc = ptl_proc;
ptl_proc->proc_peers[ptl_proc->proc_peer_count] = ptl_peer;
/* construct the remote endpoint addr */
retry_connect:
mx_status = mx_connect( ptl_peer->peer_ptl->mx_endpoint, remote->nic_id, remote->endpoint_id,
mca_ptl_mx_component.mx_filter, 1, &(ptl_peer->peer_addr) );
if( MX_SUCCESS != mx_status ) {
if( MX_TIMEOUT == mx_status )
if( num_retry++ < 5 )
goto retry_connect;
opal_output( 0, "mx_connect fail for peer %d remote %lx %d filter %x with error %s\n",
ptl_proc->proc_peer_count,
remote->nic_id, remote->endpoint_id, mca_ptl_mx_component.mx_filter,
mx_strerror(mx_status) );
return OMPI_ERROR;
}
ptl_proc->proc_peer_count++;
return OMPI_SUCCESS;
}
/*
* Remove a peer from the proc array and indicate the address is
* no longer in use.
*/
int mca_ptl_mx_proc_remove(mca_ptl_mx_proc_t* ptl_proc, mca_ptl_base_peer_t* ptl_peer)
{
size_t i;
OPAL_THREAD_LOCK(&ptl_proc->proc_lock);
for(i=0; i<ptl_proc->proc_peer_count; i++) {
if(ptl_proc->proc_peers[i] == ptl_peer) {
memmove(ptl_proc->proc_peers+i, ptl_proc->proc_peers+i+1,
(ptl_proc->proc_peer_count-i-1)*sizeof(mca_ptl_mx_peer_t*));
ptl_proc->proc_peer_count--;
break;
}
}
OPAL_THREAD_UNLOCK(&ptl_proc->proc_lock);
return OMPI_SUCCESS;
}

72
ompi/mca/ptl/mx/ptl_mx_proc.h
Просмотреть файл

@ -1,72 +0,0 @@
/*
* Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
* University Research and Technology
* Corporation. All rights reserved.
* Copyright (c) 2004-2005 The University of Tennessee and The University
* of Tennessee Research Foundation. All rights
* reserved.
* Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
* University of Stuttgart. All rights reserved.
* Copyright (c) 2004-2005 The Regents of the University of California.
* All rights reserved.
* $COPYRIGHT$
*
* Additional copyrights may follow
*
* $HEADER$
*/
/**
* @file
*/
#ifndef MCA_PTL_MX_PROC_H
#define MCA_PTL_MX_PROC_H
#include "ompi_config.h"
#include <myriexpress.h>
#include "orte/mca/ns/ns.h"
#include "opal/class/opal_object.h"
#include "ompi/proc/proc.h"
#include "ptl_mx.h"
#include "ptl_mx_peer.h"
#if defined(c_plusplus) || defined(__cplusplus)
extern "C" {
#endif
typedef struct mca_ptl_mx_endpoint_t {
uint64_t nic_id;
uint32_t endpoint_id;
} mca_ptl_mx_endpoint_t;
/**
* Represents the state of a remote process and the set of addresses
* that it exports. Also cache an instance of mca_ptl_base_peer_t for each
* PTL instance that attempts to open a connection to the process.
*/
struct mca_ptl_mx_proc_t {
opal_list_item_t super; /**< allow proc to be placed on a list */
ompi_proc_t *proc_ompi; /**< pointer to corresponding ompi_proc_t */
orte_process_name_t proc_name; /**< globally unique identifier for the process */
mca_ptl_mx_endpoint_t *proc_addrs; /**< peer endpoint address */
size_t proc_addr_count; /**< number of addresses published by peer */
mca_ptl_mx_peer_t **proc_peers; /**< array of peers that have been created to access this proc */
size_t proc_peer_count; /**< number of peers */
opal_mutex_t proc_lock; /**< lock to protect against concurrent access to proc state */
};
typedef struct mca_ptl_mx_proc_t mca_ptl_mx_proc_t;
OBJ_CLASS_DECLARATION(mca_ptl_mx_proc_t);
mca_ptl_mx_proc_t* mca_ptl_mx_proc_create(ompi_proc_t* ompi_proc);
mca_ptl_mx_proc_t* mca_ptl_mx_proc_lookup(const orte_process_name_t*);
int mca_ptl_mx_proc_insert(mca_ptl_mx_proc_t* ptl_proc, mca_ptl_base_peer_t* ptl_peer);
int mca_ptl_mx_proc_remove(mca_ptl_mx_proc_t* ptl_proc, mca_ptl_base_peer_t* ptl_peer);
#if defined(c_plusplus) || defined(__cplusplus)
}
#endif
#endif

52
ompi/mca/ptl/mx/ptl_mx_recvfrag.c
Просмотреть файл

@ -1,52 +0,0 @@
/*
* Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
* University Research and Technology
* Corporation. All rights reserved.
* Copyright (c) 2004-2005 The University of Tennessee and The University
* of Tennessee Research Foundation. All rights
* reserved.
* Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
* University of Stuttgart. All rights reserved.
* Copyright (c) 2004-2005 The Regents of the University of California.
* All rights reserved.
* $COPYRIGHT$
*
* Additional copyrights may follow
*
* $HEADER$
*/
#include "ompi_config.h"
#include "ptl_mx.h"
#include "ptl_mx_recvfrag.h"
static void mca_ptl_mx_recv_frag_construct(mca_ptl_mx_recv_frag_t* frag);
static void mca_ptl_mx_recv_frag_destruct(mca_ptl_mx_recv_frag_t* frag);
OBJ_CLASS_INSTANCE(
mca_ptl_mx_recv_frag_t,
mca_ptl_base_recv_frag_t,
mca_ptl_mx_recv_frag_construct,
mca_ptl_mx_recv_frag_destruct);
/*
* MX recv fragment constructor
*/
static void mca_ptl_mx_recv_frag_construct(mca_ptl_mx_recv_frag_t* frag)
{
/* one time initialization */
frag->frag_segments[0].segment_ptr = &frag->frag_recv.frag_base.frag_header;
frag->frag_segments[0].segment_length = sizeof(frag->frag_recv.frag_base.frag_header);
}
/*
* MX recv fragment destructor
*/
static void mca_ptl_mx_recv_frag_destruct(mca_ptl_mx_recv_frag_t* frag)
{
}

122
ompi/mca/ptl/mx/ptl_mx_recvfrag.h
Просмотреть файл

@ -1,122 +0,0 @@
/*
* Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
* University Research and Technology
* Corporation. All rights reserved.
* Copyright (c) 2004-2005 The University of Tennessee and The University
* of Tennessee Research Foundation. All rights
* reserved.
* Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
* University of Stuttgart. All rights reserved.
* Copyright (c) 2004-2005 The Regents of the University of California.
* All rights reserved.
* $COPYRIGHT$
*
* Additional copyrights may follow
*
* $HEADER$
*/
/**
* @file
*/
#ifndef MCA_PTL_MX_RECV_FRAG_H
#define MCA_PTL_MX_RECV_FRAG_H
#include "ptl_mx.h"
#include "ompi/mca/ptl/base/ptl_base_recvfrag.h"
#include "ptl_mx_sendfrag.h"
/**
* MX received fragment derived type.
*/
struct mca_ptl_mx_recv_frag_t {
mca_ptl_base_recv_frag_t frag_recv; /**< base receive fragment descriptor */
mx_request_t frag_request;
mx_segment_t frag_segments[2];
uint32_t frag_segment_count;
unsigned char frag_data[32*1024];
size_t frag_size;
};
typedef struct mca_ptl_mx_recv_frag_t mca_ptl_mx_recv_frag_t;
OBJ_CLASS_DECLARATION(mca_ptl_mx_recv_frag_t);
#define MCA_PTL_MX_RECV_FRAG_ALLOC(frag, rc) \
{ \
opal_list_item_t* item; \
OMPI_FREE_LIST_GET(&mca_ptl_mx_component.mx_recv_frags, item, rc); \
frag = (mca_ptl_mx_recv_frag_t*)item; \
}
#define MCA_PTL_MX_RECV_FRAG_RETURN(frag) \
{ \
if(frag->frag_recv.frag_is_buffered && \
frag->frag_data != frag->frag_recv.frag_base.frag_addr) { \
free(frag->frag_recv.frag_base.frag_addr); \
} \
OMPI_FREE_LIST_RETURN(&mca_ptl_mx_component.mx_recv_frags, (opal_list_item_t*)frag); \
}
/**
* Callback on receipt of a match fragment.
*/
#define MCA_PTL_MX_RECV_FRAG_MATCH(frag, hdr) \
do { \
if(hdr->hdr_common.hdr_flags & MCA_PTL_FLAGS_NBO) { \
MCA_PTL_BASE_MATCH_HDR_NTOH(hdr->hdr_match); \
} \
ptl->super.ptl_match(&ptl->super, &frag->frag_recv, &hdr->hdr_match); \
} while(0)
/**
* Callback on receipt of a rendezvous fragment.
*/
#define MCA_PTL_MX_RECV_FRAG_RNDV(frag, hdr) \
do { \
if(hdr->hdr_common.hdr_flags & MCA_PTL_FLAGS_NBO) { \
MCA_PTL_BASE_RNDV_HDR_NTOH(hdr->hdr_rndv); \
} \
ptl->super.ptl_match(&ptl->super, &frag->frag_recv, &hdr->hdr_match); \
} while(0)
/**
* Process a fragment that completed.
*/
#define MCA_PTL_MX_RECV_FRAG_FRAG(frag) \
do { \
/* copy into user space */ \
if(frag->frag_recv.frag_is_buffered) { \
struct iovec iov; \
uint32_t iov_count; \
size_t max_data; \
int32_t free_after; \
\
iov.iov_base = frag->frag_recv.frag_base.frag_addr; \
iov.iov_len = frag->frag_recv.frag_base.frag_size; \
iov_count = 1; \
max_data = iov.iov_len; \
ompi_convertor_unpack( &frag->frag_recv.frag_base.frag_convertor, \
&iov, &iov_count, &max_data, &free_after ); \
frag->frag_recv.frag_base.frag_size = max_data; \
} \
\
/* progress the request */ \
frag->frag_recv.frag_base.frag_owner->ptl_recv_progress( \
frag->frag_recv.frag_base.frag_owner, \
frag->frag_recv.frag_request, \
frag->frag_size, \
frag->frag_recv.frag_base.frag_size); \
\
MCA_PTL_MX_RECV_FRAG_RETURN(frag); \
} while(0)
#endif

51
ompi/mca/ptl/mx/ptl_mx_sendfrag.c
Просмотреть файл

@ -1,51 +0,0 @@
/*
* Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
* University Research and Technology
* Corporation. All rights reserved.
* Copyright (c) 2004-2005 The University of Tennessee and The University
* of Tennessee Research Foundation. All rights
* reserved.
* Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
* University of Stuttgart. All rights reserved.
* Copyright (c) 2004-2005 The Regents of the University of California.
* All rights reserved.
* $COPYRIGHT$
*
* Additional copyrights may follow
*
* $HEADER$
*/
#include "ptl_mx.h"
#include "ptl_mx_peer.h"
#include "ptl_mx_sendfrag.h"
static void mca_ptl_mx_send_frag_construct(mca_ptl_mx_send_frag_t* frag);
static void mca_ptl_mx_send_frag_destruct(mca_ptl_mx_send_frag_t* frag);
OBJ_CLASS_INSTANCE(
mca_ptl_mx_send_frag_t,
mca_ptl_base_send_frag_t,
mca_ptl_mx_send_frag_construct,
mca_ptl_mx_send_frag_destruct);
/*
* Placeholders for send fragment constructor/destructors.
*/
static void mca_ptl_mx_send_frag_construct(mca_ptl_mx_send_frag_t* frag)
{
/* one time initialization */
frag->frag_segments[0].segment_ptr = &frag->frag_send.frag_base.frag_header;
frag->frag_segments[0].segment_length = sizeof(mca_ptl_base_header_t);
}
static void mca_ptl_mx_send_frag_destruct(mca_ptl_mx_send_frag_t* frag)
{
}

125
ompi/mca/ptl/mx/ptl_mx_sendfrag.h
Просмотреть файл

@ -1,125 +0,0 @@
/*
* Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
* University Research and Technology
* Corporation. All rights reserved.
* Copyright (c) 2004-2005 The University of Tennessee and The University
* of Tennessee Research Foundation. All rights
* reserved.
* Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
* University of Stuttgart. All rights reserved.
* Copyright (c) 2004-2005 The Regents of the University of California.
* All rights reserved.
* $COPYRIGHT$
*
* Additional copyrights may follow
*
* $HEADER$
*/
/**
* @file
*/
#ifndef MCA_PTL_MX_SEND_FRAG_H
#define MCA_PTL_MX_SEND_FRAG_H
#include "ompi_config.h"
#include "opal/sys/atomic.h"
#include "ompi/mca/ptl/base/ptl_base_sendreq.h"
#include "ompi/mca/ptl/base/ptl_base_sendfrag.h"
#include "ptl_mx.h"
#include "ptl_mx_peer.h"
/**
* MX send fragment derived type.
*/
struct mca_ptl_mx_send_frag_t {
mca_ptl_base_send_frag_t frag_send; /**< base send fragment descriptor */
int frag_free;
mx_request_t frag_request;
mx_segment_t frag_segments[2];
size_t frag_segment_count;
int32_t frag_progress;
};
typedef struct mca_ptl_mx_send_frag_t mca_ptl_mx_send_frag_t;
OBJ_CLASS_DECLARATION(mca_ptl_mx_send_frag_t);
#define MCA_PTL_MX_SEND_FRAG_ALLOC(sendfrag, rc) \
{ \
opal_list_item_t* item; \
OMPI_FREE_LIST_GET(&mca_ptl_mx_component.mx_send_frags, item, rc); \
sendfrag = (mca_ptl_mx_send_frag_t*)item; \
}
#define MCA_PTL_MX_SEND_FRAG_RETURN(sendfrag) \
{ \
int seg_free = sendfrag->frag_free; \
mx_segment_t *seg_ptr = sendfrag->frag_segments+1; \
while(seg_free) { \
if(seg_free & 1) { \
free(seg_ptr->segment_ptr); \
} \
seg_free >>= 1; \
seg_ptr++; \
} \
OMPI_FREE_LIST_RETURN(&mca_ptl_mx_component.mx_send_frags, (opal_list_item_t*)sendfrag); \
}
#define MCA_PTL_MX_SEND_FRAG_INIT_ACK(ack,ptl,frag) \
{ \
mca_ptl_base_header_t* hdr = &((ack)->frag_send.frag_base.frag_header); \
mca_pml_base_recv_request_t* request = &((frag)->frag_recv.frag_request->req_recv); \
hdr->hdr_common.hdr_type = MCA_PTL_HDR_TYPE_ACK; \
hdr->hdr_common.hdr_flags = 0; \
hdr->hdr_ack.hdr_src_ptr = (frag)->frag_recv.frag_base.frag_header.hdr_rndv.hdr_src_ptr; \
hdr->hdr_ack.hdr_dst_match.lval = 0; /* for VALGRIND/PURIFY - REPLACE WITH MACRO */ \
hdr->hdr_ack.hdr_dst_match.pval = request; \
hdr->hdr_ack.hdr_dst_addr.lval = 0; /* for VALGRIND/PURIFY - REPLACE WITH MACRO */ \
hdr->hdr_ack.hdr_dst_addr.pval = request->req_base.req_addr; \
hdr->hdr_ack.hdr_dst_size = request->req_bytes_packed; \
(ack)->frag_send.frag_request = NULL; \
(ack)->frag_send.frag_base.frag_peer = (frag)->frag_recv.frag_base.frag_peer; \
(ack)->frag_send.frag_base.frag_owner = ptl; \
(ack)->frag_send.frag_base.frag_addr = NULL; \
(ack)->frag_send.frag_base.frag_size = 0; \
(ack)->frag_segments[0].segment_length = sizeof(mca_ptl_base_ack_header_t); \
(ack)->frag_segment_count = 1; \
(ack)->frag_free = 0; \
}
#define MCA_PTL_MX_SEND_FRAG_PROGRESS(frag) \
do { \
mca_ptl_base_send_request_t* request = frag->frag_send.frag_request; \
bool frag_ack; \
\
/* if this is an ack - simply return to pool */ \
if(request == NULL) { \
MCA_PTL_MX_SEND_FRAG_RETURN(frag); \
break; \
} \
\
/* Done when: \
* (1) ack is not required and send completes \
* (2) ack is received and send has completed \
*/ \
frag_ack = (frag->frag_send.frag_base.frag_header. \
hdr_common.hdr_flags & MCA_PTL_FLAGS_ACK) ? true : false; \
if(frag_ack == false || opal_atomic_add_32(&frag->frag_progress,1) == 2) { \
\
/* update request status */ \
frag->frag_send.frag_base.frag_owner->ptl_send_progress( \
frag->frag_send.frag_base.frag_owner, \
request, \
frag->frag_send.frag_base.frag_size); \
\
/* return any fragment that didnt come from the cache */ \
if (request->req_cached == false || \
frag->frag_send.frag_base.frag_header.hdr_common.hdr_type == MCA_PTL_HDR_TYPE_FRAG) { \
MCA_PTL_MX_SEND_FRAG_RETURN(frag); \
} \
} \
} while (0)
#endif

0
ompi/mca/ptl/portals/.ompi_ignore
Просмотреть файл

Некоторые файлы не были показаны из-за слишком большого количества измененных файлов Показать больше