moved up
This commit was SVN r5656.
Этот коммит содержится в:
Tim Woodall
родитель
3d3151abb3
Коммит
10c8338a04
@ -1,40 +0,0 @@
|
||||
# -*- makefile -*-
|
||||
#
|
||||
# Copyright (c) 2004-2005 The Trustees of Indiana University.
|
||||
# All rights reserved.
|
||||
# Copyright (c) 2004-2005 The Trustees of the University of Tennessee.
|
||||
# All rights reserved.
|
||||
# Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
|
||||
# University of Stuttgart. All rights reserved.
|
||||
# Copyright (c) 2004-2005 The Regents of the University of California.
|
||||
# All rights reserved.
|
||||
# $COPYRIGHT$
|
||||
#
|
||||
# Additional copyrights may follow
|
||||
#
|
||||
# $HEADER$
|
||||
#
|
||||
|
||||
sources += \
|
||||
src/pml_teg.c \
|
||||
src/pml_teg.h \
|
||||
src/pml_teg_cancel.c \
|
||||
src/pml_teg_component.c \
|
||||
src/pml_teg_component.h \
|
||||
src/pml_teg_iprobe.c \
|
||||
src/pml_teg_irecv.c \
|
||||
src/pml_teg_isend.c \
|
||||
src/pml_teg_ptl.c \
|
||||
src/pml_teg_ptl.h \
|
||||
src/pml_teg_proc.c \
|
||||
src/pml_teg_proc.h \
|
||||
src/pml_teg_progress.c \
|
||||
src/pml_teg_recvfrag.c \
|
||||
src/pml_teg_recvfrag.h \
|
||||
src/pml_teg_recvreq.c \
|
||||
src/pml_teg_recvreq.h \
|
||||
src/pml_teg_sendreq.c \
|
||||
src/pml_teg_sendreq.h \
|
||||
src/pml_teg_start.c \
|
||||
src/pml_ptl_array.c \
|
||||
src/pml_ptl_array.h
|
||||
@ -1,61 +0,0 @@
|
||||
/*
|
||||
* Copyright (c) 2004-2005 The Trustees of Indiana University.
|
||||
* All rights reserved.
|
||||
* Copyright (c) 2004-2005 The Trustees of the University of Tennessee.
|
||||
* All rights reserved.
|
||||
* Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
|
||||
* University of Stuttgart. All rights reserved.
|
||||
* Copyright (c) 2004-2005 The Regents of the University of California.
|
||||
* All rights reserved.
|
||||
* $COPYRIGHT$
|
||||
*
|
||||
* Additional copyrights may follow
|
||||
*
|
||||
* $HEADER$
|
||||
*/
|
||||
|
||||
#include "ompi_config.h"
|
||||
|
||||
#include <string.h>
|
||||
|
||||
#include "mca/pml/pml.h"
|
||||
#include "pml_ptl_array.h"
|
||||
|
||||
|
||||
static void mca_ptl_array_construct(mca_ptl_array_t* array)
|
||||
{
|
||||
array->ptl_procs = 0;
|
||||
array->ptl_size = 0;
|
||||
array->ptl_index = 0;
|
||||
array->ptl_reserve = 0;
|
||||
}
|
||||
|
||||
|
||||
static void mca_ptl_array_destruct(mca_ptl_array_t* array)
|
||||
{
|
||||
if(array->ptl_procs != 0)
|
||||
free(array->ptl_procs);
|
||||
}
|
||||
|
||||
OBJ_CLASS_INSTANCE(
|
||||
mca_pml_teg_ptl_array_t,
|
||||
ompi_object_t,
|
||||
mca_ptl_array_construct,
|
||||
mca_ptl_array_destruct
|
||||
);
|
||||
|
||||
int mca_ptl_array_reserve(mca_ptl_array_t* array, size_t size)
|
||||
{
|
||||
mca_ptl_proc_t *procs;
|
||||
if(array->ptl_reserve >= size)
|
||||
return OMPI_SUCCESS;
|
||||
|
||||
procs = (mca_ptl_proc_t *)realloc(array->ptl_procs, sizeof(mca_ptl_proc_t)*size);
|
||||
if(NULL == procs)
|
||||
return OMPI_ERR_OUT_OF_RESOURCE;
|
||||
array->ptl_procs = procs;
|
||||
array->ptl_reserve = size;
|
||||
memset(array->ptl_procs+array->ptl_size, 0, (size-array->ptl_size)*sizeof(mca_ptl_proc_t));
|
||||
return OMPI_SUCCESS;
|
||||
}
|
||||
|
||||
@ -1,146 +0,0 @@
|
||||
/*
|
||||
* Copyright (c) 2004-2005 The Trustees of Indiana University.
|
||||
* All rights reserved.
|
||||
* Copyright (c) 2004-2005 The Trustees of the University of Tennessee.
|
||||
* All rights reserved.
|
||||
* Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
|
||||
* University of Stuttgart. All rights reserved.
|
||||
* Copyright (c) 2004-2005 The Regents of the University of California.
|
||||
* All rights reserved.
|
||||
* $COPYRIGHT$
|
||||
*
|
||||
* Additional copyrights may follow
|
||||
*
|
||||
* $HEADER$
|
||||
*/
|
||||
/**
|
||||
* @file
|
||||
*/
|
||||
#ifndef OMPI_PTL_ARRAY_H
|
||||
#define OMPI_PTL_ARRAY_H
|
||||
|
||||
#include "util/output.h"
|
||||
#include "mca/ptl/ptl.h"
|
||||
#if defined(c_plusplus) || defined(__cplusplus)
|
||||
extern "C" {
|
||||
#endif
|
||||
|
||||
extern ompi_class_t mca_pml_teg_ptl_array_t_class;
|
||||
|
||||
/**
|
||||
* A data structure associated with a ompi_proc_t that caches
|
||||
* addressing/scheduling attributes for a specific PTL instance
|
||||
* that can be used to reach the process.
|
||||
*/
|
||||
struct mca_ptl_proc_t {
|
||||
int ptl_weight; /**< PTL weight for scheduling */
|
||||
struct mca_ptl_base_peer_t* ptl_peer; /**< PTL addressing info */
|
||||
struct mca_pml_base_ptl_t* ptl_base; /**< PML specific PTL info */
|
||||
mca_ptl_base_module_t *ptl; /**< PTL module */
|
||||
};
|
||||
typedef struct mca_ptl_proc_t mca_ptl_proc_t;
|
||||
|
||||
/**
|
||||
* A dynamically growable array of mca_ptl_proc_t instances.
|
||||
* Maintains an index into the array that is used for round-robin
|
||||
* scheduling across contents.
|
||||
*/
|
||||
struct mca_ptl_array_t {
|
||||
ompi_object_t super;
|
||||
mca_ptl_proc_t* ptl_procs; /**< array of ptl procs */
|
||||
size_t ptl_size; /**< number available */
|
||||
size_t ptl_reserve; /**< size of allocated ptl_proc array */
|
||||
size_t ptl_index; /**< last used index*/
|
||||
};
|
||||
typedef struct mca_ptl_array_t mca_ptl_array_t;
|
||||
typedef struct mca_ptl_array_t mca_pml_teg_ptl_array_t;
|
||||
|
||||
|
||||
/**
|
||||
* If required, reallocate (grow) the array to the indicate size.
|
||||
*
|
||||
* @param array (IN)
|
||||
* @param size (IN)
|
||||
*/
|
||||
int mca_ptl_array_reserve(mca_ptl_array_t*, size_t);
|
||||
|
||||
static inline size_t mca_ptl_array_get_size(mca_ptl_array_t* array)
|
||||
{
|
||||
return array->ptl_size;
|
||||
}
|
||||
|
||||
/**
|
||||
* Grow the array if required, and set the size.
|
||||
*
|
||||
* @param array (IN)
|
||||
* @param size (IN)
|
||||
*/
|
||||
static inline void mca_ptl_array_set_size(mca_ptl_array_t* array, size_t size)
|
||||
{
|
||||
if(array->ptl_size > array->ptl_reserve)
|
||||
mca_ptl_array_reserve(array, size);
|
||||
array->ptl_size = size;
|
||||
}
|
||||
|
||||
/**
|
||||
* Grow the array size by one and return the item at that index.
|
||||
*
|
||||
* @param array (IN)
|
||||
*/
|
||||
static inline mca_ptl_proc_t* mca_ptl_array_insert(mca_ptl_array_t* array)
|
||||
{
|
||||
#if OMPI_ENABLE_DEBUG
|
||||
if(array->ptl_size >= array->ptl_reserve) {
|
||||
ompi_output(0, "mca_ptl_array_insert: invalid array index %d >= %d",
|
||||
array->ptl_size, array->ptl_reserve);
|
||||
return 0;
|
||||
}
|
||||
#endif
|
||||
return &array->ptl_procs[array->ptl_size++];
|
||||
}
|
||||
|
||||
/**
|
||||
* Return an array item at the specified index.
|
||||
*
|
||||
* @param array (IN)
|
||||
* @param index (IN)
|
||||
*/
|
||||
static inline mca_ptl_proc_t* mca_ptl_array_get_index(mca_ptl_array_t* array, size_t index)
|
||||
{
|
||||
#if OMPI_ENABLE_DEBUG
|
||||
if(index >= array->ptl_size) {
|
||||
ompi_output(0, "mca_ptl_array_get_index: invalid array index %d >= %d",
|
||||
index, array->ptl_size);
|
||||
return 0;
|
||||
}
|
||||
#endif
|
||||
return &array->ptl_procs[index];
|
||||
}
|
||||
|
||||
/**
|
||||
* Return the next LRU index in the array.
|
||||
*
|
||||
* @param array (IN)
|
||||
* @param index (IN)
|
||||
*/
|
||||
static inline mca_ptl_proc_t* mca_ptl_array_get_next(mca_ptl_array_t* array)
|
||||
{
|
||||
mca_ptl_proc_t* ptl_proc;
|
||||
#if OMPI_ENABLE_DEBUG
|
||||
if(array->ptl_size == 0) {
|
||||
ompi_output(0, "mca_ptl_array_get_next: invalid array size");
|
||||
return 0;
|
||||
}
|
||||
#endif
|
||||
ptl_proc = &array->ptl_procs[array->ptl_index++];
|
||||
if(array->ptl_index == array->ptl_size)
|
||||
array->ptl_index = 0;
|
||||
return ptl_proc;
|
||||
}
|
||||
|
||||
#if defined(c_plusplus) || defined(__cplusplus)
|
||||
}
|
||||
#endif
|
||||
|
||||
#endif
|
||||
|
||||
@ -1,436 +0,0 @@
|
||||
/*
|
||||
* Copyright (c) 2004-2005 The Trustees of Indiana University.
|
||||
* All rights reserved.
|
||||
* Copyright (c) 2004-2005 The Trustees of the University of Tennessee.
|
||||
* All rights reserved.
|
||||
* Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
|
||||
* University of Stuttgart. All rights reserved.
|
||||
* Copyright (c) 2004-2005 The Regents of the University of California.
|
||||
* All rights reserved.
|
||||
* $COPYRIGHT$
|
||||
*
|
||||
* Additional copyrights may follow
|
||||
*
|
||||
* $HEADER$
|
||||
*/
|
||||
|
||||
#include "ompi_config.h"
|
||||
|
||||
#include <stdlib.h>
|
||||
#include <string.h>
|
||||
|
||||
#include "class/ompi_bitmap.h"
|
||||
#include "mca/pml/pml.h"
|
||||
#include "mca/ptl/ptl.h"
|
||||
#include "mca/ptl/base/base.h"
|
||||
#include "mca/ptl/base/ptl_base_comm.h"
|
||||
#include "mca/ptl/base/ptl_base_header.h"
|
||||
#include "mca/ptl/base/ptl_base_recvfrag.h"
|
||||
#include "mca/ptl/base/ptl_base_sendfrag.h"
|
||||
#include "pml_teg.h"
|
||||
#include "pml_teg_component.h"
|
||||
#include "pml_teg_proc.h"
|
||||
#include "pml_teg_ptl.h"
|
||||
#include "pml_teg_recvreq.h"
|
||||
#include "pml_teg_sendreq.h"
|
||||
#include "pml_teg_recvfrag.h"
|
||||
|
||||
|
||||
mca_pml_teg_t mca_pml_teg = {
|
||||
{
|
||||
mca_pml_teg_add_procs,
|
||||
mca_pml_teg_del_procs,
|
||||
mca_pml_teg_add_ptls,
|
||||
mca_pml_teg_control,
|
||||
mca_pml_teg_progress,
|
||||
mca_pml_teg_add_comm,
|
||||
mca_pml_teg_del_comm,
|
||||
mca_pml_teg_irecv_init,
|
||||
mca_pml_teg_irecv,
|
||||
mca_pml_teg_recv,
|
||||
mca_pml_teg_isend_init,
|
||||
mca_pml_teg_isend,
|
||||
mca_pml_teg_send,
|
||||
mca_pml_teg_iprobe,
|
||||
mca_pml_teg_probe,
|
||||
mca_pml_teg_start
|
||||
}
|
||||
};
|
||||
|
||||
|
||||
int mca_pml_teg_add_comm(ompi_communicator_t* comm)
|
||||
{
|
||||
/* allocate pml specific comm data */
|
||||
mca_pml_ptl_comm_t* pml_comm = OBJ_NEW(mca_pml_ptl_comm_t);
|
||||
if (NULL == pml_comm) {
|
||||
return OMPI_ERR_OUT_OF_RESOURCE;
|
||||
}
|
||||
mca_pml_ptl_comm_init_size(pml_comm, comm->c_remote_group->grp_proc_count);
|
||||
comm->c_pml_comm = pml_comm;
|
||||
return OMPI_SUCCESS;
|
||||
}
|
||||
|
||||
int mca_pml_teg_del_comm(ompi_communicator_t* comm)
|
||||
{
|
||||
OBJ_RELEASE(comm->c_pml_comm);
|
||||
comm->c_pml_comm = 0;
|
||||
return OMPI_SUCCESS;
|
||||
}
|
||||
|
||||
static int ptl_exclusivity_compare(const void* arg1, const void* arg2)
|
||||
{
|
||||
mca_ptl_base_module_t* ptl1 = *(struct mca_ptl_base_module_t**)arg1;
|
||||
mca_ptl_base_module_t* ptl2 = *(struct mca_ptl_base_module_t**)arg2;
|
||||
if( ptl1->ptl_exclusivity > ptl2->ptl_exclusivity ) {
|
||||
return -1;
|
||||
} else if (ptl1->ptl_exclusivity == ptl2->ptl_exclusivity ) {
|
||||
return 0;
|
||||
} else {
|
||||
return 1;
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
int mca_pml_teg_add_ptls(ompi_list_t *ptls)
|
||||
{
|
||||
/* build an array of ptls and ptl modules */
|
||||
mca_ptl_base_selected_module_t* selected_ptl;
|
||||
size_t num_ptls = ompi_list_get_size(ptls);
|
||||
size_t cache_bytes = 0;
|
||||
mca_pml_teg.teg_num_ptl_modules = 0;
|
||||
mca_pml_teg.teg_num_ptl_progress = 0;
|
||||
mca_pml_teg.teg_num_ptl_components = 0;
|
||||
mca_pml_teg.teg_ptl_modules = (mca_ptl_base_module_t **)malloc(sizeof(mca_ptl_base_module_t*) * num_ptls);
|
||||
mca_pml_teg.teg_ptl_progress = (mca_ptl_base_component_progress_fn_t*)malloc(sizeof(mca_ptl_base_component_progress_fn_t) * num_ptls);
|
||||
mca_pml_teg.teg_ptl_components = (mca_ptl_base_component_t **)malloc(sizeof(mca_ptl_base_component_t*) * num_ptls);
|
||||
if (NULL == mca_pml_teg.teg_ptl_modules ||
|
||||
NULL == mca_pml_teg.teg_ptl_progress ||
|
||||
NULL == mca_pml_teg.teg_ptl_components) {
|
||||
return OMPI_ERR_OUT_OF_RESOURCE;
|
||||
}
|
||||
|
||||
for(selected_ptl = (mca_ptl_base_selected_module_t*)ompi_list_get_first(ptls);
|
||||
selected_ptl != (mca_ptl_base_selected_module_t*)ompi_list_get_end(ptls);
|
||||
selected_ptl = (mca_ptl_base_selected_module_t*)ompi_list_get_next(selected_ptl)) {
|
||||
mca_ptl_base_module_t *ptl = selected_ptl->pbsm_module;
|
||||
size_t i;
|
||||
|
||||
mca_pml_teg.teg_ptl_modules[mca_pml_teg.teg_num_ptl_modules++] = ptl;
|
||||
for(i=0; i<mca_pml_teg.teg_num_ptl_components; i++) {
|
||||
if(mca_pml_teg.teg_ptl_components[i] == ptl->ptl_component) {
|
||||
break;
|
||||
}
|
||||
}
|
||||
if(i == mca_pml_teg.teg_num_ptl_components) {
|
||||
mca_pml_teg.teg_ptl_components[mca_pml_teg.teg_num_ptl_components++] = ptl->ptl_component;
|
||||
}
|
||||
|
||||
/*
|
||||
*setup ptl
|
||||
*/
|
||||
|
||||
/* set pointer to fragment matching logic routine, if this
|
||||
* not already set by the ptl */
|
||||
if( NULL == ptl->ptl_match)
|
||||
ptl->ptl_match = mca_pml_teg_recv_frag_match;
|
||||
ptl->ptl_send_progress = mca_pml_teg_send_request_progress;
|
||||
ptl->ptl_recv_progress = mca_pml_teg_recv_request_progress;
|
||||
ptl->ptl_stack = ptl;
|
||||
ptl->ptl_base = NULL;
|
||||
|
||||
/* find maximum required size for cache */
|
||||
if(ptl->ptl_cache_bytes > cache_bytes) {
|
||||
cache_bytes = ptl->ptl_cache_bytes;
|
||||
}
|
||||
}
|
||||
|
||||
/* setup send fragments based on largest required send request */
|
||||
ompi_free_list_init(
|
||||
&mca_pml_teg.teg_send_requests,
|
||||
sizeof(mca_pml_teg_send_request_t) + cache_bytes,
|
||||
OBJ_CLASS(mca_pml_teg_send_request_t),
|
||||
mca_pml_teg.teg_free_list_num,
|
||||
mca_pml_teg.teg_free_list_max,
|
||||
mca_pml_teg.teg_free_list_inc,
|
||||
NULL);
|
||||
|
||||
/* sort ptl list by exclusivity */
|
||||
qsort(mca_pml_teg.teg_ptl_modules, mca_pml_teg.teg_num_ptl_modules, sizeof(struct mca_ptl_t*), ptl_exclusivity_compare);
|
||||
return OMPI_SUCCESS;
|
||||
}
|
||||
|
||||
/*
|
||||
* Pass control information through to all PTL modules.
|
||||
*/
|
||||
|
||||
int mca_pml_teg_control(int param, void* value, size_t size)
|
||||
{
|
||||
size_t i=0;
|
||||
for(i=0; i<mca_pml_teg.teg_num_ptl_components; i++) {
|
||||
if(NULL != mca_pml_teg.teg_ptl_components[i]->ptlm_control) {
|
||||
int rc = mca_pml_teg.teg_ptl_components[i]->ptlm_control(param,value,size);
|
||||
if(rc != OMPI_SUCCESS)
|
||||
return rc;
|
||||
}
|
||||
}
|
||||
return OMPI_SUCCESS;
|
||||
}
|
||||
|
||||
/*
|
||||
* For each proc setup a datastructure that indicates the PTLs
|
||||
* that can be used to reach the destination.
|
||||
*
|
||||
*/
|
||||
|
||||
int mca_pml_teg_add_procs(ompi_proc_t** procs, size_t nprocs)
|
||||
{
|
||||
size_t p;
|
||||
ompi_bitmap_t reachable;
|
||||
struct mca_ptl_base_peer_t** ptl_peers = NULL;
|
||||
int rc;
|
||||
size_t p_index;
|
||||
|
||||
if(nprocs == 0)
|
||||
return OMPI_SUCCESS;
|
||||
|
||||
OBJ_CONSTRUCT(&reachable, ompi_bitmap_t);
|
||||
rc = ompi_bitmap_init(&reachable, nprocs);
|
||||
if(OMPI_SUCCESS != rc)
|
||||
return rc;
|
||||
|
||||
/* iterate through each of the procs and set the peers architecture */
|
||||
for(p=0; p<nprocs; p++) {
|
||||
uint32_t* proc_arch;
|
||||
size_t size = sizeof(uint32_t);
|
||||
rc = mca_base_modex_recv(&mca_pml_teg_component.pmlm_version, procs[p],
|
||||
(void**)&proc_arch, &size);
|
||||
if(rc != OMPI_SUCCESS)
|
||||
return rc;
|
||||
if(size != sizeof(uint32_t))
|
||||
return OMPI_ERROR;
|
||||
procs[p]->proc_arch = ntohl(*proc_arch);
|
||||
free(proc_arch);
|
||||
}
|
||||
|
||||
/* attempt to add all procs to each ptl */
|
||||
ptl_peers = (struct mca_ptl_base_peer_t **)malloc(nprocs * sizeof(struct mca_ptl_base_peer_t*));
|
||||
for(p_index = 0; p_index < mca_pml_teg.teg_num_ptl_modules; p_index++) {
|
||||
mca_ptl_base_module_t* ptl = mca_pml_teg.teg_ptl_modules[p_index];
|
||||
int ptl_inuse = 0;
|
||||
|
||||
/* if the ptl can reach the destination proc it sets the
|
||||
* corresponding bit (proc index) in the reachable bitmap
|
||||
* and can return addressing information for each proc
|
||||
* that is passed back to the ptl on data transfer calls
|
||||
*/
|
||||
ompi_bitmap_clear_all_bits(&reachable);
|
||||
memset(ptl_peers, 0, nprocs * sizeof(struct mca_ptl_base_peer_t*));
|
||||
rc = ptl->ptl_add_procs(ptl, nprocs, procs, ptl_peers, &reachable);
|
||||
if(OMPI_SUCCESS != rc) {
|
||||
free(ptl_peers);
|
||||
return rc;
|
||||
}
|
||||
|
||||
/* for each proc that is reachable - add the ptl to the procs array(s) */
|
||||
for(p=0; p<nprocs; p++) {
|
||||
if(ompi_bitmap_is_set_bit(&reachable, p)) {
|
||||
ompi_proc_t *proc = procs[p];
|
||||
mca_pml_proc_t* proc_pml = proc->proc_pml;
|
||||
mca_ptl_proc_t* proc_ptl;
|
||||
size_t size;
|
||||
|
||||
/* this ptl can be used */
|
||||
ptl_inuse++;
|
||||
|
||||
/* initialize each proc */
|
||||
if(NULL == proc_pml) {
|
||||
|
||||
/* allocate pml specific proc data */
|
||||
proc_pml = OBJ_NEW(mca_pml_teg_proc_t);
|
||||
if (NULL == proc_pml) {
|
||||
ompi_output(0, "mca_pml_teg_add_procs: unable to allocate resources");
|
||||
free(ptl_peers);
|
||||
return OMPI_ERR_OUT_OF_RESOURCE;
|
||||
}
|
||||
|
||||
/* preallocate space in array for max number of ptls */
|
||||
mca_ptl_array_reserve(&proc_pml->proc_ptl_first, mca_pml_teg.teg_num_ptl_modules);
|
||||
mca_ptl_array_reserve(&proc_pml->proc_ptl_next, mca_pml_teg.teg_num_ptl_modules);
|
||||
proc_pml->proc_ompi = proc;
|
||||
proc->proc_pml = proc_pml;
|
||||
}
|
||||
|
||||
/* dont allow an additional PTL with a lower exclusivity ranking */
|
||||
size = mca_ptl_array_get_size(&proc_pml->proc_ptl_next);
|
||||
if(size > 0) {
|
||||
proc_ptl = mca_ptl_array_get_index(&proc_pml->proc_ptl_next, size-1);
|
||||
/* skip this ptl if the exclusivity is less than the previous */
|
||||
if(proc_ptl->ptl->ptl_exclusivity > ptl->ptl_exclusivity) {
|
||||
if(ptl_peers[p] != NULL) {
|
||||
ptl->ptl_del_procs(ptl, 1, &proc, &ptl_peers[p]);
|
||||
}
|
||||
continue;
|
||||
}
|
||||
}
|
||||
|
||||
/* cache the ptl on the proc */
|
||||
proc_ptl = mca_ptl_array_insert(&proc_pml->proc_ptl_next);
|
||||
proc_ptl->ptl = ptl;
|
||||
proc_ptl->ptl_peer = ptl_peers[p];
|
||||
proc_ptl->ptl_weight = 0;
|
||||
proc_pml->proc_ptl_flags |= ptl->ptl_flags;
|
||||
}
|
||||
}
|
||||
if(ptl_inuse > 0 && NULL != ptl->ptl_component->ptlm_progress) {
|
||||
size_t p;
|
||||
bool found = false;
|
||||
for(p=0; p<mca_pml_teg.teg_num_ptl_progress; p++) {
|
||||
if(mca_pml_teg.teg_ptl_progress[p] == ptl->ptl_component->ptlm_progress) {
|
||||
found = true;
|
||||
break;
|
||||
}
|
||||
}
|
||||
if(found == false) {
|
||||
mca_pml_teg.teg_ptl_progress[mca_pml_teg.teg_num_ptl_progress] =
|
||||
ptl->ptl_component->ptlm_progress;
|
||||
mca_pml_teg.teg_num_ptl_progress++;
|
||||
}
|
||||
}
|
||||
}
|
||||
free(ptl_peers);
|
||||
|
||||
/* iterate back through procs and compute metrics for registered ptls */
|
||||
for(p=0; p<nprocs; p++) {
|
||||
ompi_proc_t *proc = procs[p];
|
||||
mca_pml_proc_t* proc_pml = proc->proc_pml;
|
||||
double total_bandwidth = 0;
|
||||
uint32_t latency = 0;
|
||||
size_t n_index;
|
||||
size_t n_size;
|
||||
|
||||
/* skip over procs w/ no ptls registered */
|
||||
if(NULL == proc_pml)
|
||||
continue;
|
||||
|
||||
/* (1) determine the total bandwidth available across all ptls
|
||||
* note that we need to do this here, as we may already have ptls configured
|
||||
* (2) determine the highest priority ranking for latency
|
||||
*/
|
||||
n_size = mca_ptl_array_get_size(&proc_pml->proc_ptl_next);
|
||||
for(n_index = 0; n_index < n_size; n_index++) {
|
||||
struct mca_ptl_proc_t* proc_ptl = mca_ptl_array_get_index(&proc_pml->proc_ptl_next, n_index);
|
||||
struct mca_ptl_base_module_t* ptl = proc_ptl->ptl;
|
||||
total_bandwidth += proc_ptl->ptl->ptl_bandwidth;
|
||||
if(ptl->ptl_latency > latency)
|
||||
latency = ptl->ptl_latency;
|
||||
}
|
||||
|
||||
/* (1) set the weight of each ptl as a percentage of overall bandwidth
|
||||
* (2) copy all ptl instances at the highest priority ranking into the
|
||||
* list of ptls used for first fragments
|
||||
*/
|
||||
|
||||
for(n_index = 0; n_index < n_size; n_index++) {
|
||||
struct mca_ptl_proc_t* proc_ptl = mca_ptl_array_get_index(&proc_pml->proc_ptl_next, n_index);
|
||||
struct mca_ptl_base_module_t *ptl = proc_ptl->ptl;
|
||||
double weight;
|
||||
|
||||
/* compute weighting factor for this ptl */
|
||||
if(ptl->ptl_bandwidth)
|
||||
weight = proc_ptl->ptl->ptl_bandwidth / total_bandwidth;
|
||||
else
|
||||
weight = 1.0 / n_size;
|
||||
proc_ptl->ptl_weight = (int)(weight * 100);
|
||||
|
||||
/*
|
||||
* save/create ptl extension for use by pml
|
||||
*/
|
||||
proc_ptl->ptl_base = ptl->ptl_base;
|
||||
if (NULL == proc_ptl->ptl_base &&
|
||||
ptl->ptl_cache_bytes > 0 &&
|
||||
NULL != ptl->ptl_request_init &&
|
||||
NULL != ptl->ptl_request_fini) {
|
||||
|
||||
mca_pml_base_ptl_t* ptl_base = OBJ_NEW(mca_pml_base_ptl_t);
|
||||
ptl_base->ptl = ptl;
|
||||
ptl_base->ptl_cache_size = ptl->ptl_cache_size;
|
||||
proc_ptl->ptl_base = ptl->ptl_base = ptl_base;
|
||||
}
|
||||
|
||||
/* check to see if this ptl is already in the array of ptls used for first
|
||||
* fragments - if not add it.
|
||||
*/
|
||||
if(ptl->ptl_latency == latency) {
|
||||
struct mca_ptl_proc_t* proc_new = mca_ptl_array_insert(&proc_pml->proc_ptl_first);
|
||||
*proc_new = *proc_ptl;
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
return OMPI_SUCCESS;
|
||||
}
|
||||
|
||||
/*
|
||||
* iterate through each proc and notify any PTLs associated
|
||||
* with the proc that it is/has gone away
|
||||
*/
|
||||
|
||||
int mca_pml_teg_del_procs(ompi_proc_t** procs, size_t nprocs)
|
||||
{
|
||||
size_t p;
|
||||
int rc;
|
||||
for(p = 0; p < nprocs; p++) {
|
||||
ompi_proc_t *proc = procs[p];
|
||||
mca_pml_proc_t* proc_pml = proc->proc_pml;
|
||||
size_t f_index, f_size;
|
||||
size_t n_index, n_size;
|
||||
|
||||
/* notify each ptl that the proc is going away */
|
||||
f_size = mca_ptl_array_get_size(&proc_pml->proc_ptl_first);
|
||||
for(f_index = 0; f_index < f_size; f_index++) {
|
||||
mca_ptl_proc_t* ptl_proc = mca_ptl_array_get_index(&proc_pml->proc_ptl_first, f_index);
|
||||
mca_ptl_base_module_t* ptl = ptl_proc->ptl;
|
||||
|
||||
rc = ptl->ptl_del_procs(ptl,1,&proc,&ptl_proc->ptl_peer);
|
||||
if(OMPI_SUCCESS != rc) {
|
||||
return rc;
|
||||
}
|
||||
|
||||
/* remove this from next array so that we dont call it twice w/
|
||||
* the same address pointer
|
||||
*/
|
||||
n_size = mca_ptl_array_get_size(&proc_pml->proc_ptl_first);
|
||||
for(n_index = 0; n_index < n_size; n_index++) {
|
||||
mca_ptl_proc_t* next_proc = mca_ptl_array_get_index(&proc_pml->proc_ptl_next, n_index);
|
||||
if(next_proc->ptl == ptl) {
|
||||
memset(next_proc, 0, sizeof(mca_ptl_proc_t));
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/* notify each ptl that was not in the array of ptls for first fragments */
|
||||
n_size = mca_ptl_array_get_size(&proc_pml->proc_ptl_next);
|
||||
for(n_index = 0; n_index < n_size; n_index++) {
|
||||
mca_ptl_proc_t* ptl_proc = mca_ptl_array_get_index(&proc_pml->proc_ptl_first, n_index);
|
||||
mca_ptl_base_module_t* ptl = ptl_proc->ptl;
|
||||
if (ptl != 0) {
|
||||
rc = ptl->ptl_del_procs(ptl,1,&proc,&ptl_proc->ptl_peer);
|
||||
if(OMPI_SUCCESS != rc)
|
||||
return rc;
|
||||
}
|
||||
}
|
||||
|
||||
/* do any required cleanup */
|
||||
OBJ_RELEASE(proc_pml);
|
||||
proc->proc_pml = NULL;
|
||||
}
|
||||
return OMPI_SUCCESS;
|
||||
}
|
||||
|
||||
int mca_pml_teg_component_fini(void)
|
||||
{
|
||||
/* FIX */
|
||||
return OMPI_SUCCESS;
|
||||
}
|
||||
|
||||
@ -1,268 +0,0 @@
|
||||
/*
|
||||
* Copyright (c) 2004-2005 The Trustees of Indiana University.
|
||||
* All rights reserved.
|
||||
* Copyright (c) 2004-2005 The Trustees of the University of Tennessee.
|
||||
* All rights reserved.
|
||||
* Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
|
||||
* University of Stuttgart. All rights reserved.
|
||||
* Copyright (c) 2004-2005 The Regents of the University of California.
|
||||
* All rights reserved.
|
||||
* $COPYRIGHT$
|
||||
*
|
||||
* Additional copyrights may follow
|
||||
*
|
||||
* $HEADER$
|
||||
*/
|
||||
/**
|
||||
* @file
|
||||
*/
|
||||
|
||||
#ifndef MCA_PML_TEG_H
|
||||
#define MCA_PML_TEG_H
|
||||
|
||||
#include "threads/thread.h"
|
||||
#include "threads/condition.h"
|
||||
#include "class/ompi_free_list.h"
|
||||
#include "util/cmd_line.h"
|
||||
#include "request/request.h"
|
||||
#include "mca/pml/pml.h"
|
||||
#include "mca/pml/base/pml_base_request.h"
|
||||
#include "mca/pml/base/pml_base_bsend.h"
|
||||
#include "mca/pml/base/pml_base_sendreq.h"
|
||||
#include "mca/ptl/ptl.h"
|
||||
|
||||
|
||||
#if defined(c_plusplus) || defined(__cplusplus)
|
||||
extern "C" {
|
||||
#endif
|
||||
/**
|
||||
* TEG PML module
|
||||
*/
|
||||
|
||||
struct mca_pml_teg_t {
|
||||
mca_pml_base_module_t super;
|
||||
|
||||
mca_ptl_base_component_t **teg_ptl_components;
|
||||
size_t teg_num_ptl_components;
|
||||
|
||||
mca_ptl_base_module_t** teg_ptl_modules;
|
||||
size_t teg_num_ptl_modules;
|
||||
|
||||
mca_ptl_base_component_progress_fn_t* teg_ptl_progress;
|
||||
size_t teg_num_ptl_progress;
|
||||
|
||||
ompi_list_t teg_procs;
|
||||
ompi_mutex_t teg_lock;
|
||||
|
||||
int teg_priority;
|
||||
|
||||
int teg_free_list_num; /* initial size of free list */
|
||||
int teg_free_list_max; /* maximum size of free list */
|
||||
int teg_free_list_inc; /* number of elements to grow free list */
|
||||
int teg_poll_iterations; /* number of iterations to poll for completion */
|
||||
|
||||
/* free list of requests */
|
||||
ompi_free_list_t teg_send_requests;
|
||||
ompi_free_list_t teg_recv_requests;
|
||||
|
||||
/* list of pending send requests */
|
||||
ompi_list_t teg_send_pending;
|
||||
};
|
||||
typedef struct mca_pml_teg_t mca_pml_teg_t;
|
||||
|
||||
extern mca_pml_teg_t mca_pml_teg;
|
||||
|
||||
|
||||
/*
|
||||
* PML module functions.
|
||||
*/
|
||||
|
||||
|
||||
extern int mca_pml_teg_component_open(void);
|
||||
extern int mca_pml_teg_component_close(void);
|
||||
|
||||
extern mca_pml_base_module_t* mca_pml_teg_component_init(
|
||||
int *priority,
|
||||
bool enable_progress_threads,
|
||||
bool enable_mpi_threads
|
||||
);
|
||||
|
||||
extern int mca_pml_teg_component_fini(void);
|
||||
|
||||
|
||||
|
||||
/*
|
||||
* PML interface functions.
|
||||
*/
|
||||
|
||||
extern int mca_pml_teg_add_comm(
|
||||
struct ompi_communicator_t* comm
|
||||
);
|
||||
|
||||
extern int mca_pml_teg_del_comm(
|
||||
struct ompi_communicator_t* comm
|
||||
);
|
||||
|
||||
extern int mca_pml_teg_add_procs(
|
||||
struct ompi_proc_t **procs,
|
||||
size_t nprocs
|
||||
);
|
||||
|
||||
extern int mca_pml_teg_del_procs(
|
||||
struct ompi_proc_t **procs,
|
||||
size_t nprocs
|
||||
);
|
||||
|
||||
extern int mca_pml_teg_add_ptls(
|
||||
ompi_list_t *ptls
|
||||
);
|
||||
|
||||
extern int mca_pml_teg_control(
|
||||
int param,
|
||||
void *size,
|
||||
size_t value
|
||||
);
|
||||
|
||||
extern int mca_pml_teg_progress(void);
|
||||
|
||||
extern int mca_pml_teg_iprobe(
|
||||
int dst,
|
||||
int tag,
|
||||
struct ompi_communicator_t* comm,
|
||||
int *matched,
|
||||
ompi_status_public_t* status
|
||||
);
|
||||
|
||||
extern int mca_pml_teg_probe(
|
||||
int dst,
|
||||
int tag,
|
||||
struct ompi_communicator_t* comm,
|
||||
ompi_status_public_t* status
|
||||
);
|
||||
|
||||
extern int mca_pml_teg_cancelled(
|
||||
ompi_request_t* request,
|
||||
int *flag
|
||||
);
|
||||
|
||||
|
||||
extern int mca_pml_teg_isend_init(
|
||||
void *buf,
|
||||
size_t count,
|
||||
ompi_datatype_t *datatype,
|
||||
int dst,
|
||||
int tag,
|
||||
mca_pml_base_send_mode_t mode,
|
||||
struct ompi_communicator_t* comm,
|
||||
struct ompi_request_t **request
|
||||
);
|
||||
|
||||
extern int mca_pml_teg_isend(
|
||||
void *buf,
|
||||
size_t count,
|
||||
ompi_datatype_t *datatype,
|
||||
int dst,
|
||||
int tag,
|
||||
mca_pml_base_send_mode_t mode,
|
||||
struct ompi_communicator_t* comm,
|
||||
struct ompi_request_t **request
|
||||
);
|
||||
|
||||
extern int mca_pml_teg_send(
|
||||
void *buf,
|
||||
size_t count,
|
||||
ompi_datatype_t *datatype,
|
||||
int dst,
|
||||
int tag,
|
||||
mca_pml_base_send_mode_t mode,
|
||||
struct ompi_communicator_t* comm
|
||||
);
|
||||
|
||||
extern int mca_pml_teg_irecv_init(
|
||||
void *buf,
|
||||
size_t count,
|
||||
ompi_datatype_t *datatype,
|
||||
int src,
|
||||
int tag,
|
||||
struct ompi_communicator_t* comm,
|
||||
struct ompi_request_t **request
|
||||
);
|
||||
|
||||
extern int mca_pml_teg_irecv(
|
||||
void *buf,
|
||||
size_t count,
|
||||
ompi_datatype_t *datatype,
|
||||
int src,
|
||||
int tag,
|
||||
struct ompi_communicator_t* comm,
|
||||
struct ompi_request_t **request
|
||||
);
|
||||
|
||||
extern int mca_pml_teg_recv(
|
||||
void *buf,
|
||||
size_t count,
|
||||
ompi_datatype_t *datatype,
|
||||
int src,
|
||||
int tag,
|
||||
struct ompi_communicator_t* comm,
|
||||
ompi_status_public_t* status
|
||||
);
|
||||
|
||||
extern int mca_pml_teg_progress(void);
|
||||
|
||||
extern int mca_pml_teg_start(
|
||||
size_t count,
|
||||
ompi_request_t** requests
|
||||
);
|
||||
|
||||
#if defined(c_plusplus) || defined(__cplusplus)
|
||||
}
|
||||
#endif
|
||||
#define MCA_PML_TEG_FINI(request) \
|
||||
{ \
|
||||
mca_pml_base_request_t* pml_request = *(mca_pml_base_request_t**)(request); \
|
||||
if(pml_request->req_persistent) { \
|
||||
if(pml_request->req_free_called) { \
|
||||
MCA_PML_TEG_FREE(request); \
|
||||
} else { \
|
||||
pml_request->req_ompi.req_state = OMPI_REQUEST_INACTIVE; \
|
||||
} \
|
||||
} else { \
|
||||
MCA_PML_TEG_FREE(request); \
|
||||
} \
|
||||
}
|
||||
|
||||
|
||||
#define MCA_PML_TEG_FREE(request) \
|
||||
{ \
|
||||
mca_pml_base_request_t* pml_request = *(mca_pml_base_request_t**)(request); \
|
||||
pml_request->req_free_called = true; \
|
||||
if( pml_request->req_pml_complete == true) \
|
||||
{ \
|
||||
OMPI_REQUEST_FINI(*(request)); \
|
||||
switch(pml_request->req_type) { \
|
||||
case MCA_PML_REQUEST_SEND: \
|
||||
{ \
|
||||
mca_pml_base_send_request_t* sendreq = (mca_pml_base_send_request_t*)pml_request; \
|
||||
while(sendreq->req_lock > 0); \
|
||||
if(sendreq->req_send_mode == MCA_PML_BASE_SEND_BUFFERED) { \
|
||||
mca_pml_base_bsend_request_fini((ompi_request_t*)sendreq); \
|
||||
} \
|
||||
MCA_PML_TEG_SEND_REQUEST_RETURN(sendreq); \
|
||||
break; \
|
||||
} \
|
||||
case MCA_PML_REQUEST_RECV: \
|
||||
{ \
|
||||
mca_pml_base_recv_request_t* recvreq = (mca_pml_base_recv_request_t*)pml_request; \
|
||||
MCA_PML_TEG_RECV_REQUEST_RETURN(recvreq); \
|
||||
break; \
|
||||
} \
|
||||
default: \
|
||||
break; \
|
||||
} \
|
||||
} \
|
||||
*(request) = MPI_REQUEST_NULL; \
|
||||
}
|
||||
|
||||
#endif
|
||||
|
||||
@ -1,28 +0,0 @@
|
||||
/* -*- Mode: C; c-basic-offset:4 ; -*- */
|
||||
/*
|
||||
* Copyright (c) 2004-2005 The Trustees of Indiana University.
|
||||
* All rights reserved.
|
||||
* Copyright (c) 2004-2005 The Trustees of the University of Tennessee.
|
||||
* All rights reserved.
|
||||
* Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
|
||||
* University of Stuttgart. All rights reserved.
|
||||
* Copyright (c) 2004-2005 The Regents of the University of California.
|
||||
* All rights reserved.
|
||||
* $COPYRIGHT$
|
||||
*
|
||||
* Additional copyrights may follow
|
||||
*
|
||||
* $HEADER$
|
||||
*/
|
||||
|
||||
#include "ompi_config.h"
|
||||
|
||||
#include "pml_teg.h"
|
||||
|
||||
int mca_pml_teg_cancelled(ompi_request_t* request, int* flag)
|
||||
{
|
||||
if(NULL != flag)
|
||||
*flag = (true == request->req_status._cancelled ? 1 : 0);
|
||||
return OMPI_SUCCESS;
|
||||
}
|
||||
|
||||
@ -1,177 +0,0 @@
|
||||
/*
|
||||
* Copyright (c) 2004-2005 The Trustees of Indiana University.
|
||||
* All rights reserved.
|
||||
* Copyright (c) 2004-2005 The Trustees of the University of Tennessee.
|
||||
* All rights reserved.
|
||||
* Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
|
||||
* University of Stuttgart. All rights reserved.
|
||||
* Copyright (c) 2004-2005 The Regents of the University of California.
|
||||
* All rights reserved.
|
||||
* $COPYRIGHT$
|
||||
*
|
||||
* Additional copyrights may follow
|
||||
*
|
||||
* $HEADER$
|
||||
*/
|
||||
|
||||
#include "ompi_config.h"
|
||||
|
||||
#include "event/event.h"
|
||||
#include "mpi.h"
|
||||
#include "mca/pml/pml.h"
|
||||
#include "mca/ptl/ptl.h"
|
||||
#include "mca/base/mca_base_param.h"
|
||||
#include "mca/pml/base/pml_base_bsend.h"
|
||||
#include "pml_teg.h"
|
||||
#include "pml_teg_proc.h"
|
||||
#include "pml_teg_sendreq.h"
|
||||
#include "pml_teg_recvreq.h"
|
||||
|
||||
|
||||
mca_pml_base_component_1_0_0_t mca_pml_teg_component = {
|
||||
|
||||
/* First, the mca_base_component_t struct containing meta
|
||||
information about the component itself */
|
||||
|
||||
{
|
||||
/* Indicate that we are a pml v1.0.0 component (which also implies
|
||||
a specific MCA version) */
|
||||
|
||||
MCA_PML_BASE_VERSION_1_0_0,
|
||||
|
||||
"teg", /* MCA component name */
|
||||
1, /* MCA component major version */
|
||||
0, /* MCA component minor version */
|
||||
0, /* MCA component release version */
|
||||
mca_pml_teg_component_open, /* component open */
|
||||
mca_pml_teg_component_close /* component close */
|
||||
},
|
||||
|
||||
/* Next the MCA v1.0.0 component meta data */
|
||||
|
||||
{
|
||||
/* Whether the component is checkpointable or not */
|
||||
false
|
||||
},
|
||||
|
||||
mca_pml_teg_component_init, /* component init */
|
||||
mca_pml_teg_component_fini /* component finalize */
|
||||
};
|
||||
|
||||
|
||||
|
||||
static inline int mca_pml_teg_param_register_int(
|
||||
const char* param_name,
|
||||
int default_value)
|
||||
{
|
||||
int id = mca_base_param_register_int("pml","teg",param_name,NULL,default_value);
|
||||
int param_value = default_value;
|
||||
mca_base_param_lookup_int(id,¶m_value);
|
||||
return param_value;
|
||||
}
|
||||
|
||||
|
||||
int mca_pml_teg_component_open(void)
|
||||
{
|
||||
#ifdef WIN32
|
||||
WSADATA win_sock_data;
|
||||
if (WSAStartup(MAKEWORD(2,2), &win_sock_data) != 0) {
|
||||
ompi_output (0, "mca_oob_tcp_component_init: failed to initialise windows sockets: %d\n", WSAGetLastError());
|
||||
return OMPI_ERROR;
|
||||
}
|
||||
#endif
|
||||
OBJ_CONSTRUCT(&mca_pml_teg.teg_lock, ompi_mutex_t);
|
||||
OBJ_CONSTRUCT(&mca_pml_teg.teg_send_requests, ompi_free_list_t);
|
||||
OBJ_CONSTRUCT(&mca_pml_teg.teg_recv_requests, ompi_free_list_t);
|
||||
OBJ_CONSTRUCT(&mca_pml_teg.teg_procs, ompi_list_t);
|
||||
OBJ_CONSTRUCT(&mca_pml_teg.teg_send_pending, ompi_list_t);
|
||||
|
||||
mca_pml_teg.teg_ptl_components = NULL;
|
||||
mca_pml_teg.teg_num_ptl_components = 0;
|
||||
mca_pml_teg.teg_ptl_modules = NULL;
|
||||
mca_pml_teg.teg_num_ptl_modules = 0;
|
||||
mca_pml_teg.teg_ptl_progress = NULL;
|
||||
mca_pml_teg.teg_num_ptl_progress = 0;
|
||||
|
||||
mca_pml_teg.teg_free_list_num =
|
||||
mca_pml_teg_param_register_int("free_list_num", 256);
|
||||
mca_pml_teg.teg_free_list_max =
|
||||
mca_pml_teg_param_register_int("free_list_max", -1);
|
||||
mca_pml_teg.teg_free_list_inc =
|
||||
mca_pml_teg_param_register_int("free_list_inc", 256);
|
||||
mca_pml_teg.teg_poll_iterations =
|
||||
mca_pml_teg_param_register_int("poll_iterations", 100000);
|
||||
mca_pml_teg.teg_priority =
|
||||
mca_pml_teg_param_register_int("priority", 0);
|
||||
|
||||
return OMPI_SUCCESS;
|
||||
}
|
||||
|
||||
|
||||
int mca_pml_teg_component_close(void)
|
||||
{
|
||||
#ifdef WIN32
|
||||
WSACleanup();
|
||||
#endif
|
||||
|
||||
#if OMPI_ENABLE_DEBUG
|
||||
if (mca_pml_teg.teg_recv_requests.fl_num_allocated !=
|
||||
mca_pml_teg.teg_recv_requests.super.ompi_list_length) {
|
||||
ompi_output(0, "teg recv requests: %d allocated %d returned\n",
|
||||
mca_pml_teg.teg_recv_requests.fl_num_allocated,
|
||||
mca_pml_teg.teg_recv_requests.super.ompi_list_length);
|
||||
}
|
||||
#endif
|
||||
|
||||
if(NULL != mca_pml_teg.teg_ptl_components) {
|
||||
free(mca_pml_teg.teg_ptl_components);
|
||||
}
|
||||
if(NULL != mca_pml_teg.teg_ptl_modules) {
|
||||
free(mca_pml_teg.teg_ptl_modules);
|
||||
}
|
||||
if(NULL != mca_pml_teg.teg_ptl_progress) {
|
||||
free(mca_pml_teg.teg_ptl_progress);
|
||||
}
|
||||
OBJ_DESTRUCT(&mca_pml_teg.teg_send_pending);
|
||||
OBJ_DESTRUCT(&mca_pml_teg.teg_send_requests);
|
||||
OBJ_DESTRUCT(&mca_pml_teg.teg_recv_requests);
|
||||
OBJ_DESTRUCT(&mca_pml_teg.teg_procs);
|
||||
OBJ_DESTRUCT(&mca_pml_teg.teg_lock);
|
||||
return OMPI_SUCCESS;
|
||||
}
|
||||
|
||||
|
||||
mca_pml_base_module_t* mca_pml_teg_component_init(int* priority,
|
||||
bool enable_progress_threads,
|
||||
bool enable_mpi_threads)
|
||||
{
|
||||
uint32_t proc_arch;
|
||||
int rc;
|
||||
*priority = mca_pml_teg.teg_priority;
|
||||
|
||||
/* recv requests */
|
||||
ompi_free_list_init(
|
||||
&mca_pml_teg.teg_recv_requests,
|
||||
sizeof(mca_pml_teg_recv_request_t),
|
||||
OBJ_CLASS(mca_pml_teg_recv_request_t),
|
||||
mca_pml_teg.teg_free_list_num,
|
||||
mca_pml_teg.teg_free_list_max,
|
||||
mca_pml_teg.teg_free_list_inc,
|
||||
NULL);
|
||||
|
||||
/* buffered send */
|
||||
if(OMPI_SUCCESS != mca_pml_base_bsend_init(enable_mpi_threads)) {
|
||||
ompi_output(0, "mca_pml_teg_component_init: mca_pml_bsend_init failed\n");
|
||||
return NULL;
|
||||
}
|
||||
|
||||
/* post this processes datatype */
|
||||
proc_arch = ompi_proc_local()->proc_arch;
|
||||
proc_arch = htonl(proc_arch);
|
||||
rc = mca_base_modex_send(&mca_pml_teg_component.pmlm_version, &proc_arch, sizeof(proc_arch));
|
||||
if(rc != OMPI_SUCCESS)
|
||||
return NULL;
|
||||
|
||||
return &mca_pml_teg.super;
|
||||
}
|
||||
|
||||
@ -1,27 +0,0 @@
|
||||
/*
|
||||
* Copyright (c) 2004-2005 The Trustees of Indiana University.
|
||||
* All rights reserved.
|
||||
* Copyright (c) 2004-2005 The Trustees of the University of Tennessee.
|
||||
* All rights reserved.
|
||||
* Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
|
||||
* University of Stuttgart. All rights reserved.
|
||||
* $COPYRIGHT$
|
||||
*
|
||||
* Additional copyrights may follow
|
||||
*
|
||||
* $HEADER$
|
||||
*/
|
||||
/**
|
||||
* @file
|
||||
*/
|
||||
|
||||
#ifndef MCA_PML_TEG_COMPONENT_H
|
||||
#define MCA_PML_TEG_COMPONENT_H
|
||||
|
||||
/*
|
||||
* PML module functions.
|
||||
*/
|
||||
|
||||
OMPI_COMP_EXPORT extern mca_pml_base_component_1_0_0_t mca_pml_teg_component;
|
||||
|
||||
#endif
|
||||
@ -1,93 +0,0 @@
|
||||
/*
|
||||
* Copyright (c) 2004-2005 The Trustees of Indiana University.
|
||||
* All rights reserved.
|
||||
* Copyright (c) 2004-2005 The Trustees of the University of Tennessee.
|
||||
* All rights reserved.
|
||||
* Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
|
||||
* University of Stuttgart. All rights reserved.
|
||||
* Copyright (c) 2004-2005 The Regents of the University of California.
|
||||
* All rights reserved.
|
||||
* $COPYRIGHT$
|
||||
*
|
||||
* Additional copyrights may follow
|
||||
*
|
||||
* $HEADER$
|
||||
*/
|
||||
|
||||
#include "ompi_config.h"
|
||||
#include "request/request.h"
|
||||
#include "pml_teg_recvreq.h"
|
||||
|
||||
|
||||
int mca_pml_teg_iprobe(int src,
|
||||
int tag,
|
||||
struct ompi_communicator_t *comm,
|
||||
int *matched, ompi_status_public_t * status)
|
||||
{
|
||||
int rc;
|
||||
mca_pml_base_recv_request_t recvreq;
|
||||
|
||||
OBJ_CONSTRUCT( &(recvreq), mca_pml_base_recv_request_t );
|
||||
recvreq.req_base.req_ompi.req_type = OMPI_REQUEST_PML;
|
||||
recvreq.req_base.req_type = MCA_PML_REQUEST_IPROBE;
|
||||
MCA_PML_BASE_RECV_REQUEST_INIT(&recvreq, NULL, 0, &ompi_mpi_char, src, tag, comm, true);
|
||||
|
||||
*matched = 0;
|
||||
if ((rc = mca_pml_teg_recv_request_start(&recvreq)) == OMPI_SUCCESS) {
|
||||
if( recvreq.req_base.req_ompi.req_complete == true ) {
|
||||
if( NULL != status ) {
|
||||
*status = recvreq.req_base.req_ompi.req_status;
|
||||
}
|
||||
*matched = 1;
|
||||
} else {
|
||||
/* we are supposed to progress ... */
|
||||
ompi_progress();
|
||||
}
|
||||
}
|
||||
MCA_PML_BASE_RECV_REQUEST_RETURN( &recvreq );
|
||||
return rc;
|
||||
}
|
||||
|
||||
|
||||
int mca_pml_teg_probe(int src,
|
||||
int tag,
|
||||
struct ompi_communicator_t *comm,
|
||||
ompi_status_public_t * status)
|
||||
{
|
||||
int rc;
|
||||
mca_pml_base_recv_request_t recvreq;
|
||||
|
||||
OBJ_CONSTRUCT( &(recvreq), mca_pml_base_recv_request_t );
|
||||
recvreq.req_base.req_ompi.req_type = OMPI_REQUEST_PML;
|
||||
recvreq.req_base.req_type = MCA_PML_REQUEST_PROBE;
|
||||
MCA_PML_BASE_RECV_REQUEST_INIT(&recvreq, NULL, 0, &ompi_mpi_char, src, tag, comm, true);
|
||||
|
||||
if ((rc = mca_pml_teg_recv_request_start(&recvreq)) != OMPI_SUCCESS) {
|
||||
MCA_PML_BASE_RECV_REQUEST_RETURN( &recvreq );
|
||||
return rc;
|
||||
}
|
||||
|
||||
if (recvreq.req_base.req_ompi.req_complete == false) {
|
||||
/* give up and sleep until completion */
|
||||
if (ompi_using_threads()) {
|
||||
ompi_mutex_lock(&ompi_request_lock);
|
||||
ompi_request_waiting++;
|
||||
while (recvreq.req_base.req_ompi.req_complete == false)
|
||||
ompi_condition_wait(&ompi_request_cond, &ompi_request_lock);
|
||||
ompi_request_waiting--;
|
||||
ompi_mutex_unlock(&ompi_request_lock);
|
||||
} else {
|
||||
ompi_request_waiting++;
|
||||
while (recvreq.req_base.req_ompi.req_complete == false)
|
||||
ompi_condition_wait(&ompi_request_cond, &ompi_request_lock);
|
||||
ompi_request_waiting--;
|
||||
}
|
||||
}
|
||||
|
||||
if (NULL != status) {
|
||||
*status = recvreq.req_base.req_ompi.req_status;
|
||||
}
|
||||
MCA_PML_BASE_RECV_REQUEST_RETURN( &recvreq );
|
||||
return OMPI_SUCCESS;
|
||||
}
|
||||
|
||||
@ -1,117 +0,0 @@
|
||||
/*
|
||||
* Copyright (c) 2004-2005 The Trustees of Indiana University.
|
||||
* All rights reserved.
|
||||
* Copyright (c) 2004-2005 The Trustees of the University of Tennessee.
|
||||
* All rights reserved.
|
||||
* Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
|
||||
* University of Stuttgart. All rights reserved.
|
||||
* Copyright (c) 2004-2005 The Regents of the University of California.
|
||||
* All rights reserved.
|
||||
* $COPYRIGHT$
|
||||
*
|
||||
* Additional copyrights may follow
|
||||
*
|
||||
* $HEADER$
|
||||
*/
|
||||
|
||||
#include "ompi_config.h"
|
||||
#include "request/request.h"
|
||||
#include "pml_teg_recvreq.h"
|
||||
|
||||
|
||||
int mca_pml_teg_irecv_init(void *addr,
|
||||
size_t count,
|
||||
ompi_datatype_t * datatype,
|
||||
int src,
|
||||
int tag,
|
||||
struct ompi_communicator_t *comm,
|
||||
struct ompi_request_t **request)
|
||||
{
|
||||
int rc;
|
||||
mca_pml_base_recv_request_t *recvreq;
|
||||
MCA_PML_TEG_RECV_REQUEST_ALLOC(recvreq, rc);
|
||||
if (NULL == recvreq)
|
||||
return rc;
|
||||
|
||||
MCA_PML_BASE_RECV_REQUEST_INIT(recvreq,
|
||||
addr,
|
||||
count, datatype, src, tag, comm, true);
|
||||
|
||||
*request = (ompi_request_t *) recvreq;
|
||||
return OMPI_SUCCESS;
|
||||
}
|
||||
|
||||
int mca_pml_teg_irecv(void *addr,
|
||||
size_t count,
|
||||
ompi_datatype_t * datatype,
|
||||
int src,
|
||||
int tag,
|
||||
struct ompi_communicator_t *comm,
|
||||
struct ompi_request_t **request)
|
||||
{
|
||||
int rc;
|
||||
|
||||
mca_pml_base_recv_request_t *recvreq;
|
||||
MCA_PML_TEG_RECV_REQUEST_ALLOC(recvreq, rc);
|
||||
if (NULL == recvreq)
|
||||
return rc;
|
||||
|
||||
MCA_PML_BASE_RECV_REQUEST_INIT(recvreq,
|
||||
addr,
|
||||
count, datatype, src, tag, comm, false);
|
||||
|
||||
if ((rc = mca_pml_teg_recv_request_start(recvreq)) != OMPI_SUCCESS) {
|
||||
MCA_PML_TEG_RECV_REQUEST_RETURN(recvreq);
|
||||
return rc;
|
||||
}
|
||||
*request = (ompi_request_t *) recvreq;
|
||||
return OMPI_SUCCESS;
|
||||
}
|
||||
|
||||
|
||||
int mca_pml_teg_recv(void *addr,
|
||||
size_t count,
|
||||
ompi_datatype_t * datatype,
|
||||
int src,
|
||||
int tag,
|
||||
struct ompi_communicator_t *comm,
|
||||
ompi_status_public_t * status)
|
||||
{
|
||||
int rc;
|
||||
mca_pml_base_recv_request_t *recvreq;
|
||||
MCA_PML_TEG_RECV_REQUEST_ALLOC(recvreq, rc);
|
||||
if (NULL == recvreq)
|
||||
return rc;
|
||||
|
||||
MCA_PML_BASE_RECV_REQUEST_INIT(recvreq,
|
||||
addr,
|
||||
count, datatype, src, tag, comm, false);
|
||||
|
||||
if ((rc = mca_pml_teg_recv_request_start(recvreq)) != OMPI_SUCCESS) {
|
||||
goto recv_finish;
|
||||
}
|
||||
|
||||
if (recvreq->req_base.req_ompi.req_complete == false) {
|
||||
/* give up and sleep until completion */
|
||||
if (ompi_using_threads()) {
|
||||
ompi_mutex_lock(&ompi_request_lock);
|
||||
ompi_request_waiting++;
|
||||
while (recvreq->req_base.req_ompi.req_complete == false)
|
||||
ompi_condition_wait(&ompi_request_cond, &ompi_request_lock);
|
||||
ompi_request_waiting--;
|
||||
ompi_mutex_unlock(&ompi_request_lock);
|
||||
} else {
|
||||
ompi_request_waiting++;
|
||||
while (recvreq->req_base.req_ompi.req_complete == false)
|
||||
ompi_condition_wait(&ompi_request_cond, &ompi_request_lock);
|
||||
ompi_request_waiting--;
|
||||
}
|
||||
}
|
||||
recv_finish:
|
||||
if (NULL != status) { /* return status */
|
||||
*status = recvreq->req_base.req_ompi.req_status;
|
||||
}
|
||||
|
||||
MCA_PML_TEG_RECV_REQUEST_RETURN(recvreq);
|
||||
return recvreq->req_base.req_ompi.req_status.MPI_ERROR;
|
||||
}
|
||||
@ -1,128 +0,0 @@
|
||||
/*
|
||||
* Copyright (c) 2004-2005 The Trustees of Indiana University.
|
||||
* All rights reserved.
|
||||
* Copyright (c) 2004-2005 The Trustees of the University of Tennessee.
|
||||
* All rights reserved.
|
||||
* Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
|
||||
* University of Stuttgart. All rights reserved.
|
||||
* Copyright (c) 2004-2005 The Regents of the University of California.
|
||||
* All rights reserved.
|
||||
* $COPYRIGHT$
|
||||
*
|
||||
* Additional copyrights may follow
|
||||
*
|
||||
* $HEADER$
|
||||
*/
|
||||
|
||||
#include "ompi_config.h"
|
||||
|
||||
#include "pml_teg.h"
|
||||
#include "pml_teg_proc.h"
|
||||
#include "pml_teg_sendreq.h"
|
||||
#include "pml_teg_recvreq.h"
|
||||
|
||||
|
||||
int mca_pml_teg_isend_init(void *buf,
|
||||
size_t count,
|
||||
ompi_datatype_t * datatype,
|
||||
int dst,
|
||||
int tag,
|
||||
mca_pml_base_send_mode_t sendmode,
|
||||
ompi_communicator_t * comm,
|
||||
ompi_request_t ** request)
|
||||
{
|
||||
int rc;
|
||||
|
||||
mca_pml_base_send_request_t *sendreq;
|
||||
MCA_PML_TEG_SEND_REQUEST_ALLOC(comm, dst, sendreq, rc);
|
||||
if (rc != OMPI_SUCCESS)
|
||||
return rc;
|
||||
|
||||
MCA_PML_BASE_SEND_REQUEST_INIT(sendreq,
|
||||
buf,
|
||||
count,
|
||||
datatype,
|
||||
dst, tag,
|
||||
comm, sendmode, true);
|
||||
|
||||
*request = (ompi_request_t *) sendreq;
|
||||
return OMPI_SUCCESS;
|
||||
}
|
||||
|
||||
|
||||
int mca_pml_teg_isend(void *buf,
|
||||
size_t count,
|
||||
ompi_datatype_t * datatype,
|
||||
int dst,
|
||||
int tag,
|
||||
mca_pml_base_send_mode_t sendmode,
|
||||
ompi_communicator_t * comm,
|
||||
ompi_request_t ** request)
|
||||
{
|
||||
int rc;
|
||||
mca_pml_base_send_request_t *sendreq;
|
||||
MCA_PML_TEG_SEND_REQUEST_ALLOC(comm, dst, sendreq, rc);
|
||||
if (rc != OMPI_SUCCESS)
|
||||
return rc;
|
||||
MCA_PML_BASE_SEND_REQUEST_INIT(sendreq,
|
||||
buf,
|
||||
count,
|
||||
datatype,
|
||||
dst, tag,
|
||||
comm, sendmode, false);
|
||||
|
||||
MCA_PML_TEG_SEND_REQUEST_START(sendreq, rc);
|
||||
*request = (ompi_request_t *) sendreq;
|
||||
return rc;
|
||||
}
|
||||
|
||||
|
||||
int mca_pml_teg_send(void *buf,
|
||||
size_t count,
|
||||
ompi_datatype_t * datatype,
|
||||
int dst,
|
||||
int tag,
|
||||
mca_pml_base_send_mode_t sendmode,
|
||||
ompi_communicator_t * comm)
|
||||
{
|
||||
int rc;
|
||||
mca_pml_base_send_request_t *sendreq;
|
||||
MCA_PML_TEG_SEND_REQUEST_ALLOC(comm, dst, sendreq, rc);
|
||||
if (rc != OMPI_SUCCESS)
|
||||
return rc;
|
||||
|
||||
MCA_PML_BASE_SEND_REQUEST_INIT(sendreq,
|
||||
buf,
|
||||
count,
|
||||
datatype,
|
||||
dst, tag,
|
||||
comm, sendmode, false);
|
||||
|
||||
MCA_PML_TEG_SEND_REQUEST_START(sendreq, rc);
|
||||
if (rc != OMPI_SUCCESS) {
|
||||
MCA_PML_TEG_FREE((ompi_request_t **) & sendreq);
|
||||
return rc;
|
||||
}
|
||||
|
||||
if (sendreq->req_base.req_ompi.req_complete == false) {
|
||||
/* give up and sleep until completion */
|
||||
if (ompi_using_threads()) {
|
||||
ompi_mutex_lock(&ompi_request_lock);
|
||||
ompi_request_waiting++;
|
||||
while (sendreq->req_base.req_ompi.req_complete == false)
|
||||
ompi_condition_wait(&ompi_request_cond, &ompi_request_lock);
|
||||
ompi_request_waiting--;
|
||||
ompi_mutex_unlock(&ompi_request_lock);
|
||||
} else {
|
||||
ompi_request_waiting++;
|
||||
while (sendreq->req_base.req_ompi.req_complete == false)
|
||||
ompi_condition_wait(&ompi_request_cond, &ompi_request_lock);
|
||||
ompi_request_waiting--;
|
||||
}
|
||||
}
|
||||
|
||||
/* return request to pool */
|
||||
MCA_PML_TEG_FREE((ompi_request_t **) & sendreq);
|
||||
return OMPI_SUCCESS;
|
||||
}
|
||||
|
||||
@ -1,56 +0,0 @@
|
||||
/*
|
||||
* Copyright (c) 2004-2005 The Trustees of Indiana University.
|
||||
* All rights reserved.
|
||||
* Copyright (c) 2004-2005 The Trustees of the University of Tennessee.
|
||||
* All rights reserved.
|
||||
* Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
|
||||
* University of Stuttgart. All rights reserved.
|
||||
* Copyright (c) 2004-2005 The Regents of the University of California.
|
||||
* All rights reserved.
|
||||
* $COPYRIGHT$
|
||||
*
|
||||
* Additional copyrights may follow
|
||||
*
|
||||
* $HEADER$
|
||||
*/
|
||||
|
||||
#include "ompi_config.h"
|
||||
|
||||
#include "include/sys/atomic.h"
|
||||
#include "pml_teg.h"
|
||||
#include "pml_teg_proc.h"
|
||||
#include "pml_ptl_array.h"
|
||||
|
||||
|
||||
static void mca_pml_teg_proc_construct(mca_pml_proc_t* proc)
|
||||
{
|
||||
proc->proc_ompi = NULL;
|
||||
proc->proc_ptl_flags = 0;
|
||||
OBJ_CONSTRUCT(&proc->proc_lock, ompi_mutex_t);
|
||||
OBJ_CONSTRUCT(&proc->proc_ptl_first, mca_pml_teg_ptl_array_t);
|
||||
OBJ_CONSTRUCT(&proc->proc_ptl_next, mca_pml_teg_ptl_array_t);
|
||||
|
||||
OMPI_THREAD_LOCK(&mca_pml_teg.teg_lock);
|
||||
ompi_list_append(&mca_pml_teg.teg_procs, (ompi_list_item_t*)proc);
|
||||
OMPI_THREAD_UNLOCK(&mca_pml_teg.teg_lock);
|
||||
}
|
||||
|
||||
|
||||
static void mca_pml_teg_proc_destruct(mca_pml_proc_t* proc)
|
||||
{
|
||||
OMPI_THREAD_LOCK(&mca_pml_teg.teg_lock);
|
||||
ompi_list_remove_item(&mca_pml_teg.teg_procs, (ompi_list_item_t*)proc);
|
||||
OMPI_THREAD_UNLOCK(&mca_pml_teg.teg_lock);
|
||||
|
||||
OBJ_DESTRUCT(&proc->proc_lock);
|
||||
OBJ_DESTRUCT(&proc->proc_ptl_first);
|
||||
OBJ_DESTRUCT(&proc->proc_ptl_next);
|
||||
}
|
||||
|
||||
OBJ_CLASS_INSTANCE(
|
||||
mca_pml_teg_proc_t,
|
||||
ompi_list_item_t,
|
||||
mca_pml_teg_proc_construct,
|
||||
mca_pml_teg_proc_destruct
|
||||
);
|
||||
|
||||
@ -1,107 +0,0 @@
|
||||
/*
|
||||
* Copyright (c) 2004-2005 The Trustees of Indiana University.
|
||||
* All rights reserved.
|
||||
* Copyright (c) 2004-2005 The Trustees of the University of Tennessee.
|
||||
* All rights reserved.
|
||||
* Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
|
||||
* University of Stuttgart. All rights reserved.
|
||||
* Copyright (c) 2004-2005 The Regents of the University of California.
|
||||
* All rights reserved.
|
||||
* $COPYRIGHT$
|
||||
*
|
||||
* Additional copyrights may follow
|
||||
*
|
||||
* $HEADER$
|
||||
*/
|
||||
/**
|
||||
* @file
|
||||
*/
|
||||
#ifndef MCA_PML_PROC_H
|
||||
#define MCA_PML_PROC_H
|
||||
|
||||
#include "threads/mutex.h"
|
||||
#include "communicator/communicator.h"
|
||||
#include "group/group.h"
|
||||
#include "proc/proc.h"
|
||||
#include "pml_ptl_array.h"
|
||||
|
||||
#if defined(c_plusplus) || defined(__cplusplus)
|
||||
extern "C" {
|
||||
#endif
|
||||
/**
|
||||
* Structure associated w/ ompi_proc_t that contains data specific
|
||||
* to the PML. Note that this name is not PML specific.
|
||||
*/
|
||||
struct mca_pml_proc_t {
|
||||
ompi_list_item_t super;
|
||||
ompi_proc_t *proc_ompi; /**< back-pointer to ompi_proc_t */
|
||||
ompi_mutex_t proc_lock; /**< lock to protect against concurrent access */
|
||||
mca_ptl_array_t proc_ptl_first; /**< array of ptls to use for first fragments */
|
||||
mca_ptl_array_t proc_ptl_next; /**< array of ptls to use for remaining fragments */
|
||||
uint32_t proc_ptl_flags; /**< aggregate ptl flags */
|
||||
};
|
||||
typedef struct mca_pml_proc_t mca_pml_proc_t;
|
||||
|
||||
|
||||
OMPI_COMP_EXPORT extern ompi_class_t mca_pml_teg_proc_t_class;
|
||||
typedef struct mca_pml_proc_t mca_pml_teg_proc_t;
|
||||
|
||||
/**
|
||||
* Return the mca_pml_proc_t instance cached in the communicators local group.
|
||||
*
|
||||
* @param comm Communicator
|
||||
* @param rank Peer rank
|
||||
* @return mca_pml_proc_t instance
|
||||
*/
|
||||
|
||||
static inline mca_pml_proc_t* mca_pml_teg_proc_lookup_local(ompi_communicator_t* comm, int rank)
|
||||
{
|
||||
ompi_proc_t* proc = comm->c_local_group->grp_proc_pointers[rank];
|
||||
return proc->proc_pml;
|
||||
}
|
||||
|
||||
/**
|
||||
* Return the mca_pml_proc_t instance cached on the communicators remote group.
|
||||
*
|
||||
* @param comm Communicator
|
||||
* @param rank Peer rank
|
||||
* @return mca_pml_proc_t instance
|
||||
*/
|
||||
|
||||
static inline mca_pml_proc_t* mca_pml_teg_proc_lookup_remote(ompi_communicator_t* comm, int rank)
|
||||
{
|
||||
ompi_proc_t* proc = comm->c_remote_group->grp_proc_pointers[rank];
|
||||
return proc->proc_pml;
|
||||
}
|
||||
|
||||
/**
|
||||
* Return the mca_ptl_peer_t instance corresponding to the process/ptl combination.
|
||||
*
|
||||
* @param comm Communicator
|
||||
* @param rank Peer rank
|
||||
* @return mca_pml_proc_t instance
|
||||
*/
|
||||
|
||||
static inline struct mca_ptl_base_peer_t* mca_pml_teg_proc_lookup_remote_peer(
|
||||
ompi_communicator_t* comm,
|
||||
int rank,
|
||||
struct mca_ptl_base_module_t* ptl)
|
||||
{
|
||||
ompi_proc_t* proc = comm->c_remote_group->grp_proc_pointers[rank];
|
||||
mca_pml_proc_t* proc_pml = proc->proc_pml;
|
||||
size_t i, size = mca_ptl_array_get_size(&proc_pml->proc_ptl_first);
|
||||
mca_ptl_proc_t* proc_ptl = proc_pml->proc_ptl_first.ptl_procs;
|
||||
for(i = 0; i < size; i++) {
|
||||
if(proc_ptl->ptl == ptl) {
|
||||
return proc_ptl->ptl_peer;
|
||||
}
|
||||
proc_ptl++;
|
||||
}
|
||||
return NULL;
|
||||
}
|
||||
|
||||
#if defined(c_plusplus) || defined(__cplusplus)
|
||||
}
|
||||
#endif
|
||||
#endif
|
||||
|
||||
@ -1,40 +0,0 @@
|
||||
/*
|
||||
* Copyright (c) 2004-2005 The Trustees of Indiana University.
|
||||
* All rights reserved.
|
||||
* Copyright (c) 2004-2005 The Trustees of the University of Tennessee.
|
||||
* All rights reserved.
|
||||
* Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
|
||||
* University of Stuttgart. All rights reserved.
|
||||
* Copyright (c) 2004-2005 The Regents of the University of California.
|
||||
* All rights reserved.
|
||||
* $COPYRIGHT$
|
||||
*
|
||||
* Additional copyrights may follow
|
||||
*
|
||||
* $HEADER$
|
||||
*/
|
||||
|
||||
#include "ompi_config.h"
|
||||
|
||||
#include "pml_teg.h"
|
||||
#include "pml_teg_sendreq.h"
|
||||
|
||||
|
||||
int mca_pml_teg_progress(void)
|
||||
{
|
||||
mca_ptl_tstamp_t tstamp = 0;
|
||||
size_t i;
|
||||
int count = 0;
|
||||
|
||||
/*
|
||||
* Progress each of the PTL modules
|
||||
*/
|
||||
for(i=0; i<mca_pml_teg.teg_num_ptl_progress; i++) {
|
||||
int rc = mca_pml_teg.teg_ptl_progress[i](tstamp);
|
||||
if(rc > 0) {
|
||||
count += rc;
|
||||
}
|
||||
}
|
||||
return count;
|
||||
}
|
||||
|
||||
@ -1,43 +0,0 @@
|
||||
/*
|
||||
* Copyright (c) 2004-2005 The Trustees of Indiana University.
|
||||
* All rights reserved.
|
||||
* Copyright (c) 2004-2005 The Trustees of the University of Tennessee.
|
||||
* All rights reserved.
|
||||
* Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
|
||||
* University of Stuttgart. All rights reserved.
|
||||
* Copyright (c) 2004-2005 The Regents of the University of California.
|
||||
* All rights reserved.
|
||||
* $COPYRIGHT$
|
||||
*
|
||||
* Additional copyrights may follow
|
||||
*
|
||||
* $HEADER$
|
||||
*/
|
||||
|
||||
#include "ompi_config.h"
|
||||
|
||||
#include "pml_teg_ptl.h"
|
||||
|
||||
|
||||
static void mca_pml_base_ptl_construct(mca_pml_base_ptl_t* ptl)
|
||||
{
|
||||
OBJ_CONSTRUCT(&ptl->ptl_cache, ompi_list_t);
|
||||
OBJ_CONSTRUCT(&ptl->ptl_cache_lock, ompi_mutex_t);
|
||||
ptl->ptl = NULL;
|
||||
ptl->ptl_cache_size = 0;
|
||||
ptl->ptl_cache_alloc = 0;
|
||||
}
|
||||
|
||||
static void mca_pml_base_ptl_destruct(mca_pml_base_ptl_t* ptl)
|
||||
{
|
||||
OBJ_DESTRUCT(&ptl->ptl_cache);
|
||||
OBJ_DESTRUCT(&ptl->ptl_cache_lock);
|
||||
}
|
||||
|
||||
OBJ_CLASS_INSTANCE(
|
||||
mca_pml_base_ptl_t,
|
||||
ompi_list_t,
|
||||
mca_pml_base_ptl_construct,
|
||||
mca_pml_base_ptl_destruct
|
||||
);
|
||||
|
||||
@ -1,44 +0,0 @@
|
||||
/*
|
||||
* Copyright (c) 2004-2005 The Trustees of Indiana University.
|
||||
* All rights reserved.
|
||||
* Copyright (c) 2004-2005 The Trustees of the University of Tennessee.
|
||||
* All rights reserved.
|
||||
* Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
|
||||
* University of Stuttgart. All rights reserved.
|
||||
* Copyright (c) 2004-2005 The Regents of the University of California.
|
||||
* All rights reserved.
|
||||
* $COPYRIGHT$
|
||||
*
|
||||
* Additional copyrights may follow
|
||||
*
|
||||
* $HEADER$
|
||||
*/
|
||||
|
||||
#ifndef _MCA_PML_BASE_PTL_
|
||||
#define _MCA_PML_BASE_PTL_
|
||||
|
||||
#include "mca/pml/pml.h"
|
||||
#include "mca/ptl/ptl.h"
|
||||
#include "threads/condition.h"
|
||||
#if defined(c_plusplus) || defined(__cplusplus)
|
||||
extern "C" {
|
||||
#endif
|
||||
|
||||
|
||||
struct mca_pml_base_ptl_t {
|
||||
ompi_list_t ptl_cache; /**< cache of send requests */
|
||||
size_t ptl_cache_size; /**< maximum size of cache */
|
||||
size_t ptl_cache_alloc; /**< current number of allocated items */
|
||||
ompi_mutex_t ptl_cache_lock; /**< lock for queue access */
|
||||
struct mca_ptl_base_module_t* ptl; /**< back pointer to ptl */
|
||||
};
|
||||
typedef struct mca_pml_base_ptl_t mca_pml_base_ptl_t;
|
||||
|
||||
OBJ_CLASS_DECLARATION(mca_pml_base_ptl_t);
|
||||
|
||||
#if defined(c_plusplus) || defined(__cplusplus)
|
||||
}
|
||||
#endif
|
||||
|
||||
#endif
|
||||
|
||||
@ -1,92 +0,0 @@
|
||||
/*
|
||||
* Copyright (c) 2004-2005 The Trustees of Indiana University.
|
||||
* All rights reserved.
|
||||
* Copyright (c) 2004-2005 The Trustees of the University of Tennessee.
|
||||
* All rights reserved.
|
||||
* Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
|
||||
* University of Stuttgart. All rights reserved.
|
||||
* Copyright (c) 2004-2005 The Regents of the University of California.
|
||||
* All rights reserved.
|
||||
* $COPYRIGHT$
|
||||
*
|
||||
* Additional copyrights may follow
|
||||
*
|
||||
* $HEADER$
|
||||
*/
|
||||
|
||||
/**
|
||||
* @file
|
||||
*/
|
||||
|
||||
#include "ompi_config.h"
|
||||
|
||||
#include "mca/pml/pml.h"
|
||||
#include "pml_teg_recvfrag.h"
|
||||
#include "pml_teg_proc.h"
|
||||
|
||||
|
||||
OMPI_DECLSPEC extern ompi_class_t mca_ptl_base_recv_frag_t_class;
|
||||
|
||||
|
||||
/**
|
||||
* Called by the PTL to match attempt a match for new fragments.
|
||||
*
|
||||
* @param ptl (IN) The PTL pointer
|
||||
* @param frag (IN) Receive fragment descriptor.
|
||||
* @param header (IN) Header corresponding to the receive fragment.
|
||||
* @return OMPI_SUCCESS or error status on failure.
|
||||
*/
|
||||
bool mca_pml_teg_recv_frag_match(
|
||||
mca_ptl_base_module_t* ptl,
|
||||
mca_ptl_base_recv_frag_t* frag,
|
||||
mca_ptl_base_match_header_t* header)
|
||||
{
|
||||
bool matched;
|
||||
bool matches = false;
|
||||
ompi_list_t matched_frags;
|
||||
if((matched = mca_ptl_base_match(header, frag, &matched_frags, &matches)) == false) {
|
||||
frag = (matches ? (mca_ptl_base_recv_frag_t*)ompi_list_remove_first(&matched_frags) : NULL);
|
||||
}
|
||||
|
||||
while(NULL != frag) {
|
||||
mca_ptl_base_module_t* ptl = frag->frag_base.frag_owner;
|
||||
mca_pml_base_recv_request_t *request = frag->frag_request;
|
||||
mca_ptl_base_match_header_t *header = &frag->frag_base.frag_header.hdr_match;
|
||||
|
||||
/*
|
||||
* Initialize request status.
|
||||
*/
|
||||
request->req_bytes_packed = header->hdr_msg_length;
|
||||
request->req_base.req_ompi.req_status.MPI_SOURCE = header->hdr_src;
|
||||
request->req_base.req_ompi.req_status.MPI_TAG = header->hdr_tag;
|
||||
|
||||
/*
|
||||
* If probe - signal request is complete - but don't notify PTL
|
||||
*/
|
||||
if(request->req_base.req_type == MCA_PML_REQUEST_PROBE) {
|
||||
|
||||
ptl->ptl_recv_progress( ptl,
|
||||
request,
|
||||
header->hdr_msg_length,
|
||||
header->hdr_msg_length );
|
||||
matched = mca_pml_teg_recv_frag_match( ptl, frag, header );
|
||||
|
||||
} else {
|
||||
|
||||
/* if required - setup pointer to ptls peer */
|
||||
if (NULL == frag->frag_base.frag_peer) {
|
||||
frag->frag_base.frag_peer = mca_pml_teg_proc_lookup_remote_peer(request->req_base.req_comm,header->hdr_src,ptl);
|
||||
}
|
||||
|
||||
/* notify ptl of match */
|
||||
ptl->ptl_matched(ptl, frag);
|
||||
|
||||
};
|
||||
|
||||
/* process any additional fragments that arrived out of order */
|
||||
frag = (matches ? (mca_ptl_base_recv_frag_t*)ompi_list_remove_first(&matched_frags) : NULL);
|
||||
};
|
||||
return matched;
|
||||
}
|
||||
|
||||
|
||||
@ -1,42 +0,0 @@
|
||||
/*
|
||||
* Copyright (c) 2004-2005 The Trustees of Indiana University.
|
||||
* All rights reserved.
|
||||
* Copyright (c) 2004-2005 The Trustees of the University of Tennessee.
|
||||
* All rights reserved.
|
||||
* Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
|
||||
* University of Stuttgart. All rights reserved.
|
||||
* Copyright (c) 2004-2005 The Regents of the University of California.
|
||||
* All rights reserved.
|
||||
* $COPYRIGHT$
|
||||
*
|
||||
* Additional copyrights may follow
|
||||
*
|
||||
* $HEADER$
|
||||
*/
|
||||
/**
|
||||
* @file
|
||||
*/
|
||||
|
||||
#ifndef MCA_PML_TEG_RECVFRAG_H
|
||||
#define MCA_PML_TEG_RECVFRAG_H
|
||||
|
||||
#include "mca/ptl/ptl.h"
|
||||
#include "mca/pml/base/pml_base_recvreq.h"
|
||||
#include "mca/ptl/base/ptl_base_recvfrag.h"
|
||||
|
||||
/**
|
||||
* Called by the PTL to match attempt a match for new fragments.
|
||||
*
|
||||
* @param ptl (IN) The PTL pointer
|
||||
* @param frag (IN) Receive fragment descriptor.
|
||||
* @param header (IN) Header corresponding to the receive fragment.
|
||||
* @return OMPI_SUCCESS or error status on failure.
|
||||
*/
|
||||
bool mca_pml_teg_recv_frag_match(
|
||||
mca_ptl_base_module_t* ptl,
|
||||
mca_ptl_base_recv_frag_t* frag,
|
||||
mca_ptl_base_match_header_t* header
|
||||
);
|
||||
|
||||
#endif
|
||||
|
||||
@ -1,280 +0,0 @@
|
||||
/*
|
||||
* Copyright (c) 2004-2005 The Trustees of Indiana University.
|
||||
* All rights reserved.
|
||||
* Copyright (c) 2004-2005 The Trustees of the University of Tennessee.
|
||||
* All rights reserved.
|
||||
* Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
|
||||
* University of Stuttgart. All rights reserved.
|
||||
* Copyright (c) 2004-2005 The Regents of the University of California.
|
||||
* All rights reserved.
|
||||
* $COPYRIGHT$
|
||||
*
|
||||
* Additional copyrights may follow
|
||||
*
|
||||
* $HEADER$
|
||||
*/
|
||||
|
||||
#include "ompi_config.h"
|
||||
|
||||
#include "mca/pml/pml.h"
|
||||
#include "mca/ptl/ptl.h"
|
||||
#include "mca/ptl/base/ptl_base_comm.h"
|
||||
#include "pml_teg_recvreq.h"
|
||||
#include "pml_teg_sendreq.h"
|
||||
|
||||
|
||||
static mca_ptl_base_recv_frag_t* mca_pml_teg_recv_request_match_specific_proc(
|
||||
mca_pml_base_recv_request_t* request, int proc);
|
||||
|
||||
|
||||
static int mca_pml_teg_recv_request_fini(struct ompi_request_t** request)
|
||||
{
|
||||
MCA_PML_TEG_FINI(request);
|
||||
return OMPI_SUCCESS;
|
||||
}
|
||||
|
||||
static int mca_pml_teg_recv_request_free(struct ompi_request_t** request)
|
||||
{
|
||||
MCA_PML_TEG_FREE(request);
|
||||
return OMPI_SUCCESS;
|
||||
}
|
||||
|
||||
|
||||
static int mca_pml_teg_recv_request_cancel(struct ompi_request_t* request, int complete)
|
||||
{
|
||||
mca_pml_base_request_t* teg_request = (mca_pml_base_request_t*)request;
|
||||
ompi_communicator_t* ompi_comm = teg_request->req_comm;
|
||||
mca_pml_ptl_comm_t* pml_comm = (mca_pml_ptl_comm_t*)ompi_comm->c_pml_comm;
|
||||
|
||||
if( true == request->req_complete ) { /* way to late to cancel this one */
|
||||
return OMPI_SUCCESS;
|
||||
}
|
||||
|
||||
/* The rest should be protected behind the match logic lock */
|
||||
OMPI_THREAD_LOCK(&pml_comm->c_matching_lock);
|
||||
|
||||
if( OMPI_ANY_TAG == request->req_status.MPI_TAG ) { /* the match have not been already done */
|
||||
|
||||
if( teg_request->req_peer == OMPI_ANY_SOURCE ) {
|
||||
ompi_list_remove_item( &(pml_comm->c_wild_receives),
|
||||
(ompi_list_item_t*)request );
|
||||
} else {
|
||||
ompi_list_remove_item( pml_comm->c_specific_receives + teg_request->req_peer,
|
||||
(ompi_list_item_t*)request );
|
||||
}
|
||||
}
|
||||
|
||||
OMPI_THREAD_UNLOCK(&pml_comm->c_matching_lock);
|
||||
|
||||
request->req_status._cancelled = true;
|
||||
request->req_complete = true; /* mark it as completed so all the test/wait functions
|
||||
* on this particular request will finish */
|
||||
/* Now we have a problem if we are in a multi-threaded environment. We shou ld
|
||||
* broadcast the condition on the request in order to allow the other threa ds
|
||||
* to complete their test/wait functions.
|
||||
*/
|
||||
if(ompi_request_waiting) {
|
||||
ompi_condition_broadcast(&ompi_request_cond);
|
||||
}
|
||||
return OMPI_SUCCESS;
|
||||
}
|
||||
|
||||
static void mca_pml_teg_recv_request_construct(mca_pml_base_recv_request_t* request)
|
||||
{
|
||||
request->req_recv.req_base.req_type = MCA_PML_REQUEST_RECV;
|
||||
request->req_recv.req_base.req_ompi.req_fini = mca_pml_teg_recv_request_fini;
|
||||
request->req_recv.req_base.req_ompi.req_free = mca_pml_teg_recv_request_free;
|
||||
request->req_recv.req_base.req_ompi.req_cancel = mca_pml_teg_recv_request_cancel;
|
||||
}
|
||||
|
||||
static void mca_pml_teg_recv_request_destruct(mca_pml_base_recv_request_t* request)
|
||||
{
|
||||
}
|
||||
|
||||
OBJ_CLASS_INSTANCE(
|
||||
mca_pml_teg_recv_request_t,
|
||||
mca_ptl_base_recv_request_t,
|
||||
mca_pml_teg_recv_request_construct,
|
||||
mca_pml_teg_recv_request_destruct);
|
||||
|
||||
|
||||
/*
|
||||
* Update the recv request status to reflect the number of bytes
|
||||
* received and actually delivered to the application.
|
||||
*/
|
||||
|
||||
void mca_pml_teg_recv_request_progress(
|
||||
struct mca_ptl_base_module_t* ptl,
|
||||
mca_pml_base_recv_request_t* req,
|
||||
size_t bytes_received,
|
||||
size_t bytes_delivered)
|
||||
{
|
||||
OMPI_THREAD_LOCK(&ompi_request_lock);
|
||||
req->req_bytes_received += bytes_received;
|
||||
req->req_bytes_delivered += bytes_delivered;
|
||||
if (req->req_bytes_received >= req->req_bytes_packed) {
|
||||
/* initialize request status */
|
||||
req->req_base.req_ompi.req_status._count = req->req_bytes_delivered;
|
||||
req->req_base.req_pml_complete = true;
|
||||
req->req_base.req_ompi.req_complete = true;
|
||||
if(ompi_request_waiting) {
|
||||
ompi_condition_broadcast(&ompi_request_cond);
|
||||
}
|
||||
}
|
||||
OMPI_THREAD_UNLOCK(&ompi_request_lock);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*
|
||||
* This routine is used to match a posted receive when the source process
|
||||
* is specified.
|
||||
*/
|
||||
|
||||
void mca_pml_teg_recv_request_match_specific(mca_pml_base_recv_request_t* request)
|
||||
{
|
||||
ompi_communicator_t *comm = request->req_base.req_comm;
|
||||
mca_pml_ptl_comm_t* pml_comm = comm->c_pml_comm;
|
||||
int req_peer = request->req_base.req_peer;
|
||||
mca_ptl_base_recv_frag_t* frag;
|
||||
|
||||
/* check for a specific match */
|
||||
OMPI_THREAD_LOCK(&pml_comm->c_matching_lock);
|
||||
|
||||
/* assign sequence number */
|
||||
request->req_base.req_sequence = pml_comm->c_recv_seq++;
|
||||
|
||||
if (ompi_list_get_size(&pml_comm->c_unexpected_frags[req_peer]) > 0 &&
|
||||
(frag = mca_pml_teg_recv_request_match_specific_proc(request, req_peer)) != NULL) {
|
||||
mca_ptl_base_module_t* ptl = frag->frag_base.frag_owner;
|
||||
/* setup pointer to ptls peer */
|
||||
if(NULL == frag->frag_base.frag_peer)
|
||||
frag->frag_base.frag_peer = mca_pml_teg_proc_lookup_remote_peer(comm,req_peer,ptl);
|
||||
OMPI_THREAD_UNLOCK(&pml_comm->c_matching_lock);
|
||||
if( !((MCA_PML_REQUEST_IPROBE == request->req_base.req_type) ||
|
||||
(MCA_PML_REQUEST_PROBE == request->req_base.req_type)) ) {
|
||||
ptl->ptl_matched(ptl, frag);
|
||||
}
|
||||
return; /* match found */
|
||||
}
|
||||
|
||||
/* We didn't find any matches. Record this irecv so we can match
|
||||
* it when the message comes in.
|
||||
*/
|
||||
if(request->req_base.req_type != MCA_PML_REQUEST_IPROBE) {
|
||||
ompi_list_append(pml_comm->c_specific_receives+req_peer, (ompi_list_item_t*)request);
|
||||
}
|
||||
OMPI_THREAD_UNLOCK(&pml_comm->c_matching_lock);
|
||||
}
|
||||
|
||||
|
||||
/*
|
||||
* this routine is used to try and match a wild posted receive - where
|
||||
* wild is determined by the value assigned to the source process
|
||||
*/
|
||||
|
||||
void mca_pml_teg_recv_request_match_wild(mca_pml_base_recv_request_t* request)
|
||||
{
|
||||
ompi_communicator_t *comm = request->req_base.req_comm;
|
||||
mca_pml_ptl_comm_t* pml_comm = comm->c_pml_comm;
|
||||
int proc_count = comm->c_remote_group->grp_proc_count;
|
||||
int proc;
|
||||
|
||||
/*
|
||||
* Loop over all the outstanding messages to find one that matches.
|
||||
* There is an outer loop over lists of messages from each
|
||||
* process, then an inner loop over the messages from the
|
||||
* process.
|
||||
*/
|
||||
OMPI_THREAD_LOCK(&pml_comm->c_matching_lock);
|
||||
|
||||
/* assign sequence number */
|
||||
request->req_base.req_sequence = pml_comm->c_recv_seq++;
|
||||
|
||||
for (proc = 0; proc < proc_count; proc++) {
|
||||
mca_ptl_base_recv_frag_t* frag;
|
||||
|
||||
/* continue if no frags to match */
|
||||
if (ompi_list_get_size(&pml_comm->c_unexpected_frags[proc]) == 0)
|
||||
continue;
|
||||
|
||||
/* loop over messages from the current proc */
|
||||
if ((frag = mca_pml_teg_recv_request_match_specific_proc(request, proc)) != NULL) {
|
||||
mca_ptl_base_module_t* ptl = frag->frag_base.frag_owner;
|
||||
/* if required - setup pointer to ptls peer */
|
||||
if(NULL == frag->frag_base.frag_peer)
|
||||
frag->frag_base.frag_peer = mca_pml_teg_proc_lookup_remote_peer(comm,proc,ptl);
|
||||
OMPI_THREAD_UNLOCK(&pml_comm->c_matching_lock);
|
||||
if( !((MCA_PML_REQUEST_IPROBE == request->req_base.req_type) ||
|
||||
(MCA_PML_REQUEST_PROBE == request->req_base.req_type)) ) {
|
||||
ptl->ptl_matched(ptl, frag);
|
||||
}
|
||||
return; /* match found */
|
||||
}
|
||||
}
|
||||
|
||||
/* We didn't find any matches. Record this irecv so we can match to
|
||||
* it when the message comes in.
|
||||
*/
|
||||
|
||||
if(request->req_base.req_type != MCA_PML_REQUEST_IPROBE)
|
||||
ompi_list_append(&pml_comm->c_wild_receives, (ompi_list_item_t*)request);
|
||||
OMPI_THREAD_UNLOCK(&pml_comm->c_matching_lock);
|
||||
}
|
||||
|
||||
|
||||
/*
|
||||
* this routine tries to match a posted receive. If a match is found,
|
||||
* it places the request in the appropriate matched receive list.
|
||||
*/
|
||||
|
||||
static mca_ptl_base_recv_frag_t* mca_pml_teg_recv_request_match_specific_proc(
|
||||
mca_pml_base_recv_request_t* request, int proc)
|
||||
{
|
||||
mca_pml_ptl_comm_t *pml_comm = request->req_base.req_comm->c_pml_comm;
|
||||
ompi_list_t* unexpected_frags = pml_comm->c_unexpected_frags+proc;
|
||||
mca_ptl_base_recv_frag_t* frag;
|
||||
mca_ptl_base_match_header_t* header;
|
||||
int tag = request->req_base.req_tag;
|
||||
|
||||
if( OMPI_ANY_TAG == tag ) {
|
||||
for (frag = (mca_ptl_base_recv_frag_t*)ompi_list_get_first(unexpected_frags);
|
||||
frag != (mca_ptl_base_recv_frag_t*)ompi_list_get_end(unexpected_frags);
|
||||
frag = (mca_ptl_base_recv_frag_t*)ompi_list_get_next(frag)) {
|
||||
header = &(frag->frag_base.frag_header.hdr_match);
|
||||
|
||||
/* check first frag - we assume that process matching has been done already */
|
||||
if( header->hdr_tag >= 0 ) {
|
||||
goto find_fragment;
|
||||
}
|
||||
}
|
||||
} else {
|
||||
for (frag = (mca_ptl_base_recv_frag_t*)ompi_list_get_first(unexpected_frags);
|
||||
frag != (mca_ptl_base_recv_frag_t*)ompi_list_get_end(unexpected_frags);
|
||||
frag = (mca_ptl_base_recv_frag_t*)ompi_list_get_next(frag)) {
|
||||
header = &(frag->frag_base.frag_header.hdr_match);
|
||||
|
||||
/* check first frag - we assume that process matching has been done already */
|
||||
if ( tag == header->hdr_tag ) {
|
||||
/* we assume that the tag is correct from MPI point of view (ie. >= 0 ) */
|
||||
goto find_fragment;
|
||||
}
|
||||
}
|
||||
}
|
||||
return NULL;
|
||||
find_fragment:
|
||||
request->req_bytes_packed = header->hdr_msg_length;
|
||||
request->req_base.req_ompi.req_status.MPI_TAG = header->hdr_tag;
|
||||
request->req_base.req_ompi.req_status.MPI_SOURCE = header->hdr_src;
|
||||
|
||||
if( !((MCA_PML_REQUEST_IPROBE == request->req_base.req_type) ||
|
||||
(MCA_PML_REQUEST_PROBE == request->req_base.req_type)) ) {
|
||||
ompi_list_remove_item(unexpected_frags, (ompi_list_item_t*)frag);
|
||||
frag->frag_request = request;
|
||||
} else {
|
||||
/* it's a probe, therefore report it's completion */
|
||||
mca_pml_teg_recv_request_progress( NULL, request, header->hdr_msg_length, header->hdr_msg_length );
|
||||
}
|
||||
return frag;
|
||||
}
|
||||
|
||||
@ -1,125 +0,0 @@
|
||||
/*
|
||||
* Copyright (c) 2004-2005 The Trustees of Indiana University.
|
||||
* All rights reserved.
|
||||
* Copyright (c) 2004-2005 The Trustees of the University of Tennessee.
|
||||
* All rights reserved.
|
||||
* Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
|
||||
* University of Stuttgart. All rights reserved.
|
||||
* Copyright (c) 2004-2005 The Regents of the University of California.
|
||||
* All rights reserved.
|
||||
* $COPYRIGHT$
|
||||
*
|
||||
* Additional copyrights may follow
|
||||
*
|
||||
* $HEADER$
|
||||
*/
|
||||
/**
|
||||
* @file
|
||||
*/
|
||||
#ifndef OMPI_PML_TEG_RECV_REQUEST_H
|
||||
#define OMPI_PML_TEG_RECV_REQUEST_H
|
||||
|
||||
#include "pml_teg.h"
|
||||
#include "pml_teg_proc.h"
|
||||
#include "mca/ptl/base/ptl_base_recvreq.h"
|
||||
#include "mca/ptl/base/ptl_base_recvfrag.h"
|
||||
|
||||
#if defined(c_plusplus) || defined(__cplusplus)
|
||||
extern "C" {
|
||||
#endif
|
||||
typedef mca_ptl_base_recv_request_t mca_pml_teg_recv_request_t;
|
||||
|
||||
OBJ_CLASS_DECLARATION(mca_pml_teg_recv_request_t);
|
||||
|
||||
|
||||
/**
|
||||
* Allocate a recv request from the modules free list.
|
||||
*
|
||||
* @param rc (OUT) OMPI_SUCCESS or error status on failure.
|
||||
* @return Receive request.
|
||||
*/
|
||||
#define MCA_PML_TEG_RECV_REQUEST_ALLOC(recvreq, rc) \
|
||||
do { \
|
||||
ompi_list_item_t* item; \
|
||||
OMPI_FREE_LIST_GET(&mca_pml_teg.teg_recv_requests, item, rc); \
|
||||
recvreq = (mca_pml_base_recv_request_t*)item; \
|
||||
} while(0)
|
||||
|
||||
/**
|
||||
* Return a recv request to the modules free list.
|
||||
*
|
||||
* @param request (IN) Receive request.
|
||||
*/
|
||||
#define MCA_PML_TEG_RECV_REQUEST_RETURN(request) \
|
||||
do { \
|
||||
MCA_PML_BASE_RECV_REQUEST_RETURN( request ); \
|
||||
OMPI_FREE_LIST_RETURN(&mca_pml_teg.teg_recv_requests, (ompi_list_item_t*)request); \
|
||||
} while(0)
|
||||
|
||||
/**
|
||||
* Attempt to match the request against the unexpected fragment list
|
||||
* for all source ranks w/in the communicator.
|
||||
*
|
||||
* @param request (IN) Request to match.
|
||||
*/
|
||||
void mca_pml_teg_recv_request_match_wild(mca_pml_base_recv_request_t* request);
|
||||
|
||||
/**
|
||||
* Attempt to match the request against the unexpected fragment list
|
||||
* for a specific source rank.
|
||||
*
|
||||
* @param request (IN) Request to match.
|
||||
*/
|
||||
void mca_pml_teg_recv_request_match_specific(mca_pml_base_recv_request_t* request);
|
||||
|
||||
/**
|
||||
* Start an initialized request.
|
||||
*
|
||||
* @param request Receive request.
|
||||
* @return OMPI_SUCESS or error status on failure.
|
||||
*/
|
||||
static inline int mca_pml_teg_recv_request_start(mca_pml_base_recv_request_t* request)
|
||||
{
|
||||
/* init/re-init the request */
|
||||
request->req_bytes_received = 0;
|
||||
request->req_bytes_delivered = 0;
|
||||
request->req_base.req_pml_complete = false;
|
||||
request->req_base.req_ompi.req_complete = false;
|
||||
request->req_base.req_ompi.req_state = OMPI_REQUEST_ACTIVE;
|
||||
/* always set the req_status.MPI_TAG to ANY_TAG before starting the request. This field
|
||||
* is used on the cancel part in order to find out if the request has been matched or not.
|
||||
*/
|
||||
request->req_base.req_ompi.req_status.MPI_TAG = OMPI_ANY_TAG;
|
||||
request->req_base.req_ompi.req_status.MPI_ERROR = OMPI_SUCCESS;
|
||||
request->req_base.req_ompi.req_status._cancelled = 0;
|
||||
|
||||
/* attempt to match posted recv */
|
||||
if(request->req_base.req_peer == OMPI_ANY_SOURCE) {
|
||||
mca_pml_teg_recv_request_match_wild(request);
|
||||
} else {
|
||||
mca_pml_teg_recv_request_match_specific(request);
|
||||
}
|
||||
return OMPI_SUCCESS;
|
||||
}
|
||||
|
||||
/**
|
||||
* Update status of a recv request based on the completion status of
|
||||
* the receive fragment.
|
||||
*
|
||||
* @param ptl (IN) The PTL pointer.
|
||||
* @param request (IN) Receive request.
|
||||
* @param bytes_received (IN) Bytes received from peer.
|
||||
* @param bytes_delivered (IN) Bytes delivered to application.
|
||||
*/
|
||||
void mca_pml_teg_recv_request_progress(
|
||||
struct mca_ptl_base_module_t* ptl,
|
||||
mca_pml_base_recv_request_t* request,
|
||||
size_t bytes_received,
|
||||
size_t bytes_delivered
|
||||
);
|
||||
|
||||
#if defined(c_plusplus) || defined(__cplusplus)
|
||||
}
|
||||
#endif
|
||||
#endif
|
||||
|
||||
@ -1,216 +0,0 @@
|
||||
/*
|
||||
* Copyright (c) 2004-2005 The Trustees of Indiana University.
|
||||
* All rights reserved.
|
||||
* Copyright (c) 2004-2005 The Trustees of the University of Tennessee.
|
||||
* All rights reserved.
|
||||
* Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
|
||||
* University of Stuttgart. All rights reserved.
|
||||
* Copyright (c) 2004-2005 The Regents of the University of California.
|
||||
* All rights reserved.
|
||||
* $COPYRIGHT$
|
||||
*
|
||||
* Additional copyrights may follow
|
||||
*
|
||||
* $HEADER$
|
||||
*/
|
||||
|
||||
|
||||
/*%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%*/
|
||||
|
||||
#include "ompi_config.h"
|
||||
#include "include/constants.h"
|
||||
#include "mca/pml/pml.h"
|
||||
#include "mca/ptl/ptl.h"
|
||||
#include "pml_teg.h"
|
||||
#include "pml_teg_proc.h"
|
||||
#include "pml_teg_sendreq.h"
|
||||
#include "pml_teg_recvreq.h"
|
||||
|
||||
|
||||
|
||||
static int mca_pml_teg_send_request_fini(struct ompi_request_t** request)
|
||||
{
|
||||
MCA_PML_TEG_FINI(request);
|
||||
return OMPI_SUCCESS;
|
||||
}
|
||||
|
||||
static int mca_pml_teg_send_request_free(struct ompi_request_t** request)
|
||||
{
|
||||
MCA_PML_TEG_FREE(request);
|
||||
return OMPI_SUCCESS;
|
||||
}
|
||||
|
||||
static int mca_pml_teg_send_request_cancel(struct ompi_request_t* request, int complete)
|
||||
{
|
||||
/* we dont cancel send requests by now */
|
||||
return OMPI_SUCCESS;
|
||||
}
|
||||
|
||||
|
||||
static void mca_pml_teg_send_request_construct(mca_pml_base_send_request_t* req)
|
||||
{
|
||||
req->req_base.req_type = MCA_PML_REQUEST_SEND;
|
||||
req->req_base.req_ompi.req_fini = mca_pml_teg_send_request_fini;
|
||||
req->req_base.req_ompi.req_free = mca_pml_teg_send_request_free;
|
||||
req->req_base.req_ompi.req_cancel = mca_pml_teg_send_request_cancel;
|
||||
}
|
||||
|
||||
|
||||
static void mca_pml_teg_send_request_destruct(mca_pml_base_send_request_t* req)
|
||||
{
|
||||
}
|
||||
|
||||
|
||||
OBJ_CLASS_INSTANCE(
|
||||
mca_pml_teg_send_request_t,
|
||||
mca_pml_base_send_request_t,
|
||||
mca_pml_teg_send_request_construct,
|
||||
mca_pml_teg_send_request_destruct);
|
||||
|
||||
|
||||
|
||||
/**
|
||||
* Schedule message delivery across potentially multiple PTLs.
|
||||
*
|
||||
* @param request (IN) Request to schedule
|
||||
* @return status Error status
|
||||
*
|
||||
*/
|
||||
|
||||
|
||||
int mca_pml_teg_send_request_schedule(mca_pml_base_send_request_t* req)
|
||||
{
|
||||
ompi_proc_t *proc = ompi_comm_peer_lookup(req->req_base.req_comm, req->req_base.req_peer);
|
||||
mca_pml_proc_t* proc_pml = proc->proc_pml;
|
||||
int send_count = 0;
|
||||
size_t bytes_remaining;
|
||||
size_t num_ptl_avail;
|
||||
size_t num_ptl;
|
||||
|
||||
/*
|
||||
* Only allow one thread in this routine for a given request.
|
||||
* However, we cannot block callers on a mutex, so simply keep track
|
||||
* of the number of times the routine has been called and run through
|
||||
* the scheduling logic once for every call.
|
||||
*/
|
||||
if(OMPI_THREAD_ADD32(&req->req_lock,1) == 1) {
|
||||
do {
|
||||
/* allocate remaining bytes to PTLs */
|
||||
bytes_remaining = req->req_bytes_packed - req->req_offset;
|
||||
num_ptl_avail = proc_pml->proc_ptl_next.ptl_size;
|
||||
num_ptl = 0;
|
||||
while(bytes_remaining > 0 && num_ptl++ < num_ptl_avail) {
|
||||
mca_ptl_proc_t* ptl_proc = mca_ptl_array_get_next(&proc_pml->proc_ptl_next);
|
||||
mca_ptl_base_module_t* ptl = ptl_proc->ptl;
|
||||
int rc;
|
||||
|
||||
/* if this is the last PTL that is available to use, or the number of
|
||||
* bytes remaining in the message is less than the PTLs minimum fragment
|
||||
* size, then go ahead and give the rest of the message to this PTL.
|
||||
*/
|
||||
size_t bytes_to_frag;
|
||||
if(num_ptl == num_ptl_avail || bytes_remaining < ptl->ptl_min_frag_size) {
|
||||
bytes_to_frag = bytes_remaining;
|
||||
|
||||
/* otherwise attempt to give the PTL a percentage of the message
|
||||
* based on a weighting factor. for simplicity calculate this as
|
||||
* a percentage of the overall message length (regardless of amount
|
||||
* previously assigned)
|
||||
*/
|
||||
} else {
|
||||
bytes_to_frag = (ptl_proc->ptl_weight * bytes_remaining) / 100;
|
||||
}
|
||||
|
||||
/* makes sure that we don't exceed ptl_max_frag_size */
|
||||
if(ptl->ptl_max_frag_size != 0 && bytes_to_frag > ptl->ptl_max_frag_size)
|
||||
bytes_to_frag = ptl->ptl_max_frag_size;
|
||||
|
||||
rc = ptl->ptl_put(ptl, ptl_proc->ptl_peer, req, req->req_offset, bytes_to_frag, 0);
|
||||
if(rc == OMPI_SUCCESS) {
|
||||
send_count++;
|
||||
bytes_remaining = req->req_bytes_packed - req->req_offset;
|
||||
}
|
||||
}
|
||||
|
||||
/* unable to complete send - queue for later */
|
||||
if(send_count == 0) {
|
||||
OMPI_THREAD_LOCK(&mca_pml_teg.teg_lock);
|
||||
ompi_list_append(&mca_pml_teg.teg_send_pending, (ompi_list_item_t*)req);
|
||||
OMPI_THREAD_UNLOCK(&mca_pml_teg.teg_lock);
|
||||
req->req_lock = 0;
|
||||
return OMPI_ERR_OUT_OF_RESOURCE;
|
||||
}
|
||||
|
||||
/* fragments completed while scheduling - so retry */
|
||||
} while(OMPI_THREAD_ADD32(&req->req_lock,-1) > 0);
|
||||
|
||||
/* free the request if completed while in the scheduler */
|
||||
if (req->req_base.req_free_called && req->req_base.req_pml_complete) {
|
||||
MCA_PML_TEG_FREE((ompi_request_t**)&req);
|
||||
}
|
||||
}
|
||||
return OMPI_SUCCESS;
|
||||
}
|
||||
|
||||
|
||||
/**
|
||||
* Update the status of the send request to reflect the number of bytes
|
||||
* "actually" sent (and acknowledged). This should be called by the
|
||||
* lower layer PTL after the fragment is actually delivered and has been
|
||||
* acknowledged (if required). Note that this routine should NOT be called
|
||||
* directly by the PTL, a function pointer is setup on the PTL at init to
|
||||
* enable upcalls into the PML w/out directly linking to a specific PML
|
||||
* implementation.
|
||||
*/
|
||||
|
||||
void mca_pml_teg_send_request_progress(
|
||||
struct mca_ptl_base_module_t* ptl,
|
||||
mca_pml_base_send_request_t* req,
|
||||
size_t bytes_sent)
|
||||
{
|
||||
bool schedule = false;
|
||||
|
||||
OMPI_THREAD_LOCK(&ompi_request_lock);
|
||||
req->req_bytes_sent += bytes_sent;
|
||||
if (req->req_bytes_sent >= req->req_bytes_packed) {
|
||||
req->req_base.req_pml_complete = true;
|
||||
if (req->req_base.req_ompi.req_complete == false) {
|
||||
req->req_base.req_ompi.req_status.MPI_SOURCE = req->req_base.req_comm->c_my_rank;
|
||||
req->req_base.req_ompi.req_status.MPI_TAG = req->req_base.req_tag;
|
||||
req->req_base.req_ompi.req_status.MPI_ERROR = OMPI_SUCCESS;
|
||||
req->req_base.req_ompi.req_status._count = req->req_bytes_sent;
|
||||
req->req_base.req_ompi.req_complete = true;
|
||||
if(ompi_request_waiting) {
|
||||
ompi_condition_broadcast(&ompi_request_cond);
|
||||
}
|
||||
} else if(req->req_base.req_free_called) {
|
||||
/* don't free the request if in the scheduler */
|
||||
if(req->req_lock == 0) {
|
||||
MCA_PML_TEG_FREE((ompi_request_t**)&req);
|
||||
}
|
||||
} else if (req->req_send_mode == MCA_PML_BASE_SEND_BUFFERED) {
|
||||
mca_pml_base_bsend_request_fini((ompi_request_t*)req);
|
||||
}
|
||||
/* test to see if we have scheduled the entire request */
|
||||
} else if (req->req_offset < req->req_bytes_packed) {
|
||||
schedule = true;
|
||||
}
|
||||
OMPI_THREAD_UNLOCK(&ompi_request_lock);
|
||||
|
||||
/* schedule remaining fragments of this request */
|
||||
if(schedule) {
|
||||
mca_pml_teg_send_request_schedule(req);
|
||||
}
|
||||
|
||||
/* check for pending requests that need to be progressed */
|
||||
while(ompi_list_get_size(&mca_pml_teg.teg_send_pending) != 0) {
|
||||
OMPI_THREAD_LOCK(&mca_pml_teg.teg_lock);
|
||||
req = (mca_pml_base_send_request_t*)ompi_list_remove_first(&mca_pml_teg.teg_send_pending);
|
||||
OMPI_THREAD_UNLOCK(&mca_pml_teg.teg_lock);
|
||||
if(req == NULL)
|
||||
break;
|
||||
if(mca_pml_teg_send_request_schedule(req) != OMPI_SUCCESS)
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
@ -1,187 +0,0 @@
|
||||
/*
|
||||
* Copyright (c) 2004-2005 The Trustees of Indiana University.
|
||||
* All rights reserved.
|
||||
* Copyright (c) 2004-2005 The Trustees of the University of Tennessee.
|
||||
* All rights reserved.
|
||||
* Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
|
||||
* University of Stuttgart. All rights reserved.
|
||||
* Copyright (c) 2004-2005 The Regents of the University of California.
|
||||
* All rights reserved.
|
||||
* $COPYRIGHT$
|
||||
*
|
||||
* Additional copyrights may follow
|
||||
*
|
||||
* $HEADER$
|
||||
*/
|
||||
/**
|
||||
* @file
|
||||
*/
|
||||
#ifndef OMPI_PML_TEG_SEND_REQUEST_H
|
||||
#define OMPI_PML_TEG_SEND_REQUEST_H
|
||||
|
||||
#include "mca/ptl/ptl.h"
|
||||
#include "mca/pml/base/pml_base_sendreq.h"
|
||||
#include "mca/ptl/base/ptl_base_sendfrag.h"
|
||||
#include "mca/ptl/base/ptl_base_comm.h"
|
||||
#include "pml_teg_proc.h"
|
||||
#include "pml_teg_ptl.h"
|
||||
|
||||
#if defined(c_plusplus) || defined(__cplusplus)
|
||||
extern "C" {
|
||||
#endif
|
||||
typedef mca_pml_base_send_request_t mca_pml_teg_send_request_t;
|
||||
|
||||
OBJ_CLASS_DECLARATION(mca_pml_teg_send_request_t);
|
||||
|
||||
|
||||
#define MCA_PML_TEG_SEND_REQUEST_ALLOC( \
|
||||
comm, \
|
||||
dst, \
|
||||
sendreq, \
|
||||
rc) \
|
||||
{ \
|
||||
mca_pml_proc_t *proc = mca_pml_teg_proc_lookup_remote(comm,dst); \
|
||||
mca_ptl_proc_t* ptl_proc; \
|
||||
mca_pml_base_ptl_t* ptl_base; \
|
||||
\
|
||||
if(NULL == proc) { \
|
||||
return OMPI_ERR_OUT_OF_RESOURCE; \
|
||||
} \
|
||||
OMPI_THREAD_SCOPED_LOCK(&proc->proc_lock, \
|
||||
(ptl_proc = mca_ptl_array_get_next(&proc->proc_ptl_first))); \
|
||||
ptl_base = ptl_proc->ptl_base; \
|
||||
/* \
|
||||
* check to see if there is a cache of send requests associated with \
|
||||
* this ptl - if so try the allocation from there. \
|
||||
*/ \
|
||||
if(NULL != ptl_base) { \
|
||||
OMPI_THREAD_LOCK(&ptl_base->ptl_cache_lock); \
|
||||
sendreq = (mca_pml_base_send_request_t*) \
|
||||
ompi_list_remove_first(&ptl_base->ptl_cache); \
|
||||
if(NULL != sendreq) { \
|
||||
OMPI_THREAD_UNLOCK(&ptl_base->ptl_cache_lock); \
|
||||
rc = OMPI_SUCCESS; \
|
||||
} else if (ptl_base->ptl_cache_alloc < ptl_base->ptl_cache_size) { \
|
||||
/* \
|
||||
* allocate an additional request to the cache \
|
||||
*/ \
|
||||
mca_ptl_base_module_t* ptl = ptl_base->ptl; \
|
||||
ompi_list_item_t* item; \
|
||||
OMPI_FREE_LIST_WAIT(&mca_pml_teg.teg_send_requests, item, rc); \
|
||||
sendreq = (mca_pml_base_send_request_t*)item; \
|
||||
sendreq->req_ptl = ptl; \
|
||||
if(ptl->ptl_request_init(ptl, sendreq) == OMPI_SUCCESS) { \
|
||||
sendreq->req_cached = true; \
|
||||
ptl_base->ptl_cache_alloc++; \
|
||||
} \
|
||||
OMPI_THREAD_UNLOCK(&ptl_base->ptl_cache_lock); \
|
||||
} else { \
|
||||
/* \
|
||||
* take a request from the global pool \
|
||||
*/ \
|
||||
ompi_list_item_t* item; \
|
||||
OMPI_THREAD_UNLOCK(&ptl_base->ptl_cache_lock); \
|
||||
OMPI_FREE_LIST_WAIT(&mca_pml_teg.teg_send_requests, item, rc); \
|
||||
sendreq = (mca_pml_base_send_request_t*)item; \
|
||||
sendreq->req_ptl = ptl_proc->ptl; \
|
||||
} \
|
||||
\
|
||||
/* otherwise - take the allocation from the global list */ \
|
||||
} else { \
|
||||
ompi_list_item_t* item; \
|
||||
OMPI_FREE_LIST_WAIT(&mca_pml_teg.teg_send_requests, item, rc); \
|
||||
sendreq = (mca_pml_base_send_request_t*)item; \
|
||||
sendreq->req_ptl = ptl_proc->ptl; \
|
||||
} \
|
||||
/* update request to point to current peer */ \
|
||||
sendreq->req_peer = ptl_proc->ptl_peer; \
|
||||
}
|
||||
|
||||
|
||||
#define MCA_PML_TEG_SEND_REQUEST_RETURN(sendreq) \
|
||||
{ \
|
||||
mca_ptl_base_module_t* ptl = (sendreq)->req_ptl; \
|
||||
mca_pml_base_ptl_t* ptl_base = ptl->ptl_base; \
|
||||
\
|
||||
/* Let the base handle the reference counts */ \
|
||||
MCA_PML_BASE_SEND_REQUEST_RETURN(sendreq); \
|
||||
\
|
||||
/* \
|
||||
* If there is a cache associated with the ptl - first attempt \
|
||||
* to return the send descriptor to the cache. \
|
||||
*/ \
|
||||
if(NULL != ptl->ptl_base && (sendreq)->req_cached) { \
|
||||
OMPI_THREAD_LOCK(&ptl_base->ptl_cache_lock); \
|
||||
ompi_list_prepend(&ptl_base->ptl_cache, \
|
||||
(ompi_list_item_t*)sendreq); \
|
||||
OMPI_THREAD_UNLOCK(&ptl_base->ptl_cache_lock); \
|
||||
} else { \
|
||||
OMPI_FREE_LIST_RETURN( \
|
||||
&mca_pml_teg.teg_send_requests, (ompi_list_item_t*)sendreq); \
|
||||
} \
|
||||
}
|
||||
|
||||
|
||||
/**
|
||||
* Start a send request.
|
||||
*/
|
||||
#define MCA_PML_TEG_SEND_REQUEST_START(req, rc) \
|
||||
{ \
|
||||
mca_ptl_base_module_t* ptl = req->req_ptl; \
|
||||
size_t first_fragment_size = ptl->ptl_first_frag_size; \
|
||||
int flags; \
|
||||
\
|
||||
req->req_offset = 0; \
|
||||
req->req_lock = 0; \
|
||||
req->req_bytes_sent = 0; \
|
||||
req->req_peer_match.lval = 0; \
|
||||
req->req_peer_addr.lval = 0; \
|
||||
req->req_peer_size = 0; \
|
||||
req->req_base.req_pml_complete = false; \
|
||||
req->req_base.req_ompi.req_complete = false; \
|
||||
req->req_base.req_ompi.req_state = OMPI_REQUEST_ACTIVE; \
|
||||
req->req_base.req_sequence = mca_pml_ptl_comm_send_sequence( \
|
||||
req->req_base.req_comm->c_pml_comm, req->req_base.req_peer); \
|
||||
\
|
||||
/* handle buffered send */ \
|
||||
if(req->req_send_mode == MCA_PML_BASE_SEND_BUFFERED) { \
|
||||
mca_pml_base_bsend_request_start(&req->req_base.req_ompi); \
|
||||
} \
|
||||
\
|
||||
/* start the first fragment */ \
|
||||
if (first_fragment_size == 0 || \
|
||||
req->req_bytes_packed <= first_fragment_size) { \
|
||||
first_fragment_size = req->req_bytes_packed; \
|
||||
flags = (req->req_send_mode == MCA_PML_BASE_SEND_SYNCHRONOUS) ? \
|
||||
MCA_PTL_FLAGS_ACK : 0; \
|
||||
} else { \
|
||||
/* require match for first fragment of a multi-fragment */ \
|
||||
flags = MCA_PTL_FLAGS_ACK; \
|
||||
} \
|
||||
rc = ptl->ptl_send(ptl, req->req_peer, req, 0, first_fragment_size, \
|
||||
flags); \
|
||||
}
|
||||
|
||||
|
||||
/**
|
||||
* Schedule any data that was not delivered in the first fragment
|
||||
* across the available PTLs.
|
||||
*/
|
||||
int mca_pml_teg_send_request_schedule(mca_pml_base_send_request_t* req);
|
||||
|
||||
|
||||
/**
|
||||
* Update the request to reflect the number of bytes delivered. If this
|
||||
* was the first fragment - schedule the rest of the data.
|
||||
*/
|
||||
void mca_pml_teg_send_request_progress(
|
||||
struct mca_ptl_base_module_t* ptl,
|
||||
mca_pml_base_send_request_t* send_request,
|
||||
size_t bytes_sent
|
||||
);
|
||||
|
||||
#if defined(c_plusplus) || defined(__cplusplus)
|
||||
}
|
||||
#endif
|
||||
#endif
|
||||
|
||||
@ -1,122 +0,0 @@
|
||||
/*
|
||||
* Copyright (c) 2004-2005 The Trustees of Indiana University.
|
||||
* All rights reserved.
|
||||
* Copyright (c) 2004-2005 The Trustees of the University of Tennessee.
|
||||
* All rights reserved.
|
||||
* Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
|
||||
* University of Stuttgart. All rights reserved.
|
||||
* Copyright (c) 2004-2005 The Regents of the University of California.
|
||||
* All rights reserved.
|
||||
* $COPYRIGHT$
|
||||
*
|
||||
* Additional copyrights may follow
|
||||
*
|
||||
* $HEADER$
|
||||
*/
|
||||
|
||||
#include "ompi_config.h"
|
||||
|
||||
#include "pml_teg.h"
|
||||
#include "pml_teg_recvreq.h"
|
||||
#include "pml_teg_sendreq.h"
|
||||
|
||||
|
||||
int mca_pml_teg_start(size_t count, ompi_request_t** requests)
|
||||
{
|
||||
int rc;
|
||||
size_t i;
|
||||
for(i=0; i<count; i++) {
|
||||
mca_pml_base_request_t *pml_request = (mca_pml_base_request_t*)requests[i];
|
||||
if(NULL == pml_request)
|
||||
continue;
|
||||
|
||||
/* If the persistent request is currently active - obtain the
|
||||
* request lock and verify the status is incomplete. if the
|
||||
* pml layer has not completed the request - mark the request
|
||||
* as free called - so that it will be freed when the request
|
||||
* completes - and create a new request.
|
||||
*/
|
||||
|
||||
switch(pml_request->req_ompi.req_state) {
|
||||
case OMPI_REQUEST_INACTIVE:
|
||||
if(pml_request->req_pml_complete == true)
|
||||
break;
|
||||
/* otherwise fall through */
|
||||
case OMPI_REQUEST_ACTIVE: {
|
||||
|
||||
ompi_request_t *request;
|
||||
OMPI_THREAD_LOCK(&ompi_request_lock);
|
||||
if (pml_request->req_pml_complete == false) {
|
||||
/* free request after it completes */
|
||||
pml_request->req_free_called = true;
|
||||
} else {
|
||||
/* can reuse the existing request */
|
||||
OMPI_THREAD_UNLOCK(&ompi_request_lock);
|
||||
break;
|
||||
}
|
||||
|
||||
/* allocate a new request */
|
||||
switch(pml_request->req_type) {
|
||||
case MCA_PML_REQUEST_SEND: {
|
||||
mca_pml_base_send_mode_t sendmode =
|
||||
((mca_pml_base_send_request_t*)pml_request)->req_send_mode;
|
||||
rc = mca_pml_teg_isend_init(
|
||||
pml_request->req_addr,
|
||||
pml_request->req_count,
|
||||
pml_request->req_datatype,
|
||||
pml_request->req_peer,
|
||||
pml_request->req_tag,
|
||||
sendmode,
|
||||
pml_request->req_comm,
|
||||
&request);
|
||||
break;
|
||||
}
|
||||
case MCA_PML_REQUEST_RECV:
|
||||
rc = mca_pml_teg_irecv_init(
|
||||
pml_request->req_addr,
|
||||
pml_request->req_count,
|
||||
pml_request->req_datatype,
|
||||
pml_request->req_peer,
|
||||
pml_request->req_tag,
|
||||
pml_request->req_comm,
|
||||
&request);
|
||||
break;
|
||||
default:
|
||||
rc = OMPI_ERR_REQUEST;
|
||||
break;
|
||||
}
|
||||
OMPI_THREAD_UNLOCK(&ompi_request_lock);
|
||||
if(OMPI_SUCCESS != rc)
|
||||
return rc;
|
||||
pml_request = (mca_pml_base_request_t*)request;
|
||||
requests[i] = request;
|
||||
break;
|
||||
}
|
||||
default:
|
||||
return OMPI_ERR_REQUEST;
|
||||
}
|
||||
|
||||
/* start the request */
|
||||
switch(pml_request->req_type) {
|
||||
case MCA_PML_REQUEST_SEND:
|
||||
{
|
||||
mca_pml_base_send_request_t* sendreq = (mca_pml_base_send_request_t*)pml_request;
|
||||
MCA_PML_TEG_SEND_REQUEST_START(sendreq, rc);
|
||||
if(rc != OMPI_SUCCESS)
|
||||
return rc;
|
||||
break;
|
||||
}
|
||||
case MCA_PML_REQUEST_RECV:
|
||||
{
|
||||
mca_pml_base_recv_request_t* recvreq = (mca_pml_base_recv_request_t*)pml_request;
|
||||
if((rc = mca_pml_teg_recv_request_start(recvreq)) != OMPI_SUCCESS)
|
||||
return rc;
|
||||
break;
|
||||
}
|
||||
default:
|
||||
return OMPI_ERR_REQUEST;
|
||||
}
|
||||
}
|
||||
return OMPI_SUCCESS;
|
||||
}
|
||||
|
||||
Загрузка…
x
Ссылка в новой задаче
Block a user