reorg to support new pml - move ptl/teg specific data
from pml_base_* data structures *** NOTE *** this will break gm/ib - I will fix these shortly This commit was SVN r5655.
Этот коммит содержится в:
Tim Woodall
родитель
ac60eac8f3
Коммит
3d3151abb3
@ -972,11 +972,6 @@ int ompi_comm_determine_first ( ompi_communicator_t *intercomm, int high )
|
||||
/********************************************************************************/
|
||||
int ompi_comm_dump ( ompi_communicator_t *comm )
|
||||
{
|
||||
mca_pml_ptl_comm_t *pml_comm;
|
||||
mca_ptl_sequence_t *seq;
|
||||
ompi_list_t *list;
|
||||
int i;
|
||||
|
||||
ompi_output(0, "Dumping information for comm_cid %d\n", comm->c_contextid);
|
||||
ompi_output(0," f2c index:%d cube_dim: %d\n", comm->c_f_to_c_index,
|
||||
comm->c_cube_dim);
|
||||
@ -997,23 +992,6 @@ int ompi_comm_dump ( ompi_communicator_t *comm )
|
||||
if (OMPI_COMM_IS_INTER(comm)) {
|
||||
ompi_output(0," Remote group size:%d\n", comm->c_remote_group->grp_proc_count);
|
||||
}
|
||||
|
||||
|
||||
/* Dump the c_pml_comm->c_unexpexted_frags for each process */
|
||||
pml_comm = (mca_pml_ptl_comm_t *)comm->c_pml_comm;
|
||||
seq = (mca_ptl_sequence_t *) pml_comm->c_frags_cant_match;
|
||||
for ( i = 0; i < comm->c_local_group->grp_proc_count; i++ ){
|
||||
list = (ompi_list_t *)seq+i;
|
||||
ompi_output(0,"%d: head->list_next:%p head->list_prev:%p"
|
||||
" tail->list_next:%p tail->list_next:%p\n",
|
||||
i,
|
||||
(char*)list->ompi_list_head.ompi_list_next,
|
||||
(char*)list->ompi_list_head.ompi_list_prev,
|
||||
(char*)list->ompi_list_tail.ompi_list_next,
|
||||
(char*)list->ompi_list_tail.ompi_list_prev );
|
||||
fflush(stdout);
|
||||
}
|
||||
|
||||
return MPI_SUCCESS;
|
||||
}
|
||||
/********************************************************************************/
|
||||
|
||||
@ -272,6 +272,7 @@ static void ompi_comm_construct(ompi_communicator_t* comm)
|
||||
comm->c_remote_group = NULL;
|
||||
comm->error_handler = NULL;
|
||||
comm->c_pml_comm = NULL;
|
||||
comm->c_pml_procs = NULL;
|
||||
comm->c_topo = NULL;
|
||||
comm->c_topo_component = NULL;
|
||||
comm->c_topo_comm = NULL;
|
||||
|
||||
@ -131,8 +131,8 @@ struct ompi_communicator_t {
|
||||
ompi_errhandler_type_t errhandler_type;
|
||||
|
||||
/* Hooks for PML to hang things */
|
||||
|
||||
struct mca_pml_comm_t *c_pml_comm;
|
||||
struct mca_pml_comm_t *c_pml_comm;
|
||||
struct mca_pml_proc_t **c_pml_procs;
|
||||
|
||||
mca_coll_base_module_1_0_0_t c_coll;
|
||||
/**< Selected collective module, saved by value for speed (instead
|
||||
|
||||
@ -30,8 +30,6 @@ OMPI_DECLSPEC extern ompi_class_t mca_pml_base_recv_request_t_class;
|
||||
struct mca_pml_base_recv_request_t {
|
||||
mca_pml_base_request_t req_base; /**< base request */
|
||||
size_t req_bytes_packed; /**< size of message being received */
|
||||
size_t req_bytes_received; /**< number of bytes received from network */
|
||||
size_t req_bytes_delivered; /**< number of bytes delivered to user */
|
||||
};
|
||||
typedef struct mca_pml_base_recv_request_t mca_pml_base_recv_request_t;
|
||||
|
||||
|
||||
@ -34,11 +34,11 @@ static void mca_pml_base_send_request_construct(mca_pml_base_send_request_t* req
|
||||
{
|
||||
/* no need to reinit for every send -- never changes */
|
||||
request->req_base.req_type = MCA_PML_REQUEST_SEND;
|
||||
request->req_cached = false;
|
||||
OBJ_CONSTRUCT(&request->req_convertor, ompi_convertor_t);
|
||||
}
|
||||
|
||||
static void mca_pml_base_send_request_destruct(mca_pml_base_send_request_t* req)
|
||||
{
|
||||
OBJ_DESTRUCT(&req->req_convertor);
|
||||
}
|
||||
|
||||
|
||||
@ -35,21 +35,12 @@ OMPI_DECLSPEC extern ompi_class_t mca_pml_base_send_request_t_class;
|
||||
*/
|
||||
struct mca_pml_base_send_request_t {
|
||||
mca_pml_base_request_t req_base; /** base request type - common data structure for use by wait/test */
|
||||
ompi_datatype_t* req_datatype; /**< pointer to datatype */
|
||||
void *req_addr; /**< pointer to send buffer - may not be application buffer */
|
||||
size_t req_count; /**< number of elements in send buffer */
|
||||
ompi_datatype_t* req_datatype; /**< pointer to datatype */
|
||||
size_t req_offset; /**< number of bytes that have already been assigned to a fragment */
|
||||
size_t req_bytes_packed; /**< packed size of a message given the datatype and count */
|
||||
size_t req_bytes_sent; /**< number of bytes that have been sent */
|
||||
mca_pml_base_send_mode_t req_send_mode; /**< type of send */
|
||||
struct mca_ptl_base_module_t* req_ptl; /**< PTL that is selected for first fragment */
|
||||
struct mca_ptl_base_peer_t* req_peer; /**< PTL peer instance that will be used for first fragment */
|
||||
ompi_ptr_t req_peer_match; /**< matched receive at peer */
|
||||
ompi_ptr_t req_peer_addr; /**< peers remote buffer address */
|
||||
uint64_t req_peer_size; /**< size of peers remote buffer */
|
||||
bool req_cached; /**< has this request been obtained from the ptls cache */
|
||||
ompi_convertor_t req_convertor; /**< convertor that describes this datatype */
|
||||
volatile int32_t req_lock; /**< lock used by the scheduler */
|
||||
};
|
||||
typedef struct mca_pml_base_send_request_t mca_pml_base_send_request_t;
|
||||
|
||||
@ -137,32 +128,6 @@ typedef struct mca_pml_base_send_request_t mca_pml_base_send_request_t;
|
||||
OBJ_RELEASE(request->req_base.req_datatype); \
|
||||
} while (0)
|
||||
|
||||
/**
|
||||
* Test to check if an acknowledgment has been received, with the match.
|
||||
*
|
||||
* @param request (IN) Send request.
|
||||
* return TRUE if an ack/match has been received from peer.
|
||||
*/
|
||||
|
||||
static inline bool mca_pml_base_send_request_matched(
|
||||
mca_pml_base_send_request_t* request)
|
||||
{
|
||||
return (NULL != request->req_peer_match.pval);
|
||||
}
|
||||
|
||||
/**
|
||||
* Atomically increase the request offset.
|
||||
*
|
||||
* @param request (IN) Send request.
|
||||
* @param offset (IN) Increment.
|
||||
*/
|
||||
|
||||
static inline void mca_pml_base_send_request_offset(
|
||||
mca_pml_base_send_request_t* request,
|
||||
size_t offset)
|
||||
{
|
||||
OMPI_THREAD_ADD32((int32_t*)(&request->req_offset), offset);
|
||||
}
|
||||
|
||||
#if defined(c_plusplus) || defined(__cplusplus)
|
||||
}
|
||||
|
||||
@ -14,34 +14,28 @@
|
||||
# $HEADER$
|
||||
#
|
||||
|
||||
# Use the top-level Makefile.options
|
||||
|
||||
include $(top_ompi_srcdir)/config/Makefile.options
|
||||
|
||||
sources =
|
||||
include src/Makefile.extra
|
||||
|
||||
# Make the output library in this directory, and name it either
|
||||
# mca_<type>_<name>.la (for DSO builds) or libmca_<type>_<name>.la
|
||||
# (for static builds).
|
||||
|
||||
if OMPI_BUILD_pml_teg_DSO
|
||||
lib =
|
||||
lib_sources =
|
||||
component = mca_pml_teg.la
|
||||
component_sources = $(sources)
|
||||
else
|
||||
lib = libmca_pml_teg.la
|
||||
lib_sources = $(sources)
|
||||
component =
|
||||
component_sources =
|
||||
endif
|
||||
|
||||
mcacomponentdir = $(libdir)/openmpi
|
||||
mcacomponent_LTLIBRARIES = $(component)
|
||||
mca_pml_teg_la_SOURCES = $(component_sources)
|
||||
mca_pml_teg_la_LDFLAGS = -module -avoid-version
|
||||
|
||||
noinst_LTLIBRARIES = $(lib)
|
||||
libmca_pml_teg_la_SOURCES = $(lib_sources)
|
||||
libmca_pml_teg_la_LDFLAGS = -module -avoid-version
|
||||
noinst_LTLIBRARIES = libmca_pml_teg.la
|
||||
libmca_pml_teg_la_SOURCES = \
|
||||
pml_teg.c \
|
||||
pml_teg.h \
|
||||
pml_teg_cancel.c \
|
||||
pml_teg_component.c \
|
||||
pml_teg_component.h \
|
||||
pml_teg_iprobe.c \
|
||||
pml_teg_irecv.c \
|
||||
pml_teg_isend.c \
|
||||
pml_teg_ptl.c \
|
||||
pml_teg_ptl.h \
|
||||
pml_teg_proc.c \
|
||||
pml_teg_proc.h \
|
||||
pml_teg_progress.c \
|
||||
pml_teg_recvfrag.c \
|
||||
pml_teg_recvfrag.h \
|
||||
pml_teg_recvreq.c \
|
||||
pml_teg_recvreq.h \
|
||||
pml_teg_sendreq.c \
|
||||
pml_teg_sendreq.h \
|
||||
pml_teg_start.c \
|
||||
pml_ptl_array.c
|
||||
|
||||
@ -17,6 +17,6 @@
|
||||
|
||||
# Specific to this module
|
||||
|
||||
PARAM_INIT_FILE=src/pml_teg.c
|
||||
PARAM_CONFIG_HEADER_FILE="src/teg_config.h"
|
||||
PARAM_INIT_FILE=pml_teg.c
|
||||
PARAM_CONFIG_HEADER_FILE="teg_config.h"
|
||||
PARAM_CONFIG_FILES="Makefile"
|
||||
|
||||
61
src/mca/pml/teg/pml_ptl_array.c
Обычный файл
61
src/mca/pml/teg/pml_ptl_array.c
Обычный файл
@ -0,0 +1,61 @@
|
||||
/*
|
||||
* Copyright (c) 2004-2005 The Trustees of Indiana University.
|
||||
* All rights reserved.
|
||||
* Copyright (c) 2004-2005 The Trustees of the University of Tennessee.
|
||||
* All rights reserved.
|
||||
* Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
|
||||
* University of Stuttgart. All rights reserved.
|
||||
* Copyright (c) 2004-2005 The Regents of the University of California.
|
||||
* All rights reserved.
|
||||
* $COPYRIGHT$
|
||||
*
|
||||
* Additional copyrights may follow
|
||||
*
|
||||
* $HEADER$
|
||||
*/
|
||||
|
||||
#include "ompi_config.h"
|
||||
|
||||
#include <string.h>
|
||||
|
||||
#include "mca/pml/pml.h"
|
||||
#include "pml_ptl_array.h"
|
||||
|
||||
|
||||
static void mca_ptl_array_construct(mca_ptl_array_t* array)
|
||||
{
|
||||
array->ptl_procs = 0;
|
||||
array->ptl_size = 0;
|
||||
array->ptl_index = 0;
|
||||
array->ptl_reserve = 0;
|
||||
}
|
||||
|
||||
|
||||
static void mca_ptl_array_destruct(mca_ptl_array_t* array)
|
||||
{
|
||||
if(array->ptl_procs != 0)
|
||||
free(array->ptl_procs);
|
||||
}
|
||||
|
||||
OBJ_CLASS_INSTANCE(
|
||||
mca_pml_teg_ptl_array_t,
|
||||
ompi_object_t,
|
||||
mca_ptl_array_construct,
|
||||
mca_ptl_array_destruct
|
||||
);
|
||||
|
||||
int mca_ptl_array_reserve(mca_ptl_array_t* array, size_t size)
|
||||
{
|
||||
mca_ptl_proc_t *procs;
|
||||
if(array->ptl_reserve >= size)
|
||||
return OMPI_SUCCESS;
|
||||
|
||||
procs = (mca_ptl_proc_t *)realloc(array->ptl_procs, sizeof(mca_ptl_proc_t)*size);
|
||||
if(NULL == procs)
|
||||
return OMPI_ERR_OUT_OF_RESOURCE;
|
||||
array->ptl_procs = procs;
|
||||
array->ptl_reserve = size;
|
||||
memset(array->ptl_procs+array->ptl_size, 0, (size-array->ptl_size)*sizeof(mca_ptl_proc_t));
|
||||
return OMPI_SUCCESS;
|
||||
}
|
||||
|
||||
146
src/mca/pml/teg/pml_ptl_array.h
Обычный файл
146
src/mca/pml/teg/pml_ptl_array.h
Обычный файл
@ -0,0 +1,146 @@
|
||||
/*
|
||||
* Copyright (c) 2004-2005 The Trustees of Indiana University.
|
||||
* All rights reserved.
|
||||
* Copyright (c) 2004-2005 The Trustees of the University of Tennessee.
|
||||
* All rights reserved.
|
||||
* Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
|
||||
* University of Stuttgart. All rights reserved.
|
||||
* Copyright (c) 2004-2005 The Regents of the University of California.
|
||||
* All rights reserved.
|
||||
* $COPYRIGHT$
|
||||
*
|
||||
* Additional copyrights may follow
|
||||
*
|
||||
* $HEADER$
|
||||
*/
|
||||
/**
|
||||
* @file
|
||||
*/
|
||||
#ifndef OMPI_PTL_ARRAY_H
|
||||
#define OMPI_PTL_ARRAY_H
|
||||
|
||||
#include "util/output.h"
|
||||
#include "mca/ptl/ptl.h"
|
||||
#if defined(c_plusplus) || defined(__cplusplus)
|
||||
extern "C" {
|
||||
#endif
|
||||
|
||||
extern ompi_class_t mca_pml_teg_ptl_array_t_class;
|
||||
|
||||
/**
|
||||
* A data structure associated with a ompi_proc_t that caches
|
||||
* addressing/scheduling attributes for a specific PTL instance
|
||||
* that can be used to reach the process.
|
||||
*/
|
||||
struct mca_ptl_proc_t {
|
||||
int ptl_weight; /**< PTL weight for scheduling */
|
||||
struct mca_ptl_base_peer_t* ptl_peer; /**< PTL addressing info */
|
||||
struct mca_pml_base_ptl_t* ptl_base; /**< PML specific PTL info */
|
||||
mca_ptl_base_module_t *ptl; /**< PTL module */
|
||||
};
|
||||
typedef struct mca_ptl_proc_t mca_ptl_proc_t;
|
||||
|
||||
/**
|
||||
* A dynamically growable array of mca_ptl_proc_t instances.
|
||||
* Maintains an index into the array that is used for round-robin
|
||||
* scheduling across contents.
|
||||
*/
|
||||
struct mca_ptl_array_t {
|
||||
ompi_object_t super;
|
||||
mca_ptl_proc_t* ptl_procs; /**< array of ptl procs */
|
||||
size_t ptl_size; /**< number available */
|
||||
size_t ptl_reserve; /**< size of allocated ptl_proc array */
|
||||
size_t ptl_index; /**< last used index*/
|
||||
};
|
||||
typedef struct mca_ptl_array_t mca_ptl_array_t;
|
||||
typedef struct mca_ptl_array_t mca_pml_teg_ptl_array_t;
|
||||
|
||||
|
||||
/**
|
||||
* If required, reallocate (grow) the array to the indicate size.
|
||||
*
|
||||
* @param array (IN)
|
||||
* @param size (IN)
|
||||
*/
|
||||
int mca_ptl_array_reserve(mca_ptl_array_t*, size_t);
|
||||
|
||||
static inline size_t mca_ptl_array_get_size(mca_ptl_array_t* array)
|
||||
{
|
||||
return array->ptl_size;
|
||||
}
|
||||
|
||||
/**
|
||||
* Grow the array if required, and set the size.
|
||||
*
|
||||
* @param array (IN)
|
||||
* @param size (IN)
|
||||
*/
|
||||
static inline void mca_ptl_array_set_size(mca_ptl_array_t* array, size_t size)
|
||||
{
|
||||
if(array->ptl_size > array->ptl_reserve)
|
||||
mca_ptl_array_reserve(array, size);
|
||||
array->ptl_size = size;
|
||||
}
|
||||
|
||||
/**
|
||||
* Grow the array size by one and return the item at that index.
|
||||
*
|
||||
* @param array (IN)
|
||||
*/
|
||||
static inline mca_ptl_proc_t* mca_ptl_array_insert(mca_ptl_array_t* array)
|
||||
{
|
||||
#if OMPI_ENABLE_DEBUG
|
||||
if(array->ptl_size >= array->ptl_reserve) {
|
||||
ompi_output(0, "mca_ptl_array_insert: invalid array index %d >= %d",
|
||||
array->ptl_size, array->ptl_reserve);
|
||||
return 0;
|
||||
}
|
||||
#endif
|
||||
return &array->ptl_procs[array->ptl_size++];
|
||||
}
|
||||
|
||||
/**
|
||||
* Return an array item at the specified index.
|
||||
*
|
||||
* @param array (IN)
|
||||
* @param index (IN)
|
||||
*/
|
||||
static inline mca_ptl_proc_t* mca_ptl_array_get_index(mca_ptl_array_t* array, size_t index)
|
||||
{
|
||||
#if OMPI_ENABLE_DEBUG
|
||||
if(index >= array->ptl_size) {
|
||||
ompi_output(0, "mca_ptl_array_get_index: invalid array index %d >= %d",
|
||||
index, array->ptl_size);
|
||||
return 0;
|
||||
}
|
||||
#endif
|
||||
return &array->ptl_procs[index];
|
||||
}
|
||||
|
||||
/**
|
||||
* Return the next LRU index in the array.
|
||||
*
|
||||
* @param array (IN)
|
||||
* @param index (IN)
|
||||
*/
|
||||
static inline mca_ptl_proc_t* mca_ptl_array_get_next(mca_ptl_array_t* array)
|
||||
{
|
||||
mca_ptl_proc_t* ptl_proc;
|
||||
#if OMPI_ENABLE_DEBUG
|
||||
if(array->ptl_size == 0) {
|
||||
ompi_output(0, "mca_ptl_array_get_next: invalid array size");
|
||||
return 0;
|
||||
}
|
||||
#endif
|
||||
ptl_proc = &array->ptl_procs[array->ptl_index++];
|
||||
if(array->ptl_index == array->ptl_size)
|
||||
array->ptl_index = 0;
|
||||
return ptl_proc;
|
||||
}
|
||||
|
||||
#if defined(c_plusplus) || defined(__cplusplus)
|
||||
}
|
||||
#endif
|
||||
|
||||
#endif
|
||||
|
||||
436
src/mca/pml/teg/pml_teg.c
Обычный файл
436
src/mca/pml/teg/pml_teg.c
Обычный файл
@ -0,0 +1,436 @@
|
||||
/*
|
||||
* Copyright (c) 2004-2005 The Trustees of Indiana University.
|
||||
* All rights reserved.
|
||||
* Copyright (c) 2004-2005 The Trustees of the University of Tennessee.
|
||||
* All rights reserved.
|
||||
* Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
|
||||
* University of Stuttgart. All rights reserved.
|
||||
* Copyright (c) 2004-2005 The Regents of the University of California.
|
||||
* All rights reserved.
|
||||
* $COPYRIGHT$
|
||||
*
|
||||
* Additional copyrights may follow
|
||||
*
|
||||
* $HEADER$
|
||||
*/
|
||||
|
||||
#include "ompi_config.h"
|
||||
|
||||
#include <stdlib.h>
|
||||
#include <string.h>
|
||||
|
||||
#include "class/ompi_bitmap.h"
|
||||
#include "mca/pml/pml.h"
|
||||
#include "mca/ptl/ptl.h"
|
||||
#include "mca/ptl/base/base.h"
|
||||
#include "mca/ptl/base/ptl_base_comm.h"
|
||||
#include "mca/ptl/base/ptl_base_header.h"
|
||||
#include "mca/ptl/base/ptl_base_recvfrag.h"
|
||||
#include "mca/ptl/base/ptl_base_sendfrag.h"
|
||||
#include "pml_teg.h"
|
||||
#include "pml_teg_component.h"
|
||||
#include "pml_teg_proc.h"
|
||||
#include "pml_teg_ptl.h"
|
||||
#include "pml_teg_recvreq.h"
|
||||
#include "pml_teg_sendreq.h"
|
||||
#include "pml_teg_recvfrag.h"
|
||||
|
||||
|
||||
mca_pml_teg_t mca_pml_teg = {
|
||||
{
|
||||
mca_pml_teg_add_procs,
|
||||
mca_pml_teg_del_procs,
|
||||
mca_pml_teg_add_ptls,
|
||||
mca_pml_teg_control,
|
||||
mca_pml_teg_progress,
|
||||
mca_pml_teg_add_comm,
|
||||
mca_pml_teg_del_comm,
|
||||
mca_pml_teg_irecv_init,
|
||||
mca_pml_teg_irecv,
|
||||
mca_pml_teg_recv,
|
||||
mca_pml_teg_isend_init,
|
||||
mca_pml_teg_isend,
|
||||
mca_pml_teg_send,
|
||||
mca_pml_teg_iprobe,
|
||||
mca_pml_teg_probe,
|
||||
mca_pml_teg_start
|
||||
}
|
||||
};
|
||||
|
||||
|
||||
int mca_pml_teg_add_comm(ompi_communicator_t* comm)
|
||||
{
|
||||
/* allocate pml specific comm data */
|
||||
mca_pml_ptl_comm_t* pml_comm = OBJ_NEW(mca_pml_ptl_comm_t);
|
||||
if (NULL == pml_comm) {
|
||||
return OMPI_ERR_OUT_OF_RESOURCE;
|
||||
}
|
||||
mca_pml_ptl_comm_init_size(pml_comm, comm->c_remote_group->grp_proc_count);
|
||||
comm->c_pml_comm = pml_comm;
|
||||
return OMPI_SUCCESS;
|
||||
}
|
||||
|
||||
int mca_pml_teg_del_comm(ompi_communicator_t* comm)
|
||||
{
|
||||
OBJ_RELEASE(comm->c_pml_comm);
|
||||
comm->c_pml_comm = 0;
|
||||
return OMPI_SUCCESS;
|
||||
}
|
||||
|
||||
static int ptl_exclusivity_compare(const void* arg1, const void* arg2)
|
||||
{
|
||||
mca_ptl_base_module_t* ptl1 = *(struct mca_ptl_base_module_t**)arg1;
|
||||
mca_ptl_base_module_t* ptl2 = *(struct mca_ptl_base_module_t**)arg2;
|
||||
if( ptl1->ptl_exclusivity > ptl2->ptl_exclusivity ) {
|
||||
return -1;
|
||||
} else if (ptl1->ptl_exclusivity == ptl2->ptl_exclusivity ) {
|
||||
return 0;
|
||||
} else {
|
||||
return 1;
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
int mca_pml_teg_add_ptls(ompi_list_t *ptls)
|
||||
{
|
||||
/* build an array of ptls and ptl modules */
|
||||
mca_ptl_base_selected_module_t* selected_ptl;
|
||||
size_t num_ptls = ompi_list_get_size(ptls);
|
||||
size_t cache_bytes = 0;
|
||||
mca_pml_teg.teg_num_ptl_modules = 0;
|
||||
mca_pml_teg.teg_num_ptl_progress = 0;
|
||||
mca_pml_teg.teg_num_ptl_components = 0;
|
||||
mca_pml_teg.teg_ptl_modules = (mca_ptl_base_module_t **)malloc(sizeof(mca_ptl_base_module_t*) * num_ptls);
|
||||
mca_pml_teg.teg_ptl_progress = (mca_ptl_base_component_progress_fn_t*)malloc(sizeof(mca_ptl_base_component_progress_fn_t) * num_ptls);
|
||||
mca_pml_teg.teg_ptl_components = (mca_ptl_base_component_t **)malloc(sizeof(mca_ptl_base_component_t*) * num_ptls);
|
||||
if (NULL == mca_pml_teg.teg_ptl_modules ||
|
||||
NULL == mca_pml_teg.teg_ptl_progress ||
|
||||
NULL == mca_pml_teg.teg_ptl_components) {
|
||||
return OMPI_ERR_OUT_OF_RESOURCE;
|
||||
}
|
||||
|
||||
for(selected_ptl = (mca_ptl_base_selected_module_t*)ompi_list_get_first(ptls);
|
||||
selected_ptl != (mca_ptl_base_selected_module_t*)ompi_list_get_end(ptls);
|
||||
selected_ptl = (mca_ptl_base_selected_module_t*)ompi_list_get_next(selected_ptl)) {
|
||||
mca_ptl_base_module_t *ptl = selected_ptl->pbsm_module;
|
||||
size_t i;
|
||||
|
||||
mca_pml_teg.teg_ptl_modules[mca_pml_teg.teg_num_ptl_modules++] = ptl;
|
||||
for(i=0; i<mca_pml_teg.teg_num_ptl_components; i++) {
|
||||
if(mca_pml_teg.teg_ptl_components[i] == ptl->ptl_component) {
|
||||
break;
|
||||
}
|
||||
}
|
||||
if(i == mca_pml_teg.teg_num_ptl_components) {
|
||||
mca_pml_teg.teg_ptl_components[mca_pml_teg.teg_num_ptl_components++] = ptl->ptl_component;
|
||||
}
|
||||
|
||||
/*
|
||||
*setup ptl
|
||||
*/
|
||||
|
||||
/* set pointer to fragment matching logic routine, if this
|
||||
* not already set by the ptl */
|
||||
if( NULL == ptl->ptl_match)
|
||||
ptl->ptl_match = mca_pml_teg_recv_frag_match;
|
||||
ptl->ptl_send_progress = mca_pml_teg_send_request_progress;
|
||||
ptl->ptl_recv_progress = mca_pml_teg_recv_request_progress;
|
||||
ptl->ptl_stack = ptl;
|
||||
ptl->ptl_base = NULL;
|
||||
|
||||
/* find maximum required size for cache */
|
||||
if(ptl->ptl_cache_bytes > cache_bytes) {
|
||||
cache_bytes = ptl->ptl_cache_bytes;
|
||||
}
|
||||
}
|
||||
|
||||
/* setup send fragments based on largest required send request */
|
||||
ompi_free_list_init(
|
||||
&mca_pml_teg.teg_send_requests,
|
||||
sizeof(mca_pml_teg_send_request_t) + cache_bytes,
|
||||
OBJ_CLASS(mca_pml_teg_send_request_t),
|
||||
mca_pml_teg.teg_free_list_num,
|
||||
mca_pml_teg.teg_free_list_max,
|
||||
mca_pml_teg.teg_free_list_inc,
|
||||
NULL);
|
||||
|
||||
/* sort ptl list by exclusivity */
|
||||
qsort(mca_pml_teg.teg_ptl_modules, mca_pml_teg.teg_num_ptl_modules, sizeof(struct mca_ptl_t*), ptl_exclusivity_compare);
|
||||
return OMPI_SUCCESS;
|
||||
}
|
||||
|
||||
/*
|
||||
* Pass control information through to all PTL modules.
|
||||
*/
|
||||
|
||||
int mca_pml_teg_control(int param, void* value, size_t size)
|
||||
{
|
||||
size_t i=0;
|
||||
for(i=0; i<mca_pml_teg.teg_num_ptl_components; i++) {
|
||||
if(NULL != mca_pml_teg.teg_ptl_components[i]->ptlm_control) {
|
||||
int rc = mca_pml_teg.teg_ptl_components[i]->ptlm_control(param,value,size);
|
||||
if(rc != OMPI_SUCCESS)
|
||||
return rc;
|
||||
}
|
||||
}
|
||||
return OMPI_SUCCESS;
|
||||
}
|
||||
|
||||
/*
|
||||
* For each proc setup a datastructure that indicates the PTLs
|
||||
* that can be used to reach the destination.
|
||||
*
|
||||
*/
|
||||
|
||||
int mca_pml_teg_add_procs(ompi_proc_t** procs, size_t nprocs)
|
||||
{
|
||||
size_t p;
|
||||
ompi_bitmap_t reachable;
|
||||
struct mca_ptl_base_peer_t** ptl_peers = NULL;
|
||||
int rc;
|
||||
size_t p_index;
|
||||
|
||||
if(nprocs == 0)
|
||||
return OMPI_SUCCESS;
|
||||
|
||||
OBJ_CONSTRUCT(&reachable, ompi_bitmap_t);
|
||||
rc = ompi_bitmap_init(&reachable, nprocs);
|
||||
if(OMPI_SUCCESS != rc)
|
||||
return rc;
|
||||
|
||||
/* iterate through each of the procs and set the peers architecture */
|
||||
for(p=0; p<nprocs; p++) {
|
||||
uint32_t* proc_arch;
|
||||
size_t size = sizeof(uint32_t);
|
||||
rc = mca_base_modex_recv(&mca_pml_teg_component.pmlm_version, procs[p],
|
||||
(void**)&proc_arch, &size);
|
||||
if(rc != OMPI_SUCCESS)
|
||||
return rc;
|
||||
if(size != sizeof(uint32_t))
|
||||
return OMPI_ERROR;
|
||||
procs[p]->proc_arch = ntohl(*proc_arch);
|
||||
free(proc_arch);
|
||||
}
|
||||
|
||||
/* attempt to add all procs to each ptl */
|
||||
ptl_peers = (struct mca_ptl_base_peer_t **)malloc(nprocs * sizeof(struct mca_ptl_base_peer_t*));
|
||||
for(p_index = 0; p_index < mca_pml_teg.teg_num_ptl_modules; p_index++) {
|
||||
mca_ptl_base_module_t* ptl = mca_pml_teg.teg_ptl_modules[p_index];
|
||||
int ptl_inuse = 0;
|
||||
|
||||
/* if the ptl can reach the destination proc it sets the
|
||||
* corresponding bit (proc index) in the reachable bitmap
|
||||
* and can return addressing information for each proc
|
||||
* that is passed back to the ptl on data transfer calls
|
||||
*/
|
||||
ompi_bitmap_clear_all_bits(&reachable);
|
||||
memset(ptl_peers, 0, nprocs * sizeof(struct mca_ptl_base_peer_t*));
|
||||
rc = ptl->ptl_add_procs(ptl, nprocs, procs, ptl_peers, &reachable);
|
||||
if(OMPI_SUCCESS != rc) {
|
||||
free(ptl_peers);
|
||||
return rc;
|
||||
}
|
||||
|
||||
/* for each proc that is reachable - add the ptl to the procs array(s) */
|
||||
for(p=0; p<nprocs; p++) {
|
||||
if(ompi_bitmap_is_set_bit(&reachable, p)) {
|
||||
ompi_proc_t *proc = procs[p];
|
||||
mca_pml_proc_t* proc_pml = proc->proc_pml;
|
||||
mca_ptl_proc_t* proc_ptl;
|
||||
size_t size;
|
||||
|
||||
/* this ptl can be used */
|
||||
ptl_inuse++;
|
||||
|
||||
/* initialize each proc */
|
||||
if(NULL == proc_pml) {
|
||||
|
||||
/* allocate pml specific proc data */
|
||||
proc_pml = OBJ_NEW(mca_pml_teg_proc_t);
|
||||
if (NULL == proc_pml) {
|
||||
ompi_output(0, "mca_pml_teg_add_procs: unable to allocate resources");
|
||||
free(ptl_peers);
|
||||
return OMPI_ERR_OUT_OF_RESOURCE;
|
||||
}
|
||||
|
||||
/* preallocate space in array for max number of ptls */
|
||||
mca_ptl_array_reserve(&proc_pml->proc_ptl_first, mca_pml_teg.teg_num_ptl_modules);
|
||||
mca_ptl_array_reserve(&proc_pml->proc_ptl_next, mca_pml_teg.teg_num_ptl_modules);
|
||||
proc_pml->proc_ompi = proc;
|
||||
proc->proc_pml = proc_pml;
|
||||
}
|
||||
|
||||
/* dont allow an additional PTL with a lower exclusivity ranking */
|
||||
size = mca_ptl_array_get_size(&proc_pml->proc_ptl_next);
|
||||
if(size > 0) {
|
||||
proc_ptl = mca_ptl_array_get_index(&proc_pml->proc_ptl_next, size-1);
|
||||
/* skip this ptl if the exclusivity is less than the previous */
|
||||
if(proc_ptl->ptl->ptl_exclusivity > ptl->ptl_exclusivity) {
|
||||
if(ptl_peers[p] != NULL) {
|
||||
ptl->ptl_del_procs(ptl, 1, &proc, &ptl_peers[p]);
|
||||
}
|
||||
continue;
|
||||
}
|
||||
}
|
||||
|
||||
/* cache the ptl on the proc */
|
||||
proc_ptl = mca_ptl_array_insert(&proc_pml->proc_ptl_next);
|
||||
proc_ptl->ptl = ptl;
|
||||
proc_ptl->ptl_peer = ptl_peers[p];
|
||||
proc_ptl->ptl_weight = 0;
|
||||
proc_pml->proc_ptl_flags |= ptl->ptl_flags;
|
||||
}
|
||||
}
|
||||
if(ptl_inuse > 0 && NULL != ptl->ptl_component->ptlm_progress) {
|
||||
size_t p;
|
||||
bool found = false;
|
||||
for(p=0; p<mca_pml_teg.teg_num_ptl_progress; p++) {
|
||||
if(mca_pml_teg.teg_ptl_progress[p] == ptl->ptl_component->ptlm_progress) {
|
||||
found = true;
|
||||
break;
|
||||
}
|
||||
}
|
||||
if(found == false) {
|
||||
mca_pml_teg.teg_ptl_progress[mca_pml_teg.teg_num_ptl_progress] =
|
||||
ptl->ptl_component->ptlm_progress;
|
||||
mca_pml_teg.teg_num_ptl_progress++;
|
||||
}
|
||||
}
|
||||
}
|
||||
free(ptl_peers);
|
||||
|
||||
/* iterate back through procs and compute metrics for registered ptls */
|
||||
for(p=0; p<nprocs; p++) {
|
||||
ompi_proc_t *proc = procs[p];
|
||||
mca_pml_proc_t* proc_pml = proc->proc_pml;
|
||||
double total_bandwidth = 0;
|
||||
uint32_t latency = 0;
|
||||
size_t n_index;
|
||||
size_t n_size;
|
||||
|
||||
/* skip over procs w/ no ptls registered */
|
||||
if(NULL == proc_pml)
|
||||
continue;
|
||||
|
||||
/* (1) determine the total bandwidth available across all ptls
|
||||
* note that we need to do this here, as we may already have ptls configured
|
||||
* (2) determine the highest priority ranking for latency
|
||||
*/
|
||||
n_size = mca_ptl_array_get_size(&proc_pml->proc_ptl_next);
|
||||
for(n_index = 0; n_index < n_size; n_index++) {
|
||||
struct mca_ptl_proc_t* proc_ptl = mca_ptl_array_get_index(&proc_pml->proc_ptl_next, n_index);
|
||||
struct mca_ptl_base_module_t* ptl = proc_ptl->ptl;
|
||||
total_bandwidth += proc_ptl->ptl->ptl_bandwidth;
|
||||
if(ptl->ptl_latency > latency)
|
||||
latency = ptl->ptl_latency;
|
||||
}
|
||||
|
||||
/* (1) set the weight of each ptl as a percentage of overall bandwidth
|
||||
* (2) copy all ptl instances at the highest priority ranking into the
|
||||
* list of ptls used for first fragments
|
||||
*/
|
||||
|
||||
for(n_index = 0; n_index < n_size; n_index++) {
|
||||
struct mca_ptl_proc_t* proc_ptl = mca_ptl_array_get_index(&proc_pml->proc_ptl_next, n_index);
|
||||
struct mca_ptl_base_module_t *ptl = proc_ptl->ptl;
|
||||
double weight;
|
||||
|
||||
/* compute weighting factor for this ptl */
|
||||
if(ptl->ptl_bandwidth)
|
||||
weight = proc_ptl->ptl->ptl_bandwidth / total_bandwidth;
|
||||
else
|
||||
weight = 1.0 / n_size;
|
||||
proc_ptl->ptl_weight = (int)(weight * 100);
|
||||
|
||||
/*
|
||||
* save/create ptl extension for use by pml
|
||||
*/
|
||||
proc_ptl->ptl_base = ptl->ptl_base;
|
||||
if (NULL == proc_ptl->ptl_base &&
|
||||
ptl->ptl_cache_bytes > 0 &&
|
||||
NULL != ptl->ptl_request_init &&
|
||||
NULL != ptl->ptl_request_fini) {
|
||||
|
||||
mca_pml_base_ptl_t* ptl_base = OBJ_NEW(mca_pml_base_ptl_t);
|
||||
ptl_base->ptl = ptl;
|
||||
ptl_base->ptl_cache_size = ptl->ptl_cache_size;
|
||||
proc_ptl->ptl_base = ptl->ptl_base = ptl_base;
|
||||
}
|
||||
|
||||
/* check to see if this ptl is already in the array of ptls used for first
|
||||
* fragments - if not add it.
|
||||
*/
|
||||
if(ptl->ptl_latency == latency) {
|
||||
struct mca_ptl_proc_t* proc_new = mca_ptl_array_insert(&proc_pml->proc_ptl_first);
|
||||
*proc_new = *proc_ptl;
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
return OMPI_SUCCESS;
|
||||
}
|
||||
|
||||
/*
|
||||
* iterate through each proc and notify any PTLs associated
|
||||
* with the proc that it is/has gone away
|
||||
*/
|
||||
|
||||
int mca_pml_teg_del_procs(ompi_proc_t** procs, size_t nprocs)
|
||||
{
|
||||
size_t p;
|
||||
int rc;
|
||||
for(p = 0; p < nprocs; p++) {
|
||||
ompi_proc_t *proc = procs[p];
|
||||
mca_pml_proc_t* proc_pml = proc->proc_pml;
|
||||
size_t f_index, f_size;
|
||||
size_t n_index, n_size;
|
||||
|
||||
/* notify each ptl that the proc is going away */
|
||||
f_size = mca_ptl_array_get_size(&proc_pml->proc_ptl_first);
|
||||
for(f_index = 0; f_index < f_size; f_index++) {
|
||||
mca_ptl_proc_t* ptl_proc = mca_ptl_array_get_index(&proc_pml->proc_ptl_first, f_index);
|
||||
mca_ptl_base_module_t* ptl = ptl_proc->ptl;
|
||||
|
||||
rc = ptl->ptl_del_procs(ptl,1,&proc,&ptl_proc->ptl_peer);
|
||||
if(OMPI_SUCCESS != rc) {
|
||||
return rc;
|
||||
}
|
||||
|
||||
/* remove this from next array so that we dont call it twice w/
|
||||
* the same address pointer
|
||||
*/
|
||||
n_size = mca_ptl_array_get_size(&proc_pml->proc_ptl_first);
|
||||
for(n_index = 0; n_index < n_size; n_index++) {
|
||||
mca_ptl_proc_t* next_proc = mca_ptl_array_get_index(&proc_pml->proc_ptl_next, n_index);
|
||||
if(next_proc->ptl == ptl) {
|
||||
memset(next_proc, 0, sizeof(mca_ptl_proc_t));
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/* notify each ptl that was not in the array of ptls for first fragments */
|
||||
n_size = mca_ptl_array_get_size(&proc_pml->proc_ptl_next);
|
||||
for(n_index = 0; n_index < n_size; n_index++) {
|
||||
mca_ptl_proc_t* ptl_proc = mca_ptl_array_get_index(&proc_pml->proc_ptl_first, n_index);
|
||||
mca_ptl_base_module_t* ptl = ptl_proc->ptl;
|
||||
if (ptl != 0) {
|
||||
rc = ptl->ptl_del_procs(ptl,1,&proc,&ptl_proc->ptl_peer);
|
||||
if(OMPI_SUCCESS != rc)
|
||||
return rc;
|
||||
}
|
||||
}
|
||||
|
||||
/* do any required cleanup */
|
||||
OBJ_RELEASE(proc_pml);
|
||||
proc->proc_pml = NULL;
|
||||
}
|
||||
return OMPI_SUCCESS;
|
||||
}
|
||||
|
||||
int mca_pml_teg_component_fini(void)
|
||||
{
|
||||
/* FIX */
|
||||
return OMPI_SUCCESS;
|
||||
}
|
||||
|
||||
268
src/mca/pml/teg/pml_teg.h
Обычный файл
268
src/mca/pml/teg/pml_teg.h
Обычный файл
@ -0,0 +1,268 @@
|
||||
/*
|
||||
* Copyright (c) 2004-2005 The Trustees of Indiana University.
|
||||
* All rights reserved.
|
||||
* Copyright (c) 2004-2005 The Trustees of the University of Tennessee.
|
||||
* All rights reserved.
|
||||
* Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
|
||||
* University of Stuttgart. All rights reserved.
|
||||
* Copyright (c) 2004-2005 The Regents of the University of California.
|
||||
* All rights reserved.
|
||||
* $COPYRIGHT$
|
||||
*
|
||||
* Additional copyrights may follow
|
||||
*
|
||||
* $HEADER$
|
||||
*/
|
||||
/**
|
||||
* @file
|
||||
*/
|
||||
|
||||
#ifndef MCA_PML_TEG_H
|
||||
#define MCA_PML_TEG_H
|
||||
|
||||
#include "threads/thread.h"
|
||||
#include "threads/condition.h"
|
||||
#include "class/ompi_free_list.h"
|
||||
#include "util/cmd_line.h"
|
||||
#include "request/request.h"
|
||||
#include "mca/pml/pml.h"
|
||||
#include "mca/pml/base/pml_base_request.h"
|
||||
#include "mca/pml/base/pml_base_bsend.h"
|
||||
#include "mca/ptl/base/ptl_base_sendreq.h"
|
||||
#include "mca/ptl/ptl.h"
|
||||
|
||||
|
||||
#if defined(c_plusplus) || defined(__cplusplus)
|
||||
extern "C" {
|
||||
#endif
|
||||
/**
|
||||
* TEG PML module
|
||||
*/
|
||||
|
||||
struct mca_pml_teg_t {
|
||||
mca_pml_base_module_t super;
|
||||
|
||||
mca_ptl_base_component_t **teg_ptl_components;
|
||||
size_t teg_num_ptl_components;
|
||||
|
||||
mca_ptl_base_module_t** teg_ptl_modules;
|
||||
size_t teg_num_ptl_modules;
|
||||
|
||||
mca_ptl_base_component_progress_fn_t* teg_ptl_progress;
|
||||
size_t teg_num_ptl_progress;
|
||||
|
||||
ompi_list_t teg_procs;
|
||||
ompi_mutex_t teg_lock;
|
||||
|
||||
int teg_priority;
|
||||
|
||||
int teg_free_list_num; /* initial size of free list */
|
||||
int teg_free_list_max; /* maximum size of free list */
|
||||
int teg_free_list_inc; /* number of elements to grow free list */
|
||||
int teg_poll_iterations; /* number of iterations to poll for completion */
|
||||
|
||||
/* free list of requests */
|
||||
ompi_free_list_t teg_send_requests;
|
||||
ompi_free_list_t teg_recv_requests;
|
||||
|
||||
/* list of pending send requests */
|
||||
ompi_list_t teg_send_pending;
|
||||
};
|
||||
typedef struct mca_pml_teg_t mca_pml_teg_t;
|
||||
|
||||
extern mca_pml_teg_t mca_pml_teg;
|
||||
|
||||
|
||||
/*
|
||||
* PML module functions.
|
||||
*/
|
||||
|
||||
|
||||
extern int mca_pml_teg_component_open(void);
|
||||
extern int mca_pml_teg_component_close(void);
|
||||
|
||||
extern mca_pml_base_module_t* mca_pml_teg_component_init(
|
||||
int *priority,
|
||||
bool enable_progress_threads,
|
||||
bool enable_mpi_threads
|
||||
);
|
||||
|
||||
extern int mca_pml_teg_component_fini(void);
|
||||
|
||||
|
||||
|
||||
/*
|
||||
* PML interface functions.
|
||||
*/
|
||||
|
||||
extern int mca_pml_teg_add_comm(
|
||||
struct ompi_communicator_t* comm
|
||||
);
|
||||
|
||||
extern int mca_pml_teg_del_comm(
|
||||
struct ompi_communicator_t* comm
|
||||
);
|
||||
|
||||
extern int mca_pml_teg_add_procs(
|
||||
struct ompi_proc_t **procs,
|
||||
size_t nprocs
|
||||
);
|
||||
|
||||
extern int mca_pml_teg_del_procs(
|
||||
struct ompi_proc_t **procs,
|
||||
size_t nprocs
|
||||
);
|
||||
|
||||
extern int mca_pml_teg_add_ptls(
|
||||
ompi_list_t *ptls
|
||||
);
|
||||
|
||||
extern int mca_pml_teg_control(
|
||||
int param,
|
||||
void *size,
|
||||
size_t value
|
||||
);
|
||||
|
||||
extern int mca_pml_teg_progress(void);
|
||||
|
||||
extern int mca_pml_teg_iprobe(
|
||||
int dst,
|
||||
int tag,
|
||||
struct ompi_communicator_t* comm,
|
||||
int *matched,
|
||||
ompi_status_public_t* status
|
||||
);
|
||||
|
||||
extern int mca_pml_teg_probe(
|
||||
int dst,
|
||||
int tag,
|
||||
struct ompi_communicator_t* comm,
|
||||
ompi_status_public_t* status
|
||||
);
|
||||
|
||||
extern int mca_pml_teg_cancelled(
|
||||
ompi_request_t* request,
|
||||
int *flag
|
||||
);
|
||||
|
||||
|
||||
extern int mca_pml_teg_isend_init(
|
||||
void *buf,
|
||||
size_t count,
|
||||
ompi_datatype_t *datatype,
|
||||
int dst,
|
||||
int tag,
|
||||
mca_pml_base_send_mode_t mode,
|
||||
struct ompi_communicator_t* comm,
|
||||
struct ompi_request_t **request
|
||||
);
|
||||
|
||||
extern int mca_pml_teg_isend(
|
||||
void *buf,
|
||||
size_t count,
|
||||
ompi_datatype_t *datatype,
|
||||
int dst,
|
||||
int tag,
|
||||
mca_pml_base_send_mode_t mode,
|
||||
struct ompi_communicator_t* comm,
|
||||
struct ompi_request_t **request
|
||||
);
|
||||
|
||||
extern int mca_pml_teg_send(
|
||||
void *buf,
|
||||
size_t count,
|
||||
ompi_datatype_t *datatype,
|
||||
int dst,
|
||||
int tag,
|
||||
mca_pml_base_send_mode_t mode,
|
||||
struct ompi_communicator_t* comm
|
||||
);
|
||||
|
||||
extern int mca_pml_teg_irecv_init(
|
||||
void *buf,
|
||||
size_t count,
|
||||
ompi_datatype_t *datatype,
|
||||
int src,
|
||||
int tag,
|
||||
struct ompi_communicator_t* comm,
|
||||
struct ompi_request_t **request
|
||||
);
|
||||
|
||||
extern int mca_pml_teg_irecv(
|
||||
void *buf,
|
||||
size_t count,
|
||||
ompi_datatype_t *datatype,
|
||||
int src,
|
||||
int tag,
|
||||
struct ompi_communicator_t* comm,
|
||||
struct ompi_request_t **request
|
||||
);
|
||||
|
||||
extern int mca_pml_teg_recv(
|
||||
void *buf,
|
||||
size_t count,
|
||||
ompi_datatype_t *datatype,
|
||||
int src,
|
||||
int tag,
|
||||
struct ompi_communicator_t* comm,
|
||||
ompi_status_public_t* status
|
||||
);
|
||||
|
||||
extern int mca_pml_teg_progress(void);
|
||||
|
||||
extern int mca_pml_teg_start(
|
||||
size_t count,
|
||||
ompi_request_t** requests
|
||||
);
|
||||
|
||||
#if defined(c_plusplus) || defined(__cplusplus)
|
||||
}
|
||||
#endif
|
||||
#define MCA_PML_TEG_FINI(request) \
|
||||
{ \
|
||||
mca_pml_base_request_t* pml_request = *(mca_pml_base_request_t**)(request); \
|
||||
if(pml_request->req_persistent) { \
|
||||
if(pml_request->req_free_called) { \
|
||||
MCA_PML_TEG_FREE(request); \
|
||||
} else { \
|
||||
pml_request->req_ompi.req_state = OMPI_REQUEST_INACTIVE; \
|
||||
} \
|
||||
} else { \
|
||||
MCA_PML_TEG_FREE(request); \
|
||||
} \
|
||||
}
|
||||
|
||||
|
||||
#define MCA_PML_TEG_FREE(request) \
|
||||
{ \
|
||||
mca_pml_base_request_t* pml_request = *(mca_pml_base_request_t**)(request); \
|
||||
pml_request->req_free_called = true; \
|
||||
if( pml_request->req_pml_complete == true) \
|
||||
{ \
|
||||
OMPI_REQUEST_FINI(*(request)); \
|
||||
switch(pml_request->req_type) { \
|
||||
case MCA_PML_REQUEST_SEND: \
|
||||
{ \
|
||||
mca_pml_teg_send_request_t* sendreq = (mca_pml_teg_send_request_t*)pml_request; \
|
||||
while(sendreq->req_lock > 0); \
|
||||
if(sendreq->req_send.req_send_mode == MCA_PML_BASE_SEND_BUFFERED) { \
|
||||
mca_pml_base_bsend_request_fini((ompi_request_t*)sendreq); \
|
||||
} \
|
||||
MCA_PML_TEG_SEND_REQUEST_RETURN(sendreq); \
|
||||
break; \
|
||||
} \
|
||||
case MCA_PML_REQUEST_RECV: \
|
||||
{ \
|
||||
mca_pml_teg_recv_request_t* recvreq = (mca_pml_teg_recv_request_t*)pml_request; \
|
||||
MCA_PML_TEG_RECV_REQUEST_RETURN(recvreq); \
|
||||
break; \
|
||||
} \
|
||||
default: \
|
||||
break; \
|
||||
} \
|
||||
} \
|
||||
*(request) = MPI_REQUEST_NULL; \
|
||||
}
|
||||
|
||||
#endif
|
||||
|
||||
28
src/mca/pml/teg/pml_teg_cancel.c
Обычный файл
28
src/mca/pml/teg/pml_teg_cancel.c
Обычный файл
@ -0,0 +1,28 @@
|
||||
/* -*- Mode: C; c-basic-offset:4 ; -*- */
|
||||
/*
|
||||
* Copyright (c) 2004-2005 The Trustees of Indiana University.
|
||||
* All rights reserved.
|
||||
* Copyright (c) 2004-2005 The Trustees of the University of Tennessee.
|
||||
* All rights reserved.
|
||||
* Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
|
||||
* University of Stuttgart. All rights reserved.
|
||||
* Copyright (c) 2004-2005 The Regents of the University of California.
|
||||
* All rights reserved.
|
||||
* $COPYRIGHT$
|
||||
*
|
||||
* Additional copyrights may follow
|
||||
*
|
||||
* $HEADER$
|
||||
*/
|
||||
|
||||
#include "ompi_config.h"
|
||||
|
||||
#include "pml_teg.h"
|
||||
|
||||
int mca_pml_teg_cancelled(ompi_request_t* request, int* flag)
|
||||
{
|
||||
if(NULL != flag)
|
||||
*flag = (true == request->req_status._cancelled ? 1 : 0);
|
||||
return OMPI_SUCCESS;
|
||||
}
|
||||
|
||||
177
src/mca/pml/teg/pml_teg_component.c
Обычный файл
177
src/mca/pml/teg/pml_teg_component.c
Обычный файл
@ -0,0 +1,177 @@
|
||||
/*
|
||||
* Copyright (c) 2004-2005 The Trustees of Indiana University.
|
||||
* All rights reserved.
|
||||
* Copyright (c) 2004-2005 The Trustees of the University of Tennessee.
|
||||
* All rights reserved.
|
||||
* Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
|
||||
* University of Stuttgart. All rights reserved.
|
||||
* Copyright (c) 2004-2005 The Regents of the University of California.
|
||||
* All rights reserved.
|
||||
* $COPYRIGHT$
|
||||
*
|
||||
* Additional copyrights may follow
|
||||
*
|
||||
* $HEADER$
|
||||
*/
|
||||
|
||||
#include "ompi_config.h"
|
||||
|
||||
#include "event/event.h"
|
||||
#include "mpi.h"
|
||||
#include "mca/pml/pml.h"
|
||||
#include "mca/ptl/ptl.h"
|
||||
#include "mca/base/mca_base_param.h"
|
||||
#include "mca/pml/base/pml_base_bsend.h"
|
||||
#include "pml_teg.h"
|
||||
#include "pml_teg_proc.h"
|
||||
#include "pml_teg_sendreq.h"
|
||||
#include "pml_teg_recvreq.h"
|
||||
|
||||
|
||||
mca_pml_base_component_1_0_0_t mca_pml_teg_component = {
|
||||
|
||||
/* First, the mca_base_component_t struct containing meta
|
||||
information about the component itself */
|
||||
|
||||
{
|
||||
/* Indicate that we are a pml v1.0.0 component (which also implies
|
||||
a specific MCA version) */
|
||||
|
||||
MCA_PML_BASE_VERSION_1_0_0,
|
||||
|
||||
"teg", /* MCA component name */
|
||||
1, /* MCA component major version */
|
||||
0, /* MCA component minor version */
|
||||
0, /* MCA component release version */
|
||||
mca_pml_teg_component_open, /* component open */
|
||||
mca_pml_teg_component_close /* component close */
|
||||
},
|
||||
|
||||
/* Next the MCA v1.0.0 component meta data */
|
||||
|
||||
{
|
||||
/* Whether the component is checkpointable or not */
|
||||
false
|
||||
},
|
||||
|
||||
mca_pml_teg_component_init, /* component init */
|
||||
mca_pml_teg_component_fini /* component finalize */
|
||||
};
|
||||
|
||||
|
||||
|
||||
static inline int mca_pml_teg_param_register_int(
|
||||
const char* param_name,
|
||||
int default_value)
|
||||
{
|
||||
int id = mca_base_param_register_int("pml","teg",param_name,NULL,default_value);
|
||||
int param_value = default_value;
|
||||
mca_base_param_lookup_int(id,¶m_value);
|
||||
return param_value;
|
||||
}
|
||||
|
||||
|
||||
int mca_pml_teg_component_open(void)
|
||||
{
|
||||
#ifdef WIN32
|
||||
WSADATA win_sock_data;
|
||||
if (WSAStartup(MAKEWORD(2,2), &win_sock_data) != 0) {
|
||||
ompi_output (0, "mca_oob_tcp_component_init: failed to initialise windows sockets: %d\n", WSAGetLastError());
|
||||
return OMPI_ERROR;
|
||||
}
|
||||
#endif
|
||||
OBJ_CONSTRUCT(&mca_pml_teg.teg_lock, ompi_mutex_t);
|
||||
OBJ_CONSTRUCT(&mca_pml_teg.teg_send_requests, ompi_free_list_t);
|
||||
OBJ_CONSTRUCT(&mca_pml_teg.teg_recv_requests, ompi_free_list_t);
|
||||
OBJ_CONSTRUCT(&mca_pml_teg.teg_procs, ompi_list_t);
|
||||
OBJ_CONSTRUCT(&mca_pml_teg.teg_send_pending, ompi_list_t);
|
||||
|
||||
mca_pml_teg.teg_ptl_components = NULL;
|
||||
mca_pml_teg.teg_num_ptl_components = 0;
|
||||
mca_pml_teg.teg_ptl_modules = NULL;
|
||||
mca_pml_teg.teg_num_ptl_modules = 0;
|
||||
mca_pml_teg.teg_ptl_progress = NULL;
|
||||
mca_pml_teg.teg_num_ptl_progress = 0;
|
||||
|
||||
mca_pml_teg.teg_free_list_num =
|
||||
mca_pml_teg_param_register_int("free_list_num", 256);
|
||||
mca_pml_teg.teg_free_list_max =
|
||||
mca_pml_teg_param_register_int("free_list_max", -1);
|
||||
mca_pml_teg.teg_free_list_inc =
|
||||
mca_pml_teg_param_register_int("free_list_inc", 256);
|
||||
mca_pml_teg.teg_poll_iterations =
|
||||
mca_pml_teg_param_register_int("poll_iterations", 100000);
|
||||
mca_pml_teg.teg_priority =
|
||||
mca_pml_teg_param_register_int("priority", 0);
|
||||
|
||||
return OMPI_SUCCESS;
|
||||
}
|
||||
|
||||
|
||||
int mca_pml_teg_component_close(void)
|
||||
{
|
||||
#ifdef WIN32
|
||||
WSACleanup();
|
||||
#endif
|
||||
|
||||
#if OMPI_ENABLE_DEBUG
|
||||
if (mca_pml_teg.teg_recv_requests.fl_num_allocated !=
|
||||
mca_pml_teg.teg_recv_requests.super.ompi_list_length) {
|
||||
ompi_output(0, "teg recv requests: %d allocated %d returned\n",
|
||||
mca_pml_teg.teg_recv_requests.fl_num_allocated,
|
||||
mca_pml_teg.teg_recv_requests.super.ompi_list_length);
|
||||
}
|
||||
#endif
|
||||
|
||||
if(NULL != mca_pml_teg.teg_ptl_components) {
|
||||
free(mca_pml_teg.teg_ptl_components);
|
||||
}
|
||||
if(NULL != mca_pml_teg.teg_ptl_modules) {
|
||||
free(mca_pml_teg.teg_ptl_modules);
|
||||
}
|
||||
if(NULL != mca_pml_teg.teg_ptl_progress) {
|
||||
free(mca_pml_teg.teg_ptl_progress);
|
||||
}
|
||||
OBJ_DESTRUCT(&mca_pml_teg.teg_send_pending);
|
||||
OBJ_DESTRUCT(&mca_pml_teg.teg_send_requests);
|
||||
OBJ_DESTRUCT(&mca_pml_teg.teg_recv_requests);
|
||||
OBJ_DESTRUCT(&mca_pml_teg.teg_procs);
|
||||
OBJ_DESTRUCT(&mca_pml_teg.teg_lock);
|
||||
return OMPI_SUCCESS;
|
||||
}
|
||||
|
||||
|
||||
mca_pml_base_module_t* mca_pml_teg_component_init(int* priority,
|
||||
bool enable_progress_threads,
|
||||
bool enable_mpi_threads)
|
||||
{
|
||||
uint32_t proc_arch;
|
||||
int rc;
|
||||
*priority = mca_pml_teg.teg_priority;
|
||||
|
||||
/* recv requests */
|
||||
ompi_free_list_init(
|
||||
&mca_pml_teg.teg_recv_requests,
|
||||
sizeof(mca_pml_teg_recv_request_t),
|
||||
OBJ_CLASS(mca_pml_teg_recv_request_t),
|
||||
mca_pml_teg.teg_free_list_num,
|
||||
mca_pml_teg.teg_free_list_max,
|
||||
mca_pml_teg.teg_free_list_inc,
|
||||
NULL);
|
||||
|
||||
/* buffered send */
|
||||
if(OMPI_SUCCESS != mca_pml_base_bsend_init(enable_mpi_threads)) {
|
||||
ompi_output(0, "mca_pml_teg_component_init: mca_pml_bsend_init failed\n");
|
||||
return NULL;
|
||||
}
|
||||
|
||||
/* post this processes datatype */
|
||||
proc_arch = ompi_proc_local()->proc_arch;
|
||||
proc_arch = htonl(proc_arch);
|
||||
rc = mca_base_modex_send(&mca_pml_teg_component.pmlm_version, &proc_arch, sizeof(proc_arch));
|
||||
if(rc != OMPI_SUCCESS)
|
||||
return NULL;
|
||||
|
||||
return &mca_pml_teg.super;
|
||||
}
|
||||
|
||||
27
src/mca/pml/teg/pml_teg_component.h
Обычный файл
27
src/mca/pml/teg/pml_teg_component.h
Обычный файл
@ -0,0 +1,27 @@
|
||||
/*
|
||||
* Copyright (c) 2004-2005 The Trustees of Indiana University.
|
||||
* All rights reserved.
|
||||
* Copyright (c) 2004-2005 The Trustees of the University of Tennessee.
|
||||
* All rights reserved.
|
||||
* Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
|
||||
* University of Stuttgart. All rights reserved.
|
||||
* $COPYRIGHT$
|
||||
*
|
||||
* Additional copyrights may follow
|
||||
*
|
||||
* $HEADER$
|
||||
*/
|
||||
/**
|
||||
* @file
|
||||
*/
|
||||
|
||||
#ifndef MCA_PML_TEG_COMPONENT_H
|
||||
#define MCA_PML_TEG_COMPONENT_H
|
||||
|
||||
/*
|
||||
* PML module functions.
|
||||
*/
|
||||
|
||||
OMPI_COMP_EXPORT extern mca_pml_base_component_1_0_0_t mca_pml_teg_component;
|
||||
|
||||
#endif
|
||||
93
src/mca/pml/teg/pml_teg_iprobe.c
Обычный файл
93
src/mca/pml/teg/pml_teg_iprobe.c
Обычный файл
@ -0,0 +1,93 @@
|
||||
/*
|
||||
* Copyright (c) 2004-2005 The Trustees of Indiana University.
|
||||
* All rights reserved.
|
||||
* Copyright (c) 2004-2005 The Trustees of the University of Tennessee.
|
||||
* All rights reserved.
|
||||
* Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
|
||||
* University of Stuttgart. All rights reserved.
|
||||
* Copyright (c) 2004-2005 The Regents of the University of California.
|
||||
* All rights reserved.
|
||||
* $COPYRIGHT$
|
||||
*
|
||||
* Additional copyrights may follow
|
||||
*
|
||||
* $HEADER$
|
||||
*/
|
||||
|
||||
#include "ompi_config.h"
|
||||
#include "request/request.h"
|
||||
#include "pml_teg_recvreq.h"
|
||||
|
||||
|
||||
int mca_pml_teg_iprobe(int src,
|
||||
int tag,
|
||||
struct ompi_communicator_t *comm,
|
||||
int *matched, ompi_status_public_t * status)
|
||||
{
|
||||
int rc;
|
||||
mca_ptl_base_recv_request_t recvreq;
|
||||
|
||||
OBJ_CONSTRUCT( &(recvreq), mca_ptl_base_recv_request_t );
|
||||
recvreq.req_recv.req_base.req_ompi.req_type = OMPI_REQUEST_PML;
|
||||
recvreq.req_recv.req_base.req_type = MCA_PML_REQUEST_IPROBE;
|
||||
MCA_PML_TEG_RECV_REQUEST_INIT(&recvreq, NULL, 0, &ompi_mpi_char, src, tag, comm, true);
|
||||
|
||||
*matched = 0;
|
||||
if ((rc = mca_pml_teg_recv_request_start(&recvreq)) == OMPI_SUCCESS) {
|
||||
if( recvreq.req_recv.req_base.req_ompi.req_complete == true ) {
|
||||
if( NULL != status ) {
|
||||
*status = recvreq.req_recv.req_base.req_ompi.req_status;
|
||||
}
|
||||
*matched = 1;
|
||||
} else {
|
||||
/* we are supposed to progress ... */
|
||||
ompi_progress();
|
||||
}
|
||||
}
|
||||
MCA_PML_BASE_RECV_REQUEST_RETURN((&recvreq.req_recv));
|
||||
return rc;
|
||||
}
|
||||
|
||||
|
||||
int mca_pml_teg_probe(int src,
|
||||
int tag,
|
||||
struct ompi_communicator_t *comm,
|
||||
ompi_status_public_t * status)
|
||||
{
|
||||
int rc;
|
||||
mca_ptl_base_recv_request_t recvreq;
|
||||
|
||||
OBJ_CONSTRUCT( &(recvreq), mca_ptl_base_recv_request_t );
|
||||
recvreq.req_recv.req_base.req_ompi.req_type = OMPI_REQUEST_PML;
|
||||
recvreq.req_recv.req_base.req_type = MCA_PML_REQUEST_PROBE;
|
||||
MCA_PML_TEG_RECV_REQUEST_INIT(&recvreq, NULL, 0, &ompi_mpi_char, src, tag, comm, true);
|
||||
|
||||
if ((rc = mca_pml_teg_recv_request_start(&recvreq)) != OMPI_SUCCESS) {
|
||||
MCA_PML_BASE_RECV_REQUEST_RETURN((&recvreq.req_recv));
|
||||
return rc;
|
||||
}
|
||||
|
||||
if (recvreq.req_recv.req_base.req_ompi.req_complete == false) {
|
||||
/* give up and sleep until completion */
|
||||
if (ompi_using_threads()) {
|
||||
ompi_mutex_lock(&ompi_request_lock);
|
||||
ompi_request_waiting++;
|
||||
while (recvreq.req_recv.req_base.req_ompi.req_complete == false)
|
||||
ompi_condition_wait(&ompi_request_cond, &ompi_request_lock);
|
||||
ompi_request_waiting--;
|
||||
ompi_mutex_unlock(&ompi_request_lock);
|
||||
} else {
|
||||
ompi_request_waiting++;
|
||||
while (recvreq.req_recv.req_base.req_ompi.req_complete == false)
|
||||
ompi_condition_wait(&ompi_request_cond, &ompi_request_lock);
|
||||
ompi_request_waiting--;
|
||||
}
|
||||
}
|
||||
|
||||
if (NULL != status) {
|
||||
*status = recvreq.req_recv.req_base.req_ompi.req_status;
|
||||
}
|
||||
MCA_PML_BASE_RECV_REQUEST_RETURN(&recvreq.req_recv);
|
||||
return OMPI_SUCCESS;
|
||||
}
|
||||
|
||||
117
src/mca/pml/teg/pml_teg_irecv.c
Обычный файл
117
src/mca/pml/teg/pml_teg_irecv.c
Обычный файл
@ -0,0 +1,117 @@
|
||||
/*
|
||||
* Copyright (c) 2004-2005 The Trustees of Indiana University.
|
||||
* All rights reserved.
|
||||
* Copyright (c) 2004-2005 The Trustees of the University of Tennessee.
|
||||
* All rights reserved.
|
||||
* Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
|
||||
* University of Stuttgart. All rights reserved.
|
||||
* Copyright (c) 2004-2005 The Regents of the University of California.
|
||||
* All rights reserved.
|
||||
* $COPYRIGHT$
|
||||
*
|
||||
* Additional copyrights may follow
|
||||
*
|
||||
* $HEADER$
|
||||
*/
|
||||
|
||||
#include "ompi_config.h"
|
||||
#include "request/request.h"
|
||||
#include "pml_teg_recvreq.h"
|
||||
|
||||
|
||||
int mca_pml_teg_irecv_init(void *addr,
|
||||
size_t count,
|
||||
ompi_datatype_t * datatype,
|
||||
int src,
|
||||
int tag,
|
||||
struct ompi_communicator_t *comm,
|
||||
struct ompi_request_t **request)
|
||||
{
|
||||
int rc;
|
||||
mca_ptl_base_recv_request_t *recvreq;
|
||||
MCA_PML_TEG_RECV_REQUEST_ALLOC(recvreq, rc);
|
||||
if (NULL == recvreq)
|
||||
return rc;
|
||||
|
||||
MCA_PML_TEG_RECV_REQUEST_INIT(recvreq,
|
||||
addr,
|
||||
count, datatype, src, tag, comm, true);
|
||||
|
||||
*request = (ompi_request_t *) recvreq;
|
||||
return OMPI_SUCCESS;
|
||||
}
|
||||
|
||||
int mca_pml_teg_irecv(void *addr,
|
||||
size_t count,
|
||||
ompi_datatype_t * datatype,
|
||||
int src,
|
||||
int tag,
|
||||
struct ompi_communicator_t *comm,
|
||||
struct ompi_request_t **request)
|
||||
{
|
||||
int rc;
|
||||
|
||||
mca_ptl_base_recv_request_t *recvreq;
|
||||
MCA_PML_TEG_RECV_REQUEST_ALLOC(recvreq, rc);
|
||||
if (NULL == recvreq)
|
||||
return rc;
|
||||
|
||||
MCA_PML_TEG_RECV_REQUEST_INIT(recvreq,
|
||||
addr,
|
||||
count, datatype, src, tag, comm, false);
|
||||
|
||||
if ((rc = mca_pml_teg_recv_request_start(recvreq)) != OMPI_SUCCESS) {
|
||||
MCA_PML_TEG_RECV_REQUEST_RETURN(recvreq);
|
||||
return rc;
|
||||
}
|
||||
*request = (ompi_request_t *) recvreq;
|
||||
return OMPI_SUCCESS;
|
||||
}
|
||||
|
||||
|
||||
int mca_pml_teg_recv(void *addr,
|
||||
size_t count,
|
||||
ompi_datatype_t * datatype,
|
||||
int src,
|
||||
int tag,
|
||||
struct ompi_communicator_t *comm,
|
||||
ompi_status_public_t * status)
|
||||
{
|
||||
int rc;
|
||||
mca_ptl_base_recv_request_t *recvreq;
|
||||
MCA_PML_TEG_RECV_REQUEST_ALLOC(recvreq, rc);
|
||||
if (NULL == recvreq)
|
||||
return rc;
|
||||
|
||||
MCA_PML_TEG_RECV_REQUEST_INIT(recvreq,
|
||||
addr,
|
||||
count, datatype, src, tag, comm, false);
|
||||
|
||||
if ((rc = mca_pml_teg_recv_request_start(recvreq)) != OMPI_SUCCESS) {
|
||||
goto recv_finish;
|
||||
}
|
||||
|
||||
if (recvreq->req_recv.req_base.req_ompi.req_complete == false) {
|
||||
/* give up and sleep until completion */
|
||||
if (ompi_using_threads()) {
|
||||
ompi_mutex_lock(&ompi_request_lock);
|
||||
ompi_request_waiting++;
|
||||
while (recvreq->req_recv.req_base.req_ompi.req_complete == false)
|
||||
ompi_condition_wait(&ompi_request_cond, &ompi_request_lock);
|
||||
ompi_request_waiting--;
|
||||
ompi_mutex_unlock(&ompi_request_lock);
|
||||
} else {
|
||||
ompi_request_waiting++;
|
||||
while (recvreq->req_recv.req_base.req_ompi.req_complete == false)
|
||||
ompi_condition_wait(&ompi_request_cond, &ompi_request_lock);
|
||||
ompi_request_waiting--;
|
||||
}
|
||||
}
|
||||
recv_finish:
|
||||
if (NULL != status) { /* return status */
|
||||
*status = recvreq->req_recv.req_base.req_ompi.req_status;
|
||||
}
|
||||
|
||||
MCA_PML_TEG_RECV_REQUEST_RETURN(recvreq);
|
||||
return recvreq->req_recv.req_base.req_ompi.req_status.MPI_ERROR;
|
||||
}
|
||||
128
src/mca/pml/teg/pml_teg_isend.c
Обычный файл
128
src/mca/pml/teg/pml_teg_isend.c
Обычный файл
@ -0,0 +1,128 @@
|
||||
/*
|
||||
* Copyright (c) 2004-2005 The Trustees of Indiana University.
|
||||
* All rights reserved.
|
||||
* Copyright (c) 2004-2005 The Trustees of the University of Tennessee.
|
||||
* All rights reserved.
|
||||
* Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
|
||||
* University of Stuttgart. All rights reserved.
|
||||
* Copyright (c) 2004-2005 The Regents of the University of California.
|
||||
* All rights reserved.
|
||||
* $COPYRIGHT$
|
||||
*
|
||||
* Additional copyrights may follow
|
||||
*
|
||||
* $HEADER$
|
||||
*/
|
||||
|
||||
#include "ompi_config.h"
|
||||
|
||||
#include "pml_teg.h"
|
||||
#include "pml_teg_proc.h"
|
||||
#include "pml_teg_sendreq.h"
|
||||
#include "pml_teg_recvreq.h"
|
||||
|
||||
|
||||
int mca_pml_teg_isend_init(void *buf,
|
||||
size_t count,
|
||||
ompi_datatype_t * datatype,
|
||||
int dst,
|
||||
int tag,
|
||||
mca_pml_base_send_mode_t sendmode,
|
||||
ompi_communicator_t * comm,
|
||||
ompi_request_t ** request)
|
||||
{
|
||||
int rc;
|
||||
|
||||
mca_pml_teg_send_request_t *sendreq;
|
||||
MCA_PML_TEG_SEND_REQUEST_ALLOC(comm, dst, sendreq, rc);
|
||||
if (rc != OMPI_SUCCESS)
|
||||
return rc;
|
||||
|
||||
MCA_PML_TEG_SEND_REQUEST_INIT(sendreq,
|
||||
buf,
|
||||
count,
|
||||
datatype,
|
||||
dst, tag,
|
||||
comm, sendmode, true);
|
||||
|
||||
*request = (ompi_request_t *) sendreq;
|
||||
return OMPI_SUCCESS;
|
||||
}
|
||||
|
||||
|
||||
int mca_pml_teg_isend(void *buf,
|
||||
size_t count,
|
||||
ompi_datatype_t * datatype,
|
||||
int dst,
|
||||
int tag,
|
||||
mca_pml_base_send_mode_t sendmode,
|
||||
ompi_communicator_t * comm,
|
||||
ompi_request_t ** request)
|
||||
{
|
||||
int rc;
|
||||
mca_pml_teg_send_request_t *sendreq;
|
||||
MCA_PML_TEG_SEND_REQUEST_ALLOC(comm, dst, sendreq, rc);
|
||||
if (rc != OMPI_SUCCESS)
|
||||
return rc;
|
||||
MCA_PML_TEG_SEND_REQUEST_INIT(sendreq,
|
||||
buf,
|
||||
count,
|
||||
datatype,
|
||||
dst, tag,
|
||||
comm, sendmode, false);
|
||||
|
||||
MCA_PML_TEG_SEND_REQUEST_START(sendreq, rc);
|
||||
*request = (ompi_request_t *) sendreq;
|
||||
return rc;
|
||||
}
|
||||
|
||||
|
||||
int mca_pml_teg_send(void *buf,
|
||||
size_t count,
|
||||
ompi_datatype_t * datatype,
|
||||
int dst,
|
||||
int tag,
|
||||
mca_pml_base_send_mode_t sendmode,
|
||||
ompi_communicator_t * comm)
|
||||
{
|
||||
int rc;
|
||||
mca_pml_teg_send_request_t *sendreq;
|
||||
MCA_PML_TEG_SEND_REQUEST_ALLOC(comm, dst, sendreq, rc);
|
||||
if (rc != OMPI_SUCCESS)
|
||||
return rc;
|
||||
|
||||
MCA_PML_TEG_SEND_REQUEST_INIT(sendreq,
|
||||
buf,
|
||||
count,
|
||||
datatype,
|
||||
dst, tag,
|
||||
comm, sendmode, false);
|
||||
|
||||
MCA_PML_TEG_SEND_REQUEST_START(sendreq, rc);
|
||||
if (rc != OMPI_SUCCESS) {
|
||||
MCA_PML_TEG_FREE((ompi_request_t **) & sendreq);
|
||||
return rc;
|
||||
}
|
||||
|
||||
if (sendreq->req_send.req_base.req_ompi.req_complete == false) {
|
||||
/* give up and sleep until completion */
|
||||
if (ompi_using_threads()) {
|
||||
ompi_mutex_lock(&ompi_request_lock);
|
||||
ompi_request_waiting++;
|
||||
while (sendreq->req_send.req_base.req_ompi.req_complete == false)
|
||||
ompi_condition_wait(&ompi_request_cond, &ompi_request_lock);
|
||||
ompi_request_waiting--;
|
||||
ompi_mutex_unlock(&ompi_request_lock);
|
||||
} else {
|
||||
ompi_request_waiting++;
|
||||
while (sendreq->req_send.req_base.req_ompi.req_complete == false)
|
||||
ompi_condition_wait(&ompi_request_cond, &ompi_request_lock);
|
||||
ompi_request_waiting--;
|
||||
}
|
||||
}
|
||||
|
||||
/* return request to pool */
|
||||
MCA_PML_TEG_FREE((ompi_request_t **) & sendreq);
|
||||
return OMPI_SUCCESS;
|
||||
}
|
||||
|
||||
56
src/mca/pml/teg/pml_teg_proc.c
Обычный файл
56
src/mca/pml/teg/pml_teg_proc.c
Обычный файл
@ -0,0 +1,56 @@
|
||||
/*
|
||||
* Copyright (c) 2004-2005 The Trustees of Indiana University.
|
||||
* All rights reserved.
|
||||
* Copyright (c) 2004-2005 The Trustees of the University of Tennessee.
|
||||
* All rights reserved.
|
||||
* Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
|
||||
* University of Stuttgart. All rights reserved.
|
||||
* Copyright (c) 2004-2005 The Regents of the University of California.
|
||||
* All rights reserved.
|
||||
* $COPYRIGHT$
|
||||
*
|
||||
* Additional copyrights may follow
|
||||
*
|
||||
* $HEADER$
|
||||
*/
|
||||
|
||||
#include "ompi_config.h"
|
||||
|
||||
#include "include/sys/atomic.h"
|
||||
#include "pml_teg.h"
|
||||
#include "pml_teg_proc.h"
|
||||
#include "pml_ptl_array.h"
|
||||
|
||||
|
||||
static void mca_pml_teg_proc_construct(mca_pml_proc_t* proc)
|
||||
{
|
||||
proc->proc_ompi = NULL;
|
||||
proc->proc_ptl_flags = 0;
|
||||
OBJ_CONSTRUCT(&proc->proc_lock, ompi_mutex_t);
|
||||
OBJ_CONSTRUCT(&proc->proc_ptl_first, mca_pml_teg_ptl_array_t);
|
||||
OBJ_CONSTRUCT(&proc->proc_ptl_next, mca_pml_teg_ptl_array_t);
|
||||
|
||||
OMPI_THREAD_LOCK(&mca_pml_teg.teg_lock);
|
||||
ompi_list_append(&mca_pml_teg.teg_procs, (ompi_list_item_t*)proc);
|
||||
OMPI_THREAD_UNLOCK(&mca_pml_teg.teg_lock);
|
||||
}
|
||||
|
||||
|
||||
static void mca_pml_teg_proc_destruct(mca_pml_proc_t* proc)
|
||||
{
|
||||
OMPI_THREAD_LOCK(&mca_pml_teg.teg_lock);
|
||||
ompi_list_remove_item(&mca_pml_teg.teg_procs, (ompi_list_item_t*)proc);
|
||||
OMPI_THREAD_UNLOCK(&mca_pml_teg.teg_lock);
|
||||
|
||||
OBJ_DESTRUCT(&proc->proc_lock);
|
||||
OBJ_DESTRUCT(&proc->proc_ptl_first);
|
||||
OBJ_DESTRUCT(&proc->proc_ptl_next);
|
||||
}
|
||||
|
||||
OBJ_CLASS_INSTANCE(
|
||||
mca_pml_teg_proc_t,
|
||||
ompi_list_item_t,
|
||||
mca_pml_teg_proc_construct,
|
||||
mca_pml_teg_proc_destruct
|
||||
);
|
||||
|
||||
107
src/mca/pml/teg/pml_teg_proc.h
Обычный файл
107
src/mca/pml/teg/pml_teg_proc.h
Обычный файл
@ -0,0 +1,107 @@
|
||||
/*
|
||||
* Copyright (c) 2004-2005 The Trustees of Indiana University.
|
||||
* All rights reserved.
|
||||
* Copyright (c) 2004-2005 The Trustees of the University of Tennessee.
|
||||
* All rights reserved.
|
||||
* Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
|
||||
* University of Stuttgart. All rights reserved.
|
||||
* Copyright (c) 2004-2005 The Regents of the University of California.
|
||||
* All rights reserved.
|
||||
* $COPYRIGHT$
|
||||
*
|
||||
* Additional copyrights may follow
|
||||
*
|
||||
* $HEADER$
|
||||
*/
|
||||
/**
|
||||
* @file
|
||||
*/
|
||||
#ifndef MCA_PML_PROC_H
|
||||
#define MCA_PML_PROC_H
|
||||
|
||||
#include "threads/mutex.h"
|
||||
#include "communicator/communicator.h"
|
||||
#include "group/group.h"
|
||||
#include "proc/proc.h"
|
||||
#include "pml_ptl_array.h"
|
||||
|
||||
#if defined(c_plusplus) || defined(__cplusplus)
|
||||
extern "C" {
|
||||
#endif
|
||||
/**
|
||||
* Structure associated w/ ompi_proc_t that contains data specific
|
||||
* to the PML. Note that this name is not PML specific.
|
||||
*/
|
||||
struct mca_pml_proc_t {
|
||||
ompi_list_item_t super;
|
||||
ompi_proc_t *proc_ompi; /**< back-pointer to ompi_proc_t */
|
||||
ompi_mutex_t proc_lock; /**< lock to protect against concurrent access */
|
||||
mca_ptl_array_t proc_ptl_first; /**< array of ptls to use for first fragments */
|
||||
mca_ptl_array_t proc_ptl_next; /**< array of ptls to use for remaining fragments */
|
||||
uint32_t proc_ptl_flags; /**< aggregate ptl flags */
|
||||
};
|
||||
typedef struct mca_pml_proc_t mca_pml_proc_t;
|
||||
|
||||
|
||||
OMPI_COMP_EXPORT extern ompi_class_t mca_pml_teg_proc_t_class;
|
||||
typedef struct mca_pml_proc_t mca_pml_teg_proc_t;
|
||||
|
||||
/**
|
||||
* Return the mca_pml_proc_t instance cached in the communicators local group.
|
||||
*
|
||||
* @param comm Communicator
|
||||
* @param rank Peer rank
|
||||
* @return mca_pml_proc_t instance
|
||||
*/
|
||||
|
||||
static inline mca_pml_proc_t* mca_pml_teg_proc_lookup_local(ompi_communicator_t* comm, int rank)
|
||||
{
|
||||
ompi_proc_t* proc = comm->c_local_group->grp_proc_pointers[rank];
|
||||
return proc->proc_pml;
|
||||
}
|
||||
|
||||
/**
|
||||
* Return the mca_pml_proc_t instance cached on the communicators remote group.
|
||||
*
|
||||
* @param comm Communicator
|
||||
* @param rank Peer rank
|
||||
* @return mca_pml_proc_t instance
|
||||
*/
|
||||
|
||||
static inline mca_pml_proc_t* mca_pml_teg_proc_lookup_remote(ompi_communicator_t* comm, int rank)
|
||||
{
|
||||
ompi_proc_t* proc = comm->c_remote_group->grp_proc_pointers[rank];
|
||||
return proc->proc_pml;
|
||||
}
|
||||
|
||||
/**
|
||||
* Return the mca_ptl_peer_t instance corresponding to the process/ptl combination.
|
||||
*
|
||||
* @param comm Communicator
|
||||
* @param rank Peer rank
|
||||
* @return mca_pml_proc_t instance
|
||||
*/
|
||||
|
||||
static inline struct mca_ptl_base_peer_t* mca_pml_teg_proc_lookup_remote_peer(
|
||||
ompi_communicator_t* comm,
|
||||
int rank,
|
||||
struct mca_ptl_base_module_t* ptl)
|
||||
{
|
||||
ompi_proc_t* proc = comm->c_remote_group->grp_proc_pointers[rank];
|
||||
mca_pml_proc_t* proc_pml = proc->proc_pml;
|
||||
size_t i, size = mca_ptl_array_get_size(&proc_pml->proc_ptl_first);
|
||||
mca_ptl_proc_t* proc_ptl = proc_pml->proc_ptl_first.ptl_procs;
|
||||
for(i = 0; i < size; i++) {
|
||||
if(proc_ptl->ptl == ptl) {
|
||||
return proc_ptl->ptl_peer;
|
||||
}
|
||||
proc_ptl++;
|
||||
}
|
||||
return NULL;
|
||||
}
|
||||
|
||||
#if defined(c_plusplus) || defined(__cplusplus)
|
||||
}
|
||||
#endif
|
||||
#endif
|
||||
|
||||
40
src/mca/pml/teg/pml_teg_progress.c
Обычный файл
40
src/mca/pml/teg/pml_teg_progress.c
Обычный файл
@ -0,0 +1,40 @@
|
||||
/*
|
||||
* Copyright (c) 2004-2005 The Trustees of Indiana University.
|
||||
* All rights reserved.
|
||||
* Copyright (c) 2004-2005 The Trustees of the University of Tennessee.
|
||||
* All rights reserved.
|
||||
* Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
|
||||
* University of Stuttgart. All rights reserved.
|
||||
* Copyright (c) 2004-2005 The Regents of the University of California.
|
||||
* All rights reserved.
|
||||
* $COPYRIGHT$
|
||||
*
|
||||
* Additional copyrights may follow
|
||||
*
|
||||
* $HEADER$
|
||||
*/
|
||||
|
||||
#include "ompi_config.h"
|
||||
|
||||
#include "pml_teg.h"
|
||||
#include "pml_teg_sendreq.h"
|
||||
|
||||
|
||||
int mca_pml_teg_progress(void)
|
||||
{
|
||||
mca_ptl_tstamp_t tstamp = 0;
|
||||
size_t i;
|
||||
int count = 0;
|
||||
|
||||
/*
|
||||
* Progress each of the PTL modules
|
||||
*/
|
||||
for(i=0; i<mca_pml_teg.teg_num_ptl_progress; i++) {
|
||||
int rc = mca_pml_teg.teg_ptl_progress[i](tstamp);
|
||||
if(rc > 0) {
|
||||
count += rc;
|
||||
}
|
||||
}
|
||||
return count;
|
||||
}
|
||||
|
||||
43
src/mca/pml/teg/pml_teg_ptl.c
Обычный файл
43
src/mca/pml/teg/pml_teg_ptl.c
Обычный файл
@ -0,0 +1,43 @@
|
||||
/*
|
||||
* Copyright (c) 2004-2005 The Trustees of Indiana University.
|
||||
* All rights reserved.
|
||||
* Copyright (c) 2004-2005 The Trustees of the University of Tennessee.
|
||||
* All rights reserved.
|
||||
* Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
|
||||
* University of Stuttgart. All rights reserved.
|
||||
* Copyright (c) 2004-2005 The Regents of the University of California.
|
||||
* All rights reserved.
|
||||
* $COPYRIGHT$
|
||||
*
|
||||
* Additional copyrights may follow
|
||||
*
|
||||
* $HEADER$
|
||||
*/
|
||||
|
||||
#include "ompi_config.h"
|
||||
|
||||
#include "pml_teg_ptl.h"
|
||||
|
||||
|
||||
static void mca_pml_base_ptl_construct(mca_pml_base_ptl_t* ptl)
|
||||
{
|
||||
OBJ_CONSTRUCT(&ptl->ptl_cache, ompi_list_t);
|
||||
OBJ_CONSTRUCT(&ptl->ptl_cache_lock, ompi_mutex_t);
|
||||
ptl->ptl = NULL;
|
||||
ptl->ptl_cache_size = 0;
|
||||
ptl->ptl_cache_alloc = 0;
|
||||
}
|
||||
|
||||
static void mca_pml_base_ptl_destruct(mca_pml_base_ptl_t* ptl)
|
||||
{
|
||||
OBJ_DESTRUCT(&ptl->ptl_cache);
|
||||
OBJ_DESTRUCT(&ptl->ptl_cache_lock);
|
||||
}
|
||||
|
||||
OBJ_CLASS_INSTANCE(
|
||||
mca_pml_base_ptl_t,
|
||||
ompi_list_t,
|
||||
mca_pml_base_ptl_construct,
|
||||
mca_pml_base_ptl_destruct
|
||||
);
|
||||
|
||||
44
src/mca/pml/teg/pml_teg_ptl.h
Обычный файл
44
src/mca/pml/teg/pml_teg_ptl.h
Обычный файл
@ -0,0 +1,44 @@
|
||||
/*
|
||||
* Copyright (c) 2004-2005 The Trustees of Indiana University.
|
||||
* All rights reserved.
|
||||
* Copyright (c) 2004-2005 The Trustees of the University of Tennessee.
|
||||
* All rights reserved.
|
||||
* Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
|
||||
* University of Stuttgart. All rights reserved.
|
||||
* Copyright (c) 2004-2005 The Regents of the University of California.
|
||||
* All rights reserved.
|
||||
* $COPYRIGHT$
|
||||
*
|
||||
* Additional copyrights may follow
|
||||
*
|
||||
* $HEADER$
|
||||
*/
|
||||
|
||||
#ifndef _MCA_PML_BASE_PTL_
|
||||
#define _MCA_PML_BASE_PTL_
|
||||
|
||||
#include "mca/pml/pml.h"
|
||||
#include "mca/ptl/ptl.h"
|
||||
#include "threads/condition.h"
|
||||
#if defined(c_plusplus) || defined(__cplusplus)
|
||||
extern "C" {
|
||||
#endif
|
||||
|
||||
|
||||
struct mca_pml_base_ptl_t {
|
||||
ompi_list_t ptl_cache; /**< cache of send requests */
|
||||
size_t ptl_cache_size; /**< maximum size of cache */
|
||||
size_t ptl_cache_alloc; /**< current number of allocated items */
|
||||
ompi_mutex_t ptl_cache_lock; /**< lock for queue access */
|
||||
struct mca_ptl_base_module_t* ptl; /**< back pointer to ptl */
|
||||
};
|
||||
typedef struct mca_pml_base_ptl_t mca_pml_base_ptl_t;
|
||||
|
||||
OBJ_CLASS_DECLARATION(mca_pml_base_ptl_t);
|
||||
|
||||
#if defined(c_plusplus) || defined(__cplusplus)
|
||||
}
|
||||
#endif
|
||||
|
||||
#endif
|
||||
|
||||
93
src/mca/pml/teg/pml_teg_recvfrag.c
Обычный файл
93
src/mca/pml/teg/pml_teg_recvfrag.c
Обычный файл
@ -0,0 +1,93 @@
|
||||
/*
|
||||
* Copyright (c) 2004-2005 The Trustees of Indiana University.
|
||||
* All rights reserved.
|
||||
* Copyright (c) 2004-2005 The Trustees of the University of Tennessee.
|
||||
* All rights reserved.
|
||||
* Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
|
||||
* University of Stuttgart. All rights reserved.
|
||||
* Copyright (c) 2004-2005 The Regents of the University of California.
|
||||
* All rights reserved.
|
||||
* $COPYRIGHT$
|
||||
*
|
||||
* Additional copyrights may follow
|
||||
*
|
||||
* $HEADER$
|
||||
*/
|
||||
|
||||
/**
|
||||
* @file
|
||||
*/
|
||||
|
||||
#include "ompi_config.h"
|
||||
|
||||
#include "mca/pml/pml.h"
|
||||
#include "pml_teg_recvfrag.h"
|
||||
#include "pml_teg_proc.h"
|
||||
|
||||
|
||||
OMPI_DECLSPEC extern ompi_class_t mca_ptl_base_recv_frag_t_class;
|
||||
|
||||
|
||||
/**
|
||||
* Called by the PTL to match attempt a match for new fragments.
|
||||
*
|
||||
* @param ptl (IN) The PTL pointer
|
||||
* @param frag (IN) Receive fragment descriptor.
|
||||
* @param header (IN) Header corresponding to the receive fragment.
|
||||
* @return OMPI_SUCCESS or error status on failure.
|
||||
*/
|
||||
bool mca_pml_teg_recv_frag_match(
|
||||
mca_ptl_base_module_t* ptl,
|
||||
mca_ptl_base_recv_frag_t* frag,
|
||||
mca_ptl_base_match_header_t* header)
|
||||
{
|
||||
bool matched;
|
||||
bool matches = false;
|
||||
ompi_list_t matched_frags;
|
||||
if((matched = mca_ptl_base_match(header, frag, &matched_frags, &matches)) == false) {
|
||||
frag = (matches ? (mca_ptl_base_recv_frag_t*)ompi_list_remove_first(&matched_frags) : NULL);
|
||||
}
|
||||
|
||||
while(NULL != frag) {
|
||||
mca_ptl_base_module_t* ptl = frag->frag_base.frag_owner;
|
||||
mca_ptl_base_recv_request_t *request = frag->frag_request;
|
||||
mca_ptl_base_match_header_t *header = &frag->frag_base.frag_header.hdr_match;
|
||||
|
||||
/*
|
||||
* Initialize request status.
|
||||
*/
|
||||
request->req_recv.req_bytes_packed = header->hdr_msg_length;
|
||||
request->req_recv.req_base.req_ompi.req_status.MPI_SOURCE = header->hdr_src;
|
||||
request->req_recv.req_base.req_ompi.req_status.MPI_TAG = header->hdr_tag;
|
||||
|
||||
/*
|
||||
* If probe - signal request is complete - but don't notify PTL
|
||||
*/
|
||||
if(request->req_recv.req_base.req_type == MCA_PML_REQUEST_PROBE) {
|
||||
|
||||
ptl->ptl_recv_progress( ptl,
|
||||
request,
|
||||
header->hdr_msg_length,
|
||||
header->hdr_msg_length );
|
||||
matched = mca_pml_teg_recv_frag_match( ptl, frag, header );
|
||||
|
||||
} else {
|
||||
|
||||
/* if required - setup pointer to ptls peer */
|
||||
if (NULL == frag->frag_base.frag_peer) {
|
||||
frag->frag_base.frag_peer = mca_pml_teg_proc_lookup_remote_peer(
|
||||
request->req_recv.req_base.req_comm,header->hdr_src,ptl);
|
||||
}
|
||||
|
||||
/* notify ptl of match */
|
||||
ptl->ptl_matched(ptl, frag);
|
||||
|
||||
};
|
||||
|
||||
/* process any additional fragments that arrived out of order */
|
||||
frag = (matches ? (mca_ptl_base_recv_frag_t*)ompi_list_remove_first(&matched_frags) : NULL);
|
||||
};
|
||||
return matched;
|
||||
}
|
||||
|
||||
|
||||
42
src/mca/pml/teg/pml_teg_recvfrag.h
Обычный файл
42
src/mca/pml/teg/pml_teg_recvfrag.h
Обычный файл
@ -0,0 +1,42 @@
|
||||
/*
|
||||
* Copyright (c) 2004-2005 The Trustees of Indiana University.
|
||||
* All rights reserved.
|
||||
* Copyright (c) 2004-2005 The Trustees of the University of Tennessee.
|
||||
* All rights reserved.
|
||||
* Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
|
||||
* University of Stuttgart. All rights reserved.
|
||||
* Copyright (c) 2004-2005 The Regents of the University of California.
|
||||
* All rights reserved.
|
||||
* $COPYRIGHT$
|
||||
*
|
||||
* Additional copyrights may follow
|
||||
*
|
||||
* $HEADER$
|
||||
*/
|
||||
/**
|
||||
* @file
|
||||
*/
|
||||
|
||||
#ifndef MCA_PML_TEG_RECVFRAG_H
|
||||
#define MCA_PML_TEG_RECVFRAG_H
|
||||
|
||||
#include "mca/ptl/ptl.h"
|
||||
#include "mca/pml/base/pml_base_recvreq.h"
|
||||
#include "mca/ptl/base/ptl_base_recvfrag.h"
|
||||
|
||||
/**
|
||||
* Called by the PTL to match attempt a match for new fragments.
|
||||
*
|
||||
* @param ptl (IN) The PTL pointer
|
||||
* @param frag (IN) Receive fragment descriptor.
|
||||
* @param header (IN) Header corresponding to the receive fragment.
|
||||
* @return OMPI_SUCCESS or error status on failure.
|
||||
*/
|
||||
bool mca_pml_teg_recv_frag_match(
|
||||
mca_ptl_base_module_t* ptl,
|
||||
mca_ptl_base_recv_frag_t* frag,
|
||||
mca_ptl_base_match_header_t* header
|
||||
);
|
||||
|
||||
#endif
|
||||
|
||||
280
src/mca/pml/teg/pml_teg_recvreq.c
Обычный файл
280
src/mca/pml/teg/pml_teg_recvreq.c
Обычный файл
@ -0,0 +1,280 @@
|
||||
/*
|
||||
* Copyright (c) 2004-2005 The Trustees of Indiana University.
|
||||
* All rights reserved.
|
||||
* Copyright (c) 2004-2005 The Trustees of the University of Tennessee.
|
||||
* All rights reserved.
|
||||
* Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
|
||||
* University of Stuttgart. All rights reserved.
|
||||
* Copyright (c) 2004-2005 The Regents of the University of California.
|
||||
* All rights reserved.
|
||||
* $COPYRIGHT$
|
||||
*
|
||||
* Additional copyrights may follow
|
||||
*
|
||||
* $HEADER$
|
||||
*/
|
||||
|
||||
#include "ompi_config.h"
|
||||
|
||||
#include "mca/pml/pml.h"
|
||||
#include "mca/ptl/ptl.h"
|
||||
#include "mca/ptl/base/ptl_base_comm.h"
|
||||
#include "pml_teg_recvreq.h"
|
||||
#include "pml_teg_sendreq.h"
|
||||
|
||||
|
||||
static mca_ptl_base_recv_frag_t* mca_pml_teg_recv_request_match_specific_proc(
|
||||
mca_ptl_base_recv_request_t* request, int proc);
|
||||
|
||||
|
||||
static int mca_pml_teg_recv_request_fini(struct ompi_request_t** request)
|
||||
{
|
||||
MCA_PML_TEG_FINI(request);
|
||||
return OMPI_SUCCESS;
|
||||
}
|
||||
|
||||
static int mca_pml_teg_recv_request_free(struct ompi_request_t** request)
|
||||
{
|
||||
MCA_PML_TEG_FREE(request);
|
||||
return OMPI_SUCCESS;
|
||||
}
|
||||
|
||||
|
||||
static int mca_pml_teg_recv_request_cancel(struct ompi_request_t* request, int complete)
|
||||
{
|
||||
mca_pml_base_request_t* teg_request = (mca_pml_base_request_t*)request;
|
||||
ompi_communicator_t* ompi_comm = teg_request->req_comm;
|
||||
mca_pml_ptl_comm_t* pml_comm = (mca_pml_ptl_comm_t*)ompi_comm->c_pml_comm;
|
||||
|
||||
if( true == request->req_complete ) { /* way to late to cancel this one */
|
||||
return OMPI_SUCCESS;
|
||||
}
|
||||
|
||||
/* The rest should be protected behind the match logic lock */
|
||||
OMPI_THREAD_LOCK(&pml_comm->c_matching_lock);
|
||||
|
||||
if( OMPI_ANY_TAG == request->req_status.MPI_TAG ) { /* the match have not been already done */
|
||||
|
||||
if( teg_request->req_peer == OMPI_ANY_SOURCE ) {
|
||||
ompi_list_remove_item( &(pml_comm->c_wild_receives),
|
||||
(ompi_list_item_t*)request );
|
||||
} else {
|
||||
ompi_list_remove_item( pml_comm->c_specific_receives + teg_request->req_peer,
|
||||
(ompi_list_item_t*)request );
|
||||
}
|
||||
}
|
||||
|
||||
OMPI_THREAD_UNLOCK(&pml_comm->c_matching_lock);
|
||||
|
||||
request->req_status._cancelled = true;
|
||||
request->req_complete = true; /* mark it as completed so all the test/wait functions
|
||||
* on this particular request will finish */
|
||||
/* Now we have a problem if we are in a multi-threaded environment. We shou ld
|
||||
* broadcast the condition on the request in order to allow the other threa ds
|
||||
* to complete their test/wait functions.
|
||||
*/
|
||||
if(ompi_request_waiting) {
|
||||
ompi_condition_broadcast(&ompi_request_cond);
|
||||
}
|
||||
return OMPI_SUCCESS;
|
||||
}
|
||||
|
||||
static void mca_pml_teg_recv_request_construct(mca_ptl_base_recv_request_t* request)
|
||||
{
|
||||
request->req_recv.req_base.req_type = MCA_PML_REQUEST_RECV;
|
||||
request->req_recv.req_base.req_ompi.req_fini = mca_pml_teg_recv_request_fini;
|
||||
request->req_recv.req_base.req_ompi.req_free = mca_pml_teg_recv_request_free;
|
||||
request->req_recv.req_base.req_ompi.req_cancel = mca_pml_teg_recv_request_cancel;
|
||||
}
|
||||
|
||||
static void mca_pml_teg_recv_request_destruct(mca_ptl_base_recv_request_t* request)
|
||||
{
|
||||
}
|
||||
|
||||
OBJ_CLASS_INSTANCE(
|
||||
mca_pml_teg_recv_request_t,
|
||||
mca_ptl_base_recv_request_t,
|
||||
mca_pml_teg_recv_request_construct,
|
||||
mca_pml_teg_recv_request_destruct);
|
||||
|
||||
|
||||
/*
|
||||
* Update the recv request status to reflect the number of bytes
|
||||
* received and actually delivered to the application.
|
||||
*/
|
||||
|
||||
void mca_pml_teg_recv_request_progress(
|
||||
struct mca_ptl_base_module_t* ptl,
|
||||
mca_ptl_base_recv_request_t* req,
|
||||
size_t bytes_received,
|
||||
size_t bytes_delivered)
|
||||
{
|
||||
OMPI_THREAD_LOCK(&ompi_request_lock);
|
||||
req->req_bytes_received += bytes_received;
|
||||
req->req_bytes_delivered += bytes_delivered;
|
||||
if (req->req_bytes_received >= req->req_recv.req_bytes_packed) {
|
||||
/* initialize request status */
|
||||
req->req_recv.req_base.req_ompi.req_status._count = req->req_bytes_delivered;
|
||||
req->req_recv.req_base.req_pml_complete = true;
|
||||
req->req_recv.req_base.req_ompi.req_complete = true;
|
||||
if(ompi_request_waiting) {
|
||||
ompi_condition_broadcast(&ompi_request_cond);
|
||||
}
|
||||
}
|
||||
OMPI_THREAD_UNLOCK(&ompi_request_lock);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*
|
||||
* This routine is used to match a posted receive when the source process
|
||||
* is specified.
|
||||
*/
|
||||
|
||||
void mca_pml_teg_recv_request_match_specific(mca_ptl_base_recv_request_t* request)
|
||||
{
|
||||
ompi_communicator_t *comm = request->req_recv.req_base.req_comm;
|
||||
mca_pml_ptl_comm_t* pml_comm = comm->c_pml_comm;
|
||||
int req_peer = request->req_recv.req_base.req_peer;
|
||||
mca_ptl_base_recv_frag_t* frag;
|
||||
|
||||
/* check for a specific match */
|
||||
OMPI_THREAD_LOCK(&pml_comm->c_matching_lock);
|
||||
|
||||
/* assign sequence number */
|
||||
request->req_recv.req_base.req_sequence = pml_comm->c_recv_seq++;
|
||||
|
||||
if (ompi_list_get_size(&pml_comm->c_unexpected_frags[req_peer]) > 0 &&
|
||||
(frag = mca_pml_teg_recv_request_match_specific_proc(request, req_peer)) != NULL) {
|
||||
mca_ptl_base_module_t* ptl = frag->frag_base.frag_owner;
|
||||
/* setup pointer to ptls peer */
|
||||
if(NULL == frag->frag_base.frag_peer)
|
||||
frag->frag_base.frag_peer = mca_pml_teg_proc_lookup_remote_peer(comm,req_peer,ptl);
|
||||
OMPI_THREAD_UNLOCK(&pml_comm->c_matching_lock);
|
||||
if( !((MCA_PML_REQUEST_IPROBE == request->req_recv.req_base.req_type) ||
|
||||
(MCA_PML_REQUEST_PROBE == request->req_recv.req_base.req_type)) ) {
|
||||
ptl->ptl_matched(ptl, frag);
|
||||
}
|
||||
return; /* match found */
|
||||
}
|
||||
|
||||
/* We didn't find any matches. Record this irecv so we can match
|
||||
* it when the message comes in.
|
||||
*/
|
||||
if(request->req_recv.req_base.req_type != MCA_PML_REQUEST_IPROBE) {
|
||||
ompi_list_append(pml_comm->c_specific_receives+req_peer, (ompi_list_item_t*)request);
|
||||
}
|
||||
OMPI_THREAD_UNLOCK(&pml_comm->c_matching_lock);
|
||||
}
|
||||
|
||||
|
||||
/*
|
||||
* this routine is used to try and match a wild posted receive - where
|
||||
* wild is determined by the value assigned to the source process
|
||||
*/
|
||||
|
||||
void mca_pml_teg_recv_request_match_wild(mca_ptl_base_recv_request_t* request)
|
||||
{
|
||||
ompi_communicator_t *comm = request->req_recv.req_base.req_comm;
|
||||
mca_pml_ptl_comm_t* pml_comm = comm->c_pml_comm;
|
||||
int proc_count = comm->c_remote_group->grp_proc_count;
|
||||
int proc;
|
||||
|
||||
/*
|
||||
* Loop over all the outstanding messages to find one that matches.
|
||||
* There is an outer loop over lists of messages from each
|
||||
* process, then an inner loop over the messages from the
|
||||
* process.
|
||||
*/
|
||||
OMPI_THREAD_LOCK(&pml_comm->c_matching_lock);
|
||||
|
||||
/* assign sequence number */
|
||||
request->req_recv.req_base.req_sequence = pml_comm->c_recv_seq++;
|
||||
|
||||
for (proc = 0; proc < proc_count; proc++) {
|
||||
mca_ptl_base_recv_frag_t* frag;
|
||||
|
||||
/* continue if no frags to match */
|
||||
if (ompi_list_get_size(&pml_comm->c_unexpected_frags[proc]) == 0)
|
||||
continue;
|
||||
|
||||
/* loop over messages from the current proc */
|
||||
if ((frag = mca_pml_teg_recv_request_match_specific_proc(request, proc)) != NULL) {
|
||||
mca_ptl_base_module_t* ptl = frag->frag_base.frag_owner;
|
||||
/* if required - setup pointer to ptls peer */
|
||||
if(NULL == frag->frag_base.frag_peer)
|
||||
frag->frag_base.frag_peer = mca_pml_teg_proc_lookup_remote_peer(comm,proc,ptl);
|
||||
OMPI_THREAD_UNLOCK(&pml_comm->c_matching_lock);
|
||||
if( !((MCA_PML_REQUEST_IPROBE == request->req_recv.req_base.req_type) ||
|
||||
(MCA_PML_REQUEST_PROBE == request->req_recv.req_base.req_type)) ) {
|
||||
ptl->ptl_matched(ptl, frag);
|
||||
}
|
||||
return; /* match found */
|
||||
}
|
||||
}
|
||||
|
||||
/* We didn't find any matches. Record this irecv so we can match to
|
||||
* it when the message comes in.
|
||||
*/
|
||||
|
||||
if(request->req_recv.req_base.req_type != MCA_PML_REQUEST_IPROBE)
|
||||
ompi_list_append(&pml_comm->c_wild_receives, (ompi_list_item_t*)request);
|
||||
OMPI_THREAD_UNLOCK(&pml_comm->c_matching_lock);
|
||||
}
|
||||
|
||||
|
||||
/*
|
||||
* this routine tries to match a posted receive. If a match is found,
|
||||
* it places the request in the appropriate matched receive list.
|
||||
*/
|
||||
|
||||
static mca_ptl_base_recv_frag_t* mca_pml_teg_recv_request_match_specific_proc(
|
||||
mca_ptl_base_recv_request_t* request, int proc)
|
||||
{
|
||||
mca_pml_ptl_comm_t *pml_comm = request->req_recv.req_base.req_comm->c_pml_comm;
|
||||
ompi_list_t* unexpected_frags = pml_comm->c_unexpected_frags+proc;
|
||||
mca_ptl_base_recv_frag_t* frag;
|
||||
mca_ptl_base_match_header_t* header;
|
||||
int tag = request->req_recv.req_base.req_tag;
|
||||
|
||||
if( OMPI_ANY_TAG == tag ) {
|
||||
for (frag = (mca_ptl_base_recv_frag_t*)ompi_list_get_first(unexpected_frags);
|
||||
frag != (mca_ptl_base_recv_frag_t*)ompi_list_get_end(unexpected_frags);
|
||||
frag = (mca_ptl_base_recv_frag_t*)ompi_list_get_next(frag)) {
|
||||
header = &(frag->frag_base.frag_header.hdr_match);
|
||||
|
||||
/* check first frag - we assume that process matching has been done already */
|
||||
if( header->hdr_tag >= 0 ) {
|
||||
goto find_fragment;
|
||||
}
|
||||
}
|
||||
} else {
|
||||
for (frag = (mca_ptl_base_recv_frag_t*)ompi_list_get_first(unexpected_frags);
|
||||
frag != (mca_ptl_base_recv_frag_t*)ompi_list_get_end(unexpected_frags);
|
||||
frag = (mca_ptl_base_recv_frag_t*)ompi_list_get_next(frag)) {
|
||||
header = &(frag->frag_base.frag_header.hdr_match);
|
||||
|
||||
/* check first frag - we assume that process matching has been done already */
|
||||
if ( tag == header->hdr_tag ) {
|
||||
/* we assume that the tag is correct from MPI point of view (ie. >= 0 ) */
|
||||
goto find_fragment;
|
||||
}
|
||||
}
|
||||
}
|
||||
return NULL;
|
||||
find_fragment:
|
||||
request->req_recv.req_bytes_packed = header->hdr_msg_length;
|
||||
request->req_recv.req_base.req_ompi.req_status.MPI_TAG = header->hdr_tag;
|
||||
request->req_recv.req_base.req_ompi.req_status.MPI_SOURCE = header->hdr_src;
|
||||
|
||||
if( !((MCA_PML_REQUEST_IPROBE == request->req_recv.req_base.req_type) ||
|
||||
(MCA_PML_REQUEST_PROBE == request->req_recv.req_base.req_type)) ) {
|
||||
ompi_list_remove_item(unexpected_frags, (ompi_list_item_t*)frag);
|
||||
frag->frag_request = request;
|
||||
} else {
|
||||
/* it's a probe, therefore report it's completion */
|
||||
mca_pml_teg_recv_request_progress( NULL, request, header->hdr_msg_length, header->hdr_msg_length );
|
||||
}
|
||||
return frag;
|
||||
}
|
||||
|
||||
152
src/mca/pml/teg/pml_teg_recvreq.h
Обычный файл
152
src/mca/pml/teg/pml_teg_recvreq.h
Обычный файл
@ -0,0 +1,152 @@
|
||||
/*
|
||||
* Copyright (c) 2004-2005 The Trustees of Indiana University.
|
||||
* All rights reserved.
|
||||
* Copyright (c) 2004-2005 The Trustees of the University of Tennessee.
|
||||
* All rights reserved.
|
||||
* Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
|
||||
* University of Stuttgart. All rights reserved.
|
||||
* Copyright (c) 2004-2005 The Regents of the University of California.
|
||||
* All rights reserved.
|
||||
* $COPYRIGHT$
|
||||
*
|
||||
* Additional copyrights may follow
|
||||
*
|
||||
* $HEADER$
|
||||
*/
|
||||
/**
|
||||
* @file
|
||||
*/
|
||||
#ifndef OMPI_PML_TEG_RECV_REQUEST_H
|
||||
#define OMPI_PML_TEG_RECV_REQUEST_H
|
||||
|
||||
#include "pml_teg.h"
|
||||
#include "pml_teg_proc.h"
|
||||
#include "mca/ptl/base/ptl_base_recvreq.h"
|
||||
#include "mca/ptl/base/ptl_base_recvfrag.h"
|
||||
|
||||
#if defined(c_plusplus) || defined(__cplusplus)
|
||||
extern "C" {
|
||||
#endif
|
||||
typedef mca_ptl_base_recv_request_t mca_pml_teg_recv_request_t;
|
||||
|
||||
OBJ_CLASS_DECLARATION(mca_pml_teg_recv_request_t);
|
||||
|
||||
|
||||
/**
|
||||
* Allocate a recv request from the modules free list.
|
||||
*
|
||||
* @param rc (OUT) OMPI_SUCCESS or error status on failure.
|
||||
* @return Receive request.
|
||||
*/
|
||||
#define MCA_PML_TEG_RECV_REQUEST_ALLOC(recvreq, rc) \
|
||||
do { \
|
||||
ompi_list_item_t* item; \
|
||||
OMPI_FREE_LIST_GET(&mca_pml_teg.teg_recv_requests, item, rc); \
|
||||
recvreq = (mca_ptl_base_recv_request_t*)item; \
|
||||
} while(0)
|
||||
|
||||
|
||||
/**
|
||||
* Initialize a recv request.
|
||||
*/
|
||||
#define MCA_PML_TEG_RECV_REQUEST_INIT( \
|
||||
request, \
|
||||
addr, \
|
||||
count, \
|
||||
datatype, \
|
||||
src, \
|
||||
tag, \
|
||||
comm, \
|
||||
persistent) \
|
||||
{ \
|
||||
MCA_PML_BASE_RECV_REQUEST_INIT( \
|
||||
(&(request)->req_recv), \
|
||||
addr, \
|
||||
count, \
|
||||
datatype, \
|
||||
src, \
|
||||
tag, \
|
||||
comm, \
|
||||
persistent \
|
||||
); \
|
||||
}
|
||||
|
||||
|
||||
/**
|
||||
* Return a recv request to the modules free list.
|
||||
*
|
||||
* @param request (IN) Receive request.
|
||||
*/
|
||||
#define MCA_PML_TEG_RECV_REQUEST_RETURN(request) \
|
||||
do { \
|
||||
MCA_PML_BASE_RECV_REQUEST_RETURN( &request->req_recv ); \
|
||||
OMPI_FREE_LIST_RETURN(&mca_pml_teg.teg_recv_requests, (ompi_list_item_t*)request); \
|
||||
} while(0)
|
||||
|
||||
/**
|
||||
* Attempt to match the request against the unexpected fragment list
|
||||
* for all source ranks w/in the communicator.
|
||||
*
|
||||
* @param request (IN) Request to match.
|
||||
*/
|
||||
void mca_pml_teg_recv_request_match_wild(mca_ptl_base_recv_request_t* request);
|
||||
|
||||
/**
|
||||
* Attempt to match the request against the unexpected fragment list
|
||||
* for a specific source rank.
|
||||
*
|
||||
* @param request (IN) Request to match.
|
||||
*/
|
||||
void mca_pml_teg_recv_request_match_specific(mca_ptl_base_recv_request_t* request);
|
||||
|
||||
/**
|
||||
* Start an initialized request.
|
||||
*
|
||||
* @param request Receive request.
|
||||
* @return OMPI_SUCESS or error status on failure.
|
||||
*/
|
||||
static inline int mca_pml_teg_recv_request_start(mca_ptl_base_recv_request_t* request)
|
||||
{
|
||||
/* init/re-init the request */
|
||||
request->req_bytes_received = 0;
|
||||
request->req_bytes_delivered = 0;
|
||||
request->req_recv.req_base.req_pml_complete = false;
|
||||
request->req_recv.req_base.req_ompi.req_complete = false;
|
||||
request->req_recv.req_base.req_ompi.req_state = OMPI_REQUEST_ACTIVE;
|
||||
/* always set the req_status.MPI_TAG to ANY_TAG before starting the request. This field
|
||||
* is used on the cancel part in order to find out if the request has been matched or not.
|
||||
*/
|
||||
request->req_recv.req_base.req_ompi.req_status.MPI_TAG = OMPI_ANY_TAG;
|
||||
request->req_recv.req_base.req_ompi.req_status.MPI_ERROR = OMPI_SUCCESS;
|
||||
request->req_recv.req_base.req_ompi.req_status._cancelled = 0;
|
||||
|
||||
/* attempt to match posted recv */
|
||||
if(request->req_recv.req_base.req_peer == OMPI_ANY_SOURCE) {
|
||||
mca_pml_teg_recv_request_match_wild(request);
|
||||
} else {
|
||||
mca_pml_teg_recv_request_match_specific(request);
|
||||
}
|
||||
return OMPI_SUCCESS;
|
||||
}
|
||||
|
||||
/**
|
||||
* Update status of a recv request based on the completion status of
|
||||
* the receive fragment.
|
||||
*
|
||||
* @param ptl (IN) The PTL pointer.
|
||||
* @param request (IN) Receive request.
|
||||
* @param bytes_received (IN) Bytes received from peer.
|
||||
* @param bytes_delivered (IN) Bytes delivered to application.
|
||||
*/
|
||||
void mca_pml_teg_recv_request_progress(
|
||||
struct mca_ptl_base_module_t* ptl,
|
||||
mca_ptl_base_recv_request_t* request,
|
||||
size_t bytes_received,
|
||||
size_t bytes_delivered
|
||||
);
|
||||
|
||||
#if defined(c_plusplus) || defined(__cplusplus)
|
||||
}
|
||||
#endif
|
||||
#endif
|
||||
|
||||
215
src/mca/pml/teg/pml_teg_sendreq.c
Обычный файл
215
src/mca/pml/teg/pml_teg_sendreq.c
Обычный файл
@ -0,0 +1,215 @@
|
||||
/*
|
||||
* Copyright (c) 2004-2005 The Trustees of Indiana University.
|
||||
* All rights reserved.
|
||||
* Copyright (c) 2004-2005 The Trustees of the University of Tennessee.
|
||||
* All rights reserved.
|
||||
* Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
|
||||
* University of Stuttgart. All rights reserved.
|
||||
* Copyright (c) 2004-2005 The Regents of the University of California.
|
||||
* All rights reserved.
|
||||
* $COPYRIGHT$
|
||||
*
|
||||
* Additional copyrights may follow
|
||||
*
|
||||
* $HEADER$
|
||||
*/
|
||||
|
||||
|
||||
/*%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%*/
|
||||
|
||||
#include "ompi_config.h"
|
||||
#include "include/constants.h"
|
||||
#include "mca/pml/pml.h"
|
||||
#include "mca/ptl/ptl.h"
|
||||
#include "pml_teg.h"
|
||||
#include "pml_teg_proc.h"
|
||||
#include "pml_teg_sendreq.h"
|
||||
#include "pml_teg_recvreq.h"
|
||||
|
||||
|
||||
|
||||
static int mca_pml_teg_send_request_fini(struct ompi_request_t** request)
|
||||
{
|
||||
MCA_PML_TEG_FINI(request);
|
||||
return OMPI_SUCCESS;
|
||||
}
|
||||
|
||||
static int mca_pml_teg_send_request_free(struct ompi_request_t** request)
|
||||
{
|
||||
MCA_PML_TEG_FREE(request);
|
||||
return OMPI_SUCCESS;
|
||||
}
|
||||
|
||||
static int mca_pml_teg_send_request_cancel(struct ompi_request_t* request, int complete)
|
||||
{
|
||||
/* we dont cancel send requests by now */
|
||||
return OMPI_SUCCESS;
|
||||
}
|
||||
|
||||
|
||||
static void mca_pml_teg_send_request_construct(mca_pml_teg_send_request_t* req)
|
||||
{
|
||||
req->req_cached = false;
|
||||
req->req_send.req_base.req_ompi.req_fini = mca_pml_teg_send_request_fini;
|
||||
req->req_send.req_base.req_ompi.req_free = mca_pml_teg_send_request_free;
|
||||
req->req_send.req_base.req_ompi.req_cancel = mca_pml_teg_send_request_cancel;
|
||||
}
|
||||
|
||||
|
||||
static void mca_pml_teg_send_request_destruct(mca_pml_teg_send_request_t* req)
|
||||
{
|
||||
}
|
||||
|
||||
|
||||
OBJ_CLASS_INSTANCE(
|
||||
mca_pml_teg_send_request_t,
|
||||
mca_ptl_base_send_request_t,
|
||||
mca_pml_teg_send_request_construct,
|
||||
mca_pml_teg_send_request_destruct);
|
||||
|
||||
|
||||
/**
|
||||
* Schedule message delivery across potentially multiple PTLs.
|
||||
*
|
||||
* @param request (IN) Request to schedule
|
||||
* @return status Error status
|
||||
*
|
||||
*/
|
||||
|
||||
|
||||
int mca_pml_teg_send_request_schedule(mca_ptl_base_send_request_t* req)
|
||||
{
|
||||
ompi_proc_t *proc = ompi_comm_peer_lookup(req->req_send.req_base.req_comm, req->req_send.req_base.req_peer);
|
||||
mca_pml_proc_t* proc_pml = proc->proc_pml;
|
||||
int send_count = 0;
|
||||
size_t bytes_remaining;
|
||||
size_t num_ptl_avail;
|
||||
size_t num_ptl;
|
||||
|
||||
/*
|
||||
* Only allow one thread in this routine for a given request.
|
||||
* However, we cannot block callers on a mutex, so simply keep track
|
||||
* of the number of times the routine has been called and run through
|
||||
* the scheduling logic once for every call.
|
||||
*/
|
||||
if(OMPI_THREAD_ADD32(&req->req_lock,1) == 1) {
|
||||
do {
|
||||
/* allocate remaining bytes to PTLs */
|
||||
bytes_remaining = req->req_send.req_bytes_packed - req->req_offset;
|
||||
num_ptl_avail = proc_pml->proc_ptl_next.ptl_size;
|
||||
num_ptl = 0;
|
||||
while(bytes_remaining > 0 && num_ptl++ < num_ptl_avail) {
|
||||
mca_ptl_proc_t* ptl_proc = mca_ptl_array_get_next(&proc_pml->proc_ptl_next);
|
||||
mca_ptl_base_module_t* ptl = ptl_proc->ptl;
|
||||
int rc;
|
||||
|
||||
/* if this is the last PTL that is available to use, or the number of
|
||||
* bytes remaining in the message is less than the PTLs minimum fragment
|
||||
* size, then go ahead and give the rest of the message to this PTL.
|
||||
*/
|
||||
size_t bytes_to_frag;
|
||||
if(num_ptl == num_ptl_avail || bytes_remaining < ptl->ptl_min_frag_size) {
|
||||
bytes_to_frag = bytes_remaining;
|
||||
|
||||
/* otherwise attempt to give the PTL a percentage of the message
|
||||
* based on a weighting factor. for simplicity calculate this as
|
||||
* a percentage of the overall message length (regardless of amount
|
||||
* previously assigned)
|
||||
*/
|
||||
} else {
|
||||
bytes_to_frag = (ptl_proc->ptl_weight * bytes_remaining) / 100;
|
||||
}
|
||||
|
||||
/* makes sure that we don't exceed ptl_max_frag_size */
|
||||
if(ptl->ptl_max_frag_size != 0 && bytes_to_frag > ptl->ptl_max_frag_size)
|
||||
bytes_to_frag = ptl->ptl_max_frag_size;
|
||||
|
||||
rc = ptl->ptl_put(ptl, ptl_proc->ptl_peer, req, req->req_offset, bytes_to_frag, 0);
|
||||
if(rc == OMPI_SUCCESS) {
|
||||
send_count++;
|
||||
bytes_remaining = req->req_send.req_bytes_packed - req->req_offset;
|
||||
}
|
||||
}
|
||||
|
||||
/* unable to complete send - queue for later */
|
||||
if(send_count == 0) {
|
||||
OMPI_THREAD_LOCK(&mca_pml_teg.teg_lock);
|
||||
ompi_list_append(&mca_pml_teg.teg_send_pending, (ompi_list_item_t*)req);
|
||||
OMPI_THREAD_UNLOCK(&mca_pml_teg.teg_lock);
|
||||
req->req_lock = 0;
|
||||
return OMPI_ERR_OUT_OF_RESOURCE;
|
||||
}
|
||||
|
||||
/* fragments completed while scheduling - so retry */
|
||||
} while(OMPI_THREAD_ADD32(&req->req_lock,-1) > 0);
|
||||
|
||||
/* free the request if completed while in the scheduler */
|
||||
if (req->req_send.req_base.req_free_called && req->req_send.req_base.req_pml_complete) {
|
||||
MCA_PML_TEG_FREE((ompi_request_t**)&req);
|
||||
}
|
||||
}
|
||||
return OMPI_SUCCESS;
|
||||
}
|
||||
|
||||
|
||||
/**
|
||||
* Update the status of the send request to reflect the number of bytes
|
||||
* "actually" sent (and acknowledged). This should be called by the
|
||||
* lower layer PTL after the fragment is actually delivered and has been
|
||||
* acknowledged (if required). Note that this routine should NOT be called
|
||||
* directly by the PTL, a function pointer is setup on the PTL at init to
|
||||
* enable upcalls into the PML w/out directly linking to a specific PML
|
||||
* implementation.
|
||||
*/
|
||||
|
||||
void mca_pml_teg_send_request_progress(
|
||||
struct mca_ptl_base_module_t* ptl,
|
||||
mca_ptl_base_send_request_t* req,
|
||||
size_t bytes_sent)
|
||||
{
|
||||
bool schedule = false;
|
||||
|
||||
OMPI_THREAD_LOCK(&ompi_request_lock);
|
||||
req->req_bytes_sent += bytes_sent;
|
||||
if (req->req_bytes_sent >= req->req_send.req_bytes_packed) {
|
||||
req->req_send.req_base.req_pml_complete = true;
|
||||
if (req->req_send.req_base.req_ompi.req_complete == false) {
|
||||
req->req_send.req_base.req_ompi.req_status.MPI_SOURCE = req->req_send.req_base.req_comm->c_my_rank;
|
||||
req->req_send.req_base.req_ompi.req_status.MPI_TAG = req->req_send.req_base.req_tag;
|
||||
req->req_send.req_base.req_ompi.req_status.MPI_ERROR = OMPI_SUCCESS;
|
||||
req->req_send.req_base.req_ompi.req_status._count = req->req_bytes_sent;
|
||||
req->req_send.req_base.req_ompi.req_complete = true;
|
||||
if(ompi_request_waiting) {
|
||||
ompi_condition_broadcast(&ompi_request_cond);
|
||||
}
|
||||
} else if(req->req_send.req_base.req_free_called) {
|
||||
/* don't free the request if in the scheduler */
|
||||
if(req->req_lock == 0) {
|
||||
MCA_PML_TEG_FREE((ompi_request_t**)&req);
|
||||
}
|
||||
} else if (req->req_send.req_send_mode == MCA_PML_BASE_SEND_BUFFERED) {
|
||||
mca_pml_base_bsend_request_fini((ompi_request_t*)req);
|
||||
}
|
||||
/* test to see if we have scheduled the entire request */
|
||||
} else if (req->req_offset < req->req_send.req_bytes_packed) {
|
||||
schedule = true;
|
||||
}
|
||||
OMPI_THREAD_UNLOCK(&ompi_request_lock);
|
||||
|
||||
/* schedule remaining fragments of this request */
|
||||
if(schedule) {
|
||||
mca_pml_teg_send_request_schedule(req);
|
||||
}
|
||||
|
||||
/* check for pending requests that need to be progressed */
|
||||
while(ompi_list_get_size(&mca_pml_teg.teg_send_pending) != 0) {
|
||||
OMPI_THREAD_LOCK(&mca_pml_teg.teg_lock);
|
||||
req = (mca_ptl_base_send_request_t*)ompi_list_remove_first(&mca_pml_teg.teg_send_pending);
|
||||
OMPI_THREAD_UNLOCK(&mca_pml_teg.teg_lock);
|
||||
if(req == NULL)
|
||||
break;
|
||||
if(mca_pml_teg_send_request_schedule(req) != OMPI_SUCCESS)
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
211
src/mca/pml/teg/pml_teg_sendreq.h
Обычный файл
211
src/mca/pml/teg/pml_teg_sendreq.h
Обычный файл
@ -0,0 +1,211 @@
|
||||
/*
|
||||
* Copyright (c) 2004-2005 The Trustees of Indiana University.
|
||||
* All rights reserved.
|
||||
* Copyright (c) 2004-2005 The Trustees of the University of Tennessee.
|
||||
* All rights reserved.
|
||||
* Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
|
||||
* University of Stuttgart. All rights reserved.
|
||||
* Copyright (c) 2004-2005 The Regents of the University of California.
|
||||
* All rights reserved.
|
||||
* $COPYRIGHT$
|
||||
*
|
||||
* Additional copyrights may follow
|
||||
*
|
||||
* $HEADER$
|
||||
*/
|
||||
/**
|
||||
* @file
|
||||
*/
|
||||
#ifndef OMPI_PML_TEG_SEND_REQUEST_H
|
||||
#define OMPI_PML_TEG_SEND_REQUEST_H
|
||||
|
||||
#include "mca/ptl/ptl.h"
|
||||
#include "mca/ptl/base/ptl_base_sendreq.h"
|
||||
#include "mca/ptl/base/ptl_base_sendfrag.h"
|
||||
#include "mca/ptl/base/ptl_base_comm.h"
|
||||
#include "pml_teg_proc.h"
|
||||
#include "pml_teg_ptl.h"
|
||||
|
||||
#if defined(c_plusplus) || defined(__cplusplus)
|
||||
extern "C" {
|
||||
#endif
|
||||
|
||||
|
||||
typedef mca_ptl_base_send_request_t mca_pml_teg_send_request_t;
|
||||
OBJ_CLASS_DECLARATION(mca_pml_teg_send_request_t);
|
||||
|
||||
|
||||
#define MCA_PML_TEG_SEND_REQUEST_ALLOC( \
|
||||
comm, \
|
||||
dst, \
|
||||
sendreq, \
|
||||
rc) \
|
||||
{ \
|
||||
mca_pml_proc_t *proc = mca_pml_teg_proc_lookup_remote(comm,dst); \
|
||||
mca_ptl_proc_t* ptl_proc; \
|
||||
mca_pml_base_ptl_t* ptl_base; \
|
||||
\
|
||||
if(NULL == proc) { \
|
||||
return OMPI_ERR_OUT_OF_RESOURCE; \
|
||||
} \
|
||||
OMPI_THREAD_SCOPED_LOCK(&proc->proc_lock, \
|
||||
(ptl_proc = mca_ptl_array_get_next(&proc->proc_ptl_first))); \
|
||||
ptl_base = ptl_proc->ptl_base; \
|
||||
/* \
|
||||
* check to see if there is a cache of send requests associated with \
|
||||
* this ptl - if so try the allocation from there. \
|
||||
*/ \
|
||||
if(NULL != ptl_base) { \
|
||||
OMPI_THREAD_LOCK(&ptl_base->ptl_cache_lock); \
|
||||
sendreq = (mca_pml_teg_send_request_t*) \
|
||||
ompi_list_remove_first(&ptl_base->ptl_cache); \
|
||||
if(NULL != sendreq) { \
|
||||
OMPI_THREAD_UNLOCK(&ptl_base->ptl_cache_lock); \
|
||||
rc = OMPI_SUCCESS; \
|
||||
} else if (ptl_base->ptl_cache_alloc < ptl_base->ptl_cache_size) { \
|
||||
/* \
|
||||
* allocate an additional request to the cache \
|
||||
*/ \
|
||||
mca_ptl_base_module_t* ptl = ptl_base->ptl; \
|
||||
ompi_list_item_t* item; \
|
||||
OMPI_FREE_LIST_WAIT(&mca_pml_teg.teg_send_requests, item, rc); \
|
||||
sendreq = (mca_pml_teg_send_request_t*)item; \
|
||||
sendreq->req_ptl = ptl; \
|
||||
if(ptl->ptl_request_init(ptl, sendreq) == OMPI_SUCCESS) { \
|
||||
sendreq->req_cached = true; \
|
||||
ptl_base->ptl_cache_alloc++; \
|
||||
} \
|
||||
OMPI_THREAD_UNLOCK(&ptl_base->ptl_cache_lock); \
|
||||
} else { \
|
||||
/* \
|
||||
* take a request from the global pool \
|
||||
*/ \
|
||||
ompi_list_item_t* item; \
|
||||
OMPI_THREAD_UNLOCK(&ptl_base->ptl_cache_lock); \
|
||||
OMPI_FREE_LIST_WAIT(&mca_pml_teg.teg_send_requests, item, rc); \
|
||||
sendreq = (mca_pml_teg_send_request_t*)item; \
|
||||
sendreq->req_ptl = ptl_proc->ptl; \
|
||||
} \
|
||||
\
|
||||
/* otherwise - take the allocation from the global list */ \
|
||||
} else { \
|
||||
ompi_list_item_t* item; \
|
||||
OMPI_FREE_LIST_WAIT(&mca_pml_teg.teg_send_requests, item, rc); \
|
||||
sendreq = (mca_pml_teg_send_request_t*)item; \
|
||||
sendreq->req_ptl = ptl_proc->ptl; \
|
||||
} \
|
||||
/* update request to point to current peer */ \
|
||||
sendreq->req_peer = ptl_proc->ptl_peer; \
|
||||
}
|
||||
|
||||
|
||||
#define MCA_PML_TEG_SEND_REQUEST_INIT( request, \
|
||||
addr, \
|
||||
count, \
|
||||
datatype, \
|
||||
peer, \
|
||||
tag, \
|
||||
comm, \
|
||||
mode, \
|
||||
persistent) \
|
||||
{ \
|
||||
MCA_PML_BASE_SEND_REQUEST_INIT((&request->req_send), \
|
||||
addr, \
|
||||
count, \
|
||||
datatype, \
|
||||
peer, \
|
||||
tag, \
|
||||
comm, \
|
||||
mode, \
|
||||
persistent \
|
||||
); \
|
||||
}
|
||||
|
||||
|
||||
#define MCA_PML_TEG_SEND_REQUEST_RETURN(sendreq) \
|
||||
{ \
|
||||
mca_ptl_base_module_t* ptl = (sendreq)->req_ptl; \
|
||||
mca_pml_base_ptl_t* ptl_base = ptl->ptl_base; \
|
||||
\
|
||||
/* Let the base handle the reference counts */ \
|
||||
MCA_PML_BASE_SEND_REQUEST_RETURN((&sendreq->req_send)); \
|
||||
\
|
||||
/* \
|
||||
* If there is a cache associated with the ptl - first attempt \
|
||||
* to return the send descriptor to the cache. \
|
||||
*/ \
|
||||
if(NULL != ptl->ptl_base && (sendreq)->req_cached) { \
|
||||
OMPI_THREAD_LOCK(&ptl_base->ptl_cache_lock); \
|
||||
ompi_list_prepend(&ptl_base->ptl_cache, \
|
||||
(ompi_list_item_t*)sendreq); \
|
||||
OMPI_THREAD_UNLOCK(&ptl_base->ptl_cache_lock); \
|
||||
} else { \
|
||||
OMPI_FREE_LIST_RETURN( \
|
||||
&mca_pml_teg.teg_send_requests, (ompi_list_item_t*)sendreq); \
|
||||
} \
|
||||
}
|
||||
|
||||
|
||||
/**
|
||||
* Start a send request.
|
||||
*/
|
||||
#define MCA_PML_TEG_SEND_REQUEST_START(req, rc) \
|
||||
{ \
|
||||
mca_ptl_base_module_t* ptl = req->req_ptl; \
|
||||
size_t first_fragment_size = ptl->ptl_first_frag_size; \
|
||||
int flags; \
|
||||
\
|
||||
req->req_lock = 0; \
|
||||
req->req_bytes_sent = 0; \
|
||||
req->req_peer_match.lval = 0; \
|
||||
req->req_peer_addr.lval = 0; \
|
||||
req->req_peer_size = 0; \
|
||||
req->req_offset = 0; \
|
||||
req->req_send.req_base.req_pml_complete = false; \
|
||||
req->req_send.req_base.req_ompi.req_complete = false; \
|
||||
req->req_send.req_base.req_ompi.req_state = OMPI_REQUEST_ACTIVE; \
|
||||
req->req_send.req_base.req_sequence = mca_pml_ptl_comm_send_sequence( \
|
||||
req->req_send.req_base.req_comm->c_pml_comm, req->req_send.req_base.req_peer); \
|
||||
\
|
||||
/* handle buffered send */ \
|
||||
if(req->req_send.req_send_mode == MCA_PML_BASE_SEND_BUFFERED) { \
|
||||
mca_pml_base_bsend_request_start(&req->req_send.req_base.req_ompi); \
|
||||
} \
|
||||
\
|
||||
/* start the first fragment */ \
|
||||
if (first_fragment_size == 0 || \
|
||||
req->req_send.req_bytes_packed <= first_fragment_size) { \
|
||||
first_fragment_size = req->req_send.req_bytes_packed; \
|
||||
flags = (req->req_send.req_send_mode == MCA_PML_BASE_SEND_SYNCHRONOUS) ? \
|
||||
MCA_PTL_FLAGS_ACK : 0; \
|
||||
} else { \
|
||||
/* require match for first fragment of a multi-fragment */ \
|
||||
flags = MCA_PTL_FLAGS_ACK; \
|
||||
} \
|
||||
rc = ptl->ptl_send(ptl, req->req_peer, req, 0, first_fragment_size, \
|
||||
flags); \
|
||||
}
|
||||
|
||||
|
||||
/**
|
||||
* Schedule any data that was not delivered in the first fragment
|
||||
* across the available PTLs.
|
||||
*/
|
||||
int mca_pml_teg_send_request_schedule(mca_ptl_base_send_request_t* req);
|
||||
|
||||
|
||||
/**
|
||||
* Update the request to reflect the number of bytes delivered. If this
|
||||
* was the first fragment - schedule the rest of the data.
|
||||
*/
|
||||
void mca_pml_teg_send_request_progress(
|
||||
struct mca_ptl_base_module_t* ptl,
|
||||
mca_ptl_base_send_request_t* send_request,
|
||||
size_t bytes_sent
|
||||
);
|
||||
|
||||
#if defined(c_plusplus) || defined(__cplusplus)
|
||||
}
|
||||
#endif
|
||||
#endif
|
||||
|
||||
122
src/mca/pml/teg/pml_teg_start.c
Обычный файл
122
src/mca/pml/teg/pml_teg_start.c
Обычный файл
@ -0,0 +1,122 @@
|
||||
/*
|
||||
* Copyright (c) 2004-2005 The Trustees of Indiana University.
|
||||
* All rights reserved.
|
||||
* Copyright (c) 2004-2005 The Trustees of the University of Tennessee.
|
||||
* All rights reserved.
|
||||
* Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
|
||||
* University of Stuttgart. All rights reserved.
|
||||
* Copyright (c) 2004-2005 The Regents of the University of California.
|
||||
* All rights reserved.
|
||||
* $COPYRIGHT$
|
||||
*
|
||||
* Additional copyrights may follow
|
||||
*
|
||||
* $HEADER$
|
||||
*/
|
||||
|
||||
#include "ompi_config.h"
|
||||
|
||||
#include "pml_teg.h"
|
||||
#include "pml_teg_recvreq.h"
|
||||
#include "pml_teg_sendreq.h"
|
||||
|
||||
|
||||
int mca_pml_teg_start(size_t count, ompi_request_t** requests)
|
||||
{
|
||||
int rc;
|
||||
size_t i;
|
||||
for(i=0; i<count; i++) {
|
||||
mca_pml_base_request_t *pml_request = (mca_pml_base_request_t*)requests[i];
|
||||
if(NULL == pml_request)
|
||||
continue;
|
||||
|
||||
/* If the persistent request is currently active - obtain the
|
||||
* request lock and verify the status is incomplete. if the
|
||||
* pml layer has not completed the request - mark the request
|
||||
* as free called - so that it will be freed when the request
|
||||
* completes - and create a new request.
|
||||
*/
|
||||
|
||||
switch(pml_request->req_ompi.req_state) {
|
||||
case OMPI_REQUEST_INACTIVE:
|
||||
if(pml_request->req_pml_complete == true)
|
||||
break;
|
||||
/* otherwise fall through */
|
||||
case OMPI_REQUEST_ACTIVE: {
|
||||
|
||||
ompi_request_t *request;
|
||||
OMPI_THREAD_LOCK(&ompi_request_lock);
|
||||
if (pml_request->req_pml_complete == false) {
|
||||
/* free request after it completes */
|
||||
pml_request->req_free_called = true;
|
||||
} else {
|
||||
/* can reuse the existing request */
|
||||
OMPI_THREAD_UNLOCK(&ompi_request_lock);
|
||||
break;
|
||||
}
|
||||
|
||||
/* allocate a new request */
|
||||
switch(pml_request->req_type) {
|
||||
case MCA_PML_REQUEST_SEND: {
|
||||
mca_pml_base_send_mode_t sendmode =
|
||||
((mca_pml_base_send_request_t*)pml_request)->req_send_mode;
|
||||
rc = mca_pml_teg_isend_init(
|
||||
pml_request->req_addr,
|
||||
pml_request->req_count,
|
||||
pml_request->req_datatype,
|
||||
pml_request->req_peer,
|
||||
pml_request->req_tag,
|
||||
sendmode,
|
||||
pml_request->req_comm,
|
||||
&request);
|
||||
break;
|
||||
}
|
||||
case MCA_PML_REQUEST_RECV:
|
||||
rc = mca_pml_teg_irecv_init(
|
||||
pml_request->req_addr,
|
||||
pml_request->req_count,
|
||||
pml_request->req_datatype,
|
||||
pml_request->req_peer,
|
||||
pml_request->req_tag,
|
||||
pml_request->req_comm,
|
||||
&request);
|
||||
break;
|
||||
default:
|
||||
rc = OMPI_ERR_REQUEST;
|
||||
break;
|
||||
}
|
||||
OMPI_THREAD_UNLOCK(&ompi_request_lock);
|
||||
if(OMPI_SUCCESS != rc)
|
||||
return rc;
|
||||
pml_request = (mca_pml_base_request_t*)request;
|
||||
requests[i] = request;
|
||||
break;
|
||||
}
|
||||
default:
|
||||
return OMPI_ERR_REQUEST;
|
||||
}
|
||||
|
||||
/* start the request */
|
||||
switch(pml_request->req_type) {
|
||||
case MCA_PML_REQUEST_SEND:
|
||||
{
|
||||
mca_ptl_base_send_request_t* sendreq = (mca_ptl_base_send_request_t*)pml_request;
|
||||
MCA_PML_TEG_SEND_REQUEST_START(sendreq, rc);
|
||||
if(rc != OMPI_SUCCESS)
|
||||
return rc;
|
||||
break;
|
||||
}
|
||||
case MCA_PML_REQUEST_RECV:
|
||||
{
|
||||
mca_ptl_base_recv_request_t* recvreq = (mca_ptl_base_recv_request_t*)pml_request;
|
||||
if((rc = mca_pml_teg_recv_request_start(recvreq)) != OMPI_SUCCESS)
|
||||
return rc;
|
||||
break;
|
||||
}
|
||||
default:
|
||||
return OMPI_ERR_REQUEST;
|
||||
}
|
||||
}
|
||||
return OMPI_SUCCESS;
|
||||
}
|
||||
|
||||
@ -81,10 +81,10 @@ static int mca_pml_teg_recv_request_cancel(struct ompi_request_t* request, int c
|
||||
|
||||
static void mca_pml_teg_recv_request_construct(mca_pml_base_recv_request_t* request)
|
||||
{
|
||||
request->req_base.req_type = MCA_PML_REQUEST_RECV;
|
||||
request->req_base.req_ompi.req_fini = mca_pml_teg_recv_request_fini;
|
||||
request->req_base.req_ompi.req_free = mca_pml_teg_recv_request_free;
|
||||
request->req_base.req_ompi.req_cancel = mca_pml_teg_recv_request_cancel;
|
||||
request->req_recv.req_base.req_type = MCA_PML_REQUEST_RECV;
|
||||
request->req_recv.req_base.req_ompi.req_fini = mca_pml_teg_recv_request_fini;
|
||||
request->req_recv.req_base.req_ompi.req_free = mca_pml_teg_recv_request_free;
|
||||
request->req_recv.req_base.req_ompi.req_cancel = mca_pml_teg_recv_request_cancel;
|
||||
}
|
||||
|
||||
static void mca_pml_teg_recv_request_destruct(mca_pml_base_recv_request_t* request)
|
||||
@ -93,7 +93,7 @@ static void mca_pml_teg_recv_request_destruct(mca_pml_base_recv_request_t* reque
|
||||
|
||||
OBJ_CLASS_INSTANCE(
|
||||
mca_pml_teg_recv_request_t,
|
||||
mca_pml_base_recv_request_t,
|
||||
mca_ptl_base_recv_request_t,
|
||||
mca_pml_teg_recv_request_construct,
|
||||
mca_pml_teg_recv_request_destruct);
|
||||
|
||||
|
||||
@ -21,13 +21,13 @@
|
||||
|
||||
#include "pml_teg.h"
|
||||
#include "pml_teg_proc.h"
|
||||
#include "mca/pml/base/pml_base_recvreq.h"
|
||||
#include "mca/ptl/base/ptl_base_recvreq.h"
|
||||
#include "mca/ptl/base/ptl_base_recvfrag.h"
|
||||
|
||||
#if defined(c_plusplus) || defined(__cplusplus)
|
||||
extern "C" {
|
||||
#endif
|
||||
typedef mca_pml_base_recv_request_t mca_pml_teg_recv_request_t;
|
||||
typedef mca_ptl_base_recv_request_t mca_pml_teg_recv_request_t;
|
||||
|
||||
OBJ_CLASS_DECLARATION(mca_pml_teg_recv_request_t);
|
||||
|
||||
|
||||
@ -25,24 +25,28 @@ AM_CPPFLAGS = -I$(top_builddir)/src
|
||||
# Source code files
|
||||
|
||||
headers = \
|
||||
base.h \
|
||||
base.h \
|
||||
ptl_base_comm.h \
|
||||
ptl_base_fragment.h \
|
||||
ptl_base_header.h \
|
||||
ptl_base_match.h \
|
||||
ptl_base_recvfrag.h \
|
||||
ptl_base_sendfrag.h
|
||||
ptl_base_recvreq.h \
|
||||
ptl_base_sendfrag.h \
|
||||
ptl_base_sendreq.h
|
||||
|
||||
libmca_ptl_base_la_SOURCES = \
|
||||
$(headers) \
|
||||
ptl_base_close.c \
|
||||
ptl_base_comm.c \
|
||||
ptl_base_fragment.c \
|
||||
$(headers) \
|
||||
ptl_base_close.c \
|
||||
ptl_base_comm.c \
|
||||
ptl_base_fragment.c \
|
||||
ptl_base_match.c \
|
||||
ptl_base_open.c \
|
||||
ptl_base_recvfrag.c \
|
||||
ptl_base_select.c \
|
||||
ptl_base_sendfrag.c
|
||||
ptl_base_open.c \
|
||||
ptl_base_recvfrag.c \
|
||||
ptl_base_recvreq.c \
|
||||
ptl_base_select.c \
|
||||
ptl_base_sendfrag.c \
|
||||
ptl_base_sendreq.c
|
||||
|
||||
# Conditionally install the header files
|
||||
|
||||
|
||||
@ -50,7 +50,7 @@
|
||||
do { \
|
||||
/* local parameters */ \
|
||||
ompi_list_t* wild_receives = &pml_comm->c_wild_receives; \
|
||||
mca_pml_base_recv_request_t *wild_recv; \
|
||||
mca_ptl_base_recv_request_t *wild_recv; \
|
||||
int frag_tag,recv_tag; \
|
||||
\
|
||||
/* initialization */ \
|
||||
@ -61,14 +61,14 @@ do { \
|
||||
* locking is protecting from having other threads trying to \
|
||||
* change this list. \
|
||||
*/ \
|
||||
for(wild_recv = (mca_pml_base_recv_request_t *) \
|
||||
for(wild_recv = (mca_ptl_base_recv_request_t *) \
|
||||
ompi_list_get_first(wild_receives); \
|
||||
wild_recv != (mca_pml_base_recv_request_t *) \
|
||||
wild_recv != (mca_ptl_base_recv_request_t *) \
|
||||
ompi_list_get_end(wild_receives); \
|
||||
wild_recv = (mca_pml_base_recv_request_t *) \
|
||||
wild_recv = (mca_ptl_base_recv_request_t *) \
|
||||
((ompi_list_item_t *)wild_recv)->ompi_list_next) { \
|
||||
\
|
||||
recv_tag = wild_recv->req_base.req_tag; \
|
||||
recv_tag = wild_recv->req_recv.req_base.req_tag; \
|
||||
if ( \
|
||||
/* exact tag match */ \
|
||||
(frag_tag == recv_tag) || \
|
||||
@ -112,7 +112,7 @@ do { \
|
||||
do { \
|
||||
/* local variables */ \
|
||||
ompi_list_t* specific_receives = (pml_comm->c_specific_receives)+frag_src; \
|
||||
mca_pml_base_recv_request_t *specific_recv; \
|
||||
mca_ptl_base_recv_request_t *specific_recv; \
|
||||
int frag_src,recv_tag,frag_tag; \
|
||||
\
|
||||
/* initialization */ \
|
||||
@ -122,16 +122,16 @@ do { \
|
||||
/* \
|
||||
* Loop over the specific irecvs. \
|
||||
*/ \
|
||||
for(specific_recv = (mca_pml_base_recv_request_t *) \
|
||||
for(specific_recv = (mca_ptl_base_recv_request_t *) \
|
||||
ompi_list_get_first(specific_receives); \
|
||||
specific_recv != (mca_pml_base_recv_request_t *) \
|
||||
specific_recv != (mca_ptl_base_recv_request_t *) \
|
||||
ompi_list_get_end(specific_receives); \
|
||||
specific_recv = (mca_pml_base_recv_request_t *) \
|
||||
specific_recv = (mca_ptl_base_recv_request_t *) \
|
||||
((ompi_list_item_t *)specific_recv)->ompi_list_next) { \
|
||||
/* \
|
||||
* Check for a match \
|
||||
*/ \
|
||||
recv_tag = specific_recv->req_base.req_tag; \
|
||||
recv_tag = specific_recv->req_recv.req_base.req_tag; \
|
||||
if ( (frag_tag == recv_tag) || \
|
||||
( (recv_tag == OMPI_ANY_TAG) && (0 <= frag_tag) ) ) { \
|
||||
\
|
||||
@ -170,7 +170,7 @@ do { \
|
||||
frag_header, pml_comm, return_match) \
|
||||
do { \
|
||||
/* local variables */ \
|
||||
mca_pml_base_recv_request_t *specific_recv, *wild_recv; \
|
||||
mca_ptl_base_recv_request_t *specific_recv, *wild_recv; \
|
||||
mca_ptl_sequence_t wild_recv_seq, specific_recv_seq; \
|
||||
int frag_src,frag_tag, wild_recv_tag, specific_recv_tag; \
|
||||
\
|
||||
@ -182,13 +182,13 @@ do { \
|
||||
* We know that when this is called, both specific and wild irecvs \
|
||||
* have been posted. \
|
||||
*/ \
|
||||
specific_recv = (mca_pml_base_recv_request_t *) \
|
||||
specific_recv = (mca_ptl_base_recv_request_t *) \
|
||||
ompi_list_get_first((pml_comm->c_specific_receives)+frag_src); \
|
||||
wild_recv = (mca_pml_base_recv_request_t *) \
|
||||
wild_recv = (mca_ptl_base_recv_request_t *) \
|
||||
ompi_list_get_first(&(pml_comm->c_wild_receives)); \
|
||||
\
|
||||
specific_recv_seq = specific_recv->req_base.req_sequence; \
|
||||
wild_recv_seq = wild_recv->req_base.req_sequence; \
|
||||
specific_recv_seq = specific_recv->req_recv.req_base.req_sequence; \
|
||||
wild_recv_seq = wild_recv->req_recv.req_base.req_sequence; \
|
||||
\
|
||||
while (true) { \
|
||||
if (wild_recv_seq < specific_recv_seq) { \
|
||||
@ -198,7 +198,7 @@ do { \
|
||||
/* \
|
||||
* try and match \
|
||||
*/ \
|
||||
wild_recv_tag = wild_recv->req_base.req_tag; \
|
||||
wild_recv_tag = wild_recv->req_recv.req_base.req_tag; \
|
||||
if ( (frag_tag == wild_recv_tag) || \
|
||||
( (wild_recv_tag == OMPI_ANY_TAG) && (0 <= frag_tag) ) ) { \
|
||||
/* \
|
||||
@ -215,14 +215,14 @@ do { \
|
||||
/* \
|
||||
* No match, go to the next. \
|
||||
*/ \
|
||||
wild_recv=(mca_pml_base_recv_request_t *) \
|
||||
wild_recv=(mca_ptl_base_recv_request_t *) \
|
||||
((ompi_list_item_t *)wild_recv)->ompi_list_next; \
|
||||
\
|
||||
/* \
|
||||
* If that was the last wild one, just look at the \
|
||||
* rest of the specific ones. \
|
||||
*/ \
|
||||
if (wild_recv == (mca_pml_base_recv_request_t *) \
|
||||
if (wild_recv == (mca_ptl_base_recv_request_t *) \
|
||||
ompi_list_get_end(&(pml_comm->c_wild_receives)) ) \
|
||||
{ \
|
||||
MCA_PTL_BASE_CHECK_SPECIFIC_RECEIVES_FOR_MATCH(frag_header, pml_comm, return_match); \
|
||||
@ -233,13 +233,13 @@ do { \
|
||||
* Get the sequence number for this recv, and go \
|
||||
* back to the top of the loop. \
|
||||
*/ \
|
||||
wild_recv_seq = wild_recv->req_base.req_sequence; \
|
||||
wild_recv_seq = wild_recv->req_recv.req_base.req_sequence; \
|
||||
\
|
||||
} else { \
|
||||
/* \
|
||||
* specific recv is earlier than the wild one. \
|
||||
*/ \
|
||||
specific_recv_tag=specific_recv->req_base.req_tag; \
|
||||
specific_recv_tag=specific_recv->req_recv.req_base.req_tag; \
|
||||
if ( (frag_tag == specific_recv_tag) || \
|
||||
( (specific_recv_tag == OMPI_ANY_TAG) && (0<=frag_tag)) ) \
|
||||
{ \
|
||||
@ -256,14 +256,14 @@ do { \
|
||||
/* \
|
||||
* No match, go on to the next specific irecv. \
|
||||
*/ \
|
||||
specific_recv = (mca_pml_base_recv_request_t *) \
|
||||
specific_recv = (mca_ptl_base_recv_request_t *) \
|
||||
((ompi_list_item_t *)specific_recv)->ompi_list_next; \
|
||||
\
|
||||
/* \
|
||||
* If that was the last specific irecv, process the \
|
||||
* rest of the wild ones. \
|
||||
*/ \
|
||||
if (specific_recv == (mca_pml_base_recv_request_t *) \
|
||||
if (specific_recv == (mca_ptl_base_recv_request_t *) \
|
||||
ompi_list_get_end((pml_comm->c_specific_receives)+frag_src) ) \
|
||||
{ \
|
||||
MCA_PTL_BASE_CHECK_WILD_RECEIVES_FOR_MATCH(frag_header, pml_comm, return_match); \
|
||||
@ -273,7 +273,7 @@ do { \
|
||||
* Get the sequence number for this recv, and go \
|
||||
* back to the top of the loop. \
|
||||
*/ \
|
||||
specific_recv_seq = specific_recv->req_base.req_sequence; \
|
||||
specific_recv_seq = specific_recv->req_recv.req_base.req_sequence; \
|
||||
} \
|
||||
} \
|
||||
} while(0)
|
||||
@ -328,7 +328,7 @@ bool mca_ptl_base_match(
|
||||
uint16_t next_msg_seq_expected, frag_msg_seq;
|
||||
|
||||
ompi_communicator_t *comm_ptr;
|
||||
mca_pml_base_recv_request_t *matched_receive = NULL;
|
||||
mca_ptl_base_recv_request_t *matched_receive = NULL;
|
||||
mca_pml_ptl_comm_t *pml_comm;
|
||||
int frag_src;
|
||||
bool match_made=false;
|
||||
@ -398,12 +398,12 @@ bool mca_ptl_base_match(
|
||||
frag_desc->frag_request=matched_receive;
|
||||
|
||||
/* set lenght of incoming message */
|
||||
matched_receive->req_bytes_packed = frag_header->hdr_msg_length;
|
||||
matched_receive->req_recv.req_bytes_packed = frag_header->hdr_msg_length;
|
||||
|
||||
/*
|
||||
* update delivered sequence number information, if needed.
|
||||
*/
|
||||
if( (matched_receive->req_base.req_type == MCA_PML_REQUEST_PROBE) ) {
|
||||
if( (matched_receive->req_recv.req_base.req_type == MCA_PML_REQUEST_PROBE) ) {
|
||||
/* Match a probe, rollback the next expected sequence number */
|
||||
(pml_comm->c_next_msg_seq[frag_src])--;
|
||||
}
|
||||
@ -461,7 +461,7 @@ static bool mca_ptl_base_check_cantmatch_for_match(ompi_list_t *additional_match
|
||||
int match_found;
|
||||
uint16_t next_msg_seq_expected, frag_seq;
|
||||
mca_ptl_base_recv_frag_t *frag_desc;
|
||||
mca_pml_base_recv_request_t *matched_receive = NULL;
|
||||
mca_ptl_base_recv_request_t *matched_receive = NULL;
|
||||
bool match_made = false;
|
||||
|
||||
/*
|
||||
@ -604,7 +604,7 @@ bool mca_ptl_base_match_in_order_network_delivery(
|
||||
{
|
||||
/* local variables */
|
||||
ompi_communicator_t *comm_ptr;
|
||||
mca_pml_base_recv_request_t *matched_receive = NULL;
|
||||
mca_ptl_base_recv_request_t *matched_receive = NULL;
|
||||
mca_pml_ptl_comm_t *pml_comm;
|
||||
int frag_src;
|
||||
|
||||
@ -658,7 +658,7 @@ bool mca_ptl_base_match_in_order_network_delivery(
|
||||
frag_desc->frag_request=matched_receive;
|
||||
|
||||
/* set lenght of incoming message */
|
||||
matched_receive->req_bytes_packed=frag_header->hdr_msg_length;
|
||||
matched_receive->req_recv.req_bytes_packed=frag_header->hdr_msg_length;
|
||||
|
||||
} else {
|
||||
/* if no match found, place on unexpected queue */
|
||||
|
||||
@ -20,7 +20,7 @@
|
||||
#define MCA_PTL_BASE_RECVFRAG_H
|
||||
|
||||
#include "mca/ptl/ptl.h"
|
||||
#include "mca/pml/base/pml_base_recvreq.h"
|
||||
#include "mca/ptl/base/ptl_base_recvreq.h"
|
||||
#include "mca/ptl/base/ptl_base_fragment.h"
|
||||
#include "mca/ptl/base/ptl_base_match.h"
|
||||
|
||||
@ -34,7 +34,7 @@ OMPI_DECLSPEC extern ompi_class_t mca_ptl_base_recv_frag_t_class;
|
||||
*/
|
||||
struct mca_ptl_base_recv_frag_t {
|
||||
mca_ptl_base_frag_t frag_base; /**< base fragment descriptor */
|
||||
mca_pml_base_recv_request_t *frag_request; /**< matched posted receive */
|
||||
mca_ptl_base_recv_request_t *frag_request; /**< matched posted receive */
|
||||
bool frag_is_buffered; /**< does fragment need to be unpacked into users buffer */
|
||||
};
|
||||
typedef struct mca_ptl_base_recv_frag_t mca_ptl_base_recv_frag_t;
|
||||
|
||||
10
src/mca/ptl/base/ptl_base_recvreq.c
Обычный файл
10
src/mca/ptl/base/ptl_base_recvreq.c
Обычный файл
@ -0,0 +1,10 @@
|
||||
#include "ptl_base_recvreq.h"
|
||||
|
||||
|
||||
OBJ_CLASS_INSTANCE(
|
||||
mca_ptl_base_recv_request_t,
|
||||
mca_pml_base_recv_request_t,
|
||||
NULL,
|
||||
NULL
|
||||
);
|
||||
|
||||
47
src/mca/ptl/base/ptl_base_recvreq.h
Обычный файл
47
src/mca/ptl/base/ptl_base_recvreq.h
Обычный файл
@ -0,0 +1,47 @@
|
||||
/*
|
||||
* Copyright (c) 2004-2005 The Trustees of Indiana University.
|
||||
* All rights reserved.
|
||||
* Copyright (c) 2004-2005 The Trustees of the University of Tennessee.
|
||||
* All rights reserved.
|
||||
* Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
|
||||
* University of Stuttgart. All rights reserved.
|
||||
* Copyright (c) 2004-2005 The Regents of the University of California.
|
||||
* All rights reserved.
|
||||
* $COPYRIGHT$
|
||||
*
|
||||
* Additional copyrights may follow
|
||||
*
|
||||
* $HEADER$
|
||||
*/
|
||||
/**
|
||||
* @file
|
||||
*/
|
||||
#ifndef MCA_PTL_BASE_RECV_REQUEST_H
|
||||
#define MCA_PTL_BASE_RECV_REQUEST_H
|
||||
|
||||
#include "mca/ptl/ptl.h"
|
||||
#include "mca/pml/base/pml_base_recvreq.h"
|
||||
|
||||
#if defined(c_plusplus) || defined(__cplusplus)
|
||||
extern "C" {
|
||||
#endif
|
||||
|
||||
/**
|
||||
* Base type for recv requests
|
||||
*/
|
||||
struct mca_ptl_base_recv_request_t {
|
||||
mca_pml_base_recv_request_t req_recv;
|
||||
size_t req_bytes_received; /**< number of bytes received from network */
|
||||
size_t req_bytes_delivered; /**< number of bytes delivered to user */
|
||||
};
|
||||
typedef struct mca_ptl_base_recv_request_t mca_ptl_base_recv_request_t;
|
||||
|
||||
|
||||
OMPI_DECLSPEC OBJ_CLASS_DECLARATION(mca_ptl_base_recv_request_t);
|
||||
|
||||
|
||||
#if defined(c_plusplus) || defined(__cplusplus)
|
||||
}
|
||||
#endif
|
||||
#endif
|
||||
|
||||
@ -32,7 +32,7 @@ OMPI_DECLSPEC extern ompi_class_t mca_ptl_base_send_frag_t_class;
|
||||
*/
|
||||
struct mca_ptl_base_send_frag_t {
|
||||
mca_ptl_base_frag_t frag_base; /**< base fragment descriptor */
|
||||
struct mca_pml_base_send_request_t *frag_request; /**< pointer to send request */
|
||||
struct mca_ptl_base_send_request_t *frag_request; /**< pointer to send request */
|
||||
};
|
||||
typedef struct mca_ptl_base_send_frag_t mca_ptl_base_send_frag_t;
|
||||
|
||||
|
||||
10
src/mca/ptl/base/ptl_base_sendreq.c
Обычный файл
10
src/mca/ptl/base/ptl_base_sendreq.c
Обычный файл
@ -0,0 +1,10 @@
|
||||
#include "ptl_base_sendreq.h"
|
||||
|
||||
|
||||
OBJ_CLASS_INSTANCE(
|
||||
mca_ptl_base_send_request_t,
|
||||
mca_pml_base_send_request_t,
|
||||
NULL,
|
||||
NULL
|
||||
);
|
||||
|
||||
69
src/mca/ptl/base/ptl_base_sendreq.h
Обычный файл
69
src/mca/ptl/base/ptl_base_sendreq.h
Обычный файл
@ -0,0 +1,69 @@
|
||||
/*
|
||||
* Copyright (c) 2004-2005 The Trustees of Indiana University.
|
||||
* All rights reserved.
|
||||
* Copyright (c) 2004-2005 The Trustees of the University of Tennessee.
|
||||
* All rights reserved.
|
||||
* Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
|
||||
* University of Stuttgart. All rights reserved.
|
||||
* Copyright (c) 2004-2005 The Regents of the University of California.
|
||||
* All rights reserved.
|
||||
* $COPYRIGHT$
|
||||
*
|
||||
* Additional copyrights may follow
|
||||
*
|
||||
* $HEADER$
|
||||
*/
|
||||
/**
|
||||
* @file
|
||||
*/
|
||||
#ifndef MCA_PTL_BASE_SEND_REQUEST_H
|
||||
#define MCA_PTL_BASE_SEND_REQUEST_H
|
||||
|
||||
#include "mca/ptl/ptl.h"
|
||||
#include "mca/pml/base/pml_base_sendreq.h"
|
||||
|
||||
#if defined(c_plusplus) || defined(__cplusplus)
|
||||
extern "C" {
|
||||
#endif
|
||||
|
||||
/**
|
||||
* Base type for send requests
|
||||
*/
|
||||
struct mca_ptl_base_send_request_t {
|
||||
mca_pml_base_send_request_t req_send;
|
||||
size_t req_offset; /**< number of bytes that have been scheduled */
|
||||
size_t req_bytes_sent; /**< number of bytes that have been sent */
|
||||
ompi_ptr_t req_peer_match; /**< matched receive at peer */
|
||||
ompi_ptr_t req_peer_addr; /**< peers remote buffer address */
|
||||
uint64_t req_peer_size; /**< size of peers remote buffer */
|
||||
bool req_cached; /**< has this request been obtained from the ptls cache */
|
||||
volatile int32_t req_lock; /**< lock used by the scheduler */
|
||||
struct mca_ptl_base_module_t* req_ptl; /**< ptl allocated for first fragment */
|
||||
struct mca_ptl_base_peer_t* req_peer; /**< peer associated w/ this ptl */
|
||||
};
|
||||
typedef struct mca_ptl_base_send_request_t mca_ptl_base_send_request_t;
|
||||
|
||||
|
||||
OMPI_DECLSPEC OBJ_CLASS_DECLARATION(mca_ptl_base_send_request_t);
|
||||
|
||||
|
||||
/**
|
||||
* Atomically increase the request offset.
|
||||
*
|
||||
* @param request (IN) Send request.
|
||||
* @param offset (IN) Increment.
|
||||
*/
|
||||
|
||||
static inline void mca_ptl_base_send_request_offset(
|
||||
mca_ptl_base_send_request_t* request,
|
||||
size_t offset)
|
||||
{
|
||||
OMPI_THREAD_ADD32((int32_t*)(&request->req_offset), offset);
|
||||
}
|
||||
|
||||
|
||||
#if defined(c_plusplus) || defined(__cplusplus)
|
||||
}
|
||||
#endif
|
||||
#endif
|
||||
|
||||
@ -89,7 +89,7 @@ static void *mca_ptl_mx_alloc(size_t *size)
|
||||
*
|
||||
*/
|
||||
|
||||
int mca_ptl_mx_request_init(struct mca_ptl_base_module_t* ptl, mca_pml_base_send_request_t* request)
|
||||
int mca_ptl_mx_request_init(struct mca_ptl_base_module_t* ptl, mca_ptl_base_send_request_t* request)
|
||||
{
|
||||
OBJ_CONSTRUCT(request+1, mca_ptl_mx_send_frag_t);
|
||||
return OMPI_SUCCESS;
|
||||
@ -109,7 +109,7 @@ int mca_ptl_mx_request_init(struct mca_ptl_base_module_t* ptl, mca_pml_base_send
|
||||
* descriptor by the PTL.
|
||||
*/
|
||||
|
||||
void mca_ptl_mx_request_fini(struct mca_ptl_base_module_t* ptl, mca_pml_base_send_request_t* request)
|
||||
void mca_ptl_mx_request_fini(struct mca_ptl_base_module_t* ptl, mca_ptl_base_send_request_t* request)
|
||||
{
|
||||
OBJ_DESTRUCT(request+1);
|
||||
}
|
||||
@ -142,7 +142,7 @@ void mca_ptl_mx_request_fini(struct mca_ptl_base_module_t* ptl, mca_pml_base_sen
|
||||
int mca_ptl_mx_send(
|
||||
struct mca_ptl_base_module_t* ptl,
|
||||
struct mca_ptl_base_peer_t* ptl_peer,
|
||||
struct mca_pml_base_send_request_t* sendreq,
|
||||
struct mca_ptl_base_send_request_t* sendreq,
|
||||
size_t offset,
|
||||
size_t size,
|
||||
int flags)
|
||||
@ -177,13 +177,13 @@ int mca_ptl_mx_send(
|
||||
int rc;
|
||||
|
||||
convertor = &sendfrag->frag_send.frag_base.frag_convertor;
|
||||
ompi_convertor_copy(&sendreq->req_convertor, convertor);
|
||||
ompi_convertor_copy(&sendreq->req_send.req_convertor, convertor);
|
||||
ompi_convertor_init_for_send(
|
||||
convertor,
|
||||
0,
|
||||
sendreq->req_datatype,
|
||||
sendreq->req_count,
|
||||
sendreq->req_addr,
|
||||
sendreq->req_send.req_datatype,
|
||||
sendreq->req_send.req_count,
|
||||
sendreq->req_send.req_addr,
|
||||
offset,
|
||||
mca_ptl_mx_alloc );
|
||||
|
||||
@ -218,12 +218,12 @@ int mca_ptl_mx_send(
|
||||
/* first fragment - need to try and match at the receiver */
|
||||
if(offset == 0) {
|
||||
hdr->hdr_common.hdr_flags = flags;
|
||||
hdr->hdr_match.hdr_contextid = sendreq->req_base.req_comm->c_contextid;
|
||||
hdr->hdr_match.hdr_src = sendreq->req_base.req_comm->c_my_rank;
|
||||
hdr->hdr_match.hdr_dst = sendreq->req_base.req_peer;
|
||||
hdr->hdr_match.hdr_tag = sendreq->req_base.req_tag;
|
||||
hdr->hdr_match.hdr_msg_length = sendreq->req_bytes_packed;
|
||||
hdr->hdr_match.hdr_msg_seq = sendreq->req_base.req_sequence;
|
||||
hdr->hdr_match.hdr_contextid = sendreq->req_send.req_base.req_comm->c_contextid;
|
||||
hdr->hdr_match.hdr_src = sendreq->req_send.req_base.req_comm->c_my_rank;
|
||||
hdr->hdr_match.hdr_dst = sendreq->req_send.req_base.req_peer;
|
||||
hdr->hdr_match.hdr_tag = sendreq->req_send.req_base.req_tag;
|
||||
hdr->hdr_match.hdr_msg_length = sendreq->req_send.req_bytes_packed;
|
||||
hdr->hdr_match.hdr_msg_seq = sendreq->req_send.req_base.req_sequence;
|
||||
|
||||
/* for the first 32K - send header for matching + data */
|
||||
segments = sendfrag->frag_segments;
|
||||
@ -292,7 +292,7 @@ int mca_ptl_mx_send(
|
||||
/* must update the offset after actual fragment size is determined
|
||||
* before attempting to send the fragment
|
||||
*/
|
||||
mca_pml_base_send_request_offset(sendreq,
|
||||
mca_ptl_base_send_request_offset(sendreq,
|
||||
sendfrag->frag_send.frag_base.frag_size);
|
||||
|
||||
/* start the fragment */
|
||||
@ -329,7 +329,7 @@ int mca_ptl_mx_send(
|
||||
int mca_ptl_mx_send_continue(
|
||||
struct mca_ptl_base_module_t* ptl,
|
||||
struct mca_ptl_base_peer_t* ptl_peer,
|
||||
struct mca_pml_base_send_request_t* sendreq,
|
||||
struct mca_ptl_base_send_request_t* sendreq,
|
||||
size_t offset,
|
||||
size_t size,
|
||||
int flags)
|
||||
@ -353,13 +353,13 @@ int mca_ptl_mx_send_continue(
|
||||
|
||||
/* initialize convertor */
|
||||
convertor = &sendfrag->frag_send.frag_base.frag_convertor;
|
||||
ompi_convertor_copy(&sendreq->req_convertor, convertor);
|
||||
ompi_convertor_copy(&sendreq->req_send.req_convertor, convertor);
|
||||
ompi_convertor_init_for_send(
|
||||
convertor,
|
||||
0,
|
||||
sendreq->req_datatype,
|
||||
sendreq->req_count,
|
||||
sendreq->req_addr,
|
||||
sendreq->req_send.req_datatype,
|
||||
sendreq->req_send.req_count,
|
||||
sendreq->req_send.req_addr,
|
||||
offset,
|
||||
mca_ptl_mx_alloc );
|
||||
|
||||
@ -400,7 +400,7 @@ int mca_ptl_mx_send_continue(
|
||||
/* must update the offset after actual fragment size is determined
|
||||
* before attempting to send the fragment
|
||||
*/
|
||||
mca_pml_base_send_request_offset(sendreq, size);
|
||||
mca_ptl_base_send_request_offset(sendreq, size);
|
||||
|
||||
/* start the fragment */
|
||||
match.sval.uval = sendreq->req_peer_match.ival;
|
||||
@ -446,7 +446,7 @@ void mca_ptl_mx_matched(
|
||||
mca_ptl_base_recv_frag_t* frag)
|
||||
{
|
||||
mca_ptl_base_header_t* hdr = &frag->frag_base.frag_header;
|
||||
mca_pml_base_recv_request_t* request = frag->frag_request;
|
||||
mca_ptl_base_recv_request_t* request = frag->frag_request;
|
||||
mca_ptl_mx_module_t* mx_ptl = (mca_ptl_mx_module_t*)ptl;
|
||||
mca_ptl_mx_recv_frag_t* mx_frag = (mca_ptl_mx_recv_frag_t*)frag;
|
||||
unsigned int bytes_delivered = mx_frag->frag_size;
|
||||
@ -493,8 +493,8 @@ void mca_ptl_mx_matched(
|
||||
struct iovec iov;
|
||||
unsigned int iov_count = 1;
|
||||
int free_after = 0;
|
||||
ompi_proc_t *proc = ompi_comm_peer_lookup(request->req_base.req_comm,
|
||||
request->req_base.req_ompi.req_status.MPI_SOURCE);
|
||||
ompi_proc_t *proc = ompi_comm_peer_lookup(request->req_recv.req_base.req_comm,
|
||||
request->req_recv.req_base.req_ompi.req_status.MPI_SOURCE);
|
||||
ompi_convertor_t* convertor = &frag->frag_base.frag_convertor;
|
||||
|
||||
/* initialize receive convertor */
|
||||
@ -502,9 +502,9 @@ void mca_ptl_mx_matched(
|
||||
ompi_convertor_init_for_recv(
|
||||
convertor, /* convertor */
|
||||
0, /* flags */
|
||||
request->req_base.req_datatype, /* datatype */
|
||||
request->req_base.req_count, /* count elements */
|
||||
request->req_base.req_addr, /* users buffer */
|
||||
request->req_recv.req_base.req_datatype, /* datatype */
|
||||
request->req_recv.req_base.req_count, /* count elements */
|
||||
request->req_recv.req_base.req_addr, /* users buffer */
|
||||
0, /* offset in bytes into packed buffer */
|
||||
NULL ); /* not allocating memory */
|
||||
/*ompi_convertor_get_packed_size(convertor, &request->req_bytes_packed); */
|
||||
|
||||
@ -249,7 +249,7 @@ extern int mca_ptl_mx_del_procs(
|
||||
|
||||
extern int mca_ptl_mx_request_init(
|
||||
struct mca_ptl_base_module_t* ptl,
|
||||
struct mca_pml_base_send_request_t*
|
||||
struct mca_ptl_base_send_request_t*
|
||||
);
|
||||
|
||||
|
||||
@ -268,7 +268,7 @@ extern int mca_ptl_mx_request_init(
|
||||
|
||||
extern void mca_ptl_mx_request_fini(
|
||||
struct mca_ptl_base_module_t* ptl,
|
||||
struct mca_pml_base_send_request_t*
|
||||
struct mca_ptl_base_send_request_t*
|
||||
);
|
||||
|
||||
|
||||
@ -325,7 +325,7 @@ extern void mca_ptl_mx_matched(
|
||||
extern int mca_ptl_mx_send(
|
||||
struct mca_ptl_base_module_t* ptl,
|
||||
struct mca_ptl_base_peer_t* ptl_peer,
|
||||
struct mca_pml_base_send_request_t*,
|
||||
struct mca_ptl_base_send_request_t*,
|
||||
size_t offset,
|
||||
size_t size,
|
||||
int flags
|
||||
@ -348,7 +348,7 @@ extern int mca_ptl_mx_send(
|
||||
extern int mca_ptl_mx_send_continue(
|
||||
struct mca_ptl_base_module_t* ptl,
|
||||
struct mca_ptl_base_peer_t* ptl_peer,
|
||||
struct mca_pml_base_send_request_t*,
|
||||
struct mca_ptl_base_send_request_t*,
|
||||
size_t offset,
|
||||
size_t size,
|
||||
int flags
|
||||
|
||||
@ -178,7 +178,7 @@ static void* mca_ptl_mx_thread(ompi_object_t *arg)
|
||||
static void mca_ptl_mx_match(void* context, uint64_t match_value, int size)
|
||||
{
|
||||
mca_ptl_mx_module_t* ptl = (mca_ptl_mx_module_t*)context;
|
||||
mca_pml_base_recv_request_t* request;
|
||||
mca_ptl_base_recv_request_t* request;
|
||||
mca_ptl_mx_recv_frag_t *frag;
|
||||
mx_return_t mx_return;
|
||||
ompi_ptr_t match;
|
||||
@ -193,10 +193,10 @@ static void mca_ptl_mx_match(void* context, uint64_t match_value, int size)
|
||||
|
||||
/* otherwise extract request pointer and offset */
|
||||
match.lval = match_value;
|
||||
request = (mca_pml_base_recv_request_t*)match.sval.uval;
|
||||
request = (mca_ptl_base_recv_request_t*)match.sval.uval;
|
||||
offset = match.sval.lval;
|
||||
proc = ompi_comm_peer_lookup(request->req_base.req_comm,
|
||||
request->req_base.req_ompi.req_status.MPI_SOURCE);
|
||||
request->req_recv.req_base.req_ompi.req_status.MPI_SOURCE);
|
||||
|
||||
/* allocate a fragment for receive */
|
||||
MCA_PTL_MX_RECV_FRAG_ALLOC(frag, rc);
|
||||
@ -220,15 +220,15 @@ static void mca_ptl_mx_match(void* context, uint64_t match_value, int size)
|
||||
ompi_convertor_init_for_recv(
|
||||
convertor,
|
||||
0, /* flags */
|
||||
request->req_base.req_datatype, /* datatype */
|
||||
request->req_base.req_count, /* count elements */
|
||||
request->req_base.req_addr, /* users buffer */
|
||||
request->req_recv.req_base.req_datatype, /* datatype */
|
||||
request->req_recv.req_base.req_count, /* count elements */
|
||||
request->req_recv.req_base.req_addr, /* users buffer */
|
||||
offset, /* offset in bytes into packed buffer */
|
||||
mca_ptl_mx_mem_alloc ); /* not allocating memory */
|
||||
|
||||
/* non-contiguous - allocate buffer for receive */
|
||||
if( 1 == ompi_convertor_need_buffers( convertor ) ||
|
||||
request->req_bytes_packed < offset + frag->frag_size ) {
|
||||
request->req_recv.req_bytes_packed < offset + frag->frag_size ) {
|
||||
|
||||
/* TODO - use a fixed fragment size for non-contigous and convert
|
||||
* this to a free-list of buffers.
|
||||
@ -242,14 +242,14 @@ static void mca_ptl_mx_match(void* context, uint64_t match_value, int size)
|
||||
}
|
||||
|
||||
/* check for sending more than receiving */
|
||||
if( offset > request->req_bytes_packed ) {
|
||||
if( offset > request->req_recv.req_bytes_packed ) {
|
||||
frag->frag_recv.frag_base.frag_size = 0;
|
||||
} else if (offset + frag->frag_size > request->req_bytes_packed ) {
|
||||
frag->frag_recv.frag_base.frag_size = request->req_bytes_packed - offset;
|
||||
} else if (offset + frag->frag_size > request->req_recv.req_bytes_packed ) {
|
||||
frag->frag_recv.frag_base.frag_size = request->req_recv.req_bytes_packed - offset;
|
||||
}
|
||||
/* calculate offset into users buffer */
|
||||
} else {
|
||||
frag->frag_recv.frag_base.frag_addr = ((unsigned char*)request->req_base.req_addr) + offset;
|
||||
frag->frag_recv.frag_base.frag_addr = ((unsigned char*)request->req_recv.req_base.req_addr) + offset;
|
||||
frag->frag_recv.frag_is_buffered = false;
|
||||
}
|
||||
|
||||
|
||||
@ -110,7 +110,7 @@ do {
|
||||
case MCA_PTL_HDR_TYPE_ACK: \
|
||||
{ \
|
||||
mca_ptl_mx_send_frag_t* sendfrag; \
|
||||
mca_pml_base_send_request_t* sendreq; \
|
||||
mca_ptl_base_send_request_t* sendreq; \
|
||||
sendfrag = (mca_ptl_mx_send_frag_t*) \
|
||||
hdr->hdr_ack.hdr_src_ptr.pval; \
|
||||
sendreq = sendfrag->frag_send.frag_request; \
|
||||
|
||||
@ -21,7 +21,7 @@
|
||||
|
||||
#include "ompi_config.h"
|
||||
#include "include/sys/atomic.h"
|
||||
#include "mca/pml/base/pml_base_sendreq.h"
|
||||
#include "mca/ptl/base/ptl_base_sendreq.h"
|
||||
#include "mca/ptl/base/ptl_base_sendfrag.h"
|
||||
#include "ptl_mx.h"
|
||||
#include "ptl_mx_peer.h"
|
||||
@ -88,7 +88,7 @@ OBJ_CLASS_DECLARATION(mca_ptl_mx_send_frag_t);
|
||||
|
||||
#define MCA_PTL_MX_SEND_FRAG_PROGRESS(frag) \
|
||||
do { \
|
||||
mca_pml_base_send_request_t* request = frag->frag_send.frag_request; \
|
||||
mca_ptl_base_send_request_t* request = frag->frag_send.frag_request; \
|
||||
bool frag_ack; \
|
||||
\
|
||||
/* if this is an ack - simply return to pool */ \
|
||||
|
||||
@ -105,7 +105,7 @@
|
||||
* PTLs.
|
||||
*
|
||||
* The PML is responsible for managing the state (allocation, initialization,
|
||||
* and release) of send request descriptors (mca_pml_base_send_request_t).
|
||||
* and release) of send request descriptors (mca_ptl_base_send_request_t).
|
||||
* However, to minimize the latency associated with allocating resources to
|
||||
* a request, the PML provides the capability to cache send requests
|
||||
* descriptors on a per-PTL basis. Each PTL exports two variables
|
||||
@ -117,7 +117,7 @@
|
||||
* is grown by the PML up to the specified limit. The PTL variable,
|
||||
* ptl_cache_bytes, can be used to specify that additional memory should be
|
||||
* allocated by the PML in one contigous block along with the base send request
|
||||
* (mca_pml_base_send_request_t) for use by the PTL. The PTLs ptl_request_init()
|
||||
* (mca_ptl_base_send_request_t) for use by the PTL. The PTLs ptl_request_init()
|
||||
* method is then called to initialize this additional memory and associating
|
||||
* any PTL specific resources with the request. If a request is removed from
|
||||
* the cache, the ptl_request_fini() method will be called to allow the PTL
|
||||
@ -129,7 +129,7 @@
|
||||
* number of bytes. However, this may not be possible due to resource
|
||||
* contraints or datatype alignment/offset. The PTL is responsible for
|
||||
* updating the number of bytes actually fragmented and queued for delivery
|
||||
* on the send request (mca_pml_base_send_request.req_offset) to reflect
|
||||
* on the send request (mca_ptl_base_send_request.req_offset) to reflect
|
||||
* the current offset into the send buffer.
|
||||
*
|
||||
* If the request is larger than ptl_first_frag_size, the remainder of
|
||||
@ -258,8 +258,8 @@
|
||||
struct mca_ptl_base_module_t;
|
||||
struct mca_ptl_base_peer_t;
|
||||
struct mca_ptl_base_fragment_t;
|
||||
struct mca_pml_base_recv_request_t;
|
||||
struct mca_pml_base_send_request_t;
|
||||
struct mca_ptl_base_recv_request_t;
|
||||
struct mca_ptl_base_send_request_t;
|
||||
struct mca_ptl_base_recv_frag_t;
|
||||
struct mca_ptl_base_send_frag_t;
|
||||
struct mca_ptl_base_match_header_t;
|
||||
@ -458,7 +458,7 @@ typedef int (*mca_ptl_base_module_del_procs_fn_t)(
|
||||
*/
|
||||
typedef int (*mca_ptl_base_module_request_init_fn_t)(
|
||||
struct mca_ptl_base_module_t* ptl,
|
||||
struct mca_pml_base_send_request_t* request
|
||||
struct mca_ptl_base_send_request_t* request
|
||||
);
|
||||
|
||||
|
||||
@ -477,7 +477,7 @@ typedef int (*mca_ptl_base_module_request_init_fn_t)(
|
||||
|
||||
typedef void (*mca_ptl_base_module_request_fini_fn_t)(
|
||||
struct mca_ptl_base_module_t* ptl,
|
||||
struct mca_pml_base_send_request_t* request
|
||||
struct mca_ptl_base_send_request_t* request
|
||||
);
|
||||
|
||||
/**
|
||||
@ -506,7 +506,7 @@ typedef void (*mca_ptl_base_module_request_fini_fn_t)(
|
||||
typedef int (*mca_ptl_base_module_send_fn_t)(
|
||||
struct mca_ptl_base_module_t* ptl,
|
||||
struct mca_ptl_base_peer_t* ptl_base_peer,
|
||||
struct mca_pml_base_send_request_t* request,
|
||||
struct mca_ptl_base_send_request_t* request,
|
||||
size_t offset,
|
||||
size_t size,
|
||||
int flags
|
||||
@ -541,7 +541,7 @@ typedef int (*mca_ptl_base_module_send_fn_t)(
|
||||
typedef int (*mca_ptl_base_module_put_fn_t)(
|
||||
struct mca_ptl_base_module_t* ptl,
|
||||
struct mca_ptl_base_peer_t* ptl_base_peer,
|
||||
struct mca_pml_base_send_request_t* request,
|
||||
struct mca_ptl_base_send_request_t* request,
|
||||
size_t offset,
|
||||
size_t size,
|
||||
int flags
|
||||
@ -566,7 +566,7 @@ typedef int (*mca_ptl_base_module_put_fn_t)(
|
||||
typedef int (*mca_ptl_base_module_get_fn_t)(
|
||||
struct mca_ptl_base_module_t* ptl,
|
||||
struct mca_ptl_base_peer_t* ptl_base_peer,
|
||||
struct mca_pml_base_recv_request_t* request,
|
||||
struct mca_ptl_base_recv_request_t* request,
|
||||
size_t offset,
|
||||
size_t size,
|
||||
int flags
|
||||
@ -637,7 +637,7 @@ typedef void (*mca_ptl_base_module_matched_fn_t)(
|
||||
*/
|
||||
typedef void (*mca_ptl_base_module_recv_progress_fn_t)(
|
||||
struct mca_ptl_base_module_t* ptl,
|
||||
struct mca_pml_base_recv_request_t* recv_request,
|
||||
struct mca_ptl_base_recv_request_t* recv_request,
|
||||
size_t bytes_received,
|
||||
size_t bytes_delivered
|
||||
);
|
||||
@ -657,7 +657,7 @@ typedef void (*mca_ptl_base_module_recv_progress_fn_t)(
|
||||
|
||||
typedef void (*mca_ptl_base_module_send_progress_fn_t)(
|
||||
struct mca_ptl_base_module_t* ptl,
|
||||
struct mca_pml_base_send_request_t* send_request,
|
||||
struct mca_ptl_base_send_request_t* send_request,
|
||||
size_t bytes_sent
|
||||
);
|
||||
|
||||
|
||||
@ -101,13 +101,13 @@ int mca_ptl_self_finalize(struct mca_ptl_base_module_t* ptl)
|
||||
return OMPI_SUCCESS;
|
||||
}
|
||||
|
||||
int mca_ptl_self_request_init(struct mca_ptl_base_module_t* ptl, mca_pml_base_send_request_t* request)
|
||||
int mca_ptl_self_request_init(struct mca_ptl_base_module_t* ptl, mca_ptl_base_send_request_t* request)
|
||||
{
|
||||
OBJ_CONSTRUCT(request+1, mca_ptl_base_recv_frag_t);
|
||||
return OMPI_SUCCESS;
|
||||
}
|
||||
|
||||
void mca_ptl_self_request_fini(struct mca_ptl_base_module_t* ptl, mca_pml_base_send_request_t* request)
|
||||
void mca_ptl_self_request_fini(struct mca_ptl_base_module_t* ptl, mca_ptl_base_send_request_t* request)
|
||||
{
|
||||
OBJ_DESTRUCT(request+1);
|
||||
}
|
||||
@ -122,7 +122,7 @@ void mca_ptl_self_request_fini(struct mca_ptl_base_module_t* ptl, mca_pml_base_s
|
||||
int mca_ptl_self_send(
|
||||
struct mca_ptl_base_module_t* ptl,
|
||||
struct mca_ptl_base_peer_t* ptl_base_peer,
|
||||
struct mca_pml_base_send_request_t* request,
|
||||
struct mca_ptl_base_send_request_t* request,
|
||||
size_t offset,
|
||||
size_t size,
|
||||
int flags )
|
||||
@ -133,16 +133,16 @@ int mca_ptl_self_send(
|
||||
|
||||
hdr->hdr_common.hdr_type = MCA_PTL_HDR_TYPE_MATCH;
|
||||
hdr->hdr_common.hdr_flags = flags;
|
||||
hdr->hdr_match.hdr_contextid = request->req_base.req_comm->c_contextid;
|
||||
hdr->hdr_match.hdr_src = request->req_base.req_comm->c_my_rank;
|
||||
hdr->hdr_match.hdr_dst = request->req_base.req_peer;
|
||||
hdr->hdr_match.hdr_tag = request->req_base.req_tag;
|
||||
hdr->hdr_match.hdr_msg_length = request->req_bytes_packed;
|
||||
hdr->hdr_match.hdr_msg_seq = request->req_base.req_sequence;
|
||||
hdr->hdr_match.hdr_contextid = request->req_send.req_base.req_comm->c_contextid;
|
||||
hdr->hdr_match.hdr_src = request->req_send.req_base.req_comm->c_my_rank;
|
||||
hdr->hdr_match.hdr_dst = request->req_send.req_base.req_peer;
|
||||
hdr->hdr_match.hdr_tag = request->req_send.req_base.req_tag;
|
||||
hdr->hdr_match.hdr_msg_length = request->req_send.req_bytes_packed;
|
||||
hdr->hdr_match.hdr_msg_seq = request->req_send.req_base.req_sequence;
|
||||
hdr->hdr_rndv.hdr_src_ptr.lval = 0;
|
||||
hdr->hdr_rndv.hdr_src_ptr.pval = request;
|
||||
req->req_frag.frag_base.frag_peer = ptl_base_peer;
|
||||
req->req_frag.frag_base.frag_size = request->req_bytes_packed;
|
||||
req->req_frag.frag_base.frag_size = request->req_send.req_bytes_packed;
|
||||
req->req_frag.frag_base.frag_owner = &mca_ptl_self_module;
|
||||
req->req_frag.frag_request = NULL;
|
||||
req->req_frag.frag_is_buffered = 0;
|
||||
@ -169,10 +169,10 @@ void mca_ptl_self_matched( mca_ptl_base_module_t* ptl,
|
||||
{
|
||||
mca_ptl_self_send_request_t* sendreq = (mca_ptl_self_send_request_t*)
|
||||
frag->frag_base.frag_header.hdr_rndv.hdr_src_ptr.pval;
|
||||
mca_pml_base_recv_request_t* recvreq = frag->frag_request;
|
||||
mca_ptl_base_recv_request_t* recvreq = frag->frag_request;
|
||||
|
||||
if( (recvreq->req_base.req_count != 0) &&
|
||||
(sendreq->req_send.req_base.req_count != 0) ) {
|
||||
if( (recvreq->req_recv.req_base.req_count != 0) &&
|
||||
(sendreq->req_ptl.req_send.req_base.req_count != 0) ) {
|
||||
/* Did you have the same datatype or not ? If yes we can use an optimized version
|
||||
* for the copy function, if not we have to use a temporary buffer to pack/unpack
|
||||
*
|
||||
@ -180,12 +180,12 @@ void mca_ptl_self_matched( mca_ptl_base_module_t* ptl,
|
||||
* a contigous buffer and the convertor on the send request initialized to point
|
||||
* into this buffer.
|
||||
*/
|
||||
if( sendreq->req_send.req_datatype == recvreq->req_base.req_datatype ) {
|
||||
ompi_ddt_copy_content_same_ddt( recvreq->req_base.req_datatype,
|
||||
recvreq->req_base.req_count > sendreq->req_send.req_count ?
|
||||
sendreq->req_send.req_count : recvreq->req_base.req_count,
|
||||
(char *)recvreq->req_base.req_addr,
|
||||
(const char *)sendreq->req_send.req_addr );
|
||||
if( sendreq->req_ptl.req_send.req_datatype == recvreq->req_recv.req_base.req_datatype ) {
|
||||
ompi_ddt_copy_content_same_ddt( recvreq->req_recv.req_base.req_datatype,
|
||||
recvreq->req_recv.req_base.req_count > sendreq->req_ptl.req_send.req_count ?
|
||||
sendreq->req_ptl.req_send.req_count : recvreq->req_recv.req_base.req_count,
|
||||
(char *)recvreq->req_recv.req_base.req_addr,
|
||||
(const char *)sendreq->req_ptl.req_send.req_addr );
|
||||
} else {
|
||||
ompi_convertor_t *pSendConvertor, *pRecvConvertor;
|
||||
struct iovec iov[1];
|
||||
@ -199,10 +199,10 @@ void mca_ptl_self_matched( mca_ptl_base_module_t* ptl,
|
||||
|
||||
ompi_convertor_init_for_recv( &frag->frag_base.frag_convertor,
|
||||
0,
|
||||
recvreq->req_base.req_datatype,
|
||||
recvreq->req_base.req_count,
|
||||
recvreq->req_base.req_addr, 0, NULL );
|
||||
pSendConvertor = &(sendreq->req_send.req_convertor);
|
||||
recvreq->req_recv.req_base.req_datatype,
|
||||
recvreq->req_recv.req_base.req_count,
|
||||
recvreq->req_recv.req_base.req_addr, 0, NULL );
|
||||
pSendConvertor = &(sendreq->req_ptl.req_send.req_convertor);
|
||||
pRecvConvertor = &(frag->frag_base.frag_convertor);
|
||||
completed = 0;
|
||||
freeAfter = 0;
|
||||
@ -221,8 +221,8 @@ void mca_ptl_self_matched( mca_ptl_base_module_t* ptl,
|
||||
free( buf );
|
||||
}
|
||||
}
|
||||
ptl->ptl_send_progress( ptl, &sendreq->req_send,
|
||||
sendreq->req_send.req_bytes_packed );
|
||||
ptl->ptl_send_progress( ptl, &sendreq->req_ptl,
|
||||
sendreq->req_ptl.req_send.req_bytes_packed );
|
||||
ptl->ptl_recv_progress( ptl,
|
||||
recvreq,
|
||||
frag->frag_base.frag_header.hdr_match.hdr_msg_length,
|
||||
|
||||
@ -24,8 +24,8 @@
|
||||
#include "event/event.h"
|
||||
#include "mca/pml/pml.h"
|
||||
#include "mca/ptl/ptl.h"
|
||||
#include "mca/pml/base/pml_base_sendreq.h"
|
||||
#include "mca/pml/base/pml_base_recvreq.h"
|
||||
#include "mca/ptl/base/ptl_base_sendreq.h"
|
||||
#include "mca/ptl/base/ptl_base_recvreq.h"
|
||||
#include "mca/ptl/base/ptl_base_recvfrag.h"
|
||||
#if defined(c_plusplus) || defined(__cplusplus)
|
||||
extern "C" {
|
||||
@ -54,7 +54,7 @@ typedef struct mca_ptl_self_component_t mca_ptl_self_component_t;
|
||||
* base send request, and the base receive fragment which will be used to do the match.
|
||||
*/
|
||||
struct mca_ptl_self_send_request_t {
|
||||
mca_pml_base_send_request_t req_send;
|
||||
mca_ptl_base_send_request_t req_ptl;
|
||||
mca_ptl_base_recv_frag_t req_frag; /* first fragment */
|
||||
};
|
||||
typedef struct mca_ptl_self_send_request_t mca_ptl_self_send_request_t;
|
||||
@ -91,9 +91,9 @@ extern mca_ptl_base_module_t** mca_ptl_self_component_init(
|
||||
int mca_ptl_self_add_proc(struct mca_ptl_base_module_t* ptl, size_t nprocs, struct ompi_proc_t **ompi_proc, struct mca_ptl_base_peer_t** peer_ret, struct ompi_bitmap_t* reachable);
|
||||
int mca_ptl_self_del_proc(struct mca_ptl_base_module_t* ptl, size_t nprocs, struct ompi_proc_t **proc, struct mca_ptl_base_peer_t** ptl_peer);
|
||||
int mca_ptl_self_finalize(struct mca_ptl_base_module_t* ptl);
|
||||
int mca_ptl_self_request_init(struct mca_ptl_base_module_t* ptl, struct mca_pml_base_send_request_t* request);
|
||||
void mca_ptl_self_request_fini(struct mca_ptl_base_module_t* ptl, struct mca_pml_base_send_request_t* request);
|
||||
int mca_ptl_self_send( struct mca_ptl_base_module_t* ptl, struct mca_ptl_base_peer_t* ptl_base_peer, struct mca_pml_base_send_request_t* request,
|
||||
int mca_ptl_self_request_init(struct mca_ptl_base_module_t* ptl, struct mca_ptl_base_send_request_t* request);
|
||||
void mca_ptl_self_request_fini(struct mca_ptl_base_module_t* ptl, struct mca_ptl_base_send_request_t* request);
|
||||
int mca_ptl_self_send( struct mca_ptl_base_module_t* ptl, struct mca_ptl_base_peer_t* ptl_base_peer, struct mca_ptl_base_send_request_t* request,
|
||||
size_t offset, size_t size, int flags );
|
||||
void mca_ptl_self_matched( mca_ptl_base_module_t* ptl, mca_ptl_base_recv_frag_t* frag );
|
||||
|
||||
|
||||
@ -27,8 +27,8 @@
|
||||
#include "util/if.h"
|
||||
#include "mca/pml/pml.h"
|
||||
#include "mca/ptl/ptl.h"
|
||||
#include "mca/pml/base/pml_base_sendreq.h"
|
||||
#include "mca/pml/base/pml_base_recvreq.h"
|
||||
#include "mca/ptl/base/ptl_base_sendreq.h"
|
||||
#include "mca/ptl/base/ptl_base_recvreq.h"
|
||||
#include "mca/ptl/base/ptl_base_header.h"
|
||||
#include "mca/ptl/base/ptl_base_sendfrag.h"
|
||||
#include "mca/ptl/base/ptl_base_recvfrag.h"
|
||||
@ -55,7 +55,7 @@ mca_ptl_sm_t mca_ptl_sm[2] = {
|
||||
&mca_ptl_sm_component.super,
|
||||
20, /* number of elements in the send descriptor cache */
|
||||
sizeof(mca_ptl_sm_send_request_t) -
|
||||
sizeof(mca_pml_base_send_request_t), /* size of shared memory send
|
||||
sizeof(mca_ptl_base_send_request_t), /* size of shared memory send
|
||||
descriptor */
|
||||
1, /* ptl_first_frag_size */
|
||||
0, /* ptl_min_frag_size */
|
||||
@ -80,7 +80,7 @@ mca_ptl_sm_t mca_ptl_sm[2] = {
|
||||
&mca_ptl_sm_component.super,
|
||||
20, /* number of elements in the send descriptor cache */
|
||||
sizeof(mca_ptl_sm_send_request_t) -
|
||||
sizeof(mca_pml_base_send_request_t), /* size of shared memory
|
||||
sizeof(mca_ptl_base_send_request_t), /* size of shared memory
|
||||
send descriptor */
|
||||
1, /* ptl_first_frag_size */
|
||||
0, /* ptl_min_frag_size */
|
||||
@ -711,7 +711,7 @@ int mca_ptl_sm_finalize(struct mca_ptl_base_module_t* ptl)
|
||||
}
|
||||
|
||||
|
||||
int mca_ptl_sm_request_alloc(struct mca_ptl_base_module_t* ptl, struct mca_pml_base_send_request_t* request)
|
||||
int mca_ptl_sm_request_alloc(struct mca_ptl_base_module_t* ptl, struct mca_ptl_base_send_request_t* request)
|
||||
{
|
||||
mca_ptl_sm_send_request_t *sm_request;
|
||||
ompi_list_item_t* item;
|
||||
@ -731,7 +731,7 @@ int mca_ptl_sm_request_alloc(struct mca_ptl_base_module_t* ptl, struct mca_pml_b
|
||||
}
|
||||
|
||||
|
||||
void mca_ptl_sm_request_return(struct mca_ptl_base_module_t* ptl, struct mca_pml_base_send_request_t* request)
|
||||
void mca_ptl_sm_request_return(struct mca_ptl_base_module_t* ptl, struct mca_ptl_base_send_request_t* request)
|
||||
{
|
||||
mca_ptl_sm_send_request_t *sm_request;
|
||||
ompi_list_item_t* item;
|
||||
@ -763,7 +763,7 @@ void mca_ptl_sm_request_return(struct mca_ptl_base_module_t* ptl, struct mca_pml
|
||||
int mca_ptl_sm_send(
|
||||
struct mca_ptl_base_module_t* ptl,
|
||||
struct mca_ptl_base_peer_t* ptl_peer,
|
||||
struct mca_pml_base_send_request_t* sendreq,
|
||||
struct mca_ptl_base_send_request_t* sendreq,
|
||||
size_t offset,
|
||||
size_t size,
|
||||
int flags)
|
||||
@ -794,11 +794,11 @@ int mca_ptl_sm_send(
|
||||
int free_after=0;
|
||||
struct iovec address;
|
||||
|
||||
convertor = &sendreq->req_convertor;
|
||||
convertor = &sendreq->req_send.req_convertor;
|
||||
ompi_convertor_init_for_send( convertor, 0,
|
||||
sendreq->req_datatype,
|
||||
sendreq->req_count,
|
||||
sendreq->req_addr,
|
||||
sendreq->req_send.req_datatype,
|
||||
sendreq->req_send.req_count,
|
||||
sendreq->req_send.req_addr,
|
||||
offset, NULL);
|
||||
|
||||
sm_data_ptr=sm_request->req_frag->buff;
|
||||
@ -822,11 +822,11 @@ int mca_ptl_sm_send(
|
||||
hdr = &(send_frag->super.frag_base.frag_header);
|
||||
hdr->hdr_common.hdr_type = MCA_PTL_HDR_TYPE_MATCH;
|
||||
hdr->hdr_common.hdr_flags = flags;
|
||||
hdr->hdr_match.hdr_contextid = sendreq->req_base.req_comm->c_contextid;
|
||||
hdr->hdr_match.hdr_src = sendreq->req_base.req_comm->c_my_rank;
|
||||
hdr->hdr_match.hdr_dst = sendreq->req_base.req_peer;
|
||||
hdr->hdr_match.hdr_tag = sendreq->req_base.req_tag;
|
||||
hdr->hdr_match.hdr_msg_length = sendreq->req_bytes_packed;
|
||||
hdr->hdr_match.hdr_contextid = sendreq->req_send.req_base.req_comm->c_contextid;
|
||||
hdr->hdr_match.hdr_src = sendreq->req_send.req_base.req_comm->c_my_rank;
|
||||
hdr->hdr_match.hdr_dst = sendreq->req_send.req_base.req_peer;
|
||||
hdr->hdr_match.hdr_tag = sendreq->req_send.req_base.req_tag;
|
||||
hdr->hdr_match.hdr_msg_length = sendreq->req_send.req_bytes_packed;
|
||||
|
||||
/* update the offset within the payload */
|
||||
sendreq->req_offset += size;
|
||||
@ -879,8 +879,8 @@ int mca_ptl_sm_send(
|
||||
ompi_atomic_unlock(&send_fifo->head_lock);
|
||||
|
||||
/* if this is the entire message - signal request is complete */
|
||||
if(sendreq->req_bytes_packed == size) {
|
||||
ompi_request_complete( &(sendreq->req_base.req_ompi) );
|
||||
if(sendreq->req_send.req_bytes_packed == size) {
|
||||
ompi_request_complete( &(sendreq->req_send.req_base.req_ompi) );
|
||||
}
|
||||
|
||||
/* return */
|
||||
@ -897,7 +897,7 @@ int mca_ptl_sm_send(
|
||||
int mca_ptl_sm_send_continue(
|
||||
struct mca_ptl_base_module_t* ptl,
|
||||
struct mca_ptl_base_peer_t* ptl_peer,
|
||||
struct mca_pml_base_send_request_t* sendreq,
|
||||
struct mca_ptl_base_send_request_t* sendreq,
|
||||
size_t offset,
|
||||
size_t size,
|
||||
int flags)
|
||||
@ -928,11 +928,11 @@ int mca_ptl_sm_send_continue(
|
||||
}
|
||||
|
||||
/* pack data in payload buffer */
|
||||
convertor = &sendreq->req_convertor;
|
||||
convertor = &sendreq->req_send.req_convertor;
|
||||
ompi_convertor_init_for_send( convertor, 0,
|
||||
sendreq->req_datatype,
|
||||
sendreq->req_count,
|
||||
sendreq->req_addr,
|
||||
sendreq->req_send.req_datatype,
|
||||
sendreq->req_send.req_count,
|
||||
sendreq->req_send.req_addr,
|
||||
offset, NULL);
|
||||
sm_data_ptr=send_frag->buff;
|
||||
|
||||
|
||||
@ -274,7 +274,7 @@ extern int mca_ptl_sm_del_procs(
|
||||
*/
|
||||
extern int mca_ptl_sm_request_alloc(
|
||||
struct mca_ptl_base_module_t* ptl,
|
||||
struct mca_pml_base_send_request_t*
|
||||
struct mca_ptl_base_send_request_t*
|
||||
);
|
||||
|
||||
/**
|
||||
@ -286,7 +286,7 @@ extern int mca_ptl_sm_request_alloc(
|
||||
*/
|
||||
extern void mca_ptl_sm_request_return(
|
||||
struct mca_ptl_base_module_t* ptl,
|
||||
struct mca_pml_base_send_request_t*
|
||||
struct mca_ptl_base_send_request_t*
|
||||
);
|
||||
|
||||
/**
|
||||
@ -330,7 +330,7 @@ extern int mca_ptl_sm_del_procs(
|
||||
*/
|
||||
extern int mca_ptl_sm_request_alloc(
|
||||
struct mca_ptl_base_module_t* ptl,
|
||||
struct mca_pml_base_send_request_t*
|
||||
struct mca_ptl_base_send_request_t*
|
||||
);
|
||||
|
||||
/**
|
||||
@ -342,7 +342,7 @@ extern int mca_ptl_sm_request_alloc(
|
||||
*/
|
||||
extern void mca_ptl_sm_request_return(
|
||||
struct mca_ptl_base_module_t* ptl,
|
||||
struct mca_pml_base_send_request_t*
|
||||
struct mca_ptl_base_send_request_t*
|
||||
);
|
||||
|
||||
/**
|
||||
@ -388,7 +388,7 @@ extern void mca_ptl_sm_matched_common_base_addr(
|
||||
extern int mca_ptl_sm_send(
|
||||
struct mca_ptl_base_module_t* ptl,
|
||||
struct mca_ptl_base_peer_t* ptl_peer,
|
||||
struct mca_pml_base_send_request_t*,
|
||||
struct mca_ptl_base_send_request_t*,
|
||||
size_t offset,
|
||||
size_t size,
|
||||
int flags
|
||||
@ -407,7 +407,7 @@ extern int mca_ptl_sm_send(
|
||||
extern int mca_ptl_sm_send_continue(
|
||||
struct mca_ptl_base_module_t* ptl,
|
||||
struct mca_ptl_base_peer_t* ptl_peer,
|
||||
struct mca_pml_base_send_request_t*,
|
||||
struct mca_ptl_base_send_request_t*,
|
||||
size_t offset,
|
||||
size_t size,
|
||||
int flags
|
||||
|
||||
@ -293,7 +293,7 @@ mca_ptl_base_module_t** mca_ptl_sm_component_init(
|
||||
for( i=0 ; i < 2 ; i++ ) {
|
||||
mca_ptl_sm[i].super.ptl_cache_size=mca_ptl_sm_component.sm_first_frag_free_list_max;
|
||||
mca_ptl_sm[i].super.ptl_cache_bytes=sizeof(mca_ptl_sm_send_request_t) -
|
||||
sizeof(mca_pml_base_send_request_t);
|
||||
sizeof(mca_ptl_base_send_request_t);
|
||||
mca_ptl_sm[i].super.ptl_first_frag_size=mca_ptl_sm_component.first_fragment_size;
|
||||
mca_ptl_sm[i].super.ptl_min_frag_size=mca_ptl_sm_component.max_fragment_size;
|
||||
mca_ptl_sm[i].super.ptl_max_frag_size=mca_ptl_sm_component.max_fragment_size;
|
||||
@ -363,7 +363,7 @@ int mca_ptl_sm_component_progress(mca_ptl_tstamp_t tstamp)
|
||||
ompi_fifo_t *send_fifo = NULL;
|
||||
bool frag_matched;
|
||||
mca_ptl_base_match_header_t *matching_header;
|
||||
mca_pml_base_send_request_t *base_send_req;
|
||||
mca_ptl_base_send_request_t *base_send_req;
|
||||
ompi_list_item_t *item;
|
||||
int return_status = 0;
|
||||
|
||||
@ -434,10 +434,10 @@ int mca_ptl_sm_component_progress(mca_ptl_tstamp_t tstamp)
|
||||
if( frag_matched ) {
|
||||
/* deliver data, and ack */
|
||||
mca_ptl_base_match_header_t *hdr = &header_ptr->super.frag_base.frag_header.hdr_match;
|
||||
mca_pml_base_recv_request_t *request = header_ptr->super.frag_request;
|
||||
request->req_bytes_packed = hdr->hdr_msg_length;
|
||||
request->req_base.req_ompi.req_status.MPI_SOURCE = hdr->hdr_src;
|
||||
request->req_base.req_ompi.req_status.MPI_TAG = hdr->hdr_tag;
|
||||
mca_ptl_base_recv_request_t *request = header_ptr->super.frag_request;
|
||||
request->req_recv.req_bytes_packed = hdr->hdr_msg_length;
|
||||
request->req_recv.req_base.req_ompi.req_status.MPI_SOURCE = hdr->hdr_src;
|
||||
request->req_recv.req_base.req_ompi.req_status.MPI_TAG = hdr->hdr_tag;
|
||||
mca_ptl_sm_matched_same_base_addr(
|
||||
(mca_ptl_base_module_t *)&mca_ptl_sm,
|
||||
(mca_ptl_base_recv_frag_t *)header_ptr);
|
||||
@ -548,10 +548,10 @@ int mca_ptl_sm_component_progress(mca_ptl_tstamp_t tstamp)
|
||||
if( frag_matched ) {
|
||||
/* deliver data, and ack */
|
||||
mca_ptl_base_match_header_t *hdr = &header_ptr->super.frag_base.frag_header.hdr_match;
|
||||
mca_pml_base_recv_request_t *request = header_ptr->super.frag_request;
|
||||
request->req_bytes_packed = hdr->hdr_msg_length;
|
||||
request->req_base.req_ompi.req_status.MPI_SOURCE = hdr->hdr_src;
|
||||
request->req_base.req_ompi.req_status.MPI_TAG = hdr->hdr_tag;
|
||||
mca_ptl_base_recv_request_t *request = header_ptr->super.frag_request;
|
||||
request->req_recv.req_bytes_packed = hdr->hdr_msg_length;
|
||||
request->req_recv.req_base.req_ompi.req_status.MPI_SOURCE = hdr->hdr_src;
|
||||
request->req_recv.req_base.req_ompi.req_status.MPI_TAG = hdr->hdr_tag;
|
||||
mca_ptl_sm_matched((mca_ptl_base_module_t *)&mca_ptl_sm,
|
||||
(mca_ptl_base_recv_frag_t *)header_ptr);
|
||||
}
|
||||
|
||||
@ -47,7 +47,7 @@ struct mca_ptl_sm_frag_t {
|
||||
int queue_index; /**< local process index, cached for fast
|
||||
acking */
|
||||
struct mca_ptl_base_module_t* send_ptl; /**< PTL that is selected for first fragment */
|
||||
struct mca_pml_base_send_request_t* send_req;
|
||||
struct mca_ptl_base_send_request_t* send_req;
|
||||
size_t send_offset;
|
||||
void *buff; /**< pointer to buffer */
|
||||
};
|
||||
|
||||
@ -67,7 +67,7 @@ void mca_ptl_sm_matched(
|
||||
mca_ptl_base_module_t* ptl,
|
||||
mca_ptl_base_recv_frag_t* frag)
|
||||
{
|
||||
mca_pml_base_recv_request_t* recv_desc;
|
||||
mca_ptl_base_recv_request_t* recv_desc;
|
||||
mca_ptl_sm_frag_t *sm_frag_desc;
|
||||
struct iovec iov;
|
||||
ompi_convertor_t frag_convertor;
|
||||
@ -92,7 +92,7 @@ void mca_ptl_sm_matched(
|
||||
* Initialize convertor and use it to unpack data
|
||||
*/
|
||||
OBJ_CONSTRUCT(&frag_convertor, ompi_convertor_t);
|
||||
proc = ompi_comm_peer_lookup(recv_desc->req_base.req_comm,
|
||||
proc = ompi_comm_peer_lookup(recv_desc->req_recv.req_base.req_comm,
|
||||
frag->frag_base.frag_header.hdr_match.hdr_src);
|
||||
/* write over converter set on the send side */
|
||||
ompi_convertor_copy(proc->proc_convertor,
|
||||
@ -100,9 +100,9 @@ void mca_ptl_sm_matched(
|
||||
ompi_convertor_init_for_recv(
|
||||
&frag_convertor, /* convertor */
|
||||
0, /* flags */
|
||||
recv_desc->req_base.req_datatype, /* datatype */
|
||||
recv_desc->req_base.req_count, /* count elements */
|
||||
recv_desc->req_base.req_addr, /* users buffer */
|
||||
recv_desc->req_recv.req_base.req_datatype, /* datatype */
|
||||
recv_desc->req_recv.req_base.req_count, /* count elements */
|
||||
recv_desc->req_recv.req_base.req_addr, /* users buffer */
|
||||
sm_frag_desc->send_offset, /* offset in bytes into packed buffer */
|
||||
NULL ); /* dont allocate memory */
|
||||
|
||||
|
||||
@ -21,7 +21,7 @@
|
||||
|
||||
#include <sys/types.h>
|
||||
#include "ompi_config.h"
|
||||
#include "mca/pml/base/pml_base_sendreq.h"
|
||||
#include "mca/ptl/base/ptl_base_sendreq.h"
|
||||
#include "mca/ptl/base/ptl_base_sendfrag.h"
|
||||
#include "ptl_sm.h"
|
||||
|
||||
@ -56,7 +56,7 @@ typedef struct mca_ptl_sm_send_frag_t mca_ptl_sm_send_frag_t;
|
||||
int mca_ptl_sm_send_frag_init(
|
||||
mca_ptl_sm_send_frag_t*,
|
||||
struct mca_ptl_base_peer_t*,
|
||||
struct mca_pml_base_send_request_t*,
|
||||
struct mca_ptl_base_send_request_t*,
|
||||
size_t offset,
|
||||
size_t* size,
|
||||
int flags);
|
||||
|
||||
@ -30,7 +30,7 @@ static void mca_ptl_sm_send_request_destruct(mca_ptl_sm_send_request_t*);
|
||||
|
||||
OBJ_CLASS_INSTANCE(
|
||||
mca_ptl_sm_send_request_t,
|
||||
mca_pml_base_send_request_t,
|
||||
mca_ptl_base_send_request_t,
|
||||
mca_ptl_sm_send_request_construct,
|
||||
mca_ptl_sm_send_request_destruct
|
||||
);
|
||||
@ -54,7 +54,7 @@ void mca_ptl_sm_send_request_destruct(mca_ptl_sm_send_request_t* request)
|
||||
* to allocate fragment descriptor and payload memory
|
||||
*/
|
||||
int mca_ptl_sm_send_request_init(struct mca_ptl_base_module_t* ptl,
|
||||
struct mca_pml_base_send_request_t* request)
|
||||
struct mca_ptl_base_send_request_t* request)
|
||||
{
|
||||
|
||||
mca_ptl_sm_send_request_t *sm_request;
|
||||
|
||||
@ -22,7 +22,7 @@
|
||||
|
||||
#include <sys/types.h>
|
||||
#include "ompi_config.h"
|
||||
#include "mca/pml/base/pml_base_sendreq.h"
|
||||
#include "mca/ptl/base/ptl_base_sendreq.h"
|
||||
#include "ptl_sm_frag.h"
|
||||
|
||||
#if defined(c_plusplus) || defined(__cplusplus)
|
||||
@ -38,7 +38,7 @@ OBJ_CLASS_DECLARATION(mca_ptl_sm_send_request_t);
|
||||
struct mca_ptl_sm_send_request_t {
|
||||
|
||||
/* base send descriptor */
|
||||
mca_pml_base_send_request_t super;
|
||||
mca_ptl_base_send_request_t super;
|
||||
|
||||
/* pointer to first fragment descriptor */
|
||||
mca_ptl_sm_frag_t *req_frag;
|
||||
@ -50,7 +50,7 @@ typedef struct mca_ptl_sm_send_request_t mca_ptl_sm_send_request_t;
|
||||
* memory send request is initialized.
|
||||
*/
|
||||
int mca_ptl_sm_send_request_init(struct mca_ptl_base_module_t* ptl,
|
||||
struct mca_pml_base_send_request_t* request);
|
||||
struct mca_ptl_base_send_request_t* request);
|
||||
|
||||
#if defined(c_plusplus) || defined(__cplusplus)
|
||||
}
|
||||
|
||||
@ -18,7 +18,7 @@
|
||||
sources += \
|
||||
src/ptl_tcp.c \
|
||||
src/ptl_tcp.h \
|
||||
src/ptl_tcp_addr.h \
|
||||
src/ptl_tcp_addr.h \
|
||||
src/ptl_tcp_component.c \
|
||||
src/ptl_tcp_peer.c \
|
||||
src/ptl_tcp_peer.h \
|
||||
|
||||
@ -22,9 +22,9 @@
|
||||
#include "mca/pml/pml.h"
|
||||
#include "mca/ptl/ptl.h"
|
||||
#include "mca/ptl/base/ptl_base_header.h"
|
||||
#include "mca/pml/base/pml_base_sendreq.h"
|
||||
#include "mca/ptl/base/ptl_base_sendreq.h"
|
||||
#include "mca/ptl/base/ptl_base_sendfrag.h"
|
||||
#include "mca/pml/base/pml_base_recvreq.h"
|
||||
#include "mca/ptl/base/ptl_base_recvreq.h"
|
||||
#include "mca/ptl/base/ptl_base_recvfrag.h"
|
||||
#include "mca/base/mca_base_module_exchange.h"
|
||||
#include "ptl_tcp.h"
|
||||
@ -39,7 +39,7 @@ mca_ptl_tcp_module_t mca_ptl_tcp_module = {
|
||||
{
|
||||
&mca_ptl_tcp_component.super,
|
||||
16, /* max size of request cache */
|
||||
sizeof(mca_ptl_tcp_send_request_t) - sizeof(mca_pml_base_send_request_t), /* bytes required by ptl for a request */
|
||||
sizeof(mca_ptl_tcp_send_request_t) - sizeof(mca_ptl_base_send_request_t), /* bytes required by ptl for a request */
|
||||
0, /* max size of first fragment */
|
||||
0, /* min fragment size */
|
||||
0, /* max fragment size */
|
||||
@ -179,7 +179,7 @@ int mca_ptl_tcp_finalize(struct mca_ptl_base_module_t* ptl)
|
||||
* along w/ the ptl to cache the first fragment control information.
|
||||
*/
|
||||
|
||||
int mca_ptl_tcp_request_init(struct mca_ptl_base_module_t* ptl, struct mca_pml_base_send_request_t* request)
|
||||
int mca_ptl_tcp_request_init(struct mca_ptl_base_module_t* ptl, struct mca_ptl_base_send_request_t* request)
|
||||
{
|
||||
OBJ_CONSTRUCT(request+1, mca_ptl_tcp_send_frag_t);
|
||||
return OMPI_SUCCESS;
|
||||
@ -190,7 +190,7 @@ int mca_ptl_tcp_request_init(struct mca_ptl_base_module_t* ptl, struct mca_pml_b
|
||||
* Cleanup any resources cached along w/ the request.
|
||||
*/
|
||||
|
||||
void mca_ptl_tcp_request_fini(struct mca_ptl_base_module_t* ptl, struct mca_pml_base_send_request_t* request)
|
||||
void mca_ptl_tcp_request_fini(struct mca_ptl_base_module_t* ptl, struct mca_ptl_base_send_request_t* request)
|
||||
{
|
||||
OBJ_DESTRUCT(request+1);
|
||||
}
|
||||
@ -240,7 +240,7 @@ void mca_ptl_tcp_send_frag_return(struct mca_ptl_base_module_t* ptl, struct mca_
|
||||
int mca_ptl_tcp_send(
|
||||
struct mca_ptl_base_module_t* ptl,
|
||||
struct mca_ptl_base_peer_t* ptl_peer,
|
||||
struct mca_pml_base_send_request_t* sendreq,
|
||||
struct mca_ptl_base_send_request_t* sendreq,
|
||||
size_t offset,
|
||||
size_t size,
|
||||
int flags)
|
||||
@ -261,7 +261,7 @@ int mca_ptl_tcp_send(
|
||||
/* must update the offset after actual fragment size is determined -- and very important --
|
||||
* before attempting to send the fragment
|
||||
*/
|
||||
mca_pml_base_send_request_offset(sendreq, size);
|
||||
mca_ptl_base_send_request_offset(sendreq, size);
|
||||
return mca_ptl_tcp_peer_send(ptl_peer, sendfrag, offset);
|
||||
}
|
||||
|
||||
|
||||
@ -197,7 +197,7 @@ extern int mca_ptl_tcp_del_procs(
|
||||
*/
|
||||
extern int mca_ptl_tcp_request_init(
|
||||
struct mca_ptl_base_module_t* ptl,
|
||||
struct mca_pml_base_send_request_t*
|
||||
struct mca_ptl_base_send_request_t*
|
||||
);
|
||||
|
||||
/**
|
||||
@ -209,7 +209,7 @@ extern int mca_ptl_tcp_request_init(
|
||||
*/
|
||||
extern void mca_ptl_tcp_request_fini(
|
||||
struct mca_ptl_base_module_t* ptl,
|
||||
struct mca_pml_base_send_request_t*
|
||||
struct mca_ptl_base_send_request_t*
|
||||
);
|
||||
|
||||
/**
|
||||
@ -237,7 +237,7 @@ extern void mca_ptl_tcp_matched(
|
||||
extern int mca_ptl_tcp_send(
|
||||
struct mca_ptl_base_module_t* ptl,
|
||||
struct mca_ptl_base_peer_t* ptl_peer,
|
||||
struct mca_pml_base_send_request_t*,
|
||||
struct mca_ptl_base_send_request_t*,
|
||||
size_t offset,
|
||||
size_t size,
|
||||
int flags
|
||||
|
||||
@ -21,7 +21,7 @@
|
||||
#include <sys/types.h>
|
||||
#endif
|
||||
#include "include/ompi_socket_errno.h"
|
||||
#include "mca/pml/base/pml_base_sendreq.h"
|
||||
#include "mca/ptl/base/ptl_base_sendreq.h"
|
||||
#include "ptl_tcp.h"
|
||||
#include "ptl_tcp_peer.h"
|
||||
#include "ptl_tcp_recvfrag.h"
|
||||
@ -184,7 +184,7 @@ static bool mca_ptl_tcp_recv_frag_header(mca_ptl_tcp_recv_frag_t* frag, int sd,
|
||||
static bool mca_ptl_tcp_recv_frag_ack(mca_ptl_tcp_recv_frag_t* frag, int sd)
|
||||
{
|
||||
mca_ptl_tcp_send_frag_t* sendfrag;
|
||||
mca_pml_base_send_request_t* sendreq;
|
||||
mca_ptl_base_send_request_t* sendreq;
|
||||
sendfrag = (mca_ptl_tcp_send_frag_t*)frag->frag_recv.frag_base.frag_header.hdr_ack.hdr_src_ptr.pval;
|
||||
sendreq = sendfrag->frag_send.frag_request;
|
||||
sendreq->req_peer_match = frag->frag_recv.frag_base.frag_header.hdr_ack.hdr_dst_match;
|
||||
@ -227,7 +227,7 @@ static bool mca_ptl_tcp_recv_frag_frag(mca_ptl_tcp_recv_frag_t* frag, int sd)
|
||||
{
|
||||
/* get request from header */
|
||||
if(frag->frag_msg_cnt == 0) {
|
||||
frag->frag_recv.frag_request = (mca_pml_base_recv_request_t *)frag->frag_recv.frag_base.frag_header.hdr_frag.hdr_dst_ptr.pval;
|
||||
frag->frag_recv.frag_request = (mca_ptl_base_recv_request_t *)frag->frag_recv.frag_base.frag_header.hdr_frag.hdr_dst_ptr.pval;
|
||||
mca_ptl_tcp_recv_frag_matched(frag,
|
||||
frag->frag_recv.frag_base.frag_header.hdr_frag.hdr_frag_offset,
|
||||
frag->frag_recv.frag_base.frag_header.hdr_frag.hdr_frag_length);
|
||||
|
||||
@ -89,21 +89,21 @@ static inline void mca_ptl_tcp_recv_frag_matched(
|
||||
size_t frag_offset,
|
||||
size_t frag_length)
|
||||
{
|
||||
mca_pml_base_recv_request_t* request = frag->frag_recv.frag_request;
|
||||
mca_ptl_base_recv_request_t* request = frag->frag_recv.frag_request;
|
||||
|
||||
/* if there is data associated with the fragment -- setup to receive */
|
||||
if(frag_length > 0) {
|
||||
/* initialize receive convertor */
|
||||
ompi_proc_t *proc =
|
||||
ompi_comm_peer_lookup(request->req_base.req_comm,
|
||||
request->req_base.req_ompi.req_status.MPI_SOURCE);
|
||||
ompi_comm_peer_lookup(request->req_recv.req_base.req_comm,
|
||||
request->req_recv.req_base.req_ompi.req_status.MPI_SOURCE);
|
||||
ompi_convertor_copy(proc->proc_convertor, &frag->frag_recv.frag_base.frag_convertor);
|
||||
ompi_convertor_init_for_recv(
|
||||
&frag->frag_recv.frag_base.frag_convertor, /* convertor */
|
||||
0, /* flags */
|
||||
request->req_base.req_datatype, /* datatype */
|
||||
request->req_base.req_count, /* count elements */
|
||||
request->req_base.req_addr, /* users buffer */
|
||||
request->req_recv.req_base.req_datatype, /* datatype */
|
||||
request->req_recv.req_base.req_count, /* count elements */
|
||||
request->req_recv.req_base.req_addr, /* users buffer */
|
||||
0, /* offset in bytes into packed buffer */
|
||||
mca_ptl_tcp_memalloc ); /* not allocating memory */
|
||||
|
||||
@ -115,16 +115,16 @@ static inline void mca_ptl_tcp_recv_frag_matched(
|
||||
} else {
|
||||
long true_lb, true_extent;
|
||||
|
||||
ompi_ddt_get_true_extent( request->req_base.req_datatype, &true_lb, &true_extent );
|
||||
frag->frag_recv.frag_base.frag_addr = ((unsigned char*)request->req_base.req_addr) +
|
||||
ompi_ddt_get_true_extent( request->req_recv.req_base.req_datatype, &true_lb, &true_extent );
|
||||
frag->frag_recv.frag_base.frag_addr = ((unsigned char*)request->req_recv.req_base.req_addr) +
|
||||
frag_offset + true_lb;
|
||||
}
|
||||
frag->frag_size = frag_length;
|
||||
if(frag_offset + frag_length > request->req_bytes_packed) {
|
||||
if(frag_offset > request->req_bytes_packed)
|
||||
if(frag_offset + frag_length > request->req_recv.req_bytes_packed) {
|
||||
if(frag_offset > request->req_recv.req_bytes_packed)
|
||||
frag->frag_recv.frag_base.frag_size = 0;
|
||||
else
|
||||
frag->frag_recv.frag_base.frag_size = request->req_bytes_packed - frag_offset;
|
||||
frag->frag_recv.frag_base.frag_size = request->req_recv.req_bytes_packed - frag_offset;
|
||||
} else {
|
||||
frag->frag_recv.frag_base.frag_size = frag_length;
|
||||
}
|
||||
@ -151,7 +151,7 @@ static inline void mca_ptl_tcp_recv_frag_progress(mca_ptl_tcp_recv_frag_t* frag)
|
||||
(frag->frag_recv.frag_base.frag_header.hdr_common.hdr_type != MCA_PTL_HDR_TYPE_FRAG &&
|
||||
frag_progressed == 2 )) {
|
||||
|
||||
mca_pml_base_recv_request_t* request = frag->frag_recv.frag_request;
|
||||
mca_ptl_base_recv_request_t* request = frag->frag_recv.frag_request;
|
||||
if(frag->frag_recv.frag_is_buffered) {
|
||||
mca_ptl_base_frag_header_t* header = &(frag)->frag_recv.frag_base.frag_header.hdr_frag;
|
||||
size_t offset = (header->hdr_common.hdr_type == MCA_PTL_HDR_TYPE_FRAG) ?
|
||||
@ -162,15 +162,15 @@ static inline void mca_ptl_tcp_recv_frag_progress(mca_ptl_tcp_recv_frag_t* frag)
|
||||
*/
|
||||
struct iovec iov;
|
||||
ompi_proc_t *proc =
|
||||
ompi_comm_peer_lookup(request->req_base.req_comm,
|
||||
request->req_base.req_ompi.req_status.MPI_SOURCE);
|
||||
ompi_comm_peer_lookup(request->req_recv.req_base.req_comm,
|
||||
request->req_recv.req_base.req_ompi.req_status.MPI_SOURCE);
|
||||
ompi_convertor_copy(proc->proc_convertor, &frag->frag_recv.frag_base.frag_convertor);
|
||||
ompi_convertor_init_for_recv(
|
||||
&frag->frag_recv.frag_base.frag_convertor, /* convertor */
|
||||
0, /* flags */
|
||||
request->req_base.req_datatype, /* datatype */
|
||||
request->req_base.req_count, /* count elements */
|
||||
request->req_base.req_addr, /* users buffer */
|
||||
request->req_recv.req_base.req_datatype, /* datatype */
|
||||
request->req_recv.req_base.req_count, /* count elements */
|
||||
request->req_recv.req_base.req_addr, /* users buffer */
|
||||
offset, /* offset in bytes into packed buffer */
|
||||
NULL ); /* dont allocate memory */
|
||||
|
||||
|
||||
@ -78,7 +78,7 @@ void* mca_ptl_tcp_memalloc( size_t* length )
|
||||
int mca_ptl_tcp_send_frag_init(
|
||||
mca_ptl_tcp_send_frag_t* sendfrag,
|
||||
mca_ptl_base_peer_t* ptl_peer,
|
||||
mca_pml_base_send_request_t* sendreq,
|
||||
mca_ptl_base_send_request_t* sendreq,
|
||||
size_t offset,
|
||||
size_t* size,
|
||||
int flags)
|
||||
@ -96,13 +96,13 @@ int mca_ptl_tcp_send_frag_init(
|
||||
int rc;
|
||||
|
||||
convertor = &sendfrag->frag_convertor;
|
||||
ompi_convertor_copy(&sendreq->req_convertor, convertor);
|
||||
ompi_convertor_copy(&sendreq->req_send.req_convertor, convertor);
|
||||
ompi_convertor_init_for_send(
|
||||
convertor,
|
||||
0,
|
||||
sendreq->req_datatype,
|
||||
sendreq->req_count,
|
||||
sendreq->req_addr,
|
||||
sendreq->req_send.req_datatype,
|
||||
sendreq->req_send.req_count,
|
||||
sendreq->req_send.req_addr,
|
||||
offset,
|
||||
mca_ptl_tcp_memalloc );
|
||||
|
||||
@ -130,12 +130,12 @@ int mca_ptl_tcp_send_frag_init(
|
||||
if(offset == 0) {
|
||||
hdr->hdr_common.hdr_type = (flags & MCA_PTL_FLAGS_ACK) ? MCA_PTL_HDR_TYPE_MATCH : MCA_PTL_HDR_TYPE_RNDV;
|
||||
hdr->hdr_common.hdr_flags = flags;
|
||||
hdr->hdr_match.hdr_contextid = sendreq->req_base.req_comm->c_contextid;
|
||||
hdr->hdr_match.hdr_src = sendreq->req_base.req_comm->c_my_rank;
|
||||
hdr->hdr_match.hdr_dst = sendreq->req_base.req_peer;
|
||||
hdr->hdr_match.hdr_tag = sendreq->req_base.req_tag;
|
||||
hdr->hdr_match.hdr_msg_length = sendreq->req_bytes_packed;
|
||||
hdr->hdr_match.hdr_msg_seq = sendreq->req_base.req_sequence;
|
||||
hdr->hdr_match.hdr_contextid = sendreq->req_send.req_base.req_comm->c_contextid;
|
||||
hdr->hdr_match.hdr_src = sendreq->req_send.req_base.req_comm->c_my_rank;
|
||||
hdr->hdr_match.hdr_dst = sendreq->req_send.req_base.req_peer;
|
||||
hdr->hdr_match.hdr_tag = sendreq->req_send.req_base.req_tag;
|
||||
hdr->hdr_match.hdr_msg_length = sendreq->req_send.req_bytes_packed;
|
||||
hdr->hdr_match.hdr_msg_seq = sendreq->req_send.req_base.req_sequence;
|
||||
hdr->hdr_rndv.hdr_frag_length = size_out;
|
||||
hdr->hdr_rndv.hdr_src_ptr.lval = 0; /* for VALGRIND/PURIFY - REPLACE WITH MACRO */
|
||||
hdr->hdr_rndv.hdr_src_ptr.pval = sendfrag;
|
||||
|
||||
@ -30,7 +30,7 @@
|
||||
#include <netinet/in.h>
|
||||
#endif
|
||||
#include "include/sys/atomic.h"
|
||||
#include "mca/pml/base/pml_base_sendreq.h"
|
||||
#include "mca/ptl/base/ptl_base_sendreq.h"
|
||||
#include "mca/ptl/base/ptl_base_sendfrag.h"
|
||||
#include "ptl_tcp.h"
|
||||
#include "ptl_tcp_recvfrag.h"
|
||||
@ -77,7 +77,7 @@ bool mca_ptl_tcp_send_frag_handler(mca_ptl_tcp_send_frag_t*, int sd);
|
||||
int mca_ptl_tcp_send_frag_init(
|
||||
mca_ptl_tcp_send_frag_t*,
|
||||
struct mca_ptl_base_peer_t*,
|
||||
struct mca_pml_base_send_request_t*,
|
||||
struct mca_ptl_base_send_request_t*,
|
||||
size_t offset,
|
||||
size_t* size,
|
||||
int flags);
|
||||
@ -92,7 +92,7 @@ int mca_ptl_tcp_send_frag_init(
|
||||
|
||||
static inline void mca_ptl_tcp_send_frag_progress(mca_ptl_tcp_send_frag_t* frag)
|
||||
{
|
||||
mca_pml_base_send_request_t* request = frag->frag_send.frag_request;
|
||||
mca_ptl_base_send_request_t* request = frag->frag_send.frag_request;
|
||||
bool frag_ack;
|
||||
|
||||
/* if this is an ack - simply return to pool */
|
||||
@ -133,15 +133,15 @@ static inline void mca_ptl_tcp_send_frag_init_ack(
|
||||
mca_ptl_tcp_recv_frag_t* frag)
|
||||
{
|
||||
mca_ptl_base_header_t* hdr = &ack->frag_send.frag_base.frag_header;
|
||||
mca_pml_base_recv_request_t* request = frag->frag_recv.frag_request;
|
||||
mca_ptl_base_recv_request_t* request = frag->frag_recv.frag_request;
|
||||
hdr->hdr_common.hdr_type = MCA_PTL_HDR_TYPE_ACK;
|
||||
hdr->hdr_common.hdr_flags = 0;
|
||||
hdr->hdr_ack.hdr_src_ptr = frag->frag_recv.frag_base.frag_header.hdr_rndv.hdr_src_ptr;
|
||||
hdr->hdr_ack.hdr_dst_match.lval = 0; /* for VALGRIND/PURIFY - REPLACE WITH MACRO */
|
||||
hdr->hdr_ack.hdr_dst_match.pval = request;
|
||||
hdr->hdr_ack.hdr_dst_addr.lval = 0; /* for VALGRIND/PURIFY - REPLACE WITH MACRO */
|
||||
hdr->hdr_ack.hdr_dst_addr.pval = request->req_base.req_addr;
|
||||
hdr->hdr_ack.hdr_dst_size = request->req_bytes_packed;
|
||||
hdr->hdr_ack.hdr_dst_addr.pval = request->req_recv.req_base.req_addr;
|
||||
hdr->hdr_ack.hdr_dst_size = request->req_recv.req_bytes_packed;
|
||||
ack->frag_send.frag_request = 0;
|
||||
ack->frag_send.frag_base.frag_peer = ptl_peer;
|
||||
ack->frag_send.frag_base.frag_owner = ptl;
|
||||
|
||||
@ -22,7 +22,7 @@
|
||||
#endif
|
||||
#include "include/ompi_socket_errno.h"
|
||||
#include "include/types.h"
|
||||
#include "mca/pml/base/pml_base_sendreq.h"
|
||||
#include "mca/ptl/base/ptl_base_sendreq.h"
|
||||
#include "ptl_tcp.h"
|
||||
#include "ptl_tcp_sendreq.h"
|
||||
|
||||
@ -33,7 +33,7 @@ static void mca_ptl_tcp_send_request_destruct(mca_ptl_tcp_send_request_t*);
|
||||
|
||||
ompi_class_t mca_ptl_tcp_send_request_t_class = {
|
||||
"mca_ptl_tcp_send_request_t",
|
||||
OBJ_CLASS(mca_pml_base_send_request_t),
|
||||
OBJ_CLASS(mca_ptl_base_send_request_t),
|
||||
(ompi_construct_t)mca_ptl_tcp_send_request_construct,
|
||||
(ompi_destruct_t)mca_ptl_tcp_send_request_destruct
|
||||
};
|
||||
|
||||
@ -30,7 +30,7 @@
|
||||
#include <netinet/in.h>
|
||||
#endif
|
||||
#include "ompi_config.h"
|
||||
#include "mca/pml/base/pml_base_sendreq.h"
|
||||
#include "mca/ptl/base/ptl_base_sendreq.h"
|
||||
#include "ptl_tcp_sendfrag.h"
|
||||
#if defined(c_plusplus) || defined(__cplusplus)
|
||||
extern "C" {
|
||||
@ -43,7 +43,7 @@ extern ompi_class_t mca_ptl_tcp_send_request_t_class;
|
||||
* fragment on every send request.
|
||||
*/
|
||||
struct mca_ptl_tcp_send_request_t {
|
||||
mca_pml_base_send_request_t super;
|
||||
mca_ptl_base_send_request_t super;
|
||||
mca_ptl_tcp_send_frag_t req_frag; /* first fragment */
|
||||
};
|
||||
typedef struct mca_ptl_tcp_send_request_t mca_ptl_tcp_send_request_t;
|
||||
|
||||
Загрузка…
Ссылка в новой задаче
Block a user