ports/openmpi
ports/openmpi
1
1
Форкнуть 0
Код Релизы Активность

Very initial IB PTL implementation.

Просмотр исходного кода 
Stuff accomplished:

1. configure.stub and other Makefile stuff.
2. Layout of initial ptl_ib datastructures.
3. module_open and module_init are working.
4. Set up a UD interface on module init.
5. OOB is used to exchange the UD queue pairs.

HACK ALERT!
-----------
* Forcibly opening only one ptl inside module init. Our machines are
equipped with multiple HCAs ...
* Using #defines for some constants. These should be runtime parameters
but ignoring these for now ...

still fiddling around with ompi_proc and other ib_peer datastructures.

-Sayantan.

This commit was SVN r1759.
Этот коммит содержится в:
Sayantan Sur 2004-07-16 14:28:51 +00:00
родитель 62317978a4
Коммит 93115f589e
18 изменённых файлов: 1482 добавлений и 0 удалений
Показать статистику Скачать Patch файл Скачать Diff файл Показать все файлы Свернуть все файлы

34
src/mca/ptl/ib/Makefile.am Обычный файл
Просмотреть файл

@ -0,0 +1,34 @@
#
# $HEADER$
#
# Use the top-level Makefile.options
include $(top_ompi_srcdir)/config/Makefile.options
SUBDIRS = src
EXTRA_DIST = VERSION
# Make the output library in this directory, and name it either
# mca_<type>_<name>.la (for DSO builds) or libmca_<type>_<name>.la
# (for static builds).
if OMPI_BUILD_ptl_ib_DSO
component_noinst =
component_install = mca_ptl_ib.la
else
component_noinst = libmca_ptl_ib.la
component_install =
endif
mcacomponentdir = $(libdir)/openmpi
mcacomponent_LTLIBRARIES = $(component_install)
mca_ptl_ib_la_SOURCES =
mca_ptl_ib_la_LIBADD = src/libmca_ptl_ib.la
mca_ptl_ib_la_LDFLAGS = -module -avoid-version
noinst_LTLIBRARIES = $(component_noinst)
libmca_ptl_ib_la_SOURCES =
libmca_ptl_ib_la_LIBADD = src/libmca_ptl_ib.la
libmca_ptl_ib_la_LDFLAGS = -module -avoid-version

6
src/mca/ptl/ib/VERSION Обычный файл
Просмотреть файл

@ -0,0 +1,6 @@
major=1
minor=0
release=0
alpha=0
beta=0
svn=1

10
src/mca/ptl/ib/configure.params Обычный файл
Просмотреть файл

@ -0,0 +1,10 @@
# -*- shell-script -*-
#
# $HEADER$
#
# Specific to this module
PARAM_INIT_FILE=src/ptl_ib.c
PARAM_CONFIG_HEADER_FILE="ptl_ib_config.h"
PARAM_CONFIG_FILES="Makefile src/Makefile"

78
src/mca/ptl/ib/configure.stub Обычный файл
Просмотреть файл

@ -0,0 +1,78 @@
# -*- shell-script -*-
#
# $HEADER$
#
#
# Main function. This will be invoked in the middle of the templated
# configure script.
#
AC_DEFUN([MCA_CONFIGURE_STUB],[
# Additional --with flags that can be specified
AC_ARG_WITH(ptl-ib,
AC_HELP_STRING([--with-ptl-ib=DIR],
[Specify the installation directory of IB]))
AC_ARG_WITH(ptl-ib-libdir,
AC_HELP_STRING([--with-ptl-ib-libdir=DIR],
[directory where the IB library can be found, if it is not in \$IBDIR/lib or \$IBDIR/binary/lib]))
# Add to CPPFLAGS if necessary
EXTRA_CPPFLAGS=
if test -n "$with_ptl_ib"; then
if test -d "$with_ptl_ib/include"; then
EXTRA_CPPFLAGS="-I$with_ptl_ib/include"
else
AC_MSG_WARN([*** Warning: cannot find $with_ptl_ib/include])
AC_MSG_WARN([*** Will still try to configure ib ptl anyway...])
fi
if test -d "$with_ptl_ib/wrap"; then
EXTRA_CPPFLAGS="-I$with_ptl_ib/wrap $EXTRA_CPPFLAGS"
else
AC_MSG_WARN([*** Warning: cannot find $with_ptl_ib/wrap])
AC_MSG_WARN([*** Will still try to configure ib ptl anyway...])
fi
fi
# See if we can find vapi.h
CPPFLAGS="$CPPFLAGS $EXTRA_CPPFLAGS"
AC_CHECK_HEADERS(vapi.h,,
AC_MSG_ERROR([*** Cannot find working vapi.h]))
# Add to LDFLAGS if necessary
EXTRA_LDFLAGS=
if test -n "$with_ptl_ib_libdir"; then
if test -d "$with_ptl_ib_libdir/lib"; then
EXTRA_LDFLAGS="-L$with_ptl_ib_libdir/lib"
else
AC_MSG_WARN([*** Warning: cannot find $with_ptl_ib_libdir/lib])
AC_MSG_WARN([*** Will still try to configure ib ptl anyway...])
fi
elif test -n "$with_ptl_ib"; then
if test -d "$with_ptl_ib/lib"; then
EXTRA_LDFLAGS="-L$with_ptl_ib/lib"
else
AC_MSG_WARN([*** Warning: cannot find $with_ptl_ib/lib])
AC_MSG_WARN([*** Will still try to configure ib ptl anyway...])
fi
fi
# Try to find libvapi
LDFLAGS="$LDFLAGS $EXTRA_LDFLAGS"
AC_CHECK_LIB([vapi], [VAPI_open_hca], [],
AC_MSG_ERROR([*** Cannot find libvapi]), [-lmtl_common -lmpga])
LIBS="$LIBS -lmtl_common -lmpga"
#
# Save extra compiler/linker flags so that they can be added in
# the wrapper compilers, if necessary
#
WRAPPER_EXTRA_LDFLAGS="$EXTRA_LDFLAGS"
WRAPPER_EXTRA_LIBS="-lvapi -lmtl_common -lmpga"
])dnl

14
src/mca/ptl/ib/src/Makefile.am Обычный файл
Просмотреть файл

@ -0,0 +1,14 @@
# -*- makefile -*-
#
# $HEADER$
#
include $(top_ompi_srcdir)/config/Makefile.options
noinst_LTLIBRARIES = libmca_ptl_ib.la
libmca_ptl_ib_la_SOURCES = \
ptl_ib.c \
ptl_ib.h \
ptl_ib_module.c \
ptl_ib_priv.c \
ptl_ib_proc.c

180
src/mca/ptl/ib/src/ptl_ib.c Обычный файл
Просмотреть файл

@ -0,0 +1,180 @@
/*
* $HEADER$
*/
#include <string.h>
#include "util/output.h"
#include "util/if.h"
#include "mca/pml/pml.h"
#include "mca/ptl/ptl.h"
#include "mca/ptl/base/ptl_base_header.h"
#include "mca/pml/base/pml_base_sendreq.h"
#include "mca/ptl/base/ptl_base_sendfrag.h"
#include "mca/pml/base/pml_base_recvreq.h"
#include "mca/ptl/base/ptl_base_recvfrag.h"
#include "mca/base/mca_base_module_exchange.h"
#include "ptl_ib.h"
mca_ptl_ib_t mca_ptl_ib = {
{
&mca_ptl_ib_module.super,
0, /* ptl_exclusivity */
0, /* ptl_latency */
0, /* ptl_andwidth */
0, /* ptl_frag_first_size */
0, /* ptl_frag_min_size */
0, /* ptl_frag_max_size */
MCA_PTL_PUT, /* ptl flags */
mca_ptl_ib_add_procs,
mca_ptl_ib_del_procs,
mca_ptl_ib_finalize,
mca_ptl_ib_send,
NULL,
mca_ptl_ib_matched,
mca_ptl_ib_request_alloc,
mca_ptl_ib_request_return
}
};
OBJ_CLASS_INSTANCE(mca_ptl_ib_recv_frag_t,
mca_ptl_base_recv_frag_t,
NULL, NULL);
OBJ_CLASS_INSTANCE(mca_ptl_ib_send_request_t,
mca_pml_base_send_request_t,
NULL, NULL);
OBJ_CLASS_INSTANCE(mca_ptl_ib_peer_t,
ompi_list_item_t,
NULL, NULL);
OBJ_CLASS_INSTANCE(mca_ptl_ib_proc_t,
ompi_list_item_t,
NULL, NULL);
int mca_ptl_ib_add_procs(
struct mca_ptl_t* ptl,
size_t nprocs,
struct ompi_proc_t **ompi_procs,
struct mca_ptl_base_peer_t** peers,
ompi_bitmap_t* reachable)
{
int i, rc;
fprintf(stderr,"[%s:%d] %s\n",
__FILE__, __LINE__, __func__);
for(i = 0; i < nprocs; i++) {
struct ompi_proc_t* ompi_proc = ompi_procs[i];
mca_ptl_ib_proc_t* ptl_proc = mca_ptl_ib_proc_create(ompi_proc);
mca_ptl_base_peer_t* ptl_peer;
if(NULL == ptl_proc) {
return OMPI_ERR_OUT_OF_RESOURCE;
}
/*
* Check to make sure that the peer has at least as many interface
* addresses exported as we are trying to use. If not, then
* don't bind this PTL instance to the proc.
*/
OMPI_THREAD_LOCK(&ptl_proc->proc_lock);
if(ptl_proc->proc_addr_count == ptl_proc->proc_peer_count) {
OMPI_THREAD_UNLOCK(&ptl_proc->proc_lock);
return OMPI_ERR_UNREACH;
}
/* The ptl_proc datastructure is shared by all TCP PTL
* instances that are trying to reach this destination.
* Cache the peer instance on the ptl_proc.
*/
ptl_peer = OBJ_NEW(mca_ptl_ib_peer_t);
if(NULL == ptl_peer) {
OMPI_THREAD_UNLOCK(&ptl_proc->proc_lock);
return OMPI_ERR_OUT_OF_RESOURCE;
}
ptl_peer->peer_ptl = (mca_ptl_ib_t*)ptl;
/*
rc = mca_ptl_ib_proc_insert(ptl_proc, ptl_peer);
if(rc != OMPI_SUCCESS) {
OBJ_RELEASE(ptl_peer);
OMPI_THREAD_UNLOCK(&ptl_proc->proc_lock);
return rc;
}
*/
ompi_bitmap_set_bit(reachable, i);
OMPI_THREAD_UNLOCK(&ptl_proc->proc_lock);
peers[i] = ptl_peer;
}
return OMPI_SUCCESS;
}
int mca_ptl_ib_del_procs(struct mca_ptl_t* ptl,
size_t nprocs,
struct ompi_proc_t **procs,
struct mca_ptl_base_peer_t ** peers)
{
/* Stub */
fprintf(stderr,"[%s][%d]\n", __FILE__, __LINE__);
return OMPI_SUCCESS;
}
int mca_ptl_ib_finalize(struct mca_ptl_t* ptl)
{
/* Stub */
fprintf(stderr,"[%s][%d]\n", __FILE__, __LINE__);
return OMPI_SUCCESS;
}
int mca_ptl_ib_request_alloc(struct mca_ptl_t* ptl,
struct mca_pml_base_send_request_t** request)
{
/* Stub */
fprintf(stderr,"[%s][%d]\n", __FILE__, __LINE__);
return OMPI_SUCCESS;
}
void mca_ptl_ib_request_return(struct mca_ptl_t* ptl,
struct mca_pml_base_send_request_t* request)
{
/* Stub */
fprintf(stderr,"[%s][%d]\n", __FILE__, __LINE__);
}
/*
* Initiate a send. If this is the first fragment, use the fragment
* descriptor allocated with the send requests, otherwise obtain
* one from the free list. Initialize the fragment and foward
* on to the peer.
*/
int mca_ptl_ib_send(
struct mca_ptl_t* ptl,
struct mca_ptl_base_peer_t* ptl_peer,
struct mca_pml_base_send_request_t* sendreq,
size_t offset,
size_t size,
int flags)
{
/* Stub */
fprintf(stderr,"[%s][%d]\n", __FILE__, __LINE__);
return OMPI_SUCCESS;
}
/*
* A posted receive has been matched - if required send an
* ack back to the peer and process the fragment.
*/
void mca_ptl_ib_matched(
mca_ptl_t* ptl,
mca_ptl_base_recv_frag_t* frag)
{
fprintf(stderr,"[%s][%d]\n", __FILE__, __LINE__);
/* Stub */
}

258
src/mca/ptl/ib/src/ptl_ib.h Обычный файл
Просмотреть файл

@ -0,0 +1,258 @@
/*
* $HEADER$
*/
/**
* @file
*/
#ifndef MCA_PTL_IB_H
#define MCA_PTL_IB_H
/* Standard system includes */
#include <sys/types.h>
#include <stdint.h>
#include <string.h>
/* Open MPI includes */
#include "class/ompi_free_list.h"
#include "event/event.h"
#include "mca/pml/pml.h"
#include "mca/ptl/ptl.h"
#include "util/output.h"
/* InfiniBand VAPI includes */
#include "ptl_ib_vapi.h"
#include "ptl_ib_addr.h"
#include "ptl_ib_proc.h"
#include "ptl_ib_peer.h"
/* Other IB ptl includes */
#include "ptl_ib_sendreq.h"
#include "ptl_ib_recvfrag.h"
/**
* IB PTL module.
*/
struct mca_ptl_ib_module_1_0_0_t {
mca_ptl_base_module_1_0_0_t super; /**< base PTL module */
struct mca_ptl_ib_t** ib_ptls; /**< array of available PTLs */
uint32_t ib_num_ptls; /**< number of ptls actually used */
uint32_t ib_max_ptls; /**< maximum number of ptls */
int ib_free_list_num; /**< initial size of free lists */
int ib_free_list_max; /**< maximum size of free lists */
int ib_free_list_inc; /**< number of elements to alloc when growing free lists */
ompi_free_list_t ib_send_requests; /**< free list of ib send requests -- sendreq + IB */
ompi_free_list_t ib_recv_frags; /**< free list of ib recv fragments */
ompi_event_t ib_send_event; /**< event structure for sends */
ompi_event_t ib_recv_event; /**< event structure for recvs */
ompi_mutex_t ib_lock; /**< lock for accessing module state */
uint32_t ib_num_hcas; /* number of hcas available to the IB module */
};
typedef struct mca_ptl_ib_module_1_0_0_t mca_ptl_ib_module_1_0_0_t;
typedef struct mca_ptl_ib_module_1_0_0_t mca_ptl_ib_module_t;
struct mca_ptl_ib_recv_frag_t;
extern mca_ptl_ib_module_1_0_0_t mca_ptl_ib_module;
/**
* IB PTL Interface
*/
struct mca_ptl_ib_t {
mca_ptl_t super; /**< base PTL interface */
VAPI_hca_id_t hca_id; /* ID of HCA this PTL is tied to */
VAPI_hca_port_t port; /* InfiniBand port of this PTL */
VAPI_hca_hndl_t nic; /* NIC handle */
VAPI_pd_hndl_t ptag; /* Protection Domain tag */
VAPI_cq_hndl_t cq_hndl; /* Completion Queue handle */
VAPI_qp_hndl_t *qp_hndl; /* Array of Queue Pair handles */
VAPI_qp_hndl_t ud_scq_hndl;/* UD send completion queue handle */
VAPI_qp_hndl_t ud_rcq_hndl;/* US recv completion queue handle */
VAPI_qp_hndl_t ud_qp_hndl; /* UD queue pair handle */
VAPI_qp_prop_t ud_qp_prop; /* UD queue pair properties */
VAPI_rr_desc_t* ud_rr_hndl; /* UD receive descriptor pool */
};
typedef struct mca_ptl_ib_t mca_ptl_ib_t;
extern mca_ptl_ib_t mca_ptl_ib;
/**
* Register IB module parameters with the MCA framework
*/
extern int mca_ptl_ib_module_open(void);
/**
* Any final cleanup before being unloaded.
*/
extern int mca_ptl_ib_module_close(void);
/**
* IB module initialization.
*
* @param num_ptls (OUT) Number of PTLs returned in PTL array.
* @param allow_multi_user_threads (OUT) Flag indicating wether PTL supports user threads (TRUE)
* @param have_hidden_threads (OUT) Flag indicating wether PTL uses threads (TRUE)
*
* (1) read interface list from kernel and compare against module parameters
* then create a PTL instance for selected interfaces
* (2) setup IB listen socket for incoming connection attempts
* (3) publish PTL addressing info
*
*/
extern mca_ptl_t** mca_ptl_ib_module_init(
int *num_ptls,
bool *allow_multi_user_threads,
bool *have_hidden_threads
);
/**
* IB module control.
*/
extern int mca_ptl_ib_module_control(
int param,
void* value,
size_t size
);
/**
* IB module progress.
*/
extern int mca_ptl_ib_module_progress(
mca_ptl_tstamp_t tstamp
);
/**
* Cleanup any resources held by the PTL.
*
* @param ptl PTL instance.
* @return OMPI_SUCCESS or error status on failure.
*/
extern int mca_ptl_ib_finalize(
struct mca_ptl_t* ptl
);
/**
* PML->PTL notification of change in the process list.
*
* @param ptl (IN)
* @param nprocs (IN) Number of processes
* @param procs (IN) Set of processes
* @param peers (OUT) Set of (optional) peer addressing info.
* @param peers (IN/OUT) Set of processes that are reachable via this PTL.
* @return OMPI_SUCCESS or error status on failure.
*
*/
extern int mca_ptl_ib_add_procs(
struct mca_ptl_t* ptl,
size_t nprocs,
struct ompi_proc_t **procs,
struct mca_ptl_base_peer_t** peers,
ompi_bitmap_t* reachable
);
/**
* PML->PTL notification of change in the process list.
*
* @param ptl (IN) PTL instance
* @param nproc (IN) Number of processes.
* @param procs (IN) Set of processes.
* @param peers (IN) Set of peer data structures.
* @return Status indicating if cleanup was successful
*
*/
extern int mca_ptl_ib_del_procs(
struct mca_ptl_t* ptl,
size_t nprocs,
struct ompi_proc_t **procs,
struct mca_ptl_base_peer_t** peers
);
/**
* PML->PTL Allocate a send request from the PTL modules free list.
*
* @param ptl (IN) PTL instance
* @param request (OUT) Pointer to allocated request.
* @return Status indicating if allocation was successful.
*
*/
extern int mca_ptl_ib_request_alloc(
struct mca_ptl_t* ptl,
struct mca_pml_base_send_request_t**
);
/**
* PML->PTL Return a send request to the PTL modules free list.
*
* @param ptl (IN) PTL instance
* @param request (IN) Pointer to allocated request.
*
*/
extern void mca_ptl_ib_request_return(
struct mca_ptl_t* ptl,
struct mca_pml_base_send_request_t*
);
/**
* PML->PTL Notification that a receive fragment has been matched.
*
* @param ptl (IN) PTL instance
* @param recv_frag (IN) Receive fragment
*
*/
extern void mca_ptl_ib_matched(
struct mca_ptl_t* ptl,
struct mca_ptl_base_recv_frag_t* frag
);
/**
* PML->PTL Initiate a send of the specified size.
*
* @param ptl (IN) PTL instance
* @param ptl_base_peer (IN) PTL peer addressing
* @param send_request (IN/OUT) Send request (allocated by PML via mca_ptl_base_request_alloc_fn_t)
* @param size (IN) Number of bytes PML is requesting PTL to deliver
* @param flags (IN) Flags that should be passed to the peer via the message header.
* @param request (OUT) OMPI_SUCCESS if the PTL was able to queue one or more fragments
*/
extern int mca_ptl_ib_send(
struct mca_ptl_t* ptl,
struct mca_ptl_base_peer_t* ptl_peer,
struct mca_pml_base_send_request_t*,
size_t offset,
size_t size,
int flags
);
/**
* Return a recv fragment to the modules free list.
*
* @param ptl (IN) PTL instance
* @param frag (IN) IB receive fragment
*
*/
extern void mca_ptl_ib_recv_frag_return(
struct mca_ptl_t* ptl,
struct mca_ptl_ib_recv_frag_t* frag
);
/**
* Return a send fragment to the modules free list.
*
* @param ptl (IN) PTL instance
* @param frag (IN) IB send fragment
*
*/
extern void mca_ptl_ib_send_frag_return(
struct mca_ptl_t* ptl,
struct mca_ptl_ib_send_frag_t*
);
#endif

15
src/mca/ptl/ib/src/ptl_ib_addr.h Обычный файл
Просмотреть файл

@ -0,0 +1,15 @@
#ifndef MCA_PTL_IB_ADDR_H
#define MCA_PTL_IB_ADDR_H
#include "ptl_ib.h"
/* Structure for publishing the InfiniBand
* Unreliable Datagram address to peers */
struct mca_ptl_ib_ud_addr_t {
VAPI_qp_hndl_t ud_qp; /* UD qp hndl to be published */
IB_lid_t lid; /* Local identifier */
};
typedef struct mca_ptl_ib_ud_addr_t mca_ptl_ib_ud_addr_t;
#endif

375
src/mca/ptl/ib/src/ptl_ib_module.c Обычный файл
Просмотреть файл

@ -0,0 +1,375 @@
/*
* $HEADER$
*/
/* Open MPI includes */
#include "include/constants.h"
#include "event/event.h"
#include "util/if.h"
#include "util/argv.h"
#include "util/output.h"
#include "mca/pml/pml.h"
#include "mca/ptl/ptl.h"
#include "mca/pml/base/pml_base_sendreq.h"
#include "mca/base/mca_base_param.h"
#include "mca/base/mca_base_module_exchange.h"
/* IB ptl includes */
#include "ptl_ib.h"
#include "ptl_ib_priv.h"
mca_ptl_ib_module_1_0_0_t mca_ptl_ib_module = {
{
/* First, the mca_base_module_t struct containing meta information
about the module itself */
{
/* Indicate that we are a pml v1.0.0 module (which also implies a
specific MCA version) */
MCA_PTL_BASE_VERSION_1_0_0,
"ib", /* MCA module name */
1, /* MCA module major version */
0, /* MCA module minor version */
0, /* MCA module release version */
mca_ptl_ib_module_open, /* module open */
mca_ptl_ib_module_close /* module close */
},
/* Next the MCA v1.0.0 module meta data */
{
/* Whether the module is checkpointable or not */
false
},
mca_ptl_ib_module_init,
mca_ptl_ib_module_control,
mca_ptl_ib_module_progress,
}
};
/*
* functions for receiving event callbacks
*/
static void mca_ptl_ib_module_recv_handler(int, short, void*);
/*
* utility routines for parameter registration
*/
static inline char* mca_ptl_ib_param_register_string(
const char* param_name,
const char* default_value)
{
char *param_value;
int id = mca_base_param_register_string("ptl","ib",param_name,NULL,default_value);
mca_base_param_lookup_string(id, &param_value);
return param_value;
}
static inline int mca_ptl_ib_param_register_int(
const char* param_name,
int default_value)
{
int id = mca_base_param_register_int("ptl","ib",param_name,NULL,default_value);
int param_value = default_value;
mca_base_param_lookup_int(id,&param_value);
return param_value;
}
/*
* Called by MCA framework to open the module, registers
* module parameters.
*/
int mca_ptl_ib_module_open(void)
{
fprintf(stderr,"[%s:%d] %s\n",
__FILE__, __LINE__, __func__);
fflush(stderr);
/* register super module parameters */
mca_ptl_ib.super.ptl_exclusivity =
mca_ptl_ib_param_register_int ("exclusivity", 0);
mca_ptl_ib.super.ptl_first_frag_size =
mca_ptl_ib_param_register_int ("first_frag_size",
(2048 - sizeof(mca_ptl_base_header_t))/*magic*/);
mca_ptl_ib.super.ptl_min_frag_size =
mca_ptl_ib_param_register_int ("min_frag_size",
(2048 - sizeof(mca_ptl_base_header_t))/*magic*/);
mca_ptl_ib.super.ptl_max_frag_size =
mca_ptl_ib_param_register_int ("max_frag_size", 2<<30);
/* register IB module parameters */
mca_ptl_ib_module.ib_free_list_num =
mca_ptl_ib_param_register_int ("free_list_num", 32);
mca_ptl_ib_module.ib_free_list_max =
mca_ptl_ib_param_register_int ("free_list_max", 1024);
mca_ptl_ib_module.ib_free_list_inc =
mca_ptl_ib_param_register_int ("free_list_inc", 32);
return OMPI_SUCCESS;
}
/*
* module cleanup - sanity checking of queue lengths
*/
int mca_ptl_ib_module_close(void)
{
fprintf(stderr,"[%s][%d]\n", __FILE__, __LINE__);
/* Stub */
return OMPI_SUCCESS;
}
/*
* Register IB module addressing information. The MCA framework
* will make this available to all peers.
*/
static int mca_ptl_ib_module_send(void)
{
int i, rc, size;
mca_ptl_ib_ud_addr_t* ud_qp_addr = NULL;
size = sizeof(mca_ptl_ib_ud_addr_t) * mca_ptl_ib_module.ib_num_ptls;
ud_qp_addr = (mca_ptl_ib_ud_addr_t*) malloc(size);
if(NULL == ud_qp_addr) {
return OMPI_ERROR;
}
for(i = 0; i < mca_ptl_ib_module.ib_num_ptls; i++) {
mca_ptl_ib_t* ptl = mca_ptl_ib_module.ib_ptls[i];
ud_qp_addr[i].ud_qp = ptl->ud_qp_hndl;
ud_qp_addr[i].lid = ptl->port.lid;
}
rc = mca_base_modex_send(&mca_ptl_ib_module.super.ptlm_version,
ud_qp_addr, size);
free(ud_qp_addr);
return rc;
}
/*
* IB module initialization:
* (1) read interface list from kernel and compare against module parameters
* then create a PTL instance for selected interfaces
* (2) setup IB listen socket for incoming connection attempts
* (3) register PTL parameters with the MCA
*/
mca_ptl_t** mca_ptl_ib_module_init(int *num_ptls,
bool *allow_multi_user_threads,
bool *have_hidden_threads)
{
mca_ptl_t **ptls;
int i, rc;
uint32_t num_hcas;
VAPI_ret_t ret;
VAPI_hca_id_t* hca_id = NULL;
mca_ptl_ib_t* ptl_ib = NULL;
*num_ptls = 0;
VAPI_cqe_num_t act_num_cqe;
act_num_cqe = 0;
*allow_multi_user_threads = true;
*have_hidden_threads = OMPI_HAVE_THREADS;
fprintf(stderr,"[%s:%d] %s\n",
__FILE__, __LINE__, __func__);
/* need to set ompi_using_threads() as ompi_event_init()
* will spawn a thread if supported */
if(OMPI_HAVE_THREADS) {
ompi_set_using_threads(true);
}
if((rc = ompi_event_init()) != OMPI_SUCCESS) {
ompi_output(0, "mca_ptl_ib_module_init: "
"unable to initialize event dispatch thread: %d\n", rc);
return NULL;
}
#if 0
/* initialize free lists */
ompi_free_list_init(&mca_ptl_ib_module.ib_send_requests,
sizeof(mca_ptl_ib_send_request_t),
OBJ_CLASS(mca_ptl_ib_send_request_t),
mca_ptl_ib_module.ib_free_list_num,
mca_ptl_ib_module.ib_free_list_max,
mca_ptl_ib_module.ib_free_list_inc,
NULL); /* use default allocator */
ompi_free_list_init(&mca_ptl_ib_module.ib_recv_frags,
sizeof(mca_ptl_ib_recv_frag_t),
OBJ_CLASS(mca_ptl_ib_recv_frag_t),
mca_ptl_ib_module.ib_free_list_num,
mca_ptl_ib_module.ib_free_list_max,
mca_ptl_ib_module.ib_free_list_inc,
NULL); /* use default allocator */
#endif
/* List all HCAs */
ret = EVAPI_list_hcas(0, &num_hcas, NULL);
/* Don't check for return status, it will be
* VAPI_EAGAIN, we are just trying to get the
* number of HCAs */
if (0 == num_hcas) {
return NULL;
}
hca_id = (VAPI_hca_id_t*) malloc(sizeof(VAPI_hca_id_t) * num_hcas);
if(NULL == hca_id) {
return NULL;
}
/* HACK: To avoid confusion, right now open only
* one IB PTL */
/*mca_ptl_ib_module.ib_num_hcas = num_hcas;*/
mca_ptl_ib_module.ib_num_hcas = 1;
mca_ptl_ib_module.ib_num_ptls = 1;
mca_ptl_ib_module.ib_max_ptls = 1;
/* Now get the hca_id from underlying VAPI layer */
ret = EVAPI_list_hcas(mca_ptl_ib_module.ib_num_hcas,
&num_hcas, hca_id);
/* HACK : Don't check return status now,
* just opening one ptl ... */
/*MCA_PTL_IB_VAPI_RET(NULL, ret, "EVAPI_list_hcas"); */
/* Number of PTLs are equal to number of HCAs */
ptl_ib = (mca_ptl_ib_t*) malloc(sizeof(mca_ptl_ib_t) *
mca_ptl_ib_module.ib_num_ptls);
if(NULL == ptl_ib) {
return NULL;
}
/* Copy the function pointers to the IB ptls */
for(i = 0; i< mca_ptl_ib_module.ib_num_ptls; i++) {
memcpy((void*)&ptl_ib[i],
&mca_ptl_ib,
sizeof(mca_ptl_ib));
}
/* Allocate list of IB ptl pointers */
mca_ptl_ib_module.ib_ptls = (struct mca_ptl_ib_t**)
malloc(mca_ptl_ib_module.ib_num_ptls *
sizeof(struct mca_ptl_ib_t*));
if(NULL == mca_ptl_ib_module.ib_ptls) {
return NULL;
}
/* Set the pointers for all IB ptls */
for(i = 0; i < mca_ptl_ib_module.ib_num_ptls; i++) {
mca_ptl_ib_module.ib_ptls[i] = &ptl_ib[i];
}
/* Open the HCAs asscociated with ptls */
for(i = 0; i < mca_ptl_ib_module.ib_num_ptls; i++) {
strncpy(ptl_ib[i].hca_id, hca_id[i],
sizeof(VAPI_hca_id_t));
/* Open the HCA */
ret = EVAPI_get_hca_hndl(ptl_ib[i].hca_id,
&ptl_ib[i].nic);
MCA_PTL_IB_VAPI_RET(NULL, ret, "EVAPI_get_hca_hndl");
/* Querying for port properties */
ret = VAPI_query_hca_port_prop(ptl_ib[i].nic,
(IB_port_t)DEFAULT_PORT,
(VAPI_hca_port_t *)&(ptl_ib[i].port));
MCA_PTL_IB_VAPI_RET(NULL, ret, "VAPI_query_hca_port_prop");
}
/* Create the Completion Queue & Protection handles
* before creating the UD Queue Pair */
for(i = 0; i < mca_ptl_ib_module.ib_num_ptls; i++) {
ret = VAPI_alloc_pd(ptl_ib[i].nic, &ptl_ib[i].ptag);
MCA_PTL_IB_VAPI_RET(NULL, ret, "VAPI_alloc_pd");
ret = VAPI_create_cq(ptl_ib[i].nic, DEFAULT_CQ_SIZE,
&ptl_ib[i].cq_hndl, &act_num_cqe);
MCA_PTL_IB_VAPI_RET(NULL, ret, "VAPI_create_cq");
/* If we didn't get any CQ entries, then return
* failure */
if(act_num_cqe == 0) {
return NULL;
}
ret = mca_ptl_ib_ud_cq_init(&ptl_ib[i]);
if(ret != VAPI_OK) {
return NULL;
}
ret = mca_ptl_ib_ud_qp_init(&ptl_ib[i]);
if(ret != VAPI_OK) {
return NULL;
}
}
if(mca_ptl_ib_module_send() != OMPI_SUCCESS) {
return NULL;
}
/* Allocate list of MCA ptl pointers */
ptls = (mca_ptl_t**) malloc(mca_ptl_ib_module.ib_num_ptls *
sizeof(mca_ptl_t*));
if(NULL == ptls) {
return NULL;
}
memcpy(ptls, mca_ptl_ib_module.ib_ptls,
mca_ptl_ib_module.ib_num_ptls *
sizeof(mca_ptl_ib_t*));
*num_ptls = mca_ptl_ib_module.ib_num_ptls;
fprintf(stderr,"ptls = %p, num_ptls = %d\n",
ptls, *num_ptls);
free(hca_id);
return ptls;
}
/*
* IB module control
*/
int mca_ptl_ib_module_control(int param, void* value, size_t size)
{
/* Stub */
fprintf(stderr,"[%s][%d]\n", __FILE__, __LINE__);
return OMPI_SUCCESS;
}
/*
* IB module progress.
*/
int mca_ptl_ib_module_progress(mca_ptl_tstamp_t tstamp)
{
/* Stub */
fprintf(stderr,"[%s][%d]\n", __FILE__, __LINE__);
return OMPI_SUCCESS;
}

56
src/mca/ptl/ib/src/ptl_ib_peer.h Обычный файл
Просмотреть файл

@ -0,0 +1,56 @@
#ifndef MCA_PTL_IB_PEER_H
#define MCA_PTL_IB_PEER_H
#include "class/ompi_list.h"
#include "event/event.h"
#include "mca/pml/pml.h"
#include "mca/ptl/ptl.h"
#include "ptl_ib_recvfrag.h"
#include "ptl_ib_sendfrag.h"
/**
* State of IB peer connection.
*/
typedef enum {
MCA_PTL_IB_CLOSED,
MCA_PTL_IB_CONNECTING,
MCA_PTL_IB_CONNECT_ACK,
MCA_PTL_IB_CONNECTED,
MCA_PTL_IB_FAILED
} mca_ptl_ib_state_t;
/**
* An abstraction that represents a connection to a peer process.
* An instance of mca_ptl_base_peer_t is associated w/ each process
* and PTL pair at startup. However, connections to the peer
* are established dynamically on an as-needed basis:
*/
struct mca_ptl_base_peer_t {
ompi_list_item_t super;
struct mca_ptl_ib_t* peer_ptl; /**< PTL instance that created this connection */
struct mca_ptl_ib_proc_t* peer_proc; /**< proc structure corresponding to peer */
struct mca_ptl_ib_addr_t* peer_addr; /**< address of peer */
int peer_sd; /**< socket connection to peer */
mca_ptl_ib_send_frag_t* peer_send_frag; /**< current send frag being processed */
mca_ptl_ib_recv_frag_t* peer_recv_frag; /**< current recv frag being processed */
mca_ptl_ib_state_t peer_state; /**< current state of the connection */
size_t peer_retries; /**< number of connection retries attempted */
ompi_list_t peer_frags; /**< list of pending frags to send */
ompi_mutex_t peer_send_lock; /**< lock for concurrent access to peer state */
ompi_mutex_t peer_recv_lock; /**< lock for concurrent access to peer state */
ompi_event_t peer_send_event; /**< event for async processing of send frags */
ompi_event_t peer_recv_event; /**< event for async processing of recv frags */
};
typedef struct mca_ptl_base_peer_t mca_ptl_base_peer_t;
typedef struct mca_ptl_base_peer_t mca_ptl_ib_peer_t;
/*
extern ompi_class_t mca_ptl_ib_peer_t_class;
*/
OBJ_CLASS_DECLARATION(mca_ptl_ib_peer_t);
#endif

129
src/mca/ptl/ib/src/ptl_ib_priv.c Обычный файл
Просмотреть файл

@ -0,0 +1,129 @@
#include "ptl_ib_vapi.h"
#include "ptl_ib.h"
#include "ptl_ib_priv.h"
VAPI_ret_t mca_ptl_ib_ud_cq_init(mca_ptl_ib_t* ptl_ib)
{
VAPI_ret_t ret;
VAPI_cqe_num_t act_num_cqe = 0;
ret = VAPI_create_cq(ptl_ib->nic, DEFAULT_CQ_SIZE,
&(ptl_ib->ud_scq_hndl), &act_num_cqe);
MCA_PTL_IB_VAPI_RET(ret, ret, "VAPI_create_cq");
if(act_num_cqe == 0) {
/* Couldn't give any CQ entries, not
* enough resources */
return VAPI_EAGAIN;
}
/* Send completion queue was allocated successfully,
* proceed to allocate receive completion queue */
act_num_cqe = 0;
ret = VAPI_create_cq(ptl_ib->nic, DEFAULT_CQ_SIZE,
&(ptl_ib->ud_rcq_hndl), &act_num_cqe);
MCA_PTL_IB_VAPI_RET(ret, ret, "VAPI_create_cq");
if(act_num_cqe == 0) {
/* Couldn't give any CQ entries, not
* enough resources */
return VAPI_EAGAIN;
}
return VAPI_OK;
}
/* Set up UD Completion Queue and Queue pair */
VAPI_ret_t mca_ptl_ib_ud_qp_init(mca_ptl_ib_t* ptl_ib)
{
VAPI_qp_init_attr_t qp_init_attr;
VAPI_qp_attr_t qp_attr;
VAPI_qp_cap_t qp_cap;
VAPI_qp_attr_mask_t qp_attr_mask;
VAPI_ret_t ret;
qp_init_attr.cap.max_oust_wr_rq = DEFAULT_UD_WQ_SIZE;
qp_init_attr.cap.max_oust_wr_sq = DEFAULT_UD_WQ_SIZE;
qp_init_attr.cap.max_sg_size_rq = DEFAULT_UD_SG_LIST;
qp_init_attr.cap.max_sg_size_sq = DEFAULT_UD_SG_LIST;
qp_init_attr.pd_hndl = ptl_ib->ptag;
/* We don't have Reliable Datagram Handle right now */
qp_init_attr.rdd_hndl = 0;
/* Set Send and Recv completion queues */
qp_init_attr.rq_cq_hndl = ptl_ib->ud_rcq_hndl;
qp_init_attr.sq_cq_hndl = ptl_ib->ud_scq_hndl;
/* Signal all work requests on this queue pair */
qp_init_attr.rq_sig_type = VAPI_SIGNAL_ALL_WR;
qp_init_attr.sq_sig_type = VAPI_SIGNAL_ALL_WR;
/* Use Unreliable Datagram transport service */
qp_init_attr.ts_type = VAPI_TS_UD;
ret = VAPI_create_qp(ptl_ib->nic, &qp_init_attr,
&(ptl_ib->ud_qp_hndl), &(ptl_ib->ud_qp_prop));
MCA_PTL_IB_VAPI_RET(ret, ret, "VAPI_create_qp");
D_PRINT("UD QP[%d] created ..hndl=%d\n",
ptl_ib->ud_qp_prop.qp_num,
(int)ptl_ib->ud_qp_hndl);
/* Modifying QP to INIT */
QP_ATTR_MASK_CLR_ALL(qp_attr_mask);
qp_attr.qp_state = VAPI_INIT;
QP_ATTR_MASK_SET(qp_attr_mask,QP_ATTR_QP_STATE);
qp_attr.pkey_ix = DEFAULT_PKEY_IX;
QP_ATTR_MASK_SET(qp_attr_mask,QP_ATTR_PKEY_IX);
qp_attr.port = DEFAULT_PORT;
QP_ATTR_MASK_SET(qp_attr_mask,QP_ATTR_PORT);
qp_attr.qkey = 0;
QP_ATTR_MASK_SET(qp_attr_mask,QP_ATTR_QKEY);
ret = VAPI_modify_qp(ptl_ib->nic,
ptl_ib->ud_qp_hndl, &qp_attr,
&qp_attr_mask, &qp_cap);
MCA_PTL_IB_VAPI_RET(ret, ret, "VAPI_modify_qp");
D_PRINT("Modified UD to init..Qp\n");
/*****************
* INIT --> RTR
*****************/
QP_ATTR_MASK_CLR_ALL(qp_attr_mask);
qp_attr.qp_state = VAPI_RTR;
QP_ATTR_MASK_SET(qp_attr_mask,QP_ATTR_QP_STATE);
ret = VAPI_modify_qp(ptl_ib->nic,
ptl_ib->ud_qp_hndl, &qp_attr,
&qp_attr_mask, &qp_cap);
MCA_PTL_IB_VAPI_RET(ret, ret, "VAPI_modify_qp");
D_PRINT("Modified UD to RTR..Qp\n");
/*********************
* RTR --> RTS
*********************/
QP_ATTR_MASK_CLR_ALL(qp_attr_mask);
qp_attr.qp_state = VAPI_RTS;
QP_ATTR_MASK_SET(qp_attr_mask,QP_ATTR_QP_STATE);
qp_attr.sq_psn = DEFAULT_PSN;
QP_ATTR_MASK_SET(qp_attr_mask,QP_ATTR_SQ_PSN);
ret = VAPI_modify_qp(ptl_ib->nic,
ptl_ib->ud_qp_hndl, &qp_attr,
&qp_attr_mask, &qp_cap);
MCA_PTL_IB_VAPI_RET(ret, ret, "VAPI_modify_qp");
D_PRINT("Modified UD to RTS..Qp\n");
/* Everything was fine ... return success! */
return VAPI_OK;
}

10
src/mca/ptl/ib/src/ptl_ib_priv.h Обычный файл
Просмотреть файл

@ -0,0 +1,10 @@
#ifndef MCA_PTL_IB_PRIV_H
#define MCA_PTL_IB_PRIV_H
#include "ptl_ib_vapi.h"
#include "ptl_ib.h"
VAPI_ret_t mca_ptl_ib_ud_cq_init(mca_ptl_ib_t*);
VAPI_ret_t mca_ptl_ib_ud_qp_init(mca_ptl_ib_t*);
#endif

147
src/mca/ptl/ib/src/ptl_ib_proc.c Обычный файл
Просмотреть файл

@ -0,0 +1,147 @@
#include "include/atomic.h"
#include "class/ompi_hash_table.h"
#include "mca/base/mca_base_module_exchange.h"
#include "ptl_ib.h"
#include "ptl_ib_vapi.h"
#include "ptl_ib_proc.h"
/*
* Look for an existing IB process instances based on the associated
* ompi_proc_t instance.
*/
/*
mca_ptl_ib_proc_t* mca_ptl_ib_proc_lookup_ompi(ompi_proc_t* ompi_proc)
{
mca_ptl_ib_proc_t* ib_proc;
OMPI_THREAD_LOCK(&mca_ptl_ib_module.ib_lock);
for(ib_proc = (mca_ptl_ib_proc_t*)
ompi_list_get_first(&mca_ptl_ib_module.ib_procs);
ib_proc != (mca_ptl_ib_proc_t*)
ompi_list_get_end(&mca_ptl_ib_module.ib_procs);
ib_proc = (mca_ptl_ib_proc_t*)ompi_list_get_next(ib_proc)) {
if(ib_proc->proc_ompi == ompi_proc) {
OMPI_THREAD_UNLOCK(&mca_ptl_ib_module.ib_lock);
return ib_proc;
}
}
OMPI_THREAD_UNLOCK(&mca_ptl_ib_module.ib_lock);
return NULL;
}
*/
/*
* Create a IB process structure. There is a one-to-one correspondence
* between a ompi_proc_t and a mca_ptl_ib_proc_t instance. We cache
* additional data (specifically the list of mca_ptl_ib_peer_t instances,
* and published addresses) associated w/ a given destination on this
* datastructure.
*/
mca_ptl_ib_proc_t* mca_ptl_ib_proc_create(ompi_proc_t* ompi_proc)
{
int rc;
size_t size;
mca_ptl_ib_proc_t* ptl_proc = NULL;
fprintf(stderr,"[%s:%d] %s\n",
__FILE__, __LINE__, __func__);
/*
mca_ptl_ib_proc_t* ptl_proc =
mca_ptl_ib_proc_lookup_ompi(ompi_proc);
if(ptl_proc != NULL) {
return ptl_proc;
}
*/
ptl_proc = OBJ_NEW(mca_ptl_ib_proc_t);
ptl_proc->proc_ompi = ompi_proc;
/* build a unique identifier (of arbitrary
* size) to represent the proc */
ptl_proc->proc_guid = ompi_proc->proc_name;
/* lookup ib parameters exported by
* this proc */
rc = mca_base_modex_recv(
&mca_ptl_ib_module.super.ptlm_version,
ompi_proc,
(void**)&ptl_proc->proc_addrs,
&size);
if(rc != OMPI_SUCCESS) {
ompi_output(0, "mca_ptl_ib_proc_create: mca_base_modex_recv: "
"failed with return value=%d", rc);
OBJ_RELEASE(ptl_proc);
return NULL;
}
if(0 != (size % sizeof(mca_ptl_ib_ud_addr_t))) {
ompi_output(0, "mca_ptl_ib_proc_create: mca_base_modex_recv: "
"invalid size %d\n", size);
return NULL;
}
ptl_proc->proc_addr_count = size / sizeof(mca_ptl_ib_ud_addr_t);
/* allocate space for peer array - one for
* each exported address
*/
ptl_proc->proc_peers = (mca_ptl_base_peer_t**)
malloc(ptl_proc->proc_addr_count * sizeof(mca_ptl_base_peer_t*));
if(NULL == ptl_proc->proc_peers) {
OBJ_RELEASE(ptl_proc);
return NULL;
}
return ptl_proc;
}
/*
* Note that this routine must be called with the lock on the process
* already held. Insert a ptl instance into the proc array and assign
* it an address.
*/
int mca_ptl_ib_proc_insert(mca_ptl_ib_proc_t* ptl_proc,
mca_ptl_base_peer_t* ptl_peer)
{
struct mca_ptl_ib_t *ptl_ib = ptl_peer->peer_ptl;
int i;
/* insert into peer array */
ptl_peer->peer_proc = ptl_proc;
ptl_proc->proc_peers[ptl_proc->proc_peer_count++] = ptl_peer;
/*
* Look through the proc instance for an address that is on the
* directly attached network. If we don't find one, pick the first
* unused address.
*/
for(i=0; i<ptl_proc->proc_addr_count; i++) {
/*
mca_ptl_ib_ud_addr_t* peer_addr = ptl_proc->proc_addrs + i;
*/
#if 0
unsigned long net1 = ptl_ib->ptl_ifaddr.sin_addr.s_addr & ptl_ib->ptl_ifmask.sin_addr.s_addr;
unsigned long net2 = peer_addr->addr_inet.s_addr & ptl_ib->ptl_ifmask.sin_addr.s_addr;
if(peer_addr->addr_inuse != 0)
continue;
if(net1 == net2) {
ptl_peer->peer_addr = peer_addr;
break;
} else if(ptl_peer->peer_addr != 0)
ptl_peer->peer_addr = peer_addr;
#endif
}
/*
ptl_peer->peer_addr->addr_inuse++;
*/
return OMPI_SUCCESS;
}

56
src/mca/ptl/ib/src/ptl_ib_proc.h Обычный файл
Просмотреть файл

@ -0,0 +1,56 @@
#ifndef MCA_PTL_IB_PROC_H
#define MCA_PTL_IB_PROC_H
#include "mca/ns/ns.h"
#include "class/ompi_object.h"
#include "proc/proc.h"
#include "ptl_ib.h"
#include "ptl_ib_vapi.h"
#include "ptl_ib_addr.h"
#include "ptl_ib_peer.h"
/*
extern ompi_class_t mca_ptl_ib_proc_t_class;
*/
OBJ_CLASS_DECLARATION(mca_ptl_ib_proc_t);
/**
* Represents the state of a remote process and the set of addresses
* that it exports. Also cache an instance of mca_ptl_base_peer_t for
* each
* PTL instance that attempts to open a connection to the process.
*/
struct mca_ptl_ib_proc_t {
ompi_list_item_t super;
/**< allow proc to be placed on a list */
ompi_proc_t *proc_ompi;
/**< pointer to corresponding ompi_proc_t */
ompi_process_name_t proc_guid;
/**< globally unique identifier for the process */
struct mca_ptl_ib_ud_addr_t *proc_addrs;
/**< array of addresses published by peer */
size_t proc_addr_count;
/**< number of addresses published by peer */
struct mca_ptl_base_peer_t **proc_peers;
/**< array of peers that have been created to access this proc */
size_t proc_peer_count;
/**< number of peers */
ompi_mutex_t proc_lock;
/**< lock to protect against concurrent access to proc state */
};
typedef struct mca_ptl_ib_proc_t mca_ptl_ib_proc_t;
mca_ptl_ib_proc_t* mca_ptl_ib_proc_create(ompi_proc_t* ompi_proc);
/*
mca_ptl_ib_proc_t* mca_ptl_ib_proc_lookup(ompi_process_name_t*);
*/
int mca_ptl_ib_proc_insert(mca_ptl_ib_proc_t*, mca_ptl_base_peer_t*);
#endif

18
src/mca/ptl/ib/src/ptl_ib_recvfrag.h Обычный файл
Просмотреть файл

@ -0,0 +1,18 @@
#ifndef MCA_PTL_IB_RECV_FRAG_H
#define MCA_PTL_IB_RECV_FRAG_H
#include "os/atomic.h"
#include "mca/ptl/ptl.h"
#include "mca/ptl/base/ptl_base_recvfrag.h"
OBJ_CLASS_DECLARATION(mca_ptl_ib_recv_frag_t);
/**
* IB received fragment derived type.
*/
struct mca_ptl_ib_recv_frag_t {
mca_ptl_base_recv_frag_t super; /**< base receive fragment descriptor */
};
typedef struct mca_ptl_ib_recv_frag_t mca_ptl_ib_recv_frag_t;
#endif

24
src/mca/ptl/ib/src/ptl_ib_sendfrag.h Обычный файл
Просмотреть файл

@ -0,0 +1,24 @@
#ifndef MCA_PTL_IB_SEND_FRAG_H
#define MCA_PTL_IB_SEND_FRAG_H
#include "os/atomic.h"
#include "ompi_config.h"
#include "mca/pml/base/pml_base_sendreq.h"
#include "mca/ptl/base/ptl_base_sendfrag.h"
#include "ptl_ib_vapi.h"
extern ompi_class_t mca_ptl_ib_send_frag_t_class;
struct mca_ptl_base_peer_t;
/**
* IB send fragment derived type.
*/
struct mca_ptl_ib_send_frag_t {
mca_ptl_base_send_frag_t super; /**< base send fragment descriptor */
};
typedef struct mca_ptl_ib_send_frag_t mca_ptl_ib_send_frag_t;
#endif

26
src/mca/ptl/ib/src/ptl_ib_sendreq.h Обычный файл
Просмотреть файл

@ -0,0 +1,26 @@
#ifndef MCA_PTL_IB_SEND_REQUEST_H
#define MCA_PTL_IB_SEND_REQUEST_H
#include "ompi_config.h"
#include "mca/pml/base/pml_base_sendreq.h"
#include "ptl_ib_sendfrag.h"
OBJ_CLASS_DECLARATION(mca_ptl_ib_send_request_t);
/**
* IB send request derived type. The send request contains both the
* base send request, and space for the first IB send fragment
* descriptor.
* This avoids the overhead of a second allocation for the initial send
* fragment on every send request.
*/
struct mca_ptl_ib_send_request_t {
mca_pml_base_send_request_t super;
mca_ptl_ib_send_frag_t req_frag; /* first fragment */
};
typedef struct mca_ptl_ib_send_request_t mca_ptl_ib_send_request_t;
#endif

46
src/mca/ptl/ib/src/ptl_ib_vapi.h Обычный файл
Просмотреть файл

@ -0,0 +1,46 @@
#ifndef MCA_PTL_IB_VAPI_H
#define MCA_PTL_IB_VAPI_H
#include <vapi.h>
#include <mpga.h>
#include <mtl_common.h>
#include <vapi_common.h>
/* HACK: Alert, these are dumb defines,
* all this stuff should be runtime. Ignoring for now.
*/
#define DEFAULT_PORT (1)
#define DEFAULT_CQ_SIZE (40000)
#define DEFAULT_UD_WQ_SIZE (10000)
#define DEFAULT_UD_SG_LIST (1)
#define DEFAULT_PKEY_IX (0)
#define DEFAULT_PSN (0)
/* This is a convinence macro.
*
* ret : The value to return if call failed
* vapi_ret : The value which was returned from the last VAPI call
* func_name : The VAPI function which was called
*/
#define MCA_PTL_IB_VAPI_RET(ret, vapi_ret, func_name) { \
if(vapi_ret != VAPI_OK) { \
ompi_output(0,"[%s:%d]", __FILE__, __LINE__); \
ompi_output(0,"%s : %s",func_name,VAPI_strerror(vapi_ret)); \
return ret; \
} \
}
/* Debug Print */
#if 1
#define D_PRINT(fmt, args...) { \
fprintf(stderr, "[%s:%d]", __FILE__, __LINE__); \
fprintf(stderr, fmt, ## args); \
fflush(stderr); \
}
#else
#define D_PRINT(fmt, args...)
#endif
#endif