Checkpoint:
- Borrow configure.m4 from the mvapi btl. One of the uDAPL headers emits a warning when -pedantic is enabled, so strip it out. - Change function check in ompi_check_dapl.m4 from dat_ia_open to dat_registry_list_providers.. dat_ia_open wasn't working right - Make the references to prepare_dst, put, and get NULL for now - Add opal_output() calls in all the udapl interface functions for debugging - Add evd_qlen component parameter to control event dispatcher queue length - First stab at component_init and module_init - Misc cleanups - whitespace, dead code removal - Update copyrights to 2006 This commit was SVN r8701.
Этот коммит содержится в:
Andrew Friedley
родитель
edfdf13096
Коммит
a4abe3bdbe
@ -49,7 +49,7 @@ AC_DEFUN([OMPI_CHECK_UDAPL],[
|
||||
OMPI_CHECK_PACKAGE([$1],
|
||||
[dat/udat.h],
|
||||
[dapl],
|
||||
[dat_ia_open],
|
||||
[dat_registry_list_providers],
|
||||
[-ldat],
|
||||
[$ompi_check_udapl_dir],
|
||||
[$ompi_check_udapl_libdir],
|
||||
|
||||
@ -1,5 +1,5 @@
|
||||
/*
|
||||
* Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
|
||||
* Copyright (c) 2004-2006 The Trustees of Indiana University and Indiana
|
||||
* University Research and Technology
|
||||
* Corporation. All rights reserved.
|
||||
* Copyright (c) 2004-2005 The University of Tennessee and The University
|
||||
@ -52,9 +52,9 @@ mca_btl_udapl_module_t mca_btl_udapl_module = {
|
||||
mca_btl_udapl_alloc,
|
||||
mca_btl_udapl_free,
|
||||
mca_btl_udapl_prepare_src,
|
||||
mca_btl_udapl_prepare_dst,
|
||||
NULL, /* prepare_dst */
|
||||
mca_btl_udapl_send,
|
||||
mca_btl_udapl_put,
|
||||
NULL, /* put */
|
||||
NULL /* get */
|
||||
}
|
||||
};
|
||||
@ -74,6 +74,8 @@ int mca_btl_udapl_add_procs(
|
||||
mca_btl_udapl_module_t* udapl_btl = (mca_btl_udapl_module_t*)btl;
|
||||
int i, rc;
|
||||
|
||||
opal_output(0, "udapl_add_procs\n");
|
||||
|
||||
for(i = 0; i < (int) nprocs; i++) {
|
||||
|
||||
struct ompi_proc_t* ompi_proc = ompi_procs[i];
|
||||
@ -124,6 +126,7 @@ int mca_btl_udapl_del_procs(struct mca_btl_base_module_t* btl,
|
||||
struct ompi_proc_t **procs,
|
||||
struct mca_btl_base_endpoint_t ** peers)
|
||||
{
|
||||
opal_output(0, "udapl_del_procs\n");
|
||||
/* TODO */
|
||||
return OMPI_SUCCESS;
|
||||
}
|
||||
@ -142,6 +145,8 @@ int mca_btl_udapl_register(
|
||||
mca_btl_udapl_module_t* udapl_btl = (mca_btl_udapl_module_t*) btl;
|
||||
udapl_btl->udapl_reg[tag].cbfunc = cbfunc;
|
||||
udapl_btl->udapl_reg[tag].cbdata = cbdata;
|
||||
|
||||
opal_output(0, "udapl_register\n");
|
||||
return OMPI_SUCCESS;
|
||||
}
|
||||
|
||||
@ -160,6 +165,8 @@ mca_btl_base_descriptor_t* mca_btl_udapl_alloc(
|
||||
mca_btl_udapl_module_t* udapl_btl = (mca_btl_udapl_module_t*) btl;
|
||||
mca_btl_udapl_frag_t* frag;
|
||||
int rc;
|
||||
|
||||
opal_output(0, "udapl_alloc\n");
|
||||
|
||||
if(size <= btl->btl_eager_limit) {
|
||||
MCA_BTL_UDAPL_FRAG_ALLOC_EAGER(udapl_btl, frag, rc);
|
||||
@ -190,7 +197,10 @@ int mca_btl_udapl_free(
|
||||
struct mca_btl_base_module_t* btl,
|
||||
mca_btl_base_descriptor_t* des)
|
||||
{
|
||||
mca_btl_udapl_frag_t* frag = (mca_btl_udapl_frag_t*)des;
|
||||
mca_btl_udapl_frag_t* frag = (mca_btl_udapl_frag_t*)des;
|
||||
|
||||
opal_output(0, "udapl_free\n");
|
||||
|
||||
if(frag->size == 0) {
|
||||
btl->btl_mpool->mpool_release(btl->btl_mpool, frag->registration);
|
||||
MCA_BTL_UDAPL_FRAG_RETURN_USER(btl, frag);
|
||||
@ -228,6 +238,8 @@ mca_btl_base_descriptor_t* mca_btl_udapl_prepare_src(
|
||||
int32_t free_after;
|
||||
int rc;
|
||||
|
||||
opal_output(0, "udapl_prepare_src\n");
|
||||
|
||||
/*
|
||||
* If the data has already been pinned and is contigous than we can
|
||||
* use it in place.
|
||||
@ -378,6 +390,8 @@ mca_btl_base_descriptor_t* mca_btl_udapl_prepare_dst(
|
||||
long lb;
|
||||
int rc;
|
||||
|
||||
opal_output(0, "udapl_prepare_dst\n");
|
||||
|
||||
MCA_BTL_UDAPL_FRAG_ALLOC_USER(btl, frag, rc);
|
||||
if(NULL == frag) {
|
||||
return NULL;
|
||||
@ -437,6 +451,7 @@ int mca_btl_udapl_send(
|
||||
mca_btl_base_tag_t tag)
|
||||
|
||||
{
|
||||
opal_output(0, "udapl_send\n");
|
||||
return OMPI_ERR_NOT_IMPLEMENTED;
|
||||
}
|
||||
|
||||
@ -455,6 +470,7 @@ int mca_btl_udapl_put(
|
||||
mca_btl_base_endpoint_t* endpoint,
|
||||
mca_btl_base_descriptor_t* des)
|
||||
{
|
||||
opal_output(0, "udapl_put\n");
|
||||
return OMPI_ERR_NOT_IMPLEMENTED;
|
||||
}
|
||||
|
||||
@ -474,6 +490,7 @@ int mca_btl_udapl_get(
|
||||
mca_btl_base_endpoint_t* endpoint,
|
||||
mca_btl_base_descriptor_t* des)
|
||||
{
|
||||
opal_output(0, "udapl_get\n");
|
||||
return OMPI_ERR_NOT_IMPLEMENTED;
|
||||
}
|
||||
|
||||
@ -486,6 +503,7 @@ int mca_btl_udapl_finalize(struct mca_btl_base_module_t* btl)
|
||||
{
|
||||
mca_btl_udapl_module_t* udapl_btl = (mca_btl_udapl_module_t*) btl;
|
||||
|
||||
opal_output(0, "udapl_finalize\n");
|
||||
OBJ_DESTRUCT(&udapl_btl->udapl_lock);
|
||||
OBJ_DESTRUCT(&udapl_btl->udapl_frag_eager);
|
||||
OBJ_DESTRUCT(&udapl_btl->udapl_frag_max);
|
||||
|
||||
@ -1,6 +1,5 @@
|
||||
|
||||
/*
|
||||
* Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
|
||||
* Copyright (c) 2004-2006 The Trustees of Indiana University and Indiana
|
||||
* University Research and Technology
|
||||
* Corporation. All rights reserved.
|
||||
* Copyright (c) 2004-2005 The University of Tennessee and The University
|
||||
@ -43,10 +42,6 @@
|
||||
extern "C" {
|
||||
#endif
|
||||
|
||||
/*
|
||||
#define GM_BUFFER_SIZE 7
|
||||
#define GM_BUFFER_LENGTH gm_max_length_for_size(GM_BUFFER_SIZE)
|
||||
*/
|
||||
|
||||
/**
|
||||
* uDAPL BTL component.
|
||||
@ -59,6 +54,7 @@ struct mca_btl_udapl_component_t {
|
||||
size_t udapl_max_btls; /**< maximum number of supported hcas */
|
||||
struct mca_btl_udapl_module_t **udapl_btls; /**< array of available BTL modules */
|
||||
size_t udapl_num_mru;
|
||||
size_t udapl_evd_qlen;
|
||||
size_t udapl_eager_frag_size;
|
||||
size_t udapl_max_frag_size;
|
||||
char* udapl_port_name;
|
||||
@ -86,11 +82,16 @@ extern mca_btl_udapl_component_t mca_btl_udapl_component;
|
||||
struct mca_btl_udapl_module_t {
|
||||
mca_btl_base_module_t super; /**< base BTL interface */
|
||||
mca_btl_base_recv_reg_t udapl_reg[256];
|
||||
|
||||
/* local port handle/address */
|
||||
/* struct gm_port *port; */
|
||||
mca_btl_udapl_addr_t udapl_addr;
|
||||
|
||||
/* interface handle */
|
||||
DAT_IA_HANDLE udapl_ia;
|
||||
|
||||
/* event dispatchers - default, data transfer, connection negotiation */
|
||||
DAT_EVD_HANDLE udapl_evd_dflt;
|
||||
DAT_EVD_HANDLE udapl_evd_dto;
|
||||
DAT_EVD_HANDLE udapl_evd_conn;
|
||||
|
||||
/* free list of fragment descriptors */
|
||||
ompi_free_list_t udapl_frag_eager;
|
||||
ompi_free_list_t udapl_frag_max;
|
||||
|
||||
@ -1,5 +1,5 @@
|
||||
/*
|
||||
* Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
|
||||
* Copyright (c) 2004-2006 The Trustees of Indiana University and Indiana
|
||||
* University Research and Technology
|
||||
* Corporation. All rights reserved.
|
||||
* Copyright (c) 2004-2005 The University of Tennessee and The University
|
||||
@ -105,6 +105,8 @@ int mca_btl_udapl_component_open(void)
|
||||
{
|
||||
int param, value;
|
||||
|
||||
opal_output(0, "udapl_component_open\n");
|
||||
|
||||
/* initialize state */
|
||||
mca_btl_udapl_component.udapl_num_btls=0;
|
||||
mca_btl_udapl_component.udapl_btls=NULL;
|
||||
@ -121,11 +123,13 @@ int mca_btl_udapl_component_open(void)
|
||||
mca_btl_udapl_component.udapl_free_list_inc =
|
||||
mca_btl_udapl_param_register_int ("free_list_inc", 8);
|
||||
mca_btl_udapl_component.udapl_debug =
|
||||
mca_btl_udapl_param_register_int("debug", 0);
|
||||
mca_btl_udapl_param_register_int("debug", 1);
|
||||
mca_btl_udapl_component.udapl_mpool_name =
|
||||
mca_btl_udapl_param_register_string("mpool", "udapl");
|
||||
mca_btl_udapl_component.udapl_max_btls =
|
||||
mca_btl_udapl_param_register_int("max_modules", 4);
|
||||
mca_btl_udapl_param_register_int("max_modules", 4);
|
||||
mca_btl_udapl_component.udapl_evd_qlen =
|
||||
mca_btl_udapl_param_register_int("evd_qlen", 8);
|
||||
mca_btl_udapl_component.udapl_num_high_priority =
|
||||
mca_btl_udapl_param_register_int("num_high_priority", 8);
|
||||
mca_btl_udapl_component.udapl_num_repost =
|
||||
@ -184,22 +188,73 @@ int mca_btl_udapl_component_open(void)
|
||||
|
||||
int mca_btl_udapl_component_close(void)
|
||||
{
|
||||
opal_output(0, "udapl_component_close\n");
|
||||
|
||||
/* TODO - clean up each btl module */
|
||||
return OMPI_SUCCESS;
|
||||
}
|
||||
|
||||
|
||||
/**
|
||||
* Report a uDAPL error - for debugging
|
||||
*/
|
||||
|
||||
static void
|
||||
mca_btl_udapl_error(DAT_RETURN ret, char* str)
|
||||
{
|
||||
char* major;
|
||||
char* minor;
|
||||
|
||||
/* don't output anything if debug is not set */
|
||||
if(0 == mca_btl_udapl_component.udapl_debug) {
|
||||
return;
|
||||
}
|
||||
|
||||
if(DAT_SUCCESS != dat_strerror(ret,
|
||||
(const char**)&major, (const char**)&minor))
|
||||
{
|
||||
printf("dat_strerror failed! ret is %d\n", ret);
|
||||
exit(-1);
|
||||
}
|
||||
|
||||
opal_output(0, "ERROR: %s %s %s\n", str, major, minor);
|
||||
}
|
||||
|
||||
|
||||
/**
|
||||
* Initialize module instance
|
||||
*/
|
||||
|
||||
#if 0
|
||||
static int
|
||||
mca_btl_udapl_module_init (mca_btl_udapl_module_t * btl)
|
||||
mca_btl_udapl_module_init (DAT_NAME_PTR ia_name,
|
||||
mca_btl_udapl_module_t * btl)
|
||||
{
|
||||
/*mca_mpool_base_resources_t resources;*/
|
||||
/*int32_t num_high_priority;
|
||||
int32_t i;
|
||||
int rc;*/
|
||||
DAT_RETURN rc;
|
||||
|
||||
/* open the uDAPL interface */
|
||||
rc = dat_ia_open(ia_name, mca_btl_udapl_component.udapl_evd_qlen,
|
||||
&btl->udapl_evd_dflt, &btl->udapl_ia);
|
||||
if(DAT_SUCCESS != rc) {
|
||||
mca_btl_udapl_error(rc, "dat_ia_open");
|
||||
return OMPI_ERROR;
|
||||
}
|
||||
|
||||
/* set up evd's */
|
||||
rc = dat_evd_create(btl->udapl_ia,
|
||||
mca_btl_udapl_component.udapl_evd_qlen, DAT_HANDLE_NULL,
|
||||
DAT_EVD_DTO_FLAG | DAT_EVD_RMR_BIND_FLAG, &btl->udapl_evd_dto);
|
||||
if(DAT_SUCCESS != rc) {
|
||||
mca_btl_udapl_error(rc, "dat_evd_create (dto)");
|
||||
return OMPI_ERROR;
|
||||
}
|
||||
|
||||
rc = dat_evd_create(btl->udapl_ia,
|
||||
mca_btl_udapl_component.udapl_evd_qlen, DAT_HANDLE_NULL,
|
||||
DAT_EVD_DTO_FLAG | DAT_EVD_RMR_BIND_FLAG, &btl->udapl_evd_conn);
|
||||
if(DAT_SUCCESS != rc) {
|
||||
mca_btl_udapl_error(rc, "dat_evd_create (conn)");
|
||||
return OMPI_ERROR;
|
||||
}
|
||||
|
||||
/* initialize objects */
|
||||
OBJ_CONSTRUCT(&btl->udapl_frag_eager, ompi_free_list_t);
|
||||
@ -209,20 +264,9 @@ mca_btl_udapl_module_init (mca_btl_udapl_module_t * btl)
|
||||
OBJ_CONSTRUCT(&btl->udapl_repost, opal_list_t);
|
||||
OBJ_CONSTRUCT(&btl->udapl_mru_reg, opal_list_t);
|
||||
OBJ_CONSTRUCT(&btl->udapl_lock, opal_mutex_t);
|
||||
|
||||
/* query nic tokens */
|
||||
|
||||
/* initialize memory pool */
|
||||
|
||||
/* initialize free lists */
|
||||
|
||||
/* post receive buffers */
|
||||
|
||||
/* enable rdma */
|
||||
|
||||
return OMPI_SUCCESS;
|
||||
}
|
||||
#endif
|
||||
|
||||
/*
|
||||
* Register uDAPL component addressing information. The MCA framework
|
||||
@ -237,6 +281,8 @@ mca_btl_udapl_modex_send(void)
|
||||
size_t size;
|
||||
mca_btl_udapl_addr_t *addrs = NULL;
|
||||
|
||||
opal_output(0, "udapl_modex_send\n");
|
||||
|
||||
size = mca_btl_udapl_component.udapl_num_btls * sizeof (mca_btl_udapl_addr_t);
|
||||
if (0 != size) {
|
||||
addrs = (mca_btl_udapl_addr_t *)malloc (size);
|
||||
@ -256,7 +302,6 @@ mca_btl_udapl_modex_send(void)
|
||||
return rc;
|
||||
}
|
||||
|
||||
|
||||
/*
|
||||
* Initialize the uDAPL component,
|
||||
* check how many interfaces are available and create a btl module for each.
|
||||
@ -269,17 +314,61 @@ mca_btl_udapl_component_init (int *num_btl_modules,
|
||||
{
|
||||
DAT_PROVIDER_INFO* datinfo;
|
||||
mca_btl_base_module_t **btls;
|
||||
*num_btl_modules = 0;
|
||||
mca_btl_udapl_module_t *btl;
|
||||
size_t i;
|
||||
|
||||
opal_output(0, "udapl_component_init\n");
|
||||
|
||||
/* enumerate uDAPL interfaces */
|
||||
datinfo = malloc(mca_btl_udapl_component.udapl_max_btls * sizeof(DAT_PROVIDER_INFO));
|
||||
if(NULL == datinfo) {
|
||||
return NULL;
|
||||
}
|
||||
if(DAT_SUCCESS != dat_registry_list_providers(mca_btl_udapl_component.udapl_max_btls,
|
||||
(DAT_COUNT*)&mca_btl_udapl_component.udapl_num_btls, &datinfo)) {
|
||||
free(datinfo);
|
||||
return NULL;
|
||||
}
|
||||
|
||||
|
||||
/* Make sure we have some interfaces */
|
||||
if(0 == mca_btl_udapl_component.udapl_num_btls) {
|
||||
mca_btl_base_error_no_nics("uDAPL", "NIC");
|
||||
free(datinfo);
|
||||
return NULL;
|
||||
}
|
||||
|
||||
/* create a BTL module for each interface */
|
||||
mca_btl_udapl_component.udapl_btls =
|
||||
malloc(mca_btl_udapl_component.udapl_num_btls *
|
||||
sizeof(mca_btl_udapl_module_t *));
|
||||
if(NULL == mca_btl_udapl_component.udapl_btls) {
|
||||
free(datinfo);
|
||||
return NULL;
|
||||
}
|
||||
|
||||
for(i = 0; i < mca_btl_udapl_component.udapl_num_btls; i++) {
|
||||
opal_output(0, "udapl creating btl for %s\n", datinfo[i].ia_name);
|
||||
|
||||
btl = malloc(sizeof(mca_btl_udapl_module_t));
|
||||
if(NULL == btl) {
|
||||
free(datinfo);
|
||||
free(mca_btl_udapl_component.udapl_btls);
|
||||
return NULL;
|
||||
}
|
||||
|
||||
/* copy default values into the new BTL */
|
||||
memcpy(btl, &mca_btl_udapl_module, sizeof(mca_btl_udapl_module_t));
|
||||
|
||||
if(OMPI_SUCCESS != mca_btl_udapl_module_init(datinfo[i].ia_name, btl)) {
|
||||
opal_output(0, "udapl module init for %s failed\n",
|
||||
datinfo[i].ia_name);
|
||||
/*TODO - how do i correctly handle an error here? */
|
||||
free(btl);
|
||||
}
|
||||
|
||||
/* successful btl creation */
|
||||
mca_btl_udapl_component.udapl_btls[i] = btl;
|
||||
}
|
||||
|
||||
/* finished with datinfo */
|
||||
free(datinfo);
|
||||
@ -304,7 +393,7 @@ mca_btl_udapl_component_init (int *num_btl_modules,
|
||||
|
||||
|
||||
/*
|
||||
* GM component progress.
|
||||
* uDAPL component progress.
|
||||
*/
|
||||
|
||||
|
||||
@ -313,6 +402,8 @@ int mca_btl_udapl_component_progress()
|
||||
static int32_t inprogress = 0;
|
||||
int count = 0;
|
||||
size_t i;
|
||||
|
||||
opal_output(0, "udapl_component_progress\n");
|
||||
|
||||
/* could get into deadlock in this case as we post recvs after callback completes */
|
||||
if(OPAL_THREAD_ADD32(&inprogress, 1) > 1) {
|
||||
|
||||
@ -1,5 +1,5 @@
|
||||
/*
|
||||
* Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
|
||||
* Copyright (c) 2004-2006 The Trustees of Indiana University and Indiana
|
||||
* University Research and Technology
|
||||
* Corporation. All rights reserved.
|
||||
* Copyright (c) 2004-2005 The University of Tennessee and The University
|
||||
@ -34,15 +34,7 @@ extern "C" {
|
||||
* Structure used to publish uDAPL id information to peers.
|
||||
*/
|
||||
struct mca_btl_udapl_addr_t {
|
||||
#if 0
|
||||
#if GM_API_VERSION > 0x200
|
||||
unsigned int global_id;
|
||||
#else
|
||||
char global_id[GM_MAX_HOST_NAME_LEN];
|
||||
#endif /* GM_API_VERSION > 0x200 */
|
||||
#endif
|
||||
unsigned int node_id;
|
||||
unsigned int port_id;
|
||||
int foo; /* placeholder to prevent division by 0 */
|
||||
};
|
||||
typedef struct mca_btl_udapl_addr_t mca_btl_udapl_addr_t;
|
||||
|
||||
|
||||
47
ompi/mca/btl/udapl/configure.m4
Обычный файл
47
ompi/mca/btl/udapl/configure.m4
Обычный файл
@ -0,0 +1,47 @@
|
||||
# -*- shell-script -*-
|
||||
#
|
||||
# Copyright (c) 2004-2006 The Trustees of Indiana University and Indiana
|
||||
# University Research and Technology
|
||||
# Corporation. All rights reserved.
|
||||
# Copyright (c) 2004-2005 The University of Tennessee and The University
|
||||
# of Tennessee Research Foundation. All rights
|
||||
# reserved.
|
||||
# Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
|
||||
# University of Stuttgart. All rights reserved.
|
||||
# Copyright (c) 2004-2005 The Regents of the University of California.
|
||||
# All rights reserved.
|
||||
# $COPYRIGHT$
|
||||
#
|
||||
# Additional copyrights may follow
|
||||
#
|
||||
# $HEADER$
|
||||
#
|
||||
|
||||
|
||||
# MCA_btl_udapl_CONFIG([action-if-can-compile],
|
||||
# [action-if-cant-compile])
|
||||
# ------------------------------------------------
|
||||
AC_DEFUN([MCA_btl_udapl_CONFIG],[
|
||||
OMPI_CHECK_UDAPL([btl_udapl],
|
||||
[btl_udapl_happy="yes"],
|
||||
[btl_udapl_happy="no"])
|
||||
|
||||
AS_IF([test "$btl_udapl_happy" = "yes"],
|
||||
[btl_udapl_WRAPPER_EXTRA_LDFLAGS="$btl_udapl_LDFLAGS"
|
||||
btl_udapl_WRAPPER_EXTRA_LIBS="$btl_udapl_LIBS"
|
||||
$1],
|
||||
[$2])
|
||||
|
||||
# Borrowed from MVAPI BTL - a data structure in the uDAPL headers
|
||||
# is not fully ISO C. Remove -pedantic to silence a warning.
|
||||
btl_udapl_CFLAGS="`echo $CFLAGS | sed 's/-pedantic//g'`"
|
||||
AS_IF([test "$btl_udapl_CFLAGS" != "$CFLAGS" -a "$btl_udapl_happy" = "yes"],
|
||||
[AC_MSG_WARN([Removed -pedantic from CFLAGS for
|
||||
uDAPL component because the uDAPL headers are not fully ISO C: $btl_udapl_CFLAGS])])
|
||||
|
||||
# substitute in the things needed to build udapl
|
||||
AC_SUBST([btl_udapl_CFLAGS])
|
||||
AC_SUBST([btl_udapl_CPPFLAGS])
|
||||
AC_SUBST([btl_udapl_LDFLAGS])
|
||||
AC_SUBST([btl_udapl_LIBS])
|
||||
])dnl
|
||||
Загрузка…
x
Ссылка в новой задаче
Block a user