ports/openmpi
ports/openmpi
1
1
Форкнуть 0
Код Релизы Активность

btl/template: update for BTL 3.0 interface

Просмотр исходного кода 
Signed-off-by: Nathan Hjelm <hjelmn@lanl.gov>
Этот коммит содержится в:
Nathan Hjelm 2015-01-05 15:25:13 -07:00 коммит произвёл Nathan Hjelm
родитель f241b6e0a7
Коммит 4972d97b8b
2 изменённых файлов: 167 добавлений и 88 удалений
Показать статистику Скачать Patch файл Скачать Diff файл Показать все файлы Свернуть все файлы

121
opal/mca/btl/template/btl_template.c
Просмотреть файл

@ -10,7 +10,7 @@
* University of Stuttgart. All rights reserved. * University of Stuttgart. All rights reserved.
* Copyright (c) 2004-2005 The Regents of the University of California. * Copyright (c) 2004-2005 The Regents of the University of California.
* All rights reserved. * All rights reserved.
* Copyright (c) 2014 Los Alamos National Security, LLC. All rights * Copyright (c) 2014-2015 Los Alamos National Security, LLC. All rights
* reserved. * reserved.
* $COPYRIGHT$ * $COPYRIGHT$
* *
@ -43,9 +43,11 @@ mca_btl_template_module_t mca_btl_template_module = {
.btl_alloc = mca_btl_template_alloc, .btl_alloc = mca_btl_template_alloc,
.btl_free = mca_btl_template_free, .btl_free = mca_btl_template_free,
.btl_prepare_src = mca_btl_template_prepare_src, .btl_prepare_src = mca_btl_template_prepare_src,
.btl_prepare_dst = mca_btl_template_prepare_dst,
.btl_send = mca_btl_template_send, .btl_send = mca_btl_template_send,
.btl_put = mca_btl_template_put, .btl_put = mca_btl_template_put,
.btl_get = mca_btl_template_get,
.btl_register_mem = mca_btl_template_register_mem,
.btl_deregister_mem = mca_btl_template_deregister_mem,
.btl_ft_event = mca_btl_template_ft_event .btl_ft_event = mca_btl_template_ft_event
} }
}; };
@ -206,7 +208,6 @@ int mca_btl_template_free(
mca_btl_base_descriptor_t* mca_btl_template_prepare_src( mca_btl_base_descriptor_t* mca_btl_template_prepare_src(
struct mca_btl_base_module_t* btl, struct mca_btl_base_module_t* btl,
struct mca_btl_base_endpoint_t* endpoint, struct mca_btl_base_endpoint_t* endpoint,
struct mca_mpool_base_registration_t* registration,
struct opal_convertor_t* convertor, struct opal_convertor_t* convertor,
uint8_t order, uint8_t order,
size_t reserve, size_t reserve,
@ -270,49 +271,8 @@ mca_btl_base_descriptor_t* mca_btl_template_prepare_src(
frag->segment.seg_len = max_data + reserve; frag->segment.seg_len = max_data + reserve;
} }
frag->base.des_local = &frag->segment; frag->base.des_segments = &frag->segment;
frag->base.des_local_count = 1; frag->base.des_segment_count = 1;
frag->base.des_flags = 0;
return &frag->base;
}
/**
* Prepare a descriptor for send/rdma using the supplied
* convertor. If the convertor references data that is contigous,
* the descriptor may simply point to the user buffer. Otherwise,
* this routine is responsible for allocating buffer space and
* packing if required.
*
* @param btl (IN) BTL module
* @param endpoint (IN) BTL peer addressing
* @param convertor (IN) Data type convertor
* @param reserve (IN) Additional bytes requested by upper layer to precede user data
* @param size (IN/OUT) Number of bytes to prepare (IN), number of bytes actually prepared (OUT)
*/
mca_btl_base_descriptor_t* mca_btl_template_prepare_dst(
struct mca_btl_base_module_t* btl,
struct mca_btl_base_endpoint_t* endpoint,
struct mca_mpool_base_registration_t* registration,
struct opal_convertor_t* convertor,
uint8_t order,
size_t reserve,
size_t* size,
uint32_t flags)
{
mca_btl_template_frag_t* frag;
MCA_BTL_TEMPLATE_FRAG_ALLOC_USER(btl, frag);
if(OPAL_UNLIKELY(NULL == frag)) {
return NULL;
}
frag->segment.seg_len = *size;
opal_convertor_get_current_pointer( convertor, (void**)&(frag->segment.seg_addr.pval) );
frag->base.des_local = &frag->segment;
frag->base.des_local_count = 1;
frag->base.des_flags = 0; frag->base.des_flags = 0;
return &frag->base; return &frag->base;
} }
@ -350,14 +310,13 @@ int mca_btl_template_send(
* @param descriptor (IN) Description of the data to be transferred * @param descriptor (IN) Description of the data to be transferred
*/ */
int mca_btl_template_put( int mca_btl_template_put (struct mca_btl_base_module_t *btl,
mca_btl_base_module_t* btl, struct mca_btl_base_endpoint_t *endpoint, void *local_address,
mca_btl_base_endpoint_t* endpoint, uint64_t remote_address, struct mca_btl_base_registration_handle_t *local_handle,
mca_btl_base_descriptor_t* descriptor) struct mca_btl_base_registration_handle_t *remote_handle, size_t size, int flags,
int order, mca_btl_base_rdma_completion_fn_t cbfunc, void *cbcontext, void *cbdata)
{ {
/* mca_btl_template_module_t* template_btl = (mca_btl_template_module_t*) btl; */ /* mca_btl_template_module_t* template_btl = (mca_btl_template_module_t*) btl; */
mca_btl_template_frag_t* frag = (mca_btl_template_frag_t*) descriptor;
frag->endpoint = endpoint;
/* TODO */ /* TODO */
return OPAL_ERR_NOT_IMPLEMENTED; return OPAL_ERR_NOT_IMPLEMENTED;
} }
@ -372,18 +331,64 @@ int mca_btl_template_put(
* *
*/ */
int mca_btl_template_get( int mca_btl_template_get (struct mca_btl_base_module_t *btl,
mca_btl_base_module_t* btl, struct mca_btl_base_endpoint_t *endpoint, void *local_address,
mca_btl_base_endpoint_t* endpoint, uint64_t remote_address, struct mca_btl_base_registration_handle_t *local_handle,
mca_btl_base_descriptor_t* descriptor) struct mca_btl_base_registration_handle_t *remote_handle, size_t size, int flags,
int order, mca_btl_base_rdma_completion_fn_t cbfunc, void *cbcontext, void *cbdata)
{ {
/* mca_btl_template_module_t* template_btl = (mca_btl_template_module_t*) btl; */ /* mca_btl_template_module_t* template_btl = (mca_btl_template_module_t*) btl; */
mca_btl_template_frag_t* frag = (mca_btl_template_frag_t*) descriptor;
frag->endpoint = endpoint;
/* TODO */ /* TODO */
return OPAL_ERR_NOT_IMPLEMENTED; return OPAL_ERR_NOT_IMPLEMENTED;
} }
/**
* @brief Register a memory region for put/get/atomic operations.
*
* @param btl (IN) BTL module
* @param endpoint(IN) BTL addressing information (or NULL for all endpoints)
* @param base (IN) Pointer to start of region
* @param size (IN) Size of region
* @param flags (IN) Flags indicating what operation will be performed. Valid
* values are MCA_BTL_DES_FLAGS_PUT, MCA_BTL_DES_FLAGS_GET,
* and MCA_BTL_DES_FLAGS_ATOMIC
*
* @returns a memory registration handle valid for both local and remote operations
* @returns NULL if the region could not be registered
*
* This function registers the specified region with the hardware for use with
* the btl_put, btl_get, btl_atomic_cas, btl_atomic_op, and btl_atomic_fop
* functions. Care should be taken to not hold an excessive number of registrations
* as they may use limited system/NIC resources.
*/
struct mca_btl_base_registration_handle_t *mca_btl_template_register_mem (
struct mca_btl_base_module_t* btl, struct mca_btl_base_endpoint_t *endpoint, void *base,
size_t size, uint32_t flags)
{
/* mca_btl_template_module_t* template_btl = (mca_btl_template_module_t*) btl; */
/* TODO */
return NULL;
}
/**
* @brief Deregister a memory region
*
* @param btl (IN) BTL module region was registered with
* @param handle (IN) BTL registration handle to deregister
*
* This function deregisters the memory region associated with the specified handle. Care
* should be taken to not perform any RDMA or atomic operation on this memory region
* after it is deregistered. It is erroneous to specify a memory handle associated with
* a remote node.
*/
int mca_btl_template_deregister_mem (struct mca_btl_base_module_t* btl,
struct mca_btl_base_registration_handle_t *handle)
{
/* mca_btl_template_module_t* template_btl = (mca_btl_template_module_t*) btl; */
/* TODO */
return OPAL_ERR_NOT_IMPLEMENTED;
}
/* /*
* Cleanup/release module resources. * Cleanup/release module resources.

134
opal/mca/btl/template/btl_template.h
Просмотреть файл

@ -1,3 +1,4 @@
/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
/* /*
* Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
* University Research and Technology * University Research and Technology
@ -9,6 +10,8 @@
* University of Stuttgart. All rights reserved. * University of Stuttgart. All rights reserved.
* Copyright (c) 2004-2005 The Regents of the University of California. * Copyright (c) 2004-2005 The Regents of the University of California.
* All rights reserved. * All rights reserved.
* Copyright (c) 2015 Los Alamos National Security, LLC. All rights
* reserved.
* $COPYRIGHT$ * $COPYRIGHT$
* *
* Additional copyrights may follow * Additional copyrights may follow
@ -41,7 +44,7 @@ BEGIN_C_DECLS
*/ */
struct mca_btl_template_component_t { struct mca_btl_template_component_t {
mca_btl_base_component_2_0_0_t super; /**< base BTL component */ mca_btl_base_component_3_0_0_t super; /**< base BTL component */
uint32_t template_num_btls; uint32_t template_num_btls;
/**< number of hcas available to the TEMPLATE component */ /**< number of hcas available to the TEMPLATE component */
@ -187,32 +190,114 @@ extern int mca_btl_template_send(
/** /**
* Initiate an asynchronous put. * Initiate an asynchronous put.
* Completion Semantics: if this function returns a 1 then the operation
* is complete. a return of OPAL_SUCCESS indicates
* the put operation has been queued with the
* network. the local_handle can not be deregistered
* until all outstanding operations on that handle
* have been completed.
* *
* @param btl (IN) BTL module * @param btl (IN) BTL module
* @param endpoint (IN) BTL addressing information * @param endpoint (IN) BTL addressing information
* @param descriptor (IN) Description of the data to be transferred * @param local_address (IN) Local address to put from (registered)
* @param remote_address (IN) Remote address to put to (registered remotely)
* @param local_handle (IN) Registration handle for region containing
* (local_address, local_address + size)
* @param remote_handle (IN) Remote registration handle for region containing
* (remote_address, remote_address + size)
* @param size (IN) Number of bytes to put
* @param flags (IN) Flags for this put operation
* @param order (IN) Ordering
* @param cbfunc (IN) Function to call on completion (if queued)
* @param cbcontext (IN) Context for the callback
* @param cbdata (IN) Data for callback
*
* @retval OPAL_SUCCESS The descriptor was successfully queued for a put
* @retval OPAL_ERROR The descriptor was NOT successfully queued for a put
* @retval OPAL_ERR_OUT_OF_RESOURCE Insufficient resources to queue the put
* operation. Try again later
* @retval OPAL_ERR_NOT_AVAILABLE Put can not be performed due to size or
* alignment restrictions.
*/ */
int mca_btl_template_put (struct mca_btl_base_module_t *btl,
extern int mca_btl_template_put( struct mca_btl_base_endpoint_t *endpoint, void *local_address,
struct mca_btl_base_module_t* btl, uint64_t remote_address, struct mca_btl_base_registration_handle_t *local_handle,
struct mca_btl_base_endpoint_t* btl_peer, struct mca_btl_base_registration_handle_t *remote_handle, size_t size, int flags,
struct mca_btl_base_descriptor_t* decriptor int order, mca_btl_base_rdma_completion_fn_t cbfunc, void *cbcontext, void *cbdata);
);
/** /**
* Initiate an asynchronous get. * Initiate an asynchronous get.
* Completion Semantics: if this function returns a 1 then the operation
* is complete. a return of OPAL_SUCCESS indicates
* the get operation has been queued with the
* network. the local_handle can not be deregistered
* until all outstanding operations on that handle
* have been completed.
*
* @param btl (IN) BTL module
* @param endpoint (IN) BTL addressing information
* @param local_address (IN) Local address to put from (registered)
* @param remote_address (IN) Remote address to put to (registered remotely)
* @param local_handle (IN) Registration handle for region containing
* (local_address, local_address + size)
* @param remote_handle (IN) Remote registration handle for region containing
* (remote_address, remote_address + size)
* @param size (IN) Number of bytes to put
* @param flags (IN) Flags for this put operation
* @param order (IN) Ordering
* @param cbfunc (IN) Function to call on completion (if queued)
* @param cbcontext (IN) Context for the callback
* @param cbdata (IN) Data for callback
*
* @retval OPAL_SUCCESS The descriptor was successfully queued for a put
* @retval OPAL_ERROR The descriptor was NOT successfully queued for a put
* @retval OPAL_ERR_OUT_OF_RESOURCE Insufficient resources to queue the put
* operation. Try again later
* @retval OPAL_ERR_NOT_AVAILABLE Put can not be performed due to size or
* alignment restrictions.
*/
int mca_btl_template_get (struct mca_btl_base_module_t *btl,
struct mca_btl_base_endpoint_t *endpoint, void *local_address,
uint64_t remote_address, struct mca_btl_base_registration_handle_t *local_handle,
struct mca_btl_base_registration_handle_t *remote_handle, size_t size, int flags,
int order, mca_btl_base_rdma_completion_fn_t cbfunc, void *cbcontext, void *cbdata);
/**
* @brief Register a memory region for put/get/atomic operations.
* *
* @param btl (IN) BTL module * @param btl (IN) BTL module
* @param endpoint (IN) BTL addressing information * @param endpoint(IN) BTL addressing information (or NULL for all endpoints)
* @param descriptor (IN) Description of the data to be transferred * @param base (IN) Pointer to start of region
* @param size (IN) Size of region
* @param flags (IN) Flags indicating what operation will be performed. Valid
* values are MCA_BTL_DES_FLAGS_PUT, MCA_BTL_DES_FLAGS_GET,
* and MCA_BTL_DES_FLAGS_ATOMIC
*
* @returns a memory registration handle valid for both local and remote operations
* @returns NULL if the region could not be registered
*
* This function registers the specified region with the hardware for use with
* the btl_put, btl_get, btl_atomic_cas, btl_atomic_op, and btl_atomic_fop
* functions. Care should be taken to not hold an excessive number of registrations
* as they may use limited system/NIC resources.
*/ */
struct mca_btl_base_registration_handle_t *mca_btl_template_register_mem (
extern int mca_btl_template_get( struct mca_btl_base_module_t* btl, struct mca_btl_base_endpoint_t *endpoint, void *base,
struct mca_btl_base_module_t* btl, size_t size, uint32_t flags);
struct mca_btl_base_endpoint_t* btl_peer,
struct mca_btl_base_descriptor_t* decriptor /**
); * @brief Deregister a memory region
*
* @param btl (IN) BTL module region was registered with
* @param handle (IN) BTL registration handle to deregister
*
* This function deregisters the memory region associated with the specified handle. Care
* should be taken to not perform any RDMA or atomic operation on this memory region
* after it is deregistered. It is erroneous to specify a memory handle associated with
* a remote node.
*/
int mca_btl_template_deregister_mem (struct mca_btl_base_module_t* btl,
struct mca_btl_base_registration_handle_t *handle);
/** /**
* Register a callback function that is called on receipt * Register a callback function that is called on receipt
@ -275,7 +360,6 @@ extern int mca_btl_template_free(
mca_btl_base_descriptor_t* mca_btl_template_prepare_src( mca_btl_base_descriptor_t* mca_btl_template_prepare_src(
struct mca_btl_base_module_t* btl, struct mca_btl_base_module_t* btl,
struct mca_btl_base_endpoint_t* peer, struct mca_btl_base_endpoint_t* peer,
struct mca_mpool_base_registration_t*,
struct opal_convertor_t* convertor, struct opal_convertor_t* convertor,
uint8_t order, uint8_t order,
size_t reserve, size_t reserve,
@ -283,16 +367,6 @@ mca_btl_base_descriptor_t* mca_btl_template_prepare_src(
uint32_t flags uint32_t flags
); );
extern mca_btl_base_descriptor_t* mca_btl_template_prepare_dst(
struct mca_btl_base_module_t* btl,
struct mca_btl_base_endpoint_t* peer,
struct mca_mpool_base_registration_t*,
struct opal_convertor_t* convertor,
uint8_t order,
size_t reserve,
size_t* size,
uint32_t flags);
/** /**
* Fault Tolerance Event Notification Function * Fault Tolerance Event Notification Function
* @param state Checkpoint Stae * @param state Checkpoint Stae