diff --git a/opal/mca/btl/template/btl_template.c b/opal/mca/btl/template/btl_template.c index 7ad3777173..8387c6e239 100644 --- a/opal/mca/btl/template/btl_template.c +++ b/opal/mca/btl/template/btl_template.c @@ -10,7 +10,7 @@ * University of Stuttgart. All rights reserved. * Copyright (c) 2004-2005 The Regents of the University of California. * All rights reserved. - * Copyright (c) 2014 Los Alamos National Security, LLC. All rights + * Copyright (c) 2014-2015 Los Alamos National Security, LLC. All rights * reserved. * $COPYRIGHT$ * @@ -43,9 +43,11 @@ mca_btl_template_module_t mca_btl_template_module = { .btl_alloc = mca_btl_template_alloc, .btl_free = mca_btl_template_free, .btl_prepare_src = mca_btl_template_prepare_src, - .btl_prepare_dst = mca_btl_template_prepare_dst, .btl_send = mca_btl_template_send, .btl_put = mca_btl_template_put, + .btl_get = mca_btl_template_get, + .btl_register_mem = mca_btl_template_register_mem, + .btl_deregister_mem = mca_btl_template_deregister_mem, .btl_ft_event = mca_btl_template_ft_event } }; @@ -206,7 +208,6 @@ int mca_btl_template_free( mca_btl_base_descriptor_t* mca_btl_template_prepare_src( struct mca_btl_base_module_t* btl, struct mca_btl_base_endpoint_t* endpoint, - struct mca_mpool_base_registration_t* registration, struct opal_convertor_t* convertor, uint8_t order, size_t reserve, @@ -270,49 +271,8 @@ mca_btl_base_descriptor_t* mca_btl_template_prepare_src( frag->segment.seg_len = max_data + reserve; } - frag->base.des_local = &frag->segment; - frag->base.des_local_count = 1; - frag->base.des_flags = 0; - return &frag->base; -} - - -/** - * Prepare a descriptor for send/rdma using the supplied - * convertor. If the convertor references data that is contigous, - * the descriptor may simply point to the user buffer. Otherwise, - * this routine is responsible for allocating buffer space and - * packing if required. - * - * @param btl (IN) BTL module - * @param endpoint (IN) BTL peer addressing - * @param convertor (IN) Data type convertor - * @param reserve (IN) Additional bytes requested by upper layer to precede user data - * @param size (IN/OUT) Number of bytes to prepare (IN), number of bytes actually prepared (OUT) - */ - -mca_btl_base_descriptor_t* mca_btl_template_prepare_dst( - struct mca_btl_base_module_t* btl, - struct mca_btl_base_endpoint_t* endpoint, - struct mca_mpool_base_registration_t* registration, - struct opal_convertor_t* convertor, - uint8_t order, - size_t reserve, - size_t* size, - uint32_t flags) -{ - mca_btl_template_frag_t* frag; - - MCA_BTL_TEMPLATE_FRAG_ALLOC_USER(btl, frag); - if(OPAL_UNLIKELY(NULL == frag)) { - return NULL; - } - - frag->segment.seg_len = *size; - opal_convertor_get_current_pointer( convertor, (void**)&(frag->segment.seg_addr.pval) ); - - frag->base.des_local = &frag->segment; - frag->base.des_local_count = 1; + frag->base.des_segments = &frag->segment; + frag->base.des_segment_count = 1; frag->base.des_flags = 0; return &frag->base; } @@ -350,14 +310,13 @@ int mca_btl_template_send( * @param descriptor (IN) Description of the data to be transferred */ -int mca_btl_template_put( - mca_btl_base_module_t* btl, - mca_btl_base_endpoint_t* endpoint, - mca_btl_base_descriptor_t* descriptor) +int mca_btl_template_put (struct mca_btl_base_module_t *btl, + struct mca_btl_base_endpoint_t *endpoint, void *local_address, + uint64_t remote_address, struct mca_btl_base_registration_handle_t *local_handle, + struct mca_btl_base_registration_handle_t *remote_handle, size_t size, int flags, + int order, mca_btl_base_rdma_completion_fn_t cbfunc, void *cbcontext, void *cbdata) { /* mca_btl_template_module_t* template_btl = (mca_btl_template_module_t*) btl; */ - mca_btl_template_frag_t* frag = (mca_btl_template_frag_t*) descriptor; - frag->endpoint = endpoint; /* TODO */ return OPAL_ERR_NOT_IMPLEMENTED; } @@ -372,18 +331,64 @@ int mca_btl_template_put( * */ -int mca_btl_template_get( - mca_btl_base_module_t* btl, - mca_btl_base_endpoint_t* endpoint, - mca_btl_base_descriptor_t* descriptor) +int mca_btl_template_get (struct mca_btl_base_module_t *btl, + struct mca_btl_base_endpoint_t *endpoint, void *local_address, + uint64_t remote_address, struct mca_btl_base_registration_handle_t *local_handle, + struct mca_btl_base_registration_handle_t *remote_handle, size_t size, int flags, + int order, mca_btl_base_rdma_completion_fn_t cbfunc, void *cbcontext, void *cbdata) { /* mca_btl_template_module_t* template_btl = (mca_btl_template_module_t*) btl; */ - mca_btl_template_frag_t* frag = (mca_btl_template_frag_t*) descriptor; - frag->endpoint = endpoint; /* TODO */ return OPAL_ERR_NOT_IMPLEMENTED; } +/** + * @brief Register a memory region for put/get/atomic operations. + * + * @param btl (IN) BTL module + * @param endpoint(IN) BTL addressing information (or NULL for all endpoints) + * @param base (IN) Pointer to start of region + * @param size (IN) Size of region + * @param flags (IN) Flags indicating what operation will be performed. Valid + * values are MCA_BTL_DES_FLAGS_PUT, MCA_BTL_DES_FLAGS_GET, + * and MCA_BTL_DES_FLAGS_ATOMIC + * + * @returns a memory registration handle valid for both local and remote operations + * @returns NULL if the region could not be registered + * + * This function registers the specified region with the hardware for use with + * the btl_put, btl_get, btl_atomic_cas, btl_atomic_op, and btl_atomic_fop + * functions. Care should be taken to not hold an excessive number of registrations + * as they may use limited system/NIC resources. + */ +struct mca_btl_base_registration_handle_t *mca_btl_template_register_mem ( + struct mca_btl_base_module_t* btl, struct mca_btl_base_endpoint_t *endpoint, void *base, + size_t size, uint32_t flags) +{ + /* mca_btl_template_module_t* template_btl = (mca_btl_template_module_t*) btl; */ + /* TODO */ + return NULL; +} + +/** + * @brief Deregister a memory region + * + * @param btl (IN) BTL module region was registered with + * @param handle (IN) BTL registration handle to deregister + * + * This function deregisters the memory region associated with the specified handle. Care + * should be taken to not perform any RDMA or atomic operation on this memory region + * after it is deregistered. It is erroneous to specify a memory handle associated with + * a remote node. + */ +int mca_btl_template_deregister_mem (struct mca_btl_base_module_t* btl, + struct mca_btl_base_registration_handle_t *handle) +{ + /* mca_btl_template_module_t* template_btl = (mca_btl_template_module_t*) btl; */ + /* TODO */ + return OPAL_ERR_NOT_IMPLEMENTED; +} + /* * Cleanup/release module resources. diff --git a/opal/mca/btl/template/btl_template.h b/opal/mca/btl/template/btl_template.h index c7f6a1929c..cfe7231230 100644 --- a/opal/mca/btl/template/btl_template.h +++ b/opal/mca/btl/template/btl_template.h @@ -1,3 +1,4 @@ +/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */ /* * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana * University Research and Technology @@ -9,6 +10,8 @@ * University of Stuttgart. All rights reserved. * Copyright (c) 2004-2005 The Regents of the University of California. * All rights reserved. + * Copyright (c) 2015 Los Alamos National Security, LLC. All rights + * reserved. * $COPYRIGHT$ * * Additional copyrights may follow @@ -41,7 +44,7 @@ BEGIN_C_DECLS */ struct mca_btl_template_component_t { - mca_btl_base_component_2_0_0_t super; /**< base BTL component */ + mca_btl_base_component_3_0_0_t super; /**< base BTL component */ uint32_t template_num_btls; /**< number of hcas available to the TEMPLATE component */ @@ -187,32 +190,114 @@ extern int mca_btl_template_send( /** * Initiate an asynchronous put. + * Completion Semantics: if this function returns a 1 then the operation + * is complete. a return of OPAL_SUCCESS indicates + * the put operation has been queued with the + * network. the local_handle can not be deregistered + * until all outstanding operations on that handle + * have been completed. * - * @param btl (IN) BTL module - * @param endpoint (IN) BTL addressing information - * @param descriptor (IN) Description of the data to be transferred + * @param btl (IN) BTL module + * @param endpoint (IN) BTL addressing information + * @param local_address (IN) Local address to put from (registered) + * @param remote_address (IN) Remote address to put to (registered remotely) + * @param local_handle (IN) Registration handle for region containing + * (local_address, local_address + size) + * @param remote_handle (IN) Remote registration handle for region containing + * (remote_address, remote_address + size) + * @param size (IN) Number of bytes to put + * @param flags (IN) Flags for this put operation + * @param order (IN) Ordering + * @param cbfunc (IN) Function to call on completion (if queued) + * @param cbcontext (IN) Context for the callback + * @param cbdata (IN) Data for callback + * + * @retval OPAL_SUCCESS The descriptor was successfully queued for a put + * @retval OPAL_ERROR The descriptor was NOT successfully queued for a put + * @retval OPAL_ERR_OUT_OF_RESOURCE Insufficient resources to queue the put + * operation. Try again later + * @retval OPAL_ERR_NOT_AVAILABLE Put can not be performed due to size or + * alignment restrictions. */ - -extern int mca_btl_template_put( - struct mca_btl_base_module_t* btl, - struct mca_btl_base_endpoint_t* btl_peer, - struct mca_btl_base_descriptor_t* decriptor -); - +int mca_btl_template_put (struct mca_btl_base_module_t *btl, + struct mca_btl_base_endpoint_t *endpoint, void *local_address, + uint64_t remote_address, struct mca_btl_base_registration_handle_t *local_handle, + struct mca_btl_base_registration_handle_t *remote_handle, size_t size, int flags, + int order, mca_btl_base_rdma_completion_fn_t cbfunc, void *cbcontext, void *cbdata); /** * Initiate an asynchronous get. + * Completion Semantics: if this function returns a 1 then the operation + * is complete. a return of OPAL_SUCCESS indicates + * the get operation has been queued with the + * network. the local_handle can not be deregistered + * until all outstanding operations on that handle + * have been completed. + * + * @param btl (IN) BTL module + * @param endpoint (IN) BTL addressing information + * @param local_address (IN) Local address to put from (registered) + * @param remote_address (IN) Remote address to put to (registered remotely) + * @param local_handle (IN) Registration handle for region containing + * (local_address, local_address + size) + * @param remote_handle (IN) Remote registration handle for region containing + * (remote_address, remote_address + size) + * @param size (IN) Number of bytes to put + * @param flags (IN) Flags for this put operation + * @param order (IN) Ordering + * @param cbfunc (IN) Function to call on completion (if queued) + * @param cbcontext (IN) Context for the callback + * @param cbdata (IN) Data for callback + * + * @retval OPAL_SUCCESS The descriptor was successfully queued for a put + * @retval OPAL_ERROR The descriptor was NOT successfully queued for a put + * @retval OPAL_ERR_OUT_OF_RESOURCE Insufficient resources to queue the put + * operation. Try again later + * @retval OPAL_ERR_NOT_AVAILABLE Put can not be performed due to size or + * alignment restrictions. + */ +int mca_btl_template_get (struct mca_btl_base_module_t *btl, + struct mca_btl_base_endpoint_t *endpoint, void *local_address, + uint64_t remote_address, struct mca_btl_base_registration_handle_t *local_handle, + struct mca_btl_base_registration_handle_t *remote_handle, size_t size, int flags, + int order, mca_btl_base_rdma_completion_fn_t cbfunc, void *cbcontext, void *cbdata); + +/** + * @brief Register a memory region for put/get/atomic operations. * * @param btl (IN) BTL module - * @param endpoint (IN) BTL addressing information - * @param descriptor (IN) Description of the data to be transferred + * @param endpoint(IN) BTL addressing information (or NULL for all endpoints) + * @param base (IN) Pointer to start of region + * @param size (IN) Size of region + * @param flags (IN) Flags indicating what operation will be performed. Valid + * values are MCA_BTL_DES_FLAGS_PUT, MCA_BTL_DES_FLAGS_GET, + * and MCA_BTL_DES_FLAGS_ATOMIC + * + * @returns a memory registration handle valid for both local and remote operations + * @returns NULL if the region could not be registered + * + * This function registers the specified region with the hardware for use with + * the btl_put, btl_get, btl_atomic_cas, btl_atomic_op, and btl_atomic_fop + * functions. Care should be taken to not hold an excessive number of registrations + * as they may use limited system/NIC resources. */ - -extern int mca_btl_template_get( - struct mca_btl_base_module_t* btl, - struct mca_btl_base_endpoint_t* btl_peer, - struct mca_btl_base_descriptor_t* decriptor -); +struct mca_btl_base_registration_handle_t *mca_btl_template_register_mem ( + struct mca_btl_base_module_t* btl, struct mca_btl_base_endpoint_t *endpoint, void *base, + size_t size, uint32_t flags); + +/** + * @brief Deregister a memory region + * + * @param btl (IN) BTL module region was registered with + * @param handle (IN) BTL registration handle to deregister + * + * This function deregisters the memory region associated with the specified handle. Care + * should be taken to not perform any RDMA or atomic operation on this memory region + * after it is deregistered. It is erroneous to specify a memory handle associated with + * a remote node. + */ +int mca_btl_template_deregister_mem (struct mca_btl_base_module_t* btl, + struct mca_btl_base_registration_handle_t *handle); /** * Register a callback function that is called on receipt @@ -275,7 +360,6 @@ extern int mca_btl_template_free( mca_btl_base_descriptor_t* mca_btl_template_prepare_src( struct mca_btl_base_module_t* btl, struct mca_btl_base_endpoint_t* peer, - struct mca_mpool_base_registration_t*, struct opal_convertor_t* convertor, uint8_t order, size_t reserve, @@ -283,16 +367,6 @@ mca_btl_base_descriptor_t* mca_btl_template_prepare_src( uint32_t flags ); -extern mca_btl_base_descriptor_t* mca_btl_template_prepare_dst( - struct mca_btl_base_module_t* btl, - struct mca_btl_base_endpoint_t* peer, - struct mca_mpool_base_registration_t*, - struct opal_convertor_t* convertor, - uint8_t order, - size_t reserve, - size_t* size, - uint32_t flags); - /** * Fault Tolerance Event Notification Function * @param state Checkpoint Stae