From 02d5381c2580fbdbaf682ead3863abbc97ad44bb Mon Sep 17 00:00:00 2001
From: Mitch Sukalski <mwsukal@ca.sandia.gov>
Date: Thu, 9 Dec 2004 16:28:38 +0000
Subject: [PATCH] updated IB memory registry support with new public interfaces
 for the other IB PTL code

This commit was SVN r3762.
---
 src/mca/ptl/ib/src/ptl_ib_memory.c | 45 +++++++++++++++++++++++++++++-
 src/mca/ptl/ib/src/ptl_ib_memory.h | 31 ++++++++++++++++++++
 2 files changed, 75 insertions(+), 1 deletion(-)

diff --git a/src/mca/ptl/ib/src/ptl_ib_memory.c b/src/mca/ptl/ib/src/ptl_ib_memory.c
index 1275651400..f5707d54d8 100644
--- a/src/mca/ptl/ib/src/ptl_ib_memory.c
+++ b/src/mca/ptl/ib/src/ptl_ib_memory.c
@@ -142,7 +142,7 @@ mca_ptl_ib_mem_registry_info_t *mca_ptl_ib_mem_registry_register(
         /* create new entry and register memory region */
         item = (ompi_list_item_t *)info;
         OMPI_FREE_LIST_GET(&(registry->info_free_list), item, rc);
-        if (info == (mca_ptl_ib_mem_registry_info_t *)NULL) {
+        if (OMPI_SUCCESS != rc) {
             /* error - return null pointer */
             return info;
         }
@@ -165,9 +165,14 @@ mca_ptl_ib_mem_registry_info_t *mca_ptl_ib_mem_registry_register(
                     item = (ompi_list_item_t *)info;
                     OMPI_FREE_LIST_RETURN(&(registry->info_free_list), item);
                     info = NULL;
+                    return info;
                 }
             }
         } while ((VAPI_OK != vapi_result) && (NULL != info));
+        /* insert a reference to this information into the red/black tree */
+        rc = ompi_rb_tree_insert(&(registry->rb_tree), &(info->reply), info);
+        /* aargh! what do we do if the tree insert fails... */
+        mca_ptl_ib_mem_registry_insert_hint(registry, &(info->reply), info);
     }
     else {
         (info->ref_cnt)++;
@@ -176,6 +181,44 @@ mca_ptl_ib_mem_registry_info_t *mca_ptl_ib_mem_registry_register(
     return info;
 }
 
+mca_ptl_ib_mem_registry_info_t *mca_ptl_ib_register_mem_with_registry(
+    mca_ptl_ib_state_t *ib_state,
+    void *addr, size_t len)
+{
+    mca_ptl_ib_mem_registry_info_t *info;
+    VAPI_mr_t mr;
+
+    mr.acl = VAPI_EN_LOCAL_WRITE | VAPI_EN_REMOTE_WRITE;
+    mr.l_key = 0;
+    mr.r_key = 0;
+    mr.pd_hndl = ib_state->ptag;
+    mr.size = len;
+    mr.start = (VAPI_virt_addr_t) (MT_virt_addr_t) addr;
+    mr.type = VAPI_MR;
+
+    info = mca_ptl_ib_mem_registry_register(&(ib_state->mem_registry),&mr);
+    return info;
+}
+
+int mca_ptl_ib_deregister_mem_with_registry(
+    mca_ptl_ib_state_t *ib_state,
+    void *addr, size_t len)
+{
+    VAPI_mr_t mr;
+    int rc;
+
+    mr.acl = VAPI_EN_LOCAL_WRITE | VAPI_EN_REMOTE_WRITE;
+    mr.l_key = 0;
+    mr.r_key = 0;
+    mr.pd_hndl = ib_state->ptag;
+    mr.size = len;
+    mr.start = (VAPI_virt_addr_t) (MT_virt_addr_t) addr;
+    mr.type = VAPI_MR;
+
+    rc =  mca_ptl_ib_mem_registry_deregister(&(ib_state->mem_registry),&mr);
+    return rc;
+}
+
 static int mca_ptl_ib_mem_registry_real_deregister(
     mca_ptl_ib_mem_registry_t *registry,
     mca_ptl_ib_mem_registry_info_t *info)
diff --git a/src/mca/ptl/ib/src/ptl_ib_memory.h b/src/mca/ptl/ib/src/ptl_ib_memory.h
index 93d175c7db..411997cc82 100644
--- a/src/mca/ptl/ib/src/ptl_ib_memory.h
+++ b/src/mca/ptl/ib/src/ptl_ib_memory.h
@@ -65,6 +65,27 @@ struct mca_ptl_ib_mem_registry_t {
 
 OBJ_CLASS_DECLARATION(mca_ptl_ib_mem_registry_t);
 
+static inline void mca_ptl_ib_mem_registry_insert_hint(
+    mca_ptl_ib_mem_registry_t *registry, VAPI_mr_t *key, 
+    mca_ptl_ib_mem_registry_info_t *info)
+{
+    uint64_t hints_hash = 0, addrll;
+
+    if (registry->hints_size) {
+        addrll = (uint64_t)(key->start);
+    
+        /* calculate hash index for hints array - hash is (hints_log_size - 1) bits of key 
+         * from first non-zero least significant bit
+         */
+        hints_hash = addrll & (-addrll);
+        hints_hash = (((hints_hash << registry->hints_log_size) - hints_hash) & addrll) / 
+            hints_hash;
+
+        registry->hints[hints_hash].pval = info;
+    }
+    return;
+}
+
 /* find information on a registered memory region for a given address,
  * region size, and access permissions
  *
@@ -105,10 +126,20 @@ static inline mca_ptl_ib_mem_registry_info_t *mca_ptl_ib_mem_registry_find(
     return info;
 }
 
+/* prototypes */
+
 mca_ptl_ib_mem_registry_info_t *mca_ptl_ib_mem_registry_register(
     mca_ptl_ib_mem_registry_t *registry,
     VAPI_mr_t *mr);
 
+mca_ptl_ib_mem_registry_info_t *mca_ptl_ib_register_mem_with_registry(
+    struct mca_ptl_ib_state_t *ib_state,
+    void *addr, size_t len);
+
+int mca_ptl_ib_deregister_mem_with_registry(
+    struct mca_ptl_ib_state_t *ib_state,
+    void *addr, size_t len);
+
 int mca_ptl_ib_mem_registry_deregister(
     mca_ptl_ib_mem_registry_t *registry,
     VAPI_mr_t *mr);