re-reg from min of bases and max of bounds

add byte counting for total registered memory 

This commit was SVN r7546.

ompi/mca/mpool/gm/mpool_gm_module.c

@@ -24,7 +24,7 @@
 
 extern uint32_t mca_mpool_base_page_size;
 extern uint32_t mca_mpool_base_page_size_log; 
-
+uint64_t mca_mpool_gm_mem_registered; 
 
 /* 
  *  Initializes the mpool module.
@@ -45,6 +45,7 @@ void mca_mpool_gm_module_init(mca_mpool_gm_module_t* mpool)
     mpool->super.rcache = 
         mca_rcache_base_module_create(mca_mpool_gm_component.rcache_name);
     mpool->super.flags = MCA_MPOOL_FLAGS_MPI_ALLOC_MEM; 
+    mca_mpool_gm_mem_registered = 0;
 }
 
 
@@ -95,10 +96,13 @@ int mca_mpool_gm_register(
 
     if((rc = gm_register_memory(gm_mpool->port, reg->base, reg->bound - reg->base + 1)) != GM_SUCCESS) {
         opal_output(0, "[%s:%d] error(%d) registering gm memory\n", __FILE__, __LINE__, rc);
+        assert(0);
         OBJ_RELEASE(reg);
         
         return OMPI_ERR_OUT_OF_RESOURCE;
     }
+    mca_mpool_gm_mem_registered += reg->bound - reg->base + 1;
+
     
     OPAL_THREAD_ADD32(&reg->ref_count,1);
 
@@ -119,12 +123,20 @@ int mca_mpool_gm_register(
 int mca_mpool_gm_deregister(mca_mpool_base_module_t* mpool, 
                               mca_mpool_base_registration_t* reg)
 {
+    int rc;
     if(reg->flags & (MCA_MPOOL_FLAGS_CACHE | MCA_MPOOL_FLAGS_PERSIST)) { 
         mpool->rcache->rcache_delete(mpool->rcache, 
                                      reg, 
                                      reg->flags); 
     }
-    return mca_mpool_gm_release(mpool, reg); 
+    if((rc = mca_mpool_gm_release(mpool, reg)) != GM_SUCCESS) { 
+        opal_output(0, "[%s:%d] error(%d) deregistering gm memory\n", __FILE__, __LINE__, rc);
+        assert(0);
+        return OMPI_ERR_OUT_OF_RESOURCE;
+
+    }
+    
+    return OMPI_SUCCESS;
 }
 
 /**
@@ -184,12 +196,13 @@ int mca_mpool_gm_release(
             return OMPI_ERROR; 
         }
         /* opal_output(0,"deregistering gm memory\n"); */
+        mca_mpool_gm_mem_registered -= reg->bound - reg->base + 1;
         OBJ_RELEASE(reg);
     }
     else { 
         /* opal_output(0, "release says ref_count is %d\n", reg->ref_count);  */
     }
-
+    
     return OMPI_SUCCESS; 
 }
 

ompi/mca/pml/ob1/pml_ob1_rdma.c

@@ -73,7 +73,7 @@ size_t mca_pml_ob1_rdma_btls(
                               size,
                               &regs, 
                               &reg_cnt); 
-        
+        assert(reg_cnt <= 1);
         /* shortcut for one entry - the typical case */
         if(reg_cnt == 1) {
             mca_mpool_base_registration_t* reg  = ompi_pointer_array_get_item(&regs, 0); 
@@ -90,11 +90,15 @@ size_t mca_pml_ob1_rdma_btls(
                 num_btls_used++;
 
             /* otherwise if leave_pinned re-register */
-            } else if(mca_pml_ob1.leave_pinned) {
+            } else if( mca_pml_ob1.leave_pinned ) {
+                mca_mpool_base_registration_t reg_temp;
                 unsigned char* new_base = reg->base < base ? reg->base : base;
-                size_t new_len = (base - new_base) + size;                
-                /* printf("re-reg 2: base %p size %d new_base %p new_len %d\n", base, size, new_base, new_len);  */
+                unsigned char* new_bound = reg->bound > (base + size - 1) ? reg->bound : (base + size - 1);  
+                size_t new_len = new_bound - new_base + 1;
+                
+                /* printf("re-re5Bg 2: base %p size %d new_base %p new_len %d\n", base, size, new_base, new_len);  */
                 assert(new_len >= size);
+                reg_temp = *reg; 
                 btl_mpool->mpool_deregister(btl_mpool, reg); 
                 btl_mpool->mpool_register(btl_mpool, 
                     new_base, 
@@ -220,7 +224,7 @@ mca_mpool_base_registration_t* mca_pml_ob1_rdma_registration(
         size,
         &regs, 
         &reg_cnt); 
-   
+    assert(reg_cnt <= 1);
     for(r = 0; r < reg_cnt; r++) { 
         mca_mpool_base_registration_t* reg  = ompi_pointer_array_get_item(&regs, r); 
         size_t reg_len = reg->bound - base + 1;
@@ -256,16 +260,18 @@ mca_mpool_base_registration_t* mca_pml_ob1_rdma_registration(
        /* a registration exists but is not large enough */
        } else {
            unsigned char* new_base = largest->base < base ? largest->base: base;
-           size_t new_len = (base - new_base) + size;
+           unsigned char* new_bound = largest->bound > (base + size - 1) ? largest->bound : (base + size - 1);  
+           size_t new_len = new_bound - new_base + 1;
+           
            /* printf("re-reg 2: base %p size %d new_base %p new_len %d\n", base, size, new_base, new_len);  */
-
-           btl_mpool->mpool_deregister(btl_mpool, largest); 
+           
+           btl_mpool->mpool_deregister(btl_mpool, largest);
            assert(new_len >= size);
-           btl_mpool->mpool_register(btl_mpool, 
-               new_base, 
-               new_len, 
+           btl_mpool->mpool_register(btl_mpool,
+               new_base,
+               new_len,
                MCA_MPOOL_FLAGS_CACHE,
-               &fit); 
+               &fit);
            assert(fit->ref_count >= 3);
         }
     }