From 05e6e51fec1296379ff7ef141fc23f83925d572c Mon Sep 17 00:00:00 2001 From: Galen Shipman Date: Thu, 29 Sep 2005 21:28:54 +0000 Subject: [PATCH] re-reg from min of bases and max of bounds add byte counting for total registered memory This commit was SVN r7546. --- ompi/mca/mpool/gm/mpool_gm_module.c | 19 +++++++++++++++--- ompi/mca/pml/ob1/pml_ob1_rdma.c | 30 +++++++++++++++++------------ 2 files changed, 34 insertions(+), 15 deletions(-) diff --git a/ompi/mca/mpool/gm/mpool_gm_module.c b/ompi/mca/mpool/gm/mpool_gm_module.c index 262f50f3d6..8ca82e92cf 100644 --- a/ompi/mca/mpool/gm/mpool_gm_module.c +++ b/ompi/mca/mpool/gm/mpool_gm_module.c @@ -24,7 +24,7 @@ extern uint32_t mca_mpool_base_page_size; extern uint32_t mca_mpool_base_page_size_log; - +uint64_t mca_mpool_gm_mem_registered; /* * Initializes the mpool module. @@ -45,6 +45,7 @@ void mca_mpool_gm_module_init(mca_mpool_gm_module_t* mpool) mpool->super.rcache = mca_rcache_base_module_create(mca_mpool_gm_component.rcache_name); mpool->super.flags = MCA_MPOOL_FLAGS_MPI_ALLOC_MEM; + mca_mpool_gm_mem_registered = 0; } @@ -95,10 +96,13 @@ int mca_mpool_gm_register( if((rc = gm_register_memory(gm_mpool->port, reg->base, reg->bound - reg->base + 1)) != GM_SUCCESS) { opal_output(0, "[%s:%d] error(%d) registering gm memory\n", __FILE__, __LINE__, rc); + assert(0); OBJ_RELEASE(reg); return OMPI_ERR_OUT_OF_RESOURCE; } + mca_mpool_gm_mem_registered += reg->bound - reg->base + 1; + OPAL_THREAD_ADD32(®->ref_count,1); @@ -119,12 +123,20 @@ int mca_mpool_gm_register( int mca_mpool_gm_deregister(mca_mpool_base_module_t* mpool, mca_mpool_base_registration_t* reg) { + int rc; if(reg->flags & (MCA_MPOOL_FLAGS_CACHE | MCA_MPOOL_FLAGS_PERSIST)) { mpool->rcache->rcache_delete(mpool->rcache, reg, reg->flags); } - return mca_mpool_gm_release(mpool, reg); + if((rc = mca_mpool_gm_release(mpool, reg)) != GM_SUCCESS) { + opal_output(0, "[%s:%d] error(%d) deregistering gm memory\n", __FILE__, __LINE__, rc); + assert(0); + return OMPI_ERR_OUT_OF_RESOURCE; + + } + + return OMPI_SUCCESS; } /** @@ -184,12 +196,13 @@ int mca_mpool_gm_release( return OMPI_ERROR; } /* opal_output(0,"deregistering gm memory\n"); */ + mca_mpool_gm_mem_registered -= reg->bound - reg->base + 1; OBJ_RELEASE(reg); } else { /* opal_output(0, "release says ref_count is %d\n", reg->ref_count); */ } - + return OMPI_SUCCESS; } diff --git a/ompi/mca/pml/ob1/pml_ob1_rdma.c b/ompi/mca/pml/ob1/pml_ob1_rdma.c index e75af56683..55c60148f8 100644 --- a/ompi/mca/pml/ob1/pml_ob1_rdma.c +++ b/ompi/mca/pml/ob1/pml_ob1_rdma.c @@ -73,7 +73,7 @@ size_t mca_pml_ob1_rdma_btls( size, ®s, ®_cnt); - + assert(reg_cnt <= 1); /* shortcut for one entry - the typical case */ if(reg_cnt == 1) { mca_mpool_base_registration_t* reg = ompi_pointer_array_get_item(®s, 0); @@ -90,11 +90,15 @@ size_t mca_pml_ob1_rdma_btls( num_btls_used++; /* otherwise if leave_pinned re-register */ - } else if(mca_pml_ob1.leave_pinned) { + } else if( mca_pml_ob1.leave_pinned ) { + mca_mpool_base_registration_t reg_temp; unsigned char* new_base = reg->base < base ? reg->base : base; - size_t new_len = (base - new_base) + size; - /* printf("re-reg 2: base %p size %d new_base %p new_len %d\n", base, size, new_base, new_len); */ + unsigned char* new_bound = reg->bound > (base + size - 1) ? reg->bound : (base + size - 1); + size_t new_len = new_bound - new_base + 1; + + /* printf("re-re5Bg 2: base %p size %d new_base %p new_len %d\n", base, size, new_base, new_len); */ assert(new_len >= size); + reg_temp = *reg; btl_mpool->mpool_deregister(btl_mpool, reg); btl_mpool->mpool_register(btl_mpool, new_base, @@ -220,7 +224,7 @@ mca_mpool_base_registration_t* mca_pml_ob1_rdma_registration( size, ®s, ®_cnt); - + assert(reg_cnt <= 1); for(r = 0; r < reg_cnt; r++) { mca_mpool_base_registration_t* reg = ompi_pointer_array_get_item(®s, r); size_t reg_len = reg->bound - base + 1; @@ -256,16 +260,18 @@ mca_mpool_base_registration_t* mca_pml_ob1_rdma_registration( /* a registration exists but is not large enough */ } else { unsigned char* new_base = largest->base < base ? largest->base: base; - size_t new_len = (base - new_base) + size; + unsigned char* new_bound = largest->bound > (base + size - 1) ? largest->bound : (base + size - 1); + size_t new_len = new_bound - new_base + 1; + /* printf("re-reg 2: base %p size %d new_base %p new_len %d\n", base, size, new_base, new_len); */ - - btl_mpool->mpool_deregister(btl_mpool, largest); + + btl_mpool->mpool_deregister(btl_mpool, largest); assert(new_len >= size); - btl_mpool->mpool_register(btl_mpool, - new_base, - new_len, + btl_mpool->mpool_register(btl_mpool, + new_base, + new_len, MCA_MPOOL_FLAGS_CACHE, - &fit); + &fit); assert(fit->ref_count >= 3); } }