oshmem: code review fixes

Signed-off-by: Alex Mikheev <alexm@mellanox.com>

oshmem/mca/memheap/base/base.h

@@ -162,24 +162,22 @@ extern int mca_memheap_seg_cmp(const void *k, const void *v);
 
 extern mca_memheap_map_t* memheap_map;
 
-static inline int map_segment_is_va_in(map_base_segment_t *s, const void *va)
+static inline int map_segment_is_va_in(map_base_segment_t *s, void *va)
 {
-    return ((uintptr_t)va >= (uintptr_t)s->va_base &&
+    return (va >= s->va_base && va < s->va_end);
-            (uintptr_t)va <  (uintptr_t)s->va_end);
 }
 
-static inline map_segment_t *memheap_find_seg(const int segno)
+static inline map_segment_t *memheap_find_seg(int segno)
 {
     return &mca_memheap_base_map.mem_segs[segno];
 }
 
-static inline int memheap_is_va_in_segment(const void *va, const int segno) 
+static inline int memheap_is_va_in_segment(void *va, int segno) 
 {
-
     return map_segment_is_va_in(&memheap_find_seg(segno)->super, va);
 }
 
-static inline int memheap_find_segnum(const void *va)
+static inline int memheap_find_segnum(void *va)
 {
     if (OPAL_LIKELY(memheap_is_va_in_segment(va, SYMB_SEG_INDEX))) {
         return SYMB_SEG_INDEX;
@@ -189,19 +187,19 @@ static inline int memheap_find_segnum(const void *va)
     return MEMHEAP_SEG_INVALID;
 }
 
-static inline void* memheap_va2rva(const void* va, const void* local_base, const void* remote_base)
+static inline void* memheap_va2rva(void* va, void* local_base, void* remote_base)
 {
     return (void*) (remote_base > local_base ?
             (uintptr_t)va + ((uintptr_t)remote_base - (uintptr_t)local_base) :
             (uintptr_t)va - ((uintptr_t)local_base - (uintptr_t)remote_base));
 }
 
-static inline void *map_segment_va2rva(mkey_segment_t *seg, const void *va)
+static inline void *map_segment_va2rva(mkey_segment_t *seg, void *va)
 {
     return memheap_va2rva(va, seg->super.va_base, seg->rva_base);
 }
 
-static inline map_base_segment_t *map_segment_find_va(map_base_segment_t *segs, size_t elem_size, const void *va) 
+static inline map_base_segment_t *map_segment_find_va(map_base_segment_t *segs, size_t elem_size, void *va) 
 {
     map_base_segment_t *rseg;
 
@@ -220,8 +218,7 @@ static inline map_base_segment_t *map_segment_find_va(map_base_segment_t *segs,
 
 void mkey_segment_init(mkey_segment_t *seg, sshmem_mkey_t *mkey, uint32_t segno);
 
-
+static inline map_segment_t *memheap_find_va(void* va)
-static inline map_segment_t *memheap_find_va(const void* va)
 {
     map_segment_t *s;
 
@@ -296,7 +293,6 @@ static inline void* mca_memheap_seg2base_va(int seg)
     return memheap_map->mem_segs[seg].super.va_base;
 }
 
-
 END_C_DECLS
 
 #endif /* MCA_MEMHEAP_BASE_H */

oshmem/mca/memheap/base/memheap_base_mkey.c

@@ -85,7 +85,7 @@ int mca_memheap_seg_cmp(const void *k, const void *v)
 
 static int pack_local_mkeys(opal_buffer_t *msg, int pe, int seg)
 {
-    int i, n, tr_id;
+    int i, n;
     sshmem_mkey_t *mkey;
 
     /* go over all transports and pack mkeys */
@@ -93,14 +93,13 @@ static int pack_local_mkeys(opal_buffer_t *msg, int pe, int seg)
     opal_dss.pack(msg, &n, 1, OPAL_UINT32);
     MEMHEAP_VERBOSE(5, "found %d transports to %d", n, pe);
     for (i = 0; i < n; i++) {
-        tr_id = i;
+        mkey = mca_memheap_base_get_mkey(mca_memheap_seg2base_va(seg), i);
-        mkey = mca_memheap_base_get_mkey(mca_memheap_seg2base_va(seg), tr_id);
         if (!mkey) {
             MEMHEAP_ERROR("seg#%d tr_id: %d failed to find local mkey",
-                          seg, tr_id);
+                          seg, i);
             return OSHMEM_ERROR;
         }
-        opal_dss.pack(msg, &tr_id, 1, OPAL_UINT32);
+        opal_dss.pack(msg, &i, 1, OPAL_UINT32);
         opal_dss.pack(msg, &mkey->va_base, 1, OPAL_UINT64);
         if (0 == mkey->va_base) {
             opal_dss.pack(msg, &mkey->u.key, 1, OPAL_UINT64);
@@ -112,7 +111,7 @@ static int pack_local_mkeys(opal_buffer_t *msg, int pe, int seg)
         }
         MEMHEAP_VERBOSE(5,
                         "seg#%d tr_id: %d %s",
-                        seg, tr_id, mca_spml_base_mkey2str(mkey));
+                        seg, i, mca_spml_base_mkey2str(mkey));
     }
     return OSHMEM_SUCCESS;
 }
@@ -470,13 +469,6 @@ static int memheap_oob_get_mkeys(int pe, uint32_t seg, sshmem_mkey_t *mkeys)
                             pe,
                             i,
                             mca_spml_base_mkey2str(&mkeys[i]));
-            int my_pe = oshmem_my_proc_id();
-            if (my_pe == 0) 
-            printf(
-                            "MKEY CALCULATED BY LOCAL SPML: pe: %d tr_id: %d %s\n",
-                            pe,
-                            i,
-                            mca_spml_base_mkey2str(&mkeys[i]));
         }
         return OSHMEM_SUCCESS;
     }
@@ -742,7 +734,7 @@ uint64_t mca_memheap_base_find_offset(int pe,
 
 int mca_memheap_base_is_symmetric_addr(const void* va)
 {
-    return (memheap_find_va(va) ? 1 : 0);
+    return (memheap_find_va((void *)va) ? 1 : 0);
 }
 
 int mca_memheap_base_detect_addr_type(void* va)

oshmem/mca/spml/ikrit/spml_ikrit.c

@@ -491,7 +491,7 @@ sshmem_mkey_t *mca_spml_ikrit_register(void* addr,
         }
         SPML_VERBOSE(5,
                      "rank %d ptl %d addr %p size %llu %s",
-		     my_rank, i, addr, (unsigned long long)size,
+                     my_rank, i, addr, (unsigned long long)size,
                      mca_spml_base_mkey2str(&mkeys[i]));
 
         mca_spml_ikrit_cache_mkeys(&mkeys[i], memheap_find_segnum(addr), my_rank, i);
@@ -687,12 +687,11 @@ static inline int mca_spml_ikrit_get_async(void *src_addr,
 
     get_req = alloc_get_req();
 
-    if (OSHMEM_SUCCESS
+    if (OSHMEM_SUCCESS != mca_spml_ikrit_get_helper(&get_req->mxm_req,
-            != mca_spml_ikrit_get_helper(&get_req->mxm_req,
+                                                    src_addr,
-                                         src_addr,
+                                                    size,
-                                         size,
+                                                    dst_addr,
-                                         dst_addr,
+                                                    src)) {
-                                         src)) {
         oshmem_shmem_abort(-1);
         return OSHMEM_ERROR;
     }
@@ -818,16 +817,6 @@ static inline int mca_spml_ikrit_put_internal(void* dst_addr,
     put_req->mxm_req.base.mq = mca_spml_ikrit.mxm_mq;
     /* request immediate responce if we are getting low on send buffers. We only get responce from remote on ack timeout.
      * Also request explicit ack once in a while  */
-#if 0
-    put_req->mxm_req.opcode = MXM_REQ_OP_PUT;
-    if (mca_spml_ikrit.free_list_max - mca_spml_ikrit.n_active_puts <= SPML_IKRIT_PUT_LOW_WATER ||
-            (mca_spml_ikrit.mxm_peers[dst]->n_active_puts + 1) % SPML_IKRIT_PACKETS_PER_SYNC == 0) {
-        put_req->mxm_req.base.flags = MXM_REQ_FLAG_SEND_SYNC;
-        need_progress = 1;
-    } else  {
-        put_req->mxm_req.base.flags = MXM_REQ_FLAG_SEND_LAZY|MXM_REQ_FLAG_SEND_SYNC;
-    }
-#endif
     put_req->mxm_req.flags = 0;
     if (mca_spml_ikrit.free_list_max - mca_spml_ikrit.n_active_puts <= SPML_IKRIT_PUT_LOW_WATER ||
             (int)opal_list_get_size(&mca_spml_ikrit.active_peers) > mca_spml_ikrit.unsync_conn_max ||
@@ -1007,11 +996,7 @@ int mca_spml_ikrit_fence(void)
         mca_spml_ikrit_mxm_fence(peer - mca_spml_ikrit.mxm_peers);
     }
 
-    while (0 < mca_spml_ikrit.n_mxm_fences) {
+    while (0 < mca_spml_ikrit.n_mxm_fences || 0 < mca_spml_ikrit.n_active_gets) {
-        opal_progress();
-    }
-
-    while (0 < mca_spml_ikrit.n_active_gets) {
         opal_progress();
     }