diff --git a/ompi/mca/btl/vader/btl_vader.c b/ompi/mca/btl/vader/btl_vader.c index 57124ce7bc..e6d550ed61 100644 --- a/ompi/mca/btl/vader/btl_vader.c +++ b/ompi/mca/btl/vader/btl_vader.c @@ -12,7 +12,7 @@ * All rights reserved. * Copyright (c) 2006-2007 Voltaire. All rights reserved. * Copyright (c) 2009 Cisco Systems, Inc. All rights reserved. - * Copyright (c) 2010-2011 Los Alamos National Security, LLC. + * Copyright (c) 2010-2012 Los Alamos National Security, LLC. * All rights reserved. * $COPYRIGHT$ * @@ -217,16 +217,8 @@ static int vader_btl_first_time_init(mca_btl_vader_t *vader_btl, int n) vader_fifo_t *my_fifos; ompi_proc_t **procs; size_t num_procs; - xpmem_segid_t my_segid; int i, rc; - /* make the entire memory space of this process accessible */ - my_segid = xpmem_make (0, 0xffffffffffffffffll, XPMEM_PERMIT_MODE, - (void *)0666); - if (-1 == my_segid) { - return OMPI_ERROR; - } - rc = vader_init_mpool (vader_btl, n); if (OMPI_SUCCESS != rc) { return rc; @@ -273,7 +265,7 @@ static int vader_btl_first_time_init(mca_btl_vader_t *vader_btl, int n) /* set the base of the shared memory segment */ component->shm_bases[component->my_smp_rank] = (char *)component->vader_mpool_base; - component->shm_seg_ids[component->my_smp_rank] = my_segid; + component->shm_seg_ids[component->my_smp_rank] = component->my_seg_id; /* initialize the fifo and fast boxes "owned" by this process */ posix_memalign ((void **)&my_fifos, getpagesize (), (n + 1) * getpagesize ()); diff --git a/ompi/mca/btl/vader/btl_vader.h b/ompi/mca/btl/vader/btl_vader.h index 0533016f09..15bac7e139 100644 --- a/ompi/mca/btl/vader/btl_vader.h +++ b/ompi/mca/btl/vader/btl_vader.h @@ -12,7 +12,7 @@ * All rights reserved. * Copyright (c) 2006-2007 Voltaire. All rights reserved. * Copyright (c) 2009-2010 Cisco Systems, Inc. All rights reserved. - * Copyright (c) 2010-2011 Los Alamos National Security, LLC. + * Copyright (c) 2010-2012 Los Alamos National Security, LLC. * All rights reserved. * $COPYRIGHT$ * @@ -67,7 +67,6 @@ extern int mca_btl_vader_memcpy_limit; extern int mca_btl_vader_log_align; extern int mca_btl_vader_max_inline_send; -#define VADER_FIFO_FREE (void *) (-2) /* We can't use opal_cache_line_size here because we need a compile-time constant for padding the struct. We can't really have a compile-time constant that is portable, either (e.g., compile on @@ -102,6 +101,7 @@ struct mca_btl_vader_component_t { * shared memory */ char **shm_bases; /**< pointer to base pointers in * shared memory */ + xpmem_segid_t my_seg_id; /* this rank's xpmem segment id */ xpmem_segid_t *shm_seg_ids; /* xpmem segment ids */ struct vader_fifo_t **fifo; /**< cached copy of the pointer to * the 2D fifo array. */ @@ -145,8 +145,8 @@ OMPI_MODULE_DECLSPEC extern mca_btl_vader_t mca_btl_vader; * we define macros to translate between relative addresses and * virtual addresses. */ -#define VIRTUAL2RELATIVE(VADDR ) ((long)(VADDR) - (long)mca_btl_vader_component.shm_bases[mca_btl_vader_component.my_smp_rank]) -#define RELATIVE2VIRTUAL(OFFSET) ((long)(OFFSET) + (long)mca_btl_vader_component.shm_bases[mca_btl_vader_component.my_smp_rank]) +#define VIRTUAL2RELATIVE(VADDR ) ((intptr_t)(VADDR) - (intptr_t)mca_btl_vader_component.shm_bases[mca_btl_vader_component.my_smp_rank]) +#define RELATIVE2VIRTUAL(OFFSET) ((intptr_t)(OFFSET) + (intptr_t)mca_btl_vader_component.shm_bases[mca_btl_vader_component.my_smp_rank]) /* look up the remote pointer in the peer rcache and attach if * necessary */ diff --git a/ompi/mca/btl/vader/btl_vader_component.c b/ompi/mca/btl/vader/btl_vader_component.c index d66d7810d8..a648712b6e 100644 --- a/ompi/mca/btl/vader/btl_vader_component.c +++ b/ompi/mca/btl/vader/btl_vader_component.c @@ -12,7 +12,7 @@ * All rights reserved. * Copyright (c) 2006-2007 Voltaire. All rights reserved. * Copyright (c) 2009-2010 Cisco Systems, Inc. All rights reserved. - * Copyright (c) 2010-2011 Los Alamos National Security, LLC. + * Copyright (c) 2010-2012 Los Alamos National Security, LLC. * All rights reserved. * Copyright (c) 2011 NVIDIA Corporation. All rights reserved. * $COPYRIGHT$ @@ -241,6 +241,14 @@ static mca_btl_base_module_t **mca_btl_vader_component_init (int *num_btls, return NULL; } + /* create an xpmem segment for the entire memory space */ + component->my_seg_id = xpmem_make (0, 0xffffffffffffffffll, XPMEM_PERMIT_MODE, + (void *)0666); + if (-1 == component->my_seg_id) { + free (btls); + return NULL; + } + *num_btls = 1; /* get pointer to the btls */ @@ -303,15 +311,11 @@ static int mca_btl_vader_component_progress (void) mca_btl_vader_progress_sends (); /* poll the fifo once */ - hdr = (mca_btl_vader_hdr_t *) vader_fifo_read (fifo); - if (VADER_FIFO_FREE == hdr) { + hdr = vader_fifo_read (fifo); + if (NULL == hdr) { return 0; } - /* change the address from address relative to the shared - * memory address, to a true virtual address */ - hdr = (mca_btl_vader_hdr_t *) RELATIVE2VIRTUAL(hdr); - reg = mca_btl_base_active_message_trigger + hdr->tag; frag.base.des_dst = segments; diff --git a/ompi/mca/btl/vader/btl_vader_fifo.h b/ompi/mca/btl/vader/btl_vader_fifo.h index fb72ef4ffa..0fdd3e24c0 100644 --- a/ompi/mca/btl/vader/btl_vader_fifo.h +++ b/ompi/mca/btl/vader/btl_vader_fifo.h @@ -12,7 +12,7 @@ * All rights reserved. * Copyright (c) 2006-2007 Voltaire. All rights reserved. * Copyright (c) 2009-2010 Cisco Systems, Inc. All rights reserved. - * Copyright (c) 2010-2011 Los Alamos National Security, LLC. + * Copyright (c) 2010-2012 Los Alamos National Security, LLC. * All rights reserved. * $COPYRIGHT$ * @@ -30,6 +30,8 @@ #include "btl_vader_endpoint.h" #include "btl_vader_frag.h" +#define VADER_FIFO_FREE ((intptr_t)-2) + /* * Shared Memory FIFOs * @@ -48,9 +50,9 @@ /* lock free fifo */ struct vader_fifo_t { - volatile void *fifo_head; - volatile void *fifo_tail; - char pad[VADER_CACHE_LINE_PAD - 2 * sizeof (void *)]; + volatile intptr_t fifo_head; + volatile intptr_t fifo_tail; + char pad[VADER_CACHE_LINE_PAD - 2 * sizeof (intptr_t)]; }; typedef struct vader_fifo_t vader_fifo_t; @@ -61,47 +63,45 @@ static inline int vader_fifo_init (vader_fifo_t *fifo) return OMPI_SUCCESS; } -static inline void vader_fifo_write (void *value, vader_fifo_t *fifo) +static inline void vader_fifo_write (mca_btl_vader_hdr_t *hdr, int rank) { - mca_btl_vader_hdr_t *hdr; - void *prev; + vader_fifo_t *fifo = mca_btl_vader_component.fifo[rank]; + intptr_t prev, value = VIRTUAL2RELATIVE(hdr); - opal_atomic_rmb (); - - hdr = (mca_btl_vader_hdr_t *) RELATIVE2VIRTUAL(value); hdr->next = VADER_FIFO_FREE; - prev = (void *) opal_atomic_swap_ptr (&fifo->fifo_tail, (intptr_t) value); + prev = opal_atomic_swap_ptr (&fifo->fifo_tail, value); - opal_atomic_wmb (); + opal_atomic_rmb (); if (OPAL_LIKELY(VADER_FIFO_FREE != prev)) { hdr = (mca_btl_vader_hdr_t *) RELATIVE2VIRTUAL(prev); hdr->next = value; } else { - fifo->fifo_head = value; + fifo->fifo_head = value; } opal_atomic_wmb (); } -static inline void *vader_fifo_read (vader_fifo_t *fifo) +static inline mca_btl_vader_hdr_t *vader_fifo_read (vader_fifo_t *fifo) { mca_btl_vader_hdr_t *hdr; - void *value; + intptr_t value; opal_atomic_rmb (); - value = (void *) opal_atomic_swap_ptr (&fifo->fifo_head, (intptr_t) VADER_FIFO_FREE); + value = opal_atomic_swap_ptr (&fifo->fifo_head, VADER_FIFO_FREE); if (VADER_FIFO_FREE == value) { /* fifo is empty or we lost the race with another thread */ - return value; + return NULL; } hdr = (mca_btl_vader_hdr_t *) RELATIVE2VIRTUAL(value); if (OPAL_UNLIKELY(VADER_FIFO_FREE == hdr->next)) { - if (!opal_atomic_cmpset_ptr (&fifo->fifo_tail, value, VADER_FIFO_FREE)) { + if (!opal_atomic_cmpset_ptr (&fifo->fifo_tail, (void *)value, + (void *)VADER_FIFO_FREE)) { while (VADER_FIFO_FREE == hdr->next) { opal_atomic_rmb (); } @@ -114,7 +114,7 @@ static inline void *vader_fifo_read (vader_fifo_t *fifo) opal_atomic_wmb (); - return value; + return hdr; } #else /* defined(OPAL_HAVE_ATOMIC_SWAP_64) */ diff --git a/ompi/mca/btl/vader/btl_vader_frag.h b/ompi/mca/btl/vader/btl_vader_frag.h index fb927aef1e..5079099161 100644 --- a/ompi/mca/btl/vader/btl_vader_frag.h +++ b/ompi/mca/btl/vader/btl_vader_frag.h @@ -12,7 +12,7 @@ * All rights reserved. * Copyright (c) 2008 Sun Microsystems, Inc. All rights reserved. * Copyright (c) 2009 Cisco Systems, Inc. All rights reserved. - * Copyright (c) 2011 Los Alamos National Security, LLC. All rights + * Copyright (c) 2011-2012 Los Alamos National Security, LLC. All rights * reserved. * $COPYRIGHT$ * @@ -31,7 +31,7 @@ #define MCA_BTL_VADER_FLAG_FBOX 2 struct mca_btl_vader_hdr_t { - volatile void *next; /* next item in fifo. many peers may touch this */ + volatile uintptr_t next; /* next item in fifo. many peers may touch this */ volatile bool complete; /* fragment completion (usually 1 byte) */ mca_btl_base_tag_t tag; /* tag associated with this fragment (used to lookup callback) */ char pad[2]; @@ -48,8 +48,7 @@ struct mca_btl_vader_frag_t { mca_btl_base_descriptor_t base; mca_btl_base_segment_t segment; struct mca_btl_base_endpoint_t *endpoint; - /* pointer written to the FIFO, this is the base of the shared memory region */ - mca_btl_vader_hdr_t *hdr; + mca_btl_vader_hdr_t *hdr; /* in the shared memory region */ ompi_free_list_t *my_list; }; diff --git a/ompi/mca/btl/vader/btl_vader_send.c b/ompi/mca/btl/vader/btl_vader_send.c index 10025b59ea..6829f03329 100644 --- a/ompi/mca/btl/vader/btl_vader_send.c +++ b/ompi/mca/btl/vader/btl_vader_send.c @@ -12,7 +12,7 @@ * All rights reserved. * Copyright (c) 2006-2007 Voltaire. All rights reserved. * Copyright (c) 2009 Cisco Systems, Inc. All rights reserved. - * Copyright (c) 2010-2011 Los Alamos National Security, LLC. + * Copyright (c) 2010-2012 Los Alamos National Security, LLC. * All rights reserved. * $COPYRIGHT$ * @@ -59,8 +59,7 @@ int mca_btl_vader_send (struct mca_btl_base_module_t *btl, opal_list_append (&mca_btl_vader_component.active_sends, (opal_list_item_t *) frag); /* post the relative address of the descriptor into the peer's fifo */ - vader_fifo_write ((void *) VIRTUAL2RELATIVE(frag->hdr), - mca_btl_vader_component.fifo[endpoint->peer_smp_rank]); + vader_fifo_write (frag->hdr, endpoint->peer_smp_rank); if (frag->hdr->flags & MCA_BTL_VADER_FLAG_SINGLE_COPY) { frag->base.des_flags |= MCA_BTL_DES_SEND_ALWAYS_CALLBACK; diff --git a/ompi/mca/btl/vader/btl_vader_sendi.c b/ompi/mca/btl/vader/btl_vader_sendi.c index aafd9977b1..701a296bb6 100644 --- a/ompi/mca/btl/vader/btl_vader_sendi.c +++ b/ompi/mca/btl/vader/btl_vader_sendi.c @@ -12,7 +12,7 @@ * All rights reserved. * Copyright (c) 2006-2007 Voltaire. All rights reserved. * Copyright (c) 2009 Cisco Systems, Inc. All rights reserved. - * Copyright (c) 2010-2011 Los Alamos National Security, LLC. + * Copyright (c) 2010-2012 Los Alamos National Security, LLC. * All rights reserved. * $COPYRIGHT$ * @@ -103,8 +103,7 @@ int mca_btl_vader_sendi (struct mca_btl_base_module_t *btl, opal_list_append (&mca_btl_vader_component.active_sends, (opal_list_item_t *) frag); /* write the fragment pointer to peer's the FIFO */ - vader_fifo_write ((void *) VIRTUAL2RELATIVE(frag->hdr), - mca_btl_vader_component.fifo[endpoint->peer_smp_rank]); + vader_fifo_write (frag->hdr, endpoint->peer_smp_rank); /* the progress function will return the fragment */