diff --git a/opal/config/opal_config_asm.m4 b/opal/config/opal_config_asm.m4 index 723d62df88..439cbab850 100644 --- a/opal/config/opal_config_asm.m4 +++ b/opal/config/opal_config_asm.m4 @@ -915,6 +915,28 @@ AC_DEFUN([OPAL_CONFIG_ASM],[ armv7*) ompi_cv_asm_arch="ARM" OPAL_ASM_SUPPORT_64BIT=1 + OPAL_ASM_ARM_VERSION=7 + AC_DEFINE_UNQUOTED([OPAL_ASM_ARM_VERSION], [$OPAL_ASM_ARM_VERSION], + [What ARM assembly version to use]) + OMPI_GCC_INLINE_ASSIGN='"mov %0, #0" : "=&r"(ret)' + ;; + + armv6*) + ompi_cv_asm_arch="ARM" + OPAL_ASM_SUPPORT_64BIT=0 + OPAL_ASM_ARM_VERSION=6 + AC_DEFINE_UNQUOTED([OPAL_ASM_ARM_VERSION], [$OPAL_ASM_ARM_VERSION], + [What ARM assembly version to use]) + OMPI_GCC_INLINE_ASSIGN='"mov %0, #0" : "=&r"(ret)' + ;; + + armv5*linux*|armv4*linux*) + # uses Linux kernel helpers for some atomic operations + ompi_cv_asm_arch="ARM" + OPAL_ASM_SUPPORT_64BIT=0 + OPAL_ASM_ARM_VERSION=5 + AC_DEFINE_UNQUOTED([OPAL_ASM_ARM_VERSION], [$OPAL_ASM_ARM_VERSION], + [What ARM assembly version to use]) OMPI_GCC_INLINE_ASSIGN='"mov %0, #0" : "=&r"(ret)' ;; diff --git a/opal/include/opal/sys/arm/atomic.h b/opal/include/opal/sys/arm/atomic.h index 27bf4239cf..72ae2149fe 100644 --- a/opal/include/opal/sys/arm/atomic.h +++ b/opal/include/opal/sys/arm/atomic.h @@ -18,46 +18,67 @@ * $HEADER$ */ +/* + * ARMv5 and earlier lack robust atomic operations and therefore this file uses + * Linux kernel support where needed. The kernel also provides memory barriers + * and this file uses them for ARMv5 and earlier processors, which lack the + * memory barrier instruction. These kernel functions are available on kernel + * versions 2.6.15 and greater; using them will result in undefined behavior on + * older kernels. + * See Documentation/arm/kernel_user_helpers.txt in the kernel tree for details + */ + #ifndef OMPI_SYS_ARCH_ATOMIC_H #define OMPI_SYS_ARCH_ATOMIC_H 1 #if OPAL_WANT_SMP_LOCKS +#if (OPAL_ASM_ARM_VERSION >= 7) + +#define OPAL_HAVE_ATOMIC_MEM_BARRIER 1 +/* use the DMB instruction if available... */ + #define MB() __asm__ __volatile__ ("dmb" : : : "memory") #define RMB() __asm__ __volatile__ ("dmb" : : : "memory") #define WMB() __asm__ __volatile__ ("dmb" : : : "memory") +#elif (OPAL_ASM_ARM_VERSION == 6) + +#define OPAL_HAVE_ATOMIC_MEM_BARRIER 1 +/* ...or the v6-specific equivalent... */ + +#define MB() __asm__ __volatile__ ("mcr p15, 0, r0, c7, c10, 5" : : : "memory") +#define RMB() MB() +#define WMB() MB() + +#else + +#define OPAL_HAVE_ATOMIC_MEM_BARRIER 1 +/* ...otherwise use the Linux kernel-provided barrier */ + +#define MB() (*((void (*)(void))(0xffff0fa0)))() +#define RMB() MB() +#define WMB() MB() + +#endif + #else #define MB() #define RMB() #define WMB() -#endif +#endif /* OPAL_WANT_SMP_LOCKS */ -/********************************************************************** - * - * Define constants for ARMv7 - * - *********************************************************************/ -#define OPAL_HAVE_ATOMIC_MEM_BARRIER 1 - -#define OPAL_HAVE_ATOMIC_CMPSET_32 1 - -#define OPAL_HAVE_ATOMIC_CMPSET_64 1 - -#define OPAL_HAVE_ATOMIC_MATH_32 1 -#define OPAL_HAVE_ATOMIC_ADD_32 1 -#define OPAL_HAVE_ATOMIC_SUB_32 1 - /********************************************************************** * * Memory Barriers * *********************************************************************/ -#if OMPI_GCC_INLINE_ASSEMBLY + +#if (OPAL_HAVE_ATOMIC_MEM_BARRIER == 1) static inline void opal_atomic_mb(void) @@ -79,6 +100,8 @@ void opal_atomic_wmb(void) WMB(); } +#endif + /********************************************************************** * @@ -86,6 +109,10 @@ void opal_atomic_wmb(void) * *********************************************************************/ +#if (OMPI_GCC_INLINE_ASSEMBLY && (OPAL_ASM_ARM_VERSION >= 6)) + +#define OPAL_HAVE_ATOMIC_CMPSET_32 1 +#define OPAL_HAVE_ATOMIC_MATH_32 1 static inline int opal_atomic_cmpset_32(volatile int32_t *addr, int32_t oldval, int32_t newval) { @@ -131,7 +158,9 @@ static inline int opal_atomic_cmpset_rel_32(volatile int32_t *addr, return opal_atomic_cmpset_32(addr, oldval, newval); } +#if (OPAL_ASM_SUPPORT_64BIT == 1) +#define OPAL_HAVE_ATOMIC_CMPSET_64 1 static inline int opal_atomic_cmpset_64(volatile int64_t *addr, int64_t oldval, int64_t newval) { @@ -181,7 +210,10 @@ static inline int opal_atomic_cmpset_rel_64(volatile int64_t *addr, return opal_atomic_cmpset_64(addr, oldval, newval); } +#endif + +#define OPAL_HAVE_ATOMIC_ADD_32 1 static inline int32_t opal_atomic_add_32(volatile int32_t* v, int inc) { int32_t t; @@ -202,7 +234,7 @@ static inline int32_t opal_atomic_add_32(volatile int32_t* v, int inc) return t; } - +#define OPAL_HAVE_ATOMIC_SUB_32 1 static inline int32_t opal_atomic_sub_32(volatile int32_t* v, int dec) { int32_t t; @@ -222,7 +254,30 @@ static inline int32_t opal_atomic_sub_32(volatile int32_t* v, int dec) return t; } +#else /* OPAL_ASM_ARM_VERSION <=5 or no GCC inline assembly */ -#endif /* OMPI_GCC_INLINE_ASSEMBLY */ +#define OPAL_HAVE_ATOMIC_CMPSET_32 1 +#define __kuser_cmpxchg (*((int (*)(int, int, volatile int*))(0xffff0fc0))) +static inline int opal_atomic_cmpset_32(volatile int32_t *addr, + int32_t oldval, int32_t newval) +{ + return !(__kuser_cmpxchg(oldval, newval, addr)); +} + +static inline int opal_atomic_cmpset_acq_32(volatile int32_t *addr, + int32_t oldval, int32_t newval) +{ + /* kernel function includes all necessary memory barriers */ + return opal_atomic_cmpset_32(addr, oldval, newval); +} + +static inline int opal_atomic_cmpset_rel_32(volatile int32_t *addr, + int32_t oldval, int32_t newval) +{ + /* kernel function includes all necessary memory barriers */ + return opal_atomic_cmpset_32(addr, oldval, newval); +} + +#endif #endif /* ! OMPI_SYS_ARCH_ATOMIC_H */