From 41f00b746529a4d28d4b1ffe6c3952e1b5bc9cb4 Mon Sep 17 00:00:00 2001 From: Nathan Hjelm Date: Wed, 4 May 2016 12:46:42 -0600 Subject: [PATCH 1/5] memory/patcher: initialize patcher framework when needed This commit moves the patcher framework initialization to the memory/patcher component. Signed-off-by: Nathan Hjelm --- .../memory/patcher/memory_patcher_component.c | 20 ++++++++++++++++--- opal/runtime/opal_finalize.c | 2 -- opal/runtime/opal_init.c | 8 -------- 3 files changed, 17 insertions(+), 13 deletions(-) diff --git a/opal/mca/memory/patcher/memory_patcher_component.c b/opal/mca/memory/patcher/memory_patcher_component.c index 6e317c919e..7e1be7d790 100644 --- a/opal/mca/memory/patcher/memory_patcher_component.c +++ b/opal/mca/memory/patcher/memory_patcher_component.c @@ -31,6 +31,7 @@ #include "opal/mca/memory/base/empty.h" #include "opal/mca/memory/base/base.h" #include "opal/memoryhooks/memory.h" +#include "opal/mca/patcher/base/base.h" #include #include @@ -395,11 +396,16 @@ static int patcher_register (void) static int patcher_query (int *priority) { - if (opal_patcher->patch_symbol) { - *priority = mca_memory_patcher_priority; - } else { + int rc; + + rc = mca_base_framework_open (&opal_patcher_base_framework, 0); + if (OPAL_SUCCESS != rc) { *priority = -1; + return OPAL_SUCCESS; } + + *priority = mca_memory_patcher_priority; + return OPAL_SUCCESS; } @@ -414,6 +420,12 @@ static int patcher_open (void) was_executed_already = 1; + rc = opal_patcher_base_select (); + if (OPAL_SUCCESS != rc) { + mca_base_framework_close (&opal_patcher_base_framework); + return OPAL_ERR_NOT_AVAILABLE; + } + /* set memory hooks support level */ opal_mem_hooks_set_support (OPAL_MEMORY_FREE_SUPPORT | OPAL_MEMORY_MUNMAP_SUPPORT); @@ -461,6 +473,8 @@ static int patcher_open (void) static int patcher_close(void) { + mca_base_framework_close (&opal_patcher_base_framework); + /* Note that we don't need to unpatch any symbols here; the patcher framework will take care of all of that for us. */ return OPAL_SUCCESS; diff --git a/opal/runtime/opal_finalize.c b/opal/runtime/opal_finalize.c index 04a580ecac..83beb5ac53 100644 --- a/opal/runtime/opal_finalize.c +++ b/opal/runtime/opal_finalize.c @@ -40,7 +40,6 @@ #include "opal/mca/installdirs/base/base.h" #include "opal/mca/memchecker/base/base.h" #include "opal/mca/memcpy/base/base.h" -#include "opal/mca/patcher/base/base.h" #include "opal/mca/backtrace/base/base.h" #include "opal/mca/sec/base/base.h" #include "opal/mca/timer/base/base.h" @@ -153,7 +152,6 @@ opal_finalize(void) (void) mca_base_framework_close(&opal_backtrace_base_framework); (void) mca_base_framework_close(&opal_memchecker_base_framework); - (void) mca_base_framework_close(&opal_patcher_base_framework); /* close the memcpy framework */ (void) mca_base_framework_close(&opal_memcpy_base_framework); diff --git a/opal/runtime/opal_init.c b/opal/runtime/opal_init.c index 6cf2f1ba04..34a850d156 100644 --- a/opal/runtime/opal_init.c +++ b/opal/runtime/opal_init.c @@ -431,14 +431,6 @@ opal_init(int* pargc, char*** pargv) goto return_error; } - if (OPAL_SUCCESS != (ret = mca_base_framework_open(&opal_patcher_base_framework, 0))) { - error = "opal_patcher_base_open"; - goto return_error; - } - - /* select a patcher module. if a patcher module can not be found it is not an error. */ - (void) opal_patcher_base_select (); - /* initialize the memory manager / tracker */ if (OPAL_SUCCESS != (ret = opal_mem_hooks_init())) { error = "opal_mem_hooks_init"; From 71be36d380ccfeb3939ace71c084bf619d478b12 Mon Sep 17 00:00:00 2001 From: Nathan Hjelm Date: Wed, 4 May 2016 12:48:52 -0600 Subject: [PATCH 2/5] patcher: fix ppc32 support The table of contents (TOC) code only appears to only apply to ppc64. The code was incorrectly assuming the existence of the TOC on ppc32. This commit updates the necessary code to only apply to ppc64. Signed-off-by: Nathan Hjelm --- opal/mca/patcher/base/base.h | 2 +- opal/mca/patcher/base/patcher_base_patch.c | 10 +++++----- opal/mca/patcher/overwrite/patcher_overwrite_module.c | 2 +- opal/mca/patcher/patcher.h | 2 +- 4 files changed, 8 insertions(+), 8 deletions(-) diff --git a/opal/mca/patcher/base/base.h b/opal/mca/patcher/base/base.h index 65b48fc00c..0be3cf1dd7 100644 --- a/opal/mca/patcher/base/base.h +++ b/opal/mca/patcher/base/base.h @@ -68,7 +68,7 @@ OPAL_DECLSPEC int mca_patcher_base_patch_hook (mca_patcher_base_module_t *module OPAL_DECLSPEC void mca_base_patcher_patch_apply_binary (mca_patcher_base_patch_t *patch); static inline uintptr_t mca_patcher_base_addr_text (uintptr_t addr) { -#if (defined(__PPC64__) || defined(__powerpc64__) || defined(__PPC__)) && _CALL_ELF != 2 +#if (OPAL_ASSEMBLY_ARCH == OPAL_POWERPC64) && (!defined (_CALL_ELF) || (_CALL_ELF != 2)) struct odp_t { uintptr_t text; uintptr_t toc; diff --git a/opal/mca/patcher/base/patcher_base_patch.c b/opal/mca/patcher/base/patcher_base_patch.c index e94826302f..b805e8befa 100644 --- a/opal/mca/patcher/base/patcher_base_patch.c +++ b/opal/mca/patcher/base/patcher_base_patch.c @@ -122,7 +122,7 @@ static void ModifyMemoryProtection (uintptr_t addr, size_t length, int prot) if (mprotect((void *)base, page_size, prot)) perror("MemHook: mprotect failed"); base += page_size; - } while (base < addr + length); + } while (base < bound); #else if (mprotect((void *) base, length, prot)) { perror("MemHook: mprotect failed"); @@ -156,10 +156,9 @@ void mca_base_patcher_patch_apply_binary (mca_patcher_base_patch_t *patch) int mca_patcher_base_patch_hook (mca_patcher_base_module_t *module, uintptr_t hook_addr) { -#if defined(__PPC64__) || defined(__powerpc64__) || defined(__PPC__) +#if (OPAL_ASSEMBLY_ARCH == OPAL_POWERPC64) mca_patcher_base_patch_t *hook_patch; const unsigned int nop = 0x60000000; - unsigned int *nop_addr; hook_patch = OBJ_NEW(mca_patcher_base_patch_t); if (OPAL_UNLIKELY(NULL == hook_patch)) { @@ -167,15 +166,16 @@ int mca_patcher_base_patch_hook (mca_patcher_base_module_t *module, uintptr_t ho } // locate reserved code space in hook function - for (nop_addr = (unsigned int *)hook_addr ; ; nop_addr++) { + for (unsigned int *nop_addr = (unsigned int *)hook_addr ; ; nop_addr++) { if (nop_addr[0] == nop && nop_addr[1] == nop && nop_addr[2] == nop && nop_addr[3] == nop && nop_addr[4] == nop) { + hook_patch->patch_orig = (uintptr_t) nop_addr; break; } } + // generate code to restore TOC register unsigned long toc asm("r2"); - hook_patch->patch_orig = (uintptr_t) nop_addr; hook_patch->patch_data_size = PatchLoadImm((uintptr_t)hook_patch->patch_data, 2, toc); /* put the hook patch on the patch list so it will be undone on finalize */ diff --git a/opal/mca/patcher/overwrite/patcher_overwrite_module.c b/opal/mca/patcher/overwrite/patcher_overwrite_module.c index 15b64ebd2d..88052091cb 100644 --- a/opal/mca/patcher/overwrite/patcher_overwrite_module.c +++ b/opal/mca/patcher/overwrite/patcher_overwrite_module.c @@ -214,7 +214,7 @@ static int mca_patcher_overwrite_apply_patch (mca_patcher_base_patch_t *patch) return rc; } -#if _CALL_ELF == 2 +#if defined(_CALL_ELF) && (_CALL_ELF == 2) sys_addr += 8; hook_addr += 8; #endif /* _CALL_ELF == 2*/ diff --git a/opal/mca/patcher/patcher.h b/opal/mca/patcher/patcher.h index 25af9b1376..8e8d13a1c9 100644 --- a/opal/mca/patcher/patcher.h +++ b/opal/mca/patcher/patcher.h @@ -21,7 +21,7 @@ /* Any function being patched in as a hook must use SYMBOLPATCH_BEGIN at the top, * and SYMBOLPATCH_END before it returns (this is just for PPC). */ -#if (defined(__PPC64__) || defined(__powerpc64__) || defined(__PPC__)) && defined(OPAL_GCC_INLINE_ASSEMBLY) +#if (OPAL_ASSEMBLY_ARCH == OPAL_POWERPC64) /* special processing for ppc64 to save and restore TOC (r2) * Reference: "64-bit PowerPC ELF Application Binary Interface Supplement 1.9" */ From 6ad68da4078929888e23b0be275d34e2c3e7b6c4 Mon Sep 17 00:00:00 2001 From: Nathan Hjelm Date: Wed, 4 May 2016 12:50:51 -0600 Subject: [PATCH 3/5] patcher/linux: disable the linux patcher component This commit disables the linux patcher component due to a limitation in loader patching. While this component is effective in patching calls made within Open MPI and by the application it fails to hook calls made within glibc. This means the munmap call made by free is not correctly hooked. Until this problem can be resolved this component will remain disabled. If it can't be resolved this component should probably be removed. Signed-off-by: Nathan Hjelm --- opal/mca/patcher/linux/configure.m4 | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/opal/mca/patcher/linux/configure.m4 b/opal/mca/patcher/linux/configure.m4 index 6cfef2ca1a..99160e4289 100644 --- a/opal/mca/patcher/linux/configure.m4 +++ b/opal/mca/patcher/linux/configure.m4 @@ -48,6 +48,10 @@ AC_DEFUN([MCA_opal_patcher_linux_CONFIG],[ fi fi + # this component can not be used until we can determine a way to hook munmap, etc inside + # glibc. this is needed to catch munmap called by free + opal_patcher_linux_happy=no + AS_IF([test $opal_patcher_linux_happy = yes], [$1], [$2]) OPAL_VAR_SCOPE_POP ]) From 6c9a0e1c5561dee72bbe6dc610be720052d7b710 Mon Sep 17 00:00:00 2001 From: Nathan Hjelm Date: Wed, 4 May 2016 12:53:24 -0600 Subject: [PATCH 4/5] patcher/overwrite: disable ia64 support for now Signed-off-by: Nathan Hjelm --- opal/mca/patcher/overwrite/configure.m4 | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/opal/mca/patcher/overwrite/configure.m4 b/opal/mca/patcher/overwrite/configure.m4 index 02394cc958..2449472685 100644 --- a/opal/mca/patcher/overwrite/configure.m4 +++ b/opal/mca/patcher/overwrite/configure.m4 @@ -30,8 +30,9 @@ AC_DEFUN([MCA_opal_patcher_overwrite_CONFIG],[ opal_patcher_overwrite_happy=no if test $OPAL_ENABLE_DLOPEN_SUPPORT = 1; then +# Disable ia64 for now. We can revive it later if anyone cares AC_COMPILE_IFELSE([AC_LANG_PROGRAM([[ -#if !defined(__i386__) && !defined(__x86_64__) && !defined(__ia64__) && !defined(__PPC__) +#if !defined(__i386__) && !defined(__x86_64__) && !defined(__PPC__) #error "platform not supported" #endif ]],[])],[opal_patcher_overwrite_happy=yes],[]) From ff2a54bd37939d2d3507765af21883ae211c2fcf Mon Sep 17 00:00:00 2001 From: Nathan Hjelm Date: Wed, 4 May 2016 12:53:45 -0600 Subject: [PATCH 5/5] patcher/linux: code cleanup Update based on cleanup made to the upstream version on OpenUCX. Signed-off-by: Nathan Hjelm --- opal/mca/patcher/linux/patcher_linux_module.c | 143 +++++------------- 1 file changed, 37 insertions(+), 106 deletions(-) diff --git a/opal/mca/patcher/linux/patcher_linux_module.c b/opal/mca/patcher/linux/patcher_linux_module.c index 90cd497a10..29f7dfc553 100644 --- a/opal/mca/patcher/linux/patcher_linux_module.c +++ b/opal/mca/patcher/linux/patcher_linux_module.c @@ -39,21 +39,6 @@ static void *mca_patcher_linux_dlopen(const char *filename, int flag); -typedef struct mca_patcher_linux_elf_strtab { - char *tab; - ElfW(Xword) size; -} mca_patcher_linux_elf_strtab_t; - -typedef struct mca_patcher_linux_elf_jmpreltab { - ElfW(Rela) *tab; - ElfW(Xword) size; -} mca_patcher_linux_elf_jmprel_t; - -typedef struct mca_patcher_linux_elf_symtab { - ElfW(Sym) *tab; - ElfW(Xword) entsz; -} mca_patcher_linux_elf_symtab_t; - typedef struct mca_patcher_linux_dl_iter_context { mca_patcher_linux_patch_t *patch; bool remove; @@ -81,93 +66,54 @@ static void *(*orig_dlopen) (const char *, int); static const ElfW(Phdr) * mca_patcher_linux_get_phdr_dynamic(const ElfW(Phdr) *phdr, uint16_t phnum, int phent) { - for (uint16_t i = 0; i < phnum; ++i) { + for (uint16_t i = 0 ; i < phnum ; ++i, phdr = (ElfW(Phdr)*)((intptr_t) phdr + phent)) { if (phdr->p_type == PT_DYNAMIC) { return phdr; } - phdr = (ElfW(Phdr)*)((char*)phdr + phent); } + return NULL; } -#if SIZEOF_VOID_P == 8 -static const ElfW(Dyn)* -mca_patcher_linux_get_dynentry(ElfW(Addr) base, const ElfW(Phdr) *pdyn, int64_t type) -#else -static const ElfW(Dyn)* -mca_patcher_linux_get_dynentry(ElfW(Addr) base, const ElfW(Phdr) *pdyn, int32_t type) -#endif +static void *mca_patcher_linux_get_dynentry(ElfW(Addr) base, const ElfW(Phdr) *pdyn, ElfW(Sxword) type) { for (ElfW(Dyn) *dyn = (ElfW(Dyn)*)(base + pdyn->p_vaddr); dyn->d_tag; ++dyn) { if (dyn->d_tag == type) { - return dyn; + return (void *) (uintptr_t) dyn->d_un.d_val; } } + return NULL; } -static void mca_patcher_linux_get_jmprel(ElfW(Addr) base, const ElfW(Phdr) *pdyn, - mca_patcher_linux_elf_jmprel_t *table) -{ - const ElfW(Dyn) *dyn; - - dyn = mca_patcher_linux_get_dynentry(base, pdyn, DT_JMPREL); - table->tab = (dyn == NULL) ? NULL : (ElfW(Rela)*)dyn->d_un.d_ptr; - dyn = mca_patcher_linux_get_dynentry(base, pdyn, DT_PLTRELSZ); - table->size = (dyn == NULL) ? 0 : dyn->d_un.d_val; -} - -static void mca_patcher_linux_get_symtab(ElfW(Addr) base, const ElfW(Phdr) *pdyn, - mca_patcher_linux_elf_symtab_t *table) -{ - const ElfW(Dyn) *dyn; - - dyn = mca_patcher_linux_get_dynentry(base, pdyn, DT_SYMTAB); - table->tab = (dyn == NULL) ? NULL : (ElfW(Sym)*)dyn->d_un.d_ptr; - dyn = mca_patcher_linux_get_dynentry(base, pdyn, DT_SYMENT); - table->entsz = (dyn == NULL) ? 0 : dyn->d_un.d_val; -} - -static void mca_patcher_linux_get_strtab(ElfW(Addr) base, const ElfW(Phdr) *pdyn, - mca_patcher_linux_elf_strtab_t *table) -{ - const ElfW(Dyn) *dyn; - - dyn = mca_patcher_linux_get_dynentry(base, pdyn, DT_STRTAB); - table->tab = (dyn == NULL) ? NULL : (char *)dyn->d_un.d_ptr; - dyn = mca_patcher_linux_get_dynentry(base, pdyn, DT_STRSZ); - table->size = (dyn == NULL) ? 0 : dyn->d_un.d_val; -} - static void * mca_patcher_linux_get_got_entry (ElfW(Addr) base, const ElfW(Phdr) *phdr, int16_t phnum, int phent, const char *symbol) { - mca_patcher_linux_elf_jmprel_t jmprel; - mca_patcher_linux_elf_symtab_t symtab; - mca_patcher_linux_elf_strtab_t strtab; - ElfW(Rela) *rela, *relaend; const ElfW(Phdr) *dphdr; - const char *relsymname; - uint32_t relsymidx; + void *jmprel, *strtab; + ElfW(Sym) *symtab; + size_t pltrelsz; dphdr = mca_patcher_linux_get_phdr_dynamic (phdr, phnum, phent); - mca_patcher_linux_get_jmprel (base, dphdr, &jmprel); - mca_patcher_linux_get_symtab (base, dphdr, &symtab); - mca_patcher_linux_get_strtab (base, dphdr, &strtab); + jmprel = mca_patcher_linux_get_dynentry (base, dphdr, DT_JMPREL); + symtab = (ElfW(Sym) *) mca_patcher_linux_get_dynentry (base, dphdr, DT_SYMTAB); + strtab = mca_patcher_linux_get_dynentry (base, dphdr, DT_STRTAB); + pltrelsz = (size_t) (uintptr_t) mca_patcher_linux_get_dynentry (base, dphdr, DT_PLTRELSZ); - relaend = (ElfW(Rela) *)((char *)jmprel.tab + jmprel.size); - for (rela = jmprel.tab; rela < relaend; ++rela) { + for (ElfW(Rela) *reloc = jmprel; (intptr_t) reloc < (intptr_t) jmprel + pltrelsz; ++reloc) { #if SIZEOF_VOID_P == 8 - relsymidx = ELF64_R_SYM(rela->r_info); + uint32_t relsymidx = ELF64_R_SYM(reloc->r_info); #else - relsymidx = ELF32_R_SYM(rela->r_info); + uint32_t relsymidx = ELF32_R_SYM(reloc->r_info); #endif - relsymname = strtab.tab + symtab.tab[relsymidx].st_name; - if (!strcmp(symbol, relsymname)) { - return (void *)(base + rela->r_offset); + char *elf_sym = (char *) strtab + symtab[relsymidx].st_name; + + if (0 == strcmp (symbol, elf_sym)) { + return (void *)(base + reloc->r_offset); } } + return NULL; } @@ -177,11 +123,7 @@ static int mca_patcher_linux_get_aux_phent (void) #define MCA_PATCHER_LINUX_AUXV_BUF_LEN 16 static const char *proc_auxv_filename = "/proc/self/auxv"; static int phent = 0; -#if SIZEOF_VOID_P == 8 - Elf64_auxv_t buffer[MCA_PATCHER_LINUX_AUXV_BUF_LEN]; -#else - Elf32_auxv_t buffer[MCA_PATCHER_LINUX_AUXV_BUF_LEN]; -#endif + ElfW(auxv_t) buffer[MCA_PATCHER_LINUX_AUXV_BUF_LEN]; unsigned count; ssize_t nread; int fd; @@ -231,8 +173,7 @@ mca_patcher_linux_modify_got (ElfW(Addr) base, const ElfW(Phdr) *phdr, const cha int16_t phnum, int phent, mca_patcher_linux_dl_iter_context_t *ctx) { long page_size = opal_getpagesize (); - void **entry; - void *page; + void **entry, *page; int ret; entry = mca_patcher_linux_get_got_entry (base, phdr, phnum, phent, ctx->patch->super.patch_symbol); @@ -267,18 +208,18 @@ mca_patcher_linux_modify_got (ElfW(Addr) base, const ElfW(Phdr) *phdr, const cha *entry = (void *) ctx->patch->super.patch_value; } } else { - if (*entry == (void *) ctx->patch->super.patch_value) { - /* find the appropriate entry and restore the original value */ - mca_patcher_linux_patch_got_t *patch_got; - OPAL_LIST_FOREACH_REV(patch_got, &ctx->patch->patch_got_list, mca_patcher_linux_patch_got_t) { - if (patch_got->got_entry == entry) { - opal_output_verbose (MCA_BASE_VERBOSE_TRACE, opal_patcher_base_framework.framework_output, - "restoring got entry %p with original value %p\n", (void *) entry, patch_got->got_orig); + /* find the appropriate entry and restore the original value */ + mca_patcher_linux_patch_got_t *patch_got; + OPAL_LIST_FOREACH_REV(patch_got, &ctx->patch->patch_got_list, mca_patcher_linux_patch_got_t) { + if (patch_got->got_entry == entry) { + opal_output_verbose (MCA_BASE_VERBOSE_TRACE, opal_patcher_base_framework.framework_output, + "restoring got entry %p with original value %p\n", (void *) entry, patch_got->got_orig); + if (*entry == (void *) ctx->patch->super.patch_value) { *entry = patch_got->got_orig; - opal_list_remove_item (&ctx->patch->patch_got_list, &patch_got->super); - OBJ_RELEASE(patch_got); - break; } + opal_list_remove_item (&ctx->patch->patch_got_list, &patch_got->super); + OBJ_RELEASE(patch_got); + break; } } } @@ -321,12 +262,7 @@ static int mca_patcher_linux_apply_patch (mca_patcher_linux_patch_t *patch) /* Avoid locks here because we don't modify ELF data structures. * Worst case the same symbol will be written more than once. */ - (void)dl_iterate_phdr(mca_patcher_linux_phdr_iterator, &ctx); - if (ctx.status == OPAL_SUCCESS) { - opal_output_verbose (MCA_BASE_VERBOSE_INFO, opal_patcher_base_framework.framework_output, - "modified '%s' to %" PRIxPTR , ctx.patch->super.patch_symbol, - ctx.patch->super.patch_value); - } + (void) dl_iterate_phdr(mca_patcher_linux_phdr_iterator, &ctx); return ctx.status; } @@ -342,11 +278,7 @@ static int mca_patcher_linux_remove_patch (mca_patcher_linux_patch_t *patch) /* Avoid locks here because we don't modify ELF data structures. * Worst case the same symbol will be written more than once. */ - (void)dl_iterate_phdr(mca_patcher_linux_phdr_iterator, &ctx); - if (ctx.status == OPAL_SUCCESS) { - opal_output_verbose (MCA_BASE_VERBOSE_INFO, opal_patcher_base_framework.framework_output, - "modified '%s' to 0x%lx", ctx.patch->super.patch_symbol, ctx.patch->super.patch_value); - } + (void) dl_iterate_phdr(mca_patcher_linux_phdr_iterator, &ctx); return ctx.status; } @@ -357,7 +289,6 @@ static void *mca_patcher_linux_dlopen(const char *filename, int flag) mca_patcher_linux_patch_t *patch; void *handle; - assert (orig_dlopen); handle = orig_dlopen (filename, flag); if (handle != NULL) { /* @@ -368,10 +299,10 @@ static void *mca_patcher_linux_dlopen(const char *filename, int flag) */ opal_mutex_lock (&mca_patcher_linux_module.patch_list_mutex); OPAL_LIST_FOREACH(patch, &mca_patcher_linux_module.patch_list, mca_patcher_linux_patch_t) { - opal_output_verbose (MCA_BASE_VERBOSE_INFO, opal_patcher_base_framework.framework_output, - "in dlopen(), re-applying '%s' to %p", patch->super.patch_symbol, (void *) patch->super.patch_value); - /* ignore hook binary patches */ if (!patch->super.patch_data_size) { + opal_output_verbose (MCA_BASE_VERBOSE_INFO, opal_patcher_base_framework.framework_output, + "in dlopen(), re-applying '%s' to %p", patch->super.patch_symbol, (void *) patch->super.patch_value); + /* ignore hook binary patches */ mca_patcher_linux_apply_patch (patch); } }