diff options
Diffstat (limited to 'ldso/dynlink.c')
-rw-r--r-- | ldso/dynlink.c | 237 |
1 files changed, 186 insertions, 51 deletions
diff --git a/ldso/dynlink.c b/ldso/dynlink.c index 7810356b..324aa859 100644 --- a/ldso/dynlink.c +++ b/ldso/dynlink.c @@ -1,6 +1,5 @@ #define _GNU_SOURCE #define SYSCALL_NO_TLS 1 -#include <stdio.h> #include <stdlib.h> #include <stdarg.h> #include <stddef.h> @@ -21,11 +20,24 @@ #include <semaphore.h> #include <sys/membarrier.h> #include "pthread_impl.h" -#include "libc.h" +#include "fork_impl.h" #include "dynlink.h" -#include "malloc_impl.h" -static void error(const char *, ...); +static size_t ldso_page_size; +#ifndef PAGE_SIZE +#define PAGE_SIZE ldso_page_size +#endif + +#include "libc.h" + +#define malloc __libc_malloc +#define calloc __libc_calloc +#define realloc __libc_realloc +#define free __libc_free + +static void error_impl(const char *, ...); +static void error_noop(const char *, ...); +static void (*error)(const char *, ...) = error_noop; #define MAXP2(a,b) (-(-(a)&-(b))) #define ALIGN(x,y) ((x)+(y)-1 & -(y)) @@ -79,7 +91,7 @@ struct dso { struct dso **deps, *needed_by; size_t ndeps_direct; size_t next_dep; - int ctor_visitor; + pthread_t ctor_visitor; char *rpath_orig, *rpath; struct tls_module tls; size_t tls_id; @@ -107,6 +119,8 @@ struct symdef { struct dso *dso; }; +typedef void (*stage3_func)(size_t *, size_t *); + static struct builtin_tls { char c; struct pthread pt; @@ -143,6 +157,8 @@ static struct fdpic_dummy_loadmap app_dummy_loadmap; struct debug *_dl_debug_addr = &debug; +extern weak hidden char __ehdr_start[]; + extern hidden int __malloc_replaced; hidden void (*const __init_array_start)(void)=0, (*const __fini_array_start)(void)=0; @@ -183,8 +199,14 @@ static void *laddr_pg(const struct dso *p, size_t v) } return (void *)(v - p->loadmap->segs[j].p_vaddr + p->loadmap->segs[j].addr); } -#define fpaddr(p, v) ((void (*)())&(struct funcdesc){ \ - laddr(p, v), (p)->got }) +static void (*fdbarrier(void *p))() +{ + void (*fd)(); + __asm__("" : "=r"(fd) : "0"(p)); + return fd; +} +#define fpaddr(p, v) fdbarrier((&(struct funcdesc){ \ + laddr(p, v), (p)->got })) #else #define laddr(p, v) (void *)((p)->base + (v)) #define laddr_pg(p, v) laddr(p, v) @@ -196,7 +218,8 @@ static void decode_vec(size_t *v, size_t *a, size_t cnt) size_t i; for (i=0; i<cnt; i++) a[i] = 0; for (; v[0]; v+=2) if (v[0]-1<cnt-1) { - a[0] |= 1UL<<v[0]; + if (v[0] < 8*sizeof(long)) + a[0] |= 1UL<<v[0]; a[v[0]] = v[1]; } } @@ -322,6 +345,40 @@ static struct symdef find_sym(struct dso *dso, const char *s, int need_def) return find_sym2(dso, s, need_def, 0); } +static struct symdef get_lfs64(const char *name) +{ + const char *p; + static const char lfs64_list[] = + "aio_cancel\0aio_error\0aio_fsync\0aio_read\0aio_return\0" + "aio_suspend\0aio_write\0alphasort\0creat\0fallocate\0" + "fgetpos\0fopen\0freopen\0fseeko\0fsetpos\0fstat\0" + "fstatat\0fstatfs\0fstatvfs\0ftello\0ftruncate\0ftw\0" + "getdents\0getrlimit\0glob\0globfree\0lio_listio\0" + "lockf\0lseek\0lstat\0mkostemp\0mkostemps\0mkstemp\0" + "mkstemps\0mmap\0nftw\0open\0openat\0posix_fadvise\0" + "posix_fallocate\0pread\0preadv\0prlimit\0pwrite\0" + "pwritev\0readdir\0scandir\0sendfile\0setrlimit\0" + "stat\0statfs\0statvfs\0tmpfile\0truncate\0versionsort\0" + "__fxstat\0__fxstatat\0__lxstat\0__xstat\0"; + size_t l; + char buf[16]; + for (l=0; name[l]; l++) { + if (l >= sizeof buf) goto nomatch; + buf[l] = name[l]; + } + if (!strcmp(name, "readdir64_r")) + return find_sym(&ldso, "readdir_r", 1); + if (l<2 || name[l-2]!='6' || name[l-1]!='4') + goto nomatch; + buf[l-=2] = 0; + for (p=lfs64_list; *p; p++) { + if (!strcmp(buf, p)) return find_sym(&ldso, buf, 1); + while (*p) p++; + } +nomatch: + return (struct symdef){ 0 }; +} + static void do_relocs(struct dso *dso, size_t *rel, size_t rel_size, size_t stride) { unsigned char *base = dso->base; @@ -375,6 +432,7 @@ static void do_relocs(struct dso *dso, size_t *rel, size_t rel_size, size_t stri def = (sym->st_info>>4) == STB_LOCAL ? (struct symdef){ .dso = dso, .sym = sym } : find_sym(ctx, name, type==REL_PLT); + if (!def.sym) def = get_lfs64(name); if (!def.sym && (sym->st_shndx != SHN_UNDEF || sym->st_info>>4 != STB_WEAK)) { if (dso->lazy && (type==REL_PLT || type==REL_GOT)) { @@ -407,8 +465,6 @@ static void do_relocs(struct dso *dso, size_t *rel, size_t rel_size, size_t stri } switch(type) { - case REL_NONE: - break; case REL_OFFSET: addend -= (size_t)reloc_addr; case REL_SYMBOLIC: @@ -461,7 +517,7 @@ static void do_relocs(struct dso *dso, size_t *rel, size_t rel_size, size_t stri break; #endif case REL_TLSDESC: - if (stride<3) addend = reloc_addr[1]; + if (stride<3) addend = reloc_addr[!TLSDESC_BACKWARDS]; if (def.dso->tls_id > static_tls_cnt) { struct td_index *new = malloc(sizeof *new); if (!new) { @@ -486,13 +542,13 @@ static void do_relocs(struct dso *dso, size_t *rel, size_t rel_size, size_t stri + addend; #endif } -#ifdef TLSDESC_BACKWARDS /* Some archs (32-bit ARM at least) invert the order of * the descriptor members. Fix them up here. */ - size_t tmp = reloc_addr[0]; - reloc_addr[0] = reloc_addr[1]; - reloc_addr[1] = tmp; -#endif + if (TLSDESC_BACKWARDS) { + size_t tmp = reloc_addr[0]; + reloc_addr[0] = reloc_addr[1]; + reloc_addr[1] = tmp; + } break; default: error("Error relocating %s: unsupported relocation type %d", @@ -503,6 +559,24 @@ static void do_relocs(struct dso *dso, size_t *rel, size_t rel_size, size_t stri } } +static void do_relr_relocs(struct dso *dso, size_t *relr, size_t relr_size) +{ + if (dso == &ldso) return; /* self-relocation was done in _dlstart */ + unsigned char *base = dso->base; + size_t *reloc_addr; + for (; relr_size; relr++, relr_size-=sizeof(size_t)) + if ((relr[0]&1) == 0) { + reloc_addr = laddr(dso, relr[0]); + *reloc_addr++ += (size_t)base; + } else { + int i = 0; + for (size_t bitmap=relr[0]; (bitmap>>=1); i++) + if (bitmap&1) + reloc_addr[i] += (size_t)base; + reloc_addr += 8*sizeof(size_t)-1; + } +} + static void redo_lazy_relocs() { struct dso *p = lazy_head, *next; @@ -551,10 +625,25 @@ static void reclaim_gaps(struct dso *dso) } } +static ssize_t read_loop(int fd, void *p, size_t n) +{ + for (size_t i=0; i<n; ) { + ssize_t l = read(fd, (char *)p+i, n-i); + if (l<0) { + if (errno==EINTR) continue; + else return -1; + } + if (l==0) return i; + i += l; + } + return n; +} + static void *mmap_fixed(void *p, size_t n, int prot, int flags, int fd, off_t off) { static int no_map_fixed; char *q; + if (!n) return p; if (!no_map_fixed) { q = mmap(p, n, prot, flags|MAP_FIXED, fd, off); if (!DL_NOMMU_SUPPORT || q != MAP_FAILED || errno != EINVAL) @@ -841,7 +930,7 @@ static int fixup_rpath(struct dso *p, char *buf, size_t buf_size) case ENOENT: case ENOTDIR: case EACCES: - break; + return 0; default: return -1; } @@ -1055,13 +1144,17 @@ static struct dso *load_library(const char *name, struct dso *needed_by) snprintf(etc_ldso_path, sizeof etc_ldso_path, "%.*s/etc/ld-musl-" LDSO_ARCH ".path", (int)prefix_len, prefix); - FILE *f = fopen(etc_ldso_path, "rbe"); - if (f) { - if (getdelim(&sys_path, (size_t[1]){0}, 0, f) <= 0) { + fd = open(etc_ldso_path, O_RDONLY|O_CLOEXEC); + if (fd>=0) { + size_t n = 0; + if (!fstat(fd, &st)) n = st.st_size; + if ((sys_path = malloc(n+1))) + sys_path[n] = 0; + if (!sys_path || read_loop(fd, sys_path, n)<0) { free(sys_path); sys_path = ""; } - fclose(f); + close(fd); } else if (errno != ENOENT) { sys_path = ""; } @@ -1326,13 +1419,17 @@ static void reloc_all(struct dso *p) 2+(dyn[DT_PLTREL]==DT_RELA)); do_relocs(p, laddr(p, dyn[DT_REL]), dyn[DT_RELSZ], 2); do_relocs(p, laddr(p, dyn[DT_RELA]), dyn[DT_RELASZ], 3); - - if (head != &ldso && p->relro_start != p->relro_end && - mprotect(laddr(p, p->relro_start), p->relro_end-p->relro_start, PROT_READ) - && errno != ENOSYS) { - error("Error relocating %s: RELRO protection failed: %m", - p->name); - if (runtime) longjmp(*rtld_fail, 1); + if (!DL_FDPIC) + do_relr_relocs(p, laddr(p, dyn[DT_RELR]), dyn[DT_RELRSZ]); + + if (head != &ldso && p->relro_start != p->relro_end) { + long ret = __syscall(SYS_mprotect, laddr(p, p->relro_start), + p->relro_end-p->relro_start, PROT_READ); + if (ret != 0 && ret != -ENOSYS) { + error("Error relocating %s: RELRO protection failed: %m", + p->name); + if (runtime) longjmp(*rtld_fail, 1); + } } p->relocated = 1; @@ -1373,7 +1470,7 @@ void __libc_exit_fini() { struct dso *p; size_t dyn[DYN_CNT]; - int self = __pthread_self()->tid; + pthread_t self = __pthread_self(); /* Take both locks before setting shutting_down, so that * either lock is sufficient to read its value. The lock @@ -1399,6 +1496,17 @@ void __libc_exit_fini() } } +void __ldso_atfork(int who) +{ + if (who<0) { + pthread_rwlock_wrlock(&lock); + pthread_mutex_lock(&init_fini_lock); + } else { + pthread_mutex_unlock(&init_fini_lock); + pthread_rwlock_unlock(&lock); + } +} + static struct dso **queue_ctors(struct dso *dso) { size_t cnt, qpos, spos, i; @@ -1457,6 +1565,13 @@ static struct dso **queue_ctors(struct dso *dso) } queue[qpos] = 0; for (i=0; i<qpos; i++) queue[i]->mark = 0; + for (i=0; i<qpos; i++) + if (queue[i]->ctor_visitor && queue[i]->ctor_visitor->tid < 0) { + error("State of %s is inconsistent due to multithreaded fork\n", + queue[i]->name); + free(queue); + if (runtime) longjmp(*rtld_fail, 1); + } return queue; } @@ -1465,7 +1580,7 @@ static void do_init_fini(struct dso **queue) { struct dso *p; size_t dyn[DYN_CNT], i; - int self = __pthread_self()->tid; + pthread_t self = __pthread_self(); pthread_mutex_lock(&init_fini_lock); for (i=0; (p=queue[i]); i++) { @@ -1574,7 +1689,7 @@ static void install_new_tls(void) /* Install new dtv for each thread. */ for (j=0, td=self; !j || td!=self; j++, td=td->next) { - td->dtv = td->dtv_copy = newdtv[j]; + td->dtv = newdtv[j]; } __tl_unlock(); @@ -1594,13 +1709,14 @@ static void install_new_tls(void) hidden void __dls2(unsigned char *base, size_t *sp) { + size_t *auxv; + for (auxv=sp+1+*sp+1; *auxv; auxv++); + auxv++; if (DL_FDPIC) { void *p1 = (void *)sp[-2]; void *p2 = (void *)sp[-1]; if (!p1) { - size_t *auxv, aux[AUX_CNT]; - for (auxv=sp+1+*sp+1; *auxv; auxv++); - auxv++; + size_t aux[AUX_CNT]; decode_vec(auxv, aux, AUX_CNT); if (aux[AT_BASE]) ldso.base = (void *)aux[AT_BASE]; else ldso.base = (void *)(aux[AT_PHDR] & -4096); @@ -1611,11 +1727,12 @@ hidden void __dls2(unsigned char *base, size_t *sp) } else { ldso.base = base; } - Ehdr *ehdr = (void *)ldso.base; + Ehdr *ehdr = __ehdr_start ? (void *)__ehdr_start : (void *)ldso.base; ldso.name = ldso.shortname = "libc.so"; ldso.phnum = ehdr->e_phnum; ldso.phdr = laddr(&ldso, ehdr->e_phoff); ldso.phentsize = ehdr->e_phentsize; + search_vec(auxv, &ldso_page_size, AT_PAGESZ); kernel_mapped_dso(&ldso); decode_dyn(&ldso); @@ -1646,8 +1763,8 @@ hidden void __dls2(unsigned char *base, size_t *sp) * symbolically as a barrier against moving the address * load across the above relocation processing. */ struct symdef dls2b_def = find_sym(&ldso, "__dls2b", 0); - if (DL_FDPIC) ((stage3_func)&ldso.funcdescs[dls2b_def.sym-ldso.syms])(sp); - else ((stage3_func)laddr(&ldso, dls2b_def.sym->st_value))(sp); + if (DL_FDPIC) ((stage3_func)&ldso.funcdescs[dls2b_def.sym-ldso.syms])(sp, auxv); + else ((stage3_func)laddr(&ldso, dls2b_def.sym->st_value))(sp, auxv); } /* Stage 2b sets up a valid thread pointer, which requires relocations @@ -1656,11 +1773,13 @@ hidden void __dls2(unsigned char *base, size_t *sp) * so that loads of the thread pointer and &errno can be pure/const and * thereby hoistable. */ -void __dls2b(size_t *sp) +void __dls2b(size_t *sp, size_t *auxv) { /* Setup early thread pointer in builtin_tls for ldso/libc itself to * use during dynamic linking. If possible it will also serve as the * thread pointer at runtime. */ + search_vec(auxv, &__hwcap, AT_HWCAP); + libc.auxv = auxv; libc.tls_size = sizeof builtin_tls; libc.tls_align = tls_align; if (__init_tp(__copy_tls((void *)builtin_tls)) < 0) { @@ -1668,8 +1787,8 @@ void __dls2b(size_t *sp) } struct symdef dls3_def = find_sym(&ldso, "__dls3", 0); - if (DL_FDPIC) ((stage3_func)&ldso.funcdescs[dls3_def.sym-ldso.syms])(sp); - else ((stage3_func)laddr(&ldso, dls3_def.sym->st_value))(sp); + if (DL_FDPIC) ((stage3_func)&ldso.funcdescs[dls3_def.sym-ldso.syms])(sp, auxv); + else ((stage3_func)laddr(&ldso, dls3_def.sym->st_value))(sp, auxv); } /* Stage 3 of the dynamic linker is called with the dynamic linker/libc @@ -1677,10 +1796,10 @@ void __dls2b(size_t *sp) * process dependencies and relocations for the main application and * transfer control to its entry point. */ -void __dls3(size_t *sp) +void __dls3(size_t *sp, size_t *auxv) { static struct dso app, vdso; - size_t aux[AUX_CNT], *auxv; + size_t aux[AUX_CNT]; size_t i; char *env_preload=0; char *replace_argv0=0; @@ -1693,10 +1812,7 @@ void __dls3(size_t *sp) /* Find aux vector just past environ[] and use it to initialize * global data that may be needed before we can make syscalls. */ __environ = envp; - for (i=argc+1; argv[i]; i++); - libc.auxv = auxv = (void *)(argv+i+1); decode_vec(auxv, aux, AUX_CNT); - __hwcap = aux[AT_HWCAP]; search_vec(auxv, &__sysinfo, AT_SYSINFO); __pthread_self()->sysinfo = __sysinfo; libc.page_size = aux[AT_PAGESZ]; @@ -1709,6 +1825,9 @@ void __dls3(size_t *sp) env_preload = getenv("LD_PRELOAD"); } + /* Activate error handler function */ + error = error_impl; + /* If the main program was already loaded by the kernel, * AT_PHDR will point to some location other than the dynamic * linker's program headers. */ @@ -1784,7 +1903,7 @@ void __dls3(size_t *sp) dprintf(2, "%s: cannot load %s: %s\n", ldname, argv[0], strerror(errno)); _exit(1); } - Ehdr *ehdr = (void *)map_library(fd, &app); + Ehdr *ehdr = map_library(fd, &app); if (!ehdr) { dprintf(2, "%s: %s: Not a valid dynamic program\n", ldname, argv[0]); _exit(1); @@ -1876,6 +1995,10 @@ void __dls3(size_t *sp) size_t *ptr = (size_t *) app.dynv[i+1]; *ptr = (size_t)&debug; } + if (app.dynv[i]==DT_DEBUG_INDIRECT_REL) { + size_t *ptr = (size_t *)((size_t)&app.dynv[i] + app.dynv[i+1]); + *ptr = (size_t)&debug; + } } /* This must be done before final relocations, since it calls @@ -1930,6 +2053,8 @@ void __dls3(size_t *sp) * possibility of incomplete replacement. */ if (find_sym(head, "malloc", 1).dso != &ldso) __malloc_replaced = 1; + if (find_sym(head, "aligned_alloc", 1).dso != &ldso) + __aligned_alloc_replaced = 1; /* Switch to runtime mode: any further failures in the dynamic * linker are a reportable failure rather than a fatal startup @@ -1940,7 +2065,7 @@ void __dls3(size_t *sp) debug.bp = dl_debug_state; debug.head = head; debug.base = ldso.base; - debug.state = 0; + debug.state = RT_CONSISTENT; _dl_debug_state(); if (replace_argv0) argv[0] = replace_argv0; @@ -1989,6 +2114,9 @@ void *dlopen(const char *file, int mode) pthread_rwlock_wrlock(&lock); __inhibit_ptc(); + debug.state = RT_ADD; + _dl_debug_state(); + p = 0; if (shutting_down) { error("Cannot dlopen while program is exiting."); @@ -2048,8 +2176,9 @@ void *dlopen(const char *file, int mode) load_deps(p); extend_bfs_deps(p); pthread_mutex_lock(&init_fini_lock); - if (!p->constructed) ctor_queue = queue_ctors(p); + int constructed = p->constructed; pthread_mutex_unlock(&init_fini_lock); + if (!constructed) ctor_queue = queue_ctors(p); if (!p->relocated && (mode & RTLD_LAZY)) { prepare_lazy(p); for (i=0; p->deps[i]; i++) @@ -2081,9 +2210,10 @@ void *dlopen(const char *file, int mode) update_tls_size(); if (tls_cnt != orig_tls_cnt) install_new_tls(); - _dl_debug_state(); orig_tail = tail; end: + debug.state = RT_CONSISTENT; + _dl_debug_state(); __release_ptc(); if (p) gencnt++; pthread_rwlock_unlock(&lock); @@ -2277,7 +2407,8 @@ int dl_iterate_phdr(int(*callback)(struct dl_phdr_info *info, size_t size, void info.dlpi_adds = gencnt; info.dlpi_subs = 0; info.dlpi_tls_modid = current->tls_id; - info.dlpi_tls_data = current->tls.image; + info.dlpi_tls_data = !current->tls_id ? 0 : + __tls_get_addr((tls_mod_off_t[]){current->tls_id,0}); ret = (callback)(&info, sizeof (info), data); @@ -2290,7 +2421,7 @@ int dl_iterate_phdr(int(*callback)(struct dl_phdr_info *info, size_t size, void return ret; } -static void error(const char *fmt, ...) +static void error_impl(const char *fmt, ...) { va_list ap; va_start(ap, fmt); @@ -2304,3 +2435,7 @@ static void error(const char *fmt, ...) __dl_vseterr(fmt, ap); va_end(ap); } + +static void error_noop(const char *fmt, ...) +{ +} |