summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
Diffstat
-rw-r--r--ldso/dynlink.c121
-rw-r--r--src/internal/pthread_impl.h1
-rw-r--r--src/ldso/aarch64/tlsdesc.s59
-rw-r--r--src/ldso/arm/tlsdesc.S19
-rw-r--r--src/ldso/i386/tlsdesc.s8
-rw-r--r--src/ldso/x86_64/tlsdesc.s21
-rw-r--r--src/thread/__tls_get_addr.c7
-rw-r--r--src/thread/i386/tls.s8
-rw-r--r--src/thread/pthread_create.c2
9 files changed, 86 insertions, 160 deletions
diff --git a/ldso/dynlink.c b/ldso/dynlink.c
index ec921dfd..9e2adb21 100644
--- a/ldso/dynlink.c
+++ b/ldso/dynlink.c
@@ -17,6 +17,7 @@
#include <pthread.h>
#include <ctype.h>
#include <dlfcn.h>
+#include <semaphore.h>
#include "pthread_impl.h"
#include "libc.h"
#include "dynlink.h"
@@ -1338,48 +1339,6 @@ void __init_tls(size_t *auxv)
{
}
-hidden void *__tls_get_new(tls_mod_off_t *v)
-{
- pthread_t self = __pthread_self();
-
- /* Block signals to make accessing new TLS async-signal-safe */
- sigset_t set;
- __block_all_sigs(&set);
- if (v[0] <= self->dtv[0]) {
- __restore_sigs(&set);
- return (void *)(self->dtv[v[0]] + v[1]);
- }
-
- /* This is safe without any locks held because, if the caller
- * is able to request the Nth entry of the DTV, the DSO list
- * must be valid at least that far out and it was synchronized
- * at program startup or by an already-completed call to dlopen. */
- struct dso *p;
- for (p=head; p->tls_id != v[0]; p=p->next);
-
- /* Get new DTV space from new DSO */
- uintptr_t *newdtv = p->new_dtv +
- (v[0]+1)*a_fetch_add(&p->new_dtv_idx,1);
- memcpy(newdtv, self->dtv, (self->dtv[0]+1) * sizeof(uintptr_t));
- newdtv[0] = v[0];
- self->dtv = self->dtv_copy = newdtv;
-
- /* Get new TLS memory from all new DSOs up to the requested one */
- unsigned char *mem;
- for (p=head; ; p=p->next) {
- if (!p->tls_id || self->dtv[p->tls_id]) continue;
- mem = p->new_tls + (p->tls.size + p->tls.align)
- * a_fetch_add(&p->new_tls_idx,1);
- mem += ((uintptr_t)p->tls.image - (uintptr_t)mem)
- & (p->tls.align-1);
- self->dtv[p->tls_id] = (uintptr_t)mem + DTP_OFFSET;
- memcpy(mem, p->tls.image, p->tls.len);
- if (p->tls_id == v[0]) break;
- }
- __restore_sigs(&set);
- return mem + v[1] + DTP_OFFSET;
-}
-
static void update_tls_size()
{
libc.tls_cnt = tls_cnt;
@@ -1392,6 +1351,82 @@ static void update_tls_size()
tls_align);
}
+void __dl_prepare_for_threads(void)
+{
+ /* MEMBARRIER_CMD_REGISTER_PRIVATE_EXPEDITED */
+ __syscall(SYS_membarrier, 1<<4, 0);
+}
+
+static sem_t barrier_sem;
+static void bcast_barrier(int s)
+{
+ sem_post(&barrier_sem);
+}
+
+static void install_new_tls(void)
+{
+ sigset_t set;
+ pthread_t self = __pthread_self(), td;
+ uintptr_t (*newdtv)[tls_cnt+1] = (void *)tail->new_dtv;
+ struct dso *p;
+ size_t i, j;
+ size_t old_cnt = self->dtv[0];
+
+ __block_app_sigs(&set);
+ __tl_lock();
+ /* Copy existing dtv contents from all existing threads. */
+ for (i=0, td=self; !i || td!=self; i++, td=td->next) {
+ memcpy(newdtv+i, td->dtv,
+ (old_cnt+1)*sizeof(uintptr_t));
+ newdtv[i][0] = tls_cnt;
+ }
+ /* Install new dtls into the enlarged, uninstalled dtv copies. */
+ for (p=head; ; p=p->next) {
+ if (!p->tls_id || self->dtv[p->tls_id]) continue;
+ unsigned char *mem = p->new_tls;
+ for (j=0; j<i; j++) {
+ unsigned char *new = mem;
+ new += ((uintptr_t)p->tls.image - (uintptr_t)mem)
+ & (p->tls.align-1);
+ memcpy(new, p->tls.image, p->tls.len);
+ newdtv[j][p->tls_id] =
+ (uintptr_t)new + DTP_OFFSET;
+ mem += p->tls.size + p->tls.align;
+ }
+ if (p->tls_id == tls_cnt) break;
+ }
+
+ /* Broadcast barrier to ensure contents of new dtv is visible
+ * if the new dtv pointer is. Use SYS_membarrier if it works,
+ * otherwise emulate with a signal. */
+
+ /* MEMBARRIER_CMD_PRIVATE_EXPEDITED */
+ if (__syscall(SYS_membarrier, 1<<3, 0)) {
+ sem_init(&barrier_sem, 0, 0);
+ struct sigaction sa = {
+ .sa_flags = SA_RESTART,
+ .sa_handler = bcast_barrier
+ };
+ memset(&sa.sa_mask, -1, sizeof sa.sa_mask);
+ __libc_sigaction(SIGSYNCCALL, &sa, 0);
+ for (td=self->next; td!=self; td=td->next)
+ if (j) __syscall(SYS_tkill, td->tid, SIGSYNCCALL);
+ for (td=self->next; td!=self; td=td->next)
+ sem_wait(&barrier_sem);
+ sa.sa_handler = SIG_IGN;
+ __libc_sigaction(SIGSYNCCALL, &sa, 0);
+ sem_destroy(&barrier_sem);
+ }
+
+ /* Install new dtv for each thread. */
+ for (j=0, td=self; !j || td!=self; j++, td=td->next) {
+ td->dtv = td->dtv_copy = newdtv[j];
+ }
+
+ __tl_unlock();
+ __restore_sigs(&set);
+}
+
/* Stage 1 of the dynamic linker is defined in dlstart.c. It calls the
* following stage 2 and stage 3 functions via primitive symbolic lookup
* since it does not have access to their addresses to begin with. */
@@ -1864,6 +1899,8 @@ void *dlopen(const char *file, int mode)
redo_lazy_relocs();
update_tls_size();
+ if (tls_cnt != orig_tls_cnt)
+ install_new_tls();
_dl_debug_state();
orig_tail = tail;
end:
diff --git a/src/internal/pthread_impl.h b/src/internal/pthread_impl.h
index d5d969ec..de089967 100644
--- a/src/internal/pthread_impl.h
+++ b/src/internal/pthread_impl.h
@@ -130,6 +130,7 @@ hidden int __init_tp(void *);
hidden void *__copy_tls(unsigned char *);
hidden void __reset_tls();
+hidden void __dl_prepare_for_threads(void);
hidden void __dl_thread_cleanup(void);
hidden void __testcancel();
hidden void __do_cleanup_push(struct __ptcb *);
diff --git a/src/ldso/aarch64/tlsdesc.s b/src/ldso/aarch64/tlsdesc.s
index 8e4004d7..c91baa45 100644
--- a/src/ldso/aarch64/tlsdesc.s
+++ b/src/ldso/aarch64/tlsdesc.s
@@ -29,67 +29,10 @@ __tlsdesc_dynamic:
ldr x0,[x0,#8] // p
ldr x2,[x0] // p->modidx
ldr x3,[x1,#-8] // dtv
- ldr x4,[x3] // dtv[0]
- cmp x2,x4
- b.hi 1f
ldr x2,[x3,x2,lsl #3] // dtv[p->modidx]
ldr x0,[x0,#8] // p->off
add x0,x0,x2
-2: sub x0,x0,x1
+ sub x0,x0,x1
ldp x3,x4,[sp,#16]
ldp x1,x2,[sp],#32
ret
-
- // save all registers __tls_get_new may clobber
- // update sp in two steps because offset must be in [-512,509]
-1: stp x29,x30,[sp,#-160]!
- stp x5,x6,[sp,#16]
- stp x7,x8,[sp,#32]
- stp x9,x10,[sp,#48]
- stp x11,x12,[sp,#64]
- stp x13,x14,[sp,#80]
- stp x15,x16,[sp,#96]
- stp x17,x18,[sp,#112]
- stp q0,q1,[sp,#128]
- stp q2,q3,[sp,#-480]!
- stp q4,q5,[sp,#32]
- stp q6,q7,[sp,#64]
- stp q8,q9,[sp,#96]
- stp q10,q11,[sp,#128]
- stp q12,q13,[sp,#160]
- stp q14,q15,[sp,#192]
- stp q16,q17,[sp,#224]
- stp q18,q19,[sp,#256]
- stp q20,q21,[sp,#288]
- stp q22,q23,[sp,#320]
- stp q24,q25,[sp,#352]
- stp q26,q27,[sp,#384]
- stp q28,q29,[sp,#416]
- stp q30,q31,[sp,#448]
- bl __tls_get_new
- mrs x1,tpidr_el0
- ldp q4,q5,[sp,#32]
- ldp q6,q7,[sp,#64]
- ldp q8,q9,[sp,#96]
- ldp q10,q11,[sp,#128]
- ldp q12,q13,[sp,#160]
- ldp q14,q15,[sp,#192]
- ldp q16,q17,[sp,#224]
- ldp q18,q19,[sp,#256]
- ldp q20,q21,[sp,#288]
- ldp q22,q23,[sp,#320]
- ldp q24,q25,[sp,#352]
- ldp q26,q27,[sp,#384]
- ldp q28,q29,[sp,#416]
- ldp q30,q31,[sp,#448]
- ldp q2,q3,[sp],#480
- ldp x5,x6,[sp,#16]
- ldp x7,x8,[sp,#32]
- ldp x9,x10,[sp,#48]
- ldp x11,x12,[sp,#64]
- ldp x13,x14,[sp,#80]
- ldp x15,x16,[sp,#96]
- ldp x17,x18,[sp,#112]
- ldp q0,q1,[sp,#128]
- ldp x29,x30,[sp],#160
- b 2b
diff --git a/src/ldso/arm/tlsdesc.S b/src/ldso/arm/tlsdesc.S
index 4e67c3e2..455eac1d 100644
--- a/src/ldso/arm/tlsdesc.S
+++ b/src/ldso/arm/tlsdesc.S
@@ -35,13 +35,9 @@ __tlsdesc_dynamic:
#endif
#endif
ldr r3,[r0,#-4] // r3 = dtv
- ldr ip,[r3] // ip = dtv slot count
- cmp r1,ip
- bhi 3f
ldr ip,[r3,r1,LSL #2]
sub r0,ip,r0
add r0,r0,r2 // r0 = r3[r1]-r0+r2
-4:
#if __ARM_ARCH >= 5
pop {r2,r3,ip,pc}
#else
@@ -49,21 +45,6 @@ __tlsdesc_dynamic:
bx lr
#endif
-3:
-#if __ARM_PCS_VFP || !__SOFTFP__
- .fpu vfp
- vpush {d0-d7}
-#endif
- push {r0-r3}
- add r0,sp,#4
- bl __tls_get_new
- pop {r1-r3,ip}
-#if __ARM_PCS_VFP || !__SOFTFP__
- vpop {d0-d7}
-#endif
- sub r0,r0,r1 // r0 = retval-tp
- b 4b
-
#if ((__ARM_ARCH_6K__ || __ARM_ARCH_6KZ__ || __ARM_ARCH_6ZK__) && !__thumb__) \
|| __ARM_ARCH_7A__ || __ARM_ARCH_7R__ || __ARM_ARCH >= 7
#else
diff --git a/src/ldso/i386/tlsdesc.s b/src/ldso/i386/tlsdesc.s
index 4a553bce..a5c0100c 100644
--- a/src/ldso/i386/tlsdesc.s
+++ b/src/ldso/i386/tlsdesc.s
@@ -17,15 +17,9 @@ __tlsdesc_dynamic:
mov %gs:4,%edx
push %ecx
mov (%eax),%ecx
- cmp %ecx,(%edx)
- jc 1f
mov 4(%eax),%eax
add (%edx,%ecx,4),%eax
-2: pop %ecx
+ pop %ecx
sub %gs:0,%eax
pop %edx
ret
-1: push %eax
- call __tls_get_new
- pop %ecx
- jmp 2b
diff --git a/src/ldso/x86_64/tlsdesc.s b/src/ldso/x86_64/tlsdesc.s
index 8238c3eb..0151d15c 100644
--- a/src/ldso/x86_64/tlsdesc.s
+++ b/src/ldso/x86_64/tlsdesc.s
@@ -17,28 +17,9 @@ __tlsdesc_dynamic:
mov %fs:8,%rdx
push %rcx
mov (%rax),%rcx
- cmp %rcx,(%rdx)
- jc 1f
mov 8(%rax),%rax
add (%rdx,%rcx,8),%rax
-2: pop %rcx
+ pop %rcx
sub %fs:0,%rax
pop %rdx
ret
-1: push %rdi
- push %rdi
- push %rsi
- push %r8
- push %r9
- push %r10
- push %r11
- mov %rax,%rdi
- call __tls_get_new
- pop %r11
- pop %r10
- pop %r9
- pop %r8
- pop %rsi
- pop %rdi
- pop %rdi
- jmp 2b
diff --git a/src/thread/__tls_get_addr.c b/src/thread/__tls_get_addr.c
index d7afdabd..19524fe0 100644
--- a/src/thread/__tls_get_addr.c
+++ b/src/thread/__tls_get_addr.c
@@ -1,12 +1,7 @@
-#include <stddef.h>
#include "pthread_impl.h"
void *__tls_get_addr(tls_mod_off_t *v)
{
pthread_t self = __pthread_self();
- if (v[0] <= self->dtv[0])
- return (void *)(self->dtv[v[0]] + v[1]);
- return __tls_get_new(v);
+ return (void *)(self->dtv[v[0]] + v[1]);
}
-
-weak_alias(__tls_get_addr, __tls_get_new);
diff --git a/src/thread/i386/tls.s b/src/thread/i386/tls.s
index 76d5d462..6e4c4cb9 100644
--- a/src/thread/i386/tls.s
+++ b/src/thread/i386/tls.s
@@ -4,14 +4,6 @@
___tls_get_addr:
mov %gs:4,%edx
mov (%eax),%ecx
- cmp %ecx,(%edx)
- jc 1f
mov 4(%eax),%eax
add (%edx,%ecx,4),%eax
ret
-1: push %eax
-.weak __tls_get_new
-.hidden __tls_get_new
- call __tls_get_new
- pop %edx
- ret
diff --git a/src/thread/pthread_create.c b/src/thread/pthread_create.c
index cec82157..0142b347 100644
--- a/src/thread/pthread_create.c
+++ b/src/thread/pthread_create.c
@@ -15,6 +15,7 @@ weak_alias(dummy_0, __release_ptc);
weak_alias(dummy_0, __pthread_tsd_run_dtors);
weak_alias(dummy_0, __do_orphaned_stdio_locks);
weak_alias(dummy_0, __dl_thread_cleanup);
+weak_alias(dummy_0, __dl_prepare_for_threads);
void __tl_lock(void)
{
@@ -235,6 +236,7 @@ int __pthread_create(pthread_t *restrict res, const pthread_attr_t *restrict att
init_file_lock(__stderr_used);
__syscall(SYS_rt_sigprocmask, SIG_UNBLOCK, SIGPT_SET, 0, _NSIG/8);
self->tsd = (void **)__pthread_tsd_main;
+ __dl_prepare_for_threads();
libc.threaded = 1;
}
if (attrp && !c11) attr = *attrp;
generated by cgit v1.2.1 (git 2.18.0) at 2025-10-31 04:59:29 +0000