diff options
Diffstat (limited to 'src/thread')
30 files changed, 266 insertions, 49 deletions
diff --git a/src/thread/aarch64/__set_thread_area.c b/src/thread/aarch64/__set_thread_area.c new file mode 100644 index 00000000..2ec788e8 --- /dev/null +++ b/src/thread/aarch64/__set_thread_area.c @@ -0,0 +1,27 @@ +#include <elf.h> +#include "libc.h" + +#define BITRANGE(a,b) (2*(1UL<<(b))-(1UL<<(a))) + +int __set_thread_area(void *p) +{ + __asm__ __volatile__ ("msr tpidr_el0,%0" : : "r"(p) : "memory"); + + /* Mask off hwcap bits for SME and unknown future features. This is + * necessary because SME is not safe to use without libc support for + * it, and we do not (yet) have such support. */ + for (size_t *v = libc.auxv; *v; v+=2) { + if (v[0]==AT_HWCAP) { + v[1] &= ~BITRANGE(42,63); /* 42-47 are SME */ + } else if (v[0]==AT_HWCAP2) { + v[1] &= ~(BITRANGE(23,30) + | BITRANGE(37,42) + | BITRANGE(57,62)); + } else if (v[0]==AT_HWCAP3 || v[0]==AT_HWCAP4) { + v[0] = AT_IGNORE; + v[1] = 0; + } + } + + return 0; +} diff --git a/src/thread/aarch64/clone.s b/src/thread/aarch64/clone.s index e3c83395..aff8155b 100644 --- a/src/thread/aarch64/clone.s +++ b/src/thread/aarch64/clone.s @@ -24,7 +24,8 @@ __clone: // parent ret // child -1: ldp x1,x0,[sp],#16 +1: mov x29, 0 + ldp x1,x0,[sp],#16 blr x1 mov x8,#93 // SYS_exit svc #0 diff --git a/src/thread/arm/clone.s b/src/thread/arm/clone.s index bb0965da..4ff0c0e8 100644 --- a/src/thread/arm/clone.s +++ b/src/thread/arm/clone.s @@ -19,7 +19,8 @@ __clone: ldmfd sp!,{r4,r5,r6,r7} bx lr -1: mov r0,r6 +1: mov fp,#0 + mov r0,r6 bl 3f 2: mov r7,#1 svc 0 diff --git a/src/thread/aarch64/__set_thread_area.s b/src/thread/loongarch64/__set_thread_area.s index fd0df34b..021307fc 100644 --- a/src/thread/aarch64/__set_thread_area.s +++ b/src/thread/loongarch64/__set_thread_area.s @@ -2,6 +2,6 @@ .hidden __set_thread_area .type __set_thread_area,@function __set_thread_area: - msr tpidr_el0,x0 - mov w0,#0 - ret + move $tp, $a0 + move $a0, $zero + jr $ra diff --git a/src/thread/loongarch64/__unmapself.s b/src/thread/loongarch64/__unmapself.s new file mode 100644 index 00000000..719ad056 --- /dev/null +++ b/src/thread/loongarch64/__unmapself.s @@ -0,0 +1,7 @@ +.global __unmapself +.type __unmapself, @function +__unmapself: + li.d $a7, 215 # call munmap + syscall 0 + li.d $a7, 93 # call exit + syscall 0 diff --git a/src/thread/loongarch64/clone.s b/src/thread/loongarch64/clone.s new file mode 100644 index 00000000..cb4aacfc --- /dev/null +++ b/src/thread/loongarch64/clone.s @@ -0,0 +1,30 @@ +#__clone(func, stack, flags, arg, ptid, tls, ctid) +# a0, a1, a2, a3, a4, a5, a6 +# sys_clone(flags, stack, ptid, ctid, tls) +# a0, a1, a2, a3, a4 + +.global __clone +.hidden __clone +.type __clone,@function +__clone: + bstrins.d $a1, $zero, 3, 0 #stack to 16 align + # Save function pointer and argument pointer on new thread stack + addi.d $a1, $a1, -16 + st.d $a0, $a1, 0 # save function pointer + st.d $a3, $a1, 8 # save argument pointer + or $a0, $a2, $zero + or $a2, $a4, $zero + or $a3, $a6, $zero + or $a4, $a5, $zero + ori $a7, $zero, 220 + syscall 0 # call clone + + beqz $a0, 1f # whether child process + jirl $zero, $ra, 0 # parent process return +1: + move $fp, $zero + ld.d $t8, $sp, 0 # function pointer + ld.d $a0, $sp, 8 # argument pointer + jirl $ra, $t8, 0 # call the user's function + ori $a7, $zero, 93 + syscall 0 # child process exit diff --git a/src/thread/loongarch64/syscall_cp.s b/src/thread/loongarch64/syscall_cp.s new file mode 100644 index 00000000..c057a97b --- /dev/null +++ b/src/thread/loongarch64/syscall_cp.s @@ -0,0 +1,29 @@ +.global __cp_begin +.hidden __cp_begin +.global __cp_end +.hidden __cp_end +.global __cp_cancel +.hidden __cp_cancel +.hidden __cancel +.global __syscall_cp_asm +.hidden __syscall_cp_asm +.type __syscall_cp_asm,@function + +__syscall_cp_asm: +__cp_begin: + ld.w $a0, $a0, 0 + bnez $a0, __cp_cancel + move $t8, $a1 # reserve system call number + move $a0, $a2 + move $a1, $a3 + move $a2, $a4 + move $a3, $a5 + move $a4, $a6 + move $a5, $a7 + move $a7, $t8 + syscall 0 +__cp_end: + jr $ra +__cp_cancel: + la.local $t8, __cancel + jr $t8 diff --git a/src/thread/m68k/clone.s b/src/thread/m68k/clone.s index f6dfa06f..0134cf4e 100644 --- a/src/thread/m68k/clone.s +++ b/src/thread/m68k/clone.s @@ -18,7 +18,8 @@ __clone: beq 1f movem.l (%sp)+,%d2-%d5 rts -1: move.l %a1,-(%sp) +1: suba.l %fp,%fp + move.l %a1,-(%sp) jsr (%a0) move.l #1,%d0 trap #0 diff --git a/src/thread/microblaze/clone.s b/src/thread/microblaze/clone.s index b68cc5fc..64e3f074 100644 --- a/src/thread/microblaze/clone.s +++ b/src/thread/microblaze/clone.s @@ -22,7 +22,8 @@ __clone: rtsd r15, 8 nop -1: lwi r3, r1, 0 +1: add r19, r0, r0 + lwi r3, r1, 0 lwi r5, r1, 4 brald r15, r3 nop diff --git a/src/thread/mips/clone.s b/src/thread/mips/clone.s index 04463385..229b987e 100644 --- a/src/thread/mips/clone.s +++ b/src/thread/mips/clone.s @@ -27,7 +27,8 @@ __clone: addu $sp, $sp, 16 jr $ra nop -1: lw $25, 0($sp) +1: move $fp, $0 + lw $25, 0($sp) lw $4, 4($sp) jalr $25 nop diff --git a/src/thread/mips64/clone.s b/src/thread/mips64/clone.s index 2d86899a..8de3db6c 100644 --- a/src/thread/mips64/clone.s +++ b/src/thread/mips64/clone.s @@ -25,7 +25,8 @@ __clone: nop jr $ra nop -1: ld $25, 0($sp) # function pointer +1: move $fp, $0 + ld $25, 0($sp) # function pointer ld $4, 8($sp) # argument pointer jalr $25 # call the user's function nop diff --git a/src/thread/mipsn32/clone.s b/src/thread/mipsn32/clone.s index 4d3c8c7a..9571231a 100644 --- a/src/thread/mipsn32/clone.s +++ b/src/thread/mipsn32/clone.s @@ -25,7 +25,8 @@ __clone: nop jr $ra nop -1: lw $25, 0($sp) # function pointer +1: move $fp, $0 + lw $25, 0($sp) # function pointer lw $4, 4($sp) # argument pointer jalr $25 # call the user's function nop diff --git a/src/thread/or1k/clone.s b/src/thread/or1k/clone.s index 2473ac20..b41488a2 100644 --- a/src/thread/or1k/clone.s +++ b/src/thread/or1k/clone.s @@ -6,6 +6,8 @@ .hidden __clone .type __clone,@function __clone: + l.xori r11, r0, -4 + l.and r4, r4, r11 l.addi r4, r4, -8 l.sw 0(r4), r3 l.sw 4(r4), r6 @@ -23,7 +25,8 @@ __clone: l.jr r9 l.nop -1: l.lwz r11, 0(r1) +1: l.ori r2, r0, 0 + l.lwz r11, 0(r1) l.jalr r11 l.lwz r3, 4(r1) diff --git a/src/thread/pthread_atfork.c b/src/thread/pthread_atfork.c index 76497401..26d32543 100644 --- a/src/thread/pthread_atfork.c +++ b/src/thread/pthread_atfork.c @@ -1,7 +1,13 @@ #include <pthread.h> +#include <errno.h> #include "libc.h" #include "lock.h" +#define malloc __libc_malloc +#define calloc undef +#define realloc undef +#define free undef + static struct atfork_funcs { void (*prepare)(void); void (*parent)(void); @@ -34,7 +40,7 @@ void __fork_handler(int who) int pthread_atfork(void (*prepare)(void), void (*parent)(void), void (*child)(void)) { struct atfork_funcs *new = malloc(sizeof *new); - if (!new) return -1; + if (!new) return ENOMEM; LOCK(lock); new->next = funcs; diff --git a/src/thread/pthread_cancel.c b/src/thread/pthread_cancel.c index 2f9d5e97..139a6fc8 100644 --- a/src/thread/pthread_cancel.c +++ b/src/thread/pthread_cancel.c @@ -56,7 +56,12 @@ static void cancel_handler(int sig, siginfo_t *si, void *ctx) _sigaddset(&uc->uc_sigmask, SIGCANCEL); - if (self->cancelasync || pc >= (uintptr_t)__cp_begin && pc < (uintptr_t)__cp_end) { + if (self->cancelasync) { + pthread_sigmask(SIG_SETMASK, &uc->uc_sigmask, 0); + __cancel(); + } + + if (pc >= (uintptr_t)__cp_begin && pc < (uintptr_t)__cp_end) { uc->uc_mcontext.MC_PC = (uintptr_t)__cp_cancel; #ifdef CANCEL_GOT uc->uc_mcontext.MC_GOT = CANCEL_GOT; @@ -77,7 +82,7 @@ void __testcancel() static void init_cancellation() { struct sigaction sa = { - .sa_flags = SA_SIGINFO | SA_RESTART, + .sa_flags = SA_SIGINFO | SA_RESTART | SA_ONSTACK, .sa_sigaction = cancel_handler }; memset(&sa.sa_mask, -1, _NSIG/8); diff --git a/src/thread/pthread_create.c b/src/thread/pthread_create.c index 6f187ee8..087f6206 100644 --- a/src/thread/pthread_create.c +++ b/src/thread/pthread_create.c @@ -107,6 +107,16 @@ _Noreturn void __pthread_exit(void *result) /* At this point we are committed to thread termination. */ + /* After the kernel thread exits, its tid may be reused. Clear it + * to prevent inadvertent use and inform functions that would use + * it that it's no longer available. At this point the killlock + * may be released, since functions that use it will consistently + * see the thread as having exited. Release it now so that no + * remaining locks (except thread list) are held if we end up + * resetting need_locks below. */ + self->tid = 0; + UNLOCK(self->killlock); + /* Process robust list in userspace to handle non-pshared mutexes * and the detached thread case where the robust list head will * be invalid when the kernel would process it. */ @@ -159,12 +169,6 @@ _Noreturn void __pthread_exit(void *result) a_store(&self->detach_state, DT_EXITED); __wake(&self->detach_state, 1, 1); - /* After the kernel thread exits, its tid may be reused. Clear it - * to prevent inadvertent use and inform functions that would use - * it that it's no longer available. */ - self->tid = 0; - UNLOCK(self->killlock); - for (;;) __syscall(SYS_exit, 0); } diff --git a/src/thread/pthread_detach.c b/src/thread/pthread_detach.c index 77772af2..d73a500e 100644 --- a/src/thread/pthread_detach.c +++ b/src/thread/pthread_detach.c @@ -5,8 +5,12 @@ static int __pthread_detach(pthread_t t) { /* If the cas fails, detach state is either already-detached * or exiting/exited, and pthread_join will trap or cleanup. */ - if (a_cas(&t->detach_state, DT_JOINABLE, DT_DETACHED) != DT_JOINABLE) - return __pthread_join(t, 0); + if (a_cas(&t->detach_state, DT_JOINABLE, DT_DETACHED) != DT_JOINABLE) { + int cs; + __pthread_setcancelstate(PTHREAD_CANCEL_DISABLE, &cs); + __pthread_join(t, 0); + __pthread_setcancelstate(cs, 0); + } return 0; } diff --git a/src/thread/pthread_getname_np.c b/src/thread/pthread_getname_np.c index 48d1a294..85504e45 100644 --- a/src/thread/pthread_getname_np.c +++ b/src/thread/pthread_getname_np.c @@ -17,7 +17,7 @@ int pthread_getname_np(pthread_t thread, char *name, size_t len) snprintf(f, sizeof f, "/proc/self/task/%d/comm", thread->tid); pthread_setcancelstate(PTHREAD_CANCEL_DISABLE, &cs); - if ((fd = open(f, O_RDONLY|O_CLOEXEC)) < 0 || (len = read(fd, name, len)) < 0) status = errno; + if ((fd = open(f, O_RDONLY|O_CLOEXEC)) < 0 || (len = read(fd, name, len)) == -1) status = errno; else name[len-1] = 0; /* remove trailing new line only if successful */ if (fd >= 0) close(fd); pthread_setcancelstate(cs, 0); diff --git a/src/thread/pthread_key_create.c b/src/thread/pthread_key_create.c index d1120941..39770c7a 100644 --- a/src/thread/pthread_key_create.c +++ b/src/thread/pthread_key_create.c @@ -1,4 +1,5 @@ #include "pthread_impl.h" +#include "fork_impl.h" volatile size_t __pthread_tsd_size = sizeof(void *) * PTHREAD_KEYS_MAX; void *__pthread_tsd_main[PTHREAD_KEYS_MAX] = { 0 }; @@ -20,6 +21,13 @@ static void dummy_0(void) weak_alias(dummy_0, __tl_lock); weak_alias(dummy_0, __tl_unlock); +void __pthread_key_atfork(int who) +{ + if (who<0) __pthread_rwlock_rdlock(&key_lock); + else if (!who) __pthread_rwlock_unlock(&key_lock); + else key_lock = (pthread_rwlock_t)PTHREAD_RWLOCK_INITIALIZER; +} + int __pthread_key_create(pthread_key_t *k, void (*dtor)(void *)) { pthread_t self = __pthread_self(); diff --git a/src/thread/riscv32/__set_thread_area.s b/src/thread/riscv32/__set_thread_area.s new file mode 100644 index 00000000..828154d2 --- /dev/null +++ b/src/thread/riscv32/__set_thread_area.s @@ -0,0 +1,6 @@ +.global __set_thread_area +.type __set_thread_area, %function +__set_thread_area: + mv tp, a0 + li a0, 0 + ret diff --git a/src/thread/riscv32/__unmapself.s b/src/thread/riscv32/__unmapself.s new file mode 100644 index 00000000..2849119c --- /dev/null +++ b/src/thread/riscv32/__unmapself.s @@ -0,0 +1,7 @@ +.global __unmapself +.type __unmapself, %function +__unmapself: + li a7, 215 # SYS_munmap + ecall + li a7, 93 # SYS_exit + ecall diff --git a/src/thread/riscv32/clone.s b/src/thread/riscv32/clone.s new file mode 100644 index 00000000..e2116e33 --- /dev/null +++ b/src/thread/riscv32/clone.s @@ -0,0 +1,35 @@ +# __clone(func, stack, flags, arg, ptid, tls, ctid) +# a0, a1, a2, a3, a4, a5, a6 + +# syscall(SYS_clone, flags, stack, ptid, tls, ctid) +# a7 a0, a1, a2, a3, a4 + +.global __clone +.type __clone, %function +__clone: + # Save func and arg to stack + andi a1, a1, -16 + addi a1, a1, -16 + sw a0, 0(a1) + sw a3, 4(a1) + + # Call SYS_clone + mv a0, a2 + mv a2, a4 + mv a3, a5 + mv a4, a6 + li a7, 220 # SYS_clone + ecall + + beqz a0, 1f + # Parent + ret + + # Child +1: lw a1, 0(sp) + lw a0, 4(sp) + jalr a1 + + # Exit + li a7, 93 # SYS_exit + ecall diff --git a/src/thread/riscv32/syscall_cp.s b/src/thread/riscv32/syscall_cp.s new file mode 100644 index 00000000..079d1ba0 --- /dev/null +++ b/src/thread/riscv32/syscall_cp.s @@ -0,0 +1,29 @@ +.global __cp_begin +.hidden __cp_begin +.global __cp_end +.hidden __cp_end +.global __cp_cancel +.hidden __cp_cancel +.hidden __cancel +.global __syscall_cp_asm +.hidden __syscall_cp_asm +.type __syscall_cp_asm, %function +__syscall_cp_asm: +__cp_begin: + lw t0, 0(a0) + bnez t0, __cp_cancel + + mv t0, a1 + mv a0, a2 + mv a1, a3 + mv a2, a4 + mv a3, a5 + mv a4, a6 + mv a5, a7 + lw a6, 0(sp) + mv a7, t0 + ecall +__cp_end: + ret +__cp_cancel: + tail __cancel diff --git a/src/thread/riscv64/clone.s b/src/thread/riscv64/clone.s index db908248..0e6f41a8 100644 --- a/src/thread/riscv64/clone.s +++ b/src/thread/riscv64/clone.s @@ -8,6 +8,7 @@ .type __clone, %function __clone: # Save func and arg to stack + andi a1, a1, -16 addi a1, a1, -16 sd a0, 0(a1) sd a3, 8(a1) diff --git a/src/thread/s390x/__tls_get_offset.s b/src/thread/s390x/__tls_get_offset.s index 8ee92de8..056c9110 100644 --- a/src/thread/s390x/__tls_get_offset.s +++ b/src/thread/s390x/__tls_get_offset.s @@ -1,17 +1,17 @@ .global __tls_get_offset .type __tls_get_offset,%function __tls_get_offset: - stmg %r14, %r15, 112(%r15) - aghi %r15, -160 + ear %r3, %a0 + sllg %r3, %r3, 32 + ear %r3, %a1 - la %r2, 0(%r2, %r12) - brasl %r14, __tls_get_addr + la %r1, 0(%r2, %r12) - ear %r1, %a0 - sllg %r1, %r1, 32 - ear %r1, %a1 + lg %r0, 0(%r1) + sllg %r4, %r0, 3 + lg %r5, 8(%r3) + lg %r2, 0(%r4, %r5) + ag %r2, 8(%r1) + sgr %r2, %r3 - sgr %r2, %r1 - - lmg %r14, %r15, 272(%r15) br %r14 diff --git a/src/thread/sem_getvalue.c b/src/thread/sem_getvalue.c index d9d83071..c0b7762d 100644 --- a/src/thread/sem_getvalue.c +++ b/src/thread/sem_getvalue.c @@ -1,8 +1,9 @@ #include <semaphore.h> +#include <limits.h> int sem_getvalue(sem_t *restrict sem, int *restrict valp) { int val = sem->__val[0]; - *valp = val < 0 ? 0 : val; + *valp = val & SEM_VALUE_MAX; return 0; } diff --git a/src/thread/sem_post.c b/src/thread/sem_post.c index 31e3293d..1c8f763c 100644 --- a/src/thread/sem_post.c +++ b/src/thread/sem_post.c @@ -1,17 +1,21 @@ #include <semaphore.h> +#include <limits.h> #include "pthread_impl.h" int sem_post(sem_t *sem) { - int val, waiters, priv = sem->__val[2]; + int val, new, waiters, priv = sem->__val[2]; do { val = sem->__val[0]; waiters = sem->__val[1]; - if (val == SEM_VALUE_MAX) { + if ((val & SEM_VALUE_MAX) == SEM_VALUE_MAX) { errno = EOVERFLOW; return -1; } - } while (a_cas(sem->__val, val, val+1+(val<0)) != val); - if (val<0 || waiters) __wake(sem->__val, 1, priv); + new = val + 1; + if (waiters <= 1) + new &= ~0x80000000; + } while (a_cas(sem->__val, val, new) != val); + if (val<0 || waiters) __wake(sem->__val, waiters>1 ? 1 : -1, priv); return 0; } diff --git a/src/thread/sem_timedwait.c b/src/thread/sem_timedwait.c index 58d3ebfe..aa67376c 100644 --- a/src/thread/sem_timedwait.c +++ b/src/thread/sem_timedwait.c @@ -1,4 +1,5 @@ #include <semaphore.h> +#include <limits.h> #include "pthread_impl.h" static void cleanup(void *p) @@ -13,14 +14,15 @@ int sem_timedwait(sem_t *restrict sem, const struct timespec *restrict at) if (!sem_trywait(sem)) return 0; int spins = 100; - while (spins-- && sem->__val[0] <= 0 && !sem->__val[1]) a_spin(); + while (spins-- && !(sem->__val[0] & SEM_VALUE_MAX) && !sem->__val[1]) + a_spin(); while (sem_trywait(sem)) { - int r; + int r, priv = sem->__val[2]; a_inc(sem->__val+1); - a_cas(sem->__val, 0, -1); + a_cas(sem->__val, 0, 0x80000000); pthread_cleanup_push(cleanup, (void *)(sem->__val+1)); - r = __timedwait_cp(sem->__val, -1, CLOCK_REALTIME, at, sem->__val[2]); + r = __timedwait_cp(sem->__val, 0x80000000, CLOCK_REALTIME, at, priv); pthread_cleanup_pop(1); if (r) { errno = r; diff --git a/src/thread/sem_trywait.c b/src/thread/sem_trywait.c index 04edf46b..beb435da 100644 --- a/src/thread/sem_trywait.c +++ b/src/thread/sem_trywait.c @@ -1,12 +1,12 @@ #include <semaphore.h> +#include <limits.h> #include "pthread_impl.h" int sem_trywait(sem_t *sem) { int val; - while ((val=sem->__val[0]) > 0) { - int new = val-1-(val==1 && sem->__val[1]); - if (a_cas(sem->__val, val, new)==val) return 0; + while ((val=sem->__val[0]) & SEM_VALUE_MAX) { + if (a_cas(sem->__val, val, val-1)==val) return 0; } errno = EAGAIN; return -1; diff --git a/src/thread/synccall.c b/src/thread/synccall.c index d58c851f..38597254 100644 --- a/src/thread/synccall.c +++ b/src/thread/synccall.c @@ -11,7 +11,7 @@ weak_alias(dummy_0, __tl_unlock); static int target_tid; static void (*callback)(void *), *context; -static sem_t target_sem, caller_sem; +static sem_t target_sem, caller_sem, exit_sem; static void dummy(void *p) { @@ -33,7 +33,7 @@ static void handler(int sig) /* Inform caller we've complered the callback and wait * for the caller to release us to return. */ sem_post(&caller_sem); - sem_wait(&target_sem); + sem_wait(&exit_sem); /* Inform caller we are returning and state is destroyable. */ sem_post(&caller_sem); @@ -45,7 +45,7 @@ void __synccall(void (*func)(void *), void *ctx) { sigset_t oldmask; int cs, i, r; - struct sigaction sa = { .sa_flags = SA_RESTART, .sa_handler = handler }; + struct sigaction sa = { .sa_flags = SA_RESTART | SA_ONSTACK, .sa_handler = handler }; pthread_t self = __pthread_self(), td; int count = 0; @@ -62,6 +62,7 @@ void __synccall(void (*func)(void *), void *ctx) sem_init(&target_sem, 0, 0); sem_init(&caller_sem, 0, 0); + sem_init(&exit_sem, 0, 0); if (!libc.threads_minus_1 || __syscall(SYS_gettid) != self->tid) goto single_threaded; @@ -107,12 +108,13 @@ single_threaded: /* Only release the caught threads once all threads, including the * caller, have returned from the callback function. */ for (i=0; i<count; i++) - sem_post(&target_sem); + sem_post(&exit_sem); for (i=0; i<count; i++) sem_wait(&caller_sem); sem_destroy(&caller_sem); sem_destroy(&target_sem); + sem_destroy(&exit_sem); pthread_setcancelstate(cs, 0); __tl_unlock(); |