diff options
Diffstat (limited to 'src/thread')
65 files changed, 835 insertions, 453 deletions
diff --git a/src/thread/__lock.c b/src/thread/__lock.c index 45557c88..60eece49 100644 --- a/src/thread/__lock.c +++ b/src/thread/__lock.c @@ -18,9 +18,11 @@ void __lock(volatile int *l) { - if (!libc.threads_minus_1) return; + int need_locks = libc.need_locks; + if (!need_locks) return; /* fast path: INT_MIN for the lock, +1 for the congestion */ int current = a_cas(l, 0, INT_MIN + 1); + if (need_locks < 0) libc.need_locks = 0; if (!current) return; /* A first spin loop, for medium congestion. */ for (unsigned i = 0; i < 10; ++i) { diff --git a/src/thread/__syscall_cp.c b/src/thread/__syscall_cp.c index af666f06..42a01674 100644 --- a/src/thread/__syscall_cp.c +++ b/src/thread/__syscall_cp.c @@ -7,7 +7,7 @@ static long sccp(syscall_arg_t nr, syscall_arg_t u, syscall_arg_t v, syscall_arg_t w, syscall_arg_t x, syscall_arg_t y, syscall_arg_t z) { - return (__syscall)(nr, u, v, w, x, y, z); + return __syscall(nr, u, v, w, x, y, z); } weak_alias(sccp, __syscall_cp_c); diff --git a/src/thread/__timedwait.c b/src/thread/__timedwait.c index 229db313..666093be 100644 --- a/src/thread/__timedwait.c +++ b/src/thread/__timedwait.c @@ -5,6 +5,30 @@ #include "syscall.h" #include "pthread_impl.h" +#define IS32BIT(x) !((x)+0x80000000ULL>>32) +#define CLAMP(x) (int)(IS32BIT(x) ? (x) : 0x7fffffffU+((0ULL+(x))>>63)) + +static int __futex4_cp(volatile void *addr, int op, int val, const struct timespec *to) +{ + int r; +#ifdef SYS_futex_time64 + time_t s = to ? to->tv_sec : 0; + long ns = to ? to->tv_nsec : 0; + r = -ENOSYS; + if (SYS_futex == SYS_futex_time64 || !IS32BIT(s)) + r = __syscall_cp(SYS_futex_time64, addr, op, val, + to ? ((long long[]){s, ns}) : 0); + if (SYS_futex == SYS_futex_time64 || r!=-ENOSYS) return r; + to = to ? (void *)(long[]){CLAMP(s), ns} : 0; +#endif + r = __syscall_cp(SYS_futex, addr, op, val, to); + if (r != -ENOSYS) return r; + return __syscall_cp(SYS_futex, addr, op & ~FUTEX_PRIVATE, val, to); +} + +static volatile int dummy = 0; +weak_alias(dummy, __eintr_valid_flag); + int __timedwait_cp(volatile int *addr, int val, clockid_t clk, const struct timespec *at, int priv) { @@ -25,9 +49,13 @@ int __timedwait_cp(volatile int *addr, int val, top = &to; } - r = -__syscall_cp(SYS_futex, addr, FUTEX_WAIT|priv, val, top); - if (r == ENOSYS) r = -__syscall_cp(SYS_futex, addr, FUTEX_WAIT, val, top); + r = -__futex4_cp(addr, FUTEX_WAIT|priv, val, top); if (r != EINTR && r != ETIMEDOUT && r != ECANCELED) r = 0; + /* Mitigate bug in old kernels wrongly reporting EINTR for non- + * interrupting (SA_RESTART) signal handlers. This is only practical + * when NO interrupting signal handlers have been installed, and + * works by sigaction tracking whether that's the case. */ + if (r == EINTR && !__eintr_valid_flag) r = 0; return r; } diff --git a/src/thread/__tls_get_addr.c b/src/thread/__tls_get_addr.c index d7afdabd..19524fe0 100644 --- a/src/thread/__tls_get_addr.c +++ b/src/thread/__tls_get_addr.c @@ -1,12 +1,7 @@ -#include <stddef.h> #include "pthread_impl.h" void *__tls_get_addr(tls_mod_off_t *v) { pthread_t self = __pthread_self(); - if (v[0] <= self->dtv[0]) - return (void *)(self->dtv[v[0]] + v[1]); - return __tls_get_new(v); + return (void *)(self->dtv[v[0]] + v[1]); } - -weak_alias(__tls_get_addr, __tls_get_new); diff --git a/src/thread/__unmapself.c b/src/thread/__unmapself.c index 1d3bee1d..31d94e67 100644 --- a/src/thread/__unmapself.c +++ b/src/thread/__unmapself.c @@ -4,7 +4,6 @@ /* cheat and reuse CRTJMP macro from dynlink code */ #include "dynlink.h" -static volatile int lock; static void *unmap_base; static size_t unmap_size; static char shared_stack[256]; @@ -17,12 +16,8 @@ static void do_unmap() void __unmapself(void *base, size_t size) { - int tid=__pthread_self()->tid; char *stack = shared_stack + sizeof shared_stack; stack -= (uintptr_t)stack % 16; - while (lock || a_cas(&lock, 0, tid)) - a_spin(); - __syscall(SYS_set_tid_address, &lock); unmap_base = base; unmap_size = size; CRTJMP(do_unmap, stack); diff --git a/src/thread/arm/atomics.s b/src/thread/arm/atomics.s index 101ad391..da50508d 100644 --- a/src/thread/arm/atomics.s +++ b/src/thread/arm/atomics.s @@ -15,10 +15,10 @@ __a_barrier_oldkuser: mov r1,r0 mov r2,sp ldr ip,=0xffff0fc0 - mov lr,pc - mov pc,ip + bl 1f pop {r0,r1,r2,r3,ip,lr} bx lr +1: bx ip .global __a_barrier_v6 .hidden __a_barrier_v6 diff --git a/src/thread/arm/clone.s b/src/thread/arm/clone.s index e16b1326..bb0965da 100644 --- a/src/thread/arm/clone.s +++ b/src/thread/arm/clone.s @@ -20,13 +20,9 @@ __clone: bx lr 1: mov r0,r6 - tst r5,#1 - bne 1f - mov lr,pc - mov pc,r5 + bl 3f 2: mov r7,#1 svc 0 - -1: mov lr,pc - bx r5 b 2b + +3: bx r5 diff --git a/src/thread/arm/syscall_cp.s b/src/thread/arm/syscall_cp.s index a5730c08..e607dd42 100644 --- a/src/thread/arm/syscall_cp.s +++ b/src/thread/arm/syscall_cp.s @@ -11,19 +11,19 @@ .type __syscall_cp_asm,%function __syscall_cp_asm: mov ip,sp - stmfd sp!,{r4,r5,r6,r7,lr} + stmfd sp!,{r4,r5,r6,r7} __cp_begin: ldr r0,[r0] cmp r0,#0 - blne __cp_cancel + bne __cp_cancel mov r7,r1 mov r0,r2 mov r1,r3 ldmfd ip,{r2,r3,r4,r5,r6} svc 0 __cp_end: - ldmfd sp!,{r4,r5,r6,r7,lr} + ldmfd sp!,{r4,r5,r6,r7} bx lr __cp_cancel: - ldmfd sp!,{r4,r5,r6,r7,lr} + ldmfd sp!,{r4,r5,r6,r7} b __cancel diff --git a/src/thread/i386/__set_thread_area.s b/src/thread/i386/__set_thread_area.s index c2c21dd5..aa6852be 100644 --- a/src/thread/i386/__set_thread_area.s +++ b/src/thread/i386/__set_thread_area.s @@ -28,6 +28,7 @@ __set_thread_area: ret 2: mov %ebx,%ecx + xor %eax,%eax xor %ebx,%ebx xor %edx,%edx mov %ebx,(%esp) diff --git a/src/thread/i386/tls.s b/src/thread/i386/tls.s index 76d5d462..6e4c4cb9 100644 --- a/src/thread/i386/tls.s +++ b/src/thread/i386/tls.s @@ -4,14 +4,6 @@ ___tls_get_addr: mov %gs:4,%edx mov (%eax),%ecx - cmp %ecx,(%edx) - jc 1f mov 4(%eax),%eax add (%edx,%ecx,4),%eax ret -1: push %eax -.weak __tls_get_new -.hidden __tls_get_new - call __tls_get_new - pop %edx - ret diff --git a/src/thread/loongarch64/__set_thread_area.s b/src/thread/loongarch64/__set_thread_area.s new file mode 100644 index 00000000..021307fc --- /dev/null +++ b/src/thread/loongarch64/__set_thread_area.s @@ -0,0 +1,7 @@ +.global __set_thread_area +.hidden __set_thread_area +.type __set_thread_area,@function +__set_thread_area: + move $tp, $a0 + move $a0, $zero + jr $ra diff --git a/src/thread/loongarch64/__unmapself.s b/src/thread/loongarch64/__unmapself.s new file mode 100644 index 00000000..719ad056 --- /dev/null +++ b/src/thread/loongarch64/__unmapself.s @@ -0,0 +1,7 @@ +.global __unmapself +.type __unmapself, @function +__unmapself: + li.d $a7, 215 # call munmap + syscall 0 + li.d $a7, 93 # call exit + syscall 0 diff --git a/src/thread/loongarch64/clone.s b/src/thread/loongarch64/clone.s new file mode 100644 index 00000000..a165b365 --- /dev/null +++ b/src/thread/loongarch64/clone.s @@ -0,0 +1,29 @@ +#__clone(func, stack, flags, arg, ptid, tls, ctid) +# a0, a1, a2, a3, a4, a5, a6 +# sys_clone(flags, stack, ptid, ctid, tls) +# a0, a1, a2, a3, a4 + +.global __clone +.hidden __clone +.type __clone,@function +__clone: + bstrins.d $a1, $zero, 3, 0 #stack to 16 align + # Save function pointer and argument pointer on new thread stack + addi.d $a1, $a1, -16 + st.d $a0, $a1, 0 # save function pointer + st.d $a3, $a1, 8 # save argument pointer + or $a0, $a2, $zero + or $a2, $a4, $zero + or $a3, $a6, $zero + or $a4, $a5, $zero + ori $a7, $zero, 220 + syscall 0 # call clone + + beqz $a0, 1f # whether child process + jirl $zero, $ra, 0 # parent process return +1: + ld.d $t8, $sp, 0 # function pointer + ld.d $a0, $sp, 8 # argument pointer + jirl $ra, $t8, 0 # call the user's function + ori $a7, $zero, 93 + syscall 0 # child process exit diff --git a/src/thread/loongarch64/syscall_cp.s b/src/thread/loongarch64/syscall_cp.s new file mode 100644 index 00000000..c057a97b --- /dev/null +++ b/src/thread/loongarch64/syscall_cp.s @@ -0,0 +1,29 @@ +.global __cp_begin +.hidden __cp_begin +.global __cp_end +.hidden __cp_end +.global __cp_cancel +.hidden __cp_cancel +.hidden __cancel +.global __syscall_cp_asm +.hidden __syscall_cp_asm +.type __syscall_cp_asm,@function + +__syscall_cp_asm: +__cp_begin: + ld.w $a0, $a0, 0 + bnez $a0, __cp_cancel + move $t8, $a1 # reserve system call number + move $a0, $a2 + move $a1, $a3 + move $a2, $a4 + move $a3, $a5 + move $a4, $a6 + move $a5, $a7 + move $a7, $t8 + syscall 0 +__cp_end: + jr $ra +__cp_cancel: + la.local $t8, __cancel + jr $t8 diff --git a/src/thread/pthread_atfork.c b/src/thread/pthread_atfork.c index 76497401..26d32543 100644 --- a/src/thread/pthread_atfork.c +++ b/src/thread/pthread_atfork.c @@ -1,7 +1,13 @@ #include <pthread.h> +#include <errno.h> #include "libc.h" #include "lock.h" +#define malloc __libc_malloc +#define calloc undef +#define realloc undef +#define free undef + static struct atfork_funcs { void (*prepare)(void); void (*parent)(void); @@ -34,7 +40,7 @@ void __fork_handler(int who) int pthread_atfork(void (*prepare)(void), void (*parent)(void), void (*child)(void)) { struct atfork_funcs *new = malloc(sizeof *new); - if (!new) return -1; + if (!new) return ENOMEM; LOCK(lock); new->next = funcs; diff --git a/src/thread/pthread_attr_get.c b/src/thread/pthread_attr_get.c index 4aa5afdb..f12ff442 100644 --- a/src/thread/pthread_attr_get.c +++ b/src/thread/pthread_attr_get.c @@ -70,7 +70,7 @@ int pthread_condattr_getpshared(const pthread_condattr_t *restrict a, int *restr int pthread_mutexattr_getprotocol(const pthread_mutexattr_t *restrict a, int *restrict protocol) { - *protocol = PTHREAD_PRIO_NONE; + *protocol = a->__attr / 8U % 2; return 0; } int pthread_mutexattr_getpshared(const pthread_mutexattr_t *restrict a, int *restrict pshared) diff --git a/src/thread/pthread_attr_setinheritsched.c b/src/thread/pthread_attr_setinheritsched.c index 6a648376..ca264be7 100644 --- a/src/thread/pthread_attr_setinheritsched.c +++ b/src/thread/pthread_attr_setinheritsched.c @@ -1,25 +1,6 @@ #include "pthread_impl.h" #include "syscall.h" -hidden void *__start_sched(void *p) -{ - struct start_sched_args *ssa = p; - void *start_arg = ssa->start_arg; - void *(*start_fn)(void *) = ssa->start_fn; - pthread_t self = __pthread_self(); - - int ret = -__syscall(SYS_sched_setscheduler, self->tid, - ssa->attr->_a_policy, &ssa->attr->_a_prio); - if (!ret) __restore_sigs(&ssa->mask); - a_store(&ssa->futex, ret); - __wake(&ssa->futex, 1, 1); - if (ret) { - self->detach_state = DT_DYNAMIC; - return 0; - } - return start_fn(start_arg); -} - int pthread_attr_setinheritsched(pthread_attr_t *a, int inherit) { if (inherit > 1U) return EINVAL; diff --git a/src/thread/pthread_cancel.c b/src/thread/pthread_cancel.c index 2f9d5e97..139a6fc8 100644 --- a/src/thread/pthread_cancel.c +++ b/src/thread/pthread_cancel.c @@ -56,7 +56,12 @@ static void cancel_handler(int sig, siginfo_t *si, void *ctx) _sigaddset(&uc->uc_sigmask, SIGCANCEL); - if (self->cancelasync || pc >= (uintptr_t)__cp_begin && pc < (uintptr_t)__cp_end) { + if (self->cancelasync) { + pthread_sigmask(SIG_SETMASK, &uc->uc_sigmask, 0); + __cancel(); + } + + if (pc >= (uintptr_t)__cp_begin && pc < (uintptr_t)__cp_end) { uc->uc_mcontext.MC_PC = (uintptr_t)__cp_cancel; #ifdef CANCEL_GOT uc->uc_mcontext.MC_GOT = CANCEL_GOT; @@ -77,7 +82,7 @@ void __testcancel() static void init_cancellation() { struct sigaction sa = { - .sa_flags = SA_SIGINFO | SA_RESTART, + .sa_flags = SA_SIGINFO | SA_RESTART | SA_ONSTACK, .sa_sigaction = cancel_handler }; memset(&sa.sa_mask, -1, _NSIG/8); diff --git a/src/thread/pthread_cond_timedwait.c b/src/thread/pthread_cond_timedwait.c index d1501240..6b761455 100644 --- a/src/thread/pthread_cond_timedwait.c +++ b/src/thread/pthread_cond_timedwait.c @@ -146,14 +146,18 @@ relock: if (oldstate == WAITING) goto done; - if (!node.next) a_inc(&m->_m_waiters); + if (!node.next && !(m->_m_type & 8)) + a_inc(&m->_m_waiters); /* Unlock the barrier that's holding back the next waiter, and * either wake it or requeue it to the mutex. */ - if (node.prev) - unlock_requeue(&node.prev->barrier, &m->_m_lock, m->_m_type & 128); - else - a_dec(&m->_m_waiters); + if (node.prev) { + int val = m->_m_lock; + if (val>0) a_cas(&m->_m_lock, val, val|0x80000000); + unlock_requeue(&node.prev->barrier, &m->_m_lock, m->_m_type & (8|128)); + } else if (!(m->_m_type & 8)) { + a_dec(&m->_m_waiters); + } /* Since a signal was consumed, cancellation is not permitted. */ if (e == ECANCELED) e = 0; diff --git a/src/thread/pthread_create.c b/src/thread/pthread_create.c index 3da7db14..087f6206 100644 --- a/src/thread/pthread_create.c +++ b/src/thread/pthread_create.c @@ -15,12 +15,41 @@ weak_alias(dummy_0, __release_ptc); weak_alias(dummy_0, __pthread_tsd_run_dtors); weak_alias(dummy_0, __do_orphaned_stdio_locks); weak_alias(dummy_0, __dl_thread_cleanup); +weak_alias(dummy_0, __membarrier_init); -static void *dummy_1(void *p) +static int tl_lock_count; +static int tl_lock_waiters; + +void __tl_lock(void) { - return 0; + int tid = __pthread_self()->tid; + int val = __thread_list_lock; + if (val == tid) { + tl_lock_count++; + return; + } + while ((val = a_cas(&__thread_list_lock, 0, tid))) + __wait(&__thread_list_lock, &tl_lock_waiters, val, 0); +} + +void __tl_unlock(void) +{ + if (tl_lock_count) { + tl_lock_count--; + return; + } + a_store(&__thread_list_lock, 0); + if (tl_lock_waiters) __wake(&__thread_list_lock, 1, 0); +} + +void __tl_sync(pthread_t td) +{ + a_barrier(); + int val = __thread_list_lock; + if (!val) return; + __wait(&__thread_list_lock, &tl_lock_waiters, val, 0); + if (tl_lock_waiters) __wake(&__thread_list_lock, 1, 0); } -weak_alias(dummy_1, __start_sched); _Noreturn void __pthread_exit(void *result) { @@ -40,30 +69,54 @@ _Noreturn void __pthread_exit(void *result) __pthread_tsd_run_dtors(); + __block_app_sigs(&set); + + /* This atomic potentially competes with a concurrent pthread_detach + * call; the loser is responsible for freeing thread resources. */ + int state = a_cas(&self->detach_state, DT_JOINABLE, DT_EXITING); + + if (state==DT_DETACHED && self->map_base) { + /* Since __unmapself bypasses the normal munmap code path, + * explicitly wait for vmlock holders first. This must be + * done before any locks are taken, to avoid lock ordering + * issues that could lead to deadlock. */ + __vm_wait(); + } + /* Access to target the exiting thread with syscalls that use * its kernel tid is controlled by killlock. For detached threads, * any use past this point would have undefined behavior, but for - * joinable threads it's a valid usage that must be handled. */ + * joinable threads it's a valid usage that must be handled. + * Signals must be blocked since pthread_kill must be AS-safe. */ LOCK(self->killlock); - /* Block all signals before decrementing the live thread count. - * This is important to ensure that dynamically allocated TLS - * is not under-allocated/over-committed, and possibly for other - * reasons as well. */ - __block_all_sigs(&set); - - /* It's impossible to determine whether this is "the last thread" - * until performing the atomic decrement, since multiple threads - * could exit at the same time. For the last thread, revert the - * decrement, restore the tid, and unblock signals to give the - * atexit handlers and stdio cleanup code a consistent state. */ - if (a_fetch_add(&libc.threads_minus_1, -1)==0) { - libc.threads_minus_1 = 0; + /* The thread list lock must be AS-safe, and thus depends on + * application signals being blocked above. */ + __tl_lock(); + + /* If this is the only thread in the list, don't proceed with + * termination of the thread, but restore the previous lock and + * signal state to prepare for exit to call atexit handlers. */ + if (self->next == self) { + __tl_unlock(); UNLOCK(self->killlock); + self->detach_state = state; __restore_sigs(&set); exit(0); } + /* At this point we are committed to thread termination. */ + + /* After the kernel thread exits, its tid may be reused. Clear it + * to prevent inadvertent use and inform functions that would use + * it that it's no longer available. At this point the killlock + * may be released, since functions that use it will consistently + * see the thread as having exited. Release it now so that no + * remaining locks (except thread list) are held if we end up + * resetting need_locks below. */ + self->tid = 0; + UNLOCK(self->killlock); + /* Process robust list in userspace to handle non-pshared mutexes * and the detached thread case where the robust list head will * be invalid when the kernel would process it. */ @@ -86,39 +139,35 @@ _Noreturn void __pthread_exit(void *result) __do_orphaned_stdio_locks(); __dl_thread_cleanup(); - /* This atomic potentially competes with a concurrent pthread_detach - * call; the loser is responsible for freeing thread resources. */ - int state = a_cas(&self->detach_state, DT_JOINABLE, DT_EXITING); - - if (state>=DT_DETACHED && self->map_base) { - /* Detached threads must avoid the kernel clear_child_tid - * feature, since the virtual address will have been - * unmapped and possibly already reused by a new mapping - * at the time the kernel would perform the write. In - * the case of threads that started out detached, the - * initial clone flags are correct, but if the thread was - * detached later, we need to clear it here. */ - if (state == DT_DYNAMIC) __syscall(SYS_set_tid_address, 0); + /* Last, unlink thread from the list. This change will not be visible + * until the lock is released, which only happens after SYS_exit + * has been called, via the exit futex address pointing at the lock. + * This needs to happen after any possible calls to LOCK() that might + * skip locking if process appears single-threaded. */ + if (!--libc.threads_minus_1) libc.need_locks = -1; + self->next->prev = self->prev; + self->prev->next = self->next; + self->prev = self->next = self; + + if (state==DT_DETACHED && self->map_base) { + /* Detached threads must block even implementation-internal + * signals, since they will not have a stack in their last + * moments of existence. */ + __block_all_sigs(&set); /* Robust list will no longer be valid, and was already * processed above, so unregister it with the kernel. */ if (self->robust_list.off) __syscall(SYS_set_robust_list, 0, 3*sizeof(long)); - /* Since __unmapself bypasses the normal munmap code path, - * explicitly wait for vmlock holders first. */ - __vm_wait(); - /* The following call unmaps the thread's stack mapping * and then exits without touching the stack. */ __unmapself(self->map_base, self->map_size); } - /* After the kernel thread exits, its tid may be reused. Clear it - * to prevent inadvertent use and inform functions that would use - * it that it's no longer available. */ - self->tid = 0; - UNLOCK(self->killlock); + /* Wake any joiner. */ + a_store(&self->detach_state, DT_EXITED); + __wake(&self->detach_state, 1, 1); for (;;) __syscall(SYS_exit, 0); } @@ -135,21 +184,35 @@ void __do_cleanup_pop(struct __ptcb *cb) __pthread_self()->cancelbuf = cb->__next; } +struct start_args { + void *(*start_func)(void *); + void *start_arg; + volatile int control; + unsigned long sig_mask[_NSIG/8/sizeof(long)]; +}; + static int start(void *p) { - pthread_t self = p; - if (self->unblock_cancel) - __syscall(SYS_rt_sigprocmask, SIG_UNBLOCK, - SIGPT_SET, 0, _NSIG/8); - __pthread_exit(self->start(self->start_arg)); + struct start_args *args = p; + int state = args->control; + if (state) { + if (a_cas(&args->control, 1, 2)==1) + __wait(&args->control, 0, 2, 1); + if (args->control) { + __syscall(SYS_set_tid_address, &args->control); + for (;;) __syscall(SYS_exit, 0); + } + } + __syscall(SYS_rt_sigprocmask, SIG_SETMASK, &args->sig_mask, 0, _NSIG/8); + __pthread_exit(args->start_func(args->start_arg)); return 0; } static int start_c11(void *p) { - pthread_t self = p; - int (*start)(void*) = (int(*)(void*)) self->start; - __pthread_exit((void *)(uintptr_t)start(self->start_arg)); + struct start_args *args = p; + int (*start)(void*) = (int(*)(void*)) args->start_func; + __pthread_exit((void *)(uintptr_t)start(args->start_arg)); return 0; } @@ -161,8 +224,6 @@ weak_alias(dummy, __pthread_tsd_size); static void *dummy_tsd[1] = { 0 }; weak_alias(dummy_tsd, __pthread_tsd_main); -volatile int __block_new_threads = 0; - static FILE *volatile dummy_file = 0; weak_alias(dummy_file, __stdin_used); weak_alias(dummy_file, __stdout_used); @@ -182,9 +243,8 @@ int __pthread_create(pthread_t *restrict res, const pthread_attr_t *restrict att unsigned flags = CLONE_VM | CLONE_FS | CLONE_FILES | CLONE_SIGHAND | CLONE_THREAD | CLONE_SYSVSEM | CLONE_SETTLS | CLONE_PARENT_SETTID | CLONE_CHILD_CLEARTID | CLONE_DETACHED; - int do_sched = 0; pthread_attr_t attr = { 0 }; - struct start_sched_args ssa; + sigset_t set; if (!libc.can_do_threads) return ENOSYS; self = __pthread_self(); @@ -197,6 +257,7 @@ int __pthread_create(pthread_t *restrict res, const pthread_attr_t *restrict att init_file_lock(__stderr_used); __syscall(SYS_rt_sigprocmask, SIG_UNBLOCK, SIGPT_SET, 0, _NSIG/8); self->tsd = (void **)__pthread_tsd_main; + __membarrier_init(); libc.threaded = 1; } if (attrp && !c11) attr = *attrp; @@ -207,8 +268,6 @@ int __pthread_create(pthread_t *restrict res, const pthread_attr_t *restrict att attr._a_guardsize = __default_guardsize; } - if (__block_new_threads) __wait(&__block_new_threads, 0, 1, 1); - if (attr._a_stackaddr) { size_t need = libc.tls_size + __pthread_tsd_size; size = attr._a_stacksize; @@ -257,50 +316,74 @@ int __pthread_create(pthread_t *restrict res, const pthread_attr_t *restrict att new->stack = stack; new->stack_size = stack - stack_limit; new->guard_size = guard; - new->start = entry; - new->start_arg = arg; new->self = new; new->tsd = (void *)tsd; new->locale = &libc.global_locale; if (attr._a_detach) { new->detach_state = DT_DETACHED; - flags -= CLONE_CHILD_CLEARTID; } else { new->detach_state = DT_JOINABLE; } - if (attr._a_sched) { - do_sched = 1; - ssa.futex = -1; - ssa.start_fn = new->start; - ssa.start_arg = new->start_arg; - ssa.attr = &attr; - new->start = __start_sched; - new->start_arg = &ssa; - __block_app_sigs(&ssa.mask); - } new->robust_list.head = &new->robust_list.head; - new->unblock_cancel = self->cancel; - new->CANARY = self->CANARY; - - a_inc(&libc.threads_minus_1); - ret = __clone((c11 ? start_c11 : start), stack, flags, new, &new->tid, TP_ADJ(new), &new->detach_state); - - __release_ptc(); + new->canary = self->canary; + new->sysinfo = self->sysinfo; + + /* Setup argument structure for the new thread on its stack. + * It's safe to access from the caller only until the thread + * list is unlocked. */ + stack -= (uintptr_t)stack % sizeof(uintptr_t); + stack -= sizeof(struct start_args); + struct start_args *args = (void *)stack; + args->start_func = entry; + args->start_arg = arg; + args->control = attr._a_sched ? 1 : 0; + + /* Application signals (but not the synccall signal) must be + * blocked before the thread list lock can be taken, to ensure + * that the lock is AS-safe. */ + __block_app_sigs(&set); + + /* Ensure SIGCANCEL is unblocked in new thread. This requires + * working with a copy of the set so we can restore the + * original mask in the calling thread. */ + memcpy(&args->sig_mask, &set, sizeof args->sig_mask); + args->sig_mask[(SIGCANCEL-1)/8/sizeof(long)] &= + ~(1UL<<((SIGCANCEL-1)%(8*sizeof(long)))); + + __tl_lock(); + if (!libc.threads_minus_1++) libc.need_locks = 1; + ret = __clone((c11 ? start_c11 : start), stack, flags, args, &new->tid, TP_ADJ(new), &__thread_list_lock); + + /* All clone failures translate to EAGAIN. If explicit scheduling + * was requested, attempt it before unlocking the thread list so + * that the failed thread is never exposed and so that we can + * clean up all transient resource usage before returning. */ + if (ret < 0) { + ret = -EAGAIN; + } else if (attr._a_sched) { + ret = __syscall(SYS_sched_setscheduler, + new->tid, attr._a_policy, &attr._a_prio); + if (a_swap(&args->control, ret ? 3 : 0)==2) + __wake(&args->control, 1, 1); + if (ret) + __wait(&args->control, 0, 3, 0); + } - if (do_sched) { - __restore_sigs(&ssa.mask); + if (ret >= 0) { + new->next = self->next; + new->prev = self; + new->next->prev = new; + new->prev->next = new; + } else { + if (!--libc.threads_minus_1) libc.need_locks = 0; } + __tl_unlock(); + __restore_sigs(&set); + __release_ptc(); if (ret < 0) { - a_dec(&libc.threads_minus_1); if (map) __munmap(map, size); - return EAGAIN; - } - - if (do_sched) { - __futexwait(&ssa.futex, -1, 1); - ret = ssa.futex; - if (ret) return ret; + return -ret; } *res = new; diff --git a/src/thread/pthread_detach.c b/src/thread/pthread_detach.c index 16b0552d..d73a500e 100644 --- a/src/thread/pthread_detach.c +++ b/src/thread/pthread_detach.c @@ -5,8 +5,12 @@ static int __pthread_detach(pthread_t t) { /* If the cas fails, detach state is either already-detached * or exiting/exited, and pthread_join will trap or cleanup. */ - if (a_cas(&t->detach_state, DT_JOINABLE, DT_DYNAMIC) != DT_JOINABLE) - return __pthread_join(t, 0); + if (a_cas(&t->detach_state, DT_JOINABLE, DT_DETACHED) != DT_JOINABLE) { + int cs; + __pthread_setcancelstate(PTHREAD_CANCEL_DISABLE, &cs); + __pthread_join(t, 0); + __pthread_setcancelstate(cs, 0); + } return 0; } diff --git a/src/thread/pthread_getname_np.c b/src/thread/pthread_getname_np.c new file mode 100644 index 00000000..85504e45 --- /dev/null +++ b/src/thread/pthread_getname_np.c @@ -0,0 +1,25 @@ +#define _GNU_SOURCE +#include <fcntl.h> +#include <unistd.h> +#include <sys/prctl.h> + +#include "pthread_impl.h" + +int pthread_getname_np(pthread_t thread, char *name, size_t len) +{ + int fd, cs, status = 0; + char f[sizeof "/proc/self/task//comm" + 3*sizeof(int)]; + + if (len < 16) return ERANGE; + + if (thread == pthread_self()) + return prctl(PR_GET_NAME, (unsigned long)name, 0UL, 0UL, 0UL) ? errno : 0; + + snprintf(f, sizeof f, "/proc/self/task/%d/comm", thread->tid); + pthread_setcancelstate(PTHREAD_CANCEL_DISABLE, &cs); + if ((fd = open(f, O_RDONLY|O_CLOEXEC)) < 0 || (len = read(fd, name, len)) == -1) status = errno; + else name[len-1] = 0; /* remove trailing new line only if successful */ + if (fd >= 0) close(fd); + pthread_setcancelstate(cs, 0); + return status; +} diff --git a/src/thread/pthread_getschedparam.c b/src/thread/pthread_getschedparam.c index 1cba073d..c098befb 100644 --- a/src/thread/pthread_getschedparam.c +++ b/src/thread/pthread_getschedparam.c @@ -4,6 +4,8 @@ int pthread_getschedparam(pthread_t t, int *restrict policy, struct sched_param *restrict param) { int r; + sigset_t set; + __block_app_sigs(&set); LOCK(t->killlock); if (!t->tid) { r = ESRCH; @@ -14,5 +16,6 @@ int pthread_getschedparam(pthread_t t, int *restrict policy, struct sched_param } } UNLOCK(t->killlock); + __restore_sigs(&set); return r; } diff --git a/src/thread/pthread_join.c b/src/thread/pthread_join.c index 54d81039..17dae85d 100644 --- a/src/thread/pthread_join.c +++ b/src/thread/pthread_join.c @@ -1,6 +1,12 @@ +#define _GNU_SOURCE #include "pthread_impl.h" #include <sys/mman.h> +static void dummy1(pthread_t t) +{ +} +weak_alias(dummy1, __tl_sync); + static int __pthread_timedjoin_np(pthread_t t, void **res, const struct timespec *at) { int state, cs, r = 0; @@ -9,11 +15,11 @@ static int __pthread_timedjoin_np(pthread_t t, void **res, const struct timespec if (cs == PTHREAD_CANCEL_ENABLE) __pthread_setcancelstate(cs, 0); while ((state = t->detach_state) && r != ETIMEDOUT && r != EINVAL) { if (state >= DT_DETACHED) a_crash(); - r = __timedwait_cp(&t->detach_state, state, CLOCK_REALTIME, at, 0); + r = __timedwait_cp(&t->detach_state, state, CLOCK_REALTIME, at, 1); } __pthread_setcancelstate(cs, 0); if (r == ETIMEDOUT || r == EINVAL) return r; - a_barrier(); + __tl_sync(t); if (res) *res = t->result; if (t->map_base) __munmap(t->map_base, t->map_size); return 0; diff --git a/src/thread/pthread_key_create.c b/src/thread/pthread_key_create.c index e26f199c..39770c7a 100644 --- a/src/thread/pthread_key_create.c +++ b/src/thread/pthread_key_create.c @@ -1,4 +1,5 @@ #include "pthread_impl.h" +#include "fork_impl.h" volatile size_t __pthread_tsd_size = sizeof(void *) * PTHREAD_KEYS_MAX; void *__pthread_tsd_main[PTHREAD_KEYS_MAX] = { 0 }; @@ -13,43 +14,23 @@ static void nodtor(void *dummy) { } -static void dirty(void *dummy) +static void dummy_0(void) { } -struct cleanup_args { - pthread_t caller; - int ret; -}; +weak_alias(dummy_0, __tl_lock); +weak_alias(dummy_0, __tl_unlock); -static void clean_dirty_tsd_callback(void *p) +void __pthread_key_atfork(int who) { - struct cleanup_args *args = p; - pthread_t self = __pthread_self(); - pthread_key_t i; - for (i=0; i<PTHREAD_KEYS_MAX; i++) { - if (keys[i] == dirty && self->tsd[i]) - self->tsd[i] = 0; - } - /* Arbitrary choice to avoid data race. */ - if (args->caller == self) args->ret = 0; -} - -static int clean_dirty_tsd(void) -{ - struct cleanup_args args = { - .caller = __pthread_self(), - .ret = EAGAIN - }; - __pthread_key_delete_synccall(clean_dirty_tsd_callback, &args); - return args.ret; + if (who<0) __pthread_rwlock_rdlock(&key_lock); + else if (!who) __pthread_rwlock_unlock(&key_lock); + else key_lock = (pthread_rwlock_t)PTHREAD_RWLOCK_INITIALIZER; } int __pthread_key_create(pthread_key_t *k, void (*dtor)(void *)) { - pthread_key_t j = next_key; pthread_t self = __pthread_self(); - int found_dirty = 0; /* This can only happen in the main thread before * pthread_create has been called. */ @@ -58,46 +39,38 @@ int __pthread_key_create(pthread_key_t *k, void (*dtor)(void *)) /* Purely a sentinel value since null means slot is free. */ if (!dtor) dtor = nodtor; - pthread_rwlock_wrlock(&key_lock); + __pthread_rwlock_wrlock(&key_lock); + pthread_key_t j = next_key; do { if (!keys[j]) { keys[next_key = *k = j] = dtor; - pthread_rwlock_unlock(&key_lock); + __pthread_rwlock_unlock(&key_lock); return 0; - } else if (keys[j] == dirty) { - found_dirty = 1; } } while ((j=(j+1)%PTHREAD_KEYS_MAX) != next_key); - /* It's possible that all slots are in use or __synccall fails. */ - if (!found_dirty || clean_dirty_tsd()) { - pthread_rwlock_unlock(&key_lock); - return EAGAIN; - } - - /* If this point is reached there is necessarily a newly-cleaned - * slot to allocate to satisfy the caller's request. Find it and - * mark any additional previously-dirty slots clean. */ - for (j=0; j<PTHREAD_KEYS_MAX; j++) { - if (keys[j] == dirty) { - if (dtor) { - keys[next_key = *k = j] = dtor; - dtor = 0; - } else { - keys[j] = 0; - } - } - } - - pthread_rwlock_unlock(&key_lock); - return 0; + __pthread_rwlock_unlock(&key_lock); + return EAGAIN; } -int __pthread_key_delete_impl(pthread_key_t k) +int __pthread_key_delete(pthread_key_t k) { - pthread_rwlock_wrlock(&key_lock); - keys[k] = dirty; - pthread_rwlock_unlock(&key_lock); + sigset_t set; + pthread_t self = __pthread_self(), td=self; + + __block_app_sigs(&set); + __pthread_rwlock_wrlock(&key_lock); + + __tl_lock(); + do td->tsd[k] = 0; + while ((td=td->next)!=self); + __tl_unlock(); + + keys[k] = 0; + + __pthread_rwlock_unlock(&key_lock); + __restore_sigs(&set); + return 0; } @@ -106,20 +79,21 @@ void __pthread_tsd_run_dtors() pthread_t self = __pthread_self(); int i, j; for (j=0; self->tsd_used && j<PTHREAD_DESTRUCTOR_ITERATIONS; j++) { - pthread_rwlock_rdlock(&key_lock); + __pthread_rwlock_rdlock(&key_lock); self->tsd_used = 0; for (i=0; i<PTHREAD_KEYS_MAX; i++) { void *val = self->tsd[i]; void (*dtor)(void *) = keys[i]; self->tsd[i] = 0; - if (val && dtor && dtor != nodtor && dtor != dirty) { - pthread_rwlock_unlock(&key_lock); + if (val && dtor && dtor != nodtor) { + __pthread_rwlock_unlock(&key_lock); dtor(val); - pthread_rwlock_rdlock(&key_lock); + __pthread_rwlock_rdlock(&key_lock); } } - pthread_rwlock_unlock(&key_lock); + __pthread_rwlock_unlock(&key_lock); } } weak_alias(__pthread_key_create, pthread_key_create); +weak_alias(__pthread_key_delete, pthread_key_delete); diff --git a/src/thread/pthread_key_delete.c b/src/thread/pthread_key_delete.c deleted file mode 100644 index 012fe2da..00000000 --- a/src/thread/pthread_key_delete.c +++ /dev/null @@ -1,14 +0,0 @@ -#include "pthread_impl.h" -#include "libc.h" - -void __pthread_key_delete_synccall(void (*f)(void *), void *p) -{ - __synccall(f, p); -} - -int __pthread_key_delete(pthread_key_t k) -{ - return __pthread_key_delete_impl(k); -} - -weak_alias(__pthread_key_delete, pthread_key_delete); diff --git a/src/thread/pthread_kill.c b/src/thread/pthread_kill.c index 3d9395cb..79ddb209 100644 --- a/src/thread/pthread_kill.c +++ b/src/thread/pthread_kill.c @@ -4,9 +4,15 @@ int pthread_kill(pthread_t t, int sig) { int r; + sigset_t set; + /* Block not just app signals, but internal ones too, since + * pthread_kill is used to implement pthread_cancel, which + * must be async-cancel-safe. */ + __block_all_sigs(&set); LOCK(t->killlock); r = t->tid ? -__syscall(SYS_tkill, t->tid, sig) : (sig+0U >= _NSIG ? EINVAL : 0); UNLOCK(t->killlock); + __restore_sigs(&set); return r; } diff --git a/src/thread/pthread_mutex_consistent.c b/src/thread/pthread_mutex_consistent.c index 96b83b52..27c74e5b 100644 --- a/src/thread/pthread_mutex_consistent.c +++ b/src/thread/pthread_mutex_consistent.c @@ -1,10 +1,14 @@ #include "pthread_impl.h" +#include "atomic.h" int pthread_mutex_consistent(pthread_mutex_t *m) { - if (!(m->_m_type & 8)) return EINVAL; - if ((m->_m_lock & 0x7fffffff) != __pthread_self()->tid) + int old = m->_m_lock; + int own = old & 0x3fffffff; + if (!(m->_m_type & 4) || !own || !(old & 0x40000000)) + return EINVAL; + if (own != __pthread_self()->tid) return EPERM; - m->_m_type &= ~8U; + a_and(&m->_m_lock, ~0x40000000); return 0; } diff --git a/src/thread/pthread_mutex_destroy.c b/src/thread/pthread_mutex_destroy.c index 6d49e689..8d1bf77b 100644 --- a/src/thread/pthread_mutex_destroy.c +++ b/src/thread/pthread_mutex_destroy.c @@ -1,6 +1,10 @@ -#include <pthread.h> +#include "pthread_impl.h" int pthread_mutex_destroy(pthread_mutex_t *mutex) { + /* If the mutex being destroyed is process-shared and has nontrivial + * type (tracking ownership), it might be in the pending slot of a + * robust_list; wait for quiescence. */ + if (mutex->_m_type > 128) __vm_wait(); return 0; } diff --git a/src/thread/pthread_mutex_timedlock.c b/src/thread/pthread_mutex_timedlock.c index 9867f389..9279fc54 100644 --- a/src/thread/pthread_mutex_timedlock.c +++ b/src/thread/pthread_mutex_timedlock.c @@ -1,5 +1,58 @@ #include "pthread_impl.h" +#define IS32BIT(x) !((x)+0x80000000ULL>>32) +#define CLAMP(x) (int)(IS32BIT(x) ? (x) : 0x7fffffffU+((0ULL+(x))>>63)) + +static int __futex4(volatile void *addr, int op, int val, const struct timespec *to) +{ +#ifdef SYS_futex_time64 + time_t s = to ? to->tv_sec : 0; + long ns = to ? to->tv_nsec : 0; + int r = -ENOSYS; + if (SYS_futex == SYS_futex_time64 || !IS32BIT(s)) + r = __syscall(SYS_futex_time64, addr, op, val, + to ? ((long long[]){s, ns}) : 0); + if (SYS_futex == SYS_futex_time64 || r!=-ENOSYS) return r; + to = to ? (void *)(long[]){CLAMP(s), ns} : 0; +#endif + return __syscall(SYS_futex, addr, op, val, to); +} + +static int pthread_mutex_timedlock_pi(pthread_mutex_t *restrict m, const struct timespec *restrict at) +{ + int type = m->_m_type; + int priv = (type & 128) ^ 128; + pthread_t self = __pthread_self(); + int e; + + if (!priv) self->robust_list.pending = &m->_m_next; + + do e = -__futex4(&m->_m_lock, FUTEX_LOCK_PI|priv, 0, at); + while (e==EINTR); + if (e) self->robust_list.pending = 0; + + switch (e) { + case 0: + /* Catch spurious success for non-robust mutexes. */ + if (!(type&4) && ((m->_m_lock & 0x40000000) || m->_m_waiters)) { + a_store(&m->_m_waiters, -1); + __syscall(SYS_futex, &m->_m_lock, FUTEX_UNLOCK_PI|priv); + self->robust_list.pending = 0; + break; + } + /* Signal to trylock that we already have the lock. */ + m->_m_count = -1; + return __pthread_mutex_trylock(m); + case ETIMEDOUT: + return e; + case EDEADLK: + if ((type&3) == PTHREAD_MUTEX_ERRORCHECK) return e; + } + do e = __timedwait(&(int){0}, 0, CLOCK_REALTIME, at, 1); + while (e != ETIMEDOUT); + return e; +} + int __pthread_mutex_timedlock(pthread_mutex_t *restrict m, const struct timespec *restrict at) { if ((m->_m_type&15) == PTHREAD_MUTEX_NORMAL @@ -9,17 +62,21 @@ int __pthread_mutex_timedlock(pthread_mutex_t *restrict m, const struct timespec int type = m->_m_type; int r, t, priv = (type & 128) ^ 128; - r = pthread_mutex_trylock(m); + r = __pthread_mutex_trylock(m); if (r != EBUSY) return r; + + if (type&8) return pthread_mutex_timedlock_pi(m, at); int spins = 100; while (spins-- && m->_m_lock && !m->_m_waiters) a_spin(); while ((r=__pthread_mutex_trylock(m)) == EBUSY) { - if (!(r=m->_m_lock) || ((r&0x40000000) && (type&4))) + r = m->_m_lock; + int own = r & 0x3fffffff; + if (!own && (!r || (type&4))) continue; if ((type&3) == PTHREAD_MUTEX_ERRORCHECK - && (r&0x7fffffff) == __pthread_self()->tid) + && own == __pthread_self()->tid) return EDEADLK; a_inc(&m->_m_waiters); diff --git a/src/thread/pthread_mutex_trylock.c b/src/thread/pthread_mutex_trylock.c index 783ca0c4..a24e7c58 100644 --- a/src/thread/pthread_mutex_trylock.c +++ b/src/thread/pthread_mutex_trylock.c @@ -3,21 +3,28 @@ int __pthread_mutex_trylock_owner(pthread_mutex_t *m) { int old, own; - int type = m->_m_type & 15; + int type = m->_m_type; pthread_t self = __pthread_self(); int tid = self->tid; old = m->_m_lock; - own = old & 0x7fffffff; - if (own == tid && (type&3) == PTHREAD_MUTEX_RECURSIVE) { - if ((unsigned)m->_m_count >= INT_MAX) return EAGAIN; - m->_m_count++; - return 0; + own = old & 0x3fffffff; + if (own == tid) { + if ((type&8) && m->_m_count<0) { + old &= 0x40000000; + m->_m_count = 0; + goto success; + } + if ((type&3) == PTHREAD_MUTEX_RECURSIVE) { + if ((unsigned)m->_m_count >= INT_MAX) return EAGAIN; + m->_m_count++; + return 0; + } } - if (own == 0x7fffffff) return ENOTRECOVERABLE; - if (own && (!(own & 0x40000000) || !(type & 4))) return EBUSY; + if (own == 0x3fffffff) return ENOTRECOVERABLE; + if (own || (old && !(type & 4))) return EBUSY; - if (m->_m_type & 128) { + if (type & 128) { if (!self->robust_list.off) { self->robust_list.off = (char*)&m->_m_lock-(char *)&m->_m_next; __syscall(SYS_set_robust_list, &self->robust_list, 3*sizeof(long)); @@ -25,12 +32,22 @@ int __pthread_mutex_trylock_owner(pthread_mutex_t *m) if (m->_m_waiters) tid |= 0x80000000; self->robust_list.pending = &m->_m_next; } + tid |= old & 0x40000000; if (a_cas(&m->_m_lock, old, tid) != old) { self->robust_list.pending = 0; + if ((type&12)==12 && m->_m_waiters) return ENOTRECOVERABLE; return EBUSY; } +success: + if ((type&8) && m->_m_waiters) { + int priv = (type & 128) ^ 128; + __syscall(SYS_futex, &m->_m_lock, FUTEX_UNLOCK_PI|priv); + self->robust_list.pending = 0; + return (type&4) ? ENOTRECOVERABLE : EBUSY; + } + volatile void *next = self->robust_list.head; m->_m_next = next; m->_m_prev = &self->robust_list.head; @@ -39,9 +56,8 @@ int __pthread_mutex_trylock_owner(pthread_mutex_t *m) self->robust_list.head = &m->_m_next; self->robust_list.pending = 0; - if (own) { + if (old) { m->_m_count = 0; - m->_m_type |= 8; return EOWNERDEAD; } diff --git a/src/thread/pthread_mutex_unlock.c b/src/thread/pthread_mutex_unlock.c index 7dd00d27..b66423e6 100644 --- a/src/thread/pthread_mutex_unlock.c +++ b/src/thread/pthread_mutex_unlock.c @@ -7,13 +7,19 @@ int __pthread_mutex_unlock(pthread_mutex_t *m) int cont; int type = m->_m_type & 15; int priv = (m->_m_type & 128) ^ 128; + int new = 0; + int old; if (type != PTHREAD_MUTEX_NORMAL) { self = __pthread_self(); - if ((m->_m_lock&0x7fffffff) != self->tid) + old = m->_m_lock; + int own = old & 0x3fffffff; + if (own != self->tid) return EPERM; if ((type&3) == PTHREAD_MUTEX_RECURSIVE && m->_m_count) return m->_m_count--, 0; + if ((type&4) && (old&0x40000000)) + new = 0x7fffffff; if (!priv) { self->robust_list.pending = &m->_m_next; __vm_lock(); @@ -24,7 +30,16 @@ int __pthread_mutex_unlock(pthread_mutex_t *m) if (next != &self->robust_list.head) *(volatile void *volatile *) ((char *)next - sizeof(void *)) = prev; } - cont = a_swap(&m->_m_lock, (type & 8) ? 0x7fffffff : 0); + if (type&8) { + if (old<0 || a_cas(&m->_m_lock, old, new)!=old) { + if (new) a_store(&m->_m_waiters, -1); + __syscall(SYS_futex, &m->_m_lock, FUTEX_UNLOCK_PI|priv); + } + cont = 0; + waiters = 0; + } else { + cont = a_swap(&m->_m_lock, new); + } if (type != PTHREAD_MUTEX_NORMAL && !priv) { self->robust_list.pending = 0; __vm_unlock(); diff --git a/src/thread/pthread_mutexattr_setprotocol.c b/src/thread/pthread_mutexattr_setprotocol.c index c92a31c8..8b80c1ce 100644 --- a/src/thread/pthread_mutexattr_setprotocol.c +++ b/src/thread/pthread_mutexattr_setprotocol.c @@ -1,7 +1,28 @@ #include "pthread_impl.h" +#include "syscall.h" + +static volatile int check_pi_result = -1; int pthread_mutexattr_setprotocol(pthread_mutexattr_t *a, int protocol) { - if (protocol) return ENOTSUP; - return 0; + int r; + switch (protocol) { + case PTHREAD_PRIO_NONE: + a->__attr &= ~8; + return 0; + case PTHREAD_PRIO_INHERIT: + r = check_pi_result; + if (r < 0) { + volatile int lk = 0; + r = -__syscall(SYS_futex, &lk, FUTEX_LOCK_PI, 0, 0); + a_store(&check_pi_result, r); + } + if (r) return r; + a->__attr |= 8; + return 0; + case PTHREAD_PRIO_PROTECT: + return ENOTSUP; + default: + return EINVAL; + } } diff --git a/src/thread/pthread_mutexattr_setrobust.c b/src/thread/pthread_mutexattr_setrobust.c index 04db92a6..30a9ac3b 100644 --- a/src/thread/pthread_mutexattr_setrobust.c +++ b/src/thread/pthread_mutexattr_setrobust.c @@ -1,22 +1,20 @@ #include "pthread_impl.h" #include "syscall.h" -static pthread_once_t check_robust_once; -static int check_robust_result; - -static void check_robust() -{ - void *p; - size_t l; - check_robust_result = -__syscall(SYS_get_robust_list, 0, &p, &l); -} +static volatile int check_robust_result = -1; int pthread_mutexattr_setrobust(pthread_mutexattr_t *a, int robust) { if (robust > 1U) return EINVAL; if (robust) { - pthread_once(&check_robust_once, check_robust); - if (check_robust_result) return check_robust_result; + int r = check_robust_result; + if (r < 0) { + void *p; + size_t l; + r = -__syscall(SYS_get_robust_list, 0, &p, &l); + a_store(&check_robust_result, r); + } + if (r) return r; a->__attr |= 4; return 0; } diff --git a/src/thread/pthread_rwlock_rdlock.c b/src/thread/pthread_rwlock_rdlock.c index 0800d21f..8546c07d 100644 --- a/src/thread/pthread_rwlock_rdlock.c +++ b/src/thread/pthread_rwlock_rdlock.c @@ -1,6 +1,8 @@ #include "pthread_impl.h" -int pthread_rwlock_rdlock(pthread_rwlock_t *rw) +int __pthread_rwlock_rdlock(pthread_rwlock_t *rw) { - return pthread_rwlock_timedrdlock(rw, 0); + return __pthread_rwlock_timedrdlock(rw, 0); } + +weak_alias(__pthread_rwlock_rdlock, pthread_rwlock_rdlock); diff --git a/src/thread/pthread_rwlock_timedrdlock.c b/src/thread/pthread_rwlock_timedrdlock.c index 0d5d0d6c..8cdd8ecf 100644 --- a/src/thread/pthread_rwlock_timedrdlock.c +++ b/src/thread/pthread_rwlock_timedrdlock.c @@ -1,6 +1,6 @@ #include "pthread_impl.h" -int pthread_rwlock_timedrdlock(pthread_rwlock_t *restrict rw, const struct timespec *restrict at) +int __pthread_rwlock_timedrdlock(pthread_rwlock_t *restrict rw, const struct timespec *restrict at) { int r, t; @@ -10,7 +10,7 @@ int pthread_rwlock_timedrdlock(pthread_rwlock_t *restrict rw, const struct times int spins = 100; while (spins-- && rw->_rw_lock && !rw->_rw_waiters) a_spin(); - while ((r=pthread_rwlock_tryrdlock(rw))==EBUSY) { + while ((r=__pthread_rwlock_tryrdlock(rw))==EBUSY) { if (!(r=rw->_rw_lock) || (r&0x7fffffff)!=0x7fffffff) continue; t = r | 0x80000000; a_inc(&rw->_rw_waiters); @@ -21,3 +21,5 @@ int pthread_rwlock_timedrdlock(pthread_rwlock_t *restrict rw, const struct times } return r; } + +weak_alias(__pthread_rwlock_timedrdlock, pthread_rwlock_timedrdlock); diff --git a/src/thread/pthread_rwlock_timedwrlock.c b/src/thread/pthread_rwlock_timedwrlock.c index 7f26dad1..d77706e6 100644 --- a/src/thread/pthread_rwlock_timedwrlock.c +++ b/src/thread/pthread_rwlock_timedwrlock.c @@ -1,6 +1,6 @@ #include "pthread_impl.h" -int pthread_rwlock_timedwrlock(pthread_rwlock_t *restrict rw, const struct timespec *restrict at) +int __pthread_rwlock_timedwrlock(pthread_rwlock_t *restrict rw, const struct timespec *restrict at) { int r, t; @@ -10,7 +10,7 @@ int pthread_rwlock_timedwrlock(pthread_rwlock_t *restrict rw, const struct times int spins = 100; while (spins-- && rw->_rw_lock && !rw->_rw_waiters) a_spin(); - while ((r=pthread_rwlock_trywrlock(rw))==EBUSY) { + while ((r=__pthread_rwlock_trywrlock(rw))==EBUSY) { if (!(r=rw->_rw_lock)) continue; t = r | 0x80000000; a_inc(&rw->_rw_waiters); @@ -21,3 +21,5 @@ int pthread_rwlock_timedwrlock(pthread_rwlock_t *restrict rw, const struct times } return r; } + +weak_alias(__pthread_rwlock_timedwrlock, pthread_rwlock_timedwrlock); diff --git a/src/thread/pthread_rwlock_tryrdlock.c b/src/thread/pthread_rwlock_tryrdlock.c index fa271fcc..c13bc9cc 100644 --- a/src/thread/pthread_rwlock_tryrdlock.c +++ b/src/thread/pthread_rwlock_tryrdlock.c @@ -1,6 +1,6 @@ #include "pthread_impl.h" -int pthread_rwlock_tryrdlock(pthread_rwlock_t *rw) +int __pthread_rwlock_tryrdlock(pthread_rwlock_t *rw) { int val, cnt; do { @@ -11,3 +11,5 @@ int pthread_rwlock_tryrdlock(pthread_rwlock_t *rw) } while (a_cas(&rw->_rw_lock, val, val+1) != val); return 0; } + +weak_alias(__pthread_rwlock_tryrdlock, pthread_rwlock_tryrdlock); diff --git a/src/thread/pthread_rwlock_trywrlock.c b/src/thread/pthread_rwlock_trywrlock.c index bb3d3a99..64d9d312 100644 --- a/src/thread/pthread_rwlock_trywrlock.c +++ b/src/thread/pthread_rwlock_trywrlock.c @@ -1,7 +1,9 @@ #include "pthread_impl.h" -int pthread_rwlock_trywrlock(pthread_rwlock_t *rw) +int __pthread_rwlock_trywrlock(pthread_rwlock_t *rw) { if (a_cas(&rw->_rw_lock, 0, 0x7fffffff)) return EBUSY; return 0; } + +weak_alias(__pthread_rwlock_trywrlock, pthread_rwlock_trywrlock); diff --git a/src/thread/pthread_rwlock_unlock.c b/src/thread/pthread_rwlock_unlock.c index 7b5eec84..9ae27ad2 100644 --- a/src/thread/pthread_rwlock_unlock.c +++ b/src/thread/pthread_rwlock_unlock.c @@ -1,6 +1,6 @@ #include "pthread_impl.h" -int pthread_rwlock_unlock(pthread_rwlock_t *rw) +int __pthread_rwlock_unlock(pthread_rwlock_t *rw) { int val, cnt, waiters, new, priv = rw->_rw_shared^128; @@ -16,3 +16,5 @@ int pthread_rwlock_unlock(pthread_rwlock_t *rw) return 0; } + +weak_alias(__pthread_rwlock_unlock, pthread_rwlock_unlock); diff --git a/src/thread/pthread_rwlock_wrlock.c b/src/thread/pthread_rwlock_wrlock.c index 7f33535c..46a3b3a5 100644 --- a/src/thread/pthread_rwlock_wrlock.c +++ b/src/thread/pthread_rwlock_wrlock.c @@ -1,6 +1,8 @@ #include "pthread_impl.h" -int pthread_rwlock_wrlock(pthread_rwlock_t *rw) +int __pthread_rwlock_wrlock(pthread_rwlock_t *rw) { - return pthread_rwlock_timedwrlock(rw, 0); + return __pthread_rwlock_timedwrlock(rw, 0); } + +weak_alias(__pthread_rwlock_wrlock, pthread_rwlock_wrlock); diff --git a/src/thread/pthread_setname_np.c b/src/thread/pthread_setname_np.c index 82d35e17..fc2d2306 100644 --- a/src/thread/pthread_setname_np.c +++ b/src/thread/pthread_setname_np.c @@ -19,7 +19,7 @@ int pthread_setname_np(pthread_t thread, const char *name) snprintf(f, sizeof f, "/proc/self/task/%d/comm", thread->tid); pthread_setcancelstate(PTHREAD_CANCEL_DISABLE, &cs); - if ((fd = open(f, O_WRONLY)) < 0 || write(fd, name, len) < 0) status = errno; + if ((fd = open(f, O_WRONLY|O_CLOEXEC)) < 0 || write(fd, name, len) < 0) status = errno; if (fd >= 0) close(fd); pthread_setcancelstate(cs, 0); return status; diff --git a/src/thread/pthread_setschedparam.c b/src/thread/pthread_setschedparam.c index 038d13d8..76d4d45a 100644 --- a/src/thread/pthread_setschedparam.c +++ b/src/thread/pthread_setschedparam.c @@ -4,8 +4,11 @@ int pthread_setschedparam(pthread_t t, int policy, const struct sched_param *param) { int r; + sigset_t set; + __block_app_sigs(&set); LOCK(t->killlock); r = !t->tid ? ESRCH : -__syscall(SYS_sched_setscheduler, t->tid, policy, param); UNLOCK(t->killlock); + __restore_sigs(&set); return r; } diff --git a/src/thread/pthread_setschedprio.c b/src/thread/pthread_setschedprio.c index 5bf4a019..fc2e13dd 100644 --- a/src/thread/pthread_setschedprio.c +++ b/src/thread/pthread_setschedprio.c @@ -4,8 +4,11 @@ int pthread_setschedprio(pthread_t t, int prio) { int r; + sigset_t set; + __block_app_sigs(&set); LOCK(t->killlock); r = !t->tid ? ESRCH : -__syscall(SYS_sched_setparam, t->tid, &prio); UNLOCK(t->killlock); + __restore_sigs(&set); return r; } diff --git a/src/thread/pthread_sigmask.c b/src/thread/pthread_sigmask.c index 88c333f6..f188782a 100644 --- a/src/thread/pthread_sigmask.c +++ b/src/thread/pthread_sigmask.c @@ -5,7 +5,7 @@ int pthread_sigmask(int how, const sigset_t *restrict set, sigset_t *restrict old) { int ret; - if ((unsigned)how - SIG_BLOCK > 2U) return EINVAL; + if (set && (unsigned)how - SIG_BLOCK > 2U) return EINVAL; ret = -__syscall(SYS_rt_sigprocmask, how, set, old, _NSIG/8); if (!ret && old) { if (sizeof old->__bits[0] == 8) { diff --git a/src/thread/riscv32/__set_thread_area.s b/src/thread/riscv32/__set_thread_area.s new file mode 100644 index 00000000..828154d2 --- /dev/null +++ b/src/thread/riscv32/__set_thread_area.s @@ -0,0 +1,6 @@ +.global __set_thread_area +.type __set_thread_area, %function +__set_thread_area: + mv tp, a0 + li a0, 0 + ret diff --git a/src/thread/riscv32/__unmapself.s b/src/thread/riscv32/__unmapself.s new file mode 100644 index 00000000..2849119c --- /dev/null +++ b/src/thread/riscv32/__unmapself.s @@ -0,0 +1,7 @@ +.global __unmapself +.type __unmapself, %function +__unmapself: + li a7, 215 # SYS_munmap + ecall + li a7, 93 # SYS_exit + ecall diff --git a/src/thread/riscv32/clone.s b/src/thread/riscv32/clone.s new file mode 100644 index 00000000..3102239d --- /dev/null +++ b/src/thread/riscv32/clone.s @@ -0,0 +1,34 @@ +# __clone(func, stack, flags, arg, ptid, tls, ctid) +# a0, a1, a2, a3, a4, a5, a6 + +# syscall(SYS_clone, flags, stack, ptid, tls, ctid) +# a7 a0, a1, a2, a3, a4 + +.global __clone +.type __clone, %function +__clone: + # Save func and arg to stack + addi a1, a1, -16 + sw a0, 0(a1) + sw a3, 4(a1) + + # Call SYS_clone + mv a0, a2 + mv a2, a4 + mv a3, a5 + mv a4, a6 + li a7, 220 # SYS_clone + ecall + + beqz a0, 1f + # Parent + ret + + # Child +1: lw a1, 0(sp) + lw a0, 4(sp) + jalr a1 + + # Exit + li a7, 93 # SYS_exit + ecall diff --git a/src/thread/riscv32/syscall_cp.s b/src/thread/riscv32/syscall_cp.s new file mode 100644 index 00000000..079d1ba0 --- /dev/null +++ b/src/thread/riscv32/syscall_cp.s @@ -0,0 +1,29 @@ +.global __cp_begin +.hidden __cp_begin +.global __cp_end +.hidden __cp_end +.global __cp_cancel +.hidden __cp_cancel +.hidden __cancel +.global __syscall_cp_asm +.hidden __syscall_cp_asm +.type __syscall_cp_asm, %function +__syscall_cp_asm: +__cp_begin: + lw t0, 0(a0) + bnez t0, __cp_cancel + + mv t0, a1 + mv a0, a2 + mv a1, a3 + mv a2, a4 + mv a3, a5 + mv a4, a6 + mv a5, a7 + lw a6, 0(sp) + mv a7, t0 + ecall +__cp_end: + ret +__cp_cancel: + tail __cancel diff --git a/src/thread/riscv64/__set_thread_area.s b/src/thread/riscv64/__set_thread_area.s new file mode 100644 index 00000000..828154d2 --- /dev/null +++ b/src/thread/riscv64/__set_thread_area.s @@ -0,0 +1,6 @@ +.global __set_thread_area +.type __set_thread_area, %function +__set_thread_area: + mv tp, a0 + li a0, 0 + ret diff --git a/src/thread/riscv64/__unmapself.s b/src/thread/riscv64/__unmapself.s new file mode 100644 index 00000000..2849119c --- /dev/null +++ b/src/thread/riscv64/__unmapself.s @@ -0,0 +1,7 @@ +.global __unmapself +.type __unmapself, %function +__unmapself: + li a7, 215 # SYS_munmap + ecall + li a7, 93 # SYS_exit + ecall diff --git a/src/thread/riscv64/clone.s b/src/thread/riscv64/clone.s new file mode 100644 index 00000000..db908248 --- /dev/null +++ b/src/thread/riscv64/clone.s @@ -0,0 +1,34 @@ +# __clone(func, stack, flags, arg, ptid, tls, ctid) +# a0, a1, a2, a3, a4, a5, a6 + +# syscall(SYS_clone, flags, stack, ptid, tls, ctid) +# a7 a0, a1, a2, a3, a4 + +.global __clone +.type __clone, %function +__clone: + # Save func and arg to stack + addi a1, a1, -16 + sd a0, 0(a1) + sd a3, 8(a1) + + # Call SYS_clone + mv a0, a2 + mv a2, a4 + mv a3, a5 + mv a4, a6 + li a7, 220 # SYS_clone + ecall + + beqz a0, 1f + # Parent + ret + + # Child +1: ld a1, 0(sp) + ld a0, 8(sp) + jalr a1 + + # Exit + li a7, 93 # SYS_exit + ecall diff --git a/src/thread/riscv64/syscall_cp.s b/src/thread/riscv64/syscall_cp.s new file mode 100644 index 00000000..eeef6391 --- /dev/null +++ b/src/thread/riscv64/syscall_cp.s @@ -0,0 +1,29 @@ +.global __cp_begin +.hidden __cp_begin +.global __cp_end +.hidden __cp_end +.global __cp_cancel +.hidden __cp_cancel +.hidden __cancel +.global __syscall_cp_asm +.hidden __syscall_cp_asm +.type __syscall_cp_asm, %function +__syscall_cp_asm: +__cp_begin: + lw t0, 0(a0) + bnez t0, __cp_cancel + + mv t0, a1 + mv a0, a2 + mv a1, a3 + mv a2, a4 + mv a3, a5 + mv a4, a6 + mv a5, a7 + ld a6, 0(sp) + mv a7, t0 + ecall +__cp_end: + ret +__cp_cancel: + tail __cancel diff --git a/src/thread/s390x/clone.s b/src/thread/s390x/clone.s index 577748ea..2125f20b 100644 --- a/src/thread/s390x/clone.s +++ b/src/thread/s390x/clone.s @@ -17,6 +17,9 @@ __clone: # if (!tid) syscall(SYS_exit, a(d)); # return tid; + # preserve call-saved register used as syscall arg + stg %r6, 48(%r15) + # create initial stack frame for new thread nill %r3, 0xfff8 aghi %r3, -160 @@ -35,6 +38,9 @@ __clone: lg %r6, 160(%r15) svc 120 + # restore call-saved register + lg %r6, 48(%r15) + # if error or if we're the parent, return ltgr %r2, %r2 bnzr %r14 diff --git a/src/thread/s390x/syscall_cp.s b/src/thread/s390x/syscall_cp.s index c1da40de..d094cbf5 100644 --- a/src/thread/s390x/syscall_cp.s +++ b/src/thread/s390x/syscall_cp.s @@ -14,6 +14,7 @@ __cp_begin: icm %r2, 15, 0(%r2) jne __cp_cancel + stg %r6, 48(%r15) stg %r7, 56(%r15) lgr %r1, %r3 lgr %r2, %r4 @@ -26,6 +27,7 @@ __cp_begin: __cp_end: lg %r7, 56(%r15) + lg %r6, 48(%r15) br %r14 __cp_cancel: diff --git a/src/thread/sem_getvalue.c b/src/thread/sem_getvalue.c index d9d83071..c0b7762d 100644 --- a/src/thread/sem_getvalue.c +++ b/src/thread/sem_getvalue.c @@ -1,8 +1,9 @@ #include <semaphore.h> +#include <limits.h> int sem_getvalue(sem_t *restrict sem, int *restrict valp) { int val = sem->__val[0]; - *valp = val < 0 ? 0 : val; + *valp = val & SEM_VALUE_MAX; return 0; } diff --git a/src/thread/sem_open.c b/src/thread/sem_open.c index de8555c5..0ad29de9 100644 --- a/src/thread/sem_open.c +++ b/src/thread/sem_open.c @@ -12,6 +12,12 @@ #include <stdlib.h> #include <pthread.h> #include "lock.h" +#include "fork_impl.h" + +#define malloc __libc_malloc +#define calloc __libc_calloc +#define realloc undef +#define free undef static struct { ino_t ino; @@ -19,6 +25,7 @@ static struct { int refcnt; } *semtab; static volatile int lock[1]; +volatile int *const __sem_open_lockptr = lock; #define FLAGS (O_RDWR|O_NOFOLLOW|O_CLOEXEC|O_NONBLOCK) @@ -163,10 +170,12 @@ int sem_close(sem_t *sem) int i; LOCK(lock); for (i=0; i<SEM_NSEMS_MAX && semtab[i].sem != sem; i++); - if (!--semtab[i].refcnt) { - semtab[i].sem = 0; - semtab[i].ino = 0; + if (--semtab[i].refcnt) { + UNLOCK(lock); + return 0; } + semtab[i].sem = 0; + semtab[i].ino = 0; UNLOCK(lock); munmap(sem, sizeof *sem); return 0; diff --git a/src/thread/sem_post.c b/src/thread/sem_post.c index 31e3293d..5c2517f2 100644 --- a/src/thread/sem_post.c +++ b/src/thread/sem_post.c @@ -1,17 +1,21 @@ #include <semaphore.h> +#include <limits.h> #include "pthread_impl.h" int sem_post(sem_t *sem) { - int val, waiters, priv = sem->__val[2]; + int val, new, waiters, priv = sem->__val[2]; do { val = sem->__val[0]; waiters = sem->__val[1]; - if (val == SEM_VALUE_MAX) { + if ((val & SEM_VALUE_MAX) == SEM_VALUE_MAX) { errno = EOVERFLOW; return -1; } - } while (a_cas(sem->__val, val, val+1+(val<0)) != val); - if (val<0 || waiters) __wake(sem->__val, 1, priv); + new = val + 1; + if (waiters <= 1) + new &= ~0x80000000; + } while (a_cas(sem->__val, val, new) != val); + if (val<0) __wake(sem->__val, waiters>1 ? 1 : -1, priv); return 0; } diff --git a/src/thread/sem_timedwait.c b/src/thread/sem_timedwait.c index 8132eb1b..aa67376c 100644 --- a/src/thread/sem_timedwait.c +++ b/src/thread/sem_timedwait.c @@ -1,4 +1,5 @@ #include <semaphore.h> +#include <limits.h> #include "pthread_impl.h" static void cleanup(void *p) @@ -13,16 +14,17 @@ int sem_timedwait(sem_t *restrict sem, const struct timespec *restrict at) if (!sem_trywait(sem)) return 0; int spins = 100; - while (spins-- && sem->__val[0] <= 0 && !sem->__val[1]) a_spin(); + while (spins-- && !(sem->__val[0] & SEM_VALUE_MAX) && !sem->__val[1]) + a_spin(); while (sem_trywait(sem)) { - int r; + int r, priv = sem->__val[2]; a_inc(sem->__val+1); - a_cas(sem->__val, 0, -1); + a_cas(sem->__val, 0, 0x80000000); pthread_cleanup_push(cleanup, (void *)(sem->__val+1)); - r = __timedwait_cp(sem->__val, -1, CLOCK_REALTIME, at, sem->__val[2]); + r = __timedwait_cp(sem->__val, 0x80000000, CLOCK_REALTIME, at, priv); pthread_cleanup_pop(1); - if (r && r != EINTR) { + if (r) { errno = r; return -1; } diff --git a/src/thread/sem_trywait.c b/src/thread/sem_trywait.c index 04edf46b..beb435da 100644 --- a/src/thread/sem_trywait.c +++ b/src/thread/sem_trywait.c @@ -1,12 +1,12 @@ #include <semaphore.h> +#include <limits.h> #include "pthread_impl.h" int sem_trywait(sem_t *sem) { int val; - while ((val=sem->__val[0]) > 0) { - int new = val-1-(val==1 && sem->__val[1]); - if (a_cas(sem->__val, val, new)==val) return 0; + while ((val=sem->__val[0]) & SEM_VALUE_MAX) { + if (a_cas(sem->__val, val, val-1)==val) return 0; } errno = EAGAIN; return -1; diff --git a/src/thread/synccall.c b/src/thread/synccall.c index cc66bd24..38597254 100644 --- a/src/thread/synccall.c +++ b/src/thread/synccall.c @@ -1,46 +1,42 @@ #include "pthread_impl.h" #include <semaphore.h> -#include <unistd.h> -#include <dirent.h> #include <string.h> -#include <ctype.h> -#include "futex.h" -#include "atomic.h" -#include "../dirent/__dirent.h" -#include "lock.h" - -static struct chain { - struct chain *next; - int tid; - sem_t target_sem, caller_sem; -} *volatile head; - -static volatile int synccall_lock[1]; -static volatile int target_tid; + +static void dummy_0(void) +{ +} + +weak_alias(dummy_0, __tl_lock); +weak_alias(dummy_0, __tl_unlock); + +static int target_tid; static void (*callback)(void *), *context; -static volatile int dummy = 0; -weak_alias(dummy, __block_new_threads); +static sem_t target_sem, caller_sem, exit_sem; + +static void dummy(void *p) +{ +} static void handler(int sig) { - struct chain ch; - int old_errno = errno; + if (__pthread_self()->tid != target_tid) return; - sem_init(&ch.target_sem, 0, 0); - sem_init(&ch.caller_sem, 0, 0); + int old_errno = errno; - ch.tid = __syscall(SYS_gettid); + /* Inform caller we have received signal and wait for + * the caller to let us make the callback. */ + sem_post(&caller_sem); + sem_wait(&target_sem); - do ch.next = head; - while (a_cas_p(&head, ch.next, &ch) != ch.next); + callback(context); - if (a_cas(&target_tid, ch.tid, 0) == (ch.tid | 0x80000000)) - __syscall(SYS_futex, &target_tid, FUTEX_UNLOCK_PI|FUTEX_PRIVATE); + /* Inform caller we've complered the callback and wait + * for the caller to release us to return. */ + sem_post(&caller_sem); + sem_wait(&exit_sem); - sem_wait(&ch.target_sem); - callback(context); - sem_post(&ch.caller_sem); - sem_wait(&ch.target_sem); + /* Inform caller we are returning and state is destroyable. */ + sem_post(&caller_sem); errno = old_errno; } @@ -48,12 +44,10 @@ static void handler(int sig) void __synccall(void (*func)(void *), void *ctx) { sigset_t oldmask; - int cs, i, r, pid, self;; - DIR dir = {0}; - struct dirent *de; - struct sigaction sa = { .sa_flags = SA_RESTART, .sa_handler = handler }; - struct chain *cp, *next; - struct timespec ts; + int cs, i, r; + struct sigaction sa = { .sa_flags = SA_RESTART | SA_ONSTACK, .sa_handler = handler }; + pthread_t self = __pthread_self(), td; + int count = 0; /* Blocking signals in two steps, first only app-level signals * before taking the lock, then all signals after taking the lock, @@ -62,98 +56,47 @@ void __synccall(void (*func)(void *), void *ctx) * any until after the lock would allow re-entry in the same thread * with the lock already held. */ __block_app_sigs(&oldmask); - LOCK(synccall_lock); + __tl_lock(); __block_all_sigs(0); pthread_setcancelstate(PTHREAD_CANCEL_DISABLE, &cs); - head = 0; + sem_init(&target_sem, 0, 0); + sem_init(&caller_sem, 0, 0); + sem_init(&exit_sem, 0, 0); - if (!libc.threaded) goto single_threaded; + if (!libc.threads_minus_1 || __syscall(SYS_gettid) != self->tid) + goto single_threaded; callback = func; context = ctx; - /* This atomic store ensures that any signaled threads will see the - * above stores, and prevents more than a bounded number of threads, - * those already in pthread_create, from creating new threads until - * the value is cleared to zero again. */ - a_store(&__block_new_threads, 1); - /* Block even implementation-internal signals, so that nothing * interrupts the SIGSYNCCALL handlers. The main possible source * of trouble is asynchronous cancellation. */ memset(&sa.sa_mask, -1, sizeof sa.sa_mask); __libc_sigaction(SIGSYNCCALL, &sa, 0); - pid = __syscall(SYS_getpid); - self = __syscall(SYS_gettid); - - /* Since opendir is not AS-safe, the DIR needs to be setup manually - * in automatic storage. Thankfully this is easy. */ - dir.fd = open("/proc/self/task", O_RDONLY|O_DIRECTORY|O_CLOEXEC); - if (dir.fd < 0) goto out; - - /* Initially send one signal per counted thread. But since we can't - * synchronize with thread creation/exit here, there could be too - * few signals. This initial signaling is just an optimization, not - * part of the logic. */ - for (i=libc.threads_minus_1; i; i--) - __syscall(SYS_kill, pid, SIGSYNCCALL); - - /* Loop scanning the kernel-provided thread list until it shows no - * threads that have not already replied to the signal. */ - for (;;) { - int miss_cnt = 0; - while ((de = readdir(&dir))) { - if (!isdigit(de->d_name[0])) continue; - int tid = atoi(de->d_name); - if (tid == self || !tid) continue; - - /* Set the target thread as the PI futex owner before - * checking if it's in the list of caught threads. If it - * adds itself to the list after we check for it, then - * it will see its own tid in the PI futex and perform - * the unlock operation. */ - a_store(&target_tid, tid); - - /* Thread-already-caught is a success condition. */ - for (cp = head; cp && cp->tid != tid; cp=cp->next); - if (cp) continue; - - r = -__syscall(SYS_tgkill, pid, tid, SIGSYNCCALL); - - /* Target thread exit is a success condition. */ - if (r == ESRCH) continue; - - /* The FUTEX_LOCK_PI operation is used to loan priority - * to the target thread, which otherwise may be unable - * to run. Timeout is necessary because there is a race - * condition where the tid may be reused by a different - * process. */ - clock_gettime(CLOCK_REALTIME, &ts); - ts.tv_nsec += 10000000; - if (ts.tv_nsec >= 1000000000) { - ts.tv_sec++; - ts.tv_nsec -= 1000000000; - } - r = -__syscall(SYS_futex, &target_tid, - FUTEX_LOCK_PI|FUTEX_PRIVATE, 0, &ts); - - /* Obtaining the lock means the thread responded. ESRCH - * means the target thread exited, which is okay too. */ - if (!r || r == ESRCH) continue; - - miss_cnt++; + + for (td=self->next; td!=self; td=td->next) { + target_tid = td->tid; + while ((r = -__syscall(SYS_tkill, td->tid, SIGSYNCCALL)) == EAGAIN); + if (r) { + /* If we failed to signal any thread, nop out the + * callback to abort the synccall and just release + * any threads already caught. */ + callback = func = dummy; + break; } - if (!miss_cnt) break; - rewinddir(&dir); + sem_wait(&caller_sem); + count++; } - close(dir.fd); + target_tid = 0; - /* Serialize execution of callback in caught threads. */ - for (cp=head; cp; cp=cp->next) { - sem_post(&cp->target_sem); - sem_wait(&cp->caller_sem); + /* Serialize execution of callback in caught threads, or just + * release them all if synccall is being aborted. */ + for (i=0; i<count; i++) { + sem_post(&target_sem); + sem_wait(&caller_sem); } sa.sa_handler = SIG_IGN; @@ -164,16 +107,16 @@ single_threaded: /* Only release the caught threads once all threads, including the * caller, have returned from the callback function. */ - for (cp=head; cp; cp=next) { - next = cp->next; - sem_post(&cp->target_sem); - } + for (i=0; i<count; i++) + sem_post(&exit_sem); + for (i=0; i<count; i++) + sem_wait(&caller_sem); -out: - a_store(&__block_new_threads, 0); - __wake(&__block_new_threads, -1, 1); + sem_destroy(&caller_sem); + sem_destroy(&target_sem); + sem_destroy(&exit_sem); pthread_setcancelstate(cs, 0); - UNLOCK(synccall_lock); + __tl_unlock(); __restore_sigs(&oldmask); } diff --git a/src/thread/thrd_sleep.c b/src/thread/thrd_sleep.c index e8dfe400..97de5345 100644 --- a/src/thread/thrd_sleep.c +++ b/src/thread/thrd_sleep.c @@ -1,10 +1,11 @@ #include <threads.h> +#include <time.h> #include <errno.h> #include "syscall.h" int thrd_sleep(const struct timespec *req, struct timespec *rem) { - int ret = __syscall(SYS_nanosleep, req, rem); + int ret = -__clock_nanosleep(CLOCK_REALTIME, 0, req, rem); switch (ret) { case 0: return 0; case -EINTR: return -1; /* value specified by C11 */ diff --git a/src/thread/vmlock.c b/src/thread/vmlock.c index 75f3cb76..fa0a8e3c 100644 --- a/src/thread/vmlock.c +++ b/src/thread/vmlock.c @@ -1,6 +1,8 @@ #include "pthread_impl.h" +#include "fork_impl.h" static volatile int vmlock[2]; +volatile int *const __vmlock_lockptr = vmlock; void __vm_wait() { diff --git a/src/thread/x32/syscall_cp.s b/src/thread/x32/syscall_cp.s index 9805af0a..4f101716 100644 --- a/src/thread/x32/syscall_cp.s +++ b/src/thread/x32/syscall_cp.s @@ -6,10 +6,10 @@ .global __cp_cancel .hidden __cp_cancel .hidden __cancel -.global __syscall_cp_internal -.hidden __syscall_cp_internal -.type __syscall_cp_internal,@function -__syscall_cp_internal: +.global __syscall_cp_asm +.hidden __syscall_cp_asm +.type __syscall_cp_asm,@function +__syscall_cp_asm: __cp_begin: mov (%rdi),%eax diff --git a/src/thread/x32/syscall_cp_fixup.c b/src/thread/x32/syscall_cp_fixup.c deleted file mode 100644 index 4956610f..00000000 --- a/src/thread/x32/syscall_cp_fixup.c +++ /dev/null @@ -1,39 +0,0 @@ -#include <sys/syscall.h> -#include <features.h> - -hidden long __syscall_cp_internal(volatile void*, long long, long long, - long long, long long, long long, - long long, long long); - -struct __timespec { long long tv_sec; long tv_nsec; }; -struct __timespec_kernel { long long tv_sec; long long tv_nsec; }; -#define __tsc(X) ((struct __timespec*)(unsigned long)(X)) -#define __fixup(X) do { if(X) { \ - ts->tv_sec = __tsc(X)->tv_sec; \ - ts->tv_nsec = __tsc(X)->tv_nsec; \ - (X) = (unsigned long)ts; } } while(0) - -hidden long __syscall_cp_asm (volatile void * foo, long long n, long long a1, - long long a2, long long a3, long long a4, - long long a5, long long a6) -{ - struct __timespec_kernel ts[1]; - switch (n) { - case SYS_mq_timedsend: case SYS_mq_timedreceive: case SYS_pselect6: - __fixup(a5); - break; - case SYS_futex: - if((a2 & (~128 /* FUTEX_PRIVATE_FLAG */)) == 0 /* FUTEX_WAIT */) - __fixup(a4); - break; - case SYS_clock_nanosleep: - case SYS_rt_sigtimedwait: case SYS_ppoll: - __fixup(a3); - break; - case SYS_nanosleep: - __fixup(a1); - break; - } - return __syscall_cp_internal(foo, n, a1, a2, a3, a4, a5, a6); -} - |