summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
-rw-r--r--.mailmap1
-rw-r--r--COPYRIGHT9
-rw-r--r--Makefile3
-rw-r--r--VERSION2
-rw-r--r--WHATSNEW72
-rw-r--r--arch/aarch64/bits/hwcap.h10
-rw-r--r--arch/aarch64/bits/syscall.h.in5
-rw-r--r--arch/aarch64/pthread_arch.h9
-rw-r--r--arch/arm/bits/syscall.h.in25
-rw-r--r--arch/arm/pthread_arch.h17
-rw-r--r--arch/arm/syscall_arch.h6
-rw-r--r--arch/generic/bits/dirent.h11
-rw-r--r--arch/generic/bits/fcntl.h6
-rw-r--r--arch/i386/bits/syscall.h.in25
-rw-r--r--arch/i386/pthread_arch.h10
-rw-r--r--arch/i386/syscall_arch.h2
-rw-r--r--arch/m68k/bits/syscall.h.in25
-rw-r--r--arch/m68k/pthread_arch.h7
-rw-r--r--arch/m68k/syscall_arch.h1
-rw-r--r--arch/microblaze/bits/syscall.h.in25
-rw-r--r--arch/microblaze/pthread_arch.h10
-rw-r--r--arch/microblaze/syscall_arch.h2
-rw-r--r--arch/mips/bits/hwcap.h11
-rw-r--r--arch/mips/bits/signal.h10
-rw-r--r--arch/mips/bits/syscall.h.in25
-rw-r--r--arch/mips/pthread_arch.h10
-rw-r--r--arch/mips/syscall_arch.h82
-rw-r--r--arch/mips64/bits/fcntl.h2
-rw-r--r--arch/mips64/bits/signal.h2
-rw-r--r--arch/mips64/bits/syscall.h.in5
-rw-r--r--arch/mips64/pthread_arch.h10
-rw-r--r--arch/mips64/reloc.h2
-rw-r--r--arch/mips64/syscall_arch.h70
-rw-r--r--arch/mipsn32/bits/signal.h2
-rw-r--r--arch/mipsn32/bits/syscall.h.in25
-rw-r--r--arch/mipsn32/pthread_arch.h10
-rw-r--r--arch/mipsn32/syscall_arch.h75
-rw-r--r--arch/or1k/bits/syscall.h.in25
-rw-r--r--arch/or1k/pthread_arch.h10
-rw-r--r--arch/powerpc/bits/mman.h1
-rw-r--r--arch/powerpc/bits/syscall.h.in25
-rw-r--r--arch/powerpc/pthread_arch.h10
-rw-r--r--arch/powerpc64/bits/mman.h1
-rw-r--r--arch/powerpc64/bits/syscall.h.in5
-rw-r--r--arch/powerpc64/pthread_arch.h10
-rw-r--r--arch/riscv64/atomic_arch.h2
-rw-r--r--arch/riscv64/bits/fcntl.h38
-rw-r--r--arch/riscv64/bits/reg.h6
-rw-r--r--arch/riscv64/bits/signal.h9
-rw-r--r--arch/riscv64/bits/syscall.h.in5
-rw-r--r--arch/riscv64/pthread_arch.h7
-rw-r--r--arch/s390x/bits/syscall.h.in5
-rw-r--r--arch/s390x/pthread_arch.h10
-rw-r--r--arch/s390x/syscall_arch.h2
-rw-r--r--arch/sh/bits/syscall.h.in25
-rw-r--r--arch/sh/pthread_arch.h9
-rw-r--r--arch/x32/bits/fcntl.h2
-rw-r--r--arch/x32/bits/syscall.h.in6
-rw-r--r--arch/x32/pthread_arch.h12
-rw-r--r--arch/x86_64/bits/fcntl.h40
-rw-r--r--arch/x86_64/bits/syscall.h.in5
-rw-r--r--arch/x86_64/pthread_arch.h10
-rw-r--r--compat/time32/setitimer_time32.c10
-rwxr-xr-xconfigure58
-rw-r--r--include/alloca.h2
-rw-r--r--include/alltypes.h.in3
-rw-r--r--include/arpa/inet.h5
-rw-r--r--include/dirent.h14
-rw-r--r--include/elf.h2
-rw-r--r--include/limits.h2
-rw-r--r--include/netinet/if_ether.h2
-rw-r--r--include/netinet/in.h6
-rw-r--r--include/netinet/ip.h2
-rw-r--r--include/netinet/tcp.h21
-rw-r--r--include/netinet/udp.h1
-rw-r--r--include/pthread.h3
-rw-r--r--include/sched.h1
-rw-r--r--include/sys/fanotify.h4
-rw-r--r--include/sys/ioctl.h10
-rw-r--r--include/sys/mman.h3
-rw-r--r--include/sys/prctl.h7
-rw-r--r--include/sys/procfs.h7
-rw-r--r--include/sys/ptrace.h29
-rw-r--r--include/sys/random.h1
-rw-r--r--include/sys/resource.h3
-rw-r--r--include/sys/socket.h1
-rw-r--r--include/sys/ttydefaults.h7
-rw-r--r--include/sys/wait.h3
-rw-r--r--include/termios.h4
-rw-r--r--include/unistd.h1
-rw-r--r--include/utmpx.h7
-rw-r--r--ldso/dynlink.c45
-rw-r--r--src/complex/catanf.c14
-rw-r--r--src/complex/catanl.c14
-rw-r--r--src/ctype/wcwidth.c2
-rw-r--r--src/env/__init_tls.c2
-rw-r--r--src/env/__stack_chk_fail.c2
-rw-r--r--src/errno/__strerror.h13
-rw-r--r--src/errno/strerror.c41
-rw-r--r--src/fenv/riscv64/fenv.S5
-rw-r--r--src/fenv/sh/fenv.S2
-rw-r--r--src/internal/atomic.h15
-rw-r--r--src/internal/dynlink.h6
-rw-r--r--src/internal/libc.h9
-rw-r--r--src/internal/libm.h7
-rw-r--r--src/internal/pthread_impl.h32
-rw-r--r--src/internal/shgetc.c2
-rw-r--r--src/internal/stdio_impl.h2
-rw-r--r--src/internal/syscall.h78
-rw-r--r--src/ipc/msgctl.c10
-rw-r--r--src/ipc/semctl.c10
-rw-r--r--src/ipc/shmctl.c10
-rw-r--r--src/linux/clock_adjtime.c95
-rw-r--r--src/linux/gettid.c8
-rw-r--r--src/locale/locale_map.c2
-rw-r--r--src/malloc/DESIGN22
-rw-r--r--src/malloc/aligned_alloc.c7
-rw-r--r--src/malloc/calloc.c45
-rw-r--r--src/malloc/expand_heap.c71
-rw-r--r--src/malloc/lite_malloc.c100
-rw-r--r--src/malloc/mallocng/aligned_alloc.c57
-rw-r--r--src/malloc/mallocng/donate.c39
-rw-r--r--src/malloc/mallocng/free.c143
-rw-r--r--src/malloc/mallocng/glue.h77
-rw-r--r--src/malloc/mallocng/malloc.c387
-rw-r--r--src/malloc/mallocng/malloc_usable_size.c12
-rw-r--r--src/malloc/mallocng/meta.h288
-rw-r--r--src/malloc/mallocng/realloc.c51
-rw-r--r--src/malloc/memalign.c53
-rw-r--r--src/malloc/oldmalloc/aligned_alloc.c53
-rw-r--r--src/malloc/oldmalloc/malloc.c (renamed from src/malloc/malloc.c)357
-rw-r--r--src/malloc/oldmalloc/malloc_impl.h (renamed from src/internal/malloc_impl.h)9
-rw-r--r--src/malloc/oldmalloc/malloc_usable_size.c (renamed from src/malloc/malloc_usable_size.c)0
-rw-r--r--src/malloc/posix_memalign.c3
-rw-r--r--src/malloc/replaced.c4
-rw-r--r--src/math/__expo2.c5
-rw-r--r--src/math/__expo2f.c5
-rw-r--r--src/math/__math_invalidl.c9
-rw-r--r--src/math/__rem_pio2.c15
-rw-r--r--src/math/__rem_pio2f.c13
-rw-r--r--src/math/__rem_pio2l.c16
-rw-r--r--src/math/cosh.c2
-rw-r--r--src/math/coshf.c2
-rw-r--r--src/math/i386/acos.s16
-rw-r--r--src/math/i386/acosf.s17
-rw-r--r--src/math/i386/acosl.s15
-rw-r--r--src/math/i386/asin.s32
-rw-r--r--src/math/i386/asinf.s24
-rw-r--r--src/math/i386/asinl.s13
-rw-r--r--src/math/i386/atan.s2
-rw-r--r--src/math/i386/atan2.s3
-rw-r--r--src/math/i386/atan2f.s3
-rw-r--r--src/math/i386/atanf.s2
-rw-r--r--src/math/i386/exp2.s1
-rw-r--r--src/math/i386/exp2f.s1
-rw-r--r--src/math/i386/exp2l.s2
-rw-r--r--src/math/i386/exp_ld.s (renamed from src/math/i386/exp.s)55
-rw-r--r--src/math/i386/expf.s1
-rw-r--r--src/math/i386/expm1.s1
-rw-r--r--src/math/i386/expm1f.s1
-rw-r--r--src/math/i386/expm1l.s2
-rw-r--r--src/math/i386/fabs.c7
-rw-r--r--src/math/i386/fabs.s6
-rw-r--r--src/math/i386/fabsf.c7
-rw-r--r--src/math/i386/fabsf.s6
-rw-r--r--src/math/i386/fabsl.c7
-rw-r--r--src/math/i386/fabsl.s6
-rw-r--r--src/math/i386/fmod.c10
-rw-r--r--src/math/i386/fmod.s11
-rw-r--r--src/math/i386/fmodf.c10
-rw-r--r--src/math/i386/fmodf.s11
-rw-r--r--src/math/i386/fmodl.c9
-rw-r--r--src/math/i386/fmodl.s11
-rw-r--r--src/math/i386/llrint.c8
-rw-r--r--src/math/i386/llrint.s8
-rw-r--r--src/math/i386/llrintf.c8
-rw-r--r--src/math/i386/llrintf.s9
-rw-r--r--src/math/i386/llrintl.c8
-rw-r--r--src/math/i386/llrintl.s8
-rw-r--r--src/math/i386/log.s2
-rw-r--r--src/math/i386/log10.s2
-rw-r--r--src/math/i386/log10f.s2
-rw-r--r--src/math/i386/log1p.s4
-rw-r--r--src/math/i386/log1pf.s4
-rw-r--r--src/math/i386/log2.s2
-rw-r--r--src/math/i386/log2f.s2
-rw-r--r--src/math/i386/logf.s2
-rw-r--r--src/math/i386/lrint.c8
-rw-r--r--src/math/i386/lrint.s7
-rw-r--r--src/math/i386/lrintf.c8
-rw-r--r--src/math/i386/lrintf.s7
-rw-r--r--src/math/i386/lrintl.c8
-rw-r--r--src/math/i386/lrintl.s7
-rw-r--r--src/math/i386/remainder.c12
-rw-r--r--src/math/i386/remainder.s14
-rw-r--r--src/math/i386/remainderf.c12
-rw-r--r--src/math/i386/remainderf.s14
-rw-r--r--src/math/i386/remainderl.c9
-rw-r--r--src/math/i386/remainderl.s11
-rw-r--r--src/math/i386/rint.c7
-rw-r--r--src/math/i386/rint.s6
-rw-r--r--src/math/i386/rintf.c7
-rw-r--r--src/math/i386/rintf.s6
-rw-r--r--src/math/i386/rintl.c7
-rw-r--r--src/math/i386/rintl.s6
-rw-r--r--src/math/i386/sqrt.c15
-rw-r--r--src/math/i386/sqrt.s21
-rw-r--r--src/math/i386/sqrtf.c12
-rw-r--r--src/math/i386/sqrtf.s7
-rw-r--r--src/math/i386/sqrtl.c7
-rw-r--r--src/math/i386/sqrtl.s5
-rw-r--r--src/math/m68k/sqrtl.c15
-rw-r--r--src/math/powerpc/fabs.c2
-rw-r--r--src/math/powerpc/fma.c2
-rw-r--r--src/math/sinh.c2
-rw-r--r--src/math/sinhf.c2
-rw-r--r--src/math/sqrt.c320
-rw-r--r--src/math/sqrt_data.c19
-rw-r--r--src/math/sqrt_data.h13
-rw-r--r--src/math/sqrtf.c140
-rw-r--r--src/math/sqrtl.c254
-rw-r--r--src/math/x32/lrintl.s4
-rw-r--r--src/math/x86_64/fabs.c10
-rw-r--r--src/math/x86_64/fabs.s9
-rw-r--r--src/math/x86_64/fabsf.c10
-rw-r--r--src/math/x86_64/fabsf.s7
-rw-r--r--src/math/x86_64/fabsl.c7
-rw-r--r--src/math/x86_64/fabsl.s6
-rw-r--r--src/math/x86_64/fmodl.c9
-rw-r--r--src/math/x86_64/fmodl.s11
-rw-r--r--src/math/x86_64/llrint.c8
-rw-r--r--src/math/x86_64/llrint.s5
-rw-r--r--src/math/x86_64/llrintf.c8
-rw-r--r--src/math/x86_64/llrintf.s5
-rw-r--r--src/math/x86_64/llrintl.c8
-rw-r--r--src/math/x86_64/llrintl.s7
-rw-r--r--src/math/x86_64/lrint.c8
-rw-r--r--src/math/x86_64/lrint.s5
-rw-r--r--src/math/x86_64/lrintf.c8
-rw-r--r--src/math/x86_64/lrintf.s5
-rw-r--r--src/math/x86_64/lrintl.c8
-rw-r--r--src/math/x86_64/lrintl.s7
-rw-r--r--src/math/x86_64/remainderl.c9
-rw-r--r--src/math/x86_64/remainderl.s11
-rw-r--r--src/math/x86_64/remquol.c32
-rw-r--r--src/math/x86_64/rintl.c7
-rw-r--r--src/math/x86_64/rintl.s6
-rw-r--r--src/math/x86_64/sqrt.c7
-rw-r--r--src/math/x86_64/sqrt.s4
-rw-r--r--src/math/x86_64/sqrtf.c7
-rw-r--r--src/math/x86_64/sqrtf.s4
-rw-r--r--src/math/x86_64/sqrtl.c7
-rw-r--r--src/math/x86_64/sqrtl.s5
-rw-r--r--src/misc/getentropy.c2
-rw-r--r--src/misc/ioctl.c136
-rw-r--r--src/misc/nftw.c22
-rw-r--r--src/network/getnameinfo.c1
-rw-r--r--src/network/getsockopt.c9
-rw-r--r--src/network/h_errno.c4
-rw-r--r--src/network/herror.c2
-rw-r--r--src/network/lookup_name.c23
-rw-r--r--src/network/recvmmsg.c14
-rw-r--r--src/network/recvmsg.c47
-rw-r--r--src/network/res_mkquery.c1
-rw-r--r--src/network/res_query.c16
-rw-r--r--src/network/res_send.c2
-rw-r--r--src/network/setsockopt.c9
-rw-r--r--src/network/socket.c10
-rw-r--r--src/process/fork.c1
-rw-r--r--src/setjmp/aarch64/longjmp.s7
-rw-r--r--src/setjmp/i386/longjmp.s12
-rw-r--r--src/setjmp/x32/longjmp.s14
-rw-r--r--src/setjmp/x32/setjmp.s2
-rw-r--r--src/setjmp/x86_64/longjmp.s14
-rw-r--r--src/setjmp/x86_64/setjmp.s2
-rw-r--r--src/signal/arm/sigsetjmp.s5
-rw-r--r--src/stat/fchmodat.c3
-rw-r--r--src/stdio/__string_read.c16
-rw-r--r--src/stdio/fmemopen.c6
-rw-r--r--src/stdio/tempnam.c5
-rw-r--r--src/stdio/tmpnam.c5
-rw-r--r--src/stdio/vdprintf.c7
-rw-r--r--src/stdio/vfscanf.c5
-rw-r--r--src/stdio/vsscanf.c16
-rw-r--r--src/stdlib/wcstod.c3
-rw-r--r--src/stdlib/wcstol.c3
-rw-r--r--src/string/aarch64/memcpy.S186
-rw-r--r--src/string/aarch64/memset.S115
-rw-r--r--src/string/arm/memcpy.S (renamed from src/string/arm/memcpy_le.S)110
-rw-r--r--src/string/arm/memcpy.c3
-rw-r--r--src/string/memccpy.c2
-rw-r--r--src/string/memmem.c8
-rw-r--r--src/string/strsignal.c10
-rw-r--r--src/string/strstr.c8
-rw-r--r--src/termios/tcgetwinsize.c8
-rw-r--r--src/termios/tcsetwinsize.c8
-rw-r--r--src/thread/__lock.c4
-rw-r--r--src/thread/i386/__set_thread_area.s1
-rw-r--r--src/thread/pthread_create.c36
-rw-r--r--src/thread/pthread_getschedparam.c3
-rw-r--r--src/thread/pthread_kill.c6
-rw-r--r--src/thread/pthread_setschedparam.c3
-rw-r--r--src/thread/pthread_setschedprio.c3
-rw-r--r--src/thread/synccall.c3
-rw-r--r--src/time/__map_file.c3
-rw-r--r--src/time/__tz.c12
-rw-r--r--src/unistd/faccessat.c11
-rw-r--r--tools/add-cfi.i386.awk2
-rw-r--r--tools/add-cfi.x86_64.awk2
309 files changed, 4177 insertions, 1795 deletions
diff --git a/.mailmap b/.mailmap
new file mode 100644
index 00000000..aede9ec8
--- /dev/null
+++ b/.mailmap
@@ -0,0 +1 @@
+Ada Worcester <oss@ada.pikhq.com> <josiahw@gmail.com>
diff --git a/COPYRIGHT b/COPYRIGHT
index 67802d11..c1628e9a 100644
--- a/COPYRIGHT
+++ b/COPYRIGHT
@@ -1,7 +1,7 @@
musl as a whole is licensed under the following standard MIT license:
----------------------------------------------------------------------
-Copyright © 2005-2019 Rich Felker, et al.
+Copyright © 2005-2020 Rich Felker, et al.
Permission is hereby granted, free of charge, to any person obtaining
a copy of this software and associated documentation files (the
@@ -26,6 +26,7 @@ SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
Authors/contributors include:
A. Wilcox
+Ada Worcester
Alex Dowad
Alex Suykov
Alexander Monakov
@@ -65,7 +66,6 @@ Jeremy Huntwork
Jo-Philipp Wich
Joakim Sindholt
John Spencer
-Josiah Worcester
Julien Ramseier
Justin Cormack
Kaarle Ritvanen
@@ -127,10 +127,13 @@ Copyright © 2017-2018 Arm Limited
and labelled as such in comments in the individual source files. All
have been licensed under extremely permissive terms.
-The ARM memcpy code (src/string/arm/memcpy_el.S) is Copyright © 2008
+The ARM memcpy code (src/string/arm/memcpy.S) is Copyright © 2008
The Android Open Source Project and is licensed under a two-clause BSD
license. It was taken from Bionic libc, used on Android.
+The AArch64 memcpy and memset code (src/string/aarch64/*) are
+Copyright © 1999-2019, Arm Limited.
+
The implementation of DES for crypt (src/crypt/crypt_des.c) is
Copyright © 1994 David Burren. It is licensed under a BSD license.
diff --git a/Makefile b/Makefile
index bd8f5c38..e8cc4436 100644
--- a/Makefile
+++ b/Makefile
@@ -17,7 +17,8 @@ includedir = $(prefix)/include
libdir = $(prefix)/lib
syslibdir = /lib
-SRC_DIRS = $(addprefix $(srcdir)/,src/* crt ldso $(COMPAT_SRC_DIRS))
+MALLOC_DIR = mallocng
+SRC_DIRS = $(addprefix $(srcdir)/,src/* src/malloc/$(MALLOC_DIR) crt ldso $(COMPAT_SRC_DIRS))
BASE_GLOBS = $(addsuffix /*.c,$(SRC_DIRS))
ARCH_GLOBS = $(addsuffix /$(ARCH)/*.[csS],$(SRC_DIRS))
BASE_SRCS = $(sort $(wildcard $(BASE_GLOBS)))
diff --git a/VERSION b/VERSION
index 3fe3e58a..6085e946 100644
--- a/VERSION
+++ b/VERSION
@@ -1 +1 @@
-1.1.24
+1.2.1
diff --git a/WHATSNEW b/WHATSNEW
index f9879bd5..d9826fc0 100644
--- a/WHATSNEW
+++ b/WHATSNEW
@@ -2164,3 +2164,75 @@ arch-specific bugs fixed:
- riscv64 atomics had broken asm constraints (missing earlyclobber flag)
- arm clone() was broken when compiled as thumb if start function returned
- mipsr6 setjmp/longjmp did not preserve fpu register state correctly
+
+
+
+1.2.0 release notes
+
+new features:
+- time_t is now 64-bit on all archs (not just 64-bit archs)
+- character type & case mapping data updated to Unicode 12.1.0
+- header-level support for new linux features in 5.3 and 5.4
+
+performance:
+- new O(1) wchar_t case mapping implementation
+- i386 now uses C math code for exp, faster than old asm
+- mips math asm
+
+compatibility & conformance:
+- endian.h now aims to conform to future POSIX definition
+- support older compilers that don't accept powerpc math asm constraints
+- fdpic code in ldso was incompatible with valid optimizations in gcc 9+
+- RLIMIT_RTTIME was missing from sys/resource.h
+
+bugs fixed:
+- wcwidth wrongly returned 0 for most of planes 4 and up
+- missing case mapping between U+03F3 and U+037F
+- wrong cacosh results for arguments with negative imaginary part
+- wrong catanf/catanl results for various classes of arguments
+- wrong return value for ungetc with argument outside [0,UCHAR_MAX]
+- posix_openpt with no ptys available produced wrong errno
+
+arch-specific bugs fixed:
+- sigcontext/regset definition mistakes & omissions on m68k, powerpc64
+- fesetenv(FE_DFL_ENV) crashed on riscv64
+- sh2 dynamic linker was broken since 1.1.21 (crash in stage 2b)
+- arm dynamic linker chose wrong tls/atomic variants since 1.1.21
+- some math library functions returned excess precision on i386
+- unconfirmed regression in fchmodat AT_SYMLINK_NOFOLLOW on mips*
+
+
+
+1.2.1 release notes
+
+major changes:
+- new malloc implementation (mallocng & overhauled bump allocator)
+
+new features:
+- DNS queries via res_* now set AD flag, report zone signedness (DNSSEC)
+- PTHREAD_NULL macro (POSIX-future)
+
+performance:
+- optimized memcpy and memset for aarch64
+- optimized memcpy for arm now supports big endian
+- optimized x86_64 remquol
+- improved strerror without linear search
+
+bugs fixed:
+- lock-skipping for processes that returned to single-threaded was wrong
+- AF_UNSPEC dns lookups mishandled single failure in paired A+AAAA
+- res_send and res_query returned wrong value on errors from nameserver
+- corrupted sysvipc timestamps on 32-bit archs with old kernels
+- incorrect parsing of timezone offsets after overly-long zone name
+- clock_adjtime was broken on 32-bit archs (time64)
+- pthread_kill as not async-signal-safe
+- pthread_cancel was not async-cancel-safe
+- large-ulp errors in various math functions in non-default rounding modes
+
+arch-specific bugs fixed:
+- arm clock_gettime was broken on some hw due to bad time64 vdso
+- m68k sqrtl lacked long double precision
+- mips* syscall mechanism regressions on older kernels
+- mips* had negated error codes for some syscalls (kernel bug)
+- mips* SIGEMT was wrongly called SIGSTKFLT
+- sh fesetround didn't work correctly on sh
diff --git a/arch/aarch64/bits/hwcap.h b/arch/aarch64/bits/hwcap.h
index a7484028..7ab73f99 100644
--- a/arch/aarch64/bits/hwcap.h
+++ b/arch/aarch64/bits/hwcap.h
@@ -38,3 +38,13 @@
#define HWCAP2_SVEBITPERM (1 << 4)
#define HWCAP2_SVESHA3 (1 << 5)
#define HWCAP2_SVESM4 (1 << 6)
+#define HWCAP2_FLAGM2 (1 << 7)
+#define HWCAP2_FRINT (1 << 8)
+#define HWCAP2_SVEI8MM (1 << 9)
+#define HWCAP2_SVEF32MM (1 << 10)
+#define HWCAP2_SVEF64MM (1 << 11)
+#define HWCAP2_SVEBF16 (1 << 12)
+#define HWCAP2_I8MM (1 << 13)
+#define HWCAP2_BF16 (1 << 14)
+#define HWCAP2_DGH (1 << 15)
+#define HWCAP2_RNG (1 << 16)
diff --git a/arch/aarch64/bits/syscall.h.in b/arch/aarch64/bits/syscall.h.in
index 955e2cab..ac3eaf80 100644
--- a/arch/aarch64/bits/syscall.h.in
+++ b/arch/aarch64/bits/syscall.h.in
@@ -287,4 +287,9 @@
#define __NR_fsconfig 431
#define __NR_fsmount 432
#define __NR_fspick 433
+#define __NR_pidfd_open 434
+#define __NR_clone3 435
+#define __NR_openat2 437
+#define __NR_pidfd_getfd 438
+#define __NR_faccessat2 439
diff --git a/arch/aarch64/pthread_arch.h b/arch/aarch64/pthread_arch.h
index e64b126d..3909616c 100644
--- a/arch/aarch64/pthread_arch.h
+++ b/arch/aarch64/pthread_arch.h
@@ -1,12 +1,11 @@
-static inline struct pthread *__pthread_self()
+static inline uintptr_t __get_tp()
{
- char *self;
- __asm__ ("mrs %0,tpidr_el0" : "=r"(self));
- return (void*)(self - sizeof(struct pthread));
+ uintptr_t tp;
+ __asm__ ("mrs %0,tpidr_el0" : "=r"(tp));
+ return tp;
}
#define TLS_ABOVE_TP
#define GAP_ABOVE_TP 16
-#define TP_ADJ(p) ((char *)(p) + sizeof(struct pthread))
#define MC_PC pc
diff --git a/arch/arm/bits/syscall.h.in b/arch/arm/bits/syscall.h.in
index a565a4ee..5b4e6791 100644
--- a/arch/arm/bits/syscall.h.in
+++ b/arch/arm/bits/syscall.h.in
@@ -55,8 +55,8 @@
#define __NR_sethostname 74
#define __NR_setrlimit 75
#define __NR_getrusage 77
-#define __NR_gettimeofday 78
-#define __NR_settimeofday 79
+#define __NR_gettimeofday_time32 78
+#define __NR_settimeofday_time32 79
#define __NR_getgroups 80
#define __NR_setgroups 81
#define __NR_symlink 83
@@ -211,14 +211,14 @@
#define __NR_remap_file_pages 253
#define __NR_set_tid_address 256
#define __NR_timer_create 257
-#define __NR_timer_settime 258
-#define __NR_timer_gettime 259
+#define __NR_timer_settime32 258
+#define __NR_timer_gettime32 259
#define __NR_timer_getoverrun 260
#define __NR_timer_delete 261
-#define __NR_clock_settime 262
-#define __NR_clock_gettime 263
-#define __NR_clock_getres 264
-#define __NR_clock_nanosleep 265
+#define __NR_clock_settime32 262
+#define __NR_clock_gettime32 263
+#define __NR_clock_getres_time32 264
+#define __NR_clock_nanosleep_time32 265
#define __NR_statfs64 266
#define __NR_fstatfs64 267
#define __NR_tgkill 268
@@ -308,8 +308,8 @@
#define __NR_timerfd_create 350
#define __NR_eventfd 351
#define __NR_fallocate 352
-#define __NR_timerfd_settime 353
-#define __NR_timerfd_gettime 354
+#define __NR_timerfd_settime32 353
+#define __NR_timerfd_gettime32 354
#define __NR_signalfd4 355
#define __NR_eventfd2 356
#define __NR_epoll_create1 357
@@ -387,6 +387,11 @@
#define __NR_fsconfig 431
#define __NR_fsmount 432
#define __NR_fspick 433
+#define __NR_pidfd_open 434
+#define __NR_clone3 435
+#define __NR_openat2 437
+#define __NR_pidfd_getfd 438
+#define __NR_faccessat2 439
#define __ARM_NR_breakpoint 0x0f0001
#define __ARM_NR_cacheflush 0x0f0002
diff --git a/arch/arm/pthread_arch.h b/arch/arm/pthread_arch.h
index e689ea21..157e2eae 100644
--- a/arch/arm/pthread_arch.h
+++ b/arch/arm/pthread_arch.h
@@ -1,11 +1,11 @@
#if ((__ARM_ARCH_6K__ || __ARM_ARCH_6KZ__ || __ARM_ARCH_6ZK__) && !__thumb__) \
|| __ARM_ARCH_7A__ || __ARM_ARCH_7R__ || __ARM_ARCH >= 7
-static inline pthread_t __pthread_self()
+static inline uintptr_t __get_tp()
{
- char *p;
- __asm__ ( "mrc p15,0,%0,c13,c0,3" : "=r"(p) );
- return (void *)(p-sizeof(struct pthread));
+ uintptr_t tp;
+ __asm__ ( "mrc p15,0,%0,c13,c0,3" : "=r"(tp) );
+ return tp;
}
#else
@@ -16,18 +16,17 @@ static inline pthread_t __pthread_self()
#define BLX "blx"
#endif
-static inline pthread_t __pthread_self()
+static inline uintptr_t __get_tp()
{
extern hidden uintptr_t __a_gettp_ptr;
- register uintptr_t p __asm__("r0");
- __asm__ ( BLX " %1" : "=r"(p) : "r"(__a_gettp_ptr) : "cc", "lr" );
- return (void *)(p-sizeof(struct pthread));
+ register uintptr_t tp __asm__("r0");
+ __asm__ ( BLX " %1" : "=r"(tp) : "r"(__a_gettp_ptr) : "cc", "lr" );
+ return tp;
}
#endif
#define TLS_ABOVE_TP
#define GAP_ABOVE_TP 8
-#define TP_ADJ(p) ((char *)(p) + sizeof(struct pthread))
#define MC_PC arm_pc
diff --git a/arch/arm/syscall_arch.h b/arch/arm/syscall_arch.h
index 4b08762d..a877b2cf 100644
--- a/arch/arm/syscall_arch.h
+++ b/arch/arm/syscall_arch.h
@@ -98,12 +98,6 @@ static inline long __syscall6(long n, long a, long b, long c, long d, long e, lo
__asm_syscall(R7_OPERAND, "0"(r0), "r"(r1), "r"(r2), "r"(r3), "r"(r4), "r"(r5));
}
-#define VDSO_USEFUL
-#define VDSO_CGT32_SYM "__vdso_clock_gettime"
-#define VDSO_CGT32_VER "LINUX_2.6"
-#define VDSO_CGT_SYM "__vdso_clock_gettime64"
-#define VDSO_CGT_VER "LINUX_2.6"
-
#define SYSCALL_FADVISE_6_ARG
#define SYSCALL_IPC_BROKEN_MODE
diff --git a/arch/generic/bits/dirent.h b/arch/generic/bits/dirent.h
new file mode 100644
index 00000000..c845fe82
--- /dev/null
+++ b/arch/generic/bits/dirent.h
@@ -0,0 +1,11 @@
+#define _DIRENT_HAVE_D_RECLEN
+#define _DIRENT_HAVE_D_OFF
+#define _DIRENT_HAVE_D_TYPE
+
+struct dirent {
+ ino_t d_ino;
+ off_t d_off;
+ unsigned short d_reclen;
+ unsigned char d_type;
+ char d_name[256];
+};
diff --git a/arch/generic/bits/fcntl.h b/arch/generic/bits/fcntl.h
index ae233cc0..730a98cf 100644
--- a/arch/generic/bits/fcntl.h
+++ b/arch/generic/bits/fcntl.h
@@ -30,9 +30,15 @@
#define F_SETSIG 10
#define F_GETSIG 11
+#if __LONG_MAX == 0x7fffffffL
#define F_GETLK 12
#define F_SETLK 13
#define F_SETLKW 14
+#else
+#define F_GETLK 5
+#define F_SETLK 6
+#define F_SETLKW 7
+#endif
#define F_SETOWN_EX 15
#define F_GETOWN_EX 16
diff --git a/arch/i386/bits/syscall.h.in b/arch/i386/bits/syscall.h.in
index c95e108e..fb562db5 100644
--- a/arch/i386/bits/syscall.h.in
+++ b/arch/i386/bits/syscall.h.in
@@ -76,8 +76,8 @@
#define __NR_setrlimit 75
#define __NR_getrlimit 76 /* Back compatible 2Gig limited rlimit */
#define __NR_getrusage 77
-#define __NR_gettimeofday 78
-#define __NR_settimeofday 79
+#define __NR_gettimeofday_time32 78
+#define __NR_settimeofday_time32 79
#define __NR_getgroups 80
#define __NR_setgroups 81
#define __NR_select 82
@@ -257,14 +257,14 @@
#define __NR_remap_file_pages 257
#define __NR_set_tid_address 258
#define __NR_timer_create 259
-#define __NR_timer_settime (__NR_timer_create+1)
-#define __NR_timer_gettime (__NR_timer_create+2)
+#define __NR_timer_settime32 (__NR_timer_create+1)
+#define __NR_timer_gettime32 (__NR_timer_create+2)
#define __NR_timer_getoverrun (__NR_timer_create+3)
#define __NR_timer_delete (__NR_timer_create+4)
-#define __NR_clock_settime (__NR_timer_create+5)
-#define __NR_clock_gettime (__NR_timer_create+6)
-#define __NR_clock_getres (__NR_timer_create+7)
-#define __NR_clock_nanosleep (__NR_timer_create+8)
+#define __NR_clock_settime32 (__NR_timer_create+5)
+#define __NR_clock_gettime32 (__NR_timer_create+6)
+#define __NR_clock_getres_time32 (__NR_timer_create+7)
+#define __NR_clock_nanosleep_time32 (__NR_timer_create+8)
#define __NR_statfs64 268
#define __NR_fstatfs64 269
#define __NR_tgkill 270
@@ -322,8 +322,8 @@
#define __NR_timerfd_create 322
#define __NR_eventfd 323
#define __NR_fallocate 324
-#define __NR_timerfd_settime 325
-#define __NR_timerfd_gettime 326
+#define __NR_timerfd_settime32 325
+#define __NR_timerfd_gettime32 326
#define __NR_signalfd4 327
#define __NR_eventfd2 328
#define __NR_epoll_create1 329
@@ -424,4 +424,9 @@
#define __NR_fsconfig 431
#define __NR_fsmount 432
#define __NR_fspick 433
+#define __NR_pidfd_open 434
+#define __NR_clone3 435
+#define __NR_openat2 437
+#define __NR_pidfd_getfd 438
+#define __NR_faccessat2 439
diff --git a/arch/i386/pthread_arch.h b/arch/i386/pthread_arch.h
index 6f600b9e..a639c382 100644
--- a/arch/i386/pthread_arch.h
+++ b/arch/i386/pthread_arch.h
@@ -1,10 +1,8 @@
-static inline struct pthread *__pthread_self()
+static inline uintptr_t __get_tp()
{
- struct pthread *self;
- __asm__ ("movl %%gs:0,%0" : "=r" (self) );
- return self;
+ uintptr_t tp;
+ __asm__ ("movl %%gs:0,%0" : "=r" (tp) );
+ return tp;
}
-#define TP_ADJ(p) (p)
-
#define MC_PC gregs[REG_EIP]
diff --git a/arch/i386/syscall_arch.h b/arch/i386/syscall_arch.h
index 69642e57..f92b7aa9 100644
--- a/arch/i386/syscall_arch.h
+++ b/arch/i386/syscall_arch.h
@@ -87,5 +87,3 @@ static inline long __syscall6(long n, long a1, long a2, long a3, long a4, long a
#define VDSO_CGT32_VER "LINUX_2.6"
#define VDSO_CGT_SYM "__vdso_clock_gettime64"
#define VDSO_CGT_VER "LINUX_2.6"
-
-#define SYSCALL_USE_SOCKETCALL
diff --git a/arch/m68k/bits/syscall.h.in b/arch/m68k/bits/syscall.h.in
index fb522b48..93703b46 100644
--- a/arch/m68k/bits/syscall.h.in
+++ b/arch/m68k/bits/syscall.h.in
@@ -67,8 +67,8 @@
#define __NR_setrlimit 75
#define __NR_getrlimit 76
#define __NR_getrusage 77
-#define __NR_gettimeofday 78
-#define __NR_settimeofday 79
+#define __NR_gettimeofday_time32 78
+#define __NR_settimeofday_time32 79
#define __NR_getgroups 80
#define __NR_setgroups 81
#define __NR_select 82
@@ -235,14 +235,14 @@
#define __NR_remap_file_pages 252
#define __NR_set_tid_address 253
#define __NR_timer_create 254
-#define __NR_timer_settime 255
-#define __NR_timer_gettime 256
+#define __NR_timer_settime32 255
+#define __NR_timer_gettime32 256
#define __NR_timer_getoverrun 257
#define __NR_timer_delete 258
-#define __NR_clock_settime 259
-#define __NR_clock_gettime 260
-#define __NR_clock_getres 261
-#define __NR_clock_nanosleep 262
+#define __NR_clock_settime32 259
+#define __NR_clock_gettime32 260
+#define __NR_clock_getres_time32 261
+#define __NR_clock_nanosleep_time32 262
#define __NR_statfs64 263
#define __NR_fstatfs64 264
#define __NR_tgkill 265
@@ -300,8 +300,8 @@
#define __NR_timerfd_create 318
#define __NR_eventfd 319
#define __NR_fallocate 320
-#define __NR_timerfd_settime 321
-#define __NR_timerfd_gettime 322
+#define __NR_timerfd_settime32 321
+#define __NR_timerfd_gettime32 322
#define __NR_signalfd4 323
#define __NR_eventfd2 324
#define __NR_epoll_create1 325
@@ -404,3 +404,8 @@
#define __NR_fsconfig 431
#define __NR_fsmount 432
#define __NR_fspick 433
+#define __NR_pidfd_open 434
+#define __NR_clone3 435
+#define __NR_openat2 437
+#define __NR_pidfd_getfd 438
+#define __NR_faccessat2 439
diff --git a/arch/m68k/pthread_arch.h b/arch/m68k/pthread_arch.h
index 02d5b8a0..5bea4e1a 100644
--- a/arch/m68k/pthread_arch.h
+++ b/arch/m68k/pthread_arch.h
@@ -1,13 +1,12 @@
-static inline struct pthread *__pthread_self()
+static inline uintptr_t __get_tp()
{
- uintptr_t tp = __syscall(SYS_get_thread_area);
- return (pthread_t)(tp - 0x7000 - sizeof(struct pthread));
+ return __syscall(SYS_get_thread_area);
}
#define TLS_ABOVE_TP
#define GAP_ABOVE_TP 0
-#define TP_ADJ(p) ((char *)(p) + sizeof(struct pthread) + 0x7000)
+#define TP_OFFSET 0x7000
#define DTP_OFFSET 0x8000
#define MC_PC gregs[R_PC]
diff --git a/arch/m68k/syscall_arch.h b/arch/m68k/syscall_arch.h
index af79c306..6a9d0ae8 100644
--- a/arch/m68k/syscall_arch.h
+++ b/arch/m68k/syscall_arch.h
@@ -87,5 +87,4 @@ static inline long __syscall6(long n, long a, long b, long c, long d, long e, lo
return d0;
}
-#define SYSCALL_USE_SOCKETCALL
#define SYSCALL_IPC_BROKEN_MODE
diff --git a/arch/microblaze/bits/syscall.h.in b/arch/microblaze/bits/syscall.h.in
index 59f86236..1e78dfde 100644
--- a/arch/microblaze/bits/syscall.h.in
+++ b/arch/microblaze/bits/syscall.h.in
@@ -76,8 +76,8 @@
#define __NR_setrlimit 75
#define __NR_getrlimit 76
#define __NR_getrusage 77
-#define __NR_gettimeofday 78
-#define __NR_settimeofday 79
+#define __NR_gettimeofday_time32 78
+#define __NR_settimeofday_time32 79
#define __NR_getgroups 80
#define __NR_setgroups 81
#define __NR_select 82
@@ -255,14 +255,14 @@
#define __NR_remap_file_pages 257
#define __NR_set_tid_address 258
#define __NR_timer_create 259
-#define __NR_timer_settime 260
-#define __NR_timer_gettime 261
+#define __NR_timer_settime32 260
+#define __NR_timer_gettime32 261
#define __NR_timer_getoverrun 262
#define __NR_timer_delete 263
-#define __NR_clock_settime 264
-#define __NR_clock_gettime 265
-#define __NR_clock_getres 266
-#define __NR_clock_nanosleep 267
+#define __NR_clock_settime32 264
+#define __NR_clock_gettime32 265
+#define __NR_clock_getres_time32 266
+#define __NR_clock_nanosleep_time32 267
#define __NR_statfs64 268
#define __NR_fstatfs64 269
#define __NR_tgkill 270
@@ -320,8 +320,8 @@
#define __NR_eventfd 323
#define __NR_fallocate 324
#define __NR_semtimedop 325
-#define __NR_timerfd_settime 326
-#define __NR_timerfd_gettime 327
+#define __NR_timerfd_settime32 326
+#define __NR_timerfd_gettime32 327
#define __NR_semctl 328
#define __NR_semget 329
#define __NR_semop 330
@@ -425,4 +425,9 @@
#define __NR_fsconfig 431
#define __NR_fsmount 432
#define __NR_fspick 433
+#define __NR_pidfd_open 434
+#define __NR_clone3 435
+#define __NR_openat2 437
+#define __NR_pidfd_getfd 438
+#define __NR_faccessat2 439
diff --git a/arch/microblaze/pthread_arch.h b/arch/microblaze/pthread_arch.h
index f6ba8de9..ff26624e 100644
--- a/arch/microblaze/pthread_arch.h
+++ b/arch/microblaze/pthread_arch.h
@@ -1,10 +1,8 @@
-static inline struct pthread *__pthread_self()
+static inline uintptr_t __get_tp()
{
- struct pthread *self;
- __asm__ ("ori %0, r21, 0" : "=r" (self) );
- return self;
+ uintptr_t tp;
+ __asm__ ("ori %0, r21, 0" : "=r" (tp) );
+ return tp;
}
-#define TP_ADJ(p) (p)
-
#define MC_PC regs.pc
diff --git a/arch/microblaze/syscall_arch.h b/arch/microblaze/syscall_arch.h
index 169013f8..61d8248e 100644
--- a/arch/microblaze/syscall_arch.h
+++ b/arch/microblaze/syscall_arch.h
@@ -95,3 +95,5 @@ static inline long __syscall6(long n, long a, long b, long c, long d, long e, lo
}
#define SYSCALL_IPC_BROKEN_MODE
+
+#undef SYS_socketcall
diff --git a/arch/mips/bits/hwcap.h b/arch/mips/bits/hwcap.h
index 13e86fe7..7986deb7 100644
--- a/arch/mips/bits/hwcap.h
+++ b/arch/mips/bits/hwcap.h
@@ -1,3 +1,14 @@
#define HWCAP_MIPS_R6 (1 << 0)
#define HWCAP_MIPS_MSA (1 << 1)
#define HWCAP_MIPS_CRC32 (1 << 2)
+#define HWCAP_MIPS_MIPS16 (1 << 3)
+#define HWCAP_MIPS_MDMX (1 << 4)
+#define HWCAP_MIPS_MIPS3D (1 << 5)
+#define HWCAP_MIPS_SMARTMIPS (1 << 6)
+#define HWCAP_MIPS_DSP (1 << 7)
+#define HWCAP_MIPS_DSP2 (1 << 8)
+#define HWCAP_MIPS_DSP3 (1 << 9)
+#define HWCAP_MIPS_MIPS16E2 (1 << 10)
+#define HWCAP_LOONGSON_MMI (1 << 11)
+#define HWCAP_LOONGSON_EXT (1 << 12)
+#define HWCAP_LOONGSON_EXT2 (1 << 13)
diff --git a/arch/mips/bits/signal.h b/arch/mips/bits/signal.h
index 1a84de59..1b69e762 100644
--- a/arch/mips/bits/signal.h
+++ b/arch/mips/bits/signal.h
@@ -19,14 +19,18 @@ typedef struct {
} fpregset_t;
struct sigcontext {
unsigned sc_regmask, sc_status;
- unsigned long long sc_pc, sc_regs[32], sc_fpregs[32];
+ unsigned long long sc_pc;
+ gregset_t sc_regs;
+ fpregset_t sc_fpregs;
unsigned sc_ownedfp, sc_fpc_csr, sc_fpc_eir, sc_used_math, sc_dsp;
unsigned long long sc_mdhi, sc_mdlo;
unsigned long sc_hi1, sc_lo1, sc_hi2, sc_lo2, sc_hi3, sc_lo3;
};
typedef struct {
unsigned regmask, status;
- unsigned long long pc, gregs[32], fpregs[32];
+ unsigned long long pc;
+ gregset_t gregs;
+ fpregset_t fpregs;
unsigned ownedfp, fpc_csr, fpc_eir, used_math, dsp;
unsigned long long mdhi, mdlo;
unsigned long hi1, lo1, hi2, lo2, hi3, lo3;
@@ -89,7 +93,7 @@ typedef struct __ucontext {
#define SIGTRAP 5
#define SIGABRT 6
#define SIGIOT SIGABRT
-#define SIGSTKFLT 7
+#define SIGEMT 7
#define SIGFPE 8
#define SIGKILL 9
#define SIGBUS 10
diff --git a/arch/mips/bits/syscall.h.in b/arch/mips/bits/syscall.h.in
index 582fa3b5..5b2066ef 100644
--- a/arch/mips/bits/syscall.h.in
+++ b/arch/mips/bits/syscall.h.in
@@ -76,8 +76,8 @@
#define __NR_setrlimit 4075
#define __NR_getrlimit 4076
#define __NR_getrusage 4077
-#define __NR_gettimeofday 4078
-#define __NR_settimeofday 4079
+#define __NR_gettimeofday_time32 4078
+#define __NR_settimeofday_time32 4079
#define __NR_getgroups 4080
#define __NR_setgroups 4081
#define __NR_reserved82 4082
@@ -256,14 +256,14 @@
#define __NR_statfs64 4255
#define __NR_fstatfs64 4256
#define __NR_timer_create 4257
-#define __NR_timer_settime 4258
-#define __NR_timer_gettime 4259
+#define __NR_timer_settime32 4258
+#define __NR_timer_gettime32 4259
#define __NR_timer_getoverrun 4260
#define __NR_timer_delete 4261
-#define __NR_clock_settime 4262
-#define __NR_clock_gettime 4263
-#define __NR_clock_getres 4264
-#define __NR_clock_nanosleep 4265
+#define __NR_clock_settime32 4262
+#define __NR_clock_gettime32 4263
+#define __NR_clock_getres_time32 4264
+#define __NR_clock_nanosleep_time32 4265
#define __NR_tgkill 4266
#define __NR_utimes 4267
#define __NR_mbind 4268
@@ -319,8 +319,8 @@
#define __NR_eventfd 4319
#define __NR_fallocate 4320
#define __NR_timerfd_create 4321
-#define __NR_timerfd_gettime 4322
-#define __NR_timerfd_settime 4323
+#define __NR_timerfd_gettime32 4322
+#define __NR_timerfd_settime32 4323
#define __NR_signalfd4 4324
#define __NR_eventfd2 4325
#define __NR_epoll_create1 4326
@@ -406,4 +406,9 @@
#define __NR_fsconfig 4431
#define __NR_fsmount 4432
#define __NR_fspick 4433
+#define __NR_pidfd_open 4434
+#define __NR_clone3 4435
+#define __NR_openat2 4437
+#define __NR_pidfd_getfd 4438
+#define __NR_faccessat2 4439
diff --git a/arch/mips/pthread_arch.h b/arch/mips/pthread_arch.h
index 1e7839ea..c45347ab 100644
--- a/arch/mips/pthread_arch.h
+++ b/arch/mips/pthread_arch.h
@@ -1,19 +1,19 @@
-static inline struct pthread *__pthread_self()
+static inline uintptr_t __get_tp()
{
#if __mips_isa_rev < 2
- register char *tp __asm__("$3");
+ register uintptr_t tp __asm__("$3");
__asm__ (".word 0x7c03e83b" : "=r" (tp) );
#else
- char *tp;
+ uintptr_t tp;
__asm__ ("rdhwr %0, $29" : "=r" (tp) );
#endif
- return (pthread_t)(tp - 0x7000 - sizeof(struct pthread));
+ return tp;
}
#define TLS_ABOVE_TP
#define GAP_ABOVE_TP 0
-#define TP_ADJ(p) ((char *)(p) + sizeof(struct pthread) + 0x7000)
+#define TP_OFFSET 0x7000
#define DTP_OFFSET 0x8000
#define MC_PC pc
diff --git a/arch/mips/syscall_arch.h b/arch/mips/syscall_arch.h
index f821e73f..5b7c38de 100644
--- a/arch/mips/syscall_arch.h
+++ b/arch/mips/syscall_arch.h
@@ -18,26 +18,26 @@
static inline long __syscall0(long n)
{
register long r7 __asm__("$7");
- register long r2 __asm__("$2") = n;
+ register long r2 __asm__("$2");
__asm__ __volatile__ (
- "syscall"
- : "+r"(r2), "=r"(r7)
- :
+ "addu $2,$0,%2 ; syscall"
+ : "=&r"(r2), "=r"(r7)
+ : "ir"(n), "0"(r2)
: SYSCALL_CLOBBERLIST, "$8", "$9", "$10");
- return r7 ? -r2 : r2;
+ return r7 && r2>0 ? -r2 : r2;
}
static inline long __syscall1(long n, long a)
{
register long r4 __asm__("$4") = a;
register long r7 __asm__("$7");
- register long r2 __asm__("$2") = n;
+ register long r2 __asm__("$2");
__asm__ __volatile__ (
- "syscall"
- : "+r"(r2), "=r"(r7)
- : "r"(r4)
+ "addu $2,$0,%2 ; syscall"
+ : "=&r"(r2), "=r"(r7)
+ : "ir"(n), "0"(r2), "r"(r4)
: SYSCALL_CLOBBERLIST, "$8", "$9", "$10");
- return r7 ? -r2 : r2;
+ return r7 && r2>0 ? -r2 : r2;
}
static inline long __syscall2(long n, long a, long b)
@@ -45,13 +45,13 @@ static inline long __syscall2(long n, long a, long b)
register long r4 __asm__("$4") = a;
register long r5 __asm__("$5") = b;
register long r7 __asm__("$7");
- register long r2 __asm__("$2") = n;
+ register long r2 __asm__("$2");
__asm__ __volatile__ (
- "syscall"
- : "+r"(r2), "=r"(r7)
- : "r"(r4), "r"(r5)
+ "addu $2,$0,%2 ; syscall"
+ : "=&r"(r2), "=r"(r7)
+ : "ir"(n), "0"(r2), "r"(r4), "r"(r5)
: SYSCALL_CLOBBERLIST, "$8", "$9", "$10");
- return r7 ? -r2 : r2;
+ return r7 && r2>0 ? -r2 : r2;
}
static inline long __syscall3(long n, long a, long b, long c)
@@ -60,13 +60,13 @@ static inline long __syscall3(long n, long a, long b, long c)
register long r5 __asm__("$5") = b;
register long r6 __asm__("$6") = c;
register long r7 __asm__("$7");
- register long r2 __asm__("$2") = n;
+ register long r2 __asm__("$2");
__asm__ __volatile__ (
- "syscall"
- : "+r"(r2), "=r"(r7)
- : "r"(r4), "r"(r5), "r"(r6)
+ "addu $2,$0,%2 ; syscall"
+ : "=&r"(r2), "=r"(r7)
+ : "ir"(n), "0"(r2), "r"(r4), "r"(r5), "r"(r6)
: SYSCALL_CLOBBERLIST, "$8", "$9", "$10");
- return r7 ? -r2 : r2;
+ return r7 && r2>0 ? -r2 : r2;
}
static inline long __syscall4(long n, long a, long b, long c, long d)
@@ -75,13 +75,13 @@ static inline long __syscall4(long n, long a, long b, long c, long d)
register long r5 __asm__("$5") = b;
register long r6 __asm__("$6") = c;
register long r7 __asm__("$7") = d;
- register long r2 __asm__("$2") = n;
+ register long r2 __asm__("$2");
__asm__ __volatile__ (
- "syscall"
- : "+r"(r2), "+r"(r7)
- : "r"(r4), "r"(r5), "r"(r6)
+ "addu $2,$0,%2 ; syscall"
+ : "=&r"(r2), "+r"(r7)
+ : "ir"(n), "0"(r2), "r"(r4), "r"(r5), "r"(r6)
: SYSCALL_CLOBBERLIST, "$8", "$9", "$10");
- return r7 ? -r2 : r2;
+ return r7 && r2>0 ? -r2 : r2;
}
static inline long __syscall5(long n, long a, long b, long c, long d, long e)
@@ -91,15 +91,15 @@ static inline long __syscall5(long n, long a, long b, long c, long d, long e)
register long r6 __asm__("$6") = c;
register long r7 __asm__("$7") = d;
register long r8 __asm__("$8") = e;
- register long r2 __asm__("$2") = n;
+ register long r2 __asm__("$2");
__asm__ __volatile__ (
"subu $sp,$sp,32 ; sw $8,16($sp) ; "
- "syscall ;"
+ "addu $2,$0,%3 ; syscall ;"
"addu $sp,$sp,32"
- : "+r"(r2), "+r"(r7), "+r"(r8)
- : "r"(r4), "r"(r5), "r"(r6)
+ : "=&r"(r2), "+r"(r7), "+r"(r8)
+ : "ir"(n), "0"(r2), "r"(r4), "r"(r5), "r"(r6)
: SYSCALL_CLOBBERLIST, "$9", "$10");
- return r7 ? -r2 : r2;
+ return r7 && r2>0 ? -r2 : r2;
}
static inline long __syscall6(long n, long a, long b, long c, long d, long e, long f)
@@ -110,15 +110,15 @@ static inline long __syscall6(long n, long a, long b, long c, long d, long e, lo
register long r7 __asm__("$7") = d;
register long r8 __asm__("$8") = e;
register long r9 __asm__("$9") = f;
- register long r2 __asm__("$2") = n;
+ register long r2 __asm__("$2");
__asm__ __volatile__ (
"subu $sp,$sp,32 ; sw $8,16($sp) ; sw $9,20($sp) ; "
- "syscall ;"
+ "addu $2,$0,%4 ; syscall ;"
"addu $sp,$sp,32"
- : "+r"(r2), "+r"(r7), "+r"(r8), "+r"(r9)
- : "r"(r4), "r"(r5), "r"(r6)
+ : "=&r"(r2), "+r"(r7), "+r"(r8), "+r"(r9)
+ : "ir"(n), "0"(r2), "r"(r4), "r"(r5), "r"(r6)
: SYSCALL_CLOBBERLIST, "$10");
- return r7 ? -r2 : r2;
+ return r7 && r2>0 ? -r2 : r2;
}
static inline long __syscall7(long n, long a, long b, long c, long d, long e, long f, long g)
@@ -130,15 +130,15 @@ static inline long __syscall7(long n, long a, long b, long c, long d, long e, lo
register long r8 __asm__("$8") = e;
register long r9 __asm__("$9") = f;
register long r10 __asm__("$10") = g;
- register long r2 __asm__("$2") = n;
+ register long r2 __asm__("$2");
__asm__ __volatile__ (
"subu $sp,$sp,32 ; sw $8,16($sp) ; sw $9,20($sp) ; sw $10,24($sp) ; "
- "syscall ;"
+ "addu $2,$0,%5 ; syscall ;"
"addu $sp,$sp,32"
- : "+r"(r2), "+r"(r7), "+r"(r8), "+r"(r9), "+r"(r10)
- : "r"(r4), "r"(r5), "r"(r6)
+ : "=&r"(r2), "+r"(r7), "+r"(r8), "+r"(r9), "+r"(r10)
+ : "ir"(n), "0"(r2), "r"(r4), "r"(r5), "r"(r6)
: SYSCALL_CLOBBERLIST);
- return r7 ? -r2 : r2;
+ return r7 && r2>0 ? -r2 : r2;
}
#define VDSO_USEFUL
@@ -149,3 +149,5 @@ static inline long __syscall7(long n, long a, long b, long c, long d, long e, lo
#define SO_SNDTIMEO_OLD 0x1005
#define SO_RCVTIMEO_OLD 0x1006
+
+#undef SYS_socketcall
diff --git a/arch/mips64/bits/fcntl.h b/arch/mips64/bits/fcntl.h
index 3bcec15e..5da1eef8 100644
--- a/arch/mips64/bits/fcntl.h
+++ b/arch/mips64/bits/fcntl.h
@@ -13,7 +13,7 @@
#define O_ASYNC 010000
#define O_DIRECT 0100000
-#define O_LARGEFILE 0
+#define O_LARGEFILE 020000
#define O_NOATIME 01000000
#define O_PATH 010000000
#define O_TMPFILE 020200000
diff --git a/arch/mips64/bits/signal.h b/arch/mips64/bits/signal.h
index c31ad07e..4f91c9fc 100644
--- a/arch/mips64/bits/signal.h
+++ b/arch/mips64/bits/signal.h
@@ -112,7 +112,7 @@ typedef struct __ucontext {
#define SIGTRAP 5
#define SIGABRT 6
#define SIGIOT SIGABRT
-#define SIGSTKFLT 7
+#define SIGEMT 7
#define SIGFPE 8
#define SIGKILL 9
#define SIGBUS 10
diff --git a/arch/mips64/bits/syscall.h.in b/arch/mips64/bits/syscall.h.in
index 34b9752e..30cb321f 100644
--- a/arch/mips64/bits/syscall.h.in
+++ b/arch/mips64/bits/syscall.h.in
@@ -336,4 +336,9 @@
#define __NR_fsconfig 5431
#define __NR_fsmount 5432
#define __NR_fspick 5433
+#define __NR_pidfd_open 5434
+#define __NR_clone3 5435
+#define __NR_openat2 5437
+#define __NR_pidfd_getfd 5438
+#define __NR_faccessat2 5439
diff --git a/arch/mips64/pthread_arch.h b/arch/mips64/pthread_arch.h
index 1e7839ea..c45347ab 100644
--- a/arch/mips64/pthread_arch.h
+++ b/arch/mips64/pthread_arch.h
@@ -1,19 +1,19 @@
-static inline struct pthread *__pthread_self()
+static inline uintptr_t __get_tp()
{
#if __mips_isa_rev < 2
- register char *tp __asm__("$3");
+ register uintptr_t tp __asm__("$3");
__asm__ (".word 0x7c03e83b" : "=r" (tp) );
#else
- char *tp;
+ uintptr_t tp;
__asm__ ("rdhwr %0, $29" : "=r" (tp) );
#endif
- return (pthread_t)(tp - 0x7000 - sizeof(struct pthread));
+ return tp;
}
#define TLS_ABOVE_TP
#define GAP_ABOVE_TP 0
-#define TP_ADJ(p) ((char *)(p) + sizeof(struct pthread) + 0x7000)
+#define TP_OFFSET 0x7000
#define DTP_OFFSET 0x8000
#define MC_PC pc
diff --git a/arch/mips64/reloc.h b/arch/mips64/reloc.h
index 7e61e225..fdb5edc9 100644
--- a/arch/mips64/reloc.h
+++ b/arch/mips64/reloc.h
@@ -27,6 +27,8 @@
#define REL_DTPOFF R_MIPS_TLS_DTPREL64
#define REL_TPOFF R_MIPS_TLS_TPREL64
+#include <endian.h>
+
#undef R_TYPE
#undef R_SYM
#undef R_INFO
diff --git a/arch/mips64/syscall_arch.h b/arch/mips64/syscall_arch.h
index 69c429b8..ae6532fc 100644
--- a/arch/mips64/syscall_arch.h
+++ b/arch/mips64/syscall_arch.h
@@ -16,26 +16,26 @@
static inline long __syscall0(long n)
{
register long r7 __asm__("$7");
- register long r2 __asm__("$2") = n;
+ register long r2 __asm__("$2");
__asm__ __volatile__ (
- "syscall"
- : "+&r"(r2), "=r"(r7)
- :
+ "daddu $2,$0,%2 ; syscall"
+ : "=&r"(r2), "=r"(r7)
+ : "ir"(n), "0"(r2)
: SYSCALL_CLOBBERLIST);
- return r7 ? -r2 : r2;
+ return r7 && r2>0 ? -r2 : r2;
}
static inline long __syscall1(long n, long a)
{
register long r4 __asm__("$4") = a;
register long r7 __asm__("$7");
- register long r2 __asm__("$2") = n;
+ register long r2 __asm__("$2");
__asm__ __volatile__ (
- "syscall"
- : "+&r"(r2), "=r"(r7)
- : "r"(r4)
+ "daddu $2,$0,%2 ; syscall"
+ : "=&r"(r2), "=r"(r7)
+ : "ir"(n), "0"(r2), "r"(r4)
: SYSCALL_CLOBBERLIST);
- return r7 ? -r2 : r2;
+ return r7 && r2>0 ? -r2 : r2;
}
static inline long __syscall2(long n, long a, long b)
@@ -43,14 +43,14 @@ static inline long __syscall2(long n, long a, long b)
register long r4 __asm__("$4") = a;
register long r5 __asm__("$5") = b;
register long r7 __asm__("$7");
- register long r2 __asm__("$2") = n;
+ register long r2 __asm__("$2");
__asm__ __volatile__ (
- "syscall"
- : "+&r"(r2), "=r"(r7)
- : "r"(r4), "r"(r5)
+ "daddu $2,$0,%2 ; syscall"
+ : "=&r"(r2), "=r"(r7)
+ : "ir"(n), "0"(r2), "r"(r4), "r"(r5)
: SYSCALL_CLOBBERLIST);
- return r7 ? -r2 : r2;
+ return r7 && r2>0 ? -r2 : r2;
}
static inline long __syscall3(long n, long a, long b, long c)
@@ -59,14 +59,14 @@ static inline long __syscall3(long n, long a, long b, long c)
register long r5 __asm__("$5") = b;
register long r6 __asm__("$6") = c;
register long r7 __asm__("$7");
- register long r2 __asm__("$2") = n;
+ register long r2 __asm__("$2");
__asm__ __volatile__ (
- "syscall"
- : "+&r"(r2), "=r"(r7)
- : "r"(r4), "r"(r5), "r"(r6)
+ "daddu $2,$0,%2 ; syscall"
+ : "=&r"(r2), "=r"(r7)
+ : "ir"(n), "0"(r2), "r"(r4), "r"(r5), "r"(r6)
: SYSCALL_CLOBBERLIST);
- return r7 ? -r2 : r2;
+ return r7 && r2>0 ? -r2 : r2;
}
static inline long __syscall4(long n, long a, long b, long c, long d)
@@ -75,14 +75,14 @@ static inline long __syscall4(long n, long a, long b, long c, long d)
register long r5 __asm__("$5") = b;
register long r6 __asm__("$6") = c;
register long r7 __asm__("$7") = d;
- register long r2 __asm__("$2") = n;
+ register long r2 __asm__("$2");
__asm__ __volatile__ (
- "syscall"
- : "+&r"(r2), "+r"(r7)
- : "r"(r4), "r"(r5), "r"(r6)
+ "daddu $2,$0,%2 ; syscall"
+ : "=&r"(r2), "+r"(r7)
+ : "ir"(n), "0"(r2), "r"(r4), "r"(r5), "r"(r6)
: SYSCALL_CLOBBERLIST);
- return r7 ? -r2 : r2;
+ return r7 && r2>0 ? -r2 : r2;
}
static inline long __syscall5(long n, long a, long b, long c, long d, long e)
@@ -92,14 +92,14 @@ static inline long __syscall5(long n, long a, long b, long c, long d, long e)
register long r6 __asm__("$6") = c;
register long r7 __asm__("$7") = d;
register long r8 __asm__("$8") = e;
- register long r2 __asm__("$2") = n;
+ register long r2 __asm__("$2");
__asm__ __volatile__ (
- "syscall"
- : "+&r"(r2), "+r"(r7)
- : "r"(r4), "r"(r5), "r"(r6), "r"(r8)
+ "daddu $2,$0,%2 ; syscall"
+ : "=&r"(r2), "+r"(r7)
+ : "ir"(n), "0"(r2), "r"(r4), "r"(r5), "r"(r6), "r"(r8)
: SYSCALL_CLOBBERLIST);
- return r7 ? -r2 : r2;
+ return r7 && r2>0 ? -r2 : r2;
}
static inline long __syscall6(long n, long a, long b, long c, long d, long e, long f)
@@ -110,14 +110,14 @@ static inline long __syscall6(long n, long a, long b, long c, long d, long e, lo
register long r7 __asm__("$7") = d;
register long r8 __asm__("$8") = e;
register long r9 __asm__("$9") = f;
- register long r2 __asm__("$2") = n;
+ register long r2 __asm__("$2");
__asm__ __volatile__ (
- "syscall"
- : "+&r"(r2), "+r"(r7)
- : "r"(r4), "r"(r5), "r"(r6), "r"(r8), "r"(r9)
+ "daddu $2,$0,%2 ; syscall"
+ : "=&r"(r2), "+r"(r7)
+ : "ir"(n), "0"(r2), "r"(r4), "r"(r5), "r"(r6), "r"(r8), "r"(r9)
: SYSCALL_CLOBBERLIST);
- return r7 ? -r2 : r2;
+ return r7 && r2>0 ? -r2 : r2;
}
#define VDSO_USEFUL
diff --git a/arch/mipsn32/bits/signal.h b/arch/mipsn32/bits/signal.h
index c31ad07e..4f91c9fc 100644
--- a/arch/mipsn32/bits/signal.h
+++ b/arch/mipsn32/bits/signal.h
@@ -112,7 +112,7 @@ typedef struct __ucontext {
#define SIGTRAP 5
#define SIGABRT 6
#define SIGIOT SIGABRT
-#define SIGSTKFLT 7
+#define SIGEMT 7
#define SIGFPE 8
#define SIGKILL 9
#define SIGBUS 10
diff --git a/arch/mipsn32/bits/syscall.h.in b/arch/mipsn32/bits/syscall.h.in
index d80cafa8..12eae034 100644
--- a/arch/mipsn32/bits/syscall.h.in
+++ b/arch/mipsn32/bits/syscall.h.in
@@ -92,7 +92,7 @@
#define __NR_fchown 6091
#define __NR_lchown 6092
#define __NR_umask 6093
-#define __NR_gettimeofday 6094
+#define __NR_gettimeofday_time32 6094
#define __NR_getrlimit 6095
#define __NR_getrusage 6096
#define __NR_sysinfo 6097
@@ -157,7 +157,7 @@
#define __NR_chroot 6156
#define __NR_sync 6157
#define __NR_acct 6158
-#define __NR_settimeofday 6159
+#define __NR_settimeofday_time32 6159
#define __NR_mount 6160
#define __NR_umount2 6161
#define __NR_swapon 6162
@@ -219,14 +219,14 @@
#define __NR_fstatfs64 6218
#define __NR_sendfile64 6219
#define __NR_timer_create 6220
-#define __NR_timer_settime 6221
-#define __NR_timer_gettime 6222
+#define __NR_timer_settime32 6221
+#define __NR_timer_gettime32 6222
#define __NR_timer_getoverrun 6223
#define __NR_timer_delete 6224
-#define __NR_clock_settime 6225
-#define __NR_clock_gettime 6226
-#define __NR_clock_getres 6227
-#define __NR_clock_nanosleep 6228
+#define __NR_clock_settime32 6225
+#define __NR_clock_gettime32 6226
+#define __NR_clock_getres_time32 6227
+#define __NR_clock_nanosleep_time32 6228
#define __NR_tgkill 6229
#define __NR_utimes 6230
#define __NR_mbind 6231
@@ -282,8 +282,8 @@
#define __NR_eventfd 6282
#define __NR_fallocate 6283
#define __NR_timerfd_create 6284
-#define __NR_timerfd_gettime 6285
-#define __NR_timerfd_settime 6286
+#define __NR_timerfd_gettime32 6285
+#define __NR_timerfd_settime32 6286
#define __NR_signalfd4 6287
#define __NR_eventfd2 6288
#define __NR_epoll_create1 6289
@@ -360,4 +360,9 @@
#define __NR_fsconfig 6431
#define __NR_fsmount 6432
#define __NR_fspick 6433
+#define __NR_pidfd_open 6434
+#define __NR_clone3 6435
+#define __NR_openat2 6437
+#define __NR_pidfd_getfd 6438
+#define __NR_faccessat2 6439
diff --git a/arch/mipsn32/pthread_arch.h b/arch/mipsn32/pthread_arch.h
index 1e7839ea..c45347ab 100644
--- a/arch/mipsn32/pthread_arch.h
+++ b/arch/mipsn32/pthread_arch.h
@@ -1,19 +1,19 @@
-static inline struct pthread *__pthread_self()
+static inline uintptr_t __get_tp()
{
#if __mips_isa_rev < 2
- register char *tp __asm__("$3");
+ register uintptr_t tp __asm__("$3");
__asm__ (".word 0x7c03e83b" : "=r" (tp) );
#else
- char *tp;
+ uintptr_t tp;
__asm__ ("rdhwr %0, $29" : "=r" (tp) );
#endif
- return (pthread_t)(tp - 0x7000 - sizeof(struct pthread));
+ return tp;
}
#define TLS_ABOVE_TP
#define GAP_ABOVE_TP 0
-#define TP_ADJ(p) ((char *)(p) + sizeof(struct pthread) + 0x7000)
+#define TP_OFFSET 0x7000
#define DTP_OFFSET 0x8000
#define MC_PC pc
diff --git a/arch/mipsn32/syscall_arch.h b/arch/mipsn32/syscall_arch.h
index c1a4b7da..c681905d 100644
--- a/arch/mipsn32/syscall_arch.h
+++ b/arch/mipsn32/syscall_arch.h
@@ -16,26 +16,26 @@
static inline long __syscall0(long n)
{
register long r7 __asm__("$7");
- register long r2 __asm__("$2") = n;
+ register long r2 __asm__("$2");
__asm__ __volatile__ (
- "syscall"
- : "+&r"(r2), "=r"(r7)
- :
+ "daddu $2,$0,%2 ; syscall"
+ : "=&r"(r2), "=r"(r7)
+ : "ir"(n), "0"(r2)
: SYSCALL_CLOBBERLIST);
- return r7 ? -r2 : r2;
+ return r7 && r2>0 ? -r2 : r2;
}
static inline long __syscall1(long n, long a)
{
register long r4 __asm__("$4") = a;
register long r7 __asm__("$7");
- register long r2 __asm__("$2") = n;
+ register long r2 __asm__("$2");
__asm__ __volatile__ (
- "syscall"
- : "+&r"(r2), "=r"(r7)
- : "r"(r4)
+ "daddu $2,$0,%2 ; syscall"
+ : "=&r"(r2), "=r"(r7)
+ : "ir"(n), "0"(r2), "r"(r4)
: SYSCALL_CLOBBERLIST);
- return r7 ? -r2 : r2;
+ return r7 && r2>0 ? -r2 : r2;
}
static inline long __syscall2(long n, long a, long b)
@@ -43,13 +43,14 @@ static inline long __syscall2(long n, long a, long b)
register long r4 __asm__("$4") = a;
register long r5 __asm__("$5") = b;
register long r7 __asm__("$7");
- register long r2 __asm__("$2") = n;
+ register long r2 __asm__("$2");
+
__asm__ __volatile__ (
- "syscall"
- : "+&r"(r2), "=r"(r7)
- : "r"(r4), "r"(r5)
+ "daddu $2,$0,%2 ; syscall"
+ : "=&r"(r2), "=r"(r7)
+ : "ir"(n), "0"(r2), "r"(r4), "r"(r5)
: SYSCALL_CLOBBERLIST);
- return r7 ? -r2 : r2;
+ return r7 && r2>0 ? -r2 : r2;
}
static inline long __syscall3(long n, long a, long b, long c)
@@ -58,13 +59,14 @@ static inline long __syscall3(long n, long a, long b, long c)
register long r5 __asm__("$5") = b;
register long r6 __asm__("$6") = c;
register long r7 __asm__("$7");
- register long r2 __asm__("$2") = n;
+ register long r2 __asm__("$2");
+
__asm__ __volatile__ (
- "syscall"
- : "+&r"(r2), "=r"(r7)
- : "r"(r4), "r"(r5), "r"(r6)
+ "daddu $2,$0,%2 ; syscall"
+ : "=&r"(r2), "=r"(r7)
+ : "ir"(n), "0"(r2), "r"(r4), "r"(r5), "r"(r6)
: SYSCALL_CLOBBERLIST);
- return r7 ? -r2 : r2;
+ return r7 && r2>0 ? -r2 : r2;
}
static inline long __syscall4(long n, long a, long b, long c, long d)
@@ -73,13 +75,14 @@ static inline long __syscall4(long n, long a, long b, long c, long d)
register long r5 __asm__("$5") = b;
register long r6 __asm__("$6") = c;
register long r7 __asm__("$7") = d;
- register long r2 __asm__("$2") = n;
+ register long r2 __asm__("$2");
+
__asm__ __volatile__ (
- "syscall"
- : "+&r"(r2), "+r"(r7)
- : "r"(r4), "r"(r5), "r"(r6)
+ "daddu $2,$0,%2 ; syscall"
+ : "=&r"(r2), "+r"(r7)
+ : "ir"(n), "0"(r2), "r"(r4), "r"(r5), "r"(r6)
: SYSCALL_CLOBBERLIST);
- return r7 ? -r2 : r2;
+ return r7 && r2>0 ? -r2 : r2;
}
static inline long __syscall5(long n, long a, long b, long c, long d, long e)
@@ -89,13 +92,14 @@ static inline long __syscall5(long n, long a, long b, long c, long d, long e)
register long r6 __asm__("$6") = c;
register long r7 __asm__("$7") = d;
register long r8 __asm__("$8") = e;
- register long r2 __asm__("$2") = n;
+ register long r2 __asm__("$2");
+
__asm__ __volatile__ (
- "syscall"
- : "+&r"(r2), "+r"(r7)
- : "r"(r4), "r"(r5), "r"(r6), "r"(r8)
+ "daddu $2,$0,%2 ; syscall"
+ : "=&r"(r2), "+r"(r7)
+ : "ir"(n), "0"(r2), "r"(r4), "r"(r5), "r"(r6), "r"(r8)
: SYSCALL_CLOBBERLIST);
- return r7 ? -r2 : r2;
+ return r7 && r2>0 ? -r2 : r2;
}
static inline long __syscall6(long n, long a, long b, long c, long d, long e, long f)
@@ -106,13 +110,14 @@ static inline long __syscall6(long n, long a, long b, long c, long d, long e, lo
register long r7 __asm__("$7") = d;
register long r8 __asm__("$8") = e;
register long r9 __asm__("$9") = f;
- register long r2 __asm__("$2") = n;
+ register long r2 __asm__("$2");
+
__asm__ __volatile__ (
- "syscall"
- : "+&r"(r2), "+r"(r7)
- : "r"(r4), "r"(r5), "r"(r6), "r"(r8), "r"(r9)
+ "daddu $2,$0,%2 ; syscall"
+ : "=&r"(r2), "+r"(r7)
+ : "ir"(n), "0"(r2), "r"(r4), "r"(r5), "r"(r6), "r"(r8), "r"(r9)
: SYSCALL_CLOBBERLIST);
- return r7 ? -r2 : r2;
+ return r7 && r2>0 ? -r2 : r2;
}
#define VDSO_USEFUL
diff --git a/arch/or1k/bits/syscall.h.in b/arch/or1k/bits/syscall.h.in
index 8ffcfae8..bc9def13 100644
--- a/arch/or1k/bits/syscall.h.in
+++ b/arch/or1k/bits/syscall.h.in
@@ -85,8 +85,8 @@
#define __NR_fdatasync 83
#define __NR_sync_file_range 84
#define __NR_timerfd_create 85
-#define __NR_timerfd_settime 86
-#define __NR_timerfd_gettime 87
+#define __NR_timerfd_settime32 86
+#define __NR_timerfd_gettime32 87
#define __NR_utimensat 88
#define __NR_acct 89
#define __NR_capget 90
@@ -107,14 +107,14 @@
#define __NR_init_module 105
#define __NR_delete_module 106
#define __NR_timer_create 107
-#define __NR_timer_gettime 108
+#define __NR_timer_gettime32 108
#define __NR_timer_getoverrun 109
-#define __NR_timer_settime 110
+#define __NR_timer_settime32 110
#define __NR_timer_delete 111
-#define __NR_clock_settime 112
-#define __NR_clock_gettime 113
-#define __NR_clock_getres 114
-#define __NR_clock_nanosleep 115
+#define __NR_clock_settime32 112
+#define __NR_clock_gettime32 113
+#define __NR_clock_getres_time32 114
+#define __NR_clock_nanosleep_time32 115
#define __NR_syslog 116
#define __NR_ptrace 117
#define __NR_sched_setparam 118
@@ -168,8 +168,8 @@
#define __NR_umask 166
#define __NR_prctl 167
#define __NR_getcpu 168
-#define __NR_gettimeofday 169
-#define __NR_settimeofday 170
+#define __NR_gettimeofday_time32 169
+#define __NR_settimeofday_time32 170
#define __NR_adjtimex 171
#define __NR_getpid 172
#define __NR_getppid 173
@@ -309,4 +309,9 @@
#define __NR_fsconfig 431
#define __NR_fsmount 432
#define __NR_fspick 433
+#define __NR_pidfd_open 434
+#define __NR_clone3 435
+#define __NR_openat2 437
+#define __NR_pidfd_getfd 438
+#define __NR_faccessat2 439
diff --git a/arch/or1k/pthread_arch.h b/arch/or1k/pthread_arch.h
index 1b806f89..f75ea7e4 100644
--- a/arch/or1k/pthread_arch.h
+++ b/arch/or1k/pthread_arch.h
@@ -1,18 +1,16 @@
-/* or1k use variant I, but with the twist that tp points to the end of TCB */
-static inline struct pthread *__pthread_self()
+static inline uintptr_t __get_tp()
{
#ifdef __clang__
- char *tp;
+ uintptr_t tp;
__asm__ ("l.ori %0, r10, 0" : "=r" (tp) );
#else
- register char *tp __asm__("r10");
+ register uintptr_t tp __asm__("r10");
__asm__ ("" : "=r" (tp) );
#endif
- return (struct pthread *) (tp - sizeof(struct pthread));
+ return tp;
}
#define TLS_ABOVE_TP
#define GAP_ABOVE_TP 0
-#define TP_ADJ(p) ((char *)(p) + sizeof(struct pthread))
#define MC_PC regs.pc
diff --git a/arch/powerpc/bits/mman.h b/arch/powerpc/bits/mman.h
index b3a675a8..95ec4358 100644
--- a/arch/powerpc/bits/mman.h
+++ b/arch/powerpc/bits/mman.h
@@ -4,7 +4,6 @@
#define MAP_NORESERVE 0x40
#undef MAP_LOCKED
#define MAP_LOCKED 0x80
-#undef MAP_SYNC
#undef MCL_CURRENT
#define MCL_CURRENT 0x2000
diff --git a/arch/powerpc/bits/syscall.h.in b/arch/powerpc/bits/syscall.h.in
index adcf63db..2d4c5dfc 100644
--- a/arch/powerpc/bits/syscall.h.in
+++ b/arch/powerpc/bits/syscall.h.in
@@ -76,8 +76,8 @@
#define __NR_setrlimit 75
#define __NR_getrlimit 76
#define __NR_getrusage 77
-#define __NR_gettimeofday 78
-#define __NR_settimeofday 79
+#define __NR_gettimeofday_time32 78
+#define __NR_settimeofday_time32 79
#define __NR_getgroups 80
#define __NR_setgroups 81
#define __NR_select 82
@@ -238,14 +238,14 @@
#define __NR_epoll_wait 238
#define __NR_remap_file_pages 239
#define __NR_timer_create 240
-#define __NR_timer_settime 241
-#define __NR_timer_gettime 242
+#define __NR_timer_settime32 241
+#define __NR_timer_gettime32 242
#define __NR_timer_getoverrun 243
#define __NR_timer_delete 244
-#define __NR_clock_settime 245
-#define __NR_clock_gettime 246
-#define __NR_clock_getres 247
-#define __NR_clock_nanosleep 248
+#define __NR_clock_settime32 245
+#define __NR_clock_gettime32 246
+#define __NR_clock_getres_time32 247
+#define __NR_clock_nanosleep_time32 248
#define __NR_swapcontext 249
#define __NR_tgkill 250
#define __NR_utimes 251
@@ -307,8 +307,8 @@
#define __NR_sync_file_range2 308
#define __NR_fallocate 309
#define __NR_subpage_prot 310
-#define __NR_timerfd_settime 311
-#define __NR_timerfd_gettime 312
+#define __NR_timerfd_settime32 311
+#define __NR_timerfd_gettime32 312
#define __NR_signalfd4 313
#define __NR_eventfd2 314
#define __NR_epoll_create1 315
@@ -413,4 +413,9 @@
#define __NR_fsconfig 431
#define __NR_fsmount 432
#define __NR_fspick 433
+#define __NR_pidfd_open 434
+#define __NR_clone3 435
+#define __NR_openat2 437
+#define __NR_pidfd_getfd 438
+#define __NR_faccessat2 439
diff --git a/arch/powerpc/pthread_arch.h b/arch/powerpc/pthread_arch.h
index ae0f28d6..42e88b07 100644
--- a/arch/powerpc/pthread_arch.h
+++ b/arch/powerpc/pthread_arch.h
@@ -1,18 +1,16 @@
-static inline struct pthread *__pthread_self()
+static inline uintptr_t __get_tp()
{
- register char *tp __asm__("r2");
+ register uintptr_t tp __asm__("r2");
__asm__ ("" : "=r" (tp) );
- return (pthread_t)(tp - 0x7000 - sizeof(struct pthread));
+ return tp;
}
#define TLS_ABOVE_TP
#define GAP_ABOVE_TP 0
-#define TP_ADJ(p) ((char *)(p) + sizeof(struct pthread) + 0x7000)
+#define TP_OFFSET 0x7000
#define DTP_OFFSET 0x8000
// the kernel calls the ip "nip", it's the first saved value after the 32
// GPRs.
#define MC_PC gregs[32]
-
-#define CANARY canary_at_end
diff --git a/arch/powerpc64/bits/mman.h b/arch/powerpc64/bits/mman.h
index b3a675a8..95ec4358 100644
--- a/arch/powerpc64/bits/mman.h
+++ b/arch/powerpc64/bits/mman.h
@@ -4,7 +4,6 @@
#define MAP_NORESERVE 0x40
#undef MAP_LOCKED
#define MAP_LOCKED 0x80
-#undef MAP_SYNC
#undef MCL_CURRENT
#define MCL_CURRENT 0x2000
diff --git a/arch/powerpc64/bits/syscall.h.in b/arch/powerpc64/bits/syscall.h.in
index 32545b39..2a5c7034 100644
--- a/arch/powerpc64/bits/syscall.h.in
+++ b/arch/powerpc64/bits/syscall.h.in
@@ -385,4 +385,9 @@
#define __NR_fsconfig 431
#define __NR_fsmount 432
#define __NR_fspick 433
+#define __NR_pidfd_open 434
+#define __NR_clone3 435
+#define __NR_openat2 437
+#define __NR_pidfd_getfd 438
+#define __NR_faccessat2 439
diff --git a/arch/powerpc64/pthread_arch.h b/arch/powerpc64/pthread_arch.h
index 79c3ecd8..1b7b9079 100644
--- a/arch/powerpc64/pthread_arch.h
+++ b/arch/powerpc64/pthread_arch.h
@@ -1,18 +1,16 @@
-static inline struct pthread *__pthread_self()
+static inline uintptr_t __get_tp()
{
- register char *tp __asm__("r13");
+ register uintptr_t tp __asm__("r13");
__asm__ ("" : "=r" (tp) );
- return (pthread_t)(tp - 0x7000 - sizeof(struct pthread));
+ return tp;
}
#define TLS_ABOVE_TP
#define GAP_ABOVE_TP 0
-#define TP_ADJ(p) ((char *)(p) + sizeof(struct pthread) + 0x7000)
+#define TP_OFFSET 0x7000
#define DTP_OFFSET 0x8000
// the kernel calls the ip "nip", it's the first saved value after the 32
// GPRs.
#define MC_PC gp_regs[32]
-
-#define CANARY canary_at_end
diff --git a/arch/riscv64/atomic_arch.h b/arch/riscv64/atomic_arch.h
index 41ad4d04..0c382588 100644
--- a/arch/riscv64/atomic_arch.h
+++ b/arch/riscv64/atomic_arch.h
@@ -15,7 +15,7 @@ static inline int a_cas(volatile int *p, int t, int s)
" bnez %1, 1b\n"
"1:"
: "=&r"(old), "=&r"(tmp)
- : "r"(p), "r"(t), "r"(s)
+ : "r"(p), "r"((long)t), "r"((long)s)
: "memory");
return old;
}
diff --git a/arch/riscv64/bits/fcntl.h b/arch/riscv64/bits/fcntl.h
deleted file mode 100644
index ecb4d18f..00000000
--- a/arch/riscv64/bits/fcntl.h
+++ /dev/null
@@ -1,38 +0,0 @@
-#define O_CREAT 0100
-#define O_EXCL 0200
-#define O_NOCTTY 0400
-#define O_TRUNC 01000
-#define O_APPEND 02000
-#define O_NONBLOCK 04000
-#define O_DSYNC 010000
-#define O_SYNC 04010000
-#define O_RSYNC 04010000
-#define O_DIRECTORY 0200000
-#define O_NOFOLLOW 0400000
-#define O_CLOEXEC 02000000
-
-#define O_ASYNC 020000
-#define O_DIRECT 040000
-#define O_LARGEFILE 0100000
-#define O_NOATIME 01000000
-#define O_PATH 010000000
-#define O_TMPFILE 020200000
-#define O_NDELAY O_NONBLOCK
-
-#define F_DUPFD 0
-#define F_GETFD 1
-#define F_SETFD 2
-#define F_GETFL 3
-#define F_SETFL 4
-#define F_GETLK 5
-#define F_SETLK 6
-#define F_SETLKW 7
-#define F_SETOWN 8
-#define F_GETOWN 9
-#define F_SETSIG 10
-#define F_GETSIG 11
-
-#define F_SETOWN_EX 15
-#define F_GETOWN_EX 16
-
-#define F_GETOWNER_UIDS 17
diff --git a/arch/riscv64/bits/reg.h b/arch/riscv64/bits/reg.h
index c800788c..2633f39d 100644
--- a/arch/riscv64/bits/reg.h
+++ b/arch/riscv64/bits/reg.h
@@ -1,8 +1,2 @@
#undef __WORDSIZE
#define __WORDSIZE 64
-#define REG_PC 0
-#define REG_RA 1
-#define REG_SP 2
-#define REG_TP 4
-#define REG_S0 8
-#define REG_A0 10
diff --git a/arch/riscv64/bits/signal.h b/arch/riscv64/bits/signal.h
index 2ff4be30..b006334f 100644
--- a/arch/riscv64/bits/signal.h
+++ b/arch/riscv64/bits/signal.h
@@ -35,6 +35,15 @@ typedef struct mcontext_t {
union __riscv_mc_fp_state __fpregs;
} mcontext_t;
+#if defined(_GNU_SOURCE)
+#define REG_PC 0
+#define REG_RA 1
+#define REG_SP 2
+#define REG_TP 4
+#define REG_S0 8
+#define REG_A0 10
+#endif
+
#if defined(_GNU_SOURCE) || defined(_BSD_SOURCE)
typedef unsigned long greg_t;
typedef unsigned long gregset_t[32];
diff --git a/arch/riscv64/bits/syscall.h.in b/arch/riscv64/bits/syscall.h.in
index 1db70cfb..439712a4 100644
--- a/arch/riscv64/bits/syscall.h.in
+++ b/arch/riscv64/bits/syscall.h.in
@@ -287,6 +287,11 @@
#define __NR_fsconfig 431
#define __NR_fsmount 432
#define __NR_fspick 433
+#define __NR_pidfd_open 434
+#define __NR_clone3 435
+#define __NR_openat2 437
+#define __NR_pidfd_getfd 438
+#define __NR_faccessat2 439
#define __NR_sysriscv __NR_arch_specific_syscall
#define __NR_riscv_flush_icache (__NR_sysriscv + 15)
diff --git a/arch/riscv64/pthread_arch.h b/arch/riscv64/pthread_arch.h
index db414b17..a20d7fba 100644
--- a/arch/riscv64/pthread_arch.h
+++ b/arch/riscv64/pthread_arch.h
@@ -1,13 +1,12 @@
-static inline struct pthread *__pthread_self()
+static inline uintptr_t __get_tp()
{
- char *tp;
+ uintptr_t tp;
__asm__ __volatile__("mv %0, tp" : "=r"(tp));
- return (void *)(tp - sizeof(struct pthread));
+ return tp;
}
#define TLS_ABOVE_TP
#define GAP_ABOVE_TP 0
-#define TP_ADJ(p) ((char *)p + sizeof(struct pthread))
#define DTP_OFFSET 0x800
diff --git a/arch/s390x/bits/syscall.h.in b/arch/s390x/bits/syscall.h.in
index c4c70474..4c04abc5 100644
--- a/arch/s390x/bits/syscall.h.in
+++ b/arch/s390x/bits/syscall.h.in
@@ -350,4 +350,9 @@
#define __NR_fsconfig 431
#define __NR_fsmount 432
#define __NR_fspick 433
+#define __NR_pidfd_open 434
+#define __NR_clone3 435
+#define __NR_openat2 437
+#define __NR_pidfd_getfd 438
+#define __NR_faccessat2 439
diff --git a/arch/s390x/pthread_arch.h b/arch/s390x/pthread_arch.h
index e2251f1f..e54fec3f 100644
--- a/arch/s390x/pthread_arch.h
+++ b/arch/s390x/pthread_arch.h
@@ -1,14 +1,12 @@
-static inline struct pthread *__pthread_self()
+static inline uintptr_t __get_tp()
{
- struct pthread *self;
+ uintptr_t tp;
__asm__ (
"ear %0, %%a0\n"
"sllg %0, %0, 32\n"
"ear %0, %%a1\n"
- : "=r"(self));
- return self;
+ : "=r"(tp));
+ return tp;
}
-#define TP_ADJ(p) (p)
-
#define MC_PC psw.addr
diff --git a/arch/s390x/syscall_arch.h b/arch/s390x/syscall_arch.h
index afb99852..83cc9a27 100644
--- a/arch/s390x/syscall_arch.h
+++ b/arch/s390x/syscall_arch.h
@@ -72,5 +72,3 @@ static inline long __syscall6(long n, long a, long b, long c, long d, long e, lo
register long r7 __asm__("r7") = f;
__asm_syscall("+r"(r2), "r"(r1), "r"(r3), "r"(r4), "r"(r5), "r"(r6), "r"(r7));
}
-
-#define SYSCALL_USE_SOCKETCALL
diff --git a/arch/sh/bits/syscall.h.in b/arch/sh/bits/syscall.h.in
index 4705ef93..3942dea2 100644
--- a/arch/sh/bits/syscall.h.in
+++ b/arch/sh/bits/syscall.h.in
@@ -67,8 +67,8 @@
#define __NR_setrlimit 75
#define __NR_getrlimit 76
#define __NR_getrusage 77
-#define __NR_gettimeofday 78
-#define __NR_settimeofday 79
+#define __NR_gettimeofday_time32 78
+#define __NR_settimeofday_time32 79
#define __NR_getgroups 80
#define __NR_setgroups 81
#define __NR_symlink 83
@@ -231,14 +231,14 @@
#define __NR_remap_file_pages 257
#define __NR_set_tid_address 258
#define __NR_timer_create 259
-#define __NR_timer_settime 260
-#define __NR_timer_gettime 261
+#define __NR_timer_settime32 260
+#define __NR_timer_gettime32 261
#define __NR_timer_getoverrun 262
#define __NR_timer_delete 263
-#define __NR_clock_settime 264
-#define __NR_clock_gettime 265
-#define __NR_clock_getres 266
-#define __NR_clock_nanosleep 267
+#define __NR_clock_settime32 264
+#define __NR_clock_gettime32 265
+#define __NR_clock_getres_time32 266
+#define __NR_clock_nanosleep_time32 267
#define __NR_statfs64 268
#define __NR_fstatfs64 269
#define __NR_tgkill 270
@@ -294,8 +294,8 @@
#define __NR_timerfd_create 322
#define __NR_eventfd 323
#define __NR_fallocate 324
-#define __NR_timerfd_settime 325
-#define __NR_timerfd_gettime 326
+#define __NR_timerfd_settime32 325
+#define __NR_timerfd_gettime32 326
#define __NR_signalfd4 327
#define __NR_eventfd2 328
#define __NR_epoll_create1 329
@@ -397,4 +397,9 @@
#define __NR_fsconfig 431
#define __NR_fsmount 432
#define __NR_fspick 433
+#define __NR_pidfd_open 434
+#define __NR_clone3 435
+#define __NR_openat2 437
+#define __NR_pidfd_getfd 438
+#define __NR_faccessat2 439
diff --git a/arch/sh/pthread_arch.h b/arch/sh/pthread_arch.h
index 3ee9c1a9..0fcf70d2 100644
--- a/arch/sh/pthread_arch.h
+++ b/arch/sh/pthread_arch.h
@@ -1,13 +1,12 @@
-static inline struct pthread *__pthread_self()
+static inline uintptr_t __get_tp()
{
- char *self;
- __asm__ ("stc gbr,%0" : "=r" (self) );
- return (struct pthread *) (self - sizeof(struct pthread));
+ uintptr_t tp;
+ __asm__ ("stc gbr,%0" : "=r" (tp) );
+ return tp;
}
#define TLS_ABOVE_TP
#define GAP_ABOVE_TP 8
-#define TP_ADJ(p) ((char *)(p) + sizeof(struct pthread))
#define MC_PC sc_pc
diff --git a/arch/x32/bits/fcntl.h b/arch/x32/bits/fcntl.h
index 1b88ad39..08627f81 100644
--- a/arch/x32/bits/fcntl.h
+++ b/arch/x32/bits/fcntl.h
@@ -13,7 +13,7 @@
#define O_ASYNC 020000
#define O_DIRECT 040000
-#define O_LARGEFILE 0
+#define O_LARGEFILE 0100000
#define O_NOATIME 01000000
#define O_PATH 010000000
#define O_TMPFILE 020200000
diff --git a/arch/x32/bits/syscall.h.in b/arch/x32/bits/syscall.h.in
index 4d72852e..e4c4bd06 100644
--- a/arch/x32/bits/syscall.h.in
+++ b/arch/x32/bits/syscall.h.in
@@ -296,6 +296,12 @@
#define __NR_fsconfig (0x40000000 + 431)
#define __NR_fsmount (0x40000000 + 432)
#define __NR_fspick (0x40000000 + 433)
+#define __NR_pidfd_open (0x40000000 + 434)
+#define __NR_clone3 (0x40000000 + 435)
+#define __NR_openat2 (0x40000000 + 437)
+#define __NR_pidfd_getfd (0x40000000 + 438)
+#define __NR_faccessat2 (0x40000000 + 439)
+
#define __NR_rt_sigaction (0x40000000 + 512)
#define __NR_rt_sigreturn (0x40000000 + 513)
diff --git a/arch/x32/pthread_arch.h b/arch/x32/pthread_arch.h
index f640a1a1..c1e7716d 100644
--- a/arch/x32/pthread_arch.h
+++ b/arch/x32/pthread_arch.h
@@ -1,14 +1,12 @@
-static inline struct pthread *__pthread_self()
+static inline uintptr_t __get_tp()
{
- struct pthread *self;
- __asm__ ("mov %%fs:0,%0" : "=r" (self) );
- return self;
+ uintptr_t tp;
+ __asm__ ("mov %%fs:0,%0" : "=r" (tp) );
+ return tp;
}
-#define TP_ADJ(p) (p)
-
#define MC_PC gregs[REG_RIP]
-#define CANARY canary2
+#define CANARY_PAD
#define tls_mod_off_t unsigned long long
diff --git a/arch/x86_64/bits/fcntl.h b/arch/x86_64/bits/fcntl.h
deleted file mode 100644
index 1b88ad39..00000000
--- a/arch/x86_64/bits/fcntl.h
+++ /dev/null
@@ -1,40 +0,0 @@
-#define O_CREAT 0100
-#define O_EXCL 0200
-#define O_NOCTTY 0400
-#define O_TRUNC 01000
-#define O_APPEND 02000
-#define O_NONBLOCK 04000
-#define O_DSYNC 010000
-#define O_SYNC 04010000
-#define O_RSYNC 04010000
-#define O_DIRECTORY 0200000
-#define O_NOFOLLOW 0400000
-#define O_CLOEXEC 02000000
-
-#define O_ASYNC 020000
-#define O_DIRECT 040000
-#define O_LARGEFILE 0
-#define O_NOATIME 01000000
-#define O_PATH 010000000
-#define O_TMPFILE 020200000
-#define O_NDELAY O_NONBLOCK
-
-#define F_DUPFD 0
-#define F_GETFD 1
-#define F_SETFD 2
-#define F_GETFL 3
-#define F_SETFL 4
-
-#define F_SETOWN 8
-#define F_GETOWN 9
-#define F_SETSIG 10
-#define F_GETSIG 11
-
-#define F_GETLK 5
-#define F_SETLK 6
-#define F_SETLKW 7
-
-#define F_SETOWN_EX 15
-#define F_GETOWN_EX 16
-
-#define F_GETOWNER_UIDS 17
diff --git a/arch/x86_64/bits/syscall.h.in b/arch/x86_64/bits/syscall.h.in
index 2d4634f6..12a86980 100644
--- a/arch/x86_64/bits/syscall.h.in
+++ b/arch/x86_64/bits/syscall.h.in
@@ -343,4 +343,9 @@
#define __NR_fsconfig 431
#define __NR_fsmount 432
#define __NR_fspick 433
+#define __NR_pidfd_open 434
+#define __NR_clone3 435
+#define __NR_openat2 437
+#define __NR_pidfd_getfd 438
+#define __NR_faccessat2 439
diff --git a/arch/x86_64/pthread_arch.h b/arch/x86_64/pthread_arch.h
index 65e880c6..c8c63f2e 100644
--- a/arch/x86_64/pthread_arch.h
+++ b/arch/x86_64/pthread_arch.h
@@ -1,10 +1,8 @@
-static inline struct pthread *__pthread_self()
+static inline uintptr_t __get_tp()
{
- struct pthread *self;
- __asm__ ("mov %%fs:0,%0" : "=r" (self) );
- return self;
+ uintptr_t tp;
+ __asm__ ("mov %%fs:0,%0" : "=r" (tp) );
+ return tp;
}
-#define TP_ADJ(p) (p)
-
#define MC_PC gregs[REG_RIP]
diff --git a/compat/time32/setitimer_time32.c b/compat/time32/setitimer_time32.c
index 4651dacb..2475fd8c 100644
--- a/compat/time32/setitimer_time32.c
+++ b/compat/time32/setitimer_time32.c
@@ -15,9 +15,11 @@ int __setitimer_time32(int which, const struct itimerval32 *restrict new32, stru
* timer setting, so we can't fail on out-of-range old value.
* Since these are relative times, values large enough to overflow
* don't make sense anyway. */
- old32->it_interval.tv_sec = old.it_interval.tv_sec;
- old32->it_interval.tv_usec = old.it_interval.tv_usec;
- old32->it_value.tv_sec = old.it_value.tv_sec;
- old32->it_value.tv_usec = old.it_value.tv_usec;
+ if (old32) {
+ old32->it_interval.tv_sec = old.it_interval.tv_sec;
+ old32->it_interval.tv_usec = old.it_interval.tv_usec;
+ old32->it_value.tv_sec = old.it_value.tv_sec;
+ old32->it_value.tv_usec = old.it_value.tv_usec;
+ }
return 0;
}
diff --git a/configure b/configure
index 86801281..947adf41 100755
--- a/configure
+++ b/configure
@@ -30,11 +30,14 @@ System types:
Optional features:
--enable-optimize=... optimize listed components for speed over size [auto]
--enable-debug build with debugging information [disabled]
- --enable-warnings build with recommended warnings flags [disabled]
+ --disable-warnings build with recommended warnings flags [enabled]
--enable-wrapper=... build given musl toolchain wrapper [auto]
--disable-shared inhibit building shared library [enabled]
--disable-static inhibit building static library [enabled]
+Optional packages:
+ --with-malloc=... choose malloc implementation [mallocng]
+
Some influential environment variables:
CC C compiler command [detected]
CFLAGS C compiler flags [-Os -pipe ...]
@@ -133,12 +136,13 @@ build=
target=
optimize=auto
debug=no
-warnings=no
+warnings=yes
shared=auto
static=yes
wrapper=auto
gcc_wrapper=no
clang_wrapper=no
+malloc_dir=mallocng
for arg ; do
case "$arg" in
@@ -168,6 +172,7 @@ case "$arg" in
--disable-wrapper|--enable-wrapper=no) wrapper=no ;;
--enable-gcc-wrapper|--enable-gcc-wrapper=yes) wrapper=yes ; gcc_wrapper=yes ;;
--disable-gcc-wrapper|--enable-gcc-wrapper=no) wrapper=no ;;
+--with-malloc=*) malloc_dir=${arg#*=} ;;
--enable-*|--disable-*|--with-*|--without-*|--*dir=*) ;;
--host=*|--target=*) target=${arg#*=} ;;
--build=*) build=${arg#*=} ;;
@@ -215,6 +220,12 @@ set +C
trap 'rm "$tmpc"' EXIT INT QUIT TERM HUP
#
+# Check that the requested malloc implementation exists
+#
+test -d "$srcdir/src/malloc/$malloc_dir" \
+|| fail "$0: error: chosen malloc implementation '$malloc_dir' does not exist"
+
+#
# Check whether we are cross-compiling, and set a default
# CROSS_COMPILE prefix if none was provided.
#
@@ -495,6 +506,16 @@ fnmatch '-mtune=*|*\ -mtune=*' "$CC $CFLAGS" || tryldflag CFLAGS_AUTO -mtune=gen
fi
#
+# GCC defines -w as overriding any -W options, regardless of order, but
+# clang has a bunch of annoying warnings enabled by default and needs -w
+# to start from a clean slate. So use -w if building with clang. Also
+# turn off a common on-by-default cast warning regardless of compiler.
+#
+test "$cc_family" = clang && tryflag CFLAGS_AUTO -w
+
+tryflag CFLAGS_AUTO -Wno-pointer-to-int-cast
+
+#
# Even with -std=c99, gcc accepts some constructs which are constraint
# violations. We want to treat these as errors regardless of whether
# other purely stylistic warnings are enabled -- especially implicit
@@ -504,6 +525,10 @@ tryflag CFLAGS_AUTO -Werror=implicit-function-declaration
tryflag CFLAGS_AUTO -Werror=implicit-int
tryflag CFLAGS_AUTO -Werror=pointer-sign
tryflag CFLAGS_AUTO -Werror=pointer-arith
+tryflag CFLAGS_AUTO -Werror=int-conversion
+tryflag CFLAGS_AUTO -Werror=incompatible-pointer-types
+tryflag CFLAGS_AUTO -Werror=discarded-qualifiers
+tryflag CFLAGS_AUTO -Werror=discarded-array-qualifiers
#
# GCC ignores unused arguements by default, but Clang needs this extra
@@ -513,14 +538,17 @@ tryflag CFLAGS_AUTO -Werror=pointer-arith
test "$cc_family" = clang && tryflag CFLAGS_AUTO -Qunused-arguments
if test "x$warnings" = xyes ; then
-tryflag CFLAGS_AUTO -Wall
-tryflag CFLAGS_AUTO -Wno-parentheses
-tryflag CFLAGS_AUTO -Wno-uninitialized
-tryflag CFLAGS_AUTO -Wno-missing-braces
-tryflag CFLAGS_AUTO -Wno-unused-value
-tryflag CFLAGS_AUTO -Wno-unused-but-set-variable
-tryflag CFLAGS_AUTO -Wno-unknown-pragmas
-tryflag CFLAGS_AUTO -Wno-pointer-to-int-cast
+tryflag CFLAGS_AUTO -Waddress
+tryflag CFLAGS_AUTO -Warray-bounds
+tryflag CFLAGS_AUTO -Wchar-subscripts
+tryflag CFLAGS_AUTO -Wduplicate-decl-specifier
+tryflag CFLAGS_AUTO -Winit-self
+tryflag CFLAGS_AUTO -Wreturn-type
+tryflag CFLAGS_AUTO -Wsequence-point
+tryflag CFLAGS_AUTO -Wstrict-aliasing
+tryflag CFLAGS_AUTO -Wunused-function
+tryflag CFLAGS_AUTO -Wunused-label
+tryflag CFLAGS_AUTO -Wunused-variable
fi
# Determine if the compiler produces position-independent code (PIC)
@@ -646,6 +674,15 @@ if test "$ARCH" = "powerpc" ; then
trycppif "__NO_FPRS__ && !_SOFT_FLOAT" "$t" && fail \
"$0: error: compiler's floating point configuration is unsupported"
trycppif _SOFT_FLOAT "$t" && SUBARCH=${SUBARCH}-sf
+printf "checking whether compiler can use 'd' constraint in asm... "
+echo 'double f(double x) { __asm__ ("fabs %0, %1" : "=d"(x) : "d"(x)); return x; }' > "$tmpc"
+if $CC $CFLAGS_C99FSE $CPPFLAGS $CFLAGS -c -o /dev/null "$tmpc" >/dev/null 2>&1 ; then
+printf "yes\n"
+else
+printf "no\n"
+CFLAGS_AUTO="$CFLAGS_AUTO -DBROKEN_PPC_D_ASM"
+CFLAGS_AUTO="${CFLAGS_AUTO# }"
+fi
fi
test "$ARCH" = "microblaze" && trycppif __MICROBLAZEEL__ "$t" \
@@ -763,6 +800,7 @@ OPTIMIZE_GLOBS = $OPTIMIZE_GLOBS
ALL_TOOLS = $tools
TOOL_LIBS = $tool_libs
ADD_CFI = $ADD_CFI
+MALLOC_DIR = $malloc_dir
EOF
test "x$static" = xno && echo "STATIC_LIBS ="
test "x$shared" = xno && echo "SHARED_LIBS ="
diff --git a/include/alloca.h b/include/alloca.h
index d2e6f1c6..b8d183d1 100644
--- a/include/alloca.h
+++ b/include/alloca.h
@@ -10,9 +10,7 @@ extern "C" {
void *alloca(size_t);
-#ifdef __GNUC__
#define alloca __builtin_alloca
-#endif
#ifdef __cplusplus
}
diff --git a/include/alltypes.h.in b/include/alltypes.h.in
index 94aa2089..d47aeea9 100644
--- a/include/alltypes.h.in
+++ b/include/alltypes.h.in
@@ -1,5 +1,6 @@
#define __LITTLE_ENDIAN 1234
#define __BIG_ENDIAN 4321
+#define __USE_TIME_BITS64 1
TYPEDEF unsigned _Addr size_t;
TYPEDEF unsigned _Addr uintptr_t;
@@ -76,6 +77,8 @@ TYPEDEF struct __sigset_t { unsigned long __bits[128/sizeof(long)]; } sigset_t;
STRUCT iovec { void *iov_base; size_t iov_len; };
+STRUCT winsize { unsigned short ws_row, ws_col, ws_xpixel, ws_ypixel; };
+
TYPEDEF unsigned socklen_t;
TYPEDEF unsigned short sa_family_t;
diff --git a/include/arpa/inet.h b/include/arpa/inet.h
index 37f8c11e..9d20a15b 100644
--- a/include/arpa/inet.h
+++ b/include/arpa/inet.h
@@ -24,11 +24,6 @@ struct in_addr inet_makeaddr(in_addr_t, in_addr_t);
in_addr_t inet_lnaof(struct in_addr);
in_addr_t inet_netof(struct in_addr);
-#undef INET_ADDRSTRLEN
-#undef INET6_ADDRSTRLEN
-#define INET_ADDRSTRLEN 16
-#define INET6_ADDRSTRLEN 46
-
#ifdef __cplusplus
}
#endif
diff --git a/include/dirent.h b/include/dirent.h
index e0a8fe6a..650ecf64 100644
--- a/include/dirent.h
+++ b/include/dirent.h
@@ -15,19 +15,9 @@ extern "C" {
#include <bits/alltypes.h>
-typedef struct __dirstream DIR;
-
-#define _DIRENT_HAVE_D_RECLEN
-#define _DIRENT_HAVE_D_OFF
-#define _DIRENT_HAVE_D_TYPE
+#include <bits/dirent.h>
-struct dirent {
- ino_t d_ino;
- off_t d_off;
- unsigned short d_reclen;
- unsigned char d_type;
- char d_name[256];
-};
+typedef struct __dirstream DIR;
#define d_fileno d_ino
diff --git a/include/elf.h b/include/elf.h
index 549f92c1..b5e7befb 100644
--- a/include/elf.h
+++ b/include/elf.h
@@ -603,6 +603,7 @@ typedef struct {
#define PT_GNU_EH_FRAME 0x6474e550
#define PT_GNU_STACK 0x6474e551
#define PT_GNU_RELRO 0x6474e552
+#define PT_GNU_PROPERTY 0x6474e553
#define PT_LOSUNW 0x6ffffffa
#define PT_SUNWBSS 0x6ffffffa
#define PT_SUNWSTACK 0x6ffffffb
@@ -1085,6 +1086,7 @@ typedef struct {
#define NT_GNU_BUILD_ID 3
#define NT_GNU_GOLD_VERSION 4
+#define NT_GNU_PROPERTY_TYPE_0 5
diff --git a/include/limits.h b/include/limits.h
index 1499eaae..53a27b9d 100644
--- a/include/limits.h
+++ b/include/limits.h
@@ -41,7 +41,9 @@
#define PIPE_BUF 4096
#define FILESIZEBITS 64
+#ifndef NAME_MAX
#define NAME_MAX 255
+#endif
#define PATH_MAX 4096
#define NGROUPS_MAX 32
#define ARG_MAX 131072
diff --git a/include/netinet/if_ether.h b/include/netinet/if_ether.h
index 8af47dbe..55a2ff1b 100644
--- a/include/netinet/if_ether.h
+++ b/include/netinet/if_ether.h
@@ -58,6 +58,8 @@
#define ETH_P_ERSPAN 0x88BE
#define ETH_P_PREAUTH 0x88C7
#define ETH_P_TIPC 0x88CA
+#define ETH_P_LLDP 0x88CC
+#define ETH_P_MRP 0x88E3
#define ETH_P_MACSEC 0x88E5
#define ETH_P_8021AH 0x88E7
#define ETH_P_MVRP 0x88F5
diff --git a/include/netinet/in.h b/include/netinet/in.h
index 5b8b21e6..36a2013a 100644
--- a/include/netinet/in.h
+++ b/include/netinet/in.h
@@ -60,8 +60,6 @@ struct ipv6_mreq {
extern const struct in6_addr in6addr_any, in6addr_loopback;
-#undef INET_ADDRSTRLEN
-#undef INET6_ADDRSTRLEN
#define INET_ADDRSTRLEN 16
#define INET6_ADDRSTRLEN 46
@@ -103,8 +101,10 @@ uint16_t ntohs(uint16_t);
#define IPPROTO_MH 135
#define IPPROTO_UDPLITE 136
#define IPPROTO_MPLS 137
+#define IPPROTO_ETHERNET 143
#define IPPROTO_RAW 255
-#define IPPROTO_MAX 256
+#define IPPROTO_MPTCP 262
+#define IPPROTO_MAX 263
#define IN6_IS_ADDR_UNSPECIFIED(a) \
(((uint32_t *) (a))[0] == 0 && ((uint32_t *) (a))[1] == 0 && \
diff --git a/include/netinet/ip.h b/include/netinet/ip.h
index 8b6d7fef..0ae132a5 100644
--- a/include/netinet/ip.h
+++ b/include/netinet/ip.h
@@ -190,6 +190,8 @@ struct ip_timestamp {
#define IP_MSS 576
+#define __UAPI_DEF_IPHDR 0
+
#ifdef __cplusplus
}
#endif
diff --git a/include/netinet/tcp.h b/include/netinet/tcp.h
index adcd45e7..b7b997f5 100644
--- a/include/netinet/tcp.h
+++ b/include/netinet/tcp.h
@@ -38,6 +38,7 @@
#define TCP_FASTOPEN_NO_COOKIE 34
#define TCP_ZEROCOPY_RECEIVE 35
#define TCP_INQ 36
+#define TCP_TX_DELAY 37
#define TCP_CM_INQ TCP_INQ
@@ -77,6 +78,8 @@ enum {
TCP_NLA_DSACK_DUPS,
TCP_NLA_REORD_SEEN,
TCP_NLA_SRTT,
+ TCP_NLA_TIMEOUT_REHASH,
+ TCP_NLA_BYTES_NOTSENT,
};
#if defined(_GNU_SOURCE) || defined(_BSD_SOURCE)
@@ -180,6 +183,13 @@ struct tcphdr {
#define TCP_CA_Recovery 3
#define TCP_CA_Loss 4
+enum tcp_fastopen_client_fail {
+ TFO_STATUS_UNSPEC,
+ TFO_COOKIE_UNAVAILABLE,
+ TFO_DATA_NOT_ACKED,
+ TFO_SYN_RETRANSMITTED,
+};
+
struct tcp_info {
uint8_t tcpi_state;
uint8_t tcpi_ca_state;
@@ -188,7 +198,7 @@ struct tcp_info {
uint8_t tcpi_backoff;
uint8_t tcpi_options;
uint8_t tcpi_snd_wscale : 4, tcpi_rcv_wscale : 4;
- uint8_t tcpi_delivery_rate_app_limited : 1;
+ uint8_t tcpi_delivery_rate_app_limited : 1, tcpi_fastopen_client_fail : 2;
uint32_t tcpi_rto;
uint32_t tcpi_ato;
uint32_t tcpi_snd_mss;
@@ -233,18 +243,21 @@ struct tcp_info {
uint64_t tcpi_bytes_retrans;
uint32_t tcpi_dsack_dups;
uint32_t tcpi_reord_seen;
+ uint32_t tcpi_rcv_ooopack;
+ uint32_t tcpi_snd_wnd;
};
#define TCP_MD5SIG_MAXKEYLEN 80
-#define TCP_MD5SIG_FLAG_PREFIX 1
+#define TCP_MD5SIG_FLAG_PREFIX 0x1
+#define TCP_MD5SIG_FLAG_IFINDEX 0x2
struct tcp_md5sig {
struct sockaddr_storage tcpm_addr;
uint8_t tcpm_flags;
uint8_t tcpm_prefixlen;
uint16_t tcpm_keylen;
- uint32_t __tcpm_pad;
+ int tcpm_ifindex;
uint8_t tcpm_key[TCP_MD5SIG_MAXKEYLEN];
};
@@ -272,6 +285,8 @@ struct tcp_zerocopy_receive {
uint64_t address;
uint32_t length;
uint32_t recv_skip_hint;
+ uint32_t inq;
+ int32_t err;
};
#endif
diff --git a/include/netinet/udp.h b/include/netinet/udp.h
index ffd89079..40c3f203 100644
--- a/include/netinet/udp.h
+++ b/include/netinet/udp.h
@@ -35,6 +35,7 @@ struct udphdr {
#define UDP_ENCAP_GTP0 4
#define UDP_ENCAP_GTP1U 5
#define UDP_ENCAP_RXRPC 6
+#define TCP_ENCAP_ESPINTCP 7
#define SOL_UDP 17
diff --git a/include/pthread.h b/include/pthread.h
index 984db680..0492f26a 100644
--- a/include/pthread.h
+++ b/include/pthread.h
@@ -74,6 +74,9 @@ extern "C" {
#define PTHREAD_BARRIER_SERIAL_THREAD (-1)
+#define PTHREAD_NULL ((pthread_t)0)
+
+
int pthread_create(pthread_t *__restrict, const pthread_attr_t *__restrict, void *(*)(void *), void *__restrict);
int pthread_detach(pthread_t);
_Noreturn void pthread_exit(void *);
diff --git a/include/sched.h b/include/sched.h
index 822f464e..fda4b484 100644
--- a/include/sched.h
+++ b/include/sched.h
@@ -49,6 +49,7 @@ int sched_yield(void);
#ifdef _GNU_SOURCE
#define CSIGNAL 0x000000ff
+#define CLONE_NEWTIME 0x00000080
#define CLONE_VM 0x00000100
#define CLONE_FS 0x00000200
#define CLONE_FILES 0x00000400
diff --git a/include/sys/fanotify.h b/include/sys/fanotify.h
index b637c8f5..75766790 100644
--- a/include/sys/fanotify.h
+++ b/include/sys/fanotify.h
@@ -55,8 +55,9 @@ struct fanotify_response {
#define FAN_OPEN_PERM 0x10000
#define FAN_ACCESS_PERM 0x20000
#define FAN_OPEN_EXEC_PERM 0x40000
-#define FAN_ONDIR 0x40000000
+#define FAN_DIR_MODIFY 0x00080000
#define FAN_EVENT_ON_CHILD 0x08000000
+#define FAN_ONDIR 0x40000000
#define FAN_CLOSE (FAN_CLOSE_WRITE | FAN_CLOSE_NOWRITE)
#define FAN_MOVE (FAN_MOVED_FROM | FAN_MOVED_TO)
#define FAN_CLOEXEC 0x01
@@ -88,6 +89,7 @@ struct fanotify_response {
#define FAN_ALL_OUTGOING_EVENTS (FAN_ALL_EVENTS | FAN_ALL_PERM_EVENTS | FAN_Q_OVERFLOW)
#define FANOTIFY_METADATA_VERSION 3
#define FAN_EVENT_INFO_TYPE_FID 1
+#define FAN_EVENT_INFO_TYPE_DFID_NAME 2
#define FAN_ALLOW 0x01
#define FAN_DENY 0x02
#define FAN_AUDIT 0x10
diff --git a/include/sys/ioctl.h b/include/sys/ioctl.h
index 372e3ddc..a9a2346e 100644
--- a/include/sys/ioctl.h
+++ b/include/sys/ioctl.h
@@ -4,6 +4,9 @@
extern "C" {
#endif
+#define __NEED_struct_winsize
+
+#include <bits/alltypes.h>
#include <bits/ioctl.h>
#define N_TTY 0
@@ -46,13 +49,6 @@ extern "C" {
#define TIOCSER_TEMT 1
-struct winsize {
- unsigned short ws_row;
- unsigned short ws_col;
- unsigned short ws_xpixel;
- unsigned short ws_ypixel;
-};
-
#define SIOCADDRT 0x890B
#define SIOCDELRT 0x890C
#define SIOCRTMSG 0x890D
diff --git a/include/sys/mman.h b/include/sys/mman.h
index d0761b18..4d603e91 100644
--- a/include/sys/mman.h
+++ b/include/sys/mman.h
@@ -92,6 +92,8 @@ extern "C" {
#define MADV_DODUMP 17
#define MADV_WIPEONFORK 18
#define MADV_KEEPONFORK 19
+#define MADV_COLD 20
+#define MADV_PAGEOUT 21
#define MADV_HWPOISON 100
#define MADV_SOFT_OFFLINE 101
#endif
@@ -99,6 +101,7 @@ extern "C" {
#ifdef _GNU_SOURCE
#define MREMAP_MAYMOVE 1
#define MREMAP_FIXED 2
+#define MREMAP_DONTUNMAP 4
#define MLOCK_ONFAULT 0x01
diff --git a/include/sys/prctl.h b/include/sys/prctl.h
index 07f0d73d..4b9fcc05 100644
--- a/include/sys/prctl.h
+++ b/include/sys/prctl.h
@@ -154,6 +154,13 @@ struct prctl_mm_map {
#define PR_PAC_APDBKEY (1UL << 3)
#define PR_PAC_APGAKEY (1UL << 4)
+#define PR_SET_TAGGED_ADDR_CTRL 55
+#define PR_GET_TAGGED_ADDR_CTRL 56
+#define PR_TAGGED_ADDR_ENABLE (1UL << 0)
+
+#define PR_SET_IO_FLUSHER 57
+#define PR_GET_IO_FLUSHER 58
+
int prctl (int, ...);
#ifdef __cplusplus
diff --git a/include/sys/procfs.h b/include/sys/procfs.h
index e23bf1ad..38e58c16 100644
--- a/include/sys/procfs.h
+++ b/include/sys/procfs.h
@@ -23,10 +23,9 @@ struct elf_prstatus {
pid_t pr_ppid;
pid_t pr_pgrp;
pid_t pr_sid;
- struct timeval pr_utime;
- struct timeval pr_stime;
- struct timeval pr_cutime;
- struct timeval pr_cstime;
+ struct {
+ long tv_sec, tv_usec;
+ } pr_utime, pr_stime, pr_cutime, pr_cstime;
elf_gregset_t pr_reg;
int pr_fpvalid;
};
diff --git a/include/sys/ptrace.h b/include/sys/ptrace.h
index 229e1f3d..5d62a985 100644
--- a/include/sys/ptrace.h
+++ b/include/sys/ptrace.h
@@ -41,6 +41,7 @@ extern "C" {
#define PTRACE_SETSIGMASK 0x420b
#define PTRACE_SECCOMP_GET_FILTER 0x420c
#define PTRACE_SECCOMP_GET_METADATA 0x420d
+#define PTRACE_GET_SYSCALL_INFO 0x420e
#define PT_READ_I PTRACE_PEEKTEXT
#define PT_READ_D PTRACE_PEEKDATA
@@ -88,6 +89,11 @@ extern "C" {
#define PTRACE_PEEKSIGINFO_SHARED 1
+#define PTRACE_SYSCALL_INFO_NONE 0
+#define PTRACE_SYSCALL_INFO_ENTRY 1
+#define PTRACE_SYSCALL_INFO_EXIT 2
+#define PTRACE_SYSCALL_INFO_SECCOMP 3
+
#include <bits/ptrace.h>
struct __ptrace_peeksiginfo_args {
@@ -101,6 +107,29 @@ struct __ptrace_seccomp_metadata {
uint64_t flags;
};
+struct __ptrace_syscall_info {
+ uint8_t op;
+ uint8_t __pad[3];
+ uint32_t arch;
+ uint64_t instruction_pointer;
+ uint64_t stack_pointer;
+ union {
+ struct {
+ uint64_t nr;
+ uint64_t args[6];
+ } entry;
+ struct {
+ int64_t rval;
+ uint8_t is_error;
+ } exit;
+ struct {
+ uint64_t nr;
+ uint64_t args[6];
+ uint32_t ret_data;
+ } seccomp;
+ };
+};
+
long ptrace(int, ...);
#ifdef __cplusplus
diff --git a/include/sys/random.h b/include/sys/random.h
index 4ee7bf2c..59e40ab8 100644
--- a/include/sys/random.h
+++ b/include/sys/random.h
@@ -10,6 +10,7 @@ extern "C" {
#define GRND_NONBLOCK 0x0001
#define GRND_RANDOM 0x0002
+#define GRND_INSECURE 0x0004
ssize_t getrandom(void *, size_t, unsigned);
diff --git a/include/sys/resource.h b/include/sys/resource.h
index e0c86ae3..3068328d 100644
--- a/include/sys/resource.h
+++ b/include/sys/resource.h
@@ -90,7 +90,8 @@ int prlimit(pid_t, int, const struct rlimit *, struct rlimit *);
#define RLIMIT_MSGQUEUE 12
#define RLIMIT_NICE 13
#define RLIMIT_RTPRIO 14
-#define RLIMIT_NLIMITS 15
+#define RLIMIT_RTTIME 15
+#define RLIMIT_NLIMITS 16
#define RLIM_NLIMITS RLIMIT_NLIMITS
diff --git a/include/sys/socket.h b/include/sys/socket.h
index a1c0b01c..38f5bb17 100644
--- a/include/sys/socket.h
+++ b/include/sys/socket.h
@@ -288,6 +288,7 @@ struct linger {
#define SO_TXTIME 61
#define SCM_TXTIME SO_TXTIME
#define SO_BINDTOIFINDEX 62
+#define SO_DETACH_REUSEPORT_BPF 68
#ifndef SOL_SOCKET
#define SOL_SOCKET 1
diff --git a/include/sys/ttydefaults.h b/include/sys/ttydefaults.h
index d251b715..edb55bc4 100644
--- a/include/sys/ttydefaults.h
+++ b/include/sys/ttydefaults.h
@@ -6,16 +6,11 @@
#define TTYDEF_LFLAG (ECHO | ICANON | ISIG | IEXTEN | ECHOE|ECHOKE|ECHOCTL)
#define TTYDEF_CFLAG (CREAD | CS7 | PARENB | HUPCL)
#define TTYDEF_SPEED (B9600)
-#define CTRL(x) (x&037)
+#define CTRL(x) ((x)&037)
#define CEOF CTRL('d')
-#ifdef _POSIX_VDISABLE
-#define CEOL _POSIX_VDISABLE
-#define CSTATUS _POSIX_VDISABLE
-#else
#define CEOL '\0'
#define CSTATUS '\0'
-#endif
#define CERASE 0177
#define CINTR CTRL('c')
diff --git a/include/sys/wait.h b/include/sys/wait.h
index d9adbdec..d4b1f2e1 100644
--- a/include/sys/wait.h
+++ b/include/sys/wait.h
@@ -13,7 +13,8 @@ extern "C" {
typedef enum {
P_ALL = 0,
P_PID = 1,
- P_PGID = 2
+ P_PGID = 2,
+ P_PIDFD = 3
} idtype_t;
pid_t wait (int *);
diff --git a/include/termios.h b/include/termios.h
index d73c780d..cbb53301 100644
--- a/include/termios.h
+++ b/include/termios.h
@@ -8,6 +8,7 @@ extern "C" {
#include <features.h>
#define __NEED_pid_t
+#define __NEED_struct_winsize
#include <bits/alltypes.h>
@@ -27,6 +28,9 @@ int cfsetispeed (struct termios *, speed_t);
int tcgetattr (int, struct termios *);
int tcsetattr (int, int, const struct termios *);
+int tcgetwinsize (int, struct winsize *);
+int tcsetwinsize (int, const struct winsize *);
+
int tcsendbreak (int, int);
int tcdrain (int);
int tcflush (int, int);
diff --git a/include/unistd.h b/include/unistd.h
index 7bcbff94..07584a23 100644
--- a/include/unistd.h
+++ b/include/unistd.h
@@ -190,6 +190,7 @@ int syncfs(int);
int euidaccess(const char *, int);
int eaccess(const char *, int);
ssize_t copy_file_range(int, off_t *, int, off_t *, size_t, unsigned);
+pid_t gettid(void);
#endif
#if defined(_LARGEFILE64_SOURCE) || defined(_GNU_SOURCE)
diff --git a/include/utmpx.h b/include/utmpx.h
index 0429014d..b293f427 100644
--- a/include/utmpx.h
+++ b/include/utmpx.h
@@ -16,6 +16,7 @@ extern "C" {
struct utmpx {
short ut_type;
+ short __ut_pad1;
pid_t ut_pid;
char ut_line[32];
char ut_id[4];
@@ -25,7 +26,11 @@ struct utmpx {
short __e_termination;
short __e_exit;
} ut_exit;
- long ut_session;
+#if __BYTE_ORDER == 1234
+ int ut_session, __ut_pad2;
+#else
+ int __ut_pad2, ut_session;
+#endif
struct timeval ut_tv;
unsigned ut_addr_v6[4];
char __unused[20];
diff --git a/ldso/dynlink.c b/ldso/dynlink.c
index 7810356b..f7474743 100644
--- a/ldso/dynlink.c
+++ b/ldso/dynlink.c
@@ -23,7 +23,6 @@
#include "pthread_impl.h"
#include "libc.h"
#include "dynlink.h"
-#include "malloc_impl.h"
static void error(const char *, ...);
@@ -107,6 +106,8 @@ struct symdef {
struct dso *dso;
};
+typedef void (*stage3_func)(size_t *, size_t *);
+
static struct builtin_tls {
char c;
struct pthread pt;
@@ -183,8 +184,14 @@ static void *laddr_pg(const struct dso *p, size_t v)
}
return (void *)(v - p->loadmap->segs[j].p_vaddr + p->loadmap->segs[j].addr);
}
-#define fpaddr(p, v) ((void (*)())&(struct funcdesc){ \
- laddr(p, v), (p)->got })
+static void (*fdbarrier(void *p))()
+{
+ void (*fd)();
+ __asm__("" : "=r"(fd) : "0"(p));
+ return fd;
+}
+#define fpaddr(p, v) fdbarrier((&(struct funcdesc){ \
+ laddr(p, v), (p)->got }))
#else
#define laddr(p, v) (void *)((p)->base + (v))
#define laddr_pg(p, v) laddr(p, v)
@@ -407,8 +414,6 @@ static void do_relocs(struct dso *dso, size_t *rel, size_t rel_size, size_t stri
}
switch(type) {
- case REL_NONE:
- break;
case REL_OFFSET:
addend -= (size_t)reloc_addr;
case REL_SYMBOLIC:
@@ -1574,7 +1579,7 @@ static void install_new_tls(void)
/* Install new dtv for each thread. */
for (j=0, td=self; !j || td!=self; j++, td=td->next) {
- td->dtv = td->dtv_copy = newdtv[j];
+ td->dtv = newdtv[j];
}
__tl_unlock();
@@ -1594,13 +1599,14 @@ static void install_new_tls(void)
hidden void __dls2(unsigned char *base, size_t *sp)
{
+ size_t *auxv;
+ for (auxv=sp+1+*sp+1; *auxv; auxv++);
+ auxv++;
if (DL_FDPIC) {
void *p1 = (void *)sp[-2];
void *p2 = (void *)sp[-1];
if (!p1) {
- size_t *auxv, aux[AUX_CNT];
- for (auxv=sp+1+*sp+1; *auxv; auxv++);
- auxv++;
+ size_t aux[AUX_CNT];
decode_vec(auxv, aux, AUX_CNT);
if (aux[AT_BASE]) ldso.base = (void *)aux[AT_BASE];
else ldso.base = (void *)(aux[AT_PHDR] & -4096);
@@ -1646,8 +1652,8 @@ hidden void __dls2(unsigned char *base, size_t *sp)
* symbolically as a barrier against moving the address
* load across the above relocation processing. */
struct symdef dls2b_def = find_sym(&ldso, "__dls2b", 0);
- if (DL_FDPIC) ((stage3_func)&ldso.funcdescs[dls2b_def.sym-ldso.syms])(sp);
- else ((stage3_func)laddr(&ldso, dls2b_def.sym->st_value))(sp);
+ if (DL_FDPIC) ((stage3_func)&ldso.funcdescs[dls2b_def.sym-ldso.syms])(sp, auxv);
+ else ((stage3_func)laddr(&ldso, dls2b_def.sym->st_value))(sp, auxv);
}
/* Stage 2b sets up a valid thread pointer, which requires relocations
@@ -1656,11 +1662,13 @@ hidden void __dls2(unsigned char *base, size_t *sp)
* so that loads of the thread pointer and &errno can be pure/const and
* thereby hoistable. */
-void __dls2b(size_t *sp)
+void __dls2b(size_t *sp, size_t *auxv)
{
/* Setup early thread pointer in builtin_tls for ldso/libc itself to
* use during dynamic linking. If possible it will also serve as the
* thread pointer at runtime. */
+ search_vec(auxv, &__hwcap, AT_HWCAP);
+ libc.auxv = auxv;
libc.tls_size = sizeof builtin_tls;
libc.tls_align = tls_align;
if (__init_tp(__copy_tls((void *)builtin_tls)) < 0) {
@@ -1668,8 +1676,8 @@ void __dls2b(size_t *sp)
}
struct symdef dls3_def = find_sym(&ldso, "__dls3", 0);
- if (DL_FDPIC) ((stage3_func)&ldso.funcdescs[dls3_def.sym-ldso.syms])(sp);
- else ((stage3_func)laddr(&ldso, dls3_def.sym->st_value))(sp);
+ if (DL_FDPIC) ((stage3_func)&ldso.funcdescs[dls3_def.sym-ldso.syms])(sp, auxv);
+ else ((stage3_func)laddr(&ldso, dls3_def.sym->st_value))(sp, auxv);
}
/* Stage 3 of the dynamic linker is called with the dynamic linker/libc
@@ -1677,10 +1685,10 @@ void __dls2b(size_t *sp)
* process dependencies and relocations for the main application and
* transfer control to its entry point. */
-void __dls3(size_t *sp)
+void __dls3(size_t *sp, size_t *auxv)
{
static struct dso app, vdso;
- size_t aux[AUX_CNT], *auxv;
+ size_t aux[AUX_CNT];
size_t i;
char *env_preload=0;
char *replace_argv0=0;
@@ -1693,10 +1701,7 @@ void __dls3(size_t *sp)
/* Find aux vector just past environ[] and use it to initialize
* global data that may be needed before we can make syscalls. */
__environ = envp;
- for (i=argc+1; argv[i]; i++);
- libc.auxv = auxv = (void *)(argv+i+1);
decode_vec(auxv, aux, AUX_CNT);
- __hwcap = aux[AT_HWCAP];
search_vec(auxv, &__sysinfo, AT_SYSINFO);
__pthread_self()->sysinfo = __sysinfo;
libc.page_size = aux[AT_PAGESZ];
@@ -1930,6 +1935,8 @@ void __dls3(size_t *sp)
* possibility of incomplete replacement. */
if (find_sym(head, "malloc", 1).dso != &ldso)
__malloc_replaced = 1;
+ if (find_sym(head, "aligned_alloc", 1).dso != &ldso)
+ __aligned_alloc_replaced = 1;
/* Switch to runtime mode: any further failures in the dynamic
* linker are a reportable failure rather than a fatal startup
diff --git a/src/complex/catanf.c b/src/complex/catanf.c
index e10d9c09..ef3907a5 100644
--- a/src/complex/catanf.c
+++ b/src/complex/catanf.c
@@ -87,29 +87,17 @@ float complex catanf(float complex z)
x = crealf(z);
y = cimagf(z);
- if ((x == 0.0f) && (y > 1.0f))
- goto ovrf;
-
x2 = x * x;
a = 1.0f - x2 - (y * y);
- if (a == 0.0f)
- goto ovrf;
t = 0.5f * atan2f(2.0f * x, a);
w = _redupif(t);
t = y - 1.0f;
a = x2 + (t * t);
- if (a == 0.0f)
- goto ovrf;
t = y + 1.0f;
a = (x2 + (t * t))/a;
- w = w + (0.25f * logf (a)) * I;
- return w;
-
-ovrf:
- // FIXME
- w = MAXNUMF + MAXNUMF * I;
+ w = CMPLXF(w, 0.25f * logf(a));
return w;
}
diff --git a/src/complex/catanl.c b/src/complex/catanl.c
index a9fc02db..e62526c0 100644
--- a/src/complex/catanl.c
+++ b/src/complex/catanl.c
@@ -97,30 +97,18 @@ long double complex catanl(long double complex z)
x = creall(z);
y = cimagl(z);
- if ((x == 0.0L) && (y > 1.0L))
- goto ovrf;
-
x2 = x * x;
a = 1.0L - x2 - (y * y);
- if (a == 0.0L)
- goto ovrf;
t = atan2l(2.0L * x, a) * 0.5L;
w = redupil(t);
t = y - 1.0L;
a = x2 + (t * t);
- if (a == 0.0L)
- goto ovrf;
t = y + 1.0L;
a = (x2 + (t * t)) / a;
- w = w + (0.25L * logl(a)) * I;
- return w;
-
-ovrf:
- // FIXME
- w = LDBL_MAX + LDBL_MAX * I;
+ w = CMPLXF(w, 0.25L * logl(a));
return w;
}
#endif
diff --git a/src/ctype/wcwidth.c b/src/ctype/wcwidth.c
index 49c40eea..36256a53 100644
--- a/src/ctype/wcwidth.c
+++ b/src/ctype/wcwidth.c
@@ -23,7 +23,7 @@ int wcwidth(wchar_t wc)
return -1;
if (wc-0x20000U < 0x20000)
return 2;
- if (wc == 0xe0001 || wc-0xe0020U < 0x5f || wc-0xe0100 < 0xef)
+ if (wc == 0xe0001 || wc-0xe0020U < 0x5f || wc-0xe0100U < 0xef)
return 0;
return 1;
}
diff --git a/src/env/__init_tls.c b/src/env/__init_tls.c
index 772baba3..a93141ed 100644
--- a/src/env/__init_tls.c
+++ b/src/env/__init_tls.c
@@ -67,7 +67,7 @@ void *__copy_tls(unsigned char *mem)
}
#endif
dtv[0] = libc.tls_cnt;
- td->dtv = td->dtv_copy = dtv;
+ td->dtv = dtv;
return td;
}
diff --git a/src/env/__stack_chk_fail.c b/src/env/__stack_chk_fail.c
index e32596d1..bf5a280a 100644
--- a/src/env/__stack_chk_fail.c
+++ b/src/env/__stack_chk_fail.c
@@ -9,7 +9,7 @@ void __init_ssp(void *entropy)
if (entropy) memcpy(&__stack_chk_guard, entropy, sizeof(uintptr_t));
else __stack_chk_guard = (uintptr_t)&__stack_chk_guard * 1103515245;
- __pthread_self()->CANARY = __stack_chk_guard;
+ __pthread_self()->canary = __stack_chk_guard;
}
void __stack_chk_fail(void)
diff --git a/src/errno/__strerror.h b/src/errno/__strerror.h
index 2f04d400..2d992da5 100644
--- a/src/errno/__strerror.h
+++ b/src/errno/__strerror.h
@@ -1,8 +1,9 @@
-/* This file is sorted such that 'errors' which represent exceptional
- * conditions under which a correct program may fail come first, followed
- * by messages that indicate an incorrect program or system failure. The
- * macro E() along with double-inclusion is used to ensure that ordering
- * of the strings remains synchronized. */
+/* The first entry is a catch-all for codes not enumerated here.
+ * This file is included multiple times to declare and define a structure
+ * with these messages, and then to define a lookup table translating
+ * error codes to offsets of corresponding fields in the structure. */
+
+E(0, "No error information")
E(EILSEQ, "Illegal byte sequence")
E(EDOM, "Domain error")
@@ -101,5 +102,3 @@ E(EDQUOT, "Quota exceeded")
E(ENOMEDIUM, "No medium found")
E(EMEDIUMTYPE, "Wrong medium type")
E(EMULTIHOP, "Multihop attempted")
-
-E(0, "No error information")
diff --git a/src/errno/strerror.c b/src/errno/strerror.c
index e3ed771a..7f926432 100644
--- a/src/errno/strerror.c
+++ b/src/errno/strerror.c
@@ -1,30 +1,41 @@
#include <errno.h>
+#include <stddef.h>
#include <string.h>
#include "locale_impl.h"
-#define E(a,b) ((unsigned char)a),
-static const unsigned char errid[] = {
+/* mips has one error code outside of the 8-bit range due to a
+ * historical typo, so we just remap it. */
+#if EDQUOT==1133
+#define EDQUOT_ORIG 1133
+#undef EDQUOT
+#define EDQUOT 109
+#endif
+
+static const struct errmsgstr_t {
+#define E(n, s) char str##n[sizeof(s)];
+#include "__strerror.h"
+#undef E
+} errmsgstr = {
+#define E(n, s) s,
#include "__strerror.h"
+#undef E
};
-#undef E
-#define E(a,b) b "\0"
-static const char errmsg[] =
+static const unsigned short errmsgidx[] = {
+#define E(n, s) [n] = offsetof(struct errmsgstr_t, str##n),
#include "__strerror.h"
-;
+#undef E
+};
char *__strerror_l(int e, locale_t loc)
{
const char *s;
- int i;
- /* mips has one error code outside of the 8-bit range due to a
- * historical typo, so we just remap it. */
- if (EDQUOT==1133) {
- if (e==109) e=-1;
- else if (e==EDQUOT) e=109;
- }
- for (i=0; errid[i] && errid[i] != e; i++);
- for (s=errmsg; i; s++, i--) for (; *s; s++);
+#ifdef EDQUOT_ORIG
+ if (e==EDQUOT) e=0;
+ else if (e==EDQUOT_ORIG) e=EDQUOT;
+#endif
+ if (e >= sizeof errmsgidx / sizeof *errmsgidx) e = 0;
+ s = (char *)&errmsgstr + errmsgidx[e];
return (char *)LCTRANS(s, LC_MESSAGES, loc);
}
diff --git a/src/fenv/riscv64/fenv.S b/src/fenv/riscv64/fenv.S
index f149003d..0ea78bf9 100644
--- a/src/fenv/riscv64/fenv.S
+++ b/src/fenv/riscv64/fenv.S
@@ -45,8 +45,11 @@ fegetenv:
.global fesetenv
.type fesetenv, %function
fesetenv:
+ li t2, -1
+ li t1, 0
+ beq a0, t2, 1f
lw t1, 0(a0)
- fscsr t0, t1
+1: fscsr t1
li a0, 0
ret
diff --git a/src/fenv/sh/fenv.S b/src/fenv/sh/fenv.S
index 907aefc0..b3b7d66a 100644
--- a/src/fenv/sh/fenv.S
+++ b/src/fenv/sh/fenv.S
@@ -12,6 +12,8 @@ fegetround:
.type __fesetround, @function
__fesetround:
sts fpscr, r0
+ mov #-4, r1
+ and r1, r0
or r4, r0
lds r0, fpscr
rts
diff --git a/src/internal/atomic.h b/src/internal/atomic.h
index f938879b..96c1552d 100644
--- a/src/internal/atomic.h
+++ b/src/internal/atomic.h
@@ -315,4 +315,19 @@ static inline int a_clz_64(uint64_t x)
}
#endif
+#ifndef a_clz_32
+#define a_clz_32 a_clz_32
+static inline int a_clz_32(uint32_t x)
+{
+ x >>= 1;
+ x |= x >> 1;
+ x |= x >> 2;
+ x |= x >> 4;
+ x |= x >> 8;
+ x |= x >> 16;
+ x++;
+ return 31-a_ctz_32(x);
+}
+#endif
+
#endif
diff --git a/src/internal/dynlink.h b/src/internal/dynlink.h
index ffd06b04..51c0639f 100644
--- a/src/internal/dynlink.h
+++ b/src/internal/dynlink.h
@@ -96,7 +96,6 @@ struct fdpic_dummy_loadmap {
#define DYN_CNT 32
typedef void (*stage2_func)(unsigned char *, size_t *);
-typedef void (*stage3_func)(size_t *);
hidden void *__dlsym(void *restrict, const char *restrict, void *restrict);
@@ -106,4 +105,9 @@ hidden void __dl_vseterr(const char *, va_list);
hidden ptrdiff_t __tlsdesc_static(), __tlsdesc_dynamic();
+hidden extern int __malloc_replaced;
+hidden extern int __aligned_alloc_replaced;
+hidden void __malloc_donate(char *, char *);
+hidden int __malloc_allzerop(void *);
+
#endif
diff --git a/src/internal/libc.h b/src/internal/libc.h
index ac97dc7e..619bba86 100644
--- a/src/internal/libc.h
+++ b/src/internal/libc.h
@@ -18,10 +18,11 @@ struct tls_module {
};
struct __libc {
- int can_do_threads;
- int threaded;
- int secure;
- volatile int threads_minus_1;
+ char can_do_threads;
+ char threaded;
+ char secure;
+ volatile signed char need_locks;
+ int threads_minus_1;
size_t *auxv;
struct tls_module *tls_head;
size_t tls_size, tls_align, tls_cnt;
diff --git a/src/internal/libm.h b/src/internal/libm.h
index b5bd26b8..72ad17d8 100644
--- a/src/internal/libm.h
+++ b/src/internal/libm.h
@@ -236,13 +236,13 @@ hidden int __rem_pio2(double,double*);
hidden double __sin(double,double,int);
hidden double __cos(double,double);
hidden double __tan(double,double,int);
-hidden double __expo2(double);
+hidden double __expo2(double,double);
hidden int __rem_pio2f(float,double*);
hidden float __sindf(double);
hidden float __cosdf(double);
hidden float __tandf(double,int);
-hidden float __expo2f(float);
+hidden float __expo2f(float,float);
hidden int __rem_pio2l(long double, long double *);
hidden long double __sinl(long double, long double, int);
@@ -267,5 +267,8 @@ hidden double __math_uflow(uint32_t);
hidden double __math_oflow(uint32_t);
hidden double __math_divzero(uint32_t);
hidden double __math_invalid(double);
+#if LDBL_MANT_DIG != DBL_MANT_DIG
+hidden long double __math_invalidl(long double);
+#endif
#endif
diff --git a/src/internal/pthread_impl.h b/src/internal/pthread_impl.h
index 5742dfc5..4d709bbc 100644
--- a/src/internal/pthread_impl.h
+++ b/src/internal/pthread_impl.h
@@ -11,16 +11,25 @@
#include "atomic.h"
#include "futex.h"
+#include "pthread_arch.h"
+
#define pthread __pthread
struct pthread {
/* Part 1 -- these fields may be external or
* internal (accessed via asm) ABI. Do not change. */
struct pthread *self;
+#ifndef TLS_ABOVE_TP
uintptr_t *dtv;
+#endif
struct pthread *prev, *next; /* non-ABI */
uintptr_t sysinfo;
- uintptr_t canary, canary2;
+#ifndef TLS_ABOVE_TP
+#ifdef CANARY_PAD
+ uintptr_t canary_pad;
+#endif
+ uintptr_t canary;
+#endif
/* Part 2 -- implementation details, non-ABI. */
int tid;
@@ -43,6 +52,7 @@ struct pthread {
long off;
volatile void *volatile pending;
} robust_list;
+ int h_errno_val;
volatile int timer_id;
locale_t locale;
volatile int killlock[1];
@@ -51,8 +61,10 @@ struct pthread {
/* Part 3 -- the positions of these fields relative to
* the end of the structure is external and internal ABI. */
- uintptr_t canary_at_end;
- uintptr_t *dtv_copy;
+#ifdef TLS_ABOVE_TP
+ uintptr_t canary;
+ uintptr_t *dtv;
+#endif
};
enum {
@@ -98,16 +110,22 @@ struct __timer {
#define _b_waiters2 __u.__vi[4]
#define _b_inst __u.__p[3]
-#include "pthread_arch.h"
-
-#ifndef CANARY
-#define CANARY canary
+#ifndef TP_OFFSET
+#define TP_OFFSET 0
#endif
#ifndef DTP_OFFSET
#define DTP_OFFSET 0
#endif
+#ifdef TLS_ABOVE_TP
+#define TP_ADJ(p) ((char *)(p) + sizeof(struct pthread) + TP_OFFSET)
+#define __pthread_self() ((pthread_t)(__get_tp() - sizeof(struct __pthread) - TP_OFFSET))
+#else
+#define TP_ADJ(p) (p)
+#define __pthread_self() ((pthread_t)__get_tp())
+#endif
+
#ifndef tls_mod_off_t
#define tls_mod_off_t size_t
#endif
diff --git a/src/internal/shgetc.c b/src/internal/shgetc.c
index a4a9c633..7455d2f0 100644
--- a/src/internal/shgetc.c
+++ b/src/internal/shgetc.c
@@ -32,6 +32,6 @@ int __shgetc(FILE *f)
else
f->shend = f->rend;
f->shcnt = f->buf - f->rpos + cnt;
- if (f->rpos[-1] != c) f->rpos[-1] = c;
+ if (f->rpos <= f->buf) f->rpos[-1] = c;
return c;
}
diff --git a/src/internal/stdio_impl.h b/src/internal/stdio_impl.h
index d7398f59..0b2438d6 100644
--- a/src/internal/stdio_impl.h
+++ b/src/internal/stdio_impl.h
@@ -60,8 +60,6 @@ hidden size_t __stdout_write(FILE *, const unsigned char *, size_t);
hidden off_t __stdio_seek(FILE *, off_t, int);
hidden int __stdio_close(FILE *);
-hidden size_t __string_read(FILE *, unsigned char *, size_t);
-
hidden int __toread(FILE *);
hidden int __towrite(FILE *);
diff --git a/src/internal/syscall.h b/src/internal/syscall.h
index 9f2784db..d5f294d4 100644
--- a/src/internal/syscall.h
+++ b/src/internal/syscall.h
@@ -2,6 +2,7 @@
#define _INTERNAL_SYSCALL_H
#include <features.h>
+#include <errno.h>
#include <sys/syscall.h>
#include "syscall_arch.h"
@@ -57,15 +58,22 @@ hidden long __syscall_ret(unsigned long),
#define __syscall_cp(...) __SYSCALL_DISP(__syscall_cp,__VA_ARGS__)
#define syscall_cp(...) __syscall_ret(__syscall_cp(__VA_ARGS__))
-#ifndef SYSCALL_USE_SOCKETCALL
-#define __socketcall(nm,a,b,c,d,e,f) __syscall(SYS_##nm, a, b, c, d, e, f)
-#define __socketcall_cp(nm,a,b,c,d,e,f) __syscall_cp(SYS_##nm, a, b, c, d, e, f)
-#else
-#define __socketcall(nm,a,b,c,d,e,f) __syscall(SYS_socketcall, __SC_##nm, \
- ((long [6]){ (long)a, (long)b, (long)c, (long)d, (long)e, (long)f }))
-#define __socketcall_cp(nm,a,b,c,d,e,f) __syscall_cp(SYS_socketcall, __SC_##nm, \
- ((long [6]){ (long)a, (long)b, (long)c, (long)d, (long)e, (long)f }))
-#endif
+static inline long __alt_socketcall(int sys, int sock, int cp, long a, long b, long c, long d, long e, long f)
+{
+ long r;
+ if (cp) r = __syscall_cp(sys, a, b, c, d, e, f);
+ else r = __syscall(sys, a, b, c, d, e, f);
+ if (r != -ENOSYS) return r;
+#ifdef SYS_socketcall
+ if (cp) r = __syscall_cp(SYS_socketcall, sock, ((long[6]){a, b, c, d, e, f}));
+ else r = __syscall(SYS_socketcall, sock, ((long[6]){a, b, c, d, e, f}));
+#endif
+ return r;
+}
+#define __socketcall(nm, a, b, c, d, e, f) __alt_socketcall(SYS_##nm, __SC_##nm, 0, \
+ (long)(a), (long)(b), (long)(c), (long)(d), (long)(e), (long)(f))
+#define __socketcall_cp(nm, a, b, c, d, e, f) __alt_socketcall(SYS_##nm, __SC_##nm, 1, \
+ (long)(a), (long)(b), (long)(c), (long)(d), (long)(e), (long)(f))
/* fixup legacy 16-bit junk */
@@ -193,6 +201,45 @@ hidden long __syscall_ret(unsigned long),
#define SYS_sendfile SYS_sendfile64
#endif
+#ifndef SYS_timer_settime
+#define SYS_timer_settime SYS_timer_settime32
+#endif
+
+#ifndef SYS_timer_gettime
+#define SYS_timer_gettime SYS_timer_gettime32
+#endif
+
+#ifndef SYS_timerfd_settime
+#define SYS_timerfd_settime SYS_timerfd_settime32
+#endif
+
+#ifndef SYS_timerfd_gettime
+#define SYS_timerfd_gettime SYS_timerfd_gettime32
+#endif
+
+#ifndef SYS_clock_settime
+#define SYS_clock_settime SYS_clock_settime32
+#endif
+
+#ifndef SYS_clock_gettime
+#define SYS_clock_gettime SYS_clock_gettime32
+#endif
+
+#ifndef SYS_clock_getres
+#define SYS_clock_getres SYS_clock_getres_time32
+#endif
+
+#ifndef SYS_clock_nanosleep
+#define SYS_clock_nanosleep SYS_clock_nanosleep_time32
+#endif
+
+#ifndef SYS_gettimeofday
+#define SYS_gettimeofday SYS_gettimeofday_time32
+#endif
+
+#ifndef SYS_settimeofday
+#define SYS_settimeofday SYS_settimeofday_time32
+#endif
/* Ensure that the plain syscall names are defined even for "time64-only"
* archs. These facilitate callers passing null time arguments, and make
@@ -299,6 +346,12 @@ hidden long __syscall_ret(unsigned long),
#define __SC_recvmmsg 19
#define __SC_sendmmsg 20
+/* This is valid only because all socket syscalls are made via
+ * socketcall, which always fills unused argument slots with zeros. */
+#ifndef SYS_accept
+#define SYS_accept SYS_accept4
+#endif
+
#ifndef SO_RCVTIMEO_OLD
#define SO_RCVTIMEO_OLD 20
#endif
@@ -306,6 +359,13 @@ hidden long __syscall_ret(unsigned long),
#define SO_SNDTIMEO_OLD 21
#endif
+#define SO_TIMESTAMP_OLD 29
+#define SO_TIMESTAMPNS_OLD 35
+#define SO_TIMESTAMPING_OLD 37
+#define SCM_TIMESTAMP_OLD SO_TIMESTAMP_OLD
+#define SCM_TIMESTAMPNS_OLD SO_TIMESTAMPNS_OLD
+#define SCM_TIMESTAMPING_OLD SO_TIMESTAMPING_OLD
+
#ifndef SIOCGSTAMP_OLD
#define SIOCGSTAMP_OLD 0x8906
#endif
diff --git a/src/ipc/msgctl.c b/src/ipc/msgctl.c
index b043041a..9c114406 100644
--- a/src/ipc/msgctl.c
+++ b/src/ipc/msgctl.c
@@ -9,6 +9,14 @@
int msgctl(int q, int cmd, struct msqid_ds *buf)
{
+#if IPC_TIME64
+ struct msqid_ds out, *orig;
+ if (cmd&IPC_TIME64) {
+ out = (struct msqid_ds){0};
+ orig = buf;
+ buf = &out;
+ }
+#endif
#ifdef SYSCALL_IPC_BROKEN_MODE
struct msqid_ds tmp;
if (cmd == IPC_SET) {
@@ -32,6 +40,8 @@ int msgctl(int q, int cmd, struct msqid_ds *buf)
#endif
#if IPC_TIME64
if (r >= 0 && (cmd&IPC_TIME64)) {
+ buf = orig;
+ *buf = out;
IPC_HILO(buf, msg_stime);
IPC_HILO(buf, msg_rtime);
IPC_HILO(buf, msg_ctime);
diff --git a/src/ipc/semctl.c b/src/ipc/semctl.c
index ed982747..bbb97d7a 100644
--- a/src/ipc/semctl.c
+++ b/src/ipc/semctl.c
@@ -28,6 +28,14 @@ int semctl(int id, int num, int cmd, ...)
arg = va_arg(ap, union semun);
va_end(ap);
}
+#if IPC_TIME64
+ struct semid_ds out, *orig;
+ if (cmd&IPC_TIME64) {
+ out = (struct semid_ds){0};
+ orig = arg.buf;
+ arg.buf = &out;
+ }
+#endif
#ifdef SYSCALL_IPC_BROKEN_MODE
struct semid_ds tmp;
if (cmd == IPC_SET) {
@@ -51,6 +59,8 @@ int semctl(int id, int num, int cmd, ...)
#endif
#if IPC_TIME64
if (r >= 0 && (cmd&IPC_TIME64)) {
+ arg.buf = orig;
+ *arg.buf = out;
IPC_HILO(arg.buf, sem_otime);
IPC_HILO(arg.buf, sem_ctime);
}
diff --git a/src/ipc/shmctl.c b/src/ipc/shmctl.c
index de3ce9d4..1c9f78c2 100644
--- a/src/ipc/shmctl.c
+++ b/src/ipc/shmctl.c
@@ -9,6 +9,14 @@
int shmctl(int id, int cmd, struct shmid_ds *buf)
{
+#if IPC_TIME64
+ struct shmid_ds out, *orig;
+ if (cmd&IPC_TIME64) {
+ out = (struct shmid_ds){0};
+ orig = buf;
+ buf = &out;
+ }
+#endif
#ifdef SYSCALL_IPC_BROKEN_MODE
struct shmid_ds tmp;
if (cmd == IPC_SET) {
@@ -32,6 +40,8 @@ int shmctl(int id, int cmd, struct shmid_ds *buf)
#endif
#if IPC_TIME64
if (r >= 0 && (cmd&IPC_TIME64)) {
+ buf = orig;
+ *buf = out;
IPC_HILO(buf, shm_atime);
IPC_HILO(buf, shm_dtime);
IPC_HILO(buf, shm_ctime);
diff --git a/src/linux/clock_adjtime.c b/src/linux/clock_adjtime.c
index 23eb8729..d4d03d24 100644
--- a/src/linux/clock_adjtime.c
+++ b/src/linux/clock_adjtime.c
@@ -38,55 +38,52 @@ int clock_adjtime (clockid_t clock_id, struct timex *utx)
{
int r = -ENOSYS;
#ifdef SYS_clock_adjtime64
- if (SYS_clock_adjtime == SYS_clock_adjtime64 ||
- (utx->modes & ADJ_SETOFFSET) && !IS32BIT(utx->time.tv_sec)) {
- struct ktimex64 ktx = {
- .modes = utx->modes,
- .offset = utx->offset,
- .freq = utx->freq,
- .maxerror = utx->maxerror,
- .esterror = utx->esterror,
- .status = utx->status,
- .constant = utx->constant,
- .precision = utx->precision,
- .tolerance = utx->tolerance,
- .time_sec = utx->time.tv_sec,
- .time_usec = utx->time.tv_usec,
- .tick = utx->tick,
- .ppsfreq = utx->ppsfreq,
- .jitter = utx->jitter,
- .shift = utx->shift,
- .stabil = utx->stabil,
- .jitcnt = utx->jitcnt,
- .calcnt = utx->calcnt,
- .errcnt = utx->errcnt,
- .stbcnt = utx->stbcnt,
- .tai = utx->tai,
- };
- r = __syscall(SYS_clock_adjtime, clock_id, &ktx);
- if (r>=0) {
- utx->modes = ktx.modes;
- utx->offset = ktx.offset;
- utx->freq = ktx.freq;
- utx->maxerror = ktx.maxerror;
- utx->esterror = ktx.esterror;
- utx->status = ktx.status;
- utx->constant = ktx.constant;
- utx->precision = ktx.precision;
- utx->tolerance = ktx.tolerance;
- utx->time.tv_sec = ktx.time_sec;
- utx->time.tv_usec = ktx.time_usec;
- utx->tick = ktx.tick;
- utx->ppsfreq = ktx.ppsfreq;
- utx->jitter = ktx.jitter;
- utx->shift = ktx.shift;
- utx->stabil = ktx.stabil;
- utx->jitcnt = ktx.jitcnt;
- utx->calcnt = ktx.calcnt;
- utx->errcnt = ktx.errcnt;
- utx->stbcnt = ktx.stbcnt;
- utx->tai = ktx.tai;
- }
+ struct ktimex64 ktx = {
+ .modes = utx->modes,
+ .offset = utx->offset,
+ .freq = utx->freq,
+ .maxerror = utx->maxerror,
+ .esterror = utx->esterror,
+ .status = utx->status,
+ .constant = utx->constant,
+ .precision = utx->precision,
+ .tolerance = utx->tolerance,
+ .time_sec = utx->time.tv_sec,
+ .time_usec = utx->time.tv_usec,
+ .tick = utx->tick,
+ .ppsfreq = utx->ppsfreq,
+ .jitter = utx->jitter,
+ .shift = utx->shift,
+ .stabil = utx->stabil,
+ .jitcnt = utx->jitcnt,
+ .calcnt = utx->calcnt,
+ .errcnt = utx->errcnt,
+ .stbcnt = utx->stbcnt,
+ .tai = utx->tai,
+ };
+ r = __syscall(SYS_clock_adjtime64, clock_id, &ktx);
+ if (r>=0) {
+ utx->modes = ktx.modes;
+ utx->offset = ktx.offset;
+ utx->freq = ktx.freq;
+ utx->maxerror = ktx.maxerror;
+ utx->esterror = ktx.esterror;
+ utx->status = ktx.status;
+ utx->constant = ktx.constant;
+ utx->precision = ktx.precision;
+ utx->tolerance = ktx.tolerance;
+ utx->time.tv_sec = ktx.time_sec;
+ utx->time.tv_usec = ktx.time_usec;
+ utx->tick = ktx.tick;
+ utx->ppsfreq = ktx.ppsfreq;
+ utx->jitter = ktx.jitter;
+ utx->shift = ktx.shift;
+ utx->stabil = ktx.stabil;
+ utx->jitcnt = ktx.jitcnt;
+ utx->calcnt = ktx.calcnt;
+ utx->errcnt = ktx.errcnt;
+ utx->stbcnt = ktx.stbcnt;
+ utx->tai = ktx.tai;
}
if (SYS_clock_adjtime == SYS_clock_adjtime64 || r!=-ENOSYS)
return __syscall_ret(r);
diff --git a/src/linux/gettid.c b/src/linux/gettid.c
new file mode 100644
index 00000000..70767137
--- /dev/null
+++ b/src/linux/gettid.c
@@ -0,0 +1,8 @@
+#define _GNU_SOURCE
+#include <unistd.h>
+#include "pthread_impl.h"
+
+pid_t gettid(void)
+{
+ return __pthread_self()->tid;
+}
diff --git a/src/locale/locale_map.c b/src/locale/locale_map.c
index 2321bac0..e7eede62 100644
--- a/src/locale/locale_map.c
+++ b/src/locale/locale_map.c
@@ -67,7 +67,7 @@ const struct __locale_map *__get_locale(int cat, const char *val)
if (path) for (; *path; path=z+!!*z) {
z = __strchrnul(path, ':');
- l = z - path - !!*z;
+ l = z - path;
if (l >= sizeof buf - n - 2) continue;
memcpy(buf, path, l);
buf[l] = '/';
diff --git a/src/malloc/DESIGN b/src/malloc/DESIGN
deleted file mode 100644
index 58b0523f..00000000
--- a/src/malloc/DESIGN
+++ /dev/null
@@ -1,22 +0,0 @@
-
-
-In principle, this memory allocator is roughly equivalent to Doug
-Lea's dlmalloc with fine-grained locking.
-
-
-
-malloc:
-
-Uses a freelist binned by chunk size, with a bitmap to optimize
-searching for the smallest non-empty bin which can satisfy an
-allocation. If no free chunks are available, it creates a new chunk of
-the requested size and attempts to merge it with any existing free
-chunk immediately below the newly created chunk.
-
-Whether the chunk was obtained from a bin or newly created, it's
-likely to be larger than the requested allocation. malloc always
-finishes its work by passing the new chunk to realloc, which will
-split it into two chunks and free the tail portion.
-
-
-
diff --git a/src/malloc/aligned_alloc.c b/src/malloc/aligned_alloc.c
deleted file mode 100644
index b6143f30..00000000
--- a/src/malloc/aligned_alloc.c
+++ /dev/null
@@ -1,7 +0,0 @@
-#include <stdlib.h>
-#include "malloc_impl.h"
-
-void *aligned_alloc(size_t align, size_t len)
-{
- return __memalign(align, len);
-}
diff --git a/src/malloc/calloc.c b/src/malloc/calloc.c
new file mode 100644
index 00000000..bf6bddca
--- /dev/null
+++ b/src/malloc/calloc.c
@@ -0,0 +1,45 @@
+#include <stdlib.h>
+#include <stdint.h>
+#include <string.h>
+#include <errno.h>
+#include "dynlink.h"
+
+static size_t mal0_clear(char *p, size_t n)
+{
+ const size_t pagesz = 4096; /* arbitrary */
+ if (n < pagesz) return n;
+#ifdef __GNUC__
+ typedef uint64_t __attribute__((__may_alias__)) T;
+#else
+ typedef unsigned char T;
+#endif
+ char *pp = p + n;
+ size_t i = (uintptr_t)pp & (pagesz - 1);
+ for (;;) {
+ pp = memset(pp - i, 0, i);
+ if (pp - p < pagesz) return pp - p;
+ for (i = pagesz; i; i -= 2*sizeof(T), pp -= 2*sizeof(T))
+ if (((T *)pp)[-1] | ((T *)pp)[-2])
+ break;
+ }
+}
+
+static int allzerop(void *p)
+{
+ return 0;
+}
+weak_alias(allzerop, __malloc_allzerop);
+
+void *calloc(size_t m, size_t n)
+{
+ if (n && m > (size_t)-1/n) {
+ errno = ENOMEM;
+ return 0;
+ }
+ n *= m;
+ void *p = malloc(n);
+ if (!p || (!__malloc_replaced && __malloc_allzerop(p)))
+ return p;
+ n = mal0_clear(p, n);
+ return memset(p, 0, n);
+}
diff --git a/src/malloc/expand_heap.c b/src/malloc/expand_heap.c
deleted file mode 100644
index e6a3d7a0..00000000
--- a/src/malloc/expand_heap.c
+++ /dev/null
@@ -1,71 +0,0 @@
-#include <limits.h>
-#include <stdint.h>
-#include <errno.h>
-#include <sys/mman.h>
-#include "libc.h"
-#include "syscall.h"
-#include "malloc_impl.h"
-
-/* This function returns true if the interval [old,new]
- * intersects the 'len'-sized interval below &libc.auxv
- * (interpreted as the main-thread stack) or below &b
- * (the current stack). It is used to defend against
- * buggy brk implementations that can cross the stack. */
-
-static int traverses_stack_p(uintptr_t old, uintptr_t new)
-{
- const uintptr_t len = 8<<20;
- uintptr_t a, b;
-
- b = (uintptr_t)libc.auxv;
- a = b > len ? b-len : 0;
- if (new>a && old<b) return 1;
-
- b = (uintptr_t)&b;
- a = b > len ? b-len : 0;
- if (new>a && old<b) return 1;
-
- return 0;
-}
-
-/* Expand the heap in-place if brk can be used, or otherwise via mmap,
- * using an exponential lower bound on growth by mmap to make
- * fragmentation asymptotically irrelevant. The size argument is both
- * an input and an output, since the caller needs to know the size
- * allocated, which will be larger than requested due to page alignment
- * and mmap minimum size rules. The caller is responsible for locking
- * to prevent concurrent calls. */
-
-void *__expand_heap(size_t *pn)
-{
- static uintptr_t brk;
- static unsigned mmap_step;
- size_t n = *pn;
-
- if (n > SIZE_MAX/2 - PAGE_SIZE) {
- errno = ENOMEM;
- return 0;
- }
- n += -n & PAGE_SIZE-1;
-
- if (!brk) {
- brk = __syscall(SYS_brk, 0);
- brk += -brk & PAGE_SIZE-1;
- }
-
- if (n < SIZE_MAX-brk && !traverses_stack_p(brk, brk+n)
- && __syscall(SYS_brk, brk+n)==brk+n) {
- *pn = n;
- brk += n;
- return (void *)(brk-n);
- }
-
- size_t min = (size_t)PAGE_SIZE << mmap_step/2;
- if (n < min) n = min;
- void *area = __mmap(0, n, PROT_READ|PROT_WRITE,
- MAP_PRIVATE|MAP_ANONYMOUS, -1, 0);
- if (area == MAP_FAILED) return 0;
- *pn = n;
- mmap_step++;
- return area;
-}
diff --git a/src/malloc/lite_malloc.c b/src/malloc/lite_malloc.c
index 050d84f6..f8931ba5 100644
--- a/src/malloc/lite_malloc.c
+++ b/src/malloc/lite_malloc.c
@@ -2,58 +2,102 @@
#include <stdint.h>
#include <limits.h>
#include <errno.h>
+#include <sys/mman.h>
+#include "libc.h"
#include "lock.h"
-#include "malloc_impl.h"
+#include "syscall.h"
#define ALIGN 16
+/* This function returns true if the interval [old,new]
+ * intersects the 'len'-sized interval below &libc.auxv
+ * (interpreted as the main-thread stack) or below &b
+ * (the current stack). It is used to defend against
+ * buggy brk implementations that can cross the stack. */
+
+static int traverses_stack_p(uintptr_t old, uintptr_t new)
+{
+ const uintptr_t len = 8<<20;
+ uintptr_t a, b;
+
+ b = (uintptr_t)libc.auxv;
+ a = b > len ? b-len : 0;
+ if (new>a && old<b) return 1;
+
+ b = (uintptr_t)&b;
+ a = b > len ? b-len : 0;
+ if (new>a && old<b) return 1;
+
+ return 0;
+}
+
static void *__simple_malloc(size_t n)
{
- static char *cur, *end;
+ static uintptr_t brk, cur, end;
static volatile int lock[1];
- size_t align=1, pad;
+ static unsigned mmap_step;
+ size_t align=1;
void *p;
+ if (n > SIZE_MAX/2) {
+ errno = ENOMEM;
+ return 0;
+ }
+
if (!n) n++;
while (align<n && align<ALIGN)
align += align;
LOCK(lock);
- pad = -(uintptr_t)cur & align-1;
-
- if (n <= SIZE_MAX/2 + ALIGN) n += pad;
+ cur += -cur & align-1;
if (n > end-cur) {
- size_t m = n;
- char *new = __expand_heap(&m);
- if (!new) {
- UNLOCK(lock);
- return 0;
+ size_t req = n - (end-cur) + PAGE_SIZE-1 & -PAGE_SIZE;
+
+ if (!cur) {
+ brk = __syscall(SYS_brk, 0);
+ brk += -brk & PAGE_SIZE-1;
+ cur = end = brk;
}
- if (new != end) {
- cur = new;
- n -= pad;
- pad = 0;
+
+ if (brk == end && req < SIZE_MAX-brk
+ && !traverses_stack_p(brk, brk+req)
+ && __syscall(SYS_brk, brk+req)==brk+req) {
+ brk = end += req;
+ } else {
+ int new_area = 0;
+ req = n + PAGE_SIZE-1 & -PAGE_SIZE;
+ /* Only make a new area rather than individual mmap
+ * if wasted space would be over 1/8 of the map. */
+ if (req-n > req/8) {
+ /* Geometric area size growth up to 64 pages,
+ * bounding waste by 1/8 of the area. */
+ size_t min = PAGE_SIZE<<(mmap_step/2);
+ if (min-n > end-cur) {
+ if (req < min) {
+ req = min;
+ if (mmap_step < 12)
+ mmap_step++;
+ }
+ new_area = 1;
+ }
+ }
+ void *mem = __mmap(0, req, PROT_READ|PROT_WRITE,
+ MAP_PRIVATE|MAP_ANONYMOUS, -1, 0);
+ if (mem == MAP_FAILED || !new_area) {
+ UNLOCK(lock);
+ return mem==MAP_FAILED ? 0 : mem;
+ }
+ cur = (uintptr_t)mem;
+ end = cur + req;
}
- end = new + m;
}
- p = cur + pad;
+ p = (void *)cur;
cur += n;
UNLOCK(lock);
return p;
}
weak_alias(__simple_malloc, malloc);
-
-static void *__simple_calloc(size_t m, size_t n)
-{
- if (n && m > (size_t)-1/n) {
- errno = ENOMEM;
- return 0;
- }
- return __simple_malloc(n * m);
-}
-
-weak_alias(__simple_calloc, calloc);
diff --git a/src/malloc/mallocng/aligned_alloc.c b/src/malloc/mallocng/aligned_alloc.c
new file mode 100644
index 00000000..34116896
--- /dev/null
+++ b/src/malloc/mallocng/aligned_alloc.c
@@ -0,0 +1,57 @@
+#include <stdlib.h>
+#include <errno.h>
+#include "meta.h"
+
+void *aligned_alloc(size_t align, size_t len)
+{
+ if ((align & -align) != align) {
+ errno = EINVAL;
+ return 0;
+ }
+
+ if (len > SIZE_MAX - align || align >= (1ULL<<31)*UNIT) {
+ errno = ENOMEM;
+ return 0;
+ }
+
+ if (DISABLE_ALIGNED_ALLOC) {
+ errno = ENOMEM;
+ return 0;
+ }
+
+ if (align <= UNIT) align = UNIT;
+
+ unsigned char *p = malloc(len + align - UNIT);
+ struct meta *g = get_meta(p);
+ int idx = get_slot_index(p);
+ size_t stride = get_stride(g);
+ unsigned char *start = g->mem->storage + stride*idx;
+ unsigned char *end = g->mem->storage + stride*(idx+1) - IB;
+ size_t adj = -(uintptr_t)p & (align-1);
+
+ if (!adj) {
+ set_size(p, end, len);
+ return p;
+ }
+ p += adj;
+ uint32_t offset = (size_t)(p-g->mem->storage)/UNIT;
+ if (offset <= 0xffff) {
+ *(uint16_t *)(p-2) = offset;
+ p[-4] = 0;
+ } else {
+ // use a 32-bit offset if 16-bit doesn't fit. for this,
+ // 16-bit field must be zero, [-4] byte nonzero.
+ *(uint16_t *)(p-2) = 0;
+ *(uint32_t *)(p-8) = offset;
+ p[-4] = 1;
+ }
+ p[-3] = idx;
+ set_size(p, end, len);
+ // store offset to aligned enframing. this facilitates cycling
+ // offset and also iteration of heap for debugging/measurement.
+ // for extreme overalignment it won't fit but these are classless
+ // allocations anyway.
+ *(uint16_t *)(start - 2) = (size_t)(p-start)/UNIT;
+ start[-3] = 7<<5;
+ return p;
+}
diff --git a/src/malloc/mallocng/donate.c b/src/malloc/mallocng/donate.c
new file mode 100644
index 00000000..41d850f3
--- /dev/null
+++ b/src/malloc/mallocng/donate.c
@@ -0,0 +1,39 @@
+#include <stdlib.h>
+#include <stdint.h>
+#include <limits.h>
+#include <string.h>
+#include <sys/mman.h>
+#include <errno.h>
+
+#include "meta.h"
+
+static void donate(unsigned char *base, size_t len)
+{
+ uintptr_t a = (uintptr_t)base;
+ uintptr_t b = a + len;
+ a += -a & (UNIT-1);
+ b -= b & (UNIT-1);
+ memset(base, 0, len);
+ for (int sc=47; sc>0 && b>a; sc-=4) {
+ if (b-a < (size_classes[sc]+1)*UNIT) continue;
+ struct meta *m = alloc_meta();
+ m->avail_mask = 0;
+ m->freed_mask = 1;
+ m->mem = (void *)a;
+ m->mem->meta = m;
+ m->last_idx = 0;
+ m->freeable = 0;
+ m->sizeclass = sc;
+ m->maplen = 0;
+ *((unsigned char *)m->mem+UNIT-4) = 0;
+ *((unsigned char *)m->mem+UNIT-3) = 255;
+ m->mem->storage[size_classes[sc]*UNIT-4] = 0;
+ queue(&ctx.active[sc], m);
+ a += (size_classes[sc]+1)*UNIT;
+ }
+}
+
+void __malloc_donate(char *start, char *end)
+{
+ donate((void *)start, end-start);
+}
diff --git a/src/malloc/mallocng/free.c b/src/malloc/mallocng/free.c
new file mode 100644
index 00000000..40745f97
--- /dev/null
+++ b/src/malloc/mallocng/free.c
@@ -0,0 +1,143 @@
+#define _BSD_SOURCE
+#include <stdlib.h>
+#include <sys/mman.h>
+
+#include "meta.h"
+
+struct mapinfo {
+ void *base;
+ size_t len;
+};
+
+static struct mapinfo nontrivial_free(struct meta *, int);
+
+static struct mapinfo free_group(struct meta *g)
+{
+ struct mapinfo mi = { 0 };
+ int sc = g->sizeclass;
+ if (sc < 48) {
+ ctx.usage_by_class[sc] -= g->last_idx+1;
+ }
+ if (g->maplen) {
+ step_seq();
+ record_seq(sc);
+ mi.base = g->mem;
+ mi.len = g->maplen*4096UL;
+ } else {
+ void *p = g->mem;
+ struct meta *m = get_meta(p);
+ int idx = get_slot_index(p);
+ g->mem->meta = 0;
+ // not checking size/reserved here; it's intentionally invalid
+ mi = nontrivial_free(m, idx);
+ }
+ free_meta(g);
+ return mi;
+}
+
+static int okay_to_free(struct meta *g)
+{
+ int sc = g->sizeclass;
+
+ if (!g->freeable) return 0;
+
+ // always free individual mmaps not suitable for reuse
+ if (sc >= 48 || get_stride(g) < UNIT*size_classes[sc])
+ return 1;
+
+ // always free groups allocated inside another group's slot
+ // since recreating them should not be expensive and they
+ // might be blocking freeing of a much larger group.
+ if (!g->maplen) return 1;
+
+ // if there is another non-full group, free this one to
+ // consolidate future allocations, reduce fragmentation.
+ if (g->next != g) return 1;
+
+ // free any group in a size class that's not bouncing
+ if (!is_bouncing(sc)) return 1;
+
+ size_t cnt = g->last_idx+1;
+ size_t usage = ctx.usage_by_class[sc];
+
+ // if usage is high enough that a larger count should be
+ // used, free the low-count group so a new one will be made.
+ if (9*cnt <= usage && cnt < 20)
+ return 1;
+
+ // otherwise, keep the last group in a bouncing class.
+ return 0;
+}
+
+static struct mapinfo nontrivial_free(struct meta *g, int i)
+{
+ uint32_t self = 1u<<i;
+ int sc = g->sizeclass;
+ uint32_t mask = g->freed_mask | g->avail_mask;
+
+ if (mask+self == (2u<<g->last_idx)-1 && okay_to_free(g)) {
+ // any multi-slot group is necessarily on an active list
+ // here, but single-slot groups might or might not be.
+ if (g->next) {
+ assert(sc < 48);
+ int activate_new = (ctx.active[sc]==g);
+ dequeue(&ctx.active[sc], g);
+ if (activate_new && ctx.active[sc])
+ activate_group(ctx.active[sc]);
+ }
+ return free_group(g);
+ } else if (!mask) {
+ assert(sc < 48);
+ // might still be active if there were no allocations
+ // after last available slot was taken.
+ if (ctx.active[sc] != g) {
+ queue(&ctx.active[sc], g);
+ }
+ }
+ a_or(&g->freed_mask, self);
+ return (struct mapinfo){ 0 };
+}
+
+void free(void *p)
+{
+ if (!p) return;
+
+ struct meta *g = get_meta(p);
+ int idx = get_slot_index(p);
+ size_t stride = get_stride(g);
+ unsigned char *start = g->mem->storage + stride*idx;
+ unsigned char *end = start + stride - IB;
+ get_nominal_size(p, end);
+ uint32_t self = 1u<<idx, all = (2u<<g->last_idx)-1;
+ ((unsigned char *)p)[-3] = 255;
+ // invalidate offset to group header, and cycle offset of
+ // used region within slot if current offset is zero.
+ *(uint16_t *)((char *)p-2) = 0;
+
+ // release any whole pages contained in the slot to be freed
+ // unless it's a single-slot group that will be unmapped.
+ if (((uintptr_t)(start-1) ^ (uintptr_t)end) >= 2*PGSZ && g->last_idx) {
+ unsigned char *base = start + (-(uintptr_t)start & (PGSZ-1));
+ size_t len = (end-base) & -PGSZ;
+ if (len) madvise(base, len, MADV_FREE);
+ }
+
+ // atomic free without locking if this is neither first or last slot
+ for (;;) {
+ uint32_t freed = g->freed_mask;
+ uint32_t avail = g->avail_mask;
+ uint32_t mask = freed | avail;
+ assert(!(mask&self));
+ if (!freed || mask+self==all) break;
+ if (!MT)
+ g->freed_mask = freed+self;
+ else if (a_cas(&g->freed_mask, freed, freed+self)!=freed)
+ continue;
+ return;
+ }
+
+ wrlock();
+ struct mapinfo mi = nontrivial_free(g, idx);
+ unlock();
+ if (mi.len) munmap(mi.base, mi.len);
+}
diff --git a/src/malloc/mallocng/glue.h b/src/malloc/mallocng/glue.h
new file mode 100644
index 00000000..16acd1ea
--- /dev/null
+++ b/src/malloc/mallocng/glue.h
@@ -0,0 +1,77 @@
+#ifndef MALLOC_GLUE_H
+#define MALLOC_GLUE_H
+
+#include <stdint.h>
+#include <sys/mman.h>
+#include <pthread.h>
+#include <unistd.h>
+#include <elf.h>
+#include <string.h>
+#include "atomic.h"
+#include "syscall.h"
+#include "libc.h"
+#include "lock.h"
+#include "dynlink.h"
+
+// use macros to appropriately namespace these.
+#define size_classes __malloc_size_classes
+#define ctx __malloc_context
+#define alloc_meta __malloc_alloc_meta
+#define is_allzero __malloc_allzerop
+#define dump_heap __dump_heap
+
+#if USE_REAL_ASSERT
+#include <assert.h>
+#else
+#undef assert
+#define assert(x) do { if (!(x)) a_crash(); } while(0)
+#endif
+
+#define brk(p) ((uintptr_t)__syscall(SYS_brk, p))
+
+#define mmap __mmap
+#define madvise __madvise
+#define mremap __mremap
+
+#define DISABLE_ALIGNED_ALLOC (__malloc_replaced && !__aligned_alloc_replaced)
+
+static inline uint64_t get_random_secret()
+{
+ uint64_t secret = (uintptr_t)&secret * 1103515245;
+ for (size_t i=0; libc.auxv[i]; i+=2)
+ if (libc.auxv[i]==AT_RANDOM)
+ memcpy(&secret, (char *)libc.auxv[i+1]+8, sizeof secret);
+ return secret;
+}
+
+#ifndef PAGESIZE
+#define PAGESIZE PAGE_SIZE
+#endif
+
+#define MT (libc.need_locks)
+
+#define RDLOCK_IS_EXCLUSIVE 1
+
+__attribute__((__visibility__("hidden")))
+extern int __malloc_lock[1];
+
+#define LOCK_OBJ_DEF \
+int __malloc_lock[1];
+
+static inline void rdlock()
+{
+ if (MT) LOCK(__malloc_lock);
+}
+static inline void wrlock()
+{
+ if (MT) LOCK(__malloc_lock);
+}
+static inline void unlock()
+{
+ UNLOCK(__malloc_lock);
+}
+static inline void upgradelock()
+{
+}
+
+#endif
diff --git a/src/malloc/mallocng/malloc.c b/src/malloc/mallocng/malloc.c
new file mode 100644
index 00000000..d695ab8e
--- /dev/null
+++ b/src/malloc/mallocng/malloc.c
@@ -0,0 +1,387 @@
+#include <stdlib.h>
+#include <stdint.h>
+#include <limits.h>
+#include <string.h>
+#include <sys/mman.h>
+#include <errno.h>
+
+#include "meta.h"
+
+LOCK_OBJ_DEF;
+
+const uint16_t size_classes[] = {
+ 1, 2, 3, 4, 5, 6, 7, 8,
+ 9, 10, 12, 15,
+ 18, 20, 25, 31,
+ 36, 42, 50, 63,
+ 72, 84, 102, 127,
+ 146, 170, 204, 255,
+ 292, 340, 409, 511,
+ 584, 682, 818, 1023,
+ 1169, 1364, 1637, 2047,
+ 2340, 2730, 3276, 4095,
+ 4680, 5460, 6552, 8191,
+};
+
+static const uint8_t small_cnt_tab[][3] = {
+ { 30, 30, 30 },
+ { 31, 15, 15 },
+ { 20, 10, 10 },
+ { 31, 15, 7 },
+ { 25, 12, 6 },
+ { 21, 10, 5 },
+ { 18, 8, 4 },
+ { 31, 15, 7 },
+ { 28, 14, 6 },
+};
+
+static const uint8_t med_cnt_tab[4] = { 28, 24, 20, 32 };
+
+struct malloc_context ctx = { 0 };
+
+struct meta *alloc_meta(void)
+{
+ struct meta *m;
+ unsigned char *p;
+ if (!ctx.init_done) {
+#ifndef PAGESIZE
+ ctx.pagesize = get_page_size();
+#endif
+ ctx.secret = get_random_secret();
+ ctx.init_done = 1;
+ }
+ size_t pagesize = PGSZ;
+ if (pagesize < 4096) pagesize = 4096;
+ if ((m = dequeue_head(&ctx.free_meta_head))) return m;
+ if (!ctx.avail_meta_count) {
+ int need_unprotect = 1;
+ if (!ctx.avail_meta_area_count && ctx.brk!=-1) {
+ uintptr_t new = ctx.brk + pagesize;
+ int need_guard = 0;
+ if (!ctx.brk) {
+ need_guard = 1;
+ ctx.brk = brk(0);
+ // some ancient kernels returned _ebss
+ // instead of next page as initial brk.
+ ctx.brk += -ctx.brk & (pagesize-1);
+ new = ctx.brk + 2*pagesize;
+ }
+ if (brk(new) != new) {
+ ctx.brk = -1;
+ } else {
+ if (need_guard) mmap((void *)ctx.brk, pagesize,
+ PROT_NONE, MAP_ANON|MAP_PRIVATE|MAP_FIXED, -1, 0);
+ ctx.brk = new;
+ ctx.avail_meta_areas = (void *)(new - pagesize);
+ ctx.avail_meta_area_count = pagesize>>12;
+ need_unprotect = 0;
+ }
+ }
+ if (!ctx.avail_meta_area_count) {
+ size_t n = 2UL << ctx.meta_alloc_shift;
+ p = mmap(0, n*pagesize, PROT_NONE,
+ MAP_PRIVATE|MAP_ANON, -1, 0);
+ if (p==MAP_FAILED) return 0;
+ ctx.avail_meta_areas = p + pagesize;
+ ctx.avail_meta_area_count = (n-1)*(pagesize>>12);
+ ctx.meta_alloc_shift++;
+ }
+ p = ctx.avail_meta_areas;
+ if ((uintptr_t)p & (pagesize-1)) need_unprotect = 0;
+ if (need_unprotect)
+ if (mprotect(p, pagesize, PROT_READ|PROT_WRITE)
+ && errno != ENOSYS)
+ return 0;
+ ctx.avail_meta_area_count--;
+ ctx.avail_meta_areas = p + 4096;
+ if (ctx.meta_area_tail) {
+ ctx.meta_area_tail->next = (void *)p;
+ } else {
+ ctx.meta_area_head = (void *)p;
+ }
+ ctx.meta_area_tail = (void *)p;
+ ctx.meta_area_tail->check = ctx.secret;
+ ctx.avail_meta_count = ctx.meta_area_tail->nslots
+ = (4096-sizeof(struct meta_area))/sizeof *m;
+ ctx.avail_meta = ctx.meta_area_tail->slots;
+ }
+ ctx.avail_meta_count--;
+ m = ctx.avail_meta++;
+ m->prev = m->next = 0;
+ return m;
+}
+
+static uint32_t try_avail(struct meta **pm)
+{
+ struct meta *m = *pm;
+ uint32_t first;
+ if (!m) return 0;
+ uint32_t mask = m->avail_mask;
+ if (!mask) {
+ if (!m) return 0;
+ if (!m->freed_mask) {
+ dequeue(pm, m);
+ m = *pm;
+ if (!m) return 0;
+ } else {
+ m = m->next;
+ *pm = m;
+ }
+
+ mask = m->freed_mask;
+
+ // skip fully-free group unless it's the only one
+ // or it's a permanently non-freeable group
+ if (mask == (2u<<m->last_idx)-1 && m->freeable) {
+ m = m->next;
+ *pm = m;
+ mask = m->freed_mask;
+ }
+
+ // activate more slots in a not-fully-active group
+ // if needed, but only as a last resort. prefer using
+ // any other group with free slots. this avoids
+ // touching & dirtying as-yet-unused pages.
+ if (!(mask & ((2u<<m->mem->active_idx)-1))) {
+ if (m->next != m) {
+ m = m->next;
+ *pm = m;
+ } else {
+ int cnt = m->mem->active_idx + 2;
+ int size = size_classes[m->sizeclass]*UNIT;
+ int span = UNIT + size*cnt;
+ // activate up to next 4k boundary
+ while ((span^(span+size-1)) < 4096) {
+ cnt++;
+ span += size;
+ }
+ if (cnt > m->last_idx+1)
+ cnt = m->last_idx+1;
+ m->mem->active_idx = cnt-1;
+ }
+ }
+ mask = activate_group(m);
+ assert(mask);
+ decay_bounces(m->sizeclass);
+ }
+ first = mask&-mask;
+ m->avail_mask = mask-first;
+ return first;
+}
+
+static int alloc_slot(int, size_t);
+
+static struct meta *alloc_group(int sc, size_t req)
+{
+ size_t size = UNIT*size_classes[sc];
+ int i = 0, cnt;
+ unsigned char *p;
+ struct meta *m = alloc_meta();
+ if (!m) return 0;
+ size_t usage = ctx.usage_by_class[sc];
+ size_t pagesize = PGSZ;
+ int active_idx;
+ if (sc < 9) {
+ while (i<2 && 4*small_cnt_tab[sc][i] > usage)
+ i++;
+ cnt = small_cnt_tab[sc][i];
+ } else {
+ // lookup max number of slots fitting in power-of-two size
+ // from a table, along with number of factors of two we
+ // can divide out without a remainder or reaching 1.
+ cnt = med_cnt_tab[sc&3];
+
+ // reduce cnt to avoid excessive eagar allocation.
+ while (!(cnt&1) && 4*cnt > usage)
+ cnt >>= 1;
+
+ // data structures don't support groups whose slot offsets
+ // in units don't fit in 16 bits.
+ while (size*cnt >= 65536*UNIT)
+ cnt >>= 1;
+ }
+
+ // If we selected a count of 1 above but it's not sufficient to use
+ // mmap, increase to 2. Then it might be; if not it will nest.
+ if (cnt==1 && size*cnt+UNIT <= pagesize/2) cnt = 2;
+
+ // All choices of size*cnt are "just below" a power of two, so anything
+ // larger than half the page size should be allocated as whole pages.
+ if (size*cnt+UNIT > pagesize/2) {
+ // check/update bounce counter to start/increase retention
+ // of freed maps, and inhibit use of low-count, odd-size
+ // small mappings and single-slot groups if activated.
+ int nosmall = is_bouncing(sc);
+ account_bounce(sc);
+ step_seq();
+
+ // since the following count reduction opportunities have
+ // an absolute memory usage cost, don't overdo them. count
+ // coarse usage as part of usage.
+ if (!(sc&1) && sc<32) usage += ctx.usage_by_class[sc+1];
+
+ // try to drop to a lower count if the one found above
+ // increases usage by more than 25%. these reduced counts
+ // roughly fill an integral number of pages, just not a
+ // power of two, limiting amount of unusable space.
+ if (4*cnt > usage && !nosmall) {
+ if (0);
+ else if ((sc&3)==1 && size*cnt>8*pagesize) cnt = 2;
+ else if ((sc&3)==2 && size*cnt>4*pagesize) cnt = 3;
+ else if ((sc&3)==0 && size*cnt>8*pagesize) cnt = 3;
+ else if ((sc&3)==0 && size*cnt>2*pagesize) cnt = 5;
+ }
+ size_t needed = size*cnt + UNIT;
+ needed += -needed & (pagesize-1);
+
+ // produce an individually-mmapped allocation if usage is low,
+ // bounce counter hasn't triggered, and either it saves memory
+ // or it avoids eagar slot allocation without wasting too much.
+ if (!nosmall && cnt<=7) {
+ req += IB + UNIT;
+ req += -req & (pagesize-1);
+ if (req<size+UNIT || (req>=4*pagesize && 2*cnt>usage)) {
+ cnt = 1;
+ needed = req;
+ }
+ }
+
+ p = mmap(0, needed, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANON, -1, 0);
+ if (p==MAP_FAILED) {
+ free_meta(m);
+ return 0;
+ }
+ m->maplen = needed>>12;
+ ctx.mmap_counter++;
+ active_idx = (4096-UNIT)/size-1;
+ if (active_idx > cnt-1) active_idx = cnt-1;
+ if (active_idx < 0) active_idx = 0;
+ } else {
+ int j = size_to_class(UNIT+cnt*size-IB);
+ int idx = alloc_slot(j, UNIT+cnt*size-IB);
+ if (idx < 0) {
+ free_meta(m);
+ return 0;
+ }
+ struct meta *g = ctx.active[j];
+ p = enframe(g, idx, UNIT*size_classes[j]-IB, ctx.mmap_counter);
+ m->maplen = 0;
+ p[-3] = (p[-3]&31) | (6<<5);
+ for (int i=0; i<=cnt; i++)
+ p[UNIT+i*size-4] = 0;
+ active_idx = cnt-1;
+ }
+ ctx.usage_by_class[sc] += cnt;
+ m->avail_mask = (2u<<active_idx)-1;
+ m->freed_mask = (2u<<(cnt-1))-1 - m->avail_mask;
+ m->mem = (void *)p;
+ m->mem->meta = m;
+ m->mem->active_idx = active_idx;
+ m->last_idx = cnt-1;
+ m->freeable = 1;
+ m->sizeclass = sc;
+ return m;
+}
+
+static int alloc_slot(int sc, size_t req)
+{
+ uint32_t first = try_avail(&ctx.active[sc]);
+ if (first) return a_ctz_32(first);
+
+ struct meta *g = alloc_group(sc, req);
+ if (!g) return -1;
+
+ g->avail_mask--;
+ queue(&ctx.active[sc], g);
+ return 0;
+}
+
+void *malloc(size_t n)
+{
+ if (size_overflows(n)) return 0;
+ struct meta *g;
+ uint32_t mask, first;
+ int sc;
+ int idx;
+ int ctr;
+
+ if (n >= MMAP_THRESHOLD) {
+ size_t needed = n + IB + UNIT;
+ void *p = mmap(0, needed, PROT_READ|PROT_WRITE,
+ MAP_PRIVATE|MAP_ANON, -1, 0);
+ if (p==MAP_FAILED) return 0;
+ wrlock();
+ step_seq();
+ g = alloc_meta();
+ if (!g) {
+ unlock();
+ munmap(p, needed);
+ return 0;
+ }
+ g->mem = p;
+ g->mem->meta = g;
+ g->last_idx = 0;
+ g->freeable = 1;
+ g->sizeclass = 63;
+ g->maplen = (needed+4095)/4096;
+ g->avail_mask = g->freed_mask = 0;
+ // use a global counter to cycle offset in
+ // individually-mmapped allocations.
+ ctx.mmap_counter++;
+ idx = 0;
+ goto success;
+ }
+
+ sc = size_to_class(n);
+
+ rdlock();
+ g = ctx.active[sc];
+
+ // use coarse size classes initially when there are not yet
+ // any groups of desired size. this allows counts of 2 or 3
+ // to be allocated at first rather than having to start with
+ // 7 or 5, the min counts for even size classes.
+ if (!g && sc>=4 && sc<32 && sc!=6 && !(sc&1) && !ctx.usage_by_class[sc]) {
+ size_t usage = ctx.usage_by_class[sc|1];
+ // if a new group may be allocated, count it toward
+ // usage in deciding if we can use coarse class.
+ if (!ctx.active[sc|1] || (!ctx.active[sc|1]->avail_mask
+ && !ctx.active[sc|1]->freed_mask))
+ usage += 3;
+ if (usage <= 12)
+ sc |= 1;
+ g = ctx.active[sc];
+ }
+
+ for (;;) {
+ mask = g ? g->avail_mask : 0;
+ first = mask&-mask;
+ if (!first) break;
+ if (RDLOCK_IS_EXCLUSIVE || !MT)
+ g->avail_mask = mask-first;
+ else if (a_cas(&g->avail_mask, mask, mask-first)!=mask)
+ continue;
+ idx = a_ctz_32(first);
+ goto success;
+ }
+ upgradelock();
+
+ idx = alloc_slot(sc, n);
+ if (idx < 0) {
+ unlock();
+ return 0;
+ }
+ g = ctx.active[sc];
+
+success:
+ ctr = ctx.mmap_counter;
+ unlock();
+ return enframe(g, idx, n, ctr);
+}
+
+int is_allzero(void *p)
+{
+ struct meta *g = get_meta(p);
+ return g->sizeclass >= 48 ||
+ get_stride(g) < UNIT*size_classes[g->sizeclass];
+}
diff --git a/src/malloc/mallocng/malloc_usable_size.c b/src/malloc/mallocng/malloc_usable_size.c
new file mode 100644
index 00000000..a440a4ea
--- /dev/null
+++ b/src/malloc/mallocng/malloc_usable_size.c
@@ -0,0 +1,12 @@
+#include <stdlib.h>
+#include "meta.h"
+
+size_t malloc_usable_size(void *p)
+{
+ struct meta *g = get_meta(p);
+ int idx = get_slot_index(p);
+ size_t stride = get_stride(g);
+ unsigned char *start = g->mem->storage + stride*idx;
+ unsigned char *end = start + stride - IB;
+ return get_nominal_size(p, end);
+}
diff --git a/src/malloc/mallocng/meta.h b/src/malloc/mallocng/meta.h
new file mode 100644
index 00000000..61ec53f9
--- /dev/null
+++ b/src/malloc/mallocng/meta.h
@@ -0,0 +1,288 @@
+#ifndef MALLOC_META_H
+#define MALLOC_META_H
+
+#include <stdint.h>
+#include <errno.h>
+#include <limits.h>
+#include "glue.h"
+
+__attribute__((__visibility__("hidden")))
+extern const uint16_t size_classes[];
+
+#define MMAP_THRESHOLD 131052
+
+#define UNIT 16
+#define IB 4
+
+struct group {
+ struct meta *meta;
+ unsigned char active_idx:5;
+ char pad[UNIT - sizeof(struct meta *) - 1];
+ unsigned char storage[];
+};
+
+struct meta {
+ struct meta *prev, *next;
+ struct group *mem;
+ volatile int avail_mask, freed_mask;
+ uintptr_t last_idx:5;
+ uintptr_t freeable:1;
+ uintptr_t sizeclass:6;
+ uintptr_t maplen:8*sizeof(uintptr_t)-12;
+};
+
+struct meta_area {
+ uint64_t check;
+ struct meta_area *next;
+ int nslots;
+ struct meta slots[];
+};
+
+struct malloc_context {
+ uint64_t secret;
+#ifndef PAGESIZE
+ size_t pagesize;
+#endif
+ int init_done;
+ unsigned mmap_counter;
+ struct meta *free_meta_head;
+ struct meta *avail_meta;
+ size_t avail_meta_count, avail_meta_area_count, meta_alloc_shift;
+ struct meta_area *meta_area_head, *meta_area_tail;
+ unsigned char *avail_meta_areas;
+ struct meta *active[48];
+ size_t usage_by_class[48];
+ uint8_t unmap_seq[32], bounces[32];
+ uint8_t seq;
+ uintptr_t brk;
+};
+
+__attribute__((__visibility__("hidden")))
+extern struct malloc_context ctx;
+
+#ifdef PAGESIZE
+#define PGSZ PAGESIZE
+#else
+#define PGSZ ctx.pagesize
+#endif
+
+__attribute__((__visibility__("hidden")))
+struct meta *alloc_meta(void);
+
+__attribute__((__visibility__("hidden")))
+int is_allzero(void *);
+
+static inline void queue(struct meta **phead, struct meta *m)
+{
+ assert(!m->next);
+ assert(!m->prev);
+ if (*phead) {
+ struct meta *head = *phead;
+ m->next = head;
+ m->prev = head->prev;
+ m->next->prev = m->prev->next = m;
+ } else {
+ m->prev = m->next = m;
+ *phead = m;
+ }
+}
+
+static inline void dequeue(struct meta **phead, struct meta *m)
+{
+ if (m->next != m) {
+ m->prev->next = m->next;
+ m->next->prev = m->prev;
+ if (*phead == m) *phead = m->next;
+ } else {
+ *phead = 0;
+ }
+ m->prev = m->next = 0;
+}
+
+static inline struct meta *dequeue_head(struct meta **phead)
+{
+ struct meta *m = *phead;
+ if (m) dequeue(phead, m);
+ return m;
+}
+
+static inline void free_meta(struct meta *m)
+{
+ *m = (struct meta){0};
+ queue(&ctx.free_meta_head, m);
+}
+
+static inline uint32_t activate_group(struct meta *m)
+{
+ assert(!m->avail_mask);
+ uint32_t mask, act = (2u<<m->mem->active_idx)-1;
+ do mask = m->freed_mask;
+ while (a_cas(&m->freed_mask, mask, mask&~act)!=mask);
+ return m->avail_mask = mask & act;
+}
+
+static inline int get_slot_index(const unsigned char *p)
+{
+ return p[-3] & 31;
+}
+
+static inline struct meta *get_meta(const unsigned char *p)
+{
+ assert(!((uintptr_t)p & 15));
+ int offset = *(const uint16_t *)(p - 2);
+ int index = get_slot_index(p);
+ if (p[-4]) {
+ assert(!offset);
+ offset = *(uint32_t *)(p - 8);
+ assert(offset > 0xffff);
+ }
+ const struct group *base = (const void *)(p - UNIT*offset - UNIT);
+ const struct meta *meta = base->meta;
+ assert(meta->mem == base);
+ assert(index <= meta->last_idx);
+ assert(!(meta->avail_mask & (1u<<index)));
+ assert(!(meta->freed_mask & (1u<<index)));
+ const struct meta_area *area = (void *)((uintptr_t)meta & -4096);
+ assert(area->check == ctx.secret);
+ if (meta->sizeclass < 48) {
+ assert(offset >= size_classes[meta->sizeclass]*index);
+ assert(offset < size_classes[meta->sizeclass]*(index+1));
+ } else {
+ assert(meta->sizeclass == 63);
+ }
+ if (meta->maplen) {
+ assert(offset <= meta->maplen*4096UL/UNIT - 1);
+ }
+ return (struct meta *)meta;
+}
+
+static inline size_t get_nominal_size(const unsigned char *p, const unsigned char *end)
+{
+ size_t reserved = p[-3] >> 5;
+ if (reserved >= 5) {
+ assert(reserved == 5);
+ reserved = *(const uint32_t *)(end-4);
+ assert(reserved >= 5);
+ assert(!end[-5]);
+ }
+ assert(reserved <= end-p);
+ assert(!*(end-reserved));
+ // also check the slot's overflow byte
+ assert(!*end);
+ return end-reserved-p;
+}
+
+static inline size_t get_stride(const struct meta *g)
+{
+ if (!g->last_idx && g->maplen) {
+ return g->maplen*4096UL - UNIT;
+ } else {
+ return UNIT*size_classes[g->sizeclass];
+ }
+}
+
+static inline void set_size(unsigned char *p, unsigned char *end, size_t n)
+{
+ int reserved = end-p-n;
+ if (reserved) end[-reserved] = 0;
+ if (reserved >= 5) {
+ *(uint32_t *)(end-4) = reserved;
+ end[-5] = 0;
+ reserved = 5;
+ }
+ p[-3] = (p[-3]&31) + (reserved<<5);
+}
+
+static inline void *enframe(struct meta *g, int idx, size_t n, int ctr)
+{
+ size_t stride = get_stride(g);
+ size_t slack = (stride-IB-n)/UNIT;
+ unsigned char *p = g->mem->storage + stride*idx;
+ unsigned char *end = p+stride-IB;
+ // cycle offset within slot to increase interval to address
+ // reuse, facilitate trapping double-free.
+ int off = (p[-3] ? *(uint16_t *)(p-2) + 1 : ctr) & 255;
+ assert(!p[-4]);
+ if (off > slack) {
+ size_t m = slack;
+ m |= m>>1; m |= m>>2; m |= m>>4;
+ off &= m;
+ if (off > slack) off -= slack+1;
+ assert(off <= slack);
+ }
+ if (off) {
+ // store offset in unused header at offset zero
+ // if enframing at non-zero offset.
+ *(uint16_t *)(p-2) = off;
+ p[-3] = 7<<5;
+ p += UNIT*off;
+ // for nonzero offset there is no permanent check
+ // byte, so make one.
+ p[-4] = 0;
+ }
+ *(uint16_t *)(p-2) = (size_t)(p-g->mem->storage)/UNIT;
+ p[-3] = idx;
+ set_size(p, end, n);
+ return p;
+}
+
+static inline int size_to_class(size_t n)
+{
+ n = (n+IB-1)>>4;
+ if (n<10) return n;
+ n++;
+ int i = (28-a_clz_32(n))*4 + 8;
+ if (n>size_classes[i+1]) i+=2;
+ if (n>size_classes[i]) i++;
+ return i;
+}
+
+static inline int size_overflows(size_t n)
+{
+ if (n >= SIZE_MAX/2 - 4096) {
+ errno = ENOMEM;
+ return 1;
+ }
+ return 0;
+}
+
+static inline void step_seq(void)
+{
+ if (ctx.seq==255) {
+ for (int i=0; i<32; i++) ctx.unmap_seq[i] = 0;
+ ctx.seq = 1;
+ } else {
+ ctx.seq++;
+ }
+}
+
+static inline void record_seq(int sc)
+{
+ if (sc-7U < 32) ctx.unmap_seq[sc-7] = ctx.seq;
+}
+
+static inline void account_bounce(int sc)
+{
+ if (sc-7U < 32) {
+ int seq = ctx.unmap_seq[sc-7];
+ if (seq && ctx.seq-seq < 10) {
+ if (ctx.bounces[sc-7]+1 < 100)
+ ctx.bounces[sc-7]++;
+ else
+ ctx.bounces[sc-7] = 150;
+ }
+ }
+}
+
+static inline void decay_bounces(int sc)
+{
+ if (sc-7U < 32 && ctx.bounces[sc-7])
+ ctx.bounces[sc-7]--;
+}
+
+static inline int is_bouncing(int sc)
+{
+ return (sc-7U < 32 && ctx.bounces[sc-7] >= 100);
+}
+
+#endif
diff --git a/src/malloc/mallocng/realloc.c b/src/malloc/mallocng/realloc.c
new file mode 100644
index 00000000..18769f42
--- /dev/null
+++ b/src/malloc/mallocng/realloc.c
@@ -0,0 +1,51 @@
+#define _GNU_SOURCE
+#include <stdlib.h>
+#include <sys/mman.h>
+#include <string.h>
+#include "meta.h"
+
+void *realloc(void *p, size_t n)
+{
+ if (!p) return malloc(n);
+ if (size_overflows(n)) return 0;
+
+ struct meta *g = get_meta(p);
+ int idx = get_slot_index(p);
+ size_t stride = get_stride(g);
+ unsigned char *start = g->mem->storage + stride*idx;
+ unsigned char *end = start + stride - IB;
+ size_t old_size = get_nominal_size(p, end);
+ size_t avail_size = end-(unsigned char *)p;
+ void *new;
+
+ // only resize in-place if size class matches
+ if (n <= avail_size && n<MMAP_THRESHOLD
+ && size_to_class(n)+1 >= g->sizeclass) {
+ set_size(p, end, n);
+ return p;
+ }
+
+ // use mremap if old and new size are both mmap-worthy
+ if (g->sizeclass>=48 && n>=MMAP_THRESHOLD) {
+ assert(g->sizeclass==63);
+ size_t base = (unsigned char *)p-start;
+ size_t needed = (n + base + UNIT + IB + 4095) & -4096;
+ new = g->maplen*4096UL == needed ? g->mem :
+ mremap(g->mem, g->maplen*4096UL, needed, MREMAP_MAYMOVE);
+ if (new!=MAP_FAILED) {
+ g->mem = new;
+ g->maplen = needed/4096;
+ p = g->mem->storage + base;
+ end = g->mem->storage + (needed - UNIT) - IB;
+ *end = 0;
+ set_size(p, end, n);
+ return p;
+ }
+ }
+
+ new = malloc(n);
+ if (!new) return 0;
+ memcpy(new, p, n < old_size ? n : old_size);
+ free(p);
+ return new;
+}
diff --git a/src/malloc/memalign.c b/src/malloc/memalign.c
index cf9dfbda..32cd87d8 100644
--- a/src/malloc/memalign.c
+++ b/src/malloc/memalign.c
@@ -1,54 +1,7 @@
+#define _BSD_SOURCE
#include <stdlib.h>
-#include <stdint.h>
-#include <errno.h>
-#include "malloc_impl.h"
-void *__memalign(size_t align, size_t len)
+void *memalign(size_t align, size_t len)
{
- unsigned char *mem, *new;
-
- if ((align & -align) != align) {
- errno = EINVAL;
- return 0;
- }
-
- if (len > SIZE_MAX - align || __malloc_replaced) {
- errno = ENOMEM;
- return 0;
- }
-
- if (align <= SIZE_ALIGN)
- return malloc(len);
-
- if (!(mem = malloc(len + align-1)))
- return 0;
-
- new = (void *)((uintptr_t)mem + align-1 & -align);
- if (new == mem) return mem;
-
- struct chunk *c = MEM_TO_CHUNK(mem);
- struct chunk *n = MEM_TO_CHUNK(new);
-
- if (IS_MMAPPED(c)) {
- /* Apply difference between aligned and original
- * address to the "extra" field of mmapped chunk. */
- n->psize = c->psize + (new-mem);
- n->csize = c->csize - (new-mem);
- return new;
- }
-
- struct chunk *t = NEXT_CHUNK(c);
-
- /* Split the allocated chunk into two chunks. The aligned part
- * that will be used has the size in its footer reduced by the
- * difference between the aligned and original addresses, and
- * the resulting size copied to its header. A new header and
- * footer are written for the split-off part to be freed. */
- n->psize = c->csize = C_INUSE | (new-mem);
- n->csize = t->psize -= new-mem;
-
- __bin_chunk(c);
- return new;
+ return aligned_alloc(align, len);
}
-
-weak_alias(__memalign, memalign);
diff --git a/src/malloc/oldmalloc/aligned_alloc.c b/src/malloc/oldmalloc/aligned_alloc.c
new file mode 100644
index 00000000..4adca3b4
--- /dev/null
+++ b/src/malloc/oldmalloc/aligned_alloc.c
@@ -0,0 +1,53 @@
+#include <stdlib.h>
+#include <stdint.h>
+#include <errno.h>
+#include "malloc_impl.h"
+
+void *aligned_alloc(size_t align, size_t len)
+{
+ unsigned char *mem, *new;
+
+ if ((align & -align) != align) {
+ errno = EINVAL;
+ return 0;
+ }
+
+ if (len > SIZE_MAX - align ||
+ (__malloc_replaced && !__aligned_alloc_replaced)) {
+ errno = ENOMEM;
+ return 0;
+ }
+
+ if (align <= SIZE_ALIGN)
+ return malloc(len);
+
+ if (!(mem = malloc(len + align-1)))
+ return 0;
+
+ new = (void *)((uintptr_t)mem + align-1 & -align);
+ if (new == mem) return mem;
+
+ struct chunk *c = MEM_TO_CHUNK(mem);
+ struct chunk *n = MEM_TO_CHUNK(new);
+
+ if (IS_MMAPPED(c)) {
+ /* Apply difference between aligned and original
+ * address to the "extra" field of mmapped chunk. */
+ n->psize = c->psize + (new-mem);
+ n->csize = c->csize - (new-mem);
+ return new;
+ }
+
+ struct chunk *t = NEXT_CHUNK(c);
+
+ /* Split the allocated chunk into two chunks. The aligned part
+ * that will be used has the size in its footer reduced by the
+ * difference between the aligned and original addresses, and
+ * the resulting size copied to its header. A new header and
+ * footer are written for the split-off part to be freed. */
+ n->psize = c->csize = C_INUSE | (new-mem);
+ n->csize = t->psize -= new-mem;
+
+ __bin_chunk(c);
+ return new;
+}
diff --git a/src/malloc/malloc.c b/src/malloc/oldmalloc/malloc.c
index 96982596..c0997ad8 100644
--- a/src/malloc/malloc.c
+++ b/src/malloc/oldmalloc/malloc.c
@@ -17,17 +17,18 @@
static struct {
volatile uint64_t binmap;
struct bin bins[64];
- volatile int free_lock[2];
+ volatile int split_merge_lock[2];
} mal;
-int __malloc_replaced;
-
/* Synchronization tools */
static inline void lock(volatile int *lk)
{
- if (libc.threads_minus_1)
+ int need_locks = libc.need_locks;
+ if (need_locks) {
while(a_swap(lk, 1)) __wait(lk, lk+1, 1, 1);
+ if (need_locks < 0) libc.need_locks = 0;
+ }
}
static inline void unlock(volatile int *lk)
@@ -123,9 +124,72 @@ void __dump_heap(int x)
}
#endif
+/* This function returns true if the interval [old,new]
+ * intersects the 'len'-sized interval below &libc.auxv
+ * (interpreted as the main-thread stack) or below &b
+ * (the current stack). It is used to defend against
+ * buggy brk implementations that can cross the stack. */
+
+static int traverses_stack_p(uintptr_t old, uintptr_t new)
+{
+ const uintptr_t len = 8<<20;
+ uintptr_t a, b;
+
+ b = (uintptr_t)libc.auxv;
+ a = b > len ? b-len : 0;
+ if (new>a && old<b) return 1;
+
+ b = (uintptr_t)&b;
+ a = b > len ? b-len : 0;
+ if (new>a && old<b) return 1;
+
+ return 0;
+}
+
+/* Expand the heap in-place if brk can be used, or otherwise via mmap,
+ * using an exponential lower bound on growth by mmap to make
+ * fragmentation asymptotically irrelevant. The size argument is both
+ * an input and an output, since the caller needs to know the size
+ * allocated, which will be larger than requested due to page alignment
+ * and mmap minimum size rules. The caller is responsible for locking
+ * to prevent concurrent calls. */
+
+static void *__expand_heap(size_t *pn)
+{
+ static uintptr_t brk;
+ static unsigned mmap_step;
+ size_t n = *pn;
+
+ if (n > SIZE_MAX/2 - PAGE_SIZE) {
+ errno = ENOMEM;
+ return 0;
+ }
+ n += -n & PAGE_SIZE-1;
+
+ if (!brk) {
+ brk = __syscall(SYS_brk, 0);
+ brk += -brk & PAGE_SIZE-1;
+ }
+
+ if (n < SIZE_MAX-brk && !traverses_stack_p(brk, brk+n)
+ && __syscall(SYS_brk, brk+n)==brk+n) {
+ *pn = n;
+ brk += n;
+ return (void *)(brk-n);
+ }
+
+ size_t min = (size_t)PAGE_SIZE << mmap_step/2;
+ if (n < min) n = min;
+ void *area = __mmap(0, n, PROT_READ|PROT_WRITE,
+ MAP_PRIVATE|MAP_ANONYMOUS, -1, 0);
+ if (area == MAP_FAILED) return 0;
+ *pn = n;
+ mmap_step++;
+ return area;
+}
+
static struct chunk *expand_heap(size_t n)
{
- static int heap_lock[2];
static void *end;
void *p;
struct chunk *w;
@@ -135,13 +199,8 @@ static struct chunk *expand_heap(size_t n)
* we need room for an extra zero-sized sentinel chunk. */
n += SIZE_ALIGN;
- lock(heap_lock);
-
p = __expand_heap(&n);
- if (!p) {
- unlock(heap_lock);
- return 0;
- }
+ if (!p) return 0;
/* If not just expanding existing space, we need to make a
* new sentinel chunk below the allocated space. */
@@ -164,8 +223,6 @@ static struct chunk *expand_heap(size_t n)
w = MEM_TO_CHUNK(p);
w->csize = n | C_INUSE;
- unlock(heap_lock);
-
return w;
}
@@ -195,96 +252,44 @@ static void unbin(struct chunk *c, int i)
NEXT_CHUNK(c)->psize |= C_INUSE;
}
-static int alloc_fwd(struct chunk *c)
+static void bin_chunk(struct chunk *self, int i)
{
- int i;
- size_t k;
- while (!((k=c->csize) & C_INUSE)) {
- i = bin_index(k);
- lock_bin(i);
- if (c->csize == k) {
- unbin(c, i);
- unlock_bin(i);
- return 1;
- }
- unlock_bin(i);
- }
- return 0;
-}
-
-static int alloc_rev(struct chunk *c)
-{
- int i;
- size_t k;
- while (!((k=c->psize) & C_INUSE)) {
- i = bin_index(k);
- lock_bin(i);
- if (c->psize == k) {
- unbin(PREV_CHUNK(c), i);
- unlock_bin(i);
- return 1;
- }
- unlock_bin(i);
- }
- return 0;
+ self->next = BIN_TO_CHUNK(i);
+ self->prev = mal.bins[i].tail;
+ self->next->prev = self;
+ self->prev->next = self;
+ if (self->prev == BIN_TO_CHUNK(i))
+ a_or_64(&mal.binmap, 1ULL<<i);
}
-
-/* pretrim - trims a chunk _prior_ to removing it from its bin.
- * Must be called with i as the ideal bin for size n, j the bin
- * for the _free_ chunk self, and bin j locked. */
-static int pretrim(struct chunk *self, size_t n, int i, int j)
+static void trim(struct chunk *self, size_t n)
{
- size_t n1;
+ size_t n1 = CHUNK_SIZE(self);
struct chunk *next, *split;
- /* We cannot pretrim if it would require re-binning. */
- if (j < 40) return 0;
- if (j < i+3) {
- if (j != 63) return 0;
- n1 = CHUNK_SIZE(self);
- if (n1-n <= MMAP_THRESHOLD) return 0;
- } else {
- n1 = CHUNK_SIZE(self);
- }
- if (bin_index(n1-n) != j) return 0;
+ if (n >= n1 - DONTCARE) return;
next = NEXT_CHUNK(self);
split = (void *)((char *)self + n);
- split->prev = self->prev;
- split->next = self->next;
- split->prev->next = split;
- split->next->prev = split;
split->psize = n | C_INUSE;
split->csize = n1-n;
next->psize = n1-n;
self->csize = n | C_INUSE;
- return 1;
-}
-
-static void trim(struct chunk *self, size_t n)
-{
- size_t n1 = CHUNK_SIZE(self);
- struct chunk *next, *split;
-
- if (n >= n1 - DONTCARE) return;
- next = NEXT_CHUNK(self);
- split = (void *)((char *)self + n);
+ int i = bin_index(n1-n);
+ lock_bin(i);
- split->psize = n | C_INUSE;
- split->csize = n1-n | C_INUSE;
- next->psize = n1-n | C_INUSE;
- self->csize = n | C_INUSE;
+ bin_chunk(split, i);
- __bin_chunk(split);
+ unlock_bin(i);
}
void *malloc(size_t n)
{
struct chunk *c;
int i, j;
+ uint64_t mask;
if (adjust_size(&n) < 0) return 0;
@@ -300,70 +305,43 @@ void *malloc(size_t n)
}
i = bin_index_up(n);
- for (;;) {
- uint64_t mask = mal.binmap & -(1ULL<<i);
- if (!mask) {
- c = expand_heap(n);
- if (!c) return 0;
- if (alloc_rev(c)) {
- struct chunk *x = c;
- c = PREV_CHUNK(c);
- NEXT_CHUNK(x)->psize = c->csize =
- x->csize + CHUNK_SIZE(c);
- }
- break;
+ if (i<63 && (mal.binmap & (1ULL<<i))) {
+ lock_bin(i);
+ c = mal.bins[i].head;
+ if (c != BIN_TO_CHUNK(i) && CHUNK_SIZE(c)-n <= DONTCARE) {
+ unbin(c, i);
+ unlock_bin(i);
+ return CHUNK_TO_MEM(c);
}
+ unlock_bin(i);
+ }
+ lock(mal.split_merge_lock);
+ for (mask = mal.binmap & -(1ULL<<i); mask; mask -= (mask&-mask)) {
j = first_set(mask);
lock_bin(j);
c = mal.bins[j].head;
if (c != BIN_TO_CHUNK(j)) {
- if (!pretrim(c, n, i, j)) unbin(c, j);
+ unbin(c, j);
unlock_bin(j);
break;
}
unlock_bin(j);
}
-
- /* Now patch up in case we over-allocated */
+ if (!mask) {
+ c = expand_heap(n);
+ if (!c) {
+ unlock(mal.split_merge_lock);
+ return 0;
+ }
+ }
trim(c, n);
-
+ unlock(mal.split_merge_lock);
return CHUNK_TO_MEM(c);
}
-static size_t mal0_clear(char *p, size_t pagesz, size_t n)
+int __malloc_allzerop(void *p)
{
-#ifdef __GNUC__
- typedef uint64_t __attribute__((__may_alias__)) T;
-#else
- typedef unsigned char T;
-#endif
- char *pp = p + n;
- size_t i = (uintptr_t)pp & (pagesz - 1);
- for (;;) {
- pp = memset(pp - i, 0, i);
- if (pp - p < pagesz) return pp - p;
- for (i = pagesz; i; i -= 2*sizeof(T), pp -= 2*sizeof(T))
- if (((T *)pp)[-1] | ((T *)pp)[-2])
- break;
- }
-}
-
-void *calloc(size_t m, size_t n)
-{
- if (n && m > (size_t)-1/n) {
- errno = ENOMEM;
- return 0;
- }
- n *= m;
- void *p = malloc(n);
- if (!p) return p;
- if (!__malloc_replaced) {
- if (IS_MMAPPED(MEM_TO_CHUNK(p)))
- return p;
- if (n >= PAGE_SIZE)
- n = mal0_clear(p, PAGE_SIZE, n);
- }
- return memset(p, 0, n);
+ return IS_MMAPPED(MEM_TO_CHUNK(p));
}
void *realloc(void *p, size_t n)
@@ -379,6 +357,8 @@ void *realloc(void *p, size_t n)
self = MEM_TO_CHUNK(p);
n1 = n0 = CHUNK_SIZE(self);
+ if (n<=n0 && n0-n<=DONTCARE) return p;
+
if (IS_MMAPPED(self)) {
size_t extra = self->psize;
char *base = (char *)self - extra;
@@ -405,34 +385,43 @@ void *realloc(void *p, size_t n)
/* Crash on corrupted footer (likely from buffer overflow) */
if (next->psize != self->csize) a_crash();
- /* Merge adjacent chunks if we need more space. This is not
- * a waste of time even if we fail to get enough space, because our
- * subsequent call to free would otherwise have to do the merge. */
- if (n > n1 && alloc_fwd(next)) {
- n1 += CHUNK_SIZE(next);
- next = NEXT_CHUNK(next);
- }
- /* FIXME: find what's wrong here and reenable it..? */
- if (0 && n > n1 && alloc_rev(self)) {
- self = PREV_CHUNK(self);
- n1 += CHUNK_SIZE(self);
+ if (n < n0) {
+ int i = bin_index_up(n);
+ int j = bin_index(n0);
+ if (i<j && (mal.binmap & (1ULL << i)))
+ goto copy_realloc;
+ struct chunk *split = (void *)((char *)self + n);
+ self->csize = split->psize = n | C_INUSE;
+ split->csize = next->psize = n0-n | C_INUSE;
+ __bin_chunk(split);
+ return CHUNK_TO_MEM(self);
}
- self->csize = n1 | C_INUSE;
- next->psize = n1 | C_INUSE;
- /* If we got enough space, split off the excess and return */
- if (n <= n1) {
- //memmove(CHUNK_TO_MEM(self), p, n0-OVERHEAD);
- trim(self, n);
- return CHUNK_TO_MEM(self);
+ lock(mal.split_merge_lock);
+
+ size_t nsize = next->csize & C_INUSE ? 0 : CHUNK_SIZE(next);
+ if (n0+nsize >= n) {
+ int i = bin_index(nsize);
+ lock_bin(i);
+ if (!(next->csize & C_INUSE)) {
+ unbin(next, i);
+ unlock_bin(i);
+ next = NEXT_CHUNK(next);
+ self->csize = next->psize = n0+nsize | C_INUSE;
+ trim(self, n);
+ unlock(mal.split_merge_lock);
+ return CHUNK_TO_MEM(self);
+ }
+ unlock_bin(i);
}
+ unlock(mal.split_merge_lock);
copy_realloc:
/* As a last resort, allocate a new chunk and copy to it. */
new = malloc(n-OVERHEAD);
if (!new) return 0;
copy_free_ret:
- memcpy(new, p, n0-OVERHEAD);
+ memcpy(new, p, (n<n0 ? n : n0) - OVERHEAD);
free(CHUNK_TO_MEM(self));
return new;
}
@@ -440,59 +429,51 @@ copy_free_ret:
void __bin_chunk(struct chunk *self)
{
struct chunk *next = NEXT_CHUNK(self);
- size_t final_size, new_size, size;
- int reclaim=0;
- int i;
-
- final_size = new_size = CHUNK_SIZE(self);
/* Crash on corrupted footer (likely from buffer overflow) */
if (next->psize != self->csize) a_crash();
- for (;;) {
- if (self->psize & next->csize & C_INUSE) {
- self->csize = final_size | C_INUSE;
- next->psize = final_size | C_INUSE;
- i = bin_index(final_size);
- lock_bin(i);
- lock(mal.free_lock);
- if (self->psize & next->csize & C_INUSE)
- break;
- unlock(mal.free_lock);
- unlock_bin(i);
- }
+ lock(mal.split_merge_lock);
- if (alloc_rev(self)) {
- self = PREV_CHUNK(self);
- size = CHUNK_SIZE(self);
- final_size += size;
- if (new_size+size > RECLAIM && (new_size+size^size) > size)
- reclaim = 1;
- }
+ size_t osize = CHUNK_SIZE(self), size = osize;
+
+ /* Since we hold split_merge_lock, only transition from free to
+ * in-use can race; in-use to free is impossible */
+ size_t psize = self->psize & C_INUSE ? 0 : CHUNK_PSIZE(self);
+ size_t nsize = next->csize & C_INUSE ? 0 : CHUNK_SIZE(next);
- if (alloc_fwd(next)) {
- size = CHUNK_SIZE(next);
- final_size += size;
- if (new_size+size > RECLAIM && (new_size+size^size) > size)
- reclaim = 1;
+ if (psize) {
+ int i = bin_index(psize);
+ lock_bin(i);
+ if (!(self->psize & C_INUSE)) {
+ struct chunk *prev = PREV_CHUNK(self);
+ unbin(prev, i);
+ self = prev;
+ size += psize;
+ }
+ unlock_bin(i);
+ }
+ if (nsize) {
+ int i = bin_index(nsize);
+ lock_bin(i);
+ if (!(next->csize & C_INUSE)) {
+ unbin(next, i);
next = NEXT_CHUNK(next);
+ size += nsize;
}
+ unlock_bin(i);
}
- if (!(mal.binmap & 1ULL<<i))
- a_or_64(&mal.binmap, 1ULL<<i);
-
- self->csize = final_size;
- next->psize = final_size;
- unlock(mal.free_lock);
+ int i = bin_index(size);
+ lock_bin(i);
- self->next = BIN_TO_CHUNK(i);
- self->prev = mal.bins[i].tail;
- self->next->prev = self;
- self->prev->next = self;
+ self->csize = size;
+ next->psize = size;
+ bin_chunk(self, i);
+ unlock(mal.split_merge_lock);
/* Replace middle of large chunks with fresh zero pages */
- if (reclaim) {
+ if (size > RECLAIM && (size^(size-osize)) > size-osize) {
uintptr_t a = (uintptr_t)self + SIZE_ALIGN+PAGE_SIZE-1 & -PAGE_SIZE;
uintptr_t b = (uintptr_t)next - SIZE_ALIGN & -PAGE_SIZE;
#if 1
diff --git a/src/internal/malloc_impl.h b/src/malloc/oldmalloc/malloc_impl.h
index 59785a7f..e1cf4774 100644
--- a/src/internal/malloc_impl.h
+++ b/src/malloc/oldmalloc/malloc_impl.h
@@ -2,12 +2,7 @@
#define MALLOC_IMPL_H
#include <sys/mman.h>
-
-hidden void *__expand_heap(size_t *);
-
-hidden void __malloc_donate(char *, char *);
-
-hidden void *__memalign(size_t, size_t);
+#include "dynlink.h"
struct chunk {
size_t psize, csize;
@@ -41,6 +36,4 @@ struct bin {
hidden void __bin_chunk(struct chunk *);
-hidden extern int __malloc_replaced;
-
#endif
diff --git a/src/malloc/malloc_usable_size.c b/src/malloc/oldmalloc/malloc_usable_size.c
index 672b518a..672b518a 100644
--- a/src/malloc/malloc_usable_size.c
+++ b/src/malloc/oldmalloc/malloc_usable_size.c
diff --git a/src/malloc/posix_memalign.c b/src/malloc/posix_memalign.c
index 2ea8bd8a..ad4d8f47 100644
--- a/src/malloc/posix_memalign.c
+++ b/src/malloc/posix_memalign.c
@@ -1,11 +1,10 @@
#include <stdlib.h>
#include <errno.h>
-#include "malloc_impl.h"
int posix_memalign(void **res, size_t align, size_t len)
{
if (align < sizeof(void *)) return EINVAL;
- void *mem = __memalign(align, len);
+ void *mem = aligned_alloc(align, len);
if (!mem) return errno;
*res = mem;
return 0;
diff --git a/src/malloc/replaced.c b/src/malloc/replaced.c
new file mode 100644
index 00000000..07fce61e
--- /dev/null
+++ b/src/malloc/replaced.c
@@ -0,0 +1,4 @@
+#include "dynlink.h"
+
+int __malloc_replaced;
+int __aligned_alloc_replaced;
diff --git a/src/math/__expo2.c b/src/math/__expo2.c
index 740ac680..248f052b 100644
--- a/src/math/__expo2.c
+++ b/src/math/__expo2.c
@@ -5,12 +5,13 @@ static const int k = 2043;
static const double kln2 = 0x1.62066151add8bp+10;
/* exp(x)/2 for x >= log(DBL_MAX), slightly better than 0.5*exp(x/2)*exp(x/2) */
-double __expo2(double x)
+double __expo2(double x, double sign)
{
double scale;
/* note that k is odd and scale*scale overflows */
INSERT_WORDS(scale, (uint32_t)(0x3ff + k/2) << 20, 0);
/* exp(x - k ln2) * 2**(k-1) */
- return exp(x - kln2) * scale * scale;
+ /* in directed rounding correct sign before rounding or overflow is important */
+ return exp(x - kln2) * (sign * scale) * scale;
}
diff --git a/src/math/__expo2f.c b/src/math/__expo2f.c
index 5163e418..538eb09c 100644
--- a/src/math/__expo2f.c
+++ b/src/math/__expo2f.c
@@ -5,12 +5,13 @@ static const int k = 235;
static const float kln2 = 0x1.45c778p+7f;
/* expf(x)/2 for x >= log(FLT_MAX), slightly better than 0.5f*expf(x/2)*expf(x/2) */
-float __expo2f(float x)
+float __expo2f(float x, float sign)
{
float scale;
/* note that k is odd and scale*scale overflows */
SET_FLOAT_WORD(scale, (uint32_t)(0x7f + k/2) << 23);
/* exp(x - k ln2) * 2**(k-1) */
- return expf(x - kln2) * scale * scale;
+ /* in directed rounding correct sign before rounding or overflow is important */
+ return expf(x - kln2) * (sign * scale) * scale;
}
diff --git a/src/math/__math_invalidl.c b/src/math/__math_invalidl.c
new file mode 100644
index 00000000..1fca99de
--- /dev/null
+++ b/src/math/__math_invalidl.c
@@ -0,0 +1,9 @@
+#include <float.h>
+#include "libm.h"
+
+#if LDBL_MANT_DIG != DBL_MANT_DIG
+long double __math_invalidl(long double x)
+{
+ return (x - x) / (x - x);
+}
+#endif
diff --git a/src/math/__rem_pio2.c b/src/math/__rem_pio2.c
index d403f81c..dcf672fb 100644
--- a/src/math/__rem_pio2.c
+++ b/src/math/__rem_pio2.c
@@ -36,6 +36,7 @@
*/
static const double
toint = 1.5/EPS,
+pio4 = 0x1.921fb54442d18p-1,
invpio2 = 6.36619772367581382433e-01, /* 0x3FE45F30, 0x6DC9C883 */
pio2_1 = 1.57079632673412561417e+00, /* 0x3FF921FB, 0x54400000 */
pio2_1t = 6.07710050650619224932e-11, /* 0x3DD0B461, 0x1A626331 */
@@ -117,11 +118,23 @@ int __rem_pio2(double x, double *y)
}
if (ix < 0x413921fb) { /* |x| ~< 2^20*(pi/2), medium size */
medium:
- /* rint(x/(pi/2)), Assume round-to-nearest. */
+ /* rint(x/(pi/2)) */
fn = (double_t)x*invpio2 + toint - toint;
n = (int32_t)fn;
r = x - fn*pio2_1;
w = fn*pio2_1t; /* 1st round, good to 85 bits */
+ /* Matters with directed rounding. */
+ if (predict_false(r - w < -pio4)) {
+ n--;
+ fn--;
+ r = x - fn*pio2_1;
+ w = fn*pio2_1t;
+ } else if (predict_false(r - w > pio4)) {
+ n++;
+ fn++;
+ r = x - fn*pio2_1;
+ w = fn*pio2_1t;
+ }
y[0] = r - w;
u.f = y[0];
ey = u.i>>52 & 0x7ff;
diff --git a/src/math/__rem_pio2f.c b/src/math/__rem_pio2f.c
index 4473c1c4..e6765643 100644
--- a/src/math/__rem_pio2f.c
+++ b/src/math/__rem_pio2f.c
@@ -35,6 +35,7 @@
*/
static const double
toint = 1.5/EPS,
+pio4 = 0x1.921fb6p-1,
invpio2 = 6.36619772367581382433e-01, /* 0x3FE45F30, 0x6DC9C883 */
pio2_1 = 1.57079631090164184570e+00, /* 0x3FF921FB, 0x50000000 */
pio2_1t = 1.58932547735281966916e-08; /* 0x3E5110b4, 0x611A6263 */
@@ -50,10 +51,20 @@ int __rem_pio2f(float x, double *y)
ix = u.i & 0x7fffffff;
/* 25+53 bit pi is good enough for medium size */
if (ix < 0x4dc90fdb) { /* |x| ~< 2^28*(pi/2), medium size */
- /* Use a specialized rint() to get fn. Assume round-to-nearest. */
+ /* Use a specialized rint() to get fn. */
fn = (double_t)x*invpio2 + toint - toint;
n = (int32_t)fn;
*y = x - fn*pio2_1 - fn*pio2_1t;
+ /* Matters with directed rounding. */
+ if (predict_false(*y < -pio4)) {
+ n--;
+ fn--;
+ *y = x - fn*pio2_1 - fn*pio2_1t;
+ } else if (predict_false(*y > pio4)) {
+ n++;
+ fn++;
+ *y = x - fn*pio2_1 - fn*pio2_1t;
+ }
return n;
}
if(ix>=0x7f800000) { /* x is inf or NaN */
diff --git a/src/math/__rem_pio2l.c b/src/math/__rem_pio2l.c
index 77255bd8..236b2def 100644
--- a/src/math/__rem_pio2l.c
+++ b/src/math/__rem_pio2l.c
@@ -44,6 +44,7 @@ pio2_1 = 1.57079632679597125389e+00, /* 0x3FF921FB, 0x54444000 */
pio2_2 = -1.07463465549783099519e-12, /* -0x12e7b967674000.0p-92 */
pio2_3 = 6.36831716351370313614e-25; /* 0x18a2e037074000.0p-133 */
static const long double
+pio4 = 0x1.921fb54442d1846ap-1L,
invpio2 = 6.36619772367581343076e-01L, /* 0xa2f9836e4e44152a.0p-64 */
pio2_1t = -1.07463465549719416346e-12L, /* -0x973dcb3b399d747f.0p-103 */
pio2_2t = 6.36831716351095013979e-25L, /* 0xc51701b839a25205.0p-144 */
@@ -57,6 +58,7 @@ pio2_3t = -2.75299651904407171810e-37L; /* -0xbb5bf6c7ddd660ce.0p-185 */
#define NX 5
#define NY 3
static const long double
+pio4 = 0x1.921fb54442d18469898cc51701b8p-1L,
invpio2 = 6.3661977236758134307553505349005747e-01L, /* 0x145f306dc9c882a53f84eafa3ea6a.0p-113 */
pio2_1 = 1.5707963267948966192292994253909555e+00L, /* 0x1921fb54442d18469800000000000.0p-112 */
pio2_1t = 2.0222662487959507323996846200947577e-21L, /* 0x13198a2e03707344a4093822299f3.0p-181 */
@@ -76,11 +78,23 @@ int __rem_pio2l(long double x, long double *y)
u.f = x;
ex = u.i.se & 0x7fff;
if (SMALL(u)) {
- /* rint(x/(pi/2)), Assume round-to-nearest. */
+ /* rint(x/(pi/2)) */
fn = x*invpio2 + toint - toint;
n = QUOBITS(fn);
r = x-fn*pio2_1;
w = fn*pio2_1t; /* 1st round good to 102/180 bits (ld80/ld128) */
+ /* Matters with directed rounding. */
+ if (predict_false(r - w < -pio4)) {
+ n--;
+ fn--;
+ r = x - fn*pio2_1;
+ w = fn*pio2_1t;
+ } else if (predict_false(r - w > pio4)) {
+ n++;
+ fn++;
+ r = x - fn*pio2_1;
+ w = fn*pio2_1t;
+ }
y[0] = r-w;
u.f = y[0];
ey = u.i.se & 0x7fff;
diff --git a/src/math/cosh.c b/src/math/cosh.c
index 100f8231..490c15fb 100644
--- a/src/math/cosh.c
+++ b/src/math/cosh.c
@@ -35,6 +35,6 @@ double cosh(double x)
/* |x| > log(DBL_MAX) or nan */
/* note: the result is stored to handle overflow */
- t = __expo2(x);
+ t = __expo2(x, 1.0);
return t;
}
diff --git a/src/math/coshf.c b/src/math/coshf.c
index b09f2ee5..e739cff9 100644
--- a/src/math/coshf.c
+++ b/src/math/coshf.c
@@ -28,6 +28,6 @@ float coshf(float x)
}
/* |x| > log(FLT_MAX) or nan */
- t = __expo2f(x);
+ t = __expo2f(x, 1.0f);
return t;
}
diff --git a/src/math/i386/acos.s b/src/math/i386/acos.s
index 47f365ef..af423a2f 100644
--- a/src/math/i386/acos.s
+++ b/src/math/i386/acos.s
@@ -1,22 +1,10 @@
# use acos(x) = atan2(fabs(sqrt((1-x)*(1+x))), x)
-.global acosf
-.type acosf,@function
-acosf:
- flds 4(%esp)
- jmp 1f
-
-.global acosl
-.type acosl,@function
-acosl:
- fldt 4(%esp)
- jmp 1f
-
.global acos
.type acos,@function
acos:
fldl 4(%esp)
-1: fld %st(0)
+ fld %st(0)
fld1
fsub %st(0),%st(1)
fadd %st(2)
@@ -25,4 +13,6 @@ acos:
fabs # fix sign of zero (matters in downward rounding mode)
fxch %st(1)
fpatan
+ fstpl 4(%esp)
+ fldl 4(%esp)
ret
diff --git a/src/math/i386/acosf.s b/src/math/i386/acosf.s
index 6c95509f..d2cdfdbf 100644
--- a/src/math/i386/acosf.s
+++ b/src/math/i386/acosf.s
@@ -1 +1,16 @@
-# see acos.s
+.global acosf
+.type acosf,@function
+acosf:
+ flds 4(%esp)
+ fld %st(0)
+ fld1
+ fsub %st(0),%st(1)
+ fadd %st(2)
+ fmulp
+ fsqrt
+ fabs # fix sign of zero (matters in downward rounding mode)
+ fxch %st(1)
+ fpatan
+ fstps 4(%esp)
+ flds 4(%esp)
+ ret
diff --git a/src/math/i386/acosl.s b/src/math/i386/acosl.s
index 6c95509f..599c8230 100644
--- a/src/math/i386/acosl.s
+++ b/src/math/i386/acosl.s
@@ -1 +1,14 @@
-# see acos.s
+.global acosl
+.type acosl,@function
+acosl:
+ fldt 4(%esp)
+ fld %st(0)
+ fld1
+ fsub %st(0),%st(1)
+ fadd %st(2)
+ fmulp
+ fsqrt
+ fabs # fix sign of zero (matters in downward rounding mode)
+ fxch %st(1)
+ fpatan
+ ret
diff --git a/src/math/i386/asin.s b/src/math/i386/asin.s
index 920d967a..2bc8356f 100644
--- a/src/math/i386/asin.s
+++ b/src/math/i386/asin.s
@@ -1,23 +1,3 @@
-.global asinf
-.type asinf,@function
-asinf:
- flds 4(%esp)
- mov 4(%esp),%eax
- add %eax,%eax
- cmp $0x01000000,%eax
- jae 1f
- # subnormal x, return x with underflow
- fld %st(0)
- fmul %st(1)
- fstps 4(%esp)
- ret
-
-.global asinl
-.type asinl,@function
-asinl:
- fldt 4(%esp)
- jmp 1f
-
.global asin
.type asin,@function
asin:
@@ -25,15 +5,17 @@ asin:
mov 8(%esp),%eax
add %eax,%eax
cmp $0x00200000,%eax
- jae 1f
- # subnormal x, return x with underflow
- fsts 4(%esp)
- ret
-1: fld %st(0)
+ jb 1f
+ fld %st(0)
fld1
fsub %st(0),%st(1)
fadd %st(2)
fmulp
fsqrt
fpatan
+ fstpl 4(%esp)
+ fldl 4(%esp)
+ ret
+ # subnormal x, return x with underflow
+1: fsts 4(%esp)
ret
diff --git a/src/math/i386/asinf.s b/src/math/i386/asinf.s
index e07bf599..05909753 100644
--- a/src/math/i386/asinf.s
+++ b/src/math/i386/asinf.s
@@ -1 +1,23 @@
-# see asin.s
+.global asinf
+.type asinf,@function
+asinf:
+ flds 4(%esp)
+ mov 4(%esp),%eax
+ add %eax,%eax
+ cmp $0x01000000,%eax
+ jb 1f
+ fld %st(0)
+ fld1
+ fsub %st(0),%st(1)
+ fadd %st(2)
+ fmulp
+ fsqrt
+ fpatan
+ fstps 4(%esp)
+ flds 4(%esp)
+ ret
+ # subnormal x, return x with underflow
+1: fld %st(0)
+ fmul %st(1)
+ fstps 4(%esp)
+ ret
diff --git a/src/math/i386/asinl.s b/src/math/i386/asinl.s
index e07bf599..e973fc85 100644
--- a/src/math/i386/asinl.s
+++ b/src/math/i386/asinl.s
@@ -1 +1,12 @@
-# see asin.s
+.global asinl
+.type asinl,@function
+asinl:
+ fldt 4(%esp)
+ fld %st(0)
+ fld1
+ fsub %st(0),%st(1)
+ fadd %st(2)
+ fmulp
+ fsqrt
+ fpatan
+ ret
diff --git a/src/math/i386/atan.s b/src/math/i386/atan.s
index a26feae1..2c57f6b3 100644
--- a/src/math/i386/atan.s
+++ b/src/math/i386/atan.s
@@ -8,6 +8,8 @@ atan:
jb 1f
fld1
fpatan
+ fstpl 4(%esp)
+ fldl 4(%esp)
ret
# subnormal x, return x with underflow
1: fsts 4(%esp)
diff --git a/src/math/i386/atan2.s b/src/math/i386/atan2.s
index 76b95f31..8bc441b1 100644
--- a/src/math/i386/atan2.s
+++ b/src/math/i386/atan2.s
@@ -4,7 +4,8 @@ atan2:
fldl 4(%esp)
fldl 12(%esp)
fpatan
- fstl 4(%esp)
+ fstpl 4(%esp)
+ fldl 4(%esp)
mov 8(%esp),%eax
add %eax,%eax
cmp $0x00200000,%eax
diff --git a/src/math/i386/atan2f.s b/src/math/i386/atan2f.s
index c9408a90..3908c86d 100644
--- a/src/math/i386/atan2f.s
+++ b/src/math/i386/atan2f.s
@@ -4,7 +4,8 @@ atan2f:
flds 4(%esp)
flds 8(%esp)
fpatan
- fsts 4(%esp)
+ fstps 4(%esp)
+ flds 4(%esp)
mov 4(%esp),%eax
add %eax,%eax
cmp $0x01000000,%eax
diff --git a/src/math/i386/atanf.s b/src/math/i386/atanf.s
index 893beac5..c2cbe2e0 100644
--- a/src/math/i386/atanf.s
+++ b/src/math/i386/atanf.s
@@ -8,6 +8,8 @@ atanf:
jb 1f
fld1
fpatan
+ fstps 4(%esp)
+ flds 4(%esp)
ret
# subnormal x, return x with underflow
1: fld %st(0)
diff --git a/src/math/i386/exp2.s b/src/math/i386/exp2.s
deleted file mode 100644
index f335a3e5..00000000
--- a/src/math/i386/exp2.s
+++ /dev/null
@@ -1 +0,0 @@
-# see exp.s
diff --git a/src/math/i386/exp2f.s b/src/math/i386/exp2f.s
deleted file mode 100644
index f335a3e5..00000000
--- a/src/math/i386/exp2f.s
+++ /dev/null
@@ -1 +0,0 @@
-# see exp.s
diff --git a/src/math/i386/exp2l.s b/src/math/i386/exp2l.s
index f335a3e5..8125761d 100644
--- a/src/math/i386/exp2l.s
+++ b/src/math/i386/exp2l.s
@@ -1 +1 @@
-# see exp.s
+# see exp_ld.s
diff --git a/src/math/i386/exp.s b/src/math/i386/exp_ld.s
index df87c497..99cba01f 100644
--- a/src/math/i386/exp.s
+++ b/src/math/i386/exp_ld.s
@@ -1,35 +1,8 @@
-.global expm1f
-.type expm1f,@function
-expm1f:
- flds 4(%esp)
- mov 4(%esp),%eax
- add %eax,%eax
- cmp $0x01000000,%eax
- jae 1f
- # subnormal x, return x with underflow
- fld %st(0)
- fmul %st(1)
- fstps 4(%esp)
- ret
-
.global expm1l
.type expm1l,@function
expm1l:
fldt 4(%esp)
- jmp 1f
-
-.global expm1
-.type expm1,@function
-expm1:
- fldl 4(%esp)
- mov 8(%esp),%eax
- add %eax,%eax
- cmp $0x00200000,%eax
- jae 1f
- # subnormal x, return x with underflow
- fsts 4(%esp)
- ret
-1: fldl2e
+ fldl2e
fmulp
mov $0xc2820000,%eax
push %eax
@@ -59,12 +32,6 @@ expm1:
fsubrp
ret
-.global exp2f
-.type exp2f,@function
-exp2f:
- flds 4(%esp)
- jmp 1f
-
.global exp2l
.global __exp2l
.hidden __exp2l
@@ -72,26 +39,6 @@ exp2f:
exp2l:
__exp2l:
fldt 4(%esp)
- jmp 1f
-
-.global expf
-.type expf,@function
-expf:
- flds 4(%esp)
- jmp 2f
-
-.global exp
-.type exp,@function
-exp:
- fldl 4(%esp)
-2: fldl2e
- fmulp
- jmp 1f
-
-.global exp2
-.type exp2,@function
-exp2:
- fldl 4(%esp)
1: sub $12,%esp
fld %st(0)
fstpt (%esp)
diff --git a/src/math/i386/expf.s b/src/math/i386/expf.s
deleted file mode 100644
index f335a3e5..00000000
--- a/src/math/i386/expf.s
+++ /dev/null
@@ -1 +0,0 @@
-# see exp.s
diff --git a/src/math/i386/expm1.s b/src/math/i386/expm1.s
deleted file mode 100644
index f335a3e5..00000000
--- a/src/math/i386/expm1.s
+++ /dev/null
@@ -1 +0,0 @@
-# see exp.s
diff --git a/src/math/i386/expm1f.s b/src/math/i386/expm1f.s
deleted file mode 100644
index f335a3e5..00000000
--- a/src/math/i386/expm1f.s
+++ /dev/null
@@ -1 +0,0 @@
-# see exp.s
diff --git a/src/math/i386/expm1l.s b/src/math/i386/expm1l.s
index f335a3e5..8125761d 100644
--- a/src/math/i386/expm1l.s
+++ b/src/math/i386/expm1l.s
@@ -1 +1 @@
-# see exp.s
+# see exp_ld.s
diff --git a/src/math/i386/fabs.c b/src/math/i386/fabs.c
new file mode 100644
index 00000000..39672786
--- /dev/null
+++ b/src/math/i386/fabs.c
@@ -0,0 +1,7 @@
+#include <math.h>
+
+double fabs(double x)
+{
+ __asm__ ("fabs" : "+t"(x));
+ return x;
+}
diff --git a/src/math/i386/fabs.s b/src/math/i386/fabs.s
deleted file mode 100644
index d66ea9a1..00000000
--- a/src/math/i386/fabs.s
+++ /dev/null
@@ -1,6 +0,0 @@
-.global fabs
-.type fabs,@function
-fabs:
- fldl 4(%esp)
- fabs
- ret
diff --git a/src/math/i386/fabsf.c b/src/math/i386/fabsf.c
new file mode 100644
index 00000000..d882eee3
--- /dev/null
+++ b/src/math/i386/fabsf.c
@@ -0,0 +1,7 @@
+#include <math.h>
+
+float fabsf(float x)
+{
+ __asm__ ("fabs" : "+t"(x));
+ return x;
+}
diff --git a/src/math/i386/fabsf.s b/src/math/i386/fabsf.s
deleted file mode 100644
index a981c422..00000000
--- a/src/math/i386/fabsf.s
+++ /dev/null
@@ -1,6 +0,0 @@
-.global fabsf
-.type fabsf,@function
-fabsf:
- flds 4(%esp)
- fabs
- ret
diff --git a/src/math/i386/fabsl.c b/src/math/i386/fabsl.c
new file mode 100644
index 00000000..cc1c9ed9
--- /dev/null
+++ b/src/math/i386/fabsl.c
@@ -0,0 +1,7 @@
+#include <math.h>
+
+long double fabsl(long double x)
+{
+ __asm__ ("fabs" : "+t"(x));
+ return x;
+}
diff --git a/src/math/i386/fabsl.s b/src/math/i386/fabsl.s
deleted file mode 100644
index ceef9e4c..00000000
--- a/src/math/i386/fabsl.s
+++ /dev/null
@@ -1,6 +0,0 @@
-.global fabsl
-.type fabsl,@function
-fabsl:
- fldt 4(%esp)
- fabs
- ret
diff --git a/src/math/i386/fmod.c b/src/math/i386/fmod.c
new file mode 100644
index 00000000..ea0c58d9
--- /dev/null
+++ b/src/math/i386/fmod.c
@@ -0,0 +1,10 @@
+#include <math.h>
+
+double fmod(double x, double y)
+{
+ unsigned short fpsr;
+ // fprem does not introduce excess precision into x
+ do __asm__ ("fprem; fnstsw %%ax" : "+t"(x), "=a"(fpsr) : "u"(y));
+ while (fpsr & 0x400);
+ return x;
+}
diff --git a/src/math/i386/fmod.s b/src/math/i386/fmod.s
deleted file mode 100644
index 2113b3c5..00000000
--- a/src/math/i386/fmod.s
+++ /dev/null
@@ -1,11 +0,0 @@
-.global fmod
-.type fmod,@function
-fmod:
- fldl 12(%esp)
- fldl 4(%esp)
-1: fprem
- fnstsw %ax
- sahf
- jp 1b
- fstp %st(1)
- ret
diff --git a/src/math/i386/fmodf.c b/src/math/i386/fmodf.c
new file mode 100644
index 00000000..90b56ab0
--- /dev/null
+++ b/src/math/i386/fmodf.c
@@ -0,0 +1,10 @@
+#include <math.h>
+
+float fmodf(float x, float y)
+{
+ unsigned short fpsr;
+ // fprem does not introduce excess precision into x
+ do __asm__ ("fprem; fnstsw %%ax" : "+t"(x), "=a"(fpsr) : "u"(y));
+ while (fpsr & 0x400);
+ return x;
+}
diff --git a/src/math/i386/fmodf.s b/src/math/i386/fmodf.s
deleted file mode 100644
index e04e2a56..00000000
--- a/src/math/i386/fmodf.s
+++ /dev/null
@@ -1,11 +0,0 @@
-.global fmodf
-.type fmodf,@function
-fmodf:
- flds 8(%esp)
- flds 4(%esp)
-1: fprem
- fnstsw %ax
- sahf
- jp 1b
- fstp %st(1)
- ret
diff --git a/src/math/i386/fmodl.c b/src/math/i386/fmodl.c
new file mode 100644
index 00000000..3daeab06
--- /dev/null
+++ b/src/math/i386/fmodl.c
@@ -0,0 +1,9 @@
+#include <math.h>
+
+long double fmodl(long double x, long double y)
+{
+ unsigned short fpsr;
+ do __asm__ ("fprem; fnstsw %%ax" : "+t"(x), "=a"(fpsr) : "u"(y));
+ while (fpsr & 0x400);
+ return x;
+}
diff --git a/src/math/i386/fmodl.s b/src/math/i386/fmodl.s
deleted file mode 100644
index 0cb3fe9b..00000000
--- a/src/math/i386/fmodl.s
+++ /dev/null
@@ -1,11 +0,0 @@
-.global fmodl
-.type fmodl,@function
-fmodl:
- fldt 16(%esp)
- fldt 4(%esp)
-1: fprem
- fnstsw %ax
- sahf
- jp 1b
- fstp %st(1)
- ret
diff --git a/src/math/i386/llrint.c b/src/math/i386/llrint.c
new file mode 100644
index 00000000..aa400817
--- /dev/null
+++ b/src/math/i386/llrint.c
@@ -0,0 +1,8 @@
+#include <math.h>
+
+long long llrint(double x)
+{
+ long long r;
+ __asm__ ("fistpll %0" : "=m"(r) : "t"(x) : "st");
+ return r;
+}
diff --git a/src/math/i386/llrint.s b/src/math/i386/llrint.s
deleted file mode 100644
index 8e89cd91..00000000
--- a/src/math/i386/llrint.s
+++ /dev/null
@@ -1,8 +0,0 @@
-.global llrint
-.type llrint,@function
-llrint:
- fldl 4(%esp)
- fistpll 4(%esp)
- mov 4(%esp),%eax
- mov 8(%esp),%edx
- ret
diff --git a/src/math/i386/llrintf.c b/src/math/i386/llrintf.c
new file mode 100644
index 00000000..c41a317b
--- /dev/null
+++ b/src/math/i386/llrintf.c
@@ -0,0 +1,8 @@
+#include <math.h>
+
+long long llrintf(float x)
+{
+ long long r;
+ __asm__ ("fistpll %0" : "=m"(r) : "t"(x) : "st");
+ return r;
+}
diff --git a/src/math/i386/llrintf.s b/src/math/i386/llrintf.s
deleted file mode 100644
index aa850c6c..00000000
--- a/src/math/i386/llrintf.s
+++ /dev/null
@@ -1,9 +0,0 @@
-.global llrintf
-.type llrintf,@function
-llrintf:
- sub $8,%esp
- flds 12(%esp)
- fistpll (%esp)
- pop %eax
- pop %edx
- ret
diff --git a/src/math/i386/llrintl.c b/src/math/i386/llrintl.c
new file mode 100644
index 00000000..c439ef28
--- /dev/null
+++ b/src/math/i386/llrintl.c
@@ -0,0 +1,8 @@
+#include <math.h>
+
+long long llrintl(long double x)
+{
+ long long r;
+ __asm__ ("fistpll %0" : "=m"(r) : "t"(x) : "st");
+ return r;
+}
diff --git a/src/math/i386/llrintl.s b/src/math/i386/llrintl.s
deleted file mode 100644
index 1cfb56f1..00000000
--- a/src/math/i386/llrintl.s
+++ /dev/null
@@ -1,8 +0,0 @@
-.global llrintl
-.type llrintl,@function
-llrintl:
- fldt 4(%esp)
- fistpll 4(%esp)
- mov 4(%esp),%eax
- mov 8(%esp),%edx
- ret
diff --git a/src/math/i386/log.s b/src/math/i386/log.s
index fcccf030..08c59924 100644
--- a/src/math/i386/log.s
+++ b/src/math/i386/log.s
@@ -4,4 +4,6 @@ log:
fldln2
fldl 4(%esp)
fyl2x
+ fstpl 4(%esp)
+ fldl 4(%esp)
ret
diff --git a/src/math/i386/log10.s b/src/math/i386/log10.s
index 28eb5b2f..120e91ec 100644
--- a/src/math/i386/log10.s
+++ b/src/math/i386/log10.s
@@ -4,4 +4,6 @@ log10:
fldlg2
fldl 4(%esp)
fyl2x
+ fstpl 4(%esp)
+ fldl 4(%esp)
ret
diff --git a/src/math/i386/log10f.s b/src/math/i386/log10f.s
index c0c0c67e..b055493a 100644
--- a/src/math/i386/log10f.s
+++ b/src/math/i386/log10f.s
@@ -4,4 +4,6 @@ log10f:
fldlg2
flds 4(%esp)
fyl2x
+ fstps 4(%esp)
+ flds 4(%esp)
ret
diff --git a/src/math/i386/log1p.s b/src/math/i386/log1p.s
index 354f391a..f3c95f83 100644
--- a/src/math/i386/log1p.s
+++ b/src/math/i386/log1p.s
@@ -10,10 +10,14 @@ log1p:
cmp $0x00100000,%eax
jb 2f
fyl2xp1
+ fstpl 4(%esp)
+ fldl 4(%esp)
ret
1: fld1
faddp
fyl2x
+ fstpl 4(%esp)
+ fldl 4(%esp)
ret
# subnormal x, return x with underflow
2: fsts 4(%esp)
diff --git a/src/math/i386/log1pf.s b/src/math/i386/log1pf.s
index 4d3484cd..9f13d95f 100644
--- a/src/math/i386/log1pf.s
+++ b/src/math/i386/log1pf.s
@@ -10,10 +10,14 @@ log1pf:
cmp $0x00800000,%eax
jb 2f
fyl2xp1
+ fstps 4(%esp)
+ flds 4(%esp)
ret
1: fld1
faddp
fyl2x
+ fstps 4(%esp)
+ flds 4(%esp)
ret
# subnormal x, return x with underflow
2: fxch
diff --git a/src/math/i386/log2.s b/src/math/i386/log2.s
index 15088037..7eff0b61 100644
--- a/src/math/i386/log2.s
+++ b/src/math/i386/log2.s
@@ -4,4 +4,6 @@ log2:
fld1
fldl 4(%esp)
fyl2x
+ fstpl 4(%esp)
+ fldl 4(%esp)
ret
diff --git a/src/math/i386/log2f.s b/src/math/i386/log2f.s
index 00cdce75..b32fa2f7 100644
--- a/src/math/i386/log2f.s
+++ b/src/math/i386/log2f.s
@@ -4,4 +4,6 @@ log2f:
fld1
flds 4(%esp)
fyl2x
+ fstps 4(%esp)
+ flds 4(%esp)
ret
diff --git a/src/math/i386/logf.s b/src/math/i386/logf.s
index da7ff3ae..4d0346a4 100644
--- a/src/math/i386/logf.s
+++ b/src/math/i386/logf.s
@@ -4,4 +4,6 @@ logf:
fldln2
flds 4(%esp)
fyl2x
+ fstps 4(%esp)
+ flds 4(%esp)
ret
diff --git a/src/math/i386/lrint.c b/src/math/i386/lrint.c
new file mode 100644
index 00000000..89563ab2
--- /dev/null
+++ b/src/math/i386/lrint.c
@@ -0,0 +1,8 @@
+#include <math.h>
+
+long lrint(double x)
+{
+ long r;
+ __asm__ ("fistpl %0" : "=m"(r) : "t"(x) : "st");
+ return r;
+}
diff --git a/src/math/i386/lrint.s b/src/math/i386/lrint.s
deleted file mode 100644
index 02b83d9f..00000000
--- a/src/math/i386/lrint.s
+++ /dev/null
@@ -1,7 +0,0 @@
-.global lrint
-.type lrint,@function
-lrint:
- fldl 4(%esp)
- fistpl 4(%esp)
- mov 4(%esp),%eax
- ret
diff --git a/src/math/i386/lrintf.c b/src/math/i386/lrintf.c
new file mode 100644
index 00000000..0bbf29de
--- /dev/null
+++ b/src/math/i386/lrintf.c
@@ -0,0 +1,8 @@
+#include <math.h>
+
+long lrintf(float x)
+{
+ long r;
+ __asm__ ("fistpl %0" : "=m"(r) : "t"(x) : "st");
+ return r;
+}
diff --git a/src/math/i386/lrintf.s b/src/math/i386/lrintf.s
deleted file mode 100644
index 907aac29..00000000
--- a/src/math/i386/lrintf.s
+++ /dev/null
@@ -1,7 +0,0 @@
-.global lrintf
-.type lrintf,@function
-lrintf:
- flds 4(%esp)
- fistpl 4(%esp)
- mov 4(%esp),%eax
- ret
diff --git a/src/math/i386/lrintl.c b/src/math/i386/lrintl.c
new file mode 100644
index 00000000..eb8c0902
--- /dev/null
+++ b/src/math/i386/lrintl.c
@@ -0,0 +1,8 @@
+#include <math.h>
+
+long lrintl(long double x)
+{
+ long r;
+ __asm__ ("fistpl %0" : "=m"(r) : "t"(x) : "st");
+ return r;
+}
diff --git a/src/math/i386/lrintl.s b/src/math/i386/lrintl.s
deleted file mode 100644
index 3ae05aac..00000000
--- a/src/math/i386/lrintl.s
+++ /dev/null
@@ -1,7 +0,0 @@
-.global lrintl
-.type lrintl,@function
-lrintl:
- fldt 4(%esp)
- fistpl 4(%esp)
- mov 4(%esp),%eax
- ret
diff --git a/src/math/i386/remainder.c b/src/math/i386/remainder.c
new file mode 100644
index 00000000..c083df90
--- /dev/null
+++ b/src/math/i386/remainder.c
@@ -0,0 +1,12 @@
+#include <math.h>
+
+double remainder(double x, double y)
+{
+ unsigned short fpsr;
+ // fprem1 does not introduce excess precision into x
+ do __asm__ ("fprem1; fnstsw %%ax" : "+t"(x), "=a"(fpsr) : "u"(y));
+ while (fpsr & 0x400);
+ return x;
+}
+
+weak_alias(remainder, drem);
diff --git a/src/math/i386/remainder.s b/src/math/i386/remainder.s
deleted file mode 100644
index ab1da95d..00000000
--- a/src/math/i386/remainder.s
+++ /dev/null
@@ -1,14 +0,0 @@
-.global remainder
-.type remainder,@function
-remainder:
-.weak drem
-.type drem,@function
-drem:
- fldl 12(%esp)
- fldl 4(%esp)
-1: fprem1
- fnstsw %ax
- sahf
- jp 1b
- fstp %st(1)
- ret
diff --git a/src/math/i386/remainderf.c b/src/math/i386/remainderf.c
new file mode 100644
index 00000000..280207d2
--- /dev/null
+++ b/src/math/i386/remainderf.c
@@ -0,0 +1,12 @@
+#include <math.h>
+
+float remainderf(float x, float y)
+{
+ unsigned short fpsr;
+ // fprem1 does not introduce excess precision into x
+ do __asm__ ("fprem1; fnstsw %%ax" : "+t"(x), "=a"(fpsr) : "u"(y));
+ while (fpsr & 0x400);
+ return x;
+}
+
+weak_alias(remainderf, dremf);
diff --git a/src/math/i386/remainderf.s b/src/math/i386/remainderf.s
deleted file mode 100644
index 6a7378a3..00000000
--- a/src/math/i386/remainderf.s
+++ /dev/null
@@ -1,14 +0,0 @@
-.global remainderf
-.type remainderf,@function
-remainderf:
-.weak dremf
-.type dremf,@function
-dremf:
- flds 8(%esp)
- flds 4(%esp)
-1: fprem1
- fnstsw %ax
- sahf
- jp 1b
- fstp %st(1)
- ret
diff --git a/src/math/i386/remainderl.c b/src/math/i386/remainderl.c
new file mode 100644
index 00000000..8cf75071
--- /dev/null
+++ b/src/math/i386/remainderl.c
@@ -0,0 +1,9 @@
+#include <math.h>
+
+long double remainderl(long double x, long double y)
+{
+ unsigned short fpsr;
+ do __asm__ ("fprem1; fnstsw %%ax" : "+t"(x), "=a"(fpsr) : "u"(y));
+ while (fpsr & 0x400);
+ return x;
+}
diff --git a/src/math/i386/remainderl.s b/src/math/i386/remainderl.s
deleted file mode 100644
index b41518ed..00000000
--- a/src/math/i386/remainderl.s
+++ /dev/null
@@ -1,11 +0,0 @@
-.global remainderl
-.type remainderl,@function
-remainderl:
- fldt 16(%esp)
- fldt 4(%esp)
-1: fprem1
- fnstsw %ax
- sahf
- jp 1b
- fstp %st(1)
- ret
diff --git a/src/math/i386/rint.c b/src/math/i386/rint.c
new file mode 100644
index 00000000..a5276a60
--- /dev/null
+++ b/src/math/i386/rint.c
@@ -0,0 +1,7 @@
+#include <math.h>
+
+double rint(double x)
+{
+ __asm__ ("frndint" : "+t"(x));
+ return x;
+}
diff --git a/src/math/i386/rint.s b/src/math/i386/rint.s
deleted file mode 100644
index bb99a11c..00000000
--- a/src/math/i386/rint.s
+++ /dev/null
@@ -1,6 +0,0 @@
-.global rint
-.type rint,@function
-rint:
- fldl 4(%esp)
- frndint
- ret
diff --git a/src/math/i386/rintf.c b/src/math/i386/rintf.c
new file mode 100644
index 00000000..bb4121a4
--- /dev/null
+++ b/src/math/i386/rintf.c
@@ -0,0 +1,7 @@
+#include <math.h>
+
+float rintf(float x)
+{
+ __asm__ ("frndint" : "+t"(x));
+ return x;
+}
diff --git a/src/math/i386/rintf.s b/src/math/i386/rintf.s
deleted file mode 100644
index bce4c5a6..00000000
--- a/src/math/i386/rintf.s
+++ /dev/null
@@ -1,6 +0,0 @@
-.global rintf
-.type rintf,@function
-rintf:
- flds 4(%esp)
- frndint
- ret
diff --git a/src/math/i386/rintl.c b/src/math/i386/rintl.c
new file mode 100644
index 00000000..e1a92077
--- /dev/null
+++ b/src/math/i386/rintl.c
@@ -0,0 +1,7 @@
+#include <math.h>
+
+long double rintl(long double x)
+{
+ __asm__ ("frndint" : "+t"(x));
+ return x;
+}
diff --git a/src/math/i386/rintl.s b/src/math/i386/rintl.s
deleted file mode 100644
index cd2bf9a9..00000000
--- a/src/math/i386/rintl.s
+++ /dev/null
@@ -1,6 +0,0 @@
-.global rintl
-.type rintl,@function
-rintl:
- fldt 4(%esp)
- frndint
- ret
diff --git a/src/math/i386/sqrt.c b/src/math/i386/sqrt.c
new file mode 100644
index 00000000..934fbcca
--- /dev/null
+++ b/src/math/i386/sqrt.c
@@ -0,0 +1,15 @@
+#include "libm.h"
+
+double sqrt(double x)
+{
+ union ldshape ux;
+ unsigned fpsr;
+ __asm__ ("fsqrt; fnstsw %%ax": "=t"(ux.f), "=a"(fpsr) : "0"(x));
+ if ((ux.i.m & 0x7ff) != 0x400)
+ return (double)ux.f;
+ /* Rounding to double would have encountered an exact halfway case.
+ Adjust mantissa downwards if fsqrt rounded up, else upwards.
+ (result of fsqrt could not have been exact) */
+ ux.i.m ^= (fpsr & 0x200) + 0x300;
+ return (double)ux.f;
+}
diff --git a/src/math/i386/sqrt.s b/src/math/i386/sqrt.s
deleted file mode 100644
index 57837e25..00000000
--- a/src/math/i386/sqrt.s
+++ /dev/null
@@ -1,21 +0,0 @@
-.global sqrt
-.type sqrt,@function
-sqrt: fldl 4(%esp)
- fsqrt
- fnstsw %ax
- sub $12,%esp
- fld %st(0)
- fstpt (%esp)
- mov (%esp),%ecx
- and $0x7ff,%ecx
- cmp $0x400,%ecx
- jnz 1f
- and $0x200,%eax
- sub $0x100,%eax
- sub %eax,(%esp)
- fstp %st(0)
- fldt (%esp)
-1: add $12,%esp
- fstpl 4(%esp)
- fldl 4(%esp)
- ret
diff --git a/src/math/i386/sqrtf.c b/src/math/i386/sqrtf.c
new file mode 100644
index 00000000..41c65c2b
--- /dev/null
+++ b/src/math/i386/sqrtf.c
@@ -0,0 +1,12 @@
+#include <math.h>
+
+float sqrtf(float x)
+{
+ long double t;
+ /* The long double result has sufficient precision so that
+ * second rounding to float still keeps the returned value
+ * correctly rounded, see Pierre Roux, "Innocuous Double
+ * Rounding of Basic Arithmetic Operations". */
+ __asm__ ("fsqrt" : "=t"(t) : "0"(x));
+ return (float)t;
+}
diff --git a/src/math/i386/sqrtf.s b/src/math/i386/sqrtf.s
deleted file mode 100644
index 9e944f45..00000000
--- a/src/math/i386/sqrtf.s
+++ /dev/null
@@ -1,7 +0,0 @@
-.global sqrtf
-.type sqrtf,@function
-sqrtf: flds 4(%esp)
- fsqrt
- fstps 4(%esp)
- flds 4(%esp)
- ret
diff --git a/src/math/i386/sqrtl.c b/src/math/i386/sqrtl.c
new file mode 100644
index 00000000..864cfcc4
--- /dev/null
+++ b/src/math/i386/sqrtl.c
@@ -0,0 +1,7 @@
+#include <math.h>
+
+long double sqrtl(long double x)
+{
+ __asm__ ("fsqrt" : "+t"(x));
+ return x;
+}
diff --git a/src/math/i386/sqrtl.s b/src/math/i386/sqrtl.s
deleted file mode 100644
index e0d42616..00000000
--- a/src/math/i386/sqrtl.s
+++ /dev/null
@@ -1,5 +0,0 @@
-.global sqrtl
-.type sqrtl,@function
-sqrtl: fldt 4(%esp)
- fsqrt
- ret
diff --git a/src/math/m68k/sqrtl.c b/src/math/m68k/sqrtl.c
new file mode 100644
index 00000000..b1c303c7
--- /dev/null
+++ b/src/math/m68k/sqrtl.c
@@ -0,0 +1,15 @@
+#include <math.h>
+
+#if __HAVE_68881__
+
+long double sqrtl(long double x)
+{
+ __asm__ ("fsqrt.x %1,%0" : "=f"(x) : "fm"(x));
+ return x;
+}
+
+#else
+
+#include "../sqrtl.c"
+
+#endif
diff --git a/src/math/powerpc/fabs.c b/src/math/powerpc/fabs.c
index f6ec4433..0efc21ef 100644
--- a/src/math/powerpc/fabs.c
+++ b/src/math/powerpc/fabs.c
@@ -1,6 +1,6 @@
#include <math.h>
-#ifdef _SOFT_FLOAT
+#if defined(_SOFT_FLOAT) || defined(BROKEN_PPC_D_ASM)
#include "../fabs.c"
diff --git a/src/math/powerpc/fma.c b/src/math/powerpc/fma.c
index fd268f5f..135c9903 100644
--- a/src/math/powerpc/fma.c
+++ b/src/math/powerpc/fma.c
@@ -1,6 +1,6 @@
#include <math.h>
-#ifdef _SOFT_FLOAT
+#if defined(_SOFT_FLOAT) || defined(BROKEN_PPC_D_ASM)
#include "../fma.c"
diff --git a/src/math/sinh.c b/src/math/sinh.c
index 00022c4e..a01951ae 100644
--- a/src/math/sinh.c
+++ b/src/math/sinh.c
@@ -34,6 +34,6 @@ double sinh(double x)
/* |x| > log(DBL_MAX) or nan */
/* note: the result is stored to handle overflow */
- t = 2*h*__expo2(absx);
+ t = __expo2(absx, 2*h);
return t;
}
diff --git a/src/math/sinhf.c b/src/math/sinhf.c
index 6ad19ea2..b9caa793 100644
--- a/src/math/sinhf.c
+++ b/src/math/sinhf.c
@@ -26,6 +26,6 @@ float sinhf(float x)
}
/* |x| > logf(FLT_MAX) or nan */
- t = 2*h*__expo2f(absx);
+ t = __expo2f(absx, 2*h);
return t;
}
diff --git a/src/math/sqrt.c b/src/math/sqrt.c
index f1f6d76c..5ba26559 100644
--- a/src/math/sqrt.c
+++ b/src/math/sqrt.c
@@ -1,184 +1,158 @@
-/* origin: FreeBSD /usr/src/lib/msun/src/e_sqrt.c */
-/*
- * ====================================================
- * Copyright (C) 1993 by Sun Microsystems, Inc. All rights reserved.
- *
- * Developed at SunSoft, a Sun Microsystems, Inc. business.
- * Permission to use, copy, modify, and distribute this
- * software is freely granted, provided that this notice
- * is preserved.
- * ====================================================
- */
-/* sqrt(x)
- * Return correctly rounded sqrt.
- * ------------------------------------------
- * | Use the hardware sqrt if you have one |
- * ------------------------------------------
- * Method:
- * Bit by bit method using integer arithmetic. (Slow, but portable)
- * 1. Normalization
- * Scale x to y in [1,4) with even powers of 2:
- * find an integer k such that 1 <= (y=x*2^(2k)) < 4, then
- * sqrt(x) = 2^k * sqrt(y)
- * 2. Bit by bit computation
- * Let q = sqrt(y) truncated to i bit after binary point (q = 1),
- * i 0
- * i+1 2
- * s = 2*q , and y = 2 * ( y - q ). (1)
- * i i i i
- *
- * To compute q from q , one checks whether
- * i+1 i
- *
- * -(i+1) 2
- * (q + 2 ) <= y. (2)
- * i
- * -(i+1)
- * If (2) is false, then q = q ; otherwise q = q + 2 .
- * i+1 i i+1 i
- *
- * With some algebric manipulation, it is not difficult to see
- * that (2) is equivalent to
- * -(i+1)
- * s + 2 <= y (3)
- * i i
- *
- * The advantage of (3) is that s and y can be computed by
- * i i
- * the following recurrence formula:
- * if (3) is false
- *
- * s = s , y = y ; (4)
- * i+1 i i+1 i
- *
- * otherwise,
- * -i -(i+1)
- * s = s + 2 , y = y - s - 2 (5)
- * i+1 i i+1 i i
- *
- * One may easily use induction to prove (4) and (5).
- * Note. Since the left hand side of (3) contain only i+2 bits,
- * it does not necessary to do a full (53-bit) comparison
- * in (3).
- * 3. Final rounding
- * After generating the 53 bits result, we compute one more bit.
- * Together with the remainder, we can decide whether the
- * result is exact, bigger than 1/2ulp, or less than 1/2ulp
- * (it will never equal to 1/2ulp).
- * The rounding mode can be detected by checking whether
- * huge + tiny is equal to huge, and whether huge - tiny is
- * equal to huge for some floating point number "huge" and "tiny".
- *
- * Special cases:
- * sqrt(+-0) = +-0 ... exact
- * sqrt(inf) = inf
- * sqrt(-ve) = NaN ... with invalid signal
- * sqrt(NaN) = NaN ... with invalid signal for signaling NaN
- */
-
+#include <stdint.h>
+#include <math.h>
#include "libm.h"
+#include "sqrt_data.h"
-static const double tiny = 1.0e-300;
+#define FENV_SUPPORT 1
-double sqrt(double x)
+/* returns a*b*2^-32 - e, with error 0 <= e < 1. */
+static inline uint32_t mul32(uint32_t a, uint32_t b)
{
- double z;
- int32_t sign = (int)0x80000000;
- int32_t ix0,s0,q,m,t,i;
- uint32_t r,t1,s1,ix1,q1;
+ return (uint64_t)a*b >> 32;
+}
- EXTRACT_WORDS(ix0, ix1, x);
+/* returns a*b*2^-64 - e, with error 0 <= e < 3. */
+static inline uint64_t mul64(uint64_t a, uint64_t b)
+{
+ uint64_t ahi = a>>32;
+ uint64_t alo = a&0xffffffff;
+ uint64_t bhi = b>>32;
+ uint64_t blo = b&0xffffffff;
+ return ahi*bhi + (ahi*blo >> 32) + (alo*bhi >> 32);
+}
- /* take care of Inf and NaN */
- if ((ix0&0x7ff00000) == 0x7ff00000) {
- return x*x + x; /* sqrt(NaN)=NaN, sqrt(+inf)=+inf, sqrt(-inf)=sNaN */
- }
- /* take care of zero */
- if (ix0 <= 0) {
- if (((ix0&~sign)|ix1) == 0)
- return x; /* sqrt(+-0) = +-0 */
- if (ix0 < 0)
- return (x-x)/(x-x); /* sqrt(-ve) = sNaN */
- }
- /* normalize x */
- m = ix0>>20;
- if (m == 0) { /* subnormal x */
- while (ix0 == 0) {
- m -= 21;
- ix0 |= (ix1>>11);
- ix1 <<= 21;
- }
- for (i=0; (ix0&0x00100000) == 0; i++)
- ix0<<=1;
- m -= i - 1;
- ix0 |= ix1>>(32-i);
- ix1 <<= i;
- }
- m -= 1023; /* unbias exponent */
- ix0 = (ix0&0x000fffff)|0x00100000;
- if (m & 1) { /* odd m, double x to make it even */
- ix0 += ix0 + ((ix1&sign)>>31);
- ix1 += ix1;
- }
- m >>= 1; /* m = [m/2] */
-
- /* generate sqrt(x) bit by bit */
- ix0 += ix0 + ((ix1&sign)>>31);
- ix1 += ix1;
- q = q1 = s0 = s1 = 0; /* [q,q1] = sqrt(x) */
- r = 0x00200000; /* r = moving bit from right to left */
-
- while (r != 0) {
- t = s0 + r;
- if (t <= ix0) {
- s0 = t + r;
- ix0 -= t;
- q += r;
- }
- ix0 += ix0 + ((ix1&sign)>>31);
- ix1 += ix1;
- r >>= 1;
- }
+double sqrt(double x)
+{
+ uint64_t ix, top, m;
- r = sign;
- while (r != 0) {
- t1 = s1 + r;
- t = s0;
- if (t < ix0 || (t == ix0 && t1 <= ix1)) {
- s1 = t1 + r;
- if ((t1&sign) == sign && (s1&sign) == 0)
- s0++;
- ix0 -= t;
- if (ix1 < t1)
- ix0--;
- ix1 -= t1;
- q1 += r;
- }
- ix0 += ix0 + ((ix1&sign)>>31);
- ix1 += ix1;
- r >>= 1;
+ /* special case handling. */
+ ix = asuint64(x);
+ top = ix >> 52;
+ if (predict_false(top - 0x001 >= 0x7ff - 0x001)) {
+ /* x < 0x1p-1022 or inf or nan. */
+ if (ix * 2 == 0)
+ return x;
+ if (ix == 0x7ff0000000000000)
+ return x;
+ if (ix > 0x7ff0000000000000)
+ return __math_invalid(x);
+ /* x is subnormal, normalize it. */
+ ix = asuint64(x * 0x1p52);
+ top = ix >> 52;
+ top -= 52;
}
- /* use floating add to find out rounding direction */
- if ((ix0|ix1) != 0) {
- z = 1.0 - tiny; /* raise inexact flag */
- if (z >= 1.0) {
- z = 1.0 + tiny;
- if (q1 == (uint32_t)0xffffffff) {
- q1 = 0;
- q++;
- } else if (z > 1.0) {
- if (q1 == (uint32_t)0xfffffffe)
- q++;
- q1 += 2;
- } else
- q1 += q1 & 1;
- }
+ /* argument reduction:
+ x = 4^e m; with integer e, and m in [1, 4)
+ m: fixed point representation [2.62]
+ 2^e is the exponent part of the result. */
+ int even = top & 1;
+ m = (ix << 11) | 0x8000000000000000;
+ if (even) m >>= 1;
+ top = (top + 0x3ff) >> 1;
+
+ /* approximate r ~ 1/sqrt(m) and s ~ sqrt(m) when m in [1,4)
+
+ initial estimate:
+ 7bit table lookup (1bit exponent and 6bit significand).
+
+ iterative approximation:
+ using 2 goldschmidt iterations with 32bit int arithmetics
+ and a final iteration with 64bit int arithmetics.
+
+ details:
+
+ the relative error (e = r0 sqrt(m)-1) of a linear estimate
+ (r0 = a m + b) is |e| < 0.085955 ~ 0x1.6p-4 at best,
+ a table lookup is faster and needs one less iteration
+ 6 bit lookup table (128b) gives |e| < 0x1.f9p-8
+ 7 bit lookup table (256b) gives |e| < 0x1.fdp-9
+ for single and double prec 6bit is enough but for quad
+ prec 7bit is needed (or modified iterations). to avoid
+ one more iteration >=13bit table would be needed (16k).
+
+ a newton-raphson iteration for r is
+ w = r*r
+ u = 3 - m*w
+ r = r*u/2
+ can use a goldschmidt iteration for s at the end or
+ s = m*r
+
+ first goldschmidt iteration is
+ s = m*r
+ u = 3 - s*r
+ r = r*u/2
+ s = s*u/2
+ next goldschmidt iteration is
+ u = 3 - s*r
+ r = r*u/2
+ s = s*u/2
+ and at the end r is not computed only s.
+
+ they use the same amount of operations and converge at the
+ same quadratic rate, i.e. if
+ r1 sqrt(m) - 1 = e, then
+ r2 sqrt(m) - 1 = -3/2 e^2 - 1/2 e^3
+ the advantage of goldschmidt is that the mul for s and r
+ are independent (computed in parallel), however it is not
+ "self synchronizing": it only uses the input m in the
+ first iteration so rounding errors accumulate. at the end
+ or when switching to larger precision arithmetics rounding
+ errors dominate so the first iteration should be used.
+
+ the fixed point representations are
+ m: 2.30 r: 0.32, s: 2.30, d: 2.30, u: 2.30, three: 2.30
+ and after switching to 64 bit
+ m: 2.62 r: 0.64, s: 2.62, d: 2.62, u: 2.62, three: 2.62 */
+
+ static const uint64_t three = 0xc0000000;
+ uint64_t r, s, d, u, i;
+
+ i = (ix >> 46) % 128;
+ r = (uint32_t)__rsqrt_tab[i] << 16;
+ /* |r sqrt(m) - 1| < 0x1.fdp-9 */
+ s = mul32(m>>32, r);
+ /* |s/sqrt(m) - 1| < 0x1.fdp-9 */
+ d = mul32(s, r);
+ u = three - d;
+ r = mul32(r, u) << 1;
+ /* |r sqrt(m) - 1| < 0x1.7bp-16 */
+ s = mul32(s, u) << 1;
+ /* |s/sqrt(m) - 1| < 0x1.7bp-16 */
+ d = mul32(s, r);
+ u = three - d;
+ r = mul32(r, u) << 1;
+ /* |r sqrt(m) - 1| < 0x1.3704p-29 (measured worst-case) */
+ r = r << 32;
+ s = mul64(m, r);
+ d = mul64(s, r);
+ u = (three<<32) - d;
+ s = mul64(s, u); /* repr: 3.61 */
+ /* -0x1p-57 < s - sqrt(m) < 0x1.8001p-61 */
+ s = (s - 2) >> 9; /* repr: 12.52 */
+ /* -0x1.09p-52 < s - sqrt(m) < -0x1.fffcp-63 */
+
+ /* s < sqrt(m) < s + 0x1.09p-52,
+ compute nearest rounded result:
+ the nearest result to 52 bits is either s or s+0x1p-52,
+ we can decide by comparing (2^52 s + 0.5)^2 to 2^104 m. */
+ uint64_t d0, d1, d2;
+ double y, t;
+ d0 = (m << 42) - s*s;
+ d1 = s - d0;
+ d2 = d1 + s + 1;
+ s += d1 >> 63;
+ s &= 0x000fffffffffffff;
+ s |= top << 52;
+ y = asdouble(s);
+ if (FENV_SUPPORT) {
+ /* handle rounding modes and inexact exception:
+ only (s+1)^2 == 2^42 m case is exact otherwise
+ add a tiny value to cause the fenv effects. */
+ uint64_t tiny = predict_false(d2==0) ? 0 : 0x0010000000000000;
+ tiny |= (d1^d2) & 0x8000000000000000;
+ t = asdouble(tiny);
+ y = eval_as_double(y + t);
}
- ix0 = (q>>1) + 0x3fe00000;
- ix1 = q1>>1;
- if (q&1)
- ix1 |= sign;
- INSERT_WORDS(z, ix0 + ((uint32_t)m << 20), ix1);
- return z;
+ return y;
}
diff --git a/src/math/sqrt_data.c b/src/math/sqrt_data.c
new file mode 100644
index 00000000..61bc22f4
--- /dev/null
+++ b/src/math/sqrt_data.c
@@ -0,0 +1,19 @@
+#include "sqrt_data.h"
+const uint16_t __rsqrt_tab[128] = {
+0xb451,0xb2f0,0xb196,0xb044,0xaef9,0xadb6,0xac79,0xab43,
+0xaa14,0xa8eb,0xa7c8,0xa6aa,0xa592,0xa480,0xa373,0xa26b,
+0xa168,0xa06a,0x9f70,0x9e7b,0x9d8a,0x9c9d,0x9bb5,0x9ad1,
+0x99f0,0x9913,0x983a,0x9765,0x9693,0x95c4,0x94f8,0x9430,
+0x936b,0x92a9,0x91ea,0x912e,0x9075,0x8fbe,0x8f0a,0x8e59,
+0x8daa,0x8cfe,0x8c54,0x8bac,0x8b07,0x8a64,0x89c4,0x8925,
+0x8889,0x87ee,0x8756,0x86c0,0x862b,0x8599,0x8508,0x8479,
+0x83ec,0x8361,0x82d8,0x8250,0x81c9,0x8145,0x80c2,0x8040,
+0xff02,0xfd0e,0xfb25,0xf947,0xf773,0xf5aa,0xf3ea,0xf234,
+0xf087,0xeee3,0xed47,0xebb3,0xea27,0xe8a3,0xe727,0xe5b2,
+0xe443,0xe2dc,0xe17a,0xe020,0xdecb,0xdd7d,0xdc34,0xdaf1,
+0xd9b3,0xd87b,0xd748,0xd61a,0xd4f1,0xd3cd,0xd2ad,0xd192,
+0xd07b,0xcf69,0xce5b,0xcd51,0xcc4a,0xcb48,0xca4a,0xc94f,
+0xc858,0xc764,0xc674,0xc587,0xc49d,0xc3b7,0xc2d4,0xc1f4,
+0xc116,0xc03c,0xbf65,0xbe90,0xbdbe,0xbcef,0xbc23,0xbb59,
+0xba91,0xb9cc,0xb90a,0xb84a,0xb78c,0xb6d0,0xb617,0xb560,
+};
diff --git a/src/math/sqrt_data.h b/src/math/sqrt_data.h
new file mode 100644
index 00000000..260c7f9c
--- /dev/null
+++ b/src/math/sqrt_data.h
@@ -0,0 +1,13 @@
+#ifndef _SQRT_DATA_H
+#define _SQRT_DATA_H
+
+#include <features.h>
+#include <stdint.h>
+
+/* if x in [1,2): i = (int)(64*x);
+ if x in [2,4): i = (int)(32*x-64);
+ __rsqrt_tab[i]*2^-16 is estimating 1/sqrt(x) with small relative error:
+ |__rsqrt_tab[i]*0x1p-16*sqrt(x) - 1| < -0x1.fdp-9 < 2^-8 */
+extern hidden const uint16_t __rsqrt_tab[128];
+
+#endif
diff --git a/src/math/sqrtf.c b/src/math/sqrtf.c
index d6ace38a..740d81cb 100644
--- a/src/math/sqrtf.c
+++ b/src/math/sqrtf.c
@@ -1,83 +1,83 @@
-/* origin: FreeBSD /usr/src/lib/msun/src/e_sqrtf.c */
-/*
- * Conversion to float by Ian Lance Taylor, Cygnus Support, ian@cygnus.com.
- */
-/*
- * ====================================================
- * Copyright (C) 1993 by Sun Microsystems, Inc. All rights reserved.
- *
- * Developed at SunPro, a Sun Microsystems, Inc. business.
- * Permission to use, copy, modify, and distribute this
- * software is freely granted, provided that this notice
- * is preserved.
- * ====================================================
- */
-
+#include <stdint.h>
+#include <math.h>
#include "libm.h"
+#include "sqrt_data.h"
-static const float tiny = 1.0e-30;
+#define FENV_SUPPORT 1
-float sqrtf(float x)
+static inline uint32_t mul32(uint32_t a, uint32_t b)
{
- float z;
- int32_t sign = (int)0x80000000;
- int32_t ix,s,q,m,t,i;
- uint32_t r;
+ return (uint64_t)a*b >> 32;
+}
- GET_FLOAT_WORD(ix, x);
+/* see sqrt.c for more detailed comments. */
- /* take care of Inf and NaN */
- if ((ix&0x7f800000) == 0x7f800000)
- return x*x + x; /* sqrt(NaN)=NaN, sqrt(+inf)=+inf, sqrt(-inf)=sNaN */
+float sqrtf(float x)
+{
+ uint32_t ix, m, m1, m0, even, ey;
- /* take care of zero */
- if (ix <= 0) {
- if ((ix&~sign) == 0)
- return x; /* sqrt(+-0) = +-0 */
- if (ix < 0)
- return (x-x)/(x-x); /* sqrt(-ve) = sNaN */
- }
- /* normalize x */
- m = ix>>23;
- if (m == 0) { /* subnormal x */
- for (i = 0; (ix&0x00800000) == 0; i++)
- ix<<=1;
- m -= i - 1;
+ ix = asuint(x);
+ if (predict_false(ix - 0x00800000 >= 0x7f800000 - 0x00800000)) {
+ /* x < 0x1p-126 or inf or nan. */
+ if (ix * 2 == 0)
+ return x;
+ if (ix == 0x7f800000)
+ return x;
+ if (ix > 0x7f800000)
+ return __math_invalidf(x);
+ /* x is subnormal, normalize it. */
+ ix = asuint(x * 0x1p23f);
+ ix -= 23 << 23;
}
- m -= 127; /* unbias exponent */
- ix = (ix&0x007fffff)|0x00800000;
- if (m&1) /* odd m, double x to make it even */
- ix += ix;
- m >>= 1; /* m = [m/2] */
- /* generate sqrt(x) bit by bit */
- ix += ix;
- q = s = 0; /* q = sqrt(x) */
- r = 0x01000000; /* r = moving bit from right to left */
+ /* x = 4^e m; with int e and m in [1, 4). */
+ even = ix & 0x00800000;
+ m1 = (ix << 8) | 0x80000000;
+ m0 = (ix << 7) & 0x7fffffff;
+ m = even ? m0 : m1;
- while (r != 0) {
- t = s + r;
- if (t <= ix) {
- s = t+r;
- ix -= t;
- q += r;
- }
- ix += ix;
- r >>= 1;
- }
+ /* 2^e is the exponent part of the return value. */
+ ey = ix >> 1;
+ ey += 0x3f800000 >> 1;
+ ey &= 0x7f800000;
+
+ /* compute r ~ 1/sqrt(m), s ~ sqrt(m) with 2 goldschmidt iterations. */
+ static const uint32_t three = 0xc0000000;
+ uint32_t r, s, d, u, i;
+ i = (ix >> 17) % 128;
+ r = (uint32_t)__rsqrt_tab[i] << 16;
+ /* |r*sqrt(m) - 1| < 0x1p-8 */
+ s = mul32(m, r);
+ /* |s/sqrt(m) - 1| < 0x1p-8 */
+ d = mul32(s, r);
+ u = three - d;
+ r = mul32(r, u) << 1;
+ /* |r*sqrt(m) - 1| < 0x1.7bp-16 */
+ s = mul32(s, u) << 1;
+ /* |s/sqrt(m) - 1| < 0x1.7bp-16 */
+ d = mul32(s, r);
+ u = three - d;
+ s = mul32(s, u);
+ /* -0x1.03p-28 < s/sqrt(m) - 1 < 0x1.fp-31 */
+ s = (s - 1)>>6;
+ /* s < sqrt(m) < s + 0x1.08p-23 */
- /* use floating add to find out rounding direction */
- if (ix != 0) {
- z = 1.0f - tiny; /* raise inexact flag */
- if (z >= 1.0f) {
- z = 1.0f + tiny;
- if (z > 1.0f)
- q += 2;
- else
- q += q & 1;
- }
+ /* compute nearest rounded result. */
+ uint32_t d0, d1, d2;
+ float y, t;
+ d0 = (m << 16) - s*s;
+ d1 = s - d0;
+ d2 = d1 + s + 1;
+ s += d1 >> 31;
+ s &= 0x007fffff;
+ s |= ey;
+ y = asfloat(s);
+ if (FENV_SUPPORT) {
+ /* handle rounding and inexact exception. */
+ uint32_t tiny = predict_false(d2==0) ? 0 : 0x01000000;
+ tiny |= (d1^d2) & 0x80000000;
+ t = asfloat(tiny);
+ y = eval_as_float(y + t);
}
- ix = (q>>1) + 0x3f000000;
- SET_FLOAT_WORD(z, ix + ((uint32_t)m << 23));
- return z;
+ return y;
}
diff --git a/src/math/sqrtl.c b/src/math/sqrtl.c
index 83a8f80c..1b9f19c7 100644
--- a/src/math/sqrtl.c
+++ b/src/math/sqrtl.c
@@ -1,7 +1,259 @@
+#include <stdint.h>
#include <math.h>
+#include <float.h>
+#include "libm.h"
+#if LDBL_MANT_DIG == 53 && LDBL_MAX_EXP == 1024
long double sqrtl(long double x)
{
- /* FIXME: implement in C, this is for LDBL_MANT_DIG == 64 only */
return sqrt(x);
}
+#elif (LDBL_MANT_DIG == 113 || LDBL_MANT_DIG == 64) && LDBL_MAX_EXP == 16384
+#include "sqrt_data.h"
+
+#define FENV_SUPPORT 1
+
+typedef struct {
+ uint64_t hi;
+ uint64_t lo;
+} u128;
+
+/* top: 16 bit sign+exponent, x: significand. */
+static inline long double mkldbl(uint64_t top, u128 x)
+{
+ union ldshape u;
+#if LDBL_MANT_DIG == 113
+ u.i2.hi = x.hi;
+ u.i2.lo = x.lo;
+ u.i2.hi &= 0x0000ffffffffffff;
+ u.i2.hi |= top << 48;
+#elif LDBL_MANT_DIG == 64
+ u.i.se = top;
+ u.i.m = x.lo;
+ /* force the top bit on non-zero (and non-subnormal) results. */
+ if (top & 0x7fff)
+ u.i.m |= 0x8000000000000000;
+#endif
+ return u.f;
+}
+
+/* return: top 16 bit is sign+exp and following bits are the significand. */
+static inline u128 asu128(long double x)
+{
+ union ldshape u = {.f=x};
+ u128 r;
+#if LDBL_MANT_DIG == 113
+ r.hi = u.i2.hi;
+ r.lo = u.i2.lo;
+#elif LDBL_MANT_DIG == 64
+ r.lo = u.i.m<<49;
+ /* ignore the top bit: pseudo numbers are not handled. */
+ r.hi = u.i.m>>15;
+ r.hi &= 0x0000ffffffffffff;
+ r.hi |= (uint64_t)u.i.se << 48;
+#endif
+ return r;
+}
+
+/* returns a*b*2^-32 - e, with error 0 <= e < 1. */
+static inline uint32_t mul32(uint32_t a, uint32_t b)
+{
+ return (uint64_t)a*b >> 32;
+}
+
+/* returns a*b*2^-64 - e, with error 0 <= e < 3. */
+static inline uint64_t mul64(uint64_t a, uint64_t b)
+{
+ uint64_t ahi = a>>32;
+ uint64_t alo = a&0xffffffff;
+ uint64_t bhi = b>>32;
+ uint64_t blo = b&0xffffffff;
+ return ahi*bhi + (ahi*blo >> 32) + (alo*bhi >> 32);
+}
+
+static inline u128 add64(u128 a, uint64_t b)
+{
+ u128 r;
+ r.lo = a.lo + b;
+ r.hi = a.hi;
+ if (r.lo < a.lo)
+ r.hi++;
+ return r;
+}
+
+static inline u128 add128(u128 a, u128 b)
+{
+ u128 r;
+ r.lo = a.lo + b.lo;
+ r.hi = a.hi + b.hi;
+ if (r.lo < a.lo)
+ r.hi++;
+ return r;
+}
+
+static inline u128 sub64(u128 a, uint64_t b)
+{
+ u128 r;
+ r.lo = a.lo - b;
+ r.hi = a.hi;
+ if (a.lo < b)
+ r.hi--;
+ return r;
+}
+
+static inline u128 sub128(u128 a, u128 b)
+{
+ u128 r;
+ r.lo = a.lo - b.lo;
+ r.hi = a.hi - b.hi;
+ if (a.lo < b.lo)
+ r.hi--;
+ return r;
+}
+
+/* a<<n, 0 <= n <= 127 */
+static inline u128 lsh(u128 a, int n)
+{
+ if (n == 0)
+ return a;
+ if (n >= 64) {
+ a.hi = a.lo<<(n-64);
+ a.lo = 0;
+ } else {
+ a.hi = (a.hi<<n) | (a.lo>>(64-n));
+ a.lo = a.lo<<n;
+ }
+ return a;
+}
+
+/* a>>n, 0 <= n <= 127 */
+static inline u128 rsh(u128 a, int n)
+{
+ if (n == 0)
+ return a;
+ if (n >= 64) {
+ a.lo = a.hi>>(n-64);
+ a.hi = 0;
+ } else {
+ a.lo = (a.lo>>n) | (a.hi<<(64-n));
+ a.hi = a.hi>>n;
+ }
+ return a;
+}
+
+/* returns a*b exactly. */
+static inline u128 mul64_128(uint64_t a, uint64_t b)
+{
+ u128 r;
+ uint64_t ahi = a>>32;
+ uint64_t alo = a&0xffffffff;
+ uint64_t bhi = b>>32;
+ uint64_t blo = b&0xffffffff;
+ uint64_t lo1 = ((ahi*blo)&0xffffffff) + ((alo*bhi)&0xffffffff) + (alo*blo>>32);
+ uint64_t lo2 = (alo*blo)&0xffffffff;
+ r.hi = ahi*bhi + (ahi*blo>>32) + (alo*bhi>>32) + (lo1>>32);
+ r.lo = (lo1<<32) + lo2;
+ return r;
+}
+
+/* returns a*b*2^-128 - e, with error 0 <= e < 7. */
+static inline u128 mul128(u128 a, u128 b)
+{
+ u128 hi = mul64_128(a.hi, b.hi);
+ uint64_t m1 = mul64(a.hi, b.lo);
+ uint64_t m2 = mul64(a.lo, b.hi);
+ return add64(add64(hi, m1), m2);
+}
+
+/* returns a*b % 2^128. */
+static inline u128 mul128_tail(u128 a, u128 b)
+{
+ u128 lo = mul64_128(a.lo, b.lo);
+ lo.hi += a.hi*b.lo + a.lo*b.hi;
+ return lo;
+}
+
+
+/* see sqrt.c for detailed comments. */
+
+long double sqrtl(long double x)
+{
+ u128 ix, ml;
+ uint64_t top;
+
+ ix = asu128(x);
+ top = ix.hi >> 48;
+ if (predict_false(top - 0x0001 >= 0x7fff - 0x0001)) {
+ /* x < 0x1p-16382 or inf or nan. */
+ if (2*ix.hi == 0 && ix.lo == 0)
+ return x;
+ if (ix.hi == 0x7fff000000000000 && ix.lo == 0)
+ return x;
+ if (top >= 0x7fff)
+ return __math_invalidl(x);
+ /* x is subnormal, normalize it. */
+ ix = asu128(x * 0x1p112);
+ top = ix.hi >> 48;
+ top -= 112;
+ }
+
+ /* x = 4^e m; with int e and m in [1, 4) */
+ int even = top & 1;
+ ml = lsh(ix, 15);
+ ml.hi |= 0x8000000000000000;
+ if (even) ml = rsh(ml, 1);
+ top = (top + 0x3fff) >> 1;
+
+ /* r ~ 1/sqrt(m) */
+ static const uint64_t three = 0xc0000000;
+ uint64_t r, s, d, u, i;
+ i = (ix.hi >> 42) % 128;
+ r = (uint32_t)__rsqrt_tab[i] << 16;
+ /* |r sqrt(m) - 1| < 0x1p-8 */
+ s = mul32(ml.hi>>32, r);
+ d = mul32(s, r);
+ u = three - d;
+ r = mul32(u, r) << 1;
+ /* |r sqrt(m) - 1| < 0x1.7bp-16, switch to 64bit */
+ r = r<<32;
+ s = mul64(ml.hi, r);
+ d = mul64(s, r);
+ u = (three<<32) - d;
+ r = mul64(u, r) << 1;
+ /* |r sqrt(m) - 1| < 0x1.a5p-31 */
+ s = mul64(u, s) << 1;
+ d = mul64(s, r);
+ u = (three<<32) - d;
+ r = mul64(u, r) << 1;
+ /* |r sqrt(m) - 1| < 0x1.c001p-59, switch to 128bit */
+
+ static const u128 threel = {.hi=three<<32, .lo=0};
+ u128 rl, sl, dl, ul;
+ rl.hi = r;
+ rl.lo = 0;
+ sl = mul128(ml, rl);
+ dl = mul128(sl, rl);
+ ul = sub128(threel, dl);
+ sl = mul128(ul, sl); /* repr: 3.125 */
+ /* -0x1p-116 < s - sqrt(m) < 0x3.8001p-125 */
+ sl = rsh(sub64(sl, 4), 125-(LDBL_MANT_DIG-1));
+ /* s < sqrt(m) < s + 1 ULP + tiny */
+
+ long double y;
+ u128 d2, d1, d0;
+ d0 = sub128(lsh(ml, 2*(LDBL_MANT_DIG-1)-126), mul128_tail(sl,sl));
+ d1 = sub128(sl, d0);
+ d2 = add128(add64(sl, 1), d1);
+ sl = add64(sl, d1.hi >> 63);
+ y = mkldbl(top, sl);
+ if (FENV_SUPPORT) {
+ /* handle rounding modes and inexact exception. */
+ top = predict_false((d2.hi|d2.lo)==0) ? 0 : 1;
+ top |= ((d1.hi^d2.hi)&0x8000000000000000) >> 48;
+ y += mkldbl(top, (u128){0});
+ }
+ return y;
+}
+#else
+#error unsupported long double format
+#endif
diff --git a/src/math/x32/lrintl.s b/src/math/x32/lrintl.s
index ee97d1cf..d4355c32 100644
--- a/src/math/x32/lrintl.s
+++ b/src/math/x32/lrintl.s
@@ -2,6 +2,6 @@
.type lrintl,@function
lrintl:
fldt 8(%esp)
- fistpll 8(%esp)
- mov 8(%esp),%rax
+ fistpl 8(%esp)
+ movl 8(%esp),%eax
ret
diff --git a/src/math/x86_64/fabs.c b/src/math/x86_64/fabs.c
new file mode 100644
index 00000000..16562477
--- /dev/null
+++ b/src/math/x86_64/fabs.c
@@ -0,0 +1,10 @@
+#include <math.h>
+
+double fabs(double x)
+{
+ double t;
+ __asm__ ("pcmpeqd %0, %0" : "=x"(t)); // t = ~0
+ __asm__ ("psrlq $1, %0" : "+x"(t)); // t >>= 1
+ __asm__ ("andps %1, %0" : "+x"(x) : "x"(t)); // x &= t
+ return x;
+}
diff --git a/src/math/x86_64/fabs.s b/src/math/x86_64/fabs.s
deleted file mode 100644
index 5715005e..00000000
--- a/src/math/x86_64/fabs.s
+++ /dev/null
@@ -1,9 +0,0 @@
-.global fabs
-.type fabs,@function
-fabs:
- xor %eax,%eax
- dec %rax
- shr %rax
- movq %rax,%xmm1
- andpd %xmm1,%xmm0
- ret
diff --git a/src/math/x86_64/fabsf.c b/src/math/x86_64/fabsf.c
new file mode 100644
index 00000000..36ea7481
--- /dev/null
+++ b/src/math/x86_64/fabsf.c
@@ -0,0 +1,10 @@
+#include <math.h>
+
+float fabsf(float x)
+{
+ float t;
+ __asm__ ("pcmpeqd %0, %0" : "=x"(t)); // t = ~0
+ __asm__ ("psrld $1, %0" : "+x"(t)); // t >>= 1
+ __asm__ ("andps %1, %0" : "+x"(x) : "x"(t)); // x &= t
+ return x;
+}
diff --git a/src/math/x86_64/fabsf.s b/src/math/x86_64/fabsf.s
deleted file mode 100644
index 501a1f17..00000000
--- a/src/math/x86_64/fabsf.s
+++ /dev/null
@@ -1,7 +0,0 @@
-.global fabsf
-.type fabsf,@function
-fabsf:
- mov $0x7fffffff,%eax
- movq %rax,%xmm1
- andps %xmm1,%xmm0
- ret
diff --git a/src/math/x86_64/fabsl.c b/src/math/x86_64/fabsl.c
new file mode 100644
index 00000000..cc1c9ed9
--- /dev/null
+++ b/src/math/x86_64/fabsl.c
@@ -0,0 +1,7 @@
+#include <math.h>
+
+long double fabsl(long double x)
+{
+ __asm__ ("fabs" : "+t"(x));
+ return x;
+}
diff --git a/src/math/x86_64/fabsl.s b/src/math/x86_64/fabsl.s
deleted file mode 100644
index 4e7ab525..00000000
--- a/src/math/x86_64/fabsl.s
+++ /dev/null
@@ -1,6 +0,0 @@
-.global fabsl
-.type fabsl,@function
-fabsl:
- fldt 8(%rsp)
- fabs
- ret
diff --git a/src/math/x86_64/fmodl.c b/src/math/x86_64/fmodl.c
new file mode 100644
index 00000000..3daeab06
--- /dev/null
+++ b/src/math/x86_64/fmodl.c
@@ -0,0 +1,9 @@
+#include <math.h>
+
+long double fmodl(long double x, long double y)
+{
+ unsigned short fpsr;
+ do __asm__ ("fprem; fnstsw %%ax" : "+t"(x), "=a"(fpsr) : "u"(y));
+ while (fpsr & 0x400);
+ return x;
+}
diff --git a/src/math/x86_64/fmodl.s b/src/math/x86_64/fmodl.s
deleted file mode 100644
index ea07b402..00000000
--- a/src/math/x86_64/fmodl.s
+++ /dev/null
@@ -1,11 +0,0 @@
-.global fmodl
-.type fmodl,@function
-fmodl:
- fldt 24(%rsp)
- fldt 8(%rsp)
-1: fprem
- fnstsw %ax
- testb $4,%ah
- jnz 1b
- fstp %st(1)
- ret
diff --git a/src/math/x86_64/llrint.c b/src/math/x86_64/llrint.c
new file mode 100644
index 00000000..dd38a722
--- /dev/null
+++ b/src/math/x86_64/llrint.c
@@ -0,0 +1,8 @@
+#include <math.h>
+
+long long llrint(double x)
+{
+ long long r;
+ __asm__ ("cvtsd2si %1, %0" : "=r"(r) : "x"(x));
+ return r;
+}
diff --git a/src/math/x86_64/llrint.s b/src/math/x86_64/llrint.s
deleted file mode 100644
index bf476498..00000000
--- a/src/math/x86_64/llrint.s
+++ /dev/null
@@ -1,5 +0,0 @@
-.global llrint
-.type llrint,@function
-llrint:
- cvtsd2si %xmm0,%rax
- ret
diff --git a/src/math/x86_64/llrintf.c b/src/math/x86_64/llrintf.c
new file mode 100644
index 00000000..fc8625e8
--- /dev/null
+++ b/src/math/x86_64/llrintf.c
@@ -0,0 +1,8 @@
+#include <math.h>
+
+long long llrintf(float x)
+{
+ long long r;
+ __asm__ ("cvtss2si %1, %0" : "=r"(r) : "x"(x));
+ return r;
+}
diff --git a/src/math/x86_64/llrintf.s b/src/math/x86_64/llrintf.s
deleted file mode 100644
index d7204ac0..00000000
--- a/src/math/x86_64/llrintf.s
+++ /dev/null
@@ -1,5 +0,0 @@
-.global llrintf
-.type llrintf,@function
-llrintf:
- cvtss2si %xmm0,%rax
- ret
diff --git a/src/math/x86_64/llrintl.c b/src/math/x86_64/llrintl.c
new file mode 100644
index 00000000..c439ef28
--- /dev/null
+++ b/src/math/x86_64/llrintl.c
@@ -0,0 +1,8 @@
+#include <math.h>
+
+long long llrintl(long double x)
+{
+ long long r;
+ __asm__ ("fistpll %0" : "=m"(r) : "t"(x) : "st");
+ return r;
+}
diff --git a/src/math/x86_64/llrintl.s b/src/math/x86_64/llrintl.s
deleted file mode 100644
index 1ec0817d..00000000
--- a/src/math/x86_64/llrintl.s
+++ /dev/null
@@ -1,7 +0,0 @@
-.global llrintl
-.type llrintl,@function
-llrintl:
- fldt 8(%rsp)
- fistpll 8(%rsp)
- mov 8(%rsp),%rax
- ret
diff --git a/src/math/x86_64/lrint.c b/src/math/x86_64/lrint.c
new file mode 100644
index 00000000..a742fec6
--- /dev/null
+++ b/src/math/x86_64/lrint.c
@@ -0,0 +1,8 @@
+#include <math.h>
+
+long lrint(double x)
+{
+ long r;
+ __asm__ ("cvtsd2si %1, %0" : "=r"(r) : "x"(x));
+ return r;
+}
diff --git a/src/math/x86_64/lrint.s b/src/math/x86_64/lrint.s
deleted file mode 100644
index 15fc2454..00000000
--- a/src/math/x86_64/lrint.s
+++ /dev/null
@@ -1,5 +0,0 @@
-.global lrint
-.type lrint,@function
-lrint:
- cvtsd2si %xmm0,%rax
- ret
diff --git a/src/math/x86_64/lrintf.c b/src/math/x86_64/lrintf.c
new file mode 100644
index 00000000..2ba5639d
--- /dev/null
+++ b/src/math/x86_64/lrintf.c
@@ -0,0 +1,8 @@
+#include <math.h>
+
+long lrintf(float x)
+{
+ long r;
+ __asm__ ("cvtss2si %1, %0" : "=r"(r) : "x"(x));
+ return r;
+}
diff --git a/src/math/x86_64/lrintf.s b/src/math/x86_64/lrintf.s
deleted file mode 100644
index 488423d2..00000000
--- a/src/math/x86_64/lrintf.s
+++ /dev/null
@@ -1,5 +0,0 @@
-.global lrintf
-.type lrintf,@function
-lrintf:
- cvtss2si %xmm0,%rax
- ret
diff --git a/src/math/x86_64/lrintl.c b/src/math/x86_64/lrintl.c
new file mode 100644
index 00000000..068e2e4d
--- /dev/null
+++ b/src/math/x86_64/lrintl.c
@@ -0,0 +1,8 @@
+#include <math.h>
+
+long lrintl(long double x)
+{
+ long r;
+ __asm__ ("fistpll %0" : "=m"(r) : "t"(x) : "st");
+ return r;
+}
diff --git a/src/math/x86_64/lrintl.s b/src/math/x86_64/lrintl.s
deleted file mode 100644
index d587b12b..00000000
--- a/src/math/x86_64/lrintl.s
+++ /dev/null
@@ -1,7 +0,0 @@
-.global lrintl
-.type lrintl,@function
-lrintl:
- fldt 8(%rsp)
- fistpll 8(%rsp)
- mov 8(%rsp),%rax
- ret
diff --git a/src/math/x86_64/remainderl.c b/src/math/x86_64/remainderl.c
new file mode 100644
index 00000000..8cf75071
--- /dev/null
+++ b/src/math/x86_64/remainderl.c
@@ -0,0 +1,9 @@
+#include <math.h>
+
+long double remainderl(long double x, long double y)
+{
+ unsigned short fpsr;
+ do __asm__ ("fprem1; fnstsw %%ax" : "+t"(x), "=a"(fpsr) : "u"(y));
+ while (fpsr & 0x400);
+ return x;
+}
diff --git a/src/math/x86_64/remainderl.s b/src/math/x86_64/remainderl.s
deleted file mode 100644
index cb3857b4..00000000
--- a/src/math/x86_64/remainderl.s
+++ /dev/null
@@ -1,11 +0,0 @@
-.global remainderl
-.type remainderl,@function
-remainderl:
- fldt 24(%rsp)
- fldt 8(%rsp)
-1: fprem1
- fnstsw %ax
- testb $4,%ah
- jnz 1b
- fstp %st(1)
- ret
diff --git a/src/math/x86_64/remquol.c b/src/math/x86_64/remquol.c
new file mode 100644
index 00000000..60eef089
--- /dev/null
+++ b/src/math/x86_64/remquol.c
@@ -0,0 +1,32 @@
+#include <math.h>
+
+long double remquol(long double x, long double y, int *quo)
+{
+ signed char *cx = (void *)&x, *cy = (void *)&y;
+ /* By ensuring that addresses of x and y cannot be discarded,
+ * this empty asm guides GCC into representing extraction of
+ * their sign bits as memory loads rather than making x and y
+ * not-address-taken internally and using bitfield operations,
+ * which in the end wouldn't work out, as extraction from FPU
+ * registers needs to go through memory anyway. This way GCC
+ * should manage to use incoming stack slots without spills. */
+ __asm__ ("" :: "X"(cx), "X"(cy));
+
+ long double t = x;
+ unsigned fpsr;
+ do __asm__ ("fprem1; fnstsw %%ax" : "+t"(t), "=a"(fpsr) : "u"(y));
+ while (fpsr & 0x400);
+ /* C0, C1, C3 flags in x87 status word carry low bits of quotient:
+ * 15 14 13 12 11 10 9 8
+ * . C3 . . . C2 C1 C0
+ * . b1 . . . 0 b0 b2 */
+ unsigned char i = fpsr >> 8;
+ i = i>>4 | i<<4;
+ /* i[5:2] is now {b0 b2 ? b1}. Retrieve {0 b2 b1 b0} via
+ * in-register table lookup. */
+ unsigned qbits = 0x7575313164642020 >> (i & 60);
+ qbits &= 7;
+
+ *quo = (cx[9]^cy[9]) < 0 ? -qbits : qbits;
+ return t;
+}
diff --git a/src/math/x86_64/rintl.c b/src/math/x86_64/rintl.c
new file mode 100644
index 00000000..e1a92077
--- /dev/null
+++ b/src/math/x86_64/rintl.c
@@ -0,0 +1,7 @@
+#include <math.h>
+
+long double rintl(long double x)
+{
+ __asm__ ("frndint" : "+t"(x));
+ return x;
+}
diff --git a/src/math/x86_64/rintl.s b/src/math/x86_64/rintl.s
deleted file mode 100644
index 64e663cd..00000000
--- a/src/math/x86_64/rintl.s
+++ /dev/null
@@ -1,6 +0,0 @@
-.global rintl
-.type rintl,@function
-rintl:
- fldt 8(%rsp)
- frndint
- ret
diff --git a/src/math/x86_64/sqrt.c b/src/math/x86_64/sqrt.c
new file mode 100644
index 00000000..657e09e3
--- /dev/null
+++ b/src/math/x86_64/sqrt.c
@@ -0,0 +1,7 @@
+#include <math.h>
+
+double sqrt(double x)
+{
+ __asm__ ("sqrtsd %1, %0" : "=x"(x) : "x"(x));
+ return x;
+}
diff --git a/src/math/x86_64/sqrt.s b/src/math/x86_64/sqrt.s
deleted file mode 100644
index d3c609f9..00000000
--- a/src/math/x86_64/sqrt.s
+++ /dev/null
@@ -1,4 +0,0 @@
-.global sqrt
-.type sqrt,@function
-sqrt: sqrtsd %xmm0, %xmm0
- ret
diff --git a/src/math/x86_64/sqrtf.c b/src/math/x86_64/sqrtf.c
new file mode 100644
index 00000000..720baec6
--- /dev/null
+++ b/src/math/x86_64/sqrtf.c
@@ -0,0 +1,7 @@
+#include <math.h>
+
+float sqrtf(float x)
+{
+ __asm__ ("sqrtss %1, %0" : "=x"(x) : "x"(x));
+ return x;
+}
diff --git a/src/math/x86_64/sqrtf.s b/src/math/x86_64/sqrtf.s
deleted file mode 100644
index eec48c60..00000000
--- a/src/math/x86_64/sqrtf.s
+++ /dev/null
@@ -1,4 +0,0 @@
-.global sqrtf
-.type sqrtf,@function
-sqrtf: sqrtss %xmm0, %xmm0
- ret
diff --git a/src/math/x86_64/sqrtl.c b/src/math/x86_64/sqrtl.c
new file mode 100644
index 00000000..864cfcc4
--- /dev/null
+++ b/src/math/x86_64/sqrtl.c
@@ -0,0 +1,7 @@
+#include <math.h>
+
+long double sqrtl(long double x)
+{
+ __asm__ ("fsqrt" : "+t"(x));
+ return x;
+}
diff --git a/src/math/x86_64/sqrtl.s b/src/math/x86_64/sqrtl.s
deleted file mode 100644
index 23cd687d..00000000
--- a/src/math/x86_64/sqrtl.s
+++ /dev/null
@@ -1,5 +0,0 @@
-.global sqrtl
-.type sqrtl,@function
-sqrtl: fldt 8(%rsp)
- fsqrt
- ret
diff --git a/src/misc/getentropy.c b/src/misc/getentropy.c
index d2f282ce..651ea95f 100644
--- a/src/misc/getentropy.c
+++ b/src/misc/getentropy.c
@@ -6,7 +6,7 @@
int getentropy(void *buffer, size_t len)
{
- int cs, ret;
+ int cs, ret = 0;
char *pos = buffer;
if (len > 256) {
diff --git a/src/misc/ioctl.c b/src/misc/ioctl.c
index 6f31d4bc..89477511 100644
--- a/src/misc/ioctl.c
+++ b/src/misc/ioctl.c
@@ -3,8 +3,115 @@
#include <errno.h>
#include <time.h>
#include <sys/time.h>
+#include <stddef.h>
+#include <string.h>
#include "syscall.h"
+#define alignof(t) offsetof(struct { char c; t x; }, x)
+
+#define W 1
+#define R 2
+#define WR 3
+
+struct ioctl_compat_map {
+ int new_req, old_req;
+ unsigned char old_size, dir, force_align, noffs;
+ unsigned char offsets[8];
+};
+
+#define NINTH(a,b,c,d,e,f,g,h,i,...) i
+#define COUNT(...) NINTH(__VA_ARGS__,8,7,6,5,4,3,2,1,0)
+#define OFFS(...) COUNT(__VA_ARGS__), { __VA_ARGS__ }
+
+/* yields a type for a struct with original size n, with a misaligned
+ * timeval/timespec expanded from 32- to 64-bit. for use with ioctl
+ * number producing macros; only size of result is meaningful. */
+#define new_misaligned(n) struct { int i; time_t t; char c[(n)-4]; }
+
+static const struct ioctl_compat_map compat_map[] = {
+ { SIOCGSTAMP, SIOCGSTAMP_OLD, 8, R, 0, OFFS(0, 4) },
+ { SIOCGSTAMPNS, SIOCGSTAMPNS_OLD, 8, R, 0, OFFS(0, 4) },
+
+ /* SNDRV_TIMER_IOCTL_STATUS */
+ { _IOR('T', 0x14, char[96]), _IOR('T', 0x14, 88), 88, R, 0, OFFS(0,4) },
+
+ /* SNDRV_PCM_IOCTL_STATUS[_EXT] */
+ { _IOR('A', 0x20, char[128]), _IOR('A', 0x20, char[108]), 108, R, 1, OFFS(4,8,12,16,52,56,60,64) },
+ { _IOWR('A', 0x24, char[128]), _IOWR('A', 0x24, char[108]), 108, WR, 1, OFFS(4,8,12,16,52,56,60,64) },
+
+ /* SNDRV_RAWMIDI_IOCTL_STATUS */
+ { _IOWR('W', 0x20, char[48]), _IOWR('W', 0x20, char[36]), 36, WR, 1, OFFS(4,8) },
+
+ /* SNDRV_PCM_IOCTL_SYNC_PTR - with 3 subtables */
+ { _IOWR('A', 0x23, char[136]), _IOWR('A', 0x23, char[132]), 0, WR, 1, 0 },
+ { 0, 0, 4, WR, 1, 0 }, /* snd_pcm_sync_ptr (flags only) */
+ { 0, 0, 32, WR, 1, OFFS(8,12,16,24,28) }, /* snd_pcm_mmap_status */
+ { 0, 0, 8, WR, 1, OFFS(0,4) }, /* snd_pcm_mmap_control */
+
+ /* VIDIOC_QUERYBUF, VIDIOC_QBUF, VIDIOC_DQBUF, VIDIOC_PREPARE_BUF */
+ { _IOWR('V', 9, new_misaligned(72)), _IOWR('V', 9, char[72]), 72, WR, 0, OFFS(20) },
+ { _IOWR('V', 15, new_misaligned(72)), _IOWR('V', 15, char[72]), 72, WR, 0, OFFS(20) },
+ { _IOWR('V', 17, new_misaligned(72)), _IOWR('V', 17, char[72]), 72, WR, 0, OFFS(20) },
+ { _IOWR('V', 93, new_misaligned(72)), _IOWR('V', 93, char[72]), 72, WR, 0, OFFS(20) },
+
+ /* VIDIOC_DQEVENT */
+ { _IOR('V', 89, new_misaligned(96)), _IOR('V', 89, char[96]), 96, R, 0, OFFS(76,80) },
+
+ /* VIDIOC_OMAP3ISP_STAT_REQ */
+ { _IOWR('V', 192+6, char[32]), _IOWR('V', 192+6, char[24]), 22, WR, 0, OFFS(0,4) },
+
+ /* PPPIOCGIDLE */
+ { _IOR('t', 63, char[16]), _IOR('t', 63, char[8]), 8, R, 0, OFFS(0,4) },
+
+ /* PPGETTIME, PPSETTIME */
+ { _IOR('p', 0x95, char[16]), _IOR('p', 0x95, char[8]), 8, R, 0, OFFS(0,4) },
+ { _IOW('p', 0x96, char[16]), _IOW('p', 0x96, char[8]), 8, W, 0, OFFS(0,4) },
+
+ /* LPSETTIMEOUT */
+ { _IOW(0x6, 0xf, char[16]), 0x060f, 8, W, 0, OFFS(0,4) },
+};
+
+static void convert_ioctl_struct(const struct ioctl_compat_map *map, char *old, char *new, int dir)
+{
+ int new_offset = 0;
+ int old_offset = 0;
+ int old_size = map->old_size;
+ if (!(dir & map->dir)) return;
+ if (!map->old_size) {
+ /* offsets hard-coded for SNDRV_PCM_IOCTL_SYNC_PTR;
+ * if another exception appears this needs changing. */
+ convert_ioctl_struct(map+1, old, new, dir);
+ convert_ioctl_struct(map+2, old+4, new+8, dir);
+ convert_ioctl_struct(map+3, old+68, new+72, dir);
+ return;
+ }
+ for (int i=0; i < map->noffs; i++) {
+ int ts_offset = map->offsets[i];
+ int len = ts_offset-old_offset;
+ if (dir==W) memcpy(old+old_offset, new+new_offset, len);
+ else memcpy(new+new_offset, old+old_offset, len);
+ new_offset += len;
+ old_offset += len;
+ long long new_ts;
+ long old_ts;
+ int align = map->force_align ? sizeof(time_t) : alignof(time_t);
+ new_offset += (align-1) & -new_offset;
+ if (dir==W) {
+ memcpy(&new_ts, new+new_offset, sizeof new_ts);
+ old_ts = new_ts;
+ memcpy(old+old_offset, &old_ts, sizeof old_ts);
+ } else {
+ memcpy(&old_ts, old+old_offset, sizeof old_ts);
+ new_ts = old_ts;
+ memcpy(new+new_offset, &new_ts, sizeof new_ts);
+ }
+ new_offset += sizeof new_ts;
+ old_offset += sizeof old_ts;
+ }
+ if (dir==W) memcpy(old+old_offset, new+new_offset, old_size-old_offset);
+ else memcpy(new+new_offset, old+old_offset, old_size-old_offset);
+}
+
int ioctl(int fd, int req, ...)
{
void *arg;
@@ -13,23 +120,18 @@ int ioctl(int fd, int req, ...)
arg = va_arg(ap, void *);
va_end(ap);
int r = __syscall(SYS_ioctl, fd, req, arg);
- if (r==-ENOTTY) switch (req) {
- case SIOCGSTAMP:
- case SIOCGSTAMPNS:
- if (SIOCGSTAMP==SIOCGSTAMP_OLD) break;
- if (req==SIOCGSTAMP) req=SIOCGSTAMP_OLD;
- if (req==SIOCGSTAMPNS) req=SIOCGSTAMPNS_OLD;
- long t32[2];
- r = __syscall(SYS_ioctl, fd, req, t32);
- if (r<0) break;
- if (req==SIOCGSTAMP_OLD) {
- struct timeval *tv = arg;
- tv->tv_sec = t32[0];
- tv->tv_usec = t32[1];
- } else {
- struct timespec *ts = arg;
- ts->tv_sec = t32[0];
- ts->tv_nsec = t32[1];
+ if (SIOCGSTAMP != SIOCGSTAMP_OLD && req && r==-ENOTTY) {
+ for (int i=0; i<sizeof compat_map/sizeof *compat_map; i++) {
+ if (compat_map[i].new_req != req) continue;
+ union {
+ long long align;
+ char buf[256];
+ } u;
+ convert_ioctl_struct(&compat_map[i], u.buf, arg, W);
+ r = __syscall(SYS_ioctl, fd, compat_map[i].old_req, u.buf);
+ if (r<0) break;
+ convert_ioctl_struct(&compat_map[i], u.buf, arg, R);
+ break;
}
}
return __syscall_ret(r);
diff --git a/src/misc/nftw.c b/src/misc/nftw.c
index 0a464100..8dcff7fe 100644
--- a/src/misc/nftw.c
+++ b/src/misc/nftw.c
@@ -1,5 +1,6 @@
#include <ftw.h>
#include <dirent.h>
+#include <fcntl.h>
#include <sys/stat.h>
#include <errno.h>
#include <unistd.h>
@@ -26,6 +27,8 @@ static int do_nftw(char *path, int (*fn)(const char *, const struct stat *, int,
struct history new;
int type;
int r;
+ int dfd;
+ int err;
struct FTW lev;
if ((flags & FTW_PHYS) ? lstat(path, &st) : stat(path, &st) < 0) {
@@ -34,8 +37,7 @@ static int do_nftw(char *path, int (*fn)(const char *, const struct stat *, int,
else if (errno != EACCES) return -1;
else type = FTW_NS;
} else if (S_ISDIR(st.st_mode)) {
- if (access(path, R_OK) < 0) type = FTW_DNR;
- else if (flags & FTW_DEPTH) type = FTW_DP;
+ if (flags & FTW_DEPTH) type = FTW_DP;
else type = FTW_D;
} else if (S_ISLNK(st.st_mode)) {
if (flags & FTW_PHYS) type = FTW_SL;
@@ -63,6 +65,13 @@ static int do_nftw(char *path, int (*fn)(const char *, const struct stat *, int,
lev.base = k;
}
+ if (type == FTW_D || type == FTW_DP) {
+ dfd = open(path, O_RDONLY);
+ err = errno;
+ if (dfd < 0 && err == EACCES) type = FTW_DNR;
+ if (!fd_limit) close(dfd);
+ }
+
if (!(flags & FTW_DEPTH) && (r=fn(path, &st, type, &lev)))
return r;
@@ -71,7 +80,11 @@ static int do_nftw(char *path, int (*fn)(const char *, const struct stat *, int,
return 0;
if ((type == FTW_D || type == FTW_DP) && fd_limit) {
- DIR *d = opendir(path);
+ if (dfd < 0) {
+ errno = err;
+ return -1;
+ }
+ DIR *d = fdopendir(dfd);
if (d) {
struct dirent *de;
while ((de = readdir(d))) {
@@ -92,7 +105,8 @@ static int do_nftw(char *path, int (*fn)(const char *, const struct stat *, int,
}
}
closedir(d);
- } else if (errno != EACCES) {
+ } else {
+ close(dfd);
return -1;
}
}
diff --git a/src/network/getnameinfo.c b/src/network/getnameinfo.c
index f77e73ad..949e1811 100644
--- a/src/network/getnameinfo.c
+++ b/src/network/getnameinfo.c
@@ -158,6 +158,7 @@ int getnameinfo(const struct sockaddr *restrict sa, socklen_t sl,
unsigned char query[18+PTR_MAX], reply[512];
int qlen = __res_mkquery(0, ptr, 1, RR_PTR,
0, 0, 0, query, sizeof query);
+ query[3] = 0; /* don't need AD flag */
int rlen = __res_send(query, qlen, reply, sizeof reply);
buf[0] = 0;
if (rlen > 0)
diff --git a/src/network/getsockopt.c b/src/network/getsockopt.c
index e871d624..d3640d9c 100644
--- a/src/network/getsockopt.c
+++ b/src/network/getsockopt.c
@@ -26,6 +26,15 @@ int getsockopt(int fd, int level, int optname, void *restrict optval, socklen_t
tv->tv_sec = tv32[0];
tv->tv_usec = tv32[1];
*optlen = sizeof *tv;
+ break;
+ case SO_TIMESTAMP:
+ case SO_TIMESTAMPNS:
+ if (SO_TIMESTAMP == SO_TIMESTAMP_OLD) break;
+ if (optname==SO_TIMESTAMP) optname=SO_TIMESTAMP_OLD;
+ if (optname==SO_TIMESTAMPNS) optname=SO_TIMESTAMPNS_OLD;
+ r = __socketcall(getsockopt, fd, level,
+ optname, optval, optlen, 0);
+ break;
}
}
return __syscall_ret(r);
diff --git a/src/network/h_errno.c b/src/network/h_errno.c
index 4f700cea..638f7718 100644
--- a/src/network/h_errno.c
+++ b/src/network/h_errno.c
@@ -1,9 +1,11 @@
#include <netdb.h>
+#include "pthread_impl.h"
#undef h_errno
int h_errno;
int *__h_errno_location(void)
{
- return &h_errno;
+ if (!__pthread_self()->stack) return &h_errno;
+ return &__pthread_self()->h_errno_val;
}
diff --git a/src/network/herror.c b/src/network/herror.c
index 65f25ff3..87f8cff4 100644
--- a/src/network/herror.c
+++ b/src/network/herror.c
@@ -4,5 +4,5 @@
void herror(const char *msg)
{
- fprintf(stderr, "%s%s%s", msg?msg:"", msg?": ":"", hstrerror(h_errno));
+ fprintf(stderr, "%s%s%s\n", msg?msg:"", msg?": ":"", hstrerror(h_errno));
}
diff --git a/src/network/lookup_name.c b/src/network/lookup_name.c
index c93263a9..aa558c19 100644
--- a/src/network/lookup_name.c
+++ b/src/network/lookup_name.c
@@ -50,7 +50,7 @@ static int name_from_hosts(struct address buf[static MAXADDRS], char canon[stati
{
char line[512];
size_t l = strlen(name);
- int cnt = 0, badfam = 0;
+ int cnt = 0, badfam = 0, have_canon = 0;
unsigned char _buf[1032];
FILE _f, *f = __fopen_rb_ca("/etc/hosts", &_f, _buf, sizeof _buf);
if (!f) switch (errno) {
@@ -80,14 +80,19 @@ static int name_from_hosts(struct address buf[static MAXADDRS], char canon[stati
continue;
default:
badfam = EAI_NONAME;
- continue;
+ break;
}
+ if (have_canon) continue;
+
/* Extract first name as canonical name */
for (; *p && isspace(*p); p++);
for (z=p; *z && !isspace(*z); z++);
*z = 0;
- if (is_valid_hostname(p)) memcpy(canon, p, z-p+1);
+ if (is_valid_hostname(p)) {
+ have_canon = 1;
+ memcpy(canon, p, z-p+1);
+ }
}
__fclose_ca(f);
return cnt ? cnt : badfam;
@@ -149,6 +154,7 @@ static int name_from_dns(struct address buf[static MAXADDRS], char canon[static
0, 0, 0, qbuf[nq], sizeof *qbuf);
if (qlens[nq] == -1)
return EAI_NONAME;
+ qbuf[nq][3] = 0; /* don't need AD flag */
nq++;
}
}
@@ -156,14 +162,17 @@ static int name_from_dns(struct address buf[static MAXADDRS], char canon[static
if (__res_msend_rc(nq, qp, qlens, ap, alens, sizeof *abuf, conf) < 0)
return EAI_SYSTEM;
+ for (i=0; i<nq; i++) {
+ if (alens[i] < 4 || (abuf[i][3] & 15) == 2) return EAI_AGAIN;
+ if ((abuf[i][3] & 15) == 3) return 0;
+ if ((abuf[i][3] & 15) != 0) return EAI_FAIL;
+ }
+
for (i=0; i<nq; i++)
__dns_parse(abuf[i], alens[i], dns_parse_callback, &ctx);
if (ctx.cnt) return ctx.cnt;
- if (alens[0] < 4 || (abuf[0][3] & 15) == 2) return EAI_AGAIN;
- if ((abuf[0][3] & 15) == 0) return EAI_NONAME;
- if ((abuf[0][3] & 15) == 3) return 0;
- return EAI_FAIL;
+ return EAI_NONAME;
}
static int name_from_dns_search(struct address buf[static MAXADDRS], char canon[static 256], const char *name, int family)
diff --git a/src/network/recvmmsg.c b/src/network/recvmmsg.c
index d5dc6b51..2978e2f6 100644
--- a/src/network/recvmmsg.c
+++ b/src/network/recvmmsg.c
@@ -8,6 +8,8 @@
#define IS32BIT(x) !((x)+0x80000000ULL>>32)
#define CLAMP(x) (int)(IS32BIT(x) ? (x) : 0x7fffffffU+((0ULL+(x))>>63))
+hidden void __convert_scm_timestamps(struct msghdr *, socklen_t);
+
int recvmmsg(int fd, struct mmsghdr *msgvec, unsigned int vlen, unsigned int flags, struct timespec *timeout)
{
#if LONG_MAX > INT_MAX
@@ -19,14 +21,18 @@ int recvmmsg(int fd, struct mmsghdr *msgvec, unsigned int vlen, unsigned int fla
#ifdef SYS_recvmmsg_time64
time_t s = timeout ? timeout->tv_sec : 0;
long ns = timeout ? timeout->tv_nsec : 0;
- int r = -ENOSYS;
- if (SYS_recvmmsg == SYS_recvmmsg_time64 || !IS32BIT(s))
- r = __syscall_cp(SYS_recvmmsg_time64, fd, msgvec, vlen, flags,
+ int r = __syscall_cp(SYS_recvmmsg_time64, fd, msgvec, vlen, flags,
timeout ? ((long long[]){s, ns}) : 0);
if (SYS_recvmmsg == SYS_recvmmsg_time64 || r!=-ENOSYS)
return __syscall_ret(r);
- return syscall_cp(SYS_recvmmsg, fd, msgvec, vlen, flags,
+ if (vlen > IOV_MAX) vlen = IOV_MAX;
+ socklen_t csize[vlen];
+ for (int i=0; i<vlen; i++) csize[i] = msgvec[i].msg_hdr.msg_controllen;
+ r = __syscall_cp(SYS_recvmmsg, fd, msgvec, vlen, flags,
timeout ? ((long[]){CLAMP(s), ns}) : 0);
+ for (int i=0; i<r; i++)
+ __convert_scm_timestamps(&msgvec[i].msg_hdr, csize[i]);
+ return __syscall_ret(r);
#else
return syscall_cp(SYS_recvmmsg, fd, msgvec, vlen, flags, timeout);
#endif
diff --git a/src/network/recvmsg.c b/src/network/recvmsg.c
index 4ca7da8b..03641625 100644
--- a/src/network/recvmsg.c
+++ b/src/network/recvmsg.c
@@ -1,10 +1,56 @@
#include <sys/socket.h>
#include <limits.h>
+#include <time.h>
+#include <sys/time.h>
+#include <string.h>
#include "syscall.h"
+hidden void __convert_scm_timestamps(struct msghdr *, socklen_t);
+
+void __convert_scm_timestamps(struct msghdr *msg, socklen_t csize)
+{
+ if (SCM_TIMESTAMP == SCM_TIMESTAMP_OLD) return;
+ if (!msg->msg_control || !msg->msg_controllen) return;
+
+ struct cmsghdr *cmsg, *last=0;
+ long tmp;
+ long long tvts[2];
+ int type = 0;
+
+ for (cmsg=CMSG_FIRSTHDR(msg); cmsg; cmsg=CMSG_NXTHDR(msg, cmsg)) {
+ if (cmsg->cmsg_level==SOL_SOCKET) switch (cmsg->cmsg_type) {
+ case SCM_TIMESTAMP_OLD:
+ if (type) break;
+ type = SCM_TIMESTAMP;
+ goto common;
+ case SCM_TIMESTAMPNS_OLD:
+ type = SCM_TIMESTAMPNS;
+ common:
+ memcpy(&tmp, CMSG_DATA(cmsg), sizeof tmp);
+ tvts[0] = tmp;
+ memcpy(&tmp, CMSG_DATA(cmsg) + sizeof tmp, sizeof tmp);
+ tvts[1] = tmp;
+ break;
+ }
+ last = cmsg;
+ }
+ if (!last || !type) return;
+ if (CMSG_SPACE(sizeof tvts) > csize-msg->msg_controllen) {
+ msg->msg_flags |= MSG_CTRUNC;
+ return;
+ }
+ msg->msg_controllen += CMSG_SPACE(sizeof tvts);
+ cmsg = CMSG_NXTHDR(msg, last);
+ cmsg->cmsg_level = SOL_SOCKET;
+ cmsg->cmsg_type = type;
+ cmsg->cmsg_len = CMSG_LEN(sizeof tvts);
+ memcpy(CMSG_DATA(cmsg), &tvts, sizeof tvts);
+}
+
ssize_t recvmsg(int fd, struct msghdr *msg, int flags)
{
ssize_t r;
+ socklen_t orig_controllen = msg->msg_controllen;
#if LONG_MAX > INT_MAX
struct msghdr h, *orig = msg;
if (msg) {
@@ -14,6 +60,7 @@ ssize_t recvmsg(int fd, struct msghdr *msg, int flags)
}
#endif
r = socketcall_cp(recvmsg, fd, msg, flags, 0, 0, 0);
+ if (r >= 0) __convert_scm_timestamps(msg, orig_controllen);
#if LONG_MAX > INT_MAX
if (orig) *orig = h;
#endif
diff --git a/src/network/res_mkquery.c b/src/network/res_mkquery.c
index 6fa04a5c..33f50cb9 100644
--- a/src/network/res_mkquery.c
+++ b/src/network/res_mkquery.c
@@ -20,6 +20,7 @@ int __res_mkquery(int op, const char *dname, int class, int type,
/* Construct query template - ID will be filled later */
memset(q, 0, n);
q[2] = op*8 + 1;
+ q[3] = 32; /* AD */
q[5] = 1;
memcpy((char *)q+13, dname, l);
for (i=13; q[i]; i=j+1) {
diff --git a/src/network/res_query.c b/src/network/res_query.c
index 2f4da2e2..506dc231 100644
--- a/src/network/res_query.c
+++ b/src/network/res_query.c
@@ -1,3 +1,4 @@
+#define _BSD_SOURCE
#include <resolv.h>
#include <netdb.h>
@@ -6,7 +7,20 @@ int res_query(const char *name, int class, int type, unsigned char *dest, int le
unsigned char q[280];
int ql = __res_mkquery(0, name, class, type, 0, 0, 0, q, sizeof q);
if (ql < 0) return ql;
- return __res_send(q, ql, dest, len);
+ int r = __res_send(q, ql, dest, len);
+ if (r<12) {
+ h_errno = TRY_AGAIN;
+ return -1;
+ }
+ if ((dest[3] & 15) == 3) {
+ h_errno = HOST_NOT_FOUND;
+ return -1;
+ }
+ if ((dest[3] & 15) == 0 && !dest[6] && !dest[7]) {
+ h_errno = NO_DATA;
+ return -1;
+ }
+ return r;
}
weak_alias(res_query, res_search);
diff --git a/src/network/res_send.c b/src/network/res_send.c
index b9cea0bf..ee4abf1f 100644
--- a/src/network/res_send.c
+++ b/src/network/res_send.c
@@ -3,7 +3,7 @@
int __res_send(const unsigned char *msg, int msglen, unsigned char *answer, int anslen)
{
int r = __res_msend(1, &msg, &msglen, &answer, &anslen, anslen);
- return r<0 ? r : anslen;
+ return r<0 || !anslen ? -1 : anslen;
}
weak_alias(__res_send, res_send);
diff --git a/src/network/setsockopt.c b/src/network/setsockopt.c
index 2c188a96..612a1947 100644
--- a/src/network/setsockopt.c
+++ b/src/network/setsockopt.c
@@ -31,6 +31,15 @@ int setsockopt(int fd, int level, int optname, const void *optval, socklen_t opt
r = __socketcall(setsockopt, fd, level, optname,
((long[]){s, CLAMP(us)}), 2*sizeof(long), 0);
+ break;
+ case SO_TIMESTAMP:
+ case SO_TIMESTAMPNS:
+ if (SO_TIMESTAMP == SO_TIMESTAMP_OLD) break;
+ if (optname==SO_TIMESTAMP) optname=SO_TIMESTAMP_OLD;
+ if (optname==SO_TIMESTAMPNS) optname=SO_TIMESTAMPNS_OLD;
+ r = __socketcall(setsockopt, fd, level,
+ optname, optval, optlen, 0);
+ break;
}
}
return __syscall_ret(r);
diff --git a/src/network/socket.c b/src/network/socket.c
index a2e92d90..afa1a7f3 100644
--- a/src/network/socket.c
+++ b/src/network/socket.c
@@ -5,17 +5,17 @@
int socket(int domain, int type, int protocol)
{
- int s = socketcall(socket, domain, type, protocol, 0, 0, 0);
- if (s<0 && (errno==EINVAL || errno==EPROTONOSUPPORT)
+ int s = __socketcall(socket, domain, type, protocol, 0, 0, 0);
+ if ((s==-EINVAL || s==-EPROTONOSUPPORT)
&& (type&(SOCK_CLOEXEC|SOCK_NONBLOCK))) {
- s = socketcall(socket, domain,
+ s = __socketcall(socket, domain,
type & ~(SOCK_CLOEXEC|SOCK_NONBLOCK),
protocol, 0, 0, 0);
- if (s < 0) return s;
+ if (s < 0) return __syscall_ret(s);
if (type & SOCK_CLOEXEC)
__syscall(SYS_fcntl, s, F_SETFD, FD_CLOEXEC);
if (type & SOCK_NONBLOCK)
__syscall(SYS_fcntl, s, F_SETFL, O_NONBLOCK);
}
- return s;
+ return __syscall_ret(s);
}
diff --git a/src/process/fork.c b/src/process/fork.c
index fb42478a..7e984ff8 100644
--- a/src/process/fork.c
+++ b/src/process/fork.c
@@ -30,6 +30,7 @@ pid_t fork(void)
self->next = self->prev = self;
__thread_list_lock = 0;
libc.threads_minus_1 = 0;
+ if (libc.need_locks) libc.need_locks = -1;
}
__restore_sigs(&set);
__fork_handler(!ret);
diff --git a/src/setjmp/aarch64/longjmp.s b/src/setjmp/aarch64/longjmp.s
index 7c4655fa..0af9c50e 100644
--- a/src/setjmp/aarch64/longjmp.s
+++ b/src/setjmp/aarch64/longjmp.s
@@ -18,7 +18,6 @@ longjmp:
ldp d12, d13, [x0,#144]
ldp d14, d15, [x0,#160]
- mov x0, x1
- cbnz x1, 1f
- mov x0, #1
-1: br x30
+ cmp w1, 0
+ csinc w0, w1, wzr, ne
+ br x30
diff --git a/src/setjmp/i386/longjmp.s b/src/setjmp/i386/longjmp.s
index 772d28dd..8188f06b 100644
--- a/src/setjmp/i386/longjmp.s
+++ b/src/setjmp/i386/longjmp.s
@@ -6,15 +6,11 @@ _longjmp:
longjmp:
mov 4(%esp),%edx
mov 8(%esp),%eax
- test %eax,%eax
- jnz 1f
- inc %eax
-1:
+ cmp $1,%eax
+ adc $0, %al
mov (%edx),%ebx
mov 4(%edx),%esi
mov 8(%edx),%edi
mov 12(%edx),%ebp
- mov 16(%edx),%ecx
- mov %ecx,%esp
- mov 20(%edx),%ecx
- jmp *%ecx
+ mov 16(%edx),%esp
+ jmp *20(%edx)
diff --git a/src/setjmp/x32/longjmp.s b/src/setjmp/x32/longjmp.s
index e175a4b9..1b2661c3 100644
--- a/src/setjmp/x32/longjmp.s
+++ b/src/setjmp/x32/longjmp.s
@@ -5,18 +5,14 @@
.type longjmp,@function
_longjmp:
longjmp:
- mov %rsi,%rax /* val will be longjmp return */
- test %rax,%rax
- jnz 1f
- inc %rax /* if val==0, val=1 per longjmp semantics */
-1:
+ xor %eax,%eax
+ cmp $1,%esi /* CF = val ? 0 : 1 */
+ adc %esi,%eax /* eax = val + !val */
mov (%rdi),%rbx /* rdi is the jmp_buf, restore regs from it */
mov 8(%rdi),%rbp
mov 16(%rdi),%r12
mov 24(%rdi),%r13
mov 32(%rdi),%r14
mov 40(%rdi),%r15
- mov 48(%rdi),%rdx /* this ends up being the stack pointer */
- mov %rdx,%rsp
- mov 56(%rdi),%rdx /* this is the instruction pointer */
- jmp *%rdx /* goto saved address without altering rsp */
+ mov 48(%rdi),%rsp
+ jmp *56(%rdi) /* goto saved address without altering rsp */
diff --git a/src/setjmp/x32/setjmp.s b/src/setjmp/x32/setjmp.s
index 98f58b8d..d95e4853 100644
--- a/src/setjmp/x32/setjmp.s
+++ b/src/setjmp/x32/setjmp.s
@@ -18,5 +18,5 @@ setjmp:
mov %rdx,48(%rdi)
mov (%rsp),%rdx /* save return addr ptr for new rip */
mov %rdx,56(%rdi)
- xor %rax,%rax /* always return 0 */
+ xor %eax,%eax /* always return 0 */
ret
diff --git a/src/setjmp/x86_64/longjmp.s b/src/setjmp/x86_64/longjmp.s
index e175a4b9..1b2661c3 100644
--- a/src/setjmp/x86_64/longjmp.s
+++ b/src/setjmp/x86_64/longjmp.s
@@ -5,18 +5,14 @@
.type longjmp,@function
_longjmp:
longjmp:
- mov %rsi,%rax /* val will be longjmp return */
- test %rax,%rax
- jnz 1f
- inc %rax /* if val==0, val=1 per longjmp semantics */
-1:
+ xor %eax,%eax
+ cmp $1,%esi /* CF = val ? 0 : 1 */
+ adc %esi,%eax /* eax = val + !val */
mov (%rdi),%rbx /* rdi is the jmp_buf, restore regs from it */
mov 8(%rdi),%rbp
mov 16(%rdi),%r12
mov 24(%rdi),%r13
mov 32(%rdi),%r14
mov 40(%rdi),%r15
- mov 48(%rdi),%rdx /* this ends up being the stack pointer */
- mov %rdx,%rsp
- mov 56(%rdi),%rdx /* this is the instruction pointer */
- jmp *%rdx /* goto saved address without altering rsp */
+ mov 48(%rdi),%rsp
+ jmp *56(%rdi) /* goto saved address without altering rsp */
diff --git a/src/setjmp/x86_64/setjmp.s b/src/setjmp/x86_64/setjmp.s
index 98f58b8d..d95e4853 100644
--- a/src/setjmp/x86_64/setjmp.s
+++ b/src/setjmp/x86_64/setjmp.s
@@ -18,5 +18,5 @@ setjmp:
mov %rdx,48(%rdi)
mov (%rsp),%rdx /* save return addr ptr for new rip */
mov %rdx,56(%rdi)
- xor %rax,%rax /* always return 0 */
+ xor %eax,%eax /* always return 0 */
ret
diff --git a/src/signal/arm/sigsetjmp.s b/src/signal/arm/sigsetjmp.s
index 318addba..69ebbf49 100644
--- a/src/signal/arm/sigsetjmp.s
+++ b/src/signal/arm/sigsetjmp.s
@@ -6,9 +6,10 @@
sigsetjmp:
__sigsetjmp:
tst r1,r1
- beq setjmp
+ bne 1f
+ b setjmp
- str lr,[r0,#256]
+1: str lr,[r0,#256]
str r4,[r0,#260+8]
mov r4,r0
diff --git a/src/stat/fchmodat.c b/src/stat/fchmodat.c
index be61bdf3..4ee00b0a 100644
--- a/src/stat/fchmodat.c
+++ b/src/stat/fchmodat.c
@@ -2,6 +2,7 @@
#include <fcntl.h>
#include <errno.h>
#include "syscall.h"
+#include "kstat.h"
int fchmodat(int fd, const char *path, mode_t mode, int flag)
{
@@ -10,7 +11,7 @@ int fchmodat(int fd, const char *path, mode_t mode, int flag)
if (flag != AT_SYMLINK_NOFOLLOW)
return __syscall_ret(-EINVAL);
- struct stat st;
+ struct kstat st;
int ret, fd2;
char proc[15+3*sizeof(int)];
diff --git a/src/stdio/__string_read.c b/src/stdio/__string_read.c
deleted file mode 100644
index 7b50a7e1..00000000
--- a/src/stdio/__string_read.c
+++ /dev/null
@@ -1,16 +0,0 @@
-#include "stdio_impl.h"
-#include <string.h>
-
-size_t __string_read(FILE *f, unsigned char *buf, size_t len)
-{
- char *src = f->cookie;
- size_t k = len+256;
- char *end = memchr(src, 0, k);
- if (end) k = end-src;
- if (k < len) len = k;
- memcpy(buf, src, len);
- f->rpos = (void *)(src+len);
- f->rend = (void *)(src+k);
- f->cookie = src+k;
- return len;
-}
diff --git a/src/stdio/fmemopen.c b/src/stdio/fmemopen.c
index 5685092e..343e3e3f 100644
--- a/src/stdio/fmemopen.c
+++ b/src/stdio/fmemopen.c
@@ -2,6 +2,7 @@
#include <errno.h>
#include <string.h>
#include <stdlib.h>
+#include <stddef.h>
#include <inttypes.h>
#include "libc.h"
@@ -95,18 +96,17 @@ FILE *fmemopen(void *restrict buf, size_t size, const char *restrict mode)
f = malloc(sizeof *f + (buf?0:size));
if (!f) return 0;
- memset(&f->f, 0, sizeof f->f);
+ memset(f, 0, offsetof(struct mem_FILE, buf));
f->f.cookie = &f->c;
f->f.fd = -1;
f->f.lbf = EOF;
f->f.buf = f->buf + UNGET;
f->f.buf_size = sizeof f->buf - UNGET;
if (!buf) {
- buf = f->buf2;;
+ buf = f->buf2;
memset(buf, 0, size);
}
- memset(&f->c, 0, sizeof f->c);
f->c.buf = buf;
f->c.size = size;
f->c.mode = *mode;
diff --git a/src/stdio/tempnam.c b/src/stdio/tempnam.c
index 84f91978..565df6b6 100644
--- a/src/stdio/tempnam.c
+++ b/src/stdio/tempnam.c
@@ -6,6 +6,7 @@
#include <string.h>
#include <stdlib.h>
#include "syscall.h"
+#include "kstat.h"
#define MAXTRIES 100
@@ -37,10 +38,10 @@ char *tempnam(const char *dir, const char *pfx)
for (try=0; try<MAXTRIES; try++) {
__randname(s+l-6);
#ifdef SYS_lstat
- r = __syscall(SYS_lstat, s, &(struct stat){0});
+ r = __syscall(SYS_lstat, s, &(struct kstat){0});
#else
r = __syscall(SYS_fstatat, AT_FDCWD, s,
- &(struct stat){0}, AT_SYMLINK_NOFOLLOW);
+ &(struct kstat){0}, AT_SYMLINK_NOFOLLOW);
#endif
if (r == -ENOENT) return strdup(s);
}
diff --git a/src/stdio/tmpnam.c b/src/stdio/tmpnam.c
index 6c7c253a..d667a836 100644
--- a/src/stdio/tmpnam.c
+++ b/src/stdio/tmpnam.c
@@ -5,6 +5,7 @@
#include <string.h>
#include <stdlib.h>
#include "syscall.h"
+#include "kstat.h"
#define MAXTRIES 100
@@ -17,10 +18,10 @@ char *tmpnam(char *buf)
for (try=0; try<MAXTRIES; try++) {
__randname(s+12);
#ifdef SYS_lstat
- r = __syscall(SYS_lstat, s, &(struct stat){0});
+ r = __syscall(SYS_lstat, s, &(struct kstat){0});
#else
r = __syscall(SYS_fstatat, AT_FDCWD, s,
- &(struct stat){0}, AT_SYMLINK_NOFOLLOW);
+ &(struct kstat){0}, AT_SYMLINK_NOFOLLOW);
#endif
if (r == -ENOENT) return strcpy(buf ? buf : internal, s);
}
diff --git a/src/stdio/vdprintf.c b/src/stdio/vdprintf.c
index c35d9b4f..3b9c093b 100644
--- a/src/stdio/vdprintf.c
+++ b/src/stdio/vdprintf.c
@@ -1,14 +1,9 @@
#include "stdio_impl.h"
-static size_t wrap_write(FILE *f, const unsigned char *buf, size_t len)
-{
- return __stdio_write(f, buf, len);
-}
-
int vdprintf(int fd, const char *restrict fmt, va_list ap)
{
FILE f = {
- .fd = fd, .lbf = EOF, .write = wrap_write,
+ .fd = fd, .lbf = EOF, .write = __stdio_write,
.buf = (void *)fmt, .buf_size = 0,
.lock = -1
};
diff --git a/src/stdio/vfscanf.c b/src/stdio/vfscanf.c
index 9e030fc4..b78a374d 100644
--- a/src/stdio/vfscanf.c
+++ b/src/stdio/vfscanf.c
@@ -57,7 +57,7 @@ int vfscanf(FILE *restrict f, const char *restrict fmt, va_list ap)
{
int width;
int size;
- int alloc;
+ int alloc = 0;
int base;
const unsigned char *p;
int c, t;
@@ -76,6 +76,9 @@ int vfscanf(FILE *restrict f, const char *restrict fmt, va_list ap)
FLOCK(f);
+ if (!f->rpos) __toread(f);
+ if (!f->rpos) goto input_fail;
+
for (p=(const unsigned char *)fmt; *p; p++) {
alloc = 0;
diff --git a/src/stdio/vsscanf.c b/src/stdio/vsscanf.c
index 98500225..4d6d259b 100644
--- a/src/stdio/vsscanf.c
+++ b/src/stdio/vsscanf.c
@@ -1,15 +1,25 @@
#include "stdio_impl.h"
+#include <string.h>
-static size_t do_read(FILE *f, unsigned char *buf, size_t len)
+static size_t string_read(FILE *f, unsigned char *buf, size_t len)
{
- return __string_read(f, buf, len);
+ char *src = f->cookie;
+ size_t k = len+256;
+ char *end = memchr(src, 0, k);
+ if (end) k = end-src;
+ if (k < len) len = k;
+ memcpy(buf, src, len);
+ f->rpos = (void *)(src+len);
+ f->rend = (void *)(src+k);
+ f->cookie = src+k;
+ return len;
}
int vsscanf(const char *restrict s, const char *restrict fmt, va_list ap)
{
FILE f = {
.buf = (void *)s, .cookie = (void *)s,
- .read = do_read, .lock = -1
+ .read = string_read, .lock = -1
};
return vfscanf(&f, fmt, ap);
}
diff --git a/src/stdlib/wcstod.c b/src/stdlib/wcstod.c
index 26fe9af8..0deb7010 100644
--- a/src/stdlib/wcstod.c
+++ b/src/stdlib/wcstod.c
@@ -33,8 +33,7 @@ static long double wcstox(const wchar_t *s, wchar_t **p, int prec)
unsigned char buf[64];
FILE f = {0};
f.flags = 0;
- f.rpos = f.rend = 0;
- f.buf = buf + 4;
+ f.rpos = f.rend = f.buf = buf + 4;
f.buf_size = sizeof buf - 4;
f.lock = -1;
f.read = do_read;
diff --git a/src/stdlib/wcstol.c b/src/stdlib/wcstol.c
index 4443f577..1eeb495f 100644
--- a/src/stdlib/wcstol.c
+++ b/src/stdlib/wcstol.c
@@ -35,8 +35,7 @@ static unsigned long long wcstox(const wchar_t *s, wchar_t **p, int base, unsign
unsigned char buf[64];
FILE f = {0};
f.flags = 0;
- f.rpos = f.rend = 0;
- f.buf = buf + 4;
+ f.rpos = f.rend = f.buf = buf + 4;
f.buf_size = sizeof buf - 4;
f.lock = -1;
f.read = do_read;
diff --git a/src/string/aarch64/memcpy.S b/src/string/aarch64/memcpy.S
new file mode 100644
index 00000000..48bb8a8d
--- /dev/null
+++ b/src/string/aarch64/memcpy.S
@@ -0,0 +1,186 @@
+/*
+ * memcpy - copy memory area
+ *
+ * Copyright (c) 2012-2020, Arm Limited.
+ * SPDX-License-Identifier: MIT
+ */
+
+/* Assumptions:
+ *
+ * ARMv8-a, AArch64, unaligned accesses.
+ *
+ */
+
+#define dstin x0
+#define src x1
+#define count x2
+#define dst x3
+#define srcend x4
+#define dstend x5
+#define A_l x6
+#define A_lw w6
+#define A_h x7
+#define B_l x8
+#define B_lw w8
+#define B_h x9
+#define C_l x10
+#define C_lw w10
+#define C_h x11
+#define D_l x12
+#define D_h x13
+#define E_l x14
+#define E_h x15
+#define F_l x16
+#define F_h x17
+#define G_l count
+#define G_h dst
+#define H_l src
+#define H_h srcend
+#define tmp1 x14
+
+/* This implementation of memcpy uses unaligned accesses and branchless
+ sequences to keep the code small, simple and improve performance.
+
+ Copies are split into 3 main cases: small copies of up to 32 bytes, medium
+ copies of up to 128 bytes, and large copies. The overhead of the overlap
+ check is negligible since it is only required for large copies.
+
+ Large copies use a software pipelined loop processing 64 bytes per iteration.
+ The destination pointer is 16-byte aligned to minimize unaligned accesses.
+ The loop tail is handled by always copying 64 bytes from the end.
+*/
+
+.global memcpy
+.type memcpy,%function
+memcpy:
+ add srcend, src, count
+ add dstend, dstin, count
+ cmp count, 128
+ b.hi .Lcopy_long
+ cmp count, 32
+ b.hi .Lcopy32_128
+
+ /* Small copies: 0..32 bytes. */
+ cmp count, 16
+ b.lo .Lcopy16
+ ldp A_l, A_h, [src]
+ ldp D_l, D_h, [srcend, -16]
+ stp A_l, A_h, [dstin]
+ stp D_l, D_h, [dstend, -16]
+ ret
+
+ /* Copy 8-15 bytes. */
+.Lcopy16:
+ tbz count, 3, .Lcopy8
+ ldr A_l, [src]
+ ldr A_h, [srcend, -8]
+ str A_l, [dstin]
+ str A_h, [dstend, -8]
+ ret
+
+ .p2align 3
+ /* Copy 4-7 bytes. */
+.Lcopy8:
+ tbz count, 2, .Lcopy4
+ ldr A_lw, [src]
+ ldr B_lw, [srcend, -4]
+ str A_lw, [dstin]
+ str B_lw, [dstend, -4]
+ ret
+
+ /* Copy 0..3 bytes using a branchless sequence. */
+.Lcopy4:
+ cbz count, .Lcopy0
+ lsr tmp1, count, 1
+ ldrb A_lw, [src]
+ ldrb C_lw, [srcend, -1]
+ ldrb B_lw, [src, tmp1]
+ strb A_lw, [dstin]
+ strb B_lw, [dstin, tmp1]
+ strb C_lw, [dstend, -1]
+.Lcopy0:
+ ret
+
+ .p2align 4
+ /* Medium copies: 33..128 bytes. */
+.Lcopy32_128:
+ ldp A_l, A_h, [src]
+ ldp B_l, B_h, [src, 16]
+ ldp C_l, C_h, [srcend, -32]
+ ldp D_l, D_h, [srcend, -16]
+ cmp count, 64
+ b.hi .Lcopy128
+ stp A_l, A_h, [dstin]
+ stp B_l, B_h, [dstin, 16]
+ stp C_l, C_h, [dstend, -32]
+ stp D_l, D_h, [dstend, -16]
+ ret
+
+ .p2align 4
+ /* Copy 65..128 bytes. */
+.Lcopy128:
+ ldp E_l, E_h, [src, 32]
+ ldp F_l, F_h, [src, 48]
+ cmp count, 96
+ b.ls .Lcopy96
+ ldp G_l, G_h, [srcend, -64]
+ ldp H_l, H_h, [srcend, -48]
+ stp G_l, G_h, [dstend, -64]
+ stp H_l, H_h, [dstend, -48]
+.Lcopy96:
+ stp A_l, A_h, [dstin]
+ stp B_l, B_h, [dstin, 16]
+ stp E_l, E_h, [dstin, 32]
+ stp F_l, F_h, [dstin, 48]
+ stp C_l, C_h, [dstend, -32]
+ stp D_l, D_h, [dstend, -16]
+ ret
+
+ .p2align 4
+ /* Copy more than 128 bytes. */
+.Lcopy_long:
+
+ /* Copy 16 bytes and then align dst to 16-byte alignment. */
+
+ ldp D_l, D_h, [src]
+ and tmp1, dstin, 15
+ bic dst, dstin, 15
+ sub src, src, tmp1
+ add count, count, tmp1 /* Count is now 16 too large. */
+ ldp A_l, A_h, [src, 16]
+ stp D_l, D_h, [dstin]
+ ldp B_l, B_h, [src, 32]
+ ldp C_l, C_h, [src, 48]
+ ldp D_l, D_h, [src, 64]!
+ subs count, count, 128 + 16 /* Test and readjust count. */
+ b.ls .Lcopy64_from_end
+
+.Lloop64:
+ stp A_l, A_h, [dst, 16]
+ ldp A_l, A_h, [src, 16]
+ stp B_l, B_h, [dst, 32]
+ ldp B_l, B_h, [src, 32]
+ stp C_l, C_h, [dst, 48]
+ ldp C_l, C_h, [src, 48]
+ stp D_l, D_h, [dst, 64]!
+ ldp D_l, D_h, [src, 64]!
+ subs count, count, 64
+ b.hi .Lloop64
+
+ /* Write the last iteration and copy 64 bytes from the end. */
+.Lcopy64_from_end:
+ ldp E_l, E_h, [srcend, -64]
+ stp A_l, A_h, [dst, 16]
+ ldp A_l, A_h, [srcend, -48]
+ stp B_l, B_h, [dst, 32]
+ ldp B_l, B_h, [srcend, -32]
+ stp C_l, C_h, [dst, 48]
+ ldp C_l, C_h, [srcend, -16]
+ stp D_l, D_h, [dst, 64]
+ stp E_l, E_h, [dstend, -64]
+ stp A_l, A_h, [dstend, -48]
+ stp B_l, B_h, [dstend, -32]
+ stp C_l, C_h, [dstend, -16]
+ ret
+
+.size memcpy,.-memcpy
diff --git a/src/string/aarch64/memset.S b/src/string/aarch64/memset.S
new file mode 100644
index 00000000..f0d29b7f
--- /dev/null
+++ b/src/string/aarch64/memset.S
@@ -0,0 +1,115 @@
+/*
+ * memset - fill memory with a constant byte
+ *
+ * Copyright (c) 2012-2020, Arm Limited.
+ * SPDX-License-Identifier: MIT
+ */
+
+/* Assumptions:
+ *
+ * ARMv8-a, AArch64, Advanced SIMD, unaligned accesses.
+ *
+ */
+
+#define dstin x0
+#define val x1
+#define valw w1
+#define count x2
+#define dst x3
+#define dstend x4
+#define zva_val x5
+
+.global memset
+.type memset,%function
+memset:
+
+ dup v0.16B, valw
+ add dstend, dstin, count
+
+ cmp count, 96
+ b.hi .Lset_long
+ cmp count, 16
+ b.hs .Lset_medium
+ mov val, v0.D[0]
+
+ /* Set 0..15 bytes. */
+ tbz count, 3, 1f
+ str val, [dstin]
+ str val, [dstend, -8]
+ ret
+ nop
+1: tbz count, 2, 2f
+ str valw, [dstin]
+ str valw, [dstend, -4]
+ ret
+2: cbz count, 3f
+ strb valw, [dstin]
+ tbz count, 1, 3f
+ strh valw, [dstend, -2]
+3: ret
+
+ /* Set 17..96 bytes. */
+.Lset_medium:
+ str q0, [dstin]
+ tbnz count, 6, .Lset96
+ str q0, [dstend, -16]
+ tbz count, 5, 1f
+ str q0, [dstin, 16]
+ str q0, [dstend, -32]
+1: ret
+
+ .p2align 4
+ /* Set 64..96 bytes. Write 64 bytes from the start and
+ 32 bytes from the end. */
+.Lset96:
+ str q0, [dstin, 16]
+ stp q0, q0, [dstin, 32]
+ stp q0, q0, [dstend, -32]
+ ret
+
+ .p2align 4
+.Lset_long:
+ and valw, valw, 255
+ bic dst, dstin, 15
+ str q0, [dstin]
+ cmp count, 160
+ ccmp valw, 0, 0, hs
+ b.ne .Lno_zva
+
+#ifndef SKIP_ZVA_CHECK
+ mrs zva_val, dczid_el0
+ and zva_val, zva_val, 31
+ cmp zva_val, 4 /* ZVA size is 64 bytes. */
+ b.ne .Lno_zva
+#endif
+ str q0, [dst, 16]
+ stp q0, q0, [dst, 32]
+ bic dst, dst, 63
+ sub count, dstend, dst /* Count is now 64 too large. */
+ sub count, count, 128 /* Adjust count and bias for loop. */
+
+ .p2align 4
+.Lzva_loop:
+ add dst, dst, 64
+ dc zva, dst
+ subs count, count, 64
+ b.hi .Lzva_loop
+ stp q0, q0, [dstend, -64]
+ stp q0, q0, [dstend, -32]
+ ret
+
+.Lno_zva:
+ sub count, dstend, dst /* Count is 16 too large. */
+ sub dst, dst, 16 /* Dst is biased by -32. */
+ sub count, count, 64 + 16 /* Adjust count and bias for loop. */
+.Lno_zva_loop:
+ stp q0, q0, [dst, 32]
+ stp q0, q0, [dst, 64]!
+ subs count, count, 64
+ b.hi .Lno_zva_loop
+ stp q0, q0, [dstend, -64]
+ stp q0, q0, [dstend, -32]
+ ret
+
+.size memset,.-memset
+
diff --git a/src/string/arm/memcpy_le.S b/src/string/arm/memcpy.S
index 9cfbcb2a..869e3448 100644
--- a/src/string/arm/memcpy_le.S
+++ b/src/string/arm/memcpy.S
@@ -1,5 +1,3 @@
-#if !__ARMEB__ && !__thumb__
-
/*
* Copyright (C) 2008 The Android Open Source Project
* All rights reserved.
@@ -40,8 +38,9 @@
* This file has been modified from the original for use in musl libc.
* The main changes are: addition of .type memcpy,%function to make the
* code safely callable from thumb mode, adjusting the return
- * instructions to be compatible with pre-thumb ARM cpus, and removal
- * of prefetch code that is not compatible with older cpus.
+ * instructions to be compatible with pre-thumb ARM cpus, removal of
+ * prefetch code that is not compatible with older cpus and support for
+ * building as thumb 2 and big-endian.
*/
.syntax unified
@@ -226,23 +225,45 @@ non_congruent:
* becomes aligned to 32 bits (r5 = nb of words to copy for alignment)
*/
movs r5, r5, lsl #31
+
+#if __ARMEB__
+ movmi r3, r3, ror #24
+ strbmi r3, [r0], #1
+ movcs r3, r3, ror #24
+ strbcs r3, [r0], #1
+ movcs r3, r3, ror #24
+ strbcs r3, [r0], #1
+#else
strbmi r3, [r0], #1
movmi r3, r3, lsr #8
strbcs r3, [r0], #1
movcs r3, r3, lsr #8
strbcs r3, [r0], #1
movcs r3, r3, lsr #8
+#endif
cmp r2, #4
blo partial_word_tail
+#if __ARMEB__
+ mov r3, r3, lsr r12
+ mov r3, r3, lsl r12
+#endif
+
/* Align destination to 32 bytes (cache line boundary) */
1: tst r0, #0x1c
beq 2f
ldr r5, [r1], #4
sub r2, r2, #4
- orr r4, r3, r5, lsl lr
+#if __ARMEB__
+ mov r4, r5, lsr lr
+ orr r4, r4, r3
+ mov r3, r5, lsl r12
+#else
+ mov r4, r5, lsl lr
+ orr r4, r4, r3
mov r3, r5, lsr r12
+#endif
str r4, [r0], #4
cmp r2, #4
bhs 1b
@@ -268,6 +289,25 @@ loop16:
ldmia r1!, { r5,r6,r7, r8,r9,r10,r11}
subs r2, r2, #32
ldrhs r12, [r1], #4
+#if __ARMEB__
+ orr r3, r3, r4, lsr #16
+ mov r4, r4, lsl #16
+ orr r4, r4, r5, lsr #16
+ mov r5, r5, lsl #16
+ orr r5, r5, r6, lsr #16
+ mov r6, r6, lsl #16
+ orr r6, r6, r7, lsr #16
+ mov r7, r7, lsl #16
+ orr r7, r7, r8, lsr #16
+ mov r8, r8, lsl #16
+ orr r8, r8, r9, lsr #16
+ mov r9, r9, lsl #16
+ orr r9, r9, r10, lsr #16
+ mov r10, r10, lsl #16
+ orr r10, r10, r11, lsr #16
+ stmia r0!, {r3,r4,r5,r6, r7,r8,r9,r10}
+ mov r3, r11, lsl #16
+#else
orr r3, r3, r4, lsl #16
mov r4, r4, lsr #16
orr r4, r4, r5, lsl #16
@@ -285,6 +325,7 @@ loop16:
orr r10, r10, r11, lsl #16
stmia r0!, {r3,r4,r5,r6, r7,r8,r9,r10}
mov r3, r11, lsr #16
+#endif
bhs 1b
b less_than_thirtytwo
@@ -294,6 +335,25 @@ loop8:
ldmia r1!, { r5,r6,r7, r8,r9,r10,r11}
subs r2, r2, #32
ldrhs r12, [r1], #4
+#if __ARMEB__
+ orr r3, r3, r4, lsr #24
+ mov r4, r4, lsl #8
+ orr r4, r4, r5, lsr #24
+ mov r5, r5, lsl #8
+ orr r5, r5, r6, lsr #24
+ mov r6, r6, lsl #8
+ orr r6, r6, r7, lsr #24
+ mov r7, r7, lsl #8
+ orr r7, r7, r8, lsr #24
+ mov r8, r8, lsl #8
+ orr r8, r8, r9, lsr #24
+ mov r9, r9, lsl #8
+ orr r9, r9, r10, lsr #24
+ mov r10, r10, lsl #8
+ orr r10, r10, r11, lsr #24
+ stmia r0!, {r3,r4,r5,r6, r7,r8,r9,r10}
+ mov r3, r11, lsl #8
+#else
orr r3, r3, r4, lsl #24
mov r4, r4, lsr #8
orr r4, r4, r5, lsl #24
@@ -311,6 +371,7 @@ loop8:
orr r10, r10, r11, lsl #24
stmia r0!, {r3,r4,r5,r6, r7,r8,r9,r10}
mov r3, r11, lsr #8
+#endif
bhs 1b
b less_than_thirtytwo
@@ -320,6 +381,25 @@ loop24:
ldmia r1!, { r5,r6,r7, r8,r9,r10,r11}
subs r2, r2, #32
ldrhs r12, [r1], #4
+#if __ARMEB__
+ orr r3, r3, r4, lsr #8
+ mov r4, r4, lsl #24
+ orr r4, r4, r5, lsr #8
+ mov r5, r5, lsl #24
+ orr r5, r5, r6, lsr #8
+ mov r6, r6, lsl #24
+ orr r6, r6, r7, lsr #8
+ mov r7, r7, lsl #24
+ orr r7, r7, r8, lsr #8
+ mov r8, r8, lsl #24
+ orr r8, r8, r9, lsr #8
+ mov r9, r9, lsl #24
+ orr r9, r9, r10, lsr #8
+ mov r10, r10, lsl #24
+ orr r10, r10, r11, lsr #8
+ stmia r0!, {r3,r4,r5,r6, r7,r8,r9,r10}
+ mov r3, r11, lsl #24
+#else
orr r3, r3, r4, lsl #8
mov r4, r4, lsr #24
orr r4, r4, r5, lsl #8
@@ -337,6 +417,7 @@ loop24:
orr r10, r10, r11, lsl #8
stmia r0!, {r3,r4,r5,r6, r7,r8,r9,r10}
mov r3, r11, lsr #24
+#endif
bhs 1b
less_than_thirtytwo:
@@ -348,8 +429,15 @@ less_than_thirtytwo:
1: ldr r5, [r1], #4
sub r2, r2, #4
- orr r4, r3, r5, lsl lr
+#if __ARMEB__
+ mov r4, r5, lsr lr
+ orr r4, r4, r3
+ mov r3, r5, lsl r12
+#else
+ mov r4, r5, lsl lr
+ orr r4, r4, r3
mov r3, r5, lsr r12
+#endif
str r4, [r0], #4
cmp r2, #4
bhs 1b
@@ -357,11 +445,20 @@ less_than_thirtytwo:
partial_word_tail:
/* we have a partial word in the input buffer */
movs r5, lr, lsl #(31-3)
+#if __ARMEB__
+ movmi r3, r3, ror #24
+ strbmi r3, [r0], #1
+ movcs r3, r3, ror #24
+ strbcs r3, [r0], #1
+ movcs r3, r3, ror #24
+ strbcs r3, [r0], #1
+#else
strbmi r3, [r0], #1
movmi r3, r3, lsr #8
strbcs r3, [r0], #1
movcs r3, r3, lsr #8
strbcs r3, [r0], #1
+#endif
/* Refill spilled registers from the stack. Don't update sp. */
ldmfd sp, {r5-r11}
@@ -380,4 +477,3 @@ copy_last_3_and_return:
ldmfd sp!, {r0, r4, lr}
bx lr
-#endif
diff --git a/src/string/arm/memcpy.c b/src/string/arm/memcpy.c
deleted file mode 100644
index f703c9bd..00000000
--- a/src/string/arm/memcpy.c
+++ /dev/null
@@ -1,3 +0,0 @@
-#if __ARMEB__ || __thumb__
-#include "../memcpy.c"
-#endif
diff --git a/src/string/memccpy.c b/src/string/memccpy.c
index 00c18e2b..3b0a3700 100644
--- a/src/string/memccpy.c
+++ b/src/string/memccpy.c
@@ -29,6 +29,6 @@ void *memccpy(void *restrict dest, const void *restrict src, int c, size_t n)
#endif
for (; n && (*d=*s)!=c; n--, s++, d++);
tail:
- if (n && *s==c) return d+1;
+ if (n) return d+1;
return 0;
}
diff --git a/src/string/memmem.c b/src/string/memmem.c
index 58a21fcd..11eff86e 100644
--- a/src/string/memmem.c
+++ b/src/string/memmem.c
@@ -12,8 +12,8 @@ static char *twobyte_memmem(const unsigned char *h, size_t k, const unsigned cha
static char *threebyte_memmem(const unsigned char *h, size_t k, const unsigned char *n)
{
- uint32_t nw = n[0]<<24 | n[1]<<16 | n[2]<<8;
- uint32_t hw = h[0]<<24 | h[1]<<16 | h[2]<<8;
+ uint32_t nw = (uint32_t)n[0]<<24 | n[1]<<16 | n[2]<<8;
+ uint32_t hw = (uint32_t)h[0]<<24 | h[1]<<16 | h[2]<<8;
for (h+=3, k-=3; k; k--, hw = (hw|*h++)<<8)
if (hw == nw) return (char *)h-3;
return hw == nw ? (char *)h-3 : 0;
@@ -21,8 +21,8 @@ static char *threebyte_memmem(const unsigned char *h, size_t k, const unsigned c
static char *fourbyte_memmem(const unsigned char *h, size_t k, const unsigned char *n)
{
- uint32_t nw = n[0]<<24 | n[1]<<16 | n[2]<<8 | n[3];
- uint32_t hw = h[0]<<24 | h[1]<<16 | h[2]<<8 | h[3];
+ uint32_t nw = (uint32_t)n[0]<<24 | n[1]<<16 | n[2]<<8 | n[3];
+ uint32_t hw = (uint32_t)h[0]<<24 | h[1]<<16 | h[2]<<8 | h[3];
for (h+=4, k-=4; k; k--, hw = hw<<8 | *h++)
if (hw == nw) return (char *)h-4;
return hw == nw ? (char *)h-4 : 0;
diff --git a/src/string/strsignal.c b/src/string/strsignal.c
index 96bfe841..5156366e 100644
--- a/src/string/strsignal.c
+++ b/src/string/strsignal.c
@@ -31,7 +31,11 @@ static const char map[] = {
[SIGPIPE] = 13,
[SIGALRM] = 14,
[SIGTERM] = 15,
+#if defined(SIGSTKFLT)
[SIGSTKFLT] = 16,
+#elif defined(SIGEMT)
+ [SIGEMT] = 16,
+#endif
[SIGCHLD] = 17,
[SIGCONT] = 18,
[SIGSTOP] = 19,
@@ -70,7 +74,13 @@ static const char strings[] =
"Broken pipe\0"
"Alarm clock\0"
"Terminated\0"
+#if defined(SIGSTKFLT)
"Stack fault\0"
+#elif defined(SIGEMT)
+ "Emulator trap\0"
+#else
+ "Unknown signal\0"
+#endif
"Child process status\0"
"Continued\0"
"Stopped (signal)\0"
diff --git a/src/string/strstr.c b/src/string/strstr.c
index 55ba1c7b..43a0207a 100644
--- a/src/string/strstr.c
+++ b/src/string/strstr.c
@@ -10,16 +10,16 @@ static char *twobyte_strstr(const unsigned char *h, const unsigned char *n)
static char *threebyte_strstr(const unsigned char *h, const unsigned char *n)
{
- uint32_t nw = n[0]<<24 | n[1]<<16 | n[2]<<8;
- uint32_t hw = h[0]<<24 | h[1]<<16 | h[2]<<8;
+ uint32_t nw = (uint32_t)n[0]<<24 | n[1]<<16 | n[2]<<8;
+ uint32_t hw = (uint32_t)h[0]<<24 | h[1]<<16 | h[2]<<8;
for (h+=2; *h && hw != nw; hw = (hw|*++h)<<8);
return *h ? (char *)h-2 : 0;
}
static char *fourbyte_strstr(const unsigned char *h, const unsigned char *n)
{
- uint32_t nw = n[0]<<24 | n[1]<<16 | n[2]<<8 | n[3];
- uint32_t hw = h[0]<<24 | h[1]<<16 | h[2]<<8 | h[3];
+ uint32_t nw = (uint32_t)n[0]<<24 | n[1]<<16 | n[2]<<8 | n[3];
+ uint32_t hw = (uint32_t)h[0]<<24 | h[1]<<16 | h[2]<<8 | h[3];
for (h+=3; *h && hw != nw; hw = hw<<8 | *++h);
return *h ? (char *)h-3 : 0;
}
diff --git a/src/termios/tcgetwinsize.c b/src/termios/tcgetwinsize.c
new file mode 100644
index 00000000..9b3a65a4
--- /dev/null
+++ b/src/termios/tcgetwinsize.c
@@ -0,0 +1,8 @@
+#include <termios.h>
+#include <sys/ioctl.h>
+#include "syscall.h"
+
+int tcgetwinsize(int fd, struct winsize *wsz)
+{
+ return syscall(SYS_ioctl, fd, TIOCGWINSZ, wsz);
+}
diff --git a/src/termios/tcsetwinsize.c b/src/termios/tcsetwinsize.c
new file mode 100644
index 00000000..e01d0e25
--- /dev/null
+++ b/src/termios/tcsetwinsize.c
@@ -0,0 +1,8 @@
+#include <termios.h>
+#include <sys/ioctl.h>
+#include "syscall.h"
+
+int tcsetwinsize(int fd, const struct winsize *wsz)
+{
+ return syscall(SYS_ioctl, fd, TIOCSWINSZ, wsz);
+}
diff --git a/src/thread/__lock.c b/src/thread/__lock.c
index 45557c88..60eece49 100644
--- a/src/thread/__lock.c
+++ b/src/thread/__lock.c
@@ -18,9 +18,11 @@
void __lock(volatile int *l)
{
- if (!libc.threads_minus_1) return;
+ int need_locks = libc.need_locks;
+ if (!need_locks) return;
/* fast path: INT_MIN for the lock, +1 for the congestion */
int current = a_cas(l, 0, INT_MIN + 1);
+ if (need_locks < 0) libc.need_locks = 0;
if (!current) return;
/* A first spin loop, for medium congestion. */
for (unsigned i = 0; i < 10; ++i) {
diff --git a/src/thread/i386/__set_thread_area.s b/src/thread/i386/__set_thread_area.s
index c2c21dd5..aa6852be 100644
--- a/src/thread/i386/__set_thread_area.s
+++ b/src/thread/i386/__set_thread_area.s
@@ -28,6 +28,7 @@ __set_thread_area:
ret
2:
mov %ebx,%ecx
+ xor %eax,%eax
xor %ebx,%ebx
xor %edx,%edx
mov %ebx,(%esp)
diff --git a/src/thread/pthread_create.c b/src/thread/pthread_create.c
index 5f491092..55744155 100644
--- a/src/thread/pthread_create.c
+++ b/src/thread/pthread_create.c
@@ -72,12 +72,13 @@ _Noreturn void __pthread_exit(void *result)
/* Access to target the exiting thread with syscalls that use
* its kernel tid is controlled by killlock. For detached threads,
* any use past this point would have undefined behavior, but for
- * joinable threads it's a valid usage that must be handled. */
+ * joinable threads it's a valid usage that must be handled.
+ * Signals must be blocked since pthread_kill must be AS-safe. */
+ __block_app_sigs(&set);
LOCK(self->killlock);
- /* The thread list lock must be AS-safe, and thus requires
- * application signals to be blocked before it can be taken. */
- __block_app_sigs(&set);
+ /* The thread list lock must be AS-safe, and thus depends on
+ * application signals being blocked above. */
__tl_lock();
/* If this is the only thread in the list, don't proceed with
@@ -85,19 +86,12 @@ _Noreturn void __pthread_exit(void *result)
* signal state to prepare for exit to call atexit handlers. */
if (self->next == self) {
__tl_unlock();
- __restore_sigs(&set);
UNLOCK(self->killlock);
+ __restore_sigs(&set);
exit(0);
}
- /* At this point we are committed to thread termination. Unlink
- * the thread from the list. This change will not be visible
- * until the lock is released, which only happens after SYS_exit
- * has been called, via the exit futex address pointing at the lock. */
- libc.threads_minus_1--;
- self->next->prev = self->prev;
- self->prev->next = self->next;
- self->prev = self->next = self;
+ /* At this point we are committed to thread termination. */
/* Process robust list in userspace to handle non-pshared mutexes
* and the detached thread case where the robust list head will
@@ -121,6 +115,16 @@ _Noreturn void __pthread_exit(void *result)
__do_orphaned_stdio_locks();
__dl_thread_cleanup();
+ /* Last, unlink thread from the list. This change will not be visible
+ * until the lock is released, which only happens after SYS_exit
+ * has been called, via the exit futex address pointing at the lock.
+ * This needs to happen after any possible calls to LOCK() that might
+ * skip locking if process appears single-threaded. */
+ if (!--libc.threads_minus_1) libc.need_locks = -1;
+ self->next->prev = self->prev;
+ self->prev->next = self->next;
+ self->prev = self->next = self;
+
/* This atomic potentially competes with a concurrent pthread_detach
* call; the loser is responsible for freeing thread resources. */
int state = a_cas(&self->detach_state, DT_JOINABLE, DT_EXITING);
@@ -310,7 +314,7 @@ int __pthread_create(pthread_t *restrict res, const pthread_attr_t *restrict att
new->detach_state = DT_JOINABLE;
}
new->robust_list.head = &new->robust_list.head;
- new->CANARY = self->CANARY;
+ new->canary = self->canary;
new->sysinfo = self->sysinfo;
/* Setup argument structure for the new thread on its stack.
@@ -336,7 +340,7 @@ int __pthread_create(pthread_t *restrict res, const pthread_attr_t *restrict att
~(1UL<<((SIGCANCEL-1)%(8*sizeof(long))));
__tl_lock();
- libc.threads_minus_1++;
+ if (!libc.threads_minus_1++) libc.need_locks = 1;
ret = __clone((c11 ? start_c11 : start), stack, flags, args, &new->tid, TP_ADJ(new), &__thread_list_lock);
/* All clone failures translate to EAGAIN. If explicit scheduling
@@ -360,7 +364,7 @@ int __pthread_create(pthread_t *restrict res, const pthread_attr_t *restrict att
new->next->prev = new;
new->prev->next = new;
} else {
- libc.threads_minus_1--;
+ if (!--libc.threads_minus_1) libc.need_locks = 0;
}
__tl_unlock();
__restore_sigs(&set);
diff --git a/src/thread/pthread_getschedparam.c b/src/thread/pthread_getschedparam.c
index 1cba073d..c098befb 100644
--- a/src/thread/pthread_getschedparam.c
+++ b/src/thread/pthread_getschedparam.c
@@ -4,6 +4,8 @@
int pthread_getschedparam(pthread_t t, int *restrict policy, struct sched_param *restrict param)
{
int r;
+ sigset_t set;
+ __block_app_sigs(&set);
LOCK(t->killlock);
if (!t->tid) {
r = ESRCH;
@@ -14,5 +16,6 @@ int pthread_getschedparam(pthread_t t, int *restrict policy, struct sched_param
}
}
UNLOCK(t->killlock);
+ __restore_sigs(&set);
return r;
}
diff --git a/src/thread/pthread_kill.c b/src/thread/pthread_kill.c
index 3d9395cb..79ddb209 100644
--- a/src/thread/pthread_kill.c
+++ b/src/thread/pthread_kill.c
@@ -4,9 +4,15 @@
int pthread_kill(pthread_t t, int sig)
{
int r;
+ sigset_t set;
+ /* Block not just app signals, but internal ones too, since
+ * pthread_kill is used to implement pthread_cancel, which
+ * must be async-cancel-safe. */
+ __block_all_sigs(&set);
LOCK(t->killlock);
r = t->tid ? -__syscall(SYS_tkill, t->tid, sig)
: (sig+0U >= _NSIG ? EINVAL : 0);
UNLOCK(t->killlock);
+ __restore_sigs(&set);
return r;
}
diff --git a/src/thread/pthread_setschedparam.c b/src/thread/pthread_setschedparam.c
index 038d13d8..76d4d45a 100644
--- a/src/thread/pthread_setschedparam.c
+++ b/src/thread/pthread_setschedparam.c
@@ -4,8 +4,11 @@
int pthread_setschedparam(pthread_t t, int policy, const struct sched_param *param)
{
int r;
+ sigset_t set;
+ __block_app_sigs(&set);
LOCK(t->killlock);
r = !t->tid ? ESRCH : -__syscall(SYS_sched_setscheduler, t->tid, policy, param);
UNLOCK(t->killlock);
+ __restore_sigs(&set);
return r;
}
diff --git a/src/thread/pthread_setschedprio.c b/src/thread/pthread_setschedprio.c
index 5bf4a019..fc2e13dd 100644
--- a/src/thread/pthread_setschedprio.c
+++ b/src/thread/pthread_setschedprio.c
@@ -4,8 +4,11 @@
int pthread_setschedprio(pthread_t t, int prio)
{
int r;
+ sigset_t set;
+ __block_app_sigs(&set);
LOCK(t->killlock);
r = !t->tid ? ESRCH : -__syscall(SYS_sched_setparam, t->tid, &prio);
UNLOCK(t->killlock);
+ __restore_sigs(&set);
return r;
}
diff --git a/src/thread/synccall.c b/src/thread/synccall.c
index 648a6ad4..d58c851f 100644
--- a/src/thread/synccall.c
+++ b/src/thread/synccall.c
@@ -63,7 +63,8 @@ void __synccall(void (*func)(void *), void *ctx)
sem_init(&target_sem, 0, 0);
sem_init(&caller_sem, 0, 0);
- if (!libc.threads_minus_1) goto single_threaded;
+ if (!libc.threads_minus_1 || __syscall(SYS_gettid) != self->tid)
+ goto single_threaded;
callback = func;
context = ctx;
diff --git a/src/time/__map_file.c b/src/time/__map_file.c
index 9d376222..d3cefa82 100644
--- a/src/time/__map_file.c
+++ b/src/time/__map_file.c
@@ -2,10 +2,11 @@
#include <fcntl.h>
#include <sys/stat.h>
#include "syscall.h"
+#include "kstat.h"
const char unsigned *__map_file(const char *pathname, size_t *size)
{
- struct stat st;
+ struct kstat st;
const unsigned char *map = MAP_FAILED;
int fd = sys_open(pathname, O_RDONLY|O_CLOEXEC|O_NONBLOCK);
if (fd < 0) return 0;
diff --git a/src/time/__tz.c b/src/time/__tz.c
index 185642e8..49a7371e 100644
--- a/src/time/__tz.c
+++ b/src/time/__tz.c
@@ -86,15 +86,15 @@ static void getname(char *d, const char **p)
int i;
if (**p == '<') {
++*p;
- for (i=0; (*p)[i]!='>' && i<TZNAME_MAX; i++)
- d[i] = (*p)[i];
- ++*p;
+ for (i=0; (*p)[i] && (*p)[i]!='>'; i++)
+ if (i<TZNAME_MAX) d[i] = (*p)[i];
+ if ((*p)[i]) ++*p;
} else {
- for (i=0; ((*p)[i]|32)-'a'<26U && i<TZNAME_MAX; i++)
- d[i] = (*p)[i];
+ for (i=0; ((*p)[i]|32)-'a'<26U; i++)
+ if (i<TZNAME_MAX) d[i] = (*p)[i];
}
*p += i;
- d[i] = 0;
+ d[i<TZNAME_MAX?i:TZNAME_MAX] = 0;
}
#define VEC(...) ((const unsigned char[]){__VA_ARGS__})
diff --git a/src/unistd/faccessat.c b/src/unistd/faccessat.c
index 76bbd4c7..557503eb 100644
--- a/src/unistd/faccessat.c
+++ b/src/unistd/faccessat.c
@@ -25,12 +25,17 @@ static int checker(void *p)
int faccessat(int fd, const char *filename, int amode, int flag)
{
- if (!flag || (flag==AT_EACCESS && getuid()==geteuid() && getgid()==getegid()))
- return syscall(SYS_faccessat, fd, filename, amode, flag);
+ if (flag) {
+ int ret = __syscall(SYS_faccessat2, fd, filename, amode, flag);
+ if (ret != -ENOSYS) return __syscall_ret(ret);
+ }
- if (flag != AT_EACCESS)
+ if (flag & ~AT_EACCESS)
return __syscall_ret(-EINVAL);
+ if (!flag || (getuid()==geteuid() && getgid()==getegid()))
+ return syscall(SYS_faccessat, fd, filename, amode);
+
char stack[1024];
sigset_t set;
pid_t pid;
diff --git a/tools/add-cfi.i386.awk b/tools/add-cfi.i386.awk
index 9162e309..d05037de 100644
--- a/tools/add-cfi.i386.awk
+++ b/tools/add-cfi.i386.awk
@@ -81,7 +81,7 @@ function adjust_sp_offset(delta) {
in_function = 0
}
}
-/^\.type [a-zA-Z0-9_]+,\@function/ {
+/^\.type [a-zA-Z0-9_]+,@function/ {
functions[substr($2, 1, length($2)-10)] = 1
}
# not interested in assembler directives beyond this, just pass them through
diff --git a/tools/add-cfi.x86_64.awk b/tools/add-cfi.x86_64.awk
index bbc90daa..7e1513d6 100644
--- a/tools/add-cfi.x86_64.awk
+++ b/tools/add-cfi.x86_64.awk
@@ -76,7 +76,7 @@ function adjust_sp_offset(delta) {
in_function = 0
}
}
-/^\.type [a-zA-Z0-9_]+,\@function/ {
+/^\.type [a-zA-Z0-9_]+,@function/ {
functions[substr($2, 1, length($2)-10)] = 1
}
# not interested in assembler directives beyond this, just pass them through