diff options
Diffstat (limited to 'src')
474 files changed, 6820 insertions, 2590 deletions
diff --git a/src/aio/aio.c b/src/aio/aio.c index 6d34fa86..d7e063bf 100644 --- a/src/aio/aio.c +++ b/src/aio/aio.c @@ -9,6 +9,12 @@ #include "syscall.h" #include "atomic.h" #include "pthread_impl.h" +#include "aio_impl.h" + +#define malloc __libc_malloc +#define calloc __libc_calloc +#define realloc __libc_realloc +#define free __libc_free /* The following is a threads-based implementation of AIO with minimal * dependence on implementation details. Most synchronization is @@ -70,8 +76,14 @@ static struct aio_queue *****map; static volatile int aio_fd_cnt; volatile int __aio_fut; +static size_t io_thread_stack_size; + +#define MAX(a,b) ((a)>(b) ? (a) : (b)) + static struct aio_queue *__aio_get_queue(int fd, int need) { + sigset_t allmask, origmask; + int masked = 0; if (fd < 0) { errno = EBADF; return 0; @@ -83,7 +95,14 @@ static struct aio_queue *__aio_get_queue(int fd, int need) if ((!map || !map[a] || !map[a][b] || !map[a][b][c] || !(q=map[a][b][c][d])) && need) { pthread_rwlock_unlock(&maplock); if (fcntl(fd, F_GETFD) < 0) return 0; + sigfillset(&allmask); + masked = 1; + pthread_sigmask(SIG_BLOCK, &allmask, &origmask); pthread_rwlock_wrlock(&maplock); + if (!io_thread_stack_size) { + unsigned long val = __getauxval(AT_MINSIGSTKSZ); + io_thread_stack_size = MAX(MINSIGSTKSZ+2048, val+512); + } if (!map) map = calloc(sizeof *map, (-1U/2+1)>>24); if (!map) goto out; if (!map[a]) map[a] = calloc(sizeof **map, 256); @@ -105,6 +124,7 @@ static struct aio_queue *__aio_get_queue(int fd, int need) if (q) pthread_mutex_lock(&q->lock); out: pthread_rwlock_unlock(&maplock); + if (masked) pthread_sigmask(SIG_SETMASK, &origmask, 0); return q; } @@ -259,15 +279,6 @@ static void *io_thread_func(void *ctx) return 0; } -static size_t io_thread_stack_size = MINSIGSTKSZ+2048; -static pthread_once_t init_stack_size_once; - -static void init_stack_size() -{ - unsigned long val = __getauxval(AT_MINSIGSTKSZ); - if (val > MINSIGSTKSZ) io_thread_stack_size = val + 512; -} - static int submit(struct aiocb *cb, int op) { int ret = 0; @@ -293,7 +304,6 @@ static int submit(struct aiocb *cb, int op) else pthread_attr_init(&a); } else { - pthread_once(&init_stack_size_once, init_stack_size); pthread_attr_init(&a); pthread_attr_setstacksize(&a, io_thread_stack_size); pthread_attr_setguardsize(&a, 0); @@ -392,9 +402,31 @@ int __aio_close(int fd) return fd; } -weak_alias(aio_cancel, aio_cancel64); -weak_alias(aio_error, aio_error64); -weak_alias(aio_fsync, aio_fsync64); -weak_alias(aio_read, aio_read64); -weak_alias(aio_write, aio_write64); -weak_alias(aio_return, aio_return64); +void __aio_atfork(int who) +{ + if (who<0) { + pthread_rwlock_rdlock(&maplock); + return; + } else if (!who) { + pthread_rwlock_unlock(&maplock); + return; + } + aio_fd_cnt = 0; + if (pthread_rwlock_tryrdlock(&maplock)) { + /* Obtaining lock may fail if _Fork was called nor via + * fork. In this case, no further aio is possible from + * child and we can just null out map so __aio_close + * does not attempt to do anything. */ + map = 0; + return; + } + if (map) for (int a=0; a<(-1U/2+1)>>24; a++) + if (map[a]) for (int b=0; b<256; b++) + if (map[a][b]) for (int c=0; c<256; c++) + if (map[a][b][c]) for (int d=0; d<256; d++) + map[a][b][c][d] = 0; + /* Re-initialize the rwlock rather than unlocking since there + * may have been more than one reference on it in the parent. + * We are not a lock holder anyway; the thread in the parent was. */ + pthread_rwlock_init(&maplock, 0); +} diff --git a/src/aio/aio_suspend.c b/src/aio/aio_suspend.c index 9b24b6af..1f0c9aaa 100644 --- a/src/aio/aio_suspend.c +++ b/src/aio/aio_suspend.c @@ -3,12 +3,13 @@ #include <time.h> #include "atomic.h" #include "pthread_impl.h" +#include "aio_impl.h" int aio_suspend(const struct aiocb *const cbs[], int cnt, const struct timespec *ts) { int i, tid = 0, ret, expect = 0; struct timespec at; - volatile int dummy_fut, *pfut; + volatile int dummy_fut = 0, *pfut; int nzcnt = 0; const struct aiocb *cb = 0; @@ -72,5 +73,3 @@ int aio_suspend(const struct aiocb *const cbs[], int cnt, const struct timespec } } } - -weak_alias(aio_suspend, aio_suspend64); diff --git a/src/aio/lio_listio.c b/src/aio/lio_listio.c index 0799c15d..a672812f 100644 --- a/src/aio/lio_listio.c +++ b/src/aio/lio_listio.c @@ -139,5 +139,3 @@ int lio_listio(int mode, struct aiocb *restrict const *restrict cbs, int cnt, st return 0; } - -weak_alias(lio_listio, lio_listio64); diff --git a/src/complex/cacosf.c b/src/complex/cacosf.c index 2e048540..ed8acf0f 100644 --- a/src/complex/cacosf.c +++ b/src/complex/cacosf.c @@ -2,8 +2,10 @@ // FIXME +static const float float_pi_2 = M_PI_2; + float complex cacosf(float complex z) { z = casinf(z); - return CMPLXF((float)M_PI_2 - crealf(z), -cimagf(z)); + return CMPLXF(float_pi_2 - crealf(z), -cimagf(z)); } diff --git a/src/complex/cacosh.c b/src/complex/cacosh.c index 76127f75..55b857ce 100644 --- a/src/complex/cacosh.c +++ b/src/complex/cacosh.c @@ -1,6 +1,6 @@ #include "complex_impl.h" -/* acosh(z) = i acos(z) */ +/* acosh(z) = ±i acos(z) */ double complex cacosh(double complex z) { diff --git a/src/complex/catanf.c b/src/complex/catanf.c index e10d9c09..1d569f2d 100644 --- a/src/complex/catanf.c +++ b/src/complex/catanf.c @@ -61,13 +61,15 @@ static const double DP1 = 3.140625; static const double DP2 = 9.67502593994140625E-4; static const double DP3 = 1.509957990978376432E-7; +static const float float_pi = M_PI; + static float _redupif(float xx) { float x, t; long i; x = xx; - t = x/(float)M_PI; + t = x/float_pi; if (t >= 0.0f) t += 0.5f; else @@ -87,29 +89,17 @@ float complex catanf(float complex z) x = crealf(z); y = cimagf(z); - if ((x == 0.0f) && (y > 1.0f)) - goto ovrf; - x2 = x * x; a = 1.0f - x2 - (y * y); - if (a == 0.0f) - goto ovrf; t = 0.5f * atan2f(2.0f * x, a); w = _redupif(t); t = y - 1.0f; a = x2 + (t * t); - if (a == 0.0f) - goto ovrf; t = y + 1.0f; a = (x2 + (t * t))/a; - w = w + (0.25f * logf (a)) * I; - return w; - -ovrf: - // FIXME - w = MAXNUMF + MAXNUMF * I; + w = CMPLXF(w, 0.25f * logf(a)); return w; } diff --git a/src/complex/catanl.c b/src/complex/catanl.c index a9fc02db..e62526c0 100644 --- a/src/complex/catanl.c +++ b/src/complex/catanl.c @@ -97,30 +97,18 @@ long double complex catanl(long double complex z) x = creall(z); y = cimagl(z); - if ((x == 0.0L) && (y > 1.0L)) - goto ovrf; - x2 = x * x; a = 1.0L - x2 - (y * y); - if (a == 0.0L) - goto ovrf; t = atan2l(2.0L * x, a) * 0.5L; w = redupil(t); t = y - 1.0L; a = x2 + (t * t); - if (a == 0.0L) - goto ovrf; t = y + 1.0L; a = (x2 + (t * t)) / a; - w = w + (0.25L * logl(a)) * I; - return w; - -ovrf: - // FIXME - w = LDBL_MAX + LDBL_MAX * I; + w = CMPLXF(w, 0.25L * logl(a)); return w; } #endif diff --git a/src/complex/cproj.c b/src/complex/cproj.c index 9ae1e17c..d2b8f5a9 100644 --- a/src/complex/cproj.c +++ b/src/complex/cproj.c @@ -3,6 +3,6 @@ double complex cproj(double complex z) { if (isinf(creal(z)) || isinf(cimag(z))) - return CMPLX(INFINITY, copysign(0.0, creal(z))); + return CMPLX(INFINITY, copysign(0.0, cimag(z))); return z; } diff --git a/src/complex/cprojf.c b/src/complex/cprojf.c index 03fab339..15a874bb 100644 --- a/src/complex/cprojf.c +++ b/src/complex/cprojf.c @@ -3,6 +3,6 @@ float complex cprojf(float complex z) { if (isinf(crealf(z)) || isinf(cimagf(z))) - return CMPLXF(INFINITY, copysignf(0.0, crealf(z))); + return CMPLXF(INFINITY, copysignf(0.0, cimagf(z))); return z; } diff --git a/src/complex/cprojl.c b/src/complex/cprojl.c index 38a494c5..531ffa1c 100644 --- a/src/complex/cprojl.c +++ b/src/complex/cprojl.c @@ -9,7 +9,7 @@ long double complex cprojl(long double complex z) long double complex cprojl(long double complex z) { if (isinf(creall(z)) || isinf(cimagl(z))) - return CMPLXL(INFINITY, copysignl(0.0, creall(z))); + return CMPLXL(INFINITY, copysignl(0.0, cimagl(z))); return z; } #endif diff --git a/src/conf/confstr.c b/src/conf/confstr.c index 02cb1aa2..3d417284 100644 --- a/src/conf/confstr.c +++ b/src/conf/confstr.c @@ -7,7 +7,7 @@ size_t confstr(int name, char *buf, size_t len) const char *s = ""; if (!name) { s = "/bin:/usr/bin"; - } else if ((name&~4U)!=1 && name-_CS_POSIX_V6_ILP32_OFF32_CFLAGS>33U) { + } else if ((name&~4U)!=1 && name-_CS_POSIX_V6_ILP32_OFF32_CFLAGS>35U) { errno = EINVAL; return 0; } diff --git a/src/conf/sysconf.c b/src/conf/sysconf.c index 3baaed32..60d3e745 100644 --- a/src/conf/sysconf.c +++ b/src/conf/sysconf.c @@ -4,6 +4,7 @@ #include <sys/resource.h> #include <signal.h> #include <sys/sysinfo.h> +#include <sys/auxv.h> #include "syscall.h" #include "libc.h" @@ -19,6 +20,8 @@ #define JT_AVPHYS_PAGES JT(9) #define JT_ZERO JT(10) #define JT_DELAYTIMER_MAX JT(11) +#define JT_MINSIGSTKSZ JT(12) +#define JT_SIGSTKSZ JT(13) #define RLIM(x) (-32768|(RLIMIT_ ## x)) @@ -165,6 +168,9 @@ long sysconf(int name) [_SC_XOPEN_STREAMS] = JT_ZERO, [_SC_THREAD_ROBUST_PRIO_INHERIT] = -1, [_SC_THREAD_ROBUST_PRIO_PROTECT] = -1, + + [_SC_MINSIGSTKSZ] = JT_MINSIGSTKSZ, + [_SC_SIGSTKSZ] = JT_SIGSTKSZ, }; if (name >= sizeof(values)/sizeof(values[0]) || !values[name]) { @@ -212,6 +218,13 @@ long sysconf(int name) mem *= si.mem_unit; mem /= PAGE_SIZE; return (mem > LONG_MAX) ? LONG_MAX : mem; + case JT_MINSIGSTKSZ & 255: + case JT_SIGSTKSZ & 255: ; + long val = __getauxval(AT_MINSIGSTKSZ); + if (val < MINSIGSTKSZ) val = MINSIGSTKSZ; + if (values[name] == JT_SIGSTKSZ) + val += SIGSTKSZ - MINSIGSTKSZ; + return val; case JT_ZERO & 255: return 0; } diff --git a/src/crypt/crypt_blowfish.c b/src/crypt/crypt_blowfish.c index d3f79851..d722607b 100644 --- a/src/crypt/crypt_blowfish.c +++ b/src/crypt/crypt_blowfish.c @@ -15,7 +15,7 @@ * No copyright is claimed, and the software is hereby placed in the public * domain. In case this attempt to disclaim copyright and place the software * in the public domain is deemed null and void, then the software is - * Copyright (c) 1998-2012 Solar Designer and it is hereby released to the + * Copyright (c) 1998-2014 Solar Designer and it is hereby released to the * general public under the following terms: * * Redistribution and use in source and binary forms, with or without @@ -31,12 +31,12 @@ * you place this code and any modifications you make under a license * of your choice. * - * This implementation is mostly compatible with OpenBSD's bcrypt.c (prefix - * "$2a$") by Niels Provos <provos at citi.umich.edu>, and uses some of his - * ideas. The password hashing algorithm was designed by David Mazieres - * <dm at lcs.mit.edu>. For more information on the level of compatibility, - * please refer to the comments in BF_set_key() below and to the included - * crypt(3) man page. + * This implementation is fully compatible with OpenBSD's bcrypt.c for prefix + * "$2b$", originally by Niels Provos <provos at citi.umich.edu>, and it uses + * some of his ideas. The password hashing algorithm was designed by David + * Mazieres <dm at lcs.mit.edu>. For information on the level of + * compatibility for bcrypt hash prefixes other than "$2b$", please refer to + * the comments in BF_set_key() below and to the included crypt(3) man page. * * There's a paper on the algorithm that explains its design decisions: * @@ -533,6 +533,7 @@ static void BF_set_key(const char *key, BF_key expanded, BF_key initial, * Valid combinations of settings are: * * Prefix "$2a$": bug = 0, safety = 0x10000 + * Prefix "$2b$": bug = 0, safety = 0 * Prefix "$2x$": bug = 1, safety = 0 * Prefix "$2y$": bug = 0, safety = 0 */ @@ -596,12 +597,14 @@ static void BF_set_key(const char *key, BF_key expanded, BF_key initial, initial[0] ^= sign; } +static const unsigned char flags_by_subtype[26] = { + 2, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4, 0 +}; + static char *BF_crypt(const char *key, const char *setting, char *output, BF_word min) { - static const unsigned char flags_by_subtype[26] = - {2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4, 0}; struct { BF_ctx ctx; BF_key expanded_key; @@ -746,9 +749,11 @@ char *__crypt_blowfish(const char *key, const char *setting, char *output) { const char *test_key = "8b \xd0\xc1\xd2\xcf\xcc\xd8"; const char *test_setting = "$2a$00$abcdefghijklmnopqrstuu"; - static const char test_hash[2][34] = - {"VUrPmXD6q/nVSSp7pNDhCR9071IfIRe\0\x55", /* $2x$ */ - "i1D709vfamulimlGcq0qq3UvuUasvEa\0\x55"}; /* $2a$, $2y$ */ + static const char test_hashes[2][34] = { + "i1D709vfamulimlGcq0qq3UvuUasvEa\0\x55", /* 'a', 'b', 'y' */ + "VUrPmXD6q/nVSSp7pNDhCR9071IfIRe\0\x55", /* 'x' */ + }; + const char *test_hash = test_hashes[0]; char *retval; const char *p; int ok; @@ -768,8 +773,11 @@ char *__crypt_blowfish(const char *key, const char *setting, char *output) * detected by the self-test. */ memcpy(buf.s, test_setting, sizeof(buf.s)); - if (retval) + if (retval) { + unsigned int flags = flags_by_subtype[setting[2] - 'a']; + test_hash = test_hashes[flags & 1]; buf.s[2] = setting[2]; + } memset(buf.o, 0x55, sizeof(buf.o)); buf.o[sizeof(buf.o) - 1] = 0; p = BF_crypt(test_key, buf.s, buf.o, 1); @@ -777,7 +785,7 @@ char *__crypt_blowfish(const char *key, const char *setting, char *output) ok = (p == buf.o && !memcmp(p, buf.s, 7 + 22) && !memcmp(p + (7 + 22), - test_hash[buf.s[2] & 1], + test_hash, 31 + 1 + 1 + 1)); { diff --git a/src/ctype/alpha.h b/src/ctype/alpha.h index 299277c7..4167f387 100644 --- a/src/ctype/alpha.h +++ b/src/ctype/alpha.h @@ -8,17 +8,17 @@ 17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17, 17,17,17,17,17,17,17,63,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16, 16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,17,64,65,17,66,67, -68,69,70,71,72,73,74,17,75,76,77,78,79,80,16,16,16,81,82,83,84,85,86,87,88,89, -16,90,16,91,92,16,16,17,17,17,93,94,95,16,16,16,16,16,16,16,16,16,16,17,17,17, -17,96,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,17,17,97,16,16,16,16,16,16, +68,69,70,71,72,73,74,17,75,76,77,78,79,80,81,16,82,83,84,85,86,87,88,89,90,91, +92,93,16,94,95,96,16,17,17,17,97,98,99,16,16,16,16,16,16,16,16,16,16,17,17,17, +17,100,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,17,17,101,16,16,16,16,16, 16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16, -16,17,17,98,99,16,16,16,100,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17, -17,17,17,17,17,17,17,101,17,17,102,16,16,16,16,16,16,16,16,16,16,16,16,16,16, -16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,17,103, -104,16,16,16,16,16,16,16,16,16,105,16,16,16,16,16,16,16,16,16,16,16,16,16,16, -16,16,16,16,16,16,16,16,16,106,107,108,109,16,16,16,16,16,16,16,16,110,16,16, -16,16,16,16,16,111,112,16,16,16,16,113,16,16,114,16,16,16,16,16,16,16,16,16, -16,16,16,16, +16,16,17,17,102,103,16,16,104,105,17,17,17,17,17,17,17,17,17,17,17,17,17,17, +17,17,17,17,17,17,17,17,17,106,17,17,107,16,16,16,16,16,16,16,16,16,16,16,16, +16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,17, +108,109,16,16,16,16,16,16,16,16,16,110,16,16,16,16,16,16,16,16,16,16,16,16,16, +16,16,16,16,16,16,16,16,16,16,111,112,113,114,16,16,16,16,16,16,16,16,115,116, +117,16,16,16,16,16,118,119,16,16,16,16,120,16,16,121,16,16,16,16,16,16,16,16, +16,16,16,16,16, 16,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,255,255, 255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255, 255,255,255,255,255,255,255,255,255,255,255,0,0,0,0,0,0,0,0,254,255,255,7,254, @@ -27,8 +27,8 @@ 255,195,255,3,0,31,80,0,0,0,0,0,0,0,0,0,0,32,0,0,0,0,0,223,188,64,215,255,255, 251,255,255,255,255,255,255,255,255,255,191,255,255,255,255,255,255,255,255, 255,255,255,255,255,255,255,255,255,3,252,255,255,255,255,255,255,255,255,255, -255,255,255,255,255,255,255,255,255,255,255,254,255,255,255,127,2,254,255,255, -255,255,0,0,0,0,0,255,191,182,0,255,255,255,7,7,0,0,0,255,7,255,255,255,255, +255,255,255,255,255,255,255,255,255,255,255,254,255,255,255,127,2,255,255,255, +255,255,1,0,0,0,0,255,191,182,0,255,255,255,135,7,0,0,0,255,7,255,255,255,255, 255,255,255,254,255,195,255,255,255,255,255,255,255,255,255,255,255,255,239, 31,254,225,255, 159,0,0,255,255,255,255,255,255,0,224,255,255,255,255,255,255,255,255,255,255, @@ -42,54 +42,55 @@ 255,0,0,239,223,253,255,255,253,239,227,223,29,96,64,207,255,6,0,239,223,253, 255,255,255,255,231,223,93,240,128,207,255,0,252,236,255,127,252,255,255,251, 47,127,128,95,255,192,255,12,0,254,255,255,255,255,127,255,7,63,32,255,3,0,0, -0,0,150,37,240,254,174,236,255,59,95,32,255,243,0,0,0, +0,0,214,247,255,255,175,255,255,59,95,32,255,243,0,0,0, 0,1,0,0,0,255,3,0,0,255,254,255,255,255,31,254,255,3,255,255,254,255,255,255, -31,0,0,0,0,0,0,0,0,255,255,255,255,255,255,127,249,255,3,255,255,231,193,255, -255,127,64,255,51,255,255,255,255,191,32,255,255,255,255,255,247,255,255,255, +31,0,0,0,0,0,0,0,0,255,255,255,255,255,255,127,249,255,3,255,255,255,255,255, +255,255,255,255,63,255,255,255,255,191,32,255,255,255,255,255,247,255,255,255, 255,255,255,255,255,255,61,127,61,255,255,255,255,255,61,255,255,255,255,61, 127,61,255,127,255,255,255,255,255,255,255,61,255,255,255,255,255,255,255,255, -135,0,0,0,0,255,255,0,0,255,255,255,255,255,255,255,255,255,255,63,63,254,255, +7,0,0,0,0,255,255,0,0,255,255,255,255,255,255,255,255,255,255,63,63,254,255, 255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255, 255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255, 255,255,255,255,255,159,255,255,254,255,255,7,255,255,255,255,255,255,255,255, 255,199,255,1,255,223,15,0,255,255,15,0,255,255,15,0,255,223,13,0,255,255,255, 255,255,255,207,255,255,1,128,16,255,3,0,0,0,0,255,3,255,255,255,255,255,255, -255,255,255,255,255,0,255,255,255,255,255,7,255,255,255,255,255,255,255,255, +255,255,255,255,255,1,255,255,255,255,255,7,255,255,255,255,255,255,255,255, 63, 0,255,255,255,127,255,15,255,1,192,255,255,255,255,63,31,0,255,255,255,255, 255,15,255,255,255,3,255,3,0,0,0,0,255,255,255,15,255,255,255,255,255,255,255, 127,254,255,31,0,255,3,255,3,128,0,0,0,0,0,0,0,0,0,0,0,255,255,255,255,255, 255,239,255,239,15,255,3,0,0,0,0,255,255,255,255,255,243,255,255,255,255,255, -255,191,255,3,0,255,255,255,255,255,255,63,0,255,227,255,255,255,255,255,63, -255,1,0,0,0,0,0,0,0,0,0,0,0,222,111,0,255,255,255,255,255,255,255,255,255,255, -255,255,255,255,255,255,255,255,255,255,255,255,255,255,0,0,0,0,128,255,31,0, -255,255,63,63,255,255,255,255,63,63,255,170,255,255,255,63,255,255,255,255, -255,255,223,95,220,31,207,15,255,31,220,31,0,0,0,0,0,0,0,0,0,0,0,0,0,0,2,128, -0,0,255,31,0,0,0,0,0,0,0,0,0,0,0,0,132,252,47,62,80,189,255,243,224,67,0,0, -255,255,255,255,255,1,0,0,0,0,0,0,0,0,0,0,0,0,0, +255,191,255,3,0,255,255,255,255,255,255,127,0,255,227,255,255,255,255,255,63, +255,1,255,255,255,255,255,231,0,0,0,0,0,222,111,4,255,255,255,255,255,255,255, +255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,0,0,0,0, +128,255,31,0,255,255,63,63,255,255,255,255,63,63,255,170,255,255,255,63,255, +255,255,255,255,255,223,95,220,31,207,15,255,31,220,31,0,0,0,0,0,0,0,0,0,0,0, +0,0,0,2,128,0,0,255,31,0,0,0,0,0,0,0,0,0,0,0,0,132,252,47,62,80,189,255,243, +224,67,0,0,255,255,255,255,255,1,0,0,0,0,0,0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,192,255,255,255,255,255,255,3,0, 0,255,255,255,255,255,127,255,255,255,255,255,127,255,255,255,255,255,255,255, 255,255,255,255,255,255,255,255,255,31,120,12,0,255,255,255,255,191,32,255, 255,255,255,255,255,255,128,0,0,255,255,127,0,127,127,127,127,127,127,127,127, 255,255,255,255,0,0,0,0,0,128,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, 0,0,224,0,0,0,254,3,62,31,254,255,255,255,255,255,255,255,255,255,127,224,254, -255,255,255,255,255,255,255,255,255,255,247,224,255,255,255,255,127,254,255, +255,255,255,255,255,255,255,255,255,255,247,224,255,255,255,255,255,254,255, 255,255,255,255,255,255,255,255,255,127,0,0,255,255,255,7,0,0,0,0,0,0,255,255, 255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255, 255,255,255,63,0,0,0,0,0,0,0,0,0,255,255,255,255,255,255,255,255,255,255,255, -255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,7,0, +255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,0, 0,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,31,0,0, 0,0,0,0,0,0,255,255,255,255,255,63,255,31,255,255,255,15,0,0,255,255,255,255, 255,127,240,143,255,255,255,255,255,255,255,255,255,255,255,255,255,255,0,0,0, 0,128,255,252,255,255,255,255,255,255,255,255,255,255,255,255,249,255,255,255, -127,255,0,0,0,0,0,0,0,128,255,187,247,255,255,255,0,0,0,255,255,255,255,255, -255,15,0,255,255,255,255,255,255,255,255,47,0,255,3,0,0,252,40,255,255,255, -255,255,7,255,255,255,255,7,0,255,255,255,31,255,255,255,255,255,255,247,255, -0,128,255,3,223,255,255,127,255,255,255,255,255,255,127,0,255,63,255,3,255, -255,127,196,255,255,255,255,255,255,255,127,5,0,0,56,255,255,60,0,126,126,126, -0,127,127,255,255,255,255,255,247,63,0,255,255,255,255,255,255,255,255,255, -255,255,255,255,255,255,7,255,3,255,255,255,255,255,255,255,255,255,255,255, -255,255,255,255,255,255,255,255,255,15,0,255,255,127,248,255,255,255,255,255, +255,255,255,124,0,0,0,0,0,128,255,191,255,255,255,255,0,0,0,255,255,255,255, +255,255,15,0,255,255,255,255,255,255,255,255,47,0,255,3,0,0,252,232,255,255, +255,255,255,7,255,255,255,255,7,0,255,255,255,31,255,255,255,255,255,255,247, +255,0,128,255,3,255,255,255,127,255,255,255,255,255,255,127,0,255,63,255,3, +255,255,127,252,255,255,255,255,255,255,255,127,5,0,0,56,255,255,60,0,126,126, +126,0,127,127,255,255,255,255,255,247,255,0,255,255,255,255,255,255,255,255, +255,255,255,255,255,255,255,7,255,3,255,255,255,255,255,255,255,255,255,255, +255,255,255,255,255,255,255,255,255,255,15,0,255,255,127,248,255,255,255,255, +255, 15,255,255,255,255,255,255,255,255,255,255,255,255,255,63,255,255,255,255,255, 255,255,255,255,255,255,255,255,3,0,0,0,0,127,0,248,224,255,253,127,95,219, 255,255,255,255,255,255,255,255,255,255,255,255,255,3,0,0,0,248,255,255,255, @@ -109,55 +110,63 @@ 0,0,0,0,0,0,0,0,0,0,0,0,63,253,255,255,255,255,191,145,255,255,63,0,255,255, 127,0,255,255,255,127,0,0,0,0,0,0,0,0,255,255,55,0,255,255,63,0,255,255,255,3, 0,0,0,0,0,0,0,0,255,255,255,255,255,255,255,192,0,0,0,0,0,0,0,0,111,240,239, -254,255,255,15,0,0,0,0,0,255,255,255,31,255,255,255,31,0,0,0,0,255,254,255, +254,255,255,63,0,0,0,0,0,255,255,255,31,255,255,255,31,0,0,0,0,255,254,255, 255,31,0,0,0,255,255,255,255,255,255,63,0,255,255,63,0,255,255,7,0,255,255,3, 0,0,0,0,0,0,0,0,0,0,0,0, 0,255,255,255,255,255,255,255,255,255,1,0,0,0,0,0,0,255,255,255,255,255,255,7, -0,255,255,255,255,255,255,7,0,255,255,255,255,255,255,255,255,63,0,0,0,192, -255,0,0,252,255,255,255,255,255,255,1,0,0,255,255,255,1,255,3,255,255,255,255, -255,255,199,255,0,0,255,255,255,255,71,0,255,255,255,255,255,255,255,255,30,0, -255,23,0,0,0,0,255,255,251,255,255,255,159,64,0,0,0,0,0,0,0,0,127,189,255,191, -255,1,255,255,255,255,255,255,255,1,255,3,239,159,249,255,255,253,237,227,159, -25,129,224,15,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,255,255,255,255,255,255, -255,255,187,7,255,3,0,0,0,0,255,255,255,255,255,255,255,255,179,0,255,3,0,0,0, +0,255,255,255,255,255,255,7,0,255,255,255,255,255,0,255,3,0,0,0,0,0,0,0,0,0,0, +0,0,0,0,0,0,0,0,0,0,0,0,0,0,255,255,255,31,128,0,255,255,63,0,0,0,0,0,0,0,0,0, +0,0,0,0,0,0,0,0,0,0,255,255,127,0,255,255,255,255,255,255,255,255,63,0,0,0, +192,255,0,0,252,255,255,255,255,255,255,1,0,0,255,255,255,1,255,3,255,255,255, +255,255,255,199,255,112,0,255,255,255,255,71,0,255,255,255,255,255,255,255, +255,30,0,255,23,0,0,0,0,255,255,251,255,255,255,159,64,0,0,0,0,0,0,0,0,127, +189,255,191,255,1,255,255,255,255,255,255,255,1,255,3,239,159,249,255,255,253, +237,227,159,25,129,224,15,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,255,255,255, +255,255,255,255,255,187,7,255,131,0,0,0,0,255,255,255,255,255,255,255,255,179, +0,255,3,0,0,0, 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,255,255,255,255,255,255,63,127,0,0,0,63,0,0, 0,0,255,255,255,255,255,255,255,127,17,0,255,3,0,0,0,0,255,255,255,255,255, -255,63,0,255,3,0,0,0,0,0, -0,255,255,255,227,255,7,255,3,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, -0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,255,255,255,255,255,255,255,255,255,3, -0,128,255,255,255,255,255,255,231,127,0,0,255,255,255,255,255,255,207,255,255, -0,0,0,0,0,255,255,255,255,255,255,255,1,255,253,255,255,255,255,127,127,1,0, -255,3,0,0,252,255,255,255,252,255,255,254,127,0,0,0,0,0,0,0,0,0,127,251,255, -255,255,255,127,180,203,0,255,3,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,255, +255,63,1,255,3,0,0,0,0,0,0,255,255,255,231,255,7,255,3,0,0,0,0,0,0,0,0,0,0,0, +0,0,0,0,0,0,0,0,0,0,0,0,0,255,255,255,255,255,255,255,1,0,0,0,0,0,0,0,0,0,0,0, +0,255,255,255,255,255,255,255,255,255,3,0,128,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, +0,0,0,0,255,252,255,255,255,255,255,252,26,0,0,0,255,255,255,255,255,255,231, +127,0,0,255,255,255,255,255,255,255,255,255,32,0,0,0,0,255,255,255,255,255, +255,255,1,255,253,255,255,255,255,127,127,1,0,255,3,0,0,252,255,255,255,252, +255,255,254,127,0,0,0,0,0,0,0,0,0,127,251,255,255,255,255,127,180,203,0,255,3, +191,253,255,255,255,127,123,1,255,3,0,0,0,0,0,0,0,0,0, +0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,255,255,127,0,255, 255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,3,0,0, 0,0,0,0,0,0,0,0,0,0,255,255,255,255,255,255,255,255,255,255,255,255,255,127,0, 0,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255, -255,255,255,255,255,15,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, -0,255,255,255,255,255,127,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, -255,255,255,255,255,255,255,255,127,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, -0,0,255,255,255,255,255,255,255,1,255,255,255,127,255,3,0,0,0,0,0,0,0,0,0,0,0, -0,255,255,255,63,0,0,255,255,255,255,255,255,127,0,15,0,255,3,248,255,255,224, -255,255,0,0,0,0,0,0,0,0,0,0,0,0,0,0,255,255,255,255,255,255,255,255,31,0,255, -255,255,255,255,127,0,0,248,255,0,0,0,0,0,0,0,0,3,0,0,0,255,255,255,255,255, +255,255,255,255,255,15,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,255,255, +255,255,255,127,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,255,255, +255,255,255,255,255,255,127,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,255, +255,255,255,255,255,255,1,255,255,255,127,255,3,0,0,0,0,0,0,0,0,0,0,0,0,255, +255,255,63,0,0,255,255,255,255,255,255,0,0,15,0,255,3,248,255,255,224,255,255, +0,0,0,0,0,0,0,0,0,0,0,0,0, +0,0,0,0,0,0,0,0,0,255,255,255,255,255,255,255,255,0,0,0,0,0,0,0,0,0,0,0,0,0,0, +0,0,255,255,255,255,255,255,255,255,255,135,255,255,255,255,255,255,255,128, +255,255,0,0,0,0,0,0,0,0,11,0,0,0,255,255,255,255,255,255,255,255,255,255,255, 255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255, -255,255,255,255,255,31,0,0,255,255,255,255,255,255,255,255,255,255,255,255, -255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,7,0, -255,255,255,127,0,0,0,0,0,0,0,0,0,0,255,255,255,255,255,255,255,255,255,255, -255,255,255,255,255,255,255, +255,0,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255, +255,255,255,255,255,255,255,255,255,255,255,255,7,0,255,255,255,127,0,0,0,0,0, +0,7,0,240,0,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255, 255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255, -255,255,255,255,255,255,255,255,255,255,255,255,255,15,255,255,255,255,255, -255,255,255,255,255,255,255,255,7,255,31,255,1,255,67,0,0,0,0,0,0,0,0,0,0,0,0, -255,255,255,255,255,255,255,255,255,255,223,255,255,255,255,255,255,255,255, -223,100,222,255,235,239,255,255,255,255,255,255,255,191,231,223,223,255,255, -255,123,95,252,253,255,255,255,255,255,255,255,255,255,255,255,255,255,255, +255,255,255,255,255,255,255,255,255,255,255,255,255,255,15,255,255,255,255, +255,255,255,255,255,255,255,255,255,7,255,31,255,1,255,67,0,0,0,0,0,0,0,0,0,0, +0,0,255,255,255,255,255,255,255,255,255,255,223,255,255,255,255,255,255,255, +255,223,100,222,255,235,239,255,255,255,255,255,255, +255,191,231,223,223,255,255,255,123,95,252,253,255,255,255,255,255,255,255, 255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255, -255,255,255,255,255,255,255,255,63,255,255,255,253,255,255,247,255,255,255, -247,255,255,223,255,255,255,223,255,255,127,255,255,255,127,255,255,255,253, -255,255,255,253,255,255,247,207,255,255,255,255,255,255,127,255,255,249,219,7, -0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,255,255,255,255,255,255, -255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,31,0, -0,0,0,0,0, -0,255,255,255,255,255,255,255,255,143,0,255,3,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, -0,0,0,0,239,255,255,255,150,254,247,10,132,234,150,170,150,247,247,94,255,251, -255,15,238,251,255,15,0,0,0,0,0,0,0,0,0,0,0,0,0,0,255,255,255,3,255,255,255,3, -255,255,255,3,0,0,0,0,0,0,0,0,0,0,0,0,0,0, +255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,63,255,255,255, +253,255,255,247,255,255,255,247,255,255,223,255,255,255,223,255,255,127,255, +255,255,127,255,255,255,253,255,255,255,253,255,255,247,207,255,255,255,255, +255,255,127,255,255,249,219,7,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, +0,0,255,255,255,255,255,31,128,63,255,67,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, +0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,255,255,255,255,255, +15,255,3,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255, +255,255,255,255,255,255,255,31,0,0,0,0,0,0,0,255,255,255,255,255,255,255,255, +143,8,255,3,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, +0,239,255,255,255,150,254,247,10,132,234,150,170,150,247,247,94,255,251,255, +15,238,251,255,15,0,0,0,0,0,0,0,0,0,0,0,0,0,0,255,255,255,3,255,255,255,3,255, +255,255,3,0,0,0,0,0,0,0,0,0,0,0,0,0,0, diff --git a/src/ctype/casemap.h b/src/ctype/casemap.h new file mode 100644 index 00000000..6ee1209b --- /dev/null +++ b/src/ctype/casemap.h @@ -0,0 +1,297 @@ +static const unsigned char tab[] = { + 7, 8, 9, 10, 11, 12, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, + 13, 6, 6, 14, 6, 6, 6, 6, 6, 6, 6, 6, 15, 16, 17, 18, + 6, 19, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 20, 21, 6, 6, + 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, + 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, + 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, + 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, + 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, + 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, + 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, + 6, 6, 6, 6, 6, 6, 22, 23, 6, 6, 6, 24, 6, 6, 6, 6, + 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, + 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, + 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, + 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, + 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 25, + 6, 6, 6, 6, 26, 6, 6, 6, 6, 6, 6, 6, 27, 6, 6, 6, + 6, 6, 6, 6, 6, 6, 6, 6, 28, 6, 6, 6, 6, 6, 6, 6, + 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, + 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, + 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, + 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, + 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 29, 6, + 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, + 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, + 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, + 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, + 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, + 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, + 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, + 6, 6, 6, 6, 6, 6, 6, 6, 6, 30, 6, 6, 6, 6, 6, 6, + 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 36, + 43, 43, 43, 43, 43, 43, 43, 43, 1, 0, 84, 86, 86, 86, 86, 86, + 86, 86, 86, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 24, 0, 0, 0, 43, 43, 43, 43, 43, 43, + 43, 7, 43, 43, 91, 86, 86, 86, 86, 86, 86, 86, 74, 86, 86, 5, + 49, 80, 49, 80, 49, 80, 49, 80, 49, 80, 49, 80, 49, 80, 49, 80, + 36, 80, 121, 49, 80, 49, 80, 49, 56, 80, 49, 80, 49, 80, 49, 80, + 49, 80, 49, 80, 49, 80, 49, 80, 78, 49, 2, 78, 13, 13, 78, 3, + 78, 0, 36, 110, 0, 78, 49, 38, 110, 81, 78, 36, 80, 78, 57, 20, + 129, 27, 29, 29, 83, 49, 80, 49, 80, 13, 49, 80, 49, 80, 49, 80, + 27, 83, 36, 80, 49, 2, 92, 123, 92, 123, 92, 123, 92, 123, 92, 123, + 20, 121, 92, 123, 92, 123, 92, 45, 43, 73, 3, 72, 3, 120, 92, 123, + 20, 0, 150, 10, 1, 43, 40, 6, 6, 0, 42, 6, 42, 42, 43, 7, + 187, 181, 43, 30, 0, 43, 7, 43, 43, 43, 1, 43, 43, 43, 43, 43, + 43, 43, 43, 43, 43, 43, 43, 43, 43, 43, 43, 43, 43, 43, 43, 43, + 43, 43, 43, 43, 43, 43, 43, 43, 43, 43, 43, 1, 43, 43, 43, 43, + 43, 43, 43, 43, 43, 43, 43, 43, 43, 43, 43, 43, 43, 43, 43, 43, + 43, 43, 43, 42, 43, 43, 43, 43, 43, 43, 43, 43, 43, 43, 43, 43, + 43, 205, 70, 205, 43, 0, 37, 43, 7, 1, 6, 1, 85, 86, 86, 86, + 86, 86, 85, 86, 86, 2, 36, 129, 129, 129, 129, 129, 21, 129, 129, 129, + 0, 0, 43, 0, 178, 209, 178, 209, 178, 209, 178, 209, 0, 0, 205, 204, + 1, 0, 215, 215, 215, 215, 215, 131, 129, 129, 129, 129, 129, 129, 129, 129, + 129, 129, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 28, 0, 0, 0, + 0, 0, 49, 80, 49, 80, 49, 80, 49, 80, 49, 80, 49, 2, 0, 0, + 49, 80, 49, 80, 49, 80, 49, 80, 49, 80, 49, 80, 49, 80, 49, 80, + 49, 80, 78, 49, 80, 49, 80, 78, 49, 80, 49, 80, 49, 80, 49, 80, + 49, 80, 49, 80, 49, 80, 49, 2, 135, 166, 135, 166, 135, 166, 135, 166, + 135, 166, 135, 166, 135, 166, 135, 166, 42, 43, 43, 43, 43, 43, 43, 43, + 43, 43, 43, 43, 43, 0, 0, 0, 84, 86, 86, 86, 86, 86, 86, 86, + 86, 86, 86, 86, 86, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 84, 86, 86, 86, 86, 86, 86, 86, 86, 86, 86, 86, 86, + 12, 0, 12, 42, 43, 43, 43, 43, 43, 43, 43, 43, 43, 43, 43, 43, + 43, 7, 42, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 42, 43, 43, 43, 43, 43, 43, + 43, 43, 43, 43, 43, 43, 43, 43, 43, 43, 43, 43, 43, 43, 43, 43, + 43, 43, 43, 43, 86, 86, 108, 129, 21, 0, 43, 43, 43, 43, 43, 43, + 43, 43, 43, 43, 43, 43, 43, 43, 43, 43, 43, 43, 43, 43, 43, 43, + 43, 43, 43, 43, 43, 43, 43, 43, 43, 43, 43, 43, 43, 43, 43, 43, + 43, 43, 43, 43, 7, 108, 3, 65, 43, 43, 86, 86, 86, 86, 86, 86, + 86, 86, 86, 86, 86, 86, 86, 86, 44, 86, 43, 43, 43, 43, 43, 43, + 43, 43, 43, 43, 43, 43, 43, 43, 43, 43, 43, 43, 43, 43, 43, 1, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 12, 108, 0, 0, 0, 0, 0, 6, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 6, 37, 6, 37, 6, 37, 6, 37, 6, 37, + 6, 37, 6, 37, 6, 37, 6, 37, 6, 37, 6, 37, 6, 37, 6, 37, + 6, 37, 6, 37, 6, 37, 6, 37, 6, 37, 6, 37, 6, 37, 6, 37, + 6, 37, 6, 37, 6, 37, 6, 37, 86, 122, 158, 38, 6, 37, 6, 37, + 6, 37, 6, 37, 6, 37, 6, 37, 6, 37, 6, 37, 6, 37, 6, 37, + 6, 37, 6, 37, 6, 37, 6, 37, 6, 37, 6, 1, 43, 43, 79, 86, + 86, 44, 43, 127, 86, 86, 57, 43, 43, 85, 86, 86, 43, 43, 79, 86, + 86, 44, 43, 127, 86, 86, 129, 55, 117, 91, 123, 92, 43, 43, 79, 86, + 86, 2, 172, 4, 0, 0, 57, 43, 43, 85, 86, 86, 43, 43, 79, 86, + 86, 44, 43, 43, 86, 86, 50, 19, 129, 87, 0, 111, 129, 126, 201, 215, + 126, 45, 129, 129, 14, 126, 57, 127, 111, 87, 0, 129, 129, 126, 21, 0, + 126, 3, 43, 43, 43, 43, 43, 43, 43, 43, 43, 43, 43, 43, 7, 43, + 36, 43, 151, 43, 43, 43, 43, 43, 43, 43, 43, 43, 42, 43, 43, 43, + 43, 43, 86, 86, 86, 86, 86, 128, 129, 129, 129, 129, 57, 187, 42, 43, + 43, 43, 43, 43, 43, 43, 43, 43, 43, 43, 43, 43, 43, 43, 43, 43, + 43, 43, 43, 43, 43, 43, 43, 43, 43, 43, 43, 43, 43, 43, 43, 43, + 43, 43, 43, 43, 43, 43, 43, 1, 129, 129, 129, 129, 129, 129, 129, 129, + 129, 129, 129, 129, 129, 129, 129, 201, 172, 172, 172, 172, 172, 172, 172, 172, + 172, 172, 172, 172, 172, 172, 172, 208, 13, 0, 78, 49, 2, 180, 193, 193, + 215, 215, 36, 80, 49, 80, 49, 80, 49, 80, 49, 80, 49, 80, 49, 80, + 49, 80, 49, 80, 49, 80, 49, 80, 49, 80, 49, 80, 49, 80, 49, 80, + 49, 80, 49, 80, 215, 215, 83, 193, 71, 212, 215, 215, 215, 5, 43, 43, + 43, 43, 43, 43, 43, 43, 43, 43, 43, 43, 7, 1, 0, 1, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 78, 49, 80, 49, 80, 49, 80, + 49, 80, 49, 80, 49, 80, 49, 80, 13, 0, 0, 0, 0, 0, 36, 80, + 49, 80, 49, 80, 49, 80, 49, 80, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 43, 43, 43, 43, 43, 43, + 43, 43, 43, 43, 43, 121, 92, 123, 92, 123, 79, 123, 92, 123, 92, 123, + 92, 123, 92, 123, 92, 123, 92, 123, 92, 123, 92, 123, 92, 123, 92, 45, + 43, 43, 121, 20, 92, 123, 92, 45, 121, 42, 92, 39, 92, 123, 92, 123, + 92, 123, 164, 0, 10, 180, 92, 123, 92, 123, 79, 3, 42, 43, 43, 43, + 43, 43, 43, 43, 43, 43, 43, 43, 43, 43, 43, 43, 43, 43, 43, 1, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 72, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 42, 43, 43, 43, 43, 43, 43, 43, 43, 43, 43, + 43, 43, 43, 43, 43, 43, 43, 43, 43, 43, 43, 43, 43, 43, 43, 43, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 43, 43, 43, 43, 43, 43, 43, 43, 7, 0, 72, 86, 86, 86, 86, + 86, 86, 86, 86, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 43, 43, 43, 43, + 43, 43, 43, 43, 43, 43, 43, 43, 43, 85, 86, 86, 86, 86, 86, 86, + 86, 86, 86, 86, 86, 86, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 36, 43, 43, 43, 43, 43, 43, 43, 43, 43, + 43, 43, 7, 0, 86, 86, 86, 86, 86, 86, 86, 86, 86, 86, 86, 86, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 36, 43, 43, 43, + 43, 43, 43, 43, 43, 43, 43, 43, 43, 43, 43, 43, 43, 7, 0, 0, + 0, 0, 86, 86, 86, 86, 86, 86, 86, 86, 86, 86, 86, 86, 86, 86, + 86, 86, 86, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 42, 43, 43, + 43, 43, 43, 43, 43, 43, 43, 43, 86, 86, 86, 86, 86, 86, 86, 86, + 86, 86, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 42, 43, 43, 43, 43, 43, 43, 43, 43, 43, 43, 86, 86, + 86, 86, 86, 86, 86, 86, 86, 86, 14, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 43, 43, 43, 43, 43, 43, 43, 43, 43, 43, 43, 85, + 86, 86, 86, 86, 86, 86, 86, 86, 86, 86, 14, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, +}; +static const int rules[] = { + 0x0, 0x2001, -0x2000, 0x1dbf00, 0x2e700, 0x7900, + 0x2402, 0x101, -0x100, 0x0, 0x201, -0x200, + -0xc6ff, -0xe800, -0x78ff, -0x12c00, 0xc300, 0xd201, + 0xce01, 0xcd01, 0x4f01, 0xca01, 0xcb01, 0xcf01, + 0x6100, 0xd301, 0xd101, 0xa300, 0xd501, 0x8200, + 0xd601, 0xda01, 0xd901, 0xdb01, 0x3800, 0x3, + -0x4f00, -0x60ff, -0x37ff, 0x242802, 0x0, 0x101, + -0x100, -0xcd00, -0xda00, -0x81ff, 0x2a2b01, -0xa2ff, + 0x2a2801, 0x2a3f00, -0xc2ff, 0x4501, 0x4701, 0x2a1f00, + 0x2a1c00, 0x2a1e00, -0xd200, -0xce00, -0xca00, -0xcb00, + 0xa54f00, 0xa54b00, -0xcf00, 0xa52800, 0xa54400, -0xd100, + -0xd300, 0x29f700, 0xa54100, 0x29fd00, -0xd500, -0xd600, + 0x29e700, 0xa54300, 0xa52a00, -0x4500, -0xd900, -0x4700, + -0xdb00, 0xa51500, 0xa51200, 0x4c2402, 0x0, 0x2001, + -0x2000, 0x101, -0x100, 0x5400, 0x7401, 0x2601, + 0x2501, 0x4001, 0x3f01, -0x2600, -0x2500, -0x1f00, + -0x4000, -0x3f00, 0x801, -0x3e00, -0x3900, -0x2f00, + -0x3600, -0x800, -0x5600, -0x5000, 0x700, -0x7400, + -0x3bff, -0x6000, -0x6ff, 0x701a02, 0x101, -0x100, + 0x2001, -0x2000, 0x5001, 0xf01, -0xf00, 0x0, + 0x3001, -0x3000, 0x101, -0x100, 0x0, 0xbc000, + 0x1c6001, 0x0, 0x97d001, 0x801, -0x800, 0x8a0502, + 0x0, -0xbbfff, -0x186200, 0x89c200, -0x182500, -0x186e00, + -0x186d00, -0x186400, -0x186300, -0x185c00, 0x0, 0x8a3800, + 0x8a0400, 0xee600, 0x101, -0x100, 0x0, -0x3b00, + -0x1dbeff, 0x8f1d02, 0x800, -0x7ff, 0x0, 0x5600, + -0x55ff, 0x4a00, 0x6400, 0x8000, 0x7000, 0x7e00, + 0x900, -0x49ff, -0x8ff, -0x1c2500, -0x63ff, -0x6fff, + -0x7fff, -0x7dff, 0xac0502, 0x0, 0x1001, -0x1000, + 0x1c01, 0x101, -0x1d5cff, -0x20beff, -0x2045ff, -0x1c00, + 0xb10b02, 0x101, -0x100, 0x3001, -0x3000, 0x0, + -0x29f6ff, -0xee5ff, -0x29e6ff, -0x2a2b00, -0x2a2800, -0x2a1bff, + -0x29fcff, -0x2a1eff, -0x2a1dff, -0x2a3eff, 0x0, -0x1c6000, + 0x0, 0x101, -0x100, 0xbc0c02, 0x0, 0x101, + -0x100, -0xa543ff, 0x3a001, -0x8a03ff, -0xa527ff, 0x3000, + -0xa54eff, -0xa54aff, -0xa540ff, -0xa511ff, -0xa529ff, -0xa514ff, + -0x2fff, -0xa542ff, -0x8a37ff, 0x0, -0x97d000, -0x3a000, + 0x0, 0x2001, -0x2000, 0x0, 0x2801, -0x2800, + 0x0, 0x4001, -0x4000, 0x0, 0x2001, -0x2000, + 0x0, 0x2001, -0x2000, 0x0, 0x2201, -0x2200, +}; +static const unsigned char rulebases[] = { + 0, 6, 39, 81, 111, 119, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 124, 0, 0, 127, 0, 0, 0, 0, 0, 0, 0, 0, 131, 142, 146, 151, + 0, 170, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 180, 196, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 198, 201, 0, 0, 0, 219, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 222, + 0, 0, 0, 0, 225, 0, 0, 0, 0, 0, 0, 0, 228, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 231, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 234, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 237, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, +}; +static const unsigned char exceptions[][2] = { + { 48, 12 }, { 49, 13 }, { 120, 14 }, { 127, 15 }, + { 128, 16 }, { 129, 17 }, { 134, 18 }, { 137, 19 }, + { 138, 19 }, { 142, 20 }, { 143, 21 }, { 144, 22 }, + { 147, 19 }, { 148, 23 }, { 149, 24 }, { 150, 25 }, + { 151, 26 }, { 154, 27 }, { 156, 25 }, { 157, 28 }, + { 158, 29 }, { 159, 30 }, { 166, 31 }, { 169, 31 }, + { 174, 31 }, { 177, 32 }, { 178, 32 }, { 183, 33 }, + { 191, 34 }, { 197, 35 }, { 200, 35 }, { 203, 35 }, + { 221, 36 }, { 242, 35 }, { 246, 37 }, { 247, 38 }, + { 32, 45 }, { 58, 46 }, { 61, 47 }, { 62, 48 }, + { 63, 49 }, { 64, 49 }, { 67, 50 }, { 68, 51 }, + { 69, 52 }, { 80, 53 }, { 81, 54 }, { 82, 55 }, + { 83, 56 }, { 84, 57 }, { 89, 58 }, { 91, 59 }, + { 92, 60 }, { 97, 61 }, { 99, 62 }, { 101, 63 }, + { 102, 64 }, { 104, 65 }, { 105, 66 }, { 106, 64 }, + { 107, 67 }, { 108, 68 }, { 111, 66 }, { 113, 69 }, + { 114, 70 }, { 117, 71 }, { 125, 72 }, { 130, 73 }, + { 135, 74 }, { 137, 75 }, { 138, 76 }, { 139, 76 }, + { 140, 77 }, { 146, 78 }, { 157, 79 }, { 158, 80 }, + { 69, 87 }, { 123, 29 }, { 124, 29 }, { 125, 29 }, + { 127, 88 }, { 134, 89 }, { 136, 90 }, { 137, 90 }, + { 138, 90 }, { 140, 91 }, { 142, 92 }, { 143, 92 }, + { 172, 93 }, { 173, 94 }, { 174, 94 }, { 175, 94 }, + { 194, 95 }, { 204, 96 }, { 205, 97 }, { 206, 97 }, + { 207, 98 }, { 208, 99 }, { 209, 100 }, { 213, 101 }, + { 214, 102 }, { 215, 103 }, { 240, 104 }, { 241, 105 }, + { 242, 106 }, { 243, 107 }, { 244, 108 }, { 245, 109 }, + { 249, 110 }, { 253, 45 }, { 254, 45 }, { 255, 45 }, + { 80, 105 }, { 81, 105 }, { 82, 105 }, { 83, 105 }, + { 84, 105 }, { 85, 105 }, { 86, 105 }, { 87, 105 }, + { 88, 105 }, { 89, 105 }, { 90, 105 }, { 91, 105 }, + { 92, 105 }, { 93, 105 }, { 94, 105 }, { 95, 105 }, + { 130, 0 }, { 131, 0 }, { 132, 0 }, { 133, 0 }, + { 134, 0 }, { 135, 0 }, { 136, 0 }, { 137, 0 }, + { 192, 117 }, { 207, 118 }, { 128, 137 }, { 129, 138 }, + { 130, 139 }, { 133, 140 }, { 134, 141 }, { 112, 157 }, + { 113, 157 }, { 118, 158 }, { 119, 158 }, { 120, 159 }, + { 121, 159 }, { 122, 160 }, { 123, 160 }, { 124, 161 }, + { 125, 161 }, { 179, 162 }, { 186, 163 }, { 187, 163 }, + { 188, 164 }, { 190, 165 }, { 195, 162 }, { 204, 164 }, + { 218, 166 }, { 219, 166 }, { 229, 106 }, { 234, 167 }, + { 235, 167 }, { 236, 110 }, { 243, 162 }, { 248, 168 }, + { 249, 168 }, { 250, 169 }, { 251, 169 }, { 252, 164 }, + { 38, 176 }, { 42, 177 }, { 43, 178 }, { 78, 179 }, + { 132, 8 }, { 98, 186 }, { 99, 187 }, { 100, 188 }, + { 101, 189 }, { 102, 190 }, { 109, 191 }, { 110, 192 }, + { 111, 193 }, { 112, 194 }, { 126, 195 }, { 127, 195 }, + { 125, 207 }, { 141, 208 }, { 148, 209 }, { 171, 210 }, + { 172, 211 }, { 173, 212 }, { 176, 213 }, { 177, 214 }, + { 178, 215 }, { 196, 216 }, { 197, 217 }, { 198, 218 }, +}; diff --git a/src/ctype/nonspacing.h b/src/ctype/nonspacing.h index 48231e73..7746f3b6 100644 --- a/src/ctype/nonspacing.h +++ b/src/ctype/nonspacing.h @@ -1,23 +1,23 @@ -16,16,16,18,19,20,21,22,23,24,25,26,27,28,29,30,31,16,16,32,16,16,16,33,34,35, -36,37,38,39,16,16,40,16,16,16,16,16,16,16,16,16,16,16,41,42,16,16,43,16,16,16, +16,16,16,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32,16,33,16,16,16,34,35,36, +37,38,39,40,16,16,41,16,16,16,16,16,16,16,16,16,16,16,42,43,16,16,44,16,16,16, 16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16, 16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16, 16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16, 16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16, -16,16,16,16,16,16,16,16,16,16,44,16,45,46,47,48,16,16,16,16,16,16,16,16,16,16, +16,16,16,16,16,16,16,16,16,16,45,16,46,47,48,49,16,16,16,16,16,16,16,16,16,16, 16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16, +16,16,16,16,16,16,16,50,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16, +16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,51,16,16,52, +53,16,54,55,56,16,16,16,16,16,16,57,16,16,58,16,59,60,61,62,63,64,65,66,67,68, +69,70,16,71,72,73,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16, +16,74,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16, 16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16, -16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,49,16,16,50, -51,16,52,53,54,16,16,16,16,16,16,55,16,16,16,16,16,56,57,58,59,60,61,62,63,16, -16,64,16,65,66,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16, +16,16,16,75,76,16,16,16,77,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16, 16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16, 16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16, -16,16,16,67,68,16,16,16,69,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16, -16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16, -16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16, -16,16,16,16,16,16,16,70,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16, -16,16,71,72,16,16,16,16,16,16,16,73,16,16,16,16,16,74,16,16,16,16,16,16,16,75, -76,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16, +16,16,16,16,16,16,16,78,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16, +16,16,79,80,16,16,16,16,16,16,16,81,16,16,16,16,16,82,83,84,16,16,16,16,16,85, +86,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16, 16,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,255,255, 255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255, 255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255, @@ -25,57 +25,67 @@ 0,0,0,0,0,0,0,248,3,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, 0,0,0,254,255,255,255,255,191,182,0,0,0,0,0,0,0,63,0,255,23,0,0,0,0,0,248,255, 255,0,0,1,0,0,0,0,0,0,0,0,0,0,0,192,191,159,61,0,0,0,128,2,0,0,0,255,255,255, -7,0,0,0,0,0,0,0,0,0,0,192,255,1,0,0,0,0,0,0,248,15,0,0,0,192,251,239,62,0,0,0, -0,0,14,0,0,0,0,0,0,0,0,0,0,0,0,0,0,240,255,255,255,255, +7,0,0,0,0,0,0,0,0,0,0,192,255,1,0,0,0,0,0,0,248,15,32,0,0,192,251,239,62,0,0, +0,0,0,14,0,0,0,0,0,0,0,0,0,0,0,0,0,0,248,255,255,255,255, 255,7,0,0,0,0,0,0,20,254,33,254,0,12,0,0,0,2,0,0,0,0,0,0,16,30,32,0,0,12,0,0, -0,6,0,0,0,0,0,0,16,134,57,2,0,0,0,35,0,6,0,0,0,0,0,0,16,190,33,0,0,12,0,0,252, -2,0,0,0,0,0,0,144,30,32,64,0,12,0,0,0,4,0,0,0,0,0,0,0,1,32,0,0,0,0,0,0,1,0,0, -0,0,0,0,192,193,61,96,0,12,0,0,0,2,0,0,0,0,0,0,144,64,48,0,0,12,0,0,0,3,0,0,0, -0,0,0,24,30,32,0,0,12,0,0,0,0,0,0,0,0,0,0,0,0,4,92,0,0,0,0,0,0,0,0,0,0,0,242, -7,128,127,0,0,0,0,0,0,0,0,0,0,0,0,242,27,0,63,0,0,0,0,0,0,0,0,0,3,0,0,160,2,0, -0,0,0,0,0,254,127,223,224,255,254,255,255,255,31,64,0,0,0,0,0,0,0,0,0,0,0,0, -224,253,102,0,0,0,195,1,0,30,0,100,32,0,32,0,0,0,0,0,0,0,0,0,0,0, -0,0,0,0,0,0,0,0,0,0,0,0,224,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,28,0, -0,0,28,0,0,0,12,0,0,0,12,0,0,0,0,0,0,0,176,63,64,254,15,32,0,0,0,0,0,120,0,0, -0,0,0,0,0,0,0,0,0,0,0,0,96,0,0,0,0,2,0,0,0,0,0,0,0,0,0,0,0,0,0,0,135,1,4,14,0, -0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,128,9,0,0,0,0,0,0,64,127, -229,31,248,159,0,0,0,0,0,0,255,127,0,0,0,0,0,0,0,0,15,0,0,0,0,0,208,23,4,0,0, -0,0,248,15,0,3,0,0,0,60,59,0,0,0,0,0,0,64,163,3,0,0,0,0,0,0,240,207,0,0,0,0,0, -0,0,0,0,0,0,0,0,0,0,0,0,0,0,247,255,253,33,16,3,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, -0,0,0,0,0,0,0,0,0,255,255,255,255,255,255,255, +64,6,0,0,0,0,0,0,16,134,57,2,0,0,0,35,0,6,0,0,0,0,0,0,16,190,33,0,0,12,0,0, +252,2,0,0,0,0,0,0,144,30,32,64,0,12,0,0,0,4,0,0,0,0,0,0,0,1,32,0,0,0,0,0,0,17, +0,0,0,0,0,0,192,193,61,96,0,12,0,0,0,2,0,0,0,0,0,0,144,64,48,0,0,12,0,0,0,3,0, +0,0,0,0,0,24,30,32,0,0,12,0,0,0,0,0,0,0,0,0,0,0,0,4,92,0,0,0,0,0,0,0,0,0,0,0, +242,7,128,127,0,0,0,0,0,0,0,0,0,0,0,0,242,31,0,63,0,0,0,0,0,0,0,0,0,3,0,0,160, +2,0,0,0,0,0,0,254,127,223,224,255,254,255,255,255,31,64,0,0,0,0,0,0,0,0,0,0,0, +0,224,253,102,0,0,0,195,1,0,30,0,100,32,0,32,0,0,0,0,0,0,0,0,0,0,0, +0,0,0,0,0,0,0,0,0,0,0,0,0,255,255,255,255,255,255,255,255,255,255,255,255,255, +255,255,255,255,255,255,255,0,0,0,0,0,0,0,0,0,0,0,224,0,0,0,0,0,0,0,0,0,0,0,0, +0,0,0,0,0,0,0,0,0,0,28,0,0,0,28,0,0,0,12,0,0,0,12,0,0,0,0,0,0,0,176,63,64,254, +15,32,0,0,0,0,0,120,0,0,0,0,0,0,0,0,0,0,0,0,0,0,96,0,0,0,0,2,0,0,0,0,0,0,0,0, +0,0,0,0,0,0,135,1,4,14,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, +128,9,0,0,0,0,0,0,64,127,229,31,248,159,0,0,0,0,0,0,255,127,0,0,0,0,0,0,0,0, +15,0,0,0,0,0,208,23,4,0,0,0,0,248,15,0,3,0,0,0,60,59,0,0,0,0,0,0,64,163,3,0,0, +0,0,0,0,240,207,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,247,255,253,33,16, +3,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,255,255,255,255,255,255,255, 251,0,248,0,0,0,124,0,0,0,0,0,0,223,255,0,0,0,0,0,0,0,0,0,0,0,0,255,255,255, 255,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,128,3,0,0,0, 0,0,0,0,0,0,0,0,0,0,0,0,0,128,0,0,0,0,0,0,0,0,0,0,0,0,255,255,255,255,0,0,0,0, 0,60,0,0,0,0,0,0,0,0,0,0,0,0,0,6,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, 0,0,0,128,247,63,0,0,0,192,0,0,0,0,0,0,0,0,0,0,3,0,68,8,0,0,96,0,0,0,0,0,0,0, -0,0,0,0,0,0,0,0,0,0,0,0,48,0,0,0,255,255,3,0,0,0,0,0,192,63,0,0,128,255,3,0,0, -0,0,0,7,0,0,0,0,0,200,19,0,0,0,0,32,0,0,0,0,0,0,0,0,126,102,0,8,16,0,0,0,0,0, -16,0,0,0,0,0,0,157,193,2,0,0,0,0,48,64, -0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,32,33,0,0,0,0,0,64, -0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,255,255,0,0,255,255,0, -0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,128,0,0,0,0,0,0,0,0,0,0,0,0,0, -0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,14,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, -0,0,0,0,0,0,0,0,0,0,0,0,32,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, -0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,192,7,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, -0,110,240,0,0,0,0,0,135,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,96,0,0, -0,2,0,0,0,0,0,0,255,127,0,0,0,0,0,0,128,3,0,0,0,0,0,120,38,0,0,0,0,0,0,0,0,7, -0,0,0,128,239,31,0,0,0,0,0,0,0,8,0,3,0,0,0,0,0,192,127,0,28,0,0,0,0,0,0,0,0,0, -0,0,128,211,64,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,128,248,7,0,0,3,0,0,0,0, -0,0,16,1,0,0,0,192,31,31,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,255, -92,0,0,0,0,0,0,0,0,0,0,0,0,0,248,133,13,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, -0,0,0,0,0,0,0,0,0,0,60,176,1,0,0,48,0,0,0,0,0,0,0,0,0,0,248,167,1,0,0,0,0,0,0, -0,0,0,0,0,0,40,191,0,0,0,0,0,0,0,0,0,0,0,0,224,188,15,0,0,0,0,0,0,0,0,0,0,0,0, -0,0,0,0,0,0,0,0,0,0,0,0,0, -0,126,6,0,0,0,0,248,121,128,0,126,14,0,0,0,0,0,252,127,3,0,0,0,0,0,0,0,0,0,0, -0,0,0,0,0,0,0,0,127,191,0,0,0,0,0,0,0,0,0,0,252,255,255,252,109,0,0,0,0,0,0,0, -0,0,0,0,0,0,0,0,126,180,191,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, -0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,31,0,0,0,0,0,0,0,127, +0,0,0,0,0,0,0,0,0,0,0,0,48,0,0,0,255,255,3,128,0,0,0,0,192,63,0,0,128,255,3,0, +0,0,0,0,7,0,0,0,0,0,200,51,0,0,0,0,32,0,0, +0,0,0,0,0,0,126,102,0,8,16,0,0,0,0,0,16,0,0,0,0,0,0,157,193,2,0,0,0,0,48,64,0, +0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,32,33,0,0,0,0,0,0,0,0, +0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,255,255,255,255,255,255,255,255,255,255,0,0,0, +64,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,255,255,0,0,255, +255,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,128,0,0,0,0,0,0,0,0,0,0, +0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,14,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, +0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,32,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, +0,0,0,0,0,0,1,0,0, +0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,192,7,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,110,240,0, +0,0,0,0,135,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,96,0,0,0,0,0,0,0,240,0,0, +0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,192,255,1,0, +0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,2,0,0,0,0,0,0,255,127,0,0,0,0,0,0,128, +3,0,0,0,0,0,120,38,0,32,0,0,0,0,0,0,7,0,0,0,128,239,31,0,0,0,0,0,0,0,8,0,3,0, +0,0,0,0,192,127,0,30,0,0,0,0,0,0,0,0,0,0,0,128,211,64,0,0,0,0,0,0,0,0,0,0,0,0, +0,0,0,0,0,0,0,128,248,7,0,0,3,0,0,0,0,0,0,24,1,0,0,0,192,31,31,0,0,0,0,0,0,0, +0,0,0,0,0,0,0,0,0, +0,0,0,0,0,0,0,0,255,92,0,0,64,0,0,0,0,0,0,0,0,0,0,248,133,13,0,0,0,0,0,0,0,0, +0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,60,176,1,0,0,48,0,0,0,0,0,0,0,0,0,0, +248,167,1,0,0,0,0,0,0,0,0,0,0,0,0,40,191,0,0,0,0,0,0,0,0,0,0,0,0,224,188,15,0, +0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,128,255,6,0,0,0,0, 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, -0,0,0,128,7,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, -96,15,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,128,3,248,255,231,15,0,0, -0,60,0,0,0,0,0,0,0,0,0, +0,0,0,0,0,0,0,240,12,1,0,0,0,254,7,0,0,0,0,248,121,128,0,126,14,0,0,0,0,0,252, +127,3,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,127,191,0,0,0,0,0,0,0,0,0,0,252,255, +255,252,109,0,0,0,0,0,0,0,0, +0,0,0,0,0,0,0,126,180,191,0,0,0,0,0,0,0,0,0,163,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, +0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,24,0,0,0,0,0,0,0,255, +1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, +0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,31,0,0,0,0,0,0,0,127,0,0,0,0,0,0,0,0,0,0,0,0, +0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,128,0,0,0,0,0,0,0,128,7,0,0,0,0,0, +0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,96,15,0,0,0,0,0,0,0,0,0, +0,0,0,0,0,0,0,0,0,0,0,0,0,0,128,3,248,255,231,15,0,0,0,60,0,0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,0,28,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,255,255, 255,255,255,255,127,248,255,255,255,255,255,31,32,0,16,0,0,248,254,255,0,0,0, 0,0,0,0,0,0,0,127,255,255,249,219,7,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, -0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,127,0,0,0,0,0,0, -0,0,0,0,0,0,0,240,7,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, +0,0,0,0,0,0,0,0,0,0,0,127,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, +0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,240,0,0,0,0,0,0,0,0,0, +0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,127,0,0,0,0,0,0,0,0,0,0,0,0,0,240,7,0,0, +0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, diff --git a/src/ctype/punct.h b/src/ctype/punct.h index 7a623940..67929470 100644 --- a/src/ctype/punct.h +++ b/src/ctype/punct.h @@ -8,17 +8,17 @@ 16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16, 16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,64,16,16,16,16,16,16,16,16,16, 16,16,16,16,16,16,16,16,16,16,16,16,16,16,65,16,16,66,16,67,68, -69,16,70,71,72,16,73,16,16,74,75,76,77,78,16,79,16,80,81,82,83,84,85,86,87,88, -16,89,16,90,91,16,16,16,16,16,16,92,16,16,16,16,16,16,16,16,16,16,16,16,16,16, +69,16,70,71,72,16,73,16,16,74,75,76,77,78,16,79,80,81,82,83,84,85,86,87,88,89, +90,91,16,92,93,94,95,16,16,16,16,96,16,16,16,16,16,16,16,16,16,16,16,16,16,16, +16,97,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16, 16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16, +16,16,16,98,99,16,16,100,101,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16, 16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16, -16,16,16,93,94,16,16,16,95,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16, 16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16, -16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16, -16,16,16,16,16,16,16,96,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16, -16,97,98,99,100,16,16,101,102,17,17,103,16,16,16,16,16,16,16,16,16,16,16,16, -16,104,105,16,16,16,16,106,16,107,108,109,17,17,17,110,111,112,113,16,16,16, -16,16, +16,16,16,16,16,16,16,16,102,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16, +16,16,16,103,104,105,106,16,16,107,108,17,17,109,16,16,16,16,16,16,110,111,16, +16,16,16,16,112,113,16,16,114,115,116,16,117,118,119,17,17,17,120,121,122,123, +124,16,16,16,16, 16,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,255,255, 255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255, 255,255,255,255,255,255,255,255,255,255,255,0,0,0,0,254,255,0,252,1,0,0,248,1, @@ -28,25 +28,25 @@ 0,0,0,0,0,0,0,0,0,0,0,0,0,0,252,3,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, 0,0,0,252,0,0,0,0,0,230,254,255,255,255,0,64,73,0,0,0,0,0,24,0,255,255,0,216, 0,0,0,0,0,0,0,1,0,60,0,0,0,0,0,0,0,0,0,0,0,0,16,224,1,30,0, -96,255,191,0,0,0,0,0,0,255,7,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,248,207,3, -0,0,0,3,0,32,255,127,0,0,0,78,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,7,252,0,0,0,0,0, -0,0,0,0,16,0,32,30,0,48,0,1,0,0,0,0,0,0,0,0,16,0,32,0,0,0,0,252,47,0,0,0,0,0, -0,0,16,0,32,0,0,0,0,0,0,0,0,0,0,0,0,0,16,0,32,0,0,0,0,3,224,0,0,0,0,0,0,0,16, -0,32,0,0,0,0,253,0,0,0,0,0,0,0,0,0,0,32,0,0,0,0,255,7,0,0,0,0,0,0,0,0,0,32,0, -0,0,0,0,255,0,0,0,0,0,0,0,16,0,32,0,0,0,0,0,0,0,0,0,0,0,0,0,24,0,160,0,127,0, -0,255,3,0,0,0,0,0,0,0,0,0,4,0,0,0,0,16,0,0,0,0,0,0,128,0,128,192,223,0,12,0,0, -0,0,0,0,0,0,0,0,0,0,0,31,0,0,0,0,0, +96,255,191,0,0,0,0,0,0,255,7,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,248,207, +227,0,0,0,3,0,32,255,127,0,0,0,78,0,0,0,0,0,0,0,0,0,0,0,0,0,0,8,0,7,252,0,0,0, +0,0,0,0,0,0,16,0,32,30,0,48,0,1,0,0,0,0,0,0,0,0,16,0,32,0,0,0,0,252,111,0,0,0, +0,0,0,0,16,0,32,0,0,0,0,64,0,0,0,0,0,0,0,0,16,0,32,0,0,0,0,3,224,0,0,0,0,0,0, +0,16,0,32,0,0,0,0,253,0,0,0,0,0,0,0,0,0,0,32,0,0,0,0,255,7,16,0,0,0,0,0,0,0,0, +32,0,0,0,0,128,255,16,0,0,0,0,0,0,16,0,32,0,0,0,0,0,0,0,0,0,0,0,0,0,24,0,160, +0,127,0,0,255,3,0,0,0,0,0,0,0,0,0,4,0,0,0,0,16,0,0,0,0,0,0,128,0,128,192,223, +0,12,0,0,0,0,0,0,0,0,0,0,0,4,0,31,0,0,0,0,0, 0,254,255,255,255,0,252,255,255,0,0,0,0,0,0,0,0,252,0,0,0,0,0,0,192,255,223, -255,7,0,0,0,0,0,0,0,0,0,0,128,6,0,252,0,0,24,62,0,0,128,191,0,204,0,0,0,0,0,0, -0,0,0,0,0,8,0,0,0,0,0,0,0,0,0,0,0,96,255,255,255,31,0,0,255,3,0,0,0,0,0,0,0,0, -0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, -0,0,0,0,0,0,0,0,0,0,96,0,0,1,0,0,24,0,0,0,0,0,0,0,0,0,56,0,0,0,0,16,0,0,0,112, -0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,48,0,0,254,127,47,0,0,255,3,255,127,0,0,0,0,0,0, -0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,14,49,0,0,0,0,0, -0,0,0,0,0,0,0,0,0,0,0,0,0,196,255,255,255, +255,7,0,0,0,0,0,0,0,0,0,0,128,6,0,252,0,0,0,0,0,0,0,0,0,192,0,0,0,0,0,0,0,0,0, +0,0,8,0,0,0,0,0,0,0,0,0,0,0,224,255,255,255,31,0,0,255,3,0,0,0,0,0,0,0,0,0,0, +0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, +0,0,0,0,0,0,0,0,96,0,0,1,0,0,24,0,0,0,0,0,0,0,0,0,56,0,0,0,0,16,0,0,0,112,0,0, +0,0,0,0,0,0,0,0,0,0,0,0,0,48,0,0,254,127,47,0,0,255,3,255,127,0,0,0,0,0,0,0,0, +0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,14,49,0,0,0,0,0,0,0, +0,0,0,0,0,0,0,0,0,0,0,196,255,255,255, 255,0,0,0,192,0,0,0,0,0,0,0,0,1,0,224,159,0,0,0,0,127,63,255,127,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,16,0,16,0,0,252,255,255,255,31,0,0,0,0,0,12,0,0,0,0,0,0,64,0, -12,240,0,0,0,0,0,0,192,248,0,0,0,0,0,0,0,192,0,0,0,0,0,0,0,0,255,0,255,255, +12,240,0,0,0,0,0,0,128,248,0,0,0,0,0,0,0,192,0,0,0,0,0,0,0,0,255,0,255,255, 255,33,144,3,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,255,255,255,255, 127,0,224,251,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,160,3,224,0,224,0, 224,0,96,128,248,255,255,255,252,255,255,255,255,255,127,223,255,241,127,255, @@ -55,22 +55,23 @@ 255,255,255,255,255,127,0,0,0,255,7,0,0,255,255,255,255,255,255,255,255,255, 255,63,0,0,0,0,0,0,252,255, 255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,207,255,255,255, -63,255,255,255,255,227,255,253,7,0,0,240,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, -0,0,0,0,0,0,0,0,0,0,0,0,224,135,3,254,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,128,0,0,0, -0,0,0,0,0,0,0,0,0,0,0,0,0,255,255,255,255,255,127,255,255,255,3,0,0,0,0,0,0, -255,255,255,251,255,255,255,255,255,255,255,255,255,255,15,0,255,255,255,255, +63,255,255,255,255,255,255,255,255,255,255,255,255,255,0,0,0,0,0,0,0,0,0,0,0, +0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,224,135,3,254,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1, +128,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,255,255,255,255,255,127,255,255,255,255,0, +0,0,0,0,0,255,255,255,251,255,255,255,255,255,255,255,255,255,255,15,0,255, 255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255, -255,255,255,63,0,0,0,255,15,30,255,255,255,1,252,193,224,0,0,0,0,0,0,0,0,0,0, -0,30,1,0,0,0,0,0,0,0,0,0,0,8,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,255,255,0,0, -0,0,255,255,255,255,15,0,0,0,255,255,255,127,255,255,255,255,255,255,255,255, +255,255,255,255,255,255,63,0,0,0,255,15,30,255,255,255,1,252,193,224,0,0,0,0, +0,0,0,0,0,0,0,30,1,0,0,0,0,0,0,0,0,0,0,8,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, +255,255,0,0,0,0,255,255,255,255,15,0,0,0,255,255,255,127,255,255,255,255,255, 255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255, -127,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,255,255,255,255,255,255, +255,255,255, +255,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,255,255,255,255,255,255, 255,255,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,255,255,255,255,255,255,127,0,0,0, 0,0,0,192,0,224,0,0,0,0,0,0,0,0,0,0,0,128,15,112,0,0,0,0,0,0,0,0,0,0,0,0,0,0, 255,0,255,255,127,0,3,0,0,0,0,0,0,0,0,0,0,0,0,6,0,0,0,0,0,0,0,0,0,0,0,0,0,0, -68,8,0,0,0,15,255,3,0,0,0,0,0,0,240,0,0,0,0,0,0,0,0,0,16,192,0,0,255,255,3,23, -0,0,0,0,0,248,0,0,0,0,8,128,0,0,0,0,0,0,0,0,0,0,8,0,255,63,0,192,32,0,0,0,0,0, -0,0,0,0,0,0,0,0,0,240,0,0,128,59,0,0,0,0,0,0,0,128,2,0,0,192,0,0,67,0,0,0,0,0, +64,0,0,0,0,15,255,3,0,0,0,0,0,0,240,0,0,0,0,0,0,0,0,0,16,192,0,0,255,255,3,23, +0,0,0,0,0,248,0,0,0,0,8,128,0,0,0,0,0,0,0,0,0,0,8,0,255,63,0,192,0,0,0,0,0,0, +0,0,0,0,0,0,0,0,0,240,0,0,128,3,0,0,0,0,0,0,0,128,2,0,0,192,0,0,67,0,0,0,0,0, 0,0,0,0,0,0,0,8,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,56,0, 0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,128,0,0,0,0,0,2,0,0,0,0,0,0, @@ -84,46 +85,57 @@ 0,0,0,0,0,0,0,0,0,128,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, 128,255,0,0,128,255,0,0,0,0,128,255,0,0,0,0,0,0,0,0,0,248,0,0,192,143,0,0,0, 128,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,48,255,255,252,255,255,255,255,255,0,0,0,0, -0,0,0,135,255,0,255,1,0,0,0,224,0,0,0,224,0,0,0,0,0,1,0,0,96,248,127,0,0,0,0, +0,0,0,135,255,1,255,1,0,0,0,224,0,0,0,224,0,0,0,0,0,1,0,0,96,248,127,0,0,0,0, 0,0,0,0,254,0,0,0,255,0,0,0,255,0,0,0,30,0,254,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,252,0,0,0,0,0,0,0,0,0,0,0, 0,255,255,255,127,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, -0,0,0,0,0,0,0,0,0,192,63,252,255,63,0,0,128,3,0,0,0,0,0,0,254,3,0,0,0,0,0,0,0, -0,0,0,0,0,0,24,0,15,0,0,0,0,0,56,0,0,0,0,0,0,0,0,0,225,63,0,232,254,255,31,0, -0,0,0,0,0,0,96,63,0,0,0,0,0,0,0,0,0,0,0,0,0,2,0,0,0,0,0,0,0,6,0,0,0,0,0,0,0,0, -0,16,0,32,0,0,192,31,31,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,68, -248,0,40,0,0,0,0,0,0,0,0,0,0,0,0,76,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, -0,0,0,0,0,0,0,0,0,128,255,255,255,0,0,0,0,0,0,0,0,0,0,0,0,128,14,0,0,0,255,31, -0,0,0,0,0,0,0,0,192,0,0,0,0,0,0,0,0,0,0,0,0,0,0,8,0,252,0,0,0,0,0,0,0,0,0,0,0, +0,0,0,0,224,127,0,0,0,192,255,255,3,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, +0,0,0,0,0,0,0,192,63,252,255,63,0,0,128,3,0,0,0,0,0,0,254,3,32,0,0,0,0,0,0,0, +0,0,0,0,0,24,0,15,0,0,0,0,0,56,0,0,0,0,0,0,0,0,0,225,63,0,232,254,255,31,0,0, +0,0,0,0,0,96,63,0,0,0,0,0,0,0,0,0,0,0,0,0,2,0,0,0,0,0,0,0,6,0,0,0,0,0,0,0,0,0, +24,0,32,0,0,192,31,31,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,68, +248,0,104,0,0,0,0,0,0,0,0,0,0,0,0,76,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, +0,0,0,0,0,0,0,0,0,0,128,255,255,255,0,0,0,0,0,0,0,0,0,0,0,0,128,14,0,0,0,255, +31,0,0,0,0,0,0,0,0,192,0,0,0,0,0,0,0,0, +0,0,0,0,0,0,8,0,252,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, +0,0,14,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,252,7,0,0,0,0,0,0,0,0,0,0,0, +0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,5,0,0,0,0,0,0,0,0,0,24,128,255,0,0,0,0,0, +0,0,0,0,0,223,7,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,128,62,0,0,252,255,31,3,0, +0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,52,0,0,0,0,0,0,0,0,0,128,0,0, +0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, +0,0,128,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,255,255,255,255,255, +255,3, +128,0,0,0,0,0,0,0,0,0,0,0,0,0,0,31,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, +0,0,255,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, +0,0,0,192,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,63,0,0,0,0,0,0,0,255,255,48,0,0,248, +3,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,255, +255,255,7,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, +0,0,0,0,0,0,4,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,176,15,0,0,0,0,0,0, +0,0,0,0,0,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255, +255,255,255,255,255,255,255,255,255,255,255,255,255,63, +0,255,255,255,255,127,254,255,255,255,255,255,255,255,255,255,255,255,255,255, +255,255,255,255,255,255,255,255,255,255,1,0,0,255,255,255,255,255,255,255,255, +63,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,255,255,15,0,255,255,255,255,255,255, +255,255,255,255,127,0,255,255,255,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, +0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,2,0,0,8,0,0,0,8,0,0,32,0,0,0,32,0,0,128, +0,0,0,128,0,0,0,2,0,0,0,2,0,0,8,0,0,0,0,0,0,0,255,255,255,255,255,255,255,255, +255,255,255,255,255,255,255,255,255,15,0,248,254,255,0,0,0,0,0,0,0,0,0,0,0,0, +0,0,0,0,127,0,0,128,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, +0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,240,0, +128,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,128,255,127,0,0,0,0,0,0,0, +0,0,0,0,0,0,112,7,0,192,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, +0,0,0,0,0,0,0,254,255,255,255,255,255,255,255,31,0,0,0,0,0,0,0,0,0,254,255, +255,255,255,255,255,63,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, +0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,3,0,255,255,255,255,255, +15,255,255,255,255,255,255,255,255,255,255,255,255,15,0,255,127,254,255,254, +255,254,255,255,255,63,0,255,31,255,255,255,255,0,0,0,252,0,0,0,28,0,0,0,252, +255,255,255,31,0,0,0,0,0,0,192,255,255,255,7,0,255,255,255,255,255,15,255,1,3, +0,63,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, +0,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255, +255,255,255,255,255,255,255,63,0,255,31,255,7,255,255,255,255,255,255,255,255, +255,255,255,255,255,255,15,0,255,255,255,255,255,255,255,255,255,255,255,1, +255,15,0,0,255,15,255,255,255,255,255,255,255,0,255,3,255,255,255,255,255,0, +255,255,255,63,0,0,0,0,0,0,0,0,0,0,255,239,255,255,255,255,255,255,255,255, +255,255,255,255,123,252,255,255,255,255,231,199,255,255,255,231,255,255,255, +255,255,255,255,255,255,255,255,255,255,255,255,255,15,0,255,63,15,7,7,0,63,0, 0,0,0,0,0,0,0,0,0,0,0,0, -0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,252,7,0,0,0,0,0,0, -0,24,128,255,0,0,0,0,0,0,0,0,0,0,223,7,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, -128,62,0,0,252,255,31,3,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,52, -0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,31, -0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,192,0,0,0,0,0,0,0, -0,0,0,0,0,0,0,0,0,63,0,0,0,0,0,0,0,128,255,48,0,0,248,3,0,0,0,0,0,0,0,0,0,0,0, -0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,128,7,0,0,0,0,0,0,0,0,0,0,0, -0, -0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,176,15,0,0,0,0,0,0,0,0,0,0,0,255,255, -255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255, -255,255,255,255,255,255,255,255,255,63,0,255,255,255,255,127,254,255,255,255, -255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255, -255,1,0,0,255,255,255,255,255,255,255,255,63,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, -0,0,0,0,0,0,0,255,255,255,255,255,255,255,255,255,255,127,0,255,255,3,0,0,0,0, -0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,2,0, -0,8,0,0,0,8,0,0,32,0,0,0,32,0,0,128,0,0,0,128,0,0,0,2,0,0,0,2,0,0,8,0,0,0,0,0, -0,0,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,15,0, -248,254,255,0,0,0,0,0,0,0,0,0, -0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,128,255,127,0,0,0,0,0,0,0,0, -0,0,0,0,0,112,7,0,192,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, -0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,3,0,255,255,255,255,255,15,255, -255,255,255,255,255,255,255,255,255,255,255,15,0,255,127,254,255,254,255,254, -255,255,255,63,0,255,31,255,255,255,127,0,0,0,252,0,0,0,12,0,0,0,252,255,255, -255,31,0,0,0,0,0,0,192,255,255,255,7,0,255,255,255,255,255,15,255,1,3,0,63,0, -0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,255,255,255,255,255,255,255,255,255,255, -255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,31,0,255,31, -255,1,255,255,255,255,255,255,255,255,255,255,255,255,255,255,15,0,255,255, -255,255,255,255,255,255,255,255,31,0,0,0,0, -0,255,15,255,255,255,255,255,255,255,0,255,3,255,255,255,255,255,0,255,255, -255,63,0,0,0,0,0,0,0,0,0,0,255,15,255,255,255,255,255,127,255,31,255,255,255, -15,0,0,255,255,255,0,0,0,0,0,1,0,255,255,127,0,0,0, diff --git a/src/ctype/towctrans.c b/src/ctype/towctrans.c index 8f681018..76d13769 100644 --- a/src/ctype/towctrans.c +++ b/src/ctype/towctrans.c @@ -1,307 +1,73 @@ -#include <ctype.h> -#include <stddef.h> #include <wctype.h> -#define CASEMAP(u1,u2,l) { (u1), (l)-(u1), (u2)-(u1)+1 } -#define CASELACE(u1,u2) CASEMAP((u1),(u2),(u1)+1) +static const unsigned char tab[]; -static const struct { - unsigned short upper; - signed char lower; - unsigned char len; -} casemaps[] = { - CASEMAP(0xc0,0xde,0xe0), +static const unsigned char rulebases[512]; +static const int rules[]; - CASELACE(0x0100,0x012e), - CASELACE(0x0132,0x0136), - CASELACE(0x0139,0x0147), - CASELACE(0x014a,0x0176), - CASELACE(0x0179,0x017d), +static const unsigned char exceptions[][2]; - CASELACE(0x370,0x372), - CASEMAP(0x391,0x3a1,0x3b1), - CASEMAP(0x3a3,0x3ab,0x3c3), - CASEMAP(0x400,0x40f,0x450), - CASEMAP(0x410,0x42f,0x430), +#include "casemap.h" - CASELACE(0x460,0x480), - CASELACE(0x48a,0x4be), - CASELACE(0x4c1,0x4cd), - CASELACE(0x4d0,0x50e), - - CASELACE(0x514,0x52e), - CASEMAP(0x531,0x556,0x561), - - CASELACE(0x01a0,0x01a4), - CASELACE(0x01b3,0x01b5), - CASELACE(0x01cd,0x01db), - CASELACE(0x01de,0x01ee), - CASELACE(0x01f8,0x021e), - CASELACE(0x0222,0x0232), - CASELACE(0x03d8,0x03ee), - - CASELACE(0x1e00,0x1e94), - CASELACE(0x1ea0,0x1efe), - - CASEMAP(0x1f08,0x1f0f,0x1f00), - CASEMAP(0x1f18,0x1f1d,0x1f10), - CASEMAP(0x1f28,0x1f2f,0x1f20), - CASEMAP(0x1f38,0x1f3f,0x1f30), - CASEMAP(0x1f48,0x1f4d,0x1f40), - - CASEMAP(0x1f68,0x1f6f,0x1f60), - CASEMAP(0x1f88,0x1f8f,0x1f80), - CASEMAP(0x1f98,0x1f9f,0x1f90), - CASEMAP(0x1fa8,0x1faf,0x1fa0), - CASEMAP(0x1fb8,0x1fb9,0x1fb0), - CASEMAP(0x1fba,0x1fbb,0x1f70), - CASEMAP(0x1fc8,0x1fcb,0x1f72), - CASEMAP(0x1fd8,0x1fd9,0x1fd0), - CASEMAP(0x1fda,0x1fdb,0x1f76), - CASEMAP(0x1fe8,0x1fe9,0x1fe0), - CASEMAP(0x1fea,0x1feb,0x1f7a), - CASEMAP(0x1ff8,0x1ff9,0x1f78), - CASEMAP(0x1ffa,0x1ffb,0x1f7c), - - CASEMAP(0x13f0,0x13f5,0x13f8), - CASELACE(0xa698,0xa69a), - CASELACE(0xa796,0xa79e), - - CASELACE(0x246,0x24e), - CASELACE(0x510,0x512), - CASEMAP(0x2160,0x216f,0x2170), - CASEMAP(0x2c00,0x2c2e,0x2c30), - CASELACE(0x2c67,0x2c6b), - CASELACE(0x2c80,0x2ce2), - CASELACE(0x2ceb,0x2ced), - - CASELACE(0xa640,0xa66c), - CASELACE(0xa680,0xa696), - - CASELACE(0xa722,0xa72e), - CASELACE(0xa732,0xa76e), - CASELACE(0xa779,0xa77b), - CASELACE(0xa77e,0xa786), - - CASELACE(0xa790,0xa792), - CASELACE(0xa7a0,0xa7a8), - - CASELACE(0xa7b4,0xa7b6), - - CASEMAP(0xff21,0xff3a,0xff41), - { 0,0,0 } -}; - -static const unsigned short pairs[][2] = { - { 'I', 0x0131 }, - { 'S', 0x017f }, - { 0x0130, 'i' }, - { 0x0178, 0x00ff }, - { 0x0181, 0x0253 }, - { 0x0182, 0x0183 }, - { 0x0184, 0x0185 }, - { 0x0186, 0x0254 }, - { 0x0187, 0x0188 }, - { 0x0189, 0x0256 }, - { 0x018a, 0x0257 }, - { 0x018b, 0x018c }, - { 0x018e, 0x01dd }, - { 0x018f, 0x0259 }, - { 0x0190, 0x025b }, - { 0x0191, 0x0192 }, - { 0x0193, 0x0260 }, - { 0x0194, 0x0263 }, - { 0x0196, 0x0269 }, - { 0x0197, 0x0268 }, - { 0x0198, 0x0199 }, - { 0x019c, 0x026f }, - { 0x019d, 0x0272 }, - { 0x019f, 0x0275 }, - { 0x01a6, 0x0280 }, - { 0x01a7, 0x01a8 }, - { 0x01a9, 0x0283 }, - { 0x01ac, 0x01ad }, - { 0x01ae, 0x0288 }, - { 0x01af, 0x01b0 }, - { 0x01b1, 0x028a }, - { 0x01b2, 0x028b }, - { 0x01b7, 0x0292 }, - { 0x01b8, 0x01b9 }, - { 0x01bc, 0x01bd }, - { 0x01c4, 0x01c6 }, - { 0x01c4, 0x01c5 }, - { 0x01c5, 0x01c6 }, - { 0x01c7, 0x01c9 }, - { 0x01c7, 0x01c8 }, - { 0x01c8, 0x01c9 }, - { 0x01ca, 0x01cc }, - { 0x01ca, 0x01cb }, - { 0x01cb, 0x01cc }, - { 0x01f1, 0x01f3 }, - { 0x01f1, 0x01f2 }, - { 0x01f2, 0x01f3 }, - { 0x01f4, 0x01f5 }, - { 0x01f6, 0x0195 }, - { 0x01f7, 0x01bf }, - { 0x0220, 0x019e }, - { 0x0386, 0x03ac }, - { 0x0388, 0x03ad }, - { 0x0389, 0x03ae }, - { 0x038a, 0x03af }, - { 0x038c, 0x03cc }, - { 0x038e, 0x03cd }, - { 0x038f, 0x03ce }, - { 0x0399, 0x0345 }, - { 0x0399, 0x1fbe }, - { 0x03a3, 0x03c2 }, - { 0x03f7, 0x03f8 }, - { 0x03fa, 0x03fb }, - { 0x1e60, 0x1e9b }, - { 0x1e9e, 0xdf }, - - { 0x1f59, 0x1f51 }, - { 0x1f5b, 0x1f53 }, - { 0x1f5d, 0x1f55 }, - { 0x1f5f, 0x1f57 }, - { 0x1fbc, 0x1fb3 }, - { 0x1fcc, 0x1fc3 }, - { 0x1fec, 0x1fe5 }, - { 0x1ffc, 0x1ff3 }, - - { 0x23a, 0x2c65 }, - { 0x23b, 0x23c }, - { 0x23d, 0x19a }, - { 0x23e, 0x2c66 }, - { 0x241, 0x242 }, - { 0x243, 0x180 }, - { 0x244, 0x289 }, - { 0x245, 0x28c }, - { 0x3f4, 0x3b8 }, - { 0x3f9, 0x3f2 }, - { 0x3fd, 0x37b }, - { 0x3fe, 0x37c }, - { 0x3ff, 0x37d }, - { 0x4c0, 0x4cf }, - - { 0x2126, 0x3c9 }, - { 0x212a, 'k' }, - { 0x212b, 0xe5 }, - { 0x2132, 0x214e }, - { 0x2183, 0x2184 }, - { 0x2c60, 0x2c61 }, - { 0x2c62, 0x26b }, - { 0x2c63, 0x1d7d }, - { 0x2c64, 0x27d }, - { 0x2c6d, 0x251 }, - { 0x2c6e, 0x271 }, - { 0x2c6f, 0x250 }, - { 0x2c70, 0x252 }, - { 0x2c72, 0x2c73 }, - { 0x2c75, 0x2c76 }, - { 0x2c7e, 0x23f }, - { 0x2c7f, 0x240 }, - { 0x2cf2, 0x2cf3 }, - - { 0xa77d, 0x1d79 }, - { 0xa78b, 0xa78c }, - { 0xa78d, 0x265 }, - { 0xa7aa, 0x266 }, - - { 0x10c7, 0x2d27 }, - { 0x10cd, 0x2d2d }, - - /* bogus greek 'symbol' letters */ - { 0x376, 0x377 }, - { 0x39c, 0xb5 }, - { 0x392, 0x3d0 }, - { 0x398, 0x3d1 }, - { 0x3a6, 0x3d5 }, - { 0x3a0, 0x3d6 }, - { 0x39a, 0x3f0 }, - { 0x3a1, 0x3f1 }, - { 0x395, 0x3f5 }, - { 0x3cf, 0x3d7 }, - - { 0xa7ab, 0x25c }, - { 0xa7ac, 0x261 }, - { 0xa7ad, 0x26c }, - { 0xa7ae, 0x26a }, - { 0xa7b0, 0x29e }, - { 0xa7b1, 0x287 }, - { 0xa7b2, 0x29d }, - { 0xa7b3, 0xab53 }, - - /* special cyrillic lowercase forms */ - { 0x412, 0x1c80 }, - { 0x414, 0x1c81 }, - { 0x41e, 0x1c82 }, - { 0x421, 0x1c83 }, - { 0x422, 0x1c84 }, - { 0x422, 0x1c85 }, - { 0x42a, 0x1c86 }, - { 0x462, 0x1c87 }, - { 0xa64a, 0x1c88 }, - - { 0,0 } -}; - - -static wchar_t __towcase(wchar_t wc, int lower) +static int casemap(unsigned c, int dir) { - int i; - int lmul = 2*lower-1; - int lmask = lower-1; - /* no letters with case in these large ranges */ - if (!iswalpha(wc) - || (unsigned)wc - 0x0600 <= 0x0fff-0x0600 - || (unsigned)wc - 0x2e00 <= 0xa63f-0x2e00 - || (unsigned)wc - 0xa800 <= 0xab52-0xa800 - || (unsigned)wc - 0xabc0 <= 0xfeff-0xabc0) - return wc; - /* special case because the diff between upper/lower is too big */ - if (lower && (unsigned)wc - 0x10a0 < 0x2e) - if (wc>0x10c5 && wc != 0x10c7 && wc != 0x10cd) return wc; - else return wc + 0x2d00 - 0x10a0; - if (!lower && (unsigned)wc - 0x2d00 < 0x26) - if (wc>0x2d25 && wc != 0x2d27 && wc != 0x2d2d) return wc; - else return wc + 0x10a0 - 0x2d00; - if (lower && (unsigned)wc - 0x13a0 < 0x50) - return wc + 0xab70 - 0x13a0; - if (!lower && (unsigned)wc - 0xab70 < 0x50) - return wc + 0x13a0 - 0xab70; - for (i=0; casemaps[i].len; i++) { - int base = casemaps[i].upper + (lmask & casemaps[i].lower); - if ((unsigned)wc-base < casemaps[i].len) { - if (casemaps[i].lower == 1) - return wc + lower - ((wc-casemaps[i].upper)&1); - return wc + lmul*casemaps[i].lower; + unsigned b, x, y, v, rt, xb, xn; + int r, rd, c0 = c; + + if (c >= 0x20000) return c; + + b = c>>8; + c &= 255; + x = c/3; + y = c%3; + + /* lookup entry in two-level base-6 table */ + v = tab[tab[b]*86+x]; + static const int mt[] = { 2048, 342, 57 }; + v = (v*mt[y]>>11)%6; + + /* use the bit vector out of the tables as an index into + * a block-specific set of rules and decode the rule into + * a type and a case-mapping delta. */ + r = rules[rulebases[b]+v]; + rt = r & 255; + rd = r >> 8; + + /* rules 0/1 are simple lower/upper case with a delta. + * apply according to desired mapping direction. */ + if (rt < 2) return c0 + (rd & -(rt^dir)); + + /* binary search. endpoints of the binary search for + * this block are stored in the rule delta field. */ + xn = rd & 0xff; + xb = (unsigned)rd >> 8; + while (xn) { + unsigned try = exceptions[xb+xn/2][0]; + if (try == c) { + r = rules[exceptions[xb+xn/2][1]]; + rt = r & 255; + rd = r >> 8; + if (rt < 2) return c0 + (rd & -(rt^dir)); + /* Hard-coded for the four exceptional titlecase */ + return c0 + (dir ? -1 : 1); + } else if (try > c) { + xn /= 2; + } else { + xb += xn/2; + xn -= xn/2; } } - for (i=0; pairs[i][1-lower]; i++) { - if (pairs[i][1-lower] == wc) - return pairs[i][lower]; - } - if ((unsigned)wc - (0x10428 - 0x28*lower) < 0x28) - return wc - 0x28 + 0x50*lower; - if ((unsigned)wc - (0x104d8 - 0x28*lower) < 0x24) - return wc - 0x28 + 0x50*lower; - if ((unsigned)wc - (0x10cc0 - 0x40*lower) < 0x33) - return wc - 0x40 + 0x80*lower; - if ((unsigned)wc - (0x118c0 - 0x20*lower) < 0x20) - return wc - 0x20 + 0x40*lower; - if ((unsigned)wc - (0x1e922 - 0x22*lower) < 0x22) - return wc - 0x22 + 0x44*lower; - return wc; + return c0; } -wint_t towupper(wint_t wc) +wint_t towlower(wint_t wc) { - return (unsigned)wc < 128 ? toupper(wc) : __towcase(wc, 0); + return casemap(wc, 0); } -wint_t towlower(wint_t wc) +wint_t towupper(wint_t wc) { - return (unsigned)wc < 128 ? tolower(wc) : __towcase(wc, 1); + return casemap(wc, 1); } wint_t __towupper_l(wint_t c, locale_t l) diff --git a/src/ctype/wcwidth.c b/src/ctype/wcwidth.c index 49c40eea..36256a53 100644 --- a/src/ctype/wcwidth.c +++ b/src/ctype/wcwidth.c @@ -23,7 +23,7 @@ int wcwidth(wchar_t wc) return -1; if (wc-0x20000U < 0x20000) return 2; - if (wc == 0xe0001 || wc-0xe0020U < 0x5f || wc-0xe0100 < 0xef) + if (wc == 0xe0001 || wc-0xe0020U < 0x5f || wc-0xe0100U < 0xef) return 0; return 1; } diff --git a/src/ctype/wide.h b/src/ctype/wide.h index e4672b23..e403c9a5 100644 --- a/src/ctype/wide.h +++ b/src/ctype/wide.h @@ -17,7 +17,7 @@ 16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,17,38,39,16,16, 16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16, 16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16, -16,16,16,16,16,16,16,40,41,42,43,44,45,46,16,16,47,16,16,16,16,16, +16,16,16,16,16,16,16,40,41,42,43,44,45,46,47,16,48,49,16,16,16,16, 16,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,255,255, 255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255, 255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255, @@ -31,10 +31,10 @@ 255,255,255,15,0,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255, 255,255,255,255,255,255,255,255,255,255,255,63,0,0,0,255,15,255,255,255,255, 255,255,255,127,254,255,255,255,255,255,255,255,255,255,127,254,255,255,255, -255,255,255,255,255,255,255,255,255,224,255,255,255,255,127,254,255,255,255, +255,255,255,255,255,255,255,255,255,224,255,255,255,255,255,254,255,255,255, 255,255,255,255,255,255,255,127,255,255,255,255,255,7,255,255,255,255,15,0, 255,255,255,255,255,127,255,255,255,255,255,0,255,255,255,255,255,255,255,255, -255,255,255,255,255,255,255,255,255,255,255,255,255,127,255,255,255,255,255, +255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255, 255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,0, 0,0,0,0,0,0,0,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255, 255,31,255,255,255,255,255,255,127,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,255, @@ -43,13 +43,13 @@ 255,15,0,0,0,0,0,0,0,0,0,0,0,0,0,255,3,0,0,255,255,255,255,247,255,127,15,0,0, 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,254,255,255,255,255,255,255,255,255,255,255, 255,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,127,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, -0,0,0,0,0,0,0,0,0,0,0,0,3,0,0,0,255,255,255,255,255,255,255,255,255,255,255, -255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,31,0, -0,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255, -255,255,255,255,255,255,255,255,255,255,255,7,0,255,255,255,127,0,0,0,0,0,0,0, -0,0,0,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255, +0,0,0,0,0,0,0,0,0,0,0,0,15,0,0,0,255,255,255,255,255,255,255,255,255,255,255, +255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255, +255,0,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255, +255,255,255,255,255,255,255,255,255,255,255,255,7,0,255,255,255,127,0,0,0,0,0, +0,7,0,240,0,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255, 255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255, -255,255,255,255,255,255,255,255,255,255,255,255, +255,255,255,255,255,255,255,255,255,255,255,255,255,255, 15,16,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,128,0,0,0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,0,0,0,0,0,64,254,7,0,0,0,0,0,0,0,0,0,0,0,0,7,0,255,255,255, 255,255,15,255,1,3,0,63,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,255,255,255,255, @@ -58,6 +58,8 @@ 255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255, 159,255,255,255,255,255,255,255,63,0,120,255,255,255,0,0,4,0,0,96,0,16,0,0,0, 0,0,0,0,0,0,0,248,255,255,255,255,255,255,255,255,255,255,0,0,0,0,0,0,255,255, -255,255,255,255,255,255,63,16,7,0,0,24,240,1,0,0,255,255,255,255,255,127,255, -31,255,255,255,15,0,0,255,255,255,0,0,0,0,0,1,0,255,255,127,0,0, -0, +255,255,255,255,255,255,63,16,39,0,0,24,240,7,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, +0,0,0,0,0,0,0,0,0,0,0,0,255,15,0, +0,0,224,255,255,255,255,255,255,255,255,255,255,255,255,123,252,255,255,255, +255,231,199,255,255,255,231,255,255,255,255,255,255,0,0,0,0,0,0,0,0,0,0,0,0,0, +0,15,7,7,0,63,0,0,0,0,0,0,0,0,0,0,0,0,0, diff --git a/src/dirent/alphasort.c b/src/dirent/alphasort.c index bee672eb..ab2624e2 100644 --- a/src/dirent/alphasort.c +++ b/src/dirent/alphasort.c @@ -5,5 +5,3 @@ int alphasort(const struct dirent **a, const struct dirent **b) { return strcoll((*a)->d_name, (*b)->d_name); } - -weak_alias(alphasort, alphasort64); diff --git a/src/dirent/readdir.c b/src/dirent/readdir.c index 569fc705..5a03b363 100644 --- a/src/dirent/readdir.c +++ b/src/dirent/readdir.c @@ -25,5 +25,3 @@ struct dirent *readdir(DIR *dir) dir->tell = de->d_off; return de; } - -weak_alias(readdir, readdir64); diff --git a/src/dirent/readdir_r.c b/src/dirent/readdir_r.c index e2a818f3..0d5de5f5 100644 --- a/src/dirent/readdir_r.c +++ b/src/dirent/readdir_r.c @@ -25,5 +25,3 @@ int readdir_r(DIR *restrict dir, struct dirent *restrict buf, struct dirent **re *result = buf; return 0; } - -weak_alias(readdir_r, readdir64_r); diff --git a/src/dirent/scandir.c b/src/dirent/scandir.c index 7ee195dd..7456b9b8 100644 --- a/src/dirent/scandir.c +++ b/src/dirent/scandir.c @@ -43,5 +43,3 @@ int scandir(const char *path, struct dirent ***res, *res = names; return cnt; } - -weak_alias(scandir, scandir64); diff --git a/src/dirent/versionsort.c b/src/dirent/versionsort.c index d4c48923..97696105 100644 --- a/src/dirent/versionsort.c +++ b/src/dirent/versionsort.c @@ -6,6 +6,3 @@ int versionsort(const struct dirent **a, const struct dirent **b) { return strverscmp((*a)->d_name, (*b)->d_name); } - -#undef versionsort64 -weak_alias(versionsort, versionsort64); diff --git a/src/env/__init_tls.c b/src/env/__init_tls.c index 772baba3..a93141ed 100644 --- a/src/env/__init_tls.c +++ b/src/env/__init_tls.c @@ -67,7 +67,7 @@ void *__copy_tls(unsigned char *mem) } #endif dtv[0] = libc.tls_cnt; - td->dtv = td->dtv_copy = dtv; + td->dtv = dtv; return td; } diff --git a/src/env/__libc_start_main.c b/src/env/__libc_start_main.c index 8fbe5262..c5b277bd 100644 --- a/src/env/__libc_start_main.c +++ b/src/env/__libc_start_main.c @@ -69,7 +69,8 @@ weak_alias(libc_start_init, __libc_start_init); typedef int lsm2_fn(int (*)(int,char **,char **), int, char **); static lsm2_fn libc_start_main_stage2; -int __libc_start_main(int (*main)(int,char **,char **), int argc, char **argv) +int __libc_start_main(int (*main)(int,char **,char **), int argc, char **argv, + void (*init_dummy)(), void(*fini_dummy)(), void(*ldso_dummy)()) { char **envp = argv+argc+1; diff --git a/src/env/__stack_chk_fail.c b/src/env/__stack_chk_fail.c index e32596d1..e5352602 100644 --- a/src/env/__stack_chk_fail.c +++ b/src/env/__stack_chk_fail.c @@ -9,7 +9,16 @@ void __init_ssp(void *entropy) if (entropy) memcpy(&__stack_chk_guard, entropy, sizeof(uintptr_t)); else __stack_chk_guard = (uintptr_t)&__stack_chk_guard * 1103515245; - __pthread_self()->CANARY = __stack_chk_guard; +#if UINTPTR_MAX >= 0xffffffffffffffff + /* Sacrifice 8 bits of entropy on 64bit to prevent leaking/ + * overwriting the canary via string-manipulation functions. + * The NULL byte is on the second byte so that off-by-ones can + * still be detected. Endianness is taken care of + * automatically. */ + ((char *)&__stack_chk_guard)[1] = 0; +#endif + + __pthread_self()->canary = __stack_chk_guard; } void __stack_chk_fail(void) diff --git a/src/errno/__strerror.h b/src/errno/__strerror.h index 2f04d400..14925907 100644 --- a/src/errno/__strerror.h +++ b/src/errno/__strerror.h @@ -1,8 +1,9 @@ -/* This file is sorted such that 'errors' which represent exceptional - * conditions under which a correct program may fail come first, followed - * by messages that indicate an incorrect program or system failure. The - * macro E() along with double-inclusion is used to ensure that ordering - * of the strings remains synchronized. */ +/* The first entry is a catch-all for codes not enumerated here. + * This file is included multiple times to declare and define a structure + * with these messages, and then to define a lookup table translating + * error codes to offsets of corresponding fields in the structure. */ + +E(0, "No error information") E(EILSEQ, "Illegal byte sequence") E(EDOM, "Domain error") @@ -101,5 +102,7 @@ E(EDQUOT, "Quota exceeded") E(ENOMEDIUM, "No medium found") E(EMEDIUMTYPE, "Wrong medium type") E(EMULTIHOP, "Multihop attempted") - -E(0, "No error information") +E(ENOKEY, "Required key not available") +E(EKEYEXPIRED, "Key has expired") +E(EKEYREVOKED, "Key has been revoked") +E(EKEYREJECTED, "Key was rejected by service") diff --git a/src/errno/strerror.c b/src/errno/strerror.c index e3ed771a..7f926432 100644 --- a/src/errno/strerror.c +++ b/src/errno/strerror.c @@ -1,30 +1,41 @@ #include <errno.h> +#include <stddef.h> #include <string.h> #include "locale_impl.h" -#define E(a,b) ((unsigned char)a), -static const unsigned char errid[] = { +/* mips has one error code outside of the 8-bit range due to a + * historical typo, so we just remap it. */ +#if EDQUOT==1133 +#define EDQUOT_ORIG 1133 +#undef EDQUOT +#define EDQUOT 109 +#endif + +static const struct errmsgstr_t { +#define E(n, s) char str##n[sizeof(s)]; +#include "__strerror.h" +#undef E +} errmsgstr = { +#define E(n, s) s, #include "__strerror.h" +#undef E }; -#undef E -#define E(a,b) b "\0" -static const char errmsg[] = +static const unsigned short errmsgidx[] = { +#define E(n, s) [n] = offsetof(struct errmsgstr_t, str##n), #include "__strerror.h" -; +#undef E +}; char *__strerror_l(int e, locale_t loc) { const char *s; - int i; - /* mips has one error code outside of the 8-bit range due to a - * historical typo, so we just remap it. */ - if (EDQUOT==1133) { - if (e==109) e=-1; - else if (e==EDQUOT) e=109; - } - for (i=0; errid[i] && errid[i] != e; i++); - for (s=errmsg; i; s++, i--) for (; *s; s++); +#ifdef EDQUOT_ORIG + if (e==EDQUOT) e=0; + else if (e==EDQUOT_ORIG) e=EDQUOT; +#endif + if (e >= sizeof errmsgidx / sizeof *errmsgidx) e = 0; + s = (char *)&errmsgstr + errmsgidx[e]; return (char *)LCTRANS(s, LC_MESSAGES, loc); } diff --git a/src/exit/abort.c b/src/exit/abort.c index e1980f10..f21f458e 100644 --- a/src/exit/abort.c +++ b/src/exit/abort.c @@ -6,8 +6,6 @@ #include "lock.h" #include "ksigaction.h" -hidden volatile int __abort_lock[1]; - _Noreturn void abort(void) { raise(SIGABRT); diff --git a/src/exit/abort_lock.c b/src/exit/abort_lock.c new file mode 100644 index 00000000..3af72c7b --- /dev/null +++ b/src/exit/abort_lock.c @@ -0,0 +1,3 @@ +#include "pthread_impl.h" + +volatile int __abort_lock[1]; diff --git a/src/exit/assert.c b/src/exit/assert.c index 49b0dc3e..94edd827 100644 --- a/src/exit/assert.c +++ b/src/exit/assert.c @@ -4,6 +4,5 @@ _Noreturn void __assert_fail(const char *expr, const char *file, int line, const char *func) { fprintf(stderr, "Assertion failed: %s (%s: %s: %d)\n", expr, file, func, line); - fflush(NULL); abort(); } diff --git a/src/exit/at_quick_exit.c b/src/exit/at_quick_exit.c index d3ce6522..e4b5d78d 100644 --- a/src/exit/at_quick_exit.c +++ b/src/exit/at_quick_exit.c @@ -1,12 +1,14 @@ #include <stdlib.h> #include "libc.h" #include "lock.h" +#include "fork_impl.h" #define COUNT 32 static void (*funcs[COUNT])(void); static int count; static volatile int lock[1]; +volatile int *const __at_quick_exit_lockptr = lock; void __funcs_on_quick_exit() { diff --git a/src/exit/atexit.c b/src/exit/atexit.c index 160d277a..854e9fdd 100644 --- a/src/exit/atexit.c +++ b/src/exit/atexit.c @@ -2,6 +2,12 @@ #include <stdint.h> #include "libc.h" #include "lock.h" +#include "fork_impl.h" + +#define malloc __libc_malloc +#define calloc __libc_calloc +#define realloc undef +#define free undef /* Ensure that at least 32 atexit handlers can be registered without malloc */ #define COUNT 32 @@ -15,6 +21,7 @@ static struct fl static int slot; static volatile int lock[1]; +volatile int *const __atexit_lockptr = lock; void __funcs_on_exit() { diff --git a/src/fcntl/creat.c b/src/fcntl/creat.c index 8f8aab64..c9c43910 100644 --- a/src/fcntl/creat.c +++ b/src/fcntl/creat.c @@ -4,5 +4,3 @@ int creat(const char *filename, mode_t mode) { return open(filename, O_CREAT|O_WRONLY|O_TRUNC, mode); } - -weak_alias(creat, creat64); diff --git a/src/fcntl/open.c b/src/fcntl/open.c index 1d817a2d..4c3c8275 100644 --- a/src/fcntl/open.c +++ b/src/fcntl/open.c @@ -19,5 +19,3 @@ int open(const char *filename, int flags, ...) return __syscall_ret(fd); } - -weak_alias(open, open64); diff --git a/src/fcntl/openat.c b/src/fcntl/openat.c index ad165ec3..83a9e0d0 100644 --- a/src/fcntl/openat.c +++ b/src/fcntl/openat.c @@ -15,5 +15,3 @@ int openat(int fd, const char *filename, int flags, ...) return syscall_cp(SYS_openat, fd, filename, flags|O_LARGEFILE, mode); } - -weak_alias(openat, openat64); diff --git a/src/fcntl/posix_fadvise.c b/src/fcntl/posix_fadvise.c index 75b8e1ae..07346d21 100644 --- a/src/fcntl/posix_fadvise.c +++ b/src/fcntl/posix_fadvise.c @@ -14,5 +14,3 @@ int posix_fadvise(int fd, off_t base, off_t len, int advice) __SYSCALL_LL_E(len), advice); #endif } - -weak_alias(posix_fadvise, posix_fadvise64); diff --git a/src/fcntl/posix_fallocate.c b/src/fcntl/posix_fallocate.c index c57a24ae..80a65cbf 100644 --- a/src/fcntl/posix_fallocate.c +++ b/src/fcntl/posix_fallocate.c @@ -6,5 +6,3 @@ int posix_fallocate(int fd, off_t base, off_t len) return -__syscall(SYS_fallocate, fd, 0, __SYSCALL_LL_E(base), __SYSCALL_LL_E(len)); } - -weak_alias(posix_fallocate, posix_fallocate64); diff --git a/src/fenv/loongarch64/fenv.S b/src/fenv/loongarch64/fenv.S new file mode 100644 index 00000000..9c38599e --- /dev/null +++ b/src/fenv/loongarch64/fenv.S @@ -0,0 +1,78 @@ +#ifndef __loongarch_soft_float + +#ifdef BROKEN_LOONGARCH_FCSR_ASM +#define FCSR $r0 +#else +#define FCSR $fcsr0 +#endif + +.global feclearexcept +.type feclearexcept,@function +feclearexcept: + li.w $t0, 0x1f0000 + and $a0, $a0, $t0 + movfcsr2gr $t1, FCSR + andn $t1, $t1, $a0 + movgr2fcsr FCSR, $t1 + li.w $a0, 0 + jr $ra + +.global feraiseexcept +.type feraiseexcept,@function +feraiseexcept: + li.w $t0, 0x1f0000 + and $a0, $a0, $t0 + movfcsr2gr $t1, FCSR + or $t1, $t1, $a0 + movgr2fcsr FCSR, $t1 + li.w $a0, 0 + jr $ra + +.global fetestexcept +.type fetestexcept,@function +fetestexcept: + li.w $t0, 0x1f0000 + and $a0, $a0, $t0 + movfcsr2gr $t1, FCSR + and $a0, $t1, $a0 + jr $ra + +.global fegetround +.type fegetround,@function +fegetround: + movfcsr2gr $t0, FCSR + andi $a0, $t0, 0x300 + jr $ra + +.global __fesetround +.hidden __fesetround +.type __fesetround,@function +__fesetround: + li.w $t0, 0x300 + and $a0, $a0, $t0 + movfcsr2gr $t1, FCSR + andn $t1, $t1, $t0 + or $t1, $t1, $a0 + movgr2fcsr FCSR, $t1 + li.w $a0, 0 + jr $ra + +.global fegetenv +.type fegetenv,@function +fegetenv: + movfcsr2gr $t0, FCSR + st.w $t0, $a0, 0 + li.w $a0, 0 + jr $ra + +.global fesetenv +.type fesetenv,@function +fesetenv: + addi.d $t0, $a0, 1 + beq $t0, $r0, 1f + ld.w $t0, $a0, 0 +1: movgr2fcsr FCSR, $t0 + li.w $a0, 0 + jr $ra + +#endif diff --git a/src/fenv/powerpc/fenv-sf.c b/src/fenv/powerpc/fenv-sf.c index 85bef40f..d4248f26 100644 --- a/src/fenv/powerpc/fenv-sf.c +++ b/src/fenv/powerpc/fenv-sf.c @@ -1,3 +1,3 @@ -#ifdef _SOFT_FLOAT +#if defined(_SOFT_FLOAT) || defined(__NO_FPRS__) #include "../fenv.c" #endif diff --git a/src/fenv/powerpc/fenv.S b/src/fenv/powerpc/fenv.S index 22cea216..55055d0b 100644 --- a/src/fenv/powerpc/fenv.S +++ b/src/fenv/powerpc/fenv.S @@ -1,4 +1,4 @@ -#ifndef _SOFT_FLOAT +#if !defined(_SOFT_FLOAT) && !defined(__NO_FPRS__) .global feclearexcept .type feclearexcept,@function feclearexcept: diff --git a/src/fenv/riscv32/fenv-sf.c b/src/fenv/riscv32/fenv-sf.c new file mode 100644 index 00000000..ecd3cb5c --- /dev/null +++ b/src/fenv/riscv32/fenv-sf.c @@ -0,0 +1,3 @@ +#ifndef __riscv_flen +#include "../fenv.c" +#endif diff --git a/src/fenv/riscv32/fenv.S b/src/fenv/riscv32/fenv.S new file mode 100644 index 00000000..0ea78bf9 --- /dev/null +++ b/src/fenv/riscv32/fenv.S @@ -0,0 +1,56 @@ +#ifdef __riscv_flen + +.global feclearexcept +.type feclearexcept, %function +feclearexcept: + csrc fflags, a0 + li a0, 0 + ret + +.global feraiseexcept +.type feraiseexcept, %function +feraiseexcept: + csrs fflags, a0 + li a0, 0 + ret + +.global fetestexcept +.type fetestexcept, %function +fetestexcept: + frflags t0 + and a0, t0, a0 + ret + +.global fegetround +.type fegetround, %function +fegetround: + frrm a0 + ret + +.global __fesetround +.type __fesetround, %function +__fesetround: + fsrm t0, a0 + li a0, 0 + ret + +.global fegetenv +.type fegetenv, %function +fegetenv: + frcsr t0 + sw t0, 0(a0) + li a0, 0 + ret + +.global fesetenv +.type fesetenv, %function +fesetenv: + li t2, -1 + li t1, 0 + beq a0, t2, 1f + lw t1, 0(a0) +1: fscsr t1 + li a0, 0 + ret + +#endif diff --git a/src/fenv/riscv64/fenv.S b/src/fenv/riscv64/fenv.S index f149003d..0ea78bf9 100644 --- a/src/fenv/riscv64/fenv.S +++ b/src/fenv/riscv64/fenv.S @@ -45,8 +45,11 @@ fegetenv: .global fesetenv .type fesetenv, %function fesetenv: + li t2, -1 + li t1, 0 + beq a0, t2, 1f lw t1, 0(a0) - fscsr t0, t1 +1: fscsr t1 li a0, 0 ret diff --git a/src/fenv/sh/fenv.S b/src/fenv/sh/fenv.S index 907aefc0..b3b7d66a 100644 --- a/src/fenv/sh/fenv.S +++ b/src/fenv/sh/fenv.S @@ -12,6 +12,8 @@ fegetround: .type __fesetround, @function __fesetround: sts fpscr, r0 + mov #-4, r1 + and r1, r0 or r4, r0 lds r0, fpscr rts diff --git a/src/include/stdlib.h b/src/include/stdlib.h index d38a5417..812b04de 100644 --- a/src/include/stdlib.h +++ b/src/include/stdlib.h @@ -8,5 +8,12 @@ hidden void __env_rm_add(char *, char *); hidden int __mkostemps(char *, int, int); hidden int __ptsname_r(int, char *, size_t); hidden char *__randname(char *); +hidden void __qsort_r (void *, size_t, size_t, int (*)(const void *, const void *, void *), void *); + +hidden void *__libc_malloc(size_t); +hidden void *__libc_malloc_impl(size_t); +hidden void *__libc_calloc(size_t, size_t); +hidden void *__libc_realloc(void *, size_t); +hidden void __libc_free(void *); #endif diff --git a/src/include/sys/stat.h b/src/include/sys/stat.h new file mode 100644 index 00000000..59339bee --- /dev/null +++ b/src/include/sys/stat.h @@ -0,0 +1,9 @@ +#ifndef SYS_STAT_H +#define SYS_STAT_H + +#include "../../../include/sys/stat.h" + +hidden int __fstat(int, struct stat *); +hidden int __fstatat(int, const char *restrict, struct stat *restrict, int); + +#endif diff --git a/src/include/unistd.h b/src/include/unistd.h index 1b4605c7..7b52a924 100644 --- a/src/include/unistd.h +++ b/src/include/unistd.h @@ -8,7 +8,6 @@ extern char **__environ; hidden int __dup3(int, int, int); hidden int __mkostemps(char *, int, int); hidden int __execvpe(const char *, char *const *, char *const *); -hidden int __aio_close(int); hidden off_t __lseek(int, off_t, int); #endif diff --git a/src/internal/aio_impl.h b/src/internal/aio_impl.h new file mode 100644 index 00000000..a8657665 --- /dev/null +++ b/src/internal/aio_impl.h @@ -0,0 +1,9 @@ +#ifndef AIO_IMPL_H +#define AIO_IMPL_H + +extern hidden volatile int __aio_fut; + +extern hidden int __aio_close(int); +extern hidden void __aio_atfork(int); + +#endif diff --git a/src/internal/atomic.h b/src/internal/atomic.h index f938879b..8f71c8cd 100644 --- a/src/internal/atomic.h +++ b/src/internal/atomic.h @@ -194,7 +194,7 @@ static inline void a_store(volatile int *p, int v) #ifndef a_barrier #define a_barrier a_barrier -static void a_barrier() +static inline void a_barrier() { volatile int tmp = 0; a_cas(&tmp, 0, 0); @@ -315,4 +315,19 @@ static inline int a_clz_64(uint64_t x) } #endif +#ifndef a_clz_32 +#define a_clz_32 a_clz_32 +static inline int a_clz_32(uint32_t x) +{ + x >>= 1; + x |= x >> 1; + x |= x >> 2; + x |= x >> 4; + x |= x >> 8; + x |= x >> 16; + x++; + return 31-a_ctz_32(x); +} +#endif + #endif diff --git a/src/internal/dynlink.h b/src/internal/dynlink.h index ffd06b04..40c743e2 100644 --- a/src/internal/dynlink.h +++ b/src/internal/dynlink.h @@ -73,6 +73,10 @@ struct fdpic_dummy_loadmap { #define DL_NOMMU_SUPPORT 0 #endif +#ifndef TLSDESC_BACKWARDS +#define TLSDESC_BACKWARDS 0 +#endif + #if !DL_FDPIC #define IS_RELATIVE(x,s) ( \ (R_TYPE(x) == REL_RELATIVE) || \ @@ -92,11 +96,14 @@ struct fdpic_dummy_loadmap { #define DT_DEBUG_INDIRECT 0 #endif +#ifndef DT_DEBUG_INDIRECT_REL +#define DT_DEBUG_INDIRECT_REL 0 +#endif + #define AUX_CNT 32 -#define DYN_CNT 32 +#define DYN_CNT 37 typedef void (*stage2_func)(unsigned char *, size_t *); -typedef void (*stage3_func)(size_t *); hidden void *__dlsym(void *restrict, const char *restrict, void *restrict); @@ -106,4 +113,9 @@ hidden void __dl_vseterr(const char *, va_list); hidden ptrdiff_t __tlsdesc_static(), __tlsdesc_dynamic(); +hidden extern int __malloc_replaced; +hidden extern int __aligned_alloc_replaced; +hidden void __malloc_donate(char *, char *); +hidden int __malloc_allzerop(void *); + #endif diff --git a/src/internal/emulate_wait4.c b/src/internal/emulate_wait4.c new file mode 100644 index 00000000..f6303412 --- /dev/null +++ b/src/internal/emulate_wait4.c @@ -0,0 +1,55 @@ +#include <sys/wait.h> +#include "syscall.h" + +#ifndef SYS_wait4 +hidden long __emulate_wait4(int pid, int *status, int options, void *kru, int cp) +{ + idtype_t t; + int r; + siginfo_t info; + + info.si_pid = 0; + if (pid < -1) { + t = P_PGID; + pid = -pid; + } else if (pid == -1) { + t = P_ALL; + } else if (pid == 0) { + t = P_PGID; + } else { + t = P_PID; + } + + if (cp) r = __syscall_cp(SYS_waitid, t, pid, &info, options|WEXITED, kru); + else r = __syscall(SYS_waitid, t, pid, &info, options|WEXITED, kru); + + if (r<0) return r; + + if (info.si_pid && status) { + int sw=0; + switch (info.si_code) { + case CLD_CONTINUED: + sw = 0xffff; + break; + case CLD_DUMPED: + sw = info.si_status&0x7f | 0x80; + break; + case CLD_EXITED: + sw = (info.si_status&0xff) << 8; + break; + case CLD_KILLED: + sw = info.si_status&0x7f; + break; + case CLD_STOPPED: + case CLD_TRAPPED: + /* see ptrace(2); the high bits of si_status can contain */ + /* PTRACE_EVENT_ values which must be preserved */ + sw = (info.si_status << 8) + 0x7f; + break; + } + *status = sw; + } + + return info.si_pid; +} +#endif diff --git a/src/internal/fork_impl.h b/src/internal/fork_impl.h new file mode 100644 index 00000000..f995fce2 --- /dev/null +++ b/src/internal/fork_impl.h @@ -0,0 +1,21 @@ +#include <features.h> + +extern hidden volatile int *const __at_quick_exit_lockptr; +extern hidden volatile int *const __atexit_lockptr; +extern hidden volatile int *const __gettext_lockptr; +extern hidden volatile int *const __locale_lockptr; +extern hidden volatile int *const __random_lockptr; +extern hidden volatile int *const __sem_open_lockptr; +extern hidden volatile int *const __stdio_ofl_lockptr; +extern hidden volatile int *const __syslog_lockptr; +extern hidden volatile int *const __timezone_lockptr; + +extern hidden volatile int *const __bump_lockptr; + +extern hidden volatile int *const __vmlock_lockptr; + +hidden void __malloc_atfork(int); +hidden void __ldso_atfork(int); +hidden void __pthread_key_atfork(int); + +hidden void __post_Fork(int); diff --git a/src/internal/ksigaction.h b/src/internal/ksigaction.h index 8ebd5938..ef333f33 100644 --- a/src/internal/ksigaction.h +++ b/src/internal/ksigaction.h @@ -6,8 +6,13 @@ struct k_sigaction { void (*handler)(int); unsigned long flags; +#ifdef SA_RESTORER void (*restorer)(void); +#endif unsigned mask[2]; +#ifndef SA_RESTORER + void *unused; +#endif }; hidden void __restore(), __restore_rt(); diff --git a/src/internal/libc.h b/src/internal/libc.h index ac97dc7e..619bba86 100644 --- a/src/internal/libc.h +++ b/src/internal/libc.h @@ -18,10 +18,11 @@ struct tls_module { }; struct __libc { - int can_do_threads; - int threaded; - int secure; - volatile int threads_minus_1; + char can_do_threads; + char threaded; + char secure; + volatile signed char need_locks; + int threads_minus_1; size_t *auxv; struct tls_module *tls_head; size_t tls_size, tls_align, tls_cnt; diff --git a/src/internal/libm.h b/src/internal/libm.h index b5bd26b8..72ad17d8 100644 --- a/src/internal/libm.h +++ b/src/internal/libm.h @@ -236,13 +236,13 @@ hidden int __rem_pio2(double,double*); hidden double __sin(double,double,int); hidden double __cos(double,double); hidden double __tan(double,double,int); -hidden double __expo2(double); +hidden double __expo2(double,double); hidden int __rem_pio2f(float,double*); hidden float __sindf(double); hidden float __cosdf(double); hidden float __tandf(double,int); -hidden float __expo2f(float); +hidden float __expo2f(float,float); hidden int __rem_pio2l(long double, long double *); hidden long double __sinl(long double, long double, int); @@ -267,5 +267,8 @@ hidden double __math_uflow(uint32_t); hidden double __math_oflow(uint32_t); hidden double __math_divzero(uint32_t); hidden double __math_invalid(double); +#if LDBL_MANT_DIG != DBL_MANT_DIG +hidden long double __math_invalidl(long double); +#endif #endif diff --git a/src/internal/locale_impl.h b/src/internal/locale_impl.h index 741a71c4..4431a92e 100644 --- a/src/internal/locale_impl.h +++ b/src/internal/locale_impl.h @@ -15,6 +15,8 @@ struct __locale_map { const struct __locale_map *next; }; +extern hidden volatile int __locale_lock[1]; + extern hidden const struct __locale_map __c_dot_utf8; extern hidden const struct __locale_struct __c_locale; extern hidden const struct __locale_struct __c_dot_utf8_locale; diff --git a/src/internal/pthread_impl.h b/src/internal/pthread_impl.h index 5742dfc5..de2b9d8b 100644 --- a/src/internal/pthread_impl.h +++ b/src/internal/pthread_impl.h @@ -11,16 +11,25 @@ #include "atomic.h" #include "futex.h" +#include "pthread_arch.h" + #define pthread __pthread struct pthread { /* Part 1 -- these fields may be external or * internal (accessed via asm) ABI. Do not change. */ struct pthread *self; +#ifndef TLS_ABOVE_TP uintptr_t *dtv; +#endif struct pthread *prev, *next; /* non-ABI */ uintptr_t sysinfo; - uintptr_t canary, canary2; +#ifndef TLS_ABOVE_TP +#ifdef CANARY_PAD + uintptr_t canary_pad; +#endif + uintptr_t canary; +#endif /* Part 2 -- implementation details, non-ABI. */ int tid; @@ -43,6 +52,7 @@ struct pthread { long off; volatile void *volatile pending; } robust_list; + int h_errno_val; volatile int timer_id; locale_t locale; volatile int killlock[1]; @@ -51,21 +61,19 @@ struct pthread { /* Part 3 -- the positions of these fields relative to * the end of the structure is external and internal ABI. */ - uintptr_t canary_at_end; - uintptr_t *dtv_copy; +#ifdef TLS_ABOVE_TP + uintptr_t canary; + uintptr_t *dtv; +#endif }; enum { - DT_EXITING = 0, + DT_EXITED = 0, + DT_EXITING, DT_JOINABLE, DT_DETACHED, }; -struct __timer { - int timerid; - pthread_t thread; -}; - #define __SU (sizeof(size_t)/sizeof(int)) #define _a_stacksize __u.__s[0] @@ -98,16 +106,22 @@ struct __timer { #define _b_waiters2 __u.__vi[4] #define _b_inst __u.__p[3] -#include "pthread_arch.h" - -#ifndef CANARY -#define CANARY canary +#ifndef TP_OFFSET +#define TP_OFFSET 0 #endif #ifndef DTP_OFFSET #define DTP_OFFSET 0 #endif +#ifdef TLS_ABOVE_TP +#define TP_ADJ(p) ((char *)(p) + sizeof(struct pthread) + TP_OFFSET) +#define __pthread_self() ((pthread_t)(__get_tp() - sizeof(struct __pthread) - TP_OFFSET)) +#else +#define TP_ADJ(p) (p) +#define __pthread_self() ((pthread_t)__get_tp()) +#endif + #ifndef tls_mod_off_t #define tls_mod_off_t size_t #endif @@ -141,7 +155,6 @@ hidden int __pthread_key_delete_impl(pthread_key_t); extern hidden volatile size_t __pthread_tsd_size; extern hidden void *__pthread_tsd_main[]; -extern hidden volatile int __aio_fut; extern hidden volatile int __eintr_valid_flag; hidden int __clone(int (*)(void *), void *, int, void *, ...); @@ -176,6 +189,8 @@ hidden void __tl_sync(pthread_t); extern hidden volatile int __thread_list_lock; +extern hidden volatile int __abort_lock[1]; + extern hidden unsigned __default_stacksize; extern hidden unsigned __default_guardsize; diff --git a/src/internal/shgetc.c b/src/internal/shgetc.c index a4a9c633..7455d2f0 100644 --- a/src/internal/shgetc.c +++ b/src/internal/shgetc.c @@ -32,6 +32,6 @@ int __shgetc(FILE *f) else f->shend = f->rend; f->shcnt = f->buf - f->rpos + cnt; - if (f->rpos[-1] != c) f->rpos[-1] = c; + if (f->rpos <= f->buf) f->rpos[-1] = c; return c; } diff --git a/src/internal/stdio_impl.h b/src/internal/stdio_impl.h index d7398f59..0b2438d6 100644 --- a/src/internal/stdio_impl.h +++ b/src/internal/stdio_impl.h @@ -60,8 +60,6 @@ hidden size_t __stdout_write(FILE *, const unsigned char *, size_t); hidden off_t __stdio_seek(FILE *, off_t, int); hidden int __stdio_close(FILE *); -hidden size_t __string_read(FILE *, unsigned char *, size_t); - hidden int __toread(FILE *); hidden int __towrite(FILE *); diff --git a/src/internal/syscall.h b/src/internal/syscall.h index 9f2784db..33d981f9 100644 --- a/src/internal/syscall.h +++ b/src/internal/syscall.h @@ -2,6 +2,7 @@ #define _INTERNAL_SYSCALL_H #include <features.h> +#include <errno.h> #include <sys/syscall.h> #include "syscall_arch.h" @@ -57,15 +58,22 @@ hidden long __syscall_ret(unsigned long), #define __syscall_cp(...) __SYSCALL_DISP(__syscall_cp,__VA_ARGS__) #define syscall_cp(...) __syscall_ret(__syscall_cp(__VA_ARGS__)) -#ifndef SYSCALL_USE_SOCKETCALL -#define __socketcall(nm,a,b,c,d,e,f) __syscall(SYS_##nm, a, b, c, d, e, f) -#define __socketcall_cp(nm,a,b,c,d,e,f) __syscall_cp(SYS_##nm, a, b, c, d, e, f) -#else -#define __socketcall(nm,a,b,c,d,e,f) __syscall(SYS_socketcall, __SC_##nm, \ - ((long [6]){ (long)a, (long)b, (long)c, (long)d, (long)e, (long)f })) -#define __socketcall_cp(nm,a,b,c,d,e,f) __syscall_cp(SYS_socketcall, __SC_##nm, \ - ((long [6]){ (long)a, (long)b, (long)c, (long)d, (long)e, (long)f })) -#endif +static inline long __alt_socketcall(int sys, int sock, int cp, syscall_arg_t a, syscall_arg_t b, syscall_arg_t c, syscall_arg_t d, syscall_arg_t e, syscall_arg_t f) +{ + long r; + if (cp) r = __syscall_cp(sys, a, b, c, d, e, f); + else r = __syscall(sys, a, b, c, d, e, f); + if (r != -ENOSYS) return r; +#ifdef SYS_socketcall + if (cp) r = __syscall_cp(SYS_socketcall, sock, ((long[6]){a, b, c, d, e, f})); + else r = __syscall(SYS_socketcall, sock, ((long[6]){a, b, c, d, e, f})); +#endif + return r; +} +#define __socketcall(nm, a, b, c, d, e, f) __alt_socketcall(SYS_##nm, __SC_##nm, 0, \ + __scc(a), __scc(b), __scc(c), __scc(d), __scc(e), __scc(f)) +#define __socketcall_cp(nm, a, b, c, d, e, f) __alt_socketcall(SYS_##nm, __SC_##nm, 1, \ + __scc(a), __scc(b), __scc(c), __scc(d), __scc(e), __scc(f)) /* fixup legacy 16-bit junk */ @@ -193,6 +201,45 @@ hidden long __syscall_ret(unsigned long), #define SYS_sendfile SYS_sendfile64 #endif +#ifdef SYS_timer_settime32 +#define SYS_timer_settime SYS_timer_settime32 +#endif + +#ifdef SYS_timer_gettime32 +#define SYS_timer_gettime SYS_timer_gettime32 +#endif + +#ifdef SYS_timerfd_settime32 +#define SYS_timerfd_settime SYS_timerfd_settime32 +#endif + +#ifdef SYS_timerfd_gettime32 +#define SYS_timerfd_gettime SYS_timerfd_gettime32 +#endif + +#ifdef SYS_clock_settime32 +#define SYS_clock_settime SYS_clock_settime32 +#endif + +#ifdef SYS_clock_gettime32 +#define SYS_clock_gettime SYS_clock_gettime32 +#endif + +#ifdef SYS_clock_getres_time32 +#define SYS_clock_getres SYS_clock_getres_time32 +#endif + +#ifdef SYS_clock_nanosleep_time32 +#define SYS_clock_nanosleep SYS_clock_nanosleep_time32 +#endif + +#ifdef SYS_gettimeofday_time32 +#define SYS_gettimeofday SYS_gettimeofday_time32 +#endif + +#ifdef SYS_settimeofday_time32 +#define SYS_settimeofday SYS_settimeofday_time32 +#endif /* Ensure that the plain syscall names are defined even for "time64-only" * archs. These facilitate callers passing null time arguments, and make @@ -299,6 +346,12 @@ hidden long __syscall_ret(unsigned long), #define __SC_recvmmsg 19 #define __SC_sendmmsg 20 +/* This is valid only because all socket syscalls are made via + * socketcall, which always fills unused argument slots with zeros. */ +#ifndef SYS_accept +#define SYS_accept SYS_accept4 +#endif + #ifndef SO_RCVTIMEO_OLD #define SO_RCVTIMEO_OLD 20 #endif @@ -306,6 +359,13 @@ hidden long __syscall_ret(unsigned long), #define SO_SNDTIMEO_OLD 21 #endif +#define SO_TIMESTAMP_OLD 29 +#define SO_TIMESTAMPNS_OLD 35 +#define SO_TIMESTAMPING_OLD 37 +#define SCM_TIMESTAMP_OLD SO_TIMESTAMP_OLD +#define SCM_TIMESTAMPNS_OLD SO_TIMESTAMPNS_OLD +#define SCM_TIMESTAMPING_OLD SO_TIMESTAMPING_OLD + #ifndef SIOCGSTAMP_OLD #define SIOCGSTAMP_OLD 0x8906 #endif @@ -331,6 +391,18 @@ hidden long __syscall_ret(unsigned long), #define __sys_open_cp(...) __SYSCALL_DISP(__sys_open_cp,,__VA_ARGS__) #define sys_open_cp(...) __syscall_ret(__sys_open_cp(__VA_ARGS__)) +#ifdef SYS_wait4 +#define __sys_wait4(a,b,c,d) __syscall(SYS_wait4,a,b,c,d) +#define __sys_wait4_cp(a,b,c,d) __syscall_cp(SYS_wait4,a,b,c,d) +#else +hidden long __emulate_wait4(int, int *, int, void *, int); +#define __sys_wait4(a,b,c,d) __emulate_wait4(a,b,c,d,0) +#define __sys_wait4_cp(a,b,c,d) __emulate_wait4(a,b,c,d,1) +#endif + +#define sys_wait4(a,b,c,d) __syscall_ret(__sys_wait4(a,b,c,d)) +#define sys_wait4_cp(a,b,c,d) __syscall_ret(__sys_wait4_cp(a,b,c,d)) + hidden void __procfdname(char __buf[static 15+3*sizeof(int)], unsigned); hidden void *__vdsosym(const char *, const char *); diff --git a/src/ipc/msgctl.c b/src/ipc/msgctl.c index b043041a..9c114406 100644 --- a/src/ipc/msgctl.c +++ b/src/ipc/msgctl.c @@ -9,6 +9,14 @@ int msgctl(int q, int cmd, struct msqid_ds *buf) { +#if IPC_TIME64 + struct msqid_ds out, *orig; + if (cmd&IPC_TIME64) { + out = (struct msqid_ds){0}; + orig = buf; + buf = &out; + } +#endif #ifdef SYSCALL_IPC_BROKEN_MODE struct msqid_ds tmp; if (cmd == IPC_SET) { @@ -32,6 +40,8 @@ int msgctl(int q, int cmd, struct msqid_ds *buf) #endif #if IPC_TIME64 if (r >= 0 && (cmd&IPC_TIME64)) { + buf = orig; + *buf = out; IPC_HILO(buf, msg_stime); IPC_HILO(buf, msg_rtime); IPC_HILO(buf, msg_ctime); diff --git a/src/ipc/semctl.c b/src/ipc/semctl.c index ed982747..bbb97d7a 100644 --- a/src/ipc/semctl.c +++ b/src/ipc/semctl.c @@ -28,6 +28,14 @@ int semctl(int id, int num, int cmd, ...) arg = va_arg(ap, union semun); va_end(ap); } +#if IPC_TIME64 + struct semid_ds out, *orig; + if (cmd&IPC_TIME64) { + out = (struct semid_ds){0}; + orig = arg.buf; + arg.buf = &out; + } +#endif #ifdef SYSCALL_IPC_BROKEN_MODE struct semid_ds tmp; if (cmd == IPC_SET) { @@ -51,6 +59,8 @@ int semctl(int id, int num, int cmd, ...) #endif #if IPC_TIME64 if (r >= 0 && (cmd&IPC_TIME64)) { + arg.buf = orig; + *arg.buf = out; IPC_HILO(arg.buf, sem_otime); IPC_HILO(arg.buf, sem_ctime); } diff --git a/src/ipc/semtimedop.c b/src/ipc/semtimedop.c index 1632e7b0..a104af21 100644 --- a/src/ipc/semtimedop.c +++ b/src/ipc/semtimedop.c @@ -7,7 +7,8 @@ #define IS32BIT(x) !((x)+0x80000000ULL>>32) #define CLAMP(x) (int)(IS32BIT(x) ? (x) : 0x7fffffffU+((0ULL+(x))>>63)) -#if !defined(SYS_semtimedop) && !defined(SYS_ipc) +#if !defined(SYS_semtimedop) && !defined(SYS_ipc) || \ + SYS_semtimedop == SYS_semtimedop_time64 #define NO_TIME32 1 #else #define NO_TIME32 0 diff --git a/src/ipc/shmctl.c b/src/ipc/shmctl.c index de3ce9d4..1c9f78c2 100644 --- a/src/ipc/shmctl.c +++ b/src/ipc/shmctl.c @@ -9,6 +9,14 @@ int shmctl(int id, int cmd, struct shmid_ds *buf) { +#if IPC_TIME64 + struct shmid_ds out, *orig; + if (cmd&IPC_TIME64) { + out = (struct shmid_ds){0}; + orig = buf; + buf = &out; + } +#endif #ifdef SYSCALL_IPC_BROKEN_MODE struct shmid_ds tmp; if (cmd == IPC_SET) { @@ -32,6 +40,8 @@ int shmctl(int id, int cmd, struct shmid_ds *buf) #endif #if IPC_TIME64 if (r >= 0 && (cmd&IPC_TIME64)) { + buf = orig; + *buf = out; IPC_HILO(buf, shm_atime); IPC_HILO(buf, shm_dtime); IPC_HILO(buf, shm_ctime); diff --git a/src/ldso/__dlsym.c b/src/ldso/__dlsym.c index 8ac0a334..0384f97e 100644 --- a/src/ldso/__dlsym.c +++ b/src/ldso/__dlsym.c @@ -8,3 +8,7 @@ static void *stub_dlsym(void *restrict p, const char *restrict s, void *restrict } weak_alias(stub_dlsym, __dlsym); + +#if _REDIR_TIME64 +weak_alias(stub_dlsym, __dlsym_redir_time64); +#endif diff --git a/src/ldso/arm/dlsym_time64.S b/src/ldso/arm/dlsym_time64.S new file mode 100644 index 00000000..bb2e7040 --- /dev/null +++ b/src/ldso/arm/dlsym_time64.S @@ -0,0 +1,3 @@ +#define __dlsym __dlsym_redir_time64 +#define dlsym __dlsym_time64 +#include "dlsym.s" diff --git a/src/ldso/dl_iterate_phdr.c b/src/ldso/dl_iterate_phdr.c index 86c87ef8..9546dd36 100644 --- a/src/ldso/dl_iterate_phdr.c +++ b/src/ldso/dl_iterate_phdr.c @@ -1,5 +1,6 @@ #include <elf.h> #include <link.h> +#include "pthread_impl.h" #include "libc.h" #define AUX_CNT 38 @@ -35,7 +36,7 @@ static int static_dl_iterate_phdr(int(*callback)(struct dl_phdr_info *info, size info.dlpi_subs = 0; if (tls_phdr) { info.dlpi_tls_modid = 1; - info.dlpi_tls_data = (void *)(base + tls_phdr->p_vaddr); + info.dlpi_tls_data = __tls_get_addr((tls_mod_off_t[]){1,0}); } else { info.dlpi_tls_modid = 0; info.dlpi_tls_data = 0; diff --git a/src/ldso/dlerror.c b/src/ldso/dlerror.c index 3fcc7779..dae0f3a9 100644 --- a/src/ldso/dlerror.c +++ b/src/ldso/dlerror.c @@ -3,7 +3,12 @@ #include <stdarg.h> #include "pthread_impl.h" #include "dynlink.h" -#include "lock.h" +#include "atomic.h" + +#define malloc __libc_malloc +#define calloc __libc_calloc +#define realloc __libc_realloc +#define free __libc_free char *dlerror() { @@ -17,30 +22,37 @@ char *dlerror() return s; } -static volatile int freebuf_queue_lock[1]; -static void **freebuf_queue; +/* Atomic singly-linked list, used to store list of thread-local dlerror + * buffers for deferred free. They cannot be freed at thread exit time + * because, by the time it's known they can be freed, the exiting thread + * is in a highly restrictive context where it cannot call (even the + * libc-internal) free. It also can't take locks; thus the atomic list. */ + +static void *volatile freebuf_queue; void __dl_thread_cleanup(void) { pthread_t self = __pthread_self(); - if (self->dlerror_buf && self->dlerror_buf != (void *)-1) { - LOCK(freebuf_queue_lock); - void **p = (void **)self->dlerror_buf; - *p = freebuf_queue; - freebuf_queue = p; - UNLOCK(freebuf_queue_lock); - } + if (!self->dlerror_buf || self->dlerror_buf == (void *)-1) + return; + void *h; + do { + h = freebuf_queue; + *(void **)self->dlerror_buf = h; + } while (a_cas_p(&freebuf_queue, h, self->dlerror_buf) != h); } hidden void __dl_vseterr(const char *fmt, va_list ap) { - LOCK(freebuf_queue_lock); - while (freebuf_queue) { - void **p = freebuf_queue; - freebuf_queue = *p; - free(p); + void **q; + do q = freebuf_queue; + while (q && a_cas_p(&freebuf_queue, q, 0) != q); + + while (q) { + void **p = *q; + free(q); + q = p; } - UNLOCK(freebuf_queue_lock); va_list ap2; va_copy(ap2, ap); diff --git a/src/ldso/i386/dlsym_time64.S b/src/ldso/i386/dlsym_time64.S new file mode 100644 index 00000000..bb2e7040 --- /dev/null +++ b/src/ldso/i386/dlsym_time64.S @@ -0,0 +1,3 @@ +#define __dlsym __dlsym_redir_time64 +#define dlsym __dlsym_time64 +#include "dlsym.s" diff --git a/src/ldso/loongarch64/dlsym.s b/src/ldso/loongarch64/dlsym.s new file mode 100644 index 00000000..26fabcdb --- /dev/null +++ b/src/ldso/loongarch64/dlsym.s @@ -0,0 +1,7 @@ +.global dlsym +.hidden __dlsym +.type dlsym,@function +dlsym: + move $a2, $ra + la.global $t0, __dlsym + jr $t0 diff --git a/src/ldso/m68k/dlsym_time64.S b/src/ldso/m68k/dlsym_time64.S new file mode 100644 index 00000000..bb2e7040 --- /dev/null +++ b/src/ldso/m68k/dlsym_time64.S @@ -0,0 +1,3 @@ +#define __dlsym __dlsym_redir_time64 +#define dlsym __dlsym_time64 +#include "dlsym.s" diff --git a/src/ldso/microblaze/dlsym_time64.S b/src/ldso/microblaze/dlsym_time64.S new file mode 100644 index 00000000..bb2e7040 --- /dev/null +++ b/src/ldso/microblaze/dlsym_time64.S @@ -0,0 +1,3 @@ +#define __dlsym __dlsym_redir_time64 +#define dlsym __dlsym_time64 +#include "dlsym.s" diff --git a/src/ldso/mips/dlsym_time64.S b/src/ldso/mips/dlsym_time64.S new file mode 100644 index 00000000..bb2e7040 --- /dev/null +++ b/src/ldso/mips/dlsym_time64.S @@ -0,0 +1,3 @@ +#define __dlsym __dlsym_redir_time64 +#define dlsym __dlsym_time64 +#include "dlsym.s" diff --git a/src/ldso/mipsn32/dlsym_time64.S b/src/ldso/mipsn32/dlsym_time64.S new file mode 100644 index 00000000..bb2e7040 --- /dev/null +++ b/src/ldso/mipsn32/dlsym_time64.S @@ -0,0 +1,3 @@ +#define __dlsym __dlsym_redir_time64 +#define dlsym __dlsym_time64 +#include "dlsym.s" diff --git a/src/ldso/or1k/dlsym_time64.S b/src/ldso/or1k/dlsym_time64.S new file mode 100644 index 00000000..bb2e7040 --- /dev/null +++ b/src/ldso/or1k/dlsym_time64.S @@ -0,0 +1,3 @@ +#define __dlsym __dlsym_redir_time64 +#define dlsym __dlsym_time64 +#include "dlsym.s" diff --git a/src/ldso/powerpc/dlsym_time64.S b/src/ldso/powerpc/dlsym_time64.S new file mode 100644 index 00000000..bb2e7040 --- /dev/null +++ b/src/ldso/powerpc/dlsym_time64.S @@ -0,0 +1,3 @@ +#define __dlsym __dlsym_redir_time64 +#define dlsym __dlsym_time64 +#include "dlsym.s" diff --git a/src/ldso/riscv32/dlsym.s b/src/ldso/riscv32/dlsym.s new file mode 100644 index 00000000..2bafd72d --- /dev/null +++ b/src/ldso/riscv32/dlsym.s @@ -0,0 +1,6 @@ +.global dlsym +.hidden __dlsym +.type dlsym, %function +dlsym: + mv a2, ra + tail __dlsym diff --git a/src/ldso/riscv64/tlsdesc.s b/src/ldso/riscv64/tlsdesc.s new file mode 100644 index 00000000..bef8b322 --- /dev/null +++ b/src/ldso/riscv64/tlsdesc.s @@ -0,0 +1,32 @@ +.text +.global __tlsdesc_static +.hidden __tlsdesc_static +.type __tlsdesc_static,%function +__tlsdesc_static: + ld a0,8(a0) + jr t0 + +.global __tlsdesc_dynamic +.hidden __tlsdesc_dynamic +.type __tlsdesc_dynamic,%function +__tlsdesc_dynamic: + add sp,sp,-16 + sd t1,(sp) + sd t2,8(sp) + + ld t2,-8(tp) # t2=dtv + + ld a0,8(a0) # a0=&{modidx,off} + ld t1,8(a0) # t1=off + ld a0,(a0) # a0=modidx + sll a0,a0,3 # a0=8*modidx + + add a0,a0,t2 # a0=dtv+8*modidx + ld a0,(a0) # a0=dtv[modidx] + add a0,a0,t1 # a0=dtv[modidx]+off + sub a0,a0,tp # a0=dtv[modidx]+off-tp + + ld t1,(sp) + ld t2,8(sp) + add sp,sp,16 + jr t0 diff --git a/src/ldso/sh/dlsym.s b/src/ldso/sh/dlsym.s index 11a6fff5..34f3c35c 100644 --- a/src/ldso/sh/dlsym.s +++ b/src/ldso/sh/dlsym.s @@ -5,7 +5,7 @@ dlsym: mov.l L1, r0 1: braf r0 - mov.l @r15, r6 + sts pr, r6 .align 2 L1: .long __dlsym@PLT-(1b+4-.) diff --git a/src/ldso/sh/dlsym_time64.S b/src/ldso/sh/dlsym_time64.S new file mode 100644 index 00000000..bb2e7040 --- /dev/null +++ b/src/ldso/sh/dlsym_time64.S @@ -0,0 +1,3 @@ +#define __dlsym __dlsym_redir_time64 +#define dlsym __dlsym_time64 +#include "dlsym.s" diff --git a/src/legacy/cuserid.c b/src/legacy/cuserid.c index 4e78798d..dcaf73d4 100644 --- a/src/legacy/cuserid.c +++ b/src/legacy/cuserid.c @@ -2,13 +2,21 @@ #include <pwd.h> #include <stdio.h> #include <unistd.h> +#include <string.h> char *cuserid(char *buf) { + static char usridbuf[L_cuserid]; struct passwd pw, *ppw; long pwb[256]; - if (getpwuid_r(geteuid(), &pw, (void *)pwb, sizeof pwb, &ppw)) - return 0; - snprintf(buf, L_cuserid, "%s", pw.pw_name); + if (buf) *buf = 0; + getpwuid_r(geteuid(), &pw, (void *)pwb, sizeof pwb, &ppw); + if (!ppw) + return buf; + size_t len = strnlen(pw.pw_name, L_cuserid); + if (len == L_cuserid) + return buf; + if (!buf) buf = usridbuf; + memcpy(buf, pw.pw_name, len+1); return buf; } diff --git a/src/legacy/ftw.c b/src/legacy/ftw.c index 506bd29c..e757fc6f 100644 --- a/src/legacy/ftw.c +++ b/src/legacy/ftw.c @@ -7,5 +7,3 @@ int ftw(const char *path, int (*fn)(const char *, const struct stat *, int), int * actually undefined, but works on all real-world machines. */ return nftw(path, (int (*)())fn, fd_limit, FTW_PHYS); } - -weak_alias(ftw, ftw64); diff --git a/src/legacy/lutimes.c b/src/legacy/lutimes.c index 2e5502d1..dd465923 100644 --- a/src/legacy/lutimes.c +++ b/src/legacy/lutimes.c @@ -6,9 +6,11 @@ int lutimes(const char *filename, const struct timeval tv[2]) { struct timespec times[2]; - times[0].tv_sec = tv[0].tv_sec; - times[0].tv_nsec = tv[0].tv_usec * 1000; - times[1].tv_sec = tv[1].tv_sec; - times[1].tv_nsec = tv[1].tv_usec * 1000; - return utimensat(AT_FDCWD, filename, times, AT_SYMLINK_NOFOLLOW); + if (tv) { + times[0].tv_sec = tv[0].tv_sec; + times[0].tv_nsec = tv[0].tv_usec * 1000; + times[1].tv_sec = tv[1].tv_sec; + times[1].tv_nsec = tv[1].tv_usec * 1000; + } + return utimensat(AT_FDCWD, filename, tv ? times : 0, AT_SYMLINK_NOFOLLOW); } diff --git a/src/linux/cache.c b/src/linux/cache.c index 0eb051c2..e76f7812 100644 --- a/src/linux/cache.c +++ b/src/linux/cache.c @@ -21,7 +21,7 @@ weak_alias(__cachectl, cachectl); #ifdef SYS_riscv_flush_icache #define VDSO_FLUSH_ICACHE_SYM "__vdso_flush_icache" -#define VDSO_FLUSH_ICACHE_VER "LINUX_4.5" +#define VDSO_FLUSH_ICACHE_VER "LINUX_4.15" static void *volatile vdso_func; @@ -45,6 +45,7 @@ int __riscv_flush_icache(void *start, void *end, unsigned long int flags) if (!r) return r; if (r != -ENOSYS) return __syscall_ret(r); } + return syscall(SYS_riscv_flush_icache, start, end, flags); } weak_alias(__riscv_flush_icache, riscv_flush_icache); #endif diff --git a/src/linux/clock_adjtime.c b/src/linux/clock_adjtime.c index 23eb8729..d4d03d24 100644 --- a/src/linux/clock_adjtime.c +++ b/src/linux/clock_adjtime.c @@ -38,55 +38,52 @@ int clock_adjtime (clockid_t clock_id, struct timex *utx) { int r = -ENOSYS; #ifdef SYS_clock_adjtime64 - if (SYS_clock_adjtime == SYS_clock_adjtime64 || - (utx->modes & ADJ_SETOFFSET) && !IS32BIT(utx->time.tv_sec)) { - struct ktimex64 ktx = { - .modes = utx->modes, - .offset = utx->offset, - .freq = utx->freq, - .maxerror = utx->maxerror, - .esterror = utx->esterror, - .status = utx->status, - .constant = utx->constant, - .precision = utx->precision, - .tolerance = utx->tolerance, - .time_sec = utx->time.tv_sec, - .time_usec = utx->time.tv_usec, - .tick = utx->tick, - .ppsfreq = utx->ppsfreq, - .jitter = utx->jitter, - .shift = utx->shift, - .stabil = utx->stabil, - .jitcnt = utx->jitcnt, - .calcnt = utx->calcnt, - .errcnt = utx->errcnt, - .stbcnt = utx->stbcnt, - .tai = utx->tai, - }; - r = __syscall(SYS_clock_adjtime, clock_id, &ktx); - if (r>=0) { - utx->modes = ktx.modes; - utx->offset = ktx.offset; - utx->freq = ktx.freq; - utx->maxerror = ktx.maxerror; - utx->esterror = ktx.esterror; - utx->status = ktx.status; - utx->constant = ktx.constant; - utx->precision = ktx.precision; - utx->tolerance = ktx.tolerance; - utx->time.tv_sec = ktx.time_sec; - utx->time.tv_usec = ktx.time_usec; - utx->tick = ktx.tick; - utx->ppsfreq = ktx.ppsfreq; - utx->jitter = ktx.jitter; - utx->shift = ktx.shift; - utx->stabil = ktx.stabil; - utx->jitcnt = ktx.jitcnt; - utx->calcnt = ktx.calcnt; - utx->errcnt = ktx.errcnt; - utx->stbcnt = ktx.stbcnt; - utx->tai = ktx.tai; - } + struct ktimex64 ktx = { + .modes = utx->modes, + .offset = utx->offset, + .freq = utx->freq, + .maxerror = utx->maxerror, + .esterror = utx->esterror, + .status = utx->status, + .constant = utx->constant, + .precision = utx->precision, + .tolerance = utx->tolerance, + .time_sec = utx->time.tv_sec, + .time_usec = utx->time.tv_usec, + .tick = utx->tick, + .ppsfreq = utx->ppsfreq, + .jitter = utx->jitter, + .shift = utx->shift, + .stabil = utx->stabil, + .jitcnt = utx->jitcnt, + .calcnt = utx->calcnt, + .errcnt = utx->errcnt, + .stbcnt = utx->stbcnt, + .tai = utx->tai, + }; + r = __syscall(SYS_clock_adjtime64, clock_id, &ktx); + if (r>=0) { + utx->modes = ktx.modes; + utx->offset = ktx.offset; + utx->freq = ktx.freq; + utx->maxerror = ktx.maxerror; + utx->esterror = ktx.esterror; + utx->status = ktx.status; + utx->constant = ktx.constant; + utx->precision = ktx.precision; + utx->tolerance = ktx.tolerance; + utx->time.tv_sec = ktx.time_sec; + utx->time.tv_usec = ktx.time_usec; + utx->tick = ktx.tick; + utx->ppsfreq = ktx.ppsfreq; + utx->jitter = ktx.jitter; + utx->shift = ktx.shift; + utx->stabil = ktx.stabil; + utx->jitcnt = ktx.jitcnt; + utx->calcnt = ktx.calcnt; + utx->errcnt = ktx.errcnt; + utx->stbcnt = ktx.stbcnt; + utx->tai = ktx.tai; } if (SYS_clock_adjtime == SYS_clock_adjtime64 || r!=-ENOSYS) return __syscall_ret(r); diff --git a/src/linux/clone.c b/src/linux/clone.c index 8c1af7d3..257c1cec 100644 --- a/src/linux/clone.c +++ b/src/linux/clone.c @@ -4,18 +4,62 @@ #include <sched.h> #include "pthread_impl.h" #include "syscall.h" +#include "lock.h" +#include "fork_impl.h" + +struct clone_start_args { + int (*func)(void *); + void *arg; + sigset_t sigmask; +}; + +static int clone_start(void *arg) +{ + struct clone_start_args *csa = arg; + __post_Fork(0); + __restore_sigs(&csa->sigmask); + return csa->func(csa->arg); +} int clone(int (*func)(void *), void *stack, int flags, void *arg, ...) { + struct clone_start_args csa; va_list ap; - pid_t *ptid, *ctid; - void *tls; + pid_t *ptid = 0, *ctid = 0; + void *tls = 0; + + /* Flags that produce an invalid thread/TLS state are disallowed. */ + int badflags = CLONE_THREAD | CLONE_SETTLS | CLONE_CHILD_CLEARTID; + + if ((flags & badflags) || !stack) + return __syscall_ret(-EINVAL); va_start(ap, arg); - ptid = va_arg(ap, pid_t *); - tls = va_arg(ap, void *); - ctid = va_arg(ap, pid_t *); + if (flags & (CLONE_PIDFD | CLONE_PARENT_SETTID | CLONE_CHILD_SETTID)) + ptid = va_arg(ap, pid_t *); + if (flags & CLONE_CHILD_SETTID) { + tls = va_arg(ap, void *); + ctid = va_arg(ap, pid_t *); + } va_end(ap); - return __syscall_ret(__clone(func, stack, flags, arg, ptid, tls, ctid)); + /* If CLONE_VM is used, it's impossible to give the child a consistent + * thread structure. In this case, the best we can do is assume the + * caller is content with an extremely restrictive execution context + * like the one vfork() would provide. */ + if (flags & CLONE_VM) return __syscall_ret( + __clone(func, stack, flags, arg, ptid, tls, ctid)); + + __block_all_sigs(&csa.sigmask); + LOCK(__abort_lock); + + /* Setup the a wrapper start function for the child process to do + * mimic _Fork in producing a consistent execution state. */ + csa.func = func; + csa.arg = arg; + int ret = __clone(clone_start, stack, flags, &csa, ptid, tls, ctid); + + __post_Fork(ret); + __restore_sigs(&csa.sigmask); + return __syscall_ret(ret); } diff --git a/src/linux/epoll.c b/src/linux/epoll.c index deff5b10..e56e8f4c 100644 --- a/src/linux/epoll.c +++ b/src/linux/epoll.c @@ -5,6 +5,7 @@ int epoll_create(int size) { + if (size<=0) return __syscall_ret(-EINVAL); return epoll_create1(0); } @@ -24,9 +25,9 @@ int epoll_ctl(int fd, int op, int fd2, struct epoll_event *ev) int epoll_pwait(int fd, struct epoll_event *ev, int cnt, int to, const sigset_t *sigs) { - int r = __syscall(SYS_epoll_pwait, fd, ev, cnt, to, sigs, _NSIG/8); + int r = __syscall_cp(SYS_epoll_pwait, fd, ev, cnt, to, sigs, _NSIG/8); #ifdef SYS_epoll_wait - if (r==-ENOSYS && !sigs) r = __syscall(SYS_epoll_wait, fd, ev, cnt, to); + if (r==-ENOSYS && !sigs) r = __syscall_cp(SYS_epoll_wait, fd, ev, cnt, to); #endif return __syscall_ret(r); } diff --git a/src/linux/fallocate.c b/src/linux/fallocate.c index 7d68bc8f..9146350e 100644 --- a/src/linux/fallocate.c +++ b/src/linux/fallocate.c @@ -7,6 +7,3 @@ int fallocate(int fd, int mode, off_t base, off_t len) return syscall(SYS_fallocate, fd, mode, __SYSCALL_LL_E(base), __SYSCALL_LL_E(len)); } - -#undef fallocate64 -weak_alias(fallocate, fallocate64); diff --git a/src/linux/getdents.c b/src/linux/getdents.c index 796c1e5c..97f76e14 100644 --- a/src/linux/getdents.c +++ b/src/linux/getdents.c @@ -8,5 +8,3 @@ int getdents(int fd, struct dirent *buf, size_t len) if (len>INT_MAX) len = INT_MAX; return syscall(SYS_getdents, fd, buf, len); } - -weak_alias(getdents, getdents64); diff --git a/src/linux/gettid.c b/src/linux/gettid.c new file mode 100644 index 00000000..70767137 --- /dev/null +++ b/src/linux/gettid.c @@ -0,0 +1,8 @@ +#define _GNU_SOURCE +#include <unistd.h> +#include "pthread_impl.h" + +pid_t gettid(void) +{ + return __pthread_self()->tid; +} diff --git a/src/linux/membarrier.c b/src/linux/membarrier.c index 9ebe906e..f64fe7e1 100644 --- a/src/linux/membarrier.c +++ b/src/linux/membarrier.c @@ -9,13 +9,8 @@ static void dummy_0(void) { } -static void dummy_1(pthread_t t) -{ -} - weak_alias(dummy_0, __tl_lock); weak_alias(dummy_0, __tl_unlock); -weak_alias(dummy_1, __tl_sync); static sem_t barrier_sem; @@ -40,7 +35,7 @@ int __membarrier(int cmd, int flags) __tl_lock(); sem_init(&barrier_sem, 0, 0); struct sigaction sa = { - .sa_flags = SA_RESTART, + .sa_flags = SA_RESTART | SA_ONSTACK, .sa_handler = bcast_barrier }; memset(&sa.sa_mask, -1, sizeof sa.sa_mask); diff --git a/src/linux/preadv2.c b/src/linux/preadv2.c new file mode 100644 index 00000000..5e7ab70f --- /dev/null +++ b/src/linux/preadv2.c @@ -0,0 +1,17 @@ +#define _GNU_SOURCE +#include <sys/uio.h> +#include <unistd.h> +#include "syscall.h" + +ssize_t preadv2(int fd, const struct iovec *iov, int count, off_t ofs, int flags) +{ +#ifdef SYS_preadv + if (!flags) { + if (ofs==-1) return readv(fd, iov, count); + return syscall_cp(SYS_preadv, fd, iov, count, + (long)(ofs), (long)(ofs>>32)); + } +#endif + return syscall_cp(SYS_preadv2, fd, iov, count, + (long)(ofs), (long)(ofs>>32), flags); +} diff --git a/src/linux/prlimit.c b/src/linux/prlimit.c index 3df9ffba..fcf45aab 100644 --- a/src/linux/prlimit.c +++ b/src/linux/prlimit.c @@ -21,6 +21,3 @@ int prlimit(pid_t pid, int resource, const struct rlimit *new_limit, struct rlim } return r; } - -#undef prlimit64 -weak_alias(prlimit, prlimit64); diff --git a/src/linux/pwritev2.c b/src/linux/pwritev2.c new file mode 100644 index 00000000..ece90d7c --- /dev/null +++ b/src/linux/pwritev2.c @@ -0,0 +1,17 @@ +#define _GNU_SOURCE +#include <sys/uio.h> +#include <unistd.h> +#include "syscall.h" + +ssize_t pwritev2(int fd, const struct iovec *iov, int count, off_t ofs, int flags) +{ +#ifdef SYS_pwritev + if (!flags) { + if (ofs==-1) return writev(fd, iov, count); + return syscall_cp(SYS_pwritev, fd, iov, count, + (long)(ofs), (long)(ofs>>32)); + } +#endif + return syscall_cp(SYS_pwritev2, fd, iov, count, + (long)(ofs), (long)(ofs>>32), flags); +} diff --git a/src/linux/sendfile.c b/src/linux/sendfile.c index 9afe6dd6..fc1577d3 100644 --- a/src/linux/sendfile.c +++ b/src/linux/sendfile.c @@ -5,5 +5,3 @@ ssize_t sendfile(int out_fd, int in_fd, off_t *ofs, size_t count) { return syscall(SYS_sendfile, out_fd, in_fd, ofs, count); } - -weak_alias(sendfile, sendfile64); diff --git a/src/linux/setgroups.c b/src/linux/setgroups.c index 1248fdbf..47142f14 100644 --- a/src/linux/setgroups.c +++ b/src/linux/setgroups.c @@ -1,8 +1,36 @@ #define _GNU_SOURCE #include <unistd.h> +#include <signal.h> #include "syscall.h" +#include "libc.h" + +struct ctx { + size_t count; + const gid_t *list; + int ret; +}; + +static void do_setgroups(void *p) +{ + struct ctx *c = p; + if (c->ret<0) return; + int ret = __syscall(SYS_setgroups, c->count, c->list); + if (ret && !c->ret) { + /* If one thread fails to set groups after another has already + * succeeded, forcibly killing the process is the only safe + * thing to do. State is inconsistent and dangerous. Use + * SIGKILL because it is uncatchable. */ + __block_all_sigs(0); + __syscall(SYS_kill, __syscall(SYS_getpid), SIGKILL); + } + c->ret = ret; +} int setgroups(size_t count, const gid_t list[]) { - return syscall(SYS_setgroups, count, list); + /* ret is initially nonzero so that failure of the first thread does not + * trigger the safety kill above. */ + struct ctx c = { .count = count, .list = list, .ret = 1 }; + __synccall(do_setgroups, &c); + return __syscall_ret(c.ret); } diff --git a/src/linux/statx.c b/src/linux/statx.c new file mode 100644 index 00000000..4616bff4 --- /dev/null +++ b/src/linux/statx.c @@ -0,0 +1,42 @@ +#define _GNU_SOURCE +#include <sys/stat.h> +#include <string.h> +#include <syscall.h> +#include <sys/sysmacros.h> +#include <errno.h> + +int statx(int dirfd, const char *restrict path, int flags, unsigned mask, struct statx *restrict stx) +{ + int ret = __syscall(SYS_statx, dirfd, path, flags, mask, stx); + +#ifndef SYS_fstatat + return __syscall_ret(ret); +#endif + + if (ret != -ENOSYS) return __syscall_ret(ret); + + struct stat st; + ret = fstatat(dirfd, path, &st, flags); + if (ret) return ret; + + stx->stx_dev_major = major(st.st_dev); + stx->stx_dev_minor = minor(st.st_dev); + stx->stx_ino = st.st_ino; + stx->stx_mode = st.st_mode; + stx->stx_nlink = st.st_nlink; + stx->stx_uid = st.st_uid; + stx->stx_gid = st.st_gid; + stx->stx_size = st.st_size; + stx->stx_blksize = st.st_blksize; + stx->stx_blocks = st.st_blocks; + stx->stx_atime.tv_sec = st.st_atim.tv_sec; + stx->stx_atime.tv_nsec = st.st_atim.tv_nsec; + stx->stx_mtime.tv_sec = st.st_mtim.tv_sec; + stx->stx_mtime.tv_nsec = st.st_mtim.tv_nsec; + stx->stx_ctime.tv_sec = st.st_ctim.tv_sec; + stx->stx_ctime.tv_nsec = st.st_ctim.tv_nsec; + stx->stx_btime = (struct statx_timestamp){.tv_sec=0, .tv_nsec=0}; + stx->stx_mask = STATX_BASIC_STATS; + + return 0; +} diff --git a/src/linux/wait4.c b/src/linux/wait4.c index 83650e34..fb08c0d0 100644 --- a/src/linux/wait4.c +++ b/src/linux/wait4.c @@ -12,7 +12,7 @@ pid_t wait4(pid_t pid, int *status, int options, struct rusage *ru) if (ru) { long long kru64[18]; r = __syscall(SYS_wait4_time64, pid, status, options, kru64); - if (!r) { + if (r > 0) { ru->ru_utime = (struct timeval) { .tv_sec = kru64[0], .tv_usec = kru64[1] }; ru->ru_stime = (struct timeval) @@ -26,7 +26,7 @@ pid_t wait4(pid_t pid, int *status, int options, struct rusage *ru) } #endif char *dest = ru ? (char *)&ru->ru_maxrss - 4*sizeof(long) : 0; - r = __syscall(SYS_wait4, pid, status, options, dest); + r = __sys_wait4(pid, status, options, dest); if (r>0 && ru && sizeof(time_t) > sizeof(long)) { long kru[4]; memcpy(kru, dest, 4*sizeof(long)); diff --git a/src/locale/dcngettext.c b/src/locale/dcngettext.c index 4c304393..0b53286d 100644 --- a/src/locale/dcngettext.c +++ b/src/locale/dcngettext.c @@ -10,6 +10,12 @@ #include "atomic.h" #include "pleval.h" #include "lock.h" +#include "fork_impl.h" + +#define malloc __libc_malloc +#define calloc __libc_calloc +#define realloc undef +#define free undef struct binding { struct binding *next; @@ -34,9 +40,11 @@ static char *gettextdir(const char *domainname, size_t *dirlen) return 0; } +static volatile int lock[1]; +volatile int *const __gettext_lockptr = lock; + char *bindtextdomain(const char *domainname, const char *dirname) { - static volatile int lock[1]; struct binding *p, *q; if (!domainname) return 0; @@ -124,6 +132,9 @@ char *dcngettext(const char *domainname, const char *msgid1, const char *msgid2, struct binding *q; int old_errno = errno; + /* match gnu gettext behaviour */ + if (!msgid1) goto notrans; + if ((unsigned)category >= LC_ALL) goto notrans; if (!domainname) domainname = __gettextdomain(); diff --git a/src/locale/duplocale.c b/src/locale/duplocale.c index 030b64cb..5ce33ae6 100644 --- a/src/locale/duplocale.c +++ b/src/locale/duplocale.c @@ -3,6 +3,11 @@ #include "locale_impl.h" #include "libc.h" +#define malloc __libc_malloc +#define calloc undef +#define realloc undef +#define free undef + locale_t __duplocale(locale_t old) { locale_t new = malloc(sizeof *new); diff --git a/src/locale/freelocale.c b/src/locale/freelocale.c index 802b8bfe..385d1206 100644 --- a/src/locale/freelocale.c +++ b/src/locale/freelocale.c @@ -1,6 +1,11 @@ #include <stdlib.h> #include "locale_impl.h" +#define malloc undef +#define calloc undef +#define realloc undef +#define free __libc_free + void freelocale(locale_t l) { if (__loc_is_allocated(l)) free(l); diff --git a/src/locale/iconv.c b/src/locale/iconv.c index 3047c27b..7fb2e1ef 100644 --- a/src/locale/iconv.c +++ b/src/locale/iconv.c @@ -49,10 +49,10 @@ static const unsigned char charmaps[] = "ucs4\0utf32\0\0\313" "ucs2\0\0\314" "eucjp\0\0\320" -"shiftjis\0sjis\0\0\321" +"shiftjis\0sjis\0cp932\0\0\321" "iso2022jp\0\0\322" "gb18030\0\0\330" -"gbk\0\0\331" +"gbk\0cp936\0windows936\0\0\331" "gb2312\0\0\332" "big5\0bigfive\0cp950\0big5hkscs\0\0\340" "euckr\0ksc5601\0ksx1001\0cp949\0\0\350" @@ -340,6 +340,7 @@ size_t iconv(iconv_t cd, char **restrict in, size_t *restrict inb, char **restri c++; d -= 159; } + if (c>=84) goto ilseq; c = jis0208[c][d]; if (!c) goto ilseq; break; @@ -403,6 +404,10 @@ size_t iconv(iconv_t cd, char **restrict in, size_t *restrict inb, char **restri if (c < 128) break; if (c < 0xa1) goto ilseq; case GBK: + if (c == 128) { + c = 0x20ac; + break; + } case GB18030: if (c < 128) break; c -= 0x81; diff --git a/src/locale/locale_map.c b/src/locale/locale_map.c index 2321bac0..da61f7fc 100644 --- a/src/locale/locale_map.c +++ b/src/locale/locale_map.c @@ -1,9 +1,16 @@ #include <locale.h> #include <string.h> #include <sys/mman.h> +#include <stdlib.h> #include "locale_impl.h" #include "libc.h" #include "lock.h" +#include "fork_impl.h" + +#define malloc __libc_malloc +#define calloc undef +#define realloc undef +#define free undef const char *__lctrans_impl(const char *msg, const struct __locale_map *lm) { @@ -21,9 +28,11 @@ static const char envvars[][12] = { "LC_MESSAGES", }; +volatile int __locale_lock[1]; +volatile int *const __locale_lockptr = __locale_lock; + const struct __locale_map *__get_locale(int cat, const char *val) { - static volatile int lock[1]; static void *volatile loc_head; const struct __locale_map *p; struct __locale_map *new = 0; @@ -54,20 +63,12 @@ const struct __locale_map *__get_locale(int cat, const char *val) for (p=loc_head; p; p=p->next) if (!strcmp(val, p->name)) return p; - LOCK(lock); - - for (p=loc_head; p; p=p->next) - if (!strcmp(val, p->name)) { - UNLOCK(lock); - return p; - } - if (!libc.secure) path = getenv("MUSL_LOCPATH"); /* FIXME: add a default path? */ if (path) for (; *path; path=z+!!*z) { z = __strchrnul(path, ':'); - l = z - path - !!*z; + l = z - path; if (l >= sizeof buf - n - 2) continue; memcpy(buf, path, l); buf[l] = '/'; @@ -108,6 +109,5 @@ const struct __locale_map *__get_locale(int cat, const char *val) * requested name was "C" or "POSIX". */ if (!new && cat == LC_CTYPE) new = (void *)&__c_dot_utf8; - UNLOCK(lock); return new; } diff --git a/src/locale/newlocale.c b/src/locale/newlocale.c index d20a8489..9ac3cd38 100644 --- a/src/locale/newlocale.c +++ b/src/locale/newlocale.c @@ -2,16 +2,15 @@ #include <string.h> #include <pthread.h> #include "locale_impl.h" +#include "lock.h" -static pthread_once_t default_locale_once; -static struct __locale_struct default_locale, default_ctype_locale; +#define malloc __libc_malloc +#define calloc undef +#define realloc undef +#define free undef -static void default_locale_init(void) -{ - for (int i=0; i<LC_ALL; i++) - default_locale.cat[i] = __get_locale(i, ""); - default_ctype_locale.cat[LC_CTYPE] = default_locale.cat[LC_CTYPE]; -} +static int default_locale_init_done; +static struct __locale_struct default_locale, default_ctype_locale; int __loc_is_allocated(locale_t loc) { @@ -19,7 +18,7 @@ int __loc_is_allocated(locale_t loc) && loc != &default_locale && loc != &default_ctype_locale; } -locale_t __newlocale(int mask, const char *name, locale_t loc) +static locale_t do_newlocale(int mask, const char *name, locale_t loc) { struct __locale_struct tmp; @@ -44,7 +43,12 @@ locale_t __newlocale(int mask, const char *name, locale_t loc) /* And provide builtins for the initial default locale, and a * variant of the C locale honoring the default locale's encoding. */ - pthread_once(&default_locale_once, default_locale_init); + if (!default_locale_init_done) { + for (int i=0; i<LC_ALL; i++) + default_locale.cat[i] = __get_locale(i, ""); + default_ctype_locale.cat[LC_CTYPE] = default_locale.cat[LC_CTYPE]; + default_locale_init_done = 1; + } if (!memcmp(&tmp, &default_locale, sizeof tmp)) return &default_locale; if (!memcmp(&tmp, &default_ctype_locale, sizeof tmp)) return &default_ctype_locale; @@ -55,4 +59,12 @@ locale_t __newlocale(int mask, const char *name, locale_t loc) return loc; } +locale_t __newlocale(int mask, const char *name, locale_t loc) +{ + LOCK(__locale_lock); + loc = do_newlocale(mask, name, loc); + UNLOCK(__locale_lock); + return loc; +} + weak_alias(__newlocale, newlocale); diff --git a/src/locale/setlocale.c b/src/locale/setlocale.c index 2bc7b500..360c4437 100644 --- a/src/locale/setlocale.c +++ b/src/locale/setlocale.c @@ -9,12 +9,11 @@ static char buf[LC_ALL*(LOCALE_NAME_MAX+1)]; char *setlocale(int cat, const char *name) { - static volatile int lock[1]; const struct __locale_map *lm; if ((unsigned)cat > LC_ALL) return 0; - LOCK(lock); + LOCK(__locale_lock); /* For LC_ALL, setlocale is required to return a string which * encodes the current setting for all categories. The format of @@ -36,7 +35,7 @@ char *setlocale(int cat, const char *name) } lm = __get_locale(i, part); if (lm == LOC_MAP_FAILED) { - UNLOCK(lock); + UNLOCK(__locale_lock); return 0; } tmp_locale.cat[i] = lm; @@ -57,14 +56,14 @@ char *setlocale(int cat, const char *name) s += l+1; } *--s = 0; - UNLOCK(lock); + UNLOCK(__locale_lock); return same==LC_ALL ? (char *)part : buf; } if (name) { lm = __get_locale(cat, name); if (lm == LOC_MAP_FAILED) { - UNLOCK(lock); + UNLOCK(__locale_lock); return 0; } libc.global_locale.cat[cat] = lm; @@ -73,7 +72,7 @@ char *setlocale(int cat, const char *name) } char *ret = lm ? (char *)lm->name : "C"; - UNLOCK(lock); + UNLOCK(__locale_lock); return ret; } diff --git a/src/locale/strtod_l.c b/src/locale/strtod_l.c new file mode 100644 index 00000000..574ba148 --- /dev/null +++ b/src/locale/strtod_l.c @@ -0,0 +1,22 @@ +#define _GNU_SOURCE +#include <stdlib.h> +#include <locale.h> + +float strtof_l(const char *restrict s, char **restrict p, locale_t l) +{ + return strtof(s, p); +} + +double strtod_l(const char *restrict s, char **restrict p, locale_t l) +{ + return strtod(s, p); +} + +long double strtold_l(const char *restrict s, char **restrict p, locale_t l) +{ + return strtold(s, p); +} + +weak_alias(strtof_l, __strtof_l); +weak_alias(strtod_l, __strtod_l); +weak_alias(strtold_l, __strtold_l); diff --git a/src/malloc/DESIGN b/src/malloc/DESIGN deleted file mode 100644 index 58b0523f..00000000 --- a/src/malloc/DESIGN +++ /dev/null @@ -1,22 +0,0 @@ - - -In principle, this memory allocator is roughly equivalent to Doug -Lea's dlmalloc with fine-grained locking. - - - -malloc: - -Uses a freelist binned by chunk size, with a bitmap to optimize -searching for the smallest non-empty bin which can satisfy an -allocation. If no free chunks are available, it creates a new chunk of -the requested size and attempts to merge it with any existing free -chunk immediately below the newly created chunk. - -Whether the chunk was obtained from a bin or newly created, it's -likely to be larger than the requested allocation. malloc always -finishes its work by passing the new chunk to realloc, which will -split it into two chunks and free the tail portion. - - - diff --git a/src/malloc/aligned_alloc.c b/src/malloc/aligned_alloc.c deleted file mode 100644 index b6143f30..00000000 --- a/src/malloc/aligned_alloc.c +++ /dev/null @@ -1,7 +0,0 @@ -#include <stdlib.h> -#include "malloc_impl.h" - -void *aligned_alloc(size_t align, size_t len) -{ - return __memalign(align, len); -} diff --git a/src/malloc/calloc.c b/src/malloc/calloc.c new file mode 100644 index 00000000..bf6bddca --- /dev/null +++ b/src/malloc/calloc.c @@ -0,0 +1,45 @@ +#include <stdlib.h> +#include <stdint.h> +#include <string.h> +#include <errno.h> +#include "dynlink.h" + +static size_t mal0_clear(char *p, size_t n) +{ + const size_t pagesz = 4096; /* arbitrary */ + if (n < pagesz) return n; +#ifdef __GNUC__ + typedef uint64_t __attribute__((__may_alias__)) T; +#else + typedef unsigned char T; +#endif + char *pp = p + n; + size_t i = (uintptr_t)pp & (pagesz - 1); + for (;;) { + pp = memset(pp - i, 0, i); + if (pp - p < pagesz) return pp - p; + for (i = pagesz; i; i -= 2*sizeof(T), pp -= 2*sizeof(T)) + if (((T *)pp)[-1] | ((T *)pp)[-2]) + break; + } +} + +static int allzerop(void *p) +{ + return 0; +} +weak_alias(allzerop, __malloc_allzerop); + +void *calloc(size_t m, size_t n) +{ + if (n && m > (size_t)-1/n) { + errno = ENOMEM; + return 0; + } + n *= m; + void *p = malloc(n); + if (!p || (!__malloc_replaced && __malloc_allzerop(p))) + return p; + n = mal0_clear(p, n); + return memset(p, 0, n); +} diff --git a/src/malloc/expand_heap.c b/src/malloc/expand_heap.c deleted file mode 100644 index e6a3d7a0..00000000 --- a/src/malloc/expand_heap.c +++ /dev/null @@ -1,71 +0,0 @@ -#include <limits.h> -#include <stdint.h> -#include <errno.h> -#include <sys/mman.h> -#include "libc.h" -#include "syscall.h" -#include "malloc_impl.h" - -/* This function returns true if the interval [old,new] - * intersects the 'len'-sized interval below &libc.auxv - * (interpreted as the main-thread stack) or below &b - * (the current stack). It is used to defend against - * buggy brk implementations that can cross the stack. */ - -static int traverses_stack_p(uintptr_t old, uintptr_t new) -{ - const uintptr_t len = 8<<20; - uintptr_t a, b; - - b = (uintptr_t)libc.auxv; - a = b > len ? b-len : 0; - if (new>a && old<b) return 1; - - b = (uintptr_t)&b; - a = b > len ? b-len : 0; - if (new>a && old<b) return 1; - - return 0; -} - -/* Expand the heap in-place if brk can be used, or otherwise via mmap, - * using an exponential lower bound on growth by mmap to make - * fragmentation asymptotically irrelevant. The size argument is both - * an input and an output, since the caller needs to know the size - * allocated, which will be larger than requested due to page alignment - * and mmap minimum size rules. The caller is responsible for locking - * to prevent concurrent calls. */ - -void *__expand_heap(size_t *pn) -{ - static uintptr_t brk; - static unsigned mmap_step; - size_t n = *pn; - - if (n > SIZE_MAX/2 - PAGE_SIZE) { - errno = ENOMEM; - return 0; - } - n += -n & PAGE_SIZE-1; - - if (!brk) { - brk = __syscall(SYS_brk, 0); - brk += -brk & PAGE_SIZE-1; - } - - if (n < SIZE_MAX-brk && !traverses_stack_p(brk, brk+n) - && __syscall(SYS_brk, brk+n)==brk+n) { - *pn = n; - brk += n; - return (void *)(brk-n); - } - - size_t min = (size_t)PAGE_SIZE << mmap_step/2; - if (n < min) n = min; - void *area = __mmap(0, n, PROT_READ|PROT_WRITE, - MAP_PRIVATE|MAP_ANONYMOUS, -1, 0); - if (area == MAP_FAILED) return 0; - *pn = n; - mmap_step++; - return area; -} diff --git a/src/malloc/free.c b/src/malloc/free.c new file mode 100644 index 00000000..3944f7b2 --- /dev/null +++ b/src/malloc/free.c @@ -0,0 +1,6 @@ +#include <stdlib.h> + +void free(void *p) +{ + __libc_free(p); +} diff --git a/src/malloc/libc_calloc.c b/src/malloc/libc_calloc.c new file mode 100644 index 00000000..d25eabea --- /dev/null +++ b/src/malloc/libc_calloc.c @@ -0,0 +1,4 @@ +#define calloc __libc_calloc +#define malloc __libc_malloc + +#include "calloc.c" diff --git a/src/malloc/lite_malloc.c b/src/malloc/lite_malloc.c index 050d84f6..43a988fb 100644 --- a/src/malloc/lite_malloc.c +++ b/src/malloc/lite_malloc.c @@ -2,58 +2,117 @@ #include <stdint.h> #include <limits.h> #include <errno.h> +#include <sys/mman.h> +#include "libc.h" #include "lock.h" -#include "malloc_impl.h" +#include "syscall.h" +#include "fork_impl.h" #define ALIGN 16 +/* This function returns true if the interval [old,new] + * intersects the 'len'-sized interval below &libc.auxv + * (interpreted as the main-thread stack) or below &b + * (the current stack). It is used to defend against + * buggy brk implementations that can cross the stack. */ + +static int traverses_stack_p(uintptr_t old, uintptr_t new) +{ + const uintptr_t len = 8<<20; + uintptr_t a, b; + + b = (uintptr_t)libc.auxv; + a = b > len ? b-len : 0; + if (new>a && old<b) return 1; + + b = (uintptr_t)&b; + a = b > len ? b-len : 0; + if (new>a && old<b) return 1; + + return 0; +} + +static volatile int lock[1]; +volatile int *const __bump_lockptr = lock; + static void *__simple_malloc(size_t n) { - static char *cur, *end; - static volatile int lock[1]; - size_t align=1, pad; + static uintptr_t brk, cur, end; + static unsigned mmap_step; + size_t align=1; void *p; + if (n > SIZE_MAX/2) { + errno = ENOMEM; + return 0; + } + if (!n) n++; while (align<n && align<ALIGN) align += align; LOCK(lock); - pad = -(uintptr_t)cur & align-1; - - if (n <= SIZE_MAX/2 + ALIGN) n += pad; + cur += -cur & align-1; if (n > end-cur) { - size_t m = n; - char *new = __expand_heap(&m); - if (!new) { - UNLOCK(lock); - return 0; + size_t req = n - (end-cur) + PAGE_SIZE-1 & -PAGE_SIZE; + + if (!cur) { + brk = __syscall(SYS_brk, 0); + brk += -brk & PAGE_SIZE-1; + cur = end = brk; } - if (new != end) { - cur = new; - n -= pad; - pad = 0; + + if (brk == end && req < SIZE_MAX-brk + && !traverses_stack_p(brk, brk+req) + && __syscall(SYS_brk, brk+req)==brk+req) { + brk = end += req; + } else { + int new_area = 0; + req = n + PAGE_SIZE-1 & -PAGE_SIZE; + /* Only make a new area rather than individual mmap + * if wasted space would be over 1/8 of the map. */ + if (req-n > req/8) { + /* Geometric area size growth up to 64 pages, + * bounding waste by 1/8 of the area. */ + size_t min = PAGE_SIZE<<(mmap_step/2); + if (min-n > end-cur) { + if (req < min) { + req = min; + if (mmap_step < 12) + mmap_step++; + } + new_area = 1; + } + } + void *mem = __mmap(0, req, PROT_READ|PROT_WRITE, + MAP_PRIVATE|MAP_ANONYMOUS, -1, 0); + if (mem == MAP_FAILED || !new_area) { + UNLOCK(lock); + return mem==MAP_FAILED ? 0 : mem; + } + cur = (uintptr_t)mem; + end = cur + req; } - end = new + m; } - p = cur + pad; + p = (void *)cur; cur += n; UNLOCK(lock); return p; } -weak_alias(__simple_malloc, malloc); +weak_alias(__simple_malloc, __libc_malloc_impl); -static void *__simple_calloc(size_t m, size_t n) +void *__libc_malloc(size_t n) { - if (n && m > (size_t)-1/n) { - errno = ENOMEM; - return 0; - } - return __simple_malloc(n * m); + return __libc_malloc_impl(n); +} + +static void *default_malloc(size_t n) +{ + return __libc_malloc_impl(n); } -weak_alias(__simple_calloc, calloc); +weak_alias(default_malloc, malloc); diff --git a/src/malloc/mallocng/aligned_alloc.c b/src/malloc/mallocng/aligned_alloc.c new file mode 100644 index 00000000..e0862a83 --- /dev/null +++ b/src/malloc/mallocng/aligned_alloc.c @@ -0,0 +1,60 @@ +#include <stdlib.h> +#include <errno.h> +#include "meta.h" + +void *aligned_alloc(size_t align, size_t len) +{ + if ((align & -align) != align) { + errno = EINVAL; + return 0; + } + + if (len > SIZE_MAX - align || align >= (1ULL<<31)*UNIT) { + errno = ENOMEM; + return 0; + } + + if (DISABLE_ALIGNED_ALLOC) { + errno = ENOMEM; + return 0; + } + + if (align <= UNIT) align = UNIT; + + unsigned char *p = malloc(len + align - UNIT); + if (!p) + return 0; + + struct meta *g = get_meta(p); + int idx = get_slot_index(p); + size_t stride = get_stride(g); + unsigned char *start = g->mem->storage + stride*idx; + unsigned char *end = g->mem->storage + stride*(idx+1) - IB; + size_t adj = -(uintptr_t)p & (align-1); + + if (!adj) { + set_size(p, end, len); + return p; + } + p += adj; + uint32_t offset = (size_t)(p-g->mem->storage)/UNIT; + if (offset <= 0xffff) { + *(uint16_t *)(p-2) = offset; + p[-4] = 0; + } else { + // use a 32-bit offset if 16-bit doesn't fit. for this, + // 16-bit field must be zero, [-4] byte nonzero. + *(uint16_t *)(p-2) = 0; + *(uint32_t *)(p-8) = offset; + p[-4] = 1; + } + p[-3] = idx; + set_size(p, end, len); + // store offset to aligned enframing. this facilitates cycling + // offset and also iteration of heap for debugging/measurement. + // for extreme overalignment it won't fit but these are classless + // allocations anyway. + *(uint16_t *)(start - 2) = (size_t)(p-start)/UNIT; + start[-3] = 7<<5; + return p; +} diff --git a/src/malloc/mallocng/donate.c b/src/malloc/mallocng/donate.c new file mode 100644 index 00000000..41d850f3 --- /dev/null +++ b/src/malloc/mallocng/donate.c @@ -0,0 +1,39 @@ +#include <stdlib.h> +#include <stdint.h> +#include <limits.h> +#include <string.h> +#include <sys/mman.h> +#include <errno.h> + +#include "meta.h" + +static void donate(unsigned char *base, size_t len) +{ + uintptr_t a = (uintptr_t)base; + uintptr_t b = a + len; + a += -a & (UNIT-1); + b -= b & (UNIT-1); + memset(base, 0, len); + for (int sc=47; sc>0 && b>a; sc-=4) { + if (b-a < (size_classes[sc]+1)*UNIT) continue; + struct meta *m = alloc_meta(); + m->avail_mask = 0; + m->freed_mask = 1; + m->mem = (void *)a; + m->mem->meta = m; + m->last_idx = 0; + m->freeable = 0; + m->sizeclass = sc; + m->maplen = 0; + *((unsigned char *)m->mem+UNIT-4) = 0; + *((unsigned char *)m->mem+UNIT-3) = 255; + m->mem->storage[size_classes[sc]*UNIT-4] = 0; + queue(&ctx.active[sc], m); + a += (size_classes[sc]+1)*UNIT; + } +} + +void __malloc_donate(char *start, char *end) +{ + donate((void *)start, end-start); +} diff --git a/src/malloc/mallocng/free.c b/src/malloc/mallocng/free.c new file mode 100644 index 00000000..43f32aad --- /dev/null +++ b/src/malloc/mallocng/free.c @@ -0,0 +1,151 @@ +#define _BSD_SOURCE +#include <stdlib.h> +#include <sys/mman.h> + +#include "meta.h" + +struct mapinfo { + void *base; + size_t len; +}; + +static struct mapinfo nontrivial_free(struct meta *, int); + +static struct mapinfo free_group(struct meta *g) +{ + struct mapinfo mi = { 0 }; + int sc = g->sizeclass; + if (sc < 48) { + ctx.usage_by_class[sc] -= g->last_idx+1; + } + if (g->maplen) { + step_seq(); + record_seq(sc); + mi.base = g->mem; + mi.len = g->maplen*4096UL; + } else { + void *p = g->mem; + struct meta *m = get_meta(p); + int idx = get_slot_index(p); + g->mem->meta = 0; + // not checking size/reserved here; it's intentionally invalid + mi = nontrivial_free(m, idx); + } + free_meta(g); + return mi; +} + +static int okay_to_free(struct meta *g) +{ + int sc = g->sizeclass; + + if (!g->freeable) return 0; + + // always free individual mmaps not suitable for reuse + if (sc >= 48 || get_stride(g) < UNIT*size_classes[sc]) + return 1; + + // always free groups allocated inside another group's slot + // since recreating them should not be expensive and they + // might be blocking freeing of a much larger group. + if (!g->maplen) return 1; + + // if there is another non-full group, free this one to + // consolidate future allocations, reduce fragmentation. + if (g->next != g) return 1; + + // free any group in a size class that's not bouncing + if (!is_bouncing(sc)) return 1; + + size_t cnt = g->last_idx+1; + size_t usage = ctx.usage_by_class[sc]; + + // if usage is high enough that a larger count should be + // used, free the low-count group so a new one will be made. + if (9*cnt <= usage && cnt < 20) + return 1; + + // otherwise, keep the last group in a bouncing class. + return 0; +} + +static struct mapinfo nontrivial_free(struct meta *g, int i) +{ + uint32_t self = 1u<<i; + int sc = g->sizeclass; + uint32_t mask = g->freed_mask | g->avail_mask; + + if (mask+self == (2u<<g->last_idx)-1 && okay_to_free(g)) { + // any multi-slot group is necessarily on an active list + // here, but single-slot groups might or might not be. + if (g->next) { + assert(sc < 48); + int activate_new = (ctx.active[sc]==g); + dequeue(&ctx.active[sc], g); + if (activate_new && ctx.active[sc]) + activate_group(ctx.active[sc]); + } + return free_group(g); + } else if (!mask) { + assert(sc < 48); + // might still be active if there were no allocations + // after last available slot was taken. + if (ctx.active[sc] != g) { + queue(&ctx.active[sc], g); + } + } + a_or(&g->freed_mask, self); + return (struct mapinfo){ 0 }; +} + +void free(void *p) +{ + if (!p) return; + + struct meta *g = get_meta(p); + int idx = get_slot_index(p); + size_t stride = get_stride(g); + unsigned char *start = g->mem->storage + stride*idx; + unsigned char *end = start + stride - IB; + get_nominal_size(p, end); + uint32_t self = 1u<<idx, all = (2u<<g->last_idx)-1; + ((unsigned char *)p)[-3] = 255; + // invalidate offset to group header, and cycle offset of + // used region within slot if current offset is zero. + *(uint16_t *)((char *)p-2) = 0; + + // release any whole pages contained in the slot to be freed + // unless it's a single-slot group that will be unmapped. + if (((uintptr_t)(start-1) ^ (uintptr_t)end) >= 2*PGSZ && g->last_idx) { + unsigned char *base = start + (-(uintptr_t)start & (PGSZ-1)); + size_t len = (end-base) & -PGSZ; + if (len && USE_MADV_FREE) { + int e = errno; + madvise(base, len, MADV_FREE); + errno = e; + } + } + + // atomic free without locking if this is neither first or last slot + for (;;) { + uint32_t freed = g->freed_mask; + uint32_t avail = g->avail_mask; + uint32_t mask = freed | avail; + assert(!(mask&self)); + if (!freed || mask+self==all) break; + if (!MT) + g->freed_mask = freed+self; + else if (a_cas(&g->freed_mask, freed, freed+self)!=freed) + continue; + return; + } + + wrlock(); + struct mapinfo mi = nontrivial_free(g, idx); + unlock(); + if (mi.len) { + int e = errno; + munmap(mi.base, mi.len); + errno = e; + } +} diff --git a/src/malloc/mallocng/glue.h b/src/malloc/mallocng/glue.h new file mode 100644 index 00000000..77f4c812 --- /dev/null +++ b/src/malloc/mallocng/glue.h @@ -0,0 +1,95 @@ +#ifndef MALLOC_GLUE_H +#define MALLOC_GLUE_H + +#include <stdint.h> +#include <sys/mman.h> +#include <pthread.h> +#include <unistd.h> +#include <elf.h> +#include <string.h> +#include "atomic.h" +#include "syscall.h" +#include "libc.h" +#include "lock.h" +#include "dynlink.h" + +// use macros to appropriately namespace these. +#define size_classes __malloc_size_classes +#define ctx __malloc_context +#define alloc_meta __malloc_alloc_meta +#define is_allzero __malloc_allzerop +#define dump_heap __dump_heap + +#define malloc __libc_malloc_impl +#define realloc __libc_realloc +#define free __libc_free + +#define USE_MADV_FREE 0 + +#if USE_REAL_ASSERT +#include <assert.h> +#else +#undef assert +#define assert(x) do { if (!(x)) a_crash(); } while(0) +#endif + +#define brk(p) ((uintptr_t)__syscall(SYS_brk, p)) + +#define mmap __mmap +#define madvise __madvise +#define mremap __mremap + +#define DISABLE_ALIGNED_ALLOC (__malloc_replaced && !__aligned_alloc_replaced) + +static inline uint64_t get_random_secret() +{ + uint64_t secret = (uintptr_t)&secret * 1103515245; + for (size_t i=0; libc.auxv[i]; i+=2) + if (libc.auxv[i]==AT_RANDOM) + memcpy(&secret, (char *)libc.auxv[i+1]+8, sizeof secret); + return secret; +} + +#ifndef PAGESIZE +#define PAGESIZE PAGE_SIZE +#endif + +#define MT (libc.need_locks) + +#define RDLOCK_IS_EXCLUSIVE 1 + +__attribute__((__visibility__("hidden"))) +extern int __malloc_lock[1]; + +#define LOCK_OBJ_DEF \ +int __malloc_lock[1]; \ +void __malloc_atfork(int who) { malloc_atfork(who); } + +static inline void rdlock() +{ + if (MT) LOCK(__malloc_lock); +} +static inline void wrlock() +{ + if (MT) LOCK(__malloc_lock); +} +static inline void unlock() +{ + UNLOCK(__malloc_lock); +} +static inline void upgradelock() +{ +} +static inline void resetlock() +{ + __malloc_lock[0] = 0; +} + +static inline void malloc_atfork(int who) +{ + if (who<0) rdlock(); + else if (who>0) resetlock(); + else unlock(); +} + +#endif diff --git a/src/malloc/mallocng/malloc.c b/src/malloc/mallocng/malloc.c new file mode 100644 index 00000000..d695ab8e --- /dev/null +++ b/src/malloc/mallocng/malloc.c @@ -0,0 +1,387 @@ +#include <stdlib.h> +#include <stdint.h> +#include <limits.h> +#include <string.h> +#include <sys/mman.h> +#include <errno.h> + +#include "meta.h" + +LOCK_OBJ_DEF; + +const uint16_t size_classes[] = { + 1, 2, 3, 4, 5, 6, 7, 8, + 9, 10, 12, 15, + 18, 20, 25, 31, + 36, 42, 50, 63, + 72, 84, 102, 127, + 146, 170, 204, 255, + 292, 340, 409, 511, + 584, 682, 818, 1023, + 1169, 1364, 1637, 2047, + 2340, 2730, 3276, 4095, + 4680, 5460, 6552, 8191, +}; + +static const uint8_t small_cnt_tab[][3] = { + { 30, 30, 30 }, + { 31, 15, 15 }, + { 20, 10, 10 }, + { 31, 15, 7 }, + { 25, 12, 6 }, + { 21, 10, 5 }, + { 18, 8, 4 }, + { 31, 15, 7 }, + { 28, 14, 6 }, +}; + +static const uint8_t med_cnt_tab[4] = { 28, 24, 20, 32 }; + +struct malloc_context ctx = { 0 }; + +struct meta *alloc_meta(void) +{ + struct meta *m; + unsigned char *p; + if (!ctx.init_done) { +#ifndef PAGESIZE + ctx.pagesize = get_page_size(); +#endif + ctx.secret = get_random_secret(); + ctx.init_done = 1; + } + size_t pagesize = PGSZ; + if (pagesize < 4096) pagesize = 4096; + if ((m = dequeue_head(&ctx.free_meta_head))) return m; + if (!ctx.avail_meta_count) { + int need_unprotect = 1; + if (!ctx.avail_meta_area_count && ctx.brk!=-1) { + uintptr_t new = ctx.brk + pagesize; + int need_guard = 0; + if (!ctx.brk) { + need_guard = 1; + ctx.brk = brk(0); + // some ancient kernels returned _ebss + // instead of next page as initial brk. + ctx.brk += -ctx.brk & (pagesize-1); + new = ctx.brk + 2*pagesize; + } + if (brk(new) != new) { + ctx.brk = -1; + } else { + if (need_guard) mmap((void *)ctx.brk, pagesize, + PROT_NONE, MAP_ANON|MAP_PRIVATE|MAP_FIXED, -1, 0); + ctx.brk = new; + ctx.avail_meta_areas = (void *)(new - pagesize); + ctx.avail_meta_area_count = pagesize>>12; + need_unprotect = 0; + } + } + if (!ctx.avail_meta_area_count) { + size_t n = 2UL << ctx.meta_alloc_shift; + p = mmap(0, n*pagesize, PROT_NONE, + MAP_PRIVATE|MAP_ANON, -1, 0); + if (p==MAP_FAILED) return 0; + ctx.avail_meta_areas = p + pagesize; + ctx.avail_meta_area_count = (n-1)*(pagesize>>12); + ctx.meta_alloc_shift++; + } + p = ctx.avail_meta_areas; + if ((uintptr_t)p & (pagesize-1)) need_unprotect = 0; + if (need_unprotect) + if (mprotect(p, pagesize, PROT_READ|PROT_WRITE) + && errno != ENOSYS) + return 0; + ctx.avail_meta_area_count--; + ctx.avail_meta_areas = p + 4096; + if (ctx.meta_area_tail) { + ctx.meta_area_tail->next = (void *)p; + } else { + ctx.meta_area_head = (void *)p; + } + ctx.meta_area_tail = (void *)p; + ctx.meta_area_tail->check = ctx.secret; + ctx.avail_meta_count = ctx.meta_area_tail->nslots + = (4096-sizeof(struct meta_area))/sizeof *m; + ctx.avail_meta = ctx.meta_area_tail->slots; + } + ctx.avail_meta_count--; + m = ctx.avail_meta++; + m->prev = m->next = 0; + return m; +} + +static uint32_t try_avail(struct meta **pm) +{ + struct meta *m = *pm; + uint32_t first; + if (!m) return 0; + uint32_t mask = m->avail_mask; + if (!mask) { + if (!m) return 0; + if (!m->freed_mask) { + dequeue(pm, m); + m = *pm; + if (!m) return 0; + } else { + m = m->next; + *pm = m; + } + + mask = m->freed_mask; + + // skip fully-free group unless it's the only one + // or it's a permanently non-freeable group + if (mask == (2u<<m->last_idx)-1 && m->freeable) { + m = m->next; + *pm = m; + mask = m->freed_mask; + } + + // activate more slots in a not-fully-active group + // if needed, but only as a last resort. prefer using + // any other group with free slots. this avoids + // touching & dirtying as-yet-unused pages. + if (!(mask & ((2u<<m->mem->active_idx)-1))) { + if (m->next != m) { + m = m->next; + *pm = m; + } else { + int cnt = m->mem->active_idx + 2; + int size = size_classes[m->sizeclass]*UNIT; + int span = UNIT + size*cnt; + // activate up to next 4k boundary + while ((span^(span+size-1)) < 4096) { + cnt++; + span += size; + } + if (cnt > m->last_idx+1) + cnt = m->last_idx+1; + m->mem->active_idx = cnt-1; + } + } + mask = activate_group(m); + assert(mask); + decay_bounces(m->sizeclass); + } + first = mask&-mask; + m->avail_mask = mask-first; + return first; +} + +static int alloc_slot(int, size_t); + +static struct meta *alloc_group(int sc, size_t req) +{ + size_t size = UNIT*size_classes[sc]; + int i = 0, cnt; + unsigned char *p; + struct meta *m = alloc_meta(); + if (!m) return 0; + size_t usage = ctx.usage_by_class[sc]; + size_t pagesize = PGSZ; + int active_idx; + if (sc < 9) { + while (i<2 && 4*small_cnt_tab[sc][i] > usage) + i++; + cnt = small_cnt_tab[sc][i]; + } else { + // lookup max number of slots fitting in power-of-two size + // from a table, along with number of factors of two we + // can divide out without a remainder or reaching 1. + cnt = med_cnt_tab[sc&3]; + + // reduce cnt to avoid excessive eagar allocation. + while (!(cnt&1) && 4*cnt > usage) + cnt >>= 1; + + // data structures don't support groups whose slot offsets + // in units don't fit in 16 bits. + while (size*cnt >= 65536*UNIT) + cnt >>= 1; + } + + // If we selected a count of 1 above but it's not sufficient to use + // mmap, increase to 2. Then it might be; if not it will nest. + if (cnt==1 && size*cnt+UNIT <= pagesize/2) cnt = 2; + + // All choices of size*cnt are "just below" a power of two, so anything + // larger than half the page size should be allocated as whole pages. + if (size*cnt+UNIT > pagesize/2) { + // check/update bounce counter to start/increase retention + // of freed maps, and inhibit use of low-count, odd-size + // small mappings and single-slot groups if activated. + int nosmall = is_bouncing(sc); + account_bounce(sc); + step_seq(); + + // since the following count reduction opportunities have + // an absolute memory usage cost, don't overdo them. count + // coarse usage as part of usage. + if (!(sc&1) && sc<32) usage += ctx.usage_by_class[sc+1]; + + // try to drop to a lower count if the one found above + // increases usage by more than 25%. these reduced counts + // roughly fill an integral number of pages, just not a + // power of two, limiting amount of unusable space. + if (4*cnt > usage && !nosmall) { + if (0); + else if ((sc&3)==1 && size*cnt>8*pagesize) cnt = 2; + else if ((sc&3)==2 && size*cnt>4*pagesize) cnt = 3; + else if ((sc&3)==0 && size*cnt>8*pagesize) cnt = 3; + else if ((sc&3)==0 && size*cnt>2*pagesize) cnt = 5; + } + size_t needed = size*cnt + UNIT; + needed += -needed & (pagesize-1); + + // produce an individually-mmapped allocation if usage is low, + // bounce counter hasn't triggered, and either it saves memory + // or it avoids eagar slot allocation without wasting too much. + if (!nosmall && cnt<=7) { + req += IB + UNIT; + req += -req & (pagesize-1); + if (req<size+UNIT || (req>=4*pagesize && 2*cnt>usage)) { + cnt = 1; + needed = req; + } + } + + p = mmap(0, needed, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANON, -1, 0); + if (p==MAP_FAILED) { + free_meta(m); + return 0; + } + m->maplen = needed>>12; + ctx.mmap_counter++; + active_idx = (4096-UNIT)/size-1; + if (active_idx > cnt-1) active_idx = cnt-1; + if (active_idx < 0) active_idx = 0; + } else { + int j = size_to_class(UNIT+cnt*size-IB); + int idx = alloc_slot(j, UNIT+cnt*size-IB); + if (idx < 0) { + free_meta(m); + return 0; + } + struct meta *g = ctx.active[j]; + p = enframe(g, idx, UNIT*size_classes[j]-IB, ctx.mmap_counter); + m->maplen = 0; + p[-3] = (p[-3]&31) | (6<<5); + for (int i=0; i<=cnt; i++) + p[UNIT+i*size-4] = 0; + active_idx = cnt-1; + } + ctx.usage_by_class[sc] += cnt; + m->avail_mask = (2u<<active_idx)-1; + m->freed_mask = (2u<<(cnt-1))-1 - m->avail_mask; + m->mem = (void *)p; + m->mem->meta = m; + m->mem->active_idx = active_idx; + m->last_idx = cnt-1; + m->freeable = 1; + m->sizeclass = sc; + return m; +} + +static int alloc_slot(int sc, size_t req) +{ + uint32_t first = try_avail(&ctx.active[sc]); + if (first) return a_ctz_32(first); + + struct meta *g = alloc_group(sc, req); + if (!g) return -1; + + g->avail_mask--; + queue(&ctx.active[sc], g); + return 0; +} + +void *malloc(size_t n) +{ + if (size_overflows(n)) return 0; + struct meta *g; + uint32_t mask, first; + int sc; + int idx; + int ctr; + + if (n >= MMAP_THRESHOLD) { + size_t needed = n + IB + UNIT; + void *p = mmap(0, needed, PROT_READ|PROT_WRITE, + MAP_PRIVATE|MAP_ANON, -1, 0); + if (p==MAP_FAILED) return 0; + wrlock(); + step_seq(); + g = alloc_meta(); + if (!g) { + unlock(); + munmap(p, needed); + return 0; + } + g->mem = p; + g->mem->meta = g; + g->last_idx = 0; + g->freeable = 1; + g->sizeclass = 63; + g->maplen = (needed+4095)/4096; + g->avail_mask = g->freed_mask = 0; + // use a global counter to cycle offset in + // individually-mmapped allocations. + ctx.mmap_counter++; + idx = 0; + goto success; + } + + sc = size_to_class(n); + + rdlock(); + g = ctx.active[sc]; + + // use coarse size classes initially when there are not yet + // any groups of desired size. this allows counts of 2 or 3 + // to be allocated at first rather than having to start with + // 7 or 5, the min counts for even size classes. + if (!g && sc>=4 && sc<32 && sc!=6 && !(sc&1) && !ctx.usage_by_class[sc]) { + size_t usage = ctx.usage_by_class[sc|1]; + // if a new group may be allocated, count it toward + // usage in deciding if we can use coarse class. + if (!ctx.active[sc|1] || (!ctx.active[sc|1]->avail_mask + && !ctx.active[sc|1]->freed_mask)) + usage += 3; + if (usage <= 12) + sc |= 1; + g = ctx.active[sc]; + } + + for (;;) { + mask = g ? g->avail_mask : 0; + first = mask&-mask; + if (!first) break; + if (RDLOCK_IS_EXCLUSIVE || !MT) + g->avail_mask = mask-first; + else if (a_cas(&g->avail_mask, mask, mask-first)!=mask) + continue; + idx = a_ctz_32(first); + goto success; + } + upgradelock(); + + idx = alloc_slot(sc, n); + if (idx < 0) { + unlock(); + return 0; + } + g = ctx.active[sc]; + +success: + ctr = ctx.mmap_counter; + unlock(); + return enframe(g, idx, n, ctr); +} + +int is_allzero(void *p) +{ + struct meta *g = get_meta(p); + return g->sizeclass >= 48 || + get_stride(g) < UNIT*size_classes[g->sizeclass]; +} diff --git a/src/malloc/mallocng/malloc_usable_size.c b/src/malloc/mallocng/malloc_usable_size.c new file mode 100644 index 00000000..ce6a960c --- /dev/null +++ b/src/malloc/mallocng/malloc_usable_size.c @@ -0,0 +1,13 @@ +#include <stdlib.h> +#include "meta.h" + +size_t malloc_usable_size(void *p) +{ + if (!p) return 0; + struct meta *g = get_meta(p); + int idx = get_slot_index(p); + size_t stride = get_stride(g); + unsigned char *start = g->mem->storage + stride*idx; + unsigned char *end = start + stride - IB; + return get_nominal_size(p, end); +} diff --git a/src/malloc/mallocng/meta.h b/src/malloc/mallocng/meta.h new file mode 100644 index 00000000..61ec53f9 --- /dev/null +++ b/src/malloc/mallocng/meta.h @@ -0,0 +1,288 @@ +#ifndef MALLOC_META_H +#define MALLOC_META_H + +#include <stdint.h> +#include <errno.h> +#include <limits.h> +#include "glue.h" + +__attribute__((__visibility__("hidden"))) +extern const uint16_t size_classes[]; + +#define MMAP_THRESHOLD 131052 + +#define UNIT 16 +#define IB 4 + +struct group { + struct meta *meta; + unsigned char active_idx:5; + char pad[UNIT - sizeof(struct meta *) - 1]; + unsigned char storage[]; +}; + +struct meta { + struct meta *prev, *next; + struct group *mem; + volatile int avail_mask, freed_mask; + uintptr_t last_idx:5; + uintptr_t freeable:1; + uintptr_t sizeclass:6; + uintptr_t maplen:8*sizeof(uintptr_t)-12; +}; + +struct meta_area { + uint64_t check; + struct meta_area *next; + int nslots; + struct meta slots[]; +}; + +struct malloc_context { + uint64_t secret; +#ifndef PAGESIZE + size_t pagesize; +#endif + int init_done; + unsigned mmap_counter; + struct meta *free_meta_head; + struct meta *avail_meta; + size_t avail_meta_count, avail_meta_area_count, meta_alloc_shift; + struct meta_area *meta_area_head, *meta_area_tail; + unsigned char *avail_meta_areas; + struct meta *active[48]; + size_t usage_by_class[48]; + uint8_t unmap_seq[32], bounces[32]; + uint8_t seq; + uintptr_t brk; +}; + +__attribute__((__visibility__("hidden"))) +extern struct malloc_context ctx; + +#ifdef PAGESIZE +#define PGSZ PAGESIZE +#else +#define PGSZ ctx.pagesize +#endif + +__attribute__((__visibility__("hidden"))) +struct meta *alloc_meta(void); + +__attribute__((__visibility__("hidden"))) +int is_allzero(void *); + +static inline void queue(struct meta **phead, struct meta *m) +{ + assert(!m->next); + assert(!m->prev); + if (*phead) { + struct meta *head = *phead; + m->next = head; + m->prev = head->prev; + m->next->prev = m->prev->next = m; + } else { + m->prev = m->next = m; + *phead = m; + } +} + +static inline void dequeue(struct meta **phead, struct meta *m) +{ + if (m->next != m) { + m->prev->next = m->next; + m->next->prev = m->prev; + if (*phead == m) *phead = m->next; + } else { + *phead = 0; + } + m->prev = m->next = 0; +} + +static inline struct meta *dequeue_head(struct meta **phead) +{ + struct meta *m = *phead; + if (m) dequeue(phead, m); + return m; +} + +static inline void free_meta(struct meta *m) +{ + *m = (struct meta){0}; + queue(&ctx.free_meta_head, m); +} + +static inline uint32_t activate_group(struct meta *m) +{ + assert(!m->avail_mask); + uint32_t mask, act = (2u<<m->mem->active_idx)-1; + do mask = m->freed_mask; + while (a_cas(&m->freed_mask, mask, mask&~act)!=mask); + return m->avail_mask = mask & act; +} + +static inline int get_slot_index(const unsigned char *p) +{ + return p[-3] & 31; +} + +static inline struct meta *get_meta(const unsigned char *p) +{ + assert(!((uintptr_t)p & 15)); + int offset = *(const uint16_t *)(p - 2); + int index = get_slot_index(p); + if (p[-4]) { + assert(!offset); + offset = *(uint32_t *)(p - 8); + assert(offset > 0xffff); + } + const struct group *base = (const void *)(p - UNIT*offset - UNIT); + const struct meta *meta = base->meta; + assert(meta->mem == base); + assert(index <= meta->last_idx); + assert(!(meta->avail_mask & (1u<<index))); + assert(!(meta->freed_mask & (1u<<index))); + const struct meta_area *area = (void *)((uintptr_t)meta & -4096); + assert(area->check == ctx.secret); + if (meta->sizeclass < 48) { + assert(offset >= size_classes[meta->sizeclass]*index); + assert(offset < size_classes[meta->sizeclass]*(index+1)); + } else { + assert(meta->sizeclass == 63); + } + if (meta->maplen) { + assert(offset <= meta->maplen*4096UL/UNIT - 1); + } + return (struct meta *)meta; +} + +static inline size_t get_nominal_size(const unsigned char *p, const unsigned char *end) +{ + size_t reserved = p[-3] >> 5; + if (reserved >= 5) { + assert(reserved == 5); + reserved = *(const uint32_t *)(end-4); + assert(reserved >= 5); + assert(!end[-5]); + } + assert(reserved <= end-p); + assert(!*(end-reserved)); + // also check the slot's overflow byte + assert(!*end); + return end-reserved-p; +} + +static inline size_t get_stride(const struct meta *g) +{ + if (!g->last_idx && g->maplen) { + return g->maplen*4096UL - UNIT; + } else { + return UNIT*size_classes[g->sizeclass]; + } +} + +static inline void set_size(unsigned char *p, unsigned char *end, size_t n) +{ + int reserved = end-p-n; + if (reserved) end[-reserved] = 0; + if (reserved >= 5) { + *(uint32_t *)(end-4) = reserved; + end[-5] = 0; + reserved = 5; + } + p[-3] = (p[-3]&31) + (reserved<<5); +} + +static inline void *enframe(struct meta *g, int idx, size_t n, int ctr) +{ + size_t stride = get_stride(g); + size_t slack = (stride-IB-n)/UNIT; + unsigned char *p = g->mem->storage + stride*idx; + unsigned char *end = p+stride-IB; + // cycle offset within slot to increase interval to address + // reuse, facilitate trapping double-free. + int off = (p[-3] ? *(uint16_t *)(p-2) + 1 : ctr) & 255; + assert(!p[-4]); + if (off > slack) { + size_t m = slack; + m |= m>>1; m |= m>>2; m |= m>>4; + off &= m; + if (off > slack) off -= slack+1; + assert(off <= slack); + } + if (off) { + // store offset in unused header at offset zero + // if enframing at non-zero offset. + *(uint16_t *)(p-2) = off; + p[-3] = 7<<5; + p += UNIT*off; + // for nonzero offset there is no permanent check + // byte, so make one. + p[-4] = 0; + } + *(uint16_t *)(p-2) = (size_t)(p-g->mem->storage)/UNIT; + p[-3] = idx; + set_size(p, end, n); + return p; +} + +static inline int size_to_class(size_t n) +{ + n = (n+IB-1)>>4; + if (n<10) return n; + n++; + int i = (28-a_clz_32(n))*4 + 8; + if (n>size_classes[i+1]) i+=2; + if (n>size_classes[i]) i++; + return i; +} + +static inline int size_overflows(size_t n) +{ + if (n >= SIZE_MAX/2 - 4096) { + errno = ENOMEM; + return 1; + } + return 0; +} + +static inline void step_seq(void) +{ + if (ctx.seq==255) { + for (int i=0; i<32; i++) ctx.unmap_seq[i] = 0; + ctx.seq = 1; + } else { + ctx.seq++; + } +} + +static inline void record_seq(int sc) +{ + if (sc-7U < 32) ctx.unmap_seq[sc-7] = ctx.seq; +} + +static inline void account_bounce(int sc) +{ + if (sc-7U < 32) { + int seq = ctx.unmap_seq[sc-7]; + if (seq && ctx.seq-seq < 10) { + if (ctx.bounces[sc-7]+1 < 100) + ctx.bounces[sc-7]++; + else + ctx.bounces[sc-7] = 150; + } + } +} + +static inline void decay_bounces(int sc) +{ + if (sc-7U < 32 && ctx.bounces[sc-7]) + ctx.bounces[sc-7]--; +} + +static inline int is_bouncing(int sc) +{ + return (sc-7U < 32 && ctx.bounces[sc-7] >= 100); +} + +#endif diff --git a/src/malloc/mallocng/realloc.c b/src/malloc/mallocng/realloc.c new file mode 100644 index 00000000..18769f42 --- /dev/null +++ b/src/malloc/mallocng/realloc.c @@ -0,0 +1,51 @@ +#define _GNU_SOURCE +#include <stdlib.h> +#include <sys/mman.h> +#include <string.h> +#include "meta.h" + +void *realloc(void *p, size_t n) +{ + if (!p) return malloc(n); + if (size_overflows(n)) return 0; + + struct meta *g = get_meta(p); + int idx = get_slot_index(p); + size_t stride = get_stride(g); + unsigned char *start = g->mem->storage + stride*idx; + unsigned char *end = start + stride - IB; + size_t old_size = get_nominal_size(p, end); + size_t avail_size = end-(unsigned char *)p; + void *new; + + // only resize in-place if size class matches + if (n <= avail_size && n<MMAP_THRESHOLD + && size_to_class(n)+1 >= g->sizeclass) { + set_size(p, end, n); + return p; + } + + // use mremap if old and new size are both mmap-worthy + if (g->sizeclass>=48 && n>=MMAP_THRESHOLD) { + assert(g->sizeclass==63); + size_t base = (unsigned char *)p-start; + size_t needed = (n + base + UNIT + IB + 4095) & -4096; + new = g->maplen*4096UL == needed ? g->mem : + mremap(g->mem, g->maplen*4096UL, needed, MREMAP_MAYMOVE); + if (new!=MAP_FAILED) { + g->mem = new; + g->maplen = needed/4096; + p = g->mem->storage + base; + end = g->mem->storage + (needed - UNIT) - IB; + *end = 0; + set_size(p, end, n); + return p; + } + } + + new = malloc(n); + if (!new) return 0; + memcpy(new, p, n < old_size ? n : old_size); + free(p); + return new; +} diff --git a/src/malloc/memalign.c b/src/malloc/memalign.c index cf9dfbda..32cd87d8 100644 --- a/src/malloc/memalign.c +++ b/src/malloc/memalign.c @@ -1,54 +1,7 @@ +#define _BSD_SOURCE #include <stdlib.h> -#include <stdint.h> -#include <errno.h> -#include "malloc_impl.h" -void *__memalign(size_t align, size_t len) +void *memalign(size_t align, size_t len) { - unsigned char *mem, *new; - - if ((align & -align) != align) { - errno = EINVAL; - return 0; - } - - if (len > SIZE_MAX - align || __malloc_replaced) { - errno = ENOMEM; - return 0; - } - - if (align <= SIZE_ALIGN) - return malloc(len); - - if (!(mem = malloc(len + align-1))) - return 0; - - new = (void *)((uintptr_t)mem + align-1 & -align); - if (new == mem) return mem; - - struct chunk *c = MEM_TO_CHUNK(mem); - struct chunk *n = MEM_TO_CHUNK(new); - - if (IS_MMAPPED(c)) { - /* Apply difference between aligned and original - * address to the "extra" field of mmapped chunk. */ - n->psize = c->psize + (new-mem); - n->csize = c->csize - (new-mem); - return new; - } - - struct chunk *t = NEXT_CHUNK(c); - - /* Split the allocated chunk into two chunks. The aligned part - * that will be used has the size in its footer reduced by the - * difference between the aligned and original addresses, and - * the resulting size copied to its header. A new header and - * footer are written for the split-off part to be freed. */ - n->psize = c->csize = C_INUSE | (new-mem); - n->csize = t->psize -= new-mem; - - __bin_chunk(c); - return new; + return aligned_alloc(align, len); } - -weak_alias(__memalign, memalign); diff --git a/src/malloc/oldmalloc/aligned_alloc.c b/src/malloc/oldmalloc/aligned_alloc.c new file mode 100644 index 00000000..4adca3b4 --- /dev/null +++ b/src/malloc/oldmalloc/aligned_alloc.c @@ -0,0 +1,53 @@ +#include <stdlib.h> +#include <stdint.h> +#include <errno.h> +#include "malloc_impl.h" + +void *aligned_alloc(size_t align, size_t len) +{ + unsigned char *mem, *new; + + if ((align & -align) != align) { + errno = EINVAL; + return 0; + } + + if (len > SIZE_MAX - align || + (__malloc_replaced && !__aligned_alloc_replaced)) { + errno = ENOMEM; + return 0; + } + + if (align <= SIZE_ALIGN) + return malloc(len); + + if (!(mem = malloc(len + align-1))) + return 0; + + new = (void *)((uintptr_t)mem + align-1 & -align); + if (new == mem) return mem; + + struct chunk *c = MEM_TO_CHUNK(mem); + struct chunk *n = MEM_TO_CHUNK(new); + + if (IS_MMAPPED(c)) { + /* Apply difference between aligned and original + * address to the "extra" field of mmapped chunk. */ + n->psize = c->psize + (new-mem); + n->csize = c->csize - (new-mem); + return new; + } + + struct chunk *t = NEXT_CHUNK(c); + + /* Split the allocated chunk into two chunks. The aligned part + * that will be used has the size in its footer reduced by the + * difference between the aligned and original addresses, and + * the resulting size copied to its header. A new header and + * footer are written for the split-off part to be freed. */ + n->psize = c->csize = C_INUSE | (new-mem); + n->csize = t->psize -= new-mem; + + __bin_chunk(c); + return new; +} diff --git a/src/malloc/malloc.c b/src/malloc/oldmalloc/malloc.c index 96982596..25d00d44 100644 --- a/src/malloc/malloc.c +++ b/src/malloc/oldmalloc/malloc.c @@ -9,6 +9,11 @@ #include "atomic.h" #include "pthread_impl.h" #include "malloc_impl.h" +#include "fork_impl.h" + +#define malloc __libc_malloc_impl +#define realloc __libc_realloc +#define free __libc_free #if defined(__GNUC__) && defined(__PIC__) #define inline inline __attribute__((always_inline)) @@ -17,17 +22,18 @@ static struct { volatile uint64_t binmap; struct bin bins[64]; - volatile int free_lock[2]; + volatile int split_merge_lock[2]; } mal; -int __malloc_replaced; - /* Synchronization tools */ static inline void lock(volatile int *lk) { - if (libc.threads_minus_1) + int need_locks = libc.need_locks; + if (need_locks) { while(a_swap(lk, 1)) __wait(lk, lk+1, 1, 1); + if (need_locks < 0) libc.need_locks = 0; + } } static inline void unlock(volatile int *lk) @@ -123,9 +129,72 @@ void __dump_heap(int x) } #endif +/* This function returns true if the interval [old,new] + * intersects the 'len'-sized interval below &libc.auxv + * (interpreted as the main-thread stack) or below &b + * (the current stack). It is used to defend against + * buggy brk implementations that can cross the stack. */ + +static int traverses_stack_p(uintptr_t old, uintptr_t new) +{ + const uintptr_t len = 8<<20; + uintptr_t a, b; + + b = (uintptr_t)libc.auxv; + a = b > len ? b-len : 0; + if (new>a && old<b) return 1; + + b = (uintptr_t)&b; + a = b > len ? b-len : 0; + if (new>a && old<b) return 1; + + return 0; +} + +/* Expand the heap in-place if brk can be used, or otherwise via mmap, + * using an exponential lower bound on growth by mmap to make + * fragmentation asymptotically irrelevant. The size argument is both + * an input and an output, since the caller needs to know the size + * allocated, which will be larger than requested due to page alignment + * and mmap minimum size rules. The caller is responsible for locking + * to prevent concurrent calls. */ + +static void *__expand_heap(size_t *pn) +{ + static uintptr_t brk; + static unsigned mmap_step; + size_t n = *pn; + + if (n > SIZE_MAX/2 - PAGE_SIZE) { + errno = ENOMEM; + return 0; + } + n += -n & PAGE_SIZE-1; + + if (!brk) { + brk = __syscall(SYS_brk, 0); + brk += -brk & PAGE_SIZE-1; + } + + if (n < SIZE_MAX-brk && !traverses_stack_p(brk, brk+n) + && __syscall(SYS_brk, brk+n)==brk+n) { + *pn = n; + brk += n; + return (void *)(brk-n); + } + + size_t min = (size_t)PAGE_SIZE << mmap_step/2; + if (n < min) n = min; + void *area = __mmap(0, n, PROT_READ|PROT_WRITE, + MAP_PRIVATE|MAP_ANONYMOUS, -1, 0); + if (area == MAP_FAILED) return 0; + *pn = n; + mmap_step++; + return area; +} + static struct chunk *expand_heap(size_t n) { - static int heap_lock[2]; static void *end; void *p; struct chunk *w; @@ -135,13 +204,8 @@ static struct chunk *expand_heap(size_t n) * we need room for an extra zero-sized sentinel chunk. */ n += SIZE_ALIGN; - lock(heap_lock); - p = __expand_heap(&n); - if (!p) { - unlock(heap_lock); - return 0; - } + if (!p) return 0; /* If not just expanding existing space, we need to make a * new sentinel chunk below the allocated space. */ @@ -164,8 +228,6 @@ static struct chunk *expand_heap(size_t n) w = MEM_TO_CHUNK(p); w->csize = n | C_INUSE; - unlock(heap_lock); - return w; } @@ -195,96 +257,44 @@ static void unbin(struct chunk *c, int i) NEXT_CHUNK(c)->psize |= C_INUSE; } -static int alloc_fwd(struct chunk *c) +static void bin_chunk(struct chunk *self, int i) { - int i; - size_t k; - while (!((k=c->csize) & C_INUSE)) { - i = bin_index(k); - lock_bin(i); - if (c->csize == k) { - unbin(c, i); - unlock_bin(i); - return 1; - } - unlock_bin(i); - } - return 0; -} - -static int alloc_rev(struct chunk *c) -{ - int i; - size_t k; - while (!((k=c->psize) & C_INUSE)) { - i = bin_index(k); - lock_bin(i); - if (c->psize == k) { - unbin(PREV_CHUNK(c), i); - unlock_bin(i); - return 1; - } - unlock_bin(i); - } - return 0; + self->next = BIN_TO_CHUNK(i); + self->prev = mal.bins[i].tail; + self->next->prev = self; + self->prev->next = self; + if (self->prev == BIN_TO_CHUNK(i)) + a_or_64(&mal.binmap, 1ULL<<i); } - -/* pretrim - trims a chunk _prior_ to removing it from its bin. - * Must be called with i as the ideal bin for size n, j the bin - * for the _free_ chunk self, and bin j locked. */ -static int pretrim(struct chunk *self, size_t n, int i, int j) +static void trim(struct chunk *self, size_t n) { - size_t n1; + size_t n1 = CHUNK_SIZE(self); struct chunk *next, *split; - /* We cannot pretrim if it would require re-binning. */ - if (j < 40) return 0; - if (j < i+3) { - if (j != 63) return 0; - n1 = CHUNK_SIZE(self); - if (n1-n <= MMAP_THRESHOLD) return 0; - } else { - n1 = CHUNK_SIZE(self); - } - if (bin_index(n1-n) != j) return 0; + if (n >= n1 - DONTCARE) return; next = NEXT_CHUNK(self); split = (void *)((char *)self + n); - split->prev = self->prev; - split->next = self->next; - split->prev->next = split; - split->next->prev = split; split->psize = n | C_INUSE; split->csize = n1-n; next->psize = n1-n; self->csize = n | C_INUSE; - return 1; -} -static void trim(struct chunk *self, size_t n) -{ - size_t n1 = CHUNK_SIZE(self); - struct chunk *next, *split; + int i = bin_index(n1-n); + lock_bin(i); - if (n >= n1 - DONTCARE) return; + bin_chunk(split, i); - next = NEXT_CHUNK(self); - split = (void *)((char *)self + n); - - split->psize = n | C_INUSE; - split->csize = n1-n | C_INUSE; - next->psize = n1-n | C_INUSE; - self->csize = n | C_INUSE; - - __bin_chunk(split); + unlock_bin(i); } void *malloc(size_t n) { struct chunk *c; int i, j; + uint64_t mask; if (adjust_size(&n) < 0) return 0; @@ -300,70 +310,43 @@ void *malloc(size_t n) } i = bin_index_up(n); - for (;;) { - uint64_t mask = mal.binmap & -(1ULL<<i); - if (!mask) { - c = expand_heap(n); - if (!c) return 0; - if (alloc_rev(c)) { - struct chunk *x = c; - c = PREV_CHUNK(c); - NEXT_CHUNK(x)->psize = c->csize = - x->csize + CHUNK_SIZE(c); - } - break; + if (i<63 && (mal.binmap & (1ULL<<i))) { + lock_bin(i); + c = mal.bins[i].head; + if (c != BIN_TO_CHUNK(i) && CHUNK_SIZE(c)-n <= DONTCARE) { + unbin(c, i); + unlock_bin(i); + return CHUNK_TO_MEM(c); } + unlock_bin(i); + } + lock(mal.split_merge_lock); + for (mask = mal.binmap & -(1ULL<<i); mask; mask -= (mask&-mask)) { j = first_set(mask); lock_bin(j); c = mal.bins[j].head; if (c != BIN_TO_CHUNK(j)) { - if (!pretrim(c, n, i, j)) unbin(c, j); + unbin(c, j); unlock_bin(j); break; } unlock_bin(j); } - - /* Now patch up in case we over-allocated */ + if (!mask) { + c = expand_heap(n); + if (!c) { + unlock(mal.split_merge_lock); + return 0; + } + } trim(c, n); - + unlock(mal.split_merge_lock); return CHUNK_TO_MEM(c); } -static size_t mal0_clear(char *p, size_t pagesz, size_t n) +int __malloc_allzerop(void *p) { -#ifdef __GNUC__ - typedef uint64_t __attribute__((__may_alias__)) T; -#else - typedef unsigned char T; -#endif - char *pp = p + n; - size_t i = (uintptr_t)pp & (pagesz - 1); - for (;;) { - pp = memset(pp - i, 0, i); - if (pp - p < pagesz) return pp - p; - for (i = pagesz; i; i -= 2*sizeof(T), pp -= 2*sizeof(T)) - if (((T *)pp)[-1] | ((T *)pp)[-2]) - break; - } -} - -void *calloc(size_t m, size_t n) -{ - if (n && m > (size_t)-1/n) { - errno = ENOMEM; - return 0; - } - n *= m; - void *p = malloc(n); - if (!p) return p; - if (!__malloc_replaced) { - if (IS_MMAPPED(MEM_TO_CHUNK(p))) - return p; - if (n >= PAGE_SIZE) - n = mal0_clear(p, PAGE_SIZE, n); - } - return memset(p, 0, n); + return IS_MMAPPED(MEM_TO_CHUNK(p)); } void *realloc(void *p, size_t n) @@ -379,6 +362,8 @@ void *realloc(void *p, size_t n) self = MEM_TO_CHUNK(p); n1 = n0 = CHUNK_SIZE(self); + if (n<=n0 && n0-n<=DONTCARE) return p; + if (IS_MMAPPED(self)) { size_t extra = self->psize; char *base = (char *)self - extra; @@ -405,34 +390,43 @@ void *realloc(void *p, size_t n) /* Crash on corrupted footer (likely from buffer overflow) */ if (next->psize != self->csize) a_crash(); - /* Merge adjacent chunks if we need more space. This is not - * a waste of time even if we fail to get enough space, because our - * subsequent call to free would otherwise have to do the merge. */ - if (n > n1 && alloc_fwd(next)) { - n1 += CHUNK_SIZE(next); - next = NEXT_CHUNK(next); - } - /* FIXME: find what's wrong here and reenable it..? */ - if (0 && n > n1 && alloc_rev(self)) { - self = PREV_CHUNK(self); - n1 += CHUNK_SIZE(self); + if (n < n0) { + int i = bin_index_up(n); + int j = bin_index(n0); + if (i<j && (mal.binmap & (1ULL << i))) + goto copy_realloc; + struct chunk *split = (void *)((char *)self + n); + self->csize = split->psize = n | C_INUSE; + split->csize = next->psize = n0-n | C_INUSE; + __bin_chunk(split); + return CHUNK_TO_MEM(self); } - self->csize = n1 | C_INUSE; - next->psize = n1 | C_INUSE; - /* If we got enough space, split off the excess and return */ - if (n <= n1) { - //memmove(CHUNK_TO_MEM(self), p, n0-OVERHEAD); - trim(self, n); - return CHUNK_TO_MEM(self); + lock(mal.split_merge_lock); + + size_t nsize = next->csize & C_INUSE ? 0 : CHUNK_SIZE(next); + if (n0+nsize >= n) { + int i = bin_index(nsize); + lock_bin(i); + if (!(next->csize & C_INUSE)) { + unbin(next, i); + unlock_bin(i); + next = NEXT_CHUNK(next); + self->csize = next->psize = n0+nsize | C_INUSE; + trim(self, n); + unlock(mal.split_merge_lock); + return CHUNK_TO_MEM(self); + } + unlock_bin(i); } + unlock(mal.split_merge_lock); copy_realloc: /* As a last resort, allocate a new chunk and copy to it. */ new = malloc(n-OVERHEAD); if (!new) return 0; copy_free_ret: - memcpy(new, p, n0-OVERHEAD); + memcpy(new, p, (n<n0 ? n : n0) - OVERHEAD); free(CHUNK_TO_MEM(self)); return new; } @@ -440,67 +434,61 @@ copy_free_ret: void __bin_chunk(struct chunk *self) { struct chunk *next = NEXT_CHUNK(self); - size_t final_size, new_size, size; - int reclaim=0; - int i; - - final_size = new_size = CHUNK_SIZE(self); /* Crash on corrupted footer (likely from buffer overflow) */ if (next->psize != self->csize) a_crash(); - for (;;) { - if (self->psize & next->csize & C_INUSE) { - self->csize = final_size | C_INUSE; - next->psize = final_size | C_INUSE; - i = bin_index(final_size); - lock_bin(i); - lock(mal.free_lock); - if (self->psize & next->csize & C_INUSE) - break; - unlock(mal.free_lock); - unlock_bin(i); - } + lock(mal.split_merge_lock); - if (alloc_rev(self)) { - self = PREV_CHUNK(self); - size = CHUNK_SIZE(self); - final_size += size; - if (new_size+size > RECLAIM && (new_size+size^size) > size) - reclaim = 1; - } + size_t osize = CHUNK_SIZE(self), size = osize; + + /* Since we hold split_merge_lock, only transition from free to + * in-use can race; in-use to free is impossible */ + size_t psize = self->psize & C_INUSE ? 0 : CHUNK_PSIZE(self); + size_t nsize = next->csize & C_INUSE ? 0 : CHUNK_SIZE(next); - if (alloc_fwd(next)) { - size = CHUNK_SIZE(next); - final_size += size; - if (new_size+size > RECLAIM && (new_size+size^size) > size) - reclaim = 1; + if (psize) { + int i = bin_index(psize); + lock_bin(i); + if (!(self->psize & C_INUSE)) { + struct chunk *prev = PREV_CHUNK(self); + unbin(prev, i); + self = prev; + size += psize; + } + unlock_bin(i); + } + if (nsize) { + int i = bin_index(nsize); + lock_bin(i); + if (!(next->csize & C_INUSE)) { + unbin(next, i); next = NEXT_CHUNK(next); + size += nsize; } + unlock_bin(i); } - if (!(mal.binmap & 1ULL<<i)) - a_or_64(&mal.binmap, 1ULL<<i); + int i = bin_index(size); + lock_bin(i); - self->csize = final_size; - next->psize = final_size; - unlock(mal.free_lock); - - self->next = BIN_TO_CHUNK(i); - self->prev = mal.bins[i].tail; - self->next->prev = self; - self->prev->next = self; + self->csize = size; + next->psize = size; + bin_chunk(self, i); + unlock(mal.split_merge_lock); /* Replace middle of large chunks with fresh zero pages */ - if (reclaim) { + if (size > RECLAIM && (size^(size-osize)) > size-osize) { uintptr_t a = (uintptr_t)self + SIZE_ALIGN+PAGE_SIZE-1 & -PAGE_SIZE; uintptr_t b = (uintptr_t)next - SIZE_ALIGN & -PAGE_SIZE; + int e = errno; #if 1 __madvise((void *)a, b-a, MADV_DONTNEED); #else __mmap((void *)a, b-a, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS|MAP_FIXED, -1, 0); #endif + errno = e; } unlock_bin(i); @@ -513,7 +501,9 @@ static void unmap_chunk(struct chunk *self) size_t len = CHUNK_SIZE(self) + extra; /* Crash on double free */ if (extra & 1) a_crash(); + int e = errno; __munmap(base, len); + errno = e; } void free(void *p) @@ -546,3 +536,21 @@ void __malloc_donate(char *start, char *end) c->csize = n->psize = C_INUSE | (end-start); __bin_chunk(c); } + +void __malloc_atfork(int who) +{ + if (who<0) { + lock(mal.split_merge_lock); + for (int i=0; i<64; i++) + lock(mal.bins[i].lock); + } else if (!who) { + for (int i=0; i<64; i++) + unlock(mal.bins[i].lock); + unlock(mal.split_merge_lock); + } else { + for (int i=0; i<64; i++) + mal.bins[i].lock[0] = mal.bins[i].lock[1] = 0; + mal.split_merge_lock[1] = 0; + mal.split_merge_lock[0] = 0; + } +} diff --git a/src/internal/malloc_impl.h b/src/malloc/oldmalloc/malloc_impl.h index 59785a7f..e1cf4774 100644 --- a/src/internal/malloc_impl.h +++ b/src/malloc/oldmalloc/malloc_impl.h @@ -2,12 +2,7 @@ #define MALLOC_IMPL_H #include <sys/mman.h> - -hidden void *__expand_heap(size_t *); - -hidden void __malloc_donate(char *, char *); - -hidden void *__memalign(size_t, size_t); +#include "dynlink.h" struct chunk { size_t psize, csize; @@ -41,6 +36,4 @@ struct bin { hidden void __bin_chunk(struct chunk *); -hidden extern int __malloc_replaced; - #endif diff --git a/src/malloc/malloc_usable_size.c b/src/malloc/oldmalloc/malloc_usable_size.c index 672b518a..672b518a 100644 --- a/src/malloc/malloc_usable_size.c +++ b/src/malloc/oldmalloc/malloc_usable_size.c diff --git a/src/malloc/posix_memalign.c b/src/malloc/posix_memalign.c index 2ea8bd8a..ad4d8f47 100644 --- a/src/malloc/posix_memalign.c +++ b/src/malloc/posix_memalign.c @@ -1,11 +1,10 @@ #include <stdlib.h> #include <errno.h> -#include "malloc_impl.h" int posix_memalign(void **res, size_t align, size_t len) { if (align < sizeof(void *)) return EINVAL; - void *mem = __memalign(align, len); + void *mem = aligned_alloc(align, len); if (!mem) return errno; *res = mem; return 0; diff --git a/src/malloc/realloc.c b/src/malloc/realloc.c new file mode 100644 index 00000000..fb0e8b7c --- /dev/null +++ b/src/malloc/realloc.c @@ -0,0 +1,6 @@ +#include <stdlib.h> + +void *realloc(void *p, size_t n) +{ + return __libc_realloc(p, n); +} diff --git a/src/malloc/reallocarray.c b/src/malloc/reallocarray.c new file mode 100644 index 00000000..4a6ebe46 --- /dev/null +++ b/src/malloc/reallocarray.c @@ -0,0 +1,13 @@ +#define _BSD_SOURCE +#include <errno.h> +#include <stdlib.h> + +void *reallocarray(void *ptr, size_t m, size_t n) +{ + if (n && m > -1 / n) { + errno = ENOMEM; + return 0; + } + + return realloc(ptr, m * n); +} diff --git a/src/malloc/replaced.c b/src/malloc/replaced.c new file mode 100644 index 00000000..07fce61e --- /dev/null +++ b/src/malloc/replaced.c @@ -0,0 +1,4 @@ +#include "dynlink.h" + +int __malloc_replaced; +int __aligned_alloc_replaced; diff --git a/src/math/__expo2.c b/src/math/__expo2.c index 740ac680..248f052b 100644 --- a/src/math/__expo2.c +++ b/src/math/__expo2.c @@ -5,12 +5,13 @@ static const int k = 2043; static const double kln2 = 0x1.62066151add8bp+10; /* exp(x)/2 for x >= log(DBL_MAX), slightly better than 0.5*exp(x/2)*exp(x/2) */ -double __expo2(double x) +double __expo2(double x, double sign) { double scale; /* note that k is odd and scale*scale overflows */ INSERT_WORDS(scale, (uint32_t)(0x3ff + k/2) << 20, 0); /* exp(x - k ln2) * 2**(k-1) */ - return exp(x - kln2) * scale * scale; + /* in directed rounding correct sign before rounding or overflow is important */ + return exp(x - kln2) * (sign * scale) * scale; } diff --git a/src/math/__expo2f.c b/src/math/__expo2f.c index 5163e418..538eb09c 100644 --- a/src/math/__expo2f.c +++ b/src/math/__expo2f.c @@ -5,12 +5,13 @@ static const int k = 235; static const float kln2 = 0x1.45c778p+7f; /* expf(x)/2 for x >= log(FLT_MAX), slightly better than 0.5f*expf(x/2)*expf(x/2) */ -float __expo2f(float x) +float __expo2f(float x, float sign) { float scale; /* note that k is odd and scale*scale overflows */ SET_FLOAT_WORD(scale, (uint32_t)(0x7f + k/2) << 23); /* exp(x - k ln2) * 2**(k-1) */ - return expf(x - kln2) * scale * scale; + /* in directed rounding correct sign before rounding or overflow is important */ + return expf(x - kln2) * (sign * scale) * scale; } diff --git a/src/math/__math_invalidl.c b/src/math/__math_invalidl.c new file mode 100644 index 00000000..1fca99de --- /dev/null +++ b/src/math/__math_invalidl.c @@ -0,0 +1,9 @@ +#include <float.h> +#include "libm.h" + +#if LDBL_MANT_DIG != DBL_MANT_DIG +long double __math_invalidl(long double x) +{ + return (x - x) / (x - x); +} +#endif diff --git a/src/math/__rem_pio2.c b/src/math/__rem_pio2.c index d403f81c..dcf672fb 100644 --- a/src/math/__rem_pio2.c +++ b/src/math/__rem_pio2.c @@ -36,6 +36,7 @@ */ static const double toint = 1.5/EPS, +pio4 = 0x1.921fb54442d18p-1, invpio2 = 6.36619772367581382433e-01, /* 0x3FE45F30, 0x6DC9C883 */ pio2_1 = 1.57079632673412561417e+00, /* 0x3FF921FB, 0x54400000 */ pio2_1t = 6.07710050650619224932e-11, /* 0x3DD0B461, 0x1A626331 */ @@ -117,11 +118,23 @@ int __rem_pio2(double x, double *y) } if (ix < 0x413921fb) { /* |x| ~< 2^20*(pi/2), medium size */ medium: - /* rint(x/(pi/2)), Assume round-to-nearest. */ + /* rint(x/(pi/2)) */ fn = (double_t)x*invpio2 + toint - toint; n = (int32_t)fn; r = x - fn*pio2_1; w = fn*pio2_1t; /* 1st round, good to 85 bits */ + /* Matters with directed rounding. */ + if (predict_false(r - w < -pio4)) { + n--; + fn--; + r = x - fn*pio2_1; + w = fn*pio2_1t; + } else if (predict_false(r - w > pio4)) { + n++; + fn++; + r = x - fn*pio2_1; + w = fn*pio2_1t; + } y[0] = r - w; u.f = y[0]; ey = u.i>>52 & 0x7ff; diff --git a/src/math/__rem_pio2f.c b/src/math/__rem_pio2f.c index 4473c1c4..e6765643 100644 --- a/src/math/__rem_pio2f.c +++ b/src/math/__rem_pio2f.c @@ -35,6 +35,7 @@ */ static const double toint = 1.5/EPS, +pio4 = 0x1.921fb6p-1, invpio2 = 6.36619772367581382433e-01, /* 0x3FE45F30, 0x6DC9C883 */ pio2_1 = 1.57079631090164184570e+00, /* 0x3FF921FB, 0x50000000 */ pio2_1t = 1.58932547735281966916e-08; /* 0x3E5110b4, 0x611A6263 */ @@ -50,10 +51,20 @@ int __rem_pio2f(float x, double *y) ix = u.i & 0x7fffffff; /* 25+53 bit pi is good enough for medium size */ if (ix < 0x4dc90fdb) { /* |x| ~< 2^28*(pi/2), medium size */ - /* Use a specialized rint() to get fn. Assume round-to-nearest. */ + /* Use a specialized rint() to get fn. */ fn = (double_t)x*invpio2 + toint - toint; n = (int32_t)fn; *y = x - fn*pio2_1 - fn*pio2_1t; + /* Matters with directed rounding. */ + if (predict_false(*y < -pio4)) { + n--; + fn--; + *y = x - fn*pio2_1 - fn*pio2_1t; + } else if (predict_false(*y > pio4)) { + n++; + fn++; + *y = x - fn*pio2_1 - fn*pio2_1t; + } return n; } if(ix>=0x7f800000) { /* x is inf or NaN */ diff --git a/src/math/__rem_pio2l.c b/src/math/__rem_pio2l.c index 77255bd8..236b2def 100644 --- a/src/math/__rem_pio2l.c +++ b/src/math/__rem_pio2l.c @@ -44,6 +44,7 @@ pio2_1 = 1.57079632679597125389e+00, /* 0x3FF921FB, 0x54444000 */ pio2_2 = -1.07463465549783099519e-12, /* -0x12e7b967674000.0p-92 */ pio2_3 = 6.36831716351370313614e-25; /* 0x18a2e037074000.0p-133 */ static const long double +pio4 = 0x1.921fb54442d1846ap-1L, invpio2 = 6.36619772367581343076e-01L, /* 0xa2f9836e4e44152a.0p-64 */ pio2_1t = -1.07463465549719416346e-12L, /* -0x973dcb3b399d747f.0p-103 */ pio2_2t = 6.36831716351095013979e-25L, /* 0xc51701b839a25205.0p-144 */ @@ -57,6 +58,7 @@ pio2_3t = -2.75299651904407171810e-37L; /* -0xbb5bf6c7ddd660ce.0p-185 */ #define NX 5 #define NY 3 static const long double +pio4 = 0x1.921fb54442d18469898cc51701b8p-1L, invpio2 = 6.3661977236758134307553505349005747e-01L, /* 0x145f306dc9c882a53f84eafa3ea6a.0p-113 */ pio2_1 = 1.5707963267948966192292994253909555e+00L, /* 0x1921fb54442d18469800000000000.0p-112 */ pio2_1t = 2.0222662487959507323996846200947577e-21L, /* 0x13198a2e03707344a4093822299f3.0p-181 */ @@ -76,11 +78,23 @@ int __rem_pio2l(long double x, long double *y) u.f = x; ex = u.i.se & 0x7fff; if (SMALL(u)) { - /* rint(x/(pi/2)), Assume round-to-nearest. */ + /* rint(x/(pi/2)) */ fn = x*invpio2 + toint - toint; n = QUOBITS(fn); r = x-fn*pio2_1; w = fn*pio2_1t; /* 1st round good to 102/180 bits (ld80/ld128) */ + /* Matters with directed rounding. */ + if (predict_false(r - w < -pio4)) { + n--; + fn--; + r = x - fn*pio2_1; + w = fn*pio2_1t; + } else if (predict_false(r - w > pio4)) { + n++; + fn++; + r = x - fn*pio2_1; + w = fn*pio2_1t; + } y[0] = r-w; u.f = y[0]; ey = u.i.se & 0x7fff; diff --git a/src/math/acoshf.c b/src/math/acoshf.c index 8a4ec4d5..b773d48e 100644 --- a/src/math/acoshf.c +++ b/src/math/acoshf.c @@ -15,12 +15,12 @@ float acoshf(float x) uint32_t a = u.i & 0x7fffffff; if (a < 0x3f800000+(1<<23)) - /* |x| < 2, invalid if x < 1 or nan */ + /* |x| < 2, invalid if x < 1 */ /* up to 2ulp error in [1,1.125] */ return log1pf(x-1 + sqrtf((x-1)*(x-1)+2*(x-1))); - if (a < 0x3f800000+(12<<23)) - /* |x| < 0x1p12 */ + if (u.i < 0x3f800000+(12<<23)) + /* 2 <= x < 0x1p12 */ return logf(2*x - 1/(x+sqrtf(x*x-1))); - /* x >= 0x1p12 */ + /* x >= 0x1p12 or x <= -2 or nan */ return logf(x) + 0.693147180559945309417232121458176568f; } diff --git a/src/math/acoshl.c b/src/math/acoshl.c index 8d4b43f6..943cec17 100644 --- a/src/math/acoshl.c +++ b/src/math/acoshl.c @@ -10,14 +10,18 @@ long double acoshl(long double x) long double acoshl(long double x) { union ldshape u = {x}; - int e = u.i.se & 0x7fff; + int e = u.i.se; if (e < 0x3fff + 1) - /* |x| < 2, invalid if x < 1 or nan */ + /* 0 <= x < 2, invalid if x < 1 */ return log1pl(x-1 + sqrtl((x-1)*(x-1)+2*(x-1))); if (e < 0x3fff + 32) - /* |x| < 0x1p32 */ + /* 2 <= x < 0x1p32 */ return logl(2*x - 1/(x+sqrtl(x*x-1))); + if (e & 0x8000) + /* x < 0 or x = -0, invalid */ + return (x - x) / (x - x); + /* 0x1p32 <= x or nan */ return logl(x) + 0.693147180559945309417232121458176568L; } #elif LDBL_MANT_DIG == 113 && LDBL_MAX_EXP == 16384 diff --git a/src/math/arm/fabs.c b/src/math/arm/fabs.c index f890520a..6e1d367d 100644 --- a/src/math/arm/fabs.c +++ b/src/math/arm/fabs.c @@ -1,6 +1,6 @@ #include <math.h> -#if __ARM_PCS_VFP +#if __ARM_PCS_VFP && __ARM_FP&8 double fabs(double x) { diff --git a/src/math/arm/sqrt.c b/src/math/arm/sqrt.c index 874af960..567e2e91 100644 --- a/src/math/arm/sqrt.c +++ b/src/math/arm/sqrt.c @@ -1,6 +1,6 @@ #include <math.h> -#if __ARM_PCS_VFP || (__VFP_FP__ && !__SOFTFP__) +#if (__ARM_PCS_VFP || (__VFP_FP__ && !__SOFTFP__)) && (__ARM_FP&8) double sqrt(double x) { diff --git a/src/math/cosh.c b/src/math/cosh.c index 100f8231..490c15fb 100644 --- a/src/math/cosh.c +++ b/src/math/cosh.c @@ -35,6 +35,6 @@ double cosh(double x) /* |x| > log(DBL_MAX) or nan */ /* note: the result is stored to handle overflow */ - t = __expo2(x); + t = __expo2(x, 1.0); return t; } diff --git a/src/math/coshf.c b/src/math/coshf.c index b09f2ee5..e739cff9 100644 --- a/src/math/coshf.c +++ b/src/math/coshf.c @@ -28,6 +28,6 @@ float coshf(float x) } /* |x| > log(FLT_MAX) or nan */ - t = __expo2f(x); + t = __expo2f(x, 1.0f); return t; } diff --git a/src/math/expm1f.c b/src/math/expm1f.c index 297e0b44..09a41afe 100644 --- a/src/math/expm1f.c +++ b/src/math/expm1f.c @@ -16,7 +16,6 @@ #include "libm.h" static const float -o_threshold = 8.8721679688e+01, /* 0x42b17180 */ ln2_hi = 6.9313812256e-01, /* 0x3f317180 */ ln2_lo = 9.0580006145e-06, /* 0x3717f7d1 */ invln2 = 1.4426950216e+00, /* 0x3fb8aa3b */ @@ -41,7 +40,7 @@ float expm1f(float x) return x; if (sign) return -1; - if (x > o_threshold) { + if (hx > 0x42b17217) { /* x > log(FLT_MAX) */ x *= 0x1p127f; return x; } diff --git a/src/math/fma.c b/src/math/fma.c index 0c6f90c9..adfadca8 100644 --- a/src/math/fma.c +++ b/src/math/fma.c @@ -53,7 +53,7 @@ double fma(double x, double y, double z) return x*y + z; if (nz.e >= ZEROINFNAN) { if (nz.e > ZEROINFNAN) /* z==0 */ - return x*y + z; + return x*y; return z; } diff --git a/src/math/fmaf.c b/src/math/fmaf.c index 80f5cd8a..7c65acf1 100644 --- a/src/math/fmaf.c +++ b/src/math/fmaf.c @@ -77,17 +77,16 @@ float fmaf(float x, float y, float z) * If result is inexact, and exactly halfway between two float values, * we need to adjust the low-order bit in the direction of the error. */ -#ifdef FE_TOWARDZERO - fesetround(FE_TOWARDZERO); -#endif - volatile double vxy = xy; /* XXX work around gcc CSE bug */ - double adjusted_result = vxy + z; - fesetround(FE_TONEAREST); - if (result == adjusted_result) { - u.f = adjusted_result; + double err; + int neg = u.i >> 63; + if (neg == (z > xy)) + err = xy - result + z; + else + err = z - result + xy; + if (neg == (err < 0)) u.i++; - adjusted_result = u.f; - } - z = adjusted_result; + else + u.i--; + z = u.f; return z; } diff --git a/src/math/i386/acos.s b/src/math/i386/acos.s index 47f365ef..af423a2f 100644 --- a/src/math/i386/acos.s +++ b/src/math/i386/acos.s @@ -1,22 +1,10 @@ # use acos(x) = atan2(fabs(sqrt((1-x)*(1+x))), x) -.global acosf -.type acosf,@function -acosf: - flds 4(%esp) - jmp 1f - -.global acosl -.type acosl,@function -acosl: - fldt 4(%esp) - jmp 1f - .global acos .type acos,@function acos: fldl 4(%esp) -1: fld %st(0) + fld %st(0) fld1 fsub %st(0),%st(1) fadd %st(2) @@ -25,4 +13,6 @@ acos: fabs # fix sign of zero (matters in downward rounding mode) fxch %st(1) fpatan + fstpl 4(%esp) + fldl 4(%esp) ret diff --git a/src/math/i386/acosf.s b/src/math/i386/acosf.s index 6c95509f..d2cdfdbf 100644 --- a/src/math/i386/acosf.s +++ b/src/math/i386/acosf.s @@ -1 +1,16 @@ -# see acos.s +.global acosf +.type acosf,@function +acosf: + flds 4(%esp) + fld %st(0) + fld1 + fsub %st(0),%st(1) + fadd %st(2) + fmulp + fsqrt + fabs # fix sign of zero (matters in downward rounding mode) + fxch %st(1) + fpatan + fstps 4(%esp) + flds 4(%esp) + ret diff --git a/src/math/i386/acosl.s b/src/math/i386/acosl.s index 6c95509f..599c8230 100644 --- a/src/math/i386/acosl.s +++ b/src/math/i386/acosl.s @@ -1 +1,14 @@ -# see acos.s +.global acosl +.type acosl,@function +acosl: + fldt 4(%esp) + fld %st(0) + fld1 + fsub %st(0),%st(1) + fadd %st(2) + fmulp + fsqrt + fabs # fix sign of zero (matters in downward rounding mode) + fxch %st(1) + fpatan + ret diff --git a/src/math/i386/asin.s b/src/math/i386/asin.s index 920d967a..2bc8356f 100644 --- a/src/math/i386/asin.s +++ b/src/math/i386/asin.s @@ -1,23 +1,3 @@ -.global asinf -.type asinf,@function -asinf: - flds 4(%esp) - mov 4(%esp),%eax - add %eax,%eax - cmp $0x01000000,%eax - jae 1f - # subnormal x, return x with underflow - fld %st(0) - fmul %st(1) - fstps 4(%esp) - ret - -.global asinl -.type asinl,@function -asinl: - fldt 4(%esp) - jmp 1f - .global asin .type asin,@function asin: @@ -25,15 +5,17 @@ asin: mov 8(%esp),%eax add %eax,%eax cmp $0x00200000,%eax - jae 1f - # subnormal x, return x with underflow - fsts 4(%esp) - ret -1: fld %st(0) + jb 1f + fld %st(0) fld1 fsub %st(0),%st(1) fadd %st(2) fmulp fsqrt fpatan + fstpl 4(%esp) + fldl 4(%esp) + ret + # subnormal x, return x with underflow +1: fsts 4(%esp) ret diff --git a/src/math/i386/asinf.s b/src/math/i386/asinf.s index e07bf599..05909753 100644 --- a/src/math/i386/asinf.s +++ b/src/math/i386/asinf.s @@ -1 +1,23 @@ -# see asin.s +.global asinf +.type asinf,@function +asinf: + flds 4(%esp) + mov 4(%esp),%eax + add %eax,%eax + cmp $0x01000000,%eax + jb 1f + fld %st(0) + fld1 + fsub %st(0),%st(1) + fadd %st(2) + fmulp + fsqrt + fpatan + fstps 4(%esp) + flds 4(%esp) + ret + # subnormal x, return x with underflow +1: fld %st(0) + fmul %st(1) + fstps 4(%esp) + ret diff --git a/src/math/i386/asinl.s b/src/math/i386/asinl.s index e07bf599..e973fc85 100644 --- a/src/math/i386/asinl.s +++ b/src/math/i386/asinl.s @@ -1 +1,12 @@ -# see asin.s +.global asinl +.type asinl,@function +asinl: + fldt 4(%esp) + fld %st(0) + fld1 + fsub %st(0),%st(1) + fadd %st(2) + fmulp + fsqrt + fpatan + ret diff --git a/src/math/i386/atan.s b/src/math/i386/atan.s index a26feae1..2c57f6b3 100644 --- a/src/math/i386/atan.s +++ b/src/math/i386/atan.s @@ -8,6 +8,8 @@ atan: jb 1f fld1 fpatan + fstpl 4(%esp) + fldl 4(%esp) ret # subnormal x, return x with underflow 1: fsts 4(%esp) diff --git a/src/math/i386/atan2.s b/src/math/i386/atan2.s index 76b95f31..8bc441b1 100644 --- a/src/math/i386/atan2.s +++ b/src/math/i386/atan2.s @@ -4,7 +4,8 @@ atan2: fldl 4(%esp) fldl 12(%esp) fpatan - fstl 4(%esp) + fstpl 4(%esp) + fldl 4(%esp) mov 8(%esp),%eax add %eax,%eax cmp $0x00200000,%eax diff --git a/src/math/i386/atan2f.s b/src/math/i386/atan2f.s index c9408a90..3908c86d 100644 --- a/src/math/i386/atan2f.s +++ b/src/math/i386/atan2f.s @@ -4,7 +4,8 @@ atan2f: flds 4(%esp) flds 8(%esp) fpatan - fsts 4(%esp) + fstps 4(%esp) + flds 4(%esp) mov 4(%esp),%eax add %eax,%eax cmp $0x01000000,%eax diff --git a/src/math/i386/atanf.s b/src/math/i386/atanf.s index 893beac5..c2cbe2e0 100644 --- a/src/math/i386/atanf.s +++ b/src/math/i386/atanf.s @@ -8,6 +8,8 @@ atanf: jb 1f fld1 fpatan + fstps 4(%esp) + flds 4(%esp) ret # subnormal x, return x with underflow 1: fld %st(0) diff --git a/src/math/i386/exp2.s b/src/math/i386/exp2.s deleted file mode 100644 index f335a3e5..00000000 --- a/src/math/i386/exp2.s +++ /dev/null @@ -1 +0,0 @@ -# see exp.s diff --git a/src/math/i386/exp2f.s b/src/math/i386/exp2f.s deleted file mode 100644 index f335a3e5..00000000 --- a/src/math/i386/exp2f.s +++ /dev/null @@ -1 +0,0 @@ -# see exp.s diff --git a/src/math/i386/exp2l.s b/src/math/i386/exp2l.s index f335a3e5..8125761d 100644 --- a/src/math/i386/exp2l.s +++ b/src/math/i386/exp2l.s @@ -1 +1 @@ -# see exp.s +# see exp_ld.s diff --git a/src/math/i386/exp.s b/src/math/i386/exp_ld.s index df87c497..99cba01f 100644 --- a/src/math/i386/exp.s +++ b/src/math/i386/exp_ld.s @@ -1,35 +1,8 @@ -.global expm1f -.type expm1f,@function -expm1f: - flds 4(%esp) - mov 4(%esp),%eax - add %eax,%eax - cmp $0x01000000,%eax - jae 1f - # subnormal x, return x with underflow - fld %st(0) - fmul %st(1) - fstps 4(%esp) - ret - .global expm1l .type expm1l,@function expm1l: fldt 4(%esp) - jmp 1f - -.global expm1 -.type expm1,@function -expm1: - fldl 4(%esp) - mov 8(%esp),%eax - add %eax,%eax - cmp $0x00200000,%eax - jae 1f - # subnormal x, return x with underflow - fsts 4(%esp) - ret -1: fldl2e + fldl2e fmulp mov $0xc2820000,%eax push %eax @@ -59,12 +32,6 @@ expm1: fsubrp ret -.global exp2f -.type exp2f,@function -exp2f: - flds 4(%esp) - jmp 1f - .global exp2l .global __exp2l .hidden __exp2l @@ -72,26 +39,6 @@ exp2f: exp2l: __exp2l: fldt 4(%esp) - jmp 1f - -.global expf -.type expf,@function -expf: - flds 4(%esp) - jmp 2f - -.global exp -.type exp,@function -exp: - fldl 4(%esp) -2: fldl2e - fmulp - jmp 1f - -.global exp2 -.type exp2,@function -exp2: - fldl 4(%esp) 1: sub $12,%esp fld %st(0) fstpt (%esp) diff --git a/src/math/i386/expf.s b/src/math/i386/expf.s deleted file mode 100644 index f335a3e5..00000000 --- a/src/math/i386/expf.s +++ /dev/null @@ -1 +0,0 @@ -# see exp.s diff --git a/src/math/i386/expm1.s b/src/math/i386/expm1.s deleted file mode 100644 index f335a3e5..00000000 --- a/src/math/i386/expm1.s +++ /dev/null @@ -1 +0,0 @@ -# see exp.s diff --git a/src/math/i386/expm1f.s b/src/math/i386/expm1f.s deleted file mode 100644 index f335a3e5..00000000 --- a/src/math/i386/expm1f.s +++ /dev/null @@ -1 +0,0 @@ -# see exp.s diff --git a/src/math/i386/expm1l.s b/src/math/i386/expm1l.s index f335a3e5..8125761d 100644 --- a/src/math/i386/expm1l.s +++ b/src/math/i386/expm1l.s @@ -1 +1 @@ -# see exp.s +# see exp_ld.s diff --git a/src/math/i386/fabs.c b/src/math/i386/fabs.c new file mode 100644 index 00000000..39672786 --- /dev/null +++ b/src/math/i386/fabs.c @@ -0,0 +1,7 @@ +#include <math.h> + +double fabs(double x) +{ + __asm__ ("fabs" : "+t"(x)); + return x; +} diff --git a/src/math/i386/fabs.s b/src/math/i386/fabs.s deleted file mode 100644 index d66ea9a1..00000000 --- a/src/math/i386/fabs.s +++ /dev/null @@ -1,6 +0,0 @@ -.global fabs -.type fabs,@function -fabs: - fldl 4(%esp) - fabs - ret diff --git a/src/math/i386/fabsf.c b/src/math/i386/fabsf.c new file mode 100644 index 00000000..d882eee3 --- /dev/null +++ b/src/math/i386/fabsf.c @@ -0,0 +1,7 @@ +#include <math.h> + +float fabsf(float x) +{ + __asm__ ("fabs" : "+t"(x)); + return x; +} diff --git a/src/math/i386/fabsf.s b/src/math/i386/fabsf.s deleted file mode 100644 index a981c422..00000000 --- a/src/math/i386/fabsf.s +++ /dev/null @@ -1,6 +0,0 @@ -.global fabsf -.type fabsf,@function -fabsf: - flds 4(%esp) - fabs - ret diff --git a/src/math/i386/fabsl.c b/src/math/i386/fabsl.c new file mode 100644 index 00000000..cc1c9ed9 --- /dev/null +++ b/src/math/i386/fabsl.c @@ -0,0 +1,7 @@ +#include <math.h> + +long double fabsl(long double x) +{ + __asm__ ("fabs" : "+t"(x)); + return x; +} diff --git a/src/math/i386/fabsl.s b/src/math/i386/fabsl.s deleted file mode 100644 index ceef9e4c..00000000 --- a/src/math/i386/fabsl.s +++ /dev/null @@ -1,6 +0,0 @@ -.global fabsl -.type fabsl,@function -fabsl: - fldt 4(%esp) - fabs - ret diff --git a/src/math/i386/fmod.c b/src/math/i386/fmod.c new file mode 100644 index 00000000..ea0c58d9 --- /dev/null +++ b/src/math/i386/fmod.c @@ -0,0 +1,10 @@ +#include <math.h> + +double fmod(double x, double y) +{ + unsigned short fpsr; + // fprem does not introduce excess precision into x + do __asm__ ("fprem; fnstsw %%ax" : "+t"(x), "=a"(fpsr) : "u"(y)); + while (fpsr & 0x400); + return x; +} diff --git a/src/math/i386/fmod.s b/src/math/i386/fmod.s deleted file mode 100644 index 2113b3c5..00000000 --- a/src/math/i386/fmod.s +++ /dev/null @@ -1,11 +0,0 @@ -.global fmod -.type fmod,@function -fmod: - fldl 12(%esp) - fldl 4(%esp) -1: fprem - fnstsw %ax - sahf - jp 1b - fstp %st(1) - ret diff --git a/src/math/i386/fmodf.c b/src/math/i386/fmodf.c new file mode 100644 index 00000000..90b56ab0 --- /dev/null +++ b/src/math/i386/fmodf.c @@ -0,0 +1,10 @@ +#include <math.h> + +float fmodf(float x, float y) +{ + unsigned short fpsr; + // fprem does not introduce excess precision into x + do __asm__ ("fprem; fnstsw %%ax" : "+t"(x), "=a"(fpsr) : "u"(y)); + while (fpsr & 0x400); + return x; +} diff --git a/src/math/i386/fmodf.s b/src/math/i386/fmodf.s deleted file mode 100644 index e04e2a56..00000000 --- a/src/math/i386/fmodf.s +++ /dev/null @@ -1,11 +0,0 @@ -.global fmodf -.type fmodf,@function -fmodf: - flds 8(%esp) - flds 4(%esp) -1: fprem - fnstsw %ax - sahf - jp 1b - fstp %st(1) - ret diff --git a/src/math/i386/fmodl.c b/src/math/i386/fmodl.c new file mode 100644 index 00000000..3daeab06 --- /dev/null +++ b/src/math/i386/fmodl.c @@ -0,0 +1,9 @@ +#include <math.h> + +long double fmodl(long double x, long double y) +{ + unsigned short fpsr; + do __asm__ ("fprem; fnstsw %%ax" : "+t"(x), "=a"(fpsr) : "u"(y)); + while (fpsr & 0x400); + return x; +} diff --git a/src/math/i386/fmodl.s b/src/math/i386/fmodl.s deleted file mode 100644 index 0cb3fe9b..00000000 --- a/src/math/i386/fmodl.s +++ /dev/null @@ -1,11 +0,0 @@ -.global fmodl -.type fmodl,@function -fmodl: - fldt 16(%esp) - fldt 4(%esp) -1: fprem - fnstsw %ax - sahf - jp 1b - fstp %st(1) - ret diff --git a/src/math/i386/llrint.c b/src/math/i386/llrint.c new file mode 100644 index 00000000..aa400817 --- /dev/null +++ b/src/math/i386/llrint.c @@ -0,0 +1,8 @@ +#include <math.h> + +long long llrint(double x) +{ + long long r; + __asm__ ("fistpll %0" : "=m"(r) : "t"(x) : "st"); + return r; +} diff --git a/src/math/i386/llrint.s b/src/math/i386/llrint.s deleted file mode 100644 index 8e89cd91..00000000 --- a/src/math/i386/llrint.s +++ /dev/null @@ -1,8 +0,0 @@ -.global llrint -.type llrint,@function -llrint: - fldl 4(%esp) - fistpll 4(%esp) - mov 4(%esp),%eax - mov 8(%esp),%edx - ret diff --git a/src/math/i386/llrintf.c b/src/math/i386/llrintf.c new file mode 100644 index 00000000..c41a317b --- /dev/null +++ b/src/math/i386/llrintf.c @@ -0,0 +1,8 @@ +#include <math.h> + +long long llrintf(float x) +{ + long long r; + __asm__ ("fistpll %0" : "=m"(r) : "t"(x) : "st"); + return r; +} diff --git a/src/math/i386/llrintf.s b/src/math/i386/llrintf.s deleted file mode 100644 index aa850c6c..00000000 --- a/src/math/i386/llrintf.s +++ /dev/null @@ -1,9 +0,0 @@ -.global llrintf -.type llrintf,@function -llrintf: - sub $8,%esp - flds 12(%esp) - fistpll (%esp) - pop %eax - pop %edx - ret diff --git a/src/math/i386/llrintl.c b/src/math/i386/llrintl.c new file mode 100644 index 00000000..c439ef28 --- /dev/null +++ b/src/math/i386/llrintl.c @@ -0,0 +1,8 @@ +#include <math.h> + +long long llrintl(long double x) +{ + long long r; + __asm__ ("fistpll %0" : "=m"(r) : "t"(x) : "st"); + return r; +} diff --git a/src/math/i386/llrintl.s b/src/math/i386/llrintl.s deleted file mode 100644 index 1cfb56f1..00000000 --- a/src/math/i386/llrintl.s +++ /dev/null @@ -1,8 +0,0 @@ -.global llrintl -.type llrintl,@function -llrintl: - fldt 4(%esp) - fistpll 4(%esp) - mov 4(%esp),%eax - mov 8(%esp),%edx - ret diff --git a/src/math/i386/log.s b/src/math/i386/log.s index fcccf030..08c59924 100644 --- a/src/math/i386/log.s +++ b/src/math/i386/log.s @@ -4,4 +4,6 @@ log: fldln2 fldl 4(%esp) fyl2x + fstpl 4(%esp) + fldl 4(%esp) ret diff --git a/src/math/i386/log10.s b/src/math/i386/log10.s index 28eb5b2f..120e91ec 100644 --- a/src/math/i386/log10.s +++ b/src/math/i386/log10.s @@ -4,4 +4,6 @@ log10: fldlg2 fldl 4(%esp) fyl2x + fstpl 4(%esp) + fldl 4(%esp) ret diff --git a/src/math/i386/log10f.s b/src/math/i386/log10f.s index c0c0c67e..b055493a 100644 --- a/src/math/i386/log10f.s +++ b/src/math/i386/log10f.s @@ -4,4 +4,6 @@ log10f: fldlg2 flds 4(%esp) fyl2x + fstps 4(%esp) + flds 4(%esp) ret diff --git a/src/math/i386/log1p.s b/src/math/i386/log1p.s index 354f391a..f3c95f83 100644 --- a/src/math/i386/log1p.s +++ b/src/math/i386/log1p.s @@ -10,10 +10,14 @@ log1p: cmp $0x00100000,%eax jb 2f fyl2xp1 + fstpl 4(%esp) + fldl 4(%esp) ret 1: fld1 faddp fyl2x + fstpl 4(%esp) + fldl 4(%esp) ret # subnormal x, return x with underflow 2: fsts 4(%esp) diff --git a/src/math/i386/log1pf.s b/src/math/i386/log1pf.s index 4d3484cd..9f13d95f 100644 --- a/src/math/i386/log1pf.s +++ b/src/math/i386/log1pf.s @@ -10,10 +10,14 @@ log1pf: cmp $0x00800000,%eax jb 2f fyl2xp1 + fstps 4(%esp) + flds 4(%esp) ret 1: fld1 faddp fyl2x + fstps 4(%esp) + flds 4(%esp) ret # subnormal x, return x with underflow 2: fxch diff --git a/src/math/i386/log2.s b/src/math/i386/log2.s index 15088037..7eff0b61 100644 --- a/src/math/i386/log2.s +++ b/src/math/i386/log2.s @@ -4,4 +4,6 @@ log2: fld1 fldl 4(%esp) fyl2x + fstpl 4(%esp) + fldl 4(%esp) ret diff --git a/src/math/i386/log2f.s b/src/math/i386/log2f.s index 00cdce75..b32fa2f7 100644 --- a/src/math/i386/log2f.s +++ b/src/math/i386/log2f.s @@ -4,4 +4,6 @@ log2f: fld1 flds 4(%esp) fyl2x + fstps 4(%esp) + flds 4(%esp) ret diff --git a/src/math/i386/logf.s b/src/math/i386/logf.s index da7ff3ae..4d0346a4 100644 --- a/src/math/i386/logf.s +++ b/src/math/i386/logf.s @@ -4,4 +4,6 @@ logf: fldln2 flds 4(%esp) fyl2x + fstps 4(%esp) + flds 4(%esp) ret diff --git a/src/math/i386/lrint.c b/src/math/i386/lrint.c new file mode 100644 index 00000000..89563ab2 --- /dev/null +++ b/src/math/i386/lrint.c @@ -0,0 +1,8 @@ +#include <math.h> + +long lrint(double x) +{ + long r; + __asm__ ("fistpl %0" : "=m"(r) : "t"(x) : "st"); + return r; +} diff --git a/src/math/i386/lrint.s b/src/math/i386/lrint.s deleted file mode 100644 index 02b83d9f..00000000 --- a/src/math/i386/lrint.s +++ /dev/null @@ -1,7 +0,0 @@ -.global lrint -.type lrint,@function -lrint: - fldl 4(%esp) - fistpl 4(%esp) - mov 4(%esp),%eax - ret diff --git a/src/math/i386/lrintf.c b/src/math/i386/lrintf.c new file mode 100644 index 00000000..0bbf29de --- /dev/null +++ b/src/math/i386/lrintf.c @@ -0,0 +1,8 @@ +#include <math.h> + +long lrintf(float x) +{ + long r; + __asm__ ("fistpl %0" : "=m"(r) : "t"(x) : "st"); + return r; +} diff --git a/src/math/i386/lrintf.s b/src/math/i386/lrintf.s deleted file mode 100644 index 907aac29..00000000 --- a/src/math/i386/lrintf.s +++ /dev/null @@ -1,7 +0,0 @@ -.global lrintf -.type lrintf,@function -lrintf: - flds 4(%esp) - fistpl 4(%esp) - mov 4(%esp),%eax - ret diff --git a/src/math/i386/lrintl.c b/src/math/i386/lrintl.c new file mode 100644 index 00000000..eb8c0902 --- /dev/null +++ b/src/math/i386/lrintl.c @@ -0,0 +1,8 @@ +#include <math.h> + +long lrintl(long double x) +{ + long r; + __asm__ ("fistpl %0" : "=m"(r) : "t"(x) : "st"); + return r; +} diff --git a/src/math/i386/lrintl.s b/src/math/i386/lrintl.s deleted file mode 100644 index 3ae05aac..00000000 --- a/src/math/i386/lrintl.s +++ /dev/null @@ -1,7 +0,0 @@ -.global lrintl -.type lrintl,@function -lrintl: - fldt 4(%esp) - fistpl 4(%esp) - mov 4(%esp),%eax - ret diff --git a/src/math/i386/remainder.c b/src/math/i386/remainder.c new file mode 100644 index 00000000..c083df90 --- /dev/null +++ b/src/math/i386/remainder.c @@ -0,0 +1,12 @@ +#include <math.h> + +double remainder(double x, double y) +{ + unsigned short fpsr; + // fprem1 does not introduce excess precision into x + do __asm__ ("fprem1; fnstsw %%ax" : "+t"(x), "=a"(fpsr) : "u"(y)); + while (fpsr & 0x400); + return x; +} + +weak_alias(remainder, drem); diff --git a/src/math/i386/remainder.s b/src/math/i386/remainder.s deleted file mode 100644 index ab1da95d..00000000 --- a/src/math/i386/remainder.s +++ /dev/null @@ -1,14 +0,0 @@ -.global remainder -.type remainder,@function -remainder: -.weak drem -.type drem,@function -drem: - fldl 12(%esp) - fldl 4(%esp) -1: fprem1 - fnstsw %ax - sahf - jp 1b - fstp %st(1) - ret diff --git a/src/math/i386/remainderf.c b/src/math/i386/remainderf.c new file mode 100644 index 00000000..280207d2 --- /dev/null +++ b/src/math/i386/remainderf.c @@ -0,0 +1,12 @@ +#include <math.h> + +float remainderf(float x, float y) +{ + unsigned short fpsr; + // fprem1 does not introduce excess precision into x + do __asm__ ("fprem1; fnstsw %%ax" : "+t"(x), "=a"(fpsr) : "u"(y)); + while (fpsr & 0x400); + return x; +} + +weak_alias(remainderf, dremf); diff --git a/src/math/i386/remainderf.s b/src/math/i386/remainderf.s deleted file mode 100644 index 6a7378a3..00000000 --- a/src/math/i386/remainderf.s +++ /dev/null @@ -1,14 +0,0 @@ -.global remainderf -.type remainderf,@function -remainderf: -.weak dremf -.type dremf,@function -dremf: - flds 8(%esp) - flds 4(%esp) -1: fprem1 - fnstsw %ax - sahf - jp 1b - fstp %st(1) - ret diff --git a/src/math/i386/remainderl.c b/src/math/i386/remainderl.c new file mode 100644 index 00000000..8cf75071 --- /dev/null +++ b/src/math/i386/remainderl.c @@ -0,0 +1,9 @@ +#include <math.h> + +long double remainderl(long double x, long double y) +{ + unsigned short fpsr; + do __asm__ ("fprem1; fnstsw %%ax" : "+t"(x), "=a"(fpsr) : "u"(y)); + while (fpsr & 0x400); + return x; +} diff --git a/src/math/i386/remainderl.s b/src/math/i386/remainderl.s deleted file mode 100644 index b41518ed..00000000 --- a/src/math/i386/remainderl.s +++ /dev/null @@ -1,11 +0,0 @@ -.global remainderl -.type remainderl,@function -remainderl: - fldt 16(%esp) - fldt 4(%esp) -1: fprem1 - fnstsw %ax - sahf - jp 1b - fstp %st(1) - ret diff --git a/src/math/i386/rint.c b/src/math/i386/rint.c new file mode 100644 index 00000000..a5276a60 --- /dev/null +++ b/src/math/i386/rint.c @@ -0,0 +1,7 @@ +#include <math.h> + +double rint(double x) +{ + __asm__ ("frndint" : "+t"(x)); + return x; +} diff --git a/src/math/i386/rint.s b/src/math/i386/rint.s deleted file mode 100644 index bb99a11c..00000000 --- a/src/math/i386/rint.s +++ /dev/null @@ -1,6 +0,0 @@ -.global rint -.type rint,@function -rint: - fldl 4(%esp) - frndint - ret diff --git a/src/math/i386/rintf.c b/src/math/i386/rintf.c new file mode 100644 index 00000000..bb4121a4 --- /dev/null +++ b/src/math/i386/rintf.c @@ -0,0 +1,7 @@ +#include <math.h> + +float rintf(float x) +{ + __asm__ ("frndint" : "+t"(x)); + return x; +} diff --git a/src/math/i386/rintf.s b/src/math/i386/rintf.s deleted file mode 100644 index bce4c5a6..00000000 --- a/src/math/i386/rintf.s +++ /dev/null @@ -1,6 +0,0 @@ -.global rintf -.type rintf,@function -rintf: - flds 4(%esp) - frndint - ret diff --git a/src/math/i386/rintl.c b/src/math/i386/rintl.c new file mode 100644 index 00000000..e1a92077 --- /dev/null +++ b/src/math/i386/rintl.c @@ -0,0 +1,7 @@ +#include <math.h> + +long double rintl(long double x) +{ + __asm__ ("frndint" : "+t"(x)); + return x; +} diff --git a/src/math/i386/rintl.s b/src/math/i386/rintl.s deleted file mode 100644 index cd2bf9a9..00000000 --- a/src/math/i386/rintl.s +++ /dev/null @@ -1,6 +0,0 @@ -.global rintl -.type rintl,@function -rintl: - fldt 4(%esp) - frndint - ret diff --git a/src/math/i386/sqrt.c b/src/math/i386/sqrt.c new file mode 100644 index 00000000..934fbcca --- /dev/null +++ b/src/math/i386/sqrt.c @@ -0,0 +1,15 @@ +#include "libm.h" + +double sqrt(double x) +{ + union ldshape ux; + unsigned fpsr; + __asm__ ("fsqrt; fnstsw %%ax": "=t"(ux.f), "=a"(fpsr) : "0"(x)); + if ((ux.i.m & 0x7ff) != 0x400) + return (double)ux.f; + /* Rounding to double would have encountered an exact halfway case. + Adjust mantissa downwards if fsqrt rounded up, else upwards. + (result of fsqrt could not have been exact) */ + ux.i.m ^= (fpsr & 0x200) + 0x300; + return (double)ux.f; +} diff --git a/src/math/i386/sqrt.s b/src/math/i386/sqrt.s deleted file mode 100644 index 57837e25..00000000 --- a/src/math/i386/sqrt.s +++ /dev/null @@ -1,21 +0,0 @@ -.global sqrt -.type sqrt,@function -sqrt: fldl 4(%esp) - fsqrt - fnstsw %ax - sub $12,%esp - fld %st(0) - fstpt (%esp) - mov (%esp),%ecx - and $0x7ff,%ecx - cmp $0x400,%ecx - jnz 1f - and $0x200,%eax - sub $0x100,%eax - sub %eax,(%esp) - fstp %st(0) - fldt (%esp) -1: add $12,%esp - fstpl 4(%esp) - fldl 4(%esp) - ret diff --git a/src/math/i386/sqrtf.c b/src/math/i386/sqrtf.c new file mode 100644 index 00000000..41c65c2b --- /dev/null +++ b/src/math/i386/sqrtf.c @@ -0,0 +1,12 @@ +#include <math.h> + +float sqrtf(float x) +{ + long double t; + /* The long double result has sufficient precision so that + * second rounding to float still keeps the returned value + * correctly rounded, see Pierre Roux, "Innocuous Double + * Rounding of Basic Arithmetic Operations". */ + __asm__ ("fsqrt" : "=t"(t) : "0"(x)); + return (float)t; +} diff --git a/src/math/i386/sqrtf.s b/src/math/i386/sqrtf.s deleted file mode 100644 index 9e944f45..00000000 --- a/src/math/i386/sqrtf.s +++ /dev/null @@ -1,7 +0,0 @@ -.global sqrtf -.type sqrtf,@function -sqrtf: flds 4(%esp) - fsqrt - fstps 4(%esp) - flds 4(%esp) - ret diff --git a/src/math/i386/sqrtl.c b/src/math/i386/sqrtl.c new file mode 100644 index 00000000..864cfcc4 --- /dev/null +++ b/src/math/i386/sqrtl.c @@ -0,0 +1,7 @@ +#include <math.h> + +long double sqrtl(long double x) +{ + __asm__ ("fsqrt" : "+t"(x)); + return x; +} diff --git a/src/math/i386/sqrtl.s b/src/math/i386/sqrtl.s deleted file mode 100644 index e0d42616..00000000 --- a/src/math/i386/sqrtl.s +++ /dev/null @@ -1,5 +0,0 @@ -.global sqrtl -.type sqrtl,@function -sqrtl: fldt 4(%esp) - fsqrt - ret diff --git a/src/math/logf.c b/src/math/logf.c index 7ee5d7fe..e4c2237c 100644 --- a/src/math/logf.c +++ b/src/math/logf.c @@ -53,7 +53,7 @@ float logf(float x) tmp = ix - OFF; i = (tmp >> (23 - LOGF_TABLE_BITS)) % N; k = (int32_t)tmp >> 23; /* arithmetic shift */ - iz = ix - (tmp & 0x1ff << 23); + iz = ix - (tmp & 0xff800000); invc = T[i].invc; logc = T[i].logc; z = (double_t)asfloat(iz); diff --git a/src/math/m68k/sqrtl.c b/src/math/m68k/sqrtl.c new file mode 100644 index 00000000..b1c303c7 --- /dev/null +++ b/src/math/m68k/sqrtl.c @@ -0,0 +1,15 @@ +#include <math.h> + +#if __HAVE_68881__ + +long double sqrtl(long double x) +{ + __asm__ ("fsqrt.x %1,%0" : "=f"(x) : "fm"(x)); + return x; +} + +#else + +#include "../sqrtl.c" + +#endif diff --git a/src/math/powerpc/fabs.c b/src/math/powerpc/fabs.c index f6ec4433..9453a3aa 100644 --- a/src/math/powerpc/fabs.c +++ b/src/math/powerpc/fabs.c @@ -1,6 +1,6 @@ #include <math.h> -#ifdef _SOFT_FLOAT +#if defined(_SOFT_FLOAT) || defined(__NO_FPRS__) || defined(BROKEN_PPC_D_ASM) #include "../fabs.c" diff --git a/src/math/powerpc/fabsf.c b/src/math/powerpc/fabsf.c index d88b5911..2e9da588 100644 --- a/src/math/powerpc/fabsf.c +++ b/src/math/powerpc/fabsf.c @@ -1,6 +1,6 @@ #include <math.h> -#ifdef _SOFT_FLOAT +#if defined(_SOFT_FLOAT) || defined(__NO_FPRS__) #include "../fabsf.c" diff --git a/src/math/powerpc/fma.c b/src/math/powerpc/fma.c index fd268f5f..0eb2ba1e 100644 --- a/src/math/powerpc/fma.c +++ b/src/math/powerpc/fma.c @@ -1,6 +1,6 @@ #include <math.h> -#ifdef _SOFT_FLOAT +#if defined(_SOFT_FLOAT) || defined(__NO_FPRS__) || defined(BROKEN_PPC_D_ASM) #include "../fma.c" diff --git a/src/math/powerpc/fmaf.c b/src/math/powerpc/fmaf.c index a99a2a3b..dc1a749d 100644 --- a/src/math/powerpc/fmaf.c +++ b/src/math/powerpc/fmaf.c @@ -1,6 +1,6 @@ #include <math.h> -#ifdef _SOFT_FLOAT +#if defined(_SOFT_FLOAT) || defined(__NO_FPRS__) #include "../fmaf.c" diff --git a/src/math/powl.c b/src/math/powl.c index 5b6da07b..6f64ea71 100644 --- a/src/math/powl.c +++ b/src/math/powl.c @@ -212,25 +212,33 @@ long double powl(long double x, long double y) } if (x == 1.0) return 1.0; /* 1**y = 1, even if y is nan */ - if (x == -1.0 && !isfinite(y)) - return 1.0; /* -1**inf = 1 */ if (y == 0.0) return 1.0; /* x**0 = 1, even if x is nan */ if (y == 1.0) return x; - if (y >= LDBL_MAX) { - if (x > 1.0 || x < -1.0) - return INFINITY; - if (x != 0.0) - return 0.0; - } - if (y <= -LDBL_MAX) { - if (x > 1.0 || x < -1.0) + /* if y*log2(x) < log2(LDBL_TRUE_MIN)-1 then x^y uflows to 0 + if y*log2(x) > -log2(LDBL_TRUE_MIN)+1 > LDBL_MAX_EXP then x^y oflows + if |x|!=1 then |log2(x)| > |log(x)| > LDBL_EPSILON/2 so + x^y oflows/uflows if |y|*LDBL_EPSILON/2 > -log2(LDBL_TRUE_MIN)+1 */ + if (fabsl(y) > 2*(-LDBL_MIN_EXP+LDBL_MANT_DIG+1)/LDBL_EPSILON) { + /* y is not an odd int */ + if (x == -1.0) + return 1.0; + if (y == INFINITY) { + if (x > 1.0 || x < -1.0) + return INFINITY; return 0.0; - if (x != 0.0 || y == -INFINITY) + } + if (y == -INFINITY) { + if (x > 1.0 || x < -1.0) + return 0.0; return INFINITY; + } + if ((x > 1.0 || x < -1.0) == (y > 0)) + return huge * huge; + return twom10000 * twom10000; } - if (x >= LDBL_MAX) { + if (x == INFINITY) { if (y > 0.0) return INFINITY; return 0.0; @@ -253,7 +261,7 @@ long double powl(long double x, long double y) yoddint = 1; } - if (x <= -LDBL_MAX) { + if (x == -INFINITY) { if (y > 0.0) { if (yoddint) return -INFINITY; diff --git a/src/math/riscv32/copysign.c b/src/math/riscv32/copysign.c new file mode 100644 index 00000000..c7854178 --- /dev/null +++ b/src/math/riscv32/copysign.c @@ -0,0 +1,15 @@ +#include <math.h> + +#if __riscv_flen >= 64 + +double copysign(double x, double y) +{ + __asm__ ("fsgnj.d %0, %1, %2" : "=f"(x) : "f"(x), "f"(y)); + return x; +} + +#else + +#include "../copysign.c" + +#endif diff --git a/src/math/riscv32/copysignf.c b/src/math/riscv32/copysignf.c new file mode 100644 index 00000000..a125611a --- /dev/null +++ b/src/math/riscv32/copysignf.c @@ -0,0 +1,15 @@ +#include <math.h> + +#if __riscv_flen >= 32 + +float copysignf(float x, float y) +{ + __asm__ ("fsgnj.s %0, %1, %2" : "=f"(x) : "f"(x), "f"(y)); + return x; +} + +#else + +#include "../copysignf.c" + +#endif diff --git a/src/math/riscv32/fabs.c b/src/math/riscv32/fabs.c new file mode 100644 index 00000000..5290b6f0 --- /dev/null +++ b/src/math/riscv32/fabs.c @@ -0,0 +1,15 @@ +#include <math.h> + +#if __riscv_flen >= 64 + +double fabs(double x) +{ + __asm__ ("fabs.d %0, %1" : "=f"(x) : "f"(x)); + return x; +} + +#else + +#include "../fabs.c" + +#endif diff --git a/src/math/riscv32/fabsf.c b/src/math/riscv32/fabsf.c new file mode 100644 index 00000000..f5032e35 --- /dev/null +++ b/src/math/riscv32/fabsf.c @@ -0,0 +1,15 @@ +#include <math.h> + +#if __riscv_flen >= 32 + +float fabsf(float x) +{ + __asm__ ("fabs.s %0, %1" : "=f"(x) : "f"(x)); + return x; +} + +#else + +#include "../fabsf.c" + +#endif diff --git a/src/math/riscv32/fma.c b/src/math/riscv32/fma.c new file mode 100644 index 00000000..99b05713 --- /dev/null +++ b/src/math/riscv32/fma.c @@ -0,0 +1,15 @@ +#include <math.h> + +#if __riscv_flen >= 64 + +double fma(double x, double y, double z) +{ + __asm__ ("fmadd.d %0, %1, %2, %3" : "=f"(x) : "f"(x), "f"(y), "f"(z)); + return x; +} + +#else + +#include "../fma.c" + +#endif diff --git a/src/math/riscv32/fmaf.c b/src/math/riscv32/fmaf.c new file mode 100644 index 00000000..f9dc47ed --- /dev/null +++ b/src/math/riscv32/fmaf.c @@ -0,0 +1,15 @@ +#include <math.h> + +#if __riscv_flen >= 32 + +float fmaf(float x, float y, float z) +{ + __asm__ ("fmadd.s %0, %1, %2, %3" : "=f"(x) : "f"(x), "f"(y), "f"(z)); + return x; +} + +#else + +#include "../fmaf.c" + +#endif diff --git a/src/math/riscv32/fmax.c b/src/math/riscv32/fmax.c new file mode 100644 index 00000000..023709cd --- /dev/null +++ b/src/math/riscv32/fmax.c @@ -0,0 +1,15 @@ +#include <math.h> + +#if __riscv_flen >= 64 + +double fmax(double x, double y) +{ + __asm__ ("fmax.d %0, %1, %2" : "=f"(x) : "f"(x), "f"(y)); + return x; +} + +#else + +#include "../fmax.c" + +#endif diff --git a/src/math/riscv32/fmaxf.c b/src/math/riscv32/fmaxf.c new file mode 100644 index 00000000..863d2bd1 --- /dev/null +++ b/src/math/riscv32/fmaxf.c @@ -0,0 +1,15 @@ +#include <math.h> + +#if __riscv_flen >= 32 + +float fmaxf(float x, float y) +{ + __asm__ ("fmax.s %0, %1, %2" : "=f"(x) : "f"(x), "f"(y)); + return x; +} + +#else + +#include "../fmaxf.c" + +#endif diff --git a/src/math/riscv32/fmin.c b/src/math/riscv32/fmin.c new file mode 100644 index 00000000..a4e3b067 --- /dev/null +++ b/src/math/riscv32/fmin.c @@ -0,0 +1,15 @@ +#include <math.h> + +#if __riscv_flen >= 64 + +double fmin(double x, double y) +{ + __asm__ ("fmin.d %0, %1, %2" : "=f"(x) : "f"(x), "f"(y)); + return x; +} + +#else + +#include "../fmin.c" + +#endif diff --git a/src/math/riscv32/fminf.c b/src/math/riscv32/fminf.c new file mode 100644 index 00000000..32156e80 --- /dev/null +++ b/src/math/riscv32/fminf.c @@ -0,0 +1,15 @@ +#include <math.h> + +#if __riscv_flen >= 32 + +float fminf(float x, float y) +{ + __asm__ ("fmin.s %0, %1, %2" : "=f"(x) : "f"(x), "f"(y)); + return x; +} + +#else + +#include "../fminf.c" + +#endif diff --git a/src/math/riscv32/sqrt.c b/src/math/riscv32/sqrt.c new file mode 100644 index 00000000..867a504c --- /dev/null +++ b/src/math/riscv32/sqrt.c @@ -0,0 +1,15 @@ +#include <math.h> + +#if __riscv_flen >= 64 + +double sqrt(double x) +{ + __asm__ ("fsqrt.d %0, %1" : "=f"(x) : "f"(x)); + return x; +} + +#else + +#include "../sqrt.c" + +#endif diff --git a/src/math/riscv32/sqrtf.c b/src/math/riscv32/sqrtf.c new file mode 100644 index 00000000..610c2cf8 --- /dev/null +++ b/src/math/riscv32/sqrtf.c @@ -0,0 +1,15 @@ +#include <math.h> + +#if __riscv_flen >= 32 + +float sqrtf(float x) +{ + __asm__ ("fsqrt.s %0, %1" : "=f"(x) : "f"(x)); + return x; +} + +#else + +#include "../sqrtf.c" + +#endif diff --git a/src/math/sinh.c b/src/math/sinh.c index 00022c4e..a01951ae 100644 --- a/src/math/sinh.c +++ b/src/math/sinh.c @@ -34,6 +34,6 @@ double sinh(double x) /* |x| > log(DBL_MAX) or nan */ /* note: the result is stored to handle overflow */ - t = 2*h*__expo2(absx); + t = __expo2(absx, 2*h); return t; } diff --git a/src/math/sinhf.c b/src/math/sinhf.c index 6ad19ea2..b9caa793 100644 --- a/src/math/sinhf.c +++ b/src/math/sinhf.c @@ -26,6 +26,6 @@ float sinhf(float x) } /* |x| > logf(FLT_MAX) or nan */ - t = 2*h*__expo2f(absx); + t = __expo2f(absx, 2*h); return t; } diff --git a/src/math/sqrt.c b/src/math/sqrt.c index f1f6d76c..5ba26559 100644 --- a/src/math/sqrt.c +++ b/src/math/sqrt.c @@ -1,184 +1,158 @@ -/* origin: FreeBSD /usr/src/lib/msun/src/e_sqrt.c */ -/* - * ==================================================== - * Copyright (C) 1993 by Sun Microsystems, Inc. All rights reserved. - * - * Developed at SunSoft, a Sun Microsystems, Inc. business. - * Permission to use, copy, modify, and distribute this - * software is freely granted, provided that this notice - * is preserved. - * ==================================================== - */ -/* sqrt(x) - * Return correctly rounded sqrt. - * ------------------------------------------ - * | Use the hardware sqrt if you have one | - * ------------------------------------------ - * Method: - * Bit by bit method using integer arithmetic. (Slow, but portable) - * 1. Normalization - * Scale x to y in [1,4) with even powers of 2: - * find an integer k such that 1 <= (y=x*2^(2k)) < 4, then - * sqrt(x) = 2^k * sqrt(y) - * 2. Bit by bit computation - * Let q = sqrt(y) truncated to i bit after binary point (q = 1), - * i 0 - * i+1 2 - * s = 2*q , and y = 2 * ( y - q ). (1) - * i i i i - * - * To compute q from q , one checks whether - * i+1 i - * - * -(i+1) 2 - * (q + 2 ) <= y. (2) - * i - * -(i+1) - * If (2) is false, then q = q ; otherwise q = q + 2 . - * i+1 i i+1 i - * - * With some algebric manipulation, it is not difficult to see - * that (2) is equivalent to - * -(i+1) - * s + 2 <= y (3) - * i i - * - * The advantage of (3) is that s and y can be computed by - * i i - * the following recurrence formula: - * if (3) is false - * - * s = s , y = y ; (4) - * i+1 i i+1 i - * - * otherwise, - * -i -(i+1) - * s = s + 2 , y = y - s - 2 (5) - * i+1 i i+1 i i - * - * One may easily use induction to prove (4) and (5). - * Note. Since the left hand side of (3) contain only i+2 bits, - * it does not necessary to do a full (53-bit) comparison - * in (3). - * 3. Final rounding - * After generating the 53 bits result, we compute one more bit. - * Together with the remainder, we can decide whether the - * result is exact, bigger than 1/2ulp, or less than 1/2ulp - * (it will never equal to 1/2ulp). - * The rounding mode can be detected by checking whether - * huge + tiny is equal to huge, and whether huge - tiny is - * equal to huge for some floating point number "huge" and "tiny". - * - * Special cases: - * sqrt(+-0) = +-0 ... exact - * sqrt(inf) = inf - * sqrt(-ve) = NaN ... with invalid signal - * sqrt(NaN) = NaN ... with invalid signal for signaling NaN - */ - +#include <stdint.h> +#include <math.h> #include "libm.h" +#include "sqrt_data.h" -static const double tiny = 1.0e-300; +#define FENV_SUPPORT 1 -double sqrt(double x) +/* returns a*b*2^-32 - e, with error 0 <= e < 1. */ +static inline uint32_t mul32(uint32_t a, uint32_t b) { - double z; - int32_t sign = (int)0x80000000; - int32_t ix0,s0,q,m,t,i; - uint32_t r,t1,s1,ix1,q1; + return (uint64_t)a*b >> 32; +} - EXTRACT_WORDS(ix0, ix1, x); +/* returns a*b*2^-64 - e, with error 0 <= e < 3. */ +static inline uint64_t mul64(uint64_t a, uint64_t b) +{ + uint64_t ahi = a>>32; + uint64_t alo = a&0xffffffff; + uint64_t bhi = b>>32; + uint64_t blo = b&0xffffffff; + return ahi*bhi + (ahi*blo >> 32) + (alo*bhi >> 32); +} - /* take care of Inf and NaN */ - if ((ix0&0x7ff00000) == 0x7ff00000) { - return x*x + x; /* sqrt(NaN)=NaN, sqrt(+inf)=+inf, sqrt(-inf)=sNaN */ - } - /* take care of zero */ - if (ix0 <= 0) { - if (((ix0&~sign)|ix1) == 0) - return x; /* sqrt(+-0) = +-0 */ - if (ix0 < 0) - return (x-x)/(x-x); /* sqrt(-ve) = sNaN */ - } - /* normalize x */ - m = ix0>>20; - if (m == 0) { /* subnormal x */ - while (ix0 == 0) { - m -= 21; - ix0 |= (ix1>>11); - ix1 <<= 21; - } - for (i=0; (ix0&0x00100000) == 0; i++) - ix0<<=1; - m -= i - 1; - ix0 |= ix1>>(32-i); - ix1 <<= i; - } - m -= 1023; /* unbias exponent */ - ix0 = (ix0&0x000fffff)|0x00100000; - if (m & 1) { /* odd m, double x to make it even */ - ix0 += ix0 + ((ix1&sign)>>31); - ix1 += ix1; - } - m >>= 1; /* m = [m/2] */ - - /* generate sqrt(x) bit by bit */ - ix0 += ix0 + ((ix1&sign)>>31); - ix1 += ix1; - q = q1 = s0 = s1 = 0; /* [q,q1] = sqrt(x) */ - r = 0x00200000; /* r = moving bit from right to left */ - - while (r != 0) { - t = s0 + r; - if (t <= ix0) { - s0 = t + r; - ix0 -= t; - q += r; - } - ix0 += ix0 + ((ix1&sign)>>31); - ix1 += ix1; - r >>= 1; - } +double sqrt(double x) +{ + uint64_t ix, top, m; - r = sign; - while (r != 0) { - t1 = s1 + r; - t = s0; - if (t < ix0 || (t == ix0 && t1 <= ix1)) { - s1 = t1 + r; - if ((t1&sign) == sign && (s1&sign) == 0) - s0++; - ix0 -= t; - if (ix1 < t1) - ix0--; - ix1 -= t1; - q1 += r; - } - ix0 += ix0 + ((ix1&sign)>>31); - ix1 += ix1; - r >>= 1; + /* special case handling. */ + ix = asuint64(x); + top = ix >> 52; + if (predict_false(top - 0x001 >= 0x7ff - 0x001)) { + /* x < 0x1p-1022 or inf or nan. */ + if (ix * 2 == 0) + return x; + if (ix == 0x7ff0000000000000) + return x; + if (ix > 0x7ff0000000000000) + return __math_invalid(x); + /* x is subnormal, normalize it. */ + ix = asuint64(x * 0x1p52); + top = ix >> 52; + top -= 52; } - /* use floating add to find out rounding direction */ - if ((ix0|ix1) != 0) { - z = 1.0 - tiny; /* raise inexact flag */ - if (z >= 1.0) { - z = 1.0 + tiny; - if (q1 == (uint32_t)0xffffffff) { - q1 = 0; - q++; - } else if (z > 1.0) { - if (q1 == (uint32_t)0xfffffffe) - q++; - q1 += 2; - } else - q1 += q1 & 1; - } + /* argument reduction: + x = 4^e m; with integer e, and m in [1, 4) + m: fixed point representation [2.62] + 2^e is the exponent part of the result. */ + int even = top & 1; + m = (ix << 11) | 0x8000000000000000; + if (even) m >>= 1; + top = (top + 0x3ff) >> 1; + + /* approximate r ~ 1/sqrt(m) and s ~ sqrt(m) when m in [1,4) + + initial estimate: + 7bit table lookup (1bit exponent and 6bit significand). + + iterative approximation: + using 2 goldschmidt iterations with 32bit int arithmetics + and a final iteration with 64bit int arithmetics. + + details: + + the relative error (e = r0 sqrt(m)-1) of a linear estimate + (r0 = a m + b) is |e| < 0.085955 ~ 0x1.6p-4 at best, + a table lookup is faster and needs one less iteration + 6 bit lookup table (128b) gives |e| < 0x1.f9p-8 + 7 bit lookup table (256b) gives |e| < 0x1.fdp-9 + for single and double prec 6bit is enough but for quad + prec 7bit is needed (or modified iterations). to avoid + one more iteration >=13bit table would be needed (16k). + + a newton-raphson iteration for r is + w = r*r + u = 3 - m*w + r = r*u/2 + can use a goldschmidt iteration for s at the end or + s = m*r + + first goldschmidt iteration is + s = m*r + u = 3 - s*r + r = r*u/2 + s = s*u/2 + next goldschmidt iteration is + u = 3 - s*r + r = r*u/2 + s = s*u/2 + and at the end r is not computed only s. + + they use the same amount of operations and converge at the + same quadratic rate, i.e. if + r1 sqrt(m) - 1 = e, then + r2 sqrt(m) - 1 = -3/2 e^2 - 1/2 e^3 + the advantage of goldschmidt is that the mul for s and r + are independent (computed in parallel), however it is not + "self synchronizing": it only uses the input m in the + first iteration so rounding errors accumulate. at the end + or when switching to larger precision arithmetics rounding + errors dominate so the first iteration should be used. + + the fixed point representations are + m: 2.30 r: 0.32, s: 2.30, d: 2.30, u: 2.30, three: 2.30 + and after switching to 64 bit + m: 2.62 r: 0.64, s: 2.62, d: 2.62, u: 2.62, three: 2.62 */ + + static const uint64_t three = 0xc0000000; + uint64_t r, s, d, u, i; + + i = (ix >> 46) % 128; + r = (uint32_t)__rsqrt_tab[i] << 16; + /* |r sqrt(m) - 1| < 0x1.fdp-9 */ + s = mul32(m>>32, r); + /* |s/sqrt(m) - 1| < 0x1.fdp-9 */ + d = mul32(s, r); + u = three - d; + r = mul32(r, u) << 1; + /* |r sqrt(m) - 1| < 0x1.7bp-16 */ + s = mul32(s, u) << 1; + /* |s/sqrt(m) - 1| < 0x1.7bp-16 */ + d = mul32(s, r); + u = three - d; + r = mul32(r, u) << 1; + /* |r sqrt(m) - 1| < 0x1.3704p-29 (measured worst-case) */ + r = r << 32; + s = mul64(m, r); + d = mul64(s, r); + u = (three<<32) - d; + s = mul64(s, u); /* repr: 3.61 */ + /* -0x1p-57 < s - sqrt(m) < 0x1.8001p-61 */ + s = (s - 2) >> 9; /* repr: 12.52 */ + /* -0x1.09p-52 < s - sqrt(m) < -0x1.fffcp-63 */ + + /* s < sqrt(m) < s + 0x1.09p-52, + compute nearest rounded result: + the nearest result to 52 bits is either s or s+0x1p-52, + we can decide by comparing (2^52 s + 0.5)^2 to 2^104 m. */ + uint64_t d0, d1, d2; + double y, t; + d0 = (m << 42) - s*s; + d1 = s - d0; + d2 = d1 + s + 1; + s += d1 >> 63; + s &= 0x000fffffffffffff; + s |= top << 52; + y = asdouble(s); + if (FENV_SUPPORT) { + /* handle rounding modes and inexact exception: + only (s+1)^2 == 2^42 m case is exact otherwise + add a tiny value to cause the fenv effects. */ + uint64_t tiny = predict_false(d2==0) ? 0 : 0x0010000000000000; + tiny |= (d1^d2) & 0x8000000000000000; + t = asdouble(tiny); + y = eval_as_double(y + t); } - ix0 = (q>>1) + 0x3fe00000; - ix1 = q1>>1; - if (q&1) - ix1 |= sign; - INSERT_WORDS(z, ix0 + ((uint32_t)m << 20), ix1); - return z; + return y; } diff --git a/src/math/sqrt_data.c b/src/math/sqrt_data.c new file mode 100644 index 00000000..61bc22f4 --- /dev/null +++ b/src/math/sqrt_data.c @@ -0,0 +1,19 @@ +#include "sqrt_data.h" +const uint16_t __rsqrt_tab[128] = { +0xb451,0xb2f0,0xb196,0xb044,0xaef9,0xadb6,0xac79,0xab43, +0xaa14,0xa8eb,0xa7c8,0xa6aa,0xa592,0xa480,0xa373,0xa26b, +0xa168,0xa06a,0x9f70,0x9e7b,0x9d8a,0x9c9d,0x9bb5,0x9ad1, +0x99f0,0x9913,0x983a,0x9765,0x9693,0x95c4,0x94f8,0x9430, +0x936b,0x92a9,0x91ea,0x912e,0x9075,0x8fbe,0x8f0a,0x8e59, +0x8daa,0x8cfe,0x8c54,0x8bac,0x8b07,0x8a64,0x89c4,0x8925, +0x8889,0x87ee,0x8756,0x86c0,0x862b,0x8599,0x8508,0x8479, +0x83ec,0x8361,0x82d8,0x8250,0x81c9,0x8145,0x80c2,0x8040, +0xff02,0xfd0e,0xfb25,0xf947,0xf773,0xf5aa,0xf3ea,0xf234, +0xf087,0xeee3,0xed47,0xebb3,0xea27,0xe8a3,0xe727,0xe5b2, +0xe443,0xe2dc,0xe17a,0xe020,0xdecb,0xdd7d,0xdc34,0xdaf1, +0xd9b3,0xd87b,0xd748,0xd61a,0xd4f1,0xd3cd,0xd2ad,0xd192, +0xd07b,0xcf69,0xce5b,0xcd51,0xcc4a,0xcb48,0xca4a,0xc94f, +0xc858,0xc764,0xc674,0xc587,0xc49d,0xc3b7,0xc2d4,0xc1f4, +0xc116,0xc03c,0xbf65,0xbe90,0xbdbe,0xbcef,0xbc23,0xbb59, +0xba91,0xb9cc,0xb90a,0xb84a,0xb78c,0xb6d0,0xb617,0xb560, +}; diff --git a/src/math/sqrt_data.h b/src/math/sqrt_data.h new file mode 100644 index 00000000..260c7f9c --- /dev/null +++ b/src/math/sqrt_data.h @@ -0,0 +1,13 @@ +#ifndef _SQRT_DATA_H +#define _SQRT_DATA_H + +#include <features.h> +#include <stdint.h> + +/* if x in [1,2): i = (int)(64*x); + if x in [2,4): i = (int)(32*x-64); + __rsqrt_tab[i]*2^-16 is estimating 1/sqrt(x) with small relative error: + |__rsqrt_tab[i]*0x1p-16*sqrt(x) - 1| < -0x1.fdp-9 < 2^-8 */ +extern hidden const uint16_t __rsqrt_tab[128]; + +#endif diff --git a/src/math/sqrtf.c b/src/math/sqrtf.c index d6ace38a..740d81cb 100644 --- a/src/math/sqrtf.c +++ b/src/math/sqrtf.c @@ -1,83 +1,83 @@ -/* origin: FreeBSD /usr/src/lib/msun/src/e_sqrtf.c */ -/* - * Conversion to float by Ian Lance Taylor, Cygnus Support, ian@cygnus.com. - */ -/* - * ==================================================== - * Copyright (C) 1993 by Sun Microsystems, Inc. All rights reserved. - * - * Developed at SunPro, a Sun Microsystems, Inc. business. - * Permission to use, copy, modify, and distribute this - * software is freely granted, provided that this notice - * is preserved. - * ==================================================== - */ - +#include <stdint.h> +#include <math.h> #include "libm.h" +#include "sqrt_data.h" -static const float tiny = 1.0e-30; +#define FENV_SUPPORT 1 -float sqrtf(float x) +static inline uint32_t mul32(uint32_t a, uint32_t b) { - float z; - int32_t sign = (int)0x80000000; - int32_t ix,s,q,m,t,i; - uint32_t r; + return (uint64_t)a*b >> 32; +} - GET_FLOAT_WORD(ix, x); +/* see sqrt.c for more detailed comments. */ - /* take care of Inf and NaN */ - if ((ix&0x7f800000) == 0x7f800000) - return x*x + x; /* sqrt(NaN)=NaN, sqrt(+inf)=+inf, sqrt(-inf)=sNaN */ +float sqrtf(float x) +{ + uint32_t ix, m, m1, m0, even, ey; - /* take care of zero */ - if (ix <= 0) { - if ((ix&~sign) == 0) - return x; /* sqrt(+-0) = +-0 */ - if (ix < 0) - return (x-x)/(x-x); /* sqrt(-ve) = sNaN */ - } - /* normalize x */ - m = ix>>23; - if (m == 0) { /* subnormal x */ - for (i = 0; (ix&0x00800000) == 0; i++) - ix<<=1; - m -= i - 1; + ix = asuint(x); + if (predict_false(ix - 0x00800000 >= 0x7f800000 - 0x00800000)) { + /* x < 0x1p-126 or inf or nan. */ + if (ix * 2 == 0) + return x; + if (ix == 0x7f800000) + return x; + if (ix > 0x7f800000) + return __math_invalidf(x); + /* x is subnormal, normalize it. */ + ix = asuint(x * 0x1p23f); + ix -= 23 << 23; } - m -= 127; /* unbias exponent */ - ix = (ix&0x007fffff)|0x00800000; - if (m&1) /* odd m, double x to make it even */ - ix += ix; - m >>= 1; /* m = [m/2] */ - /* generate sqrt(x) bit by bit */ - ix += ix; - q = s = 0; /* q = sqrt(x) */ - r = 0x01000000; /* r = moving bit from right to left */ + /* x = 4^e m; with int e and m in [1, 4). */ + even = ix & 0x00800000; + m1 = (ix << 8) | 0x80000000; + m0 = (ix << 7) & 0x7fffffff; + m = even ? m0 : m1; - while (r != 0) { - t = s + r; - if (t <= ix) { - s = t+r; - ix -= t; - q += r; - } - ix += ix; - r >>= 1; - } + /* 2^e is the exponent part of the return value. */ + ey = ix >> 1; + ey += 0x3f800000 >> 1; + ey &= 0x7f800000; + + /* compute r ~ 1/sqrt(m), s ~ sqrt(m) with 2 goldschmidt iterations. */ + static const uint32_t three = 0xc0000000; + uint32_t r, s, d, u, i; + i = (ix >> 17) % 128; + r = (uint32_t)__rsqrt_tab[i] << 16; + /* |r*sqrt(m) - 1| < 0x1p-8 */ + s = mul32(m, r); + /* |s/sqrt(m) - 1| < 0x1p-8 */ + d = mul32(s, r); + u = three - d; + r = mul32(r, u) << 1; + /* |r*sqrt(m) - 1| < 0x1.7bp-16 */ + s = mul32(s, u) << 1; + /* |s/sqrt(m) - 1| < 0x1.7bp-16 */ + d = mul32(s, r); + u = three - d; + s = mul32(s, u); + /* -0x1.03p-28 < s/sqrt(m) - 1 < 0x1.fp-31 */ + s = (s - 1)>>6; + /* s < sqrt(m) < s + 0x1.08p-23 */ - /* use floating add to find out rounding direction */ - if (ix != 0) { - z = 1.0f - tiny; /* raise inexact flag */ - if (z >= 1.0f) { - z = 1.0f + tiny; - if (z > 1.0f) - q += 2; - else - q += q & 1; - } + /* compute nearest rounded result. */ + uint32_t d0, d1, d2; + float y, t; + d0 = (m << 16) - s*s; + d1 = s - d0; + d2 = d1 + s + 1; + s += d1 >> 31; + s &= 0x007fffff; + s |= ey; + y = asfloat(s); + if (FENV_SUPPORT) { + /* handle rounding and inexact exception. */ + uint32_t tiny = predict_false(d2==0) ? 0 : 0x01000000; + tiny |= (d1^d2) & 0x80000000; + t = asfloat(tiny); + y = eval_as_float(y + t); } - ix = (q>>1) + 0x3f000000; - SET_FLOAT_WORD(z, ix + ((uint32_t)m << 23)); - return z; + return y; } diff --git a/src/math/sqrtl.c b/src/math/sqrtl.c index 83a8f80c..a231b3f2 100644 --- a/src/math/sqrtl.c +++ b/src/math/sqrtl.c @@ -1,7 +1,259 @@ +#include <stdint.h> #include <math.h> +#include <float.h> +#include "libm.h" +#if LDBL_MANT_DIG == 53 && LDBL_MAX_EXP == 1024 long double sqrtl(long double x) { - /* FIXME: implement in C, this is for LDBL_MANT_DIG == 64 only */ return sqrt(x); } +#elif (LDBL_MANT_DIG == 113 || LDBL_MANT_DIG == 64) && LDBL_MAX_EXP == 16384 +#include "sqrt_data.h" + +#define FENV_SUPPORT 1 + +typedef struct { + uint64_t hi; + uint64_t lo; +} u128; + +/* top: 16 bit sign+exponent, x: significand. */ +static inline long double mkldbl(uint64_t top, u128 x) +{ + union ldshape u; +#if LDBL_MANT_DIG == 113 + u.i2.hi = x.hi; + u.i2.lo = x.lo; + u.i2.hi &= 0x0000ffffffffffff; + u.i2.hi |= top << 48; +#elif LDBL_MANT_DIG == 64 + u.i.se = top; + u.i.m = x.lo; + /* force the top bit on non-zero (and non-subnormal) results. */ + if (top & 0x7fff) + u.i.m |= 0x8000000000000000; +#endif + return u.f; +} + +/* return: top 16 bit is sign+exp and following bits are the significand. */ +static inline u128 asu128(long double x) +{ + union ldshape u = {.f=x}; + u128 r; +#if LDBL_MANT_DIG == 113 + r.hi = u.i2.hi; + r.lo = u.i2.lo; +#elif LDBL_MANT_DIG == 64 + r.lo = u.i.m<<49; + /* ignore the top bit: pseudo numbers are not handled. */ + r.hi = u.i.m>>15; + r.hi &= 0x0000ffffffffffff; + r.hi |= (uint64_t)u.i.se << 48; +#endif + return r; +} + +/* returns a*b*2^-32 - e, with error 0 <= e < 1. */ +static inline uint32_t mul32(uint32_t a, uint32_t b) +{ + return (uint64_t)a*b >> 32; +} + +/* returns a*b*2^-64 - e, with error 0 <= e < 3. */ +static inline uint64_t mul64(uint64_t a, uint64_t b) +{ + uint64_t ahi = a>>32; + uint64_t alo = a&0xffffffff; + uint64_t bhi = b>>32; + uint64_t blo = b&0xffffffff; + return ahi*bhi + (ahi*blo >> 32) + (alo*bhi >> 32); +} + +static inline u128 add64(u128 a, uint64_t b) +{ + u128 r; + r.lo = a.lo + b; + r.hi = a.hi; + if (r.lo < a.lo) + r.hi++; + return r; +} + +static inline u128 add128(u128 a, u128 b) +{ + u128 r; + r.lo = a.lo + b.lo; + r.hi = a.hi + b.hi; + if (r.lo < a.lo) + r.hi++; + return r; +} + +static inline u128 sub64(u128 a, uint64_t b) +{ + u128 r; + r.lo = a.lo - b; + r.hi = a.hi; + if (a.lo < b) + r.hi--; + return r; +} + +static inline u128 sub128(u128 a, u128 b) +{ + u128 r; + r.lo = a.lo - b.lo; + r.hi = a.hi - b.hi; + if (a.lo < b.lo) + r.hi--; + return r; +} + +/* a<<n, 0 <= n <= 127 */ +static inline u128 lsh(u128 a, int n) +{ + if (n == 0) + return a; + if (n >= 64) { + a.hi = a.lo<<(n-64); + a.lo = 0; + } else { + a.hi = (a.hi<<n) | (a.lo>>(64-n)); + a.lo = a.lo<<n; + } + return a; +} + +/* a>>n, 0 <= n <= 127 */ +static inline u128 rsh(u128 a, int n) +{ + if (n == 0) + return a; + if (n >= 64) { + a.lo = a.hi>>(n-64); + a.hi = 0; + } else { + a.lo = (a.lo>>n) | (a.hi<<(64-n)); + a.hi = a.hi>>n; + } + return a; +} + +/* returns a*b exactly. */ +static inline u128 mul64_128(uint64_t a, uint64_t b) +{ + u128 r; + uint64_t ahi = a>>32; + uint64_t alo = a&0xffffffff; + uint64_t bhi = b>>32; + uint64_t blo = b&0xffffffff; + uint64_t lo1 = ((ahi*blo)&0xffffffff) + ((alo*bhi)&0xffffffff) + (alo*blo>>32); + uint64_t lo2 = (alo*blo)&0xffffffff; + r.hi = ahi*bhi + (ahi*blo>>32) + (alo*bhi>>32) + (lo1>>32); + r.lo = (lo1<<32) + lo2; + return r; +} + +/* returns a*b*2^-128 - e, with error 0 <= e < 7. */ +static inline u128 mul128(u128 a, u128 b) +{ + u128 hi = mul64_128(a.hi, b.hi); + uint64_t m1 = mul64(a.hi, b.lo); + uint64_t m2 = mul64(a.lo, b.hi); + return add64(add64(hi, m1), m2); +} + +/* returns a*b % 2^128. */ +static inline u128 mul128_tail(u128 a, u128 b) +{ + u128 lo = mul64_128(a.lo, b.lo); + lo.hi += a.hi*b.lo + a.lo*b.hi; + return lo; +} + + +/* see sqrt.c for detailed comments. */ + +long double sqrtl(long double x) +{ + u128 ix, ml; + uint64_t top; + + ix = asu128(x); + top = ix.hi >> 48; + if (predict_false(top - 0x0001 >= 0x7fff - 0x0001)) { + /* x < 0x1p-16382 or inf or nan. */ + if (2*ix.hi == 0 && ix.lo == 0) + return x; + if (ix.hi == 0x7fff000000000000 && ix.lo == 0) + return x; + if (top >= 0x7fff) + return __math_invalidl(x); + /* x is subnormal, normalize it. */ + ix = asu128(x * 0x1p112); + top = ix.hi >> 48; + top -= 112; + } + + /* x = 4^e m; with int e and m in [1, 4) */ + int even = top & 1; + ml = lsh(ix, 15); + ml.hi |= 0x8000000000000000; + if (even) ml = rsh(ml, 1); + top = (top + 0x3fff) >> 1; + + /* r ~ 1/sqrt(m) */ + const uint64_t three = 0xc0000000; + uint64_t r, s, d, u, i; + i = (ix.hi >> 42) % 128; + r = (uint32_t)__rsqrt_tab[i] << 16; + /* |r sqrt(m) - 1| < 0x1p-8 */ + s = mul32(ml.hi>>32, r); + d = mul32(s, r); + u = three - d; + r = mul32(u, r) << 1; + /* |r sqrt(m) - 1| < 0x1.7bp-16, switch to 64bit */ + r = r<<32; + s = mul64(ml.hi, r); + d = mul64(s, r); + u = (three<<32) - d; + r = mul64(u, r) << 1; + /* |r sqrt(m) - 1| < 0x1.a5p-31 */ + s = mul64(u, s) << 1; + d = mul64(s, r); + u = (three<<32) - d; + r = mul64(u, r) << 1; + /* |r sqrt(m) - 1| < 0x1.c001p-59, switch to 128bit */ + + const u128 threel = {.hi=three<<32, .lo=0}; + u128 rl, sl, dl, ul; + rl.hi = r; + rl.lo = 0; + sl = mul128(ml, rl); + dl = mul128(sl, rl); + ul = sub128(threel, dl); + sl = mul128(ul, sl); /* repr: 3.125 */ + /* -0x1p-116 < s - sqrt(m) < 0x3.8001p-125 */ + sl = rsh(sub64(sl, 4), 125-(LDBL_MANT_DIG-1)); + /* s < sqrt(m) < s + 1 ULP + tiny */ + + long double y; + u128 d2, d1, d0; + d0 = sub128(lsh(ml, 2*(LDBL_MANT_DIG-1)-126), mul128_tail(sl,sl)); + d1 = sub128(sl, d0); + d2 = add128(add64(sl, 1), d1); + sl = add64(sl, d1.hi >> 63); + y = mkldbl(top, sl); + if (FENV_SUPPORT) { + /* handle rounding modes and inexact exception. */ + top = predict_false((d2.hi|d2.lo)==0) ? 0 : 1; + top |= ((d1.hi^d2.hi)&0x8000000000000000) >> 48; + y += mkldbl(top, (u128){0}); + } + return y; +} +#else +#error unsupported long double format +#endif diff --git a/src/math/x32/lrintl.s b/src/math/x32/lrintl.s index ee97d1cf..d4355c32 100644 --- a/src/math/x32/lrintl.s +++ b/src/math/x32/lrintl.s @@ -2,6 +2,6 @@ .type lrintl,@function lrintl: fldt 8(%esp) - fistpll 8(%esp) - mov 8(%esp),%rax + fistpl 8(%esp) + movl 8(%esp),%eax ret diff --git a/src/math/x86_64/fabs.c b/src/math/x86_64/fabs.c new file mode 100644 index 00000000..16562477 --- /dev/null +++ b/src/math/x86_64/fabs.c @@ -0,0 +1,10 @@ +#include <math.h> + +double fabs(double x) +{ + double t; + __asm__ ("pcmpeqd %0, %0" : "=x"(t)); // t = ~0 + __asm__ ("psrlq $1, %0" : "+x"(t)); // t >>= 1 + __asm__ ("andps %1, %0" : "+x"(x) : "x"(t)); // x &= t + return x; +} diff --git a/src/math/x86_64/fabs.s b/src/math/x86_64/fabs.s deleted file mode 100644 index 5715005e..00000000 --- a/src/math/x86_64/fabs.s +++ /dev/null @@ -1,9 +0,0 @@ -.global fabs -.type fabs,@function -fabs: - xor %eax,%eax - dec %rax - shr %rax - movq %rax,%xmm1 - andpd %xmm1,%xmm0 - ret diff --git a/src/math/x86_64/fabsf.c b/src/math/x86_64/fabsf.c new file mode 100644 index 00000000..36ea7481 --- /dev/null +++ b/src/math/x86_64/fabsf.c @@ -0,0 +1,10 @@ +#include <math.h> + +float fabsf(float x) +{ + float t; + __asm__ ("pcmpeqd %0, %0" : "=x"(t)); // t = ~0 + __asm__ ("psrld $1, %0" : "+x"(t)); // t >>= 1 + __asm__ ("andps %1, %0" : "+x"(x) : "x"(t)); // x &= t + return x; +} diff --git a/src/math/x86_64/fabsf.s b/src/math/x86_64/fabsf.s deleted file mode 100644 index 501a1f17..00000000 --- a/src/math/x86_64/fabsf.s +++ /dev/null @@ -1,7 +0,0 @@ -.global fabsf -.type fabsf,@function -fabsf: - mov $0x7fffffff,%eax - movq %rax,%xmm1 - andps %xmm1,%xmm0 - ret diff --git a/src/math/x86_64/fabsl.c b/src/math/x86_64/fabsl.c new file mode 100644 index 00000000..cc1c9ed9 --- /dev/null +++ b/src/math/x86_64/fabsl.c @@ -0,0 +1,7 @@ +#include <math.h> + +long double fabsl(long double x) +{ + __asm__ ("fabs" : "+t"(x)); + return x; +} diff --git a/src/math/x86_64/fabsl.s b/src/math/x86_64/fabsl.s deleted file mode 100644 index 4e7ab525..00000000 --- a/src/math/x86_64/fabsl.s +++ /dev/null @@ -1,6 +0,0 @@ -.global fabsl -.type fabsl,@function -fabsl: - fldt 8(%rsp) - fabs - ret diff --git a/src/math/x86_64/fmodl.c b/src/math/x86_64/fmodl.c new file mode 100644 index 00000000..3daeab06 --- /dev/null +++ b/src/math/x86_64/fmodl.c @@ -0,0 +1,9 @@ +#include <math.h> + +long double fmodl(long double x, long double y) +{ + unsigned short fpsr; + do __asm__ ("fprem; fnstsw %%ax" : "+t"(x), "=a"(fpsr) : "u"(y)); + while (fpsr & 0x400); + return x; +} diff --git a/src/math/x86_64/fmodl.s b/src/math/x86_64/fmodl.s deleted file mode 100644 index ea07b402..00000000 --- a/src/math/x86_64/fmodl.s +++ /dev/null @@ -1,11 +0,0 @@ -.global fmodl -.type fmodl,@function -fmodl: - fldt 24(%rsp) - fldt 8(%rsp) -1: fprem - fnstsw %ax - testb $4,%ah - jnz 1b - fstp %st(1) - ret diff --git a/src/math/x86_64/llrint.c b/src/math/x86_64/llrint.c new file mode 100644 index 00000000..dd38a722 --- /dev/null +++ b/src/math/x86_64/llrint.c @@ -0,0 +1,8 @@ +#include <math.h> + +long long llrint(double x) +{ + long long r; + __asm__ ("cvtsd2si %1, %0" : "=r"(r) : "x"(x)); + return r; +} diff --git a/src/math/x86_64/llrint.s b/src/math/x86_64/llrint.s deleted file mode 100644 index bf476498..00000000 --- a/src/math/x86_64/llrint.s +++ /dev/null @@ -1,5 +0,0 @@ -.global llrint -.type llrint,@function -llrint: - cvtsd2si %xmm0,%rax - ret diff --git a/src/math/x86_64/llrintf.c b/src/math/x86_64/llrintf.c new file mode 100644 index 00000000..fc8625e8 --- /dev/null +++ b/src/math/x86_64/llrintf.c @@ -0,0 +1,8 @@ +#include <math.h> + +long long llrintf(float x) +{ + long long r; + __asm__ ("cvtss2si %1, %0" : "=r"(r) : "x"(x)); + return r; +} diff --git a/src/math/x86_64/llrintf.s b/src/math/x86_64/llrintf.s deleted file mode 100644 index d7204ac0..00000000 --- a/src/math/x86_64/llrintf.s +++ /dev/null @@ -1,5 +0,0 @@ -.global llrintf -.type llrintf,@function -llrintf: - cvtss2si %xmm0,%rax - ret diff --git a/src/math/x86_64/llrintl.c b/src/math/x86_64/llrintl.c new file mode 100644 index 00000000..c439ef28 --- /dev/null +++ b/src/math/x86_64/llrintl.c @@ -0,0 +1,8 @@ +#include <math.h> + +long long llrintl(long double x) +{ + long long r; + __asm__ ("fistpll %0" : "=m"(r) : "t"(x) : "st"); + return r; +} diff --git a/src/math/x86_64/llrintl.s b/src/math/x86_64/llrintl.s deleted file mode 100644 index 1ec0817d..00000000 --- a/src/math/x86_64/llrintl.s +++ /dev/null @@ -1,7 +0,0 @@ -.global llrintl -.type llrintl,@function -llrintl: - fldt 8(%rsp) - fistpll 8(%rsp) - mov 8(%rsp),%rax - ret diff --git a/src/math/x86_64/lrint.c b/src/math/x86_64/lrint.c new file mode 100644 index 00000000..a742fec6 --- /dev/null +++ b/src/math/x86_64/lrint.c @@ -0,0 +1,8 @@ +#include <math.h> + +long lrint(double x) +{ + long r; + __asm__ ("cvtsd2si %1, %0" : "=r"(r) : "x"(x)); + return r; +} diff --git a/src/math/x86_64/lrint.s b/src/math/x86_64/lrint.s deleted file mode 100644 index 15fc2454..00000000 --- a/src/math/x86_64/lrint.s +++ /dev/null @@ -1,5 +0,0 @@ -.global lrint -.type lrint,@function -lrint: - cvtsd2si %xmm0,%rax - ret diff --git a/src/math/x86_64/lrintf.c b/src/math/x86_64/lrintf.c new file mode 100644 index 00000000..2ba5639d --- /dev/null +++ b/src/math/x86_64/lrintf.c @@ -0,0 +1,8 @@ +#include <math.h> + +long lrintf(float x) +{ + long r; + __asm__ ("cvtss2si %1, %0" : "=r"(r) : "x"(x)); + return r; +} diff --git a/src/math/x86_64/lrintf.s b/src/math/x86_64/lrintf.s deleted file mode 100644 index 488423d2..00000000 --- a/src/math/x86_64/lrintf.s +++ /dev/null @@ -1,5 +0,0 @@ -.global lrintf -.type lrintf,@function -lrintf: - cvtss2si %xmm0,%rax - ret diff --git a/src/math/x86_64/lrintl.c b/src/math/x86_64/lrintl.c new file mode 100644 index 00000000..068e2e4d --- /dev/null +++ b/src/math/x86_64/lrintl.c @@ -0,0 +1,8 @@ +#include <math.h> + +long lrintl(long double x) +{ + long r; + __asm__ ("fistpll %0" : "=m"(r) : "t"(x) : "st"); + return r; +} diff --git a/src/math/x86_64/lrintl.s b/src/math/x86_64/lrintl.s deleted file mode 100644 index d587b12b..00000000 --- a/src/math/x86_64/lrintl.s +++ /dev/null @@ -1,7 +0,0 @@ -.global lrintl -.type lrintl,@function -lrintl: - fldt 8(%rsp) - fistpll 8(%rsp) - mov 8(%rsp),%rax - ret diff --git a/src/math/x86_64/remainderl.c b/src/math/x86_64/remainderl.c new file mode 100644 index 00000000..8cf75071 --- /dev/null +++ b/src/math/x86_64/remainderl.c @@ -0,0 +1,9 @@ +#include <math.h> + +long double remainderl(long double x, long double y) +{ + unsigned short fpsr; + do __asm__ ("fprem1; fnstsw %%ax" : "+t"(x), "=a"(fpsr) : "u"(y)); + while (fpsr & 0x400); + return x; +} diff --git a/src/math/x86_64/remainderl.s b/src/math/x86_64/remainderl.s deleted file mode 100644 index cb3857b4..00000000 --- a/src/math/x86_64/remainderl.s +++ /dev/null @@ -1,11 +0,0 @@ -.global remainderl -.type remainderl,@function -remainderl: - fldt 24(%rsp) - fldt 8(%rsp) -1: fprem1 - fnstsw %ax - testb $4,%ah - jnz 1b - fstp %st(1) - ret diff --git a/src/math/x86_64/remquol.c b/src/math/x86_64/remquol.c new file mode 100644 index 00000000..60eef089 --- /dev/null +++ b/src/math/x86_64/remquol.c @@ -0,0 +1,32 @@ +#include <math.h> + +long double remquol(long double x, long double y, int *quo) +{ + signed char *cx = (void *)&x, *cy = (void *)&y; + /* By ensuring that addresses of x and y cannot be discarded, + * this empty asm guides GCC into representing extraction of + * their sign bits as memory loads rather than making x and y + * not-address-taken internally and using bitfield operations, + * which in the end wouldn't work out, as extraction from FPU + * registers needs to go through memory anyway. This way GCC + * should manage to use incoming stack slots without spills. */ + __asm__ ("" :: "X"(cx), "X"(cy)); + + long double t = x; + unsigned fpsr; + do __asm__ ("fprem1; fnstsw %%ax" : "+t"(t), "=a"(fpsr) : "u"(y)); + while (fpsr & 0x400); + /* C0, C1, C3 flags in x87 status word carry low bits of quotient: + * 15 14 13 12 11 10 9 8 + * . C3 . . . C2 C1 C0 + * . b1 . . . 0 b0 b2 */ + unsigned char i = fpsr >> 8; + i = i>>4 | i<<4; + /* i[5:2] is now {b0 b2 ? b1}. Retrieve {0 b2 b1 b0} via + * in-register table lookup. */ + unsigned qbits = 0x7575313164642020 >> (i & 60); + qbits &= 7; + + *quo = (cx[9]^cy[9]) < 0 ? -qbits : qbits; + return t; +} diff --git a/src/math/x86_64/rintl.c b/src/math/x86_64/rintl.c new file mode 100644 index 00000000..e1a92077 --- /dev/null +++ b/src/math/x86_64/rintl.c @@ -0,0 +1,7 @@ +#include <math.h> + +long double rintl(long double x) +{ + __asm__ ("frndint" : "+t"(x)); + return x; +} diff --git a/src/math/x86_64/rintl.s b/src/math/x86_64/rintl.s deleted file mode 100644 index 64e663cd..00000000 --- a/src/math/x86_64/rintl.s +++ /dev/null @@ -1,6 +0,0 @@ -.global rintl -.type rintl,@function -rintl: - fldt 8(%rsp) - frndint - ret diff --git a/src/math/x86_64/sqrt.c b/src/math/x86_64/sqrt.c new file mode 100644 index 00000000..657e09e3 --- /dev/null +++ b/src/math/x86_64/sqrt.c @@ -0,0 +1,7 @@ +#include <math.h> + +double sqrt(double x) +{ + __asm__ ("sqrtsd %1, %0" : "=x"(x) : "x"(x)); + return x; +} diff --git a/src/math/x86_64/sqrt.s b/src/math/x86_64/sqrt.s deleted file mode 100644 index d3c609f9..00000000 --- a/src/math/x86_64/sqrt.s +++ /dev/null @@ -1,4 +0,0 @@ -.global sqrt -.type sqrt,@function -sqrt: sqrtsd %xmm0, %xmm0 - ret diff --git a/src/math/x86_64/sqrtf.c b/src/math/x86_64/sqrtf.c new file mode 100644 index 00000000..720baec6 --- /dev/null +++ b/src/math/x86_64/sqrtf.c @@ -0,0 +1,7 @@ +#include <math.h> + +float sqrtf(float x) +{ + __asm__ ("sqrtss %1, %0" : "=x"(x) : "x"(x)); + return x; +} diff --git a/src/math/x86_64/sqrtf.s b/src/math/x86_64/sqrtf.s deleted file mode 100644 index eec48c60..00000000 --- a/src/math/x86_64/sqrtf.s +++ /dev/null @@ -1,4 +0,0 @@ -.global sqrtf -.type sqrtf,@function -sqrtf: sqrtss %xmm0, %xmm0 - ret diff --git a/src/math/x86_64/sqrtl.c b/src/math/x86_64/sqrtl.c new file mode 100644 index 00000000..864cfcc4 --- /dev/null +++ b/src/math/x86_64/sqrtl.c @@ -0,0 +1,7 @@ +#include <math.h> + +long double sqrtl(long double x) +{ + __asm__ ("fsqrt" : "+t"(x)); + return x; +} diff --git a/src/math/x86_64/sqrtl.s b/src/math/x86_64/sqrtl.s deleted file mode 100644 index 23cd687d..00000000 --- a/src/math/x86_64/sqrtl.s +++ /dev/null @@ -1,5 +0,0 @@ -.global sqrtl -.type sqrtl,@function -sqrtl: fldt 8(%rsp) - fsqrt - ret diff --git a/src/misc/getentropy.c b/src/misc/getentropy.c index d2f282ce..651ea95f 100644 --- a/src/misc/getentropy.c +++ b/src/misc/getentropy.c @@ -6,7 +6,7 @@ int getentropy(void *buffer, size_t len) { - int cs, ret; + int cs, ret = 0; char *pos = buffer; if (len > 256) { diff --git a/src/misc/getopt.c b/src/misc/getopt.c index c3f66995..b02b81c3 100644 --- a/src/misc/getopt.c +++ b/src/misc/getopt.c @@ -87,7 +87,8 @@ int getopt(int argc, char * const argv[], const char *optstring) if (optstring[i] == ':') { optarg = 0; if (optstring[i+1] != ':' || optpos) { - optarg = argv[optind++] + optpos; + optarg = argv[optind++]; + if (optpos) optarg += optpos; optpos = 0; } if (optind > argc) { diff --git a/src/misc/getrlimit.c b/src/misc/getrlimit.c index 2ab2f0f4..a5558d81 100644 --- a/src/misc/getrlimit.c +++ b/src/misc/getrlimit.c @@ -6,12 +6,13 @@ int getrlimit(int resource, struct rlimit *rlim) { - unsigned long k_rlim[2]; int ret = syscall(SYS_prlimit64, 0, resource, 0, rlim); if (!ret) { FIX(rlim->rlim_cur); FIX(rlim->rlim_max); } +#ifdef SYS_getrlimit + unsigned long k_rlim[2]; if (!ret || errno != ENOSYS) return ret; if (syscall(SYS_getrlimit, resource, k_rlim) < 0) @@ -21,6 +22,7 @@ int getrlimit(int resource, struct rlimit *rlim) FIX(rlim->rlim_cur); FIX(rlim->rlim_max); return 0; +#else + return ret; +#endif } - -weak_alias(getrlimit, getrlimit64); diff --git a/src/misc/ioctl.c b/src/misc/ioctl.c index 6f31d4bc..35804f02 100644 --- a/src/misc/ioctl.c +++ b/src/misc/ioctl.c @@ -3,8 +3,128 @@ #include <errno.h> #include <time.h> #include <sys/time.h> +#include <stddef.h> +#include <stdint.h> +#include <string.h> +#include <endian.h> #include "syscall.h" +#define alignof(t) offsetof(struct { char c; t x; }, x) + +#define W 1 +#define R 2 +#define WR 3 + +struct ioctl_compat_map { + int new_req, old_req; + unsigned char old_size, dir, force_align, noffs; + unsigned char offsets[8]; +}; + +#define NINTH(a,b,c,d,e,f,g,h,i,...) i +#define COUNT(...) NINTH(__VA_ARGS__,8,7,6,5,4,3,2,1,0) +#define OFFS(...) COUNT(__VA_ARGS__), { __VA_ARGS__ } + +/* yields a type for a struct with original size n, with a misaligned + * timeval/timespec expanded from 32- to 64-bit. for use with ioctl + * number producing macros; only size of result is meaningful. */ +#define new_misaligned(n) struct { int i; time_t t; char c[(n)-4]; } + +struct v4l2_event { + uint32_t a; + uint64_t b[8]; + uint32_t c[2], ts[2], d[9]; +}; + +static const struct ioctl_compat_map compat_map[] = { + { SIOCGSTAMP, SIOCGSTAMP_OLD, 8, R, 0, OFFS(0, 4) }, + { SIOCGSTAMPNS, SIOCGSTAMPNS_OLD, 8, R, 0, OFFS(0, 4) }, + + /* SNDRV_TIMER_IOCTL_STATUS */ + { _IOR('T', 0x14, char[96]), _IOR('T', 0x14, 88), 88, R, 0, OFFS(0,4) }, + + /* SNDRV_PCM_IOCTL_STATUS[_EXT] */ + { _IOR('A', 0x20, char[128]), _IOR('A', 0x20, char[108]), 108, R, 1, OFFS(4,8,12,16,52,56,60,64) }, + { _IOWR('A', 0x24, char[128]), _IOWR('A', 0x24, char[108]), 108, WR, 1, OFFS(4,8,12,16,52,56,60,64) }, + + /* SNDRV_RAWMIDI_IOCTL_STATUS */ + { _IOWR('W', 0x20, char[48]), _IOWR('W', 0x20, char[36]), 36, WR, 1, OFFS(4,8) }, + + /* SNDRV_PCM_IOCTL_SYNC_PTR - with 3 subtables */ + { _IOWR('A', 0x23, char[136]), _IOWR('A', 0x23, char[132]), 0, WR, 1, 0 }, + { 0, 0, 4, WR, 1, 0 }, /* snd_pcm_sync_ptr (flags only) */ + { 0, 0, 32, WR, 1, OFFS(8,12,16,24,28) }, /* snd_pcm_mmap_status */ + { 0, 0, 4, WR, 1, 0 }, /* snd_pcm_mmap_control (each member) */ + + /* VIDIOC_QUERYBUF, VIDIOC_QBUF, VIDIOC_DQBUF, VIDIOC_PREPARE_BUF */ + { _IOWR('V', 9, new_misaligned(68)), _IOWR('V', 9, char[68]), 68, WR, 1, OFFS(20, 24) }, + { _IOWR('V', 15, new_misaligned(68)), _IOWR('V', 15, char[68]), 68, WR, 1, OFFS(20, 24) }, + { _IOWR('V', 17, new_misaligned(68)), _IOWR('V', 17, char[68]), 68, WR, 1, OFFS(20, 24) }, + { _IOWR('V', 93, new_misaligned(68)), _IOWR('V', 93, char[68]), 68, WR, 1, OFFS(20, 24) }, + + /* VIDIOC_DQEVENT */ + { _IOR('V', 89, new_misaligned(120)), _IOR('V', 89, struct v4l2_event), sizeof(struct v4l2_event), + R, 0, OFFS(offsetof(struct v4l2_event, ts[0]), offsetof(struct v4l2_event, ts[1])) }, + + /* VIDIOC_OMAP3ISP_STAT_REQ */ + { _IOWR('V', 192+6, char[32]), _IOWR('V', 192+6, char[24]), 22, WR, 0, OFFS(0,4) }, + + /* PPPIOCGIDLE */ + { _IOR('t', 63, char[16]), _IOR('t', 63, char[8]), 8, R, 0, OFFS(0,4) }, + + /* PPGETTIME, PPSETTIME */ + { _IOR('p', 0x95, char[16]), _IOR('p', 0x95, char[8]), 8, R, 0, OFFS(0,4) }, + { _IOW('p', 0x96, char[16]), _IOW('p', 0x96, char[8]), 8, W, 0, OFFS(0,4) }, + + /* LPSETTIMEOUT */ + { _IOW(0x6, 0xf, char[16]), 0x060f, 8, W, 0, OFFS(0,4) }, +}; + +static void convert_ioctl_struct(const struct ioctl_compat_map *map, char *old, char *new, int dir) +{ + int new_offset = 0; + int old_offset = 0; + int old_size = map->old_size; + if (!(dir & map->dir)) return; + if (!map->old_size) { + /* offsets hard-coded for SNDRV_PCM_IOCTL_SYNC_PTR; + * if another exception appears this needs changing. */ + convert_ioctl_struct(map+1, old, new, dir); + convert_ioctl_struct(map+2, old+4, new+8, dir); + /* snd_pcm_mmap_control, special-cased due to kernel + * type definition having been botched. */ + int adj = BYTE_ORDER==BIG_ENDIAN ? 4 : 0; + convert_ioctl_struct(map+3, old+68, new+72+adj, dir); + convert_ioctl_struct(map+3, old+72, new+76+3*adj, dir); + return; + } + for (int i=0; i < map->noffs; i++) { + int ts_offset = map->offsets[i]; + int len = ts_offset-old_offset; + if (dir==W) memcpy(old+old_offset, new+new_offset, len); + else memcpy(new+new_offset, old+old_offset, len); + new_offset += len; + old_offset += len; + long long new_ts; + long old_ts; + int align = map->force_align ? sizeof(time_t) : alignof(time_t); + new_offset += (align-1) & -new_offset; + if (dir==W) { + memcpy(&new_ts, new+new_offset, sizeof new_ts); + old_ts = new_ts; + memcpy(old+old_offset, &old_ts, sizeof old_ts); + } else { + memcpy(&old_ts, old+old_offset, sizeof old_ts); + new_ts = old_ts; + memcpy(new+new_offset, &new_ts, sizeof new_ts); + } + new_offset += sizeof new_ts; + old_offset += sizeof old_ts; + } + if (dir==W) memcpy(old+old_offset, new+new_offset, old_size-old_offset); + else memcpy(new+new_offset, old+old_offset, old_size-old_offset); +} + int ioctl(int fd, int req, ...) { void *arg; @@ -13,23 +133,18 @@ int ioctl(int fd, int req, ...) arg = va_arg(ap, void *); va_end(ap); int r = __syscall(SYS_ioctl, fd, req, arg); - if (r==-ENOTTY) switch (req) { - case SIOCGSTAMP: - case SIOCGSTAMPNS: - if (SIOCGSTAMP==SIOCGSTAMP_OLD) break; - if (req==SIOCGSTAMP) req=SIOCGSTAMP_OLD; - if (req==SIOCGSTAMPNS) req=SIOCGSTAMPNS_OLD; - long t32[2]; - r = __syscall(SYS_ioctl, fd, req, t32); - if (r<0) break; - if (req==SIOCGSTAMP_OLD) { - struct timeval *tv = arg; - tv->tv_sec = t32[0]; - tv->tv_usec = t32[1]; - } else { - struct timespec *ts = arg; - ts->tv_sec = t32[0]; - ts->tv_nsec = t32[1]; + if (SIOCGSTAMP != SIOCGSTAMP_OLD && req && r==-ENOTTY) { + for (int i=0; i<sizeof compat_map/sizeof *compat_map; i++) { + if (compat_map[i].new_req != req) continue; + union { + long long align; + char buf[256]; + } u; + convert_ioctl_struct(&compat_map[i], u.buf, arg, W); + r = __syscall(SYS_ioctl, fd, compat_map[i].old_req, u.buf); + if (r<0) break; + convert_ioctl_struct(&compat_map[i], u.buf, arg, R); + break; } } return __syscall_ret(r); diff --git a/src/misc/lockf.c b/src/misc/lockf.c index 16a80bec..0162442b 100644 --- a/src/misc/lockf.c +++ b/src/misc/lockf.c @@ -28,5 +28,3 @@ int lockf(int fd, int op, off_t size) errno = EINVAL; return -1; } - -weak_alias(lockf, lockf64); diff --git a/src/misc/mntent.c b/src/misc/mntent.c index eabb8200..78bf0cd0 100644 --- a/src/misc/mntent.c +++ b/src/misc/mntent.c @@ -2,6 +2,7 @@ #include <string.h> #include <mntent.h> #include <errno.h> +#include <limits.h> static char *internal_buf; static size_t internal_bufsize; @@ -19,9 +20,46 @@ int endmntent(FILE *f) return 1; } +static char *unescape_ent(char *beg) +{ + char *dest = beg; + const char *src = beg; + while (*src) { + const char *val; + unsigned char cval = 0; + if (*src != '\\') { + *dest++ = *src++; + continue; + } + if (src[1] == '\\') { + ++src; + *dest++ = *src++; + continue; + } + val = src + 1; + for (int i = 0; i < 3; ++i) { + if (*val >= '0' && *val <= '7') { + cval <<= 3; + cval += *val++ - '0'; + } else { + break; + } + } + if (cval) { + *dest++ = cval; + src = val; + } else { + *dest++ = *src++; + } + } + *dest = 0; + return beg; +} + struct mntent *getmntent_r(FILE *f, struct mntent *mnt, char *linebuf, int buflen) { - int cnt, n[8], use_internal = (linebuf == SENTINEL); + int n[8], use_internal = (linebuf == SENTINEL); + size_t len, i; mnt->mnt_freq = 0; mnt->mnt_passno = 0; @@ -39,20 +77,24 @@ struct mntent *getmntent_r(FILE *f, struct mntent *mnt, char *linebuf, int bufle errno = ERANGE; return 0; } - cnt = sscanf(linebuf, " %n%*s%n %n%*s%n %n%*s%n %n%*s%n %d %d", + + len = strlen(linebuf); + if (len > INT_MAX) continue; + for (i = 0; i < sizeof n / sizeof *n; i++) n[i] = len; + sscanf(linebuf, " %n%*[^ \t]%n %n%*[^ \t]%n %n%*[^ \t]%n %n%*[^ \t]%n %d %d", n, n+1, n+2, n+3, n+4, n+5, n+6, n+7, &mnt->mnt_freq, &mnt->mnt_passno); - } while (cnt < 2 || linebuf[n[0]] == '#'); + } while (linebuf[n[0]] == '#' || n[1]==len); linebuf[n[1]] = 0; linebuf[n[3]] = 0; linebuf[n[5]] = 0; linebuf[n[7]] = 0; - mnt->mnt_fsname = linebuf+n[0]; - mnt->mnt_dir = linebuf+n[2]; - mnt->mnt_type = linebuf+n[4]; - mnt->mnt_opts = linebuf+n[6]; + mnt->mnt_fsname = unescape_ent(linebuf+n[0]); + mnt->mnt_dir = unescape_ent(linebuf+n[2]); + mnt->mnt_type = unescape_ent(linebuf+n[4]); + mnt->mnt_opts = unescape_ent(linebuf+n[6]); return mnt; } diff --git a/src/misc/nftw.c b/src/misc/nftw.c index 0a464100..71bc62ee 100644 --- a/src/misc/nftw.c +++ b/src/misc/nftw.c @@ -1,5 +1,6 @@ #include <ftw.h> #include <dirent.h> +#include <fcntl.h> #include <sys/stat.h> #include <errno.h> #include <unistd.h> @@ -26,16 +27,19 @@ static int do_nftw(char *path, int (*fn)(const char *, const struct stat *, int, struct history new; int type; int r; + int dfd; + int err; struct FTW lev; + st.st_dev = st.st_ino = 0; + if ((flags & FTW_PHYS) ? lstat(path, &st) : stat(path, &st) < 0) { if (!(flags & FTW_PHYS) && errno==ENOENT && !lstat(path, &st)) type = FTW_SLN; else if (errno != EACCES) return -1; else type = FTW_NS; } else if (S_ISDIR(st.st_mode)) { - if (access(path, R_OK) < 0) type = FTW_DNR; - else if (flags & FTW_DEPTH) type = FTW_DP; + if (flags & FTW_DEPTH) type = FTW_DP; else type = FTW_D; } else if (S_ISLNK(st.st_mode)) { if (flags & FTW_PHYS) type = FTW_SL; @@ -44,7 +48,7 @@ static int do_nftw(char *path, int (*fn)(const char *, const struct stat *, int, type = FTW_F; } - if ((flags & FTW_MOUNT) && h && st.st_dev != h->dev) + if ((flags & FTW_MOUNT) && h && type != FTW_NS && st.st_dev != h->dev) return 0; new.chain = h; @@ -63,6 +67,13 @@ static int do_nftw(char *path, int (*fn)(const char *, const struct stat *, int, lev.base = k; } + if (type == FTW_D || type == FTW_DP) { + dfd = open(path, O_RDONLY); + err = errno; + if (dfd < 0 && err == EACCES) type = FTW_DNR; + if (!fd_limit) close(dfd); + } + if (!(flags & FTW_DEPTH) && (r=fn(path, &st, type, &lev))) return r; @@ -71,7 +82,11 @@ static int do_nftw(char *path, int (*fn)(const char *, const struct stat *, int, return 0; if ((type == FTW_D || type == FTW_DP) && fd_limit) { - DIR *d = opendir(path); + if (dfd < 0) { + errno = err; + return -1; + } + DIR *d = fdopendir(dfd); if (d) { struct dirent *de; while ((de = readdir(d))) { @@ -92,7 +107,8 @@ static int do_nftw(char *path, int (*fn)(const char *, const struct stat *, int, } } closedir(d); - } else if (errno != EACCES) { + } else { + close(dfd); return -1; } } @@ -124,5 +140,3 @@ int nftw(const char *path, int (*fn)(const char *, const struct stat *, int, str pthread_setcancelstate(cs, 0); return r; } - -weak_alias(nftw, nftw64); diff --git a/src/misc/pty.c b/src/misc/pty.c index b9cb5eaa..a0577147 100644 --- a/src/misc/pty.c +++ b/src/misc/pty.c @@ -7,7 +7,9 @@ int posix_openpt(int flags) { - return open("/dev/ptmx", flags); + int r = open("/dev/ptmx", flags); + if (r < 0 && errno == ENOSPC) errno = EAGAIN; + return r; } int grantpt(int fd) diff --git a/src/misc/realpath.c b/src/misc/realpath.c index d2708e59..db8b74dc 100644 --- a/src/misc/realpath.c +++ b/src/misc/realpath.c @@ -1,43 +1,156 @@ #include <stdlib.h> #include <limits.h> -#include <sys/stat.h> -#include <fcntl.h> #include <errno.h> #include <unistd.h> #include <string.h> -#include "syscall.h" + +static size_t slash_len(const char *s) +{ + const char *s0 = s; + while (*s == '/') s++; + return s-s0; +} char *realpath(const char *restrict filename, char *restrict resolved) { - int fd; - ssize_t r; - struct stat st1, st2; - char buf[15+3*sizeof(int)]; - char tmp[PATH_MAX]; + char stack[PATH_MAX+1]; + char output[PATH_MAX]; + size_t p, q, l, l0, cnt=0, nup=0; + int check_dir=0; if (!filename) { errno = EINVAL; return 0; } + l = strnlen(filename, sizeof stack); + if (!l) { + errno = ENOENT; + return 0; + } + if (l >= PATH_MAX) goto toolong; + p = sizeof stack - l - 1; + q = 0; + memcpy(stack+p, filename, l+1); + + /* Main loop. Each iteration pops the next part from stack of + * remaining path components and consumes any slashes that follow. + * If not a link, it's moved to output; if a link, contents are + * pushed to the stack. */ +restart: + for (; ; p+=slash_len(stack+p)) { + /* If stack starts with /, the whole component is / or // + * and the output state must be reset. */ + if (stack[p] == '/') { + check_dir=0; + nup=0; + q=0; + output[q++] = '/'; + p++; + /* Initial // is special. */ + if (stack[p] == '/' && stack[p+1] != '/') + output[q++] = '/'; + continue; + } + + char *z = __strchrnul(stack+p, '/'); + l0 = l = z-(stack+p); - fd = sys_open(filename, O_PATH|O_NONBLOCK|O_CLOEXEC); - if (fd < 0) return 0; - __procfdname(buf, fd); + if (!l && !check_dir) break; - r = readlink(buf, tmp, sizeof tmp - 1); - if (r < 0) goto err; - tmp[r] = 0; + /* Skip any . component but preserve check_dir status. */ + if (l==1 && stack[p]=='.') { + p += l; + continue; + } - fstat(fd, &st1); - r = stat(tmp, &st2); - if (r<0 || st1.st_dev != st2.st_dev || st1.st_ino != st2.st_ino) { - if (!r) errno = ELOOP; - goto err; + /* Copy next component onto output at least temporarily, to + * call readlink, but wait to advance output position until + * determining it's not a link. */ + if (q && output[q-1] != '/') { + if (!p) goto toolong; + stack[--p] = '/'; + l++; + } + if (q+l >= PATH_MAX) goto toolong; + memcpy(output+q, stack+p, l); + output[q+l] = 0; + p += l; + + int up = 0; + if (l0==2 && stack[p-2]=='.' && stack[p-1]=='.') { + up = 1; + /* Any non-.. path components we could cancel start + * after nup repetitions of the 3-byte string "../"; + * if there are none, accumulate .. components to + * later apply to cwd, if needed. */ + if (q <= 3*nup) { + nup++; + q += l; + continue; + } + /* When previous components are already known to be + * directories, processing .. can skip readlink. */ + if (!check_dir) goto skip_readlink; + } + ssize_t k = readlink(output, stack, p); + if (k==p) goto toolong; + if (!k) { + errno = ENOENT; + return 0; + } + if (k<0) { + if (errno != EINVAL) return 0; +skip_readlink: + check_dir = 0; + if (up) { + while(q && output[q-1]!='/') q--; + if (q>1 && (q>2 || output[0]!='/')) q--; + continue; + } + if (l0) q += l; + check_dir = stack[p]; + continue; + } + if (++cnt == SYMLOOP_MAX) { + errno = ELOOP; + return 0; + } + + /* If link contents end in /, strip any slashes already on + * stack to avoid /->// or //->/// or spurious toolong. */ + if (stack[k-1]=='/') while (stack[p]=='/') p++; + p -= k; + memmove(stack+p, stack, k); + + /* Skip the stack advancement in case we have a new + * absolute base path. */ + goto restart; } - __syscall(SYS_close, fd); - return resolved ? strcpy(resolved, tmp) : strdup(tmp); -err: - __syscall(SYS_close, fd); + output[q] = 0; + + if (output[0] != '/') { + if (!getcwd(stack, sizeof stack)) return 0; + l = strlen(stack); + /* Cancel any initial .. components. */ + p = 0; + while (nup--) { + while(l>1 && stack[l-1]!='/') l--; + if (l>1) l--; + p += 2; + if (p<q) p++; + } + if (q-p && stack[l-1]!='/') stack[l++] = '/'; + if (l + (q-p) + 1 >= PATH_MAX) goto toolong; + memmove(output + l, output + p, q - p + 1); + memcpy(output, stack, l); + q = l + q-p; + } + + if (resolved) return memcpy(resolved, output, q+1); + else return strdup(output); + +toolong: + errno = ENAMETOOLONG; return 0; } diff --git a/src/misc/setrlimit.c b/src/misc/setrlimit.c index 7a66ab29..edb413fa 100644 --- a/src/misc/setrlimit.c +++ b/src/misc/setrlimit.c @@ -6,45 +6,46 @@ #define MIN(a, b) ((a)<(b) ? (a) : (b)) #define FIX(x) do{ if ((x)>=SYSCALL_RLIM_INFINITY) (x)=RLIM_INFINITY; }while(0) -static int __setrlimit(int resource, const struct rlimit *rlim) -{ - unsigned long k_rlim[2]; - struct rlimit tmp; - if (SYSCALL_RLIM_INFINITY != RLIM_INFINITY) { - tmp = *rlim; - FIX(tmp.rlim_cur); - FIX(tmp.rlim_max); - rlim = &tmp; - } - int ret = __syscall(SYS_prlimit64, 0, resource, rlim, 0); - if (ret != -ENOSYS) return ret; - k_rlim[0] = MIN(rlim->rlim_cur, MIN(-1UL, SYSCALL_RLIM_INFINITY)); - k_rlim[1] = MIN(rlim->rlim_max, MIN(-1UL, SYSCALL_RLIM_INFINITY)); - return __syscall(SYS_setrlimit, resource, k_rlim); -} - struct ctx { - const struct rlimit *rlim; + unsigned long lim[2]; int res; int err; }; +#ifdef SYS_setrlimit static void do_setrlimit(void *p) { struct ctx *c = p; if (c->err>0) return; - c->err = -__setrlimit(c->res, c->rlim); + c->err = -__syscall(SYS_setrlimit, c->res, c->lim); } +#endif int setrlimit(int resource, const struct rlimit *rlim) { - struct ctx c = { .res = resource, .rlim = rlim, .err = -1 }; + struct rlimit tmp; + if (SYSCALL_RLIM_INFINITY != RLIM_INFINITY) { + tmp = *rlim; + FIX(tmp.rlim_cur); + FIX(tmp.rlim_max); + rlim = &tmp; + } + int ret = __syscall(SYS_prlimit64, 0, resource, rlim, 0); +#ifdef SYS_setrlimit + if (ret != -ENOSYS) return __syscall_ret(ret); + + struct ctx c = { + .lim[0] = MIN(rlim->rlim_cur, MIN(-1UL, SYSCALL_RLIM_INFINITY)), + .lim[1] = MIN(rlim->rlim_max, MIN(-1UL, SYSCALL_RLIM_INFINITY)), + .res = resource, .err = -1 + }; __synccall(do_setrlimit, &c); if (c.err) { if (c.err>0) errno = c.err; return -1; } return 0; +#else + return __syscall_ret(ret); +#endif } - -weak_alias(setrlimit, setrlimit64); diff --git a/src/misc/syslog.c b/src/misc/syslog.c index 13d4b0a6..710202f9 100644 --- a/src/misc/syslog.c +++ b/src/misc/syslog.c @@ -10,6 +10,8 @@ #include <errno.h> #include <fcntl.h> #include "lock.h" +#include "fork_impl.h" +#include "locale_impl.h" static volatile int lock[1]; static char log_ident[32]; @@ -17,6 +19,7 @@ static int log_opt; static int log_facility = LOG_USER; static int log_mask = 0xff; static int log_fd = -1; +volatile int *const __syslog_lockptr = lock; int setlogmask(int maskpri) { @@ -97,7 +100,7 @@ static void _vsyslog(int priority, const char *message, va_list ap) now = time(NULL); gmtime_r(&now, &tm); - strftime(timebuf, sizeof timebuf, "%b %e %T", &tm); + strftime_l(timebuf, sizeof timebuf, "%b %e %T", &tm, C_LOCALE); pid = (log_opt & LOG_PID) ? getpid() : 0; l = snprintf(buf, sizeof buf, "<%d>%s %n%s%s%.0d%s: ", diff --git a/src/mman/mmap.c b/src/mman/mmap.c index eff88d82..43e5e029 100644 --- a/src/mman/mmap.c +++ b/src/mman/mmap.c @@ -37,5 +37,3 @@ void *__mmap(void *start, size_t len, int prot, int flags, int fd, off_t off) } weak_alias(__mmap, mmap); - -weak_alias(mmap, mmap64); diff --git a/src/mq/mq_notify.c b/src/mq/mq_notify.c index 221591c7..0e1e6c7a 100644 --- a/src/mq/mq_notify.c +++ b/src/mq/mq_notify.c @@ -4,11 +4,14 @@ #include <sys/socket.h> #include <signal.h> #include <unistd.h> +#include <semaphore.h> #include "syscall.h" struct args { - pthread_barrier_t barrier; + sem_t sem; int sock; + mqd_t mqd; + int err; const struct sigevent *sev; }; @@ -20,8 +23,19 @@ static void *start(void *p) int s = args->sock; void (*func)(union sigval) = args->sev->sigev_notify_function; union sigval val = args->sev->sigev_value; + struct sigevent sev2; + static const char zeros[32]; + int err; + + sev2.sigev_notify = SIGEV_THREAD; + sev2.sigev_signo = s; + sev2.sigev_value.sival_ptr = (void *)&zeros; + + args->err = err = -__syscall(SYS_mq_notify, args->mqd, &sev2); + sem_post(&args->sem); + if (err) return 0; - pthread_barrier_wait(&args->barrier); + pthread_detach(pthread_self()); n = recv(s, buf, sizeof(buf), MSG_NOSIGNAL|MSG_WAITALL); close(s); if (n==sizeof buf && buf[sizeof buf - 1] == 1) @@ -35,8 +49,8 @@ int mq_notify(mqd_t mqd, const struct sigevent *sev) pthread_attr_t attr; pthread_t td; int s; - struct sigevent sev2; - static const char zeros[32]; + int cs; + sigset_t allmask, origmask; if (!sev || sev->sigev_notify != SIGEV_THREAD) return syscall(SYS_mq_notify, mqd, sev); @@ -44,30 +58,35 @@ int mq_notify(mqd_t mqd, const struct sigevent *sev) s = socket(AF_NETLINK, SOCK_RAW|SOCK_CLOEXEC, 0); if (s < 0) return -1; args.sock = s; + args.mqd = mqd; if (sev->sigev_notify_attributes) attr = *sev->sigev_notify_attributes; else pthread_attr_init(&attr); - pthread_attr_setdetachstate(&attr, PTHREAD_CREATE_DETACHED); - pthread_barrier_init(&args.barrier, 0, 2); + pthread_attr_setdetachstate(&attr, PTHREAD_CREATE_JOINABLE); + sem_init(&args.sem, 0, 0); + sigfillset(&allmask); + pthread_sigmask(SIG_BLOCK, &allmask, &origmask); if (pthread_create(&td, &attr, start, &args)) { __syscall(SYS_close, s); + pthread_sigmask(SIG_SETMASK, &origmask, 0); errno = EAGAIN; return -1; } + pthread_sigmask(SIG_SETMASK, &origmask, 0); - pthread_barrier_wait(&args.barrier); - pthread_barrier_destroy(&args.barrier); - - sev2.sigev_notify = SIGEV_THREAD; - sev2.sigev_signo = s; - sev2.sigev_value.sival_ptr = (void *)&zeros; + pthread_setcancelstate(PTHREAD_CANCEL_DISABLE, &cs); + sem_wait(&args.sem); + sem_destroy(&args.sem); - if (syscall(SYS_mq_notify, mqd, &sev2) < 0) { - pthread_cancel(td); + if (args.err) { __syscall(SYS_close, s); + pthread_join(td, 0); + pthread_setcancelstate(cs, 0); + errno = args.err; return -1; } + pthread_setcancelstate(cs, 0); return 0; } diff --git a/src/multibyte/mbrtowc.c b/src/multibyte/mbrtowc.c index c94819e7..7824997e 100644 --- a/src/multibyte/mbrtowc.c +++ b/src/multibyte/mbrtowc.c @@ -8,7 +8,7 @@ size_t mbrtowc(wchar_t *restrict wc, const char *restrict src, size_t n, mbstate static unsigned internal_state; unsigned c; const unsigned char *s = (const void *)src; - const unsigned N = n; + const size_t N = n; wchar_t dummy; if (!st) st = (void *)&internal_state; diff --git a/src/multibyte/wcsnrtombs.c b/src/multibyte/wcsnrtombs.c index 676932b5..95e25e70 100644 --- a/src/multibyte/wcsnrtombs.c +++ b/src/multibyte/wcsnrtombs.c @@ -1,41 +1,33 @@ #include <wchar.h> +#include <limits.h> +#include <string.h> size_t wcsnrtombs(char *restrict dst, const wchar_t **restrict wcs, size_t wn, size_t n, mbstate_t *restrict st) { - size_t l, cnt=0, n2; - char *s, buf[256]; const wchar_t *ws = *wcs; - const wchar_t *tmp_ws; - - if (!dst) s = buf, n = sizeof buf; - else s = dst; - - while ( ws && n && ( (n2=wn)>=n || n2>32 ) ) { - if (n2>=n) n2=n; - tmp_ws = ws; - l = wcsrtombs(s, &ws, n2, 0); - if (!(l+1)) { - cnt = l; - n = 0; + size_t cnt = 0; + if (!dst) n=0; + while (ws && wn) { + char tmp[MB_LEN_MAX]; + size_t l = wcrtomb(n<MB_LEN_MAX ? tmp : dst, *ws, 0); + if (l==-1) { + cnt = -1; break; } - if (s != buf) { - s += l; + if (dst) { + if (n<MB_LEN_MAX) { + if (l>n) break; + memcpy(dst, tmp, l); + } + dst += l; n -= l; } - wn = ws ? wn - (ws - tmp_ws) : 0; - cnt += l; - } - if (ws) while (n && wn) { - l = wcrtomb(s, *ws, 0); - if ((l+1)<=1) { - if (!l) ws = 0; - else cnt = l; + if (!*ws) { + ws = 0; break; } - ws++; wn--; - /* safe - this loop runs fewer than sizeof(buf) times */ - s+=l; n-=l; + ws++; + wn--; cnt += l; } if (dst) *wcs = ws; diff --git a/src/network/accept4.c b/src/network/accept4.c index 59ab1726..765a38ed 100644 --- a/src/network/accept4.c +++ b/src/network/accept4.c @@ -9,6 +9,10 @@ int accept4(int fd, struct sockaddr *restrict addr, socklen_t *restrict len, int if (!flg) return accept(fd, addr, len); int ret = socketcall_cp(accept4, fd, addr, len, flg, 0, 0); if (ret>=0 || (errno != ENOSYS && errno != EINVAL)) return ret; + if (flg & ~(SOCK_CLOEXEC|SOCK_NONBLOCK)) { + errno = EINVAL; + return -1; + } ret = accept(fd, addr, len); if (ret<0) return ret; if (flg & SOCK_CLOEXEC) diff --git a/src/network/dns_parse.c b/src/network/dns_parse.c index e6ee19d9..09813112 100644 --- a/src/network/dns_parse.c +++ b/src/network/dns_parse.c @@ -1,7 +1,7 @@ #include <string.h> #include "lookup.h" -int __dns_parse(const unsigned char *r, int rlen, int (*callback)(void *, int, const void *, int, const void *), void *ctx) +int __dns_parse(const unsigned char *r, int rlen, int (*callback)(void *, int, const void *, int, const void *, int), void *ctx) { int qdcount, ancount; const unsigned char *p; @@ -12,21 +12,20 @@ int __dns_parse(const unsigned char *r, int rlen, int (*callback)(void *, int, c p = r+12; qdcount = r[4]*256 + r[5]; ancount = r[6]*256 + r[7]; - if (qdcount+ancount > 64) return -1; while (qdcount--) { while (p-r < rlen && *p-1U < 127) p++; - if (*p>193 || (*p==193 && p[1]>254) || p>r+rlen-6) + if (p>r+rlen-6) return -1; p += 5 + !!*p; } while (ancount--) { while (p-r < rlen && *p-1U < 127) p++; - if (*p>193 || (*p==193 && p[1]>254) || p>r+rlen-6) + if (p>r+rlen-12) return -1; p += 1 + !!*p; len = p[8]*256 + p[9]; - if (p+len > r+rlen) return -1; - if (callback(ctx, p[1], p+10, len, r) < 0) return -1; + if (len+10 > r+rlen-p) return -1; + if (callback(ctx, p[1], p+10, len, r, rlen) < 0) return -1; p += 10 + len; } return 0; diff --git a/src/network/gai_strerror.c b/src/network/gai_strerror.c index 9596580e..56b71503 100644 --- a/src/network/gai_strerror.c +++ b/src/network/gai_strerror.c @@ -6,7 +6,7 @@ static const char msgs[] = "Name does not resolve\0" "Try again\0" "Non-recoverable error\0" - "Unknown error\0" + "Name has no usable address\0" "Unrecognized address family or invalid length\0" "Unrecognized socket type\0" "Unrecognized service\0" diff --git a/src/network/getaddrinfo.c b/src/network/getaddrinfo.c index efaab306..64ad259a 100644 --- a/src/network/getaddrinfo.c +++ b/src/network/getaddrinfo.c @@ -16,6 +16,7 @@ int getaddrinfo(const char *restrict host, const char *restrict serv, const stru char canon[256], *outcanon; int nservs, naddrs, nais, canon_len, i, j, k; int family = AF_UNSPEC, flags = 0, proto = 0, socktype = 0; + int no_family = 0; struct aibuf *out; if (!host && !serv) return EAI_NONAME; @@ -66,9 +67,11 @@ int getaddrinfo(const char *restrict host, const char *restrict serv, const stru pthread_setcancelstate( PTHREAD_CANCEL_DISABLE, &cs); int r = connect(s, ta[i], tl[i]); + int saved_errno = errno; pthread_setcancelstate(cs, 0); close(s); if (!r) continue; + errno = saved_errno; } switch (errno) { case EADDRNOTAVAIL: @@ -80,7 +83,7 @@ int getaddrinfo(const char *restrict host, const char *restrict serv, const stru default: return EAI_SYSTEM; } - if (family == tf[i]) return EAI_NONAME; + if (family == tf[i]) no_family = 1; family = tf[1-i]; } } @@ -91,6 +94,8 @@ int getaddrinfo(const char *restrict host, const char *restrict serv, const stru naddrs = __lookup_name(addrs, canon, host, family, flags); if (naddrs < 0) return naddrs; + if (no_family) return EAI_NODATA; + nais = nservs * naddrs; canon_len = strlen(canon); out = calloc(1, nais * sizeof(*out) + canon_len + 1); diff --git a/src/network/gethostbyaddr.c b/src/network/gethostbyaddr.c index 598e2241..c3cacaac 100644 --- a/src/network/gethostbyaddr.c +++ b/src/network/gethostbyaddr.c @@ -20,5 +20,5 @@ struct hostent *gethostbyaddr(const void *a, socklen_t l, int af) err = gethostbyaddr_r(a, l, af, h, (void *)(h+1), size-sizeof *h, &res, &h_errno); } while (err == ERANGE); - return err ? 0 : h; + return res; } diff --git a/src/network/gethostbyaddr_r.c b/src/network/gethostbyaddr_r.c index 0f1e61aa..ceaf3935 100644 --- a/src/network/gethostbyaddr_r.c +++ b/src/network/gethostbyaddr_r.c @@ -54,10 +54,11 @@ int gethostbyaddr_r(const void *a, socklen_t l, int af, case EAI_OVERFLOW: return ERANGE; default: - case EAI_MEMORY: - case EAI_SYSTEM: case EAI_FAIL: *err = NO_RECOVERY; + return EBADMSG; + case EAI_SYSTEM: + *err = NO_RECOVERY; return errno; case 0: break; diff --git a/src/network/gethostbyname2.c b/src/network/gethostbyname2.c index dc9d6621..bd0da7f8 100644 --- a/src/network/gethostbyname2.c +++ b/src/network/gethostbyname2.c @@ -21,5 +21,5 @@ struct hostent *gethostbyname2(const char *name, int af) err = gethostbyname2_r(name, af, h, (void *)(h+1), size-sizeof *h, &res, &h_errno); } while (err == ERANGE); - return err ? 0 : h; + return res; } diff --git a/src/network/gethostbyname2_r.c b/src/network/gethostbyname2_r.c index fc894877..a5eb67fe 100644 --- a/src/network/gethostbyname2_r.c +++ b/src/network/gethostbyname2_r.c @@ -22,7 +22,10 @@ int gethostbyname2_r(const char *name, int af, if (cnt<0) switch (cnt) { case EAI_NONAME: *err = HOST_NOT_FOUND; - return ENOENT; + return 0; + case EAI_NODATA: + *err = NO_DATA; + return 0; case EAI_AGAIN: *err = TRY_AGAIN; return EAGAIN; @@ -30,7 +33,6 @@ int gethostbyname2_r(const char *name, int af, case EAI_FAIL: *err = NO_RECOVERY; return EBADMSG; - case EAI_MEMORY: case EAI_SYSTEM: *err = NO_RECOVERY; return errno; diff --git a/src/network/getifaddrs.c b/src/network/getifaddrs.c index fed75bd8..74df4d6c 100644 --- a/src/network/getifaddrs.c +++ b/src/network/getifaddrs.c @@ -39,8 +39,8 @@ struct ifaddrs_storage { }; struct ifaddrs_ctx { - struct ifaddrs_storage *first; - struct ifaddrs_storage *last; + struct ifaddrs *first; + struct ifaddrs *last; struct ifaddrs_storage *hash[IFADDRS_HASH_SIZE]; }; @@ -195,9 +195,9 @@ static int netlink_msg_to_ifaddr(void *pctx, struct nlmsghdr *h) } if (ifs->ifa.ifa_name) { - if (!ctx->first) ctx->first = ifs; - if (ctx->last) ctx->last->ifa.ifa_next = &ifs->ifa; - ctx->last = ifs; + if (!ctx->first) ctx->first = &ifs->ifa; + if (ctx->last) ctx->last->ifa_next = &ifs->ifa; + ctx->last = &ifs->ifa; } else { free(ifs); } @@ -210,7 +210,7 @@ int getifaddrs(struct ifaddrs **ifap) int r; memset(ctx, 0, sizeof *ctx); r = __rtnetlink_enumerate(AF_UNSPEC, AF_UNSPEC, netlink_msg_to_ifaddr, ctx); - if (r == 0) *ifap = &ctx->first->ifa; - else freeifaddrs(&ctx->first->ifa); + if (r == 0) *ifap = ctx->first; + else freeifaddrs(ctx->first); return r; } diff --git a/src/network/getnameinfo.c b/src/network/getnameinfo.c index f77e73ad..133c15b3 100644 --- a/src/network/getnameinfo.c +++ b/src/network/getnameinfo.c @@ -58,6 +58,7 @@ static void reverse_hosts(char *buf, const unsigned char *a, unsigned scopeid, i if ((p=strchr(line, '#'))) *p++='\n', *p=0; for (p=line; *p && !isspace(*p); p++); + if (!*p) continue; *p++ = 0; if (__lookup_ipliteral(&iplit, line, AF_UNSPEC)<=0) continue; @@ -108,10 +109,10 @@ static void reverse_services(char *buf, int port, int dgram) __fclose_ca(f); } -static int dns_parse_callback(void *c, int rr, const void *data, int len, const void *packet) +static int dns_parse_callback(void *c, int rr, const void *data, int len, const void *packet, int plen) { if (rr != RR_PTR) return 0; - if (__dn_expand(packet, (const unsigned char *)packet + 512, + if (__dn_expand(packet, (const unsigned char *)packet + plen, data, c, 256) <= 0) *(char *)c = 0; return 0; @@ -158,10 +159,13 @@ int getnameinfo(const struct sockaddr *restrict sa, socklen_t sl, unsigned char query[18+PTR_MAX], reply[512]; int qlen = __res_mkquery(0, ptr, 1, RR_PTR, 0, 0, 0, query, sizeof query); + query[3] = 0; /* don't need AD flag */ int rlen = __res_send(query, qlen, reply, sizeof reply); buf[0] = 0; - if (rlen > 0) + if (rlen > 0) { + if (rlen > sizeof reply) rlen = sizeof reply; __dns_parse(reply, rlen, dns_parse_callback, buf); + } } if (!*buf) { if (flags & NI_NAMEREQD) return EAI_NONAME; diff --git a/src/network/getservbyport_r.c b/src/network/getservbyport_r.c index b7f21c6b..e4cc3079 100644 --- a/src/network/getservbyport_r.c +++ b/src/network/getservbyport_r.c @@ -26,7 +26,7 @@ int getservbyport_r(int port, const char *prots, /* Align buffer */ i = (uintptr_t)buf & sizeof(char *)-1; if (!i) i = sizeof(char *); - if (buflen < 3*sizeof(char *)-i) + if (buflen <= 3*sizeof(char *)-i) return ERANGE; buf += sizeof(char *)-i; buflen -= sizeof(char *)-i; @@ -46,6 +46,8 @@ int getservbyport_r(int port, const char *prots, case EAI_MEMORY: case EAI_SYSTEM: return ENOMEM; + case EAI_OVERFLOW: + return ERANGE; default: return ENOENT; case 0: diff --git a/src/network/getsockopt.c b/src/network/getsockopt.c index e871d624..d3640d9c 100644 --- a/src/network/getsockopt.c +++ b/src/network/getsockopt.c @@ -26,6 +26,15 @@ int getsockopt(int fd, int level, int optname, void *restrict optval, socklen_t tv->tv_sec = tv32[0]; tv->tv_usec = tv32[1]; *optlen = sizeof *tv; + break; + case SO_TIMESTAMP: + case SO_TIMESTAMPNS: + if (SO_TIMESTAMP == SO_TIMESTAMP_OLD) break; + if (optname==SO_TIMESTAMP) optname=SO_TIMESTAMP_OLD; + if (optname==SO_TIMESTAMPNS) optname=SO_TIMESTAMPNS_OLD; + r = __socketcall(getsockopt, fd, level, + optname, optval, optlen, 0); + break; } } return __syscall_ret(r); diff --git a/src/network/h_errno.c b/src/network/h_errno.c index 4f700cea..638f7718 100644 --- a/src/network/h_errno.c +++ b/src/network/h_errno.c @@ -1,9 +1,11 @@ #include <netdb.h> +#include "pthread_impl.h" #undef h_errno int h_errno; int *__h_errno_location(void) { - return &h_errno; + if (!__pthread_self()->stack) return &h_errno; + return &__pthread_self()->h_errno_val; } diff --git a/src/network/herror.c b/src/network/herror.c index 65f25ff3..87f8cff4 100644 --- a/src/network/herror.c +++ b/src/network/herror.c @@ -4,5 +4,5 @@ void herror(const char *msg) { - fprintf(stderr, "%s%s%s", msg?msg:"", msg?": ":"", hstrerror(h_errno)); + fprintf(stderr, "%s%s%s\n", msg?msg:"", msg?": ":"", hstrerror(h_errno)); } diff --git a/src/network/inet_pton.c b/src/network/inet_pton.c index d36c3689..bcbdd9ef 100644 --- a/src/network/inet_pton.c +++ b/src/network/inet_pton.c @@ -54,6 +54,7 @@ int inet_pton(int af, const char *restrict s, void *restrict a0) if (s[j]!='.' || (i<6 && brk<0)) return 0; need_v4=1; i++; + ip[i&7]=0; break; } s += j+1; diff --git a/src/network/lookup.h b/src/network/lookup.h index ef662725..54b2f8b5 100644 --- a/src/network/lookup.h +++ b/src/network/lookup.h @@ -50,6 +50,6 @@ hidden int __lookup_ipliteral(struct address buf[static 1], const char *name, in hidden int __get_resolv_conf(struct resolvconf *, char *, size_t); hidden int __res_msend_rc(int, const unsigned char *const *, const int *, unsigned char *const *, int *, int, const struct resolvconf *); -hidden int __dns_parse(const unsigned char *, int, int (*)(void *, int, const void *, int, const void *), void *); +hidden int __dns_parse(const unsigned char *, int, int (*)(void *, int, const void *, int, const void *, int), void *); #endif diff --git a/src/network/lookup_ipliteral.c b/src/network/lookup_ipliteral.c index 2fddab73..1e766206 100644 --- a/src/network/lookup_ipliteral.c +++ b/src/network/lookup_ipliteral.c @@ -15,7 +15,7 @@ int __lookup_ipliteral(struct address buf[static 1], const char *name, int famil struct in6_addr a6; if (__inet_aton(name, &a4) > 0) { if (family == AF_INET6) /* wrong family */ - return EAI_NONAME; + return EAI_NODATA; memcpy(&buf[0].addr, &a4, sizeof a4); buf[0].family = AF_INET; buf[0].scopeid = 0; @@ -34,7 +34,7 @@ int __lookup_ipliteral(struct address buf[static 1], const char *name, int famil if (inet_pton(AF_INET6, name, &a6) <= 0) return 0; if (family == AF_INET) /* wrong family */ - return EAI_NONAME; + return EAI_NODATA; memcpy(&buf[0].addr, &a6, sizeof a6); buf[0].family = AF_INET6; diff --git a/src/network/lookup_name.c b/src/network/lookup_name.c index c93263a9..35218185 100644 --- a/src/network/lookup_name.c +++ b/src/network/lookup_name.c @@ -50,7 +50,7 @@ static int name_from_hosts(struct address buf[static MAXADDRS], char canon[stati { char line[512]; size_t l = strlen(name); - int cnt = 0, badfam = 0; + int cnt = 0, badfam = 0, have_canon = 0; unsigned char _buf[1032]; FILE _f, *f = __fopen_rb_ca("/etc/hosts", &_f, _buf, sizeof _buf); if (!f) switch (errno) { @@ -79,15 +79,20 @@ static int name_from_hosts(struct address buf[static MAXADDRS], char canon[stati case 0: continue; default: - badfam = EAI_NONAME; - continue; + badfam = EAI_NODATA; + break; } + if (have_canon) continue; + /* Extract first name as canonical name */ for (; *p && isspace(*p); p++); for (z=p; *z && !isspace(*z); z++); *z = 0; - if (is_valid_hostname(p)) memcpy(canon, p, z-p+1); + if (is_valid_hostname(p)) { + have_canon = 1; + memcpy(canon, p, z-p+1); + } } __fclose_ca(f); return cnt ? cnt : badfam; @@ -97,45 +102,50 @@ struct dpc_ctx { struct address *addrs; char *canon; int cnt; + int rrtype; }; #define RR_A 1 #define RR_CNAME 5 #define RR_AAAA 28 -static int dns_parse_callback(void *c, int rr, const void *data, int len, const void *packet) +#define ABUF_SIZE 4800 + +static int dns_parse_callback(void *c, int rr, const void *data, int len, const void *packet, int plen) { char tmp[256]; + int family; struct dpc_ctx *ctx = c; - if (ctx->cnt >= MAXADDRS) return -1; + if (rr == RR_CNAME) { + if (__dn_expand(packet, (const unsigned char *)packet + plen, + data, tmp, sizeof tmp) > 0 && is_valid_hostname(tmp)) + strcpy(ctx->canon, tmp); + return 0; + } + if (ctx->cnt >= MAXADDRS) return 0; + if (rr != ctx->rrtype) return 0; switch (rr) { case RR_A: if (len != 4) return -1; - ctx->addrs[ctx->cnt].family = AF_INET; - ctx->addrs[ctx->cnt].scopeid = 0; - memcpy(ctx->addrs[ctx->cnt++].addr, data, 4); + family = AF_INET; break; case RR_AAAA: if (len != 16) return -1; - ctx->addrs[ctx->cnt].family = AF_INET6; - ctx->addrs[ctx->cnt].scopeid = 0; - memcpy(ctx->addrs[ctx->cnt++].addr, data, 16); - break; - case RR_CNAME: - if (__dn_expand(packet, (const unsigned char *)packet + 512, - data, tmp, sizeof tmp) > 0 && is_valid_hostname(tmp)) - strcpy(ctx->canon, tmp); + family = AF_INET6; break; } + ctx->addrs[ctx->cnt].family = family; + ctx->addrs[ctx->cnt].scopeid = 0; + memcpy(ctx->addrs[ctx->cnt++].addr, data, len); return 0; } static int name_from_dns(struct address buf[static MAXADDRS], char canon[static 256], const char *name, int family, const struct resolvconf *conf) { - unsigned char qbuf[2][280], abuf[2][512]; + unsigned char qbuf[2][280], abuf[2][ABUF_SIZE]; const unsigned char *qp[2] = { qbuf[0], qbuf[1] }; unsigned char *ap[2] = { abuf[0], abuf[1] }; - int qlens[2], alens[2]; + int qlens[2], alens[2], qtypes[2]; int i, nq = 0; struct dpc_ctx ctx = { .addrs = buf, .canon = canon }; static const struct { int af; int rr; } afrr[2] = { @@ -148,7 +158,12 @@ static int name_from_dns(struct address buf[static MAXADDRS], char canon[static qlens[nq] = __res_mkquery(0, name, 1, afrr[i].rr, 0, 0, 0, qbuf[nq], sizeof *qbuf); if (qlens[nq] == -1) - return EAI_NONAME; + return 0; + qtypes[nq] = afrr[i].rr; + qbuf[nq][3] = 0; /* don't need AD flag */ + /* Ensure query IDs are distinct. */ + if (nq && qbuf[nq][0] == qbuf[0][0]) + qbuf[nq][0]++; nq++; } } @@ -156,14 +171,20 @@ static int name_from_dns(struct address buf[static MAXADDRS], char canon[static if (__res_msend_rc(nq, qp, qlens, ap, alens, sizeof *abuf, conf) < 0) return EAI_SYSTEM; - for (i=0; i<nq; i++) + for (i=0; i<nq; i++) { + if (alens[i] < 4 || (abuf[i][3] & 15) == 2) return EAI_AGAIN; + if ((abuf[i][3] & 15) == 3) return 0; + if ((abuf[i][3] & 15) != 0) return EAI_FAIL; + } + + for (i=nq-1; i>=0; i--) { + ctx.rrtype = qtypes[i]; + if (alens[i] > sizeof(abuf[i])) alens[i] = sizeof abuf[i]; __dns_parse(abuf[i], alens[i], dns_parse_callback, &ctx); + } if (ctx.cnt) return ctx.cnt; - if (alens[0] < 4 || (abuf[0][3] & 15) == 2) return EAI_AGAIN; - if ((abuf[0][3] & 15) == 0) return EAI_NONAME; - if ((abuf[0][3] & 15) == 3) return 0; - return EAI_FAIL; + return EAI_NODATA; } static int name_from_dns_search(struct address buf[static MAXADDRS], char canon[static 256], const char *name, int family) diff --git a/src/network/netlink.h b/src/network/netlink.h index 38acb178..873fabe2 100644 --- a/src/network/netlink.h +++ b/src/network/netlink.h @@ -86,7 +86,7 @@ struct ifaddrmsg { #define RTA_DATALEN(rta) ((rta)->rta_len-sizeof(struct rtattr)) #define RTA_DATAEND(rta) ((char*)(rta)+(rta)->rta_len) #define RTA_NEXT(rta) (struct rtattr*)((char*)(rta)+NETLINK_ALIGN((rta)->rta_len)) -#define RTA_OK(nlh,end) ((char*)(end)-(char*)(rta) >= sizeof(struct rtattr)) +#define RTA_OK(rta,end) ((char*)(end)-(char*)(rta) >= sizeof(struct rtattr)) #define NLMSG_RTA(nlh,len) ((void*)((char*)(nlh)+sizeof(struct nlmsghdr)+NETLINK_ALIGN(len))) #define NLMSG_RTAOK(rta,nlh) RTA_OK(rta,NLMSG_DATAEND(nlh)) diff --git a/src/network/recvmmsg.c b/src/network/recvmmsg.c index d5dc6b51..2978e2f6 100644 --- a/src/network/recvmmsg.c +++ b/src/network/recvmmsg.c @@ -8,6 +8,8 @@ #define IS32BIT(x) !((x)+0x80000000ULL>>32) #define CLAMP(x) (int)(IS32BIT(x) ? (x) : 0x7fffffffU+((0ULL+(x))>>63)) +hidden void __convert_scm_timestamps(struct msghdr *, socklen_t); + int recvmmsg(int fd, struct mmsghdr *msgvec, unsigned int vlen, unsigned int flags, struct timespec *timeout) { #if LONG_MAX > INT_MAX @@ -19,14 +21,18 @@ int recvmmsg(int fd, struct mmsghdr *msgvec, unsigned int vlen, unsigned int fla #ifdef SYS_recvmmsg_time64 time_t s = timeout ? timeout->tv_sec : 0; long ns = timeout ? timeout->tv_nsec : 0; - int r = -ENOSYS; - if (SYS_recvmmsg == SYS_recvmmsg_time64 || !IS32BIT(s)) - r = __syscall_cp(SYS_recvmmsg_time64, fd, msgvec, vlen, flags, + int r = __syscall_cp(SYS_recvmmsg_time64, fd, msgvec, vlen, flags, timeout ? ((long long[]){s, ns}) : 0); if (SYS_recvmmsg == SYS_recvmmsg_time64 || r!=-ENOSYS) return __syscall_ret(r); - return syscall_cp(SYS_recvmmsg, fd, msgvec, vlen, flags, + if (vlen > IOV_MAX) vlen = IOV_MAX; + socklen_t csize[vlen]; + for (int i=0; i<vlen; i++) csize[i] = msgvec[i].msg_hdr.msg_controllen; + r = __syscall_cp(SYS_recvmmsg, fd, msgvec, vlen, flags, timeout ? ((long[]){CLAMP(s), ns}) : 0); + for (int i=0; i<r; i++) + __convert_scm_timestamps(&msgvec[i].msg_hdr, csize[i]); + return __syscall_ret(r); #else return syscall_cp(SYS_recvmmsg, fd, msgvec, vlen, flags, timeout); #endif diff --git a/src/network/recvmsg.c b/src/network/recvmsg.c index 4ca7da8b..03641625 100644 --- a/src/network/recvmsg.c +++ b/src/network/recvmsg.c @@ -1,10 +1,56 @@ #include <sys/socket.h> #include <limits.h> +#include <time.h> +#include <sys/time.h> +#include <string.h> #include "syscall.h" +hidden void __convert_scm_timestamps(struct msghdr *, socklen_t); + +void __convert_scm_timestamps(struct msghdr *msg, socklen_t csize) +{ + if (SCM_TIMESTAMP == SCM_TIMESTAMP_OLD) return; + if (!msg->msg_control || !msg->msg_controllen) return; + + struct cmsghdr *cmsg, *last=0; + long tmp; + long long tvts[2]; + int type = 0; + + for (cmsg=CMSG_FIRSTHDR(msg); cmsg; cmsg=CMSG_NXTHDR(msg, cmsg)) { + if (cmsg->cmsg_level==SOL_SOCKET) switch (cmsg->cmsg_type) { + case SCM_TIMESTAMP_OLD: + if (type) break; + type = SCM_TIMESTAMP; + goto common; + case SCM_TIMESTAMPNS_OLD: + type = SCM_TIMESTAMPNS; + common: + memcpy(&tmp, CMSG_DATA(cmsg), sizeof tmp); + tvts[0] = tmp; + memcpy(&tmp, CMSG_DATA(cmsg) + sizeof tmp, sizeof tmp); + tvts[1] = tmp; + break; + } + last = cmsg; + } + if (!last || !type) return; + if (CMSG_SPACE(sizeof tvts) > csize-msg->msg_controllen) { + msg->msg_flags |= MSG_CTRUNC; + return; + } + msg->msg_controllen += CMSG_SPACE(sizeof tvts); + cmsg = CMSG_NXTHDR(msg, last); + cmsg->cmsg_level = SOL_SOCKET; + cmsg->cmsg_type = type; + cmsg->cmsg_len = CMSG_LEN(sizeof tvts); + memcpy(CMSG_DATA(cmsg), &tvts, sizeof tvts); +} + ssize_t recvmsg(int fd, struct msghdr *msg, int flags) { ssize_t r; + socklen_t orig_controllen = msg->msg_controllen; #if LONG_MAX > INT_MAX struct msghdr h, *orig = msg; if (msg) { @@ -14,6 +60,7 @@ ssize_t recvmsg(int fd, struct msghdr *msg, int flags) } #endif r = socketcall_cp(recvmsg, fd, msg, flags, 0, 0, 0); + if (r >= 0) __convert_scm_timestamps(msg, orig_controllen); #if LONG_MAX > INT_MAX if (orig) *orig = h; #endif diff --git a/src/network/res_mkquery.c b/src/network/res_mkquery.c index 6fa04a5c..614bf786 100644 --- a/src/network/res_mkquery.c +++ b/src/network/res_mkquery.c @@ -13,6 +13,7 @@ int __res_mkquery(int op, const char *dname, int class, int type, int n; if (l && dname[l-1]=='.') l--; + if (l && dname[l-1]=='.') return -1; n = 17+l+!!l; if (l>253 || buflen<n || op>15u || class>255u || type>255u) return -1; @@ -20,6 +21,7 @@ int __res_mkquery(int op, const char *dname, int class, int type, /* Construct query template - ID will be filled later */ memset(q, 0, n); q[2] = op*8 + 1; + q[3] = 32; /* AD */ q[5] = 1; memcpy((char *)q+13, dname, l); for (i=13; q[i]; i=j+1) { diff --git a/src/network/res_msend.c b/src/network/res_msend.c index 3e018009..86c2fcf4 100644 --- a/src/network/res_msend.c +++ b/src/network/res_msend.c @@ -1,5 +1,6 @@ #include <sys/socket.h> #include <netinet/in.h> +#include <netinet/tcp.h> #include <netdb.h> #include <arpa/inet.h> #include <stdint.h> @@ -16,17 +17,65 @@ static void cleanup(void *p) { - __syscall(SYS_close, (intptr_t)p); + struct pollfd *pfd = p; + for (int i=0; pfd[i].fd >= -1; i++) + if (pfd[i].fd >= 0) __syscall(SYS_close, pfd[i].fd); } static unsigned long mtime() { struct timespec ts; - clock_gettime(CLOCK_REALTIME, &ts); + if (clock_gettime(CLOCK_MONOTONIC, &ts) < 0 && errno == ENOSYS) + clock_gettime(CLOCK_REALTIME, &ts); return (unsigned long)ts.tv_sec * 1000 + ts.tv_nsec / 1000000; } +static int start_tcp(struct pollfd *pfd, int family, const void *sa, socklen_t sl, const unsigned char *q, int ql) +{ + struct msghdr mh = { + .msg_name = (void *)sa, + .msg_namelen = sl, + .msg_iovlen = 2, + .msg_iov = (struct iovec [2]){ + { .iov_base = (uint8_t[]){ ql>>8, ql }, .iov_len = 2 }, + { .iov_base = (void *)q, .iov_len = ql } } + }; + int r; + int fd = socket(family, SOCK_STREAM|SOCK_CLOEXEC|SOCK_NONBLOCK, 0); + pfd->fd = fd; + pfd->events = POLLOUT; + if (!setsockopt(fd, IPPROTO_TCP, TCP_FASTOPEN_CONNECT, + &(int){1}, sizeof(int))) { + r = sendmsg(fd, &mh, MSG_FASTOPEN|MSG_NOSIGNAL); + if (r == ql+2) pfd->events = POLLIN; + if (r >= 0) return r; + if (errno == EINPROGRESS) return 0; + } + r = connect(fd, sa, sl); + if (!r || errno == EINPROGRESS) return 0; + close(fd); + pfd->fd = -1; + return -1; +} + +static void step_mh(struct msghdr *mh, size_t n) +{ + /* Adjust iovec in msghdr to skip first n bytes. */ + while (mh->msg_iovlen && n >= mh->msg_iov->iov_len) { + n -= mh->msg_iov->iov_len; + mh->msg_iov++; + mh->msg_iovlen--; + } + if (!mh->msg_iovlen) return; + mh->msg_iov->iov_base = (char *)mh->msg_iov->iov_base + n; + mh->msg_iov->iov_len -= n; +} + +/* Internal contract for __res_msend[_rc]: asize must be >=512, nqueries + * must be sufficiently small to be safe as VLA size. In practice it's + * either 1 or 2, anyway. */ + int __res_msend_rc(int nqueries, const unsigned char *const *queries, const int *qlens, unsigned char *const *answers, int *alens, int asize, const struct resolvconf *conf) @@ -44,7 +93,10 @@ int __res_msend_rc(int nqueries, const unsigned char *const *queries, int next; int i, j; int cs; - struct pollfd pfd; + struct pollfd pfd[nqueries+2]; + int qpos[nqueries], apos[nqueries]; + unsigned char alen_buf[nqueries][2]; + int r; unsigned long t0, t1, t2; pthread_setcancelstate(PTHREAD_CANCEL_DISABLE, &cs); @@ -68,29 +120,22 @@ int __res_msend_rc(int nqueries, const unsigned char *const *queries, } /* Get local address and open/bind a socket */ - sa.sin.sin_family = family; fd = socket(family, SOCK_DGRAM|SOCK_CLOEXEC|SOCK_NONBLOCK, 0); /* Handle case where system lacks IPv6 support */ if (fd < 0 && family == AF_INET6 && errno == EAFNOSUPPORT) { + for (i=0; i<nns && conf->ns[nns].family == AF_INET6; i++); + if (i==nns) { + pthread_setcancelstate(cs, 0); + return -1; + } fd = socket(AF_INET, SOCK_DGRAM|SOCK_CLOEXEC|SOCK_NONBLOCK, 0); family = AF_INET; + sl = sizeof sa.sin; } - if (fd < 0 || bind(fd, (void *)&sa, sl) < 0) { - if (fd >= 0) close(fd); - pthread_setcancelstate(cs, 0); - return -1; - } - - /* Past this point, there are no errors. Each individual query will - * yield either no reply (indicated by zero length) or an answer - * packet which is up to the caller to interpret. */ - - pthread_cleanup_push(cleanup, (void *)(intptr_t)fd); - pthread_setcancelstate(cs, 0); /* Convert any IPv4 addresses in a mixed environment to v4-mapped */ - if (family == AF_INET6) { + if (fd >= 0 && family == AF_INET6) { setsockopt(fd, IPPROTO_IPV6, IPV6_V6ONLY, &(int){0}, sizeof 0); for (i=0; i<nns; i++) { if (ns[i].sin.sin_family != AF_INET) continue; @@ -104,16 +149,38 @@ int __res_msend_rc(int nqueries, const unsigned char *const *queries, } } + sa.sin.sin_family = family; + if (fd < 0 || bind(fd, (void *)&sa, sl) < 0) { + if (fd >= 0) close(fd); + pthread_setcancelstate(cs, 0); + return -1; + } + + /* Past this point, there are no errors. Each individual query will + * yield either no reply (indicated by zero length) or an answer + * packet which is up to the caller to interpret. */ + + for (i=0; i<nqueries; i++) pfd[i].fd = -1; + pfd[nqueries].fd = fd; + pfd[nqueries].events = POLLIN; + pfd[nqueries+1].fd = -2; + + pthread_cleanup_push(cleanup, pfd); + pthread_setcancelstate(cs, 0); + memset(alens, 0, sizeof *alens * nqueries); - pfd.fd = fd; - pfd.events = POLLIN; retry_interval = timeout / attempts; next = 0; t0 = t2 = mtime(); t1 = t2 - retry_interval; for (; t2-t0 < timeout; t2=mtime()) { + /* This is the loop exit condition: that all queries + * have an accepted answer. */ + for (i=0; i<nqueries && alens[i]>0; i++); + if (i==nqueries) break; + if (t2-t1 >= retry_interval) { /* Query all configured namservers in parallel */ for (i=0; i<nqueries; i++) @@ -127,10 +194,20 @@ int __res_msend_rc(int nqueries, const unsigned char *const *queries, } /* Wait for a response, or until time to retry */ - if (poll(&pfd, 1, t1+retry_interval-t2) <= 0) continue; + if (poll(pfd, nqueries+1, t1+retry_interval-t2) <= 0) continue; - while ((rlen = recvfrom(fd, answers[next], asize, 0, - (void *)&sa, (socklen_t[1]){sl})) >= 0) { + while (next < nqueries) { + struct msghdr mh = { + .msg_name = (void *)&sa, + .msg_namelen = sl, + .msg_iovlen = 1, + .msg_iov = (struct iovec []){ + { .iov_base = (void *)answers[next], + .iov_len = asize } + } + }; + rlen = recvmsg(fd, &mh, 0); + if (rlen < 0) break; /* Ignore non-identifiable packets */ if (rlen < 4) continue; @@ -170,12 +247,72 @@ int __res_msend_rc(int nqueries, const unsigned char *const *queries, else memcpy(answers[i], answers[next], rlen); - if (next == nqueries) goto out; + /* Ignore further UDP if all slots full or TCP-mode */ + if (next == nqueries) pfd[nqueries].events = 0; + + /* If answer is truncated (TC bit), fallback to TCP */ + if ((answers[i][2] & 2) || (mh.msg_flags & MSG_TRUNC)) { + alens[i] = -1; + pthread_setcancelstate(PTHREAD_CANCEL_DISABLE, 0); + r = start_tcp(pfd+i, family, ns+j, sl, queries[i], qlens[i]); + pthread_setcancelstate(cs, 0); + if (r >= 0) { + qpos[i] = r; + apos[i] = 0; + } + continue; + } + } + + for (i=0; i<nqueries; i++) if (pfd[i].revents & POLLOUT) { + struct msghdr mh = { + .msg_iovlen = 2, + .msg_iov = (struct iovec [2]){ + { .iov_base = (uint8_t[]){ qlens[i]>>8, qlens[i] }, .iov_len = 2 }, + { .iov_base = (void *)queries[i], .iov_len = qlens[i] } } + }; + step_mh(&mh, qpos[i]); + r = sendmsg(pfd[i].fd, &mh, MSG_NOSIGNAL); + if (r < 0) goto out; + qpos[i] += r; + if (qpos[i] == qlens[i]+2) + pfd[i].events = POLLIN; + } + + for (i=0; i<nqueries; i++) if (pfd[i].revents & POLLIN) { + struct msghdr mh = { + .msg_iovlen = 2, + .msg_iov = (struct iovec [2]){ + { .iov_base = alen_buf[i], .iov_len = 2 }, + { .iov_base = answers[i], .iov_len = asize } } + }; + step_mh(&mh, apos[i]); + r = recvmsg(pfd[i].fd, &mh, 0); + if (r <= 0) goto out; + apos[i] += r; + if (apos[i] < 2) continue; + int alen = alen_buf[i][0]*256 + alen_buf[i][1]; + if (alen < 13) goto out; + if (apos[i] < alen+2 && apos[i] < asize+2) + continue; + int rcode = answers[i][3] & 15; + if (rcode != 0 && rcode != 3) + goto out; + + /* Storing the length here commits the accepted answer. + * Immediately close TCP socket so as not to consume + * resources we no longer need. */ + alens[i] = alen; + __syscall(SYS_close, pfd[i].fd); + pfd[i].fd = -1; } } out: pthread_cleanup_pop(1); + /* Disregard any incomplete TCP results */ + for (i=0; i<nqueries; i++) if (alens[i]<0) alens[i] = 0; + return 0; } diff --git a/src/network/res_query.c b/src/network/res_query.c index 2f4da2e2..506dc231 100644 --- a/src/network/res_query.c +++ b/src/network/res_query.c @@ -1,3 +1,4 @@ +#define _BSD_SOURCE #include <resolv.h> #include <netdb.h> @@ -6,7 +7,20 @@ int res_query(const char *name, int class, int type, unsigned char *dest, int le unsigned char q[280]; int ql = __res_mkquery(0, name, class, type, 0, 0, 0, q, sizeof q); if (ql < 0) return ql; - return __res_send(q, ql, dest, len); + int r = __res_send(q, ql, dest, len); + if (r<12) { + h_errno = TRY_AGAIN; + return -1; + } + if ((dest[3] & 15) == 3) { + h_errno = HOST_NOT_FOUND; + return -1; + } + if ((dest[3] & 15) == 0 && !dest[6] && !dest[7]) { + h_errno = NO_DATA; + return -1; + } + return r; } weak_alias(res_query, res_search); diff --git a/src/network/res_send.c b/src/network/res_send.c index b9cea0bf..9593164d 100644 --- a/src/network/res_send.c +++ b/src/network/res_send.c @@ -1,9 +1,17 @@ #include <resolv.h> +#include <string.h> int __res_send(const unsigned char *msg, int msglen, unsigned char *answer, int anslen) { - int r = __res_msend(1, &msg, &msglen, &answer, &anslen, anslen); - return r<0 ? r : anslen; + int r; + if (anslen < 512) { + unsigned char buf[512]; + r = __res_send(msg, msglen, buf, sizeof buf); + if (r >= 0) memcpy(answer, buf, r < anslen ? r : anslen); + return r; + } + r = __res_msend(1, &msg, &msglen, &answer, &anslen, anslen); + return r<0 || !anslen ? -1 : anslen; } weak_alias(__res_send, res_send); diff --git a/src/network/sendmsg.c b/src/network/sendmsg.c index 80cc5f41..acdfdf29 100644 --- a/src/network/sendmsg.c +++ b/src/network/sendmsg.c @@ -8,13 +8,16 @@ ssize_t sendmsg(int fd, const struct msghdr *msg, int flags) { #if LONG_MAX > INT_MAX struct msghdr h; - struct cmsghdr chbuf[1024/sizeof(struct cmsghdr)+1], *c; + /* Kernels before 2.6.38 set SCM_MAX_FD to 255, allocate enough + * space to support an SCM_RIGHTS ancillary message with 255 fds. + * Kernels since 2.6.38 set SCM_MAX_FD to 253. */ + struct cmsghdr chbuf[CMSG_SPACE(255*sizeof(int))/sizeof(struct cmsghdr)+1], *c; if (msg) { h = *msg; h.__pad1 = h.__pad2 = 0; msg = &h; if (h.msg_controllen) { - if (h.msg_controllen > 1024) { + if (h.msg_controllen > sizeof chbuf) { errno = ENOMEM; return -1; } diff --git a/src/network/setsockopt.c b/src/network/setsockopt.c index 2c188a96..612a1947 100644 --- a/src/network/setsockopt.c +++ b/src/network/setsockopt.c @@ -31,6 +31,15 @@ int setsockopt(int fd, int level, int optname, const void *optval, socklen_t opt r = __socketcall(setsockopt, fd, level, optname, ((long[]){s, CLAMP(us)}), 2*sizeof(long), 0); + break; + case SO_TIMESTAMP: + case SO_TIMESTAMPNS: + if (SO_TIMESTAMP == SO_TIMESTAMP_OLD) break; + if (optname==SO_TIMESTAMP) optname=SO_TIMESTAMP_OLD; + if (optname==SO_TIMESTAMPNS) optname=SO_TIMESTAMPNS_OLD; + r = __socketcall(setsockopt, fd, level, + optname, optval, optlen, 0); + break; } } return __syscall_ret(r); diff --git a/src/network/socket.c b/src/network/socket.c index a2e92d90..afa1a7f3 100644 --- a/src/network/socket.c +++ b/src/network/socket.c @@ -5,17 +5,17 @@ int socket(int domain, int type, int protocol) { - int s = socketcall(socket, domain, type, protocol, 0, 0, 0); - if (s<0 && (errno==EINVAL || errno==EPROTONOSUPPORT) + int s = __socketcall(socket, domain, type, protocol, 0, 0, 0); + if ((s==-EINVAL || s==-EPROTONOSUPPORT) && (type&(SOCK_CLOEXEC|SOCK_NONBLOCK))) { - s = socketcall(socket, domain, + s = __socketcall(socket, domain, type & ~(SOCK_CLOEXEC|SOCK_NONBLOCK), protocol, 0, 0, 0); - if (s < 0) return s; + if (s < 0) return __syscall_ret(s); if (type & SOCK_CLOEXEC) __syscall(SYS_fcntl, s, F_SETFD, FD_CLOEXEC); if (type & SOCK_NONBLOCK) __syscall(SYS_fcntl, s, F_SETFL, O_NONBLOCK); } - return s; + return __syscall_ret(s); } diff --git a/src/passwd/getgrouplist.c b/src/passwd/getgrouplist.c index 43e51824..301824ce 100644 --- a/src/passwd/getgrouplist.c +++ b/src/passwd/getgrouplist.c @@ -31,7 +31,8 @@ int getgrouplist(const char *user, gid_t gid, gid_t *groups, int *ngroups) if (resp[INITGRFOUND]) { nscdbuf = calloc(resp[INITGRNGRPS], sizeof(uint32_t)); if (!nscdbuf) goto cleanup; - if (!fread(nscdbuf, sizeof(*nscdbuf)*resp[INITGRNGRPS], 1, f)) { + size_t nbytes = sizeof(*nscdbuf)*resp[INITGRNGRPS]; + if (nbytes && !fread(nscdbuf, nbytes, 1, f)) { if (!ferror(f)) errno = EIO; goto cleanup; } diff --git a/src/passwd/nscd_query.c b/src/passwd/nscd_query.c index d38e371b..dc3406b8 100644 --- a/src/passwd/nscd_query.c +++ b/src/passwd/nscd_query.c @@ -40,7 +40,15 @@ retry: buf[0] = NSCDVERSION; fd = socket(PF_UNIX, SOCK_STREAM | SOCK_CLOEXEC, 0); - if (fd < 0) return NULL; + if (fd < 0) { + if (errno == EAFNOSUPPORT) { + f = fopen("/dev/null", "re"); + if (f) + errno = errno_save; + return f; + } + return 0; + } if(!(f = fdopen(fd, "r"))) { close(fd); diff --git a/src/prng/random.c b/src/prng/random.c index 633a17f6..d3780fa7 100644 --- a/src/prng/random.c +++ b/src/prng/random.c @@ -1,6 +1,7 @@ #include <stdlib.h> #include <stdint.h> #include "lock.h" +#include "fork_impl.h" /* this code uses the same lagged fibonacci generator as the @@ -23,6 +24,7 @@ static int i = 3; static int j = 0; static uint32_t *x = init+1; static volatile int lock[1]; +volatile int *const __random_lockptr = lock; static uint32_t lcg31(uint32_t x) { return (1103515245*x + 12345) & 0x7fffffff; diff --git a/src/process/_Fork.c b/src/process/_Fork.c new file mode 100644 index 00000000..9c07792d --- /dev/null +++ b/src/process/_Fork.c @@ -0,0 +1,43 @@ +#include <unistd.h> +#include <signal.h> +#include "syscall.h" +#include "libc.h" +#include "lock.h" +#include "pthread_impl.h" +#include "aio_impl.h" +#include "fork_impl.h" + +static void dummy(int x) { } +weak_alias(dummy, __aio_atfork); + +void __post_Fork(int ret) +{ + if (!ret) { + pthread_t self = __pthread_self(); + self->tid = __syscall(SYS_set_tid_address, &__thread_list_lock); + self->robust_list.off = 0; + self->robust_list.pending = 0; + self->next = self->prev = self; + __thread_list_lock = 0; + libc.threads_minus_1 = 0; + if (libc.need_locks) libc.need_locks = -1; + } + UNLOCK(__abort_lock); + if (!ret) __aio_atfork(1); +} + +pid_t _Fork(void) +{ + pid_t ret; + sigset_t set; + __block_all_sigs(&set); + LOCK(__abort_lock); +#ifdef SYS_fork + ret = __syscall(SYS_fork); +#else + ret = __syscall(SYS_clone, SIGCHLD, 0); +#endif + __post_Fork(ret); + __restore_sigs(&set); + return __syscall_ret(ret); +} diff --git a/src/process/aarch64/vfork.s b/src/process/aarch64/vfork.s new file mode 100644 index 00000000..429bec8c --- /dev/null +++ b/src/process/aarch64/vfork.s @@ -0,0 +1,9 @@ +.global vfork +.type vfork,%function +vfork: + mov x8, 220 // SYS_clone + mov x0, 0x4111 // SIGCHLD | CLONE_VM | CLONE_VFORK + mov x1, 0 + svc 0 + .hidden __syscall_ret + b __syscall_ret diff --git a/src/process/fdop.h b/src/process/fdop.h index 5adf1443..7cf733b2 100644 --- a/src/process/fdop.h +++ b/src/process/fdop.h @@ -10,3 +10,8 @@ struct fdop { mode_t mode; char path[]; }; + +#define malloc __libc_malloc +#define calloc __libc_calloc +#define realloc undef +#define free __libc_free diff --git a/src/process/fork.c b/src/process/fork.c index fb42478a..56f19313 100644 --- a/src/process/fork.c +++ b/src/process/fork.c @@ -1,37 +1,90 @@ #include <unistd.h> -#include <string.h> -#include <signal.h> -#include "syscall.h" +#include <errno.h> #include "libc.h" +#include "lock.h" #include "pthread_impl.h" +#include "fork_impl.h" -static void dummy(int x) -{ -} +static volatile int *const dummy_lockptr = 0; + +weak_alias(dummy_lockptr, __at_quick_exit_lockptr); +weak_alias(dummy_lockptr, __atexit_lockptr); +weak_alias(dummy_lockptr, __gettext_lockptr); +weak_alias(dummy_lockptr, __locale_lockptr); +weak_alias(dummy_lockptr, __random_lockptr); +weak_alias(dummy_lockptr, __sem_open_lockptr); +weak_alias(dummy_lockptr, __stdio_ofl_lockptr); +weak_alias(dummy_lockptr, __syslog_lockptr); +weak_alias(dummy_lockptr, __timezone_lockptr); +weak_alias(dummy_lockptr, __bump_lockptr); + +weak_alias(dummy_lockptr, __vmlock_lockptr); +static volatile int *const *const atfork_locks[] = { + &__at_quick_exit_lockptr, + &__atexit_lockptr, + &__gettext_lockptr, + &__locale_lockptr, + &__random_lockptr, + &__sem_open_lockptr, + &__stdio_ofl_lockptr, + &__syslog_lockptr, + &__timezone_lockptr, + &__bump_lockptr, +}; + +static void dummy(int x) { } weak_alias(dummy, __fork_handler); +weak_alias(dummy, __malloc_atfork); +weak_alias(dummy, __aio_atfork); +weak_alias(dummy, __pthread_key_atfork); +weak_alias(dummy, __ldso_atfork); + +static void dummy_0(void) { } +weak_alias(dummy_0, __tl_lock); +weak_alias(dummy_0, __tl_unlock); pid_t fork(void) { - pid_t ret; sigset_t set; __fork_handler(-1); - __block_all_sigs(&set); -#ifdef SYS_fork - ret = __syscall(SYS_fork); -#else - ret = __syscall(SYS_clone, SIGCHLD, 0); -#endif - if (!ret) { - pthread_t self = __pthread_self(); - self->tid = __syscall(SYS_gettid); - self->robust_list.off = 0; - self->robust_list.pending = 0; - self->next = self->prev = self; - __thread_list_lock = 0; - libc.threads_minus_1 = 0; + __block_app_sigs(&set); + int need_locks = libc.need_locks > 0; + if (need_locks) { + __ldso_atfork(-1); + __pthread_key_atfork(-1); + __aio_atfork(-1); + __inhibit_ptc(); + for (int i=0; i<sizeof atfork_locks/sizeof *atfork_locks; i++) + if (*atfork_locks[i]) LOCK(*atfork_locks[i]); + __malloc_atfork(-1); + __tl_lock(); + } + pthread_t self=__pthread_self(), next=self->next; + pid_t ret = _Fork(); + int errno_save = errno; + if (need_locks) { + if (!ret) { + for (pthread_t td=next; td!=self; td=td->next) + td->tid = -1; + if (__vmlock_lockptr) { + __vmlock_lockptr[0] = 0; + __vmlock_lockptr[1] = 0; + } + } + __tl_unlock(); + __malloc_atfork(!ret); + for (int i=0; i<sizeof atfork_locks/sizeof *atfork_locks; i++) + if (*atfork_locks[i]) + if (ret) UNLOCK(*atfork_locks[i]); + else **atfork_locks[i] = 0; + __release_ptc(); + if (ret) __aio_atfork(0); + __pthread_key_atfork(!ret); + __ldso_atfork(!ret); } __restore_sigs(&set); __fork_handler(!ret); - return __syscall_ret(ret); + if (ret<0) errno = errno_save; + return ret; } diff --git a/src/process/posix_spawn.c b/src/process/posix_spawn.c index 29652197..8294598b 100644 --- a/src/process/posix_spawn.c +++ b/src/process/posix_spawn.c @@ -4,8 +4,10 @@ #include <unistd.h> #include <signal.h> #include <fcntl.h> +#include <errno.h> #include <sys/wait.h> #include "syscall.h" +#include "lock.h" #include "pthread_impl.h" #include "fdop.h" @@ -155,7 +157,11 @@ static int child(void *args_vp) fail: /* Since sizeof errno < PIPE_BUF, the write is atomic. */ ret = -ret; - if (ret) while (__syscall(SYS_write, p, &ret, sizeof ret) < 0); + if (ret) { + int r; + do r = __syscall(SYS_write, p, &ret, sizeof ret); + while (r<0 && r!=-EPIPE); + } _exit(127); } @@ -170,9 +176,6 @@ int posix_spawn(pid_t *restrict res, const char *restrict path, int ec=0, cs; struct args args; - if (pipe2(args.p, O_CLOEXEC)) - return errno; - pthread_setcancelstate(PTHREAD_CANCEL_DISABLE, &cs); args.path = path; @@ -182,9 +185,20 @@ int posix_spawn(pid_t *restrict res, const char *restrict path, args.envp = envp; pthread_sigmask(SIG_BLOCK, SIGALL_SET, &args.oldmask); + /* The lock guards both against seeing a SIGABRT disposition change + * by abort and against leaking the pipe fd to fork-without-exec. */ + LOCK(__abort_lock); + + if (pipe2(args.p, O_CLOEXEC)) { + UNLOCK(__abort_lock); + ec = errno; + goto fail; + } + pid = __clone(child, stack+sizeof stack, CLONE_VM|CLONE_VFORK|SIGCHLD, &args); close(args.p[1]); + UNLOCK(__abort_lock); if (pid > 0) { if (read(args.p[0], &ec, sizeof ec) != sizeof ec) ec = 0; @@ -197,6 +211,7 @@ int posix_spawn(pid_t *restrict res, const char *restrict path, if (!ec && res) *res = pid; +fail: pthread_sigmask(SIG_SETMASK, &args.oldmask, 0); pthread_setcancelstate(cs, 0); diff --git a/src/process/posix_spawn_file_actions_addclose.c b/src/process/posix_spawn_file_actions_addclose.c index cdda5979..0c2ef8fa 100644 --- a/src/process/posix_spawn_file_actions_addclose.c +++ b/src/process/posix_spawn_file_actions_addclose.c @@ -5,6 +5,7 @@ int posix_spawn_file_actions_addclose(posix_spawn_file_actions_t *fa, int fd) { + if (fd < 0) return EBADF; struct fdop *op = malloc(sizeof *op); if (!op) return ENOMEM; op->cmd = FDOP_CLOSE; diff --git a/src/process/posix_spawn_file_actions_adddup2.c b/src/process/posix_spawn_file_actions_adddup2.c index 0367498f..addca4d4 100644 --- a/src/process/posix_spawn_file_actions_adddup2.c +++ b/src/process/posix_spawn_file_actions_adddup2.c @@ -5,6 +5,7 @@ int posix_spawn_file_actions_adddup2(posix_spawn_file_actions_t *fa, int srcfd, int fd) { + if (srcfd < 0 || fd < 0) return EBADF; struct fdop *op = malloc(sizeof *op); if (!op) return ENOMEM; op->cmd = FDOP_DUP2; diff --git a/src/process/posix_spawn_file_actions_addfchdir.c b/src/process/posix_spawn_file_actions_addfchdir.c index 436c683d..e89ede8c 100644 --- a/src/process/posix_spawn_file_actions_addfchdir.c +++ b/src/process/posix_spawn_file_actions_addfchdir.c @@ -6,6 +6,7 @@ int posix_spawn_file_actions_addfchdir_np(posix_spawn_file_actions_t *fa, int fd) { + if (fd < 0) return EBADF; struct fdop *op = malloc(sizeof *op); if (!op) return ENOMEM; op->cmd = FDOP_FCHDIR; diff --git a/src/process/posix_spawn_file_actions_addopen.c b/src/process/posix_spawn_file_actions_addopen.c index 368922c7..82bbcec9 100644 --- a/src/process/posix_spawn_file_actions_addopen.c +++ b/src/process/posix_spawn_file_actions_addopen.c @@ -6,6 +6,7 @@ int posix_spawn_file_actions_addopen(posix_spawn_file_actions_t *restrict fa, int fd, const char *restrict path, int flags, mode_t mode) { + if (fd < 0) return EBADF; struct fdop *op = malloc(sizeof *op + strlen(path) + 1); if (!op) return ENOMEM; op->cmd = FDOP_OPEN; diff --git a/src/process/riscv64/vfork.s b/src/process/riscv64/vfork.s new file mode 100644 index 00000000..c93dca23 --- /dev/null +++ b/src/process/riscv64/vfork.s @@ -0,0 +1,12 @@ +.global vfork +.type vfork,@function +vfork: + /* riscv does not have SYS_vfork, so we must use clone instead */ + /* note: riscv's clone = clone(flags, sp, ptidptr, tls, ctidptr) */ + li a7, 220 + li a0, 0x100 | 0x4000 | 17 /* flags = CLONE_VM | CLONE_VFORK | SIGCHLD */ + mv a1, sp + /* the other arguments are ignoreable */ + ecall + .hidden __syscall_ret + j __syscall_ret diff --git a/src/process/waitpid.c b/src/process/waitpid.c index 1b65bf05..80231862 100644 --- a/src/process/waitpid.c +++ b/src/process/waitpid.c @@ -3,5 +3,5 @@ pid_t waitpid(pid_t pid, int *status, int options) { - return syscall_cp(SYS_wait4, pid, status, options, 0); + return sys_wait4_cp(pid, status, options, 0); } diff --git a/src/regex/glob.c b/src/regex/glob.c index 9de080ed..87bae084 100644 --- a/src/regex/glob.c +++ b/src/regex/glob.c @@ -265,7 +265,7 @@ int glob(const char *restrict pat, int flags, int (*errfunc)(const char *path, i if (append(&tail, pat, strlen(pat), 0)) return GLOB_NOSPACE; cnt++; - } else + } else if (!error) return GLOB_NOMATCH; } @@ -306,6 +306,3 @@ void globfree(glob_t *g) g->gl_pathc = 0; g->gl_pathv = NULL; } - -weak_alias(glob, glob64); -weak_alias(globfree, globfree64); diff --git a/src/search/hsearch.c b/src/search/hsearch.c index b3ac8796..2634a67f 100644 --- a/src/search/hsearch.c +++ b/src/search/hsearch.c @@ -41,9 +41,9 @@ static int resize(size_t nel, struct hsearch_data *htab) { size_t newsize; size_t i, j; + size_t oldsize = htab->__tab->mask + 1; ENTRY *e, *newe; ENTRY *oldtab = htab->__tab->entries; - ENTRY *oldend = htab->__tab->entries + htab->__tab->mask + 1; if (nel > MAXSIZE) nel = MAXSIZE; @@ -56,7 +56,7 @@ static int resize(size_t nel, struct hsearch_data *htab) htab->__tab->mask = newsize - 1; if (!oldtab) return 1; - for (e = oldtab; e < oldend; e++) + for (e = oldtab; e < oldtab + oldsize; e++) if (e->key) { for (i=keyhash(e->key),j=1; ; i+=j++) { newe = htab->__tab->entries + (i & htab->__tab->mask); diff --git a/src/select/poll.c b/src/select/poll.c index c84c8a99..7883dfab 100644 --- a/src/select/poll.c +++ b/src/select/poll.c @@ -8,8 +8,13 @@ int poll(struct pollfd *fds, nfds_t n, int timeout) #ifdef SYS_poll return syscall_cp(SYS_poll, fds, n, timeout); #else +#if SYS_ppoll_time64 == SYS_ppoll + typedef long long ppoll_ts_t[2]; +#else + typedef long ppoll_ts_t[2]; +#endif return syscall_cp(SYS_ppoll, fds, n, timeout>=0 ? - &((struct timespec){ .tv_sec = timeout/1000, - .tv_nsec = timeout%1000*1000000 }) : 0, 0, _NSIG/8); + ((ppoll_ts_t){ timeout/1000, timeout%1000*1000000 }) : 0, + 0, _NSIG/8); #endif } diff --git a/src/linux/ppoll.c b/src/select/ppoll.c index e614600a..9a0bf929 100644 --- a/src/linux/ppoll.c +++ b/src/select/ppoll.c @@ -1,4 +1,4 @@ -#define _GNU_SOURCE +#define _BSD_SOURCE #include <poll.h> #include <signal.h> #include <errno.h> diff --git a/src/select/select.c b/src/select/select.c index 8a786884..f1d72863 100644 --- a/src/select/select.c +++ b/src/select/select.c @@ -33,6 +33,7 @@ int select(int n, fd_set *restrict rfds, fd_set *restrict wfds, fd_set *restrict ((syscall_arg_t[]){ 0, _NSIG/8 })); if (SYS_pselect6 == SYS_pselect6_time64 || r!=-ENOSYS) return __syscall_ret(r); + s = CLAMP(s); #endif #ifdef SYS_select return syscall_cp(SYS_select, n, rfds, wfds, efds, diff --git a/src/setjmp/aarch64/longjmp.s b/src/setjmp/aarch64/longjmp.s index 7c4655fa..0af9c50e 100644 --- a/src/setjmp/aarch64/longjmp.s +++ b/src/setjmp/aarch64/longjmp.s @@ -18,7 +18,6 @@ longjmp: ldp d12, d13, [x0,#144] ldp d14, d15, [x0,#160] - mov x0, x1 - cbnz x1, 1f - mov x0, #1 -1: br x30 + cmp w1, 0 + csinc w0, w1, wzr, ne + br x30 diff --git a/src/setjmp/i386/longjmp.s b/src/setjmp/i386/longjmp.s index 772d28dd..8188f06b 100644 --- a/src/setjmp/i386/longjmp.s +++ b/src/setjmp/i386/longjmp.s @@ -6,15 +6,11 @@ _longjmp: longjmp: mov 4(%esp),%edx mov 8(%esp),%eax - test %eax,%eax - jnz 1f - inc %eax -1: + cmp $1,%eax + adc $0, %al mov (%edx),%ebx mov 4(%edx),%esi mov 8(%edx),%edi mov 12(%edx),%ebp - mov 16(%edx),%ecx - mov %ecx,%esp - mov 20(%edx),%ecx - jmp *%ecx + mov 16(%edx),%esp + jmp *20(%edx) diff --git a/src/setjmp/loongarch64/longjmp.S b/src/setjmp/loongarch64/longjmp.S new file mode 100644 index 00000000..896d2e26 --- /dev/null +++ b/src/setjmp/loongarch64/longjmp.S @@ -0,0 +1,32 @@ +.global _longjmp +.global longjmp +.type _longjmp,@function +.type longjmp,@function +_longjmp: +longjmp: + ld.d $ra, $a0, 0 + ld.d $sp, $a0, 8 + ld.d $r21,$a0, 16 + ld.d $fp, $a0, 24 + ld.d $s0, $a0, 32 + ld.d $s1, $a0, 40 + ld.d $s2, $a0, 48 + ld.d $s3, $a0, 56 + ld.d $s4, $a0, 64 + ld.d $s5, $a0, 72 + ld.d $s6, $a0, 80 + ld.d $s7, $a0, 88 + ld.d $s8, $a0, 96 +#ifndef __loongarch_soft_float + fld.d $fs0, $a0, 104 + fld.d $fs1, $a0, 112 + fld.d $fs2, $a0, 120 + fld.d $fs3, $a0, 128 + fld.d $fs4, $a0, 136 + fld.d $fs5, $a0, 144 + fld.d $fs6, $a0, 152 + fld.d $fs7, $a0, 160 +#endif + sltui $a0, $a1, 1 + add.d $a0, $a0, $a1 + jr $ra diff --git a/src/setjmp/loongarch64/setjmp.S b/src/setjmp/loongarch64/setjmp.S new file mode 100644 index 00000000..d158a3d2 --- /dev/null +++ b/src/setjmp/loongarch64/setjmp.S @@ -0,0 +1,34 @@ +.global __setjmp +.global _setjmp +.global setjmp +.type __setjmp,@function +.type _setjmp,@function +.type setjmp,@function +__setjmp: +_setjmp: +setjmp: + st.d $ra, $a0, 0 + st.d $sp, $a0, 8 + st.d $r21,$a0, 16 + st.d $fp, $a0, 24 + st.d $s0, $a0, 32 + st.d $s1, $a0, 40 + st.d $s2, $a0, 48 + st.d $s3, $a0, 56 + st.d $s4, $a0, 64 + st.d $s5, $a0, 72 + st.d $s6, $a0, 80 + st.d $s7, $a0, 88 + st.d $s8, $a0, 96 +#ifndef __loongarch_soft_float + fst.d $fs0, $a0, 104 + fst.d $fs1, $a0, 112 + fst.d $fs2, $a0, 120 + fst.d $fs3, $a0, 128 + fst.d $fs4, $a0, 136 + fst.d $fs5, $a0, 144 + fst.d $fs6, $a0, 152 + fst.d $fs7, $a0, 160 +#endif + move $a0, $zero + jr $ra diff --git a/src/setjmp/powerpc/longjmp.S b/src/setjmp/powerpc/longjmp.S index e598bd05..465e4cd7 100644 --- a/src/setjmp/powerpc/longjmp.S +++ b/src/setjmp/powerpc/longjmp.S @@ -37,7 +37,37 @@ longjmp: lwz 29, 72(3) lwz 30, 76(3) lwz 31, 80(3) -#ifndef _SOFT_FLOAT +#if defined(_SOFT_FLOAT) || defined(__NO_FPRS__) + mflr 0 + bl 1f + .hidden __hwcap + .long __hwcap-. +1: mflr 6 + lwz 5, 0(6) + lwzx 6, 6, 5 + andis. 6, 6, 0x80 + beq 1f + .long 0x11c35b01 /* evldd 14,88(3) */ + .long 0x11e36301 /* ... */ + .long 0x12036b01 + .long 0x12237301 + .long 0x12437b01 + .long 0x12638301 + .long 0x12838b01 + .long 0x12a39301 + .long 0x12c39b01 + .long 0x12e3a301 + .long 0x1303ab01 + .long 0x1323b301 + .long 0x1343bb01 + .long 0x1363c301 + .long 0x1383cb01 + .long 0x13a3d301 + .long 0x13c3db01 + .long 0x13e3e301 /* evldd 31,224(3) */ + .long 0x11a3eb01 /* evldd 13,232(3) */ +1: mtlr 0 +#else lfd 14,88(3) lfd 15,96(3) lfd 16,104(3) diff --git a/src/setjmp/powerpc/setjmp.S b/src/setjmp/powerpc/setjmp.S index cd91a207..f1fcce33 100644 --- a/src/setjmp/powerpc/setjmp.S +++ b/src/setjmp/powerpc/setjmp.S @@ -37,7 +37,37 @@ setjmp: stw 29, 72(3) stw 30, 76(3) stw 31, 80(3) -#ifndef _SOFT_FLOAT +#if defined(_SOFT_FLOAT) || defined(__NO_FPRS__) + mflr 0 + bl 1f + .hidden __hwcap + .long __hwcap-. +1: mflr 4 + lwz 5, 0(4) + lwzx 4, 4, 5 + andis. 4, 4, 0x80 + beq 1f + .long 0x11c35b21 /* evstdd 14,88(3) */ + .long 0x11e36321 /* ... */ + .long 0x12036b21 + .long 0x12237321 + .long 0x12437b21 + .long 0x12638321 + .long 0x12838b21 + .long 0x12a39321 + .long 0x12c39b21 + .long 0x12e3a321 + .long 0x1303ab21 + .long 0x1323b321 + .long 0x1343bb21 + .long 0x1363c321 + .long 0x1383cb21 + .long 0x13a3d321 + .long 0x13c3db21 + .long 0x13e3e321 /* evstdd 31,224(3) */ + .long 0x11a3eb21 /* evstdd 13,232(3) */ +1: mtlr 0 +#else stfd 14,88(3) stfd 15,96(3) stfd 16,104(3) diff --git a/src/setjmp/riscv32/longjmp.S b/src/setjmp/riscv32/longjmp.S new file mode 100644 index 00000000..f9cb3318 --- /dev/null +++ b/src/setjmp/riscv32/longjmp.S @@ -0,0 +1,42 @@ +.global __longjmp +.global _longjmp +.global longjmp +.type __longjmp, %function +.type _longjmp, %function +.type longjmp, %function +__longjmp: +_longjmp: +longjmp: + lw s0, 0(a0) + lw s1, 4(a0) + lw s2, 8(a0) + lw s3, 12(a0) + lw s4, 16(a0) + lw s5, 20(a0) + lw s6, 24(a0) + lw s7, 28(a0) + lw s8, 32(a0) + lw s9, 36(a0) + lw s10, 40(a0) + lw s11, 44(a0) + lw sp, 48(a0) + lw ra, 52(a0) + +#ifndef __riscv_float_abi_soft + fld fs0, 56(a0) + fld fs1, 64(a0) + fld fs2, 72(a0) + fld fs3, 80(a0) + fld fs4, 88(a0) + fld fs5, 96(a0) + fld fs6, 104(a0) + fld fs7, 112(a0) + fld fs8, 120(a0) + fld fs9, 128(a0) + fld fs10, 136(a0) + fld fs11, 144(a0) +#endif + + seqz a0, a1 + add a0, a0, a1 + ret diff --git a/src/setjmp/riscv32/setjmp.S b/src/setjmp/riscv32/setjmp.S new file mode 100644 index 00000000..8a75cf55 --- /dev/null +++ b/src/setjmp/riscv32/setjmp.S @@ -0,0 +1,41 @@ +.global __setjmp +.global _setjmp +.global setjmp +.type __setjmp, %function +.type _setjmp, %function +.type setjmp, %function +__setjmp: +_setjmp: +setjmp: + sw s0, 0(a0) + sw s1, 4(a0) + sw s2, 8(a0) + sw s3, 12(a0) + sw s4, 16(a0) + sw s5, 20(a0) + sw s6, 24(a0) + sw s7, 28(a0) + sw s8, 32(a0) + sw s9, 36(a0) + sw s10, 40(a0) + sw s11, 44(a0) + sw sp, 48(a0) + sw ra, 52(a0) + +#ifndef __riscv_float_abi_soft + fsd fs0, 56(a0) + fsd fs1, 64(a0) + fsd fs2, 72(a0) + fsd fs3, 80(a0) + fsd fs4, 88(a0) + fsd fs5, 96(a0) + fsd fs6, 104(a0) + fsd fs7, 112(a0) + fsd fs8, 120(a0) + fsd fs9, 128(a0) + fsd fs10, 136(a0) + fsd fs11, 144(a0) +#endif + + li a0, 0 + ret diff --git a/src/setjmp/x32/longjmp.s b/src/setjmp/x32/longjmp.s index e175a4b9..1b2661c3 100644 --- a/src/setjmp/x32/longjmp.s +++ b/src/setjmp/x32/longjmp.s @@ -5,18 +5,14 @@ .type longjmp,@function _longjmp: longjmp: - mov %rsi,%rax /* val will be longjmp return */ - test %rax,%rax - jnz 1f - inc %rax /* if val==0, val=1 per longjmp semantics */ -1: + xor %eax,%eax + cmp $1,%esi /* CF = val ? 0 : 1 */ + adc %esi,%eax /* eax = val + !val */ mov (%rdi),%rbx /* rdi is the jmp_buf, restore regs from it */ mov 8(%rdi),%rbp mov 16(%rdi),%r12 mov 24(%rdi),%r13 mov 32(%rdi),%r14 mov 40(%rdi),%r15 - mov 48(%rdi),%rdx /* this ends up being the stack pointer */ - mov %rdx,%rsp - mov 56(%rdi),%rdx /* this is the instruction pointer */ - jmp *%rdx /* goto saved address without altering rsp */ + mov 48(%rdi),%rsp + jmp *56(%rdi) /* goto saved address without altering rsp */ diff --git a/src/setjmp/x32/setjmp.s b/src/setjmp/x32/setjmp.s index 98f58b8d..d95e4853 100644 --- a/src/setjmp/x32/setjmp.s +++ b/src/setjmp/x32/setjmp.s @@ -18,5 +18,5 @@ setjmp: mov %rdx,48(%rdi) mov (%rsp),%rdx /* save return addr ptr for new rip */ mov %rdx,56(%rdi) - xor %rax,%rax /* always return 0 */ + xor %eax,%eax /* always return 0 */ ret diff --git a/src/setjmp/x86_64/longjmp.s b/src/setjmp/x86_64/longjmp.s index e175a4b9..1b2661c3 100644 --- a/src/setjmp/x86_64/longjmp.s +++ b/src/setjmp/x86_64/longjmp.s @@ -5,18 +5,14 @@ .type longjmp,@function _longjmp: longjmp: - mov %rsi,%rax /* val will be longjmp return */ - test %rax,%rax - jnz 1f - inc %rax /* if val==0, val=1 per longjmp semantics */ -1: + xor %eax,%eax + cmp $1,%esi /* CF = val ? 0 : 1 */ + adc %esi,%eax /* eax = val + !val */ mov (%rdi),%rbx /* rdi is the jmp_buf, restore regs from it */ mov 8(%rdi),%rbp mov 16(%rdi),%r12 mov 24(%rdi),%r13 mov 32(%rdi),%r14 mov 40(%rdi),%r15 - mov 48(%rdi),%rdx /* this ends up being the stack pointer */ - mov %rdx,%rsp - mov 56(%rdi),%rdx /* this is the instruction pointer */ - jmp *%rdx /* goto saved address without altering rsp */ + mov 48(%rdi),%rsp + jmp *56(%rdi) /* goto saved address without altering rsp */ diff --git a/src/setjmp/x86_64/setjmp.s b/src/setjmp/x86_64/setjmp.s index 98f58b8d..d95e4853 100644 --- a/src/setjmp/x86_64/setjmp.s +++ b/src/setjmp/x86_64/setjmp.s @@ -18,5 +18,5 @@ setjmp: mov %rdx,48(%rdi) mov (%rsp),%rdx /* save return addr ptr for new rip */ mov %rdx,56(%rdi) - xor %rax,%rax /* always return 0 */ + xor %eax,%eax /* always return 0 */ ret diff --git a/src/signal/arm/sigsetjmp.s b/src/signal/arm/sigsetjmp.s index 318addba..69ebbf49 100644 --- a/src/signal/arm/sigsetjmp.s +++ b/src/signal/arm/sigsetjmp.s @@ -6,9 +6,10 @@ sigsetjmp: __sigsetjmp: tst r1,r1 - beq setjmp + bne 1f + b setjmp - str lr,[r0,#256] +1: str lr,[r0,#256] str r4,[r0,#260+8] mov r4,r0 diff --git a/src/signal/block.c b/src/signal/block.c index d7f61001..cc8698f0 100644 --- a/src/signal/block.c +++ b/src/signal/block.c @@ -3,9 +3,9 @@ #include <signal.h> static const unsigned long all_mask[] = { -#if ULONG_MAX == 0xffffffff && _NSIG == 129 +#if ULONG_MAX == 0xffffffff && _NSIG > 65 -1UL, -1UL, -1UL, -1UL -#elif ULONG_MAX == 0xffffffff +#elif ULONG_MAX == 0xffffffff || _NSIG > 65 -1UL, -1UL #else -1UL diff --git a/src/signal/mips/restore.s b/src/signal/loongarch64/restore.s index b6dadce0..d90a8ebb 100644 --- a/src/signal/mips/restore.s +++ b/src/signal/loongarch64/restore.s @@ -1,15 +1,10 @@ -.set noreorder - .global __restore_rt -.hidden __restore_rt -.type __restore_rt,@function -__restore_rt: - li $2, 4193 - syscall - .global __restore +.hidden __restore_rt .hidden __restore +.type __restore_rt,@function .type __restore,@function +__restore_rt: __restore: - li $2, 4119 - syscall + li.w $a7, 139 + syscall 0 diff --git a/src/signal/loongarch64/sigsetjmp.s b/src/signal/loongarch64/sigsetjmp.s new file mode 100644 index 00000000..9c0e3ae2 --- /dev/null +++ b/src/signal/loongarch64/sigsetjmp.s @@ -0,0 +1,25 @@ +.global sigsetjmp +.global __sigsetjmp +.type sigsetjmp,@function +.type __sigsetjmp,@function +sigsetjmp: +__sigsetjmp: + beq $a1, $zero, 1f + st.d $ra, $a0, 184 + st.d $s0, $a0, 200 #184+8+8 + move $s0, $a0 + + la.global $t0, setjmp + jirl $ra, $t0, 0 + + move $a1, $a0 # Return from 'setjmp' or 'longjmp' + move $a0, $s0 + ld.d $ra, $a0, 184 + ld.d $s0, $a0, 200 #184+8+8 + +.hidden __sigsetjmp_tail + la.global $t0, __sigsetjmp_tail + jr $t0 +1: + la.global $t0, setjmp + jr $t0 diff --git a/src/signal/mips64/restore.s b/src/signal/mips64/restore.s deleted file mode 100644 index 401f8e73..00000000 --- a/src/signal/mips64/restore.s +++ /dev/null @@ -1,11 +0,0 @@ -.set noreorder -.global __restore_rt -.global __restore -.hidden __restore_rt -.hidden __restore -.type __restore_rt,@function -.type __restore,@function -__restore_rt: -__restore: - li $2,5211 - syscall diff --git a/src/signal/mipsn32/restore.s b/src/signal/mipsn32/restore.s deleted file mode 100644 index 4cd4e1b4..00000000 --- a/src/signal/mipsn32/restore.s +++ /dev/null @@ -1,11 +0,0 @@ -.set noreorder -.global __restore_rt -.global __restore -.hidden __restore_rt -.hidden __restore -.type __restore_rt,@function -.type __restore,@function -__restore_rt: -__restore: - li $2,6211 - syscall diff --git a/src/signal/riscv32/restore.s b/src/signal/riscv32/restore.s new file mode 100644 index 00000000..40012c75 --- /dev/null +++ b/src/signal/riscv32/restore.s @@ -0,0 +1,8 @@ +.global __restore +.type __restore, %function +__restore: +.global __restore_rt +.type __restore_rt, %function +__restore_rt: + li a7, 139 # SYS_rt_sigreturn + ecall diff --git a/src/signal/riscv32/sigsetjmp.s b/src/signal/riscv32/sigsetjmp.s new file mode 100644 index 00000000..c1caeab1 --- /dev/null +++ b/src/signal/riscv32/sigsetjmp.s @@ -0,0 +1,23 @@ +.global sigsetjmp +.global __sigsetjmp +.type sigsetjmp, %function +.type __sigsetjmp, %function +sigsetjmp: +__sigsetjmp: + bnez a1, 1f + tail setjmp +1: + + sw ra, 152(a0) + sw s0, 164(a0) + mv s0, a0 + + call setjmp + + mv a1, a0 + mv a0, s0 + lw s0, 164(a0) + lw ra, 152(a0) + +.hidden __sigsetjmp_tail + tail __sigsetjmp_tail diff --git a/src/signal/sh/sigsetjmp.s b/src/signal/sh/sigsetjmp.s index 1e2270be..f0f604e2 100644 --- a/src/signal/sh/sigsetjmp.s +++ b/src/signal/sh/sigsetjmp.s @@ -27,7 +27,7 @@ __sigsetjmp: mov.l 3f, r0 4: braf r0 - mov.l @(4+8,r4), r8 + mov.l @(4+8,r6), r8 9: mov.l 5f, r0 6: braf r0 diff --git a/src/signal/sigaction.c b/src/signal/sigaction.c index c109bea0..e45308fa 100644 --- a/src/signal/sigaction.c +++ b/src/signal/sigaction.c @@ -7,12 +7,6 @@ #include "lock.h" #include "ksigaction.h" -static volatile int dummy_lock[1] = { 0 }; - -extern hidden volatile int __abort_lock[1]; - -weak_alias(dummy_lock, __abort_lock); - static int unmask_done; static unsigned long handler_set[_NSIG/(8*sizeof(long))]; @@ -26,7 +20,6 @@ volatile int __eintr_valid_flag; int __libc_sigaction(int sig, const struct sigaction *restrict sa, struct sigaction *restrict old) { struct k_sigaction ksa, ksa_old; - unsigned long set[_NSIG/(8*sizeof(long))]; if (sa) { if ((uintptr_t)sa->sa_handler > 1UL) { a_or_l(handler_set+(sig-1)/(8*sizeof(long)), @@ -50,24 +43,15 @@ int __libc_sigaction(int sig, const struct sigaction *restrict sa, struct sigact a_store(&__eintr_valid_flag, 1); } } - /* Changing the disposition of SIGABRT to anything but - * SIG_DFL requires a lock, so that it cannot be changed - * while abort is terminating the process after simply - * calling raise(SIGABRT) failed to do so. */ - if (sa->sa_handler != SIG_DFL && sig == SIGABRT) { - __block_all_sigs(&set); - LOCK(__abort_lock); - } ksa.handler = sa->sa_handler; - ksa.flags = sa->sa_flags | SA_RESTORER; + ksa.flags = sa->sa_flags; +#ifdef SA_RESTORER + ksa.flags |= SA_RESTORER; ksa.restorer = (sa->sa_flags & SA_SIGINFO) ? __restore_rt : __restore; +#endif memcpy(&ksa.mask, &sa->sa_mask, _NSIG/8); } int r = __syscall(SYS_rt_sigaction, sig, sa?&ksa:0, old?&ksa_old:0, _NSIG/8); - if (sig == SIGABRT && sa && sa->sa_handler != SIG_DFL) { - UNLOCK(__abort_lock); - __restore_sigs(&set); - } if (old && !r) { old->sa_handler = ksa_old.handler; old->sa_flags = ksa_old.flags; @@ -78,11 +62,26 @@ int __libc_sigaction(int sig, const struct sigaction *restrict sa, struct sigact int __sigaction(int sig, const struct sigaction *restrict sa, struct sigaction *restrict old) { + unsigned long set[_NSIG/(8*sizeof(long))]; + if (sig-32U < 3 || sig-1U >= _NSIG-1) { errno = EINVAL; return -1; } - return __libc_sigaction(sig, sa, old); + + /* Doing anything with the disposition of SIGABRT requires a lock, + * so that it cannot be changed while abort is terminating the + * process and so any change made by abort can't be observed. */ + if (sig == SIGABRT) { + __block_all_sigs(&set); + LOCK(__abort_lock); + } + int r = __libc_sigaction(sig, sa, old); + if (sig == SIGABRT) { + UNLOCK(__abort_lock); + __restore_sigs(&set); + } + return r; } weak_alias(__sigaction, sigaction); diff --git a/src/stat/__xstat.c b/src/stat/__xstat.c index f6303430..b4560df7 100644 --- a/src/stat/__xstat.c +++ b/src/stat/__xstat.c @@ -1,5 +1,7 @@ #include <sys/stat.h> +#if !_REDIR_TIME64 + int __fxstat(int ver, int fd, struct stat *buf) { return fstat(fd, buf); @@ -20,10 +22,7 @@ int __xstat(int ver, const char *path, struct stat *buf) return stat(path, buf); } -weak_alias(__fxstat, __fxstat64); -weak_alias(__fxstatat, __fxstatat64); -weak_alias(__lxstat, __lxstat64); -weak_alias(__xstat, __xstat64); +#endif int __xmknod(int ver, const char *path, mode_t mode, dev_t *dev) { diff --git a/src/stat/fchmodat.c b/src/stat/fchmodat.c index be61bdf3..92c9d1b0 100644 --- a/src/stat/fchmodat.c +++ b/src/stat/fchmodat.c @@ -5,17 +5,20 @@ int fchmodat(int fd, const char *path, mode_t mode, int flag) { - if (!flag) return syscall(SYS_fchmodat, fd, path, mode, flag); + if (!flag) return syscall(SYS_fchmodat, fd, path, mode); + + int ret = __syscall(SYS_fchmodat2, fd, path, mode, flag); + if (ret != -ENOSYS) return __syscall_ret(ret); if (flag != AT_SYMLINK_NOFOLLOW) return __syscall_ret(-EINVAL); struct stat st; - int ret, fd2; + int fd2; char proc[15+3*sizeof(int)]; - if ((ret = __syscall(SYS_fstatat, fd, path, &st, flag))) - return __syscall_ret(ret); + if (fstatat(fd, path, &st, flag)) + return -1; if (S_ISLNK(st.st_mode)) return __syscall_ret(-EOPNOTSUPP); @@ -26,12 +29,12 @@ int fchmodat(int fd, const char *path, mode_t mode, int flag) } __procfdname(proc, fd2); - ret = __syscall(SYS_fstatat, AT_FDCWD, proc, &st, 0); + ret = stat(proc, &st); if (!ret) { - if (S_ISLNK(st.st_mode)) ret = -EOPNOTSUPP; - else ret = __syscall(SYS_fchmodat, AT_FDCWD, proc, mode); + if (S_ISLNK(st.st_mode)) ret = __syscall_ret(-EOPNOTSUPP); + else ret = syscall(SYS_fchmodat, AT_FDCWD, proc, mode); } __syscall(SYS_close, fd2); - return __syscall_ret(ret); + return ret; } diff --git a/src/stat/fstat.c b/src/stat/fstat.c index 07f9a5de..fd28b8ac 100644 --- a/src/stat/fstat.c +++ b/src/stat/fstat.c @@ -4,10 +4,10 @@ #include <fcntl.h> #include "syscall.h" -int fstat(int fd, struct stat *st) +int __fstat(int fd, struct stat *st) { if (fd<0) return __syscall_ret(-EBADF); - return fstatat(fd, "", st, AT_EMPTY_PATH); + return __fstatat(fd, "", st, AT_EMPTY_PATH); } -weak_alias(fstat, fstat64); +weak_alias(__fstat, fstat); diff --git a/src/stat/fstatat.c b/src/stat/fstatat.c index d915fa10..9eed063b 100644 --- a/src/stat/fstatat.c +++ b/src/stat/fstatat.c @@ -6,7 +6,6 @@ #include <stdint.h> #include <sys/sysmacros.h> #include "syscall.h" -#include "kstat.h" struct statx { uint32_t stx_mask; @@ -37,6 +36,7 @@ static int fstatat_statx(int fd, const char *restrict path, struct stat *restric { struct statx stx; + flag |= AT_NO_AUTOMOUNT; int ret = __syscall(SYS_statx, fd, path, flag, 0x7ff, &stx); if (ret) return ret; @@ -57,10 +57,22 @@ static int fstatat_statx(int fd, const char *restrict path, struct stat *restric .st_mtim.tv_nsec = stx.stx_mtime.tv_nsec, .st_ctim.tv_sec = stx.stx_ctime.tv_sec, .st_ctim.tv_nsec = stx.stx_ctime.tv_nsec, +#if _REDIR_TIME64 + .__st_atim32.tv_sec = stx.stx_atime.tv_sec, + .__st_atim32.tv_nsec = stx.stx_atime.tv_nsec, + .__st_mtim32.tv_sec = stx.stx_mtime.tv_sec, + .__st_mtim32.tv_nsec = stx.stx_mtime.tv_nsec, + .__st_ctim32.tv_sec = stx.stx_ctime.tv_sec, + .__st_ctim32.tv_nsec = stx.stx_ctime.tv_nsec, +#endif }; return 0; } +#ifdef SYS_fstatat + +#include "kstat.h" + static int fstatat_kstat(int fd, const char *restrict path, struct stat *restrict st, int flag) { int ret; @@ -110,20 +122,33 @@ static int fstatat_kstat(int fd, const char *restrict path, struct stat *restric .st_mtim.tv_nsec = kst.st_mtime_nsec, .st_ctim.tv_sec = kst.st_ctime_sec, .st_ctim.tv_nsec = kst.st_ctime_nsec, +#if _REDIR_TIME64 + .__st_atim32.tv_sec = kst.st_atime_sec, + .__st_atim32.tv_nsec = kst.st_atime_nsec, + .__st_mtim32.tv_sec = kst.st_mtime_sec, + .__st_mtim32.tv_nsec = kst.st_mtime_nsec, + .__st_ctim32.tv_sec = kst.st_ctime_sec, + .__st_ctim32.tv_nsec = kst.st_ctime_nsec, +#endif }; return 0; } +#endif -int fstatat(int fd, const char *restrict path, struct stat *restrict st, int flag) +int __fstatat(int fd, const char *restrict path, struct stat *restrict st, int flag) { int ret; +#ifdef SYS_fstatat if (sizeof((struct kstat){0}.st_atime_sec) < sizeof(time_t)) { ret = fstatat_statx(fd, path, st, flag); if (ret!=-ENOSYS) return __syscall_ret(ret); } ret = fstatat_kstat(fd, path, st, flag); +#else + ret = fstatat_statx(fd, path, st, flag); +#endif return __syscall_ret(ret); } -weak_alias(fstatat, fstatat64); +weak_alias(__fstatat, fstatat); diff --git a/src/stat/lstat.c b/src/stat/lstat.c index 9f95218a..6822fcae 100644 --- a/src/stat/lstat.c +++ b/src/stat/lstat.c @@ -5,5 +5,3 @@ int lstat(const char *restrict path, struct stat *restrict buf) { return fstatat(AT_FDCWD, path, buf, AT_SYMLINK_NOFOLLOW); } - -weak_alias(lstat, lstat64); diff --git a/src/stat/stat.c b/src/stat/stat.c index 528870d2..23570e7a 100644 --- a/src/stat/stat.c +++ b/src/stat/stat.c @@ -5,5 +5,3 @@ int stat(const char *restrict path, struct stat *restrict buf) { return fstatat(AT_FDCWD, path, buf, 0); } - -weak_alias(stat, stat64); diff --git a/src/stat/statvfs.c b/src/stat/statvfs.c index f65d1b54..bc12da8b 100644 --- a/src/stat/statvfs.c +++ b/src/stat/statvfs.c @@ -39,6 +39,7 @@ static void fixup(struct statvfs *out, const struct statfs *in) out->f_fsid = in->f_fsid.__val[0]; out->f_flag = in->f_flags; out->f_namemax = in->f_namelen; + out->f_type = in->f_type; } int statvfs(const char *restrict path, struct statvfs *restrict buf) @@ -56,8 +57,3 @@ int fstatvfs(int fd, struct statvfs *buf) fixup(buf, &kbuf); return 0; } - -weak_alias(statvfs, statvfs64); -weak_alias(statfs, statfs64); -weak_alias(fstatvfs, fstatvfs64); -weak_alias(fstatfs, fstatfs64); diff --git a/src/stdio/__stdio_close.c b/src/stdio/__stdio_close.c index 79452bdb..30291328 100644 --- a/src/stdio/__stdio_close.c +++ b/src/stdio/__stdio_close.c @@ -1,4 +1,5 @@ #include "stdio_impl.h" +#include "aio_impl.h" static int dummy(int fd) { diff --git a/src/stdio/__string_read.c b/src/stdio/__string_read.c deleted file mode 100644 index 7b50a7e1..00000000 --- a/src/stdio/__string_read.c +++ /dev/null @@ -1,16 +0,0 @@ -#include "stdio_impl.h" -#include <string.h> - -size_t __string_read(FILE *f, unsigned char *buf, size_t len) -{ - char *src = f->cookie; - size_t k = len+256; - char *end = memchr(src, 0, k); - if (end) k = end-src; - if (k < len) len = k; - memcpy(buf, src, len); - f->rpos = (void *)(src+len); - f->rend = (void *)(src+k); - f->cookie = src+k; - return len; -} diff --git a/src/stdio/fgetpos.c b/src/stdio/fgetpos.c index 50813d2c..392f7323 100644 --- a/src/stdio/fgetpos.c +++ b/src/stdio/fgetpos.c @@ -7,5 +7,3 @@ int fgetpos(FILE *restrict f, fpos_t *restrict pos) *(long long *)pos = off; return 0; } - -weak_alias(fgetpos, fgetpos64); diff --git a/src/stdio/fgets.c b/src/stdio/fgets.c index 6171f398..4a100b39 100644 --- a/src/stdio/fgets.c +++ b/src/stdio/fgets.c @@ -12,13 +12,14 @@ char *fgets(char *restrict s, int n, FILE *restrict f) FLOCK(f); - if (n--<=1) { + if (n<=1) { f->mode |= f->mode-1; FUNLOCK(f); - if (n) return 0; + if (n<1) return 0; *s = 0; return s; } + n--; while (n) { if (f->rpos != f->rend) { diff --git a/src/stdio/fgetws.c b/src/stdio/fgetws.c index b08b3049..195cb435 100644 --- a/src/stdio/fgetws.c +++ b/src/stdio/fgetws.c @@ -1,6 +1,5 @@ #include "stdio_impl.h" #include <wchar.h> -#include <errno.h> wint_t __fgetwc_unlocked(FILE *); @@ -12,10 +11,6 @@ wchar_t *fgetws(wchar_t *restrict s, int n, FILE *restrict f) FLOCK(f); - /* Setup a dummy errno so we can detect EILSEQ. This is - * the only way to catch encoding errors in the form of a - * partial character just before EOF. */ - errno = EAGAIN; for (; n; n--) { wint_t c = __fgetwc_unlocked(f); if (c == WEOF) break; @@ -23,7 +18,7 @@ wchar_t *fgetws(wchar_t *restrict s, int n, FILE *restrict f) if (c == '\n') break; } *p = 0; - if (ferror(f) || errno==EILSEQ) p = s; + if (ferror(f)) p = s; FUNLOCK(f); diff --git a/src/stdio/fmemopen.c b/src/stdio/fmemopen.c index 5685092e..343e3e3f 100644 --- a/src/stdio/fmemopen.c +++ b/src/stdio/fmemopen.c @@ -2,6 +2,7 @@ #include <errno.h> #include <string.h> #include <stdlib.h> +#include <stddef.h> #include <inttypes.h> #include "libc.h" @@ -95,18 +96,17 @@ FILE *fmemopen(void *restrict buf, size_t size, const char *restrict mode) f = malloc(sizeof *f + (buf?0:size)); if (!f) return 0; - memset(&f->f, 0, sizeof f->f); + memset(f, 0, offsetof(struct mem_FILE, buf)); f->f.cookie = &f->c; f->f.fd = -1; f->f.lbf = EOF; f->f.buf = f->buf + UNGET; f->f.buf_size = sizeof f->buf - UNGET; if (!buf) { - buf = f->buf2;; + buf = f->buf2; memset(buf, 0, size); } - memset(&f->c, 0, sizeof f->c); f->c.buf = buf; f->c.size = size; f->c.mode = *mode; diff --git a/src/stdio/fopen.c b/src/stdio/fopen.c index e1b91e12..80bc341e 100644 --- a/src/stdio/fopen.c +++ b/src/stdio/fopen.c @@ -29,5 +29,3 @@ FILE *fopen(const char *restrict filename, const char *restrict mode) __syscall(SYS_close, fd); return 0; } - -weak_alias(fopen, fopen64); diff --git a/src/stdio/freopen.c b/src/stdio/freopen.c index 615d4b47..1641a4c5 100644 --- a/src/stdio/freopen.c +++ b/src/stdio/freopen.c @@ -40,6 +40,8 @@ FILE *freopen(const char *restrict filename, const char *restrict mode, FILE *re fclose(f2); } + f->mode = 0; + f->locale = 0; FUNLOCK(f); return f; @@ -49,5 +51,3 @@ fail: fclose(f); return NULL; } - -weak_alias(freopen, freopen64); diff --git a/src/stdio/fseek.c b/src/stdio/fseek.c index 439308f7..c7425802 100644 --- a/src/stdio/fseek.c +++ b/src/stdio/fseek.c @@ -1,7 +1,14 @@ #include "stdio_impl.h" +#include <errno.h> int __fseeko_unlocked(FILE *f, off_t off, int whence) { + /* Fail immediately for invalid whence argument. */ + if (whence != SEEK_CUR && whence != SEEK_SET && whence != SEEK_END) { + errno = EINVAL; + return -1; + } + /* Adjust relative offset for unread data in buffer, if any. */ if (whence == SEEK_CUR && f->rend) off -= f->rend - f->rpos; @@ -39,5 +46,3 @@ int fseek(FILE *f, long off, int whence) } weak_alias(__fseeko, fseeko); - -weak_alias(fseeko, fseeko64); diff --git a/src/stdio/fsetpos.c b/src/stdio/fsetpos.c index 77ab8d82..779cb3cc 100644 --- a/src/stdio/fsetpos.c +++ b/src/stdio/fsetpos.c @@ -4,5 +4,3 @@ int fsetpos(FILE *f, const fpos_t *pos) { return __fseeko(f, *(const long long *)pos, SEEK_SET); } - -weak_alias(fsetpos, fsetpos64); diff --git a/src/stdio/ftell.c b/src/stdio/ftell.c index 1a2afbbc..1e1a08d8 100644 --- a/src/stdio/ftell.c +++ b/src/stdio/ftell.c @@ -37,5 +37,3 @@ long ftell(FILE *f) } weak_alias(__ftello, ftello); - -weak_alias(ftello, ftello64); diff --git a/src/stdio/getdelim.c b/src/stdio/getdelim.c index d2f5b15a..df114441 100644 --- a/src/stdio/getdelim.c +++ b/src/stdio/getdelim.c @@ -55,9 +55,11 @@ ssize_t getdelim(char **restrict s, size_t *restrict n, int delim, FILE *restric *s = tmp; *n = m; } - memcpy(*s+i, f->rpos, k); - f->rpos += k; - i += k; + if (k) { + memcpy(*s+i, f->rpos, k); + f->rpos += k; + i += k; + } if (z) break; if ((c = getc_unlocked(f)) == EOF) { if (!i || !feof(f)) { diff --git a/src/stdio/ofl.c b/src/stdio/ofl.c index f2d3215a..aad3d171 100644 --- a/src/stdio/ofl.c +++ b/src/stdio/ofl.c @@ -1,8 +1,10 @@ #include "stdio_impl.h" #include "lock.h" +#include "fork_impl.h" static FILE *ofl_head; static volatile int ofl_lock[1]; +volatile int *const __stdio_ofl_lockptr = ofl_lock; FILE **__ofl_lock() { diff --git a/src/stdio/open_wmemstream.c b/src/stdio/open_wmemstream.c index ed1b561d..b8ae4a79 100644 --- a/src/stdio/open_wmemstream.c +++ b/src/stdio/open_wmemstream.c @@ -40,8 +40,12 @@ fail: static size_t wms_write(FILE *f, const unsigned char *buf, size_t len) { struct cookie *c = f->cookie; - size_t len2; + size_t len2 = f->wpos - f->wbase; wchar_t *newbuf; + if (len2) { + f->wpos = f->wbase; + if (wms_write(f, f->wbase, len2) < len2) return 0; + } if (len + c->pos >= c->space) { len2 = 2*c->space+1 | c->pos+len+1; if (len2 > SSIZE_MAX/4) return 0; diff --git a/src/stdio/pclose.c b/src/stdio/pclose.c index 080a4262..c64da405 100644 --- a/src/stdio/pclose.c +++ b/src/stdio/pclose.c @@ -7,7 +7,7 @@ int pclose(FILE *f) int status, r; pid_t pid = f->pipe_pid; fclose(f); - while ((r=__syscall(SYS_wait4, pid, &status, 0, 0)) == -EINTR); + while ((r=__sys_wait4(pid, &status, 0, 0)) == -EINTR); if (r<0) return __syscall_ret(r); return status; } diff --git a/src/stdio/popen.c b/src/stdio/popen.c index 92cb57ee..3ec83394 100644 --- a/src/stdio/popen.c +++ b/src/stdio/popen.c @@ -31,25 +31,12 @@ FILE *popen(const char *cmd, const char *mode) __syscall(SYS_close, p[1]); return NULL; } - FLOCK(f); - - /* If the child's end of the pipe happens to already be on the final - * fd number to which it will be assigned (either 0 or 1), it must - * be moved to a different fd. Otherwise, there is no safe way to - * remove the close-on-exec flag in the child without also creating - * a file descriptor leak race condition in the parent. */ - if (p[1-op] == 1-op) { - int tmp = fcntl(1-op, F_DUPFD_CLOEXEC, 0); - if (tmp < 0) { - e = errno; - goto fail; - } - __syscall(SYS_close, p[1-op]); - p[1-op] = tmp; - } e = ENOMEM; if (!posix_spawn_file_actions_init(&fa)) { + for (FILE *l = *__ofl_lock(); l; l=l->next) + if (l->pipe_pid && posix_spawn_file_actions_addclose(&fa, l->fd)) + goto fail; if (!posix_spawn_file_actions_adddup2(&fa, p[1-op], 1-op)) { if (!(e = posix_spawn(&pid, "/bin/sh", &fa, 0, (char *[]){ "sh", "-c", (char *)cmd, 0 }, __environ))) { @@ -58,13 +45,14 @@ FILE *popen(const char *cmd, const char *mode) if (!strchr(mode, 'e')) fcntl(p[op], F_SETFD, 0); __syscall(SYS_close, p[1-op]); - FUNLOCK(f); + __ofl_unlock(); return f; } } +fail: + __ofl_unlock(); posix_spawn_file_actions_destroy(&fa); } -fail: fclose(f); __syscall(SYS_close, p[1-op]); diff --git a/src/stdio/tempnam.c b/src/stdio/tempnam.c index 84f91978..0c65b1f0 100644 --- a/src/stdio/tempnam.c +++ b/src/stdio/tempnam.c @@ -36,11 +36,10 @@ char *tempnam(const char *dir, const char *pfx) for (try=0; try<MAXTRIES; try++) { __randname(s+l-6); -#ifdef SYS_lstat - r = __syscall(SYS_lstat, s, &(struct stat){0}); +#ifdef SYS_readlink + r = __syscall(SYS_readlink, s, (char[1]){0}, 1); #else - r = __syscall(SYS_fstatat, AT_FDCWD, s, - &(struct stat){0}, AT_SYMLINK_NOFOLLOW); + r = __syscall(SYS_readlinkat, AT_FDCWD, s, (char[1]){0}, 1); #endif if (r == -ENOENT) return strdup(s); } diff --git a/src/stdio/tmpfile.c b/src/stdio/tmpfile.c index ae493987..2fa8803f 100644 --- a/src/stdio/tmpfile.c +++ b/src/stdio/tmpfile.c @@ -27,5 +27,3 @@ FILE *tmpfile(void) } return 0; } - -weak_alias(tmpfile, tmpfile64); diff --git a/src/stdio/tmpnam.c b/src/stdio/tmpnam.c index 6c7c253a..71dc8bb1 100644 --- a/src/stdio/tmpnam.c +++ b/src/stdio/tmpnam.c @@ -16,11 +16,10 @@ char *tmpnam(char *buf) int r; for (try=0; try<MAXTRIES; try++) { __randname(s+12); -#ifdef SYS_lstat - r = __syscall(SYS_lstat, s, &(struct stat){0}); +#ifdef SYS_readlink + r = __syscall(SYS_readlink, s, (char[1]){0}, 1); #else - r = __syscall(SYS_fstatat, AT_FDCWD, s, - &(struct stat){0}, AT_SYMLINK_NOFOLLOW); + r = __syscall(SYS_readlinkat, AT_FDCWD, s, (char[1]){0}, 1); #endif if (r == -ENOENT) return strcpy(buf ? buf : internal, s); } diff --git a/src/stdio/vdprintf.c b/src/stdio/vdprintf.c index c35d9b4f..3b9c093b 100644 --- a/src/stdio/vdprintf.c +++ b/src/stdio/vdprintf.c @@ -1,14 +1,9 @@ #include "stdio_impl.h" -static size_t wrap_write(FILE *f, const unsigned char *buf, size_t len) -{ - return __stdio_write(f, buf, len); -} - int vdprintf(int fd, const char *restrict fmt, va_list ap) { FILE f = { - .fd = fd, .lbf = EOF, .write = wrap_write, + .fd = fd, .lbf = EOF, .write = __stdio_write, .buf = (void *)fmt, .buf_size = 0, .lock = -1 }; diff --git a/src/stdio/vfprintf.c b/src/stdio/vfprintf.c index 9b961e7f..497c5e19 100644 --- a/src/stdio/vfprintf.c +++ b/src/stdio/vfprintf.c @@ -52,7 +52,7 @@ static const unsigned char states[]['z'-'A'+1] = { S('o') = UINT, S('u') = UINT, S('x') = UINT, S('X') = UINT, S('e') = DBL, S('f') = DBL, S('g') = DBL, S('a') = DBL, S('E') = DBL, S('F') = DBL, S('G') = DBL, S('A') = DBL, - S('c') = CHAR, S('C') = INT, + S('c') = INT, S('C') = UINT, S('s') = PTR, S('S') = PTR, S('p') = UIPTR, S('n') = PTR, S('m') = NOARG, S('l') = LPRE, S('h') = HPRE, S('L') = BIGLPRE, @@ -62,7 +62,7 @@ static const unsigned char states[]['z'-'A'+1] = { S('o') = ULONG, S('u') = ULONG, S('x') = ULONG, S('X') = ULONG, S('e') = DBL, S('f') = DBL, S('g') = DBL, S('a') = DBL, S('E') = DBL, S('F') = DBL, S('G') = DBL, S('A') = DBL, - S('c') = INT, S('s') = PTR, S('n') = PTR, + S('c') = UINT, S('s') = PTR, S('n') = PTR, S('l') = LLPRE, }, { /* 2: ll-prefixed */ S('d') = LLONG, S('i') = LLONG, @@ -132,7 +132,7 @@ static void pop_arg(union arg *arg, int type, va_list *ap) static void out(FILE *f, const char *s, size_t l) { - if (!(f->flags & F_ERR)) __fwritex((void *)s, l, f); + if (!ferror(f)) __fwritex((void *)s, l, f); } static void pad(FILE *f, char c, int w, int l, int fl) @@ -437,7 +437,7 @@ static int printf_core(FILE *f, const char *fmt, va_list *ap, union arg *nl_arg, unsigned st, ps; int cnt=0, l=0; size_t i; - char buf[sizeof(uintmax_t)*3+3+LDBL_MANT_DIG/4]; + char buf[sizeof(uintmax_t)*3]; const char *prefix; int t, pl; wchar_t wc[2], *ws; @@ -478,8 +478,8 @@ static int printf_core(FILE *f, const char *fmt, va_list *ap, union arg *nl_arg, if (*s=='*') { if (isdigit(s[1]) && s[2]=='$') { l10n=1; - nl_type[s[1]-'0'] = INT; - w = nl_arg[s[1]-'0'].i; + if (!f) nl_type[s[1]-'0'] = INT, w = 0; + else w = nl_arg[s[1]-'0'].i; s+=3; } else if (!l10n) { w = f ? va_arg(*ap, int) : 0; @@ -491,8 +491,8 @@ static int printf_core(FILE *f, const char *fmt, va_list *ap, union arg *nl_arg, /* Read precision */ if (*s=='.' && s[1]=='*') { if (isdigit(s[2]) && s[3]=='$') { - nl_type[s[2]-'0'] = INT; - p = nl_arg[s[2]-'0'].i; + if (!f) nl_type[s[2]-'0'] = INT, p = 0; + else p = nl_arg[s[2]-'0'].i; s+=4; } else if (!l10n) { p = f ? va_arg(*ap, int) : 0; @@ -521,13 +521,18 @@ static int printf_core(FILE *f, const char *fmt, va_list *ap, union arg *nl_arg, if (st==NOARG) { if (argpos>=0) goto inval; } else { - if (argpos>=0) nl_type[argpos]=st, arg=nl_arg[argpos]; - else if (f) pop_arg(&arg, st, ap); + if (argpos>=0) { + if (!f) nl_type[argpos]=st; + else arg=nl_arg[argpos]; + } else if (f) pop_arg(&arg, st, ap); else return 0; } if (!f) continue; + /* Do not process any new directives once in error state. */ + if (ferror(f)) return -1; + z = buf + sizeof(buf); prefix = "-+ 0X0x"; pl = 0; @@ -583,6 +588,7 @@ static int printf_core(FILE *f, const char *fmt, va_list *ap, union arg *nl_arg, } p = MAX(p, z-a + !arg.i); break; + narrow_c: case 'c': *(a=z-(p=1))=arg.i; fl &= ~ZERO_PAD; @@ -597,6 +603,7 @@ static int printf_core(FILE *f, const char *fmt, va_list *ap, union arg *nl_arg, fl &= ~ZERO_PAD; break; case 'C': + if (!arg.i) goto narrow_c; wc[0] = arg.i; wc[1] = 0; arg.p = wc; @@ -672,7 +679,7 @@ int vfprintf(FILE *restrict f, const char *restrict fmt, va_list ap) FLOCK(f); olderr = f->flags & F_ERR; - if (f->mode < 1) f->flags &= ~F_ERR; + f->flags &= ~F_ERR; if (!f->buf_size) { saved_buf = f->buf; f->buf = internal_buf; @@ -688,7 +695,7 @@ int vfprintf(FILE *restrict f, const char *restrict fmt, va_list ap) f->buf_size = 0; f->wpos = f->wbase = f->wend = 0; } - if (f->flags & F_ERR) ret = -1; + if (ferror(f)) ret = -1; f->flags |= olderr; FUNLOCK(f); va_end(ap2); diff --git a/src/stdio/vfscanf.c b/src/stdio/vfscanf.c index 9e030fc4..b78a374d 100644 --- a/src/stdio/vfscanf.c +++ b/src/stdio/vfscanf.c @@ -57,7 +57,7 @@ int vfscanf(FILE *restrict f, const char *restrict fmt, va_list ap) { int width; int size; - int alloc; + int alloc = 0; int base; const unsigned char *p; int c, t; @@ -76,6 +76,9 @@ int vfscanf(FILE *restrict f, const char *restrict fmt, va_list ap) FLOCK(f); + if (!f->rpos) __toread(f); + if (!f->rpos) goto input_fail; + for (p=(const unsigned char *)fmt; *p; p++) { alloc = 0; diff --git a/src/stdio/vfwprintf.c b/src/stdio/vfwprintf.c index 85b036c3..59d5471b 100644 --- a/src/stdio/vfwprintf.c +++ b/src/stdio/vfwprintf.c @@ -45,7 +45,7 @@ static const unsigned char states[]['z'-'A'+1] = { S('o') = UINT, S('u') = UINT, S('x') = UINT, S('X') = UINT, S('e') = DBL, S('f') = DBL, S('g') = DBL, S('a') = DBL, S('E') = DBL, S('F') = DBL, S('G') = DBL, S('A') = DBL, - S('c') = CHAR, S('C') = INT, + S('c') = INT, S('C') = UINT, S('s') = PTR, S('S') = PTR, S('p') = UIPTR, S('n') = PTR, S('m') = NOARG, S('l') = LPRE, S('h') = HPRE, S('L') = BIGLPRE, @@ -55,7 +55,7 @@ static const unsigned char states[]['z'-'A'+1] = { S('o') = ULONG, S('u') = ULONG, S('x') = ULONG, S('X') = ULONG, S('e') = DBL, S('f') = DBL, S('g') = DBL, S('a') = DBL, S('E') = DBL, S('F') = DBL, S('G') = DBL, S('A') = DBL, - S('c') = INT, S('s') = PTR, S('n') = PTR, + S('c') = UINT, S('s') = PTR, S('n') = PTR, S('l') = LLPRE, }, { /* 2: ll-prefixed */ S('d') = LLONG, S('i') = LLONG, @@ -125,7 +125,13 @@ static void pop_arg(union arg *arg, int type, va_list *ap) static void out(FILE *f, const wchar_t *s, size_t l) { - while (l-- && !(f->flags & F_ERR)) fputwc(*s++, f); + while (l-- && !ferror(f)) fputwc(*s++, f); +} + +static void pad(FILE *f, int n, int fl) +{ + if ((fl & LEFT_ADJ) || !n || ferror(f)) return; + fprintf(f, "%*s", n, ""); } static int getint(wchar_t **s) { @@ -242,6 +248,10 @@ static int wprintf_core(FILE *f, const wchar_t *fmt, va_list *ap, union arg *nl_ } if (!f) continue; + + /* Do not process any new directives once in error state. */ + if (ferror(f)) return -1; + t = s[-1]; if (ps && (t&15)==3) t&=~32; @@ -258,25 +268,22 @@ static int wprintf_core(FILE *f, const wchar_t *fmt, va_list *ap, union arg *nl_ } continue; case 'c': + case 'C': if (w<1) w=1; - if (w>1 && !(fl&LEFT_ADJ)) fprintf(f, "%*s", w-1, ""); - fputwc(btowc(arg.i), f); - if (w>1 && (fl&LEFT_ADJ)) fprintf(f, "%*s", w-1, ""); + pad(f, w-1, fl); + out(f, &(wchar_t){t=='C' ? arg.i : btowc(arg.i)}, 1); + pad(f, w-1, fl^LEFT_ADJ); l = w; continue; - case 'C': - fputwc(arg.i, f); - l = 1; - continue; case 'S': a = arg.p; z = a + wcsnlen(a, p<0 ? INT_MAX : p); if (p<0 && *z) goto overflow; p = z-a; if (w<p) w=p; - if (!(fl&LEFT_ADJ)) fprintf(f, "%*s", w-p, ""); + pad(f, w-p, fl); out(f, a, p); - if ((fl&LEFT_ADJ)) fprintf(f, "%*s", w-p, ""); + pad(f, w-p, fl^LEFT_ADJ); l=w; continue; case 'm': @@ -289,14 +296,14 @@ static int wprintf_core(FILE *f, const wchar_t *fmt, va_list *ap, union arg *nl_ if (p<0 && *bs) goto overflow; p=l; if (w<p) w=p; - if (!(fl&LEFT_ADJ)) fprintf(f, "%*s", w-p, ""); + pad(f, w-p, fl); bs = arg.p; while (l--) { i=mbtowc(&wc, bs, MB_LEN_MAX); bs+=i; - fputwc(wc, f); + out(f, &wc, 1); } - if ((fl&LEFT_ADJ)) fprintf(f, "%*s", w-p, ""); + pad(f, w-p, fl^LEFT_ADJ); l=w; continue; } @@ -340,8 +347,8 @@ overflow: int vfwprintf(FILE *restrict f, const wchar_t *restrict fmt, va_list ap) { va_list ap2; - int nl_type[NL_ARGMAX] = {0}; - union arg nl_arg[NL_ARGMAX]; + int nl_type[NL_ARGMAX+1] = {0}; + union arg nl_arg[NL_ARGMAX+1]; int olderr; int ret; @@ -357,7 +364,7 @@ int vfwprintf(FILE *restrict f, const wchar_t *restrict fmt, va_list ap) olderr = f->flags & F_ERR; f->flags &= ~F_ERR; ret = wprintf_core(f, fmt, &ap2, nl_arg, nl_type); - if (f->flags & F_ERR) ret = -1; + if (ferror(f)) ret = -1; f->flags |= olderr; FUNLOCK(f); va_end(ap2); diff --git a/src/stdio/vsnprintf.c b/src/stdio/vsnprintf.c index b3510a63..409b9c85 100644 --- a/src/stdio/vsnprintf.c +++ b/src/stdio/vsnprintf.c @@ -45,11 +45,6 @@ int vsnprintf(char *restrict s, size_t n, const char *restrict fmt, va_list ap) .cookie = &c, }; - if (n > INT_MAX) { - errno = EOVERFLOW; - return -1; - } - *c.s = 0; return vfprintf(&f, fmt, ap); } diff --git a/src/stdio/vsscanf.c b/src/stdio/vsscanf.c index 98500225..4d6d259b 100644 --- a/src/stdio/vsscanf.c +++ b/src/stdio/vsscanf.c @@ -1,15 +1,25 @@ #include "stdio_impl.h" +#include <string.h> -static size_t do_read(FILE *f, unsigned char *buf, size_t len) +static size_t string_read(FILE *f, unsigned char *buf, size_t len) { - return __string_read(f, buf, len); + char *src = f->cookie; + size_t k = len+256; + char *end = memchr(src, 0, k); + if (end) k = end-src; + if (k < len) len = k; + memcpy(buf, src, len); + f->rpos = (void *)(src+len); + f->rend = (void *)(src+k); + f->cookie = src+k; + return len; } int vsscanf(const char *restrict s, const char *restrict fmt, va_list ap) { FILE f = { .buf = (void *)s, .cookie = (void *)s, - .read = do_read, .lock = -1 + .read = string_read, .lock = -1 }; return vfscanf(&f, fmt, ap); } diff --git a/src/stdio/vswprintf.c b/src/stdio/vswprintf.c index 7f98c5c9..5e9a4dad 100644 --- a/src/stdio/vswprintf.c +++ b/src/stdio/vswprintf.c @@ -18,6 +18,7 @@ static size_t sw_write(FILE *f, const unsigned char *s, size_t l) if (s!=f->wbase && sw_write(f, f->wbase, f->wpos-f->wbase)==-1) return -1; while (c->l && l && (i=mbtowc(c->ws, (void *)s, l))>=0) { + if (!i) i=1; s+=i; l-=i; c->l--; @@ -50,9 +51,6 @@ int vswprintf(wchar_t *restrict s, size_t n, const wchar_t *restrict fmt, va_lis if (!n) { return -1; - } else if (n > INT_MAX) { - errno = EOVERFLOW; - return -1; } r = vfwprintf(&f, fmt, ap); sw_write(&f, 0, 0); diff --git a/src/stdlib/qsort.c b/src/stdlib/qsort.c index da58fd31..314ddc29 100644 --- a/src/stdlib/qsort.c +++ b/src/stdlib/qsort.c @@ -24,6 +24,7 @@ /* Smoothsort, an adaptive variant of Heapsort. Memory usage: O(1). Run time: Worst case O(n log n), close to O(n) in the mostly-sorted case. */ +#define _BSD_SOURCE #include <stdint.h> #include <stdlib.h> #include <string.h> @@ -31,7 +32,7 @@ #include "atomic.h" #define ntz(x) a_ctz_l((x)) -typedef int (*cmpfun)(const void *, const void *); +typedef int (*cmpfun)(const void *, const void *, void *); static inline int pntz(size_t p[2]) { int r = ntz(p[0] - 1); @@ -88,7 +89,7 @@ static inline void shr(size_t p[2], int n) p[1] >>= n; } -static void sift(unsigned char *head, size_t width, cmpfun cmp, int pshift, size_t lp[]) +static void sift(unsigned char *head, size_t width, cmpfun cmp, void *arg, int pshift, size_t lp[]) { unsigned char *rt, *lf; unsigned char *ar[14 * sizeof(size_t) + 1]; @@ -99,10 +100,10 @@ static void sift(unsigned char *head, size_t width, cmpfun cmp, int pshift, size rt = head - width; lf = head - width - lp[pshift - 2]; - if((*cmp)(ar[0], lf) >= 0 && (*cmp)(ar[0], rt) >= 0) { + if(cmp(ar[0], lf, arg) >= 0 && cmp(ar[0], rt, arg) >= 0) { break; } - if((*cmp)(lf, rt) >= 0) { + if(cmp(lf, rt, arg) >= 0) { ar[i++] = lf; head = lf; pshift -= 1; @@ -115,7 +116,7 @@ static void sift(unsigned char *head, size_t width, cmpfun cmp, int pshift, size cycle(width, ar, i); } -static void trinkle(unsigned char *head, size_t width, cmpfun cmp, size_t pp[2], int pshift, int trusty, size_t lp[]) +static void trinkle(unsigned char *head, size_t width, cmpfun cmp, void *arg, size_t pp[2], int pshift, int trusty, size_t lp[]) { unsigned char *stepson, *rt, *lf; @@ -130,13 +131,13 @@ static void trinkle(unsigned char *head, size_t width, cmpfun cmp, size_t pp[2], ar[0] = head; while(p[0] != 1 || p[1] != 0) { stepson = head - lp[pshift]; - if((*cmp)(stepson, ar[0]) <= 0) { + if(cmp(stepson, ar[0], arg) <= 0) { break; } if(!trusty && pshift > 1) { rt = head - width; lf = head - width - lp[pshift - 2]; - if((*cmp)(rt, stepson) >= 0 || (*cmp)(lf, stepson) >= 0) { + if(cmp(rt, stepson, arg) >= 0 || cmp(lf, stepson, arg) >= 0) { break; } } @@ -150,11 +151,11 @@ static void trinkle(unsigned char *head, size_t width, cmpfun cmp, size_t pp[2], } if(!trusty) { cycle(width, ar, i); - sift(head, width, cmp, pshift, lp); + sift(head, width, cmp, arg, pshift, lp); } } -void qsort(void *base, size_t nel, size_t width, cmpfun cmp) +void __qsort_r(void *base, size_t nel, size_t width, cmpfun cmp, void *arg) { size_t lp[12*sizeof(size_t)]; size_t i, size = width * nel; @@ -173,16 +174,16 @@ void qsort(void *base, size_t nel, size_t width, cmpfun cmp) while(head < high) { if((p[0] & 3) == 3) { - sift(head, width, cmp, pshift, lp); + sift(head, width, cmp, arg, pshift, lp); shr(p, 2); pshift += 2; } else { if(lp[pshift - 1] >= high - head) { - trinkle(head, width, cmp, p, pshift, 0, lp); + trinkle(head, width, cmp, arg, p, pshift, 0, lp); } else { - sift(head, width, cmp, pshift, lp); + sift(head, width, cmp, arg, pshift, lp); } - + if(pshift == 1) { shl(p, 1); pshift = 0; @@ -191,12 +192,12 @@ void qsort(void *base, size_t nel, size_t width, cmpfun cmp) pshift = 1; } } - + p[0] |= 1; head += width; } - trinkle(head, width, cmp, p, pshift, 0, lp); + trinkle(head, width, cmp, arg, p, pshift, 0, lp); while(pshift != 1 || p[0] != 1 || p[1] != 0) { if(pshift <= 1) { @@ -208,11 +209,13 @@ void qsort(void *base, size_t nel, size_t width, cmpfun cmp) pshift -= 2; p[0] ^= 7; shr(p, 1); - trinkle(head - lp[pshift] - width, width, cmp, p, pshift + 1, 1, lp); + trinkle(head - lp[pshift] - width, width, cmp, arg, p, pshift + 1, 1, lp); shl(p, 1); p[0] |= 1; - trinkle(head - width, width, cmp, p, pshift, 1, lp); + trinkle(head - width, width, cmp, arg, p, pshift, 1, lp); } head -= width; } } + +weak_alias(__qsort_r, qsort_r); diff --git a/src/stdlib/qsort_nr.c b/src/stdlib/qsort_nr.c new file mode 100644 index 00000000..8ffe71d0 --- /dev/null +++ b/src/stdlib/qsort_nr.c @@ -0,0 +1,14 @@ +#define _BSD_SOURCE +#include <stdlib.h> + +typedef int (*cmpfun)(const void *, const void *); + +static int wrapper_cmp(const void *v1, const void *v2, void *cmp) +{ + return ((cmpfun)cmp)(v1, v2); +} + +void qsort(void *base, size_t nel, size_t width, cmpfun cmp) +{ + __qsort_r(base, nel, width, wrapper_cmp, (void *)cmp); +} diff --git a/src/stdlib/strtod.c b/src/stdlib/strtod.c index a5d0118a..39b9daad 100644 --- a/src/stdlib/strtod.c +++ b/src/stdlib/strtod.c @@ -28,10 +28,3 @@ long double strtold(const char *restrict s, char **restrict p) { return strtox(s, p, 2); } - -weak_alias(strtof, strtof_l); -weak_alias(strtod, strtod_l); -weak_alias(strtold, strtold_l); -weak_alias(strtof, __strtof_l); -weak_alias(strtod, __strtod_l); -weak_alias(strtold, __strtold_l); diff --git a/src/stdlib/wcstod.c b/src/stdlib/wcstod.c index 26fe9af8..0deb7010 100644 --- a/src/stdlib/wcstod.c +++ b/src/stdlib/wcstod.c @@ -33,8 +33,7 @@ static long double wcstox(const wchar_t *s, wchar_t **p, int prec) unsigned char buf[64]; FILE f = {0}; f.flags = 0; - f.rpos = f.rend = 0; - f.buf = buf + 4; + f.rpos = f.rend = f.buf = buf + 4; f.buf_size = sizeof buf - 4; f.lock = -1; f.read = do_read; diff --git a/src/stdlib/wcstol.c b/src/stdlib/wcstol.c index 4443f577..1eeb495f 100644 --- a/src/stdlib/wcstol.c +++ b/src/stdlib/wcstol.c @@ -35,8 +35,7 @@ static unsigned long long wcstox(const wchar_t *s, wchar_t **p, int base, unsign unsigned char buf[64]; FILE f = {0}; f.flags = 0; - f.rpos = f.rend = 0; - f.buf = buf + 4; + f.rpos = f.rend = f.buf = buf + 4; f.buf_size = sizeof buf - 4; f.lock = -1; f.read = do_read; diff --git a/src/string/aarch64/memcpy.S b/src/string/aarch64/memcpy.S new file mode 100644 index 00000000..48bb8a8d --- /dev/null +++ b/src/string/aarch64/memcpy.S @@ -0,0 +1,186 @@ +/* + * memcpy - copy memory area + * + * Copyright (c) 2012-2020, Arm Limited. + * SPDX-License-Identifier: MIT + */ + +/* Assumptions: + * + * ARMv8-a, AArch64, unaligned accesses. + * + */ + +#define dstin x0 +#define src x1 +#define count x2 +#define dst x3 +#define srcend x4 +#define dstend x5 +#define A_l x6 +#define A_lw w6 +#define A_h x7 +#define B_l x8 +#define B_lw w8 +#define B_h x9 +#define C_l x10 +#define C_lw w10 +#define C_h x11 +#define D_l x12 +#define D_h x13 +#define E_l x14 +#define E_h x15 +#define F_l x16 +#define F_h x17 +#define G_l count +#define G_h dst +#define H_l src +#define H_h srcend +#define tmp1 x14 + +/* This implementation of memcpy uses unaligned accesses and branchless + sequences to keep the code small, simple and improve performance. + + Copies are split into 3 main cases: small copies of up to 32 bytes, medium + copies of up to 128 bytes, and large copies. The overhead of the overlap + check is negligible since it is only required for large copies. + + Large copies use a software pipelined loop processing 64 bytes per iteration. + The destination pointer is 16-byte aligned to minimize unaligned accesses. + The loop tail is handled by always copying 64 bytes from the end. +*/ + +.global memcpy +.type memcpy,%function +memcpy: + add srcend, src, count + add dstend, dstin, count + cmp count, 128 + b.hi .Lcopy_long + cmp count, 32 + b.hi .Lcopy32_128 + + /* Small copies: 0..32 bytes. */ + cmp count, 16 + b.lo .Lcopy16 + ldp A_l, A_h, [src] + ldp D_l, D_h, [srcend, -16] + stp A_l, A_h, [dstin] + stp D_l, D_h, [dstend, -16] + ret + + /* Copy 8-15 bytes. */ +.Lcopy16: + tbz count, 3, .Lcopy8 + ldr A_l, [src] + ldr A_h, [srcend, -8] + str A_l, [dstin] + str A_h, [dstend, -8] + ret + + .p2align 3 + /* Copy 4-7 bytes. */ +.Lcopy8: + tbz count, 2, .Lcopy4 + ldr A_lw, [src] + ldr B_lw, [srcend, -4] + str A_lw, [dstin] + str B_lw, [dstend, -4] + ret + + /* Copy 0..3 bytes using a branchless sequence. */ +.Lcopy4: + cbz count, .Lcopy0 + lsr tmp1, count, 1 + ldrb A_lw, [src] + ldrb C_lw, [srcend, -1] + ldrb B_lw, [src, tmp1] + strb A_lw, [dstin] + strb B_lw, [dstin, tmp1] + strb C_lw, [dstend, -1] +.Lcopy0: + ret + + .p2align 4 + /* Medium copies: 33..128 bytes. */ +.Lcopy32_128: + ldp A_l, A_h, [src] + ldp B_l, B_h, [src, 16] + ldp C_l, C_h, [srcend, -32] + ldp D_l, D_h, [srcend, -16] + cmp count, 64 + b.hi .Lcopy128 + stp A_l, A_h, [dstin] + stp B_l, B_h, [dstin, 16] + stp C_l, C_h, [dstend, -32] + stp D_l, D_h, [dstend, -16] + ret + + .p2align 4 + /* Copy 65..128 bytes. */ +.Lcopy128: + ldp E_l, E_h, [src, 32] + ldp F_l, F_h, [src, 48] + cmp count, 96 + b.ls .Lcopy96 + ldp G_l, G_h, [srcend, -64] + ldp H_l, H_h, [srcend, -48] + stp G_l, G_h, [dstend, -64] + stp H_l, H_h, [dstend, -48] +.Lcopy96: + stp A_l, A_h, [dstin] + stp B_l, B_h, [dstin, 16] + stp E_l, E_h, [dstin, 32] + stp F_l, F_h, [dstin, 48] + stp C_l, C_h, [dstend, -32] + stp D_l, D_h, [dstend, -16] + ret + + .p2align 4 + /* Copy more than 128 bytes. */ +.Lcopy_long: + + /* Copy 16 bytes and then align dst to 16-byte alignment. */ + + ldp D_l, D_h, [src] + and tmp1, dstin, 15 + bic dst, dstin, 15 + sub src, src, tmp1 + add count, count, tmp1 /* Count is now 16 too large. */ + ldp A_l, A_h, [src, 16] + stp D_l, D_h, [dstin] + ldp B_l, B_h, [src, 32] + ldp C_l, C_h, [src, 48] + ldp D_l, D_h, [src, 64]! + subs count, count, 128 + 16 /* Test and readjust count. */ + b.ls .Lcopy64_from_end + +.Lloop64: + stp A_l, A_h, [dst, 16] + ldp A_l, A_h, [src, 16] + stp B_l, B_h, [dst, 32] + ldp B_l, B_h, [src, 32] + stp C_l, C_h, [dst, 48] + ldp C_l, C_h, [src, 48] + stp D_l, D_h, [dst, 64]! + ldp D_l, D_h, [src, 64]! + subs count, count, 64 + b.hi .Lloop64 + + /* Write the last iteration and copy 64 bytes from the end. */ +.Lcopy64_from_end: + ldp E_l, E_h, [srcend, -64] + stp A_l, A_h, [dst, 16] + ldp A_l, A_h, [srcend, -48] + stp B_l, B_h, [dst, 32] + ldp B_l, B_h, [srcend, -32] + stp C_l, C_h, [dst, 48] + ldp C_l, C_h, [srcend, -16] + stp D_l, D_h, [dst, 64] + stp E_l, E_h, [dstend, -64] + stp A_l, A_h, [dstend, -48] + stp B_l, B_h, [dstend, -32] + stp C_l, C_h, [dstend, -16] + ret + +.size memcpy,.-memcpy diff --git a/src/string/aarch64/memset.S b/src/string/aarch64/memset.S new file mode 100644 index 00000000..f0d29b7f --- /dev/null +++ b/src/string/aarch64/memset.S @@ -0,0 +1,115 @@ +/* + * memset - fill memory with a constant byte + * + * Copyright (c) 2012-2020, Arm Limited. + * SPDX-License-Identifier: MIT + */ + +/* Assumptions: + * + * ARMv8-a, AArch64, Advanced SIMD, unaligned accesses. + * + */ + +#define dstin x0 +#define val x1 +#define valw w1 +#define count x2 +#define dst x3 +#define dstend x4 +#define zva_val x5 + +.global memset +.type memset,%function +memset: + + dup v0.16B, valw + add dstend, dstin, count + + cmp count, 96 + b.hi .Lset_long + cmp count, 16 + b.hs .Lset_medium + mov val, v0.D[0] + + /* Set 0..15 bytes. */ + tbz count, 3, 1f + str val, [dstin] + str val, [dstend, -8] + ret + nop +1: tbz count, 2, 2f + str valw, [dstin] + str valw, [dstend, -4] + ret +2: cbz count, 3f + strb valw, [dstin] + tbz count, 1, 3f + strh valw, [dstend, -2] +3: ret + + /* Set 17..96 bytes. */ +.Lset_medium: + str q0, [dstin] + tbnz count, 6, .Lset96 + str q0, [dstend, -16] + tbz count, 5, 1f + str q0, [dstin, 16] + str q0, [dstend, -32] +1: ret + + .p2align 4 + /* Set 64..96 bytes. Write 64 bytes from the start and + 32 bytes from the end. */ +.Lset96: + str q0, [dstin, 16] + stp q0, q0, [dstin, 32] + stp q0, q0, [dstend, -32] + ret + + .p2align 4 +.Lset_long: + and valw, valw, 255 + bic dst, dstin, 15 + str q0, [dstin] + cmp count, 160 + ccmp valw, 0, 0, hs + b.ne .Lno_zva + +#ifndef SKIP_ZVA_CHECK + mrs zva_val, dczid_el0 + and zva_val, zva_val, 31 + cmp zva_val, 4 /* ZVA size is 64 bytes. */ + b.ne .Lno_zva +#endif + str q0, [dst, 16] + stp q0, q0, [dst, 32] + bic dst, dst, 63 + sub count, dstend, dst /* Count is now 64 too large. */ + sub count, count, 128 /* Adjust count and bias for loop. */ + + .p2align 4 +.Lzva_loop: + add dst, dst, 64 + dc zva, dst + subs count, count, 64 + b.hi .Lzva_loop + stp q0, q0, [dstend, -64] + stp q0, q0, [dstend, -32] + ret + +.Lno_zva: + sub count, dstend, dst /* Count is 16 too large. */ + sub dst, dst, 16 /* Dst is biased by -32. */ + sub count, count, 64 + 16 /* Adjust count and bias for loop. */ +.Lno_zva_loop: + stp q0, q0, [dst, 32] + stp q0, q0, [dst, 64]! + subs count, count, 64 + b.hi .Lno_zva_loop + stp q0, q0, [dstend, -64] + stp q0, q0, [dstend, -32] + ret + +.size memset,.-memset + diff --git a/src/string/arm/memcpy_le.S b/src/string/arm/memcpy.S index 9cfbcb2a..869e3448 100644 --- a/src/string/arm/memcpy_le.S +++ b/src/string/arm/memcpy.S @@ -1,5 +1,3 @@ -#if !__ARMEB__ && !__thumb__ - /* * Copyright (C) 2008 The Android Open Source Project * All rights reserved. @@ -40,8 +38,9 @@ * This file has been modified from the original for use in musl libc. * The main changes are: addition of .type memcpy,%function to make the * code safely callable from thumb mode, adjusting the return - * instructions to be compatible with pre-thumb ARM cpus, and removal - * of prefetch code that is not compatible with older cpus. + * instructions to be compatible with pre-thumb ARM cpus, removal of + * prefetch code that is not compatible with older cpus and support for + * building as thumb 2 and big-endian. */ .syntax unified @@ -226,23 +225,45 @@ non_congruent: * becomes aligned to 32 bits (r5 = nb of words to copy for alignment) */ movs r5, r5, lsl #31 + +#if __ARMEB__ + movmi r3, r3, ror #24 + strbmi r3, [r0], #1 + movcs r3, r3, ror #24 + strbcs r3, [r0], #1 + movcs r3, r3, ror #24 + strbcs r3, [r0], #1 +#else strbmi r3, [r0], #1 movmi r3, r3, lsr #8 strbcs r3, [r0], #1 movcs r3, r3, lsr #8 strbcs r3, [r0], #1 movcs r3, r3, lsr #8 +#endif cmp r2, #4 blo partial_word_tail +#if __ARMEB__ + mov r3, r3, lsr r12 + mov r3, r3, lsl r12 +#endif + /* Align destination to 32 bytes (cache line boundary) */ 1: tst r0, #0x1c beq 2f ldr r5, [r1], #4 sub r2, r2, #4 - orr r4, r3, r5, lsl lr +#if __ARMEB__ + mov r4, r5, lsr lr + orr r4, r4, r3 + mov r3, r5, lsl r12 +#else + mov r4, r5, lsl lr + orr r4, r4, r3 mov r3, r5, lsr r12 +#endif str r4, [r0], #4 cmp r2, #4 bhs 1b @@ -268,6 +289,25 @@ loop16: ldmia r1!, { r5,r6,r7, r8,r9,r10,r11} subs r2, r2, #32 ldrhs r12, [r1], #4 +#if __ARMEB__ + orr r3, r3, r4, lsr #16 + mov r4, r4, lsl #16 + orr r4, r4, r5, lsr #16 + mov r5, r5, lsl #16 + orr r5, r5, r6, lsr #16 + mov r6, r6, lsl #16 + orr r6, r6, r7, lsr #16 + mov r7, r7, lsl #16 + orr r7, r7, r8, lsr #16 + mov r8, r8, lsl #16 + orr r8, r8, r9, lsr #16 + mov r9, r9, lsl #16 + orr r9, r9, r10, lsr #16 + mov r10, r10, lsl #16 + orr r10, r10, r11, lsr #16 + stmia r0!, {r3,r4,r5,r6, r7,r8,r9,r10} + mov r3, r11, lsl #16 +#else orr r3, r3, r4, lsl #16 mov r4, r4, lsr #16 orr r4, r4, r5, lsl #16 @@ -285,6 +325,7 @@ loop16: orr r10, r10, r11, lsl #16 stmia r0!, {r3,r4,r5,r6, r7,r8,r9,r10} mov r3, r11, lsr #16 +#endif bhs 1b b less_than_thirtytwo @@ -294,6 +335,25 @@ loop8: ldmia r1!, { r5,r6,r7, r8,r9,r10,r11} subs r2, r2, #32 ldrhs r12, [r1], #4 +#if __ARMEB__ + orr r3, r3, r4, lsr #24 + mov r4, r4, lsl #8 + orr r4, r4, r5, lsr #24 + mov r5, r5, lsl #8 + orr r5, r5, r6, lsr #24 + mov r6, r6, lsl #8 + orr r6, r6, r7, lsr #24 + mov r7, r7, lsl #8 + orr r7, r7, r8, lsr #24 + mov r8, r8, lsl #8 + orr r8, r8, r9, lsr #24 + mov r9, r9, lsl #8 + orr r9, r9, r10, lsr #24 + mov r10, r10, lsl #8 + orr r10, r10, r11, lsr #24 + stmia r0!, {r3,r4,r5,r6, r7,r8,r9,r10} + mov r3, r11, lsl #8 +#else orr r3, r3, r4, lsl #24 mov r4, r4, lsr #8 orr r4, r4, r5, lsl #24 @@ -311,6 +371,7 @@ loop8: orr r10, r10, r11, lsl #24 stmia r0!, {r3,r4,r5,r6, r7,r8,r9,r10} mov r3, r11, lsr #8 +#endif bhs 1b b less_than_thirtytwo @@ -320,6 +381,25 @@ loop24: ldmia r1!, { r5,r6,r7, r8,r9,r10,r11} subs r2, r2, #32 ldrhs r12, [r1], #4 +#if __ARMEB__ + orr r3, r3, r4, lsr #8 + mov r4, r4, lsl #24 + orr r4, r4, r5, lsr #8 + mov r5, r5, lsl #24 + orr r5, r5, r6, lsr #8 + mov r6, r6, lsl #24 + orr r6, r6, r7, lsr #8 + mov r7, r7, lsl #24 + orr r7, r7, r8, lsr #8 + mov r8, r8, lsl #24 + orr r8, r8, r9, lsr #8 + mov r9, r9, lsl #24 + orr r9, r9, r10, lsr #8 + mov r10, r10, lsl #24 + orr r10, r10, r11, lsr #8 + stmia r0!, {r3,r4,r5,r6, r7,r8,r9,r10} + mov r3, r11, lsl #24 +#else orr r3, r3, r4, lsl #8 mov r4, r4, lsr #24 orr r4, r4, r5, lsl #8 @@ -337,6 +417,7 @@ loop24: orr r10, r10, r11, lsl #8 stmia r0!, {r3,r4,r5,r6, r7,r8,r9,r10} mov r3, r11, lsr #24 +#endif bhs 1b less_than_thirtytwo: @@ -348,8 +429,15 @@ less_than_thirtytwo: 1: ldr r5, [r1], #4 sub r2, r2, #4 - orr r4, r3, r5, lsl lr +#if __ARMEB__ + mov r4, r5, lsr lr + orr r4, r4, r3 + mov r3, r5, lsl r12 +#else + mov r4, r5, lsl lr + orr r4, r4, r3 mov r3, r5, lsr r12 +#endif str r4, [r0], #4 cmp r2, #4 bhs 1b @@ -357,11 +445,20 @@ less_than_thirtytwo: partial_word_tail: /* we have a partial word in the input buffer */ movs r5, lr, lsl #(31-3) +#if __ARMEB__ + movmi r3, r3, ror #24 + strbmi r3, [r0], #1 + movcs r3, r3, ror #24 + strbcs r3, [r0], #1 + movcs r3, r3, ror #24 + strbcs r3, [r0], #1 +#else strbmi r3, [r0], #1 movmi r3, r3, lsr #8 strbcs r3, [r0], #1 movcs r3, r3, lsr #8 strbcs r3, [r0], #1 +#endif /* Refill spilled registers from the stack. Don't update sp. */ ldmfd sp, {r5-r11} @@ -380,4 +477,3 @@ copy_last_3_and_return: ldmfd sp!, {r0, r4, lr} bx lr -#endif diff --git a/src/string/arm/memcpy.c b/src/string/arm/memcpy.c deleted file mode 100644 index f703c9bd..00000000 --- a/src/string/arm/memcpy.c +++ /dev/null @@ -1,3 +0,0 @@ -#if __ARMEB__ || __thumb__ -#include "../memcpy.c" -#endif diff --git a/src/string/memccpy.c b/src/string/memccpy.c index 00c18e2b..3b0a3700 100644 --- a/src/string/memccpy.c +++ b/src/string/memccpy.c @@ -29,6 +29,6 @@ void *memccpy(void *restrict dest, const void *restrict src, int c, size_t n) #endif for (; n && (*d=*s)!=c; n--, s++, d++); tail: - if (n && *s==c) return d+1; + if (n) return d+1; return 0; } diff --git a/src/string/memmem.c b/src/string/memmem.c index 58a21fcd..11eff86e 100644 --- a/src/string/memmem.c +++ b/src/string/memmem.c @@ -12,8 +12,8 @@ static char *twobyte_memmem(const unsigned char *h, size_t k, const unsigned cha static char *threebyte_memmem(const unsigned char *h, size_t k, const unsigned char *n) { - uint32_t nw = n[0]<<24 | n[1]<<16 | n[2]<<8; - uint32_t hw = h[0]<<24 | h[1]<<16 | h[2]<<8; + uint32_t nw = (uint32_t)n[0]<<24 | n[1]<<16 | n[2]<<8; + uint32_t hw = (uint32_t)h[0]<<24 | h[1]<<16 | h[2]<<8; for (h+=3, k-=3; k; k--, hw = (hw|*h++)<<8) if (hw == nw) return (char *)h-3; return hw == nw ? (char *)h-3 : 0; @@ -21,8 +21,8 @@ static char *threebyte_memmem(const unsigned char *h, size_t k, const unsigned c static char *fourbyte_memmem(const unsigned char *h, size_t k, const unsigned char *n) { - uint32_t nw = n[0]<<24 | n[1]<<16 | n[2]<<8 | n[3]; - uint32_t hw = h[0]<<24 | h[1]<<16 | h[2]<<8 | h[3]; + uint32_t nw = (uint32_t)n[0]<<24 | n[1]<<16 | n[2]<<8 | n[3]; + uint32_t hw = (uint32_t)h[0]<<24 | h[1]<<16 | h[2]<<8 | h[3]; for (h+=4, k-=4; k; k--, hw = hw<<8 | *h++) if (hw == nw) return (char *)h-4; return hw == nw ? (char *)h-4 : 0; diff --git a/src/string/strsignal.c b/src/string/strsignal.c index 96bfe841..5156366e 100644 --- a/src/string/strsignal.c +++ b/src/string/strsignal.c @@ -31,7 +31,11 @@ static const char map[] = { [SIGPIPE] = 13, [SIGALRM] = 14, [SIGTERM] = 15, +#if defined(SIGSTKFLT) [SIGSTKFLT] = 16, +#elif defined(SIGEMT) + [SIGEMT] = 16, +#endif [SIGCHLD] = 17, [SIGCONT] = 18, [SIGSTOP] = 19, @@ -70,7 +74,13 @@ static const char strings[] = "Broken pipe\0" "Alarm clock\0" "Terminated\0" +#if defined(SIGSTKFLT) "Stack fault\0" +#elif defined(SIGEMT) + "Emulator trap\0" +#else + "Unknown signal\0" +#endif "Child process status\0" "Continued\0" "Stopped (signal)\0" diff --git a/src/string/strstr.c b/src/string/strstr.c index 55ba1c7b..96657bc2 100644 --- a/src/string/strstr.c +++ b/src/string/strstr.c @@ -10,16 +10,16 @@ static char *twobyte_strstr(const unsigned char *h, const unsigned char *n) static char *threebyte_strstr(const unsigned char *h, const unsigned char *n) { - uint32_t nw = n[0]<<24 | n[1]<<16 | n[2]<<8; - uint32_t hw = h[0]<<24 | h[1]<<16 | h[2]<<8; + uint32_t nw = (uint32_t)n[0]<<24 | n[1]<<16 | n[2]<<8; + uint32_t hw = (uint32_t)h[0]<<24 | h[1]<<16 | h[2]<<8; for (h+=2; *h && hw != nw; hw = (hw|*++h)<<8); return *h ? (char *)h-2 : 0; } static char *fourbyte_strstr(const unsigned char *h, const unsigned char *n) { - uint32_t nw = n[0]<<24 | n[1]<<16 | n[2]<<8 | n[3]; - uint32_t hw = h[0]<<24 | h[1]<<16 | h[2]<<8 | h[3]; + uint32_t nw = (uint32_t)n[0]<<24 | n[1]<<16 | n[2]<<8 | n[3]; + uint32_t hw = (uint32_t)h[0]<<24 | h[1]<<16 | h[2]<<8 | h[3]; for (h+=3; *h && hw != nw; hw = hw<<8 | *++h); return *h ? (char *)h-3 : 0; } @@ -96,7 +96,7 @@ static char *twoway_strstr(const unsigned char *h, const unsigned char *n) for (;;) { /* Update incremental end-of-haystack pointer */ if (z-h < l) { - /* Fast estimate for MIN(l,63) */ + /* Fast estimate for MAX(l,63) */ size_t grow = l | 63; const unsigned char *z2 = memchr(z, 0, grow); if (z2) { diff --git a/src/string/strverscmp.c b/src/string/strverscmp.c index 4daf276d..16c1da22 100644 --- a/src/string/strverscmp.c +++ b/src/string/strverscmp.c @@ -18,9 +18,9 @@ int strverscmp(const char *l0, const char *r0) else if (c!='0') z=0; } - if (l[dp]!='0' && r[dp]!='0') { - /* If we're not looking at a digit sequence that began - * with a zero, longest digit string is greater. */ + if (l[dp]-'1'<9U && r[dp]-'1'<9U) { + /* If we're looking at non-degenerate digit sequences starting + * with nonzero digits, longest digit string is greater. */ for (j=i; isdigit(l[j]); j++) if (!isdigit(r[j])) return 1; if (isdigit(r[j])) return -1; diff --git a/src/string/wcscmp.c b/src/string/wcscmp.c index 26eeee70..286ec3ea 100644 --- a/src/string/wcscmp.c +++ b/src/string/wcscmp.c @@ -3,5 +3,5 @@ int wcscmp(const wchar_t *l, const wchar_t *r) { for (; *l==*r && *l && *r; l++, r++); - return *l - *r; + return *l < *r ? -1 : *l > *r; } diff --git a/src/string/wcsncmp.c b/src/string/wcsncmp.c index 4ab32a92..2b3558bf 100644 --- a/src/string/wcsncmp.c +++ b/src/string/wcsncmp.c @@ -3,5 +3,5 @@ int wcsncmp(const wchar_t *l, const wchar_t *r, size_t n) { for (; n && *l==*r && *l && *r; n--, l++, r++); - return n ? *l - *r : 0; + return n ? (*l < *r ? -1 : *l > *r) : 0; } diff --git a/src/string/wmemcmp.c b/src/string/wmemcmp.c index 2a193263..717d77b1 100644 --- a/src/string/wmemcmp.c +++ b/src/string/wmemcmp.c @@ -3,5 +3,5 @@ int wmemcmp(const wchar_t *l, const wchar_t *r, size_t n) { for (; n && *l==*r; n--, l++, r++); - return n ? *l-*r : 0; + return n ? (*l < *r ? -1 : *l > *r) : 0; } diff --git a/src/temp/__randname.c b/src/temp/__randname.c index 2bce37a0..e9b970f1 100644 --- a/src/temp/__randname.c +++ b/src/temp/__randname.c @@ -1,5 +1,6 @@ #include <time.h> #include <stdint.h> +#include "pthread_impl.h" /* This assumes that a check for the template size has already been made */ @@ -10,7 +11,7 @@ char *__randname(char *template) unsigned long r; __clock_gettime(CLOCK_REALTIME, &ts); - r = ts.tv_nsec*65537 ^ (uintptr_t)&ts / 16 + (uintptr_t)template; + r = ts.tv_sec + ts.tv_nsec + __pthread_self()->tid * 65537UL; for (i=0; i<6; i++, r>>=5) template[i] = 'A'+(r&15)+(r&16)*2; diff --git a/src/temp/mkostemp.c b/src/temp/mkostemp.c index d8dcb805..e3dfdd91 100644 --- a/src/temp/mkostemp.c +++ b/src/temp/mkostemp.c @@ -5,5 +5,3 @@ int mkostemp(char *template, int flags) { return __mkostemps(template, 0, flags); } - -weak_alias(mkostemp, mkostemp64); diff --git a/src/temp/mkostemps.c b/src/temp/mkostemps.c index ef24eeae..093d2380 100644 --- a/src/temp/mkostemps.c +++ b/src/temp/mkostemps.c @@ -26,4 +26,3 @@ int __mkostemps(char *template, int len, int flags) } weak_alias(__mkostemps, mkostemps); -weak_alias(__mkostemps, mkostemps64); diff --git a/src/temp/mkstemp.c b/src/temp/mkstemp.c index 166b8afe..76c835bb 100644 --- a/src/temp/mkstemp.c +++ b/src/temp/mkstemp.c @@ -4,5 +4,3 @@ int mkstemp(char *template) { return __mkostemps(template, 0, 0); } - -weak_alias(mkstemp, mkstemp64); diff --git a/src/temp/mkstemps.c b/src/temp/mkstemps.c index 6b7531b5..f8eabfec 100644 --- a/src/temp/mkstemps.c +++ b/src/temp/mkstemps.c @@ -5,5 +5,3 @@ int mkstemps(char *template, int len) { return __mkostemps(template, len, 0); } - -weak_alias(mkstemps, mkstemps64); diff --git a/src/termios/tcgetwinsize.c b/src/termios/tcgetwinsize.c new file mode 100644 index 00000000..9b3a65a4 --- /dev/null +++ b/src/termios/tcgetwinsize.c @@ -0,0 +1,8 @@ +#include <termios.h> +#include <sys/ioctl.h> +#include "syscall.h" + +int tcgetwinsize(int fd, struct winsize *wsz) +{ + return syscall(SYS_ioctl, fd, TIOCGWINSZ, wsz); +} diff --git a/src/termios/tcsetwinsize.c b/src/termios/tcsetwinsize.c new file mode 100644 index 00000000..e01d0e25 --- /dev/null +++ b/src/termios/tcsetwinsize.c @@ -0,0 +1,8 @@ +#include <termios.h> +#include <sys/ioctl.h> +#include "syscall.h" + +int tcsetwinsize(int fd, const struct winsize *wsz) +{ + return syscall(SYS_ioctl, fd, TIOCSWINSZ, wsz); +} diff --git a/src/thread/__lock.c b/src/thread/__lock.c index 45557c88..60eece49 100644 --- a/src/thread/__lock.c +++ b/src/thread/__lock.c @@ -18,9 +18,11 @@ void __lock(volatile int *l) { - if (!libc.threads_minus_1) return; + int need_locks = libc.need_locks; + if (!need_locks) return; /* fast path: INT_MIN for the lock, +1 for the congestion */ int current = a_cas(l, 0, INT_MIN + 1); + if (need_locks < 0) libc.need_locks = 0; if (!current) return; /* A first spin loop, for medium congestion. */ for (unsigned i = 0; i < 10; ++i) { diff --git a/src/thread/i386/__set_thread_area.s b/src/thread/i386/__set_thread_area.s index c2c21dd5..aa6852be 100644 --- a/src/thread/i386/__set_thread_area.s +++ b/src/thread/i386/__set_thread_area.s @@ -28,6 +28,7 @@ __set_thread_area: ret 2: mov %ebx,%ecx + xor %eax,%eax xor %ebx,%ebx xor %edx,%edx mov %ebx,(%esp) diff --git a/src/thread/loongarch64/__set_thread_area.s b/src/thread/loongarch64/__set_thread_area.s new file mode 100644 index 00000000..021307fc --- /dev/null +++ b/src/thread/loongarch64/__set_thread_area.s @@ -0,0 +1,7 @@ +.global __set_thread_area +.hidden __set_thread_area +.type __set_thread_area,@function +__set_thread_area: + move $tp, $a0 + move $a0, $zero + jr $ra diff --git a/src/thread/loongarch64/__unmapself.s b/src/thread/loongarch64/__unmapself.s new file mode 100644 index 00000000..719ad056 --- /dev/null +++ b/src/thread/loongarch64/__unmapself.s @@ -0,0 +1,7 @@ +.global __unmapself +.type __unmapself, @function +__unmapself: + li.d $a7, 215 # call munmap + syscall 0 + li.d $a7, 93 # call exit + syscall 0 diff --git a/src/thread/loongarch64/clone.s b/src/thread/loongarch64/clone.s new file mode 100644 index 00000000..a165b365 --- /dev/null +++ b/src/thread/loongarch64/clone.s @@ -0,0 +1,29 @@ +#__clone(func, stack, flags, arg, ptid, tls, ctid) +# a0, a1, a2, a3, a4, a5, a6 +# sys_clone(flags, stack, ptid, ctid, tls) +# a0, a1, a2, a3, a4 + +.global __clone +.hidden __clone +.type __clone,@function +__clone: + bstrins.d $a1, $zero, 3, 0 #stack to 16 align + # Save function pointer and argument pointer on new thread stack + addi.d $a1, $a1, -16 + st.d $a0, $a1, 0 # save function pointer + st.d $a3, $a1, 8 # save argument pointer + or $a0, $a2, $zero + or $a2, $a4, $zero + or $a3, $a6, $zero + or $a4, $a5, $zero + ori $a7, $zero, 220 + syscall 0 # call clone + + beqz $a0, 1f # whether child process + jirl $zero, $ra, 0 # parent process return +1: + ld.d $t8, $sp, 0 # function pointer + ld.d $a0, $sp, 8 # argument pointer + jirl $ra, $t8, 0 # call the user's function + ori $a7, $zero, 93 + syscall 0 # child process exit diff --git a/src/thread/loongarch64/syscall_cp.s b/src/thread/loongarch64/syscall_cp.s new file mode 100644 index 00000000..c057a97b --- /dev/null +++ b/src/thread/loongarch64/syscall_cp.s @@ -0,0 +1,29 @@ +.global __cp_begin +.hidden __cp_begin +.global __cp_end +.hidden __cp_end +.global __cp_cancel +.hidden __cp_cancel +.hidden __cancel +.global __syscall_cp_asm +.hidden __syscall_cp_asm +.type __syscall_cp_asm,@function + +__syscall_cp_asm: +__cp_begin: + ld.w $a0, $a0, 0 + bnez $a0, __cp_cancel + move $t8, $a1 # reserve system call number + move $a0, $a2 + move $a1, $a3 + move $a2, $a4 + move $a3, $a5 + move $a4, $a6 + move $a5, $a7 + move $a7, $t8 + syscall 0 +__cp_end: + jr $ra +__cp_cancel: + la.local $t8, __cancel + jr $t8 diff --git a/src/thread/pthread_atfork.c b/src/thread/pthread_atfork.c index 76497401..26d32543 100644 --- a/src/thread/pthread_atfork.c +++ b/src/thread/pthread_atfork.c @@ -1,7 +1,13 @@ #include <pthread.h> +#include <errno.h> #include "libc.h" #include "lock.h" +#define malloc __libc_malloc +#define calloc undef +#define realloc undef +#define free undef + static struct atfork_funcs { void (*prepare)(void); void (*parent)(void); @@ -34,7 +40,7 @@ void __fork_handler(int who) int pthread_atfork(void (*prepare)(void), void (*parent)(void), void (*child)(void)) { struct atfork_funcs *new = malloc(sizeof *new); - if (!new) return -1; + if (!new) return ENOMEM; LOCK(lock); new->next = funcs; diff --git a/src/thread/pthread_attr_get.c b/src/thread/pthread_attr_get.c index 4aa5afdb..f12ff442 100644 --- a/src/thread/pthread_attr_get.c +++ b/src/thread/pthread_attr_get.c @@ -70,7 +70,7 @@ int pthread_condattr_getpshared(const pthread_condattr_t *restrict a, int *restr int pthread_mutexattr_getprotocol(const pthread_mutexattr_t *restrict a, int *restrict protocol) { - *protocol = PTHREAD_PRIO_NONE; + *protocol = a->__attr / 8U % 2; return 0; } int pthread_mutexattr_getpshared(const pthread_mutexattr_t *restrict a, int *restrict pshared) diff --git a/src/thread/pthread_cancel.c b/src/thread/pthread_cancel.c index 2f9d5e97..139a6fc8 100644 --- a/src/thread/pthread_cancel.c +++ b/src/thread/pthread_cancel.c @@ -56,7 +56,12 @@ static void cancel_handler(int sig, siginfo_t *si, void *ctx) _sigaddset(&uc->uc_sigmask, SIGCANCEL); - if (self->cancelasync || pc >= (uintptr_t)__cp_begin && pc < (uintptr_t)__cp_end) { + if (self->cancelasync) { + pthread_sigmask(SIG_SETMASK, &uc->uc_sigmask, 0); + __cancel(); + } + + if (pc >= (uintptr_t)__cp_begin && pc < (uintptr_t)__cp_end) { uc->uc_mcontext.MC_PC = (uintptr_t)__cp_cancel; #ifdef CANCEL_GOT uc->uc_mcontext.MC_GOT = CANCEL_GOT; @@ -77,7 +82,7 @@ void __testcancel() static void init_cancellation() { struct sigaction sa = { - .sa_flags = SA_SIGINFO | SA_RESTART, + .sa_flags = SA_SIGINFO | SA_RESTART | SA_ONSTACK, .sa_sigaction = cancel_handler }; memset(&sa.sa_mask, -1, _NSIG/8); diff --git a/src/thread/pthread_cond_timedwait.c b/src/thread/pthread_cond_timedwait.c index d1501240..6b761455 100644 --- a/src/thread/pthread_cond_timedwait.c +++ b/src/thread/pthread_cond_timedwait.c @@ -146,14 +146,18 @@ relock: if (oldstate == WAITING) goto done; - if (!node.next) a_inc(&m->_m_waiters); + if (!node.next && !(m->_m_type & 8)) + a_inc(&m->_m_waiters); /* Unlock the barrier that's holding back the next waiter, and * either wake it or requeue it to the mutex. */ - if (node.prev) - unlock_requeue(&node.prev->barrier, &m->_m_lock, m->_m_type & 128); - else - a_dec(&m->_m_waiters); + if (node.prev) { + int val = m->_m_lock; + if (val>0) a_cas(&m->_m_lock, val, val|0x80000000); + unlock_requeue(&node.prev->barrier, &m->_m_lock, m->_m_type & (8|128)); + } else if (!(m->_m_type & 8)) { + a_dec(&m->_m_waiters); + } /* Since a signal was consumed, cancellation is not permitted. */ if (e == ECANCELED) e = 0; diff --git a/src/thread/pthread_create.c b/src/thread/pthread_create.c index 5f491092..087f6206 100644 --- a/src/thread/pthread_create.c +++ b/src/thread/pthread_create.c @@ -69,15 +69,29 @@ _Noreturn void __pthread_exit(void *result) __pthread_tsd_run_dtors(); + __block_app_sigs(&set); + + /* This atomic potentially competes with a concurrent pthread_detach + * call; the loser is responsible for freeing thread resources. */ + int state = a_cas(&self->detach_state, DT_JOINABLE, DT_EXITING); + + if (state==DT_DETACHED && self->map_base) { + /* Since __unmapself bypasses the normal munmap code path, + * explicitly wait for vmlock holders first. This must be + * done before any locks are taken, to avoid lock ordering + * issues that could lead to deadlock. */ + __vm_wait(); + } + /* Access to target the exiting thread with syscalls that use * its kernel tid is controlled by killlock. For detached threads, * any use past this point would have undefined behavior, but for - * joinable threads it's a valid usage that must be handled. */ + * joinable threads it's a valid usage that must be handled. + * Signals must be blocked since pthread_kill must be AS-safe. */ LOCK(self->killlock); - /* The thread list lock must be AS-safe, and thus requires - * application signals to be blocked before it can be taken. */ - __block_app_sigs(&set); + /* The thread list lock must be AS-safe, and thus depends on + * application signals being blocked above. */ __tl_lock(); /* If this is the only thread in the list, don't proceed with @@ -85,19 +99,23 @@ _Noreturn void __pthread_exit(void *result) * signal state to prepare for exit to call atexit handlers. */ if (self->next == self) { __tl_unlock(); - __restore_sigs(&set); UNLOCK(self->killlock); + self->detach_state = state; + __restore_sigs(&set); exit(0); } - /* At this point we are committed to thread termination. Unlink - * the thread from the list. This change will not be visible - * until the lock is released, which only happens after SYS_exit - * has been called, via the exit futex address pointing at the lock. */ - libc.threads_minus_1--; - self->next->prev = self->prev; - self->prev->next = self->next; - self->prev = self->next = self; + /* At this point we are committed to thread termination. */ + + /* After the kernel thread exits, its tid may be reused. Clear it + * to prevent inadvertent use and inform functions that would use + * it that it's no longer available. At this point the killlock + * may be released, since functions that use it will consistently + * see the thread as having exited. Release it now so that no + * remaining locks (except thread list) are held if we end up + * resetting need_locks below. */ + self->tid = 0; + UNLOCK(self->killlock); /* Process robust list in userspace to handle non-pshared mutexes * and the detached thread case where the robust list head will @@ -121,9 +139,15 @@ _Noreturn void __pthread_exit(void *result) __do_orphaned_stdio_locks(); __dl_thread_cleanup(); - /* This atomic potentially competes with a concurrent pthread_detach - * call; the loser is responsible for freeing thread resources. */ - int state = a_cas(&self->detach_state, DT_JOINABLE, DT_EXITING); + /* Last, unlink thread from the list. This change will not be visible + * until the lock is released, which only happens after SYS_exit + * has been called, via the exit futex address pointing at the lock. + * This needs to happen after any possible calls to LOCK() that might + * skip locking if process appears single-threaded. */ + if (!--libc.threads_minus_1) libc.need_locks = -1; + self->next->prev = self->prev; + self->prev->next = self->next; + self->prev = self->next = self; if (state==DT_DETACHED && self->map_base) { /* Detached threads must block even implementation-internal @@ -136,24 +160,15 @@ _Noreturn void __pthread_exit(void *result) if (self->robust_list.off) __syscall(SYS_set_robust_list, 0, 3*sizeof(long)); - /* Since __unmapself bypasses the normal munmap code path, - * explicitly wait for vmlock holders first. */ - __vm_wait(); - /* The following call unmaps the thread's stack mapping * and then exits without touching the stack. */ __unmapself(self->map_base, self->map_size); } /* Wake any joiner. */ + a_store(&self->detach_state, DT_EXITED); __wake(&self->detach_state, 1, 1); - /* After the kernel thread exits, its tid may be reused. Clear it - * to prevent inadvertent use and inform functions that would use - * it that it's no longer available. */ - self->tid = 0; - UNLOCK(self->killlock); - for (;;) __syscall(SYS_exit, 0); } @@ -310,7 +325,7 @@ int __pthread_create(pthread_t *restrict res, const pthread_attr_t *restrict att new->detach_state = DT_JOINABLE; } new->robust_list.head = &new->robust_list.head; - new->CANARY = self->CANARY; + new->canary = self->canary; new->sysinfo = self->sysinfo; /* Setup argument structure for the new thread on its stack. @@ -336,7 +351,7 @@ int __pthread_create(pthread_t *restrict res, const pthread_attr_t *restrict att ~(1UL<<((SIGCANCEL-1)%(8*sizeof(long)))); __tl_lock(); - libc.threads_minus_1++; + if (!libc.threads_minus_1++) libc.need_locks = 1; ret = __clone((c11 ? start_c11 : start), stack, flags, args, &new->tid, TP_ADJ(new), &__thread_list_lock); /* All clone failures translate to EAGAIN. If explicit scheduling @@ -360,7 +375,7 @@ int __pthread_create(pthread_t *restrict res, const pthread_attr_t *restrict att new->next->prev = new; new->prev->next = new; } else { - libc.threads_minus_1--; + if (!--libc.threads_minus_1) libc.need_locks = 0; } __tl_unlock(); __restore_sigs(&set); diff --git a/src/thread/pthread_detach.c b/src/thread/pthread_detach.c index 77772af2..d73a500e 100644 --- a/src/thread/pthread_detach.c +++ b/src/thread/pthread_detach.c @@ -5,8 +5,12 @@ static int __pthread_detach(pthread_t t) { /* If the cas fails, detach state is either already-detached * or exiting/exited, and pthread_join will trap or cleanup. */ - if (a_cas(&t->detach_state, DT_JOINABLE, DT_DETACHED) != DT_JOINABLE) - return __pthread_join(t, 0); + if (a_cas(&t->detach_state, DT_JOINABLE, DT_DETACHED) != DT_JOINABLE) { + int cs; + __pthread_setcancelstate(PTHREAD_CANCEL_DISABLE, &cs); + __pthread_join(t, 0); + __pthread_setcancelstate(cs, 0); + } return 0; } diff --git a/src/thread/pthread_getname_np.c b/src/thread/pthread_getname_np.c new file mode 100644 index 00000000..85504e45 --- /dev/null +++ b/src/thread/pthread_getname_np.c @@ -0,0 +1,25 @@ +#define _GNU_SOURCE +#include <fcntl.h> +#include <unistd.h> +#include <sys/prctl.h> + +#include "pthread_impl.h" + +int pthread_getname_np(pthread_t thread, char *name, size_t len) +{ + int fd, cs, status = 0; + char f[sizeof "/proc/self/task//comm" + 3*sizeof(int)]; + + if (len < 16) return ERANGE; + + if (thread == pthread_self()) + return prctl(PR_GET_NAME, (unsigned long)name, 0UL, 0UL, 0UL) ? errno : 0; + + snprintf(f, sizeof f, "/proc/self/task/%d/comm", thread->tid); + pthread_setcancelstate(PTHREAD_CANCEL_DISABLE, &cs); + if ((fd = open(f, O_RDONLY|O_CLOEXEC)) < 0 || (len = read(fd, name, len)) == -1) status = errno; + else name[len-1] = 0; /* remove trailing new line only if successful */ + if (fd >= 0) close(fd); + pthread_setcancelstate(cs, 0); + return status; +} diff --git a/src/thread/pthread_getschedparam.c b/src/thread/pthread_getschedparam.c index 1cba073d..c098befb 100644 --- a/src/thread/pthread_getschedparam.c +++ b/src/thread/pthread_getschedparam.c @@ -4,6 +4,8 @@ int pthread_getschedparam(pthread_t t, int *restrict policy, struct sched_param *restrict param) { int r; + sigset_t set; + __block_app_sigs(&set); LOCK(t->killlock); if (!t->tid) { r = ESRCH; @@ -14,5 +16,6 @@ int pthread_getschedparam(pthread_t t, int *restrict policy, struct sched_param } } UNLOCK(t->killlock); + __restore_sigs(&set); return r; } diff --git a/src/thread/pthread_key_create.c b/src/thread/pthread_key_create.c index d1120941..39770c7a 100644 --- a/src/thread/pthread_key_create.c +++ b/src/thread/pthread_key_create.c @@ -1,4 +1,5 @@ #include "pthread_impl.h" +#include "fork_impl.h" volatile size_t __pthread_tsd_size = sizeof(void *) * PTHREAD_KEYS_MAX; void *__pthread_tsd_main[PTHREAD_KEYS_MAX] = { 0 }; @@ -20,6 +21,13 @@ static void dummy_0(void) weak_alias(dummy_0, __tl_lock); weak_alias(dummy_0, __tl_unlock); +void __pthread_key_atfork(int who) +{ + if (who<0) __pthread_rwlock_rdlock(&key_lock); + else if (!who) __pthread_rwlock_unlock(&key_lock); + else key_lock = (pthread_rwlock_t)PTHREAD_RWLOCK_INITIALIZER; +} + int __pthread_key_create(pthread_key_t *k, void (*dtor)(void *)) { pthread_t self = __pthread_self(); diff --git a/src/thread/pthread_kill.c b/src/thread/pthread_kill.c index 3d9395cb..79ddb209 100644 --- a/src/thread/pthread_kill.c +++ b/src/thread/pthread_kill.c @@ -4,9 +4,15 @@ int pthread_kill(pthread_t t, int sig) { int r; + sigset_t set; + /* Block not just app signals, but internal ones too, since + * pthread_kill is used to implement pthread_cancel, which + * must be async-cancel-safe. */ + __block_all_sigs(&set); LOCK(t->killlock); r = t->tid ? -__syscall(SYS_tkill, t->tid, sig) : (sig+0U >= _NSIG ? EINVAL : 0); UNLOCK(t->killlock); + __restore_sigs(&set); return r; } diff --git a/src/thread/pthread_mutex_destroy.c b/src/thread/pthread_mutex_destroy.c index 6d49e689..8d1bf77b 100644 --- a/src/thread/pthread_mutex_destroy.c +++ b/src/thread/pthread_mutex_destroy.c @@ -1,6 +1,10 @@ -#include <pthread.h> +#include "pthread_impl.h" int pthread_mutex_destroy(pthread_mutex_t *mutex) { + /* If the mutex being destroyed is process-shared and has nontrivial + * type (tracking ownership), it might be in the pending slot of a + * robust_list; wait for quiescence. */ + if (mutex->_m_type > 128) __vm_wait(); return 0; } diff --git a/src/thread/pthread_mutexattr_setprotocol.c b/src/thread/pthread_mutexattr_setprotocol.c index 511cc32d..8b80c1ce 100644 --- a/src/thread/pthread_mutexattr_setprotocol.c +++ b/src/thread/pthread_mutexattr_setprotocol.c @@ -1,24 +1,23 @@ #include "pthread_impl.h" #include "syscall.h" -static pthread_once_t check_pi_once; -static int check_pi_result; - -static void check_pi() -{ - volatile int lk = 0; - check_pi_result = -__syscall(SYS_futex, &lk, FUTEX_LOCK_PI, 0, 0); -} +static volatile int check_pi_result = -1; int pthread_mutexattr_setprotocol(pthread_mutexattr_t *a, int protocol) { + int r; switch (protocol) { case PTHREAD_PRIO_NONE: a->__attr &= ~8; return 0; case PTHREAD_PRIO_INHERIT: - pthread_once(&check_pi_once, check_pi); - if (check_pi_result) return check_pi_result; + r = check_pi_result; + if (r < 0) { + volatile int lk = 0; + r = -__syscall(SYS_futex, &lk, FUTEX_LOCK_PI, 0, 0); + a_store(&check_pi_result, r); + } + if (r) return r; a->__attr |= 8; return 0; case PTHREAD_PRIO_PROTECT: diff --git a/src/thread/pthread_mutexattr_setrobust.c b/src/thread/pthread_mutexattr_setrobust.c index 04db92a6..30a9ac3b 100644 --- a/src/thread/pthread_mutexattr_setrobust.c +++ b/src/thread/pthread_mutexattr_setrobust.c @@ -1,22 +1,20 @@ #include "pthread_impl.h" #include "syscall.h" -static pthread_once_t check_robust_once; -static int check_robust_result; - -static void check_robust() -{ - void *p; - size_t l; - check_robust_result = -__syscall(SYS_get_robust_list, 0, &p, &l); -} +static volatile int check_robust_result = -1; int pthread_mutexattr_setrobust(pthread_mutexattr_t *a, int robust) { if (robust > 1U) return EINVAL; if (robust) { - pthread_once(&check_robust_once, check_robust); - if (check_robust_result) return check_robust_result; + int r = check_robust_result; + if (r < 0) { + void *p; + size_t l; + r = -__syscall(SYS_get_robust_list, 0, &p, &l); + a_store(&check_robust_result, r); + } + if (r) return r; a->__attr |= 4; return 0; } diff --git a/src/thread/pthread_setname_np.c b/src/thread/pthread_setname_np.c index 82d35e17..fc2d2306 100644 --- a/src/thread/pthread_setname_np.c +++ b/src/thread/pthread_setname_np.c @@ -19,7 +19,7 @@ int pthread_setname_np(pthread_t thread, const char *name) snprintf(f, sizeof f, "/proc/self/task/%d/comm", thread->tid); pthread_setcancelstate(PTHREAD_CANCEL_DISABLE, &cs); - if ((fd = open(f, O_WRONLY)) < 0 || write(fd, name, len) < 0) status = errno; + if ((fd = open(f, O_WRONLY|O_CLOEXEC)) < 0 || write(fd, name, len) < 0) status = errno; if (fd >= 0) close(fd); pthread_setcancelstate(cs, 0); return status; diff --git a/src/thread/pthread_setschedparam.c b/src/thread/pthread_setschedparam.c index 038d13d8..76d4d45a 100644 --- a/src/thread/pthread_setschedparam.c +++ b/src/thread/pthread_setschedparam.c @@ -4,8 +4,11 @@ int pthread_setschedparam(pthread_t t, int policy, const struct sched_param *param) { int r; + sigset_t set; + __block_app_sigs(&set); LOCK(t->killlock); r = !t->tid ? ESRCH : -__syscall(SYS_sched_setscheduler, t->tid, policy, param); UNLOCK(t->killlock); + __restore_sigs(&set); return r; } diff --git a/src/thread/pthread_setschedprio.c b/src/thread/pthread_setschedprio.c index 5bf4a019..fc2e13dd 100644 --- a/src/thread/pthread_setschedprio.c +++ b/src/thread/pthread_setschedprio.c @@ -4,8 +4,11 @@ int pthread_setschedprio(pthread_t t, int prio) { int r; + sigset_t set; + __block_app_sigs(&set); LOCK(t->killlock); r = !t->tid ? ESRCH : -__syscall(SYS_sched_setparam, t->tid, &prio); UNLOCK(t->killlock); + __restore_sigs(&set); return r; } diff --git a/src/thread/riscv32/__set_thread_area.s b/src/thread/riscv32/__set_thread_area.s new file mode 100644 index 00000000..828154d2 --- /dev/null +++ b/src/thread/riscv32/__set_thread_area.s @@ -0,0 +1,6 @@ +.global __set_thread_area +.type __set_thread_area, %function +__set_thread_area: + mv tp, a0 + li a0, 0 + ret diff --git a/src/thread/riscv32/__unmapself.s b/src/thread/riscv32/__unmapself.s new file mode 100644 index 00000000..2849119c --- /dev/null +++ b/src/thread/riscv32/__unmapself.s @@ -0,0 +1,7 @@ +.global __unmapself +.type __unmapself, %function +__unmapself: + li a7, 215 # SYS_munmap + ecall + li a7, 93 # SYS_exit + ecall diff --git a/src/thread/riscv32/clone.s b/src/thread/riscv32/clone.s new file mode 100644 index 00000000..3102239d --- /dev/null +++ b/src/thread/riscv32/clone.s @@ -0,0 +1,34 @@ +# __clone(func, stack, flags, arg, ptid, tls, ctid) +# a0, a1, a2, a3, a4, a5, a6 + +# syscall(SYS_clone, flags, stack, ptid, tls, ctid) +# a7 a0, a1, a2, a3, a4 + +.global __clone +.type __clone, %function +__clone: + # Save func and arg to stack + addi a1, a1, -16 + sw a0, 0(a1) + sw a3, 4(a1) + + # Call SYS_clone + mv a0, a2 + mv a2, a4 + mv a3, a5 + mv a4, a6 + li a7, 220 # SYS_clone + ecall + + beqz a0, 1f + # Parent + ret + + # Child +1: lw a1, 0(sp) + lw a0, 4(sp) + jalr a1 + + # Exit + li a7, 93 # SYS_exit + ecall diff --git a/src/thread/riscv32/syscall_cp.s b/src/thread/riscv32/syscall_cp.s new file mode 100644 index 00000000..079d1ba0 --- /dev/null +++ b/src/thread/riscv32/syscall_cp.s @@ -0,0 +1,29 @@ +.global __cp_begin +.hidden __cp_begin +.global __cp_end +.hidden __cp_end +.global __cp_cancel +.hidden __cp_cancel +.hidden __cancel +.global __syscall_cp_asm +.hidden __syscall_cp_asm +.type __syscall_cp_asm, %function +__syscall_cp_asm: +__cp_begin: + lw t0, 0(a0) + bnez t0, __cp_cancel + + mv t0, a1 + mv a0, a2 + mv a1, a3 + mv a2, a4 + mv a3, a5 + mv a4, a6 + mv a5, a7 + lw a6, 0(sp) + mv a7, t0 + ecall +__cp_end: + ret +__cp_cancel: + tail __cancel diff --git a/src/thread/s390x/clone.s b/src/thread/s390x/clone.s index 577748ea..2125f20b 100644 --- a/src/thread/s390x/clone.s +++ b/src/thread/s390x/clone.s @@ -17,6 +17,9 @@ __clone: # if (!tid) syscall(SYS_exit, a(d)); # return tid; + # preserve call-saved register used as syscall arg + stg %r6, 48(%r15) + # create initial stack frame for new thread nill %r3, 0xfff8 aghi %r3, -160 @@ -35,6 +38,9 @@ __clone: lg %r6, 160(%r15) svc 120 + # restore call-saved register + lg %r6, 48(%r15) + # if error or if we're the parent, return ltgr %r2, %r2 bnzr %r14 diff --git a/src/thread/s390x/syscall_cp.s b/src/thread/s390x/syscall_cp.s index c1da40de..d094cbf5 100644 --- a/src/thread/s390x/syscall_cp.s +++ b/src/thread/s390x/syscall_cp.s @@ -14,6 +14,7 @@ __cp_begin: icm %r2, 15, 0(%r2) jne __cp_cancel + stg %r6, 48(%r15) stg %r7, 56(%r15) lgr %r1, %r3 lgr %r2, %r4 @@ -26,6 +27,7 @@ __cp_begin: __cp_end: lg %r7, 56(%r15) + lg %r6, 48(%r15) br %r14 __cp_cancel: diff --git a/src/thread/sem_getvalue.c b/src/thread/sem_getvalue.c index d9d83071..c0b7762d 100644 --- a/src/thread/sem_getvalue.c +++ b/src/thread/sem_getvalue.c @@ -1,8 +1,9 @@ #include <semaphore.h> +#include <limits.h> int sem_getvalue(sem_t *restrict sem, int *restrict valp) { int val = sem->__val[0]; - *valp = val < 0 ? 0 : val; + *valp = val & SEM_VALUE_MAX; return 0; } diff --git a/src/thread/sem_open.c b/src/thread/sem_open.c index de8555c5..0ad29de9 100644 --- a/src/thread/sem_open.c +++ b/src/thread/sem_open.c @@ -12,6 +12,12 @@ #include <stdlib.h> #include <pthread.h> #include "lock.h" +#include "fork_impl.h" + +#define malloc __libc_malloc +#define calloc __libc_calloc +#define realloc undef +#define free undef static struct { ino_t ino; @@ -19,6 +25,7 @@ static struct { int refcnt; } *semtab; static volatile int lock[1]; +volatile int *const __sem_open_lockptr = lock; #define FLAGS (O_RDWR|O_NOFOLLOW|O_CLOEXEC|O_NONBLOCK) @@ -163,10 +170,12 @@ int sem_close(sem_t *sem) int i; LOCK(lock); for (i=0; i<SEM_NSEMS_MAX && semtab[i].sem != sem; i++); - if (!--semtab[i].refcnt) { - semtab[i].sem = 0; - semtab[i].ino = 0; + if (--semtab[i].refcnt) { + UNLOCK(lock); + return 0; } + semtab[i].sem = 0; + semtab[i].ino = 0; UNLOCK(lock); munmap(sem, sizeof *sem); return 0; diff --git a/src/thread/sem_post.c b/src/thread/sem_post.c index 31e3293d..5c2517f2 100644 --- a/src/thread/sem_post.c +++ b/src/thread/sem_post.c @@ -1,17 +1,21 @@ #include <semaphore.h> +#include <limits.h> #include "pthread_impl.h" int sem_post(sem_t *sem) { - int val, waiters, priv = sem->__val[2]; + int val, new, waiters, priv = sem->__val[2]; do { val = sem->__val[0]; waiters = sem->__val[1]; - if (val == SEM_VALUE_MAX) { + if ((val & SEM_VALUE_MAX) == SEM_VALUE_MAX) { errno = EOVERFLOW; return -1; } - } while (a_cas(sem->__val, val, val+1+(val<0)) != val); - if (val<0 || waiters) __wake(sem->__val, 1, priv); + new = val + 1; + if (waiters <= 1) + new &= ~0x80000000; + } while (a_cas(sem->__val, val, new) != val); + if (val<0) __wake(sem->__val, waiters>1 ? 1 : -1, priv); return 0; } diff --git a/src/thread/sem_timedwait.c b/src/thread/sem_timedwait.c index 58d3ebfe..aa67376c 100644 --- a/src/thread/sem_timedwait.c +++ b/src/thread/sem_timedwait.c @@ -1,4 +1,5 @@ #include <semaphore.h> +#include <limits.h> #include "pthread_impl.h" static void cleanup(void *p) @@ -13,14 +14,15 @@ int sem_timedwait(sem_t *restrict sem, const struct timespec *restrict at) if (!sem_trywait(sem)) return 0; int spins = 100; - while (spins-- && sem->__val[0] <= 0 && !sem->__val[1]) a_spin(); + while (spins-- && !(sem->__val[0] & SEM_VALUE_MAX) && !sem->__val[1]) + a_spin(); while (sem_trywait(sem)) { - int r; + int r, priv = sem->__val[2]; a_inc(sem->__val+1); - a_cas(sem->__val, 0, -1); + a_cas(sem->__val, 0, 0x80000000); pthread_cleanup_push(cleanup, (void *)(sem->__val+1)); - r = __timedwait_cp(sem->__val, -1, CLOCK_REALTIME, at, sem->__val[2]); + r = __timedwait_cp(sem->__val, 0x80000000, CLOCK_REALTIME, at, priv); pthread_cleanup_pop(1); if (r) { errno = r; diff --git a/src/thread/sem_trywait.c b/src/thread/sem_trywait.c index 04edf46b..beb435da 100644 --- a/src/thread/sem_trywait.c +++ b/src/thread/sem_trywait.c @@ -1,12 +1,12 @@ #include <semaphore.h> +#include <limits.h> #include "pthread_impl.h" int sem_trywait(sem_t *sem) { int val; - while ((val=sem->__val[0]) > 0) { - int new = val-1-(val==1 && sem->__val[1]); - if (a_cas(sem->__val, val, new)==val) return 0; + while ((val=sem->__val[0]) & SEM_VALUE_MAX) { + if (a_cas(sem->__val, val, val-1)==val) return 0; } errno = EAGAIN; return -1; diff --git a/src/thread/synccall.c b/src/thread/synccall.c index 648a6ad4..38597254 100644 --- a/src/thread/synccall.c +++ b/src/thread/synccall.c @@ -11,7 +11,7 @@ weak_alias(dummy_0, __tl_unlock); static int target_tid; static void (*callback)(void *), *context; -static sem_t target_sem, caller_sem; +static sem_t target_sem, caller_sem, exit_sem; static void dummy(void *p) { @@ -33,7 +33,7 @@ static void handler(int sig) /* Inform caller we've complered the callback and wait * for the caller to release us to return. */ sem_post(&caller_sem); - sem_wait(&target_sem); + sem_wait(&exit_sem); /* Inform caller we are returning and state is destroyable. */ sem_post(&caller_sem); @@ -45,7 +45,7 @@ void __synccall(void (*func)(void *), void *ctx) { sigset_t oldmask; int cs, i, r; - struct sigaction sa = { .sa_flags = SA_RESTART, .sa_handler = handler }; + struct sigaction sa = { .sa_flags = SA_RESTART | SA_ONSTACK, .sa_handler = handler }; pthread_t self = __pthread_self(), td; int count = 0; @@ -62,8 +62,10 @@ void __synccall(void (*func)(void *), void *ctx) sem_init(&target_sem, 0, 0); sem_init(&caller_sem, 0, 0); + sem_init(&exit_sem, 0, 0); - if (!libc.threads_minus_1) goto single_threaded; + if (!libc.threads_minus_1 || __syscall(SYS_gettid) != self->tid) + goto single_threaded; callback = func; context = ctx; @@ -106,12 +108,13 @@ single_threaded: /* Only release the caught threads once all threads, including the * caller, have returned from the callback function. */ for (i=0; i<count; i++) - sem_post(&target_sem); + sem_post(&exit_sem); for (i=0; i<count; i++) sem_wait(&caller_sem); sem_destroy(&caller_sem); sem_destroy(&target_sem); + sem_destroy(&exit_sem); pthread_setcancelstate(cs, 0); __tl_unlock(); diff --git a/src/thread/vmlock.c b/src/thread/vmlock.c index 75f3cb76..fa0a8e3c 100644 --- a/src/thread/vmlock.c +++ b/src/thread/vmlock.c @@ -1,6 +1,8 @@ #include "pthread_impl.h" +#include "fork_impl.h" static volatile int vmlock[2]; +volatile int *const __vmlock_lockptr = vmlock; void __vm_wait() { diff --git a/src/time/__map_file.c b/src/time/__map_file.c index 9d376222..c2b29fe8 100644 --- a/src/time/__map_file.c +++ b/src/time/__map_file.c @@ -9,7 +9,7 @@ const char unsigned *__map_file(const char *pathname, size_t *size) const unsigned char *map = MAP_FAILED; int fd = sys_open(pathname, O_RDONLY|O_CLOEXEC|O_NONBLOCK); if (fd < 0) return 0; - if (!syscall(SYS_fstat, fd, &st)) { + if (!__fstat(fd, &st)) { map = __mmap(0, st.st_size, PROT_READ, MAP_SHARED, fd, 0); *size = st.st_size; } diff --git a/src/time/__tz.c b/src/time/__tz.c index 185642e8..c34b3eb7 100644 --- a/src/time/__tz.c +++ b/src/time/__tz.c @@ -4,8 +4,15 @@ #include <stdlib.h> #include <string.h> #include <sys/mman.h> +#include <ctype.h> #include "libc.h" #include "lock.h" +#include "fork_impl.h" + +#define malloc __libc_malloc +#define calloc undef +#define realloc undef +#define free undef long __timezone = 0; int __daylight = 0; @@ -30,6 +37,7 @@ static char *old_tz = old_tz_buf; static size_t old_tz_size = sizeof old_tz_buf; static volatile int lock[1]; +volatile int *const __timezone_lockptr = lock; static int getint(const char **p) { @@ -86,15 +94,15 @@ static void getname(char *d, const char **p) int i; if (**p == '<') { ++*p; - for (i=0; (*p)[i]!='>' && i<TZNAME_MAX; i++) - d[i] = (*p)[i]; - ++*p; + for (i=0; (*p)[i] && (*p)[i]!='>'; i++) + if (i<TZNAME_MAX) d[i] = (*p)[i]; + if ((*p)[i]) ++*p; } else { - for (i=0; ((*p)[i]|32)-'a'<26U && i<TZNAME_MAX; i++) - d[i] = (*p)[i]; + for (i=0; ((*p)[i]|32)-'a'<26U; i++) + if (i<TZNAME_MAX) d[i] = (*p)[i]; } *p += i; - d[i] = 0; + d[i<TZNAME_MAX?i:TZNAME_MAX] = 0; } #define VEC(...) ((const unsigned char[]){__VA_ARGS__}) @@ -147,10 +155,21 @@ static void do_tzset() } if (old_tz) memcpy(old_tz, s, i+1); + int posix_form = 0; + if (*s != ':') { + p = s; + char dummy_name[TZNAME_MAX+1]; + getname(dummy_name, &p); + if (p!=s && (*p == '+' || *p == '-' || isdigit(*p) + || !strcmp(dummy_name, "UTC") + || !strcmp(dummy_name, "GMT"))) + posix_form = 1; + } + /* Non-suid can use an absolute tzfile pathname or a relative * pathame beginning with "."; in secure mode, only the * standard path will be searched. */ - if (*s == ':' || ((p=strchr(s, '/')) && !memchr(s, ',', p-s))) { + if (!posix_form) { if (*s == ':') s++; if (*s == '/' || *s == '.') { if (!libc.secure || !strcmp(s, "/etc/localtime")) @@ -178,7 +197,7 @@ static void do_tzset() zi = map; if (map) { int scale = 2; - if (sizeof(time_t) > 4 && map[4]=='2') { + if (map[4]!='1') { size_t skip = zi_dotprod(zi+20, VEC(1,1,8,5,6,1), 6); trans = zi+skip+44+44; scale++; @@ -274,22 +293,20 @@ static size_t scan_trans(long long t, int local, size_t *alt) n = (index-trans)>>scale; if (a == n-1) return -1; if (a == 0) { - x = zi_read32(trans + (a<<scale)); - if (scale == 3) x = x<<32 | zi_read32(trans + (a<<scale) + 4); + x = zi_read32(trans); + if (scale == 3) x = x<<32 | zi_read32(trans + 4); else x = (int32_t)x; - if (local) off = (int32_t)zi_read32(types + 6 * index[a-1]); + /* Find the lowest non-DST type, or 0 if none. */ + size_t j = 0; + for (size_t i=abbrevs-types; i; i-=6) { + if (!types[i-6+4]) j = i-6; + } + if (local) off = (int32_t)zi_read32(types + j); + /* If t is before first transition, use the above-found type + * and the index-zero (after transition) type as the alt. */ if (t - off < (int64_t)x) { - for (a=0; a<(abbrevs-types)/6; a++) { - if (types[6*a+4] != types[4]) break; - } - if (a == (abbrevs-types)/6) a = 0; - if (types[6*a+4]) { - *alt = a; - return 0; - } else { - *alt = 0; - return a; - } + if (alt) *alt = index[0]; + return j/6; } } diff --git a/src/time/__year_to_secs.c b/src/time/__year_to_secs.c index 2824ec6d..b42f5a6d 100644 --- a/src/time/__year_to_secs.c +++ b/src/time/__year_to_secs.c @@ -10,9 +10,9 @@ long long __year_to_secs(long long year, int *is_leap) return 31536000*(y-70) + 86400*leaps; } - int cycles, centuries, leaps, rem; + int cycles, centuries, leaps, rem, dummy; - if (!is_leap) is_leap = &(int){0}; + if (!is_leap) is_leap = &dummy; cycles = (year-100) / 400; rem = (year-100) % 400; if (rem < 0) { diff --git a/src/time/clock_getcpuclockid.c b/src/time/clock_getcpuclockid.c index 8a0e2d4c..bce1e8ab 100644 --- a/src/time/clock_getcpuclockid.c +++ b/src/time/clock_getcpuclockid.c @@ -8,6 +8,7 @@ int clock_getcpuclockid(pid_t pid, clockid_t *clk) struct timespec ts; clockid_t id = (-pid-1)*8U + 2; int ret = __syscall(SYS_clock_getres, id, &ts); + if (ret == -EINVAL) ret = -ESRCH; if (ret) return -ret; *clk = id; return 0; diff --git a/src/time/clock_gettime.c b/src/time/clock_gettime.c index 3e1d0975..4d2ec22f 100644 --- a/src/time/clock_gettime.c +++ b/src/time/clock_gettime.c @@ -42,6 +42,9 @@ static int cgt_init(clockid_t clk, struct timespec *ts) p = cgt_time32_wrap; } } +#ifdef VDSO_CGT_WORKAROUND + if (!__vdsosym(VDSO_CGT32_VER, VDSO_CGT32_SYM)) p = 0; +#endif #endif int (*f)(clockid_t, struct timespec *) = (int (*)(clockid_t, struct timespec *))p; @@ -80,10 +83,12 @@ int __clock_gettime(clockid_t clk, struct timespec *ts) return __syscall_ret(r); long ts32[2]; r = __syscall(SYS_clock_gettime, clk, ts32); +#ifdef SYS_gettimeofday if (r==-ENOSYS && clk==CLOCK_REALTIME) { r = __syscall(SYS_gettimeofday, ts32, 0); ts32[1] *= 1000; } +#endif if (!r) { ts->tv_sec = ts32[0]; ts->tv_nsec = ts32[1]; @@ -92,6 +97,7 @@ int __clock_gettime(clockid_t clk, struct timespec *ts) return __syscall_ret(r); #else r = __syscall(SYS_clock_gettime, clk, ts); +#ifdef SYS_gettimeofday if (r == -ENOSYS) { if (clk == CLOCK_REALTIME) { __syscall(SYS_gettimeofday, ts, 0); @@ -100,6 +106,7 @@ int __clock_gettime(clockid_t clk, struct timespec *ts) } r = -EINVAL; } +#endif return __syscall_ret(r); #endif } diff --git a/src/time/strftime.c b/src/time/strftime.c index cc53d536..c40246db 100644 --- a/src/time/strftime.c +++ b/src/time/strftime.c @@ -3,6 +3,7 @@ #include <string.h> #include <langinfo.h> #include <locale.h> +#include <ctype.h> #include <time.h> #include <limits.h> #include "locale_impl.h" @@ -233,7 +234,12 @@ size_t __strftime_l(char *restrict s, size_t n, const char *restrict f, const st pad = 0; if (*f == '-' || *f == '_' || *f == '0') pad = *f++; if ((plus = (*f == '+'))) f++; - width = strtoul(f, &p, 10); + if (isdigit(*f)) { + width = strtoul(f, &p, 10); + } else { + width = 0; + p = (void *)f; + } if (*p == 'C' || *p == 'F' || *p == 'G' || *p == 'Y') { if (!width && p!=f) width = 1; } else { diff --git a/src/time/timer_create.c b/src/time/timer_create.c index 455d49fc..9216b3ab 100644 --- a/src/time/timer_create.c +++ b/src/time/timer_create.c @@ -2,6 +2,7 @@ #include <setjmp.h> #include <limits.h> #include "pthread_impl.h" +#include "atomic.h" struct ksigevent { union sigval sigev_value; @@ -32,19 +33,6 @@ static void cleanup_fromsig(void *p) longjmp(p, 1); } -static void timer_handler(int sig, siginfo_t *si, void *ctx) -{ -} - -static void install_handler() -{ - struct sigaction sa = { - .sa_sigaction = timer_handler, - .sa_flags = SA_SIGINFO | SA_RESTART - }; - __libc_sigaction(SIGTIMER, &sa, 0); -} - static void *start(void *arg) { pthread_t self = __pthread_self(); @@ -55,6 +43,8 @@ static void *start(void *arg) union sigval val = args->sev->sigev_value; pthread_barrier_wait(&args->b); + if (self->cancel) + return 0; for (;;) { siginfo_t si; while (sigwaitinfo(SIGTIMER_SET, &si) < 0); @@ -71,7 +61,7 @@ static void *start(void *arg) int timer_create(clockid_t clk, struct sigevent *restrict evp, timer_t *restrict res) { - static pthread_once_t once = PTHREAD_ONCE_INIT; + static volatile int init = 0; pthread_t td; pthread_attr_t attr; int r; @@ -83,11 +73,15 @@ int timer_create(clockid_t clk, struct sigevent *restrict evp, timer_t *restrict switch (evp ? evp->sigev_notify : SIGEV_SIGNAL) { case SIGEV_NONE: case SIGEV_SIGNAL: + case SIGEV_THREAD_ID: if (evp) { ksev.sigev_value = evp->sigev_value; ksev.sigev_signo = evp->sigev_signo; ksev.sigev_notify = evp->sigev_notify; - ksev.sigev_tid = 0; + if (evp->sigev_notify == SIGEV_THREAD_ID) + ksev.sigev_tid = evp->sigev_notify_thread_id; + else + ksev.sigev_tid = 0; ksevp = &ksev; } if (syscall(SYS_timer_create, clk, ksevp, &timerid) < 0) @@ -95,7 +89,11 @@ int timer_create(clockid_t clk, struct sigevent *restrict evp, timer_t *restrict *res = (void *)(intptr_t)timerid; break; case SIGEV_THREAD: - pthread_once(&once, install_handler); + if (!init) { + struct sigaction sa = { .sa_handler = SIG_DFL }; + __libc_sigaction(SIGTIMER, &sa, 0); + a_store(&init, 1); + } if (evp->sigev_notify_attributes) attr = *evp->sigev_notify_attributes; else @@ -115,10 +113,12 @@ int timer_create(clockid_t clk, struct sigevent *restrict evp, timer_t *restrict ksev.sigev_value.sival_ptr = 0; ksev.sigev_signo = SIGTIMER; - ksev.sigev_notify = 4; /* SIGEV_THREAD_ID */ + ksev.sigev_notify = SIGEV_THREAD_ID; ksev.sigev_tid = td->tid; - if (syscall(SYS_timer_create, clk, &ksev, &timerid) < 0) + if (syscall(SYS_timer_create, clk, &ksev, &timerid) < 0) { timerid = -1; + td->cancel = 1; + } td->timer_id = timerid; pthread_barrier_wait(&args.b); if (timerid < 0) return -1; diff --git a/src/unistd/close.c b/src/unistd/close.c index 5b38e019..a2105f50 100644 --- a/src/unistd/close.c +++ b/src/unistd/close.c @@ -1,5 +1,6 @@ #include <unistd.h> #include <errno.h> +#include "aio_impl.h" #include "syscall.h" static int dummy(int fd) diff --git a/src/unistd/dup3.c b/src/unistd/dup3.c index f919f791..40798bde 100644 --- a/src/unistd/dup3.c +++ b/src/unistd/dup3.c @@ -9,12 +9,14 @@ int __dup3(int old, int new, int flags) int r; #ifdef SYS_dup2 if (old==new) return __syscall_ret(-EINVAL); - if (flags & O_CLOEXEC) { + if (flags) { while ((r=__syscall(SYS_dup3, old, new, flags))==-EBUSY); if (r!=-ENOSYS) return __syscall_ret(r); + if (flags & ~O_CLOEXEC) return __syscall_ret(-EINVAL); } while ((r=__syscall(SYS_dup2, old, new))==-EBUSY); - if (flags & O_CLOEXEC) __syscall(SYS_fcntl, new, F_SETFD, FD_CLOEXEC); + if (r >= 0 && (flags & O_CLOEXEC)) + __syscall(SYS_fcntl, new, F_SETFD, FD_CLOEXEC); #else while ((r=__syscall(SYS_dup3, old, new, flags))==-EBUSY); #endif diff --git a/src/unistd/faccessat.c b/src/unistd/faccessat.c index 76bbd4c7..43052dd7 100644 --- a/src/unistd/faccessat.c +++ b/src/unistd/faccessat.c @@ -25,12 +25,17 @@ static int checker(void *p) int faccessat(int fd, const char *filename, int amode, int flag) { - if (!flag || (flag==AT_EACCESS && getuid()==geteuid() && getgid()==getegid())) - return syscall(SYS_faccessat, fd, filename, amode, flag); + if (flag) { + int ret = __syscall(SYS_faccessat2, fd, filename, amode, flag); + if (ret != -ENOSYS) return __syscall_ret(ret); + } - if (flag != AT_EACCESS) + if (flag & ~AT_EACCESS) return __syscall_ret(-EINVAL); + if (!flag || (getuid()==geteuid() && getgid()==getegid())) + return syscall(SYS_faccessat, fd, filename, amode); + char stack[1024]; sigset_t set; pid_t pid; @@ -48,7 +53,7 @@ int faccessat(int fd, const char *filename, int amode, int flag) if (pid<0 || __syscall(SYS_read, p[0], &ret, sizeof ret) != sizeof(ret)) ret = -EBUSY; __syscall(SYS_close, p[0]); - __syscall(SYS_wait4, pid, &status, __WCLONE, 0); + __sys_wait4(pid, &status, __WCLONE, 0); __restore_sigs(&set); diff --git a/src/unistd/ftruncate.c b/src/unistd/ftruncate.c index b41be0fa..54ff34bc 100644 --- a/src/unistd/ftruncate.c +++ b/src/unistd/ftruncate.c @@ -5,5 +5,3 @@ int ftruncate(int fd, off_t length) { return syscall(SYS_ftruncate, fd, __SYSCALL_LL_O(length)); } - -weak_alias(ftruncate, ftruncate64); diff --git a/src/unistd/lseek.c b/src/unistd/lseek.c index b4984f3e..f5b66682 100644 --- a/src/unistd/lseek.c +++ b/src/unistd/lseek.c @@ -12,4 +12,3 @@ off_t __lseek(int fd, off_t offset, int whence) } weak_alias(__lseek, lseek); -weak_alias(__lseek, lseek64); diff --git a/src/unistd/mipsn32/lseek.c b/src/unistd/mipsn32/lseek.c index 60e74a51..0f6cbcaa 100644 --- a/src/unistd/mipsn32/lseek.c +++ b/src/unistd/mipsn32/lseek.c @@ -17,4 +17,3 @@ off_t __lseek(int fd, off_t offset, int whence) } weak_alias(__lseek, lseek); -weak_alias(__lseek, lseek64); diff --git a/src/unistd/nice.c b/src/unistd/nice.c index 6c25c8c3..1c2295ff 100644 --- a/src/unistd/nice.c +++ b/src/unistd/nice.c @@ -1,4 +1,5 @@ #include <unistd.h> +#include <errno.h> #include <sys/resource.h> #include <limits.h> #include "syscall.h" @@ -12,5 +13,11 @@ int nice(int inc) prio += getpriority(PRIO_PROCESS, 0); if (prio > NZERO-1) prio = NZERO-1; if (prio < -NZERO) prio = -NZERO; - return setpriority(PRIO_PROCESS, 0, prio) ? -1 : prio; + if (setpriority(PRIO_PROCESS, 0, prio)) { + if (errno == EACCES) + errno = EPERM; + return -1; + } else { + return prio; + } } diff --git a/src/unistd/pipe2.c b/src/unistd/pipe2.c index f24f74fb..a096990b 100644 --- a/src/unistd/pipe2.c +++ b/src/unistd/pipe2.c @@ -8,6 +8,7 @@ int pipe2(int fd[2], int flag) if (!flag) return pipe(fd); int ret = __syscall(SYS_pipe2, fd, flag); if (ret != -ENOSYS) return __syscall_ret(ret); + if (flag & ~(O_CLOEXEC|O_NONBLOCK)) return __syscall_ret(-EINVAL); ret = pipe(fd); if (ret) return ret; if (flag & O_CLOEXEC) { diff --git a/src/unistd/pread.c b/src/unistd/pread.c index 5681b045..b03fb0ad 100644 --- a/src/unistd/pread.c +++ b/src/unistd/pread.c @@ -5,5 +5,3 @@ ssize_t pread(int fd, void *buf, size_t size, off_t ofs) { return syscall_cp(SYS_pread, fd, buf, size, __SYSCALL_LL_PRW(ofs)); } - -weak_alias(pread, pread64); diff --git a/src/unistd/preadv.c b/src/unistd/preadv.c index 8376d60f..890ab403 100644 --- a/src/unistd/preadv.c +++ b/src/unistd/preadv.c @@ -8,5 +8,3 @@ ssize_t preadv(int fd, const struct iovec *iov, int count, off_t ofs) return syscall_cp(SYS_preadv, fd, iov, count, (long)(ofs), (long)(ofs>>32)); } - -weak_alias(preadv, preadv64); diff --git a/src/unistd/pwrite.c b/src/unistd/pwrite.c index ca376576..a008b3ec 100644 --- a/src/unistd/pwrite.c +++ b/src/unistd/pwrite.c @@ -1,9 +1,18 @@ +#define _GNU_SOURCE #include <unistd.h> +#include <sys/uio.h> +#include <fcntl.h> #include "syscall.h" ssize_t pwrite(int fd, const void *buf, size_t size, off_t ofs) { + if (ofs == -1) ofs--; + int r = __syscall_cp(SYS_pwritev2, fd, + (&(struct iovec){ .iov_base = (void *)buf, .iov_len = size }), + 1, (long)(ofs), (long)(ofs>>32), RWF_NOAPPEND); + if (r != -EOPNOTSUPP && r != -ENOSYS) + return __syscall_ret(r); + if (fcntl(fd, F_GETFL) & O_APPEND) + return __syscall_ret(-EOPNOTSUPP); return syscall_cp(SYS_pwrite, fd, buf, size, __SYSCALL_LL_PRW(ofs)); } - -weak_alias(pwrite, pwrite64); diff --git a/src/unistd/pwritev.c b/src/unistd/pwritev.c index f5a612c4..44a53d85 100644 --- a/src/unistd/pwritev.c +++ b/src/unistd/pwritev.c @@ -1,12 +1,18 @@ -#define _BSD_SOURCE +#define _GNU_SOURCE #include <sys/uio.h> #include <unistd.h> +#include <fcntl.h> #include "syscall.h" ssize_t pwritev(int fd, const struct iovec *iov, int count, off_t ofs) { + if (ofs == -1) ofs--; + int r = __syscall_cp(SYS_pwritev2, fd, iov, count, + (long)(ofs), (long)(ofs>>32), RWF_NOAPPEND); + if (r != -EOPNOTSUPP && r != -ENOSYS) + return __syscall_ret(r); + if (fcntl(fd, F_GETFL) & O_APPEND) + return __syscall_ret(-EOPNOTSUPP); return syscall_cp(SYS_pwritev, fd, iov, count, (long)(ofs), (long)(ofs>>32)); } - -weak_alias(pwritev, pwritev64); diff --git a/src/unistd/readlink.c b/src/unistd/readlink.c index a152d524..32f4537f 100644 --- a/src/unistd/readlink.c +++ b/src/unistd/readlink.c @@ -4,9 +4,16 @@ ssize_t readlink(const char *restrict path, char *restrict buf, size_t bufsize) { + char dummy[1]; + if (!bufsize) { + buf = dummy; + bufsize = 1; + } #ifdef SYS_readlink - return syscall(SYS_readlink, path, buf, bufsize); + int r = __syscall(SYS_readlink, path, buf, bufsize); #else - return syscall(SYS_readlinkat, AT_FDCWD, path, buf, bufsize); + int r = __syscall(SYS_readlinkat, AT_FDCWD, path, buf, bufsize); #endif + if (buf == dummy && r > 0) r = 0; + return __syscall_ret(r); } diff --git a/src/unistd/readlinkat.c b/src/unistd/readlinkat.c index 9af45cd5..f79d3d14 100644 --- a/src/unistd/readlinkat.c +++ b/src/unistd/readlinkat.c @@ -3,5 +3,12 @@ ssize_t readlinkat(int fd, const char *restrict path, char *restrict buf, size_t bufsize) { - return syscall(SYS_readlinkat, fd, path, buf, bufsize); + char dummy[1]; + if (!bufsize) { + buf = dummy; + bufsize = 1; + } + int r = __syscall(SYS_readlinkat, fd, path, buf, bufsize); + if (buf == dummy && r > 0) r = 0; + return __syscall_ret(r); } diff --git a/src/unistd/setxid.c b/src/unistd/setxid.c index 0239f8af..a629ed4b 100644 --- a/src/unistd/setxid.c +++ b/src/unistd/setxid.c @@ -1,20 +1,19 @@ #include <unistd.h> -#include <errno.h> +#include <signal.h> #include "syscall.h" #include "libc.h" -#include "pthread_impl.h" struct ctx { int id, eid, sid; - int nr, err; + int nr, ret; }; static void do_setxid(void *p) { struct ctx *c = p; - if (c->err>0) return; - int ret = -__syscall(c->nr, c->id, c->eid, c->sid); - if (ret && !c->err) { + if (c->ret<0) return; + int ret = __syscall(c->nr, c->id, c->eid, c->sid); + if (ret && !c->ret) { /* If one thread fails to set ids after another has already * succeeded, forcibly killing the process is the only safe * thing to do. State is inconsistent and dangerous. Use @@ -22,18 +21,14 @@ static void do_setxid(void *p) __block_all_sigs(0); __syscall(SYS_kill, __syscall(SYS_getpid), SIGKILL); } - c->err = ret; + c->ret = ret; } int __setxid(int nr, int id, int eid, int sid) { - /* err is initially nonzero so that failure of the first thread does not + /* ret is initially nonzero so that failure of the first thread does not * trigger the safety kill above. */ - struct ctx c = { .nr = nr, .id = id, .eid = eid, .sid = sid, .err = -1 }; + struct ctx c = { .nr = nr, .id = id, .eid = eid, .sid = sid, .ret = 1 }; __synccall(do_setxid, &c); - if (c.err) { - if (c.err>0) errno = c.err; - return -1; - } - return 0; + return __syscall_ret(c.ret > 0 ? -EAGAIN : c.ret); } diff --git a/src/unistd/truncate.c b/src/unistd/truncate.c index 97296800..077351e1 100644 --- a/src/unistd/truncate.c +++ b/src/unistd/truncate.c @@ -5,5 +5,3 @@ int truncate(const char *path, off_t length) { return syscall(SYS_truncate, path, __SYSCALL_LL_O(length)); } - -weak_alias(truncate, truncate64); diff --git a/src/unistd/x32/lseek.c b/src/unistd/x32/lseek.c index 32636429..5f93292f 100644 --- a/src/unistd/x32/lseek.c +++ b/src/unistd/x32/lseek.c @@ -12,4 +12,3 @@ off_t __lseek(int fd, off_t offset, int whence) } weak_alias(__lseek, lseek); -weak_alias(__lseek, lseek64); |