summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorRich Felker <dalias@aerifal.cx>2011-05-11 19:58:03 -0400
committerRich Felker <dalias@aerifal.cx>2011-05-11 19:58:03 -0400
commit8628eff9128d66cc69dbc301341dc55130a9817e (patch)
treef6eaf02a2b338a200303de07054f3a5414326a12
parent15b77d52c950c375514d2e8152e7112fd018e20e (diff)
downloadmusl-8628eff9128d66cc69dbc301341dc55130a9817e.tar.gz
fix the last known rounding bug in floating point printing
the observed symptom was that the code was incorrectly rounding up 1.0625 to 1.063 despite the rounding mode being round-to-nearest with ties broken by rounding to even last place. however, the code was just not right in many respects, and i'm surprised it worked as well as it did. this time i tested the values that end up in the variables round, small, and the expression round+small, and all look good.
-rw-r--r--src/stdio/vfprintf.c7
1 files changed, 4 insertions, 3 deletions
diff --git a/src/stdio/vfprintf.c b/src/stdio/vfprintf.c
index f19058d3..f13fbe10 100644
--- a/src/stdio/vfprintf.c
+++ b/src/stdio/vfprintf.c
@@ -326,9 +326,10 @@ static int fmt_fp(FILE *f, long double y, int w, int p, int fl, int t)
if (x || d+1!=z) {
long double round = CONCAT(0x1p,LDBL_MANT_DIG);
long double small;
- if (x<i/2) small=0x01p-1;
- else if (i==i/2 && d+1==z) small=0x10p-1;
- else small=0x11p-1;
+ if (*d/i & 1) round += 2;
+ if (x<i/2) small=0x0.8p0;
+ else if (x==i/2 && d+1==z) small=0x1.0p0;
+ else small=0x1.8p0;
if (pl && *prefix=='-') round*=-1, small*=-1;
*d -= x;
/* Decide whether to round by probing round+small */