qsort: fix shift UB in shl and shrHEADmaster

if shl() or shr() are called with n==8*sizeof(size_t), n is adjusted
to 0. the shift by (sizeof(size_t) * 8 - n) that then follows will
consequently shift by the width of size_t, which is UB and in practice
produces an incorrect result.

return early in this case. the bitvector p was already shifted by the
required amount.

1 files changed, 2 insertions, 0 deletions

diff --git a/src/stdlib/qsort.c b/src/stdlib/qsort.c
index e4bce9f7..28607450 100644
--- a/src/stdlib/qsort.c
+++ b/src/stdlib/qsort.c
@@ -71,6 +71,7 @@ static inline void shl(size_t p[2], int n)
 		n -= 8 * sizeof(size_t);
 		p[1] = p[0];
 		p[0] = 0;
+		if (!n) return;
 	}
 	p[1] <<= n;
 	p[1] |= p[0] >> (sizeof(size_t) * 8 - n);
@@ -83,6 +84,7 @@ static inline void shr(size_t p[2], int n)
 		n -= 8 * sizeof(size_t);
 		p[0] = p[1];
 		p[1] = 0;
+		if (!n) return;
 	}
 	p[0] >>= n;
 	p[0] |= p[1] << (sizeof(size_t) * 8 - n);