make fseek detect and produce an error for invalid whence arguments

this is a POSIX requirement. we previously relied on the underlying fd
(or other backend) seek operation to produce the error, but since
linux lseek now supports other seek modes (SEEK_DATA and SEEK_HOLE)
which do not interact well with stdio buffering, this is insufficient.
instead, explicitly check whence before performing any operations.

1 files changed, 7 insertions, 0 deletions

diff --git a/src/stdio/fseek.c b/src/stdio/fseek.c
index 439308f7..c07f7e95 100644
--- a/src/stdio/fseek.c
+++ b/src/stdio/fseek.c
@@ -1,7 +1,14 @@
 #include "stdio_impl.h"
+#include <errno.h>
 
 int __fseeko_unlocked(FILE *f, off_t off, int whence)
 {
+	/* Fail immediately for invalid whence argument. */
+	if (whence != SEEK_CUR && whence != SEEK_SET && whence != SEEK_END) {
+		errno = EINVAL;
+		return -1;
+	}
+
 	/* Adjust relative offset for unread data in buffer, if any. */
 	if (whence == SEEK_CUR && f->rend) off -= f->rend - f->rpos;