fix stdio lock dependency on read-after-free not faulting

instead of using a waiters count, add a bit to the lock field
indicating that the lock may have waiters. threads which obtain the
lock after contending for it will perform a potentially-spurious wake
when they release the lock.

1 files changed, 6 insertions, 3 deletions

diff --git a/src/stdio/ftrylockfile.c b/src/stdio/ftrylockfile.c
index eb13c839..3b1d5f20 100644
--- a/src/stdio/ftrylockfile.c
+++ b/src/stdio/ftrylockfile.c
@@ -2,6 +2,8 @@
 #include "pthread_impl.h"
 #include <limits.h>
 
+#define MAYBE_WAITERS 0x40000000
+
 void __do_orphaned_stdio_locks()
 {
 	FILE *f;
@@ -22,14 +24,15 @@ int ftrylockfile(FILE *f)
 {
 	pthread_t self = __pthread_self();
 	int tid = self->tid;
-	if (f->lock == tid) {
+	int owner = f->lock;
+	if ((owner & ~MAYBE_WAITERS) == tid) {
 		if (f->lockcount == LONG_MAX)
 			return -1;
 		f->lockcount++;
 		return 0;
 	}
-	if (f->lock < 0) f->lock = 0;
-	if (f->lock || a_cas(&f->lock, 0, tid))
+	if (owner < 0) f->lock = owner = 0;
+	if (owner || a_cas(&f->lock, 0, tid))
 		return -1;
 	f->lockcount = 1;
 	f->prev_locked = 0;