overhaul system() and popen() to use vfork; fix various related bugs

since we target systems without overcommit, special care should be
taken that system() and popen(), like posix_spawn(), do not fail in
processes whose commit charges are too high to allow ordinary forking.

this in turn requires special precautions to ensure that the parent
process's signal handlers do not end up running in the shared-memory
child, where they could corrupt the state of the parent process.

popen has also been updated to use pipe2, so it does not have a
fd-leak race in multi-threaded programs. since pipe2 is missing on
older kernels, (non-atomic) emulation has been added.

some silly bugs in the old code should be gone too.

2 files changed, 49 insertions, 29 deletions

diff --git a/src/process/posix_spawn.c b/src/process/posix_spawn.c
index 8a6ff6db..e8557487 100644
--- a/src/process/posix_spawn.c
+++ b/src/process/posix_spawn.c
@@ -4,6 +4,7 @@
 #include <stdint.h>
 #include <fcntl.h>
 #include "syscall.h"
+#include "pthread_impl.h"
 #include "fdop.h"
 #include "libc.h"
 
@@ -30,7 +31,7 @@ int __posix_spawnx(pid_t *restrict res, const char *restrict path,
 
 	if (!attr) attr = &dummy_attr;
 
-	sigprocmask(SIG_BLOCK, (void *)(uint64_t []){-1}, &oldmask);
+	sigprocmask(SIG_BLOCK, SIGALL_SET, &oldmask);
 
 	__acquire_ptc();
 	pid = __vfork();
@@ -43,14 +44,14 @@ int __posix_spawnx(pid_t *restrict res, const char *restrict path,
 		return 0;
 	}
 
-	for (i=1; i<=64; i++) {
+	for (i=1; i<=8*__SYSCALL_SSLEN; i++) {
 		struct sigaction sa;
-		sigaction(i, 0, &sa);
-		if (sa.sa_handler!=SIG_IGN ||
+		__libc_sigaction(i, 0, &sa);
+		if (sa.sa_handler!=SIG_DFL && (sa.sa_handler!=SIG_IGN ||
 		    ((attr->__flags & POSIX_SPAWN_SETSIGDEF)
-		     && sigismember(&attr->__def, i) )) {
+		     && sigismember(&attr->__def, i) ))) {
 			sa.sa_handler = SIG_DFL;
-			sigaction(i, &sa, 0);
+			__libc_sigaction(i, &sa, 0);
 		}
 	}
 
diff --git a/src/process/system.c b/src/process/system.c
index 0f1c07b5..c8f26008 100644
--- a/src/process/system.c
+++ b/src/process/system.c
@@ -3,43 +3,62 @@
 #include <signal.h>
 #include <sys/wait.h>
 #include <errno.h>
+#include "pthread_impl.h"
+#include "libc.h"
+
+static void dummy_0()
+{
+}
+weak_alias(dummy_0, __acquire_ptc);
+weak_alias(dummy_0, __release_ptc);
+
+pid_t __vfork(void);
 
 int system(const char *cmd)
 {
 	pid_t pid;
-	sigset_t old, new;
-	struct sigaction sa, oldint, oldquit;
-	int status;
+	sigset_t old;
+	struct sigaction sa = { .sa_handler = SIG_IGN }, oldint, oldquit;
+	int status = -1, i;
 
 	if (!cmd) return 1;
 
-	sa.sa_handler = SIG_IGN;
-	sigemptyset(&sa.sa_mask);
-	sa.sa_flags = 0;
-
 	sigaction(SIGINT, &sa, &oldint);
 	sigaction(SIGQUIT, &sa, &oldquit);
-	sigaddset(&sa.sa_mask, SIGCHLD);
-	sigprocmask(SIG_BLOCK, &new, &old);
+	sigprocmask(SIG_BLOCK, SIGALL_SET, &old);
+
+	__acquire_ptc();
+	pid = __vfork();
+	__release_ptc();
 
-	pid = fork();
-	if (pid <= 0) {
+	if (pid > 0) {
+		sigset_t new = old;
+		sigaddset(&new, SIGCHLD);
+		sigprocmask(SIG_BLOCK, &new, 0);
+		while (waitpid(pid, &status, 0) && errno == EINTR);
+	}
+
+	if (pid) {
 		sigaction(SIGINT, &oldint, NULL);
 		sigaction(SIGQUIT, &oldquit, NULL);
 		sigprocmask(SIG_SETMASK, &old, NULL);
-		if (pid == 0) {
-			execl("/bin/sh", "sh", "-c", cmd, (char *)0);
-			_exit(127);
-		}
-		return -1;
+		return status;
 	}
-	while (waitpid(pid, &status, 0) == -1)
-		if (errno != EINTR) {
-			status = -1;
-			break;
+
+	/* Before we can unblock signals in the child, all signal
+	 * handlers must be eliminated -- even implementation-internal
+	 * ones. Otherwise, a signal handler could run in the child
+	 * and clobber the parent's memory (due to vfork). */
+	for (i=1; i<=8*__SYSCALL_SSLEN; i++) {
+		struct sigaction sa;
+		__libc_sigaction(i, 0, &sa);
+		if (sa.sa_handler!=SIG_IGN && sa.sa_handler!=SIG_DFL) {
+			sa.sa_handler = SIG_DFL;
+			__libc_sigaction(i, &sa, 0);
 		}
-	sigaction(SIGINT, &oldint, NULL);
-	sigaction(SIGQUIT, &oldquit, NULL);
+	}
+
 	sigprocmask(SIG_SETMASK, &old, NULL);
-	return status;
+	execl("/bin/sh", "sh", "-c", cmd, (char *)0);
+	_exit(127);
 }