fix for broken kernel side RLIM_INFINITY on mips

On 32 bit mips the kernel uses -1UL/2 to mark RLIM_INFINITY (and
this is the definition in the userspace api), but since it is in
the middle of the valid range of limits and limits are often
compared with relational operators, various kernel side logic is
broken if larger than -1UL/2 limits are used. So we truncate the
limits to -1UL/2 in get/setrlimit and prlimit.

Even if the kernel side logic consistently treated -1UL/2 as greater
than any other limit value, there wouldn't be any clean workaround
that allowed using large limits:
* using -1UL/2 as RLIM_INFINITY in userspace would mean different
infinity value for get/setrlimt and prlimit (where infinity is always
-1ULL) and userspace logic could break easily (just like the kernel
is broken now) and more special case code would be needed for mips.
* translating -1UL/2 kernel side value to -1ULL in userspace would
mean that -1UL/2 limit cannot be set (eg. -1UL/2+1 had to be passed
to the kernel instead).

(cherry picked from commit 8258014fd1e34e942a549c88c7e022a00445c352)

5 files changed, 40 insertions, 3 deletions

diff --git a/arch/mips/syscall_arch.h b/arch/mips/syscall_arch.h
index c52976eb..e62c3398 100644
--- a/arch/mips/syscall_arch.h
+++ b/arch/mips/syscall_arch.h
@@ -5,6 +5,8 @@
 
 long (__syscall)(long, ...);
 
+#define SYSCALL_RLIM_INFINITY (-1UL/2)
+
 #ifndef __clang__
 
 #define __asm_syscall(...) do { \
diff --git a/src/internal/syscall.h b/src/internal/syscall.h
index d0fe107e..1c73d0de 100644
--- a/src/internal/syscall.h
+++ b/src/internal/syscall.h
@@ -4,6 +4,10 @@
 #include <sys/syscall.h>
 #include "syscall_arch.h"
 
+#ifndef SYSCALL_RLIM_INFINITY
+#define SYSCALL_RLIM_INFINITY (~0ULL)
+#endif
+
 #ifndef __scc
 #define __scc(X) ((long) (X))
 typedef long syscall_arg_t;
diff --git a/src/linux/prlimit.c b/src/linux/prlimit.c
index d1639cca..0fe28e10 100644
--- a/src/linux/prlimit.c
+++ b/src/linux/prlimit.c
@@ -3,9 +3,24 @@
 #include "syscall.h"
 #include "libc.h"
 
+#define FIX(x) do{ if ((x)>=SYSCALL_RLIM_INFINITY) (x)=RLIM_INFINITY; }while(0)
+
 int prlimit(pid_t pid, int resource, const struct rlimit *new_limit, struct rlimit *old_limit)
 {
-	return syscall(SYS_prlimit64, pid, resource, new_limit, old_limit);
+	struct rlimit tmp;
+	int r;
+	if (new_limit && SYSCALL_RLIM_INFINITY != RLIM_INFINITY) {
+		tmp = *new_limit;
+		FIX(tmp.rlim_cur);
+		FIX(tmp.rlim_max);
+		new_limit = &tmp;
+	}
+	r = syscall(SYS_prlimit64, pid, resource, new_limit, old_limit);
+	if (!r && old_limit && SYSCALL_RLIM_INFINITY != RLIM_INFINITY) {
+		FIX(old_limit->rlim_cur);
+		FIX(old_limit->rlim_max);
+	}
+	return r;
 }
 
 #undef prlimit64
diff --git a/src/misc/getrlimit.c b/src/misc/getrlimit.c
index b7bbd062..b073677f 100644
--- a/src/misc/getrlimit.c
+++ b/src/misc/getrlimit.c
@@ -3,16 +3,24 @@
 #include "syscall.h"
 #include "libc.h"
 
+#define FIX(x) do{ if ((x)>=SYSCALL_RLIM_INFINITY) (x)=RLIM_INFINITY; }while(0)
+
 int getrlimit(int resource, struct rlimit *rlim)
 {
 	unsigned long k_rlim[2];
 	int ret = syscall(SYS_prlimit64, 0, resource, 0, rlim);
+	if (!ret) {
+		FIX(rlim->rlim_cur);
+		FIX(rlim->rlim_max);
+	}
 	if (!ret || errno != ENOSYS)
 		return ret;
 	if (syscall(SYS_getrlimit, resource, k_rlim) < 0)
 		return -1;
 	rlim->rlim_cur = k_rlim[0] == -1UL ? RLIM_INFINITY : k_rlim[0];
 	rlim->rlim_max = k_rlim[1] == -1UL ? RLIM_INFINITY : k_rlim[1];
+	FIX(rlim->rlim_cur);
+	FIX(rlim->rlim_max);
 	return 0;
 }
 
diff --git a/src/misc/setrlimit.c b/src/misc/setrlimit.c
index ddc13e98..8a1b8cc6 100644
--- a/src/misc/setrlimit.c
+++ b/src/misc/setrlimit.c
@@ -4,14 +4,22 @@
 #include "libc.h"
 
 #define MIN(a, b) ((a)<(b) ? (a) : (b))
+#define FIX(x) do{ if ((x)>=SYSCALL_RLIM_INFINITY) (x)=RLIM_INFINITY; }while(0)
 
 int __setrlimit(int resource, const struct rlimit *rlim)
 {
 	unsigned long k_rlim[2];
+	struct rlimit tmp;
+	if (SYSCALL_RLIM_INFINITY != RLIM_INFINITY) {
+		tmp = *rlim;
+		FIX(tmp.rlim_cur);
+		FIX(tmp.rlim_max);
+		rlim = &tmp;
+	}
 	int ret = __syscall(SYS_prlimit64, 0, resource, rlim, 0);
 	if (ret != -ENOSYS) return ret;
-	k_rlim[0] = MIN(rlim->rlim_cur, -1UL);
-	k_rlim[1] = MIN(rlim->rlim_max, -1UL);
+	k_rlim[0] = MIN(rlim->rlim_cur, MIN(-1UL, SYSCALL_RLIM_INFINITY));
+	k_rlim[1] = MIN(rlim->rlim_max, MIN(-1UL, SYSCALL_RLIM_INFINITY));
 	return __syscall(SYS_setrlimit, resource, k_rlim);
 }