work around broken kernel struct ipc_perm on some big endian archs

the mode member of struct ipc_perm is specified by POSIX to have type
mode_t, which is uniformly defined as unsigned int. however, Linux
defines it with type __kernel_mode_t, and defines __kernel_mode_t as
unsigned short on some archs. since there is a subsequent padding
field, treating it as a 32-bit unsigned int works on little endian
archs, but the order is backwards on big endian archs with the
erroneous definition.

since multiple archs are affected, remedy the situation with fixup
code in the affected functions (shmctl, semctl, and msgctl) rather
than repeating the same shims in syscall_arch.h for every affected
arch.

7 files changed, 79 insertions, 6 deletions

diff --git a/arch/arm/syscall_arch.h b/arch/arm/syscall_arch.h
index 4db7d152..53fb155c 100644
--- a/arch/arm/syscall_arch.h
+++ b/arch/arm/syscall_arch.h
@@ -103,3 +103,5 @@ static inline long __syscall6(long n, long a, long b, long c, long d, long e, lo
 #define VDSO_CGT_VER "LINUX_2.6"
 
 #define SYSCALL_FADVISE_6_ARG
+
+#define SYSCALL_IPC_BROKEN_MODE
diff --git a/arch/m68k/syscall_arch.h b/arch/m68k/syscall_arch.h
index 53a4256f..af79c306 100644
--- a/arch/m68k/syscall_arch.h
+++ b/arch/m68k/syscall_arch.h
@@ -88,3 +88,4 @@ static inline long __syscall6(long n, long a, long b, long c, long d, long e, lo
 }
 
 #define SYSCALL_USE_SOCKETCALL
+#define SYSCALL_IPC_BROKEN_MODE
diff --git a/arch/microblaze/syscall_arch.h b/arch/microblaze/syscall_arch.h
index 8e2de7ea..6cf631ad 100644
--- a/arch/microblaze/syscall_arch.h
+++ b/arch/microblaze/syscall_arch.h
@@ -102,3 +102,5 @@ static inline long __syscall6(long n, long a, long b, long c, long d, long e, lo
 #define SYSCALL_NO_INLINE
 
 #endif
+
+#define SYSCALL_IPC_BROKEN_MODE
diff --git a/arch/sh/syscall_arch.h b/arch/sh/syscall_arch.h
index 84758fe0..48f61d94 100644
--- a/arch/sh/syscall_arch.h
+++ b/arch/sh/syscall_arch.h
@@ -86,3 +86,5 @@ static inline long __syscall6(long n, long a, long b, long c, long d, long e, lo
 	register long r1 __asm__("r1") = f;
 	__asm_syscall(22, "r"(r3), "r"(r4), "r"(r5), "r"(r6), "r"(r7), "0"(r0), "r"(r1));
 }
+
+#define SYSCALL_IPC_BROKEN_MODE
diff --git a/src/ipc/msgctl.c b/src/ipc/msgctl.c
index 4372c719..ea9b2337 100644
--- a/src/ipc/msgctl.c
+++ b/src/ipc/msgctl.c
@@ -1,12 +1,34 @@
 #include <sys/msg.h>
+#include <endian.h>
 #include "syscall.h"
 #include "ipc.h"
 
+#if __BYTE_ORDER != __BIG_ENDIAN
+#undef SYSCALL_IPC_BROKEN_MODE
+#endif
+
 int msgctl(int q, int cmd, struct msqid_ds *buf)
 {
+#ifdef SYSCALL_IPC_BROKEN_MODE
+	struct msqid_ds tmp;
+	if (cmd == IPC_SET) {
+		tmp = *buf;
+		tmp.msg_perm.mode *= 0x10000U;
+		buf = &tmp;
+	}
+#endif
 #ifdef SYS_msgctl
-	return syscall(SYS_msgctl, q, cmd | IPC_64, buf);
+	int r = __syscall(SYS_msgctl, q, cmd | IPC_64, buf);
 #else
-	return syscall(SYS_ipc, IPCOP_msgctl, q, cmd | IPC_64, 0, buf, 0);
+	int r = __syscall(SYS_ipc, IPCOP_msgctl, q, cmd | IPC_64, 0, buf, 0);
+#endif
+#ifdef SYSCALL_IPC_BROKEN_MODE
+	if (r >= 0) switch (cmd) {
+	case IPC_STAT:
+	case MSG_STAT:
+	case MSG_STAT_ANY:
+		buf->msg_perm.mode >>= 16;
+	}
 #endif
+	return __syscall_ret(r);
 }
diff --git a/src/ipc/semctl.c b/src/ipc/semctl.c
index 673a9a8c..941e2813 100644
--- a/src/ipc/semctl.c
+++ b/src/ipc/semctl.c
@@ -1,8 +1,13 @@
 #include <sys/sem.h>
 #include <stdarg.h>
+#include <endian.h>
 #include "syscall.h"
 #include "ipc.h"
 
+#if __BYTE_ORDER != __BIG_ENDIAN
+#undef SYSCALL_IPC_BROKEN_MODE
+#endif
+
 union semun {
 	int val;
 	struct semid_ds *buf;
@@ -20,9 +25,26 @@ int semctl(int id, int num, int cmd, ...)
 		arg = va_arg(ap, union semun);
 		va_end(ap);
 	}
+#ifdef SYSCALL_IPC_BROKEN_MODE
+	struct semid_ds tmp;
+	if (cmd == IPC_SET) {
+		tmp = *arg.buf;
+		tmp.sem_perm.mode *= 0x10000U;
+		arg.buf = &tmp;
+	}
+#endif
 #ifdef SYS_semctl
-	return syscall(SYS_semctl, id, num, cmd | IPC_64, arg.buf);
+	int r = __syscall(SYS_semctl, id, num, cmd | IPC_64, arg.buf);
 #else
-	return syscall(SYS_ipc, IPCOP_semctl, id, num, cmd | IPC_64, &arg.buf);
+	int r = __syscall(SYS_ipc, IPCOP_semctl, id, num, cmd | IPC_64, &arg.buf);
+#endif
+#ifdef SYSCALL_IPC_BROKEN_MODE
+	if (r >= 0) switch (cmd) {
+	case IPC_STAT:
+	case SEM_STAT:
+	case SEM_STAT_ANY:
+		arg.buf->sem_perm.mode >>= 16;
+	}
 #endif
+	return __syscall_ret(r);
 }
diff --git a/src/ipc/shmctl.c b/src/ipc/shmctl.c
index e2879f20..c951a581 100644
--- a/src/ipc/shmctl.c
+++ b/src/ipc/shmctl.c
@@ -1,12 +1,34 @@
 #include <sys/shm.h>
+#include <endian.h>
 #include "syscall.h"
 #include "ipc.h"
 
+#if __BYTE_ORDER != __BIG_ENDIAN
+#undef SYSCALL_IPC_BROKEN_MODE
+#endif
+
 int shmctl(int id, int cmd, struct shmid_ds *buf)
 {
+#ifdef SYSCALL_IPC_BROKEN_MODE
+	struct shmid_ds tmp;
+	if (cmd == IPC_SET) {
+		tmp = *buf;
+		tmp.shm_perm.mode *= 0x10000U;
+		buf = &tmp;
+	}
+#endif
 #ifdef SYS_shmctl
-	return syscall(SYS_shmctl, id, cmd | IPC_64, buf);
+	int r = __syscall(SYS_shmctl, id, cmd | IPC_64, buf);
 #else
-	return syscall(SYS_ipc, IPCOP_shmctl, id, cmd | IPC_64, 0, buf, 0);
+	int r = __syscall(SYS_ipc, IPCOP_shmctl, id, cmd | IPC_64, 0, buf, 0);
+#endif
+#ifdef SYSCALL_IPC_BROKEN_MODE
+	if (r >= 0) switch (cmd) {
+	case IPC_STAT:
+	case SHM_STAT:
+	case SHM_STAT_ANY:
+		buf->shm_perm.mode >>= 16;
+	}
 #endif
+	return __syscall_ret(r);
 }