Make cgo psx_syscall variant crash like runtime.AllThreadsSyscall

When a syscall that yields different return values is called from
the Go psx.Syscall*() API, we want to mirror the behavior of the
native golang runtime.AllThreadsSyscall() function.

The previous inconsistency was pointed out by Lorenz Bauer in:

  https://bugzilla.kernel.org/show_bug.cgi?id=215283#c8

[I decided to defer this change until 2.63, and not include this
in the bug-fix for 215283, on the grounds it is a slight
incompatibility in runtime behavior, and wanted to give folk an
opportunity to plan for it. This new behavior enforcement will
crash an unprepared go program.]

Signed-off-by: Andrew G. Morgan <morgan@kernel.org>

9 files changed, 162 insertions, 19 deletions

diff --git a/doc/Makefile b/doc/Makefile
index e78eb6e..c7d50e0 100644
--- a/doc/Makefile
+++ b/doc/Makefile
@@ -26,7 +26,8 @@ MAN3S = cap_init.3 cap_free.3 cap_dup.3 \
 	cap_iab_get_proc.3 cap_iab_get_pid.3 cap_iab_set_proc.3 \
 	cap_iab_to_text.3 cap_iab_from_text.3 cap_iab_get_vector.3 \
 	cap_iab_set_vector.3 cap_iab_fill.3 \
-	psx_syscall.3 psx_syscall3.3 psx_syscall6.3 libpsx.3
+	psx_syscall.3 psx_syscall3.3 psx_syscall6.3 psx_set_sensitivity.3 \
+	libpsx.3
 MAN8S = getcap.8 setcap.8 getpcaps.8 captree.8
 
 MANS = $(MAN1S) $(MAN3S) $(MAN8S)
diff --git a/doc/libpsx.3 b/doc/libpsx.3
index 4ba306b..d117ae8 100644
--- a/doc/libpsx.3
+++ b/doc/libpsx.3
@@ -1,12 +1,13 @@
-.TH LIBPSX 3 "2021-03-06" "" "Linux Programmer's Manual"
+.TH LIBPSX 3 "2021-12-12" "" "Linux Programmer's Manual"
 .SH NAME
-psx_syscall3, psx_syscall6 \- POSIX semantics for system calls
+psx_syscall3, psx_syscall6, psx_set_sensitivity \- POSIX semantics for system calls
 .SH SYNOPSIS
 .nf
 #include <sys/psx_syscall.h>
 
 long int psx_syscall3(long int syscall_nr, long int arg1, long int arg2, long int arg3);
 long int psx_syscall6(long int syscall_nr, long int arg1, long int arg2, long int arg3, long int arg4, long int arg5, long int arg6);
+int psx_set_sensitivity(psx_sensitivity_t sensitivity);
 .fi
 .sp
 Link with one of these:
@@ -62,6 +63,17 @@ larger. You are encouraged to use the more explicit
 and
 .BR psx_syscall6 ()
 functions as needed.
+.PP
+.BR psx_set_sensitivity ()
+changes the behavior of the mirrored system calls:
+.B PSX_IGNORE
+ensures that differences are ignored (the default behavior);
+.B PSX_WARNING
+prints a stderr notification about how the results differ; and
+.B PSX_ERROR
+prints the error details and generates a
+.B SIGSYS
+signal.
 .SH RETURN VALUE
 The return value for system call functions is generally the value
 returned by the kernel, or \-1 in the case of an error. In such cases
diff --git a/doc/psx_set_sensitivity.3 b/doc/psx_set_sensitivity.3
new file mode 100644
index 0000000..663420c
--- /dev/null
+++ b/doc/psx_set_sensitivity.3
@@ -0,0 +1 @@
+.so man3/libpsx.3
diff --git a/go/.gitignore b/go/.gitignore
index bdb47b5..9a88b0d 100644
--- a/go/.gitignore
+++ b/go/.gitignore
@@ -7,6 +7,8 @@ psx-signals-cgo
 b210613
 b215283
 b215283-cgo
+mismatch
+mismatch-cgo
 mknames
 web
 setid
diff --git a/go/Makefile b/go/Makefile
index ea1db7f..fb7d6ec 100644
--- a/go/Makefile
+++ b/go/Makefile
@@ -14,7 +14,7 @@ IMPORTDIR=kernel.org/pub/linux/libs/security/libcap
 PKGDIR=pkg/$(GOOSARCH)/$(IMPORTDIR)
 
 DEPS=../libcap/libcap.a ../libcap/libpsx.a
-TESTS=compare-cap try-launching psx-signals
+TESTS=compare-cap try-launching psx-signals mismatch
 
 all: PSXGOPACKAGE CAPGOPACKAGE web setid gowns captree
 
@@ -102,14 +102,24 @@ b215283-cgo: b215283.go CAPGOPACKAGE
 	CC="$(CC)" CGO_ENABLED="1" $(CGO_LDFLAGS_ALLOW) CGO_CFLAGS="$(CGO_CFLAGS)" CGO_LDFLAGS="$(CGO_LDFLAGS)" $(GO) build $(GO_BUILD_FLAGS) -mod=vendor -o $@ $<
 endif
 
+mismatch: mismatch.go PSXGOPACKAGE
+	CC="$(CC)" CGO_ENABLED="$(CGO_REQUIRED)" $(CGO_LDFLAGS_ALLOW) CGO_CFLAGS="$(CGO_CFLAGS)" CGO_LDFLAGS="$(CGO_LDFLAGS)" $(GO) build $(GO_BUILD_FLAGS) -mod=vendor $<
+
+ifeq ($(CGO_REQUIRED),0)
+mismatch-cgo: mismatch.go CAPGOPACKAGE
+	CC="$(CC)" CGO_ENABLED="1" $(CGO_LDFLAGS_ALLOW) CGO_CFLAGS="$(CGO_CFLAGS)" CGO_LDFLAGS="$(CGO_LDFLAGS)" $(GO) build $(GO_BUILD_FLAGS) -mod=vendor -o $@ $<
+endif
+
 test: setid gowns captree $(TESTS)
 	CC="$(CC)" CGO_ENABLED="$(CGO_REQUIRED)" $(CGO_LDFLAGS_ALLOW) $(GO) test -mod=vendor $(IMPORTDIR)/psx
 	CC="$(CC)" CGO_ENABLED="$(CGO_REQUIRED)" $(CGO_LDFLAGS_ALLOW) $(GO) test -mod=vendor $(IMPORTDIR)/cap
 	LD_LIBRARY_PATH=../libcap ./compare-cap
 	./psx-signals
+	./mismatch || exit 0 ; exit 1
 ifeq ($(CGO_REQUIRED),0)
-	$(MAKE) psx-signals-cgo
+	$(MAKE) psx-signals-cgo mismatch-cgo
 	./psx-signals-cgo
+	./mismatch-cgo || exit 0 ; exit 1
 endif
 	./setid --caps=false
 	./gowns -- -c "echo gowns runs"
@@ -157,4 +167,5 @@ clean:
 	rm -f compare-cap try-launching try-launching-cgo
 	rm -f $(topdir)/cap/*~ $(topdir)/psx/*~
 	rm -f b210613 b215283 b215283-cgo psx-signals psx-signals-cgo
+	rm -f mismatch mismatch-cgo
 	rm -fr vendor CAPGOPACKAGE PSXGOPACKAGE go.sum
diff --git a/go/mismatch.go b/go/mismatch.go
new file mode 100644
index 0000000..bbcf6eb
--- /dev/null
+++ b/go/mismatch.go
@@ -0,0 +1,15 @@
+// Program mismatch should panic because the syscall being requested
+// never returns consistent results.
+package main
+
+import (
+	"fmt"
+	"syscall"
+
+	"kernel.org/pub/linux/libs/security/libcap/psx"
+)
+
+func main() {
+	tid, _, err := psx.Syscall3(syscall.SYS_GETTID, 0, 0, 0)
+	fmt.Printf("gettid() -> %d: %v\n", tid, err)
+}
diff --git a/psx/psx.c b/psx/psx.c
index c317063..0a0817c 100644
--- a/psx/psx.c
+++ b/psx/psx.c
@@ -56,6 +56,8 @@ typedef struct registered_thread_s {
     pthread_mutex_t mu;
     int pending;
     int gone;
+    long int retval;
+    pid_t tid;
 } registered_thread_t;
 
 static pthread_once_t psx_tracker_initialized = PTHREAD_ONCE_INIT;
@@ -81,6 +83,7 @@ static struct psx_tracker_s {
     psx_tracker_state_t state;
     int initialized;
     int psx_sig;
+    psx_sensitivity_t sensitivity;
 
     struct {
 	long syscall_nr;
@@ -136,19 +139,20 @@ static void psx_posix_syscall_actor(int signum, siginfo_t *info, void *ignore) {
 	return;
     }
 
+    long int retval;
     if (!psx_tracker.cmd.six) {
-	(void) syscall(psx_tracker.cmd.syscall_nr,
-		       psx_tracker.cmd.arg1,
-		       psx_tracker.cmd.arg2,
-		       psx_tracker.cmd.arg3);
+	retval = syscall(psx_tracker.cmd.syscall_nr,
+			 psx_tracker.cmd.arg1,
+			 psx_tracker.cmd.arg2,
+			 psx_tracker.cmd.arg3);
     } else {
-	(void) syscall(psx_tracker.cmd.syscall_nr,
-		       psx_tracker.cmd.arg1,
-		       psx_tracker.cmd.arg2,
-		       psx_tracker.cmd.arg3,
-		       psx_tracker.cmd.arg4,
-		       psx_tracker.cmd.arg5,
-		       psx_tracker.cmd.arg6);
+	retval = syscall(psx_tracker.cmd.syscall_nr,
+			 psx_tracker.cmd.arg1,
+			 psx_tracker.cmd.arg2,
+			 psx_tracker.cmd.arg3,
+			 psx_tracker.cmd.arg4,
+			 psx_tracker.cmd.arg5,
+			 psx_tracker.cmd.arg6);
     }
 
     /*
@@ -160,6 +164,8 @@ static void psx_posix_syscall_actor(int signum, siginfo_t *info, void *ignore) {
     if (ref) {
 	pthread_mutex_lock(&ref->mu);
 	ref->pending = 0;
+	ref->retval = retval;
+	ref->tid = syscall(SYS_gettid);
 	pthread_mutex_unlock(&ref->mu);
     } /*
        * else thread must be dying and its psx_action_key has already
@@ -607,6 +613,7 @@ long int __psx_syscall(long int syscall_nr, ...) {
     }
     psx_unlock();
 
+    int mismatch = 0;
     for (;;) {
 	int waiting = 0;
 	psx_lock();
@@ -619,8 +626,12 @@ long int __psx_syscall(long int syscall_nr, ...) {
 	    pthread_mutex_lock(&ref->mu);
 	    int pending = ref->pending;
 	    int gone = ref->gone;
-	    if (pending && !gone) {
-		gone = (pthread_kill(ref->thread, 0) != 0);
+	    if (!gone) {
+		if (pending) {
+		    gone = (pthread_kill(ref->thread, 0) != 0);
+		} else {
+		    mismatch |= (ref->retval != ret);
+		}
 	    }
 	    pthread_mutex_unlock(&ref->mu);
 	    if (!gone) {
@@ -639,10 +650,67 @@ long int __psx_syscall(long int syscall_nr, ...) {
 	sched_yield();
     }
 
-    errno = restore_errno;
     psx_tracker.cmd.active = 0;
+    if (mismatch) {
+	psx_lock();
+	switch (psx_tracker.sensitivity) {
+	case PSX_IGNORE:
+	    break;
+	default:
+	    fprintf(stderr, "psx_syscall result differs.\n");
+	    if (psx_tracker.cmd.six) {
+		fprintf(stderr, "trap:%ld a123=[%ld,%ld,%ld]\n",
+			psx_tracker.cmd.syscall_nr,
+			psx_tracker.cmd.arg1,
+			psx_tracker.cmd.arg2,
+			psx_tracker.cmd.arg3);
+	    } else {
+		fprintf(stderr, "trap:%ld a123456=[%ld,%ld,%ld,%ld,%ld,%ld]\n",
+			psx_tracker.cmd.syscall_nr,
+			psx_tracker.cmd.arg1,
+			psx_tracker.cmd.arg2,
+			psx_tracker.cmd.arg3,
+			psx_tracker.cmd.arg4,
+			psx_tracker.cmd.arg5,
+			psx_tracker.cmd.arg6);
+	    }
+	    fprintf(stderr, "results:");
+	    for (ref = psx_tracker.root; ref; ref = next) {
+		next = ref->next;
+		if (ref->thread == self) {
+		    continue;
+		}
+		if (ret != ref->retval) {
+		    fprintf(stderr, " %d={%ld}", ref->tid, ref->retval);
+		}
+	    }
+	    fprintf(stderr, " wanted={%ld}\n", ret);
+	    if (psx_tracker.sensitivity == PSX_WARNING) {
+		break;
+	    }
+	    pthread_kill(self, SIGSYS);
+	}
+	psx_unlock();
+    }
+    errno = restore_errno;
     psx_new_state(_PSX_SYSCALL, _PSX_IDLE);
 
 defer:
     return ret;
 }
+
+/*
+ * Change the PSX sensitivity level. If the threads appear to have
+ * diverged in behavior, this can cause the library to notify the
+ * user.
+ */
+int psx_set_sensitivity(psx_sensitivity_t level) {
+    if (level < PSX_IGNORE || level > PSX_ERROR) {
+	errno = EINVAL;
+	return -1;
+    }
+    psx_lock();
+    psx_tracker.sensitivity = level;
+    psx_unlock();
+    return 0;
+}
diff --git a/psx/psx_cgo.go b/psx/psx_cgo.go
index 26aa15a..1f75137 100644
--- a/psx/psx_cgo.go
+++ b/psx/psx_cgo.go
@@ -4,6 +4,7 @@ package psx // import "kernel.org/pub/linux/libs/security/libcap/psx"
 
 import (
 	"runtime"
+	"sync"
 	"syscall"
 )
 
@@ -32,6 +33,15 @@ func setErrno(v int) int {
 	return int(C.__errno_too(C.long(v)))
 }
 
+var makeFatal sync.Once
+
+// forceFatal configures the psx_syscall mechanism to PSX_ERROR.
+func forceFatal() {
+	makeFatal.Do(func() {
+		C.psx_set_sensitivity(C.PSX_ERROR)
+	})
+}
+
 //go:uintptrescapes
 
 // Syscall3 performs a 3 argument syscall. Syscall3 differs from
@@ -45,6 +55,7 @@ func setErrno(v int) int {
 // If CGO_ENABLED=0 it redirects to the go1.16+
 // syscall.AllThreadsSyscall() function.
 func Syscall3(syscallnr, arg1, arg2, arg3 uintptr) (uintptr, uintptr, syscall.Errno) {
+	forceFatal()
 	// We lock to the OSThread here because we may need errno to
 	// be the one for this thread.
 	runtime.LockOSThread()
@@ -65,6 +76,7 @@ func Syscall3(syscallnr, arg1, arg2, arg3 uintptr) (uintptr, uintptr, syscall.Er
 // arguments, its behavior is identical to that of Syscall3() - see
 // above for the full documentation.
 func Syscall6(syscallnr, arg1, arg2, arg3, arg4, arg5, arg6 uintptr) (uintptr, uintptr, syscall.Errno) {
+	forceFatal()
 	// We lock to the OSThread here because we may need errno to
 	// be the one for this thread.
 	runtime.LockOSThread()
diff --git a/psx/psx_syscall.h b/psx/psx_syscall.h
index 3987d59..7a8c9a1 100644
--- a/psx/psx_syscall.h
+++ b/psx/psx_syscall.h
@@ -67,6 +67,27 @@ void psx_load_syscalls(long int (**syscall_fn)(long int,
 						long int, long int, long int,
 						long int, long int, long int));
 
+/*
+ * psx_sensitivity_t holds the level of paranoia for non-POSIX syscall
+ * behavior. The default is PSX_IGNORE: which is best effort - no
+ * enforcement; PSX_WARNING will dump to stderr a warning when a
+ * syscall's results differ; PSX_ERROR will dump info as per
+ * PSX_WARNING and generate a SIGSYS. The current mode can be set with
+ * psx_set_sensitivity().
+ */
+typedef enum {
+    PSX_IGNORE = 0,
+    PSX_WARNING = 1,
+    PSX_ERROR = 2,
+} psx_sensitivity_t;
+
+/*
+ * psx_set_sensitivity sets the current sensitivity of the PSX
+ * mechanism.  The function returns 0 on success and -1 if the
+ * requested level is invalid.
+ */
+int psx_set_sensitivity(psx_sensitivity_t level);
+
 #ifdef __cplusplus
 }
 #endif