From e7dc395ae9026050472907ccb846f093df737c81 Mon Sep 17 00:00:00 2001 From: alanhc Date: Thu, 3 Sep 2026 18:41:02 +0800 Subject: [PATCH] Stop faking SIGCHLD on a CLONE_THREAD exit When the last CLONE_THREAD worker exited, forkipc.c raised the futex interrupt, which makes the next blocking call in the surviving thread return EINTR. The comment gave the reason: "In real Linux, child exit delivers SIGCHLD which interrupts futex_wait with -EINTR." Linux does not. clone(2) is explicit that a thread created with CLONE_THREAD sends no signal to its parent when it terminates; the thread group's exit signal goes to the process's parent, not to a sibling. Nothing is delivered, so a sibling's wait is not interrupted and its timeout is what ends it. Measured on Linux 6.18.44 through the qemu lane, a worker exits and the main thread then parks for 300 ms with nothing to wake it: call Linux elfuse before futex(FUTEX_WAIT) ETIMEDOUT 303ms EINTR 101ms ppoll 0 303ms EINTR 0ms epoll_pwait 0 311ms EINTR 302ms nanosleep 0 309ms 0 328ms No signal is pending on either side. nanosleep agrees because it does not read the flag, which is what identifies the flag as the mechanism rather than anything process-wide. Removing the request drops the fabricated errno and nothing else. The futex waiter is not woken by the interrupt: it wakes on its own 100 ms quantum and re-checks thread_stop_requested, its woken flag, expired itimers and queued signals, of which futex_interrupt_consume is one reason among several. wakeup_pipe_signal and thread_interrupt_all stay, so threads parked on the shared pipe and inside hv_vcpu_run still get the nudge this site exists to send. Teardown keeps its interrupt. All four callers that mean it go through thread_wake_all_blocked, where teardown state marks every thread as leaving but the atomic interrupt itself is consumed by only one waiter. This site was the only one that raised it while the process carried on running. The one-shot consume in 520568c is untouched. That fixed the flag staying set, which is why foot spun on EINTR forever; it did not ask whether the flag should have been set at all. tests/test-futex-ops.c parks in each of the three calls above after a worker exit and fails on an early return. It fails three of seven before this change, passes after, and passes unchanged on the reference kernel. It lives in test-matrix.sh's run_unit_tests and not in tests/manifest.txt, per that file's scope note: every assertion is a timeout the guest can observe, so it is cross-checkable. The worker's own reap loop polls the CLEARTID store with nanosleep rather than a futex wait: a futex wait on any address, including that one, is a chance to consume the one-shot interrupt before the calls under test run, which would pass the test even with the bug back. Reconfirmed against the reintroduced bug: the reap loop's own wait no longer hides it, and the same three cases fail again. Naming follows test-thread.c and test-exec-handoff.c: child_tid, child_stack_buf, spawn_and_reap_worker. Filed as test-futex-no-phantom-eintr.c, which named the regression; renamed to test-futex-ops.c to match the file-naming scheme, where ops names the futex operations under test (test-file-ops.c is the existing precedent). --- src/runtime/forkipc.c | 11 +-- src/runtime/futex.c | 18 ++-- tests/test-futex-ops.c | 191 +++++++++++++++++++++++++++++++++++++++++ tests/test-matrix.sh | 2 + 4 files changed, 208 insertions(+), 14 deletions(-) create mode 100644 tests/test-futex-ops.c diff --git a/src/runtime/forkipc.c b/src/runtime/forkipc.c index f87c87c7..403e6485 100644 --- a/src/runtime/forkipc.c +++ b/src/runtime/forkipc.c @@ -1142,15 +1142,12 @@ static void *thread_create_and_run(void *arg) thread_deactivate(t); /* When all CLONE_THREAD workers have exited and only the main thread - * remains, interrupt its futex_wait. In real Linux, child exit delivers - * SIGCHLD which interrupts futex_wait with -EINTR. elfuse simulates this - * through the futex interrupt API. + * remains, nudge it so anything parked on host state re-checks. No EINTR + * goes with it: clone(2) sends no signal for a CLONE_THREAD exit, so a + * sibling's wait is not interrupted and its timeout is what ends it. */ if (thread_active_count() == 1) { - log_debug( - "last worker exited, interrupting " - "main thread futex_wait/poll"); - futex_interrupt_request(); + log_debug("last worker exited, waking main thread"); wakeup_pipe_signal(); thread_interrupt_all(); } diff --git a/src/runtime/futex.c b/src/runtime/futex.c index 7dc72a7f..2cec554c 100644 --- a/src/runtime/futex.c +++ b/src/runtime/futex.c @@ -63,9 +63,9 @@ #define ELFUSE_HAVE_OS_SYNC_WAIT_ON_ADDRESS 0 #endif -/* Interrupt flag: when set, futex_wait returns -EINTR. Used to simulate SIGCHLD - * delivery when all CLONE_THREAD workers exit: wakes the main thread from - * blocking futex_wait without triggering a full exit_group. +/* Interrupt flag: when set, futex_wait returns -EINTR. Raised only by teardown + * through thread_wake_all_blocked, so every blocked wait can observe that the + * process is tearing down without a full exit_group. */ static _Atomic int futex_interrupt_requested = 0; @@ -583,13 +583,17 @@ void futex_interrupt_clear(void) } /* Test-and-clear: returns 1 if the interrupt request was pending and atomically - * clears it, 0 otherwise. The interrupt is a one-shot edge: forkipc.c sets it - * when the last clone-thread exits so the main thread observes EINTR in its - * next blocking wait, mirroring how real Linux delivers SIGCHLD. Without the - * clear, the flag stays set and every subsequent epoll_pwait, ppoll, futex + * clears it, 0 otherwise. The interrupt is a one-shot edge, set by the teardown + * paths through thread_wake_all_blocked. Teardown state marks every thread as + * leaving; the atomic interrupt itself is consumed by only one waiter. Without + * the clear, the flag stays set and every subsequent epoll_pwait, ppoll, futex * wait, etc. spins on EINTR until execve clears it -- in foot's case it never * does, and the spinning main thread eventually faults in a code path the guest * never expects to reach. + * + * forkipc.c set it too, on the last clone-thread exit, for a SIGCHLD that + * clone(2) does not send. That is gone; only a process actually tearing down + * fabricates an EINTR now. */ int futex_interrupt_consume(void) { diff --git a/tests/test-futex-ops.c b/tests/test-futex-ops.c new file mode 100644 index 00000000..21d63d0d --- /dev/null +++ b/tests/test-futex-ops.c @@ -0,0 +1,191 @@ +/* + * A CLONE_THREAD worker's exit must not interrupt anybody + * + * Copyright 2026 elfuse contributors + * SPDX-License-Identifier: Apache-2.0 + * + * clone(2): a thread created with CLONE_THREAD sends no signal to its parent + * when it terminates. Nothing is delivered, so a sibling parked in a blocking + * call stays parked and its timeout is what ends the wait. + * + * Each case parks for PARK_MS with nothing to wake it. A timeout answer means + * the exit went unnoticed, which is what Linux does; EINTR means the wait was + * cut short by something the guest was never sent. + * + * Syscalls exercised: futex(98), ppoll(73), epoll_pwait(22), clone(220), + * exit(93), clock_gettime(113) + */ + +#include +#include +#include +#include + +#include "test-harness.h" +#include "raw-syscall.h" +#include "test-util.h" + +int passes = 0, fails = 0; + +/* Long enough that an immediate return is unambiguous, short enough that four + * of them do not slow the lane. + */ +#define PARK_MS 300 +#define PARK_NS (PARK_MS * 1000L * 1000L) + +/* A wait cut short by the exit returns well inside the park; the futex path + * polls on a 100 ms quantum, so anything under half the park is early. + */ +#define EARLY_MS (PARK_MS / 2) + +static int word; +static volatile int child_tid; +static char child_stack_buf[16384] __attribute__((aligned(16))); + +struct k_timespec { + int64_t tv_sec; + int64_t tv_nsec; +}; + +static long now_ms(void) +{ + struct k_timespec ts; + raw_syscall2(113, 1 /* CLOCK_MONOTONIC */, (long) &ts); + return (long) (ts.tv_sec * 1000 + ts.tv_nsec / 1000000); +} + +static void msleep(long ms) +{ + struct k_timespec ts = {0, ms * 1000L * 1000L}; + raw_syscall4(101 /* nanosleep */, (long) &ts, 0, 0, 0); +} + +/* Spawn a CLONE_THREAD worker that exits at once, and wait until it is gone. + * CLONE_PARENT_SETTID seeds child_tid so the CLEARTID store is an observable + * edge rather than a word that was already zero. + */ +static int spawn_and_reap_worker(void) +{ + unsigned long flags = 0x00010000 /* CLONE_THREAD */ + | 0x00000100 /* CLONE_VM */ + | 0x00000200 /* CLONE_FS */ + | 0x00000800 /* CLONE_SIGHAND */ + | 0x00100000 /* CLONE_PARENT_SETTID */ + | 0x00200000; /* CLONE_CHILD_CLEARTID */ + + long r = raw_syscall5(220, (long) flags, + (long) (child_stack_buf + sizeof(child_stack_buf)), + (long) &child_tid, 0, (long) &child_tid); + if (r == 0) { + raw_exit(0); + test_unreachable(); + } + if (r < 0) + return -1; + + /* Poll the CLEARTID store with msleep rather than a futex wait: any futex + * wait, including one on this address, is a chance to consume the one-shot + * phantom EINTR this test exists to catch, before the calls under test run. + * Bounded, so a teardown that never publishes is a reported failure rather + * than a driver timeout. + */ + long deadline = now_ms() + 5000; + while (__atomic_load_n((volatile int *) &child_tid, __ATOMIC_ACQUIRE) != + 0) { + msleep(100); + if (now_ms() > deadline) + return -1; + } + return 0; +} + +/* Runs one park and reports whether it lasted. rc/err are the raw answer. */ +static void expect_uninterrupted(const char *name, long rc, long elapsed) +{ + TEST(name); + if (rc == -EINTR) { + FAIL("a CLONE_THREAD exit interrupted a wait Linux leaves parked"); + return; + } + if (elapsed < EARLY_MS) { + FAIL("the wait ended early without reporting an interruption"); + return; + } + PASS(); +} + +int main(void) +{ + printf("=== phantom EINTR on CLONE_THREAD exit ===\n\n"); + + TEST("baseline park with no worker"); + { + struct k_timespec ts = {0, PARK_NS}; + long t0 = now_ms(); + long rc = + raw_syscall6(__NR_futex, (long) &word, + FUTEX_WAIT | FUTEX_PRIVATE_FLAG, 0, (long) &ts, 0, 0); + long elapsed = now_ms() - t0; + if (rc != -ETIMEDOUT) + FAIL("an unwoken park must report ETIMEDOUT"); + else if (elapsed < EARLY_MS) + FAIL("the park did not last"); + else + PASS(); + } + + /* One worker per case: the interrupt this test exists to catch is a + * one-shot edge, so a second case would find it already consumed. + */ + TEST("clone and reap a worker"); + if (spawn_and_reap_worker() < 0) { + FAIL("worker never exited"); + goto done; + } + PASS(); + + { + struct k_timespec ts = {0, PARK_NS}; + long t0 = now_ms(); + long rc = + raw_syscall6(__NR_futex, (long) &word, + FUTEX_WAIT | FUTEX_PRIVATE_FLAG, 0, (long) &ts, 0, 0); + expect_uninterrupted("futex wait survives the exit", rc, now_ms() - t0); + } + + TEST("clone and reap a worker for ppoll"); + if (spawn_and_reap_worker() < 0) { + FAIL("worker never exited"); + goto done; + } + PASS(); + + { + /* The raw ppoll writes the remaining time back, so it gets its own. */ + struct k_timespec ts = {0, PARK_NS}; + long t0 = now_ms(); + long rc = raw_syscall5(73, 0, 0, (long) &ts, 0, 8); + expect_uninterrupted("ppoll survives the exit", rc, now_ms() - t0); + } + + TEST("clone and reap a worker for epoll_pwait"); + long ep = raw_syscall1(20 /* epoll_create1 */, 0); + if (ep < 0 || spawn_and_reap_worker() < 0) { + FAIL("worker never exited"); + goto done; + } + PASS(); + + { + char evs[16]; + long t0 = now_ms(); + long rc = raw_syscall6(22 /* epoll_pwait */, ep, (long) evs, 1, PARK_MS, + 0, 8); + expect_uninterrupted("epoll_pwait survives the exit", rc, + now_ms() - t0); + } + +done: + SUMMARY("test-futex-ops"); + return fails > 0 ? 1 : 0; +} diff --git a/tests/test-matrix.sh b/tests/test-matrix.sh index 840033a5..2a0a2da1 100755 --- a/tests/test-matrix.sh +++ b/tests/test-matrix.sh @@ -893,6 +893,8 @@ run_unit_tests() test_rc "$runner" "test-futex-requeue-account" 0 \ "$bindir/test-futex-requeue-account" test_rc "$runner" "test-robust-futex" 0 "$bindir/test-robust-futex" + test_rc "$runner" "test-futex-ops" 0 \ + "$bindir/test-futex-ops" test_check "$runner" "test-shim-futex-fast" "OK" \ "$bindir/test-shim-futex-fast"