diff --git a/mk/verify.mk b/mk/verify.mk index 9d121021..28e019b3 100644 --- a/mk/verify.mk +++ b/mk/verify.mk @@ -391,6 +391,45 @@ VERIFY_FUTEXOP_SCAN := src/proved/futexop.h VERIFY_FUTEXOP_CLAIM := for ANY guest-supplied val3 word VERIFY_FUTEXOP_UNPROVED := the wake and requeue walks around them stay test-covered +VERIFY_FUTEXREQ_SRC := src/proved/futexreq.h +VERIFY_FUTEXREQ_FCTS := futex_requeue_counts_valid futex_requeue_budget +VERIFY_FUTEXREQ_MIN_GOALS ?= 14 +# typed: two scalars in, one out, no buffer and no aliasing question. +VERIFY_FUTEXREQ_MODEL := typed +VERIFY_FUTEXREQ_SCAN := src/proved/futexreq.h +VERIFY_FUTEXREQ_CLAIM := for ANY pair of guest-supplied requeue counts +VERIFY_FUTEXREQ_UNPROVED := the bucket walk the budget bounds stays test-covered + +VERIFY_FUTEXWAKEOP_SRC := src/proved/futexwakeop.h +VERIFY_FUTEXWAKEOP_FCTS := futex_wake_op_supported futex_wake_cmp_supported \ + futex_wake_op_apply futex_wake_op_cmp +VERIFY_FUTEXWAKEOP_MIN_GOALS ?= 92 +# typed: scalars in, one scalar out, no buffer and no aliasing question. +VERIFY_FUTEXWAKEOP_MODEL := typed +VERIFY_FUTEXWAKEOP_SCAN := src/proved/futexwakeop.h +VERIFY_FUTEXWAKEOP_CLAIM := for ANY guest-supplied op and comparison selector +VERIFY_FUTEXWAKEOP_UNPROVED := the wake walks the selectors gate stay test-covered + +VERIFY_FUTEXPI_SRC := src/proved/futexpi.h +VERIFY_FUTEXPI_FCTS := futex_pi_owner_tid futex_pi_unowned futex_pi_owner_died \ + futex_pi_has_waiters futex_pi_set_waiters \ + futex_pi_clear_waiters futex_pi_mark_owner_died +VERIFY_FUTEXPI_MIN_GOALS ?= 38 +# typed: one scalar in, one scalar out, no buffer and no aliasing question. +VERIFY_FUTEXPI_MODEL := typed +VERIFY_FUTEXPI_SCAN := src/proved/futexpi.h +VERIFY_FUTEXPI_CLAIM := for ANY bit pattern a guest can write to a PI lock word +VERIFY_FUTEXPI_UNPROVED := the CAS loops around them stay test-covered + +VERIFY_FUTEXWAITV_SRC := src/proved/futexwaitv.h +VERIFY_FUTEXWAITV_FCTS := futex_bucket_insert +VERIFY_FUTEXWAITV_MIN_GOALS ?= 36 +# typed: one flat array of unsigned, no aliasing question beside it. +VERIFY_FUTEXWAITV_MODEL := typed +VERIFY_FUTEXWAITV_SCAN := src/proved/futexwaitv.h +VERIFY_FUTEXWAITV_CLAIM := for ANY set of guest-chosen futex addresses +VERIFY_FUTEXWAITV_UNPROVED := the walk that calls it stays test-covered + VERIFY_PATHDEPTH_SRC := src/proved/pathdepth.h VERIFY_PATHDEPTH_FCTS := path_depth_push path_depth_pop VERIFY_PATHDEPTH_MIN_GOALS ?= 24 diff --git a/scripts/check-mutants.py b/scripts/check-mutants.py index 7b083a6b..b48538d7 100755 --- a/scripts/check-mutants.py +++ b/scripts/check-mutants.py @@ -179,6 +179,138 @@ def _load(stem, name): " FUTEX_TIMESPEC_SEC_MAX);\n", " return lts->tv_sec >= 0 && lts->tv_nsec >= 0;\n", ), + # ---- verify-futexreq --------------------------------------------------- + ( + "futexreq", + "src/proved/futexreq.h", + "futex_requeue_counts_valid", + "accept any pair (a negative count survives as a huge unsigned)", + " return nr_wake < FUTEX_COUNT_LIMIT && nr_requeue < FUTEX_COUNT_LIMIT;\n", + " return 1;\n", + ), + ( + "futexreq", + "src/proved/futexreq.h", + "futex_requeue_budget", + "bound the walk by the requeue half alone (the wake budget goes unwalked)", + " return (uint64_t) nr_wake + nr_requeue;\n", + " return (uint64_t) nr_requeue;\n", + ), + # ---- verify-futexwaitv ------------------------------------------------- + ( + "futexwaitv", + "src/proved/futexwaitv.h", + "futex_bucket_insert", + "append without scanning (the set stops being sorted)", + " unsigned pos = 0;\n", + " unsigned pos = n;\n", + ), + ( + "futexwaitv", + "src/proved/futexwaitv.h", + "futex_bucket_insert", + "drop the repeat check (one bucket is locked twice)", + " if (pos < n && ids[pos] == idx)\n return n;\n", + " if (0)\n return n;\n", + ), + ( + "futexwaitv", + "src/proved/futexwaitv.h", + "futex_bucket_insert", + "stop the scan one short (an equal entry is missed)", + " while (pos < n && ids[pos] < idx)\n", + " while (pos + 1 < n && ids[pos] < idx)\n", + ), + # ---- verify-futexpi ---------------------------------------------------- + ( + "futexpi", + "src/proved/futexpi.h", + "futex_pi_owner_tid", + "read the flag bits as part of the TID", + " return word & FUTEX_PI_TID_MASK;\n", + " return word;\n", + ), + ( + "futexpi", + "src/proved/futexpi.h", + "futex_pi_set_waiters", + "drop the remainder (a word that already had the bit doubles it away)", + " return word % FUTEX_PI_WAITERS + FUTEX_PI_WAITERS;\n", + " return word + FUTEX_PI_WAITERS;\n", + ), + ( + "futexpi", + "src/proved/futexpi.h", + "futex_pi_mark_owner_died", + "drop the waiters bit from the death transition", + " return (word >= FUTEX_PI_WAITERS ? FUTEX_PI_WAITERS : 0u) |\n FUTEX_PI_OWNER_DIED;\n", + " return FUTEX_PI_OWNER_DIED;\n", + ), + ( + "futexpi", + "src/proved/futexpi.h", + "futex_pi_unowned", + "call a word unowned whenever any bit is clear", + " return (word & FUTEX_PI_TID_MASK) == 0;\n", + " return word != 0xFFFFFFFFu;\n", + ), + ( + "futexpi", + "src/proved/futexpi.h", + "futex_pi_owner_died", + "read the waiters bit as the death flag", + " return (word & FUTEX_PI_OWNER_DIED) != 0;\n", + " return (word & FUTEX_PI_WAITERS) != 0;\n", + ), + ( + "futexpi", + "src/proved/futexpi.h", + "futex_pi_has_waiters", + "off by one at the boundary word", + " return word >= FUTEX_PI_WAITERS;\n", + " return word > FUTEX_PI_WAITERS;\n", + ), + ( + "futexpi", + "src/proved/futexpi.h", + "futex_pi_clear_waiters", + "clear the death flag along with the waiters bit", + " return word % FUTEX_PI_WAITERS;\n", + " return word % FUTEX_PI_OWNER_DIED;\n", + ), + # ---- verify-futexwakeop ------------------------------------------------ + ( + "futexwakeop", + "src/proved/futexwakeop.h", + "futex_wake_op_supported", + "accept every op (an unassigned encoding stops being ENOSYS)", + " return op <= FUTEX_WAKE_OP_MAX;\n", + " return 1;\n", + ), + ( + "futexwakeop", + "src/proved/futexwakeop.h", + "futex_wake_cmp_supported", + "accept every comparison selector", + " return cmp <= FUTEX_WAKE_CMP_MAX;\n", + " return 1;\n", + ), + ( + "futexwakeop", + "src/proved/futexwakeop.h", + "futex_wake_op_apply", + "drop the complement from ANDN (it becomes AND)", + " return old_val & ~op_val;\n", + " return old_val & op_val;\n", + ), + ( + "futexwakeop", + "src/proved/futexwakeop.h", + "futex_wake_op_cmp", + "widen LT to LE (the boundary case flips)", + " return old_val < cmp_arg;\n", + " return old_val <= cmp_arg;\n", + ), # ---- verify-futexop ---------------------------------------------------- ( "futexop", diff --git a/src/proved/futexpi.h b/src/proved/futexpi.h new file mode 100644 index 00000000..7666e175 --- /dev/null +++ b/src/proved/futexpi.h @@ -0,0 +1,107 @@ +/* + * The PI lock word, split out of the FUTEX_LOCK_PI paths and robust_list_walk + * in src/runtime/futex.c and proved here. + * + * Layout, which Linux fixes and a guest can write any bit pattern into: + * + * bits 0-29 owner TID (FUTEX_TID_MASK) + * bit 30 FUTEX_OWNER_DIED + * bit 31 FUTEX_WAITERS + * + * The death transition is the one worth a contract rather than a mask: it has + * to clear the TID, set OWNER_DIED and leave WAITERS alone, and an owner whose + * waiters bit it dropped is a lock nobody is ever woken from. + */ +#pragma once + +#include + +#define FUTEX_PI_TID_MASK 0x3FFFFFFFu +#define FUTEX_PI_OWNER_DIED 0x40000000u +#define FUTEX_PI_WAITERS 0x80000000u + +/*@ + assigns \nothing; + ensures bounded: \result <= 0x3FFFFFFF; + ensures exact: \result == (word & 0x3FFFFFFF); + */ +static inline uint32_t futex_pi_owner_tid(uint32_t word) +{ + return word & FUTEX_PI_TID_MASK; +} + +/*@ + assigns \nothing; + ensures binary: \result == 0 || \result == 1; + ensures exact: \result != 0 <==> (word & 0x3FFFFFFF) == 0; + */ +static inline int futex_pi_unowned(uint32_t word) +{ + return (word & FUTEX_PI_TID_MASK) == 0; +} + +/*@ + assigns \nothing; + ensures binary: \result == 0 || \result == 1; + ensures exact: \result != 0 <==> (word & 0x40000000) != 0; + */ +static inline int futex_pi_owner_died(uint32_t word) +{ + return (word & FUTEX_PI_OWNER_DIED) != 0; +} + +/* WAITERS is the top bit, so the three below read it as magnitude and edit it + * as a remainder. Under the bitwise spelling the "leaves the rest alone" + * clauses time out at 30s on both provers, the wall futexop.h documents; these + * forms discharge. Nothing about the shipped code needs the bit operators. + */ +/*@ + assigns \nothing; + ensures binary: \result == 0 || \result == 1; + ensures exact: \result != 0 <==> word >= 0x80000000; + */ +static inline int futex_pi_has_waiters(uint32_t word) +{ + return word >= FUTEX_PI_WAITERS; +} + +/* The two edits a waiter makes to the flag, neither of which may disturb the + * owner field between them. + */ +/*@ + assigns \nothing; + ensures set: \result >= 0x80000000; + ensures others_kept: \result % 0x80000000 == word % 0x80000000; + */ +static inline uint32_t futex_pi_set_waiters(uint32_t word) +{ + return word % FUTEX_PI_WAITERS + FUTEX_PI_WAITERS; +} + +/*@ + assigns \nothing; + ensures clear: \result < 0x80000000; + ensures others_kept: \result % 0x80000000 == word % 0x80000000; + */ +static inline uint32_t futex_pi_clear_waiters(uint32_t word) +{ + return word % FUTEX_PI_WAITERS; +} + +/* What robust_list_walk writes over a lock whose owner exited holding it. + * + * Spelled the way handle_futex_death() in kernel/futex/core.c spells it. The + * three postconditions pin every bit of the answer: the low 30 are zero, bit 30 + * is set, bit 31 is whatever it was. + */ +/*@ + assigns \nothing; + ensures tid_cleared: (\result & 0x3FFFFFFF) == 0; + ensures died_set: (\result & 0x40000000) != 0; + ensures waiters_kept: \result >= 0x80000000 <==> word >= 0x80000000; + */ +static inline uint32_t futex_pi_mark_owner_died(uint32_t word) +{ + return (word >= FUTEX_PI_WAITERS ? FUTEX_PI_WAITERS : 0u) | + FUTEX_PI_OWNER_DIED; +} diff --git a/src/proved/futexreq.h b/src/proved/futexreq.h new file mode 100644 index 00000000..6154e8d7 --- /dev/null +++ b/src/proved/futexreq.h @@ -0,0 +1,59 @@ +/* + * Requeue count validation, split out of futex_requeue in src/runtime/futex.c + * and proved here. + * + * Linux refuses a plain FUTEX_REQUEUE whose wake or requeue count is negative, + * before it takes either futex key. The counts reach elfuse as uint32_t -- + * sc_futex forwards val as (uint32_t) x2 and the requeue half out of the + * timeout slot -- so the sign the guest passed survives only as the top bit. + * + * The test compares against 2^31 rather than casting back to int32_t. + * Converting a value above INT32_MAX to a signed type is implementation-defined + * before C23, and the provers reason about the unsigned value directly, the way + * futexop.h and futexhash.h stay in the value theory for the same reason. + * + * Validation has to come first because of the budget below: the walk stops once + * it has touched nr_wake + nr_requeue waiters, and a negative pair read as + * unsigned makes that bound 2^33 - 2. Refusing the pair is what holds the sum + * inside 32 bits. The overflow CVE-2018-6927 reached went through this same + * argument pair, on the same syscall. + */ +#pragma once + +#include + +/* One past the largest count a guest can pass as a non-negative int32_t. */ +#define FUTEX_COUNT_LIMIT 0x80000000u + +/*@ + assigns \nothing; + ensures binary: \result == 0 || \result == 1; + ensures exact: + \result != 0 <==> (nr_wake < 0x80000000 && nr_requeue < 0x80000000); + */ +static inline int futex_requeue_counts_valid(uint32_t nr_wake, + uint32_t nr_requeue) +{ + return nr_wake < FUTEX_COUNT_LIMIT && nr_requeue < FUTEX_COUNT_LIMIT; +} + +/* How many waiters the call may touch. Linux bounds its walk the same way, by + * breaking once task_count reaches nr_wake + nr_requeue. + * + * The sum is taken in 64 bits so the addition itself cannot wrap whatever the + * caller passes; no_overflow is the stronger statement, that a validated pair + * leaves the result inside 32 bits. + */ +/*@ + requires valid: nr_wake < 0x80000000 && nr_requeue < 0x80000000; + assigns \nothing; + ensures sum: \result == (uint64_t) nr_wake + nr_requeue; + ensures covers_wake: \result >= nr_wake; + ensures covers_requeue: \result >= nr_requeue; + ensures no_overflow: \result < 0x100000000; + */ +static inline uint64_t futex_requeue_budget(uint32_t nr_wake, + uint32_t nr_requeue) +{ + return (uint64_t) nr_wake + nr_requeue; +} diff --git a/src/proved/futexwaitv.h b/src/proved/futexwaitv.h new file mode 100644 index 00000000..05a5d78a --- /dev/null +++ b/src/proved/futexwaitv.h @@ -0,0 +1,83 @@ +/* + * The bucket set a futex_waitv call locks, split out of waitv_collect_buckets + * in src/runtime/futex.c and proved here. + * + * sys_futex_waitv takes up to 128 guest-chosen addresses, hashes each to a + * bucket, and locks the distinct buckets in ascending index order. Two entries + * hashing alike is ordinary: there are 1024 buckets and the guest picks the + * addresses. What holds the call together is that this set comes out sorted and + * without repeats. A repeat locks one non-recursive mutex twice, and an + * unsorted set takes the bucket locks out of order against every other futex + * path. + * + * The insertion is proved rather than the walk around it: the walk's own bound + * is nr_futexes, which sys_futex_waitv checks before it gets here. + */ +#pragma once + +#include + +/*@ + predicate sorted_strict(unsigned *a, integer n) = + \forall integer i, j; 0 <= i < j < n ==> a[i] < a[j]; + + predicate holds(unsigned *a, integer n, unsigned v) = + \exists integer k; 0 <= k < n && a[k] == v; + */ + +/* Insert idx into a sorted, repeat-free prefix, and answer the new length. + * + * cap is the array's length rather than the caller's bound, so the shift below + * is inside the object for any n the precondition allows. + */ +/*@ + requires room: n < cap; + requires valid: \valid(ids + (0 .. cap - 1)); + requires sorted: sorted_strict(ids, n); + assigns ids[0 .. cap - 1]; + ensures grows_by_at_most_one: \result == n || \result == n + 1; + ensures still_sorted: sorted_strict(ids, \result); + ensures present: holds(ids, \result, idx); + ensures fresh_is_longer: + \result == n + 1 <==> !\at(holds(ids, n, idx), Pre); + */ +static inline unsigned futex_bucket_insert(unsigned *ids, + unsigned n, + unsigned cap, + unsigned idx) +{ + unsigned pos = 0; + + /*@ + loop invariant bound: 0 <= pos <= n; + loop invariant below: \forall integer i; 0 <= i < pos ==> ids[i] < idx; + loop assigns pos; + loop variant n - pos; + */ + while (pos < n && ids[pos] < idx) + pos++; + + if (pos < n && ids[pos] == idx) + return n; + + /* Nothing at or after pos equals idx either: the scan stopped at the first + * entry not below idx, and the prefix is strictly increasing. + */ + /*@ assert past: pos < n ==> ids[pos] > idx; */ + /*@ assert absent: !holds(ids, n, idx); */ + + /*@ + loop invariant bound: pos <= j <= n; + loop invariant shifted: + \forall integer k; j < k <= n ==> ids[k] == \at(ids[k - 1], LoopEntry); + loop invariant kept: + \forall integer k; 0 <= k < j ==> ids[k] == \at(ids[k], LoopEntry); + loop assigns j, ids[pos + 1 .. n]; + loop variant j - pos; + */ + for (unsigned j = n; j > pos; j--) + ids[j] = ids[j - 1]; + + ids[pos] = idx; + return n + 1; +} diff --git a/src/proved/futexwakeop.h b/src/proved/futexwakeop.h new file mode 100644 index 00000000..606a80df --- /dev/null +++ b/src/proved/futexwakeop.h @@ -0,0 +1,111 @@ +/* + * Operation and comparison selectors for FUTEX_WAKE_OP, split out of + * futex_wake_op in src/runtime/futex.c and proved here. + * + * Both selectors are guest-supplied and both have unassigned encodings. Linux + * answers ENOSYS for those, and refuses each at a different point: an op it + * does not implement stops before the read-modify-write, an unimplemented + * comparison stops after it. Neither wakes anybody. The measurements behind + * that split are in the commit message. + * + * The selectors are compared against their bounds rather than enumerated, so + * the supported set stays one number per side. + */ +#pragma once + +#include + +/* Largest selector Linux implements: FUTEX_OP_XOR and FUTEX_OP_CMP_GE. */ +#define FUTEX_WAKE_OP_MAX 4u +#define FUTEX_WAKE_CMP_MAX 5u + +/*@ + assigns \nothing; + ensures binary: \result == 0 || \result == 1; + ensures exact: \result != 0 <==> op <= 4; + */ +static inline int futex_wake_op_supported(uint32_t op) +{ + return op <= FUTEX_WAKE_OP_MAX; +} + +/*@ + assigns \nothing; + ensures binary: \result == 0 || \result == 1; + ensures exact: \result != 0 <==> cmp <= 5; + */ +static inline int futex_wake_cmp_supported(uint32_t cmp) +{ + return cmp <= FUTEX_WAKE_CMP_MAX; +} + +/* The word uaddr2 takes. Every op is modular, so a sign-extended operand is + * carried as its two's complement bits. + * + * The postconditions pin a value per op rather than a range: a body that + * returned old_val throughout, which is what an unhandled op used to do, meets + * a range and fails these. + * + * ANDN casts its complement back to uint32_t. Without the cast ACSL reads ~x as + * -x-1, and relating a negative operand of & to the code times out. + */ +/*@ + requires supported: op <= 4; + assigns \nothing; + ensures set: op == 0 ==> \result == op_val; + ensures add: op == 1 ==> \result == (uint32_t) (old_val + op_val); + ensures or: op == 2 ==> \result == (old_val | op_val); + ensures andn: op == 3 ==> \result == (old_val & (uint32_t) ~op_val); + ensures xor: op == 4 ==> \result == (old_val ^ op_val); + */ +static inline uint32_t futex_wake_op_apply(uint32_t old_val, + uint32_t op, + uint32_t op_val) +{ + switch (op) { + case 0: + return op_val; + case 1: + return old_val + op_val; + case 2: + return old_val | op_val; + case 3: + return old_val & ~op_val; + default: + return old_val ^ op_val; + } +} + +/* Whether the second wake fires. The comparisons are signed, on the word as it + * was before the modify above. + */ +/*@ + requires supported: cmp <= 5; + assigns \nothing; + ensures binary: \result == 0 || \result == 1; + ensures eq: cmp == 0 ==> (\result != 0 <==> old_val == cmp_arg); + ensures ne: cmp == 1 ==> (\result != 0 <==> old_val != cmp_arg); + ensures lt: cmp == 2 ==> (\result != 0 <==> old_val < cmp_arg); + ensures le: cmp == 3 ==> (\result != 0 <==> old_val <= cmp_arg); + ensures gt: cmp == 4 ==> (\result != 0 <==> old_val > cmp_arg); + ensures ge: cmp == 5 ==> (\result != 0 <==> old_val >= cmp_arg); + */ +static inline int futex_wake_op_cmp(int32_t old_val, + uint32_t cmp, + int32_t cmp_arg) +{ + switch (cmp) { + case 0: + return old_val == cmp_arg; + case 1: + return old_val != cmp_arg; + case 2: + return old_val < cmp_arg; + case 3: + return old_val <= cmp_arg; + case 4: + return old_val > cmp_arg; + default: + return old_val >= cmp_arg; + } +} diff --git a/src/runtime/futex.c b/src/runtime/futex.c index 7dc72a7f..d5dc8f84 100644 --- a/src/runtime/futex.c +++ b/src/runtime/futex.c @@ -42,6 +42,10 @@ #include "debug/log.h" #include "proved/futexhash.h" #include "proved/futexop.h" +#include "proved/futexpi.h" +#include "proved/futexreq.h" +#include "proved/futexwaitv.h" +#include "proved/futexwakeop.h" #include "proved/timespec.h" /* macOS 14.4+ ships os_sync_{wait_on_address_with_timeout,wake_by_address_any} @@ -106,18 +110,9 @@ _Static_assert(FUTEX_WAKE_BITSET == 10, #define FUTEX_BITSET_MATCH_ANY 0xFFFFFFFFU -/* PI futex word layout (bits): - * 0-29: TID of lock holder (0 = unlocked) - * 30: FUTEX_OWNER_DIED (set by robust_list_walk on thread exit) - * 31: FUTEX_WAITERS (at least one thread is blocked) - * - * Linux kernel: FUTEX_WAITERS=0x80000000 (bit 31), FUTEX_OWNER_DIED=0x40000000 - * (bit 30), FUTEX_TID_MASK=0x3FFFFFFF. FUTEX_OWNER_DIED=0x40000000 (bit 30) is - * set by robust_list_walk on thread exit. FUTEX_TID_MASK is 30 bits. +/* The PI word's three fields and the edits made to them are proved/futexpi.h, + * which carries the layout and Linux's own constants. */ -#define FUTEX_TID_MASK 0x3FFFFFFFU -#define FUTEX_OWNER_DIED 0x40000000U -#define FUTEX_WAITERS 0x80000000U /* Address-wait helper state. * @@ -519,9 +514,9 @@ static void futex_clear_waiters_bit(uint32_t *word) bool cleared; if (!futex_word_load(word, &v)) return; - if (!(v & FUTEX_WAITERS)) + if (!futex_pi_has_waiters(v)) return; - if (!futex_word_cas(word, &v, v & ~FUTEX_WAITERS, &cleared)) + if (!futex_word_cas(word, &v, futex_pi_clear_waiters(v), &cleared)) return; if (cleared) return; @@ -1409,6 +1404,12 @@ static int64_t futex_requeue(guest_t *g, int do_cmp, uint32_t expected) { + /* Linux refuses these before taking either key; proved/futexreq.h carries + * why the sign is only visible as the top bit here. + */ + if (!futex_requeue_counts_valid(wake_count, requeue_count)) + return -LINUX_EINVAL; + if (!futex_uaddr_is_aligned(uaddr) || !futex_uaddr_is_aligned(uaddr2)) return -LINUX_EINVAL; @@ -1439,34 +1440,28 @@ static int64_t futex_requeue(guest_t *g, } } - /* A PI waiter remains tied to its entry bucket while it retries the PI - * acquisition. FUTEX_REQUEUE has no PI-aware counterpart here, so reject an - * attempted migration before waking or moving any waiter. + /* A PI waiter stays tied to its entry bucket while it retries, and + * FUTEX_REQUEUE has no PI-aware form here, so reject one before anything + * moves. Every waiter the call could touch is checked, wake candidates + * included, matching where requeue.c makes the same decision. The budget is + * summed in 64 bits: both halves are guest-supplied and wake-all passes + * INT_MAX. */ - if (uaddr != uaddr2 && requeue_count != 0) { - uint32_t skips = wake_count; - uint32_t remaining = requeue_count; - - for (futex_waiter_t *w = b_src->head; w; w = w->next) { - if (w->uaddr != uaddr) - continue; - if (skips != 0) { - skips--; - continue; - } + uint64_t checked = futex_requeue_budget(wake_count, requeue_count); + for (futex_waiter_t *w = b_src->head; w && checked != 0; w = w->next) { + if (w->uaddr != uaddr) + continue; - /* pub_follows is false only for a PI waiter today; see where it is - * set in futex_lock_pi_inner. - */ - if (!w->pub_follows) { - if (idx_src != idx_dst) - pthread_mutex_unlock(&b_dst->lock); - pthread_mutex_unlock(&b_src->lock); - return -LINUX_EINVAL; - } - if (--remaining == 0) - break; + /* pub_follows is false only for a PI waiter today; see where it is set + * in futex_lock_pi_inner. + */ + if (!w->pub_follows) { + if (idx_src != idx_dst) + pthread_mutex_unlock(&b_dst->lock); + pthread_mutex_unlock(&b_src->lock); + return -LINUX_EINVAL; } + checked--; } int woken = 0, requeued = 0; @@ -1489,12 +1484,13 @@ static int64_t futex_requeue(guest_t *g, /* Requeue: remove from source, add to destination */ *pp = w->next; - /* Move the publication with the waiter. The destination is charged - * before the source is debited, so the shim never sees this parked - * waiter charged to no bucket. + /* Credit the destination before debiting the source, so the shim + * never sees this waiter charged to no bucket. Both halves sit + * under pub_follows: a waiter carrying no charge of its own must + * not gain one here, which is how a charge outlived its waiter. */ - shim_globals_futex_waiters_add(g, idx_dst, +1); if (w->pub_follows) { + shim_globals_futex_waiters_add(g, idx_dst, +1); shim_globals_futex_waiters_add(g, w->pub_bucket, -1); w->pub_bucket = idx_dst; } @@ -1585,6 +1581,12 @@ static int64_t futex_wake_op(guest_t *g, op_val = 1U << futex_op_shift_arg_mask(op_arg); wake_op &= 7; /* Actual operation is bits 0-2 */ + /* An op Linux does not implement stops here, before the modify and before + * any wake. proved/futexwakeop.h carries both gates. + */ + if (!futex_wake_op_supported(wake_op)) + return -LINUX_ENOSYS; + unsigned idx1 = futex_hash(uaddr); unsigned idx2 = futex_hash(uaddr2); futex_bucket_t *b1 = &buckets[idx1]; @@ -1620,26 +1622,7 @@ static int64_t futex_wake_op(guest_t *g, ok = futex_word_load(word2, &old_val); if (!ok) break; - switch (wake_op) { - case 0: - new_val = op_val; - break; /* SET */ - case 1: - new_val = old_val + op_val; - break; /* ADD */ - case 2: - new_val = old_val | op_val; - break; /* OR */ - case 3: - new_val = old_val & ~op_val; - break; /* ANDN */ - case 4: - new_val = old_val ^ op_val; - break; /* XOR */ - default: - new_val = old_val; - break; - } + new_val = futex_wake_op_apply(old_val, wake_op, op_val); ok = futex_word_cas(word2, &old_val, new_val, &swapped); } while (ok && !swapped); @@ -1650,6 +1633,16 @@ static int64_t futex_wake_op(guest_t *g, return -LINUX_EFAULT; } + /* A comparison Linux does not implement stops here: the modify above has + * already landed, and neither wake runs. + */ + if (!futex_wake_cmp_supported(wake_cmp)) { + if (idx1 != idx2) + pthread_mutex_unlock(&b2->lock); + pthread_mutex_unlock(&b1->lock); + return -LINUX_ENOSYS; + } + /* Wake up to val waiters at uaddr (unlink woken entries) */ int woken1 = 0; futex_waiter_t **pp1 = &b1->head; @@ -1663,32 +1656,8 @@ static int64_t futex_wake_op(guest_t *g, } } - /* Evaluate comparison predicate on old_val */ - int cond_met = 0; - /* Linux FUTEX_WAKE_OP uses signed comparison semantics */ - int32_t sv = (int32_t) old_val; - switch (wake_cmp) { - case 0: - cond_met = (sv == cmp_arg); - break; /* EQ */ - case 1: - cond_met = (sv != cmp_arg); - break; /* NE */ - case 2: - cond_met = (sv < cmp_arg); - break; /* LT (signed) */ - case 3: - cond_met = (sv <= cmp_arg); - break; /* LE (signed) */ - case 4: - cond_met = (sv > cmp_arg); - break; /* GT (signed) */ - case 5: - cond_met = (sv >= cmp_arg); - break; /* GE (signed) */ - default: - break; - } + /* Signed comparison on the word as it was before the modify. */ + int cond_met = futex_wake_op_cmp((int32_t) old_val, wake_cmp, cmp_arg); /* Conditionally wake up to val2 waiters at uaddr2 (unlink woken) */ int woken2 = 0; @@ -1790,7 +1759,7 @@ static int64_t futex_lock_pi_inner(guest_t *g, return 0; /* Already own it? Deadlock (Linux returns EDEADLK) */ - if ((expected & FUTEX_TID_MASK) == tid) + if (futex_pi_owner_tid(expected) == tid) return -LINUX_EDEADLK; /* Robust owner death: the robust-list walk sets FUTEX_OWNER_DIED and @@ -1801,7 +1770,7 @@ static int64_t futex_lock_pi_inner(guest_t *g, * which never sees a robust-cleaned word (TID == 0) and would otherwise * spin forever. */ - if (expected & FUTEX_OWNER_DIED) { + if (futex_pi_owner_died(expected)) { if (!futex_word_cas(word, &expected, 0, NULL)) return -LINUX_EFAULT; continue; /* Retry acquisition */ @@ -1812,7 +1781,7 @@ static int64_t futex_lock_pi_inner(guest_t *g, * FUTEX_LOCK_PI returns -ESRCH (attach_to_pi_owner -> * handle_exit_race). */ - uint32_t owner_tid = expected & FUTEX_TID_MASK; + uint32_t owner_tid = futex_pi_owner_tid(expected); if (owner_tid != 0 && !thread_find((int64_t) owner_tid)) return -LINUX_ESRCH; @@ -1824,11 +1793,11 @@ static int64_t futex_lock_pi_inner(guest_t *g, uint32_t cur; if (!futex_word_load(word, &cur)) return -LINUX_EFAULT; - if ((cur & FUTEX_TID_MASK) == 0) + if (futex_pi_unowned(cur)) break; /* Owner released; retry outer loop */ - if (cur & FUTEX_WAITERS) + if (futex_pi_has_waiters(cur)) break; /* Already set by another waiter */ - uint32_t desired = cur | FUTEX_WAITERS; + uint32_t desired = futex_pi_set_waiters(cur); bool marked; if (!futex_word_cas(word, &cur, desired, &marked)) return -LINUX_EFAULT; @@ -1840,7 +1809,7 @@ static int64_t futex_lock_pi_inner(guest_t *g, uint32_t cur; if (!futex_word_load(word, &cur)) return -LINUX_EFAULT; - if ((cur & FUTEX_TID_MASK) == 0) + if (futex_pi_unowned(cur)) continue; /* Enqueue and block */ @@ -1853,7 +1822,7 @@ static int64_t futex_lock_pi_inner(guest_t *g, pthread_mutex_unlock(&b->lock); return -LINUX_EFAULT; } - if ((cur & FUTEX_TID_MASK) == 0) { + if (futex_pi_unowned(cur)) { pthread_mutex_unlock(&b->lock); continue; } @@ -2000,11 +1969,11 @@ static int64_t futex_lock_pi_inner(guest_t *g, pthread_cond_destroy(&waiter.cond); return -LINUX_EFAULT; } - if (check & FUTEX_OWNER_DIED) { + if (futex_pi_owner_died(check)) { owner_died = true; break; } - uint32_t check_tid = check & FUTEX_TID_MASK; + uint32_t check_tid = futex_pi_owner_tid(check); if (check_tid != 0 && !thread_tid_alive((int64_t) check_tid)) { bucket_unlink_locked(b, &waiter); pthread_mutex_unlock(&b->lock); @@ -2098,7 +2067,7 @@ static int64_t futex_unlock_pi(guest_t *g, uint64_t uaddr) uint32_t cur; if (guest_read_small(g, uaddr, &cur, sizeof(cur)) != 0) return -LINUX_EFAULT; - if ((cur & FUTEX_TID_MASK) != tid) + if (futex_pi_owner_tid(cur) != tid) return -LINUX_EPERM; /* Only the owner reaches here, and an owned PI lock is always aligned @@ -2301,30 +2270,22 @@ typedef struct { #define LINUX_CLOCK_REALTIME 0 #define LINUX_CLOCK_MONOTONIC 1 +/* The distinct buckets the wait set covers, ascending. The locks below are + * taken in this order and released in reverse, so both properties of the answer + * are load-bearing: proved/futexwaitv.h carries them. + * + * nr_futexes is bounded by FUTEX_WAITV_MAX before the call, which is what keeps + * nbuckets below the array length at every insert. + */ static int waitv_collect_buckets(const linux_futex_waitv_t *elts, uint32_t nr_futexes, - unsigned bucket_ids[FUTEX_WAITV_MAX], - futex_bucket_t *bucket_ptrs[FUTEX_WAITV_MAX]) + unsigned bucket_ids[FUTEX_WAITV_MAX]) { unsigned nbuckets = 0; - for (uint32_t i = 0; i < nr_futexes; i++) { - unsigned idx = futex_hash(elts[i].uaddr); - unsigned pos = 0; - - while (pos < nbuckets && bucket_ids[pos] < idx) - pos++; - if (pos < nbuckets && bucket_ids[pos] == idx) - continue; - - for (unsigned j = nbuckets; j > pos; j--) { - bucket_ids[j] = bucket_ids[j - 1]; - bucket_ptrs[j] = bucket_ptrs[j - 1]; - } - bucket_ids[pos] = idx; - bucket_ptrs[pos] = &buckets[idx]; - nbuckets++; - } + for (uint32_t i = 0; i < nr_futexes; i++) + nbuckets = futex_bucket_insert(bucket_ids, nbuckets, FUTEX_WAITV_MAX, + futex_hash(elts[i].uaddr)); return (int) nbuckets; } @@ -2421,9 +2382,7 @@ int64_t sys_futex_waitv(guest_t *g, */ futex_waiter_t waiters[FUTEX_WAITV_MAX]; unsigned bucket_ids[FUTEX_WAITV_MAX]; - futex_bucket_t *bucket_ptrs[FUTEX_WAITV_MAX]; - int nbuckets = - waitv_collect_buckets(elts, nr_futexes, bucket_ids, bucket_ptrs); + int nbuckets = waitv_collect_buckets(elts, nr_futexes, bucket_ids); int enqueued = 0; int64_t result_err = 0; @@ -2446,7 +2405,7 @@ int64_t sys_futex_waitv(guest_t *g, } for (int i = 0; i < nbuckets; i++) - pthread_mutex_lock(&bucket_ptrs[i]->lock); + pthread_mutex_lock(&buckets[bucket_ids[i]].lock); for (uint32_t i = 0; i < nr_futexes; i++) { uint64_t uaddr = elts[i].uaddr; @@ -2475,7 +2434,7 @@ int64_t sys_futex_waitv(guest_t *g, } for (int i = nbuckets - 1; i >= 0; i--) - pthread_mutex_unlock(&bucket_ptrs[i]->lock); + pthread_mutex_unlock(&buckets[bucket_ids[i]].lock); /* All enqueued. Block on shared.cond until any wake site signals it. The * bounded sleep (capped at 100 ms or the user deadline, whichever is @@ -2558,7 +2517,7 @@ int64_t sys_futex_waitv(guest_t *g, unlock_early: for (int i = nbuckets - 1; i >= 0; i--) - pthread_mutex_unlock(&bucket_ptrs[i]->lock); + pthread_mutex_unlock(&buckets[bucket_ids[i]].lock); for (int i = enqueued - 1; i >= 0; i--) { waitv_unlink(&waiters[i]); @@ -2653,11 +2612,10 @@ void robust_list_walk(guest_t *g, thread_entry_t *t) if (guest_read_small(g, futex_gva, &futex_val, sizeof(futex_val)) == 0) { /* Only act if this thread owns the lock */ - uint32_t owner = futex_val & FUTEX_TID_MASK; + uint32_t owner = futex_pi_owner_tid(futex_val); if (owner == (uint32_t) thread_tid(t)) { /* Set FUTEX_OWNER_DIED and clear TID */ - uint32_t new_val = - (futex_val & ~FUTEX_TID_MASK) | FUTEX_OWNER_DIED; + uint32_t new_val = futex_pi_mark_owner_died(futex_val); if (guest_write_small(g, futex_gva, &new_val, sizeof(new_val)) < 0) log_debug( @@ -2697,10 +2655,9 @@ void robust_list_walk(guest_t *g, thread_entry_t *t) uint32_t futex_val; if (guest_read_small(g, futex_gva, &futex_val, sizeof(futex_val)) == 0) { - uint32_t owner = futex_val & FUTEX_TID_MASK; + uint32_t owner = futex_pi_owner_tid(futex_val); if (owner == (uint32_t) thread_tid(t)) { - uint32_t new_val = - (futex_val & ~FUTEX_TID_MASK) | FUTEX_OWNER_DIED; + uint32_t new_val = futex_pi_mark_owner_died(futex_val); if (guest_write_small(g, futex_gva, &new_val, sizeof(new_val)) < 0) log_debug( diff --git a/tests/test-futex-requeue-pi.c b/tests/test-futex-requeue-pi.c new file mode 100644 index 00000000..d6628859 --- /dev/null +++ b/tests/test-futex-requeue-pi.c @@ -0,0 +1,227 @@ +/* + * A plain FUTEX_REQUEUE must refuse a PI waiter, whatever the addresses are + * + * Copyright 2026 elfuse contributors + * SPDX-License-Identifier: Apache-2.0 + * + * Linux answers EINVAL when a requeue without FUTEX_CMP_REQUEUE_PI meets a + * waiter holding an rt_waiter or a pi_state. The test is on the waiter alone, + * so neither the addresses nor the wake budget excuses one, and this file puts + * a case to each of those. It also holds the negative counts requeue.c refuses + * before it takes either key. + * + * The requeue is retried while it reports zero, which is the answer when nobody + * is parked yet: without that loop a slow thread start would pass by never + * reaching the case. + * + * Syscalls exercised: futex(98), clone(220), gettid(178), exit(93), sched_yield + */ + +#include +#include +#include +#include +#include +#include + +#include "test-harness.h" +#include "raw-syscall.h" + +int passes = 0, fails = 0; + +/* The lock under test and the four handshake words that sequence the two + * threads around it. + */ +static int pi_lock; /* the PI futex the waiter parks on */ +static int holder_ready; /* set once the holder has settled, either way */ +static int holder_failed; /* set instead of owning it, when LOCK_PI refused */ +static int release; /* set to tell the holder to unlock */ +static int waiter_done; /* set once the waiter has taken and released it */ + +static int holder_stack[16384] __attribute__((aligned(16))); +static int waiter_stack[16384] __attribute__((aligned(16))); + +/* Every op carries FUTEX_PRIVATE_FLAG, as the neighbouring futex tests do. + * elfuse masks it off, but on a reference kernel private and shared hash to + * different keys, and raw_futex_wake sets it. + */ +static long futex_lock_pi(int *addr) +{ + return raw_syscall6(__NR_futex, (long) addr, + FUTEX_LOCK_PI | FUTEX_PRIVATE_FLAG, 0, 0, 0, 0); +} + +static long futex_unlock_pi(int *addr) +{ + return raw_syscall6(__NR_futex, (long) addr, + FUTEX_UNLOCK_PI | FUTEX_PRIVATE_FLAG, 0, 0, 0, 0); +} + +/* FUTEX_REQUEUE reads its requeue count out of the timeout slot. */ +static long futex_requeue_same(int *addr, long wake, long requeue) +{ + return raw_syscall6(__NR_futex, (long) addr, + FUTEX_REQUEUE | FUTEX_PRIVATE_FLAG, wake, requeue, + (long) addr, 0); +} + +static void set_and_wake(int *addr) +{ + __atomic_store_n(addr, 1, __ATOMIC_RELEASE); + raw_futex_wake(addr, 1); +} + +/* holder_ready is the one flag two threads wait on, main and the waiter, and it + * is set once. Waking a single one of them strands the other for good, because + * the flag never returns to zero and no second wake is coming. + * + * Both readers usually observe the store before they park, which is why the + * single wake survived every run on a host with hardware virtualization. Under + * qemu's tcg the window between the load in wait_until_set and the FUTEX_WAIT + * behind it is wide enough to lose, and the matrix's qemu-aarch64 lane hung + * there about one run in three. + * + * The other three flags keep set_and_wake. Each has exactly one reader, and + * leaving them alone is what keeps that readable. + */ +static void set_and_wake_all(int *addr) +{ + __atomic_store_n(addr, 1, __ATOMIC_RELEASE); + raw_futex_wake(addr, INT_MAX); +} + +static void wait_until_set(int *addr) +{ + while (__atomic_load_n(addr, __ATOMIC_ACQUIRE) == 0) + raw_futex_wait(addr, 0); +} + +/* Takes pi_lock, reports it, and holds it until told to let go. The waiter + * cannot park until someone else owns the word. + */ +static void holder_fn(void) +{ + if (futex_lock_pi(&pi_lock) == 0) { + set_and_wake_all(&holder_ready); + wait_until_set(&release); + futex_unlock_pi(&pi_lock); + } else { + /* Say so rather than leave main parked on holder_ready forever: a tree + * whose LOCK_PI is broken is the one this lane exists to catch. + */ + set_and_wake(&holder_failed); + set_and_wake_all(&holder_ready); + } + raw_exit(0); +} + +/* Parks in FUTEX_LOCK_PI on the held word. This is the waiter whose charge the + * requeue must not move, and the one whose bucket is left over-counted. + */ +static void waiter_fn(void) +{ + wait_until_set(&holder_ready); + if (futex_lock_pi(&pi_lock) == 0) + futex_unlock_pi(&pi_lock); + set_and_wake(&waiter_done); + raw_exit(0); +} + +int main(void) +{ + /* CLONE_VM | CLONE_FS | CLONE_FILES | CLONE_SIGHAND | CLONE_THREAD | + * CLONE_SYSVSEM, spelled as the value the way test-futex-pi.c does. No TLS + * or tid flags: a CHILD_CLEARTID wake would only add futex traffic. + */ + unsigned long flags = 0x50f00; + + printf("=== futex requeue PI rejection tests ===\n\n"); + + /* No waiter needed: on an empty address both counts are no-ops, so the zero + * an unguarded build reports is what EINVAL separates from. + */ + TEST("a negative wake count is EINVAL"); + EXPECT_RAW_ERRNO(futex_requeue_same(&pi_lock, -1, 0), -EINVAL, + "a negative wake count must be refused"); + + TEST("a negative requeue count is EINVAL"); + EXPECT_RAW_ERRNO(futex_requeue_same(&pi_lock, 0, -1), -EINVAL, + "a negative requeue count must be refused"); + + /* One clone at a time, each with its child branch immediately after it: + * issuing both first lets the first child run the second raw_clone too. + * test-thread.c and test-futex-pi.c clone this way for the same reason. + */ + TEST("clone holder and waiter"); + long holder = raw_clone(flags, holder_stack + 16384, 0, 0, 0); + if (holder == 0) { + holder_fn(); + __builtin_unreachable(); + } + if (holder < 0) { + FAIL("holder clone failed"); + goto done; + } + + long waiter = raw_clone(flags, waiter_stack + 16384, 0, 0, 0); + if (waiter == 0) { + waiter_fn(); + __builtin_unreachable(); + } + if (waiter < 0) { + FAIL("waiter clone failed"); + goto done; + } + PASS(); + + wait_until_set(&holder_ready); + + TEST("holder takes pi_lock"); + if (__atomic_load_n(&holder_failed, __ATOMIC_ACQUIRE) != 0) { + FAIL("FUTEX_LOCK_PI refused the holder, nothing to requeue against"); + goto done; + } + PASS(); + + /* Zero is the answer before the waiter parks; anything else means the call + * saw it, and EINVAL is the only correct one. + */ + TEST("requeue a parked PI waiter onto its own address"); + long rc = 0; + for (int i = 0; i < 100000 && rc == 0; i++) { + rc = futex_requeue_same(&pi_lock, 0, 1); + if (rc == 0) + raw_syscall0(__NR_sched_yield); + } + EXPECT_RAW_ERRNO(rc, -EINVAL, "requeuing a PI waiter must be EINVAL"); + + /* Still parked, so the other two shapes reuse it. The check outranks the + * wake/requeue decision, so neither budget excuses a PI waiter. + */ + TEST("PI waiter inside the wake budget"); + EXPECT_RAW_ERRNO(futex_requeue_same(&pi_lock, 1, 1), -EINVAL, + "a PI waiter within wake_count must be EINVAL, not woken"); + + TEST("wake-only requeue with a PI waiter"); + EXPECT_RAW_ERRNO(futex_requeue_same(&pi_lock, 1, 0), -EINVAL, + "requeue_count 0 does not excuse a PI waiter"); + + set_and_wake(&release); + wait_until_set(&waiter_done); + + /* Nobody is parked now, so a rejected requeue must not have left the waiter + * somewhere a later wake still finds. + */ + TEST("wake the drained address"); + for (int i = 0; i < 8; i++) { + if (raw_futex_wake(&pi_lock, 1) != 0) { + FAIL("a drained address reported a woken waiter"); + goto done; + } + } + PASS(); + +done: + SUMMARY("test-futex-requeue-pi"); + return fails > 0 ? 1 : 0; +} diff --git a/tests/test-futex-waitv-buckets.c b/tests/test-futex-waitv-buckets.c new file mode 100644 index 00000000..e237cd65 --- /dev/null +++ b/tests/test-futex-waitv-buckets.c @@ -0,0 +1,171 @@ +/* + * futex_waitv over entries that share a bucket + * + * Copyright 2026 elfuse contributors + * SPDX-License-Identifier: Apache-2.0 + * + * elfuse locks one bucket per distinct address hash, ascending, and unlocks in + * reverse. Two guest addresses hashing alike is ordinary: there are 1024 + * buckets and the guest picks the addresses. A repeat in that set locks one + * non-recursive mutex twice, which is a hang rather than an errno, so every + * case here is bounded by a deadline the call has to answer. + * + * The hash below mirrors proved/futexhash.h. It only names elfuse's buckets; a + * reference kernel buckets differently, so there the same cases are ordinary + * wait sets and still have to behave. + * + * Syscalls exercised: futex_waitv(449), futex(98), clone(220), exit(93), + * clock_gettime(113) + */ + +#include +#include +#include +#include + +#include "test-harness.h" +#include "raw-syscall.h" +#include "test-util.h" + +int passes = 0, fails = 0; + +#define __NR_futex_waitv 449 +#define FUTEX2_SIZE_U32 0x02 +#define WAITV_MAX 128 + +/* Mirrors futex_bucket_index in proved/futexhash.h with FUTEX_BUCKETS. */ +#define FUTEX_HASH_MULT 0x9E3779B97F4A7C15ULL +#define FUTEX_BUCKETS 1024u + +struct futex_waitv { + uint64_t val; + uint64_t uaddr; + uint32_t flags; + uint32_t __reserved; +}; + +struct k_timespec { + int64_t tv_sec; + int64_t tv_nsec; +}; + +#define PARK_MS 200 +#define EARLY_MS (PARK_MS / 2) + +#define ARENA_WORDS (1u << 20) +static uint32_t arena[ARENA_WORDS]; +static int32_t first_at[FUTEX_BUCKETS]; +static int32_t shared[WAITV_MAX]; +static int n_shared; + +static uint32_t bucket_of(uint64_t a) +{ + return (uint32_t) (((((a >> 2) * FUTEX_HASH_MULT) >> 32)) % FUTEX_BUCKETS); +} + +static long now_ms(void) +{ + struct k_timespec ts; + raw_syscall2(113, 1 /* CLOCK_MONOTONIC */, (long) &ts); + return (long) (ts.tv_sec * 1000 + ts.tv_nsec / 1000000); +} + +static void deadline_in(struct k_timespec *ts, long ms) +{ + raw_syscall2(113, 1, (long) ts); + ts->tv_nsec += ms * 1000L * 1000L; + while (ts->tv_nsec >= 1000000000L) { + ts->tv_nsec -= 1000000000L; + ts->tv_sec++; + } +} + +static void fill(struct futex_waitv *w, int n, uint32_t *const *addrs) +{ + memset(w, 0, sizeof(*w) * (size_t) n); + for (int i = 0; i < n; i++) { + w[i].uaddr = (uint64_t) (uintptr_t) addrs[i]; + w[i].flags = FUTEX2_SIZE_U32; + } +} + +/* Every case waits out its deadline: nothing here is woken. A repeat in the + * bucket set never gets that far. + */ +static void expect_timeout(const char *name, struct futex_waitv *w, int n) +{ + TEST(name); + struct k_timespec ts; + deadline_in(&ts, PARK_MS); + long t0 = now_ms(); + long rc = raw_syscall5(__NR_futex_waitv, (long) w, n, 0, (long) &ts, 1); + long elapsed = now_ms() - t0; + + if (rc != -ETIMEDOUT) + FAIL("a wait set nobody wakes must report ETIMEDOUT"); + else if (elapsed < EARLY_MS) + FAIL("the wait did not last"); + else + PASS(); +} + +int main(void) +{ + printf("=== futex_waitv bucket sharing ===\n\n"); + + for (uint32_t i = 0; i < FUTEX_BUCKETS; i++) + first_at[i] = -1; + + /* One bucket's worth of distinct words, enough to fill a whole wait set. */ + uint32_t target = bucket_of((uint64_t) (uintptr_t) &arena[0]); + for (uint32_t i = 0; i < ARENA_WORDS && n_shared < WAITV_MAX; i++) + if (bucket_of((uint64_t) (uintptr_t) &arena[i]) == target) + shared[n_shared++] = (int32_t) i; + + TEST("arena yields a full shared bucket"); + if (n_shared < WAITV_MAX) { + FAIL("not enough words share one bucket"); + goto done; + } + PASS(); + + struct futex_waitv w[WAITV_MAX]; + uint32_t *addrs[WAITV_MAX]; + + for (int i = 0; i < 2; i++) + addrs[i] = &arena[shared[0]]; + fill(w, 2, addrs); + expect_timeout("one address twice", w, 2); + + addrs[0] = &arena[shared[0]]; + addrs[1] = &arena[shared[1]]; + fill(w, 2, addrs); + expect_timeout("two addresses, one bucket", w, 2); + + addrs[0] = &arena[shared[1]]; + addrs[1] = &arena[shared[0]]; + fill(w, 2, addrs); + expect_timeout("the same two, descending", w, 2); + + for (int i = 0; i < WAITV_MAX; i++) + addrs[i] = &arena[shared[0]]; + fill(w, WAITV_MAX, addrs); + expect_timeout("one address 128 times", w, WAITV_MAX); + + /* Descending is the insertion's worst case: every entry goes to the front + * and shifts the whole set. + */ + for (int i = 0; i < WAITV_MAX; i++) + addrs[i] = &arena[shared[WAITV_MAX - 1 - i]]; + fill(w, WAITV_MAX, addrs); + expect_timeout("128 in one bucket, descending", w, WAITV_MAX); + + for (int i = 0; i < WAITV_MAX; i++) + addrs[i] = &arena[(uint32_t) (WAITV_MAX - 1 - i) * 977u]; + fill(w, WAITV_MAX, addrs); + expect_timeout("128 spread, descending address", w, WAITV_MAX); + +done: + SUMMARY("test-futex-waitv-buckets"); + return fails > 0 ? 1 : 0; +} diff --git a/tests/test-futex-wake-op-enosys.c b/tests/test-futex-wake-op-enosys.c new file mode 100644 index 00000000..00a4d632 --- /dev/null +++ b/tests/test-futex-wake-op-enosys.c @@ -0,0 +1,226 @@ +/* + * FUTEX_WAKE_OP must answer ENOSYS for a selector Linux does not implement + * + * Copyright 2026 elfuse contributors + * SPDX-License-Identifier: Apache-2.0 + * + * Both selectors in val3 are guest-supplied and both have unassigned encodings. + * Linux refuses each at a different point, which the modify makes visible: an + * unimplemented op stops before it, an unimplemented comparison after it. + * Neither wakes anybody. + * + * Every assertion is an errno or a word the guest can read, so the same source + * runs against a reference kernel. + * + * Syscalls exercised: futex(98), clone(220), exit(93), sched_yield(124) + */ + +#include +#include + +#include "test-harness.h" +#include "raw-syscall.h" + +int passes = 0, fails = 0; + +static int park_word; /* the address the waiter parks on */ +static int target_word; /* the address the modify lands on */ +static int waiter_parking; /* set just before the waiter enters FUTEX_WAIT */ +static int waiter_done; /* set once FUTEX_WAIT has returned for real */ + +static int waiter_stack[16384] __attribute__((aligned(16))); + +/* val3 layout: bit 31 OPARG_SHIFT, 30-28 op, 27-24 cmp, 23-12 oparg, 11-0 + * cmparg. + */ +static uint32_t encode(unsigned op, + unsigned cmp, + uint32_t oparg, + uint32_t cmparg) +{ + return ((op & 7u) << 28) | ((cmp & 0xfu) << 24) | ((oparg & 0xfffu) << 12) | + (cmparg & 0xfffu); +} + +/* FUTEX_WAKE_OP reads its second wake count out of the timeout slot. */ +static long futex_wake_op(long nr_wake, long nr_wake2, uint32_t val3) +{ + return raw_syscall6(__NR_futex, (long) &park_word, + FUTEX_WAKE_OP | FUTEX_PRIVATE_FLAG, nr_wake, nr_wake2, + (long) &target_word, (long) val3); +} + +/* SET 0x111, compared EQ against 0x111. The modify always lands; the compare is + * against the word as it was before, so the second wake does not fire. + */ +static uint32_t valid_val3(void) +{ + return encode(0, 0, 0x111, 0x111); +} + +static void set_and_wake(int *addr) +{ + __atomic_store_n(addr, 1, __ATOMIC_RELEASE); + raw_futex_wake(addr, 1); +} + +/* Parks once. EINTR is a retry rather than a wake, so a build that interrupts + * the wait does not read as one that answered it. + */ +static void waiter_fn(void) +{ + long r; + set_and_wake(&waiter_parking); + do { + r = raw_futex_wait(&park_word, 0); + } while (r == -EINTR); + set_and_wake(&waiter_done); + raw_exit(0); +} + +static int parked(void) +{ + return __atomic_load_n(&waiter_done, __ATOMIC_ACQUIRE) == 0; +} + +/* Give a wake that should not have happened room to arrive. A real one lands + * within a few yields; this is generous rather than tuned. + */ +static void settle(void) +{ + for (int i = 0; i < 10000; i++) + raw_syscall0(__NR_sched_yield); +} + +/* No waiter is needed for the errno and the modify: on an empty address both + * wake counts are no-ops. + */ +static void unsupported_op_case(const char *name, unsigned op) +{ + TEST(name); + __atomic_store_n(&target_word, 0, __ATOMIC_RELEASE); + long rc = futex_wake_op(1, 1, encode(op, 0, 0x111, 0x111)); + if (rc != -ENOSYS) { + FAIL("an unimplemented op must be ENOSYS"); + return; + } + if (__atomic_load_n(&target_word, __ATOMIC_ACQUIRE) != 0) { + FAIL("a refused op must not have modified the word"); + return; + } + PASS(); +} + +static void unsupported_cmp_case(const char *name, unsigned cmp) +{ + TEST(name); + __atomic_store_n(&target_word, 0, __ATOMIC_RELEASE); + long rc = futex_wake_op(1, 1, encode(0, cmp, 0x111, 0x111)); + if (rc != -ENOSYS) { + FAIL("an unimplemented comparison must be ENOSYS"); + return; + } + + /* The comparison is refused after the modify, so unlike the op case the + * word carries the operand. + */ + if (__atomic_load_n(&target_word, __ATOMIC_ACQUIRE) != 0x111) { + FAIL("a refused comparison must still have modified the word"); + return; + } + PASS(); +} + +int main(void) +{ + /* CLONE_VM | CLONE_FS | CLONE_FILES | CLONE_SIGHAND | CLONE_THREAD | + * CLONE_SYSVSEM, spelled as the value the way the sibling futex tests do. + */ + unsigned long flags = 0x50f00; + + printf("=== futex wake_op selector tests ===\n\n"); + + TEST("a supported pair modifies the word"); + __atomic_store_n(&target_word, 0, __ATOMIC_RELEASE); + long rc = futex_wake_op(1, 1, valid_val3()); + if (rc < 0) { + FAIL("a supported op and comparison must not be refused"); + } else if (__atomic_load_n(&target_word, __ATOMIC_ACQUIRE) != 0x111) { + FAIL("a supported op must have modified the word"); + } else { + PASS(); + } + + unsupported_op_case("op 5 is ENOSYS", 5); + unsupported_op_case("op 6 is ENOSYS", 6); + unsupported_op_case("op 7 is ENOSYS", 7); + + unsupported_cmp_case("comparison 6 is ENOSYS", 6); + unsupported_cmp_case("comparison 15 is ENOSYS", 15); + + /* Both unassigned: the op is read first, so the word stays untouched. */ + unsupported_op_case("an unsupported pair stops at the op", 5); + + TEST("clone the waiter"); + long waiter = raw_clone(flags, waiter_stack + 16384, 0, 0, 0); + if (waiter == 0) { + waiter_fn(); + __builtin_unreachable(); + } + if (waiter < 0) { + FAIL("waiter clone failed"); + goto done; + } + PASS(); + + while (__atomic_load_n(&waiter_parking, __ATOMIC_ACQUIRE) == 0) + raw_futex_wait(&waiter_parking, 0); + settle(); + + TEST("a refused op leaves the waiter parked"); + if (futex_wake_op(1, 1, encode(5, 0, 0x111, 0x111)) != -ENOSYS) { + FAIL("an unimplemented op must be ENOSYS"); + goto release; + } + settle(); + if (!parked()) { + FAIL("a refused op woke a parked waiter"); + goto release; + } + PASS(); + + TEST("a refused comparison leaves the waiter parked"); + if (futex_wake_op(1, 1, encode(0, 6, 0x111, 0x111)) != -ENOSYS) { + FAIL("an unimplemented comparison must be ENOSYS"); + goto release; + } + settle(); + if (!parked()) { + FAIL("a refused comparison woke a parked waiter"); + goto release; + } + PASS(); + + /* The waiter is still there, so a supported pair reaches it. This is also + * what keeps the two cases above from passing on a waiter that never + * parked: one that had gone leaves nothing to wake here. + */ + TEST("a supported op wakes the waiter left behind"); + __atomic_store_n(&park_word, 1, __ATOMIC_RELEASE); + if (futex_wake_op(1, 1, valid_val3()) != 1) { + FAIL("the waiter the refused calls left parked was not woken"); + goto release; + } + PASS(); + +release: + __atomic_store_n(&park_word, 1, __ATOMIC_RELEASE); + while (__atomic_load_n(&waiter_done, __ATOMIC_ACQUIRE) == 0) { + raw_futex_wake(&park_word, 1); + raw_syscall0(__NR_sched_yield); + } + +done: + SUMMARY("test-futex-wake-op-enosys"); + return fails > 0 ? 1 : 0; +} diff --git a/tests/test-matrix.sh b/tests/test-matrix.sh index 840033a5..b8e0fb97 100755 --- a/tests/test-matrix.sh +++ b/tests/test-matrix.sh @@ -892,6 +892,12 @@ run_unit_tests() "$bindir/test-futex-wake-nowaiter" test_rc "$runner" "test-futex-requeue-account" 0 \ "$bindir/test-futex-requeue-account" + test_rc "$runner" "test-futex-requeue-pi" 0 \ + "$bindir/test-futex-requeue-pi" + test_rc "$runner" "test-futex-wake-op-enosys" 0 \ + "$bindir/test-futex-wake-op-enosys" + test_rc "$runner" "test-futex-waitv-buckets" 0 \ + "$bindir/test-futex-waitv-buckets" test_rc "$runner" "test-robust-futex" 0 "$bindir/test-robust-futex" test_check "$runner" "test-shim-futex-fast" "OK" \ "$bindir/test-shim-futex-fast" diff --git a/tests/test-robust-futex.c b/tests/test-robust-futex.c index 72050bce..0f9890b9 100644 --- a/tests/test-robust-futex.c +++ b/tests/test-robust-futex.c @@ -15,7 +15,7 @@ #include #include -#include +#include #include #include "test-harness.h" @@ -51,6 +51,7 @@ static struct robust_list_head rhead __attribute__((aligned(8))); static struct robust_list entry1 __attribute__((aligned(8))); static char child_stack[8192] __attribute__((aligned(16))); +static volatile int preset_waiters; static int child_fn(void *arg) { @@ -63,8 +64,11 @@ static int child_fn(void *arg) rhead.list_op_pending = NULL; entry1.next = &rhead.list; /* circular: points back to head */ - /* "Acquire" the lock by writing the current TID */ - lock_word = (uint32_t) tid; + /* "Acquire" the lock by writing the current TID, with WAITERS already set + * when the case under test wants it there. + */ + lock_word = + (uint32_t) tid | (preset_waiters ? (uint32_t) FUTEX_WAITERS : 0u); /* Register robust list with kernel */ raw_syscall2(99, (long) &rhead, sizeof(rhead)); /* set_robust_list */ @@ -76,48 +80,104 @@ static int child_fn(void *arg) test_unreachable(); } -int main(void) +/* Wait for a cloned thread to finish tearing down. CLEARTID zeroes the address + * only after the robust walk has run, so a cleared word proves the walk + * finished and the reused child stack is free. -1 if it never clears. + */ +static int join_child(volatile int *ctid) { - TEST("robust-futex: owner-died on exit"); + /* Plain FUTEX_WAIT: the CLEARTID wake is not private. The timeout only + * turns a stuck teardown into a reported failure. + */ + struct timespec ts = {.tv_sec = 1, .tv_nsec = 0}; + + for (int i = 0; i < 10; i++) { + int seen = __atomic_load_n(ctid, __ATOMIC_SEQ_CST); + if (seen == 0) + return 0; + raw_syscall6(98, (long) ctid, FUTEX_WAIT, seen, (long) &ts, 0, 0); + } + return __atomic_load_n(ctid, __ATOMIC_SEQ_CST) == 0 ? 0 : -1; +} +/* One owner-dies run. + * + * Returns the word the robust walk left behind; on failure *err names what went + * wrong and the word means nothing. + */ +static uint32_t run_owner_death(int waiters, const char **err) +{ + *err = NULL; lock_word = 0; memset(&rhead, 0, sizeof(rhead)); memset(&entry1, 0, sizeof(entry1)); + preset_waiters = waiters; - /* Clone a thread: CLONE_THREAD | CLONE_VM | CLONE_FS | CLONE_SIGHAND | - * CLONE_CHILD_CLEARTID. CLONE_THREAD implies CLONE_VM and CLONE_SIGHAND. + /* CLONE_THREAD | CLONE_VM | CLONE_FS | CLONE_SIGHAND | CLONE_PARENT_SETTID + * | CLONE_CHILD_CLEARTID. CLONE_THREAD implies CLONE_VM and CLONE_SIGHAND. + * PARENT_SETTID seeds the word CLEARTID later zeroes, so join_child has + * something to wait on. */ - long flags = 0x00010000 /* CLONE_THREAD */ - | 0x00000100 /* CLONE_VM */ - | 0x00000200 /* CLONE_FS */ - | 0x00000800 /* CLONE_SIGHAND */ - | 0x00200000; /* CLONE_CHILD_CLEARTID */ + long flags = 0x00010000 | 0x00000100 | 0x00000200 | 0x00000800 | + 0x00100000 | 0x00200000; - /* Use raw_syscall5 for clone(flags, stack, ptid, tls, ctid) */ volatile int child_tid_addr = 0; - long ret = raw_syscall5(220, /* clone */ - flags, (long) (child_stack + sizeof(child_stack)), - 0, /* parent_tid */ - 0, /* tls */ - (long) &child_tid_addr /* child_tid */ - ); - - if (ret < 0) { - FAIL("clone failed"); - } else if (ret == 0) { - /* Child */ + long ret = + raw_syscall5(220, flags, (long) (child_stack + sizeof(child_stack)), + (long) &child_tid_addr, 0, (long) &child_tid_addr); + if (ret == 0) { child_fn(NULL); + test_unreachable(); + } + if (ret < 0) { + *err = "clone failed"; + return 0; + } + if (join_child(&child_tid_addr) != 0) { + *err = "child never cleared its CLEARTID word"; + return 0; + } + return lock_word; +} + +int main(void) +{ + const char *err; + + TEST("robust-futex: owner-died on exit"); + uint32_t plain = run_owner_death(0, &err); + if (err) { + FAIL(err); + } else { + EXPECT_TRUE(plain & FUTEX_OWNER_DIED, "FUTEX_OWNER_DIED not set"); + } + + /* The walk owes the word two more things than the flag. A TID left behind + * is an owner no LOCK_PI can displace, and it is what separates the robust + * path from an ordinary abandoned lock. A run that never started says + * nothing about either, so it reports once above. + */ + if (!err) { + TEST("robust-futex: owner-died clears the TID"); + EXPECT_TRUE((plain & FUTEX_TID_MASK) == 0, + "the dead owner's TID survived the walk"); + + TEST("robust-futex: owner-died keeps WAITERS clear"); + EXPECT_TRUE((plain & FUTEX_WAITERS) == 0, + "WAITERS appeared on a word that never had it"); + } + + /* Same transition over a word that already had waiters: the bit has to + * survive, or the parked waiter is never woken. + */ + TEST("robust-futex: owner-died keeps WAITERS set"); + uint32_t contended = run_owner_death(1, &err); + if (err) { + FAIL(err); } else { - /* Parent: wait for child to exit via CLONE_CHILD_CLEARTID futex */ - usleep(100000); /* 100ms grace period */ - - /* Check if FUTEX_OWNER_DIED was set */ - uint32_t val = lock_word; - if (val & FUTEX_OWNER_DIED) { - PASS(); - } else { - FAIL("FUTEX_OWNER_DIED not set"); - } + EXPECT_TRUE(contended == ((uint32_t) FUTEX_WAITERS | + (uint32_t) FUTEX_OWNER_DIED), + "a contended lock's word is not WAITERS|OWNER_DIED"); } TEST("robust-futex: set_robust_list returns 0");